@@ -119,7 +119,7 @@ static void __hyp_vgic_save_state(struct kvm_vcpu *vcpu)
}
}
-/* Restore VGICv3 state on non_VEH systems */
+/* Restore VGICv3 state on nVHE systems */
static void __hyp_vgic_restore_state(struct kvm_vcpu *vcpu)
{
if (static_branch_unlikely(&kvm_vgic_global_state.gicv3_cpuif)) {
@@ -166,8 +166,110 @@ static void __pmu_switch_to_host(struct kvm_cpu_context *host_ctxt)
write_sysreg(pmu->events_host, pmcntenset_el0);
}
-/* Switch to the guest for legacy non-VHE systems */
-int __kvm_vcpu_run(struct kvm_vcpu *vcpu)
+/* Switch to the non-protected guest */
+static int __kvm_vcpu_run_nvhe(struct kvm_vcpu *vcpu)
+{
+ struct vcpu_hyp_state *vcpu_hyps = &vcpu->arch.hyp_state;
+ struct kvm_cpu_context *vcpu_ctxt = &vcpu->arch.ctxt;
+ struct kvm *kvm = kern_hyp_va(vcpu->kvm);
+ struct vgic_dist *vgic = &kvm->arch.vgic;
+ struct kvm_cpu_context *host_ctxt;
+ struct kvm_cpu_context *guest_ctxt;
+ bool pmu_switch_needed;
+ u64 exit_code;
+
+ /*
+ * Having IRQs masked via PMR when entering the guest means the GIC
+ * will not signal the CPU of interrupts of lower priority, and the
+ * only way to get out will be via guest exceptions.
+ * Naturally, we want to avoid this.
+ */
+ if (system_uses_irq_prio_masking()) {
+ gic_write_pmr(GIC_PRIO_IRQON | GIC_PRIO_PSR_I_SET);
+ pmr_sync();
+ }
+
+ host_ctxt = &this_cpu_ptr(&kvm_host_data)->host_ctxt;
+ set_hyp_running_vcpu(host_ctxt, vcpu);
+ guest_ctxt = &vcpu->arch.ctxt;
+
+ pmu_switch_needed = __pmu_switch_to_guest(host_ctxt);
+
+ __sysreg_save_state_nvhe(host_ctxt);
+ /*
+ * We must flush and disable the SPE buffer for nVHE, as
+ * the translation regime(EL1&0) is going to be loaded with
+ * that of the guest. And we must do this before we change the
+ * translation regime to EL2 (via MDCR_EL2_E2PB == 0) and
+ * before we load guest Stage1.
+ */
+ __debug_save_host_buffers_nvhe(vcpu);
+
+ kvm_adjust_pc(vcpu_ctxt, vcpu_hyps);
+
+ /*
+ * We must restore the 32-bit state before the sysregs, thanks
+ * to erratum #852523 (Cortex-A57) or #853709 (Cortex-A72).
+ *
+ * Also, and in order to be able to deal with erratum #1319537 (A57)
+ * and #1319367 (A72), we must ensure that all VM-related sysreg are
+ * restored before we enable S2 translation.
+ */
+ __sysreg32_restore_state(vcpu);
+ __sysreg_restore_state_nvhe(guest_ctxt);
+
+ __load_guest_stage2(kern_hyp_va(vcpu->arch.hw_mmu));
+ __activate_traps(vcpu);
+
+ __hyp_vgic_restore_state(vcpu);
+ __timer_enable_traps();
+
+ __debug_switch_to_guest(vcpu);
+
+ do {
+ struct kvm_cpu_context *hyp_ctxt = this_cpu_ptr(&kvm_hyp_ctxt);
+ set_hyp_running_vcpu(hyp_ctxt, vcpu);
+
+ /* Jump in the fire! */
+ exit_code = __guest_enter(guest_ctxt);
+
+ /* And we're baaack! */
+ } while (fixup_guest_exit(vcpu, vgic, &exit_code));
+
+ __sysreg_save_state_nvhe(guest_ctxt);
+ __sysreg32_save_state(vcpu);
+ __timer_disable_traps();
+ __hyp_vgic_save_state(vcpu);
+
+ __deactivate_traps(vcpu_hyps);
+ __load_host_stage2();
+
+ __sysreg_restore_state_nvhe(host_ctxt);
+
+ if (hyp_state_flags(vcpu_hyps) & KVM_ARM64_FP_ENABLED)
+ __fpsimd_save_fpexc32(vcpu);
+
+ __debug_switch_to_host(vcpu);
+ /*
+ * This must come after restoring the host sysregs, since a non-VHE
+ * system may enable SPE here and make use of the TTBRs.
+ */
+ __debug_restore_host_buffers_nvhe(vcpu);
+
+ if (pmu_switch_needed)
+ __pmu_switch_to_host(host_ctxt);
+
+ /* Returning to host will clear PSR.I, remask PMR if needed */
+ if (system_uses_irq_prio_masking())
+ gic_write_pmr(GIC_PRIO_IRQOFF);
+
+ set_hyp_running_vcpu(host_ctxt, NULL);
+
+ return exit_code;
+}
+
+/* Switch to the protected guest */
+static int __kvm_vcpu_run_pvm(struct kvm_vcpu *vcpu)
{
struct vcpu_hyp_state *vcpu_hyps = &hyp_state(vcpu);
struct kvm_cpu_context *vcpu_ctxt = &vcpu_ctxt(vcpu);
@@ -268,6 +370,17 @@ int __kvm_vcpu_run(struct kvm_vcpu *vcpu)
return exit_code;
}
+/* Switch to the guest for non-VHE and protected KVM systems */
+int __kvm_vcpu_run(struct kvm_vcpu *vcpu)
+{
+ vcpu = kern_hyp_va(vcpu);
+
+ if (likely(!kvm_vm_is_protected(kern_hyp_va(vcpu->kvm))))
+ return __kvm_vcpu_run_nvhe(vcpu);
+ else
+ return __kvm_vcpu_run_pvm(vcpu);
+}
+
void __noreturn hyp_panic(void)
{
u64 spsr = read_sysreg_el2(SYS_SPSR);
Split kvm_run() for protected and non-protected VMs. Protected VMs support fewer features, separating it out will ease the refactoring and simplify the code. This patch starts only by replicated the code from the non-protected case, to make it easier to diff against future patches. Signed-off-by: Fuad Tabba <tabba@google.com> --- arch/arm64/kvm/hyp/nvhe/switch.c | 119 ++++++++++++++++++++++++++++++- 1 file changed, 116 insertions(+), 3 deletions(-)