@@ -92,13 +92,18 @@ static inline bool vcpu_mode_is_32bit(const struct kvm_vcpu *vcpu)
static inline void vcpu_set_spsr(struct kvm_vcpu *vcpu, u64 val)
{
if (vcpu_mode_is_32bit(vcpu)) {
- if (vcpu->arch.sysregs_loaded_on_cpu)
- __sysreg32_save_state(vcpu);
+ bool loaded;
+
+ preempt_disable();
+ loaded = vcpu->arch.sysregs32_loaded_on_cpu;
+ if (loaded)
+ kvm_call_hyp(__sysreg32_save_state, vcpu);
*vcpu_spsr32(vcpu) = val;
- if (vcpu->arch.sysregs_loaded_on_cpu)
- __sysreg32_restore_state(vcpu);
+ if (loaded)
+ kvm_call_hyp(__sysreg32_restore_state, vcpu);
+ preempt_enable();
}
if (vcpu->arch.sysregs_loaded_on_cpu)
@@ -283,6 +283,7 @@ struct kvm_vcpu_arch {
/* True when deferrable sysregs are loaded on the physical CPU,
* see kvm_vcpu_load_sysregs and kvm_vcpu_put_sysregs. */
bool sysregs_loaded_on_cpu;
+ bool sysregs32_loaded_on_cpu;
};
#define vcpu_gp_regs(v) (&(v)->arch.ctxt.gp_regs)
@@ -20,6 +20,7 @@
#include <asm/debug-monitors.h>
#include <asm/kvm_asm.h>
+#include <asm/kvm_emulate.h>
#include <asm/kvm_hyp.h>
#define read_debug(r,n) read_sysreg(r##n##_el1)
@@ -169,6 +170,9 @@ void __hyp_text __debug_switch_to_guest(struct kvm_vcpu *vcpu)
__debug_save_state(vcpu, host_dbg, host_ctxt);
__debug_restore_state(vcpu, guest_dbg, guest_ctxt);
+
+ if (vcpu_el1_is_32bit(vcpu))
+ write_sysreg(vcpu->arch.ctxt.sys_regs[DBGVCR32_EL2], dbgvcr32_el2);
}
void __hyp_text __debug_switch_to_host(struct kvm_vcpu *vcpu)
@@ -192,6 +196,9 @@ void __hyp_text __debug_switch_to_host(struct kvm_vcpu *vcpu)
__debug_save_state(vcpu, guest_dbg, guest_ctxt);
__debug_restore_state(vcpu, host_dbg, host_ctxt);
+ if (vcpu_el1_is_32bit(vcpu))
+ vcpu->arch.ctxt.sys_regs[DBGVCR32_EL2] = read_sysreg(dbgvcr32_el2);
+
vcpu->arch.debug_flags &= ~KVM_ARM64_DEBUG_DIRTY;
}
@@ -402,7 +402,6 @@ int __hyp_text __kvm_vcpu_run_nvhe(struct kvm_vcpu *vcpu)
* We must restore the 32-bit state before the sysregs, thanks
* to erratum #852523 (Cortex-A57) or #853709 (Cortex-A72).
*/
- __sysreg32_restore_state(vcpu);
__sysreg_restore_state_nvhe(guest_ctxt);
__debug_switch_to_guest(vcpu);
@@ -414,7 +413,6 @@ int __hyp_text __kvm_vcpu_run_nvhe(struct kvm_vcpu *vcpu)
} while (fixup_guest_exit(vcpu, &exit_code));
__sysreg_save_state_nvhe(guest_ctxt);
- __sysreg32_save_state(vcpu);
__timer_disable_traps(vcpu);
__vgic_save_state(vcpu);
@@ -181,7 +181,9 @@ void __hyp_text __sysreg32_save_state(struct kvm_vcpu *vcpu)
{
u64 *spsr, *sysreg;
- if (!vcpu_el1_is_32bit(vcpu))
+ vcpu = kern_hyp_va(vcpu);
+
+ if (!vcpu_el1_is_32bit(vcpu) || !vcpu->arch.sysregs32_loaded_on_cpu)
return;
spsr = vcpu->arch.ctxt.gp_regs.spsr;
@@ -195,15 +197,18 @@ void __hyp_text __sysreg32_save_state(struct kvm_vcpu *vcpu)
sysreg[DACR32_EL2] = read_sysreg(dacr32_el2);
sysreg[IFSR32_EL2] = read_sysreg(ifsr32_el2);
- if (vcpu->arch.debug_flags & KVM_ARM64_DEBUG_DIRTY)
- sysreg[DBGVCR32_EL2] = read_sysreg(dbgvcr32_el2);
+ sysreg[DBGVCR32_EL2] = read_sysreg(dbgvcr32_el2);
+
+ vcpu->arch.sysregs32_loaded_on_cpu = false;
}
void __hyp_text __sysreg32_restore_state(struct kvm_vcpu *vcpu)
{
u64 *spsr, *sysreg;
- if (!vcpu_el1_is_32bit(vcpu))
+ vcpu = kern_hyp_va(vcpu);
+
+ if (!vcpu_el1_is_32bit(vcpu) || vcpu->arch.sysregs32_loaded_on_cpu)
return;
spsr = vcpu->arch.ctxt.gp_regs.spsr;
@@ -217,8 +222,9 @@ void __hyp_text __sysreg32_restore_state(struct kvm_vcpu *vcpu)
write_sysreg(sysreg[DACR32_EL2], dacr32_el2);
write_sysreg(sysreg[IFSR32_EL2], ifsr32_el2);
- if (vcpu->arch.debug_flags & KVM_ARM64_DEBUG_DIRTY)
- write_sysreg(sysreg[DBGVCR32_EL2], dbgvcr32_el2);
+ write_sysreg(sysreg[DBGVCR32_EL2], dbgvcr32_el2);
+
+ vcpu->arch.sysregs32_loaded_on_cpu = true;
}
/**
@@ -237,19 +243,19 @@ void kvm_vcpu_load_sysregs(struct kvm_vcpu *vcpu)
struct kvm_cpu_context *host_ctxt = vcpu->arch.host_cpu_context;
struct kvm_cpu_context *guest_ctxt = &vcpu->arch.ctxt;
+ /*
+ * Erratum #852523 (Cortex-A57) or #853709 (Cortex-A72) requires us to
+ * restore the 32-bit state before the sysregs, which will happen on
+ * both VHE (below) and on non-VHE in the world-switch path.
+ */
+ kvm_call_hyp(__sysreg32_restore_state, vcpu);
+
if (!has_vhe())
return;
__sysreg_save_user_state(host_ctxt);
-
- /*
- * Load guest EL1 and user state
- *
- * We must restore the 32-bit state before the sysregs, thanks
- * to erratum #852523 (Cortex-A57) or #853709 (Cortex-A72).
- */
- __sysreg32_restore_state(vcpu);
+ /* Load guest EL1 and user state */
__sysreg_restore_user_state(guest_ctxt);
__sysreg_restore_el1_state(guest_ctxt);
@@ -283,12 +289,13 @@ void kvm_vcpu_put_sysregs(struct kvm_vcpu *vcpu)
vcpu->arch.guest_vfp_loaded = 0;
}
+ kvm_call_hyp(__sysreg32_save_state, vcpu);
+
if (!has_vhe())
return;
__sysreg_save_el1_state(guest_ctxt);
__sysreg_save_user_state(guest_ctxt);
- __sysreg32_save_state(vcpu);
/* Restore host user state */
__sysreg_restore_user_state(host_ctxt);