@@ -251,6 +251,18 @@ extern u32 __kvm_get_mdcr_el2(void);
ldr \vcpu, [\ctxt, #HOST_CONTEXT_VCPU]
.endm
+.macro get_vcpu_ctxt_ptr vcpu, ctxt
+ get_host_ctxt \ctxt, \vcpu
+ ldr \vcpu, [\ctxt, #HOST_CONTEXT_VCPU]
+ add \vcpu, \vcpu, #VCPU_CONTEXT
+.endm
+
+.macro get_vcpu_hyps_ptr vcpu, ctxt
+ get_host_ctxt \ctxt, \vcpu
+ ldr \vcpu, [\ctxt, #HOST_CONTEXT_VCPU]
+ add \vcpu, \vcpu, #VCPU_HYPS
+.endm
+
.macro get_loaded_vcpu vcpu, ctxt
adr_this_cpu \ctxt, kvm_hyp_ctxt, \vcpu
ldr \vcpu, [\ctxt, #HOST_CONTEXT_VCPU]
@@ -261,6 +273,18 @@ extern u32 __kvm_get_mdcr_el2(void);
str \vcpu, [\ctxt, #HOST_CONTEXT_VCPU]
.endm
+.macro get_loaded_vcpu_ctxt vcpu, ctxt
+ adr_this_cpu \ctxt, kvm_hyp_ctxt, \vcpu
+ ldr \vcpu, [\ctxt, #HOST_CONTEXT_VCPU]
+ add \vcpu, \vcpu, #VCPU_CONTEXT
+.endm
+
+.macro get_loaded_vcpu_hyps vcpu, ctxt
+ adr_this_cpu \ctxt, kvm_hyp_ctxt, \vcpu
+ ldr \vcpu, [\ctxt, #HOST_CONTEXT_VCPU]
+ add \vcpu, \vcpu, #VCPU_HYPS
+.endm
+
/*
* KVM extable for unexpected exceptions.
* In the same format _asm_extable, but output to a different section so that
@@ -230,6 +230,13 @@ struct kvm_cpu_context {
struct kvm_vcpu *__hyp_running_vcpu;
};
+#define get_hyp_running_vcpu(ctxt) (ctxt)->__hyp_running_vcpu
+#define set_hyp_running_vcpu(ctxt, vcpu) (ctxt)->__hyp_running_vcpu = (vcpu)
+#define is_hyp_running_vcpu(ctxt) (ctxt)->__hyp_running_vcpu
+
+#define get_hyp_running_ctxt(host_ctxt) (host_ctxt)->__hyp_running_vcpu ? &(host_ctxt)->__hyp_running_vcpu->arch.ctxt : NULL
+#define get_hyp_running_hyps(host_ctxt) (host_ctxt)->__hyp_running_vcpu ? &(host_ctxt)->__hyp_running_vcpu->arch.hyp_state : NULL
+
struct kvm_pmu_events {
u32 events_host;
u32 events_guest;
@@ -107,6 +107,7 @@ int main(void)
BLANK();
#ifdef CONFIG_KVM
DEFINE(VCPU_CONTEXT, offsetof(struct kvm_vcpu, arch.ctxt));
+ DEFINE(VCPU_HYPS, offsetof(struct kvm_vcpu, arch.hyp_state));
DEFINE(VCPU_FAULT_DISR, offsetof(struct kvm_vcpu, arch.hyp_state.fault.disr_el1));
DEFINE(VCPU_WORKAROUND_FLAGS, offsetof(struct kvm_vcpu, arch.workaround_flags));
DEFINE(CPU_USER_PT_REGS, offsetof(struct kvm_cpu_context, regs));
@@ -80,7 +80,7 @@ static inline void __sysreg_restore_el1_state(struct kvm_cpu_context *ctxt)
!cpus_have_final_cap(ARM64_WORKAROUND_SPECULATIVE_AT)) {
write_sysreg_el1(ctxt_sys_reg(ctxt, SCTLR_EL1), SYS_SCTLR);
write_sysreg_el1(ctxt_sys_reg(ctxt, TCR_EL1), SYS_TCR);
- } else if (!ctxt->__hyp_running_vcpu) {
+ } else if (!is_hyp_running_vcpu(ctxt)) {
/*
* Must only be done for guest registers, hence the context
* test. We're coming from the host, so SCTLR.M is already
@@ -109,7 +109,7 @@ static inline void __sysreg_restore_el1_state(struct kvm_cpu_context *ctxt)
if (!has_vhe() &&
cpus_have_final_cap(ARM64_WORKAROUND_SPECULATIVE_AT) &&
- ctxt->__hyp_running_vcpu) {
+ is_hyp_running_vcpu(ctxt)) {
/*
* Must only be done for host registers, hence the context
* test. Pairs with nVHE's __deactivate_traps().
@@ -191,7 +191,7 @@ int __kvm_vcpu_run(struct kvm_vcpu *vcpu)
}
host_ctxt = &this_cpu_ptr(&kvm_host_data)->host_ctxt;
- host_ctxt->__hyp_running_vcpu = vcpu;
+ set_hyp_running_vcpu(host_ctxt, vcpu);
guest_ctxt = &vcpu->arch.ctxt;
pmu_switch_needed = __pmu_switch_to_guest(host_ctxt);
@@ -261,7 +261,7 @@ int __kvm_vcpu_run(struct kvm_vcpu *vcpu)
if (system_uses_irq_prio_masking())
gic_write_pmr(GIC_PRIO_IRQOFF);
- host_ctxt->__hyp_running_vcpu = NULL;
+ set_hyp_running_vcpu(host_ctxt, NULL);
return exit_code;
}
@@ -274,12 +274,10 @@ void __noreturn hyp_panic(void)
struct kvm_cpu_context *host_ctxt;
struct kvm_vcpu *vcpu;
struct vcpu_hyp_state *vcpu_hyps;
- struct kvm_cpu_context *vcpu_ctxt;
host_ctxt = &this_cpu_ptr(&kvm_host_data)->host_ctxt;
- vcpu = host_ctxt->__hyp_running_vcpu;
- vcpu_hyps = &hyp_state(vcpu);
- vcpu_ctxt = &vcpu_ctxt(vcpu);
+ vcpu = get_hyp_running_vcpu(host_ctxt);
+ vcpu_hyps = get_hyp_running_hyps(host_ctxt);
if (vcpu) {
__timer_disable_traps();
@@ -117,7 +117,7 @@ static int __kvm_vcpu_run_vhe(struct kvm_vcpu *vcpu)
u64 exit_code;
host_ctxt = &this_cpu_ptr(&kvm_host_data)->host_ctxt;
- host_ctxt->__hyp_running_vcpu = vcpu;
+ set_hyp_running_vcpu(host_ctxt, vcpu);
guest_ctxt = &vcpu->arch.ctxt;
sysreg_save_host_state_vhe(host_ctxt);
@@ -205,12 +205,10 @@ static void __hyp_call_panic(u64 spsr, u64 elr, u64 par)
struct kvm_cpu_context *host_ctxt;
struct kvm_vcpu *vcpu;
struct vcpu_hyp_state *vcpu_hyps;
- struct kvm_cpu_context *vcpu_ctxt;
host_ctxt = &this_cpu_ptr(&kvm_host_data)->host_ctxt;
- vcpu = host_ctxt->__hyp_running_vcpu;
- vcpu_hyps = &hyp_state(vcpu);
- vcpu_ctxt = &vcpu_ctxt(vcpu);
+ vcpu = get_hyp_running_vcpu(host_ctxt);
+ vcpu_hyps = get_hyp_running_hyps(host_ctxt);
__deactivate_traps(vcpu_hyps);
sysreg_restore_host_state_vhe(host_ctxt);
__hyp_running_vcpu exposes struct vcpu, but all that accesses it only need the cpu_ctxt and the hyp state. Start this refactoring by first ensuring that all accesses to __hyp_running_vcpu go via accessors and not directly. Signed-off-by: Fuad Tabba <tabba@google.com> --- arch/arm64/include/asm/kvm_asm.h | 24 ++++++++++++++++++++++ arch/arm64/include/asm/kvm_host.h | 7 +++++++ arch/arm64/kernel/asm-offsets.c | 1 + arch/arm64/kvm/hyp/include/hyp/sysreg-sr.h | 4 ++-- arch/arm64/kvm/hyp/nvhe/switch.c | 10 ++++----- arch/arm64/kvm/hyp/vhe/switch.c | 8 +++----- 6 files changed, 41 insertions(+), 13 deletions(-)