@@ -314,7 +314,9 @@ void __noreturn __cold nvhe_hyp_panic_handler(u64 esr, u64 spsr, u64 elr_virt,
.endm
.macro get_ctxt_gp_regs ctxt, regs
- add \regs, \ctxt, #CPU_USER_PT_REGS
+alternative_cb ARM64_HAS_VIRT_HOST_EXTN, kvm_update_ctxt_gp_regs
+ add \regs, \ctxt, #CPU_USER_PT_REGS_STRG
+alternative_cb_end
.endm
/*
@@ -541,7 +541,9 @@ struct kvm_sysreg_masks {
};
struct kvm_cpu_context {
- struct user_pt_regs regs; /* sp = sp_el0 */
+ struct user_pt_regs *regs; /* sp = sp_el0 */
+ struct user_pt_regs regs_storage;
+ struct secretmem_area *regs_area;
u64 spsr_abt;
u64 spsr_und;
@@ -946,7 +948,25 @@ struct kvm_vcpu_arch {
#define vcpu_clear_on_unsupported_cpu(vcpu) \
vcpu_clear_flag(vcpu, ON_UNSUPPORTED_CPU)
-#define ctxt_gp_regs(ctxt) (&(ctxt)->regs)
+/* Static allocation is used if NVHE-host or if KERNEL_SECRETMEM is not enabled */
+static __inline bool kvm_use_dynamic_regs(void)
+{
+#ifndef CONFIG_KERNEL_SECRETMEM
+ return false;
+#endif
+ return cpus_have_cap(ARM64_HAS_VIRT_HOST_EXTN);
+}
+
+static __always_inline struct user_pt_regs *ctxt_gp_regs(const struct kvm_cpu_context *ctxt)
+{
+ struct user_pt_regs *regs = (void *) ctxt;
+ asm volatile(ALTERNATIVE_CB("add %0, %0, %1\n",
+ ARM64_HAS_VIRT_HOST_EXTN,
+ kvm_update_ctxt_gp_regs)
+ : "+r" (regs)
+ : "I" (offsetof(struct kvm_cpu_context, regs_storage)));
+ return regs;
+}
#define vcpu_gp_regs(v) (ctxt_gp_regs(&(v)->arch.ctxt))
/*
@@ -128,6 +128,7 @@ int main(void)
DEFINE(VCPU_FAULT_DISR, offsetof(struct kvm_vcpu, arch.fault.disr_el1));
DEFINE(VCPU_HCR_EL2, offsetof(struct kvm_vcpu, arch.hcr_el2));
DEFINE(CPU_USER_PT_REGS, offsetof(struct kvm_cpu_context, regs));
+ DEFINE(CPU_USER_PT_REGS_STRG, offsetof(struct kvm_cpu_context, regs_storage));
DEFINE(CPU_ELR_EL2, offsetof(struct kvm_cpu_context, sys_regs[ELR_EL2]));
DEFINE(CPU_RGSR_EL1, offsetof(struct kvm_cpu_context, sys_regs[RGSR_EL1]));
DEFINE(CPU_GCR_EL1, offsetof(struct kvm_cpu_context, sys_regs[GCR_EL1]));
@@ -86,6 +86,7 @@ KVM_NVHE_ALIAS(kvm_patch_vector_branch);
KVM_NVHE_ALIAS(kvm_update_va_mask);
KVM_NVHE_ALIAS(kvm_get_kimage_voffset);
KVM_NVHE_ALIAS(kvm_compute_final_ctr_el0);
+KVM_NVHE_ALIAS(kvm_update_ctxt_gp_regs);
KVM_NVHE_ALIAS(spectre_bhb_patch_loop_iter);
KVM_NVHE_ALIAS(spectre_bhb_patch_loop_mitigation_enable);
KVM_NVHE_ALIAS(spectre_bhb_patch_wa3);
@@ -16,6 +16,7 @@
#include <linux/fs.h>
#include <linux/mman.h>
#include <linux/sched.h>
+#include <linux/secretmem.h>
#include <linux/kvm.h>
#include <linux/kvm_irqfd.h>
#include <linux/irqbypass.h>
@@ -452,6 +453,7 @@ int kvm_arch_vcpu_precreate(struct kvm *kvm, unsigned int id)
int kvm_arch_vcpu_create(struct kvm_vcpu *vcpu)
{
+ unsigned long pages_needed;
int err;
spin_lock_init(&vcpu->arch.mp_state_lock);
@@ -469,6 +471,14 @@ int kvm_arch_vcpu_create(struct kvm_vcpu *vcpu)
vcpu->arch.mmu_page_cache.gfp_zero = __GFP_ZERO;
+ if (kvm_use_dynamic_regs()) {
+ pages_needed = (sizeof(*vcpu_gp_regs(vcpu)) + PAGE_SIZE - 1) / PAGE_SIZE;
+ vcpu->arch.ctxt.regs_area = secretmem_allocate_pages(fls(pages_needed - 1));
+ if (!vcpu->arch.ctxt.regs_area)
+ return -ENOMEM;
+ vcpu->arch.ctxt.regs = vcpu->arch.ctxt.regs_area->ptr;
+ }
+
/* Set up the timer */
kvm_timer_vcpu_init(vcpu);
@@ -489,9 +499,14 @@ int kvm_arch_vcpu_create(struct kvm_vcpu *vcpu)
err = kvm_vgic_vcpu_init(vcpu);
if (err)
- return err;
+ goto free_vcpu_ctxt;
return kvm_share_hyp(vcpu, vcpu + 1);
+
+free_vcpu_ctxt:
+ if (kvm_use_dynamic_regs())
+ secretmem_release_pages(vcpu->arch.ctxt.regs_area);
+ return err;
}
void kvm_arch_vcpu_postcreate(struct kvm_vcpu *vcpu)
@@ -508,6 +523,9 @@ void kvm_arch_vcpu_destroy(struct kvm_vcpu *vcpu)
kvm_pmu_vcpu_destroy(vcpu);
kvm_vgic_vcpu_destroy(vcpu);
kvm_arm_vcpu_destroy(vcpu);
+
+ if (kvm_use_dynamic_regs())
+ secretmem_release_pages(vcpu->arch.ctxt.regs_area);
}
void kvm_arch_vcpu_blocking(struct kvm_vcpu *vcpu)
@@ -2683,6 +2701,45 @@ static int __init init_hyp_mode(void)
return err;
}
+static int init_hyp_hve_mode(void)
+{
+ int cpu;
+ int err = 0;
+
+ if (!kvm_use_dynamic_regs())
+ return 0;
+
+ /* Allocate gp-regs */
+ for_each_possible_cpu(cpu) {
+ void *hyp_ctxt_regs;
+ void *kvm_host_data_regs;
+
+ hyp_ctxt_regs = kzalloc(sizeof(struct user_pt_regs), GFP_KERNEL);
+ if (!hyp_ctxt_regs) {
+ err = -ENOMEM;
+ goto free_regs;
+ }
+ per_cpu(kvm_hyp_ctxt, cpu).regs = hyp_ctxt_regs;
+
+ kvm_host_data_regs = kzalloc(sizeof(struct user_pt_regs), GFP_KERNEL);
+ if (!kvm_host_data_regs) {
+ err = -ENOMEM;
+ goto free_regs;
+ }
+ per_cpu(kvm_host_data, cpu).host_ctxt.regs = kvm_host_data_regs;
+ }
+
+ return 0;
+
+free_regs:
+ for_each_possible_cpu(cpu) {
+ kfree(per_cpu(kvm_hyp_ctxt, cpu).regs);
+ kfree(per_cpu(kvm_host_data, cpu).host_ctxt.regs);
+ }
+
+ return err;
+}
+
struct kvm_vcpu *kvm_mpidr_to_vcpu(struct kvm *kvm, unsigned long mpidr)
{
struct kvm_vcpu *vcpu = NULL;
@@ -2806,6 +2863,10 @@ static __init int kvm_arm_init(void)
err = init_hyp_mode();
if (err)
goto out_err;
+ } else {
+ err = init_hyp_hve_mode();
+ if (err)
+ goto out_err;
}
err = kvm_init_vector_slots();
@@ -185,6 +185,29 @@ void __init kvm_update_va_mask(struct alt_instr *alt,
}
}
+void __init kvm_update_ctxt_gp_regs(struct alt_instr *alt,
+ __le32 *origptr, __le32 *updptr, int nr_inst)
+{
+ u32 rd, rn, imm, insn, oinsn;
+
+ BUG_ON(nr_inst != 1);
+
+ if (!kvm_use_dynamic_regs())
+ return;
+
+ oinsn = le32_to_cpu(origptr[0]);
+ rd = aarch64_insn_decode_register(AARCH64_INSN_REGTYPE_RD, oinsn);
+ rn = aarch64_insn_decode_register(AARCH64_INSN_REGTYPE_RN, oinsn);
+ imm = offsetof(struct kvm_cpu_context, regs);
+
+ insn = aarch64_insn_gen_load_store_imm(rd, rn, imm,
+ AARCH64_INSN_SIZE_64,
+ AARCH64_INSN_LDST_LOAD_IMM_OFFSET);
+ BUG_ON(insn == AARCH64_BREAK_FAULT);
+
+ updptr[0] = cpu_to_le32(insn);
+}
+
void kvm_patch_vector_branch(struct alt_instr *alt,
__le32 *origptr, __le32 *updptr, int nr_inst)
{
To allocate the vCPU gp-regs using secret memory, we need to dynamically allocate the vCPU gp-regs first. This is tricky with NVHE (Non-Virtualization Host Extensions) since it will require adjusting the virtual address on every access. With a large shared codebase between the OS and the hypervisor, it would be cumbersome to duplicate the code with one version using `kern_hyp_va()`. To avoid this issue, and since the secret memory feature will not be enabled on NVHE systems, we're introducing the following changes: 1. Maintain a `struct user_pt_regs regs_storage` in the vCPU context struct as a fallback storage for the vCPU gp-regs. 2. Introduce a pointer `struct user_pt_regs *regs` in the vCPU context struct to hold the dynamically allocated vCPU gp-regs. If we are on an NVHE system or a VHE (Virtualization Host Extensions) system that doesn't support `KERNEL_SECRETMEM`, we will use `ctxt_storage`. Accessing the context in this case will not require a de-reference operation. If we are on a VHE system with support for `KERNEL_SECRETMEM`, we will use the `regs` pointer. In this case, we will add one de-reference operation every time the vCPU gp-reg is accessed. Accessing the gp-regs embedded in the vCPU context without de-reference is done as: add \regs, \ctxt, #CPU_USER_PT_REGS_STRG Accessing the dynamically allocated gp-regs with de-reference is done as: ldr \regs, [\ctxt, #CPU_USER_PT_REGS] By default, we are using the first version. If we are booting on a system that supports VHE and `KERNEL_SECRETMEM`, we switch to the second version. We are also allocating the needed gp-regs allocations for vCPU, kvm_hyp_ctxt and kvm_host_data structs when needed. Signed-off-by: Fares Mehanna <faresx@amazon.de> --- arch/arm64/include/asm/kvm_asm.h | 4 +- arch/arm64/include/asm/kvm_host.h | 24 +++++++++++- arch/arm64/kernel/asm-offsets.c | 1 + arch/arm64/kernel/image-vars.h | 1 + arch/arm64/kvm/arm.c | 63 ++++++++++++++++++++++++++++++- arch/arm64/kvm/va_layout.c | 23 +++++++++++ 6 files changed, 112 insertions(+), 4 deletions(-)