@@ -231,6 +231,19 @@ static int handle_sve(struct kvm_vcpu *vcpu)
return 1;
}
+/*
+ * Guest access to SME registers should be routed to this handler only
+ * when the system doesn't support SME.
+ */
+static int handle_sme(struct kvm_vcpu *vcpu)
+{
+ if (guest_hyp_sme_traps_enabled(vcpu))
+ return kvm_inject_nested_sync(vcpu, kvm_vcpu_get_esr(vcpu));
+
+ kvm_inject_undefined(vcpu);
+ return 1;
+}
+
/*
* Two possibilities to handle a trapping ptrauth instruction:
*
@@ -314,6 +327,7 @@ static exit_handle_fn arm_exit_handlers[] = {
[ESR_ELx_EC_SVC64] = handle_svc,
[ESR_ELx_EC_SYS64] = kvm_handle_sys_reg,
[ESR_ELx_EC_SVE] = handle_sve,
+ [ESR_ELx_EC_SME] = handle_sme,
[ESR_ELx_EC_ERET] = kvm_handle_eret,
[ESR_ELx_EC_IABT_LOW] = kvm_handle_guest_abort,
[ESR_ELx_EC_DABT_LOW] = kvm_handle_guest_abort,
@@ -100,8 +100,21 @@ static void fpsimd_sve_sync(struct kvm_vcpu *vcpu)
{
bool has_fpmr;
- if (!guest_owns_fp_regs())
+ if (!guest_owns_fp_regs()) {
+ /*
+ * We always restore SVCR for SME guests to ensure
+ * exceptions within the guest are delivered with the
+ * right type, always reset it to the fixed host
+ * value.
+ */
+ if (vcpu_has_sme(vcpu)) {
+ cpacr_clear_set(0, CPACR_EL1_SMEN);
+ isb();
+
+ sme_smstop();
+ }
return;
+ }
cpacr_clear_set(0, CPACR_EL1_FPEN | CPACR_EL1_ZEN | CPACR_EL1_SMEN);
isb();
@@ -249,10 +262,9 @@ static void handle___kvm_vcpu_run(struct kvm_cpu_context *host_ctxt)
struct pkvm_hyp_vcpu *hyp_vcpu = pkvm_get_loaded_hyp_vcpu();
/*
- * KVM (and pKVM) doesn't support SME guests for now, and
- * ensures that SME features aren't enabled in pstate when
- * loading a vcpu. Therefore, if SME features enabled the host
- * is misbehaving.
+ * KVM (and pKVM) refuses to run if PSTATE.{SM,ZA} are
+ * enabled. Therefore, if SME features enabled the
+ * host is misbehaving.
*/
if (unlikely(system_supports_sme() && read_sysreg_s(SYS_SVCR))) {
ret = -EINVAL;
@@ -35,12 +35,37 @@ DEFINE_PER_CPU(unsigned long, kvm_hyp_vector);
extern void kvm_nvhe_prepare_backtrace(unsigned long fp, unsigned long pc);
+static void __activate_traps_sme(struct kvm_vcpu *vcpu)
+{
+ if (!vcpu_has_sme(vcpu))
+ return;
+
+ if (__vcpu_sys_reg(vcpu, SVCR) == read_sysreg_s(SYS_SVCR))
+ return;
+
+ /*
+ * Write out the host state if it's in the registers,
+ * updating SVCR will invalidate it.
+ */
+ if (host_owns_fp_regs())
+ kvm_hyp_save_fpsimd_host(vcpu);
+
+ /*
+ * Always restore SVCR to ensure that exceptions delivered
+ * directly within the guest have the correct type.
+ */
+ write_sysreg_s(__vcpu_sys_reg(vcpu, SVCR), SYS_SVCR);
+ *host_data_ptr(fp_owner) = FP_STATE_FREE;
+}
+
static void __activate_cptr_traps(struct kvm_vcpu *vcpu)
{
u64 val = CPTR_EL2_TAM; /* Same bit irrespective of E2H */
- if (!guest_owns_fp_regs())
+ if (!guest_owns_fp_regs()) {
__activate_traps_fpsimd32(vcpu);
+ __activate_traps_sme(vcpu);
+ }
if (has_hvhe()) {
val |= CPACR_EL1_TTA;
@@ -49,17 +74,16 @@ static void __activate_cptr_traps(struct kvm_vcpu *vcpu)
val |= CPACR_EL1_FPEN;
if (vcpu_has_sve(vcpu))
val |= CPACR_EL1_ZEN;
+ if (vcpu_has_sme(vcpu))
+ val |= CPACR_EL1_SMEN;
}
write_sysreg(val, cpacr_el1);
} else {
val |= CPTR_EL2_TTA | CPTR_NVHE_EL2_RES1;
- /*
- * Always trap SME since it's not supported in KVM.
- * TSM is RES1 if SME isn't implemented.
- */
- val |= CPTR_EL2_TSM;
+ if (!vcpu_has_sme(vcpu) || !guest_owns_fp_regs())
+ val |= CPTR_EL2_TSM;
if (!vcpu_has_sve(vcpu) || !guest_owns_fp_regs())
val |= CPTR_EL2_TZ;
@@ -222,6 +246,7 @@ static const exit_handler_fn hyp_exit_handlers[] = {
[ESR_ELx_EC_CP15_32] = kvm_hyp_handle_cp15_32,
[ESR_ELx_EC_SYS64] = kvm_hyp_handle_sysreg,
[ESR_ELx_EC_SVE] = kvm_hyp_handle_fpsimd,
+ [ESR_ELx_EC_SME] = kvm_hyp_handle_fpsimd,
[ESR_ELx_EC_FP_ASIMD] = kvm_hyp_handle_fpsimd,
[ESR_ELx_EC_IABT_LOW] = kvm_hyp_handle_iabt_low,
[ESR_ELx_EC_DABT_LOW] = kvm_hyp_handle_dabt_low,
@@ -233,7 +258,8 @@ static const exit_handler_fn pvm_exit_handlers[] = {
[0 ... ESR_ELx_EC_MAX] = NULL,
[ESR_ELx_EC_SYS64] = kvm_handle_pvm_sys64,
[ESR_ELx_EC_SVE] = kvm_handle_pvm_restricted,
- [ESR_ELx_EC_FP_ASIMD] = kvm_hyp_handle_fpsimd,
+ [ESR_ELx_EC_SME] = kvm_handle_pvm_restricted,
+ [ESR_ELx_EC_FP_ASIMD] = kvm_handle_pvm_restricted,
[ESR_ELx_EC_IABT_LOW] = kvm_hyp_handle_iabt_low,
[ESR_ELx_EC_DABT_LOW] = kvm_hyp_handle_dabt_low,
[ESR_ELx_EC_WATCHPT_LOW] = kvm_hyp_handle_watchpt_low,
@@ -67,6 +67,7 @@ static u64 __compute_hcr(struct kvm_vcpu *vcpu)
static void __activate_cptr_traps(struct kvm_vcpu *vcpu)
{
+ struct kvm_cpu_context *ctxt = &vcpu->arch.ctxt;
u64 cptr;
/*
@@ -83,8 +84,17 @@ static void __activate_cptr_traps(struct kvm_vcpu *vcpu)
val |= CPACR_EL1_FPEN;
if (vcpu_has_sve(vcpu))
val |= CPACR_EL1_ZEN;
+ if (vcpu_has_sme(vcpu))
+ val |= CPACR_EL1_SMEN;
} else {
__activate_traps_fpsimd32(vcpu);
+
+ /*
+ * Streaming mode affects exception types delivered
+ * directly to lower ELs for FP operations, configure it.
+ */
+ if (vcpu_has_sme(vcpu))
+ write_sysreg_s(ctxt_sys_reg(ctxt, SVCR), SYS_SVCR);
}
if (!vcpu_has_nv(vcpu))
@@ -126,6 +136,8 @@ static void __activate_cptr_traps(struct kvm_vcpu *vcpu)
val &= ~CPACR_EL1_FPEN;
if (!(SYS_FIELD_GET(CPACR_EL1, ZEN, cptr) & BIT(0)))
val &= ~CPACR_EL1_ZEN;
+ if (!(SYS_FIELD_GET(CPACR_EL1, SMEN, cptr) & BIT(0)))
+ val &= ~CPACR_EL1_SMEN;
if (kvm_has_feat(vcpu->kvm, ID_AA64MMFR3_EL1, S2POE, IMP))
val |= cptr & CPACR_EL1_E0POE;
@@ -486,22 +498,28 @@ static bool kvm_hyp_handle_cpacr_el1(struct kvm_vcpu *vcpu, u64 *exit_code)
return true;
}
-static bool kvm_hyp_handle_zcr_el2(struct kvm_vcpu *vcpu, u64 *exit_code)
+static bool kvm_hyp_handle_vec_cr_el2(struct kvm_vcpu *vcpu, u64 *exit_code)
{
u32 sysreg = esr_sys64_to_sysreg(kvm_vcpu_get_esr(vcpu));
if (!vcpu_has_nv(vcpu))
return false;
- if (sysreg != SYS_ZCR_EL2)
+ switch (sysreg) {
+ case SYS_ZCR_EL2:
+ case SYS_SMCR_EL2:
+ break;
+ default:
return false;
+ }
if (guest_owns_fp_regs())
return false;
/*
- * ZCR_EL2 traps are handled in the slow path, with the expectation
- * that the guest's FP context has already been loaded onto the CPU.
+ * ZCR_EL2 and SMCR_EL2 traps are handled in the slow path,
+ * with the expectation that the guest's FP context has
+ * already been loaded onto the CPU.
*
* Load the guest's FP context and unconditionally forward to the
* slow path for handling (i.e. return false).
@@ -521,7 +539,7 @@ static bool kvm_hyp_handle_sysreg_vhe(struct kvm_vcpu *vcpu, u64 *exit_code)
if (kvm_hyp_handle_cpacr_el1(vcpu, exit_code))
return true;
- if (kvm_hyp_handle_zcr_el2(vcpu, exit_code))
+ if (kvm_hyp_handle_vec_cr_el2(vcpu, exit_code))
return true;
return kvm_hyp_handle_sysreg(vcpu, exit_code);
@@ -550,6 +568,7 @@ static const exit_handler_fn hyp_exit_handlers[] = {
[0 ... ESR_ELx_EC_MAX] = NULL,
[ESR_ELx_EC_CP15_32] = kvm_hyp_handle_cp15_32,
[ESR_ELx_EC_SYS64] = kvm_hyp_handle_sysreg_vhe,
+ [ESR_ELx_EC_SME] = kvm_hyp_handle_fpsimd,
[ESR_ELx_EC_SVE] = kvm_hyp_handle_fpsimd,
[ESR_ELx_EC_FP_ASIMD] = kvm_hyp_handle_fpsimd,
[ESR_ELx_EC_IABT_LOW] = kvm_hyp_handle_iabt_low,
The access control for SME follows the same structure as for the base FP and SVE extensions, with control being via CPACR_ELx.SMEN and CPTR_EL2.TSM mirroring the equivalent FPSIMD and SVE controls in those registers. Add handling for these controls and exceptions mirroring the existing handling for FPSIMD and SVE. When the hardware is in streaming mode guest operations that are invalid in in streaming mode will generate SME exceptions. In many situations these exceptions are routed directly to the lower ELs with no opportunity for the hypervisor to intercept. So that guests do not see unexpected exception types due to the hardware streaming mode configuration not being what the guest configured we update SVCR when enabling floating point traps for the guest. For pKVM this means that when changing SVCR we need to save the host floating point state before returning to the guest. Signed-off-by: Mark Brown <broonie@kernel.org> squash ctxtsync --- arch/arm64/kvm/handle_exit.c | 14 +++++++++++++ arch/arm64/kvm/hyp/nvhe/hyp-main.c | 22 ++++++++++++++++----- arch/arm64/kvm/hyp/nvhe/switch.c | 40 +++++++++++++++++++++++++++++++------- arch/arm64/kvm/hyp/vhe/switch.c | 29 ++++++++++++++++++++++----- 4 files changed, 88 insertions(+), 17 deletions(-)