@@ -87,6 +87,8 @@ extern void sve_load_from_fpsimd_state(struct user_fpsimd_state const *state,
extern unsigned int sve_get_vl(void);
+extern void sve_set_vq(unsigned long vq_minus_1);
+
struct arm64_cpu_capabilities;
extern void sve_kernel_enable(const struct arm64_cpu_capabilities *__unused);
@@ -58,6 +58,11 @@ ENTRY(sve_get_vl)
ret
ENDPROC(sve_get_vl)
+ENTRY(sve_set_vq)
+ sve_load_vq x0, x1, x2
+ ret
+ENDPROC(sve_set_vq)
+
/*
* Load SVE state from FPSIMD state.
*
@@ -815,10 +815,8 @@ void fpsimd_release_task(struct task_struct *dead_task)
/*
* Trapped SVE access
*
- * Storage is allocated for the full SVE state, the current FPSIMD
- * register contents are migrated across, and TIF_SVE is set so that
- * the SVE access trap will be disabled the next time this task
- * reaches ret_to_user.
+ * Storage is allocated for the full SVE state and rely on the return
+ * code to actually convert the FPSIMD state to SVE state.
*
* TIF_SVE should be clear on entry: otherwise, fpsimd_restore_current_state()
* would have disabled the SVE access trap for userspace during
@@ -836,15 +834,20 @@ asmlinkage void do_sve_acc(unsigned int esr, struct pt_regs *regs)
local_bh_disable();
- fpsimd_save();
- fpsimd_to_sve(current);
-
- /* Force ret_to_user to reload the registers: */
- fpsimd_flush_task_state(current);
- set_thread_flag(TIF_FOREIGN_FPSTATE);
+ set_thread_flag(TIF_SVE_NEEDS_FLUSH);
+ /*
+ * We should not be here with SVE enabled. TIF_SVE will be set
+ * before returning to userspace by fpsimd_restore_current_state().
+ */
+ WARN_ON(test_thread_flag(TIF_SVE));
- if (test_and_set_thread_flag(TIF_SVE))
- WARN_ON(1); /* SVE access shouldn't have trapped */
+ /*
+ * The return path (see fpsimd_restore_current_state) requires the
+ * Vector-Length to be loaded beforehand when the FPSIMD state is
+ * loaded.
+ */
+ if (!test_thread_flag(TIF_FOREIGN_FPSTATE))
+ sve_set_vq(sve_vq_from_vl(current->thread.sve_vl) - 1);
local_bh_enable();
}
@@ -1040,6 +1043,9 @@ void fpsimd_restore_current_state(void)
/*
* The userspace had SVE enabled on entry to the kernel
* and requires the state to be flushed.
+ *
+ * We rely on the Vector-Length to be set correctly before-hand
+ * when converting a loaded FPSIMD state to SVE state.
*/
sve_flush_live();
sve_user_enable();
SVE state will be flushed on the first SVE access trap. At the moment, the SVE state will be generated from the FPSIMD state in software and then loaded in memory. It is possible to use the newly introduce flag TIF_SVE_NEEDS_FLUSH to avoid a lot of memory access. If the FPSIMD state is in memory, the SVE state will be loaded on return to userspace from the FPSIMD state. If the FPSIMD state is loaded, then we need to the vector-length before relying on return to userspace to flush the SVE registers. This is because the vector-length is only set when loading from memory. Signed-off-by: Julien Grall <julien.grall@arm.com> --- arch/arm64/include/asm/fpsimd.h | 2 ++ arch/arm64/kernel/entry-fpsimd.S | 5 +++++ arch/arm64/kernel/fpsimd.c | 30 ++++++++++++++++++------------ 3 files changed, 25 insertions(+), 12 deletions(-)