diff mbox series

[v3,2/2] x86/copy_mc: Introduce copy_mc_generic()

Message ID 158992636214.403910.12184670538732959406.stgit@dwillia2-desk3.amr.corp.intel.com (mailing list archive)
State Superseded
Headers show
Series Renovate memcpy_mcsafe with copy_mc_to_{user, kernel} | expand

Commit Message

Dan Williams May 19, 2020, 10:12 p.m. UTC 
The original copy_mc_fragile() implementation had negative performance
implications since it did not use the fast-string instruction sequence
to perform copies. For this reason copy_mc_to_kernel() fell back to
plain memcpy() to preserve performance on platform that did not indicate
the capability to recover from machine check exceptions. However, that
capability detection was not architectural and now that some platforms
can recover from fast-string consumption of memory errors the memcpy()
fallback now causes these more capable platforms to fail.

Introduce copy_mc_generic() as the fast default implementation of
copy_mc_to_kernel() and finalize the transition of copy_mc_fragile() to
be a platform quirk to indicate 'fragility'. With this in place
copy_mc_to_kernel() is fast and recovery-ready by default regardless of
hardware capability.

Thanks to Vivek for identifying that copy_user_generic() is not suitable
as the copy_mc_to_user() backend since the #MC handler explicitly checks
ex_has_fault_handler().

Cc: x86@kernel.org
Cc: <stable@vger.kernel.org>
Cc: Ingo Molnar <mingo@redhat.com>
Cc: Borislav Petkov <bp@alien8.de>
Cc: Vivek Goyal <vgoyal@redhat.com>
Cc: Tony Luck <tony.luck@intel.com>
Cc: "H. Peter Anvin" <hpa@zytor.com>
Cc: Andy Lutomirski <luto@kernel.org>
Cc: Thomas Gleixner <tglx@linutronix.de>
Cc: Peter Zijlstra <peterz@infradead.org>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Reported-by: Erwin Tsaur <erwin.tsaur@intel.com>
Tested-by: Erwin Tsaur <erwin.tsaur@intel.com>
Fixes: 92b0729c34ca ("x86/mm, x86/mce: Add memcpy_mcsafe()")
Signed-off-by: Dan Williams <dan.j.williams@intel.com>
---
 arch/x86/include/asm/uaccess_64.h |    3 +++
 arch/x86/lib/copy_mc.c            |   12 +++++------
 arch/x86/lib/copy_mc_64.S         |   40 +++++++++++++++++++++++++++++++++++++
 tools/objtool/check.c             |    1 +
 4 files changed, 49 insertions(+), 7 deletions(-)

Comments

Vivek Goyal May 20, 2020, 7:13 p.m. UTC | #1 
On Tue, May 19, 2020 at 03:12:42PM -0700, Dan Williams wrote:
> The original copy_mc_fragile() implementation had negative performance
> implications since it did not use the fast-string instruction sequence
> to perform copies. For this reason copy_mc_to_kernel() fell back to
> plain memcpy() to preserve performance on platform that did not indicate
> the capability to recover from machine check exceptions. However, that
> capability detection was not architectural and now that some platforms
> can recover from fast-string consumption of memory errors the memcpy()
> fallback now causes these more capable platforms to fail.
> 
> Introduce copy_mc_generic() as the fast default implementation of
> copy_mc_to_kernel() and finalize the transition of copy_mc_fragile() to
> be a platform quirk to indicate 'fragility'. With this in place
> copy_mc_to_kernel() is fast and recovery-ready by default regardless of
> hardware capability.
> 
> Thanks to Vivek for identifying that copy_user_generic() is not suitable
> as the copy_mc_to_user() backend since the #MC handler explicitly checks
> ex_has_fault_handler().

/me is curious to know why #MC handler mandates use of _ASM_EXTABLE_FAULT().

[..]
> +/*
> + * copy_mc_generic - memory copy with exception handling
> + *
> + * Fast string copy + fault / exception handling. If the CPU does
> + * support machine check exception recovery, but does not support
> + * recovering from fast-string exceptions then this CPU needs to be
> + * added to the copy_mc_fragile_key set of quirks. Otherwise, absent any
> + * machine check recovery support this version should be no slower than
> + * standard memcpy.
> + */
> +SYM_FUNC_START(copy_mc_generic)
> +	ALTERNATIVE "jmp copy_mc_fragile", "", X86_FEATURE_ERMS
> +	movq %rdi, %rax
> +	movq %rdx, %rcx
> +.L_copy:
> +	rep movsb
> +	/* Copy successful. Return zero */
> +	xorl %eax, %eax
> +	ret
> +SYM_FUNC_END(copy_mc_generic)
> +EXPORT_SYMBOL_GPL(copy_mc_generic)
> +
> +	.section .fixup, "ax"
> +.E_copy:
> +	/*
> +	 * On fault %rcx is updated such that the copy instruction could
> +	 * optionally be restarted at the fault position, i.e. it
> +	 * contains 'bytes remaining'. A non-zero return indicates error
> +	 * to copy_safe() users, or indicate short transfers to

copy_safe() is vestige of terminology of previous patches?

> +	 * user-copy routines.
> +	 */
> +	movq %rcx, %rax
> +	ret
> +
> +	.previous
> +
> +	_ASM_EXTABLE_FAULT(.L_copy, .E_copy)

A question for my education purposes.

So copy_mc_generic() can handle MCE both on source and destination
addresses? (Assuming some device can generate MCE on stores too).
On the other hand copy_mc_fragile() handles MCE recovery only on
source and non-MCE recovery on destination.

Thanks
Vivek
Dan Williams May 20, 2020, 9:57 p.m. UTC | #2 
On Wed, May 20, 2020 at 12:13 PM Vivek Goyal <vgoyal@redhat.com> wrote:
>
> On Tue, May 19, 2020 at 03:12:42PM -0700, Dan Williams wrote:
> > The original copy_mc_fragile() implementation had negative performance
> > implications since it did not use the fast-string instruction sequence
> > to perform copies. For this reason copy_mc_to_kernel() fell back to
> > plain memcpy() to preserve performance on platform that did not indicate
> > the capability to recover from machine check exceptions. However, that
> > capability detection was not architectural and now that some platforms
> > can recover from fast-string consumption of memory errors the memcpy()
> > fallback now causes these more capable platforms to fail.
> >
> > Introduce copy_mc_generic() as the fast default implementation of
> > copy_mc_to_kernel() and finalize the transition of copy_mc_fragile() to
> > be a platform quirk to indicate 'fragility'. With this in place
> > copy_mc_to_kernel() is fast and recovery-ready by default regardless of
> > hardware capability.
> >
> > Thanks to Vivek for identifying that copy_user_generic() is not suitable
> > as the copy_mc_to_user() backend since the #MC handler explicitly checks
> > ex_has_fault_handler().
>
> /me is curious to know why #MC handler mandates use of _ASM_EXTABLE_FAULT().

Even though we could try to handle all faults / exceptions
generically, I think it makes sense to enforce type safety here if
only to support architectures that can only satisfy the minimum
contract of copy_mc_to_user(). For example, if there was some
destination exception other than #PF the contract implied by
copy_mc_to_user() is that exception is not intended to be permissible
in this path. See:

00c42373d397 x86-64: add warning for non-canonical user access address
dereferences
75045f77f7a7 x86/extable: Introduce _ASM_EXTABLE_UA for uaccess fixups

...for examples of other justification for being explicit in these paths.

>
> [..]
> > +/*
> > + * copy_mc_generic - memory copy with exception handling
> > + *
> > + * Fast string copy + fault / exception handling. If the CPU does
> > + * support machine check exception recovery, but does not support
> > + * recovering from fast-string exceptions then this CPU needs to be
> > + * added to the copy_mc_fragile_key set of quirks. Otherwise, absent any
> > + * machine check recovery support this version should be no slower than
> > + * standard memcpy.
> > + */
> > +SYM_FUNC_START(copy_mc_generic)
> > +     ALTERNATIVE "jmp copy_mc_fragile", "", X86_FEATURE_ERMS
> > +     movq %rdi, %rax
> > +     movq %rdx, %rcx
> > +.L_copy:
> > +     rep movsb
> > +     /* Copy successful. Return zero */
> > +     xorl %eax, %eax
> > +     ret
> > +SYM_FUNC_END(copy_mc_generic)
> > +EXPORT_SYMBOL_GPL(copy_mc_generic)
> > +
> > +     .section .fixup, "ax"
> > +.E_copy:
> > +     /*
> > +      * On fault %rcx is updated such that the copy instruction could
> > +      * optionally be restarted at the fault position, i.e. it
> > +      * contains 'bytes remaining'. A non-zero return indicates error
> > +      * to copy_safe() users, or indicate short transfers to
>
> copy_safe() is vestige of terminology of previous patches?

Thanks, yes, I missed this one.

>
> > +      * user-copy routines.
> > +      */
> > +     movq %rcx, %rax
> > +     ret
> > +
> > +     .previous
> > +
> > +     _ASM_EXTABLE_FAULT(.L_copy, .E_copy)
>
> A question for my education purposes.
>
> So copy_mc_generic() can handle MCE both on source and destination
> addresses? (Assuming some device can generate MCE on stores too).

There's no such thing as #MC on write. #MC is only signaled on consumed poison.

In this case what is specifically being handled is #MC with RIP
pointing at a movq instruction. The fault handler actually does not
know anything about source or destination, it just knows fault /
exception type and the register state.

> On the other hand copy_mc_fragile() handles MCE recovery only on
> source and non-MCE recovery on destination.

No, there's no difference in capability. #MC can only be raised on a
poison-read in both cases.
diff mbox series

Patch

diff --git a/arch/x86/include/asm/uaccess_64.h b/arch/x86/include/asm/uaccess_64.h
index 9f7e7e27f0a4..422d92460dda 100644
--- a/arch/x86/include/asm/uaccess_64.h
+++ b/arch/x86/include/asm/uaccess_64.h
@@ -58,6 +58,9 @@  copy_mc_to_user(void *to, const void *from, unsigned len);
 
 unsigned long __must_check
 copy_mc_fragile(void *dst, const void *src, unsigned cnt);
+
+unsigned long __must_check
+copy_mc_generic(void *dst, const void *src, unsigned cnt);
 #else
 static inline void enable_copy_mc_fragile(void)
 {
diff --git a/arch/x86/lib/copy_mc.c b/arch/x86/lib/copy_mc.c
index cdb8f5dc403d..9e6fac1ab72e 100644
--- a/arch/x86/lib/copy_mc.c
+++ b/arch/x86/lib/copy_mc.c
@@ -23,7 +23,7 @@  void enable_copy_mc_fragile(void)
  *
  * Call into the 'fragile' version on systems that have trouble
  * actually do machine check recovery. Everyone else can just
- * use memcpy().
+ * use copy_mc_generic().
  *
  * Return 0 for success, or number of bytes not copied if there was an
  * exception.
@@ -33,8 +33,7 @@  copy_mc_to_kernel(void *dst, const void *src, unsigned cnt)
 {
 	if (static_branch_unlikely(&copy_mc_fragile_key))
 		return copy_mc_fragile(dst, src, cnt);
-	memcpy(dst, src, cnt);
-	return 0;
+	return copy_mc_generic(dst, src, cnt);
 }
 EXPORT_SYMBOL_GPL(copy_mc_to_kernel);
 
@@ -56,11 +55,10 @@  copy_mc_to_user(void *to, const void *from, unsigned len)
 {
 	unsigned long ret;
 
-	if (!static_branch_unlikely(&copy_mc_fragile_key))
-		return copy_user_generic(to, from, len);
-
 	__uaccess_begin();
-	ret = copy_mc_fragile(to, from, len);
+	if (static_branch_unlikely(&copy_mc_fragile_key))
+		ret = copy_mc_fragile(to, from, len);
+	ret = copy_mc_generic(to, from, len);
 	__uaccess_end();
 	return ret;
 }
diff --git a/arch/x86/lib/copy_mc_64.S b/arch/x86/lib/copy_mc_64.S
index 35a67c50890b..96dc4b8fb954 100644
--- a/arch/x86/lib/copy_mc_64.S
+++ b/arch/x86/lib/copy_mc_64.S
@@ -2,7 +2,9 @@ 
 /* Copyright(c) 2016-2020 Intel Corporation. All rights reserved. */
 
 #include <linux/linkage.h>
+#include <asm/alternative-asm.h>
 #include <asm/copy_mc_test.h>
+#include <asm/cpufeatures.h>
 #include <asm/export.h>
 #include <asm/asm.h>
 
@@ -122,4 +124,42 @@  EXPORT_SYMBOL_GPL(copy_mc_fragile)
 	_ASM_EXTABLE(.L_write_leading_bytes, .E_leading_bytes)
 	_ASM_EXTABLE(.L_write_words, .E_write_words)
 	_ASM_EXTABLE(.L_write_trailing_bytes, .E_trailing_bytes)
+
+/*
+ * copy_mc_generic - memory copy with exception handling
+ *
+ * Fast string copy + fault / exception handling. If the CPU does
+ * support machine check exception recovery, but does not support
+ * recovering from fast-string exceptions then this CPU needs to be
+ * added to the copy_mc_fragile_key set of quirks. Otherwise, absent any
+ * machine check recovery support this version should be no slower than
+ * standard memcpy.
+ */
+SYM_FUNC_START(copy_mc_generic)
+	ALTERNATIVE "jmp copy_mc_fragile", "", X86_FEATURE_ERMS
+	movq %rdi, %rax
+	movq %rdx, %rcx
+.L_copy:
+	rep movsb
+	/* Copy successful. Return zero */
+	xorl %eax, %eax
+	ret
+SYM_FUNC_END(copy_mc_generic)
+EXPORT_SYMBOL_GPL(copy_mc_generic)
+
+	.section .fixup, "ax"
+.E_copy:
+	/*
+	 * On fault %rcx is updated such that the copy instruction could
+	 * optionally be restarted at the fault position, i.e. it
+	 * contains 'bytes remaining'. A non-zero return indicates error
+	 * to copy_safe() users, or indicate short transfers to
+	 * user-copy routines.
+	 */
+	movq %rcx, %rax
+	ret
+
+	.previous
+
+	_ASM_EXTABLE_FAULT(.L_copy, .E_copy)
 #endif
diff --git a/tools/objtool/check.c b/tools/objtool/check.c
index b62235004f0b..449a36101385 100644
--- a/tools/objtool/check.c
+++ b/tools/objtool/check.c
@@ -546,6 +546,7 @@  static const char *uaccess_safe_builtin[] = {
 	"__ubsan_handle_shift_out_of_bounds",
 	/* misc */
 	"csum_partial_copy_generic",
+	"copy_mc_generic",
 	"copy_mc_fragile",
 	"copy_mc_fragile_handle_tail",
 	"ftrace_likely_update", /* CONFIG_TRACE_BRANCH_PROFILING */