| Message ID | 20200710165203.31284-20-will@kernel.org (mailing list archive) |
|---|---|
| State | New, archived |
| Headers | show |
| Series | Allow architectures to override __READ_ONCE() | expand |
On Fri 2020-07-10 17:52:03, Will Deacon wrote: > When building with LTO, there is an increased risk of the compiler > converting an address dependency headed by a READ_ONCE() invocation > into a control dependency and consequently allowing for harmful > reordering by the CPU. > > Ensure that such transformations are harmless by overriding the generic > READ_ONCE() definition with one that provides acquire semantics when > building with LTO. Traditionally, READ_ONCE had only effects on compiler optimalizations, not on special semantics of the load instruction. Do you have example how LTO optimalizations break the code? Should some documentation be added? Because I believe users will need to understand what is going on there. It is not LTO-only problem and it is not arm64-only problem, right? Best regards, Pavel > +#ifdef CONFIG_AS_HAS_LDAPR > +#define __LOAD_RCPC(sfx, regs...) \ > + ALTERNATIVE( \ > + "ldar" #sfx "\t" #regs, \ > + ".arch_extension rcpc\n" \ > + "ldapr" #sfx "\t" #regs, \ > + ARM64_HAS_LDAPR) > +#else > +#define __LOAD_RCPC(sfx, regs...) "ldar" #sfx "\t" #regs > +#endif /* CONFIG_AS_HAS_LDAPR */ > + > +#define __READ_ONCE(x) \ > +({ \ > + typeof(&(x)) __x = &(x); \ > + int atomic = 1; \ > + union { __unqual_scalar_typeof(*__x) __val; char __c[1]; } __u; \ > + switch (sizeof(x)) { \ > + case 1: \ > + asm volatile(__LOAD_RCPC(b, %w0, %1) \ > + : "=r" (*(__u8 *)__u.__c) \ > + : "Q" (*__x) : "memory"); \ > + break; \ > + case 2: \ > + asm volatile(__LOAD_RCPC(h, %w0, %1) \ > + : "=r" (*(__u16 *)__u.__c) \ > + : "Q" (*__x) : "memory"); \ > + break; \ > + case 4: \ > + asm volatile(__LOAD_RCPC(, %w0, %1) \ > + : "=r" (*(__u32 *)__u.__c) \ > + : "Q" (*__x) : "memory"); \ > + break; \ > + case 8: \ > + asm volatile(__LOAD_RCPC(, %0, %1) \ > + : "=r" (*(__u64 *)__u.__c) \ > + : "Q" (*__x) : "memory"); \ > + break; \ > + default: \ > + atomic = 0; \ > + } \ > + atomic ? (typeof(*__x))__u.__val : (*(volatile typeof(__x))__x);\ > +}) > + > +#endif /* !BUILD_VDSO */ > +#endif /* CONFIG_LTO */ > + > +#include <asm-generic/rwonce.h> > + > +#endif /* __ASM_RWONCE_H */ > diff --git a/arch/arm64/kernel/vdso/Makefile b/arch/arm64/kernel/vdso/Makefile > index 45d5cfe46429..60df97f2e7de 100644 > --- a/arch/arm64/kernel/vdso/Makefile > +++ b/arch/arm64/kernel/vdso/Makefile > @@ -28,7 +28,7 @@ ldflags-y := -shared -nostdlib -soname=linux-vdso.so.1 --hash-style=sysv \ > $(btildflags-y) -T > > ccflags-y := -fno-common -fno-builtin -fno-stack-protector -ffixed-x18 > -ccflags-y += -DDISABLE_BRANCH_PROFILING > +ccflags-y += -DDISABLE_BRANCH_PROFILING -DBUILD_VDSO > > CFLAGS_REMOVE_vgettimeofday.o = $(CC_FLAGS_FTRACE) -Os $(CC_FLAGS_SCS) $(GCC_PLUGINS_CFLAGS) > KBUILD_CFLAGS += $(DISABLE_LTO) > diff --git a/arch/arm64/kernel/vdso32/Makefile b/arch/arm64/kernel/vdso32/Makefile > index d88148bef6b0..4fdf3754a058 100644 > --- a/arch/arm64/kernel/vdso32/Makefile > +++ b/arch/arm64/kernel/vdso32/Makefile > @@ -43,7 +43,7 @@ cc32-as-instr = $(call try-run,\ > # As a result we set our own flags here. > > # KBUILD_CPPFLAGS and NOSTDINC_FLAGS from top-level Makefile > -VDSO_CPPFLAGS := -D__KERNEL__ -nostdinc -isystem $(shell $(CC_COMPAT) -print-file-name=include) > +VDSO_CPPFLAGS := -DBUILD_VDSO -D__KERNEL__ -nostdinc -isystem $(shell $(CC_COMPAT) -print-file-name=include) > VDSO_CPPFLAGS += $(LINUXINCLUDE) > > # Common C and assembly flags > -- > 2.27.0.383.g050319c2ae-goog
diff --git a/arch/arm64/include/asm/rwonce.h b/arch/arm64/include/asm/rwonce.h new file mode 100644 index 000000000000..d78eb4cb795b --- /dev/null +++ b/arch/arm64/include/asm/rwonce.h @@ -0,0 +1,63 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (C) 2020 Google LLC. + */ +#ifndef __ASM_RWONCE_H +#define __ASM_RWONCE_H + +#ifdef CONFIG_LTO + +#include <linux/compiler_types.h> +#include <asm/alternative-macros.h> + +#ifndef BUILD_VDSO + +#ifdef CONFIG_AS_HAS_LDAPR +#define __LOAD_RCPC(sfx, regs...) \ + ALTERNATIVE( \ + "ldar" #sfx "\t" #regs, \ + ".arch_extension rcpc\n" \ + "ldapr" #sfx "\t" #regs, \ + ARM64_HAS_LDAPR) +#else +#define __LOAD_RCPC(sfx, regs...) "ldar" #sfx "\t" #regs +#endif /* CONFIG_AS_HAS_LDAPR */ + +#define __READ_ONCE(x) \ +({ \ + typeof(&(x)) __x = &(x); \ + int atomic = 1; \ + union { __unqual_scalar_typeof(*__x) __val; char __c[1]; } __u; \ + switch (sizeof(x)) { \ + case 1: \ + asm volatile(__LOAD_RCPC(b, %w0, %1) \ + : "=r" (*(__u8 *)__u.__c) \ + : "Q" (*__x) : "memory"); \ + break; \ + case 2: \ + asm volatile(__LOAD_RCPC(h, %w0, %1) \ + : "=r" (*(__u16 *)__u.__c) \ + : "Q" (*__x) : "memory"); \ + break; \ + case 4: \ + asm volatile(__LOAD_RCPC(, %w0, %1) \ + : "=r" (*(__u32 *)__u.__c) \ + : "Q" (*__x) : "memory"); \ + break; \ + case 8: \ + asm volatile(__LOAD_RCPC(, %0, %1) \ + : "=r" (*(__u64 *)__u.__c) \ + : "Q" (*__x) : "memory"); \ + break; \ + default: \ + atomic = 0; \ + } \ + atomic ? (typeof(*__x))__u.__val : (*(volatile typeof(__x))__x);\ +}) + +#endif /* !BUILD_VDSO */ +#endif /* CONFIG_LTO */ + +#include <asm-generic/rwonce.h> + +#endif /* __ASM_RWONCE_H */ diff --git a/arch/arm64/kernel/vdso/Makefile b/arch/arm64/kernel/vdso/Makefile index 45d5cfe46429..60df97f2e7de 100644 --- a/arch/arm64/kernel/vdso/Makefile +++ b/arch/arm64/kernel/vdso/Makefile @@ -28,7 +28,7 @@ ldflags-y := -shared -nostdlib -soname=linux-vdso.so.1 --hash-style=sysv \ $(btildflags-y) -T ccflags-y := -fno-common -fno-builtin -fno-stack-protector -ffixed-x18 -ccflags-y += -DDISABLE_BRANCH_PROFILING +ccflags-y += -DDISABLE_BRANCH_PROFILING -DBUILD_VDSO CFLAGS_REMOVE_vgettimeofday.o = $(CC_FLAGS_FTRACE) -Os $(CC_FLAGS_SCS) $(GCC_PLUGINS_CFLAGS) KBUILD_CFLAGS += $(DISABLE_LTO) diff --git a/arch/arm64/kernel/vdso32/Makefile b/arch/arm64/kernel/vdso32/Makefile index d88148bef6b0..4fdf3754a058 100644 --- a/arch/arm64/kernel/vdso32/Makefile +++ b/arch/arm64/kernel/vdso32/Makefile @@ -43,7 +43,7 @@ cc32-as-instr = $(call try-run,\ # As a result we set our own flags here. # KBUILD_CPPFLAGS and NOSTDINC_FLAGS from top-level Makefile -VDSO_CPPFLAGS := -D__KERNEL__ -nostdinc -isystem $(shell $(CC_COMPAT) -print-file-name=include) +VDSO_CPPFLAGS := -DBUILD_VDSO -D__KERNEL__ -nostdinc -isystem $(shell $(CC_COMPAT) -print-file-name=include) VDSO_CPPFLAGS += $(LINUXINCLUDE) # Common C and assembly flags
When building with LTO, there is an increased risk of the compiler converting an address dependency headed by a READ_ONCE() invocation into a control dependency and consequently allowing for harmful reordering by the CPU. Ensure that such transformations are harmless by overriding the generic READ_ONCE() definition with one that provides acquire semantics when building with LTO. Signed-off-by: Will Deacon <will@kernel.org> --- arch/arm64/include/asm/rwonce.h | 63 +++++++++++++++++++++++++++++++ arch/arm64/kernel/vdso/Makefile | 2 +- arch/arm64/kernel/vdso32/Makefile | 2 +- 3 files changed, 65 insertions(+), 2 deletions(-) create mode 100644 arch/arm64/include/asm/rwonce.h