| Message ID | 20220613134008.3760481-4-ardb@kernel.org (mailing list archive) |
|---|---|
| State | New, archived |
| Headers | show |
| Series | arm64: dynamic shadow call stack support | expand |
On Mon, Jun 13, 2022 at 03:40:08PM +0200, Ard Biesheuvel wrote: > Implement dynamic shadow call stack support on Clang, by parsing the > unwind tables at init time to locate all occurrences of PACIASP/AUTIASP > instructions, and replacing them with the shadow call stack push and pop > instructions, respectively. > > This is useful because the overhead of the shadow call stack is > difficult to justify on hardware that implements pointer authentication > (PAC), and given that the PAC instructions are executed as NOPs on > hardware that doesn't, we can just replace them without breaking > anything. As PACIASP/AUTIASP are guaranteed to be paired with respect to > manipulations of the return address, replacing them 1:1 with shadow call > stack pushes and pops is guaranteed to result in the desired behavior. Specifically, the "PAC available" benefit is the per-thread memory savings (no shadow stack needs to be allocated). Thanks for getting this working! > Signed-off-by: Ard Biesheuvel <ardb@kernel.org> Sami, can you test this for the cases you've needed this for? In the meantime, Will, can you land this for -next so we can get maximal test time?
On Mon, 13 Jun 2022 at 18:30, Kees Cook <keescook@chromium.org> wrote: > > On Mon, Jun 13, 2022 at 03:40:08PM +0200, Ard Biesheuvel wrote: > > Implement dynamic shadow call stack support on Clang, by parsing the > > unwind tables at init time to locate all occurrences of PACIASP/AUTIASP > > instructions, and replacing them with the shadow call stack push and pop > > instructions, respectively. > > > > This is useful because the overhead of the shadow call stack is > > difficult to justify on hardware that implements pointer authentication > > (PAC), and given that the PAC instructions are executed as NOPs on > > hardware that doesn't, we can just replace them without breaking > > anything. As PACIASP/AUTIASP are guaranteed to be paired with respect to > > manipulations of the return address, replacing them 1:1 with shadow call > > stack pushes and pops is guaranteed to result in the desired behavior. > > Specifically, the "PAC available" benefit is the per-thread memory > savings (no shadow stack needs to be allocated). Thanks for getting this > working! > > > Signed-off-by: Ard Biesheuvel <ardb@kernel.org> > > Sami, can you test this for the cases you've needed this for? > > In the meantime, Will, can you land this for -next so we can get maximal > test time? > I should note that this relies on Clang 15 which has not been released yet. I have been using the clang-15 and lld-15 packages from deb http://apt.llvm.org/bullseye/ llvm-toolchain-bullseye main and setting LLVM=-15 on the make command line.
On Mon, Jun 13, 2022 at 03:40:08PM +0200, Ard Biesheuvel wrote: > Implement dynamic shadow call stack support on Clang, by parsing the > unwind tables at init time to locate all occurrences of PACIASP/AUTIASP > instructions, and replacing them with the shadow call stack push and pop > instructions, respectively. > > This is useful because the overhead of the shadow call stack is > difficult to justify on hardware that implements pointer authentication > (PAC), and given that the PAC instructions are executed as NOPs on > hardware that doesn't, we can just replace them without breaking > anything. As PACIASP/AUTIASP are guaranteed to be paired with respect to > manipulations of the return address, replacing them 1:1 with shadow call > stack pushes and pops is guaranteed to result in the desired behavior. > > Signed-off-by: Ard Biesheuvel <ardb@kernel.org> > --- > arch/arm64/Kconfig | 9 + > arch/arm64/Makefile | 10 +- > arch/arm64/include/asm/scs.h | 45 ++++ > arch/arm64/kernel/Makefile | 2 + > arch/arm64/kernel/head.S | 3 + > arch/arm64/kernel/irq.c | 2 +- > arch/arm64/kernel/module.c | 8 + > arch/arm64/kernel/patch-scs.c | 257 ++++++++++++++++++++ > arch/arm64/kernel/sdei.c | 2 +- > arch/arm64/kernel/setup.c | 4 + > 10 files changed, 338 insertions(+), 4 deletions(-) > > diff --git a/arch/arm64/Kconfig b/arch/arm64/Kconfig > index 5f92344edff5..9ff72e582522 100644 > --- a/arch/arm64/Kconfig > +++ b/arch/arm64/Kconfig > @@ -369,6 +369,15 @@ config KASAN_SHADOW_OFFSET > config UNWIND_TABLES > bool > > +config UNWIND_PATCH_PAC_INTO_SCS > + bool "Enable shadow call stack dynamically using code patching" > + # needs Clang with https://reviews.llvm.org/D111780 incorporated > + depends on CC_IS_CLANG && CLANG_VERSION >= 150000 > + depends on ARM64_PTR_AUTH_KERNEL && CC_HAS_BRANCH_PROT_PAC_RET > + depends on SHADOW_CALL_STACK > + select UNWIND_TABLES > + select DYNAMIC_SCS > + > source "arch/arm64/Kconfig.platforms" > > menu "Kernel Features" > diff --git a/arch/arm64/Makefile b/arch/arm64/Makefile > index 4fbca56fa602..e439ebbd167d 100644 > --- a/arch/arm64/Makefile > +++ b/arch/arm64/Makefile > @@ -77,10 +77,16 @@ branch-prot-flags-$(CONFIG_CC_HAS_SIGN_RETURN_ADDRESS) := -msign-return-address= > # We enable additional protection for leaf functions as there is some > # narrow potential for ROP protection benefits and no substantial > # performance impact has been observed. > +PACRET-y := pac-ret+leaf > + > +# Using a shadow call stack in leaf functions is too costly, so avoid PAC there > +# as well when we may be patching PAC into SCS > +PACRET-$(CONFIG_UNWIND_PATCH_PAC_INTO_SCS) := pac-ret > + > ifeq ($(CONFIG_ARM64_BTI_KERNEL),y) > -branch-prot-flags-$(CONFIG_CC_HAS_BRANCH_PROT_PAC_RET_BTI) := -mbranch-protection=pac-ret+leaf+bti > +branch-prot-flags-$(CONFIG_CC_HAS_BRANCH_PROT_PAC_RET_BTI) := -mbranch-protection=$(PACRET-y)+bti > else > -branch-prot-flags-$(CONFIG_CC_HAS_BRANCH_PROT_PAC_RET) := -mbranch-protection=pac-ret+leaf > +branch-prot-flags-$(CONFIG_CC_HAS_BRANCH_PROT_PAC_RET) := -mbranch-protection=$(PACRET-y) > endif > # -march=armv8.3-a enables the non-nops instructions for PAC, to avoid the > # compiler to generate them and consequently to break the single image contract > diff --git a/arch/arm64/include/asm/scs.h b/arch/arm64/include/asm/scs.h > index 8297bccf0784..51fcfc96ba71 100644 > --- a/arch/arm64/include/asm/scs.h > +++ b/arch/arm64/include/asm/scs.h > @@ -24,6 +24,51 @@ > .endm > #endif /* CONFIG_SHADOW_CALL_STACK */ > > + > +#else > + > +#include <linux/scs.h> > +#include <asm/cpufeature.h> > + > +#ifdef CONFIG_UNWIND_PATCH_PAC_INTO_SCS > +static inline bool should_patch_pac_into_scs(void) > +{ > + /* > + * We only enable the shadow call stack dynamically if we are running > + * on a system that does not implement PAC or BTI. PAC and SCS provide > + * roughly the same level of protection, and BTI relies on the PACIASP > + * instructions serving as landing pads, preventing us from patching > + * those instructions into something else. > + */ > + u64 reg = read_sysreg_s(SYS_ID_AA64ISAR1_EL1); > + > + if (reg & ((0xf << ID_AA64ISAR1_APA_SHIFT) | > + (0xf << ID_AA64ISAR1_API_SHIFT))) > + return false; > + > + reg = read_sysreg_s(SYS_ID_AA64ISAR2_EL1); > + if (reg & (0xf << ID_AA64ISAR2_APA3_SHIFT)) > + return false; > + > + if (IS_ENABLED(CONFIG_ARM64_BTI_KERNEL)) { > + reg = read_sysreg_s(SYS_ID_AA64PFR1_EL1); > + if (reg & (0xf << ID_AA64PFR1_BT_SHIFT)) > + return false; > + } > + return true; > +} > + > +static inline void dynamic_scs_init(void) > +{ > + if (should_patch_pac_into_scs()) > + static_branch_enable(&dynamic_scs_enabled); > +} Should we print out a message to indicate we are actually enabling SCS at runtime? Otherwise I think the only way to know would be to look at /proc/meminfo, for example. > +#else > +static inline void dynamic_scs_init(void) {} > +#endif > + > +int scs_patch(const u8 eh_frame[], int size); > + > #endif /* __ASSEMBLY __ */ > > #endif /* _ASM_SCS_H */ > diff --git a/arch/arm64/kernel/Makefile b/arch/arm64/kernel/Makefile > index fa7981d0d917..bd5ab51f86fb 100644 > --- a/arch/arm64/kernel/Makefile > +++ b/arch/arm64/kernel/Makefile > @@ -74,6 +74,8 @@ obj-$(CONFIG_ARM64_PTR_AUTH) += pointer_auth.o > obj-$(CONFIG_ARM64_MTE) += mte.o > obj-y += vdso-wrap.o > obj-$(CONFIG_COMPAT_VDSO) += vdso32-wrap.o > +obj-$(CONFIG_UNWIND_PATCH_PAC_INTO_SCS) += patch-scs.o > +CFLAGS_patch-scs.o += -mbranch-protection=none > > # Force dependency (vdso*-wrap.S includes vdso.so through incbin) > $(obj)/vdso-wrap.o: $(obj)/vdso/vdso.so > diff --git a/arch/arm64/kernel/head.S b/arch/arm64/kernel/head.S > index 6a98f1a38c29..e9601c8a1bcd 100644 > --- a/arch/arm64/kernel/head.S > +++ b/arch/arm64/kernel/head.S > @@ -453,6 +453,9 @@ SYM_FUNC_START_LOCAL(__primary_switched) > mov x0, x21 // pass FDT address in x0 > bl early_fdt_map // Try mapping the FDT early > bl init_feature_override // Parse cpu feature overrides > +#ifdef CONFIG_UNWIND_PATCH_PAC_INTO_SCS > + bl scs_patch_vmlinux > +#endif > #ifdef CONFIG_RANDOMIZE_BASE > tst x23, ~(MIN_KIMG_ALIGN - 1) // already running randomized? > b.ne 0f > diff --git a/arch/arm64/kernel/irq.c b/arch/arm64/kernel/irq.c > index bda49430c9ea..c284ec35c27c 100644 > --- a/arch/arm64/kernel/irq.c > +++ b/arch/arm64/kernel/irq.c > @@ -39,7 +39,7 @@ static void init_irq_scs(void) > { > int cpu; > > - if (!IS_ENABLED(CONFIG_SHADOW_CALL_STACK)) > + if (!scs_is_enabled()) > return; > > for_each_possible_cpu(cpu) I applied the series on top of -rc1 and tested it in qemu. With -cpu cortex-a57 everything seems to work and PAC instructions are correctly patched into SCS push/pop: Thread 1 hit Breakpoint 1, 0xffff8000098b7920 in scs_patch_vmlinux () (gdb) x/1i scs_alloc 0xffff8000081daeb8 <scs_alloc>: paciasp (gdb) finish Run till exit from #0 0xffff8000098b7920 in scs_patch_vmlinux () 0xffff8000098b03cc in __primary_switched () (gdb) x/1i scs_alloc 0xffff8000081daeb8 <scs_alloc>: str x30, [x18], #8 However, with -cpu max I'm still seeing calls to scs_alloc despite the following: # dmesg | grep "Address authentication" [ 0.000000] CPU features: detected: Address authentication (architected QARMA5 algorithm) # zcat /proc/config.gz | grep -E '(SHADOW_|UNWIND|DYNAMIC_SCS)' CONFIG_UNWIND_TABLES=y CONFIG_UNWIND_PATCH_PAC_INTO_SCS=y CONFIG_CC_HAVE_SHADOW_CALL_STACK=y CONFIG_ARCH_SUPPORTS_SHADOW_CALL_STACK=y CONFIG_SHADOW_CALL_STACK=y CONFIG_DYNAMIC_SCS=y It looks like we're correctly leaving the PAC instructions unpatched, but scs_is_enabled must be returning true: Thread 1 hit Breakpoint 1, 0xffff8000098b7920 in scs_patch_vmlinux () (gdb) x/1i scs_alloc 0xffff8000081daeb8 <scs_alloc>: paciasp (gdb) finish Run till exit from #0 0xffff8000098b7920 in scs_patch_vmlinux () 0xffff8000098b03cc in __primary_switched () (gdb) x/1i scs_alloc 0xffff8000081daeb8 <scs_alloc>: paciasp (gdb) c Continuing. Thread 1 hit Breakpoint 2, 0xffff8000081daeb8 in scs_alloc () (gdb) bt #0 0xffff8000081daeb8 in scs_alloc () #1 0xffff800008015b60 in init_irq_scs () #2 0xffff8000098b3c5c in init_IRQ () #3 0xffff8000098b0954 in start_kernel () #4 0xffff8000098b03f4 in __primary_switched () I'm guessing this is also why I'm getting the following panic after attempting to load a module: [ 25.549517] Unhandled 64-bit el1h sync exception on CPU0, ESR 0x0000000034000003 -- BTI [ 25.549893] CPU: 0 PID: 156 Comm: modprobe Not tainted 5.19.0-rc1-00246-gef5e30ca27c0-dirty #1 [ 25.550139] Hardware name: linux,dummy-virt (DT) [ 25.550426] pstate: 80400c09 (Nzcv daif +PAN -UAO -TCO -DIT -SSBS BTYPE=j-) [ 25.550591] pc : scs_handle_fde_frame+0x80/0x17c [ 25.551341] lr : scs_handle_fde_frame+0x54/0x17c [ 25.551424] sp : ffff80000a67bb90 [ 25.551476] x29: ffff80000a67bb90 x28: 0000000000000000 x27: ffff800000ff6025 [ 25.552440] x26: 0000000000000003 x25: 0000000000000002 x24: ffff800000ff6026 [ 25.552584] x23: ffff800008ffd000 x22: 00000000d50323bf x21: 00000000d503233f [ 25.552725] x20: ffff800000ff0048 x19: ffff800000ff6014 x18: ffff80000a5d1000 [ 25.552867] x17: 0000000000000000 x16: ffff00001fe9cd88 x15: 24003032336f7470 [ 25.553020] x14: 180f0a0700000000 x13: 0000000000000040 x12: 0000000000000005 [ 25.553161] x11: 6c6300656c75646f x10: 0000000000000010 x9 : ffffffffffffffe4 [ 25.553354] x8 : ffff800008fd1870 x7 : 7f7f7f7f7f7f7f7f x6 : 16fefeff0eff1e0b [ 25.553511] x5 : 0080808000800000 x4 : 0000000000000010 x3 : 1800000010001f0c [ 25.553645] x2 : 1b011e7c0100527a x1 : 0000000000000000 x0 : ffff800000ff0048 [ 25.553980] Kernel panic - not syncing: Unhandled exception [ 25.554134] CPU: 0 PID: 156 Comm: modprobe Not tainted 5.19.0-rc1-00246-gef5e30ca27c0-dirty #1 [ 25.554254] Hardware name: linux,dummy-virt (DT) [ 25.554394] Call trace: [ 25.554512] dump_backtrace+0xe4/0x104 [ 25.554773] show_stack+0x18/0x24 [ 25.554893] dump_stack_lvl+0x64/0x7c [ 25.554983] dump_stack+0x18/0x38 [ 25.555067] panic+0x14c/0x370 [ 25.555149] el1t_64_irq_handler+0x0/0x1c [ 25.555231] el1_abort+0x0/0x5c [ 25.555314] el1h_64_sync+0x64/0x68 [ 25.555395] scs_handle_fde_frame+0x80/0x17c [ 25.555482] scs_patch+0x7c/0xa0 [ 25.555565] module_finalize+0xe0/0x100 [ 25.555647] post_relocation+0xd4/0xf0 [ 25.555730] load_module+0x924/0x11fc [ 25.555811] __arm64_sys_finit_module+0xbc/0x108 [ 25.555895] invoke_syscall+0x40/0x118 [ 25.555984] el0_svc_common+0xb4/0xf0 [ 25.556067] do_el0_svc+0x2c/0xb4 [ 25.556150] el0_svc+0x2c/0x7c [ 25.556233] el0t_64_sync_handler+0x84/0xf0 [ 25.556315] el0t_64_sync+0x190/0x194 This is with defconfig + SHADOW_CALL_STACK + UNWIND_PATCH_PAC_INTO_SCS. Any thoughts if I'm doing something wrong here? Sami
On Wed, 15 Jun 2022 at 23:33, Sami Tolvanen <samitolvanen@google.com> wrote: > > On Mon, Jun 13, 2022 at 03:40:08PM +0200, Ard Biesheuvel wrote: > > Implement dynamic shadow call stack support on Clang, by parsing the > > unwind tables at init time to locate all occurrences of PACIASP/AUTIASP > > instructions, and replacing them with the shadow call stack push and pop > > instructions, respectively. > > > > This is useful because the overhead of the shadow call stack is > > difficult to justify on hardware that implements pointer authentication > > (PAC), and given that the PAC instructions are executed as NOPs on > > hardware that doesn't, we can just replace them without breaking > > anything. As PACIASP/AUTIASP are guaranteed to be paired with respect to > > manipulations of the return address, replacing them 1:1 with shadow call > > stack pushes and pops is guaranteed to result in the desired behavior. > > > > Signed-off-by: Ard Biesheuvel <ardb@kernel.org> > > --- > > arch/arm64/Kconfig | 9 + > > arch/arm64/Makefile | 10 +- > > arch/arm64/include/asm/scs.h | 45 ++++ > > arch/arm64/kernel/Makefile | 2 + > > arch/arm64/kernel/head.S | 3 + > > arch/arm64/kernel/irq.c | 2 +- > > arch/arm64/kernel/module.c | 8 + > > arch/arm64/kernel/patch-scs.c | 257 ++++++++++++++++++++ > > arch/arm64/kernel/sdei.c | 2 +- > > arch/arm64/kernel/setup.c | 4 + > > 10 files changed, 338 insertions(+), 4 deletions(-) > > ... > > --- a/arch/arm64/include/asm/scs.h > > +++ b/arch/arm64/include/asm/scs.h ... > > + > > +static inline void dynamic_scs_init(void) > > +{ > > + if (should_patch_pac_into_scs()) > > + static_branch_enable(&dynamic_scs_enabled); > > +} > > Should we print out a message to indicate we are actually enabling SCS > at runtime? Otherwise I think the only way to know would be to look at > /proc/meminfo, for example. > Yes, good point. I will add something here. ... > > I applied the series on top of -rc1 and tested it in qemu. With -cpu > cortex-a57 everything seems to work and PAC instructions are correctly > patched into SCS push/pop: > ... > > However, with -cpu max I'm still seeing calls to scs_alloc despite the > following: > > # dmesg | grep "Address authentication" > [ 0.000000] CPU features: detected: Address authentication (architected QARMA5 algorithm) ... > I'm guessing this is also why I'm getting the following panic after > attempting to load a module: > > [ 25.549517] Unhandled 64-bit el1h sync exception on CPU0, ESR 0x0000000034000003 -- BTI ... > > This is with defconfig + SHADOW_CALL_STACK + UNWIND_PATCH_PAC_INTO_SCS. > Any thoughts if I'm doing something wrong here? > No, I made a mistake with the initial value of the dynamic_scs_enabled static key, so it's currently always true. I'll fix this in the next revision.
diff --git a/arch/arm64/Kconfig b/arch/arm64/Kconfig index 5f92344edff5..9ff72e582522 100644 --- a/arch/arm64/Kconfig +++ b/arch/arm64/Kconfig @@ -369,6 +369,15 @@ config KASAN_SHADOW_OFFSET config UNWIND_TABLES bool +config UNWIND_PATCH_PAC_INTO_SCS + bool "Enable shadow call stack dynamically using code patching" + # needs Clang with https://reviews.llvm.org/D111780 incorporated + depends on CC_IS_CLANG && CLANG_VERSION >= 150000 + depends on ARM64_PTR_AUTH_KERNEL && CC_HAS_BRANCH_PROT_PAC_RET + depends on SHADOW_CALL_STACK + select UNWIND_TABLES + select DYNAMIC_SCS + source "arch/arm64/Kconfig.platforms" menu "Kernel Features" diff --git a/arch/arm64/Makefile b/arch/arm64/Makefile index 4fbca56fa602..e439ebbd167d 100644 --- a/arch/arm64/Makefile +++ b/arch/arm64/Makefile @@ -77,10 +77,16 @@ branch-prot-flags-$(CONFIG_CC_HAS_SIGN_RETURN_ADDRESS) := -msign-return-address= # We enable additional protection for leaf functions as there is some # narrow potential for ROP protection benefits and no substantial # performance impact has been observed. +PACRET-y := pac-ret+leaf + +# Using a shadow call stack in leaf functions is too costly, so avoid PAC there +# as well when we may be patching PAC into SCS +PACRET-$(CONFIG_UNWIND_PATCH_PAC_INTO_SCS) := pac-ret + ifeq ($(CONFIG_ARM64_BTI_KERNEL),y) -branch-prot-flags-$(CONFIG_CC_HAS_BRANCH_PROT_PAC_RET_BTI) := -mbranch-protection=pac-ret+leaf+bti +branch-prot-flags-$(CONFIG_CC_HAS_BRANCH_PROT_PAC_RET_BTI) := -mbranch-protection=$(PACRET-y)+bti else -branch-prot-flags-$(CONFIG_CC_HAS_BRANCH_PROT_PAC_RET) := -mbranch-protection=pac-ret+leaf +branch-prot-flags-$(CONFIG_CC_HAS_BRANCH_PROT_PAC_RET) := -mbranch-protection=$(PACRET-y) endif # -march=armv8.3-a enables the non-nops instructions for PAC, to avoid the # compiler to generate them and consequently to break the single image contract diff --git a/arch/arm64/include/asm/scs.h b/arch/arm64/include/asm/scs.h index 8297bccf0784..51fcfc96ba71 100644 --- a/arch/arm64/include/asm/scs.h +++ b/arch/arm64/include/asm/scs.h @@ -24,6 +24,51 @@ .endm #endif /* CONFIG_SHADOW_CALL_STACK */ + +#else + +#include <linux/scs.h> +#include <asm/cpufeature.h> + +#ifdef CONFIG_UNWIND_PATCH_PAC_INTO_SCS +static inline bool should_patch_pac_into_scs(void) +{ + /* + * We only enable the shadow call stack dynamically if we are running + * on a system that does not implement PAC or BTI. PAC and SCS provide + * roughly the same level of protection, and BTI relies on the PACIASP + * instructions serving as landing pads, preventing us from patching + * those instructions into something else. + */ + u64 reg = read_sysreg_s(SYS_ID_AA64ISAR1_EL1); + + if (reg & ((0xf << ID_AA64ISAR1_APA_SHIFT) | + (0xf << ID_AA64ISAR1_API_SHIFT))) + return false; + + reg = read_sysreg_s(SYS_ID_AA64ISAR2_EL1); + if (reg & (0xf << ID_AA64ISAR2_APA3_SHIFT)) + return false; + + if (IS_ENABLED(CONFIG_ARM64_BTI_KERNEL)) { + reg = read_sysreg_s(SYS_ID_AA64PFR1_EL1); + if (reg & (0xf << ID_AA64PFR1_BT_SHIFT)) + return false; + } + return true; +} + +static inline void dynamic_scs_init(void) +{ + if (should_patch_pac_into_scs()) + static_branch_enable(&dynamic_scs_enabled); +} +#else +static inline void dynamic_scs_init(void) {} +#endif + +int scs_patch(const u8 eh_frame[], int size); + #endif /* __ASSEMBLY __ */ #endif /* _ASM_SCS_H */ diff --git a/arch/arm64/kernel/Makefile b/arch/arm64/kernel/Makefile index fa7981d0d917..bd5ab51f86fb 100644 --- a/arch/arm64/kernel/Makefile +++ b/arch/arm64/kernel/Makefile @@ -74,6 +74,8 @@ obj-$(CONFIG_ARM64_PTR_AUTH) += pointer_auth.o obj-$(CONFIG_ARM64_MTE) += mte.o obj-y += vdso-wrap.o obj-$(CONFIG_COMPAT_VDSO) += vdso32-wrap.o +obj-$(CONFIG_UNWIND_PATCH_PAC_INTO_SCS) += patch-scs.o +CFLAGS_patch-scs.o += -mbranch-protection=none # Force dependency (vdso*-wrap.S includes vdso.so through incbin) $(obj)/vdso-wrap.o: $(obj)/vdso/vdso.so diff --git a/arch/arm64/kernel/head.S b/arch/arm64/kernel/head.S index 6a98f1a38c29..e9601c8a1bcd 100644 --- a/arch/arm64/kernel/head.S +++ b/arch/arm64/kernel/head.S @@ -453,6 +453,9 @@ SYM_FUNC_START_LOCAL(__primary_switched) mov x0, x21 // pass FDT address in x0 bl early_fdt_map // Try mapping the FDT early bl init_feature_override // Parse cpu feature overrides +#ifdef CONFIG_UNWIND_PATCH_PAC_INTO_SCS + bl scs_patch_vmlinux +#endif #ifdef CONFIG_RANDOMIZE_BASE tst x23, ~(MIN_KIMG_ALIGN - 1) // already running randomized? b.ne 0f diff --git a/arch/arm64/kernel/irq.c b/arch/arm64/kernel/irq.c index bda49430c9ea..c284ec35c27c 100644 --- a/arch/arm64/kernel/irq.c +++ b/arch/arm64/kernel/irq.c @@ -39,7 +39,7 @@ static void init_irq_scs(void) { int cpu; - if (!IS_ENABLED(CONFIG_SHADOW_CALL_STACK)) + if (!scs_is_enabled()) return; for_each_possible_cpu(cpu) diff --git a/arch/arm64/kernel/module.c b/arch/arm64/kernel/module.c index f2d4bb14bfab..111dc6414e6d 100644 --- a/arch/arm64/kernel/module.c +++ b/arch/arm64/kernel/module.c @@ -15,9 +15,11 @@ #include <linux/kernel.h> #include <linux/mm.h> #include <linux/moduleloader.h> +#include <linux/scs.h> #include <linux/vmalloc.h> #include <asm/alternative.h> #include <asm/insn.h> +#include <asm/scs.h> #include <asm/sections.h> void *module_alloc(unsigned long size) @@ -529,5 +531,11 @@ int module_finalize(const Elf_Ehdr *hdr, if (s) apply_alternatives_module((void *)s->sh_addr, s->sh_size); + if (scs_is_dynamic()) { + s = find_section(hdr, sechdrs, ".init.eh_frame"); + if (s) + scs_patch((void *)s->sh_addr, s->sh_size); + } + return module_init_ftrace_plt(hdr, sechdrs, me); } diff --git a/arch/arm64/kernel/patch-scs.c b/arch/arm64/kernel/patch-scs.c new file mode 100644 index 000000000000..1b3da02d5b74 --- /dev/null +++ b/arch/arm64/kernel/patch-scs.c @@ -0,0 +1,257 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (C) 2022 - Google LLC + * Author: Ard Biesheuvel <ardb@google.com> + */ + +#include <linux/bug.h> +#include <linux/errno.h> +#include <linux/init.h> +#include <linux/linkage.h> +#include <linux/printk.h> +#include <linux/types.h> + +#include <asm/cacheflush.h> +#include <asm/scs.h> + +// +// This minimal DWARF CFI parser is partially based on the code in +// arch/arc/kernel/unwind.c, and on the document below: +// https://refspecs.linuxbase.org/LSB_4.0.0/LSB-Core-generic/LSB-Core-generic/ehframechpt.html +// + +#define DW_CFA_nop 0x00 +#define DW_CFA_set_loc 0x01 +#define DW_CFA_advance_loc1 0x02 +#define DW_CFA_advance_loc2 0x03 +#define DW_CFA_advance_loc4 0x04 +#define DW_CFA_offset_extended 0x05 +#define DW_CFA_restore_extended 0x06 +#define DW_CFA_undefined 0x07 +#define DW_CFA_same_value 0x08 +#define DW_CFA_register 0x09 +#define DW_CFA_remember_state 0x0a +#define DW_CFA_restore_state 0x0b +#define DW_CFA_def_cfa 0x0c +#define DW_CFA_def_cfa_register 0x0d +#define DW_CFA_def_cfa_offset 0x0e +#define DW_CFA_def_cfa_expression 0x0f +#define DW_CFA_expression 0x10 +#define DW_CFA_offset_extended_sf 0x11 +#define DW_CFA_def_cfa_sf 0x12 +#define DW_CFA_def_cfa_offset_sf 0x13 +#define DW_CFA_val_offset 0x14 +#define DW_CFA_val_offset_sf 0x15 +#define DW_CFA_val_expression 0x16 +#define DW_CFA_lo_user 0x1c +#define DW_CFA_negate_ra_state 0x2d +#define DW_CFA_GNU_args_size 0x2e +#define DW_CFA_GNU_negative_offset_extended 0x2f +#define DW_CFA_hi_user 0x3f + +extern const u8 __eh_frame_start[], __eh_frame_end[]; + +enum { + PACIASP = 0xd503233f, + AUTIASP = 0xd50323bf, + SCS_PUSH = 0xf800865e, + SCS_POP = 0xf85f8e5e, +}; + +static void __always_inline scs_patch_loc(u64 loc) +{ + u32 insn = le32_to_cpup((void *)loc); + + switch (insn) { + case PACIASP: + *(u32 *)loc = cpu_to_le32(SCS_PUSH); + break; + case AUTIASP: + *(u32 *)loc = cpu_to_le32(SCS_POP); + break; + default: + /* + * While the DW_CFA_negate_ra_state directive is guaranteed to + * appear right after a PACIASP/AUTIASP instruction, it may + * also appear after a DW_CFA_restore_state directive that + * restores a state that is only partially accurate, and is + * followed by DW_CFA_negate_ra_state directive to toggle the + * PAC bit again. So we permit other instructions here, and ignore + * them. + */ + return; + } + dcache_clean_pou(loc, loc + sizeof(u32)); +} + +/* + * Skip one uleb128/sleb128 encoded quantity from the opcode stream. All bytes + * except the last one have bit #7 set. + */ +static int __always_inline skip_xleb128(const u8 **opcode, int size) +{ + u8 c; + + do { + c = *(*opcode)++; + size--; + } while (c & BIT(7)); + + return size; +} + +struct eh_frame { + /* + * The size of this frame if 0 < size < U32_MAX, 0 terminates the list. + */ + u32 size; + + /* + * The first frame is a Common Information Entry (CIE) frame, followed + * by one or more Frame Description Entry (FDE) frames. In the former + * case, this field is 0, otherwise it is the negated offset relative + * to the associated CIE frame. + */ + u32 cie_id_or_pointer; + + union { + struct { // CIE + u8 version; + u8 augmentation_string[]; + }; + + struct { // FDE + s32 initial_loc; + s32 range; + u8 opcodes[]; + }; + }; +}; + +static int noinstr scs_handle_fde_frame(const struct eh_frame *frame, + bool fde_has_augmentation_data, + int code_alignment_factor) +{ + int size = frame->size - offsetof(struct eh_frame, opcodes) + 4; + u64 loc = (u64)offset_to_ptr(&frame->initial_loc); + const u8 *opcode = frame->opcodes; + + if (fde_has_augmentation_data) { + int l; + + // assume single byte uleb128_t + if (WARN_ON(*opcode & BIT(7))) + return -ENOEXEC; + + l = *opcode++; + opcode += l; + size -= l + 1; + } + + /* + * Starting from 'loc', apply the CFA opcodes that advance the location + * pointer, and identify the locations of the PAC instructions. + */ + while (size-- > 0) { + switch (*opcode++) { + case DW_CFA_nop: + case DW_CFA_remember_state: + case DW_CFA_restore_state: + break; + + case DW_CFA_advance_loc1: + loc += *opcode++ * code_alignment_factor; + size--; + break; + + case DW_CFA_advance_loc2: + loc += *opcode++ * code_alignment_factor; + loc += (*opcode++ << 8) * code_alignment_factor; + size -= 2; + break; + + case DW_CFA_def_cfa: + case DW_CFA_offset_extended: + size = skip_xleb128(&opcode, size); + fallthrough; + case DW_CFA_def_cfa_offset: + case DW_CFA_def_cfa_offset_sf: + case DW_CFA_def_cfa_register: + case DW_CFA_same_value: + case DW_CFA_restore_extended: + case 0x80 ... 0xbf: + size = skip_xleb128(&opcode, size); + break; + + case DW_CFA_negate_ra_state: + scs_patch_loc(loc - 4); + break; + + case 0x40 ... 0x7f: + // advance loc + loc += (opcode[-1] & 0x3f) * code_alignment_factor; + break; + + case 0xc0 ... 0xff: + break; + + default: + pr_err("unhandled opcode: %02x in FDE frame %lx\n", opcode[-1], (uintptr_t)frame); + return -ENOEXEC; + } + } + return 0; +} + +int noinstr scs_patch(const u8 eh_frame[], int size) +{ + const u8 *p = eh_frame; + + while (size > 4) { + const struct eh_frame *frame = (const void *)p; + bool fde_has_augmentation_data = true; + int code_alignment_factor = 1; + int ret; + + if (frame->size == 0 || + frame->size == U32_MAX || + frame->size > size) + break; + + if (frame->cie_id_or_pointer == 0) { + const u8 *p = frame->augmentation_string; + + /* a 'z' in the augmentation string must come first */ + fde_has_augmentation_data = *p == 'z'; + + /* + * The code alignment factor is a uleb128 encoded field + * but given that the only sensible values are 1 or 4, + * there is no point in decoding the whole thing. + */ + p += strlen(p) + 1; + if (!WARN_ON(*p & BIT(7))) + code_alignment_factor = *p; + } else { + ret = scs_handle_fde_frame(frame, + fde_has_augmentation_data, + code_alignment_factor); + if (ret) + return ret; + } + + p += sizeof(frame->size) + frame->size; + size -= sizeof(frame->size) + frame->size; + } + return 0; +} + +asmlinkage void __init scs_patch_vmlinux(void) +{ + if (!should_patch_pac_into_scs()) + return; + + WARN_ON(scs_patch(__eh_frame_start, __eh_frame_end - __eh_frame_start)); + icache_inval_all_pou(); + isb(); +} diff --git a/arch/arm64/kernel/sdei.c b/arch/arm64/kernel/sdei.c index d20620a1c51a..30f3c7563694 100644 --- a/arch/arm64/kernel/sdei.c +++ b/arch/arm64/kernel/sdei.c @@ -144,7 +144,7 @@ static int init_sdei_scs(void) int cpu; int err = 0; - if (!IS_ENABLED(CONFIG_SHADOW_CALL_STACK)) + if (!scs_is_enabled()) return 0; for_each_possible_cpu(cpu) { diff --git a/arch/arm64/kernel/setup.c b/arch/arm64/kernel/setup.c index cf3a759f10d4..1b4f84563006 100644 --- a/arch/arm64/kernel/setup.c +++ b/arch/arm64/kernel/setup.c @@ -30,6 +30,7 @@ #include <linux/efi.h> #include <linux/psci.h> #include <linux/sched/task.h> +#include <linux/scs.h> #include <linux/mm.h> #include <asm/acpi.h> @@ -42,6 +43,7 @@ #include <asm/cpu_ops.h> #include <asm/kasan.h> #include <asm/numa.h> +#include <asm/scs.h> #include <asm/sections.h> #include <asm/setup.h> #include <asm/smp_plat.h> @@ -313,6 +315,8 @@ void __init __no_sanitize_address setup_arch(char **cmdline_p) parse_early_param(); + dynamic_scs_init(); + /* * Unmask asynchronous aborts and fiq after bringing up possible * earlycon. (Report possible System Errors once we can report this
Implement dynamic shadow call stack support on Clang, by parsing the unwind tables at init time to locate all occurrences of PACIASP/AUTIASP instructions, and replacing them with the shadow call stack push and pop instructions, respectively. This is useful because the overhead of the shadow call stack is difficult to justify on hardware that implements pointer authentication (PAC), and given that the PAC instructions are executed as NOPs on hardware that doesn't, we can just replace them without breaking anything. As PACIASP/AUTIASP are guaranteed to be paired with respect to manipulations of the return address, replacing them 1:1 with shadow call stack pushes and pops is guaranteed to result in the desired behavior. Signed-off-by: Ard Biesheuvel <ardb@kernel.org> --- arch/arm64/Kconfig | 9 + arch/arm64/Makefile | 10 +- arch/arm64/include/asm/scs.h | 45 ++++ arch/arm64/kernel/Makefile | 2 + arch/arm64/kernel/head.S | 3 + arch/arm64/kernel/irq.c | 2 +- arch/arm64/kernel/module.c | 8 + arch/arm64/kernel/patch-scs.c | 257 ++++++++++++++++++++ arch/arm64/kernel/sdei.c | 2 +- arch/arm64/kernel/setup.c | 4 + 10 files changed, 338 insertions(+), 4 deletions(-)