| Message ID | 20191003171642.135652-9-james.morse@arm.com (mailing list archive) |
|---|---|
| State | New, archived |
| Headers | show |
| Series | arm64: Convert entry.S synchronous exception handling to C | expand |
On Thu, Oct 03, 2019 at 06:16:42PM +0100, James Morse wrote: > The previous patches mechanically transformed the assembly version of > entry.S to entry-common.c for synchronous exceptions. > > The C version of local_daif_restore() doesn't quite do the same thing > as the assembly versions if pseudo-NMI is in use. In particular, > | local_daif_restore(DAIF_PROCCTX_NOIRQ) > will still allow pNMI to be delivered. This is not the behaviour > do_el0_ia_bp_hardening() and do_sp_pc_abort() want as it should not > be possible for the PMU handler to run as an NMI until the bp-hardening > sequence has run. > > The bp-hardening calls were placed where they are because this was the > first C code to run after the relevant exceptions. As we've now moved > that point earlier, move the checks and calls earlier too. > > This makes it clearer that this stuff runs before any kind of exception, > and saves modifying PSTATE twice. > > Signed-off-by: James Morse <james.morse@arm.com> > Cc: Julien Thierry <julien.thierry.kdev@gmail.com> > --- > arch/arm64/include/asm/processor.h | 7 +++++++ > arch/arm64/kernel/entry-common.c | 18 +++++++++++++++--- > arch/arm64/mm/fault.c | 29 +---------------------------- > 3 files changed, 23 insertions(+), 31 deletions(-) > > diff --git a/arch/arm64/include/asm/processor.h b/arch/arm64/include/asm/processor.h > index 5623685c7d13..c0c28c4589a8 100644 > --- a/arch/arm64/include/asm/processor.h > +++ b/arch/arm64/include/asm/processor.h > @@ -24,6 +24,7 @@ > #include <linux/build_bug.h> > #include <linux/cache.h> > #include <linux/init.h> > +#include <linux/thread_info.h> > #include <linux/stddef.h> > #include <linux/string.h> Nit: alphabetical order please! > > @@ -214,6 +215,12 @@ static inline void start_thread(struct pt_regs *regs, unsigned long pc, > regs->sp = sp; > } > > +static inline bool is_ttbr0_addr(unsigned long addr) > +{ > + /* entry assembly clears tags for TTBR0 addrs */ > + return addr < TASK_SIZE; > +} Could we move is_ttbr1_addr() here too? I guess there might be include ordering issues, but if not it would be nice if they lived in the same place. > + > #ifdef CONFIG_COMPAT > static inline void compat_start_thread(struct pt_regs *regs, unsigned long pc, > unsigned long sp) > diff --git a/arch/arm64/kernel/entry-common.c b/arch/arm64/kernel/entry-common.c > index 176969e55677..eb73d250a081 100644 > --- a/arch/arm64/kernel/entry-common.c > +++ b/arch/arm64/kernel/entry-common.c > @@ -14,6 +14,7 @@ > #include <asm/esr.h> > #include <asm/exception.h> > #include <asm/kprobes.h> > +#include <asm/mmu.h> > #include <asm/sysreg.h> > > static void notrace el1_abort(struct pt_regs *regs, unsigned long esr) > @@ -112,9 +113,17 @@ static void notrace el0_ia(struct pt_regs *regs, unsigned long esr) > { > unsigned long far = read_sysreg(far_el1); > > + /* > + * We've taken an instruction abort from userspace and not yet > + * re-enabled IRQs. If the address is a kernel address, apply > + * BP hardening prior to enabling IRQs and pre-emption. > + */ > + if (!is_ttbr0_addr(far)) > + arm64_apply_bp_hardening(); > + > user_exit_irqoff(); > - local_daif_restore(DAIF_PROCCTX_NOIRQ); > - do_el0_ia_bp_hardening(far, esr, regs); > + local_daif_restore(DAIF_PROCCTX); > + do_mem_abort(far, esr, regs); > } > NOKPROBE_SYMBOL(el0_ia); > > @@ -154,8 +163,11 @@ static void notrace el0_pc(struct pt_regs *regs, unsigned long esr) > { > unsigned long far = read_sysreg(far_el1); > > + if (!is_ttbr0_addr(instruction_pointer(regs))) > + arm64_apply_bp_hardening(); > + > user_exit_irqoff(); > - local_daif_restore(DAIF_PROCCTX_NOIRQ); > + local_daif_restore(DAIF_PROCCTX); > do_sp_pc_abort(far, esr, regs); > } > NOKPROBE_SYMBOL(el0_pc); This is much nicer, and AFAICT is correct, so: Reviewed-by: Mark Rutland <mark.rutland@arm.com> Mark. > diff --git a/arch/arm64/mm/fault.c b/arch/arm64/mm/fault.c > index 0857c2fc38b9..88e4bd4bc103 100644 > --- a/arch/arm64/mm/fault.c > +++ b/arch/arm64/mm/fault.c > @@ -34,6 +34,7 @@ > #include <asm/esr.h> > #include <asm/kasan.h> > #include <asm/kprobes.h> > +#include <asm/processor.h> > #include <asm/sysreg.h> > #include <asm/system_misc.h> > #include <asm/pgtable.h> > @@ -102,12 +103,6 @@ static void mem_abort_decode(unsigned int esr) > data_abort_decode(esr); > } > > -static inline bool is_ttbr0_addr(unsigned long addr) > -{ > - /* entry assembly clears tags for TTBR0 addrs */ > - return addr < TASK_SIZE; > -} > - > static inline bool is_ttbr1_addr(unsigned long addr) > { > /* TTBR1 addresses may have a tag if KASAN_SW_TAGS is in use */ > @@ -749,30 +744,8 @@ void do_el0_irq_bp_hardening(void) > } > NOKPROBE_SYMBOL(do_el0_irq_bp_hardening); > > -void do_el0_ia_bp_hardening(unsigned long addr, unsigned int esr, > - struct pt_regs *regs) > -{ > - /* > - * We've taken an instruction abort from userspace and not yet > - * re-enabled IRQs. If the address is a kernel address, apply > - * BP hardening prior to enabling IRQs and pre-emption. > - */ > - if (!is_ttbr0_addr(addr)) > - arm64_apply_bp_hardening(); > - > - local_daif_restore(DAIF_PROCCTX); > - do_mem_abort(addr, esr, regs); > -} > -NOKPROBE_SYMBOL(do_el0_ia_bp_hardening); > - > void do_sp_pc_abort(unsigned long addr, unsigned int esr, struct pt_regs *regs) > { > - if (user_mode(regs)) { > - if (!is_ttbr0_addr(instruction_pointer(regs))) > - arm64_apply_bp_hardening(); > - local_daif_restore(DAIF_PROCCTX); > - } > - > arm64_notify_die("SP/PC alignment exception", regs, > SIGBUS, BUS_ADRALN, (void __user *)addr, esr); > } > -- > 2.20.1 >
Hi Mark, On 04/10/2019 14:31, Mark Rutland wrote: > On Thu, Oct 03, 2019 at 06:16:42PM +0100, James Morse wrote: >> The previous patches mechanically transformed the assembly version of >> entry.S to entry-common.c for synchronous exceptions. >> >> The C version of local_daif_restore() doesn't quite do the same thing >> as the assembly versions if pseudo-NMI is in use. In particular, >> | local_daif_restore(DAIF_PROCCTX_NOIRQ) >> will still allow pNMI to be delivered. This is not the behaviour >> do_el0_ia_bp_hardening() and do_sp_pc_abort() want as it should not >> be possible for the PMU handler to run as an NMI until the bp-hardening >> sequence has run. >> >> The bp-hardening calls were placed where they are because this was the >> first C code to run after the relevant exceptions. As we've now moved >> that point earlier, move the checks and calls earlier too. >> >> This makes it clearer that this stuff runs before any kind of exception, >> and saves modifying PSTATE twice. >> diff --git a/arch/arm64/include/asm/processor.h b/arch/arm64/include/asm/processor.h >> index 5623685c7d13..c0c28c4589a8 100644 >> --- a/arch/arm64/include/asm/processor.h >> +++ b/arch/arm64/include/asm/processor.h >> @@ -24,6 +24,7 @@ >> #include <linux/build_bug.h> >> #include <linux/cache.h> >> #include <linux/init.h> >> +#include <linux/thread_info.h> >> #include <linux/stddef.h> >> #include <linux/string.h> > > Nit: alphabetical order please! Huh, I should really learn the alphabet... >> @@ -214,6 +215,12 @@ static inline void start_thread(struct pt_regs *regs, unsigned long pc, >> regs->sp = sp; >> } >> >> +static inline bool is_ttbr0_addr(unsigned long addr) >> +{ >> + /* entry assembly clears tags for TTBR0 addrs */ >> + return addr < TASK_SIZE; >> +} > > Could we move is_ttbr1_addr() here too? > > I guess there might be include ordering issues, but if not it would be > nice if they lived in the same place. That works, I agree keeping them together makes sense. (I didn't spot there were two) >> diff --git a/arch/arm64/kernel/entry-common.c b/arch/arm64/kernel/entry-common.c >> index 176969e55677..eb73d250a081 100644 >> --- a/arch/arm64/kernel/entry-common.c >> +++ b/arch/arm64/kernel/entry-common.c >> @@ -112,9 +113,17 @@ static void notrace el0_ia(struct pt_regs *regs, unsigned long esr) >> { >> unsigned long far = read_sysreg(far_el1); >> >> + /* >> + * We've taken an instruction abort from userspace and not yet >> + * re-enabled IRQs. If the address is a kernel address, apply >> + * BP hardening prior to enabling IRQs and pre-emption. >> + */ >> + if (!is_ttbr0_addr(far)) >> + arm64_apply_bp_hardening(); >> + >> user_exit_irqoff(); >> - local_daif_restore(DAIF_PROCCTX_NOIRQ); >> - do_el0_ia_bp_hardening(far, esr, regs); >> + local_daif_restore(DAIF_PROCCTX); >> + do_mem_abort(far, esr, regs); >> } >> NOKPROBE_SYMBOL(el0_ia); >> >> @@ -154,8 +163,11 @@ static void notrace el0_pc(struct pt_regs *regs, unsigned long esr) >> { >> unsigned long far = read_sysreg(far_el1); >> >> + if (!is_ttbr0_addr(instruction_pointer(regs))) >> + arm64_apply_bp_hardening(); >> + >> user_exit_irqoff(); >> - local_daif_restore(DAIF_PROCCTX_NOIRQ); >> + local_daif_restore(DAIF_PROCCTX); >> do_sp_pc_abort(far, esr, regs); >> } >> NOKPROBE_SYMBOL(el0_pc); > > This is much nicer, and AFAICT is correct, so: > > Reviewed-by: Mark Rutland <mark.rutland@arm.com> Thanks! James
diff --git a/arch/arm64/include/asm/processor.h b/arch/arm64/include/asm/processor.h index 5623685c7d13..c0c28c4589a8 100644 --- a/arch/arm64/include/asm/processor.h +++ b/arch/arm64/include/asm/processor.h @@ -24,6 +24,7 @@ #include <linux/build_bug.h> #include <linux/cache.h> #include <linux/init.h> +#include <linux/thread_info.h> #include <linux/stddef.h> #include <linux/string.h> @@ -214,6 +215,12 @@ static inline void start_thread(struct pt_regs *regs, unsigned long pc, regs->sp = sp; } +static inline bool is_ttbr0_addr(unsigned long addr) +{ + /* entry assembly clears tags for TTBR0 addrs */ + return addr < TASK_SIZE; +} + #ifdef CONFIG_COMPAT static inline void compat_start_thread(struct pt_regs *regs, unsigned long pc, unsigned long sp) diff --git a/arch/arm64/kernel/entry-common.c b/arch/arm64/kernel/entry-common.c index 176969e55677..eb73d250a081 100644 --- a/arch/arm64/kernel/entry-common.c +++ b/arch/arm64/kernel/entry-common.c @@ -14,6 +14,7 @@ #include <asm/esr.h> #include <asm/exception.h> #include <asm/kprobes.h> +#include <asm/mmu.h> #include <asm/sysreg.h> static void notrace el1_abort(struct pt_regs *regs, unsigned long esr) @@ -112,9 +113,17 @@ static void notrace el0_ia(struct pt_regs *regs, unsigned long esr) { unsigned long far = read_sysreg(far_el1); + /* + * We've taken an instruction abort from userspace and not yet + * re-enabled IRQs. If the address is a kernel address, apply + * BP hardening prior to enabling IRQs and pre-emption. + */ + if (!is_ttbr0_addr(far)) + arm64_apply_bp_hardening(); + user_exit_irqoff(); - local_daif_restore(DAIF_PROCCTX_NOIRQ); - do_el0_ia_bp_hardening(far, esr, regs); + local_daif_restore(DAIF_PROCCTX); + do_mem_abort(far, esr, regs); } NOKPROBE_SYMBOL(el0_ia); @@ -154,8 +163,11 @@ static void notrace el0_pc(struct pt_regs *regs, unsigned long esr) { unsigned long far = read_sysreg(far_el1); + if (!is_ttbr0_addr(instruction_pointer(regs))) + arm64_apply_bp_hardening(); + user_exit_irqoff(); - local_daif_restore(DAIF_PROCCTX_NOIRQ); + local_daif_restore(DAIF_PROCCTX); do_sp_pc_abort(far, esr, regs); } NOKPROBE_SYMBOL(el0_pc); diff --git a/arch/arm64/mm/fault.c b/arch/arm64/mm/fault.c index 0857c2fc38b9..88e4bd4bc103 100644 --- a/arch/arm64/mm/fault.c +++ b/arch/arm64/mm/fault.c @@ -34,6 +34,7 @@ #include <asm/esr.h> #include <asm/kasan.h> #include <asm/kprobes.h> +#include <asm/processor.h> #include <asm/sysreg.h> #include <asm/system_misc.h> #include <asm/pgtable.h> @@ -102,12 +103,6 @@ static void mem_abort_decode(unsigned int esr) data_abort_decode(esr); } -static inline bool is_ttbr0_addr(unsigned long addr) -{ - /* entry assembly clears tags for TTBR0 addrs */ - return addr < TASK_SIZE; -} - static inline bool is_ttbr1_addr(unsigned long addr) { /* TTBR1 addresses may have a tag if KASAN_SW_TAGS is in use */ @@ -749,30 +744,8 @@ void do_el0_irq_bp_hardening(void) } NOKPROBE_SYMBOL(do_el0_irq_bp_hardening); -void do_el0_ia_bp_hardening(unsigned long addr, unsigned int esr, - struct pt_regs *regs) -{ - /* - * We've taken an instruction abort from userspace and not yet - * re-enabled IRQs. If the address is a kernel address, apply - * BP hardening prior to enabling IRQs and pre-emption. - */ - if (!is_ttbr0_addr(addr)) - arm64_apply_bp_hardening(); - - local_daif_restore(DAIF_PROCCTX); - do_mem_abort(addr, esr, regs); -} -NOKPROBE_SYMBOL(do_el0_ia_bp_hardening); - void do_sp_pc_abort(unsigned long addr, unsigned int esr, struct pt_regs *regs) { - if (user_mode(regs)) { - if (!is_ttbr0_addr(instruction_pointer(regs))) - arm64_apply_bp_hardening(); - local_daif_restore(DAIF_PROCCTX); - } - arm64_notify_die("SP/PC alignment exception", regs, SIGBUS, BUS_ADRALN, (void __user *)addr, esr); }
The previous patches mechanically transformed the assembly version of entry.S to entry-common.c for synchronous exceptions. The C version of local_daif_restore() doesn't quite do the same thing as the assembly versions if pseudo-NMI is in use. In particular, | local_daif_restore(DAIF_PROCCTX_NOIRQ) will still allow pNMI to be delivered. This is not the behaviour do_el0_ia_bp_hardening() and do_sp_pc_abort() want as it should not be possible for the PMU handler to run as an NMI until the bp-hardening sequence has run. The bp-hardening calls were placed where they are because this was the first C code to run after the relevant exceptions. As we've now moved that point earlier, move the checks and calls earlier too. This makes it clearer that this stuff runs before any kind of exception, and saves modifying PSTATE twice. Signed-off-by: James Morse <james.morse@arm.com> Cc: Julien Thierry <julien.thierry.kdev@gmail.com> --- arch/arm64/include/asm/processor.h | 7 +++++++ arch/arm64/kernel/entry-common.c | 18 +++++++++++++++--- arch/arm64/mm/fault.c | 29 +---------------------------- 3 files changed, 23 insertions(+), 31 deletions(-)