| Message ID | 20200325161249.55095-17-glider@google.com (mailing list archive) |
|---|---|
| State | New, archived |
| Headers | show |
| Series | Add KernelMemorySanitizer infrastructure | expand |
On Wed, Mar 25, 2020 at 5:13 PM <glider@google.com> wrote: > > Add assembly helpers to entry_64.S that invoke hooks from kmsan_entry.c and > notify KMSAN about interrupts. > Also call these hooks from kernel/softirq.c > This is needed to switch between several KMSAN contexts holding function > parameter metadata. > > Signed-off-by: Alexander Potapenko <glider@google.com> > To: Alexander Potapenko <glider@google.com> > Cc: Jens Axboe <axboe@kernel.dk> > Cc: Andy Lutomirski <luto@kernel.org> > Cc: Vegard Nossum <vegard.nossum@oracle.com> > Cc: Dmitry Vyukov <dvyukov@google.com> > Cc: Marco Elver <elver@google.com> > Cc: Andrey Konovalov <andreyknvl@google.com> > Cc: Christoph Hellwig <hch@lst.de> > Cc: linux-mm@kvack.org Acked-by: Andrey Konovalov <andreyknvl@google.com> > --- > > v4: > - moved softirq changes to this patch > > Change-Id: I3037d51672fe69d09e588b27adb2d9fdc6ad3a7d > --- > arch/x86/entry/entry_64.S | 16 ++++++++++++++++ > kernel/softirq.c | 5 +++++ > 2 files changed, 21 insertions(+) > > diff --git a/arch/x86/entry/entry_64.S b/arch/x86/entry/entry_64.S > index 0e9504fabe526..03f5a32b0af4d 100644 > --- a/arch/x86/entry/entry_64.S > +++ b/arch/x86/entry/entry_64.S > @@ -35,6 +35,7 @@ > #include <asm/asm.h> > #include <asm/smap.h> > #include <asm/pgtable_types.h> > +#include <asm/kmsan.h> > #include <asm/export.h> > #include <asm/frame.h> > #include <asm/nospec-branch.h> > @@ -575,6 +576,7 @@ SYM_CODE_START(interrupt_entry) > > 1: > ENTER_IRQ_STACK old_rsp=%rdi save_ret=1 > + KMSAN_INTERRUPT_ENTER > /* We entered an interrupt context - irqs are off: */ > TRACE_IRQS_OFF > > @@ -604,12 +606,14 @@ SYM_CODE_START_LOCAL(common_interrupt) > addq $-0x80, (%rsp) /* Adjust vector to [-256, -1] range */ > call interrupt_entry > UNWIND_HINT_REGS indirect=1 > + KMSAN_UNPOISON_PT_REGS > call do_IRQ /* rdi points to pt_regs */ > /* 0(%rsp): old RSP */ > ret_from_intr: > DISABLE_INTERRUPTS(CLBR_ANY) > TRACE_IRQS_OFF > > + KMSAN_INTERRUPT_EXIT > LEAVE_IRQ_STACK > > testb $3, CS(%rsp) > @@ -801,6 +805,7 @@ SYM_CODE_START(\sym) > .Lcommon_\sym: > call interrupt_entry > UNWIND_HINT_REGS indirect=1 > + KMSAN_UNPOISON_PT_REGS > call \do_sym /* rdi points to pt_regs */ > jmp ret_from_intr > SYM_CODE_END(\sym) > @@ -908,15 +913,18 @@ apicinterrupt IRQ_WORK_VECTOR irq_work_interrupt smp_irq_work_interrupt > > .if \shift_ist != -1 > subq $\ist_offset, CPU_TSS_IST(\shift_ist) > + KMSAN_IST_ENTER(\shift_ist) > .endif > > .if \read_cr2 > movq %r12, %rdx /* Move CR2 into 3rd argument */ > .endif > > + KMSAN_UNPOISON_PT_REGS > call \do_sym > > .if \shift_ist != -1 > + KMSAN_IST_EXIT(\shift_ist) > addq $\ist_offset, CPU_TSS_IST(\shift_ist) > .endif > > @@ -1079,7 +1087,9 @@ SYM_FUNC_START(do_softirq_own_stack) > pushq %rbp > mov %rsp, %rbp > ENTER_IRQ_STACK regs=0 old_rsp=%r11 > + KMSAN_SOFTIRQ_ENTER > call __do_softirq > + KMSAN_SOFTIRQ_EXIT > LEAVE_IRQ_STACK regs=0 > leaveq > ret > @@ -1466,9 +1476,12 @@ SYM_CODE_START(nmi) > * done with the NMI stack. > */ > > + KMSAN_NMI_ENTER > movq %rsp, %rdi > movq $-1, %rsi > + KMSAN_UNPOISON_PT_REGS > call do_nmi > + KMSAN_NMI_EXIT > > /* > * Return back to user mode. We must *not* do the normal exit > @@ -1678,10 +1691,13 @@ end_repeat_nmi: > call paranoid_entry > UNWIND_HINT_REGS > > + KMSAN_NMI_ENTER > /* paranoidentry do_nmi, 0; without TRACE_IRQS_OFF */ > movq %rsp, %rdi > movq $-1, %rsi > + KMSAN_UNPOISON_PT_REGS > call do_nmi > + KMSAN_NMI_EXIT > > /* Always restore stashed CR3 value (see paranoid_entry) */ > RESTORE_CR3 scratch_reg=%r15 save_reg=%r14 > diff --git a/kernel/softirq.c b/kernel/softirq.c > index 0427a86743a46..98c5f4062cbfe 100644 > --- a/kernel/softirq.c > +++ b/kernel/softirq.c > @@ -11,6 +11,7 @@ > > #include <linux/export.h> > #include <linux/kernel_stat.h> > +#include <linux/kmsan.h> > #include <linux/interrupt.h> > #include <linux/init.h> > #include <linux/mm.h> > @@ -370,7 +371,9 @@ static inline void invoke_softirq(void) > * it is the irq stack, because it should be near empty > * at this stage. > */ > + kmsan_context_enter(); > __do_softirq(); > + kmsan_context_exit(); > #else > /* > * Otherwise, irq_exit() is called on the task stack that can > @@ -600,7 +603,9 @@ static void run_ksoftirqd(unsigned int cpu) > * We can safely run softirq on inline stack, as we are not deep > * in the task stack here. > */ > + kmsan_context_enter(); > __do_softirq(); > + kmsan_context_exit(); > local_irq_enable(); > cond_resched(); > return; > -- > 2.25.1.696.g5e7596f4ac-goog >
diff --git a/arch/x86/entry/entry_64.S b/arch/x86/entry/entry_64.S index 0e9504fabe526..03f5a32b0af4d 100644 --- a/arch/x86/entry/entry_64.S +++ b/arch/x86/entry/entry_64.S @@ -35,6 +35,7 @@ #include <asm/asm.h> #include <asm/smap.h> #include <asm/pgtable_types.h> +#include <asm/kmsan.h> #include <asm/export.h> #include <asm/frame.h> #include <asm/nospec-branch.h> @@ -575,6 +576,7 @@ SYM_CODE_START(interrupt_entry) 1: ENTER_IRQ_STACK old_rsp=%rdi save_ret=1 + KMSAN_INTERRUPT_ENTER /* We entered an interrupt context - irqs are off: */ TRACE_IRQS_OFF @@ -604,12 +606,14 @@ SYM_CODE_START_LOCAL(common_interrupt) addq $-0x80, (%rsp) /* Adjust vector to [-256, -1] range */ call interrupt_entry UNWIND_HINT_REGS indirect=1 + KMSAN_UNPOISON_PT_REGS call do_IRQ /* rdi points to pt_regs */ /* 0(%rsp): old RSP */ ret_from_intr: DISABLE_INTERRUPTS(CLBR_ANY) TRACE_IRQS_OFF + KMSAN_INTERRUPT_EXIT LEAVE_IRQ_STACK testb $3, CS(%rsp) @@ -801,6 +805,7 @@ SYM_CODE_START(\sym) .Lcommon_\sym: call interrupt_entry UNWIND_HINT_REGS indirect=1 + KMSAN_UNPOISON_PT_REGS call \do_sym /* rdi points to pt_regs */ jmp ret_from_intr SYM_CODE_END(\sym) @@ -908,15 +913,18 @@ apicinterrupt IRQ_WORK_VECTOR irq_work_interrupt smp_irq_work_interrupt .if \shift_ist != -1 subq $\ist_offset, CPU_TSS_IST(\shift_ist) + KMSAN_IST_ENTER(\shift_ist) .endif .if \read_cr2 movq %r12, %rdx /* Move CR2 into 3rd argument */ .endif + KMSAN_UNPOISON_PT_REGS call \do_sym .if \shift_ist != -1 + KMSAN_IST_EXIT(\shift_ist) addq $\ist_offset, CPU_TSS_IST(\shift_ist) .endif @@ -1079,7 +1087,9 @@ SYM_FUNC_START(do_softirq_own_stack) pushq %rbp mov %rsp, %rbp ENTER_IRQ_STACK regs=0 old_rsp=%r11 + KMSAN_SOFTIRQ_ENTER call __do_softirq + KMSAN_SOFTIRQ_EXIT LEAVE_IRQ_STACK regs=0 leaveq ret @@ -1466,9 +1476,12 @@ SYM_CODE_START(nmi) * done with the NMI stack. */ + KMSAN_NMI_ENTER movq %rsp, %rdi movq $-1, %rsi + KMSAN_UNPOISON_PT_REGS call do_nmi + KMSAN_NMI_EXIT /* * Return back to user mode. We must *not* do the normal exit @@ -1678,10 +1691,13 @@ end_repeat_nmi: call paranoid_entry UNWIND_HINT_REGS + KMSAN_NMI_ENTER /* paranoidentry do_nmi, 0; without TRACE_IRQS_OFF */ movq %rsp, %rdi movq $-1, %rsi + KMSAN_UNPOISON_PT_REGS call do_nmi + KMSAN_NMI_EXIT /* Always restore stashed CR3 value (see paranoid_entry) */ RESTORE_CR3 scratch_reg=%r15 save_reg=%r14 diff --git a/kernel/softirq.c b/kernel/softirq.c index 0427a86743a46..98c5f4062cbfe 100644 --- a/kernel/softirq.c +++ b/kernel/softirq.c @@ -11,6 +11,7 @@ #include <linux/export.h> #include <linux/kernel_stat.h> +#include <linux/kmsan.h> #include <linux/interrupt.h> #include <linux/init.h> #include <linux/mm.h> @@ -370,7 +371,9 @@ static inline void invoke_softirq(void) * it is the irq stack, because it should be near empty * at this stage. */ + kmsan_context_enter(); __do_softirq(); + kmsan_context_exit(); #else /* * Otherwise, irq_exit() is called on the task stack that can @@ -600,7 +603,9 @@ static void run_ksoftirqd(unsigned int cpu) * We can safely run softirq on inline stack, as we are not deep * in the task stack here. */ + kmsan_context_enter(); __do_softirq(); + kmsan_context_exit(); local_irq_enable(); cond_resched(); return;
Add assembly helpers to entry_64.S that invoke hooks from kmsan_entry.c and notify KMSAN about interrupts. Also call these hooks from kernel/softirq.c This is needed to switch between several KMSAN contexts holding function parameter metadata. Signed-off-by: Alexander Potapenko <glider@google.com> To: Alexander Potapenko <glider@google.com> Cc: Jens Axboe <axboe@kernel.dk> Cc: Andy Lutomirski <luto@kernel.org> Cc: Vegard Nossum <vegard.nossum@oracle.com> Cc: Dmitry Vyukov <dvyukov@google.com> Cc: Marco Elver <elver@google.com> Cc: Andrey Konovalov <andreyknvl@google.com> Cc: Christoph Hellwig <hch@lst.de> Cc: linux-mm@kvack.org --- v4: - moved softirq changes to this patch Change-Id: I3037d51672fe69d09e588b27adb2d9fdc6ad3a7d --- arch/x86/entry/entry_64.S | 16 ++++++++++++++++ kernel/softirq.c | 5 +++++ 2 files changed, 21 insertions(+)