| Message ID | 1442923918-11289-1-git-send-email-jungseoklee85@gmail.com (mailing list archive) |
|---|---|
| State | New, archived |
| Headers | show |
On Sep 22, 2015, at 9:11 PM, Jungseok Lee wrote: Dear all, I cannot reply to your feedbacks for about a week. I will answer your emails as soon as possible after that time. Best Regards Jungseok Lee > Currently, kernel context and interrupts are handled using a single > kernel stack navigated by sp_el1. This forces a system to use 16KB > stack, not 8KB one. This restriction makes low memory platforms suffer > from memory pressure accompanied by performance degradation. > > This patch addresses the issue as introducing a separate percpu IRQ > stack to handle both hard and soft interrupts with two ground rules: > > - Utilize sp_el0 in EL1 context, which is not used currently > - Do not complicate current_thread_info calculation > > It is a core concept to directly retrieve struct thread_info from > sp_el0. This approach helps to prevent text section size from being > increased largely as removing masking operation using THREAD_SIZE > in tons of places. > > [Thanks to James Morse for his valuable feedbacks which greatly help > to figure out a better implementation. - Jungseok] > > Tested-by: James Morse <james.morse@arm.com> > Signed-off-by: Jungseok Lee <jungseoklee85@gmail.com> > --- > Changes since v2: > - Optmised current_thread_info function as removing masking operation > and volatile keyword per James and Catalin > - Reworked irq re-enterance check logic using top-bit comparison of > stacks per James > - Added sp_el0 update in cpu_resume per James > - Selected HAVE_IRQ_EXIT_ON_IRQ_STACK to expose this feature explicitly > - Added a Tested-by tag from James > - Added comments on sp_el0 as a helper messeage > > Changes since v1: > - Rebased on top of v4.3-rc1 > - Removed Kconfig about IRQ stack, per James > - Used PERCPU for IRQ stack, per James > - Tried to allocate IRQ stack when CPU is about to start up, per James > - Moved sp_el0 update into kernel_entry macro, per James > - Dropped S_SP removal patch, per Mark and James > > arch/arm64/Kconfig | 1 + > arch/arm64/include/asm/irq.h | 2 + > arch/arm64/include/asm/thread_info.h | 10 ++++- > arch/arm64/kernel/entry.S | 35 +++++++++++++--- > arch/arm64/kernel/head.S | 5 +++ > arch/arm64/kernel/irq.c | 21 ++++++++++ > arch/arm64/kernel/sleep.S | 3 ++ > arch/arm64/kernel/smp.c | 6 +++ > 8 files changed, 76 insertions(+), 7 deletions(-) > > diff --git a/arch/arm64/Kconfig b/arch/arm64/Kconfig > index 07d1811..9767bd9 100644 > --- a/arch/arm64/Kconfig > +++ b/arch/arm64/Kconfig > @@ -68,6 +68,7 @@ config ARM64 > select HAVE_FUNCTION_GRAPH_TRACER > select HAVE_GENERIC_DMA_COHERENT > select HAVE_HW_BREAKPOINT if PERF_EVENTS > + select HAVE_IRQ_EXIT_ON_IRQ_STACK > select HAVE_MEMBLOCK > select HAVE_PATA_PLATFORM > select HAVE_PERF_EVENTS > diff --git a/arch/arm64/include/asm/irq.h b/arch/arm64/include/asm/irq.h > index bbb251b..ba12725 100644 > --- a/arch/arm64/include/asm/irq.h > +++ b/arch/arm64/include/asm/irq.h > @@ -10,6 +10,8 @@ struct pt_regs; > extern void migrate_irqs(void); > extern void set_handle_irq(void (*handle_irq)(struct pt_regs *)); > > +extern int alloc_irq_stack(unsigned int cpu); > + > static inline void acpi_irq_init(void) > { > /* > diff --git a/arch/arm64/include/asm/thread_info.h b/arch/arm64/include/asm/thread_info.h > index dcd06d1..fa014df 100644 > --- a/arch/arm64/include/asm/thread_info.h > +++ b/arch/arm64/include/asm/thread_info.h > @@ -71,10 +71,16 @@ register unsigned long current_stack_pointer asm ("sp"); > */ > static inline struct thread_info *current_thread_info(void) __attribute_const__; > > +/* > + * struct thread_info can be accessed directly via sp_el0. > + */ > static inline struct thread_info *current_thread_info(void) > { > - return (struct thread_info *) > - (current_stack_pointer & ~(THREAD_SIZE - 1)); > + unsigned long sp_el0; > + > + asm ("mrs %0, sp_el0" : "=r" (sp_el0)); > + > + return (struct thread_info *)sp_el0; > } > > #define thread_saved_pc(tsk) \ > diff --git a/arch/arm64/kernel/entry.S b/arch/arm64/kernel/entry.S > index 4306c93..e5b8482 100644 > --- a/arch/arm64/kernel/entry.S > +++ b/arch/arm64/kernel/entry.S > @@ -88,7 +88,8 @@ > > .if \el == 0 > mrs x21, sp_el0 > - get_thread_info tsk // Ensure MDSCR_EL1.SS is clear, > + mov tsk, sp > + and tsk, tsk, #~(THREAD_SIZE - 1) // Ensure MDSCR_EL1.SS is clear, > ldr x19, [tsk, #TI_FLAGS] // since we can unmask debug > disable_step_tsk x19, x20 // exceptions when scheduling. > .else > @@ -105,6 +106,7 @@ > .if \el == 0 > mvn x21, xzr > str x21, [sp, #S_SYSCALLNO] > + msr sp_el0, tsk > .endif > > /* > @@ -164,8 +166,28 @@ alternative_endif > .endm > > .macro get_thread_info, rd > - mov \rd, sp > - and \rd, \rd, #~(THREAD_SIZE - 1) // top of stack > + mrs \rd, sp_el0 > + .endm > + > + .macro irq_stack_entry > + adr_l x21, irq_stack > + mrs x22, tpidr_el1 > + add x21, x21, x22 > + ldr x22, [x21] > + and x21, x22, #~(THREAD_SIZE - 1) > + mov x23, sp > + and x23, x23, #~(THREAD_SIZE - 1) > + cmp x21, x23 // check irq re-enterance > + mov x21, sp > + csel x23, x21, x22, eq // x22 = top of irq stack > + mov sp, x23 > + .endm > + > + /* > + * x21 is preserved between irq_stack_entry and irq_stack_exit. > + */ > + .macro irq_stack_exit > + mov sp, x21 > .endm > > /* > @@ -183,10 +205,11 @@ tsk .req x28 // current thread_info > * Interrupt handling. > */ > .macro irq_handler > - adrp x1, handle_arch_irq > - ldr x1, [x1, #:lo12:handle_arch_irq] > + ldr_l x1, handle_arch_irq > mov x0, sp > + irq_stack_entry > blr x1 > + irq_stack_exit > .endm > > .text > @@ -597,6 +620,8 @@ ENTRY(cpu_switch_to) > ldp x29, x9, [x8], #16 > ldr lr, [x8] > mov sp, x9 > + and x9, x9, #~(THREAD_SIZE - 1) > + msr sp_el0, x9 > ret > ENDPROC(cpu_switch_to) > > diff --git a/arch/arm64/kernel/head.S b/arch/arm64/kernel/head.S > index 90d09ed..dab089b 100644 > --- a/arch/arm64/kernel/head.S > +++ b/arch/arm64/kernel/head.S > @@ -441,6 +441,9 @@ __mmap_switched: > b 1b > 2: > adr_l sp, initial_sp, x4 > + mov x4, sp > + and x4, x4, #~(THREAD_SIZE - 1) > + msr sp_el0, x4 // Save thread_info > str_l x21, __fdt_pointer, x5 // Save FDT pointer > str_l x24, memstart_addr, x6 // Save PHYS_OFFSET > mov x29, #0 > @@ -618,6 +621,8 @@ ENDPROC(secondary_startup) > ENTRY(__secondary_switched) > ldr x0, [x21] // get secondary_data.stack > mov sp, x0 > + and x0, x0, #~(THREAD_SIZE - 1) > + msr sp_el0, x0 // save thread_info > mov x29, #0 > b secondary_start_kernel > ENDPROC(__secondary_switched) > diff --git a/arch/arm64/kernel/irq.c b/arch/arm64/kernel/irq.c > index 11dc3fd..5272bfd 100644 > --- a/arch/arm64/kernel/irq.c > +++ b/arch/arm64/kernel/irq.c > @@ -31,6 +31,8 @@ > > unsigned long irq_err_count; > > +DEFINE_PER_CPU(void *, irq_stack); > + > int arch_show_interrupts(struct seq_file *p, int prec) > { > show_ipi_list(p, prec); > @@ -50,6 +52,9 @@ void __init set_handle_irq(void (*handle_irq)(struct pt_regs *)) > > void __init init_IRQ(void) > { > + if (alloc_irq_stack(smp_processor_id())) > + panic("Failed to allocate IRQ stack for boot cpu"); > + > irqchip_init(); > if (!handle_arch_irq) > panic("No interrupt controller found."); > @@ -115,3 +120,19 @@ void migrate_irqs(void) > local_irq_restore(flags); > } > #endif /* CONFIG_HOTPLUG_CPU */ > + > +int alloc_irq_stack(unsigned int cpu) > +{ > + void *stack; > + > + if (per_cpu(irq_stack, cpu)) > + return 0; > + > + stack = (void *)__get_free_pages(THREADINFO_GFP, THREAD_SIZE_ORDER); > + if (!stack) > + return -ENOMEM; > + > + per_cpu(irq_stack, cpu) = stack + THREAD_START_SP; > + > + return 0; > +} > diff --git a/arch/arm64/kernel/sleep.S b/arch/arm64/kernel/sleep.S > index f586f7c..e33fe33 100644 > --- a/arch/arm64/kernel/sleep.S > +++ b/arch/arm64/kernel/sleep.S > @@ -173,6 +173,9 @@ ENTRY(cpu_resume) > /* load physical address of identity map page table in x1 */ > adrp x1, idmap_pg_dir > mov sp, x2 > + /* save thread_info */ > + and x2, x2, #~(THREAD_SIZE - 1) > + msr sp_el0, x2 > /* > * cpu_do_resume expects x0 to contain context physical address > * pointer and x1 to contain physical address of 1:1 page tables > diff --git a/arch/arm64/kernel/smp.c b/arch/arm64/kernel/smp.c > index dbdaacd..0bd7049 100644 > --- a/arch/arm64/kernel/smp.c > +++ b/arch/arm64/kernel/smp.c > @@ -97,6 +97,12 @@ int __cpu_up(unsigned int cpu, struct task_struct *idle) > secondary_data.stack = task_stack_page(idle) + THREAD_START_SP; > __flush_dcache_area(&secondary_data, sizeof(secondary_data)); > > + ret = alloc_irq_stack(cpu); > + if (ret) { > + pr_crit("CPU%u: failed to allocate IRQ stack\n", cpu); > + return ret; > + } > + > /* > * Now bring the CPU into our world. > */ > -- > 2.5.0 >
Hi, On 22/09/15 13:11, Jungseok Lee wrote: > Currently, kernel context and interrupts are handled using a single > kernel stack navigated by sp_el1. This forces a system to use 16KB > stack, not 8KB one. This restriction makes low memory platforms suffer > from memory pressure accompanied by performance degradation. > > This patch addresses the issue as introducing a separate percpu IRQ > stack to handle both hard and soft interrupts with two ground rules: > > - Utilize sp_el0 in EL1 context, which is not used currently > - Do not complicate current_thread_info calculation > > It is a core concept to directly retrieve struct thread_info from > sp_el0. This approach helps to prevent text section size from being > increased largely as removing masking operation using THREAD_SIZE > in tons of places. One observed change in behaviour: Any stack-unwinding now stops at el1_irq(), which is the bottom of the irq stack. This shows up with perf (using incantation [0]), and with any calls to dump_stack() (which actually stops the frame before el1_irq()). I don't know if this will break something, (perf still seems to work) - but it makes the panic() output less useful, as all the 'other' cpus print: > CPU3: stopping > CPU: 3 PID: 0 Comm: swapper/3 Not tainted 4.3.0-rc3+ #223 > Hardware name: ARM Juno development board (r1) (DT) > Call trace: > [<ffff800000089878>] dump_backtrace+0x0/0x164 > [<ffff8000000899f8>] show_stack+0x1c/0x28 > [<ffff8000003134d0>] dump_stack+0x88/0xc8 > [<ffff80000008edcc>] handle_IPI+0x258/0x268 > [<ffff8000000824b8>] gic_handle_irq+0x88/0xa4 > Exception stack(0xffff8009769e3fc0 to 0xffff8009769e40e0) > <...values from stack ...> > CPU4: stopping > CPU: 4 PID: 0 Comm: swapper/4 Not tainted 4.3.0-rc3+ #223 > Hardware name: ARM Juno development board (r1) (DT) > Call trace: So we don't get to see what they were doing, as the IPI-irq and subsequent switch to the irq_stack hide the state. I was trying to fix this with the other version, (see the changes to kernel/stacktrace.c), but as Akashi Takahiro pointed out, I wasn't quite right... I will try to produce a fragment to tidy this up next week. Thanks, James [0] perf record -e mem:<address of __do_softirq()>:x -ag -- sleep 10; perf report --call-graph --stdio
On Oct 3, 2015, at 1:23 AM, James Morse wrote: > Hi, Hi James, > > On 22/09/15 13:11, Jungseok Lee wrote: >> Currently, kernel context and interrupts are handled using a single >> kernel stack navigated by sp_el1. This forces a system to use 16KB >> stack, not 8KB one. This restriction makes low memory platforms suffer >> from memory pressure accompanied by performance degradation. >> >> This patch addresses the issue as introducing a separate percpu IRQ >> stack to handle both hard and soft interrupts with two ground rules: >> >> - Utilize sp_el0 in EL1 context, which is not used currently >> - Do not complicate current_thread_info calculation >> >> It is a core concept to directly retrieve struct thread_info from >> sp_el0. This approach helps to prevent text section size from being >> increased largely as removing masking operation using THREAD_SIZE >> in tons of places. > > One observed change in behaviour: > Any stack-unwinding now stops at el1_irq(), which is the bottom of the irq > stack. This shows up with perf (using incantation [0]), and with any calls > to dump_stack() (which actually stops the frame before el1_irq()). > > I don't know if this will break something, (perf still seems to work) - but > it makes the panic() output less useful, as all the 'other' cpus print: Agreed. A process stack should be walked to deliver useful information. There are two approaches I've tried as experimental. 1) Link IRQ stack to a process one via frame pointer As saving x29 and elr_el1 into IRQ stack and then updating x29, IRQ stack could be linked to a process one. It is similar to your patch except some points. However, it might complicate "stack tracer on ftrace" issue. 2) Walk a process stack followed by IRQ one This idea, which is straightforward, comes from x86 implementation [1]. The approach might be orthogonal to "stack tracer on ftrace" issue. In this case, unfortunately, a top bit comparison of stack pointer cannot be adopted due to a necessity of a final snapshot of a process stack pointer, which is struct irq_stack::thread_sp in v2 patch. Which one is your favorite? or any ideas? BTW, I have another question. Is it reasonable to introduce THREAD_SIZE as a kernel configuration option like page size for the sake of convenience because a combination of ARM64 and a small ram is not unusual in real practice? Needless to say, a patch, reducing the size, can be managed as out of mainline tree one. [1] arch/x86/kernel/dumpstack_64.c Best Regards Jungseok Lee
On 10/04/2015 11:32 PM, Jungseok Lee wrote: > On Oct 3, 2015, at 1:23 AM, James Morse wrote: > >> Hi, > > Hi James, > >> >> On 22/09/15 13:11, Jungseok Lee wrote: >>> Currently, kernel context and interrupts are handled using a single >>> kernel stack navigated by sp_el1. This forces a system to use 16KB >>> stack, not 8KB one. This restriction makes low memory platforms suffer >>> from memory pressure accompanied by performance degradation. >>> >>> This patch addresses the issue as introducing a separate percpu IRQ >>> stack to handle both hard and soft interrupts with two ground rules: >>> >>> - Utilize sp_el0 in EL1 context, which is not used currently >>> - Do not complicate current_thread_info calculation >>> >>> It is a core concept to directly retrieve struct thread_info from >>> sp_el0. This approach helps to prevent text section size from being >>> increased largely as removing masking operation using THREAD_SIZE >>> in tons of places. >> >> One observed change in behaviour: >> Any stack-unwinding now stops at el1_irq(), which is the bottom of the irq >> stack. This shows up with perf (using incantation [0]), and with any calls >> to dump_stack() (which actually stops the frame before el1_irq()). >> >> I don't know if this will break something, (perf still seems to work) - but >> it makes the panic() output less useful, as all the 'other' cpus print: > > Agreed. A process stack should be walked to deliver useful information. > > There are two approaches I've tried as experimental. > > 1) Link IRQ stack to a process one via frame pointer > As saving x29 and elr_el1 into IRQ stack and then updating x29, IRQ stack > could be linked to a process one. It is similar to your patch except some > points. However, it might complicate "stack tracer on ftrace" issue. Well, as far as object_is_on_stack() works correctly, stack tracer will not follow an interrupt stack even if unwind_frame() might traverse from an interrupt stack to a process stack. See check_stack(). Under this assumption, I'm going to simplify my "stack tracer" bugfix by removing interrupt-related nasty hacks that I described in RFC. Thanks, -Takahiro AKASHI > 2) Walk a process stack followed by IRQ one > This idea, which is straightforward, comes from x86 implementation [1]. The > approach might be orthogonal to "stack tracer on ftrace" issue. In this case, > unfortunately, a top bit comparison of stack pointer cannot be adopted due to > a necessity of a final snapshot of a process stack pointer, which is struct > irq_stack::thread_sp in v2 patch. > > Which one is your favorite? or any ideas? > > BTW, I have another question. Is it reasonable to introduce THREAD_SIZE as a > kernel configuration option like page size for the sake of convenience because > a combination of ARM64 and a small ram is not unusual in real practice? Needless > to say, a patch, reducing the size, can be managed as out of mainline tree one. > > [1] arch/x86/kernel/dumpstack_64.c > > Best Regards > Jungseok Lee >
On 05/10/15 07:37, AKASHI Takahiro wrote: > On 10/04/2015 11:32 PM, Jungseok Lee wrote: >> On Oct 3, 2015, at 1:23 AM, James Morse wrote: >>> One observed change in behaviour: >>> Any stack-unwinding now stops at el1_irq(), which is the bottom of the irq >>> stack. This shows up with perf (using incantation [0]), and with any calls >>> to dump_stack() (which actually stops the frame before el1_irq()). >>> >>> I don't know if this will break something, (perf still seems to work) - but >>> it makes the panic() output less useful, as all the 'other' cpus print: >> >> Agreed. A process stack should be walked to deliver useful information. >> >> There are two approaches I've tried as experimental. >> >> 1) Link IRQ stack to a process one via frame pointer >> As saving x29 and elr_el1 into IRQ stack and then updating x29, IRQ stack >> could be linked to a process one. It is similar to your patch except some >> points. However, it might complicate "stack tracer on ftrace" issue. > > Well, as far as object_is_on_stack() works correctly, stack tracer will not > follow an interrupt stack even if unwind_frame() might traverse from > an interrupt stack to a process stack. See check_stack(). > > Under this assumption, I'm going to simplify my "stack tracer" bugfix > by removing interrupt-related nasty hacks that I described in RFC. > > Thanks, > -Takahiro AKASHI > > >> 2) Walk a process stack followed by IRQ one >> This idea, which is straightforward, comes from x86 implementation [1]. The >> approach might be orthogonal to "stack tracer on ftrace" issue. In this >> case, x86 has to walk interrupt/exception stacks because the order may be: process -> hw_irq -> debug_exception -> double_fault. Where each of these could have its own stack, the code needs to determine the correct order to produce a correct stack trace. Our case is a lot simpler, as we could only ever have two, and know the order. We only need to walk the irq stack if we are currently using it, and we always know the process stack is last. I would go with the first option, being careful of stack corruption when stepping between them. >> unfortunately, a top bit comparison of stack pointer cannot be adopted >> due to >> a necessity of a final snapshot of a process stack pointer, which is struct >> irq_stack::thread_sp in v2 patch. I'm not sure I follow you here. We can check if regs->sp is an irq stack by comparing it with the per-cpu irq_stack value, (top bits comparison). Then we know that the last frame-pointer (in your (1) above), will point to the process stack, at which point we can walk onto that stack. >> BTW, I have another question. Is it reasonable to introduce THREAD_SIZE as a >> kernel configuration option like page size for the sake of convenience >> because >> a combination of ARM64 and a small ram is not unusual in real practice? We want the smallest safe value. It's probably best leaving as it is for now - once we have this feature, we can collect maximum stack-usage sizes for different platforms and workloads, and decide on the smallest safe value. Thanks, James
diff --git a/arch/arm64/Kconfig b/arch/arm64/Kconfig index 07d1811..9767bd9 100644 --- a/arch/arm64/Kconfig +++ b/arch/arm64/Kconfig @@ -68,6 +68,7 @@ config ARM64 select HAVE_FUNCTION_GRAPH_TRACER select HAVE_GENERIC_DMA_COHERENT select HAVE_HW_BREAKPOINT if PERF_EVENTS + select HAVE_IRQ_EXIT_ON_IRQ_STACK select HAVE_MEMBLOCK select HAVE_PATA_PLATFORM select HAVE_PERF_EVENTS diff --git a/arch/arm64/include/asm/irq.h b/arch/arm64/include/asm/irq.h index bbb251b..ba12725 100644 --- a/arch/arm64/include/asm/irq.h +++ b/arch/arm64/include/asm/irq.h @@ -10,6 +10,8 @@ struct pt_regs; extern void migrate_irqs(void); extern void set_handle_irq(void (*handle_irq)(struct pt_regs *)); +extern int alloc_irq_stack(unsigned int cpu); + static inline void acpi_irq_init(void) { /* diff --git a/arch/arm64/include/asm/thread_info.h b/arch/arm64/include/asm/thread_info.h index dcd06d1..fa014df 100644 --- a/arch/arm64/include/asm/thread_info.h +++ b/arch/arm64/include/asm/thread_info.h @@ -71,10 +71,16 @@ register unsigned long current_stack_pointer asm ("sp"); */ static inline struct thread_info *current_thread_info(void) __attribute_const__; +/* + * struct thread_info can be accessed directly via sp_el0. + */ static inline struct thread_info *current_thread_info(void) { - return (struct thread_info *) - (current_stack_pointer & ~(THREAD_SIZE - 1)); + unsigned long sp_el0; + + asm ("mrs %0, sp_el0" : "=r" (sp_el0)); + + return (struct thread_info *)sp_el0; } #define thread_saved_pc(tsk) \ diff --git a/arch/arm64/kernel/entry.S b/arch/arm64/kernel/entry.S index 4306c93..e5b8482 100644 --- a/arch/arm64/kernel/entry.S +++ b/arch/arm64/kernel/entry.S @@ -88,7 +88,8 @@ .if \el == 0 mrs x21, sp_el0 - get_thread_info tsk // Ensure MDSCR_EL1.SS is clear, + mov tsk, sp + and tsk, tsk, #~(THREAD_SIZE - 1) // Ensure MDSCR_EL1.SS is clear, ldr x19, [tsk, #TI_FLAGS] // since we can unmask debug disable_step_tsk x19, x20 // exceptions when scheduling. .else @@ -105,6 +106,7 @@ .if \el == 0 mvn x21, xzr str x21, [sp, #S_SYSCALLNO] + msr sp_el0, tsk .endif /* @@ -164,8 +166,28 @@ alternative_endif .endm .macro get_thread_info, rd - mov \rd, sp - and \rd, \rd, #~(THREAD_SIZE - 1) // top of stack + mrs \rd, sp_el0 + .endm + + .macro irq_stack_entry + adr_l x21, irq_stack + mrs x22, tpidr_el1 + add x21, x21, x22 + ldr x22, [x21] + and x21, x22, #~(THREAD_SIZE - 1) + mov x23, sp + and x23, x23, #~(THREAD_SIZE - 1) + cmp x21, x23 // check irq re-enterance + mov x21, sp + csel x23, x21, x22, eq // x22 = top of irq stack + mov sp, x23 + .endm + + /* + * x21 is preserved between irq_stack_entry and irq_stack_exit. + */ + .macro irq_stack_exit + mov sp, x21 .endm /* @@ -183,10 +205,11 @@ tsk .req x28 // current thread_info * Interrupt handling. */ .macro irq_handler - adrp x1, handle_arch_irq - ldr x1, [x1, #:lo12:handle_arch_irq] + ldr_l x1, handle_arch_irq mov x0, sp + irq_stack_entry blr x1 + irq_stack_exit .endm .text @@ -597,6 +620,8 @@ ENTRY(cpu_switch_to) ldp x29, x9, [x8], #16 ldr lr, [x8] mov sp, x9 + and x9, x9, #~(THREAD_SIZE - 1) + msr sp_el0, x9 ret ENDPROC(cpu_switch_to) diff --git a/arch/arm64/kernel/head.S b/arch/arm64/kernel/head.S index 90d09ed..dab089b 100644 --- a/arch/arm64/kernel/head.S +++ b/arch/arm64/kernel/head.S @@ -441,6 +441,9 @@ __mmap_switched: b 1b 2: adr_l sp, initial_sp, x4 + mov x4, sp + and x4, x4, #~(THREAD_SIZE - 1) + msr sp_el0, x4 // Save thread_info str_l x21, __fdt_pointer, x5 // Save FDT pointer str_l x24, memstart_addr, x6 // Save PHYS_OFFSET mov x29, #0 @@ -618,6 +621,8 @@ ENDPROC(secondary_startup) ENTRY(__secondary_switched) ldr x0, [x21] // get secondary_data.stack mov sp, x0 + and x0, x0, #~(THREAD_SIZE - 1) + msr sp_el0, x0 // save thread_info mov x29, #0 b secondary_start_kernel ENDPROC(__secondary_switched) diff --git a/arch/arm64/kernel/irq.c b/arch/arm64/kernel/irq.c index 11dc3fd..5272bfd 100644 --- a/arch/arm64/kernel/irq.c +++ b/arch/arm64/kernel/irq.c @@ -31,6 +31,8 @@ unsigned long irq_err_count; +DEFINE_PER_CPU(void *, irq_stack); + int arch_show_interrupts(struct seq_file *p, int prec) { show_ipi_list(p, prec); @@ -50,6 +52,9 @@ void __init set_handle_irq(void (*handle_irq)(struct pt_regs *)) void __init init_IRQ(void) { + if (alloc_irq_stack(smp_processor_id())) + panic("Failed to allocate IRQ stack for boot cpu"); + irqchip_init(); if (!handle_arch_irq) panic("No interrupt controller found."); @@ -115,3 +120,19 @@ void migrate_irqs(void) local_irq_restore(flags); } #endif /* CONFIG_HOTPLUG_CPU */ + +int alloc_irq_stack(unsigned int cpu) +{ + void *stack; + + if (per_cpu(irq_stack, cpu)) + return 0; + + stack = (void *)__get_free_pages(THREADINFO_GFP, THREAD_SIZE_ORDER); + if (!stack) + return -ENOMEM; + + per_cpu(irq_stack, cpu) = stack + THREAD_START_SP; + + return 0; +} diff --git a/arch/arm64/kernel/sleep.S b/arch/arm64/kernel/sleep.S index f586f7c..e33fe33 100644 --- a/arch/arm64/kernel/sleep.S +++ b/arch/arm64/kernel/sleep.S @@ -173,6 +173,9 @@ ENTRY(cpu_resume) /* load physical address of identity map page table in x1 */ adrp x1, idmap_pg_dir mov sp, x2 + /* save thread_info */ + and x2, x2, #~(THREAD_SIZE - 1) + msr sp_el0, x2 /* * cpu_do_resume expects x0 to contain context physical address * pointer and x1 to contain physical address of 1:1 page tables diff --git a/arch/arm64/kernel/smp.c b/arch/arm64/kernel/smp.c index dbdaacd..0bd7049 100644 --- a/arch/arm64/kernel/smp.c +++ b/arch/arm64/kernel/smp.c @@ -97,6 +97,12 @@ int __cpu_up(unsigned int cpu, struct task_struct *idle) secondary_data.stack = task_stack_page(idle) + THREAD_START_SP; __flush_dcache_area(&secondary_data, sizeof(secondary_data)); + ret = alloc_irq_stack(cpu); + if (ret) { + pr_crit("CPU%u: failed to allocate IRQ stack\n", cpu); + return ret; + } + /* * Now bring the CPU into our world. */