@@ -68,6 +68,7 @@ config ARM64
select HAVE_FUNCTION_GRAPH_TRACER
select HAVE_GENERIC_DMA_COHERENT
select HAVE_HW_BREAKPOINT if PERF_EVENTS
+ select HAVE_IRQ_EXIT_ON_IRQ_STACK
select HAVE_MEMBLOCK
select HAVE_PATA_PLATFORM
select HAVE_PERF_EVENTS
@@ -2,14 +2,32 @@
#define __ASM_IRQ_H
#include <linux/irqchip/arm-gic-acpi.h>
+#include <asm/stacktrace.h>
#include <asm-generic/irq.h>
+struct irq_stack {
+ void *stack;
+ struct stackframe frame;
+};
+
+DECLARE_PER_CPU(struct irq_stack, irq_stacks);
+
+static inline bool in_irq_stack(unsigned int cpu)
+{
+ unsigned long high = (unsigned long)per_cpu(irq_stacks, cpu).stack;
+
+ return (current_stack_pointer >= round_down(high, THREAD_SIZE)) &&
+ current_stack_pointer < high;
+}
+
struct pt_regs;
extern void migrate_irqs(void);
extern void set_handle_irq(void (*handle_irq)(struct pt_regs *));
+extern int alloc_irq_stack(unsigned int cpu);
+
static inline void acpi_irq_init(void)
{
/*
@@ -71,10 +71,16 @@ register unsigned long current_stack_pointer asm ("sp");
*/
static inline struct thread_info *current_thread_info(void) __attribute_const__;
+/*
+ * struct thread_info can be accessed directly via sp_el0.
+ */
static inline struct thread_info *current_thread_info(void)
{
- return (struct thread_info *)
- (current_stack_pointer & ~(THREAD_SIZE - 1));
+ unsigned long sp_el0;
+
+ asm ("mrs %0, sp_el0" : "=r" (sp_el0));
+
+ return (struct thread_info *)sp_el0;
}
#define thread_saved_pc(tsk) \
@@ -41,6 +41,11 @@ int main(void)
BLANK();
DEFINE(THREAD_CPU_CONTEXT, offsetof(struct task_struct, thread.cpu_context));
BLANK();
+ DEFINE(IRQ_STACK, offsetof(struct irq_stack, stack));
+ DEFINE(IRQ_FRAME_FP, offsetof(struct irq_stack, frame.fp));
+ DEFINE(IRQ_FRMAE_SP, offsetof(struct irq_stack, frame.sp));
+ DEFINE(IRQ_FRAME_PC, offsetof(struct irq_stack, frame.pc));
+ BLANK();
DEFINE(S_X0, offsetof(struct pt_regs, regs[0]));
DEFINE(S_X1, offsetof(struct pt_regs, regs[1]));
DEFINE(S_X2, offsetof(struct pt_regs, regs[2]));
@@ -88,7 +88,8 @@
.if \el == 0
mrs x21, sp_el0
- get_thread_info tsk // Ensure MDSCR_EL1.SS is clear,
+ mov tsk, sp
+ and tsk, tsk, #~(THREAD_SIZE - 1) // Ensure MDSCR_EL1.SS is clear,
ldr x19, [tsk, #TI_FLAGS] // since we can unmask debug
disable_step_tsk x19, x20 // exceptions when scheduling.
.else
@@ -96,6 +97,7 @@
.endif
mrs x22, elr_el1
mrs x23, spsr_el1
+ mov x24, x29
stp lr, x21, [sp, #S_LR]
stp x22, x23, [sp, #S_PC]
@@ -108,12 +110,20 @@
.endif
/*
+ * Set sp_el0 to current thread_info.
+ */
+ .if \el == 0
+ msr sp_el0, tsk
+ .endif
+
+ /*
* Registers that may be useful after this macro is invoked:
*
* x21 - aborted SP
* x22 - aborted PC
* x23 - aborted PSTATE
- */
+ * x24 - aborted FP
+ */
.endm
.macro kernel_exit, el
@@ -164,8 +174,35 @@ alternative_endif
.endm
.macro get_thread_info, rd
- mov \rd, sp
- and \rd, \rd, #~(THREAD_SIZE - 1) // top of stack
+ mrs \rd, sp_el0
+ .endm
+
+ .macro irq_stack_entry
+ adr_l x19, irq_stacks
+ mrs x20, tpidr_el1
+ add x19, x19, x20
+
+ ldr x23, [x19, #IRQ_STACK]
+ and x20, x23, #~(THREAD_SIZE - 1)
+ mov x23, sp
+ and x23, x23, #~(THREAD_SIZE - 1)
+ cmp x20, x23 // check irq re-enterance
+ beq 1f
+
+ ldr x20, [x19, #IRQ_STACK]
+ str x24, [x19, #IRQ_FRAME_FP]
+ str x21, [x19, #IRQ_FRMAE_SP]
+ str x22, [x19, #IRQ_FRAME_PC]
+1: mov x19, sp
+ csel x23, x19, x20, eq // x20 = top of irq stack
+ mov sp, x23
+ .endm
+
+ /*
+ * x19 is preserved between irq_stack_entry and irq_stack_exit.
+ */
+ .macro irq_stack_exit
+ mov sp, x19
.endm
/*
@@ -183,10 +220,11 @@ tsk .req x28 // current thread_info
* Interrupt handling.
*/
.macro irq_handler
- adrp x1, handle_arch_irq
- ldr x1, [x1, #:lo12:handle_arch_irq]
+ ldr_l x1, handle_arch_irq
mov x0, sp
+ irq_stack_entry
blr x1
+ irq_stack_exit
.endm
.text
@@ -597,6 +635,8 @@ ENTRY(cpu_switch_to)
ldp x29, x9, [x8], #16
ldr lr, [x8]
mov sp, x9
+ and x9, x9, #~(THREAD_SIZE - 1)
+ msr sp_el0, x9
ret
ENDPROC(cpu_switch_to)
@@ -441,6 +441,9 @@ __mmap_switched:
b 1b
2:
adr_l sp, initial_sp, x4
+ mov x4, sp
+ and x4, x4, #~(THREAD_SIZE - 1)
+ msr sp_el0, x4 // Save thread_info
str_l x21, __fdt_pointer, x5 // Save FDT pointer
str_l x24, memstart_addr, x6 // Save PHYS_OFFSET
mov x29, #0
@@ -618,6 +621,8 @@ ENDPROC(secondary_startup)
ENTRY(__secondary_switched)
ldr x0, [x21] // get secondary_data.stack
mov sp, x0
+ and x0, x0, #~(THREAD_SIZE - 1)
+ msr sp_el0, x0 // save thread_info
mov x29, #0
b secondary_start_kernel
ENDPROC(__secondary_switched)
@@ -31,6 +31,8 @@
unsigned long irq_err_count;
+DEFINE_PER_CPU(struct irq_stack, irq_stacks);
+
int arch_show_interrupts(struct seq_file *p, int prec)
{
show_ipi_list(p, prec);
@@ -50,6 +52,9 @@ void __init set_handle_irq(void (*handle_irq)(struct pt_regs *))
void __init init_IRQ(void)
{
+ if (alloc_irq_stack(smp_processor_id()))
+ panic("Failed to allocate IRQ stack for boot cpu");
+
irqchip_init();
if (!handle_arch_irq)
panic("No interrupt controller found.");
@@ -115,3 +120,19 @@ void migrate_irqs(void)
local_irq_restore(flags);
}
#endif /* CONFIG_HOTPLUG_CPU */
+
+int alloc_irq_stack(unsigned int cpu)
+{
+ void *stack;
+
+ if (per_cpu(irq_stacks, cpu).stack)
+ return 0;
+
+ stack = (void *)__get_free_pages(THREADINFO_GFP, THREAD_SIZE_ORDER);
+ if (!stack)
+ return -ENOMEM;
+
+ per_cpu(irq_stacks, cpu).stack = stack + THREAD_START_SP;
+
+ return 0;
+}
@@ -173,6 +173,9 @@ ENTRY(cpu_resume)
/* load physical address of identity map page table in x1 */
adrp x1, idmap_pg_dir
mov sp, x2
+ /* save thread_info */
+ and x2, x2, #~(THREAD_SIZE - 1)
+ msr sp_el0, x2
/*
* cpu_do_resume expects x0 to contain context physical address
* pointer and x1 to contain physical address of 1:1 page tables
@@ -97,6 +97,12 @@ int __cpu_up(unsigned int cpu, struct task_struct *idle)
secondary_data.stack = task_stack_page(idle) + THREAD_START_SP;
__flush_dcache_area(&secondary_data, sizeof(secondary_data));
+ ret = alloc_irq_stack(cpu);
+ if (ret) {
+ pr_crit("CPU%u: failed to allocate IRQ stack\n", cpu);
+ return ret;
+ }
+
/*
* Now bring the CPU into our world.
*/
@@ -146,6 +146,8 @@ static void dump_instr(const char *lvl, struct pt_regs *regs)
static void dump_backtrace(struct pt_regs *regs, struct task_struct *tsk)
{
struct stackframe frame;
+ unsigned int cpu = smp_processor_id();
+ bool in_irq = in_irq_stack(cpu);
pr_debug("%s(regs = %p tsk = %p)\n", __func__, regs, tsk);
@@ -170,6 +172,8 @@ static void dump_backtrace(struct pt_regs *regs, struct task_struct *tsk)
}
pr_emerg("Call trace:\n");
+repeat:
+ pr_emerg("<%s>\n", in_irq ? "IRQ" : "Process");
while (1) {
unsigned long where = frame.pc;
int ret;
@@ -179,6 +183,12 @@ static void dump_backtrace(struct pt_regs *regs, struct task_struct *tsk)
break;
dump_backtrace_entry(where, frame.sp);
}
+
+ if (in_irq) {
+ frame = per_cpu(irq_stacks, cpu).frame;
+ in_irq = false;
+ goto repeat;
+ }
}
void show_stack(struct task_struct *tsk, unsigned long *sp)