diff mbox series

[2/2] arm64: scs: use vmapped IRQ and SDEI shadow stacks

Message ID 20201022202355.3529836-3-samitolvanen@google.com (mailing list archive)
State New, archived
Headers show
Series scs: switch to vmapped shadow stacks | expand

Commit Message

Sami Tolvanen Oct. 22, 2020, 8:23 p.m. UTC 
Use scs_alloc() to allocate also IRQ and SDEI shadow stacks instead of
using statically allocated stacks.

Signed-off-by: Sami Tolvanen <samitolvanen@google.com>
---
 arch/arm64/include/asm/scs.h | 21 ++++++++++-
 arch/arm64/kernel/entry.S    |  6 ++--
 arch/arm64/kernel/irq.c      |  2 ++
 arch/arm64/kernel/scs.c      | 67 +++++++++++++++++++++++++++++++++---
 arch/arm64/kernel/sdei.c     |  7 ++++
 include/linux/scs.h          |  8 ++---
 kernel/scs.c                 |  4 +--
 7 files changed, 101 insertions(+), 14 deletions(-)

Comments

Kees Cook Oct. 22, 2020, 10:38 p.m. UTC | #1 
On Thu, Oct 22, 2020 at 01:23:55PM -0700, Sami Tolvanen wrote:
> Use scs_alloc() to allocate also IRQ and SDEI shadow stacks instead of
> using statically allocated stacks.
> 
> Signed-off-by: Sami Tolvanen <samitolvanen@google.com>

Reviewed-by: Kees Cook <keescook@chromium.org>
Will Deacon Nov. 19, 2020, 1:11 p.m. UTC | #2 
On Thu, Oct 22, 2020 at 01:23:55PM -0700, Sami Tolvanen wrote:
> Use scs_alloc() to allocate also IRQ and SDEI shadow stacks instead of
> using statically allocated stacks.
> 
> Signed-off-by: Sami Tolvanen <samitolvanen@google.com>
> ---
>  arch/arm64/include/asm/scs.h | 21 ++++++++++-
>  arch/arm64/kernel/entry.S    |  6 ++--
>  arch/arm64/kernel/irq.c      |  2 ++
>  arch/arm64/kernel/scs.c      | 67 +++++++++++++++++++++++++++++++++---
>  arch/arm64/kernel/sdei.c     |  7 ++++
>  include/linux/scs.h          |  8 ++---
>  kernel/scs.c                 |  4 +--
>  7 files changed, 101 insertions(+), 14 deletions(-)
> 
> diff --git a/arch/arm64/include/asm/scs.h b/arch/arm64/include/asm/scs.h
> index eaa2cd92e4c1..e9d2c3e67ff9 100644
> --- a/arch/arm64/include/asm/scs.h
> +++ b/arch/arm64/include/asm/scs.h
> @@ -24,6 +24,25 @@
>  	.endm
>  #endif /* CONFIG_SHADOW_CALL_STACK */
>  
> -#endif /* __ASSEMBLY __ */
> +#else /* __ASSEMBLY__ */
> +
> +#include <linux/scs.h>
> +
> +#ifdef CONFIG_SHADOW_CALL_STACK
> +
> +extern void scs_init_irq(void);
> +
> +extern void scs_free_sdei(void);

This is only called on the scs_init_sdei() failure path, so it can be
static. But see below, because I think we should move all of these functions
out of scs.c anyway.

> diff --git a/arch/arm64/kernel/irq.c b/arch/arm64/kernel/irq.c
> index 9cf2fb87584a..54ba3725bc0e 100644
> --- a/arch/arm64/kernel/irq.c
> +++ b/arch/arm64/kernel/irq.c
> @@ -20,6 +20,7 @@
>  #include <linux/seq_file.h>
>  #include <linux/vmalloc.h>
>  #include <asm/daifflags.h>
> +#include <asm/scs.h>
>  #include <asm/vmap_stack.h>
>  
>  /* Only access this in an NMI enter/exit */
> @@ -54,6 +55,7 @@ static void init_irq_stacks(void)
>  void __init init_IRQ(void)
>  {
>  	init_irq_stacks();
> +	scs_init_irq();

If we internalise this in init_irq_stacks()...

> diff --git a/arch/arm64/kernel/sdei.c b/arch/arm64/kernel/sdei.c
> index 7689f2031c0c..04519a7cb51d 100644
> --- a/arch/arm64/kernel/sdei.c
> +++ b/arch/arm64/kernel/sdei.c
> @@ -13,6 +13,7 @@
>  #include <asm/kprobes.h>
>  #include <asm/mmu.h>
>  #include <asm/ptrace.h>
> +#include <asm/scs.h>
>  #include <asm/sections.h>
>  #include <asm/stacktrace.h>
>  #include <asm/sysreg.h>
> @@ -138,6 +139,12 @@ unsigned long sdei_arch_get_entry_point(int conduit)
>  			return 0;
>  	}
>  
> +	if (scs_init_sdei()) {

... and this in init_sdei_stacks(), then I think we remove all of the code
from arch/arm64/kernel/scs.c.

> diff --git a/include/linux/scs.h b/include/linux/scs.h
> index 86e3c4b7b714..6b35a83576d4 100644
> --- a/include/linux/scs.h
> +++ b/include/linux/scs.h
> @@ -21,13 +21,11 @@
>  /* An illegal pointer value to mark the end of the shadow stack. */
>  #define SCS_END_MAGIC		(0x5f6UL + POISON_POINTER_DELTA)
>  
> -/* Allocate a static per-CPU shadow stack */
> -#define DEFINE_SCS(name)						\
> -	DEFINE_PER_CPU(unsigned long [SCS_SIZE/sizeof(long)], name)	\
> -
>  #define task_scs(tsk)		(task_thread_info(tsk)->scs_base)
>  #define task_scs_sp(tsk)	(task_thread_info(tsk)->scs_sp)
>  
> +void *scs_alloc(int node);
> +void scs_free(void *s);
>  void scs_init(void);
>  int scs_prepare(struct task_struct *tsk, int node);
>  void scs_release(struct task_struct *tsk);
> @@ -56,6 +54,8 @@ static inline bool task_scs_end_corrupted(struct task_struct *tsk)
>  
>  #else /* CONFIG_SHADOW_CALL_STACK */
>  
> +static inline void *scs_alloc(int node) { return NULL; }
> +static inline void scs_free(void *s) {}
>  static inline void scs_init(void) {}
>  static inline void scs_task_reset(struct task_struct *tsk) {}
>  static inline int scs_prepare(struct task_struct *tsk, int node) { return 0; }
> diff --git a/kernel/scs.c b/kernel/scs.c
> index 2136edba548d..8df4a92cd939 100644
> --- a/kernel/scs.c
> +++ b/kernel/scs.c
> @@ -24,7 +24,7 @@ static void __scs_account(void *s, int account)
>  #define NR_CACHED_SCS 2
>  static DEFINE_PER_CPU(void *, scs_cache[NR_CACHED_SCS]);
>  
> -static void *scs_alloc(int node)
> +void *scs_alloc(int node)
>  {
>  	int i;
>  	void *s;
> @@ -63,7 +63,7 @@ static void *scs_alloc(int node)
>  	return s;
>  }
>  
> -static void scs_free(void *s)
> +void scs_free(void *s)

Should be part of the first patch?

Will
diff mbox series

Patch

diff --git a/arch/arm64/include/asm/scs.h b/arch/arm64/include/asm/scs.h
index eaa2cd92e4c1..e9d2c3e67ff9 100644
--- a/arch/arm64/include/asm/scs.h
+++ b/arch/arm64/include/asm/scs.h
@@ -24,6 +24,25 @@ 
 	.endm
 #endif /* CONFIG_SHADOW_CALL_STACK */
 
-#endif /* __ASSEMBLY __ */
+#else /* __ASSEMBLY__ */
+
+#include <linux/scs.h>
+
+#ifdef CONFIG_SHADOW_CALL_STACK
+
+extern void scs_init_irq(void);
+
+extern void scs_free_sdei(void);
+extern int scs_init_sdei(void);
+
+#else
+
+static inline void scs_init_irq(void) {}
+static inline void scs_free_sdei(void) {}
+static inline int scs_init_sdei(void) { return -EOPNOTSUPP; }
+
+#endif
+
+#endif /* __ASSEMBLY__ */
 
 #endif /* _ASM_SCS_H */
diff --git a/arch/arm64/kernel/entry.S b/arch/arm64/kernel/entry.S
index f30007dff35f..0f76fe8142e4 100644
--- a/arch/arm64/kernel/entry.S
+++ b/arch/arm64/kernel/entry.S
@@ -438,7 +438,7 @@  SYM_CODE_END(__swpan_exit_el0)
 
 #ifdef CONFIG_SHADOW_CALL_STACK
 	/* also switch to the irq shadow stack */
-	adr_this_cpu scs_sp, irq_shadow_call_stack, x26
+	ldr_this_cpu scs_sp, irq_shadow_call_stack_ptr, x26
 #endif
 
 9998:
@@ -1094,9 +1094,9 @@  SYM_CODE_START(__sdei_asm_handler)
 #ifdef CONFIG_SHADOW_CALL_STACK
 	/* Use a separate shadow call stack for normal and critical events */
 	cbnz	w4, 3f
-	adr_this_cpu dst=scs_sp, sym=sdei_shadow_call_stack_normal, tmp=x6
+	ldr_this_cpu dst=scs_sp, sym=sdei_shadow_call_stack_normal_ptr, tmp=x6
 	b	4f
-3:	adr_this_cpu dst=scs_sp, sym=sdei_shadow_call_stack_critical, tmp=x6
+3:	ldr_this_cpu dst=scs_sp, sym=sdei_shadow_call_stack_critical_ptr, tmp=x6
 4:
 #endif
 
diff --git a/arch/arm64/kernel/irq.c b/arch/arm64/kernel/irq.c
index 9cf2fb87584a..54ba3725bc0e 100644
--- a/arch/arm64/kernel/irq.c
+++ b/arch/arm64/kernel/irq.c
@@ -20,6 +20,7 @@ 
 #include <linux/seq_file.h>
 #include <linux/vmalloc.h>
 #include <asm/daifflags.h>
+#include <asm/scs.h>
 #include <asm/vmap_stack.h>
 
 /* Only access this in an NMI enter/exit */
@@ -54,6 +55,7 @@  static void init_irq_stacks(void)
 void __init init_IRQ(void)
 {
 	init_irq_stacks();
+	scs_init_irq();
 	irqchip_init();
 	if (!handle_arch_irq)
 		panic("No interrupt controller found.");
diff --git a/arch/arm64/kernel/scs.c b/arch/arm64/kernel/scs.c
index e8f7ff45dd8f..f85cebf8122a 100644
--- a/arch/arm64/kernel/scs.c
+++ b/arch/arm64/kernel/scs.c
@@ -6,11 +6,70 @@ 
  */
 
 #include <linux/percpu.h>
-#include <linux/scs.h>
+#include <asm/scs.h>
 
-DEFINE_SCS(irq_shadow_call_stack);
+DEFINE_PER_CPU(unsigned long *, irq_shadow_call_stack_ptr);
+
+DECLARE_PER_CPU(unsigned long *, sdei_shadow_call_stack_normal_ptr);
+DECLARE_PER_CPU(unsigned long *, sdei_shadow_call_stack_critical_ptr);
 
 #ifdef CONFIG_ARM_SDE_INTERFACE
-DEFINE_SCS(sdei_shadow_call_stack_normal);
-DEFINE_SCS(sdei_shadow_call_stack_critical);
+DEFINE_PER_CPU(unsigned long *, sdei_shadow_call_stack_normal_ptr);
+DEFINE_PER_CPU(unsigned long *, sdei_shadow_call_stack_critical_ptr);
 #endif
+
+void scs_init_irq(void)
+{
+	int cpu;
+
+	for_each_possible_cpu(cpu)
+		per_cpu(irq_shadow_call_stack_ptr, cpu) =
+			scs_alloc(cpu_to_node(cpu));
+}
+
+
+void scs_free_sdei(void)
+{
+	int cpu;
+	void *s;
+
+	if (!IS_ENABLED(CONFIG_ARM_SDE_INTERFACE))
+		return;
+
+	for_each_possible_cpu(cpu) {
+		s = per_cpu(sdei_shadow_call_stack_normal_ptr, cpu);
+		if (s)
+			scs_free(s);
+
+		s = per_cpu(sdei_shadow_call_stack_critical_ptr, cpu);
+		if (s)
+			scs_free(s);
+	}
+}
+
+int scs_init_sdei(void)
+{
+	int cpu;
+	void *s;
+
+	if (!IS_ENABLED(CONFIG_ARM_SDE_INTERFACE))
+		return 0;
+
+	for_each_possible_cpu(cpu) {
+		s = scs_alloc(cpu_to_node(cpu));
+		if (!s)
+			goto err;
+		per_cpu(sdei_shadow_call_stack_normal_ptr, cpu) = s;
+
+		s = scs_alloc(cpu_to_node(cpu));
+		if (!s)
+			goto err;
+		per_cpu(sdei_shadow_call_stack_critical_ptr, cpu) = s;
+	}
+
+	return 0;
+
+err:
+	scs_free_sdei();
+	return -ENOMEM;
+}
diff --git a/arch/arm64/kernel/sdei.c b/arch/arm64/kernel/sdei.c
index 7689f2031c0c..04519a7cb51d 100644
--- a/arch/arm64/kernel/sdei.c
+++ b/arch/arm64/kernel/sdei.c
@@ -13,6 +13,7 @@ 
 #include <asm/kprobes.h>
 #include <asm/mmu.h>
 #include <asm/ptrace.h>
+#include <asm/scs.h>
 #include <asm/sections.h>
 #include <asm/stacktrace.h>
 #include <asm/sysreg.h>
@@ -138,6 +139,12 @@  unsigned long sdei_arch_get_entry_point(int conduit)
 			return 0;
 	}
 
+	if (scs_init_sdei()) {
+		if (IS_ENABLED(CONFIG_VMAP_STACK))
+			free_sdei_stacks();
+		return 0;
+	}
+
 	sdei_exit_mode = (conduit == SMCCC_CONDUIT_HVC) ? SDEI_EXIT_HVC : SDEI_EXIT_SMC;
 
 #ifdef CONFIG_UNMAP_KERNEL_AT_EL0
diff --git a/include/linux/scs.h b/include/linux/scs.h
index 86e3c4b7b714..6b35a83576d4 100644
--- a/include/linux/scs.h
+++ b/include/linux/scs.h
@@ -21,13 +21,11 @@ 
 /* An illegal pointer value to mark the end of the shadow stack. */
 #define SCS_END_MAGIC		(0x5f6UL + POISON_POINTER_DELTA)
 
-/* Allocate a static per-CPU shadow stack */
-#define DEFINE_SCS(name)						\
-	DEFINE_PER_CPU(unsigned long [SCS_SIZE/sizeof(long)], name)	\
-
 #define task_scs(tsk)		(task_thread_info(tsk)->scs_base)
 #define task_scs_sp(tsk)	(task_thread_info(tsk)->scs_sp)
 
+void *scs_alloc(int node);
+void scs_free(void *s);
 void scs_init(void);
 int scs_prepare(struct task_struct *tsk, int node);
 void scs_release(struct task_struct *tsk);
@@ -56,6 +54,8 @@  static inline bool task_scs_end_corrupted(struct task_struct *tsk)
 
 #else /* CONFIG_SHADOW_CALL_STACK */
 
+static inline void *scs_alloc(int node) { return NULL; }
+static inline void scs_free(void *s) {}
 static inline void scs_init(void) {}
 static inline void scs_task_reset(struct task_struct *tsk) {}
 static inline int scs_prepare(struct task_struct *tsk, int node) { return 0; }
diff --git a/kernel/scs.c b/kernel/scs.c
index 2136edba548d..8df4a92cd939 100644
--- a/kernel/scs.c
+++ b/kernel/scs.c
@@ -24,7 +24,7 @@  static void __scs_account(void *s, int account)
 #define NR_CACHED_SCS 2
 static DEFINE_PER_CPU(void *, scs_cache[NR_CACHED_SCS]);
 
-static void *scs_alloc(int node)
+void *scs_alloc(int node)
 {
 	int i;
 	void *s;
@@ -63,7 +63,7 @@  static void *scs_alloc(int node)
 	return s;
 }
 
-static void scs_free(void *s)
+void scs_free(void *s)
 {
 	int i;