new file mode 100644
@@ -0,0 +1,32 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * arch/arm/include/asm/kasan.h
+ *
+ * Copyright (c) 2015 Samsung Electronics Co., Ltd.
+ * Author: Andrey Ryabinin <ryabinin.a.a@gmail.com>
+ *
+ */
+
+#ifndef __ASM_KASAN_H
+#define __ASM_KASAN_H
+
+#ifdef CONFIG_KASAN
+
+#include <asm/kasan_def.h>
+
+#define KASAN_SHADOW_SCALE_SHIFT 3
+
+/*
+ * The compiler uses a shadow offset assuming that addresses start
+ * from 0. Kernel addresses don't start from 0, so shadow
+ * for kernel really starts from 'compiler's shadow offset' +
+ * ('kernel address space start' >> KASAN_SHADOW_SCALE_SHIFT)
+ */
+
+extern void kasan_init(void);
+
+#else
+static inline void kasan_init(void) { }
+#endif
+
+#endif
@@ -21,6 +21,7 @@
#define _PAGE_KERNEL_TABLE (PMD_TYPE_TABLE | PMD_BIT4 | PMD_DOMAIN(DOMAIN_KERNEL))
#ifdef CONFIG_ARM_LPAE
+#define PGD_SIZE (PTRS_PER_PGD * sizeof(pgd_t))
static inline pmd_t *pmd_alloc_one(struct mm_struct *mm, unsigned long addr)
{
@@ -39,14 +40,18 @@ static inline void pud_populate(struct mm_struct *mm, pud_t *pud, pmd_t *pmd)
}
#else /* !CONFIG_ARM_LPAE */
+#define PGD_SIZE (PAGE_SIZE << 2)
/*
* Since we have only two-level page tables, these are trivial
*/
#define pmd_alloc_one(mm,addr) ({ BUG(); ((pmd_t *)2); })
#define pmd_free(mm, pmd) do { } while (0)
-#define pud_populate(mm,pmd,pte) BUG()
-
+#ifndef CONFIG_KASAN
+#define pud_populate(mm, pmd, pte) BUG()
+#else
+#define pud_populate(mm, pmd, pte) do { } while (0)
+#endif
#endif /* CONFIG_ARM_LPAE */
extern pgd_t *pgd_alloc(struct mm_struct *mm);
@@ -13,7 +13,11 @@
#include <asm/fpstate.h>
#include <asm/page.h>
+#ifdef CONFIG_KASAN
+#define THREAD_SIZE_ORDER 2
+#else
#define THREAD_SIZE_ORDER 1
+#endif
#define THREAD_SIZE (PAGE_SIZE << THREAD_SIZE_ORDER)
#define THREAD_START_SP (THREAD_SIZE - 8)
@@ -111,6 +111,9 @@ __mmap_switched:
str r8, [r2] @ Save atags pointer
cmp r3, #0
strne r10, [r3] @ Save control register values
+#ifdef CONFIG_KASAN
+ bl kasan_early_init
+#endif
mov lr, #0
b start_kernel
ENDPROC(__mmap_switched)
@@ -58,6 +58,7 @@
#include <asm/unwind.h>
#include <asm/memblock.h>
#include <asm/virt.h>
+#include <asm/kasan.h>
#include "atags.h"
@@ -1130,6 +1131,7 @@ void __init setup_arch(char **cmdline_p)
early_ioremap_reset();
paging_init(mdesc);
+ kasan_init();
request_standard_resources(mdesc);
if (mdesc->restart)
@@ -113,3 +113,6 @@ obj-$(CONFIG_CACHE_L2X0_PMU) += cache-l2x0-pmu.o
obj-$(CONFIG_CACHE_XSC3L2) += cache-xsc3l2.o
obj-$(CONFIG_CACHE_TAUROS2) += cache-tauros2.o
obj-$(CONFIG_CACHE_UNIPHIER) += cache-uniphier.o
+
+KASAN_SANITIZE_kasan_init.o := n
+obj-$(CONFIG_KASAN) += kasan_init.o
new file mode 100644
@@ -0,0 +1,294 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * This file contains kasan initialization code for ARM.
+ *
+ * Copyright (c) 2018 Samsung Electronics Co., Ltd.
+ * Author: Andrey Ryabinin <ryabinin.a.a@gmail.com>
+ */
+
+#define pr_fmt(fmt) "kasan: " fmt
+#include <linux/kasan.h>
+#include <linux/kernel.h>
+#include <linux/memblock.h>
+#include <linux/sched/task.h>
+#include <linux/start_kernel.h>
+#include <asm/cputype.h>
+#include <asm/highmem.h>
+#include <asm/mach/map.h>
+#include <asm/memory.h>
+#include <asm/page.h>
+#include <asm/pgalloc.h>
+#include <asm/pgtable.h>
+#include <asm/procinfo.h>
+#include <asm/proc-fns.h>
+
+#include "mm.h"
+
+static pgd_t tmp_pgd_table[PTRS_PER_PGD] __initdata __aligned(PGD_SIZE);
+
+pmd_t tmp_pmd_table[PTRS_PER_PMD] __page_aligned_bss;
+
+static __init void *kasan_alloc_block(size_t size, int node)
+{
+ return memblock_alloc_try_nid(size, size, __pa(MAX_DMA_ADDRESS),
+ MEMBLOCK_ALLOC_KASAN, node);
+}
+
+static void __init kasan_early_pmd_populate(unsigned long start,
+ unsigned long end, pud_t *pud)
+{
+ unsigned long addr;
+ unsigned long next;
+ pmd_t *pmd;
+
+ pmd = pmd_offset(pud, start);
+ for (addr = start; addr < end;) {
+ pmd_populate_kernel(&init_mm, pmd, kasan_early_shadow_pte);
+ next = pmd_addr_end(addr, end);
+ addr = next;
+ flush_pmd_entry(pmd);
+ pmd++;
+ }
+}
+
+static void __init kasan_early_pud_populate(unsigned long start,
+ unsigned long end, pgd_t *pgd)
+{
+ unsigned long addr;
+ unsigned long next;
+ pud_t *pud;
+
+ pud = pud_offset(pgd, start);
+ for (addr = start; addr < end;) {
+ next = pud_addr_end(addr, end);
+ kasan_early_pmd_populate(addr, next, pud);
+ addr = next;
+ pud++;
+ }
+}
+
+void __init kasan_map_early_shadow(pgd_t *pgdp)
+{
+ int i;
+ unsigned long start = KASAN_SHADOW_START;
+ unsigned long end = KASAN_SHADOW_END;
+ unsigned long addr;
+ unsigned long next;
+ pgd_t *pgd;
+
+ for (i = 0; i < PTRS_PER_PTE; i++)
+ set_pte_at(&init_mm, KASAN_SHADOW_START + i*PAGE_SIZE,
+ &kasan_early_shadow_pte[i], pfn_pte(
+ virt_to_pfn(kasan_early_shadow_page),
+ __pgprot(_L_PTE_DEFAULT | L_PTE_DIRTY
+ | L_PTE_XN)));
+
+ pgd = pgd_offset_k(start);
+ for (addr = start; addr < end;) {
+ next = pgd_addr_end(addr, end);
+ kasan_early_pud_populate(addr, next, pgd);
+ addr = next;
+ pgd++;
+ }
+}
+
+extern struct proc_info_list *lookup_processor_type(unsigned int);
+
+void __init kasan_early_init(void)
+{
+ struct proc_info_list *list;
+
+ /*
+ * locate processor in the list of supported processor
+ * types. The linker builds this table for us from the
+ * entries in arch/arm/mm/proc-*.S
+ */
+ list = lookup_processor_type(read_cpuid_id());
+ if (list) {
+#ifdef MULTI_CPU
+ processor = *list->proc;
+#endif
+ }
+
+ BUILD_BUG_ON((KASAN_SHADOW_END - (1UL << 29)) != KASAN_SHADOW_OFFSET);
+ kasan_map_early_shadow(swapper_pg_dir);
+}
+
+static void __init clear_pgds(unsigned long start,
+ unsigned long end)
+{
+ for (; start && start < end; start += PMD_SIZE)
+ pmd_clear(pmd_off_k(start));
+}
+
+pte_t * __init kasan_pte_populate(pmd_t *pmd, unsigned long addr, int node)
+{
+ pte_t *pte = pte_offset_kernel(pmd, addr);
+
+ if (pte_none(*pte)) {
+ pte_t entry;
+ void *p = kasan_alloc_block(PAGE_SIZE, node);
+
+ if (!p)
+ return NULL;
+ entry = pfn_pte(virt_to_pfn(p),
+ __pgprot(pgprot_val(PAGE_KERNEL)));
+ set_pte_at(&init_mm, addr, pte, entry);
+ }
+ return pte;
+}
+
+pmd_t * __init kasan_pmd_populate(pud_t *pud, unsigned long addr, int node)
+{
+ pmd_t *pmd = pmd_offset(pud, addr);
+
+ if (pmd_none(*pmd)) {
+ void *p = kasan_alloc_block(PAGE_SIZE, node);
+
+ if (!p)
+ return NULL;
+ pmd_populate_kernel(&init_mm, pmd, p);
+ }
+ return pmd;
+}
+
+pud_t * __init kasan_pud_populate(pgd_t *pgd, unsigned long addr, int node)
+{
+ pud_t *pud = pud_offset(pgd, addr);
+
+ /*
+ * On non-LPAE systems using just 2-level page tables pud_none()
+ * will alwats be zero and we bail out of here.
+ */
+ if (pud_none(*pud)) {
+ void *p = kasan_alloc_block(PAGE_SIZE, node);
+
+ if (!p)
+ return NULL;
+ pr_err("populating pud addr %lx\n", addr);
+ pud_populate(&init_mm, pud, p);
+ }
+ return pud;
+}
+
+pgd_t * __init kasan_pgd_populate(unsigned long addr, int node)
+{
+ pgd_t *pgd = pgd_offset_k(addr);
+
+ if (pgd_none(*pgd)) {
+ void *p = kasan_alloc_block(PAGE_SIZE, node);
+
+ if (!p)
+ return NULL;
+ pgd_populate(&init_mm, pgd, p);
+ }
+ return pgd;
+}
+
+static int __init create_mapping(unsigned long start, unsigned long end,
+ int node)
+{
+ unsigned long addr = start;
+ pgd_t *pgd;
+ pud_t *pud;
+ pmd_t *pmd;
+ pte_t *pte;
+
+ pr_info("populating shadow for %lx, %lx\n", start, end);
+
+ for (; addr < end; addr += PAGE_SIZE) {
+ pgd = kasan_pgd_populate(addr, node);
+ if (!pgd)
+ return -ENOMEM;
+
+ pud = kasan_pud_populate(pgd, addr, node);
+ if (!pud)
+ return -ENOMEM;
+
+ pmd = kasan_pmd_populate(pud, addr, node);
+ if (!pmd)
+ return -ENOMEM;
+
+ pte = kasan_pte_populate(pmd, addr, node);
+ if (!pte)
+ return -ENOMEM;
+ }
+ return 0;
+}
+
+void __init kasan_init(void)
+{
+ struct memblock_region *reg;
+ int i;
+
+ /*
+ * We are going to perform proper setup of shadow memory.
+ *
+ * At first we should unmap early shadow (clear_pgds() call bellow).
+ * However, instrumented code couldn't execute without shadow memory.
+ *
+ * To keep the early shadow memory MMU tables around while setting up
+ * the proper shadow memory, we copy swapper_pg_dir (the initial page
+ * table) to tmp_pgd_table and use that to keep the early shadow memory
+ * mapped until the full shadow setup is finished. Then we swap back
+ * to the proper swapper_pg_dir.
+ */
+#ifdef CONFIG_ARM_LPAE
+ memcpy(tmp_pmd_table,
+ pgd_page_vaddr(*pgd_offset_k(KASAN_SHADOW_START)),
+ sizeof(tmp_pmd_table));
+ memcpy(tmp_pgd_table, swapper_pg_dir, sizeof(tmp_pgd_table));
+ set_pgd(&tmp_pgd_table[pgd_index(KASAN_SHADOW_START)],
+ __pgd(__pa(tmp_pmd_table) | PMD_TYPE_TABLE | L_PGD_SWAPPER));
+ cpu_switch_mm(tmp_pgd_table, &init_mm);
+#else
+ memcpy(tmp_pgd_table, swapper_pg_dir, sizeof(tmp_pgd_table));
+ cpu_switch_mm(tmp_pgd_table, &init_mm);
+#endif
+ clear_pgds(KASAN_SHADOW_START, KASAN_SHADOW_END);
+
+ kasan_populate_early_shadow(kasan_mem_to_shadow((void *)VMALLOC_START),
+ kasan_mem_to_shadow((void *)-1UL) + 1);
+
+ for_each_memblock(memory, reg) {
+ void *start = __va(reg->base);
+ void *end = __va(reg->base + reg->size);
+
+ if (reg->base + reg->size > arm_lowmem_limit)
+ end = __va(arm_lowmem_limit);
+ if (start >= end)
+ break;
+
+ create_mapping((unsigned long)kasan_mem_to_shadow(start),
+ (unsigned long)kasan_mem_to_shadow(end),
+ NUMA_NO_NODE);
+ }
+
+ /*
+ * 1. The module global variables are in MODULES_VADDR ~ MODULES_END,
+ * so we need to map this area.
+ * 2. PKMAP_BASE ~ PKMAP_BASE+PMD_SIZE's shadow and MODULES_VADDR
+ * ~ MODULES_END's shadow is in the same PMD_SIZE, so we can't
+ * use kasan_populate_zero_shadow.
+ */
+ create_mapping(
+ (unsigned long)kasan_mem_to_shadow((void *)MODULES_VADDR),
+ (unsigned long)kasan_mem_to_shadow((void *)(PKMAP_BASE +
+ PMD_SIZE)),
+ NUMA_NO_NODE);
+
+ /*
+ * KAsan may reuse the contents of kasan_early_shadow_pte directly, so
+ * we should make sure that it maps the zero page read-only.
+ */
+ for (i = 0; i < PTRS_PER_PTE; i++)
+ set_pte_at(&init_mm, KASAN_SHADOW_START + i*PAGE_SIZE,
+ &kasan_early_shadow_pte[i],
+ pfn_pte(virt_to_pfn(kasan_early_shadow_page),
+ __pgprot(pgprot_val(PAGE_KERNEL)
+ | L_PTE_RDONLY)));
+ memset(kasan_early_shadow_page, 0, PAGE_SIZE);
+ cpu_switch_mm(swapper_pg_dir, &init_mm);
+ pr_info("Kernel address sanitizer initialized\n");
+ init_task.kasan_depth = 0;
+}
@@ -61,7 +61,20 @@ pgd_t *pgd_alloc(struct mm_struct *mm)
new_pmd = pmd_alloc(mm, new_pud, 0);
if (!new_pmd)
goto no_pmd;
-#endif
+#ifdef CONFIG_KASAN
+ /*
+ * Copy PMD table for KASAN shadow mappings.
+ */
+ init_pgd = pgd_offset_k(TASK_SIZE);
+ init_pud = pud_offset(init_pgd, TASK_SIZE);
+ init_pmd = pmd_offset(init_pud, TASK_SIZE);
+ new_pmd = pmd_offset(new_pud, TASK_SIZE);
+ memcpy(new_pmd, init_pmd,
+ (pmd_index(MODULES_VADDR) - pmd_index(TASK_SIZE))
+ * sizeof(pmd_t));
+ clean_dcache_area(new_pmd, PTRS_PER_PMD * sizeof(pmd_t));
+#endif /* CONFIG_KASAN */
+#endif /* CONFIG_LPAE */
if (!vectors_high()) {
/*