@@ -19,28 +19,13 @@
* 64K (section size = 512M).
*/
#ifdef CONFIG_ARM64_4K_PAGES
-#define ARM64_KERNEL_USES_PMD_MAPS 1
+#define INIT_IDMAP_USES_PMD_MAPS 1
+#define INIT_IDMAP_TABLE_LEVELS (CONFIG_PGTABLE_LEVELS - 1)
#else
-#define ARM64_KERNEL_USES_PMD_MAPS 0
+#define INIT_IDMAP_USES_PMD_MAPS 0
+#define INIT_IDMAP_TABLE_LEVELS (CONFIG_PGTABLE_LEVELS)
#endif
-/*
- * The idmap and swapper page tables need some space reserved in the kernel
- * image. Both require pgd, pud (4 levels only) and pmd tables to (section)
- * map the kernel. With the 64K page configuration, swapper and idmap need to
- * map to pte level. The swapper also maps the FDT (see __create_page_tables
- * for more information). Note that the number of ID map translation levels
- * could be increased on the fly if system RAM is out of reach for the default
- * VA range, so pages required to map highest possible PA are reserved in all
- * cases.
- */
-#if ARM64_KERNEL_USES_PMD_MAPS
-#define SWAPPER_PGTABLE_LEVELS (CONFIG_PGTABLE_LEVELS - 1)
-#else
-#define SWAPPER_PGTABLE_LEVELS (CONFIG_PGTABLE_LEVELS)
-#endif
-
-
/*
* If KASLR is enabled, then an offset K is added to the kernel address
* space. The bottom 21 bits of this offset are zero to guarantee 2MB
@@ -69,14 +54,14 @@
#define EARLY_PGDS(vstart, vend, add) (EARLY_ENTRIES(vstart, vend, PGDIR_SHIFT, add))
-#if SWAPPER_PGTABLE_LEVELS > 3
+#if INIT_IDMAP_TABLE_LEVELS > 3
#define EARLY_PUDS(vstart, vend, add) (EARLY_ENTRIES(vstart, vend, PUD_SHIFT, add))
#else
#define EARLY_PUDS(vstart, vend, add) (0)
#endif
-#if SWAPPER_PGTABLE_LEVELS > 2
-#define EARLY_PMDS(vstart, vend, add) (EARLY_ENTRIES(vstart, vend, SWAPPER_TABLE_SHIFT, add))
+#if INIT_IDMAP_TABLE_LEVELS > 2
+#define EARLY_PMDS(vstart, vend, add) (EARLY_ENTRIES(vstart, vend, INIT_IDMAP_TABLE_SHIFT, add))
#else
#define EARLY_PMDS(vstart, vend, add) (0)
#endif
@@ -93,23 +78,23 @@
#else
#define INIT_IDMAP_DIR_SIZE (INIT_IDMAP_DIR_PAGES * PAGE_SIZE)
#endif
-#define INIT_IDMAP_DIR_PAGES EARLY_PAGES(KIMAGE_VADDR, _end + MAX_FDT_SIZE + SWAPPER_BLOCK_SIZE, 1)
+#define INIT_IDMAP_DIR_PAGES EARLY_PAGES(KIMAGE_VADDR, _end + MAX_FDT_SIZE + INIT_IDMAP_BLOCK_SIZE, 1)
/* Initial memory map size */
-#if ARM64_KERNEL_USES_PMD_MAPS
-#define SWAPPER_BLOCK_SHIFT PMD_SHIFT
-#define SWAPPER_BLOCK_SIZE PMD_SIZE
-#define SWAPPER_TABLE_SHIFT PUD_SHIFT
+#if INIT_IDMAP_USES_PMD_MAPS
+#define INIT_IDMAP_BLOCK_SHIFT PMD_SHIFT
+#define INIT_IDMAP_BLOCK_SIZE PMD_SIZE
+#define INIT_IDMAP_TABLE_SHIFT PUD_SHIFT
#else
-#define SWAPPER_BLOCK_SHIFT PAGE_SHIFT
-#define SWAPPER_BLOCK_SIZE PAGE_SIZE
-#define SWAPPER_TABLE_SHIFT PMD_SHIFT
+#define INIT_IDMAP_BLOCK_SHIFT PAGE_SHIFT
+#define INIT_IDMAP_BLOCK_SIZE PAGE_SIZE
+#define INIT_IDMAP_TABLE_SHIFT PMD_SHIFT
#endif
/* The number of segments in the kernel image (text, rodata, inittext, initdata, data+bss) */
#define KERNEL_SEGMENT_COUNT 5
-#if SWAPPER_BLOCK_SIZE > SEGMENT_ALIGN
+#if INIT_IDMAP_BLOCK_SIZE > SEGMENT_ALIGN
#define EARLY_SEGMENT_EXTRA_PAGES (KERNEL_SEGMENT_COUNT + 1)
#else
#define EARLY_SEGMENT_EXTRA_PAGES 0
@@ -118,15 +103,12 @@
/*
* Initial memory map attributes.
*/
-#define SWAPPER_PTE_FLAGS (PTE_TYPE_PAGE | PTE_AF | PTE_SHARED)
-#define SWAPPER_PMD_FLAGS (PMD_TYPE_SECT | PMD_SECT_AF | PMD_SECT_S)
-
-#if ARM64_KERNEL_USES_PMD_MAPS
-#define SWAPPER_RW_MMUFLAGS (PMD_ATTRINDX(MT_NORMAL) | SWAPPER_PMD_FLAGS)
-#define SWAPPER_RX_MMUFLAGS (SWAPPER_RW_MMUFLAGS | PMD_SECT_RDONLY)
+#if INIT_IDMAP_USES_PMD_MAPS
+#define INIT_IDMAP_RW_MMUFLAGS (PMD_ATTRINDX(MT_NORMAL) | PMD_TYPE_SECT | PMD_SECT_AF | PMD_SECT_S)
+#define INIT_IDMAP_RX_MMUFLAGS (INIT_IDMAP_RW_MMUFLAGS | PMD_SECT_RDONLY)
#else
-#define SWAPPER_RW_MMUFLAGS (PTE_ATTRINDX(MT_NORMAL) | SWAPPER_PTE_FLAGS)
-#define SWAPPER_RX_MMUFLAGS (SWAPPER_RW_MMUFLAGS | PTE_RDONLY)
+#define INIT_IDMAP_RW_MMUFLAGS (PTE_ATTRINDX(MT_NORMAL) | PTE_TYPE_PAGE | PTE_AF | PTE_SHARED)
+#define INIT_IDMAP_RX_MMUFLAGS (INIT_IDMAP_RW_MMUFLAGS | PTE_RDONLY)
#endif
/*
@@ -214,23 +214,23 @@ SYM_CODE_END(preserve_boot_args)
populate_entries \tbl, \rtbl, \istart, \iend, #PMD_TYPE_TABLE, #PAGE_SIZE, \tmp
mov \tbl, \sv
-#if SWAPPER_PGTABLE_LEVELS > 3
+#if INIT_IDMAP_TABLE_LEVELS > 3
compute_indices \vstart, \vend, #PUD_SHIFT, #(PAGE_SHIFT - 3), \istart, \iend, \count
mov \sv, \rtbl
populate_entries \tbl, \rtbl, \istart, \iend, #PMD_TYPE_TABLE, #PAGE_SIZE, \tmp
mov \tbl, \sv
#endif
-#if SWAPPER_PGTABLE_LEVELS > 2
- compute_indices \vstart, \vend, #SWAPPER_TABLE_SHIFT, #(PAGE_SHIFT - 3), \istart, \iend, \count
+#if INIT_IDMAP_TABLE_LEVELS > 2
+ compute_indices \vstart, \vend, #INIT_IDMAP_TABLE_SHIFT, #(PAGE_SHIFT - 3), \istart, \iend, \count
mov \sv, \rtbl
populate_entries \tbl, \rtbl, \istart, \iend, #PMD_TYPE_TABLE, #PAGE_SIZE, \tmp
mov \tbl, \sv
#endif
- compute_indices \vstart, \vend, #SWAPPER_BLOCK_SHIFT, #(PAGE_SHIFT - 3), \istart, \iend, \count
- bic \rtbl, \phys, #SWAPPER_BLOCK_SIZE - 1
- populate_entries \tbl, \rtbl, \istart, \iend, \flags, #SWAPPER_BLOCK_SIZE, \tmp
+ compute_indices \vstart, \vend, #INIT_IDMAP_BLOCK_SHIFT, #(PAGE_SHIFT - 3), \istart, \iend, \count
+ bic \rtbl, \phys, #INIT_IDMAP_BLOCK_SIZE - 1
+ populate_entries \tbl, \rtbl, \istart, \iend, \flags, #INIT_IDMAP_BLOCK_SIZE, \tmp
.endm
/*
@@ -317,8 +317,8 @@ SYM_FUNC_START_LOCAL(create_idmap)
#endif
adrp x0, init_idmap_pg_dir
adrp x3, _text
- adrp x6, _end + MAX_FDT_SIZE + SWAPPER_BLOCK_SIZE
- mov x7, SWAPPER_RX_MMUFLAGS
+ adrp x6, _end + MAX_FDT_SIZE + INIT_IDMAP_BLOCK_SIZE
+ mov x7, INIT_IDMAP_RX_MMUFLAGS
map_memory x0, x1, x3, x6, x7, x3, IDMAP_PGD_ORDER, x10, x11, x12, x13, x14, EXTRA_SHIFT
@@ -326,20 +326,20 @@ SYM_FUNC_START_LOCAL(create_idmap)
adrp x1, _text
adrp x2, __bss_start
adrp x3, _end
- bic x4, x2, #SWAPPER_BLOCK_SIZE - 1
- mov x5, SWAPPER_RW_MMUFLAGS
- mov x6, #SWAPPER_BLOCK_SHIFT
+ bic x4, x2, #INIT_IDMAP_BLOCK_SIZE - 1
+ mov x5, INIT_IDMAP_RW_MMUFLAGS
+ mov x6, #INIT_IDMAP_BLOCK_SHIFT
bl remap_region
/* Remap the FDT after the kernel image */
adrp x1, _text
- adrp x22, _end + SWAPPER_BLOCK_SIZE
- bic x2, x22, #SWAPPER_BLOCK_SIZE - 1
- bfi x22, x21, #0, #SWAPPER_BLOCK_SHIFT // remapped FDT address
- add x3, x2, #MAX_FDT_SIZE + SWAPPER_BLOCK_SIZE
- bic x4, x21, #SWAPPER_BLOCK_SIZE - 1
- mov x5, SWAPPER_RW_MMUFLAGS
- mov x6, #SWAPPER_BLOCK_SHIFT
+ adrp x22, _end + INIT_IDMAP_BLOCK_SIZE
+ bic x2, x22, #INIT_IDMAP_BLOCK_SIZE - 1
+ bfi x22, x21, #0, #INIT_IDMAP_BLOCK_SHIFT // remapped FDT address
+ add x3, x2, #MAX_FDT_SIZE + INIT_IDMAP_BLOCK_SIZE
+ bic x4, x21, #INIT_IDMAP_BLOCK_SIZE - 1
+ mov x5, INIT_IDMAP_RW_MMUFLAGS
+ mov x6, #INIT_IDMAP_BLOCK_SHIFT
bl remap_region
/*
@@ -206,7 +206,7 @@ SYM_FUNC_ALIAS(__pi_idmap_cpu_replace_ttbr1, idmap_cpu_replace_ttbr1)
#ifdef CONFIG_UNMAP_KERNEL_AT_EL0
-#define KPTI_NG_PTE_FLAGS (PTE_ATTRINDX(MT_NORMAL) | SWAPPER_PTE_FLAGS)
+#define KPTI_NG_PTE_FLAGS (PTE_ATTRINDX(MT_NORMAL) | PTE_TYPE_PAGE | PTE_AF | PTE_SHARED)
.pushsection ".idmap.text", "awx"
Initially, the kernel mapping as well as the ID map used block mappings on 4k pagesize configurations, but this hasn't been the case for a long time. Currently, only the initial ID map uses the larger granularity, to simplify the early mapping code, which is implemented in assembler. The permanent ID map as well as the kernel mapping (which is now created only once) always map the kernel down to pages. This means the SWAPPER_BLOCK_xxx and related constants are no longer named appropriately, so let's rename them to INIT_IDMAP_BLOCK_xxx instead. Get rid of a stale comment while at it. Signed-off-by: Ard Biesheuvel <ardb@kernel.org> --- arch/arm64/include/asm/kernel-pgtable.h | 60 +++++++------------- arch/arm64/kernel/head.S | 36 ++++++------ arch/arm64/mm/proc.S | 2 +- 3 files changed, 40 insertions(+), 58 deletions(-)