@@ -83,4 +83,11 @@ pmd_populate(struct mm_struct *mm, pmd_t *pmdp, pgtable_t ptep)
}
#define pmd_pgtable(pmd) pmd_page(pmd)
+extern void __create_pgd_mapping_extend(pgd_t *pgdir,
+ unsigned int entries_cnt, phys_addr_t phys,
+ unsigned long virt, phys_addr_t size,
+ pgprot_t prot,
+ phys_addr_t (*pgtable_alloc)(int),
+ int flags);
+
#endif
@@ -341,6 +341,9 @@ SYM_FUNC_START_LOCAL(__create_page_tables)
#if VA_BITS != EXTRA_SHIFT
#error "Mismatch between VA_BITS and page size/number of translation levels"
#endif
+ adr_l x4, idmap_extend_pgtable
+ mov x5, #1
+ str x5, [x4] //require expanded pagetable
mov x4, EXTRA_PTRS
create_table_entry x0, x3, EXTRA_SHIFT, x4, x5, x6
@@ -21,13 +21,17 @@
#include "./mmu_include.c"
-void __create_pgd_mapping_extend(pgd_t *pgdir, phys_addr_t phys,
- unsigned long virt, phys_addr_t size,
- pgprot_t prot,
- phys_addr_t (*pgtable_alloc)(int),
- int flags)
+void __create_pgd_mapping_extend(pgd_t *pgdir,
+ unsigned int entries_cnt,
+ phys_addr_t phys,
+ unsigned long virt,
+ phys_addr_t size,
+ pgprot_t prot,
+ phys_addr_t (*pgtable_alloc)(int),
+ int flags)
{
- __create_pgd_mapping(pgdir, phys, virt, size, prot, pgtable_alloc, flags);
+ __create_pgd_mapping(pgdir, entries_cnt, phys, virt, size, prot,
+ pgtable_alloc, flags);
}
#endif
@@ -145,6 +145,33 @@ static phys_addr_t pgd_pgtable_alloc(int shift)
#include "./mmu_include.c"
+int idmap_extend_pgtable;
+
+/*
+ * lock: no lock protection alongside this function call
+ * todo: tear down idmap. (no requirement at present)
+ */
+void create_idmap(pgd_t *pgdir, phys_addr_t phys,
+ phys_addr_t size,
+ pgprot_t prot,
+ phys_addr_t (*pgtable_alloc)(int),
+ int flags)
+{
+ u64 ptrs_per_pgd = idmap_ptrs_per_pgd;
+
+#ifdef CONFIG_IDMAP_PGTABLE_EXPAND
+ if (idmap_extend_pgtable)
+ __create_pgd_mapping_extend(pgdir, ptrs_per_pgd,
+ phys, phys, size, prot, pgtable_alloc, flags);
+ else
+ __create_pgd_mapping(pgdir, ptrs_per_pgd,
+ phys, phys, size, prot, pgtable_alloc, flags);
+#else
+ __create_pgd_mapping(pgdir, ptrs_per_pgd,
+ phys, phys, size, prot, pgtable_alloc, flags);
+#endif
+}
+
/*
* This function can only be used to modify existing table entries,
* without allocating new levels of table. Note that this permits the
@@ -158,7 +185,7 @@ static void __init create_mapping_noalloc(phys_addr_t phys, unsigned long virt,
&phys, virt);
return;
}
- __create_pgd_mapping(init_mm.pgd, phys, virt, size, prot, NULL,
+ __create_pgd_mapping(init_mm.pgd, PTRS_PER_PGD, phys, virt, size, prot, NULL,
NO_CONT_MAPPINGS);
}
@@ -173,7 +200,7 @@ void __init create_pgd_mapping(struct mm_struct *mm, phys_addr_t phys,
if (page_mappings_only)
flags = NO_BLOCK_MAPPINGS | NO_CONT_MAPPINGS;
- __create_pgd_mapping(mm->pgd, phys, virt, size, prot,
+ __create_pgd_mapping(mm->pgd, PTRS_PER_PGD, phys, virt, size, prot,
pgd_pgtable_alloc, flags);
}
@@ -186,7 +213,7 @@ static void update_mapping_prot(phys_addr_t phys, unsigned long virt,
return;
}
- __create_pgd_mapping(init_mm.pgd, phys, virt, size, prot, NULL,
+ __create_pgd_mapping(init_mm.pgd, PTRS_PER_PGD, phys, virt, size, prot, NULL,
NO_CONT_MAPPINGS);
/* flush the TLBs after updating live kernel mappings */
@@ -196,7 +223,7 @@ static void update_mapping_prot(phys_addr_t phys, unsigned long virt,
static void __init __map_memblock(pgd_t *pgdp, phys_addr_t start,
phys_addr_t end, pgprot_t prot, int flags)
{
- __create_pgd_mapping(pgdp, start, __phys_to_virt(start), end - start,
+ __create_pgd_mapping(pgdp, PTRS_PER_PGD, start, __phys_to_virt(start), end - start,
prot, early_pgtable_alloc, flags);
}
@@ -297,7 +324,7 @@ static void __init map_kernel_segment(pgd_t *pgdp, void *va_start, void *va_end,
BUG_ON(!PAGE_ALIGNED(pa_start));
BUG_ON(!PAGE_ALIGNED(size));
- __create_pgd_mapping(pgdp, pa_start, (unsigned long)va_start, size, prot,
+ __create_pgd_mapping(pgdp, PTRS_PER_PGD, pa_start, (unsigned long)va_start, size, prot,
early_pgtable_alloc, flags);
if (!(vm_flags & VM_NO_GUARD))
@@ -341,7 +368,7 @@ static int __init map_entry_trampoline(void)
/* Map only the text into the trampoline page table */
memset(tramp_pg_dir, 0, PGD_SIZE);
- __create_pgd_mapping(tramp_pg_dir, pa_start, TRAMP_VALIAS, PAGE_SIZE,
+ __create_pgd_mapping(tramp_pg_dir, PTRS_PER_PGD, pa_start, TRAMP_VALIAS, PAGE_SIZE,
prot, __pgd_pgtable_alloc, 0);
/* Map both the text and data into the kernel page table */
@@ -1233,7 +1260,7 @@ int arch_add_memory(int nid, u64 start, u64 size,
IS_ENABLED(CONFIG_KFENCE))
flags = NO_BLOCK_MAPPINGS | NO_CONT_MAPPINGS;
- __create_pgd_mapping(swapper_pg_dir, start, __phys_to_virt(start),
+ __create_pgd_mapping(swapper_pg_dir, PTRS_PER_PGD, start, __phys_to_virt(start),
size, params->pgprot, __pgd_pgtable_alloc,
flags);
@@ -241,14 +241,19 @@ static void alloc_init_pud(pgd_t *pgdp, unsigned long addr, unsigned long end,
pud_clear_fixmap();
}
-static void __create_pgd_mapping(pgd_t *pgdir, phys_addr_t phys,
+static void __create_pgd_mapping(pgd_t *pgdir, unsigned int entries_cnt, phys_addr_t phys,
unsigned long virt, phys_addr_t size,
pgprot_t prot,
phys_addr_t (*pgtable_alloc)(int),
int flags)
{
unsigned long addr, end, next;
- pgd_t *pgdp = pgd_offset_pgd(pgdir, virt);
+ pgd_t *pgdp;
+
+ if (likely(entries_cnt == PTRS_PER_PGD))
+ pgdp = pgd_offset_pgd(pgdir, virt);
+ else
+ pgdp = pgdir + ((virt >> PGDIR_SHIFT) & (entries_cnt - 1));
/*
* If the virtual and physical address don't have the same offset
As idmap_ptrs_per_pgd may have pgd entries greater than PTRS_PER_PGD, the prototype of __create_pgd_mapping() needs change to cope with that to create idmap. Now this adaption, create_idmap() API can be introduced to create idmap handly for all kinds of CONFIG_PGTABLE_LEVEL Signed-off-by: Pingfan Liu <kernelfans@gmail.com> Cc: Catalin Marinas <catalin.marinas@arm.com> Cc: Will Deacon <will@kernel.org> Cc: Marc Zyngier <maz@kernel.org> Cc: Kristina Martsenko <kristina.martsenko@arm.com> Cc: James Morse <james.morse@arm.com> Cc: Steven Price <steven.price@arm.com> Cc: Jonathan Cameron <Jonathan.Cameron@huawei.com> Cc: Pavel Tatashin <pasha.tatashin@soleen.com> Cc: Anshuman Khandual <anshuman.khandual@arm.com> Cc: Atish Patra <atish.patra@wdc.com> Cc: Mike Rapoport <rppt@kernel.org> Cc: Logan Gunthorpe <logang@deltatee.com> Cc: Mark Brown <broonie@kernel.org> To: linux-arm-kernel@lists.infradead.org --- arch/arm64/include/asm/pgalloc.h | 7 ++++++ arch/arm64/kernel/head.S | 3 +++ arch/arm64/mm/idmap_mmu.c | 16 ++++++++----- arch/arm64/mm/mmu.c | 41 ++++++++++++++++++++++++++------ arch/arm64/mm/mmu_include.c | 9 +++++-- 5 files changed, 61 insertions(+), 15 deletions(-)