@@ -7,6 +7,7 @@
#define NO_BLOCK_MAPPINGS BIT(0)
#define NO_CONT_MAPPINGS BIT(1)
+#define NO_FIXMAP BIT(2)
static bool pgattr_change_is_safe(u64 old, u64 new)
{
@@ -43,11 +44,14 @@ static bool pgattr_change_is_safe(u64 old, u64 new)
}
static void init_pte(pmd_t *pmdp, unsigned long addr, unsigned long end,
- phys_addr_t phys, pgprot_t prot)
+ phys_addr_t phys, pgprot_t prot, int flags)
{
pte_t *ptep;
- ptep = pte_set_fixmap_offset(pmdp, addr);
+ if (likely(!(flags & NO_FIXMAP)))
+ ptep = pte_set_fixmap_offset(pmdp, addr);
+ else
+ ptep = pte_offset_kernel(pmdp, addr);
do {
pte_t old_pte = READ_ONCE(*ptep);
@@ -63,7 +67,8 @@ static void init_pte(pmd_t *pmdp, unsigned long addr, unsigned long end,
phys += PAGE_SIZE;
} while (ptep++, addr += PAGE_SIZE, addr != end);
- pte_clear_fixmap();
+ if (likely(!(flags & NO_FIXMAP)))
+ pte_clear_fixmap();
}
static void alloc_init_cont_pte(pmd_t *pmdp, unsigned long addr,
@@ -97,7 +102,7 @@ static void alloc_init_cont_pte(pmd_t *pmdp, unsigned long addr,
(flags & NO_CONT_MAPPINGS) == 0)
__prot = __pgprot(pgprot_val(prot) | PTE_CONT);
- init_pte(pmdp, addr, next, phys, __prot);
+ init_pte(pmdp, addr, next, phys, __prot, flags);
phys += next - addr;
} while (addr = next, addr != end);
@@ -112,7 +117,10 @@ static void init_pmd(pud_t *pudp, unsigned long addr, unsigned long end,
unsigned long next;
pmd_t *pmdp;
- pmdp = pmd_set_fixmap_offset(pudp, addr);
+ if (likely(!(flags & NO_FIXMAP)))
+ pmdp = pmd_set_fixmap_offset(pudp, addr);
+ else
+ pmdp = pmd_offset(pudp, addr);
do {
pmd_t old_pmd = READ_ONCE(*pmdp);
@@ -139,7 +147,8 @@ static void init_pmd(pud_t *pudp, unsigned long addr, unsigned long end,
phys += next - addr;
} while (pmdp++, addr = next, addr != end);
- pmd_clear_fixmap();
+ if (likely(!(flags & NO_FIXMAP)))
+ pmd_clear_fixmap();
}
static void alloc_init_cont_pmd(pud_t *pudp, unsigned long addr,
@@ -215,7 +224,10 @@ static void alloc_init_pud(pgd_t *pgdp, unsigned long addr, unsigned long end,
}
BUG_ON(p4d_bad(p4d));
- pudp = pud_set_fixmap_offset(p4dp, addr);
+ if (likely(!(flags & NO_FIXMAP)))
+ pudp = pud_set_fixmap_offset(p4dp, addr);
+ else
+ pudp = pud_offset(p4dp, addr);
do {
pud_t old_pud = READ_ONCE(*pudp);
@@ -244,7 +256,8 @@ static void alloc_init_pud(pgd_t *pgdp, unsigned long addr, unsigned long end,
phys += next - addr;
} while (pudp++, addr = next, addr != end);
- pud_clear_fixmap();
+ if (likely(!(flags & NO_FIXMAP)))
+ pud_clear_fixmap();
}
static void __create_pgd_mapping(pgd_t *pgdir, unsigned int entries_cnt, phys_addr_t phys,
__create_pgd_mapping() is planned not only to be used in boot process, but also after fully bootup. The latter means several callers can run __create_pgd_mapping() concurrenlty. In this case, PUP/PMD/PTE fixmap should be not used, instead, the virtual addresss owned by each pgtable is used. Signed-off-by: Pingfan Liu <kernelfans@gmail.com> Cc: Catalin Marinas <catalin.marinas@arm.com> Cc: Will Deacon <will@kernel.org> Cc: Marc Zyngier <maz@kernel.org> Cc: Kristina Martsenko <kristina.martsenko@arm.com> Cc: James Morse <james.morse@arm.com> Cc: Steven Price <steven.price@arm.com> Cc: Jonathan Cameron <Jonathan.Cameron@huawei.com> Cc: Pavel Tatashin <pasha.tatashin@soleen.com> Cc: Anshuman Khandual <anshuman.khandual@arm.com> Cc: Atish Patra <atish.patra@wdc.com> Cc: Mike Rapoport <rppt@kernel.org> Cc: Logan Gunthorpe <logang@deltatee.com> Cc: Mark Brown <broonie@kernel.org> To: linux-arm-kernel@lists.infradead.org --- rfc -> v2: change allocator return type back to phys_addr_t --- arch/arm64/mm/mmu_include.c | 29 +++++++++++++++++++++-------- 1 file changed, 21 insertions(+), 8 deletions(-)