| Message ID | 20210811004258.138075-16-vgupta@kernel.org (mailing list archive) |
|---|---|
| State | New |
| Headers | show |
| Series | ARC mm updates to support 3 or 4 levels of paging | expand |
On Tue, Aug 10, 2021 at 05:42:55PM -0700, Vineet Gupta wrote: > ARCv2 MMU is software walked and Linux implements 2 levels of paging: pgd/pte. > Forthcoming hw will have multiple levels, so this change preps mm code > for same. It is also fun to try multi levels even on soft-walked code to > ensure generic mm code is robust to handle. > > overview > ________ > > 2 levels {pgd, pte} : pmd is folded but pmd_* macros are valid and operate on pgd > 3 levels {pgd, pmd, pte}: > - pud is folded and pud_* macros point to pgd > - pmd_* macros operate on actual pmd > > code changes > ____________ > > 1. #include <asm-generic/pgtable-nopud.h> > > 2. Define CONFIG_PGTABLE_LEVELS 3 > > 3a. Define PMD_SHIFT, PMD_SIZE, PMD_MASK, pmd_t > 3b. Define pmd_val() which actually deals with pmd > (pmd_offset(), pmd_index() are provided by generic code) > 3c. Define pmd_alloc_one() and pmd_free() to allocate pmd > (pmd_populate/pmd_free already exist) > > 4. Define pud_none(), pud_bad() macros based on generic pud_val() which > internally pertains to pgd now. > 4b. define pud_populate() to just setup pgd > > Signed-off-by: Vineet Gupta <vgupta@kernel.org> > --- > arch/arc/Kconfig | 4 ++ > arch/arc/include/asm/page.h | 11 +++++ > arch/arc/include/asm/pgalloc.h | 22 ++++++++++ > arch/arc/include/asm/pgtable-levels.h | 63 ++++++++++++++++++++++++--- > arch/arc/include/asm/processor.h | 2 +- > arch/arc/mm/fault.c | 4 ++ > arch/arc/mm/tlb.c | 4 +- > arch/arc/mm/tlbex.S | 9 ++++ > 8 files changed, 111 insertions(+), 8 deletions(-) > > diff --git a/arch/arc/Kconfig b/arch/arc/Kconfig > index 59d5b2a179f6..43cb8aaf57a2 100644 > --- a/arch/arc/Kconfig > +++ b/arch/arc/Kconfig > @@ -314,6 +314,10 @@ config ARC_HUGEPAGE_16M > > endchoice > > +config PGTABLE_LEVELS > + int "Number of Page table levels" > + default 2 > + > config ARC_COMPACT_IRQ_LEVELS > depends on ISA_ARCOMPACT > bool "Setup Timer IRQ as high Priority" > diff --git a/arch/arc/include/asm/page.h b/arch/arc/include/asm/page.h > index 313e6f543d2d..df3cc154ae4a 100644 > --- a/arch/arc/include/asm/page.h > +++ b/arch/arc/include/asm/page.h > @@ -41,6 +41,17 @@ typedef struct { > #define pgd_val(x) ((x).pgd) > #define __pgd(x) ((pgd_t) { (x) }) > > +#if CONFIG_PGTABLE_LEVELS > 2 > + > +typedef struct { > + unsigned long pmd; > +} pmd_t; > + > +#define pmd_val(x) ((x).pmd) > +#define __pmd(x) ((pmd_t) { (x) }) > + > +#endif > + > typedef struct { > #ifdef CONFIG_ARC_HAS_PAE40 > unsigned long long pte; > diff --git a/arch/arc/include/asm/pgalloc.h b/arch/arc/include/asm/pgalloc.h > index 0cf73431eb89..01c2d84418ed 100644 > --- a/arch/arc/include/asm/pgalloc.h > +++ b/arch/arc/include/asm/pgalloc.h > @@ -86,6 +86,28 @@ static inline void pgd_free(struct mm_struct *mm, pgd_t *pgd) > } > > > +#if CONFIG_PGTABLE_LEVELS > 2 > + > +static inline void pud_populate(struct mm_struct *mm, pud_t *pudp, pmd_t *pmdp) > +{ > + set_pud(pudp, __pud((unsigned long)pmdp)); > +} > + > +static inline pmd_t *pmd_alloc_one(struct mm_struct *mm, unsigned long addr) > +{ > + return (pmd_t *)__get_free_page( > + GFP_KERNEL | __GFP_RETRY_MAYFAIL | __GFP_ZERO); > +} > + > +static inline void pmd_free(struct mm_struct *mm, pmd_t *pmd) > +{ > + free_page((unsigned long)pmd); > +} > + > +#define __pmd_free_tlb(tlb, pmd, addr) pmd_free((tlb)->mm, pmd) > + > +#endif > + > /* > * With software-only page-tables, addr-split for traversal is tweakable and > * that directly governs how big tables would be at each level. > diff --git a/arch/arc/include/asm/pgtable-levels.h b/arch/arc/include/asm/pgtable-levels.h > index 8ece75335bb5..1c2f022d4ad0 100644 > --- a/arch/arc/include/asm/pgtable-levels.h > +++ b/arch/arc/include/asm/pgtable-levels.h > @@ -10,6 +10,8 @@ > #ifndef _ASM_ARC_PGTABLE_LEVELS_H > #define _ASM_ARC_PGTABLE_LEVELS_H > > +#if CONFIG_PGTABLE_LEVELS == 2 > + > /* > * 2 level paging setup for software walked MMUv3 (ARC700) and MMUv4 (HS) > * > @@ -37,16 +39,38 @@ > #define PGDIR_SHIFT 21 > #endif > > -#define PGDIR_SIZE BIT(PGDIR_SHIFT) /* vaddr span, not PDG sz */ > -#define PGDIR_MASK (~(PGDIR_SIZE - 1)) > +#else > + > +/* > + * A default 3 level paging testing setup in software walked MMU > + * MMUv4 (8K page): <4> : <7> : <8> : <13> > + */ > +#define PGDIR_SHIFT 28 > +#if CONFIG_PGTABLE_LEVELS > 2 > +#define PMD_SHIFT 21 > +#endif > + > +#endif > > +#define PGDIR_SIZE BIT(PGDIR_SHIFT) > +#define PGDIR_MASK (~(PGDIR_SIZE - 1)) > #define PTRS_PER_PGD BIT(32 - PGDIR_SHIFT) > > -#define PTRS_PER_PTE BIT(PGDIR_SHIFT - PAGE_SHIFT) > +#if CONFIG_PGTABLE_LEVELS > 2 > +#define PMD_SIZE BIT(PMD_SHIFT) > +#define PMD_MASK (~(PMD_SIZE - 1)) > +#define PTRS_PER_PMD BIT(PGDIR_SHIFT - PMD_SHIFT) Maybe move these into the previous #if CONFIG_PGTABLE_LEVELS > 2? > +#endif > + > +#define PTRS_PER_PTE BIT(PMD_SHIFT - PAGE_SHIFT) > > #ifndef __ASSEMBLY__ > > +#if CONFIG_PGTABLE_LEVELS > 2 > +#include <asm-generic/pgtable-nopud.h> > +#else > #include <asm-generic/pgtable-nopmd.h> > +#endif > > /* > * 1st level paging: pgd > @@ -57,9 +81,35 @@ > #define pgd_ERROR(e) \ > pr_crit("%s:%d: bad pgd %08lx.\n", __FILE__, __LINE__, pgd_val(e)) > > +#if CONFIG_PGTABLE_LEVELS > 2 > + > +/* In 3 level paging, pud_* macros work on pgd */ > +#define pud_none(x) (!pud_val(x)) > +#define pud_bad(x) ((pud_val(x) & ~PAGE_MASK)) > +#define pud_present(x) (pud_val(x)) > +#define pud_clear(xp) do { pud_val(*(xp)) = 0; } while (0) > +#define pud_pgtable(pud) ((pmd_t *)(pud_val(pud) & PAGE_MASK)) > +#define pud_page(pud) virt_to_page(pud_pgtable(pud)) > +#define set_pud(pudp, pud) (*(pudp) = pud) > + > +/* > + * 2nd level paging: pmd > + */ > +#define pmd_ERROR(e) \ > + pr_crit("%s:%d: bad pmd %08lx.\n", __FILE__, __LINE__, pmd_val(e)) > + > +#define pmd_pfn(pmd) ((pmd_val(pmd) & PMD_MASK) >> PAGE_SHIFT) > +#define pfn_pmd(pfn,prot) __pmd(((pfn) << PAGE_SHIFT) | pgprot_val(prot)) > +#define mk_pmd(page,prot) pfn_pmd(page_to_pfn(page),prot) > + > +#endif > + > /* > - * Due to the strange way generic pgtable level folding works, in a 2 level > - * setup, pmd_val() returns pgd, so these pmd_* macros actually work on pgd > + * Due to the strange way generic pgtable level folding works, the pmd_* macros > + * - are valid even for 2 levels (which supposedly only has pgd - pte) > + * - behave differently for 2 vs. 3 > + * In 2 level paging (pgd -> pte), pmd_* macros work on pgd > + * In 3+ level paging (pgd -> pmd -> pte), pmd_* macros work on pmd > */ > #define pmd_none(x) (!pmd_val(x)) > #define pmd_bad(x) ((pmd_val(x) & ~PAGE_MASK)) > @@ -70,6 +120,9 @@ > #define set_pmd(pmdp, pmd) (*(pmdp) = pmd) > #define pmd_pgtable(pmd) ((pgtable_t) pmd_page_vaddr(pmd)) > > +/* > + * 3rd level paging: pte > + */ > #define pte_ERROR(e) \ > pr_crit("%s:%d: bad pte %08lx.\n", __FILE__, __LINE__, pte_val(e)) > > diff --git a/arch/arc/include/asm/processor.h b/arch/arc/include/asm/processor.h > index e4031ecd3c8c..f28afcf5c6d1 100644 > --- a/arch/arc/include/asm/processor.h > +++ b/arch/arc/include/asm/processor.h > @@ -93,7 +93,7 @@ extern unsigned int get_wchan(struct task_struct *p); > #define VMALLOC_START (PAGE_OFFSET - (CONFIG_ARC_KVADDR_SIZE << 20)) > > /* 1 PGDIR_SIZE each for fixmap/pkmap, 2 PGDIR_SIZE gutter (see asm/highmem.h) */ > -#define VMALLOC_SIZE ((CONFIG_ARC_KVADDR_SIZE << 20) - PGDIR_SIZE * 4) > +#define VMALLOC_SIZE ((CONFIG_ARC_KVADDR_SIZE << 20) - PMD_SIZE * 4) > > #define VMALLOC_END (VMALLOC_START + VMALLOC_SIZE) > > diff --git a/arch/arc/mm/fault.c b/arch/arc/mm/fault.c > index 41f154320964..8da2f0ad8c69 100644 > --- a/arch/arc/mm/fault.c > +++ b/arch/arc/mm/fault.c > @@ -39,6 +39,8 @@ noinline static int handle_kernel_vaddr_fault(unsigned long address) > if (!pgd_present(*pgd_k)) > goto bad_area; > > + set_pgd(pgd, *pgd_k); > + > p4d = p4d_offset(pgd, address); > p4d_k = p4d_offset(pgd_k, address); > if (!p4d_present(*p4d_k)) > @@ -49,6 +51,8 @@ noinline static int handle_kernel_vaddr_fault(unsigned long address) > if (!pud_present(*pud_k)) > goto bad_area; > > + set_pud(pud, *pud_k); > + > pmd = pmd_offset(pud, address); > pmd_k = pmd_offset(pud_k, address); > if (!pmd_present(*pmd_k)) > diff --git a/arch/arc/mm/tlb.c b/arch/arc/mm/tlb.c > index 34f16e0b41e6..77da83569b36 100644 > --- a/arch/arc/mm/tlb.c > +++ b/arch/arc/mm/tlb.c > @@ -658,8 +658,8 @@ char *arc_mmu_mumbojumbo(int cpu_id, char *buf, int len) > IS_USED_CFG(CONFIG_TRANSPARENT_HUGEPAGE)); > > n += scnprintf(buf + n, len - n, > - "MMU [v%x]\t: %dk PAGE, %sJTLB %d (%dx%d), uDTLB %d, uITLB %d%s%s\n", > - p_mmu->ver, p_mmu->pg_sz_k, super_pg, > + "MMU [v%x]\t: %dk PAGE, %s, swalk %d lvl, JTLB %d (%dx%d), uDTLB %d, uITLB %d%s%s\n", > + p_mmu->ver, p_mmu->pg_sz_k, super_pg, CONFIG_PGTABLE_LEVELS, > p_mmu->sets * p_mmu->ways, p_mmu->sets, p_mmu->ways, > p_mmu->u_dtlb, p_mmu->u_itlb, > IS_AVAIL2(p_mmu->pae, ", PAE40 ", CONFIG_ARC_HAS_PAE40)); > diff --git a/arch/arc/mm/tlbex.S b/arch/arc/mm/tlbex.S > index d08bd09a0afc..5f6bfdfda1be 100644 > --- a/arch/arc/mm/tlbex.S > +++ b/arch/arc/mm/tlbex.S > @@ -173,6 +173,15 @@ ex_saved_reg1: > tst r3, r3 > bz do_slow_path_pf ; if no Page Table, do page fault > > +#if CONFIG_PGTABLE_LEVELS > 2 > + lsr r0, r2, PMD_SHIFT ; Bits for indexing into PMD > + and r0, r0, (PTRS_PER_PMD - 1) > + ld.as r1, [r3, r0] ; PMD entry > + tst r1, r1 > + bz do_slow_path_pf > + mov r3, r1 > +#endif > + > #ifdef CONFIG_TRANSPARENT_HUGEPAGE > and.f 0, r3, _PAGE_HW_SZ ; Is this Huge PMD (thp) > add2.nz r1, r1, r0 > -- > 2.25.1 >
On 8/11/21 5:24 AM, Mike Rapoport wrote: > On Tue, Aug 10, 2021 at 05:42:55PM -0700, Vineet Gupta wrote: >> ARCv2 MMU is software walked and Linux implements 2 levels of paging: pgd/pte. >> Forthcoming hw will have multiple levels, so this change preps mm code >> for same. It is also fun to try multi levels even on soft-walked code to >> ensure generic mm code is robust to handle. >> >> overview >> ________ >> >> 2 levels {pgd, pte} : pmd is folded but pmd_* macros are valid and operate on pgd >> 3 levels {pgd, pmd, pte}: >> - pud is folded and pud_* macros point to pgd >> - pmd_* macros operate on actual pmd >> >> code changes >> ____________ >> >> 1. #include <asm-generic/pgtable-nopud.h> >> >> 2. Define CONFIG_PGTABLE_LEVELS 3 >> >> 3a. Define PMD_SHIFT, PMD_SIZE, PMD_MASK, pmd_t >> 3b. Define pmd_val() which actually deals with pmd >> (pmd_offset(), pmd_index() are provided by generic code) >> 3c. Define pmd_alloc_one() and pmd_free() to allocate pmd >> (pmd_populate/pmd_free already exist) >> >> 4. Define pud_none(), pud_bad() macros based on generic pud_val() which >> internally pertains to pgd now. >> 4b. define pud_populate() to just setup pgd >> >> Signed-off-by: Vineet Gupta <vgupta@kernel.org> >> --- >> arch/arc/Kconfig | 4 ++ >> arch/arc/include/asm/page.h | 11 +++++ >> arch/arc/include/asm/pgalloc.h | 22 ++++++++++ >> arch/arc/include/asm/pgtable-levels.h | 63 ++++++++++++++++++++++++--- >> arch/arc/include/asm/processor.h | 2 +- >> arch/arc/mm/fault.c | 4 ++ >> arch/arc/mm/tlb.c | 4 +- >> arch/arc/mm/tlbex.S | 9 ++++ >> 8 files changed, 111 insertions(+), 8 deletions(-) >> >> diff --git a/arch/arc/Kconfig b/arch/arc/Kconfig >> index 59d5b2a179f6..43cb8aaf57a2 100644 >> --- a/arch/arc/Kconfig >> +++ b/arch/arc/Kconfig >> @@ -314,6 +314,10 @@ config ARC_HUGEPAGE_16M >> >> endchoice >> >> +config PGTABLE_LEVELS >> + int "Number of Page table levels" >> + default 2 >> + >> config ARC_COMPACT_IRQ_LEVELS >> depends on ISA_ARCOMPACT >> bool "Setup Timer IRQ as high Priority" >> diff --git a/arch/arc/include/asm/page.h b/arch/arc/include/asm/page.h >> index 313e6f543d2d..df3cc154ae4a 100644 >> --- a/arch/arc/include/asm/page.h >> +++ b/arch/arc/include/asm/page.h >> @@ -41,6 +41,17 @@ typedef struct { >> #define pgd_val(x) ((x).pgd) >> #define __pgd(x) ((pgd_t) { (x) }) >> >> +#if CONFIG_PGTABLE_LEVELS > 2 >> + >> +typedef struct { >> + unsigned long pmd; >> +} pmd_t; >> + >> +#define pmd_val(x) ((x).pmd) >> +#define __pmd(x) ((pmd_t) { (x) }) >> + >> +#endif >> + >> typedef struct { >> #ifdef CONFIG_ARC_HAS_PAE40 >> unsigned long long pte; >> diff --git a/arch/arc/include/asm/pgalloc.h b/arch/arc/include/asm/pgalloc.h >> index 0cf73431eb89..01c2d84418ed 100644 >> --- a/arch/arc/include/asm/pgalloc.h >> +++ b/arch/arc/include/asm/pgalloc.h >> @@ -86,6 +86,28 @@ static inline void pgd_free(struct mm_struct *mm, pgd_t *pgd) >> } >> >> >> +#if CONFIG_PGTABLE_LEVELS > 2 >> + >> +static inline void pud_populate(struct mm_struct *mm, pud_t *pudp, pmd_t *pmdp) >> +{ >> + set_pud(pudp, __pud((unsigned long)pmdp)); >> +} >> + >> +static inline pmd_t *pmd_alloc_one(struct mm_struct *mm, unsigned long addr) >> +{ >> + return (pmd_t *)__get_free_page( >> + GFP_KERNEL | __GFP_RETRY_MAYFAIL | __GFP_ZERO); >> +} >> + >> +static inline void pmd_free(struct mm_struct *mm, pmd_t *pmd) >> +{ >> + free_page((unsigned long)pmd); >> +} >> + >> +#define __pmd_free_tlb(tlb, pmd, addr) pmd_free((tlb)->mm, pmd) >> + >> +#endif >> + >> /* >> * With software-only page-tables, addr-split for traversal is tweakable and >> * that directly governs how big tables would be at each level. >> diff --git a/arch/arc/include/asm/pgtable-levels.h b/arch/arc/include/asm/pgtable-levels.h >> index 8ece75335bb5..1c2f022d4ad0 100644 >> --- a/arch/arc/include/asm/pgtable-levels.h >> +++ b/arch/arc/include/asm/pgtable-levels.h >> @@ -10,6 +10,8 @@ >> #ifndef _ASM_ARC_PGTABLE_LEVELS_H >> #define _ASM_ARC_PGTABLE_LEVELS_H >> >> +#if CONFIG_PGTABLE_LEVELS == 2 >> + >> /* >> * 2 level paging setup for software walked MMUv3 (ARC700) and MMUv4 (HS) >> * >> @@ -37,16 +39,38 @@ >> #define PGDIR_SHIFT 21 >> #endif >> >> -#define PGDIR_SIZE BIT(PGDIR_SHIFT) /* vaddr span, not PDG sz */ >> -#define PGDIR_MASK (~(PGDIR_SIZE - 1)) >> +#else >> + >> +/* >> + * A default 3 level paging testing setup in software walked MMU >> + * MMUv4 (8K page): <4> : <7> : <8> : <13> >> + */ >> +#define PGDIR_SHIFT 28 >> +#if CONFIG_PGTABLE_LEVELS > 2 >> +#define PMD_SHIFT 21 >> +#endif >> + >> +#endif >> >> +#define PGDIR_SIZE BIT(PGDIR_SHIFT) >> +#define PGDIR_MASK (~(PGDIR_SIZE - 1)) >> #define PTRS_PER_PGD BIT(32 - PGDIR_SHIFT) >> >> -#define PTRS_PER_PTE BIT(PGDIR_SHIFT - PAGE_SHIFT) >> +#if CONFIG_PGTABLE_LEVELS > 2 >> +#define PMD_SIZE BIT(PMD_SHIFT) >> +#define PMD_MASK (~(PMD_SIZE - 1)) >> +#define PTRS_PER_PMD BIT(PGDIR_SHIFT - PMD_SHIFT) > Maybe move these into the previous #if CONFIG_PGTABLE_LEVELS > 2? I've kept them separate to distinguish the "configuration" parts which can be changed (for ARCv2 where this is not hardware enforced) from the pieces which remain the same. So PMD_SHIFT can be changed for a quick hack say but the PMD_{SIZE,MASK} etc are invariants. -Vineet
diff --git a/arch/arc/Kconfig b/arch/arc/Kconfig index 59d5b2a179f6..43cb8aaf57a2 100644 --- a/arch/arc/Kconfig +++ b/arch/arc/Kconfig @@ -314,6 +314,10 @@ config ARC_HUGEPAGE_16M endchoice +config PGTABLE_LEVELS + int "Number of Page table levels" + default 2 + config ARC_COMPACT_IRQ_LEVELS depends on ISA_ARCOMPACT bool "Setup Timer IRQ as high Priority" diff --git a/arch/arc/include/asm/page.h b/arch/arc/include/asm/page.h index 313e6f543d2d..df3cc154ae4a 100644 --- a/arch/arc/include/asm/page.h +++ b/arch/arc/include/asm/page.h @@ -41,6 +41,17 @@ typedef struct { #define pgd_val(x) ((x).pgd) #define __pgd(x) ((pgd_t) { (x) }) +#if CONFIG_PGTABLE_LEVELS > 2 + +typedef struct { + unsigned long pmd; +} pmd_t; + +#define pmd_val(x) ((x).pmd) +#define __pmd(x) ((pmd_t) { (x) }) + +#endif + typedef struct { #ifdef CONFIG_ARC_HAS_PAE40 unsigned long long pte; diff --git a/arch/arc/include/asm/pgalloc.h b/arch/arc/include/asm/pgalloc.h index 0cf73431eb89..01c2d84418ed 100644 --- a/arch/arc/include/asm/pgalloc.h +++ b/arch/arc/include/asm/pgalloc.h @@ -86,6 +86,28 @@ static inline void pgd_free(struct mm_struct *mm, pgd_t *pgd) } +#if CONFIG_PGTABLE_LEVELS > 2 + +static inline void pud_populate(struct mm_struct *mm, pud_t *pudp, pmd_t *pmdp) +{ + set_pud(pudp, __pud((unsigned long)pmdp)); +} + +static inline pmd_t *pmd_alloc_one(struct mm_struct *mm, unsigned long addr) +{ + return (pmd_t *)__get_free_page( + GFP_KERNEL | __GFP_RETRY_MAYFAIL | __GFP_ZERO); +} + +static inline void pmd_free(struct mm_struct *mm, pmd_t *pmd) +{ + free_page((unsigned long)pmd); +} + +#define __pmd_free_tlb(tlb, pmd, addr) pmd_free((tlb)->mm, pmd) + +#endif + /* * With software-only page-tables, addr-split for traversal is tweakable and * that directly governs how big tables would be at each level. diff --git a/arch/arc/include/asm/pgtable-levels.h b/arch/arc/include/asm/pgtable-levels.h index 8ece75335bb5..1c2f022d4ad0 100644 --- a/arch/arc/include/asm/pgtable-levels.h +++ b/arch/arc/include/asm/pgtable-levels.h @@ -10,6 +10,8 @@ #ifndef _ASM_ARC_PGTABLE_LEVELS_H #define _ASM_ARC_PGTABLE_LEVELS_H +#if CONFIG_PGTABLE_LEVELS == 2 + /* * 2 level paging setup for software walked MMUv3 (ARC700) and MMUv4 (HS) * @@ -37,16 +39,38 @@ #define PGDIR_SHIFT 21 #endif -#define PGDIR_SIZE BIT(PGDIR_SHIFT) /* vaddr span, not PDG sz */ -#define PGDIR_MASK (~(PGDIR_SIZE - 1)) +#else + +/* + * A default 3 level paging testing setup in software walked MMU + * MMUv4 (8K page): <4> : <7> : <8> : <13> + */ +#define PGDIR_SHIFT 28 +#if CONFIG_PGTABLE_LEVELS > 2 +#define PMD_SHIFT 21 +#endif + +#endif +#define PGDIR_SIZE BIT(PGDIR_SHIFT) +#define PGDIR_MASK (~(PGDIR_SIZE - 1)) #define PTRS_PER_PGD BIT(32 - PGDIR_SHIFT) -#define PTRS_PER_PTE BIT(PGDIR_SHIFT - PAGE_SHIFT) +#if CONFIG_PGTABLE_LEVELS > 2 +#define PMD_SIZE BIT(PMD_SHIFT) +#define PMD_MASK (~(PMD_SIZE - 1)) +#define PTRS_PER_PMD BIT(PGDIR_SHIFT - PMD_SHIFT) +#endif + +#define PTRS_PER_PTE BIT(PMD_SHIFT - PAGE_SHIFT) #ifndef __ASSEMBLY__ +#if CONFIG_PGTABLE_LEVELS > 2 +#include <asm-generic/pgtable-nopud.h> +#else #include <asm-generic/pgtable-nopmd.h> +#endif /* * 1st level paging: pgd @@ -57,9 +81,35 @@ #define pgd_ERROR(e) \ pr_crit("%s:%d: bad pgd %08lx.\n", __FILE__, __LINE__, pgd_val(e)) +#if CONFIG_PGTABLE_LEVELS > 2 + +/* In 3 level paging, pud_* macros work on pgd */ +#define pud_none(x) (!pud_val(x)) +#define pud_bad(x) ((pud_val(x) & ~PAGE_MASK)) +#define pud_present(x) (pud_val(x)) +#define pud_clear(xp) do { pud_val(*(xp)) = 0; } while (0) +#define pud_pgtable(pud) ((pmd_t *)(pud_val(pud) & PAGE_MASK)) +#define pud_page(pud) virt_to_page(pud_pgtable(pud)) +#define set_pud(pudp, pud) (*(pudp) = pud) + +/* + * 2nd level paging: pmd + */ +#define pmd_ERROR(e) \ + pr_crit("%s:%d: bad pmd %08lx.\n", __FILE__, __LINE__, pmd_val(e)) + +#define pmd_pfn(pmd) ((pmd_val(pmd) & PMD_MASK) >> PAGE_SHIFT) +#define pfn_pmd(pfn,prot) __pmd(((pfn) << PAGE_SHIFT) | pgprot_val(prot)) +#define mk_pmd(page,prot) pfn_pmd(page_to_pfn(page),prot) + +#endif + /* - * Due to the strange way generic pgtable level folding works, in a 2 level - * setup, pmd_val() returns pgd, so these pmd_* macros actually work on pgd + * Due to the strange way generic pgtable level folding works, the pmd_* macros + * - are valid even for 2 levels (which supposedly only has pgd - pte) + * - behave differently for 2 vs. 3 + * In 2 level paging (pgd -> pte), pmd_* macros work on pgd + * In 3+ level paging (pgd -> pmd -> pte), pmd_* macros work on pmd */ #define pmd_none(x) (!pmd_val(x)) #define pmd_bad(x) ((pmd_val(x) & ~PAGE_MASK)) @@ -70,6 +120,9 @@ #define set_pmd(pmdp, pmd) (*(pmdp) = pmd) #define pmd_pgtable(pmd) ((pgtable_t) pmd_page_vaddr(pmd)) +/* + * 3rd level paging: pte + */ #define pte_ERROR(e) \ pr_crit("%s:%d: bad pte %08lx.\n", __FILE__, __LINE__, pte_val(e)) diff --git a/arch/arc/include/asm/processor.h b/arch/arc/include/asm/processor.h index e4031ecd3c8c..f28afcf5c6d1 100644 --- a/arch/arc/include/asm/processor.h +++ b/arch/arc/include/asm/processor.h @@ -93,7 +93,7 @@ extern unsigned int get_wchan(struct task_struct *p); #define VMALLOC_START (PAGE_OFFSET - (CONFIG_ARC_KVADDR_SIZE << 20)) /* 1 PGDIR_SIZE each for fixmap/pkmap, 2 PGDIR_SIZE gutter (see asm/highmem.h) */ -#define VMALLOC_SIZE ((CONFIG_ARC_KVADDR_SIZE << 20) - PGDIR_SIZE * 4) +#define VMALLOC_SIZE ((CONFIG_ARC_KVADDR_SIZE << 20) - PMD_SIZE * 4) #define VMALLOC_END (VMALLOC_START + VMALLOC_SIZE) diff --git a/arch/arc/mm/fault.c b/arch/arc/mm/fault.c index 41f154320964..8da2f0ad8c69 100644 --- a/arch/arc/mm/fault.c +++ b/arch/arc/mm/fault.c @@ -39,6 +39,8 @@ noinline static int handle_kernel_vaddr_fault(unsigned long address) if (!pgd_present(*pgd_k)) goto bad_area; + set_pgd(pgd, *pgd_k); + p4d = p4d_offset(pgd, address); p4d_k = p4d_offset(pgd_k, address); if (!p4d_present(*p4d_k)) @@ -49,6 +51,8 @@ noinline static int handle_kernel_vaddr_fault(unsigned long address) if (!pud_present(*pud_k)) goto bad_area; + set_pud(pud, *pud_k); + pmd = pmd_offset(pud, address); pmd_k = pmd_offset(pud_k, address); if (!pmd_present(*pmd_k)) diff --git a/arch/arc/mm/tlb.c b/arch/arc/mm/tlb.c index 34f16e0b41e6..77da83569b36 100644 --- a/arch/arc/mm/tlb.c +++ b/arch/arc/mm/tlb.c @@ -658,8 +658,8 @@ char *arc_mmu_mumbojumbo(int cpu_id, char *buf, int len) IS_USED_CFG(CONFIG_TRANSPARENT_HUGEPAGE)); n += scnprintf(buf + n, len - n, - "MMU [v%x]\t: %dk PAGE, %sJTLB %d (%dx%d), uDTLB %d, uITLB %d%s%s\n", - p_mmu->ver, p_mmu->pg_sz_k, super_pg, + "MMU [v%x]\t: %dk PAGE, %s, swalk %d lvl, JTLB %d (%dx%d), uDTLB %d, uITLB %d%s%s\n", + p_mmu->ver, p_mmu->pg_sz_k, super_pg, CONFIG_PGTABLE_LEVELS, p_mmu->sets * p_mmu->ways, p_mmu->sets, p_mmu->ways, p_mmu->u_dtlb, p_mmu->u_itlb, IS_AVAIL2(p_mmu->pae, ", PAE40 ", CONFIG_ARC_HAS_PAE40)); diff --git a/arch/arc/mm/tlbex.S b/arch/arc/mm/tlbex.S index d08bd09a0afc..5f6bfdfda1be 100644 --- a/arch/arc/mm/tlbex.S +++ b/arch/arc/mm/tlbex.S @@ -173,6 +173,15 @@ ex_saved_reg1: tst r3, r3 bz do_slow_path_pf ; if no Page Table, do page fault +#if CONFIG_PGTABLE_LEVELS > 2 + lsr r0, r2, PMD_SHIFT ; Bits for indexing into PMD + and r0, r0, (PTRS_PER_PMD - 1) + ld.as r1, [r3, r0] ; PMD entry + tst r1, r1 + bz do_slow_path_pf + mov r3, r1 +#endif + #ifdef CONFIG_TRANSPARENT_HUGEPAGE and.f 0, r3, _PAGE_HW_SZ ; Is this Huge PMD (thp) add2.nz r1, r1, r0