@@ -222,12 +222,6 @@ static inline unsigned long pgd_pfn(pgd_t pgd)
return (pgd_val(pgd) & PTE_PFN_MASK) >> PAGE_SHIFT;
}
-static inline int p4d_large(p4d_t p4d)
-{
- /* No 512 GiB pages yet */
- return 0;
-}
-
#define pte_page(pte) pfn_to_page(pte_pfn(pte))
static inline int pmd_large(pmd_t pte)
@@ -867,11 +861,6 @@ static inline int pud_bad(pud_t pud)
{
return (pud_flags(pud) & ~(_KERNPG_TABLE | _PAGE_USER)) != 0;
}
-#else
-static inline int pud_large(pud_t pud)
-{
- return 0;
-}
#endif /* CONFIG_PGTABLE_LEVELS > 2 */
static inline unsigned long pud_index(unsigned long address)
@@ -890,6 +879,12 @@ static inline int p4d_present(p4d_t p4d)
return p4d_flags(p4d) & _PAGE_PRESENT;
}
+static inline int p4d_large(p4d_t p4d)
+{
+ /* No 512 GiB pages yet */
+ return 0;
+}
+
static inline unsigned long p4d_page_vaddr(p4d_t p4d)
{
return (unsigned long)__va(p4d_val(p4d) & p4d_pfn_mask(p4d));
@@ -931,6 +926,8 @@ static inline int pgd_present(pgd_t pgd)
return pgd_flags(pgd) & _PAGE_PRESENT;
}
+static inline int pgd_large(pgd_t pgd) { return 0; }
+
static inline unsigned long pgd_page_vaddr(pgd_t pgd)
{
return (unsigned long)__va((unsigned long)pgd_val(pgd) & PTE_PFN_MASK);
@@ -1213,8 +1210,6 @@ static inline bool pgdp_maps_userspace(void *__ptr)
return (((ptr & ~PAGE_MASK) / sizeof(pgd_t)) < PGD_KERNEL_START);
}
-static inline int pgd_large(pgd_t pgd) { return 0; }
-
#ifdef CONFIG_PAGE_TABLE_ISOLATION
/*
* All top-level PAGE_TABLE_ISOLATION page tables are order-1 pages
@@ -20,6 +20,7 @@
#define pud_none(pud) 0
#define pud_bad(pud) 0
#define pud_present(pud) 1
+#define pud_large(pud) 0
#define pud_ERROR(pud) do { } while (0)
#define pud_clear(pud) pgd_clear(pud)
#define pud_val(pud) pgd_val(pud)
@@ -22,6 +22,7 @@
#define p4d_none(p4d) 0
#define p4d_bad(p4d) 0
#define p4d_present(p4d) 1
+#define p4d_large(p4d) 0
#define p4d_ERROR(p4d) do { } while (0)
#define p4d_clear(p4d) pgd_clear(p4d)
#define p4d_val(p4d) pgd_val(p4d)
@@ -27,6 +27,7 @@ typedef struct { pgd_t pgd; } pud_t;
static inline int pgd_none(pgd_t pgd) { return 0; }
static inline int pgd_bad(pgd_t pgd) { return 0; }
static inline int pgd_present(pgd_t pgd) { return 1; }
+static inline int pgd_large(pgd_t pgd) { return 0; }
static inline void pgd_clear(pgd_t *pgd) { }
#define pud_ERROR(pud) (pgd_ERROR((pud).pgd))
@@ -22,6 +22,7 @@ typedef struct { pgd_t pgd; } p4d_t;
static inline int pgd_none(pgd_t pgd) { return 0; }
static inline int pgd_bad(pgd_t pgd) { return 0; }
static inline int pgd_present(pgd_t pgd) { return 1; }
+static inline int pgd_large(pgd_t pgd) { return 0; }
static inline void pgd_clear(pgd_t *pgd) { }
#define p4d_ERROR(p4d) (pgd_ERROR((p4d).pgd))
@@ -30,6 +30,7 @@ typedef struct { pud_t pud; } pmd_t;
static inline int pud_none(pud_t pud) { return 0; }
static inline int pud_bad(pud_t pud) { return 0; }
static inline int pud_present(pud_t pud) { return 1; }
+static inline int pud_large(pud_t pud) { return 0; }
static inline void pud_clear(pud_t *pud) { }
#define pmd_ERROR(pmd) (pud_ERROR((pmd).pud))
@@ -31,6 +31,7 @@ typedef struct { p4d_t p4d; } pud_t;
static inline int p4d_none(p4d_t p4d) { return 0; }
static inline int p4d_bad(p4d_t p4d) { return 0; }
static inline int p4d_present(p4d_t p4d) { return 1; }
+static inline int p4d_large(p4d_t p4d) { return 0; }
static inline void p4d_clear(p4d_t *p4d) { }
#define pud_ERROR(pud) (p4d_ERROR((pud).p4d))
Exposing the pud/pgd levels of the page tables to walk_page_range() means we may come across the exotic large mappings that come with large areas of contiguous memory (such as the kernel's linear map). Where levels are folded we need to provide the appropriate stub implementation of p?d_large(). For x86 move the existing definitions of p?d_large() so that they are only defined when the corresponding levels are not folded. Signed-off-by: Steven Price <steven.price@arm.com> --- arch/x86/include/asm/pgtable.h | 21 ++++++++------------- include/asm-generic/4level-fixup.h | 1 + include/asm-generic/5level-fixup.h | 1 + include/asm-generic/pgtable-nop4d-hack.h | 1 + include/asm-generic/pgtable-nop4d.h | 1 + include/asm-generic/pgtable-nopmd.h | 1 + include/asm-generic/pgtable-nopud.h | 1 + 7 files changed, 14 insertions(+), 13 deletions(-)