@@ -34,7 +34,7 @@
#ifndef __ASSEMBLY__
#ifdef CONFIG_ARM64_64K_PAGES
-#include <asm/pgtable-2level-types.h>
+#include <asm/pgtable-3level-types.h>
#else
#include <asm/pgtable-4level-types.h>
#endif
@@ -26,8 +26,6 @@
#define check_pgt_cache() do { } while (0)
-#ifndef CONFIG_ARM64_64K_PAGES
-
static inline pmd_t *pmd_alloc_one(struct mm_struct *mm, unsigned long addr)
{
return (pmd_t *)get_zeroed_page(GFP_KERNEL | __GFP_REPEAT);
@@ -44,6 +42,8 @@ static inline void pud_populate(struct mm_struct *mm, pud_t *pud, pmd_t *pmd)
set_pud(pud, __pud(__pa(pmd) | PMD_TYPE_TABLE));
}
+#ifndef CONFIG_ARM64_64K_PAGES
+
static inline pud_t *pud_alloc_one(struct mm_struct *mm, unsigned long addr)
{
return (pud_t *)get_zeroed_page(GFP_KERNEL | __GFP_REPEAT);
@@ -16,6 +16,7 @@
#ifndef __ASM_PGTABLE_3LEVEL_HWDEF_H
#define __ASM_PGTABLE_3LEVEL_HWDEF_H
+#ifndef CONFIG_ARM64_64K_PAGES
/*
* With LPAE and 4KB pages, there are 3 levels of page tables. Each level has
* 512 entries of 8 bytes each, occupying a 4K page. The first level table
@@ -47,4 +48,37 @@
#define SECTION_SIZE (_AC(1, UL) << SECTION_SHIFT)
#define SECTION_MASK (~(SECTION_SIZE-1))
+#else /* !CONFIG_ARM64_64K_PAGES */
+
+/*
+ * With 64KB pages, there are 3 levels of page tables. Each level has
+ * entries of 8 bytes each, occupying a 64K page. The first level table
+ * has 64 entries and rest of them have 8192 entries.
+ */
+#define PTRS_PER_PTE 8192
+#define PTRS_PER_PMD 8192
+#define PTRS_PER_PGD 64
+
+/*
+ * PGDIR_SHIFT determines the size a top-level page table entry can map.
+ */
+#define PGDIR_SHIFT 42
+#define PGDIR_SIZE (_AC(1, UL) << PGDIR_SHIFT)
+#define PGDIR_MASK (~(PGDIR_SIZE-1))
+
+/*
+ * PMD_SHIFT determines the size a middle-level page table entry can map.
+ */
+#define PMD_SHIFT 29
+#define PMD_SIZE (_AC(1, UL) << PMD_SHIFT)
+#define PMD_MASK (~(PMD_SIZE-1))
+
+/*
+ * section address mask and size definitions.
+ */
+#define SECTION_SHIFT 29
+#define SECTION_SIZE (_AC(1, UL) << SECTION_SHIFT)
+#define SECTION_MASK (~(SECTION_SIZE-1))
+
+#endif /* CONFIG_ARM64_64K_PAGES */
#endif
@@ -17,7 +17,7 @@
#define __ASM_PGTABLE_HWDEF_H
#ifdef CONFIG_ARM64_64K_PAGES
-#include <asm/pgtable-2level-hwdef.h>
+#include <asm/pgtable-3level-hwdef.h>
#else
#include <asm/pgtable-4level-hwdef.h>
#endif
@@ -47,10 +47,10 @@ extern void __pud_error(const char *file, int line, unsigned long val);
extern void __pgd_error(const char *file, int line, unsigned long val);
#define pte_ERROR(pte) __pte_error(__FILE__, __LINE__, pte_val(pte))
-#ifndef CONFIG_ARM64_64K_PAGES
#define pmd_ERROR(pmd) __pmd_error(__FILE__, __LINE__, pmd_val(pmd))
-#endif
+#ifndef CONFIG_ARM64_64K_PAGES
#define pud_ERROR(pud) __pud_error(__FILE__, __LINE__, pud_val(pud))
+#endif
#define pgd_ERROR(pgd) __pgd_error(__FILE__, __LINE__, pgd_val(pgd))
/*
@@ -299,8 +299,6 @@ static inline pte_t *pmd_page_vaddr(pmd_t pmd)
*/
#define mk_pte(page,prot) pfn_pte(page_to_pfn(page),prot)
-#ifndef CONFIG_ARM64_64K_PAGES
-
/* Find an entry in the kernel page upper directory */
#define pud_index(addr) (((addr) >> PUD_SHIFT) & (PTRS_PER_PUD - 1))
@@ -324,8 +322,6 @@ static inline pmd_t *pud_page_vaddr(pud_t pud)
return __va(pud_val(pud) & PHYS_MASK & (s32)PAGE_MASK);
}
-#endif /* CONFIG_ARM64_64K_PAGES */
-
/* to find an entry in a page-table-directory */
#define pgd_index(addr) (((addr) >> PGDIR_SHIFT) & (PTRS_PER_PGD - 1))
@@ -338,11 +334,8 @@ static inline pmd_t *pud_page_vaddr(pud_t pud)
#define pgd_present(pgd) (pgd_val(pgd))
#define pgd_bad(pgd) (!(pgd_val(pgd) & 2))
-/* Find an entry in the second-level page table.. */
#ifndef CONFIG_ARM64_64K_PAGES
-#define pmd_index(addr) (((addr) >> PMD_SHIFT) & (PTRS_PER_PMD - 1))
-
static inline void set_pgd(pgd_t *pgdp, pgd_t pgd)
{
*pgdp = pgd;
@@ -359,16 +352,26 @@ static inline pud_t *pgd_page_vaddr(pgd_t pgd)
return __va(pgd_val(pgd) & PHYS_MASK & (s32)PAGE_MASK);
}
+static inline pud_t *pud_offset(pud_t *pud, unsigned long addr)
+{
+ return (pud_t *)pgd_page_vaddr(*pud) + pud_index(addr);
+}
+
+#endif
+
+/* Find an entry in the second-level page table.. */
+#define pmd_index(addr) (((addr) >> PMD_SHIFT) & (PTRS_PER_PMD - 1))
+
+#ifndef CONFIG_ARM64_64K_PAGES
static inline pmd_t *pmd_offset(pmd_t *pmd, unsigned long addr)
{
return (pmd_t *)pud_page_vaddr(*pmd) + pmd_index(addr);
}
-
-static inline pud_t *pud_offset(pud_t *pud, unsigned long addr)
+#else
+static inline pmd_t *pmd_offset(pud_t *pud, unsigned long addr)
{
- return (pud_t *)pgd_page_vaddr(*pud) + pud_index(addr);
+ return (pmd_t *)pud_page_vaddr(*pud) + pmd_index(addr);
}
-
#endif
/* Find an entry in the third-level page table.. */
@@ -175,14 +175,12 @@ static inline void __pte_free_tlb(struct mmu_gather *tlb, pgtable_t pte,
tlb_remove_page(tlb, pte);
}
-#ifndef CONFIG_ARM64_64K_PAGES
static inline void __pmd_free_tlb(struct mmu_gather *tlb, pmd_t *pmdp,
unsigned long addr)
{
tlb_add_flush(tlb, addr);
tlb_remove_page(tlb, virt_to_page(pmdp));
}
-#endif
#define pte_free_tlb(tlb, ptep, addr) __pte_free_tlb(tlb, ptep, addr)
#define pmd_free_tlb(tlb, pmdp, addr) __pmd_free_tlb(tlb, pmdp, addr)
@@ -45,7 +45,11 @@
#error KERNEL_RAM_VADDR must start at 0xXXX80000
#endif
+#ifdef CONFIG_ARM64_64K_PAGES
+#define create_page_entry create_pmd_entry
+#else
#define create_page_entry create_pud_entry
+#endif
#define SWAPPER_DIR_SIZE (4 * PAGE_SIZE)
#define IDMAP_DIR_SIZE (3 * PAGE_SIZE)
@@ -391,6 +395,22 @@ ENDPROC(__calc_phys_offset)
str \tmp2, [\pud, \tmp1, lsl #3]
.endm
+#ifdef CONFIG_ARM64_64K_PAGES
+/*
+ * Macro to populate the PMD for the corresponding block entry in the next
+ * level (tbl) for the given virtual address.
+ *
+ * Preserves: pmd, tbl, virt
+ * Corrupts: tmp1, tmp2
+ */
+ .macro create_pmd_entry, pud, tbl, virt, tmp1, tmp2
+ lsr \tmp1, \virt, #PMD_SHIFT
+ and \tmp1, \tmp1, #PTRS_PER_PMD - 1 // PMD index
+ orr \tmp2, \tbl, #3 // PMD entry table type
+ str \tmp2, [\pud, \tmp1, lsl #3]
+ .endm
+#endif
+
/*
* Macro to populate block entries in the page table for the start..end
* virtual range (inclusive).
@@ -489,7 +509,11 @@ __create_page_tables:
* later based earlyprintk kernel parameter.
*/
ldr x5, =EARLYCON_IOBASE // UART virtual address
+#ifndef CONFIG_ARM64_64K_PAGES
add x0, x26, #PAGE_SIZE // section table address
+#else
+ add x0, x26, #2 * PAGE_SIZE // section table address
+#endif
create_pgd_entry x26, x0, x5, x6, x7
add x1, x0, #2 * PAGE_SIZE
create_page_entry x0, x1, x5, x6, x7
@@ -336,10 +336,12 @@ void __pmd_error(const char *file, int line, unsigned long val)
printk("%s:%d: bad pmd %016lx.\n", file, line, val);
}
+#ifndef CONFIG_ARM64_64K_PAGES
void __pud_error(const char *file, int line, unsigned long val)
{
printk("%s:%d: bad pud %016lx.\n", file, line, val);
}
+#endif
void __pgd_error(const char *file, int line, unsigned long val)
{