@@ -609,11 +609,15 @@ alternative_endif
* but we have to add an offset so that the TTBR1 address corresponds with the
* pgdir entry that covers the lowest 48-bit addressable VA.
*
+ * Note that this trick only works for 64k pages - 4k pages uses an additional
+ * paging level, and on 16k pages, we would end up with a TTBR address that is
+ * not 64 byte aligned.
+ *
* orr is used as it can cover the immediate value (and is idempotent).
* ttbr: Value of ttbr to set, modified.
*/
.macro offset_ttbr1, ttbr, tmp
-#ifdef CONFIG_ARM64_VA_BITS_52
+#if defined(CONFIG_ARM64_VA_BITS_52) && defined(CONFIG_ARM64_64K_PAGES)
mrs \tmp, tcr_el1
and \tmp, \tmp, #TCR_T1SZ_MASK
cmp \tmp, #TCR_T1SZ(VA_BITS_MIN)
@@ -622,17 +626,6 @@ alternative_endif
#endif
.endm
-/*
- * Perform the reverse of offset_ttbr1.
- * bic is used as it can cover the immediate value and, in future, won't need
- * to be nop'ed out when dealing with 52-bit kernel VAs.
- */
- .macro restore_ttbr1, ttbr
-#ifdef CONFIG_ARM64_VA_BITS_52
- bic \ttbr, \ttbr, #TTBR1_BADDR_4852_OFFSET
-#endif
- .endm
-
/*
* Arrange a physical address in a TTBR register, taking care of 52-bit
* addresses.
@@ -16,6 +16,7 @@
#include <linux/refcount.h>
#include <asm/cpufeature.h>
+#include <asm/pgtable-prot.h>
typedef struct {
atomic64_t id;
@@ -72,6 +73,23 @@ extern void *fixmap_remap_fdt(phys_addr_t dt_phys, int *size, pgprot_t prot);
extern void mark_linear_text_alias_ro(void);
extern bool kaslr_requires_kpti(void);
+static inline void sync_kernel_pgdir_root_entries(pgd_t *pgdir)
+{
+ /*
+ * On 16k pages, we cannot advance the TTBR1 address to the pgdir entry
+ * that covers the start of the 48-bit addressable kernel VA space like
+ * we do on 64k pages when the hardware does not support LPA2, since the
+ * resulting address would not be 64 byte aligned. So instead, copy the
+ * pgdir entry that covers the mapping we just created to the start of
+ * the page table.
+ */
+ if (IS_ENABLED(CONFIG_ARM64_16K_PAGES) &&
+ VA_BITS > VA_BITS_MIN && !lpa2_is_enabled()) {
+ pgdir[0] = pgdir[PTRS_PER_PGD - 2];
+ pgdir[1] = pgdir[PTRS_PER_PGD - 1];
+ }
+}
+
#define INIT_MM_CONTEXT(name) \
.pgd = swapper_pg_dir,
@@ -1768,6 +1768,7 @@ kpti_install_ng_mappings(const struct arm64_cpu_capabilities *__unused)
create_kpti_ng_temp_pgd(kpti_ng_temp_pgd, __pa(alloc),
KPTI_NG_TEMP_VA, PAGE_SIZE, PAGE_KERNEL,
kpti_ng_pgd_alloc, 0);
+ sync_kernel_pgdir_root_entries(kpti_ng_temp_pgd);
}
cpu_install_idmap();
@@ -217,8 +217,8 @@ static void __init map_kernel(u64 kaslr_offset, u64 va_offset)
map_segment(init_pg_dir, &pgdp, va_offset, __initdata_begin,
__initdata_end, data_prot, false);
map_segment(init_pg_dir, &pgdp, va_offset, _data, _end, data_prot, true);
+ sync_kernel_pgdir_root_entries(init_pg_dir);
dsb(ishst);
-
idmap_cpu_replace_ttbr1(init_pg_dir);
if (twopass) {
@@ -665,6 +665,7 @@ static int __init map_entry_trampoline(void)
__create_pgd_mapping(tramp_pg_dir, pa_start, TRAMP_VALIAS,
entry_tramp_text_size(), prot,
__pgd_pgtable_alloc, NO_BLOCK_MAPPINGS);
+ sync_kernel_pgdir_root_entries(tramp_pg_dir);
/* Map both the text and data into the kernel page table */
for (i = 0; i < DIV_ROUND_UP(entry_tramp_text_size(), PAGE_SIZE); i++)
@@ -729,6 +730,7 @@ void __init paging_init(void)
idmap_t0sz = 63UL - __fls(__pa_symbol(_end) | GENMASK(VA_BITS_MIN - 1, 0));
map_mem(swapper_pg_dir);
+ sync_kernel_pgdir_root_entries(swapper_pg_dir);
memblock_allow_resize();
On LVA/64k granule configurations, we simply extend the level 1 root page table to cover 52 bits of VA space, and if the system in question only supports 48 bits, we point TTBR1 to the pgdir entry that covers the start of the 48-bit addressable part of the VA space. Sadly, we cannot use the same trick on LPA2/16k granule configurations. This is due to the fact that TTBR registers require 64 byte aligned addresses, while the 48-bit addressable entries in question will not appear at a 64 byte aligned address if the entire 52-bit VA table is aligned to its size (which is another requirement for TTBR registers). Fortunately, we are only dealing with two entries in this case: one that covers the kernel/vmalloc region and one covering the linear map. This makes it feasible to simply clone those entries into the start of the page table after the first mapping into the respective region is created. Signed-off-by: Ard Biesheuvel <ardb@kernel.org> --- arch/arm64/include/asm/assembler.h | 17 +++++------------ arch/arm64/include/asm/mmu.h | 18 ++++++++++++++++++ arch/arm64/kernel/cpufeature.c | 1 + arch/arm64/kernel/pi/map_kernel.c | 2 +- arch/arm64/mm/mmu.c | 2 ++ 5 files changed, 27 insertions(+), 13 deletions(-)