[1/7] riscv: mm: Pre-allocate PGD leaves to avoid synchronization

Commit Message

Björn Töpel May 12, 2023, 2:57 p.m. UTC

From: Björn Töpel <[e-mail]>

The RISC-V port copies PGD from init_mm to all userland pages-tables,
which means that when the PGD level of the init_mm table is changed,
other page-tables has to be updated.

One way to avoid synchronizing page-tables is to pre-allocate the
pages that are copied (need to be synchronized). For memory
hotswapping builds, prefer to waste some pages, rather than do
explicit synchronization.

Prepare the RISC-V port for memory add/remove, by getting rid of PGD
synchronization. Pre-allocate vmemmap, and direct map pages. This will
roughly waste ~128 worth of 4K pages.

Note that this is only done for memory hotswap enabled configuration.

Signed-off-by: Björn Töpel <[e-mail]>
---
 arch/riscv/include/asm/kasan.h |  4 +-
 arch/riscv/mm/init.c           | 86 ++++++++++++++++++++++++++++++++++
 2 files changed, 88 insertions(+), 2 deletions(-)

Patch

diff --git a/arch/riscv/include/asm/kasan.h b/arch/riscv/include/asm/kasan.h
index 0b85e363e778..e6a0071bdb56 100644
--- a/arch/riscv/include/asm/kasan.h
+++ b/arch/riscv/include/asm/kasan.h
@@ -6,8 +6,6 @@ 
 
 #ifndef __ASSEMBLY__
 
-#ifdef CONFIG_KASAN
-
 /*
  * The following comment was copied from arm64:
  * KASAN_SHADOW_START: beginning of the kernel virtual addresses.
@@ -34,6 +32,8 @@ 
  */
 #define KASAN_SHADOW_START	((KASAN_SHADOW_END - KASAN_SHADOW_SIZE) & PGDIR_MASK)
 #define KASAN_SHADOW_END	MODULES_LOWEST_VADDR
+
+#ifdef CONFIG_KASAN
 #define KASAN_SHADOW_OFFSET	_AC(CONFIG_KASAN_SHADOW_OFFSET, UL)
 
 void kasan_init(void);
diff --git a/arch/riscv/mm/init.c b/arch/riscv/mm/init.c
index 747e5b1ef02d..d2595cc33a1c 100644
--- a/arch/riscv/mm/init.c
+++ b/arch/riscv/mm/init.c
@@ -31,6 +31,7 @@ 
 #include <asm/io.h>
 #include <asm/ptdump.h>
 #include <asm/numa.h>
+#include <asm/kasan.h>
 
 #include "../kernel/head.h"
 
@@ -156,6 +157,90 @@  static void __init print_vm_layout(void)
 static void print_vm_layout(void) { }
 #endif /* CONFIG_DEBUG_VM */
 
+#ifdef CONFIG_MEMORY_HOTPLUG
+/*
+ * Pre-allocates page-table pages for a specific area in the kernel
+ * page-table. Only the level which needs to be synchronized between
+ * all page-tables is allocated because the synchronization can be
+ * expensive.
+ */
+static void __init preallocate_pgd_pages_range(unsigned long start, unsigned long end,
+					       const char *area)
+{
+	unsigned long addr;
+	const char *lvl;
+
+	for (addr = start; addr < end; addr = ALIGN(addr + 1, PGDIR_SIZE)) {
+		pgd_t *pgd = pgd_offset_k(addr);
+		p4d_t *p4d;
+		pud_t *pud;
+		pmd_t *pmd;
+
+		lvl = "p4d";
+		p4d = p4d_alloc(&init_mm, pgd, addr);
+		if (!p4d)
+			goto failed;
+
+		if (pgtable_l5_enabled)
+			continue;
+
+		/*
+		 * The goal here is to allocate all possibly required
+		 * hardware page tables pointed to by the top hardware
+		 * level.
+		 *
+		 * On 4-level systems, the P4D layer is folded away
+		 * and the above code does no preallocation.  Below,
+		 * go down to the pud _software_ level to ensure the
+		 * second hardware level is allocated on 4-level
+		 * systems too.
+		 */
+		lvl = "pud";
+		pud = pud_alloc(&init_mm, p4d, addr);
+		if (!pud)
+			goto failed;
+
+		if (pgtable_l4_enabled)
+			continue;
+		/*
+		 * The goal here is to allocate all possibly required
+		 * hardware page tables pointed to by the top hardware
+		 * level.
+		 *
+		 * On 3-level systems, the PUD layer is folded away
+		 * and the above code does no preallocation.  Below,
+		 * go down to the pmd _software_ level to ensure the
+		 * second hardware level is allocated on 3-level
+		 * systems too.
+		 */
+		lvl = "pmd";
+		pmd = pmd_alloc(&init_mm, pud, addr);
+		if (!pmd)
+			goto failed;
+	}
+
+	return;
+
+failed:
+
+	/*
+	 * The pages have to be there now or they will be missing in
+	 * process page-tables later.
+	 */
+	panic("Failed to pre-allocate %s pages for %s area\n", lvl, area);
+}
+
+#define PAGE_END KASAN_SHADOW_START
+#endif
+
+static void __init prepare_memory_hotplug(void)
+{
+#ifdef CONFIG_MEMORY_HOTPLUG
+	preallocate_pgd_pages_range(VMEMMAP_START, VMEMMAP_END, "vmemmap");
+	preallocate_pgd_pages_range(PAGE_OFFSET, PAGE_END, "direct map");
+#endif
+}
+
 void __init mem_init(void)
 {
 #ifdef CONFIG_FLATMEM
@@ -164,6 +249,7 @@  void __init mem_init(void)
 
 	swiotlb_init(max_pfn > PFN_DOWN(dma32_phys_limit), SWIOTLB_VERBOSE);
 	memblock_free_all();
+	prepare_memory_hotplug();
 
 	print_vm_layout();
 }