@@ -23,6 +23,9 @@ static __always_inline pgd_t *asi_pgd(struct asi *asi)
return asi ? asi->pgd : NULL;
}
+void asi_map(struct page *page, int numpages);
+void asi_unmap(struct page *page, int numpages);
+
#endif /* CONFIG_MITIGATION_ADDRESS_SPACE_ISOLATION */
#endif /* _ASM_X86_ASI_H */
@@ -1,8 +1,85 @@
// SPDX-License-Identifier: GPL-2.0
+#include <linux/bug.h>
+#include <linux/gfp.h>
+#include <linux/mm.h>
+#include <linux/mm_types.h>
+#include <linux/pgtable.h>
+#include <linux/set_memory.h>
+#include <linux/vmalloc.h>
+
#include <asm/asi.h>
+#include <asm/traps.h>
static __aligned(PAGE_SIZE) pgd_t asi_global_nonsensitive_pgd[PTRS_PER_PGD];
struct asi __asi_global_nonsensitive = {
.pgd = asi_global_nonsensitive_pgd,
};
+
+/*
+ * Map the given pages into the ASI nonsensitive physmap. The source of the
+ * mapping is the regular unrestricted page tables. Only supports mapping at
+ * pageblock granularity. Does no synchronization.
+ */
+void asi_map(struct page *page, int numpages)
+{
+ unsigned long virt;
+ unsigned long start = (size_t)(page_to_virt(page));
+ unsigned long end = start + PAGE_SIZE * numpages;
+ unsigned long page_size;
+
+ VM_BUG_ON(!IS_ALIGNED(page_to_pfn(page), pageblock_nr_pages));
+ VM_BUG_ON(!IS_ALIGNED(numpages, pageblock_nr_pages));
+
+ for (virt = start; virt < end; virt = ALIGN(virt + 1, page_size)) {
+ pte_t *pte, *pte_asi;
+ int level, level_asi;
+ pgd_t *pgd = pgd_offset_pgd(asi_global_nonsensitive_pgd, virt);
+
+ pte_asi = lookup_pgtable_in_pgd(pgd, virt, &level_asi);
+ page_size = page_level_size(level_asi);
+
+ pte = lookup_address(virt, &level);
+ if (!pte || pte_none(*pte))
+ continue;
+
+ /*
+ * Physmap should already be setup by PAT code, with no pages
+ * smaller than 2M. This function should only be called at
+ * pageblock granularity. Thus it should never be required to
+ * break up pages here.
+ */
+ if (WARN_ON_ONCE(!pte_asi) ||
+ WARN_ON_ONCE(ALIGN_DOWN(virt, page_size) < virt) ||
+ ALIGN(virt, page_size) > end)
+ continue;
+
+ /*
+ * Existing mappings should already match the structure of the
+ * unrestricted physmap.
+ */
+ if (WARN_ON_ONCE(level != level_asi))
+ continue;
+
+ set_pte(pte_asi, *pte);
+ }
+}
+
+/*
+ * Unmap pages previously mapped via asi_map().
+ *
+ * Interrupts must be enabled as this does a TLB shootdown.
+ */
+void asi_unmap(struct page *page, int numpages)
+{
+ size_t start = (size_t)page_to_virt(page);
+ size_t end = start + (PAGE_SIZE * numpages);
+ pgtbl_mod_mask mask = 0;
+
+ VM_BUG_ON(!IS_ALIGNED(page_to_pfn(page), pageblock_nr_pages));
+ VM_BUG_ON(!IS_ALIGNED(numpages, pageblock_nr_pages));
+
+ vunmap_pgd_range(asi_pgd(ASI_GLOBAL_NONSENSITIVE), start, end, &mask);
+
+ flush_tlb_kernel_range(start, end - 1);
+}
@@ -13,6 +13,8 @@
struct asi {};
static inline pgd_t *asi_pgd(struct asi *asi) { return NULL; }
+static inline void asi_map(struct page *page, int numpages) { }
+static inline void asi_unmap(struct page *page, int numpages) { }
#endif /* CONFIG_MITIGATION_ADDRESS_SPACE_ISOLATION */
#endif /* _INCLUDE_ASI_H */
@@ -11,6 +11,7 @@
#include <asm/page.h> /* pgprot_t */
#include <linux/rbtree.h>
#include <linux/overflow.h>
+#include <linux/pgtable.h>
#include <asm/vmalloc.h>
@@ -324,4 +325,7 @@ bool vmalloc_dump_obj(void *object);
static inline bool vmalloc_dump_obj(void *object) { return false; }
#endif
+void vunmap_pgd_range(pgd_t *pgd_table, unsigned long addr, unsigned long end,
+ pgtbl_mod_mask *mask);
+
#endif /* _LINUX_VMALLOC_H */
@@ -427,6 +427,24 @@ static void vunmap_p4d_range(pgd_t *pgd, unsigned long addr, unsigned long end,
} while (p4d++, addr = next, addr != end);
}
+void vunmap_pgd_range(pgd_t *pgd_table, unsigned long addr, unsigned long end,
+ pgtbl_mod_mask *mask)
+{
+ unsigned long next;
+ pgd_t *pgd = pgd_offset_pgd(pgd_table, addr);
+
+ BUG_ON(addr >= end);
+
+ do {
+ next = pgd_addr_end(addr, end);
+ if (pgd_bad(*pgd))
+ *mask |= PGTBL_PGD_MODIFIED;
+ if (pgd_none_or_clear_bad(pgd))
+ continue;
+ vunmap_p4d_range(pgd, addr, next, mask);
+ } while (pgd++, addr = next, addr != end);
+}
+
/*
* vunmap_range_noflush is similar to vunmap_range, but does not
* flush caches or TLBs.
@@ -441,21 +459,9 @@ static void vunmap_p4d_range(pgd_t *pgd, unsigned long addr, unsigned long end,
*/
void __vunmap_range_noflush(unsigned long start, unsigned long end)
{
- unsigned long next;
- pgd_t *pgd;
- unsigned long addr = start;
pgtbl_mod_mask mask = 0;
- BUG_ON(addr >= end);
- pgd = pgd_offset_k(addr);
- do {
- next = pgd_addr_end(addr, end);
- if (pgd_bad(*pgd))
- mask |= PGTBL_PGD_MODIFIED;
- if (pgd_none_or_clear_bad(pgd))
- continue;
- vunmap_p4d_range(pgd, addr, next, &mask);
- } while (pgd++, addr = next, addr != end);
+ vunmap_pgd_range(init_mm.pgd, start, end, &mask);
if (mask & ARCH_PAGE_TABLE_SYNC_MASK)
arch_sync_kernel_mappings(start, end);
Do not worry too much about the implementation of this. I am trying to implement this more neatly using the x86 PAT logic but haven't managed to get it working in time for the RFC. To enable testing & review, I'm very hastily throwing something together that basically works, based on a simplified version of what was used for the latest RFC [0]. [0] https://lore.kernel.org/linux-mm/20250110-asi-rfc-v2-v2-0-8419288bc805@google.com/ Signed-off-by: Brendan Jackman <jackmanb@google.com> --- arch/x86/include/asm/asi.h | 3 ++ arch/x86/mm/asi.c | 77 ++++++++++++++++++++++++++++++++++++++++++++++ include/linux/asi.h | 2 ++ include/linux/vmalloc.h | 4 +++ mm/vmalloc.c | 32 +++++++++++-------- 5 files changed, 105 insertions(+), 13 deletions(-)