@@ -21,6 +21,10 @@ static inline bool alloc_can_use_metadata_pages(gfp_t gfp_mask)
#define page_has_metadata(page) page_mte_tagged(page)
+static inline bool folio_has_metadata(struct folio *folio)
+{
+ return page_has_metadata(&folio->page);
+}
#endif /* CONFIG_MEMORY_METADATA */
#endif /* __ASM_MEMORY_METADATA_H */
@@ -20,6 +20,10 @@ static inline bool page_has_metadata(struct page *page)
{
return false;
}
+static inline bool folio_has_metadata(struct folio *folio)
+{
+ return false;
+}
#endif /* !CONFIG_MEMORY_METADATA */
#endif /* __ASM_GENERIC_MEMORY_METADATA_H */
@@ -103,6 +103,7 @@
#include <linux/printk.h>
#include <linux/swapops.h>
+#include <asm/memory_metadata.h>
#include <asm/tlbflush.h>
#include <asm/tlb.h>
#include <linux/uaccess.h>
@@ -1219,6 +1220,9 @@ static struct folio *new_folio(struct folio *src, unsigned long start)
if (folio_test_large(src))
gfp = GFP_TRANSHUGE;
+ if (folio_has_metadata(src))
+ gfp |= __GFP_TAGGED;
+
/*
* if !vma, vma_alloc_folio() will use task or system default policy
*/
@@ -51,6 +51,7 @@
#include <linux/sched/sysctl.h>
#include <linux/memory-tiers.h>
+#include <asm/memory_metadata.h>
#include <asm/tlbflush.h>
#include <trace/events/migrate.h>
@@ -1990,6 +1991,9 @@ struct folio *alloc_migration_target(struct folio *src, unsigned long private)
if (nid == NUMA_NO_NODE)
nid = folio_nid(src);
+ if (folio_has_metadata(src))
+ gfp_mask |= __GFP_TAGGED;
+
if (folio_test_hugetlb(src)) {
struct hstate *h = folio_hstate(src);
@@ -2476,6 +2480,8 @@ static struct folio *alloc_misplaced_dst_folio(struct folio *src,
__GFP_NOWARN;
gfp &= ~__GFP_RECLAIM;
}
+ if (folio_has_metadata(src))
+ gfp |= __GFP_TAGGED;
return __folio_alloc_node(gfp, order, nid);
}
With explicit metadata page management support, it's important to know if the source page for migration has metadata associated with it for two reasons: - The page allocator knows to skip metadata pages (which cannot have metadata) when allocating the destination page. - The associated metadata page is correctly reserved when fulfilling the allocation for the destination page. When choosing the destination during migration, keep track if the source page has metadata. The mbind() system call changes the NUMA allocation policy for the specified memory range and nodemask. If the MPOL_MF_MOVE or MPOL_MF_MOVE_ALL flags are set, then any existing allocations that fall within the range which don't conform to the specified policy will be migrated. The function that allocates the destination page for migration is new_page(), teach it too about source pages with metadata. Signed-off-by: Alexandru Elisei <alexandru.elisei@arm.com> --- arch/arm64/include/asm/memory_metadata.h | 4 ++++ include/asm-generic/memory_metadata.h | 4 ++++ mm/mempolicy.c | 4 ++++ mm/migrate.c | 6 ++++++ 4 files changed, 18 insertions(+)