[RFC,28/37] mm: sched: Introduce PF_MEMALLOC_ISOLATE

Commit Message

Alexandru Elisei Aug. 23, 2023, 1:13 p.m. UTC

On arm64, when reserving tag storage for an allocated page, if the tag
storage is in use, the tag storage must be migrated before it can be
reserved. As part of the migration process, the tag storage block is
first isolated.

Compaction also isolates the source pages before migrating them. If the
target for compaction requires metadata pages to be reserved, those
metadata pages might also need to be isolated, which, in rare
circumstances, can lead to the threshold in too_many_isolated() being
reached, and isolate_migratepages_pageblock() will get stuck in an infinite
loop.

Add the flag PF_MEMALLOC_ISOLATE for the current thread, which makes
too_many_isolated() ignore the threshold to make forward progress in
isolate_migratepages_pageblock().

For consistency, the similarly named function too_many_isolated() called
during reclaim has received the same treatment.

Signed-off-by: Alexandru Elisei <[e-mail]>
---
 arch/arm64/kernel/mte_tag_storage.c |  5 ++++-
 include/linux/sched.h               |  2 +-
 include/linux/sched/mm.h            | 13 +++++++++++++
 mm/compaction.c                     |  3 +++
 mm/vmscan.c                         |  3 +++
 5 files changed, 24 insertions(+), 2 deletions(-)

Patch

diff --git a/arch/arm64/kernel/mte_tag_storage.c b/arch/arm64/kernel/mte_tag_storage.c
index 1ab875be5f9b..ba316ffb9aef 100644
--- a/arch/arm64/kernel/mte_tag_storage.c
+++ b/arch/arm64/kernel/mte_tag_storage.c
@@ -505,9 +505,9 @@  static int order_to_num_blocks(int order)
 int reserve_metadata_storage(struct page *page, int order, gfp_t gfp)
 {
 	unsigned long start_block, end_block;
+	unsigned long flags, cflags;
 	struct tag_region *region;
 	unsigned long block;
-	unsigned long flags;
 	int i, tries;
 	int ret = 0;
 
@@ -539,6 +539,7 @@  int reserve_metadata_storage(struct page *page, int order, gfp_t gfp)
 	}
 	xa_unlock_irqrestore(&tag_blocks_reserved, flags);
 
+	cflags = memalloc_isolate_save();
 	for (block = start_block; block < end_block; block += region->block_size) {
 		/* Refcount incremented above. */
 		if (tag_storage_block_is_reserved(block))
@@ -566,6 +567,7 @@  int reserve_metadata_storage(struct page *page, int order, gfp_t gfp)
 	for (i = 0; i < (1 << order); i++)
 		set_bit(PG_tag_storage_reserved, &(page + i)->flags);
 
+	memalloc_isolate_restore(cflags);
 	mutex_unlock(&tag_blocks_lock);
 
 	return 0;
@@ -581,6 +583,7 @@  int reserve_metadata_storage(struct page *page, int order, gfp_t gfp)
 	}
 	xa_unlock_irqrestore(&tag_blocks_reserved, flags);
 
+	memalloc_isolate_restore(cflags);
 	mutex_unlock(&tag_blocks_lock);
 
 	count_vm_events(METADATA_RESERVE_FAIL, region->block_size);
diff --git a/include/linux/sched.h b/include/linux/sched.h
index 609bde814cb0..a2a930cab31a 100644
--- a/include/linux/sched.h
+++ b/include/linux/sched.h
@@ -1734,7 +1734,7 @@  extern struct pid *cad_pid;
 #define PF_USED_MATH		0x00002000	/* If unset the fpu must be initialized before use */
 #define PF_USER_WORKER		0x00004000	/* Kernel thread cloned from userspace thread */
 #define PF_NOFREEZE		0x00008000	/* This thread should not be frozen */
-#define PF__HOLE__00010000	0x00010000
+#define PF_MEMALLOC_ISOLATE	0x00010000	/* Ignore isolation limits */
 #define PF_KSWAPD		0x00020000	/* I am kswapd */
 #define PF_MEMALLOC_NOFS	0x00040000	/* All allocation requests will inherit GFP_NOFS */
 #define PF_MEMALLOC_NOIO	0x00080000	/* All allocation requests will inherit GFP_NOIO */
diff --git a/include/linux/sched/mm.h b/include/linux/sched/mm.h
index 8d89c8c4fac1..8db491208746 100644
--- a/include/linux/sched/mm.h
+++ b/include/linux/sched/mm.h
@@ -393,6 +393,19 @@  static inline void memalloc_pin_restore(unsigned int flags)
 	current->flags = (current->flags & ~PF_MEMALLOC_PIN) | flags;
 }
 
+static inline unsigned int memalloc_isolate_save(void)
+{
+	unsigned int flags = current->flags & PF_MEMALLOC_ISOLATE;
+
+	current->flags |= PF_MEMALLOC_ISOLATE;
+	return flags;
+}
+
+static inline void memalloc_isolate_restore(unsigned int flags)
+{
+	current->flags = (current->flags & ~PF_MEMALLOC_ISOLATE) | flags;
+}
+
 #ifdef CONFIG_MEMCG
 DECLARE_PER_CPU(struct mem_cgroup *, int_active_memcg);
 /**
diff --git a/mm/compaction.c b/mm/compaction.c
index 314793ec8bdb..fdb75316f0cc 100644
--- a/mm/compaction.c
+++ b/mm/compaction.c
@@ -778,6 +778,9 @@  static bool too_many_isolated(struct compact_control *cc)
 
 	unsigned long active, inactive, isolated;
 
+	if (current->flags & PF_MEMALLOC_ISOLATE)
+		return false;
+
 	inactive = node_page_state(pgdat, NR_INACTIVE_FILE) +
 			node_page_state(pgdat, NR_INACTIVE_ANON);
 	active = node_page_state(pgdat, NR_ACTIVE_FILE) +
diff --git a/mm/vmscan.c b/mm/vmscan.c
index 1080209a568b..912ebb6003a0 100644
--- a/mm/vmscan.c
+++ b/mm/vmscan.c
@@ -2453,6 +2453,9 @@  static int too_many_isolated(struct pglist_data *pgdat, int file,
 	if (current_is_kswapd())
 		return 0;
 
+	if (current->flags & PF_MEMALLOC_ISOLATE)
+		return 0;
+
 	if (!writeback_throttling_sane(sc))
 		return 0;