@@ -22,3 +22,23 @@ Description: Enable/disable demoting pages during reclaim
the guarantees of cpusets. This should not be enabled
on systems which need strict cpuset location
guarantees.
+
+What: /sys/kernel/mm/numa/pagecache_promotion_enabled
+Date: November 2024
+Contact: Linux memory management mailing list <linux-mm@kvack.org>
+Description: Enable/disable promoting pages during file access
+
+ Page migration during file access is intended for systems
+ with tiered memory configurations that have significant
+ unmapped file cache usage. By default, file cache memory
+ on slower tiers will not be opportunistically promoted by
+ normal NUMA hint faults, because the system has no way to
+ track them. This option enables opportunistic promotion
+ of pages that are accessed via syscall (e.g. read/write)
+ if multiple accesses occur in quick succession.
+
+ It may move data to a NUMA node that does not fall into
+ the cpuset of the allocating process which might be
+ construed to violate the guarantees of cpusets. This
+ should not be enabled on systems which need strict cpuset
+ location guarantees.
@@ -37,6 +37,7 @@ struct access_coordinate;
#ifdef CONFIG_NUMA
extern bool numa_demotion_enabled;
+extern bool numa_pagecache_promotion_enabled;
extern struct memory_dev_type *default_dram_type;
extern nodemask_t default_dram_nodes;
struct memory_dev_type *alloc_memory_type(int adistance);
@@ -76,6 +77,7 @@ static inline bool node_is_toptier(int node)
#else
#define numa_demotion_enabled false
+#define numa_pagecache_promotion_enabled false
#define default_dram_type NULL
#define default_dram_nodes NODE_MASK_NONE
/*
@@ -146,6 +146,7 @@ int migrate_misplaced_folio_prepare(struct folio *folio,
struct vm_area_struct *vma, int node);
int migrate_misplaced_folio(struct folio *folio, struct vm_area_struct *vma,
int node);
+void promotion_candidate(struct folio *folio);
#else
static inline int migrate_misplaced_folio_prepare(struct folio *folio,
struct vm_area_struct *vma, int node)
@@ -157,6 +158,9 @@ static inline int migrate_misplaced_folio(struct folio *folio,
{
return -EAGAIN; /* can't migrate now */
}
+static inline void promotion_candidate(struct folio *folio)
+{
+}
#endif /* CONFIG_NUMA_BALANCING */
#ifdef CONFIG_MIGRATION
@@ -1353,6 +1353,9 @@ struct task_struct {
unsigned long numa_faults_locality[3];
unsigned long numa_pages_migrated;
+
+ struct callback_head numa_promo_work;
+ struct list_head promo_list;
#endif /* CONFIG_NUMA_BALANCING */
#ifdef CONFIG_RSEQ
@@ -32,6 +32,7 @@ extern void set_numabalancing_state(bool enabled);
extern void task_numa_free(struct task_struct *p, bool final);
bool should_numa_migrate_memory(struct task_struct *p, struct folio *folio,
int src_nid, int dst_cpu);
+int numa_hint_fault_latency(struct folio *folio);
#else
static inline void task_numa_fault(int last_node, int node, int pages,
int flags)
@@ -52,6 +53,10 @@ static inline bool should_numa_migrate_memory(struct task_struct *p,
{
return true;
}
+static inline int numa_hint_fault_latency(struct folio *folio)
+{
+ return 0;
+}
#endif
#endif /* _LINUX_SCHED_NUMA_BALANCING_H */
@@ -186,6 +186,7 @@ struct task_struct init_task __aligned(L1_CACHE_BYTES) = {
.numa_preferred_nid = NUMA_NO_NODE,
.numa_group = NULL,
.numa_faults = NULL,
+ .promo_list = LIST_HEAD_INIT(init_task.promo_list),
#endif
#if defined(CONFIG_KASAN_GENERIC) || defined(CONFIG_KASAN_SW_TAGS)
.kasan_depth = 1,
@@ -42,6 +42,7 @@
#include <linux/interrupt.h>
#include <linux/memory-tiers.h>
#include <linux/mempolicy.h>
+#include <linux/migrate.h>
#include <linux/mutex_api.h>
#include <linux/profile.h>
#include <linux/psi.h>
@@ -1842,7 +1843,7 @@ static bool pgdat_free_space_enough(struct pglist_data *pgdat)
* The smaller the hint page fault latency, the higher the possibility
* for the page to be hot.
*/
-static int numa_hint_fault_latency(struct folio *folio)
+int numa_hint_fault_latency(struct folio *folio)
{
int last_time, time;
@@ -3528,6 +3529,27 @@ static void task_numa_work(struct callback_head *work)
}
}
+static void task_numa_promotion_work(struct callback_head *work)
+{
+ struct task_struct *p = current;
+ struct list_head *promo_list = &p->promo_list;
+ struct folio *folio, *tmp;
+ int nid = numa_node_id();
+
+ SCHED_WARN_ON(p != container_of(work, struct task_struct, numa_promo_work));
+
+ work->next = work;
+
+ if (list_empty(promo_list))
+ return;
+
+ list_for_each_entry_safe(folio, tmp, promo_list, lru) {
+ list_del_init(&folio->lru);
+ migrate_misplaced_folio(folio, NULL, nid);
+ }
+}
+
+
void init_numa_balancing(unsigned long clone_flags, struct task_struct *p)
{
int mm_users = 0;
@@ -3552,8 +3574,10 @@ void init_numa_balancing(unsigned long clone_flags, struct task_struct *p)
RCU_INIT_POINTER(p->numa_group, NULL);
p->last_task_numa_placement = 0;
p->last_sum_exec_runtime = 0;
+ INIT_LIST_HEAD(&p->promo_list);
init_task_work(&p->numa_work, task_numa_work);
+ init_task_work(&p->numa_promo_work, task_numa_promotion_work);
/* New address space, reset the preferred nid */
if (!(clone_flags & CLONE_VM)) {
@@ -935,6 +935,7 @@ static int __init memory_tier_init(void)
subsys_initcall(memory_tier_init);
bool numa_demotion_enabled = false;
+bool numa_pagecache_promotion_enabled;
#ifdef CONFIG_MIGRATION
#ifdef CONFIG_SYSFS
@@ -957,11 +958,37 @@ static ssize_t demotion_enabled_store(struct kobject *kobj,
return count;
}
+static ssize_t pagecache_promotion_enabled_show(struct kobject *kobj,
+ struct kobj_attribute *attr,
+ char *buf)
+{
+ return sysfs_emit(buf, "%s\n",
+ numa_pagecache_promotion_enabled ? "true" : "false");
+}
+
+static ssize_t pagecache_promotion_enabled_store(struct kobject *kobj,
+ struct kobj_attribute *attr,
+ const char *buf, size_t count)
+{
+ ssize_t ret;
+
+ ret = kstrtobool(buf, &numa_pagecache_promotion_enabled);
+ if (ret)
+ return ret;
+
+ return count;
+}
+
+
static struct kobj_attribute numa_demotion_enabled_attr =
__ATTR_RW(demotion_enabled);
+static struct kobj_attribute numa_pagecache_promotion_enabled_attr =
+ __ATTR_RW(pagecache_promotion_enabled);
+
static struct attribute *numa_attrs[] = {
&numa_demotion_enabled_attr.attr,
+ &numa_pagecache_promotion_enabled_attr.attr,
NULL,
};
@@ -44,6 +44,8 @@
#include <linux/sched/sysctl.h>
#include <linux/memory-tiers.h>
#include <linux/pagewalk.h>
+#include <linux/sched/numa_balancing.h>
+#include <linux/task_work.h>
#include <asm/tlbflush.h>
@@ -2711,5 +2713,59 @@ int migrate_misplaced_folio(struct folio *folio, struct vm_area_struct *vma,
BUG_ON(!list_empty(&migratepages));
return nr_remaining ? -EAGAIN : 0;
}
+
+/**
+ * promotion_candidate() - report a promotion candidate folio
+ *
+ * @folio: The folio reported as a candidate
+ *
+ * Records folio access time and places the folio on the task promotion list
+ * if access time is less than the threshold. The folio will be isolated from
+ * LRU if selected, and task_work will putback the folio on promotion failure.
+ *
+ * Takes a folio reference that will be released in task work.
+ */
+void promotion_candidate(struct folio *folio)
+{
+ struct task_struct *task = current;
+ struct list_head *promo_list = &task->promo_list;
+ struct callback_head *work = &task->numa_promo_work;
+ struct address_space *mapping = folio_mapping(folio);
+ bool write = mapping ? mapping->gfp_mask & __GFP_WRITE : false;
+ int nid = folio_nid(folio);
+ int flags, last_cpupid;
+
+ /*
+ * Only do this work if:
+ * 1) tiering and pagecache promotion are enabled
+ * 2) the page can actually be promoted
+ * 3) The hint-fault latency is relatively hot
+ * 4) the folio is not already isolated
+ * 5) This is not a kernel thread context
+ */
+ if (!(sysctl_numa_balancing_mode & NUMA_BALANCING_MEMORY_TIERING) ||
+ !numa_pagecache_promotion_enabled ||
+ node_is_toptier(nid) ||
+ numa_hint_fault_latency(folio) >= PAGE_ACCESS_TIME_MASK ||
+ folio_test_isolated(folio) ||
+ (current->flags & PF_KTHREAD)) {
+ return;
+ }
+
+ nid = numa_migrate_check(folio, NULL, 0, &flags, write, &last_cpupid);
+ if (nid == NUMA_NO_NODE)
+ return;
+
+ if (migrate_misplaced_folio_prepare(folio, NULL, nid))
+ return;
+
+ /* Ensure task can schedule work, otherwise we'll leak folios */
+ if (list_empty(promo_list) && task_work_add(task, work, TWA_RESUME)) {
+ folio_putback_lru(folio);
+ return;
+ }
+ list_add(&folio->lru, promo_list);
+}
+EXPORT_SYMBOL(promotion_candidate);
#endif /* CONFIG_NUMA_BALANCING */
#endif /* CONFIG_NUMA */
@@ -37,6 +37,7 @@
#include <linux/page_idle.h>
#include <linux/local_lock.h>
#include <linux/buffer_head.h>
+#include <linux/migrate.h>
#include "internal.h"
@@ -453,6 +454,8 @@ void folio_mark_accessed(struct folio *folio)
__lru_cache_activate_folio(folio);
folio_clear_referenced(folio);
workingset_activation(folio);
+ } else {
+ promotion_candidate(folio);
}
if (folio_test_idle(folio))
folio_clear_idle(folio);
adds /sys/kernel/mm/numa/pagecache_promotion_enabled When page cache lands on lower tiers, there is no way for promotion to occur unless it becomes memory-mapped and exposed to NUMA hint faults. Just adding a mechanism to promote pages unconditionally, however, opens up significant possibility of performance regressions. Similar to the `demotion_enabled` sysfs entry, provide a sysfs toggle to enable and disable page cache promotion. This option will enable opportunistic promotion of unmapped page cache during syscall access. This option is intended for operational conditions where demoted page cache will eventually contain memory which becomes hot - and where said memory likely to cause performance issues due to being trapped on the lower tier of memory. A Page Cache folio is considered a promotion candidates when: 0) tiering and pagecache-promotion are enabled 1) the folio reside on a node not in the top tier 2) the folio is already marked referenced and active. 3) Multiple accesses in (referenced & active) state occur quickly. Since promotion is not safe to execute unconditionally from within folio_mark_accessed, we defer promotion to a new task_work captured in the task_struct. This ensures that the task doing the access has some hand in promoting pages - even among deduplicated read only files. We use numa_hint_fault_latency to help identify when a folio is accessed multiple times in a short period. Along with folio flag checks, this helps us minimize promoting pages on the first few accesses. The promotion node is always the local node of the promoting cpu. Suggested-by: Johannes Weiner <hannes@cmpxchg.org> Signed-off-by: Gregory Price <gourry@gourry.net> --- .../ABI/testing/sysfs-kernel-mm-numa | 20 +++++++ include/linux/memory-tiers.h | 2 + include/linux/migrate.h | 4 ++ include/linux/sched.h | 3 + include/linux/sched/numa_balancing.h | 5 ++ init/init_task.c | 1 + kernel/sched/fair.c | 26 ++++++++- mm/memory-tiers.c | 27 +++++++++ mm/migrate.c | 56 +++++++++++++++++++ mm/swap.c | 3 + 10 files changed, 146 insertions(+), 1 deletion(-)