[RFC] SLUB: Add support for per object memory policies

Commit Message

Christoph Lameter via B4 Relay Aug. 20, 2024, 1:29 a.m. UTC

From: Christoph Lameter <[e-mail]>

The old SLAB allocator used to support memory policies on a per
allocation bases. In SLUB the memory policies are applied on a
per page frame / folio bases. Doing so avoids having to check memory
policies in critical code paths for kmalloc and friends.

This worked well on Intel/AMD/PowerPC because the
interconnect technology is mature and can minimize the latencies
through intelligent caching even if some small objects are not
placed optimally.

However, on ARM we have the emergence of new NUMA interconnect
technology based more on embedded devices. Caching of remote content
can currently be ineffective using the standard building blocks / mesh
available on that platform. Such architectures benefit if each slab
object is individually placed according to memory policies.

This patch adds another kernel parameter

	slab_strict_numa

If that is set then a static branch is activated that will cause
the hotpaths of the allocator to evaluate the current memory
allocation policy. Each object will be properly placed by
paying the price of extra processing and SLUB will no longer
defer to the page allocator to apply memory policies at the
folio level.

Signed-off-by: Christoph Lameter (Ampere) <[e-mail]>
---
This is a draft to discuss the issue. It builds but was never run.
---
 mm/slub.c | 27 +++++++++++++++++++++++++++
 1 file changed, 27 insertions(+)


---
base-commit: 6e4436539ae182dc86d57d13849862bcafaa4709
change-id: 20240819-strict_numa-fc59b33123a2

Best regards,

Patch

diff --git a/mm/slub.c b/mm/slub.c
index c9d8a2497fd6..c86e7ee9520a 100644
--- a/mm/slub.c
+++ b/mm/slub.c
@@ -218,6 +218,10 @@  DEFINE_STATIC_KEY_FALSE(slub_debug_enabled);
 #endif
 #endif		/* CONFIG_SLUB_DEBUG */
 
+#ifdef CONFIG_NUMA
+DEFINE_STATIC_KEY_FALSE(strict_numa);
+#endif
+
 /* Structure holding parameters for get_partial() call chain */
 struct partial_context {
 	gfp_t flags;
@@ -3825,6 +3829,14 @@  static __always_inline void *__slab_alloc_node(struct kmem_cache *s,
 	unsigned long tid;
 	void *object;
 
+#ifdef CONFIG_NUMA
+
+	if (static_branch_unlikely(&strict_numa) && node == NUMA_NO_NODE)
+
+		node = mempolicy_slab_node();
+
+#endif
+
 redo:
 	/*
 	 * Must read kmem_cache cpu data via this cpu ptr. Preemption is
@@ -5523,6 +5535,21 @@  static int __init setup_slub_min_objects(char *str)
 __setup("slab_min_objects=", setup_slub_min_objects);
 __setup_param("slub_min_objects=", slub_min_objects, setup_slub_min_objects, 0);
 
+#ifdef CONFIG_NUMA
+static int __init setup_slab_strict_numa(char *str)
+{
+	if (nr_node_ids > 1)
+		static_branch_enable(&strict_numa);
+	else
+		pr_warn("slab_strict_numa parameter set on non NUMA system.\n");
+
+	return 1;
+}
+
+__setup("slab_strict_numa", setup_slab_strict_numa);
+#endif
+
+
 #ifdef CONFIG_HARDENED_USERCOPY
 /*
  * Rejects incorrectly sized objects and objects that are to be copied