[v2,3/5] ksm: let ksmd auto-work with memory threshold

Commit Message

CGEL Aug. 12, 2022, 10:12 a.m. UTC

From: xu xin <[e-mail]>

When memory is sufficient, merging pages to save memory is not very
much needed, and it also inceases delays of COW for user application.

So set a memory threshold, when free memory is lower than the threshold,
ksmd will be triggered to compare and merge pages. And to avoid ping-pong
effect due to the threshold, ksmd needs to try to merge pages until free
memory is larger than (threshold + total_memory * 1/16).

Before free memory is lower than the threshold, ksmd will still scan pages
at a very low speed, to calculate their checksum but not to compare and
merge pages.

        |
        |       ----(Threshold + total_memory/16)--------
        |                              |
------Threshold------                  |
        |                              |
        |_____ksmd try to merge pages__|

We also add a new sysfs klob auto_threshold_percent for user to be able
to tune.

Signed-off-by: xu xin <[e-mail]>
Signed-off-by: CGEL <[e-mail]>
---
 mm/ksm.c | 103 +++++++++++++++++++++++++++++++++++++++++++++++++++++--
 1 file changed, 101 insertions(+), 2 deletions(-)

Patch

diff --git a/mm/ksm.c b/mm/ksm.c
index f416f168a6da..c5fd4f520f4a 100644
--- a/mm/ksm.c
+++ b/mm/ksm.c
@@ -296,6 +296,17 @@  static unsigned int scanning_factor = INIT_SCANNING_FACTOR;
 #define DEFAULT_MAX_SCANNING_FACTOR	16
 static unsigned int max_scanning_factor	= DEFAULT_MAX_SCANNING_FACTOR;
 
+/*
+ * Work in auto mode.
+ * Value: 0~100. Default 20 means "20%". When free memory is lower
+ * than this total memory * ksm_auto_threshold/100, auto_triggered
+ * will be set true.
+ */
+unsigned int ksm_auto_threshold = 20;
+
+/* Work in auto-mode. Whether trigger ksmd to compare and merge pages */
+static bool auto_triggered;
+
 #ifdef CONFIG_NUMA
 /* Zeroed when merging across nodes is not allowed */
 static unsigned int ksm_merge_across_nodes = 1;
@@ -2431,11 +2442,61 @@  static void ksm_do_scan(unsigned int scan_npages)
 		rmap_item = scan_get_next_rmap_item(&page);
 		if (!rmap_item)
 			return;
-		cmp_and_merge_page(page, rmap_item);
+		if (ksm_run & KSM_RUN_AUTO  && !auto_triggered) {
+			/*
+			 * This should happens only when ksm_run is KSM_RUN_AUTO
+			 * and free memory threshold still not reached.
+			 * The reason to calculate it's checksum is to reduce the
+			 * waiting time the rmap_item is added to unstable tree.
+			 */
+			rmap_item->oldchecksum = calc_checksum(page);
+		} else
+			cmp_and_merge_page(page, rmap_item);
+
 		put_page(page);
 	}
 }
 
+#define RIGHT_SHIFT_FOUR_BIT	4
+/* Work in auto mode, should reset auto_triggered ? */
+static bool should_stop_ksmd_to_merge(void)
+{
+	unsigned long total_ram_pages, free_pages;
+	unsigned int threshold;
+
+	total_ram_pages = totalram_pages();
+	free_pages = global_zone_page_state(NR_FREE_PAGES);
+	threshold = READ_ONCE(ksm_auto_threshold);
+
+	return free_pages > (total_ram_pages * threshold / 100) +
+		        (total_ram_pages >> RIGHT_SHIFT_FOUR_BIT);
+}
+
+/* Work in auto mode, should ksmd start to merge ? */
+static bool should_trigger_ksmd_to_merge(void)
+{
+	unsigned long total_ram_pages, free_pages;
+	unsigned int threshold;
+
+	total_ram_pages = totalram_pages();
+	free_pages = global_zone_page_state(NR_FREE_PAGES);
+	threshold = READ_ONCE(ksm_auto_threshold);
+
+	return free_pages < (total_ram_pages * threshold / 100);
+}
+
+static inline void trigger_ksmd_to_merge(void)
+{
+	if (!auto_triggered)
+		auto_triggered = true;
+}
+
+static inline void stop_ksmd_to_merge(void)
+{
+	if (auto_triggered)
+		auto_triggered = false;
+}
+
 static int ksmd_should_run(void)
 {
 	if (!list_empty(&ksm_mm_head.mm_list))
@@ -2478,6 +2539,8 @@  static unsigned int scan_enhanced_algorithm(unsigned int current_factor)
 	return next_factor;
 }
 
+#define SLOW_SCAN_PAGES	5 /* Used when ksmd is not triggered to merge*/
+
 static int ksm_scan_thread(void *nothing)
 {
 	unsigned int sleep_ms;
@@ -2490,7 +2553,10 @@  static int ksm_scan_thread(void *nothing)
 		wait_while_offlining();
 		if (ksmd_should_run()) {
 			if (ksm_run & KSM_RUN_AUTO) {
-				ksm_do_scan(ksm_thread_pages_to_scan * scanning_factor);
+				if (!auto_triggered)
+					ksm_do_scan(SLOW_SCAN_PAGES);
+				else
+					ksm_do_scan(ksm_thread_pages_to_scan * scanning_factor);
 
 				scanning_factor = scan_enhanced_algorithm(scanning_factor);
 				/*
@@ -2498,6 +2564,11 @@  static int ksm_scan_thread(void *nothing)
 				 * updating scanning_factor by scan_enhanced_algorithm.
 				 */
 				ksm_scan.new_ksmpages = 0;
+
+				if (should_trigger_ksmd_to_merge())
+					trigger_ksmd_to_merge();
+				else if (should_stop_ksmd_to_merge())
+					stop_ksmd_to_merge();
 			} else
 				ksm_do_scan(ksm_thread_pages_to_scan);
 		}
@@ -3047,6 +3118,32 @@  static ssize_t run_store(struct kobject *kobj, struct kobj_attribute *attr,
 }
 KSM_ATTR(run);
 
+static ssize_t auto_threshold_show(struct kobject *kobj,
+						struct kobj_attribute *attr, char *buf)
+{
+	return sysfs_emit(buf, "%u\n", ksm_auto_threshold);
+}
+
+static ssize_t auto_threshold_store(struct kobject *kobj,
+								struct kobj_attribute *attr,
+								const char *buf, size_t count)
+{
+	unsigned int value;
+	int err;
+
+	err = kstrtouint(buf, 10, &value);
+	if (err)
+		return -EINVAL;
+
+	if (value > 100)
+		return -EINVAL;
+
+	ksm_auto_threshold = value;
+
+	return count;
+}
+KSM_ATTR(auto_threshold);
+
 #ifdef CONFIG_NUMA
 static ssize_t merge_across_nodes_show(struct kobject *kobj,
 				       struct kobj_attribute *attr, char *buf)
@@ -3258,6 +3355,7 @@  static struct attribute *ksm_attrs[] = {
 	&pages_to_scan_attr.attr,
 	&max_scanning_factor_attr.attr,
 	&run_attr.attr,
+	&auto_threshold_attr.attr,
 	&pages_shared_attr.attr,
 	&pages_sharing_attr.attr,
 	&pages_unshared_attr.attr,
@@ -3289,6 +3387,7 @@  static int __init ksm_init(void)
 	zero_checksum = calc_checksum(ZERO_PAGE(0));
 	/* Default to false for backwards compatibility */
 	ksm_use_zero_pages = false;
+	auto_triggered = false;
 
 	err = ksm_slab_init();
 	if (err)