[2/2] overcommit: introduce mem-lock-onfault

Commit Message

Daniil Tatianin Dec. 5, 2024, 11:19 p.m. UTC

Locking the memory without MCL_ONFAULT instantly prefaults any mmaped
anonymous memory with a write-fault, which introduces a lot of extra
overhead in terms of memory usage when all you want to do is to prevent
kcompactd from migrating and compacting QEMU pages. Add an option to
only lock pages lazily as they're faulted by the process by using
MCL_ONFAULT if asked.

Signed-off-by: Daniil Tatianin <[e-mail]>
---
 include/sysemu/sysemu.h  |  1 +
 migration/postcopy-ram.c |  4 ++--
 qemu-options.hx          | 13 ++++++++++---
 system/globals.c         |  1 +
 system/vl.c              | 18 ++++++++++++++++--
 5 files changed, 30 insertions(+), 7 deletions(-)

Patch

diff --git a/include/sysemu/sysemu.h b/include/sysemu/sysemu.h
index 7ec419ce13..b6519c3c1e 100644
--- a/include/sysemu/sysemu.h
+++ b/include/sysemu/sysemu.h
@@ -44,6 +44,7 @@  extern const char *keyboard_layout;
 extern int old_param;
 extern uint8_t *boot_splash_filedata;
 extern bool enable_mlock;
+extern bool enable_mlock_onfault;
 extern bool enable_cpu_pm;
 extern QEMUClockType rtc_clock;
 
diff --git a/migration/postcopy-ram.c b/migration/postcopy-ram.c
index 36ec6a3d75..8ff8c73a27 100644
--- a/migration/postcopy-ram.c
+++ b/migration/postcopy-ram.c
@@ -651,8 +651,8 @@  int postcopy_ram_incoming_cleanup(MigrationIncomingState *mis)
         mis->have_fault_thread = false;
     }
 
-    if (enable_mlock) {
-        if (os_mlock(false) < 0) {
+    if (enable_mlock || enable_mlock_onfault) {
+        if (os_mlock(enable_mlock_onfault) < 0) {
             error_report("mlock: %s", strerror(errno));
             /*
              * It doesn't feel right to fail at this point, we have a valid
diff --git a/qemu-options.hx b/qemu-options.hx
index dacc9790a4..477e0e439a 100644
--- a/qemu-options.hx
+++ b/qemu-options.hx
@@ -4566,21 +4566,28 @@  SRST
 ERST
 
 DEF("overcommit", HAS_ARG, QEMU_OPTION_overcommit,
-    "-overcommit [mem-lock=on|off][cpu-pm=on|off]\n"
+    "-overcommit [mem-lock=on|off][mem-lock-onfault=on|off][cpu-pm=on|off]\n"
     "                run qemu with overcommit hints\n"
     "                mem-lock=on|off controls memory lock support (default: off)\n"
+    "                mem-lock-onfault=on|off controls memory lock on fault support (default: off)\n"
     "                cpu-pm=on|off controls cpu power management (default: off)\n",
     QEMU_ARCH_ALL)
 SRST
 ``-overcommit mem-lock=on|off``
   \ 
+``-overcommit mem-lock-onfault=on|off``
+  \
 ``-overcommit cpu-pm=on|off``
     Run qemu with hints about host resource overcommit. The default is
     to assume that host overcommits all resources.
 
     Locking qemu and guest memory can be enabled via ``mem-lock=on``
-    (disabled by default). This works when host memory is not
-    overcommitted and reduces the worst-case latency for guest.
+    or ``mem-lock-onfault=on`` (disabled by default). This works when
+    host memory is not overcommitted and reduces the worst-case latency for
+    guest. The on-fault option is better for reducing the memory footprint
+    since it makes allocations lazy, but the pages still get locked in place
+    once faulted by the guest or QEMU. Note that the two options are mutually
+    exclusive.
 
     Guest ability to manage power state of host cpus (increasing latency
     for other processes on the same host cpu, but decreasing latency for
diff --git a/system/globals.c b/system/globals.c
index 84ce943ac9..43501fe690 100644
--- a/system/globals.c
+++ b/system/globals.c
@@ -35,6 +35,7 @@  enum vga_retrace_method vga_retrace_method = VGA_RETRACE_DUMB;
 int display_opengl;
 const char* keyboard_layout;
 bool enable_mlock;
+bool enable_mlock_onfault;
 bool enable_cpu_pm;
 int autostart = 1;
 int vga_interface_type = VGA_NONE;
diff --git a/system/vl.c b/system/vl.c
index 03819a80ef..89477f38bc 100644
--- a/system/vl.c
+++ b/system/vl.c
@@ -349,6 +349,10 @@  static QemuOptsList qemu_overcommit_opts = {
             .name = "mem-lock",
             .type = QEMU_OPT_BOOL,
         },
+        {
+            .name = "mem-lock-onfault",
+            .type = QEMU_OPT_BOOL,
+        },
         {
             .name = "cpu-pm",
             .type = QEMU_OPT_BOOL,
@@ -792,8 +796,8 @@  static QemuOptsList qemu_run_with_opts = {
 
 static void realtime_init(void)
 {
-    if (enable_mlock) {
-        if (os_mlock(false) < 0) {
+    if (enable_mlock || enable_mlock_onfault) {
+        if (os_mlock(enable_mlock_onfault) < 0) {
             error_report("locking memory failed");
             exit(1);
         }
@@ -3537,7 +3541,17 @@  void qemu_init(int argc, char **argv)
                 if (!opts) {
                     exit(1);
                 }
+
                 enable_mlock = qemu_opt_get_bool(opts, "mem-lock", enable_mlock);
+                enable_mlock_onfault = qemu_opt_get_bool(opts,
+                                                         "mem-lock-onfault",
+                                                         enable_mlock_onfault);
+                if (enable_mlock && enable_mlock_onfault) {
+                    error_report("mem-lock and mem-lock-onfault are mutually"
+                                 "exclusive");
+                    exit(1);
+                }
+
                 enable_cpu_pm = qemu_opt_get_bool(opts, "cpu-pm", enable_cpu_pm);
                 break;
             case QEMU_OPTION_compat: