diff mbox

[v11,15/27] primary vm suspend/resume/checkpoint code

Message ID 1457080891-26054-16-git-send-email-xiecl.fnst@cn.fujitsu.com (mailing list archive)
State New, archived
Headers show

Commit Message

Changlong Xie March 4, 2016, 8:41 a.m. UTC
From: Wen Congyang <wency@cn.fujitsu.com>

We will do the following things again and again:
1. Suspend primary vm
   a. Suspend primary vm
   b. do postsuspend
   c. Read CHECKPOINT_SVM_SUSPENDED sent by secondary
2. Checkpoint
   a. Write emulator xenstore data and emulator context
   b. Write checkpoint end record
3. Resume primary vm
   a. Read CHECKPOINT_SVM_READY from slave
   b. Do presume
   c. Resume primary vm
   d. Read CHECKPOINT_SVM_RESUMED from slave
4. Wait a new checkpoint
   a. Wait a new checkpoint(not implemented)
   b. Send CHECKPOINT_NEW to slave

Signed-off-by: Wen Congyang <wency@cn.fujitsu.com>
Signed-off-by: Yang Hongyang <hongyang.yang@easystack.cn>
Signed-off-by: Changlong Xie <xiecl.fnst@cn.fujitsu.com>
---
 tools/libxl/Makefile          |   2 +-
 tools/libxl/libxl.c           |   5 +-
 tools/libxl/libxl_colo.h      |   6 +
 tools/libxl/libxl_colo_save.c | 562 ++++++++++++++++++++++++++++++++++++++++++
 tools/libxl/libxl_dom_save.c  |   7 +-
 tools/libxl/libxl_internal.h  | 157 ++++++------
 tools/libxl/libxl_types.idl   |   1 +
 7 files changed, 666 insertions(+), 74 deletions(-)
 create mode 100644 tools/libxl/libxl_colo_save.c

Comments

Ian Jackson March 4, 2016, 5:14 p.m. UTC | #1
Changlong Xie writes ("[PATCH v11 15/27] primary vm suspend/resume/checkpoint code"):
> From: Wen Congyang <wency@cn.fujitsu.com>

I would look at this on the same basis as the previous patch.

> diff --git a/tools/libxl/libxl_internal.h b/tools/libxl/libxl_internal.h
> index 48b4858..5160939 100644
> --- a/tools/libxl/libxl_internal.h
> +++ b/tools/libxl/libxl_internal.h

> +struct libxl__stream_read_state {
> +    /* filled by the user */
> +    libxl__ao *ao;
> +    libxl__domain_create_state *dcs;
> +    int fd;
> +    bool legacy;

Can you please split out this code motion into a separate patch ?
As it is it is very difficult to review.

Thanks,
Ian.
Wen Congyang March 7, 2016, 2:59 a.m. UTC | #2
On 03/05/2016 01:14 AM, Ian Jackson wrote:
> Changlong Xie writes ("[PATCH v11 15/27] primary vm suspend/resume/checkpoint code"):
>> From: Wen Congyang <wency@cn.fujitsu.com>
> 
> I would look at this on the same basis as the previous patch.
> 
>> diff --git a/tools/libxl/libxl_internal.h b/tools/libxl/libxl_internal.h
>> index 48b4858..5160939 100644
>> --- a/tools/libxl/libxl_internal.h
>> +++ b/tools/libxl/libxl_internal.h
> 
>> +struct libxl__stream_read_state {
>> +    /* filled by the user */
>> +    libxl__ao *ao;
>> +    libxl__domain_create_state *dcs;
>> +    int fd;
>> +    bool legacy;
> 
> Can you please split out this code motion into a separate patch ?
> As it is it is very difficult to review.

OK, will fix it in the next version.

Thanks
Wen Congyang

> 
> Thanks,
> Ian.
> 
> 
> .
>
diff mbox

Patch

diff --git a/tools/libxl/Makefile b/tools/libxl/Makefile
index d8612eb..2016393 100644
--- a/tools/libxl/Makefile
+++ b/tools/libxl/Makefile
@@ -65,7 +65,7 @@  LIBXL_OBJS-y += libxl_no_convert_callout.o
 endif
 
 LIBXL_OBJS-y += libxl_remus.o libxl_checkpoint_device.o libxl_remus_disk_drbd.o
-LIBXL_OBJS-y += libxl_colo_restore.o
+LIBXL_OBJS-y += libxl_colo_restore.o libxl_colo_save.o
 
 LIBXL_OBJS-$(CONFIG_X86) += libxl_cpuid.o libxl_x86.o libxl_psr.o
 LIBXL_OBJS-$(CONFIG_ARM) += libxl_nocpuid.o libxl_arm.o libxl_libfdt_compat.o
diff --git a/tools/libxl/libxl.c b/tools/libxl/libxl.c
index e9ab78c..305231b 100644
--- a/tools/libxl/libxl.c
+++ b/tools/libxl/libxl.c
@@ -880,7 +880,10 @@  int libxl_domain_remus_start(libxl_ctx *ctx, libxl_domain_remus_info *info,
     assert(info);
 
     /* Point of no return */
-    libxl__remus_setup(egc, &dss->rs);
+    if (libxl_defbool_val(info->colo))
+        libxl__colo_save_setup(egc, &dss->css);
+    else
+        libxl__remus_setup(egc, &dss->rs);
     return AO_INPROGRESS;
 
  out:
diff --git a/tools/libxl/libxl_colo.h b/tools/libxl/libxl_colo.h
index f2b98cc..feec7f1 100644
--- a/tools/libxl/libxl_colo.h
+++ b/tools/libxl/libxl_colo.h
@@ -18,6 +18,7 @@ 
 
 struct libxl__ao;
 struct libxl__egc;
+struct libxl__colo_save_state;
 
 enum {
     LIBXL_COLO_SETUPED,
@@ -52,4 +53,9 @@  extern void libxl__colo_restore_setup(struct libxl__egc *egc,
                                       libxl__colo_restore_state *crs);
 extern void libxl__colo_restore_teardown(struct libxl__egc *egc, void *dcs_void,
                                          int ret, int retval, int errnoval);
+extern void libxl__colo_save_setup(struct libxl__egc *egc,
+                                   struct libxl__colo_save_state *css);
+extern void libxl__colo_save_teardown(struct libxl__egc *egc,
+                                      struct libxl__colo_save_state *css,
+                                      int rc);
 #endif
diff --git a/tools/libxl/libxl_colo_save.c b/tools/libxl/libxl_colo_save.c
new file mode 100644
index 0000000..e318ea2
--- /dev/null
+++ b/tools/libxl/libxl_colo_save.c
@@ -0,0 +1,562 @@ 
+/*
+ * Copyright (C) 2014 FUJITSU LIMITED
+ * Author: Wen Congyang <wency@cn.fujitsu.com>
+ *         Yang Hongyang <hongyang.yang@easystack.cn>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU Lesser General Public License as published
+ * by the Free Software Foundation; version 2.1 only. with the special
+ * exception on linking described in file LICENSE.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Lesser General Public License for more details.
+ */
+
+#include "libxl_osdeps.h" /* must come before any other headers */
+
+#include "libxl_internal.h"
+
+static const libxl__checkpoint_device_instance_ops *colo_ops[] = {
+    NULL,
+};
+
+/* ================= helper functions ================= */
+static int init_device_subkind(libxl__checkpoint_devices_state *cds)
+{
+    /* init device subkind-specific state in the libxl ctx */
+    int rc;
+    STATE_AO_GC(cds->ao);
+
+    rc = 0;
+    return rc;
+}
+
+static void cleanup_device_subkind(libxl__checkpoint_devices_state *cds)
+{
+    /* cleanup device subkind-specific state in the libxl ctx */
+    STATE_AO_GC(cds->ao);
+}
+
+/* ================= colo: setup save environment ================= */
+static void colo_save_setup_done(libxl__egc *egc,
+                                 libxl__checkpoint_devices_state *cds,
+                                 int rc);
+static void colo_save_setup_failed(libxl__egc *egc,
+                                   libxl__checkpoint_devices_state *cds,
+                                   int rc);
+/*
+ * checkpoint callbacks are called in the following order:
+ * 1. suspend
+ * 2. checkpoint
+ * 3. resume
+ * 4. wait checkpoint
+ */
+static void libxl__colo_save_domain_suspend_callback(void *data);
+static void libxl__colo_save_domain_checkpoint_callback(void *data);
+static void libxl__colo_save_domain_resume_callback(void *data);
+static void libxl__colo_save_domain_wait_checkpoint_callback(void *data);
+
+void libxl__colo_save_setup(libxl__egc *egc, libxl__colo_save_state *css)
+{
+    libxl__domain_save_state *dss = CONTAINER_OF(css, *dss, css);
+
+    /* Convenience aliases */
+    libxl__checkpoint_devices_state *const cds = &dss->cds;
+    libxl__srm_save_autogen_callbacks *const callbacks =
+        &dss->sws.shs.callbacks.save.a;
+
+    STATE_AO_GC(dss->ao);
+
+    if (dss->type != LIBXL_DOMAIN_TYPE_HVM) {
+        LOG(ERROR, "COLO only supports hvm now");
+        goto out;
+    }
+
+    css->send_fd = dss->fd;
+    css->recv_fd = dss->recv_fd;
+    css->svm_running = false;
+
+    /* TODO: disk/nic support */
+    cds->device_kind_flags = 0;
+    cds->ops = colo_ops;
+    cds->callback = colo_save_setup_done;
+    cds->ao = ao;
+    cds->domid = dss->domid;
+    cds->concrete_data = css;
+
+    css->srs.ao = ao;
+    css->srs.fd = css->recv_fd;
+    css->srs.back_channel = true;
+    libxl__stream_read_start(egc, &css->srs);
+
+    if (init_device_subkind(cds))
+        goto out;
+
+    callbacks->suspend = libxl__colo_save_domain_suspend_callback;
+    callbacks->checkpoint = libxl__colo_save_domain_checkpoint_callback;
+    callbacks->postcopy = libxl__colo_save_domain_resume_callback;
+    callbacks->wait_checkpoint = libxl__colo_save_domain_wait_checkpoint_callback;
+
+    libxl__checkpoint_devices_setup(egc, &dss->cds);
+
+    return;
+
+out:
+    libxl__ao_complete(egc, ao, ERROR_FAIL);
+}
+
+static void colo_save_setup_done(libxl__egc *egc,
+                                 libxl__checkpoint_devices_state *cds,
+                                 int rc)
+{
+    libxl__colo_save_state *css = cds->concrete_data;
+    libxl__domain_save_state *dss = CONTAINER_OF(css, *dss, css);
+    EGC_GC;
+
+    if (!rc) {
+        libxl__domain_save(egc, dss);
+        return;
+    }
+
+    LOG(ERROR, "COLO: failed to setup device for guest with domid %u",
+        dss->domid);
+    cds->callback = colo_save_setup_failed;
+    libxl__checkpoint_devices_teardown(egc, cds);
+}
+
+static void colo_save_setup_failed(libxl__egc *egc,
+                                   libxl__checkpoint_devices_state *cds,
+                                   int rc)
+{
+    STATE_AO_GC(cds->ao);
+
+    if (rc)
+        LOG(ERROR, "COLO: failed to teardown device after setup failed"
+            " for guest with domid %u, rc %d", cds->domid, rc);
+
+    cleanup_device_subkind(cds);
+    libxl__ao_complete(egc, ao, rc);
+}
+
+
+/* ================= colo: teardown save environment ================= */
+static void colo_teardown_done(libxl__egc *egc,
+                               libxl__checkpoint_devices_state *cds,
+                               int rc);
+
+void libxl__colo_save_teardown(libxl__egc *egc,
+                               libxl__colo_save_state *css,
+                               int rc)
+{
+    libxl__domain_save_state *dss = CONTAINER_OF(css, *dss, css);
+
+    EGC_GC;
+
+    LOG(WARN, "COLO: Domain suspend terminated with rc %d,"
+        " teardown COLO devices...", rc);
+
+    libxl__stream_read_abort(egc, &css->srs, 1);
+
+    dss->cds.callback = colo_teardown_done;
+    libxl__checkpoint_devices_teardown(egc, &dss->cds);
+    return;
+}
+
+static void colo_teardown_done(libxl__egc *egc,
+                               libxl__checkpoint_devices_state *cds,
+                               int rc)
+{
+    libxl__colo_save_state *css = cds->concrete_data;
+    libxl__domain_save_state *dss = CONTAINER_OF(css, *dss, css);
+
+    cleanup_device_subkind(cds);
+    dss->callback(egc, dss, rc);
+}
+
+static void colo_common_write_stream_done(libxl__egc *egc,
+                                          libxl__stream_write_state *stream,
+                                          int rc);
+static void colo_common_read_stream_done(libxl__egc *egc,
+                                         libxl__stream_read_state *stream,
+                                         int rc);
+/* ===================== colo: suspend primary vm ===================== */
+
+static void colo_read_svm_suspended_done(libxl__egc *egc,
+                                         libxl__colo_save_state *css,
+                                         int id);
+/*
+ * Do the following things when suspending primary vm:
+ * 1. suspend primary vm
+ * 2. do postsuspend
+ * 3. read CHECKPOINT_SVM_SUSPENDED
+ * 4. read secondary vm's dirty pages
+ */
+static void colo_suspend_primary_vm_done(libxl__egc *egc,
+                                         libxl__domain_suspend_state *dsps,
+                                         int ok);
+static void colo_postsuspend_cb(libxl__egc *egc,
+                                libxl__checkpoint_devices_state *cds,
+                                int rc);
+
+static void libxl__colo_save_domain_suspend_callback(void *data)
+{
+    libxl__save_helper_state *shs = data;
+    libxl__egc *egc = shs->egc;
+    libxl__stream_write_state *sws = CONTAINER_OF(shs, *sws, shs);
+    libxl__domain_save_state *dss = sws->dss;
+
+    /* Convenience aliases */
+    libxl__domain_suspend_state *dsps = &dss->dsps;
+
+    dsps->callback_common_done = colo_suspend_primary_vm_done;
+    libxl__domain_suspend(egc, dsps);
+}
+
+static void colo_suspend_primary_vm_done(libxl__egc *egc,
+                                         libxl__domain_suspend_state *dsps,
+                                         int rc)
+{
+    libxl__domain_save_state *dss = CONTAINER_OF(dsps, *dss, dsps);
+
+    EGC_GC;
+
+    if (rc) {
+        LOG(ERROR, "cannot suspend primary vm");
+        goto out;
+    }
+
+    /* Convenience aliases */
+    libxl__checkpoint_devices_state *const cds = &dss->cds;
+
+    cds->callback = colo_postsuspend_cb;
+    libxl__checkpoint_devices_postsuspend(egc, cds);
+    return;
+
+out:
+    dss->rc = rc;
+    libxl__xc_domain_saverestore_async_callback_done(egc, &dss->sws.shs, !rc);
+}
+
+static void colo_postsuspend_cb(libxl__egc *egc,
+                                libxl__checkpoint_devices_state *cds,
+                                int rc)
+{
+    libxl__colo_save_state *css = cds->concrete_data;
+    libxl__domain_save_state *dss = CONTAINER_OF(css, *dss, css);
+
+    EGC_GC;
+
+    if (rc) {
+        LOG(ERROR, "postsuspend fails");
+        goto out;
+    }
+
+    if (!css->svm_running) {
+        rc = 0;
+        goto out;
+    }
+
+    /*
+     * read CHECKPOINT_SVM_SUSPENDED
+     */
+    css->callback = colo_read_svm_suspended_done;
+    css->srs.checkpoint_callback = colo_common_read_stream_done;
+    libxl__stream_read_checkpoint_state(egc, &css->srs);
+
+    return;
+
+out:
+    libxl__xc_domain_saverestore_async_callback_done(egc, &dss->sws.shs, !rc);
+}
+
+static void colo_read_svm_suspended_done(libxl__egc *egc,
+                                         libxl__colo_save_state *css,
+                                         int id)
+{
+    int ok = 0;
+    libxl__domain_save_state *dss = CONTAINER_OF(css, *dss, css);
+
+    EGC_GC;
+
+    if (id != CHECKPOINT_SVM_SUSPENDED) {
+        LOG(ERROR, "invalid section: %d, expected: %d", id,
+            CHECKPOINT_SVM_SUSPENDED);
+        goto out;
+    }
+
+    ok = 1;
+
+out:
+    libxl__xc_domain_saverestore_async_callback_done(egc, &dss->sws.shs, ok);
+}
+
+
+/* ===================== colo: send tailbuf ========================== */
+static void libxl__colo_save_domain_checkpoint_callback(void *data)
+{
+    libxl__save_helper_state *shs = data;
+    libxl__stream_write_state *sws = CONTAINER_OF(shs, *sws, shs);
+    libxl__domain_save_state *dss = sws->dss;
+
+    /* Convenience aliases */
+    libxl__colo_save_state *const css = &dss->css;
+
+    /* write emulator xenstore data, emulator context, and checkpoint end */
+    css->callback = NULL;
+    dss->sws.checkpoint_callback = colo_common_write_stream_done;
+    libxl__stream_write_start_checkpoint(shs->egc, &dss->sws);
+}
+
+/* ===================== colo: resume primary vm ===================== */
+/*
+ * Do the following things when resuming primary vm:
+ *  1. read CHECKPOINT_SVM_READY
+ *  2. do preresume
+ *  3. resume primary vm
+ *  4. read CHECKPOINT_SVM_RESUMED
+ */
+static void colo_read_svm_ready_done(libxl__egc *egc,
+                                     libxl__colo_save_state *css,
+                                     int id);
+static void colo_preresume_cb(libxl__egc *egc,
+                              libxl__checkpoint_devices_state *cds,
+                              int rc);
+static void colo_read_svm_resumed_done(libxl__egc *egc,
+                                       libxl__colo_save_state *css,
+                                       int id);
+
+static void libxl__colo_save_domain_resume_callback(void *data)
+{
+    libxl__save_helper_state *shs = data;
+    libxl__egc *egc = shs->egc;
+    libxl__stream_write_state *sws = CONTAINER_OF(shs, *sws, shs);
+    libxl__domain_save_state *dss = sws->dss;
+
+    /* Convenience aliases */
+    libxl__colo_save_state *const css = &dss->css;
+
+    EGC_GC;
+
+    /* read CHECKPOINT_SVM_READY */
+    css->callback = colo_read_svm_ready_done;
+    css->srs.checkpoint_callback = colo_common_read_stream_done;
+    libxl__stream_read_checkpoint_state(egc, &css->srs);
+}
+
+static void colo_read_svm_ready_done(libxl__egc *egc,
+                                     libxl__colo_save_state *css,
+                                     int id)
+{
+    libxl__domain_save_state *dss = CONTAINER_OF(css, *dss, css);
+
+    EGC_GC;
+
+    if (id != CHECKPOINT_SVM_READY) {
+        LOG(ERROR, "invalid section: %d, expected: %d", id,
+            CHECKPOINT_SVM_READY);
+        goto out;
+    }
+
+    css->svm_running = true;
+    dss->cds.callback = colo_preresume_cb;
+    libxl__checkpoint_devices_preresume(egc, &dss->cds);
+
+    return;
+
+out:
+    libxl__xc_domain_saverestore_async_callback_done(egc, &dss->sws.shs, 0);
+}
+
+static void colo_preresume_cb(libxl__egc *egc,
+                              libxl__checkpoint_devices_state *cds,
+                              int rc)
+{
+    libxl__colo_save_state *css = cds->concrete_data;
+    libxl__domain_save_state *dss = CONTAINER_OF(css, *dss, css);
+
+    EGC_GC;
+
+    if (rc) {
+        LOG(ERROR, "preresume fails");
+        goto out;
+    }
+
+    /* Resumes the domain and the device model */
+    if (libxl__domain_resume(gc, dss->domid, /* Fast Suspend */1)) {
+        LOG(ERROR, "cannot resume primary vm");
+        goto out;
+    }
+
+    /* read CHECKPOINT_SVM_RESUMED */
+    css->callback = colo_read_svm_resumed_done;
+    css->srs.checkpoint_callback = colo_common_read_stream_done;
+    libxl__stream_read_checkpoint_state(egc, &css->srs);
+
+    return;
+
+out:
+    libxl__xc_domain_saverestore_async_callback_done(egc, &dss->sws.shs, 0);
+}
+
+static void colo_read_svm_resumed_done(libxl__egc *egc,
+                                       libxl__colo_save_state *css,
+                                       int id)
+{
+    int ok = 0;
+    libxl__domain_save_state *dss = CONTAINER_OF(css, *dss, css);
+
+    EGC_GC;
+
+    if (id != CHECKPOINT_SVM_RESUMED) {
+        LOG(ERROR, "invalid section: %d, expected: %d", id,
+            CHECKPOINT_SVM_RESUMED);
+        goto out;
+    }
+
+    ok = 1;
+
+out:
+    libxl__xc_domain_saverestore_async_callback_done(egc, &dss->sws.shs, ok);
+}
+
+
+/* ===================== colo: wait new checkpoint ===================== */
+/*
+ * Do the following things:
+ * 1. do commit
+ * 2. wait for a new checkpoint
+ * 3. write CHECKPOINT_NEW
+ */
+static void colo_device_commit_cb(libxl__egc *egc,
+                                  libxl__checkpoint_devices_state *cds,
+                                  int rc);
+static void colo_start_new_checkpoint(libxl__egc *egc,
+                                      libxl__checkpoint_devices_state *cds,
+                                      int rc);
+
+static void libxl__colo_save_domain_wait_checkpoint_callback(void *data)
+{
+    libxl__save_helper_state *shs = data;
+    libxl__stream_write_state *sws = CONTAINER_OF(shs, *sws, shs);
+    libxl__domain_save_state *dss = sws->dss;
+    libxl__egc *egc = dss->sws.shs.egc;
+
+    /* Convenience aliases */
+    libxl__checkpoint_devices_state *const cds = &dss->cds;
+
+    cds->callback = colo_device_commit_cb;
+    libxl__checkpoint_devices_commit(egc, cds);
+}
+
+static void colo_device_commit_cb(libxl__egc *egc,
+                                  libxl__checkpoint_devices_state *cds,
+                                  int rc)
+{
+    libxl__colo_save_state *css = cds->concrete_data;
+    libxl__domain_save_state *dss = CONTAINER_OF(css, *dss, css);
+
+    EGC_GC;
+
+    if (rc) {
+        LOG(ERROR, "commit fails");
+        goto out;
+    }
+
+    /* TODO: wait a new checkpoint */
+    colo_start_new_checkpoint(egc, cds, 0);
+    return;
+
+out:
+    libxl__xc_domain_saverestore_async_callback_done(egc, &dss->sws.shs, 0);
+}
+
+static void colo_start_new_checkpoint(libxl__egc *egc,
+                                      libxl__checkpoint_devices_state *cds,
+                                      int rc)
+{
+    libxl__colo_save_state *css = cds->concrete_data;
+    libxl__domain_save_state *dss = CONTAINER_OF(css, *dss, css);
+    libxl_sr_checkpoint_state srcs = { .id = CHECKPOINT_NEW };
+
+    if (rc)
+        goto out;
+
+    /* write CHECKPOINT_NEW */
+    css->callback = NULL;
+    dss->sws.checkpoint_callback = colo_common_write_stream_done;
+    libxl__stream_write_checkpoint_state(egc, &dss->sws, &srcs);
+
+    return;
+
+out:
+    libxl__xc_domain_saverestore_async_callback_done(egc, &dss->sws.shs, 0);
+}
+
+
+/* ===================== colo: common callback ===================== */
+static void colo_common_write_stream_done(libxl__egc *egc,
+                                          libxl__stream_write_state *stream,
+                                          int rc)
+{
+    libxl__domain_save_state *dss = CONTAINER_OF(stream, *dss, sws);
+    int ok;
+
+    /* Convenience aliases */
+    libxl__colo_save_state *const css = &dss->css;
+
+    EGC_GC;
+
+    if (rc < 0) {
+        /* TODO: it may be a internal error, but we don't know */
+        LOG(ERROR, "sending data fails");
+        ok = 0;
+        goto out;
+    }
+
+    if (!css->callback) {
+        /* Everythins is OK */
+        ok = 1;
+        goto out;
+    }
+
+    css->callback(egc, css, 0);
+
+    return;
+
+out:
+    libxl__xc_domain_saverestore_async_callback_done(egc, &dss->sws.shs, ok);
+}
+
+static void colo_common_read_stream_done(libxl__egc *egc,
+                                         libxl__stream_read_state *stream,
+                                         int rc)
+{
+    libxl__colo_save_state *css = CONTAINER_OF(stream, *css, srs);
+    libxl__domain_save_state *dss = CONTAINER_OF(css, *dss, css);
+    int ok;
+
+    EGC_GC;
+
+    if (rc < 0) {
+        /* TODO: it may be a internal error, but we don't know */
+        LOG(ERROR, "reading data fails");
+        ok = 0;
+        goto out;
+    }
+
+    if (!css->callback) {
+        /* Everythins is OK */
+        ok = 1;
+        goto out;
+    }
+
+    /* rc contains the id */
+    css->callback(egc, css, rc);
+
+    return;
+
+out:
+    libxl__xc_domain_saverestore_async_callback_done(egc, &dss->sws.shs, ok);
+}
diff --git a/tools/libxl/libxl_dom_save.c b/tools/libxl/libxl_dom_save.c
index cd324bb..821f862 100644
--- a/tools/libxl/libxl_dom_save.c
+++ b/tools/libxl/libxl_dom_save.c
@@ -438,12 +438,15 @@  static void domain_save_done(libxl__egc *egc,
 
     if (dss->remus) {
         /*
-         * With Remus, if we reach this point, it means either
+         * With Remus/COLO, if we reach this point, it means either
          * backup died or some network error occurred preventing us
          * from sending checkpoints. Teardown the network buffers and
          * release netlink resources.  This is an async op.
          */
-        libxl__remus_teardown(egc, &dss->rs, rc);
+        if (libxl_defbool_val(dss->remus->colo))
+            libxl__colo_save_teardown(egc, &dss->css, rc);
+        else
+            libxl__remus_teardown(egc, &dss->rs, rc);
         return;
     }
 
diff --git a/tools/libxl/libxl_internal.h b/tools/libxl/libxl_internal.h
index 48b4858..5160939 100644
--- a/tools/libxl/libxl_internal.h
+++ b/tools/libxl/libxl_internal.h
@@ -2802,7 +2802,7 @@  typedef struct libxl__save_helper_state {
 /*
  * The abstract checkpoint device layer exposes a common
  * set of API to [external] libxl for manipulating devices attached to
- * a guest protected by Remus. The device layer also exposes a set of
+ * a guest protected by Remus/COLO. The device layer also exposes a set of
  * [internal] interfaces that every device type must implement.
  *
  * The following API are exposed to libxl:
@@ -2820,7 +2820,7 @@  typedef struct libxl__save_helper_state {
  *  +libxl__checkpoint_devices_commit
  *
  * Each device type needs to implement the interfaces specified in
- * the libxl__checkpoint_device_instance_ops if it wishes to support Remus.
+ * the libxl__checkpoint_device_instance_ops if it wishes to support Remus/COLO.
  *
  * The high-level control flow through the checkpoint device layer is shown
  * below:
@@ -2840,7 +2840,7 @@  typedef struct libxl__checkpoint_device_instance_ops libxl__checkpoint_device_in
 
 /*
  * Interfaces to be implemented by every device subkind that wishes to
- * support Remus. Functions must be implemented unless otherwise
+ * support Remus/COLO. Functions must be implemented unless otherwise
  * stated. Many of these functions are asynchronous. They call
  * dev->aodev.callback when done.  The actual implementations may be
  * synchronous and call dev->aodev.callback directly (as the last
@@ -3017,6 +3017,72 @@  static inline bool libxl__conversion_helper_inuse
                     (const libxl__conversion_helper_state *chs)
 { return libxl__ev_child_inuse(&chs->child); }
 
+/* State for manipulating a libxl migration v2 stream */
+
+typedef struct libxl__stream_read_state libxl__stream_read_state;
+
+typedef struct libxl__sr_record_buf {
+    /* private to stream read helper */
+    LIBXL_STAILQ_ENTRY(struct libxl__sr_record_buf) entry;
+    libxl__sr_rec_hdr hdr;
+    void *body; /* iff hdr.length != 0 */
+} libxl__sr_record_buf;
+
+struct libxl__stream_read_state {
+    /* filled by the user */
+    libxl__ao *ao;
+    libxl__domain_create_state *dcs;
+    int fd;
+    bool legacy;
+    bool back_channel;
+    void (*completion_callback)(libxl__egc *egc,
+                                libxl__stream_read_state *srs,
+                                int rc);
+    void (*checkpoint_callback)(libxl__egc *egc,
+                                libxl__stream_read_state *srs,
+                                int rc);
+    /* Private */
+    int rc;
+    bool running;
+    bool in_checkpoint;
+    bool sync_teardown; /* Only used to coordinate shutdown on error path. */
+    bool in_checkpoint_state;
+    libxl__save_helper_state shs;
+    libxl__conversion_helper_state chs;
+
+    /* Main stream-reading data. */
+    libxl__datacopier_state dc; /* Only used when reading a record */
+    libxl__sr_hdr hdr;
+    LIBXL_STAILQ_HEAD(, libxl__sr_record_buf) record_queue; /* NOGC */
+    enum {
+        SRS_PHASE_NORMAL,
+        SRS_PHASE_BUFFERING,
+        SRS_PHASE_UNBUFFERING,
+    } phase;
+    bool recursion_guard;
+
+    /* Only used while actively reading a record from the stream. */
+    libxl__sr_record_buf *incoming_record; /* NOGC */
+
+    /* Both only used when processing an EMULATOR record. */
+    libxl__datacopier_state emu_dc;
+    libxl__carefd *emu_carefd;
+};
+
+_hidden void libxl__stream_read_init(libxl__stream_read_state *stream);
+_hidden void libxl__stream_read_start(libxl__egc *egc,
+                                      libxl__stream_read_state *stream);
+_hidden void libxl__stream_read_start_checkpoint(libxl__egc *egc,
+                                                 libxl__stream_read_state *stream);
+_hidden void libxl__stream_read_checkpoint_state(libxl__egc *egc,
+                                                 libxl__stream_read_state *stream);
+_hidden void libxl__stream_read_abort(libxl__egc *egc,
+                                      libxl__stream_read_state *stream, int rc);
+static inline bool
+libxl__stream_read_inuse(const libxl__stream_read_state *stream)
+{
+    return stream->running;
+}
 
 /*----- Domain suspend (save) state structure -----*/
 /*
@@ -3093,6 +3159,18 @@  libxl__stream_write_inuse(const libxl__stream_write_state *stream)
     return stream->running;
 }
 
+/*----- colo related state structure -----*/
+typedef struct libxl__colo_save_state libxl__colo_save_state;
+struct libxl__colo_save_state {
+    int send_fd;
+    int recv_fd;
+
+    /* private */
+    libxl__stream_read_state srs;
+    void (*callback)(libxl__egc *, libxl__colo_save_state *, int);
+    bool svm_running;
+};
+
 typedef struct libxl__logdirty_switch {
     /* Set by caller of libxl__domain_common_switch_qemu_logdirty */
     libxl__ao *ao;
@@ -3151,7 +3229,12 @@  struct libxl__domain_save_state {
     int hvm;
     int xcflags;
     libxl__domain_suspend_state dsps;
-    libxl__remus_state rs;
+    union {
+        /* for Remus */
+        libxl__remus_state rs;
+        /* for COLO */
+        libxl__colo_save_state css;
+    };
     libxl__checkpoint_devices_state cds;
     libxl__stream_write_state sws;
     libxl__logdirty_switch logdirty;
@@ -3398,72 +3481,6 @@  _hidden int libxl__destroy_qdisk_backend(libxl__gc *gc, uint32_t domid);
 
 /*----- Domain creation -----*/
 
-/* State for manipulating a libxl migration v2 stream */
-typedef struct libxl__stream_read_state libxl__stream_read_state;
-
-typedef struct libxl__sr_record_buf {
-    /* private to stream read helper */
-    LIBXL_STAILQ_ENTRY(struct libxl__sr_record_buf) entry;
-    libxl__sr_rec_hdr hdr;
-    void *body; /* iff hdr.length != 0 */
-} libxl__sr_record_buf;
-
-struct libxl__stream_read_state {
-    /* filled by the user */
-    libxl__ao *ao;
-    libxl__domain_create_state *dcs;
-    int fd;
-    bool legacy;
-    bool back_channel;
-    void (*completion_callback)(libxl__egc *egc,
-                                libxl__stream_read_state *srs,
-                                int rc);
-    void (*checkpoint_callback)(libxl__egc *egc,
-                                libxl__stream_read_state *srs,
-                                int rc);
-    /* Private */
-    int rc;
-    bool running;
-    bool in_checkpoint;
-    bool sync_teardown; /* Only used to coordinate shutdown on error path. */
-    bool in_checkpoint_state;
-    libxl__save_helper_state shs;
-    libxl__conversion_helper_state chs;
-
-    /* Main stream-reading data. */
-    libxl__datacopier_state dc; /* Only used when reading a record */
-    libxl__sr_hdr hdr;
-    LIBXL_STAILQ_HEAD(, libxl__sr_record_buf) record_queue; /* NOGC */
-    enum {
-        SRS_PHASE_NORMAL,
-        SRS_PHASE_BUFFERING,
-        SRS_PHASE_UNBUFFERING,
-    } phase;
-    bool recursion_guard;
-
-    /* Only used while actively reading a record from the stream. */
-    libxl__sr_record_buf *incoming_record; /* NOGC */
-
-    /* Both only used when processing an EMULATOR record. */
-    libxl__datacopier_state emu_dc;
-    libxl__carefd *emu_carefd;
-};
-
-_hidden void libxl__stream_read_init(libxl__stream_read_state *stream);
-_hidden void libxl__stream_read_start(libxl__egc *egc,
-                                      libxl__stream_read_state *stream);
-_hidden void libxl__stream_read_start_checkpoint(libxl__egc *egc,
-                                                 libxl__stream_read_state *stream);
-_hidden void libxl__stream_read_checkpoint_state(libxl__egc *egc,
-                                                 libxl__stream_read_state *stream);
-_hidden void libxl__stream_read_abort(libxl__egc *egc,
-                                      libxl__stream_read_state *stream, int rc);
-static inline bool
-libxl__stream_read_inuse(const libxl__stream_read_state *stream)
-{
-    return stream->running;
-}
-
 struct libxl__domain_create_state {
     /* filled in by user */
     libxl__ao *ao;
diff --git a/tools/libxl/libxl_types.idl b/tools/libxl/libxl_types.idl
index 33f4a90..9b0a537 100644
--- a/tools/libxl/libxl_types.idl
+++ b/tools/libxl/libxl_types.idl
@@ -763,6 +763,7 @@  libxl_domain_remus_info = Struct("domain_remus_info",[
     ("netbuf",       libxl_defbool),
     ("netbufscript", string),
     ("diskbuf",      libxl_defbool),
+    ("colo",         libxl_defbool)
     ])
 
 libxl_event_type = Enumeration("event_type", [