@@ -69,6 +69,8 @@ struct libxl__colo_proxy_state {
* False means use kernel colo proxy.
*/
bool is_userspace_proxy;
+ const char *checkpoint_host;
+ const char *checkpoint_port;
};
struct libxl__colo_save_state {
@@ -18,9 +18,13 @@
#include "libxl_internal.h"
#include <netlink/netlink.h>
+#include <arpa/inet.h>
+#include <sys/socket.h>
+#include <netinet/in.h>
/* Consistent with the new COLO netlink channel in kernel side */
#define NETLINK_COLO 28
+#define COLO_DEFAULT_WAIT_TIME 500000
enum colo_netlink_op {
COLO_QUERY_CHECKPOINT = (NLMSG_MIN_TYPE + 1),
@@ -76,6 +80,63 @@ static int colo_proxy_send(libxl__colo_proxy_state *cps, uint8_t *buff,
return ret;
}
+static int colo_userspace_proxy_send(libxl__colo_proxy_state *cps,
+ uint8_t *buff,
+ uint32_t size)
+{
+ int ret = 0;
+ uint32_t len = 0;
+
+ len = htonl(size);
+ ret = send(cps->sock_fd, (uint8_t *)&len, sizeof(len), 0);
+ if (ret != sizeof(len)) {
+ goto err;
+ }
+
+ ret = send(cps->sock_fd, (uint8_t *)buff, size, 0);
+ if (ret != size) {
+ goto err;
+ }
+
+err:
+ return ret;
+}
+
+static int colo_userspace_proxy_recv(libxl__colo_proxy_state *cps,
+ char *buff,
+ unsigned int timeout_us)
+{
+ struct timeval tv;
+ int ret;
+ uint32_t len = 0;
+ uint32_t size = 0;
+
+ STATE_AO_GC(cps->ao);
+
+ if (timeout_us) {
+ tv.tv_sec = timeout_us / 1000000;
+ tv.tv_usec = timeout_us % 1000000;
+ ret = setsockopt(cps->sock_fd, SOL_SOCKET, SO_RCVTIMEO, &tv,
+ sizeof(tv));
+ if (ret < 0) {
+ LOGD(ERROR, ao->domid,
+ "colo_userspace_proxy_recv setsockopt error: %s",
+ strerror(errno));
+ }
+ }
+
+ ret = recv(cps->sock_fd, (uint8_t *)&len, sizeof(len), 0);
+ if (ret < 0) {
+ goto err;
+ }
+
+ size = ntohl(len);
+ ret = recv(cps->sock_fd, buff, size, 0);
+
+err:
+ return ret;
+}
+
/* error: return -1, otherwise return 0 */
static int64_t colo_proxy_recv(libxl__colo_proxy_state *cps, uint8_t **buff,
unsigned int timeout_us)
@@ -153,8 +214,45 @@ int colo_proxy_setup(libxl__colo_proxy_state *cps)
STATE_AO_GC(cps->ao);
/* If enable userspace proxy mode, we don't need setup kernel proxy */
- if (cps->is_userspace_proxy)
+ if (cps->is_userspace_proxy) {
+ struct sockaddr_in addr;
+ int port;
+ char recvbuff[1024];
+ const char sendbuf[] = "COLO_USERSPACE_PROXY_INIT";
+
+ memset(&addr, 0, sizeof(addr));
+ port = atoi(cps->checkpoint_port);
+ addr.sin_family = AF_INET;
+ addr.sin_port = htons(port);
+ addr.sin_addr.s_addr = inet_addr(cps->checkpoint_host);
+
+ skfd = socket(AF_INET, SOCK_STREAM, 0);
+ if (skfd < 0) {
+ LOGD(ERROR, ao->domid, "can not create a TCP socket: %s",
+ strerror(errno));
+ goto out;
+ }
+
+ cps->sock_fd = skfd;
+
+ if (connect(skfd, (struct sockaddr *)&addr, sizeof(addr)) < 0) {
+ LOGD(ERROR, ao->domid, "connect error");
+ goto out;
+ }
+
+ ret = colo_userspace_proxy_send(cps, (uint8_t *)sendbuf, strlen(sendbuf));
+ if (ret < 0)
+ goto out;
+
+ ret = colo_userspace_proxy_recv(cps, recvbuff, COLO_DEFAULT_WAIT_TIME);
+ if (ret < 0) {
+ LOGD(ERROR, ao->domid, "Can't recv msg from qemu colo-compare: %s",
+ strerror(errno));
+ goto out;
+ }
+
return 0;
+ }
skfd = socket(PF_NETLINK, SOCK_RAW, NETLINK_COLO);
if (skfd < 0) {
@@ -247,8 +345,13 @@ void colo_proxy_preresume(libxl__colo_proxy_state *cps)
* If enable userspace proxy mode,
* we don't need preresume kernel proxy
*/
- if (cps->is_userspace_proxy)
+ if (cps->is_userspace_proxy) {
+ const char sendbuf[] = "COLO_CHECKPOINT";
+ colo_userspace_proxy_send(cps,
+ (uint8_t *)sendbuf,
+ strlen(sendbuf));
return;
+ }
colo_proxy_send(cps, NULL, 0, COLO_CHECKPOINT);
/* TODO: need to handle if the call fails... */
@@ -277,6 +380,7 @@ int colo_proxy_checkpoint(libxl__colo_proxy_state *cps,
struct nlmsghdr *h;
struct colo_msg *m;
int ret = -1;
+ char recvbuff[1024];
STATE_AO_GC(cps->ao);
@@ -289,8 +393,19 @@ int colo_proxy_checkpoint(libxl__colo_proxy_state *cps,
* event.
*/
if (cps->is_userspace_proxy) {
- usleep(timeout_us);
- return 0;
+ ret = colo_userspace_proxy_recv(cps, recvbuff, timeout_us);
+ if (ret <= 0) {
+ ret = 0;
+ goto out1;
+ }
+
+ if (!strcmp(recvbuff, "DO_CHECKPOINT")) {
+ ret = 1;
+ } else {
+ LOGD(ERROR, ao->domid, "receive qemu colo-compare checkpoint error");
+ ret = 0;
+ }
+ goto out1;
}
size = colo_proxy_recv(cps, &buff, timeout_us);
@@ -318,4 +433,7 @@ int colo_proxy_checkpoint(libxl__colo_proxy_state *cps,
out:
free(buff);
return ret;
+
+out1:
+ return ret;
}
@@ -613,7 +613,8 @@ static void colo_restore_preresume_cb(libxl__egc *egc,
}
}
- colo_proxy_preresume(&crs->cps);
+ if (!crs->cps.is_userspace_proxy)
+ colo_proxy_preresume(&crs->cps);
colo_restore_resume_vm(egc, crcs);
@@ -786,9 +787,11 @@ static void colo_setup_checkpoint_devices(libxl__egc *egc,
cds->ops = colo_restore_ops;
crs->cps.ao = ao;
- if (colo_proxy_setup(&crs->cps)) {
- LOGD(ERROR, cds->domid, "COLO: failed to setup colo proxy for guest");
- goto out;
+ if (!crs->cps.is_userspace_proxy) {
+ if (colo_proxy_setup(&crs->cps)) {
+ LOGD(ERROR, cds->domid, "COLO: failed to setup colo proxy for guest");
+ goto out;
+ }
}
if (init_device_subkind(cds))
@@ -86,6 +86,7 @@ void libxl__colo_save_setup(libxl__egc *egc, libxl__colo_save_state *css)
libxl__checkpoint_devices_state *const cds = &dss->cds;
libxl__srm_save_autogen_callbacks *const callbacks =
&dss->sws.shs.callbacks.save.a;
+ libxl_device_nic *nics;
STATE_AO_GC(dss->ao);
@@ -110,19 +111,25 @@ void libxl__colo_save_setup(libxl__egc *egc, libxl__colo_save_state *css)
css->colo_proxy_script = GCSPRINTF("%s/colo-proxy-setup",
libxl__xen_script_dir_path());
- /* If enable userspace proxy mode, we don't need VIF */
- if (css->cps.is_userspace_proxy)
- cds->device_kind_flags = (1 << LIBXL__DEVICE_KIND_VBD);
- else
- cds->device_kind_flags = (1 << LIBXL__DEVICE_KIND_VIF) |
- (1 << LIBXL__DEVICE_KIND_VBD);
-
cds->ops = colo_ops;
cds->callback = colo_save_setup_done;
cds->ao = ao;
cds->domid = dss->domid;
cds->concrete_data = css;
+ /* If enable userspace proxy mode, we don't need VIF */
+ if (css->cps.is_userspace_proxy) {
+ cds->device_kind_flags = (1 << LIBXL__DEVICE_KIND_VBD);
+
+ /* Use this args we can connect to qemu colo-compare */
+ nics = libxl_device_nic_list(CTX, cds->domid, &cds->num_nics);
+ css->cps.checkpoint_host = nics->colo_checkpoint_host;
+ css->cps.checkpoint_port = nics->colo_checkpoint_port;
+ } else {
+ cds->device_kind_flags = (1 << LIBXL__DEVICE_KIND_VIF) |
+ (1 << LIBXL__DEVICE_KIND_VBD);
+ }
+
css->srs.ao = ao;
css->srs.fd = css->recv_fd;
css->srs.back_channel = true;
@@ -250,6 +250,8 @@ static void libxl__device_nic_add(libxl__egc *egc, uint32_t domid,
MAYBE_ADD_COLO_ARGS(filter_sec_redirector1_indev);
MAYBE_ADD_COLO_ARGS(filter_sec_redirector1_outdev);
MAYBE_ADD_COLO_ARGS(filter_sec_rewriter0_queue);
+ MAYBE_ADD_COLO_ARGS(checkpoint_host);
+ MAYBE_ADD_COLO_ARGS(checkpoint_port);
#undef MAYBE_ADD_COLO_ARGS
@@ -459,6 +461,8 @@ static int libxl__device_nic_from_xenstore(libxl__gc *gc,
CHECK_COLO_ARGS(filter_sec_redirector1_indev);
CHECK_COLO_ARGS(filter_sec_redirector1_outdev);
CHECK_COLO_ARGS(filter_sec_rewriter0_queue);
+ CHECK_COLO_ARGS(checkpoint_host);
+ CHECK_COLO_ARGS(checkpoint_port);
#undef CHECK_COLO_ARGS
@@ -675,7 +675,9 @@ libxl_device_nic = Struct("device_nic", [
("colo_filter_sec_redirector1_queue", string),
("colo_filter_sec_redirector1_indev", string),
("colo_filter_sec_redirector1_outdev", string),
- ("colo_filter_sec_rewriter0_queue", string)
+ ("colo_filter_sec_rewriter0_queue", string),
+ ("colo_checkpoint_host", string),
+ ("colo_checkpoint_port", string)
])
libxl_device_pci = Struct("device_pci", [
@@ -551,6 +551,10 @@ int parse_nic_config(libxl_device_nic *nic, XLU_Config **config, char *token)
replace_string(&nic->colo_filter_sec_redirector1_outdev, oparg);
} else if (MATCH_OPTION("colo_filter_sec_rewriter0_queue", token, oparg)) {
replace_string(&nic->colo_filter_sec_rewriter0_queue, oparg);
+ } else if (MATCH_OPTION("colo_checkpoint_host", token, oparg)) {
+ replace_string(&nic->colo_checkpoint_host, oparg);
+ } else if (MATCH_OPTION("colo_checkpoint_port", token, oparg)) {
+ replace_string(&nic->colo_checkpoint_port, oparg);
} else if (MATCH_OPTION("accel", token, oparg)) {
fprintf(stderr, "the accel parameter for vifs is currently not supported\n");
} else if (MATCH_OPTION("devid", token, oparg)) {
We use kernel colo proxy's way to get the checkpoint event from qemu colo-compare. Qemu colo-compare need add a API to support this(I will add this in qemu). Qemu side patch: https://lists.nongnu.org/archive/html/qemu-devel/2017-02/msg07265.html Signed-off-by: Zhang Chen <zhangchen.fnst@cn.fujitsu.com> --- tools/libxl/libxl_colo.h | 2 + tools/libxl/libxl_colo_proxy.c | 126 +++++++++++++++++++++++++++++++++++++-- tools/libxl/libxl_colo_restore.c | 11 ++-- tools/libxl/libxl_colo_save.c | 21 ++++--- tools/libxl/libxl_nic.c | 4 ++ tools/libxl/libxl_types.idl | 4 +- tools/xl/xl_parse.c | 4 ++ 7 files changed, 156 insertions(+), 16 deletions(-)