| Message ID | 1487152473-3693-8-git-send-email-zhangchen.fnst@cn.fujitsu.com (mailing list archive) |
|---|---|
| State | New, archived |
| Headers | show |
On Wed, Feb 15, 2017 at 05:54:33PM +0800, Zhang Chen wrote: > We use kernel colo proxy's way to get the checkpoint event > from qemu colo-compare. > Qemu colo-compare need add a API to support this(I will add this in qemu). > > Signed-off-by: Zhang Chen <zhangchen.fnst@cn.fujitsu.com> > --- > tools/libxl/libxl_colo.h | 2 + > tools/libxl/libxl_colo_proxy.c | 84 +++++++++++++++++++++++++++++++++++++--- > tools/libxl/libxl_colo_restore.c | 11 ++++-- > tools/libxl/libxl_colo_save.c | 22 +++++++---- > tools/libxl/libxl_nic.c | 4 ++ > tools/libxl/libxl_types.idl | 4 +- > tools/libxl/xl_cmdimpl.c | 4 ++ > 7 files changed, 113 insertions(+), 18 deletions(-) > > diff --git a/tools/libxl/libxl_colo.h b/tools/libxl/libxl_colo.h > index 4746d8c..6c01b55 100644 > --- a/tools/libxl/libxl_colo.h > +++ b/tools/libxl/libxl_colo.h > @@ -69,6 +69,8 @@ struct libxl__colo_proxy_state { > * False means use kernel colo proxy. > */ > bool is_userspace_proxy; > + const char *checkpoint_host; > + const char *checkpoint_port; > }; > > struct libxl__colo_save_state { > diff --git a/tools/libxl/libxl_colo_proxy.c b/tools/libxl/libxl_colo_proxy.c > index dd902fc..9d21cf1 100644 > --- a/tools/libxl/libxl_colo_proxy.c > +++ b/tools/libxl/libxl_colo_proxy.c > @@ -18,6 +18,9 @@ > #include "libxl_internal.h" > > #include <netlink/netlink.h> > +#include <arpa/inet.h> > +#include <sys/socket.h> > +#include <netinet/in.h> > > /* Consistent with the new COLO netlink channel in kernel side */ > #define NETLINK_COLO 28 > @@ -76,6 +79,26 @@ static int colo_proxy_send(libxl__colo_proxy_state *cps, uint8_t *buff, > return ret; > } > > +static int colo_userspace_proxy_recv(libxl__colo_proxy_state *cps, > + char *buff, > + unsigned int timeout_us) > +{ > + struct timeval tv; > + int ret; > + > + STATE_AO_GC(cps->ao); > + > + if (timeout_us) { > + tv.tv_sec = timeout_us / 1000000; > + tv.tv_usec = timeout_us % 1000000; > + setsockopt(cps->sock_fd, SOL_SOCKET, SO_RCVTIMEO, &tv, sizeof(tv)); > + } > + > + ret = recv(cps->sock_fd, buff, sizeof(buff),0); > + > + return ret; > +} > + > /* error: return -1, otherwise return 0 */ > static int64_t colo_proxy_recv(libxl__colo_proxy_state *cps, uint8_t **buff, > unsigned int timeout_us) > @@ -153,8 +176,45 @@ int colo_proxy_setup(libxl__colo_proxy_state *cps) > STATE_AO_GC(cps->ao); > > /* If enable userspace proxy mode, we don't need setup kernel proxy */ > - if (cps->is_userspace_proxy) > + if (cps->is_userspace_proxy) { > + struct sockaddr_in addr; > + int port; > + char recvbuff[1024]; > + > + memset(&addr, 0, sizeof(addr)); > + port = atoi(cps->checkpoint_port); > + addr.sin_family = AF_INET; > + addr.sin_port = htons(port); > + addr.sin_addr.s_addr = inet_addr(cps->checkpoint_host); > + > + skfd = socket(AF_INET, SOCK_STREAM, 0); > + if (skfd < 0) { > + LOGD(ERROR, ao->domid, "can not create a TCP socket: %s", > + strerror(errno)); > + goto out; > + } > + > + cps->sock_fd = skfd; > + > + if (connect(skfd, (struct sockaddr *)&addr, sizeof(addr)) < 0) { > + LOGD(ERROR, ao->domid, "connect error"); > + goto out; > + } > + > + char sendbuf[] = "COLO_USERSPACE_PROXY_INIT"; Um, could you move it right around 'char recvbuff' ? > + ret = send(skfd, sendbuf, strlen(sendbuf),0); > + if (ret < 0) > + goto out; > + > + ret = colo_userspace_proxy_recv(cps, recvbuff, 500000); That 500000 looks like a good candidate for a #define? > + if (size < 0) { > + LOGD(ERROR, ao->domid, "Can't recv msg from qemu colo-compare: %s", > + strerror(errno)); > + goto out; > + } > + > return 0; > + } > > skfd = socket(PF_NETLINK, SOCK_RAW, NETLINK_COLO); > if (skfd < 0) { > @@ -247,8 +307,11 @@ void colo_proxy_preresume(libxl__colo_proxy_state *cps) > * If enable userspace proxy mode, > * we don't need preresume kernel proxy > */ > - if (cps->is_userspace_proxy) > + if (cps->is_userspace_proxy) { > + char sendbuf[] = "COLO_CHECKPOINT"; > + send(cps->sock_fd, sendbuf, strlen(sendbuf),0); > return; > + } > > colo_proxy_send(cps, NULL, 0, COLO_CHECKPOINT); > /* TODO: need to handle if the call fails... */ > @@ -277,16 +340,25 @@ int colo_proxy_checkpoint(libxl__colo_proxy_state *cps, > struct nlmsghdr *h; > struct colo_msg *m; > int ret = -1; > + char recvbuff[1024]; > > STATE_AO_GC(cps->ao); > > /* > - * enable userspace proxy mode, tmp sleep. > - * then we will add qemu API support this func. > + * enable userspace proxy mode. > + * Then we will add qemu API support for this func. > */ > if (cps->is_userspace_proxy) { > - sleep(timeout_us / 1000000); > - return 0; > + ret = colo_userspace_proxy_recv(cps, recvbuff, timeout_us); > + if (ret <= 0) > + return 0; > + > + if (!strcmp(recvbuff, "DO_CHECKPOINT")) { > + return 1; > + } else { > + LOGD(ERROR, ao->domid, "receive qemu colo-compare checkpoint error"); > + return -1; > + } > } > > size = colo_proxy_recv(cps, &buff, timeout_us); > diff --git a/tools/libxl/libxl_colo_restore.c b/tools/libxl/libxl_colo_restore.c > index c6d239a..065ea00 100644 > --- a/tools/libxl/libxl_colo_restore.c > +++ b/tools/libxl/libxl_colo_restore.c > @@ -613,7 +613,8 @@ static void colo_restore_preresume_cb(libxl__egc *egc, > } > } > > - colo_proxy_preresume(&crs->cps); > + if (!crs->cps.is_userspace_proxy) > + colo_proxy_preresume(&crs->cps); > > colo_restore_resume_vm(egc, crcs); > > @@ -786,9 +787,11 @@ static void colo_setup_checkpoint_devices(libxl__egc *egc, > cds->ops = colo_restore_ops; > > crs->cps.ao = ao; > - if (colo_proxy_setup(&crs->cps)) { > - LOGD(ERROR, cds->domid, "COLO: failed to setup colo proxy for guest"); > - goto out; > + if (!crs->cps.is_userspace_proxy) { > + if (colo_proxy_setup(&crs->cps)) { > + LOGD(ERROR, cds->domid, "COLO: failed to setup colo proxy for guest"); > + goto out; > + } > } > > if (init_device_subkind(cds)) > diff --git a/tools/libxl/libxl_colo_save.c b/tools/libxl/libxl_colo_save.c > index 91e3fce..a0cfc5a 100644 > --- a/tools/libxl/libxl_colo_save.c > +++ b/tools/libxl/libxl_colo_save.c > @@ -86,6 +86,7 @@ void libxl__colo_save_setup(libxl__egc *egc, libxl__colo_save_state *css) > libxl__checkpoint_devices_state *const cds = &dss->cds; > libxl__srm_save_autogen_callbacks *const callbacks = > &dss->sws.shs.callbacks.save.a; > + libxl_device_nic *nics; > > STATE_AO_GC(dss->ao); > > @@ -110,24 +111,31 @@ void libxl__colo_save_setup(libxl__egc *egc, libxl__colo_save_state *css) > css->colo_proxy_script = GCSPRINTF("%s/colo-proxy-setup", > libxl__xen_script_dir_path()); > > - /* If enable userspace proxy mode, we don't need VIF */ > - if (css->cps.is_userspace_proxy) > - cds->device_kind_flags = (1 << LIBXL__DEVICE_KIND_VBD); > - else > - cds->device_kind_flags = (1 << LIBXL__DEVICE_KIND_VIF) | > - (1 << LIBXL__DEVICE_KIND_VBD); > - > cds->ops = colo_ops; > cds->callback = colo_save_setup_done; > cds->ao = ao; > cds->domid = dss->domid; > cds->concrete_data = css; > > + /* If enable userspace proxy mode, we don't need VIF */ > + if (css->cps.is_userspace_proxy) { > + cds->device_kind_flags = (1 << LIBXL__DEVICE_KIND_VBD); > + > + /* Use this args we can connect to qemu colo-compare */ > + nics = libxl_device_nic_list(CTX, cds->domid, &cds->num_nics); > + css->cps.checkpoint_host = nics->colo_checkpoint_host; > + css->cps.checkpoint_port = nics->colo_checkpoint_port; > + } else { > + cds->device_kind_flags = (1 << LIBXL__DEVICE_KIND_VIF) | > + (1 << LIBXL__DEVICE_KIND_VBD); > + } > + > css->srs.ao = ao; > css->srs.fd = css->recv_fd; > css->srs.back_channel = true; > libxl__stream_read_start(egc, &css->srs); > css->cps.ao = ao; > + > if (colo_proxy_setup(&css->cps)) { > LOGD(ERROR, cds->domid, "COLO: failed to setup colo proxy for guest"); > goto out; > diff --git a/tools/libxl/libxl_nic.c b/tools/libxl/libxl_nic.c > index 5e1fecd..6bc6146 100644 > --- a/tools/libxl/libxl_nic.c > +++ b/tools/libxl/libxl_nic.c > @@ -246,6 +246,8 @@ static void libxl__device_nic_add(libxl__egc *egc, uint32_t domid, > MAYBE_ADD_COLO_ARGS(filter_sec_redirector1_indev); > MAYBE_ADD_COLO_ARGS(filter_sec_redirector1_outdev); > MAYBE_ADD_COLO_ARGS(filter_sec_rewriter0_queue); > + MAYBE_ADD_COLO_ARGS(checkpoint_host); > + MAYBE_ADD_COLO_ARGS(checkpoint_port); > > #undef MAYBE_ADD_COLO_ARGS > > @@ -451,6 +453,8 @@ static int libxl__device_nic_from_xenstore(libxl__gc *gc, > CHECK_COLO_ARGS(filter_sec_redirector1_indev); > CHECK_COLO_ARGS(filter_sec_redirector1_outdev); > CHECK_COLO_ARGS(filter_sec_rewriter0_queue); > + CHECK_COLO_ARGS(checkpoint_host); > + CHECK_COLO_ARGS(checkpoint_port); > > #undef CHECK_COLO_ARGS > > diff --git a/tools/libxl/libxl_types.idl b/tools/libxl/libxl_types.idl > index 47e96b1..0b412ee 100644 > --- a/tools/libxl/libxl_types.idl > +++ b/tools/libxl/libxl_types.idl > @@ -671,7 +671,9 @@ libxl_device_nic = Struct("device_nic", [ > ("colo_filter_sec_redirector1_queue", string), > ("colo_filter_sec_redirector1_indev", string), > ("colo_filter_sec_redirector1_outdev", string), > - ("colo_filter_sec_rewriter0_queue", string) > + ("colo_filter_sec_rewriter0_queue", string), > + ("colo_checkpoint_host", string), > + ("colo_checkpoint_port", string) > ]) > > libxl_device_pci = Struct("device_pci", [ > diff --git a/tools/libxl/xl_cmdimpl.c b/tools/libxl/xl_cmdimpl.c > index 32a47f6..ad5e193 100644 > --- a/tools/libxl/xl_cmdimpl.c > +++ b/tools/libxl/xl_cmdimpl.c > @@ -1145,6 +1145,10 @@ static int parse_nic_config(libxl_device_nic *nic, XLU_Config **config, char *to > replace_string(&nic->colo_filter_sec_redirector1_outdev, oparg); > } else if (MATCH_OPTION("colo_filter_sec_rewriter0_queue", token, oparg)) { > replace_string(&nic->colo_filter_sec_rewriter0_queue, oparg); > + } else if (MATCH_OPTION("colo_checkpoint_host", token, oparg)) { > + replace_string(&nic->colo_checkpoint_host, oparg); > + } else if (MATCH_OPTION("colo_checkpoint_port", token, oparg)) { > + replace_string(&nic->colo_checkpoint_port, oparg); > } else if (MATCH_OPTION("accel", token, oparg)) { > fprintf(stderr, "the accel parameter for vifs is currently not supported\n"); > } else { > -- > 2.7.4 > > > > > _______________________________________________ > Xen-devel mailing list > Xen-devel@lists.xen.org > https://lists.xen.org/xen-devel
On 02/16/2017 12:24 AM, Konrad Rzeszutek Wilk wrote: > On Wed, Feb 15, 2017 at 05:54:33PM +0800, Zhang Chen wrote: >> We use kernel colo proxy's way to get the checkpoint event >> from qemu colo-compare. >> Qemu colo-compare need add a API to support this(I will add this in qemu). >> >> Signed-off-by: Zhang Chen <zhangchen.fnst@cn.fujitsu.com> >> --- >> tools/libxl/libxl_colo.h | 2 + >> tools/libxl/libxl_colo_proxy.c | 84 +++++++++++++++++++++++++++++++++++++--- >> tools/libxl/libxl_colo_restore.c | 11 ++++-- >> tools/libxl/libxl_colo_save.c | 22 +++++++---- >> tools/libxl/libxl_nic.c | 4 ++ >> tools/libxl/libxl_types.idl | 4 +- >> tools/libxl/xl_cmdimpl.c | 4 ++ >> 7 files changed, 113 insertions(+), 18 deletions(-) >> >> diff --git a/tools/libxl/libxl_colo.h b/tools/libxl/libxl_colo.h >> index 4746d8c..6c01b55 100644 >> --- a/tools/libxl/libxl_colo.h >> +++ b/tools/libxl/libxl_colo.h >> @@ -69,6 +69,8 @@ struct libxl__colo_proxy_state { >> * False means use kernel colo proxy. >> */ >> bool is_userspace_proxy; >> + const char *checkpoint_host; >> + const char *checkpoint_port; >> }; >> >> struct libxl__colo_save_state { >> diff --git a/tools/libxl/libxl_colo_proxy.c b/tools/libxl/libxl_colo_proxy.c >> index dd902fc..9d21cf1 100644 >> --- a/tools/libxl/libxl_colo_proxy.c >> +++ b/tools/libxl/libxl_colo_proxy.c >> @@ -18,6 +18,9 @@ >> #include "libxl_internal.h" >> >> #include <netlink/netlink.h> >> +#include <arpa/inet.h> >> +#include <sys/socket.h> >> +#include <netinet/in.h> >> >> /* Consistent with the new COLO netlink channel in kernel side */ >> #define NETLINK_COLO 28 >> @@ -76,6 +79,26 @@ static int colo_proxy_send(libxl__colo_proxy_state *cps, uint8_t *buff, >> return ret; >> } >> >> +static int colo_userspace_proxy_recv(libxl__colo_proxy_state *cps, >> + char *buff, >> + unsigned int timeout_us) >> +{ >> + struct timeval tv; >> + int ret; >> + >> + STATE_AO_GC(cps->ao); >> + >> + if (timeout_us) { >> + tv.tv_sec = timeout_us / 1000000; >> + tv.tv_usec = timeout_us % 1000000; >> + setsockopt(cps->sock_fd, SOL_SOCKET, SO_RCVTIMEO, &tv, sizeof(tv)); >> + } >> + >> + ret = recv(cps->sock_fd, buff, sizeof(buff),0); >> + >> + return ret; >> +} >> + >> /* error: return -1, otherwise return 0 */ >> static int64_t colo_proxy_recv(libxl__colo_proxy_state *cps, uint8_t **buff, >> unsigned int timeout_us) >> @@ -153,8 +176,45 @@ int colo_proxy_setup(libxl__colo_proxy_state *cps) >> STATE_AO_GC(cps->ao); >> >> /* If enable userspace proxy mode, we don't need setup kernel proxy */ >> - if (cps->is_userspace_proxy) >> + if (cps->is_userspace_proxy) { >> + struct sockaddr_in addr; >> + int port; >> + char recvbuff[1024]; >> + >> + memset(&addr, 0, sizeof(addr)); >> + port = atoi(cps->checkpoint_port); >> + addr.sin_family = AF_INET; >> + addr.sin_port = htons(port); >> + addr.sin_addr.s_addr = inet_addr(cps->checkpoint_host); >> + >> + skfd = socket(AF_INET, SOCK_STREAM, 0); >> + if (skfd < 0) { >> + LOGD(ERROR, ao->domid, "can not create a TCP socket: %s", >> + strerror(errno)); >> + goto out; >> + } >> + >> + cps->sock_fd = skfd; >> + >> + if (connect(skfd, (struct sockaddr *)&addr, sizeof(addr)) < 0) { >> + LOGD(ERROR, ao->domid, "connect error"); >> + goto out; >> + } >> + >> + char sendbuf[] = "COLO_USERSPACE_PROXY_INIT"; > Um, could you move it right around 'char recvbuff' ? OK, I will move it in next version. >> + ret = send(skfd, sendbuf, strlen(sendbuf),0); >> + if (ret < 0) >> + goto out; >> + >> + ret = colo_userspace_proxy_recv(cps, recvbuff, 500000); > That 500000 looks like a good candidate for a #define? I will add a macro "#define COLO_DEFAULT_WAIT_TIME 500000". Thanks Zhang Chen > >> + if (size < 0) { s/size/ret >> + LOGD(ERROR, ao->domid, "Can't recv msg from qemu colo-compare: %s", >> + strerror(errno)); >> + goto out; >> + } >> + >> return 0; >> + } >> >> skfd = socket(PF_NETLINK, SOCK_RAW, NETLINK_COLO); >> if (skfd < 0) { >> @@ -247,8 +307,11 @@ void colo_proxy_preresume(libxl__colo_proxy_state *cps) >> * If enable userspace proxy mode, >> * we don't need preresume kernel proxy >> */ >> - if (cps->is_userspace_proxy) >> + if (cps->is_userspace_proxy) { >> + char sendbuf[] = "COLO_CHECKPOINT"; >> + send(cps->sock_fd, sendbuf, strlen(sendbuf),0); >> return; >> + } >> >> colo_proxy_send(cps, NULL, 0, COLO_CHECKPOINT); >> /* TODO: need to handle if the call fails... */ >> @@ -277,16 +340,25 @@ int colo_proxy_checkpoint(libxl__colo_proxy_state *cps, >> struct nlmsghdr *h; >> struct colo_msg *m; >> int ret = -1; >> + char recvbuff[1024]; >> >> STATE_AO_GC(cps->ao); >> >> /* >> - * enable userspace proxy mode, tmp sleep. >> - * then we will add qemu API support this func. >> + * enable userspace proxy mode. >> + * Then we will add qemu API support for this func. >> */ >> if (cps->is_userspace_proxy) { >> - sleep(timeout_us / 1000000); >> - return 0; >> + ret = colo_userspace_proxy_recv(cps, recvbuff, timeout_us); >> + if (ret <= 0) >> + return 0; >> + >> + if (!strcmp(recvbuff, "DO_CHECKPOINT")) { >> + return 1; >> + } else { >> + LOGD(ERROR, ao->domid, "receive qemu colo-compare checkpoint error"); >> + return -1; >> + } >> } >> >> size = colo_proxy_recv(cps, &buff, timeout_us); >> diff --git a/tools/libxl/libxl_colo_restore.c b/tools/libxl/libxl_colo_restore.c >> index c6d239a..065ea00 100644 >> --- a/tools/libxl/libxl_colo_restore.c >> +++ b/tools/libxl/libxl_colo_restore.c >> @@ -613,7 +613,8 @@ static void colo_restore_preresume_cb(libxl__egc *egc, >> } >> } >> >> - colo_proxy_preresume(&crs->cps); >> + if (!crs->cps.is_userspace_proxy) >> + colo_proxy_preresume(&crs->cps); >> >> colo_restore_resume_vm(egc, crcs); >> >> @@ -786,9 +787,11 @@ static void colo_setup_checkpoint_devices(libxl__egc *egc, >> cds->ops = colo_restore_ops; >> >> crs->cps.ao = ao; >> - if (colo_proxy_setup(&crs->cps)) { >> - LOGD(ERROR, cds->domid, "COLO: failed to setup colo proxy for guest"); >> - goto out; >> + if (!crs->cps.is_userspace_proxy) { >> + if (colo_proxy_setup(&crs->cps)) { >> + LOGD(ERROR, cds->domid, "COLO: failed to setup colo proxy for guest"); >> + goto out; >> + } >> } >> >> if (init_device_subkind(cds)) >> diff --git a/tools/libxl/libxl_colo_save.c b/tools/libxl/libxl_colo_save.c >> index 91e3fce..a0cfc5a 100644 >> --- a/tools/libxl/libxl_colo_save.c >> +++ b/tools/libxl/libxl_colo_save.c >> @@ -86,6 +86,7 @@ void libxl__colo_save_setup(libxl__egc *egc, libxl__colo_save_state *css) >> libxl__checkpoint_devices_state *const cds = &dss->cds; >> libxl__srm_save_autogen_callbacks *const callbacks = >> &dss->sws.shs.callbacks.save.a; >> + libxl_device_nic *nics; >> >> STATE_AO_GC(dss->ao); >> >> @@ -110,24 +111,31 @@ void libxl__colo_save_setup(libxl__egc *egc, libxl__colo_save_state *css) >> css->colo_proxy_script = GCSPRINTF("%s/colo-proxy-setup", >> libxl__xen_script_dir_path()); >> >> - /* If enable userspace proxy mode, we don't need VIF */ >> - if (css->cps.is_userspace_proxy) >> - cds->device_kind_flags = (1 << LIBXL__DEVICE_KIND_VBD); >> - else >> - cds->device_kind_flags = (1 << LIBXL__DEVICE_KIND_VIF) | >> - (1 << LIBXL__DEVICE_KIND_VBD); >> - >> cds->ops = colo_ops; >> cds->callback = colo_save_setup_done; >> cds->ao = ao; >> cds->domid = dss->domid; >> cds->concrete_data = css; >> >> + /* If enable userspace proxy mode, we don't need VIF */ >> + if (css->cps.is_userspace_proxy) { >> + cds->device_kind_flags = (1 << LIBXL__DEVICE_KIND_VBD); >> + >> + /* Use this args we can connect to qemu colo-compare */ >> + nics = libxl_device_nic_list(CTX, cds->domid, &cds->num_nics); >> + css->cps.checkpoint_host = nics->colo_checkpoint_host; >> + css->cps.checkpoint_port = nics->colo_checkpoint_port; >> + } else { >> + cds->device_kind_flags = (1 << LIBXL__DEVICE_KIND_VIF) | >> + (1 << LIBXL__DEVICE_KIND_VBD); >> + } >> + >> css->srs.ao = ao; >> css->srs.fd = css->recv_fd; >> css->srs.back_channel = true; >> libxl__stream_read_start(egc, &css->srs); >> css->cps.ao = ao; >> + >> if (colo_proxy_setup(&css->cps)) { >> LOGD(ERROR, cds->domid, "COLO: failed to setup colo proxy for guest"); >> goto out; >> diff --git a/tools/libxl/libxl_nic.c b/tools/libxl/libxl_nic.c >> index 5e1fecd..6bc6146 100644 >> --- a/tools/libxl/libxl_nic.c >> +++ b/tools/libxl/libxl_nic.c >> @@ -246,6 +246,8 @@ static void libxl__device_nic_add(libxl__egc *egc, uint32_t domid, >> MAYBE_ADD_COLO_ARGS(filter_sec_redirector1_indev); >> MAYBE_ADD_COLO_ARGS(filter_sec_redirector1_outdev); >> MAYBE_ADD_COLO_ARGS(filter_sec_rewriter0_queue); >> + MAYBE_ADD_COLO_ARGS(checkpoint_host); >> + MAYBE_ADD_COLO_ARGS(checkpoint_port); >> >> #undef MAYBE_ADD_COLO_ARGS >> >> @@ -451,6 +453,8 @@ static int libxl__device_nic_from_xenstore(libxl__gc *gc, >> CHECK_COLO_ARGS(filter_sec_redirector1_indev); >> CHECK_COLO_ARGS(filter_sec_redirector1_outdev); >> CHECK_COLO_ARGS(filter_sec_rewriter0_queue); >> + CHECK_COLO_ARGS(checkpoint_host); >> + CHECK_COLO_ARGS(checkpoint_port); >> >> #undef CHECK_COLO_ARGS >> >> diff --git a/tools/libxl/libxl_types.idl b/tools/libxl/libxl_types.idl >> index 47e96b1..0b412ee 100644 >> --- a/tools/libxl/libxl_types.idl >> +++ b/tools/libxl/libxl_types.idl >> @@ -671,7 +671,9 @@ libxl_device_nic = Struct("device_nic", [ >> ("colo_filter_sec_redirector1_queue", string), >> ("colo_filter_sec_redirector1_indev", string), >> ("colo_filter_sec_redirector1_outdev", string), >> - ("colo_filter_sec_rewriter0_queue", string) >> + ("colo_filter_sec_rewriter0_queue", string), >> + ("colo_checkpoint_host", string), >> + ("colo_checkpoint_port", string) >> ]) >> >> libxl_device_pci = Struct("device_pci", [ >> diff --git a/tools/libxl/xl_cmdimpl.c b/tools/libxl/xl_cmdimpl.c >> index 32a47f6..ad5e193 100644 >> --- a/tools/libxl/xl_cmdimpl.c >> +++ b/tools/libxl/xl_cmdimpl.c >> @@ -1145,6 +1145,10 @@ static int parse_nic_config(libxl_device_nic *nic, XLU_Config **config, char *to >> replace_string(&nic->colo_filter_sec_redirector1_outdev, oparg); >> } else if (MATCH_OPTION("colo_filter_sec_rewriter0_queue", token, oparg)) { >> replace_string(&nic->colo_filter_sec_rewriter0_queue, oparg); >> + } else if (MATCH_OPTION("colo_checkpoint_host", token, oparg)) { >> + replace_string(&nic->colo_checkpoint_host, oparg); >> + } else if (MATCH_OPTION("colo_checkpoint_port", token, oparg)) { >> + replace_string(&nic->colo_checkpoint_port, oparg); >> } else if (MATCH_OPTION("accel", token, oparg)) { >> fprintf(stderr, "the accel parameter for vifs is currently not supported\n"); >> } else { >> -- >> 2.7.4 >> >> >> >> >> _______________________________________________ >> Xen-devel mailing list >> Xen-devel@lists.xen.org >> https://lists.xen.org/xen-devel > > . >
diff --git a/tools/libxl/libxl_colo.h b/tools/libxl/libxl_colo.h index 4746d8c..6c01b55 100644 --- a/tools/libxl/libxl_colo.h +++ b/tools/libxl/libxl_colo.h @@ -69,6 +69,8 @@ struct libxl__colo_proxy_state { * False means use kernel colo proxy. */ bool is_userspace_proxy; + const char *checkpoint_host; + const char *checkpoint_port; }; struct libxl__colo_save_state { diff --git a/tools/libxl/libxl_colo_proxy.c b/tools/libxl/libxl_colo_proxy.c index dd902fc..9d21cf1 100644 --- a/tools/libxl/libxl_colo_proxy.c +++ b/tools/libxl/libxl_colo_proxy.c @@ -18,6 +18,9 @@ #include "libxl_internal.h" #include <netlink/netlink.h> +#include <arpa/inet.h> +#include <sys/socket.h> +#include <netinet/in.h> /* Consistent with the new COLO netlink channel in kernel side */ #define NETLINK_COLO 28 @@ -76,6 +79,26 @@ static int colo_proxy_send(libxl__colo_proxy_state *cps, uint8_t *buff, return ret; } +static int colo_userspace_proxy_recv(libxl__colo_proxy_state *cps, + char *buff, + unsigned int timeout_us) +{ + struct timeval tv; + int ret; + + STATE_AO_GC(cps->ao); + + if (timeout_us) { + tv.tv_sec = timeout_us / 1000000; + tv.tv_usec = timeout_us % 1000000; + setsockopt(cps->sock_fd, SOL_SOCKET, SO_RCVTIMEO, &tv, sizeof(tv)); + } + + ret = recv(cps->sock_fd, buff, sizeof(buff),0); + + return ret; +} + /* error: return -1, otherwise return 0 */ static int64_t colo_proxy_recv(libxl__colo_proxy_state *cps, uint8_t **buff, unsigned int timeout_us) @@ -153,8 +176,45 @@ int colo_proxy_setup(libxl__colo_proxy_state *cps) STATE_AO_GC(cps->ao); /* If enable userspace proxy mode, we don't need setup kernel proxy */ - if (cps->is_userspace_proxy) + if (cps->is_userspace_proxy) { + struct sockaddr_in addr; + int port; + char recvbuff[1024]; + + memset(&addr, 0, sizeof(addr)); + port = atoi(cps->checkpoint_port); + addr.sin_family = AF_INET; + addr.sin_port = htons(port); + addr.sin_addr.s_addr = inet_addr(cps->checkpoint_host); + + skfd = socket(AF_INET, SOCK_STREAM, 0); + if (skfd < 0) { + LOGD(ERROR, ao->domid, "can not create a TCP socket: %s", + strerror(errno)); + goto out; + } + + cps->sock_fd = skfd; + + if (connect(skfd, (struct sockaddr *)&addr, sizeof(addr)) < 0) { + LOGD(ERROR, ao->domid, "connect error"); + goto out; + } + + char sendbuf[] = "COLO_USERSPACE_PROXY_INIT"; + ret = send(skfd, sendbuf, strlen(sendbuf),0); + if (ret < 0) + goto out; + + ret = colo_userspace_proxy_recv(cps, recvbuff, 500000); + if (size < 0) { + LOGD(ERROR, ao->domid, "Can't recv msg from qemu colo-compare: %s", + strerror(errno)); + goto out; + } + return 0; + } skfd = socket(PF_NETLINK, SOCK_RAW, NETLINK_COLO); if (skfd < 0) { @@ -247,8 +307,11 @@ void colo_proxy_preresume(libxl__colo_proxy_state *cps) * If enable userspace proxy mode, * we don't need preresume kernel proxy */ - if (cps->is_userspace_proxy) + if (cps->is_userspace_proxy) { + char sendbuf[] = "COLO_CHECKPOINT"; + send(cps->sock_fd, sendbuf, strlen(sendbuf),0); return; + } colo_proxy_send(cps, NULL, 0, COLO_CHECKPOINT); /* TODO: need to handle if the call fails... */ @@ -277,16 +340,25 @@ int colo_proxy_checkpoint(libxl__colo_proxy_state *cps, struct nlmsghdr *h; struct colo_msg *m; int ret = -1; + char recvbuff[1024]; STATE_AO_GC(cps->ao); /* - * enable userspace proxy mode, tmp sleep. - * then we will add qemu API support this func. + * enable userspace proxy mode. + * Then we will add qemu API support for this func. */ if (cps->is_userspace_proxy) { - sleep(timeout_us / 1000000); - return 0; + ret = colo_userspace_proxy_recv(cps, recvbuff, timeout_us); + if (ret <= 0) + return 0; + + if (!strcmp(recvbuff, "DO_CHECKPOINT")) { + return 1; + } else { + LOGD(ERROR, ao->domid, "receive qemu colo-compare checkpoint error"); + return -1; + } } size = colo_proxy_recv(cps, &buff, timeout_us); diff --git a/tools/libxl/libxl_colo_restore.c b/tools/libxl/libxl_colo_restore.c index c6d239a..065ea00 100644 --- a/tools/libxl/libxl_colo_restore.c +++ b/tools/libxl/libxl_colo_restore.c @@ -613,7 +613,8 @@ static void colo_restore_preresume_cb(libxl__egc *egc, } } - colo_proxy_preresume(&crs->cps); + if (!crs->cps.is_userspace_proxy) + colo_proxy_preresume(&crs->cps); colo_restore_resume_vm(egc, crcs); @@ -786,9 +787,11 @@ static void colo_setup_checkpoint_devices(libxl__egc *egc, cds->ops = colo_restore_ops; crs->cps.ao = ao; - if (colo_proxy_setup(&crs->cps)) { - LOGD(ERROR, cds->domid, "COLO: failed to setup colo proxy for guest"); - goto out; + if (!crs->cps.is_userspace_proxy) { + if (colo_proxy_setup(&crs->cps)) { + LOGD(ERROR, cds->domid, "COLO: failed to setup colo proxy for guest"); + goto out; + } } if (init_device_subkind(cds)) diff --git a/tools/libxl/libxl_colo_save.c b/tools/libxl/libxl_colo_save.c index 91e3fce..a0cfc5a 100644 --- a/tools/libxl/libxl_colo_save.c +++ b/tools/libxl/libxl_colo_save.c @@ -86,6 +86,7 @@ void libxl__colo_save_setup(libxl__egc *egc, libxl__colo_save_state *css) libxl__checkpoint_devices_state *const cds = &dss->cds; libxl__srm_save_autogen_callbacks *const callbacks = &dss->sws.shs.callbacks.save.a; + libxl_device_nic *nics; STATE_AO_GC(dss->ao); @@ -110,24 +111,31 @@ void libxl__colo_save_setup(libxl__egc *egc, libxl__colo_save_state *css) css->colo_proxy_script = GCSPRINTF("%s/colo-proxy-setup", libxl__xen_script_dir_path()); - /* If enable userspace proxy mode, we don't need VIF */ - if (css->cps.is_userspace_proxy) - cds->device_kind_flags = (1 << LIBXL__DEVICE_KIND_VBD); - else - cds->device_kind_flags = (1 << LIBXL__DEVICE_KIND_VIF) | - (1 << LIBXL__DEVICE_KIND_VBD); - cds->ops = colo_ops; cds->callback = colo_save_setup_done; cds->ao = ao; cds->domid = dss->domid; cds->concrete_data = css; + /* If enable userspace proxy mode, we don't need VIF */ + if (css->cps.is_userspace_proxy) { + cds->device_kind_flags = (1 << LIBXL__DEVICE_KIND_VBD); + + /* Use this args we can connect to qemu colo-compare */ + nics = libxl_device_nic_list(CTX, cds->domid, &cds->num_nics); + css->cps.checkpoint_host = nics->colo_checkpoint_host; + css->cps.checkpoint_port = nics->colo_checkpoint_port; + } else { + cds->device_kind_flags = (1 << LIBXL__DEVICE_KIND_VIF) | + (1 << LIBXL__DEVICE_KIND_VBD); + } + css->srs.ao = ao; css->srs.fd = css->recv_fd; css->srs.back_channel = true; libxl__stream_read_start(egc, &css->srs); css->cps.ao = ao; + if (colo_proxy_setup(&css->cps)) { LOGD(ERROR, cds->domid, "COLO: failed to setup colo proxy for guest"); goto out; diff --git a/tools/libxl/libxl_nic.c b/tools/libxl/libxl_nic.c index 5e1fecd..6bc6146 100644 --- a/tools/libxl/libxl_nic.c +++ b/tools/libxl/libxl_nic.c @@ -246,6 +246,8 @@ static void libxl__device_nic_add(libxl__egc *egc, uint32_t domid, MAYBE_ADD_COLO_ARGS(filter_sec_redirector1_indev); MAYBE_ADD_COLO_ARGS(filter_sec_redirector1_outdev); MAYBE_ADD_COLO_ARGS(filter_sec_rewriter0_queue); + MAYBE_ADD_COLO_ARGS(checkpoint_host); + MAYBE_ADD_COLO_ARGS(checkpoint_port); #undef MAYBE_ADD_COLO_ARGS @@ -451,6 +453,8 @@ static int libxl__device_nic_from_xenstore(libxl__gc *gc, CHECK_COLO_ARGS(filter_sec_redirector1_indev); CHECK_COLO_ARGS(filter_sec_redirector1_outdev); CHECK_COLO_ARGS(filter_sec_rewriter0_queue); + CHECK_COLO_ARGS(checkpoint_host); + CHECK_COLO_ARGS(checkpoint_port); #undef CHECK_COLO_ARGS diff --git a/tools/libxl/libxl_types.idl b/tools/libxl/libxl_types.idl index 47e96b1..0b412ee 100644 --- a/tools/libxl/libxl_types.idl +++ b/tools/libxl/libxl_types.idl @@ -671,7 +671,9 @@ libxl_device_nic = Struct("device_nic", [ ("colo_filter_sec_redirector1_queue", string), ("colo_filter_sec_redirector1_indev", string), ("colo_filter_sec_redirector1_outdev", string), - ("colo_filter_sec_rewriter0_queue", string) + ("colo_filter_sec_rewriter0_queue", string), + ("colo_checkpoint_host", string), + ("colo_checkpoint_port", string) ]) libxl_device_pci = Struct("device_pci", [ diff --git a/tools/libxl/xl_cmdimpl.c b/tools/libxl/xl_cmdimpl.c index 32a47f6..ad5e193 100644 --- a/tools/libxl/xl_cmdimpl.c +++ b/tools/libxl/xl_cmdimpl.c @@ -1145,6 +1145,10 @@ static int parse_nic_config(libxl_device_nic *nic, XLU_Config **config, char *to replace_string(&nic->colo_filter_sec_redirector1_outdev, oparg); } else if (MATCH_OPTION("colo_filter_sec_rewriter0_queue", token, oparg)) { replace_string(&nic->colo_filter_sec_rewriter0_queue, oparg); + } else if (MATCH_OPTION("colo_checkpoint_host", token, oparg)) { + replace_string(&nic->colo_checkpoint_host, oparg); + } else if (MATCH_OPTION("colo_checkpoint_port", token, oparg)) { + replace_string(&nic->colo_checkpoint_port, oparg); } else if (MATCH_OPTION("accel", token, oparg)) { fprintf(stderr, "the accel parameter for vifs is currently not supported\n"); } else {
We use kernel colo proxy's way to get the checkpoint event from qemu colo-compare. Qemu colo-compare need add a API to support this(I will add this in qemu). Signed-off-by: Zhang Chen <zhangchen.fnst@cn.fujitsu.com> --- tools/libxl/libxl_colo.h | 2 + tools/libxl/libxl_colo_proxy.c | 84 +++++++++++++++++++++++++++++++++++++--- tools/libxl/libxl_colo_restore.c | 11 ++++-- tools/libxl/libxl_colo_save.c | 22 +++++++---- tools/libxl/libxl_nic.c | 4 ++ tools/libxl/libxl_types.idl | 4 +- tools/libxl/xl_cmdimpl.c | 4 ++ 7 files changed, 113 insertions(+), 18 deletions(-)