@@ -148,6 +148,151 @@ static int vfio_migration_set_state(VFIODevice *vbasedev, uint32_t mask,
return 0;
}
+static void *get_data_section_size(VFIORegion *region, uint64_t data_offset,
+ uint64_t data_size, uint64_t *size)
+{
+ void *ptr = NULL;
+ uint64_t limit = 0;
+ int i;
+
+ if (!region->mmaps) {
+ if (size) {
+ *size = MIN(data_size, region->size - data_offset);
+ }
+ return ptr;
+ }
+
+ for (i = 0; i < region->nr_mmaps; i++) {
+ VFIOMmap *map = region->mmaps + i;
+
+ if ((data_offset >= map->offset) &&
+ (data_offset < map->offset + map->size)) {
+
+ /* check if data_offset is within sparse mmap areas */
+ ptr = map->mmap + data_offset - map->offset;
+ if (size) {
+ *size = MIN(data_size, map->offset + map->size - data_offset);
+ }
+ break;
+ } else if ((data_offset < map->offset) &&
+ (!limit || limit > map->offset)) {
+ /*
+ * data_offset is not within sparse mmap areas, find size of
+ * non-mapped area. Check through all list since region->mmaps list
+ * is not sorted.
+ */
+ limit = map->offset;
+ }
+ }
+
+ if (!ptr && size) {
+ *size = limit ? MIN(data_size, limit - data_offset) : data_size;
+ }
+ return ptr;
+}
+
+static int vfio_save_buffer(QEMUFile *f, VFIODevice *vbasedev, uint64_t *size)
+{
+ VFIOMigration *migration = vbasedev->migration;
+ VFIORegion *region = &migration->region;
+ uint64_t data_offset = 0, data_size = 0, sz;
+ int ret;
+
+ ret = vfio_mig_read(vbasedev, &data_offset, sizeof(data_offset),
+ region->fd_offset + VFIO_MIG_STRUCT_OFFSET(data_offset));
+ if (ret < 0) {
+ return ret;
+ }
+
+ ret = vfio_mig_read(vbasedev, &data_size, sizeof(data_size),
+ region->fd_offset + VFIO_MIG_STRUCT_OFFSET(data_size));
+ if (ret < 0) {
+ return ret;
+ }
+
+ trace_vfio_save_buffer(vbasedev->name, data_offset, data_size,
+ migration->pending_bytes);
+
+ qemu_put_be64(f, data_size);
+ sz = data_size;
+
+ while (sz) {
+ void *buf;
+ uint64_t sec_size;
+ bool buf_allocated = false;
+
+ buf = get_data_section_size(region, data_offset, sz, &sec_size);
+
+ if (!buf) {
+ buf = g_try_malloc(sec_size);
+ if (!buf) {
+ error_report("%s: Error allocating buffer ", __func__);
+ return -ENOMEM;
+ }
+ buf_allocated = true;
+
+ ret = vfio_mig_read(vbasedev, buf, sec_size,
+ region->fd_offset + data_offset);
+ if (ret < 0) {
+ g_free(buf);
+ return ret;
+ }
+ }
+
+ qemu_put_buffer(f, buf, sec_size);
+
+ if (buf_allocated) {
+ g_free(buf);
+ }
+ sz -= sec_size;
+ data_offset += sec_size;
+ }
+
+ ret = qemu_file_get_error(f);
+
+ if (!ret && size) {
+ *size = data_size;
+ }
+
+ return ret;
+}
+
+static int vfio_update_pending(VFIODevice *vbasedev)
+{
+ VFIOMigration *migration = vbasedev->migration;
+ VFIORegion *region = &migration->region;
+ uint64_t pending_bytes = 0;
+ int ret;
+
+ ret = vfio_mig_read(vbasedev, &pending_bytes, sizeof(pending_bytes),
+ region->fd_offset + VFIO_MIG_STRUCT_OFFSET(pending_bytes));
+ if (ret < 0) {
+ migration->pending_bytes = 0;
+ return ret;
+ }
+
+ migration->pending_bytes = pending_bytes;
+ trace_vfio_update_pending(vbasedev->name, pending_bytes);
+ return 0;
+}
+
+static int vfio_save_device_config_state(QEMUFile *f, void *opaque)
+{
+ VFIODevice *vbasedev = opaque;
+
+ qemu_put_be64(f, VFIO_MIG_FLAG_DEV_CONFIG_STATE);
+
+ if (vbasedev->ops && vbasedev->ops->vfio_save_config) {
+ vbasedev->ops->vfio_save_config(vbasedev, f);
+ }
+
+ qemu_put_be64(f, VFIO_MIG_FLAG_END_OF_STATE);
+
+ trace_vfio_save_device_config_state(vbasedev->name);
+
+ return qemu_file_get_error(f);
+}
+
static void vfio_migration_cleanup(VFIODevice *vbasedev)
{
VFIOMigration *migration = vbasedev->migration;
@@ -210,9 +355,140 @@ static void vfio_save_cleanup(void *opaque)
trace_vfio_save_cleanup(vbasedev->name);
}
+static void vfio_save_pending(QEMUFile *f, void *opaque,
+ uint64_t threshold_size,
+ uint64_t *res_precopy_only,
+ uint64_t *res_compatible,
+ uint64_t *res_postcopy_only)
+{
+ VFIODevice *vbasedev = opaque;
+ VFIOMigration *migration = vbasedev->migration;
+ int ret;
+
+ ret = vfio_update_pending(vbasedev);
+ if (ret) {
+ return;
+ }
+
+ *res_precopy_only += migration->pending_bytes;
+
+ trace_vfio_save_pending(vbasedev->name, *res_precopy_only,
+ *res_postcopy_only, *res_compatible);
+}
+
+static int vfio_save_iterate(QEMUFile *f, void *opaque)
+{
+ VFIODevice *vbasedev = opaque;
+ VFIOMigration *migration = vbasedev->migration;
+ uint64_t data_size;
+ int ret;
+
+ qemu_put_be64(f, VFIO_MIG_FLAG_DEV_DATA_STATE);
+
+ if (migration->pending_bytes == 0) {
+ ret = vfio_update_pending(vbasedev);
+ if (ret) {
+ return ret;
+ }
+
+ if (migration->pending_bytes == 0) {
+ qemu_put_be64(f, 0);
+ qemu_put_be64(f, VFIO_MIG_FLAG_END_OF_STATE);
+ /* indicates data finished, goto complete phase */
+ return 1;
+ }
+ }
+
+ ret = vfio_save_buffer(f, vbasedev, &data_size);
+ if (ret) {
+ error_report("%s: vfio_save_buffer failed %s", vbasedev->name,
+ strerror(errno));
+ return ret;
+ }
+
+ qemu_put_be64(f, VFIO_MIG_FLAG_END_OF_STATE);
+
+ ret = qemu_file_get_error(f);
+ if (ret) {
+ return ret;
+ }
+
+ /*
+ * Reset pending_bytes as .save_live_pending is not called during savevm or
+ * snapshot case, in such case vfio_update_pending() at the start of this
+ * function updates pending_bytes.
+ */
+ migration->pending_bytes = 0;
+ trace_vfio_save_iterate(vbasedev->name, data_size);
+ return 0;
+}
+
+static int vfio_save_complete_precopy(QEMUFile *f, void *opaque)
+{
+ VFIODevice *vbasedev = opaque;
+ VFIOMigration *migration = vbasedev->migration;
+ uint64_t data_size;
+ int ret;
+
+ ret = vfio_migration_set_state(vbasedev, ~VFIO_DEVICE_STATE_RUNNING,
+ VFIO_DEVICE_STATE_SAVING);
+ if (ret) {
+ error_report("%s: Failed to set state STOP and SAVING",
+ vbasedev->name);
+ return ret;
+ }
+
+ ret = vfio_save_device_config_state(f, opaque);
+ if (ret) {
+ return ret;
+ }
+
+ ret = vfio_update_pending(vbasedev);
+ if (ret) {
+ return ret;
+ }
+
+ while (migration->pending_bytes > 0) {
+ qemu_put_be64(f, VFIO_MIG_FLAG_DEV_DATA_STATE);
+ ret = vfio_save_buffer(f, vbasedev, &data_size);
+ if (ret < 0) {
+ error_report("%s: Failed to save buffer", vbasedev->name);
+ return ret;
+ }
+
+ if (data_size == 0) {
+ break;
+ }
+
+ ret = vfio_update_pending(vbasedev);
+ if (ret) {
+ return ret;
+ }
+ }
+
+ qemu_put_be64(f, VFIO_MIG_FLAG_END_OF_STATE);
+
+ ret = qemu_file_get_error(f);
+ if (ret) {
+ return ret;
+ }
+
+ ret = vfio_migration_set_state(vbasedev, ~VFIO_DEVICE_STATE_SAVING, 0);
+ if (ret) {
+ error_report("%s: Failed to set state STOPPED", vbasedev->name);
+ return ret;
+ }
+
+ trace_vfio_save_complete_precopy(vbasedev->name);
+ return ret;
+}
+
static SaveVMHandlers savevm_vfio_handlers = {
.save_setup = vfio_save_setup,
.save_cleanup = vfio_save_cleanup,
+ .save_live_pending = vfio_save_pending,
+ .save_live_iterate = vfio_save_iterate,
+ .save_live_complete_precopy = vfio_save_complete_precopy,
};
/* ---------------------------------------------------------------------- */
@@ -153,3 +153,9 @@ vfio_vmstate_change(const char *name, int running, const char *reason, uint32_t
vfio_migration_state_notifier(const char *name, const char *state) " (%s) state %s"
vfio_save_setup(const char *name) " (%s)"
vfio_save_cleanup(const char *name) " (%s)"
+vfio_save_buffer(const char *name, uint64_t data_offset, uint64_t data_size, uint64_t pending) " (%s) Offset 0x%"PRIx64" size 0x%"PRIx64" pending 0x%"PRIx64
+vfio_update_pending(const char *name, uint64_t pending) " (%s) pending 0x%"PRIx64
+vfio_save_device_config_state(const char *name) " (%s)"
+vfio_save_pending(const char *name, uint64_t precopy, uint64_t postcopy, uint64_t compatible) " (%s) precopy 0x%"PRIx64" postcopy 0x%"PRIx64" compatible 0x%"PRIx64
+vfio_save_iterate(const char *name, int data_size) " (%s) data_size %d"
+vfio_save_complete_precopy(const char *name) " (%s)"
@@ -65,6 +65,7 @@ typedef struct VFIOMigration {
uint32_t device_state;
int vm_running;
Notifier migration_state;
+ uint64_t pending_bytes;
} VFIOMigration;
typedef struct VFIOAddressSpace {