Message ID | 1603365127-14202-5-git-send-email-kwankhede@nvidia.com (mailing list archive) |
---|---|
State | New, archived |
Headers | show |
Series | Add migration support for VFIO devices | expand |
On Thu, 22 Oct 2020 16:41:54 +0530 Kirti Wankhede <kwankhede@nvidia.com> wrote: > Whether the VFIO device supports migration or not is decided based of > migration region query. If migration region query is successful and migration > region initialization is successful then migration is supported else > migration is blocked. > > Signed-off-by: Kirti Wankhede <kwankhede@nvidia.com> > Reviewed-by: Neo Jia <cjia@nvidia.com> > Acked-by: Dr. David Alan Gilbert <dgilbert@redhat.com> > --- > hw/vfio/meson.build | 1 + > hw/vfio/migration.c | 129 ++++++++++++++++++++++++++++++++++++++++++ > hw/vfio/trace-events | 3 + > include/hw/vfio/vfio-common.h | 9 +++ > 4 files changed, 142 insertions(+) > create mode 100644 hw/vfio/migration.c > > diff --git a/hw/vfio/meson.build b/hw/vfio/meson.build > index 37efa74018bc..da9af297a0c5 100644 > --- a/hw/vfio/meson.build > +++ b/hw/vfio/meson.build > @@ -2,6 +2,7 @@ vfio_ss = ss.source_set() > vfio_ss.add(files( > 'common.c', > 'spapr.c', > + 'migration.c', > )) > vfio_ss.add(when: 'CONFIG_VFIO_PCI', if_true: files( > 'display.c', > diff --git a/hw/vfio/migration.c b/hw/vfio/migration.c > new file mode 100644 > index 000000000000..5f74a3ad1d72 > --- /dev/null > +++ b/hw/vfio/migration.c > @@ -0,0 +1,129 @@ > +/* > + * Migration support for VFIO devices > + * > + * Copyright NVIDIA, Inc. 2020 > + * > + * This work is licensed under the terms of the GNU GPL, version 2. See > + * the COPYING file in the top-level directory. > + */ > + > +#include "qemu/osdep.h" > +#include <linux/vfio.h> > + > +#include "hw/vfio/vfio-common.h" > +#include "cpu.h" > +#include "migration/migration.h" > +#include "migration/qemu-file.h" > +#include "migration/register.h" > +#include "migration/blocker.h" > +#include "migration/misc.h" > +#include "qapi/error.h" > +#include "exec/ramlist.h" > +#include "exec/ram_addr.h" > +#include "pci.h" > +#include "trace.h" > + > +static void vfio_migration_region_exit(VFIODevice *vbasedev) > +{ > + VFIOMigration *migration = vbasedev->migration; > + > + if (!migration) { > + return; > + } > + > + if (migration->region.size) { > + vfio_region_exit(&migration->region); > + vfio_region_finalize(&migration->region); > + } > +} > + > +static int vfio_migration_init(VFIODevice *vbasedev, > + struct vfio_region_info *info) > +{ > + int ret; > + Object *obj; > + VFIOMigration *migration; > + > + if (!vbasedev->ops->vfio_get_object) { > + return -EINVAL; > + } > + > + obj = vbasedev->ops->vfio_get_object(vbasedev); > + if (!obj) { > + return -EINVAL; > + } > + > + migration = g_new0(VFIOMigration, 1); > + > + ret = vfio_region_setup(obj, vbasedev, &migration->region, > + info->index, "migration"); > + if (ret) { > + error_report("%s: Failed to setup VFIO migration region %d: %s", > + vbasedev->name, info->index, strerror(-ret)); > + goto err; > + } > + > + if (!migration->region.size) { > + error_report("%s: Invalid zero-sized of VFIO migration region %d", > + vbasedev->name, info->index); > + ret = -EINVAL; > + goto err; > + } > + > + vbasedev->migration = migration; > + return 0; > + > +err: > + vfio_migration_region_exit(vbasedev); We can't get here with vbasedev->migration set, did you intend to set vbasedev->migration before testing region.size? Thanks, Alex > + g_free(migration); > + return ret; > +} > + > +/* ---------------------------------------------------------------------- */ > + > +int vfio_migration_probe(VFIODevice *vbasedev, Error **errp) > +{ > + struct vfio_region_info *info = NULL; > + Error *local_err = NULL; > + int ret; > + > + ret = vfio_get_dev_region_info(vbasedev, VFIO_REGION_TYPE_MIGRATION, > + VFIO_REGION_SUBTYPE_MIGRATION, &info); > + if (ret) { > + goto add_blocker; > + } > + > + ret = vfio_migration_init(vbasedev, info); > + if (ret) { > + goto add_blocker; > + } > + > + g_free(info); > + trace_vfio_migration_probe(vbasedev->name, info->index); > + return 0; > + > +add_blocker: > + error_setg(&vbasedev->migration_blocker, > + "VFIO device doesn't support migration"); > + g_free(info); > + > + ret = migrate_add_blocker(vbasedev->migration_blocker, &local_err); > + if (local_err) { > + error_propagate(errp, local_err); > + error_free(vbasedev->migration_blocker); > + vbasedev->migration_blocker = NULL; > + } > + return ret; > +} > + > +void vfio_migration_finalize(VFIODevice *vbasedev) > +{ > + if (vbasedev->migration_blocker) { > + migrate_del_blocker(vbasedev->migration_blocker); > + error_free(vbasedev->migration_blocker); > + vbasedev->migration_blocker = NULL; > + } > + > + vfio_migration_region_exit(vbasedev); > + g_free(vbasedev->migration); > +} > diff --git a/hw/vfio/trace-events b/hw/vfio/trace-events > index a0c7b49a2ebc..9ced5ec6277c 100644 > --- a/hw/vfio/trace-events > +++ b/hw/vfio/trace-events > @@ -145,3 +145,6 @@ vfio_display_edid_link_up(void) "" > vfio_display_edid_link_down(void) "" > vfio_display_edid_update(uint32_t prefx, uint32_t prefy) "%ux%u" > vfio_display_edid_write_error(void) "" > + > +# migration.c > +vfio_migration_probe(const char *name, uint32_t index) " (%s) Region %d" > diff --git a/include/hw/vfio/vfio-common.h b/include/hw/vfio/vfio-common.h > index ba6169cd926e..8275c4c68f45 100644 > --- a/include/hw/vfio/vfio-common.h > +++ b/include/hw/vfio/vfio-common.h > @@ -57,6 +57,10 @@ typedef struct VFIORegion { > uint8_t nr; /* cache the region number for debug */ > } VFIORegion; > > +typedef struct VFIOMigration { > + VFIORegion region; > +} VFIOMigration; > + > typedef struct VFIOAddressSpace { > AddressSpace *as; > QLIST_HEAD(, VFIOContainer) containers; > @@ -113,6 +117,8 @@ typedef struct VFIODevice { > unsigned int num_irqs; > unsigned int num_regions; > unsigned int flags; > + VFIOMigration *migration; > + Error *migration_blocker; > } VFIODevice; > > struct VFIODeviceOps { > @@ -204,4 +210,7 @@ int vfio_spapr_create_window(VFIOContainer *container, > int vfio_spapr_remove_window(VFIOContainer *container, > hwaddr offset_within_address_space); > > +int vfio_migration_probe(VFIODevice *vbasedev, Error **errp); > +void vfio_migration_finalize(VFIODevice *vbasedev); > + > #endif /* HW_VFIO_VFIO_COMMON_H */
On 10/22/2020 7:52 PM, Alex Williamson wrote: > On Thu, 22 Oct 2020 16:41:54 +0530 > Kirti Wankhede <kwankhede@nvidia.com> wrote: > >> Whether the VFIO device supports migration or not is decided based of >> migration region query. If migration region query is successful and migration >> region initialization is successful then migration is supported else >> migration is blocked. >> >> Signed-off-by: Kirti Wankhede <kwankhede@nvidia.com> >> Reviewed-by: Neo Jia <cjia@nvidia.com> >> Acked-by: Dr. David Alan Gilbert <dgilbert@redhat.com> >> --- >> hw/vfio/meson.build | 1 + >> hw/vfio/migration.c | 129 ++++++++++++++++++++++++++++++++++++++++++ >> hw/vfio/trace-events | 3 + >> include/hw/vfio/vfio-common.h | 9 +++ >> 4 files changed, 142 insertions(+) >> create mode 100644 hw/vfio/migration.c >> >> diff --git a/hw/vfio/meson.build b/hw/vfio/meson.build >> index 37efa74018bc..da9af297a0c5 100644 >> --- a/hw/vfio/meson.build >> +++ b/hw/vfio/meson.build >> @@ -2,6 +2,7 @@ vfio_ss = ss.source_set() >> vfio_ss.add(files( >> 'common.c', >> 'spapr.c', >> + 'migration.c', >> )) >> vfio_ss.add(when: 'CONFIG_VFIO_PCI', if_true: files( >> 'display.c', >> diff --git a/hw/vfio/migration.c b/hw/vfio/migration.c >> new file mode 100644 >> index 000000000000..5f74a3ad1d72 >> --- /dev/null >> +++ b/hw/vfio/migration.c >> @@ -0,0 +1,129 @@ >> +/* >> + * Migration support for VFIO devices >> + * >> + * Copyright NVIDIA, Inc. 2020 >> + * >> + * This work is licensed under the terms of the GNU GPL, version 2. See >> + * the COPYING file in the top-level directory. >> + */ >> + >> +#include "qemu/osdep.h" >> +#include <linux/vfio.h> >> + >> +#include "hw/vfio/vfio-common.h" >> +#include "cpu.h" >> +#include "migration/migration.h" >> +#include "migration/qemu-file.h" >> +#include "migration/register.h" >> +#include "migration/blocker.h" >> +#include "migration/misc.h" >> +#include "qapi/error.h" >> +#include "exec/ramlist.h" >> +#include "exec/ram_addr.h" >> +#include "pci.h" >> +#include "trace.h" >> + >> +static void vfio_migration_region_exit(VFIODevice *vbasedev) >> +{ >> + VFIOMigration *migration = vbasedev->migration; >> + >> + if (!migration) { >> + return; >> + } >> + >> + if (migration->region.size) { >> + vfio_region_exit(&migration->region); >> + vfio_region_finalize(&migration->region); >> + } >> +} >> + >> +static int vfio_migration_init(VFIODevice *vbasedev, >> + struct vfio_region_info *info) >> +{ >> + int ret; >> + Object *obj; >> + VFIOMigration *migration; >> + >> + if (!vbasedev->ops->vfio_get_object) { >> + return -EINVAL; >> + } >> + >> + obj = vbasedev->ops->vfio_get_object(vbasedev); >> + if (!obj) { >> + return -EINVAL; >> + } >> + >> + migration = g_new0(VFIOMigration, 1); >> + >> + ret = vfio_region_setup(obj, vbasedev, &migration->region, >> + info->index, "migration"); >> + if (ret) { >> + error_report("%s: Failed to setup VFIO migration region %d: %s", >> + vbasedev->name, info->index, strerror(-ret)); >> + goto err; >> + } >> + >> + if (!migration->region.size) { >> + error_report("%s: Invalid zero-sized of VFIO migration region %d", >> + vbasedev->name, info->index); >> + ret = -EINVAL; >> + goto err; >> + } >> + >> + vbasedev->migration = migration; >> + return 0; >> + >> +err: >> + vfio_migration_region_exit(vbasedev); > > We can't get here with vbasedev->migration set, did you intend to set > vbasedev->migration before testing region.size? Thanks, > Oh yes, I missed to address this when I moved migration variable to VFIODevice. Moving vbasedev->migration before region.size check. Also removing region.size check vfio_migration_region_exit() for vfio_region_exit() and vfio_region_finalize(). Thanks, Kirti > Alex > > > >> + g_free(migration); >> + return ret; >> +} >> + >> +/* ---------------------------------------------------------------------- */ >> + >> +int vfio_migration_probe(VFIODevice *vbasedev, Error **errp) >> +{ >> + struct vfio_region_info *info = NULL; >> + Error *local_err = NULL; >> + int ret; >> + >> + ret = vfio_get_dev_region_info(vbasedev, VFIO_REGION_TYPE_MIGRATION, >> + VFIO_REGION_SUBTYPE_MIGRATION, &info); >> + if (ret) { >> + goto add_blocker; >> + } >> + >> + ret = vfio_migration_init(vbasedev, info); >> + if (ret) { >> + goto add_blocker; >> + } >> + >> + g_free(info); >> + trace_vfio_migration_probe(vbasedev->name, info->index); >> + return 0; >> + >> +add_blocker: >> + error_setg(&vbasedev->migration_blocker, >> + "VFIO device doesn't support migration"); >> + g_free(info); >> + >> + ret = migrate_add_blocker(vbasedev->migration_blocker, &local_err); >> + if (local_err) { >> + error_propagate(errp, local_err); >> + error_free(vbasedev->migration_blocker); >> + vbasedev->migration_blocker = NULL; >> + } >> + return ret; >> +} >> + >> +void vfio_migration_finalize(VFIODevice *vbasedev) >> +{ >> + if (vbasedev->migration_blocker) { >> + migrate_del_blocker(vbasedev->migration_blocker); >> + error_free(vbasedev->migration_blocker); >> + vbasedev->migration_blocker = NULL; >> + } >> + >> + vfio_migration_region_exit(vbasedev); >> + g_free(vbasedev->migration); >> +} >> diff --git a/hw/vfio/trace-events b/hw/vfio/trace-events >> index a0c7b49a2ebc..9ced5ec6277c 100644 >> --- a/hw/vfio/trace-events >> +++ b/hw/vfio/trace-events >> @@ -145,3 +145,6 @@ vfio_display_edid_link_up(void) "" >> vfio_display_edid_link_down(void) "" >> vfio_display_edid_update(uint32_t prefx, uint32_t prefy) "%ux%u" >> vfio_display_edid_write_error(void) "" >> + >> +# migration.c >> +vfio_migration_probe(const char *name, uint32_t index) " (%s) Region %d" >> diff --git a/include/hw/vfio/vfio-common.h b/include/hw/vfio/vfio-common.h >> index ba6169cd926e..8275c4c68f45 100644 >> --- a/include/hw/vfio/vfio-common.h >> +++ b/include/hw/vfio/vfio-common.h >> @@ -57,6 +57,10 @@ typedef struct VFIORegion { >> uint8_t nr; /* cache the region number for debug */ >> } VFIORegion; >> >> +typedef struct VFIOMigration { >> + VFIORegion region; >> +} VFIOMigration; >> + >> typedef struct VFIOAddressSpace { >> AddressSpace *as; >> QLIST_HEAD(, VFIOContainer) containers; >> @@ -113,6 +117,8 @@ typedef struct VFIODevice { >> unsigned int num_irqs; >> unsigned int num_regions; >> unsigned int flags; >> + VFIOMigration *migration; >> + Error *migration_blocker; >> } VFIODevice; >> >> struct VFIODeviceOps { >> @@ -204,4 +210,7 @@ int vfio_spapr_create_window(VFIOContainer *container, >> int vfio_spapr_remove_window(VFIOContainer *container, >> hwaddr offset_within_address_space); >> >> +int vfio_migration_probe(VFIODevice *vbasedev, Error **errp); >> +void vfio_migration_finalize(VFIODevice *vbasedev); >> + >> #endif /* HW_VFIO_VFIO_COMMON_H */ >
On Thu, 22 Oct 2020 16:41:54 +0530 Kirti Wankhede <kwankhede@nvidia.com> wrote: > Whether the VFIO device supports migration or not is decided based of > migration region query. If migration region query is successful and migration > region initialization is successful then migration is supported else > migration is blocked. > > Signed-off-by: Kirti Wankhede <kwankhede@nvidia.com> > Reviewed-by: Neo Jia <cjia@nvidia.com> > Acked-by: Dr. David Alan Gilbert <dgilbert@redhat.com> > --- > hw/vfio/meson.build | 1 + > hw/vfio/migration.c | 129 ++++++++++++++++++++++++++++++++++++++++++ > hw/vfio/trace-events | 3 + > include/hw/vfio/vfio-common.h | 9 +++ > 4 files changed, 142 insertions(+) > create mode 100644 hw/vfio/migration.c (...) > +static int vfio_migration_init(VFIODevice *vbasedev, > + struct vfio_region_info *info) > +{ > + int ret; > + Object *obj; > + VFIOMigration *migration; > + > + if (!vbasedev->ops->vfio_get_object) { > + return -EINVAL; > + } > + > + obj = vbasedev->ops->vfio_get_object(vbasedev); > + if (!obj) { > + return -EINVAL; > + } > + > + migration = g_new0(VFIOMigration, 1); > + > + ret = vfio_region_setup(obj, vbasedev, &migration->region, > + info->index, "migration"); > + if (ret) { > + error_report("%s: Failed to setup VFIO migration region %d: %s", > + vbasedev->name, info->index, strerror(-ret)); > + goto err; > + } > + > + if (!migration->region.size) { > + error_report("%s: Invalid zero-sized of VFIO migration region %d", s/of // > + vbasedev->name, info->index); > + ret = -EINVAL; > + goto err; > + } > + > + vbasedev->migration = migration; > + return 0; > + > +err: > + vfio_migration_region_exit(vbasedev); > + g_free(migration); > + return ret; > +} (...)
diff --git a/hw/vfio/meson.build b/hw/vfio/meson.build index 37efa74018bc..da9af297a0c5 100644 --- a/hw/vfio/meson.build +++ b/hw/vfio/meson.build @@ -2,6 +2,7 @@ vfio_ss = ss.source_set() vfio_ss.add(files( 'common.c', 'spapr.c', + 'migration.c', )) vfio_ss.add(when: 'CONFIG_VFIO_PCI', if_true: files( 'display.c', diff --git a/hw/vfio/migration.c b/hw/vfio/migration.c new file mode 100644 index 000000000000..5f74a3ad1d72 --- /dev/null +++ b/hw/vfio/migration.c @@ -0,0 +1,129 @@ +/* + * Migration support for VFIO devices + * + * Copyright NVIDIA, Inc. 2020 + * + * This work is licensed under the terms of the GNU GPL, version 2. See + * the COPYING file in the top-level directory. + */ + +#include "qemu/osdep.h" +#include <linux/vfio.h> + +#include "hw/vfio/vfio-common.h" +#include "cpu.h" +#include "migration/migration.h" +#include "migration/qemu-file.h" +#include "migration/register.h" +#include "migration/blocker.h" +#include "migration/misc.h" +#include "qapi/error.h" +#include "exec/ramlist.h" +#include "exec/ram_addr.h" +#include "pci.h" +#include "trace.h" + +static void vfio_migration_region_exit(VFIODevice *vbasedev) +{ + VFIOMigration *migration = vbasedev->migration; + + if (!migration) { + return; + } + + if (migration->region.size) { + vfio_region_exit(&migration->region); + vfio_region_finalize(&migration->region); + } +} + +static int vfio_migration_init(VFIODevice *vbasedev, + struct vfio_region_info *info) +{ + int ret; + Object *obj; + VFIOMigration *migration; + + if (!vbasedev->ops->vfio_get_object) { + return -EINVAL; + } + + obj = vbasedev->ops->vfio_get_object(vbasedev); + if (!obj) { + return -EINVAL; + } + + migration = g_new0(VFIOMigration, 1); + + ret = vfio_region_setup(obj, vbasedev, &migration->region, + info->index, "migration"); + if (ret) { + error_report("%s: Failed to setup VFIO migration region %d: %s", + vbasedev->name, info->index, strerror(-ret)); + goto err; + } + + if (!migration->region.size) { + error_report("%s: Invalid zero-sized of VFIO migration region %d", + vbasedev->name, info->index); + ret = -EINVAL; + goto err; + } + + vbasedev->migration = migration; + return 0; + +err: + vfio_migration_region_exit(vbasedev); + g_free(migration); + return ret; +} + +/* ---------------------------------------------------------------------- */ + +int vfio_migration_probe(VFIODevice *vbasedev, Error **errp) +{ + struct vfio_region_info *info = NULL; + Error *local_err = NULL; + int ret; + + ret = vfio_get_dev_region_info(vbasedev, VFIO_REGION_TYPE_MIGRATION, + VFIO_REGION_SUBTYPE_MIGRATION, &info); + if (ret) { + goto add_blocker; + } + + ret = vfio_migration_init(vbasedev, info); + if (ret) { + goto add_blocker; + } + + g_free(info); + trace_vfio_migration_probe(vbasedev->name, info->index); + return 0; + +add_blocker: + error_setg(&vbasedev->migration_blocker, + "VFIO device doesn't support migration"); + g_free(info); + + ret = migrate_add_blocker(vbasedev->migration_blocker, &local_err); + if (local_err) { + error_propagate(errp, local_err); + error_free(vbasedev->migration_blocker); + vbasedev->migration_blocker = NULL; + } + return ret; +} + +void vfio_migration_finalize(VFIODevice *vbasedev) +{ + if (vbasedev->migration_blocker) { + migrate_del_blocker(vbasedev->migration_blocker); + error_free(vbasedev->migration_blocker); + vbasedev->migration_blocker = NULL; + } + + vfio_migration_region_exit(vbasedev); + g_free(vbasedev->migration); +} diff --git a/hw/vfio/trace-events b/hw/vfio/trace-events index a0c7b49a2ebc..9ced5ec6277c 100644 --- a/hw/vfio/trace-events +++ b/hw/vfio/trace-events @@ -145,3 +145,6 @@ vfio_display_edid_link_up(void) "" vfio_display_edid_link_down(void) "" vfio_display_edid_update(uint32_t prefx, uint32_t prefy) "%ux%u" vfio_display_edid_write_error(void) "" + +# migration.c +vfio_migration_probe(const char *name, uint32_t index) " (%s) Region %d" diff --git a/include/hw/vfio/vfio-common.h b/include/hw/vfio/vfio-common.h index ba6169cd926e..8275c4c68f45 100644 --- a/include/hw/vfio/vfio-common.h +++ b/include/hw/vfio/vfio-common.h @@ -57,6 +57,10 @@ typedef struct VFIORegion { uint8_t nr; /* cache the region number for debug */ } VFIORegion; +typedef struct VFIOMigration { + VFIORegion region; +} VFIOMigration; + typedef struct VFIOAddressSpace { AddressSpace *as; QLIST_HEAD(, VFIOContainer) containers; @@ -113,6 +117,8 @@ typedef struct VFIODevice { unsigned int num_irqs; unsigned int num_regions; unsigned int flags; + VFIOMigration *migration; + Error *migration_blocker; } VFIODevice; struct VFIODeviceOps { @@ -204,4 +210,7 @@ int vfio_spapr_create_window(VFIOContainer *container, int vfio_spapr_remove_window(VFIOContainer *container, hwaddr offset_within_address_space); +int vfio_migration_probe(VFIODevice *vbasedev, Error **errp); +void vfio_migration_finalize(VFIODevice *vbasedev); + #endif /* HW_VFIO_VFIO_COMMON_H */