@@ -586,11 +586,12 @@ static void vfio_listener_region_add(MemoryListener *listener,
{
VFIOContainerBase *bcontainer = container_of(listener, VFIOContainerBase,
listener);
- vfio_container_region_add(bcontainer, section);
+ vfio_container_region_add(bcontainer, section, false);
}
void vfio_container_region_add(VFIOContainerBase *bcontainer,
- MemoryRegionSection *section)
+ MemoryRegionSection *section,
+ bool remap)
{
hwaddr iova, end;
Int128 llend, llsize;
@@ -626,6 +627,30 @@ void vfio_container_region_add(VFIOContainerBase *bcontainer,
int iommu_idx;
trace_vfio_listener_region_add_iommu(iova, end);
+
+ /*
+ * If remap, then VFIO_DMA_UNMAP_FLAG_VADDR has been called, and we
+ * want to remap the vaddr. vfio_container_region_add was already
+ * called in the past, so the giommu already exists. Find it and
+ * replay it, which calls vfio_dma_map further down the stack.
+ */
+
+ if (remap) {
+ hwaddr as_offset = section->offset_within_address_space;
+ hwaddr iommu_offset = as_offset - section->offset_within_region;
+
+ QLIST_FOREACH(giommu, &bcontainer->giommu_list, giommu_next) {
+ if (giommu->iommu_mr == iommu_mr &&
+ giommu->iommu_offset == iommu_offset) {
+ memory_region_iommu_replay(giommu->iommu_mr, &giommu->n);
+ return;
+ }
+ }
+ error_report("Container cannot find iommu region %s offset %lx",
+ memory_region_name(section->mr), iommu_offset);
+ goto fail;
+ }
+
/*
* FIXME: For VFIO iommu types which have KVM acceleration to
* avoid bouncing all map/unmaps through qemu this way, this
@@ -676,7 +701,21 @@ void vfio_container_region_add(VFIOContainerBase *bcontainer,
* about changes.
*/
if (memory_region_has_ram_discard_manager(section->mr)) {
- vfio_register_ram_discard_listener(bcontainer, section);
+ /*
+ * If remap, then VFIO_DMA_UNMAP_FLAG_VADDR has been called, and we
+ * want to remap the vaddr. vfio_container_region_add was already
+ * called in the past, so the ram discard listener already exists.
+ * Call its populate function directly, which calls vfio_dma_map.
+ */
+ if (remap) {
+ VFIORamDiscardListener *vrdl =
+ vfio_find_ram_discard_listener(bcontainer, section);
+ if (vrdl->listener.notify_populate(&vrdl->listener, section)) {
+ error_report("listener.notify_populate failed");
+ }
+ } else {
+ vfio_register_ram_discard_listener(bcontainer, section);
+ }
return;
}
@@ -29,9 +29,18 @@ static bool vfio_dma_unmap_vaddr_all(VFIOContainer *container, Error **errp)
error_setg_errno(errp, errno, "vfio_dma_unmap_vaddr_all");
return false;
}
+ container->vaddr_unmapped = true;
return true;
}
+static void vfio_region_remap(MemoryListener *listener,
+ MemoryRegionSection *section)
+{
+ VFIOContainerBase *bcontainer = container_of(listener, VFIOContainerBase,
+ remap_listener);
+ vfio_container_region_add(bcontainer, section, true);
+}
+
static bool vfio_can_cpr_exec(VFIOContainer *container, Error **errp)
{
if (!ioctl(container->fd, VFIO_CHECK_EXTENSION, VFIO_UPDATE_VADDR)) {
@@ -95,6 +104,37 @@ static const VMStateDescription vfio_container_vmstate = {
}
};
+static int vfio_cpr_fail_notifier(NotifierWithReturn *notifier,
+ MigrationEvent *e, Error **errp)
+{
+ VFIOContainer *container =
+ container_of(notifier, VFIOContainer, cpr_exec_notifier);
+ VFIOContainerBase *bcontainer = &container->bcontainer;
+
+ if (e->type != MIG_EVENT_PRECOPY_FAILED) {
+ return 0;
+ }
+
+ if (container->vaddr_unmapped) {
+ /*
+ * Force a call to vfio_region_remap for each mapped section by
+ * temporarily registering a listener, which calls vfio_dma_map
+ * further down the stack. Set reused so vfio_dma_map restores vaddr.
+ */
+ bcontainer->reused = true;
+ bcontainer->remap_listener = (MemoryListener) {
+ .name = "vfio recover",
+ .region_add = vfio_region_remap
+ };
+ memory_listener_register(&bcontainer->remap_listener,
+ bcontainer->space->as);
+ memory_listener_unregister(&bcontainer->remap_listener);
+ bcontainer->reused = false;
+ container->vaddr_unmapped = false;
+ }
+ return 0;
+}
+
bool vfio_legacy_cpr_register_container(VFIOContainerBase *bcontainer,
Error **errp)
{
@@ -107,6 +147,9 @@ bool vfio_legacy_cpr_register_container(VFIOContainerBase *bcontainer,
vmstate_register(NULL, -1, &vfio_container_vmstate, container);
+ migration_add_notifier_mode(&container->cpr_exec_notifier,
+ vfio_cpr_fail_notifier,
+ MIG_MODE_CPR_EXEC);
return true;
}
@@ -115,4 +158,5 @@ void vfio_legacy_cpr_unregister_container(VFIOContainerBase *bcontainer)
VFIOContainer *container = VFIO_CONTAINER(bcontainer);
vmstate_unregister(NULL, &vfio_container_vmstate, container);
+ migration_remove_notifier(&container->cpr_exec_notifier);
}
@@ -81,6 +81,8 @@ typedef struct VFIOContainer {
VFIOContainerBase bcontainer;
int fd; /* /dev/vfio/vfio, empowered by the attached groups */
unsigned iommu_type;
+ NotifierWithReturn cpr_exec_notifier;
+ bool vaddr_unmapped;
QLIST_HEAD(, VFIOGroup) group_list;
} VFIOContainer;
@@ -292,7 +294,7 @@ int vfio_get_dirty_bitmap(const VFIOContainerBase *bcontainer, uint64_t iova,
uint64_t size, ram_addr_t ram_addr, Error **errp);
void vfio_container_region_add(VFIOContainerBase *bcontainer,
- MemoryRegionSection *section);
+ MemoryRegionSection *section, bool remap);
void vfio_listener_register(VFIOContainerBase *bcontainer);
/* Returns 0 on success, or a negative errno. */
@@ -37,6 +37,7 @@ typedef struct VFIOContainerBase {
Object parent;
VFIOAddressSpace *space;
MemoryListener listener;
+ MemoryListener remap_listener;
Error *error;
bool initialized;
bool reused;
If there are multiple containers and unmap-all fails for some container, we need to remap vaddr for the other containers for which unmap-all succeeded. Recover by walking all address ranges of all containers to restore the vaddr for each. Do so by invoking the vfio listener callback, and passing a new "remap" flag that tells it to restore a mapping without re-allocating new userland data structures. Signed-off-by: Steve Sistare <steven.sistare@oracle.com> --- hw/vfio/common.c | 45 ++++++++++++++++++++++++++++++++--- hw/vfio/cpr-legacy.c | 44 ++++++++++++++++++++++++++++++++++ include/hw/vfio/vfio-common.h | 4 +++- include/hw/vfio/vfio-container-base.h | 1 + 4 files changed, 90 insertions(+), 4 deletions(-)