@@ -90,6 +90,7 @@ struct vfio_dma {
bool iommu_mapped;
bool lock_cap; /* capable(CAP_IPC_LOCK) */
struct task_struct *task;
+ struct mutex pfn_list_lock;
struct rb_root pfn_list; /* Ex-user pinned pfn list */
};
@@ -539,7 +540,7 @@ static int vfio_iommu_type1_pin_pages(void *iommu_data,
if (!iommu->v2)
return -EACCES;
- down_write(&iommu->lock);
+ down_read(&iommu->lock);
/* Fail if notifier list is empty */
if (!iommu->notifier.head) {
@@ -570,8 +571,11 @@ static int vfio_iommu_type1_pin_pages(void *iommu_data,
goto pin_unwind;
}
+ mutex_lock(&dma->pfn_list_lock);
+
vpfn = vfio_iova_get_vfio_pfn(dma, iova);
if (vpfn) {
+ mutex_unlock(&dma->pfn_list_lock);
phys_pfn[i] = vpfn->pfn;
continue;
}
@@ -579,14 +583,19 @@ static int vfio_iommu_type1_pin_pages(void *iommu_data,
remote_vaddr = dma->vaddr + iova - dma->iova;
ret = vfio_pin_page_external(dma, remote_vaddr, &phys_pfn[i],
do_accounting);
- if (ret)
+ if (ret) {
+ mutex_unlock(&dma->pfn_list_lock);
goto pin_unwind;
+ }
ret = vfio_add_to_pfn_list(dma, iova, phys_pfn[i]);
if (ret) {
vfio_unpin_page_external(dma, iova, do_accounting);
+ mutex_unlock(&dma->pfn_list_lock);
goto pin_unwind;
}
+
+ mutex_unlock(&dma->pfn_list_lock);
}
ret = i;
@@ -599,11 +608,13 @@ static int vfio_iommu_type1_pin_pages(void *iommu_data,
iova = user_pfn[j] << PAGE_SHIFT;
dma = vfio_find_dma(iommu, iova, PAGE_SIZE);
+ mutex_lock(&dma->pfn_list_lock);
vfio_unpin_page_external(dma, iova, do_accounting);
+ mutex_unlock(&dma->pfn_list_lock);
phys_pfn[j] = 0;
}
pin_done:
- up_write(&iommu->lock);
+ up_read(&iommu->lock);
return ret;
}
@@ -622,7 +633,7 @@ static int vfio_iommu_type1_unpin_pages(void *iommu_data,
if (!iommu->v2)
return -EACCES;
- down_write(&iommu->lock);
+ down_read(&iommu->lock);
do_accounting = !IS_IOMMU_CAP_DOMAIN_IN_CONTAINER(iommu);
for (i = 0; i < npage; i++) {
@@ -633,11 +644,13 @@ static int vfio_iommu_type1_unpin_pages(void *iommu_data,
dma = vfio_find_dma(iommu, iova, PAGE_SIZE);
if (!dma)
goto unpin_exit;
+ mutex_lock(&dma->pfn_list_lock);
vfio_unpin_page_external(dma, iova, do_accounting);
+ mutex_unlock(&dma->pfn_list_lock);
}
unpin_exit:
- up_write(&iommu->lock);
+ up_read(&iommu->lock);
return i > npage ? npage : (i > 0 ? i : -EINVAL);
}
@@ -1109,6 +1122,7 @@ static int vfio_dma_do_map(struct vfio_iommu *iommu,
dma->iova = iova;
dma->vaddr = vaddr;
dma->prot = prot;
+ mutex_init(&dma->pfn_list_lock);
/*
* We need to be able to both add to a task's locked memory and test
We can promote external page {un}pinning to a reader lock, allowing concurrency since these don't change the vfio_iommu state. We do need to protect the vpfn list per vfio_dma in place of that serialization though. Signed-off-by: Alex Williamson <alex.williamson@redhat.com> --- drivers/vfio/vfio_iommu_type1.c | 24 +++++++++++++++++++----- 1 file changed, 19 insertions(+), 5 deletions(-)