@@ -1305,9 +1305,19 @@ static const struct pci_device_id hisi_acc_vfio_pci_table[] = {
MODULE_DEVICE_TABLE(pci, hisi_acc_vfio_pci_table);
+static pci_ers_result_t hisi_acc_aer_err_detected(struct pci_dev *pdev,
+ pci_channel_state_t state)
+{
+ struct hisi_acc_vf_core_device *hisi_acc_vdev =
+ dev_get_drvdata(&pdev->dev);
+
+ return vfio_pci_core_aer_err_detected(&hisi_acc_vdev->core_device,
+ state);
+}
+
static const struct pci_error_handlers hisi_acc_vf_err_handlers = {
.reset_done = hisi_acc_vf_pci_aer_reset_done,
- .error_detected = vfio_pci_core_aer_err_detected,
+ .error_detected = hisi_acc_aer_err_detected,
};
static struct pci_driver hisi_acc_vfio_pci_driver = {
@@ -643,9 +643,17 @@ static const struct pci_device_id mlx5vf_pci_table[] = {
MODULE_DEVICE_TABLE(pci, mlx5vf_pci_table);
+static pci_ers_result_t mlx5vf_pci_aer_err_detected(struct pci_dev *pdev,
+ pci_channel_state_t state)
+{
+ struct mlx5vf_pci_core_device *mvdev = dev_get_drvdata(&pdev->dev);
+
+ return vfio_pci_core_aer_err_detected(&mvdev->core_device, state);
+}
+
static const struct pci_error_handlers mlx5vf_err_handlers = {
.reset_done = mlx5vf_pci_aer_reset_done,
- .error_detected = vfio_pci_core_aer_err_detected,
+ .error_detected = mlx5vf_pci_aer_err_detected,
};
static struct pci_driver mlx5vf_pci_driver = {
@@ -174,10 +174,12 @@ static void vfio_pci_remove(struct pci_dev *pdev)
static int vfio_pci_sriov_configure(struct pci_dev *pdev, int nr_virtfn)
{
+ struct vfio_pci_core_device *vdev = dev_get_drvdata(&pdev->dev);
+
if (!enable_sriov)
return -ENOENT;
- return vfio_pci_core_sriov_configure(pdev, nr_virtfn);
+ return vfio_pci_core_sriov_configure(vdev, nr_virtfn);
}
static const struct pci_device_id vfio_pci_table[] = {
@@ -187,13 +189,25 @@ static const struct pci_device_id vfio_pci_table[] = {
MODULE_DEVICE_TABLE(pci, vfio_pci_table);
+static pci_ers_result_t vfio_pci_aer_err_detected(struct pci_dev *pdev,
+ pci_channel_state_t state)
+{
+ struct vfio_pci_core_device *vdev = dev_get_drvdata(&pdev->dev);
+
+ return vfio_pci_core_aer_err_detected(vdev, state);
+}
+
+static const struct pci_error_handlers vfio_pci_err_handlers = {
+ .error_detected = vfio_pci_aer_err_detected,
+};
+
static struct pci_driver vfio_pci_driver = {
.name = "vfio-pci",
.id_table = vfio_pci_table,
.probe = vfio_pci_probe,
.remove = vfio_pci_remove,
.sriov_configure = vfio_pci_sriov_configure,
- .err_handler = &vfio_pci_core_err_handlers,
+ .err_handler = &vfio_pci_err_handlers,
};
static void __init vfio_pci_fill_ids(void)
@@ -1890,9 +1890,7 @@ EXPORT_SYMBOL_GPL(vfio_pci_core_register_device);
void vfio_pci_core_unregister_device(struct vfio_pci_core_device *vdev)
{
- struct pci_dev *pdev = vdev->pdev;
-
- vfio_pci_core_sriov_configure(pdev, 0);
+ vfio_pci_core_sriov_configure(vdev, 0);
vfio_unregister_group_dev(&vdev->vdev);
@@ -1904,18 +1902,10 @@ void vfio_pci_core_unregister_device(struct vfio_pci_core_device *vdev)
}
EXPORT_SYMBOL_GPL(vfio_pci_core_unregister_device);
-pci_ers_result_t vfio_pci_core_aer_err_detected(struct pci_dev *pdev,
- pci_channel_state_t state)
+pci_ers_result_t
+vfio_pci_core_aer_err_detected(struct vfio_pci_core_device *vdev,
+ pci_channel_state_t state)
{
- struct vfio_pci_core_device *vdev;
- struct vfio_device *device;
-
- device = vfio_device_get_from_dev(&pdev->dev);
- if (device == NULL)
- return PCI_ERS_RESULT_DISCONNECT;
-
- vdev = container_of(device, struct vfio_pci_core_device, vdev);
-
mutex_lock(&vdev->igate);
if (vdev->err_trigger)
@@ -1923,26 +1913,18 @@ pci_ers_result_t vfio_pci_core_aer_err_detected(struct pci_dev *pdev,
mutex_unlock(&vdev->igate);
- vfio_device_put(device);
-
return PCI_ERS_RESULT_CAN_RECOVER;
}
EXPORT_SYMBOL_GPL(vfio_pci_core_aer_err_detected);
-int vfio_pci_core_sriov_configure(struct pci_dev *pdev, int nr_virtfn)
+int vfio_pci_core_sriov_configure(struct vfio_pci_core_device *vdev,
+ int nr_virtfn)
{
- struct vfio_pci_core_device *vdev;
- struct vfio_device *device;
+ struct pci_dev *pdev = vdev->pdev;
int ret = 0;
device_lock_assert(&pdev->dev);
- device = vfio_device_get_from_dev(&pdev->dev);
- if (!device)
- return -ENODEV;
-
- vdev = container_of(device, struct vfio_pci_core_device, vdev);
-
if (nr_virtfn) {
mutex_lock(&vfio_pci_sriov_pfs_mutex);
/*
@@ -1960,8 +1942,7 @@ int vfio_pci_core_sriov_configure(struct pci_dev *pdev, int nr_virtfn)
ret = pci_enable_sriov(pdev, nr_virtfn);
if (ret)
goto out_del;
- ret = nr_virtfn;
- goto out_put;
+ return ret;
}
pci_disable_sriov(pdev);
@@ -1971,17 +1952,10 @@ int vfio_pci_core_sriov_configure(struct pci_dev *pdev, int nr_virtfn)
list_del_init(&vdev->sriov_pfs_item);
out_unlock:
mutex_unlock(&vfio_pci_sriov_pfs_mutex);
-out_put:
- vfio_device_put(device);
return ret;
}
EXPORT_SYMBOL_GPL(vfio_pci_core_sriov_configure);
-const struct pci_error_handlers vfio_pci_core_err_handlers = {
- .error_detected = vfio_pci_core_aer_err_detected,
-};
-EXPORT_SYMBOL_GPL(vfio_pci_core_err_handlers);
-
static bool vfio_dev_in_groups(struct vfio_pci_core_device *vdev,
struct vfio_pci_group_info *groups)
{
@@ -492,12 +492,11 @@ static struct vfio_group *vfio_group_get_from_dev(struct device *dev)
* Device objects - create, release, get, put, search
*/
/* Device reference always implies a group reference */
-void vfio_device_put(struct vfio_device *device)
+static void vfio_device_put(struct vfio_device *device)
{
if (refcount_dec_and_test(&device->refcount))
complete(&device->comp);
}
-EXPORT_SYMBOL_GPL(vfio_device_put);
static bool vfio_device_try_get(struct vfio_device *device)
{
@@ -831,29 +830,6 @@ int vfio_register_emulated_iommu_dev(struct vfio_device *device)
}
EXPORT_SYMBOL_GPL(vfio_register_emulated_iommu_dev);
-/*
- * Get a reference to the vfio_device for a device. Even if the
- * caller thinks they own the device, they could be racing with a
- * release call path, so we can't trust drvdata for the shortcut.
- * Go the long way around, from the iommu_group to the vfio_group
- * to the vfio_device.
- */
-struct vfio_device *vfio_device_get_from_dev(struct device *dev)
-{
- struct vfio_group *group;
- struct vfio_device *device;
-
- group = vfio_group_get_from_dev(dev);
- if (!group)
- return NULL;
-
- device = vfio_group_get_device(group, dev);
- vfio_group_put(group);
-
- return device;
-}
-EXPORT_SYMBOL_GPL(vfio_device_get_from_dev);
-
static struct vfio_device *vfio_device_get_from_name(struct vfio_group *group,
char *buf)
{
@@ -125,8 +125,6 @@ void vfio_uninit_group_dev(struct vfio_device *device);
int vfio_register_group_dev(struct vfio_device *device);
int vfio_register_emulated_iommu_dev(struct vfio_device *device);
void vfio_unregister_group_dev(struct vfio_device *device);
-extern struct vfio_device *vfio_device_get_from_dev(struct device *dev);
-extern void vfio_device_put(struct vfio_device *device);
int vfio_assign_device_set(struct vfio_device *device, void *set_id);
@@ -227,8 +227,8 @@ void vfio_pci_core_init_device(struct vfio_pci_core_device *vdev,
int vfio_pci_core_register_device(struct vfio_pci_core_device *vdev);
void vfio_pci_core_uninit_device(struct vfio_pci_core_device *vdev);
void vfio_pci_core_unregister_device(struct vfio_pci_core_device *vdev);
-int vfio_pci_core_sriov_configure(struct pci_dev *pdev, int nr_virtfn);
-extern const struct pci_error_handlers vfio_pci_core_err_handlers;
+int vfio_pci_core_sriov_configure(struct vfio_pci_core_device *vdev,
+ int nr_virtfn);
long vfio_pci_core_ioctl(struct vfio_device *core_vdev, unsigned int cmd,
unsigned long arg);
int vfio_pci_core_ioctl_feature(struct vfio_device *device, u32 flags,
@@ -243,8 +243,9 @@ int vfio_pci_core_match(struct vfio_device *core_vdev, char *buf);
int vfio_pci_core_enable(struct vfio_pci_core_device *vdev);
void vfio_pci_core_disable(struct vfio_pci_core_device *vdev);
void vfio_pci_core_finish_enable(struct vfio_pci_core_device *vdev);
-pci_ers_result_t vfio_pci_core_aer_err_detected(struct pci_dev *pdev,
- pci_channel_state_t state);
+pci_ers_result_t
+vfio_pci_core_aer_err_detected(struct vfio_pci_core_device *vdev,
+ pci_channel_state_t state);
static inline bool vfio_pci_is_vga(struct pci_dev *pdev)
{
The last user of this function is in PCI callbacks that want to convert their struct pci_dev to a vfio_device. Instead of searching use the vfio_device available trivially through the drvdata. When a callback in the device_driver is called, the caller must hold the device_lock() on dev. The purpose of the device_lock is to prevent remove() from being called (see __device_release_driver), and allow the driver to safely interact with its drvdata without races. The PCI core correctly follows this and holds the device_lock() when calling error_detected (see report_error_detected) and sriov_configure (see sriov_numvfs_store). Further, since the drvdata holds a positive refcount on the vfio_device any access of the drvdata, under the driver_lock, from a driver callback needs no further protection or refcounting. Thus the remark in the vfio_device_get_from_dev() comment does not apply here, VFIO PCI drivers all call vfio_unregister_group_dev() from their remove callbacks under the driver lock and cannot race with the remaining callers. Signed-off-by: Jason Gunthorpe <jgg@nvidia.com> --- .../vfio/pci/hisilicon/hisi_acc_vfio_pci.c | 12 +++++- drivers/vfio/pci/mlx5/main.c | 10 ++++- drivers/vfio/pci/vfio_pci.c | 18 +++++++- drivers/vfio/pci/vfio_pci_core.c | 42 ++++--------------- drivers/vfio/vfio.c | 26 +----------- include/linux/vfio.h | 2 - include/linux/vfio_pci_core.h | 9 ++-- 7 files changed, 50 insertions(+), 69 deletions(-) This depends on the gvt rework from Christoph. base-commit: 1d0b96c15ee71167d8020446b608a0ed6a70d4b3