@@ -113,7 +113,7 @@ static void vfio_pci_probe_mmaps(struct vfio_pci_device *vdev)
int bar;
struct vfio_pci_dummy_resource *dummy_res;
- INIT_LIST_HEAD(&vdev->dummy_resources_list);
+ INIT_LIST_HEAD(&vdev->priv->dummy_resources_list);
for (bar = PCI_STD_RESOURCES; bar <= PCI_STD_RESOURCE_END; bar++) {
res = vdev->pdev->resource + bar;
@@ -133,7 +133,7 @@ static void vfio_pci_probe_mmaps(struct vfio_pci_device *vdev)
goto no_mmap;
if (resource_size(res) >= PAGE_SIZE) {
- vdev->bar_mmap_supported[bar] = true;
+ vdev->priv->bar_mmap_supported[bar] = true;
continue;
}
@@ -158,8 +158,8 @@ static void vfio_pci_probe_mmaps(struct vfio_pci_device *vdev)
}
dummy_res->index = bar;
list_add(&dummy_res->res_next,
- &vdev->dummy_resources_list);
- vdev->bar_mmap_supported[bar] = true;
+ &vdev->priv->dummy_resources_list);
+ vdev->priv->bar_mmap_supported[bar] = true;
continue;
}
/*
@@ -171,7 +171,7 @@ static void vfio_pci_probe_mmaps(struct vfio_pci_device *vdev)
* the BAR's location in a page.
*/
no_mmap:
- vdev->bar_mmap_supported[bar] = false;
+ vdev->priv->bar_mmap_supported[bar] = false;
}
}
@@ -217,7 +217,7 @@ static void vfio_pci_probe_power_state(struct vfio_pci_device *vdev)
pci_read_config_word(pdev, pdev->pm_cap + PCI_PM_CTRL, &pmcsr);
- vdev->needs_pm_restore = !(pmcsr & PCI_PM_CTRL_NO_SOFT_RESET);
+ vdev->priv->needs_pm_restore = !(pmcsr & PCI_PM_CTRL_NO_SOFT_RESET);
}
/*
@@ -233,7 +233,7 @@ int vfio_pci_set_power_state(struct vfio_pci_device *vdev, pci_power_t state)
bool needs_restore = false, needs_save = false;
int ret;
- if (vdev->needs_pm_restore) {
+ if (vdev->priv->needs_pm_restore) {
if (pdev->current_state < PCI_D3hot && state >= PCI_D3hot) {
pci_save_state(pdev);
needs_save = true;
@@ -248,9 +248,10 @@ int vfio_pci_set_power_state(struct vfio_pci_device *vdev, pci_power_t state)
if (!ret) {
/* D3 might be unsupported via quirk, skip unless in D3 */
if (needs_save && pdev->current_state >= PCI_D3hot) {
- vdev->pm_save = pci_store_saved_state(pdev);
+ vdev->priv->pm_save = pci_store_saved_state(pdev);
} else if (needs_restore) {
- pci_load_and_free_saved_state(pdev, &vdev->pm_save);
+ pci_load_and_free_saved_state(pdev,
+ &vdev->priv->pm_save);
pci_restore_state(pdev);
}
}
@@ -281,31 +282,31 @@ static int vfio_pci_enable(struct vfio_pci_device *vdev)
return ret;
}
- vdev->reset_works = !ret;
+ vdev->priv->reset_works = !ret;
pci_save_state(pdev);
- vdev->pci_saved_state = pci_store_saved_state(pdev);
- if (!vdev->pci_saved_state)
+ vdev->priv->pci_saved_state = pci_store_saved_state(pdev);
+ if (!vdev->priv->pci_saved_state)
pci_dbg(pdev, "%s: Couldn't store saved state\n", __func__);
if (likely(!nointxmask)) {
if (vfio_pci_nointx(pdev)) {
pci_info(pdev, "Masking broken INTx support\n");
- vdev->nointx = true;
+ vdev->priv->nointx = true;
pci_intx(pdev, 0);
} else
- vdev->pci_2_3 = pci_intx_mask_supported(pdev);
+ vdev->priv->pci_2_3 = pci_intx_mask_supported(pdev);
}
pci_read_config_word(pdev, PCI_COMMAND, &cmd);
- if (vdev->pci_2_3 && (cmd & PCI_COMMAND_INTX_DISABLE)) {
+ if (vdev->priv->pci_2_3 && (cmd & PCI_COMMAND_INTX_DISABLE)) {
cmd &= ~PCI_COMMAND_INTX_DISABLE;
pci_write_config_word(pdev, PCI_COMMAND, cmd);
}
ret = vfio_config_init(vdev);
if (ret) {
- kfree(vdev->pci_saved_state);
- vdev->pci_saved_state = NULL;
+ kfree(vdev->priv->pci_saved_state);
+ vdev->priv->pci_saved_state = NULL;
pci_disable_device(pdev);
return ret;
}
@@ -318,14 +319,15 @@ static int vfio_pci_enable(struct vfio_pci_device *vdev)
pci_read_config_word(pdev, msix_pos + PCI_MSIX_FLAGS, &flags);
pci_read_config_dword(pdev, msix_pos + PCI_MSIX_TABLE, &table);
- vdev->msix_bar = table & PCI_MSIX_TABLE_BIR;
- vdev->msix_offset = table & PCI_MSIX_TABLE_OFFSET;
- vdev->msix_size = ((flags & PCI_MSIX_FLAGS_QSIZE) + 1) * 16;
+ vdev->priv->msix_bar = table & PCI_MSIX_TABLE_BIR;
+ vdev->priv->msix_offset = table & PCI_MSIX_TABLE_OFFSET;
+ vdev->priv->msix_size =
+ ((flags & PCI_MSIX_FLAGS_QSIZE) + 1) * 16;
} else
- vdev->msix_bar = 0xFF;
+ vdev->priv->msix_bar = 0xFF;
if (!vfio_vga_disabled() && vfio_pci_is_vga(pdev))
- vdev->has_vga = true;
+ vdev->priv->has_vga = true;
if (vfio_pci_is_vga(pdev) &&
@@ -381,40 +383,41 @@ static void vfio_pci_disable(struct vfio_pci_device *vdev)
/* Device closed, don't need mutex here */
list_for_each_entry_safe(ioeventfd, ioeventfd_tmp,
- &vdev->ioeventfds_list, next) {
+ &vdev->priv->ioeventfds_list, next) {
vfio_virqfd_disable(&ioeventfd->virqfd);
list_del(&ioeventfd->next);
kfree(ioeventfd);
}
- vdev->ioeventfds_nr = 0;
+ vdev->priv->ioeventfds_nr = 0;
- vdev->virq_disabled = false;
+ vdev->priv->virq_disabled = false;
for (i = 0; i < vdev->num_regions; i++)
- vdev->region[i].ops->release(vdev, &vdev->region[i]);
+ vdev->priv->region[i].ops->release(vdev,
+ &vdev->priv->region[i]);
vdev->num_regions = 0;
- kfree(vdev->region);
- vdev->region = NULL; /* don't krealloc a freed pointer */
+ kfree(vdev->priv->region);
+ vdev->priv->region = NULL; /* don't krealloc a freed pointer */
vfio_config_free(vdev);
for (bar = PCI_STD_RESOURCES; bar <= PCI_STD_RESOURCE_END; bar++) {
- if (!vdev->barmap[bar])
+ if (!vdev->priv->barmap[bar])
continue;
- pci_iounmap(pdev, vdev->barmap[bar]);
+ pci_iounmap(pdev, vdev->priv->barmap[bar]);
pci_release_selected_regions(pdev, 1 << bar);
- vdev->barmap[bar] = NULL;
+ vdev->priv->barmap[bar] = NULL;
}
list_for_each_entry_safe(dummy_res, tmp,
- &vdev->dummy_resources_list, res_next) {
+ &vdev->priv->dummy_resources_list, res_next) {
list_del(&dummy_res->res_next);
release_resource(&dummy_res->resource);
kfree(dummy_res);
}
- vdev->needs_reset = true;
+ vdev->priv->needs_reset = true;
/*
* If we have saved state, restore it. If we can reset the device,
@@ -422,10 +425,10 @@ static void vfio_pci_disable(struct vfio_pci_device *vdev)
* nothing, but saving and restoring current state without reset
* is just busy work.
*/
- if (pci_load_and_free_saved_state(pdev, &vdev->pci_saved_state)) {
+ if (pci_load_and_free_saved_state(pdev, &vdev->priv->pci_saved_state)) {
pci_info(pdev, "%s: Couldn't reload saved state\n", __func__);
- if (!vdev->reset_works)
+ if (!vdev->priv->reset_works)
goto out;
pci_save_state(pdev);
@@ -444,10 +447,10 @@ static void vfio_pci_disable(struct vfio_pci_device *vdev)
* We can not use the "try" reset interface here, which will
* overwrite the previously restored configuration information.
*/
- if (vdev->reset_works && pci_cfg_access_trylock(pdev)) {
+ if (vdev->priv->reset_works && pci_cfg_access_trylock(pdev)) {
if (device_trylock(&pdev->dev)) {
if (!__pci_reset_function_locked(pdev))
- vdev->needs_reset = false;
+ vdev->priv->needs_reset = false;
device_unlock(&pdev->dev);
}
pci_cfg_access_unlock(pdev);
@@ -467,14 +470,14 @@ static void vfio_pci_release(void *device_data)
{
struct vfio_pci_device *vdev = device_data;
- mutex_lock(&vdev->reflck->lock);
+ mutex_lock(&vdev->priv->reflck->lock);
- if (!(--vdev->refcnt)) {
+ if (!(--vdev->priv->refcnt)) {
vfio_spapr_pci_eeh_release(vdev->pdev);
vfio_pci_disable(vdev);
}
- mutex_unlock(&vdev->reflck->lock);
+ mutex_unlock(&vdev->priv->reflck->lock);
module_put(THIS_MODULE);
}
@@ -487,18 +490,18 @@ static int vfio_pci_open(void *device_data)
if (!try_module_get(THIS_MODULE))
return -ENODEV;
- mutex_lock(&vdev->reflck->lock);
+ mutex_lock(&vdev->priv->reflck->lock);
- if (!vdev->refcnt) {
+ if (!vdev->priv->refcnt) {
ret = vfio_pci_enable(vdev);
if (ret)
goto error;
vfio_spapr_pci_eeh_open(vdev->pdev);
}
- vdev->refcnt++;
+ vdev->priv->refcnt++;
error:
- mutex_unlock(&vdev->reflck->lock);
+ mutex_unlock(&vdev->priv->reflck->lock);
if (ret)
module_put(THIS_MODULE);
return ret;
@@ -510,7 +513,7 @@ static int vfio_pci_get_irq_count(struct vfio_pci_device *vdev, int irq_type)
u8 pin;
if (!IS_ENABLED(CONFIG_VFIO_PCI_INTX) ||
- vdev->nointx || vdev->pdev->is_virtfn)
+ vdev->priv->nointx || vdev->pdev->is_virtfn)
return 0;
pci_read_config_byte(vdev->pdev, PCI_INTERRUPT_PIN, &pin);
@@ -669,19 +672,19 @@ int vfio_pci_register_dev_region(struct vfio_pci_device *vdev,
{
struct vfio_pci_region *region;
- region = krealloc(vdev->region,
+ region = krealloc(vdev->priv->region,
(vdev->num_regions + 1) * sizeof(*region),
GFP_KERNEL);
if (!region)
return -ENOMEM;
- vdev->region = region;
- vdev->region[vdev->num_regions].type = type;
- vdev->region[vdev->num_regions].subtype = subtype;
- vdev->region[vdev->num_regions].ops = ops;
- vdev->region[vdev->num_regions].size = size;
- vdev->region[vdev->num_regions].flags = flags;
- vdev->region[vdev->num_regions].data = data;
+ vdev->priv->region = region;
+ vdev->priv->region[vdev->num_regions].type = type;
+ vdev->priv->region[vdev->num_regions].subtype = subtype;
+ vdev->priv->region[vdev->num_regions].ops = ops;
+ vdev->priv->region[vdev->num_regions].size = size;
+ vdev->priv->region[vdev->num_regions].flags = flags;
+ vdev->priv->region[vdev->num_regions].data = data;
vdev->num_regions++;
@@ -707,7 +710,7 @@ static long vfio_pci_ioctl(void *device_data,
info.flags = VFIO_DEVICE_FLAGS_PCI;
- if (vdev->reset_works)
+ if (vdev->priv->reset_works)
info.flags |= VFIO_DEVICE_FLAGS_RESET;
info.num_regions = VFIO_PCI_NUM_REGIONS + vdev->num_regions;
@@ -747,9 +750,9 @@ static long vfio_pci_ioctl(void *device_data,
info.flags = VFIO_REGION_INFO_FLAG_READ |
VFIO_REGION_INFO_FLAG_WRITE;
- if (vdev->bar_mmap_supported[info.index]) {
+ if (vdev->priv->bar_mmap_supported[info.index]) {
info.flags |= VFIO_REGION_INFO_FLAG_MMAP;
- if (info.index == vdev->msix_bar) {
+ if (info.index == vdev->priv->msix_bar) {
ret = msix_mmappable_cap(vdev, &caps);
if (ret)
return ret;
@@ -797,7 +800,7 @@ static long vfio_pci_ioctl(void *device_data,
break;
}
case VFIO_PCI_VGA_REGION_INDEX:
- if (!vdev->has_vga)
+ if (!vdev->priv->has_vga)
return -EINVAL;
info.offset = VFIO_PCI_INDEX_TO_OFFSET(info.index);
@@ -811,6 +814,7 @@ static long vfio_pci_ioctl(void *device_data,
struct vfio_region_info_cap_type cap_type = {
.header.id = VFIO_REGION_INFO_CAP_TYPE,
.header.version = 1 };
+ struct vfio_pci_region *region;
if (info.index >=
VFIO_PCI_NUM_REGIONS + vdev->num_regions)
@@ -821,21 +825,22 @@ static long vfio_pci_ioctl(void *device_data,
i = info.index - VFIO_PCI_NUM_REGIONS;
+ region = vdev->priv->region;
info.offset = VFIO_PCI_INDEX_TO_OFFSET(info.index);
- info.size = vdev->region[i].size;
- info.flags = vdev->region[i].flags;
+ info.size = region[i].size;
+ info.flags = region[i].flags;
- cap_type.type = vdev->region[i].type;
- cap_type.subtype = vdev->region[i].subtype;
+ cap_type.type = region[i].type;
+ cap_type.subtype = region[i].subtype;
ret = vfio_info_add_capability(&caps, &cap_type.header,
sizeof(cap_type));
if (ret)
return ret;
- if (vdev->region[i].ops->add_capability) {
- ret = vdev->region[i].ops->add_capability(vdev,
- &vdev->region[i], &caps);
+ if (region[i].ops->add_capability) {
+ ret = region[i].ops->add_capability(vdev,
+ ®ion[i], &caps);
if (ret)
return ret;
}
@@ -925,18 +930,18 @@ static long vfio_pci_ioctl(void *device_data,
return PTR_ERR(data);
}
- mutex_lock(&vdev->igate);
+ mutex_lock(&vdev->priv->igate);
ret = vfio_pci_set_irqs_ioctl(vdev, hdr.flags, hdr.index,
hdr.start, hdr.count, data);
- mutex_unlock(&vdev->igate);
+ mutex_unlock(&vdev->priv->igate);
kfree(data);
return ret;
} else if (cmd == VFIO_DEVICE_RESET) {
- return vdev->reset_works ?
+ return vdev->priv->reset_works ?
pci_try_reset_function(vdev->pdev) : -EINVAL;
} else if (cmd == VFIO_DEVICE_GET_PCI_HOT_RESET_INFO) {
@@ -1167,7 +1172,7 @@ static ssize_t vfio_pci_rw(void *device_data, char __user *buf,
return vfio_pci_vga_rw(vdev, buf, count, ppos, iswrite);
default:
index -= VFIO_PCI_NUM_REGIONS;
- return vdev->region[index].ops->rw(vdev, buf,
+ return vdev->priv->region[index].ops->rw(vdev, buf,
count, ppos, iswrite);
}
@@ -1208,7 +1213,7 @@ static int vfio_pci_mmap(void *device_data, struct vm_area_struct *vma)
return -EINVAL;
if (index >= VFIO_PCI_NUM_REGIONS) {
int regnum = index - VFIO_PCI_NUM_REGIONS;
- struct vfio_pci_region *region = vdev->region + regnum;
+ struct vfio_pci_region *region = vdev->priv->region + regnum;
if (region && region->ops && region->ops->mmap &&
(region->flags & VFIO_REGION_INFO_FLAG_MMAP))
@@ -1217,7 +1222,7 @@ static int vfio_pci_mmap(void *device_data, struct vm_area_struct *vma)
}
if (index >= VFIO_PCI_ROM_REGION_INDEX)
return -EINVAL;
- if (!vdev->bar_mmap_supported[index])
+ if (!vdev->priv->bar_mmap_supported[index])
return -EINVAL;
phys_len = PAGE_ALIGN(pci_resource_len(pdev, index));
@@ -1233,14 +1238,14 @@ static int vfio_pci_mmap(void *device_data, struct vm_area_struct *vma)
* Even though we don't make use of the barmap for the mmap,
* we need to request the region and the barmap tracks that.
*/
- if (!vdev->barmap[index]) {
+ if (!vdev->priv->barmap[index]) {
ret = pci_request_selected_regions(pdev,
1 << index, "vfio-pci");
if (ret)
return ret;
- vdev->barmap[index] = pci_iomap(pdev, index, 0);
- if (!vdev->barmap[index]) {
+ vdev->priv->barmap[index] = pci_iomap(pdev, index, 0);
+ if (!vdev->priv->barmap[index]) {
pci_release_selected_regions(pdev, 1 << index);
return -ENOMEM;
}
@@ -1259,20 +1264,20 @@ static void vfio_pci_request(void *device_data, unsigned int count)
struct vfio_pci_device *vdev = device_data;
struct pci_dev *pdev = vdev->pdev;
- mutex_lock(&vdev->igate);
+ mutex_lock(&vdev->priv->igate);
- if (vdev->req_trigger) {
+ if (vdev->priv->req_trigger) {
if (!(count % 10))
pci_notice_ratelimited(pdev,
"Relaying device request to user (#%u)\n",
count);
- eventfd_signal(vdev->req_trigger, 1);
+ eventfd_signal(vdev->priv->req_trigger, 1);
} else if (count == 0) {
pci_warn(pdev,
"No device request channel registered, blocked until released by user\n");
}
- mutex_unlock(&vdev->igate);
+ mutex_unlock(&vdev->priv->igate);
}
static const struct vfio_device_ops vfio_pci_ops = {
@@ -1321,12 +1326,18 @@ static int vfio_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id)
return -ENOMEM;
}
+ vdev->priv = kzalloc(sizeof(*vdev->priv), GFP_KERNEL);
+ if (!vdev->priv) {
+ vfio_iommu_group_put(group, &pdev->dev);
+ return -ENOMEM;
+ }
+
vdev->pdev = pdev;
vdev->irq_type = VFIO_PCI_NUM_IRQS;
- mutex_init(&vdev->igate);
- spin_lock_init(&vdev->irqlock);
- mutex_init(&vdev->ioeventfds_lock);
- INIT_LIST_HEAD(&vdev->ioeventfds_list);
+ mutex_init(&vdev->priv->igate);
+ spin_lock_init(&vdev->priv->irqlock);
+ mutex_init(&vdev->priv->ioeventfds_lock);
+ INIT_LIST_HEAD(&vdev->priv->ioeventfds_list);
ret = vfio_add_group_dev(&pdev->dev, &vfio_pci_ops, vdev);
if (ret) {
@@ -1376,16 +1387,16 @@ static void vfio_pci_remove(struct pci_dev *pdev)
if (!vdev)
return;
- vfio_pci_reflck_put(vdev->reflck);
+ vfio_pci_reflck_put(vdev->priv->reflck);
vfio_iommu_group_put(pdev->dev.iommu_group, &pdev->dev);
- kfree(vdev->region);
- mutex_destroy(&vdev->ioeventfds_lock);
+ kfree(vdev->priv->region);
+ mutex_destroy(&vdev->priv->ioeventfds_lock);
if (!disable_idle_d3)
vfio_pci_set_power_state(vdev, PCI_D0);
- kfree(vdev->pm_save);
+ kfree(vdev->priv->pm_save);
kfree(vdev);
if (vfio_pci_is_vga(pdev)) {
@@ -1412,12 +1423,12 @@ static pci_ers_result_t vfio_pci_aer_err_detected(struct pci_dev *pdev,
return PCI_ERS_RESULT_DISCONNECT;
}
- mutex_lock(&vdev->igate);
+ mutex_lock(&vdev->priv->igate);
- if (vdev->err_trigger)
- eventfd_signal(vdev->err_trigger, 1);
+ if (vdev->priv->err_trigger)
+ eventfd_signal(vdev->priv->err_trigger, 1);
- mutex_unlock(&vdev->igate);
+ mutex_unlock(&vdev->priv->igate);
vfio_device_put(device);
@@ -1474,9 +1485,9 @@ static int vfio_pci_reflck_find(struct pci_dev *pdev, void *data)
vdev = vfio_device_data(device);
- if (vdev->reflck) {
- vfio_pci_reflck_get(vdev->reflck);
- *preflck = vdev->reflck;
+ if (vdev->priv->reflck) {
+ vfio_pci_reflck_get(vdev->priv->reflck);
+ *preflck = vdev->priv->reflck;
vfio_device_put(device);
return 1;
}
@@ -1493,12 +1504,12 @@ static int vfio_pci_reflck_attach(struct vfio_pci_device *vdev)
if (pci_is_root_bus(vdev->pdev->bus) ||
vfio_pci_for_each_slot_or_bus(vdev->pdev, vfio_pci_reflck_find,
- &vdev->reflck, slot) <= 0)
- vdev->reflck = vfio_pci_reflck_alloc();
+ &vdev->priv->reflck, slot) <= 0)
+ vdev->priv->reflck = vfio_pci_reflck_alloc();
mutex_unlock(&reflck_lock);
- return PTR_ERR_OR_ZERO(vdev->reflck);
+ return PTR_ERR_OR_ZERO(vdev->priv->reflck);
}
static void vfio_pci_reflck_release(struct kref *kref)
@@ -1543,7 +1554,7 @@ static int vfio_pci_get_unused_devs(struct pci_dev *pdev, void *data)
vdev = vfio_device_data(device);
/* Fault if the device is not unused */
- if (vdev->refcnt) {
+ if (vdev->priv->refcnt) {
vfio_device_put(device);
return -EBUSY;
}
@@ -1559,7 +1570,7 @@ static int vfio_pci_get_unused_devs(struct pci_dev *pdev, void *data)
* - At least one of the affected devices is marked dirty via
* needs_reset (such as by lack of FLR support)
* Then attempt to perform that bus or slot reset. Callers are required
- * to hold vdev->reflck->lock, protecting the bus/slot reset group from
+ * to hold vdev->priv->reflck->lock, protecting the bus/slot reset group from
* concurrent opens. A vfio_device reference is acquired for each device
* to prevent unbinds during the reset operation.
*
@@ -1597,7 +1608,7 @@ static void vfio_pci_try_bus_reset(struct vfio_pci_device *vdev)
/* Does at least one need a reset? */
for (i = 0; i < devs.cur_index; i++) {
tmp = vfio_device_data(devs.devices[i]);
- if (tmp->needs_reset) {
+ if (tmp->priv->needs_reset) {
ret = pci_reset_bus(vdev->pdev);
break;
}
@@ -1615,7 +1626,7 @@ static void vfio_pci_try_bus_reset(struct vfio_pci_device *vdev)
* the power state.
*/
if (!ret) {
- tmp->needs_reset = false;
+ tmp->priv->needs_reset = false;
if (tmp != vdev && !disable_idle_d3)
vfio_pci_set_power_state(tmp, PCI_D3hot);
@@ -177,7 +177,7 @@ static int vfio_default_config_read(struct vfio_pci_device *vdev, int pos,
{
__le32 virt = 0;
- memcpy(val, vdev->vconfig + pos, count);
+ memcpy(val, vdev->priv->vconfig + pos, count);
memcpy(&virt, perm->virt + offset, count);
@@ -214,12 +214,12 @@ static int vfio_default_config_write(struct vfio_pci_device *vdev, int pos,
if (write & virt) {
__le32 virt_val = 0;
- memcpy(&virt_val, vdev->vconfig + pos, count);
+ memcpy(&virt_val, vdev->priv->vconfig + pos, count);
virt_val &= ~(write & virt);
virt_val |= (val & (write & virt));
- memcpy(vdev->vconfig + pos, &virt_val, count);
+ memcpy(vdev->priv->vconfig + pos, &virt_val, count);
}
/* Non-virtualzed and writable bits go to hardware */
@@ -256,13 +256,13 @@ static int vfio_direct_config_read(struct vfio_pci_device *vdev, int pos,
if (pos >= PCI_CFG_SPACE_SIZE) { /* Extended cap header mangling */
if (offset < 4)
- memcpy(val, vdev->vconfig + pos, count);
+ memcpy(val, vdev->priv->vconfig + pos, count);
} else if (pos >= PCI_STD_HEADER_SIZEOF) { /* Std cap mangling */
if (offset == PCI_CAP_LIST_ID && count > 1)
- memcpy(val, vdev->vconfig + pos,
+ memcpy(val, vdev->priv->vconfig + pos,
min(PCI_CAP_FLAGS, count));
else if (offset == PCI_CAP_LIST_NEXT)
- memcpy(val, vdev->vconfig + pos, 1);
+ memcpy(val, vdev->priv->vconfig + pos, 1);
}
return count;
@@ -300,7 +300,7 @@ static int vfio_virt_config_write(struct vfio_pci_device *vdev, int pos,
int count, struct perm_bits *perm,
int offset, __le32 val)
{
- memcpy(vdev->vconfig + pos, &val, count);
+ memcpy(vdev->priv->vconfig + pos, &val, count);
return count;
}
@@ -308,7 +308,7 @@ static int vfio_virt_config_read(struct vfio_pci_device *vdev, int pos,
int count, struct perm_bits *perm,
int offset, __le32 *val)
{
- memcpy(val, vdev->vconfig + pos, count);
+ memcpy(val, vdev->priv->vconfig + pos, count);
return count;
}
@@ -402,7 +402,7 @@ static inline void p_setd(struct perm_bits *p, int off, u32 virt, u32 write)
static void vfio_bar_restore(struct vfio_pci_device *vdev)
{
struct pci_dev *pdev = vdev->pdev;
- u32 *rbar = vdev->rbar;
+ u32 *rbar = vdev->priv->rbar;
u16 cmd;
int i;
@@ -416,7 +416,7 @@ static void vfio_bar_restore(struct vfio_pci_device *vdev)
pci_user_write_config_dword(pdev, PCI_ROM_ADDRESS, *rbar);
- if (vdev->nointx) {
+ if (vdev->priv->nointx) {
pci_user_read_config_word(pdev, PCI_COMMAND, &cmd);
cmd |= PCI_COMMAND_INTX_DISABLE;
pci_user_write_config_word(pdev, PCI_COMMAND, cmd);
@@ -453,7 +453,7 @@ static void vfio_bar_fixup(struct vfio_pci_device *vdev)
__le32 *bar;
u64 mask;
- bar = (__le32 *)&vdev->vconfig[PCI_BASE_ADDRESS_0];
+ bar = (__le32 *)&vdev->priv->vconfig[PCI_BASE_ADDRESS_0];
for (i = PCI_STD_RESOURCES; i <= PCI_STD_RESOURCE_END; i++, bar++) {
if (!pci_resource_start(pdev, i)) {
@@ -473,7 +473,7 @@ static void vfio_bar_fixup(struct vfio_pci_device *vdev)
}
}
- bar = (__le32 *)&vdev->vconfig[PCI_ROM_ADDRESS];
+ bar = (__le32 *)&vdev->priv->vconfig[PCI_ROM_ADDRESS];
/*
* NB. REGION_INFO will have reported zero size if we weren't able
@@ -492,7 +492,7 @@ static void vfio_bar_fixup(struct vfio_pci_device *vdev)
} else
*bar = 0;
- vdev->bardirty = false;
+ vdev->priv->bardirty = false;
}
static int vfio_basic_config_read(struct vfio_pci_device *vdev, int pos,
@@ -506,7 +506,8 @@ static int vfio_basic_config_read(struct vfio_pci_device *vdev, int pos,
/* Mask in virtual memory enable for SR-IOV devices */
if (offset == PCI_COMMAND && vdev->pdev->is_virtfn) {
- u16 cmd = le16_to_cpu(*(__le16 *)&vdev->vconfig[PCI_COMMAND]);
+ u16 cmd = le16_to_cpu(*(__le16 *)
+ &vdev->priv->vconfig[PCI_COMMAND]);
u32 tmp_val = le32_to_cpu(*val);
tmp_val |= cmd & PCI_COMMAND_MEMORY;
@@ -523,9 +524,9 @@ static bool vfio_need_bar_restore(struct vfio_pci_device *vdev)
u32 bar;
for (; pos <= PCI_BASE_ADDRESS_5; i++, pos += 4) {
- if (vdev->rbar[i]) {
+ if (vdev->priv->rbar[i]) {
ret = pci_user_read_config_dword(vdev->pdev, pos, &bar);
- if (ret || vdev->rbar[i] != bar)
+ if (ret || vdev->priv->rbar[i] != bar)
return true;
}
}
@@ -542,7 +543,7 @@ static int vfio_basic_config_write(struct vfio_pci_device *vdev, int pos,
u16 new_cmd = 0;
int ret;
- virt_cmd = (__le16 *)&vdev->vconfig[PCI_COMMAND];
+ virt_cmd = (__le16 *)&vdev->priv->vconfig[PCI_COMMAND];
if (offset == PCI_COMMAND) {
bool phys_mem, virt_mem, new_mem, phys_io, virt_io, new_io;
@@ -598,17 +599,17 @@ static int vfio_basic_config_write(struct vfio_pci_device *vdev, int pos,
virt_intx_disable = !!(le16_to_cpu(*virt_cmd) &
PCI_COMMAND_INTX_DISABLE);
- if (virt_intx_disable && !vdev->virq_disabled) {
- vdev->virq_disabled = true;
+ if (virt_intx_disable && !vdev->priv->virq_disabled) {
+ vdev->priv->virq_disabled = true;
vfio_pci_intx_mask(vdev);
- } else if (!virt_intx_disable && vdev->virq_disabled) {
- vdev->virq_disabled = false;
+ } else if (!virt_intx_disable && vdev->priv->virq_disabled) {
+ vdev->priv->virq_disabled = false;
vfio_pci_intx_unmask(vdev);
}
}
if (is_bar(offset))
- vdev->bardirty = true;
+ vdev->priv->bardirty = true;
return count;
}
@@ -721,8 +722,10 @@ static int vfio_vpd_config_write(struct vfio_pci_device *vdev, int pos,
int offset, __le32 val)
{
struct pci_dev *pdev = vdev->pdev;
- __le16 *paddr = (__le16 *)(vdev->vconfig + pos - offset + PCI_VPD_ADDR);
- __le32 *pdata = (__le32 *)(vdev->vconfig + pos - offset + PCI_VPD_DATA);
+ __le16 *paddr = (__le16 *)(vdev->priv->vconfig + pos - offset +
+ PCI_VPD_ADDR);
+ __le32 *pdata = (__le32 *)(vdev->priv->vconfig + pos - offset +
+ PCI_VPD_DATA);
u16 addr;
u32 data;
@@ -802,7 +805,7 @@ static int vfio_exp_config_write(struct vfio_pci_device *vdev, int pos,
int count, struct perm_bits *perm,
int offset, __le32 val)
{
- __le16 *ctrl = (__le16 *)(vdev->vconfig + pos -
+ __le16 *ctrl = (__le16 *)(vdev->priv->vconfig + pos -
offset + PCI_EXP_DEVCTL);
int readrq = le16_to_cpu(*ctrl) & PCI_EXP_DEVCTL_READRQ;
@@ -883,7 +886,7 @@ static int vfio_af_config_write(struct vfio_pci_device *vdev, int pos,
int count, struct perm_bits *perm,
int offset, __le32 val)
{
- u8 *ctrl = vdev->vconfig + pos - offset + PCI_AF_CTRL;
+ u8 *ctrl = vdev->priv->vconfig + pos - offset + PCI_AF_CTRL;
count = vfio_default_config_write(vdev, pos, count, perm, offset, val);
if (count < 0)
@@ -1040,13 +1043,13 @@ static int vfio_find_cap_start(struct vfio_pci_device *vdev, int pos)
u8 cap;
int base = (pos >= PCI_CFG_SPACE_SIZE) ? PCI_CFG_SPACE_SIZE :
PCI_STD_HEADER_SIZEOF;
- cap = vdev->pci_config_map[pos];
+ cap = vdev->priv->pci_config_map[pos];
if (cap == PCI_CAP_ID_BASIC)
return 0;
/* XXX Can we have to abutting capabilities of the same type? */
- while (pos - 1 >= base && vdev->pci_config_map[pos - 1] == cap)
+ while (pos - 1 >= base && vdev->priv->pci_config_map[pos - 1] == cap)
pos--;
return pos;
@@ -1063,10 +1066,10 @@ static int vfio_msi_config_read(struct vfio_pci_device *vdev, int pos,
start = vfio_find_cap_start(vdev, pos);
- flags = (__le16 *)&vdev->vconfig[start];
+ flags = (__le16 *)&vdev->priv->vconfig[start];
*flags &= cpu_to_le16(~PCI_MSI_FLAGS_QMASK);
- *flags |= cpu_to_le16(vdev->msi_qmax << 1);
+ *flags |= cpu_to_le16(vdev->priv->msi_qmax << 1);
}
return vfio_default_config_read(vdev, pos, count, perm, offset, val);
@@ -1088,7 +1091,7 @@ static int vfio_msi_config_write(struct vfio_pci_device *vdev, int pos,
start = vfio_find_cap_start(vdev, pos);
- pflags = (__le16 *)&vdev->vconfig[start + PCI_MSI_FLAGS];
+ pflags = (__le16 *)&vdev->priv->vconfig[start + PCI_MSI_FLAGS];
flags = le16_to_cpu(*pflags);
@@ -1097,9 +1100,9 @@ static int vfio_msi_config_write(struct vfio_pci_device *vdev, int pos,
flags &= ~PCI_MSI_FLAGS_ENABLE;
/* Check queue size */
- if ((flags & PCI_MSI_FLAGS_QSIZE) >> 4 > vdev->msi_qmax) {
+ if ((flags & PCI_MSI_FLAGS_QSIZE) >> 4 > vdev->priv->msi_qmax) {
flags &= ~PCI_MSI_FLAGS_QSIZE;
- flags |= vdev->msi_qmax << 4;
+ flags |= vdev->priv->msi_qmax << 4;
}
/* Write back to virt and to hardware */
@@ -1168,16 +1171,16 @@ static int vfio_msi_cap_len(struct vfio_pci_device *vdev, u8 pos)
if (flags & PCI_MSI_FLAGS_MASKBIT)
len += 10;
- if (vdev->msi_perm)
+ if (vdev->priv->msi_perm)
return len;
- vdev->msi_perm = kmalloc(sizeof(struct perm_bits), GFP_KERNEL);
- if (!vdev->msi_perm)
+ vdev->priv->msi_perm = kmalloc(sizeof(struct perm_bits), GFP_KERNEL);
+ if (!vdev->priv->msi_perm)
return -ENOMEM;
- ret = init_pci_cap_msi_perm(vdev->msi_perm, len, flags);
+ ret = init_pci_cap_msi_perm(vdev->priv->msi_perm, len, flags);
if (ret) {
- kfree(vdev->msi_perm);
+ kfree(vdev->priv->msi_perm);
return ret;
}
@@ -1247,7 +1250,7 @@ static int vfio_cap_len(struct vfio_pci_device *vdev, u8 cap, u8 pos)
/* Test for extended capabilities */
pci_read_config_dword(pdev, PCI_CFG_SPACE_SIZE,
&dword);
- vdev->extended_caps = (dword != 0);
+ vdev->priv->extended_caps = (dword != 0);
}
return PCI_CAP_PCIX_SIZEOF_V2;
} else
@@ -1263,7 +1266,7 @@ static int vfio_cap_len(struct vfio_pci_device *vdev, u8 cap, u8 pos)
if (pdev->cfg_size > PCI_CFG_SPACE_SIZE) {
/* Test for extended capabilities */
pci_read_config_dword(pdev, PCI_CFG_SPACE_SIZE, &dword);
- vdev->extended_caps = (dword != 0);
+ vdev->priv->extended_caps = (dword != 0);
}
/* length based on version and type */
@@ -1390,7 +1393,7 @@ static int vfio_fill_vconfig_bytes(struct vfio_pci_device *vdev,
int filled;
if (size >= 4 && !(offset % 4)) {
- __le32 *dwordp = (__le32 *)&vdev->vconfig[offset];
+ __le32 *dwordp = (__le32 *)&vdev->priv->vconfig[offset];
u32 dword;
ret = pci_read_config_dword(pdev, offset, &dword);
@@ -1399,7 +1402,7 @@ static int vfio_fill_vconfig_bytes(struct vfio_pci_device *vdev,
*dwordp = cpu_to_le32(dword);
filled = 4;
} else if (size >= 2 && !(offset % 2)) {
- __le16 *wordp = (__le16 *)&vdev->vconfig[offset];
+ __le16 *wordp = (__le16 *)&vdev->priv->vconfig[offset];
u16 word;
ret = pci_read_config_word(pdev, offset, &word);
@@ -1408,7 +1411,7 @@ static int vfio_fill_vconfig_bytes(struct vfio_pci_device *vdev,
*wordp = cpu_to_le16(word);
filled = 2;
} else {
- u8 *byte = &vdev->vconfig[offset];
+ u8 *byte = &vdev->priv->vconfig[offset];
ret = pci_read_config_byte(pdev, offset, byte);
if (ret)
return ret;
@@ -1425,7 +1428,7 @@ static int vfio_fill_vconfig_bytes(struct vfio_pci_device *vdev,
static int vfio_cap_init(struct vfio_pci_device *vdev)
{
struct pci_dev *pdev = vdev->pdev;
- u8 *map = vdev->pci_config_map;
+ u8 *map = vdev->priv->pci_config_map;
u16 status;
u8 pos, *prev, cap;
int loops, ret, caps = 0;
@@ -1443,7 +1446,7 @@ static int vfio_cap_init(struct vfio_pci_device *vdev)
return ret;
/* Mark the previous position in case we want to skip a capability */
- prev = &vdev->vconfig[PCI_CAPABILITY_LIST];
+ prev = &vdev->priv->vconfig[PCI_CAPABILITY_LIST];
/* We can bound our loop, capabilities are dword aligned */
loops = (PCI_CFG_SPACE_SIZE - PCI_STD_HEADER_SIZEOF) / PCI_CAP_SIZEOF;
@@ -1493,14 +1496,14 @@ static int vfio_cap_init(struct vfio_pci_device *vdev)
if (ret)
return ret;
- prev = &vdev->vconfig[pos + PCI_CAP_LIST_NEXT];
+ prev = &vdev->priv->vconfig[pos + PCI_CAP_LIST_NEXT];
pos = next;
caps++;
}
/* If we didn't fill any capabilities, clear the status flag */
if (!caps) {
- __le16 *vstatus = (__le16 *)&vdev->vconfig[PCI_STATUS];
+ __le16 *vstatus = (__le16 *)&vdev->priv->vconfig[PCI_STATUS];
*vstatus &= ~cpu_to_le16(PCI_STATUS_CAP_LIST);
}
@@ -1510,12 +1513,12 @@ static int vfio_cap_init(struct vfio_pci_device *vdev)
static int vfio_ecap_init(struct vfio_pci_device *vdev)
{
struct pci_dev *pdev = vdev->pdev;
- u8 *map = vdev->pci_config_map;
+ u8 *map = vdev->priv->pci_config_map;
u16 epos;
__le32 *prev = NULL;
int loops, ret, ecaps = 0;
- if (!vdev->extended_caps)
+ if (!vdev->priv->extended_caps)
return 0;
epos = PCI_CFG_SPACE_SIZE;
@@ -1590,17 +1593,17 @@ static int vfio_ecap_init(struct vfio_pci_device *vdev)
* ecaps are absent, hope users check all the way to next.
*/
if (hidden)
- *(__le32 *)&vdev->vconfig[epos] &=
+ *(__le32 *)&vdev->priv->vconfig[epos] &=
cpu_to_le32((0xffcU << 20));
else
ecaps++;
- prev = (__le32 *)&vdev->vconfig[epos];
+ prev = (__le32 *)&vdev->priv->vconfig[epos];
epos = PCI_EXT_CAP_NEXT(header);
}
if (!ecaps)
- *(u32 *)&vdev->vconfig[PCI_CFG_SPACE_SIZE] = 0;
+ *(u32 *)&vdev->priv->vconfig[PCI_CFG_SPACE_SIZE] = 0;
return 0;
}
@@ -1632,6 +1635,7 @@ int vfio_config_init(struct vfio_pci_device *vdev)
struct pci_dev *pdev = vdev->pdev;
u8 *map, *vconfig;
int ret;
+ u32 *rbar;
/*
* Config space, caps and ecaps are all dword aligned, so we could
@@ -1649,8 +1653,8 @@ int vfio_config_init(struct vfio_pci_device *vdev)
return -ENOMEM;
}
- vdev->pci_config_map = map;
- vdev->vconfig = vconfig;
+ vdev->priv->pci_config_map = map;
+ vdev->priv->vconfig = vconfig;
memset(map, PCI_CAP_ID_BASIC, PCI_STD_HEADER_SIZEOF);
memset(map + PCI_STD_HEADER_SIZEOF, PCI_CAP_ID_INVALID,
@@ -1660,7 +1664,7 @@ int vfio_config_init(struct vfio_pci_device *vdev)
if (ret)
goto out;
- vdev->bardirty = true;
+ vdev->priv->bardirty = true;
/*
* XXX can we just pci_load_saved_state/pci_restore_state?
@@ -1668,13 +1672,14 @@ int vfio_config_init(struct vfio_pci_device *vdev)
*/
/* For restore after reset */
- vdev->rbar[0] = le32_to_cpu(*(__le32 *)&vconfig[PCI_BASE_ADDRESS_0]);
- vdev->rbar[1] = le32_to_cpu(*(__le32 *)&vconfig[PCI_BASE_ADDRESS_1]);
- vdev->rbar[2] = le32_to_cpu(*(__le32 *)&vconfig[PCI_BASE_ADDRESS_2]);
- vdev->rbar[3] = le32_to_cpu(*(__le32 *)&vconfig[PCI_BASE_ADDRESS_3]);
- vdev->rbar[4] = le32_to_cpu(*(__le32 *)&vconfig[PCI_BASE_ADDRESS_4]);
- vdev->rbar[5] = le32_to_cpu(*(__le32 *)&vconfig[PCI_BASE_ADDRESS_5]);
- vdev->rbar[6] = le32_to_cpu(*(__le32 *)&vconfig[PCI_ROM_ADDRESS]);
+ rbar = vdev->priv->rbar;
+ rbar[0] = le32_to_cpu(*(__le32 *)&vconfig[PCI_BASE_ADDRESS_0]);
+ rbar[1] = le32_to_cpu(*(__le32 *)&vconfig[PCI_BASE_ADDRESS_1]);
+ rbar[2] = le32_to_cpu(*(__le32 *)&vconfig[PCI_BASE_ADDRESS_2]);
+ rbar[3] = le32_to_cpu(*(__le32 *)&vconfig[PCI_BASE_ADDRESS_3]);
+ rbar[4] = le32_to_cpu(*(__le32 *)&vconfig[PCI_BASE_ADDRESS_4]);
+ rbar[5] = le32_to_cpu(*(__le32 *)&vconfig[PCI_BASE_ADDRESS_5]);
+ rbar[6] = le32_to_cpu(*(__le32 *)&vconfig[PCI_ROM_ADDRESS]);
if (pdev->is_virtfn) {
*(__le16 *)&vconfig[PCI_VENDOR_ID] = cpu_to_le16(pdev->vendor);
@@ -1699,7 +1704,7 @@ int vfio_config_init(struct vfio_pci_device *vdev)
vconfig[PCI_INTERRUPT_PIN] = 0; /* Gratuitous for good VFs */
}
- if (!IS_ENABLED(CONFIG_VFIO_PCI_INTX) || vdev->nointx)
+ if (!IS_ENABLED(CONFIG_VFIO_PCI_INTX) || vdev->priv->nointx)
vconfig[PCI_INTERRUPT_PIN] = 0;
ret = vfio_cap_init(vdev);
@@ -1714,20 +1719,20 @@ int vfio_config_init(struct vfio_pci_device *vdev)
out:
kfree(map);
- vdev->pci_config_map = NULL;
+ vdev->priv->pci_config_map = NULL;
kfree(vconfig);
- vdev->vconfig = NULL;
+ vdev->priv->vconfig = NULL;
return pcibios_err_to_errno(ret);
}
void vfio_config_free(struct vfio_pci_device *vdev)
{
- kfree(vdev->vconfig);
- vdev->vconfig = NULL;
- kfree(vdev->pci_config_map);
- vdev->pci_config_map = NULL;
- kfree(vdev->msi_perm);
- vdev->msi_perm = NULL;
+ kfree(vdev->priv->vconfig);
+ vdev->priv->vconfig = NULL;
+ kfree(vdev->priv->pci_config_map);
+ vdev->priv->pci_config_map = NULL;
+ kfree(vdev->priv->msi_perm);
+ vdev->priv->msi_perm = NULL;
}
/*
@@ -1737,12 +1742,14 @@ void vfio_config_free(struct vfio_pci_device *vdev)
static size_t vfio_pci_cap_remaining_dword(struct vfio_pci_device *vdev,
loff_t pos)
{
- u8 cap = vdev->pci_config_map[pos];
+ u8 *pci_config_map = vdev->priv->pci_config_map;
+ u8 cap = pci_config_map[pos];
size_t i;
- for (i = 1; (pos + i) % 4 && vdev->pci_config_map[pos + i] == cap; i++)
+ for (i = 1; (pos + i) % 4 && (pci_config_map[pos + i] == cap); i++)
/* nop */;
+
return i;
}
@@ -1774,7 +1781,7 @@ static ssize_t vfio_config_do_rw(struct vfio_pci_device *vdev, char __user *buf,
ret = count;
- cap_id = vdev->pci_config_map[*ppos];
+ cap_id = vdev->priv->pci_config_map[*ppos];
if (cap_id == PCI_CAP_ID_INVALID) {
perm = &unassigned_perms;
@@ -1794,7 +1801,7 @@ static ssize_t vfio_config_do_rw(struct vfio_pci_device *vdev, char __user *buf,
perm = &cap_perms[cap_id];
if (cap_id == PCI_CAP_ID_MSI)
- perm = vdev->msi_perm;
+ perm = vdev->priv->msi_perm;
if (cap_id > PCI_CAP_ID_BASIC)
cap_start = vfio_find_cap_start(vdev, *ppos);
@@ -25,13 +25,13 @@ static size_t vfio_pci_igd_rw(struct vfio_pci_device *vdev, char __user *buf,
size_t count, loff_t *ppos, bool iswrite)
{
unsigned int i = VFIO_PCI_OFFSET_TO_INDEX(*ppos) - VFIO_PCI_NUM_REGIONS;
- void *base = vdev->region[i].data;
+ void *base = vdev->priv->region[i].data;
loff_t pos = *ppos & VFIO_PCI_OFFSET_MASK;
- if (pos >= vdev->region[i].size || iswrite)
+ if (pos >= vdev->priv->region[i].size || iswrite)
return -EINVAL;
- count = min(count, (size_t)(vdev->region[i].size - pos));
+ count = min(count, (size_t)(vdev->priv->region[i].size - pos));
if (copy_to_user(buf, base + pos, count))
return -EFAULT;
@@ -54,7 +54,7 @@ static const struct vfio_pci_regops vfio_pci_igd_regops = {
static int vfio_pci_igd_opregion_init(struct vfio_pci_device *vdev)
{
- __le32 *dwordp = (__le32 *)(vdev->vconfig + OPREGION_PCI_ADDR);
+ __le32 *dwordp = (__le32 *)(vdev->priv->vconfig + OPREGION_PCI_ADDR);
u32 addr, size;
void *base;
int ret;
@@ -101,7 +101,7 @@ static int vfio_pci_igd_opregion_init(struct vfio_pci_device *vdev)
/* Fill vconfig with the hw value and virtualize register */
*dwordp = cpu_to_le32(addr);
- memset(vdev->pci_config_map + OPREGION_PCI_ADDR,
+ memset(vdev->priv->pci_config_map + OPREGION_PCI_ADDR,
PCI_CAP_ID_INVALID_VIRT, 4);
return ret;
@@ -112,15 +112,15 @@ static size_t vfio_pci_igd_cfg_rw(struct vfio_pci_device *vdev,
bool iswrite)
{
unsigned int i = VFIO_PCI_OFFSET_TO_INDEX(*ppos) - VFIO_PCI_NUM_REGIONS;
- struct pci_dev *pdev = vdev->region[i].data;
+ struct pci_dev *pdev = vdev->priv->region[i].data;
loff_t pos = *ppos & VFIO_PCI_OFFSET_MASK;
size_t size;
int ret;
- if (pos >= vdev->region[i].size || iswrite)
+ if (pos >= vdev->priv->region[i].size || iswrite)
return -EINVAL;
- size = count = min(count, (size_t)(vdev->region[i].size - pos));
+ size = count = min(count, (size_t)(vdev->priv->region[i].size - pos));
if ((pos & 1) && size) {
u8 val;
@@ -29,8 +29,8 @@ static void vfio_send_intx_eventfd(void *opaque, void *unused)
{
struct vfio_pci_device *vdev = opaque;
- if (likely(is_intx(vdev) && !vdev->virq_disabled))
- eventfd_signal(vdev->ctx[0].trigger, 1);
+ if (likely(is_intx(vdev) && !vdev->priv->virq_disabled))
+ eventfd_signal(vdev->priv->ctx[0].trigger, 1);
}
void vfio_pci_intx_mask(struct vfio_pci_device *vdev)
@@ -38,7 +38,7 @@ void vfio_pci_intx_mask(struct vfio_pci_device *vdev)
struct pci_dev *pdev = vdev->pdev;
unsigned long flags;
- spin_lock_irqsave(&vdev->irqlock, flags);
+ spin_lock_irqsave(&vdev->priv->irqlock, flags);
/*
* Masking can come from interrupt, ioctl, or config space
@@ -47,22 +47,22 @@ void vfio_pci_intx_mask(struct vfio_pci_device *vdev)
* try to have the physical bit follow the virtual bit.
*/
if (unlikely(!is_intx(vdev))) {
- if (vdev->pci_2_3)
+ if (vdev->priv->pci_2_3)
pci_intx(pdev, 0);
- } else if (!vdev->ctx[0].masked) {
+ } else if (!vdev->priv->ctx[0].masked) {
/*
* Can't use check_and_mask here because we always want to
* mask, not just when something is pending.
*/
- if (vdev->pci_2_3)
+ if (vdev->priv->pci_2_3)
pci_intx(pdev, 0);
else
disable_irq_nosync(pdev->irq);
- vdev->ctx[0].masked = true;
+ vdev->priv->ctx[0].masked = true;
}
- spin_unlock_irqrestore(&vdev->irqlock, flags);
+ spin_unlock_irqrestore(&vdev->priv->irqlock, flags);
}
/*
@@ -78,31 +78,31 @@ static int vfio_pci_intx_unmask_handler(void *opaque, void *unused)
unsigned long flags;
int ret = 0;
- spin_lock_irqsave(&vdev->irqlock, flags);
+ spin_lock_irqsave(&vdev->priv->irqlock, flags);
/*
* Unmasking comes from ioctl or config, so again, have the
* physical bit follow the virtual even when not using INTx.
*/
if (unlikely(!is_intx(vdev))) {
- if (vdev->pci_2_3)
+ if (vdev->priv->pci_2_3)
pci_intx(pdev, 1);
- } else if (vdev->ctx[0].masked && !vdev->virq_disabled) {
+ } else if (vdev->priv->ctx[0].masked && !vdev->priv->virq_disabled) {
/*
* A pending interrupt here would immediately trigger,
* but we can avoid that overhead by just re-sending
* the interrupt to the user.
*/
- if (vdev->pci_2_3) {
+ if (vdev->priv->pci_2_3) {
if (!pci_check_and_unmask_intx(pdev))
ret = 1;
} else
enable_irq(pdev->irq);
- vdev->ctx[0].masked = (ret > 0);
+ vdev->priv->ctx[0].masked = (ret > 0);
}
- spin_unlock_irqrestore(&vdev->irqlock, flags);
+ spin_unlock_irqrestore(&vdev->priv->irqlock, flags);
return ret;
}
@@ -119,19 +119,19 @@ static irqreturn_t vfio_intx_handler(int irq, void *dev_id)
unsigned long flags;
int ret = IRQ_NONE;
- spin_lock_irqsave(&vdev->irqlock, flags);
+ spin_lock_irqsave(&vdev->priv->irqlock, flags);
- if (!vdev->pci_2_3) {
+ if (!vdev->priv->pci_2_3) {
disable_irq_nosync(vdev->pdev->irq);
- vdev->ctx[0].masked = true;
+ vdev->priv->ctx[0].masked = true;
ret = IRQ_HANDLED;
- } else if (!vdev->ctx[0].masked && /* may be shared */
+ } else if (!vdev->priv->ctx[0].masked && /* may be shared */
pci_check_and_mask_intx(vdev->pdev)) {
- vdev->ctx[0].masked = true;
+ vdev->priv->ctx[0].masked = true;
ret = IRQ_HANDLED;
}
- spin_unlock_irqrestore(&vdev->irqlock, flags);
+ spin_unlock_irqrestore(&vdev->priv->irqlock, flags);
if (ret == IRQ_HANDLED)
vfio_send_intx_eventfd(vdev, NULL);
@@ -147,11 +147,11 @@ static int vfio_intx_enable(struct vfio_pci_device *vdev)
if (!vdev->pdev->irq)
return -ENODEV;
- vdev->ctx = kzalloc(sizeof(struct vfio_pci_irq_ctx), GFP_KERNEL);
- if (!vdev->ctx)
+ vdev->priv->ctx = kzalloc(sizeof(struct vfio_pci_irq_ctx), GFP_KERNEL);
+ if (!vdev->priv->ctx)
return -ENOMEM;
- vdev->num_ctx = 1;
+ vdev->priv->num_ctx = 1;
/*
* If the virtual interrupt is masked, restore it. Devices
@@ -159,9 +159,9 @@ static int vfio_intx_enable(struct vfio_pci_device *vdev)
* here, non-PCI-2.3 devices will have to wait until the
* interrupt is enabled.
*/
- vdev->ctx[0].masked = vdev->virq_disabled;
- if (vdev->pci_2_3)
- pci_intx(vdev->pdev, !vdev->ctx[0].masked);
+ vdev->priv->ctx[0].masked = vdev->priv->virq_disabled;
+ if (vdev->priv->pci_2_3)
+ pci_intx(vdev->pdev, !vdev->priv->ctx[0].masked);
vdev->irq_type = VFIO_PCI_INTX_IRQ_INDEX;
@@ -176,37 +176,37 @@ static int vfio_intx_set_signal(struct vfio_pci_device *vdev, int fd)
unsigned long flags;
int ret;
- if (vdev->ctx[0].trigger) {
+ if (vdev->priv->ctx[0].trigger) {
free_irq(pdev->irq, vdev);
- kfree(vdev->ctx[0].name);
- eventfd_ctx_put(vdev->ctx[0].trigger);
- vdev->ctx[0].trigger = NULL;
+ kfree(vdev->priv->ctx[0].name);
+ eventfd_ctx_put(vdev->priv->ctx[0].trigger);
+ vdev->priv->ctx[0].trigger = NULL;
}
if (fd < 0) /* Disable only */
return 0;
- vdev->ctx[0].name = kasprintf(GFP_KERNEL, "vfio-intx(%s)",
+ vdev->priv->ctx[0].name = kasprintf(GFP_KERNEL, "vfio-intx(%s)",
pci_name(pdev));
- if (!vdev->ctx[0].name)
+ if (!vdev->priv->ctx[0].name)
return -ENOMEM;
trigger = eventfd_ctx_fdget(fd);
if (IS_ERR(trigger)) {
- kfree(vdev->ctx[0].name);
+ kfree(vdev->priv->ctx[0].name);
return PTR_ERR(trigger);
}
- vdev->ctx[0].trigger = trigger;
+ vdev->priv->ctx[0].trigger = trigger;
- if (!vdev->pci_2_3)
+ if (!vdev->priv->pci_2_3)
irqflags = 0;
ret = request_irq(pdev->irq, vfio_intx_handler,
- irqflags, vdev->ctx[0].name, vdev);
+ irqflags, vdev->priv->ctx[0].name, vdev);
if (ret) {
- vdev->ctx[0].trigger = NULL;
- kfree(vdev->ctx[0].name);
+ vdev->priv->ctx[0].trigger = NULL;
+ kfree(vdev->priv->ctx[0].name);
eventfd_ctx_put(trigger);
return ret;
}
@@ -215,22 +215,22 @@ static int vfio_intx_set_signal(struct vfio_pci_device *vdev, int fd)
* INTx disable will stick across the new irq setup,
* disable_irq won't.
*/
- spin_lock_irqsave(&vdev->irqlock, flags);
- if (!vdev->pci_2_3 && vdev->ctx[0].masked)
+ spin_lock_irqsave(&vdev->priv->irqlock, flags);
+ if (!vdev->priv->pci_2_3 && vdev->priv->ctx[0].masked)
disable_irq_nosync(pdev->irq);
- spin_unlock_irqrestore(&vdev->irqlock, flags);
+ spin_unlock_irqrestore(&vdev->priv->irqlock, flags);
return 0;
}
static void vfio_intx_disable(struct vfio_pci_device *vdev)
{
- vfio_virqfd_disable(&vdev->ctx[0].unmask);
- vfio_virqfd_disable(&vdev->ctx[0].mask);
+ vfio_virqfd_disable(&vdev->priv->ctx[0].unmask);
+ vfio_virqfd_disable(&vdev->priv->ctx[0].mask);
vfio_intx_set_signal(vdev, -1);
vdev->irq_type = VFIO_PCI_NUM_IRQS;
- vdev->num_ctx = 0;
- kfree(vdev->ctx);
+ vdev->priv->num_ctx = 0;
+ kfree(vdev->priv->ctx);
}
/*
@@ -253,8 +253,9 @@ static int vfio_msi_enable(struct vfio_pci_device *vdev, int nvec, bool msix)
if (!is_irq_none(vdev))
return -EINVAL;
- vdev->ctx = kcalloc(nvec, sizeof(struct vfio_pci_irq_ctx), GFP_KERNEL);
- if (!vdev->ctx)
+ vdev->priv->ctx = kcalloc(nvec, sizeof(struct vfio_pci_irq_ctx),
+ GFP_KERNEL);
+ if (!vdev->priv->ctx)
return -ENOMEM;
/* return the number of supported vectors if we can't get all: */
@@ -262,11 +263,11 @@ static int vfio_msi_enable(struct vfio_pci_device *vdev, int nvec, bool msix)
if (ret < nvec) {
if (ret > 0)
pci_free_irq_vectors(pdev);
- kfree(vdev->ctx);
+ kfree(vdev->priv->ctx);
return ret;
}
- vdev->num_ctx = nvec;
+ vdev->priv->num_ctx = nvec;
vdev->irq_type = msix ? VFIO_PCI_MSIX_IRQ_INDEX :
VFIO_PCI_MSI_IRQ_INDEX;
@@ -275,7 +276,7 @@ static int vfio_msi_enable(struct vfio_pci_device *vdev, int nvec, bool msix)
* Compute the virtual hardware field for max msi vectors -
* it is the log base 2 of the number of vectors.
*/
- vdev->msi_qmax = fls(nvec * 2 - 1) - 1;
+ vdev->priv->msi_qmax = fls(nvec * 2 - 1) - 1;
}
return 0;
@@ -287,32 +288,34 @@ static int vfio_msi_set_vector_signal(struct vfio_pci_device *vdev,
struct pci_dev *pdev = vdev->pdev;
struct eventfd_ctx *trigger;
int irq, ret;
+ struct vfio_pci_irq_ctx *ctx;
- if (vector < 0 || vector >= vdev->num_ctx)
+ if (vector < 0 || vector >= vdev->priv->num_ctx)
return -EINVAL;
irq = pci_irq_vector(pdev, vector);
- if (vdev->ctx[vector].trigger) {
- free_irq(irq, vdev->ctx[vector].trigger);
- irq_bypass_unregister_producer(&vdev->ctx[vector].producer);
- kfree(vdev->ctx[vector].name);
- eventfd_ctx_put(vdev->ctx[vector].trigger);
- vdev->ctx[vector].trigger = NULL;
+ ctx = &vdev->priv->ctx[vector];
+ if (ctx->trigger) {
+ free_irq(irq, ctx->trigger);
+ irq_bypass_unregister_producer(&ctx->producer);
+ kfree(ctx->name);
+ eventfd_ctx_put(ctx->trigger);
+ ctx->trigger = NULL;
}
if (fd < 0)
return 0;
- vdev->ctx[vector].name = kasprintf(GFP_KERNEL, "vfio-msi%s[%d](%s)",
+ ctx->name = kasprintf(GFP_KERNEL, "vfio-msi%s[%d](%s)",
msix ? "x" : "", vector,
pci_name(pdev));
- if (!vdev->ctx[vector].name)
+ if (!ctx->name)
return -ENOMEM;
trigger = eventfd_ctx_fdget(fd);
if (IS_ERR(trigger)) {
- kfree(vdev->ctx[vector].name);
+ kfree(ctx->name);
return PTR_ERR(trigger);
}
@@ -330,23 +333,22 @@ static int vfio_msi_set_vector_signal(struct vfio_pci_device *vdev,
pci_write_msi_msg(irq, &msg);
}
- ret = request_irq(irq, vfio_msihandler, 0,
- vdev->ctx[vector].name, trigger);
+ ret = request_irq(irq, vfio_msihandler, 0, ctx->name, trigger);
if (ret) {
- kfree(vdev->ctx[vector].name);
+ kfree(vdev->priv->ctx[vector].name);
eventfd_ctx_put(trigger);
return ret;
}
- vdev->ctx[vector].producer.token = trigger;
- vdev->ctx[vector].producer.irq = irq;
- ret = irq_bypass_register_producer(&vdev->ctx[vector].producer);
+ ctx->producer.token = trigger;
+ ctx->producer.irq = irq;
+ ret = irq_bypass_register_producer(&ctx->producer);
if (unlikely(ret))
dev_info(&pdev->dev,
"irq bypass producer (token %p) registration fails: %d\n",
- vdev->ctx[vector].producer.token, ret);
+ ctx->producer.token, ret);
- vdev->ctx[vector].trigger = trigger;
+ ctx->trigger = trigger;
return 0;
}
@@ -356,7 +358,7 @@ static int vfio_msi_set_block(struct vfio_pci_device *vdev, unsigned start,
{
int i, j, ret = 0;
- if (start >= vdev->num_ctx || start + count > vdev->num_ctx)
+ if (start >= vdev->priv->num_ctx || start + count > vdev->priv->num_ctx)
return -EINVAL;
for (i = 0, j = start; i < count && !ret; i++, j++) {
@@ -377,12 +379,12 @@ static void vfio_msi_disable(struct vfio_pci_device *vdev, bool msix)
struct pci_dev *pdev = vdev->pdev;
int i;
- for (i = 0; i < vdev->num_ctx; i++) {
- vfio_virqfd_disable(&vdev->ctx[i].unmask);
- vfio_virqfd_disable(&vdev->ctx[i].mask);
+ for (i = 0; i < vdev->priv->num_ctx; i++) {
+ vfio_virqfd_disable(&vdev->priv->ctx[i].unmask);
+ vfio_virqfd_disable(&vdev->priv->ctx[i].mask);
}
- vfio_msi_set_block(vdev, 0, vdev->num_ctx, NULL, msix);
+ vfio_msi_set_block(vdev, 0, vdev->priv->num_ctx, NULL, msix);
pci_free_irq_vectors(pdev);
@@ -390,12 +392,12 @@ static void vfio_msi_disable(struct vfio_pci_device *vdev, bool msix)
* Both disable paths above use pci_intx_for_msi() to clear DisINTx
* via their shutdown paths. Restore for NoINTx devices.
*/
- if (vdev->nointx)
+ if (vdev->priv->nointx)
pci_intx(pdev, 0);
vdev->irq_type = VFIO_PCI_NUM_IRQS;
- vdev->num_ctx = 0;
- kfree(vdev->ctx);
+ vdev->priv->num_ctx = 0;
+ kfree(vdev->priv->ctx);
}
/*
@@ -420,9 +422,10 @@ static int vfio_pci_set_intx_unmask(struct vfio_pci_device *vdev,
return vfio_virqfd_enable((void *) vdev,
vfio_pci_intx_unmask_handler,
vfio_send_intx_eventfd, NULL,
- &vdev->ctx[0].unmask, fd);
+ &vdev->priv->ctx[0].unmask,
+ fd);
- vfio_virqfd_disable(&vdev->ctx[0].unmask);
+ vfio_virqfd_disable(&vdev->priv->ctx[0].unmask);
}
return 0;
@@ -525,18 +528,18 @@ static int vfio_pci_set_msi_trigger(struct vfio_pci_device *vdev,
return ret;
}
- if (!irq_is(vdev, index) || start + count > vdev->num_ctx)
+ if (!irq_is(vdev, index) || start + count > vdev->priv->num_ctx)
return -EINVAL;
for (i = start; i < start + count; i++) {
- if (!vdev->ctx[i].trigger)
+ if (!vdev->priv->ctx[i].trigger)
continue;
if (flags & VFIO_IRQ_SET_DATA_NONE) {
- eventfd_signal(vdev->ctx[i].trigger, 1);
+ eventfd_signal(vdev->priv->ctx[i].trigger, 1);
} else if (flags & VFIO_IRQ_SET_DATA_BOOL) {
uint8_t *bools = data;
if (bools[i - start])
- eventfd_signal(vdev->ctx[i].trigger, 1);
+ eventfd_signal(vdev->priv->ctx[i].trigger, 1);
}
}
return 0;
@@ -604,7 +607,7 @@ static int vfio_pci_set_err_trigger(struct vfio_pci_device *vdev,
if (index != VFIO_PCI_ERR_IRQ_INDEX || start != 0 || count > 1)
return -EINVAL;
- return vfio_pci_set_ctx_trigger_single(&vdev->err_trigger,
+ return vfio_pci_set_ctx_trigger_single(&vdev->priv->err_trigger,
count, flags, data);
}
@@ -615,7 +618,7 @@ static int vfio_pci_set_req_trigger(struct vfio_pci_device *vdev,
if (index != VFIO_PCI_REQ_IRQ_INDEX || start != 0 || count > 1)
return -EINVAL;
- return vfio_pci_set_ctx_trigger_single(&vdev->req_trigger,
+ return vfio_pci_set_ctx_trigger_single(&vdev->priv->req_trigger,
count, flags, data);
}
@@ -43,16 +43,16 @@ static size_t vfio_pci_nvgpu_rw(struct vfio_pci_device *vdev,
char __user *buf, size_t count, loff_t *ppos, bool iswrite)
{
unsigned int i = VFIO_PCI_OFFSET_TO_INDEX(*ppos) - VFIO_PCI_NUM_REGIONS;
- struct vfio_pci_nvgpu_data *data = vdev->region[i].data;
+ struct vfio_pci_nvgpu_data *data = vdev->priv->region[i].data;
loff_t pos = *ppos & VFIO_PCI_OFFSET_MASK;
loff_t posaligned = pos & PAGE_MASK, posoff = pos & ~PAGE_MASK;
size_t sizealigned;
void __iomem *ptr;
- if (pos >= vdev->region[i].size)
+ if (pos >= vdev->priv->region[i].size)
return -EINVAL;
- count = min(count, (size_t)(vdev->region[i].size - pos));
+ count = min(count, (size_t)(vdev->priv->region[i].size - pos));
/*
* We map only a bit of GPU RAM for a short time instead of mapping it
@@ -115,7 +115,7 @@ static vm_fault_t vfio_pci_nvgpu_mmap_fault(struct vm_fault *vmf)
{
vm_fault_t ret;
struct vm_area_struct *vma = vmf->vma;
- struct vfio_pci_region *region = vma->vm_private_data;
+ struct vfio_pci_region *region = vma->vm_priv_data;
struct vfio_pci_nvgpu_data *data = region->data;
unsigned long vmf_off = (vmf->address - vma->vm_start) >> PAGE_SHIFT;
unsigned long nv2pg = data->gpu_hpa >> PAGE_SHIFT;
@@ -146,7 +146,7 @@ static int vfio_pci_nvgpu_mmap(struct vfio_pci_device *vdev,
if (vma->vm_end - vma->vm_start > data->size)
return -EINVAL;
- vma->vm_private_data = region;
+ vma->vm_priv_data = region;
vma->vm_flags |= VM_PFNMAP;
vma->vm_ops = &vfio_pci_nvgpu_mmap_vmops;
@@ -306,13 +306,13 @@ static size_t vfio_pci_npu2_rw(struct vfio_pci_device *vdev,
char __user *buf, size_t count, loff_t *ppos, bool iswrite)
{
unsigned int i = VFIO_PCI_OFFSET_TO_INDEX(*ppos) - VFIO_PCI_NUM_REGIONS;
- struct vfio_pci_npu2_data *data = vdev->region[i].data;
+ struct vfio_pci_npu2_data *data = vdev->priv->region[i].data;
loff_t pos = *ppos & VFIO_PCI_OFFSET_MASK;
- if (pos >= vdev->region[i].size)
+ if (pos >= vdev->priv->region[i].size)
return -EINVAL;
- count = min(count, (size_t)(vdev->region[i].size - pos));
+ count = min(count, (size_t)(vdev->priv->region[i].size - pos));
if (iswrite) {
if (copy_from_user(data->base + pos, buf, count))
@@ -84,8 +84,7 @@ struct vfio_pci_reflck {
struct mutex lock;
};
-struct vfio_pci_device {
- struct pci_dev *pdev;
+struct vfio_pci_device_private {
void __iomem *barmap[PCI_STD_RESOURCE_END + 1];
bool bar_mmap_supported[PCI_STD_RESOURCE_END + 1];
u8 *pci_config_map;
@@ -95,8 +94,6 @@ struct vfio_pci_device {
struct mutex igate;
struct vfio_pci_irq_ctx *ctx;
int num_ctx;
- int irq_type;
- int num_regions;
struct vfio_pci_region *region;
u8 msi_qmax;
u8 msix_bar;
@@ -135,7 +135,7 @@ static int vfio_pci_setup_barmap(struct vfio_pci_device *vdev, int bar)
int ret;
void __iomem *io;
- if (vdev->barmap[bar])
+ if (vdev->priv->barmap[bar])
return 0;
ret = pci_request_selected_regions(pdev, 1 << bar, "vfio");
@@ -148,7 +148,7 @@ static int vfio_pci_setup_barmap(struct vfio_pci_device *vdev, int bar)
return -ENOMEM;
}
- vdev->barmap[bar] = io;
+ vdev->priv->barmap[bar] = io;
return 0;
}
@@ -192,12 +192,12 @@ ssize_t vfio_pci_bar_rw(struct vfio_pci_device *vdev, char __user *buf,
if (ret)
return ret;
- io = vdev->barmap[bar];
+ io = vdev->priv->barmap[bar];
}
- if (bar == vdev->msix_bar) {
- x_start = vdev->msix_offset;
- x_end = vdev->msix_offset + vdev->msix_size;
+ if (bar == vdev->priv->msix_bar) {
+ x_start = vdev->priv->msix_offset;
+ x_end = vdev->priv->msix_offset + vdev->priv->msix_size;
}
done = do_io_rw(io, buf, pos, count, x_start, x_end, iswrite);
@@ -221,7 +221,7 @@ ssize_t vfio_pci_vga_rw(struct vfio_pci_device *vdev, char __user *buf,
bool is_ioport;
ssize_t done;
- if (!vdev->has_vga)
+ if (!vdev->priv->has_vga)
return -EINVAL;
if (pos > 0xbfffful)
@@ -314,9 +314,9 @@ long vfio_pci_ioeventfd(struct vfio_pci_device *vdev, loff_t offset,
return -EINVAL;
/* Disallow ioeventfds working around MSI-X table writes */
- if (bar == vdev->msix_bar &&
- !(pos + count <= vdev->msix_offset ||
- pos >= vdev->msix_offset + vdev->msix_size))
+ if (bar == vdev->priv->msix_bar &&
+ !(pos + count <= vdev->priv->msix_offset ||
+ pos >= vdev->priv->msix_offset + vdev->priv->msix_size))
return -EINVAL;
#ifndef iowrite64
@@ -328,15 +328,15 @@ long vfio_pci_ioeventfd(struct vfio_pci_device *vdev, loff_t offset,
if (ret)
return ret;
- mutex_lock(&vdev->ioeventfds_lock);
+ mutex_lock(&vdev->priv->ioeventfds_lock);
- list_for_each_entry(ioeventfd, &vdev->ioeventfds_list, next) {
+ list_for_each_entry(ioeventfd, &vdev->priv->ioeventfds_list, next) {
if (ioeventfd->pos == pos && ioeventfd->bar == bar &&
ioeventfd->data == data && ioeventfd->count == count) {
if (fd == -1) {
vfio_virqfd_disable(&ioeventfd->virqfd);
list_del(&ioeventfd->next);
- vdev->ioeventfds_nr--;
+ vdev->priv->ioeventfds_nr--;
kfree(ioeventfd);
ret = 0;
} else
@@ -351,7 +351,7 @@ long vfio_pci_ioeventfd(struct vfio_pci_device *vdev, loff_t offset,
goto out_unlock;
}
- if (vdev->ioeventfds_nr >= VFIO_PCI_IOEVENTFD_MAX) {
+ if (vdev->priv->ioeventfds_nr >= VFIO_PCI_IOEVENTFD_MAX) {
ret = -ENOSPC;
goto out_unlock;
}
@@ -362,7 +362,7 @@ long vfio_pci_ioeventfd(struct vfio_pci_device *vdev, loff_t offset,
goto out_unlock;
}
- ioeventfd->addr = vdev->barmap[bar] + pos;
+ ioeventfd->addr = vdev->priv->barmap[bar] + pos;
ioeventfd->data = data;
ioeventfd->pos = pos;
ioeventfd->bar = bar;
@@ -375,11 +375,11 @@ long vfio_pci_ioeventfd(struct vfio_pci_device *vdev, loff_t offset,
goto out_unlock;
}
- list_add(&ioeventfd->next, &vdev->ioeventfds_list);
- vdev->ioeventfds_nr++;
+ list_add(&ioeventfd->next, &vdev->priv->ioeventfds_list);
+ vdev->priv->ioeventfds_nr++;
out_unlock:
- mutex_unlock(&vdev->ioeventfds_lock);
+ mutex_unlock(&vdev->priv->ioeventfds_lock);
return ret;
}
@@ -195,4 +195,11 @@ extern int vfio_virqfd_enable(void *opaque,
void *data, struct virqfd **pvirqfd, int fd);
extern void vfio_virqfd_disable(struct virqfd **pvirqfd);
+struct vfio_pci_device_private;
+struct vfio_pci_device {
+ struct pci_dev *pdev;
+ int num_regions;
+ int irq_type;
+ struct vfio_pci_device_private *priv;
+};
#endif /* VFIO_H */
split vfio_pci_device into two parts: (1) a public part, including pdev, num_region, irq_type which are accessible from outside of vfio. (2) a private part, a pointer to vfio_pci_device_private, only accessible within vfio Cc: Kevin Tian <kevin.tian@intel.com> Signed-off-by: Yan Zhao <yan.y.zhao@intel.com> --- drivers/vfio/pci/vfio_pci.c | 209 +++++++++++++++------------- drivers/vfio/pci/vfio_pci_config.c | 157 +++++++++++---------- drivers/vfio/pci/vfio_pci_igd.c | 16 +-- drivers/vfio/pci/vfio_pci_intrs.c | 171 ++++++++++++----------- drivers/vfio/pci/vfio_pci_nvlink2.c | 16 +-- drivers/vfio/pci/vfio_pci_private.h | 5 +- drivers/vfio/pci/vfio_pci_rdwr.c | 36 ++--- include/linux/vfio.h | 7 + 8 files changed, 321 insertions(+), 296 deletions(-)