diff mbox series

[v1,14/18] vfio/mdev: Add mdev available instance checking to the core

Message ID 20220602171948.2790690-15-farman@linux.ibm.com (mailing list archive)
State New, archived
Headers show
Series VFIO ccw/mdev rework | expand

Commit Message

Eric Farman June 2, 2022, 5:19 p.m. UTC
From: Jason Gunthorpe <jgg@nvidia.com>

Many of the mdev drivers use a simple counter for keeping track of the
available instances. Move this code to the core code and store the counter
in the mdev_type. Implement it using correct locking, fixing mdpy.

Drivers provide a get_available() callback to set the number of available
instances for their mtypes which is fixed at registration time. The core
provides a standard sysfs attribute to return the available_instances.

Cc: Kirti Wankhede <kwankhede@nvidia.com>
Cc: Jonathan Corbet <corbet@lwn.net>
Cc: Tony Krowiak <akrowiak@linux.ibm.com>
Cc: Jason Herne <jjherne@linux.ibm.com>
Reviewed-by: Christoph Hellwig <hch@lst.de>
Signed-off-by: Jason Gunthorpe <jgg@nvidia.com>
Link: https://lore.kernel.org/r/7-v3-57c1502c62fd+2190-ccw_mdev_jgg@nvidia.com/
[farman: added Cc: tags]
Signed-off-by: Eric Farman <farman@linux.ibm.com>
---
 .../driver-api/vfio-mediated-device.rst       |  4 +-
 drivers/s390/cio/vfio_ccw_drv.c               |  1 -
 drivers/s390/cio/vfio_ccw_ops.c               | 26 ++++---------
 drivers/s390/cio/vfio_ccw_private.h           |  2 -
 drivers/s390/crypto/vfio_ap_ops.c             | 32 ++++------------
 drivers/s390/crypto/vfio_ap_private.h         |  2 -
 drivers/vfio/mdev/mdev_core.c                 | 11 +++++-
 drivers/vfio/mdev/mdev_private.h              |  2 +
 drivers/vfio/mdev/mdev_sysfs.c                | 37 +++++++++++++++++++
 include/linux/mdev.h                          |  2 +
 samples/vfio-mdev/mdpy.c                      | 22 +++--------
 11 files changed, 76 insertions(+), 65 deletions(-)

Comments

Anthony Krowiak June 3, 2022, 3:02 p.m. UTC | #1
Reviewed-by: Tony Krowiak <akrowiak@linux.ibm.com>

On 6/2/22 1:19 PM, Eric Farman wrote:
> From: Jason Gunthorpe <jgg@nvidia.com>
>
> Many of the mdev drivers use a simple counter for keeping track of the
> available instances. Move this code to the core code and store the counter
> in the mdev_type. Implement it using correct locking, fixing mdpy.
>
> Drivers provide a get_available() callback to set the number of available
> instances for their mtypes which is fixed at registration time. The core
> provides a standard sysfs attribute to return the available_instances.
>
> Cc: Kirti Wankhede <kwankhede@nvidia.com>
> Cc: Jonathan Corbet <corbet@lwn.net>
> Cc: Tony Krowiak <akrowiak@linux.ibm.com>
> Cc: Jason Herne <jjherne@linux.ibm.com>
> Reviewed-by: Christoph Hellwig <hch@lst.de>
> Signed-off-by: Jason Gunthorpe <jgg@nvidia.com>
> Link: https://lore.kernel.org/r/7-v3-57c1502c62fd+2190-ccw_mdev_jgg@nvidia.com/
> [farman: added Cc: tags]
> Signed-off-by: Eric Farman <farman@linux.ibm.com>
> ---
>   .../driver-api/vfio-mediated-device.rst       |  4 +-
>   drivers/s390/cio/vfio_ccw_drv.c               |  1 -
>   drivers/s390/cio/vfio_ccw_ops.c               | 26 ++++---------
>   drivers/s390/cio/vfio_ccw_private.h           |  2 -
>   drivers/s390/crypto/vfio_ap_ops.c             | 32 ++++------------
>   drivers/s390/crypto/vfio_ap_private.h         |  2 -
>   drivers/vfio/mdev/mdev_core.c                 | 11 +++++-
>   drivers/vfio/mdev/mdev_private.h              |  2 +
>   drivers/vfio/mdev/mdev_sysfs.c                | 37 +++++++++++++++++++
>   include/linux/mdev.h                          |  2 +
>   samples/vfio-mdev/mdpy.c                      | 22 +++--------
>   11 files changed, 76 insertions(+), 65 deletions(-)
>
> diff --git a/Documentation/driver-api/vfio-mediated-device.rst b/Documentation/driver-api/vfio-mediated-device.rst
> index f410a1cd98bb..a4f7f1362fa8 100644
> --- a/Documentation/driver-api/vfio-mediated-device.rst
> +++ b/Documentation/driver-api/vfio-mediated-device.rst
> @@ -106,6 +106,7 @@ structure to represent a mediated device's driver::
>   	     int  (*probe)  (struct mdev_device *dev);
>   	     void (*remove) (struct mdev_device *dev);
>   	     struct device_driver    driver;
> +	     unsigned int (*get_available)(struct mdev_type *mtype);
>        };
>   
>   A mediated bus driver for mdev should use this structure in the function calls
> @@ -232,7 +233,8 @@ Directories and files under the sysfs for Each Physical Device
>   * available_instances
>   
>     This attribute should show the number of devices of type <type-id> that can be
> -  created.
> +  created. Drivers can supply a get_available() function pointer to have the
> +  core code create and maintain this sysfs automatically.
>   
>   * [device]
>   
> diff --git a/drivers/s390/cio/vfio_ccw_drv.c b/drivers/s390/cio/vfio_ccw_drv.c
> index e4b285953a45..6dca35f3ceba 100644
> --- a/drivers/s390/cio/vfio_ccw_drv.c
> +++ b/drivers/s390/cio/vfio_ccw_drv.c
> @@ -147,7 +147,6 @@ static struct vfio_ccw_private *vfio_ccw_alloc_private(struct subchannel *sch)
>   	INIT_LIST_HEAD(&private->crw);
>   	INIT_WORK(&private->io_work, vfio_ccw_sch_io_todo);
>   	INIT_WORK(&private->crw_work, vfio_ccw_crw_todo);
> -	atomic_set(&private->avail, 1);
>   
>   	private->cp.guest_cp = kcalloc(CCWCHAIN_LEN_MAX, sizeof(struct ccw1),
>   				       GFP_KERNEL);
> diff --git a/drivers/s390/cio/vfio_ccw_ops.c b/drivers/s390/cio/vfio_ccw_ops.c
> index 6793c8b3c58b..60a4855e8ecb 100644
> --- a/drivers/s390/cio/vfio_ccw_ops.c
> +++ b/drivers/s390/cio/vfio_ccw_ops.c
> @@ -66,20 +66,9 @@ static ssize_t name_show(struct mdev_type *mtype,
>   }
>   static MDEV_TYPE_ATTR_RO(name);
>   
> -static ssize_t available_instances_show(struct mdev_type *mtype,
> -					struct mdev_type_attribute *attr,
> -					char *buf)
> -{
> -	struct vfio_ccw_private *private =
> -		dev_get_drvdata(mtype_get_parent_dev(mtype));
> -
> -	return sprintf(buf, "%d\n", atomic_read(&private->avail));
> -}
> -static MDEV_TYPE_ATTR_RO(available_instances);
>   
>   static struct attribute *mdev_types_attrs[] = {
>   	&mdev_type_attr_name.attr,
> -	&mdev_type_attr_available_instances.attr,
>   	NULL,
>   };
>   
> @@ -101,9 +90,6 @@ static int vfio_ccw_mdev_probe(struct mdev_device *mdev)
>   	if (private->state == VFIO_CCW_STATE_NOT_OPER)
>   		return -ENODEV;
>   
> -	if (atomic_dec_if_positive(&private->avail) < 0)
> -		return -EPERM;
> -
>   	memset(&private->vdev, 0, sizeof(private->vdev));
>   	vfio_init_group_dev(&private->vdev, &mdev->dev,
>   			    &vfio_ccw_dev_ops);
> @@ -115,13 +101,12 @@ static int vfio_ccw_mdev_probe(struct mdev_device *mdev)
>   
>   	ret = vfio_register_emulated_iommu_dev(&private->vdev);
>   	if (ret)
> -		goto err_atomic;
> +		goto err_init;
>   	dev_set_drvdata(&mdev->dev, private);
>   	return 0;
>   
> -err_atomic:
> +err_init:
>   	vfio_uninit_group_dev(&private->vdev);
> -	atomic_inc(&private->avail);
>   	return ret;
>   }
>   
> @@ -143,7 +128,6 @@ static void vfio_ccw_mdev_remove(struct mdev_device *mdev)
>   	vfio_ccw_fsm_event(private, VFIO_CCW_EVENT_CLOSE);
>   
>   	vfio_uninit_group_dev(&private->vdev);
> -	atomic_inc(&private->avail);
>   }
>   
>   static int vfio_ccw_mdev_open_device(struct vfio_device *vdev)
> @@ -614,6 +598,11 @@ static void vfio_ccw_mdev_request(struct vfio_device *vdev, unsigned int count)
>   	}
>   }
>   
> +static unsigned int vfio_ccw_get_available(struct mdev_type *mtype)
> +{
> +	return 1;
> +}
> +
>   static const struct vfio_device_ops vfio_ccw_dev_ops = {
>   	.open_device = vfio_ccw_mdev_open_device,
>   	.close_device = vfio_ccw_mdev_close_device,
> @@ -631,6 +620,7 @@ struct mdev_driver vfio_ccw_mdev_driver = {
>   	},
>   	.probe = vfio_ccw_mdev_probe,
>   	.remove = vfio_ccw_mdev_remove,
> +	.get_available = vfio_ccw_get_available,
>   };
>   
>   const struct mdev_parent_ops vfio_ccw_mdev_ops = {
> diff --git a/drivers/s390/cio/vfio_ccw_private.h b/drivers/s390/cio/vfio_ccw_private.h
> index 02a4a5edd00c..e568fd6bcf2a 100644
> --- a/drivers/s390/cio/vfio_ccw_private.h
> +++ b/drivers/s390/cio/vfio_ccw_private.h
> @@ -72,7 +72,6 @@ struct vfio_ccw_crw {
>    * @sch: pointer to the subchannel
>    * @state: internal state of the device
>    * @completion: synchronization helper of the I/O completion
> - * @avail: available for creating a mediated device
>    * @nb: notifier for vfio events
>    * @io_region: MMIO region to input/output I/O arguments/results
>    * @io_mutex: protect against concurrent update of I/O regions
> @@ -95,7 +94,6 @@ struct vfio_ccw_private {
>   	struct subchannel	*sch;
>   	int			state;
>   	struct completion	*completion;
> -	atomic_t		avail;
>   	struct notifier_block	nb;
>   	struct ccw_io_region	*io_region;
>   	struct mutex		io_mutex;
> diff --git a/drivers/s390/crypto/vfio_ap_ops.c b/drivers/s390/crypto/vfio_ap_ops.c
> index 838b1a3eac8a..4c62ba9c72d8 100644
> --- a/drivers/s390/crypto/vfio_ap_ops.c
> +++ b/drivers/s390/crypto/vfio_ap_ops.c
> @@ -462,14 +462,9 @@ static int vfio_ap_mdev_probe(struct mdev_device *mdev)
>   	struct ap_matrix_mdev *matrix_mdev;
>   	int ret;
>   
> -	if ((atomic_dec_if_positive(&matrix_dev->available_instances) < 0))
> -		return -EPERM;
> -
>   	matrix_mdev = kzalloc(sizeof(*matrix_mdev), GFP_KERNEL);
> -	if (!matrix_mdev) {
> -		ret = -ENOMEM;
> -		goto err_dec_available;
> -	}
> +	if (!matrix_mdev)
> +		return -ENOMEM;
>   	vfio_init_group_dev(&matrix_mdev->vdev, &mdev->dev,
>   			    &vfio_ap_matrix_dev_ops);
>   
> @@ -492,8 +487,6 @@ static int vfio_ap_mdev_probe(struct mdev_device *mdev)
>   	mutex_unlock(&matrix_dev->lock);
>   	vfio_uninit_group_dev(&matrix_mdev->vdev);
>   	kfree(matrix_mdev);
> -err_dec_available:
> -	atomic_inc(&matrix_dev->available_instances);
>   	return ret;
>   }
>   
> @@ -509,7 +502,6 @@ static void vfio_ap_mdev_remove(struct mdev_device *mdev)
>   	mutex_unlock(&matrix_dev->lock);
>   	vfio_uninit_group_dev(&matrix_mdev->vdev);
>   	kfree(matrix_mdev);
> -	atomic_inc(&matrix_dev->available_instances);
>   }
>   
>   static ssize_t name_show(struct mdev_type *mtype,
> @@ -520,20 +512,8 @@ static ssize_t name_show(struct mdev_type *mtype,
>   
>   static MDEV_TYPE_ATTR_RO(name);
>   
> -static ssize_t available_instances_show(struct mdev_type *mtype,
> -					struct mdev_type_attribute *attr,
> -					char *buf)
> -{
> -	return sprintf(buf, "%d\n",
> -		       atomic_read(&matrix_dev->available_instances));
> -}
> -
> -static MDEV_TYPE_ATTR_RO(available_instances);
> -
> -
>   static struct attribute *vfio_ap_mdev_type_attrs[] = {
>   	&mdev_type_attr_name.attr,
> -	&mdev_type_attr_available_instances.attr,
>   	NULL,
>   };
>   
> @@ -1473,6 +1453,11 @@ static ssize_t vfio_ap_mdev_ioctl(struct vfio_device *vdev,
>   	return ret;
>   }
>   
> +static unsigned int vfio_ap_mdev_get_available(struct mdev_type *mtype)
> +{
> +	return MAX_ZDEV_ENTRIES_EXT;
> +}
> +
>   static const struct vfio_device_ops vfio_ap_matrix_dev_ops = {
>   	.open_device = vfio_ap_mdev_open_device,
>   	.close_device = vfio_ap_mdev_close_device,
> @@ -1488,6 +1473,7 @@ static struct mdev_driver vfio_ap_matrix_driver = {
>   	},
>   	.probe = vfio_ap_mdev_probe,
>   	.remove = vfio_ap_mdev_remove,
> +	.get_available = vfio_ap_mdev_get_available,
>   };
>   
>   static const struct mdev_parent_ops vfio_ap_matrix_ops = {
> @@ -1501,8 +1487,6 @@ int vfio_ap_mdev_register(void)
>   {
>   	int ret;
>   
> -	atomic_set(&matrix_dev->available_instances, MAX_ZDEV_ENTRIES_EXT);
> -
>   	ret = mdev_register_driver(&vfio_ap_matrix_driver);
>   	if (ret)
>   		return ret;
> diff --git a/drivers/s390/crypto/vfio_ap_private.h b/drivers/s390/crypto/vfio_ap_private.h
> index 648fcaf8104a..5c0cbf03074b 100644
> --- a/drivers/s390/crypto/vfio_ap_private.h
> +++ b/drivers/s390/crypto/vfio_ap_private.h
> @@ -29,7 +29,6 @@
>    * struct ap_matrix_dev - Contains the data for the matrix device.
>    *
>    * @device:	generic device structure associated with the AP matrix device
> - * @available_instances: number of mediated matrix devices that can be created
>    * @info:	the struct containing the output from the PQAP(QCI) instruction
>    * @mdev_list:	the list of mediated matrix devices created
>    * @lock:	mutex for locking the AP matrix device. This lock will be
> @@ -41,7 +40,6 @@
>    */
>   struct ap_matrix_dev {
>   	struct device device;
> -	atomic_t available_instances;
>   	struct ap_config_info info;
>   	struct list_head mdev_list;
>   	struct mutex lock;
> diff --git a/drivers/vfio/mdev/mdev_core.c b/drivers/vfio/mdev/mdev_core.c
> index c3018e8e6d32..bb27ca0db948 100644
> --- a/drivers/vfio/mdev/mdev_core.c
> +++ b/drivers/vfio/mdev/mdev_core.c
> @@ -25,7 +25,7 @@ static DEFINE_MUTEX(parent_list_lock);
>   static struct class_compat *mdev_bus_compat_class;
>   
>   static LIST_HEAD(mdev_list);
> -static DEFINE_MUTEX(mdev_list_lock);
> +DEFINE_MUTEX(mdev_list_lock);
>   
>   struct device *mdev_parent_dev(struct mdev_device *mdev)
>   {
> @@ -245,6 +245,7 @@ static void mdev_device_release(struct device *dev)
>   
>   	mutex_lock(&mdev_list_lock);
>   	list_del(&mdev->next);
> +	mdev->type->available++;
>   	mutex_unlock(&mdev_list_lock);
>   
>   	dev_dbg(&mdev->dev, "MDEV: destroying\n");
> @@ -268,6 +269,14 @@ int mdev_device_create(struct mdev_type *type, const guid_t *uuid)
>   		}
>   	}
>   
> +	if (drv && drv->get_available) {
> +		if (!type->available) {
> +			mutex_unlock(&mdev_list_lock);
> +			return -EUSERS;
> +		}
> +		type->available--;
> +	}
> +
>   	mdev = kzalloc(sizeof(*mdev), GFP_KERNEL);
>   	if (!mdev) {
>   		mutex_unlock(&mdev_list_lock);
> diff --git a/drivers/vfio/mdev/mdev_private.h b/drivers/vfio/mdev/mdev_private.h
> index afbad7b0a14a..83586b070233 100644
> --- a/drivers/vfio/mdev/mdev_private.h
> +++ b/drivers/vfio/mdev/mdev_private.h
> @@ -29,6 +29,7 @@ struct mdev_type {
>   	struct kobject *devices_kobj;
>   	struct mdev_parent *parent;
>   	struct list_head next;
> +	unsigned int available;
>   	unsigned int type_group_id;
>   };
>   
> @@ -38,6 +39,7 @@ struct mdev_type {
>   	container_of(_kobj, struct mdev_type, kobj)
>   
>   extern struct mdev_driver vfio_mdev_driver;
> +extern struct mutex mdev_list_lock;
>   
>   int  parent_create_sysfs_files(struct mdev_parent *parent);
>   void parent_remove_sysfs_files(struct mdev_parent *parent);
> diff --git a/drivers/vfio/mdev/mdev_sysfs.c b/drivers/vfio/mdev/mdev_sysfs.c
> index d4b99440d19e..b3129dfc27ef 100644
> --- a/drivers/vfio/mdev/mdev_sysfs.c
> +++ b/drivers/vfio/mdev/mdev_sysfs.c
> @@ -93,8 +93,41 @@ static struct attribute_group mdev_type_std_group = {
>   	.attrs = mdev_types_std_attrs,
>   };
>   
> +/* mdev_type attribute used by drivers that have an get_available() op */
> +static ssize_t available_instances_show(struct mdev_type *mtype,
> +					struct mdev_type_attribute *attr,
> +					char *buf)
> +{
> +	unsigned int available;
> +
> +	mutex_lock(&mdev_list_lock);
> +	available = mtype->available;
> +	mutex_unlock(&mdev_list_lock);
> +
> +	return sysfs_emit(buf, "%u\n", available);
> +}
> +static MDEV_TYPE_ATTR_RO(available_instances);
> +static umode_t available_instances_is_visible(struct kobject *kobj,
> +					      struct attribute *attr, int n)
> +{
> +	struct mdev_type *type = to_mdev_type(kobj);
> +
> +	if (!type->parent->ops->device_driver->get_available)
> +		return 0;
> +	return attr->mode;
> +}
> +static struct attribute *mdev_types_name_attrs[] = {
> +	&mdev_type_attr_available_instances.attr,
> +	NULL,
> +};
> +static struct attribute_group mdev_type_available_instances_group = {
> +	.attrs = mdev_types_name_attrs,
> +	.is_visible = available_instances_is_visible,
> +};
> +
>   static const struct attribute_group *mdev_type_groups[] = {
>   	&mdev_type_std_group,
> +	&mdev_type_available_instances_group,
>   	NULL,
>   };
>   
> @@ -136,6 +169,10 @@ static struct mdev_type *add_mdev_supported_type(struct mdev_parent *parent,
>   	mdev_get_parent(parent);
>   	type->type_group_id = type_group_id;
>   
> +	if (parent->ops->device_driver->get_available)
> +		type->available =
> +			parent->ops->device_driver->get_available(type);
> +
>   	ret = kobject_init_and_add(&type->kobj, &mdev_type_ktype, NULL,
>   				   "%s-%s", dev_driver_string(parent->dev),
>   				   group->name);
> diff --git a/include/linux/mdev.h b/include/linux/mdev.h
> index 14655215417b..0ce1bb3dabd0 100644
> --- a/include/linux/mdev.h
> +++ b/include/linux/mdev.h
> @@ -120,12 +120,14 @@ struct mdev_type_attribute {
>    * @probe: called when new device created
>    * @remove: called when device removed
>    * @driver: device driver structure
> + * @get_available: Return the max number of instances that can be created
>    *
>    **/
>   struct mdev_driver {
>   	int (*probe)(struct mdev_device *dev);
>   	void (*remove)(struct mdev_device *dev);
>   	struct device_driver driver;
> +	unsigned int (*get_available)(struct mdev_type *mtype);
>   };
>   
>   static inline void *mdev_get_drvdata(struct mdev_device *mdev)
> diff --git a/samples/vfio-mdev/mdpy.c b/samples/vfio-mdev/mdpy.c
> index 402a7ebe6563..d7da6ed35657 100644
> --- a/samples/vfio-mdev/mdpy.c
> +++ b/samples/vfio-mdev/mdpy.c
> @@ -84,7 +84,6 @@ static dev_t		mdpy_devt;
>   static struct class	*mdpy_class;
>   static struct cdev	mdpy_cdev;
>   static struct device	mdpy_dev;
> -static u32		mdpy_count;
>   static const struct vfio_device_ops mdpy_dev_ops;
>   
>   /* State of each mdev device */
> @@ -225,9 +224,6 @@ static int mdpy_probe(struct mdev_device *mdev)
>   	u32 fbsize;
>   	int ret;
>   
> -	if (mdpy_count >= max_devices)
> -		return -ENOMEM;
> -
>   	mdev_state = kzalloc(sizeof(struct mdev_state), GFP_KERNEL);
>   	if (mdev_state == NULL)
>   		return -ENOMEM;
> @@ -256,8 +252,6 @@ static int mdpy_probe(struct mdev_device *mdev)
>   	mdpy_create_config_space(mdev_state);
>   	mdpy_reset(mdev_state);
>   
> -	mdpy_count++;
> -
>   	ret = vfio_register_emulated_iommu_dev(&mdev_state->vdev);
>   	if (ret)
>   		goto err_mem;
> @@ -284,8 +278,6 @@ static void mdpy_remove(struct mdev_device *mdev)
>   	kfree(mdev_state->vconfig);
>   	vfio_uninit_group_dev(&mdev_state->vdev);
>   	kfree(mdev_state);
> -
> -	mdpy_count--;
>   }
>   
>   static ssize_t mdpy_read(struct vfio_device *vdev, char __user *buf,
> @@ -662,18 +654,10 @@ static ssize_t description_show(struct mdev_type *mtype,
>   }
>   static MDEV_TYPE_ATTR_RO(description);
>   
> -static ssize_t available_instances_show(struct mdev_type *mtype,
> -					struct mdev_type_attribute *attr,
> -					char *buf)
> -{
> -	return sprintf(buf, "%d\n", max_devices - mdpy_count);
> -}
> -static MDEV_TYPE_ATTR_RO(available_instances);
>   
>   static struct attribute *mdev_types_attrs[] = {
>   	&mdev_type_attr_name.attr,
>   	&mdev_type_attr_description.attr,
> -	&mdev_type_attr_available_instances.attr,
>   	NULL,
>   };
>   
> @@ -706,6 +690,11 @@ static const struct vfio_device_ops mdpy_dev_ops = {
>   	.mmap = mdpy_mmap,
>   };
>   
> +static unsigned int mdpy_get_available(struct mdev_type *mtype)
> +{
> +	return max_devices;
> +}
> +
>   static struct mdev_driver mdpy_driver = {
>   	.driver = {
>   		.name = "mdpy",
> @@ -715,6 +704,7 @@ static struct mdev_driver mdpy_driver = {
>   	},
>   	.probe = mdpy_probe,
>   	.remove	= mdpy_remove,
> +	.get_available = mdpy_get_available,
>   };
>   
>   static const struct mdev_parent_ops mdev_fops = {
Kirti Wankhede June 6, 2022, 8:02 p.m. UTC | #2
On 6/2/2022 10:49 PM, Eric Farman wrote:
> From: Jason Gunthorpe <jgg@nvidia.com>
> 
> Many of the mdev drivers use a simple counter for keeping track of the
> available instances. Move this code to the core code and store the counter
> in the mdev_type. Implement it using correct locking, fixing mdpy.
> 
> Drivers provide a get_available() callback to set the number of available
> instances for their mtypes which is fixed at registration time. The core
> provides a standard sysfs attribute to return the available_instances.
> 
> Cc: Kirti Wankhede <kwankhede@nvidia.com>
> Cc: Jonathan Corbet <corbet@lwn.net>
> Cc: Tony Krowiak <akrowiak@linux.ibm.com>
> Cc: Jason Herne <jjherne@linux.ibm.com>
> Reviewed-by: Christoph Hellwig <hch@lst.de>
> Signed-off-by: Jason Gunthorpe <jgg@nvidia.com>
> Link: https://lore.kernel.org/r/7-v3-57c1502c62fd+2190-ccw_mdev_jgg@nvidia.com/
> [farman: added Cc: tags]
> Signed-off-by: Eric Farman <farman@linux.ibm.com>
> ---
>   .../driver-api/vfio-mediated-device.rst       |  4 +-
>   drivers/s390/cio/vfio_ccw_drv.c               |  1 -
>   drivers/s390/cio/vfio_ccw_ops.c               | 26 ++++---------
>   drivers/s390/cio/vfio_ccw_private.h           |  2 -
>   drivers/s390/crypto/vfio_ap_ops.c             | 32 ++++------------
>   drivers/s390/crypto/vfio_ap_private.h         |  2 -
>   drivers/vfio/mdev/mdev_core.c                 | 11 +++++-
>   drivers/vfio/mdev/mdev_private.h              |  2 +
>   drivers/vfio/mdev/mdev_sysfs.c                | 37 +++++++++++++++++++
>   include/linux/mdev.h                          |  2 +
>   samples/vfio-mdev/mdpy.c                      | 22 +++--------
>   11 files changed, 76 insertions(+), 65 deletions(-)
> 
> diff --git a/Documentation/driver-api/vfio-mediated-device.rst b/Documentation/driver-api/vfio-mediated-device.rst
> index f410a1cd98bb..a4f7f1362fa8 100644
> --- a/Documentation/driver-api/vfio-mediated-device.rst
> +++ b/Documentation/driver-api/vfio-mediated-device.rst
> @@ -106,6 +106,7 @@ structure to represent a mediated device's driver::
>   	     int  (*probe)  (struct mdev_device *dev);
>   	     void (*remove) (struct mdev_device *dev);
>   	     struct device_driver    driver;
> +	     unsigned int (*get_available)(struct mdev_type *mtype);
>        };
>

This patch conflicts with Christoph Hellwig's patch. I see 
'supported_type_groups' is not is above structure, I beleive that your 
patch is applied on top of Christoph's patch series.

but then in below part of code, 'add_mdev_supported_type' has also being 
removed in Christoph's patch. So this patch would not get applied cleanly.

Thanks,
Kirti

> +/* mdev_type attribute used by drivers that have an get_available() op */
> +static ssize_t available_instances_show(struct mdev_type *mtype,
> +					struct mdev_type_attribute *attr,
> +					char *buf)
> +{
> +	unsigned int available;
> +
> +	mutex_lock(&mdev_list_lock);
> +	available = mtype->available;
> +	mutex_unlock(&mdev_list_lock);
> +
> +	return sysfs_emit(buf, "%u\n", available);
> +}
> +static MDEV_TYPE_ATTR_RO(available_instances);
> +static umode_t available_instances_is_visible(struct kobject *kobj,
> +					      struct attribute *attr, int n)
> +{
> +	struct mdev_type *type = to_mdev_type(kobj);
> +
> +	if (!type->parent->ops->device_driver->get_available)
> +		return 0;
> +	return attr->mode;
> +}
> +static struct attribute *mdev_types_name_attrs[] = {
> +	&mdev_type_attr_available_instances.attr,
> +	NULL,
> +};
> +static struct attribute_group mdev_type_available_instances_group = {
> +	.attrs = mdev_types_name_attrs,
> +	.is_visible = available_instances_is_visible,
> +};
> +
>   static const struct attribute_group *mdev_type_groups[] = {
>   	&mdev_type_std_group,
> +	&mdev_type_available_instances_group,
>   	NULL,
>   };
>   
> @@ -136,6 +169,10 @@ static struct mdev_type *add_mdev_supported_type(struct mdev_parent *parent,
>   	mdev_get_parent(parent);
>   	type->type_group_id = type_group_id;
>   
> +	if (parent->ops->device_driver->get_available)
> +		type->available =
> +			parent->ops->device_driver->get_available(type);
> +
>   	ret = kobject_init_and_add(&type->kobj, &mdev_type_ktype, NULL,
>   				   "%s-%s", dev_driver_string(parent->dev),
>   				   group->name);
> diff --git a/include/linux/mdev.h b/include/linux/mdev.h
> index 14655215417b..0ce1bb3dabd0 100644
> --- a/include/linux/mdev.h
> +++ b/include/linux/mdev.h
> @@ -120,12 +120,14 @@ struct mdev_type_attribute {
>    * @probe: called when new device created
>    * @remove: called when device removed
>    * @driver: device driver structure
> + * @get_available: Return the max number of instances that can be created
>    *
>    **/
>   struct mdev_driver {
>   	int (*probe)(struct mdev_device *dev);
>   	void (*remove)(struct mdev_device *dev);
>   	struct device_driver driver;
> +	unsigned int (*get_available)(struct mdev_type *mtype);
>   };
>
Eric Farman June 6, 2022, 8:23 p.m. UTC | #3
On Tue, 2022-06-07 at 01:32 +0530, Kirti Wankhede wrote:
> 
> On 6/2/2022 10:49 PM, Eric Farman wrote:
> > From: Jason Gunthorpe <jgg@nvidia.com>
> > 
> > Many of the mdev drivers use a simple counter for keeping track of
> > the
> > available instances. Move this code to the core code and store the
> > counter
> > in the mdev_type. Implement it using correct locking, fixing mdpy.
> > 
> > Drivers provide a get_available() callback to set the number of
> > available
> > instances for their mtypes which is fixed at registration time. The
> > core
> > provides a standard sysfs attribute to return the
> > available_instances.
> > 
> > Cc: Kirti Wankhede <kwankhede@nvidia.com>
> > Cc: Jonathan Corbet <corbet@lwn.net>
> > Cc: Tony Krowiak <akrowiak@linux.ibm.com>
> > Cc: Jason Herne <jjherne@linux.ibm.com>
> > Reviewed-by: Christoph Hellwig <hch@lst.de>
> > Signed-off-by: Jason Gunthorpe <jgg@nvidia.com>
> > Link: 
> > https://lore.kernel.org/r/7-v3-57c1502c62fd+2190-ccw_mdev_jgg@nvidia.com/
> > [farman: added Cc: tags]
> > Signed-off-by: Eric Farman <farman@linux.ibm.com>
> > ---
> >   .../driver-api/vfio-mediated-device.rst       |  4 +-
> >   drivers/s390/cio/vfio_ccw_drv.c               |  1 -
> >   drivers/s390/cio/vfio_ccw_ops.c               | 26 ++++---------
> >   drivers/s390/cio/vfio_ccw_private.h           |  2 -
> >   drivers/s390/crypto/vfio_ap_ops.c             | 32 ++++--------
> > ----
> >   drivers/s390/crypto/vfio_ap_private.h         |  2 -
> >   drivers/vfio/mdev/mdev_core.c                 | 11 +++++-
> >   drivers/vfio/mdev/mdev_private.h              |  2 +
> >   drivers/vfio/mdev/mdev_sysfs.c                | 37
> > +++++++++++++++++++
> >   include/linux/mdev.h                          |  2 +
> >   samples/vfio-mdev/mdpy.c                      | 22 +++--------
> >   11 files changed, 76 insertions(+), 65 deletions(-)
> > 
> > diff --git a/Documentation/driver-api/vfio-mediated-device.rst
> > b/Documentation/driver-api/vfio-mediated-device.rst
> > index f410a1cd98bb..a4f7f1362fa8 100644
> > --- a/Documentation/driver-api/vfio-mediated-device.rst
> > +++ b/Documentation/driver-api/vfio-mediated-device.rst
> > @@ -106,6 +106,7 @@ structure to represent a mediated device's
> > driver::
> >   	     int  (*probe)  (struct mdev_device *dev);
> >   	     void (*remove) (struct mdev_device *dev);
> >   	     struct device_driver    driver;
> > +	     unsigned int (*get_available)(struct mdev_type *mtype);
> >        };
> > 
> 
> This patch conflicts with Christoph Hellwig's patch. I see 
> 'supported_type_groups' is not is above structure, I beleive that
> your 
> patch is applied on top of Christoph's patch series.
> 
> but then in below part of code, 'add_mdev_supported_type' has also
> being 
> removed in Christoph's patch. So this patch would not get applied
> cleanly.

Apologies. This series was fit to 5.18 as the merge window progressed.
Both this patch and the previous one have to adjust to the removal of
mdev_parent_ops that came about from 

commit 6b42f491e17ce13f5ff7f2d1f49c73a0f4c47b20
Author: Jason Gunthorpe <jgg@ziepe.ca>
Date:   Mon Apr 11 16:14:01 2022 +0200

    vfio/mdev: Remove mdev_parent_ops

I have this rebased for v2.

Eric

> 
> Thanks,
> Kirti
> 
> > +/* mdev_type attribute used by drivers that have an
> > get_available() op */
> > +static ssize_t available_instances_show(struct mdev_type *mtype,
> > +					struct mdev_type_attribute
> > *attr,
> > +					char *buf)
> > +{
> > +	unsigned int available;
> > +
> > +	mutex_lock(&mdev_list_lock);
> > +	available = mtype->available;
> > +	mutex_unlock(&mdev_list_lock);
> > +
> > +	return sysfs_emit(buf, "%u\n", available);
> > +}
> > +static MDEV_TYPE_ATTR_RO(available_instances);
> > +static umode_t available_instances_is_visible(struct kobject
> > *kobj,
> > +					      struct attribute *attr,
> > int n)
> > +{
> > +	struct mdev_type *type = to_mdev_type(kobj);
> > +
> > +	if (!type->parent->ops->device_driver->get_available)
> > +		return 0;
> > +	return attr->mode;
> > +}
> > +static struct attribute *mdev_types_name_attrs[] = {
> > +	&mdev_type_attr_available_instances.attr,
> > +	NULL,
> > +};
> > +static struct attribute_group mdev_type_available_instances_group
> > = {
> > +	.attrs = mdev_types_name_attrs,
> > +	.is_visible = available_instances_is_visible,
> > +};
> > +
> >   static const struct attribute_group *mdev_type_groups[] = {
> >   	&mdev_type_std_group,
> > +	&mdev_type_available_instances_group,
> >   	NULL,
> >   };
> >   
> > @@ -136,6 +169,10 @@ static struct mdev_type
> > *add_mdev_supported_type(struct mdev_parent *parent,
> >   	mdev_get_parent(parent);
> >   	type->type_group_id = type_group_id;
> >   
> > +	if (parent->ops->device_driver->get_available)
> > +		type->available =
> > +			parent->ops->device_driver-
> > >get_available(type);
> > +
> >   	ret = kobject_init_and_add(&type->kobj, &mdev_type_ktype, NULL,
> >   				   "%s-%s", dev_driver_string(parent-
> > >dev),
> >   				   group->name);
> > diff --git a/include/linux/mdev.h b/include/linux/mdev.h
> > index 14655215417b..0ce1bb3dabd0 100644
> > --- a/include/linux/mdev.h
> > +++ b/include/linux/mdev.h
> > @@ -120,12 +120,14 @@ struct mdev_type_attribute {
> >    * @probe: called when new device created
> >    * @remove: called when device removed
> >    * @driver: device driver structure
> > + * @get_available: Return the max number of instances that can be
> > created
> >    *
> >    **/
> >   struct mdev_driver {
> >   	int (*probe)(struct mdev_device *dev);
> >   	void (*remove)(struct mdev_device *dev);
> >   	struct device_driver driver;
> > +	unsigned int (*get_available)(struct mdev_type *mtype);
> >   };
> >
Matthew Rosato June 6, 2022, 8:37 p.m. UTC | #4
On 6/6/22 4:23 PM, Eric Farman wrote:
> On Tue, 2022-06-07 at 01:32 +0530, Kirti Wankhede wrote:
>>
>> On 6/2/2022 10:49 PM, Eric Farman wrote:
>>> From: Jason Gunthorpe <jgg@nvidia.com>
>>>
>>> Many of the mdev drivers use a simple counter for keeping track of
>>> the
>>> available instances. Move this code to the core code and store the
>>> counter
>>> in the mdev_type. Implement it using correct locking, fixing mdpy.
>>>
>>> Drivers provide a get_available() callback to set the number of
>>> available
>>> instances for their mtypes which is fixed at registration time. The
>>> core
>>> provides a standard sysfs attribute to return the
>>> available_instances.
>>>
>>> Cc: Kirti Wankhede <kwankhede@nvidia.com>
>>> Cc: Jonathan Corbet <corbet@lwn.net>
>>> Cc: Tony Krowiak <akrowiak@linux.ibm.com>
>>> Cc: Jason Herne <jjherne@linux.ibm.com>
>>> Reviewed-by: Christoph Hellwig <hch@lst.de>
>>> Signed-off-by: Jason Gunthorpe <jgg@nvidia.com>
>>> Link:
>>> https://lore.kernel.org/r/7-v3-57c1502c62fd+2190-ccw_mdev_jgg@nvidia.com/
>>> [farman: added Cc: tags]
>>> Signed-off-by: Eric Farman <farman@linux.ibm.com>
>>> ---
>>>    .../driver-api/vfio-mediated-device.rst       |  4 +-
>>>    drivers/s390/cio/vfio_ccw_drv.c               |  1 -
>>>    drivers/s390/cio/vfio_ccw_ops.c               | 26 ++++---------
>>>    drivers/s390/cio/vfio_ccw_private.h           |  2 -
>>>    drivers/s390/crypto/vfio_ap_ops.c             | 32 ++++--------
>>> ----
>>>    drivers/s390/crypto/vfio_ap_private.h         |  2 -
>>>    drivers/vfio/mdev/mdev_core.c                 | 11 +++++-
>>>    drivers/vfio/mdev/mdev_private.h              |  2 +
>>>    drivers/vfio/mdev/mdev_sysfs.c                | 37
>>> +++++++++++++++++++
>>>    include/linux/mdev.h                          |  2 +
>>>    samples/vfio-mdev/mdpy.c                      | 22 +++--------
>>>    11 files changed, 76 insertions(+), 65 deletions(-)
>>>
>>> diff --git a/Documentation/driver-api/vfio-mediated-device.rst
>>> b/Documentation/driver-api/vfio-mediated-device.rst
>>> index f410a1cd98bb..a4f7f1362fa8 100644
>>> --- a/Documentation/driver-api/vfio-mediated-device.rst
>>> +++ b/Documentation/driver-api/vfio-mediated-device.rst
>>> @@ -106,6 +106,7 @@ structure to represent a mediated device's
>>> driver::
>>>    	     int  (*probe)  (struct mdev_device *dev);
>>>    	     void (*remove) (struct mdev_device *dev);
>>>    	     struct device_driver    driver;
>>> +	     unsigned int (*get_available)(struct mdev_type *mtype);
>>>         };
>>>
>>
>> This patch conflicts with Christoph Hellwig's patch. I see
>> 'supported_type_groups' is not is above structure, I beleive that
>> your
>> patch is applied on top of Christoph's patch series.
>>
>> but then in below part of code, 'add_mdev_supported_type' has also
>> being
>> removed in Christoph's patch. So this patch would not get applied
>> cleanly.
> 
> Apologies. This series was fit to 5.18 as the merge window progressed.
> Both this patch and the previous one have to adjust to the removal of
> mdev_parent_ops that came about from
> 
> commit 6b42f491e17ce13f5ff7f2d1f49c73a0f4c47b20
> Author: Jason Gunthorpe <jgg@ziepe.ca>
> Date:   Mon Apr 11 16:14:01 2022 +0200
> 
>      vfio/mdev: Remove mdev_parent_ops
> 
> I have this rebased for v2.
> 
> Eric

Thanks Eric -- FYI, I'm planning to review the entire series but will 
likely hold off until v2 on a number of patches at this point as there 
are a few other collisions with 5.19
Tian, Kevin June 10, 2022, 7:43 a.m. UTC | #5
> From: Eric Farman <farman@linux.ibm.com>
> Sent: Tuesday, June 7, 2022 4:24 AM
> 
> On Tue, 2022-06-07 at 01:32 +0530, Kirti Wankhede wrote:
> >
> > On 6/2/2022 10:49 PM, Eric Farman wrote:
> > > From: Jason Gunthorpe <jgg@nvidia.com>
> > > @@ -106,6 +106,7 @@ structure to represent a mediated device's
> > > driver::
> > >   	     int  (*probe)  (struct mdev_device *dev);
> > >   	     void (*remove) (struct mdev_device *dev);
> > >   	     struct device_driver    driver;
> > > +	     unsigned int (*get_available)(struct mdev_type *mtype);
> > >        };
> > >
> >
> > This patch conflicts with Christoph Hellwig's patch. I see
> > 'supported_type_groups' is not is above structure, I beleive that
> > your
> > patch is applied on top of Christoph's patch series.
> >
> > but then in below part of code, 'add_mdev_supported_type' has also
> > being
> > removed in Christoph's patch. So this patch would not get applied
> > cleanly.
> 
> Apologies. This series was fit to 5.18 as the merge window progressed.
> Both this patch and the previous one have to adjust to the removal of
> mdev_parent_ops that came about from
> 
> commit 6b42f491e17ce13f5ff7f2d1f49c73a0f4c47b20
> Author: Jason Gunthorpe <jgg@ziepe.ca>
> Date:   Mon Apr 11 16:14:01 2022 +0200
> 
>     vfio/mdev: Remove mdev_parent_ops
> 
> I have this rebased for v2.
> 

btw with those latest changes [1] we don't need .get_available() then,
as mdev type is now added by mdev driver one-by-one then the
available instance can be provided directly in that path.

[1] https://lore.kernel.org/all/20220607055653.GA8848@lst.de/T/#m82caa423d728bce0729b8c9e8d8aff76250ec9ac
Christoph Hellwig June 13, 2022, 6:46 a.m. UTC | #6
On Fri, Jun 10, 2022 at 07:43:46AM +0000, Tian, Kevin wrote:
> btw with those latest changes [1] we don't need .get_available() then,
> as mdev type is now added by mdev driver one-by-one then the
> available instance can be provided directly in that path.

Yes, we can probably add a helper to add the vailable attibrute, which
takes the number of instances.  Is it ok if I just add a version of this
patch and the device_api one to my series, and we rebase this series
on top of it?  I'll try to get out a new version ASAP.
Eric Farman June 13, 2022, 2:08 p.m. UTC | #7
On Mon, 2022-06-13 at 08:46 +0200, Christoph Hellwig wrote:
> On Fri, Jun 10, 2022 at 07:43:46AM +0000, Tian, Kevin wrote:
> > btw with those latest changes [1] we don't need .get_available()
> > then,
> > as mdev type is now added by mdev driver one-by-one then the
> > available instance can be provided directly in that path.
> 
> Yes, we can probably add a helper to add the vailable attibrute,
> which
> takes the number of instances.  Is it ok if I just add a version of
> this
> patch and the device_api one to my series, and we rebase this series
> on top of it?  I'll try to get out a new version ASAP.

That's fine with me. Thanks!

Eric
diff mbox series

Patch

diff --git a/Documentation/driver-api/vfio-mediated-device.rst b/Documentation/driver-api/vfio-mediated-device.rst
index f410a1cd98bb..a4f7f1362fa8 100644
--- a/Documentation/driver-api/vfio-mediated-device.rst
+++ b/Documentation/driver-api/vfio-mediated-device.rst
@@ -106,6 +106,7 @@  structure to represent a mediated device's driver::
 	     int  (*probe)  (struct mdev_device *dev);
 	     void (*remove) (struct mdev_device *dev);
 	     struct device_driver    driver;
+	     unsigned int (*get_available)(struct mdev_type *mtype);
      };
 
 A mediated bus driver for mdev should use this structure in the function calls
@@ -232,7 +233,8 @@  Directories and files under the sysfs for Each Physical Device
 * available_instances
 
   This attribute should show the number of devices of type <type-id> that can be
-  created.
+  created. Drivers can supply a get_available() function pointer to have the
+  core code create and maintain this sysfs automatically.
 
 * [device]
 
diff --git a/drivers/s390/cio/vfio_ccw_drv.c b/drivers/s390/cio/vfio_ccw_drv.c
index e4b285953a45..6dca35f3ceba 100644
--- a/drivers/s390/cio/vfio_ccw_drv.c
+++ b/drivers/s390/cio/vfio_ccw_drv.c
@@ -147,7 +147,6 @@  static struct vfio_ccw_private *vfio_ccw_alloc_private(struct subchannel *sch)
 	INIT_LIST_HEAD(&private->crw);
 	INIT_WORK(&private->io_work, vfio_ccw_sch_io_todo);
 	INIT_WORK(&private->crw_work, vfio_ccw_crw_todo);
-	atomic_set(&private->avail, 1);
 
 	private->cp.guest_cp = kcalloc(CCWCHAIN_LEN_MAX, sizeof(struct ccw1),
 				       GFP_KERNEL);
diff --git a/drivers/s390/cio/vfio_ccw_ops.c b/drivers/s390/cio/vfio_ccw_ops.c
index 6793c8b3c58b..60a4855e8ecb 100644
--- a/drivers/s390/cio/vfio_ccw_ops.c
+++ b/drivers/s390/cio/vfio_ccw_ops.c
@@ -66,20 +66,9 @@  static ssize_t name_show(struct mdev_type *mtype,
 }
 static MDEV_TYPE_ATTR_RO(name);
 
-static ssize_t available_instances_show(struct mdev_type *mtype,
-					struct mdev_type_attribute *attr,
-					char *buf)
-{
-	struct vfio_ccw_private *private =
-		dev_get_drvdata(mtype_get_parent_dev(mtype));
-
-	return sprintf(buf, "%d\n", atomic_read(&private->avail));
-}
-static MDEV_TYPE_ATTR_RO(available_instances);
 
 static struct attribute *mdev_types_attrs[] = {
 	&mdev_type_attr_name.attr,
-	&mdev_type_attr_available_instances.attr,
 	NULL,
 };
 
@@ -101,9 +90,6 @@  static int vfio_ccw_mdev_probe(struct mdev_device *mdev)
 	if (private->state == VFIO_CCW_STATE_NOT_OPER)
 		return -ENODEV;
 
-	if (atomic_dec_if_positive(&private->avail) < 0)
-		return -EPERM;
-
 	memset(&private->vdev, 0, sizeof(private->vdev));
 	vfio_init_group_dev(&private->vdev, &mdev->dev,
 			    &vfio_ccw_dev_ops);
@@ -115,13 +101,12 @@  static int vfio_ccw_mdev_probe(struct mdev_device *mdev)
 
 	ret = vfio_register_emulated_iommu_dev(&private->vdev);
 	if (ret)
-		goto err_atomic;
+		goto err_init;
 	dev_set_drvdata(&mdev->dev, private);
 	return 0;
 
-err_atomic:
+err_init:
 	vfio_uninit_group_dev(&private->vdev);
-	atomic_inc(&private->avail);
 	return ret;
 }
 
@@ -143,7 +128,6 @@  static void vfio_ccw_mdev_remove(struct mdev_device *mdev)
 	vfio_ccw_fsm_event(private, VFIO_CCW_EVENT_CLOSE);
 
 	vfio_uninit_group_dev(&private->vdev);
-	atomic_inc(&private->avail);
 }
 
 static int vfio_ccw_mdev_open_device(struct vfio_device *vdev)
@@ -614,6 +598,11 @@  static void vfio_ccw_mdev_request(struct vfio_device *vdev, unsigned int count)
 	}
 }
 
+static unsigned int vfio_ccw_get_available(struct mdev_type *mtype)
+{
+	return 1;
+}
+
 static const struct vfio_device_ops vfio_ccw_dev_ops = {
 	.open_device = vfio_ccw_mdev_open_device,
 	.close_device = vfio_ccw_mdev_close_device,
@@ -631,6 +620,7 @@  struct mdev_driver vfio_ccw_mdev_driver = {
 	},
 	.probe = vfio_ccw_mdev_probe,
 	.remove = vfio_ccw_mdev_remove,
+	.get_available = vfio_ccw_get_available,
 };
 
 const struct mdev_parent_ops vfio_ccw_mdev_ops = {
diff --git a/drivers/s390/cio/vfio_ccw_private.h b/drivers/s390/cio/vfio_ccw_private.h
index 02a4a5edd00c..e568fd6bcf2a 100644
--- a/drivers/s390/cio/vfio_ccw_private.h
+++ b/drivers/s390/cio/vfio_ccw_private.h
@@ -72,7 +72,6 @@  struct vfio_ccw_crw {
  * @sch: pointer to the subchannel
  * @state: internal state of the device
  * @completion: synchronization helper of the I/O completion
- * @avail: available for creating a mediated device
  * @nb: notifier for vfio events
  * @io_region: MMIO region to input/output I/O arguments/results
  * @io_mutex: protect against concurrent update of I/O regions
@@ -95,7 +94,6 @@  struct vfio_ccw_private {
 	struct subchannel	*sch;
 	int			state;
 	struct completion	*completion;
-	atomic_t		avail;
 	struct notifier_block	nb;
 	struct ccw_io_region	*io_region;
 	struct mutex		io_mutex;
diff --git a/drivers/s390/crypto/vfio_ap_ops.c b/drivers/s390/crypto/vfio_ap_ops.c
index 838b1a3eac8a..4c62ba9c72d8 100644
--- a/drivers/s390/crypto/vfio_ap_ops.c
+++ b/drivers/s390/crypto/vfio_ap_ops.c
@@ -462,14 +462,9 @@  static int vfio_ap_mdev_probe(struct mdev_device *mdev)
 	struct ap_matrix_mdev *matrix_mdev;
 	int ret;
 
-	if ((atomic_dec_if_positive(&matrix_dev->available_instances) < 0))
-		return -EPERM;
-
 	matrix_mdev = kzalloc(sizeof(*matrix_mdev), GFP_KERNEL);
-	if (!matrix_mdev) {
-		ret = -ENOMEM;
-		goto err_dec_available;
-	}
+	if (!matrix_mdev)
+		return -ENOMEM;
 	vfio_init_group_dev(&matrix_mdev->vdev, &mdev->dev,
 			    &vfio_ap_matrix_dev_ops);
 
@@ -492,8 +487,6 @@  static int vfio_ap_mdev_probe(struct mdev_device *mdev)
 	mutex_unlock(&matrix_dev->lock);
 	vfio_uninit_group_dev(&matrix_mdev->vdev);
 	kfree(matrix_mdev);
-err_dec_available:
-	atomic_inc(&matrix_dev->available_instances);
 	return ret;
 }
 
@@ -509,7 +502,6 @@  static void vfio_ap_mdev_remove(struct mdev_device *mdev)
 	mutex_unlock(&matrix_dev->lock);
 	vfio_uninit_group_dev(&matrix_mdev->vdev);
 	kfree(matrix_mdev);
-	atomic_inc(&matrix_dev->available_instances);
 }
 
 static ssize_t name_show(struct mdev_type *mtype,
@@ -520,20 +512,8 @@  static ssize_t name_show(struct mdev_type *mtype,
 
 static MDEV_TYPE_ATTR_RO(name);
 
-static ssize_t available_instances_show(struct mdev_type *mtype,
-					struct mdev_type_attribute *attr,
-					char *buf)
-{
-	return sprintf(buf, "%d\n",
-		       atomic_read(&matrix_dev->available_instances));
-}
-
-static MDEV_TYPE_ATTR_RO(available_instances);
-
-
 static struct attribute *vfio_ap_mdev_type_attrs[] = {
 	&mdev_type_attr_name.attr,
-	&mdev_type_attr_available_instances.attr,
 	NULL,
 };
 
@@ -1473,6 +1453,11 @@  static ssize_t vfio_ap_mdev_ioctl(struct vfio_device *vdev,
 	return ret;
 }
 
+static unsigned int vfio_ap_mdev_get_available(struct mdev_type *mtype)
+{
+	return MAX_ZDEV_ENTRIES_EXT;
+}
+
 static const struct vfio_device_ops vfio_ap_matrix_dev_ops = {
 	.open_device = vfio_ap_mdev_open_device,
 	.close_device = vfio_ap_mdev_close_device,
@@ -1488,6 +1473,7 @@  static struct mdev_driver vfio_ap_matrix_driver = {
 	},
 	.probe = vfio_ap_mdev_probe,
 	.remove = vfio_ap_mdev_remove,
+	.get_available = vfio_ap_mdev_get_available,
 };
 
 static const struct mdev_parent_ops vfio_ap_matrix_ops = {
@@ -1501,8 +1487,6 @@  int vfio_ap_mdev_register(void)
 {
 	int ret;
 
-	atomic_set(&matrix_dev->available_instances, MAX_ZDEV_ENTRIES_EXT);
-
 	ret = mdev_register_driver(&vfio_ap_matrix_driver);
 	if (ret)
 		return ret;
diff --git a/drivers/s390/crypto/vfio_ap_private.h b/drivers/s390/crypto/vfio_ap_private.h
index 648fcaf8104a..5c0cbf03074b 100644
--- a/drivers/s390/crypto/vfio_ap_private.h
+++ b/drivers/s390/crypto/vfio_ap_private.h
@@ -29,7 +29,6 @@ 
  * struct ap_matrix_dev - Contains the data for the matrix device.
  *
  * @device:	generic device structure associated with the AP matrix device
- * @available_instances: number of mediated matrix devices that can be created
  * @info:	the struct containing the output from the PQAP(QCI) instruction
  * @mdev_list:	the list of mediated matrix devices created
  * @lock:	mutex for locking the AP matrix device. This lock will be
@@ -41,7 +40,6 @@ 
  */
 struct ap_matrix_dev {
 	struct device device;
-	atomic_t available_instances;
 	struct ap_config_info info;
 	struct list_head mdev_list;
 	struct mutex lock;
diff --git a/drivers/vfio/mdev/mdev_core.c b/drivers/vfio/mdev/mdev_core.c
index c3018e8e6d32..bb27ca0db948 100644
--- a/drivers/vfio/mdev/mdev_core.c
+++ b/drivers/vfio/mdev/mdev_core.c
@@ -25,7 +25,7 @@  static DEFINE_MUTEX(parent_list_lock);
 static struct class_compat *mdev_bus_compat_class;
 
 static LIST_HEAD(mdev_list);
-static DEFINE_MUTEX(mdev_list_lock);
+DEFINE_MUTEX(mdev_list_lock);
 
 struct device *mdev_parent_dev(struct mdev_device *mdev)
 {
@@ -245,6 +245,7 @@  static void mdev_device_release(struct device *dev)
 
 	mutex_lock(&mdev_list_lock);
 	list_del(&mdev->next);
+	mdev->type->available++;
 	mutex_unlock(&mdev_list_lock);
 
 	dev_dbg(&mdev->dev, "MDEV: destroying\n");
@@ -268,6 +269,14 @@  int mdev_device_create(struct mdev_type *type, const guid_t *uuid)
 		}
 	}
 
+	if (drv && drv->get_available) {
+		if (!type->available) {
+			mutex_unlock(&mdev_list_lock);
+			return -EUSERS;
+		}
+		type->available--;
+	}
+
 	mdev = kzalloc(sizeof(*mdev), GFP_KERNEL);
 	if (!mdev) {
 		mutex_unlock(&mdev_list_lock);
diff --git a/drivers/vfio/mdev/mdev_private.h b/drivers/vfio/mdev/mdev_private.h
index afbad7b0a14a..83586b070233 100644
--- a/drivers/vfio/mdev/mdev_private.h
+++ b/drivers/vfio/mdev/mdev_private.h
@@ -29,6 +29,7 @@  struct mdev_type {
 	struct kobject *devices_kobj;
 	struct mdev_parent *parent;
 	struct list_head next;
+	unsigned int available;
 	unsigned int type_group_id;
 };
 
@@ -38,6 +39,7 @@  struct mdev_type {
 	container_of(_kobj, struct mdev_type, kobj)
 
 extern struct mdev_driver vfio_mdev_driver;
+extern struct mutex mdev_list_lock;
 
 int  parent_create_sysfs_files(struct mdev_parent *parent);
 void parent_remove_sysfs_files(struct mdev_parent *parent);
diff --git a/drivers/vfio/mdev/mdev_sysfs.c b/drivers/vfio/mdev/mdev_sysfs.c
index d4b99440d19e..b3129dfc27ef 100644
--- a/drivers/vfio/mdev/mdev_sysfs.c
+++ b/drivers/vfio/mdev/mdev_sysfs.c
@@ -93,8 +93,41 @@  static struct attribute_group mdev_type_std_group = {
 	.attrs = mdev_types_std_attrs,
 };
 
+/* mdev_type attribute used by drivers that have an get_available() op */
+static ssize_t available_instances_show(struct mdev_type *mtype,
+					struct mdev_type_attribute *attr,
+					char *buf)
+{
+	unsigned int available;
+
+	mutex_lock(&mdev_list_lock);
+	available = mtype->available;
+	mutex_unlock(&mdev_list_lock);
+
+	return sysfs_emit(buf, "%u\n", available);
+}
+static MDEV_TYPE_ATTR_RO(available_instances);
+static umode_t available_instances_is_visible(struct kobject *kobj,
+					      struct attribute *attr, int n)
+{
+	struct mdev_type *type = to_mdev_type(kobj);
+
+	if (!type->parent->ops->device_driver->get_available)
+		return 0;
+	return attr->mode;
+}
+static struct attribute *mdev_types_name_attrs[] = {
+	&mdev_type_attr_available_instances.attr,
+	NULL,
+};
+static struct attribute_group mdev_type_available_instances_group = {
+	.attrs = mdev_types_name_attrs,
+	.is_visible = available_instances_is_visible,
+};
+
 static const struct attribute_group *mdev_type_groups[] = {
 	&mdev_type_std_group,
+	&mdev_type_available_instances_group,
 	NULL,
 };
 
@@ -136,6 +169,10 @@  static struct mdev_type *add_mdev_supported_type(struct mdev_parent *parent,
 	mdev_get_parent(parent);
 	type->type_group_id = type_group_id;
 
+	if (parent->ops->device_driver->get_available)
+		type->available =
+			parent->ops->device_driver->get_available(type);
+
 	ret = kobject_init_and_add(&type->kobj, &mdev_type_ktype, NULL,
 				   "%s-%s", dev_driver_string(parent->dev),
 				   group->name);
diff --git a/include/linux/mdev.h b/include/linux/mdev.h
index 14655215417b..0ce1bb3dabd0 100644
--- a/include/linux/mdev.h
+++ b/include/linux/mdev.h
@@ -120,12 +120,14 @@  struct mdev_type_attribute {
  * @probe: called when new device created
  * @remove: called when device removed
  * @driver: device driver structure
+ * @get_available: Return the max number of instances that can be created
  *
  **/
 struct mdev_driver {
 	int (*probe)(struct mdev_device *dev);
 	void (*remove)(struct mdev_device *dev);
 	struct device_driver driver;
+	unsigned int (*get_available)(struct mdev_type *mtype);
 };
 
 static inline void *mdev_get_drvdata(struct mdev_device *mdev)
diff --git a/samples/vfio-mdev/mdpy.c b/samples/vfio-mdev/mdpy.c
index 402a7ebe6563..d7da6ed35657 100644
--- a/samples/vfio-mdev/mdpy.c
+++ b/samples/vfio-mdev/mdpy.c
@@ -84,7 +84,6 @@  static dev_t		mdpy_devt;
 static struct class	*mdpy_class;
 static struct cdev	mdpy_cdev;
 static struct device	mdpy_dev;
-static u32		mdpy_count;
 static const struct vfio_device_ops mdpy_dev_ops;
 
 /* State of each mdev device */
@@ -225,9 +224,6 @@  static int mdpy_probe(struct mdev_device *mdev)
 	u32 fbsize;
 	int ret;
 
-	if (mdpy_count >= max_devices)
-		return -ENOMEM;
-
 	mdev_state = kzalloc(sizeof(struct mdev_state), GFP_KERNEL);
 	if (mdev_state == NULL)
 		return -ENOMEM;
@@ -256,8 +252,6 @@  static int mdpy_probe(struct mdev_device *mdev)
 	mdpy_create_config_space(mdev_state);
 	mdpy_reset(mdev_state);
 
-	mdpy_count++;
-
 	ret = vfio_register_emulated_iommu_dev(&mdev_state->vdev);
 	if (ret)
 		goto err_mem;
@@ -284,8 +278,6 @@  static void mdpy_remove(struct mdev_device *mdev)
 	kfree(mdev_state->vconfig);
 	vfio_uninit_group_dev(&mdev_state->vdev);
 	kfree(mdev_state);
-
-	mdpy_count--;
 }
 
 static ssize_t mdpy_read(struct vfio_device *vdev, char __user *buf,
@@ -662,18 +654,10 @@  static ssize_t description_show(struct mdev_type *mtype,
 }
 static MDEV_TYPE_ATTR_RO(description);
 
-static ssize_t available_instances_show(struct mdev_type *mtype,
-					struct mdev_type_attribute *attr,
-					char *buf)
-{
-	return sprintf(buf, "%d\n", max_devices - mdpy_count);
-}
-static MDEV_TYPE_ATTR_RO(available_instances);
 
 static struct attribute *mdev_types_attrs[] = {
 	&mdev_type_attr_name.attr,
 	&mdev_type_attr_description.attr,
-	&mdev_type_attr_available_instances.attr,
 	NULL,
 };
 
@@ -706,6 +690,11 @@  static const struct vfio_device_ops mdpy_dev_ops = {
 	.mmap = mdpy_mmap,
 };
 
+static unsigned int mdpy_get_available(struct mdev_type *mtype)
+{
+	return max_devices;
+}
+
 static struct mdev_driver mdpy_driver = {
 	.driver = {
 		.name = "mdpy",
@@ -715,6 +704,7 @@  static struct mdev_driver mdpy_driver = {
 	},
 	.probe = mdpy_probe,
 	.remove	= mdpy_remove,
+	.get_available = mdpy_get_available,
 };
 
 static const struct mdev_parent_ops mdev_fops = {