diff mbox series

[RFC,11/13] vfio/cxl: introduce VFIO CXL device cap

Message ID 20240920223446.1908673-12-zhiw@nvidia.com
State New
Headers show
Series vfio: introduce vfio-cxl to support CXL type-2 accelerator passthrough | expand

Commit Message

Zhi Wang Sept. 20, 2024, 10:34 p.m. UTC 
The userspace needs CXL device information, e.g. HDM decoder registers
offset to know when the VM updates the HDM decoder and re-build the
mapping between GPA in the virtual HDM decoder base registers and the
HPA of the CXL region created by the vfio-cxl-core when initialize the
CXL device.

To acheive this, a new VFIO CXL device cap is required to convey those
information to the usersapce.

Introduce a new VFIO CXL device cap to expose necessary information to
the userspace. Initialize the cap with the information filled when the
CXL device is being initialized. vfio-pci-core fills the CXL cap into
the caps returned to userapce when CXL is enabled.

Signed-off-by: Zhi Wang <zhiw@nvidia.com>
---
 drivers/vfio/pci/vfio_cxl_core.c | 15 +++++++++++++++
 drivers/vfio/pci/vfio_pci_core.c | 19 ++++++++++++++++++-
 include/linux/vfio_pci_core.h    |  1 +
 include/uapi/linux/vfio.h        | 10 ++++++++++
 4 files changed, 44 insertions(+), 1 deletion(-)

Comments

Alex Williamson Oct. 11, 2024, 9:14 p.m. UTC | #1 
On Fri, 20 Sep 2024 15:34:44 -0700
Zhi Wang <zhiw@nvidia.com> wrote:

> The userspace needs CXL device information, e.g. HDM decoder registers
> offset to know when the VM updates the HDM decoder and re-build the
> mapping between GPA in the virtual HDM decoder base registers and the
> HPA of the CXL region created by the vfio-cxl-core when initialize the
> CXL device.
> 
> To acheive this, a new VFIO CXL device cap is required to convey those
> information to the usersapce.
> 
> Introduce a new VFIO CXL device cap to expose necessary information to
> the userspace. Initialize the cap with the information filled when the
> CXL device is being initialized. vfio-pci-core fills the CXL cap into
> the caps returned to userapce when CXL is enabled.
> 
> Signed-off-by: Zhi Wang <zhiw@nvidia.com>
> ---
>  drivers/vfio/pci/vfio_cxl_core.c | 15 +++++++++++++++
>  drivers/vfio/pci/vfio_pci_core.c | 19 ++++++++++++++++++-
>  include/linux/vfio_pci_core.h    |  1 +
>  include/uapi/linux/vfio.h        | 10 ++++++++++
>  4 files changed, 44 insertions(+), 1 deletion(-)
> 
> diff --git a/drivers/vfio/pci/vfio_cxl_core.c b/drivers/vfio/pci/vfio_cxl_core.c
> index d8b51f8792a2..cebc444b54b7 100644
> --- a/drivers/vfio/pci/vfio_cxl_core.c
> +++ b/drivers/vfio/pci/vfio_cxl_core.c
> @@ -367,6 +367,19 @@ static int setup_virt_comp_regs(struct vfio_pci_core_device *core_dev)
>  	return 0;
>  }
>  
> +static void init_vfio_cxl_cap(struct vfio_pci_core_device *core_dev)
> +{
> +	struct vfio_cxl *cxl = &core_dev->cxl;
> +
> +	cxl->cap.header.id = VFIO_DEVICE_INFO_CAP_CXL;
> +	cxl->cap.header.version = 1;
> +	cxl->cap.hdm_count = cxl->hdm_count;
> +	cxl->cap.hdm_reg_offset = cxl->hdm_reg_offset;
> +	cxl->cap.hdm_reg_size = cxl->hdm_reg_size;
> +	cxl->cap.hdm_reg_bar_index = cxl->comp_reg_bar;
> +	cxl->cap.dpa_size = cxl->dpa_size;
> +}
> +
>  int vfio_cxl_core_enable(struct vfio_pci_core_device *core_dev)
>  {
>  	struct vfio_cxl *cxl = &core_dev->cxl;
> @@ -401,6 +414,8 @@ int vfio_cxl_core_enable(struct vfio_pci_core_device *core_dev)
>  	if (ret)
>  		goto err_enable_cxl_device;
>  
> +	init_vfio_cxl_cap(core_dev);
> +
>  	flags = VFIO_REGION_INFO_FLAG_READ |
>  		VFIO_REGION_INFO_FLAG_WRITE |
>  		VFIO_REGION_INFO_FLAG_MMAP;
> diff --git a/drivers/vfio/pci/vfio_pci_core.c b/drivers/vfio/pci/vfio_pci_core.c
> index e0f23b538858..47e65e28a42b 100644
> --- a/drivers/vfio/pci/vfio_pci_core.c
> +++ b/drivers/vfio/pci/vfio_pci_core.c
> @@ -963,6 +963,15 @@ static int vfio_pci_info_atomic_cap(struct vfio_pci_core_device *vdev,
>  	return vfio_info_add_capability(caps, &cap.header, sizeof(cap));
>  }
>  
> +static int vfio_pci_info_cxl_cap(struct vfio_pci_core_device *vdev,
> +				 struct vfio_info_cap *caps)
> +{
> +	struct vfio_cxl *cxl = &vdev->cxl;
> +
> +	return vfio_info_add_capability(caps, &cxl->cap.header,
> +					sizeof(cxl->cap));
> +}
> +
>  static int vfio_pci_ioctl_get_info(struct vfio_pci_core_device *vdev,
>  				   struct vfio_device_info __user *arg)
>  {
> @@ -984,9 +993,17 @@ static int vfio_pci_ioctl_get_info(struct vfio_pci_core_device *vdev,
>  	if (vdev->reset_works)
>  		info.flags |= VFIO_DEVICE_FLAGS_RESET;
>  
> -	if (vdev->has_cxl)
> +	if (vdev->has_cxl) {
>  		info.flags |= VFIO_DEVICE_FLAGS_CXL;
>  
> +		ret = vfio_pci_info_cxl_cap(vdev, &caps);
> +		if (ret) {
> +			pci_warn(vdev->pdev,
> +				 "Failed to setup CXL capabilities\n");
> +			return ret;
> +		}
> +	}
> +
>  	info.num_regions = VFIO_PCI_NUM_REGIONS + vdev->num_regions;
>  	info.num_irqs = VFIO_PCI_NUM_IRQS;
>  
> diff --git a/include/linux/vfio_pci_core.h b/include/linux/vfio_pci_core.h
> index e5646aad3eb3..d79f7a91d977 100644
> --- a/include/linux/vfio_pci_core.h
> +++ b/include/linux/vfio_pci_core.h
> @@ -80,6 +80,7 @@ struct vfio_cxl {
>  	struct resource ram_res;
>  
>  	struct vfio_cxl_region region;
> +	struct vfio_device_info_cap_cxl cap;

We should create this dynamically on request, the device info ioctl is
not a fast path.  Thanks,

Alex

>  };
>  
>  struct vfio_pci_core_device {
> diff --git a/include/uapi/linux/vfio.h b/include/uapi/linux/vfio.h
> index 0895183feaac..9a5972961280 100644
> --- a/include/uapi/linux/vfio.h
> +++ b/include/uapi/linux/vfio.h
> @@ -257,6 +257,16 @@ struct vfio_device_info_cap_pci_atomic_comp {
>  	__u32 reserved;
>  };
>  
> +#define VFIO_DEVICE_INFO_CAP_CXL		6
> +struct vfio_device_info_cap_cxl {
> +	struct vfio_info_cap_header header;
> +	__u8 hdm_count;
> +	__u8 hdm_reg_bar_index;
> +	__u64 hdm_reg_size;
> +	__u64 hdm_reg_offset;
> +	__u64 dpa_size;
> +};
> +
>  /**
>   * VFIO_DEVICE_GET_REGION_INFO - _IOWR(VFIO_TYPE, VFIO_BASE + 8,
>   *				       struct vfio_region_info)
diff mbox series

Patch

diff --git a/drivers/vfio/pci/vfio_cxl_core.c b/drivers/vfio/pci/vfio_cxl_core.c
index d8b51f8792a2..cebc444b54b7 100644
--- a/drivers/vfio/pci/vfio_cxl_core.c
+++ b/drivers/vfio/pci/vfio_cxl_core.c
@@ -367,6 +367,19 @@  static int setup_virt_comp_regs(struct vfio_pci_core_device *core_dev)
 	return 0;
 }
 
+static void init_vfio_cxl_cap(struct vfio_pci_core_device *core_dev)
+{
+	struct vfio_cxl *cxl = &core_dev->cxl;
+
+	cxl->cap.header.id = VFIO_DEVICE_INFO_CAP_CXL;
+	cxl->cap.header.version = 1;
+	cxl->cap.hdm_count = cxl->hdm_count;
+	cxl->cap.hdm_reg_offset = cxl->hdm_reg_offset;
+	cxl->cap.hdm_reg_size = cxl->hdm_reg_size;
+	cxl->cap.hdm_reg_bar_index = cxl->comp_reg_bar;
+	cxl->cap.dpa_size = cxl->dpa_size;
+}
+
 int vfio_cxl_core_enable(struct vfio_pci_core_device *core_dev)
 {
 	struct vfio_cxl *cxl = &core_dev->cxl;
@@ -401,6 +414,8 @@  int vfio_cxl_core_enable(struct vfio_pci_core_device *core_dev)
 	if (ret)
 		goto err_enable_cxl_device;
 
+	init_vfio_cxl_cap(core_dev);
+
 	flags = VFIO_REGION_INFO_FLAG_READ |
 		VFIO_REGION_INFO_FLAG_WRITE |
 		VFIO_REGION_INFO_FLAG_MMAP;
diff --git a/drivers/vfio/pci/vfio_pci_core.c b/drivers/vfio/pci/vfio_pci_core.c
index e0f23b538858..47e65e28a42b 100644
--- a/drivers/vfio/pci/vfio_pci_core.c
+++ b/drivers/vfio/pci/vfio_pci_core.c
@@ -963,6 +963,15 @@  static int vfio_pci_info_atomic_cap(struct vfio_pci_core_device *vdev,
 	return vfio_info_add_capability(caps, &cap.header, sizeof(cap));
 }
 
+static int vfio_pci_info_cxl_cap(struct vfio_pci_core_device *vdev,
+				 struct vfio_info_cap *caps)
+{
+	struct vfio_cxl *cxl = &vdev->cxl;
+
+	return vfio_info_add_capability(caps, &cxl->cap.header,
+					sizeof(cxl->cap));
+}
+
 static int vfio_pci_ioctl_get_info(struct vfio_pci_core_device *vdev,
 				   struct vfio_device_info __user *arg)
 {
@@ -984,9 +993,17 @@  static int vfio_pci_ioctl_get_info(struct vfio_pci_core_device *vdev,
 	if (vdev->reset_works)
 		info.flags |= VFIO_DEVICE_FLAGS_RESET;
 
-	if (vdev->has_cxl)
+	if (vdev->has_cxl) {
 		info.flags |= VFIO_DEVICE_FLAGS_CXL;
 
+		ret = vfio_pci_info_cxl_cap(vdev, &caps);
+		if (ret) {
+			pci_warn(vdev->pdev,
+				 "Failed to setup CXL capabilities\n");
+			return ret;
+		}
+	}
+
 	info.num_regions = VFIO_PCI_NUM_REGIONS + vdev->num_regions;
 	info.num_irqs = VFIO_PCI_NUM_IRQS;
 
diff --git a/include/linux/vfio_pci_core.h b/include/linux/vfio_pci_core.h
index e5646aad3eb3..d79f7a91d977 100644
--- a/include/linux/vfio_pci_core.h
+++ b/include/linux/vfio_pci_core.h
@@ -80,6 +80,7 @@  struct vfio_cxl {
 	struct resource ram_res;
 
 	struct vfio_cxl_region region;
+	struct vfio_device_info_cap_cxl cap;
 };
 
 struct vfio_pci_core_device {
diff --git a/include/uapi/linux/vfio.h b/include/uapi/linux/vfio.h
index 0895183feaac..9a5972961280 100644
--- a/include/uapi/linux/vfio.h
+++ b/include/uapi/linux/vfio.h
@@ -257,6 +257,16 @@  struct vfio_device_info_cap_pci_atomic_comp {
 	__u32 reserved;
 };
 
+#define VFIO_DEVICE_INFO_CAP_CXL		6
+struct vfio_device_info_cap_cxl {
+	struct vfio_info_cap_header header;
+	__u8 hdm_count;
+	__u8 hdm_reg_bar_index;
+	__u64 hdm_reg_size;
+	__u64 hdm_reg_offset;
+	__u64 dpa_size;
+};
+
 /**
  * VFIO_DEVICE_GET_REGION_INFO - _IOWR(VFIO_TYPE, VFIO_BASE + 8,
  *				       struct vfio_region_info)