diff mbox

[for-next,v1,1/2] RDMA/vmw_pvrdma: Add RoCEv2 support

Message ID 91bf53451b8618acc731f6b8b4235103abae7687.1503468979.git.aditr@vmware.com (mailing list archive)
State Accepted
Headers show

Commit Message

Adit Ranadive Aug. 23, 2017, 6:19 a.m. UTC 
From: Bryan Tan <bryantan@vmware.com>

The driver version is bumped for compatibility purposes. Also, send correct
GID type during register to device. Added compatibility check macros for
the device.

Reviewed-by: Jorgen Hansen <jhansen@vmware.com>
Reviewed-by: Aditya Sarwade <asarwade@vmware.com>
Signed-off-by: Bryan Tan <bryantan@vmware.com>
Signed-off-by: Adit Ranadive <aditr@vmware.com>
---
 drivers/infiniband/hw/vmw_pvrdma/pvrdma.h         |  2 ++
 drivers/infiniband/hw/vmw_pvrdma/pvrdma_dev_api.h | 28 +++++++++++++++++-
 drivers/infiniband/hw/vmw_pvrdma/pvrdma_main.c    | 36 ++++++++++-------------
 drivers/infiniband/hw/vmw_pvrdma/pvrdma_misc.c    |  7 +++++
 4 files changed, 51 insertions(+), 22 deletions(-)

Comments

Leon Romanovsky Aug. 24, 2017, 4:33 a.m. UTC | #1 
On Tue, Aug 22, 2017 at 11:19:00PM -0700, Adit Ranadive wrote:
> From: Bryan Tan <bryantan@vmware.com>
>
> The driver version is bumped for compatibility purposes. Also, send correct
> GID type during register to device. Added compatibility check macros for
> the device.
>
> Reviewed-by: Jorgen Hansen <jhansen@vmware.com>
> Reviewed-by: Aditya Sarwade <asarwade@vmware.com>
> Signed-off-by: Bryan Tan <bryantan@vmware.com>
> Signed-off-by: Adit Ranadive <aditr@vmware.com>
> ---
>  drivers/infiniband/hw/vmw_pvrdma/pvrdma.h         |  2 ++
>  drivers/infiniband/hw/vmw_pvrdma/pvrdma_dev_api.h | 28 +++++++++++++++++-
>  drivers/infiniband/hw/vmw_pvrdma/pvrdma_main.c    | 36 ++++++++++-------------
>  drivers/infiniband/hw/vmw_pvrdma/pvrdma_misc.c    |  7 +++++
>  4 files changed, 51 insertions(+), 22 deletions(-)
>

Thanks,
Reviewed-by: Leon Romanovsky <leonro@mellanox.com>
Yuval Shaia Aug. 24, 2017, 1:51 p.m. UTC | #2 
On Tue, Aug 22, 2017 at 11:19:00PM -0700, Adit Ranadive wrote:
> From: Bryan Tan <bryantan@vmware.com>
> 
> The driver version is bumped for compatibility purposes. Also, send correct
> GID type during register to device. Added compatibility check macros for
> the device.
> 
> Reviewed-by: Jorgen Hansen <jhansen@vmware.com>
> Reviewed-by: Aditya Sarwade <asarwade@vmware.com>
> Signed-off-by: Bryan Tan <bryantan@vmware.com>
> Signed-off-by: Adit Ranadive <aditr@vmware.com>
> ---
>  drivers/infiniband/hw/vmw_pvrdma/pvrdma.h         |  2 ++
>  drivers/infiniband/hw/vmw_pvrdma/pvrdma_dev_api.h | 28 +++++++++++++++++-
>  drivers/infiniband/hw/vmw_pvrdma/pvrdma_main.c    | 36 ++++++++++-------------
>  drivers/infiniband/hw/vmw_pvrdma/pvrdma_misc.c    |  7 +++++
>  4 files changed, 51 insertions(+), 22 deletions(-)
> 
> diff --git a/drivers/infiniband/hw/vmw_pvrdma/pvrdma.h b/drivers/infiniband/hw/vmw_pvrdma/pvrdma.h
> index 8e2f0a1..663a0c3 100644
> --- a/drivers/infiniband/hw/vmw_pvrdma/pvrdma.h
> +++ b/drivers/infiniband/hw/vmw_pvrdma/pvrdma.h
> @@ -194,6 +194,7 @@ struct pvrdma_dev {
>  	void *resp_slot;
>  	unsigned long flags;
>  	struct list_head device_link;
> +	unsigned int dsr_version;
>  
>  	/* Locking and interrupt information. */
>  	spinlock_t cmd_lock; /* Command lock. */
> @@ -444,6 +445,7 @@ void pvrdma_ah_attr_to_rdma(struct rdma_ah_attr *dst,
>  			    const struct pvrdma_ah_attr *src);
>  void rdma_ah_attr_to_pvrdma(struct pvrdma_ah_attr *dst,
>  			    const struct rdma_ah_attr *src);
> +u8 ib_gid_type_to_pvrdma(enum ib_gid_type gid_type);
>  
>  int pvrdma_uar_table_init(struct pvrdma_dev *dev);
>  void pvrdma_uar_table_cleanup(struct pvrdma_dev *dev);
> diff --git a/drivers/infiniband/hw/vmw_pvrdma/pvrdma_dev_api.h b/drivers/infiniband/hw/vmw_pvrdma/pvrdma_dev_api.h
> index 09078cc..3a308ff 100644
> --- a/drivers/infiniband/hw/vmw_pvrdma/pvrdma_dev_api.h
> +++ b/drivers/infiniband/hw/vmw_pvrdma/pvrdma_dev_api.h
> @@ -50,7 +50,15 @@
>  
>  #include "pvrdma_verbs.h"
>  
> -#define PVRDMA_VERSION			17
> +/*
> + * PVRDMA version macros. Some new features require updates to PVRDMA_VERSION.
> + * These macros allow us to check for different features if necessary.
> + */
> +
> +#define PVRDMA_ROCEV1_VERSION		17
> +#define PVRDMA_ROCEV2_VERSION		18
> +#define PVRDMA_VERSION			PVRDMA_ROCEV2_VERSION
> +
>  #define PVRDMA_BOARD_ID			1
>  #define PVRDMA_REV_ID			1
>  
> @@ -123,6 +131,24 @@
>  #define PVRDMA_GID_TYPE_FLAG_ROCE_V1	BIT(0)
>  #define PVRDMA_GID_TYPE_FLAG_ROCE_V2	BIT(1)
>  
> +/*
> + * Version checks. This checks whether each version supports specific
> + * capabilities from the device.
> + */
> +
> +#define PVRDMA_IS_VERSION17(_dev)					\
> +	(_dev->dsr_version == PVRDMA_ROCEV1_VERSION &&			\
> +	 _dev->dsr->caps.gid_types == PVRDMA_GID_TYPE_FLAG_ROCE_V1)
> +
> +#define PVRDMA_IS_VERSION18(_dev)					\
> +	(_dev->dsr_version >= PVRDMA_ROCEV2_VERSION &&			\
> +	 (_dev->dsr->caps.gid_types == PVRDMA_GID_TYPE_FLAG_ROCE_V1 ||  \
> +	  _dev->dsr->caps.gid_types == PVRDMA_GID_TYPE_FLAG_ROCE_V2))	\
> +
> +#define PVRDMA_SUPPORTED(_dev)						\
> +	((_dev->dsr->caps.mode == PVRDMA_DEVICE_MODE_ROCE) &&		\
> +	 (PVRDMA_IS_VERSION17(_dev) || PVRDMA_IS_VERSION18(_dev)))
> +
>  enum pvrdma_pci_resource {
>  	PVRDMA_PCI_RESOURCE_MSIX,	/* BAR0: MSI-X, MMIO. */
>  	PVRDMA_PCI_RESOURCE_REG,	/* BAR1: Registers, MMIO. */
> diff --git a/drivers/infiniband/hw/vmw_pvrdma/pvrdma_main.c b/drivers/infiniband/hw/vmw_pvrdma/pvrdma_main.c
> index 5b00156..6ce709a 100644
> --- a/drivers/infiniband/hw/vmw_pvrdma/pvrdma_main.c
> +++ b/drivers/infiniband/hw/vmw_pvrdma/pvrdma_main.c
> @@ -128,10 +128,14 @@ static int pvrdma_init_device(struct pvrdma_dev *dev)
>  static int pvrdma_port_immutable(struct ib_device *ibdev, u8 port_num,
>  				 struct ib_port_immutable *immutable)
>  {
> +	struct pvrdma_dev *dev = to_vdev(ibdev);
>  	struct ib_port_attr attr;
>  	int err;
>  
> -	immutable->core_cap_flags = RDMA_CORE_PORT_IBA_ROCE;
> +	if (dev->dsr->caps.gid_types == PVRDMA_GID_TYPE_FLAG_ROCE_V1)
> +		immutable->core_cap_flags |= RDMA_CORE_PORT_IBA_ROCE;
> +	else if (dev->dsr->caps.gid_types == PVRDMA_GID_TYPE_FLAG_ROCE_V2)
> +		immutable->core_cap_flags |= RDMA_CORE_PORT_IBA_ROCE_UDP_ENCAP;
>  
>  	err = ib_query_port(ibdev, port_num, &attr);
>  	if (err)
> @@ -569,6 +573,7 @@ static void pvrdma_free_slots(struct pvrdma_dev *dev)
>  
>  static int pvrdma_add_gid_at_index(struct pvrdma_dev *dev,
>  				   const union ib_gid *gid,
> +				   u8 gid_type,
>  				   int index)
>  {
>  	int ret;
> @@ -586,7 +591,7 @@ static int pvrdma_add_gid_at_index(struct pvrdma_dev *dev,
>  	cmd_bind->mtu = ib_mtu_enum_to_int(IB_MTU_1024);
>  	cmd_bind->vlan = 0xfff;
>  	cmd_bind->index = index;
> -	cmd_bind->gid_type = PVRDMA_GID_TYPE_FLAG_ROCE_V1;
> +	cmd_bind->gid_type = gid_type;
>  
>  	ret = pvrdma_cmd_post(dev, &req, NULL, 0);
>  	if (ret < 0) {
> @@ -607,7 +612,9 @@ static int pvrdma_add_gid(struct ib_device *ibdev,
>  {
>  	struct pvrdma_dev *dev = to_vdev(ibdev);
>  
> -	return pvrdma_add_gid_at_index(dev, gid, index);
> +	return pvrdma_add_gid_at_index(dev, gid,
> +				       ib_gid_type_to_pvrdma(attr->gid_type),
> +				       index);
>  }
>  
>  static int pvrdma_del_gid_at_index(struct pvrdma_dev *dev, int index)
> @@ -722,7 +729,6 @@ static int pvrdma_pci_probe(struct pci_dev *pdev,
>  	int ret;
>  	unsigned long start;
>  	unsigned long len;
> -	unsigned int version;
>  	dma_addr_t slot_dma = 0;
>  
>  	dev_dbg(&pdev->dev, "initializing driver %s\n", pci_name(pdev));
> @@ -819,13 +825,9 @@ static int pvrdma_pci_probe(struct pci_dev *pdev,
>  		goto err_unmap_regs;
>  	}
>  
> -	version = pvrdma_read_reg(dev, PVRDMA_REG_VERSION);
> +	dev->dsr_version = pvrdma_read_reg(dev, PVRDMA_REG_VERSION);
>  	dev_info(&pdev->dev, "device version %d, driver version %d\n",
> -		 version, PVRDMA_VERSION);
> -	if (version < PVRDMA_VERSION) {
> -		dev_err(&pdev->dev, "incompatible device version\n");
> -		goto err_uar_unmap;
> -	}
> +		 dev->dsr_version, PVRDMA_VERSION);
>  
>  	dev->dsr = dma_alloc_coherent(&pdev->dev, sizeof(*dev->dsr),
>  				      &dev->dsrbase, GFP_KERNEL);
> @@ -896,17 +898,9 @@ static int pvrdma_pci_probe(struct pci_dev *pdev,
>  	/* Make sure the write is complete before reading status. */
>  	mb();
>  
> -	/* Currently, the driver only supports RoCE mode. */
> -	if (dev->dsr->caps.mode != PVRDMA_DEVICE_MODE_ROCE) {
> -		dev_err(&pdev->dev, "unsupported transport %d\n",
> -			dev->dsr->caps.mode);
> -		ret = -EFAULT;
> -		goto err_free_cq_ring;
> -	}
> -
> -	/* Currently, the driver only supports RoCE V1. */
> -	if (!(dev->dsr->caps.gid_types & PVRDMA_GID_TYPE_FLAG_ROCE_V1)) {
> -		dev_err(&pdev->dev, "driver needs RoCE v1 support\n");
> +	/* The driver supports RoCE V1 and V2. */
> +	if (!PVRDMA_SUPPORTED(dev)) {
> +		dev_err(&pdev->dev, "driver needs RoCE v1 or v2 support\n");
>  		ret = -EFAULT;
>  		goto err_free_cq_ring;
>  	}
> diff --git a/drivers/infiniband/hw/vmw_pvrdma/pvrdma_misc.c b/drivers/infiniband/hw/vmw_pvrdma/pvrdma_misc.c
> index ec6a4ca..fb0c5c0 100644
> --- a/drivers/infiniband/hw/vmw_pvrdma/pvrdma_misc.c
> +++ b/drivers/infiniband/hw/vmw_pvrdma/pvrdma_misc.c
> @@ -303,3 +303,10 @@ void rdma_ah_attr_to_pvrdma(struct pvrdma_ah_attr *dst,
>  	dst->port_num = rdma_ah_get_port_num(src);
>  	memcpy(&dst->dmac, src->roce.dmac, sizeof(dst->dmac));
>  }
> +
> +u8 ib_gid_type_to_pvrdma(enum ib_gid_type gid_type)
> +{
> +	return (gid_type == IB_GID_TYPE_ROCE_UDP_ENCAP) ?
> +		PVRDMA_GID_TYPE_FLAG_ROCE_V2 :
> +		PVRDMA_GID_TYPE_FLAG_ROCE_V1;
> +}
> -- 
> 2.7.4

Reviewed-by: Yuval Shaia <yuval.shaia@oracle.com>

> 
> --
> To unsubscribe from this list: send the line "unsubscribe linux-rdma" in
> the body of a message to majordomo@vger.kernel.org
> More majordomo info at  http://vger.kernel.org/majordomo-info.html
--
To unsubscribe from this list: send the line "unsubscribe linux-rdma" in
the body of a message to majordomo@vger.kernel.org
More majordomo info at  http://vger.kernel.org/majordomo-info.html
diff mbox

Patch

diff --git a/drivers/infiniband/hw/vmw_pvrdma/pvrdma.h b/drivers/infiniband/hw/vmw_pvrdma/pvrdma.h
index 8e2f0a1..663a0c3 100644
--- a/drivers/infiniband/hw/vmw_pvrdma/pvrdma.h
+++ b/drivers/infiniband/hw/vmw_pvrdma/pvrdma.h
@@ -194,6 +194,7 @@  struct pvrdma_dev {
 	void *resp_slot;
 	unsigned long flags;
 	struct list_head device_link;
+	unsigned int dsr_version;
 
 	/* Locking and interrupt information. */
 	spinlock_t cmd_lock; /* Command lock. */
@@ -444,6 +445,7 @@  void pvrdma_ah_attr_to_rdma(struct rdma_ah_attr *dst,
 			    const struct pvrdma_ah_attr *src);
 void rdma_ah_attr_to_pvrdma(struct pvrdma_ah_attr *dst,
 			    const struct rdma_ah_attr *src);
+u8 ib_gid_type_to_pvrdma(enum ib_gid_type gid_type);
 
 int pvrdma_uar_table_init(struct pvrdma_dev *dev);
 void pvrdma_uar_table_cleanup(struct pvrdma_dev *dev);
diff --git a/drivers/infiniband/hw/vmw_pvrdma/pvrdma_dev_api.h b/drivers/infiniband/hw/vmw_pvrdma/pvrdma_dev_api.h
index 09078cc..3a308ff 100644
--- a/drivers/infiniband/hw/vmw_pvrdma/pvrdma_dev_api.h
+++ b/drivers/infiniband/hw/vmw_pvrdma/pvrdma_dev_api.h
@@ -50,7 +50,15 @@ 
 
 #include "pvrdma_verbs.h"
 
-#define PVRDMA_VERSION			17
+/*
+ * PVRDMA version macros. Some new features require updates to PVRDMA_VERSION.
+ * These macros allow us to check for different features if necessary.
+ */
+
+#define PVRDMA_ROCEV1_VERSION		17
+#define PVRDMA_ROCEV2_VERSION		18
+#define PVRDMA_VERSION			PVRDMA_ROCEV2_VERSION
+
 #define PVRDMA_BOARD_ID			1
 #define PVRDMA_REV_ID			1
 
@@ -123,6 +131,24 @@ 
 #define PVRDMA_GID_TYPE_FLAG_ROCE_V1	BIT(0)
 #define PVRDMA_GID_TYPE_FLAG_ROCE_V2	BIT(1)
 
+/*
+ * Version checks. This checks whether each version supports specific
+ * capabilities from the device.
+ */
+
+#define PVRDMA_IS_VERSION17(_dev)					\
+	(_dev->dsr_version == PVRDMA_ROCEV1_VERSION &&			\
+	 _dev->dsr->caps.gid_types == PVRDMA_GID_TYPE_FLAG_ROCE_V1)
+
+#define PVRDMA_IS_VERSION18(_dev)					\
+	(_dev->dsr_version >= PVRDMA_ROCEV2_VERSION &&			\
+	 (_dev->dsr->caps.gid_types == PVRDMA_GID_TYPE_FLAG_ROCE_V1 ||  \
+	  _dev->dsr->caps.gid_types == PVRDMA_GID_TYPE_FLAG_ROCE_V2))	\
+
+#define PVRDMA_SUPPORTED(_dev)						\
+	((_dev->dsr->caps.mode == PVRDMA_DEVICE_MODE_ROCE) &&		\
+	 (PVRDMA_IS_VERSION17(_dev) || PVRDMA_IS_VERSION18(_dev)))
+
 enum pvrdma_pci_resource {
 	PVRDMA_PCI_RESOURCE_MSIX,	/* BAR0: MSI-X, MMIO. */
 	PVRDMA_PCI_RESOURCE_REG,	/* BAR1: Registers, MMIO. */
diff --git a/drivers/infiniband/hw/vmw_pvrdma/pvrdma_main.c b/drivers/infiniband/hw/vmw_pvrdma/pvrdma_main.c
index 5b00156..6ce709a 100644
--- a/drivers/infiniband/hw/vmw_pvrdma/pvrdma_main.c
+++ b/drivers/infiniband/hw/vmw_pvrdma/pvrdma_main.c
@@ -128,10 +128,14 @@  static int pvrdma_init_device(struct pvrdma_dev *dev)
 static int pvrdma_port_immutable(struct ib_device *ibdev, u8 port_num,
 				 struct ib_port_immutable *immutable)
 {
+	struct pvrdma_dev *dev = to_vdev(ibdev);
 	struct ib_port_attr attr;
 	int err;
 
-	immutable->core_cap_flags = RDMA_CORE_PORT_IBA_ROCE;
+	if (dev->dsr->caps.gid_types == PVRDMA_GID_TYPE_FLAG_ROCE_V1)
+		immutable->core_cap_flags |= RDMA_CORE_PORT_IBA_ROCE;
+	else if (dev->dsr->caps.gid_types == PVRDMA_GID_TYPE_FLAG_ROCE_V2)
+		immutable->core_cap_flags |= RDMA_CORE_PORT_IBA_ROCE_UDP_ENCAP;
 
 	err = ib_query_port(ibdev, port_num, &attr);
 	if (err)
@@ -569,6 +573,7 @@  static void pvrdma_free_slots(struct pvrdma_dev *dev)
 
 static int pvrdma_add_gid_at_index(struct pvrdma_dev *dev,
 				   const union ib_gid *gid,
+				   u8 gid_type,
 				   int index)
 {
 	int ret;
@@ -586,7 +591,7 @@  static int pvrdma_add_gid_at_index(struct pvrdma_dev *dev,
 	cmd_bind->mtu = ib_mtu_enum_to_int(IB_MTU_1024);
 	cmd_bind->vlan = 0xfff;
 	cmd_bind->index = index;
-	cmd_bind->gid_type = PVRDMA_GID_TYPE_FLAG_ROCE_V1;
+	cmd_bind->gid_type = gid_type;
 
 	ret = pvrdma_cmd_post(dev, &req, NULL, 0);
 	if (ret < 0) {
@@ -607,7 +612,9 @@  static int pvrdma_add_gid(struct ib_device *ibdev,
 {
 	struct pvrdma_dev *dev = to_vdev(ibdev);
 
-	return pvrdma_add_gid_at_index(dev, gid, index);
+	return pvrdma_add_gid_at_index(dev, gid,
+				       ib_gid_type_to_pvrdma(attr->gid_type),
+				       index);
 }
 
 static int pvrdma_del_gid_at_index(struct pvrdma_dev *dev, int index)
@@ -722,7 +729,6 @@  static int pvrdma_pci_probe(struct pci_dev *pdev,
 	int ret;
 	unsigned long start;
 	unsigned long len;
-	unsigned int version;
 	dma_addr_t slot_dma = 0;
 
 	dev_dbg(&pdev->dev, "initializing driver %s\n", pci_name(pdev));
@@ -819,13 +825,9 @@  static int pvrdma_pci_probe(struct pci_dev *pdev,
 		goto err_unmap_regs;
 	}
 
-	version = pvrdma_read_reg(dev, PVRDMA_REG_VERSION);
+	dev->dsr_version = pvrdma_read_reg(dev, PVRDMA_REG_VERSION);
 	dev_info(&pdev->dev, "device version %d, driver version %d\n",
-		 version, PVRDMA_VERSION);
-	if (version < PVRDMA_VERSION) {
-		dev_err(&pdev->dev, "incompatible device version\n");
-		goto err_uar_unmap;
-	}
+		 dev->dsr_version, PVRDMA_VERSION);
 
 	dev->dsr = dma_alloc_coherent(&pdev->dev, sizeof(*dev->dsr),
 				      &dev->dsrbase, GFP_KERNEL);
@@ -896,17 +898,9 @@  static int pvrdma_pci_probe(struct pci_dev *pdev,
 	/* Make sure the write is complete before reading status. */
 	mb();
 
-	/* Currently, the driver only supports RoCE mode. */
-	if (dev->dsr->caps.mode != PVRDMA_DEVICE_MODE_ROCE) {
-		dev_err(&pdev->dev, "unsupported transport %d\n",
-			dev->dsr->caps.mode);
-		ret = -EFAULT;
-		goto err_free_cq_ring;
-	}
-
-	/* Currently, the driver only supports RoCE V1. */
-	if (!(dev->dsr->caps.gid_types & PVRDMA_GID_TYPE_FLAG_ROCE_V1)) {
-		dev_err(&pdev->dev, "driver needs RoCE v1 support\n");
+	/* The driver supports RoCE V1 and V2. */
+	if (!PVRDMA_SUPPORTED(dev)) {
+		dev_err(&pdev->dev, "driver needs RoCE v1 or v2 support\n");
 		ret = -EFAULT;
 		goto err_free_cq_ring;
 	}
diff --git a/drivers/infiniband/hw/vmw_pvrdma/pvrdma_misc.c b/drivers/infiniband/hw/vmw_pvrdma/pvrdma_misc.c
index ec6a4ca..fb0c5c0 100644
--- a/drivers/infiniband/hw/vmw_pvrdma/pvrdma_misc.c
+++ b/drivers/infiniband/hw/vmw_pvrdma/pvrdma_misc.c
@@ -303,3 +303,10 @@  void rdma_ah_attr_to_pvrdma(struct pvrdma_ah_attr *dst,
 	dst->port_num = rdma_ah_get_port_num(src);
 	memcpy(&dst->dmac, src->roce.dmac, sizeof(dst->dmac));
 }
+
+u8 ib_gid_type_to_pvrdma(enum ib_gid_type gid_type)
+{
+	return (gid_type == IB_GID_TYPE_ROCE_UDP_ENCAP) ?
+		PVRDMA_GID_TYPE_FLAG_ROCE_V2 :
+		PVRDMA_GID_TYPE_FLAG_ROCE_V1;
+}