diff mbox series

[for-next] RDMA: Get rid of iw_cm_verbs

Message ID 20190427093903.31546-1-kamalheib1@gmail.com (mailing list archive)
State Superseded
Headers show
Series [for-next] RDMA: Get rid of iw_cm_verbs | expand

Commit Message

Kamal Heib April 27, 2019, 9:39 a.m. UTC
Integrate iw_cm_verbs data members into ib_device_ops and ib_device
structs, this is don to achieve the following:

1- Avoid memory related bugs.
2- Make the code more cleaner.
3- Reduce code duplication.

Signed-off-by: Kamal Heib <kamalheib1@gmail.com>
---
 drivers/infiniband/core/device.c            |  8 +++++
 drivers/infiniband/core/iwcm.c              | 34 ++++++++++-----------
 drivers/infiniband/hw/cxgb3/iwch_provider.c | 32 +++++++------------
 drivers/infiniband/hw/cxgb4/provider.c      | 33 ++++++++------------
 drivers/infiniband/hw/i40iw/i40iw_verbs.c   | 30 ++++++------------
 drivers/infiniband/hw/nes/nes_verbs.c       | 27 ++++++----------
 drivers/infiniband/hw/qedr/main.c           | 25 ++++++---------
 include/rdma/ib_verbs.h                     | 22 ++++++++++---
 include/rdma/iw_cm.h                        | 25 ---------------
 9 files changed, 96 insertions(+), 140 deletions(-)

Comments

Michal Kalderon April 28, 2019, 6:28 a.m. UTC | #1
> From: Kamal Heib <kamalheib1@gmail.com>
> Sent: Saturday, April 27, 2019 12:39 PM
> 
> External Email
> 
> ----------------------------------------------------------------------
> Integrate iw_cm_verbs data members into ib_device_ops and ib_device
> structs, this is don to achieve the following:
> 
> 1- Avoid memory related bugs.
> 2- Make the code more cleaner.
> 3- Reduce code duplication.
> 
> Signed-off-by: Kamal Heib <kamalheib1@gmail.com>
> ---
>  drivers/infiniband/core/device.c            |  8 +++++
>  drivers/infiniband/core/iwcm.c              | 34 ++++++++++-----------
>  drivers/infiniband/hw/cxgb3/iwch_provider.c | 32 +++++++------------
>  drivers/infiniband/hw/cxgb4/provider.c      | 33 ++++++++------------
>  drivers/infiniband/hw/i40iw/i40iw_verbs.c   | 30 ++++++------------
>  drivers/infiniband/hw/nes/nes_verbs.c       | 27 ++++++----------
>  drivers/infiniband/hw/qedr/main.c           | 25 ++++++---------
>  include/rdma/ib_verbs.h                     | 22 ++++++++++---
>  include/rdma/iw_cm.h                        | 25 ---------------
>  9 files changed, 96 insertions(+), 140 deletions(-)
> 
> diff --git a/drivers/infiniband/core/device.c
> b/drivers/infiniband/core/device.c
> index fcbf2d4c865d..b47ba4863eed 100644
> --- a/drivers/infiniband/core/device.c
> +++ b/drivers/infiniband/core/device.c
> @@ -2345,6 +2345,14 @@ void ib_set_device_ops(struct ib_device *dev,
> const struct ib_device_ops *ops)
>  	SET_DEVICE_OP(dev_ops, set_vf_guid);
>  	SET_DEVICE_OP(dev_ops, set_vf_link_state);
>  	SET_DEVICE_OP(dev_ops, unmap_fmr);
> +	SET_DEVICE_OP(dev_ops, add_ref);
> +	SET_DEVICE_OP(dev_ops, rem_ref);
> +	SET_DEVICE_OP(dev_ops, get_qp);
> +	SET_DEVICE_OP(dev_ops, connect);
> +	SET_DEVICE_OP(dev_ops, accept);
> +	SET_DEVICE_OP(dev_ops, reject);
> +	SET_DEVICE_OP(dev_ops, create_listen);
> +	SET_DEVICE_OP(dev_ops, destroy_listen);
Perhaps there's added value in differentiating these ops as iWARP? Perhaps a prefix will suffice ? but at the least I guess a comment ? 
When they were in iw_cm it was very clear. 
  
> 
>  	SET_OBJ_SIZE(dev_ops, ib_ah);
>  	SET_OBJ_SIZE(dev_ops, ib_pd);
> diff --git a/drivers/infiniband/core/iwcm.c b/drivers/infiniband/core/iwcm.c
> index 732637c913d9..bba5ec8e03da 100644
> --- a/drivers/infiniband/core/iwcm.c
> +++ b/drivers/infiniband/core/iwcm.c
> @@ -394,7 +394,7 @@ static void destroy_cm_id(struct iw_cm_id *cm_id)
>  		cm_id_priv->state = IW_CM_STATE_DESTROYING;
>  		spin_unlock_irqrestore(&cm_id_priv->lock, flags);
>  		/* destroy the listening endpoint */
> -		cm_id->device->iwcm->destroy_listen(cm_id);
> +		cm_id->device->ops.destroy_listen(cm_id);
>  		spin_lock_irqsave(&cm_id_priv->lock, flags);
>  		break;
>  	case IW_CM_STATE_ESTABLISHED:
> @@ -417,7 +417,7 @@ static void destroy_cm_id(struct iw_cm_id *cm_id)
>  		 */
>  		cm_id_priv->state = IW_CM_STATE_DESTROYING;
>  		spin_unlock_irqrestore(&cm_id_priv->lock, flags);
> -		cm_id->device->iwcm->reject(cm_id, NULL, 0);
> +		cm_id->device->ops.reject(cm_id, NULL, 0);
>  		spin_lock_irqsave(&cm_id_priv->lock, flags);
>  		break;
>  	case IW_CM_STATE_CONN_SENT:
> @@ -427,7 +427,7 @@ static void destroy_cm_id(struct iw_cm_id *cm_id)
>  		break;
>  	}
>  	if (cm_id_priv->qp) {
> -		cm_id_priv->id.device->iwcm->rem_ref(cm_id_priv->qp);
> +		cm_id_priv->id.device->ops.rem_ref(cm_id_priv->qp);
>  		cm_id_priv->qp = NULL;
>  	}
>  	spin_unlock_irqrestore(&cm_id_priv->lock, flags); @@ -504,7 +504,7
> @@ static void iw_cm_check_wildcard(struct sockaddr_storage *pm_addr,
> static int iw_cm_map(struct iw_cm_id *cm_id, bool active)  {
>  	const char *devname = dev_name(&cm_id->device->dev);
> -	const char *ifname = cm_id->device->iwcm->ifname;
> +	const char *ifname = cm_id->device->ifname;
>  	struct iwpm_dev_data pm_reg_msg = {};
>  	struct iwpm_sa_data pm_msg;
>  	int status;
> @@ -526,7 +526,7 @@ static int iw_cm_map(struct iw_cm_id *cm_id, bool
> active)
>  	cm_id->mapped = true;
>  	pm_msg.loc_addr = cm_id->local_addr;
>  	pm_msg.rem_addr = cm_id->remote_addr;
> -	pm_msg.flags = (cm_id->device->iwcm->driver_flags &
> IW_F_NO_PORT_MAP) ?
> +	pm_msg.flags = (cm_id->device->driver_flags &
> IW_F_NO_PORT_MAP) ?
>  		       IWPM_FLAGS_NO_PORT_MAP : 0;
>  	if (active)
>  		status = iwpm_add_and_query_mapping(&pm_msg,
> @@ -577,7 +577,7 @@ int iw_cm_listen(struct iw_cm_id *cm_id, int backlog)
>  		spin_unlock_irqrestore(&cm_id_priv->lock, flags);
>  		ret = iw_cm_map(cm_id, false);
>  		if (!ret)
> -			ret = cm_id->device->iwcm->create_listen(cm_id,
> backlog);
> +			ret = cm_id->device->ops.create_listen(cm_id,
> backlog);
>  		if (ret)
>  			cm_id_priv->state = IW_CM_STATE_IDLE;
>  		spin_lock_irqsave(&cm_id_priv->lock, flags); @@ -617,7
> +617,7 @@ int iw_cm_reject(struct iw_cm_id *cm_id,
>  	cm_id_priv->state = IW_CM_STATE_IDLE;
>  	spin_unlock_irqrestore(&cm_id_priv->lock, flags);
> 
> -	ret = cm_id->device->iwcm->reject(cm_id, private_data,
> +	ret = cm_id->device->ops.reject(cm_id, private_data,
>  					  private_data_len);
> 
>  	clear_bit(IWCM_F_CONNECT_WAIT, &cm_id_priv->flags); @@ -
> 653,25 +653,25 @@ int iw_cm_accept(struct iw_cm_id *cm_id,
>  		return -EINVAL;
>  	}
>  	/* Get the ib_qp given the QPN */
> -	qp = cm_id->device->iwcm->get_qp(cm_id->device, iw_param-
> >qpn);
> +	qp = cm_id->device->ops.get_qp(cm_id->device, iw_param->qpn);
>  	if (!qp) {
>  		spin_unlock_irqrestore(&cm_id_priv->lock, flags);
>  		clear_bit(IWCM_F_CONNECT_WAIT, &cm_id_priv->flags);
>  		wake_up_all(&cm_id_priv->connect_wait);
>  		return -EINVAL;
>  	}
> -	cm_id->device->iwcm->add_ref(qp);
> +	cm_id->device->ops.add_ref(qp);
>  	cm_id_priv->qp = qp;
>  	spin_unlock_irqrestore(&cm_id_priv->lock, flags);
> 
> -	ret = cm_id->device->iwcm->accept(cm_id, iw_param);
> +	ret = cm_id->device->ops.accept(cm_id, iw_param);
>  	if (ret) {
>  		/* An error on accept precludes provider events */
>  		BUG_ON(cm_id_priv->state !=
> IW_CM_STATE_CONN_RECV);
>  		cm_id_priv->state = IW_CM_STATE_IDLE;
>  		spin_lock_irqsave(&cm_id_priv->lock, flags);
>  		if (cm_id_priv->qp) {
> -			cm_id->device->iwcm->rem_ref(qp);
> +			cm_id->device->ops.rem_ref(qp);
>  			cm_id_priv->qp = NULL;
>  		}
>  		spin_unlock_irqrestore(&cm_id_priv->lock, flags); @@ -
> 712,25 +712,25 @@ int iw_cm_connect(struct iw_cm_id *cm_id, struct
> iw_cm_conn_param *iw_param)
>  	}
> 
>  	/* Get the ib_qp given the QPN */
> -	qp = cm_id->device->iwcm->get_qp(cm_id->device, iw_param-
> >qpn);
> +	qp = cm_id->device->ops.get_qp(cm_id->device, iw_param->qpn);
>  	if (!qp) {
>  		ret = -EINVAL;
>  		goto err;
>  	}
> -	cm_id->device->iwcm->add_ref(qp);
> +	cm_id->device->ops.add_ref(qp);
>  	cm_id_priv->qp = qp;
>  	cm_id_priv->state = IW_CM_STATE_CONN_SENT;
>  	spin_unlock_irqrestore(&cm_id_priv->lock, flags);
> 
>  	ret = iw_cm_map(cm_id, true);
>  	if (!ret)
> -		ret = cm_id->device->iwcm->connect(cm_id, iw_param);
> +		ret = cm_id->device->ops.connect(cm_id, iw_param);
>  	if (!ret)
>  		return 0;	/* success */
> 
>  	spin_lock_irqsave(&cm_id_priv->lock, flags);
>  	if (cm_id_priv->qp) {
> -		cm_id->device->iwcm->rem_ref(qp);
> +		cm_id->device->ops.rem_ref(qp);
>  		cm_id_priv->qp = NULL;
>  	}
>  	cm_id_priv->state = IW_CM_STATE_IDLE;
> @@ -895,7 +895,7 @@ static int cm_conn_rep_handler(struct
> iwcm_id_private *cm_id_priv,
>  		cm_id_priv->state = IW_CM_STATE_ESTABLISHED;
>  	} else {
>  		/* REJECTED or RESET */
> -		cm_id_priv->id.device->iwcm->rem_ref(cm_id_priv->qp);
> +		cm_id_priv->id.device->ops.rem_ref(cm_id_priv->qp);
>  		cm_id_priv->qp = NULL;
>  		cm_id_priv->state = IW_CM_STATE_IDLE;
>  	}
> @@ -946,7 +946,7 @@ static int cm_close_handler(struct iwcm_id_private
> *cm_id_priv,
>  	spin_lock_irqsave(&cm_id_priv->lock, flags);
> 
>  	if (cm_id_priv->qp) {
> -		cm_id_priv->id.device->iwcm->rem_ref(cm_id_priv->qp);
> +		cm_id_priv->id.device->ops.rem_ref(cm_id_priv->qp);
>  		cm_id_priv->qp = NULL;
>  	}
>  	switch (cm_id_priv->state) {
> diff --git a/drivers/infiniband/hw/cxgb3/iwch_provider.c
> b/drivers/infiniband/hw/cxgb3/iwch_provider.c
> index 62b99d26f0d3..51ea75b19939 100644
> --- a/drivers/infiniband/hw/cxgb3/iwch_provider.c
> +++ b/drivers/infiniband/hw/cxgb3/iwch_provider.c
> @@ -1304,23 +1304,29 @@ static void get_dev_fw_ver_str(struct ib_device
> *ibdev, char *str)  }
> 
>  static const struct ib_device_ops iwch_dev_ops = {
> +	.accept = iwch_accept_cr,
> +	.add_ref = iwch_qp_add_ref,
>  	.alloc_hw_stats	= iwch_alloc_stats,
>  	.alloc_mr = iwch_alloc_mr,
>  	.alloc_mw = iwch_alloc_mw,
>  	.alloc_pd = iwch_allocate_pd,
>  	.alloc_ucontext = iwch_alloc_ucontext,
> +	.connect = iwch_connect,
>  	.create_cq = iwch_create_cq,
> +	.create_listen = iwch_create_listen,
>  	.create_qp = iwch_create_qp,
>  	.dealloc_mw = iwch_dealloc_mw,
>  	.dealloc_pd = iwch_deallocate_pd,
>  	.dealloc_ucontext = iwch_dealloc_ucontext,
>  	.dereg_mr = iwch_dereg_mr,
>  	.destroy_cq = iwch_destroy_cq,
> +	.destroy_listen = iwch_destroy_listen,
>  	.destroy_qp = iwch_destroy_qp,
>  	.get_dev_fw_str = get_dev_fw_ver_str,
>  	.get_dma_mr = iwch_get_dma_mr,
>  	.get_hw_stats = iwch_get_mib,
>  	.get_port_immutable = iwch_port_immutable,
> +	.get_qp = iwch_get_qp,
>  	.map_mr_sg = iwch_map_mr_sg,
>  	.mmap = iwch_mmap,
>  	.modify_qp = iwch_ib_modify_qp,
> @@ -1332,6 +1338,8 @@ static const struct ib_device_ops iwch_dev_ops = {
>  	.query_pkey = iwch_query_pkey,
>  	.query_port = iwch_query_port,
>  	.reg_user_mr = iwch_reg_user_mr,
> +	.reject = iwch_reject_cr,
> +	.rem_ref = iwch_qp_rem_ref,
>  	.req_notify_cq = iwch_arm_cq,
>  	.resize_cq = iwch_resize_cq,
>  	INIT_RDMA_OBJ_SIZE(ib_pd, iwch_pd, ibpd), @@ -1340,8 +1348,6
> @@ static const struct ib_device_ops iwch_dev_ops = {
> 
>  int iwch_register_device(struct iwch_dev *dev)  {
> -	int ret;
> -
>  	pr_debug("%s iwch_dev %p\n", __func__, dev);
>  	memset(&dev->ibdev.node_guid, 0, sizeof(dev->ibdev.node_guid));
>  	memcpy(&dev->ibdev.node_guid, dev->rdev.t3cdev_p->lldev-
> >dev_addr, 6); @@ -1379,34 +1385,18 @@ int iwch_register_device(struct
> iwch_dev *dev)
>  	dev->ibdev.dev.parent = &dev->rdev.rnic_info.pdev->dev;
>  	dev->ibdev.uverbs_abi_ver = IWCH_UVERBS_ABI_VERSION;
> 
> -	dev->ibdev.iwcm = kzalloc(sizeof(struct iw_cm_verbs),
> GFP_KERNEL);
> -	if (!dev->ibdev.iwcm)
> -		return -ENOMEM;
> -
> -	dev->ibdev.iwcm->connect = iwch_connect;
> -	dev->ibdev.iwcm->accept = iwch_accept_cr;
> -	dev->ibdev.iwcm->reject = iwch_reject_cr;
> -	dev->ibdev.iwcm->create_listen = iwch_create_listen;
> -	dev->ibdev.iwcm->destroy_listen = iwch_destroy_listen;
> -	dev->ibdev.iwcm->add_ref = iwch_qp_add_ref;
> -	dev->ibdev.iwcm->rem_ref = iwch_qp_rem_ref;
> -	dev->ibdev.iwcm->get_qp = iwch_get_qp;
> -	memcpy(dev->ibdev.iwcm->ifname, dev->rdev.t3cdev_p->lldev-
> >name,
> -	       sizeof(dev->ibdev.iwcm->ifname));
> +	memcpy(dev->ibdev.ifname, dev->rdev.t3cdev_p->lldev->name,
> +	       sizeof(dev->ibdev.ifname));
> 
>  	dev->ibdev.driver_id = RDMA_DRIVER_CXGB3;
>  	rdma_set_device_sysfs_group(&dev->ibdev, &iwch_attr_group);
>  	ib_set_device_ops(&dev->ibdev, &iwch_dev_ops);
> -	ret = ib_register_device(&dev->ibdev, "cxgb3_%d");
> -	if (ret)
> -		kfree(dev->ibdev.iwcm);
> -	return ret;
> +	return ib_register_device(&dev->ibdev, "cxgb3_%d");
>  }
> 
>  void iwch_unregister_device(struct iwch_dev *dev)  {
>  	pr_debug("%s iwch_dev %p\n", __func__, dev);
>  	ib_unregister_device(&dev->ibdev);
> -	kfree(dev->ibdev.iwcm);
>  	return;
>  }
> diff --git a/drivers/infiniband/hw/cxgb4/provider.c
> b/drivers/infiniband/hw/cxgb4/provider.c
> index 3c5197ee77f5..81f1a8b5221b 100644
> --- a/drivers/infiniband/hw/cxgb4/provider.c
> +++ b/drivers/infiniband/hw/cxgb4/provider.c
> @@ -490,12 +490,16 @@ static int fill_res_entry(struct sk_buff *msg, struct
> rdma_restrack_entry *res)  }
> 
>  static const struct ib_device_ops c4iw_dev_ops = {
> +	.accept = c4iw_accept_cr,
> +	.add_ref = c4iw_qp_add_ref,
>  	.alloc_hw_stats = c4iw_alloc_stats,
>  	.alloc_mr = c4iw_alloc_mr,
>  	.alloc_mw = c4iw_alloc_mw,
>  	.alloc_pd = c4iw_allocate_pd,
>  	.alloc_ucontext = c4iw_alloc_ucontext,
> +	.connect = c4iw_connect,
>  	.create_cq = c4iw_create_cq,
> +	.create_listen = c4iw_create_listen,
>  	.create_qp = c4iw_create_qp,
>  	.create_srq = c4iw_create_srq,
>  	.dealloc_mw = c4iw_dealloc_mw,
> @@ -503,6 +507,7 @@ static const struct ib_device_ops c4iw_dev_ops = {
>  	.dealloc_ucontext = c4iw_dealloc_ucontext,
>  	.dereg_mr = c4iw_dereg_mr,
>  	.destroy_cq = c4iw_destroy_cq,
> +	.destroy_listen = c4iw_destroy_listen,
>  	.destroy_qp = c4iw_destroy_qp,
>  	.destroy_srq = c4iw_destroy_srq,
>  	.fill_res_entry = fill_res_entry,
> @@ -510,6 +515,7 @@ static const struct ib_device_ops c4iw_dev_ops = {
>  	.get_dma_mr = c4iw_get_dma_mr,
>  	.get_hw_stats = c4iw_get_mib,
>  	.get_port_immutable = c4iw_port_immutable,
> +	.get_qp = c4iw_get_qp,
>  	.map_mr_sg = c4iw_map_mr_sg,
>  	.mmap = c4iw_mmap,
>  	.modify_qp = c4iw_ib_modify_qp,
> @@ -524,6 +530,8 @@ static const struct ib_device_ops c4iw_dev_ops = {
>  	.query_port = c4iw_query_port,
>  	.query_qp = c4iw_ib_query_qp,
>  	.reg_user_mr = c4iw_reg_user_mr,
> +	.reject = c4iw_reject_cr,
> +	.rem_ref = c4iw_qp_rem_ref,
>  	.req_notify_cq = c4iw_arm_cq,
>  	INIT_RDMA_OBJ_SIZE(ib_pd, c4iw_pd, ibpd),
>  	INIT_RDMA_OBJ_SIZE(ib_srq, c4iw_srq, ibsrq), @@ -588,36 +596,20
> @@ void c4iw_register_device(struct work_struct *work)
>  	dev->ibdev.dev.parent = &dev->rdev.lldi.pdev->dev;
>  	dev->ibdev.uverbs_abi_ver = C4IW_UVERBS_ABI_VERSION;
> 
> -	dev->ibdev.iwcm = kzalloc(sizeof(struct iw_cm_verbs),
> GFP_KERNEL);
> -	if (!dev->ibdev.iwcm) {
> -		ret = -ENOMEM;
> -		goto err_dealloc_ctx;
> -	}
> -
> -	dev->ibdev.iwcm->connect = c4iw_connect;
> -	dev->ibdev.iwcm->accept = c4iw_accept_cr;
> -	dev->ibdev.iwcm->reject = c4iw_reject_cr;
> -	dev->ibdev.iwcm->create_listen = c4iw_create_listen;
> -	dev->ibdev.iwcm->destroy_listen = c4iw_destroy_listen;
> -	dev->ibdev.iwcm->add_ref = c4iw_qp_add_ref;
> -	dev->ibdev.iwcm->rem_ref = c4iw_qp_rem_ref;
> -	dev->ibdev.iwcm->get_qp = c4iw_get_qp;
> -	memcpy(dev->ibdev.iwcm->ifname, dev->rdev.lldi.ports[0]->name,
> -	       sizeof(dev->ibdev.iwcm->ifname));
> +	memcpy(dev->ibdev.ifname, dev->rdev.lldi.ports[0]->name,
> +	       sizeof(dev->ibdev.ifname));
> 
>  	rdma_set_device_sysfs_group(&dev->ibdev, &c4iw_attr_group);
>  	dev->ibdev.driver_id = RDMA_DRIVER_CXGB4;
>  	ib_set_device_ops(&dev->ibdev, &c4iw_dev_ops);
>  	ret = set_netdevs(&dev->ibdev, &dev->rdev);
>  	if (ret)
> -		goto err_kfree_iwcm;
> +		goto err_dealloc_ctx;
>  	ret = ib_register_device(&dev->ibdev, "cxgb4_%d");
>  	if (ret)
> -		goto err_kfree_iwcm;
> +		goto err_dealloc_ctx;
>  	return;
> 
> -err_kfree_iwcm:
> -	kfree(dev->ibdev.iwcm);
>  err_dealloc_ctx:
>  	pr_err("%s - Failed registering iwarp device: %d\n",
>  	       pci_name(ctx->lldi.pdev), ret); @@ -629,6 +621,5 @@ void
> c4iw_unregister_device(struct c4iw_dev *dev)  {
>  	pr_debug("c4iw_dev %p\n", dev);
>  	ib_unregister_device(&dev->ibdev);
> -	kfree(dev->ibdev.iwcm);
>  	return;
>  }
> diff --git a/drivers/infiniband/hw/i40iw/i40iw_verbs.c
> b/drivers/infiniband/hw/i40iw/i40iw_verbs.c
> index 7bf7fe854464..5eb9ceee812a 100644
> --- a/drivers/infiniband/hw/i40iw/i40iw_verbs.c
> +++ b/drivers/infiniband/hw/i40iw/i40iw_verbs.c
> @@ -2687,16 +2687,21 @@ static int i40iw_query_pkey(struct ib_device
> *ibdev,  }
> 
>  static const struct ib_device_ops i40iw_dev_ops = {
> +	.accept = i40iw_accept,
> +	.add_ref = i40iw_add_ref,
>  	.alloc_hw_stats = i40iw_alloc_hw_stats,
>  	.alloc_mr = i40iw_alloc_mr,
>  	.alloc_pd = i40iw_alloc_pd,
>  	.alloc_ucontext = i40iw_alloc_ucontext,
> +	.connect = i40iw_connect,
>  	.create_cq = i40iw_create_cq,
> +	.create_listen = i40iw_create_listen,
>  	.create_qp = i40iw_create_qp,
>  	.dealloc_pd = i40iw_dealloc_pd,
>  	.dealloc_ucontext = i40iw_dealloc_ucontext,
>  	.dereg_mr = i40iw_dereg_mr,
>  	.destroy_cq = i40iw_destroy_cq,
> +	.destroy_listen = i40iw_destroy_listen,
>  	.destroy_qp = i40iw_destroy_qp,
>  	.drain_rq = i40iw_drain_rq,
>  	.drain_sq = i40iw_drain_sq,
> @@ -2704,6 +2709,7 @@ static const struct ib_device_ops i40iw_dev_ops = {
>  	.get_dma_mr = i40iw_get_dma_mr,
>  	.get_hw_stats = i40iw_get_hw_stats,
>  	.get_port_immutable = i40iw_port_immutable,
> +	.get_qp = i40iw_get_qp,
>  	.map_mr_sg = i40iw_map_mr_sg,
>  	.mmap = i40iw_mmap,
>  	.modify_qp = i40iw_modify_qp,
> @@ -2716,6 +2722,8 @@ static const struct ib_device_ops i40iw_dev_ops = {
>  	.query_port = i40iw_query_port,
>  	.query_qp = i40iw_query_qp,
>  	.reg_user_mr = i40iw_reg_user_mr,
> +	.reject = i40iw_reject,
> +	.rem_ref = i40iw_rem_ref,
>  	.req_notify_cq = i40iw_req_notify_cq,
>  	INIT_RDMA_OBJ_SIZE(ib_pd, i40iw_pd, ibpd),
>  	INIT_RDMA_OBJ_SIZE(ib_ucontext, i40iw_ucontext, ibucontext),
> @@ -2767,22 +2775,8 @@ static struct i40iw_ib_device
> *i40iw_init_rdma_device(struct i40iw_device *iwdev
>  	iwibdev->ibdev.phys_port_cnt = 1;
>  	iwibdev->ibdev.num_comp_vectors = iwdev->ceqs_count;
>  	iwibdev->ibdev.dev.parent = &pcidev->dev;
> -	iwibdev->ibdev.iwcm = kzalloc(sizeof(*iwibdev->ibdev.iwcm),
> GFP_KERNEL);
> -	if (!iwibdev->ibdev.iwcm) {
> -		ib_dealloc_device(&iwibdev->ibdev);
> -		return NULL;
> -	}
> -
> -	iwibdev->ibdev.iwcm->add_ref = i40iw_add_ref;
> -	iwibdev->ibdev.iwcm->rem_ref = i40iw_rem_ref;
> -	iwibdev->ibdev.iwcm->get_qp = i40iw_get_qp;
> -	iwibdev->ibdev.iwcm->connect = i40iw_connect;
> -	iwibdev->ibdev.iwcm->accept = i40iw_accept;
> -	iwibdev->ibdev.iwcm->reject = i40iw_reject;
> -	iwibdev->ibdev.iwcm->create_listen = i40iw_create_listen;
> -	iwibdev->ibdev.iwcm->destroy_listen = i40iw_destroy_listen;
> -	memcpy(iwibdev->ibdev.iwcm->ifname, netdev->name,
> -	       sizeof(iwibdev->ibdev.iwcm->ifname));
> +	memcpy(iwibdev->ibdev.ifname, netdev->name,
> +	       sizeof(iwibdev->ibdev.ifname));
>  	ib_set_device_ops(&iwibdev->ibdev, &i40iw_dev_ops);
> 
>  	return iwibdev;
> @@ -2813,8 +2807,6 @@ void i40iw_destroy_rdma_device(struct
> i40iw_ib_device *iwibdev)
>  		return;
> 
>  	ib_unregister_device(&iwibdev->ibdev);
> -	kfree(iwibdev->ibdev.iwcm);
> -	iwibdev->ibdev.iwcm = NULL;
>  	wait_event_timeout(iwibdev->iwdev->close_wq,
>  			   !atomic64_read(&iwibdev->iwdev->use_count),
>  			   I40IW_EVENT_TIMEOUT);
> @@ -2842,8 +2834,6 @@ int i40iw_register_rdma_device(struct i40iw_device
> *iwdev)
> 
>  	return 0;
>  error:
> -	kfree(iwdev->iwibdev->ibdev.iwcm);
> -	iwdev->iwibdev->ibdev.iwcm = NULL;
>  	ib_dealloc_device(&iwdev->iwibdev->ibdev);
>  	return ret;
>  }
> diff --git a/drivers/infiniband/hw/nes/nes_verbs.c
> b/drivers/infiniband/hw/nes/nes_verbs.c
> index a3b5e8eecb98..5f10784c3a34 100644
> --- a/drivers/infiniband/hw/nes/nes_verbs.c
> +++ b/drivers/infiniband/hw/nes/nes_verbs.c
> @@ -3560,23 +3560,29 @@ static void get_dev_fw_str(struct ib_device
> *dev, char *str)  }
> 
>  static const struct ib_device_ops nes_dev_ops = {
> +	.accept = nes_accept,
> +	.add_ref = nes_add_ref,
>  	.alloc_mr = nes_alloc_mr,
>  	.alloc_mw = nes_alloc_mw,
>  	.alloc_pd = nes_alloc_pd,
>  	.alloc_ucontext = nes_alloc_ucontext,
> +	.connect = nes_connect,
>  	.create_cq = nes_create_cq,
> +	.create_listen = nes_create_listen,
>  	.create_qp = nes_create_qp,
>  	.dealloc_mw = nes_dealloc_mw,
>  	.dealloc_pd = nes_dealloc_pd,
>  	.dealloc_ucontext = nes_dealloc_ucontext,
>  	.dereg_mr = nes_dereg_mr,
>  	.destroy_cq = nes_destroy_cq,
> +	.destroy_listen = nes_destroy_listen,
>  	.destroy_qp = nes_destroy_qp,
>  	.drain_rq = nes_drain_rq,
>  	.drain_sq = nes_drain_sq,
>  	.get_dev_fw_str = get_dev_fw_str,
>  	.get_dma_mr = nes_get_dma_mr,
>  	.get_port_immutable = nes_port_immutable,
> +	.get_qp = nes_get_qp,
>  	.map_mr_sg = nes_map_mr_sg,
>  	.mmap = nes_mmap,
>  	.modify_qp = nes_modify_qp,
> @@ -3589,6 +3595,8 @@ static const struct ib_device_ops nes_dev_ops = {
>  	.query_port = nes_query_port,
>  	.query_qp = nes_query_qp,
>  	.reg_user_mr = nes_reg_user_mr,
> +	.reject = nes_reject,
> +	.rem_ref = nes_rem_ref,
>  	.req_notify_cq = nes_req_notify_cq,
>  	INIT_RDMA_OBJ_SIZE(ib_pd, nes_pd, ibpd),
>  	INIT_RDMA_OBJ_SIZE(ib_ucontext, nes_ucontext, ibucontext), @@
> -3641,23 +3649,9 @@ struct nes_ib_device *nes_init_ofa_device(struct
> net_device *netdev)
>  	nesibdev->ibdev.num_comp_vectors = 1;
>  	nesibdev->ibdev.dev.parent = &nesdev->pcidev->dev;
> 
> -	nesibdev->ibdev.iwcm = kzalloc(sizeof(*nesibdev->ibdev.iwcm),
> GFP_KERNEL);
> -	if (nesibdev->ibdev.iwcm == NULL) {
> -		ib_dealloc_device(&nesibdev->ibdev);
> -		return NULL;
> -	}
> -	nesibdev->ibdev.iwcm->add_ref = nes_add_ref;
> -	nesibdev->ibdev.iwcm->rem_ref = nes_rem_ref;
> -	nesibdev->ibdev.iwcm->get_qp = nes_get_qp;
> -	nesibdev->ibdev.iwcm->connect = nes_connect;
> -	nesibdev->ibdev.iwcm->accept = nes_accept;
> -	nesibdev->ibdev.iwcm->reject = nes_reject;
> -	nesibdev->ibdev.iwcm->create_listen = nes_create_listen;
> -	nesibdev->ibdev.iwcm->destroy_listen = nes_destroy_listen;
> -
>  	ib_set_device_ops(&nesibdev->ibdev, &nes_dev_ops);
> -	memcpy(nesibdev->ibdev.iwcm->ifname, netdev->name,
> -	       sizeof(nesibdev->ibdev.iwcm->ifname));
> +	memcpy(nesibdev->ibdev.ifname, netdev->name,
> +	       sizeof(nesibdev->ibdev.ifname));
> 
>  	return nesibdev;
>  }
> @@ -3718,7 +3712,6 @@ void nes_destroy_ofa_device(struct nes_ib_device
> *nesibdev)
> 
>  	nes_unregister_ofa_device(nesibdev);
> 
> -	kfree(nesibdev->ibdev.iwcm);
>  	ib_dealloc_device(&nesibdev->ibdev);
>  }
> 
> diff --git a/drivers/infiniband/hw/qedr/main.c
> b/drivers/infiniband/hw/qedr/main.c
> index a0a49ed26860..db9a00457a6b 100644
> --- a/drivers/infiniband/hw/qedr/main.c
> +++ b/drivers/infiniband/hw/qedr/main.c
> @@ -147,8 +147,16 @@ static const struct attribute_group qedr_attr_group
> = {  };
> 
>  static const struct ib_device_ops qedr_iw_dev_ops = {
> +	.accept = qedr_iw_accept,
> +	.add_ref = qedr_iw_qp_add_ref,
> +	.connect = qedr_iw_connect,
> +	.create_listen = qedr_iw_create_listen,
> +	.destroy_listen = qedr_iw_destroy_listen,
>  	.get_port_immutable = qedr_iw_port_immutable,
> +	.get_qp = qedr_iw_get_qp,
>  	.query_gid = qedr_iw_query_gid,
> +	.reject = qedr_iw_reject,
> +	.rem_ref = qedr_iw_qp_rem_ref,
>  };
> 
>  static int qedr_iw_register_device(struct qedr_dev *dev) @@ -157,21
> +165,8 @@ static int qedr_iw_register_device(struct qedr_dev *dev)
> 
>  	ib_set_device_ops(&dev->ibdev, &qedr_iw_dev_ops);
> 
> -	dev->ibdev.iwcm = kzalloc(sizeof(*dev->ibdev.iwcm), GFP_KERNEL);
> -	if (!dev->ibdev.iwcm)
> -		return -ENOMEM;
> -
> -	dev->ibdev.iwcm->connect = qedr_iw_connect;
> -	dev->ibdev.iwcm->accept = qedr_iw_accept;
> -	dev->ibdev.iwcm->reject = qedr_iw_reject;
> -	dev->ibdev.iwcm->create_listen = qedr_iw_create_listen;
> -	dev->ibdev.iwcm->destroy_listen = qedr_iw_destroy_listen;
> -	dev->ibdev.iwcm->add_ref = qedr_iw_qp_add_ref;
> -	dev->ibdev.iwcm->rem_ref = qedr_iw_qp_rem_ref;
> -	dev->ibdev.iwcm->get_qp = qedr_iw_get_qp;
> -
> -	memcpy(dev->ibdev.iwcm->ifname,
> -	       dev->ndev->name, sizeof(dev->ibdev.iwcm->ifname));
> +	memcpy(dev->ibdev.ifname,
> +	       dev->ndev->name, sizeof(dev->ibdev.ifname));
> 
>  	return 0;
>  }
> diff --git a/include/rdma/ib_verbs.h b/include/rdma/ib_verbs.h index
> 43a75ab8ea8a..4e0d9107ce9b 100644
> --- a/include/rdma/ib_verbs.h
> +++ b/include/rdma/ib_verbs.h
> @@ -2191,8 +2191,6 @@ struct ib_cache {
>  	struct ib_event_handler event_handler;  };
> 
> -struct iw_cm_verbs;
> -
>  struct ib_port_immutable {
>  	int                           pkey_tbl_len;
>  	int                           gid_tbl_len;
> @@ -2274,6 +2272,8 @@ struct ib_counters_read_attr {  };
> 
>  struct uverbs_attr_bundle;
> +struct iw_cm_id;
> +struct iw_cm_conn_param;
> 
>  #define INIT_RDMA_OBJ_SIZE(ib_struct, drv_struct, member)                      \
>  	.size_##ib_struct =                                                    \
> @@ -2551,6 +2551,18 @@ struct ib_device_ops {
>  	 */
>  	void (*dealloc_driver)(struct ib_device *dev);
> 
> +	/* iWarp CM callbacks */
> +	void (*add_ref)(struct ib_qp *qp);
> +	void (*rem_ref)(struct ib_qp *qp);
> +	struct ib_qp *(*get_qp)(struct ib_device *device, int qpn);
> +	int (*connect)(struct iw_cm_id *cm_id,
> +		       struct iw_cm_conn_param *conn_param);
> +	int (*accept)(struct iw_cm_id *cm_id,
> +		      struct iw_cm_conn_param *conn_param);
> +	int (*reject)(struct iw_cm_id *cm_id, const void *pdata, u8
> pdata_len);
> +	int (*create_listen)(struct iw_cm_id *cm_id, int backlog);
> +	int (*destroy_listen)(struct iw_cm_id *cm_id);
> +
>  	DECLARE_RDMA_OBJ_SIZE(ib_ah);
>  	DECLARE_RDMA_OBJ_SIZE(ib_pd);
>  	DECLARE_RDMA_OBJ_SIZE(ib_srq);
> @@ -2591,8 +2603,6 @@ struct ib_device {
> 
>  	int			      num_comp_vectors;
> 
> -	struct iw_cm_verbs	     *iwcm;
> -
>  	struct module               *owner;
>  	union {
>  		struct device		dev;
> @@ -2645,6 +2655,10 @@ struct ib_device {
>  	struct mutex compat_devs_mutex;
>  	/* Maintains compat devices for each net namespace */
>  	struct xarray compat_devs;
> +
> +	/* Used by iWarp CM */
> +	char ifname[IFNAMSIZ];
> +	u32 driver_flags;
>  };
> 
>  struct ib_client {
> diff --git a/include/rdma/iw_cm.h b/include/rdma/iw_cm.h index
> 0e1f02815643..5aa8a9c76aa0 100644
> --- a/include/rdma/iw_cm.h
> +++ b/include/rdma/iw_cm.h
> @@ -118,31 +118,6 @@ enum iw_flags {
>  	IW_F_NO_PORT_MAP = (1 << 0),
>  };
> 
> -struct iw_cm_verbs {
> -	void		(*add_ref)(struct ib_qp *qp);
> -
> -	void		(*rem_ref)(struct ib_qp *qp);
> -
> -	struct ib_qp *	(*get_qp)(struct ib_device *device,
> -				  int qpn);
> -
> -	int		(*connect)(struct iw_cm_id *cm_id,
> -				   struct iw_cm_conn_param *conn_param);
> -
> -	int		(*accept)(struct iw_cm_id *cm_id,
> -				  struct iw_cm_conn_param *conn_param);
> -
> -	int		(*reject)(struct iw_cm_id *cm_id,
> -				  const void *pdata, u8 pdata_len);
> -
> -	int		(*create_listen)(struct iw_cm_id *cm_id,
> -					 int backlog);
> -
> -	int		(*destroy_listen)(struct iw_cm_id *cm_id);
> -	char		ifname[IFNAMSIZ];
> -	enum iw_flags	driver_flags;
> -};
> -
>  /**
>   * iw_create_cm_id - Create an IW CM identifier.
>   *
> --
> 2.20.1
Leon Romanovsky April 28, 2019, 7:34 a.m. UTC | #2
On Sun, Apr 28, 2019 at 06:28:55AM +0000, Michal Kalderon wrote:
> > From: Kamal Heib <kamalheib1@gmail.com>
> > Sent: Saturday, April 27, 2019 12:39 PM
> >
> > External Email
> >
> > ----------------------------------------------------------------------
> > Integrate iw_cm_verbs data members into ib_device_ops and ib_device
> > structs, this is don to achieve the following:
> >
> > 1- Avoid memory related bugs.
> > 2- Make the code more cleaner.
> > 3- Reduce code duplication.
> >
> > Signed-off-by: Kamal Heib <kamalheib1@gmail.com>
> > ---
> >  drivers/infiniband/core/device.c            |  8 +++++
> >  drivers/infiniband/core/iwcm.c              | 34 ++++++++++-----------
> >  drivers/infiniband/hw/cxgb3/iwch_provider.c | 32 +++++++------------
> >  drivers/infiniband/hw/cxgb4/provider.c      | 33 ++++++++------------
> >  drivers/infiniband/hw/i40iw/i40iw_verbs.c   | 30 ++++++------------
> >  drivers/infiniband/hw/nes/nes_verbs.c       | 27 ++++++----------
> >  drivers/infiniband/hw/qedr/main.c           | 25 ++++++---------
> >  include/rdma/ib_verbs.h                     | 22 ++++++++++---
> >  include/rdma/iw_cm.h                        | 25 ---------------
> >  9 files changed, 96 insertions(+), 140 deletions(-)
> >
> > diff --git a/drivers/infiniband/core/device.c
> > b/drivers/infiniband/core/device.c
> > index fcbf2d4c865d..b47ba4863eed 100644
> > --- a/drivers/infiniband/core/device.c
> > +++ b/drivers/infiniband/core/device.c
> > @@ -2345,6 +2345,14 @@ void ib_set_device_ops(struct ib_device *dev,
> > const struct ib_device_ops *ops)
> >  	SET_DEVICE_OP(dev_ops, set_vf_guid);
> >  	SET_DEVICE_OP(dev_ops, set_vf_link_state);
> >  	SET_DEVICE_OP(dev_ops, unmap_fmr);
> > +	SET_DEVICE_OP(dev_ops, add_ref);
> > +	SET_DEVICE_OP(dev_ops, rem_ref);
> > +	SET_DEVICE_OP(dev_ops, get_qp);
> > +	SET_DEVICE_OP(dev_ops, connect);
> > +	SET_DEVICE_OP(dev_ops, accept);
> > +	SET_DEVICE_OP(dev_ops, reject);
> > +	SET_DEVICE_OP(dev_ops, create_listen);
> > +	SET_DEVICE_OP(dev_ops, destroy_listen);
> Perhaps there's added value in differentiating these ops as iWARP? Perhaps a prefix will suffice ? but at the least I guess a comment ?
> When they were in iw_cm it was very clear.

Completely agree with request to add some prefix.

Thanks
Kamal Heib April 28, 2019, 9:14 a.m. UTC | #3
On 4/28/19 10:34 AM, Leon Romanovsky wrote:
> On Sun, Apr 28, 2019 at 06:28:55AM +0000, Michal Kalderon wrote:
>>> From: Kamal Heib <kamalheib1@gmail.com>
>>> Sent: Saturday, April 27, 2019 12:39 PM
>>>
>>> External Email
>>>
>>> ----------------------------------------------------------------------
>>> Integrate iw_cm_verbs data members into ib_device_ops and ib_device
>>> structs, this is don to achieve the following:
>>>
>>> 1- Avoid memory related bugs.
>>> 2- Make the code more cleaner.
>>> 3- Reduce code duplication.
>>>
>>> Signed-off-by: Kamal Heib <kamalheib1@gmail.com>
>>> ---
>>>  drivers/infiniband/core/device.c            |  8 +++++
>>>  drivers/infiniband/core/iwcm.c              | 34 ++++++++++-----------
>>>  drivers/infiniband/hw/cxgb3/iwch_provider.c | 32 +++++++------------
>>>  drivers/infiniband/hw/cxgb4/provider.c      | 33 ++++++++------------
>>>  drivers/infiniband/hw/i40iw/i40iw_verbs.c   | 30 ++++++------------
>>>  drivers/infiniband/hw/nes/nes_verbs.c       | 27 ++++++----------
>>>  drivers/infiniband/hw/qedr/main.c           | 25 ++++++---------
>>>  include/rdma/ib_verbs.h                     | 22 ++++++++++---
>>>  include/rdma/iw_cm.h                        | 25 ---------------
>>>  9 files changed, 96 insertions(+), 140 deletions(-)
>>>
>>> diff --git a/drivers/infiniband/core/device.c
>>> b/drivers/infiniband/core/device.c
>>> index fcbf2d4c865d..b47ba4863eed 100644
>>> --- a/drivers/infiniband/core/device.c
>>> +++ b/drivers/infiniband/core/device.c
>>> @@ -2345,6 +2345,14 @@ void ib_set_device_ops(struct ib_device *dev,
>>> const struct ib_device_ops *ops)
>>>  	SET_DEVICE_OP(dev_ops, set_vf_guid);
>>>  	SET_DEVICE_OP(dev_ops, set_vf_link_state);
>>>  	SET_DEVICE_OP(dev_ops, unmap_fmr);
>>> +	SET_DEVICE_OP(dev_ops, add_ref);
>>> +	SET_DEVICE_OP(dev_ops, rem_ref);
>>> +	SET_DEVICE_OP(dev_ops, get_qp);
>>> +	SET_DEVICE_OP(dev_ops, connect);
>>> +	SET_DEVICE_OP(dev_ops, accept);
>>> +	SET_DEVICE_OP(dev_ops, reject);
>>> +	SET_DEVICE_OP(dev_ops, create_listen);
>>> +	SET_DEVICE_OP(dev_ops, destroy_listen);
>> Perhaps there's added value in differentiating these ops as iWARP? Perhaps a prefix will suffice ? but at the least I guess a comment ?
>> When they were in iw_cm it was very clear.
> 
> Completely agree with request to add some prefix.
> 
> Thanks
> 

OK, I'll send a V2 with iw_cm_{*} prefix.

Thanks,
Kamal
Leon Romanovsky April 28, 2019, 10:56 a.m. UTC | #4
On Sun, Apr 28, 2019 at 12:14:42PM +0300, Kamal Heib wrote:
>
>
> On 4/28/19 10:34 AM, Leon Romanovsky wrote:
> > On Sun, Apr 28, 2019 at 06:28:55AM +0000, Michal Kalderon wrote:
> >>> From: Kamal Heib <kamalheib1@gmail.com>
> >>> Sent: Saturday, April 27, 2019 12:39 PM
> >>>
> >>> External Email
> >>>
> >>> ----------------------------------------------------------------------
> >>> Integrate iw_cm_verbs data members into ib_device_ops and ib_device
> >>> structs, this is don to achieve the following:
> >>>
> >>> 1- Avoid memory related bugs.
> >>> 2- Make the code more cleaner.
> >>> 3- Reduce code duplication.
> >>>
> >>> Signed-off-by: Kamal Heib <kamalheib1@gmail.com>
> >>> ---
> >>>  drivers/infiniband/core/device.c            |  8 +++++
> >>>  drivers/infiniband/core/iwcm.c              | 34 ++++++++++-----------
> >>>  drivers/infiniband/hw/cxgb3/iwch_provider.c | 32 +++++++------------
> >>>  drivers/infiniband/hw/cxgb4/provider.c      | 33 ++++++++------------
> >>>  drivers/infiniband/hw/i40iw/i40iw_verbs.c   | 30 ++++++------------
> >>>  drivers/infiniband/hw/nes/nes_verbs.c       | 27 ++++++----------
> >>>  drivers/infiniband/hw/qedr/main.c           | 25 ++++++---------
> >>>  include/rdma/ib_verbs.h                     | 22 ++++++++++---
> >>>  include/rdma/iw_cm.h                        | 25 ---------------
> >>>  9 files changed, 96 insertions(+), 140 deletions(-)
> >>>
> >>> diff --git a/drivers/infiniband/core/device.c
> >>> b/drivers/infiniband/core/device.c
> >>> index fcbf2d4c865d..b47ba4863eed 100644
> >>> --- a/drivers/infiniband/core/device.c
> >>> +++ b/drivers/infiniband/core/device.c
> >>> @@ -2345,6 +2345,14 @@ void ib_set_device_ops(struct ib_device *dev,
> >>> const struct ib_device_ops *ops)
> >>>  	SET_DEVICE_OP(dev_ops, set_vf_guid);
> >>>  	SET_DEVICE_OP(dev_ops, set_vf_link_state);
> >>>  	SET_DEVICE_OP(dev_ops, unmap_fmr);
> >>> +	SET_DEVICE_OP(dev_ops, add_ref);
> >>> +	SET_DEVICE_OP(dev_ops, rem_ref);
> >>> +	SET_DEVICE_OP(dev_ops, get_qp);
> >>> +	SET_DEVICE_OP(dev_ops, connect);
> >>> +	SET_DEVICE_OP(dev_ops, accept);
> >>> +	SET_DEVICE_OP(dev_ops, reject);
> >>> +	SET_DEVICE_OP(dev_ops, create_listen);
> >>> +	SET_DEVICE_OP(dev_ops, destroy_listen);
> >> Perhaps there's added value in differentiating these ops as iWARP? Perhaps a prefix will suffice ? but at the least I guess a comment ?
> >> When they were in iw_cm it was very clear.
> >
> > Completely agree with request to add some prefix.
> >
> > Thanks
> >
>
> OK, I'll send a V2 with iw_cm_{*} prefix.

iw_* for all iWARP specific calls will be enough.

Thanks

>
> Thanks,
> Kamal
Jason Gunthorpe April 28, 2019, 11:54 a.m. UTC | #5
On Sun, Apr 28, 2019 at 06:28:55AM +0000, Michal Kalderon wrote:
> > From: Kamal Heib <kamalheib1@gmail.com>
> > Sent: Saturday, April 27, 2019 12:39 PM
> > 
> > External Email
> > 
> > Integrate iw_cm_verbs data members into ib_device_ops and ib_device
> > structs, this is don to achieve the following:
> > 
> > 1- Avoid memory related bugs.
> > 2- Make the code more cleaner.
> > 3- Reduce code duplication.
> > 
> > Signed-off-by: Kamal Heib <kamalheib1@gmail.com>
> >  drivers/infiniband/core/device.c            |  8 +++++
> >  drivers/infiniband/core/iwcm.c              | 34 ++++++++++-----------
> >  drivers/infiniband/hw/cxgb3/iwch_provider.c | 32 +++++++------------
> >  drivers/infiniband/hw/cxgb4/provider.c      | 33 ++++++++------------
> >  drivers/infiniband/hw/i40iw/i40iw_verbs.c   | 30 ++++++------------
> >  drivers/infiniband/hw/nes/nes_verbs.c       | 27 ++++++----------
> >  drivers/infiniband/hw/qedr/main.c           | 25 ++++++---------
> >  include/rdma/ib_verbs.h                     | 22 ++++++++++---
> >  include/rdma/iw_cm.h                        | 25 ---------------
> >  9 files changed, 96 insertions(+), 140 deletions(-)
> > 
> > diff --git a/drivers/infiniband/core/device.c
> > b/drivers/infiniband/core/device.c
> > index fcbf2d4c865d..b47ba4863eed 100644
> > +++ b/drivers/infiniband/core/device.c
> > @@ -2345,6 +2345,14 @@ void ib_set_device_ops(struct ib_device *dev,
> > const struct ib_device_ops *ops)
> >  	SET_DEVICE_OP(dev_ops, set_vf_guid);
> >  	SET_DEVICE_OP(dev_ops, set_vf_link_state);
> >  	SET_DEVICE_OP(dev_ops, unmap_fmr);
> > +	SET_DEVICE_OP(dev_ops, add_ref);
> > +	SET_DEVICE_OP(dev_ops, rem_ref);
> > +	SET_DEVICE_OP(dev_ops, get_qp);
> > +	SET_DEVICE_OP(dev_ops, connect);
> > +	SET_DEVICE_OP(dev_ops, accept);
> > +	SET_DEVICE_OP(dev_ops, reject);
> > +	SET_DEVICE_OP(dev_ops, create_listen);
> > +	SET_DEVICE_OP(dev_ops, destroy_listen);
> Perhaps there's added value in differentiating these ops as iWARP? Perhaps a prefix will suffice ? but at the least I guess a comment ? 
> When they were in iw_cm it was very clear. 

Also keep sorted...

Jason
Jason Gunthorpe April 28, 2019, 11:56 a.m. UTC | #6
On Sat, Apr 27, 2019 at 12:39:03PM +0300, Kamal Heib wrote:

> @@ -2591,8 +2603,6 @@ struct ib_device {
>  
>  	int			      num_comp_vectors;
>  
> -	struct iw_cm_verbs	     *iwcm;
> -
>  	struct module               *owner;
>  	union {
>  		struct device		dev;
> @@ -2645,6 +2655,10 @@ struct ib_device {
>  	struct mutex compat_devs_mutex;
>  	/* Maintains compat devices for each net namespace */
>  	struct xarray compat_devs;
> +
> +	/* Used by iWarp CM */
> +	char ifname[IFNAMSIZ];

iw prefix here too please

I'd also love it if someone could explain what ifname is supposed to
be in the comment

Jason
diff mbox series

Patch

diff --git a/drivers/infiniband/core/device.c b/drivers/infiniband/core/device.c
index fcbf2d4c865d..b47ba4863eed 100644
--- a/drivers/infiniband/core/device.c
+++ b/drivers/infiniband/core/device.c
@@ -2345,6 +2345,14 @@  void ib_set_device_ops(struct ib_device *dev, const struct ib_device_ops *ops)
 	SET_DEVICE_OP(dev_ops, set_vf_guid);
 	SET_DEVICE_OP(dev_ops, set_vf_link_state);
 	SET_DEVICE_OP(dev_ops, unmap_fmr);
+	SET_DEVICE_OP(dev_ops, add_ref);
+	SET_DEVICE_OP(dev_ops, rem_ref);
+	SET_DEVICE_OP(dev_ops, get_qp);
+	SET_DEVICE_OP(dev_ops, connect);
+	SET_DEVICE_OP(dev_ops, accept);
+	SET_DEVICE_OP(dev_ops, reject);
+	SET_DEVICE_OP(dev_ops, create_listen);
+	SET_DEVICE_OP(dev_ops, destroy_listen);
 
 	SET_OBJ_SIZE(dev_ops, ib_ah);
 	SET_OBJ_SIZE(dev_ops, ib_pd);
diff --git a/drivers/infiniband/core/iwcm.c b/drivers/infiniband/core/iwcm.c
index 732637c913d9..bba5ec8e03da 100644
--- a/drivers/infiniband/core/iwcm.c
+++ b/drivers/infiniband/core/iwcm.c
@@ -394,7 +394,7 @@  static void destroy_cm_id(struct iw_cm_id *cm_id)
 		cm_id_priv->state = IW_CM_STATE_DESTROYING;
 		spin_unlock_irqrestore(&cm_id_priv->lock, flags);
 		/* destroy the listening endpoint */
-		cm_id->device->iwcm->destroy_listen(cm_id);
+		cm_id->device->ops.destroy_listen(cm_id);
 		spin_lock_irqsave(&cm_id_priv->lock, flags);
 		break;
 	case IW_CM_STATE_ESTABLISHED:
@@ -417,7 +417,7 @@  static void destroy_cm_id(struct iw_cm_id *cm_id)
 		 */
 		cm_id_priv->state = IW_CM_STATE_DESTROYING;
 		spin_unlock_irqrestore(&cm_id_priv->lock, flags);
-		cm_id->device->iwcm->reject(cm_id, NULL, 0);
+		cm_id->device->ops.reject(cm_id, NULL, 0);
 		spin_lock_irqsave(&cm_id_priv->lock, flags);
 		break;
 	case IW_CM_STATE_CONN_SENT:
@@ -427,7 +427,7 @@  static void destroy_cm_id(struct iw_cm_id *cm_id)
 		break;
 	}
 	if (cm_id_priv->qp) {
-		cm_id_priv->id.device->iwcm->rem_ref(cm_id_priv->qp);
+		cm_id_priv->id.device->ops.rem_ref(cm_id_priv->qp);
 		cm_id_priv->qp = NULL;
 	}
 	spin_unlock_irqrestore(&cm_id_priv->lock, flags);
@@ -504,7 +504,7 @@  static void iw_cm_check_wildcard(struct sockaddr_storage *pm_addr,
 static int iw_cm_map(struct iw_cm_id *cm_id, bool active)
 {
 	const char *devname = dev_name(&cm_id->device->dev);
-	const char *ifname = cm_id->device->iwcm->ifname;
+	const char *ifname = cm_id->device->ifname;
 	struct iwpm_dev_data pm_reg_msg = {};
 	struct iwpm_sa_data pm_msg;
 	int status;
@@ -526,7 +526,7 @@  static int iw_cm_map(struct iw_cm_id *cm_id, bool active)
 	cm_id->mapped = true;
 	pm_msg.loc_addr = cm_id->local_addr;
 	pm_msg.rem_addr = cm_id->remote_addr;
-	pm_msg.flags = (cm_id->device->iwcm->driver_flags & IW_F_NO_PORT_MAP) ?
+	pm_msg.flags = (cm_id->device->driver_flags & IW_F_NO_PORT_MAP) ?
 		       IWPM_FLAGS_NO_PORT_MAP : 0;
 	if (active)
 		status = iwpm_add_and_query_mapping(&pm_msg,
@@ -577,7 +577,7 @@  int iw_cm_listen(struct iw_cm_id *cm_id, int backlog)
 		spin_unlock_irqrestore(&cm_id_priv->lock, flags);
 		ret = iw_cm_map(cm_id, false);
 		if (!ret)
-			ret = cm_id->device->iwcm->create_listen(cm_id, backlog);
+			ret = cm_id->device->ops.create_listen(cm_id, backlog);
 		if (ret)
 			cm_id_priv->state = IW_CM_STATE_IDLE;
 		spin_lock_irqsave(&cm_id_priv->lock, flags);
@@ -617,7 +617,7 @@  int iw_cm_reject(struct iw_cm_id *cm_id,
 	cm_id_priv->state = IW_CM_STATE_IDLE;
 	spin_unlock_irqrestore(&cm_id_priv->lock, flags);
 
-	ret = cm_id->device->iwcm->reject(cm_id, private_data,
+	ret = cm_id->device->ops.reject(cm_id, private_data,
 					  private_data_len);
 
 	clear_bit(IWCM_F_CONNECT_WAIT, &cm_id_priv->flags);
@@ -653,25 +653,25 @@  int iw_cm_accept(struct iw_cm_id *cm_id,
 		return -EINVAL;
 	}
 	/* Get the ib_qp given the QPN */
-	qp = cm_id->device->iwcm->get_qp(cm_id->device, iw_param->qpn);
+	qp = cm_id->device->ops.get_qp(cm_id->device, iw_param->qpn);
 	if (!qp) {
 		spin_unlock_irqrestore(&cm_id_priv->lock, flags);
 		clear_bit(IWCM_F_CONNECT_WAIT, &cm_id_priv->flags);
 		wake_up_all(&cm_id_priv->connect_wait);
 		return -EINVAL;
 	}
-	cm_id->device->iwcm->add_ref(qp);
+	cm_id->device->ops.add_ref(qp);
 	cm_id_priv->qp = qp;
 	spin_unlock_irqrestore(&cm_id_priv->lock, flags);
 
-	ret = cm_id->device->iwcm->accept(cm_id, iw_param);
+	ret = cm_id->device->ops.accept(cm_id, iw_param);
 	if (ret) {
 		/* An error on accept precludes provider events */
 		BUG_ON(cm_id_priv->state != IW_CM_STATE_CONN_RECV);
 		cm_id_priv->state = IW_CM_STATE_IDLE;
 		spin_lock_irqsave(&cm_id_priv->lock, flags);
 		if (cm_id_priv->qp) {
-			cm_id->device->iwcm->rem_ref(qp);
+			cm_id->device->ops.rem_ref(qp);
 			cm_id_priv->qp = NULL;
 		}
 		spin_unlock_irqrestore(&cm_id_priv->lock, flags);
@@ -712,25 +712,25 @@  int iw_cm_connect(struct iw_cm_id *cm_id, struct iw_cm_conn_param *iw_param)
 	}
 
 	/* Get the ib_qp given the QPN */
-	qp = cm_id->device->iwcm->get_qp(cm_id->device, iw_param->qpn);
+	qp = cm_id->device->ops.get_qp(cm_id->device, iw_param->qpn);
 	if (!qp) {
 		ret = -EINVAL;
 		goto err;
 	}
-	cm_id->device->iwcm->add_ref(qp);
+	cm_id->device->ops.add_ref(qp);
 	cm_id_priv->qp = qp;
 	cm_id_priv->state = IW_CM_STATE_CONN_SENT;
 	spin_unlock_irqrestore(&cm_id_priv->lock, flags);
 
 	ret = iw_cm_map(cm_id, true);
 	if (!ret)
-		ret = cm_id->device->iwcm->connect(cm_id, iw_param);
+		ret = cm_id->device->ops.connect(cm_id, iw_param);
 	if (!ret)
 		return 0;	/* success */
 
 	spin_lock_irqsave(&cm_id_priv->lock, flags);
 	if (cm_id_priv->qp) {
-		cm_id->device->iwcm->rem_ref(qp);
+		cm_id->device->ops.rem_ref(qp);
 		cm_id_priv->qp = NULL;
 	}
 	cm_id_priv->state = IW_CM_STATE_IDLE;
@@ -895,7 +895,7 @@  static int cm_conn_rep_handler(struct iwcm_id_private *cm_id_priv,
 		cm_id_priv->state = IW_CM_STATE_ESTABLISHED;
 	} else {
 		/* REJECTED or RESET */
-		cm_id_priv->id.device->iwcm->rem_ref(cm_id_priv->qp);
+		cm_id_priv->id.device->ops.rem_ref(cm_id_priv->qp);
 		cm_id_priv->qp = NULL;
 		cm_id_priv->state = IW_CM_STATE_IDLE;
 	}
@@ -946,7 +946,7 @@  static int cm_close_handler(struct iwcm_id_private *cm_id_priv,
 	spin_lock_irqsave(&cm_id_priv->lock, flags);
 
 	if (cm_id_priv->qp) {
-		cm_id_priv->id.device->iwcm->rem_ref(cm_id_priv->qp);
+		cm_id_priv->id.device->ops.rem_ref(cm_id_priv->qp);
 		cm_id_priv->qp = NULL;
 	}
 	switch (cm_id_priv->state) {
diff --git a/drivers/infiniband/hw/cxgb3/iwch_provider.c b/drivers/infiniband/hw/cxgb3/iwch_provider.c
index 62b99d26f0d3..51ea75b19939 100644
--- a/drivers/infiniband/hw/cxgb3/iwch_provider.c
+++ b/drivers/infiniband/hw/cxgb3/iwch_provider.c
@@ -1304,23 +1304,29 @@  static void get_dev_fw_ver_str(struct ib_device *ibdev, char *str)
 }
 
 static const struct ib_device_ops iwch_dev_ops = {
+	.accept = iwch_accept_cr,
+	.add_ref = iwch_qp_add_ref,
 	.alloc_hw_stats	= iwch_alloc_stats,
 	.alloc_mr = iwch_alloc_mr,
 	.alloc_mw = iwch_alloc_mw,
 	.alloc_pd = iwch_allocate_pd,
 	.alloc_ucontext = iwch_alloc_ucontext,
+	.connect = iwch_connect,
 	.create_cq = iwch_create_cq,
+	.create_listen = iwch_create_listen,
 	.create_qp = iwch_create_qp,
 	.dealloc_mw = iwch_dealloc_mw,
 	.dealloc_pd = iwch_deallocate_pd,
 	.dealloc_ucontext = iwch_dealloc_ucontext,
 	.dereg_mr = iwch_dereg_mr,
 	.destroy_cq = iwch_destroy_cq,
+	.destroy_listen = iwch_destroy_listen,
 	.destroy_qp = iwch_destroy_qp,
 	.get_dev_fw_str = get_dev_fw_ver_str,
 	.get_dma_mr = iwch_get_dma_mr,
 	.get_hw_stats = iwch_get_mib,
 	.get_port_immutable = iwch_port_immutable,
+	.get_qp = iwch_get_qp,
 	.map_mr_sg = iwch_map_mr_sg,
 	.mmap = iwch_mmap,
 	.modify_qp = iwch_ib_modify_qp,
@@ -1332,6 +1338,8 @@  static const struct ib_device_ops iwch_dev_ops = {
 	.query_pkey = iwch_query_pkey,
 	.query_port = iwch_query_port,
 	.reg_user_mr = iwch_reg_user_mr,
+	.reject = iwch_reject_cr,
+	.rem_ref = iwch_qp_rem_ref,
 	.req_notify_cq = iwch_arm_cq,
 	.resize_cq = iwch_resize_cq,
 	INIT_RDMA_OBJ_SIZE(ib_pd, iwch_pd, ibpd),
@@ -1340,8 +1348,6 @@  static const struct ib_device_ops iwch_dev_ops = {
 
 int iwch_register_device(struct iwch_dev *dev)
 {
-	int ret;
-
 	pr_debug("%s iwch_dev %p\n", __func__, dev);
 	memset(&dev->ibdev.node_guid, 0, sizeof(dev->ibdev.node_guid));
 	memcpy(&dev->ibdev.node_guid, dev->rdev.t3cdev_p->lldev->dev_addr, 6);
@@ -1379,34 +1385,18 @@  int iwch_register_device(struct iwch_dev *dev)
 	dev->ibdev.dev.parent = &dev->rdev.rnic_info.pdev->dev;
 	dev->ibdev.uverbs_abi_ver = IWCH_UVERBS_ABI_VERSION;
 
-	dev->ibdev.iwcm = kzalloc(sizeof(struct iw_cm_verbs), GFP_KERNEL);
-	if (!dev->ibdev.iwcm)
-		return -ENOMEM;
-
-	dev->ibdev.iwcm->connect = iwch_connect;
-	dev->ibdev.iwcm->accept = iwch_accept_cr;
-	dev->ibdev.iwcm->reject = iwch_reject_cr;
-	dev->ibdev.iwcm->create_listen = iwch_create_listen;
-	dev->ibdev.iwcm->destroy_listen = iwch_destroy_listen;
-	dev->ibdev.iwcm->add_ref = iwch_qp_add_ref;
-	dev->ibdev.iwcm->rem_ref = iwch_qp_rem_ref;
-	dev->ibdev.iwcm->get_qp = iwch_get_qp;
-	memcpy(dev->ibdev.iwcm->ifname, dev->rdev.t3cdev_p->lldev->name,
-	       sizeof(dev->ibdev.iwcm->ifname));
+	memcpy(dev->ibdev.ifname, dev->rdev.t3cdev_p->lldev->name,
+	       sizeof(dev->ibdev.ifname));
 
 	dev->ibdev.driver_id = RDMA_DRIVER_CXGB3;
 	rdma_set_device_sysfs_group(&dev->ibdev, &iwch_attr_group);
 	ib_set_device_ops(&dev->ibdev, &iwch_dev_ops);
-	ret = ib_register_device(&dev->ibdev, "cxgb3_%d");
-	if (ret)
-		kfree(dev->ibdev.iwcm);
-	return ret;
+	return ib_register_device(&dev->ibdev, "cxgb3_%d");
 }
 
 void iwch_unregister_device(struct iwch_dev *dev)
 {
 	pr_debug("%s iwch_dev %p\n", __func__, dev);
 	ib_unregister_device(&dev->ibdev);
-	kfree(dev->ibdev.iwcm);
 	return;
 }
diff --git a/drivers/infiniband/hw/cxgb4/provider.c b/drivers/infiniband/hw/cxgb4/provider.c
index 3c5197ee77f5..81f1a8b5221b 100644
--- a/drivers/infiniband/hw/cxgb4/provider.c
+++ b/drivers/infiniband/hw/cxgb4/provider.c
@@ -490,12 +490,16 @@  static int fill_res_entry(struct sk_buff *msg, struct rdma_restrack_entry *res)
 }
 
 static const struct ib_device_ops c4iw_dev_ops = {
+	.accept = c4iw_accept_cr,
+	.add_ref = c4iw_qp_add_ref,
 	.alloc_hw_stats = c4iw_alloc_stats,
 	.alloc_mr = c4iw_alloc_mr,
 	.alloc_mw = c4iw_alloc_mw,
 	.alloc_pd = c4iw_allocate_pd,
 	.alloc_ucontext = c4iw_alloc_ucontext,
+	.connect = c4iw_connect,
 	.create_cq = c4iw_create_cq,
+	.create_listen = c4iw_create_listen,
 	.create_qp = c4iw_create_qp,
 	.create_srq = c4iw_create_srq,
 	.dealloc_mw = c4iw_dealloc_mw,
@@ -503,6 +507,7 @@  static const struct ib_device_ops c4iw_dev_ops = {
 	.dealloc_ucontext = c4iw_dealloc_ucontext,
 	.dereg_mr = c4iw_dereg_mr,
 	.destroy_cq = c4iw_destroy_cq,
+	.destroy_listen = c4iw_destroy_listen,
 	.destroy_qp = c4iw_destroy_qp,
 	.destroy_srq = c4iw_destroy_srq,
 	.fill_res_entry = fill_res_entry,
@@ -510,6 +515,7 @@  static const struct ib_device_ops c4iw_dev_ops = {
 	.get_dma_mr = c4iw_get_dma_mr,
 	.get_hw_stats = c4iw_get_mib,
 	.get_port_immutable = c4iw_port_immutable,
+	.get_qp = c4iw_get_qp,
 	.map_mr_sg = c4iw_map_mr_sg,
 	.mmap = c4iw_mmap,
 	.modify_qp = c4iw_ib_modify_qp,
@@ -524,6 +530,8 @@  static const struct ib_device_ops c4iw_dev_ops = {
 	.query_port = c4iw_query_port,
 	.query_qp = c4iw_ib_query_qp,
 	.reg_user_mr = c4iw_reg_user_mr,
+	.reject = c4iw_reject_cr,
+	.rem_ref = c4iw_qp_rem_ref,
 	.req_notify_cq = c4iw_arm_cq,
 	INIT_RDMA_OBJ_SIZE(ib_pd, c4iw_pd, ibpd),
 	INIT_RDMA_OBJ_SIZE(ib_srq, c4iw_srq, ibsrq),
@@ -588,36 +596,20 @@  void c4iw_register_device(struct work_struct *work)
 	dev->ibdev.dev.parent = &dev->rdev.lldi.pdev->dev;
 	dev->ibdev.uverbs_abi_ver = C4IW_UVERBS_ABI_VERSION;
 
-	dev->ibdev.iwcm = kzalloc(sizeof(struct iw_cm_verbs), GFP_KERNEL);
-	if (!dev->ibdev.iwcm) {
-		ret = -ENOMEM;
-		goto err_dealloc_ctx;
-	}
-
-	dev->ibdev.iwcm->connect = c4iw_connect;
-	dev->ibdev.iwcm->accept = c4iw_accept_cr;
-	dev->ibdev.iwcm->reject = c4iw_reject_cr;
-	dev->ibdev.iwcm->create_listen = c4iw_create_listen;
-	dev->ibdev.iwcm->destroy_listen = c4iw_destroy_listen;
-	dev->ibdev.iwcm->add_ref = c4iw_qp_add_ref;
-	dev->ibdev.iwcm->rem_ref = c4iw_qp_rem_ref;
-	dev->ibdev.iwcm->get_qp = c4iw_get_qp;
-	memcpy(dev->ibdev.iwcm->ifname, dev->rdev.lldi.ports[0]->name,
-	       sizeof(dev->ibdev.iwcm->ifname));
+	memcpy(dev->ibdev.ifname, dev->rdev.lldi.ports[0]->name,
+	       sizeof(dev->ibdev.ifname));
 
 	rdma_set_device_sysfs_group(&dev->ibdev, &c4iw_attr_group);
 	dev->ibdev.driver_id = RDMA_DRIVER_CXGB4;
 	ib_set_device_ops(&dev->ibdev, &c4iw_dev_ops);
 	ret = set_netdevs(&dev->ibdev, &dev->rdev);
 	if (ret)
-		goto err_kfree_iwcm;
+		goto err_dealloc_ctx;
 	ret = ib_register_device(&dev->ibdev, "cxgb4_%d");
 	if (ret)
-		goto err_kfree_iwcm;
+		goto err_dealloc_ctx;
 	return;
 
-err_kfree_iwcm:
-	kfree(dev->ibdev.iwcm);
 err_dealloc_ctx:
 	pr_err("%s - Failed registering iwarp device: %d\n",
 	       pci_name(ctx->lldi.pdev), ret);
@@ -629,6 +621,5 @@  void c4iw_unregister_device(struct c4iw_dev *dev)
 {
 	pr_debug("c4iw_dev %p\n", dev);
 	ib_unregister_device(&dev->ibdev);
-	kfree(dev->ibdev.iwcm);
 	return;
 }
diff --git a/drivers/infiniband/hw/i40iw/i40iw_verbs.c b/drivers/infiniband/hw/i40iw/i40iw_verbs.c
index 7bf7fe854464..5eb9ceee812a 100644
--- a/drivers/infiniband/hw/i40iw/i40iw_verbs.c
+++ b/drivers/infiniband/hw/i40iw/i40iw_verbs.c
@@ -2687,16 +2687,21 @@  static int i40iw_query_pkey(struct ib_device *ibdev,
 }
 
 static const struct ib_device_ops i40iw_dev_ops = {
+	.accept = i40iw_accept,
+	.add_ref = i40iw_add_ref,
 	.alloc_hw_stats = i40iw_alloc_hw_stats,
 	.alloc_mr = i40iw_alloc_mr,
 	.alloc_pd = i40iw_alloc_pd,
 	.alloc_ucontext = i40iw_alloc_ucontext,
+	.connect = i40iw_connect,
 	.create_cq = i40iw_create_cq,
+	.create_listen = i40iw_create_listen,
 	.create_qp = i40iw_create_qp,
 	.dealloc_pd = i40iw_dealloc_pd,
 	.dealloc_ucontext = i40iw_dealloc_ucontext,
 	.dereg_mr = i40iw_dereg_mr,
 	.destroy_cq = i40iw_destroy_cq,
+	.destroy_listen = i40iw_destroy_listen,
 	.destroy_qp = i40iw_destroy_qp,
 	.drain_rq = i40iw_drain_rq,
 	.drain_sq = i40iw_drain_sq,
@@ -2704,6 +2709,7 @@  static const struct ib_device_ops i40iw_dev_ops = {
 	.get_dma_mr = i40iw_get_dma_mr,
 	.get_hw_stats = i40iw_get_hw_stats,
 	.get_port_immutable = i40iw_port_immutable,
+	.get_qp = i40iw_get_qp,
 	.map_mr_sg = i40iw_map_mr_sg,
 	.mmap = i40iw_mmap,
 	.modify_qp = i40iw_modify_qp,
@@ -2716,6 +2722,8 @@  static const struct ib_device_ops i40iw_dev_ops = {
 	.query_port = i40iw_query_port,
 	.query_qp = i40iw_query_qp,
 	.reg_user_mr = i40iw_reg_user_mr,
+	.reject = i40iw_reject,
+	.rem_ref = i40iw_rem_ref,
 	.req_notify_cq = i40iw_req_notify_cq,
 	INIT_RDMA_OBJ_SIZE(ib_pd, i40iw_pd, ibpd),
 	INIT_RDMA_OBJ_SIZE(ib_ucontext, i40iw_ucontext, ibucontext),
@@ -2767,22 +2775,8 @@  static struct i40iw_ib_device *i40iw_init_rdma_device(struct i40iw_device *iwdev
 	iwibdev->ibdev.phys_port_cnt = 1;
 	iwibdev->ibdev.num_comp_vectors = iwdev->ceqs_count;
 	iwibdev->ibdev.dev.parent = &pcidev->dev;
-	iwibdev->ibdev.iwcm = kzalloc(sizeof(*iwibdev->ibdev.iwcm), GFP_KERNEL);
-	if (!iwibdev->ibdev.iwcm) {
-		ib_dealloc_device(&iwibdev->ibdev);
-		return NULL;
-	}
-
-	iwibdev->ibdev.iwcm->add_ref = i40iw_add_ref;
-	iwibdev->ibdev.iwcm->rem_ref = i40iw_rem_ref;
-	iwibdev->ibdev.iwcm->get_qp = i40iw_get_qp;
-	iwibdev->ibdev.iwcm->connect = i40iw_connect;
-	iwibdev->ibdev.iwcm->accept = i40iw_accept;
-	iwibdev->ibdev.iwcm->reject = i40iw_reject;
-	iwibdev->ibdev.iwcm->create_listen = i40iw_create_listen;
-	iwibdev->ibdev.iwcm->destroy_listen = i40iw_destroy_listen;
-	memcpy(iwibdev->ibdev.iwcm->ifname, netdev->name,
-	       sizeof(iwibdev->ibdev.iwcm->ifname));
+	memcpy(iwibdev->ibdev.ifname, netdev->name,
+	       sizeof(iwibdev->ibdev.ifname));
 	ib_set_device_ops(&iwibdev->ibdev, &i40iw_dev_ops);
 
 	return iwibdev;
@@ -2813,8 +2807,6 @@  void i40iw_destroy_rdma_device(struct i40iw_ib_device *iwibdev)
 		return;
 
 	ib_unregister_device(&iwibdev->ibdev);
-	kfree(iwibdev->ibdev.iwcm);
-	iwibdev->ibdev.iwcm = NULL;
 	wait_event_timeout(iwibdev->iwdev->close_wq,
 			   !atomic64_read(&iwibdev->iwdev->use_count),
 			   I40IW_EVENT_TIMEOUT);
@@ -2842,8 +2834,6 @@  int i40iw_register_rdma_device(struct i40iw_device *iwdev)
 
 	return 0;
 error:
-	kfree(iwdev->iwibdev->ibdev.iwcm);
-	iwdev->iwibdev->ibdev.iwcm = NULL;
 	ib_dealloc_device(&iwdev->iwibdev->ibdev);
 	return ret;
 }
diff --git a/drivers/infiniband/hw/nes/nes_verbs.c b/drivers/infiniband/hw/nes/nes_verbs.c
index a3b5e8eecb98..5f10784c3a34 100644
--- a/drivers/infiniband/hw/nes/nes_verbs.c
+++ b/drivers/infiniband/hw/nes/nes_verbs.c
@@ -3560,23 +3560,29 @@  static void get_dev_fw_str(struct ib_device *dev, char *str)
 }
 
 static const struct ib_device_ops nes_dev_ops = {
+	.accept = nes_accept,
+	.add_ref = nes_add_ref,
 	.alloc_mr = nes_alloc_mr,
 	.alloc_mw = nes_alloc_mw,
 	.alloc_pd = nes_alloc_pd,
 	.alloc_ucontext = nes_alloc_ucontext,
+	.connect = nes_connect,
 	.create_cq = nes_create_cq,
+	.create_listen = nes_create_listen,
 	.create_qp = nes_create_qp,
 	.dealloc_mw = nes_dealloc_mw,
 	.dealloc_pd = nes_dealloc_pd,
 	.dealloc_ucontext = nes_dealloc_ucontext,
 	.dereg_mr = nes_dereg_mr,
 	.destroy_cq = nes_destroy_cq,
+	.destroy_listen = nes_destroy_listen,
 	.destroy_qp = nes_destroy_qp,
 	.drain_rq = nes_drain_rq,
 	.drain_sq = nes_drain_sq,
 	.get_dev_fw_str = get_dev_fw_str,
 	.get_dma_mr = nes_get_dma_mr,
 	.get_port_immutable = nes_port_immutable,
+	.get_qp = nes_get_qp,
 	.map_mr_sg = nes_map_mr_sg,
 	.mmap = nes_mmap,
 	.modify_qp = nes_modify_qp,
@@ -3589,6 +3595,8 @@  static const struct ib_device_ops nes_dev_ops = {
 	.query_port = nes_query_port,
 	.query_qp = nes_query_qp,
 	.reg_user_mr = nes_reg_user_mr,
+	.reject = nes_reject,
+	.rem_ref = nes_rem_ref,
 	.req_notify_cq = nes_req_notify_cq,
 	INIT_RDMA_OBJ_SIZE(ib_pd, nes_pd, ibpd),
 	INIT_RDMA_OBJ_SIZE(ib_ucontext, nes_ucontext, ibucontext),
@@ -3641,23 +3649,9 @@  struct nes_ib_device *nes_init_ofa_device(struct net_device *netdev)
 	nesibdev->ibdev.num_comp_vectors = 1;
 	nesibdev->ibdev.dev.parent = &nesdev->pcidev->dev;
 
-	nesibdev->ibdev.iwcm = kzalloc(sizeof(*nesibdev->ibdev.iwcm), GFP_KERNEL);
-	if (nesibdev->ibdev.iwcm == NULL) {
-		ib_dealloc_device(&nesibdev->ibdev);
-		return NULL;
-	}
-	nesibdev->ibdev.iwcm->add_ref = nes_add_ref;
-	nesibdev->ibdev.iwcm->rem_ref = nes_rem_ref;
-	nesibdev->ibdev.iwcm->get_qp = nes_get_qp;
-	nesibdev->ibdev.iwcm->connect = nes_connect;
-	nesibdev->ibdev.iwcm->accept = nes_accept;
-	nesibdev->ibdev.iwcm->reject = nes_reject;
-	nesibdev->ibdev.iwcm->create_listen = nes_create_listen;
-	nesibdev->ibdev.iwcm->destroy_listen = nes_destroy_listen;
-
 	ib_set_device_ops(&nesibdev->ibdev, &nes_dev_ops);
-	memcpy(nesibdev->ibdev.iwcm->ifname, netdev->name,
-	       sizeof(nesibdev->ibdev.iwcm->ifname));
+	memcpy(nesibdev->ibdev.ifname, netdev->name,
+	       sizeof(nesibdev->ibdev.ifname));
 
 	return nesibdev;
 }
@@ -3718,7 +3712,6 @@  void nes_destroy_ofa_device(struct nes_ib_device *nesibdev)
 
 	nes_unregister_ofa_device(nesibdev);
 
-	kfree(nesibdev->ibdev.iwcm);
 	ib_dealloc_device(&nesibdev->ibdev);
 }
 
diff --git a/drivers/infiniband/hw/qedr/main.c b/drivers/infiniband/hw/qedr/main.c
index a0a49ed26860..db9a00457a6b 100644
--- a/drivers/infiniband/hw/qedr/main.c
+++ b/drivers/infiniband/hw/qedr/main.c
@@ -147,8 +147,16 @@  static const struct attribute_group qedr_attr_group = {
 };
 
 static const struct ib_device_ops qedr_iw_dev_ops = {
+	.accept = qedr_iw_accept,
+	.add_ref = qedr_iw_qp_add_ref,
+	.connect = qedr_iw_connect,
+	.create_listen = qedr_iw_create_listen,
+	.destroy_listen = qedr_iw_destroy_listen,
 	.get_port_immutable = qedr_iw_port_immutable,
+	.get_qp = qedr_iw_get_qp,
 	.query_gid = qedr_iw_query_gid,
+	.reject = qedr_iw_reject,
+	.rem_ref = qedr_iw_qp_rem_ref,
 };
 
 static int qedr_iw_register_device(struct qedr_dev *dev)
@@ -157,21 +165,8 @@  static int qedr_iw_register_device(struct qedr_dev *dev)
 
 	ib_set_device_ops(&dev->ibdev, &qedr_iw_dev_ops);
 
-	dev->ibdev.iwcm = kzalloc(sizeof(*dev->ibdev.iwcm), GFP_KERNEL);
-	if (!dev->ibdev.iwcm)
-		return -ENOMEM;
-
-	dev->ibdev.iwcm->connect = qedr_iw_connect;
-	dev->ibdev.iwcm->accept = qedr_iw_accept;
-	dev->ibdev.iwcm->reject = qedr_iw_reject;
-	dev->ibdev.iwcm->create_listen = qedr_iw_create_listen;
-	dev->ibdev.iwcm->destroy_listen = qedr_iw_destroy_listen;
-	dev->ibdev.iwcm->add_ref = qedr_iw_qp_add_ref;
-	dev->ibdev.iwcm->rem_ref = qedr_iw_qp_rem_ref;
-	dev->ibdev.iwcm->get_qp = qedr_iw_get_qp;
-
-	memcpy(dev->ibdev.iwcm->ifname,
-	       dev->ndev->name, sizeof(dev->ibdev.iwcm->ifname));
+	memcpy(dev->ibdev.ifname,
+	       dev->ndev->name, sizeof(dev->ibdev.ifname));
 
 	return 0;
 }
diff --git a/include/rdma/ib_verbs.h b/include/rdma/ib_verbs.h
index 43a75ab8ea8a..4e0d9107ce9b 100644
--- a/include/rdma/ib_verbs.h
+++ b/include/rdma/ib_verbs.h
@@ -2191,8 +2191,6 @@  struct ib_cache {
 	struct ib_event_handler event_handler;
 };
 
-struct iw_cm_verbs;
-
 struct ib_port_immutable {
 	int                           pkey_tbl_len;
 	int                           gid_tbl_len;
@@ -2274,6 +2272,8 @@  struct ib_counters_read_attr {
 };
 
 struct uverbs_attr_bundle;
+struct iw_cm_id;
+struct iw_cm_conn_param;
 
 #define INIT_RDMA_OBJ_SIZE(ib_struct, drv_struct, member)                      \
 	.size_##ib_struct =                                                    \
@@ -2551,6 +2551,18 @@  struct ib_device_ops {
 	 */
 	void (*dealloc_driver)(struct ib_device *dev);
 
+	/* iWarp CM callbacks */
+	void (*add_ref)(struct ib_qp *qp);
+	void (*rem_ref)(struct ib_qp *qp);
+	struct ib_qp *(*get_qp)(struct ib_device *device, int qpn);
+	int (*connect)(struct iw_cm_id *cm_id,
+		       struct iw_cm_conn_param *conn_param);
+	int (*accept)(struct iw_cm_id *cm_id,
+		      struct iw_cm_conn_param *conn_param);
+	int (*reject)(struct iw_cm_id *cm_id, const void *pdata, u8 pdata_len);
+	int (*create_listen)(struct iw_cm_id *cm_id, int backlog);
+	int (*destroy_listen)(struct iw_cm_id *cm_id);
+
 	DECLARE_RDMA_OBJ_SIZE(ib_ah);
 	DECLARE_RDMA_OBJ_SIZE(ib_pd);
 	DECLARE_RDMA_OBJ_SIZE(ib_srq);
@@ -2591,8 +2603,6 @@  struct ib_device {
 
 	int			      num_comp_vectors;
 
-	struct iw_cm_verbs	     *iwcm;
-
 	struct module               *owner;
 	union {
 		struct device		dev;
@@ -2645,6 +2655,10 @@  struct ib_device {
 	struct mutex compat_devs_mutex;
 	/* Maintains compat devices for each net namespace */
 	struct xarray compat_devs;
+
+	/* Used by iWarp CM */
+	char ifname[IFNAMSIZ];
+	u32 driver_flags;
 };
 
 struct ib_client {
diff --git a/include/rdma/iw_cm.h b/include/rdma/iw_cm.h
index 0e1f02815643..5aa8a9c76aa0 100644
--- a/include/rdma/iw_cm.h
+++ b/include/rdma/iw_cm.h
@@ -118,31 +118,6 @@  enum iw_flags {
 	IW_F_NO_PORT_MAP = (1 << 0),
 };
 
-struct iw_cm_verbs {
-	void		(*add_ref)(struct ib_qp *qp);
-
-	void		(*rem_ref)(struct ib_qp *qp);
-
-	struct ib_qp *	(*get_qp)(struct ib_device *device,
-				  int qpn);
-
-	int		(*connect)(struct iw_cm_id *cm_id,
-				   struct iw_cm_conn_param *conn_param);
-
-	int		(*accept)(struct iw_cm_id *cm_id,
-				  struct iw_cm_conn_param *conn_param);
-
-	int		(*reject)(struct iw_cm_id *cm_id,
-				  const void *pdata, u8 pdata_len);
-
-	int		(*create_listen)(struct iw_cm_id *cm_id,
-					 int backlog);
-
-	int		(*destroy_listen)(struct iw_cm_id *cm_id);
-	char		ifname[IFNAMSIZ];
-	enum iw_flags	driver_flags;
-};
-
 /**
  * iw_create_cm_id - Create an IW CM identifier.
  *