diff mbox series

[v5,2/2] rdma_rxe: use netlink messages to add/delete links

Message ID 807020fe6f05364ca793851390252cfdfd8cfb69.1543615225.git.swise@opengridcomputing.com (mailing list archive)
State Superseded
Headers show
Series Dynamic rdma link creation | expand

Commit Message

Steve Wise Nov. 30, 2018, 9:58 p.m. UTC 
Add support for the RDMA_NLDEV_CMD_NEWLINK/DELLINK messages which allow
dynamically adding new RXE links.  Deprecate the old module options
for now.

Cc: Moni Shoua <monis@mellanox.com>
Signed-off-by: Steve Wise <swise@opengridcomputing.com>
---
 drivers/infiniband/sw/rxe/rxe.c       | 62 +++++++++++++++++++++++++++++++++--
 drivers/infiniband/sw/rxe/rxe.h       |  2 +-
 drivers/infiniband/sw/rxe/rxe_net.c   |  4 +--
 drivers/infiniband/sw/rxe/rxe_net.h   |  2 +-
 drivers/infiniband/sw/rxe/rxe_sysfs.c |  6 ++--
 drivers/infiniband/sw/rxe/rxe_verbs.c |  4 +--
 drivers/infiniband/sw/rxe/rxe_verbs.h |  2 +-
 7 files changed, 70 insertions(+), 12 deletions(-)

Comments

Zhu Yanjun Dec. 3, 2018, 6:34 a.m. UTC | #1 
On 2018/12/1 5:58, Steve Wise wrote:
> Add support for the RDMA_NLDEV_CMD_NEWLINK/DELLINK messages which allow
> dynamically adding new RXE links.  Deprecate the old module options
> for now.
>
> Cc: Moni Shoua <monis@mellanox.com>
> Signed-off-by: Steve Wise <swise@opengridcomputing.com>
> ---
>   drivers/infiniband/sw/rxe/rxe.c       | 62 +++++++++++++++++++++++++++++++++--
>   drivers/infiniband/sw/rxe/rxe.h       |  2 +-
>   drivers/infiniband/sw/rxe/rxe_net.c   |  4 +--
>   drivers/infiniband/sw/rxe/rxe_net.h   |  2 +-
>   drivers/infiniband/sw/rxe/rxe_sysfs.c |  6 ++--
>   drivers/infiniband/sw/rxe/rxe_verbs.c |  4 +--
>   drivers/infiniband/sw/rxe/rxe_verbs.h |  2 +-
>   7 files changed, 70 insertions(+), 12 deletions(-)
>
> diff --git a/drivers/infiniband/sw/rxe/rxe.c b/drivers/infiniband/sw/rxe/rxe.c
> index 383e65c7bbc0..c09534b513ff 100644
> --- a/drivers/infiniband/sw/rxe/rxe.c
> +++ b/drivers/infiniband/sw/rxe/rxe.c
> @@ -31,6 +31,7 @@
>    * SOFTWARE.
>    */
>   
> +#include <rdma/rdma_netlink.h>
>   #include <net/addrconf.h>
>   #include "rxe.h"
>   #include "rxe_loc.h"
> @@ -308,7 +309,7 @@ void rxe_set_mtu(struct rxe_dev *rxe, unsigned int ndev_mtu)
>   /* called by ifc layer to create new rxe device.
>    * The caller should allocate memory for rxe by calling ib_alloc_device.
>    */
> -int rxe_add(struct rxe_dev *rxe, unsigned int mtu)
> +int rxe_add(struct rxe_dev *rxe, unsigned int mtu, const char *ibdev_name)
>   {
>   	int err;
>   
> @@ -320,7 +321,7 @@ int rxe_add(struct rxe_dev *rxe, unsigned int mtu)
>   
>   	rxe_set_mtu(rxe, mtu);
>   
> -	err = rxe_register_device(rxe);
> +	err = rxe_register_device(rxe, ibdev_name);
>   	if (err)
>   		goto err1;
>   
> @@ -339,6 +340,59 @@ void rxe_remove(struct rxe_dev *rxe)
>   	rxe_dev_put(rxe);
>   }
>   
> +static struct ib_device *rxe_newlink(const char *ibdev_name,
> +				     const char *ndev_name)
> +{
> +	struct net_device *ndev = NULL;
> +	struct rxe_dev *rxe;
> +	int err = 0;
> +
> +	ndev = dev_get_by_name(&init_net, ndev_name);
> +	if (!ndev) {
> +		pr_err("interface %s not found\n", ndev_name);
> +		err = -ENODEV;
> +		goto err;
> +	}
> +
> +	if (net_to_rxe(ndev)) {
> +		pr_err("already configured on %s\n", ndev_name);
> +		err = -EEXIST;
> +		goto err;
> +	}
> +
> +	rxe = rxe_net_add(ibdev_name, ndev);
> +	if (!rxe) {
> +		pr_err("failed to add %s\n", ndev_name);
> +		err = -EINVAL;
> +		goto err;
> +	}
> +
> +	if (netif_running(ndev) && netif_carrier_ok(ndev))
> +		rxe_port_up(rxe);
> +	else
> +		rxe_port_down(rxe);
> +	pr_info("added %s to %s\n", rxe->ib_dev.name, ndev->name);
> +err:
> +	if (ndev)
> +		dev_put(ndev);
> +	return err ? ERR_PTR(err) : &rxe->ib_dev;
> +}
> +
> +static int rxe_dellink(struct ib_device *device)
> +{
> +	struct rxe_dev *rxe = to_rdev(device);
> +
diff --git a/drivers/infiniband/sw/rxe/rxe.c 
b/drivers/infiniband/sw/rxe/rxe.c
index c09534b513ff..e19d575d276f 100644
--- a/drivers/infiniband/sw/rxe/rxe.c
+++ b/drivers/infiniband/sw/rxe/rxe.c
@@ -382,7 +382,7 @@ static int rxe_dellink(struct ib_device *device)
  {
         struct rxe_dev *rxe = to_rdev(device);

-       list_del(&rxe->list);
+       rxe_net_remove(rxe);
         rxe_remove(rxe);
         return 0;
  }
diff --git a/drivers/infiniband/sw/rxe/rxe_net.c 
b/drivers/infiniband/sw/rxe/rxe_net.c
index 10a38b76f24b..1b57710008d3 100644
--- a/drivers/infiniband/sw/rxe/rxe_net.c
+++ b/drivers/infiniband/sw/rxe/rxe_net.c
@@ -573,6 +573,13 @@ struct rxe_dev *rxe_net_add(const char *ibdev_name, 
struct net_device *ndev)
         return rxe;
  }

+void rxe_net_remove(struct rxe_dev *rxe)
+{
+       spin_lock_bh(&dev_list_lock);
+       list_del(&rxe->list);
+       spin_unlock_bh(&dev_list_lock);
+}
+
  void rxe_remove_all(void)
  {
         spin_lock_bh(&dev_list_lock);
diff --git a/drivers/infiniband/sw/rxe/rxe_net.h 
b/drivers/infiniband/sw/rxe/rxe_net.h
index f8e00e6d5d38..45bf15fab87f 100644
--- a/drivers/infiniband/sw/rxe/rxe_net.h
+++ b/drivers/infiniband/sw/rxe/rxe_net.h
@@ -44,6 +44,7 @@ struct rxe_recv_sockets {
  };

  struct rxe_dev *rxe_net_add(const char *ibdev_name, struct net_device 
*ndev);
+void rxe_net_remove(struct rxe_dev *rxe);

  int rxe_net_init(void);
  void rxe_net_exit(void);

> +	list_del(&rxe->list);

To avoid race, list_del(&rxe->list); should be replace the above.

If I am wrong, please correct me.

Zhu Yanjun

> +	rxe_remove(rxe);
> +	return 0;
> +}
> +
> +static struct rdma_link_ops rxe_link_ops = {
> +	.type = "rxe",
> +	.newlink = rxe_newlink,
> +	.dellink = rxe_dellink,
> +};
> +
>   static int __init rxe_module_init(void)
>   {
>   	int err;
> @@ -354,12 +408,14 @@ static int __init rxe_module_init(void)
>   	if (err)
>   		return err;
>   
> +	rdma_link_register(&rxe_link_ops);
>   	pr_info("loaded\n");
>   	return 0;
>   }
>   
>   static void __exit rxe_module_exit(void)
>   {
> +	rdma_link_unregister(&rxe_link_ops);
>   	rxe_remove_all();
>   	rxe_net_exit();
>   	rxe_cache_exit();
> @@ -369,3 +425,5 @@ static void __exit rxe_module_exit(void)
>   
>   late_initcall(rxe_module_init);
>   module_exit(rxe_module_exit);
> +
> +MODULE_ALIAS_RDMA_LINK("rxe");
> diff --git a/drivers/infiniband/sw/rxe/rxe.h b/drivers/infiniband/sw/rxe/rxe.h
> index 8f79bd86d033..e6e91fb14af8 100644
> --- a/drivers/infiniband/sw/rxe/rxe.h
> +++ b/drivers/infiniband/sw/rxe/rxe.h
> @@ -95,7 +95,7 @@ static inline u32 rxe_crc32(struct rxe_dev *rxe,
>   
>   void rxe_set_mtu(struct rxe_dev *rxe, unsigned int dev_mtu);
>   
> -int rxe_add(struct rxe_dev *rxe, unsigned int mtu);
> +int rxe_add(struct rxe_dev *rxe, unsigned int mtu, const char *ibdev_name);
>   void rxe_remove(struct rxe_dev *rxe);
>   void rxe_remove_all(void);
>   
> diff --git a/drivers/infiniband/sw/rxe/rxe_net.c b/drivers/infiniband/sw/rxe/rxe_net.c
> index b26a8141f3ed..8bb908cae71b 100644
> --- a/drivers/infiniband/sw/rxe/rxe_net.c
> +++ b/drivers/infiniband/sw/rxe/rxe_net.c
> @@ -550,7 +550,7 @@ enum rdma_link_layer rxe_link_layer(struct rxe_dev *rxe, unsigned int port_num)
>   	return IB_LINK_LAYER_ETHERNET;
>   }
>   
> -struct rxe_dev *rxe_net_add(struct net_device *ndev)
> +struct rxe_dev *rxe_net_add(const char *ibdev_name, struct net_device *ndev)
>   {
>   	int err;
>   	struct rxe_dev *rxe = NULL;
> @@ -561,7 +561,7 @@ struct rxe_dev *rxe_net_add(struct net_device *ndev)
>   
>   	rxe->ndev = ndev;
>   
> -	err = rxe_add(rxe, ndev->mtu);
> +	err = rxe_add(rxe, ndev->mtu, ibdev_name);
>   	if (err) {
>   		ib_dealloc_device(&rxe->ib_dev);
>   		return NULL;
> diff --git a/drivers/infiniband/sw/rxe/rxe_net.h b/drivers/infiniband/sw/rxe/rxe_net.h
> index 106c586dbb26..f8e00e6d5d38 100644
> --- a/drivers/infiniband/sw/rxe/rxe_net.h
> +++ b/drivers/infiniband/sw/rxe/rxe_net.h
> @@ -43,7 +43,7 @@ struct rxe_recv_sockets {
>   	struct socket *sk6;
>   };
>   
> -struct rxe_dev *rxe_net_add(struct net_device *ndev);
> +struct rxe_dev *rxe_net_add(const char *ibdev_name, struct net_device *ndev);
>   
>   int rxe_net_init(void);
>   void rxe_net_exit(void);
> diff --git a/drivers/infiniband/sw/rxe/rxe_sysfs.c b/drivers/infiniband/sw/rxe/rxe_sysfs.c
> index 73a19f808e1b..629f824d8d3e 100644
> --- a/drivers/infiniband/sw/rxe/rxe_sysfs.c
> +++ b/drivers/infiniband/sw/rxe/rxe_sysfs.c
> @@ -97,7 +97,7 @@ static int rxe_param_set_add(const char *val, const struct kernel_param *kp)
>   		goto err;
>   	}
>   
> -	rxe = rxe_net_add(ndev);
> +	rxe = rxe_net_add("rxe%d", ndev);
>   	if (!rxe) {
>   		pr_err("failed to add %s\n", intf);
>   		err = -EINVAL;
> @@ -152,6 +152,6 @@ static int rxe_param_set_remove(const char *val, const struct kernel_param *kp)
>   };
>   
>   module_param_cb(add, &rxe_add_ops, NULL, 0200);
> -MODULE_PARM_DESC(add, "Create RXE device over network interface");
> +MODULE_PARM_DESC(add, "DEPRECATED.  Create RXE device over network interface");
>   module_param_cb(remove, &rxe_remove_ops, NULL, 0200);
> -MODULE_PARM_DESC(remove, "Remove RXE device over network interface");
> +MODULE_PARM_DESC(remove, "DEPRECATED.  Remove RXE device over network interface");
> diff --git a/drivers/infiniband/sw/rxe/rxe_verbs.c b/drivers/infiniband/sw/rxe/rxe_verbs.c
> index 30817c79ba96..3536ca4627cd 100644
> --- a/drivers/infiniband/sw/rxe/rxe_verbs.c
> +++ b/drivers/infiniband/sw/rxe/rxe_verbs.c
> @@ -1165,7 +1165,7 @@ static ssize_t parent_show(struct device *device,
>   	.attrs = rxe_dev_attributes,
>   };
>   
> -int rxe_register_device(struct rxe_dev *rxe)
> +int rxe_register_device(struct rxe_dev *rxe, const char *ibdev_name)
>   {
>   	int err;
>   	struct ib_device *dev = &rxe->ib_dev;
> @@ -1273,7 +1273,7 @@ int rxe_register_device(struct rxe_dev *rxe)
>   
>   	rdma_set_device_sysfs_group(dev, &rxe_attr_group);
>   	dev->driver_id = RDMA_DRIVER_RXE;
> -	err = ib_register_device(dev, "rxe%d", NULL);
> +	err = ib_register_device(dev, ibdev_name, NULL);
>   	if (err) {
>   		pr_warn("%s failed with error %d\n", __func__, err);
>   		goto err1;
> diff --git a/drivers/infiniband/sw/rxe/rxe_verbs.h b/drivers/infiniband/sw/rxe/rxe_verbs.h
> index 831381b7788d..c5f336d9aaf2 100644
> --- a/drivers/infiniband/sw/rxe/rxe_verbs.h
> +++ b/drivers/infiniband/sw/rxe/rxe_verbs.h
> @@ -466,7 +466,7 @@ static inline struct rxe_mem *to_rmw(struct ib_mw *mw)
>   	return mw ? container_of(mw, struct rxe_mem, ibmw) : NULL;
>   }
>   
> -int rxe_register_device(struct rxe_dev *rxe);
> +int rxe_register_device(struct rxe_dev *rxe, const char *ibdev_name);
>   void rxe_unregister_device(struct rxe_dev *rxe);
>   
>   void rxe_mc_cleanup(struct rxe_pool_entry *arg);
Steve Wise Dec. 3, 2018, 4:26 p.m. UTC | #2 
On 12/3/2018 12:34 AM, Yanjun Zhu wrote:
>
> On 2018/12/1 5:58, Steve Wise wrote:
>> Add support for the RDMA_NLDEV_CMD_NEWLINK/DELLINK messages which allow
>> dynamically adding new RXE links.  Deprecate the old module options
>> for now.
>>
>> Cc: Moni Shoua <monis@mellanox.com>
>> Signed-off-by: Steve Wise <swise@opengridcomputing.com>
>> ---
>>   drivers/infiniband/sw/rxe/rxe.c       | 62
>> +++++++++++++++++++++++++++++++++--
>>   drivers/infiniband/sw/rxe/rxe.h       |  2 +-
>>   drivers/infiniband/sw/rxe/rxe_net.c   |  4 +--
>>   drivers/infiniband/sw/rxe/rxe_net.h   |  2 +-
>>   drivers/infiniband/sw/rxe/rxe_sysfs.c |  6 ++--
>>   drivers/infiniband/sw/rxe/rxe_verbs.c |  4 +--
>>   drivers/infiniband/sw/rxe/rxe_verbs.h |  2 +-
>>   7 files changed, 70 insertions(+), 12 deletions(-)
>>
>> diff --git a/drivers/infiniband/sw/rxe/rxe.c
>> b/drivers/infiniband/sw/rxe/rxe.c
>> index 383e65c7bbc0..c09534b513ff 100644
>> --- a/drivers/infiniband/sw/rxe/rxe.c
>> +++ b/drivers/infiniband/sw/rxe/rxe.c
>> @@ -31,6 +31,7 @@
>>    * SOFTWARE.
>>    */
>>   +#include <rdma/rdma_netlink.h>
>>   #include <net/addrconf.h>
>>   #include "rxe.h"
>>   #include "rxe_loc.h"
>> @@ -308,7 +309,7 @@ void rxe_set_mtu(struct rxe_dev *rxe, unsigned
>> int ndev_mtu)
>>   /* called by ifc layer to create new rxe device.
>>    * The caller should allocate memory for rxe by calling
>> ib_alloc_device.
>>    */
>> -int rxe_add(struct rxe_dev *rxe, unsigned int mtu)
>> +int rxe_add(struct rxe_dev *rxe, unsigned int mtu, const char
>> *ibdev_name)
>>   {
>>       int err;
>>   @@ -320,7 +321,7 @@ int rxe_add(struct rxe_dev *rxe, unsigned int mtu)
>>         rxe_set_mtu(rxe, mtu);
>>   -    err = rxe_register_device(rxe);
>> +    err = rxe_register_device(rxe, ibdev_name);
>>       if (err)
>>           goto err1;
>>   @@ -339,6 +340,59 @@ void rxe_remove(struct rxe_dev *rxe)
>>       rxe_dev_put(rxe);
>>   }
>>   +static struct ib_device *rxe_newlink(const char *ibdev_name,
>> +                     const char *ndev_name)
>> +{
>> +    struct net_device *ndev = NULL;
>> +    struct rxe_dev *rxe;
>> +    int err = 0;
>> +
>> +    ndev = dev_get_by_name(&init_net, ndev_name);
>> +    if (!ndev) {
>> +        pr_err("interface %s not found\n", ndev_name);
>> +        err = -ENODEV;
>> +        goto err;
>> +    }
>> +
>> +    if (net_to_rxe(ndev)) {
>> +        pr_err("already configured on %s\n", ndev_name);
>> +        err = -EEXIST;
>> +        goto err;
>> +    }
>> +
>> +    rxe = rxe_net_add(ibdev_name, ndev);
>> +    if (!rxe) {
>> +        pr_err("failed to add %s\n", ndev_name);
>> +        err = -EINVAL;
>> +        goto err;
>> +    }
>> +
>> +    if (netif_running(ndev) && netif_carrier_ok(ndev))
>> +        rxe_port_up(rxe);
>> +    else
>> +        rxe_port_down(rxe);
>> +    pr_info("added %s to %s\n", rxe->ib_dev.name, ndev->name);
>> +err:
>> +    if (ndev)
>> +        dev_put(ndev);
>> +    return err ? ERR_PTR(err) : &rxe->ib_dev;
>> +}
>> +
>> +static int rxe_dellink(struct ib_device *device)
>> +{
>> +    struct rxe_dev *rxe = to_rdev(device);
>> +
> diff --git a/drivers/infiniband/sw/rxe/rxe.c
> b/drivers/infiniband/sw/rxe/rxe.c
> index c09534b513ff..e19d575d276f 100644
> --- a/drivers/infiniband/sw/rxe/rxe.c
> +++ b/drivers/infiniband/sw/rxe/rxe.c
> @@ -382,7 +382,7 @@ static int rxe_dellink(struct ib_device *device)
>  {
>         struct rxe_dev *rxe = to_rdev(device);
>
> -       list_del(&rxe->list);
> +       rxe_net_remove(rxe);
>         rxe_remove(rxe);
>         return 0;
>  }
> diff --git a/drivers/infiniband/sw/rxe/rxe_net.c
> b/drivers/infiniband/sw/rxe/rxe_net.c
> index 10a38b76f24b..1b57710008d3 100644
> --- a/drivers/infiniband/sw/rxe/rxe_net.c
> +++ b/drivers/infiniband/sw/rxe/rxe_net.c
> @@ -573,6 +573,13 @@ struct rxe_dev *rxe_net_add(const char
> *ibdev_name, struct net_device *ndev)
>         return rxe;
>  }
>
> +void rxe_net_remove(struct rxe_dev *rxe)
> +{
> +       spin_lock_bh(&dev_list_lock);
> +       list_del(&rxe->list);
> +       spin_unlock_bh(&dev_list_lock);
> +}
> +
>  void rxe_remove_all(void)
>  {
>         spin_lock_bh(&dev_list_lock);
> diff --git a/drivers/infiniband/sw/rxe/rxe_net.h
> b/drivers/infiniband/sw/rxe/rxe_net.h
> index f8e00e6d5d38..45bf15fab87f 100644
> --- a/drivers/infiniband/sw/rxe/rxe_net.h
> +++ b/drivers/infiniband/sw/rxe/rxe_net.h
> @@ -44,6 +44,7 @@ struct rxe_recv_sockets {
>  };
>
>  struct rxe_dev *rxe_net_add(const char *ibdev_name, struct net_device
> *ndev);
> +void rxe_net_remove(struct rxe_dev *rxe);
>
>  int rxe_net_init(void);
>  void rxe_net_exit(void);
>
>> +    list_del(&rxe->list);
>
> To avoid race, list_del(&rxe->list); should be replace the above.
>
> If I am wrong, please correct me.
>
> Zhu Yanjun
>

Yes, you're correct.  This code is not serialized at all for device
removal.  As Jason requested, I'll address this all in the next version
and hopefully get it right. :)

Thanks for the review.  May I add a reviewed-by tag from you?

Steve.
Zhu Yanjun Dec. 4, 2018, 1:15 a.m. UTC | #3 
On 2018/12/4 0:26, Steve Wise wrote:
>
> On 12/3/2018 12:34 AM, Yanjun Zhu wrote:
>> On 2018/12/1 5:58, Steve Wise wrote:
>>> Add support for the RDMA_NLDEV_CMD_NEWLINK/DELLINK messages which allow
>>> dynamically adding new RXE links.  Deprecate the old module options
>>> for now.
>>>
>>> Cc: Moni Shoua <monis@mellanox.com>
>>> Signed-off-by: Steve Wise <swise@opengridcomputing.com>
>>> ---
>>>    drivers/infiniband/sw/rxe/rxe.c       | 62
>>> +++++++++++++++++++++++++++++++++--
>>>    drivers/infiniband/sw/rxe/rxe.h       |  2 +-
>>>    drivers/infiniband/sw/rxe/rxe_net.c   |  4 +--
>>>    drivers/infiniband/sw/rxe/rxe_net.h   |  2 +-
>>>    drivers/infiniband/sw/rxe/rxe_sysfs.c |  6 ++--
>>>    drivers/infiniband/sw/rxe/rxe_verbs.c |  4 +--
>>>    drivers/infiniband/sw/rxe/rxe_verbs.h |  2 +-
>>>    7 files changed, 70 insertions(+), 12 deletions(-)
>>>
>>> diff --git a/drivers/infiniband/sw/rxe/rxe.c
>>> b/drivers/infiniband/sw/rxe/rxe.c
>>> index 383e65c7bbc0..c09534b513ff 100644
>>> --- a/drivers/infiniband/sw/rxe/rxe.c
>>> +++ b/drivers/infiniband/sw/rxe/rxe.c
>>> @@ -31,6 +31,7 @@
>>>     * SOFTWARE.
>>>     */
>>>    +#include <rdma/rdma_netlink.h>
>>>    #include <net/addrconf.h>
>>>    #include "rxe.h"
>>>    #include "rxe_loc.h"
>>> @@ -308,7 +309,7 @@ void rxe_set_mtu(struct rxe_dev *rxe, unsigned
>>> int ndev_mtu)
>>>    /* called by ifc layer to create new rxe device.
>>>     * The caller should allocate memory for rxe by calling
>>> ib_alloc_device.
>>>     */
>>> -int rxe_add(struct rxe_dev *rxe, unsigned int mtu)
>>> +int rxe_add(struct rxe_dev *rxe, unsigned int mtu, const char
>>> *ibdev_name)
>>>    {
>>>        int err;
>>>    @@ -320,7 +321,7 @@ int rxe_add(struct rxe_dev *rxe, unsigned int mtu)
>>>          rxe_set_mtu(rxe, mtu);
>>>    -    err = rxe_register_device(rxe);
>>> +    err = rxe_register_device(rxe, ibdev_name);
>>>        if (err)
>>>            goto err1;
>>>    @@ -339,6 +340,59 @@ void rxe_remove(struct rxe_dev *rxe)
>>>        rxe_dev_put(rxe);
>>>    }
>>>    +static struct ib_device *rxe_newlink(const char *ibdev_name,
>>> +                     const char *ndev_name)
>>> +{
>>> +    struct net_device *ndev = NULL;
>>> +    struct rxe_dev *rxe;
>>> +    int err = 0;
>>> +
>>> +    ndev = dev_get_by_name(&init_net, ndev_name);
>>> +    if (!ndev) {
>>> +        pr_err("interface %s not found\n", ndev_name);
>>> +        err = -ENODEV;
>>> +        goto err;
>>> +    }
>>> +
>>> +    if (net_to_rxe(ndev)) {
>>> +        pr_err("already configured on %s\n", ndev_name);
>>> +        err = -EEXIST;
>>> +        goto err;
>>> +    }
>>> +
>>> +    rxe = rxe_net_add(ibdev_name, ndev);
>>> +    if (!rxe) {
>>> +        pr_err("failed to add %s\n", ndev_name);
>>> +        err = -EINVAL;
>>> +        goto err;
>>> +    }
>>> +
>>> +    if (netif_running(ndev) && netif_carrier_ok(ndev))
>>> +        rxe_port_up(rxe);
>>> +    else
>>> +        rxe_port_down(rxe);
>>> +    pr_info("added %s to %s\n", rxe->ib_dev.name, ndev->name);
>>> +err:
>>> +    if (ndev)
>>> +        dev_put(ndev);
>>> +    return err ? ERR_PTR(err) : &rxe->ib_dev;
>>> +}
>>> +
>>> +static int rxe_dellink(struct ib_device *device)
>>> +{
>>> +    struct rxe_dev *rxe = to_rdev(device);
>>> +
>> diff --git a/drivers/infiniband/sw/rxe/rxe.c
>> b/drivers/infiniband/sw/rxe/rxe.c
>> index c09534b513ff..e19d575d276f 100644
>> --- a/drivers/infiniband/sw/rxe/rxe.c
>> +++ b/drivers/infiniband/sw/rxe/rxe.c
>> @@ -382,7 +382,7 @@ static int rxe_dellink(struct ib_device *device)
>>   {
>>          struct rxe_dev *rxe = to_rdev(device);
>>
>> -       list_del(&rxe->list);
>> +       rxe_net_remove(rxe);
>>          rxe_remove(rxe);
>>          return 0;
>>   }
>> diff --git a/drivers/infiniband/sw/rxe/rxe_net.c
>> b/drivers/infiniband/sw/rxe/rxe_net.c
>> index 10a38b76f24b..1b57710008d3 100644
>> --- a/drivers/infiniband/sw/rxe/rxe_net.c
>> +++ b/drivers/infiniband/sw/rxe/rxe_net.c
>> @@ -573,6 +573,13 @@ struct rxe_dev *rxe_net_add(const char
>> *ibdev_name, struct net_device *ndev)
>>          return rxe;
>>   }
>>
>> +void rxe_net_remove(struct rxe_dev *rxe)
>> +{
>> +       spin_lock_bh(&dev_list_lock);
>> +       list_del(&rxe->list);
>> +       spin_unlock_bh(&dev_list_lock);
>> +}
>> +
>>   void rxe_remove_all(void)
>>   {
>>          spin_lock_bh(&dev_list_lock);
>> diff --git a/drivers/infiniband/sw/rxe/rxe_net.h
>> b/drivers/infiniband/sw/rxe/rxe_net.h
>> index f8e00e6d5d38..45bf15fab87f 100644
>> --- a/drivers/infiniband/sw/rxe/rxe_net.h
>> +++ b/drivers/infiniband/sw/rxe/rxe_net.h
>> @@ -44,6 +44,7 @@ struct rxe_recv_sockets {
>>   };
>>
>>   struct rxe_dev *rxe_net_add(const char *ibdev_name, struct net_device
>> *ndev);
>> +void rxe_net_remove(struct rxe_dev *rxe);
>>
>>   int rxe_net_init(void);
>>   void rxe_net_exit(void);
>>
>>> +    list_del(&rxe->list);
>> To avoid race, list_del(&rxe->list); should be replace the above.
>>
>> If I am wrong, please correct me.
>>
>> Zhu Yanjun
>>
> Yes, you're correct.  This code is not serialized at all for device
> removal.  As Jason requested, I'll address this all in the next version
> and hopefully get it right. :)
>
> Thanks for the review.  May I add a reviewed-by tag from you?

Yes. You can:-).

Zhu Yanjun

>
> Steve.
diff mbox series

Patch

diff --git a/drivers/infiniband/sw/rxe/rxe.c b/drivers/infiniband/sw/rxe/rxe.c
index 383e65c7bbc0..c09534b513ff 100644
--- a/drivers/infiniband/sw/rxe/rxe.c
+++ b/drivers/infiniband/sw/rxe/rxe.c
@@ -31,6 +31,7 @@ 
  * SOFTWARE.
  */
 
+#include <rdma/rdma_netlink.h>
 #include <net/addrconf.h>
 #include "rxe.h"
 #include "rxe_loc.h"
@@ -308,7 +309,7 @@  void rxe_set_mtu(struct rxe_dev *rxe, unsigned int ndev_mtu)
 /* called by ifc layer to create new rxe device.
  * The caller should allocate memory for rxe by calling ib_alloc_device.
  */
-int rxe_add(struct rxe_dev *rxe, unsigned int mtu)
+int rxe_add(struct rxe_dev *rxe, unsigned int mtu, const char *ibdev_name)
 {
 	int err;
 
@@ -320,7 +321,7 @@  int rxe_add(struct rxe_dev *rxe, unsigned int mtu)
 
 	rxe_set_mtu(rxe, mtu);
 
-	err = rxe_register_device(rxe);
+	err = rxe_register_device(rxe, ibdev_name);
 	if (err)
 		goto err1;
 
@@ -339,6 +340,59 @@  void rxe_remove(struct rxe_dev *rxe)
 	rxe_dev_put(rxe);
 }
 
+static struct ib_device *rxe_newlink(const char *ibdev_name,
+				     const char *ndev_name)
+{
+	struct net_device *ndev = NULL;
+	struct rxe_dev *rxe;
+	int err = 0;
+
+	ndev = dev_get_by_name(&init_net, ndev_name);
+	if (!ndev) {
+		pr_err("interface %s not found\n", ndev_name);
+		err = -ENODEV;
+		goto err;
+	}
+
+	if (net_to_rxe(ndev)) {
+		pr_err("already configured on %s\n", ndev_name);
+		err = -EEXIST;
+		goto err;
+	}
+
+	rxe = rxe_net_add(ibdev_name, ndev);
+	if (!rxe) {
+		pr_err("failed to add %s\n", ndev_name);
+		err = -EINVAL;
+		goto err;
+	}
+
+	if (netif_running(ndev) && netif_carrier_ok(ndev))
+		rxe_port_up(rxe);
+	else
+		rxe_port_down(rxe);
+	pr_info("added %s to %s\n", rxe->ib_dev.name, ndev->name);
+err:
+	if (ndev)
+		dev_put(ndev);
+	return err ? ERR_PTR(err) : &rxe->ib_dev;
+}
+
+static int rxe_dellink(struct ib_device *device)
+{
+	struct rxe_dev *rxe = to_rdev(device);
+
+	list_del(&rxe->list);
+	rxe_remove(rxe);
+	return 0;
+}
+
+static struct rdma_link_ops rxe_link_ops = {
+	.type = "rxe",
+	.newlink = rxe_newlink,
+	.dellink = rxe_dellink,
+};
+
 static int __init rxe_module_init(void)
 {
 	int err;
@@ -354,12 +408,14 @@  static int __init rxe_module_init(void)
 	if (err)
 		return err;
 
+	rdma_link_register(&rxe_link_ops);
 	pr_info("loaded\n");
 	return 0;
 }
 
 static void __exit rxe_module_exit(void)
 {
+	rdma_link_unregister(&rxe_link_ops);
 	rxe_remove_all();
 	rxe_net_exit();
 	rxe_cache_exit();
@@ -369,3 +425,5 @@  static void __exit rxe_module_exit(void)
 
 late_initcall(rxe_module_init);
 module_exit(rxe_module_exit);
+
+MODULE_ALIAS_RDMA_LINK("rxe");
diff --git a/drivers/infiniband/sw/rxe/rxe.h b/drivers/infiniband/sw/rxe/rxe.h
index 8f79bd86d033..e6e91fb14af8 100644
--- a/drivers/infiniband/sw/rxe/rxe.h
+++ b/drivers/infiniband/sw/rxe/rxe.h
@@ -95,7 +95,7 @@  static inline u32 rxe_crc32(struct rxe_dev *rxe,
 
 void rxe_set_mtu(struct rxe_dev *rxe, unsigned int dev_mtu);
 
-int rxe_add(struct rxe_dev *rxe, unsigned int mtu);
+int rxe_add(struct rxe_dev *rxe, unsigned int mtu, const char *ibdev_name);
 void rxe_remove(struct rxe_dev *rxe);
 void rxe_remove_all(void);
 
diff --git a/drivers/infiniband/sw/rxe/rxe_net.c b/drivers/infiniband/sw/rxe/rxe_net.c
index b26a8141f3ed..8bb908cae71b 100644
--- a/drivers/infiniband/sw/rxe/rxe_net.c
+++ b/drivers/infiniband/sw/rxe/rxe_net.c
@@ -550,7 +550,7 @@  enum rdma_link_layer rxe_link_layer(struct rxe_dev *rxe, unsigned int port_num)
 	return IB_LINK_LAYER_ETHERNET;
 }
 
-struct rxe_dev *rxe_net_add(struct net_device *ndev)
+struct rxe_dev *rxe_net_add(const char *ibdev_name, struct net_device *ndev)
 {
 	int err;
 	struct rxe_dev *rxe = NULL;
@@ -561,7 +561,7 @@  struct rxe_dev *rxe_net_add(struct net_device *ndev)
 
 	rxe->ndev = ndev;
 
-	err = rxe_add(rxe, ndev->mtu);
+	err = rxe_add(rxe, ndev->mtu, ibdev_name);
 	if (err) {
 		ib_dealloc_device(&rxe->ib_dev);
 		return NULL;
diff --git a/drivers/infiniband/sw/rxe/rxe_net.h b/drivers/infiniband/sw/rxe/rxe_net.h
index 106c586dbb26..f8e00e6d5d38 100644
--- a/drivers/infiniband/sw/rxe/rxe_net.h
+++ b/drivers/infiniband/sw/rxe/rxe_net.h
@@ -43,7 +43,7 @@  struct rxe_recv_sockets {
 	struct socket *sk6;
 };
 
-struct rxe_dev *rxe_net_add(struct net_device *ndev);
+struct rxe_dev *rxe_net_add(const char *ibdev_name, struct net_device *ndev);
 
 int rxe_net_init(void);
 void rxe_net_exit(void);
diff --git a/drivers/infiniband/sw/rxe/rxe_sysfs.c b/drivers/infiniband/sw/rxe/rxe_sysfs.c
index 73a19f808e1b..629f824d8d3e 100644
--- a/drivers/infiniband/sw/rxe/rxe_sysfs.c
+++ b/drivers/infiniband/sw/rxe/rxe_sysfs.c
@@ -97,7 +97,7 @@  static int rxe_param_set_add(const char *val, const struct kernel_param *kp)
 		goto err;
 	}
 
-	rxe = rxe_net_add(ndev);
+	rxe = rxe_net_add("rxe%d", ndev);
 	if (!rxe) {
 		pr_err("failed to add %s\n", intf);
 		err = -EINVAL;
@@ -152,6 +152,6 @@  static int rxe_param_set_remove(const char *val, const struct kernel_param *kp)
 };
 
 module_param_cb(add, &rxe_add_ops, NULL, 0200);
-MODULE_PARM_DESC(add, "Create RXE device over network interface");
+MODULE_PARM_DESC(add, "DEPRECATED.  Create RXE device over network interface");
 module_param_cb(remove, &rxe_remove_ops, NULL, 0200);
-MODULE_PARM_DESC(remove, "Remove RXE device over network interface");
+MODULE_PARM_DESC(remove, "DEPRECATED.  Remove RXE device over network interface");
diff --git a/drivers/infiniband/sw/rxe/rxe_verbs.c b/drivers/infiniband/sw/rxe/rxe_verbs.c
index 30817c79ba96..3536ca4627cd 100644
--- a/drivers/infiniband/sw/rxe/rxe_verbs.c
+++ b/drivers/infiniband/sw/rxe/rxe_verbs.c
@@ -1165,7 +1165,7 @@  static ssize_t parent_show(struct device *device,
 	.attrs = rxe_dev_attributes,
 };
 
-int rxe_register_device(struct rxe_dev *rxe)
+int rxe_register_device(struct rxe_dev *rxe, const char *ibdev_name)
 {
 	int err;
 	struct ib_device *dev = &rxe->ib_dev;
@@ -1273,7 +1273,7 @@  int rxe_register_device(struct rxe_dev *rxe)
 
 	rdma_set_device_sysfs_group(dev, &rxe_attr_group);
 	dev->driver_id = RDMA_DRIVER_RXE;
-	err = ib_register_device(dev, "rxe%d", NULL);
+	err = ib_register_device(dev, ibdev_name, NULL);
 	if (err) {
 		pr_warn("%s failed with error %d\n", __func__, err);
 		goto err1;
diff --git a/drivers/infiniband/sw/rxe/rxe_verbs.h b/drivers/infiniband/sw/rxe/rxe_verbs.h
index 831381b7788d..c5f336d9aaf2 100644
--- a/drivers/infiniband/sw/rxe/rxe_verbs.h
+++ b/drivers/infiniband/sw/rxe/rxe_verbs.h
@@ -466,7 +466,7 @@  static inline struct rxe_mem *to_rmw(struct ib_mw *mw)
 	return mw ? container_of(mw, struct rxe_mem, ibmw) : NULL;
 }
 
-int rxe_register_device(struct rxe_dev *rxe);
+int rxe_register_device(struct rxe_dev *rxe, const char *ibdev_name);
 void rxe_unregister_device(struct rxe_dev *rxe);
 
 void rxe_mc_cleanup(struct rxe_pool_entry *arg);