| Message ID | 1582731932-26574-3-git-send-email-selvin.xavier@broadcom.com (mailing list archive) |
|---|---|
| State | Mainlined |
| Commit | 66832705c4d01e52df78570e72a9392a1271d2e9 |
| Delegated to: | Jason Gunthorpe |
| Headers | show |
| Series | RDMA/bnxt_re driver update | expand |
On Wed, Feb 26, 2020 at 07:45:32AM -0800, Selvin Xavier wrote: > @@ -1724,6 +1714,7 @@ static int bnxt_re_netdev_event(struct notifier_block *notifier, > rc = bnxt_re_add_device(&rdev, real_dev); > if (!rc) > sch_work = true; > + release = false; > break; > > case NETDEV_UNREGISTER: > @@ -1732,8 +1723,7 @@ static int bnxt_re_netdev_event(struct notifier_block *notifier, > */ > if (atomic_read(&rdev->sched_count) > 0) > goto exit; This sched_count stuff needs cleaning too. krefs should be used properly, carry the kref on the ib_device into the work and use the registration lock on the ib device to serialize instead of this sched_count stuff. This all sounds so familiar.. Oh I tried to fix this once - maybe the below will help you: commit 33d88c818d155ffb2ef4b12e72107f628c70404c Author: Jason Gunthorpe <jgg@ziepe.ca> Date: Thu Jan 10 12:05:19 2019 -0700 RDMA/bnxt_re: Use ib_device_get_by_netdev() instead of open coding The core API handles the locking correctly and is faster if there are multiple devices. Signed-off-by: Jason Gunthorpe <jgg@mellanox.com> diff --git a/drivers/infiniband/hw/bnxt_re/main.c b/drivers/infiniband/hw/bnxt_re/main.c index fa539608ffbbe0..bd67a31937ec65 100644 --- a/drivers/infiniband/hw/bnxt_re/main.c +++ b/drivers/infiniband/hw/bnxt_re/main.c @@ -504,21 +504,6 @@ static bool is_bnxt_re_dev(struct net_device *netdev) return false; } -static struct bnxt_re_dev *bnxt_re_from_netdev(struct net_device *netdev) -{ - struct bnxt_re_dev *rdev; - - rcu_read_lock(); - list_for_each_entry_rcu(rdev, &bnxt_re_dev_list, list) { - if (rdev->netdev == netdev) { - rcu_read_unlock(); - return rdev; - } - } - rcu_read_unlock(); - return NULL; -} - static void bnxt_re_dev_unprobe(struct net_device *netdev, struct bnxt_en_dev *en_dev) { @@ -1616,23 +1601,26 @@ static int bnxt_re_netdev_event(struct notifier_block *notifier, { struct net_device *real_dev, *netdev = netdev_notifier_info_to_dev(ptr); struct bnxt_re_work *re_work; - struct bnxt_re_dev *rdev; + struct bnxt_re_dev *rdev = NULL; + struct ib_device *ibdev; int rc = 0; bool sch_work = false; real_dev = rdma_vlan_dev_real_dev(netdev); if (!real_dev) real_dev = netdev; - - rdev = bnxt_re_from_netdev(real_dev); - if (!rdev && event != NETDEV_REGISTER) - goto exit; if (real_dev != netdev) goto exit; + ibdev = ib_device_get_by_netdev(real_dev, RDMA_DRIVER_BNXT_RE); + if (!ibdev && event != NETDEV_REGISTER) + goto exit; + if (ibdev) + rdev = container_of(ibdev, struct bnxt_re_dev, ibdev); + switch (event) { case NETDEV_REGISTER: - if (rdev) + if (ibdev) break; rc = bnxt_re_dev_reg(&rdev, real_dev); if (rc == -ENODEV) @@ -1676,6 +1664,9 @@ static int bnxt_re_netdev_event(struct notifier_block *notifier, } } + if (ibdev) + ib_device_put(ibdev); + exit: return NOTIFY_DONE; } commit 6c617f08e749ee0f6c7be6763ea92e49ae484712 Author: Jason Gunthorpe <jgg@ziepe.ca> Date: Thu Jan 10 14:40:16 2019 -0700 RDMA/bnxt_re: Use ib_device_try_get() There are a couple places in this driver running from a work queue that need the ib_device to be registered. Instead of using a broken internal bit rely on the new core code to guarantee device registration. Signed-off-by: Jason Gunthorpe <jgg@mellanox.com> diff --git a/drivers/infiniband/hw/bnxt_re/main.c b/drivers/infiniband/hw/bnxt_re/main.c index 666897596218d3..fa539608ffbbe0 100644 --- a/drivers/infiniband/hw/bnxt_re/main.c +++ b/drivers/infiniband/hw/bnxt_re/main.c @@ -1137,12 +1137,13 @@ static int bnxt_re_update_gid(struct bnxt_re_dev *rdev) u16 gid_idx, index; int rc = 0; - if (!test_bit(BNXT_RE_FLAG_IBDEV_REGISTERED, &rdev->flags)) + if (!ib_device_try_get(&rdev->ibdev)) return 0; if (!sgid_tbl) { dev_err(rdev_to_dev(rdev), "QPLIB: SGID table not allocated"); - return -EINVAL; + rc = -EINVAL; + goto out; } for (index = 0; index < sgid_tbl->active; index++) { @@ -1163,6 +1164,8 @@ static int bnxt_re_update_gid(struct bnxt_re_dev *rdev) rdev->qplib_res.netdev->dev_addr); } +out: + ib_device_put(&rdev->ibdev); return rc; } @@ -1545,12 +1548,7 @@ static void bnxt_re_task(struct work_struct *work) re_work = container_of(work, struct bnxt_re_work, work); rdev = re_work->rdev; - if (re_work->event != NETDEV_REGISTER && - !test_bit(BNXT_RE_FLAG_IBDEV_REGISTERED, &rdev->flags)) - goto exit; - - switch (re_work->event) { - case NETDEV_REGISTER: + if (re_work->event == NETDEV_REGISTER) { rc = bnxt_re_ib_reg(rdev); if (rc) { dev_err(rdev_to_dev(rdev), @@ -1559,7 +1557,13 @@ static void bnxt_re_task(struct work_struct *work) bnxt_re_dev_unreg(rdev); goto exit; } - break; + goto exit; + } + + if (!ib_device_try_get(&rdev->ibdev)) + goto exit; + + switch (re_work->event) { case NETDEV_UP: bnxt_re_dispatch_event(&rdev->ibdev, NULL, 1, IB_EVENT_PORT_ACTIVE); @@ -1579,6 +1583,8 @@ static void bnxt_re_task(struct work_struct *work) default: break; } + + ib_device_put(&rdev->ibdev); smp_mb__before_atomic(); atomic_dec(&rdev->sched_count); exit: commit e64da98a182a2cae3338f28f6e581f189b5f8674 Author: Jason Gunthorpe <jgg@ziepe.ca> Date: Thu Jan 10 12:02:11 2019 -0700 RDMA/bnxt_re: Fix lifetimes in bnxt_re_task A work queue cannot just rely on the ib_device not being freed, it must hold a kref on the memory so that the BNXT_RE_FLAG_IBDEV_REGISTERED check works. Also, every single work queue call has an allocated memory, and the kfree of this memory was missed sometimes. Fixes: 1ac5a4047975 ("RDMA/bnxt_re: Add bnxt_re RoCE driver") Signed-off-by: Jason Gunthorpe <jgg@mellanox.com> diff --git a/drivers/infiniband/hw/bnxt_re/main.c b/drivers/infiniband/hw/bnxt_re/main.c index 814f959c7db965..666897596218d3 100644 --- a/drivers/infiniband/hw/bnxt_re/main.c +++ b/drivers/infiniband/hw/bnxt_re/main.c @@ -1547,7 +1547,7 @@ static void bnxt_re_task(struct work_struct *work) if (re_work->event != NETDEV_REGISTER && !test_bit(BNXT_RE_FLAG_IBDEV_REGISTERED, &rdev->flags)) - return; + goto exit; switch (re_work->event) { case NETDEV_REGISTER: @@ -1582,6 +1582,7 @@ static void bnxt_re_task(struct work_struct *work) smp_mb__before_atomic(); atomic_dec(&rdev->sched_count); exit: + put_device(&rdev->ibdev.dev); kfree(re_work); } @@ -1658,6 +1659,7 @@ static int bnxt_re_netdev_event(struct notifier_block *notifier, /* Allocate for the deferred task */ re_work = kzalloc(sizeof(*re_work), GFP_ATOMIC); if (re_work) { + get_device(&rdev->ibdev.dev); re_work->rdev = rdev; re_work->event = event; re_work->vlan_dev = (real_dev == netdev ?
On Fri, Feb 28, 2020 at 10:05 PM Jason Gunthorpe <jgg@ziepe.ca> wrote: > > On Wed, Feb 26, 2020 at 07:45:32AM -0800, Selvin Xavier wrote: > > @@ -1724,6 +1714,7 @@ static int bnxt_re_netdev_event(struct notifier_block *notifier, > > rc = bnxt_re_add_device(&rdev, real_dev); > > if (!rc) > > sch_work = true; > > + release = false; > > break; > > > > case NETDEV_UNREGISTER: > > @@ -1732,8 +1723,7 @@ static int bnxt_re_netdev_event(struct notifier_block *notifier, > > */ > > if (atomic_read(&rdev->sched_count) > 0) > > goto exit; > > This sched_count stuff needs cleaning too. > > krefs should be used properly, carry the kref on the ib_device into > the work and use the registration lock on the ib device to serialize > instead of this sched_count stuff. > > This all sounds so familiar.. Oh I tried to fix this once - maybe the > below will help you: > Thanks Jason for the patches. Changes in first patch is already taken care in my series. Will test your other two patches and will get back. Thanks, Selvin > commit 33d88c818d155ffb2ef4b12e72107f628c70404c > Author: Jason Gunthorpe <jgg@ziepe.ca> > Date: Thu Jan 10 12:05:19 2019 -0700 > > RDMA/bnxt_re: Use ib_device_get_by_netdev() instead of open coding > > The core API handles the locking correctly and is faster if there > are multiple devices. > > Signed-off-by: Jason Gunthorpe <jgg@mellanox.com> > > diff --git a/drivers/infiniband/hw/bnxt_re/main.c b/drivers/infiniband/hw/bnxt_re/main.c > index fa539608ffbbe0..bd67a31937ec65 100644 > --- a/drivers/infiniband/hw/bnxt_re/main.c > +++ b/drivers/infiniband/hw/bnxt_re/main.c > @@ -504,21 +504,6 @@ static bool is_bnxt_re_dev(struct net_device *netdev) > return false; > } > > -static struct bnxt_re_dev *bnxt_re_from_netdev(struct net_device *netdev) > -{ > - struct bnxt_re_dev *rdev; > - > - rcu_read_lock(); > - list_for_each_entry_rcu(rdev, &bnxt_re_dev_list, list) { > - if (rdev->netdev == netdev) { > - rcu_read_unlock(); > - return rdev; > - } > - } > - rcu_read_unlock(); > - return NULL; > -} > - > static void bnxt_re_dev_unprobe(struct net_device *netdev, > struct bnxt_en_dev *en_dev) > { > @@ -1616,23 +1601,26 @@ static int bnxt_re_netdev_event(struct notifier_block *notifier, > { > struct net_device *real_dev, *netdev = netdev_notifier_info_to_dev(ptr); > struct bnxt_re_work *re_work; > - struct bnxt_re_dev *rdev; > + struct bnxt_re_dev *rdev = NULL; > + struct ib_device *ibdev; > int rc = 0; > bool sch_work = false; > > real_dev = rdma_vlan_dev_real_dev(netdev); > if (!real_dev) > real_dev = netdev; > - > - rdev = bnxt_re_from_netdev(real_dev); > - if (!rdev && event != NETDEV_REGISTER) > - goto exit; > if (real_dev != netdev) > goto exit; > > + ibdev = ib_device_get_by_netdev(real_dev, RDMA_DRIVER_BNXT_RE); > + if (!ibdev && event != NETDEV_REGISTER) > + goto exit; > + if (ibdev) > + rdev = container_of(ibdev, struct bnxt_re_dev, ibdev); > + > switch (event) { > case NETDEV_REGISTER: > - if (rdev) > + if (ibdev) > break; > rc = bnxt_re_dev_reg(&rdev, real_dev); > if (rc == -ENODEV) > @@ -1676,6 +1664,9 @@ static int bnxt_re_netdev_event(struct notifier_block *notifier, > } > } > > + if (ibdev) > + ib_device_put(ibdev); > + > exit: > return NOTIFY_DONE; > } > > commit 6c617f08e749ee0f6c7be6763ea92e49ae484712 > Author: Jason Gunthorpe <jgg@ziepe.ca> > Date: Thu Jan 10 14:40:16 2019 -0700 > > RDMA/bnxt_re: Use ib_device_try_get() > > There are a couple places in this driver running from a work queue that > need the ib_device to be registered. Instead of using a broken internal > bit rely on the new core code to guarantee device registration. > > Signed-off-by: Jason Gunthorpe <jgg@mellanox.com> > > diff --git a/drivers/infiniband/hw/bnxt_re/main.c b/drivers/infiniband/hw/bnxt_re/main.c > index 666897596218d3..fa539608ffbbe0 100644 > --- a/drivers/infiniband/hw/bnxt_re/main.c > +++ b/drivers/infiniband/hw/bnxt_re/main.c > @@ -1137,12 +1137,13 @@ static int bnxt_re_update_gid(struct bnxt_re_dev *rdev) > u16 gid_idx, index; > int rc = 0; > > - if (!test_bit(BNXT_RE_FLAG_IBDEV_REGISTERED, &rdev->flags)) > + if (!ib_device_try_get(&rdev->ibdev)) > return 0; > > if (!sgid_tbl) { > dev_err(rdev_to_dev(rdev), "QPLIB: SGID table not allocated"); > - return -EINVAL; > + rc = -EINVAL; > + goto out; > } > > for (index = 0; index < sgid_tbl->active; index++) { > @@ -1163,6 +1164,8 @@ static int bnxt_re_update_gid(struct bnxt_re_dev *rdev) > rdev->qplib_res.netdev->dev_addr); > } > > +out: > + ib_device_put(&rdev->ibdev); > return rc; > } > > @@ -1545,12 +1548,7 @@ static void bnxt_re_task(struct work_struct *work) > re_work = container_of(work, struct bnxt_re_work, work); > rdev = re_work->rdev; > > - if (re_work->event != NETDEV_REGISTER && > - !test_bit(BNXT_RE_FLAG_IBDEV_REGISTERED, &rdev->flags)) > - goto exit; > - > - switch (re_work->event) { > - case NETDEV_REGISTER: > + if (re_work->event == NETDEV_REGISTER) { > rc = bnxt_re_ib_reg(rdev); > if (rc) { > dev_err(rdev_to_dev(rdev), > @@ -1559,7 +1557,13 @@ static void bnxt_re_task(struct work_struct *work) > bnxt_re_dev_unreg(rdev); > goto exit; > } > - break; > + goto exit; > + } > + > + if (!ib_device_try_get(&rdev->ibdev)) > + goto exit; > + > + switch (re_work->event) { > case NETDEV_UP: > bnxt_re_dispatch_event(&rdev->ibdev, NULL, 1, > IB_EVENT_PORT_ACTIVE); > @@ -1579,6 +1583,8 @@ static void bnxt_re_task(struct work_struct *work) > default: > break; > } > + > + ib_device_put(&rdev->ibdev); > smp_mb__before_atomic(); > atomic_dec(&rdev->sched_count); > exit: > > commit e64da98a182a2cae3338f28f6e581f189b5f8674 > Author: Jason Gunthorpe <jgg@ziepe.ca> > Date: Thu Jan 10 12:02:11 2019 -0700 > > RDMA/bnxt_re: Fix lifetimes in bnxt_re_task > > A work queue cannot just rely on the ib_device not being freed, it must > hold a kref on the memory so that the BNXT_RE_FLAG_IBDEV_REGISTERED check > works. > > Also, every single work queue call has an allocated memory, and the kfree > of this memory was missed sometimes. > > Fixes: 1ac5a4047975 ("RDMA/bnxt_re: Add bnxt_re RoCE driver") > Signed-off-by: Jason Gunthorpe <jgg@mellanox.com> > > diff --git a/drivers/infiniband/hw/bnxt_re/main.c b/drivers/infiniband/hw/bnxt_re/main.c > index 814f959c7db965..666897596218d3 100644 > --- a/drivers/infiniband/hw/bnxt_re/main.c > +++ b/drivers/infiniband/hw/bnxt_re/main.c > @@ -1547,7 +1547,7 @@ static void bnxt_re_task(struct work_struct *work) > > if (re_work->event != NETDEV_REGISTER && > !test_bit(BNXT_RE_FLAG_IBDEV_REGISTERED, &rdev->flags)) > - return; > + goto exit; > > switch (re_work->event) { > case NETDEV_REGISTER: > @@ -1582,6 +1582,7 @@ static void bnxt_re_task(struct work_struct *work) > smp_mb__before_atomic(); > atomic_dec(&rdev->sched_count); > exit: > + put_device(&rdev->ibdev.dev); > kfree(re_work); > } > > @@ -1658,6 +1659,7 @@ static int bnxt_re_netdev_event(struct notifier_block *notifier, > /* Allocate for the deferred task */ > re_work = kzalloc(sizeof(*re_work), GFP_ATOMIC); > if (re_work) { > + get_device(&rdev->ibdev.dev); > re_work->rdev = rdev; > re_work->event = event; > re_work->vlan_dev = (real_dev == netdev ?
diff --git a/drivers/infiniband/hw/bnxt_re/main.c b/drivers/infiniband/hw/bnxt_re/main.c index 5f8fd74..415693f 100644 --- a/drivers/infiniband/hw/bnxt_re/main.c +++ b/drivers/infiniband/hw/bnxt_re/main.c @@ -79,7 +79,8 @@ static struct list_head bnxt_re_dev_list = LIST_HEAD_INIT(bnxt_re_dev_list); static DEFINE_MUTEX(bnxt_re_dev_lock); static struct workqueue_struct *bnxt_re_wq; static void bnxt_re_remove_device(struct bnxt_re_dev *rdev); -static void bnxt_re_ib_uninit(struct bnxt_re_dev *rdev); +static void bnxt_re_dealloc_driver(struct ib_device *ib_dev); +static void bnxt_re_stop_irq(void *handle); static void bnxt_re_destroy_chip_ctx(struct bnxt_re_dev *rdev) { @@ -237,10 +238,10 @@ static void bnxt_re_shutdown(void *p) if (!rdev) return; - - bnxt_re_ib_uninit(rdev); ASSERT_RTNL(); - bnxt_re_remove_device(rdev); + /* Release the MSIx vectors before queuing unregister */ + bnxt_re_stop_irq(rdev); + ib_unregister_device_queued(&rdev->ibdev); } static void bnxt_re_stop_irq(void *handle) @@ -542,17 +543,12 @@ static bool is_bnxt_re_dev(struct net_device *netdev) static struct bnxt_re_dev *bnxt_re_from_netdev(struct net_device *netdev) { - struct bnxt_re_dev *rdev; + struct ib_device *ibdev = + ib_device_get_by_netdev(netdev, RDMA_DRIVER_BNXT_RE); + if (!ibdev) + return NULL; - rcu_read_lock(); - list_for_each_entry_rcu(rdev, &bnxt_re_dev_list, list) { - if (rdev->netdev == netdev) { - rcu_read_unlock(); - return rdev; - } - } - rcu_read_unlock(); - return NULL; + return container_of(ibdev, struct bnxt_re_dev, ibdev); } static void bnxt_re_dev_unprobe(struct net_device *netdev, @@ -626,11 +622,6 @@ static const struct attribute_group bnxt_re_dev_attr_group = { .attrs = bnxt_re_attributes, }; -static void bnxt_re_unregister_ib(struct bnxt_re_dev *rdev) -{ - ib_unregister_device(&rdev->ibdev); -} - static const struct ib_device_ops bnxt_re_dev_ops = { .owner = THIS_MODULE, .driver_id = RDMA_DRIVER_BNXT_RE, @@ -645,6 +636,7 @@ static const struct ib_device_ops bnxt_re_dev_ops = { .create_cq = bnxt_re_create_cq, .create_qp = bnxt_re_create_qp, .create_srq = bnxt_re_create_srq, + .dealloc_driver = bnxt_re_dealloc_driver, .dealloc_pd = bnxt_re_dealloc_pd, .dealloc_ucontext = bnxt_re_dealloc_ucontext, .del_gid = bnxt_re_del_gid, @@ -741,15 +733,11 @@ static void bnxt_re_dev_remove(struct bnxt_re_dev *rdev) { dev_put(rdev->netdev); rdev->netdev = NULL; - mutex_lock(&bnxt_re_dev_lock); list_del_rcu(&rdev->list); mutex_unlock(&bnxt_re_dev_lock); synchronize_rcu(); - - ib_dealloc_device(&rdev->ibdev); - /* rdev is gone */ } static struct bnxt_re_dev *bnxt_re_dev_add(struct net_device *netdev, @@ -1320,15 +1308,6 @@ static void bnxt_re_query_hwrm_intf_version(struct bnxt_re_dev *rdev) le16_to_cpu(resp.hwrm_intf_patch); } -static void bnxt_re_ib_uninit(struct bnxt_re_dev *rdev) -{ - /* Cleanup ib dev */ - if (test_bit(BNXT_RE_FLAG_IBDEV_REGISTERED, &rdev->flags)) { - ib_unregister_device(&rdev->ibdev); - clear_bit(BNXT_RE_FLAG_IBDEV_REGISTERED, &rdev->flags); - } -} - int bnxt_re_ib_init(struct bnxt_re_dev *rdev) { int rc = 0; @@ -1359,10 +1338,6 @@ static void bnxt_re_dev_uninit(struct bnxt_re_dev *rdev) u8 type; int rc; - if (test_and_clear_bit(BNXT_RE_FLAG_IBDEV_REGISTERED, &rdev->flags)) { - /* Cleanup ib dev */ - bnxt_re_unregister_ib(rdev); - } if (test_and_clear_bit(BNXT_RE_FLAG_QOS_WORK_REG, &rdev->flags)) cancel_delayed_work_sync(&rdev->worker); @@ -1632,6 +1607,19 @@ static int bnxt_re_add_device(struct bnxt_re_dev **rdev, return rc; } +static void bnxt_re_dealloc_driver(struct ib_device *ib_dev) +{ + struct bnxt_re_dev *rdev = + container_of(ib_dev, struct bnxt_re_dev, ibdev); + + clear_bit(BNXT_RE_FLAG_IBDEV_REGISTERED, &rdev->flags); + dev_info(rdev_to_dev(rdev), "Unregistering Device"); + + rtnl_lock(); + bnxt_re_remove_device(rdev); + rtnl_unlock(); +} + /* Handle all deferred netevents tasks */ static void bnxt_re_task(struct work_struct *work) { @@ -1706,6 +1694,7 @@ static int bnxt_re_netdev_event(struct notifier_block *notifier, struct bnxt_re_dev *rdev; int rc = 0; bool sch_work = false; + bool release = true; real_dev = rdma_vlan_dev_real_dev(netdev); if (!real_dev) @@ -1713,7 +1702,8 @@ static int bnxt_re_netdev_event(struct notifier_block *notifier, rdev = bnxt_re_from_netdev(real_dev); if (!rdev && event != NETDEV_REGISTER) - goto exit; + return NOTIFY_OK; + if (real_dev != netdev) goto exit; @@ -1724,6 +1714,7 @@ static int bnxt_re_netdev_event(struct notifier_block *notifier, rc = bnxt_re_add_device(&rdev, real_dev); if (!rc) sch_work = true; + release = false; break; case NETDEV_UNREGISTER: @@ -1732,8 +1723,7 @@ static int bnxt_re_netdev_event(struct notifier_block *notifier, */ if (atomic_read(&rdev->sched_count) > 0) goto exit; - bnxt_re_ib_uninit(rdev); - bnxt_re_remove_device(rdev); + ib_unregister_device_queued(&rdev->ibdev); break; default: @@ -1755,6 +1745,8 @@ static int bnxt_re_netdev_event(struct notifier_block *notifier, } exit: + if (rdev && release) + ib_device_put(&rdev->ibdev); return NOTIFY_DONE; } @@ -1790,35 +1782,21 @@ static int __init bnxt_re_mod_init(void) static void __exit bnxt_re_mod_exit(void) { - struct bnxt_re_dev *rdev, *next; - LIST_HEAD(to_be_deleted); + struct bnxt_re_dev *rdev; - mutex_lock(&bnxt_re_dev_lock); - /* Free all adapter allocated resources */ - if (!list_empty(&bnxt_re_dev_list)) - list_splice_init(&bnxt_re_dev_list, &to_be_deleted); - mutex_unlock(&bnxt_re_dev_lock); - /* - * Cleanup the devices in reverse order so that the VF device - * cleanup is done before PF cleanup - */ - list_for_each_entry_safe_reverse(rdev, next, &to_be_deleted, list) { - ibdev_info(&rdev->ibdev, "Unregistering Device"); - /* - * Flush out any scheduled tasks before destroying the - * resources - */ - flush_workqueue(bnxt_re_wq); - bnxt_re_dev_stop(rdev); - bnxt_re_ib_uninit(rdev); - /* Acquire the rtnl_lock as the L2 resources are freed here */ - rtnl_lock(); - bnxt_re_remove_device(rdev); - rtnl_unlock(); - } unregister_netdevice_notifier(&bnxt_re_netdev_notifier); if (bnxt_re_wq) destroy_workqueue(bnxt_re_wq); + list_for_each_entry(rdev, &bnxt_re_dev_list, list) { + /* VF device removal should be called before the removal + * of PF device. Queue VFs unregister first, so that VFs + * shall be removed before the PF during the call of + * ib_unregister_driver. + */ + if (rdev->is_virtfn) + ib_unregister_device(&rdev->ibdev); + } + ib_unregister_driver(RDMA_DRIVER_BNXT_RE); } module_init(bnxt_re_mod_init);
Using the new unregister APIs provided by the core. Provide the dealloc_driver hook for the core to callback at the time of device un-registration. bnxt_re VF resources are created by the corresponding PF driver. During ib_unregister_driver, PF might get removed before VF and this could cause failure when VFs are removed. Driver is explicitly queuing the removal of VF devices before calling ib_unregister_driver. Signed-off-by: Selvin Xavier <selvin.xavier@broadcom.com> --- drivers/infiniband/hw/bnxt_re/main.c | 106 ++++++++++++++--------------------- 1 file changed, 42 insertions(+), 64 deletions(-)