[v2] RDMA/core: Avoid flush_workqueue(system_unbound_wq) usage

Commit Message

Tetsuo Handa April 19, 2022, 10:18 p.m. UTC

On 2022/04/20 6:01, Jason Gunthorpe wrote:
> The flush_work exists because of this:
> 
>> @@ -1602,7 +1604,7 @@ void ib_unregister_device_queued(struct ib_device *ib_dev)
>>  	WARN_ON(!refcount_read(&ib_dev->refcount));
>>  	WARN_ON(!ib_dev->ops.dealloc_driver);
>>  	get_device(&ib_dev->dev);
>> -	if (!queue_work(system_unbound_wq, &ib_dev->unregistration_work))
>> +	if (!queue_work(ib_unbound_wq, &ib_dev->unregistration_work))
>>  		put_device(&ib_dev->dev);
>>  }
> 
> Which fires off a work that must be completed before the module is
> unloaded.

I see. Here is an updated patch.

> 
> It seems like a big waste to create an entire WQ just for a single
> infrequent user like this.

No need to worry about resource wasting, for creating a WQ without
WQ_MEM_RECLAIM flag consumes little memory.

> 
> Is there some other solution?

No.

From 731f7f167a95946cac52da6cf5964a2904a9164c Mon Sep 17 00:00:00 2001
From: Tetsuo Handa <[e-mail]>
Date: Wed, 20 Apr 2022 07:10:50 +0900
Subject: [PATCH v2] RDMA/core: Avoid flush_workqueue(system_unbound_wq) usage

Flushing system-wide workqueues is dangerous and will be forbidden.
Replace system_unbound_wq with local ib_unbound_wq.

Link: https://lkml.kernel.org/r/[e-mail]
Signed-off-by: Tetsuo Handa <[e-mail]>
---
Changes in v2:
  Need to use local ib_unbound_wq only when unloading, pointed by Jason Gunthorpe <[e-mail]>.

 drivers/infiniband/core/device.c | 24 ++++++++++++++----------
 1 file changed, 14 insertions(+), 10 deletions(-)

Comments

Jason Gunthorpe April 25, 2022, 7:53 p.m. UTC | #1

> >From 731f7f167a95946cac52da6cf5964a2904a9164c Mon Sep 17 00:00:00 2001
> From: Tetsuo Handa <[e-mail]>
> Date: Wed, 20 Apr 2022 07:10:50 +0900
> Subject: [PATCH v2] RDMA/core: Avoid flush_workqueue(system_unbound_wq) usage
> 
> Flushing system-wide workqueues is dangerous and will be forbidden.
> Replace system_unbound_wq with local ib_unbound_wq.
> 
> Link: https://lkml.kernel.org/r/[e-mail]
> Signed-off-by: Tetsuo Handa <[e-mail]>
> Changes in v2:
>   Need to use local ib_unbound_wq only when unloading, pointed by Jason Gunthorpe <[e-mail]>.
> 
>  drivers/infiniband/core/device.c | 24 ++++++++++++++----------
>  1 file changed, 14 insertions(+), 10 deletions(-)

I renamed it to ib_unreg_wq but otherwise applied, thanks

Jason

Patch

diff --git a/drivers/infiniband/core/device.c b/drivers/infiniband/core/device.c
index 4deb60a3b43f..f0fdf4dbb458 100644
--- a/drivers/infiniband/core/device.c
+++ b/drivers/infiniband/core/device.c
@@ -58,6 +58,7 @@  struct workqueue_struct *ib_comp_wq;
 struct workqueue_struct *ib_comp_unbound_wq;
 struct workqueue_struct *ib_wq;
 EXPORT_SYMBOL_GPL(ib_wq);
+static struct workqueue_struct *ib_unbound_wq;
 
 /*
  * Each of the three rwsem locks (devices, clients, client_data) protects the
@@ -1602,7 +1603,7 @@  void ib_unregister_device_queued(struct ib_device *ib_dev)
 	WARN_ON(!refcount_read(&ib_dev->refcount));
 	WARN_ON(!ib_dev->ops.dealloc_driver);
 	get_device(&ib_dev->dev);
-	if (!queue_work(system_unbound_wq, &ib_dev->unregistration_work))
+	if (!queue_work(ib_unbound_wq, &ib_dev->unregistration_work))
 		put_device(&ib_dev->dev);
 }
 EXPORT_SYMBOL(ib_unregister_device_queued);
@@ -2751,27 +2752,28 @@  static const struct rdma_nl_cbs ibnl_ls_cb_table[RDMA_NL_LS_NUM_OPS] = {
 
 static int __init ib_core_init(void)
 {
-	int ret;
+	int ret = -ENOMEM;
 
 	ib_wq = alloc_workqueue("infiniband", 0, 0);
 	if (!ib_wq)
 		return -ENOMEM;
 
+	ib_unbound_wq = alloc_workqueue("ib-unb-wq", WQ_UNBOUND,
+					WQ_UNBOUND_MAX_ACTIVE);
+	if (!ib_unbound_wq)
+		goto err;
+
 	ib_comp_wq = alloc_workqueue("ib-comp-wq",
 			WQ_HIGHPRI | WQ_MEM_RECLAIM | WQ_SYSFS, 0);
-	if (!ib_comp_wq) {
-		ret = -ENOMEM;
-		goto err;
-	}
+	if (!ib_comp_wq)
+		goto err_unbound;
 
 	ib_comp_unbound_wq =
 		alloc_workqueue("ib-comp-unb-wq",
 				WQ_UNBOUND | WQ_HIGHPRI | WQ_MEM_RECLAIM |
 				WQ_SYSFS, WQ_UNBOUND_MAX_ACTIVE);
-	if (!ib_comp_unbound_wq) {
-		ret = -ENOMEM;
+	if (!ib_comp_unbound_wq)
 		goto err_comp;
-	}
 
 	ret = class_register(&ib_class);
 	if (ret) {
@@ -2831,6 +2833,8 @@  static int __init ib_core_init(void)
 	destroy_workqueue(ib_comp_unbound_wq);
 err_comp:
 	destroy_workqueue(ib_comp_wq);
+err_unbound:
+	destroy_workqueue(ib_unbound_wq);
 err:
 	destroy_workqueue(ib_wq);
 	return ret;
@@ -2852,7 +2856,7 @@  static void __exit ib_core_cleanup(void)
 	destroy_workqueue(ib_comp_wq);
 	/* Make sure that any pending umem accounting work is done. */
 	destroy_workqueue(ib_wq);
-	flush_workqueue(system_unbound_wq);
+	destroy_workqueue(ib_unbound_wq);
 	WARN_ON(!xa_empty(&clients));
 	WARN_ON(!xa_empty(&devices));
 }