diff mbox series

[07/16] blocK: move plug flush functions to blk-mq.c

Message ID bb6fe6de2fe6bd69ccf9bc8af049ffedcf52bda0.1634676157.git.asml.silence@gmail.com (mailing list archive)
State New, archived
Headers show
Series block optimisation round | expand

Commit Message

Pavel Begunkov Oct. 19, 2021, 9:24 p.m. UTC 
Flushing is tightly coupled with blk-mq and almost all
blk_flush_plug_list() callees are in blk-mq.c. So move the whole thing
there, so the compiler is able to apply more optimisations and inline.

Signed-off-by: Pavel Begunkov <asml.silence@gmail.com>
---
 block/blk-core.c       | 27 ---------------------------
 block/blk-mq.c         | 33 +++++++++++++++++++++++++++++----
 block/blk-mq.h         |  1 -
 include/linux/blk-mq.h |  2 --
 4 files changed, 29 insertions(+), 34 deletions(-)

Comments

Chaitanya Kulkarni Oct. 19, 2021, 10:34 p.m. UTC | #1 
On 10/19/21 2:24 PM, Pavel Begunkov wrote:
> External email: Use caution opening links or attachments
> 
> 
> Flushing is tightly coupled with blk-mq and almost all
> blk_flush_plug_list() callees are in blk-mq.c. So move the whole thing
> there, so the compiler is able to apply more optimisations and inline.
> 
> Signed-off-by: Pavel Begunkov <asml.silence@gmail.com>
> ---

Looks good.

Reviewed-by: Chaitanya Kulkarni <kch@nvidia.com>
Christoph Hellwig Oct. 20, 2021, 6:17 a.m. UTC | #2 
Spelling error in the subject.

On Tue, Oct 19, 2021 at 10:24:16PM +0100, Pavel Begunkov wrote:
> Flushing is tightly coupled with blk-mq and almost all
> blk_flush_plug_list() callees are in blk-mq.c. So move the whole thing
> there, so the compiler is able to apply more optimisations and inline.

No, it isn't.  The whole callback handling is all about bio based
drivers.
Pavel Begunkov Oct. 20, 2021, 12:23 p.m. UTC | #3 
On 10/20/21 07:17, Christoph Hellwig wrote:
> Spelling error in the subject.
> 
> On Tue, Oct 19, 2021 at 10:24:16PM +0100, Pavel Begunkov wrote:
>> Flushing is tightly coupled with blk-mq and almost all
>> blk_flush_plug_list() callees are in blk-mq.c. So move the whole thing
>> there, so the compiler is able to apply more optimisations and inline.
> 
> No, it isn't.  The whole callback handling is all about bio based
> drivers.

How about leaving flush_plug_callbacks() in blk-core.c but moving
everything else?
Christoph Hellwig Oct. 20, 2021, 12:37 p.m. UTC | #4 
On Wed, Oct 20, 2021 at 01:23:05PM +0100, Pavel Begunkov wrote:
> How about leaving flush_plug_callbacks() in blk-core.c but moving
> everything else?

That's at least a little better.  I'd still prefer to keep the wrappers
out as well.  I've been wondering a bit if the whole callback handling
needs a rework.  Let me thing about this a bit more, maybe I'll have
patches later today.
Pavel Begunkov Oct. 20, 2021, 1:18 p.m. UTC | #5 
On 10/20/21 13:37, Christoph Hellwig wrote:
> On Wed, Oct 20, 2021 at 01:23:05PM +0100, Pavel Begunkov wrote:
>> How about leaving flush_plug_callbacks() in blk-core.c but moving
>> everything else?
> 
> That's at least a little better.  I'd still prefer to keep the wrappers
> out as well.  I've been wondering a bit if the whole callback handling
> needs a rework.  Let me thing about this a bit more, maybe I'll have
> patches later today.

Sure, I'll leave it to you. If you'll be changing parts around,
you can also fold in that part of 8/16 that avoids extra list init
diff mbox series

Patch

diff --git a/block/blk-core.c b/block/blk-core.c
index 88752e51d2b6..52019b8a1487 100644
--- a/block/blk-core.c
+++ b/block/blk-core.c
@@ -1595,23 +1595,6 @@  void blk_start_plug(struct blk_plug *plug)
 }
 EXPORT_SYMBOL(blk_start_plug);
 
-static void flush_plug_callbacks(struct blk_plug *plug, bool from_schedule)
-{
-	LIST_HEAD(callbacks);
-
-	while (!list_empty(&plug->cb_list)) {
-		list_splice_init(&plug->cb_list, &callbacks);
-
-		while (!list_empty(&callbacks)) {
-			struct blk_plug_cb *cb = list_first_entry(&callbacks,
-							  struct blk_plug_cb,
-							  list);
-			list_del(&cb->list);
-			cb->callback(cb, from_schedule);
-		}
-	}
-}
-
 struct blk_plug_cb *blk_check_plugged(blk_plug_cb_fn unplug, void *data,
 				      int size)
 {
@@ -1637,16 +1620,6 @@  struct blk_plug_cb *blk_check_plugged(blk_plug_cb_fn unplug, void *data,
 }
 EXPORT_SYMBOL(blk_check_plugged);
 
-void blk_flush_plug_list(struct blk_plug *plug, bool from_schedule)
-{
-	flush_plug_callbacks(plug, from_schedule);
-
-	if (!rq_list_empty(plug->mq_list))
-		blk_mq_flush_plug_list(plug, from_schedule);
-	if (unlikely(!from_schedule && plug->cached_rq))
-		blk_mq_free_plug_rqs(plug);
-}
-
 /**
  * blk_finish_plug - mark the end of a batch of submitted I/O
  * @plug:	The &struct blk_plug passed to blk_start_plug()
diff --git a/block/blk-mq.c b/block/blk-mq.c
index 218bfaa98591..6bdbaa838030 100644
--- a/block/blk-mq.c
+++ b/block/blk-mq.c
@@ -604,7 +604,7 @@  void blk_mq_free_request(struct request *rq)
 }
 EXPORT_SYMBOL_GPL(blk_mq_free_request);
 
-void blk_mq_free_plug_rqs(struct blk_plug *plug)
+static void blk_mq_free_plug_rqs(struct blk_plug *plug)
 {
 	struct request *rq;
 
@@ -2199,15 +2199,13 @@  static void blk_mq_plug_issue_direct(struct blk_plug *plug, bool from_schedule)
 		blk_mq_commit_rqs(hctx, &queued, from_schedule);
 }
 
-void blk_mq_flush_plug_list(struct blk_plug *plug, bool from_schedule)
+static void blk_mq_flush_plug_list(struct blk_plug *plug, bool from_schedule)
 {
 	struct blk_mq_hw_ctx *this_hctx;
 	struct blk_mq_ctx *this_ctx;
 	unsigned int depth;
 	LIST_HEAD(list);
 
-	if (rq_list_empty(plug->mq_list))
-		return;
 	plug->rq_count = 0;
 
 	if (!plug->multiple_queues && !plug->has_elevator) {
@@ -2249,6 +2247,33 @@  void blk_mq_flush_plug_list(struct blk_plug *plug, bool from_schedule)
 	}
 }
 
+static void flush_plug_callbacks(struct blk_plug *plug, bool from_schedule)
+{
+	LIST_HEAD(callbacks);
+
+	while (!list_empty(&plug->cb_list)) {
+		list_splice_init(&plug->cb_list, &callbacks);
+
+		while (!list_empty(&callbacks)) {
+			struct blk_plug_cb *cb = list_first_entry(&callbacks,
+							  struct blk_plug_cb,
+							  list);
+			list_del(&cb->list);
+			cb->callback(cb, from_schedule);
+		}
+	}
+}
+
+void blk_flush_plug_list(struct blk_plug *plug, bool from_schedule)
+{
+	flush_plug_callbacks(plug, from_schedule);
+
+	if (!rq_list_empty(plug->mq_list))
+		blk_mq_flush_plug_list(plug, from_schedule);
+	if (unlikely(!from_schedule && plug->cached_rq))
+		blk_mq_free_plug_rqs(plug);
+}
+
 static void blk_mq_bio_to_request(struct request *rq, struct bio *bio,
 		unsigned int nr_segs)
 {
diff --git a/block/blk-mq.h b/block/blk-mq.h
index ebf67f4d4f2e..bab40688e59b 100644
--- a/block/blk-mq.h
+++ b/block/blk-mq.h
@@ -121,7 +121,6 @@  extern int __blk_mq_register_dev(struct device *dev, struct request_queue *q);
 extern int blk_mq_sysfs_register(struct request_queue *q);
 extern void blk_mq_sysfs_unregister(struct request_queue *q);
 extern void blk_mq_hctx_kobj_init(struct blk_mq_hw_ctx *hctx);
-void blk_mq_free_plug_rqs(struct blk_plug *plug);
 
 void blk_mq_release(struct request_queue *q);
 
diff --git a/include/linux/blk-mq.h b/include/linux/blk-mq.h
index 6cf35de151a9..e13780236550 100644
--- a/include/linux/blk-mq.h
+++ b/include/linux/blk-mq.h
@@ -656,8 +656,6 @@  int blk_mq_alloc_sq_tag_set(struct blk_mq_tag_set *set,
 		unsigned int set_flags);
 void blk_mq_free_tag_set(struct blk_mq_tag_set *set);
 
-void blk_mq_flush_plug_list(struct blk_plug *plug, bool from_schedule);
-
 void blk_mq_free_request(struct request *rq);
 
 bool blk_mq_queue_inflight(struct request_queue *q);