[v4,2/2] blk-mq: break more earlier when interate hctx

Commit Message

Weiping Zhang Oct. 26, 2020, 6:15 a.m. UTC

For blk_mq_part_is_in_inflight and blk_mq_queue_inflight they do not
care how many inflight IOs, so they stop interate other hxtc when find
a request meets their requirement. Some cpu cycles can be saved in such
way.

Signed-off-by: Weiping Zhang <[e-mail]>
---
 block/blk-mq-tag.c     | 11 +++++++++--
 block/blk-mq-tag.h     |  2 +-
 block/blk-mq.c         | 34 +++++++++++++++++++++++++++++-----
 include/linux/blk-mq.h |  1 +
 4 files changed, 40 insertions(+), 8 deletions(-)

Patch

diff --git a/block/blk-mq-tag.c b/block/blk-mq-tag.c
index 9c92053e704d..f364682dabe1 100644
--- a/block/blk-mq-tag.c
+++ b/block/blk-mq-tag.c
@@ -401,14 +401,17 @@  EXPORT_SYMBOL(blk_mq_tagset_wait_completed_request);
  *		reserved) where rq is a pointer to a request and hctx points
  *		to the hardware queue associated with the request. 'reserved'
  *		indicates whether or not @rq is a reserved request.
- * @priv:	Will be passed as third argument to @fn.
+ *@check_break: Pointer to the function that will callbed for earch hctx on @q.
+ *		@check_break will break the loop for hctx when it return false,
+ *		if you want to iterate all hctx, set it to NULL.
+ * @priv:	Will be passed as third argument to @fn, or arg to @check_break
  *
  * Note: if @q->tag_set is shared with other request queues then @fn will be
  * called for all requests on all queues that share that tag set and not only
  * for requests associated with @q.
  */
 void blk_mq_queue_tag_busy_iter(struct request_queue *q, busy_iter_fn *fn,
-		void *priv)
+		check_break_fn *check_break, void *priv)
 {
 	struct blk_mq_hw_ctx *hctx;
 	int i;
@@ -434,7 +437,11 @@  void blk_mq_queue_tag_busy_iter(struct request_queue *q, busy_iter_fn *fn,
 		if (tags->nr_reserved_tags)
 			bt_for_each(hctx, tags->breserved_tags, fn, priv, true);
 		bt_for_each(hctx, tags->bitmap_tags, fn, priv, false);
+
+		if (check_break && !check_break(priv))
+			goto out;
 	}
+out:
 	blk_queue_exit(q);
 }
 
diff --git a/block/blk-mq-tag.h b/block/blk-mq-tag.h
index 7d3e6b333a4a..d122be9f87cb 100644
--- a/block/blk-mq-tag.h
+++ b/block/blk-mq-tag.h
@@ -42,7 +42,7 @@  extern void blk_mq_tag_resize_shared_sbitmap(struct blk_mq_tag_set *set,
 
 extern void blk_mq_tag_wakeup_all(struct blk_mq_tags *tags, bool);
 void blk_mq_queue_tag_busy_iter(struct request_queue *q, busy_iter_fn *fn,
-		void *priv);
+		check_break_fn *check_break, void *priv);
 void blk_mq_all_tag_iter(struct blk_mq_tags *tags, busy_tag_iter_fn *fn,
 		void *priv);
 
diff --git a/block/blk-mq.c b/block/blk-mq.c
index 126a6a6f7035..458ade751b01 100644
--- a/block/blk-mq.c
+++ b/block/blk-mq.c
@@ -115,7 +115,7 @@  unsigned int blk_mq_in_flight(struct request_queue *q, struct hd_struct *part)
 {
 	struct mq_inflight mi = { .part = part };
 
-	blk_mq_queue_tag_busy_iter(q, blk_mq_check_inflight, &mi);
+	blk_mq_queue_tag_busy_iter(q, blk_mq_check_inflight, NULL, &mi);
 
 	return mi.inflight[0] + mi.inflight[1];
 }
@@ -125,11 +125,22 @@  void blk_mq_in_flight_rw(struct request_queue *q, struct hd_struct *part,
 {
 	struct mq_inflight mi = { .part = part };
 
-	blk_mq_queue_tag_busy_iter(q, blk_mq_check_inflight, &mi);
+	blk_mq_queue_tag_busy_iter(q, blk_mq_check_inflight, NULL, &mi);
 	inflight[0] = mi.inflight[0];
 	inflight[1] = mi.inflight[1];
 }
 
+static bool blk_mq_part_check_break(void *priv)
+{
+	struct mq_inflight *mi = priv;
+
+	/* return false to stop interate other hctx */
+	if (mi->inflight[0] || mi->inflight[1])
+		return false;
+
+	return true;
+}
+
 static bool blk_mq_part_check_inflight(struct blk_mq_hw_ctx *hctx,
 				  struct request *rq, void *priv,
 				  bool reserved)
@@ -151,7 +162,8 @@  bool blk_mq_part_is_in_flight(struct request_queue *q, struct hd_struct *part)
 
 	mi.inflight[0] = mi.inflight[1] = 0;
 
-	blk_mq_queue_tag_busy_iter(q, blk_mq_part_check_inflight, &mi);
+	blk_mq_queue_tag_busy_iter(q, blk_mq_part_check_inflight,
+					blk_mq_part_check_break, &mi);
 
 	return mi.inflight[0] + mi.inflight[1] > 0;
 }
@@ -909,11 +921,23 @@  static bool blk_mq_rq_inflight(struct blk_mq_hw_ctx *hctx, struct request *rq,
 	return true;
 }
 
+static bool blk_mq_rq_check_break(void *priv)
+{
+	bool *busy = priv;
+
+	/* return false to stop interate other hctx */
+	if (*busy)
+		return false;
+
+	return true;
+}
+
 bool blk_mq_queue_inflight(struct request_queue *q)
 {
 	bool busy = false;
 
-	blk_mq_queue_tag_busy_iter(q, blk_mq_rq_inflight, &busy);
+	blk_mq_queue_tag_busy_iter(q, blk_mq_rq_inflight,
+			blk_mq_rq_check_break, &busy);
 	return busy;
 }
 EXPORT_SYMBOL_GPL(blk_mq_queue_inflight);
@@ -1018,7 +1042,7 @@  static void blk_mq_timeout_work(struct work_struct *work)
 	if (!percpu_ref_tryget(&q->q_usage_counter))
 		return;
 
-	blk_mq_queue_tag_busy_iter(q, blk_mq_check_expired, &next);
+	blk_mq_queue_tag_busy_iter(q, blk_mq_check_expired, NULL, &next);
 
 	if (next != 0) {
 		mod_timer(&q->timeout, next);
diff --git a/include/linux/blk-mq.h b/include/linux/blk-mq.h
index b23eeca4d677..efd1e8269f0b 100644
--- a/include/linux/blk-mq.h
+++ b/include/linux/blk-mq.h
@@ -280,6 +280,7 @@  struct blk_mq_queue_data {
 typedef bool (busy_iter_fn)(struct blk_mq_hw_ctx *, struct request *, void *,
 		bool);
 typedef bool (busy_tag_iter_fn)(struct request *, void *, bool);
+typedef bool (check_break_fn)(void *);
 
 /**
  * struct blk_mq_ops - Callback functions that implements block driver