[V4,5/6] blk-mq: pass obtained budget count to blk_mq_dispatch_rq_list
diff mbox series

Message ID 20200602091502.1822499-6-ming.lei@redhat.com
State New
Headers show
Series
  • blk-mq: support batching dispatch from scheduler
Related show

Commit Message

Ming Lei June 2, 2020, 9:15 a.m. UTC
Pass obtained budget count to blk_mq_dispatch_rq_list(), and prepare
for supporting fully batching submission.

With the obtained budget count, it is easier to put extra budgets
in case of .queue_rq failure.

Meantime remove the old 'got_budget' parameter.

Cc: Sagi Grimberg <sagi@grimberg.me>
Cc: Baolin Wang <baolin.wang7@gmail.com>
Cc: Christoph Hellwig <hch@infradead.org>
Tested-by: Baolin Wang <baolin.wang7@gmail.com>
Signed-off-by: Ming Lei <ming.lei@redhat.com>
---
 block/blk-mq-sched.c |  8 ++++----
 block/blk-mq.c       | 27 +++++++++++++++++++++++----
 block/blk-mq.h       |  3 ++-
 3 files changed, 29 insertions(+), 9 deletions(-)

Comments

Damien Le Moal June 2, 2020, 9:45 a.m. UTC | #1
On 2020/06/02 18:15, Ming Lei wrote:
> Pass obtained budget count to blk_mq_dispatch_rq_list(), and prepare
> for supporting fully batching submission.
> 
> With the obtained budget count, it is easier to put extra budgets
> in case of .queue_rq failure.
> 
> Meantime remove the old 'got_budget' parameter.
> 
> Cc: Sagi Grimberg <sagi@grimberg.me>
> Cc: Baolin Wang <baolin.wang7@gmail.com>
> Cc: Christoph Hellwig <hch@infradead.org>
> Tested-by: Baolin Wang <baolin.wang7@gmail.com>
> Signed-off-by: Ming Lei <ming.lei@redhat.com>
> ---
>  block/blk-mq-sched.c |  8 ++++----
>  block/blk-mq.c       | 27 +++++++++++++++++++++++----
>  block/blk-mq.h       |  3 ++-
>  3 files changed, 29 insertions(+), 9 deletions(-)
> 
> diff --git a/block/blk-mq-sched.c b/block/blk-mq-sched.c
> index 632c6f8b63f7..4c72073830f3 100644
> --- a/block/blk-mq-sched.c
> +++ b/block/blk-mq-sched.c
> @@ -130,7 +130,7 @@ static int blk_mq_do_dispatch_sched(struct blk_mq_hw_ctx *hctx)
>  		 * in blk_mq_dispatch_rq_list().
>  		 */
>  		list_add(&rq->queuelist, &rq_list);
> -	} while (blk_mq_dispatch_rq_list(rq->mq_hctx, &rq_list, true));
> +	} while (blk_mq_dispatch_rq_list(rq->mq_hctx, &rq_list, 1));
>  
>  	return ret;
>  }
> @@ -198,7 +198,7 @@ static int blk_mq_do_dispatch_ctx(struct blk_mq_hw_ctx *hctx)
>  		/* round robin for fair dispatch */
>  		ctx = blk_mq_next_ctx(hctx, rq->mq_ctx);
>  
> -	} while (blk_mq_dispatch_rq_list(rq->mq_hctx, &rq_list, true));
> +	} while (blk_mq_dispatch_rq_list(rq->mq_hctx, &rq_list, 1));
>  
>  	WRITE_ONCE(hctx->dispatch_from, ctx);
>  	return ret;
> @@ -238,7 +238,7 @@ static int __blk_mq_sched_dispatch_requests(struct blk_mq_hw_ctx *hctx)
>  	 */
>  	if (!list_empty(&rq_list)) {
>  		blk_mq_sched_mark_restart_hctx(hctx);
> -		if (blk_mq_dispatch_rq_list(hctx, &rq_list, false)) {
> +		if (blk_mq_dispatch_rq_list(hctx, &rq_list, 0)) {
>  			if (has_sched_dispatch)
>  				ret = blk_mq_do_dispatch_sched(hctx);
>  			else
> @@ -251,7 +251,7 @@ static int __blk_mq_sched_dispatch_requests(struct blk_mq_hw_ctx *hctx)
>  		ret = blk_mq_do_dispatch_ctx(hctx);
>  	} else {
>  		blk_mq_flush_busy_ctxs(hctx, &rq_list);
> -		blk_mq_dispatch_rq_list(hctx, &rq_list, false);
> +		blk_mq_dispatch_rq_list(hctx, &rq_list, 0);
>  	}
>  
>  	return ret;
> diff --git a/block/blk-mq.c b/block/blk-mq.c
> index 0e3aab91e6c0..901ef0264e44 100644
> --- a/block/blk-mq.c
> +++ b/block/blk-mq.c
> @@ -1259,7 +1259,8 @@ static enum prep_dispatch blk_mq_prep_dispatch_rq(struct request *rq,
>  		 */
>  		if (!blk_mq_mark_tag_wait(hctx, rq)) {
>  			/* budget is always obtained before getting tag */
> -			blk_mq_put_dispatch_budget(rq->q);
> +			if (ask_budget)
> +				blk_mq_put_dispatch_budget(rq->q);
>  			return PREP_DISPATCH_NO_TAG;
>  		}
>  	}
> @@ -1267,11 +1268,21 @@ static enum prep_dispatch blk_mq_prep_dispatch_rq(struct request *rq,
>  	return PREP_DISPATCH_OK;
>  }
>  
> +static void blk_mq_release_budgets(struct request_queue *q,
> +		unsigned int nr_budgets)
> +{
> +	int i = 0;
> +
> +	/* release got budgets */
> +	while (i++ < nr_budgets)

A for loop would be simpler I think...

> +		blk_mq_put_dispatch_budget(q);
> +}
> +
>  /*
>   * Returns true if we did some work AND can potentially do more.
>   */
>  bool blk_mq_dispatch_rq_list(struct blk_mq_hw_ctx *hctx, struct list_head *list,
> -			     bool got_budget)
> +			     unsigned int nr_budgets)
>  {
>  	enum prep_dispatch prep;
>  	struct request_queue *q = hctx->queue;
> @@ -1283,7 +1294,7 @@ bool blk_mq_dispatch_rq_list(struct blk_mq_hw_ctx *hctx, struct list_head *list,
>  	if (list_empty(list))
>  		return false;
>  
> -	WARN_ON(!list_is_singular(list) && got_budget);
> +	WARN_ON(!list_is_singular(list) && nr_budgets);
>  
>  	/*
>  	 * Now process all the entries, sending them to the driver.
> @@ -1295,7 +1306,7 @@ bool blk_mq_dispatch_rq_list(struct blk_mq_hw_ctx *hctx, struct list_head *list,
>  		rq = list_first_entry(list, struct request, queuelist);
>  
>  		WARN_ON_ONCE(hctx != rq->mq_hctx);
> -		prep = blk_mq_prep_dispatch_rq(rq, !got_budget);
> +		prep = blk_mq_prep_dispatch_rq(rq, !nr_budgets);
>  		if (prep != PREP_DISPATCH_OK)
>  			break;
>  
> @@ -1314,6 +1325,12 @@ bool blk_mq_dispatch_rq_list(struct blk_mq_hw_ctx *hctx, struct list_head *list,
>  			bd.last = !blk_mq_get_driver_tag(nxt);
>  		}
>  
> +		/*
> +		 * once the request is queued to lld, no need to cover the
> +		 * budget any more
> +		 */
> +		if (nr_budgets)
> +			nr_budgets--;
>  		ret = q->mq_ops->queue_rq(hctx, &bd);
>  		if (ret == BLK_STS_RESOURCE || ret == BLK_STS_DEV_RESOURCE) {
>  			blk_mq_handle_dev_resource(rq, list);
> @@ -1353,6 +1370,8 @@ bool blk_mq_dispatch_rq_list(struct blk_mq_hw_ctx *hctx, struct list_head *list,
>  		bool no_tag = false;
>  		bool no_budget_avail = false;
>  
> +		blk_mq_release_budgets(q, nr_budgets);
> +
>  		/*
>  		 * For non-shared tags, the RESTART check
>  		 * will suffice.
> diff --git a/block/blk-mq.h b/block/blk-mq.h
> index d2d737b16e0e..f3a93acfad03 100644
> --- a/block/blk-mq.h
> +++ b/block/blk-mq.h
> @@ -40,7 +40,8 @@ struct blk_mq_ctx {
>  void blk_mq_exit_queue(struct request_queue *q);
>  int blk_mq_update_nr_requests(struct request_queue *q, unsigned int nr);
>  void blk_mq_wake_waiters(struct request_queue *q);
> -bool blk_mq_dispatch_rq_list(struct blk_mq_hw_ctx *hctx, struct list_head *, bool);
> +bool blk_mq_dispatch_rq_list(struct blk_mq_hw_ctx *hctx, struct list_head *,
> +			     unsigned int);
>  void blk_mq_add_to_requeue_list(struct request *rq, bool at_head,
>  				bool kick_requeue_list);
>  void blk_mq_flush_busy_ctxs(struct blk_mq_hw_ctx *hctx, struct list_head *list);
>

Patch
diff mbox series

diff --git a/block/blk-mq-sched.c b/block/blk-mq-sched.c
index 632c6f8b63f7..4c72073830f3 100644
--- a/block/blk-mq-sched.c
+++ b/block/blk-mq-sched.c
@@ -130,7 +130,7 @@  static int blk_mq_do_dispatch_sched(struct blk_mq_hw_ctx *hctx)
 		 * in blk_mq_dispatch_rq_list().
 		 */
 		list_add(&rq->queuelist, &rq_list);
-	} while (blk_mq_dispatch_rq_list(rq->mq_hctx, &rq_list, true));
+	} while (blk_mq_dispatch_rq_list(rq->mq_hctx, &rq_list, 1));
 
 	return ret;
 }
@@ -198,7 +198,7 @@  static int blk_mq_do_dispatch_ctx(struct blk_mq_hw_ctx *hctx)
 		/* round robin for fair dispatch */
 		ctx = blk_mq_next_ctx(hctx, rq->mq_ctx);
 
-	} while (blk_mq_dispatch_rq_list(rq->mq_hctx, &rq_list, true));
+	} while (blk_mq_dispatch_rq_list(rq->mq_hctx, &rq_list, 1));
 
 	WRITE_ONCE(hctx->dispatch_from, ctx);
 	return ret;
@@ -238,7 +238,7 @@  static int __blk_mq_sched_dispatch_requests(struct blk_mq_hw_ctx *hctx)
 	 */
 	if (!list_empty(&rq_list)) {
 		blk_mq_sched_mark_restart_hctx(hctx);
-		if (blk_mq_dispatch_rq_list(hctx, &rq_list, false)) {
+		if (blk_mq_dispatch_rq_list(hctx, &rq_list, 0)) {
 			if (has_sched_dispatch)
 				ret = blk_mq_do_dispatch_sched(hctx);
 			else
@@ -251,7 +251,7 @@  static int __blk_mq_sched_dispatch_requests(struct blk_mq_hw_ctx *hctx)
 		ret = blk_mq_do_dispatch_ctx(hctx);
 	} else {
 		blk_mq_flush_busy_ctxs(hctx, &rq_list);
-		blk_mq_dispatch_rq_list(hctx, &rq_list, false);
+		blk_mq_dispatch_rq_list(hctx, &rq_list, 0);
 	}
 
 	return ret;
diff --git a/block/blk-mq.c b/block/blk-mq.c
index 0e3aab91e6c0..901ef0264e44 100644
--- a/block/blk-mq.c
+++ b/block/blk-mq.c
@@ -1259,7 +1259,8 @@  static enum prep_dispatch blk_mq_prep_dispatch_rq(struct request *rq,
 		 */
 		if (!blk_mq_mark_tag_wait(hctx, rq)) {
 			/* budget is always obtained before getting tag */
-			blk_mq_put_dispatch_budget(rq->q);
+			if (ask_budget)
+				blk_mq_put_dispatch_budget(rq->q);
 			return PREP_DISPATCH_NO_TAG;
 		}
 	}
@@ -1267,11 +1268,21 @@  static enum prep_dispatch blk_mq_prep_dispatch_rq(struct request *rq,
 	return PREP_DISPATCH_OK;
 }
 
+static void blk_mq_release_budgets(struct request_queue *q,
+		unsigned int nr_budgets)
+{
+	int i = 0;
+
+	/* release got budgets */
+	while (i++ < nr_budgets)
+		blk_mq_put_dispatch_budget(q);
+}
+
 /*
  * Returns true if we did some work AND can potentially do more.
  */
 bool blk_mq_dispatch_rq_list(struct blk_mq_hw_ctx *hctx, struct list_head *list,
-			     bool got_budget)
+			     unsigned int nr_budgets)
 {
 	enum prep_dispatch prep;
 	struct request_queue *q = hctx->queue;
@@ -1283,7 +1294,7 @@  bool blk_mq_dispatch_rq_list(struct blk_mq_hw_ctx *hctx, struct list_head *list,
 	if (list_empty(list))
 		return false;
 
-	WARN_ON(!list_is_singular(list) && got_budget);
+	WARN_ON(!list_is_singular(list) && nr_budgets);
 
 	/*
 	 * Now process all the entries, sending them to the driver.
@@ -1295,7 +1306,7 @@  bool blk_mq_dispatch_rq_list(struct blk_mq_hw_ctx *hctx, struct list_head *list,
 		rq = list_first_entry(list, struct request, queuelist);
 
 		WARN_ON_ONCE(hctx != rq->mq_hctx);
-		prep = blk_mq_prep_dispatch_rq(rq, !got_budget);
+		prep = blk_mq_prep_dispatch_rq(rq, !nr_budgets);
 		if (prep != PREP_DISPATCH_OK)
 			break;
 
@@ -1314,6 +1325,12 @@  bool blk_mq_dispatch_rq_list(struct blk_mq_hw_ctx *hctx, struct list_head *list,
 			bd.last = !blk_mq_get_driver_tag(nxt);
 		}
 
+		/*
+		 * once the request is queued to lld, no need to cover the
+		 * budget any more
+		 */
+		if (nr_budgets)
+			nr_budgets--;
 		ret = q->mq_ops->queue_rq(hctx, &bd);
 		if (ret == BLK_STS_RESOURCE || ret == BLK_STS_DEV_RESOURCE) {
 			blk_mq_handle_dev_resource(rq, list);
@@ -1353,6 +1370,8 @@  bool blk_mq_dispatch_rq_list(struct blk_mq_hw_ctx *hctx, struct list_head *list,
 		bool no_tag = false;
 		bool no_budget_avail = false;
 
+		blk_mq_release_budgets(q, nr_budgets);
+
 		/*
 		 * For non-shared tags, the RESTART check
 		 * will suffice.
diff --git a/block/blk-mq.h b/block/blk-mq.h
index d2d737b16e0e..f3a93acfad03 100644
--- a/block/blk-mq.h
+++ b/block/blk-mq.h
@@ -40,7 +40,8 @@  struct blk_mq_ctx {
 void blk_mq_exit_queue(struct request_queue *q);
 int blk_mq_update_nr_requests(struct request_queue *q, unsigned int nr);
 void blk_mq_wake_waiters(struct request_queue *q);
-bool blk_mq_dispatch_rq_list(struct blk_mq_hw_ctx *hctx, struct list_head *, bool);
+bool blk_mq_dispatch_rq_list(struct blk_mq_hw_ctx *hctx, struct list_head *,
+			     unsigned int);
 void blk_mq_add_to_requeue_list(struct request *rq, bool at_head,
 				bool kick_requeue_list);
 void blk_mq_flush_busy_ctxs(struct blk_mq_hw_ctx *hctx, struct list_head *list);