@@ -130,7 +130,7 @@ static int blk_mq_do_dispatch_sched(struct blk_mq_hw_ctx *hctx)
* in blk_mq_dispatch_rq_list().
*/
list_add(&rq->queuelist, &rq_list);
- } while (blk_mq_dispatch_rq_list(rq->mq_hctx, &rq_list, true));
+ } while (blk_mq_dispatch_rq_list(rq->mq_hctx, &rq_list, 1));
return ret;
}
@@ -198,7 +198,7 @@ static int blk_mq_do_dispatch_ctx(struct blk_mq_hw_ctx *hctx)
/* round robin for fair dispatch */
ctx = blk_mq_next_ctx(hctx, rq->mq_ctx);
- } while (blk_mq_dispatch_rq_list(rq->mq_hctx, &rq_list, true));
+ } while (blk_mq_dispatch_rq_list(rq->mq_hctx, &rq_list, 1));
WRITE_ONCE(hctx->dispatch_from, ctx);
return ret;
@@ -238,7 +238,7 @@ static int __blk_mq_sched_dispatch_requests(struct blk_mq_hw_ctx *hctx)
*/
if (!list_empty(&rq_list)) {
blk_mq_sched_mark_restart_hctx(hctx);
- if (blk_mq_dispatch_rq_list(hctx, &rq_list, false)) {
+ if (blk_mq_dispatch_rq_list(hctx, &rq_list, 0)) {
if (has_sched_dispatch)
ret = blk_mq_do_dispatch_sched(hctx);
else
@@ -251,7 +251,7 @@ static int __blk_mq_sched_dispatch_requests(struct blk_mq_hw_ctx *hctx)
ret = blk_mq_do_dispatch_ctx(hctx);
} else {
blk_mq_flush_busy_ctxs(hctx, &rq_list);
- blk_mq_dispatch_rq_list(hctx, &rq_list, false);
+ blk_mq_dispatch_rq_list(hctx, &rq_list, 0);
}
return ret;
@@ -1256,7 +1256,8 @@ static enum prep_dispatch blk_mq_prep_dispatch_rq(struct request *rq,
*/
if (!blk_mq_mark_tag_wait(hctx, rq)) {
/* budget is always obtained before getting tag */
- blk_mq_put_dispatch_budget(rq->q);
+ if (ask_budget)
+ blk_mq_put_dispatch_budget(rq->q);
return PREP_DISPATCH_NO_TAG;
}
}
@@ -1268,7 +1269,7 @@ static enum prep_dispatch blk_mq_prep_dispatch_rq(struct request *rq,
* Returns true if we did some work AND can potentially do more.
*/
bool blk_mq_dispatch_rq_list(struct blk_mq_hw_ctx *hctx, struct list_head *list,
- bool got_budget)
+ unsigned int nr_budgets)
{
enum prep_dispatch prep;
struct request_queue *q = hctx->queue;
@@ -1280,7 +1281,7 @@ bool blk_mq_dispatch_rq_list(struct blk_mq_hw_ctx *hctx, struct list_head *list,
if (list_empty(list))
return false;
- WARN_ON(!list_is_singular(list) && got_budget);
+ WARN_ON(!list_is_singular(list) && nr_budgets);
/*
* Now process all the entries, sending them to the driver.
@@ -1292,7 +1293,7 @@ bool blk_mq_dispatch_rq_list(struct blk_mq_hw_ctx *hctx, struct list_head *list,
rq = list_first_entry(list, struct request, queuelist);
WARN_ON_ONCE(hctx != rq->mq_hctx);
- prep = blk_mq_prep_dispatch_rq(rq, !got_budget);
+ prep = blk_mq_prep_dispatch_rq(rq, !nr_budgets);
if (prep != PREP_DISPATCH_OK)
break;
@@ -1349,7 +1350,11 @@ bool blk_mq_dispatch_rq_list(struct blk_mq_hw_ctx *hctx, struct list_head *list,
bool needs_restart;
bool no_tag = false;
bool no_budget_avail = false;
+ unsigned i = 0;
+ /* release got budgets */
+ while (i++ < nr_budgets)
+ blk_mq_put_dispatch_budget(hctx->queue);
/*
* For non-shared tags, the RESTART check
* will suffice.
@@ -40,7 +40,8 @@ struct blk_mq_ctx {
void blk_mq_exit_queue(struct request_queue *q);
int blk_mq_update_nr_requests(struct request_queue *q, unsigned int nr);
void blk_mq_wake_waiters(struct request_queue *q);
-bool blk_mq_dispatch_rq_list(struct blk_mq_hw_ctx *hctx, struct list_head *, bool);
+bool blk_mq_dispatch_rq_list(struct blk_mq_hw_ctx *hctx, struct list_head *,
+ unsigned int);
void blk_mq_add_to_requeue_list(struct request *rq, bool at_head,
bool kick_requeue_list);
void blk_mq_flush_busy_ctxs(struct blk_mq_hw_ctx *hctx, struct list_head *list);
Pass obtained budget count to blk_mq_dispatch_rq_list(), and prepare for supporting fully batching submission. With the obtained budget count, it is easier to put extra budgets in case of .queue_rq failure. Meantime remove the old 'got_budget' parameter. Cc: Sagi Grimberg <sagi@grimberg.me> Cc: Baolin Wang <baolin.wang7@gmail.com> Cc: Christoph Hellwig <hch@infradead.org> Signed-off-by: Ming Lei <ming.lei@redhat.com> --- block/blk-mq-sched.c | 8 ++++---- block/blk-mq.c | 13 +++++++++---- block/blk-mq.h | 3 ++- 3 files changed, 15 insertions(+), 9 deletions(-)