@@ -1235,6 +1235,7 @@ bool blk_mq_dispatch_rq_list(struct request_queue *q, struct list_head *list,
list_del_init(&rq->queuelist);
bd.rq = rq;
+ bd.budget_token = budget_token;
/*
* Flag last if we have no more requests, or if we have more
@@ -1778,14 +1779,11 @@ static void blk_mq_bio_to_request(struct request *rq, struct bio *bio,
}
static blk_status_t __blk_mq_issue_directly(struct blk_mq_hw_ctx *hctx,
- struct request *rq,
- blk_qc_t *cookie, bool last)
+ struct blk_mq_queue_data *bd,
+ blk_qc_t *cookie)
{
+ struct request *rq = bd->rq;
struct request_queue *q = rq->q;
- struct blk_mq_queue_data bd = {
- .rq = rq,
- .last = last,
- };
blk_qc_t new_cookie;
blk_status_t ret;
@@ -1796,7 +1794,7 @@ static blk_status_t __blk_mq_issue_directly(struct blk_mq_hw_ctx *hctx,
* Any other error (busy), just add it to our list as we
* previously would have done.
*/
- ret = q->mq_ops->queue_rq(hctx, &bd);
+ ret = q->mq_ops->queue_rq(hctx, bd);
switch (ret) {
case BLK_STS_OK:
blk_mq_update_dispatch_busy(hctx, false);
@@ -1823,7 +1821,10 @@ static blk_status_t __blk_mq_try_issue_directly(struct blk_mq_hw_ctx *hctx,
{
struct request_queue *q = rq->q;
bool run_queue = true;
- int budget_token;
+ struct blk_mq_queue_data bd = {
+ .rq = rq,
+ .last = last,
+ };
/*
* RCU or SRCU read lock is needed before checking quiesced flag.
@@ -1841,16 +1842,16 @@ static blk_status_t __blk_mq_try_issue_directly(struct blk_mq_hw_ctx *hctx,
if (q->elevator && !bypass_insert)
goto insert;
- budget_token = blk_mq_get_dispatch_budget(hctx);
- if (budget_token < 0)
+ bd.budget_token = blk_mq_get_dispatch_budget(hctx);
+ if (bd.budget_token < 0)
goto insert;
if (!blk_mq_get_driver_tag(rq)) {
- blk_mq_put_dispatch_budget(hctx, budget_token);
+ blk_mq_put_dispatch_budget(hctx, bd.budget_token);
goto insert;
}
- return __blk_mq_issue_directly(hctx, rq, cookie, last);
+ return __blk_mq_issue_directly(hctx, &bd, cookie);
insert:
if (bypass_insert)
return BLK_STS_RESOURCE;
@@ -259,6 +259,7 @@ struct blk_mq_tag_set {
*/
struct blk_mq_queue_data {
struct request *rq;
+ int budget_token;
bool last;
};
Pass the budget token to driver via blk_mq_queue_dada before calling .queue_rq(), and prepare for tracing SCSI's device_busy via sbitmap_queue. Cc: Omar Sandoval <osandov@fb.com> Cc: Sathya Prakash <sathya.prakash@broadcom.com> Cc: Chaitra P B <chaitra.basappa@broadcom.com> Cc: Suganath Prabu Subramani <suganath-prabu.subramani@broadcom.com> Cc: Kashyap Desai <kashyap.desai@broadcom.com> Cc: Sumit Saxena <sumit.saxena@broadcom.com> Cc: Shivasharan S <shivasharan.srikanteshwara@broadcom.com> Cc: Ewan D. Milne <emilne@redhat.com> Cc: Hannes Reinecke <hare@suse.de> Cc: Bart Van Assche <bart.vanassche@wdc.com> Signed-off-by: Ming Lei <ming.lei@redhat.com> --- block/blk-mq.c | 25 +++++++++++++------------ include/linux/blk-mq.h | 1 + 2 files changed, 14 insertions(+), 12 deletions(-)