@@ -7,6 +7,7 @@
#include <linux/kernel.h>
#include <linux/module.h>
#include <linux/blk-mq.h>
+#include <linux/list_sort.h>
#include <trace/events/block.h>
@@ -80,6 +81,74 @@ void blk_mq_sched_restart(struct blk_mq_hw_ctx *hctx)
blk_mq_run_hw_queue(hctx, true);
}
+/*
+ * We know bfq and deadline apply single scheduler queue instead of multi
+ * queue. However, the two are often used on single queue devices, also
+ * the current @hctx should affect the real device status most of times
+ * because of locality principle.
+ *
+ * So use current hctx->dispatch_busy directly for figuring out batching
+ * dispatch count.
+ */
+static unsigned int blk_mq_sched_get_batching_nr(struct blk_mq_hw_ctx *hctx)
+{
+ if (hctx->dispatch_busy)
+ return 1;
+ return hctx->queue->nr_requests;
+}
+
+static int sched_rq_cmp(void *priv, struct list_head *a, struct list_head *b)
+{
+ struct request *rqa = container_of(a, struct request, queuelist);
+ struct request *rqb = container_of(b, struct request, queuelist);
+
+ return rqa->mq_hctx > rqb->mq_hctx;
+}
+
+static inline bool blk_mq_do_dispatch_rq_lists(struct blk_mq_hw_ctx *hctx,
+ struct list_head *lists, bool multi_hctxs, unsigned count)
+{
+ bool ret;
+
+ if (!count)
+ return false;
+
+ if (likely(!multi_hctxs))
+ return blk_mq_dispatch_rq_list(hctx, lists, count);
+
+ /*
+ * Requests from different hctx may be dequeued from some scheduler,
+ * such as bfq and deadline.
+ *
+ * Sort the requests in the list according to their hctx, dispatch
+ * batching requests from same hctx
+ */
+ list_sort(NULL, lists, sched_rq_cmp);
+
+ ret = false;
+ while (!list_empty(lists)) {
+ LIST_HEAD(list);
+ struct request *new, *rq = list_first_entry(lists,
+ struct request, queuelist);
+ unsigned cnt = 0;
+
+ list_for_each_entry(new, lists, queuelist) {
+ if (new->mq_hctx != rq->mq_hctx)
+ break;
+ cnt++;
+ }
+
+ if (new->mq_hctx == rq->mq_hctx)
+ list_splice_tail_init(lists, &list);
+ else
+ list_cut_before(&list, lists, &new->queuelist);
+
+ ret = blk_mq_dispatch_rq_list(rq->mq_hctx, &list, cnt);
+ }
+
+ return ret;
+}
+
#define BLK_MQ_BUDGET_DELAY 3 /* ms units */
/*
@@ -97,7 +166,15 @@ static int blk_mq_do_dispatch_sched(struct blk_mq_hw_ctx *hctx)
LIST_HEAD(rq_list);
int ret = 0;
struct request *rq;
-
+ int cnt;
+ unsigned int max_dispatch;
+ bool multi_hctxs, run_queue;
+
+ again:
+ /* prepare one batch for dispatch */
+ cnt = 0;
+ max_dispatch = blk_mq_sched_get_batching_nr(hctx);
+ multi_hctxs = run_queue = false;
do {
if (e->type->ops.has_work && !e->type->ops.has_work(hctx))
break;
@@ -120,7 +197,7 @@ static int blk_mq_do_dispatch_sched(struct blk_mq_hw_ctx *hctx)
* no guarantee anyone will kick the queue. Kick it
* ourselves.
*/
- blk_mq_delay_run_hw_queues(q, BLK_MQ_BUDGET_DELAY);
+ run_queue = true;
break;
}
@@ -130,7 +207,20 @@ static int blk_mq_do_dispatch_sched(struct blk_mq_hw_ctx *hctx)
* in blk_mq_dispatch_rq_list().
*/
list_add(&rq->queuelist, &rq_list);
- } while (blk_mq_dispatch_rq_list(rq->mq_hctx, &rq_list, 1));
+ cnt++;
+
+ if (rq->mq_hctx != hctx && !multi_hctxs)
+ multi_hctxs = true;
+ } while (cnt < max_dispatch);
+
+ /* dispatch more if we may do more */
+ if (blk_mq_do_dispatch_rq_lists(hctx, &rq_list, multi_hctxs, cnt) &&
+ !ret)
+ goto again;
+
+ /* in-flight request's completion can rerun queue */
+ if (!cnt && run_queue)
+ blk_mq_delay_run_hw_queues(q, BLK_MQ_BUDGET_DELAY);
return ret;
}
@@ -1294,8 +1294,6 @@ bool blk_mq_dispatch_rq_list(struct blk_mq_hw_ctx *hctx, struct list_head *list,
if (list_empty(list))
return false;
- WARN_ON(!list_is_singular(list) && nr_budgets);
-
/*
* Now process all the entries, sending them to the driver.
*/