@@ -416,6 +416,29 @@ struct request *blk_mq_alloc_request_hctx(struct request_queue *q,
}
EXPORT_SYMBOL_GPL(blk_mq_alloc_request_hctx);
+static void blk_mq_update_req_complete(struct blk_mq_hw_ctx *hctx)
+{
+ struct sbitmap_queue *sbq;
+ unsigned threshold, depth;
+
+ if (!blk_mq_hctx_stopped(hctx))
+ return;
+
+ sbq = &hctx->tags->bitmap_tags;
+ depth = sbitmap_weight(&sbq->sb);
+ threshold = READ_ONCE(hctx->avg_busy_threshold);
+
+ /*
+ * TODO: replace the 1/8 hardcode window with one
+ * intelligent way, such as exponential backoff
+ */
+ if ((depth < threshold) &&
+ (threshold - depth) >= (threshold >> 3)) {
+ clear_bit(BLK_MQ_S_STOPPED, &hctx->state);
+ blk_mq_run_hw_queue(hctx, true);
+ }
+}
+
void blk_mq_free_request(struct request *rq)
{
struct request_queue *q = rq->q;
@@ -447,6 +470,8 @@ void blk_mq_free_request(struct request *rq)
blk_mq_put_tag(hctx, hctx->sched_tags, ctx, sched_tag);
blk_mq_sched_restart(hctx);
blk_queue_exit(q);
+
+ blk_mq_update_req_complete(hctx);
}
EXPORT_SYMBOL_GPL(blk_mq_free_request);
@@ -986,6 +1011,12 @@ static void blk_mq_update_req_dispatch_busy(struct blk_mq_hw_ctx *hctx)
/* use EWMA to estimate a threshold for detecting congestion */
ewma_add(hctx->avg_busy_threshold, depth, 8, 0);
+
+ if (blk_mq_hctx_stopped(hctx))
+ return;
+
+ if (depth >= hctx->avg_busy_threshold)
+ set_bit(BLK_MQ_S_STOPPED, &hctx->state);
}
bool blk_mq_dispatch_rq_list(struct request_queue *q, struct list_head *list)
@@ -189,7 +189,6 @@ static inline void virtblk_request_done(struct request *req)
static void virtblk_done(struct virtqueue *vq)
{
struct virtio_blk *vblk = vq->vdev->priv;
- bool req_done = false;
int qid = vq->index;
struct virtblk_req *vbr;
unsigned long flags;
@@ -202,15 +201,10 @@ static void virtblk_done(struct virtqueue *vq)
struct request *req = blk_mq_rq_from_pdu(vbr);
blk_mq_complete_request(req);
- req_done = true;
}
if (unlikely(virtqueue_is_broken(vq)))
break;
} while (!virtqueue_enable_cb(vq));
-
- /* In case queue is stopped waiting for more buffers. */
- if (req_done)
- blk_mq_start_stopped_hw_queues(vblk->disk->queue, true);
spin_unlock_irqrestore(&vblk->vqs[qid].lock, flags);
}
@@ -271,7 +265,6 @@ static blk_status_t virtio_queue_rq(struct blk_mq_hw_ctx *hctx,
err = virtblk_add_req(vblk->vqs[qid].vq, vbr, vbr->sg, num);
if (err) {
virtqueue_kick(vblk->vqs[qid].vq);
- blk_mq_stop_hw_queue(hctx);
spin_unlock_irqrestore(&vblk->vqs[qid].lock, flags);
/* Out of mem doesn't actually happen, since we fall back
* to direct descriptors */
@@ -907,7 +907,6 @@ static blk_status_t blkif_queue_rq(struct blk_mq_hw_ctx *hctx,
out_busy:
spin_unlock_irqrestore(&rinfo->ring_lock, flags);
- blk_mq_stop_hw_queue(hctx);
return BLK_STS_RESOURCE;
}
@@ -1213,15 +1212,6 @@ static void xlvbd_release_gendisk(struct blkfront_info *info)
info->gd = NULL;
}
-/* Already hold rinfo->ring_lock. */
-static inline void kick_pending_request_queues_locked(struct blkfront_ring_info *rinfo)
-{
- if (!RING_FULL(&rinfo->ring)) {
- blk_mq_start_stopped_hw_queues(rinfo->dev_info->rq, true);
- blk_mq_kick_requeue_list(rinfo->dev_info->rq);
- }
-}
-
static void kick_pending_request_queues(struct blkfront_ring_info *rinfo)
{
unsigned long flags;
@@ -1659,8 +1649,6 @@ static irqreturn_t blkif_interrupt(int irq, void *dev_id)
} else
rinfo->ring.sring->rsp_event = i + 1;
- kick_pending_request_queues_locked(rinfo);
-
spin_unlock_irqrestore(&rinfo->ring_lock, flags);
return IRQ_HANDLED;
@@ -761,7 +761,6 @@ static blk_status_t dm_mq_queue_rq(struct blk_mq_hw_ctx *hctx,
/* Undo dm_start_request() before requeuing */
rq_end_stats(md, rq);
rq_completed(md, rq_data_dir(rq), false);
- blk_mq_delay_run_hw_queue(hctx, 100/*ms*/);
return BLK_STS_RESOURCE;
}
@@ -1970,10 +1970,6 @@ nvme_fc_start_fcp_op(struct nvme_fc_ctrl *ctrl, struct nvme_fc_queue *queue,
if (ret != -EBUSY)
return BLK_STS_IOERR;
- if (op->rq) {
- blk_mq_stop_hw_queues(op->rq->q);
- blk_mq_delay_queue(queue->hctx, NVMEFC_QUEUE_DELAY);
- }
return BLK_STS_RESOURCE;
}
@@ -1976,9 +1976,6 @@ static blk_status_t scsi_queue_rq(struct blk_mq_hw_ctx *hctx,
case BLK_STS_OK:
break;
case BLK_STS_RESOURCE:
- if (atomic_read(&sdev->device_busy) == 0 &&
- !scsi_device_blocked(sdev))
- blk_mq_delay_run_hw_queue(hctx, SCSI_QUEUE_DELAY);
break;
default:
/*
Now we have removed all usage of start/stop queue except for the case BLK_STS_RESOURCE. This patch implements basic congestion control so that we can stop queue when congestion is detected. After the congestion condition becomes invalid, restart the queue again. The congestion threshold is introduced in last patch, and if queue depth is bigger than this threshold, this patch considers it is a congestion. There are at least two advantage to do congestion control: - avoid to waste CPU to dispatch rq to hardware/driver when it might be busy - avoid to let drivers handle this, so we can cleanup drivers and unexport interfaces of restart/stop queue, which have casued enough trouble already One simple sequential read test(libaio, bs:4k, direct io, queue depth:64, 8 jobs) on virtio-scsi shows that: - CPU utilization decreases ~20% - IOPS increases by ~10% With this congestion control approach in blk-mq framework, we can remove the handling in drivers. Actually the handling in drivers isn't good enough too, such as: - virtio-blk/xen blk_front stops queue if one rq can't be handled, and restart the queue again if one request is completed, this way may cause trouble to dispatch request at batch, and make queue stop/restart too frequent - virtio-scsi won't stop queue, and just delay handling new requests after some delay in this case, this way wastes CPU and may affect both latency and throughput Signed-off-by: Ming Lei <ming.lei@redhat.com> --- block/blk-mq.c | 31 +++++++++++++++++++++++++++++++ drivers/block/virtio_blk.c | 7 ------- drivers/block/xen-blkfront.c | 12 ------------ drivers/md/dm-rq.c | 1 - drivers/nvme/host/fc.c | 4 ---- drivers/scsi/scsi_lib.c | 3 --- 6 files changed, 31 insertions(+), 27 deletions(-)