@@ -103,6 +103,13 @@ static void nvme_put_subsystem(struct nvme_subsystem *subsys);
static void nvme_remove_invalid_namespaces(struct nvme_ctrl *ctrl,
unsigned nsid);
+static bool nvme_fail_request(struct blk_mq_hw_ctx *hctx, struct request *req,
+ void *data, bool reserved)
+{
+ blk_mq_end_request(req, BLK_STS_IOERR);
+ return true;
+}
+
static void nvme_set_queue_dying(struct nvme_ns *ns)
{
/*
@@ -113,8 +120,7 @@ static void nvme_set_queue_dying(struct nvme_ns *ns)
return;
revalidate_disk(ns->disk);
blk_set_queue_dying(ns->queue);
- /* Forcibly unquiesce queues to avoid blocking dispatch */
- blk_mq_unquiesce_queue(ns->queue);
+ blk_mq_queue_tag_busy_iter(ns->queue, nvme_fail_request, NULL);
}
static void nvme_queue_scan(struct nvme_ctrl *ctrl)
@@ -926,13 +926,6 @@ static blk_status_t nvme_queue_rq(struct blk_mq_hw_ctx *hctx,
struct nvme_command cmnd;
blk_status_t ret;
- /*
- * We should not need to do this, but we're still using this to
- * ensure we can drain requests on a dying queue.
- */
- if (unlikely(!test_bit(NVMEQ_ENABLED, &nvmeq->flags)))
- return BLK_STS_IOERR;
-
ret = nvme_setup_cmd(ns, req, &cmnd);
if (ret)
return ret;
@@ -1408,10 +1401,6 @@ static int nvme_suspend_queue(struct nvme_queue *nvmeq)
{
if (!test_and_clear_bit(NVMEQ_ENABLED, &nvmeq->flags))
return 1;
-
- /* ensure that nvme_queue_rq() sees NVMEQ_ENABLED cleared */
- mb();
-
nvmeq->dev->online_queues--;
if (!nvmeq->qid && nvmeq->dev->ctrl.admin_q)
blk_mq_quiesce_queue(nvmeq->dev->ctrl.admin_q);
@@ -1611,15 +1600,30 @@ static const struct blk_mq_ops nvme_mq_ops = {
.poll = nvme_poll,
};
+static bool nvme_fail_queue_request(struct request *req, void *data, bool reserved)
+{
+ struct nvme_iod *iod = blk_mq_rq_to_pdu(req);
+ struct nvme_queue *nvmeq = iod->nvmeq;
+
+ if (test_bit(NVMEQ_ENABLED, &nvmeq->flags))
+ return true;
+ blk_mq_end_request(req, BLK_STS_IOERR);
+ return true;
+}
+
static void nvme_dev_remove_admin(struct nvme_dev *dev)
{
if (dev->ctrl.admin_q && !blk_queue_dying(dev->ctrl.admin_q)) {
/*
* If the controller was reset during removal, it's possible
- * user requests may be waiting on a stopped queue. Start the
- * queue to flush these to completion.
+ * user requests may be waiting on a stopped queue. End all
+ * entered requests after preventing new requests from
+ * entering.
*/
- blk_mq_unquiesce_queue(dev->ctrl.admin_q);
+ blk_set_queue_dying(dev->ctrl.admin_q);
+ blk_mq_tagset_all_iter(&dev->admin_tagset,
+ nvme_fail_queue_request,
+ NULL);
blk_cleanup_queue(dev->ctrl.admin_q);
blk_mq_free_tag_set(&dev->admin_tagset);
}
@@ -2411,6 +2415,11 @@ static void nvme_pci_disable(struct nvme_dev *dev)
}
}
+static void nvme_fail_requests(struct nvme_dev *dev)
+{
+ blk_mq_tagset_all_iter(&dev->tagset, nvme_fail_queue_request, NULL);
+}
+
static void nvme_dev_disable(struct nvme_dev *dev, bool shutdown)
{
int i;
@@ -2454,11 +2463,11 @@ static void nvme_dev_disable(struct nvme_dev *dev, bool shutdown)
/*
* The driver will not be starting up queues again if shutting down so
- * must flush all entered requests to their failed completion to avoid
+ * must end all entered requests to their failed completion to avoid
* deadlocking blk-mq hot-cpu notifier.
*/
if (shutdown)
- nvme_start_queues(&dev->ctrl);
+ nvme_fail_requests(dev);
mutex_unlock(&dev->shutdown_lock);
}
@@ -2601,6 +2610,8 @@ static void nvme_reset_work(struct work_struct *work)
nvme_remove_namespaces(&dev->ctrl);
new_state = NVME_CTRL_ADMIN_ONLY;
} else {
+ /* Fail requests that entered an hctx that no longer exists */
+ nvme_fail_requests(dev);
nvme_start_queues(&dev->ctrl);
nvme_wait_freeze(&dev->ctrl);
/* hit this only when allocate tagset fails */
The nvme driver checked the queue state on every IO so the path could drain requests. The code however declares "We shold not need to do this", so let's not do it. Instead, use blk-mq's tag iterator to terminate entered requests on dying queues so the IO path doesn't have to deal with these conditions. Signed-off-by: Keith Busch <keith.busch@intel.com> --- drivers/nvme/host/core.c | 10 ++++++++-- drivers/nvme/host/pci.c | 43 +++++++++++++++++++++++++++---------------- 2 files changed, 35 insertions(+), 18 deletions(-)