@@ -21,6 +21,7 @@
#include <linux/nvme_ioctl.h>
#include <linux/pm_qos.h>
#include <asm/unaligned.h>
+#include <linux/rcupdate_wait.h>
#include "nvme.h"
#include "fabrics.h"
@@ -4916,14 +4917,6 @@ static void nvme_start_ns_queue(struct nvme_ns *ns)
blk_mq_unquiesce_queue(ns->queue);
}
-static void nvme_stop_ns_queue(struct nvme_ns *ns)
-{
- if (!test_and_set_bit(NVME_NS_STOPPED, &ns->flags))
- blk_mq_quiesce_queue(ns->queue);
- else
- blk_mq_wait_quiesce_done(ns->queue);
-}
-
static void nvme_stop_ns_queue_nosync(struct nvme_ns *ns)
{
if (!test_and_set_bit(NVME_NS_STOPPED, &ns->flags))
@@ -4933,9 +4926,34 @@ static void nvme_stop_ns_queue_nosync(struct nvme_ns *ns)
static void nvme_stop_blocking_queues(struct nvme_ctrl *ctrl)
{
struct nvme_ns *ns;
+ int i, count;
+ struct rcu_synchronize *rcu;
- list_for_each_entry(ns, &ctrl->namespaces, list)
- nvme_stop_ns_queue(ns);
+ count = 0;
+ list_for_each_entry(ns, &ctrl->namespaces, list) {
+ nvme_stop_ns_queue_nosync(ns);
+ count++;
+ }
+
+ rcu = kvmalloc(count * sizeof(*rcu), GFP_KERNEL);
+ if (rcu) {
+ i = 0;
+ list_for_each_entry(ns, &ctrl->namespaces, list) {
+ init_rcu_head(&rcu[i].head);
+ init_completion(&rcu[i].completion);
+ call_srcu(ns->queue->srcu, &rcu[i].head, wakeme_after_rcu);
+ i++;
+ }
+
+ for (i = 0; i < count; i++) {
+ wait_for_completion(&rcu[i].completion);
+ destroy_rcu_head(&rcu[i].head);
+ }
+ kvfree(rcu);
+ } else {
+ list_for_each_entry(ns, &ctrl->namespaces, list)
+ blk_mq_wait_quiesce_done(ns->queue);
+ }
}
static void nvme_stop_nonblocking_queues(struct nvme_ctrl *ctrl)
When work with nvme multipathing, if one path failed, the requests will be canceled and retry on another path. Before cancel the requests, nvme_stop_queues quiesce queues for all namespaces, now quiesce one by one, if there is a large set of namespaces, it will take long time. Because every synchronize_srcu may need more than 10 milliseconds, the total waiting time will be long. Quiesce all queues in parallel and start syncing for srcus of all queues, and then wait for all srcus to complete synchronization. The quiesce time and I/O fail over time can be largely reduced. Signed-off-by: Chao Leng <lengchao@huawei.com> --- drivers/nvme/host/core.c | 38 ++++++++++++++++++++++++++++---------- 1 file changed, 28 insertions(+), 10 deletions(-)