@@ -2913,6 +2913,302 @@ void nvme_start_queues(struct nvme_ctrl *ctrl)
}
EXPORT_SYMBOL_GPL(nvme_start_queues);
+static int nvme_alloc_io_queues(struct nvme_ctrl *ctrl)
+{
+ unsigned int nr_io_queues;
+ int ret;
+
+ nr_io_queues = ctrl->ops->nr_hw_queues(ctrl);
+ ret = nvme_set_queue_count(ctrl, &nr_io_queues);
+ if (ret)
+ return ret;
+
+ ctrl->queue_count = nr_io_queues + 1;
+ if (ctrl->queue_count < 2)
+ return 0;
+
+ dev_info(ctrl->device,
+ "creating %d I/O queues.\n", nr_io_queues);
+
+ return ctrl->ops->alloc_io_queues(ctrl);
+}
+
+void nvme_destroy_io_queues(struct nvme_ctrl *ctrl, bool remove)
+{
+ ctrl->ops->stop_io_queues(ctrl);
+ if (remove) {
+ if (ctrl->ops->flags & NVME_F_FABRICS)
+ blk_cleanup_queue(ctrl->connect_q);
+ ctrl->ops->free_tagset(ctrl, false);
+ }
+ ctrl->ops->free_io_queues(ctrl);
+}
+EXPORT_SYMBOL_GPL(nvme_destroy_io_queues);
+
+int nvme_configure_io_queues(struct nvme_ctrl *ctrl, bool new)
+{
+ int ret;
+
+ ret = nvme_alloc_io_queues(ctrl);
+ if (ret)
+ return ret;
+
+ if (new) {
+ ctrl->tagset = ctrl->ops->alloc_tagset(ctrl, false);
+ if (IS_ERR(ctrl->tagset)) {
+ ret = PTR_ERR(ctrl->tagset);
+ goto out_free_io_queues;
+ }
+
+ if (ctrl->ops->flags & NVME_F_FABRICS) {
+ ctrl->connect_q = blk_mq_init_queue(ctrl->tagset);
+ if (IS_ERR(ctrl->connect_q)) {
+ ret = PTR_ERR(ctrl->connect_q);
+ goto out_free_tag_set;
+ }
+ }
+ } else {
+ ret = blk_mq_reinit_tagset(ctrl->tagset);
+ if (ret)
+ goto out_free_io_queues;
+
+ blk_mq_update_nr_hw_queues(ctrl->tagset,
+ ctrl->queue_count - 1);
+ }
+
+ ret = ctrl->ops->start_io_queues(ctrl);
+ if (ret)
+ goto out_cleanup_connect_q;
+
+ return 0;
+
+out_cleanup_connect_q:
+ if (new && (ctrl->ops->flags & NVME_F_FABRICS))
+ blk_cleanup_queue(ctrl->connect_q);
+out_free_tag_set:
+ if (new)
+ ctrl->ops->free_tagset(ctrl, false);
+out_free_io_queues:
+ ctrl->ops->free_io_queues(ctrl);
+ return ret;
+}
+EXPORT_SYMBOL_GPL(nvme_configure_io_queues);
+
+void nvme_destroy_admin_queue(struct nvme_ctrl *ctrl, bool remove)
+{
+ ctrl->ops->stop_admin_queue(ctrl);
+ if (remove) {
+ blk_cleanup_queue(ctrl->admin_q);
+ ctrl->ops->free_tagset(ctrl, true);
+ }
+ ctrl->ops->free_admin_queue(ctrl);
+}
+EXPORT_SYMBOL_GPL(nvme_destroy_admin_queue);
+
+int nvme_configure_admin_queue(struct nvme_ctrl *ctrl, bool new)
+{
+ int error;
+
+ error = ctrl->ops->alloc_admin_queue(ctrl);
+ if (error)
+ return error;
+
+ if (new) {
+ ctrl->admin_tagset = ctrl->ops->alloc_tagset(ctrl, true);
+ if (IS_ERR(ctrl->admin_tagset)) {
+ error = PTR_ERR(ctrl->admin_tagset);
+ goto out_free_queue;
+ }
+
+ ctrl->admin_q = blk_mq_init_queue(ctrl->admin_tagset);
+ if (IS_ERR(ctrl->admin_q)) {
+ error = PTR_ERR(ctrl->admin_q);
+ goto out_free_tagset;
+ }
+ } else {
+ error = blk_mq_reinit_tagset(ctrl->admin_tagset);
+ if (error)
+ goto out_free_queue;
+ }
+
+ error = ctrl->ops->start_admin_queue(ctrl);
+ if (error)
+ goto out_cleanup_queue;
+
+ error = ctrl->ops->reg_read64(ctrl, NVME_REG_CAP, &ctrl->cap);
+ if (error) {
+ dev_err(ctrl->device,
+ "prop_get NVME_REG_CAP failed\n");
+ goto out_cleanup_queue;
+ }
+
+ ctrl->sqsize = min_t(int, NVME_CAP_MQES(ctrl->cap), ctrl->sqsize);
+
+ error = nvme_enable_ctrl(ctrl, ctrl->cap);
+ if (error)
+ goto out_cleanup_queue;
+
+ error = nvme_init_identify(ctrl);
+ if (error)
+ goto out_cleanup_queue;
+
+ return 0;
+
+out_cleanup_queue:
+ if (new)
+ blk_cleanup_queue(ctrl->admin_q);
+out_free_tagset:
+ if (new)
+ ctrl->ops->free_tagset(ctrl, true);
+out_free_queue:
+ ctrl->ops->free_admin_queue(ctrl);
+ return error;
+}
+EXPORT_SYMBOL_GPL(nvme_configure_admin_queue);
+
+static void nvme_teardown_ctrl(struct nvme_ctrl *ctrl, bool shutdown)
+{
+ if (ctrl->queue_count > 1) {
+ nvme_stop_queues(ctrl);
+ blk_mq_tagset_busy_iter(ctrl->tagset,
+ nvme_cancel_request, ctrl);
+ nvme_destroy_io_queues(ctrl, shutdown);
+ }
+
+ if (shutdown)
+ nvme_shutdown_ctrl(ctrl);
+ else
+ nvme_disable_ctrl(ctrl, ctrl->cap);
+
+ blk_mq_quiesce_queue(ctrl->admin_q);
+ blk_mq_tagset_busy_iter(ctrl->admin_tagset,
+ nvme_cancel_request, ctrl);
+ blk_mq_unquiesce_queue(ctrl->admin_q);
+ nvme_destroy_admin_queue(ctrl, shutdown);
+}
+
+static void nvme_remove_ctrl(struct nvme_ctrl *ctrl)
+{
+ nvme_remove_namespaces(ctrl);
+ nvme_teardown_ctrl(ctrl, true);
+ nvme_uninit_ctrl(ctrl);
+ nvme_put_ctrl(ctrl);
+}
+
+static void nvme_del_ctrl_work(struct work_struct *work)
+{
+ struct nvme_ctrl *ctrl = container_of(work,
+ struct nvme_ctrl, delete_work);
+
+ nvme_stop_ctrl(ctrl);
+ nvme_remove_ctrl(ctrl);
+}
+
+int __nvme_del_ctrl(struct nvme_ctrl *ctrl)
+{
+ if (!nvme_change_ctrl_state(ctrl, NVME_CTRL_DELETING))
+ return -EBUSY;
+
+ if (!queue_work(nvme_wq, &ctrl->delete_work))
+ return -EBUSY;
+
+ return 0;
+}
+EXPORT_SYMBOL_GPL(__nvme_del_ctrl);
+
+int nvme_del_ctrl(struct nvme_ctrl *ctrl)
+{
+ int ret = 0;
+
+ /*
+ * Keep a reference until all work is flushed since
+ * __nvme_del_ctrl can free the ctrl mem
+ */
+ if (!kref_get_unless_zero(&ctrl->kref))
+ return -EBUSY;
+
+ ret = __nvme_del_ctrl(ctrl);
+ if (!ret)
+ flush_work(&ctrl->delete_work);
+
+ nvme_put_ctrl(ctrl);
+ return ret;
+}
+EXPORT_SYMBOL_GPL(nvme_del_ctrl);
+
+static void nvme_reset_ctrl_work(struct work_struct *work)
+{
+ struct nvme_ctrl *ctrl =
+ container_of(work, struct nvme_ctrl, reset_work);
+ int ret;
+ bool changed;
+
+ nvme_stop_ctrl(ctrl);
+ nvme_teardown_ctrl(ctrl, false);
+
+ ret = nvme_configure_admin_queue(ctrl, false);
+ if (ret)
+ goto out_fail;
+
+ if (ctrl->queue_count > 1) {
+ ret = nvme_configure_io_queues(ctrl, false);
+ if (ret)
+ goto out_fail;
+ }
+
+ changed = nvme_change_ctrl_state(ctrl, NVME_CTRL_LIVE);
+ WARN_ON_ONCE(!changed);
+
+ nvme_start_ctrl(ctrl);
+
+ return;
+
+out_fail:
+ dev_warn(ctrl->device, "Removing after reset failure\n");
+ nvme_remove_ctrl(ctrl);
+}
+
+int nvme_probe_ctrl(struct nvme_ctrl *ctrl, struct device *dev,
+ const struct nvme_ctrl_ops *ops, unsigned long quirks)
+{
+ bool changed;
+ int ret;
+
+ ret = nvme_init_ctrl(ctrl, dev, ops, quirks);
+ if (ret)
+ return ret;
+
+ INIT_WORK(&ctrl->delete_work, nvme_del_ctrl_work);
+ INIT_WORK(&ctrl->reset_work, nvme_reset_ctrl_work);
+
+ ret = nvme_configure_admin_queue(ctrl, true);
+ if (ret)
+ goto out_uninit_ctrl;
+
+ ret = ctrl->ops->post_configure(ctrl);
+ if (ret)
+ goto out_remove_admin_queue;
+
+ if (ctrl->queue_count > 1) {
+ ret = nvme_configure_io_queues(ctrl, true);
+ if (ret)
+ goto out_remove_admin_queue;
+ }
+
+ changed = nvme_change_ctrl_state(ctrl, NVME_CTRL_LIVE);
+ WARN_ON_ONCE(!changed);
+
+ nvme_start_ctrl(ctrl);
+
+ return 0;
+out_remove_admin_queue:
+ nvme_destroy_admin_queue(ctrl, true);
+out_uninit_ctrl:
+ nvme_uninit_ctrl(ctrl);
+ return ret;
+}
+EXPORT_SYMBOL_GPL(nvme_probe_ctrl);
+
int __init nvme_core_init(void)
{
int result;
@@ -381,6 +381,17 @@ static inline struct nvme_ns *nvme_get_ns_from_dev(struct device *dev)
return dev_to_disk(dev)->private_data;
}
+void nvme_stop_io_queues(struct nvme_ctrl *ctrl);
+void nvme_destroy_io_queues(struct nvme_ctrl *ctrl, bool remove);
+int nvme_configure_io_queues(struct nvme_ctrl *ctrl, bool new);
+void nvme_destroy_admin_queue(struct nvme_ctrl *ctrl, bool remove);
+int nvme_configure_admin_queue(struct nvme_ctrl *ctrl, bool new);
+int __nvme_del_ctrl(struct nvme_ctrl *ctrl);
+int nvme_del_ctrl(struct nvme_ctrl *ctrl);
+int nvme_reset_ctrl(struct nvme_ctrl *ctrl);
+int nvme_probe_ctrl(struct nvme_ctrl *ctrl, struct device *dev,
+ const struct nvme_ctrl_ops *ops, unsigned long quirks);
+
int __init nvme_core_init(void);
void nvme_core_exit(void);
@@ -754,17 +754,6 @@ static struct blk_mq_tag_set *nvme_rdma_alloc_tagset(struct nvme_ctrl *nctrl,
return ERR_PTR(ret);
}
-static void nvme_rdma_destroy_admin_queue(struct nvme_ctrl *ctrl,
- bool remove)
-{
- ctrl->ops->stop_admin_queue(ctrl);
- if (remove) {
- blk_cleanup_queue(ctrl->admin_q);
- ctrl->ops->free_tagset(ctrl, true);
- }
- ctrl->ops->free_admin_queue(ctrl);
-}
-
static int nvme_rdma_alloc_admin_queue(struct nvme_ctrl *nctrl)
{
struct nvme_rdma_ctrl *ctrl = to_rdma_ctrl(nctrl);
@@ -793,133 +782,6 @@ static int nvme_rdma_alloc_admin_queue(struct nvme_ctrl *nctrl)
return ret;
}
-static int nvme_rdma_configure_admin_queue(struct nvme_ctrl *ctrl,
- bool new)
-{
- int error;
-
- error = ctrl->ops->alloc_admin_queue(ctrl);
- if (error)
- return error;
-
- if (new) {
- ctrl->admin_tagset = ctrl->ops->alloc_tagset(ctrl, true);
- if (IS_ERR(ctrl->admin_tagset))
- goto out_free_queue;
-
- ctrl->admin_q = blk_mq_init_queue(ctrl->admin_tagset);
- if (IS_ERR(ctrl->admin_q)) {
- error = PTR_ERR(ctrl->admin_q);
- goto out_free_tagset;
- }
- } else {
- error = blk_mq_reinit_tagset(ctrl->admin_tagset);
- if (error)
- goto out_free_queue;
- }
-
- error = ctrl->ops->start_admin_queue(ctrl);
- if (error)
- goto out_cleanup_queue;
-
- error = ctrl->ops->reg_read64(ctrl, NVME_REG_CAP,
- &ctrl->cap);
- if (error) {
- dev_err(ctrl->device, "prop_get NVME_REG_CAP failed\n");
- goto out_cleanup_queue;
- }
-
- ctrl->sqsize = min_t(int, NVME_CAP_MQES(ctrl->cap), ctrl->sqsize);
-
- error = nvme_enable_ctrl(ctrl, ctrl->cap);
- if (error)
- goto out_cleanup_queue;
-
- error = nvme_init_identify(ctrl);
- if (error)
- goto out_cleanup_queue;
-
- return 0;
-
-out_cleanup_queue:
- if (new)
- blk_cleanup_queue(ctrl->admin_q);
-out_free_tagset:
- if (new)
- ctrl->ops->free_tagset(ctrl, true);
-out_free_queue:
- ctrl->ops->free_admin_queue(ctrl);
- return error;
-}
-
-static void nvme_rdma_destroy_io_queues(struct nvme_ctrl *ctrl,
- bool remove)
-{
- ctrl->ops->stop_io_queues(ctrl);
- if (remove) {
- blk_cleanup_queue(ctrl->connect_q);
- ctrl->ops->free_tagset(ctrl, false);
- }
- ctrl->ops->free_io_queues(ctrl);
-}
-
-static int nvme_rdma_configure_io_queues(struct nvme_ctrl *ctrl, bool new)
-{
- unsigned int nr_io_queues;
- int ret;
-
- nr_io_queues = ctrl->ops->nr_hw_queues(ctrl);
- ret = nvme_set_queue_count(ctrl, &nr_io_queues);
- if (ret)
- return ret;
-
- ctrl->queue_count = nr_io_queues + 1;
- if (ctrl->queue_count < 2)
- return 0;
-
- dev_info(ctrl->device,
- "creating %d I/O queues.\n", nr_io_queues);
-
- ret = ctrl->ops->alloc_io_queues(ctrl);
- if (ret)
- return ret;
-
- if (new) {
- ctrl->tagset = ctrl->ops->alloc_tagset(ctrl, false);
- if (IS_ERR(ctrl->tagset))
- goto out_free_io_queues;
-
- ctrl->connect_q = blk_mq_init_queue(ctrl->tagset);
- if (IS_ERR(ctrl->connect_q)) {
- ret = PTR_ERR(ctrl->connect_q);
- goto out_free_tag_set;
- }
- } else {
- ret = blk_mq_reinit_tagset(ctrl->tagset);
- if (ret)
- goto out_free_io_queues;
-
- blk_mq_update_nr_hw_queues(ctrl->tagset,
- ctrl->queue_count - 1);
- }
-
- ret = nvme_rdma_start_io_queues(ctrl);
- if (ret)
- goto out_cleanup_connect_q;
-
- return 0;
-
-out_cleanup_connect_q:
- if (new)
- blk_cleanup_queue(ctrl->connect_q);
-out_free_tag_set:
- if (new)
- ctrl->ops->free_tagset(ctrl, false);
-out_free_io_queues:
- ctrl->ops->free_io_queues(ctrl);
- return ret;
-}
-
static void nvme_rdma_free_ctrl(struct nvme_ctrl *nctrl)
{
struct nvme_rdma_ctrl *ctrl = to_rdma_ctrl(nctrl);
@@ -967,15 +829,15 @@ static void nvme_rdma_reconnect_ctrl_work(struct work_struct *work)
++ctrl->nr_reconnects;
if (ctrl->queue_count > 1)
- nvme_rdma_destroy_io_queues(ctrl, false);
+ nvme_destroy_io_queues(ctrl, false);
- nvme_rdma_destroy_admin_queue(ctrl, false);
- ret = nvme_rdma_configure_admin_queue(ctrl, false);
+ nvme_destroy_admin_queue(ctrl, false);
+ ret = nvme_configure_admin_queue(ctrl, false);
if (ret)
goto requeue;
if (ctrl->queue_count > 1) {
- ret = nvme_rdma_configure_io_queues(ctrl, false);
+ ret = nvme_configure_io_queues(ctrl, false);
if (ret)
goto requeue;
}
@@ -1764,104 +1626,6 @@ static const struct blk_mq_ops nvme_rdma_admin_mq_ops = {
.timeout = nvme_rdma_timeout,
};
-static void nvme_rdma_shutdown_ctrl(struct nvme_ctrl *ctrl, bool shutdown)
-{
- if (ctrl->queue_count > 1) {
- nvme_stop_queues(ctrl);
- blk_mq_tagset_busy_iter(ctrl->tagset,
- nvme_cancel_request, ctrl);
- nvme_rdma_destroy_io_queues(ctrl, shutdown);
- }
-
- if (shutdown)
- nvme_shutdown_ctrl(ctrl);
- else
- nvme_disable_ctrl(ctrl, ctrl->cap);
-
- blk_mq_quiesce_queue(ctrl->admin_q);
- blk_mq_tagset_busy_iter(ctrl->admin_tagset,
- nvme_cancel_request, ctrl);
- blk_mq_unquiesce_queue(ctrl->admin_q);
- nvme_rdma_destroy_admin_queue(ctrl, shutdown);
-}
-
-static void nvme_rdma_remove_ctrl(struct nvme_ctrl *ctrl)
-{
- nvme_remove_namespaces(ctrl);
- nvme_rdma_shutdown_ctrl(ctrl, true);
- nvme_uninit_ctrl(ctrl);
- nvme_put_ctrl(ctrl);
-}
-
-static void nvme_rdma_del_ctrl_work(struct work_struct *work)
-{
- struct nvme_ctrl *ctrl = container_of(work,
- struct nvme_ctrl, delete_work);
-
- nvme_stop_ctrl(ctrl);
- nvme_rdma_remove_ctrl(ctrl);
-}
-
-static int __nvme_rdma_del_ctrl(struct nvme_ctrl *ctrl)
-{
- if (!nvme_change_ctrl_state(ctrl, NVME_CTRL_DELETING))
- return -EBUSY;
-
- if (!queue_work(nvme_wq, &ctrl->delete_work))
- return -EBUSY;
-
- return 0;
-}
-
-static int nvme_rdma_del_ctrl(struct nvme_ctrl *ctrl)
-{
- int ret = 0;
-
- /*
- * Keep a reference until all work is flushed since
- * __nvme_rdma_del_ctrl can free the ctrl mem
- */
- if (!kref_get_unless_zero(&ctrl->kref))
- return -EBUSY;
- ret = __nvme_rdma_del_ctrl(ctrl);
- if (!ret)
- flush_work(&ctrl->delete_work);
- nvme_put_ctrl(ctrl);
- return ret;
-}
-
-static void nvme_rdma_reset_ctrl_work(struct work_struct *work)
-{
- struct nvme_ctrl *ctrl =
- container_of(work, struct nvme_ctrl, reset_work);
- int ret;
- bool changed;
-
- nvme_stop_ctrl(ctrl);
- nvme_rdma_shutdown_ctrl(ctrl, false);
-
- ret = nvme_rdma_configure_admin_queue(ctrl, false);
- if (ret)
- goto out_fail;
-
- if (ctrl->queue_count > 1) {
- ret = nvme_rdma_configure_io_queues(ctrl, false);
- if (ret)
- goto out_fail;
- }
-
- changed = nvme_change_ctrl_state(ctrl, NVME_CTRL_LIVE);
- WARN_ON_ONCE(!changed);
-
- nvme_start_ctrl(ctrl);
-
- return;
-
-out_fail:
- dev_warn(ctrl->device, "Removing after reset failure\n");
- nvme_rdma_remove_ctrl(ctrl);
-}
-
static int nvme_rdma_post_configure(struct nvme_ctrl *ctrl)
{
/* sanity check icdoff */
@@ -1904,7 +1668,7 @@ static const struct nvme_ctrl_ops nvme_rdma_ctrl_ops = {
.reg_write32 = nvmf_reg_write32,
.free_ctrl = nvme_rdma_free_ctrl,
.submit_async_event = nvme_rdma_submit_async_event,
- .delete_ctrl = nvme_rdma_del_ctrl,
+ .delete_ctrl = nvme_del_ctrl,
.get_address = nvmf_get_address,
.alloc_admin_queue = nvme_rdma_alloc_admin_queue,
.free_admin_queue = nvme_rdma_free_admin_queue,
@@ -1920,46 +1684,6 @@ static const struct nvme_ctrl_ops nvme_rdma_ctrl_ops = {
.nr_hw_queues = nvme_rdma_nr_io_queues,
};
-static int nvme_rdma_probe_ctrl(struct nvme_ctrl *ctrl, struct device *dev,
- const struct nvme_ctrl_ops *ops, unsigned long quirks)
-{
- bool changed;
- int ret;
-
- ret = nvme_init_ctrl(ctrl, dev, ops, quirks);
- if (ret)
- return ret;
-
- INIT_WORK(&ctrl->delete_work, nvme_rdma_del_ctrl_work);
- INIT_WORK(&ctrl->reset_work, nvme_rdma_reset_ctrl_work);
-
- ret = nvme_rdma_configure_admin_queue(ctrl, true);
- if (ret)
- goto out_uninit_ctrl;
-
- ret = ctrl->ops->post_configure(ctrl);
- if (ret)
- goto out_remove_admin_queue;
-
- if (ctrl->queue_count > 1) {
- ret = nvme_rdma_configure_io_queues(ctrl, true);
- if (ret)
- goto out_remove_admin_queue;
- }
-
- changed = nvme_change_ctrl_state(ctrl, NVME_CTRL_LIVE);
- WARN_ON_ONCE(!changed);
-
- nvme_start_ctrl(ctrl);
-
- return 0;
-out_remove_admin_queue:
- nvme_rdma_destroy_admin_queue(ctrl, true);
-out_uninit_ctrl:
- nvme_uninit_ctrl(ctrl);
- return ret;
-}
-
static struct nvme_ctrl *nvme_rdma_create_ctrl(struct device *dev,
struct nvmf_ctrl_options *opts)
{
@@ -2010,7 +1734,7 @@ static struct nvme_ctrl *nvme_rdma_create_ctrl(struct device *dev,
nvme_rdma_reconnect_ctrl_work);
INIT_WORK(&ctrl->ctrl.err_work, nvme_rdma_error_recovery_work);
- ret = nvme_rdma_probe_ctrl(&ctrl->ctrl, dev, &nvme_rdma_ctrl_ops, 0);
+ ret = nvme_probe_ctrl(&ctrl->ctrl, dev, &nvme_rdma_ctrl_ops, 0);
if (!ctrl->queues)
goto out_kfree_queues;
@@ -2059,7 +1783,7 @@ static void nvme_rdma_remove_one(struct ib_device *ib_device, void *client_data)
dev_info(ctrl->ctrl.device,
"Removing ctrl: NQN \"%s\", addr %pISp\n",
ctrl->ctrl.opts->subsysnqn, &ctrl->addr);
- __nvme_rdma_del_ctrl(&ctrl->ctrl);
+ __nvme_del_ctrl(&ctrl->ctrl);
}
mutex_unlock(&nvme_rdma_ctrl_mutex);
handle controller setup (probe), reset and delete in nvme-core and rip it our from nvme-rdma. Signed-off-by: Sagi Grimberg <sagi@grimberg.me> --- drivers/nvme/host/core.c | 296 +++++++++++++++++++++++++++++++++++++++++++++++ drivers/nvme/host/nvme.h | 11 ++ drivers/nvme/host/rdma.c | 290 ++-------------------------------------------- 3 files changed, 314 insertions(+), 283 deletions(-)