From patchwork Sun Jun 18 15:21:56 2017 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Sagi Grimberg X-Patchwork-Id: 9794891 Return-Path: Received: from mail.wl.linuxfoundation.org (pdx-wl-mail.web.codeaurora.org [172.30.200.125]) by pdx-korg-patchwork.web.codeaurora.org (Postfix) with ESMTP id 3D75A601C8 for ; Sun, 18 Jun 2017 15:22:42 +0000 (UTC) Received: from mail.wl.linuxfoundation.org (localhost [127.0.0.1]) by mail.wl.linuxfoundation.org (Postfix) with ESMTP id 2FB60283AF for ; Sun, 18 Jun 2017 15:22:42 +0000 (UTC) Received: by mail.wl.linuxfoundation.org (Postfix, from userid 486) id 246D9283C0; Sun, 18 Jun 2017 15:22:42 +0000 (UTC) X-Spam-Checker-Version: SpamAssassin 3.3.1 (2010-03-16) on pdx-wl-mail.web.codeaurora.org X-Spam-Level: X-Spam-Status: No, score=-5.1 required=2.0 tests=BAYES_00,DKIM_SIGNED, RCVD_IN_DNSWL_HI, T_DKIM_INVALID, URIBL_BLACK autolearn=ham version=3.3.1 Received: from vger.kernel.org (vger.kernel.org [209.132.180.67]) by mail.wl.linuxfoundation.org (Postfix) with ESMTP id 9AD70283AF for ; Sun, 18 Jun 2017 15:22:41 +0000 (UTC) Received: (majordomo@vger.kernel.org) by vger.kernel.org via listexpand id S1752925AbdFRPWl (ORCPT ); Sun, 18 Jun 2017 11:22:41 -0400 Received: from merlin.infradead.org ([205.233.59.134]:52226 "EHLO merlin.infradead.org" rhost-flags-OK-OK-OK-OK) by vger.kernel.org with ESMTP id S1753176AbdFRPWk (ORCPT ); Sun, 18 Jun 2017 11:22:40 -0400 DKIM-Signature: v=1; a=rsa-sha256; q=dns/txt; c=relaxed/relaxed; d=infradead.org; s=merlin.20170209; h=References:In-Reply-To:Message-Id:Date: Subject:Cc:To:From:Sender:Reply-To:MIME-Version:Content-Type: Content-Transfer-Encoding:Content-ID:Content-Description:Resent-Date: Resent-From:Resent-Sender:Resent-To:Resent-Cc:Resent-Message-ID:List-Id: List-Help:List-Unsubscribe:List-Subscribe:List-Post:List-Owner:List-Archive; bh=v/Zd22UIIMknSGMWGATggYMe8Lmp39MBzHosMqnLC5Q=; b=p5hvldH5OpP+mtXc+xEk/Q/U/ eLXEi75wap2NzkgqhkHzDf6/8bc1LFdAkbF5vyvbZsxNz9FPZjbns+KeFLQRXbsICYa7uK+6R1m+C PcB/r0+5gz8pn8xfaPdMm1UOpEcoBNBvLjB+KY0YIJvc93OyaeowT0v3rng7CjVFJ2Me+4XTRNMia 1/rVijOxxYWMmrP7PFR4ArcFi8RQ5phxSWUwetcTjrJTGmJIgYMk14qjZ6wPpBU1di0LJ0tm7kR99 +dKaAb6Pw8nvg6bowrIVBxgWHxHs1qZWEuojYIdoOY0/arHf434G8gPtNXN2GHM30HrkJf9qRUJH6 0wunuOetQ==; Received: from bzq-82-81-101-184.red.bezeqint.net ([82.81.101.184] helo=bombadil.infradead.org) by merlin.infradead.org with esmtpsa (Exim 4.87 #1 (Red Hat Linux)) id 1dMc29-0006WN-Bh; Sun, 18 Jun 2017 15:22:37 +0000 From: Sagi Grimberg To: linux-nvme@lists.infradead.org Cc: Christoph Hellwig , Keith Busch , linux-block@vger.kernel.org Subject: [PATCH rfc 22/30] nvme-rdma: Split create_ctrl to transport specific and generic parts Date: Sun, 18 Jun 2017 18:21:56 +0300 Message-Id: <1497799324-19598-23-git-send-email-sagi@grimberg.me> X-Mailer: git-send-email 2.7.4 In-Reply-To: <1497799324-19598-1-git-send-email-sagi@grimberg.me> References: <1497799324-19598-1-git-send-email-sagi@grimberg.me> Sender: linux-block-owner@vger.kernel.org Precedence: bulk List-ID: X-Mailing-List: linux-block@vger.kernel.org X-Virus-Scanned: ClamAV using ClamSMTP Most of create controller are simply setup admin queue and tags, submit a set of admin commands, allocate io tags, and io queues. We can make that generic, next we will move stuff into the core. Signed-off-by: Sagi Grimberg --- drivers/nvme/host/rdma.c | 160 +++++++++++++++++++++++++++-------------------- 1 file changed, 91 insertions(+), 69 deletions(-) diff --git a/drivers/nvme/host/rdma.c b/drivers/nvme/host/rdma.c index e656b9b17d67..0036ddcbc138 100644 --- a/drivers/nvme/host/rdma.c +++ b/drivers/nvme/host/rdma.c @@ -1839,6 +1839,41 @@ static void nvme_rdma_reset_ctrl_work(struct work_struct *work) nvme_put_ctrl(ctrl); } +static int nvme_rdma_verify_ctrl(struct nvme_ctrl *ctrl) +{ + struct nvmf_ctrl_options *opts = ctrl->opts; + + /* sanity check icdoff */ + if (ctrl->icdoff) { + dev_err(ctrl->device, "icdoff is not supported!\n"); + return -EINVAL; + } + + /* sanity check keyed sgls */ + if (!(ctrl->sgls & (1 << 20))) { + dev_err(ctrl->device, "Mandatory keyed sgls are not support\n"); + return -EINVAL; + } + + if (opts->queue_size > ctrl->maxcmd) { + /* warn if maxcmd is lower than queue_size */ + dev_warn(ctrl->device, + "queue_size %zu > ctrl maxcmd %u, clamping down\n", + opts->queue_size, ctrl->maxcmd); + opts->queue_size = ctrl->maxcmd; + } + + if (opts->queue_size > ctrl->sqsize + 1) { + /* warn if sqsize is lower than queue_size */ + dev_warn(ctrl->device, + "queue_size %zu > ctrl sqsize %u, clamping down\n", + opts->queue_size, ctrl->sqsize + 1); + opts->queue_size = ctrl->sqsize + 1; + } + + return 0; +} + static const struct nvme_ctrl_ops nvme_rdma_ctrl_ops = { .name = "rdma", .module = THIS_MODULE, @@ -1853,12 +1888,62 @@ static const struct nvme_ctrl_ops nvme_rdma_ctrl_ops = { .get_address = nvmf_get_address, }; +static int nvme_rdma_probe_ctrl(struct nvme_ctrl *ctrl, struct device *dev, + const struct nvme_ctrl_ops *ops, unsigned long quirks, + unsigned int nr_io_queues, size_t queue_size, int kato) +{ + bool changed; + int ret; + + INIT_WORK(&ctrl->delete_work, nvme_rdma_del_ctrl_work); + INIT_WORK(&ctrl->reset_work, nvme_rdma_reset_ctrl_work); + + ctrl->max_queues = nr_io_queues + 1; /* +1 for admin queue */ + ctrl->sqsize = queue_size - 1; /* 0's based */ + ctrl->kato = kato; + + ret = nvme_init_ctrl(ctrl, dev, ops, quirks); + if (ret) + return ret; + + ret = nvme_rdma_configure_admin_queue(ctrl, true); + if (ret) + goto out_uninit_ctrl; + + ret = nvme_rdma_verify_ctrl(ctrl); + if (ret) + goto out_remove_admin_queue; + + if (ctrl->max_queues > 1) { + ret = nvme_rdma_configure_io_queues(ctrl, true); + if (ret) + goto out_remove_admin_queue; + } + + changed = nvme_change_ctrl_state(ctrl, NVME_CTRL_LIVE); + WARN_ON_ONCE(!changed); + + kref_get(&ctrl->kref); + + if (ctrl->queue_count > 1) { + nvme_queue_scan(ctrl); + nvme_queue_async_events(ctrl); + } + + return 0; + +out_remove_admin_queue: + nvme_rdma_destroy_admin_queue(ctrl, true); +out_uninit_ctrl: + nvme_uninit_ctrl(ctrl); + return ret; +} + static struct nvme_ctrl *nvme_rdma_create_ctrl(struct device *dev, struct nvmf_ctrl_options *opts) { struct nvme_rdma_ctrl *ctrl; int ret; - bool changed; char *port; ctrl = kzalloc(sizeof(*ctrl), GFP_KERNEL); @@ -1866,6 +1951,7 @@ static struct nvme_ctrl *nvme_rdma_create_ctrl(struct device *dev, return ERR_PTR(-ENOMEM); INIT_LIST_HEAD(&ctrl->list); + ctrl->ctrl.opts = opts; if (opts->mask & NVMF_OPT_TRSVCID) port = opts->trsvcid; @@ -1889,97 +1975,33 @@ static struct nvme_ctrl *nvme_rdma_create_ctrl(struct device *dev, } } - ctrl->ctrl.opts = opts; - ctrl->ctrl.max_queues = opts->nr_io_queues + 1; - ctrl->ctrl.sqsize = opts->queue_size - 1; - ctrl->ctrl.kato = opts->kato; INIT_DELAYED_WORK(&ctrl->ctrl.reconnect_work, nvme_rdma_reconnect_ctrl_work); INIT_WORK(&ctrl->ctrl.err_work, nvme_rdma_error_recovery_work); INIT_WORK(&ctrl->ctrl.delete_work, nvme_rdma_del_ctrl_work); - INIT_WORK(&ctrl->ctrl.reset_work, nvme_rdma_reset_ctrl_work); - - ret = nvme_init_ctrl(&ctrl->ctrl, dev, &nvme_rdma_ctrl_ops, - 0 /* no quirks, we're perfect! */); - if (ret) - goto out_free_ctrl; ret = -ENOMEM; - ctrl->queues = kcalloc(ctrl->ctrl.max_queues, sizeof(*ctrl->queues), + ctrl->queues = kcalloc(opts->nr_io_queues + 1, sizeof(*ctrl->queues), GFP_KERNEL); if (!ctrl->queues) - goto out_uninit_ctrl; + goto out_free_ctrl; - ret = nvme_rdma_configure_admin_queue(&ctrl->ctrl, true); + ret = nvme_rdma_probe_ctrl(&ctrl->ctrl, dev, &nvme_rdma_ctrl_ops, + 0, opts->nr_io_queues, opts->queue_size, opts->kato); if (ret) goto out_kfree_queues; - /* sanity check icdoff */ - if (ctrl->ctrl.icdoff) { - dev_err(ctrl->ctrl.device, "icdoff is not supported!\n"); - ret = -EINVAL; - goto out_remove_admin_queue; - } - - /* sanity check keyed sgls */ - if (!(ctrl->ctrl.sgls & (1 << 20))) { - dev_err(ctrl->ctrl.device, "Mandatory keyed sgls are not support\n"); - ret = -EINVAL; - goto out_remove_admin_queue; - } - - if (opts->queue_size > ctrl->ctrl.maxcmd) { - /* warn if maxcmd is lower than queue_size */ - dev_warn(ctrl->ctrl.device, - "queue_size %zu > ctrl maxcmd %u, clamping down\n", - opts->queue_size, ctrl->ctrl.maxcmd); - opts->queue_size = ctrl->ctrl.maxcmd; - } - - if (opts->queue_size > ctrl->ctrl.sqsize + 1) { - /* warn if sqsize is lower than queue_size */ - dev_warn(ctrl->ctrl.device, - "queue_size %zu > ctrl sqsize %u, clamping down\n", - opts->queue_size, ctrl->ctrl.sqsize + 1); - opts->queue_size = ctrl->ctrl.sqsize + 1; - } - - if (ctrl->ctrl.max_queues > 1) { - ret = nvme_rdma_configure_io_queues(&ctrl->ctrl, true); - if (ret) - goto out_remove_admin_queue; - } - - changed = nvme_change_ctrl_state(&ctrl->ctrl, NVME_CTRL_LIVE); - WARN_ON_ONCE(!changed); - dev_info(ctrl->ctrl.device, "new ctrl: NQN \"%s\", addr %pISpcs\n", ctrl->ctrl.opts->subsysnqn, &ctrl->addr); - kref_get(&ctrl->ctrl.kref); - mutex_lock(&nvme_rdma_ctrl_mutex); list_add_tail(&ctrl->list, &nvme_rdma_ctrl_list); mutex_unlock(&nvme_rdma_ctrl_mutex); - if (ctrl->ctrl.max_queues > 1) { - nvme_queue_scan(&ctrl->ctrl); - nvme_queue_async_events(&ctrl->ctrl); - } - return &ctrl->ctrl; -out_remove_admin_queue: - nvme_stop_keep_alive(&ctrl->ctrl); - nvme_rdma_destroy_admin_queue(&ctrl->ctrl, true); out_kfree_queues: kfree(ctrl->queues); -out_uninit_ctrl: - nvme_uninit_ctrl(&ctrl->ctrl); - nvme_put_ctrl(&ctrl->ctrl); - if (ret > 0) - ret = -EIO; - return ERR_PTR(ret); out_free_ctrl: kfree(ctrl); return ERR_PTR(ret);