From patchwork Fri Sep 9 12:36:26 2016 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Christoph Hellwig X-Patchwork-Id: 9323207 Return-Path: Received: from mail.wl.linuxfoundation.org (pdx-wl-mail.web.codeaurora.org [172.30.200.125]) by pdx-korg-patchwork.web.codeaurora.org (Postfix) with ESMTP id DE3966077F for ; Fri, 9 Sep 2016 12:36:47 +0000 (UTC) Received: from mail.wl.linuxfoundation.org (localhost [127.0.0.1]) by mail.wl.linuxfoundation.org (Postfix) with ESMTP id CF63C29E67 for ; Fri, 9 Sep 2016 12:36:47 +0000 (UTC) Received: by mail.wl.linuxfoundation.org (Postfix, from userid 486) id C436229E69; Fri, 9 Sep 2016 12:36:47 +0000 (UTC) X-Spam-Checker-Version: SpamAssassin 3.3.1 (2010-03-16) on pdx-wl-mail.web.codeaurora.org X-Spam-Level: X-Spam-Status: No, score=-6.9 required=2.0 tests=BAYES_00,RCVD_IN_DNSWL_HI autolearn=ham version=3.3.1 Received: from vger.kernel.org (vger.kernel.org [209.132.180.67]) by mail.wl.linuxfoundation.org (Postfix) with ESMTP id 4153F29E67 for ; Fri, 9 Sep 2016 12:36:47 +0000 (UTC) Received: (majordomo@vger.kernel.org) by vger.kernel.org via listexpand id S1753592AbcIIMgq (ORCPT ); Fri, 9 Sep 2016 08:36:46 -0400 Received: from bombadil.infradead.org ([198.137.202.9]:38195 "EHLO bombadil.infradead.org" rhost-flags-OK-OK-OK-OK) by vger.kernel.org with ESMTP id S1752752AbcIIMgp (ORCPT ); Fri, 9 Sep 2016 08:36:45 -0400 Received: from [83.175.99.196] (helo=localhost) by bombadil.infradead.org with esmtpsa (Exim 4.85_2 #1 (Red Hat Linux)) id 1biL2y-0000Bg-Kl; Fri, 09 Sep 2016 12:36:45 +0000 From: Christoph Hellwig To: linux-rdma@vger.kernel.org Cc: sagi@grimberg.me Subject: [PATCH 5/6] IB/iser: use implicit CQ allocation Date: Fri, 9 Sep 2016 14:36:26 +0200 Message-Id: <1473424587-13818-6-git-send-email-hch@lst.de> X-Mailer: git-send-email 2.1.4 In-Reply-To: <1473424587-13818-1-git-send-email-hch@lst.de> References: <1473424587-13818-1-git-send-email-hch@lst.de> X-SRS-Rewrite: SMTP reverse-path rewritten from by bombadil.infradead.org. See http://www.infradead.org/rpr.html Sender: linux-rdma-owner@vger.kernel.org Precedence: bulk List-ID: X-Mailing-List: linux-rdma@vger.kernel.org X-Virus-Scanned: ClamAV using ClamSMTP From: Sagi Grimberg Signed-off-by: Sagi Grimberg [hch: ported to the new API] Signed-off-by: Christoph Hellwig --- drivers/infiniband/ulp/iser/iscsi_iser.h | 19 ------- drivers/infiniband/ulp/iser/iser_verbs.c | 90 ++++++-------------------------- 2 files changed, 16 insertions(+), 93 deletions(-) diff --git a/drivers/infiniband/ulp/iser/iscsi_iser.h b/drivers/infiniband/ulp/iser/iscsi_iser.h index 0be6a7c..d3cc83a 100644 --- a/drivers/infiniband/ulp/iser/iscsi_iser.h +++ b/drivers/infiniband/ulp/iser/iscsi_iser.h @@ -318,18 +318,6 @@ struct ib_conn; struct iscsi_iser_task; /** - * struct iser_comp - iSER completion context - * - * @cq: completion queue - * @active_qps: Number of active QPs attached - * to completion context - */ -struct iser_comp { - struct ib_cq *cq; - int active_qps; -}; - -/** * struct iser_device - Memory registration operations * per-device registration schemes * @@ -365,9 +353,6 @@ struct iser_reg_ops { * @event_handler: IB events handle routine * @ig_list: entry in devices list * @refcount: Reference counter, dominated by open iser connections - * @comps_used: Number of completion contexts used, Min between online - * cpus and device max completion vectors - * @comps: Dinamically allocated array of completion handlers * @reg_ops: Registration ops * @remote_inv_sup: Remote invalidate is supported on this device */ @@ -377,8 +362,6 @@ struct iser_device { struct ib_event_handler event_handler; struct list_head ig_list; int refcount; - int comps_used; - struct iser_comp *comps; const struct iser_reg_ops *reg_ops; bool remote_inv_sup; }; @@ -454,7 +437,6 @@ struct iser_fr_pool { * @sig_count: send work request signal count * @rx_wr: receive work request for batch posts * @device: reference to iser device - * @comp: iser completion context * @fr_pool: connection fast registration poool * @pi_support: Indicate device T10-PI support */ @@ -465,7 +447,6 @@ struct ib_conn { u8 sig_count; struct ib_recv_wr rx_wr[ISER_MIN_POSTED_RX]; struct iser_device *device; - struct iser_comp *comp; struct iser_fr_pool fr_pool; bool pi_support; struct ib_cqe reg_cqe; diff --git a/drivers/infiniband/ulp/iser/iser_verbs.c b/drivers/infiniband/ulp/iser/iser_verbs.c index a4b791d..901d56a 100644 --- a/drivers/infiniband/ulp/iser/iser_verbs.c +++ b/drivers/infiniband/ulp/iser/iser_verbs.c @@ -68,62 +68,32 @@ static void iser_event_handler(struct ib_event_handler *handler, static int iser_create_device_ib_res(struct iser_device *device) { struct ib_device *ib_dev = device->ib_device; - int ret, i, max_cqe; + int ret; ret = iser_assign_reg_ops(device); if (ret) return ret; - device->comps_used = min_t(int, num_online_cpus(), - ib_dev->num_comp_vectors); - - device->comps = kcalloc(device->comps_used, sizeof(*device->comps), - GFP_KERNEL); - if (!device->comps) - goto comps_err; - - max_cqe = min(ISER_MAX_CQ_LEN, ib_dev->attrs.max_cqe); - - iser_info("using %d CQs, device %s supports %d vectors max_cqe %d\n", - device->comps_used, ib_dev->name, - ib_dev->num_comp_vectors, max_cqe); - device->pd = ib_alloc_pd(ib_dev, iser_always_reg ? 0 : IB_PD_UNSAFE_GLOBAL_RKEY); - if (IS_ERR(device->pd)) - goto pd_err; - - for (i = 0; i < device->comps_used; i++) { - struct iser_comp *comp = &device->comps[i]; - - comp->cq = ib_alloc_cq(ib_dev, comp, max_cqe, i, - IB_POLL_SOFTIRQ); - if (IS_ERR(comp->cq)) { - comp->cq = NULL; - goto cq_err; - } + if (IS_ERR(device->pd)) { + ret = PTR_ERR(device->pd); + goto out; } INIT_IB_EVENT_HANDLER(&device->event_handler, ib_dev, iser_event_handler); - if (ib_register_event_handler(&device->event_handler)) - goto cq_err; + ret = ib_register_event_handler(&device->event_handler); + if (ret) + goto dealloc_pd; return 0; -cq_err: - for (i = 0; i < device->comps_used; i++) { - struct iser_comp *comp = &device->comps[i]; - - if (comp->cq) - ib_free_cq(comp->cq); - } +dealloc_pd: ib_dealloc_pd(device->pd); -pd_err: - kfree(device->comps); -comps_err: +out: iser_err("failed to allocate an IB resource\n"); - return -1; + return ret; } /** @@ -132,20 +102,8 @@ comps_err: */ static void iser_free_device_ib_res(struct iser_device *device) { - int i; - - for (i = 0; i < device->comps_used; i++) { - struct iser_comp *comp = &device->comps[i]; - - ib_free_cq(comp->cq); - comp->cq = NULL; - } - (void)ib_unregister_event_handler(&device->event_handler); ib_dealloc_pd(device->pd); - - kfree(device->comps); - device->comps = NULL; device->pd = NULL; } @@ -423,7 +381,6 @@ static int iser_create_ib_conn_res(struct ib_conn *ib_conn) struct ib_device *ib_dev; struct ib_qp_init_attr init_attr; int ret = -ENOMEM; - int index, min_index = 0; BUG_ON(ib_conn->device == NULL); @@ -431,26 +388,16 @@ static int iser_create_ib_conn_res(struct ib_conn *ib_conn) ib_dev = device->ib_device; memset(&init_attr, 0, sizeof init_attr); - - mutex_lock(&ig.connlist_mutex); - /* select the CQ with the minimal number of usages */ - for (index = 0; index < device->comps_used; index++) { - if (device->comps[index].active_qps < - device->comps[min_index].active_qps) - min_index = index; - } - ib_conn->comp = &device->comps[min_index]; - ib_conn->comp->active_qps++; - mutex_unlock(&ig.connlist_mutex); - iser_info("cq index %d used for ib_conn %p\n", min_index, ib_conn); - + init_attr.create_flags = IB_QP_CREATE_ASSIGN_CQS; init_attr.event_handler = iser_qp_event_callback; init_attr.qp_context = (void *)ib_conn; - init_attr.send_cq = ib_conn->comp->cq; - init_attr.recv_cq = ib_conn->comp->cq; + init_attr.poll_ctx = IB_POLL_SOFTIRQ; + init_attr.cap.max_recv_wr = ISER_QP_MAX_RECV_DTOS; - init_attr.cap.max_send_sge = 2; init_attr.cap.max_recv_sge = 1; + + init_attr.cap.max_send_sge = 2; + init_attr.sq_sig_type = IB_SIGNAL_REQ_WR; init_attr.qp_type = IB_QPT_RC; if (ib_conn->pi_support) { @@ -483,11 +430,7 @@ static int iser_create_ib_conn_res(struct ib_conn *ib_conn) return ret; out_err: - mutex_lock(&ig.connlist_mutex); - ib_conn->comp->active_qps--; - mutex_unlock(&ig.connlist_mutex); iser_err("unable to alloc mem or create resource, err %d\n", ret); - return ret; } @@ -597,7 +540,6 @@ static void iser_free_ib_conn_res(struct iser_conn *iser_conn, iser_conn, ib_conn->cma_id, ib_conn->qp); if (ib_conn->qp != NULL) { - ib_conn->comp->active_qps--; rdma_destroy_qp(ib_conn->cma_id); ib_conn->qp = NULL; }