From patchwork Mon Oct 23 13:17:55 2017 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Yishai Hadas X-Patchwork-Id: 10022479 Return-Path: Received: from mail.wl.linuxfoundation.org (pdx-wl-mail.web.codeaurora.org [172.30.200.125]) by pdx-korg-patchwork.web.codeaurora.org (Postfix) with ESMTP id 62A8060381 for ; Mon, 23 Oct 2017 13:18:36 +0000 (UTC) Received: from mail.wl.linuxfoundation.org (localhost [127.0.0.1]) by mail.wl.linuxfoundation.org (Postfix) with ESMTP id 5D174204FE for ; Mon, 23 Oct 2017 13:18:36 +0000 (UTC) Received: by mail.wl.linuxfoundation.org (Postfix, from userid 486) id 5213028898; Mon, 23 Oct 2017 13:18:36 +0000 (UTC) X-Spam-Checker-Version: SpamAssassin 3.3.1 (2010-03-16) on pdx-wl-mail.web.codeaurora.org X-Spam-Level: X-Spam-Status: No, score=-6.9 required=2.0 tests=BAYES_00, RCVD_IN_DNSWL_HI, UNPARSEABLE_RELAY autolearn=ham version=3.3.1 Received: from vger.kernel.org (vger.kernel.org [209.132.180.67]) by mail.wl.linuxfoundation.org (Postfix) with ESMTP id CC00A204FE for ; Mon, 23 Oct 2017 13:18:35 +0000 (UTC) Received: (majordomo@vger.kernel.org) by vger.kernel.org via listexpand id S1751260AbdJWNSd (ORCPT ); Mon, 23 Oct 2017 09:18:33 -0400 Received: from mail-il-dmz.mellanox.com ([193.47.165.129]:51866 "EHLO mellanox.co.il" rhost-flags-OK-OK-OK-FAIL) by vger.kernel.org with ESMTP id S1751332AbdJWNS3 (ORCPT ); Mon, 23 Oct 2017 09:18:29 -0400 Received: from Internal Mail-Server by MTLPINE1 (envelope-from yishaih@mellanox.com) with ESMTPS (AES256-SHA encrypted); 23 Oct 2017 15:18:23 +0200 Received: from vnc17.mtl.labs.mlnx (vnc17.mtl.labs.mlnx [10.7.2.17]) by labmailer.mlnx (8.13.8/8.13.8) with ESMTP id v9NDIMI9027873; Mon, 23 Oct 2017 16:18:22 +0300 Received: from vnc17.mtl.labs.mlnx (vnc17.mtl.labs.mlnx [127.0.0.1]) by vnc17.mtl.labs.mlnx (8.13.8/8.13.8) with ESMTP id v9NDIM5R004630; Mon, 23 Oct 2017 16:18:22 +0300 Received: (from yishaih@localhost) by vnc17.mtl.labs.mlnx (8.13.8/8.13.8/Submit) id v9NDIMOh004629; Mon, 23 Oct 2017 16:18:22 +0300 From: Yishai Hadas To: dledford@redhat.com Cc: linux-rdma@vger.kernel.org, yishaih@mellanox.com, majd@mellanox.com, artemyko@mellanox.com Subject: [PATCH rdma-core 04/10] mlx5: Add support to tag matching SRQ type Date: Mon, 23 Oct 2017 16:17:55 +0300 Message-Id: <1508764681-4531-5-git-send-email-yishaih@mellanox.com> X-Mailer: git-send-email 1.8.2.3 In-Reply-To: <1508764681-4531-1-git-send-email-yishaih@mellanox.com> References: <1508764681-4531-1-git-send-email-yishaih@mellanox.com> Sender: linux-rdma-owner@vger.kernel.org Precedence: bulk List-ID: X-Mailing-List: linux-rdma@vger.kernel.org X-Virus-Scanned: ClamAV using ClamSMTP From: Artemy Kovalyov Create command QP for a tag-matching SRQ. Command QP used for inserting/removing entries from the tag matching list. This command QP is hidden from the users in the mlx5_srq structure. New verb ibv_post_srq_ops() will be added in next patch to use it. Signed-off-by: Artemy Kovalyov Reviewed-by: Yishai Hadas --- providers/mlx5/mlx5.h | 1 + providers/mlx5/verbs.c | 140 +++++++++++++++++++++++++++++++++++++++++++------ 2 files changed, 124 insertions(+), 17 deletions(-) diff --git a/providers/mlx5/mlx5.h b/providers/mlx5/mlx5.h index 46fce28..d16b5f5 100644 --- a/providers/mlx5/mlx5.h +++ b/providers/mlx5/mlx5.h @@ -360,6 +360,7 @@ struct mlx5_srq { __be32 *db; uint16_t counter; int wq_sig; + struct ibv_qp *cmd_qp; }; struct wr_list { diff --git a/providers/mlx5/verbs.c b/providers/mlx5/verbs.c index 2315a0d..a26b631 100644 --- a/providers/mlx5/verbs.c +++ b/providers/mlx5/verbs.c @@ -739,6 +739,13 @@ int mlx5_destroy_srq(struct ibv_srq *srq) struct mlx5_srq *msrq = to_msrq(srq); struct mlx5_context *ctx = to_mctx(srq->context); + if (msrq->cmd_qp) { + ret = mlx5_destroy_qp(msrq->cmd_qp); + if (ret) + return ret; + msrq->cmd_qp = NULL; + } + ret = ibv_cmd_destroy_srq(srq); if (ret) return ret; @@ -1822,9 +1829,94 @@ int mlx5_close_xrcd(struct ibv_xrcd *ib_xrcd) return ret; } -static struct ibv_srq * -mlx5_create_xrc_srq(struct ibv_context *context, - struct ibv_srq_init_attr_ex *attr) +static struct ibv_qp * +create_cmd_qp(struct ibv_context *context, + struct ibv_srq_init_attr_ex *srq_attr, + struct ibv_srq *srq) +{ + struct ibv_qp_init_attr_ex init_attr = {}; + FILE *fp = to_mctx(context)->dbg_fp; + struct ibv_port_attr port_attr; + struct ibv_modify_qp qcmd = {}; + struct ibv_qp_attr attr = {}; + struct ibv_query_port pcmd; + struct ibv_qp *qp; + int attr_mask; + int port = 1; + int ret; + + ret = ibv_cmd_query_port(context, port, &port_attr, + &pcmd, sizeof(pcmd)); + if (ret) { + mlx5_dbg(fp, MLX5_DBG_QP, "ret %d\n", ret); + return NULL; + } + + init_attr.qp_type = IBV_QPT_RC; + init_attr.srq = srq; + /* Command QP will be used to pass MLX5_OPCODE_TAG_MATCHING messages + * to add/remove tag matching list entries. + * WQ size is based on max_ops parameter holding max number of + * outstanding list operations. + */ + init_attr.cap.max_send_wr = srq_attr->tm_cap.max_ops; + /* Tag matching list entry will point to a single sge buffer */ + init_attr.cap.max_send_sge = 1; + init_attr.comp_mask = IBV_QP_INIT_ATTR_PD; + init_attr.pd = srq_attr->pd; + init_attr.send_cq = srq_attr->cq; + init_attr.recv_cq = srq_attr->cq; + + qp = create_qp(context, &init_attr); + if (!qp) + return NULL; + + attr.qp_state = IBV_QPS_INIT; + attr.port_num = port; + attr_mask = IBV_QP_STATE | IBV_QP_PKEY_INDEX + | IBV_QP_PORT | IBV_QP_ACCESS_FLAGS; + + ret = ibv_cmd_modify_qp(qp, &attr, attr_mask, &qcmd, sizeof(qcmd)); + if (ret) { + mlx5_dbg(fp, MLX5_DBG_QP, "ret %d\n", ret); + goto err; + } + + attr.qp_state = IBV_QPS_RTR; + attr.path_mtu = IBV_MTU_256; + attr.dest_qp_num = qp->qp_num; /* Loopback */ + attr.ah_attr.dlid = port_attr.lid; + attr.ah_attr.port_num = port; + attr_mask = IBV_QP_STATE | IBV_QP_AV | IBV_QP_PATH_MTU + | IBV_QP_DEST_QPN | IBV_QP_RQ_PSN + | IBV_QP_MAX_DEST_RD_ATOMIC | IBV_QP_MIN_RNR_TIMER; + + ret = ibv_cmd_modify_qp(qp, &attr, attr_mask, &qcmd, sizeof(qcmd)); + if (ret) { + mlx5_dbg(fp, MLX5_DBG_QP, "ret %d\n", ret); + goto err; + } + + attr.qp_state = IBV_QPS_RTS; + attr_mask = IBV_QP_STATE | IBV_QP_TIMEOUT | IBV_QP_RETRY_CNT + | IBV_QP_RNR_RETRY | IBV_QP_SQ_PSN + | IBV_QP_MAX_QP_RD_ATOMIC; + + ret = ibv_cmd_modify_qp(qp, &attr, attr_mask, &qcmd, sizeof(qcmd)); + if (ret) { + mlx5_dbg(fp, MLX5_DBG_QP, "ret %d\n", ret); + goto err; + } + + return qp; + +err: + mlx5_destroy_qp(qp); + return NULL; +} + +struct ibv_srq *mlx5_create_srq_ex(struct ibv_context *context, + struct ibv_srq_init_attr_ex *attr) { int err; struct mlx5_create_srq_ex cmd; @@ -1836,6 +1928,24 @@ mlx5_create_xrc_srq(struct ibv_context *context, int uidx; FILE *fp = ctx->dbg_fp; + if (!(attr->comp_mask & IBV_SRQ_INIT_ATTR_TYPE) || + (attr->srq_type == IBV_SRQT_BASIC)) + return mlx5_create_srq(attr->pd, + (struct ibv_srq_init_attr *)attr); + + if (attr->srq_type != IBV_SRQT_XRC && + attr->srq_type != IBV_SRQT_TM) { + errno = EINVAL; + return NULL; + } + + /* An extended CQ is required to read TM information from */ + if (attr->srq_type == IBV_SRQT_TM && + !(attr->cq && (to_mcq(attr->cq)->flags & MLX5_CQ_FLAGS_EXTENDED))) { + errno = EINVAL; + return NULL; + } + msrq = calloc(1, sizeof(*msrq)); if (!msrq) return NULL; @@ -1914,10 +2024,16 @@ mlx5_create_xrc_srq(struct ibv_context *context, if (err) goto err_free_uidx; + if (attr->srq_type == IBV_SRQT_TM) { + msrq->cmd_qp = create_cmd_qp(context, attr, ibsrq); + if (!msrq->cmd_qp) + goto err_destroy; + } + if (!ctx->cqe_version) { err = mlx5_store_srq(to_mctx(context), resp.srqn, msrq); if (err) - goto err_destroy; + goto err_free_cmd; pthread_mutex_unlock(&ctx->srq_table_mutex); } @@ -1928,6 +2044,9 @@ mlx5_create_xrc_srq(struct ibv_context *context, return ibsrq; +err_free_cmd: + if (msrq->cmd_qp) + mlx5_destroy_qp(msrq->cmd_qp); err_destroy: ibv_cmd_destroy_srq(ibsrq); @@ -1950,19 +2069,6 @@ err: return NULL; } -struct ibv_srq *mlx5_create_srq_ex(struct ibv_context *context, - struct ibv_srq_init_attr_ex *attr) -{ - if (!(attr->comp_mask & IBV_SRQ_INIT_ATTR_TYPE) || - (attr->srq_type == IBV_SRQT_BASIC)) - return mlx5_create_srq(attr->pd, - (struct ibv_srq_init_attr *)attr); - else if (attr->srq_type == IBV_SRQT_XRC) - return mlx5_create_xrc_srq(context, attr); - - return NULL; -} - int mlx5_query_device_ex(struct ibv_context *context, const struct ibv_query_device_ex_input *input, struct ibv_device_attr_ex *attr,