From patchwork Sun Jan 27 15:28:57 2019 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Yishai Hadas X-Patchwork-Id: 10782911 Return-Path: Received: from mail.wl.linuxfoundation.org (pdx-wl-mail.web.codeaurora.org [172.30.200.125]) by pdx-korg-patchwork-2.web.codeaurora.org (Postfix) with ESMTP id 7011E159A for ; Sun, 27 Jan 2019 15:29:38 +0000 (UTC) Received: from mail.wl.linuxfoundation.org (localhost [127.0.0.1]) by mail.wl.linuxfoundation.org (Postfix) with ESMTP id 5DC742AA34 for ; Sun, 27 Jan 2019 15:29:38 +0000 (UTC) Received: by mail.wl.linuxfoundation.org (Postfix, from userid 486) id 522822AA36; Sun, 27 Jan 2019 15:29:38 +0000 (UTC) X-Spam-Checker-Version: SpamAssassin 3.3.1 (2010-03-16) on pdx-wl-mail.web.codeaurora.org X-Spam-Level: X-Spam-Status: No, score=-7.9 required=2.0 tests=BAYES_00,MAILING_LIST_MULTI, RCVD_IN_DNSWL_HI,UNPARSEABLE_RELAY autolearn=ham version=3.3.1 Received: from vger.kernel.org (vger.kernel.org [209.132.180.67]) by mail.wl.linuxfoundation.org (Postfix) with ESMTP id B24072AA34 for ; Sun, 27 Jan 2019 15:29:37 +0000 (UTC) Received: (majordomo@vger.kernel.org) by vger.kernel.org via listexpand id S1726397AbfA0P3h (ORCPT ); Sun, 27 Jan 2019 10:29:37 -0500 Received: from mail-il-dmz.mellanox.com ([193.47.165.129]:49267 "EHLO mellanox.co.il" rhost-flags-OK-OK-OK-FAIL) by vger.kernel.org with ESMTP id S1726360AbfA0P3g (ORCPT ); Sun, 27 Jan 2019 10:29:36 -0500 Received: from Internal Mail-Server by MTLPINE1 (envelope-from yishaih@mellanox.com) with ESMTPS (AES256-SHA encrypted); 27 Jan 2019 17:29:33 +0200 Received: from vnc17.mtl.labs.mlnx (vnc17.mtl.labs.mlnx [10.7.2.17]) by labmailer.mlnx (8.13.8/8.13.8) with ESMTP id x0RFTXvd014752; Sun, 27 Jan 2019 17:29:33 +0200 Received: from vnc17.mtl.labs.mlnx (vnc17.mtl.labs.mlnx [127.0.0.1]) by vnc17.mtl.labs.mlnx (8.13.8/8.13.8) with ESMTP id x0RFTWEm011529; Sun, 27 Jan 2019 17:29:32 +0200 Received: (from yishaih@localhost) by vnc17.mtl.labs.mlnx (8.13.8/8.13.8/Submit) id x0RFTWLv011528; Sun, 27 Jan 2019 17:29:32 +0200 From: Yishai Hadas To: linux-rdma@vger.kernel.org Cc: yishaih@mellanox.com, danitg@mellanox.com, yosefe@mellanox.com, maxg@mellanox.com, jgg@mellanox.com, majd@mellanox.com Subject: [PATCH rdma-core 3/3] rping: Add option to create QP independently Date: Sun, 27 Jan 2019 17:28:57 +0200 Message-Id: <1548602938-11370-4-git-send-email-yishaih@mellanox.com> X-Mailer: git-send-email 1.8.2.3 In-Reply-To: <1548602938-11370-1-git-send-email-yishaih@mellanox.com> References: <1548602938-11370-1-git-send-email-yishaih@mellanox.com> Sender: linux-rdma-owner@vger.kernel.org Precedence: bulk List-ID: X-Mailing-List: linux-rdma@vger.kernel.org X-Virus-Scanned: ClamAV using ClamSMTP From: Yossi Itigin The option is available via "-q" flag. Signed-off-by: Yossi Itigin Signed-off-by: Yishai Hadas --- librdmacm/examples/rping.c | 112 ++++++++++++++++++++++++++++++++++++++------- librdmacm/man/rping.1 | 3 ++ 2 files changed, 99 insertions(+), 16 deletions(-) diff --git a/librdmacm/examples/rping.c b/librdmacm/examples/rping.c index 49f5dc5..993c5ad 100644 --- a/librdmacm/examples/rping.c +++ b/librdmacm/examples/rping.c @@ -145,6 +145,7 @@ struct rping_cb { struct sockaddr_storage ssource; __be16 port; /* dst port in NBO */ int verbose; /* verbose logging */ + int self_create_qp; /* Create QP not via cma */ int count; /* ping count */ int size; /* ping data size */ int validate; /* validate ping data */ @@ -190,6 +191,12 @@ static int rping_cma_event_handler(struct rdma_cm_id *cma_id, sem_post(&cb->sem); break; + case RDMA_CM_EVENT_CONNECT_RESPONSE: + DEBUG_LOG("CONNECT_RESPONSE\n"); + cb->state = CONNECTED; + sem_post(&cb->sem); + break; + case RDMA_CM_EVENT_ESTABLISHED: DEBUG_LOG("ESTABLISHED\n"); @@ -347,13 +354,67 @@ error: return ret; } +static void rping_init_conn_param(struct rping_cb *cb, + struct rdma_conn_param *conn_param) +{ + memset(conn_param, 0, sizeof(*conn_param)); + conn_param->responder_resources = 1; + conn_param->initiator_depth = 1; + conn_param->retry_count = 7; + conn_param->rnr_retry_count = 7; + if (cb->self_create_qp) + conn_param->qp_num = cb->qp->qp_num; +} + + +static int rping_self_modify_qp(struct rping_cb *cb, struct rdma_cm_id *id) +{ + struct ibv_qp_attr qp_attr; + int qp_attr_mask, ret; + + qp_attr.qp_state = IBV_QPS_INIT; + ret = rdma_init_qp_attr(id, &qp_attr, &qp_attr_mask); + if (ret) + return ret; + + ret = ibv_modify_qp(cb->qp, &qp_attr, qp_attr_mask); + if (ret) + return ret; + + qp_attr.qp_state = IBV_QPS_RTR; + ret = rdma_init_qp_attr(id, &qp_attr, &qp_attr_mask); + if (ret) + return ret; + + ret = ibv_modify_qp(cb->qp, &qp_attr, qp_attr_mask); + if (ret) + return ret; + + qp_attr.qp_state = IBV_QPS_RTS; + ret = rdma_init_qp_attr(id, &qp_attr, &qp_attr_mask); + if (ret) + return ret; + + return ibv_modify_qp(cb->qp, &qp_attr, qp_attr_mask); +} + static int rping_accept(struct rping_cb *cb) { + struct rdma_conn_param conn_param; int ret; DEBUG_LOG("accepting client connection request\n"); - ret = rdma_accept(cb->child_cm_id, NULL); + if (cb->self_create_qp) { + ret = rping_self_modify_qp(cb, cb->child_cm_id); + if (ret) + return ret; + + rping_init_conn_param(cb, &conn_param); + ret = rdma_accept(cb->child_cm_id, &conn_param); + } else { + ret = rdma_accept(cb->child_cm_id, NULL); + } if (ret) { perror("rdma_accept"); return ret; @@ -480,6 +541,7 @@ static void rping_free_buffers(struct rping_cb *cb) static int rping_create_qp(struct rping_cb *cb) { struct ibv_qp_init_attr init_attr; + struct rdma_cm_id *id; int ret; memset(&init_attr, 0, sizeof(init_attr)); @@ -491,16 +553,21 @@ static int rping_create_qp(struct rping_cb *cb) init_attr.send_cq = cb->cq; init_attr.recv_cq = cb->cq; - if (cb->server) { - ret = rdma_create_qp(cb->child_cm_id, cb->pd, &init_attr); - if (!ret) - cb->qp = cb->child_cm_id->qp; - } else { - ret = rdma_create_qp(cb->cm_id, cb->pd, &init_attr); - if (!ret) - cb->qp = cb->cm_id->qp; + if (cb->self_create_qp) { + cb->qp = ibv_create_qp(cb->pd, &init_attr); + if (!cb->qp) { + perror("ibv_create_qp"); + return -1; + } + return 0; } + id = cb->server ? cb->child_cm_id : cb->cm_id; + ret = rdma_create_qp(id, cb->pd, &init_attr); + if (!ret) + cb->qp = id->qp; + else + perror("rdma_create_qp"); return ret; } @@ -549,7 +616,6 @@ static int rping_setup_qp(struct rping_cb *cb, struct rdma_cm_id *cm_id) ret = rping_create_qp(cb); if (ret) { - perror("rdma_create_qp"); goto err3; } DEBUG_LOG("created qp %p\n", cb->qp); @@ -1011,11 +1077,7 @@ static int rping_connect_client(struct rping_cb *cb) struct rdma_conn_param conn_param; int ret; - memset(&conn_param, 0, sizeof conn_param); - conn_param.responder_resources = 1; - conn_param.initiator_depth = 1; - conn_param.retry_count = 7; - + rping_init_conn_param(cb, &conn_param); ret = rdma_connect(cb->cm_id, &conn_param); if (ret) { perror("rdma_connect"); @@ -1028,6 +1090,20 @@ static int rping_connect_client(struct rping_cb *cb) return -1; } + if (cb->self_create_qp) { + ret = rping_self_modify_qp(cb, cb->cm_id); + if (ret) { + perror("rping_modify_qp"); + return ret; + } + + ret = rdma_establish(cb->cm_id); + if (ret) { + perror("rdma_establish"); + return ret; + } + } + DEBUG_LOG("rmda_connect successful\n"); return 0; } @@ -1160,6 +1236,7 @@ static void usage(const char *name) printf("\t-a addr\t\taddress\n"); printf("\t-p port\t\tport\n"); printf("\t-P\t\tpersistent server mode allowing multiple connections\n"); + printf("\t-q\t\tuse self-created, self-modified QP\n"); } int main(int argc, char *argv[]) @@ -1182,7 +1259,7 @@ int main(int argc, char *argv[]) sem_init(&cb->sem, 0, 0); opterr = 0; - while ((op=getopt(argc, argv, "a:I:Pp:C:S:t:scvVd")) != -1) { + while ((op = getopt(argc, argv, "a:I:Pp:C:S:t:scvVdq")) != -1) { switch (op) { case 'a': ret = get_addr(optarg, (struct sockaddr *) &cb->sin); @@ -1236,6 +1313,9 @@ int main(int argc, char *argv[]) case 'd': debug++; break; + case 'q': + cb->self_create_qp = 1; + break; default: usage("rping"); ret = EINVAL; diff --git a/librdmacm/man/rping.1 b/librdmacm/man/rping.1 index 8d4f157..7ec530e 100644 --- a/librdmacm/man/rping.1 +++ b/librdmacm/man/rping.1 @@ -52,6 +52,9 @@ The size of each message transferred, in bytes. (default 100) \-P Run the server in persistent mode. This allows multiple rping clients to connect to a single server instance. The server will run until killed. +.TP +\-q +Control QP Creation/Modification directly from the application, instead of rdma_cm. .SH "NOTES" Because this test maps RDMA resources to userspace, users must ensure that they have available system resources and permissions. See the