From patchwork Sun Feb 2 15:55:01 2014 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Or Gerlitz X-Patchwork-Id: 3566961 Return-Path: X-Original-To: patchwork-linux-rdma@patchwork.kernel.org Delivered-To: patchwork-parsemail@patchwork2.web.kernel.org Received: from mail.kernel.org (mail.kernel.org [198.145.19.201]) by patchwork2.web.kernel.org (Postfix) with ESMTP id C0EDFC02DC for ; Sun, 2 Feb 2014 15:55:34 +0000 (UTC) Received: from mail.kernel.org (localhost [127.0.0.1]) by mail.kernel.org (Postfix) with ESMTP id DAB7E201E7 for ; Sun, 2 Feb 2014 15:55:33 +0000 (UTC) Received: from vger.kernel.org (vger.kernel.org [209.132.180.67]) by mail.kernel.org (Postfix) with ESMTP id D51CC201E4 for ; Sun, 2 Feb 2014 15:55:32 +0000 (UTC) Received: (majordomo@vger.kernel.org) by vger.kernel.org via listexpand id S1751798AbaBBPz3 (ORCPT ); Sun, 2 Feb 2014 10:55:29 -0500 Received: from mailp.voltaire.com ([193.47.165.129]:49293 "EHLO mellanox.co.il" rhost-flags-OK-FAIL-OK-FAIL) by vger.kernel.org with ESMTP id S1751751AbaBBPzJ (ORCPT ); Sun, 2 Feb 2014 10:55:09 -0500 Received: from Internal Mail-Server by MTLPINE2 (envelope-from ogerlitz@mellanox.com) with SMTP; 2 Feb 2014 17:55:02 +0200 Received: from r-vnc04.mtr.labs.mlnx (r-vnc04.mtr.labs.mlnx [10.208.0.116]) by labmailer.mlnx (8.13.8/8.13.8) with ESMTP id s12Ft2B0011174; Sun, 2 Feb 2014 17:55:02 +0200 From: Or Gerlitz To: roland@kernel.org Cc: linux-rdma@vger.kernel.org, monis@mellanox.com, matanb@mellanox.com, amirv@mellanox.com, yishaih@mellanox.com, dledford@redhat.com, Or Gerlitz Subject: [PATCH libmlx4 2/2] Add ibv_query_port_ex support Date: Sun, 2 Feb 2014 17:55:01 +0200 Message-Id: <1391356501-4270-3-git-send-email-ogerlitz@mellanox.com> X-Mailer: git-send-email 1.7.8.2 In-Reply-To: <1391356501-4270-1-git-send-email-ogerlitz@mellanox.com> References: <1391356501-4270-1-git-send-email-ogerlitz@mellanox.com> Sender: linux-rdma-owner@vger.kernel.org Precedence: bulk List-ID: X-Mailing-List: linux-rdma@vger.kernel.org X-Spam-Status: No, score=-7.4 required=5.0 tests=BAYES_00, RCVD_IN_DNSWL_HI, RP_MATCHES_RCVD, UNPARSEABLE_RELAY autolearn=unavailable version=3.3.1 X-Spam-Checker-Version: SpamAssassin 3.3.1 (2010-03-16) on mail.kernel.org X-Virus-Scanned: ClamAV using ClamSMTP From: Matan Barak This patch adds the extended support for ibv_query_port in libmlx4. This allows to request fields that aren't availible by the current ibv_query_port API and avoid fetching from vendor library fields that the user doesn't need, which gives more room for optimizations. Also, it adds caching of link layer's type and the port caps. Signed-off-by: Matan Barak Signed-off-by: Or Gerlitz --- src/mlx4.c | 4 +++ src/mlx4.h | 9 +++++++ src/verbs.c | 78 ++++++++++++++++++++++++++++++++++++++++++++++++++++++---- 3 files changed, 85 insertions(+), 6 deletions(-) diff --git a/src/mlx4.c b/src/mlx4.c index ca91df4..58f5435 100644 --- a/src/mlx4.c +++ b/src/mlx4.c @@ -157,6 +157,8 @@ static int mlx4_init_context(struct verbs_device *v_device, context->qp_table_shift = ffs(context->num_qps) - 1 - MLX4_QP_TABLE_BITS; context->qp_table_mask = (1 << context->qp_table_shift) - 1; + for (i = 0; i < MLX4_PORTS_NUM; ++i) + context->port_query_cache[i].valid = 0; pthread_mutex_init(&context->qp_table_mutex, NULL); for (i = 0; i < MLX4_QP_TABLE_SIZE; ++i) @@ -205,6 +207,8 @@ static int mlx4_init_context(struct verbs_device *v_device, verbs_set_ctx_op(verbs_ctx, open_qp, mlx4_open_qp); verbs_set_ctx_op(verbs_ctx, drv_ibv_create_ah_ex, mlx4_create_ah_ex); + verbs_set_ctx_op(verbs_ctx, drv_query_port_ex, + mlx4_query_port_ex); verbs_ctx->has_comp_mask = VERBS_CONTEXT_CREATE_AH; return 0; diff --git a/src/mlx4.h b/src/mlx4.h index b7afa31..446e3b5 100644 --- a/src/mlx4.h +++ b/src/mlx4.h @@ -40,6 +40,8 @@ #include #include +#define MLX4_PORTS_NUM 2 + #ifdef HAVE_VALGRIND_MEMCHECK_H # include @@ -189,6 +191,11 @@ struct mlx4_context { pthread_mutex_t db_list_mutex; int cqe_size; struct mlx4_xsrq_table xsrq_table; + struct { + uint8_t valid; + uint8_t link_layer; + enum ibv_port_cap_flags caps; + } port_query_cache[MLX4_PORTS_NUM]; }; struct mlx4_buf { @@ -354,6 +361,8 @@ int mlx4_query_device(struct ibv_context *context, struct ibv_device_attr *attr); int mlx4_query_port(struct ibv_context *context, uint8_t port, struct ibv_port_attr *attr); +int mlx4_query_port_ex(struct ibv_context *context, uint8_t port_num, + struct ibv_port_attr_ex *port_attr); struct ibv_pd *mlx4_alloc_pd(struct ibv_context *context); int mlx4_free_pd(struct ibv_pd *pd); diff --git a/src/verbs.c b/src/verbs.c index 38fa3d3..a46e26c 100644 --- a/src/verbs.c +++ b/src/verbs.c @@ -70,8 +70,63 @@ int mlx4_query_port(struct ibv_context *context, uint8_t port, struct ibv_port_attr *attr) { struct ibv_query_port cmd; + int err; + + err = ibv_cmd_query_port(context, port, attr, &cmd, sizeof(cmd)); + if (!err) { + struct mlx4_context *mctx = to_mctx(context); + if (!mctx->port_query_cache[port - 1].valid) { + mctx->port_query_cache[port - 1].link_layer = + attr->link_layer; + mctx->port_query_cache[port - 1].caps = + attr->port_cap_flags; + mctx->port_query_cache[port - 1].valid = 1; + } + } + + return err; +} + +int mlx4_query_port_ex(struct ibv_context *context, uint8_t port_num, + struct ibv_port_attr_ex *port_attr) +{ + /* Check that only valid flags were given */ + if ((!port_attr->comp_mask & IBV_QUERY_PORT_EX_ATTR_MASK1) || + (port_attr->comp_mask & ~IBV_QUERY_PORT_EX_ATTR_MASKS) || + (port_attr->mask1 & ~IBV_QUERY_PORT_EX_MASK)) { + return -EINVAL; + } + + /* Optimize the link type query */ + if (port_attr->comp_mask == IBV_QUERY_PORT_EX_ATTR_MASK1) { + if (!(port_attr->mask1 & ~(IBV_QUERY_PORT_EX_LINK_LAYER | + IBV_QUERY_PORT_EX_CAP_FLAGS))) { + struct mlx4_context *mctx = to_mctx(context); + if (port_num > MLX4_PORTS_NUM) + return -EINVAL; + if (mctx->port_query_cache[port_num - 1].valid) { + if (port_attr->mask1 & + IBV_QUERY_PORT_EX_LINK_LAYER) + port_attr->link_layer = + mctx-> + port_query_cache[port_num - 1]. + link_layer; + if (port_attr->mask1 & + IBV_QUERY_PORT_EX_CAP_FLAGS) + port_attr->port_cap_flags = + mctx-> + port_query_cache[port_num - 1]. + caps; + return 0; + } + } + if (port_attr->mask1 & IBV_QUERY_PORT_EX_STD_MASK) { + return mlx4_query_port(context, port_num, + &port_attr->port_attr); + } + } - return ibv_cmd_query_port(context, port, attr, &cmd, sizeof cmd); + return -EOPNOTSUPP; } struct ibv_pd *mlx4_alloc_pd(struct ibv_context *context) @@ -787,9 +842,17 @@ static struct ibv_ah *mlx4_create_ah_common(struct ibv_pd *pd, struct ibv_ah_attr *attr) { struct mlx4_ah *ah; - struct ibv_port_attr port_attr; + struct mlx4_context *mctx; + struct ibv_port_attr_ex port_attr; + + mctx = to_mctx(pd->context); - if (ibv_query_port(pd->context, attr->port_num, &port_attr)) + if (attr->port_num < 1 || attr->port_num > MLX4_PORTS_NUM) + return NULL; + port_attr.comp_mask = IBV_QUERY_PORT_EX_ATTR_MASK1; + port_attr.mask1 = IBV_QUERY_PORT_EX_LINK_LAYER; + + if (ibv_query_port_ex(pd->context, attr->port_num, &port_attr)) return NULL; ah = malloc(sizeof *ah); @@ -827,12 +890,15 @@ static struct ibv_ah *mlx4_create_ah_common(struct ibv_pd *pd, struct ibv_ah *mlx4_create_ah(struct ibv_pd *pd, struct ibv_ah_attr *attr) { struct ibv_ah *ah = mlx4_create_ah_common(pd, attr); - struct ibv_port_attr port_attr; + struct ibv_port_attr_ex port_attr; + + port_attr.comp_mask = IBV_QUERY_PORT_EX_ATTR_MASK1; + port_attr.mask1 = IBV_QUERY_PORT_EX_LINK_LAYER; if (NULL != ah && - !ibv_query_port(pd->context, attr->port_num, &port_attr) && + !ibv_query_port_ex(pd->context, attr->port_num, &port_attr) && (port_attr.link_layer != IBV_LINK_LAYER_ETHERNET || - !mlx4_resolve_grh_to_l2(pd, to_mah(ah), attr))) + !mlx4_resolve_grh_to_l2(pd, to_mah(ah), attr))) return ah; if (ah)