From patchwork Wed Sep 23 10:26:55 2020 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Yishai Hadas X-Patchwork-Id: 11794763 Return-Path: Received: from mail.kernel.org (pdx-korg-mail-1.web.codeaurora.org [172.30.200.123]) by pdx-korg-patchwork-2.web.codeaurora.org (Postfix) with ESMTP id 9BED36CB for ; Wed, 23 Sep 2020 10:27:16 +0000 (UTC) Received: from vger.kernel.org (vger.kernel.org [23.128.96.18]) by mail.kernel.org (Postfix) with ESMTP id 8099923119 for ; Wed, 23 Sep 2020 10:27:16 +0000 (UTC) Authentication-Results: mail.kernel.org; dkim=pass (2048-bit key) header.d=nvidia.com header.i=@nvidia.com header.b="qvoUZhej" Received: (majordomo@vger.kernel.org) by vger.kernel.org via listexpand id S1726415AbgIWK1P (ORCPT ); Wed, 23 Sep 2020 06:27:15 -0400 Received: from hqnvemgate26.nvidia.com ([216.228.121.65]:13509 "EHLO hqnvemgate26.nvidia.com" rhost-flags-OK-OK-OK-OK) by vger.kernel.org with ESMTP id S1726310AbgIWK1P (ORCPT ); Wed, 23 Sep 2020 06:27:15 -0400 Received: from hqmail.nvidia.com (Not Verified[216.228.121.13]) by hqnvemgate26.nvidia.com (using TLS: TLSv1.2, AES256-SHA) id ; Wed, 23 Sep 2020 03:27:02 -0700 Received: from HQMAIL109.nvidia.com (172.20.187.15) by HQMAIL105.nvidia.com (172.20.187.12) with Microsoft SMTP Server (TLS) id 15.0.1473.3; Wed, 23 Sep 2020 10:27:15 +0000 Received: from dev-l-vrt-092.mtl.labs.mlnx (172.20.13.39) by mail.nvidia.com (172.20.187.15) with Microsoft SMTP Server id 15.0.1473.3 via Frontend Transport; Wed, 23 Sep 2020 10:27:13 +0000 From: Yishai Hadas To: CC: , , , Subject: [PATCH V1 rdma-core 1/8] Update kernel headers Date: Wed, 23 Sep 2020 13:26:55 +0300 Message-ID: <20200923102702.590008-2-yishaih@nvidia.com> X-Mailer: git-send-email 2.26.2 In-Reply-To: <20200923102702.590008-1-yishaih@nvidia.com> References: <20200923102702.590008-1-yishaih@nvidia.com> MIME-Version: 1.0 DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=nvidia.com; s=n1; t=1600856822; bh=XIuCYwaYCj2oDMlctjwiTI4QE40tilpmn+NRYx+mRCA=; h=From:To:CC:Subject:Date:Message-ID:X-Mailer:In-Reply-To: References:MIME-Version:Content-Transfer-Encoding:Content-Type; b=qvoUZhejKsbQOWri9aOMDXK8hgR+pD//bRgdDzFfk7ZS/NXeVhQgqPaZ/y6ZynyFC Vk6vO3UMODIvsDCE5Xj4m5fQs1IMJPsLNMqJUCHMgAOyLqEpE3RKmIp4za32HUwFPz KQ0ijP71WT1p2VUnXFoUQqaXV3NMS7gNWvWIDD3Ohh5FDm9599Hl+f1LynKETSC3UC hPMngOYShuAVxGT9afzwBEz4Mm96wXnjzeok/isSysd8U3/iaLGyZJBpv/01gCxoY/ gP3/T2Sn0egxwDCJEqRydvn4Y5yg/endUjLkamn0BLRBeCJ39m8RczogTLqMd4z2gC 2VYZCcwMYwpLA== Precedence: bulk List-ID: X-Mailing-List: linux-rdma@vger.kernel.org To commit 320a6a2fef0b ("RDMA/uverbs: Expose the new GID query API to user space") Signed-off-by: Yishai Hadas --- kernel-headers/rdma/ib_user_ioctl_cmds.h | 16 ++++++++++++++++ kernel-headers/rdma/ib_user_ioctl_verbs.h | 14 ++++++++++++++ kernel-headers/rdma/ib_user_verbs.h | 11 +++++++++++ kernel-headers/rdma/rdma_user_rxe.h | 6 +++--- 4 files changed, 44 insertions(+), 3 deletions(-) diff --git a/kernel-headers/rdma/ib_user_ioctl_cmds.h b/kernel-headers/rdma/ib_user_ioctl_cmds.h index 99dcabf..7968a18 100644 --- a/kernel-headers/rdma/ib_user_ioctl_cmds.h +++ b/kernel-headers/rdma/ib_user_ioctl_cmds.h @@ -70,6 +70,8 @@ enum uverbs_methods_device { UVERBS_METHOD_QUERY_PORT, UVERBS_METHOD_GET_CONTEXT, UVERBS_METHOD_QUERY_CONTEXT, + UVERBS_METHOD_QUERY_GID_TABLE, + UVERBS_METHOD_QUERY_GID_ENTRY, }; enum uverbs_attrs_invoke_write_cmd_attr_ids { @@ -352,4 +354,18 @@ enum uverbs_attrs_async_event_create { UVERBS_ATTR_ASYNC_EVENT_ALLOC_FD_HANDLE, }; +enum uverbs_attrs_query_gid_table_cmd_attr_ids { + UVERBS_ATTR_QUERY_GID_TABLE_ENTRY_SIZE, + UVERBS_ATTR_QUERY_GID_TABLE_FLAGS, + UVERBS_ATTR_QUERY_GID_TABLE_RESP_ENTRIES, + UVERBS_ATTR_QUERY_GID_TABLE_RESP_NUM_ENTRIES, +}; + +enum uverbs_attrs_query_gid_entry_cmd_attr_ids { + UVERBS_ATTR_QUERY_GID_ENTRY_PORT, + UVERBS_ATTR_QUERY_GID_ENTRY_GID_INDEX, + UVERBS_ATTR_QUERY_GID_ENTRY_FLAGS, + UVERBS_ATTR_QUERY_GID_ENTRY_RESP_ENTRY, +}; + #endif diff --git a/kernel-headers/rdma/ib_user_ioctl_verbs.h b/kernel-headers/rdma/ib_user_ioctl_verbs.h index 5debab4..cfea82a 100644 --- a/kernel-headers/rdma/ib_user_ioctl_verbs.h +++ b/kernel-headers/rdma/ib_user_ioctl_verbs.h @@ -250,4 +250,18 @@ enum rdma_driver_id { RDMA_DRIVER_SIW, }; +enum ib_uverbs_gid_type { + IB_UVERBS_GID_TYPE_IB, + IB_UVERBS_GID_TYPE_ROCE_V1, + IB_UVERBS_GID_TYPE_ROCE_V2, +}; + +struct ib_uverbs_gid_entry { + __aligned_u64 gid[2]; + __u32 gid_index; + __u32 port_num; + __u32 gid_type; + __u32 netdev_ifindex; /* It is 0 if there is no netdev associated with it */ +}; + #endif diff --git a/kernel-headers/rdma/ib_user_verbs.h b/kernel-headers/rdma/ib_user_verbs.h index 0474c74..456438c 100644 --- a/kernel-headers/rdma/ib_user_verbs.h +++ b/kernel-headers/rdma/ib_user_verbs.h @@ -457,6 +457,17 @@ struct ib_uverbs_poll_cq { __u32 ne; }; +enum ib_uverbs_wc_opcode { + IB_UVERBS_WC_SEND = 0, + IB_UVERBS_WC_RDMA_WRITE = 1, + IB_UVERBS_WC_RDMA_READ = 2, + IB_UVERBS_WC_COMP_SWAP = 3, + IB_UVERBS_WC_FETCH_ADD = 4, + IB_UVERBS_WC_BIND_MW = 5, + IB_UVERBS_WC_LOCAL_INV = 6, + IB_UVERBS_WC_TSO = 7, +}; + struct ib_uverbs_wc { __aligned_u64 wr_id; __u32 status; diff --git a/kernel-headers/rdma/rdma_user_rxe.h b/kernel-headers/rdma/rdma_user_rxe.h index aae2e69..d8f2e0e 100644 --- a/kernel-headers/rdma/rdma_user_rxe.h +++ b/kernel-headers/rdma/rdma_user_rxe.h @@ -99,8 +99,8 @@ struct rxe_send_wr { struct ib_mr *mr; __aligned_u64 reserved; }; - __u32 key; - __u32 access; + __u32 key; + __u32 access; } reg; } wr; }; @@ -112,7 +112,7 @@ struct rxe_sge { }; struct mminfo { - __aligned_u64 offset; + __aligned_u64 offset; __u32 size; __u32 pad; }; From patchwork Wed Sep 23 10:26:56 2020 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Yishai Hadas X-Patchwork-Id: 11794765 Return-Path: Received: from mail.kernel.org (pdx-korg-mail-1.web.codeaurora.org [172.30.200.123]) by pdx-korg-patchwork-2.web.codeaurora.org (Postfix) with ESMTP id CC8A7112C for ; Wed, 23 Sep 2020 10:27:18 +0000 (UTC) Received: from vger.kernel.org (vger.kernel.org [23.128.96.18]) by mail.kernel.org (Postfix) with ESMTP id ABA6B23119 for ; Wed, 23 Sep 2020 10:27:18 +0000 (UTC) Authentication-Results: mail.kernel.org; dkim=pass (2048-bit key) header.d=nvidia.com header.i=@nvidia.com header.b="Hv7OXSKy" Received: (majordomo@vger.kernel.org) by vger.kernel.org via listexpand id S1726419AbgIWK1S (ORCPT ); Wed, 23 Sep 2020 06:27:18 -0400 Received: from hqnvemgate25.nvidia.com ([216.228.121.64]:4527 "EHLO hqnvemgate25.nvidia.com" rhost-flags-OK-OK-OK-OK) by vger.kernel.org with ESMTP id S1726310AbgIWK1R (ORCPT ); Wed, 23 Sep 2020 06:27:17 -0400 Received: from hqmail.nvidia.com (Not Verified[216.228.121.13]) by hqnvemgate25.nvidia.com (using TLS: TLSv1.2, AES256-SHA) id ; Wed, 23 Sep 2020 03:26:30 -0700 Received: from HQMAIL109.nvidia.com (172.20.187.15) by HQMAIL109.nvidia.com (172.20.187.15) with Microsoft SMTP Server (TLS) id 15.0.1473.3; Wed, 23 Sep 2020 10:27:17 +0000 Received: from dev-l-vrt-092.mtl.labs.mlnx (172.20.13.39) by mail.nvidia.com (172.20.187.15) with Microsoft SMTP Server id 15.0.1473.3 via Frontend Transport; Wed, 23 Sep 2020 10:27:15 +0000 From: Yishai Hadas To: CC: , , , Subject: [PATCH V1 rdma-core 2/8] verbs: Change the name of enum ibv_gid_type Date: Wed, 23 Sep 2020 13:26:56 +0300 Message-ID: <20200923102702.590008-3-yishaih@nvidia.com> X-Mailer: git-send-email 2.26.2 In-Reply-To: <20200923102702.590008-1-yishaih@nvidia.com> References: <20200923102702.590008-1-yishaih@nvidia.com> MIME-Version: 1.0 DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=nvidia.com; s=n1; t=1600856790; bh=aW3Y5fXb6NAIX+eQNkb/lgVBBrt2kZ9HspHdqnFEBVE=; h=From:To:CC:Subject:Date:Message-ID:X-Mailer:In-Reply-To: References:MIME-Version:Content-Transfer-Encoding:Content-Type; b=Hv7OXSKyACTSeFhF6FJJIrl0inAAEc0JzPJhCB/QHDEhHNSA6jUB8h+um0G9s5y53 J2TfWrT1RB2avQqduG85mss2rxmmWqvhiee6g9eqonj6Bx8C0Nb0V5FtgiuKcdURrd 9yg2OSLQCURba/8nv7laoA0NyOo9GmFmf5UX1nJggy3c1pCec64rQe6XtPSA9o5eJ5 yvxEh94TITOsLpm1nMjket1unIWjCmmEWI/su/iuMSTB5cDrnhFR/HTkffmT6eEPeN jYOriTlHR92UjC6XWM5MQDMf5CH17PiZiFA8LpAwk9aufeXsLnHHnowUGTIxt1/rRu m3oPt0olj8ipA== Precedence: bulk List-ID: X-Mailing-List: linux-rdma@vger.kernel.org From: Avihai Horon Change the name of enum ibv_gid_type in order to introduce a new enum ibv_gid_type in verbs.h in the next commits, which will provide a more accurate gid type info by separating IB and RoCEv1 types. This is a preliminary step before introducing a new query GID API. Signed-off-by: Avihai Horon Signed-off-by: Yishai Hadas --- libibverbs/driver.h | 8 ++++---- libibverbs/examples/devinfo.c | 14 +++++++------- libibverbs/verbs.c | 21 +++++++++++---------- providers/mlx5/verbs.c | 2 +- pyverbs/device.pyx | 2 +- pyverbs/libibverbs.pxd | 2 +- pyverbs/libibverbs_enums.pxd | 6 +++--- tests/base.py | 3 ++- 8 files changed, 30 insertions(+), 28 deletions(-) diff --git a/libibverbs/driver.h b/libibverbs/driver.h index 4436363..046c07d 100644 --- a/libibverbs/driver.h +++ b/libibverbs/driver.h @@ -72,9 +72,9 @@ enum verbs_qp_mask { VERBS_QP_EX = 1 << 1, }; -enum ibv_gid_type { - IBV_GID_TYPE_IB_ROCE_V1, - IBV_GID_TYPE_ROCE_V2, +enum ibv_gid_type_sysfs { + IBV_GID_TYPE_SYSFS_IB_ROCE_V1, + IBV_GID_TYPE_SYSFS_ROCE_V2, }; enum ibv_mr_type { @@ -653,7 +653,7 @@ static inline bool check_comp_mask(uint64_t input, uint64_t supported) } int ibv_query_gid_type(struct ibv_context *context, uint8_t port_num, - unsigned int index, enum ibv_gid_type *type); + unsigned int index, enum ibv_gid_type_sysfs *type); static inline int ibv_check_alloc_parent_domain(struct ibv_parent_domain_init_attr *attr) diff --git a/libibverbs/examples/devinfo.c b/libibverbs/examples/devinfo.c index 00ed3f9..c245217 100644 --- a/libibverbs/examples/devinfo.c +++ b/libibverbs/examples/devinfo.c @@ -164,17 +164,17 @@ static const char *vl_str(uint8_t vl_num) } #define DEVINFO_INVALID_GID_TYPE 2 -static const char *gid_type_str(enum ibv_gid_type type) +static const char *gid_type_str(enum ibv_gid_type_sysfs type) { switch (type) { - case IBV_GID_TYPE_IB_ROCE_V1: return "RoCE v1"; - case IBV_GID_TYPE_ROCE_V2: return "RoCE v2"; + case IBV_GID_TYPE_SYSFS_IB_ROCE_V1: return "RoCE v1"; + case IBV_GID_TYPE_SYSFS_ROCE_V2: return "RoCE v2"; default: return "Invalid gid type"; } } static void print_formated_gid(union ibv_gid *gid, int i, - enum ibv_gid_type type, int ll) + enum ibv_gid_type_sysfs type, int ll) { char gid_str[INET6_ADDRSTRLEN] = {}; char str[20] = {}; @@ -182,7 +182,7 @@ static void print_formated_gid(union ibv_gid *gid, int i, if (ll == IBV_LINK_LAYER_ETHERNET) sprintf(str, ", %s", gid_type_str(type)); - if (type == IBV_GID_TYPE_IB_ROCE_V1) + if (type == IBV_GID_TYPE_SYSFS_IB_ROCE_V1) printf("\t\t\tGID[%3d]:\t\t%02x%02x:%02x%02x:%02x%02x:%02x%02x:%02x%02x:%02x%02x:%02x%02x:%02x%02x%s\n", i, gid->raw[0], gid->raw[1], gid->raw[2], gid->raw[3], gid->raw[4], gid->raw[5], gid->raw[6], @@ -190,7 +190,7 @@ static void print_formated_gid(union ibv_gid *gid, int i, gid->raw[11], gid->raw[12], gid->raw[13], gid->raw[14], gid->raw[15], str); - if (type == IBV_GID_TYPE_ROCE_V2) { + if (type == IBV_GID_TYPE_SYSFS_ROCE_V2) { inet_ntop(AF_INET6, gid->raw, gid_str, sizeof(gid_str)); printf("\t\t\tGID[%3d]:\t\t%s%s\n", i, gid_str, str); } @@ -200,7 +200,7 @@ static int print_all_port_gids(struct ibv_context *ctx, struct ibv_port_attr *port_attr, uint8_t port_num) { - enum ibv_gid_type type; + enum ibv_gid_type_sysfs type; union ibv_gid gid; int tbl_len; int rc = 0; diff --git a/libibverbs/verbs.c b/libibverbs/verbs.c index b54e2b8..9507ffd 100644 --- a/libibverbs/verbs.c +++ b/libibverbs/verbs.c @@ -704,7 +704,7 @@ LATEST_SYMVER_FUNC(ibv_create_ah, 1_1, "IBVERBS_1.1", #define V1_TYPE "IB/RoCE v1" #define V2_TYPE "RoCE v2" int ibv_query_gid_type(struct ibv_context *context, uint8_t port_num, - unsigned int index, enum ibv_gid_type *type) + unsigned int index, enum ibv_gid_type_sysfs *type) { struct verbs_device *verbs_device = verbs_get_device(context->device); char buff[11]; @@ -723,7 +723,7 @@ int ibv_query_gid_type(struct ibv_context *context, uint8_t port_num, /* In IB, this file doesn't exist and the kernel sets * errno to -EINVAL. */ - *type = IBV_GID_TYPE_IB_ROCE_V1; + *type = IBV_GID_TYPE_SYSFS_IB_ROCE_V1; return 0; } if (asprintf(&dir_path, "%s/%s/%d/%s/", @@ -738,7 +738,7 @@ int ibv_query_gid_type(struct ibv_context *context, uint8_t port_num, * we have an old kernel and all GIDs are * IB/RoCE v1 */ - *type = IBV_GID_TYPE_IB_ROCE_V1; + *type = IBV_GID_TYPE_SYSFS_IB_ROCE_V1; else return -1; } else { @@ -748,9 +748,9 @@ int ibv_query_gid_type(struct ibv_context *context, uint8_t port_num, } } else { if (!strcmp(buff, V1_TYPE)) { - *type = IBV_GID_TYPE_IB_ROCE_V1; + *type = IBV_GID_TYPE_SYSFS_IB_ROCE_V1; } else if (!strcmp(buff, V2_TYPE)) { - *type = IBV_GID_TYPE_ROCE_V2; + *type = IBV_GID_TYPE_SYSFS_ROCE_V2; } else { errno = ENOTSUP; return -1; @@ -761,9 +761,10 @@ int ibv_query_gid_type(struct ibv_context *context, uint8_t port_num, } static int ibv_find_gid_index(struct ibv_context *context, uint8_t port_num, - union ibv_gid *gid, enum ibv_gid_type gid_type) + union ibv_gid *gid, + enum ibv_gid_type_sysfs gid_type) { - enum ibv_gid_type sgid_type = 0; + enum ibv_gid_type_sysfs sgid_type = 0; union ibv_gid sgid; int i = 0, ret; @@ -863,7 +864,7 @@ static inline int set_ah_attr_by_ipv4(struct ibv_context *context, map_ipv4_addr_to_ipv6(ip4h->daddr, (struct in6_addr *)&sgid); ret = ibv_find_gid_index(context, port_num, &sgid, - IBV_GID_TYPE_ROCE_V2); + IBV_GID_TYPE_SYSFS_ROCE_V2); if (ret < 0) return ret; @@ -893,9 +894,9 @@ static inline int set_ah_attr_by_ipv6(struct ibv_context *context, ah_attr->grh.dgid = grh->sgid; if (grh->next_hdr == IPPROTO_UDP) { - sgid_type = IBV_GID_TYPE_ROCE_V2; + sgid_type = IBV_GID_TYPE_SYSFS_ROCE_V2; } else if (grh->next_hdr == IB_NEXT_HDR) { - sgid_type = IBV_GID_TYPE_IB_ROCE_V1; + sgid_type = IBV_GID_TYPE_SYSFS_IB_ROCE_V1; } else { errno = EPROTONOSUPPORT; return -1; diff --git a/providers/mlx5/verbs.c b/providers/mlx5/verbs.c index 4650250..917d057 100644 --- a/providers/mlx5/verbs.c +++ b/providers/mlx5/verbs.c @@ -2955,7 +2955,7 @@ struct ibv_ah *mlx5_create_ah(struct ibv_pd *pd, struct ibv_ah_attr *attr) attr->grh.sgid_index, &gid_type)) goto err; - if (gid_type == IBV_GID_TYPE_ROCE_V2) + if (gid_type == IBV_GID_TYPE_SYSFS_ROCE_V2) mlx5_ah_set_udp_sport(ah, attr); /* Since RoCE packets must contain GRH, this bit is reserved diff --git a/pyverbs/device.pyx b/pyverbs/device.pyx index b75fcd0..c1323cd 100755 --- a/pyverbs/device.pyx +++ b/pyverbs/device.pyx @@ -220,7 +220,7 @@ cdef class Context(PyverbsCM): return gid def query_gid_type(self, unsigned int port_num, unsigned int index): - cdef v.ibv_gid_type gid_type + cdef v.ibv_gid_type_sysfs gid_type rc = v.ibv_query_gid_type(self.context, port_num, index, &gid_type) if rc != 0: raise PyverbsRDMAErrno('Failed to query gid type of port {p} and gid index {g}' diff --git a/pyverbs/libibverbs.pxd b/pyverbs/libibverbs.pxd index dda33e7..c84b9fc 100755 --- a/pyverbs/libibverbs.pxd +++ b/pyverbs/libibverbs.pxd @@ -617,6 +617,6 @@ cdef extern from 'infiniband/verbs.h': cdef extern from 'infiniband/driver.h': int ibv_query_gid_type(ibv_context *context, uint8_t port_num, - unsigned int index, ibv_gid_type *type) + unsigned int index, ibv_gid_type_sysfs *type) int ibv_set_ece(ibv_qp *qp, ibv_ece *ece) int ibv_query_ece(ibv_qp *qp, ibv_ece *ece) diff --git a/pyverbs/libibverbs_enums.pxd b/pyverbs/libibverbs_enums.pxd index 7c1a120..83ca516 100755 --- a/pyverbs/libibverbs_enums.pxd +++ b/pyverbs/libibverbs_enums.pxd @@ -443,6 +443,6 @@ _IBV_ADVISE_MR_FLAG_FLUSH = IBV_ADVISE_MR_FLAG_FLUSH cdef extern from '': - cpdef enum ibv_gid_type: - IBV_GID_TYPE_IB_ROCE_V1 - IBV_GID_TYPE_ROCE_V2 + cpdef enum ibv_gid_type_sysfs: + IBV_GID_TYPE_SYSFS_IB_ROCE_V1 + IBV_GID_TYPE_SYSFS_ROCE_V2 diff --git a/tests/base.py b/tests/base.py index b6c389d..0ebd728 100755 --- a/tests/base.py +++ b/tests/base.py @@ -176,7 +176,8 @@ class RDMATestCase(unittest.TestCase): continue # Avoid RoCEv2 GIDs on unsupported devices if port_attrs.link_layer == e.IBV_LINK_LAYER_ETHERNET and \ - ctx.query_gid_type(port, idx) == e.IBV_GID_TYPE_ROCE_V2 and \ + ctx.query_gid_type(port, idx) == \ + e.IBV_GID_TYPE_SYSFS_ROCE_V2 and \ has_roce_hw_bug(vendor_id, vendor_pid): continue if not os.path.exists('/sys/class/infiniband/{}/device/net/'.format(dev)): From patchwork Wed Sep 23 10:26:57 2020 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Yishai Hadas X-Patchwork-Id: 11794767 Return-Path: Received: from mail.kernel.org (pdx-korg-mail-1.web.codeaurora.org [172.30.200.123]) by pdx-korg-patchwork-2.web.codeaurora.org (Postfix) with ESMTP id 4C1AE6CB for ; Wed, 23 Sep 2020 10:27:21 +0000 (UTC) Received: from vger.kernel.org (vger.kernel.org [23.128.96.18]) by mail.kernel.org (Postfix) with ESMTP id 28F0823119 for ; Wed, 23 Sep 2020 10:27:21 +0000 (UTC) Authentication-Results: mail.kernel.org; dkim=pass (2048-bit key) header.d=nvidia.com header.i=@nvidia.com header.b="fFtZzlok" Received: (majordomo@vger.kernel.org) by vger.kernel.org via listexpand id S1726513AbgIWK1U (ORCPT ); Wed, 23 Sep 2020 06:27:20 -0400 Received: from hqnvemgate26.nvidia.com ([216.228.121.65]:13512 "EHLO hqnvemgate26.nvidia.com" rhost-flags-OK-OK-OK-OK) by vger.kernel.org with ESMTP id S1726310AbgIWK1U (ORCPT ); Wed, 23 Sep 2020 06:27:20 -0400 Received: from hqmail.nvidia.com (Not Verified[216.228.121.13]) by hqnvemgate26.nvidia.com (using TLS: TLSv1.2, AES256-SHA) id ; Wed, 23 Sep 2020 03:27:06 -0700 Received: from HQMAIL109.nvidia.com (172.20.187.15) by HQMAIL101.nvidia.com (172.20.187.10) with Microsoft SMTP Server (TLS) id 15.0.1473.3; Wed, 23 Sep 2020 10:27:19 +0000 Received: from dev-l-vrt-092.mtl.labs.mlnx (172.20.13.39) by mail.nvidia.com (172.20.187.15) with Microsoft SMTP Server id 15.0.1473.3 via Frontend Transport; Wed, 23 Sep 2020 10:27:17 +0000 From: Yishai Hadas To: CC: , , , Subject: [PATCH V1 rdma-core 3/8] verbs: Introduce a new query GID entry API Date: Wed, 23 Sep 2020 13:26:57 +0300 Message-ID: <20200923102702.590008-4-yishaih@nvidia.com> X-Mailer: git-send-email 2.26.2 In-Reply-To: <20200923102702.590008-1-yishaih@nvidia.com> References: <20200923102702.590008-1-yishaih@nvidia.com> MIME-Version: 1.0 DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=nvidia.com; s=n1; t=1600856827; bh=eQ64vH0I2/IndYsvAUD55pOuucaEbARIFbiv2aH9NCc=; h=From:To:CC:Subject:Date:Message-ID:X-Mailer:In-Reply-To: References:MIME-Version:Content-Transfer-Encoding:Content-Type; b=fFtZzlok9BGymNnxsX6mEibv2b2Oe4MHQ0GCa08lUTgyact1ci5mD9VLAXVkcmIk9 7wMCKtHCMzkyQdDUtGpG6MPtEDvYXEasXbSHXeve3hoDpRt0c6rBlkbvty1FoHvAOz 6srDUGJrA+lSTt9SBpZclhaKdsY3tITFoMtosPZsbbZNP1QdFUAhMvS+FuSVSkkjf/ 2aQ0TNKPidS16oZRm8OY9ue+yvgrbX23NOUjsMdjQ0W2IYmSd3nfCAwJb6Jcn4U/bu 65V2RvGukn0San9WjFNGPsadoBJgZZwJYPiGYwOQmg8uuKhgqwZU9EVAcc2CEAlYip mqpPwqzeaGlPg== Precedence: bulk List-ID: X-Mailing-List: linux-rdma@vger.kernel.org From: Avihai Horon Introduce the ibv_query_gid_ex verb which queries a specific index in the GID table of the given port of the given device. The queried data is stored in a buffer provided by the user. If the kernel doesn't support ioctl or the needed uverbs method, the API will try to query the GID entry via sysfs. This API provides a faster way to query a GID table entry using a single call over ioctl, instead of multiple calls to open, close and read multiple sysfs files for a single GID table entry. Signed-off-by: Avihai Horon Signed-off-by: Yishai Hadas --- debian/libibverbs1.symbols | 2 + libibverbs/CMakeLists.txt | 2 +- libibverbs/cmd_device.c | 105 +++++++++++++++++++++++++++++++++++ libibverbs/libibverbs.map.in | 5 ++ libibverbs/man/CMakeLists.txt | 1 + libibverbs/man/ibv_query_gid_ex.3.md | 93 +++++++++++++++++++++++++++++++ libibverbs/verbs.h | 29 ++++++++++ 7 files changed, 236 insertions(+), 1 deletion(-) create mode 100644 libibverbs/man/ibv_query_gid_ex.3.md diff --git a/debian/libibverbs1.symbols b/debian/libibverbs1.symbols index 2efbe89..536d543 100644 --- a/debian/libibverbs1.symbols +++ b/debian/libibverbs1.symbols @@ -8,7 +8,9 @@ libibverbs.so.1 libibverbs1 #MINVER# IBVERBS_1.8@IBVERBS_1.8 28 IBVERBS_1.9@IBVERBS_1.9 30 IBVERBS_1.10@IBVERBS_1.10 31 + IBVERBS_1.11@IBVERBS_1.11 32 (symver)IBVERBS_PRIVATE_25 25 + _ibv_query_gid_ex@IBVERBS_1.11 32 ibv_ack_async_event@IBVERBS_1.0 1.1.6 ibv_ack_async_event@IBVERBS_1.1 1.1.6 ibv_ack_cq_events@IBVERBS_1.0 1.1.6 diff --git a/libibverbs/CMakeLists.txt b/libibverbs/CMakeLists.txt index 06a590f..0fe4256 100644 --- a/libibverbs/CMakeLists.txt +++ b/libibverbs/CMakeLists.txt @@ -21,7 +21,7 @@ configure_file("libibverbs.map.in" rdma_library(ibverbs "${CMAKE_CURRENT_BINARY_DIR}/libibverbs.map" # See Documentation/versioning.md - 1 1.10.${PACKAGE_VERSION} + 1 1.11.${PACKAGE_VERSION} all_providers.c cmd.c cmd_ah.c diff --git a/libibverbs/cmd_device.c b/libibverbs/cmd_device.c index a55fb10..8643cc6 100644 --- a/libibverbs/cmd_device.c +++ b/libibverbs/cmd_device.c @@ -32,6 +32,8 @@ #include +#include + static void copy_query_port_resp_to_port_attr(struct ibv_port_attr *port_attr, struct ib_uverbs_query_port_resp *resp) { @@ -202,3 +204,106 @@ int ibv_cmd_query_context(struct ibv_context *context, return 0; } + +static int is_zero_gid(union ibv_gid *gid) +{ + const union ibv_gid zgid = {}; + + return !memcmp(gid, &zgid, sizeof(*gid)); +} + +static int query_sysfs_gid_ndev_ifindex(struct ibv_context *context, + uint8_t port_num, uint32_t gid_index, + uint32_t *ndev_ifindex) +{ + struct verbs_device *verbs_device = verbs_get_device(context->device); + char buff[IF_NAMESIZE]; + + if (ibv_read_ibdev_sysfs_file(buff, sizeof(buff), verbs_device->sysfs, + "ports/%d/gid_attrs/ndevs/%d", port_num, + gid_index) <= 0) { + *ndev_ifindex = 0; + return 0; + } + + *ndev_ifindex = if_nametoindex(buff); + return *ndev_ifindex ? 0 : errno; +} + +static int query_sysfs_gid_entry(struct ibv_context *context, uint32_t port_num, + uint32_t gid_index, + struct ibv_gid_entry *entry) +{ + enum ibv_gid_type_sysfs gid_type; + struct ibv_port_attr port_attr = {}; + int ret; + + entry->gid_index = gid_index; + entry->port_num = port_num; + ret = ibv_query_gid(context, port_num, gid_index, &entry->gid); + if (ret) + return EINVAL; + + ret = ibv_query_gid_type(context, port_num, gid_index, &gid_type); + if (ret) + return EINVAL; + + if (gid_type == IBV_GID_TYPE_SYSFS_IB_ROCE_V1) { + ret = ibv_query_port(context, port_num, &port_attr); + if (ret) + goto out; + + if (port_attr.link_layer == IBV_LINK_LAYER_INFINIBAND) { + entry->gid_type = IBV_GID_TYPE_IB; + } else if (port_attr.link_layer == IBV_LINK_LAYER_ETHERNET) { + entry->gid_type = IBV_GID_TYPE_ROCE_V1; + } else { + ret = EINVAL; + goto out; + } + } else { + entry->gid_type = IBV_GID_TYPE_ROCE_V2; + } + + ret = query_sysfs_gid_ndev_ifindex(context, port_num, gid_index, + &entry->ndev_ifindex); + +out: + return ret; +} + +/* Using async_event cmd_name because query_gid_ex is not in + * verbs_context_ops while async_event is and doesn't use ioctl. + */ +#define query_gid_kernel_cap async_event +int _ibv_query_gid_ex(struct ibv_context *context, uint32_t port_num, + uint32_t gid_index, struct ibv_gid_entry *entry, + uint32_t flags, size_t entry_size) +{ + DECLARE_COMMAND_BUFFER(cmdb, UVERBS_OBJECT_DEVICE, + UVERBS_METHOD_QUERY_GID_ENTRY, 4); + int ret; + + fill_attr_const_in(cmdb, UVERBS_ATTR_QUERY_GID_ENTRY_PORT, port_num); + fill_attr_const_in(cmdb, UVERBS_ATTR_QUERY_GID_ENTRY_GID_INDEX, + gid_index); + fill_attr_in_uint32(cmdb, UVERBS_ATTR_QUERY_GID_ENTRY_FLAGS, flags); + fill_attr_out(cmdb, UVERBS_ATTR_QUERY_GID_ENTRY_RESP_ENTRY, entry, + entry_size); + + switch (execute_ioctl_fallback(context, query_gid_kernel_cap, cmdb, + &ret)) { + case TRY_WRITE: + if (flags) + return EOPNOTSUPP; + + ret = query_sysfs_gid_entry(context, port_num, gid_index, + entry); + if (ret) + return ret; + + return is_zero_gid(&entry->gid) ? ENODATA : 0; + default: + return ret; + } +} diff --git a/libibverbs/libibverbs.map.in b/libibverbs/libibverbs.map.in index 3240e00..dae4963 100644 --- a/libibverbs/libibverbs.map.in +++ b/libibverbs/libibverbs.map.in @@ -142,6 +142,11 @@ IBVERBS_1.10 { ibv_unimport_pd; } IBVERBS_1.9; +IBVERBS_1.11 { + global: + _ibv_query_gid_ex; +} IBVERBS_1.10; + /* If any symbols in this stanza change ABI then the entire staza gets a new symbol version. See the top level CMakeLists.txt for this setting. */ diff --git a/libibverbs/man/CMakeLists.txt b/libibverbs/man/CMakeLists.txt index b925004..2dea4ff 100644 --- a/libibverbs/man/CMakeLists.txt +++ b/libibverbs/man/CMakeLists.txt @@ -57,6 +57,7 @@ rdma_man_pages( ibv_query_device_ex.3 ibv_query_ece.3.md ibv_query_gid.3.md + ibv_query_gid_ex.3.md ibv_query_pkey.3.md ibv_query_port.3 ibv_query_qp.3 diff --git a/libibverbs/man/ibv_query_gid_ex.3.md b/libibverbs/man/ibv_query_gid_ex.3.md new file mode 100644 index 0000000..9e14f01 --- /dev/null +++ b/libibverbs/man/ibv_query_gid_ex.3.md @@ -0,0 +1,93 @@ +--- +date: 2020-04-24 +footer: libibverbs +header: "Libibverbs Programmer's Manual" +layout: page +license: 'Licensed under the OpenIB.org BSD license (FreeBSD Variant) - See COPYING.md' +section: 3 +title: IBV_QUERY_GID_EX +--- + +# NAME + +ibv_query_gid_ex - Query an InfiniBand port's GID table entry + +# SYNOPSIS + +```c +#include + +int ibv_query_gid_ex(struct ibv_context *context, + uint32_t port_num, + uint32_t gid_index, + struct ibv_gid_entry *entry, + uint32_t flags); +``` + +# DESCRIPTION + +**ibv_query_gid_ex()** returns the GID entry at *entry* for +*gid_index* of port *port_num* for device context *context*. + +# ARGUMENTS + +*context* +: The context of the device to query. + +*port_num* +: The number of port to query its GID table. + +*gid_index* +: The index of the GID table entry to query. + +## *entry* Argument +: An ibv_gid_entry struct, as defined in . +```c +struct ibv_gid_entry { + union ibv_gid gid; + uint32_t gid_index; + uint32_t port_num; + uint32_t gid_type; + uint32_t ndev_ifindex; +}; +``` + + *gid* +: The GID entry. + + *gid_index* +: The GID table index of this entry. + + *port_num* +: The port number that this GID belongs to. + + *gid_type* +: enum ibv_gid_type, can be one of IBV_GID_TYPE_IB, IBV_GID_TYPE_ROCE_V1 or IBV_GID_TYPE_ROCE_V2. + + *ndev_ifindex* +: The interface index of the net device associated with this GID. + It is 0 if there is no net device associated with it. + +*flags* +: Extra fields to query post *ndev_ifindex*, for now must be 0. + +# RETURN VALUE + +**ibv_query_gid_ex()** returns 0 on success or errno value on error. + +# ERRORS + +ENODATA +: *gid_index* is within the GID table size of port *port_num* but there is no data in this index. + +# SEE ALSO + +**ibv_open_device**(3), +**ibv_query_device**(3), +**ibv_query_pkey**(3), +**ibv_query_port**(3), +**ibv_query_gid_table**(3) + +# AUTHOR + +Parav Pandit diff --git a/libibverbs/verbs.h b/libibverbs/verbs.h index 2e785aa..e5bf900 100644 --- a/libibverbs/verbs.h +++ b/libibverbs/verbs.h @@ -68,6 +68,20 @@ union ibv_gid { } global; }; +enum ibv_gid_type { + IBV_GID_TYPE_IB, + IBV_GID_TYPE_ROCE_V1, + IBV_GID_TYPE_ROCE_V2, +}; + +struct ibv_gid_entry { + union ibv_gid gid; + uint32_t gid_index; + uint32_t port_num; + uint32_t gid_type; /* enum ibv_gid_type */ + uint32_t ndev_ifindex; +}; + #define vext_field_avail(type, fld, sz) (offsetof(type, fld) < (sz)) #ifdef __cplusplus @@ -2330,6 +2344,21 @@ static inline int ___ibv_query_port(struct ibv_context *context, int ibv_query_gid(struct ibv_context *context, uint8_t port_num, int index, union ibv_gid *gid); +int _ibv_query_gid_ex(struct ibv_context *context, uint32_t port_num, + uint32_t gid_index, struct ibv_gid_entry *entry, + uint32_t flags, size_t entry_size); + +/** + * ibv_query_gid_ex - Read a GID table entry + */ +static inline int ibv_query_gid_ex(struct ibv_context *context, + uint32_t port_num, uint32_t gid_index, + struct ibv_gid_entry *entry, uint32_t flags) +{ + return _ibv_query_gid_ex(context, port_num, gid_index, entry, flags, + sizeof(*entry)); +} + /** * ibv_query_pkey - Get a P_Key table entry */ From patchwork Wed Sep 23 10:26:58 2020 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Yishai Hadas X-Patchwork-Id: 11794769 Return-Path: Received: from mail.kernel.org (pdx-korg-mail-1.web.codeaurora.org [172.30.200.123]) by pdx-korg-patchwork-2.web.codeaurora.org (Postfix) with ESMTP id 5A847112C for ; Wed, 23 Sep 2020 10:27:23 +0000 (UTC) Received: from vger.kernel.org (vger.kernel.org [23.128.96.18]) by mail.kernel.org (Postfix) with ESMTP id 3576223119 for ; Wed, 23 Sep 2020 10:27:23 +0000 (UTC) Authentication-Results: mail.kernel.org; dkim=pass (2048-bit key) header.d=nvidia.com header.i=@nvidia.com header.b="rZ6Yqg4J" Received: (majordomo@vger.kernel.org) by vger.kernel.org via listexpand id S1726497AbgIWK1W (ORCPT ); Wed, 23 Sep 2020 06:27:22 -0400 Received: from hqnvemgate24.nvidia.com ([216.228.121.143]:5315 "EHLO hqnvemgate24.nvidia.com" rhost-flags-OK-OK-OK-OK) by vger.kernel.org with ESMTP id S1726310AbgIWK1W (ORCPT ); Wed, 23 Sep 2020 06:27:22 -0400 Received: from hqmail.nvidia.com (Not Verified[216.228.121.13]) by hqnvemgate24.nvidia.com (using TLS: TLSv1.2, AES256-SHA) id ; Wed, 23 Sep 2020 03:25:47 -0700 Received: from HQMAIL109.nvidia.com (172.20.187.15) by HQMAIL107.nvidia.com (172.20.187.13) with Microsoft SMTP Server (TLS) id 15.0.1473.3; Wed, 23 Sep 2020 10:27:21 +0000 Received: from dev-l-vrt-092.mtl.labs.mlnx (172.20.13.39) by mail.nvidia.com (172.20.187.15) with Microsoft SMTP Server id 15.0.1473.3 via Frontend Transport; Wed, 23 Sep 2020 10:27:19 +0000 From: Yishai Hadas To: CC: , , , Subject: [PATCH V1 rdma-core 4/8] verbs: Implement ibv_query_gid and ibv_query_gid_type over ioctl Date: Wed, 23 Sep 2020 13:26:58 +0300 Message-ID: <20200923102702.590008-5-yishaih@nvidia.com> X-Mailer: git-send-email 2.26.2 In-Reply-To: <20200923102702.590008-1-yishaih@nvidia.com> References: <20200923102702.590008-1-yishaih@nvidia.com> MIME-Version: 1.0 DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=nvidia.com; s=n1; t=1600856747; bh=XoN0rGrnqW2rx1uPx47ThQh34P4/ie4LKX1Ya78NsB8=; h=From:To:CC:Subject:Date:Message-ID:X-Mailer:In-Reply-To: References:MIME-Version:Content-Transfer-Encoding:Content-Type; b=rZ6Yqg4J9HmeCyIcuzm2S8DF3kBBYVEN5qJNRMTb0aOHMxsDHnm3jhJ9jaIzquZj5 Rzmv5nDt8s213A+0wRkQfQn45zG+eCsi8tlkaqk0nQfUCqfOOij9VkPqWZo+Ej3XML iTvtDfevZQ6UiGjR6G2UTOnXY9pffarF0lxjuoS2wVc0IhtBkjad/ERCdvKtWIuKFD 1z8n7fUZ3Lsu1mGvvxplMOCzZm5yhHCgEOBya/wKs2c527eDf6s0k4dsInods/w/nx 839KWcfBY2ZAzlbQ9zpInx4Rnj/M+2h0RArOWILt+tME8I5wGK+ywhU0DMS00FaPRB qG6DEtQHE8lPA== Precedence: bulk List-ID: X-Mailing-List: linux-rdma@vger.kernel.org From: Avihai Horon Currently ibv_query_gid and ibv_query_gid_type are implemented over sysfs. In order to improve their performance we implement them using the new query GID entry API, so now they will use ioctl and fallback to sysfs. Signed-off-by: Avihai Horon Signed-off-by: Yishai Hadas --- libibverbs/cmd_device.c | 92 +++++++++++++++++++++++++++++++++++++++++++++++-- libibverbs/verbs.c | 92 +++++++++++++++---------------------------------- 2 files changed, 117 insertions(+), 67 deletions(-) diff --git a/libibverbs/cmd_device.c b/libibverbs/cmd_device.c index 8643cc6..f707273 100644 --- a/libibverbs/cmd_device.c +++ b/libibverbs/cmd_device.c @@ -30,6 +30,10 @@ * SOFTWARE. */ +#define _GNU_SOURCE +#include +#include +#include #include #include @@ -230,6 +234,90 @@ static int query_sysfs_gid_ndev_ifindex(struct ibv_context *context, return *ndev_ifindex ? 0 : errno; } +static int query_sysfs_gid(struct ibv_context *context, uint8_t port_num, int index, + union ibv_gid *gid) +{ + struct verbs_device *verbs_device = verbs_get_device(context->device); + char attr[41]; + uint16_t val; + int i; + + if (ibv_read_ibdev_sysfs_file(attr, sizeof(attr), verbs_device->sysfs, + "ports/%d/gids/%d", port_num, index) < 0) + return -1; + + for (i = 0; i < 8; ++i) { + if (sscanf(attr + i * 5, "%hx", &val) != 1) + return -1; + gid->raw[i * 2] = val >> 8; + gid->raw[i * 2 + 1] = val & 0xff; + } + + return 0; +} + +/* GID types as appear in sysfs, no change is expected as of ABI + * compatibility. + */ +#define V1_TYPE "IB/RoCE v1" +#define V2_TYPE "RoCE v2" +static int query_sysfs_gid_type(struct ibv_context *context, uint8_t port_num, + unsigned int index, enum ibv_gid_type_sysfs *type) +{ + struct verbs_device *verbs_device = verbs_get_device(context->device); + char buff[11]; + + /* Reset errno so that we can rely on its value upon any error flow in + * ibv_read_sysfs_file. + */ + errno = 0; + if (ibv_read_ibdev_sysfs_file(buff, sizeof(buff), verbs_device->sysfs, + "ports/%d/gid_attrs/types/%d", port_num, + index) <= 0) { + char *dir_path; + DIR *dir; + + if (errno == EINVAL) { + /* In IB, this file doesn't exist and the kernel sets + * errno to -EINVAL. + */ + *type = IBV_GID_TYPE_SYSFS_IB_ROCE_V1; + return 0; + } + if (asprintf(&dir_path, "%s/%s/%d/%s/", + verbs_device->sysfs->ibdev_path, "ports", port_num, + "gid_attrs") < 0) + return -1; + dir = opendir(dir_path); + free(dir_path); + if (!dir) { + if (errno == ENOENT) + /* Assuming that if gid_attrs doesn't exist, + * we have an old kernel and all GIDs are + * IB/RoCE v1 + */ + *type = IBV_GID_TYPE_SYSFS_IB_ROCE_V1; + else + return -1; + } else { + closedir(dir); + errno = EFAULT; + return -1; + } + } else { + if (!strcmp(buff, V1_TYPE)) { + *type = IBV_GID_TYPE_SYSFS_IB_ROCE_V1; + } else if (!strcmp(buff, V2_TYPE)) { + *type = IBV_GID_TYPE_SYSFS_ROCE_V2; + } else { + errno = ENOTSUP; + return -1; + } + } + + return 0; +} + static int query_sysfs_gid_entry(struct ibv_context *context, uint32_t port_num, uint32_t gid_index, struct ibv_gid_entry *entry) @@ -240,11 +328,11 @@ static int query_sysfs_gid_entry(struct ibv_context *context, uint32_t port_num, entry->gid_index = gid_index; entry->port_num = port_num; - ret = ibv_query_gid(context, port_num, gid_index, &entry->gid); + ret = query_sysfs_gid(context, port_num, gid_index, &entry->gid); if (ret) return EINVAL; - ret = ibv_query_gid_type(context, port_num, gid_index, &gid_type); + ret = query_sysfs_gid_type(context, port_num, gid_index, &gid_type); if (ret) return EINVAL; diff --git a/libibverbs/verbs.c b/libibverbs/verbs.c index 9507ffd..99a48d5 100644 --- a/libibverbs/verbs.c +++ b/libibverbs/verbs.c @@ -221,21 +221,20 @@ LATEST_SYMVER_FUNC(ibv_query_gid, 1_1, "IBVERBS_1.1", struct ibv_context *context, uint8_t port_num, int index, union ibv_gid *gid) { - struct verbs_device *verbs_device = verbs_get_device(context->device); - char attr[41]; - uint16_t val; - int i; + struct ibv_gid_entry entry = {}; + int ret; - if (ibv_read_ibdev_sysfs_file(attr, sizeof(attr), verbs_device->sysfs, - "ports/%d/gids/%d", port_num, index) < 0) + ret = _ibv_query_gid_ex(context, port_num, index, &entry, 0, + sizeof(entry)); + /* Preserve API behavior for empty GID */ + if (ret == ENODATA) { + memset(gid, 0, sizeof(*gid)); + return 0; + } + if (ret) return -1; - for (i = 0; i < 8; ++i) { - if (sscanf(attr + i * 5, "%hx", &val) != 1) - return -1; - gid->raw[i * 2 ] = val >> 8; - gid->raw[i * 2 + 1] = val & 0xff; - } + memcpy(gid, &entry.gid, sizeof(entry.gid)); return 0; } @@ -698,64 +697,27 @@ LATEST_SYMVER_FUNC(ibv_create_ah, 1_1, "IBVERBS_1.1", return ah; } -/* GID types as appear in sysfs, no change is expected as of ABI - * compatibility. - */ -#define V1_TYPE "IB/RoCE v1" -#define V2_TYPE "RoCE v2" int ibv_query_gid_type(struct ibv_context *context, uint8_t port_num, unsigned int index, enum ibv_gid_type_sysfs *type) { - struct verbs_device *verbs_device = verbs_get_device(context->device); - char buff[11]; + struct ibv_gid_entry entry = {}; + int ret; - /* Reset errno so that we can rely on its value upon any error flow in - * ibv_read_sysfs_file. - */ - errno = 0; - if (ibv_read_ibdev_sysfs_file(buff, sizeof(buff), verbs_device->sysfs, - "ports/%d/gid_attrs/types/%d", port_num, - index) <= 0) { - char *dir_path; - DIR *dir; - - if (errno == EINVAL) { - /* In IB, this file doesn't exist and the kernel sets - * errno to -EINVAL. - */ - *type = IBV_GID_TYPE_SYSFS_IB_ROCE_V1; - return 0; - } - if (asprintf(&dir_path, "%s/%s/%d/%s/", - verbs_device->sysfs->ibdev_path, "ports", port_num, - "gid_attrs") < 0) - return -1; - dir = opendir(dir_path); - free(dir_path); - if (!dir) { - if (errno == ENOENT) - /* Assuming that if gid_attrs doesn't exist, - * we have an old kernel and all GIDs are - * IB/RoCE v1 - */ - *type = IBV_GID_TYPE_SYSFS_IB_ROCE_V1; - else - return -1; - } else { - closedir(dir); - errno = EFAULT; - return -1; - } - } else { - if (!strcmp(buff, V1_TYPE)) { - *type = IBV_GID_TYPE_SYSFS_IB_ROCE_V1; - } else if (!strcmp(buff, V2_TYPE)) { - *type = IBV_GID_TYPE_SYSFS_ROCE_V2; - } else { - errno = ENOTSUP; - return -1; - } + ret = _ibv_query_gid_ex(context, port_num, index, &entry, 0, + sizeof(entry)); + /* Preserve API behavior for empty GID */ + if (ret == ENODATA) { + *type = IBV_GID_TYPE_SYSFS_IB_ROCE_V1; + return 0; } + if (ret) + return -1; + + if (entry.gid_type == IBV_GID_TYPE_IB || + entry.gid_type == IBV_GID_TYPE_ROCE_V1) + *type = IBV_GID_TYPE_SYSFS_IB_ROCE_V1; + else + *type = IBV_GID_TYPE_SYSFS_ROCE_V2; return 0; } From patchwork Wed Sep 23 10:26:59 2020 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Yishai Hadas X-Patchwork-Id: 11794771 Return-Path: Received: from mail.kernel.org (pdx-korg-mail-1.web.codeaurora.org [172.30.200.123]) by pdx-korg-patchwork-2.web.codeaurora.org (Postfix) with ESMTP id 013BB112C for ; Wed, 23 Sep 2020 10:27:25 +0000 (UTC) Received: from vger.kernel.org (vger.kernel.org [23.128.96.18]) by mail.kernel.org (Postfix) with ESMTP id D5F9D2388A for ; Wed, 23 Sep 2020 10:27:24 +0000 (UTC) Authentication-Results: mail.kernel.org; dkim=pass (2048-bit key) header.d=nvidia.com header.i=@nvidia.com header.b="kj0WgZrl" Received: (majordomo@vger.kernel.org) by vger.kernel.org via listexpand id S1726515AbgIWK1Y (ORCPT ); Wed, 23 Sep 2020 06:27:24 -0400 Received: from hqnvemgate25.nvidia.com ([216.228.121.64]:4531 "EHLO hqnvemgate25.nvidia.com" rhost-flags-OK-OK-OK-OK) by vger.kernel.org with ESMTP id S1726310AbgIWK1Y (ORCPT ); Wed, 23 Sep 2020 06:27:24 -0400 Received: from hqmail.nvidia.com (Not Verified[216.228.121.13]) by hqnvemgate25.nvidia.com (using TLS: TLSv1.2, AES256-SHA) id ; Wed, 23 Sep 2020 03:26:37 -0700 Received: from HQMAIL109.nvidia.com (172.20.187.15) by HQMAIL105.nvidia.com (172.20.187.12) with Microsoft SMTP Server (TLS) id 15.0.1473.3; Wed, 23 Sep 2020 10:27:23 +0000 Received: from dev-l-vrt-092.mtl.labs.mlnx (172.20.13.39) by mail.nvidia.com (172.20.187.15) with Microsoft SMTP Server id 15.0.1473.3 via Frontend Transport; Wed, 23 Sep 2020 10:27:21 +0000 From: Yishai Hadas To: CC: , , , Subject: [PATCH V1 rdma-core 5/8] verbs: Optimize ibv_query_gid and ibv_query_gid_type Date: Wed, 23 Sep 2020 13:26:59 +0300 Message-ID: <20200923102702.590008-6-yishaih@nvidia.com> X-Mailer: git-send-email 2.26.2 In-Reply-To: <20200923102702.590008-1-yishaih@nvidia.com> References: <20200923102702.590008-1-yishaih@nvidia.com> MIME-Version: 1.0 DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=nvidia.com; s=n1; t=1600856797; bh=hNEHyZ+DPFvy3HP+kpONa64aaQu+yKzPzsueaDx4Gx0=; h=From:To:CC:Subject:Date:Message-ID:X-Mailer:In-Reply-To: References:MIME-Version:Content-Transfer-Encoding:Content-Type; b=kj0WgZrlcSsr1I8pQ+pv8NWLVSp0aXy/UDRI3p4WOXTkYidQnw15JD5whoQIY4bS1 Ff/BJ9SDk9xv34Y/MGzft7dpniTWCbaDza1KjZXE0mWnl5/NLZlntvmexsjLyY/lqd 0gpHOisb2sCPekyn8031jTCFS1doE0Ay08zuOvhNcu9uz/EvMMHIeF/kRiwDyi8Mhv Da3UTNXu2yWoD9c9HjuuNuiAg58daIkmi2O1+8KUNTAoUTY1T5O5cjbxfcMEE21c8K GmkU1rIZpzZDbHxVN+6RJFT1lDP4UaCMUW2+XeAgSv5ED7kFnx4h3VVxTh4sPA49x6 WSMi19vEYXZYQ== Precedence: bulk List-ID: X-Mailing-List: linux-rdma@vger.kernel.org From: Avihai Horon ibv_query_gid and ibv_query_gid_type are implemented as ioctl first and fallback to sysfs. Currently, if the fallback path is taken, all of the gid entry attributes are retrieved over sysfs. For example, if ibv_query_gid is called and the fallback path is taken, the gid type and the gid ndev ifindex will also be read over sysfs, even though we only need the gid. In order to eliminate these unnecessary sysfs reads, we add an attribute mask to ibv_cmd_query_gid_entry that will allow us to mark the specific gid entry attributes that we would like to query in fallback. Signed-off-by: Avihai Horon Signed-off-by: Yishai Hadas --- libibverbs/cmd_device.c | 76 ++++++++++++++++++++++++++++++++----------------- libibverbs/driver.h | 11 +++++++ libibverbs/verbs.c | 8 +++--- 3 files changed, 65 insertions(+), 30 deletions(-) diff --git a/libibverbs/cmd_device.c b/libibverbs/cmd_device.c index f707273..4f85010 100644 --- a/libibverbs/cmd_device.c +++ b/libibverbs/cmd_device.c @@ -320,41 +320,49 @@ static int query_sysfs_gid_type(struct ibv_context *context, uint8_t port_num, static int query_sysfs_gid_entry(struct ibv_context *context, uint32_t port_num, uint32_t gid_index, - struct ibv_gid_entry *entry) + struct ibv_gid_entry *entry, + uint32_t attr_mask) { enum ibv_gid_type_sysfs gid_type; struct ibv_port_attr port_attr = {}; - int ret; + int ret = 0; entry->gid_index = gid_index; entry->port_num = port_num; - ret = query_sysfs_gid(context, port_num, gid_index, &entry->gid); - if (ret) - return EINVAL; - - ret = query_sysfs_gid_type(context, port_num, gid_index, &gid_type); - if (ret) - return EINVAL; - if (gid_type == IBV_GID_TYPE_SYSFS_IB_ROCE_V1) { - ret = ibv_query_port(context, port_num, &port_attr); + if (attr_mask & VERBS_QUERY_GID_ATTR_GID) { + ret = query_sysfs_gid(context, port_num, gid_index, &entry->gid); if (ret) - goto out; + return EINVAL; + } - if (port_attr.link_layer == IBV_LINK_LAYER_INFINIBAND) { - entry->gid_type = IBV_GID_TYPE_IB; - } else if (port_attr.link_layer == IBV_LINK_LAYER_ETHERNET) { - entry->gid_type = IBV_GID_TYPE_ROCE_V1; + if (attr_mask & VERBS_QUERY_GID_ATTR_TYPE) { + ret = query_sysfs_gid_type(context, port_num, gid_index, &gid_type); + if (ret) + return EINVAL; + + if (gid_type == IBV_GID_TYPE_SYSFS_IB_ROCE_V1) { + ret = ibv_query_port(context, port_num, &port_attr); + if (ret) + goto out; + + if (port_attr.link_layer == IBV_LINK_LAYER_INFINIBAND) { + entry->gid_type = IBV_GID_TYPE_IB; + } else if (port_attr.link_layer == + IBV_LINK_LAYER_ETHERNET) { + entry->gid_type = IBV_GID_TYPE_ROCE_V1; + } else { + ret = EINVAL; + goto out; + } } else { - ret = EINVAL; - goto out; + entry->gid_type = IBV_GID_TYPE_ROCE_V2; } - } else { - entry->gid_type = IBV_GID_TYPE_ROCE_V2; } - ret = query_sysfs_gid_ndev_ifindex(context, port_num, gid_index, - &entry->ndev_ifindex); + if (attr_mask & VERBS_QUERY_GID_ATTR_NDEV_IFINDEX) + ret = query_sysfs_gid_ndev_ifindex(context, port_num, gid_index, + &entry->ndev_ifindex); out: return ret; @@ -364,9 +372,10 @@ out: * verbs_context_ops while async_event is and doesn't use ioctl. */ #define query_gid_kernel_cap async_event -int _ibv_query_gid_ex(struct ibv_context *context, uint32_t port_num, +int __ibv_query_gid_ex(struct ibv_context *context, uint32_t port_num, uint32_t gid_index, struct ibv_gid_entry *entry, - uint32_t flags, size_t entry_size) + uint32_t flags, size_t entry_size, + uint32_t fallback_attr_mask) { DECLARE_COMMAND_BUFFER(cmdb, UVERBS_OBJECT_DEVICE, UVERBS_METHOD_QUERY_GID_ENTRY, 4); @@ -386,12 +395,27 @@ int _ibv_query_gid_ex(struct ibv_context *context, uint32_t port_num, return EOPNOTSUPP; ret = query_sysfs_gid_entry(context, port_num, gid_index, - entry); + entry, fallback_attr_mask); if (ret) return ret; - return is_zero_gid(&entry->gid) ? ENODATA : 0; + if (fallback_attr_mask & VERBS_QUERY_GID_ATTR_GID && + is_zero_gid(&entry->gid)) + return ENODATA; + + return 0; default: return ret; } } + +int _ibv_query_gid_ex(struct ibv_context *context, uint32_t port_num, + uint32_t gid_index, struct ibv_gid_entry *entry, + uint32_t flags, size_t entry_size) +{ + return __ibv_query_gid_ex(context, port_num, gid_index, entry, + flags, entry_size, + VERBS_QUERY_GID_ATTR_GID | + VERBS_QUERY_GID_ATTR_TYPE | + VERBS_QUERY_GID_ATTR_NDEV_IFINDEX); +} diff --git a/libibverbs/driver.h b/libibverbs/driver.h index 046c07d..479e16e 100644 --- a/libibverbs/driver.h +++ b/libibverbs/driver.h @@ -77,6 +77,12 @@ enum ibv_gid_type_sysfs { IBV_GID_TYPE_SYSFS_ROCE_V2, }; +enum verbs_query_gid_attr_mask { + VERBS_QUERY_GID_ATTR_GID = 1 << 0, + VERBS_QUERY_GID_ATTR_TYPE = 1 << 1, + VERBS_QUERY_GID_ATTR_NDEV_IFINDEX = 1 << 2, +}; + enum ibv_mr_type { IBV_MR_TYPE_MR, IBV_MR_TYPE_NULL_MR, @@ -633,6 +639,11 @@ int ibv_cmd_reg_dm_mr(struct ibv_pd *pd, struct verbs_dm *dm, unsigned int access, struct verbs_mr *vmr, struct ibv_command_buffer *link); +int __ibv_query_gid_ex(struct ibv_context *context, uint32_t port_num, + uint32_t gid_index, struct ibv_gid_entry *entry, + uint32_t flags, size_t entry_size, + uint32_t fallback_attr_mask); + /* * sysfs helper functions */ diff --git a/libibverbs/verbs.c b/libibverbs/verbs.c index 99a48d5..e50ad94 100644 --- a/libibverbs/verbs.c +++ b/libibverbs/verbs.c @@ -224,8 +224,8 @@ LATEST_SYMVER_FUNC(ibv_query_gid, 1_1, "IBVERBS_1.1", struct ibv_gid_entry entry = {}; int ret; - ret = _ibv_query_gid_ex(context, port_num, index, &entry, 0, - sizeof(entry)); + ret = __ibv_query_gid_ex(context, port_num, index, &entry, 0, + sizeof(entry), VERBS_QUERY_GID_ATTR_GID); /* Preserve API behavior for empty GID */ if (ret == ENODATA) { memset(gid, 0, sizeof(*gid)); @@ -703,8 +703,8 @@ int ibv_query_gid_type(struct ibv_context *context, uint8_t port_num, struct ibv_gid_entry entry = {}; int ret; - ret = _ibv_query_gid_ex(context, port_num, index, &entry, 0, - sizeof(entry)); + ret = __ibv_query_gid_ex(context, port_num, index, &entry, 0, + sizeof(entry), VERBS_QUERY_GID_ATTR_TYPE); /* Preserve API behavior for empty GID */ if (ret == ENODATA) { *type = IBV_GID_TYPE_SYSFS_IB_ROCE_V1; From patchwork Wed Sep 23 10:27:00 2020 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Yishai Hadas X-Patchwork-Id: 11794773 Return-Path: Received: from mail.kernel.org (pdx-korg-mail-1.web.codeaurora.org [172.30.200.123]) by pdx-korg-patchwork-2.web.codeaurora.org (Postfix) with ESMTP id 2359D6CB for ; Wed, 23 Sep 2020 10:27:27 +0000 (UTC) Received: from vger.kernel.org (vger.kernel.org [23.128.96.18]) by mail.kernel.org (Postfix) with ESMTP id F24DC2388A for ; Wed, 23 Sep 2020 10:27:26 +0000 (UTC) Authentication-Results: mail.kernel.org; dkim=pass (2048-bit key) header.d=nvidia.com header.i=@nvidia.com header.b="iFB4sv36" Received: (majordomo@vger.kernel.org) by vger.kernel.org via listexpand id S1726518AbgIWK10 (ORCPT ); Wed, 23 Sep 2020 06:27:26 -0400 Received: from hqnvemgate24.nvidia.com ([216.228.121.143]:5323 "EHLO hqnvemgate24.nvidia.com" rhost-flags-OK-OK-OK-OK) by vger.kernel.org with ESMTP id S1726310AbgIWK10 (ORCPT ); Wed, 23 Sep 2020 06:27:26 -0400 Received: from hqmail.nvidia.com (Not Verified[216.228.121.13]) by hqnvemgate24.nvidia.com (using TLS: TLSv1.2, AES256-SHA) id ; Wed, 23 Sep 2020 03:25:51 -0700 Received: from HQMAIL109.nvidia.com (172.20.187.15) by HQMAIL111.nvidia.com (172.20.187.18) with Microsoft SMTP Server (TLS) id 15.0.1473.3; Wed, 23 Sep 2020 10:27:25 +0000 Received: from dev-l-vrt-092.mtl.labs.mlnx (172.20.13.39) by mail.nvidia.com (172.20.187.15) with Microsoft SMTP Server id 15.0.1473.3 via Frontend Transport; Wed, 23 Sep 2020 10:27:23 +0000 From: Yishai Hadas To: CC: , , , Subject: [PATCH V1 rdma-core 6/8] verbs: Introduce a new query GID table API Date: Wed, 23 Sep 2020 13:27:00 +0300 Message-ID: <20200923102702.590008-7-yishaih@nvidia.com> X-Mailer: git-send-email 2.26.2 In-Reply-To: <20200923102702.590008-1-yishaih@nvidia.com> References: <20200923102702.590008-1-yishaih@nvidia.com> MIME-Version: 1.0 DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=nvidia.com; s=n1; t=1600856751; bh=tHrlyLreSNEJiYj3+FcfNpxzdA+DO14aIUP2gw1zfTg=; h=From:To:CC:Subject:Date:Message-ID:X-Mailer:In-Reply-To: References:MIME-Version:Content-Transfer-Encoding:Content-Type; b=iFB4sv36ZElhfSYYYk2E+LStJvVaCxYPMJgmOhN5QRSVpgf/1sIGQ9oAImd2zRop1 uD9xL2rby0EUspcozCJmazVRpt04z7iCi0H4/UR3bK0lmptv9TtkxJBddjlBbZa1q2 5cDftswuULY3joEVucwZligLRRIxmfZnPuLdAXdMAM3bpsvy+p79+Rl/NTMjr0usbk U4FhRRGoZDAgmUcxLgM6rHdNptaZOxV2ozVVK9xTiwSrYb9epAO179d3f7/u2Wo0UH +fH3T06Q3AX7hKnHn1+oiHaNaMpwQKT9VE4ZwQaDwrkO/rSvCmzIAnM057/VodGrvO b91Wix0ZxNsWQ== Precedence: bulk List-ID: X-Mailing-List: linux-rdma@vger.kernel.org From: Avihai Horon Introduce the ibv_query_gid_table verb which queries the GID tables of the given device and stores the queried data in a buffer provided by the user. If the kernel doesn't support ioctl or the needed uverbs method, the API will try to query the GID tables via sysfs. This API provides a faster way to query the GID tables of a device using a single call over ioctl, instead of multiple calls to open, close and read multiple sysfs files for a single GID table entry. Signed-off-by: Avihai Horon Signed-off-by: Yishai Hadas --- debian/libibverbs1.symbols | 1 + libibverbs/cmd_device.c | 117 +++++++++++++++++++++++++++++--- libibverbs/libibverbs.map.in | 1 + libibverbs/man/CMakeLists.txt | 1 + libibverbs/man/ibv_query_gid_table.3.md | 73 ++++++++++++++++++++ libibverbs/verbs.h | 16 +++++ 6 files changed, 199 insertions(+), 10 deletions(-) create mode 100644 libibverbs/man/ibv_query_gid_table.3.md diff --git a/debian/libibverbs1.symbols b/debian/libibverbs1.symbols index 536d543..99257de 100644 --- a/debian/libibverbs1.symbols +++ b/debian/libibverbs1.symbols @@ -11,6 +11,7 @@ libibverbs.so.1 libibverbs1 #MINVER# IBVERBS_1.11@IBVERBS_1.11 32 (symver)IBVERBS_PRIVATE_25 25 _ibv_query_gid_ex@IBVERBS_1.11 32 + _ibv_query_gid_table@IBVERBS_1.11 32 ibv_ack_async_event@IBVERBS_1.0 1.1.6 ibv_ack_async_event@IBVERBS_1.1 1.1.6 ibv_ack_cq_events@IBVERBS_1.0 1.1.6 diff --git a/libibverbs/cmd_device.c b/libibverbs/cmd_device.c index 4f85010..8bcfbb4 100644 --- a/libibverbs/cmd_device.c +++ b/libibverbs/cmd_device.c @@ -321,7 +321,7 @@ static int query_sysfs_gid_type(struct ibv_context *context, uint8_t port_num, static int query_sysfs_gid_entry(struct ibv_context *context, uint32_t port_num, uint32_t gid_index, struct ibv_gid_entry *entry, - uint32_t attr_mask) + uint32_t attr_mask, int link_layer) { enum ibv_gid_type_sysfs gid_type; struct ibv_port_attr port_attr = {}; @@ -342,14 +342,18 @@ static int query_sysfs_gid_entry(struct ibv_context *context, uint32_t port_num, return EINVAL; if (gid_type == IBV_GID_TYPE_SYSFS_IB_ROCE_V1) { - ret = ibv_query_port(context, port_num, &port_attr); - if (ret) - goto out; + if (link_layer < 0) { + ret = ibv_query_port(context, port_num, + &port_attr); + if (ret) + goto out; + + link_layer = port_attr.link_layer; + } - if (port_attr.link_layer == IBV_LINK_LAYER_INFINIBAND) { + if (link_layer == IBV_LINK_LAYER_INFINIBAND) { entry->gid_type = IBV_GID_TYPE_IB; - } else if (port_attr.link_layer == - IBV_LINK_LAYER_ETHERNET) { + } else if (link_layer == IBV_LINK_LAYER_ETHERNET) { entry->gid_type = IBV_GID_TYPE_ROCE_V1; } else { ret = EINVAL; @@ -368,8 +372,64 @@ out: return ret; } -/* Using async_event cmd_name because query_gid_ex is not in - * verbs_context_ops while async_event is and doesn't use ioctl. +static int query_gid_table_fb(struct ibv_context *context, + struct ibv_gid_entry *entries, size_t max_entries, + uint64_t *num_entries, size_t entry_size) +{ + struct ibv_device_attr dev_attr = {}; + struct ibv_port_attr port_attr = {}; + struct ibv_gid_entry entry = {}; + int attr_mask; + void *tmp; + int i, j; + int ret; + + ret = ibv_query_device(context, &dev_attr); + if (ret) + goto out; + + tmp = entries; + *num_entries = 0; + attr_mask = VERBS_QUERY_GID_ATTR_GID | VERBS_QUERY_GID_ATTR_TYPE | + VERBS_QUERY_GID_ATTR_NDEV_IFINDEX; + for (i = 0; i < dev_attr.phys_port_cnt; i++) { + ret = ibv_query_port(context, i + 1, &port_attr); + if (ret) + goto out; + + for (j = 0; j < port_attr.gid_tbl_len; j++) { + /* In case we already reached max_entries, query to some + * temp entry, in case all other entries are zeros the + * API should succceed. + */ + if (*num_entries == max_entries) + tmp = &entry; + ret = query_sysfs_gid_entry(context, i + 1, j, + tmp, + attr_mask, + port_attr.link_layer); + if (ret) + goto out; + if (is_zero_gid(&((struct ibv_gid_entry *)tmp)->gid)) + continue; + if (*num_entries == max_entries) { + ret = EINVAL; + goto out; + } + + (*num_entries)++; + tmp += entry_size; + } + } + +out: + return ret; +} + +/* Using async_event cmd_name because query_gid_ex and query_gid_table are not + * in verbs_context_ops while async_event is and doesn't use ioctl. + * If one of them is not supported, so is the other. Hence, we can use a single + * cmd_name for both of them. */ #define query_gid_kernel_cap async_event int __ibv_query_gid_ex(struct ibv_context *context, uint32_t port_num, @@ -395,7 +455,7 @@ int __ibv_query_gid_ex(struct ibv_context *context, uint32_t port_num, return EOPNOTSUPP; ret = query_sysfs_gid_entry(context, port_num, gid_index, - entry, fallback_attr_mask); + entry, fallback_attr_mask, -1); if (ret) return ret; @@ -419,3 +479,40 @@ int _ibv_query_gid_ex(struct ibv_context *context, uint32_t port_num, VERBS_QUERY_GID_ATTR_TYPE | VERBS_QUERY_GID_ATTR_NDEV_IFINDEX); } + +ssize_t _ibv_query_gid_table(struct ibv_context *context, + struct ibv_gid_entry *entries, + size_t max_entries, uint32_t flags, + size_t entry_size) +{ + DECLARE_COMMAND_BUFFER(cmdb, UVERBS_OBJECT_DEVICE, + UVERBS_METHOD_QUERY_GID_TABLE, 4); + uint64_t num_entries; + int ret; + + fill_attr_const_in(cmdb, UVERBS_ATTR_QUERY_GID_TABLE_ENTRY_SIZE, + entry_size); + fill_attr_in_uint32(cmdb, UVERBS_ATTR_QUERY_GID_TABLE_FLAGS, flags); + fill_attr_out(cmdb, UVERBS_ATTR_QUERY_GID_TABLE_RESP_ENTRIES, entries, + _array_len(entry_size, max_entries)); + fill_attr_out_ptr(cmdb, UVERBS_ATTR_QUERY_GID_TABLE_RESP_NUM_ENTRIES, + &num_entries); + + switch (execute_ioctl_fallback(context, query_gid_kernel_cap, cmdb, + &ret)) { + case TRY_WRITE: + if (flags) + return -EOPNOTSUPP; + + ret = query_gid_table_fb(context, entries, max_entries, + &num_entries, entry_size); + break; + default: + break; + } + + if (ret) + return -ret; + + return num_entries; +} diff --git a/libibverbs/libibverbs.map.in b/libibverbs/libibverbs.map.in index dae4963..7429016 100644 --- a/libibverbs/libibverbs.map.in +++ b/libibverbs/libibverbs.map.in @@ -145,6 +145,7 @@ IBVERBS_1.10 { IBVERBS_1.11 { global: _ibv_query_gid_ex; + _ibv_query_gid_table; } IBVERBS_1.10; /* If any symbols in this stanza change ABI then the entire staza gets a new symbol diff --git a/libibverbs/man/CMakeLists.txt b/libibverbs/man/CMakeLists.txt index 2dea4ff..1fb5ac1 100644 --- a/libibverbs/man/CMakeLists.txt +++ b/libibverbs/man/CMakeLists.txt @@ -58,6 +58,7 @@ rdma_man_pages( ibv_query_ece.3.md ibv_query_gid.3.md ibv_query_gid_ex.3.md + ibv_query_gid_table.3.md ibv_query_pkey.3.md ibv_query_port.3 ibv_query_qp.3 diff --git a/libibverbs/man/ibv_query_gid_table.3.md b/libibverbs/man/ibv_query_gid_table.3.md new file mode 100644 index 0000000..e10f51c --- /dev/null +++ b/libibverbs/man/ibv_query_gid_table.3.md @@ -0,0 +1,73 @@ +--- +date: 2020-04-24 +footer: libibverbs +header: "Libibverbs Programmer's Manual" +layout: page +license: 'Licensed under the OpenIB.org BSD license (FreeBSD Variant) - See COPYING.md' +section: 3 +title: IBV_QUERY_GID_TABLE +--- + +# NAME + +ibv_query_gid_table - query an InfiniBand device's GID table + +# SYNOPSIS + +```c +#include + +ssize_t ibv_query_gid_table(struct ibv_context *context, + struct ibv_gid_entry *entries, + size_t max_entries, + uint32_t flags); +``` + +# DESCRIPTION + +**ibv_query_gid_table()** returns the valid GID table entries of the RDMA +device context *context* at the pointer *entries*. + +A caller must allocate *entries* array for the GID table entries it +desires to query. This API returns only valid GID table entries. + +A caller must pass non zero number of entries at *max_entries* that corresponds +to the size of *entries* array. + +*entries* array must be allocated such that it can contain all the valid +GID table entries of the device. If there are more valid GID entries than +the provided value of *max_entries* and *entries* array, the call will fail. +For example, if a RDMA device *context* has a total of 10 valid +GID entries, *entries* should be allocated for at least 10 entries, and +*max_entries* should be set appropriately. + +# ARGUMENTS + +*context* +: The context of the device to query. + +*entries* +: Array of ibv_gid_entry structs where the GID entries are returned. + Please see **ibv_query_gid_ex**(3) man page for *ibv_gid_entry*. + +*max_entries* +: Maximum number of entries that can be returned. + +*flags* +: Extra fields to query post *entries->ndev_ifindex*, for now must be 0. + +# RETURN VALUE + +**ibv_query_gid_table()** returns the number of entries that were read on success or negative errno value on error. +Number of entries returned is <= max_entries. + +# SEE ALSO + +**ibv_open_device**(3), +**ibv_query_device**(3), +**ibv_query_port**(3), +**ibv_query_gid_ex**(3) + +# AUTHOR + +Parav Pandit diff --git a/libibverbs/verbs.h b/libibverbs/verbs.h index e5bf900..caf626c 100644 --- a/libibverbs/verbs.h +++ b/libibverbs/verbs.h @@ -43,6 +43,7 @@ #include #include #include +#include #include #ifdef __cplusplus @@ -2359,6 +2360,21 @@ static inline int ibv_query_gid_ex(struct ibv_context *context, sizeof(*entry)); } +ssize_t _ibv_query_gid_table(struct ibv_context *context, + struct ibv_gid_entry *entries, size_t max_entries, + uint32_t flags, size_t entry_size); + +/* + * ibv_query_gid_table - Get all valid GID table entries + */ +static inline ssize_t ibv_query_gid_table(struct ibv_context *context, + struct ibv_gid_entry *entries, + size_t max_entries, uint32_t flags) +{ + return _ibv_query_gid_table(context, entries, max_entries, flags, + sizeof(*entries)); +} + /** * ibv_query_pkey - Get a P_Key table entry */ From patchwork Wed Sep 23 10:27:01 2020 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Yishai Hadas X-Patchwork-Id: 11794775 Return-Path: Received: from mail.kernel.org (pdx-korg-mail-1.web.codeaurora.org [172.30.200.123]) by pdx-korg-patchwork-2.web.codeaurora.org (Postfix) with ESMTP id 50EC6112C for ; Wed, 23 Sep 2020 10:27:29 +0000 (UTC) Received: from vger.kernel.org (vger.kernel.org [23.128.96.18]) by mail.kernel.org (Postfix) with ESMTP id 2F65623119 for ; Wed, 23 Sep 2020 10:27:29 +0000 (UTC) Authentication-Results: mail.kernel.org; dkim=pass (2048-bit key) header.d=nvidia.com header.i=@nvidia.com header.b="PO1o9mZC" Received: (majordomo@vger.kernel.org) by vger.kernel.org via listexpand id S1726514AbgIWK12 (ORCPT ); Wed, 23 Sep 2020 06:27:28 -0400 Received: from hqnvemgate26.nvidia.com ([216.228.121.65]:13522 "EHLO hqnvemgate26.nvidia.com" rhost-flags-OK-OK-OK-OK) by vger.kernel.org with ESMTP id S1726310AbgIWK12 (ORCPT ); Wed, 23 Sep 2020 06:27:28 -0400 Received: from hqmail.nvidia.com (Not Verified[216.228.121.13]) by hqnvemgate26.nvidia.com (using TLS: TLSv1.2, AES256-SHA) id ; Wed, 23 Sep 2020 03:27:15 -0700 Received: from HQMAIL109.nvidia.com (172.20.187.15) by HQMAIL101.nvidia.com (172.20.187.10) with Microsoft SMTP Server (TLS) id 15.0.1473.3; Wed, 23 Sep 2020 10:27:27 +0000 Received: from dev-l-vrt-092.mtl.labs.mlnx (172.20.13.39) by mail.nvidia.com (172.20.187.15) with Microsoft SMTP Server id 15.0.1473.3 via Frontend Transport; Wed, 23 Sep 2020 10:27:25 +0000 From: Yishai Hadas To: CC: , , , , Edward Srouji Subject: [PATCH V1 rdma-core 7/8] pyverbs: Add query_gid_table and query_gid_ex methods Date: Wed, 23 Sep 2020 13:27:01 +0300 Message-ID: <20200923102702.590008-8-yishaih@nvidia.com> X-Mailer: git-send-email 2.26.2 In-Reply-To: <20200923102702.590008-1-yishaih@nvidia.com> References: <20200923102702.590008-1-yishaih@nvidia.com> MIME-Version: 1.0 DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=nvidia.com; s=n1; t=1600856835; bh=pk68iv0MABgMawuRMZ46be1k1jyb/nHS9lvZQ+5qt+0=; h=From:To:CC:Subject:Date:Message-ID:X-Mailer:In-Reply-To: References:MIME-Version:Content-Transfer-Encoding:Content-Type; b=PO1o9mZClXyfXZr8oClSAaBl8yCkMP1Pv3YK2mDAanRm4xKv3TvptAe2SEjYu0XvU jvxHAnST+QHaX+YTQT4yBakCKh5miklK0TIe6VfUQ61AZts0JsMjyUr6sWr/IPA+Iq nIbLRSwetOn1zujkscZrsEuPDX/ZUyKm9Hwa7LIBKizjJFtig6CTgK9MxqD0C3Jmoe 0I0okmUTDJVgtQ3hxthJqX9zEgS1+RZUNDF9HKEoM9quaT5pXWI8A7SNnFQvlW393W 8JwjTiYS7x5Ehg8jHs/FJWjdqnZjrHdh6wRQNxUkIpbpLpkani0NGfKCbbKSeU5KSz hazi09vKc0BOw== Precedence: bulk List-ID: X-Mailing-List: linux-rdma@vger.kernel.org From: Avihai Horon Add two new methods to Context class: query_gid_table and query_gid_ex. query_gid_table queries all GID tables of the device and returns a list of GIDEntry objects containing all valid GID entries. query_gid_ex queries the GID table of the given port in the given index and returns a GIDEntry object. Signed-off-by: Avihai Horon Signed-off-by: Edward Srouji --- pyverbs/device.pxd | 3 ++ pyverbs/device.pyx | 106 +++++++++++++++++++++++++++++++++++++++++++ pyverbs/libibverbs.pxd | 13 ++++++ pyverbs/libibverbs_enums.pxd | 5 ++ 4 files changed, 127 insertions(+) diff --git a/pyverbs/device.pxd b/pyverbs/device.pxd index 73328d3..0519c4b 100755 --- a/pyverbs/device.pxd +++ b/pyverbs/device.pxd @@ -64,3 +64,6 @@ cdef class DM(PyverbsCM): cdef class PortAttr(PyverbsObject): cdef v.ibv_port_attr attr + +cdef class GIDEntry(PyverbsObject): + cdef v.ibv_gid_entry entry diff --git a/pyverbs/device.pyx b/pyverbs/device.pyx index c1323cd..b16d6d0 100755 --- a/pyverbs/device.pyx +++ b/pyverbs/device.pyx @@ -26,6 +26,8 @@ from libc.stdlib cimport free, malloc from libc.string cimport memset from libc.stdint cimport uint64_t from libc.stdint cimport uint16_t +from libc.stdint cimport uint32_t +from pyverbs.utils import gid_str cdef extern from 'endian.h': unsigned long be64toh(unsigned long host_64bits); @@ -240,6 +242,53 @@ cdef class Context(PyverbsCM): format(p=port_num), rc) return port_attrs + def query_gid_table(self, size_t max_entries, uint32_t flags=0): + """ + Queries the GID tables of the device for at most entries + and returns them. + :param max_entries: Maximum number of GID entries to retrieve + :param flags: Specifies new extra members of struct ibv_gid_entry to + query + :return: List of GIDEntry objects on success + """ + cdef v.ibv_gid_entry *entries + cdef v.ibv_gid_entry entry + + entries = malloc(max_entries * + sizeof(v.ibv_gid_entry)) + rc = v.ibv_query_gid_table(self.context, entries, max_entries, flags) + if rc < 0: + raise PyverbsRDMAError('Failed to query gid tables of the device', + rc) + gid_entries = [] + for i in range(rc): + entry = entries[i] + gid_entries.append(GIDEntry(entry.gid._global.subnet_prefix, + entry.gid._global.interface_id, entry.gid_index, + entry.port_num, entry.gid_type, + entry.ndev_ifindex)) + free(entries) + return gid_entries + + def query_gid_ex(self, uint32_t port_num, uint32_t gid_index, + uint32_t flags=0): + """ + Queries the GID table of port in index , and + returns the GID entry. + :param port_num: The port number to query + :param gid_index: The index in the GID table to query + :param flags: Specifies new extra members of struct ibv_gid_entry to + query + :return: GIDEntry object on success + """ + entry = GIDEntry() + rc = v.ibv_query_gid_ex(self.context, port_num, gid_index, + &entry.entry, flags) + if rc != 0: + raise PyverbsRDMAError(f'Failed to query gid table of port '\ + f'{port_num} in index {gid_index}', rc) + return entry + cdef add_ref(self, obj): if isinstance(obj, PD): self.pds.add(obj) @@ -816,6 +865,63 @@ cdef class PortAttr(PyverbsObject): print_format.format('Flags', self.attr.flags) +cdef class GIDEntry(PyverbsObject): + def __init__(self, subnet_prefix=0, interface_id=0, gid_index=0, + port_num=0, gid_type=0, ndev_ifindex=0): + super().__init__() + self.entry.gid._global.subnet_prefix = subnet_prefix + self.entry.gid._global.interface_id = interface_id + self.entry.gid_index = gid_index + self.entry.port_num = port_num + self.entry.gid_type = gid_type + self.entry.ndev_ifindex = ndev_ifindex + + @property + def gid_subnet_prefix(self): + return self.entry.gid._global.subnet_prefix + + @property + def gid_interface_id(self): + return self.entry.gid._global.interface_id + + @property + def gid_index(self): + return self.entry.gid_index + + @property + def port_num(self): + return self.entry.port_num + + @property + def gid_type(self): + return self.entry.gid_type + + @property + def ndev_ifindex(self): + return self.entry.ndev_ifindex + + def gid_str(self): + return gid_str(self.gid_subnet_prefix, self.gid_interface_id) + + def __str__(self): + print_format = '{:<24}: {:<20}\n' + return print_format.format('GID', self.gid_str()) +\ + print_format.format('GID Index', self.gid_index) +\ + print_format.format('Port number', self.port_num) +\ + print_format.format('GID type', translate_gid_type( + self.gid_type)) +\ + print_format.format('Ndev ifindex', self.ndev_ifindex) + + +def translate_gid_type(gid_type): + types = {e.IBV_GID_TYPE_IB: 'IB', e.IBV_GID_TYPE_ROCE_V1: 'RoCEv1', + e.IBV_GID_TYPE_ROCE_V2: 'RoCEv2'} + try: + return types[gid_type] + except KeyError: + return f'Unknown gid_type ({gid_type})' + + def guid_format(num): """ Get GUID representation of the given number, including change of endianness. diff --git a/pyverbs/libibverbs.pxd b/pyverbs/libibverbs.pxd index c84b9fc..6fbba54 100755 --- a/pyverbs/libibverbs.pxd +++ b/pyverbs/libibverbs.pxd @@ -483,6 +483,13 @@ cdef extern from 'infiniband/verbs.h': uint32_t options uint32_t comp_mask + cdef struct ibv_gid_entry: + ibv_gid gid + uint32_t gid_index + uint32_t port_num + uint32_t gid_type + uint32_t ndev_ifindex + ibv_device **ibv_get_device_list(int *n) int ibv_get_device_index(ibv_device *device); void ibv_free_device_list(ibv_device **list) @@ -613,6 +620,12 @@ cdef extern from 'infiniband/verbs.h': void ibv_unimport_mr(ibv_mr *mr) ibv_pd *ibv_import_pd(ibv_context *context, uint32_t handle) void ibv_unimport_pd(ibv_pd *pd) + int ibv_query_gid_ex(ibv_context *context, uint32_t port_num, + uint32_t gid_index, ibv_gid_entry *entry, + uint32_t flags) + ssize_t ibv_query_gid_table(ibv_context *context, + ibv_gid_entry *entries, size_t max_entries, + uint32_t flags) cdef extern from 'infiniband/driver.h': diff --git a/pyverbs/libibverbs_enums.pxd b/pyverbs/libibverbs_enums.pxd index 83ca516..a5c07b3 100755 --- a/pyverbs/libibverbs_enums.pxd +++ b/pyverbs/libibverbs_enums.pxd @@ -427,6 +427,11 @@ cdef extern from '': cdef void *IBV_ALLOCATOR_USE_DEFAULT + cpdef enum ibv_gid_type: + IBV_GID_TYPE_IB + IBV_GID_TYPE_ROCE_V1 + IBV_GID_TYPE_ROCE_V2 + cdef extern from "": cdef unsigned long long IBV_ADVISE_MR_ADVICE_PREFETCH From patchwork Wed Sep 23 10:27:02 2020 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Yishai Hadas X-Patchwork-Id: 11794777 Return-Path: Received: from mail.kernel.org (pdx-korg-mail-1.web.codeaurora.org [172.30.200.123]) by pdx-korg-patchwork-2.web.codeaurora.org (Postfix) with ESMTP id 7105D112C for ; Wed, 23 Sep 2020 10:27:31 +0000 (UTC) Received: from vger.kernel.org (vger.kernel.org [23.128.96.18]) by mail.kernel.org (Postfix) with ESMTP id 559A323119 for ; Wed, 23 Sep 2020 10:27:31 +0000 (UTC) Authentication-Results: mail.kernel.org; dkim=pass (2048-bit key) header.d=nvidia.com header.i=@nvidia.com header.b="dTgx2K2H" Received: (majordomo@vger.kernel.org) by vger.kernel.org via listexpand id S1726550AbgIWK1a (ORCPT ); Wed, 23 Sep 2020 06:27:30 -0400 Received: from hqnvemgate26.nvidia.com ([216.228.121.65]:13525 "EHLO hqnvemgate26.nvidia.com" rhost-flags-OK-OK-OK-OK) by vger.kernel.org with ESMTP id S1726310AbgIWK1a (ORCPT ); Wed, 23 Sep 2020 06:27:30 -0400 Received: from hqmail.nvidia.com (Not Verified[216.228.121.13]) by hqnvemgate26.nvidia.com (using TLS: TLSv1.2, AES256-SHA) id ; Wed, 23 Sep 2020 03:27:17 -0700 Received: from HQMAIL109.nvidia.com (172.20.187.15) by HQMAIL111.nvidia.com (172.20.187.18) with Microsoft SMTP Server (TLS) id 15.0.1473.3; Wed, 23 Sep 2020 10:27:29 +0000 Received: from dev-l-vrt-092.mtl.labs.mlnx (172.20.13.39) by mail.nvidia.com (172.20.187.15) with Microsoft SMTP Server id 15.0.1473.3 via Frontend Transport; Wed, 23 Sep 2020 10:27:28 +0000 From: Yishai Hadas To: CC: , , , , Edward Srouji Subject: [PATCH V1 rdma-core 8/8] tests: Add tests for ibv_query_gid_table and ibv_query_gid_ex Date: Wed, 23 Sep 2020 13:27:02 +0300 Message-ID: <20200923102702.590008-9-yishaih@nvidia.com> X-Mailer: git-send-email 2.26.2 In-Reply-To: <20200923102702.590008-1-yishaih@nvidia.com> References: <20200923102702.590008-1-yishaih@nvidia.com> MIME-Version: 1.0 DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=nvidia.com; s=n1; t=1600856837; bh=umT9RbzAGBI/oJU2rcQ1nk5EPbDbYPDi9ejy+OObqs8=; h=From:To:CC:Subject:Date:Message-ID:X-Mailer:In-Reply-To: References:MIME-Version:Content-Transfer-Encoding:Content-Type; b=dTgx2K2H3Esw0uxvf0JUPieBW08FOoSVuFutOIaGZTuEq9sJFiK2MQUXxGkZ6/hNN 6Ez8QAtDdNUoCfLoV9w9ncGgePHqC3MxGxuCyHm9Kl5YG51kdeWMGUePTWvISFWVY3 BNkox8Lv/BM0fL8dZ2IAc41SSrIsMT7Z850twqViLtBvfsBtULWwmpoW4MBL3nArpz 0FRjPKzH8OY9LA1U90e97zJgjtcev/zY3ruRkVbV4EBvuF0cGOgfabD7sbDfRbVl6l jL+f3NSYRCwSmg7PK5HJzzfmZNfLdAYidtiuLnVB3YZTRYcbRgpWcDVAZf7BnpohOg zDISyzkbxelfQ== Precedence: bulk List-ID: X-Mailing-List: linux-rdma@vger.kernel.org From: Avihai Horon Add a test for ibv_query_gid_table and another one for ibv_query_gid_ex. Signed-off-by: Avihai Horon Signed-off-by: Edward Srouji --- tests/test_device.py | 32 ++++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/tests/test_device.py b/tests/test_device.py index 1eb1a81..94c0e40 100644 --- a/tests/test_device.py +++ b/tests/test_device.py @@ -6,6 +6,7 @@ Test module for pyverbs' device module. import unittest import resource import random +import errno from pyverbs.pyverbs_error import PyverbsError, PyverbsRDMAError from tests.base import PyverbsAPITestCase @@ -80,6 +81,37 @@ class DeviceTest(PyverbsAPITestCase): with d.Context(name=dev.name.decode()) as ctx: ctx.query_gid(port_num=1, index=0) + def test_query_gid_table(self): + """ + Test ibv_query_gid_table() + """ + devs = self.get_device_list() + with d.Context(name=devs[0].name.decode()) as ctx: + device_attr = ctx.query_device() + port_attr = ctx.query_port(1) + max_entries = device_attr.phys_port_cnt * port_attr.gid_tbl_len + try: + ctx.query_gid_table(max_entries) + except PyverbsRDMAError as ex: + if ex.error_code in [errno.EOPNOTSUPP, errno.EPROTONOSUPPORT]: + raise unittest.SkipTest('ibv_query_gid_table is not'\ + ' supported on this device') + raise ex + + def test_query_gid_ex(self): + """ + Test ibv_query_gid_ex() + """ + devs = self.get_device_list() + with d.Context(name=devs[0].name.decode()) as ctx: + try: + ctx.query_gid_ex(port_num=1, gid_index=0) + except PyverbsRDMAError as ex: + if ex.error_code in [errno.EOPNOTSUPP, errno.EPROTONOSUPPORT]: + raise unittest.SkipTest('ibv_query_gid_ex is not'\ + ' supported on this device') + raise ex + @staticmethod def verify_device_attr(attr, device): """