From patchwork Sun Feb 24 13:06:20 2019 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Noa Osherovich X-Patchwork-Id: 10827925 Return-Path: Received: from mail.wl.linuxfoundation.org (pdx-wl-mail.web.codeaurora.org [172.30.200.125]) by pdx-korg-patchwork-2.web.codeaurora.org (Postfix) with ESMTP id 5B4501575 for ; Sun, 24 Feb 2019 13:08:54 +0000 (UTC) Received: from mail.wl.linuxfoundation.org (localhost [127.0.0.1]) by mail.wl.linuxfoundation.org (Postfix) with ESMTP id 3BB512B4FE for ; Sun, 24 Feb 2019 13:08:54 +0000 (UTC) Received: by mail.wl.linuxfoundation.org (Postfix, from userid 486) id 2909F2B50B; Sun, 24 Feb 2019 13:08:54 +0000 (UTC) X-Spam-Checker-Version: SpamAssassin 3.3.1 (2010-03-16) on pdx-wl-mail.web.codeaurora.org X-Spam-Level: X-Spam-Status: No, score=-7.9 required=2.0 tests=BAYES_00,MAILING_LIST_MULTI, RCVD_IN_DNSWL_HI,UNPARSEABLE_RELAY autolearn=ham version=3.3.1 Received: from vger.kernel.org (vger.kernel.org [209.132.180.67]) by mail.wl.linuxfoundation.org (Postfix) with ESMTP id C8C462846C for ; Sun, 24 Feb 2019 13:08:49 +0000 (UTC) Received: (majordomo@vger.kernel.org) by vger.kernel.org via listexpand id S1728085AbfBXNIs (ORCPT ); Sun, 24 Feb 2019 08:08:48 -0500 Received: from mail-il-dmz.mellanox.com ([193.47.165.129]:50435 "EHLO mellanox.co.il" rhost-flags-OK-OK-OK-FAIL) by vger.kernel.org with ESMTP id S1726865AbfBXNIs (ORCPT ); Sun, 24 Feb 2019 08:08:48 -0500 Received: from Internal Mail-Server by MTLPINE1 (envelope-from noaos@mellanox.com) with ESMTPS (AES256-SHA encrypted); 24 Feb 2019 15:06:48 +0200 Received: from reg-l-vrt-059-009.mtl.labs.mlnx (reg-l-vrt-059-009.mtl.labs.mlnx [10.135.59.9]) by labmailer.mlnx (8.13.8/8.13.8) with ESMTP id x1OD6mfV026210; Sun, 24 Feb 2019 15:06:48 +0200 From: Noa Osherovich To: leonro@mellanox.com, jgg@mellanox.com, dledford@redhat.com Cc: linux-rdma@vger.kernel.org, Noa Osherovich Subject: [PATCH rdma-core 01/19] pyverbs: Rename Gid class Date: Sun, 24 Feb 2019 15:06:20 +0200 Message-Id: <20190224130638.31848-2-noaos@mellanox.com> X-Mailer: git-send-email 2.17.2 In-Reply-To: <20190224130638.31848-1-noaos@mellanox.com> References: <20190224130638.31848-1-noaos@mellanox.com> Sender: linux-rdma-owner@vger.kernel.org Precedence: bulk List-ID: X-Mailing-List: linux-rdma@vger.kernel.org X-Virus-Scanned: ClamAV using ClamSMTP Pyverbs will be using an all-capital class names conventions for acronyms, fix Gid to match that convention. This means breaking pyverbs' API, but since it's not used yet and currently offers very little functionality, it shouldn't make a difference. Signed-off-by: Noa Osherovich --- pyverbs/addr.pxd | 2 +- pyverbs/addr.pyx | 2 +- pyverbs/device.pyx | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/pyverbs/addr.pxd b/pyverbs/addr.pxd index 39039379d517..313cd68fb0d2 100644 --- a/pyverbs/addr.pxd +++ b/pyverbs/addr.pxd @@ -5,5 +5,5 @@ from .base cimport PyverbsObject from pyverbs cimport libibverbs as v -cdef class Gid(PyverbsObject): +cdef class GID(PyverbsObject): cdef v.ibv_gid gid diff --git a/pyverbs/addr.pyx b/pyverbs/addr.pyx index 986521b90e82..0f171e83ea21 100644 --- a/pyverbs/addr.pyx +++ b/pyverbs/addr.pyx @@ -9,7 +9,7 @@ cdef extern from 'endian.h': unsigned long be64toh(unsigned long host_64bits) -cdef class Gid(PyverbsObject): +cdef class GID(PyverbsObject): """ GID class represents ibv_gid. It enables user to query for GIDs values. """ diff --git a/pyverbs/device.pyx b/pyverbs/device.pyx index a50bd2349445..3fad6af41ecc 100644 --- a/pyverbs/device.pyx +++ b/pyverbs/device.pyx @@ -10,7 +10,7 @@ from .pyverbs_error import PyverbsRDMAError from .pyverbs_error import PyverbsUserError from pyverbs.base import PyverbsRDMAErrno cimport pyverbs.libibverbs as v -from pyverbs.addr cimport Gid +from pyverbs.addr cimport GID cdef extern from 'errno.h': int errno @@ -134,7 +134,7 @@ cdef class Context(PyverbsObject): return dev_attr def query_gid(self, unsigned int port_num, int index): - gid = Gid() + gid = GID() rc = v.ibv_query_gid(self.context, port_num, index, &gid.gid) if rc != 0: raise PyverbsRDMAError('Failed to query gid {idx} of port {port}'. From patchwork Sun Feb 24 13:06:21 2019 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Noa Osherovich X-Patchwork-Id: 10827937 Return-Path: Received: from mail.wl.linuxfoundation.org (pdx-wl-mail.web.codeaurora.org [172.30.200.125]) by pdx-korg-patchwork-2.web.codeaurora.org (Postfix) with ESMTP id ED27213B5 for ; Sun, 24 Feb 2019 13:08:55 +0000 (UTC) Received: from mail.wl.linuxfoundation.org (localhost [127.0.0.1]) by mail.wl.linuxfoundation.org (Postfix) with ESMTP id DD1A02846C for ; Sun, 24 Feb 2019 13:08:55 +0000 (UTC) Received: by mail.wl.linuxfoundation.org (Postfix, from userid 486) id D21192B4AD; Sun, 24 Feb 2019 13:08:55 +0000 (UTC) X-Spam-Checker-Version: SpamAssassin 3.3.1 (2010-03-16) on pdx-wl-mail.web.codeaurora.org X-Spam-Level: X-Spam-Status: No, score=-7.9 required=2.0 tests=BAYES_00,MAILING_LIST_MULTI, RCVD_IN_DNSWL_HI,UNPARSEABLE_RELAY autolearn=ham version=3.3.1 Received: from vger.kernel.org (vger.kernel.org [209.132.180.67]) by mail.wl.linuxfoundation.org (Postfix) with ESMTP id 8F8552B4A6 for ; Sun, 24 Feb 2019 13:08:50 +0000 (UTC) Received: (majordomo@vger.kernel.org) by vger.kernel.org via listexpand id S1727594AbfBXNIt (ORCPT ); Sun, 24 Feb 2019 08:08:49 -0500 Received: from mail-il-dmz.mellanox.com ([193.47.165.129]:50455 "EHLO mellanox.co.il" rhost-flags-OK-OK-OK-FAIL) by vger.kernel.org with ESMTP id S1728059AbfBXNIt (ORCPT ); Sun, 24 Feb 2019 08:08:49 -0500 Received: from Internal Mail-Server by MTLPINE1 (envelope-from noaos@mellanox.com) with ESMTPS (AES256-SHA encrypted); 24 Feb 2019 15:06:48 +0200 Received: from reg-l-vrt-059-009.mtl.labs.mlnx (reg-l-vrt-059-009.mtl.labs.mlnx [10.135.59.9]) by labmailer.mlnx (8.13.8/8.13.8) with ESMTP id x1OD6mfW026210; Sun, 24 Feb 2019 15:06:48 +0200 From: Noa Osherovich To: leonro@mellanox.com, jgg@mellanox.com, dledford@redhat.com Cc: linux-rdma@vger.kernel.org, Noa Osherovich Subject: [PATCH rdma-core 02/19] pyverbs: Unify close functions Date: Sun, 24 Feb 2019 15:06:21 +0200 Message-Id: <20190224130638.31848-3-noaos@mellanox.com> X-Mailer: git-send-email 2.17.2 In-Reply-To: <20190224130638.31848-1-noaos@mellanox.com> References: <20190224130638.31848-1-noaos@mellanox.com> Sender: linux-rdma-owner@vger.kernel.org Precedence: bulk List-ID: X-Mailing-List: linux-rdma@vger.kernel.org X-Virus-Scanned: ClamAV using ClamSMTP Instead of a declaring both cdef and def function, use a single cpdef close(). The main drawback of such an apporoach is a potential performance degradation, but it will be visible only during objects teardown, which don't have performance constraints anyway. Signed-off-by: Noa Osherovich --- pyverbs/device.pxd | 2 +- pyverbs/device.pyx | 11 ++--------- 2 files changed, 3 insertions(+), 10 deletions(-) diff --git a/pyverbs/device.pxd b/pyverbs/device.pxd index 31d243a80dd5..7a6489da11bf 100644 --- a/pyverbs/device.pxd +++ b/pyverbs/device.pxd @@ -8,7 +8,7 @@ cimport pyverbs.libibverbs as v cdef class Context(PyverbsObject): cdef v.ibv_context *context cdef object name - cdef _close(self) + cpdef close(self) cdef class DeviceAttr(PyverbsObject): cdef v.ibv_device_attr dev_attr diff --git a/pyverbs/device.pyx b/pyverbs/device.pyx index 3fad6af41ecc..cb3baed38afa 100644 --- a/pyverbs/device.pyx +++ b/pyverbs/device.pyx @@ -102,16 +102,9 @@ cdef class Context(PyverbsObject): Closes the inner IB device. :return: None """ - self._close() + self.close() - def close(self): - """ - Closes the inner IB device. - :return: None - """ - self._close() - - cdef _close(self): + cpdef close(self): self.logger.debug('Closing Context') if self.context != NULL: rc = v.ibv_close_device(self.context) From patchwork Sun Feb 24 13:06:22 2019 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Noa Osherovich X-Patchwork-Id: 10827933 Return-Path: Received: from mail.wl.linuxfoundation.org (pdx-wl-mail.web.codeaurora.org [172.30.200.125]) by pdx-korg-patchwork-2.web.codeaurora.org (Postfix) with ESMTP id 73F3E188D for ; Sun, 24 Feb 2019 13:08:55 +0000 (UTC) Received: from mail.wl.linuxfoundation.org (localhost [127.0.0.1]) by mail.wl.linuxfoundation.org (Postfix) with ESMTP id 62A062B4AD for ; Sun, 24 Feb 2019 13:08:55 +0000 (UTC) Received: by mail.wl.linuxfoundation.org (Postfix, from userid 486) id 574002B525; Sun, 24 Feb 2019 13:08:55 +0000 (UTC) X-Spam-Checker-Version: SpamAssassin 3.3.1 (2010-03-16) on pdx-wl-mail.web.codeaurora.org X-Spam-Level: X-Spam-Status: No, score=-7.9 required=2.0 tests=BAYES_00,MAILING_LIST_MULTI, RCVD_IN_DNSWL_HI,UNPARSEABLE_RELAY autolearn=ham version=3.3.1 Received: from vger.kernel.org (vger.kernel.org [209.132.180.67]) by mail.wl.linuxfoundation.org (Postfix) with ESMTP id F13452B4AD for ; Sun, 24 Feb 2019 13:08:50 +0000 (UTC) Received: (majordomo@vger.kernel.org) by vger.kernel.org via listexpand id S1728204AbfBXNIu (ORCPT ); Sun, 24 Feb 2019 08:08:50 -0500 Received: from mail-il-dmz.mellanox.com ([193.47.165.129]:50424 "EHLO mellanox.co.il" rhost-flags-OK-OK-OK-FAIL) by vger.kernel.org with ESMTP id S1726835AbfBXNIu (ORCPT ); Sun, 24 Feb 2019 08:08:50 -0500 Received: from Internal Mail-Server by MTLPINE1 (envelope-from noaos@mellanox.com) with ESMTPS (AES256-SHA encrypted); 24 Feb 2019 15:06:48 +0200 Received: from reg-l-vrt-059-009.mtl.labs.mlnx (reg-l-vrt-059-009.mtl.labs.mlnx [10.135.59.9]) by labmailer.mlnx (8.13.8/8.13.8) with ESMTP id x1OD6mfX026210; Sun, 24 Feb 2019 15:06:48 +0200 From: Noa Osherovich To: leonro@mellanox.com, jgg@mellanox.com, dledford@redhat.com Cc: linux-rdma@vger.kernel.org, Noa Osherovich Subject: [PATCH rdma-core 03/19] pyverbs: Use new-style properties Date: Sun, 24 Feb 2019 15:06:22 +0200 Message-Id: <20190224130638.31848-4-noaos@mellanox.com> X-Mailer: git-send-email 2.17.2 In-Reply-To: <20190224130638.31848-1-noaos@mellanox.com> References: <20190224130638.31848-1-noaos@mellanox.com> Sender: linux-rdma-owner@vger.kernel.org Precedence: bulk List-ID: X-Mailing-List: linux-rdma@vger.kernel.org X-Virus-Scanned: ClamAV using ClamSMTP When declaring properties, declare them the same way as ordinary Python code instead of using deprecated legacy syntax. Signed-off-by: Noa Osherovich --- pyverbs/addr.pyx | 55 +++++------ pyverbs/device.pyx | 234 ++++++++++++++++++++++----------------------- 2 files changed, 145 insertions(+), 144 deletions(-) diff --git a/pyverbs/addr.pyx b/pyverbs/addr.pyx index 0f171e83ea21..5713f9392d0f 100644 --- a/pyverbs/addr.pyx +++ b/pyverbs/addr.pyx @@ -13,34 +13,35 @@ cdef class GID(PyverbsObject): """ GID class represents ibv_gid. It enables user to query for GIDs values. """ - property gid: - def __get__(self): - """ - Expose the inner GID - :return: A GID string in an 8 words format: - 'xxxx:xxxx:xxxx:xxxx:xxxx:xxxx:xxxx:xxxx' - """ - return self.__str__() - def __set__(self, val): - """ - Sets the inner GID - :param val: A GID string in an 8 words format: - 'xxxx:xxxx:xxxx:xxxx:xxxx:xxxx:xxxx:xxxx' - :return: None - """ - val = val.split(':') - if len(val) != 8: - raise PyverbsUserError("Invalid GID value ({val})".format(val=val)) - if any([len(v) != 4 for v in val]): - raise PyverbsUserError("Invalid GID value ({val})".format(val=val)) - val_int = int("".join(val), 16) - vals = [] - for i in range(8): - vals.append(val[i][0:2]) - vals.append(val[i][2:4]) + @property + def gid(self): + """ + Expose the inner GID + :return: A GID string in an 8 words format: + 'xxxx:xxxx:xxxx:xxxx:xxxx:xxxx:xxxx:xxxx' + """ + return self.__str__() + @gid.setter + def gid(self, val): + """ + Sets the inner GID + :param val: A GID string in an 8 words format: + 'xxxx:xxxx:xxxx:xxxx:xxxx:xxxx:xxxx:xxxx' + :return: None + """ + val = val.split(':') + if len(val) != 8: + raise PyverbsUserError("Invalid GID value ({val})".format(val=val)) + if any([len(v) != 4 for v in val]): + raise PyverbsUserError("Invalid GID value ({val})".format(val=val)) + val_int = int("".join(val), 16) + vals = [] + for i in range(8): + vals.append(val[i][0:2]) + vals.append(val[i][2:4]) - for i in range(16): - self.gid.raw[i] = int(vals[i],16) + for i in range(16): + self.gid.raw[i] = int(vals[i],16) def __str__(self): hex_values = '%016x%016x' % (be64toh(self.gid._global.subnet_prefix), diff --git a/pyverbs/device.pyx b/pyverbs/device.pyx index cb3baed38afa..0264a4f0bddd 100644 --- a/pyverbs/device.pyx +++ b/pyverbs/device.pyx @@ -141,123 +141,123 @@ cdef class DeviceAttr(PyverbsObject): properties (read only) and also provides an __str__() function for readability. """ - property fw_version: - def __get__(self): - return self.dev_attr.fw_ver.decode() - property node_guid: - def __get__(self): - return self.dev_attr.node_guid - property sys_image_guid: - def __get__(self): - return self.dev_attr.sys_image_guid - property max_mr_size: - def __get__(self): - return self.dev_attr.max_mr_size - property page_size_cap: - def __get__(self): - return self.dev_attr.page_size_cap - property vendor_id: - def __get__(self): - return self.dev_attr.vendor_id - property vendor_part_id: - def __get__(self): - return self.dev_attr.vendor_part_id - property hw_ver: - def __get__(self): - return self.dev_attr.hw_ver - property max_qp: - def __get__(self): - return self.dev_attr.max_qp - property max_qp_wr: - def __get__(self): - return self.dev_attr.max_qp_wr - property device_cap_flags: - def __get__(self): - return self.dev_attr.device_cap_flags - property max_sge: - def __get__(self): - return self.dev_attr.max_sge - property max_sge_rd: - def __get__(self): - return self.dev_attr.max_sge_rd - property max_cq: - def __get__(self): - return self.dev_attr.max_cq - property max_cqe: - def __get__(self): - return self.dev_attr.max_cqe - property max_mr: - def __get__(self): - return self.dev_attr.max_mr - property max_pd: - def __get__(self): - return self.dev_attr.max_pd - property max_qp_rd_atom: - def __get__(self): - return self.dev_attr.max_qp_rd_atom - property max_ee_rd_atom: - def __get__(self): - return self.dev_attr.max_ee_rd_atom - property max_res_rd_atom: - def __get__(self): - return self.dev_attr.max_res_rd_atom - property max_qp_init_rd_atom: - def __get__(self): - return self.dev_attr.max_qp_init_rd_atom - property max_ee_init_rd_atom: - def __get__(self): - return self.dev_attr.max_ee_init_rd_atom - property atomic_caps: - def __get__(self): - return self.dev_attr.atomic_cap - property max_ee: - def __get__(self): - return self.dev_attr.max_ee - property max_rdd: - def __get__(self): - return self.dev_attr.max_rdd - property max_mw: - def __get__(self): - return self.dev_attr.max_mw - property max_raw_ipv6_qps: - def __get__(self): - return self.dev_attr.max_raw_ipv6_qp - property max_raw_ethy_qp: - def __get__(self): - return self.dev_attr.max_raw_ethy_qp - property max_mcast_grp: - def __get__(self): - return self.dev_attr.max_mcast_grp - property max_mcast_qp_attach: - def __get__(self): - return self.dev_attr.max_mcast_qp_attach - property max_ah: - def __get__(self): - return self.dev_attr.max_ah - property max_fmr: - def __get__(self): - return self.dev_attr.max_fmr - property max_map_per_fmr: - def __get__(self): - return self.dev_attr.max_map_per_fmr - property max_srq: - def __get__(self): - return self.dev_attr.max_srq - property max_srq_wr: - def __get__(self): - return self.dev_attr.max_srq_wr - property max_srq_sge: - def __get__(self): - return self.dev_attr.max_srq_sge - property max_pkeys: - def __get__(self): - return self.dev_attr.max_pkeys - property local_ca_ack_delay: - def __get__(self): - return self.dev_attr.local_ca_ack_delay - property phys_port_cnt: - def __get__(self): - return self.dev_attr.phys_port_cnt + @property + def fw_version(self): + return self.dev_attr.fw_ver.decode() + @property + def node_guid(self): + return self.dev_attr.node_guid + @property + def sys_image_guid(self): + return self.dev_attr.sys_image_guid + @property + def max_mr_size(self): + return self.dev_attr.max_mr_size + @property + def page_size_cap(self): + return self.dev_attr.page_size_cap + @property + def vendor_id(self): + return self.dev_attr.vendor_id + @property + def vendor_part_id(self): + return self.dev_attr.vendor_part_id + @property + def hw_ver(self): + return self.dev_attr.hw_ver + @property + def max_qp(self): + return self.dev_attr.max_qp + @property + def max_qp_wr(self): + return self.dev_attr.max_qp_wr + @property + def device_cap_flags(self): + return self.dev_attr.device_cap_flags + @property + def max_sge(self): + return self.dev_attr.max_sge + @property + def max_sge_rd(self): + return self.dev_attr.max_sge_rd + @property + def max_cq(self): + return self.dev_attr.max_cq + @property + def max_cqe(self): + return self.dev_attr.max_cqe + @property + def max_mr(self): + return self.dev_attr.max_mr + @property + def max_pd(self): + return self.dev_attr.max_pd + @property + def max_qp_rd_atom(self): + return self.dev_attr.max_qp_rd_atom + @property + def max_ee_rd_atom(self): + return self.dev_attr.max_ee_rd_atom + @property + def max_res_rd_atom(self): + return self.dev_attr.max_res_rd_atom + @property + def max_qp_init_rd_atom(self): + return self.dev_attr.max_qp_init_rd_atom + @property + def max_ee_init_rd_atom(self): + return self.dev_attr.max_ee_init_rd_atom + @property + def atomic_caps(self): + return self.dev_attr.atomic_cap + @property + def max_ee(self): + return self.dev_attr.max_ee + @property + def max_rdd(self): + return self.dev_attr.max_rdd + @property + def max_mw(self): + return self.dev_attr.max_mw + @property + def max_raw_ipv6_qps(self): + return self.dev_attr.max_raw_ipv6_qp + @property + def max_raw_ethy_qp(self): + return self.dev_attr.max_raw_ethy_qp + @property + def max_mcast_grp(self): + return self.dev_attr.max_mcast_grp + @property + def max_mcast_qp_attach(self): + return self.dev_attr.max_mcast_qp_attach + @property + def max_ah(self): + return self.dev_attr.max_ah + @property + def max_fmr(self): + return self.dev_attr.max_fmr + @property + def max_map_per_fmr(self): + return self.dev_attr.max_map_per_fmr + @property + def max_srq(self): + return self.dev_attr.max_srq + @property + def max_srq_wr(self): + return self.dev_attr.max_srq_wr + @property + def max_srq_sge(self): + return self.dev_attr.max_srq_sge + @property + def max_pkeys(self): + return self.dev_attr.max_pkeys + @property + def local_ca_ack_delay(self): + return self.dev_attr.local_ca_ack_delay + @property + def phys_port_cnt(self): + return self.dev_attr.phys_port_cnt def __str__(self): print_format = '{:<22}: {:<20}\n' From patchwork Sun Feb 24 13:06:23 2019 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Noa Osherovich X-Patchwork-Id: 10827931 Return-Path: Received: from mail.wl.linuxfoundation.org (pdx-wl-mail.web.codeaurora.org [172.30.200.125]) by pdx-korg-patchwork-2.web.codeaurora.org (Postfix) with ESMTP id 580FA1880 for ; Sun, 24 Feb 2019 13:08:55 +0000 (UTC) Received: from mail.wl.linuxfoundation.org (localhost [127.0.0.1]) by mail.wl.linuxfoundation.org (Postfix) with ESMTP id 45A9A2846C for ; Sun, 24 Feb 2019 13:08:55 +0000 (UTC) Received: by mail.wl.linuxfoundation.org (Postfix, from userid 486) id 334342B50F; Sun, 24 Feb 2019 13:08:55 +0000 (UTC) X-Spam-Checker-Version: SpamAssassin 3.3.1 (2010-03-16) on pdx-wl-mail.web.codeaurora.org X-Spam-Level: X-Spam-Status: No, score=-7.9 required=2.0 tests=BAYES_00,MAILING_LIST_MULTI, RCVD_IN_DNSWL_HI,UNPARSEABLE_RELAY autolearn=ham version=3.3.1 Received: from vger.kernel.org (vger.kernel.org [209.132.180.67]) by mail.wl.linuxfoundation.org (Postfix) with ESMTP id 346712846C for ; Sun, 24 Feb 2019 13:08:54 +0000 (UTC) Received: (majordomo@vger.kernel.org) by vger.kernel.org via listexpand id S1726835AbfBXNIx (ORCPT ); Sun, 24 Feb 2019 08:08:53 -0500 Received: from mail-il-dmz.mellanox.com ([193.47.165.129]:50476 "EHLO mellanox.co.il" rhost-flags-OK-OK-OK-FAIL) by vger.kernel.org with ESMTP id S1728059AbfBXNIw (ORCPT ); Sun, 24 Feb 2019 08:08:52 -0500 Received: from Internal Mail-Server by MTLPINE1 (envelope-from noaos@mellanox.com) with ESMTPS (AES256-SHA encrypted); 24 Feb 2019 15:06:48 +0200 Received: from reg-l-vrt-059-009.mtl.labs.mlnx (reg-l-vrt-059-009.mtl.labs.mlnx [10.135.59.9]) by labmailer.mlnx (8.13.8/8.13.8) with ESMTP id x1OD6mfY026210; Sun, 24 Feb 2019 15:06:48 +0200 From: Noa Osherovich To: leonro@mellanox.com, jgg@mellanox.com, dledford@redhat.com Cc: linux-rdma@vger.kernel.org, Noa Osherovich Subject: [PATCH rdma-core 04/19] pyverbs: Introduce PD class Date: Sun, 24 Feb 2019 15:06:23 +0200 Message-Id: <20190224130638.31848-5-noaos@mellanox.com> X-Mailer: git-send-email 2.17.2 In-Reply-To: <20190224130638.31848-1-noaos@mellanox.com> References: <20190224130638.31848-1-noaos@mellanox.com> Sender: linux-rdma-owner@vger.kernel.org Precedence: bulk List-ID: X-Mailing-List: linux-rdma@vger.kernel.org X-Virus-Scanned: ClamAV using ClamSMTP This patch introduces PD object which represents ibv_pd. This commit also introduces the usage of weak references. In Python, a weak reference is a reference to an object not strong enough to keep it from being garbage-collected. In Pyverbs, it is used to allow an object to keep track of the objects created using it, so if an object is being destroyed (e.g. explicitly by the user), it can iterate over the dependent objects and destroy the underlying C objects. E.g.: A user opens a device and creates a PD. If the Context object is being destroyed before the PD object, the kernel will return EBUSY. To avoid that, the Context object will hold a weakref to the PD and destroy the C PD before closing itself. This renders the Python PD object useless, but it is a user's choice that we are not blocking. In order to provide a clean teardown, Pyverbs Context is also a context manager, which means itshouldbe used within a 'with' block. This way, when the block is over, teardown will be executed properly. The same applies for all other relevant pyverbs' classes. A PyverbsCM class was added as an alternative base objects to inherit from. It inherits from PyverbsObject and adds the __enter__ and __exit__ functions needed for a Python context manager. Signed-off-by: Noa Osherovich --- pyverbs/CMakeLists.txt | 7 ++--- pyverbs/base.pxd | 6 ++++- pyverbs/base.pyx | 41 ++++++++++++++++++++++++++++- pyverbs/device.pxd | 7 ++--- pyverbs/device.pyx | 21 +++++++++++---- pyverbs/libibverbs.pxd | 58 +++++++++++++++++++++++------------------- pyverbs/pd.pxd | 10 ++++++++ pyverbs/pd.pyx | 46 +++++++++++++++++++++++++++++++++ 8 files changed, 157 insertions(+), 39 deletions(-) create mode 100644 pyverbs/pd.pxd create mode 100644 pyverbs/pd.pyx diff --git a/pyverbs/CMakeLists.txt b/pyverbs/CMakeLists.txt index 8d1f8030713f..794edee1f19f 100644 --- a/pyverbs/CMakeLists.txt +++ b/pyverbs/CMakeLists.txt @@ -1,11 +1,12 @@ # SPDX-License-Identifier: (GPL-2.0 OR Linux-OpenIB) -# Copyright (c) 2018, Mellanox Technologies. All rights reserved. See COPYING file +# Copyright (c) 2019, Mellanox Technologies. All rights reserved. See COPYING file rdma_cython_module(pyverbs - enums.pyx + addr.pyx base.pyx device.pyx - addr.pyx + enums.pyx + pd.pyx ) rdma_python_module(pyverbs diff --git a/pyverbs/base.pxd b/pyverbs/base.pxd index fa5e0dad3308..fa661edb5315 100644 --- a/pyverbs/base.pxd +++ b/pyverbs/base.pxd @@ -1,5 +1,9 @@ # SPDX-License-Identifier: (GPL-2.0 OR Linux-OpenIB) -# Copyright (c) 2018, Mellanox Technologies. All rights reserved. +# Copyright (c) 2019, Mellanox Technologies. All rights reserved. cdef class PyverbsObject(object): + cdef object __weakref__ cdef object logger + +cdef class PyverbsCM(PyverbsObject): + cpdef close(self) diff --git a/pyverbs/base.pyx b/pyverbs/base.pyx index 6bcebd095c83..deee520b1260 100644 --- a/pyverbs/base.pyx +++ b/pyverbs/base.pyx @@ -1,5 +1,5 @@ # SPDX-License-Identifier: (GPL-2.0 OR Linux-OpenIB) -# Copyright (c) 2018, Mellanox Technologies. All rights reserved. +# Copyright (c) 2019, Mellanox Technologies. All rights reserved. import logging from pyverbs.pyverbs_error import PyverbsRDMAError @@ -22,3 +22,42 @@ cdef class PyverbsObject(object): def set_log_level(self, val): self.logger.setLevel(val) + + def close_weakrefs(self, iterables): + """ + For each iterable element of iterables, pop each element and + call its close() method. This method is used when an object is being + closed while other objects still hold C references to it; the object + holds weakrefs to such other object, and closes them before trying to + teardown the C resources. + :param iterables: an array of WeakSets + :return: None + """ + # None elements can be present if an object's close() was called more + # than once (e.g. GC and by another object) + for it in iterables: + if it is None: + continue + while True: + try: + tmp = it.pop() + tmp.close() + except KeyError: # popping an empty set + break + + +cdef class PyverbsCM(PyverbsObject): + """ + This is a base class for pyverbs' context manager objects. It includes + __enter__ and __exit__ functions. + close() is also declared but it should be overridden by each inheriting + class. + """ + def __enter__(self): + return self + + def __exit__(self, exc_type, exc_value, traceback): + return self.close() + + cpdef close(self): + pass diff --git a/pyverbs/device.pxd b/pyverbs/device.pxd index 7a6489da11bf..5f3ba4dfc6f8 100644 --- a/pyverbs/device.pxd +++ b/pyverbs/device.pxd @@ -1,14 +1,15 @@ # SPDX-License-Identifier: (GPL-2.0 OR Linux-OpenIB) # Copyright (c) 2018, Mellanox Technologies. All rights reserved. See COPYING file -from .base cimport PyverbsObject +from .base cimport PyverbsObject, PyverbsCM cimport pyverbs.libibverbs as v -cdef class Context(PyverbsObject): +cdef class Context(PyverbsCM): cdef v.ibv_context *context cdef object name - cpdef close(self) + cdef add_ref(self, obj) + cdef object pds cdef class DeviceAttr(PyverbsObject): cdef v.ibv_device_attr dev_attr diff --git a/pyverbs/device.pyx b/pyverbs/device.pyx index 0264a4f0bddd..000fb53ff5a2 100644 --- a/pyverbs/device.pyx +++ b/pyverbs/device.pyx @@ -6,11 +6,14 @@ Device module introduces the Context and DeviceAttr class. It allows user to open an IB device (using Context(name=) and query it, which returns a DeviceAttr object. """ -from .pyverbs_error import PyverbsRDMAError +import weakref + +from .pyverbs_error import PyverbsRDMAError, PyverbsError from .pyverbs_error import PyverbsUserError from pyverbs.base import PyverbsRDMAErrno cimport pyverbs.libibverbs as v from pyverbs.addr cimport GID +from pyverbs.pd cimport PD cdef extern from 'errno.h': int errno @@ -55,10 +58,9 @@ class Device(PyverbsObject): guid=guid_to_hex(self.guid)) -cdef class Context(PyverbsObject): +cdef class Context(PyverbsCM): """ - Context class represents the C ibv_context. It currently allows only - querying the underlying device. + Context class represents the C ibv_context. """ def __cinit__(self, **kwargs): """ @@ -71,6 +73,8 @@ cdef class Context(PyverbsObject): """ cdef int count cdef v.ibv_device **dev_list + + self.pds = weakref.WeakSet() dev_name = kwargs.get('name') if dev_name is not None: @@ -106,6 +110,7 @@ cdef class Context(PyverbsObject): cpdef close(self): self.logger.debug('Closing Context') + self.close_weakrefs([self.pds]) if self.context != NULL: rc = v.ibv_close_device(self.context) if rc != 0: @@ -131,9 +136,15 @@ cdef class Context(PyverbsObject): rc = v.ibv_query_gid(self.context, port_num, index, &gid.gid) if rc != 0: raise PyverbsRDMAError('Failed to query gid {idx} of port {port}'. - format(idx=index, port=port_num)) + format(idx=index, port=port_num)) return gid + cdef add_ref(self, obj): + if isinstance(obj, PD): + self.pds.add(obj) + else: + raise PyverbsError('Unrecognized object type') + cdef class DeviceAttr(PyverbsObject): """ diff --git a/pyverbs/libibverbs.pxd b/pyverbs/libibverbs.pxd index f212078f9048..d625ddf7215b 100644 --- a/pyverbs/libibverbs.pxd +++ b/pyverbs/libibverbs.pxd @@ -24,13 +24,13 @@ cdef extern from 'infiniband/verbs.h': cdef struct ibv_device_attr: char *fw_ver - unsigned long node_guid; - unsigned long sys_image_guid; - unsigned long max_mr_size; - unsigned long page_size_cap; - unsigned int vendor_id; - unsigned int vendor_part_id; - unsigned int hw_ver; + unsigned long node_guid + unsigned long sys_image_guid + unsigned long max_mr_size + unsigned long page_size_cap + unsigned int vendor_id + unsigned int vendor_part_id + unsigned int hw_ver unsigned int max_qp unsigned int max_qp_wr unsigned int device_cap_flags @@ -46,29 +46,35 @@ cdef extern from 'infiniband/verbs.h': unsigned int max_qp_init_rd_atom unsigned int max_ee_init_rd_atom ibv_atomic_cap atomic_cap - unsigned int max_ee; - unsigned int max_rdd; - unsigned int max_mw; - unsigned int max_raw_ipv6_qp; - unsigned int max_raw_ethy_qp; - unsigned int max_mcast_grp; - unsigned int max_mcast_qp_attach; - unsigned int max_total_mcast_qp_attach; - unsigned int max_ah; - unsigned int max_fmr; - unsigned int max_map_per_fmr; - unsigned int max_srq; - unsigned int max_srq_wr; - unsigned int max_srq_sge; - unsigned int max_pkeys; - unsigned int local_ca_ack_delay; - unsigned int phys_port_cnt; + unsigned int max_ee + unsigned int max_rdd + unsigned int max_mw + unsigned int max_raw_ipv6_qp + unsigned int max_raw_ethy_qp + unsigned int max_mcast_grp + unsigned int max_mcast_qp_attach + unsigned int max_total_mcast_qp_attach + unsigned int max_ah + unsigned int max_fmr + unsigned int max_map_per_fmr + unsigned int max_srq + unsigned int max_srq_wr + unsigned int max_srq_sge + unsigned int max_pkeys + unsigned int local_ca_ack_delay + unsigned int phys_port_cnt + + struct ibv_pd: + ibv_context *context + unsigned int handle ibv_device **ibv_get_device_list(int *n) - void ibv_free_device_list(ibv_device **list); - ibv_context *ibv_open_device(ibv_device *device); + void ibv_free_device_list(ibv_device **list) + ibv_context *ibv_open_device(ibv_device *device) int ibv_close_device(ibv_context *context) int ibv_query_device(ibv_context *context, ibv_device_attr *device_attr) unsigned long ibv_get_device_guid(ibv_device *device) int ibv_query_gid(ibv_context *context, unsigned int port_num, int index, ibv_gid *gid) + ibv_pd *ibv_alloc_pd(ibv_context *context) + int ibv_dealloc_pd(ibv_pd *pd) diff --git a/pyverbs/pd.pxd b/pyverbs/pd.pxd new file mode 100644 index 000000000000..ec25ec8841ec --- /dev/null +++ b/pyverbs/pd.pxd @@ -0,0 +1,10 @@ +# SPDX-License-Identifier: (GPL-2.0 OR Linux-OpenIB) +# Copyright (c) 2019, Mellanox Technologies. All rights reserved. +from pyverbs.device cimport Context +cimport pyverbs.libibverbs as v +from .base cimport PyverbsCM + + +cdef class PD(PyverbsCM): + cdef v.ibv_pd *pd + cdef Context ctx diff --git a/pyverbs/pd.pyx b/pyverbs/pd.pyx new file mode 100644 index 000000000000..9975f906b329 --- /dev/null +++ b/pyverbs/pd.pyx @@ -0,0 +1,46 @@ +# SPDX-License-Identifier: (GPL-2.0 OR Linux-OpenIB) +# Copyright (c) 2019, Mellanox Technologies. All rights reserved. +from pyverbs.pyverbs_error import PyverbsRDMAError +from pyverbs.base import PyverbsRDMAErrno + +cdef extern from 'errno.h': + int errno + + +cdef class PD(PyverbsCM): + def __cinit__(self, Context context not None): + """ + Initializes a PD object. A reference for the creating Context is kept + so that Python's GC will destroy the objects in the right order. + :param context: The Context object creating the PD + :return: The newly created PD on success + """ + self.pd = v.ibv_alloc_pd(context.context) + if self.pd == NULL: + raise PyverbsRDMAErrno('Failed to allocate PD', errno) + self.ctx = context + context.add_ref(self) + self.logger.debug('PD: Allocated ibv_pd') + + def __dealloc__(self): + """ + Closes the inner PD. + :return: None + """ + self.close() + + cpdef close(self): + """ + Closes the underlying C object of the PD. + PD may be deleted directly or indirectly by closing its context, which + leaves the Python PD object without the underlying C object, so during + destruction, need to check whether or not the C object exists. + :return: None + """ + self.logger.debug('Closing PD') + if self.pd != NULL: + rc = v.ibv_dealloc_pd(self.pd) + if rc != 0: + raise PyverbsRDMAErrno('Failed to dealloc PD') + self.pd = NULL + self.ctx = None From patchwork Sun Feb 24 13:06:24 2019 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Noa Osherovich X-Patchwork-Id: 10827939 Return-Path: Received: from mail.wl.linuxfoundation.org (pdx-wl-mail.web.codeaurora.org [172.30.200.125]) by pdx-korg-patchwork-2.web.codeaurora.org (Postfix) with ESMTP id CA6311575 for ; Sun, 24 Feb 2019 13:08:56 +0000 (UTC) Received: from mail.wl.linuxfoundation.org (localhost [127.0.0.1]) by mail.wl.linuxfoundation.org (Postfix) with ESMTP id 087392B4A6 for ; Sun, 24 Feb 2019 13:08:56 +0000 (UTC) Received: by mail.wl.linuxfoundation.org (Postfix, from userid 486) id F12E02846C; Sun, 24 Feb 2019 13:08:55 +0000 (UTC) X-Spam-Checker-Version: SpamAssassin 3.3.1 (2010-03-16) on pdx-wl-mail.web.codeaurora.org X-Spam-Level: X-Spam-Status: No, score=-7.9 required=2.0 tests=BAYES_00,MAILING_LIST_MULTI, RCVD_IN_DNSWL_HI,UNPARSEABLE_RELAY autolearn=ham version=3.3.1 Received: from vger.kernel.org (vger.kernel.org [209.132.180.67]) by mail.wl.linuxfoundation.org (Postfix) with ESMTP id D15B92B4FD for ; Sun, 24 Feb 2019 13:08:54 +0000 (UTC) Received: (majordomo@vger.kernel.org) by vger.kernel.org via listexpand id S1726865AbfBXNIy (ORCPT ); Sun, 24 Feb 2019 08:08:54 -0500 Received: from mail-il-dmz.mellanox.com ([193.47.165.129]:50507 "EHLO mellanox.co.il" rhost-flags-OK-OK-OK-FAIL) by vger.kernel.org with ESMTP id S1728248AbfBXNIx (ORCPT ); Sun, 24 Feb 2019 08:08:53 -0500 Received: from Internal Mail-Server by MTLPINE1 (envelope-from noaos@mellanox.com) with ESMTPS (AES256-SHA encrypted); 24 Feb 2019 15:06:49 +0200 Received: from reg-l-vrt-059-009.mtl.labs.mlnx (reg-l-vrt-059-009.mtl.labs.mlnx [10.135.59.9]) by labmailer.mlnx (8.13.8/8.13.8) with ESMTP id x1OD6mfZ026210; Sun, 24 Feb 2019 15:06:48 +0200 From: Noa Osherovich To: leonro@mellanox.com, jgg@mellanox.com, dledford@redhat.com Cc: linux-rdma@vger.kernel.org, Noa Osherovich Subject: [PATCH rdma-core 05/19] pyverbs: Add unittests for PD class Date: Sun, 24 Feb 2019 15:06:24 +0200 Message-Id: <20190224130638.31848-6-noaos@mellanox.com> X-Mailer: git-send-email 2.17.2 In-Reply-To: <20190224130638.31848-1-noaos@mellanox.com> References: <20190224130638.31848-1-noaos@mellanox.com> Sender: linux-rdma-owner@vger.kernel.org Precedence: bulk List-ID: X-Mailing-List: linux-rdma@vger.kernel.org X-Virus-Scanned: ClamAV using ClamSMTP This patch adds tests for the PD related API. Bad flow of PD creation using an already freed context is not possible since it generated a segmentation fault in rdma-core. Other bad flows are checked. Signed-off-by: Noa Osherovich --- pyverbs/CMakeLists.txt | 1 + pyverbs/tests/pd.py | 70 ++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 71 insertions(+) create mode 100644 pyverbs/tests/pd.py diff --git a/pyverbs/CMakeLists.txt b/pyverbs/CMakeLists.txt index 794edee1f19f..c1e7dceb8d9c 100644 --- a/pyverbs/CMakeLists.txt +++ b/pyverbs/CMakeLists.txt @@ -18,6 +18,7 @@ rdma_python_module(pyverbs rdma_python_test(pyverbs/tests tests/__init__.py tests/device.py + tests/pd.py ) rdma_internal_binary( diff --git a/pyverbs/tests/pd.py b/pyverbs/tests/pd.py new file mode 100644 index 000000000000..c80f0b6df80e --- /dev/null +++ b/pyverbs/tests/pd.py @@ -0,0 +1,70 @@ +# SPDX-License-Identifier: (GPL-2.0 OR Linux-OpenIB) +# Copyright (c) 2019, Mellanox Technologies. All rights reserved. See COPYING file + +import unittest +import random + +from pyverbs.base import PyverbsRDMAErrno +import pyverbs.device as d +from pyverbs.pd import PD + + +class pd_test(unittest.TestCase): + """ + Test various functionalities of the PD class. + """ + def test_alloc_pd(self): + """ + Test ibv_alloc_pd() + """ + lst = d.get_device_list() + for dev in lst: + with d.Context(name=dev.name.decode()) as ctx: + with PD(ctx): + pass + + def test_dealloc_pd(self): + """ + Test ibv_dealloc_pd() + """ + lst = d.get_device_list() + for dev in lst: + with d.Context(name=dev.name.decode()) as ctx: + with PD(ctx) as pd: + pd.close() + + def test_multiple_pd_creation(self): + """ + Test multiple creations and destructions of a PD object + """ + lst = d.get_device_list() + for dev in lst: + with d.Context(name=dev.name.decode()) as ctx: + for i in range(random.randint(1, 200)): + with PD(ctx) as pd: + pd.close() + + def test_create_pd_none_ctx(self): + """ + Verify that PD can't be created with a None context + """ + try: + pd = PD(None) + except TypeError as te: + assert 'expected pyverbs.device.Context' in te.args[0] + assert 'got NoneType' in te.args[0] + else: + raise PyverbsRDMAErrno('Created a PD with None context') + + def test_destroy_pd_twice(self): + """ + Test bad flow cases in destruction of a PD object + """ + lst = d.get_device_list() + for dev in lst: + with d.Context(name=dev.name.decode()) as ctx: + with PD(ctx) as pd: + # Pyverbs supports multiple destruction of objects, we are + # not expecting an exception here. + pd.close() + pd.close() From patchwork Sun Feb 24 13:06:25 2019 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Noa Osherovich X-Patchwork-Id: 10827927 Return-Path: Received: from mail.wl.linuxfoundation.org (pdx-wl-mail.web.codeaurora.org [172.30.200.125]) by pdx-korg-patchwork-2.web.codeaurora.org (Postfix) with ESMTP id DE2ED17E9 for ; Sun, 24 Feb 2019 13:08:54 +0000 (UTC) Received: from mail.wl.linuxfoundation.org (localhost [127.0.0.1]) by mail.wl.linuxfoundation.org (Postfix) with ESMTP id C6D862B4E5 for ; Sun, 24 Feb 2019 13:08:54 +0000 (UTC) Received: by mail.wl.linuxfoundation.org (Postfix, from userid 486) id B9E472B525; Sun, 24 Feb 2019 13:08:54 +0000 (UTC) X-Spam-Checker-Version: SpamAssassin 3.3.1 (2010-03-16) on pdx-wl-mail.web.codeaurora.org X-Spam-Level: X-Spam-Status: No, score=-7.9 required=2.0 tests=BAYES_00,MAILING_LIST_MULTI, RCVD_IN_DNSWL_HI,UNPARSEABLE_RELAY autolearn=ham version=3.3.1 Received: from vger.kernel.org (vger.kernel.org [209.132.180.67]) by mail.wl.linuxfoundation.org (Postfix) with ESMTP id CAF812B4E5 for ; Sun, 24 Feb 2019 13:08:51 +0000 (UTC) Received: (majordomo@vger.kernel.org) by vger.kernel.org via listexpand id S1728229AbfBXNIv (ORCPT ); Sun, 24 Feb 2019 08:08:51 -0500 Received: from mail-il-dmz.mellanox.com ([193.47.165.129]:50469 "EHLO mellanox.co.il" rhost-flags-OK-OK-OK-FAIL) by vger.kernel.org with ESMTP id S1726865AbfBXNIu (ORCPT ); Sun, 24 Feb 2019 08:08:50 -0500 Received: from Internal Mail-Server by MTLPINE1 (envelope-from noaos@mellanox.com) with ESMTPS (AES256-SHA encrypted); 24 Feb 2019 15:06:48 +0200 Received: from reg-l-vrt-059-009.mtl.labs.mlnx (reg-l-vrt-059-009.mtl.labs.mlnx [10.135.59.9]) by labmailer.mlnx (8.13.8/8.13.8) with ESMTP id x1OD6mfa026210; Sun, 24 Feb 2019 15:06:48 +0200 From: Noa Osherovich To: leonro@mellanox.com, jgg@mellanox.com, dledford@redhat.com Cc: linux-rdma@vger.kernel.org, Noa Osherovich Subject: [PATCH rdma-core 06/19] pyverbs: Introducing MR object Date: Sun, 24 Feb 2019 15:06:25 +0200 Message-Id: <20190224130638.31848-7-noaos@mellanox.com> X-Mailer: git-send-email 2.17.2 In-Reply-To: <20190224130638.31848-1-noaos@mellanox.com> References: <20190224130638.31848-1-noaos@mellanox.com> Sender: linux-rdma-owner@vger.kernel.org Precedence: bulk List-ID: X-Mailing-List: linux-rdma@vger.kernel.org X-Virus-Scanned: ClamAV using ClamSMTP MR represnts ibv_mr. Instead of allocating a buffer prior to calling ibv_reg_mr, the user provides the MR class the required size and flags, and pyverbs allocates the needed buffer. The MR class exposes read() and write() methods to users to allow setting and reading the underlying memory buffer. Signed-off-by: Noa Osherovich --- pyverbs/CMakeLists.txt | 1 + pyverbs/libibverbs.pxd | 11 +++++ pyverbs/mr.pxd | 12 +++++ pyverbs/mr.pyx | 109 +++++++++++++++++++++++++++++++++++++++++ pyverbs/pd.pxd | 2 + pyverbs/pd.pyx | 13 ++++- 6 files changed, 147 insertions(+), 1 deletion(-) create mode 100644 pyverbs/mr.pxd create mode 100644 pyverbs/mr.pyx diff --git a/pyverbs/CMakeLists.txt b/pyverbs/CMakeLists.txt index c1e7dceb8d9c..b660b46f170a 100644 --- a/pyverbs/CMakeLists.txt +++ b/pyverbs/CMakeLists.txt @@ -6,6 +6,7 @@ rdma_cython_module(pyverbs base.pyx device.pyx enums.pyx + mr.pyx pd.pyx ) diff --git a/pyverbs/libibverbs.pxd b/pyverbs/libibverbs.pxd index d625ddf7215b..5409ff04a649 100644 --- a/pyverbs/libibverbs.pxd +++ b/pyverbs/libibverbs.pxd @@ -68,6 +68,15 @@ cdef extern from 'infiniband/verbs.h': ibv_context *context unsigned int handle + cdef struct ibv_mr: + ibv_context *context + ibv_pd *pd + void *addr + size_t length + unsigned int handle + unsigned int lkey + unsigned int rkey + ibv_device **ibv_get_device_list(int *n) void ibv_free_device_list(ibv_device **list) ibv_context *ibv_open_device(ibv_device *device) @@ -78,3 +87,5 @@ cdef extern from 'infiniband/verbs.h': int index, ibv_gid *gid) ibv_pd *ibv_alloc_pd(ibv_context *context) int ibv_dealloc_pd(ibv_pd *pd) + ibv_mr *ibv_reg_mr(ibv_pd *pd, void *addr, size_t length, int access) + int ibv_dereg_mr(ibv_mr *mr) diff --git a/pyverbs/mr.pxd b/pyverbs/mr.pxd new file mode 100644 index 000000000000..92578021c3b3 --- /dev/null +++ b/pyverbs/mr.pxd @@ -0,0 +1,12 @@ +# SPDX-License-Identifier: (GPL-2.0 OR Linux-OpenIB) +# Copyright (c) 2019, Mellanox Technologies. All rights reserved. See COPYING file + +from pyverbs.base cimport PyverbsCM +from . cimport libibverbs as v + + +cdef class MR(PyverbsCM): + cdef object pd + cdef v.ibv_mr *mr + cdef void *buf + cpdef read(self, length, offset) diff --git a/pyverbs/mr.pyx b/pyverbs/mr.pyx new file mode 100644 index 000000000000..31949e739376 --- /dev/null +++ b/pyverbs/mr.pyx @@ -0,0 +1,109 @@ +# SPDX-License-Identifier: (GPL-2.0 OR Linux-OpenIB) +# Copyright (c) 2019, Mellanox Technologies. All rights reserved. See COPYING file + +from pyverbs.pyverbs_error import PyverbsRDMAError, PyverbsError +from pyverbs.base import PyverbsRDMAErrno +from .pd cimport PD +import resource + +cdef extern from 'stdlib.h': + int posix_memalign(void **memptr, size_t alignment, size_t size) +cdef extern from 'stdlib.h': + void free(void *ptr) +cdef extern from 'string.h': + void *memcpy(void *dest, const void *src, size_t n) + void *memset(void *s, int c, size_t n) +cdef extern from 'stdint.h': + ctypedef int uintptr_t + + +cdef class MR(PyverbsCM): + """ + MR class represents ibv_mr. Buffer allocation in done in the c'tor. Freeing + it is done in close(). + """ + def __cinit__(self, PD pd not None, length, access): + """ + Allocate a user-level buffer of length and register a Memory + Region of the given length and access flags. + :param pd: A PD object + :param length: Length in bytes + :param access: Access flags, see ibv_access_flags enum + :return: The newly created MR on success + """ + #We want to enable registering an MR of size 0 but this fails with a + #buffer of size 0, so in this case lets increase the buffer + if length == 0: + length = 10 + rc = posix_memalign(&self.buf, resource.getpagesize(), length) + if rc: + raise PyverbsRDMAError('Failed to allocate MR buffer of size {l}'. + format(l=length)) + memset(self.buf, 0, length) + self.mr = v.ibv_reg_mr(pd.pd, self.buf, length, access) + if self.mr == NULL: + raise PyverbsRDMAErrno('Failed to register a MR. length: {l}, access flags: {a}'. + format(l=length, a=access)) + self.pd = pd + pd.add_ref(self) + self.logger.debug('Registered ibv_mr. Length: {l}, access flags {a}'. + format(l=length, a=access)) + + def __dealloc__(self): + self.close() + + cpdef close(self): + """ + Closes the underlying C object of the MR and frees the memory allocated. + MR may be deleted directly or indirectly by closing its context, which + leaves the Python PD object without the underlying C object, so during + destruction, need to check whether or not the C object exists. + :return: None + """ + self.logger.debug('Closing MR') + if self.mr != NULL: + rc = v.ibv_dereg_mr(self.mr) + if rc != 0: + raise PyverbsRDMAErrno('Failed to dereg MR') + self.mr = NULL + self.pd = None + free(self.buf) + self.buf = NULL + + def write(self, data, length): + """ + Write user data to the MR's buffer using memcpy + :param data: User data to write + :param length: Length of the data to write + :return: None + """ + # If data is a string, cast it to bytes as Python3 doesn't + # automatically convert it. + if isinstance(data, str): + data = data.encode() + memcpy(self.buf, data, length) + + cpdef read(self, length, offset): + """ + Reads data from the MR's buffer + :param length: Length of data to read + :param offset: Reading offset + :return: The data on the buffer in the requested offset + """ + cdef char *data + cdef int off = offset # we can't use offset in the next line, as it is + # a Python object and not C + data = (self.buf + off) + return data[:length] + + @property + def buf(self): + return self.buf + + @property + def lkey(self): + return self.mr.lkey + + @property + def rkey(self): + return self.mr.rkey diff --git a/pyverbs/pd.pxd b/pyverbs/pd.pxd index ec25ec8841ec..8381662b21be 100644 --- a/pyverbs/pd.pxd +++ b/pyverbs/pd.pxd @@ -8,3 +8,5 @@ from .base cimport PyverbsCM cdef class PD(PyverbsCM): cdef v.ibv_pd *pd cdef Context ctx + cdef add_ref(self, obj) + cdef object mrs diff --git a/pyverbs/pd.pyx b/pyverbs/pd.pyx index 9975f906b329..a5fc063751e1 100644 --- a/pyverbs/pd.pyx +++ b/pyverbs/pd.pyx @@ -1,7 +1,10 @@ # SPDX-License-Identifier: (GPL-2.0 OR Linux-OpenIB) # Copyright (c) 2019, Mellanox Technologies. All rights reserved. -from pyverbs.pyverbs_error import PyverbsRDMAError +import weakref + +from pyverbs.pyverbs_error import PyverbsRDMAError, PyverbsError from pyverbs.base import PyverbsRDMAErrno +from .mr cimport MR cdef extern from 'errno.h': int errno @@ -21,6 +24,7 @@ cdef class PD(PyverbsCM): self.ctx = context context.add_ref(self) self.logger.debug('PD: Allocated ibv_pd') + self.mrs = weakref.WeakSet() def __dealloc__(self): """ @@ -38,9 +42,16 @@ cdef class PD(PyverbsCM): :return: None """ self.logger.debug('Closing PD') + self.close_weakrefs([self.mrs]) if self.pd != NULL: rc = v.ibv_dealloc_pd(self.pd) if rc != 0: raise PyverbsRDMAErrno('Failed to dealloc PD') self.pd = NULL self.ctx = None + + cdef add_ref(self, obj): + if isinstance(obj, MR): + self.mrs.add(obj) + else: + raise PyverbsError('Unrecognized object type') From patchwork Sun Feb 24 13:06:26 2019 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Noa Osherovich X-Patchwork-Id: 10827929 Return-Path: Received: from mail.wl.linuxfoundation.org (pdx-wl-mail.web.codeaurora.org [172.30.200.125]) by pdx-korg-patchwork-2.web.codeaurora.org (Postfix) with ESMTP id 03CDE13B5 for ; Sun, 24 Feb 2019 13:08:55 +0000 (UTC) Received: from mail.wl.linuxfoundation.org (localhost [127.0.0.1]) by mail.wl.linuxfoundation.org (Postfix) with ESMTP id E71DB2B4E5 for ; Sun, 24 Feb 2019 13:08:54 +0000 (UTC) Received: by mail.wl.linuxfoundation.org (Postfix, from userid 486) id CE0812B532; Sun, 24 Feb 2019 13:08:54 +0000 (UTC) X-Spam-Checker-Version: SpamAssassin 3.3.1 (2010-03-16) on pdx-wl-mail.web.codeaurora.org X-Spam-Level: X-Spam-Status: No, score=-7.9 required=2.0 tests=BAYES_00,MAILING_LIST_MULTI, RCVD_IN_DNSWL_HI,UNPARSEABLE_RELAY autolearn=ham version=3.3.1 Received: from vger.kernel.org (vger.kernel.org [209.132.180.67]) by mail.wl.linuxfoundation.org (Postfix) with ESMTP id A2ABD2B4FD for ; Sun, 24 Feb 2019 13:08:53 +0000 (UTC) Received: (majordomo@vger.kernel.org) by vger.kernel.org via listexpand id S1728256AbfBXNIw (ORCPT ); Sun, 24 Feb 2019 08:08:52 -0500 Received: from mail-il-dmz.mellanox.com ([193.47.165.129]:50483 "EHLO mellanox.co.il" rhost-flags-OK-OK-OK-FAIL) by vger.kernel.org with ESMTP id S1726835AbfBXNIw (ORCPT ); Sun, 24 Feb 2019 08:08:52 -0500 Received: from Internal Mail-Server by MTLPINE1 (envelope-from noaos@mellanox.com) with ESMTPS (AES256-SHA encrypted); 24 Feb 2019 15:06:49 +0200 Received: from reg-l-vrt-059-009.mtl.labs.mlnx (reg-l-vrt-059-009.mtl.labs.mlnx [10.135.59.9]) by labmailer.mlnx (8.13.8/8.13.8) with ESMTP id x1OD6mfb026210; Sun, 24 Feb 2019 15:06:48 +0200 From: Noa Osherovich To: leonro@mellanox.com, jgg@mellanox.com, dledford@redhat.com Cc: linux-rdma@vger.kernel.org, Noa Osherovich Subject: [PATCH rdma-core 07/19] pyverbs: Add unittests for MR class Date: Sun, 24 Feb 2019 15:06:26 +0200 Message-Id: <20190224130638.31848-8-noaos@mellanox.com> X-Mailer: git-send-email 2.17.2 In-Reply-To: <20190224130638.31848-1-noaos@mellanox.com> References: <20190224130638.31848-1-noaos@mellanox.com> Sender: linux-rdma-owner@vger.kernel.org Precedence: bulk List-ID: X-Mailing-List: linux-rdma@vger.kernel.org X-Virus-Scanned: ClamAV using ClamSMTP This patch adds some basic unittests for MR class' API. Signed-off-by: Noa Osherovich --- pyverbs/CMakeLists.txt | 2 + pyverbs/tests/mr.py | 146 +++++++++++++++++++++++++++++++++++++++++ pyverbs/tests/utils.py | 28 ++++++++ 3 files changed, 176 insertions(+) create mode 100644 pyverbs/tests/mr.py create mode 100644 pyverbs/tests/utils.py diff --git a/pyverbs/CMakeLists.txt b/pyverbs/CMakeLists.txt index b660b46f170a..18056d6fcd65 100644 --- a/pyverbs/CMakeLists.txt +++ b/pyverbs/CMakeLists.txt @@ -19,7 +19,9 @@ rdma_python_module(pyverbs rdma_python_test(pyverbs/tests tests/__init__.py tests/device.py + tests/mr.py tests/pd.py + tests/utils.py ) rdma_internal_binary( diff --git a/pyverbs/tests/mr.py b/pyverbs/tests/mr.py new file mode 100644 index 000000000000..409cd41e3c06 --- /dev/null +++ b/pyverbs/tests/mr.py @@ -0,0 +1,146 @@ +# SPDX-License-Identifier: (GPL-2.0 OR Linux-OpenIB) +# Copyright (c) 2019, Mellanox Technologies. All rights reserved. See COPYING file +import unittest +import random + +from pyverbs.pyverbs_error import PyverbsRDMAError +from pyverbs.base import PyverbsRDMAErrno +import pyverbs.tests.utils as u +import pyverbs.device as d +from pyverbs.mr import MR +from pyverbs.pd import PD +import pyverbs.enums as e + +MAX_IO_LEN = 1048576 + + +class mr_test(unittest.TestCase): + """ + Test various functionalities of the MR class. + """ + def test_reg_mr(self): + """ Test ibv_reg_mr() """ + lst = d.get_device_list() + for dev in lst: + with d.Context(name=dev.name.decode()) as ctx: + with PD(ctx) as pd: + with MR(pd, u.get_mr_length(), u.get_access_flags()) as mr: + pass + + def test_dereg_mr(self): + """ Test ibv_dereg_mr() """ + lst = d.get_device_list() + for dev in lst: + with d.Context(name=dev.name.decode()) as ctx: + with PD(ctx) as pd: + with MR(pd, u.get_mr_length(), u.get_access_flags()) as mr: + mr.close() + + def test_reg_mr_bad_flow(self): + """ Verify that trying to register a MR with None PD fails """ + try: + mr = MR(None, random.randint(0, 10000), u.get_access_flags()) + except TypeError as te: + assert 'expected pyverbs.pd.PD' in te.args[0] + assert 'got NoneType' in te.args[0] + else: + raise PyverbsRDMAErrno('Created a MR with None PD') + + def test_dereg_mr_twice(self): + """ Verify that explicit call to MR's close() doesn't fails """ + lst = d.get_device_list() + for dev in lst: + with d.Context(name=dev.name.decode()) as ctx: + with PD(ctx) as pd: + with MR(pd, u.get_mr_length(), u.get_access_flags()) as mr: + # Pyverbs supports multiple destruction of objects, we are + # not expecting an exception here. + mr.close() + mr.close() + + def test_reg_mr_bad_flags(self): + """ Verify that illegal flags combination fails as expected """ + lst = d.get_device_list() + for dev in lst: + with d.Context(name=dev.name.decode()) as ctx: + with PD(ctx) as pd: + flags = random.sample([e.IBV_ACCESS_REMOTE_WRITE, + e.IBV_ACCESS_REMOTE_ATOMIC], + random.randint(1, 2)) + mr_flags = 0 + for i in flags: + mr_flags += i.value + try: + mr = MR(pd, u.get_mr_length(), mr_flags) + except PyverbsRDMAError as err: + assert 'Failed to register a MR' in err.args[0] + else: + raise PyverbsRDMAError('Registered a MR with illegal falgs') + + def test_write(self): + """ + Test writing to MR's buffer + """ + lst = d.get_device_list() + for dev in lst: + with d.Context(name=dev.name.decode()) as ctx: + with PD(ctx) as pd: + mr_len = u.get_mr_length() + with MR(pd, mr_len, u.get_access_flags()) as mr: + write_len = min(random.randint(1, MAX_IO_LEN), mr_len) + mr.write(u.get_data(write_len), write_len) + + def test_read(self): + """ + Test reading from MR's buffer + """ + lst = d.get_device_list() + for dev in lst: + with d.Context(name=dev.name.decode()) as ctx: + with PD(ctx) as pd: + mr_len = u.get_mr_length() + with MR(pd, mr_len, u.get_access_flags()) as mr: + write_len = min(random.randint(1, MAX_IO_LEN), mr_len) + write_str = u.get_data(write_len) + mr.write(write_str, write_len) + read_len = random.randint(1, write_len) + offset = random.randint(0, write_len-read_len) + read_str = mr.read(read_len, offset).decode() + assert read_str in write_str + + def test_lkey(self): + """ + Test reading lkey property + """ + lst = d.get_device_list() + for dev in lst: + with d.Context(name=dev.name.decode()) as ctx: + with PD(ctx) as pd: + length = u.get_mr_length() + with MR(pd, length, u.get_access_flags()) as mr: + lkey = mr.lkey + + def test_rkey(self): + """ + Test reading rkey property + """ + lst = d.get_device_list() + for dev in lst: + with d.Context(name=dev.name.decode()) as ctx: + with PD(ctx) as pd: + length = u.get_mr_length() + with MR(pd, length, u.get_access_flags()) as mr: + rkey = mr.rkey + + def test_buffer(self): + """ + Test reading buf property + """ + lst = d.get_device_list() + for dev in lst: + with d.Context(name=dev.name.decode()) as ctx: + with PD(ctx) as pd: + length = u.get_mr_length() + with MR(pd, length, u.get_access_flags()) as mr: + buf = mr.buf + diff --git a/pyverbs/tests/utils.py b/pyverbs/tests/utils.py new file mode 100644 index 000000000000..3281a326b324 --- /dev/null +++ b/pyverbs/tests/utils.py @@ -0,0 +1,28 @@ +# SPDX-License-Identifier: (GPL-2.0 OR Linux-OpenIB) +# Copyright (c) 2019, Mellanox Technologies. All rights reserved. See COPYING file +from string import ascii_lowercase as al +import random +import pyverbs.enums as e + +MAX_MR_SIZE = 4194304 + +def get_mr_length(): + # Allocating large buffers typically fails + return random.randint(0, MAX_MR_SIZE) + + +def get_access_flags(): + vals = list(e.ibv_access_flags) + selected = random.sample(vals, random.randint(1, 7)) + # Remote write / remote atomic are not allowed without local write + if e.IBV_ACCESS_REMOTE_WRITE in selected or e.IBV_ACCESS_REMOTE_ATOMIC in selected: + if not e.IBV_ACCESS_LOCAL_WRITE in selected: + selected.append(e.IBV_ACCESS_LOCAL_WRITE) + flags = 0 + for i in selected: + flags += i.value + return flags + + +def get_data(length): + return ''.join(random.choice(al) for i in range(length)) From patchwork Sun Feb 24 13:06:27 2019 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Noa Osherovich X-Patchwork-Id: 10827935 Return-Path: Received: from mail.wl.linuxfoundation.org (pdx-wl-mail.web.codeaurora.org [172.30.200.125]) by pdx-korg-patchwork-2.web.codeaurora.org (Postfix) with ESMTP id 973551575 for ; Sun, 24 Feb 2019 13:08:55 +0000 (UTC) Received: from mail.wl.linuxfoundation.org (localhost [127.0.0.1]) by mail.wl.linuxfoundation.org (Postfix) with ESMTP id 8382C2846C for ; Sun, 24 Feb 2019 13:08:55 +0000 (UTC) Received: by mail.wl.linuxfoundation.org (Postfix, from userid 486) id 77E222B4E5; Sun, 24 Feb 2019 13:08:55 +0000 (UTC) X-Spam-Checker-Version: SpamAssassin 3.3.1 (2010-03-16) on pdx-wl-mail.web.codeaurora.org X-Spam-Level: X-Spam-Status: No, score=-7.9 required=2.0 tests=BAYES_00,MAILING_LIST_MULTI, RCVD_IN_DNSWL_HI,UNPARSEABLE_RELAY autolearn=ham version=3.3.1 Received: from vger.kernel.org (vger.kernel.org [209.132.180.67]) by mail.wl.linuxfoundation.org (Postfix) with ESMTP id A17202B4FE for ; Sun, 24 Feb 2019 13:08:54 +0000 (UTC) Received: (majordomo@vger.kernel.org) by vger.kernel.org via listexpand id S1728287AbfBXNIx (ORCPT ); Sun, 24 Feb 2019 08:08:53 -0500 Received: from mail-il-dmz.mellanox.com ([193.47.165.129]:50492 "EHLO mellanox.co.il" rhost-flags-OK-OK-OK-FAIL) by vger.kernel.org with ESMTP id S1726865AbfBXNIx (ORCPT ); Sun, 24 Feb 2019 08:08:53 -0500 Received: from Internal Mail-Server by MTLPINE1 (envelope-from noaos@mellanox.com) with ESMTPS (AES256-SHA encrypted); 24 Feb 2019 15:06:49 +0200 Received: from reg-l-vrt-059-009.mtl.labs.mlnx (reg-l-vrt-059-009.mtl.labs.mlnx [10.135.59.9]) by labmailer.mlnx (8.13.8/8.13.8) with ESMTP id x1OD6mfc026210; Sun, 24 Feb 2019 15:06:48 +0200 From: Noa Osherovich To: leonro@mellanox.com, jgg@mellanox.com, dledford@redhat.com Cc: linux-rdma@vger.kernel.org, Noa Osherovich Subject: [PATCH rdma-core 08/19] pyverbs: Add support for extended query_device Date: Sun, 24 Feb 2019 15:06:27 +0200 Message-Id: <20190224130638.31848-9-noaos@mellanox.com> X-Mailer: git-send-email 2.17.2 In-Reply-To: <20190224130638.31848-1-noaos@mellanox.com> References: <20190224130638.31848-1-noaos@mellanox.com> Sender: linux-rdma-owner@vger.kernel.org Precedence: bulk List-ID: X-Mailing-List: linux-rdma@vger.kernel.org X-Virus-Scanned: ClamAV using ClamSMTP ibv_query_device_ex() is needed for extended capabilities checks. Signed-off-by: Noa Osherovich --- pyverbs/device.pxd | 24 ++++++ pyverbs/device.pyx | 165 ++++++++++++++++++++++++++++++++++++++++ pyverbs/libibverbs.pxd | 60 ++++++++++++++- pyverbs/tests/device.py | 42 +++++++++- 4 files changed, 286 insertions(+), 5 deletions(-) diff --git a/pyverbs/device.pxd b/pyverbs/device.pxd index 5f3ba4dfc6f8..c81e8176eef9 100644 --- a/pyverbs/device.pxd +++ b/pyverbs/device.pxd @@ -13,3 +13,27 @@ cdef class Context(PyverbsCM): cdef class DeviceAttr(PyverbsObject): cdef v.ibv_device_attr dev_attr + +cdef class QueryDeviceExInput(PyverbsObject): + cdef v.ibv_query_device_ex_input input + +cdef class ODPCaps(PyverbsObject): + cdef v.ibv_odp_caps odp_caps + +cdef class RSSCaps(PyverbsObject): + cdef v.ibv_rss_caps rss_caps + +cdef class PacketPacingCaps(PyverbsObject): + cdef v.ibv_packet_pacing_caps packet_pacing_caps + +cdef class TMCaps(PyverbsObject): + cdef v.ibv_tm_caps tm_caps + +cdef class CQModerationCaps(PyverbsObject): + cdef v.ibv_cq_moderation_caps cq_mod_caps + +cdef class TSOCaps(PyverbsObject): + cdef v.ibv_tso_caps tso_caps + +cdef class DeviceAttrEx(PyverbsObject): + cdef v.ibv_device_attr_ex dev_attr diff --git a/pyverbs/device.pyx b/pyverbs/device.pyx index 000fb53ff5a2..948909753738 100644 --- a/pyverbs/device.pyx +++ b/pyverbs/device.pyx @@ -131,6 +131,22 @@ cdef class Context(PyverbsCM): format(name=self.name), errno) return dev_attr + def query_device_ex(self, QueryDeviceExInput ex_input = None): + """ + Queries the device's extended attributes. + :param ex_input: An extensible input struct for possible future + extensions + :return: DeviceAttrEx object + """ + dev_attr_ex = DeviceAttrEx() + rc = v.ibv_query_device_ex(self.context, + &ex_input.input if ex_input is not None else NULL, + &dev_attr_ex.dev_attr) + if rc != 0: + raise PyverbsRDMAErrno('Failed to query EX device {name}'. + format(name=self.name)) + return dev_attr_ex + def query_gid(self, unsigned int port_num, int index): gid = GID() rc = v.ibv_query_gid(self.context, port_num, index, &gid.gid) @@ -312,6 +328,155 @@ cdef class DeviceAttr(PyverbsObject): print_format.format('local CA ack delay', self.local_ca_ack_delay) +\ print_format.format('Phys port count', self.phys_port_cnt) + +cdef class QueryDeviceExInput(PyverbsObject): + def __cinit__(self, comp_mask): + self.ex_input.comp_mask = comp_mask + + +cdef class ODPCaps(PyverbsObject): + @property + def general_caps(self): + return self.odp_caps.general_caps + @property + def rc_odp_caps(self): + return self.odp_caps.per_transport_caps.rc_odp_caps + @property + def uc_odp_caps(self): + return self.odp_caps.per_transport_caps.uc_odp_caps + @property + def ud_odp_caps(self): + return self.odp_caps.per_transport_caps.ud_odp_caps + + +cdef class TSOCaps(PyverbsObject): + @property + def max_tso(self): + return self.tso_caps.max_tso + @property + def supported_qpts(self): + return self.tso_caps.supported_qpts + + +cdef class RSSCaps(PyverbsObject): + @property + def supported_qpts(self): + return self.rss_caps.supported_qpts + @property + def max_rwq_indirection_tables(self): + return self.rss_caps.max_rwq_indirection_tables + @property + def rx_hash_fields_mask(self): + return self.rss_caps.rx_hash_fields_mask + @property + def rx_hash_function(self): + return self.rss_caps.rx_hash_function + @property + def max_rwq_indirection_table_size(self): + return self.rss_caps.max_rwq_indirection_table_size + + +cdef class PacketPacingCaps(PyverbsObject): + @property + def qp_rate_limit_min(self): + return self.packet_pacing_caps.qp_rate_limit_min + @property + def qp_rate_limit_max(self): + return self.packet_pacing_caps.qp_rate_limit_max + @property + def supported_qpts(self): + return self.packet_pacing_caps.supported_qpts + + +cdef class TMCaps(PyverbsObject): + @property + def max_rndv_hdr_size(self): + return self.tm_caps.max_rndv_hdr_size + @property + def max_num_tags(self): + return self.tm_caps.max_num_tags + @property + def flags(self): + return self.tm_caps.flags + @property + def max_ops(self): + return self.tm_caps.max_ops + @property + def max_sge(self): + return self.tm_caps.max_sge + + +cdef class CQModerationCaps(PyverbsObject): + @property + def max_cq_count(self): + return self.cq_mod_caps.max_cq_count + @property + def max_cq_period(self): + return self.cq_mod_caps.max_cq_period + + +cdef class DeviceAttrEx(PyverbsObject): + @property + def orig_attr(self): + attr = DeviceAttr() + attr.dev_attr = self.dev_attr.orig_attr + return attr + @property + def comp_mask(self): + return self.dev_attr.comp_mask + @comp_mask.setter + def comp_mask(self, val): + self.dev_attr.comp_mask = val + @property + def odp_caps(self): + caps = ODPCaps() + caps.odp_caps = self.dev_attr.odp_caps + return caps + @property + def completion_timestamp_mask(self): + return self.dev_attr.completion_timestamp_mask + @property + def hca_core_clock(self): + return self.dev_attr.hca_core_clock + @property + def device_cap_flags_ex(self): + return self.dev_attr.device_cap_flags_ex + @property + def tso_caps(self): + caps = TSOCaps() + caps.tso_caps = self.dev_attr.tso_caps + return caps + @property + def rss_caps(self): + caps = RSSCaps() + caps.rss_caps = self.dev_attr.rss_caps + return caps + @property + def max_wq_type_rq(self): + return self.dev_attr.max_wq_type_rq + @property + def packet_pacing_caps(self): + caps = PacketPacingCaps() + caps.packet_pacing_caps = self.dev_attr.packet_pacing_caps + return caps + @property + def raw_packet_caps(self): + return self.dev_attr.raw_packet_caps + @property + def tm_caps(self): + caps = TMCaps() + caps.tm_caps = self.dev_attr.tm_caps + return caps + @property + def cq_mod_caps(self): + caps = CQModerationCaps() + caps.cq_mod_caps = self.dev_attr.cq_mod_caps + return caps + @property + def max_dm_size(self): + return self.dev_attr.max_dm_size + + def guid_format(num): """ Get GUID representation of the given number, including change of endianness. diff --git a/pyverbs/libibverbs.pxd b/pyverbs/libibverbs.pxd index 5409ff04a649..ed179383af9a 100644 --- a/pyverbs/libibverbs.pxd +++ b/pyverbs/libibverbs.pxd @@ -2,7 +2,7 @@ # Copyright (c) 2018, Mellanox Technologies. All rights reserved. See COPYING file include 'libibverbs_enums.pxd' -from libc.stdint cimport uint8_t, uint64_t +from libc.stdint cimport uint8_t, uint32_t, uint64_t cdef extern from 'infiniband/verbs.h': @@ -77,11 +77,69 @@ cdef extern from 'infiniband/verbs.h': unsigned int lkey unsigned int rkey + cdef struct ibv_query_device_ex_input: + unsigned int comp_mask + + cdef struct per_transport_caps: + uint32_t rc_odp_caps + uint32_t uc_odp_caps + uint32_t ud_odp_caps + + cdef struct ibv_odp_caps: + uint64_t general_caps + per_transport_caps per_transport_caps + + cdef struct ibv_tso_caps: + unsigned int max_tso + unsigned int supported_qpts + + cdef struct ibv_rss_caps: + unsigned int supported_qpts + unsigned int max_rwq_indirection_tables + unsigned int max_rwq_indirection_table_size + unsigned long rx_hash_fields_mask + unsigned int rx_hash_function + + cdef struct ibv_packet_pacing_caps: + unsigned int qp_rate_limit_min + unsigned int qp_rate_limit_max + unsigned int supported_qpts + + cdef struct ibv_tm_caps: + unsigned int max_rndv_hdr_size + unsigned int max_num_tags + unsigned int flags + unsigned int max_ops + unsigned int max_sge + + cdef struct ibv_cq_moderation_caps: + unsigned int max_cq_count + unsigned int max_cq_period + + cdef struct ibv_device_attr_ex: + ibv_device_attr orig_attr + unsigned int comp_mask + ibv_odp_caps odp_caps + unsigned long completion_timestamp_mask + unsigned long hca_core_clock + unsigned long device_cap_flags_ex + ibv_tso_caps tso_caps + ibv_rss_caps rss_caps + unsigned int max_wq_type_rq + ibv_packet_pacing_caps packet_pacing_caps + unsigned int raw_packet_caps + ibv_tm_caps tm_caps + ibv_cq_moderation_caps cq_mod_caps + unsigned long max_dm_size + ibv_device **ibv_get_device_list(int *n) void ibv_free_device_list(ibv_device **list) ibv_context *ibv_open_device(ibv_device *device) int ibv_close_device(ibv_context *context) int ibv_query_device(ibv_context *context, ibv_device_attr *device_attr) + int ibv_query_device_ex(ibv_context *context, + ibv_query_device_ex_input *input, + ibv_device_attr_ex *attr) unsigned long ibv_get_device_guid(ibv_device *device) int ibv_query_gid(ibv_context *context, unsigned int port_num, int index, ibv_gid *gid) diff --git a/pyverbs/tests/device.py b/pyverbs/tests/device.py index fde359688cd5..bdfa81f0dc06 100644 --- a/pyverbs/tests/device.py +++ b/pyverbs/tests/device.py @@ -1,8 +1,14 @@ # SPDX-License-Identifier: (GPL-2.0 OR Linux-OpenIB) # Copyright (c) 2018, Mellanox Technologies. All rights reserved. See COPYING file import unittest +import resource + import pyverbs.device as d + +PAGE_SIZE = resource.getpagesize() + + class device_test(unittest.TestCase): """ Test various functionalities of the Device class. @@ -27,8 +33,9 @@ class device_test(unittest.TestCase): """ lst = d.get_device_list() for dev in lst: - ctx = d.Context(name=dev.name.decode()) - ctx.query_device() + with d.Context(name=dev.name.decode()) as ctx: + attr = ctx.query_device() + self.verify_device_attr(attr) def test_query_gid(self): """ @@ -36,6 +43,33 @@ class device_test(unittest.TestCase): """ lst = d.get_device_list() for dev in lst: - ctx = d.Context(name=dev.name.decode()) - ctx.query_gid(port_num=1, index=0) + with d.Context(name=dev.name.decode()) as ctx: + ctx.query_gid(port_num=1, index=0) + @staticmethod + def verify_device_attr(attr): + assert attr.node_guid != 0 + assert attr.sys_image_guid != 0 + assert attr.max_mr_size > PAGE_SIZE + assert attr.page_size_cap > PAGE_SIZE + assert attr.vendor_id != 0 + assert attr.vendor_part_id != 0 + assert attr.max_qp > 0 + assert attr.max_qp_wr > 0 + assert attr.max_sge > 0 + assert attr.max_sge_rd > 0 + assert attr.max_cq > 0 + assert attr.max_cqe > 0 + assert attr.max_mr > 0 + assert attr.max_pd > 0 + assert attr.max_pkeys > 0 + + def test_query_device_ex(self): + """ + Test ibv_query_device_ex() + """ + lst = d.get_device_list() + for dev in lst: + with d.Context(name=dev.name.decode()) as ctx: + attr_ex = ctx.query_device_ex() + self.verify_device_attr(attr_ex.orig_attr) From patchwork Sun Feb 24 13:06:28 2019 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Noa Osherovich X-Patchwork-Id: 10827943 Return-Path: Received: from mail.wl.linuxfoundation.org (pdx-wl-mail.web.codeaurora.org [172.30.200.125]) by pdx-korg-patchwork-2.web.codeaurora.org (Postfix) with ESMTP id 5179013B5 for ; Sun, 24 Feb 2019 13:08:57 +0000 (UTC) Received: from mail.wl.linuxfoundation.org (localhost [127.0.0.1]) by mail.wl.linuxfoundation.org (Postfix) with ESMTP id 39FEE28707 for ; Sun, 24 Feb 2019 13:08:57 +0000 (UTC) Received: by mail.wl.linuxfoundation.org (Postfix, from userid 486) id 2CD3528689; Sun, 24 Feb 2019 13:08:57 +0000 (UTC) X-Spam-Checker-Version: SpamAssassin 3.3.1 (2010-03-16) on pdx-wl-mail.web.codeaurora.org X-Spam-Level: X-Spam-Status: No, score=-7.9 required=2.0 tests=BAYES_00,MAILING_LIST_MULTI, RCVD_IN_DNSWL_HI,UNPARSEABLE_RELAY autolearn=ham version=3.3.1 Received: from vger.kernel.org (vger.kernel.org [209.132.180.67]) by mail.wl.linuxfoundation.org (Postfix) with ESMTP id A62F627968 for ; Sun, 24 Feb 2019 13:08:56 +0000 (UTC) Received: (majordomo@vger.kernel.org) by vger.kernel.org via listexpand id S1728059AbfBXNIz (ORCPT ); Sun, 24 Feb 2019 08:08:55 -0500 Received: from mail-il-dmz.mellanox.com ([193.47.165.129]:50519 "EHLO mellanox.co.il" rhost-flags-OK-OK-OK-FAIL) by vger.kernel.org with ESMTP id S1728269AbfBXNIy (ORCPT ); Sun, 24 Feb 2019 08:08:54 -0500 Received: from Internal Mail-Server by MTLPINE1 (envelope-from noaos@mellanox.com) with ESMTPS (AES256-SHA encrypted); 24 Feb 2019 15:06:49 +0200 Received: from reg-l-vrt-059-009.mtl.labs.mlnx (reg-l-vrt-059-009.mtl.labs.mlnx [10.135.59.9]) by labmailer.mlnx (8.13.8/8.13.8) with ESMTP id x1OD6mfd026210; Sun, 24 Feb 2019 15:06:49 +0200 From: Noa Osherovich To: leonro@mellanox.com, jgg@mellanox.com, dledford@redhat.com Cc: linux-rdma@vger.kernel.org, Noa Osherovich Subject: [PATCH rdma-core 09/19] pyverbs: Add support for memory window creation Date: Sun, 24 Feb 2019 15:06:28 +0200 Message-Id: <20190224130638.31848-10-noaos@mellanox.com> X-Mailer: git-send-email 2.17.2 In-Reply-To: <20190224130638.31848-1-noaos@mellanox.com> References: <20190224130638.31848-1-noaos@mellanox.com> Sender: linux-rdma-owner@vger.kernel.org Precedence: bulk List-ID: X-Mailing-List: linux-rdma@vger.kernel.org X-Virus-Scanned: ClamAV using ClamSMTP Memory windows allow user applications a more flexible control over remote access to its memory. Add needed support create them using pyverbs. Signed-off-by: Noa Osherovich --- pyverbs/libibverbs.pxd | 9 +++++++++ pyverbs/mr.pxd | 4 ++++ pyverbs/mr.pyx | 45 ++++++++++++++++++++++++++++++++++++++++++ pyverbs/pd.pxd | 1 + pyverbs/pd.pyx | 7 +++++-- 5 files changed, 64 insertions(+), 2 deletions(-) diff --git a/pyverbs/libibverbs.pxd b/pyverbs/libibverbs.pxd index ed179383af9a..7eb1df2407c9 100644 --- a/pyverbs/libibverbs.pxd +++ b/pyverbs/libibverbs.pxd @@ -132,6 +132,13 @@ cdef extern from 'infiniband/verbs.h': ibv_cq_moderation_caps cq_mod_caps unsigned long max_dm_size + cdef struct ibv_mw: + ibv_context *context + ibv_pd *pd + unsigned int rkey + unsigned int handle + ibv_mw_type mw_type + ibv_device **ibv_get_device_list(int *n) void ibv_free_device_list(ibv_device **list) ibv_context *ibv_open_device(ibv_device *device) @@ -147,3 +154,5 @@ cdef extern from 'infiniband/verbs.h': int ibv_dealloc_pd(ibv_pd *pd) ibv_mr *ibv_reg_mr(ibv_pd *pd, void *addr, size_t length, int access) int ibv_dereg_mr(ibv_mr *mr) + ibv_mw *ibv_alloc_mw(ibv_pd *pd, ibv_mw_type type) + int ibv_dealloc_mw(ibv_mw *mw) diff --git a/pyverbs/mr.pxd b/pyverbs/mr.pxd index 92578021c3b3..91d662070135 100644 --- a/pyverbs/mr.pxd +++ b/pyverbs/mr.pxd @@ -10,3 +10,7 @@ cdef class MR(PyverbsCM): cdef v.ibv_mr *mr cdef void *buf cpdef read(self, length, offset) + +cdef class MW(PyverbsCM): + cdef object pd + cdef v.ibv_mw *mw diff --git a/pyverbs/mr.pyx b/pyverbs/mr.pyx index 31949e739376..1ab4bd809dee 100644 --- a/pyverbs/mr.pyx +++ b/pyverbs/mr.pyx @@ -107,3 +107,48 @@ cdef class MR(PyverbsCM): @property def rkey(self): return self.mr.rkey + + +cdef class MW(PyverbsCM): + def __cinit__(self, PD pd not None, v.ibv_mw_type mw_type): + """ + Initializes a memory window object of the given type + :param pd: A PD object + :param mw_type: Type of of the memory window, see ibv_mw_type enum + :return: + """ + self.mw = NULL + self.mw = v.ibv_alloc_mw(pd.pd, mw_type) + if self.mw == NULL: + raise PyverbsRDMAErrno('Failed to allocate MW') + self.pd = pd + pd.add_ref(self) + self.logger.debug('Allocated memory window of type {t}'. + format(t=mwtype2str(mw_type))) + + def __dealloc__(self): + self.close() + + cpdef close(self): + """ + Closes the underlaying C MW object. + MW may be deleted directly or by deleting its PD, which leaves the + Python object without the underlaying MW. + Need to check that the underlaying MW wasn't dealloced before. + :return: None + """ + self.logger.debug('Closing MW') + if self.mw is not NULL: + rc = v.ibv_dealloc_mw(self.mw) + if rc != 0: + raise PyverbsRDMAErrno('Failed to dealloc MW') + self.mw = NULL + self.pd = None + + +def mwtype2str(mw_type): + mw_types = {1:'IBV_MW_TYPE_1', 2:'IBV_MW_TYPE_2'} + try: + return mw_types[mw_type] + except KeyError: + return 'Unknown MW type ({t})'.format(t=mw_type) diff --git a/pyverbs/pd.pxd b/pyverbs/pd.pxd index 8381662b21be..fc2e405600d0 100644 --- a/pyverbs/pd.pxd +++ b/pyverbs/pd.pxd @@ -10,3 +10,4 @@ cdef class PD(PyverbsCM): cdef Context ctx cdef add_ref(self, obj) cdef object mrs + cdef object mws diff --git a/pyverbs/pd.pyx b/pyverbs/pd.pyx index a5fc063751e1..da580423cd81 100644 --- a/pyverbs/pd.pyx +++ b/pyverbs/pd.pyx @@ -4,7 +4,7 @@ import weakref from pyverbs.pyverbs_error import PyverbsRDMAError, PyverbsError from pyverbs.base import PyverbsRDMAErrno -from .mr cimport MR +from .mr cimport MR, MW cdef extern from 'errno.h': int errno @@ -25,6 +25,7 @@ cdef class PD(PyverbsCM): context.add_ref(self) self.logger.debug('PD: Allocated ibv_pd') self.mrs = weakref.WeakSet() + self.mws = weakref.WeakSet() def __dealloc__(self): """ @@ -42,7 +43,7 @@ cdef class PD(PyverbsCM): :return: None """ self.logger.debug('Closing PD') - self.close_weakrefs([self.mrs]) + self.close_weakrefs([self.mws, self.mrs]) if self.pd != NULL: rc = v.ibv_dealloc_pd(self.pd) if rc != 0: @@ -53,5 +54,7 @@ cdef class PD(PyverbsCM): cdef add_ref(self, obj): if isinstance(obj, MR): self.mrs.add(obj) + elif isinstance(obj, MW): + self.mws.add(obj) else: raise PyverbsError('Unrecognized object type') From patchwork Sun Feb 24 13:06:29 2019 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Noa Osherovich X-Patchwork-Id: 10827941 Return-Path: Received: from mail.wl.linuxfoundation.org (pdx-wl-mail.web.codeaurora.org [172.30.200.125]) by pdx-korg-patchwork-2.web.codeaurora.org (Postfix) with ESMTP id 3E1D717E9 for ; Sun, 24 Feb 2019 13:08:57 +0000 (UTC) Received: from mail.wl.linuxfoundation.org (localhost [127.0.0.1]) by mail.wl.linuxfoundation.org (Postfix) with ESMTP id 9108B20881 for ; Sun, 24 Feb 2019 13:08:56 +0000 (UTC) Received: by mail.wl.linuxfoundation.org (Postfix, from userid 486) id 848B4212DA; Sun, 24 Feb 2019 13:08:56 +0000 (UTC) X-Spam-Checker-Version: SpamAssassin 3.3.1 (2010-03-16) on pdx-wl-mail.web.codeaurora.org X-Spam-Level: X-Spam-Status: No, score=-7.9 required=2.0 tests=BAYES_00,MAILING_LIST_MULTI, RCVD_IN_DNSWL_HI,UNPARSEABLE_RELAY autolearn=ham version=3.3.1 Received: from vger.kernel.org (vger.kernel.org [209.132.180.67]) by mail.wl.linuxfoundation.org (Postfix) with ESMTP id 278C71FFE5 for ; Sun, 24 Feb 2019 13:08:56 +0000 (UTC) Received: (majordomo@vger.kernel.org) by vger.kernel.org via listexpand id S1728295AbfBXNIy (ORCPT ); Sun, 24 Feb 2019 08:08:54 -0500 Received: from mail-il-dmz.mellanox.com ([193.47.165.129]:50533 "EHLO mellanox.co.il" rhost-flags-OK-OK-OK-FAIL) by vger.kernel.org with ESMTP id S1728059AbfBXNIy (ORCPT ); Sun, 24 Feb 2019 08:08:54 -0500 Received: from Internal Mail-Server by MTLPINE1 (envelope-from noaos@mellanox.com) with ESMTPS (AES256-SHA encrypted); 24 Feb 2019 15:06:49 +0200 Received: from reg-l-vrt-059-009.mtl.labs.mlnx (reg-l-vrt-059-009.mtl.labs.mlnx [10.135.59.9]) by labmailer.mlnx (8.13.8/8.13.8) with ESMTP id x1OD6mfe026210; Sun, 24 Feb 2019 15:06:49 +0200 From: Noa Osherovich To: leonro@mellanox.com, jgg@mellanox.com, dledford@redhat.com Cc: linux-rdma@vger.kernel.org, Noa Osherovich Subject: [PATCH rdma-core 10/19] pyverbs: Add unittests for memory window control path Date: Sun, 24 Feb 2019 15:06:29 +0200 Message-Id: <20190224130638.31848-11-noaos@mellanox.com> X-Mailer: git-send-email 2.17.2 In-Reply-To: <20190224130638.31848-1-noaos@mellanox.com> References: <20190224130638.31848-1-noaos@mellanox.com> Sender: linux-rdma-owner@vger.kernel.org Precedence: bulk List-ID: X-Mailing-List: linux-rdma@vger.kernel.org X-Virus-Scanned: ClamAV using ClamSMTP This patch tests creation and destruction of memory window. Signed-off-by: Noa Osherovich --- pyverbs/tests/mr.py | 41 +++++++++++++++++++++++++++++++++++++++-- 1 file changed, 39 insertions(+), 2 deletions(-) diff --git a/pyverbs/tests/mr.py b/pyverbs/tests/mr.py index 409cd41e3c06..c97c6ac38d6d 100644 --- a/pyverbs/tests/mr.py +++ b/pyverbs/tests/mr.py @@ -3,11 +3,11 @@ import unittest import random -from pyverbs.pyverbs_error import PyverbsRDMAError +from pyverbs.pyverbs_error import PyverbsRDMAError, PyverbsError from pyverbs.base import PyverbsRDMAErrno import pyverbs.tests.utils as u +from pyverbs.mr import MR, MW import pyverbs.device as d -from pyverbs.mr import MR from pyverbs.pd import PD import pyverbs.enums as e @@ -144,3 +144,40 @@ class mr_test(unittest.TestCase): with MR(pd, length, u.get_access_flags()) as mr: buf = mr.buf + +class mw_test(unittest.TestCase): + """ + Test various functionalities of the MW class. + """ + def test_reg_mw(self): + """ Test ibv_alloc_mw() """ + lst = d.get_device_list() + for dev in lst: + with d.Context(name=dev.name.decode()) as ctx: + with PD(ctx) as pd: + with MW(pd, random.choice([e.IBV_MW_TYPE_1, e.IBV_MW_TYPE_2])) as mw: + pass + + def test_dereg_mw(self): + """ Test ibv_dealloc_mw() """ + lst = d.get_device_list() + for dev in lst: + with d.Context(name=dev.name.decode()) as ctx: + with PD(ctx) as pd: + with MW(pd, random.choice([e.IBV_MW_TYPE_1, e.IBV_MW_TYPE_2])) as mw: + mw.close() + + def test_reg_mw_wrong_type(self): + """ Test ibv_alloc_mw() """ + lst = d.get_device_list() + for dev in lst: + with d.Context(name=dev.name.decode()) as ctx: + with PD(ctx) as pd: + try: + mw_type =random.randint(3, 100) + mw = MW(pd, mw_type) + except PyverbsRDMAError as e: + pass + else: + raise PyverbsError('Created a MW with type {t}'.\ + format(t=mw_type)) From patchwork Sun Feb 24 13:06:30 2019 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Noa Osherovich X-Patchwork-Id: 10827945 Return-Path: Received: from mail.wl.linuxfoundation.org (pdx-wl-mail.web.codeaurora.org [172.30.200.125]) by pdx-korg-patchwork-2.web.codeaurora.org (Postfix) with ESMTP id D63FD1575 for ; Sun, 24 Feb 2019 13:08:58 +0000 (UTC) Received: from mail.wl.linuxfoundation.org (localhost [127.0.0.1]) by mail.wl.linuxfoundation.org (Postfix) with ESMTP id BBF772A007 for ; Sun, 24 Feb 2019 13:08:58 +0000 (UTC) Received: by mail.wl.linuxfoundation.org (Postfix, from userid 486) id AD12F29E42; Sun, 24 Feb 2019 13:08:58 +0000 (UTC) X-Spam-Checker-Version: SpamAssassin 3.3.1 (2010-03-16) on pdx-wl-mail.web.codeaurora.org X-Spam-Level: X-Spam-Status: No, score=-7.9 required=2.0 tests=BAYES_00,MAILING_LIST_MULTI, RCVD_IN_DNSWL_HI,UNPARSEABLE_RELAY autolearn=ham version=3.3.1 Received: from vger.kernel.org (vger.kernel.org [209.132.180.67]) by mail.wl.linuxfoundation.org (Postfix) with ESMTP id C2822291B4 for ; Sun, 24 Feb 2019 13:08:57 +0000 (UTC) Received: (majordomo@vger.kernel.org) by vger.kernel.org via listexpand id S1728172AbfBXNI4 (ORCPT ); Sun, 24 Feb 2019 08:08:56 -0500 Received: from mail-il-dmz.mellanox.com ([193.47.165.129]:50546 "EHLO mellanox.co.il" rhost-flags-OK-OK-OK-FAIL) by vger.kernel.org with ESMTP id S1728248AbfBXNI4 (ORCPT ); Sun, 24 Feb 2019 08:08:56 -0500 Received: from Internal Mail-Server by MTLPINE1 (envelope-from noaos@mellanox.com) with ESMTPS (AES256-SHA encrypted); 24 Feb 2019 15:06:49 +0200 Received: from reg-l-vrt-059-009.mtl.labs.mlnx (reg-l-vrt-059-009.mtl.labs.mlnx [10.135.59.9]) by labmailer.mlnx (8.13.8/8.13.8) with ESMTP id x1OD6mff026210; Sun, 24 Feb 2019 15:06:49 +0200 From: Noa Osherovich To: leonro@mellanox.com, jgg@mellanox.com, dledford@redhat.com Cc: linux-rdma@vger.kernel.org, Noa Osherovich , Daria Velikovsky Subject: [PATCH rdma-core 11/19] pyverbs: Add support for direct memory usage Date: Sun, 24 Feb 2019 15:06:30 +0200 Message-Id: <20190224130638.31848-12-noaos@mellanox.com> X-Mailer: git-send-email 2.17.2 In-Reply-To: <20190224130638.31848-1-noaos@mellanox.com> References: <20190224130638.31848-1-noaos@mellanox.com> Sender: linux-rdma-owner@vger.kernel.org Precedence: bulk List-ID: X-Mailing-List: linux-rdma@vger.kernel.org X-Virus-Scanned: ClamAV using ClamSMTP Expose DM and DMMR classes. DM represents ibv_dm. DMMR is similar to a regular MR but its buffer is allocated on the device and not by the OS. Signed-off-by: Daria Velikovsky Signed-off-by: Noa Osherovich --- pyverbs/device.pxd | 10 ++++ pyverbs/device.pyx | 112 ++++++++++++++++++++++++++++++++++++++++- pyverbs/libibverbs.pxd | 17 +++++++ pyverbs/mr.pxd | 3 ++ pyverbs/mr.pyx | 43 +++++++++++++++- pyverbs/pd.pyx | 5 +- 6 files changed, 185 insertions(+), 5 deletions(-) diff --git a/pyverbs/device.pxd b/pyverbs/device.pxd index c81e8176eef9..c9e346539e58 100644 --- a/pyverbs/device.pxd +++ b/pyverbs/device.pxd @@ -10,6 +10,7 @@ cdef class Context(PyverbsCM): cdef object name cdef add_ref(self, obj) cdef object pds + cdef object dms cdef class DeviceAttr(PyverbsObject): cdef v.ibv_device_attr dev_attr @@ -37,3 +38,12 @@ cdef class TSOCaps(PyverbsObject): cdef class DeviceAttrEx(PyverbsObject): cdef v.ibv_device_attr_ex dev_attr + +cdef class AllocDmAttr(PyverbsObject): + cdef v.ibv_alloc_dm_attr alloc_dm_attr + +cdef class DM(PyverbsCM): + cdef v.ibv_dm *dm + cdef object dm_mrs + cdef object context + cdef add_ref(self, obj) diff --git a/pyverbs/device.pyx b/pyverbs/device.pyx index 948909753738..7bfcda8bc177 100644 --- a/pyverbs/device.pyx +++ b/pyverbs/device.pyx @@ -13,13 +13,21 @@ from .pyverbs_error import PyverbsUserError from pyverbs.base import PyverbsRDMAErrno cimport pyverbs.libibverbs as v from pyverbs.addr cimport GID +from pyverbs.mr import DMMR from pyverbs.pd cimport PD cdef extern from 'errno.h': int errno - cdef extern from 'endian.h': unsigned long be64toh(unsigned long host_64bits); +cdef extern from 'stdlib.h': + void free(void *ptr) +cdef extern from 'string.h': + void *memset(void *s, int c, size_t n) +cdef extern from 'malloc.h': + void *malloc(size_t size) +cdef extern from 'stdint.h': + ctypedef int uint64_t class Device(PyverbsObject): @@ -75,6 +83,8 @@ cdef class Context(PyverbsCM): cdef v.ibv_device **dev_list self.pds = weakref.WeakSet() + self.dms = weakref.WeakSet() + dev_name = kwargs.get('name') if dev_name is not None: @@ -110,7 +120,7 @@ cdef class Context(PyverbsCM): cpdef close(self): self.logger.debug('Closing Context') - self.close_weakrefs([self.pds]) + self.close_weakrefs([self.dms, self.pds]) if self.context != NULL: rc = v.ibv_close_device(self.context) if rc != 0: @@ -158,6 +168,8 @@ cdef class Context(PyverbsCM): cdef add_ref(self, obj): if isinstance(obj, PD): self.pds.add(obj) + elif isinstance(obj, DM): + self.dms.add(obj) else: raise PyverbsError('Unrecognized object type') @@ -477,6 +489,102 @@ cdef class DeviceAttrEx(PyverbsObject): return self.dev_attr.max_dm_size +cdef class AllocDmAttr(PyverbsObject): + def __cinit__(self, length, log_align_req = 0, comp_mask = 0): + """ + Creates an AllocDmAttr object with the given parameters. This object + can than be used to create a DM object. + :param length: Length of the future device memory + :param log_align_req: log2 of address alignment requirement + :param comp_mask: compatibility mask + :return: An AllocDmAttr object + """ + self.alloc_dm_attr.length = length + self.alloc_dm_attr.log_align_req = log_align_req + self.alloc_dm_attr.comp_mask = comp_mask + + @property + def length(self): + return self.alloc_dm_attr.length + + @length.setter + def length(self, val): + self.alloc_dm_attr.length = val + + @property + def log_align_req(self): + return self.alloc_dm_attr.log_align_req + + @log_align_req.setter + def log_align_req(self, val): + self.alloc_dm_attr.log_align_req = val + + @property + def comp_mask(self): + return self.alloc_dm_attr.comp_mask + + @comp_mask.setter + def comp_mask(self, val): + self.alloc_dm_attr.comp_mask = val + + +cdef class DM(PyverbsCM): + def __cinit__(self, Context context, AllocDmAttr dm_attr not None): + """ + Allocate a device (direct) memory. + :param context: The context of the device on which to allocate memory + :param dm_attr: Attributes that define the DM + :return: A DM object on success + """ + self.dm_mrs = weakref.WeakSet() + device_attr = context.query_device_ex() + if device_attr.max_dm_size <= 0: + raise PyverbsUserError('Device doesn\'t support dm allocation') + self.dm = v.ibv_alloc_dm(context.context, + &dm_attr.alloc_dm_attr) + if self.dm == NULL: + raise PyverbsRDMAErrno('Failed to allocate device memory of size ' + '{size}. Max available size {max}.' + .format(size=dm_attr.length, + max=device_attr.max_dm_size)) + self.context = context + context.add_ref(self) + + def __dealloc__(self): + self.close() + + cpdef close(self): + self.logger.debug('Closing DM') + self.close_weakrefs([self.dm_mrs]) + if self.dm != NULL: + rc = v.ibv_free_dm(self.dm) + if rc != 0: + raise PyverbsRDMAErrno('Failed to free dm') + self.dm = NULL + self.context = None + + cdef add_ref(self, obj): + if isinstance(obj, DMMR): + self.dm_mrs.add(obj) + + def copy_to_dm(self, dm_offset, data, length): + rc = v.ibv_memcpy_to_dm(self.dm, dm_offset, + data, length) + if rc != 0: + raise PyverbsRDMAErrno('Failed to copy to dm') + + def copy_from_dm(self, dm_offset, length): + cdef char *data =malloc(length) + memset(data, 0, length) + rc = v.ibv_memcpy_from_dm(data, self.dm, + dm_offset, length) + if rc != 0: + raise PyverbsRDMAErrno('Failed to copy from dm') + res = data[:length] + free(data) + return res + + def guid_format(num): """ Get GUID representation of the given number, including change of endianness. diff --git a/pyverbs/libibverbs.pxd b/pyverbs/libibverbs.pxd index 7eb1df2407c9..5dd3ba013e5e 100644 --- a/pyverbs/libibverbs.pxd +++ b/pyverbs/libibverbs.pxd @@ -139,6 +139,15 @@ cdef extern from 'infiniband/verbs.h': unsigned int handle ibv_mw_type mw_type + cdef struct ibv_alloc_dm_attr: + size_t length + unsigned int log_align_req + unsigned int comp_mask + + cdef struct ibv_dm: + ibv_context *context + unsigned int comp_mask + ibv_device **ibv_get_device_list(int *n) void ibv_free_device_list(ibv_device **list) ibv_context *ibv_open_device(ibv_device *device) @@ -156,3 +165,11 @@ cdef extern from 'infiniband/verbs.h': int ibv_dereg_mr(ibv_mr *mr) ibv_mw *ibv_alloc_mw(ibv_pd *pd, ibv_mw_type type) int ibv_dealloc_mw(ibv_mw *mw) + ibv_dm *ibv_alloc_dm(ibv_context *context, ibv_alloc_dm_attr *attr) + int ibv_free_dm(ibv_dm *dm) + ibv_mr *ibv_reg_dm_mr(ibv_pd *pd, ibv_dm *dm, unsigned long dm_offset, + size_t length, unsigned int access) + int ibv_memcpy_to_dm(ibv_dm *dm, unsigned long dm_offset, void *host_addr, + size_t length) + int ibv_memcpy_from_dm(void *host_addr, ibv_dm *dm, unsigned long dm_offset, + size_t length) diff --git a/pyverbs/mr.pxd b/pyverbs/mr.pxd index 91d662070135..2d76f2dfbe7c 100644 --- a/pyverbs/mr.pxd +++ b/pyverbs/mr.pxd @@ -14,3 +14,6 @@ cdef class MR(PyverbsCM): cdef class MW(PyverbsCM): cdef object pd cdef v.ibv_mw *mw + +cdef class DMMR(MR): + cdef object dm diff --git a/pyverbs/mr.pyx b/pyverbs/mr.pyx index 1ab4bd809dee..92b195f75ee2 100644 --- a/pyverbs/mr.pyx +++ b/pyverbs/mr.pyx @@ -3,6 +3,7 @@ from pyverbs.pyverbs_error import PyverbsRDMAError, PyverbsError from pyverbs.base import PyverbsRDMAErrno +from pyverbs.device cimport DM from .pd cimport PD import resource @@ -22,7 +23,7 @@ cdef class MR(PyverbsCM): MR class represents ibv_mr. Buffer allocation in done in the c'tor. Freeing it is done in close(). """ - def __cinit__(self, PD pd not None, length, access): + def __cinit__(self, PD pd not None, length, access, **kwargs): """ Allocate a user-level buffer of length and register a Memory Region of the given length and access flags. @@ -31,6 +32,8 @@ cdef class MR(PyverbsCM): :param access: Access flags, see ibv_access_flags enum :return: The newly created MR on success """ + if len(kwargs) != 0: + return #We want to enable registering an MR of size 0 but this fails with a #buffer of size 0, so in this case lets increase the buffer if length == 0: @@ -146,6 +149,44 @@ cdef class MW(PyverbsCM): self.pd = None +cdef class DMMR(MR): + def __cinit__(self, PD pd not None, length, access, **kwargs): + """ + Initializes a DMMR (Device Memory Memory Region) of the given length + and access flags using the given PD and DM objects. + :param pd: A PD object + :param length: Length in bytes + :param access: Access flags, see ibv_access_flags enum + :param kwargs: see below + :return: The newly create DMMR + + :keyword Arguments: + * *dm* (DM) + A DM (device memory) object to be used for this DMMR + * *offset* + Byte offset from the beginning of the allocated device memory + buffer + """ + dm = kwargs['dm'] + offset = kwargs['offset'] + self.mr = v.ibv_reg_dm_mr(pd.pd, (dm).dm, offset, length, access) + if self.mr == NULL: + raise PyverbsRDMAErrno('Failed to register a device MR. length: {len}, access flags: {flags}'. + format(len=length, flags=access,)) + self.pd = pd + self.dm = dm + pd.add_ref(self) + dm.add_ref(self) + self.logger.debug('Registered device ibv_mr. Length: {len}, access flags {flags}'. + format(len=length, flags=access)) + + def write(self, data, length): + return self.dm.copy_to_dm(0, data, length) + + cpdef read(self, length, offset): + return self.dm.copy_from_dm(offset, length) + + def mwtype2str(mw_type): mw_types = {1:'IBV_MW_TYPE_1', 2:'IBV_MW_TYPE_2'} try: diff --git a/pyverbs/pd.pyx b/pyverbs/pd.pyx index da580423cd81..65cc851b0ecf 100644 --- a/pyverbs/pd.pyx +++ b/pyverbs/pd.pyx @@ -4,7 +4,8 @@ import weakref from pyverbs.pyverbs_error import PyverbsRDMAError, PyverbsError from pyverbs.base import PyverbsRDMAErrno -from .mr cimport MR, MW +from pyverbs.device cimport Context, DM +from .mr cimport MR, MW, DMMR cdef extern from 'errno.h': int errno @@ -52,7 +53,7 @@ cdef class PD(PyverbsCM): self.ctx = None cdef add_ref(self, obj): - if isinstance(obj, MR): + if isinstance(obj, MR) or isinstance(obj, DMMR): self.mrs.add(obj) elif isinstance(obj, MW): self.mws.add(obj) From patchwork Sun Feb 24 13:06:31 2019 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Noa Osherovich X-Patchwork-Id: 10827953 Return-Path: Received: from mail.wl.linuxfoundation.org (pdx-wl-mail.web.codeaurora.org [172.30.200.125]) by pdx-korg-patchwork-2.web.codeaurora.org (Postfix) with ESMTP id 081071575 for ; Sun, 24 Feb 2019 13:09:01 +0000 (UTC) Received: from mail.wl.linuxfoundation.org (localhost [127.0.0.1]) by mail.wl.linuxfoundation.org (Postfix) with ESMTP id E821F296ED for ; Sun, 24 Feb 2019 13:09:00 +0000 (UTC) Received: by mail.wl.linuxfoundation.org (Postfix, from userid 486) id DA8372AA01; Sun, 24 Feb 2019 13:09:00 +0000 (UTC) X-Spam-Checker-Version: SpamAssassin 3.3.1 (2010-03-16) on pdx-wl-mail.web.codeaurora.org X-Spam-Level: X-Spam-Status: No, score=-7.9 required=2.0 tests=BAYES_00,MAILING_LIST_MULTI, RCVD_IN_DNSWL_HI,UNPARSEABLE_RELAY autolearn=ham version=3.3.1 Received: from vger.kernel.org (vger.kernel.org [209.132.180.67]) by mail.wl.linuxfoundation.org (Postfix) with ESMTP id 2C623296ED for ; Sun, 24 Feb 2019 13:09:00 +0000 (UTC) Received: (majordomo@vger.kernel.org) by vger.kernel.org via listexpand id S1728318AbfBXNI5 (ORCPT ); Sun, 24 Feb 2019 08:08:57 -0500 Received: from mail-il-dmz.mellanox.com ([193.47.165.129]:50556 "EHLO mellanox.co.il" rhost-flags-OK-OK-OK-FAIL) by vger.kernel.org with ESMTP id S1728307AbfBXNI4 (ORCPT ); Sun, 24 Feb 2019 08:08:56 -0500 Received: from Internal Mail-Server by MTLPINE1 (envelope-from noaos@mellanox.com) with ESMTPS (AES256-SHA encrypted); 24 Feb 2019 15:06:49 +0200 Received: from reg-l-vrt-059-009.mtl.labs.mlnx (reg-l-vrt-059-009.mtl.labs.mlnx [10.135.59.9]) by labmailer.mlnx (8.13.8/8.13.8) with ESMTP id x1OD6mfg026210; Sun, 24 Feb 2019 15:06:49 +0200 From: Noa Osherovich To: leonro@mellanox.com, jgg@mellanox.com, dledford@redhat.com Cc: linux-rdma@vger.kernel.org, Noa Osherovich Subject: [PATCH rdma-core 12/19] pyverbs: Add direct memory related unittests Date: Sun, 24 Feb 2019 15:06:31 +0200 Message-Id: <20190224130638.31848-13-noaos@mellanox.com> X-Mailer: git-send-email 2.17.2 In-Reply-To: <20190224130638.31848-1-noaos@mellanox.com> References: <20190224130638.31848-1-noaos@mellanox.com> Sender: linux-rdma-owner@vger.kernel.org Precedence: bulk List-ID: X-Mailing-List: linux-rdma@vger.kernel.org X-Virus-Scanned: ClamAV using ClamSMTP Add tests for direct memory creation as well as DMMR. Signed-off-by: Noa Osherovich --- pyverbs/tests/device.py | 147 +++++++++++++++++++++++++++++++++++++++- pyverbs/tests/mr.py | 48 ++++++++++++- pyverbs/tests/utils.py | 16 +++++ 3 files changed, 209 insertions(+), 2 deletions(-) diff --git a/pyverbs/tests/device.py b/pyverbs/tests/device.py index bdfa81f0dc06..4a5e0a794069 100644 --- a/pyverbs/tests/device.py +++ b/pyverbs/tests/device.py @@ -2,10 +2,12 @@ # Copyright (c) 2018, Mellanox Technologies. All rights reserved. See COPYING file import unittest import resource +import random +from pyverbs.pyverbs_error import PyverbsError, PyverbsRDMAError +import pyverbs.tests.utils as u import pyverbs.device as d - PAGE_SIZE = resource.getpagesize() @@ -73,3 +75,146 @@ class device_test(unittest.TestCase): with d.Context(name=dev.name.decode()) as ctx: attr_ex = ctx.query_device_ex() self.verify_device_attr(attr_ex.orig_attr) + + +class dm_test(unittest.TestCase): + """ + Test various functionalities of the DM class. + """ + def test_create_dm(self): + """ + test ibv_alloc_dm() + """ + lst = d.get_device_list() + for dev in lst: + with d.Context(name=dev.name.decode()) as ctx: + attr = ctx.query_device_ex() + if attr.max_dm_size == 0: + return + dm_len = random.randrange(u.MIN_DM_SIZE, attr.max_dm_size, + u.DM_ALIGNMENT) + dm_attrs = u.get_dm_attrs(dm_len) + with d.DM(ctx, dm_attrs): + pass + + def test_destroy_dm(self): + """ + test ibv_free_dm() + """ + lst = d.get_device_list() + for dev in lst: + with d.Context(name=dev.name.decode()) as ctx: + attr = ctx.query_device_ex() + if attr.max_dm_size == 0: + return + dm_len = random.randrange(u.MIN_DM_SIZE, attr.max_dm_size, + u.DM_ALIGNMENT) + dm_attrs = u.get_dm_attrs(dm_len) + dm = d.DM(ctx, dm_attrs) + dm.close() + + def test_create_dm_bad_flow(self): + """ + test ibv_alloc_dm() with an illegal size and comp mask + """ + lst = d.get_device_list() + for dev in lst: + with d.Context(name=dev.name.decode()) as ctx: + attr = ctx.query_device_ex() + if attr.max_dm_size == 0: + return + dm_len = attr.max_dm_size + 1 + dm_attrs = u.get_dm_attrs(dm_len) + try: + dm = d.DM(ctx, dm_attrs) + except PyverbsRDMAError as e: + assert 'Failed to allocate device memory of size' in e.args[0] + assert 'Max available size' in e.args[0] + else: + raise PyverbsError('Created a DM with size larger than max reported') + dm_attrs.comp_mask = random.randint(1, 100) + try: + dm = d.DM(ctx, dm_attrs) + except PyverbsRDMAError as e: + assert 'Failed to allocate device memory of size' in e.args[0] + else: + raise PyverbsError('Created a DM with illegal comp mask {c}'.\ + format(c=dm_attrs.comp_mask)) + + def test_destroy_dm_bad_flow(self): + """ + test calling ibv_free_dm() twice + """ + lst = d.get_device_list() + for dev in lst: + with d.Context(name=dev.name.decode()) as ctx: + attr = ctx.query_device_ex() + if attr.max_dm_size == 0: + return + dm_len = random.randrange(u.MIN_DM_SIZE, attr.max_dm_size, u.DM_ALIGNMENT) + dm_attrs = u.get_dm_attrs(dm_len) + dm = d.DM(ctx, dm_attrs) + dm.close() + dm.close() + + def test_dm_write(self): + """ + Test writing to the device memory + """ + lst = d.get_device_list() + for dev in lst: + with d.Context(name=dev.name.decode()) as ctx: + attr = ctx.query_device_ex() + if attr.max_dm_size == 0: + return + dm_len = random.randrange(u.MIN_DM_SIZE, attr.max_dm_size, u.DM_ALIGNMENT) + dm_attrs = u.get_dm_attrs(dm_len) + with d.DM(ctx, dm_attrs) as dm: + data_length = random.randrange(4, dm_len, u.DM_ALIGNMENT) + data_offset = random.randrange(0, dm_len - data_length, u.DM_ALIGNMENT) + data = u.get_data(data_length) + dm.copy_to_dm(data_offset, data.encode(), data_length) + + def test_dm_write_bad_flow(self): + """ + Test writing to the device memory with bad offset and length + """ + lst = d.get_device_list() + for dev in lst: + with d.Context(name=dev.name.decode()) as ctx: + attr = ctx.query_device_ex() + if attr.max_dm_size == 0: + return + dm_len = random.randrange(u.MIN_DM_SIZE, attr.max_dm_size, u.DM_ALIGNMENT) + dm_attrs = u.get_dm_attrs(dm_len) + with d.DM(ctx, dm_attrs) as dm: + data_length = random.randrange(4, dm_len, u.DM_ALIGNMENT) + data_offset = random.randrange(0, dm_len - data_length, u.DM_ALIGNMENT) + data_offset += 1 # offset needs to be a multiple of 4 + data = u.get_data(data_length) + try: + dm.copy_to_dm(data_offset, data.encode(), data_length) + except PyverbsRDMAError as e: + assert 'Failed to copy to dm' in e.args[0] + else: + raise PyverbsError('Wrote to device memory with a bad offset') + + def test_dm_read(self): + """ + Test reading from the device memory + """ + lst = d.get_device_list() + for dev in lst: + with d.Context(name=dev.name.decode()) as ctx: + attr = ctx.query_device_ex() + if attr.max_dm_size == 0: + return + dm_len = random.randrange(u.MIN_DM_SIZE, attr.max_dm_size, u.DM_ALIGNMENT) + dm_attrs = u.get_dm_attrs(dm_len) + with d.DM(ctx, dm_attrs) as dm: + data_length = random.randrange(4, dm_len, u.DM_ALIGNMENT) + data_offset = random.randrange(0, dm_len - data_length, u.DM_ALIGNMENT) + data = u.get_data(data_length) + dm.copy_to_dm(data_offset, data.encode(), data_length) + read_str = dm.copy_from_dm(data_offset, data_length) + assert read_str.decode() == data diff --git a/pyverbs/tests/mr.py b/pyverbs/tests/mr.py index c97c6ac38d6d..229da54240cd 100644 --- a/pyverbs/tests/mr.py +++ b/pyverbs/tests/mr.py @@ -5,8 +5,8 @@ import random from pyverbs.pyverbs_error import PyverbsRDMAError, PyverbsError from pyverbs.base import PyverbsRDMAErrno +from pyverbs.mr import MR, MW, DMMR import pyverbs.tests.utils as u -from pyverbs.mr import MR, MW import pyverbs.device as d from pyverbs.pd import PD import pyverbs.enums as e @@ -181,3 +181,49 @@ class mw_test(unittest.TestCase): else: raise PyverbsError('Created a MW with type {t}'.\ format(t=mw_type)) + + +class dm_mr_test(unittest.TestCase): + """ + Test various functionalities of the DMMR class. + """ + def test_create_dm_mr(self): + """ + Test ibv_reg_dm_mr + """ + lst = d.get_device_list() + for dev in lst: + with d.Context(name=dev.name.decode()) as ctx: + attr = ctx.query_device_ex() + if attr.max_dm_size == 0: + return + with PD(ctx) as pd: + dm_len = random.randrange(u.MIN_DM_SIZE, attr.max_dm_size, + u.DM_ALIGNMENT) + dm_attrs = u.get_dm_attrs(dm_len) + with d.DM(ctx, dm_attrs) as dm: + dm_mr_len = random.randint(1, dm_len) + dm_mr_offset = random.randint(0, (dm_len - dm_mr_len)) + dm_mr = DMMR(pd, dm_mr_len, e.IBV_ACCESS_ZERO_BASED, dm=dm, + offset=dm_mr_offset) + + def test_destroy_dm_mr(self): + """ + Test freeing of dm_mr + """ + lst = d.get_device_list() + for dev in lst: + with d.Context(name=dev.name.decode()) as ctx: + attr = ctx.query_device_ex() + if attr.max_dm_size == 0: + return + with PD(ctx) as pd: + dm_len = random.randrange(u.MIN_DM_SIZE, attr.max_dm_size, + u.DM_ALIGNMENT) + dm_attrs = u.get_dm_attrs(dm_len) + with d.DM(ctx, dm_attrs) as dm: + dm_mr_len = random.randint(1, dm_len) + dm_mr_offset = random.randint(0, (dm_len - dm_mr_len)) + dm_mr = DMMR(pd, dm_mr_len, e.IBV_ACCESS_ZERO_BASED, dm=dm, + offset=dm_mr_offset) + dm_mr.close() diff --git a/pyverbs/tests/utils.py b/pyverbs/tests/utils.py index 3281a326b324..c8b46686b0c1 100644 --- a/pyverbs/tests/utils.py +++ b/pyverbs/tests/utils.py @@ -2,9 +2,19 @@ # Copyright (c) 2019, Mellanox Technologies. All rights reserved. See COPYING file from string import ascii_lowercase as al import random + +import pyverbs.device as d import pyverbs.enums as e MAX_MR_SIZE = 4194304 +# Some HWs limit DM address and length alignment to 4 for read and write +# operations. Use a minimal length and alignment that respect that. +# For creation purposes use random alignments. As this is log2 of address +# alignment, no need for large numbers. +MIN_DM_SIZE = 4 +DM_ALIGNMENT = 4 +MIN_DM_LOG_ALIGN = 0 +MAX_DM_LOG_ALIGN = 6 def get_mr_length(): # Allocating large buffers typically fails @@ -26,3 +36,9 @@ def get_access_flags(): def get_data(length): return ''.join(random.choice(al) for i in range(length)) + + +def get_dm_attrs(dm_len): + align = random.randint(MIN_DM_LOG_ALIGN, MAX_DM_LOG_ALIGN) + # Comp mask != 0 is not supported + return d.AllocDmAttr(dm_len, align, 0) From patchwork Sun Feb 24 13:06:32 2019 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Noa Osherovich X-Patchwork-Id: 10827961 Return-Path: Received: from mail.wl.linuxfoundation.org (pdx-wl-mail.web.codeaurora.org [172.30.200.125]) by pdx-korg-patchwork-2.web.codeaurora.org (Postfix) with ESMTP id 66B8F1575 for ; Sun, 24 Feb 2019 13:09:07 +0000 (UTC) Received: from mail.wl.linuxfoundation.org (localhost [127.0.0.1]) by mail.wl.linuxfoundation.org (Postfix) with ESMTP id 54ABB29DAD for ; Sun, 24 Feb 2019 13:09:07 +0000 (UTC) Received: by mail.wl.linuxfoundation.org (Postfix, from userid 486) id 4905B2AA01; Sun, 24 Feb 2019 13:09:07 +0000 (UTC) X-Spam-Checker-Version: SpamAssassin 3.3.1 (2010-03-16) on pdx-wl-mail.web.codeaurora.org X-Spam-Level: X-Spam-Status: No, score=-7.9 required=2.0 tests=BAYES_00,MAILING_LIST_MULTI, RCVD_IN_DNSWL_HI,UNPARSEABLE_RELAY autolearn=ham version=3.3.1 Received: from vger.kernel.org (vger.kernel.org [209.132.180.67]) by mail.wl.linuxfoundation.org (Postfix) with ESMTP id 603DA2A98F for ; Sun, 24 Feb 2019 13:09:06 +0000 (UTC) Received: (majordomo@vger.kernel.org) by vger.kernel.org via listexpand id S1728298AbfBXNJA (ORCPT ); Sun, 24 Feb 2019 08:09:00 -0500 Received: from mail-il-dmz.mellanox.com ([193.47.165.129]:50551 "EHLO mellanox.co.il" rhost-flags-OK-OK-OK-FAIL) by vger.kernel.org with ESMTP id S1728296AbfBXNJA (ORCPT ); Sun, 24 Feb 2019 08:09:00 -0500 Received: from Internal Mail-Server by MTLPINE1 (envelope-from noaos@mellanox.com) with ESMTPS (AES256-SHA encrypted); 24 Feb 2019 15:06:49 +0200 Received: from reg-l-vrt-059-009.mtl.labs.mlnx (reg-l-vrt-059-009.mtl.labs.mlnx [10.135.59.9]) by labmailer.mlnx (8.13.8/8.13.8) with ESMTP id x1OD6mfh026210; Sun, 24 Feb 2019 15:06:49 +0200 From: Noa Osherovich To: leonro@mellanox.com, jgg@mellanox.com, dledford@redhat.com Cc: linux-rdma@vger.kernel.org, Noa Osherovich Subject: [PATCH rdma-core 13/19] pyverbs: Add query_port method Date: Sun, 24 Feb 2019 15:06:32 +0200 Message-Id: <20190224130638.31848-14-noaos@mellanox.com> X-Mailer: git-send-email 2.17.2 In-Reply-To: <20190224130638.31848-1-noaos@mellanox.com> References: <20190224130638.31848-1-noaos@mellanox.com> Sender: linux-rdma-owner@vger.kernel.org Precedence: bulk List-ID: X-Mailing-List: linux-rdma@vger.kernel.org X-Virus-Scanned: ClamAV using ClamSMTP Context class can now query its ports. A PortAttr object with a user-friendly print function is returned to the user. Signed-off-by: Noa Osherovich --- pyverbs/device.pxd | 3 + pyverbs/device.pyx | 196 +++++++++++++++++++++++++++++++++++ pyverbs/libibverbs.pxd | 26 +++++ pyverbs/libibverbs_enums.pxd | 14 +++ 4 files changed, 239 insertions(+) diff --git a/pyverbs/device.pxd b/pyverbs/device.pxd index c9e346539e58..a7089972985b 100644 --- a/pyverbs/device.pxd +++ b/pyverbs/device.pxd @@ -47,3 +47,6 @@ cdef class DM(PyverbsCM): cdef object dm_mrs cdef object context cdef add_ref(self, obj) + +cdef class PortAttr(PyverbsObject): + cdef v.ibv_port_attr attr diff --git a/pyverbs/device.pyx b/pyverbs/device.pyx index 7bfcda8bc177..757eebf67c48 100644 --- a/pyverbs/device.pyx +++ b/pyverbs/device.pyx @@ -11,6 +11,7 @@ import weakref from .pyverbs_error import PyverbsRDMAError, PyverbsError from .pyverbs_error import PyverbsUserError from pyverbs.base import PyverbsRDMAErrno +cimport pyverbs.libibverbs_enums as e cimport pyverbs.libibverbs as v from pyverbs.addr cimport GID from pyverbs.mr import DMMR @@ -165,6 +166,18 @@ cdef class Context(PyverbsCM): format(idx=index, port=port_num)) return gid + def query_port(self, unsigned int port_num): + """ + Query port of the device and returns its attributes. + :param port_num: Port number to query + :return: PortAttr object on success + """ + port_attrs = PortAttr() + rc = v.ibv_query_port(self.context, port_num, &port_attrs.attr) + if rc != 0: + raise PyverbsRDMAErrno('Failed to query port {p}'.format(p=port_num)) + return port_attrs + cdef add_ref(self, obj): if isinstance(obj, PD): self.pds.add(obj) @@ -585,6 +598,99 @@ cdef class DM(PyverbsCM): return res +cdef class PortAttr(PyverbsObject): + @property + def state(self): + return self.attr.state + @property + def max_mtu(self): + return self.attr.max_mtu + @property + def active_mtu(self): + return self.attr.active_mtu + @property + def gid_tbl_len(self): + return self.attr.gid_tbl_len + @property + def port_cap_flags(self): + return self.attr.port_cap_flags + @property + def max_msg_sz(self): + return self.attr.max_msg_sz + @property + def bad_pkey_cntr(self): + return self.attr.bad_pkey_cntr + @property + def qkey_viol_cntr(self): + return self.attr.qkey_viol_cntr + @property + def pkey_tbl_len(self): + return self.attr.pkey_tbl_len + @property + def lid(self): + return self.attr.lid + @property + def sm_lid(self): + return self.attr.sm_lid + @property + def lmc(self): + return self.attr.lmc + @property + def max_vl_num(self): + return self.attr.max_vl_num + @property + def sm_sl(self): + return self.attr.sm_sl + @property + def subnet_timeout(self): + return self.attr.subnet_timeout + @property + def init_type_reply(self): + return self.attr.init_type_reply + @property + def active_width(self): + return self.attr.active_width + @property + def active_speed(self): + return self.attr.active_speed + @property + def phys_state(self): + return self.attr.phys_state + @property + def link_layer(self): + return self.attr.link_layer + @property + def flags(self): + return self.attr.flags + @property + def port_cap_flags2(self): + return self.attr.port_cap_flags2 + + def __str__(self): + print_format = '{:<24}: {:<20}\n' + return print_format.format('Port state', port_state_to_str(self.attr.state)) +\ + print_format.format('Max MTU', translate_mtu(self.attr.max_mtu)) +\ + print_format.format('Active MTU', translate_mtu(self.attr.active_mtu)) +\ + print_format.format('SM lid', self.attr.sm_lid) +\ + print_format.format('Port lid', self.attr.lid) +\ + print_format.format('lmc', hex(self.attr.lmc)) +\ + print_format.format('Link layer', translate_link_layer(self.attr.link_layer)) +\ + print_format.format('Max message size', hex(self.attr.max_msg_sz)) +\ + print_format.format('Port cap flags', translate_port_cap_flags(self.attr.port_cap_flags)) +\ + print_format.format('Port cap flags 2', translate_port_cap_flags2(self.attr.port_cap_flags2)) +\ + print_format.format('max VL num', self.attr.max_vl_num) +\ + print_format.format('Bad Pkey counter', self.attr.bad_pkey_cntr) +\ + print_format.format('Qkey violations counter', self.attr.qkey_viol_cntr) +\ + print_format.format('GID table len', self.attr.gid_tbl_len) +\ + print_format.format('Pkey table len', self.attr.pkey_tbl_len) +\ + print_format.format('SM sl', self.attr.sm_sl) +\ + print_format.format('Subnet timeout', self.attr.subnet_timeout) +\ + print_format.format('Init type reply', self.attr.init_type_reply) +\ + print_format.format('Active width', width_to_str(self.attr.active_width)) +\ + print_format.format('Active speed', speed_to_str(self.attr.active_speed)) +\ + print_format.format('Phys state', phys_state_to_str(self.attr.phys_state)) +\ + print_format.format('Flags', self.attr.flags) + def guid_format(num): """ Get GUID representation of the given number, including change of endianness. @@ -608,6 +714,96 @@ def translate_node_type(node_type): def guid_to_hex(node_guid): return hex(node_guid).replace('L', '').replace('0x', '') +def port_state_to_str(port_state): + l = {0: 'NOP', 1: 'Down', 2: 'Init', 3: 'Armed',4: 'Active', + 5: 'Active defer'} + try: + return '{s} ({n})'.format(s=l[port_state], n=port_state) + except: + return 'Invalid state' + +def translate_mtu(mtu): + l = {1: 256, 2: 512, 3: 1024, 4: 2048, 5: 4096} + try: + return '{s} ({n})'.format(s=l[mtu], n=mtu) + except: + return 'Invalid MTU' + +def translate_link_layer(ll): + l = {0: 'Unspecified', 1:'InfiniBand', 2:'Ethernet'} + try: + return l[ll] + except: + return 'Invalid link layer {ll}'.format(ll=ll) + +def translate_port_cap_flags(flags): + l = {e.IBV_PORT_SM: 'IBV_PORT_SM', + e.IBV_PORT_NOTICE_SUP: 'IBV_PORT_NOTICE_SUP', + e.IBV_PORT_TRAP_SUP: 'IBV_PORT_TRAP_SUP', + e.IBV_PORT_OPT_IPD_SUP: 'IBV_PORT_OPT_IPD_SUP', + e.IBV_PORT_AUTO_MIGR_SUP: 'IBV_PORT_AUTO_MIGR_SUP', + e.IBV_PORT_SL_MAP_SUP: 'IBV_PORT_SL_MAP_SUP', + e.IBV_PORT_MKEY_NVRAM: 'IBV_PORT_MKEY_NVRAM', + e.IBV_PORT_PKEY_NVRAM: 'IBV_PORT_PKEY_NVRAM', + e.IBV_PORT_LED_INFO_SUP: 'IBV_PORT_LED_INFO_SUP', + e.IBV_PORT_SYS_IMAGE_GUID_SUP: 'IBV_PORT_SYS_IMAGE_GUID_SUP', + e.IBV_PORT_PKEY_SW_EXT_PORT_TRAP_SUP: 'IBV_PORT_PKEY_SW_EXT_PORT_TRAP_SUP', + e.IBV_PORT_EXTENDED_SPEEDS_SUP: 'IBV_PORT_EXTENDED_SPEEDS_SUP', + e.IBV_PORT_CAP_MASK2_SUP: 'IBV_PORT_CAP_MASK2_SUP', + e.IBV_PORT_CM_SUP: 'IBV_PORT_CM_SUP', + e.IBV_PORT_SNMP_TUNNEL_SUP: 'IBV_PORT_SNMP_TUNNEL_SUP', + e.IBV_PORT_REINIT_SUP: 'IBV_PORT_REINIT_SUP', + e.IBV_PORT_DEVICE_MGMT_SUP: 'IBV_PORT_DEVICE_MGMT_SUP', + e.IBV_PORT_VENDOR_CLASS_SUP: 'IBV_PORT_VENDOR_CLASS_SUP', + e.IBV_PORT_DR_NOTICE_SUP: 'IBV_PORT_DR_NOTICE_SUP', + e.IBV_PORT_CAP_MASK_NOTICE_SUP: 'IBV_PORT_CAP_MASK_NOTICE_SUP', + e.IBV_PORT_BOOT_MGMT_SUP: 'IBV_PORT_BOOT_MGMT_SUP', + e.IBV_PORT_LINK_LATENCY_SUP: 'IBV_PORT_LINK_LATENCY_SUP', + e.IBV_PORT_CLIENT_REG_SUP: 'IBV_PORT_CLIENT_REG_SUP', + e.IBV_PORT_IP_BASED_GIDS: 'IBV_PORT_IP_BASED_GIDS'} + return str_from_flags(flags, l) + +def translate_port_cap_flags2(flags): + l = {e.IBV_PORT_SET_NODE_DESC_SUP: 'IBV_PORT_SET_NODE_DESC_SUP', + e.IBV_PORT_INFO_EXT_SUP: 'IBV_PORT_INFO_EXT_SUP', + e.IBV_PORT_VIRT_SUP: 'IBV_PORT_VIRT_SUP', + e.IBV_PORT_SWITCH_PORT_STATE_TABLE_SUP: 'IBV_PORT_SWITCH_PORT_STATE_TABLE_SUP', + e.IBV_PORT_LINK_WIDTH_2X_SUP: 'IBV_PORT_LINK_WIDTH_2X_SUP', + e.IBV_PORT_LINK_SPEED_HDR_SUP: 'IBV_PORT_LINK_SPEED_HDR_SUP'} + return str_from_flags(flags, l) + +def str_from_flags(flags, dictionary): + str_flags = "" + for bit in dictionary: + if flags & bit: + str_flags += dictionary[bit] + str_flags += ' ' + return str_flags + +def phys_state_to_str(phys): + l = {1: 'Sleep', 2: 'Polling', 3: 'Disabled', + 4: 'Port configuration training', 5: 'Link up', + 6: 'Link error recovery', 7: 'Phy test'} + try: + return '{s} ({n})'.format(s=l[phys], n=phys) + except: + return 'Invalid physical state' + +def width_to_str(width): + l = {1: '1X', 2: '4X', 4: '8X', 16: '2X'} + try: + return '{s} ({n})'.format(s=l[width], n=width) + except: + return 'Invalid width' + +def speed_to_str(speed): + l = {1: '2.5 Gbps', 2: '5.0 Gbps', 4: '5.0 Gbps', 8: '10.0 Gbps', + 16: '14.0 Gbps', 32: '25.0 Gbps', 64: '50.0 Gbps'} + try: + return '{s} ({n})'.format(s=l[speed], n=speed) + except: + return 'Invalid speed' + def get_device_list(): """ :return: list of IB_devices on current node diff --git a/pyverbs/libibverbs.pxd b/pyverbs/libibverbs.pxd index 5dd3ba013e5e..c6b564c4d487 100644 --- a/pyverbs/libibverbs.pxd +++ b/pyverbs/libibverbs.pxd @@ -148,6 +148,30 @@ cdef extern from 'infiniband/verbs.h': ibv_context *context unsigned int comp_mask + cdef struct ibv_port_attr: + ibv_port_state state + ibv_mtu max_mtu + ibv_mtu active_mtu + int gid_tbl_len + unsigned int port_cap_flags + unsigned int max_msg_sz + unsigned int bad_pkey_cntr + unsigned int qkey_viol_cntr + unsigned short pkey_tbl_len + unsigned short lid + unsigned short sm_lid + unsigned char lmc + unsigned char max_vl_num + unsigned char sm_sl + unsigned char subnet_timeout + unsigned char init_type_reply + unsigned char active_width + unsigned char active_speed + unsigned char phys_state + unsigned char link_layer + unsigned char flags + unsigned short port_cap_flags2 + ibv_device **ibv_get_device_list(int *n) void ibv_free_device_list(ibv_device **list) ibv_context *ibv_open_device(ibv_device *device) @@ -173,3 +197,5 @@ cdef extern from 'infiniband/verbs.h': size_t length) int ibv_memcpy_from_dm(void *host_addr, ibv_dm *dm, unsigned long dm_offset, size_t length) + int ibv_query_port(ibv_context *context, uint8_t port_num, + ibv_port_attr *port_attr) diff --git a/pyverbs/libibverbs_enums.pxd b/pyverbs/libibverbs_enums.pxd index d6e81199fae3..2525e869a201 100644 --- a/pyverbs/libibverbs_enums.pxd +++ b/pyverbs/libibverbs_enums.pxd @@ -3,6 +3,11 @@ cdef extern from '': + cpdef enum: + IBV_LINK_LAYER_UNSPECIFIED + IBV_LINK_LAYER_INFINIBAND + IBV_LINK_LAYER_ETHERNET + cpdef enum ibv_atomic_cap: IBV_ATOMIC_NONE IBV_ATOMIC_HCA @@ -29,6 +34,7 @@ cdef extern from '': IBV_PORT_SYS_IMAGE_GUID_SUP = 1 << 11 IBV_PORT_PKEY_SW_EXT_PORT_TRAP_SUP = 1 << 12 IBV_PORT_EXTENDED_SPEEDS_SUP = 1 << 14 + IBV_PORT_CAP_MASK2_SUP = 1 << 15, IBV_PORT_CM_SUP = 1 << 16 IBV_PORT_SNMP_TUNNEL_SUP = 1 << 17 IBV_PORT_REINIT_SUP = 1 << 18 @@ -41,6 +47,14 @@ cdef extern from '': IBV_PORT_CLIENT_REG_SUP = 1 << 25 IBV_PORT_IP_BASED_GIDS = 1 << 26 + cpdef enum ibv_port_cap_flags2: + IBV_PORT_SET_NODE_DESC_SUP = 1 << 0 + IBV_PORT_INFO_EXT_SUP = 1 << 1 + IBV_PORT_VIRT_SUP = 1 << 2 + IBV_PORT_SWITCH_PORT_STATE_TABLE_SUP = 1 << 3 + IBV_PORT_LINK_WIDTH_2X_SUP = 1 << 4 + IBV_PORT_LINK_SPEED_HDR_SUP = 1 << 5 + cpdef enum ibv_mtu: IBV_MTU_256 = 1 IBV_MTU_512 = 2 From patchwork Sun Feb 24 13:06:33 2019 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Noa Osherovich X-Patchwork-Id: 10827949 Return-Path: Received: from mail.wl.linuxfoundation.org (pdx-wl-mail.web.codeaurora.org [172.30.200.125]) by pdx-korg-patchwork-2.web.codeaurora.org (Postfix) with ESMTP id B1BD113B5 for ; Sun, 24 Feb 2019 13:08:59 +0000 (UTC) Received: from mail.wl.linuxfoundation.org (localhost [127.0.0.1]) by mail.wl.linuxfoundation.org (Postfix) with ESMTP id 9D6F9296ED for ; Sun, 24 Feb 2019 13:08:59 +0000 (UTC) Received: by mail.wl.linuxfoundation.org (Postfix, from userid 486) id 920402AA01; Sun, 24 Feb 2019 13:08:59 +0000 (UTC) X-Spam-Checker-Version: SpamAssassin 3.3.1 (2010-03-16) on pdx-wl-mail.web.codeaurora.org X-Spam-Level: X-Spam-Status: No, score=-7.9 required=2.0 tests=BAYES_00,MAILING_LIST_MULTI, RCVD_IN_DNSWL_HI,UNPARSEABLE_RELAY autolearn=ham version=3.3.1 Received: from vger.kernel.org (vger.kernel.org [209.132.180.67]) by mail.wl.linuxfoundation.org (Postfix) with ESMTP id 5B94E296ED for ; Sun, 24 Feb 2019 13:08:58 +0000 (UTC) Received: (majordomo@vger.kernel.org) by vger.kernel.org via listexpand id S1728248AbfBXNI5 (ORCPT ); Sun, 24 Feb 2019 08:08:57 -0500 Received: from mail-il-dmz.mellanox.com ([193.47.165.129]:50560 "EHLO mellanox.co.il" rhost-flags-OK-OK-OK-FAIL) by vger.kernel.org with ESMTP id S1728308AbfBXNI4 (ORCPT ); Sun, 24 Feb 2019 08:08:56 -0500 Received: from Internal Mail-Server by MTLPINE1 (envelope-from noaos@mellanox.com) with ESMTPS (AES256-SHA encrypted); 24 Feb 2019 15:06:49 +0200 Received: from reg-l-vrt-059-009.mtl.labs.mlnx (reg-l-vrt-059-009.mtl.labs.mlnx [10.135.59.9]) by labmailer.mlnx (8.13.8/8.13.8) with ESMTP id x1OD6mfi026210; Sun, 24 Feb 2019 15:06:49 +0200 From: Noa Osherovich To: leonro@mellanox.com, jgg@mellanox.com, dledford@redhat.com Cc: linux-rdma@vger.kernel.org, Noa Osherovich Subject: [PATCH rdma-core 14/19] pyverbs: Add unittests for query port Date: Sun, 24 Feb 2019 15:06:33 +0200 Message-Id: <20190224130638.31848-15-noaos@mellanox.com> X-Mailer: git-send-email 2.17.2 In-Reply-To: <20190224130638.31848-1-noaos@mellanox.com> References: <20190224130638.31848-1-noaos@mellanox.com> Sender: linux-rdma-owner@vger.kernel.org Precedence: bulk List-ID: X-Mailing-List: linux-rdma@vger.kernel.org X-Virus-Scanned: ClamAV using ClamSMTP Check query_port: - Good flow: Verify that the attributes' values are legal. - Bad flow: Use non-exiting port and expect a failure. Signed-off-by: Noa Osherovich --- pyverbs/tests/device.py | 37 +++++++++++++++++++++++++++++++++++++ 1 file changed, 37 insertions(+) diff --git a/pyverbs/tests/device.py b/pyverbs/tests/device.py index 4a5e0a794069..492448a1526b 100644 --- a/pyverbs/tests/device.py +++ b/pyverbs/tests/device.py @@ -76,6 +76,43 @@ class device_test(unittest.TestCase): attr_ex = ctx.query_device_ex() self.verify_device_attr(attr_ex.orig_attr) + @staticmethod + def verify_port_attr(attr): + assert 'Invalid' not in d.phys_state_to_str(attr.state) + assert 'Invalid' not in d.translate_mtu(attr.max_mtu) + assert 'Invalid' not in d.translate_mtu(attr.active_mtu) + assert 'Invalid' not in d.width_to_str(attr.active_width) + assert 'Invalid' not in d.speed_to_str(attr.active_speed) + assert 'Invalid' not in d.translate_link_layer(attr.link_layer) + assert attr.max_msg_sz > 0x1000 + + def test_query_port(self): + """ + Test ibv_query_port + """ + lst = d.get_device_list() + for dev in lst: + with d.Context(name=dev.name.decode()) as ctx: + num_ports = ctx.query_device().phys_port_cnt + for p in range(num_ports): + port_attr = ctx.query_port(p + 1) + self.verify_port_attr(port_attr) + + def test_query_port_bad_flow(self): + """ Verify that querying non-existing ports fails as expected """ + lst = d.get_device_list() + for dev in lst: + with d.Context(name=dev.name.decode()) as ctx: + num_ports = ctx.query_device().phys_port_cnt + try: + port = num_ports + random.randint(1, 10) + ctx.query_port(port) + except PyverbsRDMAError as e: + assert 'Failed to query port' in e.args[0] + assert 'Invalid argument' in e.args[0] + else: + raise PyverbsRDMAError('Successfully queried non-existing port {p}'.\ + format(p=port)) class dm_test(unittest.TestCase): """ From patchwork Sun Feb 24 13:06:34 2019 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Noa Osherovich X-Patchwork-Id: 10827947 Return-Path: Received: from mail.wl.linuxfoundation.org (pdx-wl-mail.web.codeaurora.org [172.30.200.125]) by pdx-korg-patchwork-2.web.codeaurora.org (Postfix) with ESMTP id 4272117E9 for ; Sun, 24 Feb 2019 13:08:59 +0000 (UTC) Received: from mail.wl.linuxfoundation.org (localhost [127.0.0.1]) by mail.wl.linuxfoundation.org (Postfix) with ESMTP id 30A6B2AA01 for ; Sun, 24 Feb 2019 13:08:59 +0000 (UTC) Received: by mail.wl.linuxfoundation.org (Postfix, from userid 486) id 23A662A98F; Sun, 24 Feb 2019 13:08:59 +0000 (UTC) X-Spam-Checker-Version: SpamAssassin 3.3.1 (2010-03-16) on pdx-wl-mail.web.codeaurora.org X-Spam-Level: X-Spam-Status: No, score=-7.9 required=2.0 tests=BAYES_00,MAILING_LIST_MULTI, RCVD_IN_DNSWL_HI,UNPARSEABLE_RELAY autolearn=ham version=3.3.1 Received: from vger.kernel.org (vger.kernel.org [209.132.180.67]) by mail.wl.linuxfoundation.org (Postfix) with ESMTP id C9DC72A1EC for ; Sun, 24 Feb 2019 13:08:58 +0000 (UTC) Received: (majordomo@vger.kernel.org) by vger.kernel.org via listexpand id S1728307AbfBXNI5 (ORCPT ); Sun, 24 Feb 2019 08:08:57 -0500 Received: from mail-il-dmz.mellanox.com ([193.47.165.129]:50568 "EHLO mellanox.co.il" rhost-flags-OK-OK-OK-FAIL) by vger.kernel.org with ESMTP id S1728121AbfBXNI5 (ORCPT ); Sun, 24 Feb 2019 08:08:57 -0500 Received: from Internal Mail-Server by MTLPINE1 (envelope-from noaos@mellanox.com) with ESMTPS (AES256-SHA encrypted); 24 Feb 2019 15:06:49 +0200 Received: from reg-l-vrt-059-009.mtl.labs.mlnx (reg-l-vrt-059-009.mtl.labs.mlnx [10.135.59.9]) by labmailer.mlnx (8.13.8/8.13.8) with ESMTP id x1OD6mfj026210; Sun, 24 Feb 2019 15:06:49 +0200 From: Noa Osherovich To: leonro@mellanox.com, jgg@mellanox.com, dledford@redhat.com Cc: linux-rdma@vger.kernel.org, Noa Osherovich Subject: [PATCH rdma-core 15/19] update MAINTAINERS Date: Sun, 24 Feb 2019 15:06:34 +0200 Message-Id: <20190224130638.31848-16-noaos@mellanox.com> X-Mailer: git-send-email 2.17.2 In-Reply-To: <20190224130638.31848-1-noaos@mellanox.com> References: <20190224130638.31848-1-noaos@mellanox.com> Sender: linux-rdma-owner@vger.kernel.org Precedence: bulk List-ID: X-Mailing-List: linux-rdma@vger.kernel.org X-Virus-Scanned: ClamAV using ClamSMTP Add pyverbs to the MAINTAINERS file. Signed-off-by: Noa Osherovich --- MAINTAINERS | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/MAINTAINERS b/MAINTAINERS index cc81916ea499..4d5c52206f74 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -177,3 +177,8 @@ M: Adit Ranadive L: pv-drivers@vmware.com S: Supported F: providers/vmw_pvrdma/ + +PYVERBS +M: Noa Osherovich +S: Supported +F: pyverbs/ From patchwork Sun Feb 24 13:06:35 2019 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Noa Osherovich X-Patchwork-Id: 10827951 Return-Path: Received: from mail.wl.linuxfoundation.org (pdx-wl-mail.web.codeaurora.org [172.30.200.125]) by pdx-korg-patchwork-2.web.codeaurora.org (Postfix) with ESMTP id CC1D61880 for ; Sun, 24 Feb 2019 13:08:59 +0000 (UTC) Received: from mail.wl.linuxfoundation.org (localhost [127.0.0.1]) by mail.wl.linuxfoundation.org (Postfix) with ESMTP id BB08E296ED for ; Sun, 24 Feb 2019 13:08:59 +0000 (UTC) Received: by mail.wl.linuxfoundation.org (Postfix, from userid 486) id AF37C2AA16; Sun, 24 Feb 2019 13:08:59 +0000 (UTC) X-Spam-Checker-Version: SpamAssassin 3.3.1 (2010-03-16) on pdx-wl-mail.web.codeaurora.org X-Spam-Level: X-Spam-Status: No, score=-7.9 required=2.0 tests=BAYES_00,MAILING_LIST_MULTI, RCVD_IN_DNSWL_HI,UNPARSEABLE_RELAY autolearn=ham version=3.3.1 Received: from vger.kernel.org (vger.kernel.org [209.132.180.67]) by mail.wl.linuxfoundation.org (Postfix) with ESMTP id 5B53B2A98F for ; Sun, 24 Feb 2019 13:08:59 +0000 (UTC) Received: (majordomo@vger.kernel.org) by vger.kernel.org via listexpand id S1728121AbfBXNI6 (ORCPT ); Sun, 24 Feb 2019 08:08:58 -0500 Received: from mail-il-dmz.mellanox.com ([193.47.165.129]:50585 "EHLO mellanox.co.il" rhost-flags-OK-OK-OK-FAIL) by vger.kernel.org with ESMTP id S1728269AbfBXNI5 (ORCPT ); Sun, 24 Feb 2019 08:08:57 -0500 Received: from Internal Mail-Server by MTLPINE1 (envelope-from noaos@mellanox.com) with ESMTPS (AES256-SHA encrypted); 24 Feb 2019 15:06:49 +0200 Received: from reg-l-vrt-059-009.mtl.labs.mlnx (reg-l-vrt-059-009.mtl.labs.mlnx [10.135.59.9]) by labmailer.mlnx (8.13.8/8.13.8) with ESMTP id x1OD6mfk026210; Sun, 24 Feb 2019 15:06:49 +0200 From: Noa Osherovich To: leonro@mellanox.com, jgg@mellanox.com, dledford@redhat.com Cc: linux-rdma@vger.kernel.org, Noa Osherovich Subject: [PATCH rdma-core 16/19] debian: Be explicit in dh_install in regards to NO_PYVERBS request Date: Sun, 24 Feb 2019 15:06:35 +0200 Message-Id: <20190224130638.31848-17-noaos@mellanox.com> X-Mailer: git-send-email 2.17.2 In-Reply-To: <20190224130638.31848-1-noaos@mellanox.com> References: <20190224130638.31848-1-noaos@mellanox.com> Sender: linux-rdma-owner@vger.kernel.org Precedence: bulk List-ID: X-Mailing-List: linux-rdma@vger.kernel.org X-Virus-Scanned: ClamAV using ClamSMTP Fix dh_install to ignore python3-pyverbs when not required and not fail build to missing packages. Signed-off-by: Noa Osherovich --- debian/rules | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/debian/rules b/debian/rules index dceb63527c0b..7fb636dc1722 100755 --- a/debian/rules +++ b/debian/rules @@ -67,12 +67,13 @@ INST_EXCLUDE := "etc/init.d/srpd" \ "etc/init.d/ibacm" \ "usr/sbin/run_srp_daemon" \ "usr/sbin/srp_daemon.sh" -ifeq ($(EXTRA_CMAKE_FLAGS), -DNO_PYVERBS=1) - INST_EXCLUDE := $(INST_EXCLUDE) "usr/lib/python3/dist-packages/pyverbs" -endif INST_EXCLUDE := $(addprefix -X,$(INST_EXCLUDE)) override_dh_install: +ifeq ($(EXTRA_CMAKE_FLAGS), -DNO_PYVERBS=1) + dh_install -Npython3-pyverbs --fail-missing $(INST_EXCLUDE) --remaining-packages +else dh_install --fail-missing $(INST_EXCLUDE) +endif # cmake installs the correct init scripts in the correct place, just setup the # pre-postrms From patchwork Sun Feb 24 13:06:36 2019 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Noa Osherovich X-Patchwork-Id: 10827955 Return-Path: Received: from mail.wl.linuxfoundation.org (pdx-wl-mail.web.codeaurora.org [172.30.200.125]) by pdx-korg-patchwork-2.web.codeaurora.org (Postfix) with ESMTP id 2372617E9 for ; Sun, 24 Feb 2019 13:09:01 +0000 (UTC) Received: from mail.wl.linuxfoundation.org (localhost [127.0.0.1]) by mail.wl.linuxfoundation.org (Postfix) with ESMTP id 11D6C29DAD for ; Sun, 24 Feb 2019 13:09:01 +0000 (UTC) Received: by mail.wl.linuxfoundation.org (Postfix, from userid 486) id 061F12AA16; Sun, 24 Feb 2019 13:09:01 +0000 (UTC) X-Spam-Checker-Version: SpamAssassin 3.3.1 (2010-03-16) on pdx-wl-mail.web.codeaurora.org X-Spam-Level: X-Spam-Status: No, score=-7.9 required=2.0 tests=BAYES_00,MAILING_LIST_MULTI, RCVD_IN_DNSWL_HI,UNPARSEABLE_RELAY autolearn=ham version=3.3.1 Received: from vger.kernel.org (vger.kernel.org [209.132.180.67]) by mail.wl.linuxfoundation.org (Postfix) with ESMTP id B286229DAD for ; Sun, 24 Feb 2019 13:09:00 +0000 (UTC) Received: (majordomo@vger.kernel.org) by vger.kernel.org via listexpand id S1728275AbfBXNI7 (ORCPT ); Sun, 24 Feb 2019 08:08:59 -0500 Received: from mail-il-dmz.mellanox.com ([193.47.165.129]:50607 "EHLO mellanox.co.il" rhost-flags-OK-OK-OK-FAIL) by vger.kernel.org with ESMTP id S1728233AbfBXNI7 (ORCPT ); Sun, 24 Feb 2019 08:08:59 -0500 Received: from Internal Mail-Server by MTLPINE1 (envelope-from noaos@mellanox.com) with ESMTPS (AES256-SHA encrypted); 24 Feb 2019 15:06:49 +0200 Received: from reg-l-vrt-059-009.mtl.labs.mlnx (reg-l-vrt-059-009.mtl.labs.mlnx [10.135.59.9]) by labmailer.mlnx (8.13.8/8.13.8) with ESMTP id x1OD6mfl026210; Sun, 24 Feb 2019 15:06:49 +0200 From: Noa Osherovich To: leonro@mellanox.com, jgg@mellanox.com, dledford@redhat.com Cc: linux-rdma@vger.kernel.org, Noa Osherovich Subject: [PATCH rdma-core 17/19] build: Disable pyverbs build for older Cython versions Date: Sun, 24 Feb 2019 15:06:36 +0200 Message-Id: <20190224130638.31848-18-noaos@mellanox.com> X-Mailer: git-send-email 2.17.2 In-Reply-To: <20190224130638.31848-1-noaos@mellanox.com> References: <20190224130638.31848-1-noaos@mellanox.com> Sender: linux-rdma-owner@vger.kernel.org Precedence: bulk List-ID: X-Mailing-List: linux-rdma@vger.kernel.org X-Virus-Scanned: ClamAV using ClamSMTP Dockers with older Cython versions fail to build pyverbs. Until we get to the bottom of this, disable pyverbs for older Cython versions. Signed-off-by: Noa Osherovich --- buildlib/Findcython.cmake | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/buildlib/Findcython.cmake b/buildlib/Findcython.cmake index 0bcc862b7b8a..ac1610c47f6d 100644 --- a/buildlib/Findcython.cmake +++ b/buildlib/Findcython.cmake @@ -18,6 +18,14 @@ if(NOT _VERSION_RESULT) from Cython.Compiler.Main import main main(command_line = 1)") execute_process(COMMAND "chmod" "a+x" "${CYTHON_EXECUTABLE}") + + # Dockers with older Cython versions fail to build pyverbs. Until we get to + # the bottom of this, disable pyverbs for older Cython versions. + if (CYTHON_VERSION_STRING VERSION_LESS "0.25") + message("Cython version < 0.25, disabling") + unset(CYTHON_EXECUTABLE) + endif() + endif() unset(_VERSION_RESULT) unset(_VERSION) From patchwork Sun Feb 24 13:06:37 2019 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Noa Osherovich X-Patchwork-Id: 10827959 Return-Path: Received: from mail.wl.linuxfoundation.org (pdx-wl-mail.web.codeaurora.org [172.30.200.125]) by pdx-korg-patchwork-2.web.codeaurora.org (Postfix) with ESMTP id B70A813B5 for ; Sun, 24 Feb 2019 13:09:06 +0000 (UTC) Received: from mail.wl.linuxfoundation.org (localhost [127.0.0.1]) by mail.wl.linuxfoundation.org (Postfix) with ESMTP id A30EF29DAD for ; Sun, 24 Feb 2019 13:09:06 +0000 (UTC) Received: by mail.wl.linuxfoundation.org (Postfix, from userid 486) id 944DD2AA16; Sun, 24 Feb 2019 13:09:06 +0000 (UTC) X-Spam-Checker-Version: SpamAssassin 3.3.1 (2010-03-16) on pdx-wl-mail.web.codeaurora.org X-Spam-Level: X-Spam-Status: No, score=-7.9 required=2.0 tests=BAYES_00,MAILING_LIST_MULTI, RCVD_IN_DNSWL_HI,UNPARSEABLE_RELAY autolearn=ham version=3.3.1 Received: from vger.kernel.org (vger.kernel.org [209.132.180.67]) by mail.wl.linuxfoundation.org (Postfix) with ESMTP id 3FCB529DAD for ; Sun, 24 Feb 2019 13:09:06 +0000 (UTC) Received: (majordomo@vger.kernel.org) by vger.kernel.org via listexpand id S1728259AbfBXNI7 (ORCPT ); Sun, 24 Feb 2019 08:08:59 -0500 Received: from mail-il-dmz.mellanox.com ([193.47.165.129]:50594 "EHLO mellanox.co.il" rhost-flags-OK-OK-OK-FAIL) by vger.kernel.org with ESMTP id S1728313AbfBXNI5 (ORCPT ); Sun, 24 Feb 2019 08:08:57 -0500 Received: from Internal Mail-Server by MTLPINE1 (envelope-from noaos@mellanox.com) with ESMTPS (AES256-SHA encrypted); 24 Feb 2019 15:06:49 +0200 Received: from reg-l-vrt-059-009.mtl.labs.mlnx (reg-l-vrt-059-009.mtl.labs.mlnx [10.135.59.9]) by labmailer.mlnx (8.13.8/8.13.8) with ESMTP id x1OD6mfm026210; Sun, 24 Feb 2019 15:06:49 +0200 From: Noa Osherovich To: leonro@mellanox.com, jgg@mellanox.com, dledford@redhat.com Cc: linux-rdma@vger.kernel.org, Noa Osherovich Subject: [PATCH rdma-core 18/19] travis: Update Cython version Date: Sun, 24 Feb 2019 15:06:37 +0200 Message-Id: <20190224130638.31848-19-noaos@mellanox.com> X-Mailer: git-send-email 2.17.2 In-Reply-To: <20190224130638.31848-1-noaos@mellanox.com> References: <20190224130638.31848-1-noaos@mellanox.com> Sender: linux-rdma-owner@vger.kernel.org Precedence: bulk List-ID: X-Mailing-List: linux-rdma@vger.kernel.org X-Virus-Scanned: ClamAV using ClamSMTP Ubuntu 16.04 has a rather old Cython version which breaks pyverbs' build. Upgrade it using pip. Signed-off-by: Noa Osherovich Signed-off-by: Jason Gunthorpe --- .travis.yml | 3 ++- buildlib/cbuild | 4 ++++ buildlib/travis-build | 2 ++ 3 files changed, 8 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index f1c1c0ea2bb8..660b306797bf 100644 --- a/.travis.yml +++ b/.travis.yml @@ -36,7 +36,7 @@ addons: # pyverbs - python3-dev - - cython3 + - python3-pip service: - docker @@ -47,6 +47,7 @@ before_script: - wget -q http://releases.linaro.org/$LATEST_GCC_LINARO_URL - mkdir $HOME/aarch64 && tar xf $LATEST_GCC_LINARO_TAR -C $HOME/aarch64 --strip 1 - rm $LATEST_GCC_LINARO_TAR + - http_proxy= pip3 install cython script: - buildlib/travis-build - buildlib/travis-checkpatch diff --git a/buildlib/cbuild b/buildlib/cbuild index 9ced0de6be07..a4f992db2888 100755 --- a/buildlib/cbuild +++ b/buildlib/cbuild @@ -297,6 +297,9 @@ class travis(APTEnvironment): (cd /usr/local/bin/ && ln -sf ../clang*/bin/clang-7 .)""".format(llvm_tar)]; return cmds; + def get_cython(self): + return ["""RUN pip3 install cython"""] + def get_docker_file(self): # First this to get apt-add-repository self.pkgs = {"software-properties-common"} @@ -315,6 +318,7 @@ class travis(APTEnvironment): res.lines.extend(self.get_before_script()) res.lines.extend(self.get_clang()) + res.lines.extend(self.get_cython()) return res; diff --git a/buildlib/travis-build b/buildlib/travis-build index b44f13e91145..f84b77fc7821 100755 --- a/buildlib/travis-build +++ b/buildlib/travis-build @@ -1,5 +1,7 @@ #!/bin/bash +PATH=/home/`whoami`/.local/bin:$PATH + # Stop on error set -e # Echo all commands to Travis log From patchwork Sun Feb 24 13:06:38 2019 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Noa Osherovich X-Patchwork-Id: 10827963 Return-Path: Received: from mail.wl.linuxfoundation.org (pdx-wl-mail.web.codeaurora.org [172.30.200.125]) by pdx-korg-patchwork-2.web.codeaurora.org (Postfix) with ESMTP id 9442417E9 for ; Sun, 24 Feb 2019 13:09:07 +0000 (UTC) Received: from mail.wl.linuxfoundation.org (localhost [127.0.0.1]) by mail.wl.linuxfoundation.org (Postfix) with ESMTP id 83213296ED for ; Sun, 24 Feb 2019 13:09:07 +0000 (UTC) Received: by mail.wl.linuxfoundation.org (Postfix, from userid 486) id 777CC2A98F; Sun, 24 Feb 2019 13:09:07 +0000 (UTC) X-Spam-Checker-Version: SpamAssassin 3.3.1 (2010-03-16) on pdx-wl-mail.web.codeaurora.org X-Spam-Level: X-Spam-Status: No, score=-7.9 required=2.0 tests=BAYES_00,MAILING_LIST_MULTI, RCVD_IN_DNSWL_HI,UNPARSEABLE_RELAY autolearn=ham version=3.3.1 Received: from vger.kernel.org (vger.kernel.org [209.132.180.67]) by mail.wl.linuxfoundation.org (Postfix) with ESMTP id 1ED95296ED for ; Sun, 24 Feb 2019 13:09:01 +0000 (UTC) Received: (majordomo@vger.kernel.org) by vger.kernel.org via listexpand id S1728233AbfBXNJA (ORCPT ); Sun, 24 Feb 2019 08:09:00 -0500 Received: from mail-il-dmz.mellanox.com ([193.47.165.129]:50612 "EHLO mellanox.co.il" rhost-flags-OK-OK-OK-FAIL) by vger.kernel.org with ESMTP id S1728308AbfBXNI7 (ORCPT ); Sun, 24 Feb 2019 08:08:59 -0500 Received: from Internal Mail-Server by MTLPINE1 (envelope-from noaos@mellanox.com) with ESMTPS (AES256-SHA encrypted); 24 Feb 2019 15:06:49 +0200 Received: from reg-l-vrt-059-009.mtl.labs.mlnx (reg-l-vrt-059-009.mtl.labs.mlnx [10.135.59.9]) by labmailer.mlnx (8.13.8/8.13.8) with ESMTP id x1OD6mfn026210; Sun, 24 Feb 2019 15:06:49 +0200 From: Noa Osherovich To: leonro@mellanox.com, jgg@mellanox.com, dledford@redhat.com Cc: linux-rdma@vger.kernel.org, Noa Osherovich Subject: [PATCH rdma-core 19/19] Documentation: update pyverbs Date: Sun, 24 Feb 2019 15:06:38 +0200 Message-Id: <20190224130638.31848-20-noaos@mellanox.com> X-Mailer: git-send-email 2.17.2 In-Reply-To: <20190224130638.31848-1-noaos@mellanox.com> References: <20190224130638.31848-1-noaos@mellanox.com> Sender: linux-rdma-owner@vger.kernel.org Precedence: bulk List-ID: X-Mailing-List: linux-rdma@vger.kernel.org X-Virus-Scanned: ClamAV using ClamSMTP Update pyverbs.md with small examples: - Query a port - Extended query device - Create a PD - Create a MR - Create a MW - Create a DM - Create a DmMr Signed-off-by: Noa Osherovich --- Documentation/pyverbs.md | 129 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 129 insertions(+) diff --git a/Documentation/pyverbs.md b/Documentation/pyverbs.md index a3e85c26574a..62faaed97b13 100644 --- a/Documentation/pyverbs.md +++ b/Documentation/pyverbs.md @@ -23,6 +23,7 @@ when needed (e.g. user buffer for memory region). The memory will be accessible to the users, but not allocated or freed by them. ## Usage Examples +Note that all examples use a hard-coded device name ('mlx5_0'). ##### Open an IB device Import the device module and open a device by name: @@ -96,3 +97,131 @@ print(gid) ``` 'gid' is Pyverbs' equivalent to ibv_gid, provided to the user by Pyverbs. + +##### Query port +The following code snippet provides an example of pyverbs' equivalent of +querying an a port. Context's query_port() command wraps ibv_query_port(). +The example below queries the first port of the device. +```python +import pyverbs.device as d +ctx=d.Context(name='mlx5_0') +port_attr = ctx.query_port(1) +print(port_attr) +Port state : Active (4) +Max MTU : 4096 (5) +Active MTU : 1024 (3) +SM lid : 0 +Port lid : 0 +lmc : 0x0 +Link layer : Ethernet +Max message size : 0x40000000 +Port cap flags : IBV_PORT_CM_SUP IBV_PORT_IP_BASED_GIDS +Port cap flags 2 : +max VL num : 0 +Bad Pkey counter : 0 +Qkey violations counter : 0 +Gid table len : 256 +Pkey table len : 1 +SM sl : 0 +Subnet timeout : 0 +Init type reply : 0 +Active width : 4X (2) +Ative speed : 25.0 Gbps (32) +Phys state : Link up (5) +Flags : 1 +``` + +##### Extended query device +The example below shows how to open a device using pyverbs and query the +extended device's attributes. +Context's query_device_ex() command wraps ibv_query_device_ex(). +```python +import pyverbs.device as d + +ctx = d.Context(name='mlx5_0') +attr = ctx.query_device_ex() +attr.max_dm_size +131072 +attr.rss_caps.max_rwq_indirection_table_size +2048 +``` + +#### Create RDMA objects +##### PD +The following example shows how to open a device and use its context to create +a PD. +```python +import pyverbs.device as d +from pyverbs.pd import PD + +with d.Context(name='mlx5_0') as ctx: + pd = PD(ctx) +``` +##### MR +The example below shows how to create a MR using pyverbs. Similar to C, a +device must be opened prior to creation and a PD has to be allocated. +```python +import pyverbs.device as d +from pyverbs.pd import PD +from pyverbs.mr import MR +import pyverbs.enums as e + +with d.Context(name='mlx5_0') as ctx: + with PD(ctx) as pd: + mr_len = 1000 + flags = e.IBV_ACCESS_LOCAL_WRITE + mr = MR(pd, mr_len, flags) +``` +##### Memory window +The following example shows the equivalent of creating a type 1 memory window. +It includes opening a device and allocating the necessary PD. +```python +import pyverbs.device as d +from pyverbs.pd import PD +from pyverbs.mr import MW +import pyverbs.enums as e + +with d.Context(name='mlx5_0') as ctx: + with PD(ctx) as pd: + mw = MW(pd, e.IBV_MW_TYPE_1) +``` +##### Device memory +The following snippet shows how to allocate a DM - a direct memory object, +using the device's memory. +```python +import random + +from pyverbs.device import DM, AllocDmAttr +import pyverbs.device as d + +with d.Context(name='mlx5_0') as ctx: + attr = ctx.query_device_ex() + if attr.max_dm_size != 0: + dm_len = random.randint(4, attr.max_dm_size) + dm_attrs = AllocDmAttr(dm_len) + dm = DM(ctx, dm_attrs) +``` + +##### DM MR +The example below shows how to open a DMMR - device memory MR, using the +device's own memory rather than a user-allocated buffer. +```python +import random + +from pyverbs.device import DM, AllocDmAttr +from pyverbs.mr import DmMr +import pyverbs.device as d +from pyverbs.pd import PD +import pyverbs.enums as e + +with d.Context(name='mlx5_0') as ctx: + attr = ctx.query_device_ex() + if attr.max_dm_size != 0: + dm_len = random.randint(4, attr.max_dm_size) + dm_attrs = AllocDmAttr(dm_len) + dm_mr_len = random.randint(4, dm_len) + with DM(ctx, dm_attrs) as dm: + with PD(ctx) as pd: + dm_mr = DmMr(pd, dm_mr_len, e.IBV_ACCESS_ZERO_BASED, dm=dm, + offset=0) +``` \ No newline at end of file