From patchwork Mon May 6 15:07:28 2019 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Noa Osherovich X-Patchwork-Id: 10931237 Return-Path: Received: from mail.wl.linuxfoundation.org (pdx-wl-mail.web.codeaurora.org [172.30.200.125]) by pdx-korg-patchwork-2.web.codeaurora.org (Postfix) with ESMTP id 935A313AD for ; Mon, 6 May 2019 15:07:47 +0000 (UTC) Received: from mail.wl.linuxfoundation.org (localhost [127.0.0.1]) by mail.wl.linuxfoundation.org (Postfix) with ESMTP id 806F2205AD for ; Mon, 6 May 2019 15:07:47 +0000 (UTC) Received: by mail.wl.linuxfoundation.org (Postfix, from userid 486) id 7450C287B6; Mon, 6 May 2019 15:07:47 +0000 (UTC) X-Spam-Checker-Version: SpamAssassin 3.3.1 (2010-03-16) on pdx-wl-mail.web.codeaurora.org X-Spam-Level: X-Spam-Status: No, score=-7.9 required=2.0 tests=BAYES_00,MAILING_LIST_MULTI, RCVD_IN_DNSWL_HI,UNPARSEABLE_RELAY autolearn=ham version=3.3.1 Received: from vger.kernel.org (vger.kernel.org [209.132.180.67]) by mail.wl.linuxfoundation.org (Postfix) with ESMTP id 1229F2878F for ; Mon, 6 May 2019 15:07:46 +0000 (UTC) Received: (majordomo@vger.kernel.org) by vger.kernel.org via listexpand id S1726736AbfEFPHp (ORCPT ); Mon, 6 May 2019 11:07:45 -0400 Received: from mail-il-dmz.mellanox.com ([193.47.165.129]:50074 "EHLO mellanox.co.il" rhost-flags-OK-OK-OK-FAIL) by vger.kernel.org with ESMTP id S1727388AbfEFPHo (ORCPT ); Mon, 6 May 2019 11:07:44 -0400 Received: from Internal Mail-Server by MTLPINE2 (envelope-from noaos@mellanox.com) with ESMTPS (AES256-SHA encrypted); 6 May 2019 18:07:40 +0300 Received: from reg-l-vrt-059-009.mtl.labs.mlnx (reg-l-vrt-059-009.mtl.labs.mlnx [10.135.59.9]) by labmailer.mlnx (8.13.8/8.13.8) with ESMTP id x46F7edX019922; Mon, 6 May 2019 18:07:40 +0300 From: Noa Osherovich To: leon@kernel.org, jgg@mellanox.com, dledford@redhat.com Cc: linux-rdma@vger.kernel.org, Noa Osherovich Subject: [PATCH rdma-core 01/11] pyverbs: Add support for address handle creation Date: Mon, 6 May 2019 18:07:28 +0300 Message-Id: <20190506150738.19477-2-noaos@mellanox.com> X-Mailer: git-send-email 2.17.2 In-Reply-To: <20190506150738.19477-1-noaos@mellanox.com> References: <20190506150738.19477-1-noaos@mellanox.com> Sender: linux-rdma-owner@vger.kernel.org Precedence: bulk List-ID: X-Mailing-List: linux-rdma@vger.kernel.org X-Virus-Scanned: ClamAV using ClamSMTP This patch adds the needed classes for Address Handle creation: GRH, GlobalRoute, AHAttr and AH. Signed-off-by: Noa Osherovich --- pyverbs/CMakeLists.txt | 3 +- pyverbs/addr.pxd | 16 +- pyverbs/addr.pyx | 389 +++++++++++++++++++++++++++++++++++++++-- pyverbs/libibverbs.pxd | 35 ++++ pyverbs/pd.pxd | 1 + pyverbs/pd.pyx | 6 +- pyverbs/utils.py | 35 ++++ 7 files changed, 466 insertions(+), 19 deletions(-) create mode 100644 pyverbs/utils.py diff --git a/pyverbs/CMakeLists.txt b/pyverbs/CMakeLists.txt index 65cd578cdff4..b30793dcbb8f 100644 --- a/pyverbs/CMakeLists.txt +++ b/pyverbs/CMakeLists.txt @@ -12,9 +12,10 @@ rdma_cython_module(pyverbs ) rdma_python_module(pyverbs - pyverbs_error.py __init__.py + pyverbs_error.py run_tests.py + utils.py ) rdma_python_test(pyverbs/tests diff --git a/pyverbs/addr.pxd b/pyverbs/addr.pxd index 313cd68fb0d2..389c2d5bdb2e 100644 --- a/pyverbs/addr.pxd +++ b/pyverbs/addr.pxd @@ -1,9 +1,23 @@ # SPDX-License-Identifier: (GPL-2.0 OR Linux-OpenIB) # Copyright (c) 2018, Mellanox Technologies. All rights reserved. See COPYING file -from .base cimport PyverbsObject +from .base cimport PyverbsObject, PyverbsCM from pyverbs cimport libibverbs as v cdef class GID(PyverbsObject): cdef v.ibv_gid gid + +cdef class GRH(PyverbsObject): + cdef v.ibv_grh grh + +cdef class GlobalRoute(PyverbsObject): + cdef v.ibv_global_route gr + +cdef class AHAttr(PyverbsObject): + cdef v.ibv_ah_attr ah_attr + +cdef class AH(PyverbsCM): + cdef v.ibv_ah *ah + cdef object pd + cpdef close(self) diff --git a/pyverbs/addr.pyx b/pyverbs/addr.pyx index 5713f9392d0f..462a45bc7a4a 100644 --- a/pyverbs/addr.pyx +++ b/pyverbs/addr.pyx @@ -1,9 +1,14 @@ # SPDX-License-Identifier: (GPL-2.0 OR Linux-OpenIB) # Copyright (c) 2018, Mellanox Technologies. All rights reserved. See COPYING file -import sys from libc.stdint cimport uint8_t + +from pyverbs.utils import gid_str_to_array, gid_str from .pyverbs_error import PyverbsUserError +from pyverbs.base import PyverbsRDMAErrno +cimport pyverbs.libibverbs as v +from pyverbs.pd cimport PD +from pyverbs.cq cimport WC cdef extern from 'endian.h': unsigned long be64toh(unsigned long host_64bits) @@ -13,6 +18,13 @@ cdef class GID(PyverbsObject): """ GID class represents ibv_gid. It enables user to query for GIDs values. """ + def __cinit__(self, val=None): + if val is not None: + vals = gid_str_to_array(val) + + for i in range(16): + self.gid.raw[i] = int(vals[i],16) + @property def gid(self): """ @@ -29,23 +41,368 @@ cdef class GID(PyverbsObject): 'xxxx:xxxx:xxxx:xxxx:xxxx:xxxx:xxxx:xxxx' :return: None """ - val = val.split(':') - if len(val) != 8: - raise PyverbsUserError("Invalid GID value ({val})".format(val=val)) - if any([len(v) != 4 for v in val]): - raise PyverbsUserError("Invalid GID value ({val})".format(val=val)) - val_int = int("".join(val), 16) - vals = [] - for i in range(8): - vals.append(val[i][0:2]) - vals.append(val[i][2:4]) + self._set_gid(val) + + def _set_gid(self, val): + vals = gid_str_to_array(val) for i in range(16): self.gid.raw[i] = int(vals[i],16) def __str__(self): - hex_values = '%016x%016x' % (be64toh(self.gid._global.subnet_prefix), - be64toh(self.gid._global.interface_id)) - return ':'.join([hex_values[0:4], hex_values[4:8], hex_values[8:12], - hex_values[12:16], hex_values[16:20], hex_values[20:24], - hex_values[24:28],hex_values[28:32]]) + return gid_str(self.gid._global.subnet_prefix, + self.gid._global.interface_id) + + +cdef class GRH(PyverbsObject): + """ + Represents ibv_grh struct. Used when creating or initializing an + Address Handle from a Work Completion. + """ + def __cinit__(self, GID sgid=None, GID dgid=None, version_tclass_flow=0, + paylen=0, next_hdr=0, hop_limit=1): + """ + Initializes a GRH object + :param sgid: Source GID + :param dgid: Destination GID + :param version_tclass_flow: A 32b big endian used to communicate + service level e.g. across subnets + :param paylen: A 16b big endian that is the packet length in bytes, + starting from the first byte after the GRH up to and + including the last byte of the ICRC + :param next_hdr: An 8b unsigned integer specifying the next header + For non-raw packets: 0x1B + For raw packets: According to IETF RFC 1700 + :param hop_limit: An 8b unsigned integer specifying the number of hops + (i.e. routers) that the packet is permitted to take + prior to being discarded + :return: A GRH object + """ + self.grh.dgid = dgid.gid + self.grh.sgid = sgid.gid + self.grh.version_tclass_flow = version_tclass_flow + self.grh.paylen = paylen + self.grh.next_hdr = next_hdr + self.grh.hop_limit = hop_limit + + @property + def dgid(self): + return gid_str(self.grh.dgid._global.subnet_prefix, + self.grh.dgid._global.interface_id) + @dgid.setter + def dgid(self, val): + vals = gid_str_to_array(val) + for i in range(16): + self.grh.dgid.raw[i] = int(vals[i],16) + + @property + def sgid(self): + return gid_str(self.grh.sgid._global.subnet_prefix, + self.grh.sgid._global.interface_id) + @sgid.setter + def sgid(self, val): + vals = gid_str_to_array(val) + for i in range(16): + self.grh.sgid.raw[i] = int(vals[i],16) + + @property + def version_tclass_flow(self): + return self.grh.version_tclass_flow + + @version_tclass_flow.setter + def version_tclass_flow(self, val): + self.grh.version_tclass_flow = val + + @property + def paylen(self): + return self.grh.paylen + @paylen.setter + def paylen(self, val): + self.grh.paylen = val + + @property + def next_hdr(self): + return self.grh.next_hdr + @next_hdr.setter + def next_hdr(self, val): + self.grh.next_hdr = val + + @property + def hop_limit(self): + return self.grh.hop_limit + @hop_limit.setter + def hop_limit(self, val): + self.grh.hop_limit = val + + def __str__(self): + print_format = '{:22}: {:<20}\n' + return print_format.format('DGID', self.dgid) +\ + print_format.format('SGID', self.sgid) +\ + print_format.format('version tclass flow', self.version_tclass_flow) +\ + print_format.format('paylen', self.paylen) +\ + print_format.format('next header', self.next_hdr) +\ + print_format.format('hop limit', self.hop_limit) + + +cdef class GlobalRoute(PyverbsObject): + """ + Represents ibv_global_route. Used in Address Handle creation and describes + the values to be used in the GRH of the packets that will be sent using + this Address Handle. + """ + def __cinit__(self, GID dgid=None, flow_label=0, sgid_index=0, hop_limit=1, + traffic_class=0): + """ + Initializes a GlobalRoute object with given parameters. + :param dgid: Destination GID + :param flow_label: A 20b value. If non-zero, gives a hint to switches + and routers that this sequence of packets must be + delivered in order + :param sgid_index: An index in the port's GID table that identifies the + originator of the packet + :param hop_limit: An 8b unsigned integer specifying the number of hops + (i.e. routers) that the packet is permitted to take + prior to being discarded + :param traffic_class: An 8b unsigned integer specifying the required + delivery priority for routers + :return: A GlobalRoute object + """ + self.gr.dgid=dgid.gid + self.gr.flow_label = flow_label + self.gr.sgid_index = sgid_index + self.gr.hop_limit = hop_limit + self.gr.traffic_class = traffic_class + + @property + def dgid(self): + return gid_str(self.gr.dgid._global.subnet_prefix, + self.gr.dgid._global.interface_id) + @dgid.setter + def dgid(self, val): + vals = gid_str_to_array(val) + for i in range(16): + self.gr.dgid.raw[i] = int(vals[i],16) + + @property + def flow_label(self): + return self.gr.flow_label + @flow_label.setter + def flow_label(self, val): + self.gr.flow_label = val + + @property + def sgid_index(self): + return self.gr.sgid_index + @sgid_index.setter + def sgid_index(self, val): + self.gr.sgid_index = val + + @property + def hop_limit(self): + return self.gr.hop_limit + @hop_limit.setter + def hop_limit(self, val): + self.gr.hop_limit = val + + @property + def traffic_class(self): + return self.gr.traffic_class + @traffic_class.setter + def traffic_class(self, val): + self.gr.traffic_class = val + + def __str__(self): + print_format = '{:22}: {:<20}\n' + return print_format.format('DGID', self.dgid) +\ + print_format.format('flow label', self.flow_label) +\ + print_format.format('sgid index', self.sgid_index) +\ + print_format.format('hop limit', self.hop_limit) +\ + print_format.format('traffic class', self.traffic_class) + + +cdef class AHAttr(PyverbsObject): + """ Represents ibv_ah_attr struct """ + def __cinit__(self, dlid=0, sl=0, src_path_bits=0, static_rate=0, + is_global=0, port_num=1, GlobalRoute gr=None): + """ + Initializes an AHAttr object. + :param dlid: Destination LID, a 16b unsigned integer + :param sl: Service level, an 8b unsigned integer + :param src_path_bits: When LMC (LID mask count) is used in the port, + packets are being sent with the port's base LID, + bitwise ORed with the value of the src_path_bits. + An 8b unsigned integer + :param static_rate: An 8b unsigned integer limiting the rate of packets + that are being sent to the subnet + :param is_global: If non-zero, GRH information exists in the Address + Handle + :param port_num: The local physical port from which the packets will be + sent + :param grh: Attributes of a global routing header. Will only be used if + is_global is non zero. + :return: An AHAttr object + """ + self.ah_attr.port_num = port_num + self.ah_attr.sl = sl + self.ah_attr.src_path_bits = src_path_bits + self.ah_attr.dlid = dlid + self.ah_attr.static_rate = static_rate + self.ah_attr.is_global = is_global + # Do not set GRH fields for a non-global AH + if is_global: + if gr is None: + raise PyverbsUserError('Global AH Attr is created but gr parameter is None') + self.ah_attr.grh.dgid = gr.gr.dgid + self.ah_attr.grh.flow_label = gr.flow_label + self.ah_attr.grh.sgid_index = gr.sgid_index + self.ah_attr.grh.hop_limit = gr.hop_limit + self.ah_attr.grh.traffic_class = gr.traffic_class + + @property + def port_num(self): + return self.ah_attr.port_num + @port_num.setter + def port_num(self, val): + self.ah_attr.port_num = val + + @property + def sl(self): + return self.ah_attr.sl + @sl.setter + def sl(self, val): + self.ah_attr.sl = val + + @property + def src_path_bits(self): + return self.ah_attr.src_path_bits + @src_path_bits.setter + def src_path_bits(self, val): + self.ah_attr.src_path_bits = val + + @property + def dlid(self): + return self.ah_attr.dlid + @dlid.setter + def dlid(self, val): + self.ah_attr.dlid = val + + @property + def static_rate(self): + return self.ah_attr.static_rate + @static_rate.setter + def static_rate(self, val): + self.ah_attr.static_rate = val + + @property + def is_global(self): + return self.ah_attr.is_global + @is_global.setter + def is_global(self, val): + self.ah_attr.is_global = val + + @property + def dgid(self): + if self.ah_attr.is_global: + return gid_str(self.ah_attr.grh.dgid._global.subnet_prefix, + self.ah_attr.grh.dgid._global.interface_id) + @dgid.setter + def dgid(self, val): + if self.ah_attr.is_global: + vals = gid_str_to_array(val) + for i in range(16): + self.ah_attr.grh.dgid.raw[i] = int(vals[i],16) + + @property + def flow_label(self): + if self.ah_attr.is_global: + return self.ah_attr.grh.flow_label + @flow_label.setter + def flow_label(self, val): + self.ah_attr.grh.flow_label = val + + @property + def sgid_index(self): + if self.ah_attr.is_global: + return self.ah_attr.grh.sgid_index + @sgid_index.setter + def sgid_index(self, val): + self.ah_attr.grh.sgid_index = val + + @property + def hop_limit(self): + if self.ah_attr.is_global: + return self.ah_attr.grh.hop_limit + @hop_limit.setter + def hop_limit(self, val): + self.ah_attr.grh.hop_limit = val + + @property + def traffic_class(self): + if self.ah_attr.is_global: + return self.ah_attr.grh.traffic_class + @traffic_class.setter + def traffic_class(self, val): + self.ah_attr.grh.traffic_class = val + + def __str__(self): + print_format = ' {:22}: {:<20}\n' + if self.is_global: + global_format = print_format.format('dgid', self.dgid) +\ + print_format.format('flow label', self.flow_label) +\ + print_format.format('sgid index', self.sgid_index) +\ + print_format.format('hop limit', self.hop_limit) +\ + print_format.format('traffic_class', self.traffic_class) + else: + global_format = '' + return print_format.format('port num', self.port_num) +\ + print_format.format('sl', self.sl) +\ + print_format.format('source path bits', self.src_path_bits) +\ + print_format.format('dlid', self.dlid) +\ + print_format.format('static rate', self.static_rate) +\ + print_format.format('is global', self.is_global) + global_format + + +cdef class AH(PyverbsCM): + def __cinit__(self, PD pd, **kwargs): + """ + Initializes an AH object with the given values. + Two creation methods are supported: + - Creation via AHAttr object (calls ibv_create_ah) + - Creation via a WC object (calls ibv_create_ah_from_wc) + :param pd: PD object this AH belongs to + :param kwargs: Arguments: + * *attr* (AHAttr) + An AHAttr object (represents ibv_ah_attr struct) + * *wc* + A WC object to use for AH initialization + * *grh* + A GRH object to use for AH initialization (when using wc) + * *port_num* + Port number to be used for this AH (when using wc) + :return: An AH object on success + """ + if len(kwargs) == 1: + # Create AH via ib_create_ah + ah_attr = kwargs['attr'] + self.ah = v.ibv_create_ah(pd.pd, &ah_attr.ah_attr) + else: + # Create AH from WC + wc = kwargs['wc'] + grh = kwargs['grh'] + port_num = kwargs['port_num'] + self.ah = v.ibv_create_ah_from_wc(pd.pd, &wc.wc, &grh.grh, port_num) + if self.ah == NULL: + raise PyverbsRDMAErrno('Failed to create AH') + pd.add_ref(self) + self.pd = pd + + def __dealloc__(self): + self.close() + + cpdef close(self): + self.logger.debug('Closing AH') + if self.ah != NULL: + if v.ibv_destroy_ah(self.ah): + raise PyverbsRDMAErrno('Failed to destroy AH') + self.ah = NULL + self.pd = None diff --git a/pyverbs/libibverbs.pxd b/pyverbs/libibverbs.pxd index 979022069887..118d9ea2c2d4 100644 --- a/pyverbs/libibverbs.pxd +++ b/pyverbs/libibverbs.pxd @@ -228,6 +228,35 @@ cdef extern from 'infiniband/verbs.h': unsigned long tag unsigned int priv + cdef struct ibv_grh: + unsigned int version_tclass_flow + unsigned short paylen + unsigned char next_hdr + unsigned char hop_limit + ibv_gid sgid + ibv_gid dgid + + cdef struct ibv_global_route: + ibv_gid dgid + unsigned int flow_label + unsigned char sgid_index + unsigned char hop_limit + unsigned char traffic_class + + cdef struct ibv_ah_attr: + ibv_global_route grh + unsigned short dlid + unsigned char sl + unsigned char src_path_bits + unsigned char static_rate + unsigned char is_global + unsigned char port_num + + cdef struct ibv_ah: + ibv_context *context + ibv_pd *pd + unsigned int handle + ibv_device **ibv_get_device_list(int *n) void ibv_free_device_list(ibv_device **list) ibv_context *ibv_open_device(ibv_device *device) @@ -287,3 +316,9 @@ cdef extern from 'infiniband/verbs.h': unsigned int ibv_wc_read_flow_tag(ibv_cq_ex *cq) void ibv_wc_read_tm_info(ibv_cq_ex *cq, ibv_wc_tm_info *tm_info) unsigned long ibv_wc_read_completion_wallclock_ns(ibv_cq_ex *cq) + ibv_ah *ibv_create_ah(ibv_pd *pd, ibv_ah_attr *attr) + int ibv_init_ah_from_wc(ibv_context *context, uint8_t port_num, + ibv_wc *wc, ibv_grh *grh, ibv_ah_attr *ah_attr) + ibv_ah *ibv_create_ah_from_wc(ibv_pd *pd, ibv_wc *wc, ibv_grh *grh, + uint8_t port_num) + int ibv_destroy_ah(ibv_ah *ah) diff --git a/pyverbs/pd.pxd b/pyverbs/pd.pxd index fc2e405600d0..0b8fb10e6c67 100644 --- a/pyverbs/pd.pxd +++ b/pyverbs/pd.pxd @@ -11,3 +11,4 @@ cdef class PD(PyverbsCM): cdef add_ref(self, obj) cdef object mrs cdef object mws + cdef object ahs diff --git a/pyverbs/pd.pyx b/pyverbs/pd.pyx index 65cc851b0ecf..c2164f8381bd 100644 --- a/pyverbs/pd.pyx +++ b/pyverbs/pd.pyx @@ -6,6 +6,7 @@ from pyverbs.pyverbs_error import PyverbsRDMAError, PyverbsError from pyverbs.base import PyverbsRDMAErrno from pyverbs.device cimport Context, DM from .mr cimport MR, MW, DMMR +from pyverbs.addr cimport AH cdef extern from 'errno.h': int errno @@ -27,6 +28,7 @@ cdef class PD(PyverbsCM): self.logger.debug('PD: Allocated ibv_pd') self.mrs = weakref.WeakSet() self.mws = weakref.WeakSet() + self.ahs = weakref.WeakSet() def __dealloc__(self): """ @@ -44,7 +46,7 @@ cdef class PD(PyverbsCM): :return: None """ self.logger.debug('Closing PD') - self.close_weakrefs([self.mws, self.mrs]) + self.close_weakrefs([self.ahs, self.mws, self.mrs]) if self.pd != NULL: rc = v.ibv_dealloc_pd(self.pd) if rc != 0: @@ -57,5 +59,7 @@ cdef class PD(PyverbsCM): self.mrs.add(obj) elif isinstance(obj, MW): self.mws.add(obj) + elif isinstance(obj, AH): + self.ahs.add(obj) else: raise PyverbsError('Unrecognized object type') diff --git a/pyverbs/utils.py b/pyverbs/utils.py new file mode 100644 index 000000000000..01adceab568c --- /dev/null +++ b/pyverbs/utils.py @@ -0,0 +1,35 @@ +# SPDX-License-Identifier: (GPL-2.0 OR Linux-OpenIB) +# Copyright (c) 2019 Mellanox Technologies, Inc. All rights reserved. See COPYING file + +import struct + +from pyverbs.pyverbs_error import PyverbsUserError + +be64toh = lambda num: struct.unpack('Q', struct.pack('!Q', num))[0] + +def gid_str(subnet_prefix, interface_id): + hex_values = '%016x%016x' % (be64toh(subnet_prefix), be64toh(interface_id)) + return ':'.join([hex_values[0:4], hex_values[4:8], hex_values[8:12], + hex_values[12:16], hex_values[16:20], hex_values[20:24], + hex_values[24:28],hex_values[28:32]]) + + +def gid_str_to_array(val): + """ + Splits a GID to an array of u8 that can be easily assigned to a GID's raw + array. + :param val: GID value in 8 words format + 'xxxx:xxxx:xxxx:xxxx:xxxx:xxxx:xxxx:xxxx' + :return: An array of format xx:xx etc. + """ + val = val.split(':') + if len(val) != 8: + raise PyverbsUserError('Invalid GID value ({val})'.format(val=val)) + if any([len(v) != 4 for v in val]): + raise PyverbsUserError('Invalid GID value ({val})'.format(val=val)) + val_int = int(''.join(val), 16) + vals = [] + for i in range(8): + vals.append(val[i][0:2]) + vals.append(val[i][2:4]) + return vals From patchwork Mon May 6 15:07:29 2019 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Noa Osherovich X-Patchwork-Id: 10931235 Return-Path: Received: from mail.wl.linuxfoundation.org (pdx-wl-mail.web.codeaurora.org [172.30.200.125]) by pdx-korg-patchwork-2.web.codeaurora.org (Postfix) with ESMTP id 4DC021575 for ; Mon, 6 May 2019 15:07:47 +0000 (UTC) Received: from mail.wl.linuxfoundation.org (localhost [127.0.0.1]) by mail.wl.linuxfoundation.org (Postfix) with ESMTP id 39656205AD for ; Mon, 6 May 2019 15:07:47 +0000 (UTC) Received: by mail.wl.linuxfoundation.org (Postfix, from userid 486) id 2A8B5287C3; Mon, 6 May 2019 15:07:47 +0000 (UTC) X-Spam-Checker-Version: SpamAssassin 3.3.1 (2010-03-16) on pdx-wl-mail.web.codeaurora.org X-Spam-Level: X-Spam-Status: No, score=-7.9 required=2.0 tests=BAYES_00,MAILING_LIST_MULTI, RCVD_IN_DNSWL_HI,UNPARSEABLE_RELAY autolearn=ham version=3.3.1 Received: from vger.kernel.org (vger.kernel.org [209.132.180.67]) by mail.wl.linuxfoundation.org (Postfix) with ESMTP id 54BFF287B6 for ; Mon, 6 May 2019 15:07:46 +0000 (UTC) Received: (majordomo@vger.kernel.org) by vger.kernel.org via listexpand id S1726836AbfEFPHo (ORCPT ); Mon, 6 May 2019 11:07:44 -0400 Received: from mail-il-dmz.mellanox.com ([193.47.165.129]:50073 "EHLO mellanox.co.il" rhost-flags-OK-OK-OK-FAIL) by vger.kernel.org with ESMTP id S1727092AbfEFPHm (ORCPT ); Mon, 6 May 2019 11:07:42 -0400 Received: from Internal Mail-Server by MTLPINE2 (envelope-from noaos@mellanox.com) with ESMTPS (AES256-SHA encrypted); 6 May 2019 18:07:40 +0300 Received: from reg-l-vrt-059-009.mtl.labs.mlnx (reg-l-vrt-059-009.mtl.labs.mlnx [10.135.59.9]) by labmailer.mlnx (8.13.8/8.13.8) with ESMTP id x46F7edY019922; Mon, 6 May 2019 18:07:40 +0300 From: Noa Osherovich To: leon@kernel.org, jgg@mellanox.com, dledford@redhat.com Cc: linux-rdma@vger.kernel.org, Noa Osherovich Subject: [PATCH rdma-core 02/11] pyverbs: Add unittests for address handle creation Date: Mon, 6 May 2019 18:07:29 +0300 Message-Id: <20190506150738.19477-3-noaos@mellanox.com> X-Mailer: git-send-email 2.17.2 In-Reply-To: <20190506150738.19477-1-noaos@mellanox.com> References: <20190506150738.19477-1-noaos@mellanox.com> Sender: linux-rdma-owner@vger.kernel.org Precedence: bulk List-ID: X-Mailing-List: linux-rdma@vger.kernel.org X-Virus-Scanned: ClamAV using ClamSMTP Check AH creation and destruction API. Also verify that AH can't be created without a GRH on RoCE (the test simply returns if link layer is Infiniband). This patch also removes the redundant rdma_python_test function from buildlib. rdma_python_module can be used instead. Signed-off-by: Noa Osherovich --- buildlib/pyverbs_functions.cmake | 9 ---- pyverbs/CMakeLists.txt | 3 +- pyverbs/tests/addr.py | 72 ++++++++++++++++++++++++++++++++ 3 files changed, 74 insertions(+), 10 deletions(-) create mode 100644 pyverbs/tests/addr.py diff --git a/buildlib/pyverbs_functions.cmake b/buildlib/pyverbs_functions.cmake index bd8e2028f9d5..1966cf3ba1a3 100644 --- a/buildlib/pyverbs_functions.cmake +++ b/buildlib/pyverbs_functions.cmake @@ -35,15 +35,6 @@ function(rdma_python_module PY_MODULE) endforeach() endfunction() -function(rdma_python_test PY_MODULE) - foreach(PY_FILE ${ARGN}) - get_filename_component(LINK "${CMAKE_CURRENT_SOURCE_DIR}/${PY_FILE}" ABSOLUTE) - rdma_create_symlink("${LINK}" "${BUILD_PYTHON}/${PY_MODULE}/${PY_FILE}") - install(FILES ${CMAKE_CURRENT_SOURCE_DIR}/${PY_FILE} - DESTINATION ${CMAKE_INSTALL_PYTHON_ARCH_LIB}/${PY_MODULE}) - endforeach() -endfunction() - # Make a python script runnable from the build/bin directory with all the # correct paths filled in function(rdma_internal_binary) diff --git a/pyverbs/CMakeLists.txt b/pyverbs/CMakeLists.txt index b30793dcbb8f..b7ec880ecd29 100644 --- a/pyverbs/CMakeLists.txt +++ b/pyverbs/CMakeLists.txt @@ -18,8 +18,9 @@ rdma_python_module(pyverbs utils.py ) -rdma_python_test(pyverbs/tests +rdma_python_module(pyverbs/tests tests/__init__.py + tests/addr.py tests/base.py tests/cq.py tests/device.py diff --git a/pyverbs/tests/addr.py b/pyverbs/tests/addr.py new file mode 100644 index 000000000000..1326eec0d0f6 --- /dev/null +++ b/pyverbs/tests/addr.py @@ -0,0 +1,72 @@ +# SPDX-License-Identifier: (GPL-2.0 OR Linux-OpenIB) +# Copyright (c) 2019 Mellanox Technologies, Inc. All rights reserved. See COPYING file + +from pyverbs.addr import GlobalRoute, AHAttr, AH +from pyverbs.pyverbs_error import PyverbsError +from pyverbs.tests.base import PyverbsTestCase +import pyverbs.device as d +import pyverbs.enums as e +from pyverbs.pd import PD +from pyverbs.cq import WC + +class AHTest(PyverbsTestCase): + """ + Test various functionalities of the AH class. + """ + def test_create_ah(self): + """ + Test ibv_create_ah. + """ + for ctx, attr, attr_ex in self.devices: + pd = PD(ctx) + for port_num in range(1, 1 + attr.phys_port_cnt): + gr = get_global_route(ctx, port_num=port_num) + ah_attr = AHAttr(gr=gr, is_global=1, port_num=port_num) + with AH(pd, attr=ah_attr): + pass + # TODO: Test ibv_create_ah_from_wc once we have traffic + + def test_create_ah_roce(self): + """ + Verify that AH can't be created without GRH in RoCE + """ + for ctx, attr, attr_ex in self.devices: + pd = PD(ctx) + for port_num in range(1, 1 + attr.phys_port_cnt): + port_attr = ctx.query_port(port_num) + if port_attr.link_layer == e.IBV_LINK_LAYER_INFINIBAND: + return + ah_attr = AHAttr(is_global=0, port_num=port_num) + try: + ah = AH(pd, attr=ah_attr) + except PyverbsError as err: + assert 'Failed to create AH' in err.args[0] + else: + raise PyverbsError('Created a non-global AH on RoCE') + + def test_destroy_ah(self): + """ + Test ibv_destroy_ah. + """ + for ctx, attr, attr_ex in self.devices: + pd = PD(ctx) + for port_num in range(1, 1 + attr.phys_port_cnt): + gr = get_global_route(ctx) + ah_attr = AHAttr(gr=gr, is_global=1, port_num=port_num) + with AH(pd, attr=ah_attr) as ah: + ah.close() + + +def get_global_route(ctx, gid_index=0, port_num=1): + """ + Queries the provided Context's gid and creates a GlobalRoute + object with sgid_index and the queried GID as dgid. + :param ctx: Context object to query + :param gid_index: GID index to query and use. Default: 0, as it's always + valid + :param port_num: Number of the port to query. Default: 1 + :return: GlobalRoute object + """ + gid = ctx.query_gid(port_num, gid_index) + gr = GlobalRoute(dgid=gid, sgid_index=gid_index) + return gr From patchwork Mon May 6 15:07:30 2019 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Noa Osherovich X-Patchwork-Id: 10931253 Return-Path: Received: from mail.wl.linuxfoundation.org (pdx-wl-mail.web.codeaurora.org [172.30.200.125]) by pdx-korg-patchwork-2.web.codeaurora.org (Postfix) with ESMTP id 3419F1575 for ; Mon, 6 May 2019 15:08:05 +0000 (UTC) Received: from mail.wl.linuxfoundation.org (localhost [127.0.0.1]) by mail.wl.linuxfoundation.org (Postfix) with ESMTP id 22ADF287C3 for ; Mon, 6 May 2019 15:08:05 +0000 (UTC) Received: by mail.wl.linuxfoundation.org (Postfix, from userid 486) id 16FF2287D3; Mon, 6 May 2019 15:08:05 +0000 (UTC) X-Spam-Checker-Version: SpamAssassin 3.3.1 (2010-03-16) on pdx-wl-mail.web.codeaurora.org X-Spam-Level: X-Spam-Status: No, score=-7.9 required=2.0 tests=BAYES_00,MAILING_LIST_MULTI, RCVD_IN_DNSWL_HI,UNPARSEABLE_RELAY autolearn=ham version=3.3.1 Received: from vger.kernel.org (vger.kernel.org [209.132.180.67]) by mail.wl.linuxfoundation.org (Postfix) with ESMTP id B1D0D287C3 for ; Mon, 6 May 2019 15:08:04 +0000 (UTC) Received: (majordomo@vger.kernel.org) by vger.kernel.org via listexpand id S1726906AbfEFPID (ORCPT ); Mon, 6 May 2019 11:08:03 -0400 Received: from mail-il-dmz.mellanox.com ([193.47.165.129]:50102 "EHLO mellanox.co.il" rhost-flags-OK-OK-OK-FAIL) by vger.kernel.org with ESMTP id S1727388AbfEFPHr (ORCPT ); Mon, 6 May 2019 11:07:47 -0400 Received: from Internal Mail-Server by MTLPINE2 (envelope-from noaos@mellanox.com) with ESMTPS (AES256-SHA encrypted); 6 May 2019 18:07:40 +0300 Received: from reg-l-vrt-059-009.mtl.labs.mlnx (reg-l-vrt-059-009.mtl.labs.mlnx [10.135.59.9]) by labmailer.mlnx (8.13.8/8.13.8) with ESMTP id x46F7edZ019922; Mon, 6 May 2019 18:07:40 +0300 From: Noa Osherovich To: leon@kernel.org, jgg@mellanox.com, dledford@redhat.com Cc: linux-rdma@vger.kernel.org, Noa Osherovich Subject: [PATCH rdma-core 03/11] Documentation: Document Address Handle creation with pyverbs Date: Mon, 6 May 2019 18:07:30 +0300 Message-Id: <20190506150738.19477-4-noaos@mellanox.com> X-Mailer: git-send-email 2.17.2 In-Reply-To: <20190506150738.19477-1-noaos@mellanox.com> References: <20190506150738.19477-1-noaos@mellanox.com> Sender: linux-rdma-owner@vger.kernel.org Precedence: bulk List-ID: X-Mailing-List: linux-rdma@vger.kernel.org X-Virus-Scanned: ClamAV using ClamSMTP Add code snippet to show AH creation. Fix a typo in DMMR snippet. Signed-off-by: Noa Osherovich --- Documentation/pyverbs.md | 32 +++++++++++++++++++++++++++++--- 1 file changed, 29 insertions(+), 3 deletions(-) diff --git a/Documentation/pyverbs.md b/Documentation/pyverbs.md index 3a20b7a95769..371fcd07b2cd 100644 --- a/Documentation/pyverbs.md +++ b/Documentation/pyverbs.md @@ -209,7 +209,7 @@ device's own memory rather than a user-allocated buffer. import random from pyverbs.device import DM, AllocDmAttr -from pyverbs.mr import DmMr +from pyverbs.mr import DMMR import pyverbs.device as d from pyverbs.pd import PD import pyverbs.enums as e @@ -222,7 +222,7 @@ with d.Context(name='mlx5_0') as ctx: dm_mr_len = random.randint(4, dm_len) with DM(ctx, dm_attrs) as dm: with PD(ctx) as pd: - dm_mr = DmMr(pd, dm_mr_len, e.IBV_ACCESS_ZERO_BASED, dm=dm, + dm_mr = DMMR(pd, dm_mr_len, e.IBV_ACCESS_ZERO_BASED, dm=dm, offset=0) ``` @@ -280,4 +280,30 @@ flags : 0 Extended CQ: Handle : 0 CQEs : 15 -``` \ No newline at end of file +``` + +##### Addressing related objects +The following code demonstrates creation of GlobalRoute, AHAttr and AH objects. +The example creates a global AH so it can also run on RoCE without +modifications. +```python + +from pyverbs.addr import GlobalRoute, AHAttr, AH +import pyverbs.device as d +from pyverbs.pd import PD + +with d.Context(name='mlx5_0') as ctx: + port_number = 1 + gid_index = 0 # GID index 0 always exists and valid + gid = ctx.query_gid(port_number, gid_index) + gr = GlobalRoute(dgid=gid, sgid_index=gid_index) + ah_attr = AHAttr(gr=gr, is_global=1, port_num=port_number) + print(ah_attr) + with PD(ctx) as pd: + ah = AH(pd, attr=ah_attr) +DGID : fe80:0000:0000:0000:9a03:9bff:fe00:e4bf +flow label : 0 +sgid index : 0 +hop limit : 1 +traffic class : 0 +``` From patchwork Mon May 6 15:07:31 2019 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Noa Osherovich X-Patchwork-Id: 10931249 Return-Path: Received: from mail.wl.linuxfoundation.org (pdx-wl-mail.web.codeaurora.org [172.30.200.125]) by pdx-korg-patchwork-2.web.codeaurora.org (Postfix) with ESMTP id A608513AD for ; Mon, 6 May 2019 15:07:53 +0000 (UTC) Received: from mail.wl.linuxfoundation.org (localhost [127.0.0.1]) by mail.wl.linuxfoundation.org (Postfix) with ESMTP id 914692878F for ; Mon, 6 May 2019 15:07:53 +0000 (UTC) Received: by mail.wl.linuxfoundation.org (Postfix, from userid 486) id 85317287C8; Mon, 6 May 2019 15:07:53 +0000 (UTC) X-Spam-Checker-Version: SpamAssassin 3.3.1 (2010-03-16) on pdx-wl-mail.web.codeaurora.org X-Spam-Level: X-Spam-Status: No, score=-7.9 required=2.0 tests=BAYES_00,MAILING_LIST_MULTI, RCVD_IN_DNSWL_HI,UNPARSEABLE_RELAY autolearn=ham version=3.3.1 Received: from vger.kernel.org (vger.kernel.org [209.132.180.67]) by mail.wl.linuxfoundation.org (Postfix) with ESMTP id 8DDAE2878F for ; Mon, 6 May 2019 15:07:52 +0000 (UTC) Received: (majordomo@vger.kernel.org) by vger.kernel.org via listexpand id S1727522AbfEFPHv (ORCPT ); Mon, 6 May 2019 11:07:51 -0400 Received: from mail-il-dmz.mellanox.com ([193.47.165.129]:50104 "EHLO mellanox.co.il" rhost-flags-OK-OK-OK-FAIL) by vger.kernel.org with ESMTP id S1727387AbfEFPHt (ORCPT ); Mon, 6 May 2019 11:07:49 -0400 Received: from Internal Mail-Server by MTLPINE2 (envelope-from noaos@mellanox.com) with ESMTPS (AES256-SHA encrypted); 6 May 2019 18:07:40 +0300 Received: from reg-l-vrt-059-009.mtl.labs.mlnx (reg-l-vrt-059-009.mtl.labs.mlnx [10.135.59.9]) by labmailer.mlnx (8.13.8/8.13.8) with ESMTP id x46F7eda019922; Mon, 6 May 2019 18:07:40 +0300 From: Noa Osherovich To: leon@kernel.org, jgg@mellanox.com, dledford@redhat.com Cc: linux-rdma@vger.kernel.org, Noa Osherovich Subject: [PATCH rdma-core 04/11] pyverbs: Add work requests related classes Date: Mon, 6 May 2019 18:07:31 +0300 Message-Id: <20190506150738.19477-5-noaos@mellanox.com> X-Mailer: git-send-email 2.17.2 In-Reply-To: <20190506150738.19477-1-noaos@mellanox.com> References: <20190506150738.19477-1-noaos@mellanox.com> Sender: linux-rdma-owner@vger.kernel.org Precedence: bulk List-ID: X-Mailing-List: linux-rdma@vger.kernel.org X-Virus-Scanned: ClamAV using ClamSMTP This patch adds SGE, RecvWR and SendWR classes. Signed-off-by: Noa Osherovich --- pyverbs/CMakeLists.txt | 1 + pyverbs/libibverbs.pxd | 68 +++++++++ pyverbs/wr.pxd | 16 +++ pyverbs/wr.pyx | 310 +++++++++++++++++++++++++++++++++++++++++ 4 files changed, 395 insertions(+) create mode 100644 pyverbs/wr.pxd create mode 100644 pyverbs/wr.pyx diff --git a/pyverbs/CMakeLists.txt b/pyverbs/CMakeLists.txt index b7ec880ecd29..9545e5052628 100644 --- a/pyverbs/CMakeLists.txt +++ b/pyverbs/CMakeLists.txt @@ -9,6 +9,7 @@ rdma_cython_module(pyverbs enums.pyx mr.pyx pd.pyx + wr.pyx ) rdma_python_module(pyverbs diff --git a/pyverbs/libibverbs.pxd b/pyverbs/libibverbs.pxd index 118d9ea2c2d4..33c655f42491 100644 --- a/pyverbs/libibverbs.pxd +++ b/pyverbs/libibverbs.pxd @@ -257,6 +257,74 @@ cdef extern from 'infiniband/verbs.h': ibv_pd *pd unsigned int handle + cdef struct ibv_sge: + unsigned long addr + unsigned int length + unsigned int lkey + + cdef struct ibv_recv_wr: + unsigned long wr_id + ibv_recv_wr *next + ibv_sge *sg_list + int num_sge + + cdef struct rdma: + unsigned long remote_addr + unsigned int rkey + + cdef struct atomic: + unsigned long remote_addr + unsigned long compare_add + unsigned long swap + unsigned int rkey + + cdef struct ud: + ibv_ah *ah + unsigned int remote_qpn + unsigned int remote_qkey + + cdef union wr: + rdma rdma + atomic atomic + ud ud + + cdef struct ibv_mw_bind_info: + ibv_mr *mr + unsigned long addr + unsigned long length + unsigned int mw_access_flags + + cdef struct bind_mw: + ibv_mw *mw + unsigned int rkey + ibv_mw_bind_info bind_info + + cdef struct tso: + void *hdr + unsigned short hdr_sz + unsigned short mss + + cdef union unnamed: + bind_mw bind_mw + tso tso + + cdef struct xrc: + unsigned int remote_srqn + + cdef union qp_type: + xrc xrc + + cdef struct ibv_send_wr: + unsigned long wr_id + ibv_send_wr *next + ibv_sge *sg_list + int num_sge + ibv_wr_opcode opcode + unsigned int send_flags + wr wr + qp_type qp_type + unnamed unnamed + ibv_device **ibv_get_device_list(int *n) void ibv_free_device_list(ibv_device **list) ibv_context *ibv_open_device(ibv_device *device) diff --git a/pyverbs/wr.pxd b/pyverbs/wr.pxd new file mode 100644 index 000000000000..64b16091116a --- /dev/null +++ b/pyverbs/wr.pxd @@ -0,0 +1,16 @@ +# SPDX-License-Identifier: (GPL-2.0 OR Linux-OpenIB) +# Copyright (c) 2019 Mellanox Technologies, Inc. All rights reserved. See COPYING file + +from .base cimport PyverbsCM +from pyverbs cimport libibverbs as v + + +cdef class SGE(PyverbsCM): + cdef v.ibv_sge *sge + cpdef read(self, length, offset) + +cdef class RecvWR(PyverbsCM): + cdef v.ibv_recv_wr recv_wr + +cdef class SendWR(PyverbsCM): + cdef v.ibv_send_wr send_wr diff --git a/pyverbs/wr.pyx b/pyverbs/wr.pyx new file mode 100644 index 000000000000..2dc766282db3 --- /dev/null +++ b/pyverbs/wr.pyx @@ -0,0 +1,310 @@ +# SPDX-License-Identifier: (GPL-2.0 OR Linux-OpenIB) +# Copyright (c) 2019 Mellanox Technologies Inc. All rights reserved. See COPYING file + +from pyverbs.pyverbs_error import PyverbsUserError, PyverbsError +from pyverbs.base import PyverbsRDMAErrno +cimport pyverbs.libibverbs_enums as e +from pyverbs.addr cimport AH + +cdef extern from 'stdlib.h': + void *malloc(size_t size) + void free(void *ptr) +cdef extern from 'string.h': + void *memcpy(void *dest, const void *src, size_t n) + + +cdef class SGE(PyverbsCM): + """ + Represents ibv_sge struct. It has a read function to allow users to keep + track of data. Write function is not provided as a scatter-gather element + can be using a MR or a DMMR. In case direct (device's) memory is used, + write can't be done using memcpy that relies on CPU-specific optimizations. + A SGE has no way to tell which memory it is using. + """ + def __cinit__(self, addr, length, lkey): + """ + Initializes a SGE object. + :param addr: The address to be used for read/write + :param length: Available buffer size + :param lkey: Local key of the used MR/DMMR + :return: A SGE object + """ + self.sge = malloc(sizeof(v.ibv_sge)) + if self.sge == NULL: + raise PyverbsError('Failed to allocate an SGE') + self.sge.addr = addr + self.sge.length = length + self.sge.lkey = lkey + + def __dealloc(self): + self.close() + + cpdef close(self): + free(self.sge) + + cpdef read(self, length, offset): + """ + Reads bytes of data starting at bytes from the + SGE's address. + :param length: How many bytes to read + :param offset: Offset from the SGE's address in bytes + :return: The data written at the SGE's address + offset + """ + cdef char *sg_data + cdef int off = offset + sg_data = (self.sge.addr + off) + return sg_data[:length] + + def _get_sge(self): + return self.sge + + def __str__(self): + print_format = '{:22}: {:<20}\n' + return print_format.format('Address', hex(self.sge.addr)) +\ + print_format.format('Length', self.sge.length) +\ + print_format.format('Key', hex(self.sge.lkey)) + + @property + def addr(self): + return self.sge.addr + @addr.setter + def addr(self, val): + self.sge.addr = val + + @property + def length(self): + return self.sge.length + @length.setter + def length(self, val): + self.sge.length = val + + @property + def lkey(self): + return self.sge.lkey + @lkey.setter + def lkey(self, val): + self.sge.lkey = val + + +cdef class RecvWR(PyverbsCM): + def __cinit__(self, wr_id=0, num_sge=0, sg=None, + RecvWR next_wr=None): + """ + Initializes a RecvWR object. + :param wr_id: A user-defined WR ID + :param num_sge: Size of the scatter-gather array + :param sg: A scatter-gather array + :param: next_wr: The next WR in the list + :return: A RecvWR object + """ + cdef v.ibv_sge *dst + if num_sge < 1 or sg is None: + raise PyverbsUserError('A WR needs at least one SGE') + self.recv_wr.sg_list = malloc(num_sge * sizeof(v.ibv_sge)) + if self.recv_wr.sg_list == NULL: + raise PyverbsRDMAErrno('Failed to malloc SG buffer') + dst = self.recv_wr.sg_list + copy_sg_array(dst, sg, num_sge) + self.recv_wr.num_sge = num_sge + self.recv_wr.wr_id = wr_id + if next_wr is not None: + self.recv_wr.next = &next_wr.recv_wr + + def __dealloc(self): + self.close() + + cpdef close(self): + free(self.recv_wr.sg_list) + + def __str__(self): + print_format = '{:22}: {:<20}\n' + return print_format.format('WR ID', self.recv_wr.wr_id) +\ + print_format.format('Num SGE', self.recv_wr.num_sge) + + @property + def next_wr(self): + if self.recv_wr.next == NULL: + return None + val = RecvWR() + val.recv_wr = self.recv_wr.next[0] + return val + @next_wr.setter + def next_wr(self, RecvWR val not None): + self.recv_wr.next = &val.recv_wr + + @property + def wr_id(self): + return self.recv_wr.wr_id + @wr_id.setter + def wr_id(self, val): + self.recv_wr.wr_id = val + + @property + def num_sge(self): + return self.recv_wr.num_sge + @num_sge.setter + def num_sge(self, val): + self.recv_wr.num_sge = val + + +cdef class SendWR(PyverbsCM): + def __cinit__(self, wr_id=0, opcode=e.IBV_WR_SEND, num_sge=0, sg = None, + send_flags=e.IBV_SEND_SIGNALED, SendWR next_wr = None): + """ + Initialize a SendWR object with user-provided or default values. + :param wr_id: A user-defined WR ID + :param opcode: The WR's opcode + :param num_sge: Number of scatter-gather elements in the WR + :param send_flags: Send flags as define in ibv_send_flags enum + :param sg: A SGE element, head of the scatter-gather list + :return: An initialized SendWR object + """ + cdef v.ibv_sge *dst + if num_sge < 1 or sg is None: + raise PyverbsUserError('A WR needs at least one SGE') + self.send_wr.sg_list = malloc(num_sge * sizeof(v.ibv_sge)) + if self.send_wr.sg_list == NULL: + raise PyverbsRDMAErrno('Failed to malloc SG buffer') + dst = self.send_wr.sg_list + copy_sg_array(dst, sg, num_sge) + self.send_wr.num_sge = num_sge + self.send_wr.wr_id = wr_id + if next_wr is not None: + self.send_wr.next = &next_wr.send_wr + self.send_wr.opcode = opcode + self.send_wr.send_flags = send_flags + + def __dealloc(self): + self.close() + + cpdef close(self): + free(self.send_wr.sg_list) + + def __str__(self): + print_format = '{:22}: {:<20}\n' + return print_format.format('WR ID', self.send_wr.wr_id) +\ + print_format.format('Num SGE', self.send_wr.num_sge) +\ + print_format.format('Opcode', self.send_wr.opcode) +\ + print_format.format('Send flags', + send_flags_to_str(self.send_wr.send_flags)) + + @property + def next_wr(self): + if self.send_wr.next == NULL: + return None + val = SendWR() + val.send_wr = self.send_wr.next[0] + return val + @next_wr.setter + def next_wr(self, SendWR val not None): + self.send_wr.next = &val.send_wr + + @property + def wr_id(self): + return self.send_wr.wr_id + @wr_id.setter + def wr_id(self, val): + self.send_wr.wr_id = val + + @property + def num_sge(self): + return self.send_wr.num_sge + @num_sge.setter + def num_sge(self, val): + self.send_wr.num_sge = val + + @property + def opcode(self): + return self.send_wr.opcode + @opcode.setter + def opcode(self, val): + self.send_wr.opcode = val + + @property + def send_flags(self): + return self.send_wr.send_flags + @send_flags.setter + def send_flags(self, val): + self.send_wr.send_flags = val + + property sg_list: + def __set__(self, SGE val not None): + self.send_wr.sg_list = val.sge + + def set_wr_ud(self, AH ah not None, rqpn, rqkey): + """ + Set the members of the ud struct in the send_wr's wr union. + :param ah: An address handle object + :param rqpn: The remote QP number + :param rqkey: The remote QKey, authorizing access to the destination QP + :return: None + """ + self.send_wr.wr.ud.ah = ah.ah + self.send_wr.wr.ud.remote_qpn = rqpn + self.send_wr.wr.ud.remote_qkey = rqkey + + def set_wr_rdma(self, rkey, addr): + """ + Set the members of the rdma struct in the send_wr's wr union, used for + RDMA extended transport header creation. + :param rkey: Key to access the specified memory address. + :param addr: Start address of the buffer + :return: None + """ + self.send_wr.wr.rdma.remote_addr = addr + self.send_wr.wr.rdma.rkey = rkey + + def set_wr_atomic(self, rkey, addr, compare_add, swap=0): + """ + Set the members of the atomic struct in the send_wr's wr union, used + for the atomic extended transport header. + :param rkey: Key to access the specified memory address. + :param addr: Start address of the buffer + :param compare_add: The data operand used in the compare portion of the + compare and swap operation + :param swap: The data operand used in atomic operations: + - In compare and swap this field is swapped into the + addressed buffer + - In fetch and add this field is added to the contents of + the addressed buffer + :return: None + """ + self.send_wr.wr.atomic.remote_addr = addr + self.send_wr.wr.atomic.rkey = rkey + self.send_wr.wr.atomic.compare_add = compare_add + self.send_wr.wr.atomic.swap = swap + + def set_qp_type_xrc(self, remote_srqn): + """ + Set the members of the xrc struct in the send_wr's qp_type union, used + for the XRC extended transport header. + :param remote_srqn: The XRC SRQ number to be used by the responder fot + this packet + :return: None + """ + self.send_wr.qp_type.xrc.remote_srqn = remote_srqn + +def send_flags_to_str(flags): + send_flags = {e.IBV_SEND_FENCE: 'IBV_SEND_FENCE', + e.IBV_SEND_SIGNALED: 'IBV_SEND_SIGNALED', + e.IBV_SEND_SOLICITED: 'IBV_SEND_SOLICITED', + e.IBV_SEND_INLINE: 'IBV_SEND_INLINE', + e.IBV_SEND_IP_CSUM: 'IBV_SEND_IP_CSUM'} + flags_str = '' + for f in send_flags: + if flags & f: + flags_str += send_flags[f] + flags_str += ' ' + return flags_str + + +cdef copy_sg_array(dst_obj, sg, num_sge): + cdef v.ibv_sge *dst = dst_obj + cdef v.ibv_sge *src + for i in range(num_sge): + # Avoid 'storing unsafe C derivative of temporary Python' errors + # that will occur if we merge the two following lines. + tmp = sg[i]._get_sge() + src = tmp + memcpy(dst, src, sizeof(v.ibv_sge)) + dst += 1 From patchwork Mon May 6 15:07:32 2019 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Noa Osherovich X-Patchwork-Id: 10931255 Return-Path: Received: from mail.wl.linuxfoundation.org (pdx-wl-mail.web.codeaurora.org [172.30.200.125]) by pdx-korg-patchwork-2.web.codeaurora.org (Postfix) with ESMTP id 7B1C9912 for ; Mon, 6 May 2019 15:08:06 +0000 (UTC) Received: from mail.wl.linuxfoundation.org (localhost [127.0.0.1]) by mail.wl.linuxfoundation.org (Postfix) with ESMTP id 62E432878F for ; Mon, 6 May 2019 15:08:06 +0000 (UTC) Received: by mail.wl.linuxfoundation.org (Postfix, from userid 486) id 56CBE287CF; Mon, 6 May 2019 15:08:06 +0000 (UTC) X-Spam-Checker-Version: SpamAssassin 3.3.1 (2010-03-16) on pdx-wl-mail.web.codeaurora.org X-Spam-Level: X-Spam-Status: No, score=-7.9 required=2.0 tests=BAYES_00,MAILING_LIST_MULTI, RCVD_IN_DNSWL_HI,UNPARSEABLE_RELAY autolearn=ham version=3.3.1 Received: from vger.kernel.org (vger.kernel.org [209.132.180.67]) by mail.wl.linuxfoundation.org (Postfix) with ESMTP id 5741A2878F for ; Mon, 6 May 2019 15:08:03 +0000 (UTC) Received: (majordomo@vger.kernel.org) by vger.kernel.org via listexpand id S1726727AbfEFPIC (ORCPT ); Mon, 6 May 2019 11:08:02 -0400 Received: from mail-il-dmz.mellanox.com ([193.47.165.129]:50112 "EHLO mellanox.co.il" rhost-flags-OK-OK-OK-FAIL) by vger.kernel.org with ESMTP id S1727508AbfEFPHv (ORCPT ); Mon, 6 May 2019 11:07:51 -0400 Received: from Internal Mail-Server by MTLPINE2 (envelope-from noaos@mellanox.com) with ESMTPS (AES256-SHA encrypted); 6 May 2019 18:07:41 +0300 Received: from reg-l-vrt-059-009.mtl.labs.mlnx (reg-l-vrt-059-009.mtl.labs.mlnx [10.135.59.9]) by labmailer.mlnx (8.13.8/8.13.8) with ESMTP id x46F7edb019922; Mon, 6 May 2019 18:07:40 +0300 From: Noa Osherovich To: leon@kernel.org, jgg@mellanox.com, dledford@redhat.com Cc: linux-rdma@vger.kernel.org, Noa Osherovich Subject: [PATCH rdma-core 05/11] pyverbs: Add QP related classes Date: Mon, 6 May 2019 18:07:32 +0300 Message-Id: <20190506150738.19477-6-noaos@mellanox.com> X-Mailer: git-send-email 2.17.2 In-Reply-To: <20190506150738.19477-1-noaos@mellanox.com> References: <20190506150738.19477-1-noaos@mellanox.com> Sender: linux-rdma-owner@vger.kernel.org Precedence: bulk List-ID: X-Mailing-List: linux-rdma@vger.kernel.org X-Virus-Scanned: ClamAV using ClamSMTP This patch adds the following classes needed to create a QP: QPCap, PInitAttr, QPInitAttrEx and QPAttr. These classes provide a user-friendly print format. Signed-off-by: Noa Osherovich --- pyverbs/CMakeLists.txt | 1 + pyverbs/libibverbs.pxd | 78 ++++ pyverbs/pd.pyx | 4 + pyverbs/qp.pxd | 21 ++ pyverbs/qp.pyx | 790 +++++++++++++++++++++++++++++++++++++++++ pyverbs/utils.py | 46 +++ 6 files changed, 940 insertions(+) create mode 100644 pyverbs/qp.pxd create mode 100644 pyverbs/qp.pyx diff --git a/pyverbs/CMakeLists.txt b/pyverbs/CMakeLists.txt index 9545e5052628..3b3838ddb8b5 100644 --- a/pyverbs/CMakeLists.txt +++ b/pyverbs/CMakeLists.txt @@ -9,6 +9,7 @@ rdma_cython_module(pyverbs enums.pyx mr.pyx pd.pyx + qp.pyx wr.pyx ) diff --git a/pyverbs/libibverbs.pxd b/pyverbs/libibverbs.pxd index 33c655f42491..039a6952a433 100644 --- a/pyverbs/libibverbs.pxd +++ b/pyverbs/libibverbs.pxd @@ -325,6 +325,84 @@ cdef extern from 'infiniband/verbs.h': qp_type qp_type unnamed unnamed + cdef struct ibv_qp_cap: + unsigned int max_send_wr + unsigned int max_recv_wr + unsigned int max_send_sge + unsigned int max_recv_sge + unsigned int max_inline_data + + cdef struct ibv_qp_init_attr: + void *qp_context + ibv_cq *send_cq + ibv_cq *recv_cq + ibv_srq *srq + ibv_qp_cap cap + ibv_qp_type qp_type + int sq_sig_all + + cdef struct ibv_xrcd: + pass + + cdef struct ibv_rwq_ind_table: + pass + + cdef struct ibv_rx_hash_conf: + pass + + cdef struct ibv_qp_init_attr_ex: + void *qp_context + ibv_cq *send_cq + ibv_cq *recv_cq + ibv_srq *srq + ibv_qp_cap cap + ibv_qp_type qp_type + int sq_sig_all + unsigned int comp_mask + ibv_pd *pd + ibv_xrcd *xrcd + unsigned int create_flags + unsigned short max_tso_header + ibv_rwq_ind_table *rwq_ind_tbl + ibv_rx_hash_conf rx_hash_conf + unsigned int source_qpn + unsigned long send_ops_flags + + cdef struct ibv_qp_attr: + ibv_qp_state qp_state + ibv_qp_state cur_qp_state + ibv_mtu path_mtu + ibv_mig_state path_mig_state + unsigned int qkey + unsigned int rq_psn + unsigned int sq_psn + unsigned int dest_qp_num + unsigned int qp_access_flags + ibv_qp_cap cap + ibv_ah_attr ah_attr + ibv_ah_attr alt_ah_attr + unsigned short pkey_index + unsigned short alt_pkey_index + unsigned char en_sqd_async_notify + unsigned char sq_draining + unsigned char max_rd_atomic + unsigned char max_dest_rd_atomic + unsigned char min_rnr_timer + unsigned char port_num + unsigned char timeout + unsigned char retry_cnt + unsigned char rnr_retry + unsigned char alt_port_num + unsigned char alt_timeout + unsigned int rate_limit + + cdef struct ibv_srq: + ibv_context *context + void *srq_context + ibv_pd *pd + unsigned int handle + unsigned int events_completed + ibv_device **ibv_get_device_list(int *n) void ibv_free_device_list(ibv_device **list) ibv_context *ibv_open_device(ibv_device *device) diff --git a/pyverbs/pd.pyx b/pyverbs/pd.pyx index c2164f8381bd..b09e3bf1a555 100644 --- a/pyverbs/pd.pyx +++ b/pyverbs/pd.pyx @@ -63,3 +63,7 @@ cdef class PD(PyverbsCM): self.ahs.add(obj) else: raise PyverbsError('Unrecognized object type') + + @property + def _pd(self): + return self.pd diff --git a/pyverbs/qp.pxd b/pyverbs/qp.pxd new file mode 100644 index 000000000000..787fda3cb789 --- /dev/null +++ b/pyverbs/qp.pxd @@ -0,0 +1,21 @@ +# SPDX-License-Identifier: (GPL-2.0 OR Linux-OpenIB) +# Copyright (c) 2019 Mellanox Technologies, Inc. All rights reserved. +from pyverbs.base cimport PyverbsObject, PyverbsCM +cimport pyverbs.libibverbs as v + +cdef class QPCap(PyverbsObject): + cdef v.ibv_qp_cap cap + +cdef class QPInitAttr(PyverbsObject): + cdef v.ibv_qp_init_attr attr + cdef object scq + cdef object rcq + +cdef class QPInitAttrEx(PyverbsObject): + cdef v.ibv_qp_init_attr_ex attr + cdef object scq + cdef object rcq + cdef object pd + +cdef class QPAttr(PyverbsObject): + cdef v.ibv_qp_attr attr diff --git a/pyverbs/qp.pyx b/pyverbs/qp.pyx new file mode 100644 index 000000000000..80aee40eae02 --- /dev/null +++ b/pyverbs/qp.pyx @@ -0,0 +1,790 @@ +# SPDX-License-Identifier: (GPL-2.0 OR Linux-OpenIB) +# Copyright (c) 2019 Mellanox Technologies, Inc. All rights reserved. +from pyverbs.utils import gid_str, qp_type_to_str, qp_state_to_str, mtu_to_str +from pyverbs.utils import access_flags_to_str, mig_state_to_str +from pyverbs.pyverbs_error import PyverbsUserError +cimport pyverbs.libibverbs_enums as e +from pyverbs.addr cimport AHAttr, GID +from pyverbs.addr cimport GlobalRoute +from pyverbs.cq cimport CQ, CQEX +cimport pyverbs.libibverbs as v +from pyverbs.pd cimport PD + + +cdef class QPCap(PyverbsObject): + def __cinit__(self, max_send_wr=1, max_recv_wr=10, max_send_sge=1, + max_recv_sge=1, max_inline_data=0): + """ + Initializes a QPCap object with user-provided or default values. + :param max_send_wr: max number of outstanding WRs in the SQ + :param max_recv_wr: max number of outstanding WRs in the RQ + :param max_send_sge: Requested max number of scatter-gather elements in + a WR in the SQ + :param max_recv_sge: Requested max number of scatter-gather elements in + a WR in the RQ + :param max_inline_data: max number of data (bytes) that can be posted + inline to the SQ, otherwise 0 + :return: + """ + self.cap.max_send_wr = max_send_wr + self.cap.max_recv_wr = max_recv_wr + self.cap.max_send_sge = max_send_sge + self.cap.max_recv_sge = max_recv_sge + self.cap.max_inline_data = max_inline_data + + @property + def max_send_wr(self): + return self.cap.max_send_wr + @max_send_wr.setter + def max_send_wr(self, val): + self.cap.max_send_wr = val + + @property + def max_recv_wr(self): + return self.cap.max_recv_wr + @max_recv_wr.setter + def max_recv_wr(self, val): + self.cap.max_recv_wr = val + + @property + def max_send_sge(self): + return self.cap.max_send_sge + @max_send_sge.setter + def max_send_sge(self, val): + self.cap.max_send_sge = val + + @property + def max_recv_sge(self): + return self.cap.max_recv_sge + @max_recv_sge.setter + def max_recv_sge(self, val): + self.cap.max_recv_sge = val + + @property + def max_inline_data(self): + return self.cap.max_inline_data + @max_inline_data.setter + def max_inline_data(self, val): + self.cap.max_inline_data = val + + def __str__(self): + print_format = '{:20}: {:<20}\n' + return print_format.format('max send wrs', self.cap.max_send_wr) +\ + print_format.format('max recv wrs', self.cap.max_recv_wr) +\ + print_format.format('max send sges', self.cap.max_send_sge) +\ + print_format.format('max recv sges', self.cap.max_recv_sge) +\ + print_format.format('max inline data', self.cap.max_inline_data) + + +cdef class QPInitAttr(PyverbsObject): + def __cinit__(self, qp_type=e.IBV_QPT_UD, qp_context=None, + PyverbsObject scq=None, PyverbsObject rcq=None, + object srq=None, QPCap cap=None, sq_sig_all=1): + """ + Initializes a QpInitAttr object representing ibv_qp_init_attr struct. + Note that SRQ object is not yet supported in pyverbs so can't be passed + as a parameter. None should be used until such support is added. + :param qp_type: The desired QP type (see enum ibv_qp_type) + :param qp_context: Associated QP context + :param scq: Send CQ to be used for this QP + :param rcq: Receive CQ to be used for this QP + :param srq: Not yet supported + :param cap: A QPCap object + :param sq_sig_all: If set, each send WR will generate a completion + entry + :return: A QpInitAttr object + """ + _copy_caps(cap, self) + self.attr.qp_context = qp_context + if scq is not None: + if type(scq) is CQ: + self.attr.send_cq = scq._cq + elif type(scq) is CQEX: + self.attr.send_cq = scq._ibv_cq + else: + raise PyverbsUserError('Expected CQ/CQEX, got {t}'.\ + format(t=type(scq))) + self.scq = scq + + if rcq is not None: + if type(rcq) is CQ: + self.attr.recv_cq = rcq._cq + elif type(rcq) is CQEX: + self.attr.recv_cq = rcq._ibv_cq + else: + raise PyverbsUserError('Expected CQ/CQEX, got {t}'.\ + format(t=type(rcq))) + self.rcq = rcq + + self.attr.srq = NULL # Until SRQ support is added + self.attr.qp_type = qp_type + self.attr.sq_sig_all = sq_sig_all + + @property + def send_cq(self): + return self.scq + @send_cq.setter + def send_cq(self, val): + if type(val) is CQ: + self.attr.send_cq = val._cq + elif type(val) is CQEX: + self.attr.send_cq = val._ibv_cq + self.scq = val + + @property + def recv_cq(self): + return self.rcq + @recv_cq.setter + def recv_cq(self, val): + if type(val) is CQ: + self.attr.recv_cq = val._cq + elif type(val) is CQEX: + self.attr.recv_cq = val._ibv_cq + self.rcq = val + + @property + def cap(self): + return QPCap(max_send_wr=self.attr.cap.max_send_wr, + max_recv_wr=self.attr.cap.max_recv_wr, + max_send_sge=self.attr.cap.max_send_sge, + max_recv_sge=self.attr.cap.max_recv_sge, + max_inline_data=self.attr.cap.max_inline_data) + @cap.setter + def cap(self, val): + _copy_caps(val, self) + + @property + def qp_type(self): + return self.attr.qp_type + @qp_type.setter + def qp_type(self, val): + self.attr.qp_type = val + + @property + def sq_sig_all(self): + return self.attr.sq_sig_all + @sq_sig_all.setter + def sq_sig_all(self, val): + self.attr.sq_sig_all = val + + def __str__(self): + print_format = '{:20}: {:<20}\n' + ident_format = ' {:20}: {:<20}\n' + return print_format.format('QP type', qp_type_to_str(self.qp_type)) +\ + print_format.format('SQ sig. all', self.sq_sig_all) +\ + 'QP caps:\n' +\ + ident_format.format('max send WR', self.attr.cap.max_send_wr) +\ + ident_format.format('max recv WR', self.attr.cap.max_recv_wr) +\ + ident_format.format('max send SGE', + self.attr.cap.max_send_sge) +\ + ident_format.format('max recv SGE', + self.attr.cap.max_recv_sge) +\ + ident_format.format('max inline data', + self.attr.cap.max_inline_data) + + +cdef class QPInitAttrEx(PyverbsObject): + def __cinit__(self, qp_type=e.IBV_QPT_UD, qp_context=None, + PyverbsObject scq=None, PyverbsObject rcq=None, + object srq=None, QPCap cap=None, sq_sig_all=0, comp_mask=0, + PD pd=None, object xrcd=None, create_flags=0, + max_tso_header=0, source_qpn=0, object hash_conf=None, + object ind_table=None): + """ + Initialize a QPInitAttrEx object with user-defined or default values. + :param qp_type: QP type to be created + :param qp_context: Associated user context + :param scq: Send CQ to be used for this QP + :param rcq: Recv CQ to be used for this QP + :param srq: Not yet supported + :param cap: A QPCap object + :param sq_sig_all: If set, each send WR will generate a completion + entry + :param comp_mask: bit mask to determine which of the following fields + are valid + :param pd: A PD object to be associated with this QP + :param xrcd: Not yet supported + :param create_flags: Creation flags for this QP + :param max_tso_header: Maximum TSO header size + :param source_qpn: Source QP number (requires IBV_QP_CREATE_SOURCE_QPN + set in create_flags) + :param hash_conf: Not yet supported + :param ind_table: Not yet supported + :return: An initialized QPInitAttrEx object + """ + _copy_caps(cap, self) + if scq is not None: + if type(scq) is CQ: + self.attr.send_cq = scq._cq + elif type(scq) is CQEX: + self.attr.send_cq = scq._ibv_cq + else: + raise PyverbsUserError('Expected CQ/CQEX, got {t}'.\ + format(t=type(scq))) + self.scq = scq + + if rcq is not None: + if type(rcq) is CQ: + self.attr.recv_cq = rcq._cq + elif type(rcq) is CQEX: + self.attr.recv_cq = rcq._ibv_cq + else: + raise PyverbsUserError('Expected CQ/CQEX, got {t}'.\ + format(type(rcq))) + self.rcq = rcq + + self.attr.srq = NULL # Until SRQ support is added + self.attr.xrcd = NULL # Until XRCD support is added + self.attr.rwq_ind_tbl = NULL # Until RSS support is added + self.attr.qp_type = qp_type + self.attr.sq_sig_all = sq_sig_all + unsupp_flags = e.IBV_QP_INIT_ATTR_XRCD | e.IBV_QP_INIT_ATTR_IND_TABLE |\ + e.IBV_QP_INIT_ATTR_RX_HASH + if comp_mask & unsupp_flags: + raise PyverbsUserError('XRCD and RSS are not yet supported in pyverbs') + self.attr.comp_mask = comp_mask + if pd is not None: + self.attr.pd = pd._pd + self.pd = pd + self.attr.create_flags = create_flags + self.attr.max_tso_header = max_tso_header + self.attr.source_qpn = source_qpn + + @property + def send_cq(self): + return self.scq + @send_cq.setter + def send_cq(self, val): + if type(val) is CQ: + self.attr.send_cq = val._cq + elif type(val) is CQEX: + self.attr.send_cq = val._ibv_cq + self.scq = val + + @property + def recv_cq(self): + return self.rcq + @recv_cq.setter + def recv_cq(self, val): + if type(val) is CQ: + self.attr.recv_cq = val._cq + elif type(val) is CQEX: + self.attr.recv_cq = val._ibv_cq + self.rcq = val + + @property + def cap(self): + return QPCap(max_send_wr=self.attr.cap.max_send_wr, + max_recv_wr=self.attr.cap.max_recv_wr, + max_send_sge=self.attr.cap.max_send_sge, + max_recv_sge=self.attr.cap.max_recv_sge, + max_inline_data=self.attr.cap.max_inline_data) + @cap.setter + def cap(self, val): + _copy_caps(val, self) + + @property + def qp_type(self): + return self.attr.qp_type + @qp_type.setter + def qp_type(self, val): + self.attr.qp_type = val + + @property + def sq_sig_all(self): + return self.attr.sq_sig_all + @sq_sig_all.setter + def sq_sig_all(self, val): + self.attr.sq_sig_all = val + + @property + def comp_mask(self): + return self.attr.comp_mask + @comp_mask.setter + def comp_mask(self, val): + self.attr.comp_mask = val + + @property + def pd(self): + return self.pd + @pd.setter + def pd(self, val): + self.attr.pd = val._pd + self.pd = val + + @property + def create_flags(self): + return self.attr.create_flags + @create_flags.setter + def create_flags(self, val): + self.attr.create_flags = val + + @property + def max_tso_header(self): + return self.attr.max_tso_header + @max_tso_header.setter + def max_tso_header(self, val): + self.attr.max_tso_header = val + + @property + def source_qpn(self): + return self.attr.source_qpn + @source_qpn.setter + def source_qpn(self, val): + self.attr.source_qpn = val + + def mask_to_str(self, mask): + comp_masks = {1: 'PD', 2: 'XRCD', 4: 'Create Flags', + 8: 'Max TSO header', 16: 'Indirection Table', + 32: 'RX hash'} + mask_str = '' + for f in comp_masks: + if mask & f: + mask_str += comp_masks[f] + mask_str += ' ' + return mask_str + + def flags_to_str(self, flags): + create_flags = {1: 'Block self mcast loopback', 2: 'Scatter FCS', + 4: 'CVLAN stripping', 8: 'Source QPN', + 16: 'PCI write end padding'} + create_str = '' + for f in create_flags: + if flags & f: + create_str += create_flags[f] + create_str += ' ' + return create_str + + def __str__(self): + print_format = '{:20}: {:<20}\n' + return print_format.format('QP type', qp_type_to_str(self.qp_type)) +\ + print_format.format('SQ sig. all', self.sq_sig_all) +\ + 'QP caps:\n' +\ + print_format.format(' max send WR', + self.attr.cap.max_send_wr) +\ + print_format.format(' max recv WR', + self.attr.cap.max_recv_wr) +\ + print_format.format(' max send SGE', + self.attr.cap.max_send_sge) +\ + print_format.format(' max recv SGE', + self.attr.cap.max_recv_sge) +\ + print_format.format(' max inline data', + self.attr.cap.max_inline_data) +\ + print_format.format('comp mask', + self.mask_to_str(self.attr.comp_mask)) +\ + print_format.format('create flags', + self.flags_to_str(self.attr.create_flags)) +\ + print_format.format('max TSO header', + self.attr.max_tso_header) +\ + print_format.format('Source QPN', self.attr.source_qpn) + + +cdef class QPAttr(PyverbsObject): + def __cinit__(self, qp_state=e.IBV_QPS_INIT, cur_qp_state=e.IBV_QPS_RESET, + port_num=1, path_mtu=e.IBV_MTU_1024): + """ + Initializes a QPQttr object which represents ibv_qp_attr structs. It + can be used to modify a QP. + This function initializes default values for reset-to-init transition. + :param qp_state: Desired QP state + :param cur_qp_state: Current QP state + :return: An initialized QpAttr object + """ + self.attr.qp_state = qp_state + self.attr.cur_qp_state = cur_qp_state + self.attr.port_num = port_num + self.attr.path_mtu = path_mtu + + @property + def qp_state(self): + return self.attr.qp_state + @qp_state.setter + def qp_state(self, val): + self.attr.qp_state = val + + @property + def cur_qp_state(self): + return self.attr.cur_qp_state + @cur_qp_state.setter + def cur_qp_state(self, val): + self.attr.cur_qp_state = val + + @property + def path_mtu(self): + return self.attr.path_mtu + @path_mtu.setter + def path_mtu(self, val): + self.attr.path_mtu = val + + @property + def path_mig_state(self): + return self.attr.path_mig_state + @path_mig_state.setter + def path_mig_state(self, val): + self.attr.path_mig_state = val + + @property + def qkey(self): + return self.attr.qkey + @qkey.setter + def qkey(self, val): + self.attr.qkey = val + + @property + def rq_psn(self): + return self.attr.rq_psn + @rq_psn.setter + def rq_psn(self, val): + self.attr.rq_psn = val + + @property + def sq_psn(self): + return self.attr.sq_psn + @sq_psn.setter + def sq_psn(self, val): + self.attr.sq_psn = val + + @property + def dest_qp_num(self): + return self.attr.dest_qp_num + @dest_qp_num.setter + def dest_qp_num(self, val): + self.attr.dest_qp_num = val + + @property + def qp_access_flags(self): + return self.attr.qp_access_flags + @qp_access_flags.setter + def qp_access_flags(self, val): + self.attr.qp_access_flags = val + + @property + def cap(self): + return QPCap(max_send_wr=self.attr.cap.max_send_wr, + max_recv_wr=self.attr.cap.max_recv_wr, + max_send_sge=self.attr.cap.max_send_sge, + max_recv_sge=self.attr.cap.max_recv_sge, + max_inline_data=self.attr.cap.max_inline_data) + @cap.setter + def cap(self, val): + _copy_caps(val, self) + + @property + def ah_attr(self): + if self.attr.ah_attr.is_global: + gid = gid_str(self.attr.ah_attr.grh.dgid._global.subnet_prefix, + self.attr.ah_attr.grh.dgid._global.interface_id) + g = GID(gid) + gr = GlobalRoute(flow_label=self.attr.ah_attr.grh.flow_label, + sgid_index=self.attr.ah_attr.grh.sgid_index, + hop_limit=self.attr.ah_attr.grh.hop_limit, dgid=g, + traffic_class=self.attr.ah_attr.grh.traffic_class) + else: + gr = None + ah = AHAttr(dlid=self.attr.ah_attr.dlid, sl=self.attr.ah_attr.sl, + src_path_bits=self.attr.ah_attr.src_path_bits, + static_rate=self.attr.ah_attr.static_rate, + is_global=self.attr.ah_attr.is_global, gr=gr) + return ah + + @ah_attr.setter + def ah_attr(self, val): + self._copy_ah(val) + + @property + def alt_ah_attr(self): + if self.attr.alt_ah_attr.is_global: + gid = gid_str(self.attr.alt_ah_attr.grh.dgid._global.subnet_prefix, + self.attr.alt_ah_attr.grh.dgid._global.interface_id) + g = GID(gid) + gr = GlobalRoute(flow_label=self.attr.alt_ah_attr.grh.flow_label, + sgid_index=self.attr.alt_ah_attr.grh.sgid_index, + hop_limit=self.attr.alt_ah_attr.grh.hop_limit, + dgid=g, + traffic_class=self.attr.alt_ah_attr.grh.traffic_class) + else: + gr = None + ah = AHAttr(dlid=self.attr.alt_ah_attr.dlid, + sl=self.attr.alt_ah_attr.sl, + src_path_bits=self.attr.alt_ah_attr.src_path_bits, + static_rate=self.attr.alt_ah_attr.static_rate, + is_global=self.attr.alt_ah_attr.is_global, gr=gr) + return ah + + @alt_ah_attr.setter + def alt_ah_attr(self, val): + self._copy_ah(val, True) + + def _copy_ah(self, AHAttr ah_attr, is_alt=False): + if ah_attr is None: + return + if not is_alt: + for i in range(16): + self.attr.ah_attr.grh.dgid.raw[i] = \ + ah_attr.ah_attr.grh.dgid.raw[i] + self.attr.ah_attr.grh.flow_label = ah_attr.ah_attr.grh.flow_label + self.attr.ah_attr.grh.sgid_index = ah_attr.ah_attr.grh.sgid_index + self.attr.ah_attr.grh.hop_limit = ah_attr.ah_attr.grh.hop_limit + self.attr.ah_attr.grh.traffic_class = \ + ah_attr.ah_attr.grh.traffic_class + self.attr.ah_attr.dlid = ah_attr.ah_attr.dlid + self.attr.ah_attr.sl = ah_attr.ah_attr.sl + self.attr.ah_attr.src_path_bits = ah_attr.ah_attr.src_path_bits + self.attr.ah_attr.static_rate = ah_attr.ah_attr.static_rate + self.attr.ah_attr.is_global = ah_attr.ah_attr.is_global + self.attr.ah_attr.port_num = ah_attr.ah_attr.port_num + else: + for i in range(16): + self.attr.alt_ah_attr.grh.dgid.raw[i] = \ + ah_attr.ah_attr.grh.dgid.raw[i] + self.attr.alt_ah_attr.grh.flow_label = \ + ah_attr.ah_attr.grh.flow_label + self.attr.alt_ah_attr.grh.sgid_index = \ + ah_attr.ah_attr.grh.sgid_index + self.attr.alt_ah_attr.grh.hop_limit = ah_attr.ah_attr.grh.hop_limit + self.attr.alt_ah_attr.grh.traffic_class = \ + ah_attr.ah_attr.grh.traffic_class + self.attr.alt_ah_attr.dlid = ah_attr.ah_attr.dlid + self.attr.alt_ah_attr.sl = ah_attr.ah_attr.sl + self.attr.alt_ah_attr.src_path_bits = ah_attr.ah_attr.src_path_bits + self.attr.alt_ah_attr.static_rate = ah_attr.ah_attr.static_rate + self.attr.alt_ah_attr.is_global = ah_attr.ah_attr.is_global + self.attr.alt_ah_attr.port_num = ah_attr.ah_attr.port_num + + @property + def pkey_index(self): + return self.attr.pkey_index + @pkey_index.setter + def pkey_index(self, val): + self.attr.pkey_index = val + + @property + def alt_pkey_index(self): + return self.attr.alt_pkey_index + @alt_pkey_index.setter + def alt_pkey_index(self, val): + self.attr.alt_pkey_index = val + + @property + def en_sqd_async_notify(self): + return self.attr.en_sqd_async_notify + @en_sqd_async_notify.setter + def en_sqd_async_notify(self, val): + self.attr.en_sqd_async_notify = val + + @property + def sq_draining(self): + return self.attr.sq_draining + @sq_draining.setter + def sq_draining(self, val): + self.attr.sq_draining = val + + @property + def max_rd_atomic(self): + return self.attr.max_rd_atomic + @max_rd_atomic.setter + def max_rd_atomic(self, val): + self.attr.max_rd_atomic = val + + @property + def max_dest_rd_atomic(self): + return self.attr.max_dest_rd_atomic + @max_dest_rd_atomic.setter + def max_dest_rd_atomic(self, val): + self.attr.max_dest_rd_atomic = val + + @property + def min_rnr_timer(self): + return self.attr.min_rnr_timer + @min_rnr_timer.setter + def min_rnr_timer(self, val): + self.attr.min_rnr_timer = val + + @property + def port_num(self): + return self.attr.port_num + @port_num.setter + def port_num(self, val): + self.attr.port_num = val + + @property + def timeout(self): + return self.attr.timeout + @timeout.setter + def timeout(self, val): + self.attr.timeout = val + + @property + def retry_cnt(self): + return self.attr.retry_cnt + @retry_cnt.setter + def retry_cnt(self, val): + self.attr.retry_cnt = val + + @property + def rnr_retry(self): + return self.attr.rnr_retry + @rnr_retry.setter + def rnr_retry(self, val): + self.attr.rnr_retry = val + + @property + def alt_port_num(self): + return self.attr.alt_port_num + @alt_port_num.setter + def alt_port_num(self, val): + self.attr.alt_port_num = val + + @property + def alt_timeout(self): + return self.attr.alt_timeout + @alt_timeout.setter + def alt_timeout(self, val): + self.attr.alt_timeout = val + + @property + def rate_limit(self): + return self.attr.rate_limit + @rate_limit.setter + def rate_limit(self, val): + self.attr.rate_limit = val + + def __str__(self): + print_format = '{:22}: {:<20}\n' + ah_format = ' {:22}: {:<20}\n' + ident_format = ' {:22}: {:<20}\n' + if self.attr.ah_attr.is_global: + global_ah = ah_format.format('dgid', + gid_str(self.attr.ah_attr.grh.dgid._global.subnet_prefix, + self.attr.ah_attr.grh.dgid._global.interface_id)) +\ + ah_format.format('flow label', + self.attr.ah_attr.grh.flow_label) +\ + ah_format.format('sgid index', + self.attr.ah_attr.grh.sgid_index) +\ + ah_format.format('hop limit', + self.attr.ah_attr.grh.hop_limit) +\ + ah_format.format('traffic_class', + self.attr.ah_attr.grh.traffic_class) + else: + global_ah = '' + if self.attr.alt_ah_attr.is_global: + alt_global_ah = ah_format.format('dgid', + gid_str(self.attr.alt_ah_attr.grh.dgid._global.subnet_prefix, + self.attr.alt_ah_attr.grh.dgid._global.interface_id)) +\ + ah_format.format('flow label', + self.attr.alt_ah_attr.grh.flow_label) +\ + ah_format.format('sgid index', + self.attr.alt_ah_attr.grh.sgid_index) +\ + ah_format.format('hop limit', + self.attr.alt_ah_attr.grh.hop_limit) +\ + ah_format.format('traffic_class', + self.attr.alt_ah_attr.grh.traffic_class) + else: + alt_global_ah = '' + return print_format.format('QP state', + qp_state_to_str(self.attr.qp_state)) +\ + print_format.format('QP current state', + qp_state_to_str(self.attr.cur_qp_state)) +\ + print_format.format('Path MTU', + mtu_to_str(self.attr.path_mtu)) +\ + print_format.format('Path mig. state', + mig_state_to_str(self.attr.path_mig_state)) +\ + print_format.format('QKey', self.attr.qkey) +\ + print_format.format('RQ PSN', self.attr.rq_psn) +\ + print_format.format('SQ PSN', self.attr.sq_psn) +\ + print_format.format('Dest QP number', self.attr.dest_qp_num) +\ + print_format.format('QP access flags', + access_flags_to_str(self.attr.qp_access_flags)) +\ + 'QP caps:\n' +\ + ident_format.format('max send WR', + self.attr.cap.max_send_wr) +\ + ident_format.format('max recv WR', + self.attr.cap.max_recv_wr) +\ + ident_format.format('max send SGE', + self.attr.cap.max_send_sge) +\ + ident_format.format('max recv SGE', + self.attr.cap.max_recv_sge) +\ + ident_format.format('max inline data', + self.attr.cap.max_inline_data) +\ + 'AH Attr:\n' +\ + ident_format.format('port num', self.attr.ah_attr.port_num) +\ + ident_format.format('sl', self.attr.ah_attr.sl) +\ + ident_format.format('source path bits', + self.attr.ah_attr.src_path_bits) +\ + ident_format.format('dlid', self.attr.ah_attr.dlid) +\ + ident_format.format('port num', self.attr.ah_attr.port_num) +\ + ident_format.format('static rate', + self.attr.ah_attr.static_rate) +\ + ident_format.format('is global', + self.attr.ah_attr.is_global) +\ + global_ah +\ + 'Alt. AH Attr:\n' +\ + ident_format.format('port num', self.attr.alt_ah_attr.port_num) +\ + ident_format.format('sl', self.attr.alt_ah_attr.sl) +\ + ident_format.format('source path bits', + self.attr.alt_ah_attr.src_path_bits) +\ + ident_format.format('dlid', self.attr.alt_ah_attr.dlid) +\ + ident_format.format('port num', self.attr.alt_ah_attr.port_num) +\ + ident_format.format('static rate', + self.attr.alt_ah_attr.static_rate) +\ + ident_format.format('is global', + self.attr.alt_ah_attr.is_global) +\ + alt_global_ah +\ + print_format.format('PKey index', self.attr.pkey_index) +\ + print_format.format('Alt. PKey index', + self.attr.alt_pkey_index) +\ + print_format.format('En. SQD async notify', + self.attr.en_sqd_async_notify) +\ + print_format.format('SQ draining', self.attr.sq_draining) +\ + print_format.format('Max RD atomic', self.attr.max_rd_atomic) +\ + print_format.format('Max dest. RD atomic', + self.attr.max_dest_rd_atomic) +\ + print_format.format('Min RNR timer', self.attr.min_rnr_timer) +\ + print_format.format('Port number', self.attr.port_num) +\ + print_format.format('Timeout', self.attr.timeout) +\ + print_format.format('Retry counter', self.attr.retry_cnt) +\ + print_format.format('RNR retry', self.attr.rnr_retry) +\ + print_format.format('Alt. port number', + self.attr.alt_port_num) +\ + print_format.format('Alt. timeout', self.attr.alt_timeout) +\ + print_format.format('Rate limit', self.attr.rate_limit) + + +def _copy_caps(QPCap src, dst): + """ + Copy the QPCaps values of src into the inner ibv_qp_cap struct of dst. + Since both ibv_qp_init_attr and ibv_qp_attr have an inner ibv_qp_cap inner + struct, they can both be used. + :param src: A QPCap object + :param dst: A QPInitAttr / QPInitAttrEx / QPAttr object + :return: None + """ + # we're assigning to C structs here, we must have type-specific objects in + # order to do that. Instead of having this function smaller but in 3 + # classes, it appears here once. + cdef QPInitAttr qia + cdef QPInitAttrEx qiae + cdef QPAttr qa + if src is None: + return + if type(dst) == QPInitAttr: + qia = dst + qia.attr.cap.max_send_wr = src.cap.max_send_wr + qia.attr.cap.max_recv_wr = src.cap.max_recv_wr + qia.attr.cap.max_send_sge = src.cap.max_send_sge + qia.attr.cap.max_recv_sge = src.cap.max_recv_sge + qia.attr.cap.max_inline_data = src.cap.max_inline_data + elif type(dst) == QPInitAttrEx: + qiae = dst + qiae.attr.cap.max_send_wr = src.cap.max_send_wr + qiae.attr.cap.max_recv_wr = src.cap.max_recv_wr + qiae.attr.cap.max_send_sge = src.cap.max_send_sge + qiae.attr.cap.max_recv_sge = src.cap.max_recv_sge + qiae.attr.cap.max_inline_data = src.cap.max_inline_data + else: + qa = dst + qa.attr.cap.max_send_wr = src.cap.max_send_wr + qa.attr.cap.max_recv_wr = src.cap.max_recv_wr + qa.attr.cap.max_send_sge = src.cap.max_send_sge + qa.attr.cap.max_recv_sge = src.cap.max_recv_sge + qa.attr.cap.max_inline_data = src.cap.max_inline_data diff --git a/pyverbs/utils.py b/pyverbs/utils.py index 01adceab568c..a59d6275fefc 100644 --- a/pyverbs/utils.py +++ b/pyverbs/utils.py @@ -33,3 +33,49 @@ def gid_str_to_array(val): vals.append(val[i][0:2]) vals.append(val[i][2:4]) return vals + + +def qp_type_to_str(qp_type): + types = {2: 'RC', 3: 'UC', 4: 'UD', 8: 'Raw Packet', 9: 'XRCD_SEND', + 10: 'XRCD_RECV', 0xff:'Driver QP'} + try: + return types[qp_type] + except KeyError: + return 'Unknown ({qpt})'.format(qpt=qp_type) + + +def qp_state_to_str(qp_state): + states = {0: 'Reset', 1: 'Init', 2: 'RTR', 3: 'RTS', 4: 'SQD', + 5: 'SQE', 6: 'Error', 7: 'Unknown'} + try: + return states[qp_state] + except KeyError: + return 'Unknown ({qps})'.format(qps=qp_state_to_str) + + +def mtu_to_str(mtu): + mtus = {1: 256, 2: 512, 3: 1024, 4: 2048, 5: 4096} + try: + return mtus[mtu] + except KeyError: + return 0 + + +def access_flags_to_str(flags): + access_flags = {1: 'Local write', 2: 'Remote write', 4: 'Remote read', + 8: 'Remote atomic', 16: 'MW bind', 32: 'Zero based', + 64: 'On demand'} + access_str = '' + for f in access_flags: + if flags & f: + access_str += access_flags[f] + access_str += ' ' + return access_str + + +def mig_state_to_str(mig): + mig_states = {0: 'Migrated', 1: 'Re-arm', 2: 'Armed'} + try: + return mig_states[mig] + except KeyError: + return 'Unknown ({m})'.format(m=mig) From patchwork Mon May 6 15:07:33 2019 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Noa Osherovich X-Patchwork-Id: 10931243 Return-Path: Received: from mail.wl.linuxfoundation.org (pdx-wl-mail.web.codeaurora.org [172.30.200.125]) by pdx-korg-patchwork-2.web.codeaurora.org (Postfix) with ESMTP id 9ED00912 for ; Mon, 6 May 2019 15:07:51 +0000 (UTC) Received: from mail.wl.linuxfoundation.org (localhost [127.0.0.1]) by mail.wl.linuxfoundation.org (Postfix) with ESMTP id 8AE3E287B6 for ; Mon, 6 May 2019 15:07:51 +0000 (UTC) Received: by mail.wl.linuxfoundation.org (Postfix, from userid 486) id 7F874287CF; Mon, 6 May 2019 15:07:51 +0000 (UTC) X-Spam-Checker-Version: SpamAssassin 3.3.1 (2010-03-16) on pdx-wl-mail.web.codeaurora.org X-Spam-Level: X-Spam-Status: No, score=-7.9 required=2.0 tests=BAYES_00,MAILING_LIST_MULTI, RCVD_IN_DNSWL_HI,UNPARSEABLE_RELAY autolearn=ham version=3.3.1 Received: from vger.kernel.org (vger.kernel.org [209.132.180.67]) by mail.wl.linuxfoundation.org (Postfix) with ESMTP id 3BD33287B6 for ; Mon, 6 May 2019 15:07:50 +0000 (UTC) Received: (majordomo@vger.kernel.org) by vger.kernel.org via listexpand id S1727095AbfEFPHt (ORCPT ); Mon, 6 May 2019 11:07:49 -0400 Received: from mail-il-dmz.mellanox.com ([193.47.165.129]:50114 "EHLO mellanox.co.il" rhost-flags-OK-OK-OK-FAIL) by vger.kernel.org with ESMTP id S1727522AbfEFPHs (ORCPT ); Mon, 6 May 2019 11:07:48 -0400 Received: from Internal Mail-Server by MTLPINE2 (envelope-from noaos@mellanox.com) with ESMTPS (AES256-SHA encrypted); 6 May 2019 18:07:41 +0300 Received: from reg-l-vrt-059-009.mtl.labs.mlnx (reg-l-vrt-059-009.mtl.labs.mlnx [10.135.59.9]) by labmailer.mlnx (8.13.8/8.13.8) with ESMTP id x46F7edc019922; Mon, 6 May 2019 18:07:40 +0300 From: Noa Osherovich To: leon@kernel.org, jgg@mellanox.com, dledford@redhat.com Cc: linux-rdma@vger.kernel.org, Noa Osherovich Subject: [PATCH rdma-core 06/11] pyverbs: Introducing QP class Date: Mon, 6 May 2019 18:07:33 +0300 Message-Id: <20190506150738.19477-7-noaos@mellanox.com> X-Mailer: git-send-email 2.17.2 In-Reply-To: <20190506150738.19477-1-noaos@mellanox.com> References: <20190506150738.19477-1-noaos@mellanox.com> Sender: linux-rdma-owner@vger.kernel.org Precedence: bulk List-ID: X-Mailing-List: linux-rdma@vger.kernel.org X-Virus-Scanned: ClamAV using ClamSMTP Allow creation, modification, querying and post recv and send operations on QPs of types RC, UC, UD and Raw Packet. Creation process allows user to get the QP in a more advanced state than INIT; by providing a QPAttr object (in addition to the mandatory QPInitAttr object), the user tells pyverbs to use the provided values and transition the QP as far as possible towards RTS. For connected QPs it means to INIT state, for UD / Raw Packet QPs it means to RTS. By default, the QPAttr's value will be None and the QP will be returned in RESET state. The QP class offers to_init/rtr/rts methods for quick and easy state transitions - they only require the user to provide a QPAttr object and the comp masks are being set according to the bits required for each QP type / transition. Similarly to querying a device, querying the QP doesn't require providing the QPAttr / QPInitAttr objects; they are created by pyverbs and are returned to the user as a tuple. This patch also adds weakref support to CQ/CQEX to keep pyverbs' reference scheme. Signed-off-by: Noa Osherovich --- pyverbs/cq.pxd | 4 + pyverbs/cq.pyx | 13 +++ pyverbs/device.pxd | 1 + pyverbs/device.pyx | 6 +- pyverbs/libibverbs.pxd | 22 ++++ pyverbs/pd.pxd | 1 + pyverbs/pd.pyx | 6 +- pyverbs/qp.pxd | 11 ++ pyverbs/qp.pyx | 256 +++++++++++++++++++++++++++++++++++++++++ 9 files changed, 318 insertions(+), 2 deletions(-) diff --git a/pyverbs/cq.pxd b/pyverbs/cq.pxd index 44abf3bae010..0e3bcdfffb7e 100644 --- a/pyverbs/cq.pxd +++ b/pyverbs/cq.pxd @@ -14,6 +14,8 @@ cdef class CQ(PyverbsCM): cdef v.ibv_cq *cq cpdef close(self) cdef object context + cdef add_ref(self, obj) + cdef object qps cdef class CqInitAttrEx(PyverbsObject): cdef v.ibv_cq_init_attr_ex attr @@ -24,6 +26,8 @@ cdef class CQEX(PyverbsCM): cdef v.ibv_cq *ibv_cq cpdef close(self) cdef object context + cdef add_ref(self, obj) + cdef object qps cdef class WC(PyverbsObject): cdef v.ibv_wc wc diff --git a/pyverbs/cq.pyx b/pyverbs/cq.pyx index dd7ab3cff8f0..dd475759337c 100644 --- a/pyverbs/cq.pyx +++ b/pyverbs/cq.pyx @@ -5,6 +5,7 @@ import weakref from pyverbs.base import PyverbsRDMAErrno cimport pyverbs.libibverbs_enums as e from pyverbs.device cimport Context +from pyverbs.qp cimport QP cdef class CompChannel(PyverbsCM): """ @@ -93,13 +94,19 @@ cdef class CQ(PyverbsCM): raise PyverbsRDMAErrno('Failed to create a CQ') self.context = context context.add_ref(self) + self.qps = weakref.WeakSet() self.logger.debug('Created a CQ') + cdef add_ref(self, obj): + if isinstance(obj, QP): + self.qps.add(obj) + def __dealloc__(self): self.close() cpdef close(self): self.logger.debug('Closing CQ') + self.close_weakrefs([self.qps]) if self.cq != NULL: rc = v.ibv_destroy_cq(self.cq) if rc != 0: @@ -267,12 +274,18 @@ cdef class CQEX(PyverbsCM): self.ibv_cq = v.ibv_cq_ex_to_cq(self.cq) self.context = context context.add_ref(self) + self.qps = weakref.WeakSet() + + cdef add_ref(self, obj): + if isinstance(obj, QP): + self.qps.add(obj) def __dealloc__(self): self.close() cpdef close(self): self.logger.debug('Closing CQEx') + self.close_weakrefs([self.qps]) if self.cq != NULL: rc = v.ibv_destroy_cq(self.cq) if rc != 0: diff --git a/pyverbs/device.pxd b/pyverbs/device.pxd index aecdc0c1d254..3cb52bde4603 100644 --- a/pyverbs/device.pxd +++ b/pyverbs/device.pxd @@ -13,6 +13,7 @@ cdef class Context(PyverbsCM): cdef object dms cdef object ccs cdef object cqs + cdef object qps cdef class DeviceAttr(PyverbsObject): cdef v.ibv_device_attr dev_attr diff --git a/pyverbs/device.pyx b/pyverbs/device.pyx index 72acc4d2829f..fb1986f0ba8f 100644 --- a/pyverbs/device.pyx +++ b/pyverbs/device.pyx @@ -17,6 +17,7 @@ cimport pyverbs.libibverbs as v from pyverbs.addr cimport GID from pyverbs.mr import DMMR from pyverbs.pd cimport PD +from pyverbs.qp cimport QP cdef extern from 'errno.h': int errno @@ -88,6 +89,7 @@ cdef class Context(PyverbsCM): self.dms = weakref.WeakSet() self.ccs = weakref.WeakSet() self.cqs = weakref.WeakSet() + self.qps = weakref.WeakSet() dev_name = kwargs.get('name') @@ -124,7 +126,7 @@ cdef class Context(PyverbsCM): cpdef close(self): self.logger.debug('Closing Context') - self.close_weakrefs([self.ccs, self.cqs, self.dms, self.pds]) + self.close_weakrefs([self.qps, self.ccs, self.cqs, self.dms, self.pds]) if self.context != NULL: rc = v.ibv_close_device(self.context) if rc != 0: @@ -194,6 +196,8 @@ cdef class Context(PyverbsCM): self.ccs.add(obj) elif isinstance(obj, CQ) or isinstance(obj, CQEX): self.cqs.add(obj) + elif isinstance(obj, QP): + self.qps.add(obj) else: raise PyverbsError('Unrecognized object type') diff --git a/pyverbs/libibverbs.pxd b/pyverbs/libibverbs.pxd index 039a6952a433..1aa5844b126e 100644 --- a/pyverbs/libibverbs.pxd +++ b/pyverbs/libibverbs.pxd @@ -403,6 +403,19 @@ cdef extern from 'infiniband/verbs.h': unsigned int handle unsigned int events_completed + cdef struct ibv_qp: + ibv_context *context; + void *qp_context; + ibv_pd *pd; + ibv_cq *send_cq; + ibv_cq *recv_cq; + ibv_srq *srq; + unsigned int handle; + unsigned int qp_num; + ibv_qp_state state; + ibv_qp_type qp_type; + unsigned int events_completed; + ibv_device **ibv_get_device_list(int *n) void ibv_free_device_list(ibv_device **list) ibv_context *ibv_open_device(ibv_device *device) @@ -468,3 +481,12 @@ cdef extern from 'infiniband/verbs.h': ibv_ah *ibv_create_ah_from_wc(ibv_pd *pd, ibv_wc *wc, ibv_grh *grh, uint8_t port_num) int ibv_destroy_ah(ibv_ah *ah) + ibv_qp *ibv_create_qp(ibv_pd *pd, ibv_qp_init_attr *qp_init_attr) + ibv_qp *ibv_create_qp_ex(ibv_context *context, + ibv_qp_init_attr_ex *qp_init_attr_ex) + int ibv_modify_qp(ibv_qp *qp, ibv_qp_attr *qp_attr, int comp_mask) + int ibv_query_qp(ibv_qp *qp, ibv_qp_attr *attr, int attr_mask, + ibv_qp_init_attr *init_attr) + int ibv_destroy_qp(ibv_qp *qp) + int ibv_post_recv(ibv_qp *qp, ibv_recv_wr *wr, ibv_recv_wr **bad_wr) + int ibv_post_send(ibv_qp *qp, ibv_send_wr *wr, ibv_send_wr **bad_wr) diff --git a/pyverbs/pd.pxd b/pyverbs/pd.pxd index 0b8fb10e6c67..07c9158b27eb 100644 --- a/pyverbs/pd.pxd +++ b/pyverbs/pd.pxd @@ -12,3 +12,4 @@ cdef class PD(PyverbsCM): cdef object mrs cdef object mws cdef object ahs + cdef object qps diff --git a/pyverbs/pd.pyx b/pyverbs/pd.pyx index b09e3bf1a555..4b5dc139c59f 100644 --- a/pyverbs/pd.pyx +++ b/pyverbs/pd.pyx @@ -7,6 +7,7 @@ from pyverbs.base import PyverbsRDMAErrno from pyverbs.device cimport Context, DM from .mr cimport MR, MW, DMMR from pyverbs.addr cimport AH +from pyverbs.qp cimport QP cdef extern from 'errno.h': int errno @@ -29,6 +30,7 @@ cdef class PD(PyverbsCM): self.mrs = weakref.WeakSet() self.mws = weakref.WeakSet() self.ahs = weakref.WeakSet() + self.qps = weakref.WeakSet() def __dealloc__(self): """ @@ -46,7 +48,7 @@ cdef class PD(PyverbsCM): :return: None """ self.logger.debug('Closing PD') - self.close_weakrefs([self.ahs, self.mws, self.mrs]) + self.close_weakrefs([self.qps, self.ahs, self.mws, self.mrs]) if self.pd != NULL: rc = v.ibv_dealloc_pd(self.pd) if rc != 0: @@ -61,6 +63,8 @@ cdef class PD(PyverbsCM): self.mws.add(obj) elif isinstance(obj, AH): self.ahs.add(obj) + elif isinstance(obj, QP): + self.qps.add(obj) else: raise PyverbsError('Unrecognized object type') diff --git a/pyverbs/qp.pxd b/pyverbs/qp.pxd index 787fda3cb789..d85bc28992ad 100644 --- a/pyverbs/qp.pxd +++ b/pyverbs/qp.pxd @@ -19,3 +19,14 @@ cdef class QPInitAttrEx(PyverbsObject): cdef class QPAttr(PyverbsObject): cdef v.ibv_qp_attr attr + +cdef class QP(PyverbsCM): + cdef v.ibv_qp *qp + cdef int type + cdef int state + cdef object pd + cdef object context + cpdef close(self) + cdef update_cqs(self, init_attr) + cdef object scq + cdef object rcq diff --git a/pyverbs/qp.pyx b/pyverbs/qp.pyx index 80aee40eae02..b33993a106e9 100644 --- a/pyverbs/qp.pyx +++ b/pyverbs/qp.pyx @@ -3,9 +3,12 @@ from pyverbs.utils import gid_str, qp_type_to_str, qp_state_to_str, mtu_to_str from pyverbs.utils import access_flags_to_str, mig_state_to_str from pyverbs.pyverbs_error import PyverbsUserError +from pyverbs.base import PyverbsRDMAErrno +from pyverbs.wr cimport RecvWR, SendWR cimport pyverbs.libibverbs_enums as e from pyverbs.addr cimport AHAttr, GID from pyverbs.addr cimport GlobalRoute +from pyverbs.device cimport Context from pyverbs.cq cimport CQ, CQEX cimport pyverbs.libibverbs as v from pyverbs.pd cimport PD @@ -750,6 +753,259 @@ cdef class QPAttr(PyverbsObject): print_format.format('Rate limit', self.attr.rate_limit) +cdef class QP(PyverbsCM): + def __cinit__(self, object creator not None, object init_attr not None, + QPAttr qp_attr=None): + """ + Initializes a QP object and performs state transitions according to + user request. + A C ibv_qp object will be created using the provided init_attr. + If a qp_attr object is provided, pyverbs will consider this a hint to + transit the QP's state as far as possible towards RTS: + - In case of UD and Raw Packet QP types, if a qp_attr is provided the + QP will be returned in RTS state. + - In case of connected QPs (RC, UC), remote QPN is needed for INIT2RTR + transition, so if a qp_attr is provided, the QP will be returned in + INIT state. + :param creator: The object creating the QP. Can be of type PD so + ibv_create_qp will be used or of type Context, so + ibv_create_qp_ex will be used. + :param init_attr: QP initial attributes of type QPInitAttr (when + created using PD) or QPInitAttrEx (when created + using Context). + :param qp_attr: Optional QPAttr object. Will be used for QP state + transitions after creation. + :return: An initialized QP object + """ + cdef PD pd + cdef Context ctx + self.update_cqs(init_attr) + # In order to use cdef'd methods, a proper casting must be done, let's + # infer the type. + if type(creator) == Context: + self._create_qp_ex(creator, init_attr) + ctx = creator + self.context = ctx + ctx.add_ref(self) + if init_attr.pd is not None: + pd = init_attr.pd + pd.add_ref(self) + self.pd = pd + else: + self._create_qp(creator, init_attr) + pd = creator + self.pd = pd + pd.add_ref(self) + self.context = None + if self.qp == NULL: + raise PyverbsRDMAErrno('Failed to create QP') + if qp_attr is not None: + funcs = {e.IBV_QPT_RC: self.to_init, e.IBV_QPT_UC: self.to_init, + e.IBV_QPT_UD: self.to_rts, + e.IBV_QPT_RAW_PACKET: self.to_rts} + funcs[self.qp.qp_type](qp_attr) + + cdef update_cqs(self, init_attr): + cdef CQ cq + cdef CQEX cqex + if init_attr.send_cq is not None: + if type(init_attr.send_cq) == CQ: + cq = init_attr.send_cq + cq.add_ref(self) + else: + cqex = init_attr.send_cq + cqex.add_ref(self) + self.scq = cq + if init_attr.send_cq != init_attr.recv_cq and init_attr.recv_cq is not None: + if type(init_attr.recv_cq) == CQ: + cq = init_attr.recv_cq + cq.add_ref(self) + else: + cqex = init_attr.recv_cq + cqex.add_ref(self) + self.rcq = cq + + def _create_qp(self, PD pd, QPInitAttr attr): + self.qp = v.ibv_create_qp(pd.pd, &attr.attr) + + def _create_qp_ex(self, Context ctx, QPInitAttrEx attr): + self.qp = v.ibv_create_qp_ex(ctx.context, &attr.attr) + + def __dealloc__(self): + self.close() + + cpdef close(self): + self.logger.debug('Closing QP') + if self.qp != NULL: + if v.ibv_destroy_qp(self.qp): + raise PyverbsRDMAErrno('Failed to destroy QP') + self.qp = NULL + self.pd = None + self.context = None + self.scq = None + self.rcq = None + + def _get_comp_mask(self, dst): + masks = {e.IBV_QPT_RC: {'INIT': e.IBV_QP_PKEY_INDEX | e.IBV_QP_PORT |\ + e.IBV_QP_ACCESS_FLAGS, 'RTR': e.IBV_QP_AV |\ + e.IBV_QP_PATH_MTU | e.IBV_QP_DEST_QPN |\ + e.IBV_QP_RQ_PSN |\ + e.IBV_QP_MAX_DEST_RD_ATOMIC |\ + e.IBV_QP_MIN_RNR_TIMER, + 'RTS': e.IBV_QP_TIMEOUT |\ + e.IBV_QP_RETRY_CNT | e.IBV_QP_RNR_RETRY |\ + e.IBV_QP_SQ_PSN | e.IBV_QP_MAX_QP_RD_ATOMIC}, + e.IBV_QPT_UC: {'INIT': e.IBV_QP_PKEY_INDEX | e.IBV_QP_PORT |\ + e.IBV_QP_ACCESS_FLAGS, 'RTR': e.IBV_QP_AV |\ + e.IBV_QP_PATH_MTU | e.IBV_QP_DEST_QPN |\ + e.IBV_QP_RQ_PSN, 'RTS': e.IBV_QP_SQ_PSN}, + e.IBV_QPT_UD: {'INIT': e.IBV_QP_PKEY_INDEX | e.IBV_QP_PORT |\ + e.IBV_QP_QKEY, 'RTR': 0, + 'RTS': e.IBV_QP_SQ_PSN}, + e.IBV_QPT_RAW_PACKET: {'INIT': e.IBV_QP_PORT, 'RTR': 0, + 'RTS': 0}} + return masks[self.qp.qp_type][dst] | e.IBV_QP_STATE + + def to_init(self, QPAttr qp_attr): + """ + Modify the current QP's state to INIT. If the current state doesn't + support transition to INIT, an exception will be raised. + The comp mask provided to the kernel includes the needed bits for 2INIT + transition for this QP type. + :param qp_attr: QPAttr object containing the needed attributes for + 2INIT transition + :return: None + """ + mask = self._get_comp_mask('INIT') + qp_attr.qp_state = e.IBV_QPS_INIT + rc = v.ibv_modify_qp(self.qp, &qp_attr.attr, mask) + if rc != 0: + raise PyverbsRDMAErrno('Failed to modify QP state to init (returned {rc})'. + format(rc=rc)) + + def to_rtr(self, QPAttr qp_attr): + """ + Modify the current QP's state to RTR. It assumes that its current + state is INIT or RESET, in which case it will attempt a transition to + INIT prior to transition to RTR. As a result, if current state doesn't + support transition to INIT, an exception will be raised. + The comp mask provided to the kernel includes the needed bits for 2RTR + transition for this QP type. + :param qp_attr: QPAttr object containing the needed attributes for + 2RTR transition. + :return: None + """ + if self.qp_state != e.IBV_QPS_INIT: #assume reset + self.to_init(qp_attr) + mask = self._get_comp_mask('RTR') + qp_attr.qp_state = e.IBV_QPS_RTR + rc = v.ibv_modify_qp(self.qp, &qp_attr.attr, mask) + if rc != 0: + raise PyverbsRDMAErrno('Failed to modify QP state to RTR (returned {rc})'. + format(rc=rc)) + + def to_rts(self, QPAttr qp_attr): + """ + Modify the current QP's state to RTS. It assumes that its current + state is either RTR, INIT or RESET. If current state is not RTR, to_rtr() + will be called. + The comp mask provided to the kernel includes the needed bits for 2RTS + transition for this QP type. + :param qp_attr: QPAttr object containing the needed attributes for + 2RTS transition. + :return: None + """ + if self.qp_state != e.IBV_QPS_RTR: #assume reset/init + self.to_rtr(qp_attr) + mask = self._get_comp_mask('RTS') + qp_attr.qp_state = e.IBV_QPS_RTS + rc = v.ibv_modify_qp(self.qp, &qp_attr.attr, mask) + if rc != 0: + raise PyverbsRDMAErrno('Failed to modify QP state to RTS (returned {rc})'. + format(rc=rc)) + + def query(self, attr_mask): + """ + Query the QP + :param attr_mask: The minimum list of attributes to retrieve. Some + devices may return additional attributes as well + (see enum ibv_qp_attr_mask) + :return: (QPAttr, QPInitAttr) tuple containing the QP requested + attributes + """ + attr = QPAttr() + init_attr = QPInitAttr() + rc = v.ibv_query_qp(self.qp, &attr.attr, attr_mask, &init_attr.attr) + if rc != 0: + raise PyverbsRDMAErrno('Failed to query QP (returned {rc})'. + format(rc=rc)) + return attr, init_attr + + def modify(self, QPAttr qp_attr not None, comp_mask): + """ + Modify the QP + :param qp_attr: A QPAttr object with updated values to be applied to + the QP + :param comp_mask: A bitmask specifying which QP attributes should be + modified (see enum ibv_qp_attr_mask) + :return: None + """ + rc = v.ibv_modify_qp(self.qp, &qp_attr.attr, comp_mask) + if rc != 0: + raise PyverbsRDMAErrno('Failed to modify QP (returned {rc})'. + format(rc=rc)) + + def post_recv(self, RecvWR wr not None, RecvWR bad_wr=None): + """ + Post a receive WR on the QP. + :param wr: The work request to post + :param bad_wr: A RecvWR object to hold the bad WR if it is available in + case of a failure + :return: None + """ + cdef v.ibv_recv_wr *my_bad_wr + rc = v.ibv_post_recv(self.qp, &wr.recv_wr, &my_bad_wr) + if rc != 0: + if bad_wr is not None: + bad_wr.wr = my_bad_wr + raise PyverbsRDMAErrno('Failed to post recv (returned {rc})'. + format(rc=rc)) + + def post_send(self, SendWR wr not None, SendWR bad_wr=None): + """ + Post a send WR on the QP. + :param wr: The work request to post + :param bad_wr: A SendWR object to hold the bad WR if it is available in + case of a failure + :return: None + """ + cdef v.ibv_send_wr *my_bad_wr + rc = v.ibv_post_send(self.qp, &wr.send_wr, &my_bad_wr) + if rc != 0: + if bad_wr is not None: + bad_wr.wr = my_bad_wr + raise PyverbsRDMAErrno('Failed to post send (returned {rc})'. + format(rc=rc)) + + @property + def qp_type(self): + return self.qp.qp_type + + @property + def qp_state(self): + return self.qp.state + + @property + def qp_num(self): + return self.qp.qp_num + + def __str__(self): + print_format = '{:22}: {:<20}\n' + return print_format.format('QP type', qp_type_to_str(self.qp_type)) +\ + print_format.format(' number', self.qp_num) +\ + print_format.format(' state', qp_state_to_str(self.qp_state)) + + def _copy_caps(QPCap src, dst): """ Copy the QPCaps values of src into the inner ibv_qp_cap struct of dst. From patchwork Mon May 6 15:07:34 2019 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Noa Osherovich X-Patchwork-Id: 10931241 Return-Path: Received: from mail.wl.linuxfoundation.org (pdx-wl-mail.web.codeaurora.org [172.30.200.125]) by pdx-korg-patchwork-2.web.codeaurora.org (Postfix) with ESMTP id 70877912 for ; Mon, 6 May 2019 15:07:50 +0000 (UTC) Received: from mail.wl.linuxfoundation.org (localhost [127.0.0.1]) by mail.wl.linuxfoundation.org (Postfix) with ESMTP id 5EBA22878F for ; Mon, 6 May 2019 15:07:50 +0000 (UTC) Received: by mail.wl.linuxfoundation.org (Postfix, from userid 486) id 53264287C8; Mon, 6 May 2019 15:07:50 +0000 (UTC) X-Spam-Checker-Version: SpamAssassin 3.3.1 (2010-03-16) on pdx-wl-mail.web.codeaurora.org X-Spam-Level: X-Spam-Status: No, score=-7.9 required=2.0 tests=BAYES_00,MAILING_LIST_MULTI, RCVD_IN_DNSWL_HI,UNPARSEABLE_RELAY autolearn=ham version=3.3.1 Received: from vger.kernel.org (vger.kernel.org [209.132.180.67]) by mail.wl.linuxfoundation.org (Postfix) with ESMTP id EC8E72878F for ; Mon, 6 May 2019 15:07:49 +0000 (UTC) Received: (majordomo@vger.kernel.org) by vger.kernel.org via listexpand id S1727531AbfEFPHs (ORCPT ); Mon, 6 May 2019 11:07:48 -0400 Received: from mail-il-dmz.mellanox.com ([193.47.165.129]:50124 "EHLO mellanox.co.il" rhost-flags-OK-OK-OK-FAIL) by vger.kernel.org with ESMTP id S1727541AbfEFPHr (ORCPT ); Mon, 6 May 2019 11:07:47 -0400 Received: from Internal Mail-Server by MTLPINE2 (envelope-from noaos@mellanox.com) with ESMTPS (AES256-SHA encrypted); 6 May 2019 18:07:41 +0300 Received: from reg-l-vrt-059-009.mtl.labs.mlnx (reg-l-vrt-059-009.mtl.labs.mlnx [10.135.59.9]) by labmailer.mlnx (8.13.8/8.13.8) with ESMTP id x46F7edd019922; Mon, 6 May 2019 18:07:41 +0300 From: Noa Osherovich To: leon@kernel.org, jgg@mellanox.com, dledford@redhat.com Cc: linux-rdma@vger.kernel.org, Noa Osherovich Subject: [PATCH rdma-core 07/11] pyverbs: Add missing device capabilities Date: Mon, 6 May 2019 18:07:34 +0300 Message-Id: <20190506150738.19477-8-noaos@mellanox.com> X-Mailer: git-send-email 2.17.2 In-Reply-To: <20190506150738.19477-1-noaos@mellanox.com> References: <20190506150738.19477-1-noaos@mellanox.com> Sender: linux-rdma-owner@vger.kernel.org Precedence: bulk List-ID: X-Mailing-List: linux-rdma@vger.kernel.org X-Virus-Scanned: ClamAV using ClamSMTP New device extended capabilities are defined outside the enum. Add them to pyverbs as well. Signed-off-by: Noa Osherovich --- pyverbs/libibverbs_enums.pxd | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/pyverbs/libibverbs_enums.pxd b/pyverbs/libibverbs_enums.pxd index b76914208074..85b5092c486f 100644 --- a/pyverbs/libibverbs_enums.pxd +++ b/pyverbs/libibverbs_enums.pxd @@ -408,6 +408,9 @@ cdef extern from '': IBV_RAW_PACKET_CAP_IP_CSUM = 1 << 2 IBV_RAW_PACKET_CAP_DELAY_DROP = 1 << 3 + cdef unsigned long long IBV_DEVICE_RAW_SCATTER_FCS + cdef unsigned long long IBV_DEVICE_PCI_WRITE_END_PADDING + cdef extern from "": cpdef enum ibv_tmh_op: @@ -415,3 +418,6 @@ cdef extern from "": IBV_TMH_RNDV = 1 IBV_TMH_FIN = 2 IBV_TMH_EAGER = 3 + +_IBV_DEVICE_RAW_SCATTER_FCS = IBV_DEVICE_RAW_SCATTER_FCS +_IBV_DEVICE_PCI_WRITE_END_PADDING = IBV_DEVICE_PCI_WRITE_END_PADDING From patchwork Mon May 6 15:07:35 2019 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Noa Osherovich X-Patchwork-Id: 10931245 Return-Path: Received: from mail.wl.linuxfoundation.org (pdx-wl-mail.web.codeaurora.org [172.30.200.125]) by pdx-korg-patchwork-2.web.codeaurora.org (Postfix) with ESMTP id 507911575 for ; Mon, 6 May 2019 15:07:52 +0000 (UTC) Received: from mail.wl.linuxfoundation.org (localhost [127.0.0.1]) by mail.wl.linuxfoundation.org (Postfix) with ESMTP id 36A3D2878F for ; Mon, 6 May 2019 15:07:52 +0000 (UTC) Received: by mail.wl.linuxfoundation.org (Postfix, from userid 486) id 2AB5B287D3; Mon, 6 May 2019 15:07:52 +0000 (UTC) X-Spam-Checker-Version: SpamAssassin 3.3.1 (2010-03-16) on pdx-wl-mail.web.codeaurora.org X-Spam-Level: X-Spam-Status: No, score=-7.9 required=2.0 tests=BAYES_00,MAILING_LIST_MULTI, RCVD_IN_DNSWL_HI,UNPARSEABLE_RELAY autolearn=ham version=3.3.1 Received: from vger.kernel.org (vger.kernel.org [209.132.180.67]) by mail.wl.linuxfoundation.org (Postfix) with ESMTP id 087CC2878F for ; Mon, 6 May 2019 15:07:51 +0000 (UTC) Received: (majordomo@vger.kernel.org) by vger.kernel.org via listexpand id S1726996AbfEFPHt (ORCPT ); Mon, 6 May 2019 11:07:49 -0400 Received: from mail-il-dmz.mellanox.com ([193.47.165.129]:50122 "EHLO mellanox.co.il" rhost-flags-OK-OK-OK-FAIL) by vger.kernel.org with ESMTP id S1727561AbfEFPHs (ORCPT ); Mon, 6 May 2019 11:07:48 -0400 Received: from Internal Mail-Server by MTLPINE2 (envelope-from noaos@mellanox.com) with ESMTPS (AES256-SHA encrypted); 6 May 2019 18:07:41 +0300 Received: from reg-l-vrt-059-009.mtl.labs.mlnx (reg-l-vrt-059-009.mtl.labs.mlnx [10.135.59.9]) by labmailer.mlnx (8.13.8/8.13.8) with ESMTP id x46F7ede019922; Mon, 6 May 2019 18:07:41 +0300 From: Noa Osherovich To: leon@kernel.org, jgg@mellanox.com, dledford@redhat.com Cc: linux-rdma@vger.kernel.org, Noa Osherovich Subject: [PATCH rdma-core 08/11] pyverbs/tests: Add control-path unittests for QP class Date: Mon, 6 May 2019 18:07:35 +0300 Message-Id: <20190506150738.19477-9-noaos@mellanox.com> X-Mailer: git-send-email 2.17.2 In-Reply-To: <20190506150738.19477-1-noaos@mellanox.com> References: <20190506150738.19477-1-noaos@mellanox.com> Sender: linux-rdma-owner@vger.kernel.org Precedence: bulk List-ID: X-Mailing-List: linux-rdma@vger.kernel.org X-Virus-Scanned: ClamAV using ClamSMTP Unittests include creation of supported QP types (RC, UD, UC, Raw Packet) using both ibv_create_qp and ibv_create_qp_ex. These unitests also verify that if a QP is created with a QPAttr object, it will be returned to the user in INIT state (for connected QPs) or RTS (UD, Raw Packet). Signed-off-by: Noa Osherovich --- pyverbs/CMakeLists.txt | 1 + pyverbs/tests/qp.py | 255 +++++++++++++++++++++++++++++++++++++++++ pyverbs/tests/utils.py | 128 ++++++++++++++++++++- 3 files changed, 382 insertions(+), 2 deletions(-) create mode 100644 pyverbs/tests/qp.py diff --git a/pyverbs/CMakeLists.txt b/pyverbs/CMakeLists.txt index 3b3838ddb8b5..328263fcc739 100644 --- a/pyverbs/CMakeLists.txt +++ b/pyverbs/CMakeLists.txt @@ -28,6 +28,7 @@ rdma_python_module(pyverbs/tests tests/device.py tests/mr.py tests/pd.py + tests/qp.py tests/utils.py ) diff --git a/pyverbs/tests/qp.py b/pyverbs/tests/qp.py new file mode 100644 index 000000000000..be152d4ca5bd --- /dev/null +++ b/pyverbs/tests/qp.py @@ -0,0 +1,255 @@ +# SPDX-License-Identifier: (GPL-2.0 OR Linux-OpenIB) +# Copyright (c) 2019 Mellanox Technologies, Inc. All rights reserved. See COPYING file +""" +Test module for pyverbs' qp module. +""" +import random + +from pyverbs.tests.base import PyverbsTestCase +from pyverbs.qp import QPInitAttr, QPAttr, QP +import pyverbs.tests.utils as u +import pyverbs.enums as e +from pyverbs.pd import PD +from pyverbs.cq import CQ + + +class QPTest(PyverbsTestCase): + """ + Test various functionalities of the QP class. + """ + + def test_create_qp_no_attr_connected(self): + """ + Test QP creation via ibv_create_qp without a QPAttr object proivded. + QP type can be either RC or UC. + """ + for ctx, attr, attr_ex in self.devices: + with PD(ctx) as pd: + with CQ(ctx, 100, None, None, 0) as cq: + qia = get_qp_init_attr(cq, [e.IBV_QPT_RC, e.IBV_QPT_UC], + attr) + with QP(pd, qia) as qp: + assert qp.qp_state == e.IBV_QPS_RESET + + def test_create_qp_no_attr(self): + """ + Test QP creation via ibv_create_qp without a QPAttr object proivded. + QP type can be either Raw Packet or UD. + """ + for ctx, attr, attr_ex in self.devices: + with PD(ctx) as pd: + with CQ(ctx, 100, None, None, 0) as cq: + for i in range(1, attr.phys_port_cnt + 1): + qpts = [e.IBV_QPT_UD, e.IBV_QPT_RAW_PACKET] \ + if is_eth(ctx, i) else [e.IBV_QPT_UD] + qia = get_qp_init_attr(cq, qpts, attr) + with QP(pd, qia) as qp: + assert qp.qp_state == e.IBV_QPS_RESET + + def test_create_qp_with_attr_connected(self): + """ + Test QP creation via ibv_create_qp without a QPAttr object proivded. + QP type can be either RC or UC. + """ + for ctx, attr, attr_ex in self.devices: + with PD(ctx) as pd: + with CQ(ctx, 100, None, None, 0) as cq: + qia = get_qp_init_attr(cq, [e.IBV_QPT_RC, e.IBV_QPT_UC], + attr) + with QP(pd, qia, QPAttr()) as qp: + assert qp.qp_state == e.IBV_QPS_INIT + + def test_create_qp_with_attr(self): + """ + Test QP creation via ibv_create_qp with a QPAttr object proivded. + QP type can be either Raw Packet or UD. + """ + for ctx, attr, attr_ex in self.devices: + with PD(ctx) as pd: + with CQ(ctx, 100, None, None, 0) as cq: + for i in range(1, attr.phys_port_cnt + 1): + qpts = [e.IBV_QPT_UD, e.IBV_QPT_RAW_PACKET] \ + if is_eth(ctx, i) else [e.IBV_QPT_UD] + qia = get_qp_init_attr(cq, qpts, attr) + with QP(pd, qia, QPAttr()) as qp: + assert qp.qp_state == e.IBV_QPS_RTS + + def test_create_qp_ex_no_attr_connected(self): + """ + Test QP creation via ibv_create_qp_ex without a QPAttr object proivded. + QP type can be either RC or UC. + """ + for ctx, attr, attr_ex in self.devices: + with PD(ctx) as pd: + with CQ(ctx, 100, None, None, 0) as cq: + qia = get_qp_init_attr_ex(cq, pd, [e.IBV_QPT_RC, + e.IBV_QPT_UC], + attr, attr_ex) + with QP(ctx, qia) as qp: + assert qp.qp_state == e.IBV_QPS_RESET + + def test_create_qp_ex_no_attr(self): + """ + Test QP creation via ibv_create_qp_ex without a QPAttr object proivded. + QP type can be either Raw Packet or UD. + """ + for ctx, attr, attr_ex in self.devices: + with PD(ctx) as pd: + with CQ(ctx, 100, None, None, 0) as cq: + for i in range(1, attr.phys_port_cnt + 1): + qpts = [e.IBV_QPT_UD, e.IBV_QPT_RAW_PACKET] \ + if is_eth(ctx, i) else [e.IBV_QPT_UD] + qia = get_qp_init_attr_ex(cq, pd, qpts, attr, + attr_ex) + with QP(ctx, qia) as qp: + assert qp.qp_state == e.IBV_QPS_RESET + + def test_create_qp_ex_with_attr_connected(self): + """ + Test QP creation via ibv_create_qp_ex with a QPAttr object proivded. + QP type can be either RC or UC. + """ + for ctx, attr, attr_ex in self.devices: + with PD(ctx) as pd: + with CQ(ctx, 100, None, None, 0) as cq: + qia = get_qp_init_attr_ex(cq, pd, [e.IBV_QPT_RC, + e.IBV_QPT_UC], + attr, attr_ex) + with QP(ctx, qia, QPAttr()) as qp: + assert qp.qp_state == e.IBV_QPS_INIT + + def test_create_qp_ex_with_attr(self): + """ + Test QP creation via ibv_create_qp_ex with a QPAttr object proivded. + QP type can be either Raw Packet or UD. + """ + for ctx, attr, attr_ex in self.devices: + with PD(ctx) as pd: + with CQ(ctx, 100, None, None, 0) as cq: + for i in range(1, attr.phys_port_cnt + 1): + qpts = [e.IBV_QPT_UD, e.IBV_QPT_RAW_PACKET] \ + if is_eth(ctx, i) else [e.IBV_QPT_UD] + qia = get_qp_init_attr_ex(cq, pd, qpts, attr, + attr_ex) + with QP(ctx, qia, QPAttr()) as qp: + assert qp.qp_state == e.IBV_QPS_RTS + + def test_query_qp(self): + """ + Queries a QP after creation. Verifies that its properties are as + expected. + """ + for ctx, attr, attr_ex in self.devices: + with PD(ctx) as pd: + with CQ(ctx, 100, None, None, 0) as cq: + for i in range(1, attr.phys_port_cnt + 1): + qpts = get_qp_types(ctx, i) + is_ex = random.choice([True, False]) + if is_ex: + qia = get_qp_init_attr_ex(cq, pd, qpts, attr, + attr_ex) + else: + qia = get_qp_init_attr(cq, qpts, attr) + caps = qia.cap # Save them to verify values later + qp = QP(ctx, qia) if is_ex else QP(pd, qia) + attr, init_attr = qp.query(e.IBV_QP_CUR_STATE | + e.IBV_QP_CAP) + verify_qp_attrs(caps, e.IBV_QPS_RESET, init_attr, + attr) + + def test_modify_qp(self): + """ + Queries a QP after calling modify(). Verifies that its properties are + as expected. + """ + for ctx, attr, attr_ex in self.devices: + with PD(ctx) as pd: + with CQ(ctx, 100, None, None, 0) as cq: + is_ex = random.choice([True, False]) + if is_ex: + qia = get_qp_init_attr_ex(cq, pd, [e.IBV_QPT_UD], + attr, attr_ex) + else: + qia = get_qp_init_attr(cq, [e.IBV_QPT_UD], attr) + qp = QP(ctx, qia) if is_ex \ + else QP(pd, qia) + qa = QPAttr() + qa.qkey = 0x123 + qp.to_init(qa) + attr, iattr = qp.query(e.IBV_QP_QKEY) + assert attr.qkey == qa.qkey + qp.to_rtr(qa) + qa.sq_psn = 0x45 + qp.to_rts(qa) + attr, iattr = qp.query(e.IBV_QP_SQ_PSN) + assert attr.sq_psn == qa.sq_psn + qa.qp_state = e.IBV_QPS_RESET + qp.modify(qa, e.IBV_QP_STATE) + assert qp.qp_state == e.IBV_QPS_RESET + + +def get_qp_types(ctx, port_num): + """ + Returns a list of the commonly used QP types. Raw Packet QP will not be + included if link layer is not Ethernet. + :param ctx: The device's Context, to query the port's link layer + :param port_num: Port number to query + :return: An array of QP types that can be created on this port + """ + qpts = [e.IBV_QPT_RC, e.IBV_QPT_UC, e.IBV_QPT_UD] + if is_eth(ctx, port_num): + qpts.append(e.IBV_QPT_RAW_PACKET) + return qpts + + +def verify_qp_attrs(orig_cap, state, init_attr, attr): + assert state == attr.cur_qp_state + assert orig_cap.max_send_wr <= init_attr.cap.max_send_wr + assert orig_cap.max_recv_wr <= init_attr.cap.max_recv_wr + assert orig_cap.max_send_sge <= init_attr.cap.max_send_sge + assert orig_cap.max_recv_sge <= init_attr.cap.max_recv_sge + assert orig_cap.max_inline_data <= init_attr.cap.max_inline_data + + +def get_qp_init_attr(cq, qpts, attr): + """ + Creates a QPInitAttr object with a QP type of the provided array and + other random values. + :param cq: CQ to be used as send and receive CQ + :param qpts: An array of possible QP types to use + :param attr: Device attributes for capability checks + :return: An initialized QPInitAttr object + """ + qp_cap = u.random_qp_cap(attr) + qpt = random.choice(qpts) + sig = random.randint(0, 1) + return QPInitAttr(qp_type=qpt, scq=cq, rcq=cq, cap=qp_cap, sq_sig_all=sig) + + +def get_qp_init_attr_ex(cq, pd, qpts, attr, attr_ex): + """ + Creates a QPInitAttrEx object with a QP type of the provided array + and other random values. + :param cq: CQ to be used as send and receive CQ + :param pd: A PD object to use + :param qpts: An array of possible QP types to use + :param attr: Device attributes for capability checks + :param attr_ex: Extended device attributes for capability checks + :return: An initialized QPInitAttrEx object + """ + qpt = random.choice(qpts) + qia = u.random_qp_init_attr_ex(attr_ex, attr, qpt) + qia.send_cq = cq + qia.recv_cq = cq + qia.pd = pd # Only XRCD can be created without a PD + return qia + + +def is_eth(ctx, port_num): + """ + Querires the device's context's port for its link layer. + :param ctx: The Context to query + :param port_num: Which Context's port to query + :return: True if the port's link layer is Ethernet, else False + """ + return ctx.query_port(port_num).link_layer == e.IBV_LINK_LAYER_ETHERNET diff --git a/pyverbs/tests/utils.py b/pyverbs/tests/utils.py index c4a28b70a2da..c84865a10a40 100644 --- a/pyverbs/tests/utils.py +++ b/pyverbs/tests/utils.py @@ -1,5 +1,5 @@ # SPDX-License-Identifier: (GPL-2.0 OR Linux-OpenIB) -# Copyright (c) 2019, Mellanox Technologies. All rights reserved. See COPYING file +# Copyright (c) 2019 Mellanox Technologies, Inc. All rights reserved. See COPYING file """ Provide some useful helper function for pyverbs' tests. """ @@ -7,10 +7,10 @@ from itertools import combinations as com from string import ascii_lowercase as al import random +from pyverbs.qp import QPCap, QPInitAttrEx import pyverbs.device as d import pyverbs.enums as e - MAX_MR_SIZE = 4194304 # Some HWs limit DM address and length alignment to 4 for read and write # operations. Use a minimal length and alignment that respect that. @@ -20,6 +20,8 @@ MIN_DM_SIZE = 4 DM_ALIGNMENT = 4 MIN_DM_LOG_ALIGN = 0 MAX_DM_LOG_ALIGN = 6 +# Raw Packet QP supports TSO header, which creates a larger send WQE. +MAX_RAW_PACKET_SEND_WR = 2500 def get_mr_length(): @@ -97,3 +99,125 @@ def sample(coll): :return: A subset of """ return random.sample(coll, int((len(coll) + 1) * random.random())) + + +def random_qp_cap(attr): + """ + Initializes a QPCap object with valid values based on the device's + attributes. + It doesn't check the max WR limits since they're reported for smaller WR + sizes. + :return: A QPCap object + """ + # We use significantly smaller values than those in device attributes. + # The attributes reported by the device don't take into account possible + # larger WQEs that include e.g. memory window. + send_wr = random.randint(1, int(attr.max_qp_wr / 8)) + recv_wr = random.randint(1, int(attr.max_qp_wr / 8)) + send_sge = random.randint(1, int(attr.max_sge / 2)) + recv_sge = random.randint(1, int(attr.max_sge / 2)) + inline = random.randint(0, 16) + return QPCap(send_wr, recv_wr, send_sge, recv_sge, inline) + + +def random_qp_create_mask(qpt, attr_ex): + """ + Select a random sublist of ibv_qp_init_attr_mask. Some of the options are + not yet supported by pyverbs and will not be returned. TSO support is + checked for the device and the QP type. If it doesn't exist, TSO will not + be set. + :param qpt: Current QP type + :param attr_ex: Extended device attributes for capability checks + :return: A sublist of ibv_qp_init_attr_mask + """ + has_tso = attr_ex.tso_caps.max_tso > 0 and \ + attr_ex.tso_caps.supported_qpts & 1 << qpt + supp_flags = [e.IBV_QP_INIT_ATTR_CREATE_FLAGS, + e.IBV_QP_INIT_ATTR_MAX_TSO_HEADER] + # Either PD or XRCD flag is needed, XRCD is not supported yet + selected = sample(supp_flags) + selected.append(e.IBV_QP_INIT_ATTR_PD) + if e.IBV_QP_INIT_ATTR_MAX_TSO_HEADER in selected and not has_tso: + selected.remove(e.IBV_QP_INIT_ATTR_MAX_TSO_HEADER) + mask = 0 + for s in selected: + mask += s.value + return mask + + +def get_create_qp_flags_raw_packet(attr_ex): + """ + Select random QP creation flags for Raw Packet QP. Filter out unsupported + flags prior to selection. + :param attr_ex: Device extended attributes to check capabilities + :return: A random combination of QP creation flags + """ + has_fcs = attr_ex.device_cap_flags_ex & e._IBV_DEVICE_RAW_SCATTER_FCS + has_cvlan = attr_ex.raw_packet_caps & e.IBV_RAW_PACKET_CAP_CVLAN_STRIPPING + has_padding = attr_ex.device_cap_flags_ex & \ + e._IBV_DEVICE_PCI_WRITE_END_PADDING + l = list(e.ibv_qp_create_flags) + l.remove(e.IBV_QP_CREATE_SOURCE_QPN) # UD only + if not has_fcs: + l.remove(e.IBV_QP_CREATE_SCATTER_FCS) + if not has_cvlan: + l.remove(e.IBV_QP_CREATE_CVLAN_STRIPPING) + if not has_padding: + l.remove(e.IBV_QP_CREATE_PCI_WRITE_END_PADDING) + flags = sample(l) + val = 0 + for i in flags: + val |= i.value + return val + + +def random_qp_create_flags(qpt, attr_ex): + """ + Select a random sublist of ibv_qp_create_flags according to the QP type. + :param qpt: Current QP type + :param attr_ex: Used for Raw Packet QP to check device capabilities + :return: A sublist of ibv_qp_create_flags + """ + if qpt == e.IBV_QPT_RAW_PACKET: + return get_create_qp_flags_raw_packet(attr_ex) + elif qpt == e.IBV_QPT_UD: + # IBV_QP_CREATE_SOURCE_QPN is only supported by mlx5 driver and is not + # to be check in unittests. + return random.choice([0, 2]) # IBV_QP_CREATE_BLOCK_SELF_MCAST_LB + else: + return 0 + + +def random_qp_init_attr_ex(attr_ex, attr, qpt=None): + """ + Create a random-valued QPInitAttrEX object with the given QP type. + QP type affects QP capabilities, so allow users to set it and still get + valid attributes. + :param attr_ex: Extended device attributes for capability checks + :param attr: Device attributes for capability checks + :param qpt: Requested QP type + :return: A valid initialized QPInitAttrEx object + """ + max_tso = 0 + if qpt is None: + qpt = random.choice([e.IBV_QPT_RC, e.IBV_QPT_UC, e.IBV_QPT_UD, + e.IBV_QPT_RAW_PACKET]) + qp_cap = random_qp_cap(attr) + if qpt == e.IBV_QPT_RAW_PACKET and \ + qp_cap.max_send_wr > MAX_RAW_PACKET_SEND_WR: + qp_cap.max_send_wr = MAX_RAW_PACKET_SEND_WR + sig = random.randint(0, 1) + mask = random_qp_create_mask(qpt, attr_ex) + if mask & e.IBV_QP_INIT_ATTR_CREATE_FLAGS: + cflags = random_qp_create_flags(qpt, attr_ex) + else: + cflags = 0 + if mask & e.IBV_QP_INIT_ATTR_MAX_TSO_HEADER: + if qpt != e.IBV_QPT_RAW_PACKET: + mask -= e.IBV_QP_INIT_ATTR_MAX_TSO_HEADER + else: + max_tso = \ + random.randint(16, int(attr_ex.tso_caps.max_tso / 400)) + qia = QPInitAttrEx(qp_type=qpt, cap=qp_cap, sq_sig_all=sig, comp_mask=mask, + create_flags=cflags, max_tso_header=max_tso) + return qia From patchwork Mon May 6 15:07:36 2019 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Noa Osherovich X-Patchwork-Id: 10931239 Return-Path: Received: from mail.wl.linuxfoundation.org (pdx-wl-mail.web.codeaurora.org [172.30.200.125]) by pdx-korg-patchwork-2.web.codeaurora.org (Postfix) with ESMTP id 5A4EA912 for ; Mon, 6 May 2019 15:07:49 +0000 (UTC) Received: from mail.wl.linuxfoundation.org (localhost [127.0.0.1]) by mail.wl.linuxfoundation.org (Postfix) with ESMTP id 490492878F for ; Mon, 6 May 2019 15:07:49 +0000 (UTC) Received: by mail.wl.linuxfoundation.org (Postfix, from userid 486) id 3D502287C3; Mon, 6 May 2019 15:07:49 +0000 (UTC) X-Spam-Checker-Version: SpamAssassin 3.3.1 (2010-03-16) on pdx-wl-mail.web.codeaurora.org X-Spam-Level: X-Spam-Status: No, score=-7.9 required=2.0 tests=BAYES_00,MAILING_LIST_MULTI, RCVD_IN_DNSWL_HI,UNPARSEABLE_RELAY autolearn=ham version=3.3.1 Received: from vger.kernel.org (vger.kernel.org [209.132.180.67]) by mail.wl.linuxfoundation.org (Postfix) with ESMTP id DE4DD2878F for ; Mon, 6 May 2019 15:07:48 +0000 (UTC) Received: (majordomo@vger.kernel.org) by vger.kernel.org via listexpand id S1727391AbfEFPHr (ORCPT ); Mon, 6 May 2019 11:07:47 -0400 Received: from mail-il-dmz.mellanox.com ([193.47.165.129]:50132 "EHLO mellanox.co.il" rhost-flags-OK-OK-OK-FAIL) by vger.kernel.org with ESMTP id S1727531AbfEFPHr (ORCPT ); Mon, 6 May 2019 11:07:47 -0400 Received: from Internal Mail-Server by MTLPINE2 (envelope-from noaos@mellanox.com) with ESMTPS (AES256-SHA encrypted); 6 May 2019 18:07:41 +0300 Received: from reg-l-vrt-059-009.mtl.labs.mlnx (reg-l-vrt-059-009.mtl.labs.mlnx [10.135.59.9]) by labmailer.mlnx (8.13.8/8.13.8) with ESMTP id x46F7edf019922; Mon, 6 May 2019 18:07:41 +0300 From: Noa Osherovich To: leon@kernel.org, jgg@mellanox.com, dledford@redhat.com Cc: linux-rdma@vger.kernel.org, Noa Osherovich Subject: [PATCH rdma-core 09/11] Documentation: Document QP creation and basic usage with pyverbs Date: Mon, 6 May 2019 18:07:36 +0300 Message-Id: <20190506150738.19477-10-noaos@mellanox.com> X-Mailer: git-send-email 2.17.2 In-Reply-To: <20190506150738.19477-1-noaos@mellanox.com> References: <20190506150738.19477-1-noaos@mellanox.com> Sender: linux-rdma-owner@vger.kernel.org Precedence: bulk List-ID: X-Mailing-List: linux-rdma@vger.kernel.org X-Virus-Scanned: ClamAV using ClamSMTP Add code to show the needed steps for QP creation as well as a simple post_send. Signed-off-by: Noa Osherovich --- Documentation/pyverbs.md | 32 ++++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/Documentation/pyverbs.md b/Documentation/pyverbs.md index 371fcd07b2cd..22c51868e025 100644 --- a/Documentation/pyverbs.md +++ b/Documentation/pyverbs.md @@ -307,3 +307,35 @@ sgid index : 0 hop limit : 1 traffic class : 0 ``` + +##### QP +The following snippets will demonstrate creation of a QP and a simple post_send +operation. For more complex examples, please see pyverbs/examples section. +```python +from pyverbs.qp import QPCap, QPInitAttr, QPAttr, QP +from pyverbs.addr import GlobalRoute +from pyverbs.addr import AH, AHAttr +import pyverbs.device as d +import pyverbs.enums as e +from pyverbs.pd import PD +from pyverbs.cq import CQ +import pyverbs.wr as pwr + + +ctx = d.Context(name='mlx5_0') +pd = PD(ctx) +cq = CQ(ctx, 100, None, None, 0) +cap = QPCap(100, 10, 1, 1, 0) +qia = QPInitAttr(cap=cap, qp_type = e.IBV_QPT_UD, scq=cq, rcq=cq) +# A UD QP will be in RTS if a QPAttr object is provided +udqp = QP(pd, qia, QPAttr()) +port_num = 1 +gid_index = 3 # Hard-coded for RoCE v2 interface +gid = ctx.query_gid(port_num, gid_index) +gr = GlobalRoute(dgid=gid, sgid_index=gid_index) +ah_attr = AHAttr(gr=gr, is_global=1, port_num=port_num) +ah=AH(pd, ah_attr) +wr = pwr.SendWR() +wr.set_wr_ud(ah, 0x1101, 0) # in real life, use real values +udqp.post_send(wr) +``` From patchwork Mon May 6 15:07:37 2019 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Noa Osherovich X-Patchwork-Id: 10931247 Return-Path: Received: from mail.wl.linuxfoundation.org (pdx-wl-mail.web.codeaurora.org [172.30.200.125]) by pdx-korg-patchwork-2.web.codeaurora.org (Postfix) with ESMTP id 35A89912 for ; Mon, 6 May 2019 15:07:53 +0000 (UTC) Received: from mail.wl.linuxfoundation.org (localhost [127.0.0.1]) by mail.wl.linuxfoundation.org (Postfix) with ESMTP id 23CA3287B6 for ; Mon, 6 May 2019 15:07:53 +0000 (UTC) Received: by mail.wl.linuxfoundation.org (Postfix, from userid 486) id 18744287D3; Mon, 6 May 2019 15:07:53 +0000 (UTC) X-Spam-Checker-Version: SpamAssassin 3.3.1 (2010-03-16) on pdx-wl-mail.web.codeaurora.org X-Spam-Level: X-Spam-Status: No, score=-7.9 required=2.0 tests=BAYES_00,MAILING_LIST_MULTI, RCVD_IN_DNSWL_HI,UNPARSEABLE_RELAY autolearn=ham version=3.3.1 Received: from vger.kernel.org (vger.kernel.org [209.132.180.67]) by mail.wl.linuxfoundation.org (Postfix) with ESMTP id EC677287B6 for ; Mon, 6 May 2019 15:07:51 +0000 (UTC) Received: (majordomo@vger.kernel.org) by vger.kernel.org via listexpand id S1727541AbfEFPHu (ORCPT ); Mon, 6 May 2019 11:07:50 -0400 Received: from mail-il-dmz.mellanox.com ([193.47.165.129]:50135 "EHLO mellanox.co.il" rhost-flags-OK-OK-OK-FAIL) by vger.kernel.org with ESMTP id S1727533AbfEFPHt (ORCPT ); Mon, 6 May 2019 11:07:49 -0400 Received: from Internal Mail-Server by MTLPINE2 (envelope-from noaos@mellanox.com) with ESMTPS (AES256-SHA encrypted); 6 May 2019 18:07:41 +0300 Received: from reg-l-vrt-059-009.mtl.labs.mlnx (reg-l-vrt-059-009.mtl.labs.mlnx [10.135.59.9]) by labmailer.mlnx (8.13.8/8.13.8) with ESMTP id x46F7edg019922; Mon, 6 May 2019 18:07:41 +0300 From: Noa Osherovich To: leon@kernel.org, jgg@mellanox.com, dledford@redhat.com Cc: linux-rdma@vger.kernel.org, Noa Osherovich Subject: [PATCH rdma-core 10/11] pyverbs/examples: Add base classes for pyverbs applications Date: Mon, 6 May 2019 18:07:37 +0300 Message-Id: <20190506150738.19477-11-noaos@mellanox.com> X-Mailer: git-send-email 2.17.2 In-Reply-To: <20190506150738.19477-1-noaos@mellanox.com> References: <20190506150738.19477-1-noaos@mellanox.com> Sender: linux-rdma-owner@vger.kernel.org Precedence: bulk List-ID: X-Mailing-List: linux-rdma@vger.kernel.org X-Virus-Scanned: ClamAV using ClamSMTP Add TCPData class, used to transfter connection details over TCP (GID, LID, QPN and PSN). Add Common and PingPong classes: Common provides common arguments (device, iterations etc.), a data exchange function and a statistics print function (iters/seconds). PingPong class inherits and extends Common. It is meant for pingpong-like applications and provides a wrap around CQ polling, data validation and some basic resources initialization as well. Signed-off-by: Noa Osherovich --- buildlib/pyverbs_functions.cmake | 2 +- pyverbs/CMakeLists.txt | 5 + pyverbs/examples/__init__.py | 0 pyverbs/examples/common.py | 331 +++++++++++++++++++++++++++++++ pyverbs/utils.py | 19 ++ 5 files changed, 356 insertions(+), 1 deletion(-) create mode 100644 pyverbs/examples/__init__.py create mode 100644 pyverbs/examples/common.py diff --git a/buildlib/pyverbs_functions.cmake b/buildlib/pyverbs_functions.cmake index 1966cf3ba1a3..6b272440a3d5 100644 --- a/buildlib/pyverbs_functions.cmake +++ b/buildlib/pyverbs_functions.cmake @@ -1,5 +1,5 @@ # SPDX-License-Identifier: (GPL-2.0 OR Linux-OpenIB) -# Copyright (c) 2018, Mellanox Technologies. All rights reserved. See COPYING file +# Copyright (c) 2018 Mellanox Technologies, Inc. All rights reserved. See COPYING file function(rdma_cython_module PY_MODULE) foreach(PYX_FILE ${ARGN}) diff --git a/pyverbs/CMakeLists.txt b/pyverbs/CMakeLists.txt index 328263fcc739..79d87ec39a20 100644 --- a/pyverbs/CMakeLists.txt +++ b/pyverbs/CMakeLists.txt @@ -32,6 +32,11 @@ rdma_python_module(pyverbs/tests tests/utils.py ) +rdma_python_module(pyverbs/examples + examples/common.py + examples/ib_devices.py + ) + rdma_internal_binary( run_tests.py ) diff --git a/pyverbs/examples/__init__.py b/pyverbs/examples/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/pyverbs/examples/common.py b/pyverbs/examples/common.py new file mode 100644 index 000000000000..f912ec85af70 --- /dev/null +++ b/pyverbs/examples/common.py @@ -0,0 +1,331 @@ +# SPDX-License-Identifier: (GPL-2.0 OR Linux-OpenIB) +# Copyright (c) 2019 Mellanox Technologies, Inc. All rights reserved. See COPYING file + +from _socket import SO_REUSEADDR, SOL_SOCKET +from time import sleep +import argparse +import random +import socket + +from pyverbs.pyverbs_error import PyverbsError, PyverbsUserError, \ + PyverbsRDMAError +from pyverbs.utils import wc_status_to_str +from pyverbs.device import Context +from pyverbs.pd import PD +import pyverbs.enums as e + +MAX_ATTEMPTS = 5 +DATA_LENGTH = 1024 + + +class TCPData: + """ + A wrapper class around the common data exchanged by ping-pong applications: + GID, LID, PSN and remote QPN. This class also provides a remote_data field + that can hold user-defined data. + """ + + def __init__(self, data=None): + self.extra_data = None + if data is None: + self.gid = None + self.lid = 0 + self.qpn = 0 + self.psn = random.getrandbits(24) + else: + remote_data = data.decode().split(',') + self.gid = remote_data[0] + self.lid = int(remote_data[1].strip()) + self.qpn = int(remote_data[2].strip()) + self.psn = int(remote_data[3].strip()) + if len(remote_data) > 4: + self.extra_data = remote_data[4].strip() + + def __str__(self): + ext = '' + if self.extra_data is not None: + ext = ', {ext}'.format(ext=self.extra_data) + return '{gid}, {lid}, {qpn}, {psn}{extra}'. \ + format(gid=self.gid, lid=self.lid, qpn=self.qpn, psn=self.psn, + extra=ext) + + +class Common: + """ + Base class for pyverbs-based applications. It provides basic args and + support for data exchange using TCPData + """ + + def __init__(self, logger=None): + """ + Initializes a Common object. + :param logger: A logger object to use for prints + """ + self.logger = logger + self.parser = argparse.ArgumentParser( + description='Parser for common examples options', + conflict_handler='resolve', + formatter_class=argparse.ArgumentDefaultsHelpFormatter) + self.parser.add_argument('-x', dest='server_name') + self.parser.add_argument('-d', '--device', default='mlx5_0', + help='IB device to use') + self.parser.add_argument('-i', '--ib-port', dest='ib_port', type=int, + default=1, help='Use of IB device') + self.parser.add_argument('-p', '--port', dest='tcp_port', type=int, + default=18515, + help='TCP port to exchange data over') + self.parser.add_argument('-s', '--size', dest='msg_size', type=int, + default=1024, + help='Size of message to exchange') + self.parser.add_argument('-r', '--rx-depth', dest='rx_depth', type=int, + default=1000) + self.parser.add_argument('-n', '--iters', dest='iters', type=int, + default=1000, help='Number of iterations') + self.tcp_data = TCPData() + self.remote_data = None + self.args = None + + @property + def rlid(self): + return self.remote_data.lid + + @property + def rgid(self): + return self.remote_data.gid + + @property + def rqpn(self): + return self.remote_data.qpn + + @property + def rpsn(self): + return self.remote_data.psn + + def base_print(self, data): + """ + Print the user-provided data into either the logger, if exists, or + to the shell. If logger exists, it is expected to have an 'info()' + method, similarly to the logger provided by Python's logging module. + :param data: Data to print + :return: None + """ + if self.logger: + self.logger.info(data) + else: + print(data) + + def print_stats(self, start, end): + """ + Print execution statistics similarly to rdma-core's ping-pong examples. + Calling function should provide the start and end time as measured by + the application. Number of iterations is taken from the test's + arguments. + :param start: A datetime object representing the execution start time + :param end: A datetime object representing the execution end time + :return: None + """ + time = end - start + total_us = time.seconds * 1000000 + time.microseconds + self.base_print('{iters} iters in {sec} seconds = {avg} usec/iter'. + format(iters=self.args.iters, sec=total_us / 1000000, + avg=total_us / self.args.iters)) + + def exchange_data(self, sync=False): + """ + Sync or exchange data between two sides. Server side opens a socket + and listens while client side attempts to connect MAX_ATTEMPTS times + before failing. + :param sync: If True, only sync between the two sides, else perform + data transfer as well. + :return: None + """ + + s = socket.socket() + s.setsockopt(SOL_SOCKET, SO_REUSEADDR, 1) + if self.args.server_name is None: + s.bind(('', self.args.tcp_port)) + s.listen(5) + c, addr = s.accept() + if not sync: + remote_data = c.recv(DATA_LENGTH) + c.send(self.tcp_data.__str__().encode()) + self.remote_data = TCPData(remote_data) + else: + attempts = 0 + connected = False + while not connected: + try: + s.connect((self.args.server_name, self.args.tcp_port)) + connected = True + except socket.error: + attempts += 1 + if attempts > MAX_ATTEMPTS: + raise PyverbsError( + 'Number of tries to connect socket exceeded') + sleep(1) + if not sync: + s.send(self.tcp_data.__str__().encode()) + remote_data = s.recv(DATA_LENGTH) + self.remote_data = TCPData(remote_data) + s.shutdown(socket.SHUT_RDWR) + s.close() + + +class PingPong(Common): + """ + Base class for pyverbs-based ping-pong applications. It provides a resource + initialization method (including querying port and GID if needed), data + validation, a wrapper around poll CQ and a basic check of the WC. + """ + + def __init__(self, logger=None): + """ + Initializes a PingPong object. + :param logger: A logger object to use for prints + """ + super(PingPong, self).__init__(logger) + self.context = None + self.recv_sg = None + self.add_grh = 0 + self.args = None + self.pd = None + self.mr = None + self.num_cq_events = 0 + self.parser.description = 'Parser for ping-pong common options' + self.parser.add_argument('-g', '--gid-index', dest='gid_index', + type=int, default=-1, + help='Source GID index') + self.parser.add_argument('-l', '--sl', dest='sl', type=int, default=0, + metavar='[0-15]', help='Service level value') + self.parser.add_argument('-v', '--data-validation', + action='store_true', dest='validate_data') + + def query_port_attr(self): + """ + Query port attributes for LID and whether or not GRH is required. GRH + is required in RoCE (as per spec) and in an environment which uses + GID-based addressing (such as IB SR-IOV). + This method provides early fail in case GRH is requires but GID index + was not provided by the user. An exception is raised in such case. + After calling this method, self.add_grh and self.tcp_data.lid will be + properly set. + :return: None + """ + port_attr = self.context.query_port(self.args.ib_port) + grh_bit_set = port_attr.flags & e.IBV_QPF_GRH_REQUIRED + + if grh_bit_set: + if self.args.gid_index == -1: + raise PyverbsUserError( + 'GRH is required but GID index was not provided') + self.add_grh = 1 + self.tcp_data.lid = port_attr.lid + + def set_gid(self): + self.tcp_data.gid =\ + self.context.query_gid(self.args.ib_port, + self.args.gid_index).__str__() + + def init_resources(self): + """ + Create Context and use it to query port attributes and GID. + Port attributes are used to: + 1. Check if GRH is required for this specific device. + 2. Acquire LID. + GID and LID will later be exchanged in order to create a connection. + PD is also initialized in this method instead of in each inheriting + class. + :return: None + """ + self.context = Context(name=self.args.device) + self.query_port_attr() + if self.add_grh: + self.set_gid() + self.pd = PD(self.context) + + def execute(self): + """ + A wrapper method to start execution properly: First initialize the + resources, then run. + :return: None + """ + self.init_resources() + self.run() + + def print_connection_details(self, is_local): + """ + Print the local and remote connection details: GID, LID, QPN and PSN. + :param is_local: If True, print prefix will be 'Local data', else + 'Remote data' + :return: None + """ + ext = '' + if self.tcp_data.extra_data is not None and is_local: + ext = 'Extra data: {e}'.format(e=self.tcp_data.extra_data) + if not is_local and self.remote_data.extra_data is not None: + ext = 'Extra data: {e}'.format(e=self.remote_data.extra_data) + if is_local: + self.base_print( + 'Local data : GID: {gid} LID: {lid} QPN: {qpn} PSN: {psn} {ex}'. + format(gid=self.tcp_data.gid, lid=self.tcp_data.lid, + qpn=self.tcp_data.qpn, psn=self.tcp_data.psn, ex=ext)) + else: + self.base_print( + 'Remote data: GID: {gid} LID: {lid} QPN: {qpn} PSN: {psn} {ex}'. + format(gid=self.rgid, lid=self.rlid, qpn=self.rqpn, + psn=self.rpsn, ex=ext)) + + @staticmethod + def check_wc(wc): + """ + Verify that the given work completion's status is success, else an + exception is raised. + :param wc: A WC object to check + :return: None + """ + if wc.status != e.IBV_WC_SUCCESS: + raise PyverbsRDMAError('Completion status is {status}'.format( + status=wc_status_to_str(wc.status))) + + def poll_cq(self, count): + """ + Poll completions from the CQ. + Note: This function calls the blocking poll() method of the CQ until + until completions were received. Alternatively, gets a single + CQ event when events are used. + :param count: How many completions to poll + :return: An array of work completions of length , None when + events are used + """ + if self.args.use_events: + if count > 1: + raise PyverbsUserError('Can\'t poll more than 1 CQE when using events') + self.cc.get_cq_event(self.cq) + self.num_cq_events += 1 + self.cq.req_notify() + while count > 0: + nc, wcs = self.cq.poll(count) + for wc in wcs: + self.check_wc(wc) + count -= nc + return wcs + + def validate(self, received_str): + """ + Validate the received buffer against the expected result. + The application should set client's send buffer to 'c's and the + server's send buffer to 's's. + If the expected buffer is different than the actual, an exception will + be raised. + :param received_str: The received buffer to check + :return: None + """ + expected_str = self.args.msg_size * ('c' if self.is_server else 's') + received_str = received_str.decode() + if received_str[0:self.args.msg_size] == \ + expected_str[0:self.args.msg_size]: + return + else: + raise PyverbsError( + 'Data validation failure: expected {exp}, received {rcv}'. + format(exp=expected_str, rcv=received_str)) diff --git a/pyverbs/utils.py b/pyverbs/utils.py index a59d6275fefc..0126801eb02f 100644 --- a/pyverbs/utils.py +++ b/pyverbs/utils.py @@ -79,3 +79,22 @@ def mig_state_to_str(mig): return mig_states[mig] except KeyError: return 'Unknown ({m})'.format(m=mig) + + +def wc_status_to_str(status): + try: + return \ + {0: 'Success', 1: 'Local length error', + 2: 'local QP operation error', 3: 'Local EEC operation error', + 4: 'Local protection error', 5: 'WR flush error', + 6: 'Memory window bind error', 7: 'Bad response error', + 8: 'Local access error', 9: 'Remote invalidate request error', + 10: 'Remote access error', 11: 'Remote operation error', + 12: 'Retry exceeded', 13: 'RNR retry exceeded', + 14: 'Local RDD violation error', + 15: 'Remote invalidate RD request error', + 16: 'Remote aort error', 17: 'Invalidate EECN error', + 18: 'Invalidate EEC state error', 19: 'Fatal error', + 20: 'Response timeout error', 21: 'General error'}[status] + except KeyError: + return 'Unknown WC status ({s})'.format(s=status) From patchwork Mon May 6 15:07:38 2019 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Noa Osherovich X-Patchwork-Id: 10931251 Return-Path: Received: from mail.wl.linuxfoundation.org (pdx-wl-mail.web.codeaurora.org [172.30.200.125]) by pdx-korg-patchwork-2.web.codeaurora.org (Postfix) with ESMTP id 3EDC81575 for ; Mon, 6 May 2019 15:07:54 +0000 (UTC) Received: from mail.wl.linuxfoundation.org (localhost [127.0.0.1]) by mail.wl.linuxfoundation.org (Postfix) with ESMTP id 2E2DC2878F for ; Mon, 6 May 2019 15:07:54 +0000 (UTC) Received: by mail.wl.linuxfoundation.org (Postfix, from userid 486) id 22B23287B6; Mon, 6 May 2019 15:07:54 +0000 (UTC) X-Spam-Checker-Version: SpamAssassin 3.3.1 (2010-03-16) on pdx-wl-mail.web.codeaurora.org X-Spam-Level: X-Spam-Status: No, score=-7.9 required=2.0 tests=BAYES_00,MAILING_LIST_MULTI, RCVD_IN_DNSWL_HI,UNPARSEABLE_RELAY autolearn=ham version=3.3.1 Received: from vger.kernel.org (vger.kernel.org [209.132.180.67]) by mail.wl.linuxfoundation.org (Postfix) with ESMTP id 3D976287C3 for ; Mon, 6 May 2019 15:07:51 +0000 (UTC) Received: (majordomo@vger.kernel.org) by vger.kernel.org via listexpand id S1727458AbfEFPHt (ORCPT ); Mon, 6 May 2019 11:07:49 -0400 Received: from mail-il-dmz.mellanox.com ([193.47.165.129]:50140 "EHLO mellanox.co.il" rhost-flags-OK-OK-OK-FAIL) by vger.kernel.org with ESMTP id S1727586AbfEFPHs (ORCPT ); Mon, 6 May 2019 11:07:48 -0400 Received: from Internal Mail-Server by MTLPINE2 (envelope-from noaos@mellanox.com) with ESMTPS (AES256-SHA encrypted); 6 May 2019 18:07:41 +0300 Received: from reg-l-vrt-059-009.mtl.labs.mlnx (reg-l-vrt-059-009.mtl.labs.mlnx [10.135.59.9]) by labmailer.mlnx (8.13.8/8.13.8) with ESMTP id x46F7edh019922; Mon, 6 May 2019 18:07:41 +0300 From: Noa Osherovich To: leon@kernel.org, jgg@mellanox.com, dledford@redhat.com Cc: linux-rdma@vger.kernel.org, Noa Osherovich Subject: [PATCH rdma-core 11/11] pyverbs/examples: RC pingpong Date: Mon, 6 May 2019 18:07:38 +0300 Message-Id: <20190506150738.19477-12-noaos@mellanox.com> X-Mailer: git-send-email 2.17.2 In-Reply-To: <20190506150738.19477-1-noaos@mellanox.com> References: <20190506150738.19477-1-noaos@mellanox.com> Sender: linux-rdma-owner@vger.kernel.org Precedence: bulk List-ID: X-Mailing-List: linux-rdma@vger.kernel.org X-Virus-Scanned: ClamAV using ClamSMTP This patch adds the RC pingpong example, similar to the one in libibverbs. Its current options are as follows: $ python3 rc_pingpong.py -h usage: rc_pingpong.py [-h] [-x SERVER_NAME] [-d DEVICE] [-i IB_PORT] [-p TCP_PORT] [-s MSG_SIZE] [-r RX_DEPTH] [-n ITERS] [-g GID_INDEX] [-l [0-15]] [-v] [-m PATH_MTU] [-j] [-o] [-e] Parser for RC pingpong options optional arguments: -h, --help show this help message and exit -x SERVER_NAME -d DEVICE, --device DEVICE IB device to use (default: mlx5_0) -i IB_PORT, --ib-port IB_PORT Use of IB device (default: 1) -p TCP_PORT, --port TCP_PORT TCP port to exchange data over (default: 18515) -s MSG_SIZE, --size MSG_SIZE Size of message to exchange (default: 1024) -r RX_DEPTH, --rx-depth RX_DEPTH -n ITERS, --iters ITERS Number of iterations (default: 1000) -g GID_INDEX, --gid-index GID_INDEX Source GID index -l [0-15], --sl [0-15] Service level value (default: 0) -v, --data-validation -m PATH_MTU, --mtu PATH_MTU -j, --dm Use device memory (default: False) -o, --odp use on demand paging (default: False) -e, --use-events Use CQ events instead of poll (default: False) Signed-off-by: Noa Osherovich --- pyverbs/CMakeLists.txt | 1 + pyverbs/examples/rc_pingpong.py | 208 ++++++++++++++++++++++++++++++++ 2 files changed, 209 insertions(+) create mode 100644 pyverbs/examples/rc_pingpong.py diff --git a/pyverbs/CMakeLists.txt b/pyverbs/CMakeLists.txt index 79d87ec39a20..26da71e554e2 100644 --- a/pyverbs/CMakeLists.txt +++ b/pyverbs/CMakeLists.txt @@ -35,6 +35,7 @@ rdma_python_module(pyverbs/tests rdma_python_module(pyverbs/examples examples/common.py examples/ib_devices.py + examples/rc_pingpong.py ) rdma_internal_binary( diff --git a/pyverbs/examples/rc_pingpong.py b/pyverbs/examples/rc_pingpong.py new file mode 100644 index 000000000000..cce72e5ee405 --- /dev/null +++ b/pyverbs/examples/rc_pingpong.py @@ -0,0 +1,208 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: (GPL-2.0 OR Linux-OpenIB) +# Copyright (c) 2019 Mellanox Technologies, Inc. All rights reserved. See COPYING file +import datetime +import sys + +from pyverbs.qp import QPCap, QPInitAttr, QPAttr, QP +from pyverbs.pyverbs_error import PyverbsUserError +from pyverbs.addr import AHAttr, GlobalRoute, GID +from pyverbs.examples.common import PingPong +from pyverbs.device import AllocDmAttr, DM +from pyverbs.wr import SGE, RecvWR, SendWR +from pyverbs.cq import CQ, CompChannel +from pyverbs.mr import MR, DMMR +import pyverbs.enums as e + + +class RCPingPong(PingPong): + def __init__(self, logger, args): + super(RCPingPong, self).__init__(logger) + self.args = None + self.parse(args) + self.qp = None + self.cq = None + self.dm = None + self.cc = None + self.recv_sge = None + self.is_server = False + self.num_cq_events = 0 + + def parse(self, args): + self.parser.add_argument('-m', '--mtu', dest='path_mtu', default=1024, + type=int) + self.parser.add_argument('-j', '--dm', action='store_true', + dest='use_dm', help='Use device memory') + self.parser.add_argument('-o', '--odp', action='store_true', + dest='odp', help='use on demand paging') + self.parser.add_argument('-e', '--use-events', action='store_true', + help='Use CQ events instead of poll') + self.parser.description = 'Parser for RC pingpong options' + self.args = self.parser.parse_args(args) + + def sanity(self): + """ + Perform basic sanity checks on the input and check if requested + features are supported by the device. + :return: None + """ + attr_ex = self.context.query_device_ex(None) + if self.args.use_dm and self.args.odp: + raise PyverbsUserError('Device memory region can\'t be on demand') + if self.args.odp: + rc_mask = e.IBV_ODP_SUPPORT_SEND | e.IBV_ODP_SUPPORT_RECV + if attr_ex.odp_caps.general_caps & e.IBV_ODP_SUPPORT == 0: + raise PyverbsUserError('The device isn\'t ODP capable') + if attr_ex.odp_caps.rc_odp_caps & rc_mask != rc_mask: + raise PyverbsUserError('RC send/recv with ODP are not supported') + if self.args.use_dm: + if attr_ex.max_dm_size == 0: + raise PyverbsUserError('Device doesn\'t support DM allocation') + if attr_ex.max_dm_size < self.args.msg_size: + raise PyverbsUserError('Device max DM allocation: {s}, requested: {r}'. + format(s=attr_ex.max_dm_size, + r=self.args.msg_size)) + + def mtu_to_enum(self): + """ + Converts the user-provided (or default) MTU (in bytes) to the matching + enum values. + :return: The enum entry that matches the given MTU + """ + mtus = {256: e.IBV_MTU_256, 512: e.IBV_MTU_512, 1024: e.IBV_MTU_1024, + 2048: e.IBV_MTU_2048, 4096: e.IBV_MTU_4096} + try: + return mtus[self.args.path_mtu] + except KeyError: + raise PyverbsUserError('Invalid MTU {m}'. + format(m=self.args.path_mtu)) + + def rc_qp_attr(self, attr): + """ + Set the QP attributes' values to arbitrary values (based on the values + used in ibv_rc_pingpong). + :param attr: QPAttr object to modify + :return: None + """ + attr.dest_qp_num = self.rqpn + attr.path_mtu = self.mtu_to_enum() + attr.max_dest_rd_atomic = 1 + attr.min_rnr_timer = 12 + attr.rq_psn = self.rpsn + attr.sq_psn = self.tcp_data.psn + attr.timeout = 14 + attr.retry_cnt = 7 + attr.rnr_retry = 7 + attr.max_rd_atomic = 1 + + def post_recv(self, num_wqes): + """ + Call the QP's post_recv() method times. + Since this is an example, the same WR is used for each post_recv(). + :param num_wqes: Number of WQEs to post + """ + self.recv_sge = SGE(self.mr.buf if self.dm is None else 0, + self.args.msg_size, self.mr.lkey) + wr_id = 1 if self.is_server else 2 + recv_wr = RecvWR(wr_id=wr_id, sg=[self.recv_sge], num_sge=1) + for i in range(num_wqes): + self.qp.post_recv(recv_wr, None) + + def validate(self): + received_str = self.recv_sge.read(self.args.msg_size, 0) \ + if self.dm is None else self.dm.copy_from_dm(0, self.args.msg_size) + super(RCPingPong, self).validate(received_str) + + def post_send(self): + """ + Post a single send WR on the QP. The content is either 's' for server + side or 'c' for client side. + :return: None + """ + length = self.args.msg_size + send_sge = SGE(self.mr.buf if self.dm is None else 0, + length, self.mr.lkey) + if self.args.validate_data: + msg = length * ('s' if self.is_server else 'c') + if self.dm is not None: + self.dm.copy_to_dm(0, msg.encode(), length) + else: + self.mr.write(msg, length) + send_wr = SendWR(wr_id=1 if self.is_server else 2, num_sge=1, + sg=[send_sge]) + self.qp.post_send(send_wr, None) + + def run(self): + self.is_server = True if self.args.server_name is None else False + self.sanity() + # Handle device memory + if self.args.use_dm: + dm_attr = AllocDmAttr(self.args.msg_size) + self.dm = DM(self.context, dm_attr) + # Other RDMA resources: CQ, MR, QP + if self.args.use_events: + self.cc = CompChannel(self.context) + else: + self.cc = None + self.cq = CQ(self.context, self.args.rx_depth + 1, None, self.cc, 0) + if self.args.use_events: + self.cq.req_notify() + + access_flags = e.IBV_ACCESS_LOCAL_WRITE + if self.dm is None: + if self.args.odp: + access_flags |= e.IBV_ACCESS_ON_DEMAND + self.mr = MR(self.pd, self.args.msg_size, access_flags) + else: + access_flags |= e.IBV_ACCESS_ZERO_BASED + self.mr = DMMR(self.pd, self.args.msg_size, access_flags, + dm=self.dm, offset=0) + qp_caps = QPCap(max_recv_wr=self.args.rx_depth) + qp_init_attr = QPInitAttr(qp_type=e.IBV_QPT_RC, scq=self.cq, + rcq=self.cq, cap=qp_caps) + qp_attr = QPAttr(port_num=self.args.ib_port) + self.qp = QP(self.pd, qp_init_attr, qp_attr) + # TCP data exchange + self.tcp_data.qpn = self.qp.qp_num + self.print_connection_details(is_local=True) + self.exchange_data() + self.print_connection_details(is_local=False) + # 2RTS + self.rc_qp_attr(qp_attr) + gid_idx = self.args.gid_index if self.add_grh else 0 + if self.add_grh: + gr = GlobalRoute(dgid=GID(self.rgid), sgid_index=gid_idx) + else: + gr = None + ah_attr = AHAttr(port_num=self.args.ib_port, is_global=self.add_grh, + gr=gr, dlid=self.rlid, sl=self.args.sl) + qp_attr.ah_attr = ah_attr + self.qp.to_rts(qp_attr) + self.post_recv(self.args.rx_depth) + self.exchange_data(sync=True) + start = datetime.datetime.now() + for i in range(self.args.iters): + if self.is_server: + self.poll_cq(1) + if self.args.validate_data: + self.validate() + self.post_send() + self.poll_cq(1) + self.post_recv(1) + else: + self.post_send() + self.poll_cq(1) + self.poll_cq(1) + self.post_recv(1) + if self.args.validate_data: + self.validate() + + if self.args.use_events: + self.cq.ack_events(self.num_cq_events) + end = datetime.datetime.now() + self.print_stats(start, end) + + +if __name__ == '__main__': + player = RCPingPong(None, sys.argv[1:]) + player.execute()