From patchwork Mon Sep 9 09:07:01 2019 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Noa Osherovich X-Patchwork-Id: 11137487 Return-Path: Received: from mail.kernel.org (pdx-korg-mail-1.web.codeaurora.org [172.30.200.123]) by pdx-korg-patchwork-2.web.codeaurora.org (Postfix) with ESMTP id 7BA431708 for ; Mon, 9 Sep 2019 09:07:20 +0000 (UTC) Received: from vger.kernel.org (vger.kernel.org [209.132.180.67]) by mail.kernel.org (Postfix) with ESMTP id 6479A2089F for ; Mon, 9 Sep 2019 09:07:20 +0000 (UTC) Received: (majordomo@vger.kernel.org) by vger.kernel.org via listexpand id S2389599AbfIIJHT (ORCPT ); Mon, 9 Sep 2019 05:07:19 -0400 Received: from mail-il-dmz.mellanox.com ([193.47.165.129]:48362 "EHLO mellanox.co.il" rhost-flags-OK-OK-OK-FAIL) by vger.kernel.org with ESMTP id S1727674AbfIIJHT (ORCPT ); Mon, 9 Sep 2019 05:07:19 -0400 Received: from Internal Mail-Server by MTLPINE1 (envelope-from noaos@mellanox.com) with ESMTPS (AES256-SHA encrypted); 9 Sep 2019 12:07:16 +0300 Received: from reg-l-vrt-059-007.mtl.labs.mlnx (reg-l-vrt-059-007.mtl.labs.mlnx [10.135.59.7]) by labmailer.mlnx (8.13.8/8.13.8) with ESMTP id x8997Fox028426; Mon, 9 Sep 2019 12:07:16 +0300 From: Noa Osherovich To: dledford@redhat.com, jgg@mellanox.com, leonro@mellanox.com Cc: linux-rdma@vger.kernel.org, Maxim Chicherin Subject: [PATCH rdma-core 01/12] pyverbs: Fix WC creation process Date: Mon, 9 Sep 2019 12:07:01 +0300 Message-Id: <20190909090712.11029-2-noaos@mellanox.com> X-Mailer: git-send-email 2.21.0 In-Reply-To: <20190909090712.11029-1-noaos@mellanox.com> References: <20190909090712.11029-1-noaos@mellanox.com> MIME-Version: 1.0 Sender: linux-rdma-owner@vger.kernel.org Precedence: bulk List-ID: X-Mailing-List: linux-rdma@vger.kernel.org From: Maxim Chicherin In WC constructor, parameters assignment was incorrect and values were not stored properly. imm_data attribute was not initiated, imm_data represents immediate data in network byte order if wc_flags & IBV_WC_WITH_IMM or stores the invalidated rkey if wc_flags & IBV_WC_WITH_INV. Signed-off-by: Maxim Chicherin --- pyverbs/cq.pyx | 33 +++++++++++++++++++++------------ 1 file changed, 21 insertions(+), 12 deletions(-) mode change 100644 => 100755 pyverbs/cq.pyx diff --git a/pyverbs/cq.pyx b/pyverbs/cq.pyx old mode 100644 new mode 100755 index dc09924e88a9..3ac5f704766b --- a/pyverbs/cq.pyx +++ b/pyverbs/cq.pyx @@ -366,18 +366,19 @@ cdef class WC(PyverbsObject): def __cinit__(self, wr_id=0, status=0, opcode=0, vendor_err=0, byte_len=0, qp_num=0, src_qp=0, imm_data=0, wc_flags=0, pkey_index=0, slid=0, sl=0, dlid_path_bits=0): - self.wr_id = wr_id - self.status = status - self.opcode = opcode - self.vendor_err = vendor_err - self.byte_len = byte_len - self.qp_num = qp_num - self.src_qp = src_qp - self.wc_flags = wc_flags - self.pkey_index = pkey_index - self.slid = slid - self.sl = sl - self.dlid_path_bits = dlid_path_bits + self.wc.wr_id = wr_id + self.wc.status = status + self.wc.opcode = opcode + self.wc.vendor_err = vendor_err + self.wc.byte_len = byte_len + self.wc.qp_num = qp_num + self.wc.src_qp = src_qp + self.wc.wc_flags = wc_flags + self.wc.pkey_index = pkey_index + self.wc.slid = slid + self.wc.imm_data = imm_data + self.wc.sl = sl + self.wc.dlid_path_bits = dlid_path_bits @property def wr_id(self): @@ -456,6 +457,13 @@ cdef class WC(PyverbsObject): def sl(self, val): self.wc.sl = val + @property + def imm_data(self): + return self.wc.imm_data + @imm_data.setter + def imm_data(self, val): + self.wc.imm_data = val + @property def dlid_path_bits(self): return self.wc.dlid_path_bits @@ -476,6 +484,7 @@ cdef class WC(PyverbsObject): print_format.format('pkey index', self.pkey_index) +\ print_format.format('slid', self.slid) +\ print_format.format('sl', self.sl) +\ + print_format.format('imm_data', self.imm_data) +\ print_format.format('dlid path bits', self.dlid_path_bits) From patchwork Mon Sep 9 09:07:02 2019 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Noa Osherovich X-Patchwork-Id: 11137509 Return-Path: Received: from mail.kernel.org (pdx-korg-mail-1.web.codeaurora.org [172.30.200.123]) by pdx-korg-patchwork-2.web.codeaurora.org (Postfix) with ESMTP id B565A1708 for ; Mon, 9 Sep 2019 09:07:25 +0000 (UTC) Received: from vger.kernel.org (vger.kernel.org [209.132.180.67]) by mail.kernel.org (Postfix) with ESMTP id 9EDD32196E for ; Mon, 9 Sep 2019 09:07:25 +0000 (UTC) Received: (majordomo@vger.kernel.org) by vger.kernel.org via listexpand id S2389623AbfIIJHY (ORCPT ); Mon, 9 Sep 2019 05:07:24 -0400 Received: from mail-il-dmz.mellanox.com ([193.47.165.129]:48369 "EHLO mellanox.co.il" rhost-flags-OK-OK-OK-FAIL) by vger.kernel.org with ESMTP id S2389617AbfIIJHT (ORCPT ); Mon, 9 Sep 2019 05:07:19 -0400 Received: from Internal Mail-Server by MTLPINE1 (envelope-from noaos@mellanox.com) with ESMTPS (AES256-SHA encrypted); 9 Sep 2019 12:07:16 +0300 Received: from reg-l-vrt-059-007.mtl.labs.mlnx (reg-l-vrt-059-007.mtl.labs.mlnx [10.135.59.7]) by labmailer.mlnx (8.13.8/8.13.8) with ESMTP id x8997Fp0028426; Mon, 9 Sep 2019 12:07:16 +0300 From: Noa Osherovich To: dledford@redhat.com, jgg@mellanox.com, leonro@mellanox.com Cc: linux-rdma@vger.kernel.org, Maxim Chicherin Subject: [PATCH rdma-core 02/12] pyverbs: Fix CQ and PD assignment in QPAttr Date: Mon, 9 Sep 2019 12:07:02 +0300 Message-Id: <20190909090712.11029-3-noaos@mellanox.com> X-Mailer: git-send-email 2.21.0 In-Reply-To: <20190909090712.11029-1-noaos@mellanox.com> References: <20190909090712.11029-1-noaos@mellanox.com> MIME-Version: 1.0 Sender: linux-rdma-owner@vger.kernel.org Precedence: bulk List-ID: X-Mailing-List: linux-rdma@vger.kernel.org From: Maxim Chicherin Fixed CQs assignment in QPInitAttr, QPInitAttrEx and QP objects: Receive cq parameter was assigned to send_cq attribute in InitAttr objects, and in QP rcq and scq attributes was not initialized properly. Fixed PD assignment in QPInitAttrEx object: In QPInitAttrEx pd pointer was not initialized with PD.pd pointer. Signed-off-by: Maxim Chicherin --- pyverbs/qp.pyx | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) mode change 100644 => 100755 pyverbs/qp.pyx diff --git a/pyverbs/qp.pyx b/pyverbs/qp.pyx old mode 100644 new mode 100755 index ad89c94c002e..732ef5094b6b --- a/pyverbs/qp.pyx +++ b/pyverbs/qp.pyx @@ -104,9 +104,9 @@ cdef class QPInitAttr(PyverbsObject): self.attr.qp_context = qp_context if scq is not None: if type(scq) is CQ: - self.attr.send_cq = (rcq).cq + self.attr.send_cq = (scq).cq elif type(scq) is CQEX: - self.attr.send_cq = (rcq).ibv_cq + self.attr.send_cq = (scq).ibv_cq else: raise PyverbsUserError('Expected CQ/CQEX, got {t}'.\ format(t=type(scq))) @@ -221,9 +221,9 @@ cdef class QPInitAttrEx(PyverbsObject): _copy_caps(cap, self) if scq is not None: if type(scq) is CQ: - self.attr.send_cq = (rcq).cq + self.attr.send_cq = (scq).cq elif type(scq) is CQEX: - self.attr.send_cq = (rcq).ibv_cq + self.attr.send_cq = (scq).ibv_cq else: raise PyverbsUserError('Expected CQ/CQEX, got {t}'.\ format(t=type(scq))) @@ -251,6 +251,7 @@ cdef class QPInitAttrEx(PyverbsObject): self.attr.comp_mask = comp_mask if pd is not None: self.pd = pd + self.attr.pd = pd.pd self.attr.create_flags = create_flags self.attr.max_tso_header = max_tso_header self.attr.source_qpn = source_qpn @@ -814,18 +815,20 @@ cdef class QP(PyverbsCM): if type(init_attr.send_cq) == CQ: cq = init_attr.send_cq cq.add_ref(self) + self.scq = cq else: cqex = init_attr.send_cq cqex.add_ref(self) - self.scq = cq + self.scq = cqex if init_attr.send_cq != init_attr.recv_cq and init_attr.recv_cq is not None: if type(init_attr.recv_cq) == CQ: cq = init_attr.recv_cq cq.add_ref(self) + self.rcq = cq else: cqex = init_attr.recv_cq cqex.add_ref(self) - self.rcq = cq + self.rcq = cqex def _create_qp(self, PD pd, QPInitAttr attr): self.qp = v.ibv_create_qp(pd.pd, &attr.attr) From patchwork Mon Sep 9 09:07:03 2019 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Noa Osherovich X-Patchwork-Id: 11137503 Return-Path: Received: from mail.kernel.org (pdx-korg-mail-1.web.codeaurora.org [172.30.200.123]) by pdx-korg-patchwork-2.web.codeaurora.org (Postfix) with ESMTP id 726BF1599 for ; Mon, 9 Sep 2019 09:07:23 +0000 (UTC) Received: from vger.kernel.org (vger.kernel.org [209.132.180.67]) by mail.kernel.org (Postfix) with ESMTP id 5ABA92089F for ; Mon, 9 Sep 2019 09:07:23 +0000 (UTC) Received: (majordomo@vger.kernel.org) by vger.kernel.org via listexpand id S1727674AbfIIJHT (ORCPT ); Mon, 9 Sep 2019 05:07:19 -0400 Received: from mail-il-dmz.mellanox.com ([193.47.165.129]:48370 "EHLO mellanox.co.il" rhost-flags-OK-OK-OK-FAIL) by vger.kernel.org with ESMTP id S2389601AbfIIJHT (ORCPT ); Mon, 9 Sep 2019 05:07:19 -0400 Received: from Internal Mail-Server by MTLPINE1 (envelope-from noaos@mellanox.com) with ESMTPS (AES256-SHA encrypted); 9 Sep 2019 12:07:16 +0300 Received: from reg-l-vrt-059-007.mtl.labs.mlnx (reg-l-vrt-059-007.mtl.labs.mlnx [10.135.59.7]) by labmailer.mlnx (8.13.8/8.13.8) with ESMTP id x8997Fp1028426; Mon, 9 Sep 2019 12:07:16 +0300 From: Noa Osherovich To: dledford@redhat.com, jgg@mellanox.com, leonro@mellanox.com Cc: linux-rdma@vger.kernel.org, Maxim Chicherin Subject: [PATCH rdma-core 03/12] pyverbs: Remove TM enums Date: Mon, 9 Sep 2019 12:07:03 +0300 Message-Id: <20190909090712.11029-4-noaos@mellanox.com> X-Mailer: git-send-email 2.21.0 In-Reply-To: <20190909090712.11029-1-noaos@mellanox.com> References: <20190909090712.11029-1-noaos@mellanox.com> MIME-Version: 1.0 Sender: linux-rdma-owner@vger.kernel.org Precedence: bulk List-ID: X-Mailing-List: linux-rdma@vger.kernel.org From: Maxim Chicherin Currently tag matching feature is not supported. Signed-off-by: Maxim Chicherin --- pyverbs/cq.pyx | 1 - pyverbs/libibverbs_enums.pxd | 25 ------------------------- tests/test_cq.py | 2 -- 3 files changed, 28 deletions(-) mode change 100644 => 100755 pyverbs/libibverbs_enums.pxd diff --git a/pyverbs/cq.pyx b/pyverbs/cq.pyx index 3ac5f704766b..32eee0a0f1fd 100755 --- a/pyverbs/cq.pyx +++ b/pyverbs/cq.pyx @@ -579,6 +579,5 @@ def create_wc_flags_to_str(flags): e.IBV_WC_EX_WITH_COMPLETION_TIMESTAMP: 'IBV_WC_EX_WITH_COMPLETION_TIMESTAMP', e.IBV_WC_EX_WITH_CVLAN: 'IBV_WC_EX_WITH_CVLAN', e.IBV_WC_EX_WITH_FLOW_TAG: 'IBV_WC_EX_WITH_FLOW_TAG', - e.IBV_WC_EX_WITH_TM_INFO: 'IBV_WC_EX_WITH_TM_INFO', e.IBV_WC_EX_WITH_COMPLETION_TIMESTAMP_WALLCLOCK: 'IBV_WC_EX_WITH_COMPLETION_TIMESTAMP_WALLCLOCK'} return flags_to_str(flags, cqe_flags) diff --git a/pyverbs/libibverbs_enums.pxd b/pyverbs/libibverbs_enums.pxd old mode 100644 new mode 100755 index c347ef31dd2b..1d437240a883 --- a/pyverbs/libibverbs_enums.pxd +++ b/pyverbs/libibverbs_enums.pxd @@ -202,7 +202,6 @@ cdef extern from '': IBV_WC_EX_WITH_COMPLETION_TIMESTAMP = 1 << 7 IBV_WC_EX_WITH_CVLAN = 1 << 8 IBV_WC_EX_WITH_FLOW_TAG = 1 << 9 - IBV_WC_EX_WITH_TM_INFO = 1 << 10 IBV_WC_EX_WITH_COMPLETION_TIMESTAMP_WALLCLOCK = 1 << 11 cpdef enum ibv_wc_flags: @@ -210,12 +209,6 @@ cdef extern from '': IBV_WC_WITH_IMM = 1 << 1 IBV_WC_IP_CSUM_OK = 1 << 2 IBV_WC_WITH_INV = 1 << 3 - IBV_WC_TM_SYNC_REQ = 1 << 4 - IBV_WC_TM_MATCH = 1 << 5 - IBV_WC_TM_DATA_VALID = 1 << 6 - - cpdef enum ibv_tm_cap_flags: - IBV_TM_CAP_RC = 1 << 0, cpdef enum ibv_srq_attr_mask: IBV_SRQ_MAX_WR = 1 << 0, @@ -224,14 +217,12 @@ cdef extern from '': cpdef enum ibv_srq_type: IBV_SRQT_BASIC IBV_SRQT_XRC - IBV_SRQT_TM cpdef enum ibv_srq_init_attr_mask: IBV_SRQ_INIT_ATTR_TYPE = 1 << 0 IBV_SRQ_INIT_ATTR_PD = 1 << 1 IBV_SRQ_INIT_ATTR_XRCD = 1 << 2 IBV_SRQ_INIT_ATTR_CQ = 1 << 3 - IBV_SRQ_INIT_ATTR_TM = 1 << 4 cpdef enum ibv_mig_state: IBV_MIG_MIGRATED @@ -313,15 +304,6 @@ cdef extern from '': IBV_RX_HASH_SRC_PORT_UDP = 1 << 6 IBV_RX_HASH_DST_PORT_UDP = 1 << 7 - cpdef enum ibv_ops_wr_opcode: - IBV_WR_TAG_ADD - IBV_WR_TAG_DEL - IBV_WR_TAG_SYNC - - cpdef enum ibv_ops_flags: - IBV_OPS_SIGNALED = 1 << 0 - IBV_OPS_TM_SYNC = 1 << 1 - cpdef enum ibv_flow_flags: IBV_FLOW_ATTR_FLAGS_ALLOW_LOOP_BACK = 1 << 0 IBV_FLOW_ATTR_FLAGS_DONT_TRAP = 1 << 1 @@ -415,12 +397,5 @@ cdef extern from '': cdef unsigned long long IBV_DEVICE_PCI_WRITE_END_PADDING -cdef extern from "": - cpdef enum ibv_tmh_op: - IBV_TMH_NO_TAG = 0 - IBV_TMH_RNDV = 1 - IBV_TMH_FIN = 2 - IBV_TMH_EAGER = 3 - _IBV_DEVICE_RAW_SCATTER_FCS = IBV_DEVICE_RAW_SCATTER_FCS _IBV_DEVICE_PCI_WRITE_END_PADDING = IBV_DEVICE_PCI_WRITE_END_PADDING diff --git a/tests/test_cq.py b/tests/test_cq.py index 7848f39c9c63..ad6bfde7b216 100644 --- a/tests/test_cq.py +++ b/tests/test_cq.py @@ -168,8 +168,6 @@ def get_attrs_ex(attr, attr_ex): wc_flags = list(e.ibv_create_cq_wc_flags) # Flow tag is not always supported, doesn't have a capability bit to check wc_flags.remove(e.IBV_WC_EX_WITH_FLOW_TAG) - if attr_ex.tm_caps.max_ops == 0: - wc_flags.remove(e.IBV_WC_EX_WITH_TM_INFO) if attr_ex.raw_packet_caps & e.IBV_RAW_PACKET_CAP_CVLAN_STRIPPING == 0: wc_flags.remove(e.IBV_WC_EX_WITH_CVLAN) sample = u.sample(wc_flags) From patchwork Mon Sep 9 09:07:04 2019 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Noa Osherovich X-Patchwork-Id: 11137505 Return-Path: Received: from mail.kernel.org (pdx-korg-mail-1.web.codeaurora.org [172.30.200.123]) by pdx-korg-patchwork-2.web.codeaurora.org (Postfix) with ESMTP id 9CED176 for ; Mon, 9 Sep 2019 09:07:23 +0000 (UTC) Received: from vger.kernel.org (vger.kernel.org [209.132.180.67]) by mail.kernel.org (Postfix) with ESMTP id 7BAF92196E for ; Mon, 9 Sep 2019 09:07:23 +0000 (UTC) Received: (majordomo@vger.kernel.org) by vger.kernel.org via listexpand id S2389611AbfIIJHV (ORCPT ); Mon, 9 Sep 2019 05:07:21 -0400 Received: from mail-il-dmz.mellanox.com ([193.47.165.129]:48364 "EHLO mellanox.co.il" rhost-flags-OK-OK-OK-FAIL) by vger.kernel.org with ESMTP id S2389597AbfIIJHV (ORCPT ); Mon, 9 Sep 2019 05:07:21 -0400 Received: from Internal Mail-Server by MTLPINE1 (envelope-from noaos@mellanox.com) with ESMTPS (AES256-SHA encrypted); 9 Sep 2019 12:07:16 +0300 Received: from reg-l-vrt-059-007.mtl.labs.mlnx (reg-l-vrt-059-007.mtl.labs.mlnx [10.135.59.7]) by labmailer.mlnx (8.13.8/8.13.8) with ESMTP id x8997Fp2028426; Mon, 9 Sep 2019 12:07:16 +0300 From: Noa Osherovich To: dledford@redhat.com, jgg@mellanox.com, leonro@mellanox.com Cc: linux-rdma@vger.kernel.org, Maxim Chicherin Subject: [PATCH rdma-core 04/12] pyverbs: Introducing XRCD class Date: Mon, 9 Sep 2019 12:07:04 +0300 Message-Id: <20190909090712.11029-5-noaos@mellanox.com> X-Mailer: git-send-email 2.21.0 In-Reply-To: <20190909090712.11029-1-noaos@mellanox.com> References: <20190909090712.11029-1-noaos@mellanox.com> MIME-Version: 1.0 Sender: linux-rdma-owner@vger.kernel.org Precedence: bulk List-ID: X-Mailing-List: linux-rdma@vger.kernel.org From: Maxim Chicherin Represents ibv_xrcd, The eXtended RC Domain. Signed-off-by: Maxim Chicherin --- pyverbs/CMakeLists.txt | 1 + pyverbs/device.pxd | 1 + pyverbs/device.pyx | 7 ++- pyverbs/libibverbs.pxd | 8 ++++ pyverbs/libibverbs_enums.pxd | 5 +++ pyverbs/qp.pxd | 2 + pyverbs/qp.pyx | 25 ++++++++--- pyverbs/xrcd.pxd | 16 +++++++ pyverbs/xrcd.pyx | 87 ++++++++++++++++++++++++++++++++++++ 9 files changed, 145 insertions(+), 7 deletions(-) mode change 100644 => 100755 pyverbs/CMakeLists.txt mode change 100644 => 100755 pyverbs/libibverbs.pxd create mode 100755 pyverbs/xrcd.pxd create mode 100755 pyverbs/xrcd.pyx diff --git a/pyverbs/CMakeLists.txt b/pyverbs/CMakeLists.txt old mode 100644 new mode 100755 index da49093c2cf0..1addf10305c0 --- a/pyverbs/CMakeLists.txt +++ b/pyverbs/CMakeLists.txt @@ -11,6 +11,7 @@ rdma_cython_module(pyverbs pd.pyx qp.pyx wr.pyx + xrcd.pyx ) rdma_python_module(pyverbs diff --git a/pyverbs/device.pxd b/pyverbs/device.pxd index 44c8bc3cbcbc..ed2f1ca3d6ef 100644 --- a/pyverbs/device.pxd +++ b/pyverbs/device.pxd @@ -16,6 +16,7 @@ cdef class Context(PyverbsCM): cdef object ccs cdef object cqs cdef object qps + cdef object xrcds cdef class DeviceAttr(PyverbsObject): cdef v.ibv_device_attr dev_attr diff --git a/pyverbs/device.pyx b/pyverbs/device.pyx index cbba8837c283..f238d37ce3a9 100644 --- a/pyverbs/device.pyx +++ b/pyverbs/device.pyx @@ -14,6 +14,7 @@ from .pyverbs_error import PyverbsUserError from pyverbs.base import PyverbsRDMAErrno cimport pyverbs.libibverbs_enums as e cimport pyverbs.libibverbs as v +from pyverbs.xrcd cimport XRCD from pyverbs.addr cimport GID from pyverbs.mr import DMMR from pyverbs.pd cimport PD @@ -90,6 +91,7 @@ cdef class Context(PyverbsCM): self.ccs = weakref.WeakSet() self.cqs = weakref.WeakSet() self.qps = weakref.WeakSet() + self.xrcds = weakref.WeakSet() dev_name = kwargs.get('name') @@ -126,7 +128,8 @@ cdef class Context(PyverbsCM): cpdef close(self): self.logger.debug('Closing Context') - self.close_weakrefs([self.qps, self.ccs, self.cqs, self.dms, self.pds]) + self.close_weakrefs([self.qps, self.ccs, self.cqs, self.dms, self.pds, + self.xrcds]) if self.context != NULL: rc = v.ibv_close_device(self.context) if rc != 0: @@ -198,6 +201,8 @@ cdef class Context(PyverbsCM): self.cqs.add(obj) elif isinstance(obj, QP): self.qps.add(obj) + elif isinstance(obj, XRCD): + self.xrcds.add(obj) else: raise PyverbsError('Unrecognized object type') diff --git a/pyverbs/libibverbs.pxd b/pyverbs/libibverbs.pxd old mode 100644 new mode 100755 index 1aa5844b126e..8a5dd446d847 --- a/pyverbs/libibverbs.pxd +++ b/pyverbs/libibverbs.pxd @@ -341,6 +341,11 @@ cdef extern from 'infiniband/verbs.h': ibv_qp_type qp_type int sq_sig_all + cdef struct ibv_xrcd_init_attr: + uint32_t comp_mask + int fd + int oflags + cdef struct ibv_xrcd: pass @@ -490,3 +495,6 @@ cdef extern from 'infiniband/verbs.h': int ibv_destroy_qp(ibv_qp *qp) int ibv_post_recv(ibv_qp *qp, ibv_recv_wr *wr, ibv_recv_wr **bad_wr) int ibv_post_send(ibv_qp *qp, ibv_send_wr *wr, ibv_send_wr **bad_wr) + ibv_xrcd *ibv_open_xrcd(ibv_context *context, + ibv_xrcd_init_attr *xrcd_init_attr) + int ibv_close_xrcd(ibv_xrcd *xrcd) diff --git a/pyverbs/libibverbs_enums.pxd b/pyverbs/libibverbs_enums.pxd index 1d437240a883..7d657b332324 100755 --- a/pyverbs/libibverbs_enums.pxd +++ b/pyverbs/libibverbs_enums.pxd @@ -393,6 +393,11 @@ cdef extern from '': IBV_RAW_PACKET_CAP_IP_CSUM = 1 << 2 IBV_RAW_PACKET_CAP_DELAY_DROP = 1 << 3 + cpdef enum ibv_xrcd_init_attr_mask: + IBV_XRCD_INIT_ATTR_FD = 1 << 0 + IBV_XRCD_INIT_ATTR_OFLAGS = 1 << 1 + IBV_XRCD_INIT_ATTR_RESERVED = 1 << 2 + cdef unsigned long long IBV_DEVICE_RAW_SCATTER_FCS cdef unsigned long long IBV_DEVICE_PCI_WRITE_END_PADDING diff --git a/pyverbs/qp.pxd b/pyverbs/qp.pxd index 29b9ec4a0221..a5c92ea673e5 100644 --- a/pyverbs/qp.pxd +++ b/pyverbs/qp.pxd @@ -18,6 +18,7 @@ cdef class QPInitAttrEx(PyverbsObject): cdef v.ibv_qp_init_attr_ex attr cdef object scq cdef object rcq + cdef object xrcd cdef object pd cdef class QPAttr(PyverbsObject): @@ -29,6 +30,7 @@ cdef class QP(PyverbsCM): cdef int state cdef object pd cdef object context + cdef object xrcd cpdef close(self) cdef update_cqs(self, init_attr) cdef object scq diff --git a/pyverbs/qp.pyx b/pyverbs/qp.pyx index 732ef5094b6b..a36d5fe9d569 100755 --- a/pyverbs/qp.pyx +++ b/pyverbs/qp.pyx @@ -11,6 +11,7 @@ from pyverbs.addr cimport GlobalRoute from pyverbs.device cimport Context from pyverbs.cq cimport CQ, CQEX cimport pyverbs.libibverbs as v +from pyverbs.xrcd cimport XRCD from pyverbs.pd cimport PD @@ -193,7 +194,7 @@ cdef class QPInitAttrEx(PyverbsObject): def __cinit__(self, qp_type=e.IBV_QPT_UD, qp_context=None, PyverbsObject scq=None, PyverbsObject rcq=None, object srq=None, QPCap cap=None, sq_sig_all=0, comp_mask=0, - PD pd=None, object xrcd=None, create_flags=0, + PD pd=None, XRCD xrcd=None, create_flags=0, max_tso_header=0, source_qpn=0, object hash_conf=None, object ind_table=None): """ @@ -209,7 +210,7 @@ cdef class QPInitAttrEx(PyverbsObject): :param comp_mask: bit mask to determine which of the following fields are valid :param pd: A PD object to be associated with this QP - :param xrcd: Not yet supported + :param xrcd: XRC domain to be used for XRC QPs :param create_flags: Creation flags for this QP :param max_tso_header: Maximum TSO header size :param source_qpn: Source QP number (requires IBV_QP_CREATE_SOURCE_QPN @@ -240,14 +241,14 @@ cdef class QPInitAttrEx(PyverbsObject): self.rcq = rcq self.attr.srq = NULL # Until SRQ support is added - self.attr.xrcd = NULL # Until XRCD support is added + self.xrcd = xrcd + self.attr.xrcd = xrcd.xrcd if xrcd else NULL self.attr.rwq_ind_tbl = NULL # Until RSS support is added self.attr.qp_type = qp_type self.attr.sq_sig_all = sq_sig_all - unsupp_flags = e.IBV_QP_INIT_ATTR_XRCD | e.IBV_QP_INIT_ATTR_IND_TABLE |\ - e.IBV_QP_INIT_ATTR_RX_HASH + unsupp_flags = e.IBV_QP_INIT_ATTR_IND_TABLE | e.IBV_QP_INIT_ATTR_RX_HASH if comp_mask & unsupp_flags: - raise PyverbsUserError('XRCD and RSS are not yet supported in pyverbs') + raise PyverbsUserError('RSS is not yet supported in pyverbs') self.attr.comp_mask = comp_mask if pd is not None: self.pd = pd @@ -318,6 +319,14 @@ cdef class QPInitAttrEx(PyverbsObject): self.attr.pd = val.pd self.pd = val + @property + def xrcd(self): + return self.xrcd + @xrcd.setter + def xrcd(self, XRCD val): + self.attr.xrcd = val.xrcd + self.xrcd = val + @property def create_flags(self): return self.attr.create_flags @@ -794,6 +803,10 @@ cdef class QP(PyverbsCM): pd = init_attr.pd pd.add_ref(self) self.pd = pd + if init_attr.xrcd is not None: + xrcd = init_attr.xrcd + xrcd.add_ref(self) + self.xrcd = xrcd else: self._create_qp(creator, init_attr) pd = creator diff --git a/pyverbs/xrcd.pxd b/pyverbs/xrcd.pxd new file mode 100755 index 000000000000..ab28dfafe1d1 --- /dev/null +++ b/pyverbs/xrcd.pxd @@ -0,0 +1,16 @@ +# SPDX-License-Identifier: (GPL-2.0 OR Linux-OpenIB) +# Copyright (c) 2019, Mellanox Technologies. All rights reserved. +from pyverbs.base cimport PyverbsCM, PyverbsObject +from pyverbs.device cimport Context +cimport pyverbs.libibverbs as v + + +cdef class XRCDInitAttr(PyverbsObject): + cdef v.ibv_xrcd_init_attr attr + + +cdef class XRCD(PyverbsCM): + cdef v.ibv_xrcd *xrcd + cdef Context ctx + cdef add_ref(self, obj) + cdef object qps diff --git a/pyverbs/xrcd.pyx b/pyverbs/xrcd.pyx new file mode 100755 index 000000000000..83d2a6b83e67 --- /dev/null +++ b/pyverbs/xrcd.pyx @@ -0,0 +1,87 @@ +# SPDX-License-Identifier: (GPL-2.0 OR Linux-OpenIB) +# Copyright (c) 2019, Mellanox Technologies. All rights reserved. +import weakref + +from pyverbs.pyverbs_error import PyverbsRDMAError, PyverbsError +from pyverbs.base import PyverbsRDMAErrno +from pyverbs.device cimport Context +from pyverbs.qp cimport QP + +cdef extern from 'errno.h': + int errno + + +cdef class XRCDInitAttr(PyverbsObject): + def __cinit__(self, comp_mask, oflags, fd): + self.attr.fd = fd + self.attr.comp_mask = comp_mask + self.attr.oflags = oflags + + @property + def fd(self): + return self.attr.fd + @fd.setter + def fd(self, val): + self.attr.fd = val + + @property + def comp_mask(self): + return self.attr.comp_mask + @comp_mask.setter + def comp_mask(self, val): + self.attr.comp_mask = val + + @property + def oflags(self): + return self.attr.oflags + @oflags.setter + def oflags(self, val): + self.attr.oflags = val + + +cdef class XRCD(PyverbsCM): + def __cinit__(self, Context context not None, XRCDInitAttr init_attr not None): + """ + Initializes a XRCD object. + :param context: The Context object creating the XRCD + :return: The newly created XRCD on success + """ + self.xrcd = v.ibv_open_xrcd( context.context, + &init_attr.attr) + if self.xrcd == NULL: + raise PyverbsRDMAErrno('Failed to allocate XRCD', errno) + self.ctx = context + context.add_ref(self) + self.logger.debug('XRCD: Allocated ibv_xrcd') + self.qps = weakref.WeakSet() + + def __dealloc__(self): + """ + Closes the inner XRCD. + :return: None + """ + self.close() + + cpdef close(self): + """ + Closes the underlying C object of the XRCD. + :return: None + """ + self.logger.debug('Closing XRCD') + self.close_weakrefs([self.qps]) + # XRCD may be deleted directly or indirectly by closing its context, + # which leaves the Python XRCD object without the underlying C object, + # so during destruction, need to check whether or not the C object + # exists. + if self.xrcd != NULL: + rc = v.ibv_close_xrcd(self.xrcd) + if rc != 0: + raise PyverbsRDMAErrno('Failed to dealloc XRCD') + self.xrcd = NULL + self.ctx = None + + cdef add_ref(self, obj): + if isinstance(obj, QP): + self.qps.add(obj) + else: + raise PyverbsError('Unrecognized object type') From patchwork Mon Sep 9 09:07:05 2019 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Noa Osherovich X-Patchwork-Id: 11137497 Return-Path: Received: from mail.kernel.org (pdx-korg-mail-1.web.codeaurora.org [172.30.200.123]) by pdx-korg-patchwork-2.web.codeaurora.org (Postfix) with ESMTP id 6AE9776 for ; Mon, 9 Sep 2019 09:07:22 +0000 (UTC) Received: from vger.kernel.org (vger.kernel.org [209.132.180.67]) by mail.kernel.org (Postfix) with ESMTP id 2D39821924 for ; Mon, 9 Sep 2019 09:07:22 +0000 (UTC) Received: (majordomo@vger.kernel.org) by vger.kernel.org via listexpand id S2389613AbfIIJHV (ORCPT ); Mon, 9 Sep 2019 05:07:21 -0400 Received: from mail-il-dmz.mellanox.com ([193.47.165.129]:48363 "EHLO mellanox.co.il" rhost-flags-OK-OK-OK-FAIL) by vger.kernel.org with ESMTP id S2389608AbfIIJHU (ORCPT ); Mon, 9 Sep 2019 05:07:20 -0400 Received: from Internal Mail-Server by MTLPINE1 (envelope-from noaos@mellanox.com) with ESMTPS (AES256-SHA encrypted); 9 Sep 2019 12:07:16 +0300 Received: from reg-l-vrt-059-007.mtl.labs.mlnx (reg-l-vrt-059-007.mtl.labs.mlnx [10.135.59.7]) by labmailer.mlnx (8.13.8/8.13.8) with ESMTP id x8997Fp3028426; Mon, 9 Sep 2019 12:07:16 +0300 From: Noa Osherovich To: dledford@redhat.com, jgg@mellanox.com, leonro@mellanox.com Cc: linux-rdma@vger.kernel.org, Maxim Chicherin , Noa Osherovich Subject: [PATCH rdma-core 05/12] pyverbs: Introducing SRQ class Date: Mon, 9 Sep 2019 12:07:05 +0300 Message-Id: <20190909090712.11029-6-noaos@mellanox.com> X-Mailer: git-send-email 2.21.0 In-Reply-To: <20190909090712.11029-1-noaos@mellanox.com> References: <20190909090712.11029-1-noaos@mellanox.com> MIME-Version: 1.0 Sender: linux-rdma-owner@vger.kernel.org Precedence: bulk List-ID: X-Mailing-List: linux-rdma@vger.kernel.org From: Maxim Chicherin SRQ class represents ibv_srq. SRQ is a shared receive queue which can be used by multiple QPs. Classes SrqInitAttrEx, SrqInitAttr and SrqAttr have been added too. Note: SRQ XRC is suppored too. Messages arriving at receive ports of XRC QPs will be channeled to SRQ only if the XRC Domain of the SRQ matches the XRC Domain of the transport QP. Signed-off-by: Maxim Chicherin Signed-off-by: Noa Osherovich --- pyverbs/CMakeLists.txt | 1 + pyverbs/cq.pxd | 2 + pyverbs/cq.pyx | 13 ++- pyverbs/libibverbs.pxd | 35 ++++++++ pyverbs/pd.pxd | 1 + pyverbs/pd.pyx | 6 +- pyverbs/qp.pxd | 2 + pyverbs/qp.pyx | 32 ++++++-- pyverbs/srq.pxd | 24 ++++++ pyverbs/srq.pyx | 176 +++++++++++++++++++++++++++++++++++++++++ pyverbs/xrcd.pxd | 1 + pyverbs/xrcd.pyx | 6 +- 12 files changed, 289 insertions(+), 10 deletions(-) create mode 100755 pyverbs/srq.pxd create mode 100755 pyverbs/srq.pyx diff --git a/pyverbs/CMakeLists.txt b/pyverbs/CMakeLists.txt index 1addf10305c0..90293982b280 100755 --- a/pyverbs/CMakeLists.txt +++ b/pyverbs/CMakeLists.txt @@ -12,6 +12,7 @@ rdma_cython_module(pyverbs qp.pyx wr.pyx xrcd.pyx + srq.pyx ) rdma_python_module(pyverbs diff --git a/pyverbs/cq.pxd b/pyverbs/cq.pxd index 9b8df5dcae39..8eeb2e1fd0c2 100644 --- a/pyverbs/cq.pxd +++ b/pyverbs/cq.pxd @@ -19,6 +19,7 @@ cdef class CQ(PyverbsCM): cdef object context cdef add_ref(self, obj) cdef object qps + cdef object srqs cdef class CqInitAttrEx(PyverbsObject): cdef v.ibv_cq_init_attr_ex attr @@ -31,6 +32,7 @@ cdef class CQEX(PyverbsCM): cdef object context cdef add_ref(self, obj) cdef object qps + cdef object srqs cdef class WC(PyverbsObject): cdef v.ibv_wc wc diff --git a/pyverbs/cq.pyx b/pyverbs/cq.pyx index 32eee0a0f1fd..eadc1fdb3140 100755 --- a/pyverbs/cq.pyx +++ b/pyverbs/cq.pyx @@ -2,9 +2,11 @@ # Copyright (c) 2019, Mellanox Technologies. All rights reserved. import weakref +from pyverbs.pyverbs_error import PyverbsError from pyverbs.base import PyverbsRDMAErrno cimport pyverbs.libibverbs_enums as e from pyverbs.device cimport Context +from pyverbs.srq cimport SRQ from pyverbs.qp cimport QP cdef class CompChannel(PyverbsCM): @@ -90,18 +92,23 @@ cdef class CQ(PyverbsCM): self.context = context context.add_ref(self) self.qps = weakref.WeakSet() + self.srqs = weakref.WeakSet() self.logger.debug('Created a CQ') cdef add_ref(self, obj): if isinstance(obj, QP): self.qps.add(obj) + elif isinstance(obj, SRQ): + self.srqs.add(obj) + else: + raise PyverbsError('Unrecognized object type') def __dealloc__(self): self.close() cpdef close(self): self.logger.debug('Closing CQ') - self.close_weakrefs([self.qps]) + self.close_weakrefs([self.qps, self.srqs]) if self.cq != NULL: rc = v.ibv_destroy_cq(self.cq) if rc != 0: @@ -270,6 +277,10 @@ cdef class CQEX(PyverbsCM): cdef add_ref(self, obj): if isinstance(obj, QP): self.qps.add(obj) + elif isinstance(obj, SRQ): + self.srqs.add(obj) + else: + raise PyverbsError('Unrecognized object type') def __dealloc__(self): self.close() diff --git a/pyverbs/libibverbs.pxd b/pyverbs/libibverbs.pxd index 8a5dd446d847..a92286d42c67 100755 --- a/pyverbs/libibverbs.pxd +++ b/pyverbs/libibverbs.pxd @@ -349,6 +349,32 @@ cdef extern from 'infiniband/verbs.h': cdef struct ibv_xrcd: pass + cdef struct ibv_srq_attr: + unsigned int max_wr + unsigned int max_sge + unsigned int srq_limit + + cdef struct ibv_srq_init_attr: + void *srq_context + ibv_srq_attr attr + + cdef struct ibv_srq_init_attr_ex: + void *srq_context + ibv_srq_attr attr + unsigned int comp_mask + ibv_srq_type srq_type + ibv_pd *pd + ibv_xrcd *xrcd + ibv_cq *cq + ibv_tm_caps tm_cap + + cdef struct ibv_srq: + ibv_context *context + void *srq_context + ibv_pd *pd + unsigned int handle + unsigned int events_completed + cdef struct ibv_rwq_ind_table: pass @@ -498,3 +524,12 @@ cdef extern from 'infiniband/verbs.h': ibv_xrcd *ibv_open_xrcd(ibv_context *context, ibv_xrcd_init_attr *xrcd_init_attr) int ibv_close_xrcd(ibv_xrcd *xrcd) + ibv_srq *ibv_create_srq(ibv_pd *pd, ibv_srq_init_attr *srq_init_attr) + ibv_srq *ibv_create_srq_ex(ibv_context *context, + ibv_srq_init_attr_ex *srq_init_attr) + int ibv_modify_srq(ibv_srq *srq, ibv_srq_attr *srq_attr, int srq_attr_mask) + int ibv_query_srq(ibv_srq *srq, ibv_srq_attr *srq_attr) + int ibv_get_srq_num(ibv_srq *srq, unsigned int *srq_num) + int ibv_destroy_srq(ibv_srq *srq) + int ibv_post_srq_recv(ibv_srq *srq, ibv_recv_wr *recv_wr, + ibv_recv_wr **bad_recv_wr) diff --git a/pyverbs/pd.pxd b/pyverbs/pd.pxd index e0861b301b7c..6dd9c2959ed3 100644 --- a/pyverbs/pd.pxd +++ b/pyverbs/pd.pxd @@ -12,6 +12,7 @@ cdef class PD(PyverbsCM): cdef v.ibv_pd *pd cdef Context ctx cdef add_ref(self, obj) + cdef object srqs cdef object mrs cdef object mws cdef object ahs diff --git a/pyverbs/pd.pyx b/pyverbs/pd.pyx index 7cd0876682b2..46cbb36009ce 100644 --- a/pyverbs/pd.pyx +++ b/pyverbs/pd.pyx @@ -6,6 +6,7 @@ from pyverbs.pyverbs_error import PyverbsRDMAError, PyverbsError from pyverbs.base import PyverbsRDMAErrno from pyverbs.device cimport Context, DM from .mr cimport MR, MW, DMMR +from pyverbs.srq cimport SRQ from pyverbs.addr cimport AH from pyverbs.qp cimport QP @@ -27,6 +28,7 @@ cdef class PD(PyverbsCM): self.ctx = context context.add_ref(self) self.logger.debug('PD: Allocated ibv_pd') + self.srqs = weakref.WeakSet() self.mrs = weakref.WeakSet() self.mws = weakref.WeakSet() self.ahs = weakref.WeakSet() @@ -48,7 +50,7 @@ cdef class PD(PyverbsCM): :return: None """ self.logger.debug('Closing PD') - self.close_weakrefs([self.qps, self.ahs, self.mws, self.mrs]) + self.close_weakrefs([self.qps, self.ahs, self.mws, self.mrs, self.srqs]) if self.pd != NULL: rc = v.ibv_dealloc_pd(self.pd) if rc != 0: @@ -65,5 +67,7 @@ cdef class PD(PyverbsCM): self.ahs.add(obj) elif isinstance(obj, QP): self.qps.add(obj) + elif isinstance(obj, SRQ): + self.srqs.add(obj) else: raise PyverbsError('Unrecognized object type') diff --git a/pyverbs/qp.pxd b/pyverbs/qp.pxd index a5c92ea673e5..55503702cfe3 100644 --- a/pyverbs/qp.pxd +++ b/pyverbs/qp.pxd @@ -13,12 +13,14 @@ cdef class QPInitAttr(PyverbsObject): cdef v.ibv_qp_init_attr attr cdef object scq cdef object rcq + cdef object srq cdef class QPInitAttrEx(PyverbsObject): cdef v.ibv_qp_init_attr_ex attr cdef object scq cdef object rcq cdef object xrcd + cdef object srq cdef object pd cdef class QPAttr(PyverbsObject): diff --git a/pyverbs/qp.pyx b/pyverbs/qp.pyx index a36d5fe9d569..0417625e4b6c 100755 --- a/pyverbs/qp.pyx +++ b/pyverbs/qp.pyx @@ -12,6 +12,7 @@ from pyverbs.device cimport Context from pyverbs.cq cimport CQ, CQEX cimport pyverbs.libibverbs as v from pyverbs.xrcd cimport XRCD +from pyverbs.srq cimport SRQ from pyverbs.pd cimport PD @@ -86,7 +87,7 @@ cdef class QPCap(PyverbsObject): cdef class QPInitAttr(PyverbsObject): def __cinit__(self, qp_type=e.IBV_QPT_UD, qp_context=None, PyverbsObject scq=None, PyverbsObject rcq=None, - object srq=None, QPCap cap=None, sq_sig_all=1): + SRQ srq=None, QPCap cap=None, sq_sig_all=1): """ Initializes a QpInitAttr object representing ibv_qp_init_attr struct. Note that SRQ object is not yet supported in pyverbs so can't be passed @@ -95,7 +96,7 @@ cdef class QPInitAttr(PyverbsObject): :param qp_context: Associated QP context :param scq: Send CQ to be used for this QP :param rcq: Receive CQ to be used for this QP - :param srq: Not yet supported + :param srq: Shared receive queue to be used as RQ in QP :param cap: A QPCap object :param sq_sig_all: If set, each send WR will generate a completion entry @@ -122,10 +123,10 @@ cdef class QPInitAttr(PyverbsObject): raise PyverbsUserError('Expected CQ/CQEX, got {t}'.\ format(t=type(rcq))) self.rcq = rcq - - self.attr.srq = NULL # Until SRQ support is added self.attr.qp_type = qp_type self.attr.sq_sig_all = sq_sig_all + self.srq = srq + self.attr.srq = srq.srq if srq else NULL @property def send_cq(self): @@ -138,6 +139,14 @@ cdef class QPInitAttr(PyverbsObject): self.attr.send_cq = (val).ibv_cq self.scq = val + @property + def srq(self): + return self.srq + @srq.setter + def srq(self, SRQ val): + self.attr.srq = val.srq + self.srq = val + @property def recv_cq(self): return self.rcq @@ -193,7 +202,7 @@ cdef class QPInitAttr(PyverbsObject): cdef class QPInitAttrEx(PyverbsObject): def __cinit__(self, qp_type=e.IBV_QPT_UD, qp_context=None, PyverbsObject scq=None, PyverbsObject rcq=None, - object srq=None, QPCap cap=None, sq_sig_all=0, comp_mask=0, + SRQ srq=None, QPCap cap=None, sq_sig_all=0, comp_mask=0, PD pd=None, XRCD xrcd=None, create_flags=0, max_tso_header=0, source_qpn=0, object hash_conf=None, object ind_table=None): @@ -203,7 +212,7 @@ cdef class QPInitAttrEx(PyverbsObject): :param qp_context: Associated user context :param scq: Send CQ to be used for this QP :param rcq: Recv CQ to be used for this QP - :param srq: Not yet supported + :param srq: Shared receive queue to be used as RQ in QP :param cap: A QPCap object :param sq_sig_all: If set, each send WR will generate a completion entry @@ -240,7 +249,8 @@ cdef class QPInitAttrEx(PyverbsObject): format(t=type(rcq))) self.rcq = rcq - self.attr.srq = NULL # Until SRQ support is added + self.srq = srq + self.attr.srq = srq.srq if srq else NULL self.xrcd = xrcd self.attr.xrcd = xrcd.xrcd if xrcd else NULL self.attr.rwq_ind_tbl = NULL # Until RSS support is added @@ -327,6 +337,14 @@ cdef class QPInitAttrEx(PyverbsObject): self.attr.xrcd = val.xrcd self.xrcd = val + @property + def srq(self): + return self.srq + @srq.setter + def srq(self, SRQ val): + self.attr.srq = val.srq + self.srq = val + @property def create_flags(self): return self.attr.create_flags diff --git a/pyverbs/srq.pxd b/pyverbs/srq.pxd new file mode 100755 index 000000000000..a7b7b34490e9 --- /dev/null +++ b/pyverbs/srq.pxd @@ -0,0 +1,24 @@ +# SPDX-License-Identifier: (GPL-2.0 OR Linux-OpenIB) +# Copyright (c) 2019 Mellanox Technologies, Inc. All rights reserved. + +#cython: language_level=3 + +from pyverbs.base cimport PyverbsObject, PyverbsCM +from . cimport libibverbs as v + +cdef class SrqAttr(PyverbsObject): + cdef v.ibv_srq_attr attr + +cdef class SrqInitAttr(PyverbsObject): + cdef v.ibv_srq_init_attr attr + +cdef class SrqInitAttrEx(PyverbsObject): + cdef v.ibv_srq_init_attr_ex attr + cdef object _cq + cdef object _pd + cdef object _xrcd + +cdef class SRQ(PyverbsCM): + cdef v.ibv_srq *srq + cdef object cq + cpdef close(self) diff --git a/pyverbs/srq.pyx b/pyverbs/srq.pyx new file mode 100755 index 000000000000..1d0d0dc4446b --- /dev/null +++ b/pyverbs/srq.pyx @@ -0,0 +1,176 @@ +from pyverbs.pyverbs_error import PyverbsRDMAError +from pyverbs.base import PyverbsRDMAErrno +from pyverbs.device cimport Context +from pyverbs.cq cimport CQEX, CQ +from pyverbs.xrcd cimport XRCD +from pyverbs.wr cimport RecvWR +from pyverbs.pd cimport PD + + +cdef extern from 'errno.h': + int errno +cdef extern from 'string.h': + void *memcpy(void *dest, const void *src, size_t n) + + +cdef class SrqAttr(PyverbsObject): + def __cinit__(self, max_wr=100, max_sge=1, srq_limit=0): + self.attr.max_wr = max_wr + self.attr.max_sge = max_sge + self.attr.srq_limit = srq_limit + + @property + def max_wr(self): + return self.attr.max_wr + @max_wr.setter + def max_wr(self, val): + self.attr.max_wr = val + + @property + def max_sge(self): + return self.attr.max_sge + @max_sge.setter + def max_sge(self, val): + self.attr.max_sge = val + + @property + def srq_limit(self): + return self.attr.srq_limit + def srq_limit(self, val): + self.attr.srq_limit = val + + +cdef class SrqInitAttr(PyverbsObject): + def __cinit__(self, SrqAttr attr = None): + if attr is not None: + self.attr.attr.max_wr = attr.max_wr + self.attr.attr.max_sge = attr.max_sge + self.attr.attr.srq_limit = attr.srq_limit + + +cdef class SrqInitAttrEx(PyverbsObject): + def __cinit__(self, max_wr=100, max_sge=1, srq_limit=0): + self.attr.attr.max_wr = max_wr + self.attr.attr.max_sge = max_sge + self.attr.attr.srq_limit = srq_limit + self._cq = None + self._pd = None + self._xrcd = None + + @property + def comp_mask(self): + return self.attr.comp_mask + @comp_mask.setter + def comp_mask(self, val): + self.attr.comp_mask = val + + @property + def srq_type(self): + return self.attr.srq_type + @srq_type.setter + def srq_type(self, val): + self.attr.srq_type = val + + @property + def pd(self): + return self._pd + @pd.setter + def pd(self, PD val): + self._pd = val + self.attr.pd = val.pd + + @property + def xrcd(self): + return self._xrcd + @xrcd.setter + def xrcd(self, XRCD val): + self._xrcd = val + self.attr.xrcd = val.xrcd + + @property + def cq(self): + return self._cq + @cq.setter + def cq(self, val): + if type(val) == CQ: + self.attr.cq = (val).cq + self._cq = val + else: + self.attr.cq = (val).ibv_cq + self._cq = val + + +cdef class SRQ(PyverbsCM): + def __cinit__(self, object creator not None, object attr not None): + self.srq = NULL + self.cq = None + if type(creator) == PD: + self._create_srq(creator, attr) + elif type(creator) == Context: + self._create_srq_ex(creator, attr) + else: + raise PyverbsRDMAError('Srq needs either Context or PD for creation') + if self.srq == NULL: + raise PyverbsRDMAErrno('Failed to create SRQ (errno is {err})'. + format(err=errno)) + self.logger.debug('SRQ Created') + + def __dealloc__(self): + self.close() + + cpdef close(self): + self.logger.debug('Closing SRQ') + if self.srq != NULL: + rc = v.ibv_destroy_srq(self.srq) + if rc != 0: + raise PyverbsRDMAErrno('Failed to destroy SRQ (errno is {err})'. + format(err=errno)) + self.srq = NULL + self.cq =None + + def _create_srq(self, PD pd, SrqInitAttr init_attr): + self.srq = v.ibv_create_srq(pd.pd, &init_attr.attr) + + def _create_srq_ex(self, Context context, SrqInitAttrEx init_attr_ex): + self.srq = v.ibv_create_srq_ex(context.context, &init_attr_ex.attr) + if init_attr_ex.cq: + cq = init_attr_ex.cq + cq.add_ref(self) + self.cq = cq + if init_attr_ex.xrcd: + xrcd = init_attr_ex.xrcd + xrcd.add_ref(self) + if init_attr_ex.pd: + pd = init_attr_ex.pd + pd.add_ref(self) + + def get_srq_num(self): + cdef unsigned int srqn + rc = v.ibv_get_srq_num(self.srq, &srqn) + if rc != 0: + raise PyverbsRDMAErrno('Failed to retrieve SRQ number (returned {rc})'. + format(rc=rc)) + return srqn + + def modify(self, SrqAttr attr, comp_mask): + rc = v.ibv_modify_srq(self.srq, &attr.attr, comp_mask) + if rc != 0: + raise PyverbsRDMAErrno('Failed to modify SRQ ({err})'. + format(err=errno)) + + def query(self): + attr = SrqAttr() + rc = v.ibv_query_srq(self.srq, &attr.attr) + if rc != 0: + raise PyverbsRDMAErrno('Failed to query SRQ ({err})'. + format(err=errno)) + return attr + + def post_recv(self, RecvWR wr not None, RecvWR bad_wr=None): + cdef v.ibv_recv_wr *my_bad_wr + rc = v.ibv_post_srq_recv(self.srq, &wr.recv_wr, &my_bad_wr) + if rc != 0: + if bad_wr: + memcpy(&bad_wr.recv_wr, my_bad_wr, sizeof(bad_wr.recv_wr)) + raise PyverbsRDMAErrno('Failed to post receive to SRQ ({err})'. + format(err=rc)) diff --git a/pyverbs/xrcd.pxd b/pyverbs/xrcd.pxd index ab28dfafe1d1..4c1c6c042764 100755 --- a/pyverbs/xrcd.pxd +++ b/pyverbs/xrcd.pxd @@ -13,4 +13,5 @@ cdef class XRCD(PyverbsCM): cdef v.ibv_xrcd *xrcd cdef Context ctx cdef add_ref(self, obj) + cdef object srqs cdef object qps diff --git a/pyverbs/xrcd.pyx b/pyverbs/xrcd.pyx index 83d2a6b83e67..51a044ca7494 100755 --- a/pyverbs/xrcd.pyx +++ b/pyverbs/xrcd.pyx @@ -5,6 +5,7 @@ import weakref from pyverbs.pyverbs_error import PyverbsRDMAError, PyverbsError from pyverbs.base import PyverbsRDMAErrno from pyverbs.device cimport Context +from pyverbs.srq cimport SRQ from pyverbs.qp cimport QP cdef extern from 'errno.h': @@ -53,6 +54,7 @@ cdef class XRCD(PyverbsCM): self.ctx = context context.add_ref(self) self.logger.debug('XRCD: Allocated ibv_xrcd') + self.srqs = weakref.WeakSet() self.qps = weakref.WeakSet() def __dealloc__(self): @@ -68,7 +70,7 @@ cdef class XRCD(PyverbsCM): :return: None """ self.logger.debug('Closing XRCD') - self.close_weakrefs([self.qps]) + self.close_weakrefs([self.qps, self.srqs]) # XRCD may be deleted directly or indirectly by closing its context, # which leaves the Python XRCD object without the underlying C object, # so during destruction, need to check whether or not the C object @@ -83,5 +85,7 @@ cdef class XRCD(PyverbsCM): cdef add_ref(self, obj): if isinstance(obj, QP): self.qps.add(obj) + elif isinstance(obj, SRQ): + self.srqs.add(obj) else: raise PyverbsError('Unrecognized object type') From patchwork Mon Sep 9 09:07:06 2019 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Noa Osherovich X-Patchwork-Id: 11137491 Return-Path: Received: from mail.kernel.org (pdx-korg-mail-1.web.codeaurora.org [172.30.200.123]) by pdx-korg-patchwork-2.web.codeaurora.org (Postfix) with ESMTP id EDEF418AC for ; Mon, 9 Sep 2019 09:07:20 +0000 (UTC) Received: from vger.kernel.org (vger.kernel.org [209.132.180.67]) by mail.kernel.org (Postfix) with ESMTP id CB2C721924 for ; Mon, 9 Sep 2019 09:07:20 +0000 (UTC) Received: (majordomo@vger.kernel.org) by vger.kernel.org via listexpand id S2389616AbfIIJHU (ORCPT ); Mon, 9 Sep 2019 05:07:20 -0400 Received: from mail-il-dmz.mellanox.com ([193.47.165.129]:48366 "EHLO mellanox.co.il" rhost-flags-OK-OK-OK-FAIL) by vger.kernel.org with ESMTP id S2389596AbfIIJHT (ORCPT ); Mon, 9 Sep 2019 05:07:19 -0400 Received: from Internal Mail-Server by MTLPINE1 (envelope-from noaos@mellanox.com) with ESMTPS (AES256-SHA encrypted); 9 Sep 2019 12:07:16 +0300 Received: from reg-l-vrt-059-007.mtl.labs.mlnx (reg-l-vrt-059-007.mtl.labs.mlnx [10.135.59.7]) by labmailer.mlnx (8.13.8/8.13.8) with ESMTP id x8997Fp4028426; Mon, 9 Sep 2019 12:07:16 +0300 From: Noa Osherovich To: dledford@redhat.com, jgg@mellanox.com, leonro@mellanox.com Cc: linux-rdma@vger.kernel.org, Maxim Chicherin Subject: [PATCH rdma-core 06/12] pyverbs: Support XRC QPs when modifying QP states Date: Mon, 9 Sep 2019 12:07:06 +0300 Message-Id: <20190909090712.11029-7-noaos@mellanox.com> X-Mailer: git-send-email 2.21.0 In-Reply-To: <20190909090712.11029-1-noaos@mellanox.com> References: <20190909090712.11029-1-noaos@mellanox.com> MIME-Version: 1.0 Sender: linux-rdma-owner@vger.kernel.org Precedence: bulk List-ID: X-Mailing-List: linux-rdma@vger.kernel.org From: Maxim Chicherin Add the needed support to allow usage of to_ methods for XRC QP types. Signed-off-by: Maxim Chicherin --- pyverbs/qp.pyx | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/pyverbs/qp.pyx b/pyverbs/qp.pyx index 0417625e4b6c..d9770608663c 100755 --- a/pyverbs/qp.pyx +++ b/pyverbs/qp.pyx @@ -836,6 +836,8 @@ cdef class QP(PyverbsCM): if qp_attr is not None: funcs = {e.IBV_QPT_RC: self.to_init, e.IBV_QPT_UC: self.to_init, e.IBV_QPT_UD: self.to_rts, + e.IBV_QPT_XRC_RECV: self.to_init, + e.IBV_QPT_XRC_SEND: self.to_init, e.IBV_QPT_RAW_PACKET: self.to_rts} funcs[self.qp.qp_type](qp_attr) @@ -899,7 +901,22 @@ cdef class QP(PyverbsCM): e.IBV_QP_QKEY, 'RTR': 0, 'RTS': e.IBV_QP_SQ_PSN}, e.IBV_QPT_RAW_PACKET: {'INIT': e.IBV_QP_PORT, 'RTR': 0, - 'RTS': 0}} + 'RTS': 0}, + e.IBV_QPT_XRC_RECV: {'INIT': e.IBV_QP_PKEY_INDEX |\ + e.IBV_QP_PORT | e.IBV_QP_ACCESS_FLAGS, + 'RTR': e.IBV_QP_AV | e.IBV_QP_PATH_MTU |\ + e.IBV_QP_DEST_QPN | e.IBV_QP_RQ_PSN | \ + e.IBV_QP_MAX_DEST_RD_ATOMIC |\ + e.IBV_QP_MIN_RNR_TIMER, + 'RTS': e.IBV_QP_TIMEOUT | e.IBV_QP_SQ_PSN }, + e.IBV_QPT_XRC_SEND: {'INIT': e.IBV_QP_PKEY_INDEX |\ + e.IBV_QP_PORT | e.IBV_QP_ACCESS_FLAGS, + 'RTR': e.IBV_QP_AV | e.IBV_QP_PATH_MTU |\ + e.IBV_QP_DEST_QPN | e.IBV_QP_RQ_PSN, + 'RTS': e.IBV_QP_TIMEOUT |\ + e.IBV_QP_RETRY_CNT | e.IBV_QP_RNR_RETRY |\ + e.IBV_QP_SQ_PSN | e.IBV_QP_MAX_QP_RD_ATOMIC}} + return masks[self.qp.qp_type][dst] | e.IBV_QP_STATE def to_init(self, QPAttr qp_attr): From patchwork Mon Sep 9 09:07:07 2019 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Noa Osherovich X-Patchwork-Id: 11137507 Return-Path: Received: from mail.kernel.org (pdx-korg-mail-1.web.codeaurora.org [172.30.200.123]) by pdx-korg-patchwork-2.web.codeaurora.org (Postfix) with ESMTP id 3AB841599 for ; Mon, 9 Sep 2019 09:07:25 +0000 (UTC) Received: from vger.kernel.org (vger.kernel.org [209.132.180.67]) by mail.kernel.org (Postfix) with ESMTP id 254BD21924 for ; Mon, 9 Sep 2019 09:07:25 +0000 (UTC) Received: (majordomo@vger.kernel.org) by vger.kernel.org via listexpand id S2389628AbfIIJHY (ORCPT ); Mon, 9 Sep 2019 05:07:24 -0400 Received: from mail-il-dmz.mellanox.com ([193.47.165.129]:48365 "EHLO mellanox.co.il" rhost-flags-OK-OK-OK-FAIL) by vger.kernel.org with ESMTP id S2389623AbfIIJHT (ORCPT ); Mon, 9 Sep 2019 05:07:19 -0400 Received: from Internal Mail-Server by MTLPINE1 (envelope-from noaos@mellanox.com) with ESMTPS (AES256-SHA encrypted); 9 Sep 2019 12:07:16 +0300 Received: from reg-l-vrt-059-007.mtl.labs.mlnx (reg-l-vrt-059-007.mtl.labs.mlnx [10.135.59.7]) by labmailer.mlnx (8.13.8/8.13.8) with ESMTP id x8997Fp5028426; Mon, 9 Sep 2019 12:07:16 +0300 From: Noa Osherovich To: dledford@redhat.com, jgg@mellanox.com, leonro@mellanox.com Cc: linux-rdma@vger.kernel.org, Maxim Chicherin Subject: [PATCH rdma-core 07/12] pyverbs: Add XRC to ODPCaps Date: Mon, 9 Sep 2019 12:07:07 +0300 Message-Id: <20190909090712.11029-8-noaos@mellanox.com> X-Mailer: git-send-email 2.21.0 In-Reply-To: <20190909090712.11029-1-noaos@mellanox.com> References: <20190909090712.11029-1-noaos@mellanox.com> MIME-Version: 1.0 Sender: linux-rdma-owner@vger.kernel.org Precedence: bulk List-ID: X-Mailing-List: linux-rdma@vger.kernel.org From: Maxim Chicherin Added PCI atomic caps and XRC ODP caps: XRC ODP caps reports about XRC ODP capabilities of the device. To make it easier for users, pyverbs reports XRC ODP caps as part of the device's odp_caps, even though they're reported in a separate field. PCI atomic caps represents ibv_pci_atomic_caps which is part of DeviceAttrEx and is now exposed to pyverbs' users. Signed-off-by: Maxim Chicherin --- pyverbs/device.pxd | 4 ++++ pyverbs/device.pyx | 42 ++++++++++++++++++++++++++++++++++++++++++ pyverbs/libibverbs.pxd | 9 ++++++++- 3 files changed, 54 insertions(+), 1 deletion(-) mode change 100644 => 100755 pyverbs/device.pxd mode change 100644 => 100755 pyverbs/device.pyx diff --git a/pyverbs/device.pxd b/pyverbs/device.pxd old mode 100644 new mode 100755 index ed2f1ca3d6ef..5cc13acb4331 --- a/pyverbs/device.pxd +++ b/pyverbs/device.pxd @@ -26,6 +26,7 @@ cdef class QueryDeviceExInput(PyverbsObject): cdef class ODPCaps(PyverbsObject): cdef v.ibv_odp_caps odp_caps + cdef object xrc_odp_caps cdef class RSSCaps(PyverbsObject): cdef v.ibv_rss_caps rss_caps @@ -33,6 +34,9 @@ cdef class RSSCaps(PyverbsObject): cdef class PacketPacingCaps(PyverbsObject): cdef v.ibv_packet_pacing_caps packet_pacing_caps +cdef class PCIAtomicCaps(PyverbsObject): + cdef v.ibv_pci_atomic_caps caps + cdef class TMCaps(PyverbsObject): cdef v.ibv_tm_caps tm_caps diff --git a/pyverbs/device.pyx b/pyverbs/device.pyx old mode 100644 new mode 100755 index f238d37ce3a9..084086bdbc69 --- a/pyverbs/device.pyx +++ b/pyverbs/device.pyx @@ -393,6 +393,42 @@ cdef class ODPCaps(PyverbsObject): @property def ud_odp_caps(self): return self.odp_caps.per_transport_caps.ud_odp_caps + @property + def xrc_odp_caps(self): + return self.xrc_odp_caps + @xrc_odp_caps.setter + def xrc_odp_caps(self, val): + self.xrc_odp_caps = val + + def __str__(self): + general_caps = {e.IBV_ODP_SUPPORT: 'IBV_ODP_SUPPORT', + e.IBV_ODP_SUPPORT_IMPLICIT: 'IBV_ODP_SUPPORT_IMPLICIT'} + + l = {e.IBV_ODP_SUPPORT_SEND: 'IBV_ODP_SUPPORT_SEND', + e.IBV_ODP_SUPPORT_RECV: 'IBV_ODP_SUPPORT_RECV', + e.IBV_ODP_SUPPORT_WRITE: 'IBV_ODP_SUPPORT_WRITE', + e.IBV_ODP_SUPPORT_READ: 'IBV_ODP_SUPPORT_READ', + e.IBV_ODP_SUPPORT_ATOMIC: 'IBV_ODP_SUPPORT_ATOMIC', + e.IBV_ODP_SUPPORT_SRQ_RECV: 'IBV_ODP_SUPPORT_SRQ_RECV'} + + print_format = '{}: {}\n' + return print_format.format('ODP General caps', str_from_flags(self.general_caps, general_caps)) +\ + print_format.format('RC ODP caps', str_from_flags(self.rc_odp_caps, l)) +\ + print_format.format('UD ODP caps', str_from_flags(self.ud_odp_caps, l)) +\ + print_format.format('UC ODP caps', str_from_flags(self.uc_odp_caps, l)) +\ + print_format.format('XRC ODP caps', str_from_flags(self.xrc_odp_caps, l)) + + +cdef class PCIAtomicCaps(PyverbsObject): + @property + def fetch_add(self): + return self.caps.fetch_add + @property + def swap(self): + return self.caps.swap + @property + def compare_swap(self): + return self.caps.compare_swap cdef class TSOCaps(PyverbsObject): @@ -477,6 +513,7 @@ cdef class DeviceAttrEx(PyverbsObject): def odp_caps(self): caps = ODPCaps() caps.odp_caps = self.dev_attr.odp_caps + caps.xrc_odp_caps = self.dev_attr.xrc_odp_caps return caps @property def completion_timestamp_mask(self): @@ -493,6 +530,11 @@ cdef class DeviceAttrEx(PyverbsObject): caps.tso_caps = self.dev_attr.tso_caps return caps @property + def pci_atomic_caps(self): + caps = PCIAtomicCaps() + caps.caps = self.dev_attr.pci_atomic_caps + return caps + @property def rss_caps(self): caps = RSSCaps() caps.rss_caps = self.dev_attr.rss_caps diff --git a/pyverbs/libibverbs.pxd b/pyverbs/libibverbs.pxd index a92286d42c67..02137d81e2d3 100755 --- a/pyverbs/libibverbs.pxd +++ b/pyverbs/libibverbs.pxd @@ -2,7 +2,7 @@ # Copyright (c) 2018, Mellanox Technologies. All rights reserved. See COPYING file include 'libibverbs_enums.pxd' -from libc.stdint cimport uint8_t, uint32_t, uint64_t +from libc.stdint cimport uint8_t, uint16_t, uint32_t, uint64_t cdef extern from 'infiniband/verbs.h': @@ -117,6 +117,11 @@ cdef extern from 'infiniband/verbs.h': unsigned int max_cq_count unsigned int max_cq_period + cdef struct ibv_pci_atomic_caps: + uint16_t fetch_add + uint16_t swap + uint16_t compare_swap + cdef struct ibv_device_attr_ex: ibv_device_attr orig_attr unsigned int comp_mask @@ -132,6 +137,8 @@ cdef extern from 'infiniband/verbs.h': ibv_tm_caps tm_caps ibv_cq_moderation_caps cq_mod_caps unsigned long max_dm_size + ibv_pci_atomic_caps pci_atomic_caps + uint32_t xrc_odp_caps cdef struct ibv_mw: ibv_context *context From patchwork Mon Sep 9 09:07:08 2019 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Noa Osherovich X-Patchwork-Id: 11137485 Return-Path: Received: from mail.kernel.org (pdx-korg-mail-1.web.codeaurora.org [172.30.200.123]) by pdx-korg-patchwork-2.web.codeaurora.org (Postfix) with ESMTP id 5B9EA76 for ; Mon, 9 Sep 2019 09:07:20 +0000 (UTC) Received: from vger.kernel.org (vger.kernel.org [209.132.180.67]) by mail.kernel.org (Postfix) with ESMTP id 440762089F for ; Mon, 9 Sep 2019 09:07:20 +0000 (UTC) Received: (majordomo@vger.kernel.org) by vger.kernel.org via listexpand id S2389624AbfIIJHT (ORCPT ); Mon, 9 Sep 2019 05:07:19 -0400 Received: from mail-il-dmz.mellanox.com ([193.47.165.129]:48367 "EHLO mellanox.co.il" rhost-flags-OK-OK-OK-FAIL) by vger.kernel.org with ESMTP id S2389599AbfIIJHT (ORCPT ); Mon, 9 Sep 2019 05:07:19 -0400 Received: from Internal Mail-Server by MTLPINE1 (envelope-from noaos@mellanox.com) with ESMTPS (AES256-SHA encrypted); 9 Sep 2019 12:07:16 +0300 Received: from reg-l-vrt-059-007.mtl.labs.mlnx (reg-l-vrt-059-007.mtl.labs.mlnx [10.135.59.7]) by labmailer.mlnx (8.13.8/8.13.8) with ESMTP id x8997Fp6028426; Mon, 9 Sep 2019 12:07:16 +0300 From: Noa Osherovich To: dledford@redhat.com, jgg@mellanox.com, leonro@mellanox.com Cc: linux-rdma@vger.kernel.org, Maxim Chicherin Subject: [PATCH rdma-core 08/12] Documentation: Document creation of XRCD and SRQ Date: Mon, 9 Sep 2019 12:07:08 +0300 Message-Id: <20190909090712.11029-9-noaos@mellanox.com> X-Mailer: git-send-email 2.21.0 In-Reply-To: <20190909090712.11029-1-noaos@mellanox.com> References: <20190909090712.11029-1-noaos@mellanox.com> MIME-Version: 1.0 Sender: linux-rdma-owner@vger.kernel.org Precedence: bulk List-ID: X-Mailing-List: linux-rdma@vger.kernel.org From: Maxim Chicherin Add code snippets to demonstrate creation of XRCD and SRQ. Signed-off-by: Maxim Chicherin --- Documentation/pyverbs.md | 51 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 51 insertions(+) mode change 100644 => 100755 Documentation/pyverbs.md diff --git a/Documentation/pyverbs.md b/Documentation/pyverbs.md old mode 100644 new mode 100755 index 22c51868e025..29ab9592c53c --- a/Documentation/pyverbs.md +++ b/Documentation/pyverbs.md @@ -339,3 +339,54 @@ wr = pwr.SendWR() wr.set_wr_ud(ah, 0x1101, 0) # in real life, use real values udqp.post_send(wr) ``` + +##### XRCD +The following code demonstrates creation of an XRCD object. +```python +from pyverbs.xrcd import XRCD, XRCDInitAttr +import pyverbs.device as d +import pyverbs.enums as e +import stat +import os + + +ctx = d.Context(name='ibp0s8f0') +xrcd_fd = os.open('/tmp/xrcd', os.O_RDONLY | os.O_CREAT, + stat.S_IRUSR | stat.S_IRGRP) +init = XRCDInitAttr(e.IBV_XRCD_INIT_ATTR_FD | e.IBV_XRCD_INIT_ATTR_OFLAGS, + os.O_CREAT, xrcd_fd) +xrcd = XRCD(ctx, init) +``` + +##### SRQ +The following code snippet will demonstrate creation of an XRC SRQ object. +For more complex examples, please see pyverbs/tests/test_odp. +```python +from pyverbs.xrcd import XRCD, XRCDInitAttr +from pyverbs.srq import SRQ, SrqInitAttrEx +import pyverbs.device as d +import pyverbs.enums as e +from pyverbs.cq import CQ +from pyverbs.pd import PD +import stat +import os + + +ctx = d.Context(name='ibp0s8f0') +pd = PD(ctx) +cq = CQ(ctx, 100, None, None, 0) +xrcd_fd = os.open('/tmp/xrcd', os.O_RDONLY | os.O_CREAT, + stat.S_IRUSR | stat.S_IRGRP) +init = XRCDInitAttr(e.IBV_XRCD_INIT_ATTR_FD | e.IBV_XRCD_INIT_ATTR_OFLAGS, + os.O_CREAT, xrcd_fd) +xrcd = XRCD(ctx, init) + +srq_attr = SrqInitAttrEx(max_wr=10) +srq_attr.srq_type = e.IBV_SRQT_XRC +srq_attr.pd = pd +srq_attr.xrcd = xrcd +srq_attr.cq = cq +srq_attr.comp_mask = e.IBV_SRQ_INIT_ATTR_TYPE | e.IBV_SRQ_INIT_ATTR_PD | \ + e.IBV_SRQ_INIT_ATTR_CQ | e.IBV_SRQ_INIT_ATTR_XRCD +srq = SRQ(ctx, srq_attr) +``` From patchwork Mon Sep 9 09:07:09 2019 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Noa Osherovich X-Patchwork-Id: 11137499 Return-Path: Received: from mail.kernel.org (pdx-korg-mail-1.web.codeaurora.org [172.30.200.123]) by pdx-korg-patchwork-2.web.codeaurora.org (Postfix) with ESMTP id 8AC001708 for ; Mon, 9 Sep 2019 09:07:22 +0000 (UTC) Received: from vger.kernel.org (vger.kernel.org [209.132.180.67]) by mail.kernel.org (Postfix) with ESMTP id 744C52196E for ; Mon, 9 Sep 2019 09:07:22 +0000 (UTC) Received: (majordomo@vger.kernel.org) by vger.kernel.org via listexpand id S2389601AbfIIJHT (ORCPT ); Mon, 9 Sep 2019 05:07:19 -0400 Received: from mail-il-dmz.mellanox.com ([193.47.165.129]:48377 "EHLO mellanox.co.il" rhost-flags-OK-OK-OK-FAIL) by vger.kernel.org with ESMTP id S2389605AbfIIJHT (ORCPT ); Mon, 9 Sep 2019 05:07:19 -0400 Received: from Internal Mail-Server by MTLPINE1 (envelope-from noaos@mellanox.com) with ESMTPS (AES256-SHA encrypted); 9 Sep 2019 12:07:16 +0300 Received: from reg-l-vrt-059-007.mtl.labs.mlnx (reg-l-vrt-059-007.mtl.labs.mlnx [10.135.59.7]) by labmailer.mlnx (8.13.8/8.13.8) with ESMTP id x8997Fp7028426; Mon, 9 Sep 2019 12:07:16 +0300 From: Noa Osherovich To: dledford@redhat.com, jgg@mellanox.com, leonro@mellanox.com Cc: linux-rdma@vger.kernel.org, Maxim Chicherin Subject: [PATCH rdma-core 09/12] tests: Add missing constant in UDResources Date: Mon, 9 Sep 2019 12:07:09 +0300 Message-Id: <20190909090712.11029-10-noaos@mellanox.com> X-Mailer: git-send-email 2.21.0 In-Reply-To: <20190909090712.11029-1-noaos@mellanox.com> References: <20190909090712.11029-1-noaos@mellanox.com> MIME-Version: 1.0 Sender: linux-rdma-owner@vger.kernel.org Precedence: bulk List-ID: X-Mailing-List: linux-rdma@vger.kernel.org From: Maxim Chicherin GRH_SIZE constant was missing in class implementation despite being used by create_mr method. Signed-off-by: Maxim Chicherin --- tests/base.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/base.py b/tests/base.py index a28e9b9dc466..b9bdaea93c96 100644 --- a/tests/base.py +++ b/tests/base.py @@ -274,6 +274,7 @@ class RCResources(TrafficResources): class UDResources(TrafficResources): UD_QKEY = 0x11111111 UD_PKEY_INDEX = 0 + GRH_SIZE = 40 def create_mr(self): self.mr = MR(self.pd, self.msg_size + self.GRH_SIZE, From patchwork Mon Sep 9 09:07:10 2019 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Noa Osherovich X-Patchwork-Id: 11137489 Return-Path: Received: from mail.kernel.org (pdx-korg-mail-1.web.codeaurora.org [172.30.200.123]) by pdx-korg-patchwork-2.web.codeaurora.org (Postfix) with ESMTP id 9F5651599 for ; Mon, 9 Sep 2019 09:07:20 +0000 (UTC) Received: from vger.kernel.org (vger.kernel.org [209.132.180.67]) by mail.kernel.org (Postfix) with ESMTP id 8815E2089F for ; Mon, 9 Sep 2019 09:07:20 +0000 (UTC) Received: (majordomo@vger.kernel.org) by vger.kernel.org via listexpand id S2389605AbfIIJHT (ORCPT ); Mon, 9 Sep 2019 05:07:19 -0400 Received: from mail-il-dmz.mellanox.com ([193.47.165.129]:48378 "EHLO mellanox.co.il" rhost-flags-OK-OK-OK-FAIL) by vger.kernel.org with ESMTP id S2389616AbfIIJHT (ORCPT ); Mon, 9 Sep 2019 05:07:19 -0400 Received: from Internal Mail-Server by MTLPINE1 (envelope-from noaos@mellanox.com) with ESMTPS (AES256-SHA encrypted); 9 Sep 2019 12:07:16 +0300 Received: from reg-l-vrt-059-007.mtl.labs.mlnx (reg-l-vrt-059-007.mtl.labs.mlnx [10.135.59.7]) by labmailer.mlnx (8.13.8/8.13.8) with ESMTP id x8997Fp8028426; Mon, 9 Sep 2019 12:07:16 +0300 From: Noa Osherovich To: dledford@redhat.com, jgg@mellanox.com, leonro@mellanox.com Cc: linux-rdma@vger.kernel.org, Maxim Chicherin Subject: [PATCH rdma-core 10/12] tests: Fixes to to_rts() in RCResources Date: Mon, 9 Sep 2019 12:07:10 +0300 Message-Id: <20190909090712.11029-11-noaos@mellanox.com> X-Mailer: git-send-email 2.21.0 In-Reply-To: <20190909090712.11029-1-noaos@mellanox.com> References: <20190909090712.11029-1-noaos@mellanox.com> MIME-Version: 1.0 Sender: linux-rdma-owner@vger.kernel.org Precedence: bulk List-ID: X-Mailing-List: linux-rdma@vger.kernel.org From: Maxim Chicherin Move RCResources constants outside of the class in order to expose them to other classes. Avoid passing parameters that are available to the class. Signed-off-by: Maxim Chicherin --- tests/base.py | 49 +++++++++++++++++++++++++++---------------------- 1 file changed, 27 insertions(+), 22 deletions(-) mode change 100644 => 100755 tests/base.py diff --git a/tests/base.py b/tests/base.py old mode 100644 new mode 100755 index b9bdaea93c96..ce9ea83b6b53 --- a/tests/base.py +++ b/tests/base.py @@ -13,6 +13,14 @@ from pyverbs.pd import PD from pyverbs.cq import CQ from pyverbs.mr import MR +PATH_MTU = e.IBV_MTU_1024 +MAX_DEST_RD_ATOMIC = 1 +MAX_RD_ATOMIC = 1 +MIN_RNR_TIMER =12 +RETRY_CNT = 7 +RNR_RETRY = 7 +TIMEOUT = 14 + class PyverbsAPITestCase(unittest.TestCase): def setUp(self): @@ -231,33 +239,24 @@ class TrafficResources(BaseResources): class RCResources(TrafficResources): - PATH_MTU = e.IBV_MTU_1024 - MAX_DEST_RD_ATOMIC = 1 - MAX_RD_ATOMIC = 1 - MIN_RNR_TIMER =12 - RETRY_CNT = 7 - RNR_RETRY = 7 - TIMEOUT = 14 - - def to_rts(self, rpsn, rqpn): + + def to_rts(self): """ Set the QP attributes' values to arbitrary values (same values used in ibv_rc_pingpong). - :param rpsn: Remote PSN (packet serial number) - :param rqpn: Remote QP number :return: None """ attr = QPAttr(port_num=self.ib_port) - attr.dest_qp_num = rqpn - attr.path_mtu = self.PATH_MTU - attr.max_dest_rd_atomic = self.MAX_DEST_RD_ATOMIC - attr.min_rnr_timer = self.MIN_RNR_TIMER - attr.rq_psn = rpsn - attr.sq_psn = self.psn - attr.timeout = self.TIMEOUT - attr.retry_cnt = self.RETRY_CNT - attr.rnr_retry = self.RNR_RETRY - attr.max_rd_atomic = self.MAX_RD_ATOMIC + attr.dest_qp_num = self.rqpn + attr.path_mtu = PATH_MTU + attr.max_dest_rd_atomic = MAX_DEST_RD_ATOMIC + attr.min_rnr_timer = MIN_RNR_TIMER + attr.rq_psn = self.psn + attr.sq_psn = self.rpsn + attr.timeout = TIMEOUT + attr.retry_cnt = RETRY_CNT + attr.rnr_retry = RNR_RETRY + attr.max_rd_atomic = MAX_RD_ATOMIC gr = GlobalRoute(dgid=self.ctx.query_gid(self.ib_port, self.gid_index), sgid_index=self.gid_index) ah_attr = AHAttr(port_num=self.ib_port, is_global=1, gr=gr, @@ -266,9 +265,15 @@ class RCResources(TrafficResources): self.qp.to_rts(attr) def pre_run(self, rpsn, rqpn): + """ + Configure Resources before running traffic + :param rpsn: Remote PSN (packet serial number) + :param rqpn: Remote QP number + :return: None + """ self.rqpn = rqpn self.rpsn = rpsn - self.to_rts(rpsn, rqpn) + self.to_rts() class UDResources(TrafficResources): From patchwork Mon Sep 9 09:07:11 2019 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Noa Osherovich X-Patchwork-Id: 11137493 Return-Path: Received: from mail.kernel.org (pdx-korg-mail-1.web.codeaurora.org [172.30.200.123]) by pdx-korg-patchwork-2.web.codeaurora.org (Postfix) with ESMTP id 427B776 for ; Mon, 9 Sep 2019 09:07:21 +0000 (UTC) Received: from vger.kernel.org (vger.kernel.org [209.132.180.67]) by mail.kernel.org (Postfix) with ESMTP id 2B3BE21924 for ; Mon, 9 Sep 2019 09:07:21 +0000 (UTC) Received: (majordomo@vger.kernel.org) by vger.kernel.org via listexpand id S2389612AbfIIJHU (ORCPT ); Mon, 9 Sep 2019 05:07:20 -0400 Received: from mail-il-dmz.mellanox.com ([193.47.165.129]:48387 "EHLO mellanox.co.il" rhost-flags-OK-OK-OK-FAIL) by vger.kernel.org with ESMTP id S2389613AbfIIJHT (ORCPT ); Mon, 9 Sep 2019 05:07:19 -0400 Received: from Internal Mail-Server by MTLPINE1 (envelope-from noaos@mellanox.com) with ESMTPS (AES256-SHA encrypted); 9 Sep 2019 12:07:17 +0300 Received: from reg-l-vrt-059-007.mtl.labs.mlnx (reg-l-vrt-059-007.mtl.labs.mlnx [10.135.59.7]) by labmailer.mlnx (8.13.8/8.13.8) with ESMTP id x8997Fp9028426; Mon, 9 Sep 2019 12:07:16 +0300 From: Noa Osherovich To: dledford@redhat.com, jgg@mellanox.com, leonro@mellanox.com Cc: linux-rdma@vger.kernel.org, Maxim Chicherin Subject: [PATCH rdma-core 11/12] tests: Add XRCResources class Date: Mon, 9 Sep 2019 12:07:11 +0300 Message-Id: <20190909090712.11029-12-noaos@mellanox.com> X-Mailer: git-send-email 2.21.0 In-Reply-To: <20190909090712.11029-1-noaos@mellanox.com> References: <20190909090712.11029-1-noaos@mellanox.com> MIME-Version: 1.0 Sender: linux-rdma-owner@vger.kernel.org Precedence: bulk List-ID: X-Mailing-List: linux-rdma@vger.kernel.org From: Maxim Chicherin Add base aggregation object for XRC communication. XRCResources contains Context, PD, MR, CQ, two XRC RECV QPs, two XRC SEND QPs, SRQ and XRCD. XRCResources constructor has optional parameter qp_count which decides how many QPs to create (for both types), default value is 2. Signed-off-by: Maxim Chicherin --- tests/base.py | 107 +++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 105 insertions(+), 2 deletions(-) diff --git a/tests/base.py b/tests/base.py index ce9ea83b6b53..dd17ddb6fc50 100755 --- a/tests/base.py +++ b/tests/base.py @@ -3,9 +3,13 @@ import unittest import random +import stat +import os -from pyverbs.qp import QPCap, QPInitAttr, QPAttr, QP +from pyverbs.qp import QPCap, QPInitAttrEx, QPInitAttr, QPAttr, QP from pyverbs.addr import AHAttr, GlobalRoute +from pyverbs.xrcd import XRCD, XRCDInitAttr +from pyverbs.srq import SRQ, SrqInitAttrEx from pyverbs.device import Context import pyverbs.device as d import pyverbs.enums as e @@ -296,4 +300,103 @@ class UDResources(TrafficResources): def pre_run(self, rpsn, rqpn): self.rqpn = rqpn - self.rpsn = rpsn \ No newline at end of file + self.rpsn = rpsn + + +class XRCResources(TrafficResources): + def __init__(self, dev_name, ib_port, gid_index, qp_count=2): + self.xrcd_fd = -1 + self.xrcd = None + self.srq = None + self.qp_count = qp_count + self.sqp_lst = [] + self.rqp_lst = [] + self.qps_num = [] + self.psns = [] + self.rqps_num = None + self.rpsns = None + super(XRCResources, self).__init__(dev_name, ib_port, gid_index) + + def close(self): + os.close(self.xrcd_fd) + + def create_qp(self): + """ + Initializes self.qp with an XRC SEND/RECV QP. + :return: None + """ + qp_attr = QPAttr(port_num=self.ib_port) + qp_attr.pkey_index = 0 + + for _ in range(self.qp_count): + attr_ex = QPInitAttrEx(qp_type=e.IBV_QPT_XRC_RECV, + comp_mask=e.IBV_QP_INIT_ATTR_XRCD, + xrcd=self.xrcd) + qp_attr.qp_access_flags = e.IBV_ACCESS_REMOTE_WRITE | \ + e.IBV_ACCESS_REMOTE_READ + recv_qp = QP(self.ctx, attr_ex, qp_attr) + self.rqp_lst.append(recv_qp) + + qp_caps = QPCap(max_send_wr=self.num_msgs, max_recv_sge=0, + max_recv_wr=0) + attr_ex = QPInitAttrEx(qp_type=e.IBV_QPT_XRC_SEND, sq_sig_all=1, + comp_mask=e.IBV_QP_INIT_ATTR_PD, + pd=self.pd, scq=self.cq, cap=qp_caps) + qp_attr.qp_access_flags = 0 + send_qp =QP(self.ctx, attr_ex, qp_attr) + self.sqp_lst.append(send_qp) + self.qps_num.append((recv_qp.qp_num, send_qp.qp_num)) + self.psns.append(random.getrandbits(24)) + + def create_xrcd(self): + """ + Initializes self.xrcd with an XRC Domain object. + :return: None + """ + self.xrcd_fd = os.open('/tmp/xrcd', os.O_RDONLY | os.O_CREAT, + stat.S_IRUSR | stat.S_IRGRP) + init = XRCDInitAttr( + e.IBV_XRCD_INIT_ATTR_FD | e.IBV_XRCD_INIT_ATTR_OFLAGS, + os.O_CREAT, self.xrcd_fd) + self.xrcd = XRCD(self.ctx, init) + + def create_srq(self): + """ + Initializes self.srq with a Shared Receive QP object. + :return: None + """ + srq_attr = SrqInitAttrEx(max_wr=self.qp_count*self.num_msgs) + srq_attr.srq_type = e.IBV_SRQT_XRC + srq_attr.pd = self.pd + srq_attr.xrcd = self.xrcd + srq_attr.cq = self.cq + srq_attr.comp_mask = e.IBV_SRQ_INIT_ATTR_TYPE | e.IBV_SRQ_INIT_ATTR_PD | \ + e.IBV_SRQ_INIT_ATTR_CQ | e.IBV_SRQ_INIT_ATTR_XRCD + self.srq = SRQ(self.ctx, srq_attr) + + def to_rts(self): + ah_attr = AHAttr(dlid=self.port_attr.lid) + qp_attr = QPAttr() + qp_attr.path_mtu = PATH_MTU + qp_attr.timeout = TIMEOUT + qp_attr.retry_cnt = RETRY_CNT + qp_attr.rnr_retry = RNR_RETRY + qp_attr.min_rnr_timer = MIN_RNR_TIMER + qp_attr.ah_attr = ah_attr + for i in range(self.qp_count): + qp_attr.dest_qp_num = self.rqps_num[i][1] + qp_attr.rq_psn = self.psns[i] + qp_attr.sq_psn = self.rpsns[i] + self.rqp_lst[i].to_rts(qp_attr) + qp_attr.dest_qp_num = self.rqps_num[i][0] + self.sqp_lst[i].to_rts(qp_attr) + + def init_resources(self): + self.create_xrcd() + super(XRCResources, self).init_resources() + self.create_srq() + + def pre_run(self, rpsns, rqps_num): + self.rqps_num = rqps_num + self.rpsns = rpsns + self.to_rts() From patchwork Mon Sep 9 09:07:12 2019 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Noa Osherovich X-Patchwork-Id: 11137501 Return-Path: Received: from mail.kernel.org (pdx-korg-mail-1.web.codeaurora.org [172.30.200.123]) by pdx-korg-patchwork-2.web.codeaurora.org (Postfix) with ESMTP id C2762186E for ; Mon, 9 Sep 2019 09:07:22 +0000 (UTC) Received: from vger.kernel.org (vger.kernel.org [209.132.180.67]) by mail.kernel.org (Postfix) with ESMTP id AB01521924 for ; Mon, 9 Sep 2019 09:07:22 +0000 (UTC) Received: (majordomo@vger.kernel.org) by vger.kernel.org via listexpand id S2389608AbfIIJHV (ORCPT ); Mon, 9 Sep 2019 05:07:21 -0400 Received: from mail-il-dmz.mellanox.com ([193.47.165.129]:48388 "EHLO mellanox.co.il" rhost-flags-OK-OK-OK-FAIL) by vger.kernel.org with ESMTP id S2389611AbfIIJHU (ORCPT ); Mon, 9 Sep 2019 05:07:20 -0400 Received: from Internal Mail-Server by MTLPINE1 (envelope-from noaos@mellanox.com) with ESMTPS (AES256-SHA encrypted); 9 Sep 2019 12:07:17 +0300 Received: from reg-l-vrt-059-007.mtl.labs.mlnx (reg-l-vrt-059-007.mtl.labs.mlnx [10.135.59.7]) by labmailer.mlnx (8.13.8/8.13.8) with ESMTP id x8997FpA028426; Mon, 9 Sep 2019 12:07:17 +0300 From: Noa Osherovich To: dledford@redhat.com, jgg@mellanox.com, leonro@mellanox.com Cc: linux-rdma@vger.kernel.org, Maxim Chicherin Subject: [PATCH rdma-core 12/12] tests: Add XRC ODP test case Date: Mon, 9 Sep 2019 12:07:12 +0300 Message-Id: <20190909090712.11029-13-noaos@mellanox.com> X-Mailer: git-send-email 2.21.0 In-Reply-To: <20190909090712.11029-1-noaos@mellanox.com> References: <20190909090712.11029-1-noaos@mellanox.com> MIME-Version: 1.0 Sender: linux-rdma-owner@vger.kernel.org Precedence: bulk List-ID: X-Mailing-List: linux-rdma@vger.kernel.org From: Maxim Chicherin This case creates client and server, each one have two SEND and RECV XRC QPs, and SRQ. By connecting SRQ and RECV XRC QPs to the same XRCD and connecting SEND XRC QPs to RECV XRC QPs of the opposite side, we run a traffic and validate the data received by the SRQ. Signed-off-by: Maxim Chicherin --- tests/base.py | 5 ++++- tests/test_odp.py | 32 +++++++++++++++++++++++-------- tests/utils.py | 48 ++++++++++++++++++++++++++++++++++++++++++++--- 3 files changed, 73 insertions(+), 12 deletions(-) mode change 100644 => 100755 tests/test_odp.py mode change 100644 => 100755 tests/utils.py diff --git a/tests/base.py b/tests/base.py index dd17ddb6fc50..57a75f797770 100755 --- a/tests/base.py +++ b/tests/base.py @@ -375,7 +375,10 @@ class XRCResources(TrafficResources): self.srq = SRQ(self.ctx, srq_attr) def to_rts(self): - ah_attr = AHAttr(dlid=self.port_attr.lid) + gid = self.ctx.query_gid(self.ib_port, self.gid_index) + gr = GlobalRoute(dgid=gid, sgid_index=self.gid_index) + ah_attr = AHAttr(port_num=self.ib_port, is_global=True, + gr=gr, dlid=self.port_attr.lid) qp_attr = QPAttr() qp_attr.path_mtu = PATH_MTU qp_attr.timeout = TIMEOUT diff --git a/tests/test_odp.py b/tests/test_odp.py old mode 100644 new mode 100755 index 922cd0d9aad5..d412a7792951 --- a/tests/test_odp.py +++ b/tests/test_odp.py @@ -1,6 +1,6 @@ +from tests.base import RCResources, UDResources, XRCResources +from tests.utils import requires_odp, traffic, xrc_traffic from tests.base import RDMATestCase -from tests.utils import requires_odp, traffic -from tests.base import RCResources, UDResources from pyverbs.mr import MR import pyverbs.enums as e @@ -8,7 +8,7 @@ import pyverbs.enums as e class OdpUD(UDResources): @requires_odp('ud') def create_mr(self): - self.mr = MR(self.pd, self.msg_size + self.GRH_SIZE , + self.mr = MR(self.pd, self.msg_size + self.GRH_SIZE, e.IBV_ACCESS_LOCAL_WRITE | e.IBV_ACCESS_ON_DEMAND) @@ -18,18 +18,30 @@ class OdpRC(RCResources): self.mr = MR(self.pd, self.msg_size, e.IBV_ACCESS_LOCAL_WRITE | e.IBV_ACCESS_ON_DEMAND) +class OdpXRC(XRCResources): + @requires_odp('xrc') + def create_mr(self): + self.mr = MR(self.pd, self.msg_size, + e.IBV_ACCESS_LOCAL_WRITE | e.IBV_ACCESS_ON_DEMAND) + class OdpTestCase(RDMATestCase): def setUp(self): super(OdpTestCase, self).setUp() self.iters = 100 - self.qp_dict = {'rc': OdpRC, 'ud': OdpUD} + self.qp_dict = {'rc': OdpRC, 'ud': OdpUD, 'xrc': OdpXRC} def create_players(self, qp_type): - client = self.qp_dict[qp_type](self.dev_name, self.ib_port, self.gid_index) - server = self.qp_dict[qp_type](self.dev_name, self.ib_port, self.gid_index) - client.pre_run(server.psn, server.qpn) - server.pre_run(client.psn, client.qpn) + client = self.qp_dict[qp_type](self.dev_name, self.ib_port, + self.gid_index) + server = self.qp_dict[qp_type](self.dev_name, self.ib_port, + self.gid_index) + if qp_type == 'xrc': + client.pre_run(server.psns, server.qps_num) + server.pre_run(client.psns, client.qps_num) + else: + client.pre_run(server.psn, server.qpn) + server.pre_run(client.psn, client.qpn) return client, server def test_odp_rc_traffic(self): @@ -39,3 +51,7 @@ class OdpTestCase(RDMATestCase): def test_odp_ud_traffic(self): client, server = self.create_players('ud') traffic(client, server, self.iters, self.gid_index, self.ib_port) + + def test_odp_xrc_traffic(self): + client, server = self.create_players('xrc') + xrc_traffic(client, server) diff --git a/tests/utils.py b/tests/utils.py old mode 100644 new mode 100755 index 881ce4d03634..785309552e25 --- a/tests/utils.py +++ b/tests/utils.py @@ -12,6 +12,7 @@ from pyverbs.pyverbs_error import PyverbsError, PyverbsRDMAError from pyverbs.addr import AHAttr, AH, GlobalRoute from pyverbs.wr import SGE, SendWR, RecvWR from pyverbs.qp import QPCap, QPInitAttrEx +from tests.base import XRCResources import pyverbs.device as d import pyverbs.enums as e @@ -256,7 +257,8 @@ def get_send_wr(agr_obj, is_server): :param is_server: Indicates whether this is server or client side :return: send wr """ - qp_type = agr_obj.qp.qp_type + qp_type = agr_obj.sqp_lst[0].qp_type if isinstance(agr_obj, XRCResources) \ + else agr_obj.qp.qp_type mr = agr_obj.mr if qp_type == e.IBV_QPT_UD: send_sge = SGE(mr.buf + GRH_SIZE, agr_obj.msg_size, mr.lkey) @@ -273,7 +275,8 @@ def get_recv_wr(agr_obj): :param agr_obj: Aggregation object which contains all resources necessary :return: recv wr """ - qp_type = agr_obj.qp.qp_type + qp_type = agr_obj.rqp_lst[0].qp_type if isinstance(agr_obj, XRCResources) \ + else agr_obj.qp.qp_type mr = agr_obj.mr if qp_type == e.IBV_QPT_UD: recv_sge = SGE(mr.buf, agr_obj.msg_size + GRH_SIZE, mr.lkey) @@ -393,6 +396,44 @@ def traffic(client, server, iters, gid_idx, port): msg_received = client.mr.read(client.msg_size, 0) validate(msg_received, False, client.msg_size) + +def xrc_traffic(client, server): + """ + Runs basic xrc traffic, this function assumes that number of QPs, which + server and client have are equal, server.send_qp[i] is connected to + client.recv_qp[i], each time server.send_qp[i] sends a message, it is + redirected to client.srq because client.recv_qp[i] and client.srq are + under the same xrcd. The traffic flow in the opposite direction is the same. + :param client: Aggregation object of the active side, should be an instance + of XRCResources class + :param server: Aggregation object of the passive side, should be an instance + of XRCResources class + :return: None + """ + client_srqn = client.srq.get_srq_num() + server_srqn = server.srq.get_srq_num() + s_recv_wr = get_recv_wr(server) + c_recv_wr = get_recv_wr(client) + post_recv(client.srq, c_recv_wr, client.qp_count*client.num_msgs) + post_recv(server.srq, s_recv_wr, server.qp_count*server.num_msgs) + for _ in range(client.num_msgs): + for i in range(server.qp_count): + c_send_wr = get_send_wr(client, False) + c_send_wr.set_qp_type_xrc(server_srqn) + client.sqp_lst[i].post_send(c_send_wr) + poll_cq(client.cq) + poll_cq(server.cq) + msg_received = server.mr.read(server.msg_size, 0) + validate(msg_received, True, server.msg_size) + s_send_wr = get_send_wr(server, True) + s_send_wr.set_qp_type_xrc(client_srqn) + server.sqp_lst[i].post_send(s_send_wr) + poll_cq(server.cq) + poll_cq(client.cq) + msg_received = client.mr.read(client.msg_size, 0) + validate(msg_received, False, client.msg_size) + + # Decorators def requires_odp(qp_type): def outer(func): @@ -415,7 +456,8 @@ def odp_supported(ctx, qp_type): raise unittest.SkipTest('ODP is not supported - No ODP caps') qp_odp_caps = getattr(odp_caps, '{}_odp_caps'.format(qp_type)) has_odp_send = qp_odp_caps & e.IBV_ODP_SUPPORT_SEND - has_odp_recv = qp_odp_caps & e.IBV_ODP_SUPPORT_RECV + has_odp_recv = qp_odp_caps & e.IBV_ODP_SUPPORT_SRQ_RECV if qp_type == 'xrc'\ + else qp_odp_caps & e.IBV_ODP_SUPPORT_RECV if has_odp_send == 0: raise unittest.SkipTest('ODP is not supported - ODP send not supported') if has_odp_recv == 0: