From patchwork Sun Mar 24 14:10:35 2019 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Noa Osherovich X-Patchwork-Id: 10867441 Return-Path: Received: from mail.wl.linuxfoundation.org (pdx-wl-mail.web.codeaurora.org [172.30.200.125]) by pdx-korg-patchwork-2.web.codeaurora.org (Postfix) with ESMTP id C8C381390 for ; Sun, 24 Mar 2019 14:11:25 +0000 (UTC) Received: from mail.wl.linuxfoundation.org (localhost [127.0.0.1]) by mail.wl.linuxfoundation.org (Postfix) with ESMTP id B1BB62940A for ; Sun, 24 Mar 2019 14:11:25 +0000 (UTC) Received: by mail.wl.linuxfoundation.org (Postfix, from userid 486) id A520B294B6; Sun, 24 Mar 2019 14:11:25 +0000 (UTC) X-Spam-Checker-Version: SpamAssassin 3.3.1 (2010-03-16) on pdx-wl-mail.web.codeaurora.org X-Spam-Level: X-Spam-Status: No, score=-7.9 required=2.0 tests=BAYES_00,MAILING_LIST_MULTI, RCVD_IN_DNSWL_HI,UNPARSEABLE_RELAY autolearn=ham version=3.3.1 Received: from vger.kernel.org (vger.kernel.org [209.132.180.67]) by mail.wl.linuxfoundation.org (Postfix) with ESMTP id 847852940A for ; Sun, 24 Mar 2019 14:11:24 +0000 (UTC) Received: (majordomo@vger.kernel.org) by vger.kernel.org via listexpand id S1726743AbfCXOLW (ORCPT ); Sun, 24 Mar 2019 10:11:22 -0400 Received: from mail-il-dmz.mellanox.com ([193.47.165.129]:53699 "EHLO mellanox.co.il" rhost-flags-OK-OK-OK-FAIL) by vger.kernel.org with ESMTP id S1726317AbfCXOLW (ORCPT ); Sun, 24 Mar 2019 10:11:22 -0400 Received: from Internal Mail-Server by MTLPINE1 (envelope-from noaos@mellanox.com) with ESMTPS (AES256-SHA encrypted); 24 Mar 2019 16:11:18 +0200 Received: from reg-l-vrt-059-009.mtl.labs.mlnx (reg-l-vrt-059-009.mtl.labs.mlnx [10.135.59.9]) by labmailer.mlnx (8.13.8/8.13.8) with ESMTP id x2OEAxHt028186; Sun, 24 Mar 2019 16:11:18 +0200 From: Noa Osherovich To: leon@kernel.org, jgg@mellanox.com, dledford@redhat.com Cc: linux-rdma@vger.kernel.org, Noa Osherovich Subject: [PATCH rdma-core 1/6] pyverbs: Introducing completions related classes Date: Sun, 24 Mar 2019 16:10:35 +0200 Message-Id: <20190324141040.22204-2-noaos@mellanox.com> X-Mailer: git-send-email 2.17.2 In-Reply-To: <20190324141040.22204-1-noaos@mellanox.com> References: <20190324141040.22204-1-noaos@mellanox.com> Sender: linux-rdma-owner@vger.kernel.org Precedence: bulk List-ID: X-Mailing-List: linux-rdma@vger.kernel.org X-Virus-Scanned: ClamAV using ClamSMTP This patch adds infrastructure for creation and destruction of the following classes: - CompChannel - CQ Also added WC class with a user-friendly print format and some useful functions for conversions of enum values to user-readable strings. Signed-off-by: Noa Osherovich Reviewed-by: Maor Gottlieb Reviewed-by: Leon Romanovsky --- pyverbs/CMakeLists.txt | 1 + pyverbs/cq.pxd | 17 +++ pyverbs/cq.pyx | 295 +++++++++++++++++++++++++++++++++++++++++ pyverbs/device.pxd | 2 + pyverbs/device.pyx | 13 +- pyverbs/libibverbs.pxd | 36 +++++ 6 files changed, 363 insertions(+), 1 deletion(-) create mode 100644 pyverbs/cq.pxd create mode 100644 pyverbs/cq.pyx diff --git a/pyverbs/CMakeLists.txt b/pyverbs/CMakeLists.txt index 18056d6fcd65..ef5114594e20 100644 --- a/pyverbs/CMakeLists.txt +++ b/pyverbs/CMakeLists.txt @@ -4,6 +4,7 @@ rdma_cython_module(pyverbs addr.pyx base.pyx + cq.pyx device.pyx enums.pyx mr.pyx diff --git a/pyverbs/cq.pxd b/pyverbs/cq.pxd new file mode 100644 index 000000000000..82607042ce6b --- /dev/null +++ b/pyverbs/cq.pxd @@ -0,0 +1,17 @@ +# SPDX-License-Identifier: (GPL-2.0 OR Linux-OpenIB) +# Copyright (c) 2019, Mellanox Technologies. All rights reserved. +from pyverbs.base cimport PyverbsObject, PyverbsCM +cimport pyverbs.libibverbs as v + +cdef class CompChannel(PyverbsCM): + cdef v.ibv_comp_channel *cc + cpdef close(self) + cdef object context + +cdef class CQ(PyverbsCM): + cdef v.ibv_cq *cq + cpdef close(self) + cdef object context + +cdef class WC(PyverbsObject): + cdef v.ibv_wc wc diff --git a/pyverbs/cq.pyx b/pyverbs/cq.pyx new file mode 100644 index 000000000000..d1e2520f6be1 --- /dev/null +++ b/pyverbs/cq.pyx @@ -0,0 +1,295 @@ +# SPDX-License-Identifier: (GPL-2.0 OR Linux-OpenIB) +# Copyright (c) 2019, Mellanox Technologies. All rights reserved. +from pyverbs.base import PyverbsRDMAErrno +cimport pyverbs.libibverbs_enums as e +from pyverbs.device cimport Context + +cdef class CompChannel(PyverbsCM): + """ + A completion channel is a file descriptor used to deliver completion + notifications to a userspace process. When a completion event is generated + for a CQ, the event is delivered via the completion channel attached to the + CQ. + """ + def __cinit__(self, Context context not None): + """ + Initializes a completion channel object on the given device. + :param context: The device's context to use + :return: A CompChannel object on success + """ + self.cc = v.ibv_create_comp_channel(context.context) + if self.cc == NULL: + raise PyverbsRDMAErrno('Failed to create a completion channel') + self.context = context + context.add_ref(self) + self.logger.debug('Created a Completion Channel') + + def __dealloc__(self): + self.close() + + cpdef close(self): + self.logger.debug('Closing completion channel') + if self.cc != NULL: + rc = v.ibv_destroy_comp_channel(self.cc) + if rc != 0: + raise PyverbsRDMAErrno('Failed to destroy a completion channel') + self.cc = NULL + + def get_cq_event(self, CQ expected_cq): + """ + Waits for the next completion event in the completion event channel + :param expected_cq: The CQ that got the event + :return: None + """ + cdef v.ibv_cq *cq + cdef void *ctx + + rc = v.ibv_get_cq_event(self.cc, &cq, &ctx) + if rc != 0: + raise PyverbsRDMAErrno('Failed to get CQ event') + if cq != expected_cq.cq: + raise PyverbsRDMAErrno('Received event on an unexpected CQ') + + +cdef class CQ(PyverbsCM): + """ + A Completion Queue is the notification mechanism for work request + completions. A CQ can have 0 or more associated QPs. + """ + def __cinit__(self, Context context not None, cqe, cq_context, + CompChannel channel, comp_vector): + """ + Initializes a CQ object with the given parameters. + :param context: The device's context on which to open the CQ + :param cqe: CQ's capacity + :param cq_context: User context's pointer + :param channel: If set, will be used to return completion events + :param comp_vector: Will be used for signaling completion events. + Must be larger than 0 and smaller than the + context's num_comp_vectors + :return: The newly created CQ + """ + self.cq = v.ibv_create_cq(context.context, cqe, cq_context, + NULL, comp_vector) + if self.cq == NULL: + raise PyverbsRDMAErrno('Failed to create a CQ') + self.context = context + context.add_ref(self) + self.logger.debug('Created a CQ') + + def __dealloc__(self): + self.close() + + cpdef close(self): + self.logger.debug('Closing CQ') + if self.cq != NULL: + rc = v.ibv_destroy_cq(self.cq) + if rc != 0: + raise PyverbsRDMAErrno('Failed to close CQ') + self.cq = NULL + self.context = None + + def poll(self, num_entries=1): + """ + Polls the CQ for completions. + :param num_entries: number of completions to pull + :return: (npolled, wcs): The number of polled completions and an array + of the polled completions + """ + cdef v.ibv_wc wc + wcs = [] + npolled = 0 + + while npolled < num_entries: + rc = v.ibv_poll_cq(self.cq, 1, &wc) + if rc < 0: + raise PyverbsRDMAErrno('Failed to poll CQ') + if rc == 0: + break; + npolled += 1 + wcs.append(WC(wr_id=wc.wr_id, status=wc.status, opcode=wc.opcode, + vendor_err=wc.vendor_err, byte_len=wc.byte_len, + qp_num=wc.qp_num, src_qp=wc.src_qp, + imm_data=wc.imm_data, wc_flags=wc.wc_flags, + pkey_index=wc.pkey_index, slid=wc.slid, sl=wc.sl, + dlid_path_bits=wc.dlid_path_bits)) + return npolled, wcs + + @property + def _cq(self): + return self.cq + + def __str__(self): + print_format = '{:22}: {:<20}\n' + return 'CQ\n' +\ + print_format.format('Handle', self.cq.handle) +\ + print_format.format('CQEs', self.cq.cqe) + + +cdef class WC(PyverbsObject): + def __cinit__(self, wr_id=0, status=0, opcode=0, vendor_err=0, byte_len=0, + qp_num=0, src_qp=0, imm_data=0, wc_flags=0, pkey_index=0, + slid=0, sl=0, dlid_path_bits=0): + self.wr_id = wr_id + self.status = status + self.opcode = opcode + self.vendor_err = vendor_err + self.byte_len = byte_len + self.qp_num = qp_num + self.src_qp = src_qp + self.wc_flags = wc_flags + self.pkey_index = pkey_index + self.slid = slid + self.sl = sl + self.dlid_path_bits = dlid_path_bits + + @property + def wr_id(self): + return self.wc.wr_id + @wr_id.setter + def wr_id(self, val): + self.wc.wr_id = val + + @property + def status(self): + return self.wc.status + @status.setter + def status(self, val): + self.wc.status = val + + @property + def opcode(self): + return self.wc.opcode + @opcode.setter + def opcode(self, val): + self.wc.opcode = val + + @property + def vendor_err(self): + return self.wc.vendor_err + @vendor_err.setter + def vendor_err(self, val): + self.wc.vendor_err = val + + @property + def byte_len(self): + return self.wc.byte_len + @byte_len.setter + def byte_len(self, val): + self.wc.byte_len = val + + @property + def qp_num(self): + return self.wc.qp_num + @qp_num.setter + def qp_num(self, val): + self.wc.qp_num = val + + @property + def src_qp(self): + return self.wc.src_qp + @src_qp.setter + def src_qp(self, val): + self.wc.src_qp = val + + @property + def wc_flags(self): + return self.wc.wc_flags + @wc_flags.setter + def wc_flags(self, val): + self.wc.wc_flags = val + + @property + def pkey_index(self): + return self.wc.pkey_index + @pkey_index.setter + def pkey_index(self, val): + self.wc.pkey_index = val + + @property + def slid(self): + return self.wc.slid + @slid.setter + def slid(self, val): + self.wc.slid = val + + @property + def sl(self): + return self.wc.sl + @sl.setter + def sl(self, val): + self.wc.sl = val + + @property + def dlid_path_bits(self): + return self.wc.dlid_path_bits + @dlid_path_bits.setter + def dlid_path_bits(self, val): + self.wc.dlid_path_bits = val + + def __str__(self): + print_format = '{:22}: {:<20}\n' + return print_format.format('WR ID', self.wr_id) +\ + print_format.format('status', cqe_status_to_str(self.status)) +\ + print_format.format('opcode', cqe_opcode_to_str(self.opcode)) +\ + print_format.format('vendor error', self.vendor_err) +\ + print_format.format('byte length', self.byte_len) +\ + print_format.format('QP num', self.qp_num) +\ + print_format.format('source QP', self.src_qp) +\ + print_format.format('WC flags', cqe_flags_to_str(self.wc_flags)) +\ + print_format.format('pkey index', self.pkey_index) +\ + print_format.format('slid', self.slid) +\ + print_format.format('sl', self.sl) +\ + print_format.format('dlid path bits', self.dlid_path_bits) + + +def cqe_status_to_str(status): + try: + return {e.IBV_WC_SUCCESS: "success", + e.IBV_WC_LOC_LEN_ERR: "local length error", + e.IBV_WC_LOC_QP_OP_ERR: "local QP op error", + e.IBV_WC_LOC_EEC_OP_ERR: "local EEC op error", + e.IBV_WC_LOC_PROT_ERR: "local protection error", + e.IBV_WC_WR_FLUSH_ERR: "WR flush error", + e.IBV_WC_MW_BIND_ERR: "memory window bind error", + e.IBV_WC_BAD_RESP_ERR: "bad response error", + e.IBV_WC_LOC_ACCESS_ERR: "local access error", + e.IBV_WC_REM_INV_REQ_ERR: "remote invalidate request error", + e.IBV_WC_REM_ACCESS_ERR: "remote access error", + e.IBV_WC_REM_OP_ERR: "remote op error", + e.IBV_WC_RETRY_EXC_ERR: "retry exceeded error", + e.IBV_WC_RNR_RETRY_EXC_ERR: "RNR retry exceeded", + e.IBV_WC_LOC_RDD_VIOL_ERR: "local RDD violation error", + e.IBV_WC_REM_INV_RD_REQ_ERR: "remote invalidate RD request error", + e.IBV_WC_REM_ABORT_ERR: "remote abort error", + e.IBV_WC_INV_EECN_ERR: "invalidate EECN error", + e.IBV_WC_INV_EEC_STATE_ERR: "invalidate EEC state error", + e.IBV_WC_FATAL_ERR: "WC fatal error", + e.IBV_WC_RESP_TIMEOUT_ERR: "response timeout error", + e.IBV_WC_GENERAL_ERR: "general error"}[status] + except KeyError: + return "Unknown CQE status" + +def cqe_opcode_to_str(opcode): + try: + return {0x0: "Send", 0x1:"RDMA write", 0x2: "RDMA read", + 0x3: "Compare and swap", 0x4: "Fetch and add", + 0x5: "Bind Memory window", 0x6: "Local invalidate", + 0x7: "TSO", 0x80: "Receive", + 0x81: "Receive RDMA with immediate", + 0x82: "Tag matching - add", 0x83: "Tag matching - delete", + 0x84: "Tag matching - sync", 0x85: "Tag matching - receive", + 0x86: "Tag matching - no tag"}[opcode] + except KeyError: + return "Unknown CQE opcode {op}".format(op=opcode) + +def cqe_flags_to_str(flags): + cqe_flags = {1: "GRH", 2: "With immediate", 4: "IP csum OK", + 8: "With invalidate", 16: "TM sync request", 32: "TM match", + 64: "TM data valid"} + flags_str = "" + for f in cqe_flags: + if flags & f: + flags_str += cqe_flags[f] + flags_str += " " + return flags_str diff --git a/pyverbs/device.pxd b/pyverbs/device.pxd index a7089972985b..aecdc0c1d254 100644 --- a/pyverbs/device.pxd +++ b/pyverbs/device.pxd @@ -11,6 +11,8 @@ cdef class Context(PyverbsCM): cdef add_ref(self, obj) cdef object pds cdef object dms + cdef object ccs + cdef object cqs cdef class DeviceAttr(PyverbsObject): cdef v.ibv_device_attr dev_attr diff --git a/pyverbs/device.pyx b/pyverbs/device.pyx index 757eebf67c48..3808f8600840 100644 --- a/pyverbs/device.pyx +++ b/pyverbs/device.pyx @@ -11,6 +11,7 @@ import weakref from .pyverbs_error import PyverbsRDMAError, PyverbsError from .pyverbs_error import PyverbsUserError from pyverbs.base import PyverbsRDMAErrno +from pyverbs.cq cimport CQ, CompChannel cimport pyverbs.libibverbs_enums as e cimport pyverbs.libibverbs as v from pyverbs.addr cimport GID @@ -85,6 +86,8 @@ cdef class Context(PyverbsCM): self.pds = weakref.WeakSet() self.dms = weakref.WeakSet() + self.ccs = weakref.WeakSet() + self.cqs = weakref.WeakSet() dev_name = kwargs.get('name') @@ -121,7 +124,7 @@ cdef class Context(PyverbsCM): cpdef close(self): self.logger.debug('Closing Context') - self.close_weakrefs([self.dms, self.pds]) + self.close_weakrefs([self.ccs, self.cqs, self.dms, self.pds]) if self.context != NULL: rc = v.ibv_close_device(self.context) if rc != 0: @@ -129,6 +132,10 @@ cdef class Context(PyverbsCM): format(dev=self.device.name), errno) self.context = NULL + @property + def num_comp_vectors(self): + return self.context.num_comp_vectors + def query_device(self): """ Queries the device's attributes. @@ -183,6 +190,10 @@ cdef class Context(PyverbsCM): self.pds.add(obj) elif isinstance(obj, DM): self.dms.add(obj) + elif isinstance(obj, CompChannel): + self.ccs.add(obj) + elif isinstance(obj, CQ): + self.cqs.add(obj) else: raise PyverbsError('Unrecognized object type') diff --git a/pyverbs/libibverbs.pxd b/pyverbs/libibverbs.pxd index c6b564c4d487..304dce5150d2 100644 --- a/pyverbs/libibverbs.pxd +++ b/pyverbs/libibverbs.pxd @@ -21,6 +21,7 @@ cdef extern from 'infiniband/verbs.h': cdef struct ibv_context: ibv_device *device + int num_comp_vectors cdef struct ibv_device_attr: char *fw_ver @@ -172,6 +173,33 @@ cdef extern from 'infiniband/verbs.h': unsigned char flags unsigned short port_cap_flags2 + cdef struct ibv_comp_channel: + ibv_context *context + unsigned int fd + unsigned int refcnt + + cdef struct ibv_cq: + ibv_context *context + ibv_comp_channel *channel + void *cq_context + int handle + int cqe + + cdef struct ibv_wc: + unsigned long wr_id + ibv_wc_status status + ibv_wc_opcode opcode + unsigned int vendor_err + unsigned int byte_len + unsigned int qp_num + unsigned int imm_data + unsigned int src_qp + int wc_flags + unsigned int pkey_index + unsigned int slid + unsigned int sl + unsigned int dlid_path_bits + ibv_device **ibv_get_device_list(int *n) void ibv_free_device_list(ibv_device **list) ibv_context *ibv_open_device(ibv_device *device) @@ -199,3 +227,11 @@ cdef extern from 'infiniband/verbs.h': size_t length) int ibv_query_port(ibv_context *context, uint8_t port_num, ibv_port_attr *port_attr) + ibv_comp_channel *ibv_create_comp_channel(ibv_context *context) + int ibv_destroy_comp_channel(ibv_comp_channel *channel) + int ibv_get_cq_event(ibv_comp_channel *channel, ibv_cq **cq, + void **cq_context) + ibv_cq *ibv_create_cq(ibv_context *context, int cqe, void *cq_context, + ibv_comp_channel *channel, int comp_vector) + int ibv_destroy_cq(ibv_cq *cq) + int ibv_poll_cq(ibv_cq *cq, int num_entries, ibv_wc *wc) From patchwork Sun Mar 24 14:10:36 2019 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Noa Osherovich X-Patchwork-Id: 10867443 Return-Path: Received: from mail.wl.linuxfoundation.org (pdx-wl-mail.web.codeaurora.org [172.30.200.125]) by pdx-korg-patchwork-2.web.codeaurora.org (Postfix) with ESMTP id 348111390 for ; Sun, 24 Mar 2019 14:11:27 +0000 (UTC) Received: from mail.wl.linuxfoundation.org (localhost [127.0.0.1]) by mail.wl.linuxfoundation.org (Postfix) with ESMTP id 207BA2940A for ; Sun, 24 Mar 2019 14:11:27 +0000 (UTC) Received: by mail.wl.linuxfoundation.org (Postfix, from userid 486) id 14C75294B6; Sun, 24 Mar 2019 14:11:27 +0000 (UTC) X-Spam-Checker-Version: SpamAssassin 3.3.1 (2010-03-16) on pdx-wl-mail.web.codeaurora.org X-Spam-Level: X-Spam-Status: No, score=-7.9 required=2.0 tests=BAYES_00,MAILING_LIST_MULTI, RCVD_IN_DNSWL_HI,UNPARSEABLE_RELAY autolearn=ham version=3.3.1 Received: from vger.kernel.org (vger.kernel.org [209.132.180.67]) by mail.wl.linuxfoundation.org (Postfix) with ESMTP id 865662940A for ; Sun, 24 Mar 2019 14:11:26 +0000 (UTC) Received: (majordomo@vger.kernel.org) by vger.kernel.org via listexpand id S1726877AbfCXOLZ (ORCPT ); Sun, 24 Mar 2019 10:11:25 -0400 Received: from mail-il-dmz.mellanox.com ([193.47.165.129]:53733 "EHLO mellanox.co.il" rhost-flags-OK-OK-OK-FAIL) by vger.kernel.org with ESMTP id S1726317AbfCXOLZ (ORCPT ); Sun, 24 Mar 2019 10:11:25 -0400 Received: from Internal Mail-Server by MTLPINE1 (envelope-from noaos@mellanox.com) with ESMTPS (AES256-SHA encrypted); 24 Mar 2019 16:11:21 +0200 Received: from reg-l-vrt-059-009.mtl.labs.mlnx (reg-l-vrt-059-009.mtl.labs.mlnx [10.135.59.9]) by labmailer.mlnx (8.13.8/8.13.8) with ESMTP id x2OEAxHu028186; Sun, 24 Mar 2019 16:11:21 +0200 From: Noa Osherovich To: leon@kernel.org, jgg@mellanox.com, dledford@redhat.com Cc: linux-rdma@vger.kernel.org, Noa Osherovich Subject: [PATCH rdma-core 2/6] pyverbs: Add unittests for completion-related classes Date: Sun, 24 Mar 2019 16:10:36 +0200 Message-Id: <20190324141040.22204-3-noaos@mellanox.com> X-Mailer: git-send-email 2.17.2 In-Reply-To: <20190324141040.22204-1-noaos@mellanox.com> References: <20190324141040.22204-1-noaos@mellanox.com> Sender: linux-rdma-owner@vger.kernel.org Precedence: bulk List-ID: X-Mailing-List: linux-rdma@vger.kernel.org X-Virus-Scanned: ClamAV using ClamSMTP Add unittests for creation and destruction of CQ an CompChannel: - Creation and deletion of a CompChannel (good flow). - Creation and deletion of a CQ (good flow), with and without a completion channel. - Bad flow checks: Verify failure for illegal comp_vector / cqe number. Signed-off-by: Noa Osherovich Reviewed-by: Maor Gottlieb Reviewed-by: Leon Romanovsky --- pyverbs/CMakeLists.txt | 1 + pyverbs/tests/cq.py | 130 +++++++++++++++++++++++++++++++++++++++++ 2 files changed, 131 insertions(+) create mode 100644 pyverbs/tests/cq.py diff --git a/pyverbs/CMakeLists.txt b/pyverbs/CMakeLists.txt index ef5114594e20..d83f77317ecb 100644 --- a/pyverbs/CMakeLists.txt +++ b/pyverbs/CMakeLists.txt @@ -19,6 +19,7 @@ rdma_python_module(pyverbs rdma_python_test(pyverbs/tests tests/__init__.py + tests/cq.py tests/device.py tests/mr.py tests/pd.py diff --git a/pyverbs/tests/cq.py b/pyverbs/tests/cq.py new file mode 100644 index 000000000000..afe732e5e93c --- /dev/null +++ b/pyverbs/tests/cq.py @@ -0,0 +1,130 @@ +# SPDX-License-Identifier: (GPL-2.0 OR Linux-OpenIB) +# Copyright (c) 2019, Mellanox Technologies. All rights reserved. See COPYING file +""" +Test module for pyverbs' cq module. +""" +import unittest +import random + +from pyverbs.pyverbs_error import PyverbsError +from pyverbs.cq import CompChannel, CQ +import pyverbs.device as d +import pyverbs.enums as e + + +class CQTest(unittest.TestCase): + """ + Test various functionalities of the CQ class. + """ + + @staticmethod + def test_create_cq(): + """ + Test ibv_create_cq() + """ + lst = d.get_device_list() + for dev in lst: + with d.Context(name=dev.name.decode()) as ctx: + cqes = get_num_cqes(ctx) + comp_vector = random.randint(0, ctx.num_comp_vectors - 1) + if random.choice([True, False]): + with CompChannel(ctx) as cc: + with CQ(ctx, cqes, None, cc, comp_vector): + pass + else: + with CQ(ctx, cqes, None, None, comp_vector): + pass + + @staticmethod + def test_create_cq_bad_flow(): + """ + Test ibv_create_cq() with a wrong comp_vector / cqe number + """ + lst = d.get_device_list() + for dev in lst: + with d.Context(name=dev.name.decode()) as ctx: + cqes = get_num_cqes(ctx) + comp_vector = random.randint(ctx.num_comp_vectors, 100) + try: + if random.choice([True, False]): + with CompChannel(ctx) as cc: + with CQ(ctx, cqes, None, cc, comp_vector): + pass + else: + with CQ(ctx, cqes, None, None, comp_vector): + pass + except PyverbsError as e: + assert 'Failed to create a CQ' in e.args[0] + assert 'Invalid argument' in e.args[0] + else: + raise PyverbsError( + 'Created a CQ with comp_vector={n} while device\'s num_comp_vectors={nc}'. + format(n=comp_vector, nc=ctx.num_comp_vectors)) + max_cqe = ctx.query_device().max_cqe + cqes = random.randint(max_cqe + 1, max_cqe + 100) + try: + if random.choice([True, False]): + with CompChannel(ctx) as cc: + with CQ(ctx, cqes, None, cc, 0): + pass + else: + with CQ(ctx, cqes, None, None, 0): + pass + except PyverbsError as err: + assert 'Failed to create a CQ' in err.args[0] + assert 'Invalid argument' in err.args[0] + else: + raise PyverbsError( + 'Created a CQ with cqe={n} while device\'s max_cqe={nc}'. + format(n=cqes, nc=max_cqe)) + + @staticmethod + def test_destroy_cq(): + """ + Test ibv_destroy_cq() + """ + lst = d.get_device_list() + for dev in lst: + with d.Context(name=dev.name.decode()) as ctx: + cqes = get_num_cqes(ctx) + comp_vector = random.randint(0, ctx.num_comp_vectors - 1) + if random.choice([True, False]): + with CompChannel(ctx) as cc: + cq = CQ(ctx, cqes, None, cc, comp_vector) + else: + cq = CQ(ctx, cqes, None, None, comp_vector) + cq.close() + + +class CCTest(unittest.TestCase): + """ + Test various functionalities of the Completion Channel class. + """ + + @staticmethod + def test_create_comp_channel(): + """ + Test ibv_create_comp_channel() + """ + lst = d.get_device_list() + for dev in lst: + with d.Context(name=dev.name.decode()) as ctx: + with CompChannel(ctx): + pass + + @staticmethod + def test_destroy_comp_channel(): + """ + Test ibv_destroy_comp_channel() + """ + lst = d.get_device_list() + for dev in lst: + with d.Context(name=dev.name.decode()) as ctx: + cc = CompChannel(ctx) + cc.close() + + +def get_num_cqes(ctx): + attr = ctx.query_device() + max_cqe = attr.max_cqe + return random.randint(0, max_cqe) From patchwork Sun Mar 24 14:10:37 2019 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Noa Osherovich X-Patchwork-Id: 10867445 Return-Path: Received: from mail.wl.linuxfoundation.org (pdx-wl-mail.web.codeaurora.org [172.30.200.125]) by pdx-korg-patchwork-2.web.codeaurora.org (Postfix) with ESMTP id 4FB121390 for ; Sun, 24 Mar 2019 14:11:29 +0000 (UTC) Received: from mail.wl.linuxfoundation.org (localhost [127.0.0.1]) by mail.wl.linuxfoundation.org (Postfix) with ESMTP id 37ED72940A for ; Sun, 24 Mar 2019 14:11:29 +0000 (UTC) Received: by mail.wl.linuxfoundation.org (Postfix, from userid 486) id 2AED4294B6; Sun, 24 Mar 2019 14:11:29 +0000 (UTC) X-Spam-Checker-Version: SpamAssassin 3.3.1 (2010-03-16) on pdx-wl-mail.web.codeaurora.org X-Spam-Level: X-Spam-Status: No, score=-7.9 required=2.0 tests=BAYES_00,MAILING_LIST_MULTI, RCVD_IN_DNSWL_HI,UNPARSEABLE_RELAY autolearn=ham version=3.3.1 Received: from vger.kernel.org (vger.kernel.org [209.132.180.67]) by mail.wl.linuxfoundation.org (Postfix) with ESMTP id 15E022940A for ; Sun, 24 Mar 2019 14:11:28 +0000 (UTC) Received: (majordomo@vger.kernel.org) by vger.kernel.org via listexpand id S1726317AbfCXOL1 (ORCPT ); Sun, 24 Mar 2019 10:11:27 -0400 Received: from mail-il-dmz.mellanox.com ([193.47.165.129]:53738 "EHLO mellanox.co.il" rhost-flags-OK-OK-OK-FAIL) by vger.kernel.org with ESMTP id S1726811AbfCXOL1 (ORCPT ); Sun, 24 Mar 2019 10:11:27 -0400 Received: from Internal Mail-Server by MTLPINE1 (envelope-from noaos@mellanox.com) with ESMTPS (AES256-SHA encrypted); 24 Mar 2019 16:11:22 +0200 Received: from reg-l-vrt-059-009.mtl.labs.mlnx (reg-l-vrt-059-009.mtl.labs.mlnx [10.135.59.9]) by labmailer.mlnx (8.13.8/8.13.8) with ESMTP id x2OEAxHv028186; Sun, 24 Mar 2019 16:11:22 +0200 From: Noa Osherovich To: leon@kernel.org, jgg@mellanox.com, dledford@redhat.com Cc: linux-rdma@vger.kernel.org, Noa Osherovich Subject: [PATCH rdma-core 3/6] pyverbs: Introducing extended completions related classes Date: Sun, 24 Mar 2019 16:10:37 +0200 Message-Id: <20190324141040.22204-4-noaos@mellanox.com> X-Mailer: git-send-email 2.17.2 In-Reply-To: <20190324141040.22204-1-noaos@mellanox.com> References: <20190324141040.22204-1-noaos@mellanox.com> Sender: linux-rdma-owner@vger.kernel.org Precedence: bulk List-ID: X-Mailing-List: linux-rdma@vger.kernel.org X-Virus-Scanned: ClamAV using ClamSMTP Add support for extended CQ which includes: - CqInitAttrEx class - CQEX Signed-off-by: Noa Osherovich Reviewed-by: Maor Gottlieb Reviewed-by: Leon Romanovsky --- pyverbs/cq.pxd | 15 +++ pyverbs/cq.pyx | 249 ++++++++++++++++++++++++++++++++++- pyverbs/device.pyx | 4 +- pyverbs/libibverbs.pxd | 50 +++++++ pyverbs/libibverbs_enums.pxd | 31 +++-- 5 files changed, 330 insertions(+), 19 deletions(-) diff --git a/pyverbs/cq.pxd b/pyverbs/cq.pxd index 82607042ce6b..181f7da668fe 100644 --- a/pyverbs/cq.pxd +++ b/pyverbs/cq.pxd @@ -13,5 +13,20 @@ cdef class CQ(PyverbsCM): cpdef close(self) cdef object context +cdef class CqInitAttrEx(PyverbsObject): + cdef v.ibv_cq_init_attr_ex attr + +cdef class CQEX(PyverbsCM): + cdef v.ibv_cq_ex *cq + cdef v.ibv_cq *ibv_cq + cpdef close(self) + cdef object context + cdef class WC(PyverbsObject): cdef v.ibv_wc wc + +cdef class PollCqAttr(PyverbsObject): + cdef v.ibv_poll_cq_attr attr + +cdef class WcTmInfo(PyverbsObject): + cdef v.ibv_wc_tm_info info diff --git a/pyverbs/cq.pyx b/pyverbs/cq.pyx index d1e2520f6be1..748e30fe9bcb 100644 --- a/pyverbs/cq.pyx +++ b/pyverbs/cq.pyx @@ -126,6 +126,199 @@ cdef class CQ(PyverbsCM): print_format.format('CQEs', self.cq.cqe) +cdef class CqInitAttrEx(PyverbsObject): + def __cinit__(self, cqe = 100, CompChannel channel = None, comp_vector = 0, + wc_flags = 0, comp_mask = 0, flags = 0): + """ + Initializes a CqInitAttrEx object with the given parameters. + :param cqe: CQ's capacity + :param channel: If set, will be used to return completion events + :param comp_vector: Will be used for signaling completion events. + Must be larger than 0 and smaller than the + context's num_comp_vectors + :param wc_flags: The wc_flags that should be returned in ibv_poll_cq_ex. + Or'ed bit of enum ibv_wc_flags_ex. + :param comp_mask: compatibility mask (extended verb) + :param flags: create cq attr flags - one or more flags from + ibv_create_cq_attr_flags enum + :return: + """ + self.attr.cqe = cqe + self.attr.cq_context = NULL + self.attr.channel = NULL if channel is None else channel.cc + self.attr.comp_vector = comp_vector + self.attr.wc_flags = wc_flags + self.attr.comp_mask = comp_mask + self.attr.flags = flags + + @property + def cqe(self): + return self.attr.cqe + @cqe.setter + def cqe(self, val): + self.attr.cqe = val + + # Setter-only properties require the older syntax + property cq_context: + def __set__(self, val): + self.attr.cq_context = val + + property channel: + def __set__(self, val): + self.attr.channel = val + + @property + def comp_vector(self): + return self.attr.comp_vector + @comp_vector.setter + def comp_vector(self, val): + self.attr.comp_vector = val + + @property + def wc_flags(self): + return self.attr.wc_flags + @wc_flags.setter + def wc_flags(self, val): + self.attr.wc_flags = val + + @property + def comp_mask(self): + return self.attr.comp_mask + @comp_mask.setter + def comp_mask(self, val): + self.attr.comp_mask = val + + @property + def flags(self): + return self.attr.flags + @flags.setter + def flags(self, val): + self.attr.flags = val + + def __str__(self): + print_format = '{:22}: {:<20}\n' + return print_format.format('Number of CQEs', self.cqe) +\ + print_format.format('WC flags', create_wc_flags_to_str(self.wc_flags)) +\ + print_format.format('comp mask', self.comp_mask) +\ + print_format.format('flags', self.flags) + + +cdef class CQEX(PyverbsCM): + def __cinit__(self, Context context not None, CqInitAttrEx init_attr): + """ + Initializes a CQEX object on the given device's context with the given + attributes. + :param context: The device's context on which to open the CQ + :param init_attr: Initial attributes that describe the CQ + :return: The newly created CQEX on success + """ + if init_attr is None: + init_attr = CqInitAttrEx() + self.cq = v.ibv_create_cq_ex(context.context, &init_attr.attr) + if self.cq == NULL: + raise PyverbsRDMAErrno('Failed to create extended CQ') + self.ibv_cq = v.ibv_cq_ex_to_cq(self.cq) + self.context = context + context.add_ref(self) + + def __dealloc__(self): + self.close() + + cpdef close(self): + self.logger.debug('Closing CQEx') + if self.cq != NULL: + rc = v.ibv_destroy_cq(self.cq) + if rc != 0: + raise PyverbsRDMAErrno('Failed to destroy CQEX') + self.cq = NULL + self.context = None + + def start_poll(self, PollCqAttr attr): + """ + Start polling a batch of work completions. + :param attr: For easy future extensions + :return: 0 on success, ENOENT when no completions are available + """ + if attr is None: + attr = PollCqAttr() + return v.ibv_start_poll(self.cq, &attr.attr) + + def poll_next(self): + """ + Get the next work completion. + :return: 0 on success, ENOENT when no completions are available + """ + return v.ibv_next_poll(self.cq) + + def end_poll(self): + """ + Indicates the end of polling batch of work completions + :return: None + """ + return v.ibv_end_poll(self.cq) + + def read_opcode(self): + return v.ibv_wc_read_opcode(self.cq) + def read_vendor_err(self): + return v.ibv_wc_read_vendor_err(self.cq) + def read_byte_len(self): + return v.ibv_wc_read_byte_len(self.cq) + def read_imm_data(self): + return v.ibv_wc_read_imm_data(self.cq) + def read_qp_num(self): + return v.ibv_wc_read_qp_num(self.cq) + def read_src_qp(self): + return v.ibv_wc_read_src_qp(self.cq) + def read_wc_flags(self): + return v.ibv_wc_read_wc_flags(self.cq) + def read_slid(self): + return v.ibv_wc_read_slid(self.cq) + def read_sl(self): + return v.ibv_wc_read_sl(self.cq) + def read_dlid_path_bits(self): + return v.ibv_wc_read_dlid_path_bits(self.cq) + def read_timestamp(self): + return v.ibv_wc_read_completion_ts(self.cq) + def read_cvlan(self): + return v.ibv_wc_read_cvlan(self.cq) + def read_flow_tag(self): + return v.ibv_wc_read_flow_tag(self.cq) + def read_tm_info(self): + info = WcTmInfo() + v.ibv_wc_read_tm_info(self.cq, &info.info) + return info + def read_completion_wallclock_ns(self): + return v.ibv_wc_read_completion_wallclock_ns(self.cq) + + @property + def status(self): + return self.cq.status + @status.setter + def status(self, val): + self.cq.status = val + + @property + def wr_id(self): + return self.cq.wr_id + @wr_id.setter + def wr_id(self, val): + self.cq.wr_id = val + + @property + def _cq(self): + return self.cq + + @property + def _ibv_cq(self): + return self.ibv_cq + + def __str__(self): + print_format = '{:<22}: {:<20}\n' + return 'Extended CQ:\n' +\ + print_format.format('Handle', self.cq.handle) +\ + print_format.format('CQEs', self.cq.cqe) + + cdef class WC(PyverbsObject): def __cinit__(self, wr_id=0, status=0, opcode=0, vendor_err=0, byte_len=0, qp_num=0, src_qp=0, imm_data=0, wc_flags=0, pkey_index=0, @@ -243,6 +436,31 @@ cdef class WC(PyverbsObject): print_format.format('dlid path bits', self.dlid_path_bits) +cdef class PollCqAttr(PyverbsObject): + @property + def comp_mask(self): + return self.attr.comp_mask + @comp_mask.setter + def comp_mask(self, val): + self.attr.comp_mask = val + + +cdef class WcTmInfo(PyverbsObject): + @property + def tag(self): + return self.info.tag + @tag.setter + def tag(self, val): + self.info.tag = val + + @property + def priv(self): + return self.info.priv + @priv.setter + def priv(self, val): + self.info.priv = val + + def cqe_status_to_str(status): try: return {e.IBV_WC_SUCCESS: "success", @@ -283,13 +501,32 @@ def cqe_opcode_to_str(opcode): except KeyError: return "Unknown CQE opcode {op}".format(op=opcode) -def cqe_flags_to_str(flags): - cqe_flags = {1: "GRH", 2: "With immediate", 4: "IP csum OK", - 8: "With invalidate", 16: "TM sync request", 32: "TM match", - 64: "TM data valid"} +def flags_to_str(flags, dictionary): flags_str = "" - for f in cqe_flags: + for f in dictionary: if flags & f: - flags_str += cqe_flags[f] + flags_str += dictionary[f] flags_str += " " return flags_str + + +def cqe_flags_to_str(flags): + cqe_flags = {1: "GRH", 2: "With immediate", 4: "IP csum OK", + 8: "With invalidate", 16: "TM sync request", 32: "TM match", + 64: "TM data valid"} + return flags_to_str(flags, cqe_flags) + +def create_wc_flags_to_str(flags): + cqe_flags = {e.IBV_WC_EX_WITH_BYTE_LEN: 'IBV_WC_EX_WITH_BYTE_LEN', + e.IBV_WC_EX_WITH_IMM: 'IBV_WC_EX_WITH_IMM', + e.IBV_WC_EX_WITH_QP_NUM: 'IBV_WC_EX_WITH_QP_NUM', + e.IBV_WC_EX_WITH_SRC_QP: 'IBV_WC_EX_WITH_SRC_QP', + e.IBV_WC_EX_WITH_SLID: 'IBV_WC_EX_WITH_SLID', + e.IBV_WC_EX_WITH_SL: 'IBV_WC_EX_WITH_SL', + e.IBV_WC_EX_WITH_DLID_PATH_BITS: 'IBV_WC_EX_WITH_DLID_PATH_BITS', + e.IBV_WC_EX_WITH_COMPLETION_TIMESTAMP: 'IBV_WC_EX_WITH_COMPLETION_TIMESTAMP', + e.IBV_WC_EX_WITH_CVLAN: 'IBV_WC_EX_WITH_CVLAN', + e.IBV_WC_EX_WITH_FLOW_TAG: 'IBV_WC_EX_WITH_FLOW_TAG', + e.IBV_WC_EX_WITH_TM_INFO: 'IBV_WC_EX_WITH_TM_INFO', + e.IBV_WC_EX_WITH_COMPLETION_TIMESTAMP_WALLCLOCK: 'IBV_WC_EX_WITH_COMPLETION_TIMESTAMP_WALLCLOCK'} + return flags_to_str(flags, cqe_flags) diff --git a/pyverbs/device.pyx b/pyverbs/device.pyx index 3808f8600840..e953b7fc2eaf 100644 --- a/pyverbs/device.pyx +++ b/pyverbs/device.pyx @@ -9,9 +9,9 @@ which returns a DeviceAttr object. import weakref from .pyverbs_error import PyverbsRDMAError, PyverbsError +from pyverbs.cq cimport CQEX, CQ, CompChannel from .pyverbs_error import PyverbsUserError from pyverbs.base import PyverbsRDMAErrno -from pyverbs.cq cimport CQ, CompChannel cimport pyverbs.libibverbs_enums as e cimport pyverbs.libibverbs as v from pyverbs.addr cimport GID @@ -192,7 +192,7 @@ cdef class Context(PyverbsCM): self.dms.add(obj) elif isinstance(obj, CompChannel): self.ccs.add(obj) - elif isinstance(obj, CQ): + elif isinstance(obj, CQ) or isinstance(obj, CQEX): self.cqs.add(obj) else: raise PyverbsError('Unrecognized object type') diff --git a/pyverbs/libibverbs.pxd b/pyverbs/libibverbs.pxd index 304dce5150d2..7facff5f8856 100644 --- a/pyverbs/libibverbs.pxd +++ b/pyverbs/libibverbs.pxd @@ -200,6 +200,34 @@ cdef extern from 'infiniband/verbs.h': unsigned int sl unsigned int dlid_path_bits + cdef struct ibv_cq_init_attr_ex: + unsigned int cqe + void *cq_context + ibv_comp_channel *channel + unsigned int comp_vector + unsigned long wc_flags + unsigned int comp_mask + unsigned int flags + + cdef struct ibv_cq_ex: + ibv_context *context + ibv_comp_channel *channel + void *cq_context + unsigned int handle + int cqe + unsigned int comp_events_completed + unsigned int async_events_completed + unsigned int comp_mask + ibv_wc_status status + unsigned long wr_id + + cdef struct ibv_poll_cq_attr: + unsigned int comp_mask + + cdef struct ibv_wc_tm_info: + unsigned long tag + unsigned int priv + ibv_device **ibv_get_device_list(int *n) void ibv_free_device_list(ibv_device **list) ibv_context *ibv_open_device(ibv_device *device) @@ -235,3 +263,25 @@ cdef extern from 'infiniband/verbs.h': ibv_comp_channel *channel, int comp_vector) int ibv_destroy_cq(ibv_cq *cq) int ibv_poll_cq(ibv_cq *cq, int num_entries, ibv_wc *wc) + ibv_cq_ex *ibv_create_cq_ex(ibv_context *context, + ibv_cq_init_attr_ex *cq_attr) + ibv_cq *ibv_cq_ex_to_cq(ibv_cq_ex *cq) + int ibv_start_poll(ibv_cq_ex *cq, ibv_poll_cq_attr *attr) + int ibv_next_poll(ibv_cq_ex *cq) + void ibv_end_poll(ibv_cq_ex *cq) + ibv_wc_opcode ibv_wc_read_opcode(ibv_cq_ex *cq) + unsigned int ibv_wc_read_vendor_err(ibv_cq_ex *cq) + unsigned int ibv_wc_read_byte_len(ibv_cq_ex *cq) + unsigned int ibv_wc_read_imm_data(ibv_cq_ex *cq) + unsigned int ibv_wc_read_invalidated_rkey(ibv_cq_ex *cq) + unsigned int ibv_wc_read_qp_num(ibv_cq_ex *cq) + unsigned int ibv_wc_read_src_qp(ibv_cq_ex *cq) + unsigned int ibv_wc_read_wc_flags(ibv_cq_ex *cq) + unsigned int ibv_wc_read_slid(ibv_cq_ex *cq) + unsigned char ibv_wc_read_sl(ibv_cq_ex *cq) + unsigned char ibv_wc_read_dlid_path_bits(ibv_cq_ex *cq) + unsigned long ibv_wc_read_completion_ts(ibv_cq_ex *cq) + unsigned short ibv_wc_read_cvlan(ibv_cq_ex *cq) + unsigned int ibv_wc_read_flow_tag(ibv_cq_ex *cq) + void ibv_wc_read_tm_info(ibv_cq_ex *cq, ibv_wc_tm_info *tm_info) + unsigned long ibv_wc_read_completion_wallclock_ns(ibv_cq_ex *cq) diff --git a/pyverbs/libibverbs_enums.pxd b/pyverbs/libibverbs_enums.pxd index 580252468e31..e429cd35cc47 100644 --- a/pyverbs/libibverbs_enums.pxd +++ b/pyverbs/libibverbs_enums.pxd @@ -173,17 +173,18 @@ cdef extern from '': IBV_WC_RECV_RDMA_WITH_IMM cpdef enum ibv_create_cq_wc_flags: - IBV_WC_EX_WITH_BYTE_LEN = 1 << 0 - IBV_WC_EX_WITH_IMM = 1 << 1 - IBV_WC_EX_WITH_QP_NUM = 1 << 2 - IBV_WC_EX_WITH_SRC_QP = 1 << 3 - IBV_WC_EX_WITH_SLID = 1 << 4 - IBV_WC_EX_WITH_SL = 1 << 5 - IBV_WC_EX_WITH_DLID_PATH_BITS = 1 << 6 - IBV_WC_EX_WITH_COMPLETION_TIMESTAMP = 1 << 7 - IBV_WC_EX_WITH_CVLAN = 1 << 8 - IBV_WC_EX_WITH_FLOW_TAG = 1 << 9 - IBV_WC_EX_WITH_TM_INFO = 1 << 10 + IBV_WC_EX_WITH_BYTE_LEN = 1 << 0 + IBV_WC_EX_WITH_IMM = 1 << 1 + IBV_WC_EX_WITH_QP_NUM = 1 << 2 + IBV_WC_EX_WITH_SRC_QP = 1 << 3 + IBV_WC_EX_WITH_SLID = 1 << 4 + IBV_WC_EX_WITH_SL = 1 << 5 + IBV_WC_EX_WITH_DLID_PATH_BITS = 1 << 6 + IBV_WC_EX_WITH_COMPLETION_TIMESTAMP = 1 << 7 + IBV_WC_EX_WITH_CVLAN = 1 << 8 + IBV_WC_EX_WITH_FLOW_TAG = 1 << 9 + IBV_WC_EX_WITH_TM_INFO = 1 << 10 + IBV_WC_EX_WITH_COMPLETION_TIMESTAMP_WALLCLOCK = 1 << 11 cpdef enum ibv_wc_flags: IBV_WC_GRH = 1 << 0 @@ -340,6 +341,14 @@ cdef extern from '': cpdef enum ibv_read_counters_flags: IBV_READ_COUNTERS_ATTR_PREFER_CACHED = 1 << 0 + cpdef enum ibv_cq_init_attr_mask: + IBV_CQ_INIT_ATTR_MASK_FLAGS = 1 << 0 + + cpdef enum ibv_create_cq_attr_flags: + IBV_CREATE_CQ_ATTR_SINGLE_THREADED = 1 << 0 + IBV_CREATE_CQ_ATTR_IGNORE_OVERRUN = 1 << 1 + + cdef extern from "": cpdef enum ibv_tmh_op: IBV_TMH_NO_TAG = 0 From patchwork Sun Mar 24 14:10:38 2019 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Noa Osherovich X-Patchwork-Id: 10867449 Return-Path: Received: from mail.wl.linuxfoundation.org (pdx-wl-mail.web.codeaurora.org [172.30.200.125]) by pdx-korg-patchwork-2.web.codeaurora.org (Postfix) with ESMTP id 00BB013B5 for ; Sun, 24 Mar 2019 14:11:32 +0000 (UTC) Received: from mail.wl.linuxfoundation.org (localhost [127.0.0.1]) by mail.wl.linuxfoundation.org (Postfix) with ESMTP id E12082940A for ; Sun, 24 Mar 2019 14:11:31 +0000 (UTC) Received: by mail.wl.linuxfoundation.org (Postfix, from userid 486) id D5A97294B6; Sun, 24 Mar 2019 14:11:31 +0000 (UTC) X-Spam-Checker-Version: SpamAssassin 3.3.1 (2010-03-16) on pdx-wl-mail.web.codeaurora.org X-Spam-Level: X-Spam-Status: No, score=-7.9 required=2.0 tests=BAYES_00,MAILING_LIST_MULTI, RCVD_IN_DNSWL_HI,UNPARSEABLE_RELAY autolearn=ham version=3.3.1 Received: from vger.kernel.org (vger.kernel.org [209.132.180.67]) by mail.wl.linuxfoundation.org (Postfix) with ESMTP id 6F7652940A for ; Sun, 24 Mar 2019 14:11:31 +0000 (UTC) Received: (majordomo@vger.kernel.org) by vger.kernel.org via listexpand id S1726889AbfCXOLa (ORCPT ); Sun, 24 Mar 2019 10:11:30 -0400 Received: from mail-il-dmz.mellanox.com ([193.47.165.129]:53762 "EHLO mellanox.co.il" rhost-flags-OK-OK-OK-FAIL) by vger.kernel.org with ESMTP id S1726811AbfCXOLa (ORCPT ); Sun, 24 Mar 2019 10:11:30 -0400 Received: from Internal Mail-Server by MTLPINE1 (envelope-from noaos@mellanox.com) with ESMTPS (AES256-SHA encrypted); 24 Mar 2019 16:11:24 +0200 Received: from reg-l-vrt-059-009.mtl.labs.mlnx (reg-l-vrt-059-009.mtl.labs.mlnx [10.135.59.9]) by labmailer.mlnx (8.13.8/8.13.8) with ESMTP id x2OEAxHw028186; Sun, 24 Mar 2019 16:11:24 +0200 From: Noa Osherovich To: leon@kernel.org, jgg@mellanox.com, dledford@redhat.com Cc: linux-rdma@vger.kernel.org, Noa Osherovich Subject: [PATCH rdma-core 4/6] pyverbs: Add unittests for extended completion-related classes Date: Sun, 24 Mar 2019 16:10:38 +0200 Message-Id: <20190324141040.22204-5-noaos@mellanox.com> X-Mailer: git-send-email 2.17.2 In-Reply-To: <20190324141040.22204-1-noaos@mellanox.com> References: <20190324141040.22204-1-noaos@mellanox.com> Sender: linux-rdma-owner@vger.kernel.org Precedence: bulk List-ID: X-Mailing-List: linux-rdma@vger.kernel.org X-Virus-Scanned: ClamAV using ClamSMTP Add unittests for extended CQ control path: - Creation and deletion of an extended CQ (good flow), with and without a completion channel. - Bad flow checks: Verify failure for illegal comp_vector / cqe number. Signed-off-by: Noa Osherovich Reviewed-by: Maor Gottlieb Reviewed-by: Leon Romanovsky --- pyverbs/tests/cq.py | 81 ++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 80 insertions(+), 1 deletion(-) diff --git a/pyverbs/tests/cq.py b/pyverbs/tests/cq.py index afe732e5e93c..4c1cea2b5296 100644 --- a/pyverbs/tests/cq.py +++ b/pyverbs/tests/cq.py @@ -6,8 +6,8 @@ Test module for pyverbs' cq module. import unittest import random +from pyverbs.cq import CompChannel, CQ, CqInitAttrEx, CQEX from pyverbs.pyverbs_error import PyverbsError -from pyverbs.cq import CompChannel, CQ import pyverbs.device as d import pyverbs.enums as e @@ -124,7 +124,86 @@ class CCTest(unittest.TestCase): cc.close() +class CQEXTest(unittest.TestCase): + """ + Test various functionalities of the CQEX class. + """ + + @staticmethod + def test_create_cq_ex(): + """ + Test ibv_create_cq_ex() + """ + lst = d.get_device_list() + for dev in lst: + with d.Context(name=dev.name.decode()) as ctx: + with CQEX(ctx, get_attrs_ex(ctx)): + pass + + @staticmethod + def test_create_cq_ex_bad_flow(): + """ + Test ibv_create_cq_ex() with wrong comp_vector / number of cqes + """ + lst = d.get_device_list() + for dev in lst: + with d.Context(name=dev.name.decode()) as ctx: + attrs_ex = get_attrs_ex(ctx) + max_cqe = ctx.query_device().max_cqe + attrs_ex.cqe = max_cqe + random.randint(1, 100) + try: + CQEX(ctx, attrs_ex) + except PyverbsError as e: + assert 'Failed to create extended CQ' in e.args[0] + assert ' Errno: 22' in e.args[0] + else: + raise PyverbsError( + 'Created a CQEX with {c} CQEs while device\'s max CQE={dc}'. + format(c=attrs_ex.cqe, dc=max_cqe)) + comp_channel = random.randint(ctx.num_comp_vectors, 100) + attrs_ex.comp_vector = comp_channel + attrs_ex.cqe = get_num_cqes(ctx) + try: + CQEX(ctx, attrs_ex) + except PyverbsError as e: + assert 'Failed to create extended CQ' in e.args[0] + assert ' Errno: 22' in e.args[0] + else: + raise PyverbsError( + 'Created a CQEX with comp_vector={c} while device\'s num_comp_vectors={dc}'. + format(c=comp_channel, dc=ctx.num_comp_vectors)) + + @staticmethod + def test_destroy_cq_ex(): + """ + Test ibv_destroy_cq() for extended CQs + """ + lst = d.get_device_list() + for dev in lst: + with d.Context(name=dev.name.decode()) as ctx: + with CQEX(ctx, get_attrs_ex(ctx)) as cq: + cq.close() + + def get_num_cqes(ctx): attr = ctx.query_device() max_cqe = attr.max_cqe return random.randint(0, max_cqe) + + +def get_attrs_ex(ctx): + cqe = get_num_cqes(ctx) + sample = random.sample(list(e.ibv_create_cq_wc_flags), + random.randint(0, 11)) + wc_flags = 0 + for flag in sample: + wc_flags |= flag + comp_mask = random.choice([0, e.IBV_CQ_INIT_ATTR_MASK_FLAGS]) + flags = 0 + if comp_mask is not 0: + sample = random.sample(list(e.ibv_create_cq_attr_flags), + random.randint(0, 2)) + for flag in sample: + flags |= flag + return CqInitAttrEx(cqe=cqe, wc_flags=wc_flags, comp_mask=comp_mask, + flags=flags) From patchwork Sun Mar 24 14:10:39 2019 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Noa Osherovich X-Patchwork-Id: 10867447 Return-Path: Received: from mail.wl.linuxfoundation.org (pdx-wl-mail.web.codeaurora.org [172.30.200.125]) by pdx-korg-patchwork-2.web.codeaurora.org (Postfix) with ESMTP id E69D91390 for ; Sun, 24 Mar 2019 14:11:30 +0000 (UTC) Received: from mail.wl.linuxfoundation.org (localhost [127.0.0.1]) by mail.wl.linuxfoundation.org (Postfix) with ESMTP id D37652940A for ; Sun, 24 Mar 2019 14:11:30 +0000 (UTC) Received: by mail.wl.linuxfoundation.org (Postfix, from userid 486) id C80E4294B6; Sun, 24 Mar 2019 14:11:30 +0000 (UTC) X-Spam-Checker-Version: SpamAssassin 3.3.1 (2010-03-16) on pdx-wl-mail.web.codeaurora.org X-Spam-Level: X-Spam-Status: No, score=-7.9 required=2.0 tests=BAYES_00,MAILING_LIST_MULTI, RCVD_IN_DNSWL_HI,UNPARSEABLE_RELAY autolearn=ham version=3.3.1 Received: from vger.kernel.org (vger.kernel.org [209.132.180.67]) by mail.wl.linuxfoundation.org (Postfix) with ESMTP id 6E7BD2940A for ; Sun, 24 Mar 2019 14:11:30 +0000 (UTC) Received: (majordomo@vger.kernel.org) by vger.kernel.org via listexpand id S1727111AbfCXOL3 (ORCPT ); Sun, 24 Mar 2019 10:11:29 -0400 Received: from mail-il-dmz.mellanox.com ([193.47.165.129]:53764 "EHLO mellanox.co.il" rhost-flags-OK-OK-OK-FAIL) by vger.kernel.org with ESMTP id S1726889AbfCXOL3 (ORCPT ); Sun, 24 Mar 2019 10:11:29 -0400 Received: from Internal Mail-Server by MTLPINE1 (envelope-from noaos@mellanox.com) with ESMTPS (AES256-SHA encrypted); 24 Mar 2019 16:11:25 +0200 Received: from reg-l-vrt-059-009.mtl.labs.mlnx (reg-l-vrt-059-009.mtl.labs.mlnx [10.135.59.9]) by labmailer.mlnx (8.13.8/8.13.8) with ESMTP id x2OEAxHx028186; Sun, 24 Mar 2019 16:11:25 +0200 From: Noa Osherovich To: leon@kernel.org, jgg@mellanox.com, dledford@redhat.com Cc: linux-rdma@vger.kernel.org, Noa Osherovich Subject: [PATCH rdma-core 5/6] Documentation: Document creation of CQs using pyverbs Date: Sun, 24 Mar 2019 16:10:39 +0200 Message-Id: <20190324141040.22204-6-noaos@mellanox.com> X-Mailer: git-send-email 2.17.2 In-Reply-To: <20190324141040.22204-1-noaos@mellanox.com> References: <20190324141040.22204-1-noaos@mellanox.com> Sender: linux-rdma-owner@vger.kernel.org Precedence: bulk List-ID: X-Mailing-List: linux-rdma@vger.kernel.org X-Virus-Scanned: ClamAV using ClamSMTP Add code snippets to demonstrate creation of a CQ and an extended CQ. Signed-off-by: Noa Osherovich Reviewed-by: Maor Gottlieb Reviewed-by: Leon Romanovsky --- Documentation/pyverbs.md | 56 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 56 insertions(+) diff --git a/Documentation/pyverbs.md b/Documentation/pyverbs.md index 112d806d9f2b..3a20b7a95769 100644 --- a/Documentation/pyverbs.md +++ b/Documentation/pyverbs.md @@ -225,3 +225,59 @@ with d.Context(name='mlx5_0') as ctx: dm_mr = DmMr(pd, dm_mr_len, e.IBV_ACCESS_ZERO_BASED, dm=dm, offset=0) ``` + +##### CQ +The following snippets show how to create CQs using pyverbs. Pyverbs supports +both CQ and extended CQ (CQEX). +As in C, a completion queue can be created with or without a completion +channel, the snippets show that. +CQ's 3rd parameter is cq_context, a user-defined context. We're using None in +our snippets. +```python +import random + +from pyverbs.cq import CompChannel, CQ +import pyverbs.device as d + +with d.Context(name='mlx5_0') as ctx: + num_cqes = random.randint(0, 200) # Just arbitrary values. Max value can be + # found in device attributes + comp_vector = 0 # An arbitrary value. comp_vector is limited by the + # context's num_comp_vectors + if random.choice([True, False]): + with CompChannel(ctx) as cc: + cq = CQ(ctx, num_cqes, None, cc, comp_vector) + else: + cq = CQ(ctx, num_cqes, None, None, comp_vector) + print(cq) +CQ +Handle : 0 +CQEs : 63 +``` + +```python +import random + +from pyverbs.cq import CqInitAttrEx, CQEX +import pyverbs.device as d +import pyverbs.enums as e + +with d.Context(name='mlx5_0') as ctx: + num_cqe = random.randint(0, 200) + wc_flags = e.IBV_WC_EX_WITH_CVLAN + comp_mask = 0 # Not using flags in this example + # completion channel is not used in this example + attrs = CqInitAttrEx(cqe=num_cqe, wc_flags=wc_flags, comp_mask=comp_mask, + flags=0) + print(attrs) + cq_ex = CQEX(ctx, attrs) + print(cq_ex) + Number of CQEs : 10 +WC flags : IBV_WC_EX_WITH_CVLAN +comp mask : 0 +flags : 0 + +Extended CQ: +Handle : 0 +CQEs : 15 +``` \ No newline at end of file From patchwork Sun Mar 24 14:10:40 2019 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Noa Osherovich X-Patchwork-Id: 10867451 Return-Path: Received: from mail.wl.linuxfoundation.org (pdx-wl-mail.web.codeaurora.org [172.30.200.125]) by pdx-korg-patchwork-2.web.codeaurora.org (Postfix) with ESMTP id 0BC1A13B5 for ; Sun, 24 Mar 2019 14:11:35 +0000 (UTC) Received: from mail.wl.linuxfoundation.org (localhost [127.0.0.1]) by mail.wl.linuxfoundation.org (Postfix) with ESMTP id E7FCE2940A for ; Sun, 24 Mar 2019 14:11:34 +0000 (UTC) Received: by mail.wl.linuxfoundation.org (Postfix, from userid 486) id D9B8B294B6; Sun, 24 Mar 2019 14:11:34 +0000 (UTC) X-Spam-Checker-Version: SpamAssassin 3.3.1 (2010-03-16) on pdx-wl-mail.web.codeaurora.org X-Spam-Level: X-Spam-Status: No, score=-7.9 required=2.0 tests=BAYES_00,MAILING_LIST_MULTI, RCVD_IN_DNSWL_HI,UNPARSEABLE_RELAY autolearn=ham version=3.3.1 Received: from vger.kernel.org (vger.kernel.org [209.132.180.67]) by mail.wl.linuxfoundation.org (Postfix) with ESMTP id B969C2940A for ; Sun, 24 Mar 2019 14:11:32 +0000 (UTC) Received: (majordomo@vger.kernel.org) by vger.kernel.org via listexpand id S1726811AbfCXOLc (ORCPT ); Sun, 24 Mar 2019 10:11:32 -0400 Received: from mail-il-dmz.mellanox.com ([193.47.165.129]:53770 "EHLO mellanox.co.il" rhost-flags-OK-OK-OK-FAIL) by vger.kernel.org with ESMTP id S1726970AbfCXOLc (ORCPT ); Sun, 24 Mar 2019 10:11:32 -0400 Received: from Internal Mail-Server by MTLPINE1 (envelope-from noaos@mellanox.com) with ESMTPS (AES256-SHA encrypted); 24 Mar 2019 16:11:26 +0200 Received: from reg-l-vrt-059-009.mtl.labs.mlnx (reg-l-vrt-059-009.mtl.labs.mlnx [10.135.59.9]) by labmailer.mlnx (8.13.8/8.13.8) with ESMTP id x2OEAxI0028186; Sun, 24 Mar 2019 16:11:26 +0200 From: Noa Osherovich To: leon@kernel.org, jgg@mellanox.com, dledford@redhat.com Cc: linux-rdma@vger.kernel.org, Noa Osherovich Subject: [PATCH rdma-core 6/6] pyverbs/tests: Pylint fixes Date: Sun, 24 Mar 2019 16:10:40 +0200 Message-Id: <20190324141040.22204-7-noaos@mellanox.com> X-Mailer: git-send-email 2.17.2 In-Reply-To: <20190324141040.22204-1-noaos@mellanox.com> References: <20190324141040.22204-1-noaos@mellanox.com> Sender: linux-rdma-owner@vger.kernel.org Precedence: bulk List-ID: X-Mailing-List: linux-rdma@vger.kernel.org X-Virus-Scanned: ClamAV using ClamSMTP Pylint is a code analysis tool for Python. It doesn't work perfectly with Cython, but some of its reported errors can be fixed: - Add missing docstrings. - Add @staticmethod annotations for test functions that do not use 'self' parameter. - Reformatting of long lines in a way that doesn't truncate user-visible strings. - Remove unused variables. - Rename test classes according to CamelCase convention. Signed-off-by: Noa Osherovich Reviewed-by: Maor Gottlieb Reviewed-by: Leon Romanovsky --- pyverbs/tests/device.py | 110 +++++++++++++++++++++++++++------------- pyverbs/tests/mr.py | 92 ++++++++++++++++++++------------- pyverbs/tests/pd.py | 23 ++++++--- pyverbs/tests/utils.py | 32 ++++++++++-- 4 files changed, 176 insertions(+), 81 deletions(-) diff --git a/pyverbs/tests/device.py b/pyverbs/tests/device.py index 492448a1526b..4f96a11a3ccb 100644 --- a/pyverbs/tests/device.py +++ b/pyverbs/tests/device.py @@ -1,5 +1,8 @@ # SPDX-License-Identifier: (GPL-2.0 OR Linux-OpenIB) # Copyright (c) 2018, Mellanox Technologies. All rights reserved. See COPYING file +""" +Test module for pyverbs' device module. +""" import unittest import resource import random @@ -11,23 +14,26 @@ import pyverbs.device as d PAGE_SIZE = resource.getpagesize() -class device_test(unittest.TestCase): +class DeviceTest(unittest.TestCase): """ Test various functionalities of the Device class. """ - def test_dev_list(self): + + @staticmethod + def test_dev_list(): """ Verify that it's possible to get IB devices list. """ - lst = d.get_device_list() + d.get_device_list() - def test_open_dev(self): + @staticmethod + def test_open_dev(): """ Test ibv_open_device() """ lst = d.get_device_list() for dev in lst: - ctx = d.Context(name=dev.name.decode()) + d.Context(name=dev.name.decode()) def test_query_device(self): """ @@ -39,7 +45,8 @@ class device_test(unittest.TestCase): attr = ctx.query_device() self.verify_device_attr(attr) - def test_query_gid(self): + @staticmethod + def test_query_gid(): """ Test ibv_query_gid() """ @@ -50,6 +57,12 @@ class device_test(unittest.TestCase): @staticmethod def verify_device_attr(attr): + """ + Helper method that verifies correctness of some members of DeviceAttr + object. + :param attr: A DeviceAttr object + :return: None + """ assert attr.node_guid != 0 assert attr.sys_image_guid != 0 assert attr.max_mr_size > PAGE_SIZE @@ -78,6 +91,12 @@ class device_test(unittest.TestCase): @staticmethod def verify_port_attr(attr): + """ + Helper method that verifies correctness of some members of PortAttr + object. + :param attr: A PortAttr object + :return: None + """ assert 'Invalid' not in d.phys_state_to_str(attr.state) assert 'Invalid' not in d.translate_mtu(attr.max_mtu) assert 'Invalid' not in d.translate_mtu(attr.active_mtu) @@ -98,7 +117,8 @@ class device_test(unittest.TestCase): port_attr = ctx.query_port(p + 1) self.verify_port_attr(port_attr) - def test_query_port_bad_flow(self): + @staticmethod + def test_query_port_bad_flow(): """ Verify that querying non-existing ports fails as expected """ lst = d.get_device_list() for dev in lst: @@ -111,14 +131,18 @@ class device_test(unittest.TestCase): assert 'Failed to query port' in e.args[0] assert 'Invalid argument' in e.args[0] else: - raise PyverbsRDMAError('Successfully queried non-existing port {p}'.\ - format(p=port)) + raise PyverbsRDMAError( + 'Successfully queried non-existing port {p}'. \ + format(p=port)) + -class dm_test(unittest.TestCase): +class DMTest(unittest.TestCase): """ Test various functionalities of the DM class. """ - def test_create_dm(self): + + @staticmethod + def test_create_dm(): """ test ibv_alloc_dm() """ @@ -134,7 +158,8 @@ class dm_test(unittest.TestCase): with d.DM(ctx, dm_attrs): pass - def test_destroy_dm(self): + @staticmethod + def test_destroy_dm(): """ test ibv_free_dm() """ @@ -150,7 +175,8 @@ class dm_test(unittest.TestCase): dm = d.DM(ctx, dm_attrs) dm.close() - def test_create_dm_bad_flow(self): + @staticmethod + def test_create_dm_bad_flow(): """ test ibv_alloc_dm() with an illegal size and comp mask """ @@ -163,22 +189,27 @@ class dm_test(unittest.TestCase): dm_len = attr.max_dm_size + 1 dm_attrs = u.get_dm_attrs(dm_len) try: - dm = d.DM(ctx, dm_attrs) + d.DM(ctx, dm_attrs) except PyverbsRDMAError as e: - assert 'Failed to allocate device memory of size' in e.args[0] + assert 'Failed to allocate device memory of size' in \ + e.args[0] assert 'Max available size' in e.args[0] else: - raise PyverbsError('Created a DM with size larger than max reported') + raise PyverbsError( + 'Created a DM with size larger than max reported') dm_attrs.comp_mask = random.randint(1, 100) try: - dm = d.DM(ctx, dm_attrs) + d.DM(ctx, dm_attrs) except PyverbsRDMAError as e: - assert 'Failed to allocate device memory of size' in e.args[0] + assert 'Failed to allocate device memory of size' in \ + e.args[0] else: - raise PyverbsError('Created a DM with illegal comp mask {c}'.\ - format(c=dm_attrs.comp_mask)) + raise PyverbsError( + 'Created a DM with illegal comp mask {c}'. \ + format(c=dm_attrs.comp_mask)) - def test_destroy_dm_bad_flow(self): + @staticmethod + def test_destroy_dm_bad_flow(): """ test calling ibv_free_dm() twice """ @@ -188,13 +219,15 @@ class dm_test(unittest.TestCase): attr = ctx.query_device_ex() if attr.max_dm_size == 0: return - dm_len = random.randrange(u.MIN_DM_SIZE, attr.max_dm_size, u.DM_ALIGNMENT) + dm_len = random.randrange(u.MIN_DM_SIZE, attr.max_dm_size, + u.DM_ALIGNMENT) dm_attrs = u.get_dm_attrs(dm_len) dm = d.DM(ctx, dm_attrs) dm.close() dm.close() - def test_dm_write(self): + @staticmethod + def test_dm_write(): """ Test writing to the device memory """ @@ -204,15 +237,18 @@ class dm_test(unittest.TestCase): attr = ctx.query_device_ex() if attr.max_dm_size == 0: return - dm_len = random.randrange(u.MIN_DM_SIZE, attr.max_dm_size, u.DM_ALIGNMENT) + dm_len = random.randrange(u.MIN_DM_SIZE, attr.max_dm_size, + u.DM_ALIGNMENT) dm_attrs = u.get_dm_attrs(dm_len) with d.DM(ctx, dm_attrs) as dm: data_length = random.randrange(4, dm_len, u.DM_ALIGNMENT) - data_offset = random.randrange(0, dm_len - data_length, u.DM_ALIGNMENT) + data_offset = random.randrange(0, dm_len - data_length, + u.DM_ALIGNMENT) data = u.get_data(data_length) dm.copy_to_dm(data_offset, data.encode(), data_length) - def test_dm_write_bad_flow(self): + @staticmethod + def test_dm_write_bad_flow(): """ Test writing to the device memory with bad offset and length """ @@ -222,21 +258,25 @@ class dm_test(unittest.TestCase): attr = ctx.query_device_ex() if attr.max_dm_size == 0: return - dm_len = random.randrange(u.MIN_DM_SIZE, attr.max_dm_size, u.DM_ALIGNMENT) + dm_len = random.randrange(u.MIN_DM_SIZE, attr.max_dm_size, + u.DM_ALIGNMENT) dm_attrs = u.get_dm_attrs(dm_len) with d.DM(ctx, dm_attrs) as dm: data_length = random.randrange(4, dm_len, u.DM_ALIGNMENT) - data_offset = random.randrange(0, dm_len - data_length, u.DM_ALIGNMENT) - data_offset += 1 # offset needs to be a multiple of 4 + data_offset = random.randrange(0, dm_len - data_length, + u.DM_ALIGNMENT) + data_offset += 1 # offset needs to be a multiple of 4 data = u.get_data(data_length) try: dm.copy_to_dm(data_offset, data.encode(), data_length) except PyverbsRDMAError as e: - assert 'Failed to copy to dm' in e.args[0] + assert 'Failed to copy to dm' in e.args[0] else: - raise PyverbsError('Wrote to device memory with a bad offset') + raise PyverbsError( + 'Wrote to device memory with a bad offset') - def test_dm_read(self): + @staticmethod + def test_dm_read(): """ Test reading from the device memory """ @@ -246,11 +286,13 @@ class dm_test(unittest.TestCase): attr = ctx.query_device_ex() if attr.max_dm_size == 0: return - dm_len = random.randrange(u.MIN_DM_SIZE, attr.max_dm_size, u.DM_ALIGNMENT) + dm_len = random.randrange(u.MIN_DM_SIZE, attr.max_dm_size, + u.DM_ALIGNMENT) dm_attrs = u.get_dm_attrs(dm_len) with d.DM(ctx, dm_attrs) as dm: data_length = random.randrange(4, dm_len, u.DM_ALIGNMENT) - data_offset = random.randrange(0, dm_len - data_length, u.DM_ALIGNMENT) + data_offset = random.randrange(0, dm_len - data_length, + u.DM_ALIGNMENT) data = u.get_data(data_length) dm.copy_to_dm(data_offset, data.encode(), data_length) read_str = dm.copy_from_dm(data_offset, data_length) diff --git a/pyverbs/tests/mr.py b/pyverbs/tests/mr.py index 229da54240cd..f27c40887f94 100644 --- a/pyverbs/tests/mr.py +++ b/pyverbs/tests/mr.py @@ -1,5 +1,8 @@ # SPDX-License-Identifier: (GPL-2.0 OR Linux-OpenIB) # Copyright (c) 2019, Mellanox Technologies. All rights reserved. See COPYING file +""" +Test module for pyverbs' mr module. +""" import unittest import random @@ -14,11 +17,12 @@ import pyverbs.enums as e MAX_IO_LEN = 1048576 -class mr_test(unittest.TestCase): +class MRTest(unittest.TestCase): """ Test various functionalities of the MR class. """ - def test_reg_mr(self): + @staticmethod + def test_reg_mr(): """ Test ibv_reg_mr() """ lst = d.get_device_list() for dev in lst: @@ -27,7 +31,8 @@ class mr_test(unittest.TestCase): with MR(pd, u.get_mr_length(), u.get_access_flags()) as mr: pass - def test_dereg_mr(self): + @staticmethod + def test_dereg_mr(): """ Test ibv_dereg_mr() """ lst = d.get_device_list() for dev in lst: @@ -36,48 +41,52 @@ class mr_test(unittest.TestCase): with MR(pd, u.get_mr_length(), u.get_access_flags()) as mr: mr.close() - def test_reg_mr_bad_flow(self): + @staticmethod + def test_reg_mr_bad_flow(): """ Verify that trying to register a MR with None PD fails """ try: - mr = MR(None, random.randint(0, 10000), u.get_access_flags()) + MR(None, random.randint(0, 10000), u.get_access_flags()) except TypeError as te: assert 'expected pyverbs.pd.PD' in te.args[0] assert 'got NoneType' in te.args[0] else: raise PyverbsRDMAErrno('Created a MR with None PD') - def test_dereg_mr_twice(self): + @staticmethod + def test_dereg_mr_twice(): """ Verify that explicit call to MR's close() doesn't fails """ lst = d.get_device_list() for dev in lst: with d.Context(name=dev.name.decode()) as ctx: with PD(ctx) as pd: with MR(pd, u.get_mr_length(), u.get_access_flags()) as mr: - # Pyverbs supports multiple destruction of objects, we are - # not expecting an exception here. + # Pyverbs supports multiple destruction of objects, + # we are not expecting an exception here. mr.close() mr.close() - def test_reg_mr_bad_flags(self): + @staticmethod + def test_reg_mr_bad_flags(): """ Verify that illegal flags combination fails as expected """ lst = d.get_device_list() for dev in lst: with d.Context(name=dev.name.decode()) as ctx: with PD(ctx) as pd: flags = random.sample([e.IBV_ACCESS_REMOTE_WRITE, - e.IBV_ACCESS_REMOTE_ATOMIC], + e.IBV_ACCESS_REMOTE_ATOMIC], random.randint(1, 2)) mr_flags = 0 for i in flags: mr_flags += i.value try: - mr = MR(pd, u.get_mr_length(), mr_flags) + MR(pd, u.get_mr_length(), mr_flags) except PyverbsRDMAError as err: assert 'Failed to register a MR' in err.args[0] else: raise PyverbsRDMAError('Registered a MR with illegal falgs') - def test_write(self): + @staticmethod + def test_write(): """ Test writing to MR's buffer """ @@ -90,7 +99,8 @@ class mr_test(unittest.TestCase): write_len = min(random.randint(1, MAX_IO_LEN), mr_len) mr.write(u.get_data(write_len), write_len) - def test_read(self): + @staticmethod + def test_read(): """ Test reading from MR's buffer """ @@ -108,7 +118,8 @@ class mr_test(unittest.TestCase): read_str = mr.read(read_len, offset).decode() assert read_str in write_str - def test_lkey(self): + @staticmethod + def test_lkey(): """ Test reading lkey property """ @@ -118,9 +129,10 @@ class mr_test(unittest.TestCase): with PD(ctx) as pd: length = u.get_mr_length() with MR(pd, length, u.get_access_flags()) as mr: - lkey = mr.lkey + mr.lkey - def test_rkey(self): + @staticmethod + def test_rkey(): """ Test reading rkey property """ @@ -130,9 +142,10 @@ class mr_test(unittest.TestCase): with PD(ctx) as pd: length = u.get_mr_length() with MR(pd, length, u.get_access_flags()) as mr: - rkey = mr.rkey + mr.rkey - def test_buffer(self): + @staticmethod + def test_buffer(): """ Test reading buf property """ @@ -142,52 +155,58 @@ class mr_test(unittest.TestCase): with PD(ctx) as pd: length = u.get_mr_length() with MR(pd, length, u.get_access_flags()) as mr: - buf = mr.buf + mr.buf -class mw_test(unittest.TestCase): +class MWTest(unittest.TestCase): """ Test various functionalities of the MW class. """ - def test_reg_mw(self): + @staticmethod + def test_reg_mw(): """ Test ibv_alloc_mw() """ lst = d.get_device_list() for dev in lst: with d.Context(name=dev.name.decode()) as ctx: with PD(ctx) as pd: - with MW(pd, random.choice([e.IBV_MW_TYPE_1, e.IBV_MW_TYPE_2])) as mw: + with MW(pd, random.choice([e.IBV_MW_TYPE_1, + e.IBV_MW_TYPE_2])): pass - def test_dereg_mw(self): + @staticmethod + def test_dereg_mw(): """ Test ibv_dealloc_mw() """ lst = d.get_device_list() for dev in lst: with d.Context(name=dev.name.decode()) as ctx: with PD(ctx) as pd: - with MW(pd, random.choice([e.IBV_MW_TYPE_1, e.IBV_MW_TYPE_2])) as mw: + with MW(pd, random.choice([e.IBV_MW_TYPE_1, + e.IBV_MW_TYPE_2])) as mw: mw.close() - def test_reg_mw_wrong_type(self): + @staticmethod + def test_reg_mw_wrong_type(): """ Test ibv_alloc_mw() """ lst = d.get_device_list() for dev in lst: with d.Context(name=dev.name.decode()) as ctx: with PD(ctx) as pd: try: - mw_type =random.randint(3, 100) - mw = MW(pd, mw_type) - except PyverbsRDMAError as e: + mw_type = random.randint(3, 100) + MW(pd, mw_type) + except PyverbsRDMAError: pass else: raise PyverbsError('Created a MW with type {t}'.\ format(t=mw_type)) -class dm_mr_test(unittest.TestCase): +class DMMRTest(unittest.TestCase): """ Test various functionalities of the DMMR class. """ - def test_create_dm_mr(self): + @staticmethod + def test_create_dm_mr(): """ Test ibv_reg_dm_mr """ @@ -204,10 +223,11 @@ class dm_mr_test(unittest.TestCase): with d.DM(ctx, dm_attrs) as dm: dm_mr_len = random.randint(1, dm_len) dm_mr_offset = random.randint(0, (dm_len - dm_mr_len)) - dm_mr = DMMR(pd, dm_mr_len, e.IBV_ACCESS_ZERO_BASED, dm=dm, - offset=dm_mr_offset) + DMMR(pd, dm_mr_len, e.IBV_ACCESS_ZERO_BASED, dm=dm, + offset=dm_mr_offset) - def test_destroy_dm_mr(self): + @staticmethod + def test_destroy_dm_mr(): """ Test freeing of dm_mr """ @@ -219,11 +239,11 @@ class dm_mr_test(unittest.TestCase): return with PD(ctx) as pd: dm_len = random.randrange(u.MIN_DM_SIZE, attr.max_dm_size, - u.DM_ALIGNMENT) + u.DM_ALIGNMENT) dm_attrs = u.get_dm_attrs(dm_len) with d.DM(ctx, dm_attrs) as dm: dm_mr_len = random.randint(1, dm_len) dm_mr_offset = random.randint(0, (dm_len - dm_mr_len)) - dm_mr = DMMR(pd, dm_mr_len, e.IBV_ACCESS_ZERO_BASED, dm=dm, - offset=dm_mr_offset) + dm_mr = DMMR(pd, dm_mr_len, e.IBV_ACCESS_ZERO_BASED, + dm=dm, offset=dm_mr_offset) dm_mr.close() diff --git a/pyverbs/tests/pd.py b/pyverbs/tests/pd.py index c80f0b6df80e..c9c6384e1bd0 100644 --- a/pyverbs/tests/pd.py +++ b/pyverbs/tests/pd.py @@ -1,6 +1,8 @@ # SPDX-License-Identifier: (GPL-2.0 OR Linux-OpenIB) # Copyright (c) 2019, Mellanox Technologies. All rights reserved. See COPYING file - +""" +Test module for pyverbs' pd module. +""" import unittest import random @@ -9,11 +11,12 @@ import pyverbs.device as d from pyverbs.pd import PD -class pd_test(unittest.TestCase): +class PDTest(unittest.TestCase): """ Test various functionalities of the PD class. """ - def test_alloc_pd(self): + @staticmethod + def test_alloc_pd(): """ Test ibv_alloc_pd() """ @@ -23,7 +26,8 @@ class pd_test(unittest.TestCase): with PD(ctx): pass - def test_dealloc_pd(self): + @staticmethod + def test_dealloc_pd(): """ Test ibv_dealloc_pd() """ @@ -33,7 +37,8 @@ class pd_test(unittest.TestCase): with PD(ctx) as pd: pd.close() - def test_multiple_pd_creation(self): + @staticmethod + def test_multiple_pd_creation(): """ Test multiple creations and destructions of a PD object """ @@ -44,19 +49,21 @@ class pd_test(unittest.TestCase): with PD(ctx) as pd: pd.close() - def test_create_pd_none_ctx(self): + @staticmethod + def test_create_pd_none_ctx(): """ Verify that PD can't be created with a None context """ try: - pd = PD(None) + PD(None) except TypeError as te: assert 'expected pyverbs.device.Context' in te.args[0] assert 'got NoneType' in te.args[0] else: raise PyverbsRDMAErrno('Created a PD with None context') - def test_destroy_pd_twice(self): + @staticmethod + def test_destroy_pd_twice(): """ Test bad flow cases in destruction of a PD object """ diff --git a/pyverbs/tests/utils.py b/pyverbs/tests/utils.py index c8b46686b0c1..84deb49093d2 100644 --- a/pyverbs/tests/utils.py +++ b/pyverbs/tests/utils.py @@ -1,11 +1,15 @@ # SPDX-License-Identifier: (GPL-2.0 OR Linux-OpenIB) # Copyright (c) 2019, Mellanox Technologies. All rights reserved. See COPYING file +""" +Provide some useful helper function for pyverbs' tests. +""" from string import ascii_lowercase as al import random import pyverbs.device as d import pyverbs.enums as e + MAX_MR_SIZE = 4194304 # Some HWs limit DM address and length alignment to 4 for read and write # operations. Use a minimal length and alignment that respect that. @@ -16,15 +20,26 @@ DM_ALIGNMENT = 4 MIN_DM_LOG_ALIGN = 0 MAX_DM_LOG_ALIGN = 6 + def get_mr_length(): - # Allocating large buffers typically fails + """ + Provide a random value for MR length. We avoid large buffers as these + allocations typically fails. + :return: A random MR length + """ return random.randint(0, MAX_MR_SIZE) def get_access_flags(): + """ + Provide random legal access flags for an MR. + Since remote write and remote atomic require local write permission, if + one of them is randomly selected without local write, local write will be + added as well. + :return: A random legal value for MR flags + """ vals = list(e.ibv_access_flags) selected = random.sample(vals, random.randint(1, 7)) - # Remote write / remote atomic are not allowed without local write if e.IBV_ACCESS_REMOTE_WRITE in selected or e.IBV_ACCESS_REMOTE_ATOMIC in selected: if not e.IBV_ACCESS_LOCAL_WRITE in selected: selected.append(e.IBV_ACCESS_LOCAL_WRITE) @@ -35,10 +50,21 @@ def get_access_flags(): def get_data(length): + """ + Randomizes data of a given length. + :param length: Length of the data + :return: A random data of the given length + """ return ''.join(random.choice(al) for i in range(length)) def get_dm_attrs(dm_len): + """ + Initializes an AllocDmAttr member with the given length and random + alignment. It currently sets comp_mask = 0 since other comp_mask values + are not supported. + :param dm_len: + :return: An initialized AllocDmAttr object + """ align = random.randint(MIN_DM_LOG_ALIGN, MAX_DM_LOG_ALIGN) - # Comp mask != 0 is not supported return d.AllocDmAttr(dm_len, align, 0)