From patchwork Sun Feb 24 13:06:25 2019 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Noa Osherovich X-Patchwork-Id: 10827927 Return-Path: Received: from mail.wl.linuxfoundation.org (pdx-wl-mail.web.codeaurora.org [172.30.200.125]) by pdx-korg-patchwork-2.web.codeaurora.org (Postfix) with ESMTP id DE2ED17E9 for ; Sun, 24 Feb 2019 13:08:54 +0000 (UTC) Received: from mail.wl.linuxfoundation.org (localhost [127.0.0.1]) by mail.wl.linuxfoundation.org (Postfix) with ESMTP id C6D862B4E5 for ; Sun, 24 Feb 2019 13:08:54 +0000 (UTC) Received: by mail.wl.linuxfoundation.org (Postfix, from userid 486) id B9E472B525; Sun, 24 Feb 2019 13:08:54 +0000 (UTC) X-Spam-Checker-Version: SpamAssassin 3.3.1 (2010-03-16) on pdx-wl-mail.web.codeaurora.org X-Spam-Level: X-Spam-Status: No, score=-7.9 required=2.0 tests=BAYES_00,MAILING_LIST_MULTI, RCVD_IN_DNSWL_HI,UNPARSEABLE_RELAY autolearn=ham version=3.3.1 Received: from vger.kernel.org (vger.kernel.org [209.132.180.67]) by mail.wl.linuxfoundation.org (Postfix) with ESMTP id CAF812B4E5 for ; Sun, 24 Feb 2019 13:08:51 +0000 (UTC) Received: (majordomo@vger.kernel.org) by vger.kernel.org via listexpand id S1728229AbfBXNIv (ORCPT ); Sun, 24 Feb 2019 08:08:51 -0500 Received: from mail-il-dmz.mellanox.com ([193.47.165.129]:50469 "EHLO mellanox.co.il" rhost-flags-OK-OK-OK-FAIL) by vger.kernel.org with ESMTP id S1726865AbfBXNIu (ORCPT ); Sun, 24 Feb 2019 08:08:50 -0500 Received: from Internal Mail-Server by MTLPINE1 (envelope-from noaos@mellanox.com) with ESMTPS (AES256-SHA encrypted); 24 Feb 2019 15:06:48 +0200 Received: from reg-l-vrt-059-009.mtl.labs.mlnx (reg-l-vrt-059-009.mtl.labs.mlnx [10.135.59.9]) by labmailer.mlnx (8.13.8/8.13.8) with ESMTP id x1OD6mfa026210; Sun, 24 Feb 2019 15:06:48 +0200 From: Noa Osherovich To: leonro@mellanox.com, jgg@mellanox.com, dledford@redhat.com Cc: linux-rdma@vger.kernel.org, Noa Osherovich Subject: [PATCH rdma-core 06/19] pyverbs: Introducing MR object Date: Sun, 24 Feb 2019 15:06:25 +0200 Message-Id: <20190224130638.31848-7-noaos@mellanox.com> X-Mailer: git-send-email 2.17.2 In-Reply-To: <20190224130638.31848-1-noaos@mellanox.com> References: <20190224130638.31848-1-noaos@mellanox.com> Sender: linux-rdma-owner@vger.kernel.org Precedence: bulk List-ID: X-Mailing-List: linux-rdma@vger.kernel.org X-Virus-Scanned: ClamAV using ClamSMTP MR represnts ibv_mr. Instead of allocating a buffer prior to calling ibv_reg_mr, the user provides the MR class the required size and flags, and pyverbs allocates the needed buffer. The MR class exposes read() and write() methods to users to allow setting and reading the underlying memory buffer. Signed-off-by: Noa Osherovich --- pyverbs/CMakeLists.txt | 1 + pyverbs/libibverbs.pxd | 11 +++++ pyverbs/mr.pxd | 12 +++++ pyverbs/mr.pyx | 109 +++++++++++++++++++++++++++++++++++++++++ pyverbs/pd.pxd | 2 + pyverbs/pd.pyx | 13 ++++- 6 files changed, 147 insertions(+), 1 deletion(-) create mode 100644 pyverbs/mr.pxd create mode 100644 pyverbs/mr.pyx diff --git a/pyverbs/CMakeLists.txt b/pyverbs/CMakeLists.txt index c1e7dceb8d9c..b660b46f170a 100644 --- a/pyverbs/CMakeLists.txt +++ b/pyverbs/CMakeLists.txt @@ -6,6 +6,7 @@ rdma_cython_module(pyverbs base.pyx device.pyx enums.pyx + mr.pyx pd.pyx ) diff --git a/pyverbs/libibverbs.pxd b/pyverbs/libibverbs.pxd index d625ddf7215b..5409ff04a649 100644 --- a/pyverbs/libibverbs.pxd +++ b/pyverbs/libibverbs.pxd @@ -68,6 +68,15 @@ cdef extern from 'infiniband/verbs.h': ibv_context *context unsigned int handle + cdef struct ibv_mr: + ibv_context *context + ibv_pd *pd + void *addr + size_t length + unsigned int handle + unsigned int lkey + unsigned int rkey + ibv_device **ibv_get_device_list(int *n) void ibv_free_device_list(ibv_device **list) ibv_context *ibv_open_device(ibv_device *device) @@ -78,3 +87,5 @@ cdef extern from 'infiniband/verbs.h': int index, ibv_gid *gid) ibv_pd *ibv_alloc_pd(ibv_context *context) int ibv_dealloc_pd(ibv_pd *pd) + ibv_mr *ibv_reg_mr(ibv_pd *pd, void *addr, size_t length, int access) + int ibv_dereg_mr(ibv_mr *mr) diff --git a/pyverbs/mr.pxd b/pyverbs/mr.pxd new file mode 100644 index 000000000000..92578021c3b3 --- /dev/null +++ b/pyverbs/mr.pxd @@ -0,0 +1,12 @@ +# SPDX-License-Identifier: (GPL-2.0 OR Linux-OpenIB) +# Copyright (c) 2019, Mellanox Technologies. All rights reserved. See COPYING file + +from pyverbs.base cimport PyverbsCM +from . cimport libibverbs as v + + +cdef class MR(PyverbsCM): + cdef object pd + cdef v.ibv_mr *mr + cdef void *buf + cpdef read(self, length, offset) diff --git a/pyverbs/mr.pyx b/pyverbs/mr.pyx new file mode 100644 index 000000000000..31949e739376 --- /dev/null +++ b/pyverbs/mr.pyx @@ -0,0 +1,109 @@ +# SPDX-License-Identifier: (GPL-2.0 OR Linux-OpenIB) +# Copyright (c) 2019, Mellanox Technologies. All rights reserved. See COPYING file + +from pyverbs.pyverbs_error import PyverbsRDMAError, PyverbsError +from pyverbs.base import PyverbsRDMAErrno +from .pd cimport PD +import resource + +cdef extern from 'stdlib.h': + int posix_memalign(void **memptr, size_t alignment, size_t size) +cdef extern from 'stdlib.h': + void free(void *ptr) +cdef extern from 'string.h': + void *memcpy(void *dest, const void *src, size_t n) + void *memset(void *s, int c, size_t n) +cdef extern from 'stdint.h': + ctypedef int uintptr_t + + +cdef class MR(PyverbsCM): + """ + MR class represents ibv_mr. Buffer allocation in done in the c'tor. Freeing + it is done in close(). + """ + def __cinit__(self, PD pd not None, length, access): + """ + Allocate a user-level buffer of length and register a Memory + Region of the given length and access flags. + :param pd: A PD object + :param length: Length in bytes + :param access: Access flags, see ibv_access_flags enum + :return: The newly created MR on success + """ + #We want to enable registering an MR of size 0 but this fails with a + #buffer of size 0, so in this case lets increase the buffer + if length == 0: + length = 10 + rc = posix_memalign(&self.buf, resource.getpagesize(), length) + if rc: + raise PyverbsRDMAError('Failed to allocate MR buffer of size {l}'. + format(l=length)) + memset(self.buf, 0, length) + self.mr = v.ibv_reg_mr(pd.pd, self.buf, length, access) + if self.mr == NULL: + raise PyverbsRDMAErrno('Failed to register a MR. length: {l}, access flags: {a}'. + format(l=length, a=access)) + self.pd = pd + pd.add_ref(self) + self.logger.debug('Registered ibv_mr. Length: {l}, access flags {a}'. + format(l=length, a=access)) + + def __dealloc__(self): + self.close() + + cpdef close(self): + """ + Closes the underlying C object of the MR and frees the memory allocated. + MR may be deleted directly or indirectly by closing its context, which + leaves the Python PD object without the underlying C object, so during + destruction, need to check whether or not the C object exists. + :return: None + """ + self.logger.debug('Closing MR') + if self.mr != NULL: + rc = v.ibv_dereg_mr(self.mr) + if rc != 0: + raise PyverbsRDMAErrno('Failed to dereg MR') + self.mr = NULL + self.pd = None + free(self.buf) + self.buf = NULL + + def write(self, data, length): + """ + Write user data to the MR's buffer using memcpy + :param data: User data to write + :param length: Length of the data to write + :return: None + """ + # If data is a string, cast it to bytes as Python3 doesn't + # automatically convert it. + if isinstance(data, str): + data = data.encode() + memcpy(self.buf, data, length) + + cpdef read(self, length, offset): + """ + Reads data from the MR's buffer + :param length: Length of data to read + :param offset: Reading offset + :return: The data on the buffer in the requested offset + """ + cdef char *data + cdef int off = offset # we can't use offset in the next line, as it is + # a Python object and not C + data = (self.buf + off) + return data[:length] + + @property + def buf(self): + return self.buf + + @property + def lkey(self): + return self.mr.lkey + + @property + def rkey(self): + return self.mr.rkey diff --git a/pyverbs/pd.pxd b/pyverbs/pd.pxd index ec25ec8841ec..8381662b21be 100644 --- a/pyverbs/pd.pxd +++ b/pyverbs/pd.pxd @@ -8,3 +8,5 @@ from .base cimport PyverbsCM cdef class PD(PyverbsCM): cdef v.ibv_pd *pd cdef Context ctx + cdef add_ref(self, obj) + cdef object mrs diff --git a/pyverbs/pd.pyx b/pyverbs/pd.pyx index 9975f906b329..a5fc063751e1 100644 --- a/pyverbs/pd.pyx +++ b/pyverbs/pd.pyx @@ -1,7 +1,10 @@ # SPDX-License-Identifier: (GPL-2.0 OR Linux-OpenIB) # Copyright (c) 2019, Mellanox Technologies. All rights reserved. -from pyverbs.pyverbs_error import PyverbsRDMAError +import weakref + +from pyverbs.pyverbs_error import PyverbsRDMAError, PyverbsError from pyverbs.base import PyverbsRDMAErrno +from .mr cimport MR cdef extern from 'errno.h': int errno @@ -21,6 +24,7 @@ cdef class PD(PyverbsCM): self.ctx = context context.add_ref(self) self.logger.debug('PD: Allocated ibv_pd') + self.mrs = weakref.WeakSet() def __dealloc__(self): """ @@ -38,9 +42,16 @@ cdef class PD(PyverbsCM): :return: None """ self.logger.debug('Closing PD') + self.close_weakrefs([self.mrs]) if self.pd != NULL: rc = v.ibv_dealloc_pd(self.pd) if rc != 0: raise PyverbsRDMAErrno('Failed to dealloc PD') self.pd = NULL self.ctx = None + + cdef add_ref(self, obj): + if isinstance(obj, MR): + self.mrs.add(obj) + else: + raise PyverbsError('Unrecognized object type')