From patchwork Sun Feb 24 13:06:30 2019 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Noa Osherovich X-Patchwork-Id: 10827945 Return-Path: Received: from mail.wl.linuxfoundation.org (pdx-wl-mail.web.codeaurora.org [172.30.200.125]) by pdx-korg-patchwork-2.web.codeaurora.org (Postfix) with ESMTP id D63FD1575 for ; Sun, 24 Feb 2019 13:08:58 +0000 (UTC) Received: from mail.wl.linuxfoundation.org (localhost [127.0.0.1]) by mail.wl.linuxfoundation.org (Postfix) with ESMTP id BBF772A007 for ; Sun, 24 Feb 2019 13:08:58 +0000 (UTC) Received: by mail.wl.linuxfoundation.org (Postfix, from userid 486) id AD12F29E42; Sun, 24 Feb 2019 13:08:58 +0000 (UTC) X-Spam-Checker-Version: SpamAssassin 3.3.1 (2010-03-16) on pdx-wl-mail.web.codeaurora.org X-Spam-Level: X-Spam-Status: No, score=-7.9 required=2.0 tests=BAYES_00,MAILING_LIST_MULTI, RCVD_IN_DNSWL_HI,UNPARSEABLE_RELAY autolearn=ham version=3.3.1 Received: from vger.kernel.org (vger.kernel.org [209.132.180.67]) by mail.wl.linuxfoundation.org (Postfix) with ESMTP id C2822291B4 for ; Sun, 24 Feb 2019 13:08:57 +0000 (UTC) Received: (majordomo@vger.kernel.org) by vger.kernel.org via listexpand id S1728172AbfBXNI4 (ORCPT ); Sun, 24 Feb 2019 08:08:56 -0500 Received: from mail-il-dmz.mellanox.com ([193.47.165.129]:50546 "EHLO mellanox.co.il" rhost-flags-OK-OK-OK-FAIL) by vger.kernel.org with ESMTP id S1728248AbfBXNI4 (ORCPT ); Sun, 24 Feb 2019 08:08:56 -0500 Received: from Internal Mail-Server by MTLPINE1 (envelope-from noaos@mellanox.com) with ESMTPS (AES256-SHA encrypted); 24 Feb 2019 15:06:49 +0200 Received: from reg-l-vrt-059-009.mtl.labs.mlnx (reg-l-vrt-059-009.mtl.labs.mlnx [10.135.59.9]) by labmailer.mlnx (8.13.8/8.13.8) with ESMTP id x1OD6mff026210; Sun, 24 Feb 2019 15:06:49 +0200 From: Noa Osherovich To: leonro@mellanox.com, jgg@mellanox.com, dledford@redhat.com Cc: linux-rdma@vger.kernel.org, Noa Osherovich , Daria Velikovsky Subject: [PATCH rdma-core 11/19] pyverbs: Add support for direct memory usage Date: Sun, 24 Feb 2019 15:06:30 +0200 Message-Id: <20190224130638.31848-12-noaos@mellanox.com> X-Mailer: git-send-email 2.17.2 In-Reply-To: <20190224130638.31848-1-noaos@mellanox.com> References: <20190224130638.31848-1-noaos@mellanox.com> Sender: linux-rdma-owner@vger.kernel.org Precedence: bulk List-ID: X-Mailing-List: linux-rdma@vger.kernel.org X-Virus-Scanned: ClamAV using ClamSMTP Expose DM and DMMR classes. DM represents ibv_dm. DMMR is similar to a regular MR but its buffer is allocated on the device and not by the OS. Signed-off-by: Daria Velikovsky Signed-off-by: Noa Osherovich --- pyverbs/device.pxd | 10 ++++ pyverbs/device.pyx | 112 ++++++++++++++++++++++++++++++++++++++++- pyverbs/libibverbs.pxd | 17 +++++++ pyverbs/mr.pxd | 3 ++ pyverbs/mr.pyx | 43 +++++++++++++++- pyverbs/pd.pyx | 5 +- 6 files changed, 185 insertions(+), 5 deletions(-) diff --git a/pyverbs/device.pxd b/pyverbs/device.pxd index c81e8176eef9..c9e346539e58 100644 --- a/pyverbs/device.pxd +++ b/pyverbs/device.pxd @@ -10,6 +10,7 @@ cdef class Context(PyverbsCM): cdef object name cdef add_ref(self, obj) cdef object pds + cdef object dms cdef class DeviceAttr(PyverbsObject): cdef v.ibv_device_attr dev_attr @@ -37,3 +38,12 @@ cdef class TSOCaps(PyverbsObject): cdef class DeviceAttrEx(PyverbsObject): cdef v.ibv_device_attr_ex dev_attr + +cdef class AllocDmAttr(PyverbsObject): + cdef v.ibv_alloc_dm_attr alloc_dm_attr + +cdef class DM(PyverbsCM): + cdef v.ibv_dm *dm + cdef object dm_mrs + cdef object context + cdef add_ref(self, obj) diff --git a/pyverbs/device.pyx b/pyverbs/device.pyx index 948909753738..7bfcda8bc177 100644 --- a/pyverbs/device.pyx +++ b/pyverbs/device.pyx @@ -13,13 +13,21 @@ from .pyverbs_error import PyverbsUserError from pyverbs.base import PyverbsRDMAErrno cimport pyverbs.libibverbs as v from pyverbs.addr cimport GID +from pyverbs.mr import DMMR from pyverbs.pd cimport PD cdef extern from 'errno.h': int errno - cdef extern from 'endian.h': unsigned long be64toh(unsigned long host_64bits); +cdef extern from 'stdlib.h': + void free(void *ptr) +cdef extern from 'string.h': + void *memset(void *s, int c, size_t n) +cdef extern from 'malloc.h': + void *malloc(size_t size) +cdef extern from 'stdint.h': + ctypedef int uint64_t class Device(PyverbsObject): @@ -75,6 +83,8 @@ cdef class Context(PyverbsCM): cdef v.ibv_device **dev_list self.pds = weakref.WeakSet() + self.dms = weakref.WeakSet() + dev_name = kwargs.get('name') if dev_name is not None: @@ -110,7 +120,7 @@ cdef class Context(PyverbsCM): cpdef close(self): self.logger.debug('Closing Context') - self.close_weakrefs([self.pds]) + self.close_weakrefs([self.dms, self.pds]) if self.context != NULL: rc = v.ibv_close_device(self.context) if rc != 0: @@ -158,6 +168,8 @@ cdef class Context(PyverbsCM): cdef add_ref(self, obj): if isinstance(obj, PD): self.pds.add(obj) + elif isinstance(obj, DM): + self.dms.add(obj) else: raise PyverbsError('Unrecognized object type') @@ -477,6 +489,102 @@ cdef class DeviceAttrEx(PyverbsObject): return self.dev_attr.max_dm_size +cdef class AllocDmAttr(PyverbsObject): + def __cinit__(self, length, log_align_req = 0, comp_mask = 0): + """ + Creates an AllocDmAttr object with the given parameters. This object + can than be used to create a DM object. + :param length: Length of the future device memory + :param log_align_req: log2 of address alignment requirement + :param comp_mask: compatibility mask + :return: An AllocDmAttr object + """ + self.alloc_dm_attr.length = length + self.alloc_dm_attr.log_align_req = log_align_req + self.alloc_dm_attr.comp_mask = comp_mask + + @property + def length(self): + return self.alloc_dm_attr.length + + @length.setter + def length(self, val): + self.alloc_dm_attr.length = val + + @property + def log_align_req(self): + return self.alloc_dm_attr.log_align_req + + @log_align_req.setter + def log_align_req(self, val): + self.alloc_dm_attr.log_align_req = val + + @property + def comp_mask(self): + return self.alloc_dm_attr.comp_mask + + @comp_mask.setter + def comp_mask(self, val): + self.alloc_dm_attr.comp_mask = val + + +cdef class DM(PyverbsCM): + def __cinit__(self, Context context, AllocDmAttr dm_attr not None): + """ + Allocate a device (direct) memory. + :param context: The context of the device on which to allocate memory + :param dm_attr: Attributes that define the DM + :return: A DM object on success + """ + self.dm_mrs = weakref.WeakSet() + device_attr = context.query_device_ex() + if device_attr.max_dm_size <= 0: + raise PyverbsUserError('Device doesn\'t support dm allocation') + self.dm = v.ibv_alloc_dm(context.context, + &dm_attr.alloc_dm_attr) + if self.dm == NULL: + raise PyverbsRDMAErrno('Failed to allocate device memory of size ' + '{size}. Max available size {max}.' + .format(size=dm_attr.length, + max=device_attr.max_dm_size)) + self.context = context + context.add_ref(self) + + def __dealloc__(self): + self.close() + + cpdef close(self): + self.logger.debug('Closing DM') + self.close_weakrefs([self.dm_mrs]) + if self.dm != NULL: + rc = v.ibv_free_dm(self.dm) + if rc != 0: + raise PyverbsRDMAErrno('Failed to free dm') + self.dm = NULL + self.context = None + + cdef add_ref(self, obj): + if isinstance(obj, DMMR): + self.dm_mrs.add(obj) + + def copy_to_dm(self, dm_offset, data, length): + rc = v.ibv_memcpy_to_dm(self.dm, dm_offset, + data, length) + if rc != 0: + raise PyverbsRDMAErrno('Failed to copy to dm') + + def copy_from_dm(self, dm_offset, length): + cdef char *data =malloc(length) + memset(data, 0, length) + rc = v.ibv_memcpy_from_dm(data, self.dm, + dm_offset, length) + if rc != 0: + raise PyverbsRDMAErrno('Failed to copy from dm') + res = data[:length] + free(data) + return res + + def guid_format(num): """ Get GUID representation of the given number, including change of endianness. diff --git a/pyverbs/libibverbs.pxd b/pyverbs/libibverbs.pxd index 7eb1df2407c9..5dd3ba013e5e 100644 --- a/pyverbs/libibverbs.pxd +++ b/pyverbs/libibverbs.pxd @@ -139,6 +139,15 @@ cdef extern from 'infiniband/verbs.h': unsigned int handle ibv_mw_type mw_type + cdef struct ibv_alloc_dm_attr: + size_t length + unsigned int log_align_req + unsigned int comp_mask + + cdef struct ibv_dm: + ibv_context *context + unsigned int comp_mask + ibv_device **ibv_get_device_list(int *n) void ibv_free_device_list(ibv_device **list) ibv_context *ibv_open_device(ibv_device *device) @@ -156,3 +165,11 @@ cdef extern from 'infiniband/verbs.h': int ibv_dereg_mr(ibv_mr *mr) ibv_mw *ibv_alloc_mw(ibv_pd *pd, ibv_mw_type type) int ibv_dealloc_mw(ibv_mw *mw) + ibv_dm *ibv_alloc_dm(ibv_context *context, ibv_alloc_dm_attr *attr) + int ibv_free_dm(ibv_dm *dm) + ibv_mr *ibv_reg_dm_mr(ibv_pd *pd, ibv_dm *dm, unsigned long dm_offset, + size_t length, unsigned int access) + int ibv_memcpy_to_dm(ibv_dm *dm, unsigned long dm_offset, void *host_addr, + size_t length) + int ibv_memcpy_from_dm(void *host_addr, ibv_dm *dm, unsigned long dm_offset, + size_t length) diff --git a/pyverbs/mr.pxd b/pyverbs/mr.pxd index 91d662070135..2d76f2dfbe7c 100644 --- a/pyverbs/mr.pxd +++ b/pyverbs/mr.pxd @@ -14,3 +14,6 @@ cdef class MR(PyverbsCM): cdef class MW(PyverbsCM): cdef object pd cdef v.ibv_mw *mw + +cdef class DMMR(MR): + cdef object dm diff --git a/pyverbs/mr.pyx b/pyverbs/mr.pyx index 1ab4bd809dee..92b195f75ee2 100644 --- a/pyverbs/mr.pyx +++ b/pyverbs/mr.pyx @@ -3,6 +3,7 @@ from pyverbs.pyverbs_error import PyverbsRDMAError, PyverbsError from pyverbs.base import PyverbsRDMAErrno +from pyverbs.device cimport DM from .pd cimport PD import resource @@ -22,7 +23,7 @@ cdef class MR(PyverbsCM): MR class represents ibv_mr. Buffer allocation in done in the c'tor. Freeing it is done in close(). """ - def __cinit__(self, PD pd not None, length, access): + def __cinit__(self, PD pd not None, length, access, **kwargs): """ Allocate a user-level buffer of length and register a Memory Region of the given length and access flags. @@ -31,6 +32,8 @@ cdef class MR(PyverbsCM): :param access: Access flags, see ibv_access_flags enum :return: The newly created MR on success """ + if len(kwargs) != 0: + return #We want to enable registering an MR of size 0 but this fails with a #buffer of size 0, so in this case lets increase the buffer if length == 0: @@ -146,6 +149,44 @@ cdef class MW(PyverbsCM): self.pd = None +cdef class DMMR(MR): + def __cinit__(self, PD pd not None, length, access, **kwargs): + """ + Initializes a DMMR (Device Memory Memory Region) of the given length + and access flags using the given PD and DM objects. + :param pd: A PD object + :param length: Length in bytes + :param access: Access flags, see ibv_access_flags enum + :param kwargs: see below + :return: The newly create DMMR + + :keyword Arguments: + * *dm* (DM) + A DM (device memory) object to be used for this DMMR + * *offset* + Byte offset from the beginning of the allocated device memory + buffer + """ + dm = kwargs['dm'] + offset = kwargs['offset'] + self.mr = v.ibv_reg_dm_mr(pd.pd, (dm).dm, offset, length, access) + if self.mr == NULL: + raise PyverbsRDMAErrno('Failed to register a device MR. length: {len}, access flags: {flags}'. + format(len=length, flags=access,)) + self.pd = pd + self.dm = dm + pd.add_ref(self) + dm.add_ref(self) + self.logger.debug('Registered device ibv_mr. Length: {len}, access flags {flags}'. + format(len=length, flags=access)) + + def write(self, data, length): + return self.dm.copy_to_dm(0, data, length) + + cpdef read(self, length, offset): + return self.dm.copy_from_dm(offset, length) + + def mwtype2str(mw_type): mw_types = {1:'IBV_MW_TYPE_1', 2:'IBV_MW_TYPE_2'} try: diff --git a/pyverbs/pd.pyx b/pyverbs/pd.pyx index da580423cd81..65cc851b0ecf 100644 --- a/pyverbs/pd.pyx +++ b/pyverbs/pd.pyx @@ -4,7 +4,8 @@ import weakref from pyverbs.pyverbs_error import PyverbsRDMAError, PyverbsError from pyverbs.base import PyverbsRDMAErrno -from .mr cimport MR, MW +from pyverbs.device cimport Context, DM +from .mr cimport MR, MW, DMMR cdef extern from 'errno.h': int errno @@ -52,7 +53,7 @@ cdef class PD(PyverbsCM): self.ctx = None cdef add_ref(self, obj): - if isinstance(obj, MR): + if isinstance(obj, MR) or isinstance(obj, DMMR): self.mrs.add(obj) elif isinstance(obj, MW): self.mws.add(obj)