@@ -6,6 +6,7 @@ rdma_cython_module(pyverbs
base.pyx
device.pyx
enums.pyx
+ mr.pyx
pd.pyx
)
@@ -68,6 +68,15 @@ cdef extern from 'infiniband/verbs.h':
ibv_context *context
unsigned int handle
+ cdef struct ibv_mr:
+ ibv_context *context
+ ibv_pd *pd
+ void *addr
+ size_t length
+ unsigned int handle
+ unsigned int lkey
+ unsigned int rkey
+
ibv_device **ibv_get_device_list(int *n)
void ibv_free_device_list(ibv_device **list)
ibv_context *ibv_open_device(ibv_device *device)
@@ -78,3 +87,5 @@ cdef extern from 'infiniband/verbs.h':
int index, ibv_gid *gid)
ibv_pd *ibv_alloc_pd(ibv_context *context)
int ibv_dealloc_pd(ibv_pd *pd)
+ ibv_mr *ibv_reg_mr(ibv_pd *pd, void *addr, size_t length, int access)
+ int ibv_dereg_mr(ibv_mr *mr)
new file mode 100644
@@ -0,0 +1,12 @@
+# SPDX-License-Identifier: (GPL-2.0 OR Linux-OpenIB)
+# Copyright (c) 2019, Mellanox Technologies. All rights reserved. See COPYING file
+
+from pyverbs.base cimport PyverbsCM
+from . cimport libibverbs as v
+
+
+cdef class MR(PyverbsCM):
+ cdef object pd
+ cdef v.ibv_mr *mr
+ cdef void *buf
+ cpdef read(self, length, offset)
new file mode 100644
@@ -0,0 +1,109 @@
+# SPDX-License-Identifier: (GPL-2.0 OR Linux-OpenIB)
+# Copyright (c) 2019, Mellanox Technologies. All rights reserved. See COPYING file
+
+from pyverbs.pyverbs_error import PyverbsRDMAError, PyverbsError
+from pyverbs.base import PyverbsRDMAErrno
+from .pd cimport PD
+import resource
+
+cdef extern from 'stdlib.h':
+ int posix_memalign(void **memptr, size_t alignment, size_t size)
+cdef extern from 'stdlib.h':
+ void free(void *ptr)
+cdef extern from 'string.h':
+ void *memcpy(void *dest, const void *src, size_t n)
+ void *memset(void *s, int c, size_t n)
+cdef extern from 'stdint.h':
+ ctypedef int uintptr_t
+
+
+cdef class MR(PyverbsCM):
+ """
+ MR class represents ibv_mr. Buffer allocation in done in the c'tor. Freeing
+ it is done in close().
+ """
+ def __cinit__(self, PD pd not None, length, access):
+ """
+ Allocate a user-level buffer of length <length> and register a Memory
+ Region of the given length and access flags.
+ :param pd: A PD object
+ :param length: Length in bytes
+ :param access: Access flags, see ibv_access_flags enum
+ :return: The newly created MR on success
+ """
+ #We want to enable registering an MR of size 0 but this fails with a
+ #buffer of size 0, so in this case lets increase the buffer
+ if length == 0:
+ length = 10
+ rc = posix_memalign(&self.buf, resource.getpagesize(), length)
+ if rc:
+ raise PyverbsRDMAError('Failed to allocate MR buffer of size {l}'.
+ format(l=length))
+ memset(self.buf, 0, length)
+ self.mr = v.ibv_reg_mr(<v.ibv_pd*>pd.pd, self.buf, length, access)
+ if self.mr == NULL:
+ raise PyverbsRDMAErrno('Failed to register a MR. length: {l}, access flags: {a}'.
+ format(l=length, a=access))
+ self.pd = pd
+ pd.add_ref(self)
+ self.logger.debug('Registered ibv_mr. Length: {l}, access flags {a}'.
+ format(l=length, a=access))
+
+ def __dealloc__(self):
+ self.close()
+
+ cpdef close(self):
+ """
+ Closes the underlying C object of the MR and frees the memory allocated.
+ MR may be deleted directly or indirectly by closing its context, which
+ leaves the Python PD object without the underlying C object, so during
+ destruction, need to check whether or not the C object exists.
+ :return: None
+ """
+ self.logger.debug('Closing MR')
+ if self.mr != NULL:
+ rc = v.ibv_dereg_mr(self.mr)
+ if rc != 0:
+ raise PyverbsRDMAErrno('Failed to dereg MR')
+ self.mr = NULL
+ self.pd = None
+ free(self.buf)
+ self.buf = NULL
+
+ def write(self, data, length):
+ """
+ Write user data to the MR's buffer using memcpy
+ :param data: User data to write
+ :param length: Length of the data to write
+ :return: None
+ """
+ # If data is a string, cast it to bytes as Python3 doesn't
+ # automatically convert it.
+ if isinstance(data, str):
+ data = data.encode()
+ memcpy(self.buf, <char *>data, length)
+
+ cpdef read(self, length, offset):
+ """
+ Reads data from the MR's buffer
+ :param length: Length of data to read
+ :param offset: Reading offset
+ :return: The data on the buffer in the requested offset
+ """
+ cdef char *data
+ cdef int off = offset # we can't use offset in the next line, as it is
+ # a Python object and not C
+ data = <char*>(self.buf + off)
+ return data[:length]
+
+ @property
+ def buf(self):
+ return <uintptr_t>self.buf
+
+ @property
+ def lkey(self):
+ return self.mr.lkey
+
+ @property
+ def rkey(self):
+ return self.mr.rkey
@@ -8,3 +8,5 @@ from .base cimport PyverbsCM
cdef class PD(PyverbsCM):
cdef v.ibv_pd *pd
cdef Context ctx
+ cdef add_ref(self, obj)
+ cdef object mrs
@@ -1,7 +1,10 @@
# SPDX-License-Identifier: (GPL-2.0 OR Linux-OpenIB)
# Copyright (c) 2019, Mellanox Technologies. All rights reserved.
-from pyverbs.pyverbs_error import PyverbsRDMAError
+import weakref
+
+from pyverbs.pyverbs_error import PyverbsRDMAError, PyverbsError
from pyverbs.base import PyverbsRDMAErrno
+from .mr cimport MR
cdef extern from 'errno.h':
int errno
@@ -21,6 +24,7 @@ cdef class PD(PyverbsCM):
self.ctx = context
context.add_ref(self)
self.logger.debug('PD: Allocated ibv_pd')
+ self.mrs = weakref.WeakSet()
def __dealloc__(self):
"""
@@ -38,9 +42,16 @@ cdef class PD(PyverbsCM):
:return: None
"""
self.logger.debug('Closing PD')
+ self.close_weakrefs([self.mrs])
if self.pd != NULL:
rc = v.ibv_dealloc_pd(self.pd)
if rc != 0:
raise PyverbsRDMAErrno('Failed to dealloc PD')
self.pd = NULL
self.ctx = None
+
+ cdef add_ref(self, obj):
+ if isinstance(obj, MR):
+ self.mrs.add(obj)
+ else:
+ raise PyverbsError('Unrecognized object type')
MR represnts ibv_mr. Instead of allocating a buffer prior to calling ibv_reg_mr, the user provides the MR class the required size and flags, and pyverbs allocates the needed buffer. The MR class exposes read() and write() methods to users to allow setting and reading the underlying memory buffer. Signed-off-by: Noa Osherovich <noaos@mellanox.com> --- pyverbs/CMakeLists.txt | 1 + pyverbs/libibverbs.pxd | 11 +++++ pyverbs/mr.pxd | 12 +++++ pyverbs/mr.pyx | 109 +++++++++++++++++++++++++++++++++++++++++ pyverbs/pd.pxd | 2 + pyverbs/pd.pyx | 13 ++++- 6 files changed, 147 insertions(+), 1 deletion(-) create mode 100644 pyverbs/mr.pxd create mode 100644 pyverbs/mr.pyx