[rdma-core,1/8] pyverbs: Add support for memory window

Commit Message

Noa Osherovich Dec. 31, 2019, 9:19 a.m. UTC

In order to use memory window, some additions were needed:
- Allow pyverbs users to create MWBindInfo, the Python representation
  of ibv_mw_bind_info.
- Expose ibv_inc_rkey, which creates a new rkey from a given one by
  increasing its 8 LSBs while keeping the same index.
- Expose the memory window's rkey and handle properties.

Signed-off-by: Ahmad Ghazawi <[e-mail]>
Signed-off-by: Noa Osherovich <[e-mail]>
Reviewed-by: Edward Srouji <[e-mail]>
---
 pyverbs/base.pyx       |  8 +++++++-
 pyverbs/libibverbs.pxd |  1 +
 pyverbs/mr.pxd         |  6 ++++++
 pyverbs/mr.pyx         | 19 +++++++++++++++++++
 4 files changed, 33 insertions(+), 1 deletion(-)

Patch

diff --git a/pyverbs/base.pyx b/pyverbs/base.pyx
index c5b16795ddb6..790ba4153dea 100644
--- a/pyverbs/base.pyx
+++ b/pyverbs/base.pyx
@@ -1,9 +1,15 @@ 
 # SPDX-License-Identifier: (GPL-2.0 OR Linux-OpenIB)
 # Copyright (c) 2019, Mellanox Technologies. All rights reserved.
 
+from libc.errno cimport errno
 import logging
+
 from pyverbs.pyverbs_error import PyverbsRDMAError
-from libc.errno cimport errno
+cimport pyverbs.libibverbs as v
+
+
+def inc_rkey(rkey):
+    return v.ibv_inc_rkey(rkey)
 
 
 cpdef PyverbsRDMAErrno(str msg):
diff --git a/pyverbs/libibverbs.pxd b/pyverbs/libibverbs.pxd
index ad8d8bacc541..fea8a1b408da 100755
--- a/pyverbs/libibverbs.pxd
+++ b/pyverbs/libibverbs.pxd
@@ -551,3 +551,4 @@  cdef extern from 'infiniband/verbs.h':
                           ibv_recv_wr **bad_recv_wr)
     ibv_pd *ibv_alloc_parent_domain(ibv_context *context,
                                     ibv_parent_domain_init_attr *attr)
+    uint32_t ibv_inc_rkey(uint32_t rkey)
diff --git a/pyverbs/mr.pxd b/pyverbs/mr.pxd
index fb46611e6f42..402df4492425 100644
--- a/pyverbs/mr.pxd
+++ b/pyverbs/mr.pxd
@@ -12,6 +12,12 @@  cdef class MR(PyverbsCM):
     cdef v.ibv_mr *mr
     cdef void *buf
     cpdef read(self, length, offset)
+    cdef add_ref(self, obj)
+    cdef object bind_infos
+
+cdef class MWBindInfo(PyverbsCM):
+    cdef v.ibv_mw_bind_info info
+    cdef object mr
 
 cdef class MW(PyverbsCM):
     cdef object pd
diff --git a/pyverbs/mr.pyx b/pyverbs/mr.pyx
index 6b28c8173ef8..9b1277f0882f 100644
--- a/pyverbs/mr.pyx
+++ b/pyverbs/mr.pyx
@@ -3,10 +3,12 @@ 
 
 import resource
 import logging
+import weakref
 
 from pyverbs.pyverbs_error import PyverbsRDMAError, PyverbsError
 from pyverbs.base import PyverbsRDMAErrno
 from posix.stdlib cimport posix_memalign
+from pyverbs.base cimport close_weakrefs
 from libc.string cimport memcpy, memset
 from libc.stdint cimport uintptr_t
 from pyverbs.device cimport DM
@@ -29,6 +31,7 @@  cdef class MR(PyverbsCM):
         :return: The newly created MR on success
         """
         super().__init__()
+        self.bind_infos = weakref.WeakSet()
         if self.mr != NULL:
             return
         #We want to enable registering an MR of size 0 but this fails with a
@@ -61,6 +64,7 @@  cdef class MR(PyverbsCM):
         :return: None
         """
         self.logger.debug('Closing MR')
+        close_weakrefs([self.bind_infos])
         if self.mr != NULL:
             rc = v.ibv_dereg_mr(self.mr)
             if rc != 0:
@@ -96,6 +100,10 @@  cdef class MR(PyverbsCM):
         data = <char*>(self.buf + off)
         return data[:length]
 
+    cdef add_ref(self, obj):
+        if isinstance(obj, MWBindInfo):
+            self.bind_infos.add(obj)
+
     @property
     def buf(self):
         return <uintptr_t>self.buf
@@ -109,6 +117,17 @@  cdef class MR(PyverbsCM):
         return self.mr.rkey
 
 
+cdef class MWBindInfo(PyverbsCM):
+    def __init__(self, MR mr not None, addr, length, mw_access_flags):
+        super().__init__()
+        self.mr = mr
+        self.info.mr = mr.mr
+        self.info.addr = addr
+        self.info.length = length
+        self.info.mw_access_flags = mw_access_flags
+        mr.add_ref(self)
+
+
 cdef class MW(PyverbsCM):
     def __init__(self, PD pd not None, v.ibv_mw_type mw_type):
         """