@@ -1,11 +1,12 @@
# SPDX-License-Identifier: (GPL-2.0 OR Linux-OpenIB)
-# Copyright (c) 2018, Mellanox Technologies. All rights reserved. See COPYING file
+# Copyright (c) 2019, Mellanox Technologies. All rights reserved. See COPYING file
rdma_cython_module(pyverbs
- enums.pyx
+ addr.pyx
base.pyx
device.pyx
- addr.pyx
+ enums.pyx
+ pd.pyx
)
rdma_python_module(pyverbs
@@ -1,5 +1,9 @@
# SPDX-License-Identifier: (GPL-2.0 OR Linux-OpenIB)
-# Copyright (c) 2018, Mellanox Technologies. All rights reserved.
+# Copyright (c) 2019, Mellanox Technologies. All rights reserved.
cdef class PyverbsObject(object):
+ cdef object __weakref__
cdef object logger
+
+cdef class PyverbsCM(PyverbsObject):
+ cpdef close(self)
@@ -1,5 +1,5 @@
# SPDX-License-Identifier: (GPL-2.0 OR Linux-OpenIB)
-# Copyright (c) 2018, Mellanox Technologies. All rights reserved.
+# Copyright (c) 2019, Mellanox Technologies. All rights reserved.
import logging
from pyverbs.pyverbs_error import PyverbsRDMAError
@@ -22,3 +22,42 @@ cdef class PyverbsObject(object):
def set_log_level(self, val):
self.logger.setLevel(val)
+
+ def close_weakrefs(self, iterables):
+ """
+ For each iterable element of iterables, pop each element and
+ call its close() method. This method is used when an object is being
+ closed while other objects still hold C references to it; the object
+ holds weakrefs to such other object, and closes them before trying to
+ teardown the C resources.
+ :param iterables: an array of WeakSets
+ :return: None
+ """
+ # None elements can be present if an object's close() was called more
+ # than once (e.g. GC and by another object)
+ for it in iterables:
+ if it is None:
+ continue
+ while True:
+ try:
+ tmp = it.pop()
+ tmp.close()
+ except KeyError: # popping an empty set
+ break
+
+
+cdef class PyverbsCM(PyverbsObject):
+ """
+ This is a base class for pyverbs' context manager objects. It includes
+ __enter__ and __exit__ functions.
+ close() is also declared but it should be overridden by each inheriting
+ class.
+ """
+ def __enter__(self):
+ return self
+
+ def __exit__(self, exc_type, exc_value, traceback):
+ return self.close()
+
+ cpdef close(self):
+ pass
@@ -1,14 +1,15 @@
# SPDX-License-Identifier: (GPL-2.0 OR Linux-OpenIB)
# Copyright (c) 2018, Mellanox Technologies. All rights reserved. See COPYING file
-from .base cimport PyverbsObject
+from .base cimport PyverbsObject, PyverbsCM
cimport pyverbs.libibverbs as v
-cdef class Context(PyverbsObject):
+cdef class Context(PyverbsCM):
cdef v.ibv_context *context
cdef object name
- cpdef close(self)
+ cdef add_ref(self, obj)
+ cdef object pds
cdef class DeviceAttr(PyverbsObject):
cdef v.ibv_device_attr dev_attr
@@ -6,11 +6,14 @@ Device module introduces the Context and DeviceAttr class.
It allows user to open an IB device (using Context(name=<name>) and query it,
which returns a DeviceAttr object.
"""
-from .pyverbs_error import PyverbsRDMAError
+import weakref
+
+from .pyverbs_error import PyverbsRDMAError, PyverbsError
from .pyverbs_error import PyverbsUserError
from pyverbs.base import PyverbsRDMAErrno
cimport pyverbs.libibverbs as v
from pyverbs.addr cimport GID
+from pyverbs.pd cimport PD
cdef extern from 'errno.h':
int errno
@@ -55,10 +58,9 @@ class Device(PyverbsObject):
guid=guid_to_hex(self.guid))
-cdef class Context(PyverbsObject):
+cdef class Context(PyverbsCM):
"""
- Context class represents the C ibv_context. It currently allows only
- querying the underlying device.
+ Context class represents the C ibv_context.
"""
def __cinit__(self, **kwargs):
"""
@@ -71,6 +73,8 @@ cdef class Context(PyverbsObject):
"""
cdef int count
cdef v.ibv_device **dev_list
+
+ self.pds = weakref.WeakSet()
dev_name = kwargs.get('name')
if dev_name is not None:
@@ -106,6 +110,7 @@ cdef class Context(PyverbsObject):
cpdef close(self):
self.logger.debug('Closing Context')
+ self.close_weakrefs([self.pds])
if self.context != NULL:
rc = v.ibv_close_device(self.context)
if rc != 0:
@@ -131,9 +136,15 @@ cdef class Context(PyverbsObject):
rc = v.ibv_query_gid(self.context, port_num, index, &gid.gid)
if rc != 0:
raise PyverbsRDMAError('Failed to query gid {idx} of port {port}'.
- format(idx=index, port=port_num))
+ format(idx=index, port=port_num))
return gid
+ cdef add_ref(self, obj):
+ if isinstance(obj, PD):
+ self.pds.add(obj)
+ else:
+ raise PyverbsError('Unrecognized object type')
+
cdef class DeviceAttr(PyverbsObject):
"""
@@ -24,13 +24,13 @@ cdef extern from 'infiniband/verbs.h':
cdef struct ibv_device_attr:
char *fw_ver
- unsigned long node_guid;
- unsigned long sys_image_guid;
- unsigned long max_mr_size;
- unsigned long page_size_cap;
- unsigned int vendor_id;
- unsigned int vendor_part_id;
- unsigned int hw_ver;
+ unsigned long node_guid
+ unsigned long sys_image_guid
+ unsigned long max_mr_size
+ unsigned long page_size_cap
+ unsigned int vendor_id
+ unsigned int vendor_part_id
+ unsigned int hw_ver
unsigned int max_qp
unsigned int max_qp_wr
unsigned int device_cap_flags
@@ -46,29 +46,35 @@ cdef extern from 'infiniband/verbs.h':
unsigned int max_qp_init_rd_atom
unsigned int max_ee_init_rd_atom
ibv_atomic_cap atomic_cap
- unsigned int max_ee;
- unsigned int max_rdd;
- unsigned int max_mw;
- unsigned int max_raw_ipv6_qp;
- unsigned int max_raw_ethy_qp;
- unsigned int max_mcast_grp;
- unsigned int max_mcast_qp_attach;
- unsigned int max_total_mcast_qp_attach;
- unsigned int max_ah;
- unsigned int max_fmr;
- unsigned int max_map_per_fmr;
- unsigned int max_srq;
- unsigned int max_srq_wr;
- unsigned int max_srq_sge;
- unsigned int max_pkeys;
- unsigned int local_ca_ack_delay;
- unsigned int phys_port_cnt;
+ unsigned int max_ee
+ unsigned int max_rdd
+ unsigned int max_mw
+ unsigned int max_raw_ipv6_qp
+ unsigned int max_raw_ethy_qp
+ unsigned int max_mcast_grp
+ unsigned int max_mcast_qp_attach
+ unsigned int max_total_mcast_qp_attach
+ unsigned int max_ah
+ unsigned int max_fmr
+ unsigned int max_map_per_fmr
+ unsigned int max_srq
+ unsigned int max_srq_wr
+ unsigned int max_srq_sge
+ unsigned int max_pkeys
+ unsigned int local_ca_ack_delay
+ unsigned int phys_port_cnt
+
+ struct ibv_pd:
+ ibv_context *context
+ unsigned int handle
ibv_device **ibv_get_device_list(int *n)
- void ibv_free_device_list(ibv_device **list);
- ibv_context *ibv_open_device(ibv_device *device);
+ void ibv_free_device_list(ibv_device **list)
+ ibv_context *ibv_open_device(ibv_device *device)
int ibv_close_device(ibv_context *context)
int ibv_query_device(ibv_context *context, ibv_device_attr *device_attr)
unsigned long ibv_get_device_guid(ibv_device *device)
int ibv_query_gid(ibv_context *context, unsigned int port_num,
int index, ibv_gid *gid)
+ ibv_pd *ibv_alloc_pd(ibv_context *context)
+ int ibv_dealloc_pd(ibv_pd *pd)
new file mode 100644
@@ -0,0 +1,10 @@
+# SPDX-License-Identifier: (GPL-2.0 OR Linux-OpenIB)
+# Copyright (c) 2019, Mellanox Technologies. All rights reserved.
+from pyverbs.device cimport Context
+cimport pyverbs.libibverbs as v
+from .base cimport PyverbsCM
+
+
+cdef class PD(PyverbsCM):
+ cdef v.ibv_pd *pd
+ cdef Context ctx
new file mode 100644
@@ -0,0 +1,46 @@
+# SPDX-License-Identifier: (GPL-2.0 OR Linux-OpenIB)
+# Copyright (c) 2019, Mellanox Technologies. All rights reserved.
+from pyverbs.pyverbs_error import PyverbsRDMAError
+from pyverbs.base import PyverbsRDMAErrno
+
+cdef extern from 'errno.h':
+ int errno
+
+
+cdef class PD(PyverbsCM):
+ def __cinit__(self, Context context not None):
+ """
+ Initializes a PD object. A reference for the creating Context is kept
+ so that Python's GC will destroy the objects in the right order.
+ :param context: The Context object creating the PD
+ :return: The newly created PD on success
+ """
+ self.pd = v.ibv_alloc_pd(<v.ibv_context*>context.context)
+ if self.pd == NULL:
+ raise PyverbsRDMAErrno('Failed to allocate PD', errno)
+ self.ctx = context
+ context.add_ref(self)
+ self.logger.debug('PD: Allocated ibv_pd')
+
+ def __dealloc__(self):
+ """
+ Closes the inner PD.
+ :return: None
+ """
+ self.close()
+
+ cpdef close(self):
+ """
+ Closes the underlying C object of the PD.
+ PD may be deleted directly or indirectly by closing its context, which
+ leaves the Python PD object without the underlying C object, so during
+ destruction, need to check whether or not the C object exists.
+ :return: None
+ """
+ self.logger.debug('Closing PD')
+ if self.pd != NULL:
+ rc = v.ibv_dealloc_pd(self.pd)
+ if rc != 0:
+ raise PyverbsRDMAErrno('Failed to dealloc PD')
+ self.pd = NULL
+ self.ctx = None
This patch introduces PD object which represents ibv_pd. This commit also introduces the usage of weak references. In Python, a weak reference is a reference to an object not strong enough to keep it from being garbage-collected. In Pyverbs, it is used to allow an object to keep track of the objects created using it, so if an object is being destroyed (e.g. explicitly by the user), it can iterate over the dependent objects and destroy the underlying C objects. E.g.: A user opens a device and creates a PD. If the Context object is being destroyed before the PD object, the kernel will return EBUSY. To avoid that, the Context object will hold a weakref to the PD and destroy the C PD before closing itself. This renders the Python PD object useless, but it is a user's choice that we are not blocking. In order to provide a clean teardown, Pyverbs Context is also a context manager, which means itshouldbe used within a 'with' block. This way, when the block is over, teardown will be executed properly. The same applies for all other relevant pyverbs' classes. A PyverbsCM class was added as an alternative base objects to inherit from. It inherits from PyverbsObject and adds the __enter__ and __exit__ functions needed for a Python context manager. Signed-off-by: Noa Osherovich <noaos@mellanox.com> --- pyverbs/CMakeLists.txt | 7 ++--- pyverbs/base.pxd | 6 ++++- pyverbs/base.pyx | 41 ++++++++++++++++++++++++++++- pyverbs/device.pxd | 7 ++--- pyverbs/device.pyx | 21 +++++++++++---- pyverbs/libibverbs.pxd | 58 +++++++++++++++++++++++------------------- pyverbs/pd.pxd | 10 ++++++++ pyverbs/pd.pyx | 46 +++++++++++++++++++++++++++++++++ 8 files changed, 157 insertions(+), 39 deletions(-) create mode 100644 pyverbs/pd.pxd create mode 100644 pyverbs/pd.pyx