@@ -12,6 +12,7 @@ rdma_cython_module(pyverbs
qp.pyx
wr.pyx
xrcd.pyx
+ srq.pyx
)
rdma_python_module(pyverbs
@@ -19,6 +19,7 @@ cdef class CQ(PyverbsCM):
cdef object context
cdef add_ref(self, obj)
cdef object qps
+ cdef object srqs
cdef class CqInitAttrEx(PyverbsObject):
cdef v.ibv_cq_init_attr_ex attr
@@ -31,6 +32,7 @@ cdef class CQEX(PyverbsCM):
cdef object context
cdef add_ref(self, obj)
cdef object qps
+ cdef object srqs
cdef class WC(PyverbsObject):
cdef v.ibv_wc wc
@@ -2,9 +2,11 @@
# Copyright (c) 2019, Mellanox Technologies. All rights reserved.
import weakref
+from pyverbs.pyverbs_error import PyverbsError
from pyverbs.base import PyverbsRDMAErrno
cimport pyverbs.libibverbs_enums as e
from pyverbs.device cimport Context
+from pyverbs.srq cimport SRQ
from pyverbs.qp cimport QP
cdef class CompChannel(PyverbsCM):
@@ -90,18 +92,23 @@ cdef class CQ(PyverbsCM):
self.context = context
context.add_ref(self)
self.qps = weakref.WeakSet()
+ self.srqs = weakref.WeakSet()
self.logger.debug('Created a CQ')
cdef add_ref(self, obj):
if isinstance(obj, QP):
self.qps.add(obj)
+ elif isinstance(obj, SRQ):
+ self.srqs.add(obj)
+ else:
+ raise PyverbsError('Unrecognized object type')
def __dealloc__(self):
self.close()
cpdef close(self):
self.logger.debug('Closing CQ')
- self.close_weakrefs([self.qps])
+ self.close_weakrefs([self.qps, self.srqs])
if self.cq != NULL:
rc = v.ibv_destroy_cq(self.cq)
if rc != 0:
@@ -270,6 +277,10 @@ cdef class CQEX(PyverbsCM):
cdef add_ref(self, obj):
if isinstance(obj, QP):
self.qps.add(obj)
+ elif isinstance(obj, SRQ):
+ self.srqs.add(obj)
+ else:
+ raise PyverbsError('Unrecognized object type')
def __dealloc__(self):
self.close()
@@ -349,6 +349,32 @@ cdef extern from 'infiniband/verbs.h':
cdef struct ibv_xrcd:
pass
+ cdef struct ibv_srq_attr:
+ unsigned int max_wr
+ unsigned int max_sge
+ unsigned int srq_limit
+
+ cdef struct ibv_srq_init_attr:
+ void *srq_context
+ ibv_srq_attr attr
+
+ cdef struct ibv_srq_init_attr_ex:
+ void *srq_context
+ ibv_srq_attr attr
+ unsigned int comp_mask
+ ibv_srq_type srq_type
+ ibv_pd *pd
+ ibv_xrcd *xrcd
+ ibv_cq *cq
+ ibv_tm_caps tm_cap
+
+ cdef struct ibv_srq:
+ ibv_context *context
+ void *srq_context
+ ibv_pd *pd
+ unsigned int handle
+ unsigned int events_completed
+
cdef struct ibv_rwq_ind_table:
pass
@@ -498,3 +524,12 @@ cdef extern from 'infiniband/verbs.h':
ibv_xrcd *ibv_open_xrcd(ibv_context *context,
ibv_xrcd_init_attr *xrcd_init_attr)
int ibv_close_xrcd(ibv_xrcd *xrcd)
+ ibv_srq *ibv_create_srq(ibv_pd *pd, ibv_srq_init_attr *srq_init_attr)
+ ibv_srq *ibv_create_srq_ex(ibv_context *context,
+ ibv_srq_init_attr_ex *srq_init_attr)
+ int ibv_modify_srq(ibv_srq *srq, ibv_srq_attr *srq_attr, int srq_attr_mask)
+ int ibv_query_srq(ibv_srq *srq, ibv_srq_attr *srq_attr)
+ int ibv_get_srq_num(ibv_srq *srq, unsigned int *srq_num)
+ int ibv_destroy_srq(ibv_srq *srq)
+ int ibv_post_srq_recv(ibv_srq *srq, ibv_recv_wr *recv_wr,
+ ibv_recv_wr **bad_recv_wr)
@@ -12,6 +12,7 @@ cdef class PD(PyverbsCM):
cdef v.ibv_pd *pd
cdef Context ctx
cdef add_ref(self, obj)
+ cdef object srqs
cdef object mrs
cdef object mws
cdef object ahs
@@ -6,6 +6,7 @@ from pyverbs.pyverbs_error import PyverbsRDMAError, PyverbsError
from pyverbs.base import PyverbsRDMAErrno
from pyverbs.device cimport Context, DM
from .mr cimport MR, MW, DMMR
+from pyverbs.srq cimport SRQ
from pyverbs.addr cimport AH
from pyverbs.qp cimport QP
@@ -27,6 +28,7 @@ cdef class PD(PyverbsCM):
self.ctx = context
context.add_ref(self)
self.logger.debug('PD: Allocated ibv_pd')
+ self.srqs = weakref.WeakSet()
self.mrs = weakref.WeakSet()
self.mws = weakref.WeakSet()
self.ahs = weakref.WeakSet()
@@ -48,7 +50,7 @@ cdef class PD(PyverbsCM):
:return: None
"""
self.logger.debug('Closing PD')
- self.close_weakrefs([self.qps, self.ahs, self.mws, self.mrs])
+ self.close_weakrefs([self.qps, self.ahs, self.mws, self.mrs, self.srqs])
if self.pd != NULL:
rc = v.ibv_dealloc_pd(self.pd)
if rc != 0:
@@ -65,5 +67,7 @@ cdef class PD(PyverbsCM):
self.ahs.add(obj)
elif isinstance(obj, QP):
self.qps.add(obj)
+ elif isinstance(obj, SRQ):
+ self.srqs.add(obj)
else:
raise PyverbsError('Unrecognized object type')
@@ -13,12 +13,14 @@ cdef class QPInitAttr(PyverbsObject):
cdef v.ibv_qp_init_attr attr
cdef object scq
cdef object rcq
+ cdef object srq
cdef class QPInitAttrEx(PyverbsObject):
cdef v.ibv_qp_init_attr_ex attr
cdef object scq
cdef object rcq
cdef object xrcd
+ cdef object srq
cdef object pd
cdef class QPAttr(PyverbsObject):
@@ -12,6 +12,7 @@ from pyverbs.device cimport Context
from pyverbs.cq cimport CQ, CQEX
cimport pyverbs.libibverbs as v
from pyverbs.xrcd cimport XRCD
+from pyverbs.srq cimport SRQ
from pyverbs.pd cimport PD
@@ -86,7 +87,7 @@ cdef class QPCap(PyverbsObject):
cdef class QPInitAttr(PyverbsObject):
def __cinit__(self, qp_type=e.IBV_QPT_UD, qp_context=None,
PyverbsObject scq=None, PyverbsObject rcq=None,
- object srq=None, QPCap cap=None, sq_sig_all=1):
+ SRQ srq=None, QPCap cap=None, sq_sig_all=1):
"""
Initializes a QpInitAttr object representing ibv_qp_init_attr struct.
Note that SRQ object is not yet supported in pyverbs so can't be passed
@@ -95,7 +96,7 @@ cdef class QPInitAttr(PyverbsObject):
:param qp_context: Associated QP context
:param scq: Send CQ to be used for this QP
:param rcq: Receive CQ to be used for this QP
- :param srq: Not yet supported
+ :param srq: Shared receive queue to be used as RQ in QP
:param cap: A QPCap object
:param sq_sig_all: If set, each send WR will generate a completion
entry
@@ -122,10 +123,10 @@ cdef class QPInitAttr(PyverbsObject):
raise PyverbsUserError('Expected CQ/CQEX, got {t}'.\
format(t=type(rcq)))
self.rcq = rcq
-
- self.attr.srq = NULL # Until SRQ support is added
self.attr.qp_type = qp_type
self.attr.sq_sig_all = sq_sig_all
+ self.srq = srq
+ self.attr.srq = srq.srq if srq else NULL
@property
def send_cq(self):
@@ -138,6 +139,14 @@ cdef class QPInitAttr(PyverbsObject):
self.attr.send_cq = (<CQEX>val).ibv_cq
self.scq = val
+ @property
+ def srq(self):
+ return self.srq
+ @srq.setter
+ def srq(self, SRQ val):
+ self.attr.srq = <v.ibv_srq*>val.srq
+ self.srq = val
+
@property
def recv_cq(self):
return self.rcq
@@ -193,7 +202,7 @@ cdef class QPInitAttr(PyverbsObject):
cdef class QPInitAttrEx(PyverbsObject):
def __cinit__(self, qp_type=e.IBV_QPT_UD, qp_context=None,
PyverbsObject scq=None, PyverbsObject rcq=None,
- object srq=None, QPCap cap=None, sq_sig_all=0, comp_mask=0,
+ SRQ srq=None, QPCap cap=None, sq_sig_all=0, comp_mask=0,
PD pd=None, XRCD xrcd=None, create_flags=0,
max_tso_header=0, source_qpn=0, object hash_conf=None,
object ind_table=None):
@@ -203,7 +212,7 @@ cdef class QPInitAttrEx(PyverbsObject):
:param qp_context: Associated user context
:param scq: Send CQ to be used for this QP
:param rcq: Recv CQ to be used for this QP
- :param srq: Not yet supported
+ :param srq: Shared receive queue to be used as RQ in QP
:param cap: A QPCap object
:param sq_sig_all: If set, each send WR will generate a completion
entry
@@ -240,7 +249,8 @@ cdef class QPInitAttrEx(PyverbsObject):
format(t=type(rcq)))
self.rcq = rcq
- self.attr.srq = NULL # Until SRQ support is added
+ self.srq = srq
+ self.attr.srq = srq.srq if srq else NULL
self.xrcd = xrcd
self.attr.xrcd = xrcd.xrcd if xrcd else NULL
self.attr.rwq_ind_tbl = NULL # Until RSS support is added
@@ -327,6 +337,14 @@ cdef class QPInitAttrEx(PyverbsObject):
self.attr.xrcd = <v.ibv_xrcd*>val.xrcd
self.xrcd = val
+ @property
+ def srq(self):
+ return self.srq
+ @srq.setter
+ def srq(self, SRQ val):
+ self.attr.srq = <v.ibv_srq*>val.srq
+ self.srq = val
+
@property
def create_flags(self):
return self.attr.create_flags
new file mode 100755
@@ -0,0 +1,24 @@
+# SPDX-License-Identifier: (GPL-2.0 OR Linux-OpenIB)
+# Copyright (c) 2019 Mellanox Technologies, Inc. All rights reserved.
+
+#cython: language_level=3
+
+from pyverbs.base cimport PyverbsObject, PyverbsCM
+from . cimport libibverbs as v
+
+cdef class SrqAttr(PyverbsObject):
+ cdef v.ibv_srq_attr attr
+
+cdef class SrqInitAttr(PyverbsObject):
+ cdef v.ibv_srq_init_attr attr
+
+cdef class SrqInitAttrEx(PyverbsObject):
+ cdef v.ibv_srq_init_attr_ex attr
+ cdef object _cq
+ cdef object _pd
+ cdef object _xrcd
+
+cdef class SRQ(PyverbsCM):
+ cdef v.ibv_srq *srq
+ cdef object cq
+ cpdef close(self)
new file mode 100755
@@ -0,0 +1,176 @@
+from pyverbs.pyverbs_error import PyverbsRDMAError
+from pyverbs.base import PyverbsRDMAErrno
+from pyverbs.device cimport Context
+from pyverbs.cq cimport CQEX, CQ
+from pyverbs.xrcd cimport XRCD
+from pyverbs.wr cimport RecvWR
+from pyverbs.pd cimport PD
+
+
+cdef extern from 'errno.h':
+ int errno
+cdef extern from 'string.h':
+ void *memcpy(void *dest, const void *src, size_t n)
+
+
+cdef class SrqAttr(PyverbsObject):
+ def __cinit__(self, max_wr=100, max_sge=1, srq_limit=0):
+ self.attr.max_wr = max_wr
+ self.attr.max_sge = max_sge
+ self.attr.srq_limit = srq_limit
+
+ @property
+ def max_wr(self):
+ return self.attr.max_wr
+ @max_wr.setter
+ def max_wr(self, val):
+ self.attr.max_wr = val
+
+ @property
+ def max_sge(self):
+ return self.attr.max_sge
+ @max_sge.setter
+ def max_sge(self, val):
+ self.attr.max_sge = val
+
+ @property
+ def srq_limit(self):
+ return self.attr.srq_limit
+ def srq_limit(self, val):
+ self.attr.srq_limit = val
+
+
+cdef class SrqInitAttr(PyverbsObject):
+ def __cinit__(self, SrqAttr attr = None):
+ if attr is not None:
+ self.attr.attr.max_wr = attr.max_wr
+ self.attr.attr.max_sge = attr.max_sge
+ self.attr.attr.srq_limit = attr.srq_limit
+
+
+cdef class SrqInitAttrEx(PyverbsObject):
+ def __cinit__(self, max_wr=100, max_sge=1, srq_limit=0):
+ self.attr.attr.max_wr = max_wr
+ self.attr.attr.max_sge = max_sge
+ self.attr.attr.srq_limit = srq_limit
+ self._cq = None
+ self._pd = None
+ self._xrcd = None
+
+ @property
+ def comp_mask(self):
+ return self.attr.comp_mask
+ @comp_mask.setter
+ def comp_mask(self, val):
+ self.attr.comp_mask = val
+
+ @property
+ def srq_type(self):
+ return self.attr.srq_type
+ @srq_type.setter
+ def srq_type(self, val):
+ self.attr.srq_type = val
+
+ @property
+ def pd(self):
+ return self._pd
+ @pd.setter
+ def pd(self, PD val):
+ self._pd = val
+ self.attr.pd = val.pd
+
+ @property
+ def xrcd(self):
+ return self._xrcd
+ @xrcd.setter
+ def xrcd(self, XRCD val):
+ self._xrcd = val
+ self.attr.xrcd = val.xrcd
+
+ @property
+ def cq(self):
+ return self._cq
+ @cq.setter
+ def cq(self, val):
+ if type(val) == CQ:
+ self.attr.cq = (<CQ>val).cq
+ self._cq = val
+ else:
+ self.attr.cq = (<CQEX>val).ibv_cq
+ self._cq = val
+
+
+cdef class SRQ(PyverbsCM):
+ def __cinit__(self, object creator not None, object attr not None):
+ self.srq = NULL
+ self.cq = None
+ if type(creator) == PD:
+ self._create_srq(creator, attr)
+ elif type(creator) == Context:
+ self._create_srq_ex(creator, attr)
+ else:
+ raise PyverbsRDMAError('Srq needs either Context or PD for creation')
+ if self.srq == NULL:
+ raise PyverbsRDMAErrno('Failed to create SRQ (errno is {err})'.
+ format(err=errno))
+ self.logger.debug('SRQ Created')
+
+ def __dealloc__(self):
+ self.close()
+
+ cpdef close(self):
+ self.logger.debug('Closing SRQ')
+ if self.srq != NULL:
+ rc = v.ibv_destroy_srq(self.srq)
+ if rc != 0:
+ raise PyverbsRDMAErrno('Failed to destroy SRQ (errno is {err})'.
+ format(err=errno))
+ self.srq = NULL
+ self.cq =None
+
+ def _create_srq(self, PD pd, SrqInitAttr init_attr):
+ self.srq = v.ibv_create_srq(pd.pd, &init_attr.attr)
+
+ def _create_srq_ex(self, Context context, SrqInitAttrEx init_attr_ex):
+ self.srq = v.ibv_create_srq_ex(context.context, &init_attr_ex.attr)
+ if init_attr_ex.cq:
+ cq = <CQ>init_attr_ex.cq
+ cq.add_ref(self)
+ self.cq = cq
+ if init_attr_ex.xrcd:
+ xrcd = <XRCD>init_attr_ex.xrcd
+ xrcd.add_ref(self)
+ if init_attr_ex.pd:
+ pd = <PD>init_attr_ex.pd
+ pd.add_ref(self)
+
+ def get_srq_num(self):
+ cdef unsigned int srqn
+ rc = v.ibv_get_srq_num(self.srq, &srqn)
+ if rc != 0:
+ raise PyverbsRDMAErrno('Failed to retrieve SRQ number (returned {rc})'.
+ format(rc=rc))
+ return srqn
+
+ def modify(self, SrqAttr attr, comp_mask):
+ rc = v.ibv_modify_srq(self.srq, &attr.attr, comp_mask)
+ if rc != 0:
+ raise PyverbsRDMAErrno('Failed to modify SRQ ({err})'.
+ format(err=errno))
+
+ def query(self):
+ attr = SrqAttr()
+ rc = v.ibv_query_srq(self.srq, &attr.attr)
+ if rc != 0:
+ raise PyverbsRDMAErrno('Failed to query SRQ ({err})'.
+ format(err=errno))
+ return attr
+
+ def post_recv(self, RecvWR wr not None, RecvWR bad_wr=None):
+ cdef v.ibv_recv_wr *my_bad_wr
+ rc = v.ibv_post_srq_recv(self.srq, &wr.recv_wr, &my_bad_wr)
+ if rc != 0:
+ if bad_wr:
+ memcpy(&bad_wr.recv_wr, my_bad_wr, sizeof(bad_wr.recv_wr))
+ raise PyverbsRDMAErrno('Failed to post receive to SRQ ({err})'.
+ format(err=rc))
@@ -13,4 +13,5 @@ cdef class XRCD(PyverbsCM):
cdef v.ibv_xrcd *xrcd
cdef Context ctx
cdef add_ref(self, obj)
+ cdef object srqs
cdef object qps
@@ -5,6 +5,7 @@ import weakref
from pyverbs.pyverbs_error import PyverbsRDMAError, PyverbsError
from pyverbs.base import PyverbsRDMAErrno
from pyverbs.device cimport Context
+from pyverbs.srq cimport SRQ
from pyverbs.qp cimport QP
cdef extern from 'errno.h':
@@ -53,6 +54,7 @@ cdef class XRCD(PyverbsCM):
self.ctx = context
context.add_ref(self)
self.logger.debug('XRCD: Allocated ibv_xrcd')
+ self.srqs = weakref.WeakSet()
self.qps = weakref.WeakSet()
def __dealloc__(self):
@@ -68,7 +70,7 @@ cdef class XRCD(PyverbsCM):
:return: None
"""
self.logger.debug('Closing XRCD')
- self.close_weakrefs([self.qps])
+ self.close_weakrefs([self.qps, self.srqs])
# XRCD may be deleted directly or indirectly by closing its context,
# which leaves the Python XRCD object without the underlying C object,
# so during destruction, need to check whether or not the C object
@@ -83,5 +85,7 @@ cdef class XRCD(PyverbsCM):
cdef add_ref(self, obj):
if isinstance(obj, QP):
self.qps.add(obj)
+ elif isinstance(obj, SRQ):
+ self.srqs.add(obj)
else:
raise PyverbsError('Unrecognized object type')