@@ -52,6 +52,12 @@ cdef extern from 'infiniband/mlx5dv.h':
mlx5dv_dc_init_attr dc_init_attr
unsigned long send_ops_flags
+ cdef struct mlx5dv_cq_init_attr:
+ unsigned long comp_mask
+ unsigned char cqe_comp_res_format
+ unsigned int flags
+ unsigned short cqe_size
+
bool mlx5dv_is_supported(v.ibv_device *device)
v.ibv_context* mlx5dv_open_device(v.ibv_device *device,
mlx5dv_context_attr *attr)
@@ -59,3 +65,6 @@ cdef extern from 'infiniband/mlx5dv.h':
v.ibv_qp *mlx5dv_create_qp(v.ibv_context *context,
v.ibv_qp_init_attr_ex *qp_attr,
mlx5dv_qp_init_attr *mlx5_qp_attr)
+ v.ibv_cq_ex *mlx5dv_create_cq(v.ibv_context *context,
+ v.ibv_cq_init_attr_ex *cq_attr,
+ mlx5dv_cq_init_attr *mlx5_cq_attr)
@@ -6,6 +6,7 @@
cimport pyverbs.providers.mlx5.libmlx5 as dv
from pyverbs.base cimport PyverbsObject
from pyverbs.device cimport Context
+from pyverbs.cq cimport CQEX
from pyverbs.qp cimport QP
@@ -26,3 +27,9 @@ cdef class Mlx5DVQPInitAttr(PyverbsObject):
cdef class Mlx5QP(QP):
cdef object dc_type
+
+cdef class Mlx5DVCQInitAttr(PyverbsObject):
+ cdef dv.mlx5dv_cq_init_attr attr
+
+cdef class Mlx5CQ(CQEX):
+ pass
@@ -7,6 +7,8 @@ cimport pyverbs.providers.mlx5.libmlx5 as dv
from pyverbs.base import PyverbsRDMAErrno
cimport pyverbs.libibverbs_enums as e
from pyverbs.qp cimport QPInitAttrEx
+from pyverbs.cq cimport CqInitAttrEx
+cimport pyverbs.libibverbs as v
from pyverbs.pd cimport PD
cdef class Mlx5DVContextAttr(PyverbsObject):
@@ -331,6 +333,107 @@ cdef class Mlx5QP(QP):
return masks[self.dc_type][dst] | e.IBV_QP_STATE
+cdef class Mlx5DVCQInitAttr(PyverbsObject):
+ """
+ Represents mlx5dv_cq_init_attr struct, initial attributes used for mlx5 CQ
+ creation.
+ """
+ def __cinit__(self, comp_mask=0, cqe_comp_res_format=0, flags=0, cqe_size=0):
+ """
+ Initializes an Mlx5CQInitAttr object with zeroes as default values.
+ :param comp_mask: Marks which of the following fields should be
+ considered. Use mlx5dv_cq_init_attr_mask enum.
+ :param cqe_comp_res_format: The various CQE response formats of the
+ responder side. Use
+ mlx5dv_cqe_comp_res_format enum.
+ :param flags: A bitwise OR of the various values described in
+ mlx5dv_cq_init_attr_flags.
+ :param cqe_size: Configure the CQE size to be 64 or 128 bytes, other
+ values will cause the CQ creation process to fail.
+ Valid when MLX5DV_CQ_INIT_ATTR_MASK_CQE_SIZE is set.
+ :return: None
+ """
+ self.attr.comp_mask = comp_mask
+ self.attr.cqe_comp_res_format = cqe_comp_res_format
+ self.attr.flags = flags
+ self.attr.cqe_size = cqe_size
+
+ @property
+ def comp_mask(self):
+ return self.attr.comp_mask
+ @comp_mask.setter
+ def comp_mask(self, val):
+ self.attr.comp_mask = val
+
+ @property
+ def cqe_comp_res_format(self):
+ return self.attr.cqe_comp_res_format
+ @cqe_comp_res_format.setter
+ def cqe_comp_res_format(self, val):
+ self.attr.cqe_comp_res_format = val
+
+ @property
+ def flags(self):
+ return self.attr.flags
+ @flags.setter
+ def flags(self, val):
+ self.attr.flags = val
+
+ @property
+ def cqe_size(self):
+ return self.attr.cqe_size
+ @cqe_size.setter
+ def cqe_size(self, val):
+ self.attr.cqe_size = val
+
+ def __str__(self):
+ print_format = '{:22}: {:<20}\n'
+ flags = {dve.MLX5DV_CQ_INIT_ATTR_FLAGS_CQE_PAD:
+ "MLX5DV_CQ_INIT_ATTR_FLAGS_CQE_PAD}"}
+ mask = {dve.MLX5DV_CQ_INIT_ATTR_MASK_COMPRESSED_CQE:
+ "MLX5DV_CQ_INIT_ATTR_MASK_COMPRESSED_CQE",
+ dve.MLX5DV_CQ_INIT_ATTR_MASK_FLAGS:
+ "MLX5DV_CQ_INIT_ATTR_MASK_FLAGS",
+ dve.MLX5DV_CQ_INIT_ATTR_MASK_CQE_SIZE:
+ "MLX5DV_CQ_INIT_ATTR_MASK_CQE_SIZE"}
+ fmt = {dve.MLX5DV_CQE_RES_FORMAT_HASH: "MLX5DV_CQE_RES_FORMAT_HASH",
+ dve.MLX5DV_CQE_RES_FORMAT_CSUM: "MLX5DV_CQE_RES_FORMAT_CSUM",
+ dve.MLX5DV_CQE_RES_FORMAT_CSUM_STRIDX:
+ "MLX5DV_CQE_RES_FORMAT_CSUM_STRIDX"}
+
+ return 'Mlx5DVCQInitAttr:\n' +\
+ print_format.format('comp_mask', bitmask_to_str(self.comp_mask,
+ mask)) +\
+ print_format.format('CQE compression format',
+ bitmask_to_str(self.cqe_comp_res_format,
+ fmt)) +\
+ print_format.format('flags', bitmask_to_str(self.flags,
+ flags)) + \
+ print_format.format('CQE size', self.cqe_size)
+
+
+cdef class Mlx5CQ(CQEX):
+ def __cinit__(self, Mlx5Context context, CqInitAttrEx init_attr,
+ Mlx5DVCQInitAttr dv_init_attr):
+ self.cq = \
+ dv.mlx5dv_create_cq(context.context, &init_attr.attr,
+ &dv_init_attr.attr if dv_init_attr is not None
+ else NULL)
+ if self.cq == NULL:
+ raise PyverbsRDMAErrno('Failed to create MLX5 CQ.\nCQInitAttrEx:\n'
+ '{}\nMLX5DVCQInitAttr:\n{}'.
+ format(init_attr, dv_init_attr))
+ self.ibv_cq = v.ibv_cq_ex_to_cq(self.cq)
+ self.context = context
+ context.add_ref(self)
+
+ def __str__(self):
+ print_format = '{:<22}: {:<20}\n'
+ return 'Mlx5 CQ:\n' +\
+ print_format.format('Handle', self.cq.handle) +\
+ print_format.format('CQEs', self.cq.cqe)
+
+
def qpts_to_str(qp_types):
numeric_types = qp_types
qpts_str = ''
@@ -66,3 +66,12 @@ cdef extern from 'infiniband/mlx5dv.h':
cpdef enum mlx5dv_qp_create_send_ops_flags:
MLX5DV_QP_EX_WITH_MR_INTERLEAVED = 1 << 0
MLX5DV_QP_EX_WITH_MR_LIST = 1 << 1
+
+ cpdef enum mlx5dv_cq_init_attr_mask:
+ MLX5DV_CQ_INIT_ATTR_MASK_COMPRESSED_CQE = 1 << 0
+ MLX5DV_CQ_INIT_ATTR_MASK_FLAGS = 1 << 1
+ MLX5DV_CQ_INIT_ATTR_MASK_CQE_SIZE = 1 << 2
+
+ cpdef enum mlx5dv_cq_init_attr_flags:
+ MLX5DV_CQ_INIT_ATTR_FLAGS_CQE_PAD = 1 << 0
+ MLX5DV_CQ_INIT_ATTR_FLAGS_RESERVED = 1 << 1
Add the needed infrastructure to allow users to create a CQ via the mlx5dv interface. Creation process requires CQInitAttrEx, similarly to the creation of an extended CQ, but requires an Mlx5DVCQInitAttr object as well (provided as kwargs, i.e. dv_init_attr=<your object>). Signed-off-by: Noa Osherovich <noaos@mellanox.com> --- pyverbs/providers/mlx5/libmlx5.pxd | 9 +++ pyverbs/providers/mlx5/mlx5dv.pxd | 7 ++ pyverbs/providers/mlx5/mlx5dv.pyx | 103 ++++++++++++++++++++++++ pyverbs/providers/mlx5/mlx5dv_enums.pxd | 9 +++ 4 files changed, 128 insertions(+)