[V1,rdma-core,4/4] mlx5: Destroy and modify a DC target QP

Commit Message

Yishai Hadas Jan. 21, 2018, 4:16 p.m. UTC

From: Moni Shoua <[e-mail]>

Changes mlx5_destroy_qp() and mlx5_modify_qp() to handle DC target QP.

Since DCT QP is a software abstraction for a DC target hardware object
the flow to destroy and modify need a special treatment. When destroying
a DCT there are no resources that needs to be released besides the
memory that this QP takes.

Also, a DC target QP is created in hardware only when it is modified to
RTR. Therefore the QPN should be taken from the response of the command
to modify a QP and put in the software QP. For that we have to use the
interface that allows to get data from the vendor channel of the
response.

Signed-off-by: Moni Shoua <[e-mail]>
Reviewed-by: Yishai Hadas <[e-mail]>
---
 providers/mlx5/mlx5-abi.h |  6 ++++
 providers/mlx5/verbs.c    | 76 ++++++++++++++++++++++++++++++++++++++++++++---
 2 files changed, 78 insertions(+), 4 deletions(-)

Patch

diff --git a/providers/mlx5/mlx5-abi.h b/providers/mlx5/mlx5-abi.h
index 6d5793f..3e4207c 100644
--- a/providers/mlx5/mlx5-abi.h
+++ b/providers/mlx5/mlx5-abi.h
@@ -329,4 +329,10 @@  struct mlx5_query_device_ex_resp {
 	__u32				reserved;
 };
 
+struct mlx5_modify_qp_resp_ex {
+	struct ibv_modify_qp_resp_ex base;
+	__u32  response_length;
+	__u32  dctn;
+};
+
 #endif /* MLX5_ABI_H */
diff --git a/providers/mlx5/verbs.c b/providers/mlx5/verbs.c
index 800a5e7..ff29b61 100644
--- a/providers/mlx5/verbs.c
+++ b/providers/mlx5/verbs.c
@@ -1915,8 +1915,16 @@  int mlx5_destroy_qp(struct ibv_qp *ibqp)
 		__mlx5_cq_clean(to_mcq(ibqp->send_cq), qp->rsc.rsn, NULL);
 
 	if (!ctx->cqe_version) {
-		if (qp->sq.wqe_cnt || qp->rq.wqe_cnt)
-			mlx5_clear_qp(ctx, ibqp->qp_num);
+		if (qp->dc_type == MLX5DV_DCTYPE_DCT) {
+			/* The QP was inserted to the tracking table only after
+			 * that it was modifed to RTR
+			 */
+			if (ibqp->state == IBV_QPS_RTR)
+				mlx5_clear_qp(ctx, ibqp->qp_num);
+		} else {
+			if (qp->sq.wqe_cnt || qp->rq.wqe_cnt)
+				mlx5_clear_qp(ctx, ibqp->qp_num);
+		}
 	}
 
 	mlx5_unlock_cqs(ibqp);
@@ -1925,8 +1933,10 @@  int mlx5_destroy_qp(struct ibv_qp *ibqp)
 	else if (!is_xrc_tgt(ibqp->qp_type))
 		mlx5_clear_uidx(ctx, qp->rsc.rsn);
 
-	mlx5_free_db(ctx, qp->db);
-	mlx5_free_qp_buf(qp);
+	if (qp->dc_type != MLX5DV_DCTYPE_DCT) {
+		mlx5_free_db(ctx, qp->db);
+		mlx5_free_qp_buf(qp);
+	}
 free:
 	if (mparent_domain)
 		atomic_fetch_sub(&mparent_domain->mpd.refcount, 1);
@@ -1963,6 +1973,61 @@  enum {
 	MLX5_MODIFY_QP_EX_ATTR_MASK = IBV_QP_RATE_LIMIT,
 };
 
+static int modify_dct(struct ibv_qp *qp, struct ibv_qp_attr *attr,
+		      int attr_mask)
+{
+	struct ibv_modify_qp_ex cmd_ex = {};
+	struct mlx5_modify_qp_resp_ex resp = {};
+	struct mlx5_qp *mqp = to_mqp(qp);
+	struct mlx5_context *context = to_mctx(qp->context);
+	int min_resp_size;
+	bool dct_create;
+	int ret;
+
+	ret = ibv_cmd_modify_qp_ex(qp, attr, attr_mask,
+				   &cmd_ex,
+				   sizeof(cmd_ex), sizeof(cmd_ex),
+				   &resp.base,
+				   sizeof(resp.base), sizeof(resp));
+	if (ret)
+		return ret;
+
+	/* dct is created in hardware and gets unique qp number when QP
+	 * is modified to RTR so operations that require QP number need
+	 * to be delayed to this time
+	 */
+	dct_create =
+		(attr_mask & IBV_QP_STATE) &&
+		(attr->qp_state == IBV_QPS_RTR);
+
+	if (!dct_create)
+		return 0;
+
+	min_resp_size =
+		offsetof(typeof(resp), dctn) +
+		sizeof(resp.dctn) -
+		sizeof(resp.base);
+
+	if (resp.response_length < min_resp_size) {
+		errno = EINVAL;
+		return errno;
+	}
+
+	qp->qp_num = resp.dctn;
+
+	if (!context->cqe_version) {
+		pthread_mutex_lock(&context->qp_table_mutex);
+		ret = mlx5_store_qp(context, qp->qp_num, mqp);
+		if (!ret)
+			mqp->rsc.rsn = qp->qp_num;
+		else
+			errno = ENOMEM;
+		pthread_mutex_unlock(&context->qp_table_mutex);
+		return ret ? errno : 0;
+	}
+	return 0;
+}
+
 int mlx5_modify_qp(struct ibv_qp *qp, struct ibv_qp_attr *attr,
 		   int attr_mask)
 {
@@ -1974,6 +2039,9 @@  int mlx5_modify_qp(struct ibv_qp *qp, struct ibv_qp_attr *attr,
 	int ret;
 	__be32 *db;
 
+	if (mqp->dc_type == MLX5DV_DCTYPE_DCT)
+		return modify_dct(qp, attr, attr_mask);
+
 	if (mqp->rss_qp)
 		return ENOSYS;