diff mbox

[libmlx5,v1,4/5] libmlx5: Added QP and XSRQ create/destroy flow with user index

Message ID 1447587059-26725-5-git-send-email-hagaya@dev-l-vrt-198-005.mtl.labs.mlnx (mailing list archive)
State Superseded
Headers show

Commit Message

Haggai Abramonvsky Nov. 15, 2015, 11:30 a.m. UTC 
From: Haggai Abramonvsky <hagaya@mellanox.com>

When wokring with CQE version 1, the library allocates a user-index for
each new QP/XSRQ, and this user-index is passed to the kernel.

Also in the destruction of a QP/XSRQ, the library needs to free the
user-index, so it can be reused.

In this stage, the library still doesn't work with CQE version 1,
therefore we prepared the user-index in the driver data,  but don't pass
it to the kernel.

Change-Id: Ifce2c2bb53536230bbbf54246d38f4b0c31de715
Signed-off-by: Haggai Abramovsky <hagaya@mellanox.com>
---
 src/cq.c       |  47 +++++++++++++++--
 src/mlx5-abi.h |   7 +++
 src/mlx5.c     |  10 ++++
 src/mlx5.h     |   4 +-
 src/verbs.c    | 155 ++++++++++++++++++++++++++++++++++++++++++---------------
 5 files changed, 177 insertions(+), 46 deletions(-)
diff mbox

Patch

diff --git a/src/cq.c b/src/cq.c
index a1fdac3..32f0dd4 100644
--- a/src/cq.c
+++ b/src/cq.c
@@ -721,6 +721,47 @@  static int is_equal_rsn(struct mlx5_cqe64 *cqe64, uint32_t rsn)
 	return rsn == (ntohl(cqe64->sop_drop_qpn) & 0xffffff);
 }
 
+static int is_equal_uidx(struct mlx5_cqe64 *cqe64, uint32_t uidx)
+{
+	return uidx == (ntohl(cqe64->srqn_uidx) & 0xffffff);
+}
+
+static inline int is_responder(uint8_t opcode)
+{
+	switch (opcode) {
+	case MLX5_CQE_RESP_WR_IMM:
+	case MLX5_CQE_RESP_SEND:
+	case MLX5_CQE_RESP_SEND_IMM:
+	case MLX5_CQE_RESP_SEND_INV:
+	case MLX5_CQE_RESP_ERR:
+		return 1;
+	}
+
+	return 0;
+}
+
+static inline int free_res_cqe(struct mlx5_cqe64 *cqe64, uint32_t rsn,
+			       struct mlx5_srq *srq, int cqe_version)
+{
+	if (cqe_version) {
+		if (is_equal_uidx(cqe64, rsn)) {
+			if (srq && is_responder(cqe64->op_own >> 4))
+				mlx5_free_srq_wqe(srq,
+						  ntohs(cqe64->wqe_counter));
+			return 1;
+		}
+	} else {
+		if (is_equal_rsn(cqe64, rsn)) {
+			if (srq && (ntohl(cqe64->srqn_uidx) & 0xffffff))
+				mlx5_free_srq_wqe(srq,
+						  ntohs(cqe64->wqe_counter));
+			return 1;
+		}
+	}
+
+	return 0;
+}
+
 void __mlx5_cq_clean(struct mlx5_cq *cq, uint32_t rsn, struct mlx5_srq *srq)
 {
 	uint32_t prod_index;
@@ -728,6 +769,7 @@  void __mlx5_cq_clean(struct mlx5_cq *cq, uint32_t rsn, struct mlx5_srq *srq)
 	struct mlx5_cqe64 *cqe64, *dest64;
 	void *cqe, *dest;
 	uint8_t owner_bit;
+	int cqe_version;
 
 	if (!cq)
 		return;
@@ -747,12 +789,11 @@  void __mlx5_cq_clean(struct mlx5_cq *cq, uint32_t rsn, struct mlx5_srq *srq)
 	 * Now sweep backwards through the CQ, removing CQ entries
 	 * that match our QP by copying older entries on top of them.
 	 */
+	cqe_version = (to_mctx(cq->ibv_cq.context))->cqe_version;
 	while ((int) --prod_index - (int) cq->cons_index >= 0) {
 		cqe = get_cqe(cq, prod_index & cq->ibv_cq.cqe);
 		cqe64 = (cq->cqe_sz == 64) ? cqe : cqe + 64;
-		if (is_equal_rsn(cqe64, rsn)) {
-			if (srq && (ntohl(cqe64->srqn_uidx) & 0xffffff))
-				mlx5_free_srq_wqe(srq, ntohs(cqe64->wqe_counter));
+		if (free_res_cqe(cqe64, rsn, srq, cqe_version)) {
 			++nfreed;
 		} else if (nfreed) {
 			dest = get_cqe(cq, (prod_index + nfreed) & cq->ibv_cq.cqe);
diff --git a/src/mlx5-abi.h b/src/mlx5-abi.h
index d0a0825..c2490a5 100644
--- a/src/mlx5-abi.h
+++ b/src/mlx5-abi.h
@@ -109,6 +109,9 @@  struct mlx5_create_srq_ex {
 	__u64				buf_addr;
 	__u64				db_addr;
 	__u32				flags;
+	__u32				reserved;
+	__u32                           uidx;
+	__u32                           reserved1;
 };
 
 struct mlx5_create_qp {
@@ -119,6 +122,8 @@  struct mlx5_create_qp {
 	__u32				rq_wqe_count;
 	__u32				rq_wqe_shift;
 	__u32				flags;
+	__u32                           uidx;
+	__u32                           reserved;
 };
 
 struct mlx5_create_qp_resp {
@@ -134,6 +139,8 @@  struct mlx5_create_qp_ex {
 	__u32				rq_wqe_count;
 	__u32				rq_wqe_shift;
 	__u32				flags;
+	__u32                           uidx;
+	__u32                           reserved;
 };
 
 struct mlx5_create_qp_resp_ex {
diff --git a/src/mlx5.c b/src/mlx5.c
index 512ded3..2ef5759 100644
--- a/src/mlx5.c
+++ b/src/mlx5.c
@@ -600,11 +600,21 @@  static int mlx5_init_context(struct verbs_device *vdev,
 	context->max_recv_wr	= resp.max_recv_wr;
 	context->max_srq_recv_wr = resp.max_srq_recv_wr;
 
+	if (context->cqe_version) {
+		if (context->cqe_version == 1)
+			mlx5_ctx_ops.poll_cq = mlx5_poll_cq_v1;
+		else
+			 context->cqe_version = 0;
+	}
+
 	pthread_mutex_init(&context->qp_table_mutex, NULL);
 	pthread_mutex_init(&context->srq_table_mutex, NULL);
 	for (i = 0; i < MLX5_QP_TABLE_SIZE; ++i)
 		context->qp_table[i].refcnt = 0;
 
+	for (i = 0; i < MLX5_QP_TABLE_SIZE; ++i)
+		context->uidx_table[i].refcnt = 0;
+
 	context->db_list = NULL;
 
 	pthread_mutex_init(&context->db_list_mutex, NULL);
diff --git a/src/mlx5.h b/src/mlx5.h
index 8b360fe..b57c7c7 100644
--- a/src/mlx5.h
+++ b/src/mlx5.h
@@ -242,8 +242,8 @@  enum mlx5_rsc_type {
 };
 
 struct mlx5_resource {
-	enum mlx5_rsc_type      type;
-	uint32_t                rsn;
+	enum mlx5_rsc_type  type;
+	uint32_t            rsn;
 };
 
 struct mlx5_device {
diff --git a/src/verbs.c b/src/verbs.c
index b99c8df..7d13ed2 100644
--- a/src/verbs.c
+++ b/src/verbs.c
@@ -53,6 +53,11 @@ 
 
 int mlx5_single_threaded = 0;
 
+static inline int is_xrc_tgt(int type)
+{
+	return type == IBV_QPT_XRC_RECV;
+}
+
 int mlx5_query_device(struct ibv_context *context, struct ibv_device_attr *attr)
 {
 	struct ibv_query_device cmd;
@@ -505,6 +510,8 @@  struct ibv_srq *mlx5_create_srq(struct ibv_pd *pd,
 	pthread_mutex_unlock(&ctx->srq_table_mutex);
 
 	srq->srqn = resp.srqn;
+	srq->rsc.rsn = resp.srqn;
+	srq->rsc.type = MLX5_RSC_TYPE_SRQ;
 
 	return ibsrq;
 
@@ -545,16 +552,22 @@  int mlx5_query_srq(struct ibv_srq *srq,
 int mlx5_destroy_srq(struct ibv_srq *srq)
 {
 	int ret;
+	struct mlx5_srq *msrq = to_msrq(srq);
+	struct mlx5_context *ctx = to_mctx(srq->context);
 
 	ret = ibv_cmd_destroy_srq(srq);
 	if (ret)
 		return ret;
 
-	mlx5_clear_srq(to_mctx(srq->context), to_msrq(srq)->srqn);
-	mlx5_free_db(to_mctx(srq->context), to_msrq(srq)->db);
-	mlx5_free_buf(&to_msrq(srq)->buf);
-	free(to_msrq(srq)->wrid);
-	free(to_msrq(srq));
+	if (ctx->cqe_version && msrq->rsc.type == MLX5_RSC_TYPE_XSRQ)
+		mlx5_clear_uidx(ctx, msrq->rsc.rsn);
+	else
+		mlx5_clear_srq(ctx, msrq->srqn);
+
+	mlx5_free_db(ctx, msrq->db);
+	mlx5_free_buf(&msrq->buf);
+	free(msrq->wrid);
+	free(msrq);
 
 	return 0;
 }
@@ -874,7 +887,8 @@  static void mlx5_free_qp_buf(struct mlx5_qp *qp)
 }
 
 static int init_attr_v2(struct ibv_context *context, struct mlx5_qp *qp,
-			struct ibv_qp_init_attr_ex *attr, uint32_t *uuar_index)
+			struct ibv_qp_init_attr_ex *attr, uint32_t *uuar_index,
+			uint32_t usr_idx)
 {
 	struct mlx5_create_qp_ex	cmd;
 	struct mlx5_create_qp_resp_ex	resp;
@@ -893,9 +907,12 @@  static int init_attr_v2(struct ibv_context *context, struct mlx5_qp *qp,
 	cmd.sq_wqe_count = qp->sq.wqe_cnt;
 	cmd.rq_wqe_count = qp->rq.wqe_cnt;
 	cmd.rq_wqe_shift = qp->rq.wqe_shift;
+	cmd.uidx = usr_idx;
 	err = ibv_cmd_create_qp_ex2(context, &qp->verbs_qp, sizeof(qp->verbs_qp),
-				    attr, &cmd.ibv_cmd, sizeof(cmd.ibv_cmd), sizeof(cmd),
-				    &resp.ibv_resp, sizeof(resp.ibv_resp), sizeof(resp));
+				    attr, &cmd.ibv_cmd, sizeof(cmd.ibv_cmd),
+				    offsetof(struct mlx5_create_qp_ex, uidx),
+				    &resp.ibv_resp, sizeof(resp.ibv_resp),
+				    sizeof(resp));
 	if (!err)
 		*uuar_index = resp.uuar_index;
 
@@ -903,7 +920,8 @@  static int init_attr_v2(struct ibv_context *context, struct mlx5_qp *qp,
 }
 
 static int init_attr_v1(struct ibv_context *context, struct mlx5_qp *qp,
-			struct ibv_qp_init_attr_ex *attr, uint32_t *uuar_index)
+			struct ibv_qp_init_attr_ex *attr, uint32_t *uuar_index,
+			uint32_t usr_idx)
 {
 	struct mlx5_create_qp		cmd;
 	struct mlx5_create_qp_resp	resp;
@@ -923,8 +941,11 @@  static int init_attr_v1(struct ibv_context *context, struct mlx5_qp *qp,
 	cmd.sq_wqe_count = qp->sq.wqe_cnt;
 	cmd.rq_wqe_count = qp->rq.wqe_cnt;
 	cmd.rq_wqe_shift = qp->rq.wqe_shift;
+	cmd.uidx = usr_idx;
+
 	err = ibv_cmd_create_qp_ex(context, &qp->verbs_qp, sizeof(qp->verbs_qp),
-				   attr, &cmd.ibv_cmd, sizeof(cmd),
+				   attr, &cmd.ibv_cmd,
+				   offsetof(struct mlx5_create_qp, uidx),
 				   &resp.ibv_resp, sizeof(resp));
 	if (!err)
 		*uuar_index = resp.uuar_index;
@@ -941,12 +962,13 @@  static int is_v2_qp(struct ibv_qp_init_attr_ex *attr)
 }
 
 static int qp_cmd(struct ibv_context *context, struct mlx5_qp *qp,
-		  struct ibv_qp_init_attr_ex *attr, uint32_t *uuar_index)
+		  struct ibv_qp_init_attr_ex *attr, uint32_t *uuar_index,
+		  uint32_t usr_idx)
 {
 	if (is_v2_qp(attr))
-		return init_attr_v2(context, qp, attr, uuar_index);
+		return init_attr_v2(context, qp, attr, uuar_index, usr_idx);
 	else
-		return init_attr_v1(context, qp, attr, uuar_index);
+		return init_attr_v1(context, qp, attr, uuar_index, usr_idx);
 
 }
 
@@ -958,6 +980,7 @@  struct ibv_qp *create_qp(struct ibv_context *context,
 	struct mlx5_context	       *ctx = to_mctx(context);
 	struct ibv_qp		       *ibqp;
 	uint32_t			uuar_index = 0;
+	uint32_t			usr_idx = 0;
 #ifdef MLX5_DEBUG
 	FILE *fp = ctx->dbg_fp;
 #endif
@@ -1001,22 +1024,33 @@  struct ibv_qp *create_qp(struct ibv_context *context,
 	qp->db[MLX5_SND_DBR] = 0;
 
 
-	pthread_mutex_lock(&ctx->qp_table_mutex);
+	if (!ctx->cqe_version) {
+		pthread_mutex_lock(&ctx->qp_table_mutex);
+	} else if (!is_xrc_tgt(attr->qp_type)) {
+		usr_idx = mlx5_store_uidx(ctx, qp);
+		if (usr_idx < 0) {
+			mlx5_dbg(fp, MLX5_DBG_QP, "Couldn't find free user index\n");
+			goto err_rq_db;
+		}
+	}
 
-	ret = qp_cmd(context, qp, attr, &uuar_index);
+	ret = qp_cmd(context, qp, attr, &uuar_index, usr_idx);
 	if (ret) {
 		mlx5_dbg(fp, MLX5_DBG_QP, "ret %d\n", ret);
-		goto err_rq_db;
+		goto err_free_uidx;
 	}
 
-	if (qp->sq.wqe_cnt || qp->rq.wqe_cnt) {
-		ret = mlx5_store_qp(ctx, ibqp->qp_num, qp);
-		if (ret) {
-			mlx5_dbg(fp, MLX5_DBG_QP, "ret %d\n", ret);
-			goto err_destroy;
+	if (!ctx->cqe_version) {
+		if (qp->sq.wqe_cnt || qp->rq.wqe_cnt) {
+			ret = mlx5_store_qp(ctx, ibqp->qp_num, qp);
+			if (ret) {
+				mlx5_dbg(fp, MLX5_DBG_QP, "ret %d\n", ret);
+				goto err_destroy;
+			}
 		}
+
+		pthread_mutex_unlock(&ctx->qp_table_mutex);
 	}
-	pthread_mutex_unlock(&ctx->qp_table_mutex);
 
 	map_uuar(context, qp, uuar_index);
 
@@ -1030,13 +1064,22 @@  struct ibv_qp *create_qp(struct ibv_context *context,
 	attr->cap.max_recv_wr = qp->rq.max_post;
 	attr->cap.max_recv_sge = qp->rq.max_gs;
 
+	qp->rsc.type = MLX5_RSC_TYPE_QP;
+	qp->rsc.rsn = (ctx->cqe_version && !is_xrc_tgt(attr->qp_type)) ?
+		      usr_idx : ibqp->qp_num;
+
 	return ibqp;
 
 err_destroy:
 	ibv_cmd_destroy_qp(ibqp);
 
+err_free_uidx:
+	if (!ctx->cqe_version)
+		pthread_mutex_unlock(&to_mctx(context)->qp_table_mutex);
+	else if (!is_xrc_tgt(attr->qp_type))
+		mlx5_clear_uidx(ctx, usr_idx);
+
 err_rq_db:
-	pthread_mutex_unlock(&to_mctx(context)->qp_table_mutex);
 	mlx5_free_db(to_mctx(context), qp->db);
 
 err_free_qp_buf:
@@ -1107,29 +1150,38 @@  static void mlx5_unlock_cqs(struct ibv_qp *qp)
 int mlx5_destroy_qp(struct ibv_qp *ibqp)
 {
 	struct mlx5_qp *qp = to_mqp(ibqp);
+	struct mlx5_context *ctx = to_mctx(ibqp->context);
 	int ret;
 
-	pthread_mutex_lock(&to_mctx(ibqp->context)->qp_table_mutex);
+	if (!ctx->cqe_version)
+		pthread_mutex_lock(&ctx->qp_table_mutex);
+
 	ret = ibv_cmd_destroy_qp(ibqp);
 	if (ret) {
-		pthread_mutex_unlock(&to_mctx(ibqp->context)->qp_table_mutex);
+		if (!ctx->cqe_version)
+			pthread_mutex_unlock(&ctx->qp_table_mutex);
 		return ret;
 	}
 
 	mlx5_lock_cqs(ibqp);
 
-	__mlx5_cq_clean(to_mcq(ibqp->recv_cq), ibqp->qp_num,
+	__mlx5_cq_clean(to_mcq(ibqp->recv_cq), qp->rsc.rsn,
 			ibqp->srq ? to_msrq(ibqp->srq) : NULL);
 	if (ibqp->send_cq != ibqp->recv_cq)
-		__mlx5_cq_clean(to_mcq(ibqp->send_cq), ibqp->qp_num, NULL);
+		__mlx5_cq_clean(to_mcq(ibqp->send_cq), qp->rsc.rsn, NULL);
 
-	if (qp->sq.wqe_cnt || qp->rq.wqe_cnt)
-		mlx5_clear_qp(to_mctx(ibqp->context), ibqp->qp_num);
+	if(!ctx->cqe_version) {
+		if (qp->sq.wqe_cnt || qp->rq.wqe_cnt)
+			mlx5_clear_qp(ctx, ibqp->qp_num);
+	}
 
 	mlx5_unlock_cqs(ibqp);
-	pthread_mutex_unlock(&to_mctx(ibqp->context)->qp_table_mutex);
+	if (!ctx->cqe_version)
+		pthread_mutex_unlock(&ctx->qp_table_mutex);
+	else if (!is_xrc_tgt(ibqp->qp_type))
+		mlx5_clear_uidx(ctx, qp->rsc.rsn);
 
-	mlx5_free_db(to_mctx(ibqp->context), qp->db);
+	mlx5_free_db(ctx, qp->db);
 	mlx5_free_qp_buf(qp);
 	free(qp);
 
@@ -1169,11 +1221,11 @@  int mlx5_modify_qp(struct ibv_qp *qp, struct ibv_qp_attr *attr,
 	    (attr_mask & IBV_QP_STATE) &&
 	    attr->qp_state == IBV_QPS_RESET) {
 		if (qp->recv_cq) {
-			mlx5_cq_clean(to_mcq(qp->recv_cq), qp->qp_num,
+			mlx5_cq_clean(to_mcq(qp->recv_cq), to_mqp(qp)->rsc.rsn,
 				      qp->srq ? to_msrq(qp->srq) : NULL);
 		}
 		if (qp->send_cq != qp->recv_cq && qp->send_cq)
-			mlx5_cq_clean(to_mcq(qp->send_cq), qp->qp_num, NULL);
+			mlx5_cq_clean(to_mcq(qp->send_cq), to_mqp(qp)->rsc.rsn, NULL);
 
 		mlx5_init_qp_indices(to_mqp(qp));
 		db = to_mqp(qp)->db;
@@ -1295,6 +1347,7 @@  mlx5_create_xrc_srq(struct ibv_context *context,
 	struct mlx5_context *ctx;
 	int max_sge;
 	struct ibv_srq *ibsrq;
+	int uidx;
 
 	msrq = calloc(1, sizeof(*msrq));
 	if (!msrq)
@@ -1358,28 +1411,48 @@  mlx5_create_xrc_srq(struct ibv_context *context,
 		cmd.flags = MLX5_SRQ_FLAG_SIGNATURE;
 
 	attr->attr.max_sge = msrq->max_gs;
-	pthread_mutex_lock(&ctx->srq_table_mutex);
+	if (ctx->cqe_version) {
+		uidx = mlx5_store_uidx(ctx, msrq);
+		if (uidx < 0) {
+			mlx5_dbg(fp, MLX5_DBG_QP, "Couldn't find free user index\n");
+			goto err_free_db;
+		}
+		cmd.uidx = uidx;
+	} else {
+		pthread_mutex_lock(&ctx->srq_table_mutex);
+	}
+
 	err = ibv_cmd_create_srq_ex(context, &msrq->vsrq, sizeof(msrq->vsrq),
-				    attr, &cmd.ibv_cmd, sizeof(cmd),
+				    attr, &cmd.ibv_cmd,
+				    offsetof(struct mlx5_create_srq_ex, uidx),
 				    &resp.ibv_resp, sizeof(resp));
 	if (err)
-		goto err_free_db;
+		goto err_free_uidx;
 
-	err = mlx5_store_srq(to_mctx(context), resp.srqn, msrq);
-	if (err)
-		goto err_destroy;
+	if (!ctx->cqe_version) {
+		err = mlx5_store_srq(to_mctx(context), resp.srqn, msrq);
+		if (err)
+			goto err_destroy;
 
-	pthread_mutex_unlock(&ctx->srq_table_mutex);
+		pthread_mutex_unlock(&ctx->srq_table_mutex);
+	}
 
 	msrq->srqn = resp.srqn;
+	msrq->rsc.type = MLX5_RSC_TYPE_XSRQ;
+	msrq->rsc.rsn = ctx->cqe_version ? cmd.uidx : resp.srqn;
 
 	return ibsrq;
 
 err_destroy:
 	ibv_cmd_destroy_srq(ibsrq);
 
+err_free_uidx:
+	if (ctx->cqe_version)
+		mlx5_clear_uidx(ctx, cmd.uidx);
+	else
+		pthread_mutex_unlock(&ctx->srq_table_mutex);
+
 err_free_db:
-	pthread_mutex_unlock(&ctx->srq_table_mutex);
 	mlx5_free_db(ctx, msrq->db);
 
 err_free: