diff mbox series

[5/7] RDMA/bnxt_re: Add extended psn structure for 57500 adapters

Message ID 1546236607-15948-6-git-send-email-devesh.sharma@broadcom.com (mailing list archive)
State Superseded
Headers show
Series Add support Broadcom's 57500 series of adapters | expand

Commit Message

Devesh Sharma Dec. 31, 2018, 6:10 a.m. UTC
The new 57500 series of adapter has bigger psn search structure.
The size of new structure is 16B. Changing the control path memory
allocation and fast path code to accommodate the new psn
structure while maintaining the backward compatibility.

There are few additional changes listed below:
 - For 57500 chip max-sge are limited to 6 for now.
 - For 57500 chip max-receive-sge should be set to 6 for now.
 - Add driver/hardware interface structure for new chip.

Signed-off-by: Selvin Xavier <selvin.xavier@broadcom.com>
Signed-off-by: Devesh Sharma <devesh.sharma@broadcom.com>
---
 drivers/infiniband/hw/bnxt_re/ib_verbs.c |  3 ++
 drivers/infiniband/hw/bnxt_re/qplib_fp.c | 52 ++++++++++++++++++++++++--------
 drivers/infiniband/hw/bnxt_re/qplib_fp.h |  2 ++
 drivers/infiniband/hw/bnxt_re/qplib_sp.c |  3 +-
 drivers/infiniband/hw/bnxt_re/roce_hsi.h | 18 +++++++++++
 5 files changed, 64 insertions(+), 14 deletions(-)

Comments

Jason Gunthorpe Jan. 2, 2019, 9:04 p.m. UTC | #1
On Mon, Dec 31, 2018 at 01:10:05AM -0500, Devesh Sharma wrote:

> @@ -902,6 +906,7 @@ int bnxt_qplib_create_qp(struct bnxt_qplib_res *res, struct bnxt_qplib_qp *qp)
>  	}
>  	hw_sq_send_ptr = (struct sq_send **)sq->hwq.pbl_ptr;
>  	if (psn_sz) {
> +
>  		psn_search_ptr = (struct sq_psn_search **)
>  				  &hw_sq_send_ptr[get_sqe_pg
>  					(sq->hwq.max_elements)];

extra new line?

Jason

> diff --git a/drivers/infiniband/hw/bnxt_re/roce_hsi.h b/drivers/infiniband/hw/bnxt_re/roce_hsi.h
> index 42dba3a..7e3bbfd 100644
> +++ b/drivers/infiniband/hw/bnxt_re/roce_hsi.h
> @@ -162,6 +162,24 @@ struct sq_psn_search {
>  	#define SQ_PSN_SEARCH_FLAGS_SFT			    24
>  };
>  
> +/* sq_psn_search_ext (size:128b/16B) */
> +struct sq_psn_search_ext {
> +	__le32  opcode_start_psn;
> +	#define SQ_PSN_SEARCH_EXT_START_PSN_MASK 0xffffffUL
> +	#define SQ_PSN_SEARCH_EXT_START_PSN_SFT 0
> +	#define SQ_PSN_SEARCH_EXT_OPCODE_MASK   0xff000000UL
> +	#define SQ_PSN_SEARCH_EXT_OPCODE_SFT    24
> +	__le32  flags_next_psn;
> +	#define SQ_PSN_SEARCH_EXT_NEXT_PSN_MASK 0xffffffUL
> +	#define SQ_PSN_SEARCH_EXT_NEXT_PSN_SFT 0
> +	#define SQ_PSN_SEARCH_EXT_FLAGS_MASK   0xff000000UL
> +	#define SQ_PSN_SEARCH_EXT_FLAGS_SFT    24
> +	__le16  start_slot_idx;
> +	__le16  reserved16;
> +	__le32  reserved32;
> +};

More creative use of white space, can you fix all of these? 

Jason
Devesh Sharma Jan. 3, 2019, 6:03 a.m. UTC | #2
On Thu, Jan 3, 2019 at 2:34 AM Jason Gunthorpe <jgg@mellanox.com> wrote:
>
> On Mon, Dec 31, 2018 at 01:10:05AM -0500, Devesh Sharma wrote:
>
> > @@ -902,6 +906,7 @@ int bnxt_qplib_create_qp(struct bnxt_qplib_res *res, struct bnxt_qplib_qp *qp)
> >       }
> >       hw_sq_send_ptr = (struct sq_send **)sq->hwq.pbl_ptr;
> >       if (psn_sz) {
> > +
> >               psn_search_ptr = (struct sq_psn_search **)
> >                                 &hw_sq_send_ptr[get_sqe_pg
> >                                       (sq->hwq.max_elements)];
>
> extra new line?
Unforced! will fix this too.
>
> Jason
>
> > diff --git a/drivers/infiniband/hw/bnxt_re/roce_hsi.h b/drivers/infiniband/hw/bnxt_re/roce_hsi.h
> > index 42dba3a..7e3bbfd 100644
> > +++ b/drivers/infiniband/hw/bnxt_re/roce_hsi.h
> > @@ -162,6 +162,24 @@ struct sq_psn_search {
> >       #define SQ_PSN_SEARCH_FLAGS_SFT                     24
> >  };
> >
> > +/* sq_psn_search_ext (size:128b/16B) */
> > +struct sq_psn_search_ext {
> > +     __le32  opcode_start_psn;
> > +     #define SQ_PSN_SEARCH_EXT_START_PSN_MASK 0xffffffUL
> > +     #define SQ_PSN_SEARCH_EXT_START_PSN_SFT 0
> > +     #define SQ_PSN_SEARCH_EXT_OPCODE_MASK   0xff000000UL
> > +     #define SQ_PSN_SEARCH_EXT_OPCODE_SFT    24
> > +     __le32  flags_next_psn;
> > +     #define SQ_PSN_SEARCH_EXT_NEXT_PSN_MASK 0xffffffUL
> > +     #define SQ_PSN_SEARCH_EXT_NEXT_PSN_SFT 0
> > +     #define SQ_PSN_SEARCH_EXT_FLAGS_MASK   0xff000000UL
> > +     #define SQ_PSN_SEARCH_EXT_FLAGS_SFT    24
> > +     __le16  start_slot_idx;
> > +     __le16  reserved16;
> > +     __le32  reserved32;
> > +};
>
> More creative use of white space, can you fix all of these?
Will fix this too.
>
> Jason
diff mbox series

Patch

diff --git a/drivers/infiniband/hw/bnxt_re/ib_verbs.c b/drivers/infiniband/hw/bnxt_re/ib_verbs.c
index f653a14..7a09ca7 100644
--- a/drivers/infiniband/hw/bnxt_re/ib_verbs.c
+++ b/drivers/infiniband/hw/bnxt_re/ib_verbs.c
@@ -1656,6 +1656,9 @@  int bnxt_re_modify_qp(struct ib_qp *ib_qp, struct ib_qp_attr *qp_attr,
 			__from_ib_access_flags(qp_attr->qp_access_flags);
 		/* LOCAL_WRITE access must be set to allow RC receive */
 		qp->qplib_qp.access |= BNXT_QPLIB_ACCESS_LOCAL_WRITE;
+		/* Temp: Set all params on QP as of now */
+		qp->qplib_qp.access |= CMDQ_MODIFY_QP_ACCESS_REMOTE_WRITE;
+		qp->qplib_qp.access |= CMDQ_MODIFY_QP_ACCESS_REMOTE_READ;
 	}
 	if (qp_attr_mask & IB_QP_PKEY_INDEX) {
 		qp->qplib_qp.modify_flags |= CMDQ_MODIFY_QP_MODIFY_MASK_PKEY;
diff --git a/drivers/infiniband/hw/bnxt_re/qplib_fp.c b/drivers/infiniband/hw/bnxt_re/qplib_fp.c
index 55083fe..0f60571 100644
--- a/drivers/infiniband/hw/bnxt_re/qplib_fp.c
+++ b/drivers/infiniband/hw/bnxt_re/qplib_fp.c
@@ -871,10 +871,11 @@  int bnxt_qplib_create_qp(struct bnxt_qplib_res *res, struct bnxt_qplib_qp *qp)
 	unsigned long int psn_search, poff = 0;
 	struct bnxt_qplib_q *sq = &qp->sq;
 	struct bnxt_qplib_q *rq = &qp->rq;
+	int i, rc, req_size, psn_sz = 0;
 	struct bnxt_qplib_hwq *xrrq;
-	int i, rc, req_size, psn_sz;
 	u16 cmd_flags = 0, max_ssge;
 	u32 sw_prod, qp_flags = 0;
+	u16 max_rsge;
 
 	RCFW_CMD_PREP(req, CREATE_QP, cmd_flags);
 
@@ -884,8 +885,11 @@  int bnxt_qplib_create_qp(struct bnxt_qplib_res *res, struct bnxt_qplib_qp *qp)
 	req.qp_handle = cpu_to_le64(qp->qp_handle);
 
 	/* SQ */
-	psn_sz = (qp->type == CMDQ_CREATE_QP_TYPE_RC) ?
-		 sizeof(struct sq_psn_search) : 0;
+	if (qp->type == CMDQ_CREATE_QP_TYPE_RC) {
+		psn_sz = bnxt_qplib_is_chip_gen_p5(res->cctx) ?
+			 sizeof(struct sq_psn_search_ext) :
+			 sizeof(struct sq_psn_search);
+	}
 	sq->hwq.max_elements = sq->max_wqe;
 	rc = bnxt_qplib_alloc_init_hwq(res->pdev, &sq->hwq, sq->sglist,
 				       sq->nmap, &sq->hwq.max_elements,
@@ -902,6 +906,7 @@  int bnxt_qplib_create_qp(struct bnxt_qplib_res *res, struct bnxt_qplib_qp *qp)
 	}
 	hw_sq_send_ptr = (struct sq_send **)sq->hwq.pbl_ptr;
 	if (psn_sz) {
+
 		psn_search_ptr = (struct sq_psn_search **)
 				  &hw_sq_send_ptr[get_sqe_pg
 					(sq->hwq.max_elements)];
@@ -915,10 +920,16 @@  int bnxt_qplib_create_qp(struct bnxt_qplib_res *res, struct bnxt_qplib_qp *qp)
 			poff = (psn_search & ~PAGE_MASK) /
 				BNXT_QPLIB_MAX_PSNE_ENTRY_SIZE;
 		}
-		for (i = 0; i < sq->hwq.max_elements; i++)
+		for (i = 0; i < sq->hwq.max_elements; i++) {
 			sq->swq[i].psn_search =
 				&psn_search_ptr[get_psne_pg(i + poff)]
 					       [get_psne_idx(i + poff)];
+			/*psns_ext will be used only for P5 chips. */
+			sq->swq[i].psn_ext =
+				(struct sq_psn_search_ext *)
+				&psn_search_ptr[get_psne_pg(i + poff)]
+					       [get_psne_idx(i + poff)];
+		}
 	}
 	pbl = &sq->hwq.pbl[PBL_LVL_0];
 	req.sq_pbl = cpu_to_le64(pbl->pg_map_arr[0]);
@@ -1017,8 +1028,9 @@  int bnxt_qplib_create_qp(struct bnxt_qplib_res *res, struct bnxt_qplib_qp *qp)
 	req.sq_fwo_sq_sge = cpu_to_le16(
 				((max_ssge & CMDQ_CREATE_QP_SQ_SGE_MASK)
 				 << CMDQ_CREATE_QP_SQ_SGE_SFT) | 0);
+	max_rsge = bnxt_qplib_is_chip_gen_p5(res->cctx) ? 6 : rq->max_sge;
 	req.rq_fwo_rq_sge = cpu_to_le16(
-				((rq->max_sge & CMDQ_CREATE_QP_RQ_SGE_MASK)
+				((max_rsge & CMDQ_CREATE_QP_RQ_SGE_MASK)
 				 << CMDQ_CREATE_QP_RQ_SGE_SFT) | 0);
 	/* ORRQ and IRRQ */
 	if (psn_sz) {
@@ -1063,6 +1075,7 @@  int bnxt_qplib_create_qp(struct bnxt_qplib_res *res, struct bnxt_qplib_qp *qp)
 
 	qp->id = le32_to_cpu(resp.xid);
 	qp->cur_qp_state = CMDQ_MODIFY_QP_NEW_STATE_RESET;
+	qp->cctx = res->cctx;
 	INIT_LIST_HEAD(&qp->sq_flush);
 	INIT_LIST_HEAD(&qp->rq_flush);
 	rcfw->qp_tbl[qp->id].qp_id = qp->id;
@@ -1752,14 +1765,26 @@  int bnxt_qplib_post_send(struct bnxt_qplib_qp *qp,
 	}
 	swq->next_psn = sq->psn & BTH_PSN_MASK;
 	if (swq->psn_search) {
-		swq->psn_search->opcode_start_psn = cpu_to_le32(
-			((swq->start_psn << SQ_PSN_SEARCH_START_PSN_SFT) &
-			 SQ_PSN_SEARCH_START_PSN_MASK) |
-			((wqe->type << SQ_PSN_SEARCH_OPCODE_SFT) &
-			 SQ_PSN_SEARCH_OPCODE_MASK));
-		swq->psn_search->flags_next_psn = cpu_to_le32(
-			((swq->next_psn << SQ_PSN_SEARCH_NEXT_PSN_SFT) &
-			 SQ_PSN_SEARCH_NEXT_PSN_MASK));
+		u32 opcd_spsn;
+		u32 flg_npsn;
+
+		opcd_spsn = ((swq->start_psn << SQ_PSN_SEARCH_START_PSN_SFT) &
+			      SQ_PSN_SEARCH_START_PSN_MASK);
+		opcd_spsn |= ((wqe->type << SQ_PSN_SEARCH_OPCODE_SFT) &
+			       SQ_PSN_SEARCH_OPCODE_MASK);
+		flg_npsn = ((swq->next_psn << SQ_PSN_SEARCH_NEXT_PSN_SFT) &
+			     SQ_PSN_SEARCH_NEXT_PSN_MASK);
+		if (bnxt_qplib_is_chip_gen_p5(qp->cctx)) {
+			swq->psn_ext->opcode_start_psn =
+						cpu_to_le32(opcd_spsn);
+			swq->psn_ext->flags_next_psn =
+						cpu_to_le32(flg_npsn);
+		} else {
+			swq->psn_search->opcode_start_psn =
+						cpu_to_le32(opcd_spsn);
+			swq->psn_search->flags_next_psn =
+						cpu_to_le32(flg_npsn);
+		}
 	}
 queue_err:
 	if (sch_handler) {
@@ -2063,6 +2088,7 @@  static int __flush_rq(struct bnxt_qplib_q *rq, struct bnxt_qplib_qp *qp,
 		opcode = CQ_BASE_CQE_TYPE_RES_RC;
 		break;
 	case CMDQ_CREATE_QP_TYPE_UD:
+	case CMDQ_CREATE_QP_TYPE_GSI:
 		opcode = CQ_BASE_CQE_TYPE_RES_UD;
 		break;
 	}
diff --git a/drivers/infiniband/hw/bnxt_re/qplib_fp.h b/drivers/infiniband/hw/bnxt_re/qplib_fp.h
index 1f32009..c9dbe7a 100644
--- a/drivers/infiniband/hw/bnxt_re/qplib_fp.h
+++ b/drivers/infiniband/hw/bnxt_re/qplib_fp.h
@@ -106,6 +106,7 @@  struct bnxt_qplib_swq {
 	u32				start_psn;
 	u32				next_psn;
 	struct sq_psn_search		*psn_search;
+	struct sq_psn_search_ext	*psn_ext;
 };
 
 struct bnxt_qplib_swqe {
@@ -254,6 +255,7 @@  struct bnxt_qplib_q {
 struct bnxt_qplib_qp {
 	struct bnxt_qplib_pd		*pd;
 	struct bnxt_qplib_dpi		*dpi;
+	struct bnxt_qplib_chip_ctx	*cctx;
 	u64				qp_handle;
 #define        BNXT_QPLIB_QP_ID_INVALID        0xFFFFFFFF
 	u32				id;
diff --git a/drivers/infiniband/hw/bnxt_re/qplib_sp.c b/drivers/infiniband/hw/bnxt_re/qplib_sp.c
index be03b57..0013998 100644
--- a/drivers/infiniband/hw/bnxt_re/qplib_sp.c
+++ b/drivers/infiniband/hw/bnxt_re/qplib_sp.c
@@ -119,7 +119,8 @@  int bnxt_qplib_get_dev_attr(struct bnxt_qplib_rcfw *rcfw,
 	 * reporting the max number
 	 */
 	attr->max_qp_wqes -= BNXT_QPLIB_RESERVED_QP_WRS;
-	attr->max_qp_sges = sb->max_sge;
+	attr->max_qp_sges = bnxt_qplib_is_chip_gen_p5(rcfw->res->cctx) ?
+			    6 : sb->max_sge;
 	attr->max_cq = le32_to_cpu(sb->max_cq);
 	attr->max_cq_wqes = le32_to_cpu(sb->max_cqe);
 	attr->max_cq_sges = attr->max_qp_sges;
diff --git a/drivers/infiniband/hw/bnxt_re/roce_hsi.h b/drivers/infiniband/hw/bnxt_re/roce_hsi.h
index 42dba3a..7e3bbfd 100644
--- a/drivers/infiniband/hw/bnxt_re/roce_hsi.h
+++ b/drivers/infiniband/hw/bnxt_re/roce_hsi.h
@@ -162,6 +162,24 @@  struct sq_psn_search {
 	#define SQ_PSN_SEARCH_FLAGS_SFT			    24
 };
 
+/* sq_psn_search_ext (size:128b/16B) */
+struct sq_psn_search_ext {
+	__le32  opcode_start_psn;
+	#define SQ_PSN_SEARCH_EXT_START_PSN_MASK 0xffffffUL
+	#define SQ_PSN_SEARCH_EXT_START_PSN_SFT 0
+	#define SQ_PSN_SEARCH_EXT_OPCODE_MASK   0xff000000UL
+	#define SQ_PSN_SEARCH_EXT_OPCODE_SFT    24
+	__le32  flags_next_psn;
+	#define SQ_PSN_SEARCH_EXT_NEXT_PSN_MASK 0xffffffUL
+	#define SQ_PSN_SEARCH_EXT_NEXT_PSN_SFT 0
+	#define SQ_PSN_SEARCH_EXT_FLAGS_MASK   0xff000000UL
+	#define SQ_PSN_SEARCH_EXT_FLAGS_SFT    24
+	__le16  start_slot_idx;
+	__le16  reserved16;
+	__le32  reserved32;
+};
+
+
 /* Send SQ WQE (40 bytes) */
 struct sq_send {
 	u8 wqe_type;