diff mbox series

[for-next,v2,1/4] RDMA/bnxt_re: Add support for optimized modify QP

Message ID 1728928561-25607-2-git-send-email-selvin.xavier@broadcom.com (mailing list archive)
State Superseded
Headers show
Series RDMA/bnxt_re: driver update | expand

Commit Message

Selvin Xavier Oct. 14, 2024, 5:55 p.m. UTC 
From: Kalesh AP <kalesh-anakkur.purayil@broadcom.com>

Modify QP improvements are for state transitions
from INIT -> RTR and RTR -> RTS.
In order to support the Modify QP Optimization feature,
the driver is expected to check for the feature support
in the CMDQ_QUERY_FUNC and register its support for this
feature with the FW in CMDQ_INITIALIZE_FIRMWARE.

Additionally, the driver is required to specify the new
fields and attribute masks for the transitions as follows:
1. INIT -> RTR:
   - New fields: srq_used, type.
   - enable srq_used when RC QP is configured to use SRQ.
   - set the type based on the QP type.
   - Mandatory masks:
     - RC: CMDQ_MODIFY_QP_MODIFY_MASK_ACCESS,
           CMDQ_MODIFY_QP_MODIFY_MASK_PKEY
     - UD QP and QP1: CMDQ_MODIFY_QP_MODIFY_MASK_PKEY,
                      CMDQ_MODIFY_QP_MODIFY_MASK_QKEY
2. RTR -> RTS:
   - New fields: type
   - set the type based on the QP type.
   - Mandatory masks:
     - RC: CMDQ_MODIFY_QP_MODIFY_MASK_ACCESS
     - UD QP and QP1: CMDQ_MODIFY_QP_MODIFY_MASK_QKEY

Reviewed-by: Saravanan Vajravel <saravanan.vajravel@broadcom.com>
Reviewed-by: Tushar Rane <tushar.rane@broadcom.com>
Signed-off-by: Kalesh AP <kalesh-anakkur.purayil@broadcom.com>
Signed-off-by: Selvin Xavier <selvin.xavier@broadcom.com>
---
 drivers/infiniband/hw/bnxt_re/qplib_fp.c   | 40 ++++++++++++++++++++++++++++++
 drivers/infiniband/hw/bnxt_re/qplib_rcfw.c |  6 ++++-
 drivers/infiniband/hw/bnxt_re/qplib_res.h  |  5 ++++
 drivers/infiniband/hw/bnxt_re/roce_hsi.h   |  3 +++
 4 files changed, 53 insertions(+), 1 deletion(-)

Comments

Leon Romanovsky Oct. 28, 2024, 11:59 a.m. UTC | #1 
On Mon, Oct 14, 2024 at 10:55:58AM -0700, Selvin Xavier wrote:
> From: Kalesh AP <kalesh-anakkur.purayil@broadcom.com>
> 
> Modify QP improvements are for state transitions
> from INIT -> RTR and RTR -> RTS.
> In order to support the Modify QP Optimization feature,
> the driver is expected to check for the feature support
> in the CMDQ_QUERY_FUNC and register its support for this
> feature with the FW in CMDQ_INITIALIZE_FIRMWARE.
> 
> Additionally, the driver is required to specify the new
> fields and attribute masks for the transitions as follows:
> 1. INIT -> RTR:
>    - New fields: srq_used, type.
>    - enable srq_used when RC QP is configured to use SRQ.
>    - set the type based on the QP type.
>    - Mandatory masks:
>      - RC: CMDQ_MODIFY_QP_MODIFY_MASK_ACCESS,
>            CMDQ_MODIFY_QP_MODIFY_MASK_PKEY
>      - UD QP and QP1: CMDQ_MODIFY_QP_MODIFY_MASK_PKEY,
>                       CMDQ_MODIFY_QP_MODIFY_MASK_QKEY
> 2. RTR -> RTS:
>    - New fields: type
>    - set the type based on the QP type.
>    - Mandatory masks:
>      - RC: CMDQ_MODIFY_QP_MODIFY_MASK_ACCESS
>      - UD QP and QP1: CMDQ_MODIFY_QP_MODIFY_MASK_QKEY
> 
> Reviewed-by: Saravanan Vajravel <saravanan.vajravel@broadcom.com>
> Reviewed-by: Tushar Rane <tushar.rane@broadcom.com>
> Signed-off-by: Kalesh AP <kalesh-anakkur.purayil@broadcom.com>
> Signed-off-by: Selvin Xavier <selvin.xavier@broadcom.com>
> ---
>  drivers/infiniband/hw/bnxt_re/qplib_fp.c   | 40 ++++++++++++++++++++++++++++++
>  drivers/infiniband/hw/bnxt_re/qplib_rcfw.c |  6 ++++-
>  drivers/infiniband/hw/bnxt_re/qplib_res.h  |  5 ++++
>  drivers/infiniband/hw/bnxt_re/roce_hsi.h   |  3 +++
>  4 files changed, 53 insertions(+), 1 deletion(-)

<...>

> diff --git a/drivers/infiniband/hw/bnxt_re/roce_hsi.h b/drivers/infiniband/hw/bnxt_re/roce_hsi.h
> index 3ec8952..69d50d7 100644
> --- a/drivers/infiniband/hw/bnxt_re/roce_hsi.h
> +++ b/drivers/infiniband/hw/bnxt_re/roce_hsi.h
> @@ -216,6 +216,8 @@ struct cmdq_initialize_fw {
>  	__le16	flags;
>  	#define CMDQ_INITIALIZE_FW_FLAGS_MRAV_RESERVATION_SPLIT          0x1UL
>  	#define CMDQ_INITIALIZE_FW_FLAGS_HW_REQUESTER_RETX_SUPPORTED     0x2UL
> +	#define CMDQ_INITIALIZE_FW_FLAGS_DRV_VERSION                     0x4UL

Where is this define used?

> +	#define CMDQ_INITIALIZE_FW_FLAGS_OPTIMIZE_MODIFY_QP_SUPPORTED    0x8UL

Thanks
Selvin Xavier Oct. 28, 2024, 4:50 p.m. UTC | #2 
On Mon, Oct 28, 2024 at 5:29 PM Leon Romanovsky <leon@kernel.org> wrote:
>
> On Mon, Oct 14, 2024 at 10:55:58AM -0700, Selvin Xavier wrote:
> > From: Kalesh AP <kalesh-anakkur.purayil@broadcom.com>
> >
> > Modify QP improvements are for state transitions
> > from INIT -> RTR and RTR -> RTS.
> > In order to support the Modify QP Optimization feature,
> > the driver is expected to check for the feature support
> > in the CMDQ_QUERY_FUNC and register its support for this
> > feature with the FW in CMDQ_INITIALIZE_FIRMWARE.
> >
> > Additionally, the driver is required to specify the new
> > fields and attribute masks for the transitions as follows:
> > 1. INIT -> RTR:
> >    - New fields: srq_used, type.
> >    - enable srq_used when RC QP is configured to use SRQ.
> >    - set the type based on the QP type.
> >    - Mandatory masks:
> >      - RC: CMDQ_MODIFY_QP_MODIFY_MASK_ACCESS,
> >            CMDQ_MODIFY_QP_MODIFY_MASK_PKEY
> >      - UD QP and QP1: CMDQ_MODIFY_QP_MODIFY_MASK_PKEY,
> >                       CMDQ_MODIFY_QP_MODIFY_MASK_QKEY
> > 2. RTR -> RTS:
> >    - New fields: type
> >    - set the type based on the QP type.
> >    - Mandatory masks:
> >      - RC: CMDQ_MODIFY_QP_MODIFY_MASK_ACCESS
> >      - UD QP and QP1: CMDQ_MODIFY_QP_MODIFY_MASK_QKEY
> >
> > Reviewed-by: Saravanan Vajravel <saravanan.vajravel@broadcom.com>
> > Reviewed-by: Tushar Rane <tushar.rane@broadcom.com>
> > Signed-off-by: Kalesh AP <kalesh-anakkur.purayil@broadcom.com>
> > Signed-off-by: Selvin Xavier <selvin.xavier@broadcom.com>
> > ---
> >  drivers/infiniband/hw/bnxt_re/qplib_fp.c   | 40 ++++++++++++++++++++++++++++++
> >  drivers/infiniband/hw/bnxt_re/qplib_rcfw.c |  6 ++++-
> >  drivers/infiniband/hw/bnxt_re/qplib_res.h  |  5 ++++
> >  drivers/infiniband/hw/bnxt_re/roce_hsi.h   |  3 +++
> >  4 files changed, 53 insertions(+), 1 deletion(-)
>
> <...>
>
> > diff --git a/drivers/infiniband/hw/bnxt_re/roce_hsi.h b/drivers/infiniband/hw/bnxt_re/roce_hsi.h
> > index 3ec8952..69d50d7 100644
> > --- a/drivers/infiniband/hw/bnxt_re/roce_hsi.h
> > +++ b/drivers/infiniband/hw/bnxt_re/roce_hsi.h
> > @@ -216,6 +216,8 @@ struct cmdq_initialize_fw {
> >       __le16  flags;
> >       #define CMDQ_INITIALIZE_FW_FLAGS_MRAV_RESERVATION_SPLIT          0x1UL
> >       #define CMDQ_INITIALIZE_FW_FLAGS_HW_REQUESTER_RETX_SUPPORTED     0x2UL
> > +     #define CMDQ_INITIALIZE_FW_FLAGS_DRV_VERSION                     0x4UL
>
> Where is this define used?
We are not using this field now. This is structure fields are copied
from an autogenerated file. Since we updated this structure, we have
copied all update to that structure. The value we
are currently interested in is 0x8UL.

Do you want me to repost the series after removing the above define?
>
> > +     #define CMDQ_INITIALIZE_FW_FLAGS_OPTIMIZE_MODIFY_QP_SUPPORTED    0x8UL
>
> Thanks
Leon Romanovsky Oct. 28, 2024, 6:58 p.m. UTC | #3 
On Mon, Oct 28, 2024 at 10:20:00PM +0530, Selvin Xavier wrote:
> On Mon, Oct 28, 2024 at 5:29 PM Leon Romanovsky <leon@kernel.org> wrote:
> >
> > On Mon, Oct 14, 2024 at 10:55:58AM -0700, Selvin Xavier wrote:
> > > From: Kalesh AP <kalesh-anakkur.purayil@broadcom.com>
> > >
> > > Modify QP improvements are for state transitions
> > > from INIT -> RTR and RTR -> RTS.
> > > In order to support the Modify QP Optimization feature,
> > > the driver is expected to check for the feature support
> > > in the CMDQ_QUERY_FUNC and register its support for this
> > > feature with the FW in CMDQ_INITIALIZE_FIRMWARE.
> > >
> > > Additionally, the driver is required to specify the new
> > > fields and attribute masks for the transitions as follows:
> > > 1. INIT -> RTR:
> > >    - New fields: srq_used, type.
> > >    - enable srq_used when RC QP is configured to use SRQ.
> > >    - set the type based on the QP type.
> > >    - Mandatory masks:
> > >      - RC: CMDQ_MODIFY_QP_MODIFY_MASK_ACCESS,
> > >            CMDQ_MODIFY_QP_MODIFY_MASK_PKEY
> > >      - UD QP and QP1: CMDQ_MODIFY_QP_MODIFY_MASK_PKEY,
> > >                       CMDQ_MODIFY_QP_MODIFY_MASK_QKEY
> > > 2. RTR -> RTS:
> > >    - New fields: type
> > >    - set the type based on the QP type.
> > >    - Mandatory masks:
> > >      - RC: CMDQ_MODIFY_QP_MODIFY_MASK_ACCESS
> > >      - UD QP and QP1: CMDQ_MODIFY_QP_MODIFY_MASK_QKEY
> > >
> > > Reviewed-by: Saravanan Vajravel <saravanan.vajravel@broadcom.com>
> > > Reviewed-by: Tushar Rane <tushar.rane@broadcom.com>
> > > Signed-off-by: Kalesh AP <kalesh-anakkur.purayil@broadcom.com>
> > > Signed-off-by: Selvin Xavier <selvin.xavier@broadcom.com>
> > > ---
> > >  drivers/infiniband/hw/bnxt_re/qplib_fp.c   | 40 ++++++++++++++++++++++++++++++
> > >  drivers/infiniband/hw/bnxt_re/qplib_rcfw.c |  6 ++++-
> > >  drivers/infiniband/hw/bnxt_re/qplib_res.h  |  5 ++++
> > >  drivers/infiniband/hw/bnxt_re/roce_hsi.h   |  3 +++
> > >  4 files changed, 53 insertions(+), 1 deletion(-)
> >
> > <...>
> >
> > > diff --git a/drivers/infiniband/hw/bnxt_re/roce_hsi.h b/drivers/infiniband/hw/bnxt_re/roce_hsi.h
> > > index 3ec8952..69d50d7 100644
> > > --- a/drivers/infiniband/hw/bnxt_re/roce_hsi.h
> > > +++ b/drivers/infiniband/hw/bnxt_re/roce_hsi.h
> > > @@ -216,6 +216,8 @@ struct cmdq_initialize_fw {
> > >       __le16  flags;
> > >       #define CMDQ_INITIALIZE_FW_FLAGS_MRAV_RESERVATION_SPLIT          0x1UL
> > >       #define CMDQ_INITIALIZE_FW_FLAGS_HW_REQUESTER_RETX_SUPPORTED     0x2UL
> > > +     #define CMDQ_INITIALIZE_FW_FLAGS_DRV_VERSION                     0x4UL
> >
> > Where is this define used?
> We are not using this field now. This is structure fields are copied
> from an autogenerated file. Since we updated this structure, we have
> copied all update to that structure. The value we
> are currently interested in is 0x8UL.
> 
> Do you want me to repost the series after removing the above define?

I will remove it by myself, no need to repost.

Thanks

> >
> > > +     #define CMDQ_INITIALIZE_FW_FLAGS_OPTIMIZE_MODIFY_QP_SUPPORTED    0x8UL
> >
> > Thanks
diff mbox series

Patch

diff --git a/drivers/infiniband/hw/bnxt_re/qplib_fp.c b/drivers/infiniband/hw/bnxt_re/qplib_fp.c
index 42e98e5..ff2340c 100644
--- a/drivers/infiniband/hw/bnxt_re/qplib_fp.c
+++ b/drivers/infiniband/hw/bnxt_re/qplib_fp.c
@@ -1277,6 +1277,40 @@  static void __filter_modify_flags(struct bnxt_qplib_qp *qp)
 	}
 }
 
+static void bnxt_set_mandatory_attributes(struct bnxt_qplib_qp *qp,
+					  struct cmdq_modify_qp *req)
+{
+	u32 mandatory_flags = 0;
+
+	if (qp->type == CMDQ_MODIFY_QP_QP_TYPE_RC)
+		mandatory_flags |= CMDQ_MODIFY_QP_MODIFY_MASK_ACCESS;
+
+	if (qp->cur_qp_state == CMDQ_MODIFY_QP_NEW_STATE_INIT &&
+	    qp->state == CMDQ_MODIFY_QP_NEW_STATE_RTR) {
+		if (qp->type == CMDQ_MODIFY_QP_QP_TYPE_RC && qp->srq)
+			req->flags = cpu_to_le16(CMDQ_MODIFY_QP_FLAGS_SRQ_USED);
+		mandatory_flags |= CMDQ_MODIFY_QP_MODIFY_MASK_PKEY;
+	}
+
+	if (qp->type == CMDQ_MODIFY_QP_QP_TYPE_UD ||
+	    qp->type == CMDQ_MODIFY_QP_QP_TYPE_GSI)
+		mandatory_flags |= CMDQ_MODIFY_QP_MODIFY_MASK_QKEY;
+
+	qp->modify_flags |= mandatory_flags;
+	req->qp_type = qp->type;
+}
+
+static bool is_optimized_state_transition(struct bnxt_qplib_qp *qp)
+{
+	if ((qp->cur_qp_state == CMDQ_MODIFY_QP_NEW_STATE_INIT &&
+	     qp->state == CMDQ_MODIFY_QP_NEW_STATE_RTR) ||
+	    (qp->cur_qp_state == CMDQ_MODIFY_QP_NEW_STATE_RTR &&
+	     qp->state == CMDQ_MODIFY_QP_NEW_STATE_RTS))
+		return true;
+
+	return false;
+}
+
 int bnxt_qplib_modify_qp(struct bnxt_qplib_res *res, struct bnxt_qplib_qp *qp)
 {
 	struct bnxt_qplib_rcfw *rcfw = res->rcfw;
@@ -1293,6 +1327,12 @@  int bnxt_qplib_modify_qp(struct bnxt_qplib_res *res, struct bnxt_qplib_qp *qp)
 
 	/* Filter out the qp_attr_mask based on the state->new transition */
 	__filter_modify_flags(qp);
+	if (qp->modify_flags & CMDQ_MODIFY_QP_MODIFY_MASK_STATE) {
+		/* Set mandatory attributes for INIT -> RTR and RTR -> RTS transition */
+		if (_is_optimize_modify_qp_supported(res->dattr->dev_cap_flags2) &&
+		    is_optimized_state_transition(qp))
+			bnxt_set_mandatory_attributes(qp, &req);
+	}
 	bmask = qp->modify_flags;
 	req.modify_mask = cpu_to_le32(qp->modify_flags);
 	req.qp_cid = cpu_to_le32(qp->id);
diff --git a/drivers/infiniband/hw/bnxt_re/qplib_rcfw.c b/drivers/infiniband/hw/bnxt_re/qplib_rcfw.c
index 3ffaef0c..f5713e3 100644
--- a/drivers/infiniband/hw/bnxt_re/qplib_rcfw.c
+++ b/drivers/infiniband/hw/bnxt_re/qplib_rcfw.c
@@ -832,6 +832,7 @@  int bnxt_qplib_init_rcfw(struct bnxt_qplib_rcfw *rcfw,
 	struct creq_initialize_fw_resp resp = {};
 	struct cmdq_initialize_fw req = {};
 	struct bnxt_qplib_cmdqmsg msg = {};
+	u16 flags = 0;
 	u8 pgsz, lvl;
 	int rc;
 
@@ -906,7 +907,10 @@  int bnxt_qplib_init_rcfw(struct bnxt_qplib_rcfw *rcfw,
 
 skip_ctx_setup:
 	if (BNXT_RE_HW_RETX(rcfw->res->dattr->dev_cap_flags))
-		req.flags |= cpu_to_le16(CMDQ_INITIALIZE_FW_FLAGS_HW_REQUESTER_RETX_SUPPORTED);
+		flags |= CMDQ_INITIALIZE_FW_FLAGS_HW_REQUESTER_RETX_SUPPORTED;
+	if (_is_optimize_modify_qp_supported(rcfw->res->dattr->dev_cap_flags2))
+		flags |= CMDQ_INITIALIZE_FW_FLAGS_OPTIMIZE_MODIFY_QP_SUPPORTED;
+	req.flags |= cpu_to_le16(flags);
 	req.stat_ctx_id = cpu_to_le32(ctx->stats.fw_id);
 	bnxt_qplib_fill_cmdqmsg(&msg, &req, &resp, NULL, sizeof(req), sizeof(resp), 0);
 	rc = bnxt_qplib_rcfw_send_message(rcfw, &msg);
diff --git a/drivers/infiniband/hw/bnxt_re/qplib_res.h b/drivers/infiniband/hw/bnxt_re/qplib_res.h
index c2f7103..ef198a6 100644
--- a/drivers/infiniband/hw/bnxt_re/qplib_res.h
+++ b/drivers/infiniband/hw/bnxt_re/qplib_res.h
@@ -576,4 +576,9 @@  static inline bool _is_relaxed_ordering_supported(u16 dev_cap_ext_flags2)
 	return dev_cap_ext_flags2 & CREQ_QUERY_FUNC_RESP_SB_MEMORY_REGION_RO_SUPPORTED;
 }
 
+static inline bool _is_optimize_modify_qp_supported(u16 dev_cap_ext_flags2)
+{
+	return dev_cap_ext_flags2 & CREQ_QUERY_FUNC_RESP_SB_OPTIMIZE_MODIFY_QP_SUPPORTED;
+}
+
 #endif /* __BNXT_QPLIB_RES_H__ */
diff --git a/drivers/infiniband/hw/bnxt_re/roce_hsi.h b/drivers/infiniband/hw/bnxt_re/roce_hsi.h
index 3ec8952..69d50d7 100644
--- a/drivers/infiniband/hw/bnxt_re/roce_hsi.h
+++ b/drivers/infiniband/hw/bnxt_re/roce_hsi.h
@@ -216,6 +216,8 @@  struct cmdq_initialize_fw {
 	__le16	flags;
 	#define CMDQ_INITIALIZE_FW_FLAGS_MRAV_RESERVATION_SPLIT          0x1UL
 	#define CMDQ_INITIALIZE_FW_FLAGS_HW_REQUESTER_RETX_SUPPORTED     0x2UL
+	#define CMDQ_INITIALIZE_FW_FLAGS_DRV_VERSION                     0x4UL
+	#define CMDQ_INITIALIZE_FW_FLAGS_OPTIMIZE_MODIFY_QP_SUPPORTED    0x8UL
 	__le16	cookie;
 	u8	resp_size;
 	u8	reserved8;
@@ -559,6 +561,7 @@  struct cmdq_modify_qp {
 	#define CMDQ_MODIFY_QP_OPCODE_LAST     CMDQ_MODIFY_QP_OPCODE_MODIFY_QP
 	u8	cmd_size;
 	__le16	flags;
+	 #define CMDQ_MODIFY_QP_FLAGS_SRQ_USED       0x1UL
 	__le16	cookie;
 	u8	resp_size;
 	u8	qp_type;