| Message ID | 20210718212703.21471-2-rpearsonhpe@gmail.com (mailing list archive) |
|---|---|
| State | Superseded |
| Headers | show |
| Series | Replace AV by AH in UD sends | expand |
On Sun, Jul 18, 2021 at 04:27:00PM -0500, Bob Pearson wrote: > Make changes to rdma_user_rxe.h to allow indexing AH objects, passing > the index in UD send WRs to the driver and returning the index to the rxe > provider. This change will allow removing handling of the AV in the user > space provider. > > In order to preserve ABI compatibility for old kernel and/or rdma-core > code keep the AV in the WQE at the same offset but move to the UD specific > part of the work request since that is the only case that uses it. > > Signed-off-by: Bob Pearson <rpearsonhpe@gmail.com> > v2: > Moved AV from rxe_send_wqe to rxe_wr. > > drivers/infiniband/sw/rxe/rxe_av.c | 2 +- > drivers/infiniband/sw/rxe/rxe_verbs.c | 3 ++- > include/uapi/rdma/rdma_user_rxe.h | 10 +++++++++- > 3 files changed, 12 insertions(+), 3 deletions(-) > > diff --git a/drivers/infiniband/sw/rxe/rxe_av.c b/drivers/infiniband/sw/rxe/rxe_av.c > index da2e867a1ed9..85580ea5eed0 100644 > +++ b/drivers/infiniband/sw/rxe/rxe_av.c > @@ -107,5 +107,5 @@ struct rxe_av *rxe_get_av(struct rxe_pkt_info *pkt) > if (qp_type(pkt->qp) == IB_QPT_RC || qp_type(pkt->qp) == IB_QPT_UC) > return &pkt->qp->pri_av; > > - return (pkt->wqe) ? &pkt->wqe->av : NULL; > + return (pkt->wqe) ? &pkt->wqe->wr.wr.ud.av : NULL; > } > diff --git a/drivers/infiniband/sw/rxe/rxe_verbs.c b/drivers/infiniband/sw/rxe/rxe_verbs.c > index c223959ac174..4176fffa7fdc 100644 > +++ b/drivers/infiniband/sw/rxe/rxe_verbs.c > @@ -605,7 +605,8 @@ static void init_send_wqe(struct rxe_qp *qp, const struct ib_send_wr *ibwr, > if (qp_type(qp) == IB_QPT_UD || > qp_type(qp) == IB_QPT_SMI || > qp_type(qp) == IB_QPT_GSI) > - memcpy(&wqe->av, &to_rah(ud_wr(ibwr)->ah)->av, sizeof(wqe->av)); > + memcpy(&wqe->wr.wr.ud.av, &to_rah(ud_wr(ibwr)->ah)->av, > + sizeof(struct rxe_av)); > > if (unlikely(ibwr->send_flags & IB_SEND_INLINE)) > copy_inline_data_to_wqe(wqe, ibwr); > diff --git a/include/uapi/rdma/rdma_user_rxe.h b/include/uapi/rdma/rdma_user_rxe.h > index e283c2220aba..ad7da77dca04 100644 > +++ b/include/uapi/rdma/rdma_user_rxe.h > @@ -98,6 +98,10 @@ struct rxe_send_wr { > __u32 remote_qpn; > __u32 remote_qkey; > __u16 pkey_index; > + __u16 reserved; > + __u32 ah_num; I'd leave this as pad[5] until the later patch when you can swap in ah_hum/pad[4] > @@ -168,6 +171,11 @@ struct rxe_recv_wqe { > struct rxe_dma_info dma; > }; > > +struct rxe_create_ah_resp { > + __u32 ah_num; > + __u32 reserved; > +}; This hunk should be moved to the later patch Jason
diff --git a/drivers/infiniband/sw/rxe/rxe_av.c b/drivers/infiniband/sw/rxe/rxe_av.c index da2e867a1ed9..85580ea5eed0 100644 --- a/drivers/infiniband/sw/rxe/rxe_av.c +++ b/drivers/infiniband/sw/rxe/rxe_av.c @@ -107,5 +107,5 @@ struct rxe_av *rxe_get_av(struct rxe_pkt_info *pkt) if (qp_type(pkt->qp) == IB_QPT_RC || qp_type(pkt->qp) == IB_QPT_UC) return &pkt->qp->pri_av; - return (pkt->wqe) ? &pkt->wqe->av : NULL; + return (pkt->wqe) ? &pkt->wqe->wr.wr.ud.av : NULL; } diff --git a/drivers/infiniband/sw/rxe/rxe_verbs.c b/drivers/infiniband/sw/rxe/rxe_verbs.c index c223959ac174..4176fffa7fdc 100644 --- a/drivers/infiniband/sw/rxe/rxe_verbs.c +++ b/drivers/infiniband/sw/rxe/rxe_verbs.c @@ -605,7 +605,8 @@ static void init_send_wqe(struct rxe_qp *qp, const struct ib_send_wr *ibwr, if (qp_type(qp) == IB_QPT_UD || qp_type(qp) == IB_QPT_SMI || qp_type(qp) == IB_QPT_GSI) - memcpy(&wqe->av, &to_rah(ud_wr(ibwr)->ah)->av, sizeof(wqe->av)); + memcpy(&wqe->wr.wr.ud.av, &to_rah(ud_wr(ibwr)->ah)->av, + sizeof(struct rxe_av)); if (unlikely(ibwr->send_flags & IB_SEND_INLINE)) copy_inline_data_to_wqe(wqe, ibwr); diff --git a/include/uapi/rdma/rdma_user_rxe.h b/include/uapi/rdma/rdma_user_rxe.h index e283c2220aba..ad7da77dca04 100644 --- a/include/uapi/rdma/rdma_user_rxe.h +++ b/include/uapi/rdma/rdma_user_rxe.h @@ -98,6 +98,10 @@ struct rxe_send_wr { __u32 remote_qpn; __u32 remote_qkey; __u16 pkey_index; + __u16 reserved; + __u32 ah_num; + __u32 pad[4]; + struct rxe_av av; /* deprecated */ } ud; struct { __aligned_u64 addr; @@ -148,7 +152,6 @@ struct rxe_dma_info { struct rxe_send_wqe { struct rxe_send_wr wr; - struct rxe_av av; __u32 status; __u32 state; __aligned_u64 iova; @@ -168,6 +171,11 @@ struct rxe_recv_wqe { struct rxe_dma_info dma; }; +struct rxe_create_ah_resp { + __u32 ah_num; + __u32 reserved; +}; + struct rxe_create_cq_resp { struct mminfo mi; };
Make changes to rdma_user_rxe.h to allow indexing AH objects, passing the index in UD send WRs to the driver and returning the index to the rxe provider. This change will allow removing handling of the AV in the user space provider. In order to preserve ABI compatibility for old kernel and/or rdma-core code keep the AV in the WQE at the same offset but move to the UD specific part of the work request since that is the only case that uses it. Signed-off-by: Bob Pearson <rpearsonhpe@gmail.com> --- v2: Moved AV from rxe_send_wqe to rxe_wr. drivers/infiniband/sw/rxe/rxe_av.c | 2 +- drivers/infiniband/sw/rxe/rxe_verbs.c | 3 ++- include/uapi/rdma/rdma_user_rxe.h | 10 +++++++++- 3 files changed, 12 insertions(+), 3 deletions(-)