From patchwork Tue Feb 2 18:50:57 2016 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Majd Dibbiny X-Patchwork-Id: 8194101 Return-Path: X-Original-To: patchwork-linux-rdma@patchwork.kernel.org Delivered-To: patchwork-parsemail@patchwork1.web.kernel.org Received: from mail.kernel.org (mail.kernel.org [198.145.29.136]) by patchwork1.web.kernel.org (Postfix) with ESMTP id 21F829F3CD for ; Tue, 2 Feb 2016 18:51:38 +0000 (UTC) Received: from mail.kernel.org (localhost [127.0.0.1]) by mail.kernel.org (Postfix) with ESMTP id BFEAB2025B for ; Tue, 2 Feb 2016 18:51:36 +0000 (UTC) Received: from vger.kernel.org (vger.kernel.org [209.132.180.67]) by mail.kernel.org (Postfix) with ESMTP id 63279202C8 for ; Tue, 2 Feb 2016 18:51:35 +0000 (UTC) Received: (majordomo@vger.kernel.org) by vger.kernel.org via listexpand id S1752018AbcBBSvc (ORCPT ); Tue, 2 Feb 2016 13:51:32 -0500 Received: from [193.47.165.129] ([193.47.165.129]:49947 "EHLO mellanox.co.il" rhost-flags-FAIL-FAIL-OK-FAIL) by vger.kernel.org with ESMTP id S964984AbcBBSv1 (ORCPT ); Tue, 2 Feb 2016 13:51:27 -0500 Received: from Internal Mail-Server by MTLPINE1 (envelope-from majd@mellanox.com) with ESMTPS (AES256-SHA encrypted); 2 Feb 2016 20:51:04 +0200 Received: from vnc21.mtl.labs.mlnx (vnc21.mtl.labs.mlnx [10.7.2.21]) by labmailer.mlnx (8.13.8/8.13.8) with ESMTP id u12Ip40W029333; Tue, 2 Feb 2016 20:51:04 +0200 From: Majd Dibbiny To: yishaih@mellanox.com Cc: linux-rdma@vger.kernel.org, matanb@mellanox.com, talal@mellanox.com, majd@mellanox.com, cl@linux.com, Haggai Abramovsky Subject: [PATCH libmlx5 3/7] Add new poll_cq according to the new CQE format Date: Tue, 2 Feb 2016 20:50:57 +0200 Message-Id: <1454439061-929-4-git-send-email-majd@mellanox.com> X-Mailer: git-send-email 1.7.11.1 In-Reply-To: <1454439061-929-1-git-send-email-majd@mellanox.com> References: <1454439061-929-1-git-send-email-majd@mellanox.com> Sender: linux-rdma-owner@vger.kernel.org Precedence: bulk List-ID: X-Mailing-List: linux-rdma@vger.kernel.org X-Spam-Status: No, score=-7.3 required=5.0 tests=BAYES_00, RCVD_IN_DNSWL_HI, RP_MATCHES_RCVD, UNPARSEABLE_RELAY autolearn=ham version=3.3.1 X-Spam-Checker-Version: SpamAssassin 3.3.1 (2010-03-16) on mail.kernel.org X-Virus-Scanned: ClamAV using ClamSMTP From: Haggai Abramovsky When working with CQE version 1, the CQE format is different, therefore we need a new poll_cq function that is compatible with the new CQE format. In the coming patches in this series, the library will decide on the cqe version when allocating the user-context and accordingly decide which poll_cq to use for this user-context. Reviewed-by:: Yishai Hadas Signed-off-by: Haggai Abramovsky Signed-off-by: Majd Dibbiny --- src/cq.c | 226 +++++++++++++++++++++++++++++++++++++++++++++++++------------ src/mlx5.h | 11 +++ 2 files changed, 194 insertions(+), 43 deletions(-) diff --git a/src/cq.c b/src/cq.c index 231b278..eadc2c7 100644 --- a/src/cq.c +++ b/src/cq.c @@ -116,7 +116,7 @@ struct mlx5_cqe64 { uint16_t slid; uint32_t flags_rqpn; uint8_t rsvd28[4]; - uint32_t srqn; + uint32_t srqn_uidx; uint32_t imm_inval_pkey; uint8_t rsvd40[4]; uint32_t byte_cnt; @@ -362,26 +362,153 @@ static void mlx5_get_cycles(uint64_t *cycles) } #endif -static int mlx5_poll_one(struct mlx5_cq *cq, - struct mlx5_qp **cur_qp, +static inline struct mlx5_qp *get_req_context(struct mlx5_context *mctx, + struct mlx5_resource **cur_rsc, + uint32_t rsn, int cqe_ver) + __attribute__((always_inline)); +static inline struct mlx5_qp *get_req_context(struct mlx5_context *mctx, + struct mlx5_resource **cur_rsc, + uint32_t rsn, int cqe_ver) +{ + if (!*cur_rsc || (rsn != (*cur_rsc)->rsn)) + *cur_rsc = cqe_ver ? mlx5_find_uidx(mctx, rsn) : + (struct mlx5_resource *)mlx5_find_qp(mctx, rsn); + + return rsc_to_mqp(*cur_rsc); +} + +static inline int get_resp_ctx_v1(struct mlx5_context *mctx, + struct mlx5_resource **cur_rsc, + struct mlx5_srq **cur_srq, + uint32_t uidx, uint8_t *is_srq) + __attribute__((always_inline)); +static inline int get_resp_ctx_v1(struct mlx5_context *mctx, + struct mlx5_resource **cur_rsc, + struct mlx5_srq **cur_srq, + uint32_t uidx, uint8_t *is_srq) +{ + struct mlx5_qp *mqp; + + if (!*cur_rsc || (uidx != (*cur_rsc)->rsn)) { + *cur_rsc = mlx5_find_uidx(mctx, uidx); + if (unlikely(!*cur_rsc)) + return CQ_POLL_ERR; + } + + switch ((*cur_rsc)->type) { + case MLX5_RSC_TYPE_QP: + mqp = rsc_to_mqp(*cur_rsc); + if (mqp->verbs_qp.qp.srq) { + *cur_srq = to_msrq(mqp->verbs_qp.qp.srq); + *is_srq = 1; + } + break; + case MLX5_RSC_TYPE_XSRQ: + *cur_srq = rsc_to_msrq(*cur_rsc); + *is_srq = 1; + break; + default: + return CQ_POLL_ERR; + } + + return CQ_OK; +} + +static inline int get_qp_ctx(struct mlx5_context *mctx, + struct mlx5_resource **cur_rsc, + uint32_t qpn) + __attribute__((always_inline)); +static inline int get_qp_ctx(struct mlx5_context *mctx, + struct mlx5_resource **cur_rsc, + uint32_t qpn) +{ + if (!*cur_rsc || (qpn != (*cur_rsc)->rsn)) { + /* + * We do not have to take the QP table lock here, + * because CQs will be locked while QPs are removed + * from the table. + */ + *cur_rsc = (struct mlx5_resource *)mlx5_find_qp(mctx, qpn); + if (unlikely(!*cur_rsc)) + return CQ_POLL_ERR; + } + + return CQ_OK; +} + +static inline int get_srq_ctx(struct mlx5_context *mctx, + struct mlx5_srq **cur_srq, + uint32_t srqn_uidx) + __attribute__((always_inline)); +static inline int get_srq_ctx(struct mlx5_context *mctx, + struct mlx5_srq **cur_srq, + uint32_t srqn) +{ + if (!*cur_srq || (srqn != (*cur_srq)->srqn)) { + *cur_srq = mlx5_find_srq(mctx, srqn); + if (unlikely(!*cur_srq)) + return CQ_POLL_ERR; + } + + return CQ_OK; +} + +static inline int get_cur_rsc(struct mlx5_context *mctx, + int cqe_ver, + uint32_t qpn, + uint32_t srqn_uidx, + struct mlx5_resource **cur_rsc, + struct mlx5_srq **cur_srq, + uint8_t *is_srq) +{ + int err; + + if (cqe_ver) { + err = get_resp_ctx_v1(mctx, cur_rsc, cur_srq, srqn_uidx, + is_srq); + } else { + if (srqn_uidx) { + *is_srq = 1; + err = get_srq_ctx(mctx, cur_srq, srqn_uidx); + } else { + err = get_qp_ctx(mctx, cur_rsc, qpn); + } + } + + return err; + +} + +static inline int mlx5_poll_one(struct mlx5_cq *cq, + struct mlx5_resource **cur_rsc, + struct mlx5_srq **cur_srq, + struct ibv_wc *wc, int cqe_ver) + __attribute__((always_inline)); +static inline int mlx5_poll_one(struct mlx5_cq *cq, + struct mlx5_resource **cur_rsc, struct mlx5_srq **cur_srq, - struct ibv_wc *wc) + struct ibv_wc *wc, int cqe_ver) { struct mlx5_cqe64 *cqe64; struct mlx5_wq *wq; uint16_t wqe_ctr; void *cqe; uint32_t qpn; - uint32_t srqn; + uint32_t srqn_uidx; int idx; uint8_t opcode; struct mlx5_err_cqe *ecqe; int err; + struct mlx5_qp *mqp; + struct mlx5_context *mctx; + uint8_t is_srq = 0; cqe = next_cqe_sw(cq); if (!cqe) return CQ_EMPTY; + mctx = to_mctx(cq->ibv_cq.context); + cqe64 = (cq->cqe_sz == 64) ? cqe : cqe + 64; ++cq->cons_index; @@ -396,51 +523,33 @@ static int mlx5_poll_one(struct mlx5_cq *cq, #ifdef MLX5_DEBUG if (mlx5_debug_mask & MLX5_DBG_CQ_CQE) { - FILE *fp = to_mctx(cq->ibv_cq.context)->dbg_fp; + FILE *fp = mctx->dbg_fp; mlx5_dbg(fp, MLX5_DBG_CQ_CQE, "dump cqe for cqn 0x%x:\n", cq->cqn); dump_cqe(fp, cqe64); } #endif - srqn = ntohl(cqe64->srqn) & 0xffffff; qpn = ntohl(cqe64->sop_drop_qpn) & 0xffffff; - if (srqn) { - if (!*cur_srq || (srqn != (*cur_srq)->srqn)) { - *cur_srq = mlx5_find_srq(to_mctx(cq->ibv_cq.context), - srqn); - if (unlikely(!*cur_srq)) - return CQ_POLL_ERR; - } - } else { - if (!*cur_qp || (qpn != (*cur_qp)->ibv_qp->qp_num)) { - /* - * We do not have to take the QP table lock here, - * because CQs will be locked while QPs are removed - * from the table. - */ - *cur_qp = mlx5_find_qp(to_mctx(cq->ibv_cq.context), - qpn); - if (unlikely(!*cur_qp)) - return CQ_POLL_ERR; - } - } - wc->wc_flags = 0; wc->qp_num = qpn; - opcode = cqe64->op_own >> 4; switch (opcode) { case MLX5_CQE_REQ: - wq = &(*cur_qp)->sq; + mqp = get_req_context(mctx, cur_rsc, + (cqe_ver ? (ntohl(cqe64->srqn_uidx) & 0xffffff) : qpn), + cqe_ver); + if (unlikely(!mqp)) + return CQ_POLL_ERR; + wq = &mqp->sq; wqe_ctr = ntohs(cqe64->wqe_counter); idx = wqe_ctr & (wq->wqe_cnt - 1); handle_good_req(wc, cqe64); if (cqe64->op_own & MLX5_INLINE_SCATTER_32) - err = mlx5_copy_to_send_wqe(*cur_qp, wqe_ctr, cqe, + err = mlx5_copy_to_send_wqe(mqp, wqe_ctr, cqe, wc->byte_len); else if (cqe64->op_own & MLX5_INLINE_SCATTER_64) - err = mlx5_copy_to_send_wqe(*cur_qp, wqe_ctr, cqe - 1, + err = mlx5_copy_to_send_wqe(mqp, wqe_ctr, cqe - 1, wc->byte_len); else err = 0; @@ -453,20 +562,27 @@ static int mlx5_poll_one(struct mlx5_cq *cq, case MLX5_CQE_RESP_SEND: case MLX5_CQE_RESP_SEND_IMM: case MLX5_CQE_RESP_SEND_INV: - wc->status = handle_responder(wc, cqe64, *cur_qp, - srqn ? *cur_srq : NULL); + srqn_uidx = ntohl(cqe64->srqn_uidx) & 0xffffff; + err = get_cur_rsc(mctx, cqe_ver, qpn, srqn_uidx, cur_rsc, + cur_srq, &is_srq); + if (unlikely(err)) + return CQ_POLL_ERR; + + wc->status = handle_responder(wc, cqe64, rsc_to_mqp(*cur_rsc), + is_srq ? *cur_srq : NULL); break; case MLX5_CQE_RESIZE_CQ: break; case MLX5_CQE_REQ_ERR: case MLX5_CQE_RESP_ERR: + srqn_uidx = ntohl(cqe64->srqn_uidx) & 0xffffff; ecqe = (struct mlx5_err_cqe *)cqe64; mlx5_handle_error_cqe(ecqe, wc); if (unlikely(ecqe->syndrome != MLX5_CQE_SYNDROME_WR_FLUSH_ERR && ecqe->syndrome != MLX5_CQE_SYNDROME_TRANSPORT_RETRY_EXC_ERR)) { - FILE *fp = to_mctx(cq->ibv_cq.context)->dbg_fp; + FILE *fp = mctx->dbg_fp; fprintf(fp, PFX "%s: got completion with error:\n", - to_mctx(cq->ibv_cq.context)->hostname); + mctx->hostname); dump_cqe(fp, ecqe); if (mlx5_freeze_on_error_cqe) { fprintf(fp, PFX "freezing at poll cq..."); @@ -476,18 +592,28 @@ static int mlx5_poll_one(struct mlx5_cq *cq, } if (opcode == MLX5_CQE_REQ_ERR) { - wq = &(*cur_qp)->sq; + mqp = get_req_context(mctx, cur_rsc, + (cqe_ver ? srqn_uidx : qpn), cqe_ver); + if (unlikely(!mqp)) + return CQ_POLL_ERR; + wq = &mqp->sq; wqe_ctr = ntohs(cqe64->wqe_counter); idx = wqe_ctr & (wq->wqe_cnt - 1); wc->wr_id = wq->wrid[idx]; wq->tail = wq->wqe_head[idx] + 1; } else { - if (*cur_srq) { + err = get_cur_rsc(mctx, cqe_ver, qpn, srqn_uidx, + cur_rsc, cur_srq, &is_srq); + if (unlikely(err)) + return CQ_POLL_ERR; + + if (is_srq) { wqe_ctr = ntohs(cqe64->wqe_counter); wc->wr_id = (*cur_srq)->wrid[wqe_ctr]; mlx5_free_srq_wqe(*cur_srq, wqe_ctr); } else { - wq = &(*cur_qp)->rq; + mqp = rsc_to_mqp(*cur_rsc); + wq = &mqp->rq; wc->wr_id = wq->wrid[wq->tail & (wq->wqe_cnt - 1)]; ++wq->tail; } @@ -498,10 +624,14 @@ static int mlx5_poll_one(struct mlx5_cq *cq, return CQ_OK; } -int mlx5_poll_cq(struct ibv_cq *ibcq, int ne, struct ibv_wc *wc) +static inline int poll_cq(struct ibv_cq *ibcq, int ne, + struct ibv_wc *wc, int cqe_ver) + __attribute__((always_inline)); +static inline int poll_cq(struct ibv_cq *ibcq, int ne, + struct ibv_wc *wc, int cqe_ver) { struct mlx5_cq *cq = to_mcq(ibcq); - struct mlx5_qp *qp = NULL; + struct mlx5_resource *rsc = NULL; struct mlx5_srq *srq = NULL; int npolled; int err = CQ_OK; @@ -519,7 +649,7 @@ int mlx5_poll_cq(struct ibv_cq *ibcq, int ne, struct ibv_wc *wc) mlx5_spin_lock(&cq->lock); for (npolled = 0; npolled < ne; ++npolled) { - err = mlx5_poll_one(cq, &qp, &srq, wc + npolled); + err = mlx5_poll_one(cq, &rsc, &srq, wc + npolled, cqe_ver); if (err != CQ_OK) break; } @@ -551,6 +681,16 @@ int mlx5_poll_cq(struct ibv_cq *ibcq, int ne, struct ibv_wc *wc) return err == CQ_POLL_ERR ? err : npolled; } +int mlx5_poll_cq(struct ibv_cq *ibcq, int ne, struct ibv_wc *wc) +{ + return poll_cq(ibcq, ne, wc, 0); +} + +int mlx5_poll_cq_v1(struct ibv_cq *ibcq, int ne, struct ibv_wc *wc) +{ + return poll_cq(ibcq, ne, wc, 1); +} + int mlx5_arm_cq(struct ibv_cq *ibvcq, int solicited) { struct mlx5_cq *cq = to_mcq(ibvcq); @@ -622,7 +762,7 @@ void __mlx5_cq_clean(struct mlx5_cq *cq, uint32_t rsn, struct mlx5_srq *srq) cqe = get_cqe(cq, prod_index & cq->ibv_cq.cqe); cqe64 = (cq->cqe_sz == 64) ? cqe : cqe + 64; if (is_equal_rsn(cqe64, rsn)) { - if (srq && (ntohl(cqe64->srqn) & 0xffffff)) + if (srq && (ntohl(cqe64->srqn_uidx) & 0xffffff)) mlx5_free_srq_wqe(srq, ntohs(cqe64->wqe_counter)); ++nfreed; } else if (nfreed) { diff --git a/src/mlx5.h b/src/mlx5.h index 2691031..8190062 100644 --- a/src/mlx5.h +++ b/src/mlx5.h @@ -544,6 +544,16 @@ static inline int max_int(int a, int b) return a > b ? a : b; } +static inline struct mlx5_qp *rsc_to_mqp(struct mlx5_resource *rsc) +{ + return (struct mlx5_qp *)rsc; +} + +static inline struct mlx5_srq *rsc_to_msrq(struct mlx5_resource *rsc) +{ + return (struct mlx5_srq *)rsc; +} + int mlx5_alloc_buf(struct mlx5_buf *buf, size_t size, int page_size); void mlx5_free_buf(struct mlx5_buf *buf); int mlx5_alloc_buf_contig(struct mlx5_context *mctx, struct mlx5_buf *buf, @@ -590,6 +600,7 @@ int mlx5_free_cq_buf(struct mlx5_context *ctx, struct mlx5_buf *buf); int mlx5_resize_cq(struct ibv_cq *cq, int cqe); int mlx5_destroy_cq(struct ibv_cq *cq); int mlx5_poll_cq(struct ibv_cq *cq, int ne, struct ibv_wc *wc); +int mlx5_poll_cq_v1(struct ibv_cq *cq, int ne, struct ibv_wc *wc); int mlx5_arm_cq(struct ibv_cq *cq, int solicited); void mlx5_cq_event(struct ibv_cq *cq); void __mlx5_cq_clean(struct mlx5_cq *cq, uint32_t qpn, struct mlx5_srq *srq);