From patchwork Mon Mar 18 12:24:17 2019 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Yishai Hadas X-Patchwork-Id: 10857539 Return-Path: Received: from mail.wl.linuxfoundation.org (pdx-wl-mail.web.codeaurora.org [172.30.200.125]) by pdx-korg-patchwork-2.web.codeaurora.org (Postfix) with ESMTP id 261D16C2 for ; Mon, 18 Mar 2019 12:25:42 +0000 (UTC) Received: from mail.wl.linuxfoundation.org (localhost [127.0.0.1]) by mail.wl.linuxfoundation.org (Postfix) with ESMTP id 05E8829222 for ; Mon, 18 Mar 2019 12:25:42 +0000 (UTC) Received: by mail.wl.linuxfoundation.org (Postfix, from userid 486) id EE274292D1; Mon, 18 Mar 2019 12:25:41 +0000 (UTC) X-Spam-Checker-Version: SpamAssassin 3.3.1 (2010-03-16) on pdx-wl-mail.web.codeaurora.org X-Spam-Level: X-Spam-Status: No, score=-7.9 required=2.0 tests=BAYES_00,MAILING_LIST_MULTI, RCVD_IN_DNSWL_HI,UNPARSEABLE_RELAY autolearn=ham version=3.3.1 Received: from vger.kernel.org (vger.kernel.org [209.132.180.67]) by mail.wl.linuxfoundation.org (Postfix) with ESMTP id BCB2E29222 for ; Mon, 18 Mar 2019 12:25:40 +0000 (UTC) Received: (majordomo@vger.kernel.org) by vger.kernel.org via listexpand id S1726969AbfCRMZj (ORCPT ); Mon, 18 Mar 2019 08:25:39 -0400 Received: from mail-il-dmz.mellanox.com ([193.47.165.129]:53318 "EHLO mellanox.co.il" rhost-flags-OK-OK-OK-FAIL) by vger.kernel.org with ESMTP id S1727220AbfCRMZj (ORCPT ); Mon, 18 Mar 2019 08:25:39 -0400 Received: from Internal Mail-Server by MTLPINE1 (envelope-from yishaih@mellanox.com) with ESMTPS (AES256-SHA encrypted); 18 Mar 2019 14:25:30 +0200 Received: from vnc17.mtl.labs.mlnx (vnc17.mtl.labs.mlnx [10.7.2.17]) by labmailer.mlnx (8.13.8/8.13.8) with ESMTP id x2ICPUIE019560; Mon, 18 Mar 2019 14:25:30 +0200 Received: from vnc17.mtl.labs.mlnx (vnc17.mtl.labs.mlnx [127.0.0.1]) by vnc17.mtl.labs.mlnx (8.13.8/8.13.8) with ESMTP id x2ICPUkM004908; Mon, 18 Mar 2019 14:25:30 +0200 Received: (from yishaih@localhost) by vnc17.mtl.labs.mlnx (8.13.8/8.13.8/Submit) id x2ICPU2P004907; Mon, 18 Mar 2019 14:25:30 +0200 From: Yishai Hadas To: linux-rdma@vger.kernel.org Cc: yishaih@mellanox.com, guyle@mellanox.com, Alexr@mellanox.com, jgg@mellanox.com, majd@mellanox.com Subject: [PATCH rdma-core 4/6] mlx5: Support raw packet QPT over new post send API Date: Mon, 18 Mar 2019 14:24:17 +0200 Message-Id: <1552911859-4073-5-git-send-email-yishaih@mellanox.com> X-Mailer: git-send-email 1.8.2.3 In-Reply-To: <1552911859-4073-1-git-send-email-yishaih@mellanox.com> References: <1552911859-4073-1-git-send-email-yishaih@mellanox.com> Sender: linux-rdma-owner@vger.kernel.org Precedence: bulk List-ID: X-Mailing-List: linux-rdma@vger.kernel.org X-Virus-Scanned: ClamAV using ClamSMTP From: Guy Levi As a complementary part for mlx5 support of new post send API, now raw packet QP transport with all its operations is supported. This completes the full support for all QP types and operations. Signed-off-by: Guy Levi Signed-off-by: Yishai Hadas --- providers/mlx5/mlx5.h | 1 + providers/mlx5/qp.c | 367 +++++++++++++++++++++++++++++++++++++++++++++----- 2 files changed, 338 insertions(+), 30 deletions(-) diff --git a/providers/mlx5/mlx5.h b/providers/mlx5/mlx5.h index b31619c..3a22fde 100644 --- a/providers/mlx5/mlx5.h +++ b/providers/mlx5/mlx5.h @@ -520,6 +520,7 @@ struct mlx5_qp { int nreq; uint32_t cur_size; uint32_t cur_post_rb; + void *cur_eth; void *cur_data; struct mlx5_wqe_ctrl_seg *cur_ctrl; /* End of new post send API specific fields */ diff --git a/providers/mlx5/qp.c b/providers/mlx5/qp.c index 8cff584..f3bce40 100644 --- a/providers/mlx5/qp.c +++ b/providers/mlx5/qp.c @@ -411,36 +411,59 @@ void *mlx5_get_atomic_laddr(struct mlx5_qp *qp, uint16_t idx, int *byte_count) } static inline int copy_eth_inline_headers(struct ibv_qp *ibqp, - struct ibv_send_wr *wr, + const void *list, + size_t nelem, struct mlx5_wqe_eth_seg *eseg, - struct mlx5_sg_copy_ptr *sg_copy_ptr) + struct mlx5_sg_copy_ptr *sg_copy_ptr, + bool is_sge) + ALWAYS_INLINE; +static inline int copy_eth_inline_headers(struct ibv_qp *ibqp, + const void *list, + size_t nelem, + struct mlx5_wqe_eth_seg *eseg, + struct mlx5_sg_copy_ptr *sg_copy_ptr, + bool is_sge) { uint32_t inl_hdr_size = to_mctx(ibqp->context)->eth_min_inline_size; - int inl_hdr_copy_size = 0; + size_t inl_hdr_copy_size = 0; int j = 0; FILE *fp = to_mctx(ibqp->context)->dbg_fp; + size_t length; + void *addr; - if (unlikely(wr->num_sge < 1)) { - mlx5_dbg(fp, MLX5_DBG_QP_SEND, "illegal num_sge: %d, minimum is 1\n", - wr->num_sge); + if (unlikely(nelem < 1)) { + mlx5_dbg(fp, MLX5_DBG_QP_SEND, + "illegal num_sge: %zu, minimum is 1\n", nelem); return EINVAL; } - if (likely(wr->sg_list[0].length >= MLX5_ETH_L2_INLINE_HEADER_SIZE)) { + if (is_sge) { + addr = (void *)(uintptr_t)((struct ibv_sge *)list)[0].addr; + length = (size_t)((struct ibv_sge *)list)[0].length; + } else { + addr = ((struct ibv_data_buf *)list)[0].addr; + length = ((struct ibv_data_buf *)list)[0].length; + } + + if (likely(length >= MLX5_ETH_L2_INLINE_HEADER_SIZE)) { inl_hdr_copy_size = inl_hdr_size; - memcpy(eseg->inline_hdr_start, - (void *)(uintptr_t)wr->sg_list[0].addr, - inl_hdr_copy_size); + memcpy(eseg->inline_hdr_start, addr, inl_hdr_copy_size); } else { uint32_t inl_hdr_size_left = inl_hdr_size; - for (j = 0; j < wr->num_sge && inl_hdr_size_left > 0; ++j) { - inl_hdr_copy_size = min(wr->sg_list[j].length, - inl_hdr_size_left); + for (j = 0; j < nelem && inl_hdr_size_left > 0; ++j) { + if (is_sge) { + addr = (void *)(uintptr_t)((struct ibv_sge *)list)[j].addr; + length = (size_t)((struct ibv_sge *)list)[j].length; + } else { + addr = ((struct ibv_data_buf *)list)[j].addr; + length = ((struct ibv_data_buf *)list)[j].length; + } + + inl_hdr_copy_size = min_t(size_t, length, inl_hdr_size_left); memcpy(eseg->inline_hdr_start + (MLX5_ETH_L2_INLINE_HEADER_SIZE - inl_hdr_size_left), - (void *)(uintptr_t)wr->sg_list[j].addr, - inl_hdr_copy_size); + addr, inl_hdr_copy_size); inl_hdr_size_left -= inl_hdr_copy_size; } if (unlikely(inl_hdr_size_left)) { @@ -456,7 +479,7 @@ static inline int copy_eth_inline_headers(struct ibv_qp *ibqp, /* If we copied all the sge into the inline-headers, then we need to * start copying from the next sge into the data-segment. */ - if (unlikely(wr->sg_list[j].length == inl_hdr_copy_size)) { + if (unlikely(length == inl_hdr_copy_size)) { ++j; inl_hdr_copy_size = 0; } @@ -619,17 +642,17 @@ static inline int set_bind_wr(struct mlx5_qp *qp, enum ibv_mw_type type, /* Copy tso header to eth segment with considering padding and WQE * wrap around in WQ buffer. */ -static inline int set_tso_eth_seg(void **seg, struct ibv_send_wr *wr, - void *qend, struct mlx5_qp *qp, int *size) +static inline int set_tso_eth_seg(void **seg, void *hdr, uint16_t hdr_sz, + uint16_t mss, + struct mlx5_qp *qp, int *size) { struct mlx5_wqe_eth_seg *eseg = *seg; int size_of_inl_hdr_start = sizeof(eseg->inline_hdr_start); uint64_t left, left_len, copy_sz; - void *pdata = wr->tso.hdr; FILE *fp = to_mctx(qp->ibv_qp->context)->dbg_fp; - if (unlikely(wr->tso.hdr_sz < MLX5_ETH_L2_MIN_HEADER_SIZE || - wr->tso.hdr_sz > qp->max_tso_header)) { + if (unlikely(hdr_sz < MLX5_ETH_L2_MIN_HEADER_SIZE || + hdr_sz > qp->max_tso_header)) { mlx5_dbg(fp, MLX5_DBG_QP_SEND, "TSO header size should be at least %d and at most %d\n", MLX5_ETH_L2_MIN_HEADER_SIZE, @@ -637,18 +660,18 @@ static inline int set_tso_eth_seg(void **seg, struct ibv_send_wr *wr, return EINVAL; } - left = wr->tso.hdr_sz; - eseg->mss = htobe16(wr->tso.mss); - eseg->inline_hdr_sz = htobe16(wr->tso.hdr_sz); + left = hdr_sz; + eseg->mss = htobe16(mss); + eseg->inline_hdr_sz = htobe16(hdr_sz); /* Check if there is space till the end of queue, if yes, * copy all in one shot, otherwise copy till the end of queue, * rollback and then copy the left */ - left_len = qend - (void *)eseg->inline_hdr_start; + left_len = qp->sq.qend - (void *)eseg->inline_hdr_start; copy_sz = min(left_len, left); - memcpy(eseg->inline_hdr_start, pdata, copy_sz); + memcpy(eseg->inline_hdr_start, hdr, copy_sz); /* The -1 is because there are already 16 bytes included in * eseg->inline_hdr[16] @@ -660,8 +683,8 @@ static inline int set_tso_eth_seg(void **seg, struct ibv_send_wr *wr, if (unlikely(copy_sz < left)) { *seg = mlx5_get_send_wqe(qp, 0); left -= copy_sz; - pdata += copy_sz; - memcpy(*seg, pdata, left); + hdr += copy_sz; + memcpy(*seg, hdr, left); *seg += align(left, 16); *size += align(left, 16) / 16; } @@ -1003,7 +1026,9 @@ static inline int _mlx5_post_send(struct ibv_qp *ibqp, struct ibv_send_wr *wr, if (wr->opcode == IBV_WR_TSO) { max_tso = qp->max_tso; - err = set_tso_eth_seg(&seg, wr, qend, qp, &size); + err = set_tso_eth_seg(&seg, wr->tso.hdr, + wr->tso.hdr_sz, + wr->tso.mss, qp, &size); if (unlikely(err)) { *bad_wr = wr; goto out; @@ -1021,7 +1046,9 @@ static inline int _mlx5_post_send(struct ibv_qp *ibqp, struct ibv_send_wr *wr, uint32_t inl_hdr_size = to_mctx(ibqp->context)->eth_min_inline_size; - err = copy_eth_inline_headers(ibqp, wr, seg, &sg_copy_ptr); + err = copy_eth_inline_headers(ibqp, wr->sg_list, + wr->num_sge, seg, + &sg_copy_ptr, 1); if (unlikely(err)) { *bad_wr = wr; mlx5_dbg(fp, MLX5_DBG_QP_SEND, @@ -1292,6 +1319,45 @@ static void mlx5_send_wr_send_other(struct ibv_qp_ex *ibqp) _mlx5_send_wr_send(ibqp, IBV_WR_SEND); } +static void mlx5_send_wr_send_eth(struct ibv_qp_ex *ibqp) +{ + uint32_t inl_hdr_size = + to_mctx(((struct ibv_qp *)ibqp)->context)->eth_min_inline_size; + struct mlx5_qp *mqp = to_mqp((struct ibv_qp *)ibqp); + struct mlx5_wqe_eth_seg *eseg; + size_t eseg_sz; + + _common_wqe_init(ibqp, IBV_WR_SEND); + + eseg = (void *)mqp->cur_ctrl + sizeof(struct mlx5_wqe_ctrl_seg); + memset(eseg, 0, sizeof(struct mlx5_wqe_eth_seg)); + if (inl_hdr_size) + mqp->cur_eth = eseg; + + if (ibqp->wr_flags & IBV_SEND_IP_CSUM) { + if (unlikely(!(mqp->qp_cap_cache & + MLX5_CSUM_SUPPORT_RAW_OVER_ETH))) { + if (!mqp->err) + mqp->err = EINVAL; + + return; + } + + eseg->cs_flags |= MLX5_ETH_WQE_L3_CSUM | MLX5_ETH_WQE_L4_CSUM; + } + + /* The eth segment size depends on the device's min inline + * header requirement which can be 0 or 18. The basic eth segment + * always includes room for first 2 inline header bytes (even if + * copy size is 0) so the additional seg size is adjusted accordingly. + */ + eseg_sz = (offsetof(struct mlx5_wqe_eth_seg, inline_hdr) + + inl_hdr_size) & ~0xf; + mqp->cur_data = (void *)eseg + eseg_sz; + mqp->cur_size = (sizeof(struct mlx5_wqe_ctrl_seg) + eseg_sz) >> 4; + mqp->nreq++; +} + static void mlx5_send_wr_send_imm(struct ibv_qp_ex *ibqp, __be32 imm_data) { struct mlx5_qp *mqp = to_mqp((struct ibv_qp *)ibqp); @@ -1311,6 +1377,48 @@ static void mlx5_send_wr_send_inv(struct ibv_qp_ex *ibqp, mqp->cur_ctrl->imm = htobe32(invalidate_rkey); } +static void mlx5_send_wr_send_tso(struct ibv_qp_ex *ibqp, void *hdr, + uint16_t hdr_sz, uint16_t mss) +{ + struct mlx5_qp *mqp = to_mqp((struct ibv_qp *)ibqp); + struct mlx5_wqe_eth_seg *eseg; + int size = 0; + int err; + + _common_wqe_init(ibqp, IBV_WR_TSO); + + eseg = (void *)mqp->cur_ctrl + sizeof(struct mlx5_wqe_ctrl_seg); + memset(eseg, 0, sizeof(struct mlx5_wqe_eth_seg)); + + if (ibqp->wr_flags & IBV_SEND_IP_CSUM) { + if (unlikely(!(mqp->qp_cap_cache & MLX5_CSUM_SUPPORT_RAW_OVER_ETH))) { + if (!mqp->err) + mqp->err = EINVAL; + + return; + } + + eseg->cs_flags |= MLX5_ETH_WQE_L3_CSUM | MLX5_ETH_WQE_L4_CSUM; + } + + err = set_tso_eth_seg((void *)&eseg, hdr, hdr_sz, mss, mqp, &size); + if (unlikely(err)) { + if (!mqp->err) + mqp->err = err; + + return; + } + + /* eseg and cur_size was updated with hdr size inside set_tso_eth_seg */ + mqp->cur_data = (void *)eseg + sizeof(struct mlx5_wqe_eth_seg); + mqp->cur_size = size + + ((sizeof(struct mlx5_wqe_ctrl_seg) + + sizeof(struct mlx5_wqe_eth_seg)) >> 4); + + mqp->cur_eth = NULL; + mqp->nreq++; +} + static inline void _mlx5_send_wr_rdma(struct ibv_qp_ex *ibqp, uint32_t rkey, uint64_t remote_addr, @@ -1513,6 +1621,36 @@ mlx5_send_wr_set_sge_ud_xrc(struct ibv_qp_ex *ibqp, uint32_t lkey, mqp->cur_setters_cnt++; } +static void +mlx5_send_wr_set_sge_eth(struct ibv_qp_ex *ibqp, uint32_t lkey, + uint64_t addr, uint32_t length) +{ + struct mlx5_qp *mqp = to_mqp((struct ibv_qp *)ibqp); + struct mlx5_wqe_eth_seg *eseg = mqp->cur_eth; + int err; + + if (eseg) { /* Inline-headers was set */ + struct mlx5_sg_copy_ptr sg_copy_ptr = {.index = 0, .offset = 0}; + struct ibv_sge sge = {.addr = addr, .length = length}; + + err = copy_eth_inline_headers((struct ibv_qp *)ibqp, &sge, 1, + eseg, &sg_copy_ptr, 1); + if (unlikely(err)) { + if (!mqp->err) + mqp->err = err; + + return; + } + + addr += sg_copy_ptr.offset; + length -= sg_copy_ptr.offset; + } + + _mlx5_send_wr_set_sge(mqp, lkey, addr, length); + + _common_wqe_finilize(mqp); +} + static inline void _mlx5_send_wr_set_sge_list(struct mlx5_qp *mqp, size_t num_sge, const struct ibv_sge *sg_list) @@ -1571,6 +1709,61 @@ mlx5_send_wr_set_sge_list_ud_xrc(struct ibv_qp_ex *ibqp, size_t num_sge, mqp->cur_setters_cnt++; } +static void +mlx5_send_wr_set_sge_list_eth(struct ibv_qp_ex *ibqp, size_t num_sge, + const struct ibv_sge *sg_list) +{ + struct mlx5_sg_copy_ptr sg_copy_ptr = {.index = 0, .offset = 0}; + struct mlx5_qp *mqp = to_mqp((struct ibv_qp *)ibqp); + struct mlx5_wqe_data_seg *dseg = mqp->cur_data; + struct mlx5_wqe_eth_seg *eseg = mqp->cur_eth; + size_t i; + + if (unlikely(num_sge > mqp->sq.max_gs)) { + FILE *fp = to_mctx(mqp->ibv_qp->context)->dbg_fp; + + mlx5_dbg(fp, MLX5_DBG_QP_SEND, "Num SGEs %zu exceeds the maximum (%d)\n", + num_sge, mqp->sq.max_gs); + + if (!mqp->err) + mqp->err = ENOMEM; + + return; + } + + if (eseg) { /* Inline-headers was set */ + int err; + + err = copy_eth_inline_headers((struct ibv_qp *)ibqp, sg_list, + num_sge, eseg, &sg_copy_ptr, 1); + if (unlikely(err)) { + if (!mqp->err) + mqp->err = err; + + return; + } + } + + for (i = sg_copy_ptr.index; i < num_sge; i++) { + uint32_t length = sg_list[i].length - sg_copy_ptr.offset; + + if (unlikely(!length)) + continue; + + if (unlikely(dseg == mqp->sq.qend)) + dseg = mlx5_get_send_wqe(mqp, 0); + + dseg->addr = htobe64(sg_list[i].addr + sg_copy_ptr.offset); + dseg->byte_count = htobe32(length); + dseg->lkey = htobe32(sg_list[i].lkey); + dseg++; + mqp->cur_size += (sizeof(*dseg) / 16); + sg_copy_ptr.offset = 0; + } + + _common_wqe_finilize(mqp); +} + static inline void memcpy_to_wqe(struct mlx5_qp *mqp, void *dest, void *src, size_t n) { @@ -1653,6 +1846,35 @@ mlx5_send_wr_set_inline_data_ud_xrc(struct ibv_qp_ex *ibqp, void *addr, mqp->cur_setters_cnt++; } +static void +mlx5_send_wr_set_inline_data_eth(struct ibv_qp_ex *ibqp, void *addr, + size_t length) +{ + struct mlx5_qp *mqp = to_mqp((struct ibv_qp *)ibqp); + struct mlx5_wqe_eth_seg *eseg = mqp->cur_eth; + + if (eseg) { /* Inline-headers was set */ + struct mlx5_sg_copy_ptr sg_copy_ptr = {.index = 0, .offset = 0}; + struct ibv_data_buf buf = {.addr = addr, .length = length}; + int err; + + err = copy_eth_inline_headers((struct ibv_qp *)ibqp, &buf, 1, + eseg, &sg_copy_ptr, 0); + if (unlikely(err)) { + if (!mqp->err) + mqp->err = err; + + return; + } + + addr += sg_copy_ptr.offset; + length -= sg_copy_ptr.offset; + } + + _mlx5_send_wr_set_inline_data(mqp, addr, length); + _common_wqe_finilize(mqp); +} + static inline void _mlx5_send_wr_set_inline_data_list(struct mlx5_qp *mqp, size_t num_buf, @@ -1720,6 +1942,66 @@ mlx5_send_wr_set_inline_data_list_ud_xrc(struct ibv_qp_ex *ibqp, } static void +mlx5_send_wr_set_inline_data_list_eth(struct ibv_qp_ex *ibqp, + size_t num_buf, + const struct ibv_data_buf *buf_list) +{ + struct mlx5_sg_copy_ptr sg_copy_ptr = {.index = 0, .offset = 0}; + struct mlx5_qp *mqp = to_mqp((struct ibv_qp *)ibqp); + struct mlx5_wqe_inline_seg *dseg = mqp->cur_data; + struct mlx5_wqe_eth_seg *eseg = mqp->cur_eth; + void *wqe = (void *)dseg + sizeof(*dseg); + size_t inl_size = 0; + size_t i; + + if (eseg) { /* Inline-headers was set */ + int err; + + err = copy_eth_inline_headers((struct ibv_qp *)ibqp, buf_list, + num_buf, eseg, &sg_copy_ptr, 0); + if (unlikely(err)) { + if (!mqp->err) + mqp->err = err; + + return; + } + } + + for (i = sg_copy_ptr.index; i < num_buf; i++) { + size_t length = buf_list[i].length - sg_copy_ptr.offset; + + inl_size += length; + + if (unlikely(inl_size > mqp->max_inline_data)) { + FILE *fp = to_mctx(mqp->ibv_qp->context)->dbg_fp; + + mlx5_dbg(fp, MLX5_DBG_QP_SEND, + "Inline data %zu exceeds the maximum (%d)\n", + inl_size, mqp->max_inline_data); + + if (!mqp->err) + mqp->err = EINVAL; + + return; + } + + memcpy_to_wqe_and_update(mqp, &wqe, + buf_list[i].addr + sg_copy_ptr.offset, + length); + + sg_copy_ptr.offset = 0; + } + + if (likely(inl_size)) { + dseg->byte_count = htobe32(inl_size | MLX5_INLINE_SEG); + mqp->cur_size += DIV_ROUND_UP(inl_size + sizeof(*dseg), 16); + } + + mqp->inl_wqe = 1; /* Encourage a BlueFlame usage */ + _common_wqe_finilize(mqp); +} + +static void mlx5_send_wr_set_ud_addr(struct ibv_qp_ex *ibqp, struct ibv_ah *ah, uint32_t remote_qpn, uint32_t remote_qkey) { @@ -1776,6 +2058,9 @@ enum { IBV_QP_EX_WITH_RDMA_WRITE_WITH_IMM | IBV_QP_EX_WITH_LOCAL_INV | IBV_QP_EX_WITH_BIND_MW, + MLX5_SUPPORTED_SEND_OPS_FLAGS_RAW_PACKET = + IBV_QP_EX_WITH_SEND | + IBV_QP_EX_WITH_TSO, }; static void fill_wr_builders_rc_xrc(struct ibv_qp_ex *ibqp) @@ -1809,6 +2094,12 @@ static void fill_wr_builders_ud(struct ibv_qp_ex *ibqp) ibqp->wr_send_imm = mlx5_send_wr_send_imm; } +static void fill_wr_builders_eth(struct ibv_qp_ex *ibqp) +{ + ibqp->wr_send = mlx5_send_wr_send_eth; + ibqp->wr_send_tso = mlx5_send_wr_send_tso; +} + static void fill_wr_setters_rc_uc(struct ibv_qp_ex *ibqp) { ibqp->wr_set_sge = mlx5_send_wr_set_sge_rc_uc; @@ -1825,6 +2116,14 @@ static void fill_wr_setters_ud_xrc(struct ibv_qp_ex *ibqp) ibqp->wr_set_inline_data_list = mlx5_send_wr_set_inline_data_list_ud_xrc; } +static void fill_wr_setters_eth(struct ibv_qp_ex *ibqp) +{ + ibqp->wr_set_sge = mlx5_send_wr_set_sge_eth; + ibqp->wr_set_sge_list = mlx5_send_wr_set_sge_list_eth; + ibqp->wr_set_inline_data = mlx5_send_wr_set_inline_data_eth; + ibqp->wr_set_inline_data_list = mlx5_send_wr_set_inline_data_list_eth; +} + int mlx5_qp_fill_wr_pfns(struct mlx5_qp *mqp, const struct ibv_qp_init_attr_ex *attr) { @@ -1879,6 +2178,14 @@ int mlx5_qp_fill_wr_pfns(struct mlx5_qp *mqp, ibqp->wr_set_ud_addr = mlx5_send_wr_set_ud_addr; break; + case IBV_QPT_RAW_PACKET: + if (ops & ~MLX5_SUPPORTED_SEND_OPS_FLAGS_RAW_PACKET) + return EOPNOTSUPP; + + fill_wr_builders_eth(ibqp); + fill_wr_setters_eth(ibqp); + break; + default: return EOPNOTSUPP; }