From patchwork Tue Dec 4 22:14:43 2018 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: "Arumugam, Kamenee" X-Patchwork-Id: 10712633 X-Patchwork-Delegate: jgg@ziepe.ca Return-Path: Received: from mail.wl.linuxfoundation.org (pdx-wl-mail.web.codeaurora.org [172.30.200.125]) by pdx-korg-patchwork-2.web.codeaurora.org (Postfix) with ESMTP id E78AA13A4 for ; Tue, 4 Dec 2018 22:14:47 +0000 (UTC) Received: from mail.wl.linuxfoundation.org (localhost [127.0.0.1]) by mail.wl.linuxfoundation.org (Postfix) with ESMTP id 5B1122C80C for ; Tue, 4 Dec 2018 22:14:45 +0000 (UTC) Received: by mail.wl.linuxfoundation.org (Postfix, from userid 486) id 4F7112C852; Tue, 4 Dec 2018 22:14:45 +0000 (UTC) X-Spam-Checker-Version: SpamAssassin 3.3.1 (2010-03-16) on pdx-wl-mail.web.codeaurora.org X-Spam-Level: X-Spam-Status: No, score=-7.9 required=2.0 tests=BAYES_00,MAILING_LIST_MULTI, RCVD_IN_DNSWL_HI autolearn=ham version=3.3.1 Received: from vger.kernel.org (vger.kernel.org [209.132.180.67]) by mail.wl.linuxfoundation.org (Postfix) with ESMTP id A1A9D2C80C for ; Tue, 4 Dec 2018 22:14:44 +0000 (UTC) Received: (majordomo@vger.kernel.org) by vger.kernel.org via listexpand id S1725919AbeLDWOo (ORCPT ); Tue, 4 Dec 2018 17:14:44 -0500 Received: from mga17.intel.com ([192.55.52.151]:57918 "EHLO mga17.intel.com" rhost-flags-OK-OK-OK-OK) by vger.kernel.org with ESMTP id S1725906AbeLDWOo (ORCPT ); Tue, 4 Dec 2018 17:14:44 -0500 X-Amp-Result: SKIPPED(no attachment in message) X-Amp-File-Uploaded: False Received: from fmsmga004.fm.intel.com ([10.253.24.48]) by fmsmga107.fm.intel.com with ESMTP/TLS/DHE-RSA-AES256-GCM-SHA384; 04 Dec 2018 14:14:43 -0800 X-ExtLoop1: 1 X-IronPort-AV: E=Sophos;i="5.56,315,1539673200"; d="scan'208";a="125093255" Received: from scymds01.sc.intel.com ([10.82.194.37]) by fmsmga004.fm.intel.com with ESMTP; 04 Dec 2018 14:14:43 -0800 Received: from scvm10.sc.intel.com (scvm10.sc.intel.com [10.82.195.27]) by scymds01.sc.intel.com with ESMTP id wB4MEhC8011921 for ; Tue, 4 Dec 2018 14:14:43 -0800 Received: from scvm10.sc.intel.com (localhost [127.0.0.1]) by scvm10.sc.intel.com with ESMTP id wB4MEhJ6011422 for ; Tue, 4 Dec 2018 14:14:43 -0800 Subject: [PATCH] hfi1verbs: Update rvt cq headers in rdma_core hfi1 provider To: linux-rdma@vger.kernel.org From: Kamenee Arumugam Date: Tue, 04 Dec 2018 14:14:43 -0800 Message-ID: <20181204221443.11392.17827.stgit@scvm10.sc.intel.com> User-Agent: StGit/0.16 MIME-Version: 1.0 Sender: linux-rdma-owner@vger.kernel.org Precedence: bulk List-ID: X-Mailing-List: linux-rdma@vger.kernel.org X-Virus-Scanned: ClamAV using ClamSMTP hfi1_wc struct is the same struct as ib_uverbs_wc and hfi1_cq_wc is not in the uapi header. Remove hfi1_wc and hfi1_cq_wc struct from providers. Include rvt-abi.h header that have rvt_cq_wc struct definition into kernel-headers. The member ib_uverbs_wc arrray in rvt_cq_wc is replaced with one in rvt-abi. Therefore,plus one to the number of cq entries. Reviewed-by: Dennis Dalessandro Reviewed-by: Mike Marciniszyn Signed-off-by: Kamenee Arumugam --- kernel-headers/CMakeLists.txt | 1 + kernel-headers/rdma/rvt-abi.h | 31 +++++++++++++++++++++++++++++++ providers/hfi1verbs/hfiverbs.h | 37 +++++-------------------------------- providers/hfi1verbs/verbs.c | 23 +++++++++++++---------- 4 files changed, 50 insertions(+), 42 deletions(-) create mode 100755 kernel-headers/rdma/rvt-abi.h diff --git a/kernel-headers/CMakeLists.txt b/kernel-headers/CMakeLists.txt index 16ba165..631db9c 100644 --- a/kernel-headers/CMakeLists.txt +++ b/kernel-headers/CMakeLists.txt @@ -24,6 +24,7 @@ publish_internal_headers(rdma rdma/rdma_user_ioctl_cmds.h rdma/rdma_user_rxe.h rdma/vmw_pvrdma-abi.h + rdma/rvt-abi.h ) publish_internal_headers(rdma/hfi diff --git a/kernel-headers/rdma/rvt-abi.h b/kernel-headers/rdma/rvt-abi.h new file mode 100755 index 0000000..c30f2fe --- /dev/null +++ b/kernel-headers/rdma/rvt-abi.h @@ -0,0 +1,31 @@ +/* SPDX-License-Identifier: (GPL-2.0 OR BSD-3-Clause) */ + +/* + * This file contains defines, structures, etc. that are used + * to communicate between kernel and user code. + */ + +#ifndef RVT_ABI_USER_H +#define RVT_ABI_USER_H + +#include +#include +#ifndef RDMA_ATOMIC_UAPI +#define RDMA_ATOMIC_UAPI(_type, _name) _type _name +#endif +/* + * This structure is used to contain the head pointer, tail pointer, + * and completion queue entries as a single memory allocation so + * it can be mmap'ed into user space. + */ +struct rvt_cq_wc { + /* index of next entry to fill */ + RDMA_ATOMIC_UAPI(__u32, head); + /* index of next ib_poll_cq() entry */ + RDMA_ATOMIC_UAPI(__u32, tail); + + /* these are actually size ibcq.cqe + 1 */ + struct ib_uverbs_wc uqueue[0]; +}; + +#endif /* RVT_ABI_USER_H */ diff --git a/providers/hfi1verbs/hfiverbs.h b/providers/hfi1verbs/hfiverbs.h index 070a01c..7ce27c8 100644 --- a/providers/hfi1verbs/hfiverbs.h +++ b/providers/hfi1verbs/hfiverbs.h @@ -65,7 +65,8 @@ #include #include - +#define RDMA_ATOMIC_UAPI(_type, _name) _Atomic(_type) _name +#include "rdma/rvt-abi.h" #define PFX "hfi1: " struct hfi1_device { @@ -77,39 +78,11 @@ struct hfi1_context { struct verbs_context ibv_ctx; }; -/* - * This structure needs to have the same size and offsets as - * the kernel's ib_wc structure since it is memory mapped. - */ -struct hfi1_wc { - uint64_t wr_id; - enum ibv_wc_status status; - enum ibv_wc_opcode opcode; - uint32_t vendor_err; - uint32_t byte_len; - uint32_t imm_data; /* in network byte order */ - uint32_t qp_num; - uint32_t src_qp; - enum ibv_wc_flags wc_flags; - uint16_t pkey_index; - uint16_t slid; - uint8_t sl; - uint8_t dlid_path_bits; - uint8_t port_num; -}; - -struct hfi1_cq_wc { - _Atomic(uint32_t) head; - _Atomic(uint32_t) tail; - struct hfi1_wc queue[1]; -}; - struct hfi1_cq { - struct ibv_cq ibv_cq; - struct hfi1_cq_wc *queue; - pthread_spinlock_t lock; + struct ibv_cq ibv_cq; + struct rvt_cq_wc *queue; + pthread_spinlock_t lock; }; - /* * Receive work request queue entry. * The size of the sg_list is determined when the QP is created and stored diff --git a/providers/hfi1verbs/verbs.c b/providers/hfi1verbs/verbs.c index ff001f6..dcf6714 100644 --- a/providers/hfi1verbs/verbs.c +++ b/providers/hfi1verbs/verbs.c @@ -63,10 +63,15 @@ #include #include #include - #include "hfiverbs.h" #include "hfi-abi.h" +static size_t hfi1_cq_size(int cqe) +{ + return sizeof(struct rvt_cq_wc) + + sizeof(struct ib_uverbs_wc) * (cqe + 1); +} + int hfi1_query_device(struct ibv_context *context, struct ibv_device_attr *attr) { @@ -186,7 +191,7 @@ struct ibv_cq *hfi1_create_cq(struct ibv_context *context, int cqe, return NULL; } - size = sizeof(struct hfi1_cq_wc) + sizeof(struct hfi1_wc) * cqe; + size = hfi1_cq_size(cqe); cq->queue = mmap(NULL, size, PROT_READ | PROT_WRITE, MAP_SHARED, context->cmd_fd, resp.offset); if ((void *) cq->queue == MAP_FAILED) { @@ -231,8 +236,7 @@ int hfi1_resize_cq(struct ibv_cq *ibcq, int cqe) memset(&resp, 0, sizeof(resp)); pthread_spin_lock(&cq->lock); /* Save the old size so we can unmmap the queue. */ - size = sizeof(struct hfi1_cq_wc) + - (sizeof(struct hfi1_wc) * cq->ibv_cq.cqe); + size = hfi1_cq_size(cq->ibv_cq.cqe); ret = ibv_cmd_resize_cq(ibcq, cqe, &cmd, sizeof cmd, &resp.ibv_resp, sizeof resp); if (ret) { @@ -240,8 +244,7 @@ int hfi1_resize_cq(struct ibv_cq *ibcq, int cqe) return ret; } (void) munmap(cq->queue, size); - size = sizeof(struct hfi1_cq_wc) + - (sizeof(struct hfi1_wc) * cq->ibv_cq.cqe); + size = hfi1_cq_size(cqe); cq->queue = mmap(NULL, size, PROT_READ | PROT_WRITE, MAP_SHARED, ibcq->context->cmd_fd, resp.offset); ret = errno; @@ -269,8 +272,8 @@ int hfi1_destroy_cq(struct ibv_cq *ibcq) if (ret) return ret; - (void) munmap(cq->queue, sizeof(struct hfi1_cq_wc) + - (sizeof(struct hfi1_wc) * cq->ibv_cq.cqe)); + (void) munmap(cq->queue, hfi1_cq_size(cq->ibv_cq.cqe)); + free(cq); return 0; } @@ -288,7 +291,7 @@ int hfi1_destroy_cq_v1(struct ibv_cq *ibcq) int hfi1_poll_cq(struct ibv_cq *ibcq, int ne, struct ibv_wc *wc) { struct hfi1_cq *cq = to_icq(ibcq); - struct hfi1_cq_wc *q; + struct rvt_cq_wc *q; int npolled; uint32_t tail; @@ -300,7 +303,7 @@ int hfi1_poll_cq(struct ibv_cq *ibcq, int ne, struct ibv_wc *wc) break; /* Make sure entry is read after head index is read. */ atomic_thread_fence(memory_order_acquire); - memcpy(wc, &q->queue[tail], sizeof(*wc)); + memcpy(wc, &q->uqueue[tail], sizeof(*wc)); if (tail == cq->ibv_cq.cqe) tail = 0; else