[rdma-core,3/5] RDMA-CORE/erdma: Add the main module of the provider

Commit Message

Cheng Xu Dec. 24, 2021, 6:55 a.m. UTC

Add the definitions of erdma provier driver.

Signed-off-by: Cheng Xu <[e-mail]>
---
 providers/erdma/erdma.c | 133 ++++++++++++++++++++++++++++++++++++++++
 providers/erdma/erdma.h |  60 ++++++++++++++++++
 2 files changed, 193 insertions(+)
 create mode 100644 providers/erdma/erdma.c
 create mode 100644 providers/erdma/erdma.h

Patch

diff --git a/providers/erdma/erdma.c b/providers/erdma/erdma.c
new file mode 100644
index 00000000..59f5ecb6
--- /dev/null
+++ b/providers/erdma/erdma.c
@@ -0,0 +1,133 @@ 
+// SPDX-License-Identifier: GPL-2.0 or OpenIB.org BSD (MIT) See COPYING file
+
+// Authors: Cheng Xu <[e-mail]>
+// Copyright (c) 2020-2021, Alibaba Group.
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <sys/mman.h>
+#include <sys/types.h>
+#include <unistd.h>
+#include <util/mmio.h>
+#include <util/udma_barrier.h>
+#include <util/util.h>
+
+#include "erdma.h"
+#include "erdma_abi.h"
+#include "erdma_hw.h"
+#include "erdma_verbs.h"
+
+static const struct verbs_context_ops erdma_context_ops = {
+	.alloc_pd = erdma_alloc_pd,
+	.create_cq = erdma_create_cq,
+	.create_qp = erdma_create_qp,
+	.dealloc_pd = erdma_free_pd,
+	.dereg_mr = erdma_dereg_mr,
+	.destroy_cq = erdma_destroy_cq,
+	.destroy_qp = erdma_destroy_qp,
+	.free_context = erdma_free_context,
+	.modify_qp = erdma_modify_qp,
+	.cq_event = erdma_cq_event,
+	.poll_cq = erdma_poll_cq,
+	.post_recv = erdma_post_recv,
+	.post_send = erdma_post_send,
+	.query_device_ex = erdma_query_device,
+	.query_port = erdma_query_port,
+	.query_qp = erdma_query_qp,
+	.reg_mr = erdma_reg_mr,
+	.req_notify_cq = erdma_notify_cq,
+};
+
+static struct verbs_context *erdma_alloc_context(struct ibv_device *device,
+						 int cmd_fd, void *private_data)
+{
+	struct erdma_context *ctx;
+	struct ibv_get_context cmd = {};
+	struct erdma_cmd_alloc_context_resp resp = {};
+	int i;
+
+	ctx = verbs_init_and_alloc_context(device, cmd_fd, ctx, ibv_ctx, RDMA_DRIVER_ERDMA);
+	if (!ctx)
+		return NULL;
+
+	pthread_mutex_init(&ctx->qp_table_mutex, NULL);
+	for (i = 0; i < ERDMA_QP_TABLE_SIZE; ++i)
+		ctx->qp_table[i].refcnt = 0;
+
+	if (ibv_cmd_get_context(&ctx->ibv_ctx, &cmd, sizeof(cmd), &resp.ibv_resp, sizeof(resp)))
+		goto fail;
+
+	verbs_set_ops(&ctx->ibv_ctx, &erdma_context_ops);
+	ctx->dev_id = resp.dev_id;
+
+	ctx->sdb_type = resp.sdb_type;
+	ctx->sdb_offset = resp.sdb_offset;
+
+	ctx->sdb = mmap(NULL, ERDMA_PAGE_SIZE, PROT_WRITE, MAP_SHARED, cmd_fd, resp.sdb);
+	if (!ctx->sdb)
+		goto fail;
+
+	ctx->rdb = mmap(NULL, ERDMA_PAGE_SIZE, PROT_WRITE, MAP_SHARED, cmd_fd, resp.rdb);
+	if (!ctx->rdb)
+		goto fail;
+
+	ctx->cdb = mmap(NULL, ERDMA_PAGE_SIZE, PROT_WRITE, MAP_SHARED, cmd_fd, resp.cdb);
+	if (!ctx->cdb)
+		goto fail;
+
+	ctx->page_size = ERDMA_PAGE_SIZE;
+	ctx->dbrecord_pages = NULL;
+	pthread_mutex_init(&ctx->dbrecord_pages_mutex, NULL);
+
+	return &ctx->ibv_ctx;
+
+fail:
+	if (ctx->sdb)
+		munmap(ctx->sdb, ERDMA_PAGE_SIZE);
+	if (ctx->rdb)
+		munmap(ctx->rdb, ERDMA_PAGE_SIZE);
+	if (ctx->cdb)
+		munmap(ctx->cdb, ERDMA_PAGE_SIZE);
+
+	verbs_uninit_context(&ctx->ibv_ctx);
+	free(ctx);
+	return NULL;
+}
+
+static struct verbs_device *erdma_device_alloc(struct verbs_sysfs_dev *sysfs_dev)
+{
+	struct erdma_device *dev;
+
+	dev = calloc(1, sizeof(*dev));
+	if (!dev)
+		return NULL;
+
+	return &dev->ibv_dev;
+}
+
+static void erdma_device_free(struct verbs_device *vdev)
+{
+	struct erdma_device *dev =
+		container_of(vdev, struct erdma_device, ibv_dev);
+
+	free(dev);
+}
+
+static const struct verbs_match_ent match_table[] = {
+	VERBS_DRIVER_ID(RDMA_DRIVER_ERDMA),
+	VERBS_PCI_MATCH(PCI_VENDOR_ID_ALIBABA, 0x107f, NULL),
+	VERBS_PCI_MATCH(PCI_VENDOR_ID_ALIBABA, 0x5007, NULL),
+	{},
+};
+
+static const struct verbs_device_ops erdma_dev_ops = {
+	.name = "erdma",
+	.match_min_abi_version = 0,
+	.match_max_abi_version = ERDMA_ABI_VERSION,
+	.match_table = match_table,
+	.alloc_device = erdma_device_alloc,
+	.uninit_device = erdma_device_free,
+	.alloc_context = erdma_alloc_context,
+};
+
+PROVIDER_DRIVER(erdma, erdma_dev_ops);
diff --git a/providers/erdma/erdma.h b/providers/erdma/erdma.h
new file mode 100644
index 00000000..a3d9f3b4
--- /dev/null
+++ b/providers/erdma/erdma.h
@@ -0,0 +1,60 @@ 
+/* SPDX-License-Identifier: GPL-2.0 or OpenIB.org BSD (MIT) See COPYING file */
+/*
+ * Authors: Cheng Xu <[e-mail]>
+ * Copyright (c) 2020-2021, Alibaba Group.
+ */
+
+#ifndef __ERDMA_H__
+#define __ERDMA_H__
+
+#include <pthread.h>
+#include <inttypes.h>
+#include <stddef.h>
+
+#include <infiniband/driver.h>
+#include <infiniband/kern-abi.h>
+
+#ifndef PCI_VENDOR_ID_ALIBABA
+#define PCI_VENDOR_ID_ALIBABA 0x1ded
+#endif
+
+#define ERDMA_PAGE_SIZE 4096
+#define ERDMA_PAGE_SHIFT 12
+#define ERDMA_SIZE_TO_NPAGE(size) (((size) + ERDMA_PAGE_SIZE - 1) >> ERDMA_PAGE_SHIFT)
+
+struct erdma_device {
+	struct verbs_device ibv_dev;
+};
+
+#define ERDMA_QP_TABLE_SIZE   4096
+#define ERDMA_QP_TABLE_SHIFT  12
+#define ERDMA_QP_TABLE_MASK   0xFFF
+
+struct erdma_context {
+	struct verbs_context ibv_ctx;
+	uint32_t dev_id;
+
+	struct {
+		struct erdma_qp **table;
+		int refcnt;
+	} qp_table[ERDMA_QP_TABLE_SIZE];
+	pthread_mutex_t qp_table_mutex;
+
+	uint8_t sdb_type;
+	uint32_t sdb_offset;
+
+	void *sdb;
+	void *rdb;
+	void *cdb;
+
+	int page_size;
+	pthread_mutex_t dbrecord_pages_mutex;
+	struct erdma_dbrecord_page *dbrecord_pages;
+};
+
+static inline struct erdma_context *to_ectx(struct ibv_context *base)
+{
+	return container_of(base, struct erdma_context, ibv_ctx.context);
+}
+
+#endif