@@ -320,6 +320,36 @@ static void free_huge_buf(struct mlx5_context *ctx, struct mlx5_buf *buf)
mlx5_spin_unlock(&ctx->hugetlb_lock);
}
+void mlx5_free_buf_extern(struct mlx5_context *ctx, struct mlx5_buf *buf)
+{
+ ibv_dofork_range(buf->buf, buf->length);
+ ctx->extern_alloc.free(buf->buf, ctx->extern_alloc.data);
+}
+
+int mlx5_alloc_buf_extern(struct mlx5_context *ctx, struct mlx5_buf *buf,
+ size_t size)
+{
+ void *addr;
+
+ addr = ctx->extern_alloc.alloc(size, ctx->extern_alloc.data);
+ if (addr || size == 0) {
+ if (ibv_dontfork_range(addr, size)) {
+ mlx5_dbg(stderr, MLX5_DBG_CONTIG,
+ "External mode dontfork_range failed\n");
+ ctx->extern_alloc.free(addr,
+ ctx->extern_alloc.data);
+ return -1;
+ }
+ buf->buf = addr;
+ buf->length = size;
+ buf->type = MLX5_ALLOC_TYPE_EXTERNAL;
+ return 0;
+ }
+
+ mlx5_dbg(stderr, MLX5_DBG_CONTIG, "External alloc failed\n");
+ return -1;
+}
+
int mlx5_alloc_prefered_buf(struct mlx5_context *mctx,
struct mlx5_buf *buf,
size_t size, int page_size,
@@ -362,6 +392,9 @@ int mlx5_alloc_prefered_buf(struct mlx5_context *mctx,
"Contig allocation failed, fallback to default mode\n");
}
+ if (type == MLX5_ALLOC_TYPE_EXTERNAL)
+ return mlx5_alloc_buf_extern(mctx, buf, size);
+
return mlx5_alloc_buf(buf, size, page_size);
}
@@ -382,6 +415,11 @@ int mlx5_free_actual_buf(struct mlx5_context *ctx, struct mlx5_buf *buf)
case MLX5_ALLOC_TYPE_CONTIG:
mlx5_free_buf_contig(ctx, buf);
break;
+
+ case MLX5_ALLOC_TYPE_EXTERNAL:
+ mlx5_free_buf_extern(ctx, buf);
+ break;
+
default:
fprintf(stderr, "Bad allocation type\n");
}
@@ -414,7 +452,13 @@ static uint32_t mlx5_get_block_order(uint32_t v)
return r;
}
-void mlx5_get_alloc_type(const char *component,
+bool mlx5_is_extern_alloc(struct mlx5_context *context)
+{
+ return context->extern_alloc.alloc && context->extern_alloc.free;
+}
+
+void mlx5_get_alloc_type(struct mlx5_context *context,
+ const char *component,
enum mlx5_alloc_type *alloc_type,
enum mlx5_alloc_type default_type)
@@ -422,6 +466,11 @@ void mlx5_get_alloc_type(const char *component,
char *env_value;
char name[128];
+ if (mlx5_is_extern_alloc(context)) {
+ *alloc_type = MLX5_ALLOC_TYPE_EXTERNAL;
+ return;
+ }
+
snprintf(name, sizeof(name), "%s_ALLOC_TYPE", component);
*alloc_type = default_type;
@@ -1497,7 +1497,7 @@ int mlx5_alloc_cq_buf(struct mlx5_context *mctx, struct mlx5_cq *cq,
if (mlx5_use_huge("HUGE_CQ"))
default_type = MLX5_ALLOC_TYPE_HUGE;
- mlx5_get_alloc_type(MLX5_CQ_PREFIX, &type, default_type);
+ mlx5_get_alloc_type(mctx, MLX5_CQ_PREFIX, &type, default_type);
ret = mlx5_alloc_prefered_buf(mctx, buf,
align(nent * cqe_sz, dev->page_size),
@@ -53,6 +53,7 @@ static struct mlx5_db_page *__add_page(struct mlx5_context *context)
int pp;
int i;
int nlong;
+ int ret;
pp = ps / context->cache_line_size;
nlong = (pp + 8 * sizeof(long) - 1) / (8 * sizeof(long));
@@ -61,7 +62,11 @@ static struct mlx5_db_page *__add_page(struct mlx5_context *context)
if (!page)
return NULL;
- if (mlx5_alloc_buf(&page->buf, ps, ps)) {
+ if (mlx5_is_extern_alloc(context))
+ ret = mlx5_alloc_buf_extern(context, &page->buf, ps);
+ else
+ ret = mlx5_alloc_buf(&page->buf, ps, ps);
+ if (ret) {
free(page);
return NULL;
}
@@ -139,7 +144,11 @@ void mlx5_free_db(struct mlx5_context *context, __be32 *db)
if (page->next)
page->next->prev = page->prev;
- mlx5_free_buf(&page->buf);
+ if (page->buf.type == MLX5_ALLOC_TYPE_EXTERNAL)
+ mlx5_free_buf_extern(context, &page->buf);
+ else
+ mlx5_free_buf(&page->buf);
+
free(page);
}
@@ -15,4 +15,5 @@ MLX5_1.1 {
MLX5_1.2 {
global:
mlx5dv_init_obj;
+ mlx5dv_set_context_attr;
} MLX5_1.1;
@@ -642,8 +642,8 @@ static int mlx5dv_get_qp(struct ibv_qp *qp_in,
struct mlx5dv_qp *qp_out)
{
struct mlx5_qp *mqp = to_mqp(qp_in);
+ uint64_t mask_out = 0;
- qp_out->comp_mask = 0;
qp_out->dbrec = mqp->db;
if (mqp->sq_buf_size)
@@ -658,13 +658,20 @@ static int mlx5dv_get_qp(struct ibv_qp *qp_in,
qp_out->rq.wqe_cnt = mqp->rq.wqe_cnt;
qp_out->rq.stride = 1 << mqp->rq.wqe_shift;
- qp_out->bf.reg = mqp->bf->reg;
+ qp_out->bf.reg = mqp->bf->reg;
+
+ if (qp_out->comp_mask & MLX5DV_QP_MASK_UAR_MMAP_OFFSET) {
+ qp_out->uar_mmap_offset = mqp->bf->uar_mmap_offset;
+ mask_out |= MLX5DV_QP_MASK_UAR_MMAP_OFFSET;
+ }
if (mqp->bf->uuarn > 0)
qp_out->bf.size = mqp->bf->buf_size;
else
qp_out->bf.size = 0;
+ qp_out->comp_mask = mask_out;
+
return 0;
}
@@ -752,6 +759,31 @@ COMPAT_SYMVER_FUNC(mlx5dv_init_obj, 1_0, "MLX5_1.0",
return ret;
}
+static off_t get_uar_mmap_offset(int idx, int page_size)
+{
+ off_t offset = 0;
+
+ set_command(MLX5_MMAP_GET_REGULAR_PAGES_CMD, &offset);
+ set_index(idx, &offset);
+ return offset * page_size;
+}
+
+int mlx5dv_set_context_attr(struct ibv_context *ibv_ctx,
+ enum mlx5dv_set_ctx_attr_type type, void *attr)
+{
+ struct mlx5_context *ctx = to_mctx(ibv_ctx);
+
+ switch (type) {
+ case MLX5DV_CTX_ATTR_BUF_ALLOCATORS:
+ ctx->extern_alloc = *((struct mlx5dv_ctx_allocators *)attr);
+ break;
+ default:
+ return ENOTSUP;
+ }
+
+ return 0;
+}
+
static void adjust_uar_info(struct mlx5_device *mdev,
struct mlx5_context *context,
struct mlx5_alloc_ucontext_resp resp)
@@ -878,11 +910,9 @@ static int mlx5_init_context(struct verbs_device *vdev,
num_sys_page_map = context->tot_uuars / (context->num_uars_per_page * MLX5_NUM_NON_FP_BFREGS_PER_UAR);
for (i = 0; i < num_sys_page_map; ++i) {
- offset = 0;
- set_command(MLX5_MMAP_GET_REGULAR_PAGES_CMD, &offset);
- set_index(i, &offset);
+ offset = get_uar_mmap_offset(i, page_size);
context->uar[i] = mmap(NULL, page_size, PROT_WRITE, MAP_SHARED,
- cmd_fd, page_size * offset);
+ cmd_fd, offset);
if (context->uar[i] == MAP_FAILED) {
context->uar[i] = NULL;
goto err_free_bf;
@@ -901,6 +931,7 @@ static int mlx5_init_context(struct verbs_device *vdev,
if (bfi)
context->bfs[bfi].buf_size = context->bf_reg_size / 2;
context->bfs[bfi].uuarn = bfi;
+ context->bfs[bfi].uar_mmap_offset = get_uar_mmap_offset(i, page_size);
}
}
}
@@ -164,6 +164,7 @@ enum mlx5_alloc_type {
MLX5_ALLOC_TYPE_CONTIG,
MLX5_ALLOC_TYPE_PREFER_HUGE,
MLX5_ALLOC_TYPE_PREFER_CONTIG,
+ MLX5_ALLOC_TYPE_EXTERNAL,
MLX5_ALLOC_TYPE_ALL
};
@@ -267,6 +268,7 @@ struct mlx5_context {
uint32_t uar_size;
uint64_t vendor_cap_flags; /* Use enum mlx5_vendor_cap_flags */
struct mlx5dv_cqe_comp_caps cqe_comp_caps;
+ struct mlx5dv_ctx_allocators extern_alloc;
};
struct mlx5_bitmap {
@@ -385,6 +387,7 @@ struct mlx5_bf {
unsigned offset;
unsigned buf_size;
unsigned uuarn;
+ off_t uar_mmap_offset;
};
struct mlx5_mr {
@@ -556,10 +559,15 @@ int mlx5_alloc_prefered_buf(struct mlx5_context *mctx,
enum mlx5_alloc_type alloc_type,
const char *component);
int mlx5_free_actual_buf(struct mlx5_context *ctx, struct mlx5_buf *buf);
-void mlx5_get_alloc_type(const char *component,
+void mlx5_get_alloc_type(struct mlx5_context *context,
+ const char *component,
enum mlx5_alloc_type *alloc_type,
enum mlx5_alloc_type default_alloc_type);
int mlx5_use_huge(const char *key);
+bool mlx5_is_extern_alloc(struct mlx5_context *context);
+int mlx5_alloc_buf_extern(struct mlx5_context *ctx, struct mlx5_buf *buf,
+ size_t size);
+void mlx5_free_buf_extern(struct mlx5_context *ctx, struct mlx5_buf *buf);
__be32 *mlx5_alloc_dbrec(struct mlx5_context *context);
void mlx5_free_db(struct mlx5_context *context, __be32 *db);
@@ -106,6 +106,10 @@ struct ibv_cq_ex *mlx5dv_create_cq(struct ibv_context *context,
int mlx5dv_query_device(struct ibv_context *ctx_in,
struct mlx5dv_context *attrs_out);
+enum mlx5dv_qp_comp_mask {
+ MLX5DV_QP_MASK_UAR_MMAP_OFFSET = 1 << 0,
+};
+
struct mlx5dv_qp {
__be32 *dbrec;
struct {
@@ -123,6 +127,7 @@ struct mlx5dv_qp {
uint32_t size;
} bf;
uint64_t comp_mask;
+ off_t uar_mmap_offset;
};
struct mlx5dv_cq {
@@ -619,4 +624,24 @@ void mlx5dv_set_eth_seg(struct mlx5_wqe_eth_seg *seg, uint8_t cs_flags,
seg->inline_hdr_sz = htobe16(inline_hdr_sz);
memcpy(seg->inline_hdr_start, inline_hdr_start, inline_hdr_sz);
}
+
+enum mlx5dv_set_ctx_attr_type {
+ MLX5DV_CTX_ATTR_BUF_ALLOCATORS = 1,
+};
+
+struct mlx5dv_ctx_allocators {
+ void *(*alloc)(size_t size, void *priv_data);
+ void (*free)(void *ptr, void *priv_data);
+ void *data;
+};
+
+/*
+ * Generic context attributes set API
+ *
+ * Returns 0 on success, or the value of errno on failure
+ * (which indicates the failure reason).
+ */
+int mlx5dv_set_context_attr(struct ibv_context *context,
+ enum mlx5dv_set_ctx_attr_type type, void *attr);
+
#endif /* _MLX5DV_H_ */
@@ -1079,7 +1079,7 @@ static int mlx5_alloc_qp_buf(struct ibv_context *context,
if (mlx5_use_huge(qp_huge_key))
default_alloc_type = MLX5_ALLOC_TYPE_HUGE;
- mlx5_get_alloc_type(MLX5_QP_PREFIX, &alloc_type,
+ mlx5_get_alloc_type(to_mctx(context), MLX5_QP_PREFIX, &alloc_type,
default_alloc_type);
err = mlx5_alloc_prefered_buf(to_mctx(context), &qp->buf,
@@ -2004,7 +2004,10 @@ static int mlx5_alloc_rwq_buf(struct ibv_context *context,
int size)
{
int err;
- enum mlx5_alloc_type default_alloc_type = MLX5_ALLOC_TYPE_PREFER_CONTIG;
+ enum mlx5_alloc_type alloc_type;
+
+ mlx5_get_alloc_type(to_mctx(context), MLX5_RWQ_PREFIX,
+ &alloc_type, MLX5_ALLOC_TYPE_ANON);
rwq->rq.wrid = malloc(rwq->rq.wqe_cnt * sizeof(uint64_t));
if (!rwq->rq.wrid) {
@@ -2016,7 +2019,7 @@ static int mlx5_alloc_rwq_buf(struct ibv_context *context,
align(rwq->buf_size, to_mdev
(context->device)->page_size),
to_mdev(context->device)->page_size,
- default_alloc_type,
+ alloc_type,
MLX5_RWQ_PREFIX);
if (err) {