@@ -84,8 +84,7 @@ enum write_fallback _execute_ioctl_fallback(struct ibv_context *ctx,
struct ibv_command_buffer *cmdb,
int *ret)
{
- struct verbs_ex_private *priv =
- container_of(ctx, struct verbs_context, context)->priv;
+ struct verbs_ex_private *priv = get_priv(ctx);
if (bitmap_test_bit(priv->unsupported_ioctls, cmd_bit))
return _check_legacy(cmdb, ret);
@@ -162,8 +162,6 @@ static struct ibv_cq_ex *
__lib_ibv_create_cq_ex(struct ibv_context *context,
struct ibv_cq_init_attr_ex *cq_attr)
{
- struct verbs_context *vctx =
- container_of(context, struct verbs_context, context);
struct ibv_cq_ex *cq;
if (cq_attr->wc_flags & ~IBV_CREATE_CQ_SUP_WC_FLAGS) {
@@ -171,7 +169,7 @@ __lib_ibv_create_cq_ex(struct ibv_context *context,
return NULL;
}
- cq = vctx->priv->create_cq_ex(context, cq_attr);
+ cq = get_ops(context)->create_cq_ex(context, cq_attr);
if (cq)
verbs_init_cq(ibv_cq_ex_to_cq(cq), context,
@@ -263,6 +261,11 @@ err_free:
return NULL;
}
+static void set_lib_ops(struct verbs_context *vctx)
+{
+ vctx->create_cq_ex = __lib_ibv_create_cq_ex;
+}
+
LATEST_SYMVER_FUNC(ibv_open_device, 1_1, "IBVERBS_1.1",
struct ibv_context *,
struct ibv_device *device)
@@ -293,10 +296,7 @@ LATEST_SYMVER_FUNC(ibv_open_device, 1_1, "IBVERBS_1.1",
if (!context_ex)
return NULL;
- if (context_ex->create_cq_ex) {
- context_ex->priv->create_cq_ex = context_ex->create_cq_ex;
- context_ex->create_cq_ex = __lib_ibv_create_cq_ex;
- }
+ set_lib_ops(context_ex);
return &context_ex->context;
}
@@ -361,7 +361,7 @@ LATEST_SYMVER_FUNC(ibv_get_async_event, 1_1, "IBVERBS_1.1",
break;
}
- context->ops.async_event(event);
+ get_ops(context)->async_event(event);
return 0;
}
@@ -504,64 +504,89 @@ const struct verbs_context_ops verbs_dummy_ops = {
void verbs_set_ops(struct verbs_context *vctx,
const struct verbs_context_ops *ops)
{
+ struct verbs_ex_private *priv = vctx->priv;
struct ibv_context_ops *ctx = &vctx->context.ops;
-#define SET_OP(ptr, name) \
+ /*
+ * We retain the function pointer for now, just as 'just-in-case' ABI
+ * compatibility. If any ever get changed incompatibly they should be
+ * set to NULL instead.
+ */
+#define SET_PRIV_OP(ptr, name) \
+ do { \
+ if (ops->name) { \
+ priv->ops.name = ops->name; \
+ (ptr)->_compat_##name = (void *(*)(void))ops->name; \
+ } \
+ } while (0)
+
+ /* Same as SET_PRIV_OP but without the compatibility pointer */
+#define SET_PRIV_OP_IC(ptr, name) \
do { \
if (ops->name) \
+ priv->ops.name = ops->name; \
+ } while (0)
+
+#define SET_OP(ptr, name) \
+ do { \
+ if (ops->name) { \
+ priv->ops.name = ops->name; \
(ptr)->name = ops->name; \
+ } \
} while (0)
#define SET_OP2(ptr, iname, name) \
do { \
- if (ops->name) \
+ if (ops->name) { \
+ priv->ops.name = ops->name; \
(ptr)->iname = ops->name; \
+ } \
} while (0)
SET_OP(vctx, alloc_dm);
SET_OP(ctx, alloc_mw);
- SET_OP(ctx, alloc_pd);
+ SET_PRIV_OP(ctx, alloc_pd);
SET_OP(vctx, alloc_parent_domain);
SET_OP(vctx, alloc_td);
- SET_OP(ctx, async_event);
SET_OP(vctx, attach_counters_point_flow);
- SET_OP(ctx, attach_mcast);
+ SET_OP(vctx, create_counters);
+ SET_PRIV_OP(ctx, async_event);
+ SET_PRIV_OP(ctx, attach_mcast);
SET_OP(ctx, bind_mw);
SET_OP(vctx, close_xrcd);
- SET_OP(ctx, cq_event);
- SET_OP(ctx, create_ah);
- SET_OP(vctx, create_counters);
- SET_OP(ctx, create_cq);
- SET_OP(vctx, create_cq_ex);
+ SET_PRIV_OP(ctx, cq_event);
+ SET_PRIV_OP(ctx, create_ah);
+ SET_PRIV_OP(ctx, create_cq);
+ SET_PRIV_OP_IC(vctx, create_cq_ex);
SET_OP2(vctx, ibv_create_flow, create_flow);
SET_OP(vctx, create_flow_action_esp);
- SET_OP(ctx, create_qp);
+ SET_PRIV_OP(ctx, create_qp);
SET_OP(vctx, create_qp_ex);
SET_OP(vctx, create_rwq_ind_table);
- SET_OP(ctx, create_srq);
+ SET_PRIV_OP(ctx, create_srq);
SET_OP(vctx, create_srq_ex);
SET_OP(vctx, create_wq);
SET_OP(ctx, dealloc_mw);
- SET_OP(ctx, dealloc_pd);
+ SET_PRIV_OP(ctx, dealloc_pd);
SET_OP(vctx, dealloc_td);
- SET_OP(ctx, dereg_mr);
- SET_OP(ctx, destroy_ah);
SET_OP(vctx, destroy_counters);
- SET_OP(ctx, destroy_cq);
+ SET_PRIV_OP(ctx, dereg_mr);
+ SET_PRIV_OP(ctx, destroy_ah);
+ SET_PRIV_OP(ctx, destroy_cq);
SET_OP2(vctx, ibv_destroy_flow, destroy_flow);
SET_OP(vctx, destroy_flow_action);
- SET_OP(ctx, destroy_qp);
+ SET_PRIV_OP(ctx, destroy_qp);
SET_OP(vctx, destroy_rwq_ind_table);
- SET_OP(ctx, destroy_srq);
+ SET_PRIV_OP(ctx, destroy_srq);
SET_OP(vctx, destroy_wq);
- SET_OP(ctx, detach_mcast);
+ SET_PRIV_OP(ctx, detach_mcast);
SET_OP(vctx, free_dm);
SET_OP(vctx, get_srq_num);
SET_OP(vctx, modify_cq);
SET_OP(vctx, modify_flow_action_esp);
- SET_OP(ctx, modify_qp);
+ SET_PRIV_OP(ctx, modify_qp);
SET_OP(vctx, modify_qp_rate_limit);
- SET_OP(ctx, modify_srq);
+ SET_PRIV_OP(ctx, modify_srq);
SET_OP(vctx, modify_wq);
SET_OP(vctx, open_qp);
SET_OP(vctx, open_xrcd);
@@ -570,18 +595,18 @@ void verbs_set_ops(struct verbs_context *vctx,
SET_OP(ctx, post_send);
SET_OP(vctx, post_srq_ops);
SET_OP(ctx, post_srq_recv);
- SET_OP(ctx, query_device);
+ SET_PRIV_OP(ctx, query_device);
SET_OP(vctx, query_device_ex);
- SET_OP(ctx, query_port);
- SET_OP(ctx, query_qp);
+ SET_PRIV_OP(ctx, query_port);
+ SET_PRIV_OP(ctx, query_qp);
SET_OP(vctx, query_rt_values);
- SET_OP(ctx, query_srq);
SET_OP(vctx, read_counters);
+ SET_PRIV_OP(ctx, query_srq);
SET_OP(vctx, reg_dm_mr);
- SET_OP(ctx, reg_mr);
+ SET_PRIV_OP(ctx, reg_mr);
SET_OP(ctx, req_notify_cq);
- SET_OP(ctx, rereg_mr);
- SET_OP(ctx, resize_cq);
+ SET_PRIV_OP(ctx, rereg_mr);
+ SET_PRIV_OP(ctx, resize_cq);
#undef SET_OP
#undef SET_OP2
@@ -61,13 +61,21 @@ void ibverbs_device_put(struct ibv_device *dev);
void ibverbs_device_hold(struct ibv_device *dev);
struct verbs_ex_private {
- struct ibv_cq_ex *(*create_cq_ex)(struct ibv_context *context,
- struct ibv_cq_init_attr_ex *init_attr);
-
BITMAP_DECLARE(unsupported_ioctls, VERBS_OPS_NUM);
uint32_t driver_id;
+ struct verbs_context_ops ops;
};
+static inline struct verbs_ex_private *get_priv(struct ibv_context *ctx)
+{
+ return container_of(ctx, struct verbs_context, context)->priv;
+}
+
+static inline const struct verbs_context_ops *get_ops(struct ibv_context *ctx)
+{
+ return &get_priv(ctx)->ops;
+}
+
#define IBV_INIT_CMD(cmd, size, opcode) \
do { \
(cmd)->hdr.command = IB_USER_VERBS_CMD_##opcode; \
@@ -141,7 +141,7 @@ LATEST_SYMVER_FUNC(ibv_query_device, 1_1, "IBVERBS_1.1",
struct ibv_context *context,
struct ibv_device_attr *device_attr)
{
- return context->ops.query_device(context, device_attr);
+ return get_ops(context)->query_device(context, device_attr);
}
LATEST_SYMVER_FUNC(ibv_query_port, 1_1, "IBVERBS_1.1",
@@ -149,7 +149,7 @@ LATEST_SYMVER_FUNC(ibv_query_port, 1_1, "IBVERBS_1.1",
struct ibv_context *context, uint8_t port_num,
struct ibv_port_attr *port_attr)
{
- return context->ops.query_port(context, port_num, port_attr);
+ return get_ops(context)->query_port(context, port_num, port_attr);
}
LATEST_SYMVER_FUNC(ibv_query_gid, 1_1, "IBVERBS_1.1",
@@ -206,7 +206,7 @@ LATEST_SYMVER_FUNC(ibv_alloc_pd, 1_1, "IBVERBS_1.1",
{
struct ibv_pd *pd;
- pd = context->ops.alloc_pd(context);
+ pd = get_ops(context)->alloc_pd(context);
if (pd)
pd->context = context;
@@ -217,7 +217,7 @@ LATEST_SYMVER_FUNC(ibv_dealloc_pd, 1_1, "IBVERBS_1.1",
int,
struct ibv_pd *pd)
{
- return pd->context->ops.dealloc_pd(pd);
+ return get_ops(pd->context)->dealloc_pd(pd);
}
LATEST_SYMVER_FUNC(ibv_reg_mr, 1_1, "IBVERBS_1.1",
@@ -230,7 +230,7 @@ LATEST_SYMVER_FUNC(ibv_reg_mr, 1_1, "IBVERBS_1.1",
if (ibv_dontfork_range(addr, length))
return NULL;
- mr = pd->context->ops.reg_mr(pd, addr, length, access);
+ mr = get_ops(pd->context)->reg_mr(pd, addr, length, access);
if (mr) {
mr->context = pd->context;
mr->pd = pd;
@@ -278,7 +278,8 @@ LATEST_SYMVER_FUNC(ibv_rereg_mr, 1_1, "IBVERBS_1.1",
old_addr = mr->addr;
old_len = mr->length;
- err = mr->context->ops.rereg_mr(mr, flags, pd, addr, length, access);
+ err = get_ops(mr->context)
+ ->rereg_mr(mr, flags, pd, addr, length, access);
if (!err) {
if (flags & IBV_REREG_MR_CHANGE_PD)
mr->pd = pd;
@@ -308,7 +309,7 @@ LATEST_SYMVER_FUNC(ibv_dereg_mr, 1_1, "IBVERBS_1.1",
void *addr = mr->addr;
size_t length = mr->length;
- ret = mr->context->ops.dereg_mr(mr);
+ ret = get_ops(mr->context)->dereg_mr(mr);
if (!ret)
ibv_dofork_range(addr, length);
@@ -370,7 +371,7 @@ LATEST_SYMVER_FUNC(ibv_create_cq, 1_1, "IBVERBS_1.1",
{
struct ibv_cq *cq;
- cq = context->ops.create_cq(context, cqe, channel, comp_vector);
+ cq = get_ops(context)->create_cq(context, cqe, channel, comp_vector);
if (cq)
verbs_init_cq(cq, context, channel, cq_context);
@@ -382,7 +383,7 @@ LATEST_SYMVER_FUNC(ibv_resize_cq, 1_1, "IBVERBS_1.1",
int,
struct ibv_cq *cq, int cqe)
{
- return cq->context->ops.resize_cq(cq, cqe);
+ return get_ops(cq->context)->resize_cq(cq, cqe);
}
LATEST_SYMVER_FUNC(ibv_destroy_cq, 1_1, "IBVERBS_1.1",
@@ -392,7 +393,7 @@ LATEST_SYMVER_FUNC(ibv_destroy_cq, 1_1, "IBVERBS_1.1",
struct ibv_comp_channel *channel = cq->channel;
int ret;
- ret = cq->context->ops.destroy_cq(cq);
+ ret = get_ops(cq->context)->destroy_cq(cq);
if (channel) {
if (!ret) {
@@ -418,7 +419,7 @@ LATEST_SYMVER_FUNC(ibv_get_cq_event, 1_1, "IBVERBS_1.1",
*cq = (struct ibv_cq *) (uintptr_t) ev.cq_handle;
*cq_context = (*cq)->cq_context;
- (*cq)->context->ops.cq_event(*cq);
+ get_ops((*cq)->context)->cq_event(*cq);
return 0;
}
@@ -440,7 +441,7 @@ LATEST_SYMVER_FUNC(ibv_create_srq, 1_1, "IBVERBS_1.1",
{
struct ibv_srq *srq;
- srq = pd->context->ops.create_srq(pd, srq_init_attr);
+ srq = get_ops(pd->context)->create_srq(pd, srq_init_attr);
if (srq) {
srq->context = pd->context;
srq->srq_context = srq_init_attr->srq_context;
@@ -459,21 +460,21 @@ LATEST_SYMVER_FUNC(ibv_modify_srq, 1_1, "IBVERBS_1.1",
struct ibv_srq_attr *srq_attr,
int srq_attr_mask)
{
- return srq->context->ops.modify_srq(srq, srq_attr, srq_attr_mask);
+ return get_ops(srq->context)->modify_srq(srq, srq_attr, srq_attr_mask);
}
LATEST_SYMVER_FUNC(ibv_query_srq, 1_1, "IBVERBS_1.1",
int,
struct ibv_srq *srq, struct ibv_srq_attr *srq_attr)
{
- return srq->context->ops.query_srq(srq, srq_attr);
+ return get_ops(srq->context)->query_srq(srq, srq_attr);
}
LATEST_SYMVER_FUNC(ibv_destroy_srq, 1_1, "IBVERBS_1.1",
int,
struct ibv_srq *srq)
{
- return srq->context->ops.destroy_srq(srq);
+ return get_ops(srq->context)->destroy_srq(srq);
}
LATEST_SYMVER_FUNC(ibv_create_qp, 1_1, "IBVERBS_1.1",
@@ -481,7 +482,7 @@ LATEST_SYMVER_FUNC(ibv_create_qp, 1_1, "IBVERBS_1.1",
struct ibv_pd *pd,
struct ibv_qp_init_attr *qp_init_attr)
{
- struct ibv_qp *qp = pd->context->ops.create_qp(pd, qp_init_attr);
+ struct ibv_qp *qp = get_ops(pd->context)->create_qp(pd, qp_init_attr);
if (qp) {
qp->context = pd->context;
@@ -508,7 +509,7 @@ LATEST_SYMVER_FUNC(ibv_query_qp, 1_1, "IBVERBS_1.1",
{
int ret;
- ret = qp->context->ops.query_qp(qp, attr, attr_mask, init_attr);
+ ret = get_ops(qp->context)->query_qp(qp, attr, attr_mask, init_attr);
if (ret)
return ret;
@@ -525,7 +526,7 @@ LATEST_SYMVER_FUNC(ibv_modify_qp, 1_1, "IBVERBS_1.1",
{
int ret;
- ret = qp->context->ops.modify_qp(qp, attr, attr_mask);
+ ret = get_ops(qp->context)->modify_qp(qp, attr, attr_mask);
if (ret)
return ret;
@@ -539,14 +540,14 @@ LATEST_SYMVER_FUNC(ibv_destroy_qp, 1_1, "IBVERBS_1.1",
int,
struct ibv_qp *qp)
{
- return qp->context->ops.destroy_qp(qp);
+ return get_ops(qp->context)->destroy_qp(qp);
}
LATEST_SYMVER_FUNC(ibv_create_ah, 1_1, "IBVERBS_1.1",
struct ibv_ah *,
struct ibv_pd *pd, struct ibv_ah_attr *attr)
{
- struct ibv_ah *ah = pd->context->ops.create_ah(pd, attr);
+ struct ibv_ah *ah = get_ops(pd->context)->create_ah(pd, attr);
if (ah) {
ah->context = pd->context;
@@ -819,21 +820,21 @@ LATEST_SYMVER_FUNC(ibv_destroy_ah, 1_1, "IBVERBS_1.1",
int,
struct ibv_ah *ah)
{
- return ah->context->ops.destroy_ah(ah);
+ return get_ops(ah->context)->destroy_ah(ah);
}
LATEST_SYMVER_FUNC(ibv_attach_mcast, 1_1, "IBVERBS_1.1",
int,
struct ibv_qp *qp, const union ibv_gid *gid, uint16_t lid)
{
- return qp->context->ops.attach_mcast(qp, gid, lid);
+ return get_ops(qp->context)->attach_mcast(qp, gid, lid);
}
LATEST_SYMVER_FUNC(ibv_detach_mcast, 1_1, "IBVERBS_1.1",
int,
struct ibv_qp *qp, const union ibv_gid *gid, uint16_t lid)
{
- return qp->context->ops.detach_mcast(qp, gid, lid);
+ return get_ops(qp->context)->detach_mcast(qp, gid, lid);
}
static inline int ipv6_addr_v4mapped(const struct in6_addr *a)
@@ -1662,61 +1662,43 @@ struct ibv_device {
};
struct ibv_context_ops {
- int (*query_device)(struct ibv_context *context,
- struct ibv_device_attr *device_attr);
- int (*query_port)(struct ibv_context *context, uint8_t port_num,
- struct ibv_port_attr *port_attr);
- struct ibv_pd * (*alloc_pd)(struct ibv_context *context);
- int (*dealloc_pd)(struct ibv_pd *pd);
- struct ibv_mr * (*reg_mr)(struct ibv_pd *pd, void *addr, size_t length,
- int access);
- int (*rereg_mr)(struct ibv_mr *mr,
- int flags,
- struct ibv_pd *pd, void *addr,
- size_t length,
- int access);
- int (*dereg_mr)(struct ibv_mr *mr);
+ void *(*_compat_query_device)(void);
+ void *(*_compat_query_port)(void);
+ void *(*_compat_alloc_pd)(void);
+ void *(*_compat_dealloc_pd)(void);
+ void *(*_compat_reg_mr)(void);
+ void *(*_compat_rereg_mr)(void);
+ void *(*_compat_dereg_mr)(void);
struct ibv_mw * (*alloc_mw)(struct ibv_pd *pd, enum ibv_mw_type type);
int (*bind_mw)(struct ibv_qp *qp, struct ibv_mw *mw,
struct ibv_mw_bind *mw_bind);
int (*dealloc_mw)(struct ibv_mw *mw);
- struct ibv_cq * (*create_cq)(struct ibv_context *context, int cqe,
- struct ibv_comp_channel *channel,
- int comp_vector);
+ void *(*_compat_create_cq)(void);
int (*poll_cq)(struct ibv_cq *cq, int num_entries, struct ibv_wc *wc);
int (*req_notify_cq)(struct ibv_cq *cq, int solicited_only);
- void (*cq_event)(struct ibv_cq *cq);
- int (*resize_cq)(struct ibv_cq *cq, int cqe);
- int (*destroy_cq)(struct ibv_cq *cq);
- struct ibv_srq * (*create_srq)(struct ibv_pd *pd,
- struct ibv_srq_init_attr *srq_init_attr);
- int (*modify_srq)(struct ibv_srq *srq,
- struct ibv_srq_attr *srq_attr,
- int srq_attr_mask);
- int (*query_srq)(struct ibv_srq *srq,
- struct ibv_srq_attr *srq_attr);
- int (*destroy_srq)(struct ibv_srq *srq);
+ void *(*_compat_cq_event)(void);
+ void *(*_compat_resize_cq)(void);
+ void *(*_compat_destroy_cq)(void);
+ void *(*_compat_create_srq)(void);
+ void *(*_compat_modify_srq)(void);
+ void *(*_compat_query_srq)(void);
+ void *(*_compat_destroy_srq)(void);
int (*post_srq_recv)(struct ibv_srq *srq,
struct ibv_recv_wr *recv_wr,
struct ibv_recv_wr **bad_recv_wr);
- struct ibv_qp * (*create_qp)(struct ibv_pd *pd, struct ibv_qp_init_attr *attr);
- int (*query_qp)(struct ibv_qp *qp, struct ibv_qp_attr *attr,
- int attr_mask,
- struct ibv_qp_init_attr *init_attr);
- int (*modify_qp)(struct ibv_qp *qp, struct ibv_qp_attr *attr,
- int attr_mask);
- int (*destroy_qp)(struct ibv_qp *qp);
+ void *(*_compat_create_qp)(void);
+ void *(*_compat_query_qp)(void);
+ void *(*_compat_modify_qp)(void);
+ void *(*_compat_destroy_qp)(void);
int (*post_send)(struct ibv_qp *qp, struct ibv_send_wr *wr,
struct ibv_send_wr **bad_wr);
int (*post_recv)(struct ibv_qp *qp, struct ibv_recv_wr *wr,
struct ibv_recv_wr **bad_wr);
- struct ibv_ah * (*create_ah)(struct ibv_pd *pd, struct ibv_ah_attr *attr);
- int (*destroy_ah)(struct ibv_ah *ah);
- int (*attach_mcast)(struct ibv_qp *qp, const union ibv_gid *gid,
- uint16_t lid);
- int (*detach_mcast)(struct ibv_qp *qp, const union ibv_gid *gid,
- uint16_t lid);
- void (*async_event)(struct ibv_async_event *event);
+ void *(*_compat_create_ah)(void);
+ void *(*_compat_destroy_ah)(void);
+ void *(*_compat_attach_mcast)(void);
+ void *(*_compat_detach_mcast)(void);
+ void *(*_compat_async_event)(void);
};
struct ibv_context {