@@ -69,6 +69,7 @@ static const struct verbs_context_ops hns_common_ops = {
.cq_event = hns_roce_u_cq_event,
.create_cq = hns_roce_u_create_cq,
.create_qp = hns_roce_u_create_qp,
+ .create_qp_ex = hns_roce_u_create_qp_ex,
.dealloc_mw = hns_roce_u_dealloc_mw,
.dealloc_pd = hns_roce_u_free_pd,
.dereg_mr = hns_roce_u_dereg_mr,
@@ -257,7 +257,7 @@ struct hns_roce_rinl_buf {
};
struct hns_roce_qp {
- struct ibv_qp ibv_qp;
+ struct verbs_qp verbs_qp;
struct hns_roce_buf buf;
int max_inline_data;
int buf_size;
@@ -336,9 +336,9 @@ static inline struct hns_roce_srq *to_hr_srq(struct ibv_srq *ibv_srq)
struct hns_roce_srq, verbs_srq);
}
-static inline struct hns_roce_qp *to_hr_qp(struct ibv_qp *ibv_qp)
+static inline struct hns_roce_qp *to_hr_qp(struct ibv_qp *ibv_qp)
{
- return container_of(ibv_qp, struct hns_roce_qp, ibv_qp);
+ return container_of(ibv_qp, struct hns_roce_qp, verbs_qp.qp);
}
static inline struct hns_roce_ah *to_hr_ah(struct ibv_ah *ibv_ah)
@@ -382,6 +382,9 @@ int hns_roce_u_query_srq(struct ibv_srq *srq, struct ibv_srq_attr *srq_attr);
int hns_roce_u_destroy_srq(struct ibv_srq *srq);
struct ibv_qp *hns_roce_u_create_qp(struct ibv_pd *pd,
struct ibv_qp_init_attr *attr);
+struct ibv_qp *
+hns_roce_u_create_qp_ex(struct ibv_context *context,
+ struct ibv_qp_init_attr_ex *qp_init_attr_ex);
int hns_roce_u_query_qp(struct ibv_qp *ibqp, struct ibv_qp_attr *attr,
int attr_mask, struct ibv_qp_init_attr *init_attr);
@@ -41,11 +41,15 @@ DECLARE_DRV_CMD(hns_roce_alloc_pd, IB_USER_VERBS_CMD_ALLOC_PD,
empty, hns_roce_ib_alloc_pd_resp);
DECLARE_DRV_CMD(hns_roce_create_cq, IB_USER_VERBS_CMD_CREATE_CQ,
hns_roce_ib_create_cq, hns_roce_ib_create_cq_resp);
-DECLARE_DRV_CMD(hns_roce_create_qp, IB_USER_VERBS_CMD_CREATE_QP,
- hns_roce_ib_create_qp, hns_roce_ib_create_qp_resp);
DECLARE_DRV_CMD(hns_roce_alloc_ucontext, IB_USER_VERBS_CMD_GET_CONTEXT,
empty, hns_roce_ib_alloc_ucontext_resp);
+DECLARE_DRV_CMD(hns_roce_create_qp, IB_USER_VERBS_CMD_CREATE_QP,
+ hns_roce_ib_create_qp, hns_roce_ib_create_qp_resp);
+
+DECLARE_DRV_CMD(hns_roce_create_qp_ex, IB_USER_VERBS_EX_CMD_CREATE_QP,
+ hns_roce_ib_create_qp, hns_roce_ib_create_qp_resp);
+
DECLARE_DRV_CMD(hns_roce_create_srq, IB_USER_VERBS_CMD_CREATE_SRQ,
hns_roce_ib_create_srq, hns_roce_ib_create_srq_resp);
@@ -268,7 +268,7 @@ static int hns_roce_v1_poll_one(struct hns_roce_cq *cq,
/* if qp is zero, it will not get the correct qpn */
if (!*cur_qp ||
- (local_qpn & HNS_ROCE_CQE_QPN_MASK) != (*cur_qp)->ibv_qp.qp_num) {
+ (local_qpn & HNS_ROCE_CQE_QPN_MASK) != (*cur_qp)->verbs_qp.qp.qp_num) {
*cur_qp = hns_roce_find_qp(to_hr_ctx(cq->ibv_cq.context),
qpn & 0xffffff);
@@ -463,7 +463,7 @@ static int hns_roce_u_v1_post_send(struct ibv_qp *ibvqp, struct ibv_send_wr *wr,
for (nreq = 0; wr; ++nreq, wr = wr->next) {
if (hns_roce_wq_overflow(&qp->sq, nreq,
- to_hr_cq(qp->ibv_qp.send_cq))) {
+ to_hr_cq(qp->verbs_qp.qp.send_cq))) {
ret = -1;
*bad_wr = wr;
goto out;
@@ -572,9 +572,10 @@ out:
if (likely(nreq)) {
qp->sq.head += nreq;
- hns_roce_update_sq_head(ctx, qp->ibv_qp.qp_num,
- qp->port_num - 1, qp->sl,
- qp->sq.head & ((qp->sq.wqe_cnt << 1) - 1));
+ hns_roce_update_sq_head(ctx, qp->verbs_qp.qp.qp_num,
+ qp->port_num - 1, qp->sl,
+ qp->sq.head & ((qp->sq.wqe_cnt << 1) -
+ 1));
}
pthread_spin_unlock(&qp->sq.lock);
@@ -740,7 +741,7 @@ static int hns_roce_u_v1_post_recv(struct ibv_qp *ibvqp, struct ibv_recv_wr *wr,
for (nreq = 0; wr; ++nreq, wr = wr->next) {
if (hns_roce_wq_overflow(&qp->rq, nreq,
- to_hr_cq(qp->ibv_qp.recv_cq))) {
+ to_hr_cq(qp->verbs_qp.qp.recv_cq))) {
ret = -1;
*bad_wr = wr;
goto out;
@@ -802,8 +803,9 @@ out:
if (nreq) {
qp->rq.head += nreq;
- hns_roce_update_rq_head(ctx, qp->ibv_qp.qp_num,
- qp->rq.head & ((qp->rq.wqe_cnt << 1) - 1));
+ hns_roce_update_rq_head(ctx, qp->verbs_qp.qp.qp_num,
+ qp->rq.head & ((qp->rq.wqe_cnt << 1) -
+ 1));
}
pthread_spin_unlock(&qp->rq.lock);
@@ -367,18 +367,15 @@ static int hns_roce_flush_cqe(struct hns_roce_qp **cur_qp, struct ibv_wc *wc)
{
struct ibv_qp_attr attr;
int attr_mask;
- int ret;
if ((wc->status != IBV_WC_SUCCESS) &&
(wc->status != IBV_WC_WR_FLUSH_ERR)) {
attr_mask = IBV_QP_STATE;
attr.qp_state = IBV_QPS_ERR;
- ret = hns_roce_u_v2_modify_qp(&(*cur_qp)->ibv_qp,
- &attr, attr_mask);
- if (ret)
- fprintf(stderr, PFX "failed to modify qp!\n");
+ hns_roce_u_v2_modify_qp(&(*cur_qp)->verbs_qp.qp, &attr,
+ attr_mask);
- (*cur_qp)->ibv_qp.state = IBV_QPS_ERR;
+ (*cur_qp)->verbs_qp.qp.state = IBV_QPS_ERR;
}
return V2_CQ_OK;
@@ -468,8 +465,8 @@ static int hns_roce_handle_recv_inl_wqe(struct hns_roce_v2_cqe *cqe,
struct hns_roce_qp **cur_qp,
struct ibv_wc *wc, uint32_t opcode)
{
- if (((*cur_qp)->ibv_qp.qp_type == IBV_QPT_RC ||
- (*cur_qp)->ibv_qp.qp_type == IBV_QPT_UC) &&
+ if (((*cur_qp)->verbs_qp.qp.qp_type == IBV_QPT_RC ||
+ (*cur_qp)->verbs_qp.qp.qp_type == IBV_QPT_UC) &&
(opcode == HNS_ROCE_RECV_OP_SEND ||
opcode == HNS_ROCE_RECV_OP_SEND_WITH_IMM ||
opcode == HNS_ROCE_RECV_OP_SEND_WITH_INV) &&
@@ -539,17 +536,16 @@ static int hns_roce_v2_poll_one(struct hns_roce_cq *cq,
HNS_ROCE_V2_CQE_IS_SQ);
/* if qp is zero, it will not get the correct qpn */
- if (!*cur_qp || qpn != (*cur_qp)->ibv_qp.qp_num) {
+ if (!*cur_qp || qpn != (*cur_qp)->verbs_qp.qp.qp_num) {
*cur_qp = hns_roce_v2_find_qp(to_hr_ctx(cq->ibv_cq.context),
qpn);
- if (!*cur_qp) {
- fprintf(stderr, PFX "can't find qp!\n");
+ if (!*cur_qp)
return V2_CQ_POLL_ERR;
- }
}
wc->qp_num = qpn;
- srq = (*cur_qp)->ibv_qp.srq ? to_hr_srq((*cur_qp)->ibv_qp.srq) : NULL;
+ srq = (*cur_qp)->verbs_qp.qp.srq ?
+ to_hr_srq((*cur_qp)->verbs_qp.qp.srq) : NULL;
if (is_send) {
wq = &(*cur_qp)->sq;
/*
@@ -710,7 +706,7 @@ static void set_sge(struct hns_roce_v2_wqe_data_seg *dseg,
/* No inner sge in UD wqe */
if (sge_info->valid_num <= HNS_ROCE_SGE_IN_WQE &&
- qp->ibv_qp.qp_type != IBV_QPT_UD) {
+ qp->verbs_qp.qp.qp_type != IBV_QPT_UD) {
set_data_seg_v2(dseg, wr->sg_list + i);
dseg++;
} else {
@@ -1159,7 +1155,7 @@ int hns_roce_u_v2_post_send(struct ibv_qp *ibvqp, struct ibv_send_wr *wr,
for (nreq = 0; wr; ++nreq, wr = wr->next) {
if (hns_roce_v2_wq_overflow(&qp->sq, nreq,
- to_hr_cq(qp->ibv_qp.send_cq))) {
+ to_hr_cq(qp->verbs_qp.qp.send_cq))) {
ret = ENOMEM;
*bad_wr = wr;
goto out;
@@ -1267,7 +1263,7 @@ static int hns_roce_u_v2_post_recv(struct ibv_qp *ibvqp, struct ibv_recv_wr *wr,
max_sge = qp->rq.max_gs - qp->rq.rsv_sge;
for (nreq = 0; wr; ++nreq, wr = wr->next) {
if (hns_roce_v2_wq_overflow(&qp->rq, nreq,
- to_hr_cq(qp->ibv_qp.recv_cq))) {
+ to_hr_cq(qp->verbs_qp.qp.recv_cq))) {
ret = ENOMEM;
*bad_wr = wr;
goto out;
@@ -555,41 +555,66 @@ int hns_roce_u_destroy_srq(struct ibv_srq *srq)
return 0;
}
-static int hns_roce_verify_qp(struct ibv_qp_init_attr *attr,
- struct hns_roce_context *context)
+enum {
+ CREATE_QP_SUP_COMP_MASK = IBV_QP_INIT_ATTR_PD,
+};
+
+static int check_qp_create_mask(struct ibv_qp_init_attr_ex *attr)
{
- struct hns_roce_device *hr_dev =
- to_hr_dev(context->ibv_ctx.context.device);
- uint32_t min_wqe_num = hr_dev->hw_version == HNS_ROCE_HW_VER1 ?
- HNS_ROCE_V1_MIN_WQE_NUM :
- HNS_ROCE_V2_MIN_WQE_NUM;
-
- if (!attr->cap.max_send_wr ||
- attr->cap.max_send_wr > context->max_qp_wr ||
- attr->cap.max_recv_wr > context->max_qp_wr ||
- attr->cap.max_send_sge > context->max_sge ||
- attr->cap.max_recv_sge > context->max_sge)
- return EINVAL;
+ if (!check_comp_mask(attr->comp_mask, CREATE_QP_SUP_COMP_MASK))
+ return -EOPNOTSUPP;
- if (attr->cap.max_send_wr < min_wqe_num)
- attr->cap.max_send_wr = min_wqe_num;
+ return 0;
+}
- if (attr->cap.max_recv_wr) {
- if (attr->cap.max_recv_wr < min_wqe_num)
- attr->cap.max_recv_wr = min_wqe_num;
+static int verify_qp_create_cap(struct hns_roce_context *ctx,
+ struct ibv_qp_init_attr_ex *attr)
+{
+ struct hns_roce_device *hr_dev = to_hr_dev(ctx->ibv_ctx.context.device);
+ struct ibv_qp_cap *cap = &attr->cap;
+ uint32_t min_wqe_num;
+
+ if (!cap->max_send_wr ||
+ cap->max_send_wr > ctx->max_qp_wr ||
+ cap->max_recv_wr > ctx->max_qp_wr ||
+ cap->max_send_sge > ctx->max_sge ||
+ cap->max_recv_sge > ctx->max_sge)
+ return -EINVAL;
+
+ min_wqe_num = hr_dev->hw_version == HNS_ROCE_HW_VER1 ?
+ HNS_ROCE_V1_MIN_WQE_NUM : HNS_ROCE_V2_MIN_WQE_NUM;
- if (!attr->cap.max_recv_sge)
+ if (cap->max_send_wr < min_wqe_num)
+ cap->max_send_wr = min_wqe_num;
+
+ if (cap->max_recv_wr) {
+ if (cap->max_recv_wr < min_wqe_num)
+ cap->max_recv_wr = min_wqe_num;
+
+ if (!cap->max_recv_sge)
return -EINVAL;
}
if (!(attr->qp_type == IBV_QPT_RC ||
(attr->qp_type == IBV_QPT_UD &&
hr_dev->hw_version >= HNS_ROCE_HW_VER3)))
- return EOPNOTSUPP;
+ return -EOPNOTSUPP;
return 0;
}
+static int verify_qp_create_attr(struct hns_roce_context *ctx,
+ struct ibv_qp_init_attr_ex *attr)
+{
+ int ret;
+
+ ret = check_qp_create_mask(attr);
+ if (ret)
+ return ret;
+
+ return verify_qp_create_cap(ctx, attr);
+}
+
static int qp_alloc_recv_inl_buf(struct ibv_qp_cap *cap,
struct hns_roce_qp *qp)
{
@@ -724,7 +749,7 @@ err_alloc:
}
static void set_extend_sge_param(struct hns_roce_device *hr_dev,
- struct ibv_qp_init_attr *attr,
+ struct ibv_qp_init_attr_ex *attr,
struct hns_roce_qp *qp, unsigned int wr_cnt)
{
int cnt = 0;
@@ -745,13 +770,14 @@ static void set_extend_sge_param(struct hns_roce_device *hr_dev,
qp->ex_sge.sge_cnt = cnt;
}
-static void hns_roce_set_qp_params(struct ibv_qp_init_attr *attr,
+static void hns_roce_set_qp_params(struct ibv_qp_init_attr_ex *attr,
struct hns_roce_qp *qp,
struct hns_roce_context *ctx)
{
struct hns_roce_device *hr_dev = to_hr_dev(ctx->ibv_ctx.context.device);
unsigned int cnt;
- qp->ibv_qp.qp_type = attr->qp_type;
+
+ qp->verbs_qp.qp.qp_type = attr->qp_type;
if (attr->cap.max_recv_wr) {
if (hr_dev->hw_version == HNS_ROCE_HW_VER2)
@@ -810,7 +836,7 @@ static void qp_free_db(struct hns_roce_qp *qp, struct hns_roce_context *ctx)
hns_roce_free_db(ctx, qp->rdb, HNS_ROCE_QP_TYPE_DB);
}
-static int qp_alloc_db(struct ibv_qp_init_attr *attr, struct hns_roce_qp *qp,
+static int qp_alloc_db(struct ibv_qp_init_attr_ex *attr, struct hns_roce_qp *qp,
struct hns_roce_context *ctx)
{
struct hns_roce_device *hr_dev = to_hr_dev(ctx->ibv_ctx.context.device);
@@ -864,30 +890,30 @@ static int hns_roce_store_qp(struct hns_roce_context *ctx, uint32_t qpn,
return 0;
}
-static int qp_exec_create_cmd(struct ibv_pd *pd,
- struct ibv_qp_init_attr *attr,
+static int qp_exec_create_cmd(struct ibv_qp_init_attr_ex *attr,
struct hns_roce_qp *qp,
struct hns_roce_context *ctx)
{
- struct hns_roce_create_qp_resp resp = {};
- struct hns_roce_create_qp cmd = {};
+ struct hns_roce_create_qp_ex_resp resp_ex = {};
+ struct hns_roce_create_qp_ex cmd_ex = {};
int ret;
- cmd.sdb_addr = (uintptr_t)qp->sdb;
- cmd.db_addr = (uintptr_t)qp->rdb;
- cmd.buf_addr = (uintptr_t)qp->buf.buf;
- cmd.log_sq_stride = qp->sq.wqe_shift;
- cmd.log_sq_bb_count = hr_ilog32(qp->sq.wqe_cnt);
+ cmd_ex.sdb_addr = (uintptr_t)qp->sdb;
+ cmd_ex.db_addr = (uintptr_t)qp->rdb;
+ cmd_ex.buf_addr = (uintptr_t)qp->buf.buf;
+ cmd_ex.log_sq_stride = qp->sq.wqe_shift;
+ cmd_ex.log_sq_bb_count = hr_ilog32(qp->sq.wqe_cnt);
+
+ ret = ibv_cmd_create_qp_ex2(&ctx->ibv_ctx.context, &qp->verbs_qp, attr,
+ &cmd_ex.ibv_cmd, sizeof(cmd_ex),
+ &resp_ex.ibv_resp, sizeof(resp_ex));
- ret = ibv_cmd_create_qp(pd, &qp->ibv_qp, attr, &cmd.ibv_cmd,
- sizeof(cmd), &resp.ibv_resp, sizeof(resp));
- if (!ret)
- qp->flags = resp.cap_flags;
+ qp->flags = resp_ex.drv_payload.cap_flags;
return ret;
}
-static void qp_setup_config(struct ibv_qp_init_attr *attr,
+static void qp_setup_config(struct ibv_qp_init_attr_ex *attr,
struct hns_roce_qp *qp,
struct hns_roce_context *ctx)
{
@@ -913,7 +939,7 @@ void hns_roce_free_qp_buf(struct hns_roce_qp *qp, struct hns_roce_context *ctx)
qp_free_wqe(qp);
}
-static int hns_roce_alloc_qp_buf(struct ibv_qp_init_attr *attr,
+static int hns_roce_alloc_qp_buf(struct ibv_qp_init_attr_ex *attr,
struct hns_roce_qp *qp,
struct hns_roce_context *ctx)
{
@@ -934,20 +960,20 @@ static int hns_roce_alloc_qp_buf(struct ibv_qp_init_attr *attr,
return ret;
}
-struct ibv_qp *hns_roce_u_create_qp(struct ibv_pd *pd,
- struct ibv_qp_init_attr *attr)
+static struct ibv_qp *create_qp(struct ibv_context *ibv_ctx,
+ struct ibv_qp_init_attr_ex *attr)
{
- struct hns_roce_context *context = to_hr_ctx(pd->context);
+ struct hns_roce_context *context = to_hr_ctx(ibv_ctx);
struct hns_roce_qp *qp;
int ret;
- ret = hns_roce_verify_qp(attr, context);
+ ret = verify_qp_create_attr(context, attr);
if (ret)
goto err;
qp = calloc(1, sizeof(*qp));
if (!qp) {
- ret = ENOMEM;
+ ret = -ENOMEM;
goto err;
}
@@ -957,20 +983,20 @@ struct ibv_qp *hns_roce_u_create_qp(struct ibv_pd *pd,
if (ret)
goto err_buf;
- ret = qp_exec_create_cmd(pd, attr, qp, context);
+ ret = qp_exec_create_cmd(attr, qp, context);
if (ret)
goto err_cmd;
- ret = hns_roce_store_qp(context, qp->ibv_qp.qp_num, qp);
+ ret = hns_roce_store_qp(context, qp->verbs_qp.qp.qp_num, qp);
if (ret)
goto err_store;
qp_setup_config(attr, qp, context);
- return &qp->ibv_qp;
+ return &qp->verbs_qp.qp;
err_store:
- ibv_cmd_destroy_qp(&qp->ibv_qp);
+ ibv_cmd_destroy_qp(&qp->verbs_qp.qp);
err_cmd:
hns_roce_free_qp_buf(qp, context);
err_buf:
@@ -983,6 +1009,29 @@ err:
return NULL;
}
+struct ibv_qp *hns_roce_u_create_qp(struct ibv_pd *pd,
+ struct ibv_qp_init_attr *attr)
+{
+ struct ibv_qp_init_attr_ex attrx = {};
+ struct ibv_qp *qp;
+
+ memcpy(&attrx, attr, sizeof(*attr));
+ attrx.comp_mask = IBV_QP_INIT_ATTR_PD;
+ attrx.pd = pd;
+
+ qp = create_qp(pd->context, &attrx);
+ if (qp)
+ memcpy(attr, &attrx, sizeof(*attr));
+
+ return qp;
+}
+
+struct ibv_qp *hns_roce_u_create_qp_ex(struct ibv_context *context,
+ struct ibv_qp_init_attr_ex *attr)
+{
+ return create_qp(context, attr);
+}
+
int hns_roce_u_query_qp(struct ibv_qp *ibqp, struct ibv_qp_attr *attr,
int attr_mask, struct ibv_qp_init_attr *init_attr)
{