@@ -12,13 +12,20 @@ int mana_ib_create_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *attr,
struct ib_device *ibdev = ibcq->device;
struct mana_ib_create_cq ucmd = {};
struct mana_ib_dev *mdev;
+ struct gdma_context *gc;
int err;
mdev = container_of(ibdev, struct mana_ib_dev, ib_dev);
+ gc = mdev->gdma_dev->gdma_context;
if (udata->inlen < sizeof(ucmd))
return -EINVAL;
+ if (attr->comp_vector > gc->max_num_queues)
+ return -EINVAL;
+
+ cq->comp_vector = attr->comp_vector;
+
err = ib_copy_from_udata(&ucmd, udata, min(sizeof(ucmd), udata->inlen));
if (err) {
ibdev_dbg(ibdev,
@@ -56,6 +63,7 @@ int mana_ib_create_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *attr,
/*
* The CQ ID is not known at this time. The ID is generated at create_qp
*/
+ cq->id = INVALID_QUEUE_ID;
return 0;
@@ -69,11 +77,33 @@ int mana_ib_destroy_cq(struct ib_cq *ibcq, struct ib_udata *udata)
struct mana_ib_cq *cq = container_of(ibcq, struct mana_ib_cq, ibcq);
struct ib_device *ibdev = ibcq->device;
struct mana_ib_dev *mdev;
+ struct gdma_context *gc;
+ int err;
mdev = container_of(ibdev, struct mana_ib_dev, ib_dev);
+ gc = mdev->gdma_dev->gdma_context;
+
+ err = mana_ib_gd_destroy_dma_region(mdev, cq->gdma_region);
+ if (err) {
+ ibdev_dbg(ibdev,
+ "Failed to destroy dma region, %d\n", err);
+ return err;
+ }
+
+ if (cq->id != INVALID_QUEUE_ID) {
+ kfree(gc->cq_table[cq->id]);
+ gc->cq_table[cq->id] = NULL;
+ }
- mana_ib_gd_destroy_dma_region(mdev, cq->gdma_region);
ib_umem_release(cq->umem);
return 0;
}
+
+void mana_ib_cq_handler(void *ctx, struct gdma_queue *gdma_cq)
+{
+ struct mana_ib_cq *cq = ctx;
+
+ if (cq->ibcq.comp_handler)
+ cq->ibcq.comp_handler(&cq->ibcq, cq->ibcq.cq_context);
+}
@@ -86,6 +86,7 @@ struct mana_ib_cq {
int cqe;
u64 gdma_region;
u64 id;
+ u32 comp_vector;
};
struct mana_ib_qp {
@@ -209,4 +210,6 @@ int mana_ib_query_gid(struct ib_device *ibdev, u32 port, int index,
void mana_ib_disassociate_ucontext(struct ib_ucontext *ibcontext);
int mana_ib_gd_query_adapter_caps(struct mana_ib_dev *mdev);
+
+void mana_ib_cq_handler(void *ctx, struct gdma_queue *gdma_cq);
#endif
@@ -102,21 +102,26 @@ static int mana_ib_create_qp_rss(struct ib_qp *ibqp, struct ib_pd *pd,
struct ib_rwq_ind_table *ind_tbl = attr->rwq_ind_tbl;
struct mana_ib_create_qp_rss_resp resp = {};
struct mana_ib_create_qp_rss ucmd = {};
+ struct gdma_queue **gdma_cq_allocated;
mana_handle_t *mana_ind_table;
struct mana_port_context *mpc;
+ struct gdma_queue *gdma_cq;
unsigned int ind_tbl_size;
struct mana_context *mc;
struct net_device *ndev;
+ struct gdma_context *gc;
struct mana_ib_cq *cq;
struct mana_ib_wq *wq;
struct gdma_dev *gd;
+ struct mana_eq *eq;
struct ib_cq *ibcq;
struct ib_wq *ibwq;
int i = 0;
u32 port;
int ret;
- gd = &mdev->gdma_dev->gdma_context->mana;
+ gc = mdev->gdma_dev->gdma_context;
+ gd = &gc->mana;
mc = gd->driver_data;
if (!udata || udata->inlen < sizeof(ucmd))
@@ -179,6 +184,13 @@ static int mana_ib_create_qp_rss(struct ib_qp *ibqp, struct ib_pd *pd,
goto fail;
}
+ gdma_cq_allocated = kcalloc(ind_tbl_size, sizeof(*gdma_cq_allocated),
+ GFP_KERNEL);
+ if (!gdma_cq_allocated) {
+ ret = -ENOMEM;
+ goto fail;
+ }
+
qp->port = port;
for (i = 0; i < ind_tbl_size; i++) {
@@ -197,12 +209,16 @@ static int mana_ib_create_qp_rss(struct ib_qp *ibqp, struct ib_pd *pd,
cq_spec.gdma_region = cq->gdma_region;
cq_spec.queue_size = cq->cqe * COMP_ENTRY_SIZE;
cq_spec.modr_ctx_id = 0;
- cq_spec.attached_eq = GDMA_CQ_NO_EQ;
+ eq = &mc->eqs[cq->comp_vector % gc->max_num_queues];
+ cq_spec.attached_eq = eq->eq->id;
ret = mana_create_wq_obj(mpc, mpc->port_handle, GDMA_RQ,
&wq_spec, &cq_spec, &wq->rx_object);
- if (ret)
+ if (ret) {
+ /* Do cleanup starting with index i-1 */
+ i--;
goto fail;
+ }
/* The GDMA regions are now owned by the WQ object */
wq->gdma_region = GDMA_INVALID_DMA_REGION;
@@ -219,6 +235,21 @@ static int mana_ib_create_qp_rss(struct ib_qp *ibqp, struct ib_pd *pd,
resp.entries[i].wqid = wq->id;
mana_ind_table[i] = wq->rx_object;
+
+ /* Create CQ table entry */
+ WARN_ON(gc->cq_table[cq->id]);
+ gdma_cq = kzalloc(sizeof(*gdma_cq), GFP_KERNEL);
+ if (!gdma_cq) {
+ ret = -ENOMEM;
+ goto fail;
+ }
+ gdma_cq_allocated[i] = gdma_cq;
+
+ gdma_cq->cq.context = cq;
+ gdma_cq->type = GDMA_CQ;
+ gdma_cq->cq.callback = mana_ib_cq_handler;
+ gdma_cq->id = cq->id;
+ gc->cq_table[cq->id] = gdma_cq;
}
resp.num_entries = i;
@@ -238,6 +269,7 @@ static int mana_ib_create_qp_rss(struct ib_qp *ibqp, struct ib_pd *pd,
goto fail;
}
+ kfree(gdma_cq_allocated);
kfree(mana_ind_table);
return 0;
@@ -245,10 +277,17 @@ static int mana_ib_create_qp_rss(struct ib_qp *ibqp, struct ib_pd *pd,
fail:
while (i-- > 0) {
ibwq = ind_tbl->ind_tbl[i];
+ ibcq = ibwq->cq;
wq = container_of(ibwq, struct mana_ib_wq, ibwq);
+ cq = container_of(ibcq, struct mana_ib_cq, ibcq);
+
+ gc->cq_table[cq->id] = NULL;
+ kfree(gdma_cq_allocated[i]);
+
mana_destroy_wq_obj(mpc, GDMA_RQ, wq->rx_object);
}
+ kfree(gdma_cq_allocated);
kfree(mana_ind_table);
return ret;
@@ -270,14 +309,17 @@ static int mana_ib_create_qp_raw(struct ib_qp *ibqp, struct ib_pd *ibpd,
struct gdma_dev *gd = &mdev->gdma_dev->gdma_context->mana;
struct mana_ib_create_qp_resp resp = {};
struct mana_ib_create_qp ucmd = {};
+ struct gdma_queue *gdma_cq = NULL;
struct mana_obj_spec wq_spec = {};
struct mana_obj_spec cq_spec = {};
struct mana_port_context *mpc;
struct mana_context *mc;
struct net_device *ndev;
struct ib_umem *umem;
- int err;
+ struct mana_eq *eq;
+ int eq_vec;
u32 port;
+ int err;
mc = gd->driver_data;
@@ -354,7 +396,9 @@ static int mana_ib_create_qp_raw(struct ib_qp *ibqp, struct ib_pd *ibpd,
cq_spec.gdma_region = send_cq->gdma_region;
cq_spec.queue_size = send_cq->cqe * COMP_ENTRY_SIZE;
cq_spec.modr_ctx_id = 0;
- cq_spec.attached_eq = GDMA_CQ_NO_EQ;
+ eq_vec = send_cq->comp_vector % gd->gdma_context->max_num_queues;
+ eq = &mc->eqs[eq_vec];
+ cq_spec.attached_eq = eq->eq->id;
err = mana_create_wq_obj(mpc, mpc->port_handle, GDMA_SQ, &wq_spec,
&cq_spec, &qp->tx_object);
@@ -372,6 +416,20 @@ static int mana_ib_create_qp_raw(struct ib_qp *ibqp, struct ib_pd *ibpd,
qp->sq_id = wq_spec.queue_index;
send_cq->id = cq_spec.queue_index;
+ /* Create CQ table entry */
+ WARN_ON(gd->gdma_context->cq_table[send_cq->id]);
+ gdma_cq = kzalloc(sizeof(*gdma_cq), GFP_KERNEL);
+ if (!gdma_cq) {
+ err = -ENOMEM;
+ goto err_destroy_wq_obj;
+ }
+
+ gdma_cq->cq.context = send_cq;
+ gdma_cq->type = GDMA_CQ;
+ gdma_cq->cq.callback = mana_ib_cq_handler;
+ gdma_cq->id = send_cq->id;
+ gd->gdma_context->cq_table[send_cq->id] = gdma_cq;
+
ibdev_dbg(&mdev->ib_dev,
"ret %d qp->tx_object 0x%llx sq id %llu cq id %llu\n", err,
qp->tx_object, qp->sq_id, send_cq->id);
@@ -391,6 +449,11 @@ static int mana_ib_create_qp_raw(struct ib_qp *ibqp, struct ib_pd *ibpd,
return 0;
err_destroy_wq_obj:
+ if (gdma_cq) {
+ kfree(gdma_cq);
+ gd->gdma_context->cq_table[send_cq->id] = NULL;
+ }
+
mana_destroy_wq_obj(mpc, GDMA_SQ, qp->tx_object);
err_destroy_dma_region: