@@ -80,7 +80,7 @@ EXPORT_SYMBOL(ib_process_cq_direct);
static void ib_cq_completion_direct(struct ib_cq *cq, void *private)
{
- WARN_ONCE(1, "got unsolicited completion for CQ 0x%p\n", cq);
+ pr_debug("got unsolicited completion for CQ 0x%p\n", cq);
}
static int ib_poll_handler(struct irq_poll *iop, int budget)
@@ -120,6 +120,33 @@ static void ib_cq_completion_workqueue(struct ib_cq *cq, void *private)
queue_work(cq->comp_wq, &cq->work);
}
+static int __ib_cq_set_ctx(struct ib_cq *cq)
+{
+ switch (cq->poll_ctx) {
+ case IB_POLL_DIRECT:
+ cq->comp_handler = ib_cq_completion_direct;
+ break;
+ case IB_POLL_SOFTIRQ:
+ cq->comp_handler = ib_cq_completion_softirq;
+
+ irq_poll_init(&cq->iop, IB_POLL_BUDGET_IRQ, ib_poll_handler);
+ ib_req_notify_cq(cq, IB_CQ_NEXT_COMP);
+ break;
+ case IB_POLL_WORKQUEUE:
+ case IB_POLL_UNBOUND_WORKQUEUE:
+ cq->comp_handler = ib_cq_completion_workqueue;
+ INIT_WORK(&cq->work, ib_cq_poll_work);
+ ib_req_notify_cq(cq, IB_CQ_NEXT_COMP);
+ cq->comp_wq = (cq->poll_ctx == IB_POLL_WORKQUEUE) ?
+ ib_comp_wq : ib_comp_unbound_wq;
+ break;
+ default:
+ return -EINVAL;
+ }
+
+ return 0;
+}
+
/**
* __ib_alloc_cq - allocate a completion queue
* @dev: device to allocate the CQ for
@@ -164,28 +191,9 @@ struct ib_cq *__ib_alloc_cq(struct ib_device *dev, void *private,
rdma_restrack_set_task(&cq->res, caller);
rdma_restrack_add(&cq->res);
- switch (cq->poll_ctx) {
- case IB_POLL_DIRECT:
- cq->comp_handler = ib_cq_completion_direct;
- break;
- case IB_POLL_SOFTIRQ:
- cq->comp_handler = ib_cq_completion_softirq;
-
- irq_poll_init(&cq->iop, IB_POLL_BUDGET_IRQ, ib_poll_handler);
- ib_req_notify_cq(cq, IB_CQ_NEXT_COMP);
- break;
- case IB_POLL_WORKQUEUE:
- case IB_POLL_UNBOUND_WORKQUEUE:
- cq->comp_handler = ib_cq_completion_workqueue;
- INIT_WORK(&cq->work, ib_cq_poll_work);
- ib_req_notify_cq(cq, IB_CQ_NEXT_COMP);
- cq->comp_wq = (cq->poll_ctx == IB_POLL_WORKQUEUE) ?
- ib_comp_wq : ib_comp_unbound_wq;
- break;
- default:
- ret = -EINVAL;
+ ret = __ib_cq_set_ctx(cq);
+ if (ret)
goto out_free_wc;
- }
return cq;
@@ -198,17 +206,8 @@ struct ib_cq *__ib_alloc_cq(struct ib_device *dev, void *private,
}
EXPORT_SYMBOL(__ib_alloc_cq);
-/**
- * ib_free_cq - free a completion queue
- * @cq: completion queue to free.
- */
-void ib_free_cq(struct ib_cq *cq)
+static void __ib_cq_clear_ctx(struct ib_cq *cq)
{
- int ret;
-
- if (WARN_ON_ONCE(atomic_read(&cq->usecnt)))
- return;
-
switch (cq->poll_ctx) {
case IB_POLL_DIRECT:
break;
@@ -222,6 +221,20 @@ void ib_free_cq(struct ib_cq *cq)
default:
WARN_ON_ONCE(1);
}
+}
+
+/**
+ * ib_free_cq - free a completion queue
+ * @cq: completion queue to free.
+ */
+void ib_free_cq(struct ib_cq *cq)
+{
+ int ret;
+
+ if (WARN_ON_ONCE(atomic_read(&cq->usecnt)))
+ return;
+
+ __ib_cq_clear_ctx(cq);
kfree(cq->wc);
rdma_restrack_del(&cq->res);
@@ -229,3 +242,28 @@ void ib_free_cq(struct ib_cq *cq)
WARN_ON_ONCE(ret);
}
EXPORT_SYMBOL(ib_free_cq);
+
+/**
+ * ib_change_cq_ctx - change completion queue polling context dynamically
+ * @cq: the completion queue
+ * @poll_ctx: new context to poll the CQ from
+ *
+ * The caller must make sure that there is no inflight I/O when calling
+ * this (otherwise its just asking for trouble). If the cq polling context
+ * change fails, the old polling context is restored.
+ */
+int ib_change_cq_ctx(struct ib_cq *cq, enum ib_poll_context poll_ctx)
+{
+ enum ib_poll_context old_ctx = cq->poll_ctx;
+ int ret;
+
+ __ib_cq_clear_ctx(cq);
+ cq->poll_ctx = poll_ctx;
+ ret = __ib_cq_set_ctx(cq);
+ if (ret) {
+ cq->poll_ctx = old_ctx;
+ __ib_cq_set_ctx(cq);
+ }
+ return ret;
+}
+EXPORT_SYMBOL(ib_change_cq_ctx);
@@ -3464,6 +3464,7 @@ struct ib_cq *__ib_alloc_cq(struct ib_device *dev, void *private,
void ib_free_cq(struct ib_cq *cq);
int ib_process_cq_direct(struct ib_cq *cq, int budget);
+int ib_change_cq_ctx(struct ib_cq *cq, enum ib_poll_context poll_ctx);
/**
* ib_create_cq - Creates a CQ on the specified device.
Allow cq consumers to modify the cq polling context online. The consumer might want to allocate the cq with softirq/workqueue polling context for async (setup time) I/O, and when completed, switch the polling context to direct polling and get all the interrupts out of the way. One example is nvme-rdma driver that hooks into the block layer infrastructure for a polling queue map for latency sensitive I/O. Every nvmf queue starts with a connect message that is the slow path at setup time, and there is no need for polling (it is actually hurtful). Instead, allocate the polling queue cq with IB_POLL_SOFTIRQ and switch it to IB_POLL_DIRECT where it makes sense. Signed-off-by: Sagi Grimberg <sagi@grimberg.me> --- drivers/infiniband/core/cq.c | 102 ++++++++++++++++++++++++----------- include/rdma/ib_verbs.h | 1 + 2 files changed, 71 insertions(+), 32 deletions(-)