@@ -47,7 +47,19 @@ static inline const void *io_uring_sqe_cmd(const struct io_uring_sqe *sqe)
#define IO_URING_INVALID_CTX_ID UINT_MAX
+enum io_uring_notifier {
+ IO_URING_NOTIFIER_CTX_EXIT,
+ IO_URING_NOTIFIER_IO_TASK_EXIT,
+};
+
+struct io_uring_notifier_data {
+ unsigned int ctx_id;
+ const struct task_struct *task;
+};
+
#if defined(CONFIG_IO_URING)
+int io_uring_cmd_register_notifier(struct notifier_block *nb);
+void io_uring_cmd_unregister_notifier(struct notifier_block *nb);
int io_uring_cmd_import_fixed(u64 ubuf, unsigned long len, int rw,
struct iov_iter *iter, void *ioucmd);
void io_uring_cmd_done(struct io_uring_cmd *cmd, ssize_t ret, ssize_t res2,
@@ -89,6 +101,13 @@ static inline void io_uring_free(struct task_struct *tsk)
}
int io_uring_cmd_sock(struct io_uring_cmd *cmd, unsigned int issue_flags);
#else
+static inline int io_uring_cmd_register_notifier(struct notifier_block *nb)
+{
+ return 0;
+}
+static inline void io_uring_cmd_unregister_notifier(struct notifier_block *nb)
+{
+}
static inline int io_uring_cmd_import_fixed(u64 ubuf, unsigned long len, int rw,
struct iov_iter *iter, void *ioucmd)
{
@@ -73,6 +73,7 @@
#include <linux/audit.h>
#include <linux/security.h>
#include <asm/shmparam.h>
+#include <linux/notifier.h>
#define CREATE_TRACE_POINTS
#include <trace/events/io_uring.h>
@@ -178,6 +179,22 @@ static struct ctl_table kernel_io_uring_disabled_table[] = {
/* mapping between io_ring_ctx instance and its ctx_id */
static DEFINE_XARRAY_FLAGS(ctx_ids, XA_FLAGS_ALLOC);
+/*
+ * Uring_cmd driver can register to be notified when ctx/io_uring_task
+ * is going away for canceling inflight commands.
+ */
+static struct srcu_notifier_head notifier_chain;
+
+int io_uring_register_notifier(struct notifier_block *nb)
+{
+ return srcu_notifier_chain_register(¬ifier_chain, nb);
+}
+
+void io_uring_unregister_notifier(struct notifier_block *nb)
+{
+ srcu_notifier_chain_unregister(¬ifier_chain, nb);
+}
+
struct sock *io_uring_get_socket(struct file *file)
{
#if defined(CONFIG_UNIX)
@@ -191,6 +208,11 @@ struct sock *io_uring_get_socket(struct file *file)
}
EXPORT_SYMBOL(io_uring_get_socket);
+struct io_ring_ctx *io_uring_id_to_ctx(unsigned int id)
+{
+ return (struct io_ring_ctx *)xa_load(&ctx_ids, id);
+}
+
static inline void io_submit_flush_completions(struct io_ring_ctx *ctx)
{
if (!wq_list_empty(&ctx->submit_state.compl_reqs) ||
@@ -3060,6 +3082,23 @@ static __cold bool io_cancel_ctx_cb(struct io_wq_work *work, void *data)
return req->ctx == data;
}
+static __cold void io_uring_cancel_notify(struct io_ring_ctx *ctx,
+ struct task_struct *task)
+{
+ struct io_uring_notifier_data notifier_data = {
+ .ctx_id = ctx->id,
+ .task = task,
+ };
+ enum io_uring_notifier notifier;
+
+ if (!task)
+ notifier = IO_URING_NOTIFIER_CTX_EXIT;
+ else
+ notifier = IO_URING_NOTIFIER_IO_TASK_EXIT;
+
+ srcu_notifier_call_chain(¬ifier_chain, notifier, ¬ifier_data);
+}
+
static __cold void io_ring_exit_work(struct work_struct *work)
{
struct io_ring_ctx *ctx = container_of(work, struct io_ring_ctx, exit_work);
@@ -3069,6 +3108,8 @@ static __cold void io_ring_exit_work(struct work_struct *work)
struct io_tctx_node *node;
int ret;
+ io_uring_cancel_notify(ctx, NULL);
+
/*
* If we're doing polled IO and end up having requests being
* submitted async (out-of-line), then completions can come in while
@@ -3346,6 +3387,11 @@ __cold void io_uring_cancel_generic(bool cancel_all, struct io_sq_data *sqd)
if (tctx->io_wq)
io_wq_exit_start(tctx->io_wq);
+ if (!cancel_all) {
+ xa_for_each(&tctx->xa, index, node)
+ io_uring_cancel_notify(node->ctx, current);
+ }
+
atomic_inc(&tctx->in_cancel);
do {
bool loop = false;
@@ -4695,6 +4741,8 @@ static int __init io_uring_init(void)
register_sysctl_init("kernel", kernel_io_uring_disabled_table);
#endif
+ srcu_init_notifier_head(¬ifier_chain);
+
return 0;
};
__initcall(io_uring_init);
@@ -38,6 +38,10 @@ enum {
IOU_STOP_MULTISHOT = -ECANCELED,
};
+struct io_ring_ctx *io_uring_id_to_ctx(unsigned int id);
+int io_uring_register_notifier(struct notifier_block *nb);
+void io_uring_unregister_notifier(struct notifier_block *nb);
+
bool io_cqe_cache_refill(struct io_ring_ctx *ctx, bool overflow);
void io_req_cqe_overflow(struct io_kiocb *req);
int io_run_task_work_sig(struct io_ring_ctx *ctx);
@@ -192,3 +192,15 @@ int io_uring_cmd_sock(struct io_uring_cmd *cmd, unsigned int issue_flags)
}
}
EXPORT_SYMBOL_GPL(io_uring_cmd_sock);
+
+int io_uring_cmd_register_notifier(struct notifier_block *nb)
+{
+ return io_uring_register_notifier(nb);
+}
+EXPORT_SYMBOL_GPL(io_uring_cmd_register_notifier);
+
+void io_uring_cmd_unregister_notifier(struct notifier_block *nb)
+{
+ io_uring_unregister_notifier(nb);
+}
+EXPORT_SYMBOL_GPL(io_uring_cmd_unregister_notifier);
Notifier callback is registered by driver to get notified, so far only for uring_cmd based driver. With this notifier, driver can cancel in-flight command when ctx is being released or io task is exiting. The main use case is ublk(or probably fuse with uring cmd support) in which uring command may never complete, so driver has to cancel this command when io task is exiting or ctx is releasing, otherwise __io_uring_cancel() may wait forever because of inflight commands. Signed-off-by: Ming Lei <ming.lei@redhat.com> --- include/linux/io_uring.h | 19 ++++++++++++++++ io_uring/io_uring.c | 48 ++++++++++++++++++++++++++++++++++++++++ io_uring/io_uring.h | 4 ++++ io_uring/uring_cmd.c | 12 ++++++++++ 4 files changed, 83 insertions(+)