@@ -206,6 +206,7 @@ struct io_submit_state {
bool need_plug;
bool cq_flush;
unsigned short submit_nr;
+ unsigned short inline_completions;
struct blk_plug plug;
};
@@ -465,6 +466,7 @@ enum {
REQ_F_BL_EMPTY_BIT,
REQ_F_BL_NO_RECYCLE_BIT,
REQ_F_BUFFERS_COMMIT_BIT,
+ REQ_F_IGNORE_INLINE_BIT,
/* not a real bit, just to check we're not overflowing the space */
__REQ_F_LAST_BIT,
@@ -541,6 +543,8 @@ enum {
REQ_F_BL_NO_RECYCLE = IO_REQ_FLAG(REQ_F_BL_NO_RECYCLE_BIT),
/* buffer ring head needs incrementing on put */
REQ_F_BUFFERS_COMMIT = IO_REQ_FLAG(REQ_F_BUFFERS_COMMIT_BIT),
+ /* if set, ignore these completions for when waiting on events */
+ REQ_F_IGNORE_INLINE = IO_REQ_FLAG(REQ_F_IGNORE_INLINE_BIT),
};
typedef void (*io_req_tw_func_t)(struct io_kiocb *req, struct io_tw_state *ts);
@@ -554,6 +554,7 @@ struct io_uring_params {
#define IORING_FEAT_REG_REG_RING (1U << 13)
#define IORING_FEAT_RECVSEND_BUNDLE (1U << 14)
#define IORING_FEAT_MIN_TIMEOUT (1U << 15)
+#define IORING_FEAT_IGNORE_INLINE (1U << 16)
/*
* io_uring_register(2) opcodes and arguments
@@ -2239,6 +2239,7 @@ static void io_submit_state_start(struct io_submit_state *state,
state->plug_started = false;
state->need_plug = max_ios > 2;
state->submit_nr = max_ios;
+ state->inline_completions = 0;
/* set only head, no need to init link_last in advance */
state->link.head = NULL;
}
@@ -3285,6 +3286,7 @@ SYSCALL_DEFINE6(io_uring_enter, unsigned int, fd, u32, to_submit,
size_t, argsz)
{
struct io_ring_ctx *ctx;
+ int inline_complete = 0;
struct file *file;
long ret;
@@ -3349,6 +3351,7 @@ SYSCALL_DEFINE6(io_uring_enter, unsigned int, fd, u32, to_submit,
mutex_unlock(&ctx->uring_lock);
goto out;
}
+ inline_complete = ctx->submit_state.inline_completions;
if (flags & IORING_ENTER_GETEVENTS) {
if (ctx->syscall_iopoll)
goto iopoll_locked;
@@ -3386,8 +3389,10 @@ SYSCALL_DEFINE6(io_uring_enter, unsigned int, fd, u32, to_submit,
ret2 = io_get_ext_arg(flags, argp, &ext_arg);
if (likely(!ret2)) {
- min_complete = min(min_complete,
- ctx->cq_entries);
+ if (min_complete > ctx->cq_entries)
+ min_complete = ctx->cq_entries;
+ else
+ min_complete += inline_complete;
ret2 = io_cqring_wait(ctx, min_complete, flags,
&ext_arg);
}
@@ -3674,7 +3679,8 @@ static __cold int io_uring_create(unsigned entries, struct io_uring_params *p,
IORING_FEAT_EXT_ARG | IORING_FEAT_NATIVE_WORKERS |
IORING_FEAT_RSRC_TAGS | IORING_FEAT_CQE_SKIP |
IORING_FEAT_LINKED_FILE | IORING_FEAT_REG_REG_RING |
- IORING_FEAT_RECVSEND_BUNDLE | IORING_FEAT_MIN_TIMEOUT;
+ IORING_FEAT_RECVSEND_BUNDLE | IORING_FEAT_MIN_TIMEOUT |
+ IORING_FEAT_IGNORE_INLINE;
if (copy_to_user(params, p, sizeof(*p))) {
ret = -EFAULT;
@@ -357,6 +357,8 @@ static inline void io_req_complete_defer(struct io_kiocb *req)
lockdep_assert_held(&req->ctx->uring_lock);
wq_list_add_tail(&req->comp_list, &state->compl_reqs);
+ if (req->flags & REQ_F_IGNORE_INLINE)
+ state->inline_completions++;
}
static inline void io_commit_cqring_flush(struct io_ring_ctx *ctx)
io_uring treats all completions the same - they post a completion event, or more, and anyone waiting on event completions will see each event as it gets posted. However, some events may be more interesting that others. For a request and response type model, it's not uncommon to have send/write events that are submitted with a recv/read type of request. While the app does want to see a successful send/write completion eventually, it need not handle it upfront as it would want to do with a recv/read, as it isn't time sensitive. Generally, a send/write completion will just mean that a buffer can get recycled/reused, whereas a recv/read completion needs acting upon (and a response sent). This can be somewhat tricky to handle if many requests and responses are being handled, and the app generally needs to track the number of pending sends/writes to be able to sanely wait on just new incoming recv/read requests. And even with that, an application would still like to see a completion for a short/failed send/write immediately. Add infrastructure to account inline completions, such that they can be deducted from the 'wait_nr' being passed in via a submit_and_wait() type of situation. Inline completions are ones that complete directly inline from submission, such as a send to a socket where there's enough space to accomodate the data being sent. No functional changes in this patch, as no opcode supports setting REQ_F_IGNORE_INLINE just yet. Signed-off-by: Jens Axboe <axboe@kernel.dk> --- include/linux/io_uring_types.h | 4 ++++ include/uapi/linux/io_uring.h | 1 + io_uring/io_uring.c | 12 +++++++++--- io_uring/io_uring.h | 2 ++ 4 files changed, 16 insertions(+), 3 deletions(-)