| Message ID | 20230112144411.2624698-2-leitao@debian.org (mailing list archive) |
|---|---|
| State | New |
| Headers | show |
| Series | [1/2] io_uring: Rename struct io_op_def | expand |
Breno Leitao <leitao@debian.org> writes: > This patch removes some "cold" fields from `struct io_issue_def`. > > The plan is to keep only highly used fields into `struct io_issue_def`, so, > it may be hot in the cache. The hot fields are basically all the bitfields > and the callback functions for .issue and .prep. > > The other less frequently used fields are now located in a secondary and > cold struct, called `io_cold_def`. > > This is the size for the structs: > > Before: io_issue_def = 56 bytes > After: io_issue_def = 24 bytes; io_cold_def = 40 bytes Does this change have an observable impact in run time? Did it show a significant decrease of dcache misses? > > Signed-off-by: Breno Leitao <leitao@debian.org> > --- > io_uring/io_uring.c | 15 +- > io_uring/opdef.c | 327 ++++++++++++++++++++++++++++++-------------- > io_uring/opdef.h | 9 +- > io_uring/rw.c | 2 +- > 4 files changed, 238 insertions(+), 115 deletions(-) > > diff --git a/io_uring/io_uring.c b/io_uring/io_uring.c > index ac7868ec9be2..0be66b026a7f 100644 > --- a/io_uring/io_uring.c > +++ b/io_uring/io_uring.c > @@ -980,7 +980,7 @@ void io_req_complete_post(struct io_kiocb *req, unsigned issue_flags) > void io_req_defer_failed(struct io_kiocb *req, s32 res) > __must_hold(&ctx->uring_lock) > { > - const struct io_issue_def *def = &io_issue_defs[req->opcode]; > + const struct io_cold_def *def = &io_cold_defs[req->opcode]; > > lockdep_assert_held(&req->ctx->uring_lock); > > @@ -1708,8 +1708,8 @@ unsigned int io_file_get_flags(struct file *file) > > bool io_alloc_async_data(struct io_kiocb *req) > { > - WARN_ON_ONCE(!io_issue_defs[req->opcode].async_size); > - req->async_data = kmalloc(io_issue_defs[req->opcode].async_size, GFP_KERNEL); > + WARN_ON_ONCE(!io_cold_defs[req->opcode].async_size); > + req->async_data = kmalloc(io_cold_defs[req->opcode].async_size, GFP_KERNEL); > if (req->async_data) { > req->flags |= REQ_F_ASYNC_DATA; > return false; > @@ -1719,20 +1719,21 @@ bool io_alloc_async_data(struct io_kiocb *req) > > int io_req_prep_async(struct io_kiocb *req) > { > + const struct io_cold_def *cdef = &io_cold_defs[req->opcode]; > const struct io_issue_def *def = &io_issue_defs[req->opcode]; > > /* assign early for deferred execution for non-fixed file */ > if (def->needs_file && !(req->flags & REQ_F_FIXED_FILE)) > req->file = io_file_get_normal(req, req->cqe.fd); > - if (!def->prep_async) > + if (!cdef->prep_async) > return 0; > if (WARN_ON_ONCE(req_has_async_data(req))) > return -EFAULT; > - if (!io_issue_defs[req->opcode].manual_alloc) { > + if (!def->manual_alloc) { > if (io_alloc_async_data(req)) > return -EAGAIN; > } > - return def->prep_async(req); > + return cdef->prep_async(req); > } > > static u32 io_get_sequence(struct io_kiocb *req) > @@ -1801,7 +1802,7 @@ static void io_clean_op(struct io_kiocb *req) > } > > if (req->flags & REQ_F_NEED_CLEANUP) { > - const struct io_issue_def *def = &io_issue_defs[req->opcode]; > + const struct io_cold_def *def = &io_cold_defs[req->opcode]; > > if (def->cleanup) > def->cleanup(req); > diff --git a/io_uring/opdef.c b/io_uring/opdef.c > index 3c95e70a625e..5238ecd7af6a 100644 > --- a/io_uring/opdef.c > +++ b/io_uring/opdef.c > @@ -50,7 +50,6 @@ const struct io_issue_def io_issue_defs[] = { > [IORING_OP_NOP] = { > .audit_skip = 1, > .iopoll = 1, > - .name = "NOP", > .prep = io_nop_prep, > .issue = io_nop, > }, > @@ -64,13 +63,8 @@ const struct io_issue_def io_issue_defs[] = { > .ioprio = 1, > .iopoll = 1, > .iopoll_queue = 1, > - .async_size = sizeof(struct io_async_rw), > - .name = "READV", > .prep = io_prep_rw, > .issue = io_read, > - .prep_async = io_readv_prep_async, > - .cleanup = io_readv_writev_cleanup, > - .fail = io_rw_fail, > }, > [IORING_OP_WRITEV] = { > .needs_file = 1, > @@ -82,18 +76,12 @@ const struct io_issue_def io_issue_defs[] = { > .ioprio = 1, > .iopoll = 1, > .iopoll_queue = 1, > - .async_size = sizeof(struct io_async_rw), > - .name = "WRITEV", > .prep = io_prep_rw, > .issue = io_write, > - .prep_async = io_writev_prep_async, > - .cleanup = io_readv_writev_cleanup, > - .fail = io_rw_fail, > }, > [IORING_OP_FSYNC] = { > .needs_file = 1, > .audit_skip = 1, > - .name = "FSYNC", > .prep = io_fsync_prep, > .issue = io_fsync, > }, > @@ -106,11 +94,8 @@ const struct io_issue_def io_issue_defs[] = { > .ioprio = 1, > .iopoll = 1, > .iopoll_queue = 1, > - .async_size = sizeof(struct io_async_rw), > - .name = "READ_FIXED", > .prep = io_prep_rw, > .issue = io_read, > - .fail = io_rw_fail, > }, > [IORING_OP_WRITE_FIXED] = { > .needs_file = 1, > @@ -122,30 +107,24 @@ const struct io_issue_def io_issue_defs[] = { > .ioprio = 1, > .iopoll = 1, > .iopoll_queue = 1, > - .async_size = sizeof(struct io_async_rw), > - .name = "WRITE_FIXED", > .prep = io_prep_rw, > .issue = io_write, > - .fail = io_rw_fail, > }, > [IORING_OP_POLL_ADD] = { > .needs_file = 1, > .unbound_nonreg_file = 1, > .audit_skip = 1, > - .name = "POLL_ADD", > .prep = io_poll_add_prep, > .issue = io_poll_add, > }, > [IORING_OP_POLL_REMOVE] = { > .audit_skip = 1, > - .name = "POLL_REMOVE", > .prep = io_poll_remove_prep, > .issue = io_poll_remove, > }, > [IORING_OP_SYNC_FILE_RANGE] = { > .needs_file = 1, > .audit_skip = 1, > - .name = "SYNC_FILE_RANGE", > .prep = io_sfr_prep, > .issue = io_sync_file_range, > }, > @@ -155,14 +134,9 @@ const struct io_issue_def io_issue_defs[] = { > .pollout = 1, > .ioprio = 1, > .manual_alloc = 1, > - .name = "SENDMSG", > #if defined(CONFIG_NET) > - .async_size = sizeof(struct io_async_msghdr), > .prep = io_sendmsg_prep, > .issue = io_sendmsg, > - .prep_async = io_sendmsg_prep_async, > - .cleanup = io_sendmsg_recvmsg_cleanup, > - .fail = io_sendrecv_fail, > #else > .prep = io_eopnotsupp_prep, > #endif > @@ -174,29 +148,21 @@ const struct io_issue_def io_issue_defs[] = { > .buffer_select = 1, > .ioprio = 1, > .manual_alloc = 1, > - .name = "RECVMSG", > #if defined(CONFIG_NET) > - .async_size = sizeof(struct io_async_msghdr), > .prep = io_recvmsg_prep, > .issue = io_recvmsg, > - .prep_async = io_recvmsg_prep_async, > - .cleanup = io_sendmsg_recvmsg_cleanup, > - .fail = io_sendrecv_fail, > #else > .prep = io_eopnotsupp_prep, > #endif > }, > [IORING_OP_TIMEOUT] = { > .audit_skip = 1, > - .async_size = sizeof(struct io_timeout_data), > - .name = "TIMEOUT", > .prep = io_timeout_prep, > .issue = io_timeout, > }, > [IORING_OP_TIMEOUT_REMOVE] = { > /* used by timeout updates' prep() */ > .audit_skip = 1, > - .name = "TIMEOUT_REMOVE", > .prep = io_timeout_remove_prep, > .issue = io_timeout_remove, > }, > @@ -206,7 +172,6 @@ const struct io_issue_def io_issue_defs[] = { > .pollin = 1, > .poll_exclusive = 1, > .ioprio = 1, /* used for flags */ > - .name = "ACCEPT", > #if defined(CONFIG_NET) > .prep = io_accept_prep, > .issue = io_accept, > @@ -216,14 +181,11 @@ const struct io_issue_def io_issue_defs[] = { > }, > [IORING_OP_ASYNC_CANCEL] = { > .audit_skip = 1, > - .name = "ASYNC_CANCEL", > .prep = io_async_cancel_prep, > .issue = io_async_cancel, > }, > [IORING_OP_LINK_TIMEOUT] = { > .audit_skip = 1, > - .async_size = sizeof(struct io_timeout_data), > - .name = "LINK_TIMEOUT", > .prep = io_link_timeout_prep, > .issue = io_no_issue, > }, > @@ -231,46 +193,36 @@ const struct io_issue_def io_issue_defs[] = { > .needs_file = 1, > .unbound_nonreg_file = 1, > .pollout = 1, > - .name = "CONNECT", > #if defined(CONFIG_NET) > - .async_size = sizeof(struct io_async_connect), > .prep = io_connect_prep, > .issue = io_connect, > - .prep_async = io_connect_prep_async, > #else > .prep = io_eopnotsupp_prep, > #endif > }, > [IORING_OP_FALLOCATE] = { > .needs_file = 1, > - .name = "FALLOCATE", > .prep = io_fallocate_prep, > .issue = io_fallocate, > }, > [IORING_OP_OPENAT] = { > - .name = "OPENAT", > .prep = io_openat_prep, > .issue = io_openat, > - .cleanup = io_open_cleanup, > }, > [IORING_OP_CLOSE] = { > - .name = "CLOSE", > .prep = io_close_prep, > .issue = io_close, > }, > [IORING_OP_FILES_UPDATE] = { > .audit_skip = 1, > .iopoll = 1, > - .name = "FILES_UPDATE", > .prep = io_files_update_prep, > .issue = io_files_update, > }, > [IORING_OP_STATX] = { > .audit_skip = 1, > - .name = "STATX", > .prep = io_statx_prep, > .issue = io_statx, > - .cleanup = io_statx_cleanup, > }, > [IORING_OP_READ] = { > .needs_file = 1, > @@ -282,11 +234,8 @@ const struct io_issue_def io_issue_defs[] = { > .ioprio = 1, > .iopoll = 1, > .iopoll_queue = 1, > - .async_size = sizeof(struct io_async_rw), > - .name = "READ", > .prep = io_prep_rw, > .issue = io_read, > - .fail = io_rw_fail, > }, > [IORING_OP_WRITE] = { > .needs_file = 1, > @@ -298,21 +247,16 @@ const struct io_issue_def io_issue_defs[] = { > .ioprio = 1, > .iopoll = 1, > .iopoll_queue = 1, > - .async_size = sizeof(struct io_async_rw), > - .name = "WRITE", > .prep = io_prep_rw, > .issue = io_write, > - .fail = io_rw_fail, > }, > [IORING_OP_FADVISE] = { > .needs_file = 1, > .audit_skip = 1, > - .name = "FADVISE", > .prep = io_fadvise_prep, > .issue = io_fadvise, > }, > [IORING_OP_MADVISE] = { > - .name = "MADVISE", > .prep = io_madvise_prep, > .issue = io_madvise, > }, > @@ -323,13 +267,9 @@ const struct io_issue_def io_issue_defs[] = { > .audit_skip = 1, > .ioprio = 1, > .manual_alloc = 1, > - .name = "SEND", > #if defined(CONFIG_NET) > - .async_size = sizeof(struct io_async_msghdr), > .prep = io_sendmsg_prep, > .issue = io_send, > - .fail = io_sendrecv_fail, > - .prep_async = io_send_prep_async, > #else > .prep = io_eopnotsupp_prep, > #endif > @@ -341,25 +281,20 @@ const struct io_issue_def io_issue_defs[] = { > .buffer_select = 1, > .audit_skip = 1, > .ioprio = 1, > - .name = "RECV", > #if defined(CONFIG_NET) > .prep = io_recvmsg_prep, > .issue = io_recv, > - .fail = io_sendrecv_fail, > #else > .prep = io_eopnotsupp_prep, > #endif > }, > [IORING_OP_OPENAT2] = { > - .name = "OPENAT2", > .prep = io_openat2_prep, > .issue = io_openat2, > - .cleanup = io_open_cleanup, > }, > [IORING_OP_EPOLL_CTL] = { > .unbound_nonreg_file = 1, > .audit_skip = 1, > - .name = "EPOLL", > #if defined(CONFIG_EPOLL) > .prep = io_epoll_ctl_prep, > .issue = io_epoll_ctl, > @@ -372,21 +307,18 @@ const struct io_issue_def io_issue_defs[] = { > .hash_reg_file = 1, > .unbound_nonreg_file = 1, > .audit_skip = 1, > - .name = "SPLICE", > .prep = io_splice_prep, > .issue = io_splice, > }, > [IORING_OP_PROVIDE_BUFFERS] = { > .audit_skip = 1, > .iopoll = 1, > - .name = "PROVIDE_BUFFERS", > .prep = io_provide_buffers_prep, > .issue = io_provide_buffers, > }, > [IORING_OP_REMOVE_BUFFERS] = { > .audit_skip = 1, > .iopoll = 1, > - .name = "REMOVE_BUFFERS", > .prep = io_remove_buffers_prep, > .issue = io_remove_buffers, > }, > @@ -395,13 +327,11 @@ const struct io_issue_def io_issue_defs[] = { > .hash_reg_file = 1, > .unbound_nonreg_file = 1, > .audit_skip = 1, > - .name = "TEE", > .prep = io_tee_prep, > .issue = io_tee, > }, > [IORING_OP_SHUTDOWN] = { > .needs_file = 1, > - .name = "SHUTDOWN", > #if defined(CONFIG_NET) > .prep = io_shutdown_prep, > .issue = io_shutdown, > @@ -410,72 +340,51 @@ const struct io_issue_def io_issue_defs[] = { > #endif > }, > [IORING_OP_RENAMEAT] = { > - .name = "RENAMEAT", > .prep = io_renameat_prep, > .issue = io_renameat, > - .cleanup = io_renameat_cleanup, > }, > [IORING_OP_UNLINKAT] = { > - .name = "UNLINKAT", > .prep = io_unlinkat_prep, > .issue = io_unlinkat, > - .cleanup = io_unlinkat_cleanup, > }, > [IORING_OP_MKDIRAT] = { > - .name = "MKDIRAT", > .prep = io_mkdirat_prep, > .issue = io_mkdirat, > - .cleanup = io_mkdirat_cleanup, > }, > [IORING_OP_SYMLINKAT] = { > - .name = "SYMLINKAT", > .prep = io_symlinkat_prep, > .issue = io_symlinkat, > - .cleanup = io_link_cleanup, > }, > [IORING_OP_LINKAT] = { > - .name = "LINKAT", > .prep = io_linkat_prep, > .issue = io_linkat, > - .cleanup = io_link_cleanup, > }, > [IORING_OP_MSG_RING] = { > .needs_file = 1, > .iopoll = 1, > - .name = "MSG_RING", > .prep = io_msg_ring_prep, > .issue = io_msg_ring, > - .cleanup = io_msg_ring_cleanup, > }, > [IORING_OP_FSETXATTR] = { > .needs_file = 1, > - .name = "FSETXATTR", > .prep = io_fsetxattr_prep, > .issue = io_fsetxattr, > - .cleanup = io_xattr_cleanup, > }, > [IORING_OP_SETXATTR] = { > - .name = "SETXATTR", > .prep = io_setxattr_prep, > .issue = io_setxattr, > - .cleanup = io_xattr_cleanup, > }, > [IORING_OP_FGETXATTR] = { > .needs_file = 1, > - .name = "FGETXATTR", > .prep = io_fgetxattr_prep, > .issue = io_fgetxattr, > - .cleanup = io_xattr_cleanup, > }, > [IORING_OP_GETXATTR] = { > - .name = "GETXATTR", > .prep = io_getxattr_prep, > .issue = io_getxattr, > - .cleanup = io_xattr_cleanup, > }, > [IORING_OP_SOCKET] = { > .audit_skip = 1, > - .name = "SOCKET", > #if defined(CONFIG_NET) > .prep = io_socket_prep, > .issue = io_socket, > @@ -486,16 +395,12 @@ const struct io_issue_def io_issue_defs[] = { > [IORING_OP_URING_CMD] = { > .needs_file = 1, > .plug = 1, > - .name = "URING_CMD", > .iopoll = 1, > .iopoll_queue = 1, > - .async_size = uring_cmd_pdu_size(1), > .prep = io_uring_cmd_prep, > .issue = io_uring_cmd, > - .prep_async = io_uring_cmd_prep_async, > }, > [IORING_OP_SEND_ZC] = { > - .name = "SEND_ZC", > .needs_file = 1, > .unbound_nonreg_file = 1, > .pollout = 1, > @@ -503,32 +408,243 @@ const struct io_issue_def io_issue_defs[] = { > .ioprio = 1, > .manual_alloc = 1, > #if defined(CONFIG_NET) > - .async_size = sizeof(struct io_async_msghdr), > .prep = io_send_zc_prep, > .issue = io_send_zc, > - .prep_async = io_send_prep_async, > - .cleanup = io_send_zc_cleanup, > - .fail = io_sendrecv_fail, > #else > .prep = io_eopnotsupp_prep, > #endif > }, > [IORING_OP_SENDMSG_ZC] = { > - .name = "SENDMSG_ZC", > .needs_file = 1, > .unbound_nonreg_file = 1, > .pollout = 1, > .ioprio = 1, > .manual_alloc = 1, > #if defined(CONFIG_NET) > - .async_size = sizeof(struct io_async_msghdr), > .prep = io_send_zc_prep, > .issue = io_sendmsg_zc, > +#else > + .prep = io_eopnotsupp_prep, > +#endif > + }, > +}; > + > + > +const struct io_cold_def io_cold_defs[] = { > + [IORING_OP_NOP] = { > + .name = "NOP", > + }, > + [IORING_OP_READV] = { > + .async_size = sizeof(struct io_async_rw), > + .name = "READV", > + .prep_async = io_readv_prep_async, > + .cleanup = io_readv_writev_cleanup, > + .fail = io_rw_fail, > + }, > + [IORING_OP_WRITEV] = { > + .async_size = sizeof(struct io_async_rw), > + .name = "WRITEV", > + .prep_async = io_writev_prep_async, > + .cleanup = io_readv_writev_cleanup, > + .fail = io_rw_fail, > + }, > + [IORING_OP_FSYNC] = { > + .name = "FSYNC", > + }, > + [IORING_OP_READ_FIXED] = { > + .async_size = sizeof(struct io_async_rw), > + .name = "READ_FIXED", > + .fail = io_rw_fail, > + }, > + [IORING_OP_WRITE_FIXED] = { > + .async_size = sizeof(struct io_async_rw), > + .name = "WRITE_FIXED", > + .fail = io_rw_fail, > + }, > + [IORING_OP_POLL_ADD] = { > + .name = "POLL_ADD", > + }, > + [IORING_OP_POLL_REMOVE] = { > + .name = "POLL_REMOVE", > + }, > + [IORING_OP_SYNC_FILE_RANGE] = { > + .name = "SYNC_FILE_RANGE", > + }, > + [IORING_OP_SENDMSG] = { > + .name = "SENDMSG", > +#if defined(CONFIG_NET) > + .async_size = sizeof(struct io_async_msghdr), > + .prep_async = io_sendmsg_prep_async, > + .cleanup = io_sendmsg_recvmsg_cleanup, > + .fail = io_sendrecv_fail, > +#endif > + }, > + [IORING_OP_RECVMSG] = { > + .name = "RECVMSG", > +#if defined(CONFIG_NET) > + .async_size = sizeof(struct io_async_msghdr), > + .prep_async = io_recvmsg_prep_async, > + .cleanup = io_sendmsg_recvmsg_cleanup, > + .fail = io_sendrecv_fail, > +#endif > + }, > + [IORING_OP_TIMEOUT] = { > + .async_size = sizeof(struct io_timeout_data), > + .name = "TIMEOUT", > + }, > + [IORING_OP_TIMEOUT_REMOVE] = { > + .name = "TIMEOUT_REMOVE", > + }, > + [IORING_OP_ACCEPT] = { > + .name = "ACCEPT", > + }, > + [IORING_OP_ASYNC_CANCEL] = { > + .name = "ASYNC_CANCEL", > + }, > + [IORING_OP_LINK_TIMEOUT] = { > + .async_size = sizeof(struct io_timeout_data), > + .name = "LINK_TIMEOUT", > + }, > + [IORING_OP_CONNECT] = { > + .name = "CONNECT", > +#if defined(CONFIG_NET) > + .async_size = sizeof(struct io_async_connect), > + .prep_async = io_connect_prep_async, > +#endif > + }, > + [IORING_OP_FALLOCATE] = { > + .name = "FALLOCATE", > + }, > + [IORING_OP_OPENAT] = { > + .name = "OPENAT", > + .cleanup = io_open_cleanup, > + }, > + [IORING_OP_CLOSE] = { > + .name = "CLOSE", > + }, > + [IORING_OP_FILES_UPDATE] = { > + .name = "FILES_UPDATE", > + }, > + [IORING_OP_STATX] = { > + .name = "STATX", > + .cleanup = io_statx_cleanup, > + }, > + [IORING_OP_READ] = { > + .async_size = sizeof(struct io_async_rw), > + .name = "READ", > + .fail = io_rw_fail, > + }, > + [IORING_OP_WRITE] = { > + .async_size = sizeof(struct io_async_rw), > + .name = "WRITE", > + .fail = io_rw_fail, > + }, > + [IORING_OP_FADVISE] = { > + .name = "FADVISE", > + }, > + [IORING_OP_MADVISE] = { > + .name = "MADVISE", > + }, > + [IORING_OP_SEND] = { > + .name = "SEND", > +#if defined(CONFIG_NET) > + .async_size = sizeof(struct io_async_msghdr), > + .fail = io_sendrecv_fail, > + .prep_async = io_send_prep_async, > +#endif > + }, > + [IORING_OP_RECV] = { > + .name = "RECV", > +#if defined(CONFIG_NET) > + .fail = io_sendrecv_fail, > +#endif > + }, > + [IORING_OP_OPENAT2] = { > + .name = "OPENAT2", > + .cleanup = io_open_cleanup, > + }, > + [IORING_OP_EPOLL_CTL] = { > + .name = "EPOLL", > + }, > + [IORING_OP_SPLICE] = { > + .name = "SPLICE", > + }, > + [IORING_OP_PROVIDE_BUFFERS] = { > + .name = "PROVIDE_BUFFERS", > + }, > + [IORING_OP_REMOVE_BUFFERS] = { > + .name = "REMOVE_BUFFERS", > + }, > + [IORING_OP_TEE] = { > + .name = "TEE", > + }, > + [IORING_OP_SHUTDOWN] = { > + .name = "SHUTDOWN", > + }, > + [IORING_OP_RENAMEAT] = { > + .name = "RENAMEAT", > + .cleanup = io_renameat_cleanup, > + }, > + [IORING_OP_UNLINKAT] = { > + .name = "UNLINKAT", > + .cleanup = io_unlinkat_cleanup, > + }, > + [IORING_OP_MKDIRAT] = { > + .name = "MKDIRAT", > + .cleanup = io_mkdirat_cleanup, > + }, > + [IORING_OP_SYMLINKAT] = { > + .name = "SYMLINKAT", > + .cleanup = io_link_cleanup, > + }, > + [IORING_OP_LINKAT] = { > + .name = "LINKAT", > + .cleanup = io_link_cleanup, > + }, > + [IORING_OP_MSG_RING] = { > + .name = "MSG_RING", > + .cleanup = io_msg_ring_cleanup, > + }, > + [IORING_OP_FSETXATTR] = { > + .name = "FSETXATTR", > + .cleanup = io_xattr_cleanup, > + }, > + [IORING_OP_SETXATTR] = { > + .name = "SETXATTR", > + .cleanup = io_xattr_cleanup, > + }, > + [IORING_OP_FGETXATTR] = { > + .name = "FGETXATTR", > + .cleanup = io_xattr_cleanup, > + }, > + [IORING_OP_GETXATTR] = { > + .name = "GETXATTR", > + .cleanup = io_xattr_cleanup, > + }, > + [IORING_OP_SOCKET] = { > + .name = "SOCKET", > + }, > + [IORING_OP_URING_CMD] = { > + .name = "URING_CMD", > + .async_size = uring_cmd_pdu_size(1), > + .prep_async = io_uring_cmd_prep_async, > + }, > + [IORING_OP_SEND_ZC] = { > + .name = "SEND_ZC", > +#if defined(CONFIG_NET) > + .async_size = sizeof(struct io_async_msghdr), > + .prep_async = io_send_prep_async, > + .cleanup = io_send_zc_cleanup, > + .fail = io_sendrecv_fail, > +#endif > + }, > + [IORING_OP_SENDMSG_ZC] = { > + .name = "SENDMSG_ZC", > +#if defined(CONFIG_NET) > + .async_size = sizeof(struct io_async_msghdr), > .prep_async = io_sendmsg_prep_async, > .cleanup = io_send_zc_cleanup, > .fail = io_sendrecv_fail, > -#else > - .prep = io_eopnotsupp_prep, > #endif > }, > }; > @@ -536,7 +652,7 @@ const struct io_issue_def io_issue_defs[] = { > const char *io_uring_get_opcode(u8 opcode) > { > if (opcode < IORING_OP_LAST) > - return io_issue_defs[opcode].name; > + return io_cold_defs[opcode].name; > return "INVALID"; > } > > @@ -544,12 +660,13 @@ void __init io_uring_optable_init(void) > { > int i; > > + BUILD_BUG_ON(ARRAY_SIZE(io_cold_defs) != IORING_OP_LAST); > BUILD_BUG_ON(ARRAY_SIZE(io_issue_defs) != IORING_OP_LAST); > > for (i = 0; i < ARRAY_SIZE(io_issue_defs); i++) { > BUG_ON(!io_issue_defs[i].prep); > if (io_issue_defs[i].prep != io_eopnotsupp_prep) > BUG_ON(!io_issue_defs[i].issue); > - WARN_ON_ONCE(!io_issue_defs[i].name); > + WARN_ON_ONCE(!io_cold_defs[i].name); > } > } > diff --git a/io_uring/opdef.h b/io_uring/opdef.h > index d718e2ab1ff7..c22c8696e749 100644 > --- a/io_uring/opdef.h > +++ b/io_uring/opdef.h > @@ -29,19 +29,24 @@ struct io_issue_def { > unsigned iopoll_queue : 1; > /* opcode specific path will handle ->async_data allocation if needed */ > unsigned manual_alloc : 1; > + > + int (*issue)(struct io_kiocb *, unsigned int); > + int (*prep)(struct io_kiocb *, const struct io_uring_sqe *); > +}; > + > +struct io_cold_def { > /* size of async data needed, if any */ > unsigned short async_size; > > const char *name; > > - int (*prep)(struct io_kiocb *, const struct io_uring_sqe *); > - int (*issue)(struct io_kiocb *, unsigned int); > int (*prep_async)(struct io_kiocb *); > void (*cleanup)(struct io_kiocb *); > void (*fail)(struct io_kiocb *); > }; > > extern const struct io_issue_def io_issue_defs[]; > +extern const struct io_cold_def io_cold_defs[]; > > void io_uring_optable_init(void); > #endif > diff --git a/io_uring/rw.c b/io_uring/rw.c > index 54b44b9b736c..a8a2eb7ee27a 100644 > --- a/io_uring/rw.c > +++ b/io_uring/rw.c > @@ -516,7 +516,7 @@ static void io_req_map_rw(struct io_kiocb *req, const struct iovec *iovec, > static int io_setup_async_rw(struct io_kiocb *req, const struct iovec *iovec, > struct io_rw_state *s, bool force) > { > - if (!force && !io_issue_defs[req->opcode].prep_async) > + if (!force && !io_cold_defs[req->opcode].prep_async) > return 0; > if (!req_has_async_data(req)) { > struct io_async_rw *iorw;
On Thu, Jan 12, 2023 at 05:35:22PM -0300, Gabriel Krisman Bertazi wrote: > Breno Leitao <leitao@debian.org> writes: > > > This patch removes some "cold" fields from `struct io_issue_def`. > > > > The plan is to keep only highly used fields into `struct io_issue_def`, so, > > it may be hot in the cache. The hot fields are basically all the bitfields > > and the callback functions for .issue and .prep. > > > > The other less frequently used fields are now located in a secondary and > > cold struct, called `io_cold_def`. > > > > This is the size for the structs: > > > > Before: io_issue_def = 56 bytes > > After: io_issue_def = 24 bytes; io_cold_def = 40 bytes > > Does this change have an observable impact in run time? Did it show > a significant decrease of dcache misses? I haven't tested it. I expect it might be hard to came up with such test. A possible test might be running io_uring heavy tests, while adding enough memory pressure. Doing this in two different instant (A/B test), might be a unpredicable and the error deviation might hide the benefit.
On 1/16/23 3:52 AM, Breno Leitao wrote: > On Thu, Jan 12, 2023 at 05:35:22PM -0300, Gabriel Krisman Bertazi wrote: >> Breno Leitao <leitao@debian.org> writes: >> >>> This patch removes some "cold" fields from `struct io_issue_def`. >>> >>> The plan is to keep only highly used fields into `struct io_issue_def`, so, >>> it may be hot in the cache. The hot fields are basically all the bitfields >>> and the callback functions for .issue and .prep. >>> >>> The other less frequently used fields are now located in a secondary and >>> cold struct, called `io_cold_def`. >>> >>> This is the size for the structs: >>> >>> Before: io_issue_def = 56 bytes >>> After: io_issue_def = 24 bytes; io_cold_def = 40 bytes >> >> Does this change have an observable impact in run time? Did it show >> a significant decrease of dcache misses? > > I haven't tested it. I expect it might be hard to came up with such test. > > A possible test might be running io_uring heavy tests, while adding > enough memory pressure. Doing this in two different instant (A/B test), > might be a unpredicable and the error deviation might hide the benefit. I think what you'd want is two (or more) io_uring ops being really busy and measuring dcache pressure while running that test. I don't think this is very feasible to accurately measure, and I also don't think that is an issue. The split into hot/cold parts of the op definitions is obviously a good idea. For ideal setups, we'll never be using the cold part at all, and having a smaller op definition for the fast path is always going to be helpful.
On 1/12/23 14:44, Breno Leitao wrote: > This patch removes some "cold" fields from `struct io_issue_def`. > > The plan is to keep only highly used fields into `struct io_issue_def`, so, > it may be hot in the cache. The hot fields are basically all the bitfields > and the callback functions for .issue and .prep. Reviewed-by: Pavel Begunkov <asml.silence@gmail.com> > The other less frequently used fields are now located in a secondary and > cold struct, called `io_cold_def`. > > This is the size for the structs: > > Before: io_issue_def = 56 bytes > After: io_issue_def = 24 bytes; io_cold_def = 40 bytes > > Signed-off-by: Breno Leitao <leitao@debian.org> > --- > io_uring/io_uring.c | 15 +- > io_uring/opdef.c | 327 ++++++++++++++++++++++++++++++-------------- > io_uring/opdef.h | 9 +- > io_uring/rw.c | 2 +- > 4 files changed, 238 insertions(+), 115 deletions(-) > > diff --git a/io_uring/io_uring.c b/io_uring/io_uring.c > index ac7868ec9be2..0be66b026a7f 100644 > --- a/io_uring/io_uring.c > +++ b/io_uring/io_uring.c > @@ -980,7 +980,7 @@ void io_req_complete_post(struct io_kiocb *req, unsigned issue_flags) > void io_req_defer_failed(struct io_kiocb *req, s32 res) > __must_hold(&ctx->uring_lock) > { > - const struct io_issue_def *def = &io_issue_defs[req->opcode]; > + const struct io_cold_def *def = &io_cold_defs[req->opcode]; > > lockdep_assert_held(&req->ctx->uring_lock); > > @@ -1708,8 +1708,8 @@ unsigned int io_file_get_flags(struct file *file) > > bool io_alloc_async_data(struct io_kiocb *req) > { > - WARN_ON_ONCE(!io_issue_defs[req->opcode].async_size); > - req->async_data = kmalloc(io_issue_defs[req->opcode].async_size, GFP_KERNEL); > + WARN_ON_ONCE(!io_cold_defs[req->opcode].async_size); > + req->async_data = kmalloc(io_cold_defs[req->opcode].async_size, GFP_KERNEL); > if (req->async_data) { > req->flags |= REQ_F_ASYNC_DATA; > return false; > @@ -1719,20 +1719,21 @@ bool io_alloc_async_data(struct io_kiocb *req) > > int io_req_prep_async(struct io_kiocb *req) > { > + const struct io_cold_def *cdef = &io_cold_defs[req->opcode]; > const struct io_issue_def *def = &io_issue_defs[req->opcode]; > > /* assign early for deferred execution for non-fixed file */ > if (def->needs_file && !(req->flags & REQ_F_FIXED_FILE)) > req->file = io_file_get_normal(req, req->cqe.fd); > - if (!def->prep_async) > + if (!cdef->prep_async) > return 0; > if (WARN_ON_ONCE(req_has_async_data(req))) > return -EFAULT; > - if (!io_issue_defs[req->opcode].manual_alloc) { > + if (!def->manual_alloc) { > if (io_alloc_async_data(req)) > return -EAGAIN; > } > - return def->prep_async(req); > + return cdef->prep_async(req); > } > > static u32 io_get_sequence(struct io_kiocb *req) > @@ -1801,7 +1802,7 @@ static void io_clean_op(struct io_kiocb *req) > } > > if (req->flags & REQ_F_NEED_CLEANUP) { > - const struct io_issue_def *def = &io_issue_defs[req->opcode]; > + const struct io_cold_def *def = &io_cold_defs[req->opcode]; > > if (def->cleanup) > def->cleanup(req); > diff --git a/io_uring/opdef.c b/io_uring/opdef.c > index 3c95e70a625e..5238ecd7af6a 100644 > --- a/io_uring/opdef.c > +++ b/io_uring/opdef.c > @@ -50,7 +50,6 @@ const struct io_issue_def io_issue_defs[] = { > [IORING_OP_NOP] = { > .audit_skip = 1, > .iopoll = 1, > - .name = "NOP", > .prep = io_nop_prep, > .issue = io_nop, > }, > @@ -64,13 +63,8 @@ const struct io_issue_def io_issue_defs[] = { > .ioprio = 1, > .iopoll = 1, > .iopoll_queue = 1, > - .async_size = sizeof(struct io_async_rw), > - .name = "READV", > .prep = io_prep_rw, > .issue = io_read, > - .prep_async = io_readv_prep_async, > - .cleanup = io_readv_writev_cleanup, > - .fail = io_rw_fail, > }, > [IORING_OP_WRITEV] = { > .needs_file = 1, > @@ -82,18 +76,12 @@ const struct io_issue_def io_issue_defs[] = { > .ioprio = 1, > .iopoll = 1, > .iopoll_queue = 1, > - .async_size = sizeof(struct io_async_rw), > - .name = "WRITEV", > .prep = io_prep_rw, > .issue = io_write, > - .prep_async = io_writev_prep_async, > - .cleanup = io_readv_writev_cleanup, > - .fail = io_rw_fail, > }, > [IORING_OP_FSYNC] = { > .needs_file = 1, > .audit_skip = 1, > - .name = "FSYNC", > .prep = io_fsync_prep, > .issue = io_fsync, > }, > @@ -106,11 +94,8 @@ const struct io_issue_def io_issue_defs[] = { > .ioprio = 1, > .iopoll = 1, > .iopoll_queue = 1, > - .async_size = sizeof(struct io_async_rw), > - .name = "READ_FIXED", > .prep = io_prep_rw, > .issue = io_read, > - .fail = io_rw_fail, > }, > [IORING_OP_WRITE_FIXED] = { > .needs_file = 1, > @@ -122,30 +107,24 @@ const struct io_issue_def io_issue_defs[] = { > .ioprio = 1, > .iopoll = 1, > .iopoll_queue = 1, > - .async_size = sizeof(struct io_async_rw), > - .name = "WRITE_FIXED", > .prep = io_prep_rw, > .issue = io_write, > - .fail = io_rw_fail, > }, > [IORING_OP_POLL_ADD] = { > .needs_file = 1, > .unbound_nonreg_file = 1, > .audit_skip = 1, > - .name = "POLL_ADD", > .prep = io_poll_add_prep, > .issue = io_poll_add, > }, > [IORING_OP_POLL_REMOVE] = { > .audit_skip = 1, > - .name = "POLL_REMOVE", > .prep = io_poll_remove_prep, > .issue = io_poll_remove, > }, > [IORING_OP_SYNC_FILE_RANGE] = { > .needs_file = 1, > .audit_skip = 1, > - .name = "SYNC_FILE_RANGE", > .prep = io_sfr_prep, > .issue = io_sync_file_range, > }, > @@ -155,14 +134,9 @@ const struct io_issue_def io_issue_defs[] = { > .pollout = 1, > .ioprio = 1, > .manual_alloc = 1, > - .name = "SENDMSG", > #if defined(CONFIG_NET) > - .async_size = sizeof(struct io_async_msghdr), > .prep = io_sendmsg_prep, > .issue = io_sendmsg, > - .prep_async = io_sendmsg_prep_async, > - .cleanup = io_sendmsg_recvmsg_cleanup, > - .fail = io_sendrecv_fail, > #else > .prep = io_eopnotsupp_prep, > #endif > @@ -174,29 +148,21 @@ const struct io_issue_def io_issue_defs[] = { > .buffer_select = 1, > .ioprio = 1, > .manual_alloc = 1, > - .name = "RECVMSG", > #if defined(CONFIG_NET) > - .async_size = sizeof(struct io_async_msghdr), > .prep = io_recvmsg_prep, > .issue = io_recvmsg, > - .prep_async = io_recvmsg_prep_async, > - .cleanup = io_sendmsg_recvmsg_cleanup, > - .fail = io_sendrecv_fail, > #else > .prep = io_eopnotsupp_prep, > #endif > }, > [IORING_OP_TIMEOUT] = { > .audit_skip = 1, > - .async_size = sizeof(struct io_timeout_data), > - .name = "TIMEOUT", > .prep = io_timeout_prep, > .issue = io_timeout, > }, > [IORING_OP_TIMEOUT_REMOVE] = { > /* used by timeout updates' prep() */ > .audit_skip = 1, > - .name = "TIMEOUT_REMOVE", > .prep = io_timeout_remove_prep, > .issue = io_timeout_remove, > }, > @@ -206,7 +172,6 @@ const struct io_issue_def io_issue_defs[] = { > .pollin = 1, > .poll_exclusive = 1, > .ioprio = 1, /* used for flags */ > - .name = "ACCEPT", > #if defined(CONFIG_NET) > .prep = io_accept_prep, > .issue = io_accept, > @@ -216,14 +181,11 @@ const struct io_issue_def io_issue_defs[] = { > }, > [IORING_OP_ASYNC_CANCEL] = { > .audit_skip = 1, > - .name = "ASYNC_CANCEL", > .prep = io_async_cancel_prep, > .issue = io_async_cancel, > }, > [IORING_OP_LINK_TIMEOUT] = { > .audit_skip = 1, > - .async_size = sizeof(struct io_timeout_data), > - .name = "LINK_TIMEOUT", > .prep = io_link_timeout_prep, > .issue = io_no_issue, > }, > @@ -231,46 +193,36 @@ const struct io_issue_def io_issue_defs[] = { > .needs_file = 1, > .unbound_nonreg_file = 1, > .pollout = 1, > - .name = "CONNECT", > #if defined(CONFIG_NET) > - .async_size = sizeof(struct io_async_connect), > .prep = io_connect_prep, > .issue = io_connect, > - .prep_async = io_connect_prep_async, > #else > .prep = io_eopnotsupp_prep, > #endif > }, > [IORING_OP_FALLOCATE] = { > .needs_file = 1, > - .name = "FALLOCATE", > .prep = io_fallocate_prep, > .issue = io_fallocate, > }, > [IORING_OP_OPENAT] = { > - .name = "OPENAT", > .prep = io_openat_prep, > .issue = io_openat, > - .cleanup = io_open_cleanup, > }, > [IORING_OP_CLOSE] = { > - .name = "CLOSE", > .prep = io_close_prep, > .issue = io_close, > }, > [IORING_OP_FILES_UPDATE] = { > .audit_skip = 1, > .iopoll = 1, > - .name = "FILES_UPDATE", > .prep = io_files_update_prep, > .issue = io_files_update, > }, > [IORING_OP_STATX] = { > .audit_skip = 1, > - .name = "STATX", > .prep = io_statx_prep, > .issue = io_statx, > - .cleanup = io_statx_cleanup, > }, > [IORING_OP_READ] = { > .needs_file = 1, > @@ -282,11 +234,8 @@ const struct io_issue_def io_issue_defs[] = { > .ioprio = 1, > .iopoll = 1, > .iopoll_queue = 1, > - .async_size = sizeof(struct io_async_rw), > - .name = "READ", > .prep = io_prep_rw, > .issue = io_read, > - .fail = io_rw_fail, > }, > [IORING_OP_WRITE] = { > .needs_file = 1, > @@ -298,21 +247,16 @@ const struct io_issue_def io_issue_defs[] = { > .ioprio = 1, > .iopoll = 1, > .iopoll_queue = 1, > - .async_size = sizeof(struct io_async_rw), > - .name = "WRITE", > .prep = io_prep_rw, > .issue = io_write, > - .fail = io_rw_fail, > }, > [IORING_OP_FADVISE] = { > .needs_file = 1, > .audit_skip = 1, > - .name = "FADVISE", > .prep = io_fadvise_prep, > .issue = io_fadvise, > }, > [IORING_OP_MADVISE] = { > - .name = "MADVISE", > .prep = io_madvise_prep, > .issue = io_madvise, > }, > @@ -323,13 +267,9 @@ const struct io_issue_def io_issue_defs[] = { > .audit_skip = 1, > .ioprio = 1, > .manual_alloc = 1, > - .name = "SEND", > #if defined(CONFIG_NET) > - .async_size = sizeof(struct io_async_msghdr), > .prep = io_sendmsg_prep, > .issue = io_send, > - .fail = io_sendrecv_fail, > - .prep_async = io_send_prep_async, > #else > .prep = io_eopnotsupp_prep, > #endif > @@ -341,25 +281,20 @@ const struct io_issue_def io_issue_defs[] = { > .buffer_select = 1, > .audit_skip = 1, > .ioprio = 1, > - .name = "RECV", > #if defined(CONFIG_NET) > .prep = io_recvmsg_prep, > .issue = io_recv, > - .fail = io_sendrecv_fail, > #else > .prep = io_eopnotsupp_prep, > #endif > }, > [IORING_OP_OPENAT2] = { > - .name = "OPENAT2", > .prep = io_openat2_prep, > .issue = io_openat2, > - .cleanup = io_open_cleanup, > }, > [IORING_OP_EPOLL_CTL] = { > .unbound_nonreg_file = 1, > .audit_skip = 1, > - .name = "EPOLL", > #if defined(CONFIG_EPOLL) > .prep = io_epoll_ctl_prep, > .issue = io_epoll_ctl, > @@ -372,21 +307,18 @@ const struct io_issue_def io_issue_defs[] = { > .hash_reg_file = 1, > .unbound_nonreg_file = 1, > .audit_skip = 1, > - .name = "SPLICE", > .prep = io_splice_prep, > .issue = io_splice, > }, > [IORING_OP_PROVIDE_BUFFERS] = { > .audit_skip = 1, > .iopoll = 1, > - .name = "PROVIDE_BUFFERS", > .prep = io_provide_buffers_prep, > .issue = io_provide_buffers, > }, > [IORING_OP_REMOVE_BUFFERS] = { > .audit_skip = 1, > .iopoll = 1, > - .name = "REMOVE_BUFFERS", > .prep = io_remove_buffers_prep, > .issue = io_remove_buffers, > }, > @@ -395,13 +327,11 @@ const struct io_issue_def io_issue_defs[] = { > .hash_reg_file = 1, > .unbound_nonreg_file = 1, > .audit_skip = 1, > - .name = "TEE", > .prep = io_tee_prep, > .issue = io_tee, > }, > [IORING_OP_SHUTDOWN] = { > .needs_file = 1, > - .name = "SHUTDOWN", > #if defined(CONFIG_NET) > .prep = io_shutdown_prep, > .issue = io_shutdown, > @@ -410,72 +340,51 @@ const struct io_issue_def io_issue_defs[] = { > #endif > }, > [IORING_OP_RENAMEAT] = { > - .name = "RENAMEAT", > .prep = io_renameat_prep, > .issue = io_renameat, > - .cleanup = io_renameat_cleanup, > }, > [IORING_OP_UNLINKAT] = { > - .name = "UNLINKAT", > .prep = io_unlinkat_prep, > .issue = io_unlinkat, > - .cleanup = io_unlinkat_cleanup, > }, > [IORING_OP_MKDIRAT] = { > - .name = "MKDIRAT", > .prep = io_mkdirat_prep, > .issue = io_mkdirat, > - .cleanup = io_mkdirat_cleanup, > }, > [IORING_OP_SYMLINKAT] = { > - .name = "SYMLINKAT", > .prep = io_symlinkat_prep, > .issue = io_symlinkat, > - .cleanup = io_link_cleanup, > }, > [IORING_OP_LINKAT] = { > - .name = "LINKAT", > .prep = io_linkat_prep, > .issue = io_linkat, > - .cleanup = io_link_cleanup, > }, > [IORING_OP_MSG_RING] = { > .needs_file = 1, > .iopoll = 1, > - .name = "MSG_RING", > .prep = io_msg_ring_prep, > .issue = io_msg_ring, > - .cleanup = io_msg_ring_cleanup, > }, > [IORING_OP_FSETXATTR] = { > .needs_file = 1, > - .name = "FSETXATTR", > .prep = io_fsetxattr_prep, > .issue = io_fsetxattr, > - .cleanup = io_xattr_cleanup, > }, > [IORING_OP_SETXATTR] = { > - .name = "SETXATTR", > .prep = io_setxattr_prep, > .issue = io_setxattr, > - .cleanup = io_xattr_cleanup, > }, > [IORING_OP_FGETXATTR] = { > .needs_file = 1, > - .name = "FGETXATTR", > .prep = io_fgetxattr_prep, > .issue = io_fgetxattr, > - .cleanup = io_xattr_cleanup, > }, > [IORING_OP_GETXATTR] = { > - .name = "GETXATTR", > .prep = io_getxattr_prep, > .issue = io_getxattr, > - .cleanup = io_xattr_cleanup, > }, > [IORING_OP_SOCKET] = { > .audit_skip = 1, > - .name = "SOCKET", > #if defined(CONFIG_NET) > .prep = io_socket_prep, > .issue = io_socket, > @@ -486,16 +395,12 @@ const struct io_issue_def io_issue_defs[] = { > [IORING_OP_URING_CMD] = { > .needs_file = 1, > .plug = 1, > - .name = "URING_CMD", > .iopoll = 1, > .iopoll_queue = 1, > - .async_size = uring_cmd_pdu_size(1), > .prep = io_uring_cmd_prep, > .issue = io_uring_cmd, > - .prep_async = io_uring_cmd_prep_async, > }, > [IORING_OP_SEND_ZC] = { > - .name = "SEND_ZC", > .needs_file = 1, > .unbound_nonreg_file = 1, > .pollout = 1, > @@ -503,32 +408,243 @@ const struct io_issue_def io_issue_defs[] = { > .ioprio = 1, > .manual_alloc = 1, > #if defined(CONFIG_NET) > - .async_size = sizeof(struct io_async_msghdr), > .prep = io_send_zc_prep, > .issue = io_send_zc, > - .prep_async = io_send_prep_async, > - .cleanup = io_send_zc_cleanup, > - .fail = io_sendrecv_fail, > #else > .prep = io_eopnotsupp_prep, > #endif > }, > [IORING_OP_SENDMSG_ZC] = { > - .name = "SENDMSG_ZC", > .needs_file = 1, > .unbound_nonreg_file = 1, > .pollout = 1, > .ioprio = 1, > .manual_alloc = 1, > #if defined(CONFIG_NET) > - .async_size = sizeof(struct io_async_msghdr), > .prep = io_send_zc_prep, > .issue = io_sendmsg_zc, > +#else > + .prep = io_eopnotsupp_prep, > +#endif > + }, > +}; > + > + > +const struct io_cold_def io_cold_defs[] = { > + [IORING_OP_NOP] = { > + .name = "NOP", > + }, > + [IORING_OP_READV] = { > + .async_size = sizeof(struct io_async_rw), > + .name = "READV", > + .prep_async = io_readv_prep_async, > + .cleanup = io_readv_writev_cleanup, > + .fail = io_rw_fail, > + }, > + [IORING_OP_WRITEV] = { > + .async_size = sizeof(struct io_async_rw), > + .name = "WRITEV", > + .prep_async = io_writev_prep_async, > + .cleanup = io_readv_writev_cleanup, > + .fail = io_rw_fail, > + }, > + [IORING_OP_FSYNC] = { > + .name = "FSYNC", > + }, > + [IORING_OP_READ_FIXED] = { > + .async_size = sizeof(struct io_async_rw), > + .name = "READ_FIXED", > + .fail = io_rw_fail, > + }, > + [IORING_OP_WRITE_FIXED] = { > + .async_size = sizeof(struct io_async_rw), > + .name = "WRITE_FIXED", > + .fail = io_rw_fail, > + }, > + [IORING_OP_POLL_ADD] = { > + .name = "POLL_ADD", > + }, > + [IORING_OP_POLL_REMOVE] = { > + .name = "POLL_REMOVE", > + }, > + [IORING_OP_SYNC_FILE_RANGE] = { > + .name = "SYNC_FILE_RANGE", > + }, > + [IORING_OP_SENDMSG] = { > + .name = "SENDMSG", > +#if defined(CONFIG_NET) > + .async_size = sizeof(struct io_async_msghdr), > + .prep_async = io_sendmsg_prep_async, > + .cleanup = io_sendmsg_recvmsg_cleanup, > + .fail = io_sendrecv_fail, > +#endif > + }, > + [IORING_OP_RECVMSG] = { > + .name = "RECVMSG", > +#if defined(CONFIG_NET) > + .async_size = sizeof(struct io_async_msghdr), > + .prep_async = io_recvmsg_prep_async, > + .cleanup = io_sendmsg_recvmsg_cleanup, > + .fail = io_sendrecv_fail, > +#endif > + }, > + [IORING_OP_TIMEOUT] = { > + .async_size = sizeof(struct io_timeout_data), > + .name = "TIMEOUT", > + }, > + [IORING_OP_TIMEOUT_REMOVE] = { > + .name = "TIMEOUT_REMOVE", > + }, > + [IORING_OP_ACCEPT] = { > + .name = "ACCEPT", > + }, > + [IORING_OP_ASYNC_CANCEL] = { > + .name = "ASYNC_CANCEL", > + }, > + [IORING_OP_LINK_TIMEOUT] = { > + .async_size = sizeof(struct io_timeout_data), > + .name = "LINK_TIMEOUT", > + }, > + [IORING_OP_CONNECT] = { > + .name = "CONNECT", > +#if defined(CONFIG_NET) > + .async_size = sizeof(struct io_async_connect), > + .prep_async = io_connect_prep_async, > +#endif > + }, > + [IORING_OP_FALLOCATE] = { > + .name = "FALLOCATE", > + }, > + [IORING_OP_OPENAT] = { > + .name = "OPENAT", > + .cleanup = io_open_cleanup, > + }, > + [IORING_OP_CLOSE] = { > + .name = "CLOSE", > + }, > + [IORING_OP_FILES_UPDATE] = { > + .name = "FILES_UPDATE", > + }, > + [IORING_OP_STATX] = { > + .name = "STATX", > + .cleanup = io_statx_cleanup, > + }, > + [IORING_OP_READ] = { > + .async_size = sizeof(struct io_async_rw), > + .name = "READ", > + .fail = io_rw_fail, > + }, > + [IORING_OP_WRITE] = { > + .async_size = sizeof(struct io_async_rw), > + .name = "WRITE", > + .fail = io_rw_fail, > + }, > + [IORING_OP_FADVISE] = { > + .name = "FADVISE", > + }, > + [IORING_OP_MADVISE] = { > + .name = "MADVISE", > + }, > + [IORING_OP_SEND] = { > + .name = "SEND", > +#if defined(CONFIG_NET) > + .async_size = sizeof(struct io_async_msghdr), > + .fail = io_sendrecv_fail, > + .prep_async = io_send_prep_async, > +#endif > + }, > + [IORING_OP_RECV] = { > + .name = "RECV", > +#if defined(CONFIG_NET) > + .fail = io_sendrecv_fail, > +#endif > + }, > + [IORING_OP_OPENAT2] = { > + .name = "OPENAT2", > + .cleanup = io_open_cleanup, > + }, > + [IORING_OP_EPOLL_CTL] = { > + .name = "EPOLL", > + }, > + [IORING_OP_SPLICE] = { > + .name = "SPLICE", > + }, > + [IORING_OP_PROVIDE_BUFFERS] = { > + .name = "PROVIDE_BUFFERS", > + }, > + [IORING_OP_REMOVE_BUFFERS] = { > + .name = "REMOVE_BUFFERS", > + }, > + [IORING_OP_TEE] = { > + .name = "TEE", > + }, > + [IORING_OP_SHUTDOWN] = { > + .name = "SHUTDOWN", > + }, > + [IORING_OP_RENAMEAT] = { > + .name = "RENAMEAT", > + .cleanup = io_renameat_cleanup, > + }, > + [IORING_OP_UNLINKAT] = { > + .name = "UNLINKAT", > + .cleanup = io_unlinkat_cleanup, > + }, > + [IORING_OP_MKDIRAT] = { > + .name = "MKDIRAT", > + .cleanup = io_mkdirat_cleanup, > + }, > + [IORING_OP_SYMLINKAT] = { > + .name = "SYMLINKAT", > + .cleanup = io_link_cleanup, > + }, > + [IORING_OP_LINKAT] = { > + .name = "LINKAT", > + .cleanup = io_link_cleanup, > + }, > + [IORING_OP_MSG_RING] = { > + .name = "MSG_RING", > + .cleanup = io_msg_ring_cleanup, > + }, > + [IORING_OP_FSETXATTR] = { > + .name = "FSETXATTR", > + .cleanup = io_xattr_cleanup, > + }, > + [IORING_OP_SETXATTR] = { > + .name = "SETXATTR", > + .cleanup = io_xattr_cleanup, > + }, > + [IORING_OP_FGETXATTR] = { > + .name = "FGETXATTR", > + .cleanup = io_xattr_cleanup, > + }, > + [IORING_OP_GETXATTR] = { > + .name = "GETXATTR", > + .cleanup = io_xattr_cleanup, > + }, > + [IORING_OP_SOCKET] = { > + .name = "SOCKET", > + }, > + [IORING_OP_URING_CMD] = { > + .name = "URING_CMD", > + .async_size = uring_cmd_pdu_size(1), > + .prep_async = io_uring_cmd_prep_async, > + }, > + [IORING_OP_SEND_ZC] = { > + .name = "SEND_ZC", > +#if defined(CONFIG_NET) > + .async_size = sizeof(struct io_async_msghdr), > + .prep_async = io_send_prep_async, > + .cleanup = io_send_zc_cleanup, > + .fail = io_sendrecv_fail, > +#endif > + }, > + [IORING_OP_SENDMSG_ZC] = { > + .name = "SENDMSG_ZC", > +#if defined(CONFIG_NET) > + .async_size = sizeof(struct io_async_msghdr), > .prep_async = io_sendmsg_prep_async, > .cleanup = io_send_zc_cleanup, > .fail = io_sendrecv_fail, > -#else > - .prep = io_eopnotsupp_prep, > #endif > }, > }; > @@ -536,7 +652,7 @@ const struct io_issue_def io_issue_defs[] = { > const char *io_uring_get_opcode(u8 opcode) > { > if (opcode < IORING_OP_LAST) > - return io_issue_defs[opcode].name; > + return io_cold_defs[opcode].name; > return "INVALID"; > } > > @@ -544,12 +660,13 @@ void __init io_uring_optable_init(void) > { > int i; > > + BUILD_BUG_ON(ARRAY_SIZE(io_cold_defs) != IORING_OP_LAST); > BUILD_BUG_ON(ARRAY_SIZE(io_issue_defs) != IORING_OP_LAST); > > for (i = 0; i < ARRAY_SIZE(io_issue_defs); i++) { > BUG_ON(!io_issue_defs[i].prep); > if (io_issue_defs[i].prep != io_eopnotsupp_prep) > BUG_ON(!io_issue_defs[i].issue); > - WARN_ON_ONCE(!io_issue_defs[i].name); > + WARN_ON_ONCE(!io_cold_defs[i].name); > } > } > diff --git a/io_uring/opdef.h b/io_uring/opdef.h > index d718e2ab1ff7..c22c8696e749 100644 > --- a/io_uring/opdef.h > +++ b/io_uring/opdef.h > @@ -29,19 +29,24 @@ struct io_issue_def { > unsigned iopoll_queue : 1; > /* opcode specific path will handle ->async_data allocation if needed */ > unsigned manual_alloc : 1; > + > + int (*issue)(struct io_kiocb *, unsigned int); > + int (*prep)(struct io_kiocb *, const struct io_uring_sqe *); > +}; > + > +struct io_cold_def { > /* size of async data needed, if any */ > unsigned short async_size; > > const char *name; > > - int (*prep)(struct io_kiocb *, const struct io_uring_sqe *); > - int (*issue)(struct io_kiocb *, unsigned int); > int (*prep_async)(struct io_kiocb *); > void (*cleanup)(struct io_kiocb *); > void (*fail)(struct io_kiocb *); > }; > > extern const struct io_issue_def io_issue_defs[]; > +extern const struct io_cold_def io_cold_defs[]; > > void io_uring_optable_init(void); > #endif > diff --git a/io_uring/rw.c b/io_uring/rw.c > index 54b44b9b736c..a8a2eb7ee27a 100644 > --- a/io_uring/rw.c > +++ b/io_uring/rw.c > @@ -516,7 +516,7 @@ static void io_req_map_rw(struct io_kiocb *req, const struct iovec *iovec, > static int io_setup_async_rw(struct io_kiocb *req, const struct iovec *iovec, > struct io_rw_state *s, bool force) > { > - if (!force && !io_issue_defs[req->opcode].prep_async) > + if (!force && !io_cold_defs[req->opcode].prep_async) > return 0; > if (!req_has_async_data(req)) { > struct io_async_rw *iorw;
diff --git a/io_uring/io_uring.c b/io_uring/io_uring.c index ac7868ec9be2..0be66b026a7f 100644 --- a/io_uring/io_uring.c +++ b/io_uring/io_uring.c @@ -980,7 +980,7 @@ void io_req_complete_post(struct io_kiocb *req, unsigned issue_flags) void io_req_defer_failed(struct io_kiocb *req, s32 res) __must_hold(&ctx->uring_lock) { - const struct io_issue_def *def = &io_issue_defs[req->opcode]; + const struct io_cold_def *def = &io_cold_defs[req->opcode]; lockdep_assert_held(&req->ctx->uring_lock); @@ -1708,8 +1708,8 @@ unsigned int io_file_get_flags(struct file *file) bool io_alloc_async_data(struct io_kiocb *req) { - WARN_ON_ONCE(!io_issue_defs[req->opcode].async_size); - req->async_data = kmalloc(io_issue_defs[req->opcode].async_size, GFP_KERNEL); + WARN_ON_ONCE(!io_cold_defs[req->opcode].async_size); + req->async_data = kmalloc(io_cold_defs[req->opcode].async_size, GFP_KERNEL); if (req->async_data) { req->flags |= REQ_F_ASYNC_DATA; return false; @@ -1719,20 +1719,21 @@ bool io_alloc_async_data(struct io_kiocb *req) int io_req_prep_async(struct io_kiocb *req) { + const struct io_cold_def *cdef = &io_cold_defs[req->opcode]; const struct io_issue_def *def = &io_issue_defs[req->opcode]; /* assign early for deferred execution for non-fixed file */ if (def->needs_file && !(req->flags & REQ_F_FIXED_FILE)) req->file = io_file_get_normal(req, req->cqe.fd); - if (!def->prep_async) + if (!cdef->prep_async) return 0; if (WARN_ON_ONCE(req_has_async_data(req))) return -EFAULT; - if (!io_issue_defs[req->opcode].manual_alloc) { + if (!def->manual_alloc) { if (io_alloc_async_data(req)) return -EAGAIN; } - return def->prep_async(req); + return cdef->prep_async(req); } static u32 io_get_sequence(struct io_kiocb *req) @@ -1801,7 +1802,7 @@ static void io_clean_op(struct io_kiocb *req) } if (req->flags & REQ_F_NEED_CLEANUP) { - const struct io_issue_def *def = &io_issue_defs[req->opcode]; + const struct io_cold_def *def = &io_cold_defs[req->opcode]; if (def->cleanup) def->cleanup(req); diff --git a/io_uring/opdef.c b/io_uring/opdef.c index 3c95e70a625e..5238ecd7af6a 100644 --- a/io_uring/opdef.c +++ b/io_uring/opdef.c @@ -50,7 +50,6 @@ const struct io_issue_def io_issue_defs[] = { [IORING_OP_NOP] = { .audit_skip = 1, .iopoll = 1, - .name = "NOP", .prep = io_nop_prep, .issue = io_nop, }, @@ -64,13 +63,8 @@ const struct io_issue_def io_issue_defs[] = { .ioprio = 1, .iopoll = 1, .iopoll_queue = 1, - .async_size = sizeof(struct io_async_rw), - .name = "READV", .prep = io_prep_rw, .issue = io_read, - .prep_async = io_readv_prep_async, - .cleanup = io_readv_writev_cleanup, - .fail = io_rw_fail, }, [IORING_OP_WRITEV] = { .needs_file = 1, @@ -82,18 +76,12 @@ const struct io_issue_def io_issue_defs[] = { .ioprio = 1, .iopoll = 1, .iopoll_queue = 1, - .async_size = sizeof(struct io_async_rw), - .name = "WRITEV", .prep = io_prep_rw, .issue = io_write, - .prep_async = io_writev_prep_async, - .cleanup = io_readv_writev_cleanup, - .fail = io_rw_fail, }, [IORING_OP_FSYNC] = { .needs_file = 1, .audit_skip = 1, - .name = "FSYNC", .prep = io_fsync_prep, .issue = io_fsync, }, @@ -106,11 +94,8 @@ const struct io_issue_def io_issue_defs[] = { .ioprio = 1, .iopoll = 1, .iopoll_queue = 1, - .async_size = sizeof(struct io_async_rw), - .name = "READ_FIXED", .prep = io_prep_rw, .issue = io_read, - .fail = io_rw_fail, }, [IORING_OP_WRITE_FIXED] = { .needs_file = 1, @@ -122,30 +107,24 @@ const struct io_issue_def io_issue_defs[] = { .ioprio = 1, .iopoll = 1, .iopoll_queue = 1, - .async_size = sizeof(struct io_async_rw), - .name = "WRITE_FIXED", .prep = io_prep_rw, .issue = io_write, - .fail = io_rw_fail, }, [IORING_OP_POLL_ADD] = { .needs_file = 1, .unbound_nonreg_file = 1, .audit_skip = 1, - .name = "POLL_ADD", .prep = io_poll_add_prep, .issue = io_poll_add, }, [IORING_OP_POLL_REMOVE] = { .audit_skip = 1, - .name = "POLL_REMOVE", .prep = io_poll_remove_prep, .issue = io_poll_remove, }, [IORING_OP_SYNC_FILE_RANGE] = { .needs_file = 1, .audit_skip = 1, - .name = "SYNC_FILE_RANGE", .prep = io_sfr_prep, .issue = io_sync_file_range, }, @@ -155,14 +134,9 @@ const struct io_issue_def io_issue_defs[] = { .pollout = 1, .ioprio = 1, .manual_alloc = 1, - .name = "SENDMSG", #if defined(CONFIG_NET) - .async_size = sizeof(struct io_async_msghdr), .prep = io_sendmsg_prep, .issue = io_sendmsg, - .prep_async = io_sendmsg_prep_async, - .cleanup = io_sendmsg_recvmsg_cleanup, - .fail = io_sendrecv_fail, #else .prep = io_eopnotsupp_prep, #endif @@ -174,29 +148,21 @@ const struct io_issue_def io_issue_defs[] = { .buffer_select = 1, .ioprio = 1, .manual_alloc = 1, - .name = "RECVMSG", #if defined(CONFIG_NET) - .async_size = sizeof(struct io_async_msghdr), .prep = io_recvmsg_prep, .issue = io_recvmsg, - .prep_async = io_recvmsg_prep_async, - .cleanup = io_sendmsg_recvmsg_cleanup, - .fail = io_sendrecv_fail, #else .prep = io_eopnotsupp_prep, #endif }, [IORING_OP_TIMEOUT] = { .audit_skip = 1, - .async_size = sizeof(struct io_timeout_data), - .name = "TIMEOUT", .prep = io_timeout_prep, .issue = io_timeout, }, [IORING_OP_TIMEOUT_REMOVE] = { /* used by timeout updates' prep() */ .audit_skip = 1, - .name = "TIMEOUT_REMOVE", .prep = io_timeout_remove_prep, .issue = io_timeout_remove, }, @@ -206,7 +172,6 @@ const struct io_issue_def io_issue_defs[] = { .pollin = 1, .poll_exclusive = 1, .ioprio = 1, /* used for flags */ - .name = "ACCEPT", #if defined(CONFIG_NET) .prep = io_accept_prep, .issue = io_accept, @@ -216,14 +181,11 @@ const struct io_issue_def io_issue_defs[] = { }, [IORING_OP_ASYNC_CANCEL] = { .audit_skip = 1, - .name = "ASYNC_CANCEL", .prep = io_async_cancel_prep, .issue = io_async_cancel, }, [IORING_OP_LINK_TIMEOUT] = { .audit_skip = 1, - .async_size = sizeof(struct io_timeout_data), - .name = "LINK_TIMEOUT", .prep = io_link_timeout_prep, .issue = io_no_issue, }, @@ -231,46 +193,36 @@ const struct io_issue_def io_issue_defs[] = { .needs_file = 1, .unbound_nonreg_file = 1, .pollout = 1, - .name = "CONNECT", #if defined(CONFIG_NET) - .async_size = sizeof(struct io_async_connect), .prep = io_connect_prep, .issue = io_connect, - .prep_async = io_connect_prep_async, #else .prep = io_eopnotsupp_prep, #endif }, [IORING_OP_FALLOCATE] = { .needs_file = 1, - .name = "FALLOCATE", .prep = io_fallocate_prep, .issue = io_fallocate, }, [IORING_OP_OPENAT] = { - .name = "OPENAT", .prep = io_openat_prep, .issue = io_openat, - .cleanup = io_open_cleanup, }, [IORING_OP_CLOSE] = { - .name = "CLOSE", .prep = io_close_prep, .issue = io_close, }, [IORING_OP_FILES_UPDATE] = { .audit_skip = 1, .iopoll = 1, - .name = "FILES_UPDATE", .prep = io_files_update_prep, .issue = io_files_update, }, [IORING_OP_STATX] = { .audit_skip = 1, - .name = "STATX", .prep = io_statx_prep, .issue = io_statx, - .cleanup = io_statx_cleanup, }, [IORING_OP_READ] = { .needs_file = 1, @@ -282,11 +234,8 @@ const struct io_issue_def io_issue_defs[] = { .ioprio = 1, .iopoll = 1, .iopoll_queue = 1, - .async_size = sizeof(struct io_async_rw), - .name = "READ", .prep = io_prep_rw, .issue = io_read, - .fail = io_rw_fail, }, [IORING_OP_WRITE] = { .needs_file = 1, @@ -298,21 +247,16 @@ const struct io_issue_def io_issue_defs[] = { .ioprio = 1, .iopoll = 1, .iopoll_queue = 1, - .async_size = sizeof(struct io_async_rw), - .name = "WRITE", .prep = io_prep_rw, .issue = io_write, - .fail = io_rw_fail, }, [IORING_OP_FADVISE] = { .needs_file = 1, .audit_skip = 1, - .name = "FADVISE", .prep = io_fadvise_prep, .issue = io_fadvise, }, [IORING_OP_MADVISE] = { - .name = "MADVISE", .prep = io_madvise_prep, .issue = io_madvise, }, @@ -323,13 +267,9 @@ const struct io_issue_def io_issue_defs[] = { .audit_skip = 1, .ioprio = 1, .manual_alloc = 1, - .name = "SEND", #if defined(CONFIG_NET) - .async_size = sizeof(struct io_async_msghdr), .prep = io_sendmsg_prep, .issue = io_send, - .fail = io_sendrecv_fail, - .prep_async = io_send_prep_async, #else .prep = io_eopnotsupp_prep, #endif @@ -341,25 +281,20 @@ const struct io_issue_def io_issue_defs[] = { .buffer_select = 1, .audit_skip = 1, .ioprio = 1, - .name = "RECV", #if defined(CONFIG_NET) .prep = io_recvmsg_prep, .issue = io_recv, - .fail = io_sendrecv_fail, #else .prep = io_eopnotsupp_prep, #endif }, [IORING_OP_OPENAT2] = { - .name = "OPENAT2", .prep = io_openat2_prep, .issue = io_openat2, - .cleanup = io_open_cleanup, }, [IORING_OP_EPOLL_CTL] = { .unbound_nonreg_file = 1, .audit_skip = 1, - .name = "EPOLL", #if defined(CONFIG_EPOLL) .prep = io_epoll_ctl_prep, .issue = io_epoll_ctl, @@ -372,21 +307,18 @@ const struct io_issue_def io_issue_defs[] = { .hash_reg_file = 1, .unbound_nonreg_file = 1, .audit_skip = 1, - .name = "SPLICE", .prep = io_splice_prep, .issue = io_splice, }, [IORING_OP_PROVIDE_BUFFERS] = { .audit_skip = 1, .iopoll = 1, - .name = "PROVIDE_BUFFERS", .prep = io_provide_buffers_prep, .issue = io_provide_buffers, }, [IORING_OP_REMOVE_BUFFERS] = { .audit_skip = 1, .iopoll = 1, - .name = "REMOVE_BUFFERS", .prep = io_remove_buffers_prep, .issue = io_remove_buffers, }, @@ -395,13 +327,11 @@ const struct io_issue_def io_issue_defs[] = { .hash_reg_file = 1, .unbound_nonreg_file = 1, .audit_skip = 1, - .name = "TEE", .prep = io_tee_prep, .issue = io_tee, }, [IORING_OP_SHUTDOWN] = { .needs_file = 1, - .name = "SHUTDOWN", #if defined(CONFIG_NET) .prep = io_shutdown_prep, .issue = io_shutdown, @@ -410,72 +340,51 @@ const struct io_issue_def io_issue_defs[] = { #endif }, [IORING_OP_RENAMEAT] = { - .name = "RENAMEAT", .prep = io_renameat_prep, .issue = io_renameat, - .cleanup = io_renameat_cleanup, }, [IORING_OP_UNLINKAT] = { - .name = "UNLINKAT", .prep = io_unlinkat_prep, .issue = io_unlinkat, - .cleanup = io_unlinkat_cleanup, }, [IORING_OP_MKDIRAT] = { - .name = "MKDIRAT", .prep = io_mkdirat_prep, .issue = io_mkdirat, - .cleanup = io_mkdirat_cleanup, }, [IORING_OP_SYMLINKAT] = { - .name = "SYMLINKAT", .prep = io_symlinkat_prep, .issue = io_symlinkat, - .cleanup = io_link_cleanup, }, [IORING_OP_LINKAT] = { - .name = "LINKAT", .prep = io_linkat_prep, .issue = io_linkat, - .cleanup = io_link_cleanup, }, [IORING_OP_MSG_RING] = { .needs_file = 1, .iopoll = 1, - .name = "MSG_RING", .prep = io_msg_ring_prep, .issue = io_msg_ring, - .cleanup = io_msg_ring_cleanup, }, [IORING_OP_FSETXATTR] = { .needs_file = 1, - .name = "FSETXATTR", .prep = io_fsetxattr_prep, .issue = io_fsetxattr, - .cleanup = io_xattr_cleanup, }, [IORING_OP_SETXATTR] = { - .name = "SETXATTR", .prep = io_setxattr_prep, .issue = io_setxattr, - .cleanup = io_xattr_cleanup, }, [IORING_OP_FGETXATTR] = { .needs_file = 1, - .name = "FGETXATTR", .prep = io_fgetxattr_prep, .issue = io_fgetxattr, - .cleanup = io_xattr_cleanup, }, [IORING_OP_GETXATTR] = { - .name = "GETXATTR", .prep = io_getxattr_prep, .issue = io_getxattr, - .cleanup = io_xattr_cleanup, }, [IORING_OP_SOCKET] = { .audit_skip = 1, - .name = "SOCKET", #if defined(CONFIG_NET) .prep = io_socket_prep, .issue = io_socket, @@ -486,16 +395,12 @@ const struct io_issue_def io_issue_defs[] = { [IORING_OP_URING_CMD] = { .needs_file = 1, .plug = 1, - .name = "URING_CMD", .iopoll = 1, .iopoll_queue = 1, - .async_size = uring_cmd_pdu_size(1), .prep = io_uring_cmd_prep, .issue = io_uring_cmd, - .prep_async = io_uring_cmd_prep_async, }, [IORING_OP_SEND_ZC] = { - .name = "SEND_ZC", .needs_file = 1, .unbound_nonreg_file = 1, .pollout = 1, @@ -503,32 +408,243 @@ const struct io_issue_def io_issue_defs[] = { .ioprio = 1, .manual_alloc = 1, #if defined(CONFIG_NET) - .async_size = sizeof(struct io_async_msghdr), .prep = io_send_zc_prep, .issue = io_send_zc, - .prep_async = io_send_prep_async, - .cleanup = io_send_zc_cleanup, - .fail = io_sendrecv_fail, #else .prep = io_eopnotsupp_prep, #endif }, [IORING_OP_SENDMSG_ZC] = { - .name = "SENDMSG_ZC", .needs_file = 1, .unbound_nonreg_file = 1, .pollout = 1, .ioprio = 1, .manual_alloc = 1, #if defined(CONFIG_NET) - .async_size = sizeof(struct io_async_msghdr), .prep = io_send_zc_prep, .issue = io_sendmsg_zc, +#else + .prep = io_eopnotsupp_prep, +#endif + }, +}; + + +const struct io_cold_def io_cold_defs[] = { + [IORING_OP_NOP] = { + .name = "NOP", + }, + [IORING_OP_READV] = { + .async_size = sizeof(struct io_async_rw), + .name = "READV", + .prep_async = io_readv_prep_async, + .cleanup = io_readv_writev_cleanup, + .fail = io_rw_fail, + }, + [IORING_OP_WRITEV] = { + .async_size = sizeof(struct io_async_rw), + .name = "WRITEV", + .prep_async = io_writev_prep_async, + .cleanup = io_readv_writev_cleanup, + .fail = io_rw_fail, + }, + [IORING_OP_FSYNC] = { + .name = "FSYNC", + }, + [IORING_OP_READ_FIXED] = { + .async_size = sizeof(struct io_async_rw), + .name = "READ_FIXED", + .fail = io_rw_fail, + }, + [IORING_OP_WRITE_FIXED] = { + .async_size = sizeof(struct io_async_rw), + .name = "WRITE_FIXED", + .fail = io_rw_fail, + }, + [IORING_OP_POLL_ADD] = { + .name = "POLL_ADD", + }, + [IORING_OP_POLL_REMOVE] = { + .name = "POLL_REMOVE", + }, + [IORING_OP_SYNC_FILE_RANGE] = { + .name = "SYNC_FILE_RANGE", + }, + [IORING_OP_SENDMSG] = { + .name = "SENDMSG", +#if defined(CONFIG_NET) + .async_size = sizeof(struct io_async_msghdr), + .prep_async = io_sendmsg_prep_async, + .cleanup = io_sendmsg_recvmsg_cleanup, + .fail = io_sendrecv_fail, +#endif + }, + [IORING_OP_RECVMSG] = { + .name = "RECVMSG", +#if defined(CONFIG_NET) + .async_size = sizeof(struct io_async_msghdr), + .prep_async = io_recvmsg_prep_async, + .cleanup = io_sendmsg_recvmsg_cleanup, + .fail = io_sendrecv_fail, +#endif + }, + [IORING_OP_TIMEOUT] = { + .async_size = sizeof(struct io_timeout_data), + .name = "TIMEOUT", + }, + [IORING_OP_TIMEOUT_REMOVE] = { + .name = "TIMEOUT_REMOVE", + }, + [IORING_OP_ACCEPT] = { + .name = "ACCEPT", + }, + [IORING_OP_ASYNC_CANCEL] = { + .name = "ASYNC_CANCEL", + }, + [IORING_OP_LINK_TIMEOUT] = { + .async_size = sizeof(struct io_timeout_data), + .name = "LINK_TIMEOUT", + }, + [IORING_OP_CONNECT] = { + .name = "CONNECT", +#if defined(CONFIG_NET) + .async_size = sizeof(struct io_async_connect), + .prep_async = io_connect_prep_async, +#endif + }, + [IORING_OP_FALLOCATE] = { + .name = "FALLOCATE", + }, + [IORING_OP_OPENAT] = { + .name = "OPENAT", + .cleanup = io_open_cleanup, + }, + [IORING_OP_CLOSE] = { + .name = "CLOSE", + }, + [IORING_OP_FILES_UPDATE] = { + .name = "FILES_UPDATE", + }, + [IORING_OP_STATX] = { + .name = "STATX", + .cleanup = io_statx_cleanup, + }, + [IORING_OP_READ] = { + .async_size = sizeof(struct io_async_rw), + .name = "READ", + .fail = io_rw_fail, + }, + [IORING_OP_WRITE] = { + .async_size = sizeof(struct io_async_rw), + .name = "WRITE", + .fail = io_rw_fail, + }, + [IORING_OP_FADVISE] = { + .name = "FADVISE", + }, + [IORING_OP_MADVISE] = { + .name = "MADVISE", + }, + [IORING_OP_SEND] = { + .name = "SEND", +#if defined(CONFIG_NET) + .async_size = sizeof(struct io_async_msghdr), + .fail = io_sendrecv_fail, + .prep_async = io_send_prep_async, +#endif + }, + [IORING_OP_RECV] = { + .name = "RECV", +#if defined(CONFIG_NET) + .fail = io_sendrecv_fail, +#endif + }, + [IORING_OP_OPENAT2] = { + .name = "OPENAT2", + .cleanup = io_open_cleanup, + }, + [IORING_OP_EPOLL_CTL] = { + .name = "EPOLL", + }, + [IORING_OP_SPLICE] = { + .name = "SPLICE", + }, + [IORING_OP_PROVIDE_BUFFERS] = { + .name = "PROVIDE_BUFFERS", + }, + [IORING_OP_REMOVE_BUFFERS] = { + .name = "REMOVE_BUFFERS", + }, + [IORING_OP_TEE] = { + .name = "TEE", + }, + [IORING_OP_SHUTDOWN] = { + .name = "SHUTDOWN", + }, + [IORING_OP_RENAMEAT] = { + .name = "RENAMEAT", + .cleanup = io_renameat_cleanup, + }, + [IORING_OP_UNLINKAT] = { + .name = "UNLINKAT", + .cleanup = io_unlinkat_cleanup, + }, + [IORING_OP_MKDIRAT] = { + .name = "MKDIRAT", + .cleanup = io_mkdirat_cleanup, + }, + [IORING_OP_SYMLINKAT] = { + .name = "SYMLINKAT", + .cleanup = io_link_cleanup, + }, + [IORING_OP_LINKAT] = { + .name = "LINKAT", + .cleanup = io_link_cleanup, + }, + [IORING_OP_MSG_RING] = { + .name = "MSG_RING", + .cleanup = io_msg_ring_cleanup, + }, + [IORING_OP_FSETXATTR] = { + .name = "FSETXATTR", + .cleanup = io_xattr_cleanup, + }, + [IORING_OP_SETXATTR] = { + .name = "SETXATTR", + .cleanup = io_xattr_cleanup, + }, + [IORING_OP_FGETXATTR] = { + .name = "FGETXATTR", + .cleanup = io_xattr_cleanup, + }, + [IORING_OP_GETXATTR] = { + .name = "GETXATTR", + .cleanup = io_xattr_cleanup, + }, + [IORING_OP_SOCKET] = { + .name = "SOCKET", + }, + [IORING_OP_URING_CMD] = { + .name = "URING_CMD", + .async_size = uring_cmd_pdu_size(1), + .prep_async = io_uring_cmd_prep_async, + }, + [IORING_OP_SEND_ZC] = { + .name = "SEND_ZC", +#if defined(CONFIG_NET) + .async_size = sizeof(struct io_async_msghdr), + .prep_async = io_send_prep_async, + .cleanup = io_send_zc_cleanup, + .fail = io_sendrecv_fail, +#endif + }, + [IORING_OP_SENDMSG_ZC] = { + .name = "SENDMSG_ZC", +#if defined(CONFIG_NET) + .async_size = sizeof(struct io_async_msghdr), .prep_async = io_sendmsg_prep_async, .cleanup = io_send_zc_cleanup, .fail = io_sendrecv_fail, -#else - .prep = io_eopnotsupp_prep, #endif }, }; @@ -536,7 +652,7 @@ const struct io_issue_def io_issue_defs[] = { const char *io_uring_get_opcode(u8 opcode) { if (opcode < IORING_OP_LAST) - return io_issue_defs[opcode].name; + return io_cold_defs[opcode].name; return "INVALID"; } @@ -544,12 +660,13 @@ void __init io_uring_optable_init(void) { int i; + BUILD_BUG_ON(ARRAY_SIZE(io_cold_defs) != IORING_OP_LAST); BUILD_BUG_ON(ARRAY_SIZE(io_issue_defs) != IORING_OP_LAST); for (i = 0; i < ARRAY_SIZE(io_issue_defs); i++) { BUG_ON(!io_issue_defs[i].prep); if (io_issue_defs[i].prep != io_eopnotsupp_prep) BUG_ON(!io_issue_defs[i].issue); - WARN_ON_ONCE(!io_issue_defs[i].name); + WARN_ON_ONCE(!io_cold_defs[i].name); } } diff --git a/io_uring/opdef.h b/io_uring/opdef.h index d718e2ab1ff7..c22c8696e749 100644 --- a/io_uring/opdef.h +++ b/io_uring/opdef.h @@ -29,19 +29,24 @@ struct io_issue_def { unsigned iopoll_queue : 1; /* opcode specific path will handle ->async_data allocation if needed */ unsigned manual_alloc : 1; + + int (*issue)(struct io_kiocb *, unsigned int); + int (*prep)(struct io_kiocb *, const struct io_uring_sqe *); +}; + +struct io_cold_def { /* size of async data needed, if any */ unsigned short async_size; const char *name; - int (*prep)(struct io_kiocb *, const struct io_uring_sqe *); - int (*issue)(struct io_kiocb *, unsigned int); int (*prep_async)(struct io_kiocb *); void (*cleanup)(struct io_kiocb *); void (*fail)(struct io_kiocb *); }; extern const struct io_issue_def io_issue_defs[]; +extern const struct io_cold_def io_cold_defs[]; void io_uring_optable_init(void); #endif diff --git a/io_uring/rw.c b/io_uring/rw.c index 54b44b9b736c..a8a2eb7ee27a 100644 --- a/io_uring/rw.c +++ b/io_uring/rw.c @@ -516,7 +516,7 @@ static void io_req_map_rw(struct io_kiocb *req, const struct iovec *iovec, static int io_setup_async_rw(struct io_kiocb *req, const struct iovec *iovec, struct io_rw_state *s, bool force) { - if (!force && !io_issue_defs[req->opcode].prep_async) + if (!force && !io_cold_defs[req->opcode].prep_async) return 0; if (!req_has_async_data(req)) { struct io_async_rw *iorw;
This patch removes some "cold" fields from `struct io_issue_def`. The plan is to keep only highly used fields into `struct io_issue_def`, so, it may be hot in the cache. The hot fields are basically all the bitfields and the callback functions for .issue and .prep. The other less frequently used fields are now located in a secondary and cold struct, called `io_cold_def`. This is the size for the structs: Before: io_issue_def = 56 bytes After: io_issue_def = 24 bytes; io_cold_def = 40 bytes Signed-off-by: Breno Leitao <leitao@debian.org> --- io_uring/io_uring.c | 15 +- io_uring/opdef.c | 327 ++++++++++++++++++++++++++++++-------------- io_uring/opdef.h | 9 +- io_uring/rw.c | 2 +- 4 files changed, 238 insertions(+), 115 deletions(-)