diff mbox series

[07/10] io_uring/rsrc: inline switch_start fast path

Message ID 9619c1717a0e01f22c5fce2f1ba2735f804da0f2.1681395792.git.asml.silence@gmail.com (mailing list archive)
State New
Headers show
Series some rsrc fixes and clean ups | expand

Commit Message

Pavel Begunkov April 13, 2023, 2:28 p.m. UTC 
Inline the part of io_rsrc_node_switch_start() that checks whether the
cache is empty or not, as most of the times it will have some number of
entries in there.

Signed-off-by: Pavel Begunkov <asml.silence@gmail.com>
---
 io_uring/rsrc.c | 12 +++++-------
 io_uring/rsrc.h |  9 ++++++++-
 2 files changed, 13 insertions(+), 8 deletions(-)
diff mbox series

Patch

diff --git a/io_uring/rsrc.c b/io_uring/rsrc.c
index 5415a18844e0..bfa0b382c6c6 100644
--- a/io_uring/rsrc.c
+++ b/io_uring/rsrc.c
@@ -230,15 +230,13 @@  void io_rsrc_node_switch(struct io_ring_ctx *ctx,
 	ctx->rsrc_node = backup;
 }
 
-int io_rsrc_node_switch_start(struct io_ring_ctx *ctx)
+int __io_rsrc_node_switch_start(struct io_ring_ctx *ctx)
 {
-	if (io_alloc_cache_empty(&ctx->rsrc_node_cache)) {
-		struct io_rsrc_node *node = kzalloc(sizeof(*node), GFP_KERNEL);
+	struct io_rsrc_node *node = kzalloc(sizeof(*node), GFP_KERNEL);
 
-		if (!node)
-			return -ENOMEM;
-		io_alloc_cache_put(&ctx->rsrc_node_cache, &node->cache);
-	}
+	if (!node)
+		return -ENOMEM;
+	io_alloc_cache_put(&ctx->rsrc_node_cache, &node->cache);
 	return 0;
 }
 
diff --git a/io_uring/rsrc.h b/io_uring/rsrc.h
index 5dd2fcb28069..732496afed4c 100644
--- a/io_uring/rsrc.h
+++ b/io_uring/rsrc.h
@@ -69,7 +69,7 @@  void io_rsrc_put_tw(struct callback_head *cb);
 void io_rsrc_node_ref_zero(struct io_rsrc_node *node);
 void io_rsrc_put_work(struct work_struct *work);
 void io_rsrc_node_destroy(struct io_ring_ctx *ctx, struct io_rsrc_node *ref_node);
-int io_rsrc_node_switch_start(struct io_ring_ctx *ctx);
+int __io_rsrc_node_switch_start(struct io_ring_ctx *ctx);
 struct io_rsrc_node *io_rsrc_node_alloc(struct io_ring_ctx *ctx);
 int io_queue_rsrc_removal(struct io_rsrc_data *data, unsigned idx,
 			  struct io_rsrc_node *node, void *rsrc);
@@ -111,6 +111,13 @@  static inline int io_scm_file_account(struct io_ring_ctx *ctx,
 	return __io_scm_file_account(ctx, file);
 }
 
+static inline int io_rsrc_node_switch_start(struct io_ring_ctx *ctx)
+{
+	if (unlikely(io_alloc_cache_empty(&ctx->rsrc_node_cache)))
+		return __io_rsrc_node_switch_start(ctx);
+	return 0;
+}
+
 int io_register_files_update(struct io_ring_ctx *ctx, void __user *arg,
 			     unsigned nr_args);
 int io_register_rsrc_update(struct io_ring_ctx *ctx, void __user *arg,