diff mbox series

[4/8] io_uring/rw: defer reg buf vec import

Message ID d0bd00d88da98bf236d92a9a45eeb69db2d3bbaf.1741014186.git.asml.silence@gmail.com (mailing list archive)
State New
Headers show
Series Add support for vectored registered buffers | expand

Commit Message

Pavel Begunkov March 3, 2025, 3:50 p.m. UTC 
Import registered buffers for vectored reads and writes later at issue
time as we now do for other fixed ops.

Signed-off-by: Pavel Begunkov <asml.silence@gmail.com>
---
 include/linux/io_uring_types.h |  3 +++
 io_uring/rw.c                  | 36 +++++++++++++++++++++++++++++-----
 2 files changed, 34 insertions(+), 5 deletions(-)

Comments

Caleb Sander Mateos March 3, 2025, 11:37 p.m. UTC | #1 
On Mon, Mar 3, 2025 at 7:52 AM Pavel Begunkov <asml.silence@gmail.com> wrote:
>
> Import registered buffers for vectored reads and writes later at issue
> time as we now do for other fixed ops.
>
> Signed-off-by: Pavel Begunkov <asml.silence@gmail.com>
> ---
>  include/linux/io_uring_types.h |  3 +++
>  io_uring/rw.c                  | 36 +++++++++++++++++++++++++++++-----
>  2 files changed, 34 insertions(+), 5 deletions(-)
>
> diff --git a/include/linux/io_uring_types.h b/include/linux/io_uring_types.h
> index b770a2b12da6..d36fccda754b 100644
> --- a/include/linux/io_uring_types.h
> +++ b/include/linux/io_uring_types.h
> @@ -502,6 +502,7 @@ enum {
>         REQ_F_BUFFERS_COMMIT_BIT,
>         REQ_F_BUF_NODE_BIT,
>         REQ_F_HAS_METADATA_BIT,
> +       REQ_F_IMPORT_BUFFER_BIT,
>
>         /* not a real bit, just to check we're not overflowing the space */
>         __REQ_F_LAST_BIT,
> @@ -584,6 +585,8 @@ enum {
>         REQ_F_BUF_NODE          = IO_REQ_FLAG(REQ_F_BUF_NODE_BIT),
>         /* request has read/write metadata assigned */
>         REQ_F_HAS_METADATA      = IO_REQ_FLAG(REQ_F_HAS_METADATA_BIT),
> +       /* resolve padded iovec to registered buffers */
> +       REQ_F_IMPORT_BUFFER     = IO_REQ_FLAG(REQ_F_IMPORT_BUFFER_BIT),
>  };
>
>  typedef void (*io_req_tw_func_t)(struct io_kiocb *req, io_tw_token_t tw);
> diff --git a/io_uring/rw.c b/io_uring/rw.c
> index 4c4229f41aaa..33a7ab2a8664 100644
> --- a/io_uring/rw.c
> +++ b/io_uring/rw.c
> @@ -381,6 +381,24 @@ int io_prep_write_fixed(struct io_kiocb *req, const struct io_uring_sqe *sqe)
>         return __io_prep_rw(req, sqe, ITER_SOURCE);
>  }
>
> +static int io_rw_import_reg_vec(struct io_kiocb *req,
> +                               struct io_async_rw *io,
> +                               int ddir, unsigned int issue_flags)
> +{
> +       struct io_rw *rw = io_kiocb_to_cmd(req, struct io_rw);
> +       unsigned uvec_segs = rw->len;
> +       unsigned iovec_off = io->vec.nr - uvec_segs;
> +       int ret;
> +
> +       ret = io_import_reg_vec(ddir, &io->iter, req, &io->vec,
> +                               uvec_segs, iovec_off, issue_flags);
> +       if (unlikely(ret))
> +               return ret;
> +       iov_iter_save_state(&io->iter, &io->iter_state);
> +       req->flags &= ~REQ_F_IMPORT_BUFFER;
> +       return 0;
> +}
> +
>  static int io_rw_prep_reg_vec(struct io_kiocb *req, int ddir)
>  {
>         struct io_rw *rw = io_kiocb_to_cmd(req, struct io_rw);
> @@ -406,10 +424,8 @@ static int io_rw_prep_reg_vec(struct io_kiocb *req, int ddir)
>         if (IS_ERR(res))
>                 return PTR_ERR(res);
>
> -       ret = io_import_reg_vec(ddir, &io->iter, req, &io->vec,
> -                               uvec_segs, iovec_off, 0);
> -       iov_iter_save_state(&io->iter, &io->iter_state);
> -       return ret;
> +       req->flags |= REQ_F_IMPORT_BUFFER;
> +       return 0;

Looks like ddir is now unused in this function?

Best,
Caleb


>  }
>
>  int io_prep_readv_fixed(struct io_kiocb *req, const struct io_uring_sqe *sqe)
> @@ -906,7 +922,11 @@ static int __io_read(struct io_kiocb *req, unsigned int issue_flags)
>         ssize_t ret;
>         loff_t *ppos;
>
> -       if (io_do_buffer_select(req)) {
> +       if (req->flags & REQ_F_IMPORT_BUFFER) {
> +               ret = io_rw_import_reg_vec(req, io, ITER_DEST, issue_flags);
> +               if (unlikely(ret))
> +                       return ret;
> +       } else if (io_do_buffer_select(req)) {
>                 ret = io_import_rw_buffer(ITER_DEST, req, io, issue_flags);
>                 if (unlikely(ret < 0))
>                         return ret;
> @@ -1117,6 +1137,12 @@ int io_write(struct io_kiocb *req, unsigned int issue_flags)
>         ssize_t ret, ret2;
>         loff_t *ppos;
>
> +       if (req->flags & REQ_F_IMPORT_BUFFER) {
> +               ret = io_rw_import_reg_vec(req, io, ITER_SOURCE, issue_flags);
> +               if (unlikely(ret))
> +                       return ret;
> +       }
> +
>         ret = io_rw_init_file(req, FMODE_WRITE, WRITE);
>         if (unlikely(ret))
>                 return ret;
> --
> 2.48.1
>
>
diff mbox series

Patch

diff --git a/include/linux/io_uring_types.h b/include/linux/io_uring_types.h
index b770a2b12da6..d36fccda754b 100644
--- a/include/linux/io_uring_types.h
+++ b/include/linux/io_uring_types.h
@@ -502,6 +502,7 @@  enum {
 	REQ_F_BUFFERS_COMMIT_BIT,
 	REQ_F_BUF_NODE_BIT,
 	REQ_F_HAS_METADATA_BIT,
+	REQ_F_IMPORT_BUFFER_BIT,
 
 	/* not a real bit, just to check we're not overflowing the space */
 	__REQ_F_LAST_BIT,
@@ -584,6 +585,8 @@  enum {
 	REQ_F_BUF_NODE		= IO_REQ_FLAG(REQ_F_BUF_NODE_BIT),
 	/* request has read/write metadata assigned */
 	REQ_F_HAS_METADATA	= IO_REQ_FLAG(REQ_F_HAS_METADATA_BIT),
+	/* resolve padded iovec to registered buffers */
+	REQ_F_IMPORT_BUFFER	= IO_REQ_FLAG(REQ_F_IMPORT_BUFFER_BIT),
 };
 
 typedef void (*io_req_tw_func_t)(struct io_kiocb *req, io_tw_token_t tw);
diff --git a/io_uring/rw.c b/io_uring/rw.c
index 4c4229f41aaa..33a7ab2a8664 100644
--- a/io_uring/rw.c
+++ b/io_uring/rw.c
@@ -381,6 +381,24 @@  int io_prep_write_fixed(struct io_kiocb *req, const struct io_uring_sqe *sqe)
 	return __io_prep_rw(req, sqe, ITER_SOURCE);
 }
 
+static int io_rw_import_reg_vec(struct io_kiocb *req,
+				struct io_async_rw *io,
+				int ddir, unsigned int issue_flags)
+{
+	struct io_rw *rw = io_kiocb_to_cmd(req, struct io_rw);
+	unsigned uvec_segs = rw->len;
+	unsigned iovec_off = io->vec.nr - uvec_segs;
+	int ret;
+
+	ret = io_import_reg_vec(ddir, &io->iter, req, &io->vec,
+				uvec_segs, iovec_off, issue_flags);
+	if (unlikely(ret))
+		return ret;
+	iov_iter_save_state(&io->iter, &io->iter_state);
+	req->flags &= ~REQ_F_IMPORT_BUFFER;
+	return 0;
+}
+
 static int io_rw_prep_reg_vec(struct io_kiocb *req, int ddir)
 {
 	struct io_rw *rw = io_kiocb_to_cmd(req, struct io_rw);
@@ -406,10 +424,8 @@  static int io_rw_prep_reg_vec(struct io_kiocb *req, int ddir)
 	if (IS_ERR(res))
 		return PTR_ERR(res);
 
-	ret = io_import_reg_vec(ddir, &io->iter, req, &io->vec,
-				uvec_segs, iovec_off, 0);
-	iov_iter_save_state(&io->iter, &io->iter_state);
-	return ret;
+	req->flags |= REQ_F_IMPORT_BUFFER;
+	return 0;
 }
 
 int io_prep_readv_fixed(struct io_kiocb *req, const struct io_uring_sqe *sqe)
@@ -906,7 +922,11 @@  static int __io_read(struct io_kiocb *req, unsigned int issue_flags)
 	ssize_t ret;
 	loff_t *ppos;
 
-	if (io_do_buffer_select(req)) {
+	if (req->flags & REQ_F_IMPORT_BUFFER) {
+		ret = io_rw_import_reg_vec(req, io, ITER_DEST, issue_flags);
+		if (unlikely(ret))
+			return ret;
+	} else if (io_do_buffer_select(req)) {
 		ret = io_import_rw_buffer(ITER_DEST, req, io, issue_flags);
 		if (unlikely(ret < 0))
 			return ret;
@@ -1117,6 +1137,12 @@  int io_write(struct io_kiocb *req, unsigned int issue_flags)
 	ssize_t ret, ret2;
 	loff_t *ppos;
 
+	if (req->flags & REQ_F_IMPORT_BUFFER) {
+		ret = io_rw_import_reg_vec(req, io, ITER_SOURCE, issue_flags);
+		if (unlikely(ret))
+			return ret;
+	}
+
 	ret = io_rw_init_file(req, FMODE_WRITE, WRITE);
 	if (unlikely(ret))
 		return ret;