diff mbox series

[1/2] fs: fork do_splice_copy_file_range() from do_splice_direct()

Message ID 20231129200709.3154370-2-amir73il@gmail.com (mailing list archive)
State New
Headers show
Series Avert possible deadlock with splice() and fanotify | expand

Commit Message

Amir Goldstein Nov. 29, 2023, 8:07 p.m. UTC
The new helper is meant to be called from context of ->copy_file_range()
methods instead of do_splice_direct().

Currently, the only difference is that do_splice_copy_file_range() does
not take a splice flags argument and it asserts that file_start_write()
was called.

Soon, do_splice_direct() will be called without file_start_write() held.

Use the new helper from __ceph_copy_file_range(), that was incorrectly
passing the copy_file_range() flags argument as splice flags argument
to do_splice_direct(). the value of flags was 0, so no actual bug fix.

Move the definition of both helpers to linux/splice.h.

Signed-off-by: Amir Goldstein <amir73il@gmail.com>
---
 fs/ceph/file.c         |  9 ++---
 fs/read_write.c        |  6 ++--
 fs/splice.c            | 82 ++++++++++++++++++++++++++++++------------
 include/linux/fs.h     |  2 --
 include/linux/splice.h | 13 ++++---
 5 files changed, 75 insertions(+), 37 deletions(-)

Comments

Amir Goldstein Nov. 30, 2023, 10:09 a.m. UTC | #1
On Wed, Nov 29, 2023 at 10:07 PM Amir Goldstein <amir73il@gmail.com> wrote:
>
> The new helper is meant to be called from context of ->copy_file_range()
> methods instead of do_splice_direct().
>
> Currently, the only difference is that do_splice_copy_file_range() does
> not take a splice flags argument and it asserts that file_start_write()
> was called.
>
> Soon, do_splice_direct() will be called without file_start_write() held.
>
> Use the new helper from __ceph_copy_file_range(), that was incorrectly
> passing the copy_file_range() flags argument as splice flags argument
> to do_splice_direct(). the value of flags was 0, so no actual bug fix.
>
> Move the definition of both helpers to linux/splice.h.
>
> Signed-off-by: Amir Goldstein <amir73il@gmail.com>
> ---
>  fs/ceph/file.c         |  9 ++---
>  fs/read_write.c        |  6 ++--
>  fs/splice.c            | 82 ++++++++++++++++++++++++++++++------------
>  include/linux/fs.h     |  2 --
>  include/linux/splice.h | 13 ++++---
>  5 files changed, 75 insertions(+), 37 deletions(-)
>
> diff --git a/fs/ceph/file.c b/fs/ceph/file.c
> index 3b5aae29e944..7c2db78e2c6e 100644
> --- a/fs/ceph/file.c
> +++ b/fs/ceph/file.c
> @@ -12,6 +12,7 @@
>  #include <linux/falloc.h>
>  #include <linux/iversion.h>
>  #include <linux/ktime.h>
> +#include <linux/splice.h>
>
>  #include "super.h"
>  #include "mds_client.h"
> @@ -3010,8 +3011,8 @@ static ssize_t __ceph_copy_file_range(struct file *src_file, loff_t src_off,
>                  * {read,write}_iter, which will get caps again.
>                  */
>                 put_rd_wr_caps(src_ci, src_got, dst_ci, dst_got);
> -               ret = do_splice_direct(src_file, &src_off, dst_file,
> -                                      &dst_off, src_objlen, flags);
> +               ret = do_splice_copy_file_range(src_file, &src_off, dst_file,
> +                                               &dst_off, src_objlen);
>                 /* Abort on short copies or on error */
>                 if (ret < (long)src_objlen) {
>                         doutc(cl, "Failed partial copy (%zd)\n", ret);
> @@ -3065,8 +3066,8 @@ static ssize_t __ceph_copy_file_range(struct file *src_file, loff_t src_off,
>          */
>         if (len && (len < src_ci->i_layout.object_size)) {
>                 doutc(cl, "Final partial copy of %zu bytes\n", len);
> -               bytes = do_splice_direct(src_file, &src_off, dst_file,
> -                                        &dst_off, len, flags);
> +               bytes = do_splice_copy_file_range(src_file, &src_off, dst_file,
> +                                                 &dst_off, len);
>                 if (bytes > 0)
>                         ret += bytes;
>                 else
> diff --git a/fs/read_write.c b/fs/read_write.c
> index f791555fa246..555514cdad53 100644
> --- a/fs/read_write.c
> +++ b/fs/read_write.c
> @@ -1423,10 +1423,8 @@ ssize_t generic_copy_file_range(struct file *file_in, loff_t pos_in,
>                                 struct file *file_out, loff_t pos_out,
>                                 size_t len, unsigned int flags)
>  {
> -       lockdep_assert(file_write_started(file_out));
> -
> -       return do_splice_direct(file_in, &pos_in, file_out, &pos_out,
> -                               len > MAX_RW_COUNT ? MAX_RW_COUNT : len, 0);
> +       return do_splice_copy_file_range(file_in, &pos_in, file_out, &pos_out,
> +                               len > MAX_RW_COUNT ? MAX_RW_COUNT : len);
>  }
>  EXPORT_SYMBOL(generic_copy_file_range);
>
> diff --git a/fs/splice.c b/fs/splice.c
> index 3fce5f6072dd..3bb4936f8b70 100644
> --- a/fs/splice.c
> +++ b/fs/splice.c
> @@ -1158,8 +1158,15 @@ static int direct_splice_actor(struct pipe_inode_info *pipe,
>  {
>         struct file *file = sd->u.file;
>
> -       return do_splice_from(pipe, file, sd->opos, sd->total_len,
> -                             sd->flags);
> +       return do_splice_from(pipe, file, sd->opos, sd->total_len, sd->flags);
> +}
> +
> +static int copy_file_range_splice_actor(struct pipe_inode_info *pipe,
> +                                       struct splice_desc *sd)
> +{
> +       struct file *file = sd->u.file;
> +
> +       return do_splice_from(pipe, file, sd->opos, sd->total_len, sd->flags);
>  }
>
>  static void direct_file_splice_eof(struct splice_desc *sd)
> @@ -1170,25 +1177,10 @@ static void direct_file_splice_eof(struct splice_desc *sd)
>                 file->f_op->splice_eof(file);
>  }
>
> -/**
> - * do_splice_direct - splices data directly between two files
> - * @in:                file to splice from
> - * @ppos:      input file offset
> - * @out:       file to splice to
> - * @opos:      output file offset
> - * @len:       number of bytes to splice
> - * @flags:     splice modifier flags
> - *
> - * Description:
> - *    For use by do_sendfile(). splice can easily emulate sendfile, but
> - *    doing it in the application would incur an extra system call
> - *    (splice in + splice out, as compared to just sendfile()). So this helper
> - *    can splice directly through a process-private pipe.
> - *
> - * Callers already called rw_verify_area() on the entire range.
> - */
> -long do_splice_direct(struct file *in, loff_t *ppos, struct file *out,
> -                     loff_t *opos, size_t len, unsigned int flags)
> +static long do_splice_direct_actor(struct file *in, loff_t *ppos,
> +                                  struct file *out, loff_t *opos,
> +                                  size_t len, unsigned int flags,
> +                                  splice_direct_actor *actor)
>  {
>         struct splice_desc sd = {
>                 .len            = len,
> @@ -1207,14 +1199,60 @@ long do_splice_direct(struct file *in, loff_t *ppos, struct file *out,
>         if (unlikely(out->f_flags & O_APPEND))
>                 return -EINVAL;
>
> -       ret = splice_direct_to_actor(in, &sd, direct_splice_actor);
> +       ret = splice_direct_to_actor(in, &sd, actor);
>         if (ret > 0)
>                 *ppos = sd.pos;
>
>         return ret;
>  }
> +/**
> + * do_splice_direct - splices data directly between two files
> + * @in:                file to splice from
> + * @ppos:      input file offset
> + * @out:       file to splice to
> + * @opos:      output file offset
> + * @len:       number of bytes to splice
> + * @flags:     splice modifier flags
> + *
> + * Description:
> + *    For use by do_sendfile(). splice can easily emulate sendfile, but
> + *    doing it in the application would incur an extra system call
> + *    (splice in + splice out, as compared to just sendfile()). So this helper
> + *    can splice directly through a process-private pipe.
> + *
> + * Callers already called rw_verify_area() on the entire range.
> + */
> +long do_splice_direct(struct file *in, loff_t *ppos, struct file *out,
> +                     loff_t *opos, size_t len, unsigned int flags)
> +{
> +       return do_splice_direct_actor(in, ppos, out, opos, len, flags,
> +                                     direct_splice_actor);
> +}
>  EXPORT_SYMBOL(do_splice_direct);
>
> +/**
> + * do_splice_copy_file_range - splices data for copy_file_range()
> + * @in:                file to splice from
> + * @ppos:      input file offset
> + * @out:       file to splice to
> + * @opos:      output file offset
> + * @len:       number of bytes to splice
> + *
> + * Description:
> + *    For use by generic_copy_file_range() and ->copy_file_range() methods.
> + *
> + * Callers already called rw_verify_area() on the entire range.
> + */
> +long do_splice_copy_file_range(struct file *in, loff_t *ppos, struct file *out,
> +                              loff_t *opos, size_t len)

FYI, I renamed do_splice_vfs_copy_file_range => splice_file_range in v2
for brevity.

Thanks,
Amir.
Christian Brauner Nov. 30, 2023, 1:18 p.m. UTC | #2
On Wed, Nov 29, 2023 at 10:07:08PM +0200, Amir Goldstein wrote:
> The new helper is meant to be called from context of ->copy_file_range()
> methods instead of do_splice_direct().
> 
> Currently, the only difference is that do_splice_copy_file_range() does
> not take a splice flags argument and it asserts that file_start_write()
> was called.
> 
> Soon, do_splice_direct() will be called without file_start_write() held.
> 
> Use the new helper from __ceph_copy_file_range(), that was incorrectly
> passing the copy_file_range() flags argument as splice flags argument
> to do_splice_direct(). the value of flags was 0, so no actual bug fix.
> 
> Move the definition of both helpers to linux/splice.h.
> 
> Signed-off-by: Amir Goldstein <amir73il@gmail.com>
> ---
>  fs/ceph/file.c         |  9 ++---
>  fs/read_write.c        |  6 ++--
>  fs/splice.c            | 82 ++++++++++++++++++++++++++++++------------
>  include/linux/fs.h     |  2 --
>  include/linux/splice.h | 13 ++++---
>  5 files changed, 75 insertions(+), 37 deletions(-)
> 
> diff --git a/fs/ceph/file.c b/fs/ceph/file.c
> index 3b5aae29e944..7c2db78e2c6e 100644
> --- a/fs/ceph/file.c
> +++ b/fs/ceph/file.c
> @@ -12,6 +12,7 @@
>  #include <linux/falloc.h>
>  #include <linux/iversion.h>
>  #include <linux/ktime.h>
> +#include <linux/splice.h>
>  
>  #include "super.h"
>  #include "mds_client.h"
> @@ -3010,8 +3011,8 @@ static ssize_t __ceph_copy_file_range(struct file *src_file, loff_t src_off,
>  		 * {read,write}_iter, which will get caps again.
>  		 */
>  		put_rd_wr_caps(src_ci, src_got, dst_ci, dst_got);
> -		ret = do_splice_direct(src_file, &src_off, dst_file,
> -				       &dst_off, src_objlen, flags);
> +		ret = do_splice_copy_file_range(src_file, &src_off, dst_file,
> +						&dst_off, src_objlen);
>  		/* Abort on short copies or on error */
>  		if (ret < (long)src_objlen) {
>  			doutc(cl, "Failed partial copy (%zd)\n", ret);
> @@ -3065,8 +3066,8 @@ static ssize_t __ceph_copy_file_range(struct file *src_file, loff_t src_off,
>  	 */
>  	if (len && (len < src_ci->i_layout.object_size)) {
>  		doutc(cl, "Final partial copy of %zu bytes\n", len);
> -		bytes = do_splice_direct(src_file, &src_off, dst_file,
> -					 &dst_off, len, flags);
> +		bytes = do_splice_copy_file_range(src_file, &src_off, dst_file,
> +						  &dst_off, len);
>  		if (bytes > 0)
>  			ret += bytes;
>  		else
> diff --git a/fs/read_write.c b/fs/read_write.c
> index f791555fa246..555514cdad53 100644
> --- a/fs/read_write.c
> +++ b/fs/read_write.c
> @@ -1423,10 +1423,8 @@ ssize_t generic_copy_file_range(struct file *file_in, loff_t pos_in,
>  				struct file *file_out, loff_t pos_out,
>  				size_t len, unsigned int flags)
>  {

Hm, the low-level helper takes a @flags argument but it's completely
ignored. I think that helper should remove it or it should check:

if (flags)
	return -EINVAL;

in case it's ever called from codepaths where @flags hasn't been
sanitized imho.

> -	lockdep_assert(file_write_started(file_out));
> -
> -	return do_splice_direct(file_in, &pos_in, file_out, &pos_out,
> -				len > MAX_RW_COUNT ? MAX_RW_COUNT : len, 0);
> +	return do_splice_copy_file_range(file_in, &pos_in, file_out, &pos_out,
> +				len > MAX_RW_COUNT ? MAX_RW_COUNT : len);

clamp(len, 0, MAX_RW_COUNT)

?
Jan Kara Nov. 30, 2023, 1:30 p.m. UTC | #3
On Thu 30-11-23 12:09:09, Amir Goldstein wrote:
> On Wed, Nov 29, 2023 at 10:07 PM Amir Goldstein <amir73il@gmail.com> wrote:
> >
> > The new helper is meant to be called from context of ->copy_file_range()
> > methods instead of do_splice_direct().
> >
> > Currently, the only difference is that do_splice_copy_file_range() does
> > not take a splice flags argument and it asserts that file_start_write()
> > was called.
> >
> > Soon, do_splice_direct() will be called without file_start_write() held.
> >
> > Use the new helper from __ceph_copy_file_range(), that was incorrectly
> > passing the copy_file_range() flags argument as splice flags argument
> > to do_splice_direct(). the value of flags was 0, so no actual bug fix.
> >
> > Move the definition of both helpers to linux/splice.h.
> >
> > Signed-off-by: Amir Goldstein <amir73il@gmail.com>
...
> > +/**
> > + * do_splice_copy_file_range - splices data for copy_file_range()
> > + * @in:                file to splice from
> > + * @ppos:      input file offset
> > + * @out:       file to splice to
> > + * @opos:      output file offset
> > + * @len:       number of bytes to splice
> > + *
> > + * Description:
> > + *    For use by generic_copy_file_range() and ->copy_file_range() methods.
> > + *
> > + * Callers already called rw_verify_area() on the entire range.
> > + */
> > +long do_splice_copy_file_range(struct file *in, loff_t *ppos, struct file *out,
> > +                              loff_t *opos, size_t len)
> 
> FYI, I renamed do_splice_vfs_copy_file_range => splice_file_range in v2
> for brevity.

Yeah, after the rename things look better :). Otherwise I didn't find any
problem so feel free to add:

Reviewed-by: Jan Kara <jack@suse.cz>

								Honza
Amir Goldstein Nov. 30, 2023, 1:37 p.m. UTC | #4
On Thu, Nov 30, 2023 at 3:18 PM Christian Brauner <brauner@kernel.org> wrote:
>
> On Wed, Nov 29, 2023 at 10:07:08PM +0200, Amir Goldstein wrote:
> > The new helper is meant to be called from context of ->copy_file_range()
> > methods instead of do_splice_direct().
> >
> > Currently, the only difference is that do_splice_copy_file_range() does
> > not take a splice flags argument and it asserts that file_start_write()
> > was called.
> >
> > Soon, do_splice_direct() will be called without file_start_write() held.
> >
> > Use the new helper from __ceph_copy_file_range(), that was incorrectly
> > passing the copy_file_range() flags argument as splice flags argument
> > to do_splice_direct(). the value of flags was 0, so no actual bug fix.
> >
> > Move the definition of both helpers to linux/splice.h.
> >
> > Signed-off-by: Amir Goldstein <amir73il@gmail.com>
> > ---
> >  fs/ceph/file.c         |  9 ++---
> >  fs/read_write.c        |  6 ++--
> >  fs/splice.c            | 82 ++++++++++++++++++++++++++++++------------
> >  include/linux/fs.h     |  2 --
> >  include/linux/splice.h | 13 ++++---
> >  5 files changed, 75 insertions(+), 37 deletions(-)
> >
> > diff --git a/fs/ceph/file.c b/fs/ceph/file.c
> > index 3b5aae29e944..7c2db78e2c6e 100644
> > --- a/fs/ceph/file.c
> > +++ b/fs/ceph/file.c
> > @@ -12,6 +12,7 @@
> >  #include <linux/falloc.h>
> >  #include <linux/iversion.h>
> >  #include <linux/ktime.h>
> > +#include <linux/splice.h>
> >
> >  #include "super.h"
> >  #include "mds_client.h"
> > @@ -3010,8 +3011,8 @@ static ssize_t __ceph_copy_file_range(struct file *src_file, loff_t src_off,
> >                * {read,write}_iter, which will get caps again.
> >                */
> >               put_rd_wr_caps(src_ci, src_got, dst_ci, dst_got);
> > -             ret = do_splice_direct(src_file, &src_off, dst_file,
> > -                                    &dst_off, src_objlen, flags);
> > +             ret = do_splice_copy_file_range(src_file, &src_off, dst_file,
> > +                                             &dst_off, src_objlen);
> >               /* Abort on short copies or on error */
> >               if (ret < (long)src_objlen) {
> >                       doutc(cl, "Failed partial copy (%zd)\n", ret);
> > @@ -3065,8 +3066,8 @@ static ssize_t __ceph_copy_file_range(struct file *src_file, loff_t src_off,
> >        */
> >       if (len && (len < src_ci->i_layout.object_size)) {
> >               doutc(cl, "Final partial copy of %zu bytes\n", len);
> > -             bytes = do_splice_direct(src_file, &src_off, dst_file,
> > -                                      &dst_off, len, flags);
> > +             bytes = do_splice_copy_file_range(src_file, &src_off, dst_file,
> > +                                               &dst_off, len);
> >               if (bytes > 0)
> >                       ret += bytes;
> >               else
> > diff --git a/fs/read_write.c b/fs/read_write.c
> > index f791555fa246..555514cdad53 100644
> > --- a/fs/read_write.c
> > +++ b/fs/read_write.c
> > @@ -1423,10 +1423,8 @@ ssize_t generic_copy_file_range(struct file *file_in, loff_t pos_in,
> >                               struct file *file_out, loff_t pos_out,
> >                               size_t len, unsigned int flags)
> >  {
>
> Hm, the low-level helper takes a @flags argument but it's completely
> ignored. I think that helper should remove it or it should check:
>
> if (flags)
>         return -EINVAL;
>

It's a good point.
The upstream code and in this v1, generic_copy_file_range() can actually
be called with flag COPY_FILE_SPLICE, but it is a mistake that
I fixed it in my branch for v2, so in v2 I can add this check.

> in case it's ever called from codepaths where @flags hasn't been
> sanitized imho.
>
> > -     lockdep_assert(file_write_started(file_out));
> > -
> > -     return do_splice_direct(file_in, &pos_in, file_out, &pos_out,
> > -                             len > MAX_RW_COUNT ? MAX_RW_COUNT : len, 0);
> > +     return do_splice_copy_file_range(file_in, &pos_in, file_out, &pos_out,
> > +                             len > MAX_RW_COUNT ? MAX_RW_COUNT : len);
>
> clamp(len, 0, MAX_RW_COUNT)
>

It is a low level helper, so I don't want to worry about negative len value.
Already changed to min_t(size_t, len, MAX_RW_COUNT) in my branch.

Thanks!
Amir.
diff mbox series

Patch

diff --git a/fs/ceph/file.c b/fs/ceph/file.c
index 3b5aae29e944..7c2db78e2c6e 100644
--- a/fs/ceph/file.c
+++ b/fs/ceph/file.c
@@ -12,6 +12,7 @@ 
 #include <linux/falloc.h>
 #include <linux/iversion.h>
 #include <linux/ktime.h>
+#include <linux/splice.h>
 
 #include "super.h"
 #include "mds_client.h"
@@ -3010,8 +3011,8 @@  static ssize_t __ceph_copy_file_range(struct file *src_file, loff_t src_off,
 		 * {read,write}_iter, which will get caps again.
 		 */
 		put_rd_wr_caps(src_ci, src_got, dst_ci, dst_got);
-		ret = do_splice_direct(src_file, &src_off, dst_file,
-				       &dst_off, src_objlen, flags);
+		ret = do_splice_copy_file_range(src_file, &src_off, dst_file,
+						&dst_off, src_objlen);
 		/* Abort on short copies or on error */
 		if (ret < (long)src_objlen) {
 			doutc(cl, "Failed partial copy (%zd)\n", ret);
@@ -3065,8 +3066,8 @@  static ssize_t __ceph_copy_file_range(struct file *src_file, loff_t src_off,
 	 */
 	if (len && (len < src_ci->i_layout.object_size)) {
 		doutc(cl, "Final partial copy of %zu bytes\n", len);
-		bytes = do_splice_direct(src_file, &src_off, dst_file,
-					 &dst_off, len, flags);
+		bytes = do_splice_copy_file_range(src_file, &src_off, dst_file,
+						  &dst_off, len);
 		if (bytes > 0)
 			ret += bytes;
 		else
diff --git a/fs/read_write.c b/fs/read_write.c
index f791555fa246..555514cdad53 100644
--- a/fs/read_write.c
+++ b/fs/read_write.c
@@ -1423,10 +1423,8 @@  ssize_t generic_copy_file_range(struct file *file_in, loff_t pos_in,
 				struct file *file_out, loff_t pos_out,
 				size_t len, unsigned int flags)
 {
-	lockdep_assert(file_write_started(file_out));
-
-	return do_splice_direct(file_in, &pos_in, file_out, &pos_out,
-				len > MAX_RW_COUNT ? MAX_RW_COUNT : len, 0);
+	return do_splice_copy_file_range(file_in, &pos_in, file_out, &pos_out,
+				len > MAX_RW_COUNT ? MAX_RW_COUNT : len);
 }
 EXPORT_SYMBOL(generic_copy_file_range);
 
diff --git a/fs/splice.c b/fs/splice.c
index 3fce5f6072dd..3bb4936f8b70 100644
--- a/fs/splice.c
+++ b/fs/splice.c
@@ -1158,8 +1158,15 @@  static int direct_splice_actor(struct pipe_inode_info *pipe,
 {
 	struct file *file = sd->u.file;
 
-	return do_splice_from(pipe, file, sd->opos, sd->total_len,
-			      sd->flags);
+	return do_splice_from(pipe, file, sd->opos, sd->total_len, sd->flags);
+}
+
+static int copy_file_range_splice_actor(struct pipe_inode_info *pipe,
+					struct splice_desc *sd)
+{
+	struct file *file = sd->u.file;
+
+	return do_splice_from(pipe, file, sd->opos, sd->total_len, sd->flags);
 }
 
 static void direct_file_splice_eof(struct splice_desc *sd)
@@ -1170,25 +1177,10 @@  static void direct_file_splice_eof(struct splice_desc *sd)
 		file->f_op->splice_eof(file);
 }
 
-/**
- * do_splice_direct - splices data directly between two files
- * @in:		file to splice from
- * @ppos:	input file offset
- * @out:	file to splice to
- * @opos:	output file offset
- * @len:	number of bytes to splice
- * @flags:	splice modifier flags
- *
- * Description:
- *    For use by do_sendfile(). splice can easily emulate sendfile, but
- *    doing it in the application would incur an extra system call
- *    (splice in + splice out, as compared to just sendfile()). So this helper
- *    can splice directly through a process-private pipe.
- *
- * Callers already called rw_verify_area() on the entire range.
- */
-long do_splice_direct(struct file *in, loff_t *ppos, struct file *out,
-		      loff_t *opos, size_t len, unsigned int flags)
+static long do_splice_direct_actor(struct file *in, loff_t *ppos,
+				   struct file *out, loff_t *opos,
+				   size_t len, unsigned int flags,
+				   splice_direct_actor *actor)
 {
 	struct splice_desc sd = {
 		.len		= len,
@@ -1207,14 +1199,60 @@  long do_splice_direct(struct file *in, loff_t *ppos, struct file *out,
 	if (unlikely(out->f_flags & O_APPEND))
 		return -EINVAL;
 
-	ret = splice_direct_to_actor(in, &sd, direct_splice_actor);
+	ret = splice_direct_to_actor(in, &sd, actor);
 	if (ret > 0)
 		*ppos = sd.pos;
 
 	return ret;
 }
+/**
+ * do_splice_direct - splices data directly between two files
+ * @in:		file to splice from
+ * @ppos:	input file offset
+ * @out:	file to splice to
+ * @opos:	output file offset
+ * @len:	number of bytes to splice
+ * @flags:	splice modifier flags
+ *
+ * Description:
+ *    For use by do_sendfile(). splice can easily emulate sendfile, but
+ *    doing it in the application would incur an extra system call
+ *    (splice in + splice out, as compared to just sendfile()). So this helper
+ *    can splice directly through a process-private pipe.
+ *
+ * Callers already called rw_verify_area() on the entire range.
+ */
+long do_splice_direct(struct file *in, loff_t *ppos, struct file *out,
+		      loff_t *opos, size_t len, unsigned int flags)
+{
+	return do_splice_direct_actor(in, ppos, out, opos, len, flags,
+				      direct_splice_actor);
+}
 EXPORT_SYMBOL(do_splice_direct);
 
+/**
+ * do_splice_copy_file_range - splices data for copy_file_range()
+ * @in:		file to splice from
+ * @ppos:	input file offset
+ * @out:	file to splice to
+ * @opos:	output file offset
+ * @len:	number of bytes to splice
+ *
+ * Description:
+ *    For use by generic_copy_file_range() and ->copy_file_range() methods.
+ *
+ * Callers already called rw_verify_area() on the entire range.
+ */
+long do_splice_copy_file_range(struct file *in, loff_t *ppos, struct file *out,
+			       loff_t *opos, size_t len)
+{
+	lockdep_assert(file_write_started(out));
+
+	return do_splice_direct_actor(in, ppos, out, opos, len, 0,
+				      copy_file_range_splice_actor);
+}
+EXPORT_SYMBOL(do_splice_copy_file_range);
+
 static int wait_for_space(struct pipe_inode_info *pipe, unsigned flags)
 {
 	for (;;) {
diff --git a/include/linux/fs.h b/include/linux/fs.h
index ae0e2fb7bcea..04422a0eccdd 100644
--- a/include/linux/fs.h
+++ b/include/linux/fs.h
@@ -3052,8 +3052,6 @@  ssize_t copy_splice_read(struct file *in, loff_t *ppos,
 			 size_t len, unsigned int flags);
 extern ssize_t iter_file_splice_write(struct pipe_inode_info *,
 		struct file *, loff_t *, size_t, unsigned int);
-extern long do_splice_direct(struct file *in, loff_t *ppos, struct file *out,
-		loff_t *opos, size_t len, unsigned int flags);
 
 
 extern void
diff --git a/include/linux/splice.h b/include/linux/splice.h
index 6c461573434d..11e62b641d69 100644
--- a/include/linux/splice.h
+++ b/include/linux/splice.h
@@ -80,11 +80,14 @@  extern ssize_t add_to_pipe(struct pipe_inode_info *,
 long vfs_splice_read(struct file *in, loff_t *ppos,
 		     struct pipe_inode_info *pipe, size_t len,
 		     unsigned int flags);
-extern ssize_t splice_direct_to_actor(struct file *, struct splice_desc *,
-				      splice_direct_actor *);
-extern long do_splice(struct file *in, loff_t *off_in,
-		      struct file *out, loff_t *off_out,
-		      size_t len, unsigned int flags);
+ssize_t splice_direct_to_actor(struct file *file, struct splice_desc *sd,
+			       splice_direct_actor *actor);
+long do_splice(struct file *in, loff_t *off_in, struct file *out,
+	       loff_t *off_out, size_t len, unsigned int flags);
+long do_splice_direct(struct file *in, loff_t *ppos, struct file *out,
+		      loff_t *opos, size_t len, unsigned int flags);
+long do_splice_copy_file_range(struct file *in, loff_t *ppos, struct file *out,
+			       loff_t *opos, size_t len);
 
 extern long do_tee(struct file *in, struct file *out, size_t len,
 		   unsigned int flags);