@@ -641,6 +641,7 @@ static int unlock_request(struct fuse_req *req)
struct fuse_copy_state {
int write;
+ unsigned int cleanup_mode; /* Page cleanup mode (0/FOLL_GET/PIN) */
struct fuse_req *req;
struct iov_iter *iter;
struct pipe_buffer *pipebufs;
@@ -661,6 +662,11 @@ static void fuse_copy_init(struct fuse_copy_state *cs, int write,
cs->iter = iter;
}
+static void fuse_release_copy_page(struct fuse_copy_state *cs, struct page *page)
+{
+ page_put_unpin(page, cs->cleanup_mode);
+}
+
/* Unmap and put previous page of userspace buffer */
static void fuse_copy_finish(struct fuse_copy_state *cs)
{
@@ -675,7 +681,7 @@ static void fuse_copy_finish(struct fuse_copy_state *cs)
flush_dcache_page(cs->pg);
set_page_dirty_lock(cs->pg);
}
- put_page(cs->pg);
+ fuse_release_copy_page(cs, cs->pg);
}
cs->pg = NULL;
}
@@ -704,6 +710,7 @@ static int fuse_copy_fill(struct fuse_copy_state *cs)
BUG_ON(!cs->nr_segs);
cs->currbuf = buf;
+ cs->cleanup_mode = FOLL_GET;
cs->pg = buf->page;
cs->offset = buf->offset;
cs->len = buf->len;
@@ -722,6 +729,7 @@ static int fuse_copy_fill(struct fuse_copy_state *cs)
buf->len = 0;
cs->currbuf = buf;
+ cs->cleanup_mode = FOLL_GET;
cs->pg = page;
cs->offset = 0;
cs->len = PAGE_SIZE;
@@ -729,15 +737,18 @@ static int fuse_copy_fill(struct fuse_copy_state *cs)
cs->nr_segs++;
}
} else {
+ unsigned int gup_flags = cs->write ? FOLL_SOURCE_BUF : FOLL_DEST_BUF;
+ struct page **pages = &cs->pg;
size_t off;
- err = iov_iter_get_pages(cs->iter, &page, PAGE_SIZE, 1, &off,
- cs->write ? FOLL_SOURCE_BUF : FOLL_DEST_BUF);
+
+ err = iov_iter_extract_pages(cs->iter, &pages, PAGE_SIZE, 1,
+ gup_flags, &off);
if (err < 0)
return err;
BUG_ON(!err);
cs->len = err;
cs->offset = off;
- cs->pg = page;
+ cs->cleanup_mode = iov_iter_extract_mode(cs->iter, gup_flags);
}
return lock_request(cs->req);
@@ -899,10 +910,12 @@ static int fuse_ref_page(struct fuse_copy_state *cs, struct page *page,
if (cs->nr_segs >= cs->pipe->max_usage)
return -EIO;
- get_page(page);
+ err = try_grab_page(page, cs->cleanup_mode);
+ if (err < 0)
+ return err;
err = unlock_request(cs->req);
if (err) {
- put_page(page);
+ fuse_release_copy_page(cs, page);
return err;
}
@@ -624,6 +624,11 @@ void fuse_read_args_fill(struct fuse_io_args *ia, struct file *file, loff_t pos,
args->out_args[0].size = count;
}
+static void fuse_release_page(struct fuse_args_pages *ap, struct page *page)
+{
+ page_put_unpin(page, ap->cleanup_mode);
+}
+
static void fuse_release_user_pages(struct fuse_args_pages *ap,
bool should_dirty)
{
@@ -632,7 +637,7 @@ static void fuse_release_user_pages(struct fuse_args_pages *ap,
for (i = 0; i < ap->num_pages; i++) {
if (should_dirty)
set_page_dirty_lock(ap->pages[i]);
- put_page(ap->pages[i]);
+ fuse_release_page(ap, ap->pages[i]);
}
}
@@ -920,7 +925,7 @@ static void fuse_readpages_end(struct fuse_mount *fm, struct fuse_args *args,
else
SetPageError(page);
unlock_page(page);
- put_page(page);
+ fuse_release_page(ap, page);
}
if (ia->ff)
fuse_file_put(ia->ff, false, false);
@@ -1153,7 +1158,7 @@ static ssize_t fuse_send_write_pages(struct fuse_io_args *ia,
}
if (ia->write.page_locked && (i == ap->num_pages - 1))
unlock_page(page);
- put_page(page);
+ fuse_release_page(ap, page);
}
return err;
@@ -1172,6 +1177,7 @@ static ssize_t fuse_fill_write_pages(struct fuse_io_args *ia,
ap->args.in_pages = true;
ap->descs[0].offset = offset;
+ ap->cleanup_mode = FOLL_GET;
do {
size_t tmp;
@@ -1200,7 +1206,7 @@ static ssize_t fuse_fill_write_pages(struct fuse_io_args *ia,
if (!tmp) {
unlock_page(page);
- put_page(page);
+ fuse_release_page(ap, page);
goto again;
}
@@ -1393,9 +1399,12 @@ static int fuse_get_user_pages(struct fuse_args_pages *ap, struct iov_iter *ii,
size_t *nbytesp, int write,
unsigned int max_pages)
{
+ unsigned int gup_flags = write ? FOLL_SOURCE_BUF : FOLL_DEST_BUF;
size_t nbytes = 0; /* # bytes already packed in req */
ssize_t ret = 0;
+ ap->cleanup_mode = iov_iter_extract_mode(ii, gup_flags);
+
/* Special case for kernel I/O: can copy directly into the buffer */
if (iov_iter_is_kvec(ii)) {
unsigned long user_addr = fuse_get_user_addr(ii);
@@ -1412,12 +1421,13 @@ static int fuse_get_user_pages(struct fuse_args_pages *ap, struct iov_iter *ii,
}
while (nbytes < *nbytesp && ap->num_pages < max_pages) {
+ struct page **pages = &ap->pages[ap->num_pages];
unsigned npages;
size_t start;
- ret = iov_iter_get_pages(ii, &ap->pages[ap->num_pages],
- *nbytesp - nbytes,
- max_pages - ap->num_pages,
- &start, write ? FOLL_SOURCE_BUF : FOLL_DEST_BUF);
+ ret = iov_iter_extract_pages(ii, &pages,
+ *nbytesp - nbytes,
+ max_pages - ap->num_pages,
+ gup_flags, &start);
if (ret < 0)
break;
@@ -271,6 +271,7 @@ struct fuse_args_pages {
struct page **pages;
struct fuse_page_desc *descs;
unsigned int num_pages;
+ unsigned int cleanup_mode;
};
#define FUSE_ARGS(args) struct fuse_args args = {}
Convert the fuse code to use iov_iter_extract_pages() instead of iov_iter_get_pages(). This will pin pages or leave them unaltered rather than getting a ref on them as appropriate to the iterator. The pages need to be pinned for DIO-read rather than having refs taken on them to prevent VM copy-on-write from malfunctioning during a concurrent fork() (the result of the I/O would otherwise end up only visible to the child process and not the parent). Signed-off-by: David Howells <dhowells@redhat.com> cc: Miklos Szeredi <miklos@szeredi.hu> cc: Al Viro <viro@zeniv.linux.org.uk> cc: Christoph Hellwig <hch@lst.de> cc: linux-fsdevel@vger.kernel.org --- fs/fuse/dev.c | 25 +++++++++++++++++++------ fs/fuse/file.c | 26 ++++++++++++++++++-------- fs/fuse/fuse_i.h | 1 + 3 files changed, 38 insertions(+), 14 deletions(-)