@@ -9,6 +9,7 @@
#include <linux/module.h>
#include <linux/fscache-cache.h>
#include <linux/slab.h>
+#include <linux/netfs.h>
#include "internal.h"
/*
@@ -79,6 +79,7 @@ static void netfs_put_read_request(struct netfs_read_request *rreq)
netfs_rreq_clear_subreqs(rreq);
if (rreq->netfs_priv)
rreq->netfs_ops->cleanup(rreq->mapping, rreq->netfs_priv);
+ fscache_end_operation(&rreq->cache_resources);
kfree(rreq);
}
}
@@ -125,6 +126,31 @@ static void netfs_clear_unread(struct netfs_read_subrequest *subreq)
iov_iter_zero(iov_iter_count(&iter), &iter);
}
+static void netfs_cache_read_terminated(void *priv, ssize_t transferred_or_error)
+{
+ struct netfs_read_subrequest *subreq = priv;
+
+ netfs_subreq_terminated(subreq, transferred_or_error);
+}
+
+/*
+ * Issue a read against the cache.
+ * - Eats the caller's ref on subreq.
+ */
+static void netfs_read_from_cache(struct netfs_read_request *rreq,
+ struct netfs_read_subrequest *subreq,
+ bool seek_data)
+{
+ struct iov_iter iter;
+
+ iov_iter_xarray(&iter, READ, &rreq->mapping->i_pages,
+ subreq->start + subreq->transferred,
+ subreq->len - subreq->transferred);
+
+ fscache_read(&rreq->cache_resources, subreq->start, &iter, seek_data,
+ netfs_cache_read_terminated, subreq);
+}
+
/*
* Fill a subrequest region with zeroes.
*/
@@ -166,6 +192,122 @@ static void netfs_rreq_completed(struct netfs_read_request *rreq)
netfs_put_read_request(rreq);
}
+/*
+ * Deal with the completion of writing the data to the cache. We have to clear
+ * the PG_fscache bits on the pages involved and release the caller's ref.
+ *
+ * May be called in softirq mode and we inherit a ref from the caller.
+ */
+static void netfs_rreq_unmark_after_write(struct netfs_read_request *rreq)
+{
+ struct netfs_read_subrequest *subreq;
+ struct page *page;
+ pgoff_t unlocked = 0;
+ bool have_unlocked = false;
+
+ rcu_read_lock();
+
+ list_for_each_entry(subreq, &rreq->subrequests, rreq_link) {
+ XA_STATE(xas, &rreq->mapping->i_pages, subreq->start / PAGE_SIZE);
+
+ xas_for_each(&xas, page, (subreq->start + subreq->len - 1) / PAGE_SIZE) {
+ /* We might have multiple writes from the same huge
+ * page, but we mustn't unlock a page more than once.
+ */
+ if (have_unlocked && page->index <= unlocked)
+ continue;
+ unlocked = page->index;
+ unlock_page_fscache(page);
+ have_unlocked = true;
+ }
+ }
+
+ rcu_read_unlock();
+ netfs_rreq_completed(rreq);
+}
+
+static void netfs_rreq_copy_terminated(void *priv, ssize_t transferred_or_error)
+{
+ struct netfs_read_subrequest *subreq = priv;
+ struct netfs_read_request *rreq = subreq->rreq;
+
+ if (IS_ERR_VALUE(transferred_or_error))
+ subreq->error = transferred_or_error;
+ else
+ subreq->error = 0;
+
+ /* If we decrement nr_wr_ops to 0, the ref belongs to us. */
+ if (atomic_dec_and_test(&rreq->nr_wr_ops))
+ netfs_rreq_unmark_after_write(rreq);
+
+ netfs_put_subrequest(subreq);
+}
+
+/*
+ * Perform any outstanding writes to the cache. We inherit a ref from the
+ * caller.
+ */
+static void netfs_rreq_do_write_to_cache(struct netfs_read_request *rreq)
+{
+ struct netfs_read_subrequest *subreq, *next, *p;
+ struct iov_iter iter;
+
+ /* We don't want terminating writes trying to wake us up whilst we're
+ * still going through the list.
+ */
+ atomic_inc(&rreq->nr_wr_ops);
+
+ list_for_each_entry_safe(subreq, p, &rreq->subrequests, rreq_link) {
+ if (!test_bit(NETFS_SREQ_WRITE_TO_CACHE, &subreq->flags)) {
+ list_del_init(&subreq->rreq_link);
+ netfs_put_subrequest(subreq);
+ }
+ }
+
+ list_for_each_entry(subreq, &rreq->subrequests, rreq_link) {
+ /* Amalgamate adjacent writes */
+ while (!list_is_last(&subreq->rreq_link, &rreq->subrequests)) {
+ next = list_next_entry(subreq, rreq_link);
+ if (next->start > subreq->start + subreq->len)
+ break;
+ subreq->len += next->len;
+ list_del_init(&next->rreq_link);
+ netfs_put_subrequest(next);
+ }
+
+ iov_iter_xarray(&iter, WRITE, &rreq->mapping->i_pages,
+ subreq->start, subreq->len);
+
+ atomic_inc(&rreq->nr_wr_ops);
+ netfs_get_read_subrequest(subreq);
+ fscache_write(&rreq->cache_resources, subreq->start, &iter,
+ netfs_rreq_copy_terminated, subreq);
+ }
+
+ /* If we decrement nr_wr_ops to 0, the usage ref belongs to us. */
+ if (atomic_dec_and_test(&rreq->nr_wr_ops))
+ netfs_rreq_unmark_after_write(rreq);
+}
+
+static void netfs_rreq_write_to_cache_work(struct work_struct *work)
+{
+ struct netfs_read_request *rreq =
+ container_of(work, struct netfs_read_request, work);
+
+ netfs_rreq_do_write_to_cache(rreq);
+}
+
+static void netfs_rreq_write_to_cache(struct netfs_read_request *rreq)
+{
+ if (in_softirq()) {
+ rreq->work.func = netfs_rreq_write_to_cache_work;
+ if (!queue_work(system_unbound_wq, &rreq->work))
+ BUG();
+ } else {
+ netfs_rreq_do_write_to_cache(rreq);
+ }
+}
+
/*
* Unlock the pages in a read operation. We need to set PG_fscache on any
* pages we're going to write back before we unlock them.
@@ -258,7 +400,10 @@ static void netfs_rreq_short_read(struct netfs_read_request *rreq,
netfs_get_read_subrequest(subreq);
atomic_inc(&rreq->nr_rd_ops);
- netfs_read_from_server(rreq, subreq);
+ if (subreq->source == NETFS_READ_FROM_CACHE)
+ netfs_read_from_cache(rreq, subreq, true);
+ else
+ netfs_read_from_server(rreq, subreq);
}
/*
@@ -299,6 +444,25 @@ static bool netfs_rreq_perform_resubmissions(struct netfs_read_request *rreq)
return false;
}
+/*
+ * Check to see if the data read is still valid.
+ */
+static void netfs_rreq_is_still_valid(struct netfs_read_request *rreq)
+{
+ struct netfs_read_subrequest *subreq;
+
+ if (!rreq->netfs_ops->is_still_valid ||
+ rreq->netfs_ops->is_still_valid(rreq))
+ return;
+
+ list_for_each_entry(subreq, &rreq->subrequests, rreq_link) {
+ if (subreq->source == NETFS_READ_FROM_CACHE) {
+ subreq->error = -ESTALE;
+ __set_bit(NETFS_RREQ_INCOMPLETE_IO, &rreq->flags);
+ }
+ }
+}
+
/*
* Assess the state of a read request and decide what to do next.
*
@@ -308,6 +472,8 @@ static bool netfs_rreq_perform_resubmissions(struct netfs_read_request *rreq)
static void netfs_rreq_assess(struct netfs_read_request *rreq)
{
again:
+ netfs_rreq_is_still_valid(rreq);
+
if (!test_bit(NETFS_RREQ_FAILED, &rreq->flags) &&
test_bit(NETFS_RREQ_INCOMPLETE_IO, &rreq->flags)) {
if (netfs_rreq_perform_resubmissions(rreq))
@@ -320,6 +486,9 @@ static void netfs_rreq_assess(struct netfs_read_request *rreq)
clear_bit_unlock(NETFS_RREQ_IN_PROGRESS, &rreq->flags);
wake_up_bit(&rreq->flags, NETFS_RREQ_IN_PROGRESS);
+ if (test_bit(NETFS_RREQ_WRITE_TO_CACHE, &rreq->flags))
+ return netfs_rreq_write_to_cache(rreq);
+
netfs_rreq_completed(rreq);
}
@@ -434,7 +603,11 @@ static enum netfs_read_source netfs_cache_prepare_read(struct netfs_read_subrequ
loff_t i_size)
{
struct netfs_read_request *rreq = subreq->rreq;
+ struct fscache_op_resources *opr = &rreq->cache_resources;
+ const struct fscache_op_ops *ops = fscache_operation_valid(opr);
+ if (ops)
+ return ops->prepare_read(subreq, i_size);
if (subreq->start >= rreq->i_size)
return NETFS_FILL_WITH_ZEROES;
return NETFS_DOWNLOAD_FROM_SERVER;
@@ -524,6 +697,9 @@ static bool netfs_rreq_submit_slice(struct netfs_read_request *rreq,
case NETFS_DOWNLOAD_FROM_SERVER:
netfs_read_from_server(rreq, subreq);
break;
+ case NETFS_READ_FROM_CACHE:
+ netfs_read_from_cache(rreq, subreq, false);
+ break;
default:
BUG();
}
@@ -536,9 +712,24 @@ static bool netfs_rreq_submit_slice(struct netfs_read_request *rreq,
return false;
}
+static void netfs_cache_expand_readahead(struct netfs_read_request *rreq,
+ loff_t *_start, size_t *_len, loff_t i_size)
+{
+ struct fscache_op_resources *opr = &rreq->cache_resources;
+ const struct fscache_op_ops *ops = fscache_operation_valid(opr);
+
+ if (ops)
+ ops->expand_readahead(opr, _start, _len, i_size);
+}
+
static void netfs_rreq_expand(struct netfs_read_request *rreq,
struct readahead_control *ractl)
{
+ /* Give the cache a chance to change the request parameters. The
+ * resultant request must contain the original region.
+ */
+ netfs_cache_expand_readahead(rreq, &rreq->start, &rreq->len, rreq->i_size);
+
/* Give the netfs a chance to change the request parameters. The
* resultant request must contain the original region.
*/
@@ -600,6 +791,8 @@ void netfs_readahead(struct readahead_control *ractl,
rreq->start = readahead_pos(ractl);
rreq->len = readahead_length(ractl);
+ if (ops->begin_cache_operation)
+ ops->begin_cache_operation(rreq);
netfs_rreq_expand(rreq, ractl);
atomic_set(&rreq->nr_rd_ops, 1);
@@ -670,6 +863,8 @@ int netfs_readpage(struct file *file,
rreq->start = page->index * PAGE_SIZE;
rreq->len = thp_size(page);
+ if (ops->begin_cache_operation)
+ ops->begin_cache_operation(rreq);
netfs_get_read_request(rreq);
atomic_set(&rreq->nr_rd_ops, 1);
@@ -52,6 +52,7 @@ struct fscache_cookie;
struct fscache_netfs;
struct fscache_op_resources;
struct fscache_op_ops;
+struct netfs_read_subrequest;
typedef void (*fscache_io_terminated_t)(void *priv, ssize_t transferred_or_error);
@@ -174,6 +175,16 @@ struct fscache_op_ops {
struct iov_iter *iter,
fscache_io_terminated_t term_func,
void *term_func_priv);
+
+ /* Expand readahead request */
+ void (*expand_readahead)(struct fscache_op_resources *opr,
+ loff_t *_start, size_t *_len, loff_t i_size);
+
+ /* Prepare a read operation, shortening it to a cached/uncached
+ * boundary as appropriate.
+ */
+ enum netfs_read_source (*prepare_read)(struct netfs_read_subrequest *subreq,
+ loff_t i_size);
};
/*
@@ -12,6 +12,8 @@
#ifndef _LINUX_NETFS_H
#define _LINUX_NETFS_H
+#include <linux/fscache.h>
+
enum netfs_read_source {
NETFS_FILL_WITH_ZEROES,
NETFS_DOWNLOAD_FROM_SERVER,
@@ -47,10 +49,12 @@ struct netfs_read_subrequest {
struct netfs_read_request {
struct inode *inode; /* The file being accessed */
struct address_space *mapping; /* The mapping being accessed */
+ struct fscache_op_resources cache_resources;
struct list_head subrequests; /* Requests to fetch I/O from disk or net */
struct work_struct work;
void *netfs_priv; /* Private data for the netfs */
atomic_t nr_rd_ops; /* Number of read ops in progress */
+ atomic_t nr_wr_ops; /* Number of write ops in progress */
size_t submitted; /* Amount submitted for I/O so far */
size_t len; /* Length of the request */
short error; /* 0 or error that occurred */
@@ -73,6 +77,7 @@ struct netfs_read_request {
*/
struct netfs_read_request_ops {
void (*init_rreq)(struct netfs_read_request *rreq, struct file *file);
+ int (*begin_cache_operation)(struct netfs_read_request *rreq);
void (*expand_readahead)(struct netfs_read_request *rreq);
bool (*clamp_length)(struct netfs_read_subrequest *subreq);
void (*issue_op)(struct netfs_read_subrequest *subreq);
Add support to the netfs support read helpers for reading data from the cache instead of downloading it from the server and support for writing data just downloaded or cleared to the cache. Signed-off-by: David Howells <dhowells@redhat.com> --- fs/fscache/io.c | 1 fs/netfs/read_helper.c | 197 +++++++++++++++++++++++++++++++++++++++++++++++ include/linux/fscache.h | 11 +++ include/linux/netfs.h | 5 + 4 files changed, 213 insertions(+), 1 deletion(-)