diff mbox

[v3,01/49] pnfs: Prepare for flexfiles by pulling out common code

Message ID 1420590534-84063-2-git-send-email-loghyr@primarydata.com (mailing list archive)
State New, archived
Headers show

Commit Message

Thomas Haynes Jan. 7, 2015, 12:28 a.m. UTC
The flexfilelayout driver will share some common code
with the filelayout driver. This set of changes refactors
that common code out to avoid any module depenencies.

Signed-off-by: Tom Haynes <loghyr@primarydata.com>
---
 fs/nfs/Makefile                   |   2 +-
 fs/nfs/filelayout/filelayout.c    | 291 ++------------------------------------
 fs/nfs/filelayout/filelayout.h    |  11 --
 fs/nfs/filelayout/filelayoutdev.c |   2 +-
 fs/nfs/pnfs.h                     |  23 +++
 fs/nfs/pnfs_nfsio.c               | 291 ++++++++++++++++++++++++++++++++++++++
 6 files changed, 330 insertions(+), 290 deletions(-)
 create mode 100644 fs/nfs/pnfs_nfsio.c

Comments

Schumaker, Anna Jan. 12, 2015, 6:59 p.m. UTC | #1
On 01/06/2015 07:28 PM, Tom Haynes wrote:
> The flexfilelayout driver will share some common code
> with the filelayout driver. This set of changes refactors
> that common code out to avoid any module depenencies.

Can you remind me why the common code has to go into nfsv4.ko?  Can't this all be moved into a filelayout_common module?

Anna

> 
> Signed-off-by: Tom Haynes <loghyr@primarydata.com>
> ---
>  fs/nfs/Makefile                   |   2 +-
>  fs/nfs/filelayout/filelayout.c    | 291 ++------------------------------------
>  fs/nfs/filelayout/filelayout.h    |  11 --
>  fs/nfs/filelayout/filelayoutdev.c |   2 +-
>  fs/nfs/pnfs.h                     |  23 +++
>  fs/nfs/pnfs_nfsio.c               | 291 ++++++++++++++++++++++++++++++++++++++
>  6 files changed, 330 insertions(+), 290 deletions(-)
>  create mode 100644 fs/nfs/pnfs_nfsio.c
> 
> diff --git a/fs/nfs/Makefile b/fs/nfs/Makefile
> index 04cb830..7973c4e3 100644
> --- a/fs/nfs/Makefile
> +++ b/fs/nfs/Makefile
> @@ -27,7 +27,7 @@ nfsv4-y := nfs4proc.o nfs4xdr.o nfs4state.o nfs4renewd.o nfs4super.o nfs4file.o
>  	  dns_resolve.o nfs4trace.o
>  nfsv4-$(CONFIG_NFS_USE_LEGACY_DNS) += cache_lib.o
>  nfsv4-$(CONFIG_SYSCTL)	+= nfs4sysctl.o
> -nfsv4-$(CONFIG_NFS_V4_1)	+= pnfs.o pnfs_dev.o
> +nfsv4-$(CONFIG_NFS_V4_1)	+= pnfs.o pnfs_dev.o pnfs_nfsio.o
>  nfsv4-$(CONFIG_NFS_V4_2)	+= nfs42proc.o
>  
>  obj-$(CONFIG_PNFS_FILE_LAYOUT) += filelayout/
> diff --git a/fs/nfs/filelayout/filelayout.c b/fs/nfs/filelayout/filelayout.c
> index 7afb52f..bc36ed3 100644
> --- a/fs/nfs/filelayout/filelayout.c
> +++ b/fs/nfs/filelayout/filelayout.c
> @@ -118,13 +118,6 @@ static void filelayout_reset_read(struct nfs_pgio_header *hdr)
>  	}
>  }
>  
> -static void filelayout_fenceme(struct inode *inode, struct pnfs_layout_hdr *lo)
> -{
> -	if (!test_and_clear_bit(NFS_LAYOUT_RETURN, &lo->plh_flags))
> -		return;
> -	pnfs_return_layout(inode);
> -}
> -
>  static int filelayout_async_handle_error(struct rpc_task *task,
>  					 struct nfs4_state *state,
>  					 struct nfs_client *clp,
> @@ -339,16 +332,6 @@ static void filelayout_read_count_stats(struct rpc_task *task, void *data)
>  	rpc_count_iostats(task, NFS_SERVER(hdr->inode)->client->cl_metrics);
>  }
>  
> -static void filelayout_read_release(void *data)
> -{
> -	struct nfs_pgio_header *hdr = data;
> -	struct pnfs_layout_hdr *lo = hdr->lseg->pls_layout;
> -
> -	filelayout_fenceme(lo->plh_inode, lo);
> -	nfs_put_client(hdr->ds_clp);
> -	hdr->mds_ops->rpc_release(data);
> -}
> -
>  static int filelayout_write_done_cb(struct rpc_task *task,
>  				struct nfs_pgio_header *hdr)
>  {
> @@ -371,17 +354,6 @@ static int filelayout_write_done_cb(struct rpc_task *task,
>  	return 0;
>  }
>  
> -/* Fake up some data that will cause nfs_commit_release to retry the writes. */
> -static void prepare_to_resend_writes(struct nfs_commit_data *data)
> -{
> -	struct nfs_page *first = nfs_list_entry(data->pages.next);
> -
> -	data->task.tk_status = 0;
> -	memcpy(&data->verf.verifier, &first->wb_verf,
> -	       sizeof(data->verf.verifier));
> -	data->verf.verifier.data[0]++; /* ensure verifier mismatch */
> -}
> -
>  static int filelayout_commit_done_cb(struct rpc_task *task,
>  				     struct nfs_commit_data *data)
>  {
> @@ -393,7 +365,7 @@ static int filelayout_commit_done_cb(struct rpc_task *task,
>  
>  	switch (err) {
>  	case -NFS4ERR_RESET_TO_MDS:
> -		prepare_to_resend_writes(data);
> +		pnfs_generic_prepare_to_resend_writes(data);
>  		return -EAGAIN;
>  	case -EAGAIN:
>  		rpc_restart_call_prepare(task);
> @@ -451,16 +423,6 @@ static void filelayout_write_count_stats(struct rpc_task *task, void *data)
>  	rpc_count_iostats(task, NFS_SERVER(hdr->inode)->client->cl_metrics);
>  }
>  
> -static void filelayout_write_release(void *data)
> -{
> -	struct nfs_pgio_header *hdr = data;
> -	struct pnfs_layout_hdr *lo = hdr->lseg->pls_layout;
> -
> -	filelayout_fenceme(lo->plh_inode, lo);
> -	nfs_put_client(hdr->ds_clp);
> -	hdr->mds_ops->rpc_release(data);
> -}
> -
>  static void filelayout_commit_prepare(struct rpc_task *task, void *data)
>  {
>  	struct nfs_commit_data *wdata = data;
> @@ -471,14 +433,6 @@ static void filelayout_commit_prepare(struct rpc_task *task, void *data)
>  			task);
>  }
>  
> -static void filelayout_write_commit_done(struct rpc_task *task, void *data)
> -{
> -	struct nfs_commit_data *wdata = data;
> -
> -	/* Note this may cause RPC to be resent */
> -	wdata->mds_ops->rpc_call_done(task, data);
> -}
> -
>  static void filelayout_commit_count_stats(struct rpc_task *task, void *data)
>  {
>  	struct nfs_commit_data *cdata = data;
> @@ -486,35 +440,25 @@ static void filelayout_commit_count_stats(struct rpc_task *task, void *data)
>  	rpc_count_iostats(task, NFS_SERVER(cdata->inode)->client->cl_metrics);
>  }
>  
> -static void filelayout_commit_release(void *calldata)
> -{
> -	struct nfs_commit_data *data = calldata;
> -
> -	data->completion_ops->completion(data);
> -	pnfs_put_lseg(data->lseg);
> -	nfs_put_client(data->ds_clp);
> -	nfs_commitdata_release(data);
> -}
> -
>  static const struct rpc_call_ops filelayout_read_call_ops = {
>  	.rpc_call_prepare = filelayout_read_prepare,
>  	.rpc_call_done = filelayout_read_call_done,
>  	.rpc_count_stats = filelayout_read_count_stats,
> -	.rpc_release = filelayout_read_release,
> +	.rpc_release = pnfs_generic_rw_release,
>  };
>  
>  static const struct rpc_call_ops filelayout_write_call_ops = {
>  	.rpc_call_prepare = filelayout_write_prepare,
>  	.rpc_call_done = filelayout_write_call_done,
>  	.rpc_count_stats = filelayout_write_count_stats,
> -	.rpc_release = filelayout_write_release,
> +	.rpc_release = pnfs_generic_rw_release,
>  };
>  
>  static const struct rpc_call_ops filelayout_commit_call_ops = {
>  	.rpc_call_prepare = filelayout_commit_prepare,
> -	.rpc_call_done = filelayout_write_commit_done,
> +	.rpc_call_done = pnfs_generic_write_commit_done,
>  	.rpc_count_stats = filelayout_commit_count_stats,
> -	.rpc_release = filelayout_commit_release,
> +	.rpc_release = pnfs_generic_commit_release,
>  };
>  
>  static enum pnfs_try_status
> @@ -1004,33 +948,6 @@ static u32 select_bucket_index(struct nfs4_filelayout_segment *fl, u32 j)
>  		return j;
>  }
>  
> -/* The generic layer is about to remove the req from the commit list.
> - * If this will make the bucket empty, it will need to put the lseg reference.
> - * Note this is must be called holding the inode (/cinfo) lock
> - */
> -static void
> -filelayout_clear_request_commit(struct nfs_page *req,
> -				struct nfs_commit_info *cinfo)
> -{
> -	struct pnfs_layout_segment *freeme = NULL;
> -
> -	if (!test_and_clear_bit(PG_COMMIT_TO_DS, &req->wb_flags))
> -		goto out;
> -	cinfo->ds->nwritten--;
> -	if (list_is_singular(&req->wb_list)) {
> -		struct pnfs_commit_bucket *bucket;
> -
> -		bucket = list_first_entry(&req->wb_list,
> -					  struct pnfs_commit_bucket,
> -					  written);
> -		freeme = bucket->wlseg;
> -		bucket->wlseg = NULL;
> -	}
> -out:
> -	nfs_request_remove_commit_list(req, cinfo);
> -	pnfs_put_lseg_locked(freeme);
> -}
> -
>  static void
>  filelayout_mark_request_commit(struct nfs_page *req,
>  			       struct pnfs_layout_segment *lseg,
> @@ -1064,7 +981,7 @@ filelayout_mark_request_commit(struct nfs_page *req,
>  		 * is normally transferred to the COMMIT call and released
>  		 * there.  It could also be released if the last req is pulled
>  		 * off due to a rewrite, in which case it will be done in
> -		 * filelayout_clear_request_commit
> +		 * pnfs_generic_clear_request_commit
>  		 */
>  		buckets[i].wlseg = pnfs_get_lseg(lseg);
>  	}
> @@ -1142,97 +1059,11 @@ static int filelayout_initiate_commit(struct nfs_commit_data *data, int how)
>  				   &filelayout_commit_call_ops, how,
>  				   RPC_TASK_SOFTCONN);
>  out_err:
> -	prepare_to_resend_writes(data);
> -	filelayout_commit_release(data);
> +	pnfs_generic_prepare_to_resend_writes(data);
> +	pnfs_generic_commit_release(data);
>  	return -EAGAIN;
>  }
>  
> -static int
> -transfer_commit_list(struct list_head *src, struct list_head *dst,
> -		     struct nfs_commit_info *cinfo, int max)
> -{
> -	struct nfs_page *req, *tmp;
> -	int ret = 0;
> -
> -	list_for_each_entry_safe(req, tmp, src, wb_list) {
> -		if (!nfs_lock_request(req))
> -			continue;
> -		kref_get(&req->wb_kref);
> -		if (cond_resched_lock(cinfo->lock))
> -			list_safe_reset_next(req, tmp, wb_list);
> -		nfs_request_remove_commit_list(req, cinfo);
> -		clear_bit(PG_COMMIT_TO_DS, &req->wb_flags);
> -		nfs_list_add_request(req, dst);
> -		ret++;
> -		if ((ret == max) && !cinfo->dreq)
> -			break;
> -	}
> -	return ret;
> -}
> -
> -/* Note called with cinfo->lock held. */
> -static int
> -filelayout_scan_ds_commit_list(struct pnfs_commit_bucket *bucket,
> -			       struct nfs_commit_info *cinfo,
> -			       int max)
> -{
> -	struct list_head *src = &bucket->written;
> -	struct list_head *dst = &bucket->committing;
> -	int ret;
> -
> -	ret = transfer_commit_list(src, dst, cinfo, max);
> -	if (ret) {
> -		cinfo->ds->nwritten -= ret;
> -		cinfo->ds->ncommitting += ret;
> -		bucket->clseg = bucket->wlseg;
> -		if (list_empty(src))
> -			bucket->wlseg = NULL;
> -		else
> -			pnfs_get_lseg(bucket->clseg);
> -	}
> -	return ret;
> -}
> -
> -/* Move reqs from written to committing lists, returning count of number moved.
> - * Note called with cinfo->lock held.
> - */
> -static int filelayout_scan_commit_lists(struct nfs_commit_info *cinfo,
> -					int max)
> -{
> -	int i, rv = 0, cnt;
> -
> -	for (i = 0; i < cinfo->ds->nbuckets && max != 0; i++) {
> -		cnt = filelayout_scan_ds_commit_list(&cinfo->ds->buckets[i],
> -						     cinfo, max);
> -		max -= cnt;
> -		rv += cnt;
> -	}
> -	return rv;
> -}
> -
> -/* Pull everything off the committing lists and dump into @dst */
> -static void filelayout_recover_commit_reqs(struct list_head *dst,
> -					   struct nfs_commit_info *cinfo)
> -{
> -	struct pnfs_commit_bucket *b;
> -	struct pnfs_layout_segment *freeme;
> -	int i;
> -
> -restart:
> -	spin_lock(cinfo->lock);
> -	for (i = 0, b = cinfo->ds->buckets; i < cinfo->ds->nbuckets; i++, b++) {
> -		if (transfer_commit_list(&b->written, dst, cinfo, 0)) {
> -			freeme = b->wlseg;
> -			b->wlseg = NULL;
> -			spin_unlock(cinfo->lock);
> -			pnfs_put_lseg(freeme);
> -			goto restart;
> -		}
> -	}
> -	cinfo->ds->nwritten = 0;
> -	spin_unlock(cinfo->lock);
> -}
> -
>  /* filelayout_search_commit_reqs - Search lists in @cinfo for the head reqest
>   *				   for @page
>   * @cinfo - commit info for current inode
> @@ -1263,108 +1094,14 @@ filelayout_search_commit_reqs(struct nfs_commit_info *cinfo, struct page *page)
>  	return NULL;
>  }
>  
> -static void filelayout_retry_commit(struct nfs_commit_info *cinfo, int idx)
> -{
> -	struct pnfs_ds_commit_info *fl_cinfo = cinfo->ds;
> -	struct pnfs_commit_bucket *bucket;
> -	struct pnfs_layout_segment *freeme;
> -	int i;
> -
> -	for (i = idx; i < fl_cinfo->nbuckets; i++) {
> -		bucket = &fl_cinfo->buckets[i];
> -		if (list_empty(&bucket->committing))
> -			continue;
> -		nfs_retry_commit(&bucket->committing, bucket->clseg, cinfo);
> -		spin_lock(cinfo->lock);
> -		freeme = bucket->clseg;
> -		bucket->clseg = NULL;
> -		spin_unlock(cinfo->lock);
> -		pnfs_put_lseg(freeme);
> -	}
> -}
> -
> -static unsigned int
> -alloc_ds_commits(struct nfs_commit_info *cinfo, struct list_head *list)
> -{
> -	struct pnfs_ds_commit_info *fl_cinfo;
> -	struct pnfs_commit_bucket *bucket;
> -	struct nfs_commit_data *data;
> -	int i;
> -	unsigned int nreq = 0;
> -
> -	fl_cinfo = cinfo->ds;
> -	bucket = fl_cinfo->buckets;
> -	for (i = 0; i < fl_cinfo->nbuckets; i++, bucket++) {
> -		if (list_empty(&bucket->committing))
> -			continue;
> -		data = nfs_commitdata_alloc();
> -		if (!data)
> -			break;
> -		data->ds_commit_index = i;
> -		spin_lock(cinfo->lock);
> -		data->lseg = bucket->clseg;
> -		bucket->clseg = NULL;
> -		spin_unlock(cinfo->lock);
> -		list_add(&data->pages, list);
> -		nreq++;
> -	}
> -
> -	/* Clean up on error */
> -	filelayout_retry_commit(cinfo, i);
> -	/* Caller will clean up entries put on list */
> -	return nreq;
> -}
> -
> -/* This follows nfs_commit_list pretty closely */
>  static int
>  filelayout_commit_pagelist(struct inode *inode, struct list_head *mds_pages,
>  			   int how, struct nfs_commit_info *cinfo)
>  {
> -	struct nfs_commit_data *data, *tmp;
> -	LIST_HEAD(list);
> -	unsigned int nreq = 0;
> -
> -	if (!list_empty(mds_pages)) {
> -		data = nfs_commitdata_alloc();
> -		if (data != NULL) {
> -			data->lseg = NULL;
> -			list_add(&data->pages, &list);
> -			nreq++;
> -		} else {
> -			nfs_retry_commit(mds_pages, NULL, cinfo);
> -			filelayout_retry_commit(cinfo, 0);
> -			cinfo->completion_ops->error_cleanup(NFS_I(inode));
> -			return -ENOMEM;
> -		}
> -	}
> -
> -	nreq += alloc_ds_commits(cinfo, &list);
> -
> -	if (nreq == 0) {
> -		cinfo->completion_ops->error_cleanup(NFS_I(inode));
> -		goto out;
> -	}
> -
> -	atomic_add(nreq, &cinfo->mds->rpcs_out);
> -
> -	list_for_each_entry_safe(data, tmp, &list, pages) {
> -		list_del_init(&data->pages);
> -		if (!data->lseg) {
> -			nfs_init_commit(data, mds_pages, NULL, cinfo);
> -			nfs_initiate_commit(NFS_CLIENT(inode), data,
> -					    data->mds_ops, how, 0);
> -		} else {
> -			struct pnfs_commit_bucket *buckets;
> -
> -			buckets = cinfo->ds->buckets;
> -			nfs_init_commit(data, &buckets[data->ds_commit_index].committing, data->lseg, cinfo);
> -			filelayout_initiate_commit(data, how);
> -		}
> -	}
> -out:
> -	cinfo->ds->ncommitting = 0;
> -	return PNFS_ATTEMPTED;
> +	return pnfs_generic_commit_pagelist(inode, mds_pages, how, cinfo,
> +					    filelayout_initiate_commit);
>  }
> +
>  static struct nfs4_deviceid_node *
>  filelayout_alloc_deviceid_node(struct nfs_server *server,
>  		struct pnfs_device *pdev, gfp_t gfp_flags)
> @@ -1421,9 +1158,9 @@ static struct pnfs_layoutdriver_type filelayout_type = {
>  	.pg_write_ops		= &filelayout_pg_write_ops,
>  	.get_ds_info		= &filelayout_get_ds_info,
>  	.mark_request_commit	= filelayout_mark_request_commit,
> -	.clear_request_commit	= filelayout_clear_request_commit,
> -	.scan_commit_lists	= filelayout_scan_commit_lists,
> -	.recover_commit_reqs	= filelayout_recover_commit_reqs,
> +	.clear_request_commit	= pnfs_generic_clear_request_commit,
> +	.scan_commit_lists	= pnfs_generic_scan_commit_lists,
> +	.recover_commit_reqs	= pnfs_generic_recover_commit_reqs,
>  	.search_commit_reqs	= filelayout_search_commit_reqs,
>  	.commit_pagelist	= filelayout_commit_pagelist,
>  	.read_pagelist		= filelayout_read_pagelist,
> diff --git a/fs/nfs/filelayout/filelayout.h b/fs/nfs/filelayout/filelayout.h
> index 7c9f800..a5ce9b4 100644
> --- a/fs/nfs/filelayout/filelayout.h
> +++ b/fs/nfs/filelayout/filelayout.h
> @@ -119,17 +119,6 @@ FILELAYOUT_DEVID_NODE(struct pnfs_layout_segment *lseg)
>  	return &FILELAYOUT_LSEG(lseg)->dsaddr->id_node;
>  }
>  
> -static inline void
> -filelayout_mark_devid_invalid(struct nfs4_deviceid_node *node)
> -{
> -	u32 *p = (u32 *)&node->deviceid;
> -
> -	printk(KERN_WARNING "NFS: Deviceid [%x%x%x%x] marked out of use.\n",
> -		p[0], p[1], p[2], p[3]);
> -
> -	set_bit(NFS_DEVICEID_INVALID, &node->flags);
> -}
> -
>  static inline bool
>  filelayout_test_devid_invalid(struct nfs4_deviceid_node *node)
>  {
> diff --git a/fs/nfs/filelayout/filelayoutdev.c b/fs/nfs/filelayout/filelayoutdev.c
> index bfecac7..d21080a 100644
> --- a/fs/nfs/filelayout/filelayoutdev.c
> +++ b/fs/nfs/filelayout/filelayoutdev.c
> @@ -708,7 +708,7 @@ nfs4_fl_prepare_ds(struct pnfs_layout_segment *lseg, u32 ds_idx)
>  	if (ds == NULL) {
>  		printk(KERN_ERR "NFS: %s: No data server for offset index %d\n",
>  			__func__, ds_idx);
> -		filelayout_mark_devid_invalid(devid);
> +		pnfs_generic_mark_devid_invalid(devid);
>  		goto out;
>  	}
>  	smp_rmb();
> diff --git a/fs/nfs/pnfs.h b/fs/nfs/pnfs.h
> index 9ae5b76..88eede0 100644
> --- a/fs/nfs/pnfs.h
> +++ b/fs/nfs/pnfs.h
> @@ -275,6 +275,23 @@ void nfs4_mark_deviceid_unavailable(struct nfs4_deviceid_node *node);
>  bool nfs4_test_deviceid_unavailable(struct nfs4_deviceid_node *node);
>  void nfs4_deviceid_purge_client(const struct nfs_client *);
>  
> +/* pnfs_nfsio.c */
> +void pnfs_generic_clear_request_commit(struct nfs_page *req,
> +				       struct nfs_commit_info *cinfo);
> +void pnfs_generic_commit_release(void *calldata);
> +void pnfs_generic_prepare_to_resend_writes(struct nfs_commit_data *data);
> +void pnfs_generic_rw_release(void *data);
> +void pnfs_generic_recover_commit_reqs(struct list_head *dst,
> +				      struct nfs_commit_info *cinfo);
> +int pnfs_generic_commit_pagelist(struct inode *inode,
> +				 struct list_head *mds_pages,
> +				 int how,
> +				 struct nfs_commit_info *cinfo,
> +				 int (*initiate_commit)(struct nfs_commit_data *data,
> +							int how));
> +int pnfs_generic_scan_commit_lists(struct nfs_commit_info *cinfo, int max);
> +void pnfs_generic_write_commit_done(struct rpc_task *task, void *data);
> +
>  static inline struct nfs4_deviceid_node *
>  nfs4_get_deviceid(struct nfs4_deviceid_node *d)
>  {
> @@ -317,6 +334,12 @@ pnfs_get_ds_info(struct inode *inode)
>  	return ld->get_ds_info(inode);
>  }
>  
> +static inline void
> +pnfs_generic_mark_devid_invalid(struct nfs4_deviceid_node *node)
> +{
> +	set_bit(NFS_DEVICEID_INVALID, &node->flags);
> +}
> +
>  static inline bool
>  pnfs_mark_request_commit(struct nfs_page *req, struct pnfs_layout_segment *lseg,
>  			 struct nfs_commit_info *cinfo)
> diff --git a/fs/nfs/pnfs_nfsio.c b/fs/nfs/pnfs_nfsio.c
> new file mode 100644
> index 0000000..e5f841c
> --- /dev/null
> +++ b/fs/nfs/pnfs_nfsio.c
> @@ -0,0 +1,291 @@
> +/*
> + * Common NFS I/O  operations for the pnfs file based
> + * layout drivers.
> + *
> + * Copyright (c) 2014, Primary Data, Inc. All rights reserved.
> + *
> + * Tom Haynes <loghyr@primarydata.com>
> + */
> +
> +#include <linux/nfs_fs.h>
> +#include <linux/nfs_page.h>
> +
> +#include "internal.h"
> +#include "pnfs.h"
> +
> +static void pnfs_generic_fenceme(struct inode *inode,
> +				 struct pnfs_layout_hdr *lo)
> +{
> +	if (!test_and_clear_bit(NFS_LAYOUT_RETURN, &lo->plh_flags))
> +		return;
> +	pnfs_return_layout(inode);
> +}
> +
> +void pnfs_generic_rw_release(void *data)
> +{
> +	struct nfs_pgio_header *hdr = data;
> +	struct pnfs_layout_hdr *lo = hdr->lseg->pls_layout;
> +
> +	pnfs_generic_fenceme(lo->plh_inode, lo);
> +	nfs_put_client(hdr->ds_clp);
> +	hdr->mds_ops->rpc_release(data);
> +}
> +EXPORT_SYMBOL_GPL(pnfs_generic_rw_release);
> +
> +/* Fake up some data that will cause nfs_commit_release to retry the writes. */
> +void pnfs_generic_prepare_to_resend_writes(struct nfs_commit_data *data)
> +{
> +	struct nfs_page *first = nfs_list_entry(data->pages.next);
> +
> +	data->task.tk_status = 0;
> +	memcpy(&data->verf.verifier, &first->wb_verf,
> +	       sizeof(data->verf.verifier));
> +	data->verf.verifier.data[0]++; /* ensure verifier mismatch */
> +}
> +EXPORT_SYMBOL_GPL(pnfs_generic_prepare_to_resend_writes);
> +
> +void pnfs_generic_write_commit_done(struct rpc_task *task, void *data)
> +{
> +	struct nfs_commit_data *wdata = data;
> +
> +	/* Note this may cause RPC to be resent */
> +	wdata->mds_ops->rpc_call_done(task, data);
> +}
> +EXPORT_SYMBOL_GPL(pnfs_generic_write_commit_done);
> +
> +void pnfs_generic_commit_release(void *calldata)
> +{
> +	struct nfs_commit_data *data = calldata;
> +
> +	data->completion_ops->completion(data);
> +	pnfs_put_lseg(data->lseg);
> +	nfs_put_client(data->ds_clp);
> +	nfs_commitdata_release(data);
> +}
> +EXPORT_SYMBOL_GPL(pnfs_generic_commit_release);
> +
> +/* The generic layer is about to remove the req from the commit list.
> + * If this will make the bucket empty, it will need to put the lseg reference.
> + * Note this is must be called holding the inode (/cinfo) lock
> + */
> +void
> +pnfs_generic_clear_request_commit(struct nfs_page *req,
> +				  struct nfs_commit_info *cinfo)
> +{
> +	struct pnfs_layout_segment *freeme = NULL;
> +
> +	if (!test_and_clear_bit(PG_COMMIT_TO_DS, &req->wb_flags))
> +		goto out;
> +	cinfo->ds->nwritten--;
> +	if (list_is_singular(&req->wb_list)) {
> +		struct pnfs_commit_bucket *bucket;
> +
> +		bucket = list_first_entry(&req->wb_list,
> +					  struct pnfs_commit_bucket,
> +					  written);
> +		freeme = bucket->wlseg;
> +		bucket->wlseg = NULL;
> +	}
> +out:
> +	nfs_request_remove_commit_list(req, cinfo);
> +	pnfs_put_lseg_locked(freeme);
> +}
> +EXPORT_SYMBOL_GPL(pnfs_generic_clear_request_commit);
> +
> +static int
> +pnfs_generic_transfer_commit_list(struct list_head *src, struct list_head *dst,
> +				  struct nfs_commit_info *cinfo, int max)
> +{
> +	struct nfs_page *req, *tmp;
> +	int ret = 0;
> +
> +	list_for_each_entry_safe(req, tmp, src, wb_list) {
> +		if (!nfs_lock_request(req))
> +			continue;
> +		kref_get(&req->wb_kref);
> +		if (cond_resched_lock(cinfo->lock))
> +			list_safe_reset_next(req, tmp, wb_list);
> +		nfs_request_remove_commit_list(req, cinfo);
> +		clear_bit(PG_COMMIT_TO_DS, &req->wb_flags);
> +		nfs_list_add_request(req, dst);
> +		ret++;
> +		if ((ret == max) && !cinfo->dreq)
> +			break;
> +	}
> +	return ret;
> +}
> +
> +/* Note called with cinfo->lock held. */
> +static int
> +pnfs_generic_scan_ds_commit_list(struct pnfs_commit_bucket *bucket,
> +				 struct nfs_commit_info *cinfo,
> +				 int max)
> +{
> +	struct list_head *src = &bucket->written;
> +	struct list_head *dst = &bucket->committing;
> +	int ret;
> +
> +	ret = pnfs_generic_transfer_commit_list(src, dst, cinfo, max);
> +	if (ret) {
> +		cinfo->ds->nwritten -= ret;
> +		cinfo->ds->ncommitting += ret;
> +		bucket->clseg = bucket->wlseg;
> +		if (list_empty(src))
> +			bucket->wlseg = NULL;
> +		else
> +			pnfs_get_lseg(bucket->clseg);
> +	}
> +	return ret;
> +}
> +
> +/* Move reqs from written to committing lists, returning count of number moved.
> + * Note called with cinfo->lock held.
> + */
> +int pnfs_generic_scan_commit_lists(struct nfs_commit_info *cinfo,
> +				   int max)
> +{
> +	int i, rv = 0, cnt;
> +
> +	for (i = 0; i < cinfo->ds->nbuckets && max != 0; i++) {
> +		cnt = pnfs_generic_scan_ds_commit_list(&cinfo->ds->buckets[i],
> +						       cinfo, max);
> +		max -= cnt;
> +		rv += cnt;
> +	}
> +	return rv;
> +}
> +EXPORT_SYMBOL_GPL(pnfs_generic_scan_commit_lists);
> +
> +/* Pull everything off the committing lists and dump into @dst */
> +void pnfs_generic_recover_commit_reqs(struct list_head *dst,
> +				      struct nfs_commit_info *cinfo)
> +{
> +	struct pnfs_commit_bucket *b;
> +	struct pnfs_layout_segment *freeme;
> +	int i;
> +
> +restart:
> +	spin_lock(cinfo->lock);
> +	for (i = 0, b = cinfo->ds->buckets; i < cinfo->ds->nbuckets; i++, b++) {
> +		if (pnfs_generic_transfer_commit_list(&b->written, dst,
> +						      cinfo, 0)) {
> +			freeme = b->wlseg;
> +			b->wlseg = NULL;
> +			spin_unlock(cinfo->lock);
> +			pnfs_put_lseg(freeme);
> +			goto restart;
> +		}
> +	}
> +	cinfo->ds->nwritten = 0;
> +	spin_unlock(cinfo->lock);
> +}
> +EXPORT_SYMBOL_GPL(pnfs_generic_recover_commit_reqs);
> +
> +static void pnfs_generic_retry_commit(struct nfs_commit_info *cinfo, int idx)
> +{
> +	struct pnfs_ds_commit_info *fl_cinfo = cinfo->ds;
> +	struct pnfs_commit_bucket *bucket;
> +	struct pnfs_layout_segment *freeme;
> +	int i;
> +
> +	for (i = idx; i < fl_cinfo->nbuckets; i++) {
> +		bucket = &fl_cinfo->buckets[i];
> +		if (list_empty(&bucket->committing))
> +			continue;
> +		nfs_retry_commit(&bucket->committing, bucket->clseg, cinfo);
> +		spin_lock(cinfo->lock);
> +		freeme = bucket->clseg;
> +		bucket->clseg = NULL;
> +		spin_unlock(cinfo->lock);
> +		pnfs_put_lseg(freeme);
> +	}
> +}
> +
> +static unsigned int
> +pnfs_generic_alloc_ds_commits(struct nfs_commit_info *cinfo,
> +			      struct list_head *list)
> +{
> +	struct pnfs_ds_commit_info *fl_cinfo;
> +	struct pnfs_commit_bucket *bucket;
> +	struct nfs_commit_data *data;
> +	int i;
> +	unsigned int nreq = 0;
> +
> +	fl_cinfo = cinfo->ds;
> +	bucket = fl_cinfo->buckets;
> +	for (i = 0; i < fl_cinfo->nbuckets; i++, bucket++) {
> +		if (list_empty(&bucket->committing))
> +			continue;
> +		data = nfs_commitdata_alloc();
> +		if (!data)
> +			break;
> +		data->ds_commit_index = i;
> +		spin_lock(cinfo->lock);
> +		data->lseg = bucket->clseg;
> +		bucket->clseg = NULL;
> +		spin_unlock(cinfo->lock);
> +		list_add(&data->pages, list);
> +		nreq++;
> +	}
> +
> +	/* Clean up on error */
> +	pnfs_generic_retry_commit(cinfo, i);
> +	return nreq;
> +}
> +
> +/* This follows nfs_commit_list pretty closely */
> +int
> +pnfs_generic_commit_pagelist(struct inode *inode, struct list_head *mds_pages,
> +			     int how, struct nfs_commit_info *cinfo,
> +			     int (*initiate_commit)(struct nfs_commit_data *data,
> +						    int how))
> +{
> +	struct nfs_commit_data *data, *tmp;
> +	LIST_HEAD(list);
> +	unsigned int nreq = 0;
> +
> +	if (!list_empty(mds_pages)) {
> +		data = nfs_commitdata_alloc();
> +		if (data != NULL) {
> +			data->lseg = NULL;
> +			list_add(&data->pages, &list);
> +			nreq++;
> +		} else {
> +			nfs_retry_commit(mds_pages, NULL, cinfo);
> +			pnfs_generic_retry_commit(cinfo, 0);
> +			cinfo->completion_ops->error_cleanup(NFS_I(inode));
> +			return -ENOMEM;
> +		}
> +	}
> +
> +	nreq += pnfs_generic_alloc_ds_commits(cinfo, &list);
> +
> +	if (nreq == 0) {
> +		cinfo->completion_ops->error_cleanup(NFS_I(inode));
> +		goto out;
> +	}
> +
> +	atomic_add(nreq, &cinfo->mds->rpcs_out);
> +
> +	list_for_each_entry_safe(data, tmp, &list, pages) {
> +		list_del_init(&data->pages);
> +		if (!data->lseg) {
> +			nfs_init_commit(data, mds_pages, NULL, cinfo);
> +			nfs_initiate_commit(NFS_CLIENT(inode), data,
> +					    data->mds_ops, how, 0);
> +		} else {
> +			struct pnfs_commit_bucket *buckets;
> +
> +			buckets = cinfo->ds->buckets;
> +			nfs_init_commit(data,
> +					&buckets[data->ds_commit_index].committing,
> +					data->lseg,
> +					cinfo);
> +			initiate_commit(data, how);
> +		}
> +	}
> +out:
> +	cinfo->ds->ncommitting = 0;
> +	return PNFS_ATTEMPTED;
> +}
> +EXPORT_SYMBOL_GPL(pnfs_generic_commit_pagelist);
> 

--
To unsubscribe from this list: send the line "unsubscribe linux-nfs" in
the body of a message to majordomo@vger.kernel.org
More majordomo info at  http://vger.kernel.org/majordomo-info.html
Thomas Haynes Jan. 12, 2015, 7:10 p.m. UTC | #2
On Mon, Jan 12, 2015 at 01:59:55PM -0500, Anna Schumaker wrote:
> On 01/06/2015 07:28 PM, Tom Haynes wrote:
> > The flexfilelayout driver will share some common code
> > with the filelayout driver. This set of changes refactors
> > that common code out to avoid any module depenencies.
> 
> Can you remind me why the common code has to go into nfsv4.ko?  Can't this all be moved into a filelayout_common module?

Yes, it could. The main dependency I wanted to avoid
was between the flex files and the files.

I did not want to add another module...

> 
> Anna
> 
> > 
> > Signed-off-by: Tom Haynes <loghyr@primarydata.com>
> > ---
> >  fs/nfs/Makefile                   |   2 +-
> >  fs/nfs/filelayout/filelayout.c    | 291 ++------------------------------------
> >  fs/nfs/filelayout/filelayout.h    |  11 --
> >  fs/nfs/filelayout/filelayoutdev.c |   2 +-
> >  fs/nfs/pnfs.h                     |  23 +++
> >  fs/nfs/pnfs_nfsio.c               | 291 ++++++++++++++++++++++++++++++++++++++
> >  6 files changed, 330 insertions(+), 290 deletions(-)
> >  create mode 100644 fs/nfs/pnfs_nfsio.c
> > 
> > diff --git a/fs/nfs/Makefile b/fs/nfs/Makefile
> > index 04cb830..7973c4e3 100644
> > --- a/fs/nfs/Makefile
> > +++ b/fs/nfs/Makefile
> > @@ -27,7 +27,7 @@ nfsv4-y := nfs4proc.o nfs4xdr.o nfs4state.o nfs4renewd.o nfs4super.o nfs4file.o
> >  	  dns_resolve.o nfs4trace.o
> >  nfsv4-$(CONFIG_NFS_USE_LEGACY_DNS) += cache_lib.o
> >  nfsv4-$(CONFIG_SYSCTL)	+= nfs4sysctl.o
> > -nfsv4-$(CONFIG_NFS_V4_1)	+= pnfs.o pnfs_dev.o
> > +nfsv4-$(CONFIG_NFS_V4_1)	+= pnfs.o pnfs_dev.o pnfs_nfsio.o
> >  nfsv4-$(CONFIG_NFS_V4_2)	+= nfs42proc.o
> >  
> >  obj-$(CONFIG_PNFS_FILE_LAYOUT) += filelayout/
> > diff --git a/fs/nfs/filelayout/filelayout.c b/fs/nfs/filelayout/filelayout.c
> > index 7afb52f..bc36ed3 100644
> > --- a/fs/nfs/filelayout/filelayout.c
> > +++ b/fs/nfs/filelayout/filelayout.c
> > @@ -118,13 +118,6 @@ static void filelayout_reset_read(struct nfs_pgio_header *hdr)
> >  	}
> >  }
> >  
> > -static void filelayout_fenceme(struct inode *inode, struct pnfs_layout_hdr *lo)
> > -{
> > -	if (!test_and_clear_bit(NFS_LAYOUT_RETURN, &lo->plh_flags))
> > -		return;
> > -	pnfs_return_layout(inode);
> > -}
> > -
> >  static int filelayout_async_handle_error(struct rpc_task *task,
> >  					 struct nfs4_state *state,
> >  					 struct nfs_client *clp,
> > @@ -339,16 +332,6 @@ static void filelayout_read_count_stats(struct rpc_task *task, void *data)
> >  	rpc_count_iostats(task, NFS_SERVER(hdr->inode)->client->cl_metrics);
> >  }
> >  
> > -static void filelayout_read_release(void *data)
> > -{
> > -	struct nfs_pgio_header *hdr = data;
> > -	struct pnfs_layout_hdr *lo = hdr->lseg->pls_layout;
> > -
> > -	filelayout_fenceme(lo->plh_inode, lo);
> > -	nfs_put_client(hdr->ds_clp);
> > -	hdr->mds_ops->rpc_release(data);
> > -}
> > -
> >  static int filelayout_write_done_cb(struct rpc_task *task,
> >  				struct nfs_pgio_header *hdr)
> >  {
> > @@ -371,17 +354,6 @@ static int filelayout_write_done_cb(struct rpc_task *task,
> >  	return 0;
> >  }
> >  
> > -/* Fake up some data that will cause nfs_commit_release to retry the writes. */
> > -static void prepare_to_resend_writes(struct nfs_commit_data *data)
> > -{
> > -	struct nfs_page *first = nfs_list_entry(data->pages.next);
> > -
> > -	data->task.tk_status = 0;
> > -	memcpy(&data->verf.verifier, &first->wb_verf,
> > -	       sizeof(data->verf.verifier));
> > -	data->verf.verifier.data[0]++; /* ensure verifier mismatch */
> > -}
> > -
> >  static int filelayout_commit_done_cb(struct rpc_task *task,
> >  				     struct nfs_commit_data *data)
> >  {
> > @@ -393,7 +365,7 @@ static int filelayout_commit_done_cb(struct rpc_task *task,
> >  
> >  	switch (err) {
> >  	case -NFS4ERR_RESET_TO_MDS:
> > -		prepare_to_resend_writes(data);
> > +		pnfs_generic_prepare_to_resend_writes(data);
> >  		return -EAGAIN;
> >  	case -EAGAIN:
> >  		rpc_restart_call_prepare(task);
> > @@ -451,16 +423,6 @@ static void filelayout_write_count_stats(struct rpc_task *task, void *data)
> >  	rpc_count_iostats(task, NFS_SERVER(hdr->inode)->client->cl_metrics);
> >  }
> >  
> > -static void filelayout_write_release(void *data)
> > -{
> > -	struct nfs_pgio_header *hdr = data;
> > -	struct pnfs_layout_hdr *lo = hdr->lseg->pls_layout;
> > -
> > -	filelayout_fenceme(lo->plh_inode, lo);
> > -	nfs_put_client(hdr->ds_clp);
> > -	hdr->mds_ops->rpc_release(data);
> > -}
> > -
> >  static void filelayout_commit_prepare(struct rpc_task *task, void *data)
> >  {
> >  	struct nfs_commit_data *wdata = data;
> > @@ -471,14 +433,6 @@ static void filelayout_commit_prepare(struct rpc_task *task, void *data)
> >  			task);
> >  }
> >  
> > -static void filelayout_write_commit_done(struct rpc_task *task, void *data)
> > -{
> > -	struct nfs_commit_data *wdata = data;
> > -
> > -	/* Note this may cause RPC to be resent */
> > -	wdata->mds_ops->rpc_call_done(task, data);
> > -}
> > -
> >  static void filelayout_commit_count_stats(struct rpc_task *task, void *data)
> >  {
> >  	struct nfs_commit_data *cdata = data;
> > @@ -486,35 +440,25 @@ static void filelayout_commit_count_stats(struct rpc_task *task, void *data)
> >  	rpc_count_iostats(task, NFS_SERVER(cdata->inode)->client->cl_metrics);
> >  }
> >  
> > -static void filelayout_commit_release(void *calldata)
> > -{
> > -	struct nfs_commit_data *data = calldata;
> > -
> > -	data->completion_ops->completion(data);
> > -	pnfs_put_lseg(data->lseg);
> > -	nfs_put_client(data->ds_clp);
> > -	nfs_commitdata_release(data);
> > -}
> > -
> >  static const struct rpc_call_ops filelayout_read_call_ops = {
> >  	.rpc_call_prepare = filelayout_read_prepare,
> >  	.rpc_call_done = filelayout_read_call_done,
> >  	.rpc_count_stats = filelayout_read_count_stats,
> > -	.rpc_release = filelayout_read_release,
> > +	.rpc_release = pnfs_generic_rw_release,
> >  };
> >  
> >  static const struct rpc_call_ops filelayout_write_call_ops = {
> >  	.rpc_call_prepare = filelayout_write_prepare,
> >  	.rpc_call_done = filelayout_write_call_done,
> >  	.rpc_count_stats = filelayout_write_count_stats,
> > -	.rpc_release = filelayout_write_release,
> > +	.rpc_release = pnfs_generic_rw_release,
> >  };
> >  
> >  static const struct rpc_call_ops filelayout_commit_call_ops = {
> >  	.rpc_call_prepare = filelayout_commit_prepare,
> > -	.rpc_call_done = filelayout_write_commit_done,
> > +	.rpc_call_done = pnfs_generic_write_commit_done,
> >  	.rpc_count_stats = filelayout_commit_count_stats,
> > -	.rpc_release = filelayout_commit_release,
> > +	.rpc_release = pnfs_generic_commit_release,
> >  };
> >  
> >  static enum pnfs_try_status
> > @@ -1004,33 +948,6 @@ static u32 select_bucket_index(struct nfs4_filelayout_segment *fl, u32 j)
> >  		return j;
> >  }
> >  
> > -/* The generic layer is about to remove the req from the commit list.
> > - * If this will make the bucket empty, it will need to put the lseg reference.
> > - * Note this is must be called holding the inode (/cinfo) lock
> > - */
> > -static void
> > -filelayout_clear_request_commit(struct nfs_page *req,
> > -				struct nfs_commit_info *cinfo)
> > -{
> > -	struct pnfs_layout_segment *freeme = NULL;
> > -
> > -	if (!test_and_clear_bit(PG_COMMIT_TO_DS, &req->wb_flags))
> > -		goto out;
> > -	cinfo->ds->nwritten--;
> > -	if (list_is_singular(&req->wb_list)) {
> > -		struct pnfs_commit_bucket *bucket;
> > -
> > -		bucket = list_first_entry(&req->wb_list,
> > -					  struct pnfs_commit_bucket,
> > -					  written);
> > -		freeme = bucket->wlseg;
> > -		bucket->wlseg = NULL;
> > -	}
> > -out:
> > -	nfs_request_remove_commit_list(req, cinfo);
> > -	pnfs_put_lseg_locked(freeme);
> > -}
> > -
> >  static void
> >  filelayout_mark_request_commit(struct nfs_page *req,
> >  			       struct pnfs_layout_segment *lseg,
> > @@ -1064,7 +981,7 @@ filelayout_mark_request_commit(struct nfs_page *req,
> >  		 * is normally transferred to the COMMIT call and released
> >  		 * there.  It could also be released if the last req is pulled
> >  		 * off due to a rewrite, in which case it will be done in
> > -		 * filelayout_clear_request_commit
> > +		 * pnfs_generic_clear_request_commit
> >  		 */
> >  		buckets[i].wlseg = pnfs_get_lseg(lseg);
> >  	}
> > @@ -1142,97 +1059,11 @@ static int filelayout_initiate_commit(struct nfs_commit_data *data, int how)
> >  				   &filelayout_commit_call_ops, how,
> >  				   RPC_TASK_SOFTCONN);
> >  out_err:
> > -	prepare_to_resend_writes(data);
> > -	filelayout_commit_release(data);
> > +	pnfs_generic_prepare_to_resend_writes(data);
> > +	pnfs_generic_commit_release(data);
> >  	return -EAGAIN;
> >  }
> >  
> > -static int
> > -transfer_commit_list(struct list_head *src, struct list_head *dst,
> > -		     struct nfs_commit_info *cinfo, int max)
> > -{
> > -	struct nfs_page *req, *tmp;
> > -	int ret = 0;
> > -
> > -	list_for_each_entry_safe(req, tmp, src, wb_list) {
> > -		if (!nfs_lock_request(req))
> > -			continue;
> > -		kref_get(&req->wb_kref);
> > -		if (cond_resched_lock(cinfo->lock))
> > -			list_safe_reset_next(req, tmp, wb_list);
> > -		nfs_request_remove_commit_list(req, cinfo);
> > -		clear_bit(PG_COMMIT_TO_DS, &req->wb_flags);
> > -		nfs_list_add_request(req, dst);
> > -		ret++;
> > -		if ((ret == max) && !cinfo->dreq)
> > -			break;
> > -	}
> > -	return ret;
> > -}
> > -
> > -/* Note called with cinfo->lock held. */
> > -static int
> > -filelayout_scan_ds_commit_list(struct pnfs_commit_bucket *bucket,
> > -			       struct nfs_commit_info *cinfo,
> > -			       int max)
> > -{
> > -	struct list_head *src = &bucket->written;
> > -	struct list_head *dst = &bucket->committing;
> > -	int ret;
> > -
> > -	ret = transfer_commit_list(src, dst, cinfo, max);
> > -	if (ret) {
> > -		cinfo->ds->nwritten -= ret;
> > -		cinfo->ds->ncommitting += ret;
> > -		bucket->clseg = bucket->wlseg;
> > -		if (list_empty(src))
> > -			bucket->wlseg = NULL;
> > -		else
> > -			pnfs_get_lseg(bucket->clseg);
> > -	}
> > -	return ret;
> > -}
> > -
> > -/* Move reqs from written to committing lists, returning count of number moved.
> > - * Note called with cinfo->lock held.
> > - */
> > -static int filelayout_scan_commit_lists(struct nfs_commit_info *cinfo,
> > -					int max)
> > -{
> > -	int i, rv = 0, cnt;
> > -
> > -	for (i = 0; i < cinfo->ds->nbuckets && max != 0; i++) {
> > -		cnt = filelayout_scan_ds_commit_list(&cinfo->ds->buckets[i],
> > -						     cinfo, max);
> > -		max -= cnt;
> > -		rv += cnt;
> > -	}
> > -	return rv;
> > -}
> > -
> > -/* Pull everything off the committing lists and dump into @dst */
> > -static void filelayout_recover_commit_reqs(struct list_head *dst,
> > -					   struct nfs_commit_info *cinfo)
> > -{
> > -	struct pnfs_commit_bucket *b;
> > -	struct pnfs_layout_segment *freeme;
> > -	int i;
> > -
> > -restart:
> > -	spin_lock(cinfo->lock);
> > -	for (i = 0, b = cinfo->ds->buckets; i < cinfo->ds->nbuckets; i++, b++) {
> > -		if (transfer_commit_list(&b->written, dst, cinfo, 0)) {
> > -			freeme = b->wlseg;
> > -			b->wlseg = NULL;
> > -			spin_unlock(cinfo->lock);
> > -			pnfs_put_lseg(freeme);
> > -			goto restart;
> > -		}
> > -	}
> > -	cinfo->ds->nwritten = 0;
> > -	spin_unlock(cinfo->lock);
> > -}
> > -
> >  /* filelayout_search_commit_reqs - Search lists in @cinfo for the head reqest
> >   *				   for @page
> >   * @cinfo - commit info for current inode
> > @@ -1263,108 +1094,14 @@ filelayout_search_commit_reqs(struct nfs_commit_info *cinfo, struct page *page)
> >  	return NULL;
> >  }
> >  
> > -static void filelayout_retry_commit(struct nfs_commit_info *cinfo, int idx)
> > -{
> > -	struct pnfs_ds_commit_info *fl_cinfo = cinfo->ds;
> > -	struct pnfs_commit_bucket *bucket;
> > -	struct pnfs_layout_segment *freeme;
> > -	int i;
> > -
> > -	for (i = idx; i < fl_cinfo->nbuckets; i++) {
> > -		bucket = &fl_cinfo->buckets[i];
> > -		if (list_empty(&bucket->committing))
> > -			continue;
> > -		nfs_retry_commit(&bucket->committing, bucket->clseg, cinfo);
> > -		spin_lock(cinfo->lock);
> > -		freeme = bucket->clseg;
> > -		bucket->clseg = NULL;
> > -		spin_unlock(cinfo->lock);
> > -		pnfs_put_lseg(freeme);
> > -	}
> > -}
> > -
> > -static unsigned int
> > -alloc_ds_commits(struct nfs_commit_info *cinfo, struct list_head *list)
> > -{
> > -	struct pnfs_ds_commit_info *fl_cinfo;
> > -	struct pnfs_commit_bucket *bucket;
> > -	struct nfs_commit_data *data;
> > -	int i;
> > -	unsigned int nreq = 0;
> > -
> > -	fl_cinfo = cinfo->ds;
> > -	bucket = fl_cinfo->buckets;
> > -	for (i = 0; i < fl_cinfo->nbuckets; i++, bucket++) {
> > -		if (list_empty(&bucket->committing))
> > -			continue;
> > -		data = nfs_commitdata_alloc();
> > -		if (!data)
> > -			break;
> > -		data->ds_commit_index = i;
> > -		spin_lock(cinfo->lock);
> > -		data->lseg = bucket->clseg;
> > -		bucket->clseg = NULL;
> > -		spin_unlock(cinfo->lock);
> > -		list_add(&data->pages, list);
> > -		nreq++;
> > -	}
> > -
> > -	/* Clean up on error */
> > -	filelayout_retry_commit(cinfo, i);
> > -	/* Caller will clean up entries put on list */
> > -	return nreq;
> > -}
> > -
> > -/* This follows nfs_commit_list pretty closely */
> >  static int
> >  filelayout_commit_pagelist(struct inode *inode, struct list_head *mds_pages,
> >  			   int how, struct nfs_commit_info *cinfo)
> >  {
> > -	struct nfs_commit_data *data, *tmp;
> > -	LIST_HEAD(list);
> > -	unsigned int nreq = 0;
> > -
> > -	if (!list_empty(mds_pages)) {
> > -		data = nfs_commitdata_alloc();
> > -		if (data != NULL) {
> > -			data->lseg = NULL;
> > -			list_add(&data->pages, &list);
> > -			nreq++;
> > -		} else {
> > -			nfs_retry_commit(mds_pages, NULL, cinfo);
> > -			filelayout_retry_commit(cinfo, 0);
> > -			cinfo->completion_ops->error_cleanup(NFS_I(inode));
> > -			return -ENOMEM;
> > -		}
> > -	}
> > -
> > -	nreq += alloc_ds_commits(cinfo, &list);
> > -
> > -	if (nreq == 0) {
> > -		cinfo->completion_ops->error_cleanup(NFS_I(inode));
> > -		goto out;
> > -	}
> > -
> > -	atomic_add(nreq, &cinfo->mds->rpcs_out);
> > -
> > -	list_for_each_entry_safe(data, tmp, &list, pages) {
> > -		list_del_init(&data->pages);
> > -		if (!data->lseg) {
> > -			nfs_init_commit(data, mds_pages, NULL, cinfo);
> > -			nfs_initiate_commit(NFS_CLIENT(inode), data,
> > -					    data->mds_ops, how, 0);
> > -		} else {
> > -			struct pnfs_commit_bucket *buckets;
> > -
> > -			buckets = cinfo->ds->buckets;
> > -			nfs_init_commit(data, &buckets[data->ds_commit_index].committing, data->lseg, cinfo);
> > -			filelayout_initiate_commit(data, how);
> > -		}
> > -	}
> > -out:
> > -	cinfo->ds->ncommitting = 0;
> > -	return PNFS_ATTEMPTED;
> > +	return pnfs_generic_commit_pagelist(inode, mds_pages, how, cinfo,
> > +					    filelayout_initiate_commit);
> >  }
> > +
> >  static struct nfs4_deviceid_node *
> >  filelayout_alloc_deviceid_node(struct nfs_server *server,
> >  		struct pnfs_device *pdev, gfp_t gfp_flags)
> > @@ -1421,9 +1158,9 @@ static struct pnfs_layoutdriver_type filelayout_type = {
> >  	.pg_write_ops		= &filelayout_pg_write_ops,
> >  	.get_ds_info		= &filelayout_get_ds_info,
> >  	.mark_request_commit	= filelayout_mark_request_commit,
> > -	.clear_request_commit	= filelayout_clear_request_commit,
> > -	.scan_commit_lists	= filelayout_scan_commit_lists,
> > -	.recover_commit_reqs	= filelayout_recover_commit_reqs,
> > +	.clear_request_commit	= pnfs_generic_clear_request_commit,
> > +	.scan_commit_lists	= pnfs_generic_scan_commit_lists,
> > +	.recover_commit_reqs	= pnfs_generic_recover_commit_reqs,
> >  	.search_commit_reqs	= filelayout_search_commit_reqs,
> >  	.commit_pagelist	= filelayout_commit_pagelist,
> >  	.read_pagelist		= filelayout_read_pagelist,
> > diff --git a/fs/nfs/filelayout/filelayout.h b/fs/nfs/filelayout/filelayout.h
> > index 7c9f800..a5ce9b4 100644
> > --- a/fs/nfs/filelayout/filelayout.h
> > +++ b/fs/nfs/filelayout/filelayout.h
> > @@ -119,17 +119,6 @@ FILELAYOUT_DEVID_NODE(struct pnfs_layout_segment *lseg)
> >  	return &FILELAYOUT_LSEG(lseg)->dsaddr->id_node;
> >  }
> >  
> > -static inline void
> > -filelayout_mark_devid_invalid(struct nfs4_deviceid_node *node)
> > -{
> > -	u32 *p = (u32 *)&node->deviceid;
> > -
> > -	printk(KERN_WARNING "NFS: Deviceid [%x%x%x%x] marked out of use.\n",
> > -		p[0], p[1], p[2], p[3]);
> > -
> > -	set_bit(NFS_DEVICEID_INVALID, &node->flags);
> > -}
> > -
> >  static inline bool
> >  filelayout_test_devid_invalid(struct nfs4_deviceid_node *node)
> >  {
> > diff --git a/fs/nfs/filelayout/filelayoutdev.c b/fs/nfs/filelayout/filelayoutdev.c
> > index bfecac7..d21080a 100644
> > --- a/fs/nfs/filelayout/filelayoutdev.c
> > +++ b/fs/nfs/filelayout/filelayoutdev.c
> > @@ -708,7 +708,7 @@ nfs4_fl_prepare_ds(struct pnfs_layout_segment *lseg, u32 ds_idx)
> >  	if (ds == NULL) {
> >  		printk(KERN_ERR "NFS: %s: No data server for offset index %d\n",
> >  			__func__, ds_idx);
> > -		filelayout_mark_devid_invalid(devid);
> > +		pnfs_generic_mark_devid_invalid(devid);
> >  		goto out;
> >  	}
> >  	smp_rmb();
> > diff --git a/fs/nfs/pnfs.h b/fs/nfs/pnfs.h
> > index 9ae5b76..88eede0 100644
> > --- a/fs/nfs/pnfs.h
> > +++ b/fs/nfs/pnfs.h
> > @@ -275,6 +275,23 @@ void nfs4_mark_deviceid_unavailable(struct nfs4_deviceid_node *node);
> >  bool nfs4_test_deviceid_unavailable(struct nfs4_deviceid_node *node);
> >  void nfs4_deviceid_purge_client(const struct nfs_client *);
> >  
> > +/* pnfs_nfsio.c */
> > +void pnfs_generic_clear_request_commit(struct nfs_page *req,
> > +				       struct nfs_commit_info *cinfo);
> > +void pnfs_generic_commit_release(void *calldata);
> > +void pnfs_generic_prepare_to_resend_writes(struct nfs_commit_data *data);
> > +void pnfs_generic_rw_release(void *data);
> > +void pnfs_generic_recover_commit_reqs(struct list_head *dst,
> > +				      struct nfs_commit_info *cinfo);
> > +int pnfs_generic_commit_pagelist(struct inode *inode,
> > +				 struct list_head *mds_pages,
> > +				 int how,
> > +				 struct nfs_commit_info *cinfo,
> > +				 int (*initiate_commit)(struct nfs_commit_data *data,
> > +							int how));
> > +int pnfs_generic_scan_commit_lists(struct nfs_commit_info *cinfo, int max);
> > +void pnfs_generic_write_commit_done(struct rpc_task *task, void *data);
> > +
> >  static inline struct nfs4_deviceid_node *
> >  nfs4_get_deviceid(struct nfs4_deviceid_node *d)
> >  {
> > @@ -317,6 +334,12 @@ pnfs_get_ds_info(struct inode *inode)
> >  	return ld->get_ds_info(inode);
> >  }
> >  
> > +static inline void
> > +pnfs_generic_mark_devid_invalid(struct nfs4_deviceid_node *node)
> > +{
> > +	set_bit(NFS_DEVICEID_INVALID, &node->flags);
> > +}
> > +
> >  static inline bool
> >  pnfs_mark_request_commit(struct nfs_page *req, struct pnfs_layout_segment *lseg,
> >  			 struct nfs_commit_info *cinfo)
> > diff --git a/fs/nfs/pnfs_nfsio.c b/fs/nfs/pnfs_nfsio.c
> > new file mode 100644
> > index 0000000..e5f841c
> > --- /dev/null
> > +++ b/fs/nfs/pnfs_nfsio.c
> > @@ -0,0 +1,291 @@
> > +/*
> > + * Common NFS I/O  operations for the pnfs file based
> > + * layout drivers.
> > + *
> > + * Copyright (c) 2014, Primary Data, Inc. All rights reserved.
> > + *
> > + * Tom Haynes <loghyr@primarydata.com>
> > + */
> > +
> > +#include <linux/nfs_fs.h>
> > +#include <linux/nfs_page.h>
> > +
> > +#include "internal.h"
> > +#include "pnfs.h"
> > +
> > +static void pnfs_generic_fenceme(struct inode *inode,
> > +				 struct pnfs_layout_hdr *lo)
> > +{
> > +	if (!test_and_clear_bit(NFS_LAYOUT_RETURN, &lo->plh_flags))
> > +		return;
> > +	pnfs_return_layout(inode);
> > +}
> > +
> > +void pnfs_generic_rw_release(void *data)
> > +{
> > +	struct nfs_pgio_header *hdr = data;
> > +	struct pnfs_layout_hdr *lo = hdr->lseg->pls_layout;
> > +
> > +	pnfs_generic_fenceme(lo->plh_inode, lo);
> > +	nfs_put_client(hdr->ds_clp);
> > +	hdr->mds_ops->rpc_release(data);
> > +}
> > +EXPORT_SYMBOL_GPL(pnfs_generic_rw_release);
> > +
> > +/* Fake up some data that will cause nfs_commit_release to retry the writes. */
> > +void pnfs_generic_prepare_to_resend_writes(struct nfs_commit_data *data)
> > +{
> > +	struct nfs_page *first = nfs_list_entry(data->pages.next);
> > +
> > +	data->task.tk_status = 0;
> > +	memcpy(&data->verf.verifier, &first->wb_verf,
> > +	       sizeof(data->verf.verifier));
> > +	data->verf.verifier.data[0]++; /* ensure verifier mismatch */
> > +}
> > +EXPORT_SYMBOL_GPL(pnfs_generic_prepare_to_resend_writes);
> > +
> > +void pnfs_generic_write_commit_done(struct rpc_task *task, void *data)
> > +{
> > +	struct nfs_commit_data *wdata = data;
> > +
> > +	/* Note this may cause RPC to be resent */
> > +	wdata->mds_ops->rpc_call_done(task, data);
> > +}
> > +EXPORT_SYMBOL_GPL(pnfs_generic_write_commit_done);
> > +
> > +void pnfs_generic_commit_release(void *calldata)
> > +{
> > +	struct nfs_commit_data *data = calldata;
> > +
> > +	data->completion_ops->completion(data);
> > +	pnfs_put_lseg(data->lseg);
> > +	nfs_put_client(data->ds_clp);
> > +	nfs_commitdata_release(data);
> > +}
> > +EXPORT_SYMBOL_GPL(pnfs_generic_commit_release);
> > +
> > +/* The generic layer is about to remove the req from the commit list.
> > + * If this will make the bucket empty, it will need to put the lseg reference.
> > + * Note this is must be called holding the inode (/cinfo) lock
> > + */
> > +void
> > +pnfs_generic_clear_request_commit(struct nfs_page *req,
> > +				  struct nfs_commit_info *cinfo)
> > +{
> > +	struct pnfs_layout_segment *freeme = NULL;
> > +
> > +	if (!test_and_clear_bit(PG_COMMIT_TO_DS, &req->wb_flags))
> > +		goto out;
> > +	cinfo->ds->nwritten--;
> > +	if (list_is_singular(&req->wb_list)) {
> > +		struct pnfs_commit_bucket *bucket;
> > +
> > +		bucket = list_first_entry(&req->wb_list,
> > +					  struct pnfs_commit_bucket,
> > +					  written);
> > +		freeme = bucket->wlseg;
> > +		bucket->wlseg = NULL;
> > +	}
> > +out:
> > +	nfs_request_remove_commit_list(req, cinfo);
> > +	pnfs_put_lseg_locked(freeme);
> > +}
> > +EXPORT_SYMBOL_GPL(pnfs_generic_clear_request_commit);
> > +
> > +static int
> > +pnfs_generic_transfer_commit_list(struct list_head *src, struct list_head *dst,
> > +				  struct nfs_commit_info *cinfo, int max)
> > +{
> > +	struct nfs_page *req, *tmp;
> > +	int ret = 0;
> > +
> > +	list_for_each_entry_safe(req, tmp, src, wb_list) {
> > +		if (!nfs_lock_request(req))
> > +			continue;
> > +		kref_get(&req->wb_kref);
> > +		if (cond_resched_lock(cinfo->lock))
> > +			list_safe_reset_next(req, tmp, wb_list);
> > +		nfs_request_remove_commit_list(req, cinfo);
> > +		clear_bit(PG_COMMIT_TO_DS, &req->wb_flags);
> > +		nfs_list_add_request(req, dst);
> > +		ret++;
> > +		if ((ret == max) && !cinfo->dreq)
> > +			break;
> > +	}
> > +	return ret;
> > +}
> > +
> > +/* Note called with cinfo->lock held. */
> > +static int
> > +pnfs_generic_scan_ds_commit_list(struct pnfs_commit_bucket *bucket,
> > +				 struct nfs_commit_info *cinfo,
> > +				 int max)
> > +{
> > +	struct list_head *src = &bucket->written;
> > +	struct list_head *dst = &bucket->committing;
> > +	int ret;
> > +
> > +	ret = pnfs_generic_transfer_commit_list(src, dst, cinfo, max);
> > +	if (ret) {
> > +		cinfo->ds->nwritten -= ret;
> > +		cinfo->ds->ncommitting += ret;
> > +		bucket->clseg = bucket->wlseg;
> > +		if (list_empty(src))
> > +			bucket->wlseg = NULL;
> > +		else
> > +			pnfs_get_lseg(bucket->clseg);
> > +	}
> > +	return ret;
> > +}
> > +
> > +/* Move reqs from written to committing lists, returning count of number moved.
> > + * Note called with cinfo->lock held.
> > + */
> > +int pnfs_generic_scan_commit_lists(struct nfs_commit_info *cinfo,
> > +				   int max)
> > +{
> > +	int i, rv = 0, cnt;
> > +
> > +	for (i = 0; i < cinfo->ds->nbuckets && max != 0; i++) {
> > +		cnt = pnfs_generic_scan_ds_commit_list(&cinfo->ds->buckets[i],
> > +						       cinfo, max);
> > +		max -= cnt;
> > +		rv += cnt;
> > +	}
> > +	return rv;
> > +}
> > +EXPORT_SYMBOL_GPL(pnfs_generic_scan_commit_lists);
> > +
> > +/* Pull everything off the committing lists and dump into @dst */
> > +void pnfs_generic_recover_commit_reqs(struct list_head *dst,
> > +				      struct nfs_commit_info *cinfo)
> > +{
> > +	struct pnfs_commit_bucket *b;
> > +	struct pnfs_layout_segment *freeme;
> > +	int i;
> > +
> > +restart:
> > +	spin_lock(cinfo->lock);
> > +	for (i = 0, b = cinfo->ds->buckets; i < cinfo->ds->nbuckets; i++, b++) {
> > +		if (pnfs_generic_transfer_commit_list(&b->written, dst,
> > +						      cinfo, 0)) {
> > +			freeme = b->wlseg;
> > +			b->wlseg = NULL;
> > +			spin_unlock(cinfo->lock);
> > +			pnfs_put_lseg(freeme);
> > +			goto restart;
> > +		}
> > +	}
> > +	cinfo->ds->nwritten = 0;
> > +	spin_unlock(cinfo->lock);
> > +}
> > +EXPORT_SYMBOL_GPL(pnfs_generic_recover_commit_reqs);
> > +
> > +static void pnfs_generic_retry_commit(struct nfs_commit_info *cinfo, int idx)
> > +{
> > +	struct pnfs_ds_commit_info *fl_cinfo = cinfo->ds;
> > +	struct pnfs_commit_bucket *bucket;
> > +	struct pnfs_layout_segment *freeme;
> > +	int i;
> > +
> > +	for (i = idx; i < fl_cinfo->nbuckets; i++) {
> > +		bucket = &fl_cinfo->buckets[i];
> > +		if (list_empty(&bucket->committing))
> > +			continue;
> > +		nfs_retry_commit(&bucket->committing, bucket->clseg, cinfo);
> > +		spin_lock(cinfo->lock);
> > +		freeme = bucket->clseg;
> > +		bucket->clseg = NULL;
> > +		spin_unlock(cinfo->lock);
> > +		pnfs_put_lseg(freeme);
> > +	}
> > +}
> > +
> > +static unsigned int
> > +pnfs_generic_alloc_ds_commits(struct nfs_commit_info *cinfo,
> > +			      struct list_head *list)
> > +{
> > +	struct pnfs_ds_commit_info *fl_cinfo;
> > +	struct pnfs_commit_bucket *bucket;
> > +	struct nfs_commit_data *data;
> > +	int i;
> > +	unsigned int nreq = 0;
> > +
> > +	fl_cinfo = cinfo->ds;
> > +	bucket = fl_cinfo->buckets;
> > +	for (i = 0; i < fl_cinfo->nbuckets; i++, bucket++) {
> > +		if (list_empty(&bucket->committing))
> > +			continue;
> > +		data = nfs_commitdata_alloc();
> > +		if (!data)
> > +			break;
> > +		data->ds_commit_index = i;
> > +		spin_lock(cinfo->lock);
> > +		data->lseg = bucket->clseg;
> > +		bucket->clseg = NULL;
> > +		spin_unlock(cinfo->lock);
> > +		list_add(&data->pages, list);
> > +		nreq++;
> > +	}
> > +
> > +	/* Clean up on error */
> > +	pnfs_generic_retry_commit(cinfo, i);
> > +	return nreq;
> > +}
> > +
> > +/* This follows nfs_commit_list pretty closely */
> > +int
> > +pnfs_generic_commit_pagelist(struct inode *inode, struct list_head *mds_pages,
> > +			     int how, struct nfs_commit_info *cinfo,
> > +			     int (*initiate_commit)(struct nfs_commit_data *data,
> > +						    int how))
> > +{
> > +	struct nfs_commit_data *data, *tmp;
> > +	LIST_HEAD(list);
> > +	unsigned int nreq = 0;
> > +
> > +	if (!list_empty(mds_pages)) {
> > +		data = nfs_commitdata_alloc();
> > +		if (data != NULL) {
> > +			data->lseg = NULL;
> > +			list_add(&data->pages, &list);
> > +			nreq++;
> > +		} else {
> > +			nfs_retry_commit(mds_pages, NULL, cinfo);
> > +			pnfs_generic_retry_commit(cinfo, 0);
> > +			cinfo->completion_ops->error_cleanup(NFS_I(inode));
> > +			return -ENOMEM;
> > +		}
> > +	}
> > +
> > +	nreq += pnfs_generic_alloc_ds_commits(cinfo, &list);
> > +
> > +	if (nreq == 0) {
> > +		cinfo->completion_ops->error_cleanup(NFS_I(inode));
> > +		goto out;
> > +	}
> > +
> > +	atomic_add(nreq, &cinfo->mds->rpcs_out);
> > +
> > +	list_for_each_entry_safe(data, tmp, &list, pages) {
> > +		list_del_init(&data->pages);
> > +		if (!data->lseg) {
> > +			nfs_init_commit(data, mds_pages, NULL, cinfo);
> > +			nfs_initiate_commit(NFS_CLIENT(inode), data,
> > +					    data->mds_ops, how, 0);
> > +		} else {
> > +			struct pnfs_commit_bucket *buckets;
> > +
> > +			buckets = cinfo->ds->buckets;
> > +			nfs_init_commit(data,
> > +					&buckets[data->ds_commit_index].committing,
> > +					data->lseg,
> > +					cinfo);
> > +			initiate_commit(data, how);
> > +		}
> > +	}
> > +out:
> > +	cinfo->ds->ncommitting = 0;
> > +	return PNFS_ATTEMPTED;
> > +}
> > +EXPORT_SYMBOL_GPL(pnfs_generic_commit_pagelist);
> > 
> 
> --
> To unsubscribe from this list: send the line "unsubscribe linux-nfs" in
> the body of a message to majordomo@vger.kernel.org
> More majordomo info at  http://vger.kernel.org/majordomo-info.html
--
To unsubscribe from this list: send the line "unsubscribe linux-nfs" in
the body of a message to majordomo@vger.kernel.org
More majordomo info at  http://vger.kernel.org/majordomo-info.html
Christoph Hellwig Jan. 12, 2015, 7:13 p.m. UTC | #3
On Mon, Jan 12, 2015 at 11:10:05AM -0800, Tom Haynes wrote:
> > Can you remind me why the common code has to go into nfsv4.ko?  Can't this all be moved into a filelayout_common module?
> 
> Yes, it could. The main dependency I wanted to avoid
> was between the flex files and the files.
> 
> I did not want to add another module...

And for the tiny amount of shared code the module seems a bit pointless.
I think for now we should make sure it's nicely separated at the source
level, that way we can easily decided to move it into another module
as needed.
--
To unsubscribe from this list: send the line "unsubscribe linux-nfs" in
the body of a message to majordomo@vger.kernel.org
More majordomo info at  http://vger.kernel.org/majordomo-info.html
diff mbox

Patch

diff --git a/fs/nfs/Makefile b/fs/nfs/Makefile
index 04cb830..7973c4e3 100644
--- a/fs/nfs/Makefile
+++ b/fs/nfs/Makefile
@@ -27,7 +27,7 @@  nfsv4-y := nfs4proc.o nfs4xdr.o nfs4state.o nfs4renewd.o nfs4super.o nfs4file.o
 	  dns_resolve.o nfs4trace.o
 nfsv4-$(CONFIG_NFS_USE_LEGACY_DNS) += cache_lib.o
 nfsv4-$(CONFIG_SYSCTL)	+= nfs4sysctl.o
-nfsv4-$(CONFIG_NFS_V4_1)	+= pnfs.o pnfs_dev.o
+nfsv4-$(CONFIG_NFS_V4_1)	+= pnfs.o pnfs_dev.o pnfs_nfsio.o
 nfsv4-$(CONFIG_NFS_V4_2)	+= nfs42proc.o
 
 obj-$(CONFIG_PNFS_FILE_LAYOUT) += filelayout/
diff --git a/fs/nfs/filelayout/filelayout.c b/fs/nfs/filelayout/filelayout.c
index 7afb52f..bc36ed3 100644
--- a/fs/nfs/filelayout/filelayout.c
+++ b/fs/nfs/filelayout/filelayout.c
@@ -118,13 +118,6 @@  static void filelayout_reset_read(struct nfs_pgio_header *hdr)
 	}
 }
 
-static void filelayout_fenceme(struct inode *inode, struct pnfs_layout_hdr *lo)
-{
-	if (!test_and_clear_bit(NFS_LAYOUT_RETURN, &lo->plh_flags))
-		return;
-	pnfs_return_layout(inode);
-}
-
 static int filelayout_async_handle_error(struct rpc_task *task,
 					 struct nfs4_state *state,
 					 struct nfs_client *clp,
@@ -339,16 +332,6 @@  static void filelayout_read_count_stats(struct rpc_task *task, void *data)
 	rpc_count_iostats(task, NFS_SERVER(hdr->inode)->client->cl_metrics);
 }
 
-static void filelayout_read_release(void *data)
-{
-	struct nfs_pgio_header *hdr = data;
-	struct pnfs_layout_hdr *lo = hdr->lseg->pls_layout;
-
-	filelayout_fenceme(lo->plh_inode, lo);
-	nfs_put_client(hdr->ds_clp);
-	hdr->mds_ops->rpc_release(data);
-}
-
 static int filelayout_write_done_cb(struct rpc_task *task,
 				struct nfs_pgio_header *hdr)
 {
@@ -371,17 +354,6 @@  static int filelayout_write_done_cb(struct rpc_task *task,
 	return 0;
 }
 
-/* Fake up some data that will cause nfs_commit_release to retry the writes. */
-static void prepare_to_resend_writes(struct nfs_commit_data *data)
-{
-	struct nfs_page *first = nfs_list_entry(data->pages.next);
-
-	data->task.tk_status = 0;
-	memcpy(&data->verf.verifier, &first->wb_verf,
-	       sizeof(data->verf.verifier));
-	data->verf.verifier.data[0]++; /* ensure verifier mismatch */
-}
-
 static int filelayout_commit_done_cb(struct rpc_task *task,
 				     struct nfs_commit_data *data)
 {
@@ -393,7 +365,7 @@  static int filelayout_commit_done_cb(struct rpc_task *task,
 
 	switch (err) {
 	case -NFS4ERR_RESET_TO_MDS:
-		prepare_to_resend_writes(data);
+		pnfs_generic_prepare_to_resend_writes(data);
 		return -EAGAIN;
 	case -EAGAIN:
 		rpc_restart_call_prepare(task);
@@ -451,16 +423,6 @@  static void filelayout_write_count_stats(struct rpc_task *task, void *data)
 	rpc_count_iostats(task, NFS_SERVER(hdr->inode)->client->cl_metrics);
 }
 
-static void filelayout_write_release(void *data)
-{
-	struct nfs_pgio_header *hdr = data;
-	struct pnfs_layout_hdr *lo = hdr->lseg->pls_layout;
-
-	filelayout_fenceme(lo->plh_inode, lo);
-	nfs_put_client(hdr->ds_clp);
-	hdr->mds_ops->rpc_release(data);
-}
-
 static void filelayout_commit_prepare(struct rpc_task *task, void *data)
 {
 	struct nfs_commit_data *wdata = data;
@@ -471,14 +433,6 @@  static void filelayout_commit_prepare(struct rpc_task *task, void *data)
 			task);
 }
 
-static void filelayout_write_commit_done(struct rpc_task *task, void *data)
-{
-	struct nfs_commit_data *wdata = data;
-
-	/* Note this may cause RPC to be resent */
-	wdata->mds_ops->rpc_call_done(task, data);
-}
-
 static void filelayout_commit_count_stats(struct rpc_task *task, void *data)
 {
 	struct nfs_commit_data *cdata = data;
@@ -486,35 +440,25 @@  static void filelayout_commit_count_stats(struct rpc_task *task, void *data)
 	rpc_count_iostats(task, NFS_SERVER(cdata->inode)->client->cl_metrics);
 }
 
-static void filelayout_commit_release(void *calldata)
-{
-	struct nfs_commit_data *data = calldata;
-
-	data->completion_ops->completion(data);
-	pnfs_put_lseg(data->lseg);
-	nfs_put_client(data->ds_clp);
-	nfs_commitdata_release(data);
-}
-
 static const struct rpc_call_ops filelayout_read_call_ops = {
 	.rpc_call_prepare = filelayout_read_prepare,
 	.rpc_call_done = filelayout_read_call_done,
 	.rpc_count_stats = filelayout_read_count_stats,
-	.rpc_release = filelayout_read_release,
+	.rpc_release = pnfs_generic_rw_release,
 };
 
 static const struct rpc_call_ops filelayout_write_call_ops = {
 	.rpc_call_prepare = filelayout_write_prepare,
 	.rpc_call_done = filelayout_write_call_done,
 	.rpc_count_stats = filelayout_write_count_stats,
-	.rpc_release = filelayout_write_release,
+	.rpc_release = pnfs_generic_rw_release,
 };
 
 static const struct rpc_call_ops filelayout_commit_call_ops = {
 	.rpc_call_prepare = filelayout_commit_prepare,
-	.rpc_call_done = filelayout_write_commit_done,
+	.rpc_call_done = pnfs_generic_write_commit_done,
 	.rpc_count_stats = filelayout_commit_count_stats,
-	.rpc_release = filelayout_commit_release,
+	.rpc_release = pnfs_generic_commit_release,
 };
 
 static enum pnfs_try_status
@@ -1004,33 +948,6 @@  static u32 select_bucket_index(struct nfs4_filelayout_segment *fl, u32 j)
 		return j;
 }
 
-/* The generic layer is about to remove the req from the commit list.
- * If this will make the bucket empty, it will need to put the lseg reference.
- * Note this is must be called holding the inode (/cinfo) lock
- */
-static void
-filelayout_clear_request_commit(struct nfs_page *req,
-				struct nfs_commit_info *cinfo)
-{
-	struct pnfs_layout_segment *freeme = NULL;
-
-	if (!test_and_clear_bit(PG_COMMIT_TO_DS, &req->wb_flags))
-		goto out;
-	cinfo->ds->nwritten--;
-	if (list_is_singular(&req->wb_list)) {
-		struct pnfs_commit_bucket *bucket;
-
-		bucket = list_first_entry(&req->wb_list,
-					  struct pnfs_commit_bucket,
-					  written);
-		freeme = bucket->wlseg;
-		bucket->wlseg = NULL;
-	}
-out:
-	nfs_request_remove_commit_list(req, cinfo);
-	pnfs_put_lseg_locked(freeme);
-}
-
 static void
 filelayout_mark_request_commit(struct nfs_page *req,
 			       struct pnfs_layout_segment *lseg,
@@ -1064,7 +981,7 @@  filelayout_mark_request_commit(struct nfs_page *req,
 		 * is normally transferred to the COMMIT call and released
 		 * there.  It could also be released if the last req is pulled
 		 * off due to a rewrite, in which case it will be done in
-		 * filelayout_clear_request_commit
+		 * pnfs_generic_clear_request_commit
 		 */
 		buckets[i].wlseg = pnfs_get_lseg(lseg);
 	}
@@ -1142,97 +1059,11 @@  static int filelayout_initiate_commit(struct nfs_commit_data *data, int how)
 				   &filelayout_commit_call_ops, how,
 				   RPC_TASK_SOFTCONN);
 out_err:
-	prepare_to_resend_writes(data);
-	filelayout_commit_release(data);
+	pnfs_generic_prepare_to_resend_writes(data);
+	pnfs_generic_commit_release(data);
 	return -EAGAIN;
 }
 
-static int
-transfer_commit_list(struct list_head *src, struct list_head *dst,
-		     struct nfs_commit_info *cinfo, int max)
-{
-	struct nfs_page *req, *tmp;
-	int ret = 0;
-
-	list_for_each_entry_safe(req, tmp, src, wb_list) {
-		if (!nfs_lock_request(req))
-			continue;
-		kref_get(&req->wb_kref);
-		if (cond_resched_lock(cinfo->lock))
-			list_safe_reset_next(req, tmp, wb_list);
-		nfs_request_remove_commit_list(req, cinfo);
-		clear_bit(PG_COMMIT_TO_DS, &req->wb_flags);
-		nfs_list_add_request(req, dst);
-		ret++;
-		if ((ret == max) && !cinfo->dreq)
-			break;
-	}
-	return ret;
-}
-
-/* Note called with cinfo->lock held. */
-static int
-filelayout_scan_ds_commit_list(struct pnfs_commit_bucket *bucket,
-			       struct nfs_commit_info *cinfo,
-			       int max)
-{
-	struct list_head *src = &bucket->written;
-	struct list_head *dst = &bucket->committing;
-	int ret;
-
-	ret = transfer_commit_list(src, dst, cinfo, max);
-	if (ret) {
-		cinfo->ds->nwritten -= ret;
-		cinfo->ds->ncommitting += ret;
-		bucket->clseg = bucket->wlseg;
-		if (list_empty(src))
-			bucket->wlseg = NULL;
-		else
-			pnfs_get_lseg(bucket->clseg);
-	}
-	return ret;
-}
-
-/* Move reqs from written to committing lists, returning count of number moved.
- * Note called with cinfo->lock held.
- */
-static int filelayout_scan_commit_lists(struct nfs_commit_info *cinfo,
-					int max)
-{
-	int i, rv = 0, cnt;
-
-	for (i = 0; i < cinfo->ds->nbuckets && max != 0; i++) {
-		cnt = filelayout_scan_ds_commit_list(&cinfo->ds->buckets[i],
-						     cinfo, max);
-		max -= cnt;
-		rv += cnt;
-	}
-	return rv;
-}
-
-/* Pull everything off the committing lists and dump into @dst */
-static void filelayout_recover_commit_reqs(struct list_head *dst,
-					   struct nfs_commit_info *cinfo)
-{
-	struct pnfs_commit_bucket *b;
-	struct pnfs_layout_segment *freeme;
-	int i;
-
-restart:
-	spin_lock(cinfo->lock);
-	for (i = 0, b = cinfo->ds->buckets; i < cinfo->ds->nbuckets; i++, b++) {
-		if (transfer_commit_list(&b->written, dst, cinfo, 0)) {
-			freeme = b->wlseg;
-			b->wlseg = NULL;
-			spin_unlock(cinfo->lock);
-			pnfs_put_lseg(freeme);
-			goto restart;
-		}
-	}
-	cinfo->ds->nwritten = 0;
-	spin_unlock(cinfo->lock);
-}
-
 /* filelayout_search_commit_reqs - Search lists in @cinfo for the head reqest
  *				   for @page
  * @cinfo - commit info for current inode
@@ -1263,108 +1094,14 @@  filelayout_search_commit_reqs(struct nfs_commit_info *cinfo, struct page *page)
 	return NULL;
 }
 
-static void filelayout_retry_commit(struct nfs_commit_info *cinfo, int idx)
-{
-	struct pnfs_ds_commit_info *fl_cinfo = cinfo->ds;
-	struct pnfs_commit_bucket *bucket;
-	struct pnfs_layout_segment *freeme;
-	int i;
-
-	for (i = idx; i < fl_cinfo->nbuckets; i++) {
-		bucket = &fl_cinfo->buckets[i];
-		if (list_empty(&bucket->committing))
-			continue;
-		nfs_retry_commit(&bucket->committing, bucket->clseg, cinfo);
-		spin_lock(cinfo->lock);
-		freeme = bucket->clseg;
-		bucket->clseg = NULL;
-		spin_unlock(cinfo->lock);
-		pnfs_put_lseg(freeme);
-	}
-}
-
-static unsigned int
-alloc_ds_commits(struct nfs_commit_info *cinfo, struct list_head *list)
-{
-	struct pnfs_ds_commit_info *fl_cinfo;
-	struct pnfs_commit_bucket *bucket;
-	struct nfs_commit_data *data;
-	int i;
-	unsigned int nreq = 0;
-
-	fl_cinfo = cinfo->ds;
-	bucket = fl_cinfo->buckets;
-	for (i = 0; i < fl_cinfo->nbuckets; i++, bucket++) {
-		if (list_empty(&bucket->committing))
-			continue;
-		data = nfs_commitdata_alloc();
-		if (!data)
-			break;
-		data->ds_commit_index = i;
-		spin_lock(cinfo->lock);
-		data->lseg = bucket->clseg;
-		bucket->clseg = NULL;
-		spin_unlock(cinfo->lock);
-		list_add(&data->pages, list);
-		nreq++;
-	}
-
-	/* Clean up on error */
-	filelayout_retry_commit(cinfo, i);
-	/* Caller will clean up entries put on list */
-	return nreq;
-}
-
-/* This follows nfs_commit_list pretty closely */
 static int
 filelayout_commit_pagelist(struct inode *inode, struct list_head *mds_pages,
 			   int how, struct nfs_commit_info *cinfo)
 {
-	struct nfs_commit_data *data, *tmp;
-	LIST_HEAD(list);
-	unsigned int nreq = 0;
-
-	if (!list_empty(mds_pages)) {
-		data = nfs_commitdata_alloc();
-		if (data != NULL) {
-			data->lseg = NULL;
-			list_add(&data->pages, &list);
-			nreq++;
-		} else {
-			nfs_retry_commit(mds_pages, NULL, cinfo);
-			filelayout_retry_commit(cinfo, 0);
-			cinfo->completion_ops->error_cleanup(NFS_I(inode));
-			return -ENOMEM;
-		}
-	}
-
-	nreq += alloc_ds_commits(cinfo, &list);
-
-	if (nreq == 0) {
-		cinfo->completion_ops->error_cleanup(NFS_I(inode));
-		goto out;
-	}
-
-	atomic_add(nreq, &cinfo->mds->rpcs_out);
-
-	list_for_each_entry_safe(data, tmp, &list, pages) {
-		list_del_init(&data->pages);
-		if (!data->lseg) {
-			nfs_init_commit(data, mds_pages, NULL, cinfo);
-			nfs_initiate_commit(NFS_CLIENT(inode), data,
-					    data->mds_ops, how, 0);
-		} else {
-			struct pnfs_commit_bucket *buckets;
-
-			buckets = cinfo->ds->buckets;
-			nfs_init_commit(data, &buckets[data->ds_commit_index].committing, data->lseg, cinfo);
-			filelayout_initiate_commit(data, how);
-		}
-	}
-out:
-	cinfo->ds->ncommitting = 0;
-	return PNFS_ATTEMPTED;
+	return pnfs_generic_commit_pagelist(inode, mds_pages, how, cinfo,
+					    filelayout_initiate_commit);
 }
+
 static struct nfs4_deviceid_node *
 filelayout_alloc_deviceid_node(struct nfs_server *server,
 		struct pnfs_device *pdev, gfp_t gfp_flags)
@@ -1421,9 +1158,9 @@  static struct pnfs_layoutdriver_type filelayout_type = {
 	.pg_write_ops		= &filelayout_pg_write_ops,
 	.get_ds_info		= &filelayout_get_ds_info,
 	.mark_request_commit	= filelayout_mark_request_commit,
-	.clear_request_commit	= filelayout_clear_request_commit,
-	.scan_commit_lists	= filelayout_scan_commit_lists,
-	.recover_commit_reqs	= filelayout_recover_commit_reqs,
+	.clear_request_commit	= pnfs_generic_clear_request_commit,
+	.scan_commit_lists	= pnfs_generic_scan_commit_lists,
+	.recover_commit_reqs	= pnfs_generic_recover_commit_reqs,
 	.search_commit_reqs	= filelayout_search_commit_reqs,
 	.commit_pagelist	= filelayout_commit_pagelist,
 	.read_pagelist		= filelayout_read_pagelist,
diff --git a/fs/nfs/filelayout/filelayout.h b/fs/nfs/filelayout/filelayout.h
index 7c9f800..a5ce9b4 100644
--- a/fs/nfs/filelayout/filelayout.h
+++ b/fs/nfs/filelayout/filelayout.h
@@ -119,17 +119,6 @@  FILELAYOUT_DEVID_NODE(struct pnfs_layout_segment *lseg)
 	return &FILELAYOUT_LSEG(lseg)->dsaddr->id_node;
 }
 
-static inline void
-filelayout_mark_devid_invalid(struct nfs4_deviceid_node *node)
-{
-	u32 *p = (u32 *)&node->deviceid;
-
-	printk(KERN_WARNING "NFS: Deviceid [%x%x%x%x] marked out of use.\n",
-		p[0], p[1], p[2], p[3]);
-
-	set_bit(NFS_DEVICEID_INVALID, &node->flags);
-}
-
 static inline bool
 filelayout_test_devid_invalid(struct nfs4_deviceid_node *node)
 {
diff --git a/fs/nfs/filelayout/filelayoutdev.c b/fs/nfs/filelayout/filelayoutdev.c
index bfecac7..d21080a 100644
--- a/fs/nfs/filelayout/filelayoutdev.c
+++ b/fs/nfs/filelayout/filelayoutdev.c
@@ -708,7 +708,7 @@  nfs4_fl_prepare_ds(struct pnfs_layout_segment *lseg, u32 ds_idx)
 	if (ds == NULL) {
 		printk(KERN_ERR "NFS: %s: No data server for offset index %d\n",
 			__func__, ds_idx);
-		filelayout_mark_devid_invalid(devid);
+		pnfs_generic_mark_devid_invalid(devid);
 		goto out;
 	}
 	smp_rmb();
diff --git a/fs/nfs/pnfs.h b/fs/nfs/pnfs.h
index 9ae5b76..88eede0 100644
--- a/fs/nfs/pnfs.h
+++ b/fs/nfs/pnfs.h
@@ -275,6 +275,23 @@  void nfs4_mark_deviceid_unavailable(struct nfs4_deviceid_node *node);
 bool nfs4_test_deviceid_unavailable(struct nfs4_deviceid_node *node);
 void nfs4_deviceid_purge_client(const struct nfs_client *);
 
+/* pnfs_nfsio.c */
+void pnfs_generic_clear_request_commit(struct nfs_page *req,
+				       struct nfs_commit_info *cinfo);
+void pnfs_generic_commit_release(void *calldata);
+void pnfs_generic_prepare_to_resend_writes(struct nfs_commit_data *data);
+void pnfs_generic_rw_release(void *data);
+void pnfs_generic_recover_commit_reqs(struct list_head *dst,
+				      struct nfs_commit_info *cinfo);
+int pnfs_generic_commit_pagelist(struct inode *inode,
+				 struct list_head *mds_pages,
+				 int how,
+				 struct nfs_commit_info *cinfo,
+				 int (*initiate_commit)(struct nfs_commit_data *data,
+							int how));
+int pnfs_generic_scan_commit_lists(struct nfs_commit_info *cinfo, int max);
+void pnfs_generic_write_commit_done(struct rpc_task *task, void *data);
+
 static inline struct nfs4_deviceid_node *
 nfs4_get_deviceid(struct nfs4_deviceid_node *d)
 {
@@ -317,6 +334,12 @@  pnfs_get_ds_info(struct inode *inode)
 	return ld->get_ds_info(inode);
 }
 
+static inline void
+pnfs_generic_mark_devid_invalid(struct nfs4_deviceid_node *node)
+{
+	set_bit(NFS_DEVICEID_INVALID, &node->flags);
+}
+
 static inline bool
 pnfs_mark_request_commit(struct nfs_page *req, struct pnfs_layout_segment *lseg,
 			 struct nfs_commit_info *cinfo)
diff --git a/fs/nfs/pnfs_nfsio.c b/fs/nfs/pnfs_nfsio.c
new file mode 100644
index 0000000..e5f841c
--- /dev/null
+++ b/fs/nfs/pnfs_nfsio.c
@@ -0,0 +1,291 @@ 
+/*
+ * Common NFS I/O  operations for the pnfs file based
+ * layout drivers.
+ *
+ * Copyright (c) 2014, Primary Data, Inc. All rights reserved.
+ *
+ * Tom Haynes <loghyr@primarydata.com>
+ */
+
+#include <linux/nfs_fs.h>
+#include <linux/nfs_page.h>
+
+#include "internal.h"
+#include "pnfs.h"
+
+static void pnfs_generic_fenceme(struct inode *inode,
+				 struct pnfs_layout_hdr *lo)
+{
+	if (!test_and_clear_bit(NFS_LAYOUT_RETURN, &lo->plh_flags))
+		return;
+	pnfs_return_layout(inode);
+}
+
+void pnfs_generic_rw_release(void *data)
+{
+	struct nfs_pgio_header *hdr = data;
+	struct pnfs_layout_hdr *lo = hdr->lseg->pls_layout;
+
+	pnfs_generic_fenceme(lo->plh_inode, lo);
+	nfs_put_client(hdr->ds_clp);
+	hdr->mds_ops->rpc_release(data);
+}
+EXPORT_SYMBOL_GPL(pnfs_generic_rw_release);
+
+/* Fake up some data that will cause nfs_commit_release to retry the writes. */
+void pnfs_generic_prepare_to_resend_writes(struct nfs_commit_data *data)
+{
+	struct nfs_page *first = nfs_list_entry(data->pages.next);
+
+	data->task.tk_status = 0;
+	memcpy(&data->verf.verifier, &first->wb_verf,
+	       sizeof(data->verf.verifier));
+	data->verf.verifier.data[0]++; /* ensure verifier mismatch */
+}
+EXPORT_SYMBOL_GPL(pnfs_generic_prepare_to_resend_writes);
+
+void pnfs_generic_write_commit_done(struct rpc_task *task, void *data)
+{
+	struct nfs_commit_data *wdata = data;
+
+	/* Note this may cause RPC to be resent */
+	wdata->mds_ops->rpc_call_done(task, data);
+}
+EXPORT_SYMBOL_GPL(pnfs_generic_write_commit_done);
+
+void pnfs_generic_commit_release(void *calldata)
+{
+	struct nfs_commit_data *data = calldata;
+
+	data->completion_ops->completion(data);
+	pnfs_put_lseg(data->lseg);
+	nfs_put_client(data->ds_clp);
+	nfs_commitdata_release(data);
+}
+EXPORT_SYMBOL_GPL(pnfs_generic_commit_release);
+
+/* The generic layer is about to remove the req from the commit list.
+ * If this will make the bucket empty, it will need to put the lseg reference.
+ * Note this is must be called holding the inode (/cinfo) lock
+ */
+void
+pnfs_generic_clear_request_commit(struct nfs_page *req,
+				  struct nfs_commit_info *cinfo)
+{
+	struct pnfs_layout_segment *freeme = NULL;
+
+	if (!test_and_clear_bit(PG_COMMIT_TO_DS, &req->wb_flags))
+		goto out;
+	cinfo->ds->nwritten--;
+	if (list_is_singular(&req->wb_list)) {
+		struct pnfs_commit_bucket *bucket;
+
+		bucket = list_first_entry(&req->wb_list,
+					  struct pnfs_commit_bucket,
+					  written);
+		freeme = bucket->wlseg;
+		bucket->wlseg = NULL;
+	}
+out:
+	nfs_request_remove_commit_list(req, cinfo);
+	pnfs_put_lseg_locked(freeme);
+}
+EXPORT_SYMBOL_GPL(pnfs_generic_clear_request_commit);
+
+static int
+pnfs_generic_transfer_commit_list(struct list_head *src, struct list_head *dst,
+				  struct nfs_commit_info *cinfo, int max)
+{
+	struct nfs_page *req, *tmp;
+	int ret = 0;
+
+	list_for_each_entry_safe(req, tmp, src, wb_list) {
+		if (!nfs_lock_request(req))
+			continue;
+		kref_get(&req->wb_kref);
+		if (cond_resched_lock(cinfo->lock))
+			list_safe_reset_next(req, tmp, wb_list);
+		nfs_request_remove_commit_list(req, cinfo);
+		clear_bit(PG_COMMIT_TO_DS, &req->wb_flags);
+		nfs_list_add_request(req, dst);
+		ret++;
+		if ((ret == max) && !cinfo->dreq)
+			break;
+	}
+	return ret;
+}
+
+/* Note called with cinfo->lock held. */
+static int
+pnfs_generic_scan_ds_commit_list(struct pnfs_commit_bucket *bucket,
+				 struct nfs_commit_info *cinfo,
+				 int max)
+{
+	struct list_head *src = &bucket->written;
+	struct list_head *dst = &bucket->committing;
+	int ret;
+
+	ret = pnfs_generic_transfer_commit_list(src, dst, cinfo, max);
+	if (ret) {
+		cinfo->ds->nwritten -= ret;
+		cinfo->ds->ncommitting += ret;
+		bucket->clseg = bucket->wlseg;
+		if (list_empty(src))
+			bucket->wlseg = NULL;
+		else
+			pnfs_get_lseg(bucket->clseg);
+	}
+	return ret;
+}
+
+/* Move reqs from written to committing lists, returning count of number moved.
+ * Note called with cinfo->lock held.
+ */
+int pnfs_generic_scan_commit_lists(struct nfs_commit_info *cinfo,
+				   int max)
+{
+	int i, rv = 0, cnt;
+
+	for (i = 0; i < cinfo->ds->nbuckets && max != 0; i++) {
+		cnt = pnfs_generic_scan_ds_commit_list(&cinfo->ds->buckets[i],
+						       cinfo, max);
+		max -= cnt;
+		rv += cnt;
+	}
+	return rv;
+}
+EXPORT_SYMBOL_GPL(pnfs_generic_scan_commit_lists);
+
+/* Pull everything off the committing lists and dump into @dst */
+void pnfs_generic_recover_commit_reqs(struct list_head *dst,
+				      struct nfs_commit_info *cinfo)
+{
+	struct pnfs_commit_bucket *b;
+	struct pnfs_layout_segment *freeme;
+	int i;
+
+restart:
+	spin_lock(cinfo->lock);
+	for (i = 0, b = cinfo->ds->buckets; i < cinfo->ds->nbuckets; i++, b++) {
+		if (pnfs_generic_transfer_commit_list(&b->written, dst,
+						      cinfo, 0)) {
+			freeme = b->wlseg;
+			b->wlseg = NULL;
+			spin_unlock(cinfo->lock);
+			pnfs_put_lseg(freeme);
+			goto restart;
+		}
+	}
+	cinfo->ds->nwritten = 0;
+	spin_unlock(cinfo->lock);
+}
+EXPORT_SYMBOL_GPL(pnfs_generic_recover_commit_reqs);
+
+static void pnfs_generic_retry_commit(struct nfs_commit_info *cinfo, int idx)
+{
+	struct pnfs_ds_commit_info *fl_cinfo = cinfo->ds;
+	struct pnfs_commit_bucket *bucket;
+	struct pnfs_layout_segment *freeme;
+	int i;
+
+	for (i = idx; i < fl_cinfo->nbuckets; i++) {
+		bucket = &fl_cinfo->buckets[i];
+		if (list_empty(&bucket->committing))
+			continue;
+		nfs_retry_commit(&bucket->committing, bucket->clseg, cinfo);
+		spin_lock(cinfo->lock);
+		freeme = bucket->clseg;
+		bucket->clseg = NULL;
+		spin_unlock(cinfo->lock);
+		pnfs_put_lseg(freeme);
+	}
+}
+
+static unsigned int
+pnfs_generic_alloc_ds_commits(struct nfs_commit_info *cinfo,
+			      struct list_head *list)
+{
+	struct pnfs_ds_commit_info *fl_cinfo;
+	struct pnfs_commit_bucket *bucket;
+	struct nfs_commit_data *data;
+	int i;
+	unsigned int nreq = 0;
+
+	fl_cinfo = cinfo->ds;
+	bucket = fl_cinfo->buckets;
+	for (i = 0; i < fl_cinfo->nbuckets; i++, bucket++) {
+		if (list_empty(&bucket->committing))
+			continue;
+		data = nfs_commitdata_alloc();
+		if (!data)
+			break;
+		data->ds_commit_index = i;
+		spin_lock(cinfo->lock);
+		data->lseg = bucket->clseg;
+		bucket->clseg = NULL;
+		spin_unlock(cinfo->lock);
+		list_add(&data->pages, list);
+		nreq++;
+	}
+
+	/* Clean up on error */
+	pnfs_generic_retry_commit(cinfo, i);
+	return nreq;
+}
+
+/* This follows nfs_commit_list pretty closely */
+int
+pnfs_generic_commit_pagelist(struct inode *inode, struct list_head *mds_pages,
+			     int how, struct nfs_commit_info *cinfo,
+			     int (*initiate_commit)(struct nfs_commit_data *data,
+						    int how))
+{
+	struct nfs_commit_data *data, *tmp;
+	LIST_HEAD(list);
+	unsigned int nreq = 0;
+
+	if (!list_empty(mds_pages)) {
+		data = nfs_commitdata_alloc();
+		if (data != NULL) {
+			data->lseg = NULL;
+			list_add(&data->pages, &list);
+			nreq++;
+		} else {
+			nfs_retry_commit(mds_pages, NULL, cinfo);
+			pnfs_generic_retry_commit(cinfo, 0);
+			cinfo->completion_ops->error_cleanup(NFS_I(inode));
+			return -ENOMEM;
+		}
+	}
+
+	nreq += pnfs_generic_alloc_ds_commits(cinfo, &list);
+
+	if (nreq == 0) {
+		cinfo->completion_ops->error_cleanup(NFS_I(inode));
+		goto out;
+	}
+
+	atomic_add(nreq, &cinfo->mds->rpcs_out);
+
+	list_for_each_entry_safe(data, tmp, &list, pages) {
+		list_del_init(&data->pages);
+		if (!data->lseg) {
+			nfs_init_commit(data, mds_pages, NULL, cinfo);
+			nfs_initiate_commit(NFS_CLIENT(inode), data,
+					    data->mds_ops, how, 0);
+		} else {
+			struct pnfs_commit_bucket *buckets;
+
+			buckets = cinfo->ds->buckets;
+			nfs_init_commit(data,
+					&buckets[data->ds_commit_index].committing,
+					data->lseg,
+					cinfo);
+			initiate_commit(data, how);
+		}
+	}
+out:
+	cinfo->ds->ncommitting = 0;
+	return PNFS_ATTEMPTED;
+}
+EXPORT_SYMBOL_GPL(pnfs_generic_commit_pagelist);