| Message ID | 1414520695-1701-1-git-send-email-jlayton@primarydata.com (mailing list archive) |
|---|---|
| State | New, archived |
| Headers | show |
On Tue, 28 Oct 2014 14:24:55 -0400 Jeff Layton <jlayton@primarydata.com> wrote: > It's possible to get a dump of the RPC task queue by writing a value to > /proc/sys/sunrpc/rpc_debug. If you write any value to that file, you get > a dump of the RPC client task list into the log buffer. This is a rather > inconvenient interface however, and makes it hard to get immediate info > about the task queue. > > Add a new file under debugfs that shows similar info. There are some > small differences -- we avoid printing raw kernel addresses in favor of > symbolic names and the XID is also displayed. > > Signed-off-by: Jeff Layton <jlayton@primarydata.com> > --- > include/linux/sunrpc/clnt.h | 1 + > include/linux/sunrpc/debug.h | 13 ++++ > net/sunrpc/Kconfig | 1 + > net/sunrpc/Makefile | 1 + > net/sunrpc/clnt.c | 3 +- > net/sunrpc/debugfs.c | 153 +++++++++++++++++++++++++++++++++++++++++++ > net/sunrpc/sunrpc_syms.c | 8 +++ > 7 files changed, 179 insertions(+), 1 deletion(-) > create mode 100644 net/sunrpc/debugfs.c > > diff --git a/include/linux/sunrpc/clnt.h b/include/linux/sunrpc/clnt.h > index 70736b98c721..265d40bd3109 100644 > --- a/include/linux/sunrpc/clnt.h > +++ b/include/linux/sunrpc/clnt.h > @@ -176,5 +176,6 @@ size_t rpc_peeraddr(struct rpc_clnt *, struct sockaddr *, size_t); > const char *rpc_peeraddr2str(struct rpc_clnt *, enum rpc_display_format_t); > int rpc_localaddr(struct rpc_clnt *, struct sockaddr *, size_t); > > +const char *rpc_proc_name(const struct rpc_task *task); > #endif /* __KERNEL__ */ > #endif /* _LINUX_SUNRPC_CLNT_H */ > diff --git a/include/linux/sunrpc/debug.h b/include/linux/sunrpc/debug.h > index 9385bd74c860..7c1ef9530087 100644 > --- a/include/linux/sunrpc/debug.h > +++ b/include/linux/sunrpc/debug.h > @@ -68,6 +68,19 @@ extern unsigned int nlm_debug; > #ifdef RPC_DEBUG > void rpc_register_sysctl(void); > void rpc_unregister_sysctl(void); > +int sunrpc_debugfs_init(void); > +void sunrpc_debugfs_exit(void); > +#else > +static inline int > +sunrpc_debugfs_init(void) > +{ > + return 0; > +} > +static inline void > +sunrpc_debugfs_exit(void) > +{ > + return; > +} > #endif > > #endif /* _LINUX_SUNRPC_DEBUG_H_ */ > diff --git a/net/sunrpc/Kconfig b/net/sunrpc/Kconfig > index 0754d0f466d2..fb78117b896c 100644 > --- a/net/sunrpc/Kconfig > +++ b/net/sunrpc/Kconfig > @@ -35,6 +35,7 @@ config RPCSEC_GSS_KRB5 > config SUNRPC_DEBUG > bool "RPC: Enable dprintk debugging" > depends on SUNRPC && SYSCTL > + select DEBUG_FS > help > This option enables a sysctl-based debugging interface > that is be used by the 'rpcdebug' utility to turn on or off > diff --git a/net/sunrpc/Makefile b/net/sunrpc/Makefile > index e5a7a1cac8f3..15e6f6c23c5d 100644 > --- a/net/sunrpc/Makefile > +++ b/net/sunrpc/Makefile > @@ -14,6 +14,7 @@ sunrpc-y := clnt.o xprt.o socklib.o xprtsock.o sched.o \ > addr.o rpcb_clnt.o timer.o xdr.o \ > sunrpc_syms.o cache.o rpc_pipe.o \ > svc_xprt.o > +sunrpc-$(CONFIG_SUNRPC_DEBUG) += debugfs.o > sunrpc-$(CONFIG_SUNRPC_BACKCHANNEL) += backchannel_rqst.o bc_svc.o > sunrpc-$(CONFIG_PROC_FS) += stats.o > sunrpc-$(CONFIG_SYSCTL) += sysctl.o > diff --git a/net/sunrpc/clnt.c b/net/sunrpc/clnt.c > index 9acd6ce88db7..5b2e2d3d37c1 100644 > --- a/net/sunrpc/clnt.c > +++ b/net/sunrpc/clnt.c > @@ -1397,7 +1397,8 @@ rpc_restart_call(struct rpc_task *task) > EXPORT_SYMBOL_GPL(rpc_restart_call); > > #ifdef RPC_DEBUG > -static const char *rpc_proc_name(const struct rpc_task *task) > +const char > +*rpc_proc_name(const struct rpc_task *task) > { > const struct rpc_procinfo *proc = task->tk_msg.rpc_proc; > > diff --git a/net/sunrpc/debugfs.c b/net/sunrpc/debugfs.c > new file mode 100644 > index 000000000000..48769338172e > --- /dev/null > +++ b/net/sunrpc/debugfs.c > @@ -0,0 +1,153 @@ > +/** > + * debugfs interface for sunrpc > + * > + * (c) 2014 Jeff Layton <jlayton@primarydata.com> > + */ > + > +#include <linux/debugfs.h> > +#include <linux/sunrpc/sched.h> > +#include <linux/sunrpc/clnt.h> > +#include "netns.h" > + > +static struct dentry *topdir; > + > +static int > +tasks_show(struct seq_file *f, void *v) > +{ > + u32 xid = 0; > + struct rpc_task *task = v; > + struct rpc_clnt *clnt = task->tk_client; > + const char *rpc_waitq = "none"; > + > + if (RPC_IS_QUEUED(task)) > + rpc_waitq = rpc_qname(task->tk_waitqueue); > + > + if (task->tk_rqstp) > + xid = be32_to_cpu(task->tk_rqstp->rq_xid); > + > + seq_printf(f, "%5u %04x %6d 0x%x 0x%x %8ld %ps %sv%u %s a:%ps q:%s\n", > + task->tk_pid, task->tk_flags, task->tk_status, > + clnt->cl_clid, xid, task->tk_timeout, task->tk_ops, > + clnt->cl_program->name, clnt->cl_vers, rpc_proc_name(task), > + task->tk_action, rpc_waitq); > + return 0; > +} > + > +static void * > +tasks_start(struct seq_file *f, loff_t *ppos) > + __acquires(&sn->rpc_client_lock) > + __acquires(&clnt->cl_lock) > +{ > + loff_t *p = f->private, pos = *ppos; > + struct rpc_clnt *clnt; > + struct rpc_task *task; > + struct sunrpc_net *sn = net_generic(current->nsproxy->net_ns, > + sunrpc_net_id); > + > + *p = pos + 1; > + spin_lock(&sn->rpc_client_lock); > + list_for_each_entry(clnt, &sn->all_clients, cl_clients) { > + spin_lock(&clnt->cl_lock); > + list_for_each_entry(task, &clnt->cl_tasks, tk_task) > + if (pos-- == 0) > + return task; > + spin_unlock(&clnt->cl_lock); > + } > + return NULL; > +} > + > +static void * > +tasks_next(struct seq_file *f, void *v, loff_t *pos) > + __releases(&clnt->cl_lock) > + __acquires(&clnt->cl_lock) > +{ > + loff_t *p = f->private; > + struct rpc_task *task = v; > + struct rpc_clnt *clnt = task->tk_client; > + struct list_head *next = task->tk_task.next; > + struct sunrpc_net *sn = net_generic(current->nsproxy->net_ns, > + sunrpc_net_id); > + > + ++*p; > + ++*pos; > + > + /* If there's another task on list, return it */ > + if (next != &clnt->cl_tasks) > + return list_entry(next, struct rpc_task, tk_task); > + > + /* We're finished with this client, move on */ > + spin_unlock(&clnt->cl_lock); > +next_client: > + next = clnt->cl_clients.next; > + if (next == &sn->all_clients) > + return NULL; > + > + clnt = list_entry(next, struct rpc_clnt, cl_clients); > + if (list_empty(&clnt->cl_tasks)) > + goto next_client; > + spin_lock(&clnt->cl_lock); > + if (list_empty(&clnt->cl_tasks)) { > + spin_unlock(&clnt->cl_lock); > + goto next_client; > + } > + return list_first_entry(&clnt->cl_tasks, struct rpc_task, tk_task); > +} > + > +static void > +tasks_stop(struct seq_file *f, void *v) > + __releases(&sn->rpc_client_lock) > + __releases(&clnt->cl_lock) > +{ > + struct rpc_task *task = v; > + struct rpc_clnt *clnt; > + struct sunrpc_net *sn = net_generic(current->nsproxy->net_ns, > + sunrpc_net_id); > + > + if (v) { > + clnt = task->tk_client; > + spin_unlock(&clnt->cl_lock); > + } Full disclosure here... sparse complains about this patch: net/sunrpc/debugfs.c:36:13: warning: context imbalance in 'tasks_start' - different lock contexts for basic block net/sunrpc/debugfs.c:59:13: warning: context imbalance in 'tasks_next' - different lock contexts for basic block net/sunrpc/debugfs.c:110:20: warning: context imbalance in 'tasks_stop' - different lock contexts for basic block The __acquires/__releases annotations don't help in this case since tasks_start and tasks_next can conditionally return with the cl_lock held or not. If tasks_start/_next return a valid object, then the lock will be held. If it returns NULL then it won't be. There should be no actual lock imbalances however. > + spin_unlock(&sn->rpc_client_lock); > +} > + > +static const struct seq_operations tasks_seq_operations = { > + .start = tasks_start, > + .next = tasks_next, > + .stop = tasks_stop, > + .show = tasks_show, > +}; > + > +static int tasks_open(struct inode *inode, struct file *filp) > +{ > + return seq_open_private(filp, &tasks_seq_operations, > + sizeof(loff_t)); > +} > + > +static const struct file_operations tasks_fops = { > + .owner = THIS_MODULE, > + .open = tasks_open, > + .read = seq_read, > + .llseek = seq_lseek, > + .release = seq_release_private, > +}; > + > +void __exit > +sunrpc_debugfs_exit(void) > +{ > + debugfs_remove_recursive(topdir); > +} > + > +int __init > +sunrpc_debugfs_init(void) > +{ > + topdir = debugfs_create_dir("sunrpc", NULL); > + if (!topdir) > + return -ENOMEM; > + > + if (!debugfs_create_file("client_tasks", S_IFREG | S_IRUSR, topdir, > + NULL, &tasks_fops)) { > + debugfs_remove_recursive(topdir); > + return -ENOMEM; > + } > + return 0; > +} > diff --git a/net/sunrpc/sunrpc_syms.c b/net/sunrpc/sunrpc_syms.c > index cd30120de9e4..32583adf3477 100644 > --- a/net/sunrpc/sunrpc_syms.c > +++ b/net/sunrpc/sunrpc_syms.c > @@ -97,6 +97,11 @@ init_sunrpc(void) > err = register_rpc_pipefs(); > if (err) > goto out4; > + > + err = sunrpc_debugfs_init(); > + if (err) > + goto out5; > + > #ifdef RPC_DEBUG > rpc_register_sysctl(); > #endif > @@ -104,6 +109,8 @@ init_sunrpc(void) > init_socket_xprt(); /* clnt sock transport */ > return 0; > > +out5: > + unregister_rpc_pipefs(); > out4: > unregister_pernet_subsys(&sunrpc_net_ops); > out3: > @@ -120,6 +127,7 @@ cleanup_sunrpc(void) > rpcauth_remove_module(); > cleanup_socket_xprt(); > svc_cleanup_xprt_sock(); > + sunrpc_debugfs_exit(); > unregister_rpc_pipefs(); > rpc_destroy_mempool(); > unregister_pernet_subsys(&sunrpc_net_ops);
On Tue, Oct 28, 2014 at 2:30 PM, Jeff Layton <jeff.layton@primarydata.com> wrote: > On Tue, 28 Oct 2014 14:24:55 -0400 > Jeff Layton <jlayton@primarydata.com> wrote: > >> It's possible to get a dump of the RPC task queue by writing a value to >> /proc/sys/sunrpc/rpc_debug. If you write any value to that file, you get >> a dump of the RPC client task list into the log buffer. This is a rather >> inconvenient interface however, and makes it hard to get immediate info >> about the task queue. >> >> Add a new file under debugfs that shows similar info. There are some >> small differences -- we avoid printing raw kernel addresses in favor of >> symbolic names and the XID is also displayed. >> >> Signed-off-by: Jeff Layton <jlayton@primarydata.com> >> --- >> include/linux/sunrpc/clnt.h | 1 + >> include/linux/sunrpc/debug.h | 13 ++++ >> net/sunrpc/Kconfig | 1 + >> net/sunrpc/Makefile | 1 + >> net/sunrpc/clnt.c | 3 +- >> net/sunrpc/debugfs.c | 153 +++++++++++++++++++++++++++++++++++++++++++ >> net/sunrpc/sunrpc_syms.c | 8 +++ >> 7 files changed, 179 insertions(+), 1 deletion(-) >> create mode 100644 net/sunrpc/debugfs.c >> >> diff --git a/include/linux/sunrpc/clnt.h b/include/linux/sunrpc/clnt.h >> index 70736b98c721..265d40bd3109 100644 >> --- a/include/linux/sunrpc/clnt.h >> +++ b/include/linux/sunrpc/clnt.h >> @@ -176,5 +176,6 @@ size_t rpc_peeraddr(struct rpc_clnt *, struct sockaddr *, size_t); >> const char *rpc_peeraddr2str(struct rpc_clnt *, enum rpc_display_format_t); >> int rpc_localaddr(struct rpc_clnt *, struct sockaddr *, size_t); >> >> +const char *rpc_proc_name(const struct rpc_task *task); >> #endif /* __KERNEL__ */ >> #endif /* _LINUX_SUNRPC_CLNT_H */ >> diff --git a/include/linux/sunrpc/debug.h b/include/linux/sunrpc/debug.h >> index 9385bd74c860..7c1ef9530087 100644 >> --- a/include/linux/sunrpc/debug.h >> +++ b/include/linux/sunrpc/debug.h >> @@ -68,6 +68,19 @@ extern unsigned int nlm_debug; >> #ifdef RPC_DEBUG >> void rpc_register_sysctl(void); >> void rpc_unregister_sysctl(void); >> +int sunrpc_debugfs_init(void); >> +void sunrpc_debugfs_exit(void); >> +#else >> +static inline int >> +sunrpc_debugfs_init(void) >> +{ >> + return 0; >> +} >> +static inline void >> +sunrpc_debugfs_exit(void) >> +{ >> + return; >> +} >> #endif >> >> #endif /* _LINUX_SUNRPC_DEBUG_H_ */ >> diff --git a/net/sunrpc/Kconfig b/net/sunrpc/Kconfig >> index 0754d0f466d2..fb78117b896c 100644 >> --- a/net/sunrpc/Kconfig >> +++ b/net/sunrpc/Kconfig >> @@ -35,6 +35,7 @@ config RPCSEC_GSS_KRB5 >> config SUNRPC_DEBUG >> bool "RPC: Enable dprintk debugging" >> depends on SUNRPC && SYSCTL >> + select DEBUG_FS >> help >> This option enables a sysctl-based debugging interface >> that is be used by the 'rpcdebug' utility to turn on or off >> diff --git a/net/sunrpc/Makefile b/net/sunrpc/Makefile >> index e5a7a1cac8f3..15e6f6c23c5d 100644 >> --- a/net/sunrpc/Makefile >> +++ b/net/sunrpc/Makefile >> @@ -14,6 +14,7 @@ sunrpc-y := clnt.o xprt.o socklib.o xprtsock.o sched.o \ >> addr.o rpcb_clnt.o timer.o xdr.o \ >> sunrpc_syms.o cache.o rpc_pipe.o \ >> svc_xprt.o >> +sunrpc-$(CONFIG_SUNRPC_DEBUG) += debugfs.o >> sunrpc-$(CONFIG_SUNRPC_BACKCHANNEL) += backchannel_rqst.o bc_svc.o >> sunrpc-$(CONFIG_PROC_FS) += stats.o >> sunrpc-$(CONFIG_SYSCTL) += sysctl.o >> diff --git a/net/sunrpc/clnt.c b/net/sunrpc/clnt.c >> index 9acd6ce88db7..5b2e2d3d37c1 100644 >> --- a/net/sunrpc/clnt.c >> +++ b/net/sunrpc/clnt.c >> @@ -1397,7 +1397,8 @@ rpc_restart_call(struct rpc_task *task) >> EXPORT_SYMBOL_GPL(rpc_restart_call); >> >> #ifdef RPC_DEBUG >> -static const char *rpc_proc_name(const struct rpc_task *task) >> +const char >> +*rpc_proc_name(const struct rpc_task *task) >> { >> const struct rpc_procinfo *proc = task->tk_msg.rpc_proc; >> >> diff --git a/net/sunrpc/debugfs.c b/net/sunrpc/debugfs.c >> new file mode 100644 >> index 000000000000..48769338172e >> --- /dev/null >> +++ b/net/sunrpc/debugfs.c >> @@ -0,0 +1,153 @@ >> +/** >> + * debugfs interface for sunrpc >> + * >> + * (c) 2014 Jeff Layton <jlayton@primarydata.com> >> + */ >> + >> +#include <linux/debugfs.h> >> +#include <linux/sunrpc/sched.h> >> +#include <linux/sunrpc/clnt.h> >> +#include "netns.h" >> + >> +static struct dentry *topdir; >> + >> +static int >> +tasks_show(struct seq_file *f, void *v) >> +{ >> + u32 xid = 0; >> + struct rpc_task *task = v; >> + struct rpc_clnt *clnt = task->tk_client; >> + const char *rpc_waitq = "none"; >> + >> + if (RPC_IS_QUEUED(task)) >> + rpc_waitq = rpc_qname(task->tk_waitqueue); >> + >> + if (task->tk_rqstp) >> + xid = be32_to_cpu(task->tk_rqstp->rq_xid); >> + >> + seq_printf(f, "%5u %04x %6d 0x%x 0x%x %8ld %ps %sv%u %s a:%ps q:%s\n", >> + task->tk_pid, task->tk_flags, task->tk_status, >> + clnt->cl_clid, xid, task->tk_timeout, task->tk_ops, >> + clnt->cl_program->name, clnt->cl_vers, rpc_proc_name(task), >> + task->tk_action, rpc_waitq); >> + return 0; >> +} >> + >> +static void * >> +tasks_start(struct seq_file *f, loff_t *ppos) >> + __acquires(&sn->rpc_client_lock) >> + __acquires(&clnt->cl_lock) >> +{ >> + loff_t *p = f->private, pos = *ppos; >> + struct rpc_clnt *clnt; >> + struct rpc_task *task; >> + struct sunrpc_net *sn = net_generic(current->nsproxy->net_ns, >> + sunrpc_net_id); >> + >> + *p = pos + 1; >> + spin_lock(&sn->rpc_client_lock); >> + list_for_each_entry(clnt, &sn->all_clients, cl_clients) { >> + spin_lock(&clnt->cl_lock); >> + list_for_each_entry(task, &clnt->cl_tasks, tk_task) >> + if (pos-- == 0) >> + return task; >> + spin_unlock(&clnt->cl_lock); >> + } >> + return NULL; >> +} Instead of having a single pseudofile that iterates through all rpc_clients and all rpc_tasks, should we perhaps have a hierarchy of subdirectories? I'm thinking we might want a directory per rpc_client, each containing a pseudo-file that iterates through all rpc_tasks for that rpc_client only. The reason for doing so would be so that we can add pseudofiles to display more per-rpc_client debugging info, such as cl_nodename, cl_program->name, cl_prog, cl_vers. Maybe also with a symlink to another subdirectory that displays per-xprt debugging information? i.e. a hierarchy of the form debugfs/ rpc_clients/ 1/ cl_tasks cl_xprt->../../rpc_xprt/1 ..... 2/ cl_tasks cl_xprt->../../rpc_xprt/1 ..... rpc_xprt/ 1/ servername .....
On Mon, 24 Nov 2014 14:20:25 -0500 Trond Myklebust <trond.myklebust@primarydata.com> wrote: > On Tue, Oct 28, 2014 at 2:30 PM, Jeff Layton > <jeff.layton@primarydata.com> wrote: > > On Tue, 28 Oct 2014 14:24:55 -0400 > > Jeff Layton <jlayton@primarydata.com> wrote: > > > >> It's possible to get a dump of the RPC task queue by writing a value to > >> /proc/sys/sunrpc/rpc_debug. If you write any value to that file, you get > >> a dump of the RPC client task list into the log buffer. This is a rather > >> inconvenient interface however, and makes it hard to get immediate info > >> about the task queue. > >> > >> Add a new file under debugfs that shows similar info. There are some > >> small differences -- we avoid printing raw kernel addresses in favor of > >> symbolic names and the XID is also displayed. > >> > >> Signed-off-by: Jeff Layton <jlayton@primarydata.com> > >> --- > >> include/linux/sunrpc/clnt.h | 1 + > >> include/linux/sunrpc/debug.h | 13 ++++ > >> net/sunrpc/Kconfig | 1 + > >> net/sunrpc/Makefile | 1 + > >> net/sunrpc/clnt.c | 3 +- > >> net/sunrpc/debugfs.c | 153 +++++++++++++++++++++++++++++++++++++++++++ > >> net/sunrpc/sunrpc_syms.c | 8 +++ > >> 7 files changed, 179 insertions(+), 1 deletion(-) > >> create mode 100644 net/sunrpc/debugfs.c > >> > >> diff --git a/include/linux/sunrpc/clnt.h b/include/linux/sunrpc/clnt.h > >> index 70736b98c721..265d40bd3109 100644 > >> --- a/include/linux/sunrpc/clnt.h > >> +++ b/include/linux/sunrpc/clnt.h > >> @@ -176,5 +176,6 @@ size_t rpc_peeraddr(struct rpc_clnt *, struct sockaddr *, size_t); > >> const char *rpc_peeraddr2str(struct rpc_clnt *, enum rpc_display_format_t); > >> int rpc_localaddr(struct rpc_clnt *, struct sockaddr *, size_t); > >> > >> +const char *rpc_proc_name(const struct rpc_task *task); > >> #endif /* __KERNEL__ */ > >> #endif /* _LINUX_SUNRPC_CLNT_H */ > >> diff --git a/include/linux/sunrpc/debug.h b/include/linux/sunrpc/debug.h > >> index 9385bd74c860..7c1ef9530087 100644 > >> --- a/include/linux/sunrpc/debug.h > >> +++ b/include/linux/sunrpc/debug.h > >> @@ -68,6 +68,19 @@ extern unsigned int nlm_debug; > >> #ifdef RPC_DEBUG > >> void rpc_register_sysctl(void); > >> void rpc_unregister_sysctl(void); > >> +int sunrpc_debugfs_init(void); > >> +void sunrpc_debugfs_exit(void); > >> +#else > >> +static inline int > >> +sunrpc_debugfs_init(void) > >> +{ > >> + return 0; > >> +} > >> +static inline void > >> +sunrpc_debugfs_exit(void) > >> +{ > >> + return; > >> +} > >> #endif > >> > >> #endif /* _LINUX_SUNRPC_DEBUG_H_ */ > >> diff --git a/net/sunrpc/Kconfig b/net/sunrpc/Kconfig > >> index 0754d0f466d2..fb78117b896c 100644 > >> --- a/net/sunrpc/Kconfig > >> +++ b/net/sunrpc/Kconfig > >> @@ -35,6 +35,7 @@ config RPCSEC_GSS_KRB5 > >> config SUNRPC_DEBUG > >> bool "RPC: Enable dprintk debugging" > >> depends on SUNRPC && SYSCTL > >> + select DEBUG_FS > >> help > >> This option enables a sysctl-based debugging interface > >> that is be used by the 'rpcdebug' utility to turn on or off > >> diff --git a/net/sunrpc/Makefile b/net/sunrpc/Makefile > >> index e5a7a1cac8f3..15e6f6c23c5d 100644 > >> --- a/net/sunrpc/Makefile > >> +++ b/net/sunrpc/Makefile > >> @@ -14,6 +14,7 @@ sunrpc-y := clnt.o xprt.o socklib.o xprtsock.o sched.o \ > >> addr.o rpcb_clnt.o timer.o xdr.o \ > >> sunrpc_syms.o cache.o rpc_pipe.o \ > >> svc_xprt.o > >> +sunrpc-$(CONFIG_SUNRPC_DEBUG) += debugfs.o > >> sunrpc-$(CONFIG_SUNRPC_BACKCHANNEL) += backchannel_rqst.o bc_svc.o > >> sunrpc-$(CONFIG_PROC_FS) += stats.o > >> sunrpc-$(CONFIG_SYSCTL) += sysctl.o > >> diff --git a/net/sunrpc/clnt.c b/net/sunrpc/clnt.c > >> index 9acd6ce88db7..5b2e2d3d37c1 100644 > >> --- a/net/sunrpc/clnt.c > >> +++ b/net/sunrpc/clnt.c > >> @@ -1397,7 +1397,8 @@ rpc_restart_call(struct rpc_task *task) > >> EXPORT_SYMBOL_GPL(rpc_restart_call); > >> > >> #ifdef RPC_DEBUG > >> -static const char *rpc_proc_name(const struct rpc_task *task) > >> +const char > >> +*rpc_proc_name(const struct rpc_task *task) > >> { > >> const struct rpc_procinfo *proc = task->tk_msg.rpc_proc; > >> > >> diff --git a/net/sunrpc/debugfs.c b/net/sunrpc/debugfs.c > >> new file mode 100644 > >> index 000000000000..48769338172e > >> --- /dev/null > >> +++ b/net/sunrpc/debugfs.c > >> @@ -0,0 +1,153 @@ > >> +/** > >> + * debugfs interface for sunrpc > >> + * > >> + * (c) 2014 Jeff Layton <jlayton@primarydata.com> > >> + */ > >> + > >> +#include <linux/debugfs.h> > >> +#include <linux/sunrpc/sched.h> > >> +#include <linux/sunrpc/clnt.h> > >> +#include "netns.h" > >> + > >> +static struct dentry *topdir; > >> + > >> +static int > >> +tasks_show(struct seq_file *f, void *v) > >> +{ > >> + u32 xid = 0; > >> + struct rpc_task *task = v; > >> + struct rpc_clnt *clnt = task->tk_client; > >> + const char *rpc_waitq = "none"; > >> + > >> + if (RPC_IS_QUEUED(task)) > >> + rpc_waitq = rpc_qname(task->tk_waitqueue); > >> + > >> + if (task->tk_rqstp) > >> + xid = be32_to_cpu(task->tk_rqstp->rq_xid); > >> + > >> + seq_printf(f, "%5u %04x %6d 0x%x 0x%x %8ld %ps %sv%u %s a:%ps q:%s\n", > >> + task->tk_pid, task->tk_flags, task->tk_status, > >> + clnt->cl_clid, xid, task->tk_timeout, task->tk_ops, > >> + clnt->cl_program->name, clnt->cl_vers, rpc_proc_name(task), > >> + task->tk_action, rpc_waitq); > >> + return 0; > >> +} > >> + > >> +static void * > >> +tasks_start(struct seq_file *f, loff_t *ppos) > >> + __acquires(&sn->rpc_client_lock) > >> + __acquires(&clnt->cl_lock) > >> +{ > >> + loff_t *p = f->private, pos = *ppos; > >> + struct rpc_clnt *clnt; > >> + struct rpc_task *task; > >> + struct sunrpc_net *sn = net_generic(current->nsproxy->net_ns, > >> + sunrpc_net_id); > >> + > >> + *p = pos + 1; > >> + spin_lock(&sn->rpc_client_lock); > >> + list_for_each_entry(clnt, &sn->all_clients, cl_clients) { > >> + spin_lock(&clnt->cl_lock); > >> + list_for_each_entry(task, &clnt->cl_tasks, tk_task) > >> + if (pos-- == 0) > >> + return task; > >> + spin_unlock(&clnt->cl_lock); > >> + } > >> + return NULL; > >> +} > > Instead of having a single pseudofile that iterates through all > rpc_clients and all rpc_tasks, should we perhaps have a hierarchy of > subdirectories? > I'm thinking we might want a directory per rpc_client, each containing > a pseudo-file that iterates through all rpc_tasks for that rpc_client > only. The reason for doing so would be so that we can add pseudofiles > to display more per-rpc_client debugging info, such as cl_nodename, > cl_program->name, cl_prog, cl_vers. Maybe also with a symlink to > another subdirectory that displays per-xprt debugging information? > > i.e. a hierarchy of the form > > debugfs/ I'd suggest that we have a top level "sunrpc/" directory here to keep things neat and tidy. > rpc_clients/ > 1/ > cl_tasks > cl_xprt->../../rpc_xprt/1 > ..... > 2/ > cl_tasks > cl_xprt->../../rpc_xprt/1 > ..... > rpc_xprt/ > 1/ > servername > ..... > Hmm...that looks doable, but is a much larger piece of work than this patch. Let me see what I can come up with and I'll get back to you...
This is a respin of this patch that I sent a few weeks ago:
sunrpc: add debugfs file for displaying client rpc_task queue
...to address Trond's concerns about how to lay out this directory.
I've also included a respin of the patches to eliminate the unneeded
#define redirection in the sunrpc debugging code. I had originally based
those patches on top of the debugfs patch, so if we drop that patch
there are merge conflicts with these.
In this set, I've reversed the order so if there are any more respins
needed on the debugfs work, I shouldn't need to resend the whole set.
Jeff Layton (4):
lockd: eliminate LOCKD_DEBUG
sunrpc: eliminate RPC_DEBUG
sunrpc: eliminate RPC_TRACEPOINTS
sunrpc: add debugfs file for displaying client rpc_task queue
fs/lockd/svclock.c | 2 +-
include/linux/lockd/debug.h | 6 +-
include/linux/sunrpc/auth.h | 2 +-
include/linux/sunrpc/clnt.h | 4 +
include/linux/sunrpc/debug.h | 49 ++++++---
include/linux/sunrpc/sched.h | 8 +-
include/uapi/linux/nfsd/debug.h | 2 +-
net/sunrpc/Kconfig | 1 +
net/sunrpc/Makefile | 1 +
net/sunrpc/auth.c | 4 +-
net/sunrpc/auth_generic.c | 2 +-
net/sunrpc/auth_gss/auth_gss.c | 2 +-
net/sunrpc/auth_gss/gss_generic_token.c | 2 +-
net/sunrpc/auth_gss/gss_krb5_crypto.c | 2 +-
net/sunrpc/auth_gss/gss_krb5_keys.c | 2 +-
net/sunrpc/auth_gss/gss_krb5_mech.c | 2 +-
net/sunrpc/auth_gss/gss_krb5_seal.c | 2 +-
net/sunrpc/auth_gss/gss_krb5_seqnum.c | 2 +-
net/sunrpc/auth_gss/gss_krb5_unseal.c | 2 +-
net/sunrpc/auth_gss/gss_krb5_wrap.c | 2 +-
net/sunrpc/auth_gss/gss_mech_switch.c | 2 +-
net/sunrpc/auth_gss/gss_rpc_xdr.h | 2 +-
net/sunrpc/auth_gss/svcauth_gss.c | 2 +-
net/sunrpc/auth_null.c | 4 +-
net/sunrpc/auth_unix.c | 2 +-
net/sunrpc/backchannel_rqst.c | 2 +-
net/sunrpc/clnt.c | 15 ++-
net/sunrpc/debugfs.c | 173 ++++++++++++++++++++++++++++++++
net/sunrpc/rpcb_clnt.c | 2 +-
net/sunrpc/sched.c | 4 +-
net/sunrpc/sunrpc_syms.c | 12 ++-
net/sunrpc/svc.c | 2 +-
net/sunrpc/sysctl.c | 2 +-
net/sunrpc/xprt.c | 2 +-
net/sunrpc/xprtrdma/rpc_rdma.c | 4 +-
net/sunrpc/xprtrdma/transport.c | 8 +-
net/sunrpc/xprtrdma/verbs.c | 8 +-
net/sunrpc/xprtsock.c | 8 +-
38 files changed, 281 insertions(+), 72 deletions(-)
create mode 100644 net/sunrpc/debugfs.c
v3: - add a patch that creates and populates a rpc_xprt directory - take reference on rpc_clnt when "tasks" file is opened - minor cleanups v2: - add a whole new directory hierarchy instead of a single client_tasks file - respin patches to remove LOCKD_DEBUG/RPC_DEBUG/RPC_TRACEPOINTS defines Ok, here is the latest (and hopefully last!) respin of the patches to add some debugfs files to dump info about rpc_clnt and rpc_xprt objects. This set ensures that references are properly acquired so that the objects don't disappear while we're working on them. I've also included the patches to eliminate the RPC_DEBUG defines too as they should be merged first to prevent conflicts. Trond, can you consider these for v3.19? Thanks! Jeff Layton (5): lockd: eliminate LOCKD_DEBUG sunrpc: eliminate RPC_DEBUG sunrpc: eliminate RPC_TRACEPOINTS sunrpc: add debugfs file for displaying client rpc_task queue sunrpc: add a debugfs rpc_xprt directory with an info file in it fs/lockd/svclock.c | 2 +- include/linux/lockd/debug.h | 6 +- include/linux/sunrpc/auth.h | 2 +- include/linux/sunrpc/clnt.h | 4 + include/linux/sunrpc/debug.h | 64 +++++-- include/linux/sunrpc/sched.h | 8 +- include/linux/sunrpc/xprt.h | 3 + include/uapi/linux/nfsd/debug.h | 2 +- net/sunrpc/Kconfig | 1 + net/sunrpc/Makefile | 1 + net/sunrpc/auth.c | 4 +- net/sunrpc/auth_generic.c | 2 +- net/sunrpc/auth_gss/auth_gss.c | 2 +- net/sunrpc/auth_gss/gss_generic_token.c | 2 +- net/sunrpc/auth_gss/gss_krb5_crypto.c | 2 +- net/sunrpc/auth_gss/gss_krb5_keys.c | 2 +- net/sunrpc/auth_gss/gss_krb5_mech.c | 2 +- net/sunrpc/auth_gss/gss_krb5_seal.c | 2 +- net/sunrpc/auth_gss/gss_krb5_seqnum.c | 2 +- net/sunrpc/auth_gss/gss_krb5_unseal.c | 2 +- net/sunrpc/auth_gss/gss_krb5_wrap.c | 2 +- net/sunrpc/auth_gss/gss_mech_switch.c | 2 +- net/sunrpc/auth_gss/gss_rpc_xdr.h | 2 +- net/sunrpc/auth_gss/svcauth_gss.c | 2 +- net/sunrpc/auth_null.c | 4 +- net/sunrpc/auth_unix.c | 2 +- net/sunrpc/backchannel_rqst.c | 2 +- net/sunrpc/clnt.c | 16 +- net/sunrpc/debugfs.c | 292 ++++++++++++++++++++++++++++++++ net/sunrpc/rpcb_clnt.c | 2 +- net/sunrpc/sched.c | 4 +- net/sunrpc/sunrpc_syms.c | 12 +- net/sunrpc/svc.c | 2 +- net/sunrpc/sysctl.c | 2 +- net/sunrpc/xprt.c | 10 +- net/sunrpc/xprtrdma/rpc_rdma.c | 4 +- net/sunrpc/xprtrdma/transport.c | 8 +- net/sunrpc/xprtrdma/verbs.c | 8 +- net/sunrpc/xprtsock.c | 8 +- 39 files changed, 427 insertions(+), 72 deletions(-) create mode 100644 net/sunrpc/debugfs.c
diff --git a/include/linux/sunrpc/clnt.h b/include/linux/sunrpc/clnt.h index 70736b98c721..265d40bd3109 100644 --- a/include/linux/sunrpc/clnt.h +++ b/include/linux/sunrpc/clnt.h @@ -176,5 +176,6 @@ size_t rpc_peeraddr(struct rpc_clnt *, struct sockaddr *, size_t); const char *rpc_peeraddr2str(struct rpc_clnt *, enum rpc_display_format_t); int rpc_localaddr(struct rpc_clnt *, struct sockaddr *, size_t); +const char *rpc_proc_name(const struct rpc_task *task); #endif /* __KERNEL__ */ #endif /* _LINUX_SUNRPC_CLNT_H */ diff --git a/include/linux/sunrpc/debug.h b/include/linux/sunrpc/debug.h index 9385bd74c860..7c1ef9530087 100644 --- a/include/linux/sunrpc/debug.h +++ b/include/linux/sunrpc/debug.h @@ -68,6 +68,19 @@ extern unsigned int nlm_debug; #ifdef RPC_DEBUG void rpc_register_sysctl(void); void rpc_unregister_sysctl(void); +int sunrpc_debugfs_init(void); +void sunrpc_debugfs_exit(void); +#else +static inline int +sunrpc_debugfs_init(void) +{ + return 0; +} +static inline void +sunrpc_debugfs_exit(void) +{ + return; +} #endif #endif /* _LINUX_SUNRPC_DEBUG_H_ */ diff --git a/net/sunrpc/Kconfig b/net/sunrpc/Kconfig index 0754d0f466d2..fb78117b896c 100644 --- a/net/sunrpc/Kconfig +++ b/net/sunrpc/Kconfig @@ -35,6 +35,7 @@ config RPCSEC_GSS_KRB5 config SUNRPC_DEBUG bool "RPC: Enable dprintk debugging" depends on SUNRPC && SYSCTL + select DEBUG_FS help This option enables a sysctl-based debugging interface that is be used by the 'rpcdebug' utility to turn on or off diff --git a/net/sunrpc/Makefile b/net/sunrpc/Makefile index e5a7a1cac8f3..15e6f6c23c5d 100644 --- a/net/sunrpc/Makefile +++ b/net/sunrpc/Makefile @@ -14,6 +14,7 @@ sunrpc-y := clnt.o xprt.o socklib.o xprtsock.o sched.o \ addr.o rpcb_clnt.o timer.o xdr.o \ sunrpc_syms.o cache.o rpc_pipe.o \ svc_xprt.o +sunrpc-$(CONFIG_SUNRPC_DEBUG) += debugfs.o sunrpc-$(CONFIG_SUNRPC_BACKCHANNEL) += backchannel_rqst.o bc_svc.o sunrpc-$(CONFIG_PROC_FS) += stats.o sunrpc-$(CONFIG_SYSCTL) += sysctl.o diff --git a/net/sunrpc/clnt.c b/net/sunrpc/clnt.c index 9acd6ce88db7..5b2e2d3d37c1 100644 --- a/net/sunrpc/clnt.c +++ b/net/sunrpc/clnt.c @@ -1397,7 +1397,8 @@ rpc_restart_call(struct rpc_task *task) EXPORT_SYMBOL_GPL(rpc_restart_call); #ifdef RPC_DEBUG -static const char *rpc_proc_name(const struct rpc_task *task) +const char +*rpc_proc_name(const struct rpc_task *task) { const struct rpc_procinfo *proc = task->tk_msg.rpc_proc; diff --git a/net/sunrpc/debugfs.c b/net/sunrpc/debugfs.c new file mode 100644 index 000000000000..48769338172e --- /dev/null +++ b/net/sunrpc/debugfs.c @@ -0,0 +1,153 @@ +/** + * debugfs interface for sunrpc + * + * (c) 2014 Jeff Layton <jlayton@primarydata.com> + */ + +#include <linux/debugfs.h> +#include <linux/sunrpc/sched.h> +#include <linux/sunrpc/clnt.h> +#include "netns.h" + +static struct dentry *topdir; + +static int +tasks_show(struct seq_file *f, void *v) +{ + u32 xid = 0; + struct rpc_task *task = v; + struct rpc_clnt *clnt = task->tk_client; + const char *rpc_waitq = "none"; + + if (RPC_IS_QUEUED(task)) + rpc_waitq = rpc_qname(task->tk_waitqueue); + + if (task->tk_rqstp) + xid = be32_to_cpu(task->tk_rqstp->rq_xid); + + seq_printf(f, "%5u %04x %6d 0x%x 0x%x %8ld %ps %sv%u %s a:%ps q:%s\n", + task->tk_pid, task->tk_flags, task->tk_status, + clnt->cl_clid, xid, task->tk_timeout, task->tk_ops, + clnt->cl_program->name, clnt->cl_vers, rpc_proc_name(task), + task->tk_action, rpc_waitq); + return 0; +} + +static void * +tasks_start(struct seq_file *f, loff_t *ppos) + __acquires(&sn->rpc_client_lock) + __acquires(&clnt->cl_lock) +{ + loff_t *p = f->private, pos = *ppos; + struct rpc_clnt *clnt; + struct rpc_task *task; + struct sunrpc_net *sn = net_generic(current->nsproxy->net_ns, + sunrpc_net_id); + + *p = pos + 1; + spin_lock(&sn->rpc_client_lock); + list_for_each_entry(clnt, &sn->all_clients, cl_clients) { + spin_lock(&clnt->cl_lock); + list_for_each_entry(task, &clnt->cl_tasks, tk_task) + if (pos-- == 0) + return task; + spin_unlock(&clnt->cl_lock); + } + return NULL; +} + +static void * +tasks_next(struct seq_file *f, void *v, loff_t *pos) + __releases(&clnt->cl_lock) + __acquires(&clnt->cl_lock) +{ + loff_t *p = f->private; + struct rpc_task *task = v; + struct rpc_clnt *clnt = task->tk_client; + struct list_head *next = task->tk_task.next; + struct sunrpc_net *sn = net_generic(current->nsproxy->net_ns, + sunrpc_net_id); + + ++*p; + ++*pos; + + /* If there's another task on list, return it */ + if (next != &clnt->cl_tasks) + return list_entry(next, struct rpc_task, tk_task); + + /* We're finished with this client, move on */ + spin_unlock(&clnt->cl_lock); +next_client: + next = clnt->cl_clients.next; + if (next == &sn->all_clients) + return NULL; + + clnt = list_entry(next, struct rpc_clnt, cl_clients); + if (list_empty(&clnt->cl_tasks)) + goto next_client; + spin_lock(&clnt->cl_lock); + if (list_empty(&clnt->cl_tasks)) { + spin_unlock(&clnt->cl_lock); + goto next_client; + } + return list_first_entry(&clnt->cl_tasks, struct rpc_task, tk_task); +} + +static void +tasks_stop(struct seq_file *f, void *v) + __releases(&sn->rpc_client_lock) + __releases(&clnt->cl_lock) +{ + struct rpc_task *task = v; + struct rpc_clnt *clnt; + struct sunrpc_net *sn = net_generic(current->nsproxy->net_ns, + sunrpc_net_id); + + if (v) { + clnt = task->tk_client; + spin_unlock(&clnt->cl_lock); + } + spin_unlock(&sn->rpc_client_lock); +} + +static const struct seq_operations tasks_seq_operations = { + .start = tasks_start, + .next = tasks_next, + .stop = tasks_stop, + .show = tasks_show, +}; + +static int tasks_open(struct inode *inode, struct file *filp) +{ + return seq_open_private(filp, &tasks_seq_operations, + sizeof(loff_t)); +} + +static const struct file_operations tasks_fops = { + .owner = THIS_MODULE, + .open = tasks_open, + .read = seq_read, + .llseek = seq_lseek, + .release = seq_release_private, +}; + +void __exit +sunrpc_debugfs_exit(void) +{ + debugfs_remove_recursive(topdir); +} + +int __init +sunrpc_debugfs_init(void) +{ + topdir = debugfs_create_dir("sunrpc", NULL); + if (!topdir) + return -ENOMEM; + + if (!debugfs_create_file("client_tasks", S_IFREG | S_IRUSR, topdir, + NULL, &tasks_fops)) { + debugfs_remove_recursive(topdir); + return -ENOMEM; + } + return 0; +} diff --git a/net/sunrpc/sunrpc_syms.c b/net/sunrpc/sunrpc_syms.c index cd30120de9e4..32583adf3477 100644 --- a/net/sunrpc/sunrpc_syms.c +++ b/net/sunrpc/sunrpc_syms.c @@ -97,6 +97,11 @@ init_sunrpc(void) err = register_rpc_pipefs(); if (err) goto out4; + + err = sunrpc_debugfs_init(); + if (err) + goto out5; + #ifdef RPC_DEBUG rpc_register_sysctl(); #endif @@ -104,6 +109,8 @@ init_sunrpc(void) init_socket_xprt(); /* clnt sock transport */ return 0; +out5: + unregister_rpc_pipefs(); out4: unregister_pernet_subsys(&sunrpc_net_ops); out3: @@ -120,6 +127,7 @@ cleanup_sunrpc(void) rpcauth_remove_module(); cleanup_socket_xprt(); svc_cleanup_xprt_sock(); + sunrpc_debugfs_exit(); unregister_rpc_pipefs(); rpc_destroy_mempool(); unregister_pernet_subsys(&sunrpc_net_ops);
It's possible to get a dump of the RPC task queue by writing a value to /proc/sys/sunrpc/rpc_debug. If you write any value to that file, you get a dump of the RPC client task list into the log buffer. This is a rather inconvenient interface however, and makes it hard to get immediate info about the task queue. Add a new file under debugfs that shows similar info. There are some small differences -- we avoid printing raw kernel addresses in favor of symbolic names and the XID is also displayed. Signed-off-by: Jeff Layton <jlayton@primarydata.com> --- include/linux/sunrpc/clnt.h | 1 + include/linux/sunrpc/debug.h | 13 ++++ net/sunrpc/Kconfig | 1 + net/sunrpc/Makefile | 1 + net/sunrpc/clnt.c | 3 +- net/sunrpc/debugfs.c | 153 +++++++++++++++++++++++++++++++++++++++++++ net/sunrpc/sunrpc_syms.c | 8 +++ 7 files changed, 179 insertions(+), 1 deletion(-) create mode 100644 net/sunrpc/debugfs.c