| Message ID | 20230126000936.3017683-1-namhyung@kernel.org (mailing list archive) |
|---|---|
| State | Not Applicable |
| Delegated to: | BPF |
| Headers | show |
| Series | perf lock contention: Add -S/--callstack-filter option | expand |
Em Wed, Jan 25, 2023 at 04:09:36PM -0800, Namhyung Kim escreveu: > The -S/--callstack-filter is to limit display entries having the given > string in the callstack (not only in the caller in the output). > > The following example shows lock contention results if the callstack > has 'net' substring somewhere. Note that the caller '__dev_queue_xmit' > does not match to it, but it has 'inet6_csk_xmit' in the callstack. Looks useful! Thanks, applied. - Arnaldo > This applies even if you don't use -v option to show the full callstack. > > $ sudo ./perf lock con -abv -S net sleep 1 > ... > contended total wait max wait avg wait type caller > > 5 70.20 us 16.13 us 14.04 us spinlock __dev_queue_xmit+0xb6d > 0xffffffffa5dd1c60 _raw_spin_lock+0x30 > 0xffffffffa5b8f6ed __dev_queue_xmit+0xb6d > 0xffffffffa5cd8267 ip6_finish_output2+0x2c7 > 0xffffffffa5cdac14 ip6_finish_output+0x1d4 > 0xffffffffa5cdb477 ip6_xmit+0x457 > 0xffffffffa5d1fd17 inet6_csk_xmit+0xd7 > 0xffffffffa5c5f4aa __tcp_transmit_skb+0x54a > 0xffffffffa5c6467d tcp_keepalive_timer+0x2fd > > Signed-off-by: Namhyung Kim <namhyung@kernel.org> > --- > tools/perf/Documentation/perf-lock.txt | 6 +++ > tools/perf/builtin-lock.c | 68 +++++++++++++++++++++++++- > tools/perf/util/bpf_lock_contention.c | 2 +- > tools/perf/util/lock-contention.h | 1 + > 4 files changed, 75 insertions(+), 2 deletions(-) > > diff --git a/tools/perf/Documentation/perf-lock.txt b/tools/perf/Documentation/perf-lock.txt > index 0f9f720e599d..11b8901d8d13 100644 > --- a/tools/perf/Documentation/perf-lock.txt > +++ b/tools/perf/Documentation/perf-lock.txt > @@ -187,6 +187,12 @@ CONTENTION OPTIONS > --lock-filter=<value>:: > Show lock contention only for given lock addresses or names (comma separated list). > > +-S:: > +--callstack-filter=<value>:: > + Show lock contention only if the callstack contains the given string. > + Note that it matches the substring so 'rq' would match both 'raw_spin_rq_lock' > + and 'irq_enter_rcu'. > + > > SEE ALSO > -------- > diff --git a/tools/perf/builtin-lock.c b/tools/perf/builtin-lock.c > index 506c2fe42d52..216a9a252bf4 100644 > --- a/tools/perf/builtin-lock.c > +++ b/tools/perf/builtin-lock.c > @@ -63,11 +63,22 @@ static unsigned long bpf_map_entries = 10240; > static int max_stack_depth = CONTENTION_STACK_DEPTH; > static int stack_skip = CONTENTION_STACK_SKIP; > static int print_nr_entries = INT_MAX / 2; > +static LIST_HEAD(callstack_filters); > + > +struct callstack_filter { > + struct list_head list; > + char name[]; > +}; > > static struct lock_filter filters; > > static enum lock_aggr_mode aggr_mode = LOCK_AGGR_ADDR; > > +static bool needs_callstack(void) > +{ > + return verbose > 0 || !list_empty(&callstack_filters); > +} > + > static struct thread_stat *thread_stat_find(u32 tid) > { > struct rb_node *node; > @@ -1060,7 +1071,7 @@ static int report_lock_contention_begin_event(struct evsel *evsel, > if (!ls) > return -ENOMEM; > > - if (aggr_mode == LOCK_AGGR_CALLER && verbose > 0) { > + if (aggr_mode == LOCK_AGGR_CALLER && needs_callstack()) { > ls->callstack = get_callstack(sample, max_stack_depth); > if (ls->callstack == NULL) > return -ENOMEM; > @@ -1595,6 +1606,31 @@ static void print_contention_result(struct lock_contention *con) > if (!st->wait_time_total) > continue; > > + if (aggr_mode == LOCK_AGGR_CALLER && !list_empty(&callstack_filters)) { > + struct map *kmap; > + struct symbol *sym; > + u64 ip; > + > + for (int i = 0; i < max_stack_depth; i++) { > + struct callstack_filter *filter; > + > + if (!st->callstack || !st->callstack[i]) > + break; > + > + ip = st->callstack[i]; > + sym = machine__find_kernel_symbol(con->machine, ip, &kmap); > + if (sym == NULL) > + continue; > + > + list_for_each_entry(filter, &callstack_filters, list) { > + if (strstr(sym->name, filter->name)) > + goto found; > + } > + } > + continue; > + } > + > +found: > list_for_each_entry(key, &lock_keys, list) { > key->print(key, st); > pr_info(" "); > @@ -1743,6 +1779,7 @@ static int __cmd_contention(int argc, const char **argv) > .max_stack = max_stack_depth, > .stack_skip = stack_skip, > .filters = &filters, > + .save_callstack = needs_callstack(), > }; > > session = perf_session__new(use_bpf ? NULL : &data, &eops); > @@ -2123,6 +2160,33 @@ static int parse_lock_addr(const struct option *opt __maybe_unused, const char * > return ret; > } > > +static int parse_call_stack(const struct option *opt __maybe_unused, const char *str, > + int unset __maybe_unused) > +{ > + char *s, *tmp, *tok; > + int ret = 0; > + > + s = strdup(str); > + if (s == NULL) > + return -1; > + > + for (tok = strtok_r(s, ", ", &tmp); tok; tok = strtok_r(NULL, ", ", &tmp)) { > + struct callstack_filter *entry; > + > + entry = malloc(sizeof(*entry) + strlen(tok) + 1); > + if (entry == NULL) { > + pr_err("Memory allocation failure\n"); > + return -1; > + } > + > + strcpy(entry->name, tok); > + list_add_tail(&entry->list, &callstack_filters); > + } > + > + free(s); > + return ret; > +} > + > int cmd_lock(int argc, const char **argv) > { > const struct option lock_options[] = { > @@ -2190,6 +2254,8 @@ int cmd_lock(int argc, const char **argv) > "Filter specific type of locks", parse_lock_type), > OPT_CALLBACK('L', "lock-filter", NULL, "ADDRS/NAMES", > "Filter specific address/symbol of locks", parse_lock_addr), > + OPT_CALLBACK('S', "callstack-filter", NULL, "NAMES", > + "Filter specific function in the callstack", parse_call_stack), > OPT_PARENT(lock_options) > }; > > diff --git a/tools/perf/util/bpf_lock_contention.c b/tools/perf/util/bpf_lock_contention.c > index 0236334fd69b..4902ac331f41 100644 > --- a/tools/perf/util/bpf_lock_contention.c > +++ b/tools/perf/util/bpf_lock_contention.c > @@ -268,7 +268,7 @@ int lock_contention_read(struct lock_contention *con) > break; > } > > - if (verbose > 0) { > + if (con->save_callstack) { > st->callstack = memdup(stack_trace, stack_size); > if (st->callstack == NULL) > break; > diff --git a/tools/perf/util/lock-contention.h b/tools/perf/util/lock-contention.h > index b99e83fccf5c..17e594d57a61 100644 > --- a/tools/perf/util/lock-contention.h > +++ b/tools/perf/util/lock-contention.h > @@ -128,6 +128,7 @@ struct lock_contention { > int max_stack; > int stack_skip; > int aggr_mode; > + bool save_callstack; > }; > > #ifdef HAVE_BPF_SKEL > -- > 2.39.1.456.gfc5497dd1b-goog >
diff --git a/tools/perf/Documentation/perf-lock.txt b/tools/perf/Documentation/perf-lock.txt index 0f9f720e599d..11b8901d8d13 100644 --- a/tools/perf/Documentation/perf-lock.txt +++ b/tools/perf/Documentation/perf-lock.txt @@ -187,6 +187,12 @@ CONTENTION OPTIONS --lock-filter=<value>:: Show lock contention only for given lock addresses or names (comma separated list). +-S:: +--callstack-filter=<value>:: + Show lock contention only if the callstack contains the given string. + Note that it matches the substring so 'rq' would match both 'raw_spin_rq_lock' + and 'irq_enter_rcu'. + SEE ALSO -------- diff --git a/tools/perf/builtin-lock.c b/tools/perf/builtin-lock.c index 506c2fe42d52..216a9a252bf4 100644 --- a/tools/perf/builtin-lock.c +++ b/tools/perf/builtin-lock.c @@ -63,11 +63,22 @@ static unsigned long bpf_map_entries = 10240; static int max_stack_depth = CONTENTION_STACK_DEPTH; static int stack_skip = CONTENTION_STACK_SKIP; static int print_nr_entries = INT_MAX / 2; +static LIST_HEAD(callstack_filters); + +struct callstack_filter { + struct list_head list; + char name[]; +}; static struct lock_filter filters; static enum lock_aggr_mode aggr_mode = LOCK_AGGR_ADDR; +static bool needs_callstack(void) +{ + return verbose > 0 || !list_empty(&callstack_filters); +} + static struct thread_stat *thread_stat_find(u32 tid) { struct rb_node *node; @@ -1060,7 +1071,7 @@ static int report_lock_contention_begin_event(struct evsel *evsel, if (!ls) return -ENOMEM; - if (aggr_mode == LOCK_AGGR_CALLER && verbose > 0) { + if (aggr_mode == LOCK_AGGR_CALLER && needs_callstack()) { ls->callstack = get_callstack(sample, max_stack_depth); if (ls->callstack == NULL) return -ENOMEM; @@ -1595,6 +1606,31 @@ static void print_contention_result(struct lock_contention *con) if (!st->wait_time_total) continue; + if (aggr_mode == LOCK_AGGR_CALLER && !list_empty(&callstack_filters)) { + struct map *kmap; + struct symbol *sym; + u64 ip; + + for (int i = 0; i < max_stack_depth; i++) { + struct callstack_filter *filter; + + if (!st->callstack || !st->callstack[i]) + break; + + ip = st->callstack[i]; + sym = machine__find_kernel_symbol(con->machine, ip, &kmap); + if (sym == NULL) + continue; + + list_for_each_entry(filter, &callstack_filters, list) { + if (strstr(sym->name, filter->name)) + goto found; + } + } + continue; + } + +found: list_for_each_entry(key, &lock_keys, list) { key->print(key, st); pr_info(" "); @@ -1743,6 +1779,7 @@ static int __cmd_contention(int argc, const char **argv) .max_stack = max_stack_depth, .stack_skip = stack_skip, .filters = &filters, + .save_callstack = needs_callstack(), }; session = perf_session__new(use_bpf ? NULL : &data, &eops); @@ -2123,6 +2160,33 @@ static int parse_lock_addr(const struct option *opt __maybe_unused, const char * return ret; } +static int parse_call_stack(const struct option *opt __maybe_unused, const char *str, + int unset __maybe_unused) +{ + char *s, *tmp, *tok; + int ret = 0; + + s = strdup(str); + if (s == NULL) + return -1; + + for (tok = strtok_r(s, ", ", &tmp); tok; tok = strtok_r(NULL, ", ", &tmp)) { + struct callstack_filter *entry; + + entry = malloc(sizeof(*entry) + strlen(tok) + 1); + if (entry == NULL) { + pr_err("Memory allocation failure\n"); + return -1; + } + + strcpy(entry->name, tok); + list_add_tail(&entry->list, &callstack_filters); + } + + free(s); + return ret; +} + int cmd_lock(int argc, const char **argv) { const struct option lock_options[] = { @@ -2190,6 +2254,8 @@ int cmd_lock(int argc, const char **argv) "Filter specific type of locks", parse_lock_type), OPT_CALLBACK('L', "lock-filter", NULL, "ADDRS/NAMES", "Filter specific address/symbol of locks", parse_lock_addr), + OPT_CALLBACK('S', "callstack-filter", NULL, "NAMES", + "Filter specific function in the callstack", parse_call_stack), OPT_PARENT(lock_options) }; diff --git a/tools/perf/util/bpf_lock_contention.c b/tools/perf/util/bpf_lock_contention.c index 0236334fd69b..4902ac331f41 100644 --- a/tools/perf/util/bpf_lock_contention.c +++ b/tools/perf/util/bpf_lock_contention.c @@ -268,7 +268,7 @@ int lock_contention_read(struct lock_contention *con) break; } - if (verbose > 0) { + if (con->save_callstack) { st->callstack = memdup(stack_trace, stack_size); if (st->callstack == NULL) break; diff --git a/tools/perf/util/lock-contention.h b/tools/perf/util/lock-contention.h index b99e83fccf5c..17e594d57a61 100644 --- a/tools/perf/util/lock-contention.h +++ b/tools/perf/util/lock-contention.h @@ -128,6 +128,7 @@ struct lock_contention { int max_stack; int stack_skip; int aggr_mode; + bool save_callstack; }; #ifdef HAVE_BPF_SKEL
The -S/--callstack-filter is to limit display entries having the given string in the callstack (not only in the caller in the output). The following example shows lock contention results if the callstack has 'net' substring somewhere. Note that the caller '__dev_queue_xmit' does not match to it, but it has 'inet6_csk_xmit' in the callstack. This applies even if you don't use -v option to show the full callstack. $ sudo ./perf lock con -abv -S net sleep 1 ... contended total wait max wait avg wait type caller 5 70.20 us 16.13 us 14.04 us spinlock __dev_queue_xmit+0xb6d 0xffffffffa5dd1c60 _raw_spin_lock+0x30 0xffffffffa5b8f6ed __dev_queue_xmit+0xb6d 0xffffffffa5cd8267 ip6_finish_output2+0x2c7 0xffffffffa5cdac14 ip6_finish_output+0x1d4 0xffffffffa5cdb477 ip6_xmit+0x457 0xffffffffa5d1fd17 inet6_csk_xmit+0xd7 0xffffffffa5c5f4aa __tcp_transmit_skb+0x54a 0xffffffffa5c6467d tcp_keepalive_timer+0x2fd Signed-off-by: Namhyung Kim <namhyung@kernel.org> --- tools/perf/Documentation/perf-lock.txt | 6 +++ tools/perf/builtin-lock.c | 68 +++++++++++++++++++++++++- tools/perf/util/bpf_lock_contention.c | 2 +- tools/perf/util/lock-contention.h | 1 + 4 files changed, 75 insertions(+), 2 deletions(-)