| Message ID | 20230630083344.984305-17-jolsa@kernel.org (mailing list archive) |
|---|---|
| State | Changes Requested |
| Delegated to: | BPF |
| Headers | show |
| Series | bpf: Add multi uprobe link | expand |
On Fri, Jun 30, 2023 at 1:37 AM Jiri Olsa <jolsa@kernel.org> wrote: > > Adding support for usdt_manager_attach_usdt to use uprobe_multi > link to attach to usdt probes. > > The uprobe_multi support is detected before the usdt program is > loaded and its expected_attach_type is set accordingly. > > If uprobe_multi support is detected the usdt_manager_attach_usdt > gathers uprobes info and calls bpf_program__attach_uprobe to > create all needed uprobes. > > If uprobe_multi support is not detected the old behaviour stays. > > Also adding usdt.s program section for sleepable usdt probes. > > Signed-off-by: Jiri Olsa <jolsa@kernel.org> > --- > tools/lib/bpf/libbpf.c | 13 +++++-- > tools/lib/bpf/usdt.c | 78 ++++++++++++++++++++++++++++++++++-------- > 2 files changed, 75 insertions(+), 16 deletions(-) > > diff --git a/tools/lib/bpf/libbpf.c b/tools/lib/bpf/libbpf.c > index 4f61f9dc1748..e234c2e860f9 100644 > --- a/tools/lib/bpf/libbpf.c > +++ b/tools/lib/bpf/libbpf.c > @@ -365,6 +365,8 @@ enum sec_def_flags { > SEC_SLEEPABLE = 8, > /* BPF program support non-linear XDP buffer */ > SEC_XDP_FRAGS = 16, > + /* Setup proper attach type for usdt probes. */ > + SEC_USDT = 32, > }; > > struct bpf_sec_def { > @@ -6807,6 +6809,10 @@ static int libbpf_prepare_prog_load(struct bpf_program *prog, > if (prog->type == BPF_PROG_TYPE_XDP && (def & SEC_XDP_FRAGS)) > opts->prog_flags |= BPF_F_XDP_HAS_FRAGS; > > + /* special check for usdt to use uprobe_multi link */ > + if ((def & SEC_USDT) && kernel_supports(NULL, FEAT_UPROBE_MULTI_LINK)) please pass prog->obj to kernel_supports(), it will be especially important later with BPF token stuff > + prog->expected_attach_type = BPF_TRACE_UPROBE_MULTI; > + > if ((def & SEC_ATTACH_BTF) && !prog->attach_btf_id) { > int btf_obj_fd = 0, btf_type_id = 0, err; > const char *attach_name; > @@ -6875,7 +6881,6 @@ static int bpf_object_load_prog(struct bpf_object *obj, struct bpf_program *prog > if (!insns || !insns_cnt) > return -EINVAL; > > - load_attr.expected_attach_type = prog->expected_attach_type; > if (kernel_supports(obj, FEAT_PROG_NAME)) > prog_name = prog->name; > load_attr.attach_prog_fd = prog->attach_prog_fd; > @@ -6911,6 +6916,9 @@ static int bpf_object_load_prog(struct bpf_object *obj, struct bpf_program *prog > insns_cnt = prog->insns_cnt; > } > > + /* allow prog_prepare_load_fn to change expected_attach_type */ > + load_attr.expected_attach_type = prog->expected_attach_type; > + > if (obj->gen_loader) { > bpf_gen__prog_load(obj->gen_loader, prog->type, prog->name, > license, insns, insns_cnt, &load_attr, > @@ -8711,7 +8719,8 @@ static const struct bpf_sec_def section_defs[] = { > SEC_DEF("uretprobe.multi.s+", KPROBE, BPF_TRACE_UPROBE_MULTI, SEC_SLEEPABLE, attach_uprobe_multi), > SEC_DEF("ksyscall+", KPROBE, 0, SEC_NONE, attach_ksyscall), > SEC_DEF("kretsyscall+", KPROBE, 0, SEC_NONE, attach_ksyscall), > - SEC_DEF("usdt+", KPROBE, 0, SEC_NONE, attach_usdt), > + SEC_DEF("usdt+", KPROBE, 0, SEC_USDT, attach_usdt), > + SEC_DEF("usdt.s+", KPROBE, 0, SEC_USDT|SEC_SLEEPABLE, attach_usdt), spaces around | > SEC_DEF("tc", SCHED_CLS, 0, SEC_NONE), > SEC_DEF("classifier", SCHED_CLS, 0, SEC_NONE), > SEC_DEF("action", SCHED_ACT, 0, SEC_NONE), > diff --git a/tools/lib/bpf/usdt.c b/tools/lib/bpf/usdt.c > index 9fa883ebc0bd..6ff66a8eaf85 100644 > --- a/tools/lib/bpf/usdt.c > +++ b/tools/lib/bpf/usdt.c > @@ -809,6 +809,8 @@ struct bpf_link_usdt { > long abs_ip; > struct bpf_link *link; > } *uprobes; > + > + struct bpf_link *multi_link; > }; > > static int bpf_link_usdt_detach(struct bpf_link *link) > @@ -817,6 +819,9 @@ static int bpf_link_usdt_detach(struct bpf_link *link) > struct usdt_manager *man = usdt_link->usdt_man; > int i; > > + /* When having multi_link, uprobe_cnt is 0 */ misplaced comment, move down to for() loop? > + bpf_link__destroy(usdt_link->multi_link); > + > for (i = 0; i < usdt_link->uprobe_cnt; i++) { > /* detach underlying uprobe link */ > bpf_link__destroy(usdt_link->uprobes[i].link); > @@ -944,11 +949,13 @@ struct bpf_link *usdt_manager_attach_usdt(struct usdt_manager *man, const struct > const char *usdt_provider, const char *usdt_name, > __u64 usdt_cookie) > { > + unsigned long *offsets = NULL, *ref_ctr_offsets = NULL; > int i, err, spec_map_fd, ip_map_fd; > LIBBPF_OPTS(bpf_uprobe_opts, opts); > struct hashmap *specs_hash = NULL; > struct bpf_link_usdt *link = NULL; > struct usdt_target *targets = NULL; > + __u64 *cookies = NULL; > struct elf_fd elf_fd; > size_t target_cnt; > > @@ -995,10 +1002,21 @@ struct bpf_link *usdt_manager_attach_usdt(struct usdt_manager *man, const struct > link->link.detach = &bpf_link_usdt_detach; > link->link.dealloc = &bpf_link_usdt_dealloc; > > - link->uprobes = calloc(target_cnt, sizeof(*link->uprobes)); > - if (!link->uprobes) { > - err = -ENOMEM; > - goto err_out; > + if (kernel_supports(NULL, FEAT_UPROBE_MULTI_LINK)) { see how we feature-detect has_sema_refcnt and has_bpf_cookie, let's do the same with UPROBE_MULTI_LINK, detect once, remember, consistently use it (it also matters later for BPF token) > + offsets = calloc(target_cnt, sizeof(*offsets)); > + cookies = calloc(target_cnt, sizeof(*cookies)); > + ref_ctr_offsets = calloc(target_cnt, sizeof(*ref_ctr_offsets)); > + > + if (!offsets || !ref_ctr_offsets || !cookies) { > + err = -ENOMEM; > + goto err_out; > + } > + } else { > + link->uprobes = calloc(target_cnt, sizeof(*link->uprobes)); > + if (!link->uprobes) { > + err = -ENOMEM; > + goto err_out; > + } > } > > for (i = 0; i < target_cnt; i++) { > @@ -1039,20 +1057,48 @@ struct bpf_link *usdt_manager_attach_usdt(struct usdt_manager *man, const struct > goto err_out; > } > > - opts.ref_ctr_offset = target->sema_off; > - opts.bpf_cookie = man->has_bpf_cookie ? spec_id : 0; > - uprobe_link = bpf_program__attach_uprobe_opts(prog, pid, path, > - target->rel_ip, &opts); > - err = libbpf_get_error(uprobe_link); > + if (kernel_supports(NULL, FEAT_UPROBE_MULTI_LINK)) { > + offsets[i] = target->rel_ip; > + ref_ctr_offsets[i] = target->sema_off; > + cookies[i] = spec_id; > + } else { > + opts.ref_ctr_offset = target->sema_off; > + opts.bpf_cookie = man->has_bpf_cookie ? spec_id : 0; > + uprobe_link = bpf_program__attach_uprobe_opts(prog, pid, path, > + target->rel_ip, &opts); > + err = libbpf_get_error(uprobe_link); > + if (err) { > + pr_warn("usdt: failed to attach uprobe #%d for '%s:%s' in '%s': %d\n", > + i, usdt_provider, usdt_name, path, err); > + goto err_out; > + } > + > + link->uprobes[i].link = uprobe_link; > + link->uprobes[i].abs_ip = target->abs_ip; > + link->uprobe_cnt++; > + } > + } > + > + if (kernel_supports(NULL, FEAT_UPROBE_MULTI_LINK)) { same as above, we should feature-detect once per usdt_manager while we have associated bpf_object > + LIBBPF_OPTS(bpf_uprobe_multi_opts, opts_multi, > + .cnt = target_cnt, > + .offsets = offsets, > + .ref_ctr_offsets = ref_ctr_offsets, > + .cookies = cookies, > + ); > + > + link->multi_link = bpf_program__attach_uprobe_multi(prog, pid, path, > + NULL, &opts_multi); > + err = libbpf_get_error(link->multi_link); let's not use libbpf_get_error() in new code, there is no need, just `err = -errno` and `if (!link->multi_link)` > if (err) { > - pr_warn("usdt: failed to attach uprobe #%d for '%s:%s' in '%s': %d\n", > - i, usdt_provider, usdt_name, path, err); > + pr_warn("usdt: failed to attach uprobe multi for '%s:%s' in '%s': %d\n", > + usdt_provider, usdt_name, path, err); > goto err_out; > } > > - link->uprobes[i].link = uprobe_link; > - link->uprobes[i].abs_ip = target->abs_ip; > - link->uprobe_cnt++; > + free(offsets); > + free(ref_ctr_offsets); > + free(cookies); > } > > free(targets); > @@ -1061,6 +1107,10 @@ struct bpf_link *usdt_manager_attach_usdt(struct usdt_manager *man, const struct > return &link->link; > > err_out: > + free(offsets); > + free(ref_ctr_offsets); > + free(cookies); > + > if (link) > bpf_link__destroy(&link->link); > free(targets); > -- > 2.41.0 >
On Thu, Jul 06, 2023 at 09:29:35PM -0700, Andrii Nakryiko wrote: > On Fri, Jun 30, 2023 at 1:37 AM Jiri Olsa <jolsa@kernel.org> wrote: > > > > Adding support for usdt_manager_attach_usdt to use uprobe_multi > > link to attach to usdt probes. > > > > The uprobe_multi support is detected before the usdt program is > > loaded and its expected_attach_type is set accordingly. > > > > If uprobe_multi support is detected the usdt_manager_attach_usdt > > gathers uprobes info and calls bpf_program__attach_uprobe to > > create all needed uprobes. > > > > If uprobe_multi support is not detected the old behaviour stays. > > > > Also adding usdt.s program section for sleepable usdt probes. > > > > Signed-off-by: Jiri Olsa <jolsa@kernel.org> > > --- > > tools/lib/bpf/libbpf.c | 13 +++++-- > > tools/lib/bpf/usdt.c | 78 ++++++++++++++++++++++++++++++++++-------- > > 2 files changed, 75 insertions(+), 16 deletions(-) > > > > diff --git a/tools/lib/bpf/libbpf.c b/tools/lib/bpf/libbpf.c > > index 4f61f9dc1748..e234c2e860f9 100644 > > --- a/tools/lib/bpf/libbpf.c > > +++ b/tools/lib/bpf/libbpf.c > > @@ -365,6 +365,8 @@ enum sec_def_flags { > > SEC_SLEEPABLE = 8, > > /* BPF program support non-linear XDP buffer */ > > SEC_XDP_FRAGS = 16, > > + /* Setup proper attach type for usdt probes. */ > > + SEC_USDT = 32, > > }; > > > > struct bpf_sec_def { > > @@ -6807,6 +6809,10 @@ static int libbpf_prepare_prog_load(struct bpf_program *prog, > > if (prog->type == BPF_PROG_TYPE_XDP && (def & SEC_XDP_FRAGS)) > > opts->prog_flags |= BPF_F_XDP_HAS_FRAGS; > > > > + /* special check for usdt to use uprobe_multi link */ > > + if ((def & SEC_USDT) && kernel_supports(NULL, FEAT_UPROBE_MULTI_LINK)) > > please pass prog->obj to kernel_supports(), it will be especially > important later with BPF token stuff did not realize I have it in prog->obj ;-) ok > > > + prog->expected_attach_type = BPF_TRACE_UPROBE_MULTI; > > + > > if ((def & SEC_ATTACH_BTF) && !prog->attach_btf_id) { > > int btf_obj_fd = 0, btf_type_id = 0, err; > > const char *attach_name; > > @@ -6875,7 +6881,6 @@ static int bpf_object_load_prog(struct bpf_object *obj, struct bpf_program *prog > > if (!insns || !insns_cnt) > > return -EINVAL; > > > > - load_attr.expected_attach_type = prog->expected_attach_type; > > if (kernel_supports(obj, FEAT_PROG_NAME)) > > prog_name = prog->name; > > load_attr.attach_prog_fd = prog->attach_prog_fd; > > @@ -6911,6 +6916,9 @@ static int bpf_object_load_prog(struct bpf_object *obj, struct bpf_program *prog > > insns_cnt = prog->insns_cnt; > > } > > > > + /* allow prog_prepare_load_fn to change expected_attach_type */ > > + load_attr.expected_attach_type = prog->expected_attach_type; > > + > > if (obj->gen_loader) { > > bpf_gen__prog_load(obj->gen_loader, prog->type, prog->name, > > license, insns, insns_cnt, &load_attr, > > @@ -8711,7 +8719,8 @@ static const struct bpf_sec_def section_defs[] = { > > SEC_DEF("uretprobe.multi.s+", KPROBE, BPF_TRACE_UPROBE_MULTI, SEC_SLEEPABLE, attach_uprobe_multi), > > SEC_DEF("ksyscall+", KPROBE, 0, SEC_NONE, attach_ksyscall), > > SEC_DEF("kretsyscall+", KPROBE, 0, SEC_NONE, attach_ksyscall), > > - SEC_DEF("usdt+", KPROBE, 0, SEC_NONE, attach_usdt), > > + SEC_DEF("usdt+", KPROBE, 0, SEC_USDT, attach_usdt), > > + SEC_DEF("usdt.s+", KPROBE, 0, SEC_USDT|SEC_SLEEPABLE, attach_usdt), > > spaces around | ook > > > SEC_DEF("tc", SCHED_CLS, 0, SEC_NONE), > > SEC_DEF("classifier", SCHED_CLS, 0, SEC_NONE), > > SEC_DEF("action", SCHED_ACT, 0, SEC_NONE), > > diff --git a/tools/lib/bpf/usdt.c b/tools/lib/bpf/usdt.c > > index 9fa883ebc0bd..6ff66a8eaf85 100644 > > --- a/tools/lib/bpf/usdt.c > > +++ b/tools/lib/bpf/usdt.c > > @@ -809,6 +809,8 @@ struct bpf_link_usdt { > > long abs_ip; > > struct bpf_link *link; > > } *uprobes; > > + > > + struct bpf_link *multi_link; > > }; > > > > static int bpf_link_usdt_detach(struct bpf_link *link) > > @@ -817,6 +819,9 @@ static int bpf_link_usdt_detach(struct bpf_link *link) > > struct usdt_manager *man = usdt_link->usdt_man; > > int i; > > > > + /* When having multi_link, uprobe_cnt is 0 */ > > misplaced comment, move down to for() loop? right, will move > > > + bpf_link__destroy(usdt_link->multi_link); > > + > > for (i = 0; i < usdt_link->uprobe_cnt; i++) { > > /* detach underlying uprobe link */ > > bpf_link__destroy(usdt_link->uprobes[i].link); > > @@ -944,11 +949,13 @@ struct bpf_link *usdt_manager_attach_usdt(struct usdt_manager *man, const struct > > const char *usdt_provider, const char *usdt_name, > > __u64 usdt_cookie) > > { > > + unsigned long *offsets = NULL, *ref_ctr_offsets = NULL; > > int i, err, spec_map_fd, ip_map_fd; > > LIBBPF_OPTS(bpf_uprobe_opts, opts); > > struct hashmap *specs_hash = NULL; > > struct bpf_link_usdt *link = NULL; > > struct usdt_target *targets = NULL; > > + __u64 *cookies = NULL; > > struct elf_fd elf_fd; > > size_t target_cnt; > > > > @@ -995,10 +1002,21 @@ struct bpf_link *usdt_manager_attach_usdt(struct usdt_manager *man, const struct > > link->link.detach = &bpf_link_usdt_detach; > > link->link.dealloc = &bpf_link_usdt_dealloc; > > > > - link->uprobes = calloc(target_cnt, sizeof(*link->uprobes)); > > - if (!link->uprobes) { > > - err = -ENOMEM; > > - goto err_out; > > + if (kernel_supports(NULL, FEAT_UPROBE_MULTI_LINK)) { > > see how we feature-detect has_sema_refcnt and has_bpf_cookie, let's do > the same with UPROBE_MULTI_LINK, detect once, remember, consistently > use it (it also matters later for BPF token) ok > > > + offsets = calloc(target_cnt, sizeof(*offsets)); > > + cookies = calloc(target_cnt, sizeof(*cookies)); > > + ref_ctr_offsets = calloc(target_cnt, sizeof(*ref_ctr_offsets)); > > + > > + if (!offsets || !ref_ctr_offsets || !cookies) { > > + err = -ENOMEM; > > + goto err_out; > > + } > > + } else { > > + link->uprobes = calloc(target_cnt, sizeof(*link->uprobes)); > > + if (!link->uprobes) { > > + err = -ENOMEM; > > + goto err_out; > > + } > > } > > > > for (i = 0; i < target_cnt; i++) { > > @@ -1039,20 +1057,48 @@ struct bpf_link *usdt_manager_attach_usdt(struct usdt_manager *man, const struct > > goto err_out; > > } > > > > - opts.ref_ctr_offset = target->sema_off; > > - opts.bpf_cookie = man->has_bpf_cookie ? spec_id : 0; > > - uprobe_link = bpf_program__attach_uprobe_opts(prog, pid, path, > > - target->rel_ip, &opts); > > - err = libbpf_get_error(uprobe_link); > > + if (kernel_supports(NULL, FEAT_UPROBE_MULTI_LINK)) { > > + offsets[i] = target->rel_ip; > > + ref_ctr_offsets[i] = target->sema_off; > > + cookies[i] = spec_id; > > + } else { > > + opts.ref_ctr_offset = target->sema_off; > > + opts.bpf_cookie = man->has_bpf_cookie ? spec_id : 0; > > + uprobe_link = bpf_program__attach_uprobe_opts(prog, pid, path, > > + target->rel_ip, &opts); > > + err = libbpf_get_error(uprobe_link); > > + if (err) { > > + pr_warn("usdt: failed to attach uprobe #%d for '%s:%s' in '%s': %d\n", > > + i, usdt_provider, usdt_name, path, err); > > + goto err_out; > > + } > > + > > + link->uprobes[i].link = uprobe_link; > > + link->uprobes[i].abs_ip = target->abs_ip; > > + link->uprobe_cnt++; > > + } > > + } > > + > > + if (kernel_supports(NULL, FEAT_UPROBE_MULTI_LINK)) { > > same as above, we should feature-detect once per usdt_manager while we > have associated bpf_object ook > > > + LIBBPF_OPTS(bpf_uprobe_multi_opts, opts_multi, > > + .cnt = target_cnt, > > + .offsets = offsets, > > + .ref_ctr_offsets = ref_ctr_offsets, > > + .cookies = cookies, > > + ); > > + > > + link->multi_link = bpf_program__attach_uprobe_multi(prog, pid, path, > > + NULL, &opts_multi); > > + err = libbpf_get_error(link->multi_link); > > let's not use libbpf_get_error() in new code, there is no need, just > `err = -errno` and `if (!link->multi_link)` ah right, did not see its comment that says it's no longer recomended, will change thanks, jirka
diff --git a/tools/lib/bpf/libbpf.c b/tools/lib/bpf/libbpf.c index 4f61f9dc1748..e234c2e860f9 100644 --- a/tools/lib/bpf/libbpf.c +++ b/tools/lib/bpf/libbpf.c @@ -365,6 +365,8 @@ enum sec_def_flags { SEC_SLEEPABLE = 8, /* BPF program support non-linear XDP buffer */ SEC_XDP_FRAGS = 16, + /* Setup proper attach type for usdt probes. */ + SEC_USDT = 32, }; struct bpf_sec_def { @@ -6807,6 +6809,10 @@ static int libbpf_prepare_prog_load(struct bpf_program *prog, if (prog->type == BPF_PROG_TYPE_XDP && (def & SEC_XDP_FRAGS)) opts->prog_flags |= BPF_F_XDP_HAS_FRAGS; + /* special check for usdt to use uprobe_multi link */ + if ((def & SEC_USDT) && kernel_supports(NULL, FEAT_UPROBE_MULTI_LINK)) + prog->expected_attach_type = BPF_TRACE_UPROBE_MULTI; + if ((def & SEC_ATTACH_BTF) && !prog->attach_btf_id) { int btf_obj_fd = 0, btf_type_id = 0, err; const char *attach_name; @@ -6875,7 +6881,6 @@ static int bpf_object_load_prog(struct bpf_object *obj, struct bpf_program *prog if (!insns || !insns_cnt) return -EINVAL; - load_attr.expected_attach_type = prog->expected_attach_type; if (kernel_supports(obj, FEAT_PROG_NAME)) prog_name = prog->name; load_attr.attach_prog_fd = prog->attach_prog_fd; @@ -6911,6 +6916,9 @@ static int bpf_object_load_prog(struct bpf_object *obj, struct bpf_program *prog insns_cnt = prog->insns_cnt; } + /* allow prog_prepare_load_fn to change expected_attach_type */ + load_attr.expected_attach_type = prog->expected_attach_type; + if (obj->gen_loader) { bpf_gen__prog_load(obj->gen_loader, prog->type, prog->name, license, insns, insns_cnt, &load_attr, @@ -8711,7 +8719,8 @@ static const struct bpf_sec_def section_defs[] = { SEC_DEF("uretprobe.multi.s+", KPROBE, BPF_TRACE_UPROBE_MULTI, SEC_SLEEPABLE, attach_uprobe_multi), SEC_DEF("ksyscall+", KPROBE, 0, SEC_NONE, attach_ksyscall), SEC_DEF("kretsyscall+", KPROBE, 0, SEC_NONE, attach_ksyscall), - SEC_DEF("usdt+", KPROBE, 0, SEC_NONE, attach_usdt), + SEC_DEF("usdt+", KPROBE, 0, SEC_USDT, attach_usdt), + SEC_DEF("usdt.s+", KPROBE, 0, SEC_USDT|SEC_SLEEPABLE, attach_usdt), SEC_DEF("tc", SCHED_CLS, 0, SEC_NONE), SEC_DEF("classifier", SCHED_CLS, 0, SEC_NONE), SEC_DEF("action", SCHED_ACT, 0, SEC_NONE), diff --git a/tools/lib/bpf/usdt.c b/tools/lib/bpf/usdt.c index 9fa883ebc0bd..6ff66a8eaf85 100644 --- a/tools/lib/bpf/usdt.c +++ b/tools/lib/bpf/usdt.c @@ -809,6 +809,8 @@ struct bpf_link_usdt { long abs_ip; struct bpf_link *link; } *uprobes; + + struct bpf_link *multi_link; }; static int bpf_link_usdt_detach(struct bpf_link *link) @@ -817,6 +819,9 @@ static int bpf_link_usdt_detach(struct bpf_link *link) struct usdt_manager *man = usdt_link->usdt_man; int i; + /* When having multi_link, uprobe_cnt is 0 */ + bpf_link__destroy(usdt_link->multi_link); + for (i = 0; i < usdt_link->uprobe_cnt; i++) { /* detach underlying uprobe link */ bpf_link__destroy(usdt_link->uprobes[i].link); @@ -944,11 +949,13 @@ struct bpf_link *usdt_manager_attach_usdt(struct usdt_manager *man, const struct const char *usdt_provider, const char *usdt_name, __u64 usdt_cookie) { + unsigned long *offsets = NULL, *ref_ctr_offsets = NULL; int i, err, spec_map_fd, ip_map_fd; LIBBPF_OPTS(bpf_uprobe_opts, opts); struct hashmap *specs_hash = NULL; struct bpf_link_usdt *link = NULL; struct usdt_target *targets = NULL; + __u64 *cookies = NULL; struct elf_fd elf_fd; size_t target_cnt; @@ -995,10 +1002,21 @@ struct bpf_link *usdt_manager_attach_usdt(struct usdt_manager *man, const struct link->link.detach = &bpf_link_usdt_detach; link->link.dealloc = &bpf_link_usdt_dealloc; - link->uprobes = calloc(target_cnt, sizeof(*link->uprobes)); - if (!link->uprobes) { - err = -ENOMEM; - goto err_out; + if (kernel_supports(NULL, FEAT_UPROBE_MULTI_LINK)) { + offsets = calloc(target_cnt, sizeof(*offsets)); + cookies = calloc(target_cnt, sizeof(*cookies)); + ref_ctr_offsets = calloc(target_cnt, sizeof(*ref_ctr_offsets)); + + if (!offsets || !ref_ctr_offsets || !cookies) { + err = -ENOMEM; + goto err_out; + } + } else { + link->uprobes = calloc(target_cnt, sizeof(*link->uprobes)); + if (!link->uprobes) { + err = -ENOMEM; + goto err_out; + } } for (i = 0; i < target_cnt; i++) { @@ -1039,20 +1057,48 @@ struct bpf_link *usdt_manager_attach_usdt(struct usdt_manager *man, const struct goto err_out; } - opts.ref_ctr_offset = target->sema_off; - opts.bpf_cookie = man->has_bpf_cookie ? spec_id : 0; - uprobe_link = bpf_program__attach_uprobe_opts(prog, pid, path, - target->rel_ip, &opts); - err = libbpf_get_error(uprobe_link); + if (kernel_supports(NULL, FEAT_UPROBE_MULTI_LINK)) { + offsets[i] = target->rel_ip; + ref_ctr_offsets[i] = target->sema_off; + cookies[i] = spec_id; + } else { + opts.ref_ctr_offset = target->sema_off; + opts.bpf_cookie = man->has_bpf_cookie ? spec_id : 0; + uprobe_link = bpf_program__attach_uprobe_opts(prog, pid, path, + target->rel_ip, &opts); + err = libbpf_get_error(uprobe_link); + if (err) { + pr_warn("usdt: failed to attach uprobe #%d for '%s:%s' in '%s': %d\n", + i, usdt_provider, usdt_name, path, err); + goto err_out; + } + + link->uprobes[i].link = uprobe_link; + link->uprobes[i].abs_ip = target->abs_ip; + link->uprobe_cnt++; + } + } + + if (kernel_supports(NULL, FEAT_UPROBE_MULTI_LINK)) { + LIBBPF_OPTS(bpf_uprobe_multi_opts, opts_multi, + .cnt = target_cnt, + .offsets = offsets, + .ref_ctr_offsets = ref_ctr_offsets, + .cookies = cookies, + ); + + link->multi_link = bpf_program__attach_uprobe_multi(prog, pid, path, + NULL, &opts_multi); + err = libbpf_get_error(link->multi_link); if (err) { - pr_warn("usdt: failed to attach uprobe #%d for '%s:%s' in '%s': %d\n", - i, usdt_provider, usdt_name, path, err); + pr_warn("usdt: failed to attach uprobe multi for '%s:%s' in '%s': %d\n", + usdt_provider, usdt_name, path, err); goto err_out; } - link->uprobes[i].link = uprobe_link; - link->uprobes[i].abs_ip = target->abs_ip; - link->uprobe_cnt++; + free(offsets); + free(ref_ctr_offsets); + free(cookies); } free(targets); @@ -1061,6 +1107,10 @@ struct bpf_link *usdt_manager_attach_usdt(struct usdt_manager *man, const struct return &link->link; err_out: + free(offsets); + free(ref_ctr_offsets); + free(cookies); + if (link) bpf_link__destroy(&link->link); free(targets);
Adding support for usdt_manager_attach_usdt to use uprobe_multi link to attach to usdt probes. The uprobe_multi support is detected before the usdt program is loaded and its expected_attach_type is set accordingly. If uprobe_multi support is detected the usdt_manager_attach_usdt gathers uprobes info and calls bpf_program__attach_uprobe to create all needed uprobes. If uprobe_multi support is not detected the old behaviour stays. Also adding usdt.s program section for sleepable usdt probes. Signed-off-by: Jiri Olsa <jolsa@kernel.org> --- tools/lib/bpf/libbpf.c | 13 +++++-- tools/lib/bpf/usdt.c | 78 ++++++++++++++++++++++++++++++++++-------- 2 files changed, 75 insertions(+), 16 deletions(-)