| Message ID | 20230620083550.690426-15-jolsa@kernel.org (mailing list archive) |
|---|---|
| State | Superseded |
| Delegated to: | BPF |
| Headers | show |
| Series | bpf: Add multi uprobe link | expand |
On Tue, Jun 20, 2023 at 1:38 AM Jiri Olsa <jolsa@kernel.org> wrote: > > Adding support for usdt_manager_attach_usdt to use uprobe_multi > link to attach to usdt probes. > > The uprobe_multi support is detected before the usdt program is > loaded and its expected_attach_type is set accordingly. > > If uprobe_multi support is detected the usdt_manager_attach_usdt > gathers uprobes info and calls bpf_program__attach_uprobe_opts to > create all needed uprobes. > > If uprobe_multi support is not detected the old behaviour stays. > > Signed-off-by: Jiri Olsa <jolsa@kernel.org> > --- > tools/lib/bpf/libbpf.c | 12 ++++- > tools/lib/bpf/usdt.c | 120 ++++++++++++++++++++++++++++++----------- > 2 files changed, 99 insertions(+), 33 deletions(-) > > diff --git a/tools/lib/bpf/libbpf.c b/tools/lib/bpf/libbpf.c > index 3d570898459e..9c7a67c5cbe8 100644 > --- a/tools/lib/bpf/libbpf.c > +++ b/tools/lib/bpf/libbpf.c > @@ -363,6 +363,8 @@ enum sec_def_flags { > SEC_SLEEPABLE = 8, > /* BPF program support non-linear XDP buffer */ > SEC_XDP_FRAGS = 16, > + /* Setup proper attach type for usdt probes. */ > + SEC_USDT = 32, > }; > > struct bpf_sec_def { > @@ -6799,6 +6801,10 @@ static int libbpf_prepare_prog_load(struct bpf_program *prog, > if (prog->type == BPF_PROG_TYPE_XDP && (def & SEC_XDP_FRAGS)) > opts->prog_flags |= BPF_F_XDP_HAS_FRAGS; > > + /* special check for usdt to use uprobe_multi link */ > + if ((def & SEC_USDT) && kernel_supports(NULL, FEAT_UPROBE_LINK)) > + prog->expected_attach_type = BPF_TRACE_UPROBE_MULTI; this is quite ugly. I think KPROBE programs do not have enforcement for expected_attach_type during BPF_PROG_LOAD, so we can set BPF_TRACE_UPROBE_MULTI unconditionally, right? > + > if ((def & SEC_ATTACH_BTF) && !prog->attach_btf_id) { > int btf_obj_fd = 0, btf_type_id = 0, err; > const char *attach_name; > @@ -6867,7 +6873,6 @@ static int bpf_object_load_prog(struct bpf_object *obj, struct bpf_program *prog > if (!insns || !insns_cnt) > return -EINVAL; > > - load_attr.expected_attach_type = prog->expected_attach_type; > if (kernel_supports(obj, FEAT_PROG_NAME)) > prog_name = prog->name; > load_attr.attach_prog_fd = prog->attach_prog_fd; > @@ -6903,6 +6908,9 @@ static int bpf_object_load_prog(struct bpf_object *obj, struct bpf_program *prog > insns_cnt = prog->insns_cnt; > } > > + /* allow prog_prepare_load_fn to change expected_attach_type */ > + load_attr.expected_attach_type = prog->expected_attach_type; > + > if (obj->gen_loader) { > bpf_gen__prog_load(obj->gen_loader, prog->type, prog->name, > license, insns, insns_cnt, &load_attr, > @@ -8703,7 +8711,7 @@ static const struct bpf_sec_def section_defs[] = { > SEC_DEF("uretprobe.multi.s+", KPROBE, BPF_TRACE_UPROBE_MULTI, SEC_SLEEPABLE, attach_uprobe_multi), > SEC_DEF("ksyscall+", KPROBE, 0, SEC_NONE, attach_ksyscall), > SEC_DEF("kretsyscall+", KPROBE, 0, SEC_NONE, attach_ksyscall), > - SEC_DEF("usdt+", KPROBE, 0, SEC_NONE, attach_usdt), > + SEC_DEF("usdt+", KPROBE, 0, SEC_USDT, attach_usdt), btw, given you are touching USDT stuff, can you please also add sleepable USDT (usdt.s+) ? > SEC_DEF("tc", SCHED_CLS, 0, SEC_NONE), > SEC_DEF("classifier", SCHED_CLS, 0, SEC_NONE), > SEC_DEF("action", SCHED_ACT, 0, SEC_NONE), > diff --git a/tools/lib/bpf/usdt.c b/tools/lib/bpf/usdt.c > index f1a141555f08..33f0a2b4cc1c 100644 > --- a/tools/lib/bpf/usdt.c > +++ b/tools/lib/bpf/usdt.c > @@ -808,6 +808,16 @@ struct bpf_link_usdt { > long abs_ip; > struct bpf_link *link; > } *uprobes; > + > + bool has_uprobe_multi; > + > + struct { > + char *path; > + unsigned long *offsets; > + unsigned long *ref_ctr_offsets; > + __u64 *cookies; you shouldn't need to persist this, this can be allocated and freed inside usdt_manager_attach_usdt(), you only need the link pointer > + struct bpf_link *link; > + } uprobe_multi; > }; > > static int bpf_link_usdt_detach(struct bpf_link *link) > @@ -816,19 +826,23 @@ static int bpf_link_usdt_detach(struct bpf_link *link) > struct usdt_manager *man = usdt_link->usdt_man; > int i; > > - for (i = 0; i < usdt_link->uprobe_cnt; i++) { > - /* detach underlying uprobe link */ > - bpf_link__destroy(usdt_link->uprobes[i].link); > - /* there is no need to update specs map because it will be > - * unconditionally overwritten on subsequent USDT attaches, > - * but if BPF cookies are not used we need to remove entry > - * from ip_to_spec_id map, otherwise we'll run into false > - * conflicting IP errors > - */ > - if (!man->has_bpf_cookie) { > - /* not much we can do about errors here */ > - (void)bpf_map_delete_elem(bpf_map__fd(man->ip_to_spec_id_map), > - &usdt_link->uprobes[i].abs_ip); > + if (usdt_link->has_uprobe_multi) { > + bpf_link__destroy(usdt_link->uprobe_multi.link); > + } else { > + for (i = 0; i < usdt_link->uprobe_cnt; i++) { > + /* detach underlying uprobe link */ > + bpf_link__destroy(usdt_link->uprobes[i].link); > + /* there is no need to update specs map because it will be > + * unconditionally overwritten on subsequent USDT attaches, > + * but if BPF cookies are not used we need to remove entry > + * from ip_to_spec_id map, otherwise we'll run into false > + * conflicting IP errors > + */ > + if (!man->has_bpf_cookie) { > + /* not much we can do about errors here */ > + (void)bpf_map_delete_elem(bpf_map__fd(man->ip_to_spec_id_map), > + &usdt_link->uprobes[i].abs_ip); > + } > } you can avoid shifting all this by keeping uprobe_cnt to zero bpf_link__destory(usdt_link->uprobe_multi.link) will work fine for NULL so just do both clean ups sequentially, knowing that only one of them will actually do anything > } > > @@ -868,9 +882,15 @@ static void bpf_link_usdt_dealloc(struct bpf_link *link) > { > struct bpf_link_usdt *usdt_link = container_of(link, struct bpf_link_usdt, link); > > - free(usdt_link->spec_ids); > - free(usdt_link->uprobes); > - free(usdt_link); > + if (usdt_link->has_uprobe_multi) { > + free(usdt_link->uprobe_multi.offsets); > + free(usdt_link->uprobe_multi.ref_ctr_offsets); > + free(usdt_link->uprobe_multi.cookies); > + } else { > + free(usdt_link->spec_ids); > + free(usdt_link->uprobes); > + free(usdt_link); > + } similar to the above, just do *all* the clean up unconditionally and rely on free() handling NULLs just fine > } > > static size_t specs_hash_fn(long key, void *ctx) > @@ -943,11 +963,13 @@ struct bpf_link *usdt_manager_attach_usdt(struct usdt_manager *man, const struct > const char *usdt_provider, const char *usdt_name, > __u64 usdt_cookie) > { > + LIBBPF_OPTS(bpf_uprobe_multi_opts, opts_multi); > int i, fd, err, spec_map_fd, ip_map_fd; > LIBBPF_OPTS(bpf_uprobe_opts, opts); > struct hashmap *specs_hash = NULL; > struct bpf_link_usdt *link = NULL; > struct usdt_target *targets = NULL; > + struct bpf_link *uprobe_link; > size_t target_cnt; > Elf *elf; > > @@ -1003,16 +1025,29 @@ struct bpf_link *usdt_manager_attach_usdt(struct usdt_manager *man, const struct > link->usdt_man = man; > link->link.detach = &bpf_link_usdt_detach; > link->link.dealloc = &bpf_link_usdt_dealloc; > + link->has_uprobe_multi = bpf_program__expected_attach_type(prog) == BPF_TRACE_UPROBE_MULTI; just use kernel_supports(), it's cleaner (and result is cached, so it's not less efficient) > > - link->uprobes = calloc(target_cnt, sizeof(*link->uprobes)); > - if (!link->uprobes) { > - err = -ENOMEM; > - goto err_out; > + if (link->has_uprobe_multi) { > + link->uprobe_multi.offsets = calloc(target_cnt, sizeof(*link->uprobe_multi.offsets)); > + link->uprobe_multi.ref_ctr_offsets = calloc(target_cnt, sizeof(*link->uprobe_multi.ref_ctr_offsets)); > + link->uprobe_multi.cookies = calloc(target_cnt, sizeof(*link->uprobe_multi.cookies)); > + > + if (!link->uprobe_multi.offsets || > + !link->uprobe_multi.ref_ctr_offsets || > + !link->uprobe_multi.cookies) { > + err = -ENOMEM; > + goto err_out; > + } > + } else { > + link->uprobes = calloc(target_cnt, sizeof(*link->uprobes)); > + if (!link->uprobes) { > + err = -ENOMEM; > + goto err_out; > + } > } > [...]
On Fri, Jun 23, 2023 at 01:40:27PM -0700, Andrii Nakryiko wrote: > On Tue, Jun 20, 2023 at 1:38 AM Jiri Olsa <jolsa@kernel.org> wrote: > > > > Adding support for usdt_manager_attach_usdt to use uprobe_multi > > link to attach to usdt probes. > > > > The uprobe_multi support is detected before the usdt program is > > loaded and its expected_attach_type is set accordingly. > > > > If uprobe_multi support is detected the usdt_manager_attach_usdt > > gathers uprobes info and calls bpf_program__attach_uprobe_opts to > > create all needed uprobes. > > > > If uprobe_multi support is not detected the old behaviour stays. > > > > Signed-off-by: Jiri Olsa <jolsa@kernel.org> > > --- > > tools/lib/bpf/libbpf.c | 12 ++++- > > tools/lib/bpf/usdt.c | 120 ++++++++++++++++++++++++++++++----------- > > 2 files changed, 99 insertions(+), 33 deletions(-) > > > > diff --git a/tools/lib/bpf/libbpf.c b/tools/lib/bpf/libbpf.c > > index 3d570898459e..9c7a67c5cbe8 100644 > > --- a/tools/lib/bpf/libbpf.c > > +++ b/tools/lib/bpf/libbpf.c > > @@ -363,6 +363,8 @@ enum sec_def_flags { > > SEC_SLEEPABLE = 8, > > /* BPF program support non-linear XDP buffer */ > > SEC_XDP_FRAGS = 16, > > + /* Setup proper attach type for usdt probes. */ > > + SEC_USDT = 32, > > }; > > > > struct bpf_sec_def { > > @@ -6799,6 +6801,10 @@ static int libbpf_prepare_prog_load(struct bpf_program *prog, > > if (prog->type == BPF_PROG_TYPE_XDP && (def & SEC_XDP_FRAGS)) > > opts->prog_flags |= BPF_F_XDP_HAS_FRAGS; > > > > + /* special check for usdt to use uprobe_multi link */ > > + if ((def & SEC_USDT) && kernel_supports(NULL, FEAT_UPROBE_LINK)) > > + prog->expected_attach_type = BPF_TRACE_UPROBE_MULTI; > > this is quite ugly. I think KPROBE programs do not have enforcement > for expected_attach_type during BPF_PROG_LOAD, so we can set > BPF_TRACE_UPROBE_MULTI unconditionally, right? t doesn't but we are adding one earlier to bpf_prog_attach_check_attach_type (the one with U/K typo): + case BPF_PROG_TYPE_KPROBE: + if (prog->expected_attach_type == BPF_TRACE_UPROBE_MULTI && + attach_type != BPF_TRACE_UPROBE_MULTI) + return -EINVAL; for same reasons we did the check for kprobe_multi in: db8eae6bc5c7 bpf: Force kprobe multi expected_attach_type for kprobe_multi link > > > + > > if ((def & SEC_ATTACH_BTF) && !prog->attach_btf_id) { > > int btf_obj_fd = 0, btf_type_id = 0, err; > > const char *attach_name; > > @@ -6867,7 +6873,6 @@ static int bpf_object_load_prog(struct bpf_object *obj, struct bpf_program *prog > > if (!insns || !insns_cnt) > > return -EINVAL; > > > > - load_attr.expected_attach_type = prog->expected_attach_type; > > if (kernel_supports(obj, FEAT_PROG_NAME)) > > prog_name = prog->name; > > load_attr.attach_prog_fd = prog->attach_prog_fd; > > @@ -6903,6 +6908,9 @@ static int bpf_object_load_prog(struct bpf_object *obj, struct bpf_program *prog > > insns_cnt = prog->insns_cnt; > > } > > > > + /* allow prog_prepare_load_fn to change expected_attach_type */ > > + load_attr.expected_attach_type = prog->expected_attach_type; > > + > > if (obj->gen_loader) { > > bpf_gen__prog_load(obj->gen_loader, prog->type, prog->name, > > license, insns, insns_cnt, &load_attr, > > @@ -8703,7 +8711,7 @@ static const struct bpf_sec_def section_defs[] = { > > SEC_DEF("uretprobe.multi.s+", KPROBE, BPF_TRACE_UPROBE_MULTI, SEC_SLEEPABLE, attach_uprobe_multi), > > SEC_DEF("ksyscall+", KPROBE, 0, SEC_NONE, attach_ksyscall), > > SEC_DEF("kretsyscall+", KPROBE, 0, SEC_NONE, attach_ksyscall), > > - SEC_DEF("usdt+", KPROBE, 0, SEC_NONE, attach_usdt), > > + SEC_DEF("usdt+", KPROBE, 0, SEC_USDT, attach_usdt), > > btw, given you are touching USDT stuff, can you please also add > sleepable USDT (usdt.s+) ? ok > > > > SEC_DEF("tc", SCHED_CLS, 0, SEC_NONE), > > SEC_DEF("classifier", SCHED_CLS, 0, SEC_NONE), > > SEC_DEF("action", SCHED_ACT, 0, SEC_NONE), > > diff --git a/tools/lib/bpf/usdt.c b/tools/lib/bpf/usdt.c > > index f1a141555f08..33f0a2b4cc1c 100644 > > --- a/tools/lib/bpf/usdt.c > > +++ b/tools/lib/bpf/usdt.c > > @@ -808,6 +808,16 @@ struct bpf_link_usdt { > > long abs_ip; > > struct bpf_link *link; > > } *uprobes; > > + > > + bool has_uprobe_multi; > > + > > + struct { > > + char *path; > > + unsigned long *offsets; > > + unsigned long *ref_ctr_offsets; > > + __u64 *cookies; > > you shouldn't need to persist this, this can be allocated and freed > inside usdt_manager_attach_usdt(), you only need the link pointer aah right, that can go away, nice > > > > + struct bpf_link *link; > > + } uprobe_multi; > > }; > > > > static int bpf_link_usdt_detach(struct bpf_link *link) > > @@ -816,19 +826,23 @@ static int bpf_link_usdt_detach(struct bpf_link *link) > > struct usdt_manager *man = usdt_link->usdt_man; > > int i; > > > > - for (i = 0; i < usdt_link->uprobe_cnt; i++) { > > - /* detach underlying uprobe link */ > > - bpf_link__destroy(usdt_link->uprobes[i].link); > > - /* there is no need to update specs map because it will be > > - * unconditionally overwritten on subsequent USDT attaches, > > - * but if BPF cookies are not used we need to remove entry > > - * from ip_to_spec_id map, otherwise we'll run into false > > - * conflicting IP errors > > - */ > > - if (!man->has_bpf_cookie) { > > - /* not much we can do about errors here */ > > - (void)bpf_map_delete_elem(bpf_map__fd(man->ip_to_spec_id_map), > > - &usdt_link->uprobes[i].abs_ip); > > + if (usdt_link->has_uprobe_multi) { > > + bpf_link__destroy(usdt_link->uprobe_multi.link); > > + } else { > > + for (i = 0; i < usdt_link->uprobe_cnt; i++) { > > + /* detach underlying uprobe link */ > > + bpf_link__destroy(usdt_link->uprobes[i].link); > > + /* there is no need to update specs map because it will be > > + * unconditionally overwritten on subsequent USDT attaches, > > + * but if BPF cookies are not used we need to remove entry > > + * from ip_to_spec_id map, otherwise we'll run into false > > + * conflicting IP errors > > + */ > > + if (!man->has_bpf_cookie) { > > + /* not much we can do about errors here */ > > + (void)bpf_map_delete_elem(bpf_map__fd(man->ip_to_spec_id_map), > > + &usdt_link->uprobes[i].abs_ip); > > + } > > } > > you can avoid shifting all this by keeping uprobe_cnt to zero > > bpf_link__destory(usdt_link->uprobe_multi.link) will work fine for NULL > > so just do both clean ups sequentially, knowing that only one of them > will actually do anything ok, good idea > > > } > > > > @@ -868,9 +882,15 @@ static void bpf_link_usdt_dealloc(struct bpf_link *link) > > { > > struct bpf_link_usdt *usdt_link = container_of(link, struct bpf_link_usdt, link); > > > > - free(usdt_link->spec_ids); > > - free(usdt_link->uprobes); > > - free(usdt_link); > > + if (usdt_link->has_uprobe_multi) { > > + free(usdt_link->uprobe_multi.offsets); > > + free(usdt_link->uprobe_multi.ref_ctr_offsets); > > + free(usdt_link->uprobe_multi.cookies); > > + } else { > > + free(usdt_link->spec_ids); > > + free(usdt_link->uprobes); > > + free(usdt_link); > > + } > > similar to the above, just do *all* the clean up unconditionally and > rely on free() handling NULLs just fine ok > > > } > > > > static size_t specs_hash_fn(long key, void *ctx) > > @@ -943,11 +963,13 @@ struct bpf_link *usdt_manager_attach_usdt(struct usdt_manager *man, const struct > > const char *usdt_provider, const char *usdt_name, > > __u64 usdt_cookie) > > { > > + LIBBPF_OPTS(bpf_uprobe_multi_opts, opts_multi); > > int i, fd, err, spec_map_fd, ip_map_fd; > > LIBBPF_OPTS(bpf_uprobe_opts, opts); > > struct hashmap *specs_hash = NULL; > > struct bpf_link_usdt *link = NULL; > > struct usdt_target *targets = NULL; > > + struct bpf_link *uprobe_link; > > size_t target_cnt; > > Elf *elf; > > > > @@ -1003,16 +1025,29 @@ struct bpf_link *usdt_manager_attach_usdt(struct usdt_manager *man, const struct > > link->usdt_man = man; > > link->link.detach = &bpf_link_usdt_detach; > > link->link.dealloc = &bpf_link_usdt_dealloc; > > + link->has_uprobe_multi = bpf_program__expected_attach_type(prog) == BPF_TRACE_UPROBE_MULTI; > > just use kernel_supports(), it's cleaner (and result is cached, so > it's not less efficient) ok thanks, jirka
diff --git a/tools/lib/bpf/libbpf.c b/tools/lib/bpf/libbpf.c index 3d570898459e..9c7a67c5cbe8 100644 --- a/tools/lib/bpf/libbpf.c +++ b/tools/lib/bpf/libbpf.c @@ -363,6 +363,8 @@ enum sec_def_flags { SEC_SLEEPABLE = 8, /* BPF program support non-linear XDP buffer */ SEC_XDP_FRAGS = 16, + /* Setup proper attach type for usdt probes. */ + SEC_USDT = 32, }; struct bpf_sec_def { @@ -6799,6 +6801,10 @@ static int libbpf_prepare_prog_load(struct bpf_program *prog, if (prog->type == BPF_PROG_TYPE_XDP && (def & SEC_XDP_FRAGS)) opts->prog_flags |= BPF_F_XDP_HAS_FRAGS; + /* special check for usdt to use uprobe_multi link */ + if ((def & SEC_USDT) && kernel_supports(NULL, FEAT_UPROBE_LINK)) + prog->expected_attach_type = BPF_TRACE_UPROBE_MULTI; + if ((def & SEC_ATTACH_BTF) && !prog->attach_btf_id) { int btf_obj_fd = 0, btf_type_id = 0, err; const char *attach_name; @@ -6867,7 +6873,6 @@ static int bpf_object_load_prog(struct bpf_object *obj, struct bpf_program *prog if (!insns || !insns_cnt) return -EINVAL; - load_attr.expected_attach_type = prog->expected_attach_type; if (kernel_supports(obj, FEAT_PROG_NAME)) prog_name = prog->name; load_attr.attach_prog_fd = prog->attach_prog_fd; @@ -6903,6 +6908,9 @@ static int bpf_object_load_prog(struct bpf_object *obj, struct bpf_program *prog insns_cnt = prog->insns_cnt; } + /* allow prog_prepare_load_fn to change expected_attach_type */ + load_attr.expected_attach_type = prog->expected_attach_type; + if (obj->gen_loader) { bpf_gen__prog_load(obj->gen_loader, prog->type, prog->name, license, insns, insns_cnt, &load_attr, @@ -8703,7 +8711,7 @@ static const struct bpf_sec_def section_defs[] = { SEC_DEF("uretprobe.multi.s+", KPROBE, BPF_TRACE_UPROBE_MULTI, SEC_SLEEPABLE, attach_uprobe_multi), SEC_DEF("ksyscall+", KPROBE, 0, SEC_NONE, attach_ksyscall), SEC_DEF("kretsyscall+", KPROBE, 0, SEC_NONE, attach_ksyscall), - SEC_DEF("usdt+", KPROBE, 0, SEC_NONE, attach_usdt), + SEC_DEF("usdt+", KPROBE, 0, SEC_USDT, attach_usdt), SEC_DEF("tc", SCHED_CLS, 0, SEC_NONE), SEC_DEF("classifier", SCHED_CLS, 0, SEC_NONE), SEC_DEF("action", SCHED_ACT, 0, SEC_NONE), diff --git a/tools/lib/bpf/usdt.c b/tools/lib/bpf/usdt.c index f1a141555f08..33f0a2b4cc1c 100644 --- a/tools/lib/bpf/usdt.c +++ b/tools/lib/bpf/usdt.c @@ -808,6 +808,16 @@ struct bpf_link_usdt { long abs_ip; struct bpf_link *link; } *uprobes; + + bool has_uprobe_multi; + + struct { + char *path; + unsigned long *offsets; + unsigned long *ref_ctr_offsets; + __u64 *cookies; + struct bpf_link *link; + } uprobe_multi; }; static int bpf_link_usdt_detach(struct bpf_link *link) @@ -816,19 +826,23 @@ static int bpf_link_usdt_detach(struct bpf_link *link) struct usdt_manager *man = usdt_link->usdt_man; int i; - for (i = 0; i < usdt_link->uprobe_cnt; i++) { - /* detach underlying uprobe link */ - bpf_link__destroy(usdt_link->uprobes[i].link); - /* there is no need to update specs map because it will be - * unconditionally overwritten on subsequent USDT attaches, - * but if BPF cookies are not used we need to remove entry - * from ip_to_spec_id map, otherwise we'll run into false - * conflicting IP errors - */ - if (!man->has_bpf_cookie) { - /* not much we can do about errors here */ - (void)bpf_map_delete_elem(bpf_map__fd(man->ip_to_spec_id_map), - &usdt_link->uprobes[i].abs_ip); + if (usdt_link->has_uprobe_multi) { + bpf_link__destroy(usdt_link->uprobe_multi.link); + } else { + for (i = 0; i < usdt_link->uprobe_cnt; i++) { + /* detach underlying uprobe link */ + bpf_link__destroy(usdt_link->uprobes[i].link); + /* there is no need to update specs map because it will be + * unconditionally overwritten on subsequent USDT attaches, + * but if BPF cookies are not used we need to remove entry + * from ip_to_spec_id map, otherwise we'll run into false + * conflicting IP errors + */ + if (!man->has_bpf_cookie) { + /* not much we can do about errors here */ + (void)bpf_map_delete_elem(bpf_map__fd(man->ip_to_spec_id_map), + &usdt_link->uprobes[i].abs_ip); + } } } @@ -868,9 +882,15 @@ static void bpf_link_usdt_dealloc(struct bpf_link *link) { struct bpf_link_usdt *usdt_link = container_of(link, struct bpf_link_usdt, link); - free(usdt_link->spec_ids); - free(usdt_link->uprobes); - free(usdt_link); + if (usdt_link->has_uprobe_multi) { + free(usdt_link->uprobe_multi.offsets); + free(usdt_link->uprobe_multi.ref_ctr_offsets); + free(usdt_link->uprobe_multi.cookies); + } else { + free(usdt_link->spec_ids); + free(usdt_link->uprobes); + free(usdt_link); + } } static size_t specs_hash_fn(long key, void *ctx) @@ -943,11 +963,13 @@ struct bpf_link *usdt_manager_attach_usdt(struct usdt_manager *man, const struct const char *usdt_provider, const char *usdt_name, __u64 usdt_cookie) { + LIBBPF_OPTS(bpf_uprobe_multi_opts, opts_multi); int i, fd, err, spec_map_fd, ip_map_fd; LIBBPF_OPTS(bpf_uprobe_opts, opts); struct hashmap *specs_hash = NULL; struct bpf_link_usdt *link = NULL; struct usdt_target *targets = NULL; + struct bpf_link *uprobe_link; size_t target_cnt; Elf *elf; @@ -1003,16 +1025,29 @@ struct bpf_link *usdt_manager_attach_usdt(struct usdt_manager *man, const struct link->usdt_man = man; link->link.detach = &bpf_link_usdt_detach; link->link.dealloc = &bpf_link_usdt_dealloc; + link->has_uprobe_multi = bpf_program__expected_attach_type(prog) == BPF_TRACE_UPROBE_MULTI; - link->uprobes = calloc(target_cnt, sizeof(*link->uprobes)); - if (!link->uprobes) { - err = -ENOMEM; - goto err_out; + if (link->has_uprobe_multi) { + link->uprobe_multi.offsets = calloc(target_cnt, sizeof(*link->uprobe_multi.offsets)); + link->uprobe_multi.ref_ctr_offsets = calloc(target_cnt, sizeof(*link->uprobe_multi.ref_ctr_offsets)); + link->uprobe_multi.cookies = calloc(target_cnt, sizeof(*link->uprobe_multi.cookies)); + + if (!link->uprobe_multi.offsets || + !link->uprobe_multi.ref_ctr_offsets || + !link->uprobe_multi.cookies) { + err = -ENOMEM; + goto err_out; + } + } else { + link->uprobes = calloc(target_cnt, sizeof(*link->uprobes)); + if (!link->uprobes) { + err = -ENOMEM; + goto err_out; + } } for (i = 0; i < target_cnt; i++) { struct usdt_target *target = &targets[i]; - struct bpf_link *uprobe_link; bool is_new; int spec_id; @@ -1048,20 +1083,43 @@ struct bpf_link *usdt_manager_attach_usdt(struct usdt_manager *man, const struct goto err_out; } - opts.ref_ctr_offset = target->sema_off; - opts.bpf_cookie = man->has_bpf_cookie ? spec_id : 0; - uprobe_link = bpf_program__attach_uprobe_opts(prog, pid, path, - target->rel_ip, &opts); + if (link->has_uprobe_multi) { + link->uprobe_multi.offsets[i] = target->rel_ip; + link->uprobe_multi.ref_ctr_offsets[i] = target->sema_off; + link->uprobe_multi.cookies[i] = spec_id; + } else { + opts.ref_ctr_offset = target->sema_off; + opts.bpf_cookie = man->has_bpf_cookie ? spec_id : 0; + uprobe_link = bpf_program__attach_uprobe_opts(prog, pid, path, + target->rel_ip, &opts); + err = libbpf_get_error(uprobe_link); + if (err) { + pr_warn("usdt: failed to attach uprobe #%d for '%s:%s' in '%s': %d\n", + i, usdt_provider, usdt_name, path, err); + goto err_out; + } + + link->uprobes[i].link = uprobe_link; + link->uprobes[i].abs_ip = target->abs_ip; + link->uprobe_cnt++; + } + } + + if (link->has_uprobe_multi) { + opts_multi.cnt = target_cnt; + opts_multi.path = path; + opts_multi.offsets = link->uprobe_multi.offsets; + opts_multi.ref_ctr_offsets = link->uprobe_multi.ref_ctr_offsets; + opts_multi.cookies = link->uprobe_multi.cookies; + + uprobe_link = bpf_program__attach_uprobe_multi_opts(prog, pid, NULL, NULL, &opts_multi); err = libbpf_get_error(uprobe_link); if (err) { - pr_warn("usdt: failed to attach uprobe #%d for '%s:%s' in '%s': %d\n", - i, usdt_provider, usdt_name, path, err); + pr_warn("usdt: failed to attach uprobe multi for '%s:%s' in '%s': %d\n", + usdt_provider, usdt_name, path, err); goto err_out; } - - link->uprobes[i].link = uprobe_link; - link->uprobes[i].abs_ip = target->abs_ip; - link->uprobe_cnt++; + link->uprobe_multi.link = uprobe_link; } free(targets);
Adding support for usdt_manager_attach_usdt to use uprobe_multi link to attach to usdt probes. The uprobe_multi support is detected before the usdt program is loaded and its expected_attach_type is set accordingly. If uprobe_multi support is detected the usdt_manager_attach_usdt gathers uprobes info and calls bpf_program__attach_uprobe_opts to create all needed uprobes. If uprobe_multi support is not detected the old behaviour stays. Signed-off-by: Jiri Olsa <jolsa@kernel.org> --- tools/lib/bpf/libbpf.c | 12 ++++- tools/lib/bpf/usdt.c | 120 ++++++++++++++++++++++++++++++----------- 2 files changed, 99 insertions(+), 33 deletions(-)