@@ -34,6 +34,21 @@
#include <pthread.h>
#define BTF_ENCODER_MAX_PROTO 512
+#define BTF_IDS_SECTION ".BTF_ids"
+#define BTF_ID_FUNC_PFX "__BTF_ID__func__"
+#define BTF_ID_SET8_PFX "__BTF_ID__set8__"
+#define BTF_SET8_KFUNCS (1 << 0)
+#define BTF_KFUNC_TYPE_TAG "bpf_kfunc"
+
+/* Adapted from include/linux/btf_ids.h */
+struct btf_id_set8 {
+ uint32_t cnt;
+ uint32_t flags;
+ struct {
+ uint32_t id;
+ uint32_t flags;
+ } pairs[];
+};
/* state used to do later encoding of saved functions */
struct btf_encoder_state {
@@ -76,6 +91,7 @@ struct btf_encoder {
verbose,
force,
gen_floats,
+ skip_encoding_decl_tag,
tag_kfuncs,
is_rel;
uint32_t array_index_id;
@@ -95,6 +111,17 @@ struct btf_encoder {
} functions;
};
+struct btf_func {
+ const char *name;
+ int type_id;
+};
+
+/* Half open interval representing range of addresses containing kfuncs */
+struct btf_kfunc_set_range {
+ uint64_t start;
+ uint64_t end;
+};
+
static LIST_HEAD(encoders);
static pthread_mutex_t encoders__lock = PTHREAD_MUTEX_INITIALIZER;
@@ -1364,8 +1391,343 @@ out:
return err;
}
+/* Returns if `sym` points to a kfunc set */
+static int is_sym_kfunc_set(GElf_Sym *sym, const char *name, Elf_Data *idlist, size_t idlist_addr)
+{
+ void *ptr = idlist->d_buf;
+ struct btf_id_set8 *set;
+ int off;
+
+ /* kfuncs are only found in BTF_SET8's */
+ if (!strstarts(name, BTF_ID_SET8_PFX))
+ return false;
+
+ off = sym->st_value - idlist_addr;
+ if (off >= idlist->d_size) {
+ fprintf(stderr, "%s: symbol '%s' out of bounds\n", __func__, name);
+ return false;
+ }
+
+ /* Check the set8 flags to see if it was marked as kfunc */
+ set = ptr + off;
+ return set->flags & BTF_SET8_KFUNCS;
+}
+
+/*
+ * Parse BTF_ID symbol and return the func name.
+ *
+ * Returns:
+ * Caller-owned string containing func name if successful.
+ * NULL if !func or on error.
+ */
+static char *get_func_name(const char *sym)
+{
+ char *func, *end;
+
+ /* Example input: __BTF_ID__func__vfs_close__1
+ *
+ * The goal is to strip the prefix and suffix such that we only
+ * return vfs_close.
+ */
+
+ if (!strstarts(sym, BTF_ID_FUNC_PFX))
+ return NULL;
+
+ /* Strip prefix and handle malformed input such as __BTF_ID__func___ */
+ func = strdup(sym + sizeof(BTF_ID_FUNC_PFX) - 1);
+ if (strlen(func) < 2) {
+ free(func);
+ return NULL;
+ }
+
+ /* Strip suffix */
+ end = strrchr(func, '_');
+ if (!end || *(end - 1) != '_') {
+ free(func);
+ return NULL;
+ }
+ *(end - 1) = '\0';
+
+ return func;
+}
+
+static int btf_func_cmp(const void *_a, const void *_b)
+{
+ const struct btf_func *a = _a;
+ const struct btf_func *b = _b;
+
+ return strcmp(a->name, b->name);
+}
+
+/*
+ * Collects all functions described in BTF.
+ * Returns non-zero on error.
+ */
+static int btf_encoder__collect_btf_funcs(struct btf_encoder *encoder, struct gobuffer *funcs)
+{
+ struct btf *btf = encoder->btf;
+ int nr_types, type_id;
+ int err = -1;
+
+ /* First collect all the func entries into an array */
+ nr_types = btf__type_cnt(btf);
+ for (type_id = 1; type_id < nr_types; type_id++) {
+ const struct btf_type *type;
+ struct btf_func func = {};
+ const char *name;
+
+ type = btf__type_by_id(btf, type_id);
+ if (!type) {
+ fprintf(stderr, "%s: malformed BTF, can't resolve type for ID %d\n",
+ __func__, type_id);
+ err = -EINVAL;
+ goto out;
+ }
+
+ if (!btf_is_func(type))
+ continue;
+
+ name = btf__name_by_offset(btf, type->name_off);
+ if (!name) {
+ fprintf(stderr, "%s: malformed BTF, can't resolve name for ID %d\n",
+ __func__, type_id);
+ err = -EINVAL;
+ goto out;
+ }
+
+ func.name = name;
+ func.type_id = type_id;
+ err = gobuffer__add(funcs, &func, sizeof(func));
+ if (err < 0)
+ goto out;
+ }
+
+ /* Now that we've collected funcs, sort them by name */
+ gobuffer__sort(funcs, sizeof(struct btf_func), btf_func_cmp);
+
+ err = 0;
+out:
+ return err;
+}
+
+static int btf_encoder__tag_kfunc(struct btf_encoder *encoder, struct gobuffer *funcs, const char *kfunc)
+{
+ struct btf_func key = { .name = kfunc };
+ struct btf *btf = encoder->btf;
+ struct btf_func *target;
+ const void *base;
+ unsigned int cnt;
+ int err = -1;
+
+ base = gobuffer__entries(funcs);
+ cnt = gobuffer__nr_entries(funcs);
+ target = bsearch(&key, base, cnt, sizeof(key), btf_func_cmp);
+ if (!target) {
+ fprintf(stderr, "%s: failed to find kfunc '%s' in BTF\n", __func__, kfunc);
+ goto out;
+ }
+
+ /* Note we are unconditionally adding the btf_decl_tag even
+ * though vmlinux may already contain btf_decl_tags for kfuncs.
+ * We are ok to do this b/c we will later btf__dedup() to remove
+ * any duplicates.
+ */
+ err = btf__add_decl_tag(btf, BTF_KFUNC_TYPE_TAG, target->type_id, -1);
+ if (err < 0) {
+ fprintf(stderr, "%s: failed to insert kfunc decl tag for '%s': %d\n",
+ __func__, kfunc, err);
+ goto out;
+ }
+
+ err = 0;
+out:
+ return err;
+}
+
+static int btf_encoder__tag_kfuncs(struct btf_encoder *encoder)
+{
+ const char *filename = encoder->source_filename;
+ struct gobuffer btf_kfunc_ranges = {};
+ struct gobuffer btf_funcs = {};
+ Elf_Data *symbols = NULL;
+ Elf_Data *idlist = NULL;
+ Elf_Scn *symscn = NULL;
+ int symbols_shndx = -1;
+ size_t idlist_addr = 0;
+ int fd = -1, err = -1;
+ int idlist_shndx = -1;
+ size_t strtabidx = 0;
+ Elf_Scn *scn = NULL;
+ Elf *elf = NULL;
+ GElf_Shdr shdr;
+ size_t strndx;
+ char *secname;
+ int nr_syms;
+ int i = 0;
+
+ fd = open(filename, O_RDONLY);
+ if (fd < 0) {
+ fprintf(stderr, "Cannot open %s\n", filename);
+ goto out;
+ }
+
+ if (elf_version(EV_CURRENT) == EV_NONE) {
+ elf_error("Cannot set libelf version");
+ goto out;
+ }
+
+ elf = elf_begin(fd, ELF_C_READ, NULL);
+ if (elf == NULL) {
+ elf_error("Cannot update ELF file");
+ goto out;
+ }
+
+ /* Locate symbol table and .BTF_ids sections */
+ if (elf_getshdrstrndx(elf, &strndx) < 0)
+ goto out;
+
+ while ((scn = elf_nextscn(elf, scn)) != NULL) {
+ Elf_Data *data;
+
+ i++;
+ if (!gelf_getshdr(scn, &shdr)) {
+ elf_error("Failed to get ELF section(%d) hdr", i);
+ goto out;
+ }
+
+ secname = elf_strptr(elf, strndx, shdr.sh_name);
+ if (!secname) {
+ elf_error("Failed to get ELF section(%d) hdr name", i);
+ goto out;
+ }
+
+ data = elf_getdata(scn, 0);
+ if (!data) {
+ elf_error("Failed to get ELF section(%d) data", i);
+ goto out;
+ }
+
+ if (shdr.sh_type == SHT_SYMTAB) {
+ symbols_shndx = i;
+ symscn = scn;
+ symbols = data;
+ strtabidx = shdr.sh_link;
+ } else if (!strcmp(secname, BTF_IDS_SECTION)) {
+ idlist_shndx = i;
+ idlist_addr = shdr.sh_addr;
+ idlist = data;
+ }
+ }
+
+ /* Cannot resolve symbol or .BTF_ids sections. Nothing to do. */
+ if (symbols_shndx == -1 || idlist_shndx == -1) {
+ err = 0;
+ goto out;
+ }
+
+ if (!gelf_getshdr(symscn, &shdr)) {
+ elf_error("Failed to get ELF symbol table header");
+ goto out;
+ }
+ nr_syms = shdr.sh_size / shdr.sh_entsize;
+
+ err = btf_encoder__collect_btf_funcs(encoder, &btf_funcs);
+ if (err) {
+ fprintf(stderr, "%s: failed to collect BTF funcs\n", __func__);
+ goto out;
+ }
+
+ /* First collect all kfunc set ranges.
+ *
+ * Note we choose not to sort these ranges and accept a linear
+ * search when doing lookups. Reasoning is that the number of
+ * sets is ~O(100) and not worth the additional code to optimize.
+ */
+ for (i = 0; i < nr_syms; i++) {
+ struct btf_kfunc_set_range range = {};
+ const char *name;
+ GElf_Sym sym;
+
+ if (!gelf_getsym(symbols, i, &sym)) {
+ elf_error("Failed to get ELF symbol(%d)", i);
+ goto out;
+ }
+
+ if (sym.st_shndx != idlist_shndx)
+ continue;
+
+ name = elf_strptr(elf, strtabidx, sym.st_name);
+ if (!is_sym_kfunc_set(&sym, name, idlist, idlist_addr))
+ continue;
+
+ range.start = sym.st_value;
+ range.end = sym.st_value + sym.st_size;
+ gobuffer__add(&btf_kfunc_ranges, &range, sizeof(range));
+ }
+
+ /* Now inject BTF with kfunc decl tag for detected kfuncs */
+ for (i = 0; i < nr_syms; i++) {
+ const struct btf_kfunc_set_range *ranges;
+ unsigned int ranges_cnt;
+ char *func, *name;
+ GElf_Sym sym;
+ bool found;
+ int err;
+ int j;
+
+ if (!gelf_getsym(symbols, i, &sym)) {
+ elf_error("Failed to get ELF symbol(%d)", i);
+ goto out;
+ }
+
+ if (sym.st_shndx != idlist_shndx)
+ continue;
+
+ name = elf_strptr(elf, strtabidx, sym.st_name);
+ func = get_func_name(name);
+ if (!func)
+ continue;
+
+ /* Check if function belongs to a kfunc set */
+ ranges = gobuffer__entries(&btf_kfunc_ranges);
+ ranges_cnt = gobuffer__nr_entries(&btf_kfunc_ranges);
+ found = false;
+ for (j = 0; j < ranges_cnt; j++) {
+ size_t addr = sym.st_value;
+
+ if (ranges[j].start <= addr && addr < ranges[j].end) {
+ found = true;
+ break;
+ }
+ }
+ if (!found) {
+ free(func);
+ continue;
+ }
+
+ err = btf_encoder__tag_kfunc(encoder, &btf_funcs, func);
+ if (err) {
+ fprintf(stderr, "%s: failed to tag kfunc '%s'\n", __func__, func);
+ free(func);
+ goto out;
+ }
+ free(func);
+ }
+
+ err = 0;
+out:
+ __gobuffer__delete(&btf_funcs);
+ __gobuffer__delete(&btf_kfunc_ranges);
+ if (elf)
+ elf_end(elf);
+ if (fd != -1)
+ close(fd);
+ return err;
+}
+
int btf_encoder__encode(struct btf_encoder *encoder)
{
+ bool should_tag_kfuncs;
int err;
/* for single-threaded case, saved funcs are added here */
@@ -1378,6 +1740,15 @@ int btf_encoder__encode(struct btf_encoder *encoder)
if (btf__type_cnt(encoder->btf) == 1)
return 0;
+ /* Note vmlinux may already contain btf_decl_tag's for kfuncs. So
+ * take care to call this before btf_dedup().
+ */
+ should_tag_kfuncs = encoder->tag_kfuncs && !encoder->skip_encoding_decl_tag;
+ if (should_tag_kfuncs && btf_encoder__tag_kfuncs(encoder)) {
+ fprintf(stderr, "%s: failed to tag kfuncs!\n", __func__);
+ return -1;
+ }
+
if (btf__dedup(encoder->btf, NULL)) {
fprintf(stderr, "%s: btf__dedup failed!\n", __func__);
return -1;
@@ -1662,6 +2033,7 @@ struct btf_encoder *btf_encoder__new(struct cu *cu, const char *detached_filenam
encoder->force = conf_load->btf_encode_force;
encoder->gen_floats = conf_load->btf_gen_floats;
encoder->skip_encoding_vars = conf_load->skip_encoding_btf_vars;
+ encoder->skip_encoding_decl_tag = conf_load->skip_encoding_btf_decl_tag;
encoder->tag_kfuncs = conf_load->btf_decl_tag_kfuncs;
encoder->verbose = verbose;
encoder->has_index_type = false;
This commit teaches pahole to parse symbols in .BTF_ids section in vmlinux and discover exported kfuncs. Pahole then takes the list of kfuncs and injects a BTF_KIND_DECL_TAG for each kfunc. Example of encoding: $ bpftool btf dump file .tmp_vmlinux.btf | rg "DECL_TAG 'bpf_kfunc'" | wc -l 121 $ bpftool btf dump file .tmp_vmlinux.btf | rg 56337 [56337] FUNC 'bpf_ct_change_timeout' type_id=56336 linkage=static [127861] DECL_TAG 'bpf_kfunc' type_id=56337 component_idx=-1 This enables downstream users and tools to dynamically discover which kfuncs are available on a system by parsing vmlinux or module BTF, both available in /sys/kernel/btf. This feature is enabled with --btf_features=decl_tag,decl_tag_kfuncs. Acked-by: Jiri Olsa <jolsa@kernel.org> Tested-by: Jiri Olsa <jolsa@kernel.org> Reviewed-by: Alan Maguire <alan.maguire@oracle.com> Tested-by: Alan Maguire <alan.maguire@oracle.com> Signed-off-by: Daniel Xu <dxu@dxuuu.xyz> --- btf_encoder.c | 372 ++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 372 insertions(+)