@@ -219,7 +219,7 @@ bool btf_member_is_reg_int(const struct btf *btf, const struct btf_type *s,
u32 expected_offset, u32 expected_size);
struct btf_record *btf_parse_fields(const struct btf *btf, const struct btf_type *t,
u32 field_mask, u32 value_size);
-int btf_check_and_fixup_fields(const struct btf *btf, struct btf_record *rec);
+int btf_check_and_fixup_fields(struct btf_record *rec);
bool btf_type_is_void(const struct btf_type *t);
s32 btf_find_by_name_kind(const struct btf *btf, const char *name, u8 kind);
s32 bpf_find_btf_id(const char *name, u32 kind, struct btf **btf_p);
@@ -569,7 +569,7 @@ static inline int register_btf_id_dtor_kfuncs(const struct btf_id_dtor_kfunc *dt
{
return 0;
}
-static inline struct btf_struct_meta *btf_find_struct_meta(const struct btf *btf, u32 btf_id)
+static inline struct btf_struct_meta *btf_find_struct_meta(u32 btf_id)
{
return NULL;
}
@@ -3296,6 +3296,7 @@ struct btf_field_info {
struct {
const char *node_name;
u32 value_btf_id;
+ const struct btf *btf;
} graph_root;
};
};
@@ -3405,7 +3406,9 @@ btf_find_graph_root(const struct btf *btf, const struct btf_type *pt,
enum btf_field_type head_type)
{
const char *node_field_name;
+ bool value_is_kptr = false;
const char *value_type;
+ struct btf *kptr_btf;
s32 id;
if (!__btf_type_is_struct(t))
@@ -3413,15 +3416,26 @@ btf_find_graph_root(const struct btf *btf, const struct btf_type *pt,
if (t->size != sz)
return BTF_FIELD_IGNORE;
value_type = btf_find_decl_tag_value(btf, pt, comp_idx, "contains:");
- if (IS_ERR(value_type))
- return -EINVAL;
+ if (!IS_ERR(value_type))
+ goto found;
+ value_type = btf_find_decl_tag_value(btf, pt, comp_idx, "contains_kptr:");
+ if (!IS_ERR(value_type)) {
+ value_is_kptr = true;
+ goto found;
+ }
+ return -EINVAL;
+found:
node_field_name = strstr(value_type, ":");
if (!node_field_name)
return -EINVAL;
value_type = kstrndup(value_type, node_field_name - value_type, GFP_KERNEL | __GFP_NOWARN);
if (!value_type)
return -ENOMEM;
- id = btf_find_by_name_kind(btf, value_type, BTF_KIND_STRUCT);
+ if (value_is_kptr)
+ id = bpf_find_btf_id(value_type, BTF_KIND_STRUCT, &kptr_btf);
+ else
+ id = btf_find_by_name_kind(btf, value_type, BTF_KIND_STRUCT);
+
kfree(value_type);
if (id < 0)
return id;
@@ -3431,6 +3445,7 @@ btf_find_graph_root(const struct btf *btf, const struct btf_type *pt,
info->type = head_type;
info->off = off;
info->graph_root.value_btf_id = id;
+ info->graph_root.btf = value_is_kptr ? kptr_btf : btf;
info->graph_root.node_name = node_field_name;
return BTF_FIELD_FOUND;
}
@@ -3722,13 +3737,13 @@ static int btf_parse_kptr(const struct btf *btf, struct btf_field *field,
return ret;
}
-static int btf_parse_graph_root(const struct btf *btf,
- struct btf_field *field,
+static int btf_parse_graph_root(struct btf_field *field,
struct btf_field_info *info,
const char *node_type_name,
size_t node_type_align)
{
const struct btf_type *t, *n = NULL;
+ const struct btf *btf = info->graph_root.btf;
const struct btf_member *member;
u32 offset;
int i;
@@ -3766,18 +3781,16 @@ static int btf_parse_graph_root(const struct btf *btf,
return 0;
}
-static int btf_parse_list_head(const struct btf *btf, struct btf_field *field,
- struct btf_field_info *info)
+static int btf_parse_list_head(struct btf_field *field, struct btf_field_info *info)
{
- return btf_parse_graph_root(btf, field, info, "bpf_list_node",
- __alignof__(struct bpf_list_node));
+ return btf_parse_graph_root(field, info, "bpf_list_node",
+ __alignof__(struct bpf_list_node));
}
-static int btf_parse_rb_root(const struct btf *btf, struct btf_field *field,
- struct btf_field_info *info)
+static int btf_parse_rb_root(struct btf_field *field, struct btf_field_info *info)
{
- return btf_parse_graph_root(btf, field, info, "bpf_rb_node",
- __alignof__(struct bpf_rb_node));
+ return btf_parse_graph_root(field, info, "bpf_rb_node",
+ __alignof__(struct bpf_rb_node));
}
static int btf_field_cmp(const void *_a, const void *_b, const void *priv)
@@ -3859,12 +3872,12 @@ struct btf_record *btf_parse_fields(const struct btf *btf, const struct btf_type
goto end;
break;
case BPF_LIST_HEAD:
- ret = btf_parse_list_head(btf, &rec->fields[i], &info_arr[i]);
+ ret = btf_parse_list_head(&rec->fields[i], &info_arr[i]);
if (ret < 0)
goto end;
break;
case BPF_RB_ROOT:
- ret = btf_parse_rb_root(btf, &rec->fields[i], &info_arr[i]);
+ ret = btf_parse_rb_root(&rec->fields[i], &info_arr[i]);
if (ret < 0)
goto end;
break;
@@ -3901,7 +3914,7 @@ struct btf_record *btf_parse_fields(const struct btf *btf, const struct btf_type
return ERR_PTR(ret);
}
-int btf_check_and_fixup_fields(const struct btf *btf, struct btf_record *rec)
+int btf_check_and_fixup_fields(struct btf_record *rec)
{
int i;
@@ -3917,11 +3930,13 @@ int btf_check_and_fixup_fields(const struct btf *btf, struct btf_record *rec)
return 0;
for (i = 0; i < rec->cnt; i++) {
struct btf_struct_meta *meta;
+ struct btf *btf;
u32 btf_id;
if (!(rec->fields[i].type & BPF_GRAPH_ROOT))
continue;
btf_id = rec->fields[i].graph_root.value_btf_id;
+ btf = rec->fields[i].graph_root.btf;
meta = btf_find_struct_meta(btf, btf_id);
if (!meta)
return -EFAULT;
@@ -5630,7 +5645,7 @@ static struct btf *btf_parse(const union bpf_attr *attr, bpfptr_t uattr, u32 uat
int i;
for (i = 0; i < struct_meta_tab->cnt; i++) {
- err = btf_check_and_fixup_fields(btf, struct_meta_tab->types[i].record);
+ err = btf_check_and_fixup_fields(struct_meta_tab->types[i].record);
if (err < 0)
goto errout_meta;
}
@@ -1157,7 +1157,7 @@ static int map_check_btf(struct bpf_map *map, struct bpf_token *token,
}
}
- ret = btf_check_and_fixup_fields(btf, map->record);
+ ret = btf_check_and_fixup_fields(map->record);
if (ret < 0)
goto free_map_tab;
@@ -7,6 +7,7 @@
#include <bpf/bpf_core_read.h>
#define __contains(name, node) __attribute__((btf_decl_tag("contains:" #name ":" #node)))
+#define __contains_kptr(name, node) __attribute__((btf_decl_tag("contains_kptr:" #name ":" #node)))
/* Description
* Allocates an object of the type represented by 'local_type_id' in
This patch teaches bpf to recognize graphs that contain kernel objects as graph values. BPF programs can use a new BTF declaration tag "contain_kptr" to signal that the value of a graph will be a kernel type. "contains_kptr" follows the same annotation format as "contains". For the implementation, when the value is a kernel type, we uses kernel BTF for node and roots as well so that we don't need to match the same type in different BTF. Since graph values can be kernel types, we can no longer make the assumption that the BTF is from programs when finding and parsing graph nodes and roots. Therefore, we record the BTF of a node in btf_field_info and use it later. No kernel object can be added to bpf graphs yet. In later patches, we will teach the verifier to allow moving kptr in and out collections. Signed-off-by: Amery Hung <amery.hung@bytedance.com> --- include/linux/btf.h | 4 +- kernel/bpf/btf.c | 49 ++++++++++++------- kernel/bpf/syscall.c | 2 +- .../testing/selftests/bpf/bpf_experimental.h | 1 + 4 files changed, 36 insertions(+), 20 deletions(-)