@@ -1638,7 +1638,6 @@ void *bpf_map_container_alloc(u64 size, int numa_node);
void *bpf_map_container_mmapable_alloc(u64 size, int numa_node,
u32 align, u32 offset);
void *bpf_map_area_alloc(u64 size, int numa_node);
-void *bpf_map_area_mmapable_alloc(u64 size, int numa_node);
void bpf_map_area_free(void *base);
void bpf_map_container_free(void *base);
bool bpf_map_write_active(const struct bpf_map *map);
@@ -126,16 +126,18 @@ static struct bpf_map *array_map_alloc(union bpf_attr *attr)
/* allocate all map elements and zero-initialize them */
if (attr->map_flags & BPF_F_MMAPABLE) {
+ u32 align = PAGE_ALIGN(sizeof(struct bpf_array));
+ u32 offset = offsetof(struct bpf_array, value);
void *data;
/* kmalloc'ed memory can't be mmap'ed, use explicit vmalloc */
- data = bpf_map_area_mmapable_alloc(array_size, numa_node);
+ data = bpf_map_container_mmapable_alloc(array_size, numa_node,
+ align, offset);
if (!data)
return ERR_PTR(-ENOMEM);
- array = data + PAGE_ALIGN(sizeof(struct bpf_array))
- - offsetof(struct bpf_array, value);
+ array = data + align - offset;
} else {
- array = bpf_map_area_alloc(array_size, numa_node);
+ array = bpf_map_container_alloc(array_size, numa_node);
}
if (!array)
return ERR_PTR(-ENOMEM);
@@ -147,7 +149,7 @@ static struct bpf_map *array_map_alloc(union bpf_attr *attr)
array->elem_size = elem_size;
if (percpu && bpf_array_alloc_percpu(array)) {
- bpf_map_area_free(array);
+ bpf_map_container_free(array);
return ERR_PTR(-ENOMEM);
}
@@ -420,6 +422,7 @@ static void array_map_free(struct bpf_map *map)
struct bpf_array *array = container_of(map, struct bpf_array, map);
int i;
+
if (map_value_has_kptrs(map)) {
for (i = 0; i < array->map.max_entries; i++)
bpf_map_free_kptrs(map, array_map_elem_ptr(array, i));
@@ -430,9 +433,9 @@ static void array_map_free(struct bpf_map *map)
bpf_array_free_percpu(array);
if (array->map.map_flags & BPF_F_MMAPABLE)
- bpf_map_area_free(array_map_vmalloc_addr(array));
+ bpf_map_container_free(array_map_vmalloc_addr(array));
else
- bpf_map_area_free(array);
+ bpf_map_container_free(array);
}
static void array_map_seq_show_elem(struct bpf_map *map, void *key,
@@ -774,7 +777,7 @@ static void fd_array_map_free(struct bpf_map *map)
for (i = 0; i < array->map.max_entries; i++)
BUG_ON(array->ptrs[i] != NULL);
- bpf_map_area_free(array);
+ bpf_map_container_free(array);
}
static void *fd_array_map_lookup_elem(struct bpf_map *map, void *key)
@@ -142,7 +142,7 @@ static struct bpf_map *bloom_map_alloc(union bpf_attr *attr)
}
bitset_bytes = roundup(bitset_bytes, sizeof(unsigned long));
- bloom = bpf_map_area_alloc(sizeof(*bloom) + bitset_bytes, numa_node);
+ bloom = bpf_map_container_alloc(sizeof(*bloom) + bitset_bytes, numa_node);
if (!bloom)
return ERR_PTR(-ENOMEM);
@@ -168,7 +168,7 @@ static void bloom_map_free(struct bpf_map *map)
struct bpf_bloom_filter *bloom =
container_of(map, struct bpf_bloom_filter, map);
- bpf_map_area_free(bloom);
+ bpf_map_container_free(bloom);
}
static void *bloom_map_lookup_elem(struct bpf_map *map, void *key)
@@ -582,7 +582,7 @@ void bpf_local_storage_map_free(struct bpf_local_storage_map *smap,
synchronize_rcu();
kvfree(smap->buckets);
- bpf_map_area_free(smap);
+ bpf_map_container_free(smap);
}
int bpf_local_storage_map_alloc_check(union bpf_attr *attr)
@@ -610,7 +610,7 @@ struct bpf_local_storage_map *bpf_local_storage_map_alloc(union bpf_attr *attr)
unsigned int i;
u32 nbuckets;
- smap = bpf_map_area_alloc(sizeof(*smap), NUMA_NO_NODE);
+ smap = bpf_map_container_alloc(sizeof(*smap), NUMA_NO_NODE);
if (!smap)
return ERR_PTR(-ENOMEM);
bpf_map_init_from_attr(&smap->map, attr);
@@ -623,7 +623,7 @@ struct bpf_local_storage_map *bpf_local_storage_map_alloc(union bpf_attr *attr)
smap->buckets = kvcalloc(sizeof(*smap->buckets), nbuckets,
GFP_USER | __GFP_NOWARN | __GFP_ACCOUNT);
if (!smap->buckets) {
- bpf_map_area_free(smap);
+ bpf_map_container_free(smap);
return ERR_PTR(-ENOMEM);
}
@@ -580,7 +580,7 @@ static void bpf_struct_ops_map_free(struct bpf_map *map)
bpf_map_area_free(st_map->links);
bpf_jit_free_exec(st_map->image);
bpf_map_area_free(st_map->uvalue);
- bpf_map_area_free(st_map);
+ bpf_map_container_free(st_map);
}
static int bpf_struct_ops_map_alloc_check(union bpf_attr *attr)
@@ -618,7 +618,7 @@ static struct bpf_map *bpf_struct_ops_map_alloc(union bpf_attr *attr)
*/
(vt->size - sizeof(struct bpf_struct_ops_value));
- st_map = bpf_map_area_alloc(st_map_size, NUMA_NO_NODE);
+ st_map = bpf_map_container_alloc(st_map_size, NUMA_NO_NODE);
if (!st_map)
return ERR_PTR(-ENOMEM);
@@ -97,7 +97,7 @@ static struct bpf_map *cpu_map_alloc(union bpf_attr *attr)
attr->map_flags & ~BPF_F_NUMA_NODE)
return ERR_PTR(-EINVAL);
- cmap = bpf_map_area_alloc(sizeof(*cmap), NUMA_NO_NODE);
+ cmap = bpf_map_container_alloc(sizeof(*cmap), NUMA_NO_NODE);
if (!cmap)
return ERR_PTR(-ENOMEM);
@@ -118,7 +118,7 @@ static struct bpf_map *cpu_map_alloc(union bpf_attr *attr)
return &cmap->map;
free_cmap:
- bpf_map_area_free(cmap);
+ bpf_map_container_free(cmap);
return ERR_PTR(err);
}
@@ -623,7 +623,7 @@ static void cpu_map_free(struct bpf_map *map)
__cpu_map_entry_replace(cmap, i, NULL); /* call_rcu */
}
bpf_map_area_free(cmap->cpu_map);
- bpf_map_area_free(cmap);
+ bpf_map_container_free(cmap);
}
/* Elements are kept alive by RCU; either by rcu_read_lock() (from syscall) or
@@ -163,13 +163,13 @@ static struct bpf_map *dev_map_alloc(union bpf_attr *attr)
if (!capable(CAP_NET_ADMIN))
return ERR_PTR(-EPERM);
- dtab = bpf_map_area_alloc(sizeof(*dtab), NUMA_NO_NODE);
+ dtab = bpf_map_container_alloc(sizeof(*dtab), NUMA_NO_NODE);
if (!dtab)
return ERR_PTR(-ENOMEM);
err = dev_map_init_map(dtab, attr);
if (err) {
- bpf_map_area_free(dtab);
+ bpf_map_container_free(dtab);
return ERR_PTR(err);
}
@@ -240,7 +240,7 @@ static void dev_map_free(struct bpf_map *map)
bpf_map_area_free(dtab->netdev_map);
}
- bpf_map_area_free(dtab);
+ bpf_map_container_free(dtab);
}
static int dev_map_get_next_key(struct bpf_map *map, void *key, void *next_key)
@@ -495,7 +495,7 @@ static struct bpf_map *htab_map_alloc(union bpf_attr *attr)
struct bpf_htab *htab;
int err, i;
- htab = bpf_map_area_alloc(sizeof(*htab), NUMA_NO_NODE);
+ htab = bpf_map_container_alloc(sizeof(*htab), NUMA_NO_NODE);
if (!htab)
return ERR_PTR(-ENOMEM);
@@ -579,7 +579,7 @@ static struct bpf_map *htab_map_alloc(union bpf_attr *attr)
bpf_map_area_free(htab->buckets);
free_htab:
lockdep_unregister_key(&htab->lockdep_key);
- bpf_map_area_free(htab);
+ bpf_map_container_free(htab);
return ERR_PTR(err);
}
@@ -1496,7 +1496,7 @@ static void htab_map_free(struct bpf_map *map)
for (i = 0; i < HASHTAB_MAP_LOCK_COUNT; i++)
free_percpu(htab->map_locked[i]);
lockdep_unregister_key(&htab->lockdep_key);
- bpf_map_area_free(htab);
+ bpf_map_container_free(htab);
}
static void htab_map_seq_show_elem(struct bpf_map *map, void *key,
@@ -313,7 +313,7 @@ static struct bpf_map *cgroup_storage_map_alloc(union bpf_attr *attr)
/* max_entries is not used and enforced to be 0 */
return ERR_PTR(-EINVAL);
- map = bpf_map_area_alloc(sizeof(struct bpf_cgroup_storage_map), numa_node);
+ map = bpf_map_container_alloc(sizeof(struct bpf_cgroup_storage_map), numa_node);
if (!map)
return ERR_PTR(-ENOMEM);
@@ -345,7 +345,7 @@ static void cgroup_storage_map_free(struct bpf_map *_map)
WARN_ON(!RB_EMPTY_ROOT(&map->root));
WARN_ON(!list_empty(&map->list));
- bpf_map_area_free(map);
+ bpf_map_container_free(map);
}
static int cgroup_storage_delete_elem(struct bpf_map *map, void *key)
@@ -558,7 +558,7 @@ static struct bpf_map *trie_alloc(union bpf_attr *attr)
attr->value_size > LPM_VAL_SIZE_MAX)
return ERR_PTR(-EINVAL);
- trie = bpf_map_area_alloc(sizeof(*trie), NUMA_NO_NODE);
+ trie = bpf_map_container_alloc(sizeof(*trie), NUMA_NO_NODE);
if (!trie)
return ERR_PTR(-ENOMEM);
@@ -609,7 +609,7 @@ static void trie_free(struct bpf_map *map)
}
out:
- bpf_map_area_free(trie);
+ bpf_map_container_free(trie);
}
static int trie_get_next_key(struct bpf_map *map, void *_key, void *_next_key)
@@ -372,7 +372,7 @@ struct bpf_map *bpf_map_offload_map_alloc(union bpf_attr *attr)
attr->map_type != BPF_MAP_TYPE_HASH)
return ERR_PTR(-EINVAL);
- offmap = kzalloc(sizeof(*offmap), GFP_USER | __GFP_NOWARN);
+ offmap = bpf_map_container_alloc(sizeof(*offmap), NUMA_NO_NODE);
if (!offmap)
return ERR_PTR(-ENOMEM);
@@ -74,7 +74,7 @@ static struct bpf_map *queue_stack_map_alloc(union bpf_attr *attr)
size = (u64) attr->max_entries + 1;
queue_size = sizeof(*qs) + size * attr->value_size;
- qs = bpf_map_area_alloc(queue_size, numa_node);
+ qs = bpf_map_container_alloc(queue_size, numa_node);
if (!qs)
return ERR_PTR(-ENOMEM);
@@ -92,7 +92,7 @@ static void queue_stack_map_free(struct bpf_map *map)
{
struct bpf_queue_stack *qs = bpf_queue_stack(map);
- bpf_map_area_free(qs);
+ bpf_map_container_free(qs);
}
static int __queue_map_get(struct bpf_map *map, void *value, bool delete)
@@ -146,7 +146,7 @@ static void reuseport_array_free(struct bpf_map *map)
* Once reaching here, all sk->sk_user_data is not
* referencing this "array". "array" can be freed now.
*/
- bpf_map_area_free(array);
+ bpf_map_container_free(array);
}
static struct bpf_map *reuseport_array_alloc(union bpf_attr *attr)
@@ -158,7 +158,7 @@ static struct bpf_map *reuseport_array_alloc(union bpf_attr *attr)
return ERR_PTR(-EPERM);
/* allocate all map elements and zero-initialize them */
- array = bpf_map_area_alloc(struct_size(array, ptrs, attr->max_entries), numa_node);
+ array = bpf_map_container_alloc(struct_size(array, ptrs, attr->max_entries), numa_node);
if (!array)
return ERR_PTR(-ENOMEM);
@@ -164,7 +164,7 @@ static struct bpf_map *ringbuf_map_alloc(union bpf_attr *attr)
return ERR_PTR(-E2BIG);
#endif
- rb_map = bpf_map_area_alloc(sizeof(*rb_map), NUMA_NO_NODE);
+ rb_map = bpf_map_container_alloc(sizeof(*rb_map), NUMA_NO_NODE);
if (!rb_map)
return ERR_PTR(-ENOMEM);
@@ -172,7 +172,7 @@ static struct bpf_map *ringbuf_map_alloc(union bpf_attr *attr)
rb_map->rb = bpf_ringbuf_alloc(attr->max_entries, rb_map->map.numa_node);
if (!rb_map->rb) {
- bpf_map_area_free(rb_map);
+ bpf_map_container_free(rb_map);
return ERR_PTR(-ENOMEM);
}
@@ -199,7 +199,7 @@ static void ringbuf_map_free(struct bpf_map *map)
rb_map = container_of(map, struct bpf_ringbuf_map, map);
bpf_ringbuf_free(rb_map->rb);
- bpf_map_area_free(rb_map);
+ bpf_map_container_free(rb_map);
}
static void *ringbuf_map_lookup_elem(struct bpf_map *map, void *key)
@@ -100,7 +100,7 @@ static struct bpf_map *stack_map_alloc(union bpf_attr *attr)
return ERR_PTR(-E2BIG);
cost = n_buckets * sizeof(struct stack_map_bucket *) + sizeof(*smap);
- smap = bpf_map_area_alloc(cost, bpf_map_attr_numa_node(attr));
+ smap = bpf_map_container_alloc(cost, bpf_map_attr_numa_node(attr));
if (!smap)
return ERR_PTR(-ENOMEM);
@@ -120,7 +120,7 @@ static struct bpf_map *stack_map_alloc(union bpf_attr *attr)
put_buffers:
put_callchain_buffers();
free_smap:
- bpf_map_area_free(smap);
+ bpf_map_container_free(smap);
return ERR_PTR(err);
}
@@ -650,7 +650,7 @@ static void stack_map_free(struct bpf_map *map)
bpf_map_area_free(smap->elems);
pcpu_freelist_destroy(&smap->freelist);
- bpf_map_area_free(smap);
+ bpf_map_container_free(smap);
put_callchain_buffers();
}
@@ -339,11 +339,6 @@ void *bpf_map_area_alloc(u64 size, int numa_node)
return __bpf_map_area_alloc(size, numa_node, false);
}
-void *bpf_map_area_mmapable_alloc(u64 size, int numa_node)
-{
- return __bpf_map_area_alloc(size, numa_node, true);
-}
-
void bpf_map_area_free(void *area)
{
kvfree(area);
@@ -669,7 +664,6 @@ static void bpf_map_free_deferred(struct work_struct *work)
security_bpf_map_free(map);
kfree(map->off_arr);
- bpf_map_release_memcg(map);
/* implementation dependent freeing, map_free callback also does
* bpf_map_free_kptr_off_tab, if needed.
*/
@@ -1218,8 +1212,6 @@ static int map_create(union bpf_attr *attr)
if (err)
goto free_map_sec;
- bpf_map_save_memcg(map);
-
err = bpf_map_new_fd(map, f_flags);
if (err < 0) {
/* failed to allocate fd.
@@ -41,7 +41,7 @@ static struct bpf_map *sock_map_alloc(union bpf_attr *attr)
attr->map_flags & ~SOCK_CREATE_FLAG_MASK)
return ERR_PTR(-EINVAL);
- stab = bpf_map_area_alloc(sizeof(*stab), NUMA_NO_NODE);
+ stab = bpf_map_container_alloc(sizeof(*stab), NUMA_NO_NODE);
if (!stab)
return ERR_PTR(-ENOMEM);
@@ -52,7 +52,7 @@ static struct bpf_map *sock_map_alloc(union bpf_attr *attr)
sizeof(struct sock *),
stab->map.numa_node);
if (!stab->sks) {
- bpf_map_area_free(stab);
+ bpf_map_container_free(stab);
return ERR_PTR(-ENOMEM);
}
@@ -361,7 +361,7 @@ static void sock_map_free(struct bpf_map *map)
synchronize_rcu();
bpf_map_area_free(stab->sks);
- bpf_map_area_free(stab);
+ bpf_map_container_free(stab);
}
static void sock_map_release_progs(struct bpf_map *map)
@@ -1076,7 +1076,7 @@ static struct bpf_map *sock_hash_alloc(union bpf_attr *attr)
if (attr->key_size > MAX_BPF_STACK)
return ERR_PTR(-E2BIG);
- htab = bpf_map_area_alloc(sizeof(*htab), NUMA_NO_NODE);
+ htab = bpf_map_container_alloc(sizeof(*htab), NUMA_NO_NODE);
if (!htab)
return ERR_PTR(-ENOMEM);
@@ -1106,7 +1106,7 @@ static struct bpf_map *sock_hash_alloc(union bpf_attr *attr)
return &htab->map;
free_htab:
- bpf_map_area_free(htab);
+ bpf_map_container_free(htab);
return ERR_PTR(err);
}
@@ -1159,7 +1159,7 @@ static void sock_hash_free(struct bpf_map *map)
synchronize_rcu();
bpf_map_area_free(htab->buckets);
- bpf_map_area_free(htab);
+ bpf_map_container_free(htab);
}
static void *sock_hash_lookup_sys(struct bpf_map *map, void *key)
@@ -75,7 +75,7 @@ static struct bpf_map *xsk_map_alloc(union bpf_attr *attr)
numa_node = bpf_map_attr_numa_node(attr);
size = struct_size(m, xsk_map, attr->max_entries);
- m = bpf_map_area_alloc(size, numa_node);
+ m = bpf_map_container_alloc(size, numa_node);
if (!m)
return ERR_PTR(-ENOMEM);
@@ -90,7 +90,7 @@ static void xsk_map_free(struct bpf_map *map)
struct xsk_map *m = container_of(map, struct xsk_map, map);
synchronize_net();
- bpf_map_area_free(m);
+ bpf_map_container_free(m);
}
static int xsk_map_get_next_key(struct bpf_map *map, void *key, void *next_key)
With the preparation of previous patch, now we can use the helper bpf_map_container_alloc() to allocate all bpf maps. Signed-off-by: Yafang Shao <laoar.shao@gmail.com> --- include/linux/bpf.h | 1 - kernel/bpf/arraymap.c | 19 +++++++++++-------- kernel/bpf/bloom_filter.c | 4 ++-- kernel/bpf/bpf_local_storage.c | 6 +++--- kernel/bpf/bpf_struct_ops.c | 4 ++-- kernel/bpf/cpumap.c | 6 +++--- kernel/bpf/devmap.c | 6 +++--- kernel/bpf/hashtab.c | 6 +++--- kernel/bpf/local_storage.c | 4 ++-- kernel/bpf/lpm_trie.c | 4 ++-- kernel/bpf/offload.c | 2 +- kernel/bpf/queue_stack_maps.c | 4 ++-- kernel/bpf/reuseport_array.c | 4 ++-- kernel/bpf/ringbuf.c | 6 +++--- kernel/bpf/stackmap.c | 6 +++--- kernel/bpf/syscall.c | 8 -------- net/core/sock_map.c | 12 ++++++------ net/xdp/xskmap.c | 4 ++-- 18 files changed, 50 insertions(+), 56 deletions(-)