[v2,bpf-next,09/12] bpf: Batch call_rcu callbacks instead of SLAB_TYPESAFE_BY_RCU.

Commit Message

Alexei Starovoitov Aug. 17, 2022, 9:04 p.m. UTC

From: Alexei Starovoitov <[e-mail]>

SLAB_TYPESAFE_BY_RCU makes kmem_caches non mergeable and slows down
kmem_cache_destroy. All bpf_mem_cache are safe to share across different maps
and programs. Convert SLAB_TYPESAFE_BY_RCU to batched call_rcu. This change
solves the memory consumption issue, avoids kmem_cache_destroy latency and
keeps bpf hash map performance the same.

Signed-off-by: Alexei Starovoitov <[e-mail]>
---
 kernel/bpf/memalloc.c | 58 ++++++++++++++++++++++++++++++++++++++++---
 kernel/bpf/syscall.c  |  5 +++-
 2 files changed, 59 insertions(+), 4 deletions(-)

Patch

diff --git a/kernel/bpf/memalloc.c b/kernel/bpf/memalloc.c
index be8262f5c9ec..ae4cdc9493c3 100644
--- a/kernel/bpf/memalloc.c
+++ b/kernel/bpf/memalloc.c
@@ -106,6 +106,11 @@  struct bpf_mem_cache {
 	/* flag to refill nmi list too */
 	bool refill_nmi_list;
 	int low_watermark, high_watermark, batch;
+
+	struct rcu_head rcu;
+	struct llist_head free_by_rcu;
+	struct llist_head waiting_for_gp;
+	atomic_t call_rcu_in_progress;
 };
 
 struct bpf_mem_caches {
@@ -214,6 +219,39 @@  static void free_one(struct bpf_mem_cache *c, void *obj)
 		kfree(obj);
 }
 
+static void __free_rcu(struct rcu_head *head)
+{
+	struct bpf_mem_cache *c = container_of(head, struct bpf_mem_cache, rcu);
+	struct llist_node *llnode = __llist_del_all(&c->waiting_for_gp);
+	struct llist_node *pos, *t;
+
+	llist_for_each_safe(pos, t, llnode)
+		free_one(c, pos);
+	atomic_set(&c->call_rcu_in_progress, 0);
+}
+
+static void enque_to_free(struct bpf_mem_cache *c, void *obj)
+{
+	struct llist_node *llnode = obj;
+
+	/* bpf_mem_cache is a per-cpu object. Freeing happens in irq_work.
+	 * Nothing races to add to free_by_rcu list.
+	 */
+	__llist_add(llnode, &c->free_by_rcu);
+}
+
+static void do_call_rcu(struct bpf_mem_cache *c)
+{
+	struct llist_node *llnode, *t;
+
+	if (atomic_xchg(&c->call_rcu_in_progress, 1))
+		return;
+
+	llist_for_each_safe(llnode, t, __llist_del_all(&c->free_by_rcu))
+		__llist_add(llnode, &c->waiting_for_gp);
+	call_rcu(&c->rcu, __free_rcu);
+}
+
 static void free_bulk(struct bpf_mem_cache *c)
 {
 	struct llist_node *llnode;
@@ -230,8 +268,9 @@  static void free_bulk(struct bpf_mem_cache *c)
 			cnt = 0;
 		if (IS_ENABLED(CONFIG_PREEMPT_RT))
 			local_irq_restore(flags);
-		free_one(c, llnode);
+		enque_to_free(c, llnode);
 	} while (cnt > (c->high_watermark + c->low_watermark) / 2);
+	do_call_rcu(c);
 }
 
 static void free_bulk_nmi(struct bpf_mem_cache *c)
@@ -245,8 +284,9 @@  static void free_bulk_nmi(struct bpf_mem_cache *c)
 			cnt = atomic_dec_return(&c->free_cnt_nmi);
 		else
 			cnt = 0;
-		free_one(c, llnode);
+		enque_to_free(c, llnode);
 	} while (cnt > (c->high_watermark + c->low_watermark) / 2);
+	do_call_rcu(c);
 }
 
 static void bpf_mem_refill(struct irq_work *work)
@@ -358,7 +398,7 @@  int bpf_mem_alloc_init(struct bpf_mem_alloc *ma, int size)
 			return -ENOMEM;
 		size += LLIST_NODE_SZ; /* room for llist_node */
 		snprintf(buf, sizeof(buf), "bpf-%u", size);
-		kmem_cache = kmem_cache_create(buf, size, 8, SLAB_TYPESAFE_BY_RCU, NULL);
+		kmem_cache = kmem_cache_create(buf, size, 8, 0, NULL);
 		if (!kmem_cache) {
 			free_percpu(pc);
 			return -ENOMEM;
@@ -400,6 +440,18 @@  static void drain_mem_cache(struct bpf_mem_cache *c)
 {
 	struct llist_node *llnode;
 
+	/* The caller has done rcu_barrier() and no progs are using this
+	 * bpf_mem_cache, but htab_map_free() called bpf_mem_cache_free() for
+	 * all remaining elements and they can be in free_by_rcu or in
+	 * waiting_for_gp lists, so drain accumulating free_by_rcu list and
+	 * optionally wait for callbacks to finish.
+	 */
+	while ((llnode = __llist_del_first(&c->free_by_rcu)))
+		free_one(c, llnode);
+	if (atomic_xchg(&c->call_rcu_in_progress, 1))
+		rcu_barrier();
+	WARN_ON_ONCE(!llist_empty(&c->waiting_for_gp));
+
 	while ((llnode = llist_del_first(&c->free_llist_nmi)))
 		free_one(c, llnode);
 	while ((llnode = __llist_del_first(&c->free_llist)))
diff --git a/kernel/bpf/syscall.c b/kernel/bpf/syscall.c
index 83c7136c5788..eeef64b27683 100644
--- a/kernel/bpf/syscall.c
+++ b/kernel/bpf/syscall.c
@@ -638,7 +638,10 @@  static void __bpf_map_put(struct bpf_map *map, bool do_idr_lock)
 		bpf_map_free_id(map, do_idr_lock);
 		btf_put(map->btf);
 		INIT_WORK(&map->work, bpf_map_free_deferred);
-		schedule_work(&map->work);
+		/* Avoid spawning kworkers, since they all might contend
+		 * for the same mutex like slab_mutex.
+		 */
+		queue_work(system_unbound_wq, &map->work);
 	}
 }