@@ -154,12 +154,18 @@ struct return_instance {
unsigned long stack; /* stack pointer */
unsigned long orig_ret_vaddr; /* original return address */
bool chained; /* true, if instance is nested */
- int consumers_cnt;
+ int cons_cnt; /* total number of session consumers */
struct return_instance *next; /* keep as stack */
struct rcu_head rcu;
- struct return_consumer consumers[] __counted_by(consumers_cnt);
+ /* singular pre-allocated return_consumer instance for common case */
+ struct return_consumer consumer;
+ /*
+ * extra return_consumer instances for rare cases of multiple session consumers,
+ * contains (cons_cnt - 1) elements
+ */
+ struct return_consumer *extra_consumers;
} ____cacheline_aligned;
enum rp_check {
@@ -1899,6 +1899,7 @@ static struct return_instance *free_ret_instance(struct return_instance *ri, boo
hprobe_finalize(&ri->hprobe, hstate);
}
+ kfree(ri->extra_consumers);
kfree_rcu(ri, rcu);
return next;
}
@@ -1974,32 +1975,34 @@ static struct uprobe_task *get_utask(void)
return current->utask;
}
-static size_t ri_size(int consumers_cnt)
-{
- struct return_instance *ri;
-
- return sizeof(*ri) + sizeof(ri->consumers[0]) * consumers_cnt;
-}
-
-#define DEF_CNT 4
-
static struct return_instance *alloc_return_instance(void)
{
struct return_instance *ri;
- ri = kzalloc(ri_size(DEF_CNT), GFP_KERNEL);
+ ri = kzalloc(sizeof(*ri), GFP_KERNEL);
if (!ri)
return ZERO_SIZE_PTR;
- ri->consumers_cnt = DEF_CNT;
return ri;
}
static struct return_instance *dup_return_instance(struct return_instance *old)
{
- size_t size = ri_size(old->consumers_cnt);
+ struct return_instance *ri;
+
+ ri = kmemdup(old, sizeof(*ri), GFP_KERNEL);
+
+ if (unlikely(old->cons_cnt > 1)) {
+ ri->extra_consumers = kmemdup(old->extra_consumers,
+ sizeof(ri->extra_consumers[0]) * (old->cons_cnt - 1),
+ GFP_KERNEL);
+ if (!ri->extra_consumers) {
+ kfree(ri);
+ return NULL;
+ }
+ }
- return kmemdup(old, size, GFP_KERNEL);
+ return ri;
}
static int dup_utask(struct task_struct *t, struct uprobe_task *o_utask)
@@ -2369,25 +2372,28 @@ static struct uprobe *find_active_uprobe_rcu(unsigned long bp_vaddr, int *is_swb
return uprobe;
}
-static struct return_instance*
-push_consumer(struct return_instance *ri, int idx, __u64 id, __u64 cookie)
+static struct return_instance *push_consumer(struct return_instance *ri, __u64 id, __u64 cookie)
{
+ struct return_consumer *ric;
+
if (unlikely(ri == ZERO_SIZE_PTR))
return ri;
- if (unlikely(idx >= ri->consumers_cnt)) {
- struct return_instance *old_ri = ri;
-
- ri->consumers_cnt += DEF_CNT;
- ri = krealloc(old_ri, ri_size(old_ri->consumers_cnt), GFP_KERNEL);
- if (!ri) {
- kfree(old_ri);
+ if (unlikely(ri->cons_cnt > 0)) {
+ ric = krealloc(ri->extra_consumers, sizeof(*ric) * ri->cons_cnt, GFP_KERNEL);
+ if (!ric) {
+ kfree(ri->extra_consumers);
+ kfree_rcu(ri, rcu);
return ZERO_SIZE_PTR;
}
+ ri->extra_consumers = ric;
}
- ri->consumers[idx].id = id;
- ri->consumers[idx].cookie = cookie;
+ ric = likely(ri->cons_cnt == 0) ? &ri->consumer : &ri->extra_consumers[ri->cons_cnt - 1];
+ ric->id = id;
+ ric->cookie = cookie;
+
+ ri->cons_cnt++;
return ri;
}
@@ -2395,14 +2401,17 @@ static struct return_consumer *
return_consumer_find(struct return_instance *ri, int *iter, int id)
{
struct return_consumer *ric;
- int idx = *iter;
+ int idx;
- for (ric = &ri->consumers[idx]; idx < ri->consumers_cnt; idx++, ric++) {
+ for (idx = *iter; idx < ri->cons_cnt; idx++)
+ {
+ ric = likely(idx == 0) ? &ri->consumer : &ri->extra_consumers[idx - 1];
if (ric->id == id) {
*iter = idx + 1;
return ric;
}
}
+
return NULL;
}
@@ -2416,7 +2425,6 @@ static void handler_chain(struct uprobe *uprobe, struct pt_regs *regs)
struct uprobe_consumer *uc;
bool has_consumers = false, remove = true;
struct return_instance *ri = NULL;
- int push_idx = 0;
current->utask->auprobe = &uprobe->arch;
@@ -2441,18 +2449,12 @@ static void handler_chain(struct uprobe *uprobe, struct pt_regs *regs)
ri = alloc_return_instance();
if (session)
- ri = push_consumer(ri, push_idx++, uc->id, cookie);
+ ri = push_consumer(ri, uc->id, cookie);
}
current->utask->auprobe = NULL;
- if (!ZERO_OR_NULL_PTR(ri)) {
- /*
- * The push_idx value has the final number of return consumers,
- * and ri->consumers_cnt has number of allocated consumers.
- */
- ri->consumers_cnt = push_idx;
+ if (!ZERO_OR_NULL_PTR(ri))
prepare_uretprobe(uprobe, regs, ri);
- }
if (remove && has_consumers) {
down_read(&uprobe->register_rwsem);
In practice, each return_instance will typically contain either zero or one return_consumer, depending on whether it has any uprobe session consumer attached or not. It's highly unlikely that more than one uprobe session consumers will be attached to any given uprobe, so there is no need to optimize for that case. But the way we currently do memory allocation and accounting is by pre-allocating the space for 4 session consumers in contiguous block of memory next to struct return_instance fixed part. This is unnecessarily wasteful. This patch changes this to keep struct return_instance fixed-sized with one pre-allocated return_consumer, while (in a highly unlikely scenario) allowing for more session consumers in a separate dynamically allocated and reallocated array. We also simplify accounting a bit by not maintaining a separate temporary capacity for consumers array, and, instead, relying on krealloc() to be a no-op if underlying memory can accommodate a slightly bigger allocation (but again, it's very uncommon scenario to even have to do this reallocation). All this gets rid of ri_size(), simplifies push_consumer() and removes confusing ri->consumers_cnt re-assignment, while containing this singular preallocated consumer logic contained within a few simple preexisting helpers. Having fixed-sized struct return_instance simplifies and speeds up return_instance reuse that we ultimately add later in this patch set, see follow up patches. Signed-off-by: Andrii Nakryiko <andrii@kernel.org> --- include/linux/uprobes.h | 10 ++++-- kernel/events/uprobes.c | 72 +++++++++++++++++++++-------------------- 2 files changed, 45 insertions(+), 37 deletions(-)