Context |
Check |
Description |
netdev/tree_selection |
success
|
Not a local patch
|
bpf/vmtest-bpf-PR |
fail
|
merge-conflict
|
bpf/vmtest-bpf-VM_Test-6 |
success
|
Logs for aarch64-gcc / test (test_maps, false, 360) / test_maps on aarch64 with gcc
|
bpf/vmtest-bpf-VM_Test-7 |
success
|
Logs for aarch64-gcc / test (test_progs, false, 360) / test_progs on aarch64 with gcc
|
bpf/vmtest-bpf-VM_Test-8 |
success
|
Logs for aarch64-gcc / test (test_progs_no_alu32, false, 360) / test_progs_no_alu32 on aarch64 with gcc
|
bpf/vmtest-bpf-VM_Test-1 |
success
|
Logs for ShellCheck
|
bpf/vmtest-bpf-VM_Test-3 |
success
|
Logs for Validate matrix.py
|
bpf/vmtest-bpf-VM_Test-9 |
success
|
Logs for aarch64-gcc / test (test_verifier, false, 360) / test_verifier on aarch64 with gcc
|
bpf/vmtest-bpf-VM_Test-5 |
success
|
Logs for aarch64-gcc / build-release
|
bpf/vmtest-bpf-VM_Test-4 |
success
|
Logs for aarch64-gcc / build / build for aarch64 with gcc
|
bpf/vmtest-bpf-VM_Test-2 |
success
|
Logs for Unittests
|
bpf/vmtest-bpf-VM_Test-10 |
success
|
Logs for aarch64-gcc / veristat
|
bpf/vmtest-bpf-VM_Test-0 |
success
|
Logs for Lint
|
bpf/vmtest-bpf-VM_Test-11 |
success
|
Logs for s390x-gcc / build / build for s390x with gcc
|
bpf/vmtest-bpf-VM_Test-12 |
success
|
Logs for s390x-gcc / build-release
|
bpf/vmtest-bpf-VM_Test-13 |
success
|
Logs for s390x-gcc / test (test_progs, false, 360) / test_progs on s390x with gcc
|
bpf/vmtest-bpf-VM_Test-14 |
pending
|
Logs for s390x-gcc / test (test_progs_no_alu32, false, 360) / test_progs_no_alu32 on s390x with gcc
|
bpf/vmtest-bpf-VM_Test-15 |
success
|
Logs for s390x-gcc / test (test_verifier, false, 360) / test_verifier on s390x with gcc
|
bpf/vmtest-bpf-VM_Test-16 |
success
|
Logs for s390x-gcc / veristat
|
bpf/vmtest-bpf-VM_Test-17 |
success
|
Logs for set-matrix
|
bpf/vmtest-bpf-VM_Test-18 |
success
|
Logs for x86_64-gcc / build / build for x86_64 with gcc
|
bpf/vmtest-bpf-VM_Test-19 |
success
|
Logs for x86_64-gcc / build-release
|
bpf/vmtest-bpf-VM_Test-20 |
success
|
Logs for x86_64-gcc / test (test_maps, false, 360) / test_maps on x86_64 with gcc
|
bpf/vmtest-bpf-VM_Test-21 |
success
|
Logs for x86_64-gcc / test (test_progs, false, 360) / test_progs on x86_64 with gcc
|
bpf/vmtest-bpf-VM_Test-22 |
success
|
Logs for x86_64-gcc / test (test_progs_no_alu32, false, 360) / test_progs_no_alu32 on x86_64 with gcc
|
bpf/vmtest-bpf-VM_Test-23 |
success
|
Logs for x86_64-gcc / test (test_progs_no_alu32_parallel, true, 30) / test_progs_no_alu32_parallel on x86_64 with gcc
|
bpf/vmtest-bpf-VM_Test-24 |
success
|
Logs for x86_64-gcc / test (test_progs_parallel, true, 30) / test_progs_parallel on x86_64 with gcc
|
bpf/vmtest-bpf-VM_Test-25 |
success
|
Logs for x86_64-gcc / test (test_verifier, false, 360) / test_verifier on x86_64 with gcc
|
bpf/vmtest-bpf-VM_Test-26 |
success
|
Logs for x86_64-gcc / veristat / veristat on x86_64 with gcc
|
bpf/vmtest-bpf-VM_Test-27 |
success
|
Logs for x86_64-llvm-17 / build / build for x86_64 with llvm-17
|
bpf/vmtest-bpf-VM_Test-28 |
success
|
Logs for x86_64-llvm-17 / build-release / build for x86_64 with llvm-17-O2
|
bpf/vmtest-bpf-VM_Test-29 |
success
|
Logs for x86_64-llvm-17 / test (test_maps, false, 360) / test_maps on x86_64 with llvm-17
|
bpf/vmtest-bpf-VM_Test-30 |
success
|
Logs for x86_64-llvm-17 / test (test_progs, false, 360) / test_progs on x86_64 with llvm-17
|
bpf/vmtest-bpf-VM_Test-31 |
success
|
Logs for x86_64-llvm-17 / test (test_progs_no_alu32, false, 360) / test_progs_no_alu32 on x86_64 with llvm-17
|
bpf/vmtest-bpf-VM_Test-32 |
success
|
Logs for x86_64-llvm-17 / test (test_verifier, false, 360) / test_verifier on x86_64 with llvm-17
|
bpf/vmtest-bpf-VM_Test-33 |
success
|
Logs for x86_64-llvm-17 / veristat
|
bpf/vmtest-bpf-VM_Test-34 |
success
|
Logs for x86_64-llvm-18 / build / build for x86_64 with llvm-18
|
bpf/vmtest-bpf-VM_Test-35 |
success
|
Logs for x86_64-llvm-18 / build-release / build for x86_64 with llvm-18-O2
|
bpf/vmtest-bpf-VM_Test-36 |
success
|
Logs for x86_64-llvm-18 / test (test_maps, false, 360) / test_maps on x86_64 with llvm-18
|
bpf/vmtest-bpf-VM_Test-37 |
success
|
Logs for x86_64-llvm-18 / test (test_progs, false, 360) / test_progs on x86_64 with llvm-18
|
bpf/vmtest-bpf-VM_Test-38 |
success
|
Logs for x86_64-llvm-18 / test (test_progs_cpuv4, false, 360) / test_progs_cpuv4 on x86_64 with llvm-18
|
bpf/vmtest-bpf-VM_Test-39 |
success
|
Logs for x86_64-llvm-18 / test (test_progs_no_alu32, false, 360) / test_progs_no_alu32 on x86_64 with llvm-18
|
bpf/vmtest-bpf-VM_Test-40 |
success
|
Logs for x86_64-llvm-18 / test (test_verifier, false, 360) / test_verifier on x86_64 with llvm-18
|
bpf/vmtest-bpf-VM_Test-41 |
success
|
Logs for x86_64-llvm-18 / veristat
|
@@ -107,7 +107,7 @@ struct uprobe_task {
};
};
- struct uprobe *active_uprobe;
+ struct hprobe active_hprobe;
struct timer_list ri_timer;
unsigned long xol_vaddr;
@@ -115,7 +115,7 @@ struct uprobe_task {
struct return_instance *return_instances;
unsigned int depth;
-};
+} ____cacheline_aligned;
struct return_instance {
unsigned long func;
@@ -1860,11 +1860,16 @@ unsigned long __weak uprobe_get_swbp_addr(struct pt_regs *regs)
return instruction_pointer(regs) - UPROBE_SWBP_INSN_SIZE;
}
+static bool utask_has_pending_sstep_uprobe(struct uprobe_task *utask)
+{
+ return utask->active_hprobe.stable != NULL;
+}
+
unsigned long uprobe_get_trap_addr(struct pt_regs *regs)
{
struct uprobe_task *utask = current->utask;
- if (unlikely(utask && utask->active_uprobe))
+ if (unlikely(utask && utask_has_pending_sstep_uprobe(utask)))
return utask->vaddr;
return instruction_pointer(regs);
@@ -1893,14 +1898,17 @@ void uprobe_free_utask(struct task_struct *t)
{
struct uprobe_task *utask = t->utask;
struct return_instance *ri;
+ struct uprobe *uprobe;
+ bool under_rcu;
if (!utask)
return;
timer_delete_sync(&utask->ri_timer);
- if (utask->active_uprobe)
- put_uprobe(utask->active_uprobe);
+ /* clean up pending single-stepped uprobe */
+ uprobe = hprobe_consume(&utask->active_hprobe, &under_rcu);
+ hprobe_finalize(&utask->active_hprobe, uprobe, under_rcu);
ri = utask->return_instances;
while (ri)
@@ -1924,6 +1932,8 @@ static void ri_timer(struct timer_list *timer)
/* RCU protects return_instance from freeing. */
guard(rcu)();
+ hprobe_expire(&utask->active_hprobe);
+
for_each_ret_instance_rcu(ri, utask->return_instances) {
hprobe_expire(&ri->hprobe);
}
@@ -2166,20 +2176,15 @@ pre_ssout(struct uprobe *uprobe, struct pt_regs *regs, unsigned long bp_vaddr)
{
struct uprobe_task *utask;
unsigned long xol_vaddr;
- int err;
+ int err, srcu_idx;
utask = get_utask();
if (!utask)
return -ENOMEM;
- if (!try_get_uprobe(uprobe))
- return -EINVAL;
-
xol_vaddr = xol_get_insn_slot(uprobe);
- if (!xol_vaddr) {
- err = -ENOMEM;
- goto err_out;
- }
+ if (!xol_vaddr)
+ return -ENOMEM;
utask->xol_vaddr = xol_vaddr;
utask->vaddr = bp_vaddr;
@@ -2187,15 +2192,18 @@ pre_ssout(struct uprobe *uprobe, struct pt_regs *regs, unsigned long bp_vaddr)
err = arch_uprobe_pre_xol(&uprobe->arch, regs);
if (unlikely(err)) {
xol_free_insn_slot(current);
- goto err_out;
+ return err;
}
- utask->active_uprobe = uprobe;
+ srcu_idx = __srcu_read_lock(&uretprobes_srcu);
+
+ hprobe_init_leased(&utask->active_hprobe, uprobe, srcu_idx);
utask->state = UTASK_SSTEP;
+
+ if (!timer_pending(&utask->ri_timer))
+ mod_timer(&utask->ri_timer, jiffies + RI_TIMER_PERIOD);
+
return 0;
-err_out:
- put_uprobe(uprobe);
- return err;
}
/*
@@ -2212,7 +2220,7 @@ bool uprobe_deny_signal(void)
struct task_struct *t = current;
struct uprobe_task *utask = t->utask;
- if (likely(!utask || !utask->active_uprobe))
+ if (likely(!utask || !utask_has_pending_sstep_uprobe(utask)))
return false;
WARN_ON_ONCE(utask->state != UTASK_SSTEP);
@@ -2528,8 +2536,10 @@ static void handle_singlestep(struct uprobe_task *utask, struct pt_regs *regs)
{
struct uprobe *uprobe;
int err = 0;
+ bool under_rcu;
+
+ uprobe = hprobe_consume(&utask->active_hprobe, &under_rcu);
- uprobe = utask->active_uprobe;
if (utask->state == UTASK_SSTEP_ACK)
err = arch_uprobe_post_xol(&uprobe->arch, regs);
else if (utask->state == UTASK_SSTEP_TRAPPED)
@@ -2537,8 +2547,8 @@ static void handle_singlestep(struct uprobe_task *utask, struct pt_regs *regs)
else
WARN_ON_ONCE(1);
- put_uprobe(uprobe);
- utask->active_uprobe = NULL;
+ hprobe_finalize(&utask->active_hprobe, uprobe, under_rcu);
+
utask->state = UTASK_RUNNING;
xol_free_insn_slot(current);
@@ -2556,7 +2566,7 @@ static void handle_singlestep(struct uprobe_task *utask, struct pt_regs *regs)
/*
* On breakpoint hit, breakpoint notifier sets the TIF_UPROBE flag and
* allows the thread to return from interrupt. After that handle_swbp()
- * sets utask->active_uprobe.
+ * sets utask->active_hprobe.
*
* On singlestep exception, singlestep notifier sets the TIF_UPROBE flag
* and allows the thread to return from interrupt.
@@ -2571,7 +2581,7 @@ void uprobe_notify_resume(struct pt_regs *regs)
clear_thread_flag(TIF_UPROBE);
utask = current->utask;
- if (utask && utask->active_uprobe)
+ if (utask && utask_has_pending_sstep_uprobe(utask))
handle_singlestep(utask, regs);
else
handle_swbp(regs);
@@ -2602,7 +2612,7 @@ int uprobe_post_sstep_notifier(struct pt_regs *regs)
{
struct uprobe_task *utask = current->utask;
- if (!current->mm || !utask || !utask->active_uprobe)
+ if (!current->mm || !utask || !utask_has_pending_sstep_uprobe(utask))
/* task is currently not uprobed */
return 0;
Similarly to how we SRCU-protect uprobe instance (and avoid refcounting it unnecessarily) when waiting for return probe hit, use hprobe approach to do the same with single-stepped uprobe. Same hprobe_* primitives are used. We also reuse ri_timer() callback to expire both pending single-step uprobe and return instances. Signed-off-by: Andrii Nakryiko <andrii@kernel.org> --- include/linux/uprobes.h | 4 +-- kernel/events/uprobes.c | 56 ++++++++++++++++++++++++----------------- 2 files changed, 35 insertions(+), 25 deletions(-)