@@ -32,10 +32,13 @@ struct prev_kprobe {
unsigned int status;
};
+#define KPROBE_NEST_MAX 2
+
/* per-cpu kprobe control block */
struct kprobe_ctlblk {
unsigned int kprobe_status;
- struct prev_kprobe prev_kprobe;
+ struct prev_kprobe prev[KPROBE_NEST_MAX];
+ int nested;
};
void arch_remove_kprobe(struct kprobe *);
@@ -187,18 +187,6 @@ void __kprobes arch_remove_kprobe(struct kprobe *p)
}
}
-static void __kprobes save_previous_kprobe(struct kprobe_ctlblk *kcb)
-{
- kcb->prev_kprobe.kp = kprobe_running();
- kcb->prev_kprobe.status = kcb->kprobe_status;
-}
-
-static void __kprobes restore_previous_kprobe(struct kprobe_ctlblk *kcb)
-{
- __this_cpu_write(current_kprobe, kcb->prev_kprobe.kp);
- kcb->kprobe_status = kcb->prev_kprobe.status;
-}
-
static void __kprobes set_current_kprobe(struct kprobe *p)
{
__this_cpu_write(current_kprobe, p);
@@ -224,6 +212,40 @@ singlestep(struct kprobe *p, struct pt_regs *regs, struct kprobe_ctlblk *kcb)
p->ainsn.insn_singlestep(p->opcode, &p->ainsn, regs);
}
+static nokprobe_inline int reenter_kprobe(struct kprobe *p,
+ struct pt_regs *regs,
+ struct kprobe_ctlblk *kcb)
+{
+ /* Kprobe is pending, so we're recursing. */
+ switch (kcb->kprobe_status) {
+ case KPROBE_HIT_ACTIVE:
+ case KPROBE_HIT_SSDONE:
+ if (kcb->nested < KPROBE_NEST_MAX - 1) {
+ save_previous_kprobe(kcb);
+ return 0;
+ }
+ fallthrough;
+ case KPROBE_HIT_SS:
+ /* A pre- or post-handler probe got us here. */
+ kprobes_inc_nmissed_count(p);
+ save_previous_kprobe(kcb);
+ set_current_kprobe(p);
+ kcb->kprobe_status = KPROBE_REENTER;
+ singlestep(p, regs, kcb);
+ restore_previous_kprobe(kcb);
+ break;
+ case KPROBE_REENTER:
+ /* A nested probe was hit in FIQ, it is a BUG */
+ pr_warn("Unrecoverable kprobe detected.\n");
+ dump_kprobe(p);
+ fallthrough;
+ default:
+ /* impossible cases */
+ BUG();
+ }
+ return 1;
+}
+
/*
* Called with IRQs disabled. IRQs must remain disabled from that point
* all the way until processing this kprobe is complete. The current
@@ -262,30 +284,9 @@ void __kprobes kprobe_handler(struct pt_regs *regs)
* In this case, we can skip recursing check too.
*/
singlestep_skip(p, regs);
- } else if (cur) {
- /* Kprobe is pending, so we're recursing. */
- switch (kcb->kprobe_status) {
- case KPROBE_HIT_ACTIVE:
- case KPROBE_HIT_SSDONE:
- case KPROBE_HIT_SS:
- /* A pre- or post-handler probe got us here. */
- kprobes_inc_nmissed_count(p);
- save_previous_kprobe(kcb);
- set_current_kprobe(p);
- kcb->kprobe_status = KPROBE_REENTER;
- singlestep(p, regs, kcb);
- restore_previous_kprobe(kcb);
- break;
- case KPROBE_REENTER:
- /* A nested probe was hit in FIQ, it is a BUG */
- pr_warn("Unrecoverable kprobe detected.\n");
- dump_kprobe(p);
- /* fall through */
- default:
- /* impossible cases */
- BUG();
- }
} else {
+ if (cur && reenter_kprobe(p, regs, kcb))
+ return;
/* Probe hit and conditional execution check ok. */
set_current_kprobe(p);
kcb->kprobe_status = KPROBE_HIT_ACTIVE;
@@ -305,7 +306,7 @@ void __kprobes kprobe_handler(struct pt_regs *regs)
p->post_handler(p, regs, 0);
}
}
- reset_current_kprobe();
+ pop_current_kprobe(kcb);
}
} else {
/*
@@ -342,11 +343,7 @@ int __kprobes kprobe_fault_handler(struct pt_regs *regs, unsigned int fsr)
* normal page fault.
*/
regs->ARM_pc = (long)cur->addr;
- if (kcb->kprobe_status == KPROBE_REENTER) {
- restore_previous_kprobe(kcb);
- } else {
- reset_current_kprobe();
- }
+ pop_current_kprobe(kcb);
break;
case KPROBE_HIT_ACTIVE:
@@ -34,6 +34,36 @@ typedef enum probes_insn (kprobe_decode_insn_t)(probes_opcode_t,
const union decode_action *,
const struct decode_checker *[]);
+
+static nokprobe_inline void save_previous_kprobe(struct kprobe_ctlblk *kcb)
+{
+ int i = kcb->nested++;
+
+ kcb->prev[i].kp = kprobe_running();
+ kcb->prev[i].status = kcb->kprobe_status;
+}
+
+static nokprobe_inline void restore_previous_kprobe(struct kprobe_ctlblk *kcb)
+{
+ int i = --kcb->nested;
+
+ __this_cpu_write(current_kprobe, kcb->prev[i].kp);
+ kcb->kprobe_status = kcb->prev[i].status;
+}
+
+static nokprobe_inline void pop_current_kprobe(struct kprobe_ctlblk *kcb)
+{
+ if (kcb->nested)
+ restore_previous_kprobe(kcb);
+ else
+ reset_current_kprobe();
+}
+
+static nokprobe_inline bool kprobe_can_nest(struct kprobe_ctlblk *kcb)
+{
+ return !kprobe_running() || (kcb->nested < KPROBE_NEST_MAX - 1);
+}
+
#ifdef CONFIG_THUMB2_KERNEL
extern const union decode_action kprobes_t32_actions[];
@@ -161,13 +161,15 @@ optimized_callback(struct optimized_kprobe *op, struct pt_regs *regs)
local_irq_save(flags);
kcb = get_kprobe_ctlblk();
- if (kprobe_running()) {
+ if (!kprobe_can_nest(kcb)) {
kprobes_inc_nmissed_count(&op->kp);
} else {
+ if (kprobe_running())
+ save_previous_kprobe(kcb);
__this_cpu_write(current_kprobe, &op->kp);
kcb->kprobe_status = KPROBE_HIT_ACTIVE;
opt_pre_handler(&op->kp, regs);
- __this_cpu_write(current_kprobe, NULL);
+ pop_current_kprobe(kcb);
}
/*
Make kprobes to accept 1-level nesting instead of missing it on arm. Any kprobes hits in kprobes pre/post handler context can be nested at once. If the other kprobes hits in the nested pre/post handler context, that will be missed. We can test this feature on the kernel with CONFIG_KPROBE_EVENTS_ON_NOTRACE=y as below. # cd /sys/kernel/debug/tracing # echo p ring_buffer_lock_reserve > kprobe_events # echo p vfs_read >> kprobe_events # echo 0 > /proc/sys/debug/kprobes-optimization # to check trap handler # echo stacktrace > events/kprobes/p_ring_buffer_lock_reserve_0/trigger # echo 1 > events/kprobes/enable # cat trace ... sh-94 [000] d... 124.945302: p_vfs_read_0: (vfs_read+0x0/0x120) sh-94 [000] d... 125.041822: p_ring_buffer_lock_reserve_0: (ring_buffer_lock_reserve+0x0/0x57c) sh-94 [000] d... 125.042102: <stack trace> => kprobe_dispatcher => aggr_pre_handler => kprobe_handler => kprobe_trap_handler => do_undefinstr => __und_svc_finish => ring_buffer_lock_reserve => kprobe_trace_func => kprobe_dispatcher => aggr_pre_handler => kprobe_handler => kprobe_trap_handler => do_undefinstr => __und_svc_finish => vfs_read => sys_read => ret_fast_syscall The trap handler is nested correctly. Signed-off-by: Masami Hiramatsu <mhiramat@kernel.org> --- arch/arm/include/asm/kprobes.h | 5 ++ arch/arm/probes/kprobes/core.c | 79 ++++++++++++++++++------------------- arch/arm/probes/kprobes/core.h | 30 ++++++++++++++ arch/arm/probes/kprobes/opt-arm.c | 6 ++- 4 files changed, 76 insertions(+), 44 deletions(-)