@@ -133,8 +133,14 @@ int notrace unwind_frame(struct task_struct *tsk, struct stackframe *frame)
}
#endif /* CONFIG_FUNCTION_GRAPH_TRACER */
#ifdef CONFIG_KRETPROBES
- if (is_kretprobe_trampoline(frame->pc))
- frame->pc = kretprobe_find_ret_addr(tsk, (void *)frame->fp, &frame->kr_cur);
+ if (is_kretprobe_trampoline(frame->pc)) {
+ void *ret = kretprobe_next_ret_addr(tsk, (void *)frame->fp, &frame->kr_cur);
+ /* There must be a bug in this unwinder or kretprobe. */
+ if (WARN_ON_ONCE(IS_ERR(ret)))
+ pr_err("Kretprobe_trampoline recovery failed (%d)\n", PTR_ERR(ret));
+ else
+ frame->pc = (unsigned long)ret;
+ }
#endif
frame->pc = ptrauth_strip_insn_pac(frame->pc);
@@ -516,6 +516,8 @@ static nokprobe_inline bool is_kretprobe_trampoline(unsigned long addr)
unsigned long kretprobe_find_ret_addr(struct task_struct *tsk, void *fp,
struct llist_node **cur);
+kprobe_opcode_t *kretprobe_next_ret_addr(struct task_struct *tsk, void *fp,
+ struct llist_node **cur);
#else
static nokprobe_inline bool is_kretprobe_trampoline(unsigned long addr)
{
@@ -1922,6 +1922,55 @@ unsigned long kretprobe_find_ret_addr(struct task_struct *tsk, void *fp,
}
NOKPROBE_SYMBOL(kretprobe_find_ret_addr);
+/**
+ * kretprobe_next_ret_addr -- Find next correct return address from @cur
+ * @tsk: Target task
+ * @fp: A framepointer to verify
+ * @cur: a storage and the base point of the loop cursor.
+ *
+ * Find the next correct return address modified by a kretprobe on @tsk from
+ * the entry which points *@cur. If it finds the next currect return address
+ * whose framepointer matches @fp, returns the return address.
+ * If the next current return address's framepointer doesn't match @fp, this
+ * returns ERR_PTR(-EILSEQ). If the *@cur is the end of the kretprobe_instance
+ * list, returns ERR_PTR(-ENOENT). If the @cur is NULL, returns ERR_PTR(-EINVAL).
+ * The @tsk must be 'current' or a task which is not running. @fp is used for
+ * verifying the framepointer which recorded with the correct return address
+ * (kretprobe_instance::fp field.)
+ * The @cur is a loop cursor for searching the kretprobe return addresses on
+ * the @tsk. If *@cur is NULL, this returns the top entry of the correct return
+ * address.
+ */
+kprobe_opcode_t *kretprobe_next_ret_addr(struct task_struct *tsk, void *fp,
+ struct llist_node **cur)
+{
+ struct kretprobe_instance *ri = NULL;
+ kprobe_opcode_t *ret;
+
+ if (WARN_ON_ONCE(!cur))
+ return ERR_PTR(-EINVAL);
+
+ if (*cur) {
+ /* This returns the next correct return address */
+ ret = __kretprobe_find_ret_addr(tsk, cur);
+ if (!ret)
+ return ERR_PTR(-ENOENT);
+ ri = container_of(*cur, struct kretprobe_instance, llist);
+ return ri->fp == fp ? ret : ERR_PTR(-EILSEQ);
+ }
+
+ /* If this is the first try, find the FP-matched entry */
+ do {
+ ret = __kretprobe_find_ret_addr(tsk, cur);
+ if (!ret)
+ return ERR_PTR(-ENOENT);
+ ri = container_of(*cur, struct kretprobe_instance, llist);
+ } while (ri->fp != fp);
+
+ return ret;
+}
+NOKPROBE_SYMBOL(kretprobe_next_ret_addr);
+
void __weak arch_kretprobe_fixup_return(struct pt_regs *regs,
kprobe_opcode_t *correct_ret_addr)
{
Add kretprobe_next_ret_addr() which can detect errors in the given parameter or the kretprobe_instance list, and call it from arm64 stacktrace. This kretprobe_next_ret_addr() will return following errors when it detects; - -EINVAL if @cur is NULL (caller issue) - -ENOENT if there is no next correct return address (either kprobes or caller issue) - -EILSEQ if the next currect return address is there but doesn't match the framepointer (maybe caller issue) Thus the caller must check the error and handle it. On arm64, this tries to handle the errors and show it on the log. Suggested-by: Mark Rutland <mark.rutland@arm.com> Signed-off-by: Masami Hiramatsu <mhiramat@kernel.org> --- arch/arm64/kernel/stacktrace.c | 10 +++++++- include/linux/kprobes.h | 2 ++ kernel/kprobes.c | 49 ++++++++++++++++++++++++++++++++++++++++ 3 files changed, 59 insertions(+), 2 deletions(-)