[bpf-next,v3,10/18] bpf: Search and add kfuncs in struct_ops prologue and epilogue

Commit Message

Amery Hung Jan. 31, 2025, 7:28 p.m. UTC

From: Amery Hung <[e-mail]>

Currently, add_kfunc_call() is only invoked once before the main
verification loop. Therefore, the verifier could not find the
bpf_kfunc_btf_tab of a new kfunc call which is not seen in user defined
struct_ops operators but introduced in gen_prologue or gen_epilogue
during do_misc_fixup(). Fix this by searching kfuncs in the patching
instruction buffer and add them to prog->aux->kfunc_tab.

Signed-off-by: Amery Hung <[e-mail]>
---
 kernel/bpf/verifier.c | 25 ++++++++++++++++++++++++-
 1 file changed, 24 insertions(+), 1 deletion(-)

Comments

Eduard Zingerman Feb. 4, 2025, 5:59 a.m. UTC | #1

On Fri, 2025-01-31 at 11:28 -0800, Amery Hung wrote:
> From: Amery Hung <[e-mail]>
> 
> Currently, add_kfunc_call() is only invoked once before the main
> verification loop. Therefore, the verifier could not find the
> bpf_kfunc_btf_tab of a new kfunc call which is not seen in user defined
> struct_ops operators but introduced in gen_prologue or gen_epilogue
> during do_misc_fixup(). Fix this by searching kfuncs in the patching
> instruction buffer and add them to prog->aux->kfunc_tab.
> 
> Signed-off-by: Amery Hung <[e-mail]>
> ---

Acked-by: Eduard Zingerman <[e-mail]>

[...]

> @@ -20397,6 +20412,10 @@ static int convert_ctx_accesses(struct bpf_verifier_env *env)
>  				return -ENOMEM;
>  			env->prog = new_prog;
>  			delta += cnt - 1;
> +
> +			ret = add_kfunc_in_insns(env, epilogue_buf, epilogue_cnt - 1);

Rant: the -1 here is a bit confusing, it is second time I forget that
      last instruction of the epilogue has to be some kind of control flow,
      and last instruction of the prologue has to be first instruction
      of the program.

> +			if (ret < 0)
> +				return ret;
>  		}
>  	}
>  

[...]

Patch

diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c
index 5bcf095e8d0c..c11d105b3c6f 100644
--- a/kernel/bpf/verifier.c
+++ b/kernel/bpf/verifier.c
@@ -3215,6 +3215,21 @@  bpf_jit_find_kfunc_model(const struct bpf_prog *prog,
 	return res ? &res->func_model : NULL;
 }
 
+static int add_kfunc_in_insns(struct bpf_verifier_env *env,
+			      struct bpf_insn *insn, int cnt)
+{
+	int i, ret;
+
+	for (i = 0; i < cnt; i++, insn++) {
+		if (bpf_pseudo_kfunc_call(insn)) {
+			ret = add_kfunc_call(env, insn->imm, insn->off);
+			if (ret < 0)
+				return ret;
+		}
+	}
+	return 0;
+}
+
 static int add_subprog_and_kfunc(struct bpf_verifier_env *env)
 {
 	struct bpf_subprog_info *subprog = env->subprog_info;
@@ -20368,7 +20383,7 @@  static int convert_ctx_accesses(struct bpf_verifier_env *env)
 {
 	struct bpf_subprog_info *subprogs = env->subprog_info;
 	const struct bpf_verifier_ops *ops = env->ops;
-	int i, cnt, size, ctx_field_size, delta = 0, epilogue_cnt = 0;
+	int i, cnt, size, ctx_field_size, ret, delta = 0, epilogue_cnt = 0;
 	const int insn_cnt = env->prog->len;
 	struct bpf_insn *epilogue_buf = env->epilogue_buf;
 	struct bpf_insn *insn_buf = env->insn_buf;
@@ -20397,6 +20412,10 @@  static int convert_ctx_accesses(struct bpf_verifier_env *env)
 				return -ENOMEM;
 			env->prog = new_prog;
 			delta += cnt - 1;
+
+			ret = add_kfunc_in_insns(env, epilogue_buf, epilogue_cnt - 1);
+			if (ret < 0)
+				return ret;
 		}
 	}
 
@@ -20417,6 +20436,10 @@  static int convert_ctx_accesses(struct bpf_verifier_env *env)
 
 			env->prog = new_prog;
 			delta += cnt - 1;
+
+			ret = add_kfunc_in_insns(env, insn_buf, cnt - 1);
+			if (ret < 0)
+				return ret;
 		}
 	}