@@ -426,6 +426,7 @@ struct bpf_verifier_state {
* while they are still in use.
*/
bool used_as_loop_entry;
+ bool in_sleepable;
/* first and last insn idx of this verifier state */
u32 first_insn_idx;
@@ -1429,6 +1429,7 @@ static int copy_verifier_state(struct bpf_verifier_state *dst_state,
}
dst_state->speculative = src->speculative;
dst_state->active_rcu_lock = src->active_rcu_lock;
+ dst_state->in_sleepable = src->in_sleepable;
dst_state->curframe = src->curframe;
dst_state->active_lock.ptr = src->active_lock.ptr;
dst_state->active_lock.id = src->active_lock.id;
@@ -2404,7 +2405,7 @@ static void init_func_state(struct bpf_verifier_env *env,
/* Similar to push_stack(), but for async callbacks */
static struct bpf_verifier_state *push_async_cb(struct bpf_verifier_env *env,
int insn_idx, int prev_insn_idx,
- int subprog)
+ int subprog, bool is_sleepable)
{
struct bpf_verifier_stack_elem *elem;
struct bpf_func_state *frame;
@@ -2431,6 +2432,7 @@ static struct bpf_verifier_state *push_async_cb(struct bpf_verifier_env *env,
* Initialize it similar to do_check_common().
*/
elem->st.branches = 1;
+ elem->st.in_sleepable = is_sleepable;
frame = kzalloc(sizeof(*frame), GFP_KERNEL);
if (!frame)
goto err;
@@ -5278,7 +5280,8 @@ static int map_kptr_match_type(struct bpf_verifier_env *env,
static bool in_sleepable(struct bpf_verifier_env *env)
{
- return env->prog->sleepable;
+ return env->prog->sleepable ||
+ (env->cur_state && env->cur_state->in_sleepable);
}
/* The non-sleepable programs and sleepable programs with explicit bpf_rcu_read_lock()
@@ -9515,7 +9518,7 @@ static int btf_check_subprog_call(struct bpf_verifier_env *env, int subprog,
}
static int push_callback_call(struct bpf_verifier_env *env, struct bpf_insn *insn,
- int insn_idx, int subprog,
+ int insn_idx, int subprog, bool is_sleepable,
set_callee_state_fn set_callee_state_cb)
{
struct bpf_verifier_state *state = env->cur_state, *callback_state;
@@ -9550,7 +9553,7 @@ static int push_callback_call(struct bpf_verifier_env *env, struct bpf_insn *ins
/* there is no real recursion here. timer callbacks are async */
env->subprog_info[subprog].is_async_cb = true;
async_cb = push_async_cb(env, env->subprog_info[subprog].start,
- insn_idx, subprog);
+ insn_idx, subprog, is_sleepable);
if (!async_cb)
return -EFAULT;
callee = async_cb->frame[0];
@@ -10389,15 +10392,15 @@ static int check_helper_call(struct bpf_verifier_env *env, struct bpf_insn *insn
}
break;
case BPF_FUNC_for_each_map_elem:
- err = push_callback_call(env, insn, insn_idx, meta.subprogno,
+ err = push_callback_call(env, insn, insn_idx, meta.subprogno, false,
set_map_elem_callback_state);
break;
case BPF_FUNC_timer_set_callback:
- err = push_callback_call(env, insn, insn_idx, meta.subprogno,
+ err = push_callback_call(env, insn, insn_idx, meta.subprogno, false,
set_timer_callback_state);
break;
case BPF_FUNC_find_vma:
- err = push_callback_call(env, insn, insn_idx, meta.subprogno,
+ err = push_callback_call(env, insn, insn_idx, meta.subprogno, false,
set_find_vma_callback_state);
break;
case BPF_FUNC_snprintf:
@@ -10412,7 +10415,7 @@ static int check_helper_call(struct bpf_verifier_env *env, struct bpf_insn *insn
if (err)
return err;
if (cur_func(env)->callback_depth < regs[BPF_REG_1].umax_value) {
- err = push_callback_call(env, insn, insn_idx, meta.subprogno,
+ err = push_callback_call(env, insn, insn_idx, meta.subprogno, false,
set_loop_callback_state);
} else {
cur_func(env)->callback_depth = 0;
@@ -10515,7 +10518,7 @@ static int check_helper_call(struct bpf_verifier_env *env, struct bpf_insn *insn
break;
}
case BPF_FUNC_user_ringbuf_drain:
- err = push_callback_call(env, insn, insn_idx, meta.subprogno,
+ err = push_callback_call(env, insn, insn_idx, meta.subprogno, false,
set_user_ringbuf_callback_state);
break;
}
@@ -12232,7 +12235,7 @@ static int check_kfunc_call(struct bpf_verifier_env *env, struct bpf_insn *insn,
return err;
if (meta.func_id == special_kfunc_list[KF_bpf_rbtree_add_impl]) {
- err = push_callback_call(env, insn, insn_idx, meta.subprogno,
+ err = push_callback_call(env, insn, insn_idx, meta.subprogno, false,
set_rbtree_add_callback_state);
if (err) {
verbose(env, "kfunc %s#%d failed callback verification\n",
@@ -17004,6 +17007,9 @@ static bool states_equal(struct bpf_verifier_env *env,
if (old->active_rcu_lock != cur->active_rcu_lock)
return false;
+ if (old->in_sleepable != cur->in_sleepable)
+ return false;
+
/* for states to be equal callsites have to be the same
* and all frame states need to be equivalent
*/
To support sleepable async callbacks, we need to tell push_callback_call() whether the cb is sleepable or not. Doing so while checking for the kfunc arguments (when we call push_callback_call()) is simpler than adding a check for a function while inside push_callback_call(). When a callback is tagged as sleepable, the verifier now knows that it is the case and can allow a sleepable callback to happen. Signed-off-by: Benjamin Tissoires <bentiss@kernel.org> --- include/linux/bpf_verifier.h | 1 + kernel/bpf/verifier.c | 26 ++++++++++++++++---------- 2 files changed, 17 insertions(+), 10 deletions(-)