@@ -1131,6 +1131,15 @@ static void emit_shiftx(u8 **pprog, u32 dst_reg, u8 src_reg, bool is64, u8 op)
*pprog = prog;
}
+static bool is_static_ja_nop(const struct bpf_insn *insn)
+{
+ u8 code = insn->code;
+
+ return (code == (BPF_JMP | BPF_JA) || code == (BPF_JMP32 | BPF_JA)) &&
+ (insn->src_reg & BPF_STATIC_BRANCH_JA) &&
+ (insn->src_reg & BPF_STATIC_BRANCH_NOP);
+}
+
#define INSN_SZ_DIFF (((addrs[i] - addrs[i - 1]) - (prog - temp)))
/* mov rax, qword ptr [rbp - rounded_stack_depth - 8] */
@@ -2016,9 +2025,15 @@ st: if (is_imm8(insn->off))
}
emit_nops(&prog, INSN_SZ_DIFF - 2);
}
- EMIT2(0xEB, jmp_offset);
+ if (is_static_ja_nop(insn))
+ emit_nops(&prog, 2);
+ else
+ EMIT2(0xEB, jmp_offset);
} else if (is_simm32(jmp_offset)) {
- EMIT1_off32(0xE9, jmp_offset);
+ if (is_static_ja_nop(insn))
+ emit_nops(&prog, 5);
+ else
+ EMIT1_off32(0xE9, jmp_offset);
} else {
pr_err("jmp gen bug %llx\n", jmp_offset);
return -EFAULT;
@@ -1372,6 +1372,16 @@ struct bpf_stack_build_id {
};
};
+/* Flags for JA insn, passed in SRC_REG */
+enum {
+ BPF_STATIC_BRANCH_JA = 1 << 0,
+ BPF_STATIC_BRANCH_NOP = 1 << 1,
+};
+
+#define BPF_STATIC_BRANCH_MASK (BPF_STATIC_BRANCH_JA | \
+ BPF_STATIC_BRANCH_NOP)
+
+
#define BPF_OBJ_NAME_LEN 16U
union bpf_attr {
@@ -15607,14 +15607,24 @@ static int visit_insn(int t, struct bpf_verifier_env *env)
else
off = insn->imm;
- /* unconditional jump with single edge */
- ret = push_insn(t, t + off + 1, FALLTHROUGH, env);
- if (ret)
- return ret;
+ if (insn->src_reg & BPF_STATIC_BRANCH_JA) {
+ /* static branch - jump with two edges */
+ mark_prune_point(env, t);
+
+ ret = push_insn(t, t + 1, FALLTHROUGH, env);
+ if (ret)
+ return ret;
- mark_prune_point(env, t + off + 1);
- mark_jmp_point(env, t + off + 1);
+ ret = push_insn(t, t + off + 1, BRANCH, env);
+ } else {
+ /* unconditional jump with single edge */
+ ret = push_insn(t, t + off + 1, FALLTHROUGH, env);
+ if (ret)
+ return ret;
+ mark_prune_point(env, t + off + 1);
+ mark_jmp_point(env, t + off + 1);
+ }
return ret;
default:
@@ -17584,8 +17594,11 @@ static int do_check(struct bpf_verifier_env *env)
mark_reg_scratched(env, BPF_REG_0);
} else if (opcode == BPF_JA) {
+ struct bpf_verifier_state *other_branch;
+ u32 jmp_offset;
+
if (BPF_SRC(insn->code) != BPF_K ||
- insn->src_reg != BPF_REG_0 ||
+ (insn->src_reg & ~BPF_STATIC_BRANCH_MASK) ||
insn->dst_reg != BPF_REG_0 ||
(class == BPF_JMP && insn->imm != 0) ||
(class == BPF_JMP32 && insn->off != 0)) {
@@ -17594,9 +17607,21 @@ static int do_check(struct bpf_verifier_env *env)
}
if (class == BPF_JMP)
- env->insn_idx += insn->off + 1;
+ jmp_offset = insn->off + 1;
else
- env->insn_idx += insn->imm + 1;
+ jmp_offset = insn->imm + 1;
+
+ /* Staic branch can either jump to +off or fall through */
+ if (insn->src_reg & BPF_STATIC_BRANCH_JA) {
+ other_branch = push_stack(env, env->insn_idx + jmp_offset,
+ env->insn_idx, false);
+ if (!other_branch)
+ return -EFAULT;
+
+ jmp_offset = 1;
+ }
+
+ env->insn_idx += jmp_offset;
continue;
} else if (opcode == BPF_EXIT) {
Add support for a new version of JA instruction, a static branch JA. Such instructions may either jump to the specified offset or act as nops. To distinguish such instructions from normal JA the BPF_STATIC_BRANCH_JA flag should be set for the SRC register. By default on program load such instructions are jitted as a normal JA. However, if the BPF_STATIC_BRANCH_NOP flag is set in the SRC register, then the instruction is jitted to a NOP. In order to generate BPF_STATIC_BRANCH_JA instructions using llvm two new instructions were added: asm volatile goto ("nop_or_gotol %l[label]" :::: label); will generate the BPF_STATIC_BRANCH_JA|BPF_STATIC_BRANCH_NOP instuction and asm volatile goto ("gotol_or_nop %l[label]" :::: label); will generate a BPF_STATIC_BRANCH_JA instruction, without an extra bit set. The reason for adding two instructions is that both are required to implement static keys functionality for BPF. The verifier logic is extended to check both possible paths: jump and nop. Signed-off-by: Anton Protopopov <aspsk@isovalent.com> --- arch/x86/net/bpf_jit_comp.c | 19 ++++++++++++++-- include/uapi/linux/bpf.h | 10 +++++++++ kernel/bpf/verifier.c | 43 +++++++++++++++++++++++++++++-------- 3 files changed, 61 insertions(+), 11 deletions(-)