@@ -1185,6 +1185,14 @@ static int build_insn(const struct bpf_insn *insn, struct jit_ctx *ctx,
*/
break;
+ /* kernel hidden stack operations */
+ case BPF_ST | BPF_STACK:
+ emit(A64_PUSH(src, src, A64_SP), ctx);
+ break;
+ case BPF_LD | BPF_STACK:
+ emit(A64_POP(dst, dst, A64_SP), ctx);
+ break;
+
/* ST: *(size *)(dst + off) = imm */
case BPF_ST | BPF_MEM | BPF_W:
case BPF_ST | BPF_MEM | BPF_H:
@@ -1324,6 +1324,14 @@ static int do_jit(struct bpf_prog *bpf_prog, int *addrs, u8 *image, u8 *rw_image
EMIT_LFENCE();
break;
+ /* kernel hidden stack operations */
+ case BPF_ST | BPF_STACK:
+ EMIT1(add_1reg(0x50, src_reg)); /* pushq */
+ break;
+ case BPF_LD | BPF_STACK:
+ EMIT1(add_1reg(0x58, dst_reg)); /* popq */
+ break;
+
/* ST: *(u8*)(dst_reg + off) = imm */
case BPF_ST | BPF_MEM | BPF_B:
if (is_ereg(dst_reg))
@@ -76,6 +76,9 @@ struct ctl_table_header;
*/
#define BPF_NOSPEC 0xc0
+/* unused opcode for kernel hidden stack operations */
+#define BPF_STACK 0xe0
+
/* As per nm, we expose JITed images as text (code) section for
* kallsyms. That way, tools like perf can find it to match
* addresses.
@@ -402,6 +405,26 @@ static inline bool insn_is_zext(const struct bpf_insn *insn)
.off = 0, \
.imm = 0 })
+/* Push SRC register value onto the stack */
+
+#define BPF_PUSH64(SRC) \
+ ((struct bpf_insn) { \
+ .code = BPF_ST | BPF_STACK, \
+ .dst_reg = 0, \
+ .src_reg = SRC, \
+ .off = 0, \
+ .imm = 0 })
+
+/* Pop stack value into DST register */
+
+#define BPF_POP64(DST) \
+ ((struct bpf_insn) { \
+ .code = BPF_LD | BPF_STACK, \
+ .dst_reg = DST, \
+ .src_reg = 0, \
+ .off = 0, \
+ .imm = 0 })
+
/* Internal classic blocks for direct assignment */
#define __BPF_STMT(CODE, K) \
@@ -214,6 +214,9 @@ void print_bpf_insn(const struct bpf_insn_cbs *cbs,
insn->off, insn->imm);
} else if (BPF_MODE(insn->code) == 0xc0 /* BPF_NOSPEC, no UAPI */) {
verbose(cbs->private_data, "(%02x) nospec\n", insn->code);
+ } else if (BPF_MODE(insn->code) == 0xe0 /* BPF_STACK, no UAPI */) {
+ verbose(cbs->private_data, "(%02x) push r%d\n",
+ insn->code, insn->src_reg);
} else {
verbose(cbs->private_data, "BUG_st_%02x\n", insn->code);
}
@@ -254,6 +257,9 @@ void print_bpf_insn(const struct bpf_insn_cbs *cbs,
insn->code, insn->dst_reg,
__func_imm_name(cbs, insn, imm,
tmp, sizeof(tmp)));
+ } else if (BPF_MODE(insn->code) == 0xe0 /* BPF_STACK, no UAPI */) {
+ verbose(cbs->private_data, "(%02x) pop r%d\n",
+ insn->code, insn->dst_reg);
} else {
verbose(cbs->private_data, "BUG_ld_%02x\n", insn->code);
return;
Implemented for: - x86_64 jit (tested) - arm64 jit (untested) Interpreter is not implemented because push/pop are currently used only with xdp kfunc and jit is required to use kfuncs. Fundamentally: BPF_ST | BPF_STACK + src_reg == store into the stack BPF_LD | BPF_STACK + dst_reg == load from the stack off/imm are unused Updated disasm code to properly dump these new instructions: 31: (e2) push r1 32: (79) r5 = *(u64 *)(r1 +56) 33: (55) if r5 != 0x0 goto pc+2 34: (b7) r0 = 0 35: (05) goto pc+1 36: (79) r0 = *(u64 *)(r5 +32) 37: (e0) pop r1 Cc: Zi Shen Lim <zlim.lnx@gmail.com> Suggested-by: Alexei Starovoitov <ast@kernel.org> Signed-off-by: Stanislav Fomichev <sdf@google.com> --- arch/arm64/net/bpf_jit_comp.c | 8 ++++++++ arch/x86/net/bpf_jit_comp.c | 8 ++++++++ include/linux/filter.h | 23 +++++++++++++++++++++++ kernel/bpf/disasm.c | 6 ++++++ 4 files changed, 45 insertions(+)