@@ -2304,6 +2304,7 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog)
if (header)
bpf_jit_binary_free(header);
prog = orig_prog;
+ header = NULL;
goto out_addrs;
}
if (image) {
@@ -2371,6 +2372,7 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog)
if (tmp_blinded)
bpf_jit_prog_release_other(prog, prog == orig_prog ?
tmp : orig_prog);
+ prog->hdr = header;
return prog;
}
@@ -584,6 +584,7 @@ struct bpf_prog {
const struct bpf_insn *insn);
struct bpf_prog_aux *aux; /* Auxiliary fields */
struct sock_fprog_kern *orig_prog; /* Original BPF program */
+ struct bpf_binary_header *hdr;
/* Instructions for interpreter */
union {
DECLARE_FLEX_ARRAY(struct sock_filter, insns);
@@ -893,9 +894,14 @@ static inline void bpf_jit_binary_lock_ro(struct bpf_binary_header *hdr)
static inline struct bpf_binary_header *
bpf_jit_binary_hdr(const struct bpf_prog *fp)
{
- unsigned long real_start = (unsigned long)fp->bpf_func;
- unsigned long addr = real_start & PAGE_MASK;
+ unsigned long real_start;
+ unsigned long addr;
+ if (fp->hdr)
+ return fp->hdr;
+
+ real_start = (unsigned long)fp->bpf_func;
+ addr = real_start & PAGE_MASK;
return (void *)addr;
}
With sub page allocation, we cannot simply use bpf_func & PAGE_MASK to find the bpf_binary_header. Add a pointer to struct bpf_prog to avoid this logic. Use this point for x86_64. If the pointer is not set by the jit engine, fall back to original logic. Signed-off-by: Song Liu <songliubraving@fb.com> --- arch/x86/net/bpf_jit_comp.c | 2 ++ include/linux/filter.h | 10 ++++++++-- 2 files changed, 10 insertions(+), 2 deletions(-)