bpf, ppc64: generalize fetching subprog into bpf_jit_get_func_addr

Make fetching of the BPF call address from ppc64 JIT generic. ppc64
was using a slightly different variant rather than through the insns'
imm field encoding as the target address would not fit into that space.
Therefore, the target subprog number was encoded into the insns' offset
and fetched through fp->aux->func[off]->bpf_func instead. Given there
are other JITs with this issue and the mechanism of fetching the address
is JIT-generic, move it into the core as a helper instead. On the JIT
side, we get information on whether the retrieved address is a fixed
one, that is, not changing through JIT passes, or a dynamic one. For
the former, JITs can optimize their imm emission because this doesn't
change jump offsets throughout JIT process.

Signed-off-by: Daniel Borkmann <daniel@iogearbox.net>
Reviewed-by: Sandipan Das <sandipan@linux.ibm.com>
Tested-by: Sandipan Das <sandipan@linux.ibm.com>
Signed-off-by: Alexei Starovoitov <ast@kernel.org>

diff --git a/arch/powerpc/net/bpf_jit_comp64.c b/arch/powerpc/net/bpf_jit_comp64.c
index 50b129785aeeead06f8d131a64d30a6ccda2576e..17482f5de3e262f53d67b4199490660362b394a0 100644 (file)
--- a/arch/powerpc/net/bpf_jit_comp64.c
+++ b/arch/powerpc/net/bpf_jit_comp64.c
@@ -166,7 +166,33 @@ static void bpf_jit_build_epilogue(u32 *image, struct codegen_context *ctx)
        PPC_BLR();
 }
 
-static void bpf_jit_emit_func_call(u32 *image, struct codegen_context *ctx, u64 func)
+static void bpf_jit_emit_func_call_hlp(u32 *image, struct codegen_context *ctx,
+                                      u64 func)
+{
+#ifdef PPC64_ELF_ABI_v1
+       /* func points to the function descriptor */
+       PPC_LI64(b2p[TMP_REG_2], func);
+       /* Load actual entry point from function descriptor */
+       PPC_BPF_LL(b2p[TMP_REG_1], b2p[TMP_REG_2], 0);
+       /* ... and move it to LR */
+       PPC_MTLR(b2p[TMP_REG_1]);
+       /*
+        * Load TOC from function descriptor at offset 8.
+        * We can clobber r2 since we get called through a
+        * function pointer (so caller will save/restore r2)
+        * and since we don't use a TOC ourself.
+        */
+       PPC_BPF_LL(2, b2p[TMP_REG_2], 8);
+#else
+       /* We can clobber r12 */
+       PPC_FUNC_ADDR(12, func);
+       PPC_MTLR(12);
+#endif
+       PPC_BLRL();
+}
+
+static void bpf_jit_emit_func_call_rel(u32 *image, struct codegen_context *ctx,
+                                      u64 func)
 {
        unsigned int i, ctx_idx = ctx->idx;
 
@@ -273,7 +299,7 @@ static int bpf_jit_build_body(struct bpf_prog *fp, u32 *image,
 {
        const struct bpf_insn *insn = fp->insnsi;
        int flen = fp->len;
-       int i;
+       int i, ret;
 
        /* Start of epilogue code - will only be valid 2nd pass onwards */
        u32 exit_addr = addrs[flen];
@@ -284,8 +310,9 @@ static int bpf_jit_build_body(struct bpf_prog *fp, u32 *image,
                u32 src_reg = b2p[insn[i].src_reg];
                s16 off = insn[i].off;
                s32 imm = insn[i].imm;
+               bool func_addr_fixed;
+               u64 func_addr;
                u64 imm64;
-               u8 *func;
                u32 true_cond;
                u32 tmp_idx;
 
@@ -711,23 +738,15 @@ emit_clear:
                case BPF_JMP | BPF_CALL:
                        ctx->seen |= SEEN_FUNC;
 
-                       /* bpf function call */
-                       if (insn[i].src_reg == BPF_PSEUDO_CALL)
-                               if (!extra_pass)
-                                       func = NULL;
-                               else if (fp->aux->func && off < fp->aux->func_cnt)
-                                       /* use the subprog id from the off
-                                        * field to lookup the callee address
-                                        */
-                                       func = (u8 *) fp->aux->func[off]->bpf_func;
-                               else
-                                       return -EINVAL;
-                       /* kernel helper call */
-                       else
-                               func = (u8 *) __bpf_call_base + imm;
-
-                       bpf_jit_emit_func_call(image, ctx, (u64)func);
+                       ret = bpf_jit_get_func_addr(fp, &insn[i], extra_pass,
+                                                   &func_addr, &func_addr_fixed);
+                       if (ret < 0)
+                               return ret;
 
+                       if (func_addr_fixed)
+                               bpf_jit_emit_func_call_hlp(image, ctx, func_addr);
+                       else
+                               bpf_jit_emit_func_call_rel(image, ctx, func_addr);
                        /* move return value from r3 to BPF_REG_0 */
                        PPC_MR(b2p[BPF_REG_0], 3);
                        break;

diff --git a/include/linux/filter.h b/include/linux/filter.h
index de629b706d1d7dda246819ed37109f098c05785e..448dcc448f1fe59cb14de60969f71f8ba6c35278 100644 (file)
--- a/include/linux/filter.h
+++ b/include/linux/filter.h
@@ -866,6 +866,10 @@ void bpf_jit_binary_free(struct bpf_binary_header *hdr);
 
 void bpf_jit_free(struct bpf_prog *fp);
 
+int bpf_jit_get_func_addr(const struct bpf_prog *prog,
+                         const struct bpf_insn *insn, bool extra_pass,
+                         u64 *func_addr, bool *func_addr_fixed);
+
 struct bpf_prog *bpf_jit_blind_constants(struct bpf_prog *fp);
 void bpf_jit_prog_release_other(struct bpf_prog *fp, struct bpf_prog *fp_other);
 

diff --git a/kernel/bpf/core.c b/kernel/bpf/core.c
index 1a796e0799ec4a524aee5c325734f0ab22a410ac..b1a3545d0ec89f747d1cd51b7140fd64ad2fe6fd 100644 (file)
--- a/kernel/bpf/core.c
+++ b/kernel/bpf/core.c
@@ -672,6 +672,40 @@ void __weak bpf_jit_free(struct bpf_prog *fp)
        bpf_prog_unlock_free(fp);
 }
 
+int bpf_jit_get_func_addr(const struct bpf_prog *prog,
+                         const struct bpf_insn *insn, bool extra_pass,
+                         u64 *func_addr, bool *func_addr_fixed)
+{
+       s16 off = insn->off;
+       s32 imm = insn->imm;
+       u8 *addr;
+
+       *func_addr_fixed = insn->src_reg != BPF_PSEUDO_CALL;
+       if (!*func_addr_fixed) {
+               /* Place-holder address till the last pass has collected
+                * all addresses for JITed subprograms in which case we
+                * can pick them up from prog->aux.
+                */
+               if (!extra_pass)
+                       addr = NULL;
+               else if (prog->aux->func &&
+                        off >= 0 && off < prog->aux->func_cnt)
+                       addr = (u8 *)prog->aux->func[off]->bpf_func;
+               else
+                       return -EINVAL;
+       } else {
+               /* Address of a BPF helper call. Since part of the core
+                * kernel, it's always at a fixed location. __bpf_call_base
+                * and the helper with imm relative to it are both in core
+                * kernel.
+                */
+               addr = (u8 *)__bpf_call_base + imm;
+       }
+
+       *func_addr = (unsigned long)addr;
+       return 0;
+}
+
 static int bpf_jit_blind_insn(const struct bpf_insn *from,
                              const struct bpf_insn *aux,
                              struct bpf_insn *to_buff)