mips, bpf: Add BPF-functions call support

BPF local functions based on the BPF_CALL and BPF_IMM-mode instructions
require extra-pass being performed on the image in order to have the
calling function addresses applied. It's done by calling the
bpf_int_jit_compile() twice: first to generate the BPF JITed image with
sub-functions stub addresses, and second, after actual function addresses
are known, to apply these address in the respective instructions. In order
for that to work correctly fix the bpf_int_jit_compile() method in a way
it's done in the rest of the eBPF JIT-capable platforms: save
platform-specific eBPF JIT context and re-use it in the extra pass.

Signed-off-by: Serge Semin <fancer.lancer@gmail.com>

4 files changed, 65 insertions, 21 deletions

diff --git a/arch/mips/net/bpf_jit_comp.c b/arch/mips/net/bpf_jit_comp.c
index 6143c8a649c82..b10270777d0f3 100644
--- a/arch/mips/net/bpf_jit_comp.c
+++ b/arch/mips/net/bpf_jit_comp.c
@@ -937,6 +937,24 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog)
 	 */
 	if (!prog->jit_requested)
 		return orig_prog;
+
+	/*
+	 * If the BPF-program has been already JITed and the JIT-context hasn't
+	 * been saved, the extra pass isn't required and just return the passed
+	 * image. Otherwise restore the context and procceed with the extra
+	 * pass.
+	 */
+	if (prog->jited && !prog->aux->jit_data) {
+		return orig_prog;
+	} else if (prog->aux->jit_data) {
+		ctx = prog->aux->jit_data;
+		ctx->extra_pass = true;
+		tmp_idx = ctx->jit_index;
+		image_ptr = (void *)ctx->target;
+		image_size = prog->jited_len;
+		goto skip_init_ctx;
+	}
+
 	/*
 	 * If constant blinding was enabled and we failed during blinding
 	 * then we must fall back to the interpreter. Otherwise, we save
@@ -1002,42 +1020,67 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog)
 
 	/* Actual pass to generate final JIT code */
 	ctx->target = (u32 *)image_ptr;
-	ctx->jit_index = 0;
 
 	/*
-	 * If building the JITed code fails somehow,
-	 * we fall back to the interpretation.
+	 * If building the JITed code fails somehow fall back to the
+	 * interpretation in both cases - initial and extra passes.
 	 */
+skip_init_ctx:
+	ctx->jit_index = 0;
 	build_prologue(ctx);
 	if (build_body(ctx) < 0)
-		goto err_free_header;
+		goto err_clean_prog;
 	build_epilogue(ctx, MIPS_R_RA);
 
-	/* Populate line info meta data */
-	set_convert_flag(ctx, false);
-	bpf_prog_fill_jited_linfo(prog, &ctx->descriptors[1]);
-
-	/* Set as read-only exec and flush instruction cache */
-	if (bpf_jit_binary_lock_ro(ctx->header))
-		goto err_free_header;
-	flush_icache_range((unsigned long)ctx->header,
-			   (unsigned long)&ctx->target[ctx->jit_index]);
-
-	if (bpf_jit_enable > 1)
-		bpf_jit_dump(prog->len, image_size, ctx->passes, ctx->target);
+	/* Sanity check extra JIT-compile pass */
+	if (ctx->extra_pass && ctx->jit_index != tmp_idx) {
+		pr_err_once("multi-func JIT bug %d != %d\n",
+			    ctx->jit_index, tmp_idx);
+		goto err_clean_prog;
+	}
 
 	prog->bpf_func = (void *)ctx->target;
 	prog->jited = 1;
 	prog->jited_len = image_size;
 
+	/*
+	 * Finalize the image and free the context after the extra JIT-compile
+	 * pass or if no extra pass implied
+	 */
+	if (ctx->extra_pass || !prog->is_func) {
+		/* Populate line info meta data */
+		set_convert_flag(ctx, false);
+		bpf_prog_fill_jited_linfo(prog, &ctx->descriptors[1]);
+
+		if (bpf_jit_enable > 1) {
+			bpf_jit_dump(prog->len, image_size,
+				     ctx->passes + ctx->extra_pass,
+				     ctx->target);
+		}
+
+		/* Set as read-only exec and flush instruction cache */
+		if (bpf_jit_binary_lock_ro(ctx->header))
+			goto err_clean_prog;
+		flush_icache_range((unsigned long)ctx->header,
+				   (unsigned long)&ctx->target[ctx->jit_index]);
+
+		/* JIT-compile context is no longer needed */
+		prog->aux->jit_data = NULL;
+		kfree(ctx);
+	} else {
+		prog->aux->jit_data = ctx;
+	}
+
 	if (prog != orig_prog)
 		bpf_jit_prog_release_other(prog, orig_prog);
 
-	kfree(ctx);
-
 	return prog;
 
-err_free_header:
+err_clean_prog:
+	orig_prog->bpf_func = NULL;
+	orig_prog->jited = 0;
+	orig_prog->jited_len = 0;
+
 	bpf_jit_binary_free(ctx->header);
 
 err_free_context:
diff --git a/arch/mips/net/bpf_jit_comp.h b/arch/mips/net/bpf_jit_comp.h
index 36f6c8b706987..e258d7ff61a05 100644
--- a/arch/mips/net/bpf_jit_comp.h
+++ b/arch/mips/net/bpf_jit_comp.h
@@ -75,6 +75,7 @@
 struct jit_context {
 	struct bpf_prog *program;     /* The eBPF program being JITed        */
 	struct bpf_binary_header *header; /* The eBPF JITed binary header    */
+	bool extra_pass;              /* Extra compile pass flag             */
 	u32 passes;                   /* Number of compile passes performed  */
 	u32 changes;                  /* Number of PC-relative branch conv   */
 	u32 accessed;                 /* Bit mask of read eBPF registers     */
diff --git a/arch/mips/net/bpf_jit_comp32.c b/arch/mips/net/bpf_jit_comp32.c
index 44e8e86b65ceb..ab1fe3b7b0c0f 100644
--- a/arch/mips/net/bpf_jit_comp32.c
+++ b/arch/mips/net/bpf_jit_comp32.c
@@ -1266,7 +1266,7 @@ static int emit_call(struct jit_context *ctx, const struct bpf_insn *insn)
 	u64 addr;
 
 	/* Decode the call address */
-	if (bpf_jit_get_func_addr(ctx->program, insn, false,
+	if (bpf_jit_get_func_addr(ctx->program, insn, ctx->extra_pass,
 				  &addr, &fixed) < 0)
 		return -1;
 
diff --git a/arch/mips/net/bpf_jit_comp64.c b/arch/mips/net/bpf_jit_comp64.c
index 13226819b5f0b..ba58870009024 100644
--- a/arch/mips/net/bpf_jit_comp64.c
+++ b/arch/mips/net/bpf_jit_comp64.c
@@ -460,7 +460,7 @@ static int emit_call(struct jit_context *ctx, const struct bpf_insn *insn)
 	u64 addr;
 
 	/* Decode the call address */
-	if (bpf_jit_get_func_addr(ctx->program, insn, false,
+	if (bpf_jit_get_func_addr(ctx->program, insn, ctx->extra_pass,
 				  &addr, &fixed) < 0)
 		return -1;