101 	struct bpf_prog			*prog;  member
125 static void vm_dump(const struct bpf_prog *prog)  in vm_dump()  argument
129 		dump_bytes((u8 *)prog->insns, 8 * prog->len, " VM  ");  in vm_dump()
147 	if (bpf_jit_enable <= 1 || !ctx->prog->jited)  in jit_dump()
155 	bpf_jit_dump(ctx->prog->len, ctx->jit.len, pass, ctx->jit.buf);  in jit_dump()
160 static int jit_ctx_init(struct jit_context *ctx, struct bpf_prog *prog)  in jit_ctx_init()  argument
164 	ctx->orig_prog = prog;  in jit_ctx_init()
167 	ctx->prog = bpf_jit_blind_constants(prog);  in jit_ctx_init()
168 	if (IS_ERR(ctx->prog))  in jit_ctx_init()
169 		return PTR_ERR(ctx->prog);  in jit_ctx_init()
170 	ctx->blinded = (ctx->prog != ctx->orig_prog);  in jit_ctx_init()
173 	ctx->do_zext = !ctx->prog->aux->verifier_zext;  in jit_ctx_init()
175 	ctx->is_extra_pass = ctx->prog->jited;  in jit_ctx_init()
176 	ctx->user_bpf_prog = ctx->prog->is_func;  in jit_ctx_init()
220 			bpf_jit_prog_release_other(ctx->prog, ctx->orig_prog);  in jit_ctx_cleanup()
222 			bpf_jit_prog_release_other(ctx->orig_prog, ctx->prog);  in jit_ctx_cleanup()
252 	const struct bpf_insn *insn = ctx->prog->insnsi;  in analyze_reg_usage()
254 	for (i = 0; i < ctx->prog->len; i++) {  in analyze_reg_usage()
264 	ctx->frame_size = ctx->prog->aux->stack_depth;  in analyze_reg_usage()
333 	return (insn - ctx->prog->insnsi);  in get_index_for_insn()
381 static inline bool is_last_insn(const struct bpf_prog *prog, u32 idx)  in is_last_insn()  argument
383 	return idx == (prog->len - 1);  in is_last_insn()
426 	return (idx >= 0 && idx < ctx->prog->len);  in check_insn_idx_valid()
675 	ret = bpf_jit_get_func_addr(ctx->prog, insn, ctx->is_extra_pass,  in handle_call()
711 	if (is_last_insn(ctx->prog, idx)) {  in handle_ld_imm64()
738 	const struct bpf_insn *insn = &ctx->prog->insnsi[idx];  in handle_insn()
1027 		if (is_last_insn(ctx->prog, idx))  in handle_insn()
1056 	const struct bpf_prog *prog = ctx->prog;  in handle_body()  local
1068 	for (u32 i = 0; i < prog->len; i++) {  in handle_body()
1115 	ctx->bpf2insn = kcalloc(ctx->prog->len, sizeof(ctx->jit.len),  in jit_prepare_early_mem_alloc()
1223 	struct bpf_prog *prog = ctx->prog;  in jit_finalize()  local
1229 		prog->aux->jit_data = (void *)ctx->jit_data;  in jit_finalize()
1242 		prog->aux->jit_data = NULL;  in jit_finalize()
1243 		bpf_prog_fill_jited_linfo(prog, ctx->bpf2insn);  in jit_finalize()
1247 	prog->bpf_func = (void *)ctx->jit.buf;  in jit_finalize()
1248 	prog->jited_len = ctx->jit.len;  in jit_finalize()
1249 	prog->jited = 1;  in jit_finalize()
1262 static inline int check_jit_context(const struct bpf_prog *prog)  in check_jit_context()  argument
1264 	if (!prog->aux->jit_data) {  in check_jit_context()
1276 		(struct arc_jit_data *)ctx->prog->aux->jit_data;  in jit_resume_context()
1283 	ctx->jit.buf = (u8 *)ctx->prog->bpf_func;  in jit_resume_context()
1284 	ctx->jit.len = ctx->prog->jited_len;  in jit_resume_context()
1309 	const struct bpf_prog *prog = ctx->prog;  in jit_patch_relocations()  local
1313 	for (u32 i = 0; i < prog->len; i++) {  in jit_patch_relocations()
1314 		const struct bpf_insn *insn = &prog->insnsi[i];  in jit_patch_relocations()
1338 static struct bpf_prog *do_normal_pass(struct bpf_prog *prog)  in do_normal_pass()  argument
1343 	if (!prog->jit_requested)  in do_normal_pass()
1344 		return prog;  in do_normal_pass()
1346 	if (jit_ctx_init(&ctx, prog)) {  in do_normal_pass()
1348 		return prog;  in do_normal_pass()
1354 		return prog;  in do_normal_pass()
1359 		return prog;  in do_normal_pass()
1364 		return prog;  in do_normal_pass()
1367 	return ctx.prog;  in do_normal_pass()
1377 static struct bpf_prog *do_extra_pass(struct bpf_prog *prog)  in do_extra_pass()  argument
1382 	if (check_jit_context(prog))  in do_extra_pass()
1383 		return prog;  in do_extra_pass()
1385 	if (jit_ctx_init(&ctx, prog)) {  in do_extra_pass()
1387 		return prog;  in do_extra_pass()
1392 		return prog;  in do_extra_pass()
1397 		return prog;  in do_extra_pass()
1402 		return prog;  in do_extra_pass()
1405 	return ctx.prog;  in do_extra_pass()
1414 struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog)  in bpf_int_jit_compile()  argument
1416 	vm_dump(prog);  in bpf_int_jit_compile()
1419 	if (!prog->jited)  in bpf_int_jit_compile()
1420 		return do_normal_pass(prog);  in bpf_int_jit_compile()
1422 		return do_extra_pass(prog);  in bpf_int_jit_compile()
1424 	return prog;  in bpf_int_jit_compile()