Lines Matching refs:prog
38 do { prog = emit_code(prog, bytes, len); } while (0)
330 u8 *prog = *pprog; in push_r12() local
333 *pprog = prog; in push_r12()
338 u8 *prog = *pprog; in push_callee_regs() local
348 *pprog = prog; in push_callee_regs()
353 u8 *prog = *pprog; in pop_r12() local
356 *pprog = prog; in pop_r12()
361 u8 *prog = *pprog; in pop_callee_regs() local
371 *pprog = prog; in pop_callee_regs()
376 u8 *prog = *pprog; in emit_nops() local
390 *pprog = prog; in emit_nops()
400 u8 *prog = *pprog; in emit_fineibt() local
409 *pprog = prog; in emit_fineibt()
414 u8 *prog = *pprog; in emit_kcfi() local
432 *pprog = prog; in emit_kcfi()
437 u8 *prog = *pprog; in emit_cfi() local
441 emit_fineibt(&prog, hash); in emit_cfi()
445 emit_kcfi(&prog, hash); in emit_cfi()
453 *pprog = prog; in emit_cfi()
458 u8 *prog = *pprog; in emit_prologue_tail_call() local
484 *pprog = prog; in emit_prologue_tail_call()
496 u8 *prog = *pprog; in emit_prologue() local
498 emit_cfi(&prog, is_subprog ? cfi_bpf_subprog_hash : cfi_bpf_hash); in emit_prologue()
502 emit_nops(&prog, X86_PATCH_SIZE); in emit_prologue()
511 emit_nops(&prog, 3); /* nop3 */ in emit_prologue()
521 pop_callee_regs(&prog, all_callee_regs_used); in emit_prologue()
522 pop_r12(&prog); in emit_prologue()
537 emit_prologue_tail_call(&prog, is_subprog); in emit_prologue()
538 *pprog = prog; in emit_prologue()
543 u8 *prog = *pprog; in emit_patch() local
552 *pprog = prog; in emit_patch()
579 u8 *prog; in __bpf_arch_text_poke() local
584 prog = old_insn; in __bpf_arch_text_poke()
586 emit_call(&prog, old_addr, ip) : in __bpf_arch_text_poke()
587 emit_jump(&prog, old_addr, ip); in __bpf_arch_text_poke()
594 prog = new_insn; in __bpf_arch_text_poke()
596 emit_call(&prog, new_addr, ip) : in __bpf_arch_text_poke()
597 emit_jump(&prog, new_addr, ip); in __bpf_arch_text_poke()
638 u8 *prog = *pprog; in emit_indirect_jump() local
646 emit_jump(&prog, &__x86_indirect_jump_thunk_array[reg], ip); in emit_indirect_jump()
648 emit_jump(&prog, &__x86_indirect_thunk_array[reg], ip); in emit_indirect_jump()
655 *pprog = prog; in emit_indirect_jump()
660 u8 *prog = *pprog; in emit_return() local
663 emit_jump(&prog, x86_return_thunk, ip); in emit_return()
670 *pprog = prog; in emit_return()
695 u8 *prog = *pprog, *start = *pprog; in emit_bpf_tail_call_indirect() local
712 offset = ctx->tail_call_indirect_label - (prog + 2 - start); in emit_bpf_tail_call_indirect()
722 offset = ctx->tail_call_indirect_label - (prog + 2 - start); in emit_bpf_tail_call_indirect()
735 offset = ctx->tail_call_indirect_label - (prog + 2 - start); in emit_bpf_tail_call_indirect()
742 pop_callee_regs(&prog, all_callee_regs_used); in emit_bpf_tail_call_indirect()
743 pop_r12(&prog); in emit_bpf_tail_call_indirect()
745 pop_callee_regs(&prog, callee_regs_used); in emit_bpf_tail_call_indirect()
747 pop_r12(&prog); in emit_bpf_tail_call_indirect()
770 emit_indirect_jump(&prog, 1 /* rcx */, ip + (prog - start)); in emit_bpf_tail_call_indirect()
773 ctx->tail_call_indirect_label = prog - start; in emit_bpf_tail_call_indirect()
774 *pprog = prog; in emit_bpf_tail_call_indirect()
784 u8 *prog = *pprog, *start = *pprog; in emit_bpf_tail_call_direct() local
794 offset = ctx->tail_call_direct_label - (prog + 2 - start); in emit_bpf_tail_call_direct()
797 poke->tailcall_bypass = ip + (prog - start); in emit_bpf_tail_call_direct()
802 emit_jump(&prog, (u8 *)poke->tailcall_target + X86_PATCH_SIZE, in emit_bpf_tail_call_direct()
809 pop_callee_regs(&prog, all_callee_regs_used); in emit_bpf_tail_call_direct()
810 pop_r12(&prog); in emit_bpf_tail_call_direct()
812 pop_callee_regs(&prog, callee_regs_used); in emit_bpf_tail_call_direct()
814 pop_r12(&prog); in emit_bpf_tail_call_direct()
826 emit_nops(&prog, X86_PATCH_SIZE); in emit_bpf_tail_call_direct()
829 ctx->tail_call_direct_label = prog - start; in emit_bpf_tail_call_direct()
831 *pprog = prog; in emit_bpf_tail_call_direct()
834 static void bpf_tail_call_direct_fixup(struct bpf_prog *prog) in bpf_tail_call_direct_fixup() argument
841 for (i = 0; i < prog->aux->size_poke_tab; i++) { in bpf_tail_call_direct_fixup()
842 poke = &prog->aux->poke_tab[i]; in bpf_tail_call_direct_fixup()
843 if (poke->aux && poke->aux != prog->aux) in bpf_tail_call_direct_fixup()
874 u8 *prog = *pprog; in emit_mov_imm32() local
908 *pprog = prog; in emit_mov_imm32()
915 u8 *prog = *pprog; in emit_mov_imm64() local
924 emit_mov_imm32(&prog, false, dst_reg, imm32_lo); in emit_mov_imm64()
926 emit_mov_imm32(&prog, true, dst_reg, imm32_lo); in emit_mov_imm64()
934 *pprog = prog; in emit_mov_imm64()
939 u8 *prog = *pprog; in emit_mov_reg() local
951 *pprog = prog; in emit_mov_reg()
957 u8 *prog = *pprog; in emit_movsx_reg() local
983 *pprog = prog; in emit_movsx_reg()
989 u8 *prog = *pprog; in emit_insn_suffix() local
1003 *pprog = prog; in emit_insn_suffix()
1008 u8 *prog = *pprog; in emit_insn_suffix_SIB() local
1015 *pprog = prog; in emit_insn_suffix_SIB()
1023 u8 *prog = *pprog; in maybe_emit_mod() local
1029 *pprog = prog; in maybe_emit_mod()
1037 u8 *prog = *pprog; in maybe_emit_1mod() local
1043 *pprog = prog; in maybe_emit_1mod()
1049 u8 *prog = *pprog; in emit_ldx() local
1072 emit_insn_suffix(&prog, src_reg, dst_reg, off); in emit_ldx()
1073 *pprog = prog; in emit_ldx()
1079 u8 *prog = *pprog; in emit_ldsx() local
1095 emit_insn_suffix(&prog, src_reg, dst_reg, off); in emit_ldsx()
1096 *pprog = prog; in emit_ldsx()
1101 u8 *prog = *pprog; in emit_ldx_index() local
1121 emit_insn_suffix_SIB(&prog, src_reg, dst_reg, index_reg, off); in emit_ldx_index()
1122 *pprog = prog; in emit_ldx_index()
1133 u8 *prog = *pprog; in emit_stx() local
1160 emit_insn_suffix(&prog, dst_reg, src_reg, off); in emit_stx()
1161 *pprog = prog; in emit_stx()
1167 u8 *prog = *pprog; in emit_stx_index() local
1187 emit_insn_suffix_SIB(&prog, dst_reg, src_reg, index_reg, off); in emit_stx_index()
1188 *pprog = prog; in emit_stx_index()
1199 u8 *prog = *pprog; in emit_st_index() local
1219 emit_insn_suffix_SIB(&prog, dst_reg, 0, index_reg, off); in emit_st_index()
1221 *pprog = prog; in emit_st_index()
1232 u8 *prog = *pprog; in emit_atomic() local
1236 maybe_emit_mod(&prog, dst_reg, src_reg, bpf_size == BPF_DW); in emit_atomic()
1264 emit_insn_suffix(&prog, dst_reg, src_reg, off); in emit_atomic()
1266 *pprog = prog; in emit_atomic()
1273 u8 *prog = *pprog; in emit_atomic_index() local
1313 emit_insn_suffix_SIB(&prog, dst_reg, src_reg, index_reg, off); in emit_atomic_index()
1314 *pprog = prog; in emit_atomic_index()
1362 u8 *prog = *pprog; in emit_3vex() local
1392 *pprog = prog; in emit_3vex()
1398 u8 *prog = *pprog; in emit_shiftx() local
1402 emit_3vex(&prog, r, false, r, m, is64, src_reg, false, op); in emit_shiftx()
1404 *pprog = prog; in emit_shiftx()
1407 #define INSN_SZ_DIFF (((addrs[i] - addrs[i - 1]) - (prog - temp)))
1427 u8 *prog = temp; in do_jit() local
1435 emit_prologue(&prog, bpf_prog->aux->stack_depth, in do_jit()
1446 push_r12(&prog); in do_jit()
1447 push_callee_regs(&prog, all_callee_regs_used); in do_jit()
1450 push_r12(&prog); in do_jit()
1451 push_callee_regs(&prog, callee_regs_used); in do_jit()
1454 emit_mov_imm64(&prog, X86_REG_R12, in do_jit()
1457 ilen = prog - temp; in do_jit()
1462 prog = temp; in do_jit()
1488 maybe_emit_mod(&prog, dst_reg, src_reg, in do_jit()
1498 emit_mov_reg(&prog, false, dst_reg, src_reg); in do_jit()
1500 maybe_emit_1mod(&prog, dst_reg, true); in do_jit()
1504 maybe_emit_1mod(&prog, dst_reg, true); in do_jit()
1511 maybe_emit_1mod(&prog, dst_reg, true); in do_jit()
1518 maybe_emit_mod(&prog, dst_reg, dst_reg, false); in do_jit()
1523 maybe_emit_mod(&prog, AUX_REG, dst_reg, true); in do_jit()
1540 emit_mov_reg(&prog, in do_jit()
1544 emit_movsx_reg(&prog, insn->off, in do_jit()
1552 maybe_emit_1mod(&prog, dst_reg, in do_jit()
1567 maybe_emit_1mod(&prog, dst_reg, in do_jit()
1607 emit_mov_imm32(&prog, BPF_CLASS(insn->code) == BPF_ALU64, in do_jit()
1612 emit_mov_imm64(&prog, dst_reg, insn[1].imm, insn[0].imm); in do_jit()
1648 emit_mov_reg(&prog, is64, BPF_REG_0, dst_reg); in do_jit()
1658 maybe_emit_1mod(&prog, src_reg, is64); in do_jit()
1667 maybe_emit_1mod(&prog, src_reg, is64); in do_jit()
1674 emit_mov_reg(&prog, is64, dst_reg, BPF_REG_3); in do_jit()
1678 emit_mov_reg(&prog, is64, dst_reg, BPF_REG_0); in do_jit()
1689 maybe_emit_mod(&prog, dst_reg, dst_reg, in do_jit()
1705 maybe_emit_mod(&prog, src_reg, dst_reg, in do_jit()
1719 maybe_emit_1mod(&prog, dst_reg, in do_jit()
1753 emit_shiftx(&prog, dst_reg, src_reg, w, op); in do_jit()
1772 maybe_emit_1mod(&prog, dst_reg, in do_jit()
1886 emit_stx(&prog, BPF_SIZE(insn->code), dst_reg, src_reg, insn->off); in do_jit()
1893 start_of_ldx = prog; in do_jit()
1894 emit_st_r12(&prog, BPF_SIZE(insn->code), dst_reg, insn->off, insn->imm); in do_jit()
1906 start_of_ldx = prog; in do_jit()
1908 emit_ldx_r12(&prog, BPF_SIZE(insn->code), dst_reg, src_reg, insn->off); in do_jit()
1910 emit_stx_r12(&prog, BPF_SIZE(insn->code), dst_reg, src_reg, insn->off); in do_jit()
1934 ex->fixup = (prog - start_of_ldx) | in do_jit()
1969 emit_mov_imm64(&prog, BPF_REG_AX, (long)VSYSCALL_ADDR >> 32, in do_jit()
1977 maybe_emit_1mod(&prog, AUX_REG, true); in do_jit()
1982 maybe_emit_mod(&prog, AUX_REG, BPF_REG_AX, true); in do_jit()
1986 emit_mov_imm64(&prog, BPF_REG_AX, (long)limit >> 32, in do_jit()
1990 maybe_emit_mod(&prog, AUX_REG, BPF_REG_AX, true); in do_jit()
1995 end_of_jmp = prog; in do_jit()
1998 emit_mov_imm32(&prog, false, dst_reg, 0); in do_jit()
2003 start_of_ldx = prog; in do_jit()
2008 emit_ldsx(&prog, BPF_SIZE(insn->code), dst_reg, src_reg, insn_off); in do_jit()
2010 emit_ldx(&prog, BPF_SIZE(insn->code), dst_reg, src_reg, insn_off); in do_jit()
2018 start_of_ldx[-1] = prog - start_of_ldx; in do_jit()
2053 ex->fixup = (prog - start_of_ldx) | (reg2pt_regs[dst_reg] << 8); in do_jit()
2073 emit_mov_reg(&prog, true, BPF_REG_AX, BPF_REG_0); in do_jit()
2079 branch_target = prog; in do_jit()
2081 emit_ldx(&prog, BPF_SIZE(insn->code), in do_jit()
2087 emit_mov_reg(&prog, is64, AUX_REG, BPF_REG_0); in do_jit()
2088 maybe_emit_mod(&prog, AUX_REG, real_src_reg, is64); in do_jit()
2092 err = emit_atomic(&prog, BPF_CMPXCHG, in do_jit()
2102 EMIT2(X86_JNE, -(prog - branch_target) - 2); in do_jit()
2104 emit_mov_reg(&prog, is64, real_src_reg, BPF_REG_0); in do_jit()
2106 emit_mov_reg(&prog, true, BPF_REG_0, BPF_REG_AX); in do_jit()
2110 err = emit_atomic(&prog, insn->imm, dst_reg, src_reg, in do_jit()
2118 start_of_ldx = prog; in do_jit()
2119 err = emit_atomic_index(&prog, insn->imm, BPF_SIZE(insn->code), in do_jit()
2136 ip += x86_call_depth_emit_accounting(&prog, func, ip); in do_jit()
2137 if (emit_call(&prog, func, ip)) in do_jit()
2146 &prog, image + addrs[i - 1], in do_jit()
2152 &prog, in do_jit()
2181 maybe_emit_mod(&prog, dst_reg, src_reg, in do_jit()
2189 maybe_emit_mod(&prog, dst_reg, src_reg, in do_jit()
2197 maybe_emit_1mod(&prog, dst_reg, in do_jit()
2224 maybe_emit_mod(&prog, dst_reg, dst_reg, in do_jit()
2231 maybe_emit_1mod(&prog, dst_reg, in do_jit()
2307 emit_nops(&prog, nops); in do_jit()
2361 emit_nops(&prog, nops); in do_jit()
2386 emit_nops(&prog, INSN_SZ_DIFF - 2); in do_jit()
2406 pop_callee_regs(&prog, all_callee_regs_used); in do_jit()
2407 pop_r12(&prog); in do_jit()
2409 pop_callee_regs(&prog, callee_regs_used); in do_jit()
2411 pop_r12(&prog); in do_jit()
2414 emit_return(&prog, image + addrs[i - 1] + (prog - temp)); in do_jit()
2428 ilen = prog - temp; in do_jit()
2452 prog = temp; in do_jit()
2467 u8 *prog; in clean_stack_garbage() local
2499 prog = *pprog; in clean_stack_garbage()
2506 *pprog = prog; in clean_stack_garbage()
2527 static void save_args(const struct btf_func_model *m, u8 **prog, in save_args() argument
2570 emit_ldx(prog, BPF_DW, BPF_REG_0, BPF_REG_FP, in save_args()
2572 emit_stx(prog, BPF_DW, BPF_REG_FP, BPF_REG_0, in save_args()
2592 emit_stx(prog, BPF_DW, BPF_REG_FP, in save_args()
2601 clean_stack_garbage(m, prog, nr_stack_slots, first_off); in save_args()
2604 static void restore_regs(const struct btf_func_model *m, u8 **prog, in restore_regs() argument
2620 emit_ldx(prog, BPF_DW, in restore_regs()
2641 u8 *prog = *pprog; in invoke_bpf_prog() local
2644 struct bpf_prog *p = l->link.prog; in invoke_bpf_prog()
2648 emit_mov_imm64(&prog, BPF_REG_1, (long) cookie >> 32, (u32) (long) cookie); in invoke_bpf_prog()
2657 emit_stx(&prog, BPF_DW, BPF_REG_FP, BPF_REG_1, -run_ctx_off + ctx_cookie_off); in invoke_bpf_prog()
2660 emit_mov_imm64(&prog, BPF_REG_1, (long) p >> 32, (u32) (long) p); in invoke_bpf_prog()
2667 if (emit_rsb_call(&prog, bpf_trampoline_enter(p), image + (prog - (u8 *)rw_image))) in invoke_bpf_prog()
2670 emit_mov_reg(&prog, true, BPF_REG_6, BPF_REG_0); in invoke_bpf_prog()
2677 jmp_insn = prog; in invoke_bpf_prog()
2678 emit_nops(&prog, 2); in invoke_bpf_prog()
2687 emit_mov_imm64(&prog, BPF_REG_2, in invoke_bpf_prog()
2691 if (emit_rsb_call(&prog, p->bpf_func, image + (prog - (u8 *)rw_image))) in invoke_bpf_prog()
2703 emit_stx(&prog, BPF_DW, BPF_REG_FP, BPF_REG_0, -8); in invoke_bpf_prog()
2707 jmp_insn[1] = prog - jmp_insn - 2; in invoke_bpf_prog()
2710 emit_mov_imm64(&prog, BPF_REG_1, (long) p >> 32, (u32) (long) p); in invoke_bpf_prog()
2712 emit_mov_reg(&prog, true, BPF_REG_2, BPF_REG_6); in invoke_bpf_prog()
2718 if (emit_rsb_call(&prog, bpf_trampoline_exit(p), image + (prog - (u8 *)rw_image))) in invoke_bpf_prog()
2721 *pprog = prog; in invoke_bpf_prog()
2727 u8 *target, *prog = *pprog; in emit_align() local
2729 target = PTR_ALIGN(prog, align); in emit_align()
2730 if (target != prog) in emit_align()
2731 emit_nops(&prog, target - prog); in emit_align()
2733 *pprog = prog; in emit_align()
2738 u8 *prog = *pprog; in emit_cond_near_jump() local
2747 *pprog = prog; in emit_cond_near_jump()
2757 u8 *prog = *pprog; in invoke_bpf() local
2760 if (invoke_bpf_prog(m, &prog, tl->links[i], stack_size, in invoke_bpf()
2764 *pprog = prog; in invoke_bpf()
2773 u8 *prog = *pprog; in invoke_bpf_mod_ret() local
2779 emit_mov_imm32(&prog, false, BPF_REG_0, 0); in invoke_bpf_mod_ret()
2780 emit_stx(&prog, BPF_DW, BPF_REG_FP, BPF_REG_0, -8); in invoke_bpf_mod_ret()
2782 if (invoke_bpf_prog(m, &prog, tl->links[i], stack_size, run_ctx_off, true, in invoke_bpf_mod_ret()
2798 branches[i] = prog; in invoke_bpf_mod_ret()
2799 emit_nops(&prog, 4 + 2); in invoke_bpf_mod_ret()
2802 *pprog = prog; in invoke_bpf_mod_ret()
2883 u8 *prog; in __arch_prepare_bpf_trampoline() local
2979 prog = rw_image; in __arch_prepare_bpf_trampoline()
2985 emit_cfi(&prog, cfi_get_func_hash(func_addr)); in __arch_prepare_bpf_trampoline()
2991 x86_call_depth_emit_accounting(&prog, NULL, image); in __arch_prepare_bpf_trampoline()
3005 emit_stx(&prog, BPF_DW, BPF_REG_FP, BPF_REG_6, -rbx_off); in __arch_prepare_bpf_trampoline()
3011 emit_mov_imm64(&prog, BPF_REG_0, 0, (u32) nr_regs); in __arch_prepare_bpf_trampoline()
3012 emit_stx(&prog, BPF_DW, BPF_REG_FP, BPF_REG_0, -nregs_off); in __arch_prepare_bpf_trampoline()
3019 emit_mov_imm64(&prog, BPF_REG_0, (long) func_addr >> 32, (u32) (long) func_addr); in __arch_prepare_bpf_trampoline()
3020 emit_stx(&prog, BPF_DW, BPF_REG_FP, BPF_REG_0, -ip_off); in __arch_prepare_bpf_trampoline()
3023 save_args(m, &prog, regs_off, false); in __arch_prepare_bpf_trampoline()
3027 emit_mov_imm64(&prog, BPF_REG_1, (long) im >> 32, (u32) (long) im); in __arch_prepare_bpf_trampoline()
3028 if (emit_rsb_call(&prog, __bpf_tramp_enter, in __arch_prepare_bpf_trampoline()
3029 image + (prog - (u8 *)rw_image))) { in __arch_prepare_bpf_trampoline()
3036 if (invoke_bpf(m, &prog, fentry, regs_off, run_ctx_off, in __arch_prepare_bpf_trampoline()
3047 if (invoke_bpf_mod_ret(m, &prog, fmod_ret, regs_off, in __arch_prepare_bpf_trampoline()
3055 restore_regs(m, &prog, regs_off); in __arch_prepare_bpf_trampoline()
3056 save_args(m, &prog, arg_stack_off, true); in __arch_prepare_bpf_trampoline()
3066 emit_ldx(&prog, BPF_DW, BPF_REG_6, BPF_REG_FP, 8); in __arch_prepare_bpf_trampoline()
3070 if (emit_rsb_call(&prog, orig_call, image + (prog - (u8 *)rw_image))) { in __arch_prepare_bpf_trampoline()
3076 emit_stx(&prog, BPF_DW, BPF_REG_FP, BPF_REG_0, -8); in __arch_prepare_bpf_trampoline()
3077 im->ip_after_call = image + (prog - (u8 *)rw_image); in __arch_prepare_bpf_trampoline()
3078 emit_nops(&prog, X86_PATCH_SIZE); in __arch_prepare_bpf_trampoline()
3087 emit_align(&prog, 16); in __arch_prepare_bpf_trampoline()
3092 emit_cond_near_jump(&branches[i], image + (prog - (u8 *)rw_image), in __arch_prepare_bpf_trampoline()
3098 if (invoke_bpf(m, &prog, fexit, regs_off, run_ctx_off, in __arch_prepare_bpf_trampoline()
3106 restore_regs(m, &prog, regs_off); in __arch_prepare_bpf_trampoline()
3113 im->ip_epilogue = image + (prog - (u8 *)rw_image); in __arch_prepare_bpf_trampoline()
3115 emit_mov_imm64(&prog, BPF_REG_1, (long) im >> 32, (u32) (long) im); in __arch_prepare_bpf_trampoline()
3116 if (emit_rsb_call(&prog, __bpf_tramp_exit, image + (prog - (u8 *)rw_image))) { in __arch_prepare_bpf_trampoline()
3129 emit_ldx(&prog, BPF_DW, BPF_REG_0, BPF_REG_FP, -8); in __arch_prepare_bpf_trampoline()
3131 emit_ldx(&prog, BPF_DW, BPF_REG_6, BPF_REG_FP, -rbx_off); in __arch_prepare_bpf_trampoline()
3137 emit_return(&prog, image + (prog - (u8 *)rw_image)); in __arch_prepare_bpf_trampoline()
3139 if (WARN_ON_ONCE(prog > (u8 *)rw_image_end - BPF_INSN_SAFETY)) { in __arch_prepare_bpf_trampoline()
3143 ret = prog - (u8 *)rw_image + BPF_INSN_SAFETY; in __arch_prepare_bpf_trampoline()
3220 u8 *jg_reloc, *prog = *pprog; in emit_bpf_dispatcher() local
3233 err = emit_cond_near_jump(&prog, /* je func */ in emit_bpf_dispatcher()
3234 (void *)progs[a], image + (prog - buf), in emit_bpf_dispatcher()
3239 emit_indirect_jump(&prog, 2 /* rdx */, image + (prog - buf)); in emit_bpf_dispatcher()
3241 *pprog = prog; in emit_bpf_dispatcher()
3261 jg_reloc = prog; in emit_bpf_dispatcher()
3263 err = emit_bpf_dispatcher(&prog, a, a + pivot, /* emit lower_part */ in emit_bpf_dispatcher()
3273 emit_align(&prog, 16); in emit_bpf_dispatcher()
3274 jg_offset = prog - jg_reloc; in emit_bpf_dispatcher()
3277 err = emit_bpf_dispatcher(&prog, a + pivot + 1, /* emit upper_part */ in emit_bpf_dispatcher()
3282 *pprog = prog; in emit_bpf_dispatcher()
3300 u8 *prog = buf; in arch_prepare_bpf_dispatcher() local
3303 return emit_bpf_dispatcher(&prog, 0, num_funcs - 1, funcs, image, buf); in arch_prepare_bpf_dispatcher()
3318 struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog) in bpf_int_jit_compile() argument
3322 struct bpf_prog *tmp, *orig_prog = prog; in bpf_int_jit_compile()
3335 if (!prog->jit_requested) in bpf_int_jit_compile()
3338 tmp = bpf_jit_blind_constants(prog); in bpf_int_jit_compile()
3345 if (tmp != prog) { in bpf_int_jit_compile()
3347 prog = tmp; in bpf_int_jit_compile()
3350 jit_data = prog->aux->jit_data; in bpf_int_jit_compile()
3354 prog = orig_prog; in bpf_int_jit_compile()
3357 prog->aux->jit_data = jit_data; in bpf_int_jit_compile()
3371 addrs = kvmalloc_array(prog->len + 1, sizeof(*addrs), GFP_KERNEL); in bpf_int_jit_compile()
3373 prog = orig_prog; in bpf_int_jit_compile()
3381 for (proglen = 0, i = 0; i <= prog->len; i++) { in bpf_int_jit_compile()
3397 proglen = do_jit(prog, addrs, image, rw_image, oldproglen, &ctx, padding); in bpf_int_jit_compile()
3407 prog = orig_prog; in bpf_int_jit_compile()
3409 prog->bpf_func = NULL; in bpf_int_jit_compile()
3410 prog->jited = 0; in bpf_int_jit_compile()
3411 prog->jited_len = 0; in bpf_int_jit_compile()
3431 u32 extable_size = prog->aux->num_exentries * in bpf_int_jit_compile()
3439 prog = orig_prog; in bpf_int_jit_compile()
3442 prog->aux->extable = (void *) image + roundup(proglen, align); in bpf_int_jit_compile()
3449 bpf_jit_dump(prog->len, proglen, pass + 1, rw_image); in bpf_int_jit_compile()
3452 if (!prog->is_func || extra_pass) { in bpf_int_jit_compile()
3466 bpf_tail_call_direct_fixup(prog); in bpf_int_jit_compile()
3482 prog->bpf_func = (void *)image + cfi_get_offset(); in bpf_int_jit_compile()
3483 prog->jited = 1; in bpf_int_jit_compile()
3484 prog->jited_len = proglen - cfi_get_offset(); in bpf_int_jit_compile()
3486 prog = orig_prog; in bpf_int_jit_compile()
3489 if (!image || !prog->is_func || extra_pass) { in bpf_int_jit_compile()
3491 bpf_prog_fill_jited_linfo(prog, addrs + 1); in bpf_int_jit_compile()
3495 prog->aux->jit_data = NULL; in bpf_int_jit_compile()
3499 bpf_jit_prog_release_other(prog, prog == orig_prog ? in bpf_int_jit_compile()
3501 return prog; in bpf_int_jit_compile()
3527 void bpf_jit_free(struct bpf_prog *prog) in bpf_jit_free() argument
3529 if (prog->jited) { in bpf_jit_free()
3530 struct x64_jit_data *jit_data = prog->aux->jit_data; in bpf_jit_free()
3544 prog->bpf_func = (void *)prog->bpf_func - cfi_get_offset(); in bpf_jit_free()
3545 hdr = bpf_jit_binary_pack_hdr(prog); in bpf_jit_free()
3547 WARN_ON_ONCE(!bpf_prog_kallsyms_verify_off(prog)); in bpf_jit_free()
3550 bpf_prog_unlock_free(prog); in bpf_jit_free()