328 static void push_r12(u8 **pprog)  in push_r12()  argument
330 	u8 *prog = *pprog;  in push_r12()
333 	*pprog = prog;  in push_r12()
336 static void push_callee_regs(u8 **pprog, bool *callee_regs_used)  in push_callee_regs()  argument
338 	u8 *prog = *pprog;  in push_callee_regs()
348 	*pprog = prog;  in push_callee_regs()
351 static void pop_r12(u8 **pprog)  in pop_r12()  argument
353 	u8 *prog = *pprog;  in pop_r12()
356 	*pprog = prog;  in pop_r12()
359 static void pop_callee_regs(u8 **pprog, bool *callee_regs_used)  in pop_callee_regs()  argument
361 	u8 *prog = *pprog;  in pop_callee_regs()
371 	*pprog = prog;  in pop_callee_regs()
374 static void emit_nops(u8 **pprog, int len)  in emit_nops()  argument
376 	u8 *prog = *pprog;  in emit_nops()
390 	*pprog = prog;  in emit_nops()
398 static void emit_fineibt(u8 **pprog, u32 hash)  in emit_fineibt()  argument
400 	u8 *prog = *pprog;  in emit_fineibt()
409 	*pprog = prog;  in emit_fineibt()
412 static void emit_kcfi(u8 **pprog, u32 hash)  in emit_kcfi()  argument
414 	u8 *prog = *pprog;  in emit_kcfi()
432 	*pprog = prog;  in emit_kcfi()
435 static void emit_cfi(u8 **pprog, u32 hash)  in emit_cfi()  argument
437 	u8 *prog = *pprog;  in emit_cfi()
453 	*pprog = prog;  in emit_cfi()
456 static void emit_prologue_tail_call(u8 **pprog, bool is_subprog)  in emit_prologue_tail_call()  argument
458 	u8 *prog = *pprog;  in emit_prologue_tail_call()
484 	*pprog = prog;  in emit_prologue_tail_call()
492 static void emit_prologue(u8 **pprog, u32 stack_depth, bool ebpf_from_cbpf,  in emit_prologue()  argument
496 	u8 *prog = *pprog;  in emit_prologue()
538 	*pprog = prog;  in emit_prologue()
541 static int emit_patch(u8 **pprog, void *func, void *ip, u8 opcode)  in emit_patch()  argument
543 	u8 *prog = *pprog;  in emit_patch()
552 	*pprog = prog;  in emit_patch()
556 static int emit_call(u8 **pprog, void *func, void *ip)  in emit_call()  argument
558 	return emit_patch(pprog, func, ip, 0xE8);  in emit_call()
561 static int emit_rsb_call(u8 **pprog, void *func, void *ip)  in emit_rsb_call()  argument
564 	ip += x86_call_depth_emit_accounting(pprog, func, ip);  in emit_rsb_call()
565 	return emit_patch(pprog, func, ip, 0xE8);  in emit_rsb_call()
568 static int emit_jump(u8 **pprog, void *func, void *ip)  in emit_jump()  argument
570 	return emit_patch(pprog, func, ip, 0xE9);  in emit_jump()
636 static void emit_indirect_jump(u8 **pprog, int reg, u8 *ip)  in emit_indirect_jump()  argument
638 	u8 *prog = *pprog;  in emit_indirect_jump()
655 	*pprog = prog;  in emit_indirect_jump()
658 static void emit_return(u8 **pprog, u8 *ip)  in emit_return()  argument
660 	u8 *prog = *pprog;  in emit_return()
670 	*pprog = prog;  in emit_return()
690 					u8 **pprog, bool *callee_regs_used,  in emit_bpf_tail_call_indirect()  argument
695 	u8 *prog = *pprog, *start = *pprog;  in emit_bpf_tail_call_indirect()
774 	*pprog = prog;  in emit_bpf_tail_call_indirect()
779 				      u8 **pprog, u8 *ip,  in emit_bpf_tail_call_direct()  argument
784 	u8 *prog = *pprog, *start = *pprog;  in emit_bpf_tail_call_direct()
831 	*pprog = prog;  in emit_bpf_tail_call_direct()
871 static void emit_mov_imm32(u8 **pprog, bool sign_propagate,  in emit_mov_imm32()  argument
874 	u8 *prog = *pprog;  in emit_mov_imm32()
908 	*pprog = prog;  in emit_mov_imm32()
911 static void emit_mov_imm64(u8 **pprog, u32 dst_reg,  in emit_mov_imm64()  argument
915 	u8 *prog = *pprog;  in emit_mov_imm64()
934 	*pprog = prog;  in emit_mov_imm64()
937 static void emit_mov_reg(u8 **pprog, bool is64, u32 dst_reg, u32 src_reg)  in emit_mov_reg()  argument
939 	u8 *prog = *pprog;  in emit_mov_reg()
951 	*pprog = prog;  in emit_mov_reg()
954 static void emit_movsx_reg(u8 **pprog, int num_bits, bool is64, u32 dst_reg,  in emit_movsx_reg()  argument
957 	u8 *prog = *pprog;  in emit_movsx_reg()
983 	*pprog = prog;  in emit_movsx_reg()
987 static void emit_insn_suffix(u8 **pprog, u32 ptr_reg, u32 val_reg, int off)  in emit_insn_suffix()  argument
989 	u8 *prog = *pprog;  in emit_insn_suffix()
1003 	*pprog = prog;  in emit_insn_suffix()
1006 static void emit_insn_suffix_SIB(u8 **pprog, u32 ptr_reg, u32 val_reg, u32 index_reg, int off)  in emit_insn_suffix_SIB()  argument
1008 	u8 *prog = *pprog;  in emit_insn_suffix_SIB()
1015 	*pprog = prog;  in emit_insn_suffix_SIB()
1021 static void maybe_emit_mod(u8 **pprog, u32 dst_reg, u32 src_reg, bool is64)  in maybe_emit_mod()  argument
1023 	u8 *prog = *pprog;  in maybe_emit_mod()
1029 	*pprog = prog;  in maybe_emit_mod()
1035 static void maybe_emit_1mod(u8 **pprog, u32 reg, bool is64)  in maybe_emit_1mod()  argument
1037 	u8 *prog = *pprog;  in maybe_emit_1mod()
1043 	*pprog = prog;  in maybe_emit_1mod()
1047 static void emit_ldx(u8 **pprog, u32 size, u32 dst_reg, u32 src_reg, int off)  in emit_ldx()  argument
1049 	u8 *prog = *pprog;  in emit_ldx()
1073 	*pprog = prog;  in emit_ldx()
1077 static void emit_ldsx(u8 **pprog, u32 size, u32 dst_reg, u32 src_reg, int off)  in emit_ldsx()  argument
1079 	u8 *prog = *pprog;  in emit_ldsx()
1096 	*pprog = prog;  in emit_ldsx()
1099 static void emit_ldx_index(u8 **pprog, u32 size, u32 dst_reg, u32 src_reg, u32 index_reg, int off)  in emit_ldx_index()  argument
1101 	u8 *prog = *pprog;  in emit_ldx_index()
1122 	*pprog = prog;  in emit_ldx_index()
1125 static void emit_ldx_r12(u8 **pprog, u32 size, u32 dst_reg, u32 src_reg, int off)  in emit_ldx_r12()  argument
1127 	emit_ldx_index(pprog, size, dst_reg, src_reg, X86_REG_R12, off);  in emit_ldx_r12()
1131 static void emit_stx(u8 **pprog, u32 size, u32 dst_reg, u32 src_reg, int off)  in emit_stx()  argument
1133 	u8 *prog = *pprog;  in emit_stx()
1161 	*pprog = prog;  in emit_stx()
1165 static void emit_stx_index(u8 **pprog, u32 size, u32 dst_reg, u32 src_reg, u32 index_reg, int off)  in emit_stx_index()  argument
1167 	u8 *prog = *pprog;  in emit_stx_index()
1188 	*pprog = prog;  in emit_stx_index()
1191 static void emit_stx_r12(u8 **pprog, u32 size, u32 dst_reg, u32 src_reg, int off)  in emit_stx_r12()  argument
1193 	emit_stx_index(pprog, size, dst_reg, src_reg, X86_REG_R12, off);  in emit_stx_r12()
1197 static void emit_st_index(u8 **pprog, u32 size, u32 dst_reg, u32 index_reg, int off, int imm)  in emit_st_index()  argument
1199 	u8 *prog = *pprog;  in emit_st_index()
1221 	*pprog = prog;  in emit_st_index()
1224 static void emit_st_r12(u8 **pprog, u32 size, u32 dst_reg, int off, int imm)  in emit_st_r12()  argument
1226 	emit_st_index(pprog, size, dst_reg, X86_REG_R12, off, imm);  in emit_st_r12()
1229 static int emit_atomic(u8 **pprog, u8 atomic_op,  in emit_atomic()  argument
1232 	u8 *prog = *pprog;  in emit_atomic()
1266 	*pprog = prog;  in emit_atomic()
1270 static int emit_atomic_index(u8 **pprog, u8 atomic_op, u32 size,  in emit_atomic_index()  argument
1273 	u8 *prog = *pprog;  in emit_atomic_index()
1314 	*pprog = prog;  in emit_atomic_index()
1359 static void emit_3vex(u8 **pprog, bool r, bool x, bool b, u8 m,  in emit_3vex()  argument
1362 	u8 *prog = *pprog;  in emit_3vex()
1392 	*pprog = prog;  in emit_3vex()
1396 static void emit_shiftx(u8 **pprog, u32 dst_reg, u8 src_reg, bool is64, u8 op)  in emit_shiftx()  argument
1398 	u8 *prog = *pprog;  in emit_shiftx()
1404 	*pprog = prog;  in emit_shiftx()
2463 				u8 **pprog, int nr_stack_slots,  in clean_stack_garbage()  argument
2499 		prog = *pprog;  in clean_stack_garbage()
2506 		*pprog = prog;  in clean_stack_garbage()
2636 static int invoke_bpf_prog(const struct btf_func_model *m, u8 **pprog,  in invoke_bpf_prog()  argument
2641 	u8 *prog = *pprog;  in invoke_bpf_prog()
2721 	*pprog = prog;  in invoke_bpf_prog()
2725 static void emit_align(u8 **pprog, u32 align)  in emit_align()  argument
2727 	u8 *target, *prog = *pprog;  in emit_align()
2733 	*pprog = prog;  in emit_align()
2736 static int emit_cond_near_jump(u8 **pprog, void *func, void *ip, u8 jmp_cond)  in emit_cond_near_jump()  argument
2738 	u8 *prog = *pprog;  in emit_cond_near_jump()
2747 	*pprog = prog;  in emit_cond_near_jump()
2751 static int invoke_bpf(const struct btf_func_model *m, u8 **pprog,  in invoke_bpf()  argument
2757 	u8 *prog = *pprog;  in invoke_bpf()
2764 	*pprog = prog;  in invoke_bpf()
2768 static int invoke_bpf_mod_ret(const struct btf_func_model *m, u8 **pprog,  in invoke_bpf_mod_ret()  argument
2773 	u8 *prog = *pprog;  in invoke_bpf_mod_ret()
2802 	*pprog = prog;  in invoke_bpf_mod_ret()
3218 static int emit_bpf_dispatcher(u8 **pprog, int a, int b, s64 *progs, u8 *image, u8 *buf)  in emit_bpf_dispatcher()  argument
3220 	u8 *jg_reloc, *prog = *pprog;  in emit_bpf_dispatcher()
3241 		*pprog = prog;  in emit_bpf_dispatcher()
3282 	*pprog = prog;  in emit_bpf_dispatcher()