65 	do { prog = emit_code(prog, bytes, len); cnt += len; } while (0)
209 	u8 *prog = *pprog;  in emit_ia32_mov_i()  local
230 	*pprog = prog;  in emit_ia32_mov_i()
237 	u8 *prog = *pprog;  in emit_ia32_mov_r()  local
251 	*pprog = prog;  in emit_ia32_mov_r()
288 	u8 *prog = *pprog;  in emit_ia32_mul_r()  local
314 	*pprog = prog;  in emit_ia32_mul_r()
321 	u8 *prog = *pprog;  in emit_ia32_to_le_r64()  local
362 	*pprog = prog;  in emit_ia32_to_le_r64()
369 	u8 *prog = *pprog;  in emit_ia32_to_be_r64()  local
428 	*pprog = prog;  in emit_ia32_to_be_r64()
438 	u8 *prog = *pprog;  in emit_ia32_div_mod_r()  local
475 	*pprog = prog;  in emit_ia32_div_mod_r()
485 	u8 *prog = *pprog;  in emit_ia32_shift_r()  local
516 	*pprog = prog;  in emit_ia32_shift_r()
527 	u8 *prog = *pprog;  in emit_ia32_alu_r()  local
573 	*pprog = prog;  in emit_ia32_alu_r()
582 	u8 *prog = *pprog;  in emit_ia32_alu_r64()  local
584 	emit_ia32_alu_r(is64, false, op, dst_lo, src_lo, dstk, sstk, &prog);  in emit_ia32_alu_r64()
587 				&prog);  in emit_ia32_alu_r64()
589 		emit_ia32_mov_i(dst_hi, 0, dstk, &prog);  in emit_ia32_alu_r64()
590 	*pprog = prog;  in emit_ia32_alu_r64()
601 	u8 *prog = *pprog;  in emit_ia32_alu_i()  local
673 	*pprog = prog;  in emit_ia32_alu_i()
682 	u8 *prog = *pprog;  in emit_ia32_alu_i64()  local
688 	emit_ia32_alu_i(is64, false, op, dst_lo, val, dstk, &prog);  in emit_ia32_alu_i64()
690 		emit_ia32_alu_i(is64, true, op, dst_hi, hi, dstk, &prog);  in emit_ia32_alu_i64()
692 		emit_ia32_mov_i(dst_hi, 0, dstk, &prog);  in emit_ia32_alu_i64()
694 	*pprog = prog;  in emit_ia32_alu_i64()
700 	u8 *prog = *pprog;  in emit_ia32_neg64()  local
727 	*pprog = prog;  in emit_ia32_neg64()
734 	u8 *prog = *pprog;  in emit_ia32_lsh_r64()  local
780 	*pprog = prog;  in emit_ia32_lsh_r64()
787 	u8 *prog = *pprog;  in emit_ia32_arsh_r64()  local
833 	*pprog = prog;  in emit_ia32_arsh_r64()
840 	u8 *prog = *pprog;  in emit_ia32_rsh_r64()  local
886 	*pprog = prog;  in emit_ia32_rsh_r64()
893 	u8 *prog = *pprog;  in emit_ia32_lsh_i64()  local
934 	*pprog = prog;  in emit_ia32_lsh_i64()
941 	u8 *prog = *pprog;  in emit_ia32_rsh_i64()  local
983 	*pprog = prog;  in emit_ia32_rsh_i64()
990 	u8 *prog = *pprog;  in emit_ia32_arsh_i64()  local
1032 	*pprog = prog;  in emit_ia32_arsh_i64()
1038 	u8 *prog = *pprog;  in emit_ia32_mul_r64()  local
1109 	*pprog = prog;  in emit_ia32_mul_r64()
1115 	u8 *prog = *pprog;  in emit_ia32_mul_i64()  local
1169 	*pprog = prog;  in emit_ia32_mul_i64()
1202 	u8 *prog = *pprog;  in emit_prologue()  local
1241 	*pprog = prog;  in emit_prologue()
1247 	u8 *prog = *pprog;  in emit_epilogue()  local
1268 	*pprog = prog;  in emit_epilogue()
1273 	u8 *prog = *pprog;  in emit_jmp_edx()  local
1281 	*pprog = prog;  in emit_jmp_edx()
1301 	u8 *prog = *pprog;  in emit_bpf_tail_call()  local
1381 	cnt += emit_jmp_edx(&prog, ip + cnt);  in emit_bpf_tail_call()
1387 	*pprog = prog;  in emit_bpf_tail_call()
1393 	u8 *prog = *pprog;  in emit_push_r64()  local
1406 	*pprog = prog;  in emit_push_r64()
1411 	u8 *prog = *pprog;  in emit_push_r32()  local
1419 	*pprog = prog;  in emit_push_r32()
1578 	u8 *prog = *pprog;  in emit_kfunc_call()  local
1600 			emit_push_r64(bpf2ia32[i], &prog);  in emit_kfunc_call()
1603 			emit_push_r32(bpf2ia32[i], &prog);  in emit_kfunc_call()
1654 	*pprog = prog;  in emit_kfunc_call()
1668 	u8 *prog = temp;  in do_jit()  local
1670 	emit_prologue(&prog, bpf_prog->aux->stack_depth);  in do_jit()
1697 					emit_ia32_mov_i(dst_hi, 0, dstk, &prog);  in do_jit()
1701 						  &prog, bpf_prog->aux);  in do_jit()
1706 						  dstk, &prog);  in do_jit()
1741 						  src, dstk, sstk, &prog,  in do_jit()
1746 						  imm32, dstk, &prog,  in do_jit()
1756 						sstk, &prog);  in do_jit()
1763 						false, &prog);  in do_jit()
1767 				emit_ia32_mov_i(dst_hi, 0, dstk, &prog);  in do_jit()
1776 						  dstk, sstk, &prog);  in do_jit()
1784 						  &prog);  in do_jit()
1788 				emit_ia32_mov_i(dst_hi, 0, dstk, &prog);  in do_jit()
1799 						    src_lo, dstk, sstk, &prog);  in do_jit()
1807 						    &prog);  in do_jit()
1811 				emit_ia32_mov_i(dst_hi, 0, dstk, &prog);  in do_jit()
1827 					  false, &prog);  in do_jit()
1829 				emit_ia32_mov_i(dst_hi, 0, dstk, &prog);  in do_jit()
1835 			emit_ia32_lsh_i64(dst, imm32, dstk, &prog);  in do_jit()
1841 			emit_ia32_rsh_i64(dst, imm32, dstk, &prog);  in do_jit()
1845 			emit_ia32_lsh_r64(dst, src, dstk, sstk, &prog);  in do_jit()
1849 			emit_ia32_rsh_r64(dst, src, dstk, sstk, &prog);  in do_jit()
1853 			emit_ia32_arsh_r64(dst, src, dstk, sstk, &prog);  in do_jit()
1859 			emit_ia32_arsh_i64(dst, imm32, dstk, &prog);  in do_jit()
1864 					dst_lo, 0, dstk, &prog);  in do_jit()
1866 				emit_ia32_mov_i(dst_hi, 0, dstk, &prog);  in do_jit()
1870 			emit_ia32_neg64(dst, dstk, &prog);  in do_jit()
1877 				emit_ia32_mul_r64(dst, src, dstk, sstk, &prog);  in do_jit()
1880 				emit_ia32_mul_i64(dst, imm32, dstk, &prog);  in do_jit()
1886 			emit_ia32_to_le_r64(dst, imm32, dstk, &prog,  in do_jit()
1891 			emit_ia32_to_be_r64(dst, imm32, dstk, &prog,  in do_jit()
1899 			emit_ia32_mov_i(dst_lo, lo, dstk, &prog);  in do_jit()
1900 			emit_ia32_mov_i(dst_hi, hi, dstk, &prog);  in do_jit()
2099 						      insn, &prog);  in do_jit()
2122 			emit_push_r64(r5, &prog);  in do_jit()
2123 			emit_push_r64(r4, &prog);  in do_jit()
2124 			emit_push_r64(r3, &prog);  in do_jit()
2125 			emit_push_r64(r2, &prog);  in do_jit()
2141 			emit_bpf_tail_call(&prog, image + addrs[i - 1]);  in do_jit()
2472 			emit_epilogue(&prog, bpf_prog->aux->stack_depth);  in do_jit()
2487 		ilen = prog - temp;  in do_jit()
2511 		prog = temp;  in do_jit()
2521 struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog)  in bpf_int_jit_compile()  argument
2524 	struct bpf_prog *tmp, *orig_prog = prog;  in bpf_int_jit_compile()
2533 	if (!prog->jit_requested)  in bpf_int_jit_compile()
2536 	tmp = bpf_jit_blind_constants(prog);  in bpf_int_jit_compile()
2543 	if (tmp != prog) {  in bpf_int_jit_compile()
2545 		prog = tmp;  in bpf_int_jit_compile()
2548 	addrs = kmalloc_array(prog->len, sizeof(*addrs), GFP_KERNEL);  in bpf_int_jit_compile()
2550 		prog = orig_prog;  in bpf_int_jit_compile()
2558 	for (proglen = 0, i = 0; i < prog->len; i++) {  in bpf_int_jit_compile()
2571 		proglen = do_jit(prog, addrs, image, oldproglen, &ctx);  in bpf_int_jit_compile()
2577 			prog = orig_prog;  in bpf_int_jit_compile()
2592 				prog = orig_prog;  in bpf_int_jit_compile()
2601 		bpf_jit_dump(prog->len, proglen, pass + 1, image);  in bpf_int_jit_compile()
2604 		prog->bpf_func = (void *)image;  in bpf_int_jit_compile()
2605 		prog->jited = 1;  in bpf_int_jit_compile()
2606 		prog->jited_len = proglen;  in bpf_int_jit_compile()
2608 		prog = orig_prog;  in bpf_int_jit_compile()
2615 		bpf_jit_prog_release_other(prog, prog == orig_prog ?  in bpf_int_jit_compile()
2617 	return prog;  in bpf_int_jit_compile()