Lines Matching refs:jit
122 static inline void reg_set_seen(struct bpf_jit *jit, u32 b1) in reg_set_seen() argument
127 jit->seen_regs |= (1 << r1); in reg_set_seen()
132 reg_set_seen(jit, b1); \
141 if (jit->prg_buf) \
142 *(u16 *) (jit->prg_buf + jit->prg) = (op); \
143 jit->prg += 2; \
155 if (jit->prg_buf) \
156 *(u32 *) (jit->prg_buf + jit->prg) = (op); \
157 jit->prg += 4; \
204 int __rel = ((target) - jit->prg) / 2; \
210 if (jit->prg_buf) { \
211 *(u32 *) (jit->prg_buf + jit->prg) = (op1); \
212 *(u16 *) (jit->prg_buf + jit->prg + 4) = (op2); \
214 jit->prg += 6; \
242 unsigned int rel = (int)((target) - jit->prg) / 2; \
251 unsigned int rel = (int)((target) - jit->prg) / 2; \
260 int rel = (addrs[(i) + (off) + 1] - jit->prg) / 2; \
268 unsigned int rel = (int)((target) - jit->prg) / 2; \
275 unsigned int rel = (int)((target) - jit->prg) / 2; \
299 ret = jit->lit32; \
300 if (jit->prg_buf) \
301 *(u32 *)(jit->prg_buf + jit->lit32) = (u32)(val);\
302 jit->lit32 += 4; \
308 jit->seen |= SEEN_LITERAL; \
309 _EMIT_CONST_U32(val) - jit->base_ip; \
315 ret = jit->lit64; \
316 if (jit->prg_buf) \
317 *(u64 *)(jit->prg_buf + jit->lit64) = (u64)(val);\
318 jit->lit64 += 8; \
324 jit->seen |= SEEN_LITERAL; \
325 _EMIT_CONST_U64(val) - jit->base_ip; \
341 static bool is_first_pass(struct bpf_jit *jit) in is_first_pass() argument
343 return jit->size == 0; in is_first_pass()
350 static bool is_codegen_pass(struct bpf_jit *jit) in is_codegen_pass() argument
352 return jit->prg_buf; in is_codegen_pass()
366 static bool can_use_rel(struct bpf_jit *jit, int off) in can_use_rel() argument
368 return is_valid_rel(off - jit->prg); in can_use_rel()
384 static bool can_use_ldisp_for_lit32(struct bpf_jit *jit) in can_use_ldisp_for_lit32() argument
386 return is_valid_ldisp(jit->lit32 - jit->base_ip); in can_use_ldisp_for_lit32()
393 static bool can_use_ldisp_for_lit64(struct bpf_jit *jit) in can_use_ldisp_for_lit64() argument
395 return is_valid_ldisp(jit->lit64 - jit->base_ip); in can_use_ldisp_for_lit64()
409 static void save_regs(struct bpf_jit *jit, u32 rs, u32 re) in save_regs() argument
424 static void restore_regs(struct bpf_jit *jit, u32 rs, u32 re, u32 stack_depth) in restore_regs() argument
428 if (jit->seen & SEEN_STACK) in restore_regs()
473 static void save_restore_regs(struct bpf_jit *jit, int op, u32 stack_depth, in save_restore_regs() argument
476 u16 seen_regs = jit->seen_regs | extra_regs; in save_restore_regs()
480 if (is_first_pass(jit)) { in save_restore_regs()
485 jit->prg += (last - re + 1) * save_restore_size; in save_restore_regs()
495 save_regs(jit, rs, re); in save_restore_regs()
497 restore_regs(jit, rs, re, stack_depth); in save_restore_regs()
502 static void bpf_skip(struct bpf_jit *jit, int size) in bpf_skip() argument
556 static void bpf_jit_prologue(struct bpf_jit *jit, struct bpf_prog *fp, in bpf_jit_prologue() argument
561 EMIT6_PCREL_RILC(0xc0040000, 0, jit->prologue_plt); in bpf_jit_prologue()
562 jit->prologue_plt_ret = jit->prg; in bpf_jit_prologue()
574 bpf_skip(jit, 6); in bpf_jit_prologue()
577 jit->tail_call_start = jit->prg; in bpf_jit_prologue()
587 jit->seen_regs |= NVREGS; in bpf_jit_prologue()
590 save_restore_regs(jit, REGS_SAVE, stack_depth, in bpf_jit_prologue()
594 if (is_first_pass(jit) || (jit->seen & SEEN_LITERAL)) { in bpf_jit_prologue()
595 if (!is_first_pass(jit) && in bpf_jit_prologue()
596 is_valid_ldisp(jit->size - (jit->prg + 2))) { in bpf_jit_prologue()
599 jit->base_ip = jit->prg; in bpf_jit_prologue()
602 EMIT6_PCREL_RILB(0xc0000000, REG_L, jit->lit32_start); in bpf_jit_prologue()
603 jit->base_ip = jit->lit32_start; in bpf_jit_prologue()
607 if (is_first_pass(jit) || (jit->seen & SEEN_STACK)) { in bpf_jit_prologue()
608 if (is_first_pass(jit) || (jit->seen & SEEN_FUNC)) in bpf_jit_prologue()
615 if (is_first_pass(jit) || (jit->seen & SEEN_FUNC)) in bpf_jit_prologue()
625 static void emit_expoline(struct bpf_jit *jit) in emit_expoline() argument
628 EMIT6_PCREL_RIL(0xc6000000, jit->prg + 10); in emit_expoline()
636 static void emit_r1_thunk(struct bpf_jit *jit) in emit_r1_thunk() argument
639 jit->r1_thunk_ip = jit->prg; in emit_r1_thunk()
640 emit_expoline(jit); in emit_r1_thunk()
649 static void call_r1(struct bpf_jit *jit) in call_r1() argument
653 EMIT6_PCREL_RILB(0xc0050000, REG_14, jit->r1_thunk_ip); in call_r1()
662 static void bpf_jit_epilogue(struct bpf_jit *jit, u32 stack_depth) in bpf_jit_epilogue() argument
664 jit->exit_ip = jit->prg; in bpf_jit_epilogue()
668 save_restore_regs(jit, REGS_RESTORE, stack_depth, 0); in bpf_jit_epilogue()
670 jit->r14_thunk_ip = jit->prg; in bpf_jit_epilogue()
672 emit_expoline(jit); in bpf_jit_epilogue()
677 if (is_first_pass(jit) || (jit->seen & SEEN_FUNC)) in bpf_jit_epilogue()
678 emit_r1_thunk(jit); in bpf_jit_epilogue()
680 jit->prg = ALIGN(jit->prg, 8); in bpf_jit_epilogue()
681 jit->prologue_plt = jit->prg; in bpf_jit_epilogue()
682 if (jit->prg_buf) in bpf_jit_epilogue()
683 bpf_jit_plt((struct bpf_plt *)(jit->prg_buf + jit->prg), in bpf_jit_epilogue()
684 jit->prg_buf + jit->prologue_plt_ret, NULL); in bpf_jit_epilogue()
685 jit->prg += sizeof(struct bpf_plt); in bpf_jit_epilogue()
720 static void bpf_jit_probe_emit_nop(struct bpf_jit *jit, in bpf_jit_probe_emit_nop() argument
727 probe->nop_prg = jit->prg; in bpf_jit_probe_emit_nop()
732 static void bpf_jit_probe_load_pre(struct bpf_jit *jit, struct bpf_insn *insn, in bpf_jit_probe_load_pre() argument
742 EMIT6_PCREL_RILB(0xc4080000, REG_W1, jit->kern_arena); in bpf_jit_probe_load_pre()
745 probe->prg = jit->prg; in bpf_jit_probe_load_pre()
749 static void bpf_jit_probe_store_pre(struct bpf_jit *jit, struct bpf_insn *insn, in bpf_jit_probe_store_pre() argument
756 EMIT6_PCREL_RILB(0xc4080000, REG_W1, jit->kern_arena); in bpf_jit_probe_store_pre()
758 probe->prg = jit->prg; in bpf_jit_probe_store_pre()
761 static void bpf_jit_probe_atomic_pre(struct bpf_jit *jit, in bpf_jit_probe_atomic_pre() argument
769 EMIT6_PCREL_RILB(0xc4080000, REG_W1, jit->kern_arena); in bpf_jit_probe_atomic_pre()
773 probe->prg = jit->prg; in bpf_jit_probe_atomic_pre()
776 static int bpf_jit_probe_post(struct bpf_jit *jit, struct bpf_prog *fp, in bpf_jit_probe_post() argument
787 bpf_jit_probe_emit_nop(jit, probe); in bpf_jit_probe_post()
791 insn = jit->prg_buf + probe->prg; in bpf_jit_probe_post()
796 if (WARN_ON_ONCE(jit->excnt >= fp->aux->num_exentries)) in bpf_jit_probe_post()
799 ex = &fp->aux->extable[jit->excnt]; in bpf_jit_probe_post()
802 delta = jit->prg_buf + prg - (u8 *)&ex->insn; in bpf_jit_probe_post()
812 delta = jit->prg_buf + jit->prg - (u8 *)&ex->fixup; in bpf_jit_probe_post()
819 jit->excnt++; in bpf_jit_probe_post()
827 static int sign_extend(struct bpf_jit *jit, int r, u8 size, u8 flags) in sign_extend() argument
858 static noinline int bpf_jit_insn(struct bpf_jit *jit, struct bpf_prog *fp, in bpf_jit_insn() argument
867 u32 *addrs = jit->addrs; in bpf_jit_insn()
908 patch_brc = jit->prg; in bpf_jit_insn()
911 EMIT6_IMM(0xc0080000, dst_reg, jit->user_arena >> 32); in bpf_jit_insn()
913 if (jit->prg_buf) in bpf_jit_insn()
914 *(u16 *)(jit->prg_buf + patch_brc + 2) = in bpf_jit_insn()
915 (jit->prg - patch_brc) >> 1; in bpf_jit_insn()
1109 if (!is_first_pass(jit) && can_use_ldisp_for_lit32(jit)) { in bpf_jit_insn()
1138 jit->seen |= SEEN_LITERAL; in bpf_jit_insn()
1148 jit->seen |= SEEN_LITERAL; in bpf_jit_insn()
1171 if (!is_first_pass(jit) && can_use_ldisp_for_lit64(jit)) { in bpf_jit_insn()
1200 jit->seen |= SEEN_LITERAL; in bpf_jit_insn()
1210 jit->seen |= SEEN_LITERAL; in bpf_jit_insn()
1238 if (!is_first_pass(jit) && can_use_ldisp_for_lit64(jit)) { in bpf_jit_insn()
1247 jit->seen |= SEEN_LITERAL; in bpf_jit_insn()
1270 if (!is_first_pass(jit) && can_use_ldisp_for_lit64(jit)) { in bpf_jit_insn()
1279 jit->seen |= SEEN_LITERAL; in bpf_jit_insn()
1304 if (!is_first_pass(jit) && can_use_ldisp_for_lit64(jit)) { in bpf_jit_insn()
1313 jit->seen |= SEEN_LITERAL; in bpf_jit_insn()
1462 bpf_jit_probe_store_pre(jit, insn, &probe); in bpf_jit_insn()
1466 err = bpf_jit_probe_post(jit, fp, &probe); in bpf_jit_insn()
1469 jit->seen |= SEEN_MEM; in bpf_jit_insn()
1473 bpf_jit_probe_store_pre(jit, insn, &probe); in bpf_jit_insn()
1477 err = bpf_jit_probe_post(jit, fp, &probe); in bpf_jit_insn()
1480 jit->seen |= SEEN_MEM; in bpf_jit_insn()
1484 bpf_jit_probe_store_pre(jit, insn, &probe); in bpf_jit_insn()
1488 err = bpf_jit_probe_post(jit, fp, &probe); in bpf_jit_insn()
1491 jit->seen |= SEEN_MEM; in bpf_jit_insn()
1495 bpf_jit_probe_store_pre(jit, insn, &probe); in bpf_jit_insn()
1499 err = bpf_jit_probe_post(jit, fp, &probe); in bpf_jit_insn()
1502 jit->seen |= SEEN_MEM; in bpf_jit_insn()
1508 bpf_jit_probe_store_pre(jit, insn, &probe); in bpf_jit_insn()
1512 err = bpf_jit_probe_post(jit, fp, &probe); in bpf_jit_insn()
1515 jit->seen |= SEEN_MEM; in bpf_jit_insn()
1521 bpf_jit_probe_store_pre(jit, insn, &probe); in bpf_jit_insn()
1525 err = bpf_jit_probe_post(jit, fp, &probe); in bpf_jit_insn()
1528 jit->seen |= SEEN_MEM; in bpf_jit_insn()
1534 bpf_jit_probe_store_pre(jit, insn, &probe); in bpf_jit_insn()
1538 err = bpf_jit_probe_post(jit, fp, &probe); in bpf_jit_insn()
1541 jit->seen |= SEEN_MEM; in bpf_jit_insn()
1547 bpf_jit_probe_store_pre(jit, insn, &probe); in bpf_jit_insn()
1551 err = bpf_jit_probe_post(jit, fp, &probe); in bpf_jit_insn()
1554 jit->seen |= SEEN_MEM; in bpf_jit_insn()
1577 bpf_jit_probe_atomic_pre(jit, insn, &probe); \ in bpf_jit_insn()
1582 err = bpf_jit_probe_post(jit, fp, &probe); \ in bpf_jit_insn()
1617 bpf_jit_probe_atomic_pre(jit, insn, &load_probe); in bpf_jit_insn()
1622 bpf_jit_probe_emit_nop(jit, &load_probe); in bpf_jit_insn()
1625 probe.prg = jit->prg; in bpf_jit_insn()
1628 loop_start = jit->prg; in bpf_jit_insn()
1632 bpf_jit_probe_emit_nop(jit, &probe); in bpf_jit_insn()
1638 err = bpf_jit_probe_post(jit, fp, &load_probe); in bpf_jit_insn()
1641 err = bpf_jit_probe_post(jit, fp, &probe); in bpf_jit_insn()
1649 bpf_jit_probe_atomic_pre(jit, insn, &probe); in bpf_jit_insn()
1654 err = bpf_jit_probe_post(jit, fp, &probe); in bpf_jit_insn()
1663 jit->seen |= SEEN_MEM; in bpf_jit_insn()
1672 bpf_jit_probe_load_pre(jit, insn, &probe); in bpf_jit_insn()
1676 err = bpf_jit_probe_post(jit, fp, &probe); in bpf_jit_insn()
1679 jit->seen |= SEEN_MEM; in bpf_jit_insn()
1685 bpf_jit_probe_load_pre(jit, insn, &probe); in bpf_jit_insn()
1688 err = bpf_jit_probe_post(jit, fp, &probe); in bpf_jit_insn()
1691 jit->seen |= SEEN_MEM; in bpf_jit_insn()
1696 bpf_jit_probe_load_pre(jit, insn, &probe); in bpf_jit_insn()
1700 err = bpf_jit_probe_post(jit, fp, &probe); in bpf_jit_insn()
1703 jit->seen |= SEEN_MEM; in bpf_jit_insn()
1709 bpf_jit_probe_load_pre(jit, insn, &probe); in bpf_jit_insn()
1712 err = bpf_jit_probe_post(jit, fp, &probe); in bpf_jit_insn()
1715 jit->seen |= SEEN_MEM; in bpf_jit_insn()
1720 bpf_jit_probe_load_pre(jit, insn, &probe); in bpf_jit_insn()
1722 jit->seen |= SEEN_MEM; in bpf_jit_insn()
1725 err = bpf_jit_probe_post(jit, fp, &probe); in bpf_jit_insn()
1733 bpf_jit_probe_load_pre(jit, insn, &probe); in bpf_jit_insn()
1735 jit->seen |= SEEN_MEM; in bpf_jit_insn()
1737 err = bpf_jit_probe_post(jit, fp, &probe); in bpf_jit_insn()
1744 bpf_jit_probe_load_pre(jit, insn, &probe); in bpf_jit_insn()
1746 jit->seen |= SEEN_MEM; in bpf_jit_insn()
1749 err = bpf_jit_probe_post(jit, fp, &probe); in bpf_jit_insn()
1769 jit->seen |= SEEN_FUNC; in bpf_jit_insn()
1791 if (sign_extend(jit, BPF_REG_1 + j, in bpf_jit_insn()
1801 call_r1(jit); in bpf_jit_insn()
1824 patch_1_clrj = jit->prg; in bpf_jit_insn()
1826 jit->prg); in bpf_jit_insn()
1833 if (jit->seen & SEEN_STACK) in bpf_jit_insn()
1842 patch_2_clij = jit->prg; in bpf_jit_insn()
1844 2, jit->prg); in bpf_jit_insn()
1860 patch_3_brc = jit->prg; in bpf_jit_insn()
1861 EMIT4_PCREL_RIC(0xa7040000, 8, jit->prg); in bpf_jit_insn()
1866 save_restore_regs(jit, REGS_RESTORE, stack_depth, 0); in bpf_jit_insn()
1876 jit->seen |= SEEN_FUNC; in bpf_jit_insn()
1878 EMIT4_IMM(0xa70b0000, REG_1, jit->tail_call_start); in bpf_jit_insn()
1880 EMIT6_PCREL_RILC(0xc0040000, 0xf, jit->r1_thunk_ip); in bpf_jit_insn()
1883 _EMIT4(0x47f01000 + jit->tail_call_start); in bpf_jit_insn()
1886 if (jit->prg_buf) { in bpf_jit_insn()
1887 *(u16 *)(jit->prg_buf + patch_1_clrj + 2) = in bpf_jit_insn()
1888 (jit->prg - patch_1_clrj) >> 1; in bpf_jit_insn()
1889 *(u16 *)(jit->prg_buf + patch_2_clij + 2) = in bpf_jit_insn()
1890 (jit->prg - patch_2_clij) >> 1; in bpf_jit_insn()
1891 *(u16 *)(jit->prg_buf + patch_3_brc + 2) = in bpf_jit_insn()
1892 (jit->prg - patch_3_brc) >> 1; in bpf_jit_insn()
1900 if (!is_first_pass(jit) && can_use_rel(jit, jit->exit_ip)) in bpf_jit_insn()
1902 EMIT4_PCREL_RIC(0xa7040000, 0xf, jit->exit_ip); in bpf_jit_insn()
1905 EMIT6_PCREL_RILC(0xc0040000, 0xf, jit->exit_ip); in bpf_jit_insn()
2044 if (!is_first_pass(jit) && in bpf_jit_insn()
2045 can_use_rel(jit, addrs[i + off + 1])) { in bpf_jit_insn()
2062 if (!is_first_pass(jit) && in bpf_jit_insn()
2063 can_use_rel(jit, addrs[i + off + 1])) { in bpf_jit_insn()
2080 if (!is_first_pass(jit) && in bpf_jit_insn()
2081 can_use_rel(jit, addrs[i + off + 1])) { in bpf_jit_insn()
2097 if (!is_first_pass(jit) && in bpf_jit_insn()
2098 can_use_rel(jit, addrs[i + branch_oc_off + 1])) { in bpf_jit_insn()
2122 static bool bpf_is_new_addr_sane(struct bpf_jit *jit, int i) in bpf_is_new_addr_sane() argument
2125 if (is_first_pass(jit)) in bpf_is_new_addr_sane()
2129 if (is_codegen_pass(jit)) in bpf_is_new_addr_sane()
2130 return jit->addrs[i] == jit->prg; in bpf_is_new_addr_sane()
2133 return jit->addrs[i] >= jit->prg; in bpf_is_new_addr_sane()
2139 static int bpf_set_addr(struct bpf_jit *jit, int i) in bpf_set_addr() argument
2143 if (is_codegen_pass(jit)) { in bpf_set_addr()
2144 delta = jit->prg - jit->addrs[i]; in bpf_set_addr()
2146 bpf_skip(jit, -delta); in bpf_set_addr()
2148 if (WARN_ON_ONCE(!bpf_is_new_addr_sane(jit, i))) in bpf_set_addr()
2150 jit->addrs[i] = jit->prg; in bpf_set_addr()
2157 static int bpf_jit_prog(struct bpf_jit *jit, struct bpf_prog *fp, in bpf_jit_prog() argument
2163 jit->lit32 = jit->lit32_start; in bpf_jit_prog()
2164 jit->lit64 = jit->lit64_start; in bpf_jit_prog()
2165 jit->prg = 0; in bpf_jit_prog()
2166 jit->excnt = 0; in bpf_jit_prog()
2170 jit->kern_arena = _EMIT_CONST_U64(kern_arena); in bpf_jit_prog()
2171 jit->user_arena = bpf_arena_get_user_vm_start(fp->aux->arena); in bpf_jit_prog()
2173 bpf_jit_prologue(jit, fp, stack_depth); in bpf_jit_prog()
2174 if (bpf_set_addr(jit, 0) < 0) in bpf_jit_prog()
2177 insn_count = bpf_jit_insn(jit, fp, i, extra_pass, stack_depth); in bpf_jit_prog()
2181 if (bpf_set_addr(jit, i + insn_count) < 0) in bpf_jit_prog()
2184 bpf_jit_epilogue(jit, stack_depth); in bpf_jit_prog()
2186 lit32_size = jit->lit32 - jit->lit32_start; in bpf_jit_prog()
2187 lit64_size = jit->lit64 - jit->lit64_start; in bpf_jit_prog()
2188 jit->lit32_start = jit->prg; in bpf_jit_prog()
2190 jit->lit32_start = ALIGN(jit->lit32_start, 4); in bpf_jit_prog()
2191 jit->lit64_start = jit->lit32_start + lit32_size; in bpf_jit_prog()
2193 jit->lit64_start = ALIGN(jit->lit64_start, 8); in bpf_jit_prog()
2194 jit->size = jit->lit64_start + lit64_size; in bpf_jit_prog()
2195 jit->size_prg = jit->prg; in bpf_jit_prog()
2198 jit->excnt != fp->aux->num_exentries)) in bpf_jit_prog()
2216 static struct bpf_binary_header *bpf_jit_alloc(struct bpf_jit *jit, in bpf_jit_alloc() argument
2242 code_size = roundup(jit->size, in bpf_jit_alloc()
2246 header = bpf_jit_binary_alloc(code_size + extable_size, &jit->prg_buf, in bpf_jit_alloc()
2251 (jit->prg_buf + code_size); in bpf_jit_alloc()
2266 struct bpf_jit jit; in bpf_int_jit_compile() local
2294 jit = jit_data->ctx; in bpf_int_jit_compile()
2301 memset(&jit, 0, sizeof(jit)); in bpf_int_jit_compile()
2302 jit.addrs = kvcalloc(fp->len + 1, sizeof(*jit.addrs), GFP_KERNEL); in bpf_int_jit_compile()
2303 if (jit.addrs == NULL) { in bpf_int_jit_compile()
2313 if (bpf_jit_prog(&jit, fp, extra_pass, stack_depth)) { in bpf_int_jit_compile()
2321 header = bpf_jit_alloc(&jit, fp); in bpf_int_jit_compile()
2327 if (bpf_jit_prog(&jit, fp, extra_pass, stack_depth)) { in bpf_int_jit_compile()
2333 bpf_jit_dump(fp->len, jit.size, pass, jit.prg_buf); in bpf_int_jit_compile()
2334 print_fn_code(jit.prg_buf, jit.size_prg); in bpf_int_jit_compile()
2344 jit_data->ctx = jit; in bpf_int_jit_compile()
2347 fp->bpf_func = (void *) jit.prg_buf; in bpf_int_jit_compile()
2349 fp->jited_len = jit.size; in bpf_int_jit_compile()
2352 bpf_prog_fill_jited_linfo(fp, jit.addrs + 1); in bpf_int_jit_compile()
2354 kvfree(jit.addrs); in bpf_int_jit_compile()
2460 static void load_imm64(struct bpf_jit *jit, int dst_reg, u64 val) in load_imm64() argument
2472 struct bpf_jit *jit = &tjit->common; in invoke_bpf_prog() local
2483 load_imm64(jit, REG_W0, tlink->cookie); in invoke_bpf_prog()
2493 load_imm64(jit, REG_1, (u64)bpf_trampoline_enter(p)); in invoke_bpf_prog()
2495 load_imm64(jit, REG_2, (u64)p); in invoke_bpf_prog()
2499 call_r1(jit); in invoke_bpf_prog()
2503 patch = jit->prg; in invoke_bpf_prog()
2511 load_imm64(jit, REG_1, (u64)p->bpf_func); in invoke_bpf_prog()
2516 load_imm64(jit, REG_3, (u64)p->insnsi); in invoke_bpf_prog()
2518 call_r1(jit); in invoke_bpf_prog()
2521 if (sign_extend(jit, REG_2, m->ret_size, m->ret_flags)) in invoke_bpf_prog()
2528 if (jit->prg_buf) in invoke_bpf_prog()
2529 *(u32 *)&jit->prg_buf[patch + 2] = (jit->prg - patch) >> 1; in invoke_bpf_prog()
2536 load_imm64(jit, REG_1, (u64)bpf_trampoline_exit(p)); in invoke_bpf_prog()
2538 load_imm64(jit, REG_2, (u64)p); in invoke_bpf_prog()
2544 call_r1(jit); in invoke_bpf_prog()
2578 struct bpf_jit *jit = &tjit->common; in __arch_prepare_bpf_trampoline() local
2706 load_imm64(jit, REG_8, (u64)func_addr + S390X_PATCH_SIZE); in __arch_prepare_bpf_trampoline()
2716 load_imm64(jit, REG_0, (u64)func_addr); in __arch_prepare_bpf_trampoline()
2733 load_imm64(jit, REG_1, (u64)__bpf_tramp_enter); in __arch_prepare_bpf_trampoline()
2735 load_imm64(jit, REG_2, (u64)im); in __arch_prepare_bpf_trampoline()
2737 call_r1(jit); in __arch_prepare_bpf_trampoline()
2792 call_r1(jit); in __arch_prepare_bpf_trampoline()
2797 im->ip_after_call = jit->prg_buf + jit->prg; in __arch_prepare_bpf_trampoline()
2808 tjit->do_fexit = jit->prg; in __arch_prepare_bpf_trampoline()
2814 im->ip_epilogue = jit->prg_buf + jit->prg; in __arch_prepare_bpf_trampoline()
2821 load_imm64(jit, REG_1, (u64)__bpf_tramp_exit); in __arch_prepare_bpf_trampoline()
2823 load_imm64(jit, REG_2, (u64)im); in __arch_prepare_bpf_trampoline()
2825 call_r1(jit); in __arch_prepare_bpf_trampoline()
2852 emit_expoline(jit); in __arch_prepare_bpf_trampoline()
2860 emit_r1_thunk(jit); in __arch_prepare_bpf_trampoline()