Lines Matching refs:tjit
2468 static int invoke_bpf_prog(struct bpf_tramp_jit *tjit, in invoke_bpf_prog() argument
2472 struct bpf_jit *jit = &tjit->common; in invoke_bpf_prog()
2473 int cookie_off = tjit->run_ctx_off + in invoke_bpf_prog()
2497 EMIT4_DISP(0x41000000, REG_3, REG_15, tjit->run_ctx_off); in invoke_bpf_prog()
2513 EMIT4_DISP(0x41000000, REG_2, REG_15, tjit->bpf_args_off); in invoke_bpf_prog()
2524 tjit->retval_off); in invoke_bpf_prog()
2542 EMIT4_DISP(0x41000000, REG_4, REG_15, tjit->run_ctx_off); in invoke_bpf_prog()
2549 static int alloc_stack(struct bpf_tramp_jit *tjit, size_t size) in alloc_stack() argument
2551 int stack_offset = tjit->stack_size; in alloc_stack()
2553 tjit->stack_size += size; in alloc_stack()
2568 struct bpf_tramp_jit *tjit, in __arch_prepare_bpf_trampoline() argument
2578 struct bpf_jit *jit = &tjit->common; in __arch_prepare_bpf_trampoline()
2620 tjit->stack_size = STACK_FRAME_OVERHEAD; in __arch_prepare_bpf_trampoline()
2621 tjit->backchain_off = tjit->stack_size - sizeof(u64); in __arch_prepare_bpf_trampoline()
2622 tjit->stack_args_off = alloc_stack(tjit, nr_stack_args * sizeof(u64)); in __arch_prepare_bpf_trampoline()
2623 tjit->reg_args_off = alloc_stack(tjit, nr_reg_args * sizeof(u64)); in __arch_prepare_bpf_trampoline()
2624 tjit->ip_off = alloc_stack(tjit, sizeof(u64)); in __arch_prepare_bpf_trampoline()
2625 tjit->arg_cnt_off = alloc_stack(tjit, sizeof(u64)); in __arch_prepare_bpf_trampoline()
2626 tjit->bpf_args_off = alloc_stack(tjit, nr_bpf_args * sizeof(u64)); in __arch_prepare_bpf_trampoline()
2627 tjit->retval_off = alloc_stack(tjit, sizeof(u64)); in __arch_prepare_bpf_trampoline()
2628 tjit->r7_r8_off = alloc_stack(tjit, 2 * sizeof(u64)); in __arch_prepare_bpf_trampoline()
2629 tjit->run_ctx_off = alloc_stack(tjit, in __arch_prepare_bpf_trampoline()
2631 tjit->tccnt_off = alloc_stack(tjit, sizeof(u64)); in __arch_prepare_bpf_trampoline()
2632 tjit->r14_off = alloc_stack(tjit, sizeof(u64) * 2); in __arch_prepare_bpf_trampoline()
2638 tjit->stack_size -= STACK_FRAME_OVERHEAD - sizeof(u64); in __arch_prepare_bpf_trampoline()
2639 tjit->orig_stack_args_off = tjit->stack_size + STACK_FRAME_OVERHEAD; in __arch_prepare_bpf_trampoline()
2644 EMIT4_IMM(0xa70b0000, REG_15, -tjit->stack_size); in __arch_prepare_bpf_trampoline()
2647 tjit->backchain_off); in __arch_prepare_bpf_trampoline()
2649 _EMIT6(0xd203f000 | tjit->tccnt_off, in __arch_prepare_bpf_trampoline()
2650 0xf000 | (tjit->stack_size + STK_OFF_TCCNT)); in __arch_prepare_bpf_trampoline()
2655 tjit->reg_args_off); in __arch_prepare_bpf_trampoline()
2660 arg = tjit->orig_stack_args_off + in __arch_prepare_bpf_trampoline()
2662 bpf_arg_off = tjit->bpf_args_off + j * sizeof(u64); in __arch_prepare_bpf_trampoline()
2690 tjit->r7_r8_off); in __arch_prepare_bpf_trampoline()
2692 EMIT6_DISP_LH(0xe3000000, 0x0024, REG_14, REG_0, REG_15, tjit->r14_off); in __arch_prepare_bpf_trampoline()
2719 tjit->ip_off); in __arch_prepare_bpf_trampoline()
2725 tjit->arg_cnt_off); in __arch_prepare_bpf_trampoline()
2741 if (invoke_bpf_prog(tjit, m, fentry->links[i], in __arch_prepare_bpf_trampoline()
2751 _EMIT6(0xd707f000 | tjit->retval_off, in __arch_prepare_bpf_trampoline()
2752 0xf000 | tjit->retval_off); in __arch_prepare_bpf_trampoline()
2755 if (invoke_bpf_prog(tjit, m, fmod_ret->links[i], true)) in __arch_prepare_bpf_trampoline()
2765 tjit->retval_off); in __arch_prepare_bpf_trampoline()
2767 EMIT6_PCREL_RILC(0xc0040000, 7, tjit->do_fexit); in __arch_prepare_bpf_trampoline()
2780 tjit->reg_args_off); in __arch_prepare_bpf_trampoline()
2785 tjit->stack_args_off, in __arch_prepare_bpf_trampoline()
2786 0xf000 | tjit->orig_stack_args_off); in __arch_prepare_bpf_trampoline()
2788 _EMIT6(0xd203f000 | STK_OFF_TCCNT, 0xf000 | tjit->tccnt_off); in __arch_prepare_bpf_trampoline()
2795 tjit->retval_off); in __arch_prepare_bpf_trampoline()
2808 tjit->do_fexit = jit->prg; in __arch_prepare_bpf_trampoline()
2810 if (invoke_bpf_prog(tjit, m, fexit->links[i], false)) in __arch_prepare_bpf_trampoline()
2832 tjit->reg_args_off); in __arch_prepare_bpf_trampoline()
2838 tjit->r7_r8_off); in __arch_prepare_bpf_trampoline()
2840 EMIT6_DISP_LH(0xe3000000, 0x0004, REG_14, REG_0, REG_15, tjit->r14_off); in __arch_prepare_bpf_trampoline()
2844 tjit->retval_off); in __arch_prepare_bpf_trampoline()
2846 _EMIT6(0xd203f000 | (tjit->stack_size + STK_OFF_TCCNT), in __arch_prepare_bpf_trampoline()
2847 0xf000 | tjit->tccnt_off); in __arch_prepare_bpf_trampoline()
2849 EMIT4_IMM(0xa70b0000, REG_15, tjit->stack_size); in __arch_prepare_bpf_trampoline()
2869 struct bpf_tramp_jit tjit; in arch_bpf_trampoline_size() local
2872 memset(&tjit, 0, sizeof(tjit)); in arch_bpf_trampoline_size()
2874 ret = __arch_prepare_bpf_trampoline(&im, &tjit, m, flags, in arch_bpf_trampoline_size()
2877 return ret < 0 ? ret : tjit.common.prg; in arch_bpf_trampoline_size()
2885 struct bpf_tramp_jit tjit; in arch_prepare_bpf_trampoline() local
2889 memset(&tjit, 0, sizeof(tjit)); in arch_prepare_bpf_trampoline()
2890 ret = __arch_prepare_bpf_trampoline(im, &tjit, m, flags, in arch_prepare_bpf_trampoline()
2895 if (tjit.common.prg > (char *)image_end - (char *)image) in arch_prepare_bpf_trampoline()
2902 tjit.common.prg = 0; in arch_prepare_bpf_trampoline()
2903 tjit.common.prg_buf = image; in arch_prepare_bpf_trampoline()
2904 ret = __arch_prepare_bpf_trampoline(im, &tjit, m, flags, in arch_prepare_bpf_trampoline()
2907 return ret < 0 ? ret : tjit.common.prg; in arch_prepare_bpf_trampoline()