Lines Matching refs:nfp_prog
22 #define nfp_for_each_insn_walk2(nfp_prog, pos, next) \ argument
23 for (pos = list_first_entry(&(nfp_prog)->insns, typeof(*pos), l), \
25 &(nfp_prog)->insns != &pos->l && \
26 &(nfp_prog)->insns != &next->l; \
30 #define nfp_for_each_insn_walk3(nfp_prog, pos, next, next2) \ argument
31 for (pos = list_first_entry(&(nfp_prog)->insns, typeof(*pos), l), \
34 &(nfp_prog)->insns != &pos->l && \
35 &(nfp_prog)->insns != &next->l && \
36 &(nfp_prog)->insns != &next2->l; \
42 nfp_meta_has_prev(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) in nfp_meta_has_prev() argument
44 return meta->l.prev != &nfp_prog->insns; in nfp_meta_has_prev()
47 static void nfp_prog_push(struct nfp_prog *nfp_prog, u64 insn) in nfp_prog_push() argument
49 if (nfp_prog->__prog_alloc_len / sizeof(u64) == nfp_prog->prog_len) { in nfp_prog_push()
51 nfp_prog->prog_len); in nfp_prog_push()
52 nfp_prog->error = -ENOSPC; in nfp_prog_push()
56 nfp_prog->prog[nfp_prog->prog_len] = insn; in nfp_prog_push()
57 nfp_prog->prog_len++; in nfp_prog_push()
60 static unsigned int nfp_prog_current_offset(struct nfp_prog *nfp_prog) in nfp_prog_current_offset() argument
62 return nfp_prog->prog_len; in nfp_prog_current_offset()
66 nfp_prog_confirm_current_offset(struct nfp_prog *nfp_prog, unsigned int off) in nfp_prog_confirm_current_offset() argument
72 if (nfp_prog->error) in nfp_prog_confirm_current_offset()
74 return !WARN_ON_ONCE(nfp_prog_current_offset(nfp_prog) != off); in nfp_prog_confirm_current_offset()
79 __emit_cmd(struct nfp_prog *nfp_prog, enum cmd_tgt_map op, in __emit_cmd() argument
96 nfp_prog_push(nfp_prog, insn); in __emit_cmd()
100 emit_cmd_any(struct nfp_prog *nfp_prog, enum cmd_tgt_map op, u8 mode, u8 xfer, in emit_cmd_any() argument
108 nfp_prog->error = err; in emit_cmd_any()
113 nfp_prog->error = -EFAULT; in emit_cmd_any()
118 nfp_prog->error = -EFAULT; in emit_cmd_any()
122 __emit_cmd(nfp_prog, op, mode, xfer, reg.areg, reg.breg, size, ctx, in emit_cmd_any()
127 emit_cmd(struct nfp_prog *nfp_prog, enum cmd_tgt_map op, u8 mode, u8 xfer, in emit_cmd() argument
130 emit_cmd_any(nfp_prog, op, mode, xfer, lreg, rreg, size, ctx, false); in emit_cmd()
134 emit_cmd_indir(struct nfp_prog *nfp_prog, enum cmd_tgt_map op, u8 mode, u8 xfer, in emit_cmd_indir() argument
137 emit_cmd_any(nfp_prog, op, mode, xfer, lreg, rreg, size, ctx, true); in emit_cmd_indir()
141 __emit_br(struct nfp_prog *nfp_prog, enum br_mask mask, enum br_ev_pip ev_pip, in __emit_br() argument
158 nfp_prog_push(nfp_prog, insn); in __emit_br()
162 emit_br_relo(struct nfp_prog *nfp_prog, enum br_mask mask, u16 addr, u8 defer, in emit_br_relo() argument
167 nfp_prog->error = -EFAULT; in emit_br_relo()
171 __emit_br(nfp_prog, mask, in emit_br_relo()
175 nfp_prog->prog[nfp_prog->prog_len - 1] |= in emit_br_relo()
180 emit_br(struct nfp_prog *nfp_prog, enum br_mask mask, u16 addr, u8 defer) in emit_br() argument
182 emit_br_relo(nfp_prog, mask, addr, defer, RELO_BR_REL); in emit_br()
186 __emit_br_bit(struct nfp_prog *nfp_prog, u16 areg, u16 breg, u16 addr, u8 defer, in __emit_br_bit() argument
204 nfp_prog_push(nfp_prog, insn); in __emit_br_bit()
208 emit_br_bit_relo(struct nfp_prog *nfp_prog, swreg src, u8 bit, u16 addr, in emit_br_bit_relo() argument
222 nfp_prog->error = err; in emit_br_bit_relo()
226 __emit_br_bit(nfp_prog, reg.areg, reg.breg, addr, defer, set, in emit_br_bit_relo()
229 nfp_prog->prog[nfp_prog->prog_len - 1] |= in emit_br_bit_relo()
234 emit_br_bset(struct nfp_prog *nfp_prog, swreg src, u8 bit, u16 addr, u8 defer) in emit_br_bset() argument
236 emit_br_bit_relo(nfp_prog, src, bit, addr, defer, true, RELO_BR_REL); in emit_br_bset()
240 __emit_br_alu(struct nfp_prog *nfp_prog, u16 areg, u16 breg, u16 imm_hi, in __emit_br_alu() argument
253 nfp_prog_push(nfp_prog, insn); in __emit_br_alu()
256 static void emit_rtn(struct nfp_prog *nfp_prog, swreg base, u8 defer) in emit_rtn() argument
263 nfp_prog->error = err; in emit_rtn()
267 __emit_br_alu(nfp_prog, reg.areg, reg.breg, 0, defer, reg.dst_lmextn, in emit_rtn()
272 __emit_immed(struct nfp_prog *nfp_prog, u16 areg, u16 breg, u16 imm_hi, in __emit_immed() argument
290 nfp_prog_push(nfp_prog, insn); in __emit_immed()
294 emit_immed(struct nfp_prog *nfp_prog, swreg dst, u16 imm, in emit_immed() argument
301 nfp_prog->error = -EFAULT; in emit_immed()
307 nfp_prog->error = err; in emit_immed()
312 __emit_immed(nfp_prog, in emit_immed()
319 __emit_shf(struct nfp_prog *nfp_prog, u16 dst, enum alu_dst_ab dst_ab, in __emit_shf() argument
327 nfp_prog->error = -EFAULT; in __emit_shf()
359 nfp_prog_push(nfp_prog, insn); in __emit_shf()
363 emit_shf(struct nfp_prog *nfp_prog, swreg dst, in emit_shf() argument
371 nfp_prog->error = err; in emit_shf()
375 __emit_shf(nfp_prog, reg.dst, reg.dst_ab, sc, shift, in emit_shf()
381 emit_shf_indir(struct nfp_prog *nfp_prog, swreg dst, in emit_shf_indir() argument
386 nfp_prog->error = -EFAULT; in emit_shf_indir()
390 emit_shf(nfp_prog, dst, lreg, op, rreg, sc, 0); in emit_shf_indir()
394 __emit_alu(struct nfp_prog *nfp_prog, u16 dst, enum alu_dst_ab dst_ab, in __emit_alu() argument
411 nfp_prog_push(nfp_prog, insn); in __emit_alu()
415 emit_alu(struct nfp_prog *nfp_prog, swreg dst, in emit_alu() argument
423 nfp_prog->error = err; in emit_alu()
427 __emit_alu(nfp_prog, reg.dst, reg.dst_ab, in emit_alu()
433 __emit_mul(struct nfp_prog *nfp_prog, enum alu_dst_ab dst_ab, u16 areg, in __emit_mul() argument
450 nfp_prog_push(nfp_prog, insn); in __emit_mul()
454 emit_mul(struct nfp_prog *nfp_prog, swreg lreg, enum mul_type type, in emit_mul() argument
462 nfp_prog->error = -EINVAL; in emit_mul()
478 nfp_prog->error = err; in emit_mul()
482 __emit_mul(nfp_prog, reg.dst_ab, areg, type, step, reg.breg, reg.swap, in emit_mul()
487 __emit_ld_field(struct nfp_prog *nfp_prog, enum shf_sc sc, in __emit_ld_field() argument
507 nfp_prog_push(nfp_prog, insn); in __emit_ld_field()
511 emit_ld_field_any(struct nfp_prog *nfp_prog, swreg dst, u8 bmask, swreg src, in emit_ld_field_any() argument
520 nfp_prog->error = err; in emit_ld_field_any()
524 __emit_ld_field(nfp_prog, sc, reg.areg, bmask, reg.breg, shift, in emit_ld_field_any()
530 emit_ld_field(struct nfp_prog *nfp_prog, swreg dst, u8 bmask, swreg src, in emit_ld_field() argument
533 emit_ld_field_any(nfp_prog, dst, bmask, src, sc, shift, false); in emit_ld_field()
537 __emit_lcsr(struct nfp_prog *nfp_prog, u16 areg, u16 breg, bool wr, u16 addr, in __emit_lcsr() argument
550 nfp_prog_push(nfp_prog, insn); in __emit_lcsr()
553 static void emit_csr_wr(struct nfp_prog *nfp_prog, swreg src, u16 addr) in emit_csr_wr() argument
570 nfp_prog->error = err; in emit_csr_wr()
574 __emit_lcsr(nfp_prog, reg.areg, reg.breg, true, addr, in emit_csr_wr()
579 static void __emit_csr_rd(struct nfp_prog *nfp_prog, u16 addr) in __emit_csr_rd() argument
581 __emit_lcsr(nfp_prog, 0, 0, false, addr, false, false); in __emit_csr_rd()
584 static void emit_nop(struct nfp_prog *nfp_prog) in emit_nop() argument
586 __emit_immed(nfp_prog, UR_REG_IMM, UR_REG_IMM, 0, 0, 0, 0, 0, 0, 0); in emit_nop()
608 static void wrp_immed(struct nfp_prog *nfp_prog, swreg dst, u32 imm) in wrp_immed() argument
614 emit_immed(nfp_prog, dst, val, IMMED_WIDTH_ALL, false, shift); in wrp_immed()
616 emit_immed(nfp_prog, dst, val, IMMED_WIDTH_ALL, true, shift); in wrp_immed()
618 emit_immed(nfp_prog, dst, imm & 0xffff, IMMED_WIDTH_ALL, in wrp_immed()
620 emit_immed(nfp_prog, dst, imm >> 16, IMMED_WIDTH_WORD, in wrp_immed()
626 wrp_zext(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta, u8 dst) in wrp_zext() argument
629 wrp_immed(nfp_prog, reg_both(dst + 1), 0); in wrp_zext()
633 wrp_immed_relo(struct nfp_prog *nfp_prog, swreg dst, u32 imm, in wrp_immed_relo() argument
638 nfp_prog->error = -EFAULT; in wrp_immed_relo()
641 emit_immed(nfp_prog, dst, imm, IMMED_WIDTH_ALL, false, IMMED_SHIFT_0B); in wrp_immed_relo()
643 nfp_prog->prog[nfp_prog->prog_len - 1] |= in wrp_immed_relo()
651 static swreg ur_load_imm_any(struct nfp_prog *nfp_prog, u32 imm, swreg tmp_reg) in ur_load_imm_any() argument
656 wrp_immed(nfp_prog, tmp_reg, imm); in ur_load_imm_any()
664 static swreg re_load_imm_any(struct nfp_prog *nfp_prog, u32 imm, swreg tmp_reg) in re_load_imm_any() argument
669 wrp_immed(nfp_prog, tmp_reg, imm); in re_load_imm_any()
673 static void wrp_nops(struct nfp_prog *nfp_prog, unsigned int count) in wrp_nops() argument
676 emit_nop(nfp_prog); in wrp_nops()
679 static void wrp_mov(struct nfp_prog *nfp_prog, swreg dst, swreg src) in wrp_mov() argument
681 emit_alu(nfp_prog, dst, reg_none(), ALU_OP_NONE, src); in wrp_mov()
684 static void wrp_reg_mov(struct nfp_prog *nfp_prog, u16 dst, u16 src) in wrp_reg_mov() argument
686 wrp_mov(nfp_prog, reg_both(dst), reg_b(src)); in wrp_reg_mov()
693 wrp_reg_subpart(struct nfp_prog *nfp_prog, swreg dst, swreg src, u8 field_len, in wrp_reg_subpart() argument
699 emit_ld_field_any(nfp_prog, dst, mask, src, sc, offset * 8, true); in wrp_reg_subpart()
706 wrp_reg_or_subpart(struct nfp_prog *nfp_prog, swreg dst, swreg src, in wrp_reg_or_subpart() argument
712 emit_ld_field(nfp_prog, dst, mask, src, sc, 32 - offset * 8); in wrp_reg_or_subpart()
716 addr40_offset(struct nfp_prog *nfp_prog, u8 src_gpr, swreg offset, in addr40_offset() argument
725 emit_alu(nfp_prog, imm_a(nfp_prog), reg_a(src_gpr), ALU_OP_ADD, offset); in addr40_offset()
726 emit_alu(nfp_prog, imm_b(nfp_prog), reg_b(src_gpr + 1), ALU_OP_ADD_C, in addr40_offset()
728 *rega = imm_a(nfp_prog); in addr40_offset()
729 *regb = imm_b(nfp_prog); in addr40_offset()
733 static int nfp_cpp_memcpy(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) in nfp_cpp_memcpy() argument
742 off = re_load_imm_any(nfp_prog, meta->insn.off, imm_b(nfp_prog)); in nfp_cpp_memcpy()
748 addr40_offset(nfp_prog, meta->insn.src_reg * 2, off, &src_base, in nfp_cpp_memcpy()
753 wrp_immed(nfp_prog, reg_none(), in nfp_cpp_memcpy()
757 emit_cmd_any(nfp_prog, CMD_TGT_READ32_SWAP, in nfp_cpp_memcpy()
763 wrp_mov(nfp_prog, reg_xfer(i), reg_xfer(i)); in nfp_cpp_memcpy()
765 off = re_load_imm_any(nfp_prog, meta->paired_st->off, imm_b(nfp_prog)); in nfp_cpp_memcpy()
769 emit_cmd(nfp_prog, CMD_TGT_WRITE8_SWAP, CMD_MODE_32b, 0, in nfp_cpp_memcpy()
774 emit_cmd(nfp_prog, CMD_TGT_WRITE32_SWAP, CMD_MODE_32b, 0, in nfp_cpp_memcpy()
779 wrp_immed(nfp_prog, reg_none(), in nfp_cpp_memcpy()
781 emit_cmd_indir(nfp_prog, CMD_TGT_WRITE8_SWAP, CMD_MODE_32b, 0, in nfp_cpp_memcpy()
786 wrp_immed(nfp_prog, reg_none(), in nfp_cpp_memcpy()
788 emit_cmd_indir(nfp_prog, CMD_TGT_WRITE32_SWAP, CMD_MODE_32b, 0, in nfp_cpp_memcpy()
795 emit_cmd(nfp_prog, CMD_TGT_WRITE32_SWAP, CMD_MODE_32b, 0, in nfp_cpp_memcpy()
799 off = re_load_imm_any(nfp_prog, meta->paired_st->off + 32, in nfp_cpp_memcpy()
800 imm_b(nfp_prog)); in nfp_cpp_memcpy()
801 emit_cmd(nfp_prog, CMD_TGT_WRITE8_SWAP, CMD_MODE_32b, 8, in nfp_cpp_memcpy()
810 wrp_immed(nfp_prog, reg_none(), in nfp_cpp_memcpy()
812 emit_cmd_indir(nfp_prog, CMD_TGT_WRITE32_SWAP, CMD_MODE_32b, 0, in nfp_cpp_memcpy()
816 off = re_load_imm_any(nfp_prog, new_off, imm_b(nfp_prog)); in nfp_cpp_memcpy()
817 emit_cmd(nfp_prog, CMD_TGT_WRITE8_SWAP, CMD_MODE_32b, in nfp_cpp_memcpy()
841 wrp_reg_subpart(nfp_prog, reg_both(meta->insn.dst_reg * 2), in nfp_cpp_memcpy()
846 wrp_reg_subpart(nfp_prog, reg_both(meta->insn.dst_reg * 2), in nfp_cpp_memcpy()
850 wrp_mov(nfp_prog, reg_both(meta->insn.dst_reg * 2), in nfp_cpp_memcpy()
854 wrp_mov(nfp_prog, reg_both(meta->insn.dst_reg * 2), in nfp_cpp_memcpy()
856 wrp_mov(nfp_prog, reg_both(meta->insn.dst_reg * 2 + 1), in nfp_cpp_memcpy()
862 wrp_immed(nfp_prog, reg_both(meta->insn.dst_reg * 2 + 1), 0); in nfp_cpp_memcpy()
868 data_ld(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta, swreg offset, in data_ld() argument
880 emit_cmd(nfp_prog, CMD_TGT_READ8, CMD_MODE_32b, 0, in data_ld()
881 pptr_reg(nfp_prog), offset, sz - 1, CMD_CTX_SWAP); in data_ld()
885 emit_shf(nfp_prog, reg_both(dst_gpr), reg_none(), SHF_OP_NONE, in data_ld()
889 wrp_mov(nfp_prog, reg_both(dst_gpr + i), reg_xfer(i)); in data_ld()
892 wrp_zext(nfp_prog, meta, dst_gpr); in data_ld()
898 data_ld_host_order(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta, in data_ld_host_order() argument
911 emit_cmd(nfp_prog, CMD_TGT_READ32_SWAP, mode, 0, in data_ld_host_order()
916 emit_ld_field_any(nfp_prog, reg_both(dst_gpr), mask, in data_ld_host_order()
920 wrp_mov(nfp_prog, reg_both(dst_gpr + i), reg_xfer(i)); in data_ld_host_order()
923 wrp_zext(nfp_prog, meta, dst_gpr); in data_ld_host_order()
929 data_ld_host_order_addr32(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta, in data_ld_host_order_addr32() argument
932 return data_ld_host_order(nfp_prog, meta, dst_gpr, reg_a(src_gpr), in data_ld_host_order_addr32()
937 data_ld_host_order_addr40(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta, in data_ld_host_order_addr40() argument
942 addr40_offset(nfp_prog, src_gpr, offset, ®a, ®b); in data_ld_host_order_addr40()
944 return data_ld_host_order(nfp_prog, meta, dst_gpr, rega, regb, in data_ld_host_order_addr40()
949 construct_data_ind_ld(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta, in construct_data_ind_ld() argument
955 tmp_reg = ur_load_imm_any(nfp_prog, offset, imm_b(nfp_prog)); in construct_data_ind_ld()
956 emit_alu(nfp_prog, imm_both(nfp_prog), reg_a(src), ALU_OP_ADD, tmp_reg); in construct_data_ind_ld()
959 emit_alu(nfp_prog, imm_a(nfp_prog), in construct_data_ind_ld()
960 imm_a(nfp_prog), ALU_OP_ADD, reg_imm(size)); in construct_data_ind_ld()
961 emit_alu(nfp_prog, reg_none(), in construct_data_ind_ld()
962 plen_reg(nfp_prog), ALU_OP_SUB, imm_a(nfp_prog)); in construct_data_ind_ld()
963 emit_br_relo(nfp_prog, BR_BLO, BR_OFF_RELO, 0, RELO_BR_GO_ABORT); in construct_data_ind_ld()
966 return data_ld(nfp_prog, meta, imm_b(nfp_prog), 0, size); in construct_data_ind_ld()
970 construct_data_ld(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta, in construct_data_ld() argument
976 tmp_reg = ur_load_imm_any(nfp_prog, offset + size, imm_a(nfp_prog)); in construct_data_ld()
977 emit_alu(nfp_prog, reg_none(), plen_reg(nfp_prog), ALU_OP_SUB, tmp_reg); in construct_data_ld()
978 emit_br_relo(nfp_prog, BR_BLO, BR_OFF_RELO, 0, RELO_BR_GO_ABORT); in construct_data_ld()
981 tmp_reg = re_load_imm_any(nfp_prog, offset, imm_b(nfp_prog)); in construct_data_ld()
982 return data_ld(nfp_prog, meta, tmp_reg, 0, size); in construct_data_ld()
986 data_stx_host_order(struct nfp_prog *nfp_prog, u8 dst_gpr, swreg offset, in data_stx_host_order() argument
992 wrp_mov(nfp_prog, reg_xfer(i), reg_a(src_gpr + i)); in data_stx_host_order()
994 emit_cmd(nfp_prog, CMD_TGT_WRITE8_SWAP, CMD_MODE_32b, 0, in data_stx_host_order()
1001 data_st_host_order(struct nfp_prog *nfp_prog, u8 dst_gpr, swreg offset, in data_st_host_order() argument
1004 wrp_immed(nfp_prog, reg_xfer(0), imm); in data_st_host_order()
1006 wrp_immed(nfp_prog, reg_xfer(1), imm >> 32); in data_st_host_order()
1008 emit_cmd(nfp_prog, CMD_TGT_WRITE8_SWAP, CMD_MODE_32b, 0, in data_st_host_order()
1015 (*lmem_step)(struct nfp_prog *nfp_prog, u8 gpr, u8 gpr_byte, s32 off,
1020 wrp_lmem_load(struct nfp_prog *nfp_prog, u8 dst, u8 dst_byte, s32 off, in wrp_lmem_load() argument
1038 wrp_mov(nfp_prog, reg_both(dst), in wrp_lmem_load()
1070 reg = imm_a(nfp_prog); in wrp_lmem_load()
1077 wrp_mov(nfp_prog, reg, reg_lm(0, idx)); in wrp_lmem_load()
1080 emit_ld_field_any(nfp_prog, reg_both(dst), mask, reg, sc, shf, new_gpr); in wrp_lmem_load()
1083 wrp_mov(nfp_prog, reg_none(), reg_lm_inc(3)); in wrp_lmem_load()
1089 wrp_lmem_store(struct nfp_prog *nfp_prog, u8 src, u8 src_byte, s32 off, in wrp_lmem_store() argument
1107 wrp_mov(nfp_prog, in wrp_lmem_store()
1140 reg = imm_a(nfp_prog); in wrp_lmem_store()
1145 wrp_mov(nfp_prog, reg, reg_lm(0, idx)); in wrp_lmem_store()
1148 emit_ld_field(nfp_prog, reg, mask, reg_b(src), sc, shf); in wrp_lmem_store()
1152 wrp_mov(nfp_prog, reg_lm(0, idx), reg); in wrp_lmem_store()
1154 wrp_mov(nfp_prog, reg_none(), reg_lm_inc(3)); in wrp_lmem_store()
1161 mem_op_stack(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta, in mem_op_stack() argument
1165 s32 off = nfp_prog->stack_frame_depth + meta->insn.off + ptr_off; in mem_op_stack()
1180 stack_off_reg = ur_load_imm_any(nfp_prog, meta->insn.off, in mem_op_stack()
1181 stack_imm(nfp_prog)); in mem_op_stack()
1183 emit_alu(nfp_prog, imm_b(nfp_prog), in mem_op_stack()
1197 stack_off_reg = ur_load_imm_any(nfp_prog, round_down(off, 32), in mem_op_stack()
1198 stack_imm(nfp_prog)); in mem_op_stack()
1199 emit_alu(nfp_prog, imm_b(nfp_prog), in mem_op_stack()
1200 stack_reg(nfp_prog), ALU_OP_ADD, stack_off_reg); in mem_op_stack()
1204 stack_off_reg = ur_load_imm_any(nfp_prog, round_down(off, 4), in mem_op_stack()
1205 stack_imm(nfp_prog)); in mem_op_stack()
1207 emit_alu(nfp_prog, imm_b(nfp_prog), in mem_op_stack()
1208 stack_reg(nfp_prog), ALU_OP_ADD, stack_off_reg); in mem_op_stack()
1218 emit_csr_wr(nfp_prog, imm_b(nfp_prog), NFP_CSR_ACT_LM_ADDR3); in mem_op_stack()
1224 wrp_nops(nfp_prog, nop_cnt); in mem_op_stack()
1228 wrp_zext(nfp_prog, meta, gpr); in mem_op_stack()
1243 ret = step(nfp_prog, gpr, gpr_byte, off, slice_size, in mem_op_stack()
1265 wrp_alu_imm(struct nfp_prog *nfp_prog, u8 dst, enum alu_op alu_op, u32 imm) in wrp_alu_imm() argument
1271 wrp_immed(nfp_prog, reg_both(dst), 0); in wrp_alu_imm()
1277 wrp_immed(nfp_prog, reg_both(dst), ~0U); in wrp_alu_imm()
1283 emit_alu(nfp_prog, reg_both(dst), reg_none(), in wrp_alu_imm()
1289 tmp_reg = ur_load_imm_any(nfp_prog, imm, imm_b(nfp_prog)); in wrp_alu_imm()
1290 emit_alu(nfp_prog, reg_both(dst), reg_a(dst), alu_op, tmp_reg); in wrp_alu_imm()
1294 wrp_alu64_imm(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta, in wrp_alu64_imm() argument
1305 wrp_alu_imm(nfp_prog, insn->dst_reg * 2, alu_op, imm & ~0U); in wrp_alu64_imm()
1306 wrp_alu_imm(nfp_prog, insn->dst_reg * 2 + 1, alu_op, imm >> 32); in wrp_alu64_imm()
1312 wrp_alu64_reg(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta, in wrp_alu64_reg() argument
1317 emit_alu(nfp_prog, reg_both(dst), reg_a(dst), alu_op, reg_b(src)); in wrp_alu64_reg()
1318 emit_alu(nfp_prog, reg_both(dst + 1), in wrp_alu64_reg()
1325 wrp_alu32_imm(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta, in wrp_alu32_imm() argument
1331 wrp_alu_imm(nfp_prog, dst, alu_op, insn->imm); in wrp_alu32_imm()
1332 wrp_zext(nfp_prog, meta, dst); in wrp_alu32_imm()
1338 wrp_alu32_reg(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta, in wrp_alu32_reg() argument
1343 emit_alu(nfp_prog, reg_both(dst), reg_a(dst), alu_op, reg_b(src)); in wrp_alu32_reg()
1344 wrp_zext(nfp_prog, meta, dst); in wrp_alu32_reg()
1350 wrp_test_reg_one(struct nfp_prog *nfp_prog, u8 dst, enum alu_op alu_op, u8 src, in wrp_test_reg_one() argument
1353 emit_alu(nfp_prog, reg_none(), reg_a(dst), alu_op, reg_b(src)); in wrp_test_reg_one()
1354 emit_br(nfp_prog, br_mask, off, 0); in wrp_test_reg_one()
1358 wrp_test_reg(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta, in wrp_test_reg() argument
1363 wrp_test_reg_one(nfp_prog, insn->dst_reg * 2, alu_op, in wrp_test_reg()
1366 wrp_test_reg_one(nfp_prog, insn->dst_reg * 2 + 1, alu_op, in wrp_test_reg()
1400 static int cmp_imm(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) in cmp_imm() argument
1416 tmp_reg = ur_load_imm_any(nfp_prog, imm & ~0U, imm_b(nfp_prog)); in cmp_imm()
1418 emit_alu(nfp_prog, reg_none(), reg_a(reg), alu_op, tmp_reg); in cmp_imm()
1420 emit_alu(nfp_prog, reg_none(), tmp_reg, alu_op, reg_a(reg)); in cmp_imm()
1423 tmp_reg = ur_load_imm_any(nfp_prog, imm >> 32, imm_b(nfp_prog)); in cmp_imm()
1425 emit_alu(nfp_prog, reg_none(), in cmp_imm()
1428 emit_alu(nfp_prog, reg_none(), in cmp_imm()
1432 emit_br(nfp_prog, code->br_mask, insn->off, 0); in cmp_imm()
1437 static int cmp_reg(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) in cmp_reg() argument
1456 emit_alu(nfp_prog, reg_none(), reg_a(areg), ALU_OP_SUB, reg_b(breg)); in cmp_reg()
1458 emit_alu(nfp_prog, reg_none(), in cmp_reg()
1460 emit_br(nfp_prog, code->br_mask, insn->off, 0); in cmp_reg()
1465 static void wrp_end32(struct nfp_prog *nfp_prog, swreg reg_in, u8 gpr_out) in wrp_end32() argument
1467 emit_ld_field(nfp_prog, reg_both(gpr_out), 0xf, reg_in, in wrp_end32()
1469 emit_ld_field(nfp_prog, reg_both(gpr_out), 0x5, reg_a(gpr_out), in wrp_end32()
1474 wrp_mul_u32(struct nfp_prog *nfp_prog, swreg dst_hi, swreg dst_lo, swreg lreg, in wrp_mul_u32() argument
1477 emit_mul(nfp_prog, lreg, MUL_TYPE_START, MUL_STEP_NONE, rreg); in wrp_mul_u32()
1478 emit_mul(nfp_prog, lreg, MUL_TYPE_STEP_32x32, MUL_STEP_1, rreg); in wrp_mul_u32()
1479 emit_mul(nfp_prog, lreg, MUL_TYPE_STEP_32x32, MUL_STEP_2, rreg); in wrp_mul_u32()
1480 emit_mul(nfp_prog, lreg, MUL_TYPE_STEP_32x32, MUL_STEP_3, rreg); in wrp_mul_u32()
1481 emit_mul(nfp_prog, lreg, MUL_TYPE_STEP_32x32, MUL_STEP_4, rreg); in wrp_mul_u32()
1482 emit_mul(nfp_prog, dst_lo, MUL_TYPE_STEP_32x32, MUL_LAST, reg_none()); in wrp_mul_u32()
1484 emit_mul(nfp_prog, dst_hi, MUL_TYPE_STEP_32x32, MUL_LAST_2, in wrp_mul_u32()
1487 wrp_immed(nfp_prog, dst_hi, 0); in wrp_mul_u32()
1491 wrp_mul_u16(struct nfp_prog *nfp_prog, swreg dst_hi, swreg dst_lo, swreg lreg, in wrp_mul_u16() argument
1494 emit_mul(nfp_prog, lreg, MUL_TYPE_START, MUL_STEP_NONE, rreg); in wrp_mul_u16()
1495 emit_mul(nfp_prog, lreg, MUL_TYPE_STEP_16x16, MUL_STEP_1, rreg); in wrp_mul_u16()
1496 emit_mul(nfp_prog, lreg, MUL_TYPE_STEP_16x16, MUL_STEP_2, rreg); in wrp_mul_u16()
1497 emit_mul(nfp_prog, dst_lo, MUL_TYPE_STEP_16x16, MUL_LAST, reg_none()); in wrp_mul_u16()
1501 wrp_mul(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta, in wrp_mul() argument
1520 multiplier = ur_load_imm_any(nfp_prog, imm, imm_b(nfp_prog)); in wrp_mul()
1524 wrp_mul_u32(nfp_prog, dst_hi, dst_lo, multiplicand, multiplier, in wrp_mul()
1527 wrp_mul_u16(nfp_prog, dst_hi, dst_lo, multiplicand, multiplier); in wrp_mul()
1532 static int wrp_div_imm(struct nfp_prog *nfp_prog, u8 dst, u64 imm) in wrp_div_imm() argument
1540 wrp_immed(nfp_prog, dst_both, 0); in wrp_div_imm()
1556 swreg tmp_b = ur_load_imm_any(nfp_prog, imm, imm_b(nfp_prog)); in wrp_div_imm()
1558 emit_alu(nfp_prog, reg_none(), dst_a, ALU_OP_SUB, tmp_b); in wrp_div_imm()
1559 wrp_immed(nfp_prog, imm_a(nfp_prog), 0); in wrp_div_imm()
1560 emit_alu(nfp_prog, dst_both, imm_a(nfp_prog), ALU_OP_ADD_C, in wrp_div_imm()
1573 magic = ur_load_imm_any(nfp_prog, rvalue.m, imm_b(nfp_prog)); in wrp_div_imm()
1575 emit_shf(nfp_prog, dst_both, reg_none(), SHF_OP_NONE, dst_b, in wrp_div_imm()
1578 wrp_mul_u32(nfp_prog, imm_both(nfp_prog), reg_none(), dst_a, in wrp_div_imm()
1580 emit_alu(nfp_prog, dst_both, dst_a, ALU_OP_SUB, in wrp_div_imm()
1581 imm_b(nfp_prog)); in wrp_div_imm()
1582 emit_shf(nfp_prog, dst_both, reg_none(), SHF_OP_NONE, dst_b, in wrp_div_imm()
1584 emit_alu(nfp_prog, dst_both, dst_a, ALU_OP_ADD, in wrp_div_imm()
1585 imm_b(nfp_prog)); in wrp_div_imm()
1586 emit_shf(nfp_prog, dst_both, reg_none(), SHF_OP_NONE, dst_b, in wrp_div_imm()
1590 emit_shf(nfp_prog, dst_both, reg_none(), SHF_OP_NONE, in wrp_div_imm()
1592 wrp_mul_u32(nfp_prog, dst_both, reg_none(), dst_a, magic, true); in wrp_div_imm()
1593 emit_shf(nfp_prog, dst_both, reg_none(), SHF_OP_NONE, in wrp_div_imm()
1600 static int adjust_head(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) in adjust_head() argument
1602 swreg tmp = imm_a(nfp_prog), tmp_len = imm_b(nfp_prog); in adjust_head()
1606 adjust_head = &nfp_prog->bpf->adjust_head; in adjust_head()
1609 if (nfp_prog->adjust_head_location != UINT_MAX) { in adjust_head()
1610 if (WARN_ON_ONCE(nfp_prog->adjust_head_location != meta->n)) in adjust_head()
1613 emit_alu(nfp_prog, pptr_reg(nfp_prog), in adjust_head()
1614 reg_a(2 * 2), ALU_OP_ADD, pptr_reg(nfp_prog)); in adjust_head()
1615 emit_alu(nfp_prog, plen_reg(nfp_prog), in adjust_head()
1616 plen_reg(nfp_prog), ALU_OP_SUB, reg_a(2 * 2)); in adjust_head()
1617 emit_alu(nfp_prog, pv_len(nfp_prog), in adjust_head()
1618 pv_len(nfp_prog), ALU_OP_SUB, reg_a(2 * 2)); in adjust_head()
1620 wrp_immed(nfp_prog, reg_both(0), 0); in adjust_head()
1621 wrp_immed(nfp_prog, reg_both(1), 0); in adjust_head()
1630 ret_einval = nfp_prog_current_offset(nfp_prog) + 14; in adjust_head()
1634 emit_alu(nfp_prog, tmp, in adjust_head()
1635 reg_a(2 * 2), ALU_OP_ADD_2B, pptr_reg(nfp_prog)); in adjust_head()
1638 emit_alu(nfp_prog, reg_none(), in adjust_head()
1640 emit_br(nfp_prog, BR_BLO, ret_einval, 0); in adjust_head()
1641 emit_alu(nfp_prog, reg_none(), in adjust_head()
1643 emit_br(nfp_prog, BR_BLO, ret_einval, 0); in adjust_head()
1646 emit_alu(nfp_prog, tmp_len, in adjust_head()
1647 plen_reg(nfp_prog), ALU_OP_SUB, reg_a(2 * 2)); in adjust_head()
1648 emit_alu(nfp_prog, reg_none(), in adjust_head()
1650 emit_br(nfp_prog, BR_BMI, ret_einval, 0); in adjust_head()
1653 wrp_immed(nfp_prog, reg_both(0), 0); in adjust_head()
1654 wrp_immed(nfp_prog, reg_both(1), 0); in adjust_head()
1657 emit_ld_field(nfp_prog, pptr_reg(nfp_prog), 0x3, tmp, SHF_SC_NONE, 0); in adjust_head()
1660 emit_br(nfp_prog, BR_UNC, end, 2); in adjust_head()
1662 emit_alu(nfp_prog, plen_reg(nfp_prog), in adjust_head()
1663 plen_reg(nfp_prog), ALU_OP_SUB, reg_a(2 * 2)); in adjust_head()
1664 emit_alu(nfp_prog, pv_len(nfp_prog), in adjust_head()
1665 pv_len(nfp_prog), ALU_OP_SUB, reg_a(2 * 2)); in adjust_head()
1668 if (!nfp_prog_confirm_current_offset(nfp_prog, ret_einval)) in adjust_head()
1671 wrp_immed(nfp_prog, reg_both(0), -22); in adjust_head()
1672 wrp_immed(nfp_prog, reg_both(1), ~0); in adjust_head()
1674 if (!nfp_prog_confirm_current_offset(nfp_prog, end)) in adjust_head()
1680 static int adjust_tail(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) in adjust_tail() argument
1685 BUILD_BUG_ON(plen_reg(nfp_prog) != reg_b(STATIC_REG_PKT_LEN)); in adjust_tail()
1687 plen = imm_a(nfp_prog); in adjust_tail()
1690 ret_einval = nfp_prog_current_offset(nfp_prog) + 9; in adjust_tail()
1691 end = nfp_prog_current_offset(nfp_prog) + 11; in adjust_tail()
1694 emit_alu(nfp_prog, plen, plen_reg(nfp_prog), ALU_OP_ADD, delta); in adjust_tail()
1698 emit_br(nfp_prog, BR_BCC, ret_einval, 0); in adjust_tail()
1701 emit_alu(nfp_prog, reg_none(), plen, ALU_OP_SUB, reg_imm(ETH_HLEN)); in adjust_tail()
1702 emit_br(nfp_prog, BR_BMI, ret_einval, 0); in adjust_tail()
1704 emit_alu(nfp_prog, plen_reg(nfp_prog), in adjust_tail()
1705 plen_reg(nfp_prog), ALU_OP_ADD, delta); in adjust_tail()
1706 emit_alu(nfp_prog, pv_len(nfp_prog), in adjust_tail()
1707 pv_len(nfp_prog), ALU_OP_ADD, delta); in adjust_tail()
1709 emit_br(nfp_prog, BR_UNC, end, 2); in adjust_tail()
1710 wrp_immed(nfp_prog, reg_both(0), 0); in adjust_tail()
1711 wrp_immed(nfp_prog, reg_both(1), 0); in adjust_tail()
1713 if (!nfp_prog_confirm_current_offset(nfp_prog, ret_einval)) in adjust_tail()
1716 wrp_immed(nfp_prog, reg_both(0), -22); in adjust_tail()
1717 wrp_immed(nfp_prog, reg_both(1), ~0); in adjust_tail()
1719 if (!nfp_prog_confirm_current_offset(nfp_prog, end)) in adjust_tail()
1726 map_call_stack_common(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) in map_call_stack_common() argument
1733 lm_off = nfp_prog->stack_frame_depth; in map_call_stack_common()
1739 emit_csr_wr(nfp_prog, reg_b(2 * 2), NFP_CSR_ACT_LM_ADDR0); in map_call_stack_common()
1741 emit_csr_wr(nfp_prog, reg_b(3 * 2), NFP_CSR_ACT_LM_ADDR2); in map_call_stack_common()
1743 emit_br_relo(nfp_prog, BR_UNC, BR_OFF_RELO + meta->func_id, in map_call_stack_common()
1745 ret_tgt = nfp_prog_current_offset(nfp_prog) + 2; in map_call_stack_common()
1748 wrp_mov(nfp_prog, reg_a(0), reg_a(2)); in map_call_stack_common()
1751 wrp_immed_relo(nfp_prog, reg_b(0), ret_tgt, RELO_IMMED_REL); in map_call_stack_common()
1753 if (!nfp_prog_confirm_current_offset(nfp_prog, ret_tgt)) in map_call_stack_common()
1760 emit_csr_wr(nfp_prog, stack_reg(nfp_prog), NFP_CSR_ACT_LM_ADDR0); in map_call_stack_common()
1761 wrp_nops(nfp_prog, 3); in map_call_stack_common()
1767 nfp_get_prandom_u32(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) in nfp_get_prandom_u32() argument
1769 __emit_csr_rd(nfp_prog, NFP_CSR_PSEUDO_RND_NUM); in nfp_get_prandom_u32()
1771 emit_immed(nfp_prog, reg_both(0), 0, in nfp_get_prandom_u32()
1773 emit_immed(nfp_prog, reg_both(1), 0, in nfp_get_prandom_u32()
1779 nfp_perf_event_output(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) in nfp_perf_event_output() argument
1784 ptr_type = ur_load_imm_any(nfp_prog, meta->arg1.type, imm_a(nfp_prog)); in nfp_perf_event_output()
1786 ret_tgt = nfp_prog_current_offset(nfp_prog) + 3; in nfp_perf_event_output()
1788 emit_br_relo(nfp_prog, BR_UNC, BR_OFF_RELO + meta->func_id, in nfp_perf_event_output()
1792 wrp_mov(nfp_prog, reg_a(1), ptr_type); in nfp_perf_event_output()
1795 wrp_immed_relo(nfp_prog, reg_b(0), ret_tgt, RELO_IMMED_REL); in nfp_perf_event_output()
1797 if (!nfp_prog_confirm_current_offset(nfp_prog, ret_tgt)) in nfp_perf_event_output()
1804 nfp_queue_select(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) in nfp_queue_select() argument
1808 jmp_tgt = nfp_prog_current_offset(nfp_prog) + 5; in nfp_queue_select()
1811 emit_alu(nfp_prog, reg_none(), reg_a(meta->insn.src_reg * 2), in nfp_queue_select()
1813 emit_br(nfp_prog, BR_BEQ, jmp_tgt, 2); in nfp_queue_select()
1816 emit_shf(nfp_prog, pv_qsel_set(nfp_prog), in nfp_queue_select()
1817 pv_qsel_set(nfp_prog), SHF_OP_OR, reg_imm(1), in nfp_queue_select()
1819 emit_ld_field(nfp_prog, in nfp_queue_select()
1820 pv_qsel_val(nfp_prog), 0x1, reg_b(meta->insn.src_reg * 2), in nfp_queue_select()
1825 emit_ld_field(nfp_prog, in nfp_queue_select()
1826 pv_qsel_val(nfp_prog), 0x1, reg_imm(NFP_NET_RXR_MAX), in nfp_queue_select()
1829 if (!nfp_prog_confirm_current_offset(nfp_prog, jmp_tgt)) in nfp_queue_select()
1836 static int mov_reg64(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) in mov_reg64() argument
1845 stack_depth_reg = ur_load_imm_any(nfp_prog, in mov_reg64()
1846 nfp_prog->stack_frame_depth, in mov_reg64()
1847 stack_imm(nfp_prog)); in mov_reg64()
1848 emit_alu(nfp_prog, reg_both(dst), stack_reg(nfp_prog), in mov_reg64()
1850 wrp_immed(nfp_prog, reg_both(dst + 1), 0); in mov_reg64()
1852 wrp_reg_mov(nfp_prog, dst, src); in mov_reg64()
1853 wrp_reg_mov(nfp_prog, dst + 1, src + 1); in mov_reg64()
1859 static int mov_imm64(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) in mov_imm64() argument
1863 wrp_immed(nfp_prog, reg_both(meta->insn.dst_reg * 2), imm & ~0U); in mov_imm64()
1864 wrp_immed(nfp_prog, reg_both(meta->insn.dst_reg * 2 + 1), imm >> 32); in mov_imm64()
1869 static int xor_reg64(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) in xor_reg64() argument
1871 return wrp_alu64_reg(nfp_prog, meta, ALU_OP_XOR); in xor_reg64()
1874 static int xor_imm64(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) in xor_imm64() argument
1876 return wrp_alu64_imm(nfp_prog, meta, ALU_OP_XOR, !meta->insn.imm); in xor_imm64()
1879 static int and_reg64(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) in and_reg64() argument
1881 return wrp_alu64_reg(nfp_prog, meta, ALU_OP_AND); in and_reg64()
1884 static int and_imm64(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) in and_imm64() argument
1886 return wrp_alu64_imm(nfp_prog, meta, ALU_OP_AND, !~meta->insn.imm); in and_imm64()
1889 static int or_reg64(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) in or_reg64() argument
1891 return wrp_alu64_reg(nfp_prog, meta, ALU_OP_OR); in or_reg64()
1894 static int or_imm64(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) in or_imm64() argument
1896 return wrp_alu64_imm(nfp_prog, meta, ALU_OP_OR, !meta->insn.imm); in or_imm64()
1899 static int add_reg64(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) in add_reg64() argument
1903 emit_alu(nfp_prog, reg_both(insn->dst_reg * 2), in add_reg64()
1906 emit_alu(nfp_prog, reg_both(insn->dst_reg * 2 + 1), in add_reg64()
1913 static int add_imm64(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) in add_imm64() argument
1918 wrp_alu_imm(nfp_prog, insn->dst_reg * 2, ALU_OP_ADD, imm & ~0U); in add_imm64()
1919 wrp_alu_imm(nfp_prog, insn->dst_reg * 2 + 1, ALU_OP_ADD_C, imm >> 32); in add_imm64()
1924 static int sub_reg64(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) in sub_reg64() argument
1928 emit_alu(nfp_prog, reg_both(insn->dst_reg * 2), in sub_reg64()
1931 emit_alu(nfp_prog, reg_both(insn->dst_reg * 2 + 1), in sub_reg64()
1938 static int sub_imm64(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) in sub_imm64() argument
1943 wrp_alu_imm(nfp_prog, insn->dst_reg * 2, ALU_OP_SUB, imm & ~0U); in sub_imm64()
1944 wrp_alu_imm(nfp_prog, insn->dst_reg * 2 + 1, ALU_OP_SUB_C, imm >> 32); in sub_imm64()
1949 static int mul_reg64(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) in mul_reg64() argument
1951 return wrp_mul(nfp_prog, meta, true, true); in mul_reg64()
1954 static int mul_imm64(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) in mul_imm64() argument
1956 return wrp_mul(nfp_prog, meta, true, false); in mul_imm64()
1959 static int div_imm64(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) in div_imm64() argument
1963 return wrp_div_imm(nfp_prog, insn->dst_reg * 2, insn->imm); in div_imm64()
1966 static int div_reg64(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) in div_reg64() argument
1971 return wrp_div_imm(nfp_prog, meta->insn.dst_reg * 2, meta->umin_src); in div_reg64()
1974 static int neg_reg64(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) in neg_reg64() argument
1978 emit_alu(nfp_prog, reg_both(insn->dst_reg * 2), reg_imm(0), in neg_reg64()
1980 emit_alu(nfp_prog, reg_both(insn->dst_reg * 2 + 1), reg_imm(0), in neg_reg64()
1996 static int __shl_imm64(struct nfp_prog *nfp_prog, u8 dst, u8 shift_amt) in __shl_imm64() argument
2002 emit_shf(nfp_prog, reg_both(dst + 1), reg_a(dst + 1), in __shl_imm64()
2005 emit_shf(nfp_prog, reg_both(dst), reg_none(), SHF_OP_NONE, in __shl_imm64()
2008 wrp_reg_mov(nfp_prog, dst + 1, dst); in __shl_imm64()
2009 wrp_immed(nfp_prog, reg_both(dst), 0); in __shl_imm64()
2011 emit_shf(nfp_prog, reg_both(dst + 1), reg_none(), SHF_OP_NONE, in __shl_imm64()
2013 wrp_immed(nfp_prog, reg_both(dst), 0); in __shl_imm64()
2019 static int shl_imm64(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) in shl_imm64() argument
2024 return __shl_imm64(nfp_prog, dst, insn->imm); in shl_imm64()
2027 static void shl_reg64_lt32_high(struct nfp_prog *nfp_prog, u8 dst, u8 src) in shl_reg64_lt32_high() argument
2029 emit_alu(nfp_prog, imm_both(nfp_prog), reg_imm(32), ALU_OP_SUB, in shl_reg64_lt32_high()
2031 emit_alu(nfp_prog, reg_none(), imm_a(nfp_prog), ALU_OP_OR, reg_imm(0)); in shl_reg64_lt32_high()
2032 emit_shf_indir(nfp_prog, reg_both(dst + 1), reg_a(dst + 1), SHF_OP_NONE, in shl_reg64_lt32_high()
2037 static void shl_reg64_lt32_low(struct nfp_prog *nfp_prog, u8 dst, u8 src) in shl_reg64_lt32_low() argument
2039 emit_alu(nfp_prog, reg_none(), reg_a(src), ALU_OP_OR, reg_imm(0)); in shl_reg64_lt32_low()
2040 emit_shf_indir(nfp_prog, reg_both(dst), reg_none(), SHF_OP_NONE, in shl_reg64_lt32_low()
2044 static void shl_reg64_lt32(struct nfp_prog *nfp_prog, u8 dst, u8 src) in shl_reg64_lt32() argument
2046 shl_reg64_lt32_high(nfp_prog, dst, src); in shl_reg64_lt32()
2047 shl_reg64_lt32_low(nfp_prog, dst, src); in shl_reg64_lt32()
2050 static void shl_reg64_ge32(struct nfp_prog *nfp_prog, u8 dst, u8 src) in shl_reg64_ge32() argument
2052 emit_alu(nfp_prog, reg_none(), reg_a(src), ALU_OP_OR, reg_imm(0)); in shl_reg64_ge32()
2053 emit_shf_indir(nfp_prog, reg_both(dst + 1), reg_none(), SHF_OP_NONE, in shl_reg64_ge32()
2055 wrp_immed(nfp_prog, reg_both(dst), 0); in shl_reg64_ge32()
2058 static int shl_reg64(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) in shl_reg64() argument
2068 return __shl_imm64(nfp_prog, dst, umin); in shl_reg64()
2072 shl_reg64_lt32(nfp_prog, dst, src); in shl_reg64()
2074 shl_reg64_ge32(nfp_prog, dst, src); in shl_reg64()
2081 label_ge32 = nfp_prog_current_offset(nfp_prog) + 7; in shl_reg64()
2082 emit_br_bset(nfp_prog, reg_a(src), 5, label_ge32, 0); in shl_reg64()
2084 shl_reg64_lt32_high(nfp_prog, dst, src); in shl_reg64()
2085 label_end = nfp_prog_current_offset(nfp_prog) + 6; in shl_reg64()
2086 emit_br(nfp_prog, BR_UNC, label_end, 2); in shl_reg64()
2088 shl_reg64_lt32_low(nfp_prog, dst, src); in shl_reg64()
2090 if (!nfp_prog_confirm_current_offset(nfp_prog, label_ge32)) in shl_reg64()
2092 shl_reg64_ge32(nfp_prog, dst, src); in shl_reg64()
2094 if (!nfp_prog_confirm_current_offset(nfp_prog, label_end)) in shl_reg64()
2111 static int __shr_imm64(struct nfp_prog *nfp_prog, u8 dst, u8 shift_amt) in __shr_imm64() argument
2117 emit_shf(nfp_prog, reg_both(dst), reg_a(dst + 1), SHF_OP_NONE, in __shr_imm64()
2119 emit_shf(nfp_prog, reg_both(dst + 1), reg_none(), SHF_OP_NONE, in __shr_imm64()
2122 wrp_reg_mov(nfp_prog, dst, dst + 1); in __shr_imm64()
2123 wrp_immed(nfp_prog, reg_both(dst + 1), 0); in __shr_imm64()
2125 emit_shf(nfp_prog, reg_both(dst), reg_none(), SHF_OP_NONE, in __shr_imm64()
2127 wrp_immed(nfp_prog, reg_both(dst + 1), 0); in __shr_imm64()
2133 static int shr_imm64(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) in shr_imm64() argument
2138 return __shr_imm64(nfp_prog, dst, insn->imm); in shr_imm64()
2142 static void shr_reg64_lt32_high(struct nfp_prog *nfp_prog, u8 dst, u8 src) in shr_reg64_lt32_high() argument
2144 emit_alu(nfp_prog, reg_none(), reg_a(src), ALU_OP_OR, reg_imm(0)); in shr_reg64_lt32_high()
2145 emit_shf_indir(nfp_prog, reg_both(dst + 1), reg_none(), SHF_OP_NONE, in shr_reg64_lt32_high()
2149 static void shr_reg64_lt32_low(struct nfp_prog *nfp_prog, u8 dst, u8 src) in shr_reg64_lt32_low() argument
2151 emit_alu(nfp_prog, reg_none(), reg_a(src), ALU_OP_OR, reg_imm(0)); in shr_reg64_lt32_low()
2152 emit_shf_indir(nfp_prog, reg_both(dst), reg_a(dst + 1), SHF_OP_NONE, in shr_reg64_lt32_low()
2156 static void shr_reg64_lt32(struct nfp_prog *nfp_prog, u8 dst, u8 src) in shr_reg64_lt32() argument
2158 shr_reg64_lt32_low(nfp_prog, dst, src); in shr_reg64_lt32()
2159 shr_reg64_lt32_high(nfp_prog, dst, src); in shr_reg64_lt32()
2162 static void shr_reg64_ge32(struct nfp_prog *nfp_prog, u8 dst, u8 src) in shr_reg64_ge32() argument
2164 emit_alu(nfp_prog, reg_none(), reg_a(src), ALU_OP_OR, reg_imm(0)); in shr_reg64_ge32()
2165 emit_shf_indir(nfp_prog, reg_both(dst), reg_none(), SHF_OP_NONE, in shr_reg64_ge32()
2167 wrp_immed(nfp_prog, reg_both(dst + 1), 0); in shr_reg64_ge32()
2170 static int shr_reg64(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) in shr_reg64() argument
2180 return __shr_imm64(nfp_prog, dst, umin); in shr_reg64()
2184 shr_reg64_lt32(nfp_prog, dst, src); in shr_reg64()
2186 shr_reg64_ge32(nfp_prog, dst, src); in shr_reg64()
2193 label_ge32 = nfp_prog_current_offset(nfp_prog) + 6; in shr_reg64()
2194 emit_br_bset(nfp_prog, reg_a(src), 5, label_ge32, 0); in shr_reg64()
2195 shr_reg64_lt32_low(nfp_prog, dst, src); in shr_reg64()
2196 label_end = nfp_prog_current_offset(nfp_prog) + 6; in shr_reg64()
2197 emit_br(nfp_prog, BR_UNC, label_end, 2); in shr_reg64()
2199 shr_reg64_lt32_high(nfp_prog, dst, src); in shr_reg64()
2201 if (!nfp_prog_confirm_current_offset(nfp_prog, label_ge32)) in shr_reg64()
2203 shr_reg64_ge32(nfp_prog, dst, src); in shr_reg64()
2205 if (!nfp_prog_confirm_current_offset(nfp_prog, label_end)) in shr_reg64()
2215 static int __ashr_imm64(struct nfp_prog *nfp_prog, u8 dst, u8 shift_amt) in __ashr_imm64() argument
2221 emit_shf(nfp_prog, reg_both(dst), reg_a(dst + 1), SHF_OP_NONE, in __ashr_imm64()
2224 emit_alu(nfp_prog, reg_none(), reg_a(dst + 1), ALU_OP_OR, in __ashr_imm64()
2226 emit_shf(nfp_prog, reg_both(dst + 1), reg_none(), SHF_OP_ASHR, in __ashr_imm64()
2230 wrp_reg_mov(nfp_prog, dst, dst + 1); in __ashr_imm64()
2231 emit_shf(nfp_prog, reg_both(dst + 1), reg_none(), SHF_OP_ASHR, in __ashr_imm64()
2234 emit_alu(nfp_prog, reg_none(), reg_a(dst + 1), ALU_OP_OR, in __ashr_imm64()
2236 emit_shf(nfp_prog, reg_both(dst), reg_none(), SHF_OP_ASHR, in __ashr_imm64()
2238 emit_shf(nfp_prog, reg_both(dst + 1), reg_none(), SHF_OP_ASHR, in __ashr_imm64()
2245 static int ashr_imm64(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) in ashr_imm64() argument
2250 return __ashr_imm64(nfp_prog, dst, insn->imm); in ashr_imm64()
2253 static void ashr_reg64_lt32_high(struct nfp_prog *nfp_prog, u8 dst, u8 src) in ashr_reg64_lt32_high() argument
2258 emit_alu(nfp_prog, reg_none(), reg_a(src), ALU_OP_OR, reg_b(dst + 1)); in ashr_reg64_lt32_high()
2259 emit_shf_indir(nfp_prog, reg_both(dst + 1), reg_none(), SHF_OP_ASHR, in ashr_reg64_lt32_high()
2263 static void ashr_reg64_lt32_low(struct nfp_prog *nfp_prog, u8 dst, u8 src) in ashr_reg64_lt32_low() argument
2268 return shr_reg64_lt32_low(nfp_prog, dst, src); in ashr_reg64_lt32_low()
2271 static void ashr_reg64_lt32(struct nfp_prog *nfp_prog, u8 dst, u8 src) in ashr_reg64_lt32() argument
2273 ashr_reg64_lt32_low(nfp_prog, dst, src); in ashr_reg64_lt32()
2274 ashr_reg64_lt32_high(nfp_prog, dst, src); in ashr_reg64_lt32()
2277 static void ashr_reg64_ge32(struct nfp_prog *nfp_prog, u8 dst, u8 src) in ashr_reg64_ge32() argument
2279 emit_alu(nfp_prog, reg_none(), reg_a(src), ALU_OP_OR, reg_b(dst + 1)); in ashr_reg64_ge32()
2280 emit_shf_indir(nfp_prog, reg_both(dst), reg_none(), SHF_OP_ASHR, in ashr_reg64_ge32()
2282 emit_shf(nfp_prog, reg_both(dst + 1), reg_none(), SHF_OP_ASHR, in ashr_reg64_ge32()
2287 static int ashr_reg64(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) in ashr_reg64() argument
2297 return __ashr_imm64(nfp_prog, dst, umin); in ashr_reg64()
2301 ashr_reg64_lt32(nfp_prog, dst, src); in ashr_reg64()
2303 ashr_reg64_ge32(nfp_prog, dst, src); in ashr_reg64()
2307 label_ge32 = nfp_prog_current_offset(nfp_prog) + 6; in ashr_reg64()
2308 emit_br_bset(nfp_prog, reg_a(src), 5, label_ge32, 0); in ashr_reg64()
2309 ashr_reg64_lt32_low(nfp_prog, dst, src); in ashr_reg64()
2310 label_end = nfp_prog_current_offset(nfp_prog) + 6; in ashr_reg64()
2311 emit_br(nfp_prog, BR_UNC, label_end, 2); in ashr_reg64()
2313 ashr_reg64_lt32_high(nfp_prog, dst, src); in ashr_reg64()
2315 if (!nfp_prog_confirm_current_offset(nfp_prog, label_ge32)) in ashr_reg64()
2317 ashr_reg64_ge32(nfp_prog, dst, src); in ashr_reg64()
2319 if (!nfp_prog_confirm_current_offset(nfp_prog, label_end)) in ashr_reg64()
2326 static int mov_reg(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) in mov_reg() argument
2330 wrp_reg_mov(nfp_prog, insn->dst_reg * 2, insn->src_reg * 2); in mov_reg()
2331 wrp_immed(nfp_prog, reg_both(insn->dst_reg * 2 + 1), 0); in mov_reg()
2336 static int mov_imm(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) in mov_imm() argument
2340 wrp_immed(nfp_prog, reg_both(insn->dst_reg * 2), insn->imm); in mov_imm()
2341 wrp_immed(nfp_prog, reg_both(insn->dst_reg * 2 + 1), 0); in mov_imm()
2346 static int xor_reg(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) in xor_reg() argument
2348 return wrp_alu32_reg(nfp_prog, meta, ALU_OP_XOR); in xor_reg()
2351 static int xor_imm(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) in xor_imm() argument
2353 return wrp_alu32_imm(nfp_prog, meta, ALU_OP_XOR); in xor_imm()
2356 static int and_reg(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) in and_reg() argument
2358 return wrp_alu32_reg(nfp_prog, meta, ALU_OP_AND); in and_reg()
2361 static int and_imm(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) in and_imm() argument
2363 return wrp_alu32_imm(nfp_prog, meta, ALU_OP_AND); in and_imm()
2366 static int or_reg(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) in or_reg() argument
2368 return wrp_alu32_reg(nfp_prog, meta, ALU_OP_OR); in or_reg()
2371 static int or_imm(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) in or_imm() argument
2373 return wrp_alu32_imm(nfp_prog, meta, ALU_OP_OR); in or_imm()
2376 static int add_reg(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) in add_reg() argument
2378 return wrp_alu32_reg(nfp_prog, meta, ALU_OP_ADD); in add_reg()
2381 static int add_imm(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) in add_imm() argument
2383 return wrp_alu32_imm(nfp_prog, meta, ALU_OP_ADD); in add_imm()
2386 static int sub_reg(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) in sub_reg() argument
2388 return wrp_alu32_reg(nfp_prog, meta, ALU_OP_SUB); in sub_reg()
2391 static int sub_imm(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) in sub_imm() argument
2393 return wrp_alu32_imm(nfp_prog, meta, ALU_OP_SUB); in sub_imm()
2396 static int mul_reg(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) in mul_reg() argument
2398 return wrp_mul(nfp_prog, meta, false, true); in mul_reg()
2401 static int mul_imm(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) in mul_imm() argument
2403 return wrp_mul(nfp_prog, meta, false, false); in mul_imm()
2406 static int div_reg(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) in div_reg() argument
2408 return div_reg64(nfp_prog, meta); in div_reg()
2411 static int div_imm(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) in div_imm() argument
2413 return div_imm64(nfp_prog, meta); in div_imm()
2416 static int neg_reg(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) in neg_reg() argument
2420 emit_alu(nfp_prog, reg_both(dst), reg_imm(0), ALU_OP_SUB, reg_b(dst)); in neg_reg()
2421 wrp_zext(nfp_prog, meta, dst); in neg_reg()
2427 __ashr_imm(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta, u8 dst, in __ashr_imm() argument
2432 emit_alu(nfp_prog, reg_none(), reg_a(dst), ALU_OP_OR, in __ashr_imm()
2434 emit_shf(nfp_prog, reg_both(dst), reg_none(), SHF_OP_ASHR, in __ashr_imm()
2437 wrp_zext(nfp_prog, meta, dst); in __ashr_imm()
2442 static int ashr_reg(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) in ashr_reg() argument
2452 return __ashr_imm(nfp_prog, meta, dst, umin); in ashr_reg()
2458 emit_alu(nfp_prog, reg_none(), reg_a(src), ALU_OP_OR, reg_b(dst)); in ashr_reg()
2459 emit_shf_indir(nfp_prog, reg_both(dst), reg_none(), SHF_OP_ASHR, in ashr_reg()
2461 wrp_zext(nfp_prog, meta, dst); in ashr_reg()
2466 static int ashr_imm(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) in ashr_imm() argument
2471 return __ashr_imm(nfp_prog, meta, dst, insn->imm); in ashr_imm()
2475 __shr_imm(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta, u8 dst, in __shr_imm() argument
2479 emit_shf(nfp_prog, reg_both(dst), reg_none(), SHF_OP_NONE, in __shr_imm()
2481 wrp_zext(nfp_prog, meta, dst); in __shr_imm()
2485 static int shr_imm(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) in shr_imm() argument
2490 return __shr_imm(nfp_prog, meta, dst, insn->imm); in shr_imm()
2493 static int shr_reg(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) in shr_reg() argument
2503 return __shr_imm(nfp_prog, meta, dst, umin); in shr_reg()
2506 emit_alu(nfp_prog, reg_none(), reg_a(src), ALU_OP_OR, reg_imm(0)); in shr_reg()
2507 emit_shf_indir(nfp_prog, reg_both(dst), reg_none(), SHF_OP_NONE, in shr_reg()
2509 wrp_zext(nfp_prog, meta, dst); in shr_reg()
2514 __shl_imm(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta, u8 dst, in __shl_imm() argument
2518 emit_shf(nfp_prog, reg_both(dst), reg_none(), SHF_OP_NONE, in __shl_imm()
2520 wrp_zext(nfp_prog, meta, dst); in __shl_imm()
2524 static int shl_imm(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) in shl_imm() argument
2529 return __shl_imm(nfp_prog, meta, dst, insn->imm); in shl_imm()
2532 static int shl_reg(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) in shl_reg() argument
2542 return __shl_imm(nfp_prog, meta, dst, umin); in shl_reg()
2545 shl_reg64_lt32_low(nfp_prog, dst, src); in shl_reg()
2546 wrp_zext(nfp_prog, meta, dst); in shl_reg()
2550 static int end_reg32(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) in end_reg32() argument
2557 emit_ld_field(nfp_prog, reg_both(gpr), 0x9, reg_b(gpr), in end_reg32()
2559 emit_ld_field(nfp_prog, reg_both(gpr), 0xe, reg_a(gpr), in end_reg32()
2562 wrp_immed(nfp_prog, reg_both(gpr + 1), 0); in end_reg32()
2565 wrp_end32(nfp_prog, reg_a(gpr), gpr); in end_reg32()
2566 wrp_immed(nfp_prog, reg_both(gpr + 1), 0); in end_reg32()
2569 wrp_mov(nfp_prog, imm_a(nfp_prog), reg_b(gpr + 1)); in end_reg32()
2571 wrp_end32(nfp_prog, reg_a(gpr), gpr + 1); in end_reg32()
2572 wrp_end32(nfp_prog, imm_a(nfp_prog), gpr); in end_reg32()
2579 static int imm_ld8_part2(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) in imm_ld8_part2() argument
2589 wrp_immed(nfp_prog, reg_both(dst), imm_lo); in imm_ld8_part2()
2593 wrp_mov(nfp_prog, reg_both(dst + 1), reg_a(dst)); in imm_ld8_part2()
2595 wrp_immed(nfp_prog, reg_both(dst + 1), imm_hi); in imm_ld8_part2()
2600 static int imm_ld8(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) in imm_ld8() argument
2606 static int data_ld1(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) in data_ld1() argument
2608 return construct_data_ld(nfp_prog, meta, meta->insn.imm, 1); in data_ld1()
2611 static int data_ld2(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) in data_ld2() argument
2613 return construct_data_ld(nfp_prog, meta, meta->insn.imm, 2); in data_ld2()
2616 static int data_ld4(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) in data_ld4() argument
2618 return construct_data_ld(nfp_prog, meta, meta->insn.imm, 4); in data_ld4()
2621 static int data_ind_ld1(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) in data_ind_ld1() argument
2623 return construct_data_ind_ld(nfp_prog, meta, meta->insn.imm, in data_ind_ld1()
2627 static int data_ind_ld2(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) in data_ind_ld2() argument
2629 return construct_data_ind_ld(nfp_prog, meta, meta->insn.imm, in data_ind_ld2()
2633 static int data_ind_ld4(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) in data_ind_ld4() argument
2635 return construct_data_ind_ld(nfp_prog, meta, meta->insn.imm, in data_ind_ld4()
2640 mem_ldx_stack(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta, in mem_ldx_stack() argument
2643 return mem_op_stack(nfp_prog, meta, size, ptr_off, in mem_ldx_stack()
2648 static int mem_ldx_skb(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta, in mem_ldx_skb() argument
2657 wrp_mov(nfp_prog, dst, plen_reg(nfp_prog)); in mem_ldx_skb()
2662 wrp_mov(nfp_prog, dst, pptr_reg(nfp_prog)); in mem_ldx_skb()
2667 emit_alu(nfp_prog, dst, in mem_ldx_skb()
2668 plen_reg(nfp_prog), ALU_OP_ADD, pptr_reg(nfp_prog)); in mem_ldx_skb()
2674 wrp_immed(nfp_prog, reg_both(meta->insn.dst_reg * 2 + 1), 0); in mem_ldx_skb()
2679 static int mem_ldx_xdp(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta, in mem_ldx_xdp() argument
2688 wrp_mov(nfp_prog, dst, pptr_reg(nfp_prog)); in mem_ldx_xdp()
2693 emit_alu(nfp_prog, dst, in mem_ldx_xdp()
2694 plen_reg(nfp_prog), ALU_OP_ADD, pptr_reg(nfp_prog)); in mem_ldx_xdp()
2700 wrp_immed(nfp_prog, reg_both(meta->insn.dst_reg * 2 + 1), 0); in mem_ldx_xdp()
2706 mem_ldx_data(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta, in mem_ldx_data() argument
2711 tmp_reg = re_load_imm_any(nfp_prog, meta->insn.off, imm_b(nfp_prog)); in mem_ldx_data()
2713 return data_ld_host_order_addr32(nfp_prog, meta, meta->insn.src_reg * 2, in mem_ldx_data()
2718 mem_ldx_emem(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta, in mem_ldx_emem() argument
2723 tmp_reg = re_load_imm_any(nfp_prog, meta->insn.off, imm_b(nfp_prog)); in mem_ldx_emem()
2725 return data_ld_host_order_addr40(nfp_prog, meta, meta->insn.src_reg * 2, in mem_ldx_emem()
2730 mem_ldx_data_init_pktcache(struct nfp_prog *nfp_prog, in mem_ldx_data_init_pktcache() argument
2739 off = re_load_imm_any(nfp_prog, range_start, imm_b(nfp_prog)); in mem_ldx_data_init_pktcache()
2747 wrp_immed(nfp_prog, reg_none(), in mem_ldx_data_init_pktcache()
2751 emit_cmd_any(nfp_prog, CMD_TGT_READ32_SWAP, CMD_MODE_32b, 0, src_base, in mem_ldx_data_init_pktcache()
2756 mem_ldx_data_from_pktcache_unaligned(struct nfp_prog *nfp_prog, in mem_ldx_data_from_pktcache_unaligned() argument
2783 wrp_reg_subpart(nfp_prog, dst_lo, src_lo, len_lo, off); in mem_ldx_data_from_pktcache_unaligned()
2786 wrp_zext(nfp_prog, meta, dst_gpr); in mem_ldx_data_from_pktcache_unaligned()
2793 wrp_reg_or_subpart(nfp_prog, dst_lo, src_mid, len_mid, len_lo); in mem_ldx_data_from_pktcache_unaligned()
2794 wrp_zext(nfp_prog, meta, dst_gpr); in mem_ldx_data_from_pktcache_unaligned()
2798 wrp_reg_or_subpart(nfp_prog, dst_lo, src_mid, in mem_ldx_data_from_pktcache_unaligned()
2800 wrp_reg_subpart(nfp_prog, dst_hi, src_mid, len_lo, in mem_ldx_data_from_pktcache_unaligned()
2802 wrp_reg_or_subpart(nfp_prog, dst_hi, src_hi, REG_WIDTH - len_lo, in mem_ldx_data_from_pktcache_unaligned()
2810 mem_ldx_data_from_pktcache_aligned(struct nfp_prog *nfp_prog, in mem_ldx_data_from_pktcache_aligned() argument
2824 wrp_reg_subpart(nfp_prog, dst_lo, src_lo, size, 0); in mem_ldx_data_from_pktcache_aligned()
2825 wrp_zext(nfp_prog, meta, dst_gpr); in mem_ldx_data_from_pktcache_aligned()
2827 wrp_mov(nfp_prog, dst_lo, src_lo); in mem_ldx_data_from_pktcache_aligned()
2828 wrp_zext(nfp_prog, meta, dst_gpr); in mem_ldx_data_from_pktcache_aligned()
2832 wrp_mov(nfp_prog, dst_lo, src_lo); in mem_ldx_data_from_pktcache_aligned()
2833 wrp_mov(nfp_prog, dst_hi, src_hi); in mem_ldx_data_from_pktcache_aligned()
2840 mem_ldx_data_from_pktcache(struct nfp_prog *nfp_prog, in mem_ldx_data_from_pktcache() argument
2846 return mem_ldx_data_from_pktcache_aligned(nfp_prog, meta, size); in mem_ldx_data_from_pktcache()
2848 return mem_ldx_data_from_pktcache_unaligned(nfp_prog, meta, size); in mem_ldx_data_from_pktcache()
2852 mem_ldx(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta, in mem_ldx() argument
2856 return nfp_cpp_memcpy(nfp_prog, meta); in mem_ldx()
2859 if (nfp_prog->type == BPF_PROG_TYPE_XDP) in mem_ldx()
2860 return mem_ldx_xdp(nfp_prog, meta, size); in mem_ldx()
2862 return mem_ldx_skb(nfp_prog, meta, size); in mem_ldx()
2868 mem_ldx_data_init_pktcache(nfp_prog, meta); in mem_ldx()
2870 return mem_ldx_data_from_pktcache(nfp_prog, meta, size); in mem_ldx()
2872 return mem_ldx_data(nfp_prog, meta, size); in mem_ldx()
2877 return mem_ldx_stack(nfp_prog, meta, size, in mem_ldx()
2881 return mem_ldx_emem(nfp_prog, meta, size); in mem_ldx()
2886 static int mem_ldx1(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) in mem_ldx1() argument
2888 return mem_ldx(nfp_prog, meta, 1); in mem_ldx1()
2891 static int mem_ldx2(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) in mem_ldx2() argument
2893 return mem_ldx(nfp_prog, meta, 2); in mem_ldx2()
2896 static int mem_ldx4(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) in mem_ldx4() argument
2898 return mem_ldx(nfp_prog, meta, 4); in mem_ldx4()
2901 static int mem_ldx8(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) in mem_ldx8() argument
2903 return mem_ldx(nfp_prog, meta, 8); in mem_ldx8()
2907 mem_st_data(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta, in mem_st_data() argument
2913 off_reg = re_load_imm_any(nfp_prog, meta->insn.off, imm_b(nfp_prog)); in mem_st_data()
2915 return data_st_host_order(nfp_prog, meta->insn.dst_reg * 2, off_reg, in mem_st_data()
2919 static int mem_st(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta, in mem_st() argument
2923 return mem_st_data(nfp_prog, meta, size); in mem_st()
2928 static int mem_st1(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) in mem_st1() argument
2930 return mem_st(nfp_prog, meta, 1); in mem_st1()
2933 static int mem_st2(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) in mem_st2() argument
2935 return mem_st(nfp_prog, meta, 2); in mem_st2()
2938 static int mem_st4(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) in mem_st4() argument
2940 return mem_st(nfp_prog, meta, 4); in mem_st4()
2943 static int mem_st8(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) in mem_st8() argument
2945 return mem_st(nfp_prog, meta, 8); in mem_st8()
2949 mem_stx_data(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta, in mem_stx_data() argument
2954 off_reg = re_load_imm_any(nfp_prog, meta->insn.off, imm_b(nfp_prog)); in mem_stx_data()
2956 return data_stx_host_order(nfp_prog, meta->insn.dst_reg * 2, off_reg, in mem_stx_data()
2961 mem_stx_stack(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta, in mem_stx_stack() argument
2964 return mem_op_stack(nfp_prog, meta, size, ptr_off, in mem_stx_stack()
2969 static int mem_stx_xdp(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) in mem_stx_xdp() argument
2973 return nfp_queue_select(nfp_prog, meta); in mem_stx_xdp()
2981 mem_stx(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta, in mem_stx() argument
2985 return mem_stx_data(nfp_prog, meta, size); in mem_stx()
2988 return mem_stx_stack(nfp_prog, meta, size, in mem_stx()
2994 static int mem_stx1(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) in mem_stx1() argument
2996 return mem_stx(nfp_prog, meta, 1); in mem_stx1()
2999 static int mem_stx2(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) in mem_stx2() argument
3001 return mem_stx(nfp_prog, meta, 2); in mem_stx2()
3004 static int mem_stx4(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) in mem_stx4() argument
3007 if (nfp_prog->type == BPF_PROG_TYPE_XDP) in mem_stx4()
3008 return mem_stx_xdp(nfp_prog, meta); in mem_stx4()
3009 return mem_stx(nfp_prog, meta, 4); in mem_stx4()
3012 static int mem_stx8(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) in mem_stx8() argument
3014 return mem_stx(nfp_prog, meta, 8); in mem_stx8()
3018 mem_xadd(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta, bool is64) in mem_xadd() argument
3025 off = ur_load_imm_any(nfp_prog, meta->insn.off, imm_b(nfp_prog)); in mem_xadd()
3032 out = nfp_prog_current_offset(nfp_prog); in mem_xadd()
3033 full_add = nfp_prog_current_offset(nfp_prog); in mem_xadd()
3052 swreg max_imm = imm_a(nfp_prog); in mem_xadd()
3054 wrp_immed(nfp_prog, max_imm, 0xffff); in mem_xadd()
3055 emit_alu(nfp_prog, reg_none(), in mem_xadd()
3057 emit_alu(nfp_prog, reg_none(), in mem_xadd()
3059 emit_br(nfp_prog, BR_BLO, full_add, meta->insn.off ? 2 : 0); in mem_xadd()
3068 emit_alu(nfp_prog, imma_a(nfp_prog), in mem_xadd()
3070 emit_alu(nfp_prog, imma_b(nfp_prog), in mem_xadd()
3072 addra = imma_a(nfp_prog); in mem_xadd()
3073 addrb = imma_b(nfp_prog); in mem_xadd()
3078 swreg prev_alu = imm_a(nfp_prog); in mem_xadd()
3080 wrp_immed(nfp_prog, prev_alu, in mem_xadd()
3084 wrp_reg_or_subpart(nfp_prog, prev_alu, reg_b(src_gpr), 2, 2); in mem_xadd()
3085 emit_cmd_indir(nfp_prog, CMD_TGT_ADD_IMM, CMD_MODE_40b_BA, 0, in mem_xadd()
3089 emit_br(nfp_prog, BR_UNC, out, 0); in mem_xadd()
3092 if (!nfp_prog_confirm_current_offset(nfp_prog, full_add)) in mem_xadd()
3097 emit_cmd(nfp_prog, CMD_TGT_ADD, CMD_MODE_40b_BA, 0, in mem_xadd()
3101 wrp_mov(nfp_prog, reg_xfer(0), reg_a(src_gpr)); in mem_xadd()
3103 wrp_mov(nfp_prog, reg_xfer(1), reg_a(src_gpr + 1)); in mem_xadd()
3106 if (!nfp_prog_confirm_current_offset(nfp_prog, out)) in mem_xadd()
3112 static int mem_atomic4(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) in mem_atomic4() argument
3117 return mem_xadd(nfp_prog, meta, false); in mem_atomic4()
3120 static int mem_atomic8(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) in mem_atomic8() argument
3125 return mem_xadd(nfp_prog, meta, true); in mem_atomic8()
3128 static int jump(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) in jump() argument
3130 emit_br(nfp_prog, BR_UNC, meta->insn.off, 0); in jump()
3135 static int jeq_imm(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) in jeq_imm() argument
3145 tmp_reg = ur_load_imm_any(nfp_prog, imm & ~0U, imm_b(nfp_prog)); in jeq_imm()
3146 emit_alu(nfp_prog, imm_a(nfp_prog), in jeq_imm()
3148 or1 = imm_a(nfp_prog); in jeq_imm()
3152 tmp_reg = ur_load_imm_any(nfp_prog, imm >> 32, imm_b(nfp_prog)); in jeq_imm()
3153 emit_alu(nfp_prog, imm_b(nfp_prog), in jeq_imm()
3155 or2 = imm_b(nfp_prog); in jeq_imm()
3158 emit_alu(nfp_prog, reg_none(), or1, ALU_OP_OR, or2); in jeq_imm()
3159 emit_br(nfp_prog, BR_BEQ, insn->off, 0); in jeq_imm()
3164 static int jeq32_imm(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) in jeq32_imm() argument
3169 tmp_reg = ur_load_imm_any(nfp_prog, insn->imm, imm_b(nfp_prog)); in jeq32_imm()
3170 emit_alu(nfp_prog, reg_none(), in jeq32_imm()
3172 emit_br(nfp_prog, BR_BEQ, insn->off, 0); in jeq32_imm()
3177 static int jset_imm(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) in jset_imm() argument
3184 tmp_reg = ur_load_imm_any(nfp_prog, imm & ~0U, imm_b(nfp_prog)); in jset_imm()
3185 emit_alu(nfp_prog, imm_b(nfp_prog), in jset_imm()
3191 emit_alu(nfp_prog, reg_none(), in jset_imm()
3192 reg_a(dst_gpr + 1), ALU_OP_OR, imm_b(nfp_prog)); in jset_imm()
3194 emit_br(nfp_prog, BR_BNE, insn->off, 0); in jset_imm()
3199 static int jne_imm(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) in jne_imm() argument
3208 emit_alu(nfp_prog, reg_none(), reg_none(), ALU_OP_NONE, in jne_imm()
3211 emit_alu(nfp_prog, reg_none(), reg_a(insn->dst_reg * 2), in jne_imm()
3213 emit_br(nfp_prog, BR_BNE, insn->off, 0); in jne_imm()
3217 tmp_reg = ur_load_imm_any(nfp_prog, imm & ~0U, imm_b(nfp_prog)); in jne_imm()
3218 emit_alu(nfp_prog, reg_none(), in jne_imm()
3220 emit_br(nfp_prog, BR_BNE, insn->off, 0); in jne_imm()
3225 tmp_reg = ur_load_imm_any(nfp_prog, imm >> 32, imm_b(nfp_prog)); in jne_imm()
3226 emit_alu(nfp_prog, reg_none(), in jne_imm()
3228 emit_br(nfp_prog, BR_BNE, insn->off, 0); in jne_imm()
3233 static int jeq_reg(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) in jeq_reg() argument
3237 emit_alu(nfp_prog, imm_a(nfp_prog), reg_a(insn->dst_reg * 2), in jeq_reg()
3240 emit_alu(nfp_prog, imm_b(nfp_prog), in jeq_reg()
3243 emit_alu(nfp_prog, reg_none(), imm_a(nfp_prog), ALU_OP_OR, in jeq_reg()
3244 imm_b(nfp_prog)); in jeq_reg()
3246 emit_br(nfp_prog, BR_BEQ, insn->off, 0); in jeq_reg()
3251 static int jset_reg(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) in jset_reg() argument
3253 return wrp_test_reg(nfp_prog, meta, ALU_OP_AND, BR_BNE); in jset_reg()
3256 static int jne_reg(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) in jne_reg() argument
3258 return wrp_test_reg(nfp_prog, meta, ALU_OP_XOR, BR_BNE); in jne_reg()
3262 bpf_to_bpf_call(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) in bpf_to_bpf_call() argument
3267 stack_depth = round_up(nfp_prog->stack_frame_depth, STACK_FRAME_ALIGN); in bpf_to_bpf_call()
3272 tmp_reg = ur_load_imm_any(nfp_prog, stack_depth, in bpf_to_bpf_call()
3273 stack_imm(nfp_prog)); in bpf_to_bpf_call()
3274 emit_alu(nfp_prog, stack_reg(nfp_prog), in bpf_to_bpf_call()
3275 stack_reg(nfp_prog), ALU_OP_ADD, tmp_reg); in bpf_to_bpf_call()
3276 emit_csr_wr(nfp_prog, stack_reg(nfp_prog), in bpf_to_bpf_call()
3315 if (nfp_prog->subprog[meta->jmp_dst->subprog_idx].needs_reg_push) { in bpf_to_bpf_call()
3316 ret_tgt = nfp_prog_current_offset(nfp_prog) + 3; in bpf_to_bpf_call()
3317 emit_br_relo(nfp_prog, BR_UNC, BR_OFF_RELO, 2, in bpf_to_bpf_call()
3319 offset_br = nfp_prog_current_offset(nfp_prog); in bpf_to_bpf_call()
3320 wrp_immed_relo(nfp_prog, imm_b(nfp_prog), 0, RELO_IMMED_REL); in bpf_to_bpf_call()
3322 ret_tgt = nfp_prog_current_offset(nfp_prog) + 2; in bpf_to_bpf_call()
3323 emit_br(nfp_prog, BR_UNC, meta->insn.imm, 1); in bpf_to_bpf_call()
3324 offset_br = nfp_prog_current_offset(nfp_prog); in bpf_to_bpf_call()
3326 wrp_immed_relo(nfp_prog, ret_reg(nfp_prog), ret_tgt, RELO_IMMED_REL); in bpf_to_bpf_call()
3328 if (!nfp_prog_confirm_current_offset(nfp_prog, ret_tgt)) in bpf_to_bpf_call()
3332 tmp_reg = ur_load_imm_any(nfp_prog, stack_depth, in bpf_to_bpf_call()
3333 stack_imm(nfp_prog)); in bpf_to_bpf_call()
3334 emit_alu(nfp_prog, stack_reg(nfp_prog), in bpf_to_bpf_call()
3335 stack_reg(nfp_prog), ALU_OP_SUB, tmp_reg); in bpf_to_bpf_call()
3336 emit_csr_wr(nfp_prog, stack_reg(nfp_prog), in bpf_to_bpf_call()
3338 wrp_nops(nfp_prog, 3); in bpf_to_bpf_call()
3341 meta->num_insns_after_br = nfp_prog_current_offset(nfp_prog); in bpf_to_bpf_call()
3347 static int helper_call(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) in helper_call() argument
3351 return adjust_head(nfp_prog, meta); in helper_call()
3353 return adjust_tail(nfp_prog, meta); in helper_call()
3357 return map_call_stack_common(nfp_prog, meta); in helper_call()
3359 return nfp_get_prandom_u32(nfp_prog, meta); in helper_call()
3361 return nfp_perf_event_output(nfp_prog, meta); in helper_call()
3368 static int call(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) in call() argument
3371 return bpf_to_bpf_call(nfp_prog, meta); in call()
3373 return helper_call(nfp_prog, meta); in call()
3381 static int goto_out(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) in goto_out() argument
3383 emit_br_relo(nfp_prog, BR_UNC, BR_OFF_RELO, 0, RELO_BR_GO_OUT); in goto_out()
3389 nfp_subprog_epilogue(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) in nfp_subprog_epilogue() argument
3391 if (nfp_prog->subprog[meta->subprog_idx].needs_reg_push) { in nfp_subprog_epilogue()
3397 emit_br_relo(nfp_prog, BR_UNC, BR_OFF_RELO, 1, in nfp_subprog_epilogue()
3400 wrp_mov(nfp_prog, ret_reg(nfp_prog), reg_lm(0, 0)); in nfp_subprog_epilogue()
3403 wrp_mov(nfp_prog, ret_reg(nfp_prog), reg_lm(0, 0)); in nfp_subprog_epilogue()
3407 emit_rtn(nfp_prog, ret_reg(nfp_prog), 0); in nfp_subprog_epilogue()
3413 static int jmp_exit(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) in jmp_exit() argument
3416 return goto_out(nfp_prog, meta); in jmp_exit()
3418 return nfp_subprog_epilogue(nfp_prog, meta); in jmp_exit()
3541 nfp_fixup_immed_relo(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta, in nfp_fixup_immed_relo() argument
3544 if (immed_get_value(nfp_prog->prog[br_idx + 1])) { in nfp_fixup_immed_relo()
3549 immed_set_value(&nfp_prog->prog[br_idx + 1], jmp_dst->off); in nfp_fixup_immed_relo()
3554 static int nfp_fixup_branches(struct nfp_prog *nfp_prog) in nfp_fixup_branches() argument
3560 list_for_each_entry(meta, &nfp_prog->insns, l) { in nfp_fixup_branches()
3571 if (list_is_last(&meta->l, &nfp_prog->insns)) in nfp_fixup_branches()
3572 br_idx = nfp_prog->last_bpf_off; in nfp_fixup_branches()
3584 if (!nfp_is_br(nfp_prog->prog[br_idx])) { in nfp_fixup_branches()
3586 br_idx, meta->insn.code, nfp_prog->prog[br_idx]); in nfp_fixup_branches()
3594 if (FIELD_GET(OP_RELO_TYPE, nfp_prog->prog[br_idx]) != in nfp_fixup_branches()
3611 nfp_prog->subprog[jmp_dst->subprog_idx].needs_reg_push) { in nfp_fixup_branches()
3612 err = nfp_fixup_immed_relo(nfp_prog, meta, in nfp_fixup_branches()
3618 if (FIELD_GET(OP_RELO_TYPE, nfp_prog->prog[br_idx]) != in nfp_fixup_branches()
3623 if (!nfp_is_br(nfp_prog->prog[idx])) in nfp_fixup_branches()
3625 br_set_offset(&nfp_prog->prog[idx], jmp_dst->off); in nfp_fixup_branches()
3632 static void nfp_intro(struct nfp_prog *nfp_prog) in nfp_intro() argument
3634 wrp_immed(nfp_prog, plen_reg(nfp_prog), GENMASK(13, 0)); in nfp_intro()
3635 emit_alu(nfp_prog, plen_reg(nfp_prog), in nfp_intro()
3636 plen_reg(nfp_prog), ALU_OP_AND, pv_len(nfp_prog)); in nfp_intro()
3640 nfp_subprog_prologue(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) in nfp_subprog_prologue() argument
3643 wrp_mov(nfp_prog, reg_lm(0, 0), ret_reg(nfp_prog)); in nfp_subprog_prologue()
3647 nfp_start_subprog(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) in nfp_start_subprog() argument
3649 unsigned int depth = nfp_prog->subprog[meta->subprog_idx].stack_depth; in nfp_start_subprog()
3651 nfp_prog->stack_frame_depth = round_up(depth, 4); in nfp_start_subprog()
3652 nfp_subprog_prologue(nfp_prog, meta); in nfp_start_subprog()
3660 static void nfp_outro_tc_da(struct nfp_prog *nfp_prog) in nfp_outro_tc_da() argument
3675 nfp_prog->tgt_abort = nfp_prog_current_offset(nfp_prog); in nfp_outro_tc_da()
3677 emit_br_relo(nfp_prog, BR_UNC, BR_OFF_RELO, 2, RELO_BR_NEXT_PKT); in nfp_outro_tc_da()
3679 wrp_mov(nfp_prog, reg_a(0), NFP_BPF_ABI_FLAGS); in nfp_outro_tc_da()
3680 emit_ld_field(nfp_prog, reg_a(0), 0xc, reg_imm(0x11), SHF_SC_L_SHF, 16); in nfp_outro_tc_da()
3683 nfp_prog->tgt_out = nfp_prog_current_offset(nfp_prog); in nfp_outro_tc_da()
3686 emit_alu(nfp_prog, reg_none(), reg_imm(7), ALU_OP_SUB, reg_b(0)); in nfp_outro_tc_da()
3687 emit_br(nfp_prog, BR_BLO, nfp_prog->tgt_abort, 0); in nfp_outro_tc_da()
3688 wrp_mov(nfp_prog, reg_a(0), NFP_BPF_ABI_FLAGS); in nfp_outro_tc_da()
3690 wrp_immed(nfp_prog, reg_b(2), 0x41221211); in nfp_outro_tc_da()
3691 wrp_immed(nfp_prog, reg_b(3), 0x41001211); in nfp_outro_tc_da()
3693 emit_shf(nfp_prog, reg_a(1), in nfp_outro_tc_da()
3696 emit_alu(nfp_prog, reg_none(), reg_a(1), ALU_OP_OR, reg_imm(0)); in nfp_outro_tc_da()
3697 emit_shf(nfp_prog, reg_a(2), in nfp_outro_tc_da()
3700 emit_alu(nfp_prog, reg_none(), reg_a(1), ALU_OP_OR, reg_imm(0)); in nfp_outro_tc_da()
3701 emit_shf(nfp_prog, reg_b(2), in nfp_outro_tc_da()
3704 emit_br_relo(nfp_prog, BR_UNC, BR_OFF_RELO, 2, RELO_BR_NEXT_PKT); in nfp_outro_tc_da()
3706 emit_shf(nfp_prog, reg_b(2), in nfp_outro_tc_da()
3708 emit_ld_field(nfp_prog, reg_a(0), 0xc, reg_b(2), SHF_SC_L_SHF, 16); in nfp_outro_tc_da()
3711 static void nfp_outro_xdp(struct nfp_prog *nfp_prog) in nfp_outro_xdp() argument
3721 nfp_prog->tgt_abort = nfp_prog_current_offset(nfp_prog); in nfp_outro_xdp()
3723 emit_br_relo(nfp_prog, BR_UNC, BR_OFF_RELO, 2, RELO_BR_NEXT_PKT); in nfp_outro_xdp()
3725 wrp_mov(nfp_prog, reg_a(0), NFP_BPF_ABI_FLAGS); in nfp_outro_xdp()
3726 emit_ld_field(nfp_prog, reg_a(0), 0xc, reg_imm(0x82), SHF_SC_L_SHF, 16); in nfp_outro_xdp()
3729 nfp_prog->tgt_out = nfp_prog_current_offset(nfp_prog); in nfp_outro_xdp()
3732 emit_alu(nfp_prog, reg_none(), reg_imm(3), ALU_OP_SUB, reg_b(0)); in nfp_outro_xdp()
3733 emit_br(nfp_prog, BR_BLO, nfp_prog->tgt_abort, 0); in nfp_outro_xdp()
3735 wrp_immed(nfp_prog, reg_b(2), 0x44112282); in nfp_outro_xdp()
3737 emit_shf(nfp_prog, reg_a(1), in nfp_outro_xdp()
3740 emit_alu(nfp_prog, reg_none(), reg_a(1), ALU_OP_OR, reg_imm(0)); in nfp_outro_xdp()
3741 emit_shf(nfp_prog, reg_b(2), in nfp_outro_xdp()
3744 emit_br_relo(nfp_prog, BR_UNC, BR_OFF_RELO, 2, RELO_BR_NEXT_PKT); in nfp_outro_xdp()
3746 wrp_mov(nfp_prog, reg_a(0), NFP_BPF_ABI_FLAGS); in nfp_outro_xdp()
3747 emit_ld_field(nfp_prog, reg_a(0), 0xc, reg_b(2), SHF_SC_L_SHF, 16); in nfp_outro_xdp()
3750 static bool nfp_prog_needs_callee_reg_save(struct nfp_prog *nfp_prog) in nfp_prog_needs_callee_reg_save() argument
3754 for (idx = 1; idx < nfp_prog->subprog_cnt; idx++) in nfp_prog_needs_callee_reg_save()
3755 if (nfp_prog->subprog[idx].needs_reg_push) in nfp_prog_needs_callee_reg_save()
3761 static void nfp_push_callee_registers(struct nfp_prog *nfp_prog) in nfp_push_callee_registers() argument
3768 nfp_prog->tgt_call_push_regs = nfp_prog_current_offset(nfp_prog); in nfp_push_callee_registers()
3776 wrp_mov(nfp_prog, reg_lm(0, 1 + idx), reg_b(adj)); in nfp_push_callee_registers()
3780 emit_rtn(nfp_prog, imm_b(nfp_prog), 3); in nfp_push_callee_registers()
3782 wrp_mov(nfp_prog, reg_lm(0, 1 + idx + 1), reg_b(adj + 1)); in nfp_push_callee_registers()
3786 static void nfp_pop_callee_registers(struct nfp_prog *nfp_prog) in nfp_pop_callee_registers() argument
3793 nfp_prog->tgt_call_pop_regs = nfp_prog_current_offset(nfp_prog); in nfp_pop_callee_registers()
3801 wrp_mov(nfp_prog, reg_both(adj), reg_lm(0, 1 + idx)); in nfp_pop_callee_registers()
3805 emit_rtn(nfp_prog, ret_reg(nfp_prog), 3); in nfp_pop_callee_registers()
3807 wrp_mov(nfp_prog, reg_both(adj + 1), reg_lm(0, 1 + idx + 1)); in nfp_pop_callee_registers()
3811 static void nfp_outro(struct nfp_prog *nfp_prog) in nfp_outro() argument
3813 switch (nfp_prog->type) { in nfp_outro()
3815 nfp_outro_tc_da(nfp_prog); in nfp_outro()
3818 nfp_outro_xdp(nfp_prog); in nfp_outro()
3824 if (!nfp_prog_needs_callee_reg_save(nfp_prog)) in nfp_outro()
3827 nfp_push_callee_registers(nfp_prog); in nfp_outro()
3828 nfp_pop_callee_registers(nfp_prog); in nfp_outro()
3831 static int nfp_translate(struct nfp_prog *nfp_prog) in nfp_translate() argument
3837 depth = nfp_prog->subprog[0].stack_depth; in nfp_translate()
3838 nfp_prog->stack_frame_depth = round_up(depth, 4); in nfp_translate()
3840 nfp_intro(nfp_prog); in nfp_translate()
3841 if (nfp_prog->error) in nfp_translate()
3842 return nfp_prog->error; in nfp_translate()
3844 list_for_each_entry(meta, &nfp_prog->insns, l) { in nfp_translate()
3847 meta->off = nfp_prog_current_offset(nfp_prog); in nfp_translate()
3850 nfp_start_subprog(nfp_prog, meta); in nfp_translate()
3851 if (nfp_prog->error) in nfp_translate()
3852 return nfp_prog->error; in nfp_translate()
3856 nfp_prog->n_translated++; in nfp_translate()
3860 if (nfp_meta_has_prev(nfp_prog, meta) && in nfp_translate()
3865 err = cb(nfp_prog, meta); in nfp_translate()
3868 if (nfp_prog->error) in nfp_translate()
3869 return nfp_prog->error; in nfp_translate()
3871 nfp_prog->n_translated++; in nfp_translate()
3874 nfp_prog->last_bpf_off = nfp_prog_current_offset(nfp_prog) - 1; in nfp_translate()
3876 nfp_outro(nfp_prog); in nfp_translate()
3877 if (nfp_prog->error) in nfp_translate()
3878 return nfp_prog->error; in nfp_translate()
3880 wrp_nops(nfp_prog, NFP_USTORE_PREFETCH_WINDOW); in nfp_translate()
3881 if (nfp_prog->error) in nfp_translate()
3882 return nfp_prog->error; in nfp_translate()
3884 return nfp_fixup_branches(nfp_prog); in nfp_translate()
3888 static void nfp_bpf_opt_reg_init(struct nfp_prog *nfp_prog) in nfp_bpf_opt_reg_init() argument
3892 list_for_each_entry(meta, &nfp_prog->insns, l) { in nfp_bpf_opt_reg_init()
3914 static void nfp_bpf_opt_neg_add_sub(struct nfp_prog *nfp_prog) in nfp_bpf_opt_neg_add_sub() argument
3918 list_for_each_entry(meta, &nfp_prog->insns, l) { in nfp_bpf_opt_neg_add_sub()
3958 static void nfp_bpf_opt_ld_mask(struct nfp_prog *nfp_prog) in nfp_bpf_opt_ld_mask() argument
3967 nfp_for_each_insn_walk2(nfp_prog, meta1, meta2) { in nfp_bpf_opt_ld_mask()
3997 static void nfp_bpf_opt_ld_shift(struct nfp_prog *nfp_prog) in nfp_bpf_opt_ld_shift() argument
4001 nfp_for_each_insn_walk3(nfp_prog, meta1, meta2, meta3) { in nfp_bpf_opt_ld_shift()
4186 static void nfp_bpf_opt_ldst_gather(struct nfp_prog *nfp_prog) in nfp_bpf_opt_ldst_gather() argument
4195 nfp_for_each_insn_walk2(nfp_prog, meta1, meta2) { in nfp_bpf_opt_ldst_gather()
4276 static void nfp_bpf_opt_pkt_cache(struct nfp_prog *nfp_prog) in nfp_bpf_opt_pkt_cache() argument
4285 list_for_each_entry(meta, &nfp_prog->insns, l) { in nfp_bpf_opt_pkt_cache()
4374 list_for_each_entry(meta, &nfp_prog->insns, l) { in nfp_bpf_opt_pkt_cache()
4390 static int nfp_bpf_optimize(struct nfp_prog *nfp_prog) in nfp_bpf_optimize() argument
4392 nfp_bpf_opt_reg_init(nfp_prog); in nfp_bpf_optimize()
4394 nfp_bpf_opt_neg_add_sub(nfp_prog); in nfp_bpf_optimize()
4395 nfp_bpf_opt_ld_mask(nfp_prog); in nfp_bpf_optimize()
4396 nfp_bpf_opt_ld_shift(nfp_prog); in nfp_bpf_optimize()
4397 nfp_bpf_opt_ldst_gather(nfp_prog); in nfp_bpf_optimize()
4398 nfp_bpf_opt_pkt_cache(nfp_prog); in nfp_bpf_optimize()
4403 static int nfp_bpf_replace_map_ptrs(struct nfp_prog *nfp_prog) in nfp_bpf_replace_map_ptrs() argument
4410 nfp_for_each_insn_walk2(nfp_prog, meta1, meta2) { in nfp_bpf_replace_map_ptrs()
4453 static void nfp_bpf_prog_trim(struct nfp_prog *nfp_prog) in nfp_bpf_prog_trim() argument
4457 prog = kvmalloc_array(nfp_prog->prog_len, sizeof(u64), GFP_KERNEL); in nfp_bpf_prog_trim()
4461 nfp_prog->__prog_alloc_len = nfp_prog->prog_len * sizeof(u64); in nfp_bpf_prog_trim()
4462 memcpy(prog, nfp_prog->prog, nfp_prog->__prog_alloc_len); in nfp_bpf_prog_trim()
4463 kvfree(nfp_prog->prog); in nfp_bpf_prog_trim()
4464 nfp_prog->prog = prog; in nfp_bpf_prog_trim()
4467 int nfp_bpf_jit(struct nfp_prog *nfp_prog) in nfp_bpf_jit() argument
4471 ret = nfp_bpf_replace_map_ptrs(nfp_prog); in nfp_bpf_jit()
4475 ret = nfp_bpf_optimize(nfp_prog); in nfp_bpf_jit()
4479 ret = nfp_translate(nfp_prog); in nfp_bpf_jit()
4482 ret, nfp_prog->n_translated); in nfp_bpf_jit()
4486 nfp_bpf_prog_trim(nfp_prog); in nfp_bpf_jit()
4491 void nfp_bpf_jit_prepare(struct nfp_prog *nfp_prog) in nfp_bpf_jit_prepare() argument
4496 list_for_each_entry(meta, &nfp_prog->insns, l) { in nfp_bpf_jit_prepare()
4519 dst_meta = nfp_bpf_goto_meta(nfp_prog, meta, dst_idx); in nfp_bpf_jit_prepare()
4534 void *nfp_bpf_relo_for_vnic(struct nfp_prog *nfp_prog, struct nfp_bpf_vnic *bv) in nfp_bpf_relo_for_vnic() argument
4540 prog = kmemdup_array(nfp_prog->prog, nfp_prog->prog_len, sizeof(u64), in nfp_bpf_relo_for_vnic()
4545 for (i = 0; i < nfp_prog->prog_len; i++) { in nfp_bpf_relo_for_vnic()
4559 nfp_prog->tgt_out + bv->start_off); in nfp_bpf_relo_for_vnic()
4563 nfp_prog->tgt_abort + bv->start_off); in nfp_bpf_relo_for_vnic()
4566 if (!nfp_prog->tgt_call_push_regs) { in nfp_bpf_relo_for_vnic()
4571 off = nfp_prog->tgt_call_push_regs + bv->start_off; in nfp_bpf_relo_for_vnic()
4575 if (!nfp_prog->tgt_call_pop_regs) { in nfp_bpf_relo_for_vnic()
4580 off = nfp_prog->tgt_call_pop_regs + bv->start_off; in nfp_bpf_relo_for_vnic()
4591 val = nfp_prog->bpf->helpers.map_lookup; in nfp_bpf_relo_for_vnic()
4594 val = nfp_prog->bpf->helpers.map_update; in nfp_bpf_relo_for_vnic()
4597 val = nfp_prog->bpf->helpers.map_delete; in nfp_bpf_relo_for_vnic()
4600 val = nfp_prog->bpf->helpers.perf_event_output; in nfp_bpf_relo_for_vnic()
4618 err = nfp_bpf_ustore_calc(prog, nfp_prog->prog_len); in nfp_bpf_relo_for_vnic()