Lines Matching full:wr
9 #include "wr.h"
54 static void set_eth_seg(const struct ib_send_wr *wr, struct mlx5_ib_qp *qp, in set_eth_seg() argument
61 if (wr->send_flags & IB_SEND_IP_CSUM) in set_eth_seg()
65 if (wr->opcode == IB_WR_LSO) { in set_eth_seg()
66 struct ib_ud_wr *ud_wr = container_of(wr, struct ib_ud_wr, wr); in set_eth_seg()
103 const struct ib_send_wr *wr) in set_datagram_seg() argument
105 memcpy(&dseg->av, &to_mah(ud_wr(wr)->ah)->av, sizeof(struct mlx5_av)); in set_datagram_seg()
107 cpu_to_be32(ud_wr(wr)->remote_qpn | MLX5_EXTENDED_UD_AV); in set_datagram_seg()
108 dseg->av.key.qkey.qkey = cpu_to_be32(ud_wr(wr)->remote_qkey); in set_datagram_seg()
228 static __be32 send_ieth(const struct ib_send_wr *wr) in send_ieth() argument
230 switch (wr->opcode) { in send_ieth()
233 return wr->ex.imm_data; in send_ieth()
236 return cpu_to_be32(wr->ex.invalidate_rkey); in send_ieth()
260 static int set_data_inl_seg(struct mlx5_ib_qp *qp, const struct ib_send_wr *wr, in set_data_inl_seg() argument
272 for (i = 0; i < wr->num_sge; i++) { in set_data_inl_seg()
273 size_t len = wr->sg_list[i].length; in set_data_inl_seg()
274 void *addr = (void *)(unsigned long)(wr->sg_list[i].addr); in set_data_inl_seg()
561 const struct ib_reg_wr *wr = reg_wr(send_wr); in set_pi_umr_wr() local
562 struct mlx5_ib_mr *sig_mr = to_mmr(wr->mr); in set_pi_umr_wr()
570 unlikely(wr->access & IB_ACCESS_REMOTE_ATOMIC) || in set_pi_umr_wr()
593 set_sig_mkey_segment(*seg, wr->mr, wr->access, xlt_size, region_len, in set_pi_umr_wr()
599 ret = set_sig_data_segment(send_wr, wr->mr, sig_attrs, qp, seg, size, in set_pi_umr_wr()
636 const struct ib_reg_wr *wr, in set_reg_wr() argument
640 struct mlx5_ib_mr *mr = to_mmr(wr->mr); in set_reg_wr()
645 bool atomic = wr->access & IB_ACCESS_REMOTE_ATOMIC; in set_reg_wr()
652 if (!mlx5r_umr_can_reconfig(dev, 0, wr->access)) { in set_reg_wr()
659 if (unlikely(wr->wr.send_flags & IB_SEND_INLINE)) { in set_reg_wr()
675 set_reg_mkey_seg(*seg, mr, wr->key, wr->access); in set_reg_wr()
750 const struct ib_send_wr *wr, unsigned int *idx, int *size, in begin_wqe() argument
754 send_ieth(wr), wr->send_flags & IB_SEND_SIGNALED, in begin_wqe()
755 wr->send_flags & IB_SEND_SOLICITED); in begin_wqe()
787 static void handle_rdma_op(const struct ib_send_wr *wr, void **seg, int *size) in handle_rdma_op() argument
789 set_raddr_seg(*seg, rdma_wr(wr)->remote_addr, rdma_wr(wr)->rkey); in handle_rdma_op()
794 static void handle_local_inv(struct mlx5_ib_qp *qp, const struct ib_send_wr *wr, in handle_local_inv() argument
799 (*ctrl)->imm = cpu_to_be32(wr->ex.invalidate_rkey); in handle_local_inv()
803 static int handle_reg_mr(struct mlx5_ib_qp *qp, const struct ib_send_wr *wr, in handle_reg_mr() argument
808 (*ctrl)->imm = cpu_to_be32(reg_wr(wr)->key); in handle_reg_mr()
809 return set_reg_wr(qp, reg_wr(wr), seg, size, cur_edge, true); in handle_reg_mr()
813 const struct ib_send_wr *wr, in handle_psv() argument
825 send_ieth(wr), false, true); in handle_psv()
836 mlx5r_finish_wqe(qp, *ctrl, *seg, *size, *cur_edge, *idx, wr->wr_id, in handle_psv()
845 const struct ib_send_wr *wr, in handle_reg_mr_integrity() argument
860 mr = to_mmr(reg_wr(wr)->mr); in handle_reg_mr_integrity()
868 reg_pi_wr.access = reg_wr(wr)->access; in handle_reg_mr_integrity()
878 wr->wr_id, nreq, fence, MLX5_OPCODE_UMR); in handle_reg_mr_integrity()
880 err = begin_wqe(qp, seg, ctrl, wr, idx, size, cur_edge, nreq); in handle_reg_mr_integrity()
904 err = set_pi_umr_wr(wr, qp, seg, size, cur_edge); in handle_reg_mr_integrity()
909 mlx5r_finish_wqe(qp, *ctrl, *seg, *size, *cur_edge, *idx, wr->wr_id, in handle_reg_mr_integrity()
913 err = handle_psv(dev, qp, wr, ctrl, seg, size, cur_edge, idx, nreq, in handle_reg_mr_integrity()
919 err = handle_psv(dev, qp, wr, ctrl, seg, size, cur_edge, idx, nreq, in handle_reg_mr_integrity()
932 const struct ib_send_wr *wr, in handle_qpt_rc() argument
939 switch (wr->opcode) { in handle_qpt_rc()
943 handle_rdma_op(wr, seg, size); in handle_qpt_rc()
954 handle_local_inv(qp, wr, ctrl, seg, size, cur_edge, *idx); in handle_qpt_rc()
959 err = handle_reg_mr(qp, wr, ctrl, seg, size, cur_edge, *idx); in handle_qpt_rc()
966 err = handle_reg_mr_integrity(dev, qp, wr, ctrl, seg, size, in handle_qpt_rc()
982 static void handle_qpt_uc(const struct ib_send_wr *wr, void **seg, int *size) in handle_qpt_uc() argument
984 switch (wr->opcode) { in handle_qpt_uc()
987 handle_rdma_op(wr, seg, size); in handle_qpt_uc()
995 const struct ib_send_wr *wr, void **seg, in handle_qpt_hw_gsi() argument
998 set_datagram_seg(*seg, wr); in handle_qpt_hw_gsi()
1004 static void handle_qpt_ud(struct mlx5_ib_qp *qp, const struct ib_send_wr *wr, in handle_qpt_ud() argument
1007 set_datagram_seg(*seg, wr); in handle_qpt_ud()
1020 set_eth_seg(wr, qp, seg, size, cur_edge); in handle_qpt_ud()
1051 int mlx5_ib_post_send(struct ib_qp *ibqp, const struct ib_send_wr *wr, in mlx5_ib_post_send() argument
1073 *bad_wr = wr; in mlx5_ib_post_send()
1078 return mlx5_ib_gsi_post_send(ibqp, wr, bad_wr); in mlx5_ib_post_send()
1082 for (nreq = 0; wr; nreq++, wr = wr->next) { in mlx5_ib_post_send()
1083 if (unlikely(wr->opcode >= ARRAY_SIZE(mlx5_ib_opcode))) { in mlx5_ib_post_send()
1086 *bad_wr = wr; in mlx5_ib_post_send()
1090 num_sge = wr->num_sge; in mlx5_ib_post_send()
1094 *bad_wr = wr; in mlx5_ib_post_send()
1098 err = begin_wqe(qp, &seg, &ctrl, wr, &idx, &size, &cur_edge, in mlx5_ib_post_send()
1103 *bad_wr = wr; in mlx5_ib_post_send()
1107 if (wr->opcode == IB_WR_REG_MR || in mlx5_ib_post_send()
1108 wr->opcode == IB_WR_REG_MR_INTEGRITY) { in mlx5_ib_post_send()
1112 if (wr->send_flags & IB_SEND_FENCE) { in mlx5_ib_post_send()
1129 err = handle_qpt_rc(dev, qp, wr, &ctrl, &seg, &size, in mlx5_ib_post_send()
1133 *bad_wr = wr; in mlx5_ib_post_send()
1135 } else if (wr->opcode == IB_WR_REG_MR_INTEGRITY) { in mlx5_ib_post_send()
1141 handle_qpt_uc(wr, &seg, &size); in mlx5_ib_post_send()
1147 *bad_wr = wr; in mlx5_ib_post_send()
1152 handle_qpt_hw_gsi(qp, wr, &seg, &size, &cur_edge); in mlx5_ib_post_send()
1155 handle_qpt_ud(qp, wr, &seg, &size, &cur_edge); in mlx5_ib_post_send()
1162 if (wr->send_flags & IB_SEND_INLINE && num_sge) { in mlx5_ib_post_send()
1163 err = set_data_inl_seg(qp, wr, &seg, &size, &cur_edge); in mlx5_ib_post_send()
1166 *bad_wr = wr; in mlx5_ib_post_send()
1173 if (unlikely(!wr->sg_list[i].length)) in mlx5_ib_post_send()
1178 wr->sg_list + i); in mlx5_ib_post_send()
1185 mlx5r_finish_wqe(qp, ctrl, seg, size, cur_edge, idx, wr->wr_id, in mlx5_ib_post_send()
1186 nreq, fence, mlx5_ib_opcode[wr->opcode]); in mlx5_ib_post_send()
1206 int mlx5_ib_post_recv(struct ib_qp *ibqp, const struct ib_recv_wr *wr, in mlx5_ib_post_recv() argument
1222 *bad_wr = wr; in mlx5_ib_post_recv()
1227 return mlx5_ib_gsi_post_recv(ibqp, wr, bad_wr); in mlx5_ib_post_recv()
1233 for (nreq = 0; wr; nreq++, wr = wr->next) { in mlx5_ib_post_recv()
1236 *bad_wr = wr; in mlx5_ib_post_recv()
1240 if (unlikely(wr->num_sge > qp->rq.max_gs)) { in mlx5_ib_post_recv()
1242 *bad_wr = wr; in mlx5_ib_post_recv()
1250 for (i = 0; i < wr->num_sge; i++) in mlx5_ib_post_recv()
1251 set_data_ptr_seg(scat + i, wr->sg_list + i); in mlx5_ib_post_recv()
1264 qp->rq.wrid[ind] = wr->wr_id; in mlx5_ib_post_recv()