Lines Matching +full:long +full:- +full:term
1 // SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause
4 /* Copyright (c) 2008-2019, IBM Corporation */
44 (void *)(uintptr_t)umem->fp_addr); in siw_rx_umem()
46 srx->skb_copied += copied; in siw_rx_umem()
47 srx->skb_new -= copied; in siw_rx_umem()
49 return -EFAULT; in siw_rx_umem()
52 bytes = min(len, (int)PAGE_SIZE - pg_off); in siw_rx_umem()
57 rv = skb_copy_bits(srx->skb, srx->skb_offset, dest + pg_off, in siw_rx_umem()
62 srx->skb_copied += copied; in siw_rx_umem()
63 srx->skb_new -= copied; in siw_rx_umem()
68 return -EFAULT; in siw_rx_umem()
70 if (srx->mpa_crc_hd) { in siw_rx_umem()
71 if (rdma_is_kernel_res(&rx_qp(srx)->base_qp.res)) { in siw_rx_umem()
72 crypto_shash_update(srx->mpa_crc_hd, in siw_rx_umem()
92 srx->skb_offset += bytes; in siw_rx_umem()
94 len -= bytes; in siw_rx_umem()
98 srx->skb_copied += copied; in siw_rx_umem()
99 srx->skb_new -= copied; in siw_rx_umem()
110 rv = skb_copy_bits(srx->skb, srx->skb_offset, kva, len); in siw_rx_kva()
117 if (srx->mpa_crc_hd) in siw_rx_kva()
118 crypto_shash_update(srx->mpa_crc_hd, (u8 *)kva, len); in siw_rx_kva()
120 srx->skb_offset += len; in siw_rx_kva()
121 srx->skb_copied += len; in siw_rx_kva()
122 srx->skb_new -= len; in siw_rx_kva()
130 struct siw_pbl *pbl = mem->pbl; in siw_rx_pbl()
131 u64 offset = addr - mem->va; in siw_rx_pbl()
146 len -= bytes; in siw_rx_pbl()
168 struct iwarp_rdma_rresp *rresp = &srx->hdr.rresp; in siw_rresp_check_ntoh()
169 struct siw_wqe *wqe = &frx->wqe_active; in siw_rresp_check_ntoh()
172 u32 sink_stag = be32_to_cpu(rresp->sink_stag); in siw_rresp_check_ntoh()
173 u64 sink_to = be64_to_cpu(rresp->sink_to); in siw_rresp_check_ntoh()
175 if (frx->first_ddp_seg) { in siw_rresp_check_ntoh()
176 srx->ddp_stag = wqe->sqe.sge[0].lkey; in siw_rresp_check_ntoh()
177 srx->ddp_to = wqe->sqe.sge[0].laddr; in siw_rresp_check_ntoh()
178 frx->pbl_idx = 0; in siw_rresp_check_ntoh()
190 if (unlikely(srx->ddp_stag != sink_stag)) { in siw_rresp_check_ntoh()
192 qp_id(rx_qp(srx)), sink_stag, srx->ddp_stag); in siw_rresp_check_ntoh()
196 if (unlikely(srx->ddp_to != sink_to)) { in siw_rresp_check_ntoh()
198 qp_id(rx_qp(srx)), (unsigned long long)sink_to, in siw_rresp_check_ntoh()
199 (unsigned long long)srx->ddp_to); in siw_rresp_check_ntoh()
203 if (unlikely(!frx->more_ddp_segs && in siw_rresp_check_ntoh()
204 (wqe->processed + srx->fpdu_part_rem != wqe->bytes))) { in siw_rresp_check_ntoh()
207 wqe->processed + srx->fpdu_part_rem, wqe->bytes); in siw_rresp_check_ntoh()
215 return -EINVAL; in siw_rresp_check_ntoh()
232 struct iwarp_rdma_write *write = &srx->hdr.rwrite; in siw_write_check_ntoh()
235 u32 sink_stag = be32_to_cpu(write->sink_stag); in siw_write_check_ntoh()
236 u64 sink_to = be64_to_cpu(write->sink_to); in siw_write_check_ntoh()
238 if (frx->first_ddp_seg) { in siw_write_check_ntoh()
239 srx->ddp_stag = sink_stag; in siw_write_check_ntoh()
240 srx->ddp_to = sink_to; in siw_write_check_ntoh()
241 frx->pbl_idx = 0; in siw_write_check_ntoh()
243 if (unlikely(srx->ddp_stag != sink_stag)) { in siw_write_check_ntoh()
246 srx->ddp_stag); in siw_write_check_ntoh()
250 if (unlikely(srx->ddp_to != sink_to)) { in siw_write_check_ntoh()
253 (unsigned long long)sink_to, in siw_write_check_ntoh()
254 (unsigned long long)srx->ddp_to); in siw_write_check_ntoh()
263 return -EINVAL; in siw_write_check_ntoh()
280 struct iwarp_send_inv *send = &srx->hdr.send_inv; in siw_send_check_ntoh()
281 struct siw_wqe *wqe = &frx->wqe_active; in siw_send_check_ntoh()
284 u32 ddp_msn = be32_to_cpu(send->ddp_msn); in siw_send_check_ntoh()
285 u32 ddp_mo = be32_to_cpu(send->ddp_mo); in siw_send_check_ntoh()
286 u32 ddp_qn = be32_to_cpu(send->ddp_qn); in siw_send_check_ntoh()
294 if (unlikely(ddp_msn != srx->ddp_msn[RDMAP_UNTAGGED_QN_SEND])) { in siw_send_check_ntoh()
297 srx->ddp_msn[RDMAP_UNTAGGED_QN_SEND]); in siw_send_check_ntoh()
301 if (unlikely(ddp_mo != wqe->processed)) { in siw_send_check_ntoh()
303 qp_id(rx_qp(srx)), ddp_mo, wqe->processed); in siw_send_check_ntoh()
307 if (frx->first_ddp_seg) { in siw_send_check_ntoh()
309 frx->sge_idx = 0; in siw_send_check_ntoh()
310 frx->sge_off = 0; in siw_send_check_ntoh()
311 frx->pbl_idx = 0; in siw_send_check_ntoh()
314 srx->inval_stag = be32_to_cpu(send->inval_stag); in siw_send_check_ntoh()
316 if (unlikely(wqe->bytes < wqe->processed + srx->fpdu_part_rem)) { in siw_send_check_ntoh()
317 siw_dbg_qp(rx_qp(srx), "receive space short: %d - %d < %d\n", in siw_send_check_ntoh()
318 wqe->bytes, wqe->processed, srx->fpdu_part_rem); in siw_send_check_ntoh()
319 wqe->wc_status = SIW_WC_LOC_LEN_ERR; in siw_send_check_ntoh()
327 return -EINVAL; in siw_send_check_ntoh()
336 unsigned long flags; in siw_rqe_get()
338 srq = qp->srq; in siw_rqe_get()
340 spin_lock_irqsave(&srq->lock, flags); in siw_rqe_get()
341 if (unlikely(!srq->num_rqe)) in siw_rqe_get()
344 rqe = &srq->recvq[srq->rq_get % srq->num_rqe]; in siw_rqe_get()
346 if (unlikely(!qp->recvq)) in siw_rqe_get()
349 rqe = &qp->recvq[qp->rq_get % qp->attrs.rq_size]; in siw_rqe_get()
351 if (likely(rqe->flags == SIW_WQE_VALID)) { in siw_rqe_get()
352 int num_sge = rqe->num_sge; in siw_rqe_get()
357 wqe = rx_wqe(&qp->rx_untagged); in siw_rqe_get()
359 wqe->wr_status = SIW_WR_INPROGRESS; in siw_rqe_get()
360 wqe->bytes = 0; in siw_rqe_get()
361 wqe->processed = 0; in siw_rqe_get()
363 wqe->rqe.id = rqe->id; in siw_rqe_get()
364 wqe->rqe.num_sge = num_sge; in siw_rqe_get()
367 wqe->rqe.sge[i].laddr = rqe->sge[i].laddr; in siw_rqe_get()
368 wqe->rqe.sge[i].lkey = rqe->sge[i].lkey; in siw_rqe_get()
369 wqe->rqe.sge[i].length = rqe->sge[i].length; in siw_rqe_get()
370 wqe->bytes += wqe->rqe.sge[i].length; in siw_rqe_get()
371 wqe->mem[i] = NULL; in siw_rqe_get()
374 /* can be re-used by appl */ in siw_rqe_get()
375 smp_store_mb(rqe->flags, 0); in siw_rqe_get()
377 siw_dbg_qp(qp, "too many sge's: %d\n", rqe->num_sge); in siw_rqe_get()
379 spin_unlock_irqrestore(&srq->lock, flags); in siw_rqe_get()
383 qp->rq_get++; in siw_rqe_get()
385 if (srq->armed) { in siw_rqe_get()
387 u32 off = (srq->rq_get + srq->limit) % in siw_rqe_get()
388 srq->num_rqe; in siw_rqe_get()
389 struct siw_rqe *rqe2 = &srq->recvq[off]; in siw_rqe_get()
391 if (!(rqe2->flags & SIW_WQE_VALID)) { in siw_rqe_get()
392 srq->armed = false; in siw_rqe_get()
396 srq->rq_get++; in siw_rqe_get()
401 spin_unlock_irqrestore(&srq->lock, flags); in siw_rqe_get()
413 if (mem_p->mem_obj == NULL) in siw_rx_data()
415 else if (!mem_p->is_pbl) in siw_rx_data()
416 rv = siw_rx_umem(srx, mem_p->umem, addr, bytes); in siw_rx_data()
433 * -EAGAIN: to be called again to finish the DDP segment
437 struct siw_rx_stream *srx = &qp->rx_stream; in siw_proc_send()
438 struct siw_rx_fpdu *frx = &qp->rx_untagged; in siw_proc_send()
444 if (frx->first_ddp_seg) { in siw_proc_send()
450 return -ENOENT; in siw_proc_send()
455 if (srx->state == SIW_GET_DATA_START) { in siw_proc_send()
461 if (!srx->fpdu_part_rem) /* zero length SEND */ in siw_proc_send()
464 data_bytes = min(srx->fpdu_part_rem, srx->skb_new); in siw_proc_send()
474 sge = &wqe->rqe.sge[frx->sge_idx]; in siw_proc_send()
476 if (!sge->length) { in siw_proc_send()
478 frx->sge_idx++; in siw_proc_send()
479 frx->sge_off = 0; in siw_proc_send()
480 frx->pbl_idx = 0; in siw_proc_send()
483 sge_bytes = min(data_bytes, sge->length - frx->sge_off); in siw_proc_send()
484 mem = &wqe->mem[frx->sge_idx]; in siw_proc_send()
489 pd = qp->srq == NULL ? qp->pd : qp->srq->base_srq.pd; in siw_proc_send()
492 frx->sge_off, sge_bytes); in siw_proc_send()
502 rv = siw_rx_data(mem_p, srx, &frx->pbl_idx, in siw_proc_send()
503 sge->laddr + frx->sge_off, sge_bytes); in siw_proc_send()
505 wqe->processed += rcvd_bytes; in siw_proc_send()
510 return -EINVAL; in siw_proc_send()
512 frx->sge_off += rv; in siw_proc_send()
514 if (frx->sge_off == sge->length) { in siw_proc_send()
515 frx->sge_idx++; in siw_proc_send()
516 frx->sge_off = 0; in siw_proc_send()
517 frx->pbl_idx = 0; in siw_proc_send()
519 data_bytes -= rv; in siw_proc_send()
522 srx->fpdu_part_rem -= rv; in siw_proc_send()
523 srx->fpdu_part_rcvd += rv; in siw_proc_send()
525 wqe->processed += rcvd_bytes; in siw_proc_send()
527 if (!srx->fpdu_part_rem) in siw_proc_send()
530 return (rv < 0) ? rv : -EAGAIN; in siw_proc_send()
543 * -EAGAIN: to be called again to finish the DDP segment
547 struct siw_rx_stream *srx = &qp->rx_stream; in siw_proc_write()
548 struct siw_rx_fpdu *frx = &qp->rx_tagged; in siw_proc_write()
552 if (srx->state == SIW_GET_DATA_START) { in siw_proc_write()
553 if (!srx->fpdu_part_rem) /* zero length WRITE */ in siw_proc_write()
562 bytes = min(srx->fpdu_part_rem, srx->skb_new); in siw_proc_write()
564 if (frx->first_ddp_seg) { in siw_proc_write()
567 rx_mem(frx) = siw_mem_id2obj(qp->sdev, srx->ddp_stag >> 8); in siw_proc_write()
571 srx->ddp_stag); in siw_proc_write()
576 return -EINVAL; in siw_proc_write()
578 wqe->rqe.num_sge = 1; in siw_proc_write()
580 wqe->wr_status = SIW_WR_INPROGRESS; in siw_proc_write()
585 * Check if application re-registered memory with different in siw_proc_write()
588 if (unlikely(mem->stag != srx->ddp_stag)) { in siw_proc_write()
592 return -EINVAL; in siw_proc_write()
594 rv = siw_check_mem(qp->pd, mem, srx->ddp_to + srx->fpdu_part_rcvd, in siw_proc_write()
598 DDP_ETYPE_TAGGED_BUF, siw_tagged_error(-rv), in siw_proc_write()
603 return -EINVAL; in siw_proc_write()
606 rv = siw_rx_data(mem, srx, &frx->pbl_idx, in siw_proc_write()
607 srx->ddp_to + srx->fpdu_part_rcvd, bytes); in siw_proc_write()
612 return -EINVAL; in siw_proc_write()
614 srx->fpdu_part_rem -= rv; in siw_proc_write()
615 srx->fpdu_part_rcvd += rv; in siw_proc_write()
617 if (!srx->fpdu_part_rem) { in siw_proc_write()
618 srx->ddp_to += srx->fpdu_part_rcvd; in siw_proc_write()
621 return -EAGAIN; in siw_proc_write()
629 struct siw_rx_stream *srx = &qp->rx_stream; in siw_proc_rreq()
631 if (!srx->fpdu_part_rem) in siw_proc_rreq()
635 be16_to_cpu(srx->hdr.ctrl.mpa_len)); in siw_proc_rreq()
637 return -EPROTO; in siw_proc_rreq()
661 uint64_t raddr = be64_to_cpu(srx->hdr.rreq.sink_to), in siw_init_rresp()
662 laddr = be64_to_cpu(srx->hdr.rreq.source_to); in siw_init_rresp()
663 uint32_t length = be32_to_cpu(srx->hdr.rreq.read_size), in siw_init_rresp()
664 lkey = be32_to_cpu(srx->hdr.rreq.source_stag), in siw_init_rresp()
665 rkey = be32_to_cpu(srx->hdr.rreq.sink_stag), in siw_init_rresp()
666 msn = be32_to_cpu(srx->hdr.rreq.ddp_msn); in siw_init_rresp()
669 unsigned long flags; in siw_init_rresp()
671 if (unlikely(msn != srx->ddp_msn[RDMAP_UNTAGGED_QN_RDMA_READ])) { in siw_init_rresp()
675 return -EPROTO; in siw_init_rresp()
677 spin_lock_irqsave(&qp->sq_lock, flags); in siw_init_rresp()
679 if (unlikely(!qp->attrs.irq_size)) { in siw_init_rresp()
683 if (tx_work->wr_status == SIW_WR_IDLE) { in siw_init_rresp()
688 tx_work->processed = 0; in siw_init_rresp()
689 tx_work->mem[0] = NULL; in siw_init_rresp()
690 tx_work->wr_status = SIW_WR_QUEUED; in siw_init_rresp()
691 resp = &tx_work->sqe; in siw_init_rresp()
697 resp->opcode = SIW_OP_READ_RESPONSE; in siw_init_rresp()
699 resp->sge[0].length = length; in siw_init_rresp()
700 resp->sge[0].laddr = laddr; in siw_init_rresp()
701 resp->sge[0].lkey = lkey; in siw_init_rresp()
706 resp->sge[1].length = msn; in siw_init_rresp()
708 resp->raddr = raddr; in siw_init_rresp()
709 resp->rkey = rkey; in siw_init_rresp()
710 resp->num_sge = length ? 1 : 0; in siw_init_rresp()
713 smp_store_mb(resp->flags, SIW_WQE_VALID); in siw_init_rresp()
717 qp_id(qp), qp->attrs.irq_size); in siw_init_rresp()
722 rv = -EPROTO; in siw_init_rresp()
725 spin_unlock_irqrestore(&qp->sq_lock, flags); in siw_init_rresp()
744 if (unlikely(!qp->attrs.orq_size)) in siw_orqe_start_rx()
745 return -EPROTO; in siw_orqe_start_rx()
751 if (READ_ONCE(orqe->flags) & SIW_WQE_VALID) { in siw_orqe_start_rx()
753 wqe = rx_wqe(&qp->rx_tagged); in siw_orqe_start_rx()
754 wqe->sqe.id = orqe->id; in siw_orqe_start_rx()
755 wqe->sqe.opcode = orqe->opcode; in siw_orqe_start_rx()
756 wqe->sqe.sge[0].laddr = orqe->sge[0].laddr; in siw_orqe_start_rx()
757 wqe->sqe.sge[0].lkey = orqe->sge[0].lkey; in siw_orqe_start_rx()
758 wqe->sqe.sge[0].length = orqe->sge[0].length; in siw_orqe_start_rx()
759 wqe->sqe.flags = orqe->flags; in siw_orqe_start_rx()
760 wqe->sqe.num_sge = 1; in siw_orqe_start_rx()
761 wqe->bytes = orqe->sge[0].length; in siw_orqe_start_rx()
762 wqe->processed = 0; in siw_orqe_start_rx()
763 wqe->mem[0] = NULL; in siw_orqe_start_rx()
766 wqe->wr_status = SIW_WR_INPROGRESS; in siw_orqe_start_rx()
770 return -EPROTO; in siw_orqe_start_rx()
784 struct siw_rx_stream *srx = &qp->rx_stream; in siw_proc_rresp()
785 struct siw_rx_fpdu *frx = &qp->rx_tagged; in siw_proc_rresp()
791 if (frx->first_ddp_seg) { in siw_proc_rresp()
792 if (unlikely(wqe->wr_status != SIW_WR_IDLE)) { in siw_proc_rresp()
794 qp_id(qp), wqe->wr_status, wqe->sqe.opcode); in siw_proc_rresp()
795 rv = -EPROTO; in siw_proc_rresp()
804 qp_id(qp), qp->attrs.orq_size); in siw_proc_rresp()
813 if (unlikely(wqe->wr_status != SIW_WR_INPROGRESS)) { in siw_proc_rresp()
815 qp_id(qp), wqe->wr_status); in siw_proc_rresp()
816 rv = -EPROTO; in siw_proc_rresp()
820 if (!srx->fpdu_part_rem) /* zero length RRESPONSE */ in siw_proc_rresp()
823 sge = wqe->sqe.sge; /* there is only one */ in siw_proc_rresp()
824 mem = &wqe->mem[0]; in siw_proc_rresp()
830 rv = siw_check_sge(qp->pd, sge, mem, IB_ACCESS_LOCAL_WRITE, 0, in siw_proc_rresp()
831 wqe->bytes); in siw_proc_rresp()
834 wqe->wc_status = SIW_WC_LOC_PROT_ERR; in siw_proc_rresp()
838 siw_tagged_error(-rv), 0); in siw_proc_rresp()
842 return -EINVAL; in siw_proc_rresp()
847 bytes = min(srx->fpdu_part_rem, srx->skb_new); in siw_proc_rresp()
848 rv = siw_rx_data(mem_p, srx, &frx->pbl_idx, in siw_proc_rresp()
849 sge->laddr + wqe->processed, bytes); in siw_proc_rresp()
851 wqe->wc_status = SIW_WC_GENERAL_ERR; in siw_proc_rresp()
852 rv = -EINVAL; in siw_proc_rresp()
855 srx->fpdu_part_rem -= rv; in siw_proc_rresp()
856 srx->fpdu_part_rcvd += rv; in siw_proc_rresp()
857 wqe->processed += rv; in siw_proc_rresp()
859 if (!srx->fpdu_part_rem) { in siw_proc_rresp()
860 srx->ddp_to += srx->fpdu_part_rcvd; in siw_proc_rresp()
863 return -EAGAIN; in siw_proc_rresp()
873 srx->skb_offset += length; in siw_update_skb_rcvd()
874 srx->skb_new -= length; in siw_update_skb_rcvd()
875 srx->skb_copied += length; in siw_update_skb_rcvd()
880 struct siw_rx_stream *srx = &qp->rx_stream; in siw_proc_terminate()
881 struct sk_buff *skb = srx->skb; in siw_proc_terminate()
882 struct iwarp_terminate *term = &srx->hdr.terminate; in siw_proc_terminate() local
889 __rdmap_term_layer(term), __rdmap_term_etype(term), in siw_proc_terminate()
890 __rdmap_term_ecode(term)); in siw_proc_terminate()
892 if (be32_to_cpu(term->ddp_qn) != RDMAP_UNTAGGED_QN_TERMINATE || in siw_proc_terminate()
893 be32_to_cpu(term->ddp_msn) != in siw_proc_terminate()
894 qp->rx_stream.ddp_msn[RDMAP_UNTAGGED_QN_TERMINATE] || in siw_proc_terminate()
895 be32_to_cpu(term->ddp_mo) != 0) { in siw_proc_terminate()
896 pr_warn("siw: rx bogus TERM [QN x%08x, MSN x%08x, MO x%08x]\n", in siw_proc_terminate()
897 be32_to_cpu(term->ddp_qn), be32_to_cpu(term->ddp_msn), in siw_proc_terminate()
898 be32_to_cpu(term->ddp_mo)); in siw_proc_terminate()
899 return -ECONNRESET; in siw_proc_terminate()
902 * Receive remaining pieces of TERM if indicated in siw_proc_terminate()
904 if (!term->flag_m) in siw_proc_terminate()
905 return -ECONNRESET; in siw_proc_terminate()
908 * TERM message in siw_proc_terminate()
910 if (srx->skb_new < sizeof(struct iwarp_ctrl_tagged)) in siw_proc_terminate()
911 return -ECONNRESET; in siw_proc_terminate()
915 skb_copy_bits(skb, srx->skb_offset, infop, to_copy); in siw_proc_terminate()
923 srx->fpdu_part_rcvd += to_copy; in siw_proc_terminate()
924 srx->fpdu_part_rem -= to_copy; in siw_proc_terminate()
926 to_copy = iwarp_pktinfo[op].hdr_len - to_copy; in siw_proc_terminate()
928 /* Again, no network fragmented TERM's */ in siw_proc_terminate()
929 if (to_copy + MPA_CRC_SIZE > srx->skb_new) in siw_proc_terminate()
930 return -ECONNRESET; in siw_proc_terminate()
932 skb_copy_bits(skb, srx->skb_offset, infop, to_copy); in siw_proc_terminate()
934 if (term->flag_r) { in siw_proc_terminate()
935 siw_dbg_qp(qp, "TERM reports RDMAP hdr type %u, len %u (%s)\n", in siw_proc_terminate()
937 term->flag_m ? "valid" : "invalid"); in siw_proc_terminate()
938 } else if (term->flag_d) { in siw_proc_terminate()
939 siw_dbg_qp(qp, "TERM reports DDP hdr type %u, len %u (%s)\n", in siw_proc_terminate()
941 term->flag_m ? "valid" : "invalid"); in siw_proc_terminate()
945 srx->fpdu_part_rcvd += to_copy; in siw_proc_terminate()
946 srx->fpdu_part_rem -= to_copy; in siw_proc_terminate()
948 return -ECONNRESET; in siw_proc_terminate()
953 struct sk_buff *skb = srx->skb; in siw_get_trailer()
954 int avail = min(srx->skb_new, srx->fpdu_part_rem); in siw_get_trailer()
955 u8 *tbuf = (u8 *)&srx->trailer.crc - srx->pad; in siw_get_trailer()
959 srx->fpdu_part_rem, srx->skb_new, srx->pad); in siw_get_trailer()
961 skb_copy_bits(skb, srx->skb_offset, tbuf, avail); in siw_get_trailer()
964 srx->fpdu_part_rem -= avail; in siw_get_trailer()
966 if (srx->fpdu_part_rem) in siw_get_trailer()
967 return -EAGAIN; in siw_get_trailer()
969 if (!srx->mpa_crc_hd) in siw_get_trailer()
972 if (srx->pad) in siw_get_trailer()
973 crypto_shash_update(srx->mpa_crc_hd, tbuf, srx->pad); in siw_get_trailer()
978 crypto_shash_final(srx->mpa_crc_hd, (u8 *)&crc_own); in siw_get_trailer()
979 crc_in = (__force __wsum)srx->trailer.crc; in siw_get_trailer()
983 crc_in, crc_own, qp->rx_stream.rdmap_op); in siw_get_trailer()
988 return -EINVAL; in siw_get_trailer()
997 struct sk_buff *skb = srx->skb; in siw_get_hdr()
999 struct iwarp_ctrl *c_hdr = &srx->hdr.ctrl; in siw_get_hdr()
1004 if (srx->fpdu_part_rcvd < MIN_DDP_HDR) { in siw_get_hdr()
1008 bytes = min_t(int, srx->skb_new, in siw_get_hdr()
1009 MIN_DDP_HDR - srx->fpdu_part_rcvd); in siw_get_hdr()
1011 skb_copy_bits(skb, srx->skb_offset, in siw_get_hdr()
1012 (char *)c_hdr + srx->fpdu_part_rcvd, bytes); in siw_get_hdr()
1015 srx->fpdu_part_rcvd += bytes; in siw_get_hdr()
1016 if (srx->fpdu_part_rcvd < MIN_DDP_HDR) in siw_get_hdr()
1017 return -EAGAIN; in siw_get_hdr()
1026 if (c_hdr->ddp_rdmap_ctrl & DDP_FLAG_TAGGED) { in siw_get_hdr()
1035 return -EINVAL; in siw_get_hdr()
1044 return -EINVAL; in siw_get_hdr()
1055 return -EINVAL; in siw_get_hdr()
1062 frx = qp->rx_fpdu; in siw_get_hdr()
1073 bytes = min_t(int, hdrlen - MIN_DDP_HDR, srx->skb_new); in siw_get_hdr()
1075 skb_copy_bits(skb, srx->skb_offset, in siw_get_hdr()
1076 (char *)c_hdr + srx->fpdu_part_rcvd, bytes); in siw_get_hdr()
1079 srx->fpdu_part_rcvd += bytes; in siw_get_hdr()
1080 if (srx->fpdu_part_rcvd < hdrlen) in siw_get_hdr()
1081 return -EAGAIN; in siw_get_hdr()
1090 * tagged and untagged RDMAP messages is supported, as long as in siw_get_hdr()
1096 if (srx->mpa_crc_hd) { in siw_get_hdr()
1100 crypto_shash_init(srx->mpa_crc_hd); in siw_get_hdr()
1101 crypto_shash_update(srx->mpa_crc_hd, (u8 *)c_hdr, in siw_get_hdr()
1102 srx->fpdu_part_rcvd); in siw_get_hdr()
1104 if (frx->more_ddp_segs) { in siw_get_hdr()
1105 frx->first_ddp_seg = 0; in siw_get_hdr()
1106 if (frx->prev_rdmap_op != opcode) { in siw_get_hdr()
1108 frx->prev_rdmap_op, opcode); in siw_get_hdr()
1117 set_rx_fpdu_context(qp, frx->prev_rdmap_op); in siw_get_hdr()
1118 __rdmap_set_opcode(c_hdr, frx->prev_rdmap_op); in siw_get_hdr()
1119 return -EPROTO; in siw_get_hdr()
1122 frx->prev_rdmap_op = opcode; in siw_get_hdr()
1123 frx->first_ddp_seg = 1; in siw_get_hdr()
1125 frx->more_ddp_segs = c_hdr->ddp_rdmap_ctrl & DDP_FLAG_LAST ? 0 : 1; in siw_get_hdr()
1135 unsigned long flags; in siw_check_tx_fence()
1137 spin_lock_irqsave(&qp->orq_lock, flags); in siw_check_tx_fence()
1141 WRITE_ONCE(rreq->flags, 0); in siw_check_tx_fence()
1143 qp->orq_get++; in siw_check_tx_fence()
1145 if (qp->tx_ctx.orq_fence) { in siw_check_tx_fence()
1146 if (unlikely(tx_waiting->wr_status != SIW_WR_QUEUED)) { in siw_check_tx_fence()
1148 qp_id(qp), tx_waiting->wr_status); in siw_check_tx_fence()
1149 rv = -EPROTO; in siw_check_tx_fence()
1153 if (tx_waiting->sqe.opcode == SIW_OP_READ || in siw_check_tx_fence()
1154 tx_waiting->sqe.opcode == SIW_OP_READ_LOCAL_INV) { in siw_check_tx_fence()
1160 rv = -EPROTO; in siw_check_tx_fence()
1163 siw_read_to_orq(rreq, &tx_waiting->sqe); in siw_check_tx_fence()
1165 qp->orq_put++; in siw_check_tx_fence()
1166 qp->tx_ctx.orq_fence = 0; in siw_check_tx_fence()
1174 qp->tx_ctx.orq_fence = 0; in siw_check_tx_fence()
1179 spin_unlock_irqrestore(&qp->orq_lock, flags); in siw_check_tx_fence()
1201 struct siw_rx_stream *srx = &qp->rx_stream; in siw_rdmap_complete()
1202 struct siw_wqe *wqe = rx_wqe(qp->rx_fpdu); in siw_rdmap_complete()
1203 enum siw_wc_status wc_status = wqe->wc_status; in siw_rdmap_complete()
1204 u8 opcode = __rdmap_get_opcode(&srx->hdr.ctrl); in siw_rdmap_complete()
1210 wqe->rqe.flags |= SIW_WQE_SOLICITED; in siw_rdmap_complete()
1215 if (wqe->wr_status == SIW_WR_IDLE) in siw_rdmap_complete()
1218 srx->ddp_msn[RDMAP_UNTAGGED_QN_SEND]++; in siw_rdmap_complete()
1228 rv = siw_invalidate_stag(qp->pd, srx->inval_stag); in siw_rdmap_complete()
1232 rv == -EACCES ? in siw_rdmap_complete()
1239 rv = siw_rqe_complete(qp, &wqe->rqe, wqe->processed, in siw_rdmap_complete()
1240 rv ? 0 : srx->inval_stag, in siw_rdmap_complete()
1243 rv = siw_rqe_complete(qp, &wqe->rqe, wqe->processed, in siw_rdmap_complete()
1250 if (wqe->wr_status == SIW_WR_IDLE) in siw_rdmap_complete()
1254 if ((srx->state == SIW_GET_HDR && in siw_rdmap_complete()
1255 qp->rx_fpdu->first_ddp_seg) || error == -ENODATA) in siw_rdmap_complete()
1261 } else if (rdma_is_kernel_res(&qp->base_qp.res) && in siw_rdmap_complete()
1266 rv = siw_invalidate_stag(qp->pd, wqe->sqe.sge[0].lkey); in siw_rdmap_complete()
1281 if ((wqe->sqe.flags & SIW_WQE_SIGNALLED) || error != 0) in siw_rdmap_complete()
1282 rv = siw_sqe_complete(qp, &wqe->sqe, wqe->processed, in siw_rdmap_complete()
1290 if (qp->attrs.orq_size) in siw_rdmap_complete()
1291 WRITE_ONCE(orq_get_current(qp)->flags, 0); in siw_rdmap_complete()
1298 srx->ddp_msn[RDMAP_UNTAGGED_QN_RDMA_READ]++; in siw_rdmap_complete()
1303 if (wqe->wr_status == SIW_WR_IDLE) in siw_rdmap_complete()
1309 * While a zero-length WRITE is allowed, in siw_rdmap_complete()
1312 if (rx_mem(&qp->rx_tagged)) { in siw_rdmap_complete()
1313 siw_mem_put(rx_mem(&qp->rx_tagged)); in siw_rdmap_complete()
1314 rx_mem(&qp->rx_tagged) = NULL; in siw_rdmap_complete()
1321 wqe->wr_status = SIW_WR_IDLE; in siw_rdmap_complete()
1334 * @len: skb->len - offset : payload in skb
1339 struct siw_qp *qp = rd_desc->arg.data; in siw_tcp_rx_data()
1340 struct siw_rx_stream *srx = &qp->rx_stream; in siw_tcp_rx_data()
1343 srx->skb = skb; in siw_tcp_rx_data()
1344 srx->skb_new = skb->len - off; in siw_tcp_rx_data()
1345 srx->skb_offset = off; in siw_tcp_rx_data()
1346 srx->skb_copied = 0; in siw_tcp_rx_data()
1348 siw_dbg_qp(qp, "new data, len %d\n", srx->skb_new); in siw_tcp_rx_data()
1350 while (srx->skb_new) { in siw_tcp_rx_data()
1353 if (unlikely(srx->rx_suspend)) { in siw_tcp_rx_data()
1355 srx->skb_copied += srx->skb_new; in siw_tcp_rx_data()
1358 switch (srx->state) { in siw_tcp_rx_data()
1362 srx->fpdu_part_rem = in siw_tcp_rx_data()
1363 be16_to_cpu(srx->hdr.ctrl.mpa_len) - in siw_tcp_rx_data()
1364 srx->fpdu_part_rcvd + MPA_HDR_SIZE; in siw_tcp_rx_data()
1366 if (srx->fpdu_part_rem) in siw_tcp_rx_data()
1367 srx->pad = -srx->fpdu_part_rem & 0x3; in siw_tcp_rx_data()
1369 srx->pad = 0; in siw_tcp_rx_data()
1371 srx->state = SIW_GET_DATA_START; in siw_tcp_rx_data()
1372 srx->fpdu_part_rcvd = 0; in siw_tcp_rx_data()
1383 qp->rx_fpdu->first_ddp_seg = 0; in siw_tcp_rx_data()
1388 * Headers will be checked by the opcode-specific in siw_tcp_rx_data()
1391 rv = iwarp_pktinfo[qp->rx_stream.rdmap_op].rx_data(qp); in siw_tcp_rx_data()
1394 be16_to_cpu(srx->hdr.ctrl.mpa_len) in siw_tcp_rx_data()
1397 srx->fpdu_part_rem = (-mpa_len & 0x3) in siw_tcp_rx_data()
1399 srx->fpdu_part_rcvd = 0; in siw_tcp_rx_data()
1400 srx->state = SIW_GET_TRAILER; in siw_tcp_rx_data()
1402 if (unlikely(rv == -ECONNRESET)) in siw_tcp_rx_data()
1405 srx->state = SIW_GET_DATA_MORE; in siw_tcp_rx_data()
1419 srx->state = SIW_GET_HDR; in siw_tcp_rx_data()
1420 srx->fpdu_part_rcvd = 0; in siw_tcp_rx_data()
1422 if (!(srx->hdr.ctrl.ddp_rdmap_ctrl & in siw_tcp_rx_data()
1434 rv = -EPROTO; in siw_tcp_rx_data()
1437 if (unlikely(rv != 0 && rv != -EAGAIN)) { in siw_tcp_rx_data()
1438 if ((srx->state > SIW_GET_HDR || in siw_tcp_rx_data()
1439 qp->rx_fpdu->more_ddp_segs) && run_completion) in siw_tcp_rx_data()
1443 srx->state); in siw_tcp_rx_data()
1451 srx->state, srx->fpdu_part_rem); in siw_tcp_rx_data()
1455 return srx->skb_copied; in siw_tcp_rx_data()