Lines Matching +full:long +full:- +full:term
1 // SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause
4 /* Copyright (c) 2008-2019, IBM Corporation */
30 * per-RDMAP message basis. Please keep order of initializer. All MPA len
36 .ctrl.mpa_len = htons(sizeof(struct iwarp_rdma_write) - 2),
44 .ctrl.mpa_len = htons(sizeof(struct iwarp_rdma_rreq) - 2),
51 .ctrl.mpa_len = htons(sizeof(struct iwarp_rdma_rresp) - 2),
59 .ctrl.mpa_len = htons(sizeof(struct iwarp_send) - 2),
66 .ctrl.mpa_len = htons(sizeof(struct iwarp_send_inv) - 2),
73 .ctrl.mpa_len = htons(sizeof(struct iwarp_send) - 2),
80 .ctrl.mpa_len = htons(sizeof(struct iwarp_send_inv) - 2),
87 .ctrl.mpa_len = htons(sizeof(struct iwarp_terminate) - 2),
100 read_lock(&sk->sk_callback_lock); in siw_qp_llp_data_ready()
102 if (unlikely(!sk->sk_user_data || !sk_to_qp(sk))) in siw_qp_llp_data_ready()
107 if (likely(!qp->rx_stream.rx_suspend && in siw_qp_llp_data_ready()
108 down_read_trylock(&qp->state_lock))) { in siw_qp_llp_data_ready()
111 if (likely(qp->attrs.state == SIW_QP_STATE_RTS)) in siw_qp_llp_data_ready()
120 up_read(&qp->state_lock); in siw_qp_llp_data_ready()
123 qp->rx_stream.rx_suspend); in siw_qp_llp_data_ready()
126 read_unlock(&sk->sk_callback_lock); in siw_qp_llp_data_ready()
132 siw_qp_state_to_string[qp->attrs.state]); in siw_qp_llp_close()
134 down_write(&qp->state_lock); in siw_qp_llp_close()
136 qp->rx_stream.rx_suspend = 1; in siw_qp_llp_close()
137 qp->tx_ctx.tx_suspend = 1; in siw_qp_llp_close()
138 qp->attrs.sk = NULL; in siw_qp_llp_close()
140 switch (qp->attrs.state) { in siw_qp_llp_close()
145 qp->attrs.state = SIW_QP_STATE_ERROR; in siw_qp_llp_close()
154 if (tx_wqe(qp)->wr_status == SIW_WR_IDLE) in siw_qp_llp_close()
155 qp->attrs.state = SIW_QP_STATE_ERROR; in siw_qp_llp_close()
157 qp->attrs.state = SIW_QP_STATE_IDLE; in siw_qp_llp_close()
162 siw_qp_state_to_string[qp->attrs.state]); in siw_qp_llp_close()
171 if (qp->cep) { in siw_qp_llp_close()
172 siw_cep_put(qp->cep); in siw_qp_llp_close()
173 qp->cep = NULL; in siw_qp_llp_close()
176 up_write(&qp->state_lock); in siw_qp_llp_close()
179 siw_qp_state_to_string[qp->attrs.state]); in siw_qp_llp_close()
190 read_lock(&sk->sk_callback_lock); in siw_qp_llp_write_space()
194 cep->sk_write_space(sk); in siw_qp_llp_write_space()
196 if (!test_bit(SOCK_NOSPACE, &sk->sk_socket->flags)) in siw_qp_llp_write_space()
197 (void)siw_sq_start(cep->qp); in siw_qp_llp_write_space()
200 read_unlock(&sk->sk_callback_lock); in siw_qp_llp_write_space()
207 qp->irq = vcalloc(irq_size, sizeof(struct siw_sqe)); in siw_qp_readq_init()
208 if (!qp->irq) { in siw_qp_readq_init()
209 qp->attrs.irq_size = 0; in siw_qp_readq_init()
210 return -ENOMEM; in siw_qp_readq_init()
215 qp->orq = vcalloc(orq_size, sizeof(struct siw_sqe)); in siw_qp_readq_init()
216 if (!qp->orq) { in siw_qp_readq_init()
217 qp->attrs.orq_size = 0; in siw_qp_readq_init()
218 qp->attrs.irq_size = 0; in siw_qp_readq_init()
219 vfree(qp->irq); in siw_qp_readq_init()
220 return -ENOMEM; in siw_qp_readq_init()
223 qp->attrs.irq_size = irq_size; in siw_qp_readq_init()
224 qp->attrs.orq_size = orq_size; in siw_qp_readq_init()
231 struct siw_rx_stream *c_rx = &qp->rx_stream; in siw_qp_enable_crc()
232 struct siw_iwarp_tx *c_tx = &qp->tx_ctx; in siw_qp_enable_crc()
236 return -ENOENT; in siw_qp_enable_crc()
241 c_tx->mpa_crc_hd = kzalloc(size, GFP_KERNEL); in siw_qp_enable_crc()
242 c_rx->mpa_crc_hd = kzalloc(size, GFP_KERNEL); in siw_qp_enable_crc()
243 if (!c_tx->mpa_crc_hd || !c_rx->mpa_crc_hd) { in siw_qp_enable_crc()
244 kfree(c_tx->mpa_crc_hd); in siw_qp_enable_crc()
245 kfree(c_rx->mpa_crc_hd); in siw_qp_enable_crc()
246 c_tx->mpa_crc_hd = NULL; in siw_qp_enable_crc()
247 c_rx->mpa_crc_hd = NULL; in siw_qp_enable_crc()
248 return -ENOMEM; in siw_qp_enable_crc()
250 c_tx->mpa_crc_hd->tfm = siw_crypto_shash; in siw_qp_enable_crc()
251 c_rx->mpa_crc_hd->tfm = siw_crypto_shash; in siw_qp_enable_crc()
266 unsigned long flags; in siw_qp_mpa_rts()
269 spin_lock_irqsave(&qp->sq_lock, flags); in siw_qp_mpa_rts()
271 if (unlikely(wqe->wr_status != SIW_WR_IDLE)) { in siw_qp_mpa_rts()
272 spin_unlock_irqrestore(&qp->sq_lock, flags); in siw_qp_mpa_rts()
273 return -EIO; in siw_qp_mpa_rts()
275 memset(wqe->mem, 0, sizeof(*wqe->mem) * SIW_MAX_SGE); in siw_qp_mpa_rts()
277 wqe->wr_status = SIW_WR_QUEUED; in siw_qp_mpa_rts()
278 wqe->sqe.flags = 0; in siw_qp_mpa_rts()
279 wqe->sqe.num_sge = 1; in siw_qp_mpa_rts()
280 wqe->sqe.sge[0].length = 0; in siw_qp_mpa_rts()
281 wqe->sqe.sge[0].laddr = 0; in siw_qp_mpa_rts()
282 wqe->sqe.sge[0].lkey = 0; in siw_qp_mpa_rts()
287 wqe->sqe.rkey = 1; in siw_qp_mpa_rts()
288 wqe->sqe.raddr = 0; in siw_qp_mpa_rts()
289 wqe->processed = 0; in siw_qp_mpa_rts()
292 wqe->sqe.opcode = SIW_OP_WRITE; in siw_qp_mpa_rts()
296 wqe->sqe.opcode = SIW_OP_READ; in siw_qp_mpa_rts()
298 spin_lock(&qp->orq_lock); in siw_qp_mpa_rts()
300 if (qp->attrs.orq_size) in siw_qp_mpa_rts()
303 siw_read_to_orq(rreq, &wqe->sqe); in siw_qp_mpa_rts()
304 qp->orq_put++; in siw_qp_mpa_rts()
306 rv = -EIO; in siw_qp_mpa_rts()
308 spin_unlock(&qp->orq_lock); in siw_qp_mpa_rts()
310 rv = -EINVAL; in siw_qp_mpa_rts()
313 wqe->wr_status = SIW_WR_IDLE; in siw_qp_mpa_rts()
315 spin_unlock_irqrestore(&qp->sq_lock, flags); in siw_qp_mpa_rts()
370 if (!qp->term_info.valid) { in siw_init_terminate()
371 memset(&qp->term_info, 0, sizeof(qp->term_info)); in siw_init_terminate()
372 qp->term_info.layer = layer; in siw_init_terminate()
373 qp->term_info.etype = etype; in siw_init_terminate()
374 qp->term_info.ecode = ecode; in siw_init_terminate()
375 qp->term_info.in_tx = in_tx; in siw_init_terminate()
376 qp->term_info.valid = 1; in siw_init_terminate()
378 siw_dbg_qp(qp, "init TERM: layer %d, type %d, code %d, in tx %s\n", in siw_init_terminate()
384 * Sending TERMINATE messages is best effort - such messages
386 * not have another outbound message in-progress, i.e. the
394 struct iwarp_terminate *term = NULL; in siw_send_terminate() local
396 struct socket *s = qp->attrs.sk; in siw_send_terminate()
397 struct siw_rx_stream *srx = &qp->rx_stream; in siw_send_terminate()
398 union iwarp_hdr *rx_hdr = &srx->hdr; in siw_send_terminate()
402 if (!qp->term_info.valid) in siw_send_terminate()
405 qp->term_info.valid = 0; in siw_send_terminate()
407 if (tx_wqe(qp)->wr_status == SIW_WR_INPROGRESS) { in siw_send_terminate()
412 if (!s && qp->cep) in siw_send_terminate()
414 s = qp->cep->sock; in siw_send_terminate()
421 term = kzalloc(sizeof(*term), GFP_KERNEL); in siw_send_terminate()
422 if (!term) in siw_send_terminate()
425 term->ddp_qn = cpu_to_be32(RDMAP_UNTAGGED_QN_TERMINATE); in siw_send_terminate()
426 term->ddp_mo = 0; in siw_send_terminate()
427 term->ddp_msn = cpu_to_be32(1); in siw_send_terminate()
429 iov[0].iov_base = term; in siw_send_terminate()
430 iov[0].iov_len = sizeof(*term); in siw_send_terminate()
432 if ((qp->term_info.layer == TERM_ERROR_LAYER_DDP) || in siw_send_terminate()
433 ((qp->term_info.layer == TERM_ERROR_LAYER_RDMAP) && in siw_send_terminate()
434 (qp->term_info.etype != RDMAP_ETYPE_CATASTROPHIC))) { in siw_send_terminate()
437 kfree(term); in siw_send_terminate()
441 memcpy(&term->ctrl, &iwarp_pktinfo[RDMAP_TERMINATE].ctrl, in siw_send_terminate()
444 __rdmap_term_set_layer(term, qp->term_info.layer); in siw_send_terminate()
445 __rdmap_term_set_etype(term, qp->term_info.etype); in siw_send_terminate()
446 __rdmap_term_set_ecode(term, qp->term_info.ecode); in siw_send_terminate()
448 switch (qp->term_info.layer) { in siw_send_terminate()
450 if (qp->term_info.etype == RDMAP_ETYPE_CATASTROPHIC) in siw_send_terminate()
454 if (qp->term_info.etype == RDMAP_ETYPE_REMOTE_PROTECTION) { in siw_send_terminate()
459 term->flag_m = 1; in siw_send_terminate()
460 term->flag_d = 1; in siw_send_terminate()
461 term->flag_r = 1; in siw_send_terminate()
463 if (qp->term_info.in_tx) { in siw_send_terminate()
474 memcpy(&rreq->ctrl, in siw_send_terminate()
478 rreq->rsvd = 0; in siw_send_terminate()
479 rreq->ddp_qn = in siw_send_terminate()
483 rreq->ddp_msn = htonl(wqe->sqe.sge[0].length); in siw_send_terminate()
485 rreq->ddp_mo = htonl(wqe->processed); in siw_send_terminate()
486 rreq->sink_stag = htonl(wqe->sqe.rkey); in siw_send_terminate()
487 rreq->sink_to = cpu_to_be64(wqe->sqe.raddr); in siw_send_terminate()
488 rreq->read_size = htonl(wqe->sqe.sge[0].length); in siw_send_terminate()
489 rreq->source_stag = htonl(wqe->sqe.sge[0].lkey); in siw_send_terminate()
490 rreq->source_to = in siw_send_terminate()
491 cpu_to_be64(wqe->sqe.sge[0].laddr); in siw_send_terminate()
503 if (__rdmap_get_opcode(&rx_hdr->ctrl) == in siw_send_terminate()
515 if ((qp->term_info.ecode == RDMAP_ECODE_VERSION) || in siw_send_terminate()
516 (qp->term_info.ecode == RDMAP_ECODE_OPCODE)) in siw_send_terminate()
522 if (rx_hdr->ctrl.ddp_rdmap_ctrl & DDP_FLAG_TAGGED) in siw_send_terminate()
528 term->flag_m = 1; in siw_send_terminate()
529 term->flag_d = 1; in siw_send_terminate()
531 term->ctrl.mpa_len = cpu_to_be16(iov[1].iov_len); in siw_send_terminate()
543 if (((qp->term_info.etype == DDP_ETYPE_TAGGED_BUF) && in siw_send_terminate()
544 (qp->term_info.ecode == DDP_ECODE_T_VERSION)) || in siw_send_terminate()
545 ((qp->term_info.etype == DDP_ETYPE_UNTAGGED_BUF) && in siw_send_terminate()
546 (qp->term_info.ecode == DDP_ECODE_UT_VERSION))) in siw_send_terminate()
551 if (rx_hdr->ctrl.ddp_rdmap_ctrl & DDP_FLAG_TAGGED) in siw_send_terminate()
556 term->flag_m = 1; in siw_send_terminate()
557 term->flag_d = 1; in siw_send_terminate()
563 if (term->flag_m || term->flag_d || term->flag_r) { in siw_send_terminate()
566 len_terminate = sizeof(*term) + iov[1].iov_len + MPA_CRC_SIZE; in siw_send_terminate()
571 len_terminate = sizeof(*term) + MPA_CRC_SIZE; in siw_send_terminate()
576 if (term->flag_m) { in siw_send_terminate()
577 u32 real_ddp_len = be16_to_cpu(rx_hdr->ctrl.mpa_len); in siw_send_terminate()
578 enum rdma_opcode op = __rdmap_get_opcode(&rx_hdr->ctrl); in siw_send_terminate()
580 real_ddp_len -= iwarp_pktinfo[op].hdr_len - MPA_HDR_SIZE; in siw_send_terminate()
581 rx_hdr->ctrl.mpa_len = cpu_to_be16(real_ddp_len); in siw_send_terminate()
584 term->ctrl.mpa_len = in siw_send_terminate()
585 cpu_to_be16(len_terminate - (MPA_HDR_SIZE + MPA_CRC_SIZE)); in siw_send_terminate()
586 if (qp->tx_ctx.mpa_crc_hd) { in siw_send_terminate()
587 crypto_shash_init(qp->tx_ctx.mpa_crc_hd); in siw_send_terminate()
588 if (crypto_shash_update(qp->tx_ctx.mpa_crc_hd, in siw_send_terminate()
594 if (crypto_shash_update(qp->tx_ctx.mpa_crc_hd, in siw_send_terminate()
599 crypto_shash_final(qp->tx_ctx.mpa_crc_hd, (u8 *)&crc); in siw_send_terminate()
603 siw_dbg_qp(qp, "sent TERM: %s, layer %d, type %d, code %d (%d bytes)\n", in siw_send_terminate()
605 __rdmap_term_layer(term), __rdmap_term_etype(term), in siw_send_terminate()
606 __rdmap_term_ecode(term), rv); in siw_send_terminate()
608 kfree(term); in siw_send_terminate()
620 if (attrs->flags & SIW_RDMA_BIND_ENABLED) in siw_qp_modify_nonstate()
621 qp->attrs.flags |= SIW_RDMA_BIND_ENABLED; in siw_qp_modify_nonstate()
623 qp->attrs.flags &= ~SIW_RDMA_BIND_ENABLED; in siw_qp_modify_nonstate()
625 if (attrs->flags & SIW_RDMA_WRITE_ENABLED) in siw_qp_modify_nonstate()
626 qp->attrs.flags |= SIW_RDMA_WRITE_ENABLED; in siw_qp_modify_nonstate()
628 qp->attrs.flags &= ~SIW_RDMA_WRITE_ENABLED; in siw_qp_modify_nonstate()
630 if (attrs->flags & SIW_RDMA_READ_ENABLED) in siw_qp_modify_nonstate()
631 qp->attrs.flags |= SIW_RDMA_READ_ENABLED; in siw_qp_modify_nonstate()
633 qp->attrs.flags &= ~SIW_RDMA_READ_ENABLED; in siw_qp_modify_nonstate()
643 switch (attrs->state) { in siw_qp_nextstate_from_idle()
645 if (attrs->flags & SIW_MPA_CRC) { in siw_qp_nextstate_from_idle()
652 rv = -EINVAL; in siw_qp_nextstate_from_idle()
657 rv = -EINVAL; in siw_qp_nextstate_from_idle()
663 qp->tx_ctx.ddp_msn[RDMAP_UNTAGGED_QN_SEND] = 0; in siw_qp_nextstate_from_idle()
664 qp->tx_ctx.ddp_msn[RDMAP_UNTAGGED_QN_RDMA_READ] = 0; in siw_qp_nextstate_from_idle()
665 qp->tx_ctx.ddp_msn[RDMAP_UNTAGGED_QN_TERMINATE] = 0; in siw_qp_nextstate_from_idle()
670 qp->rx_stream.ddp_msn[RDMAP_UNTAGGED_QN_SEND] = 1; in siw_qp_nextstate_from_idle()
671 qp->rx_stream.ddp_msn[RDMAP_UNTAGGED_QN_RDMA_READ] = 1; in siw_qp_nextstate_from_idle()
672 qp->rx_stream.ddp_msn[RDMAP_UNTAGGED_QN_TERMINATE] = 1; in siw_qp_nextstate_from_idle()
678 rv = siw_qp_readq_init(qp, attrs->irq_size, in siw_qp_nextstate_from_idle()
679 attrs->orq_size); in siw_qp_nextstate_from_idle()
683 qp->attrs.sk = attrs->sk; in siw_qp_nextstate_from_idle()
684 qp->attrs.state = SIW_QP_STATE_RTS; in siw_qp_nextstate_from_idle()
687 attrs->flags & SIW_MPA_CRC ? "y" : "n", in siw_qp_nextstate_from_idle()
688 qp->attrs.orq_size, qp->attrs.irq_size); in siw_qp_nextstate_from_idle()
693 qp->attrs.state = SIW_QP_STATE_ERROR; in siw_qp_nextstate_from_idle()
694 if (qp->cep) { in siw_qp_nextstate_from_idle()
695 siw_cep_put(qp->cep); in siw_qp_nextstate_from_idle()
696 qp->cep = NULL; in siw_qp_nextstate_from_idle()
711 switch (attrs->state) { in siw_qp_nextstate_from_rts()
721 if (tx_wqe(qp)->wr_status == SIW_WR_IDLE) { in siw_qp_nextstate_from_rts()
722 qp->attrs.state = SIW_QP_STATE_CLOSING; in siw_qp_nextstate_from_rts()
724 qp->attrs.state = SIW_QP_STATE_ERROR; in siw_qp_nextstate_from_rts()
733 qp->attrs.state = SIW_QP_STATE_TERMINATE; in siw_qp_nextstate_from_rts()
751 * Esp., how to handle the non-empty IRQ case? in siw_qp_nextstate_from_rts()
757 qp->attrs.state = SIW_QP_STATE_ERROR; in siw_qp_nextstate_from_rts()
770 switch (attrs->state) { in siw_qp_nextstate_from_term()
773 qp->attrs.state = SIW_QP_STATE_ERROR; in siw_qp_nextstate_from_term()
775 if (tx_wqe(qp)->wr_status != SIW_WR_IDLE) in siw_qp_nextstate_from_term()
789 switch (attrs->state) { in siw_qp_nextstate_from_close()
791 WARN_ON(tx_wqe(qp)->wr_status != SIW_WR_IDLE); in siw_qp_nextstate_from_close()
792 qp->attrs.state = SIW_QP_STATE_IDLE; in siw_qp_nextstate_from_close()
807 qp->attrs.state = SIW_QP_STATE_ERROR; in siw_qp_nextstate_from_close()
809 if (tx_wqe(qp)->wr_status != SIW_WR_IDLE) in siw_qp_nextstate_from_close()
817 siw_qp_state_to_string[qp->attrs.state], in siw_qp_nextstate_from_close()
818 siw_qp_state_to_string[attrs->state]); in siw_qp_nextstate_from_close()
820 rv = -ECONNABORTED; in siw_qp_nextstate_from_close()
826 * Caller must hold qp->state_lock
837 siw_qp_state_to_string[qp->attrs.state], in siw_qp_modify()
838 siw_qp_state_to_string[attrs->state]); in siw_qp_modify()
846 switch (qp->attrs.state) { in siw_qp_modify()
874 rreq->id = sqe->id; in siw_read_to_orq()
875 rreq->opcode = sqe->opcode; in siw_read_to_orq()
876 rreq->sge[0].laddr = sqe->sge[0].laddr; in siw_read_to_orq()
877 rreq->sge[0].length = sqe->sge[0].length; in siw_read_to_orq()
878 rreq->sge[0].lkey = sqe->sge[0].lkey; in siw_read_to_orq()
879 rreq->sge[1].lkey = sqe->sge[1].lkey; in siw_read_to_orq()
880 rreq->flags = sqe->flags | SIW_WQE_VALID; in siw_read_to_orq()
881 rreq->num_sge = 1; in siw_read_to_orq()
894 memset(wqe->mem, 0, sizeof(*wqe->mem) * SIW_MAX_SGE); in siw_activate_tx_from_sq()
895 wqe->wr_status = SIW_WR_QUEUED; in siw_activate_tx_from_sq()
898 memcpy(&wqe->sqe, sqe, sizeof(*sqe)); in siw_activate_tx_from_sq()
900 if (wqe->sqe.opcode >= SIW_NUM_OPCODES) { in siw_activate_tx_from_sq()
901 rv = -EINVAL; in siw_activate_tx_from_sq()
904 if (wqe->sqe.flags & SIW_WQE_INLINE) { in siw_activate_tx_from_sq()
905 if (wqe->sqe.opcode != SIW_OP_SEND && in siw_activate_tx_from_sq()
906 wqe->sqe.opcode != SIW_OP_WRITE) { in siw_activate_tx_from_sq()
907 rv = -EINVAL; in siw_activate_tx_from_sq()
910 if (wqe->sqe.sge[0].length > SIW_MAX_INLINE) { in siw_activate_tx_from_sq()
911 rv = -EINVAL; in siw_activate_tx_from_sq()
914 wqe->sqe.sge[0].laddr = (uintptr_t)&wqe->sqe.sge[1]; in siw_activate_tx_from_sq()
915 wqe->sqe.sge[0].lkey = 0; in siw_activate_tx_from_sq()
916 wqe->sqe.num_sge = 1; in siw_activate_tx_from_sq()
918 if (wqe->sqe.flags & SIW_WQE_READ_FENCE) { in siw_activate_tx_from_sq()
920 if (unlikely(wqe->sqe.opcode == SIW_OP_READ || in siw_activate_tx_from_sq()
921 wqe->sqe.opcode == in siw_activate_tx_from_sq()
924 rv = -EINVAL; in siw_activate_tx_from_sq()
927 spin_lock(&qp->orq_lock); in siw_activate_tx_from_sq()
929 if (qp->attrs.orq_size && !siw_orq_empty(qp)) { in siw_activate_tx_from_sq()
930 qp->tx_ctx.orq_fence = 1; in siw_activate_tx_from_sq()
933 spin_unlock(&qp->orq_lock); in siw_activate_tx_from_sq()
935 } else if (wqe->sqe.opcode == SIW_OP_READ || in siw_activate_tx_from_sq()
936 wqe->sqe.opcode == SIW_OP_READ_LOCAL_INV) { in siw_activate_tx_from_sq()
939 if (unlikely(!qp->attrs.orq_size)) { in siw_activate_tx_from_sq()
941 rv = -EINVAL; in siw_activate_tx_from_sq()
944 wqe->sqe.num_sge = 1; in siw_activate_tx_from_sq()
946 spin_lock(&qp->orq_lock); in siw_activate_tx_from_sq()
954 siw_read_to_orq(rreq, &wqe->sqe); in siw_activate_tx_from_sq()
955 qp->orq_put++; in siw_activate_tx_from_sq()
957 qp->tx_ctx.orq_fence = 1; in siw_activate_tx_from_sq()
960 spin_unlock(&qp->orq_lock); in siw_activate_tx_from_sq()
963 /* Clear SQE, can be re-used by application */ in siw_activate_tx_from_sq()
964 smp_store_mb(sqe->flags, 0); in siw_activate_tx_from_sq()
965 qp->sq_get++; in siw_activate_tx_from_sq()
969 wqe->wr_status = SIW_WR_IDLE; in siw_activate_tx_from_sq()
977 * the active IRQ will not be served after qp->irq_burst, if the
985 if (!qp->attrs.irq_size) in siw_activate_tx()
988 irqe = &qp->irq[qp->irq_get % qp->attrs.irq_size]; in siw_activate_tx()
990 if (!(irqe->flags & SIW_WQE_VALID)) in siw_activate_tx()
997 if (sq_get_next(qp) && ++qp->irq_burst >= SIW_IRQ_MAXBURST_SQ_ACTIVE) { in siw_activate_tx()
998 qp->irq_burst = 0; in siw_activate_tx()
1001 memset(wqe->mem, 0, sizeof(*wqe->mem) * SIW_MAX_SGE); in siw_activate_tx()
1002 wqe->wr_status = SIW_WR_QUEUED; in siw_activate_tx()
1005 wqe->sqe.opcode = SIW_OP_READ_RESPONSE; in siw_activate_tx()
1006 wqe->sqe.flags = 0; in siw_activate_tx()
1007 if (irqe->num_sge) { in siw_activate_tx()
1008 wqe->sqe.num_sge = 1; in siw_activate_tx()
1009 wqe->sqe.sge[0].length = irqe->sge[0].length; in siw_activate_tx()
1010 wqe->sqe.sge[0].laddr = irqe->sge[0].laddr; in siw_activate_tx()
1011 wqe->sqe.sge[0].lkey = irqe->sge[0].lkey; in siw_activate_tx()
1013 wqe->sqe.num_sge = 0; in siw_activate_tx()
1019 wqe->sqe.sge[1].length = irqe->sge[1].length; in siw_activate_tx()
1021 wqe->sqe.rkey = irqe->rkey; in siw_activate_tx()
1022 wqe->sqe.raddr = irqe->raddr; in siw_activate_tx()
1024 wqe->processed = 0; in siw_activate_tx()
1025 qp->irq_get++; in siw_activate_tx()
1028 smp_store_mb(irqe->flags, 0); in siw_activate_tx()
1041 if (!cq->base_cq.comp_handler) in siw_cq_notify_now()
1045 cq_notify = READ_ONCE(cq->notify->flags); in siw_cq_notify_now()
1051 * CQ notification is one-shot: Since the in siw_cq_notify_now()
1053 * the CQ gets dis-aremd and must be re-aremd in siw_cq_notify_now()
1056 WRITE_ONCE(cq->notify->flags, SIW_NOTIFY_NOT); in siw_cq_notify_now()
1066 struct siw_cq *cq = qp->scq; in siw_sqe_complete()
1070 u32 sqe_flags = sqe->flags; in siw_sqe_complete()
1073 unsigned long flags; in siw_sqe_complete()
1075 spin_lock_irqsave(&cq->lock, flags); in siw_sqe_complete()
1077 idx = cq->cq_put % cq->num_cqe; in siw_sqe_complete()
1078 cqe = &cq->queue[idx]; in siw_sqe_complete()
1080 if (!READ_ONCE(cqe->flags)) { in siw_sqe_complete()
1083 cqe->id = sqe->id; in siw_sqe_complete()
1084 cqe->opcode = sqe->opcode; in siw_sqe_complete()
1085 cqe->status = status; in siw_sqe_complete()
1086 cqe->imm_data = 0; in siw_sqe_complete()
1087 cqe->bytes = bytes; in siw_sqe_complete()
1089 if (rdma_is_kernel_res(&cq->base_cq.res)) in siw_sqe_complete()
1090 cqe->base_qp = &qp->base_qp; in siw_sqe_complete()
1092 cqe->qp_id = qp_id(qp); in siw_sqe_complete()
1095 WRITE_ONCE(cqe->flags, SIW_WQE_VALID); in siw_sqe_complete()
1097 smp_store_mb(sqe->flags, 0); in siw_sqe_complete()
1099 cq->cq_put++; in siw_sqe_complete()
1102 spin_unlock_irqrestore(&cq->lock, flags); in siw_sqe_complete()
1106 cq->base_cq.comp_handler(&cq->base_cq, in siw_sqe_complete()
1107 cq->base_cq.cq_context); in siw_sqe_complete()
1110 spin_unlock_irqrestore(&cq->lock, flags); in siw_sqe_complete()
1111 rv = -ENOMEM; in siw_sqe_complete()
1116 smp_store_mb(sqe->flags, 0); in siw_sqe_complete()
1124 struct siw_cq *cq = qp->rcq; in siw_rqe_complete()
1130 unsigned long flags; in siw_rqe_complete()
1132 spin_lock_irqsave(&cq->lock, flags); in siw_rqe_complete()
1134 idx = cq->cq_put % cq->num_cqe; in siw_rqe_complete()
1135 cqe = &cq->queue[idx]; in siw_rqe_complete()
1137 if (!READ_ONCE(cqe->flags)) { in siw_rqe_complete()
1141 cqe->id = rqe->id; in siw_rqe_complete()
1142 cqe->opcode = SIW_OP_RECEIVE; in siw_rqe_complete()
1143 cqe->status = status; in siw_rqe_complete()
1144 cqe->imm_data = 0; in siw_rqe_complete()
1145 cqe->bytes = bytes; in siw_rqe_complete()
1147 if (rdma_is_kernel_res(&cq->base_cq.res)) { in siw_rqe_complete()
1148 cqe->base_qp = &qp->base_qp; in siw_rqe_complete()
1151 cqe->inval_stag = inval_stag; in siw_rqe_complete()
1154 cqe->qp_id = qp_id(qp); in siw_rqe_complete()
1157 WRITE_ONCE(cqe->flags, cqe_flags); in siw_rqe_complete()
1159 smp_store_mb(rqe->flags, 0); in siw_rqe_complete()
1161 cq->cq_put++; in siw_rqe_complete()
1164 spin_unlock_irqrestore(&cq->lock, flags); in siw_rqe_complete()
1168 cq->base_cq.comp_handler(&cq->base_cq, in siw_rqe_complete()
1169 cq->base_cq.cq_context); in siw_rqe_complete()
1172 spin_unlock_irqrestore(&cq->lock, flags); in siw_rqe_complete()
1173 rv = -ENOMEM; in siw_rqe_complete()
1178 smp_store_mb(rqe->flags, 0); in siw_rqe_complete()
1200 while (qp->attrs.orq_size) { in siw_sq_flush()
1201 sqe = &qp->orq[qp->orq_get % qp->attrs.orq_size]; in siw_sq_flush()
1202 if (!READ_ONCE(sqe->flags)) in siw_sq_flush()
1208 WRITE_ONCE(sqe->flags, 0); in siw_sq_flush()
1209 qp->orq_get++; in siw_sq_flush()
1212 * Flush an in-progress WQE if present in siw_sq_flush()
1214 if (wqe->wr_status != SIW_WR_IDLE) { in siw_sq_flush()
1216 tx_type(wqe), wqe->wr_status); in siw_sq_flush()
1223 wqe->wr_status == SIW_WR_QUEUED)) in siw_sq_flush()
1225 * An in-progress Read Request is already in in siw_sq_flush()
1228 siw_sqe_complete(qp, &wqe->sqe, wqe->bytes, in siw_sq_flush()
1231 wqe->wr_status = SIW_WR_IDLE; in siw_sq_flush()
1236 while (qp->attrs.sq_size) { in siw_sq_flush()
1237 sqe = &qp->sendq[qp->sq_get % qp->attrs.sq_size]; in siw_sq_flush()
1238 if (!READ_ONCE(sqe->flags)) in siw_sq_flush()
1249 WRITE_ONCE(sqe->flags, 0); in siw_sq_flush()
1250 qp->sq_get++; in siw_sq_flush()
1269 struct siw_wqe *wqe = &qp->rx_untagged.wqe_active; in siw_rq_flush()
1272 * Flush an in-progress untagged operation if present in siw_rq_flush()
1274 if (wqe->wr_status != SIW_WR_IDLE) { in siw_rq_flush()
1276 rx_type(wqe), wqe->wr_status); in siw_rq_flush()
1281 siw_rqe_complete(qp, &wqe->rqe, wqe->bytes, in siw_rq_flush()
1286 siw_sqe_complete(qp, &wqe->sqe, 0, SIW_WC_WR_FLUSH_ERR); in siw_rq_flush()
1288 wqe->wr_status = SIW_WR_IDLE; in siw_rq_flush()
1290 wqe = &qp->rx_tagged.wqe_active; in siw_rq_flush()
1292 if (wqe->wr_status != SIW_WR_IDLE) { in siw_rq_flush()
1294 wqe->wr_status = SIW_WR_IDLE; in siw_rq_flush()
1299 while (qp->attrs.rq_size) { in siw_rq_flush()
1301 &qp->recvq[qp->rq_get % qp->attrs.rq_size]; in siw_rq_flush()
1303 if (!READ_ONCE(rqe->flags)) in siw_rq_flush()
1309 WRITE_ONCE(rqe->flags, 0); in siw_rq_flush()
1310 qp->rq_get++; in siw_rq_flush()
1316 int rv = xa_alloc(&sdev->qp_xa, &qp->base_qp.qp_num, qp, xa_limit_32b, in siw_qp_add()
1320 kref_init(&qp->ref); in siw_qp_add()
1321 qp->sdev = sdev; in siw_qp_add()
1330 struct siw_device *sdev = qp->sdev; in siw_free_qp()
1331 unsigned long flags; in siw_free_qp()
1333 if (qp->cep) in siw_free_qp()
1334 siw_cep_put(qp->cep); in siw_free_qp()
1336 found = xa_erase(&sdev->qp_xa, qp_id(qp)); in siw_free_qp()
1338 spin_lock_irqsave(&sdev->lock, flags); in siw_free_qp()
1339 list_del(&qp->devq); in siw_free_qp()
1340 spin_unlock_irqrestore(&sdev->lock, flags); in siw_free_qp()
1342 vfree(qp->sendq); in siw_free_qp()
1343 vfree(qp->recvq); in siw_free_qp()
1344 vfree(qp->irq); in siw_free_qp()
1345 vfree(qp->orq); in siw_free_qp()
1347 siw_put_tx_cpu(qp->tx_cpu); in siw_free_qp()
1348 complete(&qp->qp_free); in siw_free_qp()
1349 atomic_dec(&sdev->num_qp); in siw_free_qp()