Lines Matching refs:tx_ring
49 return hfi1_ipoib_txreqs(txq->tx_ring.sent_txreqs, in hfi1_ipoib_used()
50 txq->tx_ring.complete_txreqs); in hfi1_ipoib_used()
56 if (atomic_inc_return(&txq->tx_ring.stops) == 1) in hfi1_ipoib_stop_txq()
63 if (atomic_dec_and_test(&txq->tx_ring.stops)) in hfi1_ipoib_wake_txq()
70 txq->tx_ring.max_items - 1); in hfi1_ipoib_ring_hwat()
76 txq->tx_ring.max_items) >> 1; in hfi1_ipoib_ring_lwat()
81 ++txq->tx_ring.sent_txreqs; in hfi1_ipoib_check_queue_depth()
83 !atomic_xchg(&txq->tx_ring.ring_full, 1)) { in hfi1_ipoib_check_queue_depth()
108 atomic_xchg(&txq->tx_ring.ring_full, 0)) { in hfi1_ipoib_check_queue_stopped()
136 struct hfi1_ipoib_circ_buf *tx_ring = &txq->tx_ring; in hfi1_ipoib_drain_tx_ring() local
140 for (i = 0; i < tx_ring->max_items; i++) { in hfi1_ipoib_drain_tx_ring()
141 tx = hfi1_txreq_from_idx(tx_ring, i); in hfi1_ipoib_drain_tx_ring()
147 tx_ring->head = 0; in hfi1_ipoib_drain_tx_ring()
148 tx_ring->tail = 0; in hfi1_ipoib_drain_tx_ring()
149 tx_ring->complete_txreqs = 0; in hfi1_ipoib_drain_tx_ring()
150 tx_ring->sent_txreqs = 0; in hfi1_ipoib_drain_tx_ring()
151 tx_ring->avail = hfi1_ipoib_ring_hwat(txq); in hfi1_ipoib_drain_tx_ring()
158 struct hfi1_ipoib_circ_buf *tx_ring = &txq->tx_ring; in hfi1_ipoib_poll_tx_ring() local
159 u32 head = tx_ring->head; in hfi1_ipoib_poll_tx_ring()
160 u32 max_tx = tx_ring->max_items; in hfi1_ipoib_poll_tx_ring()
162 struct ipoib_txreq *tx = hfi1_txreq_from_idx(tx_ring, head); in hfi1_ipoib_poll_tx_ring()
173 tx = hfi1_txreq_from_idx(tx_ring, head); in hfi1_ipoib_poll_tx_ring()
175 tx_ring->complete_txreqs += work_done; in hfi1_ipoib_poll_tx_ring()
178 smp_store_release(&tx_ring->head, head); in hfi1_ipoib_poll_tx_ring()
330 ohdr->bth[2] = cpu_to_be32(mask_psn((u32)txp->txq->tx_ring.sent_txreqs)); in hfi1_ipoib_build_ib_tx_headers()
355 struct hfi1_ipoib_circ_buf *tx_ring = &txq->tx_ring; in hfi1_ipoib_send_dma_common() local
356 u32 tail = tx_ring->tail; in hfi1_ipoib_send_dma_common()
359 if (unlikely(!tx_ring->avail)) { in hfi1_ipoib_send_dma_common()
366 head = smp_load_acquire(&tx_ring->head); in hfi1_ipoib_send_dma_common()
367 tx_ring->avail = in hfi1_ipoib_send_dma_common()
369 CIRC_CNT(head, tail, tx_ring->max_items)); in hfi1_ipoib_send_dma_common()
371 tx_ring->avail--; in hfi1_ipoib_send_dma_common()
373 tx = hfi1_txreq_from_idx(tx_ring, tail); in hfi1_ipoib_send_dma_common()
461 struct hfi1_ipoib_circ_buf *tx_ring; in hfi1_ipoib_send_dma_single() local
479 tx_ring = &txq->tx_ring; in hfi1_ipoib_send_dma_single()
480 trace_hfi1_tx_consume(tx, tx_ring->tail); in hfi1_ipoib_send_dma_single()
482 smp_store_release(&tx_ring->tail, CIRC_NEXT(tx_ring->tail, tx_ring->max_items)); in hfi1_ipoib_send_dma_single()
512 struct hfi1_ipoib_circ_buf *tx_ring; in hfi1_ipoib_send_dma_list() local
542 tx_ring = &txq->tx_ring; in hfi1_ipoib_send_dma_list()
543 trace_hfi1_tx_consume(tx, tx_ring->tail); in hfi1_ipoib_send_dma_list()
545 smp_store_release(&tx_ring->tail, CIRC_NEXT(tx_ring->tail, tx_ring->max_items)); in hfi1_ipoib_send_dma_list()
636 if (!atomic_xchg(&txq->tx_ring.no_desc, 1)) { in hfi1_ipoib_sdma_sleep()
679 if (atomic_xchg(&txq->tx_ring.no_desc, 0)) in hfi1_ipoib_flush_txq()
687 struct hfi1_ipoib_circ_buf *tx_ring; in hfi1_ipoib_txreq_init() local
708 tx_ring = &txq->tx_ring; in hfi1_ipoib_txreq_init()
720 atomic_set(&txq->tx_ring.stops, 0); in hfi1_ipoib_txreq_init()
721 atomic_set(&txq->tx_ring.ring_full, 0); in hfi1_ipoib_txreq_init()
722 atomic_set(&txq->tx_ring.no_desc, 0); in hfi1_ipoib_txreq_init()
731 txq->tx_ring.items = in hfi1_ipoib_txreq_init()
734 if (!txq->tx_ring.items) in hfi1_ipoib_txreq_init()
737 txq->tx_ring.max_items = tx_ring_size; in hfi1_ipoib_txreq_init()
738 txq->tx_ring.shift = ilog2(tx_item_size); in hfi1_ipoib_txreq_init()
739 txq->tx_ring.avail = hfi1_ipoib_ring_hwat(txq); in hfi1_ipoib_txreq_init()
740 tx_ring = &txq->tx_ring; in hfi1_ipoib_txreq_init()
742 hfi1_txreq_from_idx(tx_ring, j)->sdma_hdr = in hfi1_ipoib_txreq_init()
745 if (!hfi1_txreq_from_idx(tx_ring, j)->sdma_hdr) in hfi1_ipoib_txreq_init()
759 tx_ring = &txq->tx_ring; in hfi1_ipoib_txreq_init()
761 kfree(hfi1_txreq_from_idx(tx_ring, j)->sdma_hdr); in hfi1_ipoib_txreq_init()
762 kvfree(tx_ring->items); in hfi1_ipoib_txreq_init()
783 txq->tx_ring.complete_txreqs++; in hfi1_ipoib_drain_tx_list()
790 hfi1_ipoib_txreqs(txq->tx_ring.sent_txreqs, in hfi1_ipoib_drain_tx_list()
791 txq->tx_ring.complete_txreqs)); in hfi1_ipoib_drain_tx_list()
800 struct hfi1_ipoib_circ_buf *tx_ring = &txq->tx_ring; in hfi1_ipoib_txreq_deinit() local
807 for (j = 0; j < tx_ring->max_items; j++) in hfi1_ipoib_txreq_deinit()
808 kfree(hfi1_txreq_from_idx(tx_ring, j)->sdma_hdr); in hfi1_ipoib_txreq_deinit()
809 kvfree(tx_ring->items); in hfi1_ipoib_txreq_deinit()
849 atomic_read(&txq->tx_ring.stops), in hfi1_ipoib_tx_timeout()
850 atomic_read(&txq->tx_ring.no_desc), in hfi1_ipoib_tx_timeout()
851 atomic_read(&txq->tx_ring.ring_full)); in hfi1_ipoib_tx_timeout()
857 txq->tx_ring.sent_txreqs, txq->tx_ring.complete_txreqs, in hfi1_ipoib_tx_timeout()
860 dev->tx_queue_len, txq->tx_ring.max_items); in hfi1_ipoib_tx_timeout()
862 txq->tx_ring.head, txq->tx_ring.tail); in hfi1_ipoib_tx_timeout()