Lines Matching refs:tx_ring
31 static int nfp_nfd3_tx_ring_should_wake(struct nfp_net_tx_ring *tx_ring) in nfp_nfd3_tx_ring_should_wake() argument
33 return !nfp_net_tx_full(tx_ring, MAX_SKB_FRAGS * 4); in nfp_nfd3_tx_ring_should_wake()
36 static int nfp_nfd3_tx_ring_should_stop(struct nfp_net_tx_ring *tx_ring) in nfp_nfd3_tx_ring_should_stop() argument
38 return nfp_net_tx_full(tx_ring, MAX_SKB_FRAGS + 1); in nfp_nfd3_tx_ring_should_stop()
52 struct nfp_net_tx_ring *tx_ring) in nfp_nfd3_tx_ring_stop() argument
58 if (unlikely(nfp_nfd3_tx_ring_should_wake(tx_ring))) in nfp_nfd3_tx_ring_stop()
259 struct nfp_net_tx_ring *tx_ring; in nfp_nfd3_tx() local
274 tx_ring = &dp->tx_rings[qidx]; in nfp_nfd3_tx()
275 r_vec = tx_ring->r_vec; in nfp_nfd3_tx()
279 if (unlikely(nfp_net_tx_full(tx_ring, nr_frags + 1))) { in nfp_nfd3_tx()
281 qidx, tx_ring->wr_p, tx_ring->rd_p); in nfp_nfd3_tx()
284 nfp_net_tx_xmit_more_flush(tx_ring); in nfp_nfd3_tx()
293 nfp_net_tx_xmit_more_flush(tx_ring); in nfp_nfd3_tx()
307 wr_idx = D_IDX(tx_ring, tx_ring->wr_p); in nfp_nfd3_tx()
310 txbuf = &tx_ring->txbufs[wr_idx]; in nfp_nfd3_tx()
318 txd = &tx_ring->txds[wr_idx]; in nfp_nfd3_tx()
355 wr_idx = D_IDX(tx_ring, wr_idx + 1); in nfp_nfd3_tx()
356 tx_ring->txbufs[wr_idx].skb = skb; in nfp_nfd3_tx()
357 tx_ring->txbufs[wr_idx].dma_addr = dma_addr; in nfp_nfd3_tx()
358 tx_ring->txbufs[wr_idx].fidx = f; in nfp_nfd3_tx()
360 txd = &tx_ring->txds[wr_idx]; in nfp_nfd3_tx()
375 nd_q = netdev_get_tx_queue(dp->netdev, tx_ring->idx); in nfp_nfd3_tx()
377 tx_ring->wr_p += nr_frags + 1; in nfp_nfd3_tx()
378 if (nfp_nfd3_tx_ring_should_stop(tx_ring)) in nfp_nfd3_tx()
379 nfp_nfd3_tx_ring_stop(nd_q, tx_ring); in nfp_nfd3_tx()
381 tx_ring->wr_ptr_add += nr_frags + 1; in nfp_nfd3_tx()
383 nfp_net_tx_xmit_more_flush(tx_ring); in nfp_nfd3_tx()
390 dma_unmap_page(dp->dev, tx_ring->txbufs[wr_idx].dma_addr, in nfp_nfd3_tx()
392 tx_ring->txbufs[wr_idx].skb = NULL; in nfp_nfd3_tx()
393 tx_ring->txbufs[wr_idx].dma_addr = 0; in nfp_nfd3_tx()
394 tx_ring->txbufs[wr_idx].fidx = -2; in nfp_nfd3_tx()
397 wr_idx += tx_ring->cnt; in nfp_nfd3_tx()
399 dma_unmap_single(dp->dev, tx_ring->txbufs[wr_idx].dma_addr, in nfp_nfd3_tx()
401 tx_ring->txbufs[wr_idx].skb = NULL; in nfp_nfd3_tx()
402 tx_ring->txbufs[wr_idx].dma_addr = 0; in nfp_nfd3_tx()
403 tx_ring->txbufs[wr_idx].fidx = -2; in nfp_nfd3_tx()
407 nfp_net_tx_xmit_more_flush(tx_ring); in nfp_nfd3_tx()
421 void nfp_nfd3_tx_complete(struct nfp_net_tx_ring *tx_ring, int budget) in nfp_nfd3_tx_complete() argument
423 struct nfp_net_r_vector *r_vec = tx_ring->r_vec; in nfp_nfd3_tx_complete()
430 if (tx_ring->wr_p == tx_ring->rd_p) in nfp_nfd3_tx_complete()
434 qcp_rd_p = nfp_net_read_tx_cmpl(tx_ring, dp); in nfp_nfd3_tx_complete()
436 if (qcp_rd_p == tx_ring->qcp_rd_p) in nfp_nfd3_tx_complete()
439 todo = D_IDX(tx_ring, qcp_rd_p - tx_ring->qcp_rd_p); in nfp_nfd3_tx_complete()
448 idx = D_IDX(tx_ring, tx_ring->rd_p++); in nfp_nfd3_tx_complete()
449 tx_buf = &tx_ring->txbufs[idx]; in nfp_nfd3_tx_complete()
481 tx_ring->qcp_rd_p = qcp_rd_p; in nfp_nfd3_tx_complete()
491 nd_q = netdev_get_tx_queue(dp->netdev, tx_ring->idx); in nfp_nfd3_tx_complete()
493 if (nfp_nfd3_tx_ring_should_wake(tx_ring)) { in nfp_nfd3_tx_complete()
501 WARN_ONCE(tx_ring->wr_p - tx_ring->rd_p > tx_ring->cnt, in nfp_nfd3_tx_complete()
503 tx_ring->rd_p, tx_ring->wr_p, tx_ring->cnt); in nfp_nfd3_tx_complete()
506 static bool nfp_nfd3_xdp_complete(struct nfp_net_tx_ring *tx_ring) in nfp_nfd3_xdp_complete() argument
508 struct nfp_net_r_vector *r_vec = tx_ring->r_vec; in nfp_nfd3_xdp_complete()
516 qcp_rd_p = nfp_net_read_tx_cmpl(tx_ring, dp); in nfp_nfd3_xdp_complete()
518 if (qcp_rd_p == tx_ring->qcp_rd_p) in nfp_nfd3_xdp_complete()
521 todo = D_IDX(tx_ring, qcp_rd_p - tx_ring->qcp_rd_p); in nfp_nfd3_xdp_complete()
526 tx_ring->qcp_rd_p = D_IDX(tx_ring, tx_ring->qcp_rd_p + todo); in nfp_nfd3_xdp_complete()
530 idx = D_IDX(tx_ring, tx_ring->rd_p); in nfp_nfd3_xdp_complete()
531 tx_ring->rd_p++; in nfp_nfd3_xdp_complete()
533 done_bytes += tx_ring->txbufs[idx].real_len; in nfp_nfd3_xdp_complete()
541 WARN_ONCE(tx_ring->wr_p - tx_ring->rd_p > tx_ring->cnt, in nfp_nfd3_xdp_complete()
543 tx_ring->rd_p, tx_ring->wr_p, tx_ring->cnt); in nfp_nfd3_xdp_complete()
837 struct nfp_net_tx_ring *tx_ring, in nfp_nfd3_tx_xdp_buf() argument
850 if (unlikely(nfp_net_tx_full(tx_ring, 1))) { in nfp_nfd3_tx_xdp_buf()
852 nfp_nfd3_xdp_complete(tx_ring); in nfp_nfd3_tx_xdp_buf()
856 if (unlikely(nfp_net_tx_full(tx_ring, 1))) { in nfp_nfd3_tx_xdp_buf()
863 wr_idx = D_IDX(tx_ring, tx_ring->wr_p); in nfp_nfd3_tx_xdp_buf()
866 txbuf = &tx_ring->txbufs[wr_idx]; in nfp_nfd3_tx_xdp_buf()
880 txd = &tx_ring->txds[wr_idx]; in nfp_nfd3_tx_xdp_buf()
890 tx_ring->wr_p++; in nfp_nfd3_tx_xdp_buf()
891 tx_ring->wr_ptr_add++; in nfp_nfd3_tx_xdp_buf()
910 struct nfp_net_tx_ring *tx_ring; in nfp_nfd3_rx() local
922 tx_ring = r_vec->xdp_ring; in nfp_nfd3_rx()
1029 tx_ring, in nfp_nfd3_rx()
1137 if (tx_ring->wr_ptr_add) in nfp_nfd3_rx()
1138 nfp_net_tx_xmit_more_flush(tx_ring); in nfp_nfd3_rx()
1139 else if (unlikely(tx_ring->wr_p != tx_ring->rd_p) && in nfp_nfd3_rx()
1141 if (!nfp_nfd3_xdp_complete(tx_ring)) in nfp_nfd3_rx()
1161 if (r_vec->tx_ring) in nfp_nfd3_poll()
1162 nfp_nfd3_tx_complete(r_vec->tx_ring, budget); in nfp_nfd3_poll()
1185 if (r_vec->nfp_net->tx_coalesce_adapt_on && r_vec->tx_ring) { in nfp_nfd3_poll()
1211 struct nfp_net_tx_ring *tx_ring; in nfp_nfd3_ctrl_tx_one() local
1219 tx_ring = r_vec->tx_ring; in nfp_nfd3_ctrl_tx_one()
1226 if (unlikely(nfp_net_tx_full(tx_ring, 1))) { in nfp_nfd3_ctrl_tx_one()
1253 wr_idx = D_IDX(tx_ring, tx_ring->wr_p); in nfp_nfd3_ctrl_tx_one()
1256 txbuf = &tx_ring->txbufs[wr_idx]; in nfp_nfd3_ctrl_tx_one()
1264 txd = &tx_ring->txds[wr_idx]; in nfp_nfd3_ctrl_tx_one()
1274 tx_ring->wr_p++; in nfp_nfd3_ctrl_tx_one()
1275 tx_ring->wr_ptr_add++; in nfp_nfd3_ctrl_tx_one()
1276 nfp_net_tx_xmit_more_flush(tx_ring); in nfp_nfd3_ctrl_tx_one()
1410 nfp_nfd3_tx_complete(r_vec->tx_ring, 0); in nfp_nfd3_ctrl_poll()