Lines Matching refs:tx_ring
18 static int nfp_nfdk_tx_ring_should_wake(struct nfp_net_tx_ring *tx_ring) in nfp_nfdk_tx_ring_should_wake() argument
20 return !nfp_net_tx_full(tx_ring, NFDK_TX_DESC_STOP_CNT * 2); in nfp_nfdk_tx_ring_should_wake()
23 static int nfp_nfdk_tx_ring_should_stop(struct nfp_net_tx_ring *tx_ring) in nfp_nfdk_tx_ring_should_stop() argument
25 return nfp_net_tx_full(tx_ring, NFDK_TX_DESC_STOP_CNT); in nfp_nfdk_tx_ring_should_stop()
29 struct nfp_net_tx_ring *tx_ring) in nfp_nfdk_tx_ring_stop() argument
35 if (unlikely(nfp_nfdk_tx_ring_should_wake(tx_ring))) in nfp_nfdk_tx_ring_stop()
119 nfp_nfdk_tx_maybe_close_block(struct nfp_net_tx_ring *tx_ring, in nfp_nfdk_tx_maybe_close_block() argument
151 if (round_down(tx_ring->wr_p, NFDK_TX_DESC_BLOCK_CNT) != in nfp_nfdk_tx_maybe_close_block()
152 round_down(tx_ring->wr_p + n_descs, NFDK_TX_DESC_BLOCK_CNT)) in nfp_nfdk_tx_maybe_close_block()
155 if ((u32)tx_ring->data_pending + skb->len > NFDK_TX_MAX_DATA_PER_BLOCK) in nfp_nfdk_tx_maybe_close_block()
161 wr_p = tx_ring->wr_p; in nfp_nfdk_tx_maybe_close_block()
164 wr_idx = D_IDX(tx_ring, wr_p); in nfp_nfdk_tx_maybe_close_block()
165 tx_ring->ktxbufs[wr_idx].skb = NULL; in nfp_nfdk_tx_maybe_close_block()
166 txd = &tx_ring->ktxds[wr_idx]; in nfp_nfdk_tx_maybe_close_block()
170 tx_ring->data_pending = 0; in nfp_nfdk_tx_maybe_close_block()
171 tx_ring->wr_p += nop_slots; in nfp_nfdk_tx_maybe_close_block()
172 tx_ring->wr_ptr_add += nop_slots; in nfp_nfdk_tx_maybe_close_block()
258 struct nfp_net_tx_ring *tx_ring; in nfp_nfdk_tx() local
273 tx_ring = &dp->tx_rings[qidx]; in nfp_nfdk_tx()
274 r_vec = tx_ring->r_vec; in nfp_nfdk_tx()
278 if (unlikely(nfp_net_tx_full(tx_ring, NFDK_TX_DESC_STOP_CNT))) { in nfp_nfdk_tx()
280 qidx, tx_ring->wr_p, tx_ring->rd_p); in nfp_nfdk_tx()
282 nfp_net_tx_xmit_more_flush(tx_ring); in nfp_nfdk_tx()
293 if (nfp_nfdk_tx_maybe_close_block(tx_ring, skb)) in nfp_nfdk_tx()
301 wr_idx = D_IDX(tx_ring, tx_ring->wr_p); in nfp_nfdk_tx()
302 txd = &tx_ring->ktxds[wr_idx]; in nfp_nfdk_tx()
303 txbuf = &tx_ring->ktxbufs[wr_idx]; in nfp_nfdk_tx()
409 cnt = txd - tx_ring->ktxds - wr_idx; in nfp_nfdk_tx()
416 tx_ring->wr_p += cnt; in nfp_nfdk_tx()
417 if (tx_ring->wr_p % NFDK_TX_DESC_BLOCK_CNT) in nfp_nfdk_tx()
418 tx_ring->data_pending += skb->len; in nfp_nfdk_tx()
420 tx_ring->data_pending = 0; in nfp_nfdk_tx()
422 if (nfp_nfdk_tx_ring_should_stop(tx_ring)) in nfp_nfdk_tx()
423 nfp_nfdk_tx_ring_stop(nd_q, tx_ring); in nfp_nfdk_tx()
425 tx_ring->wr_ptr_add += cnt; in nfp_nfdk_tx()
427 nfp_net_tx_xmit_more_flush(tx_ring); in nfp_nfdk_tx()
440 txbuf = &tx_ring->ktxbufs[wr_idx + 1]; in nfp_nfdk_tx()
458 nfp_net_tx_xmit_more_flush(tx_ring); in nfp_nfdk_tx()
471 static void nfp_nfdk_tx_complete(struct nfp_net_tx_ring *tx_ring, int budget) in nfp_nfdk_tx_complete() argument
473 struct nfp_net_r_vector *r_vec = tx_ring->r_vec; in nfp_nfdk_tx_complete()
482 rd_p = tx_ring->rd_p; in nfp_nfdk_tx_complete()
483 if (tx_ring->wr_p == rd_p) in nfp_nfdk_tx_complete()
487 qcp_rd_p = nfp_net_read_tx_cmpl(tx_ring, dp); in nfp_nfdk_tx_complete()
489 if (qcp_rd_p == tx_ring->qcp_rd_p) in nfp_nfdk_tx_complete()
492 todo = D_IDX(tx_ring, qcp_rd_p - tx_ring->qcp_rd_p); in nfp_nfdk_tx_complete()
493 ktxbufs = tx_ring->ktxbufs; in nfp_nfdk_tx_complete()
501 txbuf = &ktxbufs[D_IDX(tx_ring, rd_p)]; in nfp_nfdk_tx_complete()
544 tx_ring->rd_p = rd_p; in nfp_nfdk_tx_complete()
545 tx_ring->qcp_rd_p = qcp_rd_p; in nfp_nfdk_tx_complete()
555 nd_q = netdev_get_tx_queue(dp->netdev, tx_ring->idx); in nfp_nfdk_tx_complete()
557 if (nfp_nfdk_tx_ring_should_wake(tx_ring)) { in nfp_nfdk_tx_complete()
565 WARN_ONCE(tx_ring->wr_p - tx_ring->rd_p > tx_ring->cnt, in nfp_nfdk_tx_complete()
567 tx_ring->rd_p, tx_ring->wr_p, tx_ring->cnt); in nfp_nfdk_tx_complete()
835 static bool nfp_nfdk_xdp_complete(struct nfp_net_tx_ring *tx_ring) in nfp_nfdk_xdp_complete() argument
837 struct nfp_net_r_vector *r_vec = tx_ring->r_vec; in nfp_nfdk_xdp_complete()
845 qcp_rd_p = nfp_net_read_tx_cmpl(tx_ring, dp); in nfp_nfdk_xdp_complete()
846 if (qcp_rd_p == tx_ring->qcp_rd_p) in nfp_nfdk_xdp_complete()
849 todo = D_IDX(tx_ring, qcp_rd_p - tx_ring->qcp_rd_p); in nfp_nfdk_xdp_complete()
856 int idx = D_IDX(tx_ring, tx_ring->rd_p + done); in nfp_nfdk_xdp_complete()
860 txbuf = &tx_ring->ktxbufs[idx]; in nfp_nfdk_xdp_complete()
888 tx_ring->qcp_rd_p = D_IDX(tx_ring, tx_ring->qcp_rd_p + done); in nfp_nfdk_xdp_complete()
889 tx_ring->rd_p += done; in nfp_nfdk_xdp_complete()
891 WARN_ONCE(tx_ring->wr_p - tx_ring->rd_p > tx_ring->cnt, in nfp_nfdk_xdp_complete()
893 tx_ring->rd_p, tx_ring->wr_p, tx_ring->cnt); in nfp_nfdk_xdp_complete()
900 struct nfp_net_tx_ring *tx_ring, in nfp_nfdk_tx_xdp_buf() argument
920 if (unlikely(nfp_net_tx_full(tx_ring, NFDK_TX_DESC_STOP_CNT))) { in nfp_nfdk_tx_xdp_buf()
922 nfp_nfdk_xdp_complete(tx_ring); in nfp_nfdk_tx_xdp_buf()
926 if (unlikely(nfp_net_tx_full(tx_ring, NFDK_TX_DESC_STOP_CNT))) { in nfp_nfdk_tx_xdp_buf()
935 if ((round_down(tx_ring->wr_p, NFDK_TX_DESC_BLOCK_CNT) != in nfp_nfdk_tx_xdp_buf()
936 round_down(tx_ring->wr_p + n_descs, NFDK_TX_DESC_BLOCK_CNT)) || in nfp_nfdk_tx_xdp_buf()
937 ((u32)tx_ring->data_pending + pkt_len > in nfp_nfdk_tx_xdp_buf()
939 unsigned int nop_slots = D_BLOCK_CPL(tx_ring->wr_p); in nfp_nfdk_tx_xdp_buf()
941 wr_idx = D_IDX(tx_ring, tx_ring->wr_p); in nfp_nfdk_tx_xdp_buf()
942 txd = &tx_ring->ktxds[wr_idx]; in nfp_nfdk_tx_xdp_buf()
946 tx_ring->data_pending = 0; in nfp_nfdk_tx_xdp_buf()
947 tx_ring->wr_p += nop_slots; in nfp_nfdk_tx_xdp_buf()
948 tx_ring->wr_ptr_add += nop_slots; in nfp_nfdk_tx_xdp_buf()
951 wr_idx = D_IDX(tx_ring, tx_ring->wr_p); in nfp_nfdk_tx_xdp_buf()
953 txbuf = &tx_ring->ktxbufs[wr_idx]; in nfp_nfdk_tx_xdp_buf()
963 txd = &tx_ring->ktxds[wr_idx]; in nfp_nfdk_tx_xdp_buf()
1005 cnt = txd - tx_ring->ktxds - wr_idx; in nfp_nfdk_tx_xdp_buf()
1006 tx_ring->wr_p += cnt; in nfp_nfdk_tx_xdp_buf()
1007 if (tx_ring->wr_p % NFDK_TX_DESC_BLOCK_CNT) in nfp_nfdk_tx_xdp_buf()
1008 tx_ring->data_pending += pkt_len; in nfp_nfdk_tx_xdp_buf()
1010 tx_ring->data_pending = 0; in nfp_nfdk_tx_xdp_buf()
1012 tx_ring->wr_ptr_add += cnt; in nfp_nfdk_tx_xdp_buf()
1031 struct nfp_net_tx_ring *tx_ring; in nfp_nfdk_rx() local
1044 tx_ring = r_vec->xdp_ring; in nfp_nfdk_rx()
1148 tx_ring, in nfp_nfdk_rx()
1247 if (tx_ring->wr_ptr_add) in nfp_nfdk_rx()
1248 nfp_net_tx_xmit_more_flush(tx_ring); in nfp_nfdk_rx()
1249 else if (unlikely(tx_ring->wr_p != tx_ring->rd_p) && in nfp_nfdk_rx()
1251 if (!nfp_nfdk_xdp_complete(tx_ring)) in nfp_nfdk_rx()
1271 if (r_vec->tx_ring) in nfp_nfdk_poll()
1272 nfp_nfdk_tx_complete(r_vec->tx_ring, budget); in nfp_nfdk_poll()
1295 if (r_vec->nfp_net->tx_coalesce_adapt_on && r_vec->tx_ring) { in nfp_nfdk_poll()
1321 struct nfp_net_tx_ring *tx_ring; in nfp_nfdk_ctrl_tx_one() local
1331 tx_ring = r_vec->tx_ring; in nfp_nfdk_ctrl_tx_one()
1339 if (unlikely(nfp_net_tx_full(tx_ring, NFDK_TX_DESC_STOP_CNT))) { in nfp_nfdk_ctrl_tx_one()
1363 if (nfp_nfdk_tx_maybe_close_block(tx_ring, skb)) in nfp_nfdk_ctrl_tx_one()
1367 wr_idx = D_IDX(tx_ring, tx_ring->wr_p); in nfp_nfdk_ctrl_tx_one()
1368 txd = &tx_ring->ktxds[wr_idx]; in nfp_nfdk_ctrl_tx_one()
1369 txbuf = &tx_ring->ktxbufs[wr_idx]; in nfp_nfdk_ctrl_tx_one()
1419 cnt = txd - tx_ring->ktxds - wr_idx; in nfp_nfdk_ctrl_tx_one()
1424 tx_ring->wr_p += cnt; in nfp_nfdk_ctrl_tx_one()
1425 if (tx_ring->wr_p % NFDK_TX_DESC_BLOCK_CNT) in nfp_nfdk_ctrl_tx_one()
1426 tx_ring->data_pending += skb->len; in nfp_nfdk_ctrl_tx_one()
1428 tx_ring->data_pending = 0; in nfp_nfdk_ctrl_tx_one()
1430 tx_ring->wr_ptr_add += cnt; in nfp_nfdk_ctrl_tx_one()
1431 nfp_net_tx_xmit_more_flush(tx_ring); in nfp_nfdk_ctrl_tx_one()
1572 nfp_nfdk_tx_complete(r_vec->tx_ring, 0); in nfp_nfdk_ctrl_poll()