17 		    struct nfp_net_tx_ring *tx_ring,  in nfp_nfd3_xsk_tx_xdp()  argument
26 	if (nfp_net_tx_space(tx_ring) < 1)  in nfp_nfd3_xsk_tx_xdp()
32 	wr_idx = D_IDX(tx_ring, tx_ring->wr_p);  in nfp_nfd3_xsk_tx_xdp()
34 	txbuf = &tx_ring->txbufs[wr_idx];  in nfp_nfd3_xsk_tx_xdp()
40 	txd = &tx_ring->txds[wr_idx];  in nfp_nfd3_xsk_tx_xdp()
50 	tx_ring->wr_ptr_add++;  in nfp_nfd3_xsk_tx_xdp()
51 	tx_ring->wr_p++;  in nfp_nfd3_xsk_tx_xdp()
120 	struct nfp_net_tx_ring *tx_ring;  in nfp_nfd3_xsk_rx()  local
126 	tx_ring = r_vec->xdp_ring;  in nfp_nfd3_xsk_rx()
230 			if (!nfp_nfd3_xsk_tx_xdp(dp, r_vec, rx_ring, tx_ring,  in nfp_nfd3_xsk_rx()
261 	if (tx_ring->wr_ptr_add)  in nfp_nfd3_xsk_rx()
262 		nfp_net_tx_xmit_more_flush(tx_ring);  in nfp_nfd3_xsk_rx()
275 static bool nfp_nfd3_xsk_complete(struct nfp_net_tx_ring *tx_ring)  in nfp_nfd3_xsk_complete()  argument
277 	struct nfp_net_r_vector *r_vec = tx_ring->r_vec;  in nfp_nfd3_xsk_complete()
283 	if (tx_ring->wr_p == tx_ring->rd_p)  in nfp_nfd3_xsk_complete()
287 	qcp_rd_p = nfp_qcp_rd_ptr_read(tx_ring->qcp_q);  in nfp_nfd3_xsk_complete()
289 	if (qcp_rd_p == tx_ring->qcp_rd_p)  in nfp_nfd3_xsk_complete()
292 	todo = D_IDX(tx_ring, qcp_rd_p - tx_ring->qcp_rd_p);  in nfp_nfd3_xsk_complete()
297 	tx_ring->qcp_rd_p = D_IDX(tx_ring, tx_ring->qcp_rd_p + todo);  in nfp_nfd3_xsk_complete()
303 		idx = D_IDX(tx_ring, tx_ring->rd_p);  in nfp_nfd3_xsk_complete()
304 		tx_ring->rd_p++;  in nfp_nfd3_xsk_complete()
306 		txbuf = &tx_ring->txbufs[idx];  in nfp_nfd3_xsk_complete()
326 	WARN_ONCE(tx_ring->wr_p - tx_ring->rd_p > tx_ring->cnt,  in nfp_nfd3_xsk_complete()
328 		  tx_ring->rd_p, tx_ring->wr_p, tx_ring->cnt);  in nfp_nfd3_xsk_complete()
333 static void nfp_nfd3_xsk_tx(struct nfp_net_tx_ring *tx_ring)  in nfp_nfd3_xsk_tx()  argument
335 	struct nfp_net_r_vector *r_vec = tx_ring->r_vec;  in nfp_nfd3_xsk_tx()
344 	while (nfp_net_tx_space(tx_ring) >= NFP_NET_XSK_TX_BATCH) {  in nfp_nfd3_xsk_tx()
352 		wr_idx = D_IDX(tx_ring, tx_ring->wr_p + i);  in nfp_nfd3_xsk_tx()
353 		prefetchw(&tx_ring->txds[wr_idx]);  in nfp_nfd3_xsk_tx()
360 			wr_idx = D_IDX(tx_ring, tx_ring->wr_p + i);  in nfp_nfd3_xsk_tx()
362 			tx_ring->txbufs[wr_idx].real_len = desc[i].len;  in nfp_nfd3_xsk_tx()
363 			tx_ring->txbufs[wr_idx].is_xsk_tx = false;  in nfp_nfd3_xsk_tx()
366 			txd = &tx_ring->txds[wr_idx];  in nfp_nfd3_xsk_tx()
374 		tx_ring->wr_p += got;  in nfp_nfd3_xsk_tx()
384 	nfp_qcp_wr_ptr_add(tx_ring->qcp_q, pkts);  in nfp_nfd3_xsk_tx()
396 		if (r_vec->tx_ring)  in nfp_nfd3_xsk_poll()
397 			nfp_nfd3_tx_complete(r_vec->tx_ring, budget);  in nfp_nfd3_xsk_poll()