391 	if (!rx_ring->rx_buf)  in ice_clean_rx_ring()
406 		struct ice_rx_buf *rx_buf = &rx_ring->rx_buf[i];  in ice_clean_rx_ring()  local
408 		if (!rx_buf->page)  in ice_clean_rx_ring()
414 		dma_sync_single_range_for_cpu(dev, rx_buf->dma,  in ice_clean_rx_ring()
415 					      rx_buf->page_offset,  in ice_clean_rx_ring()
420 		dma_unmap_page_attrs(dev, rx_buf->dma, ice_rx_pg_size(rx_ring),  in ice_clean_rx_ring()
422 		__page_frag_cache_drain(rx_buf->page, rx_buf->pagecnt_bias);  in ice_clean_rx_ring()
424 		rx_buf->page = NULL;  in ice_clean_rx_ring()
425 		rx_buf->page_offset = 0;  in ice_clean_rx_ring()
432 		memset(rx_ring->rx_buf, 0, array_size(rx_ring->count, sizeof(*rx_ring->rx_buf)));  in ice_clean_rx_ring()
464 		kfree(rx_ring->rx_buf);  in ice_free_rx_ring()
465 		rx_ring->rx_buf = NULL;  in ice_free_rx_ring()
492 	WARN_ON(rx_ring->rx_buf);  in ice_setup_rx_ring()
493 	rx_ring->rx_buf =  in ice_setup_rx_ring()
494 		kcalloc(rx_ring->count, sizeof(*rx_ring->rx_buf), GFP_KERNEL);  in ice_setup_rx_ring()
495 	if (!rx_ring->rx_buf)  in ice_setup_rx_ring()
519 	kfree(rx_ring->rx_buf);  in ice_setup_rx_ring()
520 	rx_ring->rx_buf = NULL;  in ice_setup_rx_ring()
538 	    struct ice_rx_buf *rx_buf, union ice_32b_rx_flex_desc *eop_desc)  in ice_run_xdp()  argument
736 	bi = &rx_ring->rx_buf[ntu];  in ice_alloc_rx_bufs()
759 			bi = rx_ring->rx_buf;  in ice_alloc_rx_bufs()
786 ice_rx_buf_adjust_pg_offset(struct ice_rx_buf *rx_buf, unsigned int size)  in ice_rx_buf_adjust_pg_offset()  argument
790 	rx_buf->page_offset ^= size;  in ice_rx_buf_adjust_pg_offset()
793 	rx_buf->page_offset += size;  in ice_rx_buf_adjust_pg_offset()
807 ice_can_reuse_rx_page(struct ice_rx_buf *rx_buf)  in ice_can_reuse_rx_page()  argument
809 	unsigned int pagecnt_bias = rx_buf->pagecnt_bias;  in ice_can_reuse_rx_page()
810 	struct page *page = rx_buf->page;  in ice_can_reuse_rx_page()
817 	if (unlikely(rx_buf->pgcnt - pagecnt_bias > 1))  in ice_can_reuse_rx_page()
822 	if (rx_buf->page_offset > ICE_LAST_OFFSET)  in ice_can_reuse_rx_page()
832 		rx_buf->pagecnt_bias = USHRT_MAX;  in ice_can_reuse_rx_page()
850 		 struct ice_rx_buf *rx_buf, const unsigned int size)  in ice_add_xdp_frag()  argument
868 	__skb_fill_page_desc_noacc(sinfo, sinfo->nr_frags++, rx_buf->page,  in ice_add_xdp_frag()
869 				   rx_buf->page_offset, size);  in ice_add_xdp_frag()
876 	if (page_is_pfmemalloc(rx_buf->page))  in ice_add_xdp_frag()
895 	new_buf = &rx_ring->rx_buf[nta];  in ice_reuse_rx_page()
924 	struct ice_rx_buf *rx_buf;  in ice_get_rx_buf()  local
926 	rx_buf = &rx_ring->rx_buf[ntc];  in ice_get_rx_buf()
927 	rx_buf->pgcnt = page_count(rx_buf->page);  in ice_get_rx_buf()
928 	prefetchw(rx_buf->page);  in ice_get_rx_buf()
931 		return rx_buf;  in ice_get_rx_buf()
933 	dma_sync_single_range_for_cpu(rx_ring->dev, rx_buf->dma,  in ice_get_rx_buf()
934 				      rx_buf->page_offset, size,  in ice_get_rx_buf()
938 	rx_buf->pagecnt_bias--;  in ice_get_rx_buf()
940 	return rx_buf;  in ice_get_rx_buf()
1010 	struct ice_rx_buf *rx_buf;  in ice_construct_skb()  local
1028 	rx_buf = &rx_ring->rx_buf[rx_ring->first_desc];  in ice_construct_skb()
1050 		skb_add_rx_frag(skb, 0, rx_buf->page,  in ice_construct_skb()
1051 				rx_buf->page_offset + headlen, size,  in ice_construct_skb()
1059 		rx_buf->act = ICE_SKB_CONSUMED;  in ice_construct_skb()
1086 ice_put_rx_buf(struct ice_rx_ring *rx_ring, struct ice_rx_buf *rx_buf)  in ice_put_rx_buf()  argument
1088 	if (!rx_buf)  in ice_put_rx_buf()
1091 	if (ice_can_reuse_rx_page(rx_buf)) {  in ice_put_rx_buf()
1093 		ice_reuse_rx_page(rx_ring, rx_buf);  in ice_put_rx_buf()
1096 		dma_unmap_page_attrs(rx_ring->dev, rx_buf->dma,  in ice_put_rx_buf()
1099 		__page_frag_cache_drain(rx_buf->page, rx_buf->pagecnt_bias);  in ice_put_rx_buf()
1103 	rx_buf->page = NULL;  in ice_put_rx_buf()
1142 		struct ice_rx_buf *rx_buf;  in ice_clean_rx_irq()  local
1183 		rx_buf = ice_get_rx_buf(rx_ring, size, ntc);  in ice_clean_rx_irq()
1188 			hard_start = page_address(rx_buf->page) + rx_buf->page_offset -  in ice_clean_rx_irq()
1192 		} else if (ice_add_xdp_frag(rx_ring, xdp, rx_buf, size)) {  in ice_clean_rx_irq()
1202 		ice_run_xdp(rx_ring, xdp, xdp_prog, xdp_ring, rx_buf, rx_desc);  in ice_clean_rx_irq()
1203 		if (rx_buf->act == ICE_XDP_PASS)  in ice_clean_rx_irq()
1220 			rx_buf->act = ICE_XDP_CONSUMED;  in ice_clean_rx_irq()
1262 		struct ice_rx_buf *buf = &rx_ring->rx_buf[cached_ntc];  in ice_clean_rx_irq()