44 				     struct efx_rx_buffer *rx_buf,  in efx_rx_packet__check_len()  argument
48 	unsigned max_len = rx_buf->len - efx->type->rx_buffer_padding;  in efx_rx_packet__check_len()
56 	rx_buf->flags |= EFX_RX_PKT_DISCARD;  in efx_rx_packet__check_len()
68 				     struct efx_rx_buffer *rx_buf,  in efx_rx_mk_skb()  argument
84 	EFX_WARN_ON_ONCE_PARANOID(rx_buf->len < hdr_len);  in efx_rx_mk_skb()
92 	if (rx_buf->len > hdr_len) {  in efx_rx_mk_skb()
93 		rx_buf->page_offset += hdr_len;  in efx_rx_mk_skb()
94 		rx_buf->len -= hdr_len;  in efx_rx_mk_skb()
98 					rx_buf->page, rx_buf->page_offset,  in efx_rx_mk_skb()
99 					rx_buf->len, efx->rx_buffer_truesize);  in efx_rx_mk_skb()
100 			rx_buf->page = NULL;  in efx_rx_mk_skb()
105 			rx_buf = efx_rx_buf_next(&channel->rx_queue, rx_buf);  in efx_rx_mk_skb()
108 		__free_pages(rx_buf->page, efx->rx_buffer_order);  in efx_rx_mk_skb()
109 		rx_buf->page = NULL;  in efx_rx_mk_skb()
126 	struct efx_rx_buffer *rx_buf;  in efx_rx_packet()  local
130 	rx_buf = efx_rx_buffer(rx_queue, index);  in efx_rx_packet()
131 	rx_buf->flags |= flags;  in efx_rx_packet()
136 			efx_rx_packet__check_len(rx_queue, rx_buf, len);  in efx_rx_packet()
144 		WARN_ON(!(len == 0 && rx_buf->flags & EFX_RX_PKT_DISCARD));  in efx_rx_packet()
145 		rx_buf->flags |= EFX_RX_PKT_DISCARD;  in efx_rx_packet()
152 		   (rx_buf->flags & EFX_RX_PKT_CSUMMED) ? " [SUMMED]" : "",  in efx_rx_packet()
153 		   (rx_buf->flags & EFX_RX_PKT_DISCARD) ? " [DISCARD]" : "");  in efx_rx_packet()
158 	if (unlikely(rx_buf->flags & EFX_RX_PKT_DISCARD)) {  in efx_rx_packet()
160 		efx_discard_rx_packet(channel, rx_buf, n_frags);  in efx_rx_packet()
165 		rx_buf->len = len;  in efx_rx_packet()
170 	efx_sync_rx_buffer(efx, rx_buf, rx_buf->len);  in efx_rx_packet()
175 	prefetch(efx_rx_buf_va(rx_buf));  in efx_rx_packet()
177 	rx_buf->page_offset += efx->rx_prefix_size;  in efx_rx_packet()
178 	rx_buf->len -= efx->rx_prefix_size;  in efx_rx_packet()
187 			rx_buf = efx_rx_buf_next(rx_queue, rx_buf);  in efx_rx_packet()
190 			efx_sync_rx_buffer(efx, rx_buf, efx->rx_dma_len);  in efx_rx_packet()
192 		rx_buf->len = len - (n_frags - 1) * efx->rx_dma_len;  in efx_rx_packet()
193 		efx_sync_rx_buffer(efx, rx_buf, rx_buf->len);  in efx_rx_packet()
197 	rx_buf = efx_rx_buffer(rx_queue, index);  in efx_rx_packet()
198 	efx_recycle_rx_pages(channel, rx_buf, n_frags);  in efx_rx_packet()
209 			   struct efx_rx_buffer *rx_buf,  in efx_rx_deliver()  argument
213 	u16 hdr_len = min_t(u16, rx_buf->len, EFX_SKB_HEADERS);  in efx_rx_deliver()
215 	skb = efx_rx_mk_skb(channel, rx_buf, n_frags, eh, hdr_len);  in efx_rx_deliver()
220 		efx_free_rx_buffers(rx_queue, rx_buf, n_frags);  in efx_rx_deliver()
227 	if (likely(rx_buf->flags & EFX_RX_PKT_CSUMMED)) {  in efx_rx_deliver()
229 		skb->csum_level = !!(rx_buf->flags & EFX_RX_PKT_CSUM_LEVEL);  in efx_rx_deliver()
252 		       struct efx_rx_buffer *rx_buf, u8 **ehp)  in efx_do_xdp()  argument
271 		efx_free_rx_buffers(rx_queue, rx_buf,  in efx_do_xdp()
281 	dma_sync_single_for_cpu(&efx->pci_dev->dev, rx_buf->dma_addr,  in efx_do_xdp()
282 				rx_buf->len, DMA_FROM_DEVICE);  in efx_do_xdp()
292 			 rx_buf->len, false);  in efx_do_xdp()
303 			rx_buf->page_offset += offset;  in efx_do_xdp()
304 			rx_buf->len -= offset;  in efx_do_xdp()
315 			efx_free_rx_buffers(rx_queue, rx_buf, 1);  in efx_do_xdp()
329 			efx_free_rx_buffers(rx_queue, rx_buf, 1);  in efx_do_xdp()
342 		efx_free_rx_buffers(rx_queue, rx_buf, 1);  in efx_do_xdp()
351 		efx_free_rx_buffers(rx_queue, rx_buf, 1);  in efx_do_xdp()
364 	struct efx_rx_buffer *rx_buf =  in __efx_rx_packet()  local
366 	u8 *eh = efx_rx_buf_va(rx_buf);  in __efx_rx_packet()
371 	if (rx_buf->flags & EFX_RX_PKT_PREFIX_LEN) {  in __efx_rx_packet()
372 		rx_buf->len = le16_to_cpup((__le16 *)  in __efx_rx_packet()
379 		if (unlikely(!rx_buf->len)) {  in __efx_rx_packet()
380 			efx_free_rx_buffers(rx_queue, rx_buf,  in __efx_rx_packet()
391 		efx_loopback_rx_packet(efx, eh, rx_buf->len);  in __efx_rx_packet()
392 		efx_free_rx_buffers(rx_queue, rx_buf,  in __efx_rx_packet()
397 	if (!efx_do_xdp(efx, channel, rx_buf, &eh))  in __efx_rx_packet()
401 		rx_buf->flags &= ~EFX_RX_PKT_CSUMMED;  in __efx_rx_packet()
403 	if ((rx_buf->flags & EFX_RX_PKT_TCP) && !channel->type->receive_skb)  in __efx_rx_packet()
404 		efx_rx_packet_gro(channel, rx_buf, channel->rx_pkt_n_frags, eh, 0);  in __efx_rx_packet()
406 		efx_rx_deliver(channel, eh, rx_buf, channel->rx_pkt_n_frags);  in __efx_rx_packet()