15 static void octep_oq_reset_indices(struct octep_oq *oq)  in octep_oq_reset_indices()  argument
17 	oq->host_read_idx = 0;  in octep_oq_reset_indices()
18 	oq->host_refill_idx = 0;  in octep_oq_reset_indices()
19 	oq->refill_count = 0;  in octep_oq_reset_indices()
20 	oq->last_pkt_count = 0;  in octep_oq_reset_indices()
21 	oq->pkts_pending = 0;  in octep_oq_reset_indices()
32 static int octep_oq_fill_ring_buffers(struct octep_oq *oq)  in octep_oq_fill_ring_buffers()  argument
34 	struct octep_oq_desc_hw *desc_ring = oq->desc_ring;  in octep_oq_fill_ring_buffers()
38 	for (i = 0; i < oq->max_count; i++) {  in octep_oq_fill_ring_buffers()
41 			dev_err(oq->dev, "Rx buffer alloc failed\n");  in octep_oq_fill_ring_buffers()
44 		desc_ring[i].buffer_ptr = dma_map_page(oq->dev, page, 0,  in octep_oq_fill_ring_buffers()
47 		if (dma_mapping_error(oq->dev, desc_ring[i].buffer_ptr)) {  in octep_oq_fill_ring_buffers()
48 			dev_err(oq->dev,  in octep_oq_fill_ring_buffers()
50 				oq->q_no);  in octep_oq_fill_ring_buffers()
54 		oq->buff_info[i].page = page;  in octep_oq_fill_ring_buffers()
63 		dma_unmap_page(oq->dev, desc_ring[i].buffer_ptr, PAGE_SIZE, DMA_FROM_DEVICE);  in octep_oq_fill_ring_buffers()
64 		put_page(oq->buff_info[i].page);  in octep_oq_fill_ring_buffers()
65 		oq->buff_info[i].page = NULL;  in octep_oq_fill_ring_buffers()
79 static int octep_oq_refill(struct octep_device *oct, struct octep_oq *oq)  in octep_oq_refill()  argument
81 	struct octep_oq_desc_hw *desc_ring = oq->desc_ring;  in octep_oq_refill()
85 	refill_idx = oq->host_refill_idx;  in octep_oq_refill()
86 	for (i = 0; i < oq->refill_count; i++) {  in octep_oq_refill()
89 			dev_err(oq->dev, "refill: rx buffer alloc failed\n");  in octep_oq_refill()
90 			oq->stats.alloc_failures++;  in octep_oq_refill()
94 		desc_ring[refill_idx].buffer_ptr = dma_map_page(oq->dev, page, 0,  in octep_oq_refill()
96 		if (dma_mapping_error(oq->dev, desc_ring[refill_idx].buffer_ptr)) {  in octep_oq_refill()
97 			dev_err(oq->dev,  in octep_oq_refill()
99 				oq->q_no);  in octep_oq_refill()
101 			oq->stats.alloc_failures++;  in octep_oq_refill()
104 		oq->buff_info[refill_idx].page = page;  in octep_oq_refill()
106 		if (refill_idx == oq->max_count)  in octep_oq_refill()
109 	oq->host_refill_idx = refill_idx;  in octep_oq_refill()
110 	oq->refill_count -= i;  in octep_oq_refill()
125 	struct octep_oq *oq;  in octep_setup_oq()  local
128 	oq = vzalloc(sizeof(*oq));  in octep_setup_oq()
129 	if (!oq)  in octep_setup_oq()
131 	oct->oq[q_no] = oq;  in octep_setup_oq()
133 	oq->octep_dev = oct;  in octep_setup_oq()
134 	oq->netdev = oct->netdev;  in octep_setup_oq()
135 	oq->dev = &oct->pdev->dev;  in octep_setup_oq()
136 	oq->q_no = q_no;  in octep_setup_oq()
137 	oq->max_count = CFG_GET_OQ_NUM_DESC(oct->conf);  in octep_setup_oq()
138 	oq->ring_size_mask = oq->max_count - 1;  in octep_setup_oq()
139 	oq->buffer_size = CFG_GET_OQ_BUF_SIZE(oct->conf);  in octep_setup_oq()
140 	oq->max_single_buffer_size = oq->buffer_size - OCTEP_OQ_RESP_HW_SIZE;  in octep_setup_oq()
147 		oq->max_single_buffer_size -= OCTEP_OQ_RESP_HW_EXT_SIZE;  in octep_setup_oq()
149 	oq->refill_threshold = CFG_GET_OQ_REFILL_THRESHOLD(oct->conf);  in octep_setup_oq()
151 	desc_ring_size = oq->max_count * OCTEP_OQ_DESC_SIZE;  in octep_setup_oq()
152 	oq->desc_ring = dma_alloc_coherent(oq->dev, desc_ring_size,  in octep_setup_oq()
153 					   &oq->desc_ring_dma, GFP_KERNEL);  in octep_setup_oq()
155 	if (unlikely(!oq->desc_ring)) {  in octep_setup_oq()
156 		dev_err(oq->dev,  in octep_setup_oq()
161 	oq->buff_info = vcalloc(oq->max_count, OCTEP_OQ_RECVBUF_SIZE);  in octep_setup_oq()
162 	if (unlikely(!oq->buff_info)) {  in octep_setup_oq()
168 	if (octep_oq_fill_ring_buffers(oq))  in octep_setup_oq()
171 	octep_oq_reset_indices(oq);  in octep_setup_oq()
178 	vfree(oq->buff_info);  in octep_setup_oq()
179 	oq->buff_info = NULL;  in octep_setup_oq()
181 	dma_free_coherent(oq->dev, desc_ring_size,  in octep_setup_oq()
182 			  oq->desc_ring, oq->desc_ring_dma);  in octep_setup_oq()
183 	oq->desc_ring = NULL;  in octep_setup_oq()
185 	vfree(oq);  in octep_setup_oq()
186 	oct->oq[q_no] = NULL;  in octep_setup_oq()
198 static void octep_oq_free_ring_buffers(struct octep_oq *oq)  in octep_oq_free_ring_buffers()  argument
200 	struct octep_oq_desc_hw *desc_ring = oq->desc_ring;  in octep_oq_free_ring_buffers()
203 	if (!oq->desc_ring || !oq->buff_info)  in octep_oq_free_ring_buffers()
206 	for (i = 0; i < oq->max_count; i++)  {  in octep_oq_free_ring_buffers()
207 		if (oq->buff_info[i].page) {  in octep_oq_free_ring_buffers()
208 			dma_unmap_page(oq->dev, desc_ring[i].buffer_ptr,  in octep_oq_free_ring_buffers()
210 			put_page(oq->buff_info[i].page);  in octep_oq_free_ring_buffers()
211 			oq->buff_info[i].page = NULL;  in octep_oq_free_ring_buffers()
215 	octep_oq_reset_indices(oq);  in octep_oq_free_ring_buffers()
225 static int octep_free_oq(struct octep_oq *oq)  in octep_free_oq()  argument
227 	struct octep_device *oct = oq->octep_dev;  in octep_free_oq()
228 	int q_no = oq->q_no;  in octep_free_oq()
230 	octep_oq_free_ring_buffers(oq);  in octep_free_oq()
232 	vfree(oq->buff_info);  in octep_free_oq()
234 	if (oq->desc_ring)  in octep_free_oq()
235 		dma_free_coherent(oq->dev,  in octep_free_oq()
236 				  oq->max_count * OCTEP_OQ_DESC_SIZE,  in octep_free_oq()
237 				  oq->desc_ring, oq->desc_ring_dma);  in octep_free_oq()
239 	vfree(oq);  in octep_free_oq()
240 	oct->oq[q_no] = NULL;  in octep_free_oq()
270 		octep_free_oq(oct->oq[i]);  in octep_setup_oqs()
287 		writel(oct->oq[i]->max_count, oct->oq[i]->pkts_credit_reg);  in octep_oq_dbell_init()
300 		if (!oct->oq[i])  in octep_free_oqs()
302 		octep_free_oq(oct->oq[i]);  in octep_free_oqs()
317 				      struct octep_oq *oq)  in octep_oq_check_hw_for_pkts()  argument
321 	pkt_count = readl(oq->pkts_sent_reg);  in octep_oq_check_hw_for_pkts()
322 	new_pkts = pkt_count - oq->last_pkt_count;  in octep_oq_check_hw_for_pkts()
330 		writel(pkt_count, oq->pkts_sent_reg);  in octep_oq_check_hw_for_pkts()
331 		pkt_count = readl(oq->pkts_sent_reg);  in octep_oq_check_hw_for_pkts()
334 	oq->last_pkt_count = pkt_count;  in octep_oq_check_hw_for_pkts()
335 	oq->pkts_pending += new_pkts;  in octep_oq_check_hw_for_pkts()
350 static void octep_oq_next_pkt(struct octep_oq *oq,  in octep_oq_next_pkt()  argument
354 	dma_unmap_page(oq->dev, oq->desc_ring[*read_idx].buffer_ptr,  in octep_oq_next_pkt()
359 	if (*read_idx == oq->max_count)  in octep_oq_next_pkt()
372 static void octep_oq_drop_rx(struct octep_oq *oq,  in octep_oq_drop_rx()  argument
376 	int data_len = buff_info->len - oq->max_single_buffer_size;  in octep_oq_drop_rx()
379 		octep_oq_next_pkt(oq, buff_info, read_idx, desc_used);  in octep_oq_drop_rx()
380 		data_len -= oq->buffer_size;  in octep_oq_drop_rx()
398 				 struct octep_oq *oq, u16 pkts_to_process)  in __octep_oq_process_rx()  argument
401 	netdev_features_t feat = oq->netdev->features;  in __octep_oq_process_rx()
410 	read_idx = oq->host_read_idx;  in __octep_oq_process_rx()
414 		buff_info = (struct octep_rx_buffer *)&oq->buff_info[read_idx];  in __octep_oq_process_rx()
440 		octep_oq_next_pkt(oq, buff_info, &read_idx, &desc_used);  in __octep_oq_process_rx()
444 			octep_oq_drop_rx(oq, buff_info,  in __octep_oq_process_rx()
446 			oq->stats.alloc_failures++;  in __octep_oq_process_rx()
453 		if (buff_info->len <= oq->max_single_buffer_size) {  in __octep_oq_process_rx()
462 			skb_put(skb, oq->max_single_buffer_size);  in __octep_oq_process_rx()
464 			data_len = buff_info->len - oq->max_single_buffer_size;  in __octep_oq_process_rx()
467 					    &oq->buff_info[read_idx];  in __octep_oq_process_rx()
468 				if (data_len < oq->buffer_size) {  in __octep_oq_process_rx()
472 					buff_info->len = oq->buffer_size;  in __octep_oq_process_rx()
473 					data_len -= oq->buffer_size;  in __octep_oq_process_rx()
481 				octep_oq_next_pkt(oq, buff_info, &read_idx, &desc_used);  in __octep_oq_process_rx()
485 		skb->dev = oq->netdev;  in __octep_oq_process_rx()
492 		napi_gro_receive(oq->napi, skb);  in __octep_oq_process_rx()
495 	oq->host_read_idx = read_idx;  in __octep_oq_process_rx()
496 	oq->refill_count += desc_used;  in __octep_oq_process_rx()
497 	oq->stats.packets += pkt;  in __octep_oq_process_rx()
498 	oq->stats.bytes += rx_bytes;  in __octep_oq_process_rx()
514 int octep_oq_process_rx(struct octep_oq *oq, int budget)  in octep_oq_process_rx()  argument
517 	struct octep_device *oct = oq->octep_dev;  in octep_oq_process_rx()
524 		if (oq->pkts_pending == 0)  in octep_oq_process_rx()
525 			octep_oq_check_hw_for_pkts(oct, oq);  in octep_oq_process_rx()
527 				     oq->pkts_pending);  in octep_oq_process_rx()
531 		pkts_processed = __octep_oq_process_rx(oct, oq,  in octep_oq_process_rx()
533 		oq->pkts_pending -= pkts_processed;  in octep_oq_process_rx()
537 	if (oq->refill_count >= oq->refill_threshold) {  in octep_oq_process_rx()
538 		u32 desc_refilled = octep_oq_refill(oct, oq);  in octep_oq_process_rx()
542 		writel(desc_refilled, oq->pkts_credit_reg);  in octep_oq_process_rx()