25 static inline u8 *ef4_tx_get_copy_buffer(struct ef4_tx_queue *tx_queue,  in ef4_tx_get_copy_buffer()  argument
28 	unsigned int index = ef4_tx_queue_get_insert_index(tx_queue);  in ef4_tx_get_copy_buffer()
30 		&tx_queue->cb_page[index >> (PAGE_SHIFT - EF4_TX_CB_ORDER)];  in ef4_tx_get_copy_buffer()
35 	    ef4_nic_alloc_buffer(tx_queue->efx, page_buf, PAGE_SIZE,  in ef4_tx_get_copy_buffer()
43 u8 *ef4_tx_get_copy_buffer_limited(struct ef4_tx_queue *tx_queue,  in ef4_tx_get_copy_buffer_limited()  argument
48 	return ef4_tx_get_copy_buffer(tx_queue, buffer);  in ef4_tx_get_copy_buffer_limited()
51 static void ef4_dequeue_buffer(struct ef4_tx_queue *tx_queue,  in ef4_dequeue_buffer()  argument
57 		struct device *dma_dev = &tx_queue->efx->pci_dev->dev;  in ef4_dequeue_buffer()
72 		netif_vdbg(tx_queue->efx, tx_done, tx_queue->efx->net_dev,  in ef4_dequeue_buffer()
74 			   tx_queue->queue, tx_queue->read_count);  in ef4_dequeue_buffer()
148 static int ef4_enqueue_skb_copy(struct ef4_tx_queue *tx_queue,  in ef4_enqueue_skb_copy()  argument
151 	unsigned int min_len = tx_queue->tx_min_size;  in ef4_enqueue_skb_copy()
159 	buffer = ef4_tx_queue_get_insert_buffer(tx_queue);  in ef4_enqueue_skb_copy()
161 	copy_buffer = ef4_tx_get_copy_buffer(tx_queue, buffer);  in ef4_enqueue_skb_copy()
177 	++tx_queue->insert_count;  in ef4_enqueue_skb_copy()
181 static struct ef4_tx_buffer *ef4_tx_map_chunk(struct ef4_tx_queue *tx_queue,  in ef4_tx_map_chunk()  argument
185 	const struct ef4_nic_type *nic_type = tx_queue->efx->type;  in ef4_tx_map_chunk()
191 		buffer = ef4_tx_queue_get_insert_buffer(tx_queue);  in ef4_tx_map_chunk()
192 		dma_len = nic_type->tx_limit_len(tx_queue, dma_addr, len);  in ef4_tx_map_chunk()
199 		++tx_queue->insert_count;  in ef4_tx_map_chunk()
207 static int ef4_tx_map_data(struct ef4_tx_queue *tx_queue, struct sk_buff *skb)  in ef4_tx_map_data()  argument
209 	struct ef4_nic *efx = tx_queue->efx;  in ef4_tx_map_data()
234 		buffer = ef4_tx_map_chunk(tx_queue, dma_addr, len);  in ef4_tx_map_data()
269 static void ef4_enqueue_unwind(struct ef4_tx_queue *tx_queue)  in ef4_enqueue_unwind()  argument
274 	while (tx_queue->insert_count != tx_queue->write_count) {  in ef4_enqueue_unwind()
275 		--tx_queue->insert_count;  in ef4_enqueue_unwind()
276 		buffer = __ef4_tx_queue_get_insert_buffer(tx_queue);  in ef4_enqueue_unwind()
277 		ef4_dequeue_buffer(tx_queue, buffer, NULL, NULL);  in ef4_enqueue_unwind()
297 netdev_tx_t ef4_enqueue_skb(struct ef4_tx_queue *tx_queue, struct sk_buff *skb)  in ef4_enqueue_skb()  argument
305 	if (skb_len < tx_queue->tx_min_size ||  in ef4_enqueue_skb()
308 		if (ef4_enqueue_skb_copy(tx_queue, skb))  in ef4_enqueue_skb()
310 		tx_queue->cb_packets++;  in ef4_enqueue_skb()
315 	if (!data_mapped && (ef4_tx_map_data(tx_queue, skb)))  in ef4_enqueue_skb()
319 	netdev_tx_sent_queue(tx_queue->core_txq, skb_len);  in ef4_enqueue_skb()
322 	if (!netdev_xmit_more() || netif_xmit_stopped(tx_queue->core_txq)) {  in ef4_enqueue_skb()
323 		struct ef4_tx_queue *txq2 = ef4_tx_queue_partner(tx_queue);  in ef4_enqueue_skb()
332 		ef4_nic_push_buffers(tx_queue);  in ef4_enqueue_skb()
334 		tx_queue->xmit_more_available = netdev_xmit_more();  in ef4_enqueue_skb()
337 	tx_queue->tx_packets++;  in ef4_enqueue_skb()
339 	ef4_tx_maybe_stop_queue(tx_queue);  in ef4_enqueue_skb()
345 	ef4_enqueue_unwind(tx_queue);  in ef4_enqueue_skb()
355 static void ef4_dequeue_buffers(struct ef4_tx_queue *tx_queue,  in ef4_dequeue_buffers()  argument
360 	struct ef4_nic *efx = tx_queue->efx;  in ef4_dequeue_buffers()
363 	stop_index = (index + 1) & tx_queue->ptr_mask;  in ef4_dequeue_buffers()
364 	read_ptr = tx_queue->read_count & tx_queue->ptr_mask;  in ef4_dequeue_buffers()
367 		struct ef4_tx_buffer *buffer = &tx_queue->buffer[read_ptr];  in ef4_dequeue_buffers()
373 				  tx_queue->queue, read_ptr);  in ef4_dequeue_buffers()
378 		ef4_dequeue_buffer(tx_queue, buffer, pkts_compl, bytes_compl);  in ef4_dequeue_buffers()
380 		++tx_queue->read_count;  in ef4_dequeue_buffers()
381 		read_ptr = tx_queue->read_count & tx_queue->ptr_mask;  in ef4_dequeue_buffers()
398 	struct ef4_tx_queue *tx_queue;  in ef4_hard_start_xmit()  local
409 	tx_queue = ef4_get_tx_queue(efx, index, type);  in ef4_hard_start_xmit()
411 	return ef4_enqueue_skb(tx_queue, skb);  in ef4_hard_start_xmit()
414 void ef4_init_tx_queue_core_txq(struct ef4_tx_queue *tx_queue)  in ef4_init_tx_queue_core_txq()  argument
416 	struct ef4_nic *efx = tx_queue->efx;  in ef4_init_tx_queue_core_txq()
419 	tx_queue->core_txq =  in ef4_init_tx_queue_core_txq()
421 				    tx_queue->queue / EF4_TXQ_TYPES +  in ef4_init_tx_queue_core_txq()
422 				    ((tx_queue->queue & EF4_TXQ_TYPE_HIGHPRI) ?  in ef4_init_tx_queue_core_txq()
432 	struct ef4_tx_queue *tx_queue;  in ef4_setup_tc()  local
457 			ef4_for_each_possible_channel_tx_queue(tx_queue,  in ef4_setup_tc()
459 				if (!(tx_queue->queue & EF4_TXQ_TYPE_HIGHPRI))  in ef4_setup_tc()
461 				if (!tx_queue->buffer) {  in ef4_setup_tc()
462 					rc = ef4_probe_tx_queue(tx_queue);  in ef4_setup_tc()
466 				if (!tx_queue->initialised)  in ef4_setup_tc()
467 					ef4_init_tx_queue(tx_queue);  in ef4_setup_tc()
468 				ef4_init_tx_queue_core_txq(tx_queue);  in ef4_setup_tc()
492 void ef4_xmit_done(struct ef4_tx_queue *tx_queue, unsigned int index)  in ef4_xmit_done()  argument
495 	struct ef4_nic *efx = tx_queue->efx;  in ef4_xmit_done()
499 	EF4_BUG_ON_PARANOID(index > tx_queue->ptr_mask);  in ef4_xmit_done()
501 	ef4_dequeue_buffers(tx_queue, index, &pkts_compl, &bytes_compl);  in ef4_xmit_done()
502 	tx_queue->pkts_compl += pkts_compl;  in ef4_xmit_done()
503 	tx_queue->bytes_compl += bytes_compl;  in ef4_xmit_done()
506 		++tx_queue->merge_events;  in ef4_xmit_done()
513 	if (unlikely(netif_tx_queue_stopped(tx_queue->core_txq)) &&  in ef4_xmit_done()
516 		txq2 = ef4_tx_queue_partner(tx_queue);  in ef4_xmit_done()
517 		fill_level = max(tx_queue->insert_count - tx_queue->read_count,  in ef4_xmit_done()
520 			netif_tx_wake_queue(tx_queue->core_txq);  in ef4_xmit_done()
524 	if ((int)(tx_queue->read_count - tx_queue->old_write_count) >= 0) {  in ef4_xmit_done()
525 		tx_queue->old_write_count = READ_ONCE(tx_queue->write_count);  in ef4_xmit_done()
526 		if (tx_queue->read_count == tx_queue->old_write_count) {  in ef4_xmit_done()
528 			tx_queue->empty_read_count =  in ef4_xmit_done()
529 				tx_queue->read_count | EF4_EMPTY_COUNT_VALID;  in ef4_xmit_done()
534 static unsigned int ef4_tx_cb_page_count(struct ef4_tx_queue *tx_queue)  in ef4_tx_cb_page_count()  argument
536 	return DIV_ROUND_UP(tx_queue->ptr_mask + 1, PAGE_SIZE >> EF4_TX_CB_ORDER);  in ef4_tx_cb_page_count()
539 int ef4_probe_tx_queue(struct ef4_tx_queue *tx_queue)  in ef4_probe_tx_queue()  argument
541 	struct ef4_nic *efx = tx_queue->efx;  in ef4_probe_tx_queue()
548 	tx_queue->ptr_mask = entries - 1;  in ef4_probe_tx_queue()
552 		  tx_queue->queue, efx->txq_entries, tx_queue->ptr_mask);  in ef4_probe_tx_queue()
555 	tx_queue->buffer = kcalloc(entries, sizeof(*tx_queue->buffer),  in ef4_probe_tx_queue()
557 	if (!tx_queue->buffer)  in ef4_probe_tx_queue()
560 	tx_queue->cb_page = kcalloc(ef4_tx_cb_page_count(tx_queue),  in ef4_probe_tx_queue()
561 				    sizeof(tx_queue->cb_page[0]), GFP_KERNEL);  in ef4_probe_tx_queue()
562 	if (!tx_queue->cb_page) {  in ef4_probe_tx_queue()
568 	rc = ef4_nic_probe_tx(tx_queue);  in ef4_probe_tx_queue()
575 	kfree(tx_queue->cb_page);  in ef4_probe_tx_queue()
576 	tx_queue->cb_page = NULL;  in ef4_probe_tx_queue()
578 	kfree(tx_queue->buffer);  in ef4_probe_tx_queue()
579 	tx_queue->buffer = NULL;  in ef4_probe_tx_queue()
583 void ef4_init_tx_queue(struct ef4_tx_queue *tx_queue)  in ef4_init_tx_queue()  argument
585 	struct ef4_nic *efx = tx_queue->efx;  in ef4_init_tx_queue()
588 		  "initialising TX queue %d\n", tx_queue->queue);  in ef4_init_tx_queue()
590 	tx_queue->insert_count = 0;  in ef4_init_tx_queue()
591 	tx_queue->write_count = 0;  in ef4_init_tx_queue()
592 	tx_queue->old_write_count = 0;  in ef4_init_tx_queue()
593 	tx_queue->read_count = 0;  in ef4_init_tx_queue()
594 	tx_queue->old_read_count = 0;  in ef4_init_tx_queue()
595 	tx_queue->empty_read_count = 0 | EF4_EMPTY_COUNT_VALID;  in ef4_init_tx_queue()
596 	tx_queue->xmit_more_available = false;  in ef4_init_tx_queue()
599 	tx_queue->tx_min_size = EF4_WORKAROUND_15592(efx) ? 33 : 0;  in ef4_init_tx_queue()
602 	ef4_nic_init_tx(tx_queue);  in ef4_init_tx_queue()
604 	tx_queue->initialised = true;  in ef4_init_tx_queue()
607 void ef4_fini_tx_queue(struct ef4_tx_queue *tx_queue)  in ef4_fini_tx_queue()  argument
611 	netif_dbg(tx_queue->efx, drv, tx_queue->efx->net_dev,  in ef4_fini_tx_queue()
612 		  "shutting down TX queue %d\n", tx_queue->queue);  in ef4_fini_tx_queue()
614 	if (!tx_queue->buffer)  in ef4_fini_tx_queue()
618 	while (tx_queue->read_count != tx_queue->write_count) {  in ef4_fini_tx_queue()
620 		buffer = &tx_queue->buffer[tx_queue->read_count & tx_queue->ptr_mask];  in ef4_fini_tx_queue()
621 		ef4_dequeue_buffer(tx_queue, buffer, &pkts_compl, &bytes_compl);  in ef4_fini_tx_queue()
623 		++tx_queue->read_count;  in ef4_fini_tx_queue()
625 	tx_queue->xmit_more_available = false;  in ef4_fini_tx_queue()
626 	netdev_tx_reset_queue(tx_queue->core_txq);  in ef4_fini_tx_queue()
629 void ef4_remove_tx_queue(struct ef4_tx_queue *tx_queue)  in ef4_remove_tx_queue()  argument
633 	if (!tx_queue->buffer)  in ef4_remove_tx_queue()
636 	netif_dbg(tx_queue->efx, drv, tx_queue->efx->net_dev,  in ef4_remove_tx_queue()
637 		  "destroying TX queue %d\n", tx_queue->queue);  in ef4_remove_tx_queue()
638 	ef4_nic_remove_tx(tx_queue);  in ef4_remove_tx_queue()
640 	if (tx_queue->cb_page) {  in ef4_remove_tx_queue()
641 		for (i = 0; i < ef4_tx_cb_page_count(tx_queue); i++)  in ef4_remove_tx_queue()
642 			ef4_nic_free_buffer(tx_queue->efx,  in ef4_remove_tx_queue()
643 					    &tx_queue->cb_page[i]);  in ef4_remove_tx_queue()
644 		kfree(tx_queue->cb_page);  in ef4_remove_tx_queue()
645 		tx_queue->cb_page = NULL;  in ef4_remove_tx_queue()
648 	kfree(tx_queue->buffer);  in ef4_remove_tx_queue()
649 	tx_queue->buffer = NULL;  in ef4_remove_tx_queue()