299 static void mvpp2_txq_inc_get(struct mvpp2_txq_pcpu *txq_pcpu)  in mvpp2_txq_inc_get()  argument
301 	txq_pcpu->txq_get_index++;  in mvpp2_txq_inc_get()
302 	if (txq_pcpu->txq_get_index == txq_pcpu->size)  in mvpp2_txq_inc_get()
303 		txq_pcpu->txq_get_index = 0;  in mvpp2_txq_inc_get()
307 			      struct mvpp2_txq_pcpu *txq_pcpu,  in mvpp2_txq_inc_put()  argument
313 		txq_pcpu->buffs + txq_pcpu->txq_put_index;  in mvpp2_txq_inc_put()
322 	txq_pcpu->txq_put_index++;  in mvpp2_txq_inc_put()
323 	if (txq_pcpu->txq_put_index == txq_pcpu->size)  in mvpp2_txq_inc_put()
324 		txq_pcpu->txq_put_index = 0;  in mvpp2_txq_inc_put()
2545 					    struct mvpp2_txq_pcpu *txq_pcpu,  in mvpp2_txq_reserved_desc_num_proc()  argument
2551 	if (txq_pcpu->reserved_num >= num)  in mvpp2_txq_reserved_desc_num_proc()
2568 	req = max(MVPP2_CPU_DESC_CHUNK, num - txq_pcpu->reserved_num);  in mvpp2_txq_reserved_desc_num_proc()
2575 	txq_pcpu->reserved_num += mvpp2_txq_alloc_reserved_desc(port, txq, req);  in mvpp2_txq_reserved_desc_num_proc()
2578 	if (txq_pcpu->reserved_num < num)  in mvpp2_txq_reserved_desc_num_proc()
2824 				struct mvpp2_txq_pcpu *txq_pcpu, int num)  in mvpp2_txq_bufs_free()  argument
2835 			txq_pcpu->buffs + txq_pcpu->txq_get_index;  in mvpp2_txq_bufs_free()
2837 		if (!IS_TSO_HEADER(txq_pcpu, tx_buf->dma) &&  in mvpp2_txq_bufs_free()
2847 		mvpp2_txq_inc_get(txq_pcpu);  in mvpp2_txq_bufs_free()
2872 			   struct mvpp2_txq_pcpu *txq_pcpu)  in mvpp2_txq_done()  argument
2877 	if (txq_pcpu->thread != mvpp2_cpu_to_thread(port->priv, smp_processor_id()))  in mvpp2_txq_done()
2883 	mvpp2_txq_bufs_free(port, txq, txq_pcpu, tx_done);  in mvpp2_txq_done()
2885 	txq_pcpu->count -= tx_done;  in mvpp2_txq_done()
2888 		if (txq_pcpu->count <= txq_pcpu->wake_threshold)  in mvpp2_txq_done()
2896 	struct mvpp2_txq_pcpu *txq_pcpu;  in mvpp2_tx_done()  local
2904 		txq_pcpu = per_cpu_ptr(txq->pcpu, thread);  in mvpp2_tx_done()
2906 		if (txq_pcpu->count) {  in mvpp2_tx_done()
2907 			mvpp2_txq_done(port, txq, txq_pcpu);  in mvpp2_tx_done()
2908 			tx_todo += txq_pcpu->count;  in mvpp2_tx_done()
3109 	struct mvpp2_txq_pcpu *txq_pcpu;  in mvpp2_txq_init()  local
3165 		txq_pcpu = per_cpu_ptr(txq->pcpu, thread);  in mvpp2_txq_init()
3166 		txq_pcpu->size = txq->size;  in mvpp2_txq_init()
3167 		txq_pcpu->buffs = kmalloc_array(txq_pcpu->size,  in mvpp2_txq_init()
3168 						sizeof(*txq_pcpu->buffs),  in mvpp2_txq_init()
3170 		if (!txq_pcpu->buffs)  in mvpp2_txq_init()
3173 		txq_pcpu->count = 0;  in mvpp2_txq_init()
3174 		txq_pcpu->reserved_num = 0;  in mvpp2_txq_init()
3175 		txq_pcpu->txq_put_index = 0;  in mvpp2_txq_init()
3176 		txq_pcpu->txq_get_index = 0;  in mvpp2_txq_init()
3177 		txq_pcpu->tso_headers = NULL;  in mvpp2_txq_init()
3179 		txq_pcpu->stop_threshold = txq->size - MVPP2_MAX_SKB_DESCS;  in mvpp2_txq_init()
3180 		txq_pcpu->wake_threshold = txq_pcpu->stop_threshold / 2;  in mvpp2_txq_init()
3182 		txq_pcpu->tso_headers =  in mvpp2_txq_init()
3184 					   txq_pcpu->size * TSO_HEADER_SIZE,  in mvpp2_txq_init()
3185 					   &txq_pcpu->tso_headers_dma,  in mvpp2_txq_init()
3187 		if (!txq_pcpu->tso_headers)  in mvpp2_txq_init()
3198 	struct mvpp2_txq_pcpu *txq_pcpu;  in mvpp2_txq_deinit()  local
3202 		txq_pcpu = per_cpu_ptr(txq->pcpu, thread);  in mvpp2_txq_deinit()
3203 		kfree(txq_pcpu->buffs);  in mvpp2_txq_deinit()
3205 		if (txq_pcpu->tso_headers)  in mvpp2_txq_deinit()
3207 					  txq_pcpu->size * TSO_HEADER_SIZE,  in mvpp2_txq_deinit()
3208 					  txq_pcpu->tso_headers,  in mvpp2_txq_deinit()
3209 					  txq_pcpu->tso_headers_dma);  in mvpp2_txq_deinit()
3211 		txq_pcpu->tso_headers = NULL;  in mvpp2_txq_deinit()
3238 	struct mvpp2_txq_pcpu *txq_pcpu;  in mvpp2_txq_clean()  local
3272 		txq_pcpu = per_cpu_ptr(txq->pcpu, thread);  in mvpp2_txq_clean()
3275 		mvpp2_txq_bufs_free(port, txq, txq_pcpu, txq_pcpu->count);  in mvpp2_txq_clean()
3278 		txq_pcpu->count = 0;  in mvpp2_txq_clean()
3279 		txq_pcpu->txq_put_index = 0;  in mvpp2_txq_clean()
3280 		txq_pcpu->txq_get_index = 0;  in mvpp2_txq_clean()
3656 	struct mvpp2_txq_pcpu *txq_pcpu;  in mvpp2_xdp_finish_tx()  local
3661 	txq_pcpu = per_cpu_ptr(txq->pcpu, thread);  in mvpp2_xdp_finish_tx()
3665 	txq_pcpu->reserved_num -= nxmit;  in mvpp2_xdp_finish_tx()
3666 	txq_pcpu->count += nxmit;  in mvpp2_xdp_finish_tx()
3673 	if (txq_pcpu->count >= txq_pcpu->stop_threshold)  in mvpp2_xdp_finish_tx()
3677 	if (!port->has_tx_irqs && txq_pcpu->count >= txq->done_pkts_coal)  in mvpp2_xdp_finish_tx()
3678 		mvpp2_txq_done(port, txq, txq_pcpu);  in mvpp2_xdp_finish_tx()
3689 	struct mvpp2_txq_pcpu *txq_pcpu;  in mvpp2_xdp_submit_frame()  local
3697 	txq_pcpu = per_cpu_ptr(txq->pcpu, thread);  in mvpp2_xdp_submit_frame()
3702 	    mvpp2_txq_reserved_desc_num_proc(port, txq, txq_pcpu, 1)) {  in mvpp2_xdp_submit_frame()
3739 	mvpp2_txq_inc_put(port, txq_pcpu, xdpf, tx_desc, buf_type);  in mvpp2_xdp_submit_frame()
4098 	struct mvpp2_txq_pcpu *txq_pcpu = per_cpu_ptr(txq->pcpu, thread);  in tx_desc_unmap_put()  local
4104 	if (!IS_TSO_HEADER(txq_pcpu, buf_dma_addr))  in tx_desc_unmap_put()
4204 	struct mvpp2_txq_pcpu *txq_pcpu = per_cpu_ptr(txq->pcpu, thread);  in mvpp2_tx_frag_process()  local
4232 			mvpp2_txq_inc_put(port, txq_pcpu, skb, tx_desc, MVPP2_TYPE_SKB);  in mvpp2_tx_frag_process()
4236 			mvpp2_txq_inc_put(port, txq_pcpu, NULL, tx_desc, MVPP2_TYPE_SKB);  in mvpp2_tx_frag_process()
4257 				     struct mvpp2_txq_pcpu *txq_pcpu,  in mvpp2_tso_put_hdr()  argument
4268 	addr = txq_pcpu->tso_headers_dma +  in mvpp2_tso_put_hdr()
4269 	       txq_pcpu->txq_put_index * TSO_HEADER_SIZE;  in mvpp2_tso_put_hdr()
4275 	mvpp2_txq_inc_put(port, txq_pcpu, NULL, tx_desc, MVPP2_TYPE_SKB);  in mvpp2_tso_put_hdr()
4282 				     struct mvpp2_txq_pcpu *txq_pcpu,  in mvpp2_tso_put_data()  argument
4305 			mvpp2_txq_inc_put(port, txq_pcpu, skb, tx_desc, MVPP2_TYPE_SKB);  in mvpp2_tso_put_data()
4312 	mvpp2_txq_inc_put(port, txq_pcpu, NULL, tx_desc, MVPP2_TYPE_SKB);  in mvpp2_tso_put_data()
4319 			struct mvpp2_txq_pcpu *txq_pcpu)  in mvpp2_tx_tso()  argument
4327 	    mvpp2_txq_reserved_desc_num_proc(port, txq, txq_pcpu,  in mvpp2_tx_tso()
4336 		char *hdr = txq_pcpu->tso_headers +  in mvpp2_tx_tso()
4337 			    txq_pcpu->txq_put_index * TSO_HEADER_SIZE;  in mvpp2_tx_tso()
4343 		mvpp2_tso_put_hdr(skb, dev, txq, aggr_txq, txq_pcpu, hdr_sz);  in mvpp2_tx_tso()
4351 					       txq_pcpu, sz, left, len == 0))  in mvpp2_tx_tso()
4372 	struct mvpp2_txq_pcpu *txq_pcpu;  in mvpp2_tx()  local
4385 	txq_pcpu = per_cpu_ptr(txq->pcpu, thread);  in mvpp2_tx()
4392 		frags = mvpp2_tx_tso(skb, dev, txq, aggr_txq, txq_pcpu);  in mvpp2_tx()
4399 	    mvpp2_txq_reserved_desc_num_proc(port, txq, txq_pcpu, frags)) {  in mvpp2_tx()
4428 		mvpp2_txq_inc_put(port, txq_pcpu, skb, tx_desc, MVPP2_TYPE_SKB);  in mvpp2_tx()
4433 		mvpp2_txq_inc_put(port, txq_pcpu, NULL, tx_desc, MVPP2_TYPE_SKB);  in mvpp2_tx()
4447 		txq_pcpu->reserved_num -= frags;  in mvpp2_tx()
4448 		txq_pcpu->count += frags;  in mvpp2_tx()
4455 		if (txq_pcpu->count >= txq_pcpu->stop_threshold)  in mvpp2_tx()
4468 	if (!port->has_tx_irqs && txq_pcpu->count >= txq->done_pkts_coal)  in mvpp2_tx()
4469 		mvpp2_txq_done(port, txq, txq_pcpu);  in mvpp2_tx()
4472 	if (!port->has_tx_irqs && txq_pcpu->count <= frags &&  in mvpp2_tx()
4473 	    txq_pcpu->count > 0) {  in mvpp2_tx()
5960 	struct mvpp2_txq_pcpu *txq_pcpu;  in mvpp2_port_init()  local
6018 			txq_pcpu = per_cpu_ptr(txq->pcpu, thread);  in mvpp2_port_init()
6019 			txq_pcpu->thread = thread;  in mvpp2_port_init()