51 static unsigned int t7xx_dpmaif_update_drb_rd_idx(struct dpmaif_ctrl *dpmaif_ctrl,  in t7xx_dpmaif_update_drb_rd_idx()  argument
54 	struct dpmaif_tx_queue *txq = &dpmaif_ctrl->txq[q_num];  in t7xx_dpmaif_update_drb_rd_idx()
62 	new_hw_rd_idx = t7xx_dpmaif_ul_get_rd_idx(&dpmaif_ctrl->hw_info, q_num);  in t7xx_dpmaif_update_drb_rd_idx()
64 		dev_err(dpmaif_ctrl->dev, "Out of range read index: %u\n", new_hw_rd_idx);  in t7xx_dpmaif_update_drb_rd_idx()
80 static unsigned int t7xx_dpmaif_release_tx_buffer(struct dpmaif_ctrl *dpmaif_ctrl,  in t7xx_dpmaif_release_tx_buffer()  argument
83 	struct dpmaif_tx_queue *txq = &dpmaif_ctrl->txq[q_num];  in t7xx_dpmaif_release_tx_buffer()
84 	struct dpmaif_callbacks *cb = dpmaif_ctrl->callbacks;  in t7xx_dpmaif_release_tx_buffer()
103 				dma_unmap_single(dpmaif_ctrl->dev, cur_drb_skb->bus_addr,  in t7xx_dpmaif_release_tx_buffer()
108 					dev_err(dpmaif_ctrl->dev,  in t7xx_dpmaif_release_tx_buffer()
125 			cb->state_notify(dpmaif_ctrl->t7xx_dev, DMPAIF_TXQ_STATE_IRQ, txq->index);  in t7xx_dpmaif_release_tx_buffer()
129 		dev_err(dpmaif_ctrl->dev, "txq%u: DRB not marked as the last one\n", q_num);  in t7xx_dpmaif_release_tx_buffer()
134 static int t7xx_dpmaif_tx_release(struct dpmaif_ctrl *dpmaif_ctrl,  in t7xx_dpmaif_tx_release()  argument
137 	struct dpmaif_tx_queue *txq = &dpmaif_ctrl->txq[q_num];  in t7xx_dpmaif_tx_release()
141 	t7xx_dpmaif_update_drb_rd_idx(dpmaif_ctrl, q_num);  in t7xx_dpmaif_tx_release()
148 		real_rel_cnt = t7xx_dpmaif_release_tx_buffer(dpmaif_ctrl, q_num, real_rel_cnt);  in t7xx_dpmaif_tx_release()
155 	return !!t7xx_dpmaif_update_drb_rd_idx(txq->dpmaif_ctrl, txq->index);  in t7xx_dpmaif_drb_ring_not_empty()
161 	struct dpmaif_ctrl *dpmaif_ctrl = txq->dpmaif_ctrl;  in t7xx_dpmaif_tx_done()  local
165 	ret = pm_runtime_resume_and_get(dpmaif_ctrl->dev);  in t7xx_dpmaif_tx_done()
170 	t7xx_pci_disable_sleep(dpmaif_ctrl->t7xx_dev);  in t7xx_dpmaif_tx_done()
171 	if (t7xx_pci_sleep_disable_complete(dpmaif_ctrl->t7xx_dev)) {  in t7xx_dpmaif_tx_done()
172 		hw_info = &dpmaif_ctrl->hw_info;  in t7xx_dpmaif_tx_done()
173 		ret = t7xx_dpmaif_tx_release(dpmaif_ctrl, txq->index, txq->drb_size_cnt);  in t7xx_dpmaif_tx_done()
177 			queue_work(dpmaif_ctrl->txq[txq->index].worker,  in t7xx_dpmaif_tx_done()
178 				   &dpmaif_ctrl->txq[txq->index].dpmaif_tx_work);  in t7xx_dpmaif_tx_done()
187 	t7xx_pci_enable_sleep(dpmaif_ctrl->t7xx_dev);  in t7xx_dpmaif_tx_done()
188 	pm_runtime_mark_last_busy(dpmaif_ctrl->dev);  in t7xx_dpmaif_tx_done()
189 	pm_runtime_put_autosuspend(dpmaif_ctrl->dev);  in t7xx_dpmaif_tx_done()
192 static void t7xx_setup_msg_drb(struct dpmaif_ctrl *dpmaif_ctrl, unsigned int q_num,  in t7xx_setup_msg_drb()  argument
196 	struct dpmaif_drb *drb_base = dpmaif_ctrl->txq[q_num].drb_base;  in t7xx_setup_msg_drb()
208 static void t7xx_setup_payload_drb(struct dpmaif_ctrl *dpmaif_ctrl, unsigned int q_num,  in t7xx_setup_payload_drb()  argument
212 	struct dpmaif_drb *drb_base = dpmaif_ctrl->txq[q_num].drb_base;  in t7xx_setup_payload_drb()
225 static void t7xx_record_drb_skb(struct dpmaif_ctrl *dpmaif_ctrl, unsigned int q_num,  in t7xx_record_drb_skb()  argument
230 	struct dpmaif_drb_skb *drb_skb_base = dpmaif_ctrl->txq[q_num].drb_skb_base;  in t7xx_record_drb_skb()
242 static int t7xx_dpmaif_add_skb_to_ring(struct dpmaif_ctrl *dpmaif_ctrl, struct sk_buff *skb)  in t7xx_dpmaif_add_skb_to_ring()  argument
244 	struct dpmaif_callbacks *cb = dpmaif_ctrl->callbacks;  in t7xx_dpmaif_add_skb_to_ring()
253 	txq = &dpmaif_ctrl->txq[skb_cb->txq_number];  in t7xx_dpmaif_add_skb_to_ring()
254 	if (!txq->que_started || dpmaif_ctrl->state != DPMAIF_STATE_PWRON)  in t7xx_dpmaif_add_skb_to_ring()
263 		dev_warn_ratelimited(dpmaif_ctrl->dev, "frag_list not supported\n");  in t7xx_dpmaif_add_skb_to_ring()
275 	t7xx_setup_msg_drb(dpmaif_ctrl, txq->index, cur_idx, skb->len, 0, skb_cb->netif_idx);  in t7xx_dpmaif_add_skb_to_ring()
276 	t7xx_record_drb_skb(dpmaif_ctrl, txq->index, cur_idx, skb, true, 0, 0, 0, 0);  in t7xx_dpmaif_add_skb_to_ring()
297 		bus_addr = dma_map_single(dpmaif_ctrl->dev, data_addr, data_len, DMA_TO_DEVICE);  in t7xx_dpmaif_add_skb_to_ring()
298 		if (dma_mapping_error(dpmaif_ctrl->dev, bus_addr))  in t7xx_dpmaif_add_skb_to_ring()
304 		t7xx_setup_payload_drb(dpmaif_ctrl, txq->index, cur_idx, bus_addr, data_len,  in t7xx_dpmaif_add_skb_to_ring()
306 		t7xx_record_drb_skb(dpmaif_ctrl, txq->index, cur_idx, skb, false, is_frag,  in t7xx_dpmaif_add_skb_to_ring()
312 		cb->state_notify(dpmaif_ctrl->t7xx_dev, DMPAIF_TXQ_STATE_FULL, txq->index);  in t7xx_dpmaif_add_skb_to_ring()
324 		dma_unmap_single(dpmaif_ctrl->dev, drb_skb->bus_addr,  in t7xx_dpmaif_add_skb_to_ring()
334 static bool t7xx_tx_lists_are_all_empty(const struct dpmaif_ctrl *dpmaif_ctrl)  in t7xx_tx_lists_are_all_empty()  argument
339 		if (!skb_queue_empty(&dpmaif_ctrl->txq[i].tx_skb_head))  in t7xx_tx_lists_are_all_empty()
347 static struct dpmaif_tx_queue *t7xx_select_tx_queue(struct dpmaif_ctrl *dpmaif_ctrl)  in t7xx_select_tx_queue()  argument
351 	txq = &dpmaif_ctrl->txq[DPMAIF_TX_DEFAULT_QUEUE];  in t7xx_select_tx_queue()
394 		ret = t7xx_dpmaif_add_skb_to_ring(txq->dpmaif_ctrl, skb);  in t7xx_txq_burst_send_skb()
396 			dev_err(txq->dpmaif_ctrl->dev,  in t7xx_txq_burst_send_skb()
411 static void t7xx_do_tx_hw_push(struct dpmaif_ctrl *dpmaif_ctrl)  in t7xx_do_tx_hw_push()  argument
419 		txq = t7xx_select_tx_queue(dpmaif_ctrl);  in t7xx_do_tx_hw_push()
432 			if (!t7xx_pci_sleep_disable_complete(dpmaif_ctrl->t7xx_dev))  in t7xx_do_tx_hw_push()
438 		t7xx_dpmaif_ul_update_hw_drb_cnt(&dpmaif_ctrl->hw_info, txq->index,  in t7xx_do_tx_hw_push()
442 	} while (!t7xx_tx_lists_are_all_empty(dpmaif_ctrl) && !kthread_should_stop() &&  in t7xx_do_tx_hw_push()
443 		 (dpmaif_ctrl->state == DPMAIF_STATE_PWRON));  in t7xx_do_tx_hw_push()
448 	struct dpmaif_ctrl *dpmaif_ctrl = arg;  in t7xx_dpmaif_tx_hw_push_thread()  local
452 		if (t7xx_tx_lists_are_all_empty(dpmaif_ctrl) ||  in t7xx_dpmaif_tx_hw_push_thread()
453 		    dpmaif_ctrl->state != DPMAIF_STATE_PWRON) {  in t7xx_dpmaif_tx_hw_push_thread()
454 			if (wait_event_interruptible(dpmaif_ctrl->tx_wq,  in t7xx_dpmaif_tx_hw_push_thread()
455 						     (!t7xx_tx_lists_are_all_empty(dpmaif_ctrl) &&  in t7xx_dpmaif_tx_hw_push_thread()
456 						     dpmaif_ctrl->state == DPMAIF_STATE_PWRON) ||  in t7xx_dpmaif_tx_hw_push_thread()
464 		ret = pm_runtime_resume_and_get(dpmaif_ctrl->dev);  in t7xx_dpmaif_tx_hw_push_thread()
468 		t7xx_pci_disable_sleep(dpmaif_ctrl->t7xx_dev);  in t7xx_dpmaif_tx_hw_push_thread()
469 		t7xx_do_tx_hw_push(dpmaif_ctrl);  in t7xx_dpmaif_tx_hw_push_thread()
470 		t7xx_pci_enable_sleep(dpmaif_ctrl->t7xx_dev);  in t7xx_dpmaif_tx_hw_push_thread()
471 		pm_runtime_mark_last_busy(dpmaif_ctrl->dev);  in t7xx_dpmaif_tx_hw_push_thread()
472 		pm_runtime_put_autosuspend(dpmaif_ctrl->dev);  in t7xx_dpmaif_tx_hw_push_thread()
478 int t7xx_dpmaif_tx_thread_init(struct dpmaif_ctrl *dpmaif_ctrl)  in t7xx_dpmaif_tx_thread_init()  argument
480 	init_waitqueue_head(&dpmaif_ctrl->tx_wq);  in t7xx_dpmaif_tx_thread_init()
481 	dpmaif_ctrl->tx_thread = kthread_run(t7xx_dpmaif_tx_hw_push_thread,  in t7xx_dpmaif_tx_thread_init()
482 					     dpmaif_ctrl, "dpmaif_tx_hw_push");  in t7xx_dpmaif_tx_thread_init()
483 	return PTR_ERR_OR_ZERO(dpmaif_ctrl->tx_thread);  in t7xx_dpmaif_tx_thread_init()
486 void t7xx_dpmaif_tx_thread_rel(struct dpmaif_ctrl *dpmaif_ctrl)  in t7xx_dpmaif_tx_thread_rel()  argument
488 	if (dpmaif_ctrl->tx_thread)  in t7xx_dpmaif_tx_thread_rel()
489 		kthread_stop(dpmaif_ctrl->tx_thread);  in t7xx_dpmaif_tx_thread_rel()
507 int t7xx_dpmaif_tx_send_skb(struct dpmaif_ctrl *dpmaif_ctrl, unsigned int txq_number,  in t7xx_dpmaif_tx_send_skb()  argument
510 	struct dpmaif_tx_queue *txq = &dpmaif_ctrl->txq[txq_number];  in t7xx_dpmaif_tx_send_skb()
511 	struct dpmaif_callbacks *cb = dpmaif_ctrl->callbacks;  in t7xx_dpmaif_tx_send_skb()
515 		cb->state_notify(dpmaif_ctrl->t7xx_dev, DMPAIF_TXQ_STATE_FULL, txq_number);  in t7xx_dpmaif_tx_send_skb()
522 	wake_up(&dpmaif_ctrl->tx_wq);  in t7xx_dpmaif_tx_send_skb()
527 void t7xx_dpmaif_irq_tx_done(struct dpmaif_ctrl *dpmaif_ctrl, unsigned int que_mask)  in t7xx_dpmaif_irq_tx_done()  argument
533 			queue_work(dpmaif_ctrl->txq[i].worker, &dpmaif_ctrl->txq[i].dpmaif_tx_work);  in t7xx_dpmaif_irq_tx_done()
547 	txq->drb_base = dma_alloc_coherent(txq->dpmaif_ctrl->dev, brb_pd_size,  in t7xx_dpmaif_tx_drb_buf_init()
553 	txq->drb_skb_base = devm_kzalloc(txq->dpmaif_ctrl->dev, brb_skb_size, GFP_KERNEL);  in t7xx_dpmaif_tx_drb_buf_init()
555 		dma_free_coherent(txq->dpmaif_ctrl->dev, brb_pd_size,  in t7xx_dpmaif_tx_drb_buf_init()
577 			dma_unmap_single(txq->dpmaif_ctrl->dev, drb_skb->bus_addr,  in t7xx_dpmaif_tx_free_drb_skb()
590 		dma_free_coherent(txq->dpmaif_ctrl->dev,  in t7xx_dpmaif_tx_drb_buf_rel()
617 		dev_err(txq->dpmaif_ctrl->dev, "Failed to initialize DRB buffers: %d\n", ret);  in t7xx_dpmaif_txq_init()
642 void t7xx_dpmaif_tx_stop(struct dpmaif_ctrl *dpmaif_ctrl)  in t7xx_dpmaif_tx_stop()  argument
650 		txq = &dpmaif_ctrl->txq[i];  in t7xx_dpmaif_tx_stop()
658 				dev_err(dpmaif_ctrl->dev, "TX queue stop failed\n");  in t7xx_dpmaif_tx_stop()
678 void t7xx_dpmaif_tx_clear(struct dpmaif_ctrl *dpmaif_ctrl)  in t7xx_dpmaif_tx_clear()  argument
683 		t7xx_dpmaif_txq_flush_rel(&dpmaif_ctrl->txq[i]);  in t7xx_dpmaif_tx_clear()