80 static int t7xx_dpmaif_update_bat_wr_idx(struct dpmaif_ctrl *dpmaif_ctrl,  in t7xx_dpmaif_update_bat_wr_idx()  argument
83 	struct dpmaif_rx_queue *rxq = &dpmaif_ctrl->rxq[q_num];  in t7xx_dpmaif_update_bat_wr_idx()
88 		dev_err(dpmaif_ctrl->dev, "RX queue %d has not been started\n", rxq->index);  in t7xx_dpmaif_update_bat_wr_idx()
109 	dev_err(dpmaif_ctrl->dev, "RX BAT flow check fail\n");  in t7xx_dpmaif_update_bat_wr_idx()
113 static bool t7xx_alloc_and_map_skb_info(const struct dpmaif_ctrl *dpmaif_ctrl,  in t7xx_alloc_and_map_skb_info()  argument
123 	data_bus_addr = dma_map_single(dpmaif_ctrl->dev, skb->data, size, DMA_FROM_DEVICE);  in t7xx_alloc_and_map_skb_info()
124 	if (dma_mapping_error(dpmaif_ctrl->dev, data_bus_addr)) {  in t7xx_alloc_and_map_skb_info()
125 		dev_err_ratelimited(dpmaif_ctrl->dev, "DMA mapping error\n");  in t7xx_alloc_and_map_skb_info()
164 int t7xx_dpmaif_rx_buf_alloc(struct dpmaif_ctrl *dpmaif_ctrl,  in t7xx_dpmaif_rx_buf_alloc()  argument
195 		    !t7xx_alloc_and_map_skb_info(dpmaif_ctrl, bat_req->pkt_buf_sz, cur_skb))  in t7xx_dpmaif_rx_buf_alloc()
206 	ret = t7xx_dpmaif_update_bat_wr_idx(dpmaif_ctrl, q_num, i);  in t7xx_dpmaif_rx_buf_alloc()
213 		ret = t7xx_dpmaif_dl_snd_hw_bat_cnt(&dpmaif_ctrl->hw_info, i);  in t7xx_dpmaif_rx_buf_alloc()
217 		hw_wr_idx = t7xx_dpmaif_dl_get_bat_wr_idx(&dpmaif_ctrl->hw_info,  in t7xx_dpmaif_rx_buf_alloc()
221 			dev_err(dpmaif_ctrl->dev, "Write index mismatch in RX ring\n");  in t7xx_dpmaif_rx_buf_alloc()
230 		t7xx_unmap_bat_skb(dpmaif_ctrl->dev, bat_req->bat_skb, i);  in t7xx_dpmaif_rx_buf_alloc()
238 	struct dpmaif_hw_info *hw_info = &rxq->dpmaif_ctrl->hw_info;  in t7xx_dpmaifq_release_pit_entry()
246 		dev_err(rxq->dpmaif_ctrl->dev, "Invalid PIT release index\n");  in t7xx_dpmaifq_release_pit_entry()
258 		dev_err(rxq->dpmaif_ctrl->dev, "PIT release failure: %d\n", ret);  in t7xx_dpmaifq_release_pit_entry()
319 int t7xx_dpmaif_rx_frag_alloc(struct dpmaif_ctrl *dpmaif_ctrl, struct dpmaif_bat_request *bat_req,  in t7xx_dpmaif_rx_frag_alloc()  argument
333 		dev_err(dpmaif_ctrl->dev,  in t7xx_dpmaif_rx_frag_alloc()
355 			data_base_addr = dma_map_page(dpmaif_ctrl->dev, page, offset,  in t7xx_dpmaif_rx_frag_alloc()
357 			if (dma_mapping_error(dpmaif_ctrl->dev, data_base_addr)) {  in t7xx_dpmaif_rx_frag_alloc()
359 				dev_err(dpmaif_ctrl->dev, "DMA mapping fail\n");  in t7xx_dpmaif_rx_frag_alloc()
379 		t7xx_dpmaif_dl_snd_hw_frg_cnt(&dpmaif_ctrl->hw_info, i);  in t7xx_dpmaif_rx_frag_alloc()
385 				t7xx_unmap_bat_page(dpmaif_ctrl->dev, bat_req->bat_skb, i);  in t7xx_dpmaif_rx_frag_alloc()
397 	struct device *dev = rxq->dpmaif_ctrl->dev;  in t7xx_dpmaif_set_frag_to_skb()
437 		dev_err(rxq->dpmaif_ctrl->dev, "Failed to set frag data to skb: %d\n", ret);  in t7xx_dpmaif_get_frag()
503 	struct dpmaif_hw_info *hw_info = &rxq->dpmaif_ctrl->hw_info;  in t7xx_dpmaif_release_bat_entry()
581 		dev_err(rxq->dpmaif_ctrl->dev, "Release PKT BAT failed: %d\n", ret);  in t7xx_dpmaif_bat_release_and_add()
585 	ret = t7xx_dpmaif_rx_buf_alloc(rxq->dpmaif_ctrl, rxq->bat_req, rxq->index, bid_cnt, false);  in t7xx_dpmaif_bat_release_and_add()
587 		dev_err(rxq->dpmaif_ctrl->dev, "Allocate new RX buffer failed: %d\n", ret);  in t7xx_dpmaif_bat_release_and_add()
603 		dev_err(rxq->dpmaif_ctrl->dev, "Release BAT entry failed: %d\n", ret);  in t7xx_dpmaif_frag_bat_release_and_add()
607 	return t7xx_dpmaif_rx_frag_alloc(rxq->dpmaif_ctrl, rxq->bat_frag, bid_cnt, false);  in t7xx_dpmaif_frag_bat_release_and_add()
627 	struct device *dev = rxq->dpmaif_ctrl->dev;  in t7xx_dpmaif_set_data_to_skb()
671 		dev_err(rxq->dpmaif_ctrl->dev, "RX set data to skb failed: %d\n", ret);  in t7xx_dpmaif_get_rx_pkt()
681 	struct dpmaif_ctrl *dpmaif_ctrl = rxq->dpmaif_ctrl;  in t7xx_dpmaifq_rx_notify_hw()  local
684 	queue_work(dpmaif_ctrl->bat_release_wq, &dpmaif_ctrl->bat_release_work);  in t7xx_dpmaifq_rx_notify_hw()
688 		dev_err(dpmaif_ctrl->dev, "RXQ%u update PIT failed: %d\n", rxq->index, ret);  in t7xx_dpmaifq_rx_notify_hw()
696 	struct dpmaif_ctrl *dpmaif_ctrl = rxq->dpmaif_ctrl;  in t7xx_dpmaif_rx_skb()  local
714 	dpmaif_ctrl->callbacks->recv_skb(dpmaif_ctrl->t7xx_dev->ccmni_ctlb, skb, &rxq->napi);  in t7xx_dpmaif_rx_skb()
721 	struct device *dev = rxq->dpmaif_ctrl->dev;  in t7xx_dpmaif_rx_start()
805 	hw_wr_idx = t7xx_dpmaif_dl_dlq_pit_get_wr_idx(&rxq->dpmaif_ctrl->hw_info, rxq->index);  in t7xx_dpmaifq_poll_pit()
812 static int t7xx_dpmaif_napi_rx_data_collect(struct dpmaif_ctrl *dpmaif_ctrl,  in t7xx_dpmaif_napi_rx_data_collect()  argument
816 	struct dpmaif_rx_queue *rxq = &dpmaif_ctrl->rxq[q_num];  in t7xx_dpmaif_napi_rx_data_collect()
826 		dev_err(dpmaif_ctrl->dev, "dlq%u rx ERR:%d\n", rxq->index, ret);  in t7xx_dpmaif_napi_rx_data_collect()
834 	struct t7xx_pci_dev *t7xx_dev = rxq->dpmaif_ctrl->t7xx_dev;  in t7xx_dpmaif_napi_rx_poll()
843 		pm_runtime_put_autosuspend(rxq->dpmaif_ctrl->dev);  in t7xx_dpmaif_napi_rx_poll()
844 		dev_err(rxq->dpmaif_ctrl->dev, "Work RXQ: %d has not been started\n", rxq->index);  in t7xx_dpmaif_napi_rx_poll()
862 		int rx_cnt = t7xx_dpmaif_napi_rx_data_collect(rxq->dpmaif_ctrl, rxq->index,  in t7xx_dpmaif_napi_rx_poll()
873 		t7xx_dpmaif_clr_ip_busy_sts(&rxq->dpmaif_ctrl->hw_info);  in t7xx_dpmaif_napi_rx_poll()
876 		t7xx_dpmaif_clr_ip_busy_sts(&rxq->dpmaif_ctrl->hw_info);  in t7xx_dpmaif_napi_rx_poll()
877 		t7xx_dpmaif_dlq_unmask_rx_done(&rxq->dpmaif_ctrl->hw_info, rxq->index);  in t7xx_dpmaif_napi_rx_poll()
878 		t7xx_pci_enable_sleep(rxq->dpmaif_ctrl->t7xx_dev);  in t7xx_dpmaif_napi_rx_poll()
879 		pm_runtime_mark_last_busy(rxq->dpmaif_ctrl->dev);  in t7xx_dpmaif_napi_rx_poll()
880 		pm_runtime_put_autosuspend(rxq->dpmaif_ctrl->dev);  in t7xx_dpmaif_napi_rx_poll()
883 		t7xx_dpmaif_clr_ip_busy_sts(&rxq->dpmaif_ctrl->hw_info);  in t7xx_dpmaif_napi_rx_poll()
889 void t7xx_dpmaif_irq_rx_done(struct dpmaif_ctrl *dpmaif_ctrl, const unsigned int que_mask)  in t7xx_dpmaif_irq_rx_done()  argument
892 	struct dpmaif_ctrl *ctrl;  in t7xx_dpmaif_irq_rx_done()
897 		dev_err(dpmaif_ctrl->dev, "Invalid RXQ number: %u\n", qno);  in t7xx_dpmaif_irq_rx_done()
901 	rxq = &dpmaif_ctrl->rxq[qno];  in t7xx_dpmaif_irq_rx_done()
902 	ctrl = rxq->dpmaif_ctrl;  in t7xx_dpmaif_irq_rx_done()
917 static void t7xx_dpmaif_base_free(const struct dpmaif_ctrl *dpmaif_ctrl,  in t7xx_dpmaif_base_free()  argument
921 		dma_free_coherent(dpmaif_ctrl->dev,  in t7xx_dpmaif_base_free()
939 int t7xx_dpmaif_bat_alloc(const struct dpmaif_ctrl *dpmaif_ctrl, struct dpmaif_bat_request *bat_req,  in t7xx_dpmaif_bat_alloc()  argument
958 	bat_req->bat_base = dma_alloc_coherent(dpmaif_ctrl->dev,  in t7xx_dpmaif_bat_alloc()
965 	bat_req->bat_skb = devm_kzalloc(dpmaif_ctrl->dev, bat_req->bat_size_cnt * sw_buf_size,  in t7xx_dpmaif_bat_alloc()
979 	t7xx_dpmaif_base_free(dpmaif_ctrl, bat_req);  in t7xx_dpmaif_bat_alloc()
984 void t7xx_dpmaif_bat_free(const struct dpmaif_ctrl *dpmaif_ctrl, struct dpmaif_bat_request *bat_req)  in t7xx_dpmaif_bat_free()  argument
997 				t7xx_unmap_bat_page(dpmaif_ctrl->dev, bat_req->bat_skb, i);  in t7xx_dpmaif_bat_free()
999 				t7xx_unmap_bat_skb(dpmaif_ctrl->dev, bat_req->bat_skb, i);  in t7xx_dpmaif_bat_free()
1003 	t7xx_dpmaif_base_free(dpmaif_ctrl, bat_req);  in t7xx_dpmaif_bat_free()
1016 	rxq->pit_base = dma_alloc_coherent(rxq->dpmaif_ctrl->dev,  in t7xx_dpmaif_rx_alloc()
1022 	rxq->bat_req = &rxq->dpmaif_ctrl->bat_req;  in t7xx_dpmaif_rx_alloc()
1025 	rxq->bat_frag = &rxq->dpmaif_ctrl->bat_frag;  in t7xx_dpmaif_rx_alloc()
1032 	if (!rxq->dpmaif_ctrl)  in t7xx_dpmaif_rx_buf_free()
1035 	t7xx_dpmaif_bat_free(rxq->dpmaif_ctrl, rxq->bat_req);  in t7xx_dpmaif_rx_buf_free()
1036 	t7xx_dpmaif_bat_free(rxq->dpmaif_ctrl, rxq->bat_frag);  in t7xx_dpmaif_rx_buf_free()
1039 		dma_free_coherent(rxq->dpmaif_ctrl->dev,  in t7xx_dpmaif_rx_buf_free()
1050 		dev_err(queue->dpmaif_ctrl->dev, "Failed to allocate RX buffers: %d\n", ret);  in t7xx_dpmaif_rxq_init()
1062 	struct dpmaif_ctrl *dpmaif_ctrl = container_of(work, struct dpmaif_ctrl, bat_release_work);  in t7xx_dpmaif_bat_release_work()  local
1066 	ret = pm_runtime_resume_and_get(dpmaif_ctrl->dev);  in t7xx_dpmaif_bat_release_work()
1070 	t7xx_pci_disable_sleep(dpmaif_ctrl->t7xx_dev);  in t7xx_dpmaif_bat_release_work()
1073 	rxq = &dpmaif_ctrl->rxq[DPF_RX_QNO_DFT];  in t7xx_dpmaif_bat_release_work()
1074 	if (t7xx_pci_sleep_disable_complete(dpmaif_ctrl->t7xx_dev)) {  in t7xx_dpmaif_bat_release_work()
1079 	t7xx_pci_enable_sleep(dpmaif_ctrl->t7xx_dev);  in t7xx_dpmaif_bat_release_work()
1080 	pm_runtime_mark_last_busy(dpmaif_ctrl->dev);  in t7xx_dpmaif_bat_release_work()
1081 	pm_runtime_put_autosuspend(dpmaif_ctrl->dev);  in t7xx_dpmaif_bat_release_work()
1084 int t7xx_dpmaif_bat_rel_wq_alloc(struct dpmaif_ctrl *dpmaif_ctrl)  in t7xx_dpmaif_bat_rel_wq_alloc()  argument
1086 	dpmaif_ctrl->bat_release_wq = alloc_workqueue("dpmaif_bat_release_work_queue",  in t7xx_dpmaif_bat_rel_wq_alloc()
1088 	if (!dpmaif_ctrl->bat_release_wq)  in t7xx_dpmaif_bat_rel_wq_alloc()
1091 	INIT_WORK(&dpmaif_ctrl->bat_release_work, t7xx_dpmaif_bat_release_work);  in t7xx_dpmaif_bat_rel_wq_alloc()
1095 void t7xx_dpmaif_bat_wq_rel(struct dpmaif_ctrl *dpmaif_ctrl)  in t7xx_dpmaif_bat_wq_rel()  argument
1097 	flush_work(&dpmaif_ctrl->bat_release_work);  in t7xx_dpmaif_bat_wq_rel()
1099 	if (dpmaif_ctrl->bat_release_wq) {  in t7xx_dpmaif_bat_wq_rel()
1100 		destroy_workqueue(dpmaif_ctrl->bat_release_wq);  in t7xx_dpmaif_bat_wq_rel()
1101 		dpmaif_ctrl->bat_release_wq = NULL;  in t7xx_dpmaif_bat_wq_rel()
1111 void t7xx_dpmaif_rx_stop(struct dpmaif_ctrl *dpmaif_ctrl)  in t7xx_dpmaif_rx_stop()  argument
1116 		struct dpmaif_rx_queue *rxq = &dpmaif_ctrl->rxq[i];  in t7xx_dpmaif_rx_stop()
1122 			dev_err(dpmaif_ctrl->dev, "Stop RX SW failed\n");  in t7xx_dpmaif_rx_stop()
1141 			dev_err(rxq->dpmaif_ctrl->dev, "Stop RX SW failed, %d\n", cnt);  in t7xx_dpmaif_stop_rxq()
1162 void t7xx_dpmaif_rx_clear(struct dpmaif_ctrl *dpmaif_ctrl)  in t7xx_dpmaif_rx_clear()  argument
1167 		t7xx_dpmaif_stop_rxq(&dpmaif_ctrl->rxq[i]);  in t7xx_dpmaif_rx_clear()