217  * struct mtk_hsdma_device - This is the struct holding info describing HSDMA
263 static struct device *hsdma2dev(struct mtk_hsdma_device *hsdma)  in hsdma2dev()  argument
265 	return hsdma->ddev.dev;  in hsdma2dev()
268 static u32 mtk_dma_read(struct mtk_hsdma_device *hsdma, u32 reg)  in mtk_dma_read()  argument
270 	return readl(hsdma->base + reg);  in mtk_dma_read()
273 static void mtk_dma_write(struct mtk_hsdma_device *hsdma, u32 reg, u32 val)  in mtk_dma_write()  argument
275 	writel(val, hsdma->base + reg);  in mtk_dma_write()
278 static void mtk_dma_rmw(struct mtk_hsdma_device *hsdma, u32 reg,  in mtk_dma_rmw()  argument
283 	val = mtk_dma_read(hsdma, reg);  in mtk_dma_rmw()
286 	mtk_dma_write(hsdma, reg, val);  in mtk_dma_rmw()
289 static void mtk_dma_set(struct mtk_hsdma_device *hsdma, u32 reg, u32 val)  in mtk_dma_set()  argument
291 	mtk_dma_rmw(hsdma, reg, 0, val);  in mtk_dma_set()
294 static void mtk_dma_clr(struct mtk_hsdma_device *hsdma, u32 reg, u32 val)  in mtk_dma_clr()  argument
296 	mtk_dma_rmw(hsdma, reg, val, 0);  in mtk_dma_clr()
304 static int mtk_hsdma_busy_wait(struct mtk_hsdma_device *hsdma)  in mtk_hsdma_busy_wait()  argument
308 	return readl_poll_timeout(hsdma->base + MTK_HSDMA_GLO, status,  in mtk_hsdma_busy_wait()
314 static int mtk_hsdma_alloc_pchan(struct mtk_hsdma_device *hsdma,  in mtk_hsdma_alloc_pchan()  argument
327 	ring->txd = dma_alloc_coherent(hsdma2dev(hsdma), pc->sz_ring,  in mtk_hsdma_alloc_pchan()
345 	/* Disable HSDMA and wait for the completion */  in mtk_hsdma_alloc_pchan()
346 	mtk_dma_clr(hsdma, MTK_HSDMA_GLO, MTK_HSDMA_GLO_DMA);  in mtk_hsdma_alloc_pchan()
347 	err = mtk_hsdma_busy_wait(hsdma);  in mtk_hsdma_alloc_pchan()
352 	mtk_dma_set(hsdma, MTK_HSDMA_RESET,  in mtk_hsdma_alloc_pchan()
354 	mtk_dma_clr(hsdma, MTK_HSDMA_RESET,  in mtk_hsdma_alloc_pchan()
357 	/* Setup HSDMA initial pointer in the ring */  in mtk_hsdma_alloc_pchan()
358 	mtk_dma_write(hsdma, MTK_HSDMA_TX_BASE, ring->tphys);  in mtk_hsdma_alloc_pchan()
359 	mtk_dma_write(hsdma, MTK_HSDMA_TX_CNT, MTK_DMA_SIZE);  in mtk_hsdma_alloc_pchan()
360 	mtk_dma_write(hsdma, MTK_HSDMA_TX_CPU, ring->cur_tptr);  in mtk_hsdma_alloc_pchan()
361 	mtk_dma_write(hsdma, MTK_HSDMA_TX_DMA, 0);  in mtk_hsdma_alloc_pchan()
362 	mtk_dma_write(hsdma, MTK_HSDMA_RX_BASE, ring->rphys);  in mtk_hsdma_alloc_pchan()
363 	mtk_dma_write(hsdma, MTK_HSDMA_RX_CNT, MTK_DMA_SIZE);  in mtk_hsdma_alloc_pchan()
364 	mtk_dma_write(hsdma, MTK_HSDMA_RX_CPU, ring->cur_rptr);  in mtk_hsdma_alloc_pchan()
365 	mtk_dma_write(hsdma, MTK_HSDMA_RX_DMA, 0);  in mtk_hsdma_alloc_pchan()
367 	/* Enable HSDMA */  in mtk_hsdma_alloc_pchan()
368 	mtk_dma_set(hsdma, MTK_HSDMA_GLO, MTK_HSDMA_GLO_DMA);  in mtk_hsdma_alloc_pchan()
371 	mtk_dma_write(hsdma, MTK_HSDMA_DLYINT, MTK_HSDMA_DLYINT_DEFAULT);  in mtk_hsdma_alloc_pchan()
374 	mtk_dma_set(hsdma, MTK_HSDMA_INT_ENABLE, MTK_HSDMA_INT_RXDONE);  in mtk_hsdma_alloc_pchan()
382 	dma_free_coherent(hsdma2dev(hsdma),  in mtk_hsdma_alloc_pchan()
387 static void mtk_hsdma_free_pchan(struct mtk_hsdma_device *hsdma,  in mtk_hsdma_free_pchan()  argument
392 	/* Disable HSDMA and then wait for the completion */  in mtk_hsdma_free_pchan()
393 	mtk_dma_clr(hsdma, MTK_HSDMA_GLO, MTK_HSDMA_GLO_DMA);  in mtk_hsdma_free_pchan()
394 	mtk_hsdma_busy_wait(hsdma);  in mtk_hsdma_free_pchan()
397 	mtk_dma_clr(hsdma, MTK_HSDMA_INT_ENABLE, MTK_HSDMA_INT_RXDONE);  in mtk_hsdma_free_pchan()
398 	mtk_dma_write(hsdma, MTK_HSDMA_TX_BASE, 0);  in mtk_hsdma_free_pchan()
399 	mtk_dma_write(hsdma, MTK_HSDMA_TX_CNT, 0);  in mtk_hsdma_free_pchan()
400 	mtk_dma_write(hsdma, MTK_HSDMA_TX_CPU, 0);  in mtk_hsdma_free_pchan()
401 	mtk_dma_write(hsdma, MTK_HSDMA_RX_BASE, 0);  in mtk_hsdma_free_pchan()
402 	mtk_dma_write(hsdma, MTK_HSDMA_RX_CNT, 0);  in mtk_hsdma_free_pchan()
403 	mtk_dma_write(hsdma, MTK_HSDMA_RX_CPU, MTK_DMA_SIZE - 1);  in mtk_hsdma_free_pchan()
407 	dma_free_coherent(hsdma2dev(hsdma),  in mtk_hsdma_free_pchan()
411 static int mtk_hsdma_issue_pending_vdesc(struct mtk_hsdma_device *hsdma,  in mtk_hsdma_issue_pending_vdesc()  argument
421 	spin_lock_irqsave(&hsdma->lock, flags);  in mtk_hsdma_issue_pending_vdesc()
431 		spin_unlock_irqrestore(&hsdma->lock, flags);  in mtk_hsdma_issue_pending_vdesc()
452 			   hsdma->soc->ls0 | MTK_HSDMA_DESC_PLEN(tlen));  in mtk_hsdma_issue_pending_vdesc()
484 	 * Updating into hardware the pointer of TX ring lets HSDMA to take  in mtk_hsdma_issue_pending_vdesc()
487 	mtk_dma_write(hsdma, MTK_HSDMA_TX_CPU, ring->cur_tptr);  in mtk_hsdma_issue_pending_vdesc()
489 	spin_unlock_irqrestore(&hsdma->lock, flags);  in mtk_hsdma_issue_pending_vdesc()
494 static void mtk_hsdma_issue_vchan_pending(struct mtk_hsdma_device *hsdma,  in mtk_hsdma_issue_vchan_pending()  argument
508 		err = mtk_hsdma_issue_pending_vdesc(hsdma, hsdma->pc, hvd);  in mtk_hsdma_issue_vchan_pending()
532 static void mtk_hsdma_free_rooms_in_ring(struct mtk_hsdma_device *hsdma)  in mtk_hsdma_free_rooms_in_ring()  argument
545 	status = mtk_dma_read(hsdma, MTK_HSDMA_INT_STATUS);  in mtk_hsdma_free_rooms_in_ring()
549 	pc = hsdma->pc;  in mtk_hsdma_free_rooms_in_ring()
568 		if (!(desc2 & hsdma->soc->ddone))  in mtk_hsdma_free_rooms_in_ring()
573 			dev_err(hsdma2dev(hsdma), "cb->vd cannot be null\n");  in mtk_hsdma_free_rooms_in_ring()
621 	mtk_dma_write(hsdma, MTK_HSDMA_RX_CPU, pc->ring.cur_rptr);  in mtk_hsdma_free_rooms_in_ring()
629 		mtk_dma_write(hsdma, MTK_HSDMA_INT_STATUS, status);  in mtk_hsdma_free_rooms_in_ring()
632 	for (i = 0; i < hsdma->dma_requests; i++) {  in mtk_hsdma_free_rooms_in_ring()
633 		hvc = &hsdma->vc[i];  in mtk_hsdma_free_rooms_in_ring()
635 		mtk_hsdma_issue_vchan_pending(hsdma, hvc);  in mtk_hsdma_free_rooms_in_ring()
641 	mtk_dma_set(hsdma, MTK_HSDMA_INT_ENABLE, MTK_HSDMA_INT_RXDONE);  in mtk_hsdma_free_rooms_in_ring()
646 	struct mtk_hsdma_device *hsdma = devid;  in mtk_hsdma_irq()  local
652 	mtk_dma_clr(hsdma, MTK_HSDMA_INT_ENABLE, MTK_HSDMA_INT_RXDONE);  in mtk_hsdma_irq()
654 	mtk_hsdma_free_rooms_in_ring(hsdma);  in mtk_hsdma_irq()
707 	struct mtk_hsdma_device *hsdma = to_hsdma_dev(c);  in mtk_hsdma_issue_pending()  local
714 		mtk_hsdma_issue_vchan_pending(hsdma, hvc);  in mtk_hsdma_issue_pending()
809 	struct mtk_hsdma_device *hsdma = to_hsdma_dev(c);  in mtk_hsdma_alloc_chan_resources()  local
813 	 * Since HSDMA has only one PC, the resource for PC is being allocated  in mtk_hsdma_alloc_chan_resources()
817 	if (!refcount_read(&hsdma->pc_refcnt)) {  in mtk_hsdma_alloc_chan_resources()
818 		err = mtk_hsdma_alloc_pchan(hsdma, hsdma->pc);  in mtk_hsdma_alloc_chan_resources()
825 		refcount_set(&hsdma->pc_refcnt, 1);  in mtk_hsdma_alloc_chan_resources()
827 		refcount_inc(&hsdma->pc_refcnt);  in mtk_hsdma_alloc_chan_resources()
835 	struct mtk_hsdma_device *hsdma = to_hsdma_dev(c);  in mtk_hsdma_free_chan_resources()  local
841 	if (!refcount_dec_and_test(&hsdma->pc_refcnt))  in mtk_hsdma_free_chan_resources()
844 	mtk_hsdma_free_pchan(hsdma, hsdma->pc);  in mtk_hsdma_free_chan_resources()
847 static int mtk_hsdma_hw_init(struct mtk_hsdma_device *hsdma)  in mtk_hsdma_hw_init()  argument
851 	pm_runtime_enable(hsdma2dev(hsdma));  in mtk_hsdma_hw_init()
852 	pm_runtime_get_sync(hsdma2dev(hsdma));  in mtk_hsdma_hw_init()
854 	err = clk_prepare_enable(hsdma->clk);  in mtk_hsdma_hw_init()
858 	mtk_dma_write(hsdma, MTK_HSDMA_INT_ENABLE, 0);  in mtk_hsdma_hw_init()
859 	mtk_dma_write(hsdma, MTK_HSDMA_GLO, MTK_HSDMA_GLO_DEFAULT);  in mtk_hsdma_hw_init()
864 static int mtk_hsdma_hw_deinit(struct mtk_hsdma_device *hsdma)  in mtk_hsdma_hw_deinit()  argument
866 	mtk_dma_write(hsdma, MTK_HSDMA_GLO, 0);  in mtk_hsdma_hw_deinit()
868 	clk_disable_unprepare(hsdma->clk);  in mtk_hsdma_hw_deinit()
870 	pm_runtime_put_sync(hsdma2dev(hsdma));  in mtk_hsdma_hw_deinit()
871 	pm_runtime_disable(hsdma2dev(hsdma));  in mtk_hsdma_hw_deinit()
887 	{ .compatible = "mediatek,mt7623-hsdma", .data = &mt7623_soc},
888 	{ .compatible = "mediatek,mt7622-hsdma", .data = &mt7622_soc},
895 	struct mtk_hsdma_device *hsdma;  in mtk_hsdma_probe()  local
900 	hsdma = devm_kzalloc(&pdev->dev, sizeof(*hsdma), GFP_KERNEL);  in mtk_hsdma_probe()
901 	if (!hsdma)  in mtk_hsdma_probe()
904 	dd = &hsdma->ddev;  in mtk_hsdma_probe()
906 	hsdma->base = devm_platform_ioremap_resource(pdev, 0);  in mtk_hsdma_probe()
907 	if (IS_ERR(hsdma->base))  in mtk_hsdma_probe()
908 		return PTR_ERR(hsdma->base);  in mtk_hsdma_probe()
910 	hsdma->soc = of_device_get_match_data(&pdev->dev);  in mtk_hsdma_probe()
911 	if (!hsdma->soc) {  in mtk_hsdma_probe()
916 	hsdma->clk = devm_clk_get(&pdev->dev, "hsdma");  in mtk_hsdma_probe()
917 	if (IS_ERR(hsdma->clk)) {  in mtk_hsdma_probe()
920 		return PTR_ERR(hsdma->clk);  in mtk_hsdma_probe()
926 	hsdma->irq = err;  in mtk_hsdma_probe()
928 	refcount_set(&hsdma->pc_refcnt, 0);  in mtk_hsdma_probe()
929 	spin_lock_init(&hsdma->lock);  in mtk_hsdma_probe()
947 	hsdma->dma_requests = MTK_HSDMA_NR_VCHANS;  in mtk_hsdma_probe()
950 						      &hsdma->dma_requests)) {  in mtk_hsdma_probe()
956 	hsdma->pc = devm_kcalloc(&pdev->dev, MTK_HSDMA_NR_MAX_PCHANS,  in mtk_hsdma_probe()
957 				 sizeof(*hsdma->pc), GFP_KERNEL);  in mtk_hsdma_probe()
958 	if (!hsdma->pc)  in mtk_hsdma_probe()
961 	hsdma->vc = devm_kcalloc(&pdev->dev, hsdma->dma_requests,  in mtk_hsdma_probe()
962 				 sizeof(*hsdma->vc), GFP_KERNEL);  in mtk_hsdma_probe()
963 	if (!hsdma->vc)  in mtk_hsdma_probe()
966 	for (i = 0; i < hsdma->dma_requests; i++) {  in mtk_hsdma_probe()
967 		vc = &hsdma->vc[i];  in mtk_hsdma_probe()
979 					 of_dma_xlate_by_chan_id, hsdma);  in mtk_hsdma_probe()
982 			"MediaTek HSDMA OF registration failed %d\n", err);  in mtk_hsdma_probe()
986 	mtk_hsdma_hw_init(hsdma);  in mtk_hsdma_probe()
988 	err = devm_request_irq(&pdev->dev, hsdma->irq,  in mtk_hsdma_probe()
990 			       dev_name(&pdev->dev), hsdma);  in mtk_hsdma_probe()
997 	platform_set_drvdata(pdev, hsdma);  in mtk_hsdma_probe()
999 	dev_info(&pdev->dev, "MediaTek HSDMA driver registered\n");  in mtk_hsdma_probe()
1004 	mtk_hsdma_hw_deinit(hsdma);  in mtk_hsdma_probe()
1014 	struct mtk_hsdma_device *hsdma = platform_get_drvdata(pdev);  in mtk_hsdma_remove()  local
1019 	for (i = 0; i < hsdma->dma_requests; i++) {  in mtk_hsdma_remove()
1020 		vc = &hsdma->vc[i];  in mtk_hsdma_remove()
1027 	mtk_dma_write(hsdma, MTK_HSDMA_INT_ENABLE, 0);  in mtk_hsdma_remove()
1030 	synchronize_irq(hsdma->irq);  in mtk_hsdma_remove()
1033 	mtk_hsdma_hw_deinit(hsdma);  in mtk_hsdma_remove()
1035 	dma_async_device_unregister(&hsdma->ddev);  in mtk_hsdma_remove()