146 	struct hisi_dma_dev *hdma_dev;  member
345 static void hisi_dma_pause_dma(struct hisi_dma_dev *hdma_dev, u32 index,  in hisi_dma_pause_dma()  argument
350 	addr = hdma_dev->queue_base + HISI_DMA_Q_CTRL0 +  in hisi_dma_pause_dma()
355 static void hisi_dma_enable_dma(struct hisi_dma_dev *hdma_dev, u32 index,  in hisi_dma_enable_dma()  argument
360 	addr = hdma_dev->queue_base + HISI_DMA_Q_CTRL0 +  in hisi_dma_enable_dma()
365 static void hisi_dma_mask_irq(struct hisi_dma_dev *hdma_dev, u32 qp_index)  in hisi_dma_mask_irq()  argument
367 	void __iomem *q_base = hdma_dev->queue_base;  in hisi_dma_mask_irq()
369 	if (hdma_dev->reg_layout == HISI_DMA_REG_LAYOUT_HIP08)  in hisi_dma_mask_irq()
381 static void hisi_dma_unmask_irq(struct hisi_dma_dev *hdma_dev, u32 qp_index)  in hisi_dma_unmask_irq()  argument
383 	void __iomem *q_base = hdma_dev->queue_base;  in hisi_dma_unmask_irq()
385 	if (hdma_dev->reg_layout == HISI_DMA_REG_LAYOUT_HIP08) {  in hisi_dma_unmask_irq()
403 static void hisi_dma_do_reset(struct hisi_dma_dev *hdma_dev, u32 index)  in hisi_dma_do_reset()  argument
407 	addr = hdma_dev->queue_base +  in hisi_dma_do_reset()
412 static void hisi_dma_reset_qp_point(struct hisi_dma_dev *hdma_dev, u32 index)  in hisi_dma_reset_qp_point()  argument
414 	void __iomem *q_base = hdma_dev->queue_base;  in hisi_dma_reset_qp_point()
423 	struct hisi_dma_dev *hdma_dev = chan->hdma_dev;  in hisi_dma_reset_or_disable_hw_chan()  local
428 	hisi_dma_pause_dma(hdma_dev, index, true);  in hisi_dma_reset_or_disable_hw_chan()
429 	hisi_dma_enable_dma(hdma_dev, index, false);  in hisi_dma_reset_or_disable_hw_chan()
430 	hisi_dma_mask_irq(hdma_dev, index);  in hisi_dma_reset_or_disable_hw_chan()
432 	addr = hdma_dev->queue_base +  in hisi_dma_reset_or_disable_hw_chan()
439 		dev_err(&hdma_dev->pdev->dev, "disable channel timeout!\n");  in hisi_dma_reset_or_disable_hw_chan()
443 	hisi_dma_do_reset(hdma_dev, index);  in hisi_dma_reset_or_disable_hw_chan()
444 	hisi_dma_reset_qp_point(hdma_dev, index);  in hisi_dma_reset_or_disable_hw_chan()
445 	hisi_dma_pause_dma(hdma_dev, index, false);  in hisi_dma_reset_or_disable_hw_chan()
448 		hisi_dma_enable_dma(hdma_dev, index, true);  in hisi_dma_reset_or_disable_hw_chan()
449 		hisi_dma_unmask_irq(hdma_dev, index);  in hisi_dma_reset_or_disable_hw_chan()
456 		dev_err(&hdma_dev->pdev->dev, "reset channel timeout!\n");  in hisi_dma_reset_or_disable_hw_chan()
464 	struct hisi_dma_dev *hdma_dev = chan->hdma_dev;  in hisi_dma_free_chan_resources()  local
469 	memset(chan->sq, 0, sizeof(struct hisi_dma_sqe) * hdma_dev->chan_depth);  in hisi_dma_free_chan_resources()
470 	memset(chan->cq, 0, sizeof(struct hisi_dma_cqe) * hdma_dev->chan_depth);  in hisi_dma_free_chan_resources()
509 	struct hisi_dma_dev *hdma_dev = chan->hdma_dev;  in hisi_dma_start_transfer()  local
532 	chan->sq_tail = (chan->sq_tail + 1) % hdma_dev->chan_depth;  in hisi_dma_start_transfer()
535 	hisi_dma_chan_write(hdma_dev->queue_base, HISI_DMA_Q_SQ_TAIL_PTR,  in hisi_dma_start_transfer()
560 	hisi_dma_pause_dma(chan->hdma_dev, chan->qp_num, true);  in hisi_dma_terminate_all()
571 	hisi_dma_pause_dma(chan->hdma_dev, chan->qp_num, false);  in hisi_dma_terminate_all()
583 static int hisi_dma_alloc_qps_mem(struct hisi_dma_dev *hdma_dev)  in hisi_dma_alloc_qps_mem()  argument
585 	size_t sq_size = sizeof(struct hisi_dma_sqe) * hdma_dev->chan_depth;  in hisi_dma_alloc_qps_mem()
586 	size_t cq_size = sizeof(struct hisi_dma_cqe) * hdma_dev->chan_depth;  in hisi_dma_alloc_qps_mem()
587 	struct device *dev = &hdma_dev->pdev->dev;  in hisi_dma_alloc_qps_mem()
591 	for (i = 0; i < hdma_dev->chan_num; i++) {  in hisi_dma_alloc_qps_mem()
592 		chan = &hdma_dev->chan[i];  in hisi_dma_alloc_qps_mem()
607 static void hisi_dma_init_hw_qp(struct hisi_dma_dev *hdma_dev, u32 index)  in hisi_dma_init_hw_qp()  argument
609 	struct hisi_dma_chan *chan = &hdma_dev->chan[index];  in hisi_dma_init_hw_qp()
610 	void __iomem *q_base = hdma_dev->queue_base;  in hisi_dma_init_hw_qp()
611 	u32 hw_depth = hdma_dev->chan_depth - 1;  in hisi_dma_init_hw_qp()
638 	if (hdma_dev->reg_layout == HISI_DMA_REG_LAYOUT_HIP08) {  in hisi_dma_init_hw_qp()
692 static void hisi_dma_enable_qp(struct hisi_dma_dev *hdma_dev, u32 qp_index)  in hisi_dma_enable_qp()  argument
694 	hisi_dma_init_hw_qp(hdma_dev, qp_index);  in hisi_dma_enable_qp()
695 	hisi_dma_unmask_irq(hdma_dev, qp_index);  in hisi_dma_enable_qp()
696 	hisi_dma_enable_dma(hdma_dev, qp_index, true);  in hisi_dma_enable_qp()
699 static void hisi_dma_disable_qp(struct hisi_dma_dev *hdma_dev, u32 qp_index)  in hisi_dma_disable_qp()  argument
701 	hisi_dma_reset_or_disable_hw_chan(&hdma_dev->chan[qp_index], true);  in hisi_dma_disable_qp()
704 static void hisi_dma_enable_qps(struct hisi_dma_dev *hdma_dev)  in hisi_dma_enable_qps()  argument
708 	for (i = 0; i < hdma_dev->chan_num; i++) {  in hisi_dma_enable_qps()
709 		hdma_dev->chan[i].qp_num = i;  in hisi_dma_enable_qps()
710 		hdma_dev->chan[i].hdma_dev = hdma_dev;  in hisi_dma_enable_qps()
711 		hdma_dev->chan[i].vc.desc_free = hisi_dma_desc_free;  in hisi_dma_enable_qps()
712 		vchan_init(&hdma_dev->chan[i].vc, &hdma_dev->dma_dev);  in hisi_dma_enable_qps()
713 		hisi_dma_enable_qp(hdma_dev, i);  in hisi_dma_enable_qps()
717 static void hisi_dma_disable_qps(struct hisi_dma_dev *hdma_dev)  in hisi_dma_disable_qps()  argument
721 	for (i = 0; i < hdma_dev->chan_num; i++) {  in hisi_dma_disable_qps()
722 		hisi_dma_disable_qp(hdma_dev, i);  in hisi_dma_disable_qps()
723 		tasklet_kill(&hdma_dev->chan[i].vc.task);  in hisi_dma_disable_qps()
730 	struct hisi_dma_dev *hdma_dev = chan->hdma_dev;  in hisi_dma_irq()  local
739 	q_base = hdma_dev->queue_base;  in hisi_dma_irq()
741 		chan->cq_head = (chan->cq_head + 1) % hdma_dev->chan_depth;  in hisi_dma_irq()
748 			dev_err(&hdma_dev->pdev->dev, "task error!\n");  in hisi_dma_irq()
757 static int hisi_dma_request_qps_irq(struct hisi_dma_dev *hdma_dev)  in hisi_dma_request_qps_irq()  argument
759 	struct pci_dev *pdev = hdma_dev->pdev;  in hisi_dma_request_qps_irq()
762 	for (i = 0; i < hdma_dev->chan_num; i++) {  in hisi_dma_request_qps_irq()
765 				       &hdma_dev->chan[i]);  in hisi_dma_request_qps_irq()
774 static int hisi_dma_enable_hw_channels(struct hisi_dma_dev *hdma_dev)  in hisi_dma_enable_hw_channels()  argument
778 	ret = hisi_dma_alloc_qps_mem(hdma_dev);  in hisi_dma_enable_hw_channels()
780 		dev_err(&hdma_dev->pdev->dev, "fail to allocate qp memory!\n");  in hisi_dma_enable_hw_channels()
784 	ret = hisi_dma_request_qps_irq(hdma_dev);  in hisi_dma_enable_hw_channels()
786 		dev_err(&hdma_dev->pdev->dev, "fail to request qp irq!\n");  in hisi_dma_enable_hw_channels()
790 	hisi_dma_enable_qps(hdma_dev);  in hisi_dma_enable_hw_channels()
800 static void hisi_dma_set_mode(struct hisi_dma_dev *hdma_dev,  in hisi_dma_set_mode()  argument
803 	if (hdma_dev->reg_layout == HISI_DMA_REG_LAYOUT_HIP08)  in hisi_dma_set_mode()
805 			       hdma_dev->base + HISI_DMA_HIP08_MODE);  in hisi_dma_set_mode()
808 static void hisi_dma_init_hw(struct hisi_dma_dev *hdma_dev)  in hisi_dma_init_hw()  argument
813 	if (hdma_dev->reg_layout == HISI_DMA_REG_LAYOUT_HIP09) {  in hisi_dma_init_hw()
815 			addr = hdma_dev->base + HISI_DMA_HIP09_PORT_CFG_REG(i);  in hisi_dma_init_hw()
822 static void hisi_dma_init_dma_dev(struct hisi_dma_dev *hdma_dev)  in hisi_dma_init_dma_dev()  argument
826 	dma_dev = &hdma_dev->dma_dev;  in hisi_dma_init_dma_dev()
835 	dma_dev->dev = &hdma_dev->pdev->dev;  in hisi_dma_init_dma_dev()
842 static struct debugfs_reg32 *hisi_dma_get_ch_regs(struct hisi_dma_dev *hdma_dev,  in hisi_dma_get_ch_regs()  argument
845 	struct device *dev = &hdma_dev->pdev->dev;  in hisi_dma_get_ch_regs()
851 	if (hdma_dev->reg_layout == HISI_DMA_REG_LAYOUT_HIP08)  in hisi_dma_get_ch_regs()
862 	if (hdma_dev->reg_layout == HISI_DMA_REG_LAYOUT_HIP08)  in hisi_dma_get_ch_regs()
872 static int hisi_dma_create_chan_dir(struct hisi_dma_dev *hdma_dev)  in hisi_dma_create_chan_dir()  argument
883 	dev = &hdma_dev->pdev->dev;  in hisi_dma_create_chan_dir()
885 	regsets = devm_kcalloc(dev, hdma_dev->chan_num,  in hisi_dma_create_chan_dir()
890 	regs = hisi_dma_get_ch_regs(hdma_dev, &regs_sz);  in hisi_dma_create_chan_dir()
894 	for (i = 0; i < hdma_dev->chan_num; i++) {  in hisi_dma_create_chan_dir()
897 		regsets[i].base = hdma_dev->queue_base + i * HISI_DMA_Q_OFFSET;  in hisi_dma_create_chan_dir()
906 					      hdma_dev->dma_dev.dbg_dev_root);  in hisi_dma_create_chan_dir()
913 static void hisi_dma_create_debugfs(struct hisi_dma_dev *hdma_dev)  in hisi_dma_create_debugfs()  argument
919 	dev = &hdma_dev->pdev->dev;  in hisi_dma_create_debugfs()
921 	if (hdma_dev->dma_dev.dbg_dev_root == NULL)  in hisi_dma_create_debugfs()
928 	if (hdma_dev->reg_layout == HISI_DMA_REG_LAYOUT_HIP08) {  in hisi_dma_create_debugfs()
935 	regset->base = hdma_dev->base;  in hisi_dma_create_debugfs()
939 				hdma_dev->dma_dev.dbg_dev_root, regset);  in hisi_dma_create_debugfs()
941 	ret = hisi_dma_create_chan_dir(hdma_dev);  in hisi_dma_create_debugfs()
943 		dev_info(&hdma_dev->pdev->dev, "fail to create debugfs for channels!\n");  in hisi_dma_create_debugfs()
946 static void hisi_dma_create_debugfs(struct hisi_dma_dev *hdma_dev) { }  in hisi_dma_create_debugfs()  argument
954 	struct hisi_dma_dev *hdma_dev;  in hisi_dma_probe()  local
983 	hdma_dev = devm_kzalloc(dev, struct_size(hdma_dev, chan, chan_num),  in hisi_dma_probe()
985 	if (!hdma_dev)  in hisi_dma_probe()
988 	hdma_dev->base = pcim_iomap_table(pdev)[PCI_BAR_2];  in hisi_dma_probe()
989 	hdma_dev->pdev = pdev;  in hisi_dma_probe()
990 	hdma_dev->chan_depth = HISI_DMA_Q_DEPTH_VAL;  in hisi_dma_probe()
991 	hdma_dev->chan_num = chan_num;  in hisi_dma_probe()
992 	hdma_dev->reg_layout = reg_layout;  in hisi_dma_probe()
993 	hdma_dev->queue_base = hdma_dev->base + hisi_dma_get_queue_base(pdev);  in hisi_dma_probe()
995 	pci_set_drvdata(pdev, hdma_dev);  in hisi_dma_probe()
1007 	hisi_dma_init_dma_dev(hdma_dev);  in hisi_dma_probe()
1009 	hisi_dma_set_mode(hdma_dev, RC);  in hisi_dma_probe()
1011 	hisi_dma_init_hw(hdma_dev);  in hisi_dma_probe()
1013 	ret = hisi_dma_enable_hw_channels(hdma_dev);  in hisi_dma_probe()
1020 				       hdma_dev);  in hisi_dma_probe()
1024 	dma_dev = &hdma_dev->dma_dev;  in hisi_dma_probe()
1031 	hisi_dma_create_debugfs(hdma_dev);  in hisi_dma_probe()