344 #define IS_BUSY(chan)	(CIRC_SPACE(bchan->tail, bchan->head,\
464 static void bam_reset_channel(struct bam_chan *bchan)  in bam_reset_channel()  argument
466 	struct bam_device *bdev = bchan->bdev;  in bam_reset_channel()
468 	lockdep_assert_held(&bchan->vc.lock);  in bam_reset_channel()
471 	writel_relaxed(1, bam_addr(bdev, bchan->id, BAM_P_RST));  in bam_reset_channel()
472 	writel_relaxed(0, bam_addr(bdev, bchan->id, BAM_P_RST));  in bam_reset_channel()
478 	bchan->initialized = 0;  in bam_reset_channel()
488 static void bam_chan_init_hw(struct bam_chan *bchan,  in bam_chan_init_hw()  argument
491 	struct bam_device *bdev = bchan->bdev;  in bam_chan_init_hw()
495 	bam_reset_channel(bchan);  in bam_chan_init_hw()
501 	writel_relaxed(ALIGN(bchan->fifo_phys, sizeof(struct bam_desc_hw)),  in bam_chan_init_hw()
502 			bam_addr(bdev, bchan->id, BAM_P_DESC_FIFO_ADDR));  in bam_chan_init_hw()
504 			bam_addr(bdev, bchan->id, BAM_P_FIFO_SIZES));  in bam_chan_init_hw()
508 			bam_addr(bdev, bchan->id, BAM_P_IRQ_EN));  in bam_chan_init_hw()
512 	val |= BIT(bchan->id);  in bam_chan_init_hw()
523 	writel_relaxed(val, bam_addr(bdev, bchan->id, BAM_P_CTRL));  in bam_chan_init_hw()
525 	bchan->initialized = 1;  in bam_chan_init_hw()
528 	bchan->head = 0;  in bam_chan_init_hw()
529 	bchan->tail = 0;  in bam_chan_init_hw()
540 	struct bam_chan *bchan = to_bam_chan(chan);  in bam_alloc_chan()  local
541 	struct bam_device *bdev = bchan->bdev;  in bam_alloc_chan()
543 	if (bchan->fifo_virt)  in bam_alloc_chan()
547 	bchan->fifo_virt = dma_alloc_wc(bdev->dev, BAM_DESC_FIFO_SIZE,  in bam_alloc_chan()
548 					&bchan->fifo_phys, GFP_KERNEL);  in bam_alloc_chan()
550 	if (!bchan->fifo_virt) {  in bam_alloc_chan()
570 	struct bam_chan *bchan = to_bam_chan(chan);  in bam_free_chan()  local
571 	struct bam_device *bdev = bchan->bdev;  in bam_free_chan()
582 	if (!list_empty(&bchan->desc_list)) {  in bam_free_chan()
583 		dev_err(bchan->bdev->dev, "Cannot free busy channel\n");  in bam_free_chan()
587 	spin_lock_irqsave(&bchan->vc.lock, flags);  in bam_free_chan()
588 	bam_reset_channel(bchan);  in bam_free_chan()
589 	spin_unlock_irqrestore(&bchan->vc.lock, flags);  in bam_free_chan()
591 	dma_free_wc(bdev->dev, BAM_DESC_FIFO_SIZE, bchan->fifo_virt,  in bam_free_chan()
592 		    bchan->fifo_phys);  in bam_free_chan()
593 	bchan->fifo_virt = NULL;  in bam_free_chan()
597 	val &= ~BIT(bchan->id);  in bam_free_chan()
601 	writel_relaxed(0, bam_addr(bdev, bchan->id, BAM_P_IRQ_EN));  in bam_free_chan()
626 	struct bam_chan *bchan = to_bam_chan(chan);  in bam_slave_config()  local
629 	spin_lock_irqsave(&bchan->vc.lock, flag);  in bam_slave_config()
630 	memcpy(&bchan->slave, cfg, sizeof(*cfg));  in bam_slave_config()
631 	bchan->reconfigure = 1;  in bam_slave_config()
632 	spin_unlock_irqrestore(&bchan->vc.lock, flag);  in bam_slave_config()
652 	struct bam_chan *bchan = to_bam_chan(chan);  in bam_prep_slave_sg()  local
653 	struct bam_device *bdev = bchan->bdev;  in bam_prep_slave_sg()
714 	return vchan_tx_prep(&bchan->vc, &async_desc->vd, flags);  in bam_prep_slave_sg()
727 	struct bam_chan *bchan = to_bam_chan(chan);  in bam_dma_terminate_all()  local
733 	spin_lock_irqsave(&bchan->vc.lock, flag);  in bam_dma_terminate_all()
747 	if (!list_empty(&bchan->desc_list)) {  in bam_dma_terminate_all()
748 		async_desc = list_first_entry(&bchan->desc_list,  in bam_dma_terminate_all()
750 		bam_chan_init_hw(bchan, async_desc->dir);  in bam_dma_terminate_all()
754 				 &bchan->desc_list, desc_node) {  in bam_dma_terminate_all()
755 		list_add(&async_desc->vd.node, &bchan->vc.desc_issued);  in bam_dma_terminate_all()
759 	vchan_get_all_descriptors(&bchan->vc, &head);  in bam_dma_terminate_all()
760 	spin_unlock_irqrestore(&bchan->vc.lock, flag);  in bam_dma_terminate_all()
762 	vchan_dma_desc_free_list(&bchan->vc, &head);  in bam_dma_terminate_all()
774 	struct bam_chan *bchan = to_bam_chan(chan);  in bam_pause()  local
775 	struct bam_device *bdev = bchan->bdev;  in bam_pause()
783 	spin_lock_irqsave(&bchan->vc.lock, flag);  in bam_pause()
784 	writel_relaxed(1, bam_addr(bdev, bchan->id, BAM_P_HALT));  in bam_pause()
785 	bchan->paused = 1;  in bam_pause()
786 	spin_unlock_irqrestore(&bchan->vc.lock, flag);  in bam_pause()
800 	struct bam_chan *bchan = to_bam_chan(chan);  in bam_resume()  local
801 	struct bam_device *bdev = bchan->bdev;  in bam_resume()
809 	spin_lock_irqsave(&bchan->vc.lock, flag);  in bam_resume()
810 	writel_relaxed(0, bam_addr(bdev, bchan->id, BAM_P_HALT));  in bam_resume()
811 	bchan->paused = 0;  in bam_resume()
812 	spin_unlock_irqrestore(&bchan->vc.lock, flag);  in bam_resume()
839 		struct bam_chan *bchan = &bdev->channels[i];  in process_channel_irqs()  local
849 		spin_lock_irqsave(&bchan->vc.lock, flags);  in process_channel_irqs()
856 		avail = CIRC_CNT(offset, bchan->head, MAX_DESCRIPTORS + 1);  in process_channel_irqs()
858 		if (offset < bchan->head)  in process_channel_irqs()
862 					 &bchan->desc_list, desc_node) {  in process_channel_irqs()
868 			bchan->head += async_desc->xfer_len;  in process_channel_irqs()
869 			bchan->head %= MAX_DESCRIPTORS;  in process_channel_irqs()
884 					 &bchan->vc.desc_issued);  in process_channel_irqs()
889 		spin_unlock_irqrestore(&bchan->vc.lock, flags);  in process_channel_irqs()
947 	struct bam_chan *bchan = to_bam_chan(chan);  in bam_tx_status()  local
960 		return bchan->paused ? DMA_PAUSED : ret;  in bam_tx_status()
962 	spin_lock_irqsave(&bchan->vc.lock, flags);  in bam_tx_status()
963 	vd = vchan_find_desc(&bchan->vc, cookie);  in bam_tx_status()
967 		list_for_each_entry(async_desc, &bchan->desc_list, desc_node) {  in bam_tx_status()
977 	spin_unlock_irqrestore(&bchan->vc.lock, flags);  in bam_tx_status()
981 	if (ret == DMA_IN_PROGRESS && bchan->paused)  in bam_tx_status()
992 static void bam_apply_new_config(struct bam_chan *bchan,  in bam_apply_new_config()  argument
995 	struct bam_device *bdev = bchan->bdev;  in bam_apply_new_config()
1000 			maxburst = bchan->slave.src_maxburst;  in bam_apply_new_config()
1002 			maxburst = bchan->slave.dst_maxburst;  in bam_apply_new_config()
1008 	bchan->reconfigure = 0;  in bam_apply_new_config()
1015 static void bam_start_dma(struct bam_chan *bchan)  in bam_start_dma()  argument
1017 	struct virt_dma_desc *vd = vchan_next_desc(&bchan->vc);  in bam_start_dma()
1018 	struct bam_device *bdev = bchan->bdev;  in bam_start_dma()
1021 	struct bam_desc_hw *fifo = PTR_ALIGN(bchan->fifo_virt,  in bam_start_dma()
1027 	lockdep_assert_held(&bchan->vc.lock);  in bam_start_dma()
1036 	while (vd && !IS_BUSY(bchan)) {  in bam_start_dma()
1042 		if (!bchan->initialized)  in bam_start_dma()
1043 			bam_chan_init_hw(bchan, async_desc->dir);  in bam_start_dma()
1046 		if (bchan->reconfigure)  in bam_start_dma()
1047 			bam_apply_new_config(bchan, async_desc->dir);  in bam_start_dma()
1050 		avail = CIRC_SPACE(bchan->tail, bchan->head,  in bam_start_dma()
1063 		vd = vchan_next_desc(&bchan->vc);  in bam_start_dma()
1081 		if (bchan->tail + async_desc->xfer_len > MAX_DESCRIPTORS) {  in bam_start_dma()
1082 			u32 partial = MAX_DESCRIPTORS - bchan->tail;  in bam_start_dma()
1084 			memcpy(&fifo[bchan->tail], desc,  in bam_start_dma()
1090 			memcpy(&fifo[bchan->tail], desc,  in bam_start_dma()
1095 		bchan->tail += async_desc->xfer_len;  in bam_start_dma()
1096 		bchan->tail %= MAX_DESCRIPTORS;  in bam_start_dma()
1097 		list_add_tail(&async_desc->desc_node, &bchan->desc_list);  in bam_start_dma()
1102 	writel_relaxed(bchan->tail * sizeof(struct bam_desc_hw),  in bam_start_dma()
1103 			bam_addr(bdev, bchan->id, BAM_P_EVNT_REG));  in bam_start_dma()
1118 	struct bam_chan *bchan;  in dma_tasklet()  local
1124 		bchan = &bdev->channels[i];  in dma_tasklet()
1125 		spin_lock_irqsave(&bchan->vc.lock, flags);  in dma_tasklet()
1127 		if (!list_empty(&bchan->vc.desc_issued) && !IS_BUSY(bchan))  in dma_tasklet()
1128 			bam_start_dma(bchan);  in dma_tasklet()
1129 		spin_unlock_irqrestore(&bchan->vc.lock, flags);  in dma_tasklet()
1142 	struct bam_chan *bchan = to_bam_chan(chan);  in bam_issue_pending()  local
1145 	spin_lock_irqsave(&bchan->vc.lock, flags);  in bam_issue_pending()
1148 	if (vchan_issue_pending(&bchan->vc) && !IS_BUSY(bchan))  in bam_issue_pending()
1149 		bam_start_dma(bchan);  in bam_issue_pending()
1151 	spin_unlock_irqrestore(&bchan->vc.lock, flags);  in bam_issue_pending()
1216 static void bam_channel_init(struct bam_device *bdev, struct bam_chan *bchan,  in bam_channel_init()  argument
1219 	bchan->id = index;  in bam_channel_init()
1220 	bchan->bdev = bdev;  in bam_channel_init()
1222 	vchan_init(&bchan->vc, &bdev->common);  in bam_channel_init()
1223 	bchan->vc.desc_free = bam_dma_free_desc;  in bam_channel_init()
1224 	INIT_LIST_HEAD(&bchan->desc_list);  in bam_channel_init()