Lines Matching +full:dma +full:- +full:channel +full:- +full:mask
1 // SPDX-License-Identifier: GPL-2.0-only
3 * Driver for the Atmel Extensible DMA Controller (aka XDMAC on AT91 systems)
11 #include <dt-bindings/dma/at91.h>
54 #define AT_XDMAC_GIM 0x14 /* Global Interrupt Mask Register */
56 #define AT_XDMAC_GE 0x1C /* Global Channel Enable Register */
57 #define AT_XDMAC_GD 0x20 /* Global Channel Disable Register */
58 #define AT_XDMAC_GS 0x24 /* Global Channel Status Register */
61 /* Channel relative registers offsets */
62 #define AT_XDMAC_CIE 0x00 /* Channel Interrupt Enable Register */
70 #define AT_XDMAC_CID 0x04 /* Channel Interrupt Disable Register */
78 #define AT_XDMAC_CIM 0x08 /* Channel Interrupt Mask Register */
79 #define AT_XDMAC_CIM_BIM BIT(0) /* End of Block Interrupt Mask Bit */
80 #define AT_XDMAC_CIM_LIM BIT(1) /* End of Linked List Interrupt Mask Bit */
81 #define AT_XDMAC_CIM_DIM BIT(2) /* End of Disable Interrupt Mask Bit */
82 #define AT_XDMAC_CIM_FIM BIT(3) /* End of Flush Interrupt Mask Bit */
83 #define AT_XDMAC_CIM_RBEIM BIT(4) /* Read Bus Error Interrupt Mask Bit */
84 #define AT_XDMAC_CIM_WBEIM BIT(5) /* Write Bus Error Interrupt Mask Bit */
85 #define AT_XDMAC_CIM_ROIM BIT(6) /* Request Overflow Interrupt Mask Bit */
86 #define AT_XDMAC_CIS 0x0C /* Channel Interrupt Status Register */
94 #define AT_XDMAC_CSA 0x10 /* Channel Source Address Register */
95 #define AT_XDMAC_CDA 0x14 /* Channel Destination Address Register */
96 #define AT_XDMAC_CNDA 0x18 /* Channel Next Descriptor Address Register */
97 #define AT_XDMAC_CNDA_NDAIF(i) ((i) & 0x1) /* Channel x Next Descriptor Interface */
98 #define AT_XDMAC_CNDA_NDA(i) ((i) & 0xfffffffc) /* Channel x Next Descriptor Address */
99 #define AT_XDMAC_CNDC 0x1C /* Channel Next Descriptor Control Register */
100 #define AT_XDMAC_CNDC_NDE (0x1 << 0) /* Channel x Next Descriptor Enable */
101 #define AT_XDMAC_CNDC_NDSUP (0x1 << 1) /* Channel x Next Descriptor Source Update */
102 #define AT_XDMAC_CNDC_NDDUP (0x1 << 2) /* Channel x Next Descriptor Destination Update */
104 #define AT_XDMAC_CNDC_NDVIEW_NDV0 (0x0 << 3) /* Channel x Next Descriptor View 0 */
105 #define AT_XDMAC_CNDC_NDVIEW_NDV1 (0x1 << 3) /* Channel x Next Descriptor View 1 */
106 #define AT_XDMAC_CNDC_NDVIEW_NDV2 (0x2 << 3) /* Channel x Next Descriptor View 2 */
107 #define AT_XDMAC_CNDC_NDVIEW_NDV3 (0x3 << 3) /* Channel x Next Descriptor View 3 */
108 #define AT_XDMAC_CUBC 0x20 /* Channel Microblock Control Register */
109 #define AT_XDMAC_CBC 0x24 /* Channel Block Control Register */
110 #define AT_XDMAC_CC 0x28 /* Channel Configuration Register */
111 #define AT_XDMAC_CC_TYPE (0x1 << 0) /* Channel Transfer Type */
119 #define AT_XDMAC_CC_DSYNC (0x1 << 4) /* Channel Synchronization */
122 #define AT_XDMAC_CC_PROT (0x1 << 5) /* Channel Protection */
125 #define AT_XDMAC_CC_SWREQ (0x1 << 6) /* Channel Software Request Trigger */
128 #define AT_XDMAC_CC_MEMSET (0x1 << 7) /* Channel Fill Block of memory */
131 #define AT_XDMAC_CC_CSIZE(i) ((0x7 & (i)) << 8) /* Channel Chunk Size */
134 #define AT_XDMAC_CC_DWIDTH(i) ((0x3 & (i)) << AT_XDMAC_CC_DWIDTH_OFFSET) /* Channel Data Width */
139 #define AT_XDMAC_CC_SIF(i) ((0x1 & (i)) << 13) /* Channel Source Interface Identifier */
140 #define AT_XDMAC_CC_DIF(i) ((0x1 & (i)) << 14) /* Channel Destination Interface Identifier */
141 #define AT_XDMAC_CC_SAM_MASK (0x3 << 16) /* Channel Source Addressing Mode */
146 #define AT_XDMAC_CC_DAM_MASK (0x3 << 18) /* Channel Source Addressing Mode */
151 #define AT_XDMAC_CC_INITD (0x1 << 21) /* Channel Initialization Terminated (read only) */
160 #define AT_XDMAC_CC_PERID(i) ((0x7f & (i)) << 24) /* Channel Peripheral Identifier */
161 #define AT_XDMAC_CDS_MSP 0x2C /* Channel Data Stride Memory Set Pattern */
162 #define AT_XDMAC_CSUS 0x30 /* Channel Source Microblock Stride */
163 #define AT_XDMAC_CDUS 0x34 /* Channel Destination Microblock Stride */
194 /* Global Channel Read Suspend Register */
198 /* Global Channel Read Write Suspend Register */
200 /* Global Channel Read Write Resume Register */
202 /* Global Channel Software Request Register */
204 /* Global channel Software Request Status Register */
206 /* Global Channel Software Flush Request Register */
208 /* Channel reg base */
216 /* ----- Channels ----- */
220 u32 mask; /* Channel Mask */ member
221 u32 cfg; /* Channel Configuration Register */
241 /* ----- Controller ----- */
243 struct dma_device dma; member
256 /* ----- Descriptors ----- */
271 /* 64-bit alignment needed to update CNDA and CUBC registers in an atomic way. */
312 return atxdmac->regs + (atxdmac->layout->chan_cc_reg_base + chan_nb * 0x40); in at_xdmac_chan_reg_base()
315 #define at_xdmac_read(atxdmac, reg) readl_relaxed((atxdmac)->regs + (reg))
317 writel_relaxed((value), (atxdmac)->regs + (reg))
319 #define at_xdmac_chan_read(atchan, reg) readl_relaxed((atchan)->ch_regs + (reg))
320 #define at_xdmac_chan_write(atchan, reg, value) writel_relaxed((value), (atchan)->ch_regs + (reg))
329 return &chan->dev->device; in chan2dev()
334 return container_of(ddev, struct at_xdmac, dma); in to_at_xdmac()
344 return test_bit(AT_XDMAC_CHAN_IS_CYCLIC, &atchan->status); in at_xdmac_chan_is_cyclic()
349 return test_bit(AT_XDMAC_CHAN_IS_PAUSED, &atchan->status); in at_xdmac_chan_is_paused()
354 return test_bit(AT_XDMAC_CHAN_IS_PAUSED_INTERNAL, &atchan->status); in at_xdmac_chan_is_paused_internal()
370 "initial descriptors per channel (default: 64)");
375 struct at_xdmac *atxdmac = to_at_xdmac(atchan->chan.device); in at_xdmac_runtime_suspend_descriptors()
378 list_for_each_entry_safe(desc, _desc, &atchan->xfers_list, xfer_node) { in at_xdmac_runtime_suspend_descriptors()
379 if (!desc->active_xfer) in at_xdmac_runtime_suspend_descriptors()
382 pm_runtime_mark_last_busy(atxdmac->dev); in at_xdmac_runtime_suspend_descriptors()
383 pm_runtime_put_autosuspend(atxdmac->dev); in at_xdmac_runtime_suspend_descriptors()
389 struct at_xdmac *atxdmac = to_at_xdmac(atchan->chan.device); in at_xdmac_runtime_resume_descriptors()
393 list_for_each_entry_safe(desc, _desc, &atchan->xfers_list, xfer_node) { in at_xdmac_runtime_resume_descriptors()
394 if (!desc->active_xfer) in at_xdmac_runtime_resume_descriptors()
397 ret = pm_runtime_resume_and_get(atxdmac->dev); in at_xdmac_runtime_resume_descriptors()
407 struct at_xdmac *atxdmac = to_at_xdmac(atchan->chan.device); in at_xdmac_chan_is_enabled()
410 ret = pm_runtime_resume_and_get(atxdmac->dev); in at_xdmac_chan_is_enabled()
414 ret = !!(at_xdmac_chan_read(atchan, AT_XDMAC_GS) & atchan->mask); in at_xdmac_chan_is_enabled()
416 pm_runtime_mark_last_busy(atxdmac->dev); in at_xdmac_chan_is_enabled()
417 pm_runtime_put_autosuspend(atxdmac->dev); in at_xdmac_chan_is_enabled()
428 ret = pm_runtime_resume_and_get(atxdmac->dev); in at_xdmac_off()
432 at_xdmac_write(atxdmac, AT_XDMAC_GD, -1L); in at_xdmac_off()
438 at_xdmac_write(atxdmac, AT_XDMAC_GID, -1L); in at_xdmac_off()
441 if (!list_empty(&atxdmac->dma.channels) && suspend_descriptors) { in at_xdmac_off()
442 list_for_each_entry_safe(chan, _chan, &atxdmac->dma.channels, in at_xdmac_off()
449 pm_runtime_mark_last_busy(atxdmac->dev); in at_xdmac_off()
450 pm_runtime_put_autosuspend(atxdmac->dev); in at_xdmac_off()
457 struct at_xdmac *atxdmac = to_at_xdmac(atchan->chan.device); in at_xdmac_start_xfer()
461 ret = pm_runtime_resume_and_get(atxdmac->dev); in at_xdmac_start_xfer()
465 dev_vdbg(chan2dev(&atchan->chan), "%s: desc 0x%p\n", __func__, first); in at_xdmac_start_xfer()
468 first->active_xfer = true; in at_xdmac_start_xfer()
471 reg = AT_XDMAC_CNDA_NDA(first->tx_dma_desc.phys); in at_xdmac_start_xfer()
472 if (atxdmac->layout->sdif) in at_xdmac_start_xfer()
473 reg |= AT_XDMAC_CNDA_NDAIF(atchan->memif); in at_xdmac_start_xfer()
484 else if ((first->lld.mbr_ubc & in at_xdmac_start_xfer()
492 * properly. This bit can be modified only by using the channel in at_xdmac_start_xfer()
495 at_xdmac_chan_write(atchan, AT_XDMAC_CC, first->lld.mbr_cfg); in at_xdmac_start_xfer()
502 dev_vdbg(chan2dev(&atchan->chan), in at_xdmac_start_xfer()
516 if (at_xdmac_chan_is_peripheral_xfer(first->lld.mbr_cfg)) in at_xdmac_start_xfer()
520 * There is no end of list when doing cyclic dma, we need to get in at_xdmac_start_xfer()
529 at_xdmac_write(atxdmac, AT_XDMAC_GIE, atchan->mask); in at_xdmac_start_xfer()
530 dev_vdbg(chan2dev(&atchan->chan), in at_xdmac_start_xfer()
531 "%s: enable channel (0x%08x)\n", __func__, atchan->mask); in at_xdmac_start_xfer()
533 at_xdmac_write(atxdmac, AT_XDMAC_GE, atchan->mask); in at_xdmac_start_xfer()
535 dev_vdbg(chan2dev(&atchan->chan), in at_xdmac_start_xfer()
548 struct at_xdmac_chan *atchan = to_at_xdmac_chan(tx->chan); in at_xdmac_tx_submit()
552 spin_lock_irqsave(&atchan->lock, irqflags); in at_xdmac_tx_submit()
555 list_add_tail(&desc->xfer_node, &atchan->xfers_list); in at_xdmac_tx_submit()
556 spin_unlock_irqrestore(&atchan->lock, irqflags); in at_xdmac_tx_submit()
558 dev_vdbg(chan2dev(tx->chan), "%s: atchan 0x%p, add desc 0x%p to xfers_list\n", in at_xdmac_tx_submit()
568 struct at_xdmac *atxdmac = to_at_xdmac(chan->device); in at_xdmac_alloc_desc()
571 desc = dma_pool_zalloc(atxdmac->at_xdmac_desc_pool, gfp_flags, &phys); in at_xdmac_alloc_desc()
573 INIT_LIST_HEAD(&desc->descs_list); in at_xdmac_alloc_desc()
574 dma_async_tx_descriptor_init(&desc->tx_dma_desc, chan); in at_xdmac_alloc_desc()
575 desc->tx_dma_desc.tx_submit = at_xdmac_tx_submit; in at_xdmac_alloc_desc()
576 desc->tx_dma_desc.phys = phys; in at_xdmac_alloc_desc()
584 memset(&desc->lld, 0, sizeof(desc->lld)); in at_xdmac_init_used_desc()
585 INIT_LIST_HEAD(&desc->descs_list); in at_xdmac_init_used_desc()
586 desc->direction = DMA_TRANS_NONE; in at_xdmac_init_used_desc()
587 desc->xfer_size = 0; in at_xdmac_init_used_desc()
588 desc->active_xfer = false; in at_xdmac_init_used_desc()
596 if (list_empty(&atchan->free_descs_list)) { in at_xdmac_get_desc()
597 desc = at_xdmac_alloc_desc(&atchan->chan, GFP_NOWAIT); in at_xdmac_get_desc()
599 desc = list_first_entry(&atchan->free_descs_list, in at_xdmac_get_desc()
601 list_del(&desc->desc_node); in at_xdmac_get_desc()
615 prev->lld.mbr_nda = desc->tx_dma_desc.phys; in at_xdmac_queue_desc()
616 prev->lld.mbr_ubc |= AT_XDMAC_MBR_UBC_NDE; in at_xdmac_queue_desc()
619 __func__, prev, &prev->lld.mbr_nda); in at_xdmac_queue_desc()
628 desc->lld.mbr_bc++; in at_xdmac_increment_block_count()
638 struct at_xdmac *atxdmac = of_dma->of_dma_data; in at_xdmac_xlate()
641 struct device *dev = atxdmac->dma.dev; in at_xdmac_xlate()
643 if (dma_spec->args_count != 1) { in at_xdmac_xlate()
644 dev_err(dev, "dma phandler args: bad number of args\n"); in at_xdmac_xlate()
648 chan = dma_get_any_slave_channel(&atxdmac->dma); in at_xdmac_xlate()
650 dev_err(dev, "can't get a dma channel\n"); in at_xdmac_xlate()
655 atchan->memif = AT91_XDMAC_DT_GET_MEM_IF(dma_spec->args[0]); in at_xdmac_xlate()
656 atchan->perif = AT91_XDMAC_DT_GET_PER_IF(dma_spec->args[0]); in at_xdmac_xlate()
657 atchan->perid = AT91_XDMAC_DT_GET_PERID(dma_spec->args[0]); in at_xdmac_xlate()
659 atchan->memif, atchan->perif, atchan->perid); in at_xdmac_xlate()
668 struct at_xdmac *atxdmac = to_at_xdmac(atchan->chan.device); in at_xdmac_compute_chan_conf()
672 atchan->cfg = in at_xdmac_compute_chan_conf()
673 AT91_XDMAC_DT_PERID(atchan->perid) in at_xdmac_compute_chan_conf()
680 if (atxdmac->layout->sdif) in at_xdmac_compute_chan_conf()
681 atchan->cfg |= AT_XDMAC_CC_DIF(atchan->memif) | in at_xdmac_compute_chan_conf()
682 AT_XDMAC_CC_SIF(atchan->perif); in at_xdmac_compute_chan_conf()
684 csize = ffs(atchan->sconfig.src_maxburst) - 1; in at_xdmac_compute_chan_conf()
687 return -EINVAL; in at_xdmac_compute_chan_conf()
689 atchan->cfg |= AT_XDMAC_CC_CSIZE(csize); in at_xdmac_compute_chan_conf()
690 dwidth = ffs(atchan->sconfig.src_addr_width) - 1; in at_xdmac_compute_chan_conf()
693 return -EINVAL; in at_xdmac_compute_chan_conf()
695 atchan->cfg |= AT_XDMAC_CC_DWIDTH(dwidth); in at_xdmac_compute_chan_conf()
697 atchan->cfg = in at_xdmac_compute_chan_conf()
698 AT91_XDMAC_DT_PERID(atchan->perid) in at_xdmac_compute_chan_conf()
705 if (atxdmac->layout->sdif) in at_xdmac_compute_chan_conf()
706 atchan->cfg |= AT_XDMAC_CC_DIF(atchan->perif) | in at_xdmac_compute_chan_conf()
707 AT_XDMAC_CC_SIF(atchan->memif); in at_xdmac_compute_chan_conf()
709 csize = ffs(atchan->sconfig.dst_maxburst) - 1; in at_xdmac_compute_chan_conf()
712 return -EINVAL; in at_xdmac_compute_chan_conf()
714 atchan->cfg |= AT_XDMAC_CC_CSIZE(csize); in at_xdmac_compute_chan_conf()
715 dwidth = ffs(atchan->sconfig.dst_addr_width) - 1; in at_xdmac_compute_chan_conf()
718 return -EINVAL; in at_xdmac_compute_chan_conf()
720 atchan->cfg |= AT_XDMAC_CC_DWIDTH(dwidth); in at_xdmac_compute_chan_conf()
723 dev_dbg(chan2dev(chan), "%s: cfg=0x%08x\n", __func__, atchan->cfg); in at_xdmac_compute_chan_conf()
735 if ((sconfig->src_maxburst > AT_XDMAC_MAX_CSIZE) in at_xdmac_check_slave_config()
736 || (sconfig->dst_maxburst > AT_XDMAC_MAX_CSIZE)) in at_xdmac_check_slave_config()
737 return -EINVAL; in at_xdmac_check_slave_config()
739 if ((sconfig->src_addr_width > AT_XDMAC_MAX_DWIDTH) in at_xdmac_check_slave_config()
740 || (sconfig->dst_addr_width > AT_XDMAC_MAX_DWIDTH)) in at_xdmac_check_slave_config()
741 return -EINVAL; in at_xdmac_check_slave_config()
753 return -EINVAL; in at_xdmac_set_slave_config()
756 memcpy(&atchan->sconfig, sconfig, sizeof(atchan->sconfig)); in at_xdmac_set_slave_config()
778 dev_err(chan2dev(chan), "invalid DMA direction\n"); in at_xdmac_prep_slave_sg()
788 spin_lock_irqsave(&atchan->lock, irqflags); in at_xdmac_prep_slave_sg()
811 list_splice_tail_init(&first->descs_list, in at_xdmac_prep_slave_sg()
812 &atchan->free_descs_list); in at_xdmac_prep_slave_sg()
818 desc->lld.mbr_sa = atchan->sconfig.src_addr; in at_xdmac_prep_slave_sg()
819 desc->lld.mbr_da = mem; in at_xdmac_prep_slave_sg()
821 desc->lld.mbr_sa = mem; in at_xdmac_prep_slave_sg()
822 desc->lld.mbr_da = atchan->sconfig.dst_addr; in at_xdmac_prep_slave_sg()
824 dwidth = at_xdmac_get_dwidth(atchan->cfg); in at_xdmac_prep_slave_sg()
828 desc->lld.mbr_ubc = AT_XDMAC_MBR_UBC_NDV2 /* next descriptor view */ in at_xdmac_prep_slave_sg()
832 desc->lld.mbr_cfg = (atchan->cfg & ~AT_XDMAC_CC_DWIDTH_MASK) | in at_xdmac_prep_slave_sg()
836 __func__, &desc->lld.mbr_sa, &desc->lld.mbr_da, desc->lld.mbr_ubc); in at_xdmac_prep_slave_sg()
848 list_add_tail(&desc->desc_node, &first->descs_list); in at_xdmac_prep_slave_sg()
853 first->tx_dma_desc.flags = flags; in at_xdmac_prep_slave_sg()
854 first->xfer_size = xfer_size; in at_xdmac_prep_slave_sg()
855 first->direction = direction; in at_xdmac_prep_slave_sg()
856 ret = &first->tx_dma_desc; in at_xdmac_prep_slave_sg()
859 spin_unlock_irqrestore(&atchan->lock, irqflags); in at_xdmac_prep_slave_sg()
880 dev_err(chan2dev(chan), "invalid DMA direction\n"); in at_xdmac_prep_dma_cyclic()
884 if (test_and_set_bit(AT_XDMAC_CHAN_IS_CYCLIC, &atchan->status)) { in at_xdmac_prep_dma_cyclic()
885 dev_err(chan2dev(chan), "channel currently used\n"); in at_xdmac_prep_dma_cyclic()
895 spin_lock_irqsave(&atchan->lock, irqflags); in at_xdmac_prep_dma_cyclic()
900 list_splice_tail_init(&first->descs_list, in at_xdmac_prep_dma_cyclic()
901 &atchan->free_descs_list); in at_xdmac_prep_dma_cyclic()
902 spin_unlock_irqrestore(&atchan->lock, irqflags); in at_xdmac_prep_dma_cyclic()
905 spin_unlock_irqrestore(&atchan->lock, irqflags); in at_xdmac_prep_dma_cyclic()
908 __func__, desc, &desc->tx_dma_desc.phys); in at_xdmac_prep_dma_cyclic()
911 desc->lld.mbr_sa = atchan->sconfig.src_addr; in at_xdmac_prep_dma_cyclic()
912 desc->lld.mbr_da = buf_addr + i * period_len; in at_xdmac_prep_dma_cyclic()
914 desc->lld.mbr_sa = buf_addr + i * period_len; in at_xdmac_prep_dma_cyclic()
915 desc->lld.mbr_da = atchan->sconfig.dst_addr; in at_xdmac_prep_dma_cyclic()
917 desc->lld.mbr_cfg = atchan->cfg; in at_xdmac_prep_dma_cyclic()
918 desc->lld.mbr_ubc = AT_XDMAC_MBR_UBC_NDV1 in at_xdmac_prep_dma_cyclic()
921 | period_len >> at_xdmac_get_dwidth(desc->lld.mbr_cfg); in at_xdmac_prep_dma_cyclic()
925 __func__, &desc->lld.mbr_sa, &desc->lld.mbr_da, desc->lld.mbr_ubc); in at_xdmac_prep_dma_cyclic()
937 list_add_tail(&desc->desc_node, &first->descs_list); in at_xdmac_prep_dma_cyclic()
941 first->tx_dma_desc.flags = flags; in at_xdmac_prep_dma_cyclic()
942 first->xfer_size = buf_len; in at_xdmac_prep_dma_cyclic()
943 first->direction = direction; in at_xdmac_prep_dma_cyclic()
945 return &first->tx_dma_desc; in at_xdmac_prep_dma_cyclic()
990 * WARNING: The channel configuration is set here since there is no in at_xdmac_interleaved_queue_desc()
998 * match the one of another channel. If not, it could lead to spurious in at_xdmac_interleaved_queue_desc()
1009 dwidth = at_xdmac_align_width(chan, src | dst | chunk->size); in at_xdmac_interleaved_queue_desc()
1010 if (chunk->size >= (AT_XDMAC_MBR_UBC_UBLEN_MAX << dwidth)) { in at_xdmac_interleaved_queue_desc()
1013 __func__, chunk->size, in at_xdmac_interleaved_queue_desc()
1022 if (xt->src_inc) { in at_xdmac_interleaved_queue_desc()
1023 if (xt->src_sgl) in at_xdmac_interleaved_queue_desc()
1029 if (xt->dst_inc) { in at_xdmac_interleaved_queue_desc()
1030 if (xt->dst_sgl) in at_xdmac_interleaved_queue_desc()
1036 spin_lock_irqsave(&atchan->lock, flags); in at_xdmac_interleaved_queue_desc()
1038 spin_unlock_irqrestore(&atchan->lock, flags); in at_xdmac_interleaved_queue_desc()
1046 ublen = chunk->size >> dwidth; in at_xdmac_interleaved_queue_desc()
1048 desc->lld.mbr_sa = src; in at_xdmac_interleaved_queue_desc()
1049 desc->lld.mbr_da = dst; in at_xdmac_interleaved_queue_desc()
1050 desc->lld.mbr_sus = dmaengine_get_src_icg(xt, chunk); in at_xdmac_interleaved_queue_desc()
1051 desc->lld.mbr_dus = dmaengine_get_dst_icg(xt, chunk); in at_xdmac_interleaved_queue_desc()
1053 desc->lld.mbr_ubc = AT_XDMAC_MBR_UBC_NDV3 in at_xdmac_interleaved_queue_desc()
1057 desc->lld.mbr_cfg = chan_cc; in at_xdmac_interleaved_queue_desc()
1061 __func__, &desc->lld.mbr_sa, &desc->lld.mbr_da, in at_xdmac_interleaved_queue_desc()
1062 desc->lld.mbr_ubc, desc->lld.mbr_cfg); in at_xdmac_interleaved_queue_desc()
1083 if (!xt || !xt->numf || (xt->dir != DMA_MEM_TO_MEM)) in at_xdmac_prep_interleaved()
1090 if ((xt->numf > 1) && (xt->frame_size > 1)) in at_xdmac_prep_interleaved()
1094 __func__, &xt->src_start, &xt->dst_start, xt->numf, in at_xdmac_prep_interleaved()
1095 xt->frame_size, flags); in at_xdmac_prep_interleaved()
1097 src_addr = xt->src_start; in at_xdmac_prep_interleaved()
1098 dst_addr = xt->dst_start; in at_xdmac_prep_interleaved()
1100 if (xt->numf > 1) { in at_xdmac_prep_interleaved()
1104 xt, xt->sgl); in at_xdmac_prep_interleaved()
1109 for (i = 0; i < xt->numf - 1; i++) in at_xdmac_prep_interleaved()
1114 list_add_tail(&first->desc_node, &first->descs_list); in at_xdmac_prep_interleaved()
1116 for (i = 0; i < xt->frame_size; i++) { in at_xdmac_prep_interleaved()
1120 chunk = xt->sgl + i; in at_xdmac_prep_interleaved()
1125 src_skip = chunk->size + src_icg; in at_xdmac_prep_interleaved()
1126 dst_skip = chunk->size + dst_icg; in at_xdmac_prep_interleaved()
1130 __func__, chunk->size, src_icg, dst_icg); in at_xdmac_prep_interleaved()
1138 list_splice_tail_init(&first->descs_list, in at_xdmac_prep_interleaved()
1139 &atchan->free_descs_list); in at_xdmac_prep_interleaved()
1148 list_add_tail(&desc->desc_node, &first->descs_list); in at_xdmac_prep_interleaved()
1150 if (xt->src_sgl) in at_xdmac_prep_interleaved()
1153 if (xt->dst_sgl) in at_xdmac_prep_interleaved()
1156 len += chunk->size; in at_xdmac_prep_interleaved()
1161 first->tx_dma_desc.cookie = -EBUSY; in at_xdmac_prep_interleaved()
1162 first->tx_dma_desc.flags = flags; in at_xdmac_prep_interleaved()
1163 first->xfer_size = len; in at_xdmac_prep_interleaved()
1165 return &first->tx_dma_desc; in at_xdmac_prep_interleaved()
1185 * match the one of another channel. If not, it could lead to spurious in at_xdmac_prep_dma_memcpy()
1213 spin_lock_irqsave(&atchan->lock, irqflags); in at_xdmac_prep_dma_memcpy()
1215 spin_unlock_irqrestore(&atchan->lock, irqflags); in at_xdmac_prep_dma_memcpy()
1219 list_splice_tail_init(&first->descs_list, in at_xdmac_prep_dma_memcpy()
1220 &atchan->free_descs_list); in at_xdmac_prep_dma_memcpy()
1242 remaining_size -= xfer_size; in at_xdmac_prep_dma_memcpy()
1244 desc->lld.mbr_sa = src_addr; in at_xdmac_prep_dma_memcpy()
1245 desc->lld.mbr_da = dst_addr; in at_xdmac_prep_dma_memcpy()
1246 desc->lld.mbr_ubc = AT_XDMAC_MBR_UBC_NDV2 in at_xdmac_prep_dma_memcpy()
1250 desc->lld.mbr_cfg = chan_cc; in at_xdmac_prep_dma_memcpy()
1254 __func__, &desc->lld.mbr_sa, &desc->lld.mbr_da, desc->lld.mbr_ubc, desc->lld.mbr_cfg); in at_xdmac_prep_dma_memcpy()
1266 list_add_tail(&desc->desc_node, &first->descs_list); in at_xdmac_prep_dma_memcpy()
1269 first->tx_dma_desc.flags = flags; in at_xdmac_prep_dma_memcpy()
1270 first->xfer_size = len; in at_xdmac_prep_dma_memcpy()
1272 return &first->tx_dma_desc; in at_xdmac_prep_dma_memcpy()
1287 * WARNING: The channel configuration is set here since there is no in at_xdmac_memset_create_desc()
1295 * match the one of another channel. If not, it could lead to spurious in at_xdmac_memset_create_desc()
1318 spin_lock_irqsave(&atchan->lock, flags); in at_xdmac_memset_create_desc()
1320 spin_unlock_irqrestore(&atchan->lock, flags); in at_xdmac_memset_create_desc()
1333 desc->lld.mbr_da = dst_addr; in at_xdmac_memset_create_desc()
1334 desc->lld.mbr_ds = (pattern << 24) | in at_xdmac_memset_create_desc()
1338 desc->lld.mbr_ubc = AT_XDMAC_MBR_UBC_NDV3 in at_xdmac_memset_create_desc()
1342 desc->lld.mbr_cfg = chan_cc; in at_xdmac_memset_create_desc()
1346 __func__, &desc->lld.mbr_da, desc->lld.mbr_ds, desc->lld.mbr_ubc, in at_xdmac_memset_create_desc()
1347 desc->lld.mbr_cfg); in at_xdmac_memset_create_desc()
1366 list_add_tail(&desc->desc_node, &desc->descs_list); in at_xdmac_prep_dma_memset()
1368 desc->tx_dma_desc.cookie = -EBUSY; in at_xdmac_prep_dma_memset()
1369 desc->tx_dma_desc.flags = flags; in at_xdmac_prep_dma_memset()
1370 desc->xfer_size = len; in at_xdmac_prep_dma_memset()
1372 return &desc->tx_dma_desc; in at_xdmac_prep_dma_memset()
1403 list_splice_tail_init(&first->descs_list, in at_xdmac_prep_dma_memset_sg()
1404 &atchan->free_descs_list); in at_xdmac_prep_dma_memset_sg()
1412 stride = sg_dma_address(sg) - in at_xdmac_prep_dma_memset_sg()
1424 * +-------+ +-------+ +-------+ in at_xdmac_prep_dma_memset_sg()
1425 * | N-2 | | N-1 | | N | in at_xdmac_prep_dma_memset_sg()
1426 * +-------+ +-------+ +-------+ in at_xdmac_prep_dma_memset_sg()
1428 * We need all these three elements (N-2, N-1 and N) in at_xdmac_prep_dma_memset_sg()
1430 * queue N-1 or reuse N-2. in at_xdmac_prep_dma_memset_sg()
1443 * N-2 descriptor in at_xdmac_prep_dma_memset_sg()
1446 ppdesc->lld.mbr_dus = stride; in at_xdmac_prep_dma_memset_sg()
1449 * Put back the N-1 descriptor in the in at_xdmac_prep_dma_memset_sg()
1452 list_add_tail(&pdesc->desc_node, in at_xdmac_prep_dma_memset_sg()
1453 &atchan->free_descs_list); in at_xdmac_prep_dma_memset_sg()
1456 * Make our N-1 descriptor pointer in at_xdmac_prep_dma_memset_sg()
1457 * point to the N-2 since they were in at_xdmac_prep_dma_memset_sg()
1473 * Queue the N-1 descriptor after the in at_xdmac_prep_dma_memset_sg()
1474 * N-2 in at_xdmac_prep_dma_memset_sg()
1479 * Add the N-1 descriptor to the list in at_xdmac_prep_dma_memset_sg()
1483 list_add_tail(&desc->desc_node, in at_xdmac_prep_dma_memset_sg()
1484 &first->descs_list); in at_xdmac_prep_dma_memset_sg()
1498 if ((i == (sg_len - 1)) && in at_xdmac_prep_dma_memset_sg()
1505 * Increment the block count of the N-1 in at_xdmac_prep_dma_memset_sg()
1509 pdesc->lld.mbr_dus = stride; in at_xdmac_prep_dma_memset_sg()
1515 list_add_tail(&desc->desc_node, in at_xdmac_prep_dma_memset_sg()
1516 &atchan->free_descs_list); in at_xdmac_prep_dma_memset_sg()
1530 first->tx_dma_desc.cookie = -EBUSY; in at_xdmac_prep_dma_memset_sg()
1531 first->tx_dma_desc.flags = flags; in at_xdmac_prep_dma_memset_sg()
1532 first->xfer_size = len; in at_xdmac_prep_dma_memset_sg()
1534 return &first->tx_dma_desc; in at_xdmac_prep_dma_memset_sg()
1542 struct at_xdmac *atxdmac = to_at_xdmac(atchan->chan.device); in at_xdmac_tx_status()
1547 u32 cur_nda, check_nda, cur_ubc, mask, value; in at_xdmac_tx_status() local
1556 pm_status = pm_runtime_resume_and_get(atxdmac->dev); in at_xdmac_tx_status()
1560 spin_lock_irqsave(&atchan->lock, flags); in at_xdmac_tx_status()
1562 desc = list_first_entry(&atchan->xfers_list, struct at_xdmac_desc, xfer_node); in at_xdmac_tx_status()
1568 if (!desc->active_xfer) { in at_xdmac_tx_status()
1569 dma_set_residue(txstate, desc->xfer_size); in at_xdmac_tx_status()
1573 residue = desc->xfer_size; in at_xdmac_tx_status()
1581 * timeout, it requests the residue. If the data are in the DMA FIFO, in at_xdmac_tx_status()
1587 mask = AT_XDMAC_CC_TYPE | AT_XDMAC_CC_DSYNC; in at_xdmac_tx_status()
1589 if ((desc->lld.mbr_cfg & mask) == value) { in at_xdmac_tx_status()
1590 at_xdmac_write(atxdmac, atxdmac->layout->gswf, atchan->mask); in at_xdmac_tx_status()
1596 * The easiest way to compute the residue should be to pause the DMA in at_xdmac_tx_status()
1600 * - DMA is running therefore a descriptor change is possible while in at_xdmac_tx_status()
1602 * - When the block transfer is done, the value of the CUBC register in at_xdmac_tx_status()
1607 * INITD -------- ------------ in at_xdmac_tx_status()
1647 if ((desc->lld.mbr_cfg & mask) == value) { in at_xdmac_tx_status()
1648 at_xdmac_write(atxdmac, atxdmac->layout->gswf, atchan->mask); in at_xdmac_tx_status()
1658 descs_list = &desc->descs_list; in at_xdmac_tx_status()
1660 dwidth = at_xdmac_get_dwidth(iter->lld.mbr_cfg); in at_xdmac_tx_status()
1661 residue -= (iter->lld.mbr_ubc & 0xffffff) << dwidth; in at_xdmac_tx_status()
1662 if ((iter->lld.mbr_nda & 0xfffffffc) == cur_nda) { in at_xdmac_tx_status()
1673 __func__, desc, &desc->tx_dma_desc.phys, ret, cookie, residue); in at_xdmac_tx_status()
1676 spin_unlock_irqrestore(&atchan->lock, flags); in at_xdmac_tx_status()
1677 pm_runtime_mark_last_busy(atxdmac->dev); in at_xdmac_tx_status()
1678 pm_runtime_put_autosuspend(atxdmac->dev); in at_xdmac_tx_status()
1687 * If channel is enabled, do nothing, advance_work will be triggered in at_xdmac_advance_work()
1690 if (at_xdmac_chan_is_enabled(atchan) || list_empty(&atchan->xfers_list)) in at_xdmac_advance_work()
1693 desc = list_first_entry(&atchan->xfers_list, struct at_xdmac_desc, in at_xdmac_advance_work()
1695 dev_vdbg(chan2dev(&atchan->chan), "%s: desc 0x%p\n", __func__, desc); in at_xdmac_advance_work()
1696 if (!desc->active_xfer) in at_xdmac_advance_work()
1705 spin_lock_irq(&atchan->lock); in at_xdmac_handle_cyclic()
1706 dev_dbg(chan2dev(&atchan->chan), "%s: status=0x%08x\n", in at_xdmac_handle_cyclic()
1707 __func__, atchan->irq_status); in at_xdmac_handle_cyclic()
1708 if (list_empty(&atchan->xfers_list)) { in at_xdmac_handle_cyclic()
1709 spin_unlock_irq(&atchan->lock); in at_xdmac_handle_cyclic()
1712 desc = list_first_entry(&atchan->xfers_list, struct at_xdmac_desc, in at_xdmac_handle_cyclic()
1714 spin_unlock_irq(&atchan->lock); in at_xdmac_handle_cyclic()
1715 txd = &desc->tx_dma_desc; in at_xdmac_handle_cyclic()
1716 if (txd->flags & DMA_PREP_INTERRUPT) in at_xdmac_handle_cyclic()
1720 /* Called with atchan->lock held. */
1723 struct at_xdmac *atxdmac = to_at_xdmac(atchan->chan.device); in at_xdmac_handle_error()
1727 ret = pm_runtime_resume_and_get(atxdmac->dev); in at_xdmac_handle_error()
1737 if (atchan->irq_status & AT_XDMAC_CIS_RBEIS) in at_xdmac_handle_error()
1738 dev_err(chan2dev(&atchan->chan), "read bus error!!!"); in at_xdmac_handle_error()
1739 if (atchan->irq_status & AT_XDMAC_CIS_WBEIS) in at_xdmac_handle_error()
1740 dev_err(chan2dev(&atchan->chan), "write bus error!!!"); in at_xdmac_handle_error()
1741 if (atchan->irq_status & AT_XDMAC_CIS_ROIS) in at_xdmac_handle_error()
1742 dev_err(chan2dev(&atchan->chan), "request overflow error!!!"); in at_xdmac_handle_error()
1744 /* Channel must be disabled first as it's not done automatically */ in at_xdmac_handle_error()
1745 at_xdmac_write(atxdmac, AT_XDMAC_GD, atchan->mask); in at_xdmac_handle_error()
1746 while (at_xdmac_read(atxdmac, AT_XDMAC_GS) & atchan->mask) in at_xdmac_handle_error()
1749 bad_desc = list_first_entry(&atchan->xfers_list, in at_xdmac_handle_error()
1754 dev_dbg(chan2dev(&atchan->chan), in at_xdmac_handle_error()
1756 __func__, &bad_desc->lld.mbr_sa, &bad_desc->lld.mbr_da, in at_xdmac_handle_error()
1757 bad_desc->lld.mbr_ubc); in at_xdmac_handle_error()
1759 pm_runtime_mark_last_busy(atxdmac->dev); in at_xdmac_handle_error()
1760 pm_runtime_put_autosuspend(atxdmac->dev); in at_xdmac_handle_error()
1768 struct at_xdmac *atxdmac = to_at_xdmac(atchan->chan.device); in at_xdmac_tasklet()
1779 spin_lock_irq(&atchan->lock); in at_xdmac_tasklet()
1781 dev_dbg(chan2dev(&atchan->chan), "%s: status=0x%08x\n", in at_xdmac_tasklet()
1782 __func__, atchan->irq_status); in at_xdmac_tasklet()
1784 if (!(atchan->irq_status & AT_XDMAC_CIS_LIS) && in at_xdmac_tasklet()
1785 !(atchan->irq_status & error_mask)) { in at_xdmac_tasklet()
1786 spin_unlock_irq(&atchan->lock); in at_xdmac_tasklet()
1790 if (atchan->irq_status & error_mask) in at_xdmac_tasklet()
1793 desc = list_first_entry(&atchan->xfers_list, struct at_xdmac_desc, in at_xdmac_tasklet()
1795 dev_vdbg(chan2dev(&atchan->chan), "%s: desc 0x%p\n", __func__, desc); in at_xdmac_tasklet()
1796 if (!desc->active_xfer) { in at_xdmac_tasklet()
1797 dev_err(chan2dev(&atchan->chan), "Xfer not active: exiting"); in at_xdmac_tasklet()
1798 spin_unlock_irq(&atchan->lock); in at_xdmac_tasklet()
1802 txd = &desc->tx_dma_desc; in at_xdmac_tasklet()
1805 list_del(&desc->xfer_node); in at_xdmac_tasklet()
1806 spin_unlock_irq(&atchan->lock); in at_xdmac_tasklet()
1808 if (txd->flags & DMA_PREP_INTERRUPT) in at_xdmac_tasklet()
1813 spin_lock_irq(&atchan->lock); in at_xdmac_tasklet()
1815 list_splice_tail_init(&desc->descs_list, &atchan->free_descs_list); in at_xdmac_tasklet()
1817 spin_unlock_irq(&atchan->lock); in at_xdmac_tasklet()
1823 pm_runtime_mark_last_busy(atxdmac->dev); in at_xdmac_tasklet()
1824 pm_runtime_put_autosuspend(atxdmac->dev); in at_xdmac_tasklet()
1840 dev_vdbg(atxdmac->dma.dev, in at_xdmac_interrupt()
1847 /* We have to find which channel has generated the interrupt. */ in at_xdmac_interrupt()
1848 for (i = 0; i < atxdmac->dma.chancnt; i++) { in at_xdmac_interrupt()
1852 atchan = &atxdmac->chan[i]; in at_xdmac_interrupt()
1855 atchan->irq_status = chan_status & chan_imr; in at_xdmac_interrupt()
1856 dev_vdbg(atxdmac->dma.dev, in at_xdmac_interrupt()
1859 dev_vdbg(chan2dev(&atchan->chan), in at_xdmac_interrupt()
1869 if (atchan->irq_status & (AT_XDMAC_CIS_RBEIS | AT_XDMAC_CIS_WBEIS)) in at_xdmac_interrupt()
1870 at_xdmac_write(atxdmac, AT_XDMAC_GD, atchan->mask); in at_xdmac_interrupt()
1872 tasklet_schedule(&atchan->tasklet); in at_xdmac_interrupt()
1886 dev_dbg(chan2dev(&atchan->chan), "%s\n", __func__); in at_xdmac_issue_pending()
1888 spin_lock_irqsave(&atchan->lock, flags); in at_xdmac_issue_pending()
1890 spin_unlock_irqrestore(&atchan->lock, flags); in at_xdmac_issue_pending()
1904 spin_lock_irqsave(&atchan->lock, flags); in at_xdmac_device_config()
1906 spin_unlock_irqrestore(&atchan->lock, flags); in at_xdmac_device_config()
1914 at_xdmac_write(atxdmac, atxdmac->layout->grws, atchan->mask); in at_xdmac_device_pause_set()
1922 struct at_xdmac *atxdmac = to_at_xdmac(atchan->chan.device); in at_xdmac_device_pause_internal()
1925 spin_lock_irqsave(&atchan->lock, flags); in at_xdmac_device_pause_internal()
1926 set_bit(AT_XDMAC_CHAN_IS_PAUSED_INTERNAL, &atchan->status); in at_xdmac_device_pause_internal()
1928 spin_unlock_irqrestore(&atchan->lock, flags); in at_xdmac_device_pause_internal()
1934 struct at_xdmac *atxdmac = to_at_xdmac(atchan->chan.device); in at_xdmac_device_pause()
1940 if (test_and_set_bit(AT_XDMAC_CHAN_IS_PAUSED, &atchan->status)) in at_xdmac_device_pause()
1943 ret = pm_runtime_resume_and_get(atxdmac->dev); in at_xdmac_device_pause()
1947 spin_lock_irqsave(&atchan->lock, flags); in at_xdmac_device_pause()
1953 spin_unlock_irqrestore(&atchan->lock, flags); in at_xdmac_device_pause()
1955 pm_runtime_mark_last_busy(atxdmac->dev); in at_xdmac_device_pause()
1956 pm_runtime_put_autosuspend(atxdmac->dev); in at_xdmac_device_pause()
1963 struct at_xdmac *atxdmac = to_at_xdmac(atchan->chan.device); in at_xdmac_device_resume_internal()
1966 spin_lock_irqsave(&atchan->lock, flags); in at_xdmac_device_resume_internal()
1967 at_xdmac_write(atxdmac, atxdmac->layout->grwr, atchan->mask); in at_xdmac_device_resume_internal()
1968 clear_bit(AT_XDMAC_CHAN_IS_PAUSED_INTERNAL, &atchan->status); in at_xdmac_device_resume_internal()
1969 spin_unlock_irqrestore(&atchan->lock, flags); in at_xdmac_device_resume_internal()
1975 struct at_xdmac *atxdmac = to_at_xdmac(atchan->chan.device); in at_xdmac_device_resume()
1981 ret = pm_runtime_resume_and_get(atxdmac->dev); in at_xdmac_device_resume()
1985 spin_lock_irqsave(&atchan->lock, flags); in at_xdmac_device_resume()
1994 at_xdmac_write(atxdmac, atxdmac->layout->grwr, atchan->mask); in at_xdmac_device_resume()
1995 clear_bit(AT_XDMAC_CHAN_IS_PAUSED, &atchan->status); in at_xdmac_device_resume()
1998 spin_unlock_irqrestore(&atchan->lock, flags); in at_xdmac_device_resume()
1999 pm_runtime_mark_last_busy(atxdmac->dev); in at_xdmac_device_resume()
2000 pm_runtime_put_autosuspend(atxdmac->dev); in at_xdmac_device_resume()
2009 struct at_xdmac *atxdmac = to_at_xdmac(atchan->chan.device); in at_xdmac_device_terminate_all()
2015 ret = pm_runtime_resume_and_get(atxdmac->dev); in at_xdmac_device_terminate_all()
2019 spin_lock_irqsave(&atchan->lock, flags); in at_xdmac_device_terminate_all()
2020 at_xdmac_write(atxdmac, AT_XDMAC_GD, atchan->mask); in at_xdmac_device_terminate_all()
2021 while (at_xdmac_read(atxdmac, AT_XDMAC_GS) & atchan->mask) in at_xdmac_device_terminate_all()
2025 list_for_each_entry_safe(desc, _desc, &atchan->xfers_list, xfer_node) { in at_xdmac_device_terminate_all()
2026 list_del(&desc->xfer_node); in at_xdmac_device_terminate_all()
2027 list_splice_tail_init(&desc->descs_list, in at_xdmac_device_terminate_all()
2028 &atchan->free_descs_list); in at_xdmac_device_terminate_all()
2034 if (desc->active_xfer) { in at_xdmac_device_terminate_all()
2035 pm_runtime_put_autosuspend(atxdmac->dev); in at_xdmac_device_terminate_all()
2036 pm_runtime_mark_last_busy(atxdmac->dev); in at_xdmac_device_terminate_all()
2040 clear_bit(AT_XDMAC_CHAN_IS_PAUSED, &atchan->status); in at_xdmac_device_terminate_all()
2041 clear_bit(AT_XDMAC_CHAN_IS_CYCLIC, &atchan->status); in at_xdmac_device_terminate_all()
2042 spin_unlock_irqrestore(&atchan->lock, flags); in at_xdmac_device_terminate_all()
2044 pm_runtime_mark_last_busy(atxdmac->dev); in at_xdmac_device_terminate_all()
2045 pm_runtime_put_autosuspend(atxdmac->dev); in at_xdmac_device_terminate_all()
2058 "can't allocate channel resources (channel enabled)\n"); in at_xdmac_alloc_chan_resources()
2059 return -EIO; in at_xdmac_alloc_chan_resources()
2062 if (!list_empty(&atchan->free_descs_list)) { in at_xdmac_alloc_chan_resources()
2064 "can't allocate channel resources (channel not free from a previous use)\n"); in at_xdmac_alloc_chan_resources()
2065 return -EIO; in at_xdmac_alloc_chan_resources()
2074 return -EIO; in at_xdmac_alloc_chan_resources()
2080 list_add_tail(&desc->desc_node, &atchan->free_descs_list); in at_xdmac_alloc_chan_resources()
2093 struct at_xdmac *atxdmac = to_at_xdmac(chan->device); in at_xdmac_free_chan_resources()
2096 list_for_each_entry_safe(desc, _desc, &atchan->free_descs_list, desc_node) { in at_xdmac_free_chan_resources()
2098 list_del(&desc->desc_node); in at_xdmac_free_chan_resources()
2099 dma_pool_free(atxdmac->at_xdmac_desc_pool, desc, desc->tx_dma_desc.phys); in at_xdmac_free_chan_resources()
2111 if (!atxdmac->layout->axi_config) in at_xdmac_axi_config()
2114 if (!of_property_read_u32(pdev->dev.of_node, "dma-requests", in at_xdmac_axi_config()
2116 dev_info(&pdev->dev, "controller in mem2mem mode.\n"); in at_xdmac_axi_config()
2134 list_for_each_entry_safe(chan, _chan, &atxdmac->dma.channels, device_node) { in atmel_xdmac_prepare()
2139 return -EAGAIN; in atmel_xdmac_prepare()
2150 ret = pm_runtime_resume_and_get(atxdmac->dev); in atmel_xdmac_suspend()
2154 list_for_each_entry_safe(chan, _chan, &atxdmac->dma.channels, device_node) { in atmel_xdmac_suspend()
2157 atchan->save_cc = at_xdmac_chan_read(atchan, AT_XDMAC_CC); in atmel_xdmac_suspend()
2160 dev_warn(chan2dev(chan), "%s: channel %d not paused\n", in atmel_xdmac_suspend()
2161 __func__, chan->chan_id); in atmel_xdmac_suspend()
2165 atchan->save_cim = at_xdmac_chan_read(atchan, AT_XDMAC_CIM); in atmel_xdmac_suspend()
2166 atchan->save_cnda = at_xdmac_chan_read(atchan, AT_XDMAC_CNDA); in atmel_xdmac_suspend()
2167 atchan->save_cndc = at_xdmac_chan_read(atchan, AT_XDMAC_CNDC); in atmel_xdmac_suspend()
2170 atxdmac->save_gim = at_xdmac_read(atxdmac, AT_XDMAC_GIM); in atmel_xdmac_suspend()
2171 atxdmac->save_gs = at_xdmac_read(atxdmac, AT_XDMAC_GS); in atmel_xdmac_suspend()
2174 pm_runtime_mark_last_busy(atxdmac->dev); in atmel_xdmac_suspend()
2175 pm_runtime_put_noidle(atxdmac->dev); in atmel_xdmac_suspend()
2176 clk_disable_unprepare(atxdmac->clk); in atmel_xdmac_suspend()
2189 ret = clk_prepare_enable(atxdmac->clk); in atmel_xdmac_resume()
2193 pm_runtime_get_noresume(atxdmac->dev); in atmel_xdmac_resume()
2198 for (i = 0; i < atxdmac->dma.chancnt; i++) { in atmel_xdmac_resume()
2199 atchan = &atxdmac->chan[i]; in atmel_xdmac_resume()
2204 at_xdmac_write(atxdmac, AT_XDMAC_GIE, atxdmac->save_gim); in atmel_xdmac_resume()
2205 list_for_each_entry_safe(chan, _chan, &atxdmac->dma.channels, device_node) { in atmel_xdmac_resume()
2208 at_xdmac_chan_write(atchan, AT_XDMAC_CC, atchan->save_cc); in atmel_xdmac_resume()
2223 * to DMA controller is cut-off. Thus, restore the in atmel_xdmac_resume()
2229 at_xdmac_chan_write(atchan, AT_XDMAC_CNDA, atchan->save_cnda); in atmel_xdmac_resume()
2230 at_xdmac_chan_write(atchan, AT_XDMAC_CNDC, atchan->save_cndc); in atmel_xdmac_resume()
2231 at_xdmac_chan_write(atchan, AT_XDMAC_CIE, atchan->save_cim); in atmel_xdmac_resume()
2233 if (atxdmac->save_gs & atchan->mask) in atmel_xdmac_resume()
2234 at_xdmac_write(atxdmac, AT_XDMAC_GE, atchan->mask); in atmel_xdmac_resume()
2238 pm_runtime_mark_last_busy(atxdmac->dev); in atmel_xdmac_resume()
2239 pm_runtime_put_autosuspend(atxdmac->dev); in atmel_xdmac_resume()
2248 clk_disable(atxdmac->clk); in atmel_xdmac_runtime_suspend()
2257 return clk_enable(atxdmac->clk); in atmel_xdmac_runtime_resume()
2283 dev_err(&pdev->dev, "invalid number of channels (%u)\n", in at_xdmac_probe()
2285 return -EINVAL; in at_xdmac_probe()
2288 atxdmac = devm_kzalloc(&pdev->dev, in at_xdmac_probe()
2292 dev_err(&pdev->dev, "can't allocate at_xdmac structure\n"); in at_xdmac_probe()
2293 return -ENOMEM; in at_xdmac_probe()
2296 atxdmac->regs = base; in at_xdmac_probe()
2297 atxdmac->irq = irq; in at_xdmac_probe()
2298 atxdmac->dev = &pdev->dev; in at_xdmac_probe()
2300 atxdmac->layout = of_device_get_match_data(&pdev->dev); in at_xdmac_probe()
2301 if (!atxdmac->layout) in at_xdmac_probe()
2302 return -ENODEV; in at_xdmac_probe()
2304 atxdmac->clk = devm_clk_get(&pdev->dev, "dma_clk"); in at_xdmac_probe()
2305 if (IS_ERR(atxdmac->clk)) { in at_xdmac_probe()
2306 dev_err(&pdev->dev, "can't get dma_clk\n"); in at_xdmac_probe()
2307 return PTR_ERR(atxdmac->clk); in at_xdmac_probe()
2311 ret = request_irq(atxdmac->irq, at_xdmac_interrupt, 0, "at_xdmac", atxdmac); in at_xdmac_probe()
2313 dev_err(&pdev->dev, "can't request irq\n"); in at_xdmac_probe()
2317 ret = clk_prepare_enable(atxdmac->clk); in at_xdmac_probe()
2319 dev_err(&pdev->dev, "can't prepare or enable clock\n"); in at_xdmac_probe()
2323 atxdmac->at_xdmac_desc_pool = in at_xdmac_probe()
2324 dmam_pool_create(dev_name(&pdev->dev), &pdev->dev, in at_xdmac_probe()
2326 if (!atxdmac->at_xdmac_desc_pool) { in at_xdmac_probe()
2327 dev_err(&pdev->dev, "no memory for descriptors dma pool\n"); in at_xdmac_probe()
2328 ret = -ENOMEM; in at_xdmac_probe()
2332 dma_cap_set(DMA_CYCLIC, atxdmac->dma.cap_mask); in at_xdmac_probe()
2333 dma_cap_set(DMA_INTERLEAVE, atxdmac->dma.cap_mask); in at_xdmac_probe()
2334 dma_cap_set(DMA_MEMCPY, atxdmac->dma.cap_mask); in at_xdmac_probe()
2335 dma_cap_set(DMA_MEMSET, atxdmac->dma.cap_mask); in at_xdmac_probe()
2336 dma_cap_set(DMA_MEMSET_SG, atxdmac->dma.cap_mask); in at_xdmac_probe()
2337 dma_cap_set(DMA_SLAVE, atxdmac->dma.cap_mask); in at_xdmac_probe()
2340 * one channel, second allocation fails in private_candidate. in at_xdmac_probe()
2342 dma_cap_set(DMA_PRIVATE, atxdmac->dma.cap_mask); in at_xdmac_probe()
2343 atxdmac->dma.dev = &pdev->dev; in at_xdmac_probe()
2344 atxdmac->dma.device_alloc_chan_resources = at_xdmac_alloc_chan_resources; in at_xdmac_probe()
2345 atxdmac->dma.device_free_chan_resources = at_xdmac_free_chan_resources; in at_xdmac_probe()
2346 atxdmac->dma.device_tx_status = at_xdmac_tx_status; in at_xdmac_probe()
2347 atxdmac->dma.device_issue_pending = at_xdmac_issue_pending; in at_xdmac_probe()
2348 atxdmac->dma.device_prep_dma_cyclic = at_xdmac_prep_dma_cyclic; in at_xdmac_probe()
2349 atxdmac->dma.device_prep_interleaved_dma = at_xdmac_prep_interleaved; in at_xdmac_probe()
2350 atxdmac->dma.device_prep_dma_memcpy = at_xdmac_prep_dma_memcpy; in at_xdmac_probe()
2351 atxdmac->dma.device_prep_dma_memset = at_xdmac_prep_dma_memset; in at_xdmac_probe()
2352 atxdmac->dma.device_prep_dma_memset_sg = at_xdmac_prep_dma_memset_sg; in at_xdmac_probe()
2353 atxdmac->dma.device_prep_slave_sg = at_xdmac_prep_slave_sg; in at_xdmac_probe()
2354 atxdmac->dma.device_config = at_xdmac_device_config; in at_xdmac_probe()
2355 atxdmac->dma.device_pause = at_xdmac_device_pause; in at_xdmac_probe()
2356 atxdmac->dma.device_resume = at_xdmac_device_resume; in at_xdmac_probe()
2357 atxdmac->dma.device_terminate_all = at_xdmac_device_terminate_all; in at_xdmac_probe()
2358 atxdmac->dma.src_addr_widths = AT_XDMAC_DMA_BUSWIDTHS; in at_xdmac_probe()
2359 atxdmac->dma.dst_addr_widths = AT_XDMAC_DMA_BUSWIDTHS; in at_xdmac_probe()
2360 atxdmac->dma.directions = BIT(DMA_DEV_TO_MEM) | BIT(DMA_MEM_TO_DEV); in at_xdmac_probe()
2361 atxdmac->dma.residue_granularity = DMA_RESIDUE_GRANULARITY_BURST; in at_xdmac_probe()
2365 pm_runtime_set_autosuspend_delay(&pdev->dev, 500); in at_xdmac_probe()
2366 pm_runtime_use_autosuspend(&pdev->dev); in at_xdmac_probe()
2367 pm_runtime_set_active(&pdev->dev); in at_xdmac_probe()
2368 pm_runtime_enable(&pdev->dev); in at_xdmac_probe()
2369 pm_runtime_get_noresume(&pdev->dev); in at_xdmac_probe()
2372 INIT_LIST_HEAD(&atxdmac->dma.channels); in at_xdmac_probe()
2378 struct at_xdmac_chan *atchan = &atxdmac->chan[i]; in at_xdmac_probe()
2380 atchan->chan.device = &atxdmac->dma; in at_xdmac_probe()
2381 list_add_tail(&atchan->chan.device_node, in at_xdmac_probe()
2382 &atxdmac->dma.channels); in at_xdmac_probe()
2384 atchan->ch_regs = at_xdmac_chan_reg_base(atxdmac, i); in at_xdmac_probe()
2385 atchan->mask = 1 << i; in at_xdmac_probe()
2387 spin_lock_init(&atchan->lock); in at_xdmac_probe()
2388 INIT_LIST_HEAD(&atchan->xfers_list); in at_xdmac_probe()
2389 INIT_LIST_HEAD(&atchan->free_descs_list); in at_xdmac_probe()
2390 tasklet_setup(&atchan->tasklet, at_xdmac_tasklet); in at_xdmac_probe()
2397 ret = dma_async_device_register(&atxdmac->dma); in at_xdmac_probe()
2399 dev_err(&pdev->dev, "fail to register DMA engine device\n"); in at_xdmac_probe()
2403 ret = of_dma_controller_register(pdev->dev.of_node, in at_xdmac_probe()
2406 dev_err(&pdev->dev, "could not register of dma controller\n"); in at_xdmac_probe()
2410 dev_info(&pdev->dev, "%d channels, mapped at 0x%p\n", in at_xdmac_probe()
2411 nr_channels, atxdmac->regs); in at_xdmac_probe()
2415 pm_runtime_mark_last_busy(&pdev->dev); in at_xdmac_probe()
2416 pm_runtime_put_autosuspend(&pdev->dev); in at_xdmac_probe()
2421 dma_async_device_unregister(&atxdmac->dma); in at_xdmac_probe()
2423 pm_runtime_put_noidle(&pdev->dev); in at_xdmac_probe()
2424 pm_runtime_disable(&pdev->dev); in at_xdmac_probe()
2425 pm_runtime_set_suspended(&pdev->dev); in at_xdmac_probe()
2426 pm_runtime_dont_use_autosuspend(&pdev->dev); in at_xdmac_probe()
2428 clk_disable_unprepare(atxdmac->clk); in at_xdmac_probe()
2430 free_irq(atxdmac->irq, atxdmac); in at_xdmac_probe()
2440 of_dma_controller_free(pdev->dev.of_node); in at_xdmac_remove()
2441 dma_async_device_unregister(&atxdmac->dma); in at_xdmac_remove()
2442 pm_runtime_disable(atxdmac->dev); in at_xdmac_remove()
2443 pm_runtime_set_suspended(&pdev->dev); in at_xdmac_remove()
2444 pm_runtime_dont_use_autosuspend(&pdev->dev); in at_xdmac_remove()
2445 clk_disable_unprepare(atxdmac->clk); in at_xdmac_remove()
2447 free_irq(atxdmac->irq, atxdmac); in at_xdmac_remove()
2449 for (i = 0; i < atxdmac->dma.chancnt; i++) { in at_xdmac_remove()
2450 struct at_xdmac_chan *atchan = &atxdmac->chan[i]; in at_xdmac_remove()
2452 tasklet_kill(&atchan->tasklet); in at_xdmac_remove()
2453 at_xdmac_free_chan_resources(&atchan->chan); in at_xdmac_remove()
2466 .compatible = "atmel,sama5d4-dma",
2469 .compatible = "microchip,sama7g5-dma",
2499 MODULE_DESCRIPTION("Atmel Extended DMA Controller driver");