Lines Matching +full:channel +full:-
1 // SPDX-License-Identifier: GPL-2.0
5 * Based on imx-dma.c
9 * Copyright 2012 Javier Martin, Vista Silicon <javier.martin@vista-silicon.com>
13 #include <linux/dma-mapping.h>
29 #include "../virt-dma.h"
108 * -----------------------------------------------------------------------------
171 * -----------------------------------------------------------------------------
178 writel(val, dmac->base + offset); in rz_dmac_writel()
184 writel(val, dmac->ext_base + offset); in rz_dmac_ext_writel()
189 return readl(dmac->ext_base + offset); in rz_dmac_ext_readl()
192 static void rz_dmac_ch_writel(struct rz_dmac_chan *channel, unsigned int val, in rz_dmac_ch_writel() argument
196 writel(val, channel->ch_base + offset); in rz_dmac_ch_writel()
198 writel(val, channel->ch_cmn_base + offset); in rz_dmac_ch_writel()
201 static u32 rz_dmac_ch_readl(struct rz_dmac_chan *channel, in rz_dmac_ch_readl() argument
205 return readl(channel->ch_base + offset); in rz_dmac_ch_readl()
207 return readl(channel->ch_cmn_base + offset); in rz_dmac_ch_readl()
211 * -----------------------------------------------------------------------------
215 static void rz_lmdesc_setup(struct rz_dmac_chan *channel, in rz_lmdesc_setup() argument
220 channel->lmdesc.base = lmdesc; in rz_lmdesc_setup()
221 channel->lmdesc.head = lmdesc; in rz_lmdesc_setup()
222 channel->lmdesc.tail = lmdesc; in rz_lmdesc_setup()
223 nxla = channel->lmdesc.base_dma; in rz_lmdesc_setup()
224 while (lmdesc < (channel->lmdesc.base + (DMAC_NR_LMDESC - 1))) { in rz_lmdesc_setup()
225 lmdesc->header = 0; in rz_lmdesc_setup()
227 lmdesc->nxla = nxla; in rz_lmdesc_setup()
231 lmdesc->header = 0; in rz_lmdesc_setup()
232 lmdesc->nxla = channel->lmdesc.base_dma; in rz_lmdesc_setup()
236 * -----------------------------------------------------------------------------
240 static void rz_dmac_lmdesc_recycle(struct rz_dmac_chan *channel) in rz_dmac_lmdesc_recycle() argument
242 struct rz_lmdesc *lmdesc = channel->lmdesc.head; in rz_dmac_lmdesc_recycle()
244 while (!(lmdesc->header & HEADER_LV)) { in rz_dmac_lmdesc_recycle()
245 lmdesc->header = 0; in rz_dmac_lmdesc_recycle()
247 if (lmdesc >= (channel->lmdesc.base + DMAC_NR_LMDESC)) in rz_dmac_lmdesc_recycle()
248 lmdesc = channel->lmdesc.base; in rz_dmac_lmdesc_recycle()
250 channel->lmdesc.head = lmdesc; in rz_dmac_lmdesc_recycle()
253 static void rz_dmac_enable_hw(struct rz_dmac_chan *channel) in rz_dmac_enable_hw() argument
255 struct dma_chan *chan = &channel->vc.chan; in rz_dmac_enable_hw()
256 struct rz_dmac *dmac = to_rz_dmac(chan->device); in rz_dmac_enable_hw()
262 dev_dbg(dmac->dev, "%s channel %d\n", __func__, channel->index); in rz_dmac_enable_hw()
266 rz_dmac_lmdesc_recycle(channel); in rz_dmac_enable_hw()
268 nxla = channel->lmdesc.base_dma + in rz_dmac_enable_hw()
269 (sizeof(struct rz_lmdesc) * (channel->lmdesc.head - in rz_dmac_enable_hw()
270 channel->lmdesc.base)); in rz_dmac_enable_hw()
272 chstat = rz_dmac_ch_readl(channel, CHSTAT, 1); in rz_dmac_enable_hw()
274 chctrl = (channel->chctrl | CHCTRL_SETEN); in rz_dmac_enable_hw()
275 rz_dmac_ch_writel(channel, nxla, NXLA, 1); in rz_dmac_enable_hw()
276 rz_dmac_ch_writel(channel, channel->chcfg, CHCFG, 1); in rz_dmac_enable_hw()
277 rz_dmac_ch_writel(channel, CHCTRL_SWRST, CHCTRL, 1); in rz_dmac_enable_hw()
278 rz_dmac_ch_writel(channel, chctrl, CHCTRL, 1); in rz_dmac_enable_hw()
284 static void rz_dmac_disable_hw(struct rz_dmac_chan *channel) in rz_dmac_disable_hw() argument
286 struct dma_chan *chan = &channel->vc.chan; in rz_dmac_disable_hw()
287 struct rz_dmac *dmac = to_rz_dmac(chan->device); in rz_dmac_disable_hw()
290 dev_dbg(dmac->dev, "%s channel %d\n", __func__, channel->index); in rz_dmac_disable_hw()
293 rz_dmac_ch_writel(channel, CHCTRL_DEFAULT, CHCTRL, 1); in rz_dmac_disable_hw()
310 static void rz_dmac_prepare_desc_for_memcpy(struct rz_dmac_chan *channel) in rz_dmac_prepare_desc_for_memcpy() argument
312 struct dma_chan *chan = &channel->vc.chan; in rz_dmac_prepare_desc_for_memcpy()
313 struct rz_dmac *dmac = to_rz_dmac(chan->device); in rz_dmac_prepare_desc_for_memcpy()
314 struct rz_lmdesc *lmdesc = channel->lmdesc.tail; in rz_dmac_prepare_desc_for_memcpy()
315 struct rz_dmac_desc *d = channel->desc; in rz_dmac_prepare_desc_for_memcpy()
319 lmdesc->sa = d->src; in rz_dmac_prepare_desc_for_memcpy()
320 lmdesc->da = d->dest; in rz_dmac_prepare_desc_for_memcpy()
321 lmdesc->tb = d->len; in rz_dmac_prepare_desc_for_memcpy()
322 lmdesc->chcfg = chcfg; in rz_dmac_prepare_desc_for_memcpy()
323 lmdesc->chitvl = 0; in rz_dmac_prepare_desc_for_memcpy()
324 lmdesc->chext = 0; in rz_dmac_prepare_desc_for_memcpy()
325 lmdesc->header = HEADER_LV; in rz_dmac_prepare_desc_for_memcpy()
327 rz_dmac_set_dmars_register(dmac, channel->index, 0); in rz_dmac_prepare_desc_for_memcpy()
329 channel->chcfg = chcfg; in rz_dmac_prepare_desc_for_memcpy()
330 channel->chctrl = CHCTRL_STG | CHCTRL_SETEN; in rz_dmac_prepare_desc_for_memcpy()
333 static void rz_dmac_prepare_descs_for_slave_sg(struct rz_dmac_chan *channel) in rz_dmac_prepare_descs_for_slave_sg() argument
335 struct dma_chan *chan = &channel->vc.chan; in rz_dmac_prepare_descs_for_slave_sg()
336 struct rz_dmac *dmac = to_rz_dmac(chan->device); in rz_dmac_prepare_descs_for_slave_sg()
337 struct rz_dmac_desc *d = channel->desc; in rz_dmac_prepare_descs_for_slave_sg()
338 struct scatterlist *sg, *sgl = d->sg; in rz_dmac_prepare_descs_for_slave_sg()
340 unsigned int i, sg_len = d->sgcount; in rz_dmac_prepare_descs_for_slave_sg()
342 channel->chcfg |= CHCFG_SEL(channel->index) | CHCFG_DEM | CHCFG_DMS; in rz_dmac_prepare_descs_for_slave_sg()
344 if (d->direction == DMA_DEV_TO_MEM) { in rz_dmac_prepare_descs_for_slave_sg()
345 channel->chcfg |= CHCFG_SAD; in rz_dmac_prepare_descs_for_slave_sg()
346 channel->chcfg &= ~CHCFG_REQD; in rz_dmac_prepare_descs_for_slave_sg()
348 channel->chcfg |= CHCFG_DAD | CHCFG_REQD; in rz_dmac_prepare_descs_for_slave_sg()
351 lmdesc = channel->lmdesc.tail; in rz_dmac_prepare_descs_for_slave_sg()
354 if (d->direction == DMA_DEV_TO_MEM) { in rz_dmac_prepare_descs_for_slave_sg()
355 lmdesc->sa = channel->src_per_address; in rz_dmac_prepare_descs_for_slave_sg()
356 lmdesc->da = sg_dma_address(sg); in rz_dmac_prepare_descs_for_slave_sg()
358 lmdesc->sa = sg_dma_address(sg); in rz_dmac_prepare_descs_for_slave_sg()
359 lmdesc->da = channel->dst_per_address; in rz_dmac_prepare_descs_for_slave_sg()
362 lmdesc->tb = sg_dma_len(sg); in rz_dmac_prepare_descs_for_slave_sg()
363 lmdesc->chitvl = 0; in rz_dmac_prepare_descs_for_slave_sg()
364 lmdesc->chext = 0; in rz_dmac_prepare_descs_for_slave_sg()
365 if (i == (sg_len - 1)) { in rz_dmac_prepare_descs_for_slave_sg()
366 lmdesc->chcfg = (channel->chcfg & ~CHCFG_DEM); in rz_dmac_prepare_descs_for_slave_sg()
367 lmdesc->header = HEADER_LV; in rz_dmac_prepare_descs_for_slave_sg()
369 lmdesc->chcfg = channel->chcfg; in rz_dmac_prepare_descs_for_slave_sg()
370 lmdesc->header = HEADER_LV; in rz_dmac_prepare_descs_for_slave_sg()
372 if (++lmdesc >= (channel->lmdesc.base + DMAC_NR_LMDESC)) in rz_dmac_prepare_descs_for_slave_sg()
373 lmdesc = channel->lmdesc.base; in rz_dmac_prepare_descs_for_slave_sg()
376 channel->lmdesc.tail = lmdesc; in rz_dmac_prepare_descs_for_slave_sg()
378 rz_dmac_set_dmars_register(dmac, channel->index, channel->mid_rid); in rz_dmac_prepare_descs_for_slave_sg()
379 channel->chctrl = CHCTRL_SETEN; in rz_dmac_prepare_descs_for_slave_sg()
384 struct rz_dmac_desc *d = chan->desc; in rz_dmac_xfer_desc()
387 vd = vchan_next_desc(&chan->vc); in rz_dmac_xfer_desc()
391 list_del(&vd->node); in rz_dmac_xfer_desc()
393 switch (d->type) { in rz_dmac_xfer_desc()
403 return -EINVAL; in rz_dmac_xfer_desc()
412 * -----------------------------------------------------------------------------
418 struct rz_dmac_chan *channel = to_rz_dmac_chan(chan); in rz_dmac_alloc_chan_resources() local
420 while (channel->descs_allocated < RZ_DMAC_MAX_CHAN_DESCRIPTORS) { in rz_dmac_alloc_chan_resources()
427 list_add_tail(&desc->node, &channel->ld_free); in rz_dmac_alloc_chan_resources()
428 channel->descs_allocated++; in rz_dmac_alloc_chan_resources()
431 if (!channel->descs_allocated) in rz_dmac_alloc_chan_resources()
432 return -ENOMEM; in rz_dmac_alloc_chan_resources()
434 return channel->descs_allocated; in rz_dmac_alloc_chan_resources()
439 struct rz_dmac_chan *channel = to_rz_dmac_chan(chan); in rz_dmac_free_chan_resources() local
440 struct rz_dmac *dmac = to_rz_dmac(chan->device); in rz_dmac_free_chan_resources()
441 struct rz_lmdesc *lmdesc = channel->lmdesc.base; in rz_dmac_free_chan_resources()
446 spin_lock_irqsave(&channel->vc.lock, flags); in rz_dmac_free_chan_resources()
451 rz_dmac_disable_hw(channel); in rz_dmac_free_chan_resources()
452 list_splice_tail_init(&channel->ld_active, &channel->ld_free); in rz_dmac_free_chan_resources()
453 list_splice_tail_init(&channel->ld_queue, &channel->ld_free); in rz_dmac_free_chan_resources()
455 if (channel->mid_rid >= 0) { in rz_dmac_free_chan_resources()
456 clear_bit(channel->mid_rid, dmac->modules); in rz_dmac_free_chan_resources()
457 channel->mid_rid = -EINVAL; in rz_dmac_free_chan_resources()
460 spin_unlock_irqrestore(&channel->vc.lock, flags); in rz_dmac_free_chan_resources()
462 list_for_each_entry_safe(desc, _desc, &channel->ld_free, node) { in rz_dmac_free_chan_resources()
464 channel->descs_allocated--; in rz_dmac_free_chan_resources()
467 INIT_LIST_HEAD(&channel->ld_free); in rz_dmac_free_chan_resources()
468 vchan_free_chan_resources(&channel->vc); in rz_dmac_free_chan_resources()
475 struct rz_dmac_chan *channel = to_rz_dmac_chan(chan); in rz_dmac_prep_dma_memcpy() local
476 struct rz_dmac *dmac = to_rz_dmac(chan->device); in rz_dmac_prep_dma_memcpy()
479 dev_dbg(dmac->dev, "%s channel: %d src=0x%pad dst=0x%pad len=%zu\n", in rz_dmac_prep_dma_memcpy()
480 __func__, channel->index, &src, &dest, len); in rz_dmac_prep_dma_memcpy()
482 if (list_empty(&channel->ld_free)) in rz_dmac_prep_dma_memcpy()
485 desc = list_first_entry(&channel->ld_free, struct rz_dmac_desc, node); in rz_dmac_prep_dma_memcpy()
487 desc->type = RZ_DMAC_DESC_MEMCPY; in rz_dmac_prep_dma_memcpy()
488 desc->src = src; in rz_dmac_prep_dma_memcpy()
489 desc->dest = dest; in rz_dmac_prep_dma_memcpy()
490 desc->len = len; in rz_dmac_prep_dma_memcpy()
491 desc->direction = DMA_MEM_TO_MEM; in rz_dmac_prep_dma_memcpy()
493 list_move_tail(channel->ld_free.next, &channel->ld_queue); in rz_dmac_prep_dma_memcpy()
494 return vchan_tx_prep(&channel->vc, &desc->vd, flags); in rz_dmac_prep_dma_memcpy()
503 struct rz_dmac_chan *channel = to_rz_dmac_chan(chan); in rz_dmac_prep_slave_sg() local
509 if (list_empty(&channel->ld_free)) in rz_dmac_prep_slave_sg()
512 desc = list_first_entry(&channel->ld_free, struct rz_dmac_desc, node); in rz_dmac_prep_slave_sg()
518 desc->type = RZ_DMAC_DESC_SLAVE_SG; in rz_dmac_prep_slave_sg()
519 desc->sg = sgl; in rz_dmac_prep_slave_sg()
520 desc->sgcount = sg_len; in rz_dmac_prep_slave_sg()
521 desc->len = dma_length; in rz_dmac_prep_slave_sg()
522 desc->direction = direction; in rz_dmac_prep_slave_sg()
525 desc->src = channel->src_per_address; in rz_dmac_prep_slave_sg()
527 desc->dest = channel->dst_per_address; in rz_dmac_prep_slave_sg()
529 list_move_tail(channel->ld_free.next, &channel->ld_queue); in rz_dmac_prep_slave_sg()
530 return vchan_tx_prep(&channel->vc, &desc->vd, flags); in rz_dmac_prep_slave_sg()
535 struct rz_dmac_chan *channel = to_rz_dmac_chan(chan); in rz_dmac_terminate_all() local
539 rz_dmac_disable_hw(channel); in rz_dmac_terminate_all()
540 spin_lock_irqsave(&channel->vc.lock, flags); in rz_dmac_terminate_all()
541 list_splice_tail_init(&channel->ld_active, &channel->ld_free); in rz_dmac_terminate_all()
542 list_splice_tail_init(&channel->ld_queue, &channel->ld_free); in rz_dmac_terminate_all()
543 vchan_get_all_descriptors(&channel->vc, &head); in rz_dmac_terminate_all()
544 spin_unlock_irqrestore(&channel->vc.lock, flags); in rz_dmac_terminate_all()
545 vchan_dma_desc_free_list(&channel->vc, &head); in rz_dmac_terminate_all()
552 struct rz_dmac_chan *channel = to_rz_dmac_chan(chan); in rz_dmac_issue_pending() local
553 struct rz_dmac *dmac = to_rz_dmac(chan->device); in rz_dmac_issue_pending()
557 spin_lock_irqsave(&channel->vc.lock, flags); in rz_dmac_issue_pending()
559 if (!list_empty(&channel->ld_queue)) { in rz_dmac_issue_pending()
560 desc = list_first_entry(&channel->ld_queue, in rz_dmac_issue_pending()
562 channel->desc = desc; in rz_dmac_issue_pending()
563 if (vchan_issue_pending(&channel->vc)) { in rz_dmac_issue_pending()
564 if (rz_dmac_xfer_desc(channel) < 0) in rz_dmac_issue_pending()
565 dev_warn(dmac->dev, "ch: %d couldn't issue DMA xfer\n", in rz_dmac_issue_pending()
566 channel->index); in rz_dmac_issue_pending()
568 list_move_tail(channel->ld_queue.next, in rz_dmac_issue_pending()
569 &channel->ld_active); in rz_dmac_issue_pending()
573 spin_unlock_irqrestore(&channel->vc.lock, flags); in rz_dmac_issue_pending()
601 struct rz_dmac_chan *channel = to_rz_dmac_chan(chan); in rz_dmac_config() local
604 channel->dst_per_address = config->dst_addr; in rz_dmac_config()
605 channel->chcfg &= ~CHCFG_FILL_DDS_MASK; in rz_dmac_config()
606 if (channel->dst_per_address) { in rz_dmac_config()
607 val = rz_dmac_ds_to_val_mapping(config->dst_addr_width); in rz_dmac_config()
609 return -EINVAL; in rz_dmac_config()
611 channel->chcfg |= FIELD_PREP(CHCFG_FILL_DDS_MASK, val); in rz_dmac_config()
614 channel->src_per_address = config->src_addr; in rz_dmac_config()
615 channel->chcfg &= ~CHCFG_FILL_SDS_MASK; in rz_dmac_config()
616 if (channel->src_per_address) { in rz_dmac_config()
617 val = rz_dmac_ds_to_val_mapping(config->src_addr_width); in rz_dmac_config()
619 return -EINVAL; in rz_dmac_config()
621 channel->chcfg |= FIELD_PREP(CHCFG_FILL_SDS_MASK, val); in rz_dmac_config()
640 struct rz_dmac_chan *channel = to_rz_dmac_chan(chan); in rz_dmac_device_synchronize() local
641 struct rz_dmac *dmac = to_rz_dmac(chan->device); in rz_dmac_device_synchronize()
646 100, 100000, false, channel, CHSTAT, 1); in rz_dmac_device_synchronize()
648 dev_warn(dmac->dev, "DMA Timeout"); in rz_dmac_device_synchronize()
650 rz_dmac_set_dmars_register(dmac, channel->index, 0); in rz_dmac_device_synchronize()
654 * -----------------------------------------------------------------------------
658 static void rz_dmac_irq_handle_channel(struct rz_dmac_chan *channel) in rz_dmac_irq_handle_channel() argument
660 struct dma_chan *chan = &channel->vc.chan; in rz_dmac_irq_handle_channel()
661 struct rz_dmac *dmac = to_rz_dmac(chan->device); in rz_dmac_irq_handle_channel()
664 chstat = rz_dmac_ch_readl(channel, CHSTAT, 1); in rz_dmac_irq_handle_channel()
666 dev_err(dmac->dev, "DMAC err CHSTAT_%d = %08X\n", in rz_dmac_irq_handle_channel()
667 channel->index, chstat); in rz_dmac_irq_handle_channel()
668 rz_dmac_ch_writel(channel, CHCTRL_DEFAULT, CHCTRL, 1); in rz_dmac_irq_handle_channel()
672 chctrl = rz_dmac_ch_readl(channel, CHCTRL, 1); in rz_dmac_irq_handle_channel()
673 rz_dmac_ch_writel(channel, chctrl | CHCTRL_CLREND, CHCTRL, 1); in rz_dmac_irq_handle_channel()
680 struct rz_dmac_chan *channel = dev_id; in rz_dmac_irq_handler() local
682 if (channel) { in rz_dmac_irq_handler()
683 rz_dmac_irq_handle_channel(channel); in rz_dmac_irq_handler()
692 struct rz_dmac_chan *channel = dev_id; in rz_dmac_irq_handler_thread() local
696 spin_lock_irqsave(&channel->vc.lock, flags); in rz_dmac_irq_handler_thread()
698 if (list_empty(&channel->ld_active)) { in rz_dmac_irq_handler_thread()
703 desc = list_first_entry(&channel->ld_active, struct rz_dmac_desc, node); in rz_dmac_irq_handler_thread()
704 vchan_cookie_complete(&desc->vd); in rz_dmac_irq_handler_thread()
705 list_move_tail(channel->ld_active.next, &channel->ld_free); in rz_dmac_irq_handler_thread()
706 if (!list_empty(&channel->ld_queue)) { in rz_dmac_irq_handler_thread()
707 desc = list_first_entry(&channel->ld_queue, struct rz_dmac_desc, in rz_dmac_irq_handler_thread()
709 channel->desc = desc; in rz_dmac_irq_handler_thread()
710 if (rz_dmac_xfer_desc(channel) == 0) in rz_dmac_irq_handler_thread()
711 list_move_tail(channel->ld_queue.next, &channel->ld_active); in rz_dmac_irq_handler_thread()
714 spin_unlock_irqrestore(&channel->vc.lock, flags); in rz_dmac_irq_handler_thread()
720 * -----------------------------------------------------------------------------
721 * OF xlate and channel filter
726 struct rz_dmac_chan *channel = to_rz_dmac_chan(chan); in rz_dmac_chan_filter() local
727 struct rz_dmac *dmac = to_rz_dmac(chan->device); in rz_dmac_chan_filter()
731 channel->mid_rid = dma_spec->args[0] & MID_RID_MASK; in rz_dmac_chan_filter()
732 ch_cfg = (dma_spec->args[0] & CHCFG_MASK) >> 10; in rz_dmac_chan_filter()
733 channel->chcfg = CHCFG_FILL_TM(ch_cfg) | CHCFG_FILL_AM(ch_cfg) | in rz_dmac_chan_filter()
736 return !test_and_set_bit(channel->mid_rid, dmac->modules); in rz_dmac_chan_filter()
744 if (dma_spec->args_count != 1) in rz_dmac_of_xlate()
755 * -----------------------------------------------------------------------------
760 struct rz_dmac_chan *channel, in rz_dmac_chan_probe() argument
763 struct platform_device *pdev = to_platform_device(dmac->dev); in rz_dmac_chan_probe()
769 channel->index = index; in rz_dmac_chan_probe()
770 channel->mid_rid = -EINVAL; in rz_dmac_chan_probe()
772 /* Request the channel interrupt. */ in rz_dmac_chan_probe()
774 channel->irq = platform_get_irq_byname(pdev, pdev_irqname); in rz_dmac_chan_probe()
775 if (channel->irq < 0) in rz_dmac_chan_probe()
776 return channel->irq; in rz_dmac_chan_probe()
778 irqname = devm_kasprintf(dmac->dev, GFP_KERNEL, "%s:%u", in rz_dmac_chan_probe()
779 dev_name(dmac->dev), index); in rz_dmac_chan_probe()
781 return -ENOMEM; in rz_dmac_chan_probe()
783 ret = devm_request_threaded_irq(dmac->dev, channel->irq, in rz_dmac_chan_probe()
786 irqname, channel); in rz_dmac_chan_probe()
788 dev_err(dmac->dev, "failed to request IRQ %u (%d)\n", in rz_dmac_chan_probe()
789 channel->irq, ret); in rz_dmac_chan_probe()
793 /* Set io base address for each channel */ in rz_dmac_chan_probe()
795 channel->ch_base = dmac->base + CHANNEL_0_7_OFFSET + in rz_dmac_chan_probe()
797 channel->ch_cmn_base = dmac->base + CHANNEL_0_7_COMMON_BASE; in rz_dmac_chan_probe()
799 channel->ch_base = dmac->base + CHANNEL_8_15_OFFSET + in rz_dmac_chan_probe()
800 EACH_CHANNEL_OFFSET * (index - 8); in rz_dmac_chan_probe()
801 channel->ch_cmn_base = dmac->base + CHANNEL_8_15_COMMON_BASE; in rz_dmac_chan_probe()
805 lmdesc = dma_alloc_coherent(&pdev->dev, in rz_dmac_chan_probe()
807 &channel->lmdesc.base_dma, GFP_KERNEL); in rz_dmac_chan_probe()
809 dev_err(&pdev->dev, "Can't allocate memory (lmdesc)\n"); in rz_dmac_chan_probe()
810 return -ENOMEM; in rz_dmac_chan_probe()
812 rz_lmdesc_setup(channel, lmdesc); in rz_dmac_chan_probe()
814 /* Initialize register for each channel */ in rz_dmac_chan_probe()
815 rz_dmac_ch_writel(channel, CHCTRL_DEFAULT, CHCTRL, 1); in rz_dmac_chan_probe()
817 channel->vc.desc_free = rz_dmac_virt_desc_free; in rz_dmac_chan_probe()
818 vchan_init(&channel->vc, &dmac->engine); in rz_dmac_chan_probe()
819 INIT_LIST_HEAD(&channel->ld_queue); in rz_dmac_chan_probe()
820 INIT_LIST_HEAD(&channel->ld_free); in rz_dmac_chan_probe()
821 INIT_LIST_HEAD(&channel->ld_active); in rz_dmac_chan_probe()
828 struct device_node *np = dev->of_node; in rz_dmac_parse_of()
831 ret = of_property_read_u32(np, "dma-channels", &dmac->n_channels); in rz_dmac_parse_of()
833 dev_err(dev, "unable to read dma-channels property\n"); in rz_dmac_parse_of()
837 if (!dmac->n_channels || dmac->n_channels > RZ_DMAC_MAX_CHANNELS) { in rz_dmac_parse_of()
838 dev_err(dev, "invalid number of channels %u\n", dmac->n_channels); in rz_dmac_parse_of()
839 return -EINVAL; in rz_dmac_parse_of()
855 dmac = devm_kzalloc(&pdev->dev, sizeof(*dmac), GFP_KERNEL); in rz_dmac_probe()
857 return -ENOMEM; in rz_dmac_probe()
859 dmac->dev = &pdev->dev; in rz_dmac_probe()
862 ret = rz_dmac_parse_of(&pdev->dev, dmac); in rz_dmac_probe()
866 dmac->channels = devm_kcalloc(&pdev->dev, dmac->n_channels, in rz_dmac_probe()
867 sizeof(*dmac->channels), GFP_KERNEL); in rz_dmac_probe()
868 if (!dmac->channels) in rz_dmac_probe()
869 return -ENOMEM; in rz_dmac_probe()
872 dmac->base = devm_platform_ioremap_resource(pdev, 0); in rz_dmac_probe()
873 if (IS_ERR(dmac->base)) in rz_dmac_probe()
874 return PTR_ERR(dmac->base); in rz_dmac_probe()
876 dmac->ext_base = devm_platform_ioremap_resource(pdev, 1); in rz_dmac_probe()
877 if (IS_ERR(dmac->ext_base)) in rz_dmac_probe()
878 return PTR_ERR(dmac->ext_base); in rz_dmac_probe()
885 ret = devm_request_irq(&pdev->dev, irq, rz_dmac_irq_handler, 0, in rz_dmac_probe()
888 dev_err(&pdev->dev, "failed to request IRQ %u (%d)\n", in rz_dmac_probe()
894 INIT_LIST_HEAD(&dmac->engine.channels); in rz_dmac_probe()
896 dmac->rstc = devm_reset_control_array_get_exclusive(&pdev->dev); in rz_dmac_probe()
897 if (IS_ERR(dmac->rstc)) in rz_dmac_probe()
898 return dev_err_probe(&pdev->dev, PTR_ERR(dmac->rstc), in rz_dmac_probe()
901 pm_runtime_enable(&pdev->dev); in rz_dmac_probe()
902 ret = pm_runtime_resume_and_get(&pdev->dev); in rz_dmac_probe()
904 dev_err(&pdev->dev, "pm_runtime_resume_and_get failed\n"); in rz_dmac_probe()
908 ret = reset_control_deassert(dmac->rstc); in rz_dmac_probe()
912 for (i = 0; i < dmac->n_channels; i++) { in rz_dmac_probe()
913 ret = rz_dmac_chan_probe(dmac, &dmac->channels[i], i); in rz_dmac_probe()
919 ret = of_dma_controller_register(pdev->dev.of_node, rz_dmac_of_xlate, in rz_dmac_probe()
925 engine = &dmac->engine; in rz_dmac_probe()
926 dma_cap_set(DMA_SLAVE, engine->cap_mask); in rz_dmac_probe()
927 dma_cap_set(DMA_MEMCPY, engine->cap_mask); in rz_dmac_probe()
931 engine->dev = &pdev->dev; in rz_dmac_probe()
933 engine->device_alloc_chan_resources = rz_dmac_alloc_chan_resources; in rz_dmac_probe()
934 engine->device_free_chan_resources = rz_dmac_free_chan_resources; in rz_dmac_probe()
935 engine->device_tx_status = dma_cookie_status; in rz_dmac_probe()
936 engine->device_prep_slave_sg = rz_dmac_prep_slave_sg; in rz_dmac_probe()
937 engine->device_prep_dma_memcpy = rz_dmac_prep_dma_memcpy; in rz_dmac_probe()
938 engine->device_config = rz_dmac_config; in rz_dmac_probe()
939 engine->device_terminate_all = rz_dmac_terminate_all; in rz_dmac_probe()
940 engine->device_issue_pending = rz_dmac_issue_pending; in rz_dmac_probe()
941 engine->device_synchronize = rz_dmac_device_synchronize; in rz_dmac_probe()
943 engine->copy_align = DMAENGINE_ALIGN_1_BYTE; in rz_dmac_probe()
944 dma_set_max_seg_size(engine->dev, U32_MAX); in rz_dmac_probe()
948 dev_err(&pdev->dev, "unable to register\n"); in rz_dmac_probe()
954 of_dma_controller_free(pdev->dev.of_node); in rz_dmac_probe()
956 channel_num = i ? i - 1 : 0; in rz_dmac_probe()
958 struct rz_dmac_chan *channel = &dmac->channels[i]; in rz_dmac_probe() local
960 dma_free_coherent(&pdev->dev, in rz_dmac_probe()
962 channel->lmdesc.base, in rz_dmac_probe()
963 channel->lmdesc.base_dma); in rz_dmac_probe()
966 reset_control_assert(dmac->rstc); in rz_dmac_probe()
968 pm_runtime_put(&pdev->dev); in rz_dmac_probe()
970 pm_runtime_disable(&pdev->dev); in rz_dmac_probe()
980 dma_async_device_unregister(&dmac->engine); in rz_dmac_remove()
981 of_dma_controller_free(pdev->dev.of_node); in rz_dmac_remove()
982 for (i = 0; i < dmac->n_channels; i++) { in rz_dmac_remove()
983 struct rz_dmac_chan *channel = &dmac->channels[i]; in rz_dmac_remove() local
985 dma_free_coherent(&pdev->dev, in rz_dmac_remove()
987 channel->lmdesc.base, in rz_dmac_remove()
988 channel->lmdesc.base_dma); in rz_dmac_remove()
990 reset_control_assert(dmac->rstc); in rz_dmac_remove()
991 pm_runtime_put(&pdev->dev); in rz_dmac_remove()
992 pm_runtime_disable(&pdev->dev); in rz_dmac_remove()
996 { .compatible = "renesas,rz-dmac", },
1003 .name = "rz-dmac",