Lines Matching +full:fu540 +full:- +full:c000 +full:- +full:pdma
1 // SPDX-License-Identifier: GPL-2.0-or-later
3 * SiFive FU540 Platform DMA driver
7 * - drivers/dma/fsl-edma.c
8 * - drivers/dma/dw-edma/
9 * - drivers/dma/pxa-dma.c
12 * - Chapter 12 "Platform DMA Engine (PDMA)" of
13 * SiFive FU540-C000 v1.0
14 * https://static.dev.sifive.com/FU540-C000-v1.0.pdf
21 #include <linux/dma-mapping.h>
26 #include "sf-pdma.h"
63 desc->chan = chan; in sf_pdma_alloc_desc()
71 desc->xfer_type = desc->chan->pdma->transfer_type; in sf_pdma_fill_desc()
72 desc->xfer_size = size; in sf_pdma_fill_desc()
73 desc->dst_addr = dst; in sf_pdma_fill_desc()
74 desc->src_addr = src; in sf_pdma_fill_desc()
79 struct pdma_regs *regs = &chan->regs; in sf_pdma_disclaim_chan()
81 writel(PDMA_CLEAR_CTRL, regs->ctrl); in sf_pdma_disclaim_chan()
93 dev_err(chan->pdma->dma_dev.dev, in sf_pdma_prep_dma_memcpy()
102 desc->dirn = DMA_MEM_TO_MEM; in sf_pdma_prep_dma_memcpy()
103 desc->async_tx = vchan_tx_prep(&chan->vchan, &desc->vdesc, flags); in sf_pdma_prep_dma_memcpy()
105 spin_lock_irqsave(&chan->vchan.lock, iflags); in sf_pdma_prep_dma_memcpy()
107 spin_unlock_irqrestore(&chan->vchan.lock, iflags); in sf_pdma_prep_dma_memcpy()
109 return desc->async_tx; in sf_pdma_prep_dma_memcpy()
117 memcpy(&chan->cfg, cfg, sizeof(*cfg)); in sf_pdma_slave_config()
125 struct pdma_regs *regs = &chan->regs; in sf_pdma_alloc_chan_resources()
128 writel(PDMA_CLAIM_MASK, regs->ctrl); in sf_pdma_alloc_chan_resources()
135 struct pdma_regs *regs = &chan->regs; in sf_pdma_disable_request()
137 writel(readl(regs->ctrl) & ~PDMA_RUN_MASK, regs->ctrl); in sf_pdma_disable_request()
146 spin_lock_irqsave(&chan->vchan.lock, flags); in sf_pdma_free_chan_resources()
148 kfree(chan->desc); in sf_pdma_free_chan_resources()
149 chan->desc = NULL; in sf_pdma_free_chan_resources()
150 vchan_get_all_descriptors(&chan->vchan, &head); in sf_pdma_free_chan_resources()
152 spin_unlock_irqrestore(&chan->vchan.lock, flags); in sf_pdma_free_chan_resources()
153 vchan_dma_desc_free_list(&chan->vchan, &head); in sf_pdma_free_chan_resources()
160 struct pdma_regs *regs = &chan->regs; in sf_pdma_desc_residue()
166 spin_lock_irqsave(&chan->vchan.lock, flags); in sf_pdma_desc_residue()
168 list_for_each_entry(vd, &chan->vchan.desc_submitted, node) in sf_pdma_desc_residue()
169 if (vd->tx.cookie == cookie) in sf_pdma_desc_residue()
170 tx = &vd->tx; in sf_pdma_desc_residue()
175 if (cookie == tx->chan->completed_cookie) in sf_pdma_desc_residue()
178 if (cookie == tx->cookie) { in sf_pdma_desc_residue()
179 residue = readq(regs->residue); in sf_pdma_desc_residue()
181 vd = vchan_find_desc(&chan->vchan, cookie); in sf_pdma_desc_residue()
186 residue = desc->xfer_size; in sf_pdma_desc_residue()
190 spin_unlock_irqrestore(&chan->vchan.lock, flags); in sf_pdma_desc_residue()
216 spin_lock_irqsave(&chan->vchan.lock, flags); in sf_pdma_terminate_all()
218 kfree(chan->desc); in sf_pdma_terminate_all()
219 chan->desc = NULL; in sf_pdma_terminate_all()
220 chan->xfer_err = false; in sf_pdma_terminate_all()
221 vchan_get_all_descriptors(&chan->vchan, &head); in sf_pdma_terminate_all()
222 spin_unlock_irqrestore(&chan->vchan.lock, flags); in sf_pdma_terminate_all()
223 vchan_dma_desc_free_list(&chan->vchan, &head); in sf_pdma_terminate_all()
230 struct pdma_regs *regs = &chan->regs; in sf_pdma_enable_request()
238 writel(v, regs->ctrl); in sf_pdma_enable_request()
243 struct virt_dma_chan *vchan = &chan->vchan; in sf_pdma_get_first_pending_desc()
246 if (list_empty(&vchan->desc_issued)) in sf_pdma_get_first_pending_desc()
249 vdesc = list_first_entry(&vchan->desc_issued, struct virt_dma_desc, node); in sf_pdma_get_first_pending_desc()
256 struct sf_pdma_desc *desc = chan->desc; in sf_pdma_xfer_desc()
257 struct pdma_regs *regs = &chan->regs; in sf_pdma_xfer_desc()
260 dev_err(chan->pdma->dma_dev.dev, "NULL desc.\n"); in sf_pdma_xfer_desc()
264 writel(desc->xfer_type, regs->xfer_type); in sf_pdma_xfer_desc()
265 writeq(desc->xfer_size, regs->xfer_size); in sf_pdma_xfer_desc()
266 writeq(desc->dst_addr, regs->dst_addr); in sf_pdma_xfer_desc()
267 writeq(desc->src_addr, regs->src_addr); in sf_pdma_xfer_desc()
269 chan->desc = desc; in sf_pdma_xfer_desc()
270 chan->status = DMA_IN_PROGRESS; in sf_pdma_xfer_desc()
279 spin_lock_irqsave(&chan->vchan.lock, flags); in sf_pdma_issue_pending()
281 if (!chan->desc && vchan_issue_pending(&chan->vchan)) { in sf_pdma_issue_pending()
283 chan->desc = sf_pdma_get_first_pending_desc(chan); in sf_pdma_issue_pending()
287 spin_unlock_irqrestore(&chan->vchan.lock, flags); in sf_pdma_issue_pending()
303 spin_lock_irqsave(&chan->lock, flags); in sf_pdma_donebh_tasklet()
304 if (chan->xfer_err) { in sf_pdma_donebh_tasklet()
305 chan->retries = MAX_RETRY; in sf_pdma_donebh_tasklet()
306 chan->status = DMA_COMPLETE; in sf_pdma_donebh_tasklet()
307 chan->xfer_err = false; in sf_pdma_donebh_tasklet()
309 spin_unlock_irqrestore(&chan->lock, flags); in sf_pdma_donebh_tasklet()
311 spin_lock_irqsave(&chan->vchan.lock, flags); in sf_pdma_donebh_tasklet()
312 list_del(&chan->desc->vdesc.node); in sf_pdma_donebh_tasklet()
313 vchan_cookie_complete(&chan->desc->vdesc); in sf_pdma_donebh_tasklet()
315 chan->desc = sf_pdma_get_first_pending_desc(chan); in sf_pdma_donebh_tasklet()
316 if (chan->desc) in sf_pdma_donebh_tasklet()
319 spin_unlock_irqrestore(&chan->vchan.lock, flags); in sf_pdma_donebh_tasklet()
325 struct sf_pdma_desc *desc = chan->desc; in sf_pdma_errbh_tasklet()
328 spin_lock_irqsave(&chan->lock, flags); in sf_pdma_errbh_tasklet()
329 if (chan->retries <= 0) { in sf_pdma_errbh_tasklet()
331 spin_unlock_irqrestore(&chan->lock, flags); in sf_pdma_errbh_tasklet()
332 dmaengine_desc_get_callback_invoke(desc->async_tx, NULL); in sf_pdma_errbh_tasklet()
335 chan->retries--; in sf_pdma_errbh_tasklet()
336 chan->xfer_err = true; in sf_pdma_errbh_tasklet()
337 chan->status = DMA_ERROR; in sf_pdma_errbh_tasklet()
340 spin_unlock_irqrestore(&chan->lock, flags); in sf_pdma_errbh_tasklet()
347 struct pdma_regs *regs = &chan->regs; in sf_pdma_done_isr()
350 spin_lock(&chan->vchan.lock); in sf_pdma_done_isr()
351 writel((readl(regs->ctrl)) & ~PDMA_DONE_STATUS_MASK, regs->ctrl); in sf_pdma_done_isr()
352 residue = readq(regs->residue); in sf_pdma_done_isr()
355 tasklet_hi_schedule(&chan->done_tasklet); in sf_pdma_done_isr()
358 struct sf_pdma_desc *desc = chan->desc; in sf_pdma_done_isr()
360 desc->src_addr += desc->xfer_size - residue; in sf_pdma_done_isr()
361 desc->dst_addr += desc->xfer_size - residue; in sf_pdma_done_isr()
362 desc->xfer_size = residue; in sf_pdma_done_isr()
367 spin_unlock(&chan->vchan.lock); in sf_pdma_done_isr()
375 struct pdma_regs *regs = &chan->regs; in sf_pdma_err_isr()
377 spin_lock(&chan->lock); in sf_pdma_err_isr()
378 writel((readl(regs->ctrl)) & ~PDMA_ERR_STATUS_MASK, regs->ctrl); in sf_pdma_err_isr()
379 spin_unlock(&chan->lock); in sf_pdma_err_isr()
381 tasklet_schedule(&chan->err_tasklet); in sf_pdma_err_isr()
387 * sf_pdma_irq_init() - Init PDMA IRQ Handlers
389 * @pdma: pointer of PDMA engine. Caller should check NULL
392 * make sure the pointer passed in are non-NULL. This function should be called
398 * * 0 - OK to init all IRQ handlers
399 * * -EINVAL - Fail to request IRQ
401 static int sf_pdma_irq_init(struct platform_device *pdev, struct sf_pdma *pdma) in sf_pdma_irq_init() argument
406 for (i = 0; i < pdma->n_chans; i++) { in sf_pdma_irq_init()
407 chan = &pdma->chans[i]; in sf_pdma_irq_init()
411 return -EINVAL; in sf_pdma_irq_init()
413 r = devm_request_irq(&pdev->dev, irq, sf_pdma_done_isr, 0, in sf_pdma_irq_init()
414 dev_name(&pdev->dev), (void *)chan); in sf_pdma_irq_init()
416 dev_err(&pdev->dev, "Fail to attach done ISR: %d\n", r); in sf_pdma_irq_init()
417 return -EINVAL; in sf_pdma_irq_init()
420 chan->txirq = irq; in sf_pdma_irq_init()
424 return -EINVAL; in sf_pdma_irq_init()
426 r = devm_request_irq(&pdev->dev, irq, sf_pdma_err_isr, 0, in sf_pdma_irq_init()
427 dev_name(&pdev->dev), (void *)chan); in sf_pdma_irq_init()
429 dev_err(&pdev->dev, "Fail to attach err ISR: %d\n", r); in sf_pdma_irq_init()
430 return -EINVAL; in sf_pdma_irq_init()
433 chan->errirq = irq; in sf_pdma_irq_init()
440 * sf_pdma_setup_chans() - Init settings of each channel
441 * @pdma: pointer of PDMA engine. Caller should check NULL
444 * the pointer passed in are non-NULL. This function should be called only
451 static void sf_pdma_setup_chans(struct sf_pdma *pdma) in sf_pdma_setup_chans() argument
456 INIT_LIST_HEAD(&pdma->dma_dev.channels); in sf_pdma_setup_chans()
458 for (i = 0; i < pdma->n_chans; i++) { in sf_pdma_setup_chans()
459 chan = &pdma->chans[i]; in sf_pdma_setup_chans()
461 chan->regs.ctrl = in sf_pdma_setup_chans()
463 chan->regs.xfer_type = in sf_pdma_setup_chans()
465 chan->regs.xfer_size = in sf_pdma_setup_chans()
467 chan->regs.dst_addr = in sf_pdma_setup_chans()
469 chan->regs.src_addr = in sf_pdma_setup_chans()
471 chan->regs.act_type = in sf_pdma_setup_chans()
473 chan->regs.residue = in sf_pdma_setup_chans()
475 chan->regs.cur_dst_addr = in sf_pdma_setup_chans()
477 chan->regs.cur_src_addr = in sf_pdma_setup_chans()
480 chan->pdma = pdma; in sf_pdma_setup_chans()
481 chan->pm_state = RUNNING; in sf_pdma_setup_chans()
482 chan->slave_id = i; in sf_pdma_setup_chans()
483 chan->xfer_err = false; in sf_pdma_setup_chans()
484 spin_lock_init(&chan->lock); in sf_pdma_setup_chans()
486 chan->vchan.desc_free = sf_pdma_free_desc; in sf_pdma_setup_chans()
487 vchan_init(&chan->vchan, &pdma->dma_dev); in sf_pdma_setup_chans()
489 writel(PDMA_CLEAR_CTRL, chan->regs.ctrl); in sf_pdma_setup_chans()
491 tasklet_setup(&chan->done_tasklet, sf_pdma_donebh_tasklet); in sf_pdma_setup_chans()
492 tasklet_setup(&chan->err_tasklet, sf_pdma_errbh_tasklet); in sf_pdma_setup_chans()
499 struct sf_pdma *pdma; in sf_pdma_probe() local
507 ret = of_property_read_u32(pdev->dev.of_node, "dma-channels", &n_chans); in sf_pdma_probe()
509 /* backwards-compatibility for no dma-channels property */ in sf_pdma_probe()
510 dev_dbg(&pdev->dev, "set number of channels to default value: 4\n"); in sf_pdma_probe()
513 dev_err(&pdev->dev, "the number of channels exceeds the maximum\n"); in sf_pdma_probe()
514 return -EINVAL; in sf_pdma_probe()
517 pdma = devm_kzalloc(&pdev->dev, struct_size(pdma, chans, n_chans), in sf_pdma_probe()
519 if (!pdma) in sf_pdma_probe()
520 return -ENOMEM; in sf_pdma_probe()
522 pdma->n_chans = n_chans; in sf_pdma_probe()
524 pdma->transfer_type = PDMA_FULL_SPEED | PDMA_STRICT_ORDERING; in sf_pdma_probe()
526 ddata = device_get_match_data(&pdev->dev); in sf_pdma_probe()
528 if (ddata->quirks & PDMA_QUIRK_NO_STRICT_ORDERING) in sf_pdma_probe()
529 pdma->transfer_type &= ~PDMA_STRICT_ORDERING; in sf_pdma_probe()
532 pdma->membase = devm_platform_ioremap_resource(pdev, 0); in sf_pdma_probe()
533 if (IS_ERR(pdma->membase)) in sf_pdma_probe()
534 return PTR_ERR(pdma->membase); in sf_pdma_probe()
536 ret = sf_pdma_irq_init(pdev, pdma); in sf_pdma_probe()
540 sf_pdma_setup_chans(pdma); in sf_pdma_probe()
542 pdma->dma_dev.dev = &pdev->dev; in sf_pdma_probe()
545 dma_cap_set(DMA_MEMCPY, pdma->dma_dev.cap_mask); in sf_pdma_probe()
546 pdma->dma_dev.copy_align = 2; in sf_pdma_probe()
547 pdma->dma_dev.src_addr_widths = widths; in sf_pdma_probe()
548 pdma->dma_dev.dst_addr_widths = widths; in sf_pdma_probe()
549 pdma->dma_dev.directions = BIT(DMA_MEM_TO_MEM); in sf_pdma_probe()
550 pdma->dma_dev.residue_granularity = DMA_RESIDUE_GRANULARITY_DESCRIPTOR; in sf_pdma_probe()
551 pdma->dma_dev.descriptor_reuse = true; in sf_pdma_probe()
554 pdma->dma_dev.device_alloc_chan_resources = in sf_pdma_probe()
556 pdma->dma_dev.device_free_chan_resources = in sf_pdma_probe()
558 pdma->dma_dev.device_tx_status = sf_pdma_tx_status; in sf_pdma_probe()
559 pdma->dma_dev.device_prep_dma_memcpy = sf_pdma_prep_dma_memcpy; in sf_pdma_probe()
560 pdma->dma_dev.device_config = sf_pdma_slave_config; in sf_pdma_probe()
561 pdma->dma_dev.device_terminate_all = sf_pdma_terminate_all; in sf_pdma_probe()
562 pdma->dma_dev.device_issue_pending = sf_pdma_issue_pending; in sf_pdma_probe()
564 platform_set_drvdata(pdev, pdma); in sf_pdma_probe()
566 ret = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64)); in sf_pdma_probe()
568 dev_warn(&pdev->dev, in sf_pdma_probe()
571 ret = dma_async_device_register(&pdma->dma_dev); in sf_pdma_probe()
573 dev_err(&pdev->dev, in sf_pdma_probe()
578 ret = of_dma_controller_register(pdev->dev.of_node, in sf_pdma_probe()
579 of_dma_xlate_by_chan_id, pdma); in sf_pdma_probe()
581 dev_err(&pdev->dev, in sf_pdma_probe()
589 dma_async_device_unregister(&pdma->dma_dev); in sf_pdma_probe()
596 struct sf_pdma *pdma = platform_get_drvdata(pdev); in sf_pdma_remove() local
600 for (i = 0; i < pdma->n_chans; i++) { in sf_pdma_remove()
601 ch = &pdma->chans[i]; in sf_pdma_remove()
603 devm_free_irq(&pdev->dev, ch->txirq, ch); in sf_pdma_remove()
604 devm_free_irq(&pdev->dev, ch->errirq, ch); in sf_pdma_remove()
605 list_del(&ch->vchan.chan.device_node); in sf_pdma_remove()
606 tasklet_kill(&ch->vchan.task); in sf_pdma_remove()
607 tasklet_kill(&ch->done_tasklet); in sf_pdma_remove()
608 tasklet_kill(&ch->err_tasklet); in sf_pdma_remove()
611 if (pdev->dev.of_node) in sf_pdma_remove()
612 of_dma_controller_free(pdev->dev.of_node); in sf_pdma_remove()
614 dma_async_device_unregister(&pdma->dma_dev); in sf_pdma_remove()
623 .compatible = "sifive,fu540-c000-pdma",
627 .compatible = "microchip,mpfs-pdma",
638 .name = "sf-pdma",