Lines Matching +full:clk +full:- +full:csr
1 // SPDX-License-Identifier: GPL-2.0-only
10 #include <linux/clk.h>
11 #include <linux/dma-mapping.h>
24 #include "../virt-dma.h"
56 /* MISR DMA non-secure/secure masked interrupt status register */
140 CTR1_PAM_0S_LT, /* if DDW > SDW, padded with 0s else left-truncated */
141 CTR1_PAM_SE_RT, /* if DDW > SDW, sign extended else right-truncated */
163 /* CxLLR DMA channel x linked-list address register */
192 AXI64, /* 1x AXI: 64-bit port 0 */
193 AHB32, /* 1x AHB: 32-bit port 0 */
194 AHB32_AHB32, /* 2x AHB: 32-bit port 0 and 32-bit port 1 */
195 AXI64_AHB32, /* 1x AXI 64-bit port 0 and 1x AHB 32-bit port 1 */
196 AXI64_AXI64, /* 2x AXI: 64-bit port 0 and 64-bit port 1 */
197 AXI128_AHB32, /* 1x AXI 128-bit port 0 and 1x AHB 32-bit port 1 */
201 DW_32, /* 32-bit, for AHB */
202 DW_64, /* 64-bit, for AXI */
203 DW_128, /* 128-bit, for AXI */
239 /* Static linked-list data structure (depends on update bits UT1/UT2/UB1/USA/UDA/ULL) */
251 * by the pointer to the next linked-list data structure. The __aligned forces the 32-byte
258 * Linked-list items
299 struct clk *clk; member
308 return container_of(chan->vchan.chan.device, struct stm32_dma3_ddata, dma_dev); in to_stm32_dma3_ddata()
323 return &chan->vchan.chan.dev->device; in chan2dev()
330 u32 id = chan->id, offset; in stm32_dma3_chan_dump_reg()
333 dev_dbg(dev, "SECCFGR(0x%03x): %08x\n", offset, readl_relaxed(ddata->base + offset)); in stm32_dma3_chan_dump_reg()
335 dev_dbg(dev, "PRIVCFGR(0x%03x): %08x\n", offset, readl_relaxed(ddata->base + offset)); in stm32_dma3_chan_dump_reg()
337 dev_dbg(dev, "C%dCIDCFGR(0x%03x): %08x\n", id, offset, readl_relaxed(ddata->base + offset)); in stm32_dma3_chan_dump_reg()
339 dev_dbg(dev, "C%dSEMCR(0x%03x): %08x\n", id, offset, readl_relaxed(ddata->base + offset)); in stm32_dma3_chan_dump_reg()
341 dev_dbg(dev, "C%dSR(0x%03x): %08x\n", id, offset, readl_relaxed(ddata->base + offset)); in stm32_dma3_chan_dump_reg()
343 dev_dbg(dev, "C%dCR(0x%03x): %08x\n", id, offset, readl_relaxed(ddata->base + offset)); in stm32_dma3_chan_dump_reg()
345 dev_dbg(dev, "C%dTR1(0x%03x): %08x\n", id, offset, readl_relaxed(ddata->base + offset)); in stm32_dma3_chan_dump_reg()
347 dev_dbg(dev, "C%dTR2(0x%03x): %08x\n", id, offset, readl_relaxed(ddata->base + offset)); in stm32_dma3_chan_dump_reg()
349 dev_dbg(dev, "C%dBR1(0x%03x): %08x\n", id, offset, readl_relaxed(ddata->base + offset)); in stm32_dma3_chan_dump_reg()
351 dev_dbg(dev, "C%dSAR(0x%03x): %08x\n", id, offset, readl_relaxed(ddata->base + offset)); in stm32_dma3_chan_dump_reg()
353 dev_dbg(dev, "C%dDAR(0x%03x): %08x\n", id, offset, readl_relaxed(ddata->base + offset)); in stm32_dma3_chan_dump_reg()
355 dev_dbg(dev, "C%dLLR(0x%03x): %08x\n", id, offset, readl_relaxed(ddata->base + offset)); in stm32_dma3_chan_dump_reg()
357 dev_dbg(dev, "C%dLBAR(0x%03x): %08x\n", id, offset, readl_relaxed(ddata->base + offset)); in stm32_dma3_chan_dump_reg()
366 for (i = 0; i < swdesc->lli_size; i++) { in stm32_dma3_chan_dump_hwdesc()
367 hwdesc = swdesc->lli[i].hwdesc; in stm32_dma3_chan_dump_hwdesc()
370 dev_dbg(chan2dev(chan), "[%d]@%pad\n", i, &swdesc->lli[i].hwdesc_addr); in stm32_dma3_chan_dump_hwdesc()
371 dev_dbg(chan2dev(chan), "| C%dTR1: %08x\n", chan->id, hwdesc->ctr1); in stm32_dma3_chan_dump_hwdesc()
372 dev_dbg(chan2dev(chan), "| C%dTR2: %08x\n", chan->id, hwdesc->ctr2); in stm32_dma3_chan_dump_hwdesc()
373 dev_dbg(chan2dev(chan), "| C%dBR1: %08x\n", chan->id, hwdesc->cbr1); in stm32_dma3_chan_dump_hwdesc()
374 dev_dbg(chan2dev(chan), "| C%dSAR: %08x\n", chan->id, hwdesc->csar); in stm32_dma3_chan_dump_hwdesc()
375 dev_dbg(chan2dev(chan), "| C%dDAR: %08x\n", chan->id, hwdesc->cdar); in stm32_dma3_chan_dump_hwdesc()
376 dev_dbg(chan2dev(chan), "| C%dLLR: %08x\n", chan->id, hwdesc->cllr); in stm32_dma3_chan_dump_hwdesc()
379 if (swdesc->cyclic) { in stm32_dma3_chan_dump_hwdesc()
381 dev_dbg(chan2dev(chan), "-->[0]@%pad\n", &swdesc->lli[0].hwdesc_addr); in stm32_dma3_chan_dump_hwdesc()
394 * If the memory to be allocated for the number of hwdesc (6 u32 members but 32-bytes in stm32_dma3_chan_desc_alloc()
406 swdesc->lli_size = count; in stm32_dma3_chan_desc_alloc()
409 swdesc->lli[i].hwdesc = dma_pool_zalloc(chan->lli_pool, GFP_NOWAIT, in stm32_dma3_chan_desc_alloc()
410 &swdesc->lli[i].hwdesc_addr); in stm32_dma3_chan_desc_alloc()
411 if (!swdesc->lli[i].hwdesc) in stm32_dma3_chan_desc_alloc()
414 swdesc->ccr = 0; in stm32_dma3_chan_desc_alloc()
417 writel_relaxed(swdesc->lli[0].hwdesc_addr & CLBAR_LBA, in stm32_dma3_chan_desc_alloc()
418 ddata->base + STM32_DMA3_CLBAR(chan->id)); in stm32_dma3_chan_desc_alloc()
421 swdesc->ccr &= ~CCR_LAP; in stm32_dma3_chan_desc_alloc()
427 while (--i >= 0) in stm32_dma3_chan_desc_alloc()
428 dma_pool_free(chan->lli_pool, swdesc->lli[i].hwdesc, swdesc->lli[i].hwdesc_addr); in stm32_dma3_chan_desc_alloc()
439 for (i = 0; i < swdesc->lli_size; i++) in stm32_dma3_chan_desc_free()
440 dma_pool_free(chan->lli_pool, swdesc->lli[i].hwdesc, swdesc->lli[i].hwdesc_addr); in stm32_dma3_chan_desc_free()
448 struct stm32_dma3_chan *chan = to_stm32_dma3_chan(vdesc->tx.chan); in stm32_dma3_chan_vdesc_free()
457 u32 ctr1 = readl_relaxed(ddata->base + STM32_DMA3_CTR1(chan->id)); in stm32_dma3_check_user_setting()
458 u32 cbr1 = readl_relaxed(ddata->base + STM32_DMA3_CBR1(chan->id)); in stm32_dma3_check_user_setting()
459 u32 csar = readl_relaxed(ddata->base + STM32_DMA3_CSAR(chan->id)); in stm32_dma3_check_user_setting()
460 u32 cdar = readl_relaxed(ddata->base + STM32_DMA3_CDAR(chan->id)); in stm32_dma3_check_user_setting()
461 u32 cllr = readl_relaxed(ddata->base + STM32_DMA3_CLLR(chan->id)); in stm32_dma3_check_user_setting()
478 if (sdw == DMA_SLAVE_BUSWIDTH_8_BYTES && port_is_ahb(ddata->ports_max_dw[sap])) in stm32_dma3_check_user_setting()
479 dev_err(dev, "double-word source data width not supported on port %u\n", sap); in stm32_dma3_check_user_setting()
480 if (ddw == DMA_SLAVE_BUSWIDTH_8_BYTES && port_is_ahb(ddata->ports_max_dw[dap])) in stm32_dma3_check_user_setting()
481 dev_err(dev, "double-word destination data width not supported on port %u\n", dap); in stm32_dma3_check_user_setting()
493 hwdesc = swdesc->lli[curr].hwdesc; in stm32_dma3_chan_prep_hwdesc()
494 hwdesc->ctr1 = ctr1; in stm32_dma3_chan_prep_hwdesc()
495 hwdesc->ctr2 = ctr2; in stm32_dma3_chan_prep_hwdesc()
496 hwdesc->cbr1 = FIELD_PREP(CBR1_BNDT, len); in stm32_dma3_chan_prep_hwdesc()
497 hwdesc->csar = src; in stm32_dma3_chan_prep_hwdesc()
498 hwdesc->cdar = dst; in stm32_dma3_chan_prep_hwdesc()
502 next_lli = swdesc->lli[0].hwdesc_addr; in stm32_dma3_chan_prep_hwdesc()
506 next_lli = swdesc->lli[next].hwdesc_addr; in stm32_dma3_chan_prep_hwdesc()
509 hwdesc->cllr = 0; in stm32_dma3_chan_prep_hwdesc()
511 hwdesc->cllr |= CLLR_UT1 | CLLR_UT2 | CLLR_UB1; in stm32_dma3_chan_prep_hwdesc()
512 hwdesc->cllr |= CLLR_USA | CLLR_UDA | CLLR_ULL; in stm32_dma3_chan_prep_hwdesc()
513 hwdesc->cllr |= (next_lli & CLLR_LA); in stm32_dma3_chan_prep_hwdesc()
556 struct dma_device dma_device = ddata->dma_dev; in stm32_dma3_chan_prep_hw()
559 u32 ch_conf = chan->dt_config.ch_conf; in stm32_dma3_chan_prep_hw()
560 u32 tr_conf = chan->dt_config.tr_conf; in stm32_dma3_chan_prep_hw()
567 sdw = chan->dma_config.src_addr_width ? : get_chan_max_dw(sap, chan->max_burst); in stm32_dma3_chan_prep_hw()
568 ddw = chan->dma_config.dst_addr_width ? : get_chan_max_dw(dap, chan->max_burst); in stm32_dma3_chan_prep_hw()
569 sbl_max = chan->dma_config.src_maxburst ? : 1; in stm32_dma3_chan_prep_hw()
570 dbl_max = chan->dma_config.dst_maxburst ? : 1; in stm32_dma3_chan_prep_hw()
575 return -EINVAL; in stm32_dma3_chan_prep_hw()
578 if (ddata->ports_max_dw[1] == DW_INVALID && (sap || dap)) { in stm32_dma3_chan_prep_hw()
580 return -EINVAL; in stm32_dma3_chan_prep_hw()
583 sap_max_dw = ddata->ports_max_dw[sap]; in stm32_dma3_chan_prep_hw()
584 dap_max_dw = ddata->ports_max_dw[dap]; in stm32_dma3_chan_prep_hw()
590 return -EINVAL; in stm32_dma3_chan_prep_hw()
602 _ctr2 |= FIELD_PREP(CTR2_REQSEL, chan->dt_config.req_line) & ~CTR2_SWREQ; in stm32_dma3_chan_prep_hw()
611 chan->tcem = tcem; in stm32_dma3_chan_prep_hw()
613 chan->dma_config.direction = dir; in stm32_dma3_chan_prep_hw()
618 ddw = min_t(u32, ddw, stm32_dma3_get_max_dw(chan->max_burst, dap_max_dw, in stm32_dma3_chan_prep_hw()
620 dbl_max = min_t(u32, dbl_max, stm32_dma3_get_max_burst(len, ddw, chan->max_burst)); in stm32_dma3_chan_prep_hw()
623 sdw = stm32_dma3_get_max_dw(chan->max_burst, sap_max_dw, len, src_addr); in stm32_dma3_chan_prep_hw()
624 sbl_max = stm32_dma3_get_max_burst(len, sdw, chan->max_burst); in stm32_dma3_chan_prep_hw()
627 _ctr1 |= FIELD_PREP(CTR1_SBL_1, sbl_max - 1); in stm32_dma3_chan_prep_hw()
629 _ctr1 |= FIELD_PREP(CTR1_DBL_1, dbl_max - 1); in stm32_dma3_chan_prep_hw()
634 if (len & (ddw - 1)) { in stm32_dma3_chan_prep_hw()
637 return -EINVAL; in stm32_dma3_chan_prep_hw()
648 sdw = min_t(u32, sdw, stm32_dma3_get_max_dw(chan->max_burst, sap_max_dw, in stm32_dma3_chan_prep_hw()
650 sbl_max = min_t(u32, sbl_max, stm32_dma3_get_max_burst(len, sdw, chan->max_burst)); in stm32_dma3_chan_prep_hw()
653 ddw = stm32_dma3_get_max_dw(chan->max_burst, dap_max_dw, len, dst_addr); in stm32_dma3_chan_prep_hw()
654 dbl_max = stm32_dma3_get_max_burst(len, ddw, chan->max_burst); in stm32_dma3_chan_prep_hw()
657 _ctr1 |= FIELD_PREP(CTR1_SBL_1, sbl_max - 1); in stm32_dma3_chan_prep_hw()
659 _ctr1 |= FIELD_PREP(CTR1_DBL_1, dbl_max - 1); in stm32_dma3_chan_prep_hw()
664 if (len & (ddw - 1)) { in stm32_dma3_chan_prep_hw()
667 return -EINVAL; in stm32_dma3_chan_prep_hw()
680 sdw = stm32_dma3_get_max_dw(chan->max_burst, sap_max_dw, len, src_addr); in stm32_dma3_chan_prep_hw()
681 sbl_max = stm32_dma3_get_max_burst(len, sdw, chan->max_burst); in stm32_dma3_chan_prep_hw()
682 if (chan->config_set & STM32_DMA3_CFG_SET_DMA) { in stm32_dma3_chan_prep_hw()
685 stm32_dma3_get_max_burst(len, sdw, chan->max_burst)); in stm32_dma3_chan_prep_hw()
691 ddw = stm32_dma3_get_max_dw(chan->max_burst, dap_max_dw, len, dst_addr); in stm32_dma3_chan_prep_hw()
692 dbl_max = stm32_dma3_get_max_burst(len, ddw, chan->max_burst); in stm32_dma3_chan_prep_hw()
693 if (chan->config_set & STM32_DMA3_CFG_SET_DMA) { in stm32_dma3_chan_prep_hw()
696 stm32_dma3_get_max_burst(len, ddw, chan->max_burst)); in stm32_dma3_chan_prep_hw()
700 _ctr1 |= FIELD_PREP(CTR1_SBL_1, sbl_max - 1); in stm32_dma3_chan_prep_hw()
702 _ctr1 |= FIELD_PREP(CTR1_DBL_1, dbl_max - 1); in stm32_dma3_chan_prep_hw()
707 if (len & (ddw - 1)) { in stm32_dma3_chan_prep_hw()
710 return -EINVAL; in stm32_dma3_chan_prep_hw()
722 return -EINVAL; in stm32_dma3_chan_prep_hw()
740 u32 id = chan->id; in stm32_dma3_chan_start()
741 u32 csr, ccr; in stm32_dma3_chan_start() local
743 vdesc = vchan_next_desc(&chan->vchan); in stm32_dma3_chan_start()
745 chan->swdesc = NULL; in stm32_dma3_chan_start()
748 list_del(&vdesc->node); in stm32_dma3_chan_start()
750 chan->swdesc = to_stm32_dma3_swdesc(vdesc); in stm32_dma3_chan_start()
751 hwdesc = chan->swdesc->lli[0].hwdesc; in stm32_dma3_chan_start()
753 stm32_dma3_chan_dump_hwdesc(chan, chan->swdesc); in stm32_dma3_chan_start()
755 writel_relaxed(chan->swdesc->ccr, ddata->base + STM32_DMA3_CCR(id)); in stm32_dma3_chan_start()
756 writel_relaxed(hwdesc->ctr1, ddata->base + STM32_DMA3_CTR1(id)); in stm32_dma3_chan_start()
757 writel_relaxed(hwdesc->ctr2, ddata->base + STM32_DMA3_CTR2(id)); in stm32_dma3_chan_start()
758 writel_relaxed(hwdesc->cbr1, ddata->base + STM32_DMA3_CBR1(id)); in stm32_dma3_chan_start()
759 writel_relaxed(hwdesc->csar, ddata->base + STM32_DMA3_CSAR(id)); in stm32_dma3_chan_start()
760 writel_relaxed(hwdesc->cdar, ddata->base + STM32_DMA3_CDAR(id)); in stm32_dma3_chan_start()
761 writel_relaxed(hwdesc->cllr, ddata->base + STM32_DMA3_CLLR(id)); in stm32_dma3_chan_start()
764 csr = readl_relaxed(ddata->base + STM32_DMA3_CSR(id)); in stm32_dma3_chan_start()
765 if (csr & CSR_ALL_F) in stm32_dma3_chan_start()
766 writel_relaxed(csr, ddata->base + STM32_DMA3_CFCR(id)); in stm32_dma3_chan_start()
770 ccr = readl_relaxed(ddata->base + STM32_DMA3_CCR(id)); in stm32_dma3_chan_start()
771 writel_relaxed(ccr | CCR_EN, ddata->base + STM32_DMA3_CCR(id)); in stm32_dma3_chan_start()
773 chan->dma_status = DMA_IN_PROGRESS; in stm32_dma3_chan_start()
775 dev_dbg(chan2dev(chan), "vchan %pK: started\n", &chan->vchan); in stm32_dma3_chan_start()
781 u32 csr, ccr = readl_relaxed(ddata->base + STM32_DMA3_CCR(chan->id)) & ~CCR_EN; in stm32_dma3_chan_suspend() local
789 writel_relaxed(ccr, ddata->base + STM32_DMA3_CCR(chan->id)); in stm32_dma3_chan_suspend()
792 ret = readl_relaxed_poll_timeout_atomic(ddata->base + STM32_DMA3_CSR(chan->id), csr, in stm32_dma3_chan_suspend()
793 csr & CSR_SUSPF, 1, 10); in stm32_dma3_chan_suspend()
795 writel_relaxed(CFCR_SUSPF, ddata->base + STM32_DMA3_CFCR(chan->id)); in stm32_dma3_chan_suspend()
806 u32 ccr = readl_relaxed(ddata->base + STM32_DMA3_CCR(chan->id)) & ~CCR_EN; in stm32_dma3_chan_reset()
808 writel_relaxed(ccr |= CCR_RESET, ddata->base + STM32_DMA3_CCR(chan->id)); in stm32_dma3_chan_reset()
817 return swdesc->lli_size - 1; in stm32_dma3_chan_get_curr_hwdesc()
820 if (swdesc->cyclic && next_lli_offset == (swdesc->lli[0].hwdesc_addr & CLLR_LA)) in stm32_dma3_chan_get_curr_hwdesc()
821 return swdesc->lli_size - 1; in stm32_dma3_chan_get_curr_hwdesc()
824 for (i = swdesc->lli_size - 1; i > 0; i--) { in stm32_dma3_chan_get_curr_hwdesc()
825 *residue += FIELD_GET(CBR1_BNDT, swdesc->lli[i].hwdesc->cbr1); in stm32_dma3_chan_get_curr_hwdesc()
826 lli_offset = swdesc->lli[i].hwdesc_addr & CLLR_LA; in stm32_dma3_chan_get_curr_hwdesc()
828 return i - 1; in stm32_dma3_chan_get_curr_hwdesc()
831 return -EINVAL; in stm32_dma3_chan_get_curr_hwdesc()
841 u32 residue, curr_lli, csr, cdar, cbr1, cllr, bndt, fifol; in stm32_dma3_chan_set_residue() local
845 csr = readl_relaxed(ddata->base + STM32_DMA3_CSR(chan->id)); in stm32_dma3_chan_set_residue()
846 if (!(csr & CSR_IDLEF) && chan->dma_status != DMA_PAUSED) { in stm32_dma3_chan_set_residue()
848 writel_relaxed(swdesc->ccr | CCR_SUSP, ddata->base + STM32_DMA3_CCR(chan->id)); in stm32_dma3_chan_set_residue()
849 ret = readl_relaxed_poll_timeout_atomic(ddata->base + STM32_DMA3_CSR(chan->id), csr, in stm32_dma3_chan_set_residue()
850 csr & (CSR_SUSPF | CSR_IDLEF), 1, 10); in stm32_dma3_chan_set_residue()
852 if (ret || ((csr & CSR_TCF) && (csr & CSR_IDLEF))) { in stm32_dma3_chan_set_residue()
853 writel_relaxed(CFCR_SUSPF, ddata->base + STM32_DMA3_CFCR(chan->id)); in stm32_dma3_chan_set_residue()
854 writel_relaxed(swdesc->ccr, ddata->base + STM32_DMA3_CCR(chan->id)); in stm32_dma3_chan_set_residue()
856 dev_err(dev, "Channel suspension timeout, csr=%08x\n", csr); in stm32_dma3_chan_set_residue()
861 if (!(csr & CSR_IDLEF)) in stm32_dma3_chan_set_residue()
862 dev_warn(dev, "Can't get residue: channel still active, csr=%08x\n", csr); in stm32_dma3_chan_set_residue()
866 * linked-list is over, no residue in stm32_dma3_chan_set_residue()
868 if (!(csr & CSR_SUSPF) && (csr & CSR_TCF) && (csr & CSR_IDLEF)) in stm32_dma3_chan_set_residue()
872 cllr = readl_relaxed(ddata->base + STM32_DMA3_CLLR(chan->id)); in stm32_dma3_chan_set_residue()
873 cbr1 = readl_relaxed(ddata->base + STM32_DMA3_CBR1(chan->id)); in stm32_dma3_chan_set_residue()
874 cdar = readl_relaxed(ddata->base + STM32_DMA3_CDAR(chan->id)); in stm32_dma3_chan_set_residue()
877 if (csr & CSR_SUSPF) { in stm32_dma3_chan_set_residue()
878 writel_relaxed(CFCR_SUSPF, ddata->base + STM32_DMA3_CFCR(chan->id)); in stm32_dma3_chan_set_residue()
879 writel_relaxed(swdesc->ccr, ddata->base + STM32_DMA3_CCR(chan->id)); in stm32_dma3_chan_set_residue()
894 /* Read current FIFO level - in units of programmed destination data width */ in stm32_dma3_chan_set_residue()
895 hwdesc = swdesc->lli[curr_lli].hwdesc; in stm32_dma3_chan_set_residue()
896 fifol = FIELD_GET(CSR_FIFOL, csr) * (1 << FIELD_GET(CTR1_DDW_LOG2, hwdesc->ctr1)); in stm32_dma3_chan_set_residue()
898 if (fifol == (1 << (chan->fifo_size + 1))) in stm32_dma3_chan_set_residue()
912 * fifol_in_bytes = bytes_read - bytes_written. in stm32_dma3_chan_set_residue()
914 pack_unpack = !!(FIELD_GET(CTR1_PAM, hwdesc->ctr1) == CTR1_PAM_PACK_UNPACK); in stm32_dma3_chan_set_residue()
915 if (pack_unpack && (hwdesc->ctr1 & CTR1_DINC)) { in stm32_dma3_chan_set_residue()
916 int bytes_read = FIELD_GET(CBR1_BNDT, hwdesc->cbr1) - bndt; in stm32_dma3_chan_set_residue()
917 int bytes_written = cdar - hwdesc->cdar; in stm32_dma3_chan_set_residue()
920 fifol = bytes_read - bytes_written; in stm32_dma3_chan_set_residue()
933 if (chan->dma_config.direction == DMA_DEV_TO_MEM) in stm32_dma3_chan_set_residue()
945 chan->dma_status = DMA_COMPLETE; in stm32_dma3_chan_stop()
948 ccr = readl_relaxed(ddata->base + STM32_DMA3_CCR(chan->id)); in stm32_dma3_chan_stop()
949 writel_relaxed(ccr & ~(CCR_ALLIE | CCR_EN), ddata->base + STM32_DMA3_CCR(chan->id)); in stm32_dma3_chan_stop()
969 if (!chan->swdesc) in stm32_dma3_chan_complete()
972 vchan_cookie_complete(&chan->swdesc->vdesc); in stm32_dma3_chan_complete()
973 chan->swdesc = NULL; in stm32_dma3_chan_complete()
981 u32 misr, csr, ccr; in stm32_dma3_chan_irq() local
983 spin_lock(&chan->vchan.lock); in stm32_dma3_chan_irq()
985 misr = readl_relaxed(ddata->base + STM32_DMA3_MISR); in stm32_dma3_chan_irq()
986 if (!(misr & MISR_MIS(chan->id))) { in stm32_dma3_chan_irq()
987 spin_unlock(&chan->vchan.lock); in stm32_dma3_chan_irq()
991 csr = readl_relaxed(ddata->base + STM32_DMA3_CSR(chan->id)); in stm32_dma3_chan_irq()
992 ccr = readl_relaxed(ddata->base + STM32_DMA3_CCR(chan->id)) & CCR_ALLIE; in stm32_dma3_chan_irq()
994 if (csr & CSR_TCF && ccr & CCR_TCIE) { in stm32_dma3_chan_irq()
995 if (chan->swdesc->cyclic) in stm32_dma3_chan_irq()
996 vchan_cyclic_callback(&chan->swdesc->vdesc); in stm32_dma3_chan_irq()
1001 if (csr & CSR_USEF && ccr & CCR_USEIE) { in stm32_dma3_chan_irq()
1003 chan->dma_status = DMA_ERROR; in stm32_dma3_chan_irq()
1009 if (csr & CSR_ULEF && ccr & CCR_ULEIE) { in stm32_dma3_chan_irq()
1011 chan->dma_status = DMA_ERROR; in stm32_dma3_chan_irq()
1016 if (csr & CSR_DTEF && ccr & CCR_DTEIE) { in stm32_dma3_chan_irq()
1018 chan->dma_status = DMA_ERROR; in stm32_dma3_chan_irq()
1027 csr &= (ccr | CCR_HTIE); in stm32_dma3_chan_irq()
1029 if (csr) in stm32_dma3_chan_irq()
1030 writel_relaxed(csr, ddata->base + STM32_DMA3_CFCR(chan->id)); in stm32_dma3_chan_irq()
1032 spin_unlock(&chan->vchan.lock); in stm32_dma3_chan_irq()
1041 u32 id = chan->id, csemcr, ccid; in stm32_dma3_alloc_chan_resources()
1044 ret = pm_runtime_resume_and_get(ddata->dma_dev.dev); in stm32_dma3_alloc_chan_resources()
1049 if (chan->semaphore_mode && in stm32_dma3_alloc_chan_resources()
1050 readl_relaxed(ddata->base + STM32_DMA3_CSEMCR(chan->id)) & CSEMCR_SEM_MUTEX) { in stm32_dma3_alloc_chan_resources()
1051 ret = -EBUSY; in stm32_dma3_alloc_chan_resources()
1055 chan->lli_pool = dmam_pool_create(dev_name(&c->dev->device), c->device->dev, in stm32_dma3_alloc_chan_resources()
1058 if (!chan->lli_pool) { in stm32_dma3_alloc_chan_resources()
1060 ret = -ENOMEM; in stm32_dma3_alloc_chan_resources()
1065 if (chan->semaphore_mode) { in stm32_dma3_alloc_chan_resources()
1066 writel_relaxed(CSEMCR_SEM_MUTEX, ddata->base + STM32_DMA3_CSEMCR(id)); in stm32_dma3_alloc_chan_resources()
1067 csemcr = readl_relaxed(ddata->base + STM32_DMA3_CSEMCR(id)); in stm32_dma3_alloc_chan_resources()
1071 dev_err(chan2dev(chan), "Not under CID1 control (in-use by CID%d)\n", ccid); in stm32_dma3_alloc_chan_resources()
1072 ret = -EPERM; in stm32_dma3_alloc_chan_resources()
1081 dmam_pool_destroy(chan->lli_pool); in stm32_dma3_alloc_chan_resources()
1082 chan->lli_pool = NULL; in stm32_dma3_alloc_chan_resources()
1085 pm_runtime_put_sync(ddata->dma_dev.dev); in stm32_dma3_alloc_chan_resources()
1097 spin_lock_irqsave(&chan->vchan.lock, flags); in stm32_dma3_free_chan_resources()
1099 chan->swdesc = NULL; in stm32_dma3_free_chan_resources()
1100 spin_unlock_irqrestore(&chan->vchan.lock, flags); in stm32_dma3_free_chan_resources()
1104 dmam_pool_destroy(chan->lli_pool); in stm32_dma3_free_chan_resources()
1105 chan->lli_pool = NULL; in stm32_dma3_free_chan_resources()
1108 if (chan->semaphore_mode) in stm32_dma3_free_chan_resources()
1109 writel_relaxed(0, ddata->base + STM32_DMA3_CSEMCR(chan->id)); in stm32_dma3_free_chan_resources()
1111 pm_runtime_put_sync(ddata->dma_dev.dev); in stm32_dma3_free_chan_resources()
1114 memset(&chan->dt_config, 0, sizeof(chan->dt_config)); in stm32_dma3_free_chan_resources()
1115 memset(&chan->dma_config, 0, sizeof(chan->dma_config)); in stm32_dma3_free_chan_resources()
1116 chan->config_set = 0; in stm32_dma3_free_chan_resources()
1123 u32 dw = get_chan_max_dw(ddata->ports_max_dw[0], chan->max_burst); /* port 0 by default */ in stm32_dma3_init_chan_config_for_memcpy()
1124 u32 burst = chan->max_burst / dw; in stm32_dma3_init_chan_config_for_memcpy()
1126 /* Initialize dt_config if channel not pre-configured through DT */ in stm32_dma3_init_chan_config_for_memcpy()
1127 if (!(chan->config_set & STM32_DMA3_CFG_SET_DT)) { in stm32_dma3_init_chan_config_for_memcpy()
1128 chan->dt_config.ch_conf = FIELD_PREP(STM32_DMA3_DT_PRIO, CCR_PRIO_VERY_HIGH); in stm32_dma3_init_chan_config_for_memcpy()
1129 chan->dt_config.ch_conf |= FIELD_PREP(STM32_DMA3_DT_FIFO, chan->fifo_size); in stm32_dma3_init_chan_config_for_memcpy()
1130 chan->dt_config.tr_conf = STM32_DMA3_DT_SINC | STM32_DMA3_DT_DINC; in stm32_dma3_init_chan_config_for_memcpy()
1131 chan->dt_config.tr_conf |= FIELD_PREP(STM32_DMA3_DT_TCEM, CTR2_TCEM_CHANNEL); in stm32_dma3_init_chan_config_for_memcpy()
1135 if (!(chan->config_set & STM32_DMA3_CFG_SET_DMA)) { in stm32_dma3_init_chan_config_for_memcpy()
1136 chan->dma_config.src_addr_width = dw; in stm32_dma3_init_chan_config_for_memcpy()
1137 chan->dma_config.dst_addr_width = dw; in stm32_dma3_init_chan_config_for_memcpy()
1138 chan->dma_config.src_maxburst = burst; in stm32_dma3_init_chan_config_for_memcpy()
1139 chan->dma_config.dst_maxburst = burst; in stm32_dma3_init_chan_config_for_memcpy()
1140 chan->dma_config.src_addr = src; in stm32_dma3_init_chan_config_for_memcpy()
1141 chan->dma_config.dst_addr = dst; in stm32_dma3_init_chan_config_for_memcpy()
1160 if (chan->config_set != STM32_DMA3_CFG_SET_BOTH) in stm32_dma3_prep_dma_memcpy()
1167 remaining = len - offset; in stm32_dma3_prep_dma_memcpy()
1170 ret = stm32_dma3_chan_prep_hw(chan, DMA_MEM_TO_MEM, &swdesc->ccr, &ctr1, &ctr2, in stm32_dma3_prep_dma_memcpy()
1180 swdesc->ccr |= CCR_USEIE | CCR_ULEIE | CCR_DTEIE; in stm32_dma3_prep_dma_memcpy()
1182 swdesc->ccr |= CCR_TCIE; in stm32_dma3_prep_dma_memcpy()
1184 swdesc->cyclic = false; in stm32_dma3_prep_dma_memcpy()
1186 return vchan_tx_prep(&chan->vchan, &swdesc->vdesc, flags); in stm32_dma3_prep_dma_memcpy()
1212 count += DIV_ROUND_UP(len, STM32_DMA3_MAX_BLOCK_SIZE) - 1; in stm32_dma3_prep_slave_sg()
1223 dev_addr = (dir == DMA_MEM_TO_DEV) ? chan->dma_config.dst_addr : in stm32_dma3_prep_slave_sg()
1224 chan->dma_config.src_addr; in stm32_dma3_prep_slave_sg()
1234 ret = stm32_dma3_chan_prep_hw(chan, dir, &swdesc->ccr, &ctr1, &ctr2, in stm32_dma3_prep_slave_sg()
1243 ret = stm32_dma3_chan_prep_hw(chan, dir, &swdesc->ccr, &ctr1, &ctr2, in stm32_dma3_prep_slave_sg()
1254 ctr1, ctr2, j == (count - 1), false); in stm32_dma3_prep_slave_sg()
1257 len -= chunk; in stm32_dma3_prep_slave_sg()
1263 swdesc->ccr |= CCR_USEIE | CCR_ULEIE | CCR_DTEIE; in stm32_dma3_prep_slave_sg()
1265 swdesc->ccr |= CCR_TCIE; in stm32_dma3_prep_slave_sg()
1267 swdesc->cyclic = false; in stm32_dma3_prep_slave_sg()
1269 return vchan_tx_prep(&chan->vchan, &swdesc->vdesc, flags); in stm32_dma3_prep_slave_sg()
1306 dst = chan->dma_config.dst_addr; in stm32_dma3_prep_dma_cyclic()
1308 ret = stm32_dma3_chan_prep_hw(chan, DMA_MEM_TO_DEV, &swdesc->ccr, &ctr1, &ctr2, in stm32_dma3_prep_dma_cyclic()
1311 src = chan->dma_config.src_addr; in stm32_dma3_prep_dma_cyclic()
1314 ret = stm32_dma3_chan_prep_hw(chan, DMA_DEV_TO_MEM, &swdesc->ccr, &ctr1, &ctr2, in stm32_dma3_prep_dma_cyclic()
1318 ret = -EINVAL; in stm32_dma3_prep_dma_cyclic()
1327 dst = chan->dma_config.dst_addr; in stm32_dma3_prep_dma_cyclic()
1329 src = chan->dma_config.src_addr; in stm32_dma3_prep_dma_cyclic()
1334 ctr1, ctr2, i == (count - 1), true); in stm32_dma3_prep_dma_cyclic()
1338 swdesc->ccr |= CCR_USEIE | CCR_ULEIE | CCR_DTEIE; in stm32_dma3_prep_dma_cyclic()
1340 swdesc->ccr |= CCR_TCIE; in stm32_dma3_prep_dma_cyclic()
1342 swdesc->cyclic = true; in stm32_dma3_prep_dma_cyclic()
1344 return vchan_tx_prep(&chan->vchan, &swdesc->vdesc, flags); in stm32_dma3_prep_dma_cyclic()
1356 if (!chan->fifo_size) { in stm32_dma3_caps()
1357 caps->max_burst = 0; in stm32_dma3_caps()
1358 caps->src_addr_widths &= ~BIT(DMA_SLAVE_BUSWIDTH_8_BYTES); in stm32_dma3_caps()
1359 caps->dst_addr_widths &= ~BIT(DMA_SLAVE_BUSWIDTH_8_BYTES); in stm32_dma3_caps()
1362 caps->max_burst = chan->max_burst; in stm32_dma3_caps()
1363 if (caps->max_burst < DMA_SLAVE_BUSWIDTH_8_BYTES) { in stm32_dma3_caps()
1364 caps->src_addr_widths &= ~BIT(DMA_SLAVE_BUSWIDTH_8_BYTES); in stm32_dma3_caps()
1365 caps->dst_addr_widths &= ~BIT(DMA_SLAVE_BUSWIDTH_8_BYTES); in stm32_dma3_caps()
1374 memcpy(&chan->dma_config, config, sizeof(*config)); in stm32_dma3_config()
1375 chan->config_set |= STM32_DMA3_CFG_SET_DMA; in stm32_dma3_config()
1389 chan->dma_status = DMA_PAUSED; in stm32_dma3_pause()
1391 dev_dbg(chan2dev(chan), "vchan %pK: paused\n", &chan->vchan); in stm32_dma3_pause()
1402 chan->dma_status = DMA_IN_PROGRESS; in stm32_dma3_resume()
1404 dev_dbg(chan2dev(chan), "vchan %pK: resumed\n", &chan->vchan); in stm32_dma3_resume()
1415 spin_lock_irqsave(&chan->vchan.lock, flags); in stm32_dma3_terminate_all()
1417 if (chan->swdesc) { in stm32_dma3_terminate_all()
1418 vchan_terminate_vdesc(&chan->swdesc->vdesc); in stm32_dma3_terminate_all()
1419 chan->swdesc = NULL; in stm32_dma3_terminate_all()
1424 vchan_get_all_descriptors(&chan->vchan, &head); in stm32_dma3_terminate_all()
1426 spin_unlock_irqrestore(&chan->vchan.lock, flags); in stm32_dma3_terminate_all()
1427 vchan_dma_desc_free_list(&chan->vchan, &head); in stm32_dma3_terminate_all()
1429 dev_dbg(chan2dev(chan), "vchan %pK: terminated\n", &chan->vchan); in stm32_dma3_terminate_all()
1438 vchan_synchronize(&chan->vchan); in stm32_dma3_synchronize()
1455 return chan->dma_status; in stm32_dma3_tx_status()
1457 spin_lock_irqsave(&chan->vchan.lock, flags); in stm32_dma3_tx_status()
1459 vd = vchan_find_desc(&chan->vchan, cookie); in stm32_dma3_tx_status()
1462 else if (chan->swdesc && chan->swdesc->vdesc.tx.cookie == cookie) in stm32_dma3_tx_status()
1463 swdesc = chan->swdesc; in stm32_dma3_tx_status()
1469 spin_unlock_irqrestore(&chan->vchan.lock, flags); in stm32_dma3_tx_status()
1471 return chan->dma_status; in stm32_dma3_tx_status()
1479 spin_lock_irqsave(&chan->vchan.lock, flags); in stm32_dma3_issue_pending()
1481 if (vchan_issue_pending(&chan->vchan) && !chan->swdesc) { in stm32_dma3_issue_pending()
1482 dev_dbg(chan2dev(chan), "vchan %pK: issued\n", &chan->vchan); in stm32_dma3_issue_pending()
1486 spin_unlock_irqrestore(&chan->vchan.lock, flags); in stm32_dma3_issue_pending()
1497 dev_dbg(c->device->dev, "%s(%s): req_line=%d ch_conf=%08x tr_conf=%08x\n", in stm32_dma3_filter_fn()
1498 __func__, dma_chan_name(c), conf->req_line, conf->ch_conf, conf->tr_conf); in stm32_dma3_filter_fn()
1500 if (!of_property_read_u32(c->device->dev->of_node, "dma-channel-mask", &mask)) in stm32_dma3_filter_fn()
1501 if (!(mask & BIT(chan->id))) in stm32_dma3_filter_fn()
1504 ret = pm_runtime_resume_and_get(ddata->dma_dev.dev); in stm32_dma3_filter_fn()
1507 semcr = readl_relaxed(ddata->base + STM32_DMA3_CSEMCR(chan->id)); in stm32_dma3_filter_fn()
1508 pm_runtime_put_sync(ddata->dma_dev.dev); in stm32_dma3_filter_fn()
1515 if (FIELD_GET(STM32_DMA3_DT_FIFO, conf->ch_conf) != chan->fifo_size) in stm32_dma3_filter_fn()
1523 struct stm32_dma3_ddata *ddata = ofdma->of_dma_data; in stm32_dma3_of_xlate()
1524 dma_cap_mask_t mask = ddata->dma_dev.cap_mask; in stm32_dma3_of_xlate()
1529 if (dma_spec->args_count < 3) { in stm32_dma3_of_xlate()
1530 dev_err(ddata->dma_dev.dev, "Invalid args count\n"); in stm32_dma3_of_xlate()
1534 conf.req_line = dma_spec->args[0]; in stm32_dma3_of_xlate()
1535 conf.ch_conf = dma_spec->args[1]; in stm32_dma3_of_xlate()
1536 conf.tr_conf = dma_spec->args[2]; in stm32_dma3_of_xlate()
1538 if (conf.req_line >= ddata->dma_requests) { in stm32_dma3_of_xlate()
1539 dev_err(ddata->dma_dev.dev, "Invalid request line\n"); in stm32_dma3_of_xlate()
1546 dev_err(ddata->dma_dev.dev, "No suitable channel found\n"); in stm32_dma3_of_xlate()
1551 chan->dt_config = conf; in stm32_dma3_of_xlate()
1552 chan->config_set |= STM32_DMA3_CFG_SET_DT; in stm32_dma3_of_xlate()
1562 chan_reserved = readl_relaxed(ddata->base + STM32_DMA3_SECCFGR); in stm32_dma3_check_rif()
1567 * In case CID filtering is not configured, dma-channel-mask property can be used to in stm32_dma3_check_rif()
1570 of_property_read_u32(ddata->dma_dev.dev->of_node, "dma-channel-mask", &mask); in stm32_dma3_check_rif()
1572 /* Reserve !CID-filtered not in dma-channel-mask, static CID != CID1, CID1 not allowed */ in stm32_dma3_check_rif()
1573 for (i = 0; i < ddata->dma_channels; i++) { in stm32_dma3_check_rif()
1574 ccidcfgr = readl_relaxed(ddata->base + STM32_DMA3_CCIDCFGR(i)); in stm32_dma3_check_rif()
1576 if (!(ccidcfgr & CCIDCFGR_CFEN)) { /* !CID-filtered */ in stm32_dma3_check_rif()
1578 if (!(mask & BIT(i))) /* Not in dma-channel-mask */ in stm32_dma3_check_rif()
1580 } else { /* CID-filtered */ in stm32_dma3_check_rif()
1587 ddata->chans[i].semaphore_mode = true; in stm32_dma3_check_rif()
1590 dev_dbg(ddata->dma_dev.dev, "chan%d: %s mode, %s\n", i, in stm32_dma3_check_rif()
1591 !(ccidcfgr & CCIDCFGR_CFEN) ? "!CID-filtered" : in stm32_dma3_check_rif()
1592 ddata->chans[i].semaphore_mode ? "Semaphore" : "Static CID", in stm32_dma3_check_rif()
1598 dev_warn(ddata->dma_dev.dev, "chan%*pbl have invalid CID configuration\n", in stm32_dma3_check_rif()
1599 ddata->dma_channels, &invalid_cid); in stm32_dma3_check_rif()
1605 { .compatible = "st,stm32mp25-dma3", },
1612 struct device_node *np = pdev->dev.of_node; in stm32_dma3_probe()
1621 ddata = devm_kzalloc(&pdev->dev, sizeof(*ddata), GFP_KERNEL); in stm32_dma3_probe()
1623 return -ENOMEM; in stm32_dma3_probe()
1626 dma_dev = &ddata->dma_dev; in stm32_dma3_probe()
1628 ddata->base = devm_platform_ioremap_resource(pdev, 0); in stm32_dma3_probe()
1629 if (IS_ERR(ddata->base)) in stm32_dma3_probe()
1630 return PTR_ERR(ddata->base); in stm32_dma3_probe()
1632 ddata->clk = devm_clk_get(&pdev->dev, NULL); in stm32_dma3_probe()
1633 if (IS_ERR(ddata->clk)) in stm32_dma3_probe()
1634 return dev_err_probe(&pdev->dev, PTR_ERR(ddata->clk), "Failed to get clk\n"); in stm32_dma3_probe()
1636 reset = devm_reset_control_get_optional(&pdev->dev, NULL); in stm32_dma3_probe()
1638 return dev_err_probe(&pdev->dev, PTR_ERR(reset), "Failed to get reset\n"); in stm32_dma3_probe()
1640 ret = clk_prepare_enable(ddata->clk); in stm32_dma3_probe()
1642 return dev_err_probe(&pdev->dev, ret, "Failed to enable clk\n"); in stm32_dma3_probe()
1646 INIT_LIST_HEAD(&dma_dev->channels); in stm32_dma3_probe()
1648 dma_cap_set(DMA_SLAVE, dma_dev->cap_mask); in stm32_dma3_probe()
1649 dma_cap_set(DMA_PRIVATE, dma_dev->cap_mask); in stm32_dma3_probe()
1650 dma_cap_set(DMA_CYCLIC, dma_dev->cap_mask); in stm32_dma3_probe()
1651 dma_cap_set(DMA_MEMCPY, dma_dev->cap_mask); in stm32_dma3_probe()
1652 dma_dev->dev = &pdev->dev; in stm32_dma3_probe()
1654 * This controller supports up to 8-byte buswidth depending on the port used and the in stm32_dma3_probe()
1657 dma_dev->copy_align = DMAENGINE_ALIGN_8_BYTES; in stm32_dma3_probe()
1658 dma_dev->src_addr_widths = BIT(DMA_SLAVE_BUSWIDTH_1_BYTE) | in stm32_dma3_probe()
1662 dma_dev->dst_addr_widths = BIT(DMA_SLAVE_BUSWIDTH_1_BYTE) | in stm32_dma3_probe()
1666 dma_dev->directions = BIT(DMA_DEV_TO_MEM) | BIT(DMA_MEM_TO_DEV) | BIT(DMA_MEM_TO_MEM); in stm32_dma3_probe()
1668 dma_dev->descriptor_reuse = true; in stm32_dma3_probe()
1669 dma_dev->max_sg_burst = STM32_DMA3_MAX_SEG_SIZE; in stm32_dma3_probe()
1670 dma_dev->residue_granularity = DMA_RESIDUE_GRANULARITY_BURST; in stm32_dma3_probe()
1671 dma_dev->device_alloc_chan_resources = stm32_dma3_alloc_chan_resources; in stm32_dma3_probe()
1672 dma_dev->device_free_chan_resources = stm32_dma3_free_chan_resources; in stm32_dma3_probe()
1673 dma_dev->device_prep_dma_memcpy = stm32_dma3_prep_dma_memcpy; in stm32_dma3_probe()
1674 dma_dev->device_prep_slave_sg = stm32_dma3_prep_slave_sg; in stm32_dma3_probe()
1675 dma_dev->device_prep_dma_cyclic = stm32_dma3_prep_dma_cyclic; in stm32_dma3_probe()
1676 dma_dev->device_caps = stm32_dma3_caps; in stm32_dma3_probe()
1677 dma_dev->device_config = stm32_dma3_config; in stm32_dma3_probe()
1678 dma_dev->device_pause = stm32_dma3_pause; in stm32_dma3_probe()
1679 dma_dev->device_resume = stm32_dma3_resume; in stm32_dma3_probe()
1680 dma_dev->device_terminate_all = stm32_dma3_terminate_all; in stm32_dma3_probe()
1681 dma_dev->device_synchronize = stm32_dma3_synchronize; in stm32_dma3_probe()
1682 dma_dev->device_tx_status = stm32_dma3_tx_status; in stm32_dma3_probe()
1683 dma_dev->device_issue_pending = stm32_dma3_issue_pending; in stm32_dma3_probe()
1686 if (of_property_read_u32(np, "dma-channels", &ddata->dma_channels)) { in stm32_dma3_probe()
1687 hwcfgr = readl_relaxed(ddata->base + STM32_DMA3_HWCFGR1); in stm32_dma3_probe()
1688 ddata->dma_channels = FIELD_GET(G_NUM_CHANNELS, hwcfgr); in stm32_dma3_probe()
1692 if (of_property_read_u32(np, "dma-requests", &ddata->dma_requests)) { in stm32_dma3_probe()
1693 hwcfgr = readl_relaxed(ddata->base + STM32_DMA3_HWCFGR2); in stm32_dma3_probe()
1694 ddata->dma_requests = FIELD_GET(G_MAX_REQ_ID, hwcfgr) + 1; in stm32_dma3_probe()
1698 hwcfgr = readl_relaxed(ddata->base + STM32_DMA3_HWCFGR1); in stm32_dma3_probe()
1701 ddata->ports_max_dw[0] = FIELD_GET(G_M0_DATA_WIDTH_ENC, hwcfgr); in stm32_dma3_probe()
1703 ddata->ports_max_dw[1] = DW_INVALID; in stm32_dma3_probe()
1705 ddata->ports_max_dw[1] = FIELD_GET(G_M1_DATA_WIDTH_ENC, hwcfgr); in stm32_dma3_probe()
1707 ddata->chans = devm_kcalloc(&pdev->dev, ddata->dma_channels, sizeof(*ddata->chans), in stm32_dma3_probe()
1709 if (!ddata->chans) { in stm32_dma3_probe()
1710 ret = -ENOMEM; in stm32_dma3_probe()
1716 if (chan_reserved == GENMASK(ddata->dma_channels - 1, 0)) { in stm32_dma3_probe()
1717 ret = -ENODEV; in stm32_dma3_probe()
1718 dev_err_probe(&pdev->dev, ret, "No channel available, abort registration\n"); in stm32_dma3_probe()
1723 hwcfgr = readl_relaxed(ddata->base + STM32_DMA3_HWCFGR3); in stm32_dma3_probe()
1724 hwcfgr |= ((u64)readl_relaxed(ddata->base + STM32_DMA3_HWCFGR4)) << 32; in stm32_dma3_probe()
1726 for (i = 0; i < ddata->dma_channels; i++) { in stm32_dma3_probe()
1730 chan = &ddata->chans[i]; in stm32_dma3_probe()
1731 chan->id = i; in stm32_dma3_probe()
1732 chan->fifo_size = get_chan_hwcfg(i, G_FIFO_SIZE(i), hwcfgr); in stm32_dma3_probe()
1733 /* If chan->fifo_size > 0 then half of the fifo size, else no burst when no FIFO */ in stm32_dma3_probe()
1734 chan->max_burst = (chan->fifo_size) ? (1 << (chan->fifo_size + 1)) / 2 : 0; in stm32_dma3_probe()
1741 for (i = 0; i < ddata->dma_channels; i++) { in stm32_dma3_probe()
1747 chan = &ddata->chans[i]; in stm32_dma3_probe()
1748 snprintf(name, sizeof(name), "dma%dchan%d", ddata->dma_dev.dev_id, chan->id); in stm32_dma3_probe()
1750 chan->vchan.desc_free = stm32_dma3_chan_vdesc_free; in stm32_dma3_probe()
1751 vchan_init(&chan->vchan, dma_dev); in stm32_dma3_probe()
1753 ret = dma_async_device_channel_register(&ddata->dma_dev, &chan->vchan.chan, name); in stm32_dma3_probe()
1755 dev_err_probe(&pdev->dev, ret, "Failed to register channel %s\n", name); in stm32_dma3_probe()
1762 chan->irq = ret; in stm32_dma3_probe()
1764 ret = devm_request_irq(&pdev->dev, chan->irq, stm32_dma3_chan_irq, 0, in stm32_dma3_probe()
1767 dev_err_probe(&pdev->dev, ret, "Failed to request channel %s IRQ\n", in stm32_dma3_probe()
1775 dev_err_probe(&pdev->dev, ret, "Failed to register controller\n"); in stm32_dma3_probe()
1779 verr = readl_relaxed(ddata->base + STM32_DMA3_VERR); in stm32_dma3_probe()
1781 pm_runtime_set_active(&pdev->dev); in stm32_dma3_probe()
1782 pm_runtime_enable(&pdev->dev); in stm32_dma3_probe()
1783 pm_runtime_get_noresume(&pdev->dev); in stm32_dma3_probe()
1784 pm_runtime_put(&pdev->dev); in stm32_dma3_probe()
1786 dev_info(&pdev->dev, "STM32 DMA3 registered rev:%lu.%lu\n", in stm32_dma3_probe()
1792 clk_disable_unprepare(ddata->clk); in stm32_dma3_probe()
1799 pm_runtime_disable(&pdev->dev); in stm32_dma3_remove()
1806 clk_disable_unprepare(ddata->clk); in stm32_dma3_runtime_suspend()
1816 ret = clk_prepare_enable(ddata->clk); in stm32_dma3_runtime_resume()
1818 dev_err(dev, "Failed to enable clk: %d\n", ret); in stm32_dma3_runtime_resume()
1832 .name = "stm32-dma3",