Lines Matching +full:double +full:- +full:channel
1 // SPDX-License-Identifier: GPL-2.0-only
5 * Inspired by dma-jz4740.c and tegra20-apb-dma.c
9 * Pierre-Yves Mordret <pierre-yves.mordret@st.com>
16 #include <linux/dma-mapping.h>
31 #include "../virt-dma.h"
49 * If (chan->id % 4) is 2 or 3, left shift the mask by 16 bits;
65 #define STM32_DMA_SCR_CT BIT(19) /* Target in double buffer */
66 #define STM32_DMA_SCR_DBM BIT(18) /* Double Buffer Mode */
158 * struct stm32_dma_cfg - STM32 DMA custom configuration
159 * @channel_id: channel ID
161 * @stream_config: 32bit mask specifying the DMA channel configuration
197 * struct stm32_dma_mdma_config - STM32 DMA MDMA configuration
237 return container_of(chan->vchan.chan.device, struct stm32_dma_device, in stm32_dma_get_dev()
253 return &chan->vchan.chan.dev->device; in chan2dev()
258 return readl_relaxed(dmadev->base + reg); in stm32_dma_read()
263 writel_relaxed(val, dmadev->base + reg); in stm32_dma_write()
278 return -EINVAL; in stm32_dma_get_width()
297 if (buf_addr & (max_width - 1)) in stm32_dma_get_max_width()
378 return -EINVAL; in stm32_dma_get_burst()
385 chan->chan_reg.dma_sfcr &= ~STM32_DMA_SFCR_MASK; in stm32_dma_set_fifo_config()
386 chan->chan_reg.dma_scr &= ~STM32_DMA_SCR_DMEIE; in stm32_dma_set_fifo_config()
390 chan->chan_reg.dma_scr |= STM32_DMA_SCR_DMEIE; in stm32_dma_set_fifo_config()
393 chan->chan_reg.dma_sfcr |= STM32_DMA_SFCR_MASK; in stm32_dma_set_fifo_config()
402 memcpy(&chan->dma_sconfig, config, sizeof(*config)); in stm32_dma_slave_config()
405 if (config->peripheral_size) { in stm32_dma_slave_config()
406 config->peripheral_config = &chan->mdma_config; in stm32_dma_slave_config()
407 config->peripheral_size = sizeof(chan->mdma_config); in stm32_dma_slave_config()
408 chan->trig_mdma = true; in stm32_dma_slave_config()
411 chan->config_init = true; in stm32_dma_slave_config()
423 * DMA channel at the correct bit offset inside that register. in stm32_dma_irq_status()
426 dma_isr = stm32_dma_read(dmadev, STM32_DMA_ISR(chan->id)); in stm32_dma_irq_status()
427 flags = dma_isr >> STM32_DMA_FLAGS_SHIFT(chan->id); in stm32_dma_irq_status()
439 * DMA channel at the correct bit offset inside that register. in stm32_dma_irq_clear()
442 dma_ifcr = flags << STM32_DMA_FLAGS_SHIFT(chan->id); in stm32_dma_irq_clear()
444 stm32_dma_write(dmadev, STM32_DMA_IFCR(chan->id), dma_ifcr); in stm32_dma_irq_clear()
452 id = chan->id; in stm32_dma_disable_chan()
460 return readl_relaxed_poll_timeout_atomic(dmadev->base + reg, in stm32_dma_disable_chan()
475 dma_scr = stm32_dma_read(dmadev, STM32_DMA_SCR(chan->id)); in stm32_dma_stop()
477 stm32_dma_write(dmadev, STM32_DMA_SCR(chan->id), dma_scr); in stm32_dma_stop()
478 dma_sfcr = stm32_dma_read(dmadev, STM32_DMA_SFCR(chan->id)); in stm32_dma_stop()
480 stm32_dma_write(dmadev, STM32_DMA_SFCR(chan->id), dma_sfcr); in stm32_dma_stop()
495 chan->busy = false; in stm32_dma_stop()
496 chan->status = DMA_COMPLETE; in stm32_dma_stop()
505 spin_lock_irqsave(&chan->vchan.lock, flags); in stm32_dma_terminate_all()
507 if (chan->desc) { in stm32_dma_terminate_all()
508 dma_cookie_complete(&chan->desc->vdesc.tx); in stm32_dma_terminate_all()
509 vchan_terminate_vdesc(&chan->desc->vdesc); in stm32_dma_terminate_all()
510 if (chan->busy) in stm32_dma_terminate_all()
512 chan->desc = NULL; in stm32_dma_terminate_all()
515 vchan_get_all_descriptors(&chan->vchan, &head); in stm32_dma_terminate_all()
516 spin_unlock_irqrestore(&chan->vchan.lock, flags); in stm32_dma_terminate_all()
517 vchan_dma_desc_free_list(&chan->vchan, &head); in stm32_dma_terminate_all()
526 vchan_synchronize(&chan->vchan); in stm32_dma_synchronize()
532 u32 scr = stm32_dma_read(dmadev, STM32_DMA_SCR(chan->id)); in stm32_dma_dump_reg()
533 u32 ndtr = stm32_dma_read(dmadev, STM32_DMA_SNDTR(chan->id)); in stm32_dma_dump_reg()
534 u32 spar = stm32_dma_read(dmadev, STM32_DMA_SPAR(chan->id)); in stm32_dma_dump_reg()
535 u32 sm0ar = stm32_dma_read(dmadev, STM32_DMA_SM0AR(chan->id)); in stm32_dma_dump_reg()
536 u32 sm1ar = stm32_dma_read(dmadev, STM32_DMA_SM1AR(chan->id)); in stm32_dma_dump_reg()
537 u32 sfcr = stm32_dma_read(dmadev, STM32_DMA_SFCR(chan->id)); in stm32_dma_dump_reg()
549 chan->next_sg++; in stm32_dma_sg_inc()
550 if (chan->desc->cyclic && (chan->next_sg == chan->desc->num_sgs)) in stm32_dma_sg_inc()
551 chan->next_sg = 0; in stm32_dma_sg_inc()
569 if (!chan->desc) { in stm32_dma_start_transfer()
570 vdesc = vchan_next_desc(&chan->vchan); in stm32_dma_start_transfer()
574 list_del(&vdesc->node); in stm32_dma_start_transfer()
576 chan->desc = to_stm32_dma_desc(vdesc); in stm32_dma_start_transfer()
577 chan->next_sg = 0; in stm32_dma_start_transfer()
580 if (chan->next_sg == chan->desc->num_sgs) in stm32_dma_start_transfer()
581 chan->next_sg = 0; in stm32_dma_start_transfer()
583 sg_req = &chan->desc->sg_req[chan->next_sg]; in stm32_dma_start_transfer()
584 reg = &sg_req->chan_reg; in stm32_dma_start_transfer()
587 if (chan->trig_mdma && chan->dma_sconfig.direction != DMA_MEM_TO_DEV) in stm32_dma_start_transfer()
588 reg->dma_scr &= ~STM32_DMA_SCR_TCIE; in stm32_dma_start_transfer()
590 reg->dma_scr &= ~STM32_DMA_SCR_EN; in stm32_dma_start_transfer()
591 stm32_dma_write(dmadev, STM32_DMA_SCR(chan->id), reg->dma_scr); in stm32_dma_start_transfer()
592 stm32_dma_write(dmadev, STM32_DMA_SPAR(chan->id), reg->dma_spar); in stm32_dma_start_transfer()
593 stm32_dma_write(dmadev, STM32_DMA_SM0AR(chan->id), reg->dma_sm0ar); in stm32_dma_start_transfer()
594 stm32_dma_write(dmadev, STM32_DMA_SFCR(chan->id), reg->dma_sfcr); in stm32_dma_start_transfer()
595 stm32_dma_write(dmadev, STM32_DMA_SM1AR(chan->id), reg->dma_sm1ar); in stm32_dma_start_transfer()
596 stm32_dma_write(dmadev, STM32_DMA_SNDTR(chan->id), reg->dma_sndtr); in stm32_dma_start_transfer()
605 if (chan->desc->cyclic) in stm32_dma_start_transfer()
611 chan->busy = true; in stm32_dma_start_transfer()
612 chan->status = DMA_IN_PROGRESS; in stm32_dma_start_transfer()
613 reg->dma_scr |= STM32_DMA_SCR_EN; in stm32_dma_start_transfer()
614 stm32_dma_write(dmadev, STM32_DMA_SCR(chan->id), reg->dma_scr); in stm32_dma_start_transfer()
616 dev_dbg(chan2dev(chan), "vchan %pK: started\n", &chan->vchan); in stm32_dma_start_transfer()
625 id = chan->id; in stm32_dma_configure_next_sg()
628 sg_req = &chan->desc->sg_req[chan->next_sg]; in stm32_dma_configure_next_sg()
631 dma_sm0ar = sg_req->chan_reg.dma_sm0ar; in stm32_dma_configure_next_sg()
636 dma_sm1ar = sg_req->chan_reg.dma_sm1ar; in stm32_dma_configure_next_sg()
652 dma_scr = stm32_dma_read(dmadev, STM32_DMA_SCR(chan->id)); in stm32_dma_handle_chan_paused()
658 if (chan->desc && chan->desc->cyclic) { in stm32_dma_handle_chan_paused()
659 if (chan->desc->num_sgs == 1) in stm32_dma_handle_chan_paused()
664 chan->chan_reg.dma_scr = dma_scr; in stm32_dma_handle_chan_paused()
670 if (chan->desc && chan->desc->cyclic) { in stm32_dma_handle_chan_paused()
672 stm32_dma_write(dmadev, STM32_DMA_SCR(chan->id), dma_scr); in stm32_dma_handle_chan_paused()
675 chan->chan_reg.dma_sndtr = stm32_dma_read(dmadev, STM32_DMA_SNDTR(chan->id)); in stm32_dma_handle_chan_paused()
677 chan->status = DMA_PAUSED; in stm32_dma_handle_chan_paused()
679 dev_dbg(chan2dev(chan), "vchan %pK: paused\n", &chan->vchan); in stm32_dma_handle_chan_paused()
688 id = chan->id; in stm32_dma_post_resume_reconfigure()
696 if (!chan->next_sg) in stm32_dma_post_resume_reconfigure()
697 sg_req = &chan->desc->sg_req[chan->desc->num_sgs - 1]; in stm32_dma_post_resume_reconfigure()
699 sg_req = &chan->desc->sg_req[chan->next_sg - 1]; in stm32_dma_post_resume_reconfigure()
702 stm32_dma_write(dmadev, STM32_DMA_SNDTR(chan->id), sg_req->chan_reg.dma_sndtr); in stm32_dma_post_resume_reconfigure()
705 stm32_dma_write(dmadev, STM32_DMA_SPAR(id), sg_req->chan_reg.dma_spar); in stm32_dma_post_resume_reconfigure()
708 stm32_dma_write(dmadev, STM32_DMA_SM0AR(id), sg_req->chan_reg.dma_sm0ar); in stm32_dma_post_resume_reconfigure()
709 stm32_dma_write(dmadev, STM32_DMA_SM1AR(id), sg_req->chan_reg.dma_sm1ar); in stm32_dma_post_resume_reconfigure()
712 if (chan->chan_reg.dma_scr & STM32_DMA_SCR_DBM) { in stm32_dma_post_resume_reconfigure()
715 if (chan->chan_reg.dma_scr & STM32_DMA_SCR_CT) in stm32_dma_post_resume_reconfigure()
719 } else if (chan->chan_reg.dma_scr & STM32_DMA_SCR_CIRC) { in stm32_dma_post_resume_reconfigure()
722 stm32_dma_write(dmadev, STM32_DMA_SCR(chan->id), dma_scr); in stm32_dma_post_resume_reconfigure()
729 stm32_dma_write(dmadev, STM32_DMA_SCR(chan->id), dma_scr); in stm32_dma_post_resume_reconfigure()
731 dev_dbg(chan2dev(chan), "vchan %pK: reconfigured after pause/resume\n", &chan->vchan); in stm32_dma_post_resume_reconfigure()
736 if (!chan->desc) in stm32_dma_handle_chan_done()
739 if (chan->desc->cyclic) { in stm32_dma_handle_chan_done()
740 vchan_cyclic_callback(&chan->desc->vdesc); in stm32_dma_handle_chan_done()
741 if (chan->trig_mdma) in stm32_dma_handle_chan_done()
750 chan->busy = false; in stm32_dma_handle_chan_done()
751 chan->status = DMA_COMPLETE; in stm32_dma_handle_chan_done()
752 if (chan->next_sg == chan->desc->num_sgs) { in stm32_dma_handle_chan_done()
753 vchan_cookie_complete(&chan->desc->vdesc); in stm32_dma_handle_chan_done()
754 chan->desc = NULL; in stm32_dma_handle_chan_done()
766 spin_lock(&chan->vchan.lock); in stm32_dma_chan_irq()
769 scr = stm32_dma_read(dmadev, STM32_DMA_SCR(chan->id)); in stm32_dma_chan_irq()
770 sfcr = stm32_dma_read(dmadev, STM32_DMA_SFCR(chan->id)); in stm32_dma_chan_irq()
793 if (chan->status != DMA_PAUSED) in stm32_dma_chan_irq()
811 spin_unlock(&chan->vchan.lock); in stm32_dma_chan_irq()
821 spin_lock_irqsave(&chan->vchan.lock, flags); in stm32_dma_issue_pending()
822 if (vchan_issue_pending(&chan->vchan) && !chan->desc && !chan->busy) { in stm32_dma_issue_pending()
823 dev_dbg(chan2dev(chan), "vchan %pK: issued\n", &chan->vchan); in stm32_dma_issue_pending()
827 spin_unlock_irqrestore(&chan->vchan.lock, flags); in stm32_dma_issue_pending()
836 if (chan->status != DMA_IN_PROGRESS) in stm32_dma_pause()
837 return -EPERM; in stm32_dma_pause()
839 spin_lock_irqsave(&chan->vchan.lock, flags); in stm32_dma_pause()
845 spin_unlock_irqrestore(&chan->vchan.lock, flags); in stm32_dma_pause()
854 struct stm32_dma_chan_reg chan_reg = chan->chan_reg; in stm32_dma_resume()
855 u32 id = chan->id, scr, ndtr, offset, spar, sm0ar, sm1ar; in stm32_dma_resume()
859 if (chan->status != DMA_PAUSED) in stm32_dma_resume()
860 return -EPERM; in stm32_dma_resume()
864 return -EPERM; in stm32_dma_resume()
866 spin_lock_irqsave(&chan->vchan.lock, flags); in stm32_dma_resume()
869 if (!chan->next_sg) in stm32_dma_resume()
870 sg_req = &chan->desc->sg_req[chan->desc->num_sgs - 1]; in stm32_dma_resume()
872 sg_req = &chan->desc->sg_req[chan->next_sg - 1]; in stm32_dma_resume()
874 ndtr = sg_req->chan_reg.dma_sndtr; in stm32_dma_resume()
875 offset = (ndtr - chan_reg.dma_sndtr); in stm32_dma_resume()
877 spar = sg_req->chan_reg.dma_spar; in stm32_dma_resume()
878 sm0ar = sg_req->chan_reg.dma_sm0ar; in stm32_dma_resume()
879 sm1ar = sg_req->chan_reg.dma_sm1ar; in stm32_dma_resume()
918 /* The stream may then be re-enabled to restart transfer from the point it was stopped */ in stm32_dma_resume()
919 chan->status = DMA_IN_PROGRESS; in stm32_dma_resume()
923 spin_unlock_irqrestore(&chan->vchan.lock, flags); in stm32_dma_resume()
925 dev_dbg(chan2dev(chan), "vchan %pK: resumed\n", &chan->vchan); in stm32_dma_resume()
941 src_addr_width = chan->dma_sconfig.src_addr_width; in stm32_dma_set_xfer_param()
942 dst_addr_width = chan->dma_sconfig.dst_addr_width; in stm32_dma_set_xfer_param()
943 src_maxburst = chan->dma_sconfig.src_maxburst; in stm32_dma_set_xfer_param()
944 dst_maxburst = chan->dma_sconfig.dst_maxburst; in stm32_dma_set_xfer_param()
945 fifoth = chan->threshold; in stm32_dma_set_xfer_param()
967 chan->mem_width = src_addr_width; in stm32_dma_set_xfer_param()
973 * Set memory burst size - burst not possible if address is not aligned on in stm32_dma_set_xfer_param()
976 if (buf_addr & (buf_len - 1)) in stm32_dma_set_xfer_param()
995 chan->chan_reg.dma_sfcr &= ~STM32_DMA_SFCR_FTH_MASK; in stm32_dma_set_xfer_param()
997 chan->chan_reg.dma_sfcr |= FIELD_PREP(STM32_DMA_SFCR_FTH_MASK, fifoth); in stm32_dma_set_xfer_param()
1000 chan->chan_reg.dma_spar = chan->dma_sconfig.dst_addr; in stm32_dma_set_xfer_param()
1015 chan->mem_burst = src_best_burst; in stm32_dma_set_xfer_param()
1023 chan->mem_width = dst_addr_width; in stm32_dma_set_xfer_param()
1029 * Set memory burst size - burst not possible if address is not aligned on in stm32_dma_set_xfer_param()
1032 if (buf_addr & (buf_len - 1)) in stm32_dma_set_xfer_param()
1040 chan->mem_burst = dst_best_burst; in stm32_dma_set_xfer_param()
1052 chan->chan_reg.dma_sfcr &= ~STM32_DMA_SFCR_FTH_MASK; in stm32_dma_set_xfer_param()
1054 chan->chan_reg.dma_sfcr |= FIELD_PREP(STM32_DMA_SFCR_FTH_MASK, fifoth); in stm32_dma_set_xfer_param()
1057 chan->chan_reg.dma_spar = chan->dma_sconfig.src_addr; in stm32_dma_set_xfer_param()
1058 *buswidth = chan->dma_sconfig.src_addr_width; in stm32_dma_set_xfer_param()
1063 return -EINVAL; in stm32_dma_set_xfer_param()
1069 chan->chan_reg.dma_scr &= ~(STM32_DMA_SCR_DIR_MASK | in stm32_dma_set_xfer_param()
1072 chan->chan_reg.dma_scr |= dma_scr; in stm32_dma_set_xfer_param()
1094 if (!chan->config_init) { in stm32_dma_prep_slave_sg()
1095 dev_err(chan2dev(chan), "dma channel is not configured\n"); in stm32_dma_prep_slave_sg()
1107 desc->num_sgs = sg_len; in stm32_dma_prep_slave_sg()
1110 if (chan->dma_sconfig.device_fc) in stm32_dma_prep_slave_sg()
1111 chan->chan_reg.dma_scr |= STM32_DMA_SCR_PFCTRL; in stm32_dma_prep_slave_sg()
1113 chan->chan_reg.dma_scr &= ~STM32_DMA_SCR_PFCTRL; in stm32_dma_prep_slave_sg()
1115 /* Activate Double Buffer Mode if DMA triggers STM32 MDMA and more than 1 sg */ in stm32_dma_prep_slave_sg()
1116 if (chan->trig_mdma && sg_len > 1) { in stm32_dma_prep_slave_sg()
1117 chan->chan_reg.dma_scr |= STM32_DMA_SCR_DBM; in stm32_dma_prep_slave_sg()
1118 chan->chan_reg.dma_scr &= ~STM32_DMA_SCR_CT; in stm32_dma_prep_slave_sg()
1128 desc->sg_req[i].len = sg_dma_len(sg); in stm32_dma_prep_slave_sg()
1130 nb_data_items = desc->sg_req[i].len / buswidth; in stm32_dma_prep_slave_sg()
1136 stm32_dma_clear_reg(&desc->sg_req[i].chan_reg); in stm32_dma_prep_slave_sg()
1137 desc->sg_req[i].chan_reg.dma_scr = chan->chan_reg.dma_scr; in stm32_dma_prep_slave_sg()
1138 desc->sg_req[i].chan_reg.dma_sfcr = chan->chan_reg.dma_sfcr; in stm32_dma_prep_slave_sg()
1139 desc->sg_req[i].chan_reg.dma_spar = chan->chan_reg.dma_spar; in stm32_dma_prep_slave_sg()
1140 desc->sg_req[i].chan_reg.dma_sm0ar = sg_dma_address(sg); in stm32_dma_prep_slave_sg()
1141 desc->sg_req[i].chan_reg.dma_sm1ar = sg_dma_address(sg); in stm32_dma_prep_slave_sg()
1142 if (chan->trig_mdma) in stm32_dma_prep_slave_sg()
1143 desc->sg_req[i].chan_reg.dma_sm1ar += sg_dma_len(sg); in stm32_dma_prep_slave_sg()
1144 desc->sg_req[i].chan_reg.dma_sndtr = nb_data_items; in stm32_dma_prep_slave_sg()
1146 desc->cyclic = false; in stm32_dma_prep_slave_sg()
1148 return vchan_tx_prep(&chan->vchan, &desc->vdesc, flags); in stm32_dma_prep_slave_sg()
1171 if (!chan->config_init) { in stm32_dma_prep_dma_cyclic()
1172 dev_err(chan2dev(chan), "dma channel is not configured\n"); in stm32_dma_prep_dma_cyclic()
1187 if (chan->busy) { in stm32_dma_prep_dma_cyclic()
1203 /* Enable Circular mode or double buffer mode */ in stm32_dma_prep_dma_cyclic()
1205 chan->chan_reg.dma_scr |= STM32_DMA_SCR_CIRC; in stm32_dma_prep_dma_cyclic()
1207 chan->chan_reg.dma_scr |= STM32_DMA_SCR_DBM; in stm32_dma_prep_dma_cyclic()
1208 chan->chan_reg.dma_scr &= ~STM32_DMA_SCR_CT; in stm32_dma_prep_dma_cyclic()
1212 chan->chan_reg.dma_scr &= ~STM32_DMA_SCR_PFCTRL; in stm32_dma_prep_dma_cyclic()
1219 desc->num_sgs = num_periods; in stm32_dma_prep_dma_cyclic()
1222 desc->sg_req[i].len = period_len; in stm32_dma_prep_dma_cyclic()
1224 stm32_dma_clear_reg(&desc->sg_req[i].chan_reg); in stm32_dma_prep_dma_cyclic()
1225 desc->sg_req[i].chan_reg.dma_scr = chan->chan_reg.dma_scr; in stm32_dma_prep_dma_cyclic()
1226 desc->sg_req[i].chan_reg.dma_sfcr = chan->chan_reg.dma_sfcr; in stm32_dma_prep_dma_cyclic()
1227 desc->sg_req[i].chan_reg.dma_spar = chan->chan_reg.dma_spar; in stm32_dma_prep_dma_cyclic()
1228 desc->sg_req[i].chan_reg.dma_sm0ar = buf_addr; in stm32_dma_prep_dma_cyclic()
1229 desc->sg_req[i].chan_reg.dma_sm1ar = buf_addr; in stm32_dma_prep_dma_cyclic()
1230 if (chan->trig_mdma) in stm32_dma_prep_dma_cyclic()
1231 desc->sg_req[i].chan_reg.dma_sm1ar += period_len; in stm32_dma_prep_dma_cyclic()
1232 desc->sg_req[i].chan_reg.dma_sndtr = nb_data_items; in stm32_dma_prep_dma_cyclic()
1233 if (!chan->trig_mdma) in stm32_dma_prep_dma_cyclic()
1236 desc->cyclic = true; in stm32_dma_prep_dma_cyclic()
1238 return vchan_tx_prep(&chan->vchan, &desc->vdesc, flags); in stm32_dma_prep_dma_cyclic()
1256 desc->num_sgs = num_sgs; in stm32_dma_prep_dma_memcpy()
1258 threshold = chan->threshold; in stm32_dma_prep_dma_memcpy()
1261 xfer_count = min_t(size_t, len - offset, in stm32_dma_prep_dma_memcpy()
1274 stm32_dma_clear_reg(&desc->sg_req[i].chan_reg); in stm32_dma_prep_dma_memcpy()
1275 desc->sg_req[i].chan_reg.dma_scr = in stm32_dma_prep_dma_memcpy()
1283 desc->sg_req[i].chan_reg.dma_sfcr |= STM32_DMA_SFCR_MASK; in stm32_dma_prep_dma_memcpy()
1284 desc->sg_req[i].chan_reg.dma_sfcr |= FIELD_PREP(STM32_DMA_SFCR_FTH_MASK, threshold); in stm32_dma_prep_dma_memcpy()
1285 desc->sg_req[i].chan_reg.dma_spar = src + offset; in stm32_dma_prep_dma_memcpy()
1286 desc->sg_req[i].chan_reg.dma_sm0ar = dest + offset; in stm32_dma_prep_dma_memcpy()
1287 desc->sg_req[i].chan_reg.dma_sndtr = xfer_count; in stm32_dma_prep_dma_memcpy()
1288 desc->sg_req[i].len = xfer_count; in stm32_dma_prep_dma_memcpy()
1290 desc->cyclic = false; in stm32_dma_prep_dma_memcpy()
1292 return vchan_tx_prep(&chan->vchan, &desc->vdesc, flags); in stm32_dma_prep_dma_memcpy()
1300 dma_scr = stm32_dma_read(dmadev, STM32_DMA_SCR(chan->id)); in stm32_dma_get_remaining_bytes()
1302 ndtr = stm32_dma_read(dmadev, STM32_DMA_SNDTR(chan->id)); in stm32_dma_get_remaining_bytes()
1308 * stm32_dma_is_current_sg - check that expected sg_req is currently transferred
1309 * @chan: dma channel
1312 * switched on the next transfer in double buffer mode. The test is done by
1316 * Returns true if expected current transfer is still running or double
1325 id = chan->id; in stm32_dma_is_current_sg()
1332 sg_req = &chan->desc->sg_req[chan->next_sg]; in stm32_dma_is_current_sg()
1333 period_len = sg_req->len; in stm32_dma_is_current_sg()
1335 /* DBM - take care of a previous pause/resume not yet post reconfigured */ in stm32_dma_is_current_sg()
1342 return (dma_smar >= sg_req->chan_reg.dma_sm0ar && in stm32_dma_is_current_sg()
1343 dma_smar < sg_req->chan_reg.dma_sm0ar + period_len); in stm32_dma_is_current_sg()
1351 return (dma_smar >= sg_req->chan_reg.dma_sm1ar && in stm32_dma_is_current_sg()
1352 dma_smar < sg_req->chan_reg.dma_sm1ar + period_len); in stm32_dma_is_current_sg()
1362 struct stm32_dma_sg_req *sg_req = &chan->desc->sg_req[chan->next_sg]; in stm32_dma_desc_residue()
1368 * - the sg_req currently transferred in stm32_dma_desc_residue()
1369 * - the Hardware remaining position in this sg (NDTR bits field). in stm32_dma_desc_residue()
1371 * A race condition may occur if DMA is running in cyclic or double in stm32_dma_desc_residue()
1379 * - read the SxNDTR register in stm32_dma_desc_residue()
1380 * - crosscheck that hardware is still in current transfer. in stm32_dma_desc_residue()
1385 * This race condition doesn't apply for none cyclic mode, as double in stm32_dma_desc_residue()
1392 if ((chan->desc->cyclic || chan->trig_mdma) && !stm32_dma_is_current_sg(chan)) { in stm32_dma_desc_residue()
1394 if (n_sg == chan->desc->num_sgs) in stm32_dma_desc_residue()
1396 if (!chan->trig_mdma) in stm32_dma_desc_residue()
1397 residue = sg_req->len; in stm32_dma_desc_residue()
1407 if ((!chan->desc->cyclic && !chan->trig_mdma) || n_sg != 0) in stm32_dma_desc_residue()
1408 for (i = n_sg; i < desc->num_sgs; i++) in stm32_dma_desc_residue()
1409 residue += desc->sg_req[i].len; in stm32_dma_desc_residue()
1411 if (!chan->mem_burst) in stm32_dma_desc_residue()
1414 burst_size = chan->mem_burst * chan->mem_width; in stm32_dma_desc_residue()
1417 residue = residue - modulo + burst_size; in stm32_dma_desc_residue()
1436 status = chan->status; in stm32_dma_tx_status()
1441 spin_lock_irqsave(&chan->vchan.lock, flags); in stm32_dma_tx_status()
1442 vdesc = vchan_find_desc(&chan->vchan, cookie); in stm32_dma_tx_status()
1443 if (chan->desc && cookie == chan->desc->vdesc.tx.cookie) in stm32_dma_tx_status()
1444 residue = stm32_dma_desc_residue(chan, chan->desc, in stm32_dma_tx_status()
1445 chan->next_sg); in stm32_dma_tx_status()
1451 spin_unlock_irqrestore(&chan->vchan.lock, flags); in stm32_dma_tx_status()
1462 chan->config_init = false; in stm32_dma_alloc_chan_resources()
1464 ret = pm_runtime_resume_and_get(dmadev->ddev.dev); in stm32_dma_alloc_chan_resources()
1470 pm_runtime_put(dmadev->ddev.dev); in stm32_dma_alloc_chan_resources()
1481 dev_dbg(chan2dev(chan), "Freeing channel %d\n", chan->id); in stm32_dma_free_chan_resources()
1483 if (chan->busy) { in stm32_dma_free_chan_resources()
1484 spin_lock_irqsave(&chan->vchan.lock, flags); in stm32_dma_free_chan_resources()
1486 chan->desc = NULL; in stm32_dma_free_chan_resources()
1487 spin_unlock_irqrestore(&chan->vchan.lock, flags); in stm32_dma_free_chan_resources()
1490 pm_runtime_put(dmadev->ddev.dev); in stm32_dma_free_chan_resources()
1493 stm32_dma_clear_reg(&chan->chan_reg); in stm32_dma_free_chan_resources()
1494 chan->threshold = 0; in stm32_dma_free_chan_resources()
1505 stm32_dma_clear_reg(&chan->chan_reg); in stm32_dma_set_config()
1507 chan->chan_reg.dma_scr = cfg->stream_config & STM32_DMA_SCR_CFG_MASK; in stm32_dma_set_config()
1508 chan->chan_reg.dma_scr |= FIELD_PREP(STM32_DMA_SCR_REQ_MASK, cfg->request_line); in stm32_dma_set_config()
1511 chan->chan_reg.dma_scr |= STM32_DMA_SCR_TEIE | STM32_DMA_SCR_TCIE; in stm32_dma_set_config()
1513 chan->threshold = FIELD_GET(STM32_DMA_THRESHOLD_FTR_MASK, cfg->features); in stm32_dma_set_config()
1514 if (FIELD_GET(STM32_DMA_DIRECT_MODE_MASK, cfg->features)) in stm32_dma_set_config()
1515 chan->threshold = STM32_DMA_FIFO_THRESHOLD_NONE; in stm32_dma_set_config()
1516 if (FIELD_GET(STM32_DMA_ALT_ACK_MODE_MASK, cfg->features)) in stm32_dma_set_config()
1517 chan->chan_reg.dma_scr |= STM32_DMA_SCR_TRBUFF; in stm32_dma_set_config()
1518 chan->mdma_config.stream_id = FIELD_GET(STM32_DMA_MDMA_STREAM_ID_MASK, cfg->features); in stm32_dma_set_config()
1524 struct stm32_dma_device *dmadev = ofdma->of_dma_data; in stm32_dma_of_xlate()
1525 struct device *dev = dmadev->ddev.dev; in stm32_dma_of_xlate()
1530 if (dma_spec->args_count < 4) { in stm32_dma_of_xlate()
1535 cfg.channel_id = dma_spec->args[0]; in stm32_dma_of_xlate()
1536 cfg.request_line = dma_spec->args[1]; in stm32_dma_of_xlate()
1537 cfg.stream_config = dma_spec->args[2]; in stm32_dma_of_xlate()
1538 cfg.features = dma_spec->args[3]; in stm32_dma_of_xlate()
1542 dev_err(dev, "Bad channel and/or request id\n"); in stm32_dma_of_xlate()
1546 chan = &dmadev->chan[cfg.channel_id]; in stm32_dma_of_xlate()
1548 c = dma_get_slave_channel(&chan->vchan.chan); in stm32_dma_of_xlate()
1560 { .compatible = "st,stm32-dma", },
1574 dmadev = devm_kzalloc(&pdev->dev, sizeof(*dmadev), GFP_KERNEL); in stm32_dma_probe()
1576 return -ENOMEM; in stm32_dma_probe()
1578 dd = &dmadev->ddev; in stm32_dma_probe()
1580 dmadev->base = devm_platform_get_and_ioremap_resource(pdev, 0, &res); in stm32_dma_probe()
1581 if (IS_ERR(dmadev->base)) in stm32_dma_probe()
1582 return PTR_ERR(dmadev->base); in stm32_dma_probe()
1584 dmadev->clk = devm_clk_get(&pdev->dev, NULL); in stm32_dma_probe()
1585 if (IS_ERR(dmadev->clk)) in stm32_dma_probe()
1586 return dev_err_probe(&pdev->dev, PTR_ERR(dmadev->clk), "Can't get clock\n"); in stm32_dma_probe()
1588 ret = clk_prepare_enable(dmadev->clk); in stm32_dma_probe()
1590 dev_err(&pdev->dev, "clk_prep_enable error: %d\n", ret); in stm32_dma_probe()
1594 dmadev->mem2mem = of_property_read_bool(pdev->dev.of_node, in stm32_dma_probe()
1597 rst = devm_reset_control_get(&pdev->dev, NULL); in stm32_dma_probe()
1600 if (ret == -EPROBE_DEFER) in stm32_dma_probe()
1608 dma_set_max_seg_size(&pdev->dev, STM32_DMA_ALIGNED_MAX_DATA_ITEMS); in stm32_dma_probe()
1610 dma_cap_set(DMA_SLAVE, dd->cap_mask); in stm32_dma_probe()
1611 dma_cap_set(DMA_PRIVATE, dd->cap_mask); in stm32_dma_probe()
1612 dma_cap_set(DMA_CYCLIC, dd->cap_mask); in stm32_dma_probe()
1613 dd->device_alloc_chan_resources = stm32_dma_alloc_chan_resources; in stm32_dma_probe()
1614 dd->device_free_chan_resources = stm32_dma_free_chan_resources; in stm32_dma_probe()
1615 dd->device_tx_status = stm32_dma_tx_status; in stm32_dma_probe()
1616 dd->device_issue_pending = stm32_dma_issue_pending; in stm32_dma_probe()
1617 dd->device_prep_slave_sg = stm32_dma_prep_slave_sg; in stm32_dma_probe()
1618 dd->device_prep_dma_cyclic = stm32_dma_prep_dma_cyclic; in stm32_dma_probe()
1619 dd->device_config = stm32_dma_slave_config; in stm32_dma_probe()
1620 dd->device_pause = stm32_dma_pause; in stm32_dma_probe()
1621 dd->device_resume = stm32_dma_resume; in stm32_dma_probe()
1622 dd->device_terminate_all = stm32_dma_terminate_all; in stm32_dma_probe()
1623 dd->device_synchronize = stm32_dma_synchronize; in stm32_dma_probe()
1624 dd->src_addr_widths = BIT(DMA_SLAVE_BUSWIDTH_1_BYTE) | in stm32_dma_probe()
1627 dd->dst_addr_widths = BIT(DMA_SLAVE_BUSWIDTH_1_BYTE) | in stm32_dma_probe()
1630 dd->directions = BIT(DMA_DEV_TO_MEM) | BIT(DMA_MEM_TO_DEV); in stm32_dma_probe()
1631 dd->residue_granularity = DMA_RESIDUE_GRANULARITY_BURST; in stm32_dma_probe()
1632 dd->copy_align = DMAENGINE_ALIGN_32_BYTES; in stm32_dma_probe()
1633 dd->max_burst = STM32_DMA_MAX_BURST; in stm32_dma_probe()
1634 dd->max_sg_burst = STM32_DMA_ALIGNED_MAX_DATA_ITEMS; in stm32_dma_probe()
1635 dd->descriptor_reuse = true; in stm32_dma_probe()
1636 dd->dev = &pdev->dev; in stm32_dma_probe()
1637 INIT_LIST_HEAD(&dd->channels); in stm32_dma_probe()
1639 if (dmadev->mem2mem) { in stm32_dma_probe()
1640 dma_cap_set(DMA_MEMCPY, dd->cap_mask); in stm32_dma_probe()
1641 dd->device_prep_dma_memcpy = stm32_dma_prep_dma_memcpy; in stm32_dma_probe()
1642 dd->directions |= BIT(DMA_MEM_TO_MEM); in stm32_dma_probe()
1646 chan = &dmadev->chan[i]; in stm32_dma_probe()
1647 chan->id = i; in stm32_dma_probe()
1648 chan->vchan.desc_free = stm32_dma_desc_free; in stm32_dma_probe()
1649 vchan_init(&chan->vchan, dd); in stm32_dma_probe()
1651 chan->mdma_config.ifcr = res->start; in stm32_dma_probe()
1652 chan->mdma_config.ifcr += STM32_DMA_IFCR(chan->id); in stm32_dma_probe()
1654 chan->mdma_config.tcf = STM32_DMA_TCI; in stm32_dma_probe()
1655 chan->mdma_config.tcf <<= STM32_DMA_FLAGS_SHIFT(chan->id); in stm32_dma_probe()
1663 chan = &dmadev->chan[i]; in stm32_dma_probe()
1667 chan->irq = ret; in stm32_dma_probe()
1669 ret = devm_request_irq(&pdev->dev, chan->irq, in stm32_dma_probe()
1673 dev_err(&pdev->dev, in stm32_dma_probe()
1674 "request_irq failed with err %d channel %d\n", in stm32_dma_probe()
1680 ret = of_dma_controller_register(pdev->dev.of_node, in stm32_dma_probe()
1683 dev_err(&pdev->dev, in stm32_dma_probe()
1690 pm_runtime_set_active(&pdev->dev); in stm32_dma_probe()
1691 pm_runtime_enable(&pdev->dev); in stm32_dma_probe()
1692 pm_runtime_get_noresume(&pdev->dev); in stm32_dma_probe()
1693 pm_runtime_put(&pdev->dev); in stm32_dma_probe()
1695 dev_info(&pdev->dev, "STM32 DMA driver registered\n"); in stm32_dma_probe()
1702 clk_disable_unprepare(dmadev->clk); in stm32_dma_probe()
1712 clk_disable_unprepare(dmadev->clk); in stm32_dma_runtime_suspend()
1722 ret = clk_prepare_enable(dmadev->clk); in stm32_dma_runtime_resume()
1746 return -EBUSY; in stm32_dma_pm_suspend()
1771 .name = "stm32-dma",