1 // SPDX-License-Identifier: GPL-2.0-only
3  * Driver for the Analog Devices AXI-DMAC core
5  * Copyright 2013-2019 Analog Devices Inc.
6  *  Author: Lars-Peter Clausen <lars@metafoo.de>
12 #include <linux/dma-mapping.h>
25 #include <linux/fpga/adi-axi-common.h>
27 #include <dt-bindings/dma/axi-dmac.h>
30 #include "virt-dma.h"
33  * The AXI-DMAC is a soft IP core that is used in FPGA designs. The core has
44  * runtime. By extension this means that each channel is uni-directional. It can
176 	return container_of(chan->vchan.chan.device, struct axi_dmac,  in chan_to_axi_dmac()
193 	writel(val, axi_dmac->base + reg);  in axi_dmac_write()
198 	return readl(axi_dmac->base + reg);  in axi_dmac_read()
203 	return chan->src_type == AXI_DMAC_BUS_TYPE_AXI_MM;  in axi_dmac_src_is_mem()
208 	return chan->dest_type == AXI_DMAC_BUS_TYPE_AXI_MM;  in axi_dmac_dest_is_mem()
211 static bool axi_dmac_check_len(struct axi_dmac_chan *chan, unsigned int len)  in axi_dmac_check_len()  argument
213 	if (len == 0)  in axi_dmac_check_len()
215 	if ((len & chan->length_align_mask) != 0) /* Not aligned */  in axi_dmac_check_len()
222 	if ((addr & chan->address_align_mask) != 0) /* Not aligned */  in axi_dmac_check_addr()
236 	if (!chan->hw_sg) {  in axi_dmac_start_transfer()
242 	desc = chan->next_desc;  in axi_dmac_start_transfer()
245 		vdesc = vchan_next_desc(&chan->vchan);  in axi_dmac_start_transfer()
248 		list_move_tail(&vdesc->node, &chan->active_descs);  in axi_dmac_start_transfer()
251 	sg = &desc->sg[desc->num_submitted];  in axi_dmac_start_transfer()
254 	if (sg->hw->id != AXI_DMAC_SG_UNUSED) {  in axi_dmac_start_transfer()
255 		sg->schedule_when_free = true;  in axi_dmac_start_transfer()
259 	if (chan->hw_sg) {  in axi_dmac_start_transfer()
260 		chan->next_desc = NULL;  in axi_dmac_start_transfer()
261 	} else if (++desc->num_submitted == desc->num_sgs ||  in axi_dmac_start_transfer()
262 		   desc->have_partial_xfer) {  in axi_dmac_start_transfer()
263 		if (desc->cyclic)  in axi_dmac_start_transfer()
264 			desc->num_submitted = 0; /* Start again */  in axi_dmac_start_transfer()
266 			chan->next_desc = NULL;  in axi_dmac_start_transfer()
269 		chan->next_desc = desc;  in axi_dmac_start_transfer()
272 	sg->hw->id = axi_dmac_read(dmac, AXI_DMAC_REG_TRANSFER_ID);  in axi_dmac_start_transfer()
274 	if (!chan->hw_sg) {  in axi_dmac_start_transfer()
276 			axi_dmac_write(dmac, AXI_DMAC_REG_DEST_ADDRESS, sg->hw->dest_addr);  in axi_dmac_start_transfer()
277 			axi_dmac_write(dmac, AXI_DMAC_REG_DEST_STRIDE, sg->hw->dst_stride);  in axi_dmac_start_transfer()
281 			axi_dmac_write(dmac, AXI_DMAC_REG_SRC_ADDRESS, sg->hw->src_addr);  in axi_dmac_start_transfer()
282 			axi_dmac_write(dmac, AXI_DMAC_REG_SRC_STRIDE, sg->hw->src_stride);  in axi_dmac_start_transfer()
290 	if (chan->hw_cyclic && desc->cyclic && !desc->vdesc.tx.callback) {  in axi_dmac_start_transfer()
291 		if (chan->hw_sg)  in axi_dmac_start_transfer()
292 			desc->sg[desc->num_sgs - 1].hw->flags &= ~AXI_DMAC_HW_FLAG_IRQ;  in axi_dmac_start_transfer()
293 		else if (desc->num_sgs == 1)  in axi_dmac_start_transfer()
297 	if (chan->hw_partial_xfer)  in axi_dmac_start_transfer()
300 	if (chan->hw_sg) {  in axi_dmac_start_transfer()
301 		axi_dmac_write(dmac, AXI_DMAC_REG_SG_ADDRESS, (u32)sg->hw_phys);  in axi_dmac_start_transfer()
303 			       (u64)sg->hw_phys >> 32);  in axi_dmac_start_transfer()
305 		axi_dmac_write(dmac, AXI_DMAC_REG_X_LENGTH, sg->hw->x_len);  in axi_dmac_start_transfer()
306 		axi_dmac_write(dmac, AXI_DMAC_REG_Y_LENGTH, sg->hw->y_len);  in axi_dmac_start_transfer()
314 	return list_first_entry_or_null(&chan->active_descs,  in axi_dmac_active_desc()
321 	if (chan->hw_2d)  in axi_dmac_total_sg_bytes()
322 		return (sg->hw->x_len + 1) * (sg->hw->y_len + 1);  in axi_dmac_total_sg_bytes()
324 		return (sg->hw->x_len + 1);  in axi_dmac_total_sg_bytes()
332 	u32 xfer_done, len, id, i;  in axi_dmac_dequeue_partial_xfers()  local
336 		len = axi_dmac_read(dmac, AXI_DMAC_REG_PARTIAL_XFER_LEN);  in axi_dmac_dequeue_partial_xfers()
340 		list_for_each_entry(desc, &chan->active_descs, vdesc.node) {  in axi_dmac_dequeue_partial_xfers()
341 			for (i = 0; i < desc->num_sgs; i++) {  in axi_dmac_dequeue_partial_xfers()
342 				sg = &desc->sg[i];  in axi_dmac_dequeue_partial_xfers()
343 				if (sg->hw->id == AXI_DMAC_SG_UNUSED)  in axi_dmac_dequeue_partial_xfers()
345 				if (sg->hw->id == id) {  in axi_dmac_dequeue_partial_xfers()
346 					desc->have_partial_xfer = true;  in axi_dmac_dequeue_partial_xfers()
347 					sg->partial_len = len;  in axi_dmac_dequeue_partial_xfers()
357 			dev_dbg(dmac->dma_dev.dev,  in axi_dmac_dequeue_partial_xfers()
358 				"Found partial segment id=%u, len=%u\n",  in axi_dmac_dequeue_partial_xfers()
359 				id, len);  in axi_dmac_dequeue_partial_xfers()
361 			dev_warn(dmac->dma_dev.dev,  in axi_dmac_dequeue_partial_xfers()
362 				 "Not found partial segment id=%u, len=%u\n",  in axi_dmac_dequeue_partial_xfers()
363 				 id, len);  in axi_dmac_dequeue_partial_xfers()
376 	struct dmaengine_result *rslt = &active->vdesc.tx_result;  in axi_dmac_compute_residue()
377 	unsigned int start = active->num_completed - 1;  in axi_dmac_compute_residue()
381 	rslt->result = DMA_TRANS_NOERROR;  in axi_dmac_compute_residue()
382 	rslt->residue = 0;  in axi_dmac_compute_residue()
384 	if (chan->hw_sg)  in axi_dmac_compute_residue()
391 	for (i = start; i < active->num_sgs; i++) {  in axi_dmac_compute_residue()
392 		sg = &active->sg[i];  in axi_dmac_compute_residue()
394 		rslt->residue += (total - sg->partial_len);  in axi_dmac_compute_residue()
409 	if (chan->hw_partial_xfer &&  in axi_dmac_transfer_done()
413 	if (chan->hw_sg) {  in axi_dmac_transfer_done()
414 		if (active->cyclic) {  in axi_dmac_transfer_done()
415 			vchan_cyclic_callback(&active->vdesc);  in axi_dmac_transfer_done()
417 			list_del(&active->vdesc.node);  in axi_dmac_transfer_done()
418 			vchan_cookie_complete(&active->vdesc);  in axi_dmac_transfer_done()
424 			sg = &active->sg[active->num_completed];  in axi_dmac_transfer_done()
425 			if (sg->hw->id == AXI_DMAC_SG_UNUSED) /* Not yet submitted */  in axi_dmac_transfer_done()
427 			if (!(BIT(sg->hw->id) & completed_transfers))  in axi_dmac_transfer_done()
429 			active->num_completed++;  in axi_dmac_transfer_done()
430 			sg->hw->id = AXI_DMAC_SG_UNUSED;  in axi_dmac_transfer_done()
431 			if (sg->schedule_when_free) {  in axi_dmac_transfer_done()
432 				sg->schedule_when_free = false;  in axi_dmac_transfer_done()
436 			if (sg->partial_len)  in axi_dmac_transfer_done()
439 			if (active->cyclic)  in axi_dmac_transfer_done()
440 				vchan_cyclic_callback(&active->vdesc);  in axi_dmac_transfer_done()
442 			if (active->num_completed == active->num_sgs ||  in axi_dmac_transfer_done()
443 			    sg->partial_len) {  in axi_dmac_transfer_done()
444 				if (active->cyclic) {  in axi_dmac_transfer_done()
445 					active->num_completed = 0; /* wrap around */  in axi_dmac_transfer_done()
447 					list_del(&active->vdesc.node);  in axi_dmac_transfer_done()
448 					vchan_cookie_complete(&active->vdesc);  in axi_dmac_transfer_done()
470 	spin_lock(&dmac->chan.vchan.lock);  in axi_dmac_interrupt_handler()
476 		start_next = axi_dmac_transfer_done(&dmac->chan, completed);  in axi_dmac_interrupt_handler()
480 		axi_dmac_start_transfer(&dmac->chan);  in axi_dmac_interrupt_handler()
481 	spin_unlock(&dmac->chan.vchan.lock);  in axi_dmac_interrupt_handler()
493 	spin_lock_irqsave(&chan->vchan.lock, flags);  in axi_dmac_terminate_all()
495 	chan->next_desc = NULL;  in axi_dmac_terminate_all()
496 	vchan_get_all_descriptors(&chan->vchan, &head);  in axi_dmac_terminate_all()
497 	list_splice_tail_init(&chan->active_descs, &head);  in axi_dmac_terminate_all()
498 	spin_unlock_irqrestore(&chan->vchan.lock, flags);  in axi_dmac_terminate_all()
500 	vchan_dma_desc_free_list(&chan->vchan, &head);  in axi_dmac_terminate_all()
509 	vchan_synchronize(&chan->vchan);  in axi_dmac_synchronize()
519 	if (chan->hw_sg)  in axi_dmac_issue_pending()
524 	spin_lock_irqsave(&chan->vchan.lock, flags);  in axi_dmac_issue_pending()
525 	if (vchan_issue_pending(&chan->vchan))  in axi_dmac_issue_pending()
527 	spin_unlock_irqrestore(&chan->vchan.lock, flags);  in axi_dmac_issue_pending()
534 	struct device *dev = dmac->dma_dev.dev;  in axi_dmac_alloc_desc()
543 	desc->num_sgs = num_sgs;  in axi_dmac_alloc_desc()
544 	desc->chan = chan;  in axi_dmac_alloc_desc()
554 		desc->sg[i].hw = &hws[i];  in axi_dmac_alloc_desc()
555 		desc->sg[i].hw_phys = hw_phys + i * sizeof(*hws);  in axi_dmac_alloc_desc()
565 	desc->sg[num_sgs - 1].hw->flags = AXI_DMAC_HW_FLAG_LAST | AXI_DMAC_HW_FLAG_IRQ;  in axi_dmac_alloc_desc()
572 	struct axi_dmac *dmac = chan_to_axi_dmac(desc->chan);  in axi_dmac_free_desc()
573 	struct device *dev = dmac->dma_dev.dev;  in axi_dmac_free_desc()
574 	struct axi_dmac_hw_desc *hw = desc->sg[0].hw;  in axi_dmac_free_desc()
575 	dma_addr_t hw_phys = desc->sg[0].hw_phys;  in axi_dmac_free_desc()
577 	dma_free_coherent(dev, PAGE_ALIGN(desc->num_sgs * sizeof(*hw)),  in axi_dmac_free_desc()
589 	unsigned int len;  in axi_dmac_fill_linear_sg()  local
592 	num_segments = DIV_ROUND_UP(period_len, chan->max_length);  in axi_dmac_fill_linear_sg()
595 	segment_size = ((segment_size - 1) | chan->length_align_mask) + 1;  in axi_dmac_fill_linear_sg()
598 		for (len = period_len; len > segment_size; sg++) {  in axi_dmac_fill_linear_sg()
600 				sg->hw->dest_addr = addr;  in axi_dmac_fill_linear_sg()
602 				sg->hw->src_addr = addr;  in axi_dmac_fill_linear_sg()
603 			sg->hw->x_len = segment_size - 1;  in axi_dmac_fill_linear_sg()
604 			sg->hw->y_len = 0;  in axi_dmac_fill_linear_sg()
605 			sg->hw->flags = 0;  in axi_dmac_fill_linear_sg()
607 			len -= segment_size;  in axi_dmac_fill_linear_sg()
611 			sg->hw->dest_addr = addr;  in axi_dmac_fill_linear_sg()
613 			sg->hw->src_addr = addr;  in axi_dmac_fill_linear_sg()
614 		sg->hw->x_len = len - 1;  in axi_dmac_fill_linear_sg()
615 		sg->hw->y_len = 0;  in axi_dmac_fill_linear_sg()
617 		addr += len;  in axi_dmac_fill_linear_sg()
634 	if (direction != chan->direction)  in axi_dmac_prep_peripheral_dma_vec()
638 		num_sgs += DIV_ROUND_UP(vecs[i].len, chan->max_length);  in axi_dmac_prep_peripheral_dma_vec()
644 	dsg = desc->sg;  in axi_dmac_prep_peripheral_dma_vec()
648 		    !axi_dmac_check_len(chan, vecs[i].len)) {  in axi_dmac_prep_peripheral_dma_vec()
654 					      vecs[i].len, dsg);  in axi_dmac_prep_peripheral_dma_vec()
657 	desc->cyclic = false;  in axi_dmac_prep_peripheral_dma_vec()
659 	return vchan_tx_prep(&chan->vchan, &desc->vdesc, flags);  in axi_dmac_prep_peripheral_dma_vec()
674 	if (direction != chan->direction)  in axi_dmac_prep_slave_sg()
679 		num_sgs += DIV_ROUND_UP(sg_dma_len(sg), chan->max_length);  in axi_dmac_prep_slave_sg()
685 	dsg = desc->sg;  in axi_dmac_prep_slave_sg()
698 	desc->cyclic = false;  in axi_dmac_prep_slave_sg()
700 	return vchan_tx_prep(&chan->vchan, &desc->vdesc, flags);  in axi_dmac_prep_slave_sg()
712 	if (direction != chan->direction)  in axi_dmac_prep_dma_cyclic()
723 	num_segments = DIV_ROUND_UP(period_len, chan->max_length);  in axi_dmac_prep_dma_cyclic()
731 	desc->sg[num_sgs - 1].hw->next_sg_addr = desc->sg[0].hw_phys;  in axi_dmac_prep_dma_cyclic()
732 	desc->sg[num_sgs - 1].hw->flags &= ~AXI_DMAC_HW_FLAG_LAST;  in axi_dmac_prep_dma_cyclic()
735 		period_len, desc->sg);  in axi_dmac_prep_dma_cyclic()
737 	desc->cyclic = true;  in axi_dmac_prep_dma_cyclic()
739 	return vchan_tx_prep(&chan->vchan, &desc->vdesc, flags);  in axi_dmac_prep_dma_cyclic()
750 	if (xt->frame_size != 1)  in axi_dmac_prep_interleaved()
753 	if (xt->dir != chan->direction)  in axi_dmac_prep_interleaved()
757 		if (!xt->src_inc || !axi_dmac_check_addr(chan, xt->src_start))  in axi_dmac_prep_interleaved()
762 		if (!xt->dst_inc || !axi_dmac_check_addr(chan, xt->dst_start))  in axi_dmac_prep_interleaved()
766 	dst_icg = dmaengine_get_dst_icg(xt, &xt->sgl[0]);  in axi_dmac_prep_interleaved()
767 	src_icg = dmaengine_get_src_icg(xt, &xt->sgl[0]);  in axi_dmac_prep_interleaved()
769 	if (chan->hw_2d) {  in axi_dmac_prep_interleaved()
770 		if (!axi_dmac_check_len(chan, xt->sgl[0].size) ||  in axi_dmac_prep_interleaved()
771 		    xt->numf == 0)  in axi_dmac_prep_interleaved()
773 		if (xt->sgl[0].size + dst_icg > chan->max_length ||  in axi_dmac_prep_interleaved()
774 		    xt->sgl[0].size + src_icg > chan->max_length)  in axi_dmac_prep_interleaved()
779 		if (chan->max_length / xt->sgl[0].size < xt->numf)  in axi_dmac_prep_interleaved()
781 		if (!axi_dmac_check_len(chan, xt->sgl[0].size * xt->numf))  in axi_dmac_prep_interleaved()
790 		desc->sg[0].hw->src_addr = xt->src_start;  in axi_dmac_prep_interleaved()
791 		desc->sg[0].hw->src_stride = xt->sgl[0].size + src_icg;  in axi_dmac_prep_interleaved()
795 		desc->sg[0].hw->dest_addr = xt->dst_start;  in axi_dmac_prep_interleaved()
796 		desc->sg[0].hw->dst_stride = xt->sgl[0].size + dst_icg;  in axi_dmac_prep_interleaved()
799 	if (chan->hw_2d) {  in axi_dmac_prep_interleaved()
800 		desc->sg[0].hw->x_len = xt->sgl[0].size - 1;  in axi_dmac_prep_interleaved()
801 		desc->sg[0].hw->y_len = xt->numf - 1;  in axi_dmac_prep_interleaved()
803 		desc->sg[0].hw->x_len = xt->sgl[0].size * xt->numf - 1;  in axi_dmac_prep_interleaved()
804 		desc->sg[0].hw->y_len = 0;  in axi_dmac_prep_interleaved()
808 		desc->cyclic = true;  in axi_dmac_prep_interleaved()
810 	return vchan_tx_prep(&chan->vchan, &desc->vdesc, flags);  in axi_dmac_prep_interleaved()
866 	chan->address_align_mask = max(chan->dest_width, chan->src_width) - 1;  in axi_dmac_adjust_chan_params()
869 		chan->direction = DMA_MEM_TO_MEM;  in axi_dmac_adjust_chan_params()
871 		chan->direction = DMA_MEM_TO_DEV;  in axi_dmac_adjust_chan_params()
873 		chan->direction = DMA_DEV_TO_MEM;  in axi_dmac_adjust_chan_params()
875 		chan->direction = DMA_DEV_TO_DEV;  in axi_dmac_adjust_chan_params()
895 		return -EINVAL;  in axi_dmac_parse_chan_dt()
897 	ret = of_property_read_u32(of_chan, "adi,source-bus-type", &val);  in axi_dmac_parse_chan_dt()
901 		return -EINVAL;  in axi_dmac_parse_chan_dt()
902 	chan->src_type = val;  in axi_dmac_parse_chan_dt()
904 	ret = of_property_read_u32(of_chan, "adi,destination-bus-type", &val);  in axi_dmac_parse_chan_dt()
908 		return -EINVAL;  in axi_dmac_parse_chan_dt()
909 	chan->dest_type = val;  in axi_dmac_parse_chan_dt()
911 	ret = of_property_read_u32(of_chan, "adi,source-bus-width", &val);  in axi_dmac_parse_chan_dt()
914 	chan->src_width = val / 8;  in axi_dmac_parse_chan_dt()
916 	ret = of_property_read_u32(of_chan, "adi,destination-bus-width", &val);  in axi_dmac_parse_chan_dt()
919 	chan->dest_width = val / 8;  in axi_dmac_parse_chan_dt()
931 	of_channels = of_get_child_by_name(dev->of_node, "adi,channels");  in axi_dmac_parse_dt()
933 		return -ENODEV;  in axi_dmac_parse_dt()
936 		ret = axi_dmac_parse_chan_dt(of_chan, &dmac->chan);  in axi_dmac_parse_dt()
940 			return -EINVAL;  in axi_dmac_parse_dt()
950 	struct axi_dmac_chan *chan = &dmac->chan;  in axi_dmac_read_chan_config()
956 		return -EFAULT;  in axi_dmac_read_chan_config()
962 		return -EINVAL;  in axi_dmac_read_chan_config()
964 	chan->src_type = val;  in axi_dmac_read_chan_config()
969 		return -EINVAL;  in axi_dmac_read_chan_config()
971 	chan->dest_type = val;  in axi_dmac_read_chan_config()
976 		return -EINVAL;  in axi_dmac_read_chan_config()
979 	chan->src_width = 1 << val;  in axi_dmac_read_chan_config()
984 		return -EINVAL;  in axi_dmac_read_chan_config()
986 	chan->dest_width = 1 << val;  in axi_dmac_read_chan_config()
995 	struct axi_dmac_chan *chan = &dmac->chan;  in axi_dmac_detect_caps()
999 		chan->hw_cyclic = true;  in axi_dmac_detect_caps()
1003 		chan->hw_sg = true;  in axi_dmac_detect_caps()
1007 		chan->hw_2d = true;  in axi_dmac_detect_caps()
1010 	chan->max_length = axi_dmac_read(dmac, AXI_DMAC_REG_X_LENGTH);  in axi_dmac_detect_caps()
1011 	if (chan->max_length != UINT_MAX)  in axi_dmac_detect_caps()
1012 		chan->max_length++;  in axi_dmac_detect_caps()
1016 	    chan->dest_type == AXI_DMAC_BUS_TYPE_AXI_MM) {  in axi_dmac_detect_caps()
1017 		dev_err(dmac->dma_dev.dev,  in axi_dmac_detect_caps()
1018 			"Destination memory-mapped interface not supported.");  in axi_dmac_detect_caps()
1019 		return -ENODEV;  in axi_dmac_detect_caps()
1024 	    chan->src_type == AXI_DMAC_BUS_TYPE_AXI_MM) {  in axi_dmac_detect_caps()
1025 		dev_err(dmac->dma_dev.dev,  in axi_dmac_detect_caps()
1026 			"Source memory-mapped interface not supported.");  in axi_dmac_detect_caps()
1027 		return -ENODEV;  in axi_dmac_detect_caps()
1031 		chan->hw_partial_xfer = true;  in axi_dmac_detect_caps()
1035 		chan->length_align_mask =  in axi_dmac_detect_caps()
1038 		chan->length_align_mask = chan->address_align_mask;  in axi_dmac_detect_caps()
1063 	dmac = devm_kzalloc(&pdev->dev, sizeof(*dmac), GFP_KERNEL);  in axi_dmac_probe()
1065 		return -ENOMEM;  in axi_dmac_probe()
1067 	dmac->irq = platform_get_irq(pdev, 0);  in axi_dmac_probe()
1068 	if (dmac->irq < 0)  in axi_dmac_probe()
1069 		return dmac->irq;  in axi_dmac_probe()
1070 	if (dmac->irq == 0)  in axi_dmac_probe()
1071 		return -EINVAL;  in axi_dmac_probe()
1073 	dmac->base = devm_platform_ioremap_resource(pdev, 0);  in axi_dmac_probe()
1074 	if (IS_ERR(dmac->base))  in axi_dmac_probe()
1075 		return PTR_ERR(dmac->base);  in axi_dmac_probe()
1077 	dmac->clk = devm_clk_get_enabled(&pdev->dev, NULL);  in axi_dmac_probe()
1078 	if (IS_ERR(dmac->clk))  in axi_dmac_probe()
1079 		return PTR_ERR(dmac->clk);  in axi_dmac_probe()
1084 		ret = axi_dmac_read_chan_config(&pdev->dev, dmac);  in axi_dmac_probe()
1086 		ret = axi_dmac_parse_dt(&pdev->dev, dmac);  in axi_dmac_probe()
1091 	INIT_LIST_HEAD(&dmac->chan.active_descs);  in axi_dmac_probe()
1093 	dma_set_max_seg_size(&pdev->dev, UINT_MAX);  in axi_dmac_probe()
1095 	dma_dev = &dmac->dma_dev;  in axi_dmac_probe()
1096 	dma_cap_set(DMA_SLAVE, dma_dev->cap_mask);  in axi_dmac_probe()
1097 	dma_cap_set(DMA_CYCLIC, dma_dev->cap_mask);  in axi_dmac_probe()
1098 	dma_cap_set(DMA_INTERLEAVE, dma_dev->cap_mask);  in axi_dmac_probe()
1099 	dma_dev->device_free_chan_resources = axi_dmac_free_chan_resources;  in axi_dmac_probe()
1100 	dma_dev->device_tx_status = dma_cookie_status;  in axi_dmac_probe()
1101 	dma_dev->device_issue_pending = axi_dmac_issue_pending;  in axi_dmac_probe()
1102 	dma_dev->device_prep_slave_sg = axi_dmac_prep_slave_sg;  in axi_dmac_probe()
1103 	dma_dev->device_prep_peripheral_dma_vec = axi_dmac_prep_peripheral_dma_vec;  in axi_dmac_probe()
1104 	dma_dev->device_prep_dma_cyclic = axi_dmac_prep_dma_cyclic;  in axi_dmac_probe()
1105 	dma_dev->device_prep_interleaved_dma = axi_dmac_prep_interleaved;  in axi_dmac_probe()
1106 	dma_dev->device_terminate_all = axi_dmac_terminate_all;  in axi_dmac_probe()
1107 	dma_dev->device_synchronize = axi_dmac_synchronize;  in axi_dmac_probe()
1108 	dma_dev->dev = &pdev->dev;  in axi_dmac_probe()
1109 	dma_dev->src_addr_widths = BIT(dmac->chan.src_width);  in axi_dmac_probe()
1110 	dma_dev->dst_addr_widths = BIT(dmac->chan.dest_width);  in axi_dmac_probe()
1111 	dma_dev->directions = BIT(dmac->chan.direction);  in axi_dmac_probe()
1112 	dma_dev->residue_granularity = DMA_RESIDUE_GRANULARITY_DESCRIPTOR;  in axi_dmac_probe()
1113 	dma_dev->max_sg_burst = 31; /* 31 SGs maximum in one burst */  in axi_dmac_probe()
1114 	INIT_LIST_HEAD(&dma_dev->channels);  in axi_dmac_probe()
1116 	dmac->chan.vchan.desc_free = axi_dmac_desc_free;  in axi_dmac_probe()
1117 	vchan_init(&dmac->chan.vchan, dma_dev);  in axi_dmac_probe()
1123 	dma_dev->copy_align = (dmac->chan.address_align_mask + 1);  in axi_dmac_probe()
1125 	if (dmac->chan.hw_sg)  in axi_dmac_probe()
1130 	if (of_dma_is_coherent(pdev->dev.of_node)) {  in axi_dmac_probe()
1135 			dev_err(dmac->dma_dev.dev,  in axi_dmac_probe()
1137 			return -EINVAL;  in axi_dmac_probe()
1149 	ret = devm_add_action_or_reset(&pdev->dev, axi_dmac_tasklet_kill,  in axi_dmac_probe()
1150 				       &dmac->chan.vchan.task);  in axi_dmac_probe()
1154 	ret = of_dma_controller_register(pdev->dev.of_node,  in axi_dmac_probe()
1159 	ret = devm_add_action_or_reset(&pdev->dev, axi_dmac_free_dma_controller,  in axi_dmac_probe()
1160 				       pdev->dev.of_node);  in axi_dmac_probe()
1164 	ret = devm_request_irq(&pdev->dev, dmac->irq, axi_dmac_interrupt_handler,  in axi_dmac_probe()
1165 			       IRQF_SHARED, dev_name(&pdev->dev), dmac);  in axi_dmac_probe()
1169 	regmap = devm_regmap_init_mmio(&pdev->dev, dmac->base,  in axi_dmac_probe()
1176 	{ .compatible = "adi,axi-dmac-1.00.a" },
1183 		.name = "dma-axi-dmac",
1190 MODULE_AUTHOR("Lars-Peter Clausen <lars@metafoo.de>");
1191 MODULE_DESCRIPTION("DMA controller driver for the AXI-DMAC controller");