1 // SPDX-License-Identifier: GPL-2.0-or-later
3  * Copyright(c) 2004 - 2006 Intel Corporation. All rights reserved.
7  * This code implements the DMA subsystem. It provides a HW-neutral interface
8  * for other kernel code to use asynchronous memory copy capabilities,
9  * if present, and allows different HW DMA drivers to register as providing
12  * Due to the fact we are accelerating what is already a relatively fast
13  * operation, the code goes to great lengths to avoid additional overhead,
21  * A subsystem can get access to a channel by calling dmaengine_get() followed
22  * by dma_find_channel(), or if it has need for an exclusive channel it can call
23  * dma_request_channel().  Once a channel is allocated a reference is taken
24  * against its corresponding driver to disable removal.
29  * See Documentation/driver-api/dmaengine for more details
35 #include <linux/dma-mapping.h>
63 /* --- debugfs implementation --- */
71 	dma_dev->dbg_dev_root = debugfs_create_dir(dev_name(dma_dev->dev),  in dmaengine_debug_register()
73 	if (IS_ERR(dma_dev->dbg_dev_root))  in dmaengine_debug_register()
74 		dma_dev->dbg_dev_root = NULL;  in dmaengine_debug_register()
79 	debugfs_remove_recursive(dma_dev->dbg_dev_root);  in dmaengine_debug_unregister()
80 	dma_dev->dbg_dev_root = NULL;  in dmaengine_debug_unregister()
88 	list_for_each_entry(chan, &dma_dev->channels, device_node) {  in dmaengine_dbg_summary_show()
89 		if (chan->client_count) {  in dmaengine_dbg_summary_show()
90 			seq_printf(s, " %-13s| %s", dma_chan_name(chan),  in dmaengine_dbg_summary_show()
91 				   chan->dbg_client_name ?: "in-use");  in dmaengine_dbg_summary_show()
93 			if (chan->router)  in dmaengine_dbg_summary_show()
95 					dev_name(chan->router->dev));  in dmaengine_dbg_summary_show()
108 		seq_printf(s, "dma%d (%s): number of channels: %u\n",  in dmaengine_summary_show()
109 			   dma_dev->dev_id, dev_name(dma_dev->dev),  in dmaengine_summary_show()
110 			   dma_dev->chancnt);  in dmaengine_summary_show()
112 		if (dma_dev->dbg_summary_show)  in dmaengine_summary_show()
113 			dma_dev->dbg_summary_show(s, dma_dev);  in dmaengine_summary_show()
117 		if (!list_is_last(&dma_dev->global_node, &dma_device_list))  in dmaengine_summary_show()
144 /* --- sysfs implementation --- */
149  * dev_to_dma_chan - convert a device pointer to its sysfs container object
159 	return chan_dev->chan;  in dev_to_dma_chan()
174 			count += per_cpu_ptr(chan->local, i)->memcpy_count;  in memcpy_count_show()
177 		err = -ENODEV;  in memcpy_count_show()
196 			count += per_cpu_ptr(chan->local, i)->bytes_transferred;  in bytes_transferred_show()
199 		err = -ENODEV;  in bytes_transferred_show()
215 		err = sysfs_emit(buf, "%d\n", chan->client_count);  in in_use_show()
217 		err = -ENODEV;  in in_use_show()
241 	.name		= "dma",
246 /* --- client and device registration --- */
252  * struct dma_chan_tbl_ent - tracks channel allocations per core/operation
253  * @chan:	associated channel for this entry
259 /* percpu lookup table for memory-to-memory offload providers */
269 	/* 'interrupt', 'private', and 'slave' are channel capabilities,  in dma_channel_table_init()
280 			err = -ENOMEM;  in dma_channel_table_init()
296  * dma_chan_is_local - checks if the channel is in the same NUMA-node as the CPU
297  * @chan:	DMA channel to test
298  * @cpu:	CPU index which the channel should be close to
300  * Returns true if the channel is in the same NUMA-node as the CPU.
304 	int node = dev_to_node(chan->device->dev);  in dma_chan_is_local()
310  * min_chan - finds the channel with min count and in the same NUMA-node as the CPU
311  * @cap:	capability to match
312  * @cpu:	CPU index which the channel should be close to
314  * If some channels are close to the given CPU, the one with the lowest
328 		if (!dma_has_cap(cap, device->cap_mask) ||  in min_chan()
329 		    dma_has_cap(DMA_PRIVATE, device->cap_mask))  in min_chan()
331 		list_for_each_entry(chan, &device->channels, device_node) {  in min_chan()
332 			if (!chan->client_count)  in min_chan()
334 			if (!min || chan->table_count < min->table_count)  in min_chan()
339 				    chan->table_count < localmin->table_count)  in min_chan()
347 		chan->table_count++;  in min_chan()
353  * dma_channel_rebalance - redistribute the available channels
355  * Optimize for CPU isolation (each CPU gets a dedicated channel for an
357  * multi-tasking channels) in the non-SMP case.
371 			per_cpu_ptr(channel_table[cap], cpu)->chan = NULL;  in dma_channel_rebalance()
374 		if (dma_has_cap(DMA_PRIVATE, device->cap_mask))  in dma_channel_rebalance()
376 		list_for_each_entry(chan, &device->channels, device_node)  in dma_channel_rebalance()
377 			chan->table_count = 0;  in dma_channel_rebalance()
388 			per_cpu_ptr(channel_table[cap], cpu)->chan = chan;  in dma_channel_rebalance()
397 	bitmap_and(has.bits, want->bits, device->cap_mask.bits,  in dma_device_satisfies_mask()
399 	return bitmap_equal(want->bits, has.bits, DMA_TX_TYPE_END);  in dma_device_satisfies_mask()
404 	return chan->device->owner;  in dma_chan_to_owner()
408  * balance_ref_count - catch up the channel reference count
409  * @chan:	channel to balance ->client_count versus dmaengine_ref_count
417 	while (chan->client_count < dmaengine_ref_count) {  in balance_ref_count()
419 		chan->client_count++;  in balance_ref_count()
427 	list_del_rcu(&device->global_node);  in dma_device_release()
430 	if (device->device_release)  in dma_device_release()
431 		device->device_release(device);  in dma_device_release()
437 	kref_put(&device->ref, dma_device_release);  in dma_device_put()
441  * dma_chan_get - try to grab a DMA channel's parent driver module
442  * @chan:	channel to grab
451 	/* The channel is already in use, update client count */  in dma_chan_get()
452 	if (chan->client_count) {  in dma_chan_get()
454 		chan->client_count++;  in dma_chan_get()
459 		return -ENODEV;  in dma_chan_get()
461 	ret = kref_get_unless_zero(&chan->device->ref);  in dma_chan_get()
463 		ret = -ENODEV;  in dma_chan_get()
468 	if (chan->device->device_alloc_chan_resources) {  in dma_chan_get()
469 		ret = chan->device->device_alloc_chan_resources(chan);  in dma_chan_get()
474 	chan->client_count++;  in dma_chan_get()
476 	if (!dma_has_cap(DMA_PRIVATE, chan->device->cap_mask))  in dma_chan_get()
482 	dma_device_put(chan->device);  in dma_chan_get()
489  * dma_chan_put - drop a reference to a DMA channel's parent driver module
490  * @chan:	channel to release
496 	/* This channel is not in use, bail out */  in dma_chan_put()
497 	if (!chan->client_count)  in dma_chan_put()
500 	chan->client_count--;  in dma_chan_put()
502 	/* This channel is not in use anymore, free it */  in dma_chan_put()
503 	if (!chan->client_count && chan->device->device_free_chan_resources) {  in dma_chan_put()
506 		chan->device->device_free_chan_resources(chan);  in dma_chan_put()
509 	/* If the channel is used via a DMA request router, free the mapping */  in dma_chan_put()
510 	if (chan->router && chan->router->route_free) {  in dma_chan_put()
511 		chan->router->route_free(chan->router->dev, chan->route_data);  in dma_chan_put()
512 		chan->router = NULL;  in dma_chan_put()
513 		chan->route_data = NULL;  in dma_chan_put()
516 	dma_device_put(chan->device);  in dma_chan_put()
529 			dev_err(chan->device->dev, "%s: timeout!\n", __func__);  in dma_sync_wait()
542  * dma_find_channel - find a channel to carry out the operation
547 	return this_cpu_read(channel_table[tx_type]->chan);  in dma_find_channel()
552  * dma_issue_pending_all - flush all pending operations across all channels
561 		if (dma_has_cap(DMA_PRIVATE, device->cap_mask))  in dma_issue_pending_all()
563 		list_for_each_entry(chan, &device->channels, device_node)  in dma_issue_pending_all()
564 			if (chan->client_count)  in dma_issue_pending_all()
565 				device->device_issue_pending(chan);  in dma_issue_pending_all()
576 		return -EINVAL;  in dma_get_slave_caps()
578 	device = chan->device;  in dma_get_slave_caps()
580 	/* check if the channel supports slave transactions */  in dma_get_slave_caps()
581 	if (!(test_bit(DMA_SLAVE, device->cap_mask.bits) ||  in dma_get_slave_caps()
582 	      test_bit(DMA_CYCLIC, device->cap_mask.bits)))  in dma_get_slave_caps()
583 		return -ENXIO;  in dma_get_slave_caps()
590 	if (!device->directions)  in dma_get_slave_caps()
591 		return -ENXIO;  in dma_get_slave_caps()
593 	caps->src_addr_widths = device->src_addr_widths;  in dma_get_slave_caps()
594 	caps->dst_addr_widths = device->dst_addr_widths;  in dma_get_slave_caps()
595 	caps->directions = device->directions;  in dma_get_slave_caps()
596 	caps->min_burst = device->min_burst;  in dma_get_slave_caps()
597 	caps->max_burst = device->max_burst;  in dma_get_slave_caps()
598 	caps->max_sg_burst = device->max_sg_burst;  in dma_get_slave_caps()
599 	caps->residue_granularity = device->residue_granularity;  in dma_get_slave_caps()
600 	caps->descriptor_reuse = device->descriptor_reuse;  in dma_get_slave_caps()
601 	caps->cmd_pause = !!device->device_pause;  in dma_get_slave_caps()
602 	caps->cmd_resume = !!device->device_resume;  in dma_get_slave_caps()
603 	caps->cmd_terminate = !!device->device_terminate_all;  in dma_get_slave_caps()
606 	 * DMA engine device might be configured with non-uniformly  in dma_get_slave_caps()
609 	 * callback to override the generic capabilities with  in dma_get_slave_caps()
610 	 * channel-specific ones.  in dma_get_slave_caps()
612 	if (device->device_caps)  in dma_get_slave_caps()
613 		device->device_caps(chan, caps);  in dma_get_slave_caps()
626 		dev_dbg(dev->dev, "%s: wrong capabilities\n", __func__);  in private_candidate()
629 	/* devices with multiple channels need special handling as we need to  in private_candidate()
632 	if (dev->chancnt > 1 && !dma_has_cap(DMA_PRIVATE, dev->cap_mask))  in private_candidate()
633 		list_for_each_entry(chan, &dev->channels, device_node) {  in private_candidate()
635 			if (chan->client_count)  in private_candidate()
639 	list_for_each_entry(chan, &dev->channels, device_node) {  in private_candidate()
640 		if (chan->client_count) {  in private_candidate()
641 			dev_dbg(dev->dev, "%s: %s busy\n",  in private_candidate()
646 			dev_dbg(dev->dev, "%s: %s filter said false\n",  in private_candidate()
664 		/* Found a suitable channel, try to grab, prep, and return it.  in find_candidate()
665 		 * We first set DMA_PRIVATE to disable balance_ref_count as this  in find_candidate()
666 		 * channel will not be published in the general-purpose  in find_candidate()
669 		dma_cap_set(DMA_PRIVATE, device->cap_mask);  in find_candidate()
670 		device->privatecnt++;  in find_candidate()
674 			if (err == -ENODEV) {  in find_candidate()
675 				dev_dbg(device->dev, "%s: %s module removed\n",  in find_candidate()
677 				list_del_rcu(&device->global_node);  in find_candidate()
679 				dev_dbg(device->dev,  in find_candidate()
680 					"%s: failed to get %s: (%d)\n",  in find_candidate()
683 			if (--device->privatecnt == 0)  in find_candidate()
684 				dma_cap_clear(DMA_PRIVATE, device->cap_mask);  in find_candidate()
690 	return chan ? chan : ERR_PTR(-EPROBE_DEFER);  in find_candidate()
694  * dma_get_slave_channel - try to get specific channel exclusively
695  * @chan:	target channel
702 	if (chan->client_count == 0) {  in dma_get_slave_channel()
703 		struct dma_device *device = chan->device;  in dma_get_slave_channel()
706 		dma_cap_set(DMA_PRIVATE, device->cap_mask);  in dma_get_slave_channel()
707 		device->privatecnt++;  in dma_get_slave_channel()
710 			dev_dbg(chan->device->dev,  in dma_get_slave_channel()
711 				"%s: failed to get %s: (%d)\n",  in dma_get_slave_channel()
714 			if (--device->privatecnt == 0)  in dma_get_slave_channel()
715 				dma_cap_clear(DMA_PRIVATE, device->cap_mask);  in dma_get_slave_channel()
747  * __dma_request_channel - try to allocate an exclusive channel
748  * @mask:	capabilities that the channel must satisfy
749  * @fn:		optional callback to disposition available channels
750  * @fn_param:	opaque parameter to pass to dma_filter_fn()
751  * @np:		device node to look for DMA channels
753  * Returns pointer to appropriate DMA channel on success or NULL.
762 	/* Find a channel */  in __dma_request_channel()
765 		/* Finds a DMA controller with matching device node */  in __dma_request_channel()
766 		if (np && device->dev->of_node && np != device->dev->of_node)  in __dma_request_channel()
792 	if (!device->filter.mapcnt)  in dma_filter_match()
795 	for (i = 0; i < device->filter.mapcnt; i++) {  in dma_filter_match()
796 		const struct dma_slave_map *map = &device->filter.map[i];  in dma_filter_match()  local
798 		if (!strcmp(map->devname, dev_name(dev)) &&  in dma_filter_match()
799 		    !strcmp(map->slave, name))  in dma_filter_match()
800 			return map;  in dma_filter_match()
807  * dma_request_chan - try to allocate an exclusive slave channel
808  * @dev:	pointer to client device structure
809  * @name:	slave channel name
811  * Returns pointer to appropriate DMA channel on success or an error pointer.
818 	/* If device-tree is present get slave info from here */  in dma_request_chan()
819 	if (dev->of_node)  in dma_request_chan()
820 		chan = of_dma_request_slave_channel(dev->of_node, name);  in dma_request_chan()
826 	if (PTR_ERR(chan) == -EPROBE_DEFER)  in dma_request_chan()
832 	/* Try to find the channel via the DMA filter map(s) */  in dma_request_chan()
836 		const struct dma_slave_map *map = dma_filter_match(d, name, dev);  in dma_request_chan()  local
838 		if (!map)  in dma_request_chan()
844 		chan = find_candidate(d, &mask, d->filter.fn, map->param);  in dma_request_chan()
853 		return ERR_PTR(-EPROBE_DEFER);  in dma_request_chan()
857 	chan->dbg_client_name = kasprintf(GFP_KERNEL, "%s:%s", dev_name(dev),  in dma_request_chan()
861 	chan->name = kasprintf(GFP_KERNEL, "dma:%s", name);  in dma_request_chan()
862 	if (!chan->name)  in dma_request_chan()
864 	chan->slave = dev;  in dma_request_chan()
866 	if (sysfs_create_link(&chan->dev->device.kobj, &dev->kobj,  in dma_request_chan()
868 		dev_warn(dev, "Cannot create DMA %s symlink\n", DMA_SLAVE_NAME);  in dma_request_chan()
869 	if (sysfs_create_link(&dev->kobj, &chan->dev->device.kobj, chan->name))  in dma_request_chan()
870 		dev_warn(dev, "Cannot create DMA %s symlink\n", chan->name);  in dma_request_chan()
877  * dma_request_chan_by_mask - allocate a channel satisfying certain capabilities
878  * @mask:	capabilities that the channel must satisfy
880  * Returns pointer to appropriate DMA channel on success or an error pointer.
887 		return ERR_PTR(-ENODEV);  in dma_request_chan_by_mask()
893 			chan = ERR_PTR(-EPROBE_DEFER);  in dma_request_chan_by_mask()
895 			chan = ERR_PTR(-ENODEV);  in dma_request_chan_by_mask()
906 	WARN_ONCE(chan->client_count != 1,  in dma_release_channel()
907 		  "chan reference count %d != 1\n", chan->client_count);  in dma_release_channel()
910 	if (--chan->device->privatecnt == 0)  in dma_release_channel()
911 		dma_cap_clear(DMA_PRIVATE, chan->device->cap_mask);  in dma_release_channel()
913 	if (chan->slave) {  in dma_release_channel()
914 		sysfs_remove_link(&chan->dev->device.kobj, DMA_SLAVE_NAME);  in dma_release_channel()
915 		sysfs_remove_link(&chan->slave->kobj, chan->name);  in dma_release_channel()
916 		kfree(chan->name);  in dma_release_channel()
917 		chan->name = NULL;  in dma_release_channel()
918 		chan->slave = NULL;  in dma_release_channel()
922 	kfree(chan->dbg_client_name);  in dma_release_channel()
923 	chan->dbg_client_name = NULL;  in dma_release_channel()
930  * dmaengine_get - register interest in dma_channels
941 	/* try to grab channels */  in dmaengine_get()
943 		if (dma_has_cap(DMA_PRIVATE, device->cap_mask))  in dmaengine_get()
945 		list_for_each_entry(chan, &device->channels, device_node) {  in dmaengine_get()
947 			if (err == -ENODEV) {  in dmaengine_get()
949 				list_del_rcu(&device->global_node);  in dmaengine_get()
952 				dev_dbg(chan->device->dev,  in dmaengine_get()
953 					"%s: failed to get %s: (%d)\n",  in dmaengine_get()
959 	 * waiting we need to rebalance to get those channels  in dmaengine_get()
960 	 * incorporated into the channel table  in dmaengine_get()
969  * dmaengine_put - let DMA drivers be removed when ref_count == 0
977 	dmaengine_ref_count--;  in dmaengine_put()
979 	/* drop channel references */  in dmaengine_put()
981 		if (dma_has_cap(DMA_PRIVATE, device->cap_mask))  in dmaengine_put()
983 		list_for_each_entry(chan, &device->channels, device_node)  in dmaengine_put()
993 	 * an async_tx channel switch event as all possible operation types can  in device_has_all_tx_types()
997 	if (!dma_has_cap(DMA_INTERRUPT, device->cap_mask))  in device_has_all_tx_types()
1002 	if (!dma_has_cap(DMA_MEMCPY, device->cap_mask))  in device_has_all_tx_types()
1007 	if (!dma_has_cap(DMA_XOR, device->cap_mask))  in device_has_all_tx_types()
1011 	if (!dma_has_cap(DMA_XOR_VAL, device->cap_mask))  in device_has_all_tx_types()
1017 	if (!dma_has_cap(DMA_PQ, device->cap_mask))  in device_has_all_tx_types()
1021 	if (!dma_has_cap(DMA_PQ_VAL, device->cap_mask))  in device_has_all_tx_types()
1035 	device->dev_id = rc;  in get_dma_id()
1045 	chan->local = alloc_percpu(typeof(*chan->local));  in __dma_async_device_channel_register()
1046 	if (!chan->local)  in __dma_async_device_channel_register()
1047 		return -ENOMEM;  in __dma_async_device_channel_register()
1048 	chan->dev = kzalloc(sizeof(*chan->dev), GFP_KERNEL);  in __dma_async_device_channel_register()
1049 	if (!chan->dev) {  in __dma_async_device_channel_register()
1050 		rc = -ENOMEM;  in __dma_async_device_channel_register()
1056 	 * the channel. Otherwise we are static enumerating.  in __dma_async_device_channel_register()
1058 	chan->chan_id = ida_alloc(&device->chan_ida, GFP_KERNEL);  in __dma_async_device_channel_register()
1059 	if (chan->chan_id < 0) {  in __dma_async_device_channel_register()
1060 		pr_err("%s: unable to alloc ida for chan: %d\n",  in __dma_async_device_channel_register()
1061 		       __func__, chan->chan_id);  in __dma_async_device_channel_register()
1062 		rc = chan->chan_id;  in __dma_async_device_channel_register()
1066 	chan->dev->device.class = &dma_devclass;  in __dma_async_device_channel_register()
1067 	chan->dev->device.parent = device->dev;  in __dma_async_device_channel_register()
1068 	chan->dev->chan = chan;  in __dma_async_device_channel_register()
1069 	chan->dev->dev_id = device->dev_id;  in __dma_async_device_channel_register()
1071 		dev_set_name(&chan->dev->device, "dma%dchan%d", device->dev_id, chan->chan_id);  in __dma_async_device_channel_register()
1073 		dev_set_name(&chan->dev->device, "%s", name);  in __dma_async_device_channel_register()
1074 	rc = device_register(&chan->dev->device);  in __dma_async_device_channel_register()
1077 	chan->client_count = 0;  in __dma_async_device_channel_register()
1078 	device->chancnt++;  in __dma_async_device_channel_register()
1083 	ida_free(&device->chan_ida, chan->chan_id);  in __dma_async_device_channel_register()
1085 	kfree(chan->dev);  in __dma_async_device_channel_register()
1087 	free_percpu(chan->local);  in __dma_async_device_channel_register()
1088 	chan->local = NULL;  in __dma_async_device_channel_register()
1110 	if (chan->local == NULL)  in __dma_async_device_channel_unregister()
1113 	WARN_ONCE(!device->device_release && chan->client_count,  in __dma_async_device_channel_unregister()
1115 		  __func__, chan->client_count);  in __dma_async_device_channel_unregister()
1117 	device->chancnt--;  in __dma_async_device_channel_unregister()
1118 	chan->dev->chan = NULL;  in __dma_async_device_channel_unregister()
1120 	ida_free(&device->chan_ida, chan->chan_id);  in __dma_async_device_channel_unregister()
1121 	device_unregister(&chan->dev->device);  in __dma_async_device_channel_unregister()
1122 	free_percpu(chan->local);  in __dma_async_device_channel_unregister()
1134  * dma_async_device_register - registers DMA devices found
1135  * @device:	pointer to &struct dma_device
1147 		return -ENODEV;  in dma_async_device_register()
1150 	if (!device->dev) {  in dma_async_device_register()
1152 		return -EIO;  in dma_async_device_register()
1155 	device->owner = device->dev->driver->owner;  in dma_async_device_register()
1159 	if (dma_has_cap(_type, device->cap_mask) && !device->device_prep_##_name) {	\  in dma_async_device_register()
1160 		dev_err(device->dev,							\  in dma_async_device_register()
1163 		return -EIO;								\  in dma_async_device_register()
1179 	if (!device->device_tx_status) {  in dma_async_device_register()
1180 		dev_err(device->dev, "Device tx_status is not defined\n");  in dma_async_device_register()
1181 		return -EIO;  in dma_async_device_register()
1185 	if (!device->device_issue_pending) {  in dma_async_device_register()
1186 		dev_err(device->dev, "Device issue_pending is not defined\n");  in dma_async_device_register()
1187 		return -EIO;  in dma_async_device_register()
1190 	if (!device->device_release)  in dma_async_device_register()
1191 		dev_dbg(device->dev,  in dma_async_device_register()
1192 			 "WARN: Device release is not defined so it is not safe to unbind this driver while in use\n");  in dma_async_device_register()
1194 	kref_init(&device->ref);  in dma_async_device_register()
1200 		dma_cap_set(DMA_ASYNC_TX, device->cap_mask);  in dma_async_device_register()
1206 	ida_init(&device->chan_ida);  in dma_async_device_register()
1209 	list_for_each_entry(chan, &device->channels, device_node) {  in dma_async_device_register()
1217 	if (dmaengine_ref_count && !dma_has_cap(DMA_PRIVATE, device->cap_mask))  in dma_async_device_register()
1218 		list_for_each_entry(chan, &device->channels, device_node) {  in dma_async_device_register()
1220 			 * to take references on their behalf  in dma_async_device_register()
1222 			if (dma_chan_get(chan) == -ENODEV) {  in dma_async_device_register()
1224 				 * channel as the remaining channels are  in dma_async_device_register()
1225 				 * guaranteed to get a reference  in dma_async_device_register()
1227 				rc = -ENODEV;  in dma_async_device_register()
1232 	list_add_tail_rcu(&device->global_node, &dma_device_list);  in dma_async_device_register()
1233 	if (dma_has_cap(DMA_PRIVATE, device->cap_mask))  in dma_async_device_register()
1234 		device->privatecnt++;	/* Always private */  in dma_async_device_register()
1243 	/* if we never registered a channel just release the idr */  in dma_async_device_register()
1244 	if (!device->chancnt) {  in dma_async_device_register()
1245 		ida_free(&dma_ida, device->dev_id);  in dma_async_device_register()
1249 	list_for_each_entry(chan, &device->channels, device_node) {  in dma_async_device_register()
1250 		if (chan->local == NULL)  in dma_async_device_register()
1253 		chan->dev->chan = NULL;  in dma_async_device_register()
1255 		device_unregister(&chan->dev->device);  in dma_async_device_register()
1256 		free_percpu(chan->local);  in dma_async_device_register()
1263  * dma_async_device_unregister - unregister a DMA device
1264  * @device:	pointer to &struct dma_device
1266  * This routine is called by dma driver exit routines, dmaengine holds module
1267  * references to prevent it being called while channels are in use.
1275 	list_for_each_entry_safe(chan, n, &device->channels, device_node)  in dma_async_device_unregister()
1283 	dma_cap_set(DMA_PRIVATE, device->cap_mask);  in dma_async_device_unregister()
1285 	ida_free(&dma_ida, device->dev_id);  in dma_async_device_unregister()
1297  * dmaenginem_async_device_register - registers DMA devices found
1298  * @device:	pointer to &struct dma_device
1310 	return devm_add_action_or_reset(device->dev, dmaenginem_async_device_unregister, device);  in dmaenginem_async_device_register()
1321 #define __UNMAP_POOL(x) { .size = x, .name = "dmaengine-unmap-" __stringify(x) }
1355 	struct device *dev = unmap->dev;  in dmaengine_unmap()
1358 	cnt = unmap->to_cnt;  in dmaengine_unmap()
1360 		dma_unmap_page(dev, unmap->addr[i], unmap->len,  in dmaengine_unmap()
1362 	cnt += unmap->from_cnt;  in dmaengine_unmap()
1364 		dma_unmap_page(dev, unmap->addr[i], unmap->len,  in dmaengine_unmap()
1366 	cnt += unmap->bidi_cnt;  in dmaengine_unmap()
1368 		if (unmap->addr[i] == 0)  in dmaengine_unmap()
1370 		dma_unmap_page(dev, unmap->addr[i], unmap->len,  in dmaengine_unmap()
1373 	cnt = unmap->map_cnt;  in dmaengine_unmap()
1374 	mempool_free(unmap, __get_unmap_pool(cnt)->pool);  in dmaengine_unmap()
1380 		kref_put(&unmap->kref, dmaengine_unmap);  in dmaengine_unmap_put()
1391 		mempool_destroy(p->pool);  in dmaengine_destroy_unmap_pool()
1392 		p->pool = NULL;  in dmaengine_destroy_unmap_pool()
1393 		kmem_cache_destroy(p->cache);  in dmaengine_destroy_unmap_pool()
1394 		p->cache = NULL;  in dmaengine_destroy_unmap_pool()
1407 		       sizeof(dma_addr_t) * p->size;  in dmaengine_init_unmap_pool()
1409 		p->cache = kmem_cache_create(p->name, size, 0,  in dmaengine_init_unmap_pool()
1411 		if (!p->cache)  in dmaengine_init_unmap_pool()
1413 		p->pool = mempool_create_slab_pool(1, p->cache);  in dmaengine_init_unmap_pool()
1414 		if (!p->pool)  in dmaengine_init_unmap_pool()
1422 	return -ENOMEM;  in dmaengine_init_unmap_pool()
1430 	unmap = mempool_alloc(__get_unmap_pool(nr)->pool, flags);  in dmaengine_get_unmap_data()
1435 	kref_init(&unmap->kref);  in dmaengine_get_unmap_data()
1436 	unmap->dev = dev;  in dmaengine_get_unmap_data()
1437 	unmap->map_cnt = nr;  in dmaengine_get_unmap_data()
1446 	tx->chan = chan;  in dma_async_tx_descriptor_init()
1448 	spin_lock_init(&tx->lock);  in dma_async_tx_descriptor_init()
1457 	if (!desc->desc_metadata_mode) {  in desc_check_and_set_metadata_mode()
1458 		if (dmaengine_is_metadata_mode_supported(desc->chan, mode))  in desc_check_and_set_metadata_mode()
1459 			desc->desc_metadata_mode = mode;  in desc_check_and_set_metadata_mode()
1461 			return -ENOTSUPP;  in desc_check_and_set_metadata_mode()
1462 	} else if (desc->desc_metadata_mode != mode) {  in desc_check_and_set_metadata_mode()
1463 		return -EINVAL;  in desc_check_and_set_metadata_mode()
1475 		return -EINVAL;  in dmaengine_desc_attach_metadata()
1481 	if (!desc->metadata_ops || !desc->metadata_ops->attach)  in dmaengine_desc_attach_metadata()
1482 		return -ENOTSUPP;  in dmaengine_desc_attach_metadata()
1484 	return desc->metadata_ops->attach(desc, data, len);  in dmaengine_desc_attach_metadata()
1494 		return ERR_PTR(-EINVAL);  in dmaengine_desc_get_metadata_ptr()
1500 	if (!desc->metadata_ops || !desc->metadata_ops->get_ptr)  in dmaengine_desc_get_metadata_ptr()
1501 		return ERR_PTR(-ENOTSUPP);  in dmaengine_desc_get_metadata_ptr()
1503 	return desc->metadata_ops->get_ptr(desc, payload_len, max_len);  in dmaengine_desc_get_metadata_ptr()
1513 		return -EINVAL;  in dmaengine_desc_set_metadata_len()
1519 	if (!desc->metadata_ops || !desc->metadata_ops->set_len)  in dmaengine_desc_set_metadata_len()
1520 		return -ENOTSUPP;  in dmaengine_desc_set_metadata_len()
1522 	return desc->metadata_ops->set_len(desc, payload_len);  in dmaengine_desc_set_metadata_len()
1527  * dma_wait_for_async_tx - spin wait for a transaction to complete
1528  * @tx:		in-flight transaction to wait on
1538 	while (tx->cookie == -EBUSY) {  in dma_wait_for_async_tx()
1540 			dev_err(tx->chan->device->dev,  in dma_wait_for_async_tx()
1547 	return dma_sync_wait(tx->chan, tx->cookie);  in dma_wait_for_async_tx()
1552  * dma_run_dependencies - process dependent operations on the target channel
1555  * Helper routine for DMA drivers to process (start) dependent operations
1556  * on their target channel.
1567 	/* we'll submit tx->next now, so clear the link */  in dma_run_dependencies()
1569 	chan = dep->chan;  in dma_run_dependencies()
1571 	/* keep submitting up until a channel switch is detected  in dma_run_dependencies()
1579 		if (dep_next && dep_next->chan == chan)  in dma_run_dependencies()
1580 			txd_clear_next(dep); /* ->next will be submitted */  in dma_run_dependencies()
1585 		dep->tx_submit(dep);  in dma_run_dependencies()
1588 	chan->device->device_issue_pending(chan);  in dma_run_dependencies()