1 /* SPDX-License-Identifier: GPL-2.0-or-later */
3  * Copyright(c) 2004 - 2006 Intel Corporation. All rights reserved.
18  * typedef dma_cookie_t - an opaque DMA cookie
20  * if dma_cookie_t is >0 it's a DMA request cookie, <0 it's an error code
31  * enum dma_status - DMA transaction status
46  * enum dma_transaction_type - DMA transaction types/indexes
49  * automatically set as dma devices are registered.
68 /* last transaction type for creation of the capabilities mask */
73  * enum dma_transfer_direction - dma transfer mode and direction indicator
89  * ----------------------------
91  * The gap(in bytes) between two chunks is called inter-chunk-gap(ICG).
96  *  it is to be repeated and other per-transfer attributes.
103  *  |      Frame-1        |       Frame-2       | ~ |       Frame-'numf'  |
111  * struct data_chunk - Element of scatter-gather list that makes a frame.
133  * struct dma_interleaved_template - Template to convey DMAC the transfer pattern
164  * struct dma_vec - DMA vector
166  * @len: Length in bytes of the DMA vector
174  * enum dma_ctrl_flags - DMA flags to augment operation preparation,
176  * @DMA_PREP_INTERRUPT - trigger an interrupt (callback) upon completion of
178  * @DMA_CTRL_ACK - if clear, the descriptor cannot be reused until the client
181  * @DMA_PREP_PQ_DISABLE_P - prevent generation of P while generating Q
182  * @DMA_PREP_PQ_DISABLE_Q - prevent generation of Q while generating P
183  * @DMA_PREP_CONTINUE - indicate to a driver that it is reusing buffers as
186  * @DMA_PREP_FENCE - tell the driver that subsequent operations depend
190  * @DMA_PREP_CMD: tell the driver that the data passed to DMA API is command
194  *  repeated when it ends until a transaction is issued on the same channel
218  * enum sum_check_bits - bit position of pq_check_flags
226  * enum pq_check_flags - result of async_{xor,pq}_zero_sum operations
227  * @SUM_CHECK_P_RESULT - 1 if xor zero sum error, 0 otherwise
228  * @SUM_CHECK_Q_RESULT - 1 if reed-solomon zero sum error, 0 otherwise
237  * dma_cap_mask_t - capabilities bitmap modeled after cpumask_t.
243  * enum dma_desc_metadata_mode - per descriptor metadata mode types supported
244  * @DESC_METADATA_CLIENT - the metadata buffer is allocated/provided by the
249  * - DMA_MEM_TO_DEV / DEV_MEM_TO_MEM:
255  * - DMA_DEV_TO_MEM:
263  * @DESC_METADATA_ENGINE - the metadata buffer is allocated/managed by the DMA
274  * - DMA_MEM_TO_DEV / DEV_MEM_TO_MEM:
279  *   4. use dmaengine_desc_set_metadata_len()  to tell the DMA engine the amount
282  * - DMA_DEV_TO_MEM:
299  * struct dma_chan_percpu - the per-CPU part of struct dma_chan
310  * struct dma_router - DMA router structure
311  * @dev: pointer to the DMA router device
320  * struct dma_chan - devices supply DMA channels, clients use them
321  * @device: ptr to the dma device who supplies this channel, always !%NULL
322  * @slave: ptr to the device using this channel
324  * @completed_cookie: last completed cookie for this channel
325  * @chan_id: channel ID for sysfs
329  *	dev_name(requester's dev):channel name, for example: "2b00000.mcasp:tx"
331  * @local: per-cpu pointer to a struct dma_chan_percpu
332  * @client_count: how many clients are using this channel
333  * @table_count: number of appearances in the mem-to-mem allocation table
334  * @router: pointer to the DMA router structure
335  * @route_data: channel specific data for the router
336  * @private: private data for certain client-channel associations
357 	/* DMA router */
365  * struct dma_chan_dev - relate sysfs device node to backing channel device
366  * @chan: driver channel device
369  * @chan_dma_dev: The channel is using custom/different dma-mapping
380  * enum dma_slave_buswidth - defines bus width of the DMA slave
397  * struct dma_slave_config - dma slave channel runtime config
399  * channel, right now. DMA_MEM_TO_DEV and DMA_DEV_TO_MEM are
403  * @src_addr: this is the physical address where DMA slave data
406  * @dst_addr: this is the physical address where DMA slave data
410  * register where DMA data shall be read. If the source
424  * an area instead of a single register to receive the data. Typically the DMA
435  * This struct is passed in as configuration data to a DMA engine
436  * in order to set up a certain channel for DMA transport at runtime.
437  * The DMA device/engine has to provide support for an additional
442  * follows: if it is likely that more than one DMA slave controllers in
463  * enum dma_residue_granularity - Granularity of the reported transfer residue
465  *  DMA channel is only able to tell whether a descriptor has been completed or
466  *  not, which means residue reporting is not supported by this channel. The
473  *  the hardware supports scatter-gather and the segment descriptor has a field
489  * struct dma_slave_caps - expose capabilities of a slave channel only
490  * @src_addr_widths: bit mask of src addr widths the channel supports.
491  *	Width is specified in bytes, e.g. for a channel supporting
492  *	a width of 4 the mask should have BIT(4) set.
493  * @dst_addr_widths: bit mask of dst addr widths the channel supports
494  * @directions: bit mask of slave directions the channel supports.
496  *	each type, the dma controller should set BIT(<TYPE>) and same
498  * @min_burst: min burst capability per-transfer
499  * @max_burst: max burst capability per-transfer
501  *	DMA tansaction with no software intervention for reinitialization.
527 	return dev_name(&chan->dev->device);  in dma_chan_name()
531  * typedef dma_filter_fn - callback filter for dma_request_channel
532  * @chan: channel to be reviewed
536  * suitable channel is passed to this routine for further dispositioning before
537  * being returned.  Where 'suitable' indicates a non-busy channel that
538  * satisfies the given capability mask.  It returns 'true' to indicate that the
539  * channel is suitable.
547 	DMA_TRANS_READ_FAILED,		/* Source DMA read failed */
548 	DMA_TRANS_WRITE_FAILED,		/* Destination DMA write failed */
588  * struct dma_async_tx_descriptor - async transaction descriptor
589  * ---dma generic offload fields---
590  * @cookie: tracking cookie for this transaction, set to -EBUSY if
595  * @chan: target channel for this operation
603  * @metadata_ops: DMA driver provided metadata mode ops, need to be set by the
604  *	DMA driver if metadata mode is supported with the descriptor
605  * ---async_tx api specific fields---
634 	kref_get(&unmap->kref);  in dma_set_unmap()
635 	tx->unmap = unmap;  in dma_set_unmap()
658 	if (!tx->unmap)  in dma_descriptor_unmap()
661 	dmaengine_unmap_put(tx->unmap);  in dma_descriptor_unmap()
662 	tx->unmap = NULL;  in dma_descriptor_unmap()
694 	spin_lock_bh(&txd->lock);  in txd_lock()
698 	spin_unlock_bh(&txd->lock);  in txd_unlock()
702 	txd->next = next;  in txd_chain()
703 	next->parent = txd;  in txd_chain()
707 	txd->parent = NULL;  in txd_clear_parent()
711 	txd->next = NULL;  in txd_clear_next()
715 	return txd->parent;  in txd_parent()
719 	return txd->next;  in txd_next()
724  * struct dma_tx_state - filled in to report the status of
726  * @last: last completed DMA cookie
727  * @used: last issued DMA cookie (i.e. the one in progress)
731  * @in_flight_bytes: amount of data in bytes cached by the DMA.
741  * enum dmaengine_alignment - defines alignment of the DMA async tx
757  * struct dma_slave_map - associates slave device and it's slave channel with
760  * @slave: slave channel name
770  * struct dma_filter - information for slave device/channel to filter_fn/param
773  * @mapcnt: number of slave device/channel in the map
774  * @map: array of channel to filter mapping data
783  * struct dma_device - info on the entity supplying DMA services
784  * @ref: reference is taken and put every time a channel is allocated or freed
785  * @chancnt: how many DMA channels are supported
786  * @privatecnt: how many DMA channels are requested by dma_request_channel
791  * @desc_metadata_modes: supported metadata modes by the DMA device
793  * @max_pq: maximum number of PQ sources and PQ-continue capability
799  * @dev: struct device reference for dma mapping api
801  * @chan_ida: unique channel ID
802  * @src_addr_widths: bit mask of src addr widths the device supports
804  *	a width of 4 the mask should have BIT(4) set.
805  * @dst_addr_widths: bit mask of dst addr widths the device supports
806  * @directions: bit mask of slave directions the device supports.
808  *	each type, the dma controller should set BIT(<TYPE>) and same
810  * @min_burst: min burst capability per-transfer
811  * @max_burst: max burst capability per-transfer
813  *	DMA tansaction with no software intervention for reinitialization.
820  * @device_router_config: optional callback for DMA router configuration
821  * @device_free_chan_resources: release DMA channel's resources
830  * @device_prep_slave_sg: prepares a slave dma operation
831  * @device_prep_dma_cyclic: prepare a cyclic dma operation suitable for audio.
835  * @device_prep_dma_imm_data: DMA's 8 byte immediate data to the dst address
836  * @device_caps: May be used to override the generic DMA slave capabilities
837  *	with per-channel specific ones
838  * @device_config: Pushes a new configuration to a channel, return 0 or an error
840  * @device_pause: Pauses any transfer happening on a channel. Returns
842  * @device_resume: Resumes any transfer on a channel previously
844  * @device_terminate_all: Aborts all transfers on a channel. Returns 0
962 	if (chan->device->device_config)  in dmaengine_slave_config()
963 		return chan->device->device_config(chan, config);  in dmaengine_slave_config()
965 	return -ENOSYS;  in dmaengine_slave_config()
983 	if (!chan || !chan->device || !chan->device->device_prep_slave_sg)  in dmaengine_prep_slave_single()
986 	return chan->device->device_prep_slave_sg(chan, &sg, 1,  in dmaengine_prep_slave_single()
991  * dmaengine_prep_peripheral_dma_vec() - Prepare a DMA scatter-gather descriptor
992  * @chan: The channel to be used for this descriptor
993  * @vecs: The array of DMA vectors that should be transferred
994  * @nents: The number of DMA vectors in the array
996  * @flags: DMA engine flags
1002 	if (!chan || !chan->device || !chan->device->device_prep_peripheral_dma_vec)  in dmaengine_prep_peripheral_dma_vec()
1005 	return chan->device->device_prep_peripheral_dma_vec(chan, vecs, nents,  in dmaengine_prep_peripheral_dma_vec()
1013 	if (!chan || !chan->device || !chan->device->device_prep_slave_sg)  in dmaengine_prep_slave_sg()
1016 	return chan->device->device_prep_slave_sg(chan, sgl, sg_len,  in dmaengine_prep_slave_sg()
1027 	if (!chan || !chan->device || !chan->device->device_prep_slave_sg)  in dmaengine_prep_rio_sg()
1030 	return chan->device->device_prep_slave_sg(chan, sgl, sg_len,  in dmaengine_prep_rio_sg()
1040 	if (!chan || !chan->device || !chan->device->device_prep_dma_cyclic)  in dmaengine_prep_dma_cyclic()
1043 	return chan->device->device_prep_dma_cyclic(chan, buf_addr, buf_len,  in dmaengine_prep_dma_cyclic()
1051 	if (!chan || !chan->device || !chan->device->device_prep_interleaved_dma)  in dmaengine_prep_interleaved_dma()
1054 	    !test_bit(DMA_REPEAT, chan->device->cap_mask.bits))  in dmaengine_prep_interleaved_dma()
1057 	return chan->device->device_prep_interleaved_dma(chan, xt, flags);  in dmaengine_prep_interleaved_dma()
1061  * dmaengine_prep_dma_memset() - Prepare a DMA memset descriptor.
1062  * @chan: The channel to be used for this descriptor
1066  * @flags: DMA engine flags
1072 	if (!chan || !chan->device || !chan->device->device_prep_dma_memset)  in dmaengine_prep_dma_memset()
1075 	return chan->device->device_prep_dma_memset(chan, dest, value,  in dmaengine_prep_dma_memset()
1083 	if (!chan || !chan->device || !chan->device->device_prep_dma_memcpy)  in dmaengine_prep_dma_memcpy()
1086 	return chan->device->device_prep_dma_memcpy(chan, dest, src,  in dmaengine_prep_dma_memcpy()
1096 	return !!(chan->device->desc_metadata_modes & mode);  in dmaengine_is_metadata_mode_supported()
1110 	return -EINVAL;  in dmaengine_desc_attach_metadata()
1121 	return -EINVAL;  in dmaengine_desc_set_metadata_len()
1126  * dmaengine_terminate_all() - Terminate all active DMA transfers
1127  * @chan: The channel for which to terminate the transfers
1134 	if (chan->device->device_terminate_all)  in dmaengine_terminate_all()
1135 		return chan->device->device_terminate_all(chan);  in dmaengine_terminate_all()
1137 	return -ENOSYS;  in dmaengine_terminate_all()
1141  * dmaengine_terminate_async() - Terminate all active DMA transfers
1142  * @chan: The channel for which to terminate the transfers
1145  * that have previously been submitted to the channel. It is not guaranteed
1156  * complete callback of a descriptor submitted on the same channel.
1163 	if (chan->device->device_terminate_all)  in dmaengine_terminate_async()
1164 		return chan->device->device_terminate_all(chan);  in dmaengine_terminate_async()
1166 	return -EINVAL;  in dmaengine_terminate_async()
1170  * dmaengine_synchronize() - Synchronize DMA channel termination
1171  * @chan: The channel to synchronize
1173  * Synchronizes to the DMA channel termination to the current context. When this
1183  * This function must only be called from non-atomic context and must not be
1185  * channel.
1191 	if (chan->device->device_synchronize)  in dmaengine_synchronize()
1192 		chan->device->device_synchronize(chan);  in dmaengine_synchronize()
1196  * dmaengine_terminate_sync() - Terminate all active DMA transfers
1197  * @chan: The channel for which to terminate the transfers
1200  * that have previously been submitted to the channel. It is similar to
1201  * dmaengine_terminate_async() but guarantees that the DMA transfer has actually
1205  * This function must only be called from non-atomic context and must not be
1207  * channel.
1224 	if (chan->device->device_pause)  in dmaengine_pause()
1225 		return chan->device->device_pause(chan);  in dmaengine_pause()
1227 	return -ENOSYS;  in dmaengine_pause()
1232 	if (chan->device->device_resume)  in dmaengine_resume()
1233 		return chan->device->device_resume(chan);  in dmaengine_resume()
1235 	return -ENOSYS;  in dmaengine_resume()
1241 	return chan->device->device_tx_status(chan, cookie, state);  in dmaengine_tx_status()
1246 	return desc->tx_submit(desc);  in dmaengine_submit()
1252 	return !(((1 << align) - 1) & (off1 | off2 | len));  in dmaengine_check_align()
1258 	return dmaengine_check_align(dev->copy_align, off1, off2, len);  in is_dma_copy_aligned()
1264 	return dmaengine_check_align(dev->xor_align, off1, off2, len);  in is_dma_xor_aligned()
1270 	return dmaengine_check_align(dev->pq_align, off1, off2, len);  in is_dma_pq_aligned()
1276 	return dmaengine_check_align(dev->fill_align, off1, off2, len);  in is_dma_fill_aligned()
1280 dma_set_maxpq(struct dma_device *dma, int maxpq, int has_pq_continue)  in dma_set_maxpq()  argument
1282 	dma->max_pq = maxpq;  in dma_set_maxpq()
1284 		dma->max_pq |= DMA_HAS_PQ_CONTINUE;  in dma_set_maxpq()
1294 	enum dma_ctrl_flags mask = DMA_PREP_CONTINUE | DMA_PREP_PQ_DISABLE_P;  in dmaf_p_disabled_continue()  local
1296 	return (flags & mask) == mask;  in dmaf_p_disabled_continue()
1299 static inline bool dma_dev_has_pq_continue(struct dma_device *dma)  in dma_dev_has_pq_continue()  argument
1301 	return (dma->max_pq & DMA_HAS_PQ_CONTINUE) == DMA_HAS_PQ_CONTINUE;  in dma_dev_has_pq_continue()
1304 static inline unsigned short dma_dev_to_maxpq(struct dma_device *dma)  in dma_dev_to_maxpq()  argument
1306 	return dma->max_pq & ~DMA_HAS_PQ_CONTINUE;  in dma_dev_to_maxpq()
1309 /* dma_maxpq - reduce maxpq in the face of continued operations
1310  * @dma - dma device with PQ capability
1311  * @flags - to check if DMA_PREP_CONTINUE and DMA_PREP_PQ_DISABLE_P are set
1322 static inline int dma_maxpq(struct dma_device *dma, enum dma_ctrl_flags flags)  in dma_maxpq()  argument
1324 	if (dma_dev_has_pq_continue(dma) || !dmaf_continue(flags))  in dma_maxpq()
1325 		return dma_dev_to_maxpq(dma);  in dma_maxpq()
1327 		return dma_dev_to_maxpq(dma) - 1;  in dma_maxpq()
1329 		return dma_dev_to_maxpq(dma) - 3;  in dma_maxpq()
1349 	return dmaengine_get_icg(xt->dst_inc, xt->dst_sgl,  in dmaengine_get_dst_icg()
1350 				 chunk->icg, chunk->dst_icg);  in dmaengine_get_dst_icg()
1356 	return dmaengine_get_icg(xt->src_inc, xt->src_sgl,  in dmaengine_get_src_icg()
1357 				 chunk->icg, chunk->src_icg);  in dmaengine_get_src_icg()
1360 /* --- public DMA engine API --- */
1400 	tx->flags |= DMA_CTRL_ACK;  in async_tx_ack()
1405 	tx->flags &= ~DMA_CTRL_ACK;  in async_tx_clear_ack()
1410 	return (tx->flags & DMA_CTRL_ACK) == DMA_CTRL_ACK;  in async_tx_test_ack()
1413 #define dma_cap_set(tx, mask) __dma_cap_set((tx), &(mask))  argument
1417 	set_bit(tx_type, dstp->bits);  in __dma_cap_set()
1420 #define dma_cap_clear(tx, mask) __dma_cap_clear((tx), &(mask))  argument
1424 	clear_bit(tx_type, dstp->bits);  in __dma_cap_clear()
1427 #define dma_cap_zero(mask) __dma_cap_zero(&(mask))  argument
1430 	bitmap_zero(dstp->bits, DMA_TX_TYPE_END);  in __dma_cap_zero()
1433 #define dma_has_cap(tx, mask) __dma_has_cap((tx), &(mask))  argument
1437 	return test_bit(tx_type, srcp->bits);  in __dma_has_cap()
1440 #define for_each_dma_cap_mask(cap, mask) \  argument
1441 	for_each_set_bit(cap, mask.bits, DMA_TX_TYPE_END)
1444  * dma_async_issue_pending - flush pending transactions to HW
1445  * @chan: target DMA channel
1452 	chan->device->device_issue_pending(chan);  in dma_async_issue_pending()
1456  * dma_async_is_tx_complete - poll for transaction completion
1457  * @chan: DMA channel
1464  * the status of multiple cookies without re-checking hardware state.
1472 	status = chan->device->device_tx_status(chan, cookie, &state);  in dma_async_is_tx_complete()
1481  * dma_async_is_complete - test a cookie against chan state
1508 	st->last = last;  in dma_set_tx_state()
1509 	st->used = used;  in dma_set_tx_state()
1510 	st->residue = residue;  in dma_set_tx_state()
1518 struct dma_chan *__dma_request_channel(const dma_cap_mask_t *mask,
1523 struct dma_chan *dma_request_chan_by_mask(const dma_cap_mask_t *mask);
1543 static inline struct dma_chan *__dma_request_channel(const dma_cap_mask_t *mask,  in __dma_request_channel()  argument
1553 	return ERR_PTR(-ENODEV);  in dma_request_chan()
1556 						const dma_cap_mask_t *mask)  in dma_request_chan_by_mask()  argument
1558 	return ERR_PTR(-ENODEV);  in dma_request_chan_by_mask()
1566 	return -ENXIO;  in dma_get_slave_caps()
1575 	ret = dma_get_slave_caps(tx->chan, &caps);  in dmaengine_desc_set_reuse()
1580 		return -EPERM;  in dmaengine_desc_set_reuse()
1582 	tx->flags |= DMA_CTRL_REUSE;  in dmaengine_desc_set_reuse()
1588 	tx->flags &= ~DMA_CTRL_REUSE;  in dmaengine_desc_clear_reuse()
1593 	return (tx->flags & DMA_CTRL_REUSE) == DMA_CTRL_REUSE;  in dmaengine_desc_test_reuse()
1600 		return -EPERM;  in dmaengine_desc_free()
1602 	return desc->desc_free(desc);  in dmaengine_desc_free()
1605 /* --- DMA device --- */
1616 #define dma_request_channel(mask, x, y) \  argument
1617 	__dma_request_channel(&(mask), x, y, NULL)
1629 *dma_request_slave_channel_compat(const dma_cap_mask_t mask,  in dma_request_slave_channel_compat()  argument
1642 	return __dma_request_channel(&mask, fn, fn_param, NULL);  in dma_request_slave_channel_compat()
1664 	if (chan->dev->chan_dma_dev)  in dmaengine_get_dma_device()
1665 		return &chan->dev->device;  in dmaengine_get_dma_device()
1667 	return chan->device->dev;  in dmaengine_get_dma_device()