Lines Matching +full:map +full:- +full:to +full:- +full:dma +full:- +full:channel
1 // SPDX-License-Identifier: GPL-2.0
3 /* Copyright (c) 2012-2018, The Linux Foundation. All rights reserved.
4 * Copyright (C) 2019-2024 Linaro Ltd.
9 #include <linux/dma-direction.h>
24 * A GSI transaction abstracts the behavior of a GSI channel by representing
28 * by the GSI transaction core, allowing users to simply describe operations
29 * to be performed. When a transaction has completed a callback function
30 * (dependent on the type of endpoint associated with the channel) allows
33 * To perform an operation (or set of them), a user of the GSI transaction
37 * exhaustion of the available TREs in a channel ring is detected as early
38 * as possible. Any other resources that might be needed to complete a
43 * scatterlist structures are initialized by "adding" operations to the
44 * transaction. If a buffer in an operation must be mapped for DMA, this is
45 * done at the time it is added to the transaction. It is possible for a
46 * mapping error to occur when an operation is added. In this case the
50 * Once all operations have been successfully added to a transaction, the
52 * transaction to the GSI transaction core. The GSI transaction code
53 * formats the content of the scatterlist array into the channel ring
54 * buffer and informs the hardware that new TREs are available to process.
56 * The last TRE in each transaction is marked to interrupt the AP when the
59 * TRE in the transaction is sufficient to indicate the full transaction
63 * GSI code into the IPA layer, allowing it to perform any final cleanup
73 /* An entry in a channel ring */
75 __le64 addr; /* DMA address */
81 /* gsi_tre->flags mask values (in CPU byte order) */
94 return -EINVAL; in gsi_trans_pool_init()
96 return -EINVAL; in gsi_trans_pool_init()
98 return -EINVAL; in gsi_trans_pool_init()
107 alloc_size = size_mul(count + max_alloc - 1, size); in gsi_trans_pool_init()
111 return -ENOMEM; in gsi_trans_pool_init()
113 pool->base = virt; in gsi_trans_pool_init()
115 pool->count = alloc_size / size; in gsi_trans_pool_init()
116 pool->free = 0; in gsi_trans_pool_init()
117 pool->max_alloc = max_alloc; in gsi_trans_pool_init()
118 pool->size = size; in gsi_trans_pool_init()
119 pool->addr = 0; /* Only used for DMA pools */ in gsi_trans_pool_init()
126 kfree(pool->base); in gsi_trans_pool_exit()
130 /* Home-grown DMA pool. This way we can preallocate the pool, and guarantee
132 * require up to max_alloc elements from the pool. But we only allow
133 * allocation of a single element from a DMA pool at a time.
143 return -EINVAL; in gsi_trans_pool_init_dma()
145 return -EINVAL; in gsi_trans_pool_init_dma()
147 return -EINVAL; in gsi_trans_pool_init_dma()
149 /* Don't let allocations cross a power-of-two boundary */ in gsi_trans_pool_init_dma()
151 total_size = (count + max_alloc - 1) * size; in gsi_trans_pool_init_dma()
153 /* The allocator will give us a power-of-2 number of pages in gsi_trans_pool_init_dma()
154 * sufficient to satisfy our request. Round up our requested in gsi_trans_pool_init_dma()
155 * size to avoid any unused space in the allocation. This way in gsi_trans_pool_init_dma()
163 return -ENOMEM; in gsi_trans_pool_init_dma()
165 pool->base = virt; in gsi_trans_pool_init_dma()
166 pool->count = total_size / size; in gsi_trans_pool_init_dma()
167 pool->free = 0; in gsi_trans_pool_init_dma()
168 pool->size = size; in gsi_trans_pool_init_dma()
169 pool->max_alloc = max_alloc; in gsi_trans_pool_init_dma()
170 pool->addr = addr; in gsi_trans_pool_init_dma()
177 size_t total_size = pool->count * pool->size; in gsi_trans_pool_exit_dma()
179 dma_free_coherent(dev, total_size, pool->base, pool->addr); in gsi_trans_pool_exit_dma()
189 WARN_ON(count > pool->max_alloc); in gsi_trans_pool_alloc_common()
192 if (count > pool->count - pool->free) in gsi_trans_pool_alloc_common()
193 pool->free = 0; in gsi_trans_pool_alloc_common()
195 offset = pool->free * pool->size; in gsi_trans_pool_alloc_common()
196 pool->free += count; in gsi_trans_pool_alloc_common()
197 memset(pool->base + offset, 0, count * pool->size); in gsi_trans_pool_alloc_common()
205 return pool->base + gsi_trans_pool_alloc_common(pool, count); in gsi_trans_pool_alloc()
208 /* Allocate a single zeroed entry from a DMA pool */
213 *addr = pool->addr + offset; in gsi_trans_pool_alloc_dma()
215 return pool->base + offset; in gsi_trans_pool_alloc_dma()
218 /* Map a TRE ring entry index to the transaction it is associated with */
221 struct gsi_channel *channel = &trans->gsi->channel[trans->channel_id]; in gsi_trans_map() local
224 index += trans->used_count - 1; in gsi_trans_map()
227 channel->trans_info.map[index % channel->tre_ring.count] = trans; in gsi_trans_map()
230 /* Return the transaction mapped to a given ring entry */
232 gsi_channel_trans_mapped(struct gsi_channel *channel, u32 index) in gsi_channel_trans_mapped() argument
235 return channel->trans_info.map[index % channel->tre_ring.count]; in gsi_channel_trans_mapped()
238 /* Return the oldest completed transaction for a channel (or null) */
239 struct gsi_trans *gsi_channel_trans_complete(struct gsi_channel *channel) in gsi_channel_trans_complete() argument
241 struct gsi_trans_info *trans_info = &channel->trans_info; in gsi_channel_trans_complete()
242 u16 trans_id = trans_info->completed_id; in gsi_channel_trans_complete()
244 if (trans_id == trans_info->pending_id) { in gsi_channel_trans_complete()
245 gsi_channel_update(channel); in gsi_channel_trans_complete()
246 if (trans_id == trans_info->pending_id) in gsi_channel_trans_complete()
250 return &trans_info->trans[trans_id %= channel->tre_count]; in gsi_channel_trans_complete()
253 /* Move a transaction from allocated to committed state */
256 struct gsi_channel *channel = &trans->gsi->channel[trans->channel_id]; in gsi_trans_move_committed() local
257 struct gsi_trans_info *trans_info = &channel->trans_info; in gsi_trans_move_committed()
260 trans_info->allocated_id++; in gsi_trans_move_committed()
263 /* Move committed transactions to pending state */
266 struct gsi_channel *channel = &trans->gsi->channel[trans->channel_id]; in gsi_trans_move_pending() local
267 struct gsi_trans_info *trans_info = &channel->trans_info; in gsi_trans_move_pending()
268 u16 trans_index = trans - &trans_info->trans[0]; in gsi_trans_move_pending()
272 delta = trans_index - trans_info->committed_id + 1; in gsi_trans_move_pending()
273 trans_info->committed_id += delta % channel->tre_count; in gsi_trans_move_pending()
276 /* Move pending transactions to completed state */
279 struct gsi_channel *channel = &trans->gsi->channel[trans->channel_id]; in gsi_trans_move_complete() local
280 struct gsi_trans_info *trans_info = &channel->trans_info; in gsi_trans_move_complete()
281 u16 trans_index = trans - trans_info->trans; in gsi_trans_move_complete()
285 delta = trans_index - trans_info->pending_id + 1; in gsi_trans_move_complete()
286 delta %= channel->tre_count; in gsi_trans_move_complete()
287 trans_info->pending_id += delta; in gsi_trans_move_complete()
290 /* Move a transaction from completed to polled state */
293 struct gsi_channel *channel = &trans->gsi->channel[trans->channel_id]; in gsi_trans_move_polled() local
294 struct gsi_trans_info *trans_info = &channel->trans_info; in gsi_trans_move_polled()
297 trans_info->completed_id++; in gsi_trans_move_polled()
300 /* Reserve some number of TREs on a channel. Returns true if successful */
304 int avail = atomic_read(&trans_info->tre_avail); in gsi_trans_tre_reserve()
308 new = avail - (int)tre_count; in gsi_trans_tre_reserve()
311 } while (!atomic_try_cmpxchg(&trans_info->tre_avail, &avail, new)); in gsi_trans_tre_reserve()
316 /* Release previously-reserved TRE entries to a channel */
320 atomic_add(tre_count, &trans_info->tre_avail); in gsi_trans_tre_release()
329 trans_info = &gsi->channel[channel_id].trans_info; in gsi_channel_trans_idle()
331 return atomic_read(&trans_info->tre_avail) == tre_max; in gsi_channel_trans_idle()
334 /* Allocate a GSI transaction on a channel */
339 struct gsi_channel *channel = &gsi->channel[channel_id]; in gsi_channel_trans_alloc() local
344 if (WARN_ON(tre_count > channel->trans_tre_max)) in gsi_channel_trans_alloc()
347 trans_info = &channel->trans_info; in gsi_channel_trans_alloc()
353 trans_index = trans_info->free_id % channel->tre_count; in gsi_channel_trans_alloc()
354 trans = &trans_info->trans[trans_index]; in gsi_channel_trans_alloc()
357 /* Initialize non-zero fields in the transaction */ in gsi_channel_trans_alloc()
358 trans->gsi = gsi; in gsi_channel_trans_alloc()
359 trans->channel_id = channel_id; in gsi_channel_trans_alloc()
360 trans->rsvd_count = tre_count; in gsi_channel_trans_alloc()
361 init_completion(&trans->completion); in gsi_channel_trans_alloc()
364 trans->sgl = gsi_trans_pool_alloc(&trans_info->sg_pool, tre_count); in gsi_channel_trans_alloc()
365 sg_init_marker(trans->sgl, tre_count); in gsi_channel_trans_alloc()
367 trans->direction = direction; in gsi_channel_trans_alloc()
368 refcount_set(&trans->refcount, 1); in gsi_channel_trans_alloc()
371 trans_info->free_id++; in gsi_channel_trans_alloc()
376 /* Free a previously-allocated transaction */
381 if (!refcount_dec_and_test(&trans->refcount)) in gsi_trans_free()
387 trans_info = &trans->gsi->channel[trans->channel_id].trans_info; in gsi_trans_free()
388 if (!trans->used_count) { in gsi_trans_free()
389 trans_info->allocated_id++; in gsi_trans_free()
390 trans_info->committed_id++; in gsi_trans_free()
391 trans_info->pending_id++; in gsi_trans_free()
392 trans_info->completed_id++; in gsi_trans_free()
398 trans_info->polled_id++; in gsi_trans_free()
403 gsi_trans_tre_release(trans_info, trans->rsvd_count); in gsi_trans_free()
406 /* Add an immediate command to a transaction */
410 u32 which = trans->used_count++; in gsi_trans_cmd_add()
413 WARN_ON(which >= trans->rsvd_count); in gsi_trans_cmd_add()
417 * using dma_alloc_coherent(). We therefore do *not* map them in gsi_trans_cmd_add()
418 * for DMA (unlike what we do for pages and skbs). in gsi_trans_cmd_add()
422 * gsi_trans_complete() to skip the unmapping step. in gsi_trans_cmd_add()
425 * entry are the DMA address and length. We still need the SG in gsi_trans_cmd_add()
426 * table flags to be maintained though, so assign a NULL page in gsi_trans_cmd_add()
429 sg = &trans->sgl[which]; in gsi_trans_cmd_add()
434 trans->cmd_opcode[which] = opcode; in gsi_trans_cmd_add()
437 /* Add a page transfer to a transaction. It will fill the only TRE. */
441 struct scatterlist *sg = &trans->sgl[0]; in gsi_trans_page_add()
444 if (WARN_ON(trans->rsvd_count != 1)) in gsi_trans_page_add()
445 return -EINVAL; in gsi_trans_page_add()
446 if (WARN_ON(trans->used_count)) in gsi_trans_page_add()
447 return -EINVAL; in gsi_trans_page_add()
450 ret = dma_map_sg(trans->gsi->dev, sg, 1, trans->direction); in gsi_trans_page_add()
452 return -ENOMEM; in gsi_trans_page_add()
454 trans->used_count++; /* Transaction now owns the (DMA mapped) page */ in gsi_trans_page_add()
459 /* Add an SKB transfer to a transaction. No other TREs will be used. */
462 struct scatterlist *sg = &trans->sgl[0]; in gsi_trans_skb_add()
466 if (WARN_ON(trans->rsvd_count != 1)) in gsi_trans_skb_add()
467 return -EINVAL; in gsi_trans_skb_add()
468 if (WARN_ON(trans->used_count)) in gsi_trans_skb_add()
469 return -EINVAL; in gsi_trans_skb_add()
471 /* skb->len will not be 0 (checked early) */ in gsi_trans_skb_add()
472 ret = skb_to_sgvec(skb, sg, 0, skb->len); in gsi_trans_skb_add()
477 ret = dma_map_sg(trans->gsi->dev, sg, used_count, trans->direction); in gsi_trans_skb_add()
479 return -ENOMEM; in gsi_trans_skb_add()
481 /* Transaction now owns the (DMA mapped) skb */ in gsi_trans_skb_add()
482 trans->used_count += used_count; in gsi_trans_skb_add()
487 /* Compute the length/opcode value to use for a TRE */
494 /* Compute the flags value to use for a given TRE */
510 } else { /* All others indicate there's more to come */ in gsi_tre_flags()
529 * Doing the assignment this way is an attempt to make that happen. in gsi_trans_tre_fill()
535 * __gsi_trans_commit() - Common GSI transaction commit code
536 * @trans: Transaction to commit
537 * @ring_db: Whether to tell the hardware about these queued transfers
539 * Formats channel ring TRE entries based on the content of the scatterlist.
540 * Maps a transaction pointer to the last ring entry used for the transaction,
541 * so it can be recovered when it completes. Moves the transaction to
542 * pending state. Finally, updates the channel ring pointer and optionally
547 struct gsi_channel *channel = &trans->gsi->channel[trans->channel_id]; in __gsi_trans_commit() local
548 struct gsi_ring *tre_ring = &channel->tre_ring; in __gsi_trans_commit()
550 bool bei = channel->toward_ipa; in __gsi_trans_commit()
558 WARN_ON(!trans->used_count); in __gsi_trans_commit()
561 * filling them we'll switch to the beginning to finish. in __gsi_trans_commit()
565 cmd_opcode = channel->command ? &trans->cmd_opcode[0] : NULL; in __gsi_trans_commit()
566 avail = tre_ring->count - tre_ring->index % tre_ring->count; in __gsi_trans_commit()
567 dest_tre = gsi_ring_virt(tre_ring, tre_ring->index); in __gsi_trans_commit()
568 for_each_sg(trans->sgl, sg, trans->used_count, i) { in __gsi_trans_commit()
569 bool last_tre = i == trans->used_count - 1; in __gsi_trans_commit()
574 if (!avail--) in __gsi_trans_commit()
583 gsi_trans_map(trans, tre_ring->index); in __gsi_trans_commit()
585 tre_ring->index += trans->used_count; in __gsi_trans_commit()
587 trans->len = byte_count; in __gsi_trans_commit()
588 if (channel->toward_ipa) in __gsi_trans_commit()
594 if (ring_db || !atomic_read(&channel->trans_info.tre_avail)) { in __gsi_trans_commit()
595 /* Report what we're handing off to hardware for TX channels */ in __gsi_trans_commit()
596 if (channel->toward_ipa) in __gsi_trans_commit()
599 gsi_channel_doorbell(channel); in __gsi_trans_commit()
606 if (trans->used_count) in gsi_trans_commit()
612 /* Commit a GSI transaction and wait for it to complete */
615 if (!trans->used_count) in gsi_trans_commit_wait()
618 refcount_inc(&trans->refcount); in gsi_trans_commit_wait()
622 wait_for_completion(&trans->completion); in gsi_trans_commit_wait()
632 if (trans->direction != DMA_NONE) in gsi_trans_complete()
633 dma_unmap_sg(trans->gsi->dev, trans->sgl, trans->used_count, in gsi_trans_complete()
634 trans->direction); in gsi_trans_complete()
638 complete(&trans->completion); in gsi_trans_complete()
643 /* Cancel a channel's pending transactions */
644 void gsi_channel_trans_cancel_pending(struct gsi_channel *channel) in gsi_channel_trans_cancel_pending() argument
646 struct gsi_trans_info *trans_info = &channel->trans_info; in gsi_channel_trans_cancel_pending()
647 u16 trans_id = trans_info->pending_id; in gsi_channel_trans_cancel_pending()
649 /* channel->gsi->mutex is held by caller */ in gsi_channel_trans_cancel_pending()
652 if (trans_id == trans_info->committed_id) in gsi_channel_trans_cancel_pending()
659 trans = &trans_info->trans[trans_id % channel->tre_count]; in gsi_channel_trans_cancel_pending()
660 trans->cancelled = true; in gsi_channel_trans_cancel_pending()
661 } while (++trans_id != trans_info->committed_id); in gsi_channel_trans_cancel_pending()
664 trans_info->pending_id = trans_info->committed_id; in gsi_channel_trans_cancel_pending()
666 /* Schedule NAPI polling to complete the cancelled transactions */ in gsi_channel_trans_cancel_pending()
667 napi_schedule(&channel->napi); in gsi_channel_trans_cancel_pending()
670 /* Issue a command to read a single byte from a channel */
673 struct gsi_channel *channel = &gsi->channel[channel_id]; in gsi_trans_read_byte() local
674 struct gsi_ring *tre_ring = &channel->tre_ring; in gsi_trans_read_byte()
678 trans_info = &channel->trans_info; in gsi_trans_read_byte()
682 return -EBUSY; in gsi_trans_read_byte()
686 dest_tre = gsi_ring_virt(tre_ring, tre_ring->index); in gsi_trans_read_byte()
689 tre_ring->index++; in gsi_trans_read_byte()
690 gsi_channel_doorbell(channel); in gsi_trans_read_byte()
698 struct gsi_channel *channel = &gsi->channel[channel_id]; in gsi_trans_read_byte_done() local
700 gsi_trans_tre_release(&channel->trans_info, 1); in gsi_trans_read_byte_done()
703 /* Initialize a channel's GSI transaction info */
706 struct gsi_channel *channel = &gsi->channel[channel_id]; in gsi_channel_trans_init() local
707 u32 tre_count = channel->tre_count; in gsi_channel_trans_init()
712 /* Ensure the size of a channel element is what's expected */ in gsi_channel_trans_init()
715 trans_info = &channel->trans_info; in gsi_channel_trans_init()
722 tre_max = gsi_channel_tre_max(channel->gsi, channel_id); in gsi_channel_trans_init()
723 atomic_set(&trans_info->tre_avail, tre_max); in gsi_channel_trans_init()
728 * it to something a little less than that). By allocating a in gsi_channel_trans_init()
729 * power-of-two number of transactions we can use an index in gsi_channel_trans_init()
730 * modulo that number to determine the next one that's free. in gsi_channel_trans_init()
733 trans_info->trans = kcalloc(tre_count, sizeof(*trans_info->trans), in gsi_channel_trans_init()
735 if (!trans_info->trans) in gsi_channel_trans_init()
736 return -ENOMEM; in gsi_channel_trans_init()
737 trans_info->free_id = 0; /* all modulo channel->tre_count */ in gsi_channel_trans_init()
738 trans_info->allocated_id = 0; in gsi_channel_trans_init()
739 trans_info->committed_id = 0; in gsi_channel_trans_init()
740 trans_info->pending_id = 0; in gsi_channel_trans_init()
741 trans_info->completed_id = 0; in gsi_channel_trans_init()
742 trans_info->polled_id = 0; in gsi_channel_trans_init()
744 /* A completion event contains a pointer to the TRE that caused in gsi_channel_trans_init()
746 * Each entry in this map records the transaction associated in gsi_channel_trans_init()
749 trans_info->map = kcalloc(tre_count, sizeof(*trans_info->map), in gsi_channel_trans_init()
751 if (!trans_info->map) { in gsi_channel_trans_init()
752 ret = -ENOMEM; in gsi_channel_trans_init()
756 /* A transaction uses a scatterlist array to represent the data in gsi_channel_trans_init()
758 * element is used to fill a single TRE when the transaction is in gsi_channel_trans_init()
762 ret = gsi_trans_pool_init(&trans_info->sg_pool, in gsi_channel_trans_init()
764 tre_max, channel->trans_tre_max); in gsi_channel_trans_init()
772 kfree(trans_info->map); in gsi_channel_trans_init()
774 kfree(trans_info->trans); in gsi_channel_trans_init()
776 dev_err(gsi->dev, "error %d initializing channel %u transactions\n", in gsi_channel_trans_init()
783 void gsi_channel_trans_exit(struct gsi_channel *channel) in gsi_channel_trans_exit() argument
785 struct gsi_trans_info *trans_info = &channel->trans_info; in gsi_channel_trans_exit()
787 gsi_trans_pool_exit(&trans_info->sg_pool); in gsi_channel_trans_exit()
788 kfree(trans_info->trans); in gsi_channel_trans_exit()
789 kfree(trans_info->map); in gsi_channel_trans_exit()