Lines Matching +full:block +full:- +full:size
1 // SPDX-License-Identifier: GPL-2.0-only
3 * Copyright 2013-2015 Analog Devices Inc.
4 * Author: Lars-Peter Clausen <lars@metafoo.de>
18 #include <linux/iio/buffer-dma.h>
19 #include <linux/dma-buf.h>
20 #include <linux/dma-fence.h>
21 #include <linux/dma-mapping.h>
25 * For DMA buffers the storage is sub-divided into so called blocks. Each block
26 * has its own memory buffer. The size of the block is the granularity at which
28 * basic unit of data exchange from one sample to one block decreases the
31 * sample the overhead will be x for each sample. Whereas when using a block
39 * them with data. Block on the outgoing queue have been filled with data and
42 * A block can be in one of the following states:
44 * the block.
47 * * Owned by the DMA controller: The DMA controller is processing the block
52 * * Dead: A block that is dead has been marked as to be freed. It might still
55 * incoming or outgoing queue the block will be freed.
58 * with both the block structure as well as the storage memory for the block
59 * will be freed when the last reference to the block is dropped. This means a
60 * block must not be accessed without holding a reference.
68 * converter to the memory region of the block. Once the DMA transfer has been
70 * block.
72 * Prior to this it must set the bytes_used field of the block contains
74 * size of the block, but if the DMA hardware has certain alignment requirements
75 * for the transfer length it might choose to use less than the full size. In
77 * datum, i.e. the block must not contain partial samples.
79 * The driver must call iio_dma_buffer_block_done() for each block it has
81 * perform a DMA transfer for the block, e.g. because the buffer was disabled
82 * before the block transfer was started. In this case it should set bytes_used
99 struct iio_dma_buffer_block *block = container_of(kref, in iio_buffer_block_release() local
101 struct iio_dma_buffer_queue *queue = block->queue; in iio_buffer_block_release()
103 WARN_ON(block->fileio && block->state != IIO_BLOCK_STATE_DEAD); in iio_buffer_block_release()
105 if (block->fileio) { in iio_buffer_block_release()
106 dma_free_coherent(queue->dev, PAGE_ALIGN(block->size), in iio_buffer_block_release()
107 block->vaddr, block->phys_addr); in iio_buffer_block_release()
109 atomic_dec(&queue->num_dmabufs); in iio_buffer_block_release()
112 iio_buffer_put(&queue->buffer); in iio_buffer_block_release()
113 kfree(block); in iio_buffer_block_release()
116 static void iio_buffer_block_get(struct iio_dma_buffer_block *block) in iio_buffer_block_get() argument
118 kref_get(&block->kref); in iio_buffer_block_get()
121 static void iio_buffer_block_put(struct iio_dma_buffer_block *block) in iio_buffer_block_put() argument
123 kref_put(&block->kref, iio_buffer_block_release); in iio_buffer_block_put()
135 struct iio_dma_buffer_block *block, *_block; in iio_dma_buffer_cleanup_worker() local
142 list_for_each_entry_safe(block, _block, &block_list, head) in iio_dma_buffer_cleanup_worker()
143 iio_buffer_block_release(&block->kref); in iio_dma_buffer_cleanup_worker()
149 struct iio_dma_buffer_block *block; in iio_buffer_block_release_atomic() local
152 block = container_of(kref, struct iio_dma_buffer_block, kref); in iio_buffer_block_release_atomic()
155 list_add_tail(&block->head, &iio_dma_buffer_dead_blocks); in iio_buffer_block_release_atomic()
164 static void iio_buffer_block_put_atomic(struct iio_dma_buffer_block *block) in iio_buffer_block_put_atomic() argument
166 kref_put(&block->kref, iio_buffer_block_release_atomic); in iio_buffer_block_put_atomic()
175 struct iio_dma_buffer_queue *queue, size_t size, bool fileio) in iio_dma_buffer_alloc_block() argument
177 struct iio_dma_buffer_block *block; in iio_dma_buffer_alloc_block() local
179 block = kzalloc(sizeof(*block), GFP_KERNEL); in iio_dma_buffer_alloc_block()
180 if (!block) in iio_dma_buffer_alloc_block()
184 block->vaddr = dma_alloc_coherent(queue->dev, PAGE_ALIGN(size), in iio_dma_buffer_alloc_block()
185 &block->phys_addr, GFP_KERNEL); in iio_dma_buffer_alloc_block()
186 if (!block->vaddr) { in iio_dma_buffer_alloc_block()
187 kfree(block); in iio_dma_buffer_alloc_block()
192 block->fileio = fileio; in iio_dma_buffer_alloc_block()
193 block->size = size; in iio_dma_buffer_alloc_block()
194 block->state = IIO_BLOCK_STATE_DONE; in iio_dma_buffer_alloc_block()
195 block->queue = queue; in iio_dma_buffer_alloc_block()
196 INIT_LIST_HEAD(&block->head); in iio_dma_buffer_alloc_block()
197 kref_init(&block->kref); in iio_dma_buffer_alloc_block()
199 iio_buffer_get(&queue->buffer); in iio_dma_buffer_alloc_block()
202 atomic_inc(&queue->num_dmabufs); in iio_dma_buffer_alloc_block()
204 return block; in iio_dma_buffer_alloc_block()
207 static void _iio_dma_buffer_block_done(struct iio_dma_buffer_block *block) in _iio_dma_buffer_block_done() argument
209 if (block->state != IIO_BLOCK_STATE_DEAD) in _iio_dma_buffer_block_done()
210 block->state = IIO_BLOCK_STATE_DONE; in _iio_dma_buffer_block_done()
217 if (queue->buffer.direction == IIO_BUFFER_DIRECTION_IN) in iio_dma_buffer_queue_wake()
222 wake_up_interruptible_poll(&queue->buffer.pollq, flags); in iio_dma_buffer_queue_wake()
226 * iio_dma_buffer_block_done() - Indicate that a block has been completed
227 * @block: The completed block
229 * Should be called when the DMA controller has finished handling the block to
230 * pass back ownership of the block to the queue.
232 void iio_dma_buffer_block_done(struct iio_dma_buffer_block *block) in iio_dma_buffer_block_done() argument
234 struct iio_dma_buffer_queue *queue = block->queue; in iio_dma_buffer_block_done()
240 spin_lock_irqsave(&queue->list_lock, flags); in iio_dma_buffer_block_done()
241 _iio_dma_buffer_block_done(block); in iio_dma_buffer_block_done()
242 spin_unlock_irqrestore(&queue->list_lock, flags); in iio_dma_buffer_block_done()
244 if (!block->fileio) in iio_dma_buffer_block_done()
245 iio_buffer_signal_dmabuf_done(block->fence, 0); in iio_dma_buffer_block_done()
247 iio_buffer_block_put_atomic(block); in iio_dma_buffer_block_done()
254 * iio_dma_buffer_block_list_abort() - Indicate that a list block has been
260 * stopped. This will set bytes_used to 0 for each block in the list and then
266 struct iio_dma_buffer_block *block, *_block; in iio_dma_buffer_block_list_abort() local
272 spin_lock_irqsave(&queue->list_lock, flags); in iio_dma_buffer_block_list_abort()
273 list_for_each_entry_safe(block, _block, list, head) { in iio_dma_buffer_block_list_abort()
274 list_del(&block->head); in iio_dma_buffer_block_list_abort()
275 block->bytes_used = 0; in iio_dma_buffer_block_list_abort()
276 _iio_dma_buffer_block_done(block); in iio_dma_buffer_block_list_abort()
278 if (!block->fileio) in iio_dma_buffer_block_list_abort()
279 iio_buffer_signal_dmabuf_done(block->fence, -EINTR); in iio_dma_buffer_block_list_abort()
280 iio_buffer_block_put_atomic(block); in iio_dma_buffer_block_list_abort()
282 spin_unlock_irqrestore(&queue->list_lock, flags); in iio_dma_buffer_block_list_abort()
284 if (queue->fileio.enabled) in iio_dma_buffer_block_list_abort()
285 queue->fileio.enabled = false; in iio_dma_buffer_block_list_abort()
292 static bool iio_dma_block_reusable(struct iio_dma_buffer_block *block) in iio_dma_block_reusable() argument
295 * If the core owns the block it can be re-used. This should be the in iio_dma_block_reusable()
297 * not support abort and has not given back the block yet. in iio_dma_block_reusable()
299 switch (block->state) { in iio_dma_block_reusable()
311 * Note that queue->num_dmabufs cannot increase while the queue is in iio_dma_buffer_can_use_fileio()
315 return queue->fileio.enabled || !atomic_read(&queue->num_dmabufs); in iio_dma_buffer_can_use_fileio()
319 * iio_dma_buffer_request_update() - DMA buffer request_update callback
328 struct iio_dma_buffer_block *block; in iio_dma_buffer_request_update() local
330 size_t size; in iio_dma_buffer_request_update() local
336 * buffering scheme with usually one block at a time being used by the in iio_dma_buffer_request_update()
339 size = DIV_ROUND_UP(queue->buffer.bytes_per_datum * in iio_dma_buffer_request_update()
340 queue->buffer.length, 2); in iio_dma_buffer_request_update()
342 mutex_lock(&queue->lock); in iio_dma_buffer_request_update()
344 queue->fileio.enabled = iio_dma_buffer_can_use_fileio(queue); in iio_dma_buffer_request_update()
347 if (!queue->fileio.enabled) in iio_dma_buffer_request_update()
351 if (PAGE_ALIGN(queue->fileio.block_size) == PAGE_ALIGN(size)) in iio_dma_buffer_request_update()
354 queue->fileio.block_size = size; in iio_dma_buffer_request_update()
355 queue->fileio.active_block = NULL; in iio_dma_buffer_request_update()
357 spin_lock_irq(&queue->list_lock); in iio_dma_buffer_request_update()
358 for (i = 0; i < ARRAY_SIZE(queue->fileio.blocks); i++) { in iio_dma_buffer_request_update()
359 block = queue->fileio.blocks[i]; in iio_dma_buffer_request_update()
361 /* If we can't re-use it free it */ in iio_dma_buffer_request_update()
362 if (block && (!iio_dma_block_reusable(block) || !try_reuse)) in iio_dma_buffer_request_update()
363 block->state = IIO_BLOCK_STATE_DEAD; in iio_dma_buffer_request_update()
371 spin_unlock_irq(&queue->list_lock); in iio_dma_buffer_request_update()
373 INIT_LIST_HEAD(&queue->incoming); in iio_dma_buffer_request_update()
375 for (i = 0; i < ARRAY_SIZE(queue->fileio.blocks); i++) { in iio_dma_buffer_request_update()
376 if (queue->fileio.blocks[i]) { in iio_dma_buffer_request_update()
377 block = queue->fileio.blocks[i]; in iio_dma_buffer_request_update()
378 if (block->state == IIO_BLOCK_STATE_DEAD) { in iio_dma_buffer_request_update()
380 iio_buffer_block_put(block); in iio_dma_buffer_request_update()
381 block = NULL; in iio_dma_buffer_request_update()
383 block->size = size; in iio_dma_buffer_request_update()
386 block = NULL; in iio_dma_buffer_request_update()
389 if (!block) { in iio_dma_buffer_request_update()
390 block = iio_dma_buffer_alloc_block(queue, size, true); in iio_dma_buffer_request_update()
391 if (!block) { in iio_dma_buffer_request_update()
392 ret = -ENOMEM; in iio_dma_buffer_request_update()
395 queue->fileio.blocks[i] = block; in iio_dma_buffer_request_update()
399 * block->bytes_used may have been modified previously, e.g. by in iio_dma_buffer_request_update()
401 * block's so that iio_dma_buffer_io() will work. in iio_dma_buffer_request_update()
403 block->bytes_used = block->size; in iio_dma_buffer_request_update()
406 * If it's an input buffer, mark the block as queued, and in iio_dma_buffer_request_update()
410 if (queue->buffer.direction == IIO_BUFFER_DIRECTION_IN) { in iio_dma_buffer_request_update()
411 block->state = IIO_BLOCK_STATE_QUEUED; in iio_dma_buffer_request_update()
412 list_add_tail(&block->head, &queue->incoming); in iio_dma_buffer_request_update()
414 block->state = IIO_BLOCK_STATE_DONE; in iio_dma_buffer_request_update()
419 mutex_unlock(&queue->lock); in iio_dma_buffer_request_update()
429 spin_lock_irq(&queue->list_lock); in iio_dma_buffer_fileio_free()
430 for (i = 0; i < ARRAY_SIZE(queue->fileio.blocks); i++) { in iio_dma_buffer_fileio_free()
431 if (!queue->fileio.blocks[i]) in iio_dma_buffer_fileio_free()
433 queue->fileio.blocks[i]->state = IIO_BLOCK_STATE_DEAD; in iio_dma_buffer_fileio_free()
435 spin_unlock_irq(&queue->list_lock); in iio_dma_buffer_fileio_free()
437 INIT_LIST_HEAD(&queue->incoming); in iio_dma_buffer_fileio_free()
439 for (i = 0; i < ARRAY_SIZE(queue->fileio.blocks); i++) { in iio_dma_buffer_fileio_free()
440 if (!queue->fileio.blocks[i]) in iio_dma_buffer_fileio_free()
442 iio_buffer_block_put(queue->fileio.blocks[i]); in iio_dma_buffer_fileio_free()
443 queue->fileio.blocks[i] = NULL; in iio_dma_buffer_fileio_free()
445 queue->fileio.active_block = NULL; in iio_dma_buffer_fileio_free()
449 struct iio_dma_buffer_block *block) in iio_dma_buffer_submit_block() argument
454 * If the hardware has already been removed we put the block into in iio_dma_buffer_submit_block()
458 if (!queue->ops) in iio_dma_buffer_submit_block()
461 block->state = IIO_BLOCK_STATE_ACTIVE; in iio_dma_buffer_submit_block()
462 iio_buffer_block_get(block); in iio_dma_buffer_submit_block()
464 ret = queue->ops->submit(queue, block); in iio_dma_buffer_submit_block()
466 if (!block->fileio) in iio_dma_buffer_submit_block()
467 iio_buffer_signal_dmabuf_done(block->fence, ret); in iio_dma_buffer_submit_block()
471 * other then wait for the buffer to be disabled and re-enabled in iio_dma_buffer_submit_block()
479 iio_buffer_block_put(block); in iio_dma_buffer_submit_block()
484 * iio_dma_buffer_enable() - Enable DMA buffer
497 struct iio_dma_buffer_block *block, *_block; in iio_dma_buffer_enable() local
499 mutex_lock(&queue->lock); in iio_dma_buffer_enable()
500 queue->active = true; in iio_dma_buffer_enable()
501 list_for_each_entry_safe(block, _block, &queue->incoming, head) { in iio_dma_buffer_enable()
502 list_del(&block->head); in iio_dma_buffer_enable()
503 iio_dma_buffer_submit_block(queue, block); in iio_dma_buffer_enable()
505 mutex_unlock(&queue->lock); in iio_dma_buffer_enable()
512 * iio_dma_buffer_disable() - Disable DMA buffer
524 mutex_lock(&queue->lock); in iio_dma_buffer_disable()
525 queue->active = false; in iio_dma_buffer_disable()
527 if (queue->ops && queue->ops->abort) in iio_dma_buffer_disable()
528 queue->ops->abort(queue); in iio_dma_buffer_disable()
529 mutex_unlock(&queue->lock); in iio_dma_buffer_disable()
536 struct iio_dma_buffer_block *block) in iio_dma_buffer_enqueue() argument
538 if (block->state == IIO_BLOCK_STATE_DEAD) { in iio_dma_buffer_enqueue()
539 iio_buffer_block_put(block); in iio_dma_buffer_enqueue()
540 } else if (queue->active) { in iio_dma_buffer_enqueue()
541 iio_dma_buffer_submit_block(queue, block); in iio_dma_buffer_enqueue()
543 block->state = IIO_BLOCK_STATE_QUEUED; in iio_dma_buffer_enqueue()
544 list_add_tail(&block->head, &queue->incoming); in iio_dma_buffer_enqueue()
551 struct iio_dma_buffer_block *block; in iio_dma_buffer_dequeue() local
554 spin_lock_irq(&queue->list_lock); in iio_dma_buffer_dequeue()
556 idx = queue->fileio.next_dequeue; in iio_dma_buffer_dequeue()
557 block = queue->fileio.blocks[idx]; in iio_dma_buffer_dequeue()
559 if (block->state == IIO_BLOCK_STATE_DONE) { in iio_dma_buffer_dequeue()
560 idx = (idx + 1) % ARRAY_SIZE(queue->fileio.blocks); in iio_dma_buffer_dequeue()
561 queue->fileio.next_dequeue = idx; in iio_dma_buffer_dequeue()
563 block = NULL; in iio_dma_buffer_dequeue()
566 spin_unlock_irq(&queue->list_lock); in iio_dma_buffer_dequeue()
568 return block; in iio_dma_buffer_dequeue()
575 struct iio_dma_buffer_block *block; in iio_dma_buffer_io() local
579 if (n < buffer->bytes_per_datum) in iio_dma_buffer_io()
580 return -EINVAL; in iio_dma_buffer_io()
582 mutex_lock(&queue->lock); in iio_dma_buffer_io()
584 if (!queue->fileio.active_block) { in iio_dma_buffer_io()
585 block = iio_dma_buffer_dequeue(queue); in iio_dma_buffer_io()
586 if (block == NULL) { in iio_dma_buffer_io()
590 queue->fileio.pos = 0; in iio_dma_buffer_io()
591 queue->fileio.active_block = block; in iio_dma_buffer_io()
593 block = queue->fileio.active_block; in iio_dma_buffer_io()
596 n = rounddown(n, buffer->bytes_per_datum); in iio_dma_buffer_io()
597 if (n > block->bytes_used - queue->fileio.pos) in iio_dma_buffer_io()
598 n = block->bytes_used - queue->fileio.pos; in iio_dma_buffer_io()
599 addr = block->vaddr + queue->fileio.pos; in iio_dma_buffer_io()
606 ret = -EFAULT; in iio_dma_buffer_io()
610 queue->fileio.pos += n; in iio_dma_buffer_io()
612 if (queue->fileio.pos == block->bytes_used) { in iio_dma_buffer_io()
613 queue->fileio.active_block = NULL; in iio_dma_buffer_io()
614 iio_dma_buffer_enqueue(queue, block); in iio_dma_buffer_io()
620 mutex_unlock(&queue->lock); in iio_dma_buffer_io()
626 * iio_dma_buffer_read() - DMA buffer read callback
642 * iio_dma_buffer_write() - DMA buffer write callback
659 * iio_dma_buffer_usage() - DMA buffer data_available and
669 struct iio_dma_buffer_block *block; in iio_dma_buffer_usage() local
674 * For counting the available bytes we'll use the size of the block not in iio_dma_buffer_usage()
675 * the number of actual bytes available in the block. Otherwise it is in iio_dma_buffer_usage()
680 mutex_lock(&queue->lock); in iio_dma_buffer_usage()
681 if (queue->fileio.active_block) in iio_dma_buffer_usage()
682 data_available += queue->fileio.active_block->size; in iio_dma_buffer_usage()
684 spin_lock_irq(&queue->list_lock); in iio_dma_buffer_usage()
686 for (i = 0; i < ARRAY_SIZE(queue->fileio.blocks); i++) { in iio_dma_buffer_usage()
687 block = queue->fileio.blocks[i]; in iio_dma_buffer_usage()
689 if (block != queue->fileio.active_block in iio_dma_buffer_usage()
690 && block->state == IIO_BLOCK_STATE_DONE) in iio_dma_buffer_usage()
691 data_available += block->size; in iio_dma_buffer_usage()
694 spin_unlock_irq(&queue->list_lock); in iio_dma_buffer_usage()
695 mutex_unlock(&queue->lock); in iio_dma_buffer_usage()
706 struct iio_dma_buffer_block *block; in iio_dma_buffer_attach_dmabuf() local
708 guard(mutex)(&queue->lock); in iio_dma_buffer_attach_dmabuf()
714 if (queue->fileio.enabled) in iio_dma_buffer_attach_dmabuf()
715 return ERR_PTR(-EBUSY); in iio_dma_buffer_attach_dmabuf()
717 block = iio_dma_buffer_alloc_block(queue, attach->dmabuf->size, false); in iio_dma_buffer_attach_dmabuf()
718 if (!block) in iio_dma_buffer_attach_dmabuf()
719 return ERR_PTR(-ENOMEM); in iio_dma_buffer_attach_dmabuf()
724 return block; in iio_dma_buffer_attach_dmabuf()
729 struct iio_dma_buffer_block *block) in iio_dma_buffer_detach_dmabuf() argument
731 block->state = IIO_BLOCK_STATE_DEAD; in iio_dma_buffer_detach_dmabuf()
732 iio_buffer_block_put_atomic(block); in iio_dma_buffer_detach_dmabuf()
736 static int iio_dma_can_enqueue_block(struct iio_dma_buffer_block *block) in iio_dma_can_enqueue_block() argument
738 struct iio_dma_buffer_queue *queue = block->queue; in iio_dma_can_enqueue_block()
741 if (queue->fileio.enabled) in iio_dma_can_enqueue_block()
742 return -EBUSY; in iio_dma_can_enqueue_block()
744 switch (block->state) { in iio_dma_can_enqueue_block()
746 return -EPERM; in iio_dma_can_enqueue_block()
749 return -EBUSY; in iio_dma_can_enqueue_block()
758 struct iio_dma_buffer_block *block, in iio_dma_buffer_enqueue_dmabuf() argument
761 size_t size, bool cyclic) in iio_dma_buffer_enqueue_dmabuf() argument
767 WARN_ON(!mutex_is_locked(&queue->lock)); in iio_dma_buffer_enqueue_dmabuf()
771 ret = iio_dma_can_enqueue_block(block); in iio_dma_buffer_enqueue_dmabuf()
775 block->bytes_used = size; in iio_dma_buffer_enqueue_dmabuf()
776 block->cyclic = cyclic; in iio_dma_buffer_enqueue_dmabuf()
777 block->sg_table = sgt; in iio_dma_buffer_enqueue_dmabuf()
778 block->fence = fence; in iio_dma_buffer_enqueue_dmabuf()
780 iio_dma_buffer_enqueue(queue, block); in iio_dma_buffer_enqueue_dmabuf()
793 mutex_lock(&queue->lock); in iio_dma_buffer_lock_queue()
801 mutex_unlock(&queue->lock); in iio_dma_buffer_unlock_queue()
806 * iio_dma_buffer_set_bytes_per_datum() - DMA buffer set_bytes_per_datum callback
807 * @buffer: Buffer to set the bytes-per-datum for
808 * @bpd: The new bytes-per-datum value
815 buffer->bytes_per_datum = bpd; in iio_dma_buffer_set_bytes_per_datum()
822 * iio_dma_buffer_set_length - DMA buffer set_length callback
834 buffer->length = length; in iio_dma_buffer_set_length()
835 buffer->watermark = length / 2; in iio_dma_buffer_set_length()
842 * iio_dma_buffer_init() - Initialize DMA buffer queue
854 iio_buffer_init(&queue->buffer); in iio_dma_buffer_init()
855 queue->buffer.length = PAGE_SIZE; in iio_dma_buffer_init()
856 queue->buffer.watermark = queue->buffer.length / 2; in iio_dma_buffer_init()
857 queue->dev = dev; in iio_dma_buffer_init()
858 queue->ops = ops; in iio_dma_buffer_init()
860 INIT_LIST_HEAD(&queue->incoming); in iio_dma_buffer_init()
862 mutex_init(&queue->lock); in iio_dma_buffer_init()
863 spin_lock_init(&queue->list_lock); in iio_dma_buffer_init()
870 * iio_dma_buffer_exit() - Cleanup DMA buffer queue
878 mutex_lock(&queue->lock); in iio_dma_buffer_exit()
881 queue->ops = NULL; in iio_dma_buffer_exit()
883 mutex_unlock(&queue->lock); in iio_dma_buffer_exit()
888 * iio_dma_buffer_release() - Release final buffer resources
897 mutex_destroy(&queue->lock); in iio_dma_buffer_release()
901 MODULE_AUTHOR("Lars-Peter Clausen <lars@metafoo.de>");