1 /* SPDX-License-Identifier: GPL-2.0-or-later */
3  * A generic kernel FIFO implementation
12  * How to porting drivers to the new generic FIFO API:
14  * - Modify the declaration of the "struct kfifo *" object into a
15  *   in-place "struct kfifo" object
16  * - Init the in-place object with kfifo_alloc() or kfifo_init()
17  *   Note: The address of the in-place "struct kfifo" object must be
19  * - Replace the use of __kfifo_put into kfifo_in and __kfifo_get
21  * - Replace the use of kfifo_put into kfifo_in_spinlocked and kfifo_get
26  * - The formerly __kfifo_* functions are renamed into kfifo_*
31  * and one writer is using the fifo and no kfifo_reset() will be called.
40 #include <linux/dma-mapping.h>
68 #define __STRUCT_KFIFO(type, size, recsize, ptrtype) \  argument
71 	type		buf[((size < 2) || (size & (size - 1))) ? -1 : size]; \
74 #define STRUCT_KFIFO(type, size) \  argument
75 	struct __STRUCT_KFIFO(type, size, 0, type)
91 #define STRUCT_KFIFO_REC_1(size) \  argument
92 	struct __STRUCT_KFIFO(unsigned char, size, 1, void)
94 #define STRUCT_KFIFO_REC_2(size) \  argument
95 	struct __STRUCT_KFIFO(unsigned char, size, 2, void)
104  * helper macro to distinguish between real in place fifo where the fifo
105  * array is a part of the structure and the fifo type where the array is
106  * outside of the fifo structure.
108 #define	__is_kfifo_ptr(fifo) \  argument
109 	(sizeof(*fifo) == sizeof(STRUCT_KFIFO_PTR(typeof(*(fifo)->type))))
112  * DECLARE_KFIFO_PTR - macro to declare a fifo pointer object
113  * @fifo: name of the declared fifo
114  * @type: type of the fifo elements
116 #define DECLARE_KFIFO_PTR(fifo, type)	STRUCT_KFIFO_PTR(type) fifo  argument
119  * DECLARE_KFIFO - macro to declare a fifo object
120  * @fifo: name of the declared fifo
121  * @type: type of the fifo elements
122  * @size: the number of elements in the fifo, this must be a power of 2
124 #define DECLARE_KFIFO(fifo, type, size)	STRUCT_KFIFO(type, size) fifo  argument
127  * INIT_KFIFO - Initialize a fifo declared by DECLARE_KFIFO
128  * @fifo: name of the declared fifo datatype
130 #define INIT_KFIFO(fifo) \  argument
132 	typeof(&(fifo)) __tmp = &(fifo); \
133 	struct __kfifo *__kfifo = &__tmp->kfifo; \
134 	__kfifo->in = 0; \
135 	__kfifo->out = 0; \
136 	__kfifo->mask = __is_kfifo_ptr(__tmp) ? 0 : ARRAY_SIZE(__tmp->buf) - 1;\
137 	__kfifo->esize = sizeof(*__tmp->buf); \
138 	__kfifo->data = __is_kfifo_ptr(__tmp) ?  NULL : __tmp->buf; \
142  * DEFINE_KFIFO - macro to define and initialize a fifo
143  * @fifo: name of the declared fifo datatype
144  * @type: type of the fifo elements
145  * @size: the number of elements in the fifo, this must be a power of 2
147  * Note: the macro can be used for global and local fifo data type variables.
149 #define DEFINE_KFIFO(fifo, type, size) \  argument
150 	DECLARE_KFIFO(fifo, type, size) = \
151 	(typeof(fifo)) { \
156 			.mask	= __is_kfifo_ptr(&(fifo)) ? \
158 				  ARRAY_SIZE((fifo).buf) - 1, \
159 			.esize	= sizeof(*(fifo).buf), \
160 			.data	= __is_kfifo_ptr(&(fifo)) ? \
162 				(fifo).buf, \
181  * kfifo_initialized - Check if the fifo is initialized
182  * @fifo: address of the fifo to check
184  * Return %true if fifo is initialized, otherwise %false.
185  * Assumes the fifo was 0 before.
187 #define kfifo_initialized(fifo) ((fifo)->kfifo.mask)  argument
190  * kfifo_esize - returns the size of the element managed by the fifo
191  * @fifo: address of the fifo to be used
193 #define kfifo_esize(fifo)	((fifo)->kfifo.esize)  argument
196  * kfifo_recsize - returns the size of the record length field
197  * @fifo: address of the fifo to be used
199 #define kfifo_recsize(fifo)	(sizeof(*(fifo)->rectype))  argument
202  * kfifo_size - returns the size of the fifo in elements
203  * @fifo: address of the fifo to be used
205 #define kfifo_size(fifo)	((fifo)->kfifo.mask + 1)  argument
208  * kfifo_reset - removes the entire fifo content
209  * @fifo: address of the fifo to be used
212  * fifo is exclusived locked or when it is secured that no other thread is
213  * accessing the fifo.
215 #define kfifo_reset(fifo) \  argument
217 	typeof((fifo) + 1) __tmp = (fifo); \
218 	__tmp->kfifo.in = __tmp->kfifo.out = 0; \
222  * kfifo_reset_out - skip fifo content
223  * @fifo: address of the fifo to be used
229 #define kfifo_reset_out(fifo)	\  argument
231 	typeof((fifo) + 1) __tmp = (fifo); \
232 	__tmp->kfifo.out = __tmp->kfifo.in; \
236  * kfifo_len - returns the number of used elements in the fifo
237  * @fifo: address of the fifo to be used
239 #define kfifo_len(fifo) \  argument
241 	typeof((fifo) + 1) __tmpl = (fifo); \
242 	__tmpl->kfifo.in - __tmpl->kfifo.out; \
246  * kfifo_is_empty - returns true if the fifo is empty
247  * @fifo: address of the fifo to be used
249 #define	kfifo_is_empty(fifo) \  argument
251 	typeof((fifo) + 1) __tmpq = (fifo); \
252 	__tmpq->kfifo.in == __tmpq->kfifo.out; \
256  * kfifo_is_empty_spinlocked - returns true if the fifo is empty using
258  * @fifo: address of the fifo to be used
261 #define kfifo_is_empty_spinlocked(fifo, lock) \  argument
266 	__ret = kfifo_is_empty(fifo); \
272  * kfifo_is_empty_spinlocked_noirqsave  - returns true if the fifo is empty
274  * @fifo: address of the fifo to be used
277 #define kfifo_is_empty_spinlocked_noirqsave(fifo, lock) \  argument
281 	__ret = kfifo_is_empty(fifo); \
287  * kfifo_is_full - returns true if the fifo is full
288  * @fifo: address of the fifo to be used
290 #define	kfifo_is_full(fifo) \  argument
292 	typeof((fifo) + 1) __tmpq = (fifo); \
293 	kfifo_len(__tmpq) > __tmpq->kfifo.mask; \
297  * kfifo_avail - returns the number of unused elements in the fifo
298  * @fifo: address of the fifo to be used
300 #define	kfifo_avail(fifo) \  argument
303 	typeof((fifo) + 1) __tmpq = (fifo); \
304 	const size_t __recsize = sizeof(*__tmpq->rectype); \
305 	unsigned int __avail = kfifo_size(__tmpq) - kfifo_len(__tmpq); \
307 	__kfifo_max_r(__avail - __recsize, __recsize)) : \
313  * kfifo_skip_count - skip output data
314  * @fifo: address of the fifo to be used
317 #define	kfifo_skip_count(fifo, count) do { \  argument
318 	typeof((fifo) + 1) __tmp = (fifo); \
319 	const size_t __recsize = sizeof(*__tmp->rectype); \
320 	struct __kfifo *__kfifo = &__tmp->kfifo; \
324 		__kfifo->out += (count); \
328  * kfifo_skip - skip output data
329  * @fifo: address of the fifo to be used
331 #define	kfifo_skip(fifo)	kfifo_skip_count(fifo, 1)  argument
334  * kfifo_peek_len - gets the size of the next fifo record
335  * @fifo: address of the fifo to be used
337  * This function returns the size of the next fifo record in number of bytes.
339 #define kfifo_peek_len(fifo) \  argument
342 	typeof((fifo) + 1) __tmp = (fifo); \
343 	const size_t __recsize = sizeof(*__tmp->rectype); \
344 	struct __kfifo *__kfifo = &__tmp->kfifo; \
345 	(!__recsize) ? kfifo_len(__tmp) * sizeof(*__tmp->type) : \
351  * kfifo_alloc - dynamically allocates a new fifo buffer
352  * @fifo: pointer to the fifo
353  * @size: the number of elements in the fifo, this must be a power of 2
356  * This macro dynamically allocates a new fifo buffer.
358  * The number of elements will be rounded-up to a power of 2.
359  * The fifo will be release with kfifo_free().
362 #define kfifo_alloc(fifo, size, gfp_mask) \  argument
365 	typeof((fifo) + 1) __tmp = (fifo); \
366 	struct __kfifo *__kfifo = &__tmp->kfifo; \
368 	__kfifo_alloc(__kfifo, size, sizeof(*__tmp->type), gfp_mask) : \
369 	-EINVAL; \
374  * kfifo_free - frees the fifo
375  * @fifo: the fifo to be freed
377 #define kfifo_free(fifo) \  argument
379 	typeof((fifo) + 1) __tmp = (fifo); \
380 	struct __kfifo *__kfifo = &__tmp->kfifo; \
386  * kfifo_init - initialize a fifo using a preallocated buffer
387  * @fifo: the fifo to assign the buffer
389  * @size: the size of the internal buffer, this have to be a power of 2
391  * This macro initializes a fifo using a preallocated buffer.
393  * The number of elements will be rounded-up to a power of 2.
396 #define kfifo_init(fifo, buffer, size) \  argument
398 	typeof((fifo) + 1) __tmp = (fifo); \
399 	struct __kfifo *__kfifo = &__tmp->kfifo; \
401 	__kfifo_init(__kfifo, buffer, size, sizeof(*__tmp->type)) : \
402 	-EINVAL; \
406  * kfifo_put - put data into the fifo
407  * @fifo: address of the fifo to be used
410  * This macro copies the given value into the fifo.
411  * It returns 0 if the fifo was full. Otherwise it returns the number
417 #define	kfifo_put(fifo, val) \  argument
419 	typeof((fifo) + 1) __tmp = (fifo); \
420 	typeof(*__tmp->const_type) __val = (val); \
422 	size_t __recsize = sizeof(*__tmp->rectype); \
423 	struct __kfifo *__kfifo = &__tmp->kfifo; \
431 			((typeof(__tmp->type))__kfifo->data) : \
432 			(__tmp->buf) \
433 			)[__kfifo->in & __tmp->kfifo.mask] = \
434 				*(typeof(__tmp->type))&__val; \
436 			__kfifo->in++; \
443  * kfifo_get - get data from the fifo
444  * @fifo: address of the fifo to be used
447  * This macro reads the data from the fifo.
448  * It returns 0 if the fifo was empty. Otherwise it returns the number
454 #define	kfifo_get(fifo, val) \  argument
457 	typeof((fifo) + 1) __tmp = (fifo); \
458 	typeof(__tmp->ptr) __val = (val); \
460 	const size_t __recsize = sizeof(*__tmp->rectype); \
461 	struct __kfifo *__kfifo = &__tmp->kfifo; \
468 			*(typeof(__tmp->type))__val = \
470 				((typeof(__tmp->type))__kfifo->data) : \
471 				(__tmp->buf) \
472 				)[__kfifo->out & __tmp->kfifo.mask]; \
474 			__kfifo->out++; \
482  * kfifo_peek - get data from the fifo without removing
483  * @fifo: address of the fifo to be used
486  * This reads the data from the fifo without removing it from the fifo.
487  * It returns 0 if the fifo was empty. Otherwise it returns the number
493 #define	kfifo_peek(fifo, val) \  argument
496 	typeof((fifo) + 1) __tmp = (fifo); \
497 	typeof(__tmp->ptr) __val = (val); \
499 	const size_t __recsize = sizeof(*__tmp->rectype); \
500 	struct __kfifo *__kfifo = &__tmp->kfifo; \
507 			*(typeof(__tmp->type))__val = \
509 				((typeof(__tmp->type))__kfifo->data) : \
510 				(__tmp->buf) \
511 				)[__kfifo->out & __tmp->kfifo.mask]; \
520  * kfifo_in - put data into the fifo
521  * @fifo: address of the fifo to be used
525  * This macro copies the given buffer into the fifo and returns the
531 #define	kfifo_in(fifo, buf, n) \  argument
533 	typeof((fifo) + 1) __tmp = (fifo); \
534 	typeof(__tmp->ptr_const) __buf = (buf); \
536 	const size_t __recsize = sizeof(*__tmp->rectype); \
537 	struct __kfifo *__kfifo = &__tmp->kfifo; \
544  * kfifo_in_spinlocked - put data into the fifo using a spinlock for locking
545  * @fifo: address of the fifo to be used
550  * This macro copies the given values buffer into the fifo and returns the
553 #define	kfifo_in_spinlocked(fifo, buf, n, lock) \  argument
558 	__ret = kfifo_in(fifo, buf, n); \
564  * kfifo_in_spinlocked_noirqsave - put data into fifo using a spinlock for
566  * @fifo: address of the fifo to be used
574 #define kfifo_in_spinlocked_noirqsave(fifo, buf, n, lock) \  argument
578 	__ret = kfifo_in(fifo, buf, n); \
584 #define kfifo_in_locked(fifo, buf, n, lock) \  argument
585 		kfifo_in_spinlocked(fifo, buf, n, lock)
588  * kfifo_out - get data from the fifo
589  * @fifo: address of the fifo to be used
593  * This macro gets some data from the fifo and returns the numbers of elements
599 #define	kfifo_out(fifo, buf, n) \  argument
602 	typeof((fifo) + 1) __tmp = (fifo); \
603 	typeof(__tmp->ptr) __buf = (buf); \
605 	const size_t __recsize = sizeof(*__tmp->rectype); \
606 	struct __kfifo *__kfifo = &__tmp->kfifo; \
614  * kfifo_out_spinlocked - get data from the fifo using a spinlock for locking
615  * @fifo: address of the fifo to be used
620  * This macro gets the data from the fifo and returns the numbers of elements
623 #define	kfifo_out_spinlocked(fifo, buf, n, lock) \  argument
629 	__ret = kfifo_out(fifo, buf, n); \
636  * kfifo_out_spinlocked_noirqsave - get data from the fifo using a spinlock
638  * @fifo: address of the fifo to be used
646 #define kfifo_out_spinlocked_noirqsave(fifo, buf, n, lock) \  argument
651 	__ret = kfifo_out(fifo, buf, n); \
658 #define kfifo_out_locked(fifo, buf, n, lock) \  argument
659 		kfifo_out_spinlocked(fifo, buf, n, lock)
662  * kfifo_from_user - puts some data from user space into the fifo
663  * @fifo: address of the fifo to be used
669  * fifo, depending of the available space and returns -EFAULT/0.
674 #define	kfifo_from_user(fifo, from, len, copied) \  argument
677 	typeof((fifo) + 1) __tmp = (fifo); \
681 	const size_t __recsize = sizeof(*__tmp->rectype); \
682 	struct __kfifo *__kfifo = &__tmp->kfifo; \
690  * kfifo_to_user - copies data from the fifo into user space
691  * @fifo: address of the fifo to be used
693  * @len: the size of the destination buffer
696  * This macro copies at most @len bytes from the fifo into the
697  * @to buffer and returns -EFAULT/0.
702 #define	kfifo_to_user(fifo, to, len, copied) \  argument
705 	typeof((fifo) + 1) __tmp = (fifo); \
709 	const size_t __recsize = sizeof(*__tmp->rectype); \
710 	struct __kfifo *__kfifo = &__tmp->kfifo; \
718  * kfifo_dma_in_prepare_mapped - setup a scatterlist for DMA input
719  * @fifo: address of the fifo to be used
731 #define	kfifo_dma_in_prepare_mapped(fifo, sgl, nents, len, dma) \  argument
733 	typeof((fifo) + 1) __tmp = (fifo); \
737 	const size_t __recsize = sizeof(*__tmp->rectype); \
738 	struct __kfifo *__kfifo = &__tmp->kfifo; \
745 #define kfifo_dma_in_prepare(fifo, sgl, nents, len) \  argument
746 	kfifo_dma_in_prepare_mapped(fifo, sgl, nents, len, DMA_MAPPING_ERROR)
749  * kfifo_dma_in_finish - finish a DMA IN operation
750  * @fifo: address of the fifo to be used
759 #define kfifo_dma_in_finish(fifo, len) \  argument
761 	typeof((fifo) + 1) __tmp = (fifo); \
763 	const size_t __recsize = sizeof(*__tmp->rectype); \
764 	struct __kfifo *__kfifo = &__tmp->kfifo; \
768 		__kfifo->in += __len / sizeof(*__tmp->type); \
772  * kfifo_dma_out_prepare_mapped - setup a scatterlist for DMA output
773  * @fifo: address of the fifo to be used
787 #define	kfifo_dma_out_prepare_mapped(fifo, sgl, nents, len, dma) \  argument
789 	typeof((fifo) + 1) __tmp = (fifo);  \
793 	const size_t __recsize = sizeof(*__tmp->rectype); \
794 	struct __kfifo *__kfifo = &__tmp->kfifo; \
801 #define	kfifo_dma_out_prepare(fifo, sgl, nents, len) \  argument
802 	kfifo_dma_out_prepare_mapped(fifo, sgl, nents, len, DMA_MAPPING_ERROR)
805  * kfifo_dma_out_finish - finish a DMA OUT operation
806  * @fifo: address of the fifo to be used
815 #define kfifo_dma_out_finish(fifo, len) do { \  argument
816 	typeof((fifo) + 1) ___tmp = (fifo); \
817 	kfifo_skip_count(___tmp, (len) / sizeof(*___tmp->type)); \
821  * kfifo_out_peek - gets some data from the fifo
822  * @fifo: address of the fifo to be used
826  * This macro gets the data from the fifo and returns the numbers of elements
827  * copied. The data is not removed from the fifo.
832 #define	kfifo_out_peek(fifo, buf, n) \  argument
835 	typeof((fifo) + 1) __tmp = (fifo); \
836 	typeof(__tmp->ptr) __buf = (buf); \
838 	const size_t __recsize = sizeof(*__tmp->rectype); \
839 	struct __kfifo *__kfifo = &__tmp->kfifo; \
847  * kfifo_out_linear - gets a tail of/offset to available data
848  * @fifo: address of the fifo to be used
852  * This macro obtains the offset (tail) to the available data in the fifo
856  * data processing (like memcpy() of (@fifo->data + @tail) with count
862 #define kfifo_out_linear(fifo, tail, n) \  argument
865 	typeof((fifo) + 1) __tmp = (fifo); \
868 	const size_t __recsize = sizeof(*__tmp->rectype); \
869 	struct __kfifo *__kfifo = &__tmp->kfifo; \
877  * kfifo_out_linear_ptr - gets a pointer to the available data
878  * @fifo: address of the fifo to be used
883  * available data in the fifo buffer and returns the numbers of elements
891 #define kfifo_out_linear_ptr(fifo, ptr, n) \  argument
894 	typeof((fifo) + 1) ___tmp = (fifo); \
897 	*(ptr) = ___tmp->kfifo.data + ___tail * kfifo_esize(___tmp); \
903 extern int __kfifo_alloc(struct __kfifo *fifo, unsigned int size,
906 extern void __kfifo_free(struct __kfifo *fifo);
908 extern int __kfifo_init(struct __kfifo *fifo, void *buffer,
909 	unsigned int size, size_t esize);
911 extern unsigned int __kfifo_in(struct __kfifo *fifo,
914 extern unsigned int __kfifo_out(struct __kfifo *fifo,
917 extern int __kfifo_from_user(struct __kfifo *fifo,
920 extern int __kfifo_to_user(struct __kfifo *fifo,
923 extern unsigned int __kfifo_dma_in_prepare(struct __kfifo *fifo,
926 extern unsigned int __kfifo_dma_out_prepare(struct __kfifo *fifo,
929 extern unsigned int __kfifo_out_peek(struct __kfifo *fifo,
932 extern unsigned int __kfifo_out_linear(struct __kfifo *fifo,
935 extern unsigned int __kfifo_in_r(struct __kfifo *fifo,
938 extern unsigned int __kfifo_out_r(struct __kfifo *fifo,
941 extern int __kfifo_from_user_r(struct __kfifo *fifo,
945 extern int __kfifo_to_user_r(struct __kfifo *fifo, void __user *to,
948 extern unsigned int __kfifo_dma_in_prepare_r(struct __kfifo *fifo,
952 extern void __kfifo_dma_in_finish_r(struct __kfifo *fifo,
955 extern unsigned int __kfifo_dma_out_prepare_r(struct __kfifo *fifo,
959 extern unsigned int __kfifo_len_r(struct __kfifo *fifo, size_t recsize);
961 extern void __kfifo_skip_r(struct __kfifo *fifo, size_t recsize);
963 extern unsigned int __kfifo_out_peek_r(struct __kfifo *fifo,
966 extern unsigned int __kfifo_out_linear_r(struct __kfifo *fifo,