Lines Matching +full:fifo +full:- +full:size
1 // SPDX-License-Identifier: GPL-2.0-or-later
3 * A generic kernel FIFO implementation
8 #include <linux/dma-mapping.h>
18 * internal helper to calculate the unused elements in a fifo
20 static inline unsigned int kfifo_unused(struct __kfifo *fifo) in kfifo_unused() argument
22 return (fifo->mask + 1) - (fifo->in - fifo->out); in kfifo_unused()
25 int __kfifo_alloc(struct __kfifo *fifo, unsigned int size, in __kfifo_alloc() argument
32 size = roundup_pow_of_two(size); in __kfifo_alloc()
34 fifo->in = 0; in __kfifo_alloc()
35 fifo->out = 0; in __kfifo_alloc()
36 fifo->esize = esize; in __kfifo_alloc()
38 if (size < 2) { in __kfifo_alloc()
39 fifo->data = NULL; in __kfifo_alloc()
40 fifo->mask = 0; in __kfifo_alloc()
41 return -EINVAL; in __kfifo_alloc()
44 fifo->data = kmalloc_array(esize, size, gfp_mask); in __kfifo_alloc()
46 if (!fifo->data) { in __kfifo_alloc()
47 fifo->mask = 0; in __kfifo_alloc()
48 return -ENOMEM; in __kfifo_alloc()
50 fifo->mask = size - 1; in __kfifo_alloc()
56 void __kfifo_free(struct __kfifo *fifo) in __kfifo_free() argument
58 kfree(fifo->data); in __kfifo_free()
59 fifo->in = 0; in __kfifo_free()
60 fifo->out = 0; in __kfifo_free()
61 fifo->esize = 0; in __kfifo_free()
62 fifo->data = NULL; in __kfifo_free()
63 fifo->mask = 0; in __kfifo_free()
67 int __kfifo_init(struct __kfifo *fifo, void *buffer, in __kfifo_init() argument
68 unsigned int size, size_t esize) in __kfifo_init() argument
70 size /= esize; in __kfifo_init()
72 if (!is_power_of_2(size)) in __kfifo_init()
73 size = rounddown_pow_of_two(size); in __kfifo_init()
75 fifo->in = 0; in __kfifo_init()
76 fifo->out = 0; in __kfifo_init()
77 fifo->esize = esize; in __kfifo_init()
78 fifo->data = buffer; in __kfifo_init()
80 if (size < 2) { in __kfifo_init()
81 fifo->mask = 0; in __kfifo_init()
82 return -EINVAL; in __kfifo_init()
84 fifo->mask = size - 1; in __kfifo_init()
90 static void kfifo_copy_in(struct __kfifo *fifo, const void *src, in kfifo_copy_in() argument
93 unsigned int size = fifo->mask + 1; in kfifo_copy_in() local
94 unsigned int esize = fifo->esize; in kfifo_copy_in()
97 off &= fifo->mask; in kfifo_copy_in()
100 size *= esize; in kfifo_copy_in()
103 l = min(len, size - off); in kfifo_copy_in()
105 memcpy(fifo->data + off, src, l); in kfifo_copy_in()
106 memcpy(fifo->data, src + l, len - l); in kfifo_copy_in()
108 * make sure that the data in the fifo is up to date before in kfifo_copy_in()
109 * incrementing the fifo->in index counter in kfifo_copy_in()
114 unsigned int __kfifo_in(struct __kfifo *fifo, in __kfifo_in() argument
119 l = kfifo_unused(fifo); in __kfifo_in()
123 kfifo_copy_in(fifo, buf, len, fifo->in); in __kfifo_in()
124 fifo->in += len; in __kfifo_in()
129 static void kfifo_copy_out(struct __kfifo *fifo, void *dst, in kfifo_copy_out() argument
132 unsigned int size = fifo->mask + 1; in kfifo_copy_out() local
133 unsigned int esize = fifo->esize; in kfifo_copy_out()
136 off &= fifo->mask; in kfifo_copy_out()
139 size *= esize; in kfifo_copy_out()
142 l = min(len, size - off); in kfifo_copy_out()
144 memcpy(dst, fifo->data + off, l); in kfifo_copy_out()
145 memcpy(dst + l, fifo->data, len - l); in kfifo_copy_out()
148 * incrementing the fifo->out index counter in kfifo_copy_out()
153 unsigned int __kfifo_out_peek(struct __kfifo *fifo, in __kfifo_out_peek() argument
158 l = fifo->in - fifo->out; in __kfifo_out_peek()
162 kfifo_copy_out(fifo, buf, len, fifo->out); in __kfifo_out_peek()
167 unsigned int __kfifo_out_linear(struct __kfifo *fifo, in __kfifo_out_linear() argument
170 unsigned int size = fifo->mask + 1; in __kfifo_out_linear() local
171 unsigned int off = fifo->out & fifo->mask; in __kfifo_out_linear()
176 return min3(n, fifo->in - fifo->out, size - off); in __kfifo_out_linear()
180 unsigned int __kfifo_out(struct __kfifo *fifo, in __kfifo_out() argument
183 len = __kfifo_out_peek(fifo, buf, len); in __kfifo_out()
184 fifo->out += len; in __kfifo_out()
189 static unsigned long kfifo_copy_from_user(struct __kfifo *fifo, in kfifo_copy_from_user() argument
193 unsigned int size = fifo->mask + 1; in kfifo_copy_from_user() local
194 unsigned int esize = fifo->esize; in kfifo_copy_from_user()
198 off &= fifo->mask; in kfifo_copy_from_user()
201 size *= esize; in kfifo_copy_from_user()
204 l = min(len, size - off); in kfifo_copy_from_user()
206 ret = copy_from_user(fifo->data + off, from, l); in kfifo_copy_from_user()
208 ret = DIV_ROUND_UP(ret + len - l, esize); in kfifo_copy_from_user()
210 ret = copy_from_user(fifo->data, from + l, len - l); in kfifo_copy_from_user()
215 * make sure that the data in the fifo is up to date before in kfifo_copy_from_user()
216 * incrementing the fifo->in index counter in kfifo_copy_from_user()
219 *copied = len - ret * esize; in kfifo_copy_from_user()
224 int __kfifo_from_user(struct __kfifo *fifo, const void __user *from, in __kfifo_from_user() argument
229 unsigned int esize = fifo->esize; in __kfifo_from_user()
235 l = kfifo_unused(fifo); in __kfifo_from_user()
239 ret = kfifo_copy_from_user(fifo, from, len, fifo->in, copied); in __kfifo_from_user()
241 len -= ret; in __kfifo_from_user()
242 err = -EFAULT; in __kfifo_from_user()
245 fifo->in += len; in __kfifo_from_user()
250 static unsigned long kfifo_copy_to_user(struct __kfifo *fifo, void __user *to, in kfifo_copy_to_user() argument
255 unsigned int size = fifo->mask + 1; in kfifo_copy_to_user() local
256 unsigned int esize = fifo->esize; in kfifo_copy_to_user()
258 off &= fifo->mask; in kfifo_copy_to_user()
261 size *= esize; in kfifo_copy_to_user()
264 l = min(len, size - off); in kfifo_copy_to_user()
266 ret = copy_to_user(to, fifo->data + off, l); in kfifo_copy_to_user()
268 ret = DIV_ROUND_UP(ret + len - l, esize); in kfifo_copy_to_user()
270 ret = copy_to_user(to + l, fifo->data, len - l); in kfifo_copy_to_user()
276 * incrementing the fifo->out index counter in kfifo_copy_to_user()
279 *copied = len - ret * esize; in kfifo_copy_to_user()
284 int __kfifo_to_user(struct __kfifo *fifo, void __user *to, in __kfifo_to_user() argument
289 unsigned int esize = fifo->esize; in __kfifo_to_user()
295 l = fifo->in - fifo->out; in __kfifo_to_user()
298 ret = kfifo_copy_to_user(fifo, to, len, fifo->out, copied); in __kfifo_to_user()
300 len -= ret; in __kfifo_to_user()
301 err = -EFAULT; in __kfifo_to_user()
304 fifo->out += len; in __kfifo_to_user()
309 static unsigned int setup_sgl_buf(struct __kfifo *fifo, struct scatterlist *sgl, in setup_sgl_buf() argument
313 const void *buf = fifo->data + data_offset; in setup_sgl_buf()
328 static unsigned int setup_sgl(struct __kfifo *fifo, struct scatterlist *sgl, in setup_sgl() argument
331 unsigned int size = fifo->mask + 1; in setup_sgl() local
332 unsigned int esize = fifo->esize; in setup_sgl()
336 off &= fifo->mask; in setup_sgl()
339 size *= esize; in setup_sgl()
342 len_to_end = min(len, size - off); in setup_sgl()
344 n = setup_sgl_buf(fifo, sgl, off, nents, len_to_end, dma); in setup_sgl()
345 n += setup_sgl_buf(fifo, sgl + n, 0, nents - n, len - len_to_end, dma); in setup_sgl()
350 unsigned int __kfifo_dma_in_prepare(struct __kfifo *fifo, in __kfifo_dma_in_prepare() argument
356 l = kfifo_unused(fifo); in __kfifo_dma_in_prepare()
360 return setup_sgl(fifo, sgl, nents, len, fifo->in, dma); in __kfifo_dma_in_prepare()
364 unsigned int __kfifo_dma_out_prepare(struct __kfifo *fifo, in __kfifo_dma_out_prepare() argument
370 l = fifo->in - fifo->out; in __kfifo_dma_out_prepare()
374 return setup_sgl(fifo, sgl, nents, len, fifo->out, dma); in __kfifo_dma_out_prepare()
380 unsigned int max = (1 << (recsize << 3)) - 1; in __kfifo_max_r()
392 * the next record in the fifo
394 static unsigned int __kfifo_peek_n(struct __kfifo *fifo, size_t recsize) in __kfifo_peek_n() argument
397 unsigned int mask = fifo->mask; in __kfifo_peek_n()
398 unsigned char *data = fifo->data; in __kfifo_peek_n()
400 l = __KFIFO_PEEK(data, fifo->out, mask); in __kfifo_peek_n()
402 if (--recsize) in __kfifo_peek_n()
403 l |= __KFIFO_PEEK(data, fifo->out + 1, mask) << 8; in __kfifo_peek_n()
415 * the record into the fifo
417 static void __kfifo_poke_n(struct __kfifo *fifo, unsigned int n, size_t recsize) in __kfifo_poke_n() argument
419 unsigned int mask = fifo->mask; in __kfifo_poke_n()
420 unsigned char *data = fifo->data; in __kfifo_poke_n()
422 __KFIFO_POKE(data, fifo->in, mask, n); in __kfifo_poke_n()
425 __KFIFO_POKE(data, fifo->in + 1, mask, n >> 8); in __kfifo_poke_n()
428 unsigned int __kfifo_len_r(struct __kfifo *fifo, size_t recsize) in __kfifo_len_r() argument
430 return __kfifo_peek_n(fifo, recsize); in __kfifo_len_r()
434 unsigned int __kfifo_in_r(struct __kfifo *fifo, const void *buf, in __kfifo_in_r() argument
437 if (len + recsize > kfifo_unused(fifo)) in __kfifo_in_r()
440 __kfifo_poke_n(fifo, len, recsize); in __kfifo_in_r()
442 kfifo_copy_in(fifo, buf, len, fifo->in + recsize); in __kfifo_in_r()
443 fifo->in += len + recsize; in __kfifo_in_r()
448 static unsigned int kfifo_out_copy_r(struct __kfifo *fifo, in kfifo_out_copy_r() argument
451 *n = __kfifo_peek_n(fifo, recsize); in kfifo_out_copy_r()
456 kfifo_copy_out(fifo, buf, len, fifo->out + recsize); in kfifo_out_copy_r()
460 unsigned int __kfifo_out_peek_r(struct __kfifo *fifo, void *buf, in __kfifo_out_peek_r() argument
465 if (fifo->in == fifo->out) in __kfifo_out_peek_r()
468 return kfifo_out_copy_r(fifo, buf, len, recsize, &n); in __kfifo_out_peek_r()
472 unsigned int __kfifo_out_linear_r(struct __kfifo *fifo, in __kfifo_out_linear_r() argument
475 if (fifo->in == fifo->out) in __kfifo_out_linear_r()
479 *tail = fifo->out + recsize; in __kfifo_out_linear_r()
481 return min(n, __kfifo_peek_n(fifo, recsize)); in __kfifo_out_linear_r()
485 unsigned int __kfifo_out_r(struct __kfifo *fifo, void *buf, in __kfifo_out_r() argument
490 if (fifo->in == fifo->out) in __kfifo_out_r()
493 len = kfifo_out_copy_r(fifo, buf, len, recsize, &n); in __kfifo_out_r()
494 fifo->out += n + recsize; in __kfifo_out_r()
499 void __kfifo_skip_r(struct __kfifo *fifo, size_t recsize) in __kfifo_skip_r() argument
503 n = __kfifo_peek_n(fifo, recsize); in __kfifo_skip_r()
504 fifo->out += n + recsize; in __kfifo_skip_r()
508 int __kfifo_from_user_r(struct __kfifo *fifo, const void __user *from, in __kfifo_from_user_r() argument
515 if (len + recsize > kfifo_unused(fifo)) { in __kfifo_from_user_r()
520 __kfifo_poke_n(fifo, len, recsize); in __kfifo_from_user_r()
522 ret = kfifo_copy_from_user(fifo, from, len, fifo->in + recsize, copied); in __kfifo_from_user_r()
525 return -EFAULT; in __kfifo_from_user_r()
527 fifo->in += len + recsize; in __kfifo_from_user_r()
532 int __kfifo_to_user_r(struct __kfifo *fifo, void __user *to, in __kfifo_to_user_r() argument
538 if (fifo->in == fifo->out) { in __kfifo_to_user_r()
543 n = __kfifo_peek_n(fifo, recsize); in __kfifo_to_user_r()
547 ret = kfifo_copy_to_user(fifo, to, len, fifo->out + recsize, copied); in __kfifo_to_user_r()
550 return -EFAULT; in __kfifo_to_user_r()
552 fifo->out += n + recsize; in __kfifo_to_user_r()
557 unsigned int __kfifo_dma_in_prepare_r(struct __kfifo *fifo, in __kfifo_dma_in_prepare_r() argument
565 if (len + recsize > kfifo_unused(fifo)) in __kfifo_dma_in_prepare_r()
568 return setup_sgl(fifo, sgl, nents, len, fifo->in + recsize, dma); in __kfifo_dma_in_prepare_r()
572 void __kfifo_dma_in_finish_r(struct __kfifo *fifo, in __kfifo_dma_in_finish_r() argument
576 __kfifo_poke_n(fifo, len, recsize); in __kfifo_dma_in_finish_r()
577 fifo->in += len + recsize; in __kfifo_dma_in_finish_r()
581 unsigned int __kfifo_dma_out_prepare_r(struct __kfifo *fifo, in __kfifo_dma_out_prepare_r() argument
589 if (len + recsize > fifo->in - fifo->out) in __kfifo_dma_out_prepare_r()
592 return setup_sgl(fifo, sgl, nents, len, fifo->out + recsize, dma); in __kfifo_dma_out_prepare_r()