Lines Matching +full:step +full:- +full:up
1 /* SPDX-License-Identifier: GPL-2.0-or-later */
25 iov_ustep_f step) in iterate_ubuf() argument
27 void __user *base = iter->ubuf; in iterate_ubuf()
30 remain = step(base + iter->iov_offset, 0, len, priv, priv2); in iterate_ubuf()
31 progress = len - remain; in iterate_ubuf()
32 iter->iov_offset += progress; in iterate_ubuf()
33 iter->count -= progress; in iterate_ubuf()
42 iov_ustep_f step) in iterate_iovec() argument
44 const struct iovec *p = iter->__iov; in iterate_iovec()
45 size_t progress = 0, skip = iter->iov_offset; in iterate_iovec()
49 size_t part = min(len, p->iov_len - skip); in iterate_iovec()
52 remain = step(p->iov_base + skip, progress, part, priv, priv2); in iterate_iovec()
53 consumed = part - remain; in iterate_iovec()
56 len -= consumed; in iterate_iovec()
57 if (skip < p->iov_len) in iterate_iovec()
64 iter->nr_segs -= p - iter->__iov; in iterate_iovec()
65 iter->__iov = p; in iterate_iovec()
66 iter->iov_offset = skip; in iterate_iovec()
67 iter->count -= progress; in iterate_iovec()
76 iov_step_f step) in iterate_kvec() argument
78 const struct kvec *p = iter->kvec; in iterate_kvec()
79 size_t progress = 0, skip = iter->iov_offset; in iterate_kvec()
83 size_t part = min(len, p->iov_len - skip); in iterate_kvec()
86 remain = step(p->iov_base + skip, progress, part, priv, priv2); in iterate_kvec()
87 consumed = part - remain; in iterate_kvec()
90 len -= consumed; in iterate_kvec()
91 if (skip < p->iov_len) in iterate_kvec()
98 iter->nr_segs -= p - iter->kvec; in iterate_kvec()
99 iter->kvec = p; in iterate_kvec()
100 iter->iov_offset = skip; in iterate_kvec()
101 iter->count -= progress; in iterate_kvec()
110 iov_step_f step) in iterate_bvec() argument
112 const struct bio_vec *p = iter->bvec; in iterate_bvec()
113 size_t progress = 0, skip = iter->iov_offset; in iterate_bvec()
117 size_t offset = p->bv_offset + skip, part; in iterate_bvec()
118 void *kaddr = kmap_local_page(p->bv_page + offset / PAGE_SIZE); in iterate_bvec()
121 (size_t)(p->bv_len - skip), in iterate_bvec()
122 (size_t)(PAGE_SIZE - offset % PAGE_SIZE)); in iterate_bvec()
123 remain = step(kaddr + offset % PAGE_SIZE, progress, part, priv, priv2); in iterate_bvec()
125 consumed = part - remain; in iterate_bvec()
126 len -= consumed; in iterate_bvec()
129 if (skip >= p->bv_len) { in iterate_bvec()
137 iter->nr_segs -= p - iter->bvec; in iterate_bvec()
138 iter->bvec = p; in iterate_bvec()
139 iter->iov_offset = skip; in iterate_bvec()
140 iter->count -= progress; in iterate_bvec()
149 iov_step_f step) in iterate_folioq() argument
151 const struct folio_queue *folioq = iter->folioq; in iterate_folioq()
152 unsigned int slot = iter->folioq_slot; in iterate_folioq()
153 size_t progress = 0, skip = iter->iov_offset; in iterate_folioq()
157 folioq = folioq->next; in iterate_folioq()
172 part = umin(len, PAGE_SIZE - skip % PAGE_SIZE); in iterate_folioq()
173 remain = step(base, progress, part, priv, priv2); in iterate_folioq()
175 consumed = part - remain; in iterate_folioq()
176 len -= consumed; in iterate_folioq()
182 if (slot == folioq_nr_slots(folioq) && folioq->next) { in iterate_folioq()
183 folioq = folioq->next; in iterate_folioq()
191 iter->folioq_slot = slot; in iterate_folioq()
192 iter->folioq = folioq; in iterate_folioq()
193 iter->iov_offset = skip; in iterate_folioq()
194 iter->count -= progress; in iterate_folioq()
203 iov_step_f step) in iterate_xarray() argument
207 loff_t start = iter->xarray_start + iter->iov_offset; in iterate_xarray()
209 XA_STATE(xas, iter->xarray, index); in iterate_xarray()
223 flen = min(folio_size(folio) - offset, len); in iterate_xarray()
229 PAGE_SIZE - offset_in_page(offset)); in iterate_xarray()
230 remain = step(base, progress, part, priv, priv2); in iterate_xarray()
233 consumed = part - remain; in iterate_xarray()
235 len -= consumed; in iterate_xarray()
239 flen -= consumed; in iterate_xarray()
246 iter->iov_offset += progress; in iterate_xarray()
247 iter->count -= progress; in iterate_xarray()
256 iov_step_f step) in iterate_discard() argument
260 iter->count -= progress; in iterate_discard()
265 * iterate_and_advance2 - Iterate over an iterator
268 * @priv: Data for the step functions.
269 * @priv2: More data for the step functions.
271 * @step: Function for other iterators; given kernel addresses.
273 * Iterate over the next part of an iterator, up to the specified length. The
274 * buffer is presented in segments, which for kernel iteration are broken up by
277 * Two step functions, @step and @ustep, must be provided, one for handling
281 * The step functions are passed the address and length of the segment, @priv,
283 * be added to @priv to point to the right part of a second buffer). The step
292 void *priv2, iov_ustep_f ustep, iov_step_f step) in iterate_and_advance2() argument
294 if (unlikely(iter->count < len)) in iterate_and_advance2()
295 len = iter->count; in iterate_and_advance2()
304 return iterate_bvec(iter, len, priv, priv2, step); in iterate_and_advance2()
306 return iterate_kvec(iter, len, priv, priv2, step); in iterate_and_advance2()
308 return iterate_folioq(iter, len, priv, priv2, step); in iterate_and_advance2()
310 return iterate_xarray(iter, len, priv, priv2, step); in iterate_and_advance2()
311 return iterate_discard(iter, len, priv, priv2, step); in iterate_and_advance2()
315 * iterate_and_advance - Iterate over an iterator
318 * @priv: Data for the step functions.
320 * @step: Function for other iterators; given kernel addresses.
326 iov_ustep_f ustep, iov_step_f step) in iterate_and_advance() argument
328 return iterate_and_advance2(iter, len, priv, NULL, ustep, step); in iterate_and_advance()
332 * iterate_and_advance_kernel - Iterate over a kernel-internal iterator
335 * @priv: Data for the step functions.
336 * @priv2: More data for the step functions.
337 * @step: Function for other iterators; given kernel addresses.
339 * Iterate over the next part of an iterator, up to the specified length. The
340 * buffer is presented in segments, which for kernel iteration are broken up by
343 * [!] Note This will only handle BVEC, KVEC, FOLIOQ, XARRAY and DISCARD-type
344 * iterators; it will not handle UBUF or IOVEC-type iterators.
346 * A step functions, @step, must be provided, one for handling mapped kernel
350 * The step functions are passed the address and length of the segment, @priv,
352 * be added to @priv to point to the right part of a second buffer). The step
361 void *priv2, iov_step_f step) in iterate_and_advance_kernel() argument
363 if (unlikely(iter->count < len)) in iterate_and_advance_kernel()
364 len = iter->count; in iterate_and_advance_kernel()
368 return iterate_bvec(iter, len, priv, priv2, step); in iterate_and_advance_kernel()
370 return iterate_kvec(iter, len, priv, priv2, step); in iterate_and_advance_kernel()
372 return iterate_folioq(iter, len, priv, priv2, step); in iterate_and_advance_kernel()
374 return iterate_xarray(iter, len, priv, priv2, step); in iterate_and_advance_kernel()
375 return iterate_discard(iter, len, priv, priv2, step); in iterate_and_advance_kernel()