1 // SPDX-License-Identifier: GPL-2.0
14  * struct iova_bitmap_map - A bitmap representing an IOVA range
55  * struct iova_bitmap - The IOVA bitmap object
72  *    4G  -  68G-1 (64G)
73  *    68G - 128G-1 (64G)
97  * The IOVA bitmap is usually located on what tracks DMA mapped ranges or
98  * some form of IOVA range tracking that co-relates to the user passed
133 	unsigned long pgsize = 1 << bitmap->mapped.pgshift;  in iova_bitmap_offset_to_index()
135 	return iova / (BITS_PER_TYPE(*bitmap->bitmap) * pgsize);  in iova_bitmap_offset_to_index()
144 	unsigned long pgshift = bitmap->mapped.pgshift;  in iova_bitmap_index_to_offset()
146 	return (index * BITS_PER_TYPE(*bitmap->bitmap)) << pgshift;  in iova_bitmap_index_to_offset()
154 	unsigned long skip = bitmap->mapped_base_index;  in iova_bitmap_mapped_iova()
156 	return bitmap->iova + iova_bitmap_index_to_offset(bitmap, skip);  in iova_bitmap_mapped_iova()
168 	struct iova_bitmap_map *mapped = &bitmap->mapped;  in iova_bitmap_get()
176 	 * mapped. The range @mapped_base_index .. @mapped_total_index-1 is  in iova_bitmap_get()
179 	npages = DIV_ROUND_UP((bitmap->mapped_total_index -  in iova_bitmap_get()
180 			       bitmap->mapped_base_index) *  in iova_bitmap_get()
181 			       sizeof(*bitmap->bitmap), PAGE_SIZE);  in iova_bitmap_get()
187 	addr = bitmap->bitmap + bitmap->mapped_base_index;  in iova_bitmap_get()
197 				  FOLL_WRITE, mapped->pages);  in iova_bitmap_get()
199 		return -EFAULT;  in iova_bitmap_get()
201 	mapped->npages = (unsigned long)ret;  in iova_bitmap_get()
203 	mapped->iova = iova_bitmap_mapped_iova(bitmap);  in iova_bitmap_get()
206 	 * offset of the page where pinned pages bit 0 is located.  in iova_bitmap_get()
210 	mapped->pgoff = offset_in_page(addr);  in iova_bitmap_get()
211 	mapped->length = iova_bitmap_mapped_length(bitmap);  in iova_bitmap_get()
222 	struct iova_bitmap_map *mapped = &bitmap->mapped;  in iova_bitmap_put()
224 	if (mapped->npages) {  in iova_bitmap_put()
225 		unpin_user_pages(mapped->pages, mapped->npages);  in iova_bitmap_put()
226 		mapped->npages = 0;  in iova_bitmap_put()
231  * iova_bitmap_alloc() - Allocates an IOVA bitmap object
253 		return ERR_PTR(-ENOMEM);  in iova_bitmap_alloc()
255 	mapped = &bitmap->mapped;  in iova_bitmap_alloc()
256 	mapped->pgshift = __ffs(page_size);  in iova_bitmap_alloc()
257 	bitmap->bitmap = (u8 __user *)data;  in iova_bitmap_alloc()
258 	bitmap->mapped_total_index =  in iova_bitmap_alloc()
259 		iova_bitmap_offset_to_index(bitmap, length - 1) + 1;  in iova_bitmap_alloc()
260 	bitmap->iova = iova;  in iova_bitmap_alloc()
261 	bitmap->length = length;  in iova_bitmap_alloc()
262 	mapped->iova = iova;  in iova_bitmap_alloc()
263 	mapped->pages = (struct page **)__get_free_page(GFP_KERNEL);  in iova_bitmap_alloc()
264 	if (!mapped->pages) {  in iova_bitmap_alloc()
265 		rc = -ENOMEM;  in iova_bitmap_alloc()
278  * iova_bitmap_free() - Frees an IOVA bitmap object
286 	struct iova_bitmap_map *mapped = &bitmap->mapped;  in iova_bitmap_free()
290 	if (mapped->pages) {  in iova_bitmap_free()
291 		free_page((unsigned long)mapped->pages);  in iova_bitmap_free()
292 		mapped->pages = NULL;  in iova_bitmap_free()
307 	bytes = (bitmap->mapped.npages << PAGE_SHIFT) - bitmap->mapped.pgoff;  in iova_bitmap_mapped_remaining()
309 	remaining = bitmap->mapped_total_index - bitmap->mapped_base_index;  in iova_bitmap_mapped_remaining()
311 			  DIV_ROUND_UP(bytes, sizeof(*bitmap->bitmap)));  in iova_bitmap_mapped_remaining()
321 	unsigned long max_iova = bitmap->iova + bitmap->length - 1;  in iova_bitmap_mapped_length()
334 	if (iova + remaining - 1 > max_iova)  in iova_bitmap_mapped_length()
335 		remaining -= ((iova + remaining - 1) - max_iova);  in iova_bitmap_mapped_length()
341  * Returns true if [@iova..@iova+@length-1] is part of the mapped IOVA range.
346 	return mapped->npages &&  in iova_bitmap_mapped_range()
347 		(iova >= mapped->iova &&  in iova_bitmap_mapped_range()
348 		 (iova + length - 1) <= (mapped->iova + mapped->length - 1));  in iova_bitmap_mapped_range()
361 	index = iova_bitmap_offset_to_index(bitmap, iova - bitmap->iova);  in iova_bitmap_advance_to()
362 	if (index >= bitmap->mapped_total_index)  in iova_bitmap_advance_to()
363 		return -EINVAL;  in iova_bitmap_advance_to()
364 	bitmap->mapped_base_index = index;  in iova_bitmap_advance_to()
373  * iova_bitmap_for_each() - Iterates over the bitmap
388 	return fn(bitmap, bitmap->iova, bitmap->length, opaque);  in iova_bitmap_for_each()
393  * iova_bitmap_set() - Records an IOVA range in bitmap
398  * Set the bits corresponding to the range [iova .. iova+length-1] in
405 	struct iova_bitmap_map *mapped = &bitmap->mapped;  in iova_bitmap_set()
420 	last_page_idx = mapped->npages - 1;  in iova_bitmap_set()
421 	cur_bit = ((iova - mapped->iova) >>  in iova_bitmap_set()
422 		mapped->pgshift) + mapped->pgoff * BITS_PER_BYTE;  in iova_bitmap_set()
423 	last_bit = (((iova + length - 1) - mapped->iova) >>  in iova_bitmap_set()
424 		mapped->pgshift) + mapped->pgoff * BITS_PER_BYTE;  in iova_bitmap_set()
429 		unsigned int nbits = min(BITS_PER_PAGE - offset,  in iova_bitmap_set()
430 					 last_bit - cur_bit + 1);  in iova_bitmap_set()
435 				((last_bit - cur_bit + 1) << mapped->pgshift);  in iova_bitmap_set()
437 			iova += (length - left);  in iova_bitmap_set()
442 		kaddr = kmap_local_page(mapped->pages[page_idx]);  in iova_bitmap_set()