16  * Main data structure for tracking mapped user pages of bitmap data.
38 	/* mapped length */
93  * which u64 word of the bitmap is mapped, up to @mapped_total_index.
97  * The IOVA bitmap is usually located on what tracks DMA mapped ranges or
102 	/* IOVA range representing the currently mapped bitmap data */
103 	struct iova_bitmap_map mapped;  member
108 	/* u64 index that @mapped points to */
125  * Relative IOVA means relative to the bitmap::mapped base IOVA
126  * (stored in mapped::iova). All computations in this file are done using
127  * relative IOVAs and thus avoid an extra subtraction against mapped::iova.
133 	unsigned long pgsize = 1 << bitmap->mapped.pgshift;  in iova_bitmap_offset_to_index()
144 	unsigned long pgshift = bitmap->mapped.pgshift;  in iova_bitmap_index_to_offset()
150  * Returns the base IOVA of the mapped range.
168 	struct iova_bitmap_map *mapped = &bitmap->mapped;  in iova_bitmap_get()  local
174 	 * @mapped_base_index is the index of the currently mapped u64 words  in iova_bitmap_get()
176 	 * mapped. The range @mapped_base_index .. @mapped_total_index-1 is  in iova_bitmap_get()
177 	 * mapped but capped at a maximum number of pages.  in iova_bitmap_get()
197 				  FOLL_WRITE, mapped->pages);  in iova_bitmap_get()
201 	mapped->npages = (unsigned long)ret;  in iova_bitmap_get()
203 	mapped->iova = iova_bitmap_mapped_iova(bitmap);  in iova_bitmap_get()
210 	mapped->pgoff = offset_in_page(addr);  in iova_bitmap_get()
211 	mapped->length = iova_bitmap_mapped_length(bitmap);  in iova_bitmap_get()
222 	struct iova_bitmap_map *mapped = &bitmap->mapped;  in iova_bitmap_put()  local
224 	if (mapped->npages) {  in iova_bitmap_put()
225 		unpin_user_pages(mapped->pages, mapped->npages);  in iova_bitmap_put()
226 		mapped->npages = 0;  in iova_bitmap_put()
247 	struct iova_bitmap_map *mapped;  in iova_bitmap_alloc()  local
255 	mapped = &bitmap->mapped;  in iova_bitmap_alloc()
256 	mapped->pgshift = __ffs(page_size);  in iova_bitmap_alloc()
262 	mapped->iova = iova;  in iova_bitmap_alloc()
263 	mapped->pages = (struct page **)__get_free_page(GFP_KERNEL);  in iova_bitmap_alloc()
264 	if (!mapped->pages) {  in iova_bitmap_alloc()
286 	struct iova_bitmap_map *mapped = &bitmap->mapped;  in iova_bitmap_free()  local
290 	if (mapped->pages) {  in iova_bitmap_free()
291 		free_page((unsigned long)mapped->pages);  in iova_bitmap_free()
292 		mapped->pages = NULL;  in iova_bitmap_free()
307 	bytes = (bitmap->mapped.npages << PAGE_SHIFT) - bitmap->mapped.pgoff;  in iova_bitmap_mapped_remaining()
317  * Returns the length of the mapped IOVA range.
341  * Returns true if [@iova..@iova+@length-1] is part of the mapped IOVA range.
343 static bool iova_bitmap_mapped_range(struct iova_bitmap_map *mapped,  in iova_bitmap_mapped_range()  argument
346 	return mapped->npages &&  in iova_bitmap_mapped_range()
347 		(iova >= mapped->iova &&  in iova_bitmap_mapped_range()
348 		 (iova + length - 1) <= (mapped->iova + mapped->length - 1));  in iova_bitmap_mapped_range()
380  * mapped bitmap user pages into IOVA ranges to process.
405 	struct iova_bitmap_map *mapped = &bitmap->mapped;  in iova_bitmap_set()  local
409 	if (unlikely(!iova_bitmap_mapped_range(mapped, iova, length))) {  in iova_bitmap_set()
420 	last_page_idx = mapped->npages - 1;  in iova_bitmap_set()
421 	cur_bit = ((iova - mapped->iova) >>  in iova_bitmap_set()
422 		mapped->pgshift) + mapped->pgoff * BITS_PER_BYTE;  in iova_bitmap_set()
423 	last_bit = (((iova + length - 1) - mapped->iova) >>  in iova_bitmap_set()
424 		mapped->pgshift) + mapped->pgoff * BITS_PER_BYTE;  in iova_bitmap_set()
435 				((last_bit - cur_bit + 1) << mapped->pgshift);  in iova_bitmap_set()
442 		kaddr = kmap_local_page(mapped->pages[page_idx]);  in iova_bitmap_set()