Lines Matching +full:page +full:- +full:size
1 // SPDX-License-Identifier: GPL-2.0-only
3 * MMU-based software IOTLB.
5 * Copyright (C) 2020-2021 Bytedance Inc. and/or its affiliates. All rights reserved.
30 return -ENOMEM; in vduse_iotlb_add_range()
32 map_file->file = get_file(file); in vduse_iotlb_add_range()
33 map_file->offset = offset; in vduse_iotlb_add_range()
35 ret = vhost_iotlb_add_range_ctx(domain->iotlb, start, last, in vduse_iotlb_add_range()
38 fput(map_file->file); in vduse_iotlb_add_range()
51 while ((map = vhost_iotlb_itree_first(domain->iotlb, start, last))) { in vduse_iotlb_del_range()
52 map_file = (struct vdpa_map_file *)map->opaque; in vduse_iotlb_del_range()
53 fput(map_file->file); in vduse_iotlb_del_range()
55 vhost_iotlb_map_free(domain->iotlb, map); in vduse_iotlb_del_range()
67 spin_lock(&domain->iotlb_lock); in vduse_domain_set_map()
72 map_file = (struct vdpa_map_file *)map->opaque; in vduse_domain_set_map()
73 ret = vduse_iotlb_add_range(domain, map->start, map->last, in vduse_domain_set_map()
74 map->addr, map->perm, in vduse_domain_set_map()
75 map_file->file, in vduse_domain_set_map()
76 map_file->offset); in vduse_domain_set_map()
80 spin_unlock(&domain->iotlb_lock); in vduse_domain_set_map()
85 spin_unlock(&domain->iotlb_lock); in vduse_domain_set_map()
95 spin_lock(&domain->iotlb_lock); in vduse_domain_clear_map()
98 vduse_iotlb_del_range(domain, map->start, map->last); in vduse_domain_clear_map()
100 spin_unlock(&domain->iotlb_lock); in vduse_domain_clear_map()
104 u64 iova, u64 size, u64 paddr) in vduse_domain_map_bounce_page() argument
107 u64 last = iova + size - 1; in vduse_domain_map_bounce_page()
110 map = &domain->bounce_maps[iova >> PAGE_SHIFT]; in vduse_domain_map_bounce_page()
111 if (!map->bounce_page) { in vduse_domain_map_bounce_page()
112 map->bounce_page = alloc_page(GFP_ATOMIC); in vduse_domain_map_bounce_page()
113 if (!map->bounce_page) in vduse_domain_map_bounce_page()
114 return -ENOMEM; in vduse_domain_map_bounce_page()
116 map->orig_phys = paddr; in vduse_domain_map_bounce_page()
124 u64 iova, u64 size) in vduse_domain_unmap_bounce_page() argument
127 u64 last = iova + size - 1; in vduse_domain_unmap_bounce_page()
130 map = &domain->bounce_maps[iova >> PAGE_SHIFT]; in vduse_domain_unmap_bounce_page()
131 map->orig_phys = INVALID_PHYS_ADDR; in vduse_domain_unmap_bounce_page()
136 static void do_bounce(phys_addr_t orig, void *addr, size_t size, in do_bounce() argument
141 struct page *page; in do_bounce() local
144 while (size) { in do_bounce()
145 sz = min_t(size_t, PAGE_SIZE - offset, size); in do_bounce()
147 page = pfn_to_page(pfn); in do_bounce()
149 memcpy_from_page(addr, page, offset, sz); in do_bounce()
151 memcpy_to_page(page, offset, addr, sz); in do_bounce()
153 size -= sz; in do_bounce()
161 dma_addr_t iova, size_t size, in vduse_domain_bounce() argument
165 struct page *page; in vduse_domain_bounce() local
170 if (iova >= domain->bounce_size) in vduse_domain_bounce()
173 while (size) { in vduse_domain_bounce()
174 map = &domain->bounce_maps[iova >> PAGE_SHIFT]; in vduse_domain_bounce()
176 sz = min_t(size_t, PAGE_SIZE - offset, size); in vduse_domain_bounce()
178 if (WARN_ON(!map->bounce_page || in vduse_domain_bounce()
179 map->orig_phys == INVALID_PHYS_ADDR)) in vduse_domain_bounce()
182 page = domain->user_bounce_pages ? in vduse_domain_bounce()
183 map->user_bounce_page : map->bounce_page; in vduse_domain_bounce()
185 addr = kmap_local_page(page); in vduse_domain_bounce()
186 do_bounce(map->orig_phys + offset, addr + offset, sz, dir); in vduse_domain_bounce()
188 size -= sz; in vduse_domain_bounce()
193 static struct page *
197 u64 last = start + PAGE_SIZE - 1; in vduse_domain_get_coherent_page()
199 struct page *page = NULL; in vduse_domain_get_coherent_page() local
201 spin_lock(&domain->iotlb_lock); in vduse_domain_get_coherent_page()
202 map = vhost_iotlb_itree_first(domain->iotlb, start, last); in vduse_domain_get_coherent_page()
206 page = pfn_to_page((map->addr + iova - map->start) >> PAGE_SHIFT); in vduse_domain_get_coherent_page()
207 get_page(page); in vduse_domain_get_coherent_page()
209 spin_unlock(&domain->iotlb_lock); in vduse_domain_get_coherent_page()
211 return page; in vduse_domain_get_coherent_page()
214 static struct page *
218 struct page *page = NULL; in vduse_domain_get_bounce_page() local
220 read_lock(&domain->bounce_lock); in vduse_domain_get_bounce_page()
221 map = &domain->bounce_maps[iova >> PAGE_SHIFT]; in vduse_domain_get_bounce_page()
222 if (domain->user_bounce_pages || !map->bounce_page) in vduse_domain_get_bounce_page()
225 page = map->bounce_page; in vduse_domain_get_bounce_page()
226 get_page(page); in vduse_domain_get_bounce_page()
228 read_unlock(&domain->bounce_lock); in vduse_domain_get_bounce_page()
230 return page; in vduse_domain_get_bounce_page()
239 bounce_pfns = domain->bounce_size >> PAGE_SHIFT; in vduse_domain_free_kernel_bounce_pages()
242 map = &domain->bounce_maps[pfn]; in vduse_domain_free_kernel_bounce_pages()
243 if (WARN_ON(map->orig_phys != INVALID_PHYS_ADDR)) in vduse_domain_free_kernel_bounce_pages()
246 if (!map->bounce_page) in vduse_domain_free_kernel_bounce_pages()
249 __free_page(map->bounce_page); in vduse_domain_free_kernel_bounce_pages()
250 map->bounce_page = NULL; in vduse_domain_free_kernel_bounce_pages()
255 struct page **pages, int count) in vduse_domain_add_user_bounce_pages()
261 if (count != (domain->bounce_size >> PAGE_SHIFT)) in vduse_domain_add_user_bounce_pages()
262 return -EINVAL; in vduse_domain_add_user_bounce_pages()
264 write_lock(&domain->bounce_lock); in vduse_domain_add_user_bounce_pages()
265 ret = -EEXIST; in vduse_domain_add_user_bounce_pages()
266 if (domain->user_bounce_pages) in vduse_domain_add_user_bounce_pages()
270 map = &domain->bounce_maps[i]; in vduse_domain_add_user_bounce_pages()
271 if (map->bounce_page) { in vduse_domain_add_user_bounce_pages()
272 /* Copy kernel page to user page if it's in use */ in vduse_domain_add_user_bounce_pages()
273 if (map->orig_phys != INVALID_PHYS_ADDR) in vduse_domain_add_user_bounce_pages()
275 page_address(map->bounce_page), in vduse_domain_add_user_bounce_pages()
278 map->user_bounce_page = pages[i]; in vduse_domain_add_user_bounce_pages()
281 domain->user_bounce_pages = true; in vduse_domain_add_user_bounce_pages()
284 write_unlock(&domain->bounce_lock); in vduse_domain_add_user_bounce_pages()
294 write_lock(&domain->bounce_lock); in vduse_domain_remove_user_bounce_pages()
295 if (!domain->user_bounce_pages) in vduse_domain_remove_user_bounce_pages()
298 count = domain->bounce_size >> PAGE_SHIFT; in vduse_domain_remove_user_bounce_pages()
300 struct page *page = NULL; in vduse_domain_remove_user_bounce_pages() local
302 map = &domain->bounce_maps[i]; in vduse_domain_remove_user_bounce_pages()
303 if (WARN_ON(!map->user_bounce_page)) in vduse_domain_remove_user_bounce_pages()
306 /* Copy user page to kernel page if it's in use */ in vduse_domain_remove_user_bounce_pages()
307 if (map->orig_phys != INVALID_PHYS_ADDR) { in vduse_domain_remove_user_bounce_pages()
308 page = map->bounce_page; in vduse_domain_remove_user_bounce_pages()
309 memcpy_from_page(page_address(page), in vduse_domain_remove_user_bounce_pages()
310 map->user_bounce_page, 0, PAGE_SIZE); in vduse_domain_remove_user_bounce_pages()
312 put_page(map->user_bounce_page); in vduse_domain_remove_user_bounce_pages()
313 map->user_bounce_page = NULL; in vduse_domain_remove_user_bounce_pages()
315 domain->user_bounce_pages = false; in vduse_domain_remove_user_bounce_pages()
317 write_unlock(&domain->bounce_lock); in vduse_domain_remove_user_bounce_pages()
322 if (!domain->bounce_map) in vduse_domain_reset_bounce_map()
325 spin_lock(&domain->iotlb_lock); in vduse_domain_reset_bounce_map()
326 if (!domain->bounce_map) in vduse_domain_reset_bounce_map()
329 vduse_iotlb_del_range(domain, 0, domain->bounce_size - 1); in vduse_domain_reset_bounce_map()
330 domain->bounce_map = 0; in vduse_domain_reset_bounce_map()
332 spin_unlock(&domain->iotlb_lock); in vduse_domain_reset_bounce_map()
339 if (domain->bounce_map) in vduse_domain_init_bounce_map()
342 spin_lock(&domain->iotlb_lock); in vduse_domain_init_bounce_map()
343 if (domain->bounce_map) in vduse_domain_init_bounce_map()
346 ret = vduse_iotlb_add_range(domain, 0, domain->bounce_size - 1, in vduse_domain_init_bounce_map()
347 0, VHOST_MAP_RW, domain->file, 0); in vduse_domain_init_bounce_map()
351 domain->bounce_map = 1; in vduse_domain_init_bounce_map()
353 spin_unlock(&domain->iotlb_lock); in vduse_domain_init_bounce_map()
359 unsigned long size, unsigned long limit) in vduse_domain_alloc_iova() argument
362 unsigned long iova_len = iova_align(iovad, size) >> shift; in vduse_domain_alloc_iova()
371 dma_addr_t iova, size_t size) in vduse_domain_free_iova() argument
374 unsigned long iova_len = iova_align(iovad, size) >> shift; in vduse_domain_free_iova()
380 dma_addr_t dma_addr, size_t size, in vduse_domain_sync_single_for_device() argument
383 read_lock(&domain->bounce_lock); in vduse_domain_sync_single_for_device()
385 vduse_domain_bounce(domain, dma_addr, size, DMA_TO_DEVICE); in vduse_domain_sync_single_for_device()
386 read_unlock(&domain->bounce_lock); in vduse_domain_sync_single_for_device()
390 dma_addr_t dma_addr, size_t size, in vduse_domain_sync_single_for_cpu() argument
393 read_lock(&domain->bounce_lock); in vduse_domain_sync_single_for_cpu()
395 vduse_domain_bounce(domain, dma_addr, size, DMA_FROM_DEVICE); in vduse_domain_sync_single_for_cpu()
396 read_unlock(&domain->bounce_lock); in vduse_domain_sync_single_for_cpu()
400 struct page *page, unsigned long offset, in vduse_domain_map_page() argument
401 size_t size, enum dma_data_direction dir, in vduse_domain_map_page() argument
404 struct iova_domain *iovad = &domain->stream_iovad; in vduse_domain_map_page()
405 unsigned long limit = domain->bounce_size - 1; in vduse_domain_map_page()
406 phys_addr_t pa = page_to_phys(page) + offset; in vduse_domain_map_page()
407 dma_addr_t iova = vduse_domain_alloc_iova(iovad, size, limit); in vduse_domain_map_page()
415 read_lock(&domain->bounce_lock); in vduse_domain_map_page()
416 if (vduse_domain_map_bounce_page(domain, (u64)iova, (u64)size, pa)) in vduse_domain_map_page()
421 vduse_domain_bounce(domain, iova, size, DMA_TO_DEVICE); in vduse_domain_map_page()
423 read_unlock(&domain->bounce_lock); in vduse_domain_map_page()
427 read_unlock(&domain->bounce_lock); in vduse_domain_map_page()
429 vduse_domain_free_iova(iovad, iova, size); in vduse_domain_map_page()
434 dma_addr_t dma_addr, size_t size, in vduse_domain_unmap_page() argument
437 struct iova_domain *iovad = &domain->stream_iovad; in vduse_domain_unmap_page()
438 read_lock(&domain->bounce_lock); in vduse_domain_unmap_page()
441 vduse_domain_bounce(domain, dma_addr, size, DMA_FROM_DEVICE); in vduse_domain_unmap_page()
443 vduse_domain_unmap_bounce_page(domain, (u64)dma_addr, (u64)size); in vduse_domain_unmap_page()
444 read_unlock(&domain->bounce_lock); in vduse_domain_unmap_page()
445 vduse_domain_free_iova(iovad, dma_addr, size); in vduse_domain_unmap_page()
449 size_t size, dma_addr_t *dma_addr, in vduse_domain_alloc_coherent() argument
452 struct iova_domain *iovad = &domain->consistent_iovad; in vduse_domain_alloc_coherent()
453 unsigned long limit = domain->iova_limit; in vduse_domain_alloc_coherent()
454 dma_addr_t iova = vduse_domain_alloc_iova(iovad, size, limit); in vduse_domain_alloc_coherent()
455 void *orig = alloc_pages_exact(size, flag); in vduse_domain_alloc_coherent()
460 spin_lock(&domain->iotlb_lock); in vduse_domain_alloc_coherent()
461 if (vduse_iotlb_add_range(domain, (u64)iova, (u64)iova + size - 1, in vduse_domain_alloc_coherent()
463 domain->file, (u64)iova)) { in vduse_domain_alloc_coherent()
464 spin_unlock(&domain->iotlb_lock); in vduse_domain_alloc_coherent()
467 spin_unlock(&domain->iotlb_lock); in vduse_domain_alloc_coherent()
475 free_pages_exact(orig, size); in vduse_domain_alloc_coherent()
477 vduse_domain_free_iova(iovad, iova, size); in vduse_domain_alloc_coherent()
482 void vduse_domain_free_coherent(struct vduse_iova_domain *domain, size_t size, in vduse_domain_free_coherent() argument
486 struct iova_domain *iovad = &domain->consistent_iovad; in vduse_domain_free_coherent()
491 spin_lock(&domain->iotlb_lock); in vduse_domain_free_coherent()
492 map = vhost_iotlb_itree_first(domain->iotlb, (u64)dma_addr, in vduse_domain_free_coherent()
493 (u64)dma_addr + size - 1); in vduse_domain_free_coherent()
495 spin_unlock(&domain->iotlb_lock); in vduse_domain_free_coherent()
498 map_file = (struct vdpa_map_file *)map->opaque; in vduse_domain_free_coherent()
499 fput(map_file->file); in vduse_domain_free_coherent()
501 pa = map->addr; in vduse_domain_free_coherent()
502 vhost_iotlb_map_free(domain->iotlb, map); in vduse_domain_free_coherent()
503 spin_unlock(&domain->iotlb_lock); in vduse_domain_free_coherent()
505 vduse_domain_free_iova(iovad, dma_addr, size); in vduse_domain_free_coherent()
506 free_pages_exact(phys_to_virt(pa), size); in vduse_domain_free_coherent()
511 struct vduse_iova_domain *domain = vmf->vma->vm_private_data; in vduse_domain_mmap_fault()
512 unsigned long iova = vmf->pgoff << PAGE_SHIFT; in vduse_domain_mmap_fault()
513 struct page *page; in vduse_domain_mmap_fault() local
518 if (iova < domain->bounce_size) in vduse_domain_mmap_fault()
519 page = vduse_domain_get_bounce_page(domain, iova); in vduse_domain_mmap_fault()
521 page = vduse_domain_get_coherent_page(domain, iova); in vduse_domain_mmap_fault()
523 if (!page) in vduse_domain_mmap_fault()
526 vmf->page = page; in vduse_domain_mmap_fault()
537 struct vduse_iova_domain *domain = file->private_data; in vduse_domain_mmap()
540 vma->vm_private_data = domain; in vduse_domain_mmap()
541 vma->vm_ops = &vduse_domain_mmap_ops; in vduse_domain_mmap()
548 struct vduse_iova_domain *domain = file->private_data; in vduse_domain_release()
550 spin_lock(&domain->iotlb_lock); in vduse_domain_release()
554 spin_unlock(&domain->iotlb_lock); in vduse_domain_release()
555 put_iova_domain(&domain->stream_iovad); in vduse_domain_release()
556 put_iova_domain(&domain->consistent_iovad); in vduse_domain_release()
557 vhost_iotlb_free(domain->iotlb); in vduse_domain_release()
558 vfree(domain->bounce_maps); in vduse_domain_release()
572 fput(domain->file); in vduse_domain_destroy()
592 domain->iotlb = vhost_iotlb_alloc(0, 0); in vduse_domain_create()
593 if (!domain->iotlb) in vduse_domain_create()
596 domain->iova_limit = iova_limit; in vduse_domain_create()
597 domain->bounce_size = PAGE_ALIGN(bounce_size); in vduse_domain_create()
598 domain->bounce_maps = vzalloc(bounce_pfns * in vduse_domain_create()
600 if (!domain->bounce_maps) in vduse_domain_create()
604 map = &domain->bounce_maps[pfn]; in vduse_domain_create()
605 map->orig_phys = INVALID_PHYS_ADDR; in vduse_domain_create()
607 file = anon_inode_getfile("[vduse-domain]", &vduse_domain_fops, in vduse_domain_create()
612 domain->file = file; in vduse_domain_create()
613 rwlock_init(&domain->bounce_lock); in vduse_domain_create()
614 spin_lock_init(&domain->iotlb_lock); in vduse_domain_create()
615 init_iova_domain(&domain->stream_iovad, in vduse_domain_create()
617 ret = iova_domain_init_rcaches(&domain->stream_iovad); in vduse_domain_create()
620 init_iova_domain(&domain->consistent_iovad, in vduse_domain_create()
622 ret = iova_domain_init_rcaches(&domain->consistent_iovad); in vduse_domain_create()
628 put_iova_domain(&domain->stream_iovad); in vduse_domain_create()
632 vfree(domain->bounce_maps); in vduse_domain_create()
634 vhost_iotlb_free(domain->iotlb); in vduse_domain_create()