20 static int vduse_iotlb_add_range(struct vduse_iova_domain *domain,  in vduse_iotlb_add_range()  argument
35 	ret = vhost_iotlb_add_range_ctx(domain->iotlb, start, last,  in vduse_iotlb_add_range()
45 static void vduse_iotlb_del_range(struct vduse_iova_domain *domain,  in vduse_iotlb_del_range()  argument
51 	while ((map = vhost_iotlb_itree_first(domain->iotlb, start, last))) {  in vduse_iotlb_del_range()
55 		vhost_iotlb_map_free(domain->iotlb, map);  in vduse_iotlb_del_range()
59 int vduse_domain_set_map(struct vduse_iova_domain *domain,  in vduse_domain_set_map()  argument
67 	spin_lock(&domain->iotlb_lock);  in vduse_domain_set_map()
68 	vduse_iotlb_del_range(domain, start, last);  in vduse_domain_set_map()
73 		ret = vduse_iotlb_add_range(domain, map->start, map->last,  in vduse_domain_set_map()
80 	spin_unlock(&domain->iotlb_lock);  in vduse_domain_set_map()
84 	vduse_iotlb_del_range(domain, start, last);  in vduse_domain_set_map()
85 	spin_unlock(&domain->iotlb_lock);  in vduse_domain_set_map()
89 void vduse_domain_clear_map(struct vduse_iova_domain *domain,  in vduse_domain_clear_map()  argument
95 	spin_lock(&domain->iotlb_lock);  in vduse_domain_clear_map()
98 		vduse_iotlb_del_range(domain, map->start, map->last);  in vduse_domain_clear_map()
100 	spin_unlock(&domain->iotlb_lock);  in vduse_domain_clear_map()
103 static int vduse_domain_map_bounce_page(struct vduse_iova_domain *domain,  in vduse_domain_map_bounce_page()  argument
110 		map = &domain->bounce_maps[iova >> PAGE_SHIFT];  in vduse_domain_map_bounce_page()
123 static void vduse_domain_unmap_bounce_page(struct vduse_iova_domain *domain,  in vduse_domain_unmap_bounce_page()  argument
130 		map = &domain->bounce_maps[iova >> PAGE_SHIFT];  in vduse_domain_unmap_bounce_page()
160 static void vduse_domain_bounce(struct vduse_iova_domain *domain,  in vduse_domain_bounce()  argument
170 	if (iova >= domain->bounce_size)  in vduse_domain_bounce()
174 		map = &domain->bounce_maps[iova >> PAGE_SHIFT];  in vduse_domain_bounce()
182 		page = domain->user_bounce_pages ?  in vduse_domain_bounce()
194 vduse_domain_get_coherent_page(struct vduse_iova_domain *domain, u64 iova)  in vduse_domain_get_coherent_page()  argument
201 	spin_lock(&domain->iotlb_lock);  in vduse_domain_get_coherent_page()
202 	map = vhost_iotlb_itree_first(domain->iotlb, start, last);  in vduse_domain_get_coherent_page()
209 	spin_unlock(&domain->iotlb_lock);  in vduse_domain_get_coherent_page()
215 vduse_domain_get_bounce_page(struct vduse_iova_domain *domain, u64 iova)  in vduse_domain_get_bounce_page()  argument
220 	read_lock(&domain->bounce_lock);  in vduse_domain_get_bounce_page()
221 	map = &domain->bounce_maps[iova >> PAGE_SHIFT];  in vduse_domain_get_bounce_page()
222 	if (domain->user_bounce_pages || !map->bounce_page)  in vduse_domain_get_bounce_page()
228 	read_unlock(&domain->bounce_lock);  in vduse_domain_get_bounce_page()
234 vduse_domain_free_kernel_bounce_pages(struct vduse_iova_domain *domain)  in vduse_domain_free_kernel_bounce_pages()  argument
239 	bounce_pfns = domain->bounce_size >> PAGE_SHIFT;  in vduse_domain_free_kernel_bounce_pages()
242 		map = &domain->bounce_maps[pfn];  in vduse_domain_free_kernel_bounce_pages()
254 int vduse_domain_add_user_bounce_pages(struct vduse_iova_domain *domain,  in vduse_domain_add_user_bounce_pages()  argument
261 	if (count != (domain->bounce_size >> PAGE_SHIFT))  in vduse_domain_add_user_bounce_pages()
264 	write_lock(&domain->bounce_lock);  in vduse_domain_add_user_bounce_pages()
266 	if (domain->user_bounce_pages)  in vduse_domain_add_user_bounce_pages()
270 		map = &domain->bounce_maps[i];  in vduse_domain_add_user_bounce_pages()
281 	domain->user_bounce_pages = true;  in vduse_domain_add_user_bounce_pages()
284 	write_unlock(&domain->bounce_lock);  in vduse_domain_add_user_bounce_pages()
289 void vduse_domain_remove_user_bounce_pages(struct vduse_iova_domain *domain)  in vduse_domain_remove_user_bounce_pages()  argument
294 	write_lock(&domain->bounce_lock);  in vduse_domain_remove_user_bounce_pages()
295 	if (!domain->user_bounce_pages)  in vduse_domain_remove_user_bounce_pages()
298 	count = domain->bounce_size >> PAGE_SHIFT;  in vduse_domain_remove_user_bounce_pages()
302 		map = &domain->bounce_maps[i];  in vduse_domain_remove_user_bounce_pages()
315 	domain->user_bounce_pages = false;  in vduse_domain_remove_user_bounce_pages()
317 	write_unlock(&domain->bounce_lock);  in vduse_domain_remove_user_bounce_pages()
320 void vduse_domain_reset_bounce_map(struct vduse_iova_domain *domain)  in vduse_domain_reset_bounce_map()  argument
322 	if (!domain->bounce_map)  in vduse_domain_reset_bounce_map()
325 	spin_lock(&domain->iotlb_lock);  in vduse_domain_reset_bounce_map()
326 	if (!domain->bounce_map)  in vduse_domain_reset_bounce_map()
329 	vduse_iotlb_del_range(domain, 0, domain->bounce_size - 1);  in vduse_domain_reset_bounce_map()
330 	domain->bounce_map = 0;  in vduse_domain_reset_bounce_map()
332 	spin_unlock(&domain->iotlb_lock);  in vduse_domain_reset_bounce_map()
335 static int vduse_domain_init_bounce_map(struct vduse_iova_domain *domain)  in vduse_domain_init_bounce_map()  argument
339 	if (domain->bounce_map)  in vduse_domain_init_bounce_map()
342 	spin_lock(&domain->iotlb_lock);  in vduse_domain_init_bounce_map()
343 	if (domain->bounce_map)  in vduse_domain_init_bounce_map()
346 	ret = vduse_iotlb_add_range(domain, 0, domain->bounce_size - 1,  in vduse_domain_init_bounce_map()
347 				    0, VHOST_MAP_RW, domain->file, 0);  in vduse_domain_init_bounce_map()
351 	domain->bounce_map = 1;  in vduse_domain_init_bounce_map()
353 	spin_unlock(&domain->iotlb_lock);  in vduse_domain_init_bounce_map()
379 void vduse_domain_sync_single_for_device(struct vduse_iova_domain *domain,  in vduse_domain_sync_single_for_device()  argument
383 	read_lock(&domain->bounce_lock);  in vduse_domain_sync_single_for_device()
385 		vduse_domain_bounce(domain, dma_addr, size, DMA_TO_DEVICE);  in vduse_domain_sync_single_for_device()
386 	read_unlock(&domain->bounce_lock);  in vduse_domain_sync_single_for_device()
389 void vduse_domain_sync_single_for_cpu(struct vduse_iova_domain *domain,  in vduse_domain_sync_single_for_cpu()  argument
393 	read_lock(&domain->bounce_lock);  in vduse_domain_sync_single_for_cpu()
395 		vduse_domain_bounce(domain, dma_addr, size, DMA_FROM_DEVICE);  in vduse_domain_sync_single_for_cpu()
396 	read_unlock(&domain->bounce_lock);  in vduse_domain_sync_single_for_cpu()
399 dma_addr_t vduse_domain_map_page(struct vduse_iova_domain *domain,  in vduse_domain_map_page()  argument
404 	struct iova_domain *iovad = &domain->stream_iovad;  in vduse_domain_map_page()
405 	unsigned long limit = domain->bounce_size - 1;  in vduse_domain_map_page()
412 	if (vduse_domain_init_bounce_map(domain))  in vduse_domain_map_page()
415 	read_lock(&domain->bounce_lock);  in vduse_domain_map_page()
416 	if (vduse_domain_map_bounce_page(domain, (u64)iova, (u64)size, pa))  in vduse_domain_map_page()
421 		vduse_domain_bounce(domain, iova, size, DMA_TO_DEVICE);  in vduse_domain_map_page()
423 	read_unlock(&domain->bounce_lock);  in vduse_domain_map_page()
427 	read_unlock(&domain->bounce_lock);  in vduse_domain_map_page()
433 void vduse_domain_unmap_page(struct vduse_iova_domain *domain,  in vduse_domain_unmap_page()  argument
437 	struct iova_domain *iovad = &domain->stream_iovad;  in vduse_domain_unmap_page()
438 	read_lock(&domain->bounce_lock);  in vduse_domain_unmap_page()
441 		vduse_domain_bounce(domain, dma_addr, size, DMA_FROM_DEVICE);  in vduse_domain_unmap_page()
443 	vduse_domain_unmap_bounce_page(domain, (u64)dma_addr, (u64)size);  in vduse_domain_unmap_page()
444 	read_unlock(&domain->bounce_lock);  in vduse_domain_unmap_page()
448 void *vduse_domain_alloc_coherent(struct vduse_iova_domain *domain,  in vduse_domain_alloc_coherent()  argument
452 	struct iova_domain *iovad = &domain->consistent_iovad;  in vduse_domain_alloc_coherent()
453 	unsigned long limit = domain->iova_limit;  in vduse_domain_alloc_coherent()
460 	spin_lock(&domain->iotlb_lock);  in vduse_domain_alloc_coherent()
461 	if (vduse_iotlb_add_range(domain, (u64)iova, (u64)iova + size - 1,  in vduse_domain_alloc_coherent()
463 				  domain->file, (u64)iova)) {  in vduse_domain_alloc_coherent()
464 		spin_unlock(&domain->iotlb_lock);  in vduse_domain_alloc_coherent()
467 	spin_unlock(&domain->iotlb_lock);  in vduse_domain_alloc_coherent()
482 void vduse_domain_free_coherent(struct vduse_iova_domain *domain, size_t size,  in vduse_domain_free_coherent()  argument
486 	struct iova_domain *iovad = &domain->consistent_iovad;  in vduse_domain_free_coherent()
491 	spin_lock(&domain->iotlb_lock);  in vduse_domain_free_coherent()
492 	map = vhost_iotlb_itree_first(domain->iotlb, (u64)dma_addr,  in vduse_domain_free_coherent()
495 		spin_unlock(&domain->iotlb_lock);  in vduse_domain_free_coherent()
502 	vhost_iotlb_map_free(domain->iotlb, map);  in vduse_domain_free_coherent()
503 	spin_unlock(&domain->iotlb_lock);  in vduse_domain_free_coherent()
511 	struct vduse_iova_domain *domain = vmf->vma->vm_private_data;  in vduse_domain_mmap_fault()  local
515 	if (!domain)  in vduse_domain_mmap_fault()
518 	if (iova < domain->bounce_size)  in vduse_domain_mmap_fault()
519 		page = vduse_domain_get_bounce_page(domain, iova);  in vduse_domain_mmap_fault()
521 		page = vduse_domain_get_coherent_page(domain, iova);  in vduse_domain_mmap_fault()
537 	struct vduse_iova_domain *domain = file->private_data;  in vduse_domain_mmap()  local
540 	vma->vm_private_data = domain;  in vduse_domain_mmap()
548 	struct vduse_iova_domain *domain = file->private_data;  in vduse_domain_release()  local
550 	spin_lock(&domain->iotlb_lock);  in vduse_domain_release()
551 	vduse_iotlb_del_range(domain, 0, ULLONG_MAX);  in vduse_domain_release()
552 	vduse_domain_remove_user_bounce_pages(domain);  in vduse_domain_release()
553 	vduse_domain_free_kernel_bounce_pages(domain);  in vduse_domain_release()
554 	spin_unlock(&domain->iotlb_lock);  in vduse_domain_release()
555 	put_iova_domain(&domain->stream_iovad);  in vduse_domain_release()
556 	put_iova_domain(&domain->consistent_iovad);  in vduse_domain_release()
557 	vhost_iotlb_free(domain->iotlb);  in vduse_domain_release()
558 	vfree(domain->bounce_maps);  in vduse_domain_release()
559 	kfree(domain);  in vduse_domain_release()
570 void vduse_domain_destroy(struct vduse_iova_domain *domain)  in vduse_domain_destroy()  argument
572 	fput(domain->file);  in vduse_domain_destroy()
578 	struct vduse_iova_domain *domain;  in vduse_domain_create()  local
588 	domain = kzalloc(sizeof(*domain), GFP_KERNEL);  in vduse_domain_create()
589 	if (!domain)  in vduse_domain_create()
592 	domain->iotlb = vhost_iotlb_alloc(0, 0);  in vduse_domain_create()
593 	if (!domain->iotlb)  in vduse_domain_create()
596 	domain->iova_limit = iova_limit;  in vduse_domain_create()
597 	domain->bounce_size = PAGE_ALIGN(bounce_size);  in vduse_domain_create()
598 	domain->bounce_maps = vzalloc(bounce_pfns *  in vduse_domain_create()
600 	if (!domain->bounce_maps)  in vduse_domain_create()
604 		map = &domain->bounce_maps[pfn];  in vduse_domain_create()
608 				domain, O_RDWR);  in vduse_domain_create()
612 	domain->file = file;  in vduse_domain_create()
613 	rwlock_init(&domain->bounce_lock);  in vduse_domain_create()
614 	spin_lock_init(&domain->iotlb_lock);  in vduse_domain_create()
615 	init_iova_domain(&domain->stream_iovad,  in vduse_domain_create()
617 	ret = iova_domain_init_rcaches(&domain->stream_iovad);  in vduse_domain_create()
620 	init_iova_domain(&domain->consistent_iovad,  in vduse_domain_create()
622 	ret = iova_domain_init_rcaches(&domain->consistent_iovad);  in vduse_domain_create()
626 	return domain;  in vduse_domain_create()
628 	put_iova_domain(&domain->stream_iovad);  in vduse_domain_create()
632 	vfree(domain->bounce_maps);  in vduse_domain_create()
634 	vhost_iotlb_free(domain->iotlb);  in vduse_domain_create()
636 	kfree(domain);  in vduse_domain_create()