1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3  * VFIO: IOMMU DMA mapping support for Type1 IOMMU
4  *
5  * Copyright (C) 2012 Red Hat, Inc.  All rights reserved.
6  *     Author: Alex Williamson <alex.williamson@redhat.com>
7  *
8  * Derived from original vfio:
9  * Copyright 2010 Cisco Systems, Inc.  All rights reserved.
10  * Author: Tom Lyon, pugs@cisco.com
11  *
12  * We arbitrarily define a Type1 IOMMU as one matching the below code.
13  * It could be called the x86 IOMMU as it's designed for AMD-Vi & Intel
14  * VT-d, but that makes it harder to re-use as theoretically anyone
15  * implementing a similar IOMMU could make use of this.  We expect the
16  * IOMMU to support the IOMMU API and have few to no restrictions around
17  * the IOVA range that can be mapped.  The Type1 IOMMU is currently
18  * optimized for relatively static mappings of a userspace process with
19  * userspace pages pinned into memory.  We also assume devices and IOMMU
20  * domains are PCI based as the IOMMU API is still centered around a
21  * device/bus interface rather than a group interface.
22  */
23 
24 #include <linux/compat.h>
25 #include <linux/device.h>
26 #include <linux/fs.h>
27 #include <linux/highmem.h>
28 #include <linux/iommu.h>
29 #include <linux/module.h>
30 #include <linux/mm.h>
31 #include <linux/kthread.h>
32 #include <linux/rbtree.h>
33 #include <linux/sched/signal.h>
34 #include <linux/sched/mm.h>
35 #include <linux/slab.h>
36 #include <linux/uaccess.h>
37 #include <linux/vfio.h>
38 #include <linux/workqueue.h>
39 #include <linux/notifier.h>
40 #include "vfio.h"
41 
42 #define DRIVER_VERSION  "0.2"
43 #define DRIVER_AUTHOR   "Alex Williamson <alex.williamson@redhat.com>"
44 #define DRIVER_DESC     "Type1 IOMMU driver for VFIO"
45 
46 static bool allow_unsafe_interrupts;
47 module_param_named(allow_unsafe_interrupts,
48 		   allow_unsafe_interrupts, bool, S_IRUGO | S_IWUSR);
49 MODULE_PARM_DESC(allow_unsafe_interrupts,
50 		 "Enable VFIO IOMMU support for on platforms without interrupt remapping support.");
51 
52 static bool disable_hugepages;
53 module_param_named(disable_hugepages,
54 		   disable_hugepages, bool, S_IRUGO | S_IWUSR);
55 MODULE_PARM_DESC(disable_hugepages,
56 		 "Disable VFIO IOMMU support for IOMMU hugepages.");
57 
58 static unsigned int dma_entry_limit __read_mostly = U16_MAX;
59 module_param_named(dma_entry_limit, dma_entry_limit, uint, 0644);
60 MODULE_PARM_DESC(dma_entry_limit,
61 		 "Maximum number of user DMA mappings per container (65535).");
62 
63 struct vfio_iommu {
64 	struct list_head	domain_list;
65 	struct list_head	iova_list;
66 	struct mutex		lock;
67 	struct rb_root		dma_list;
68 	struct list_head	device_list;
69 	struct mutex		device_list_lock;
70 	unsigned int		dma_avail;
71 	unsigned int		vaddr_invalid_count;
72 	uint64_t		pgsize_bitmap;
73 	uint64_t		num_non_pinned_groups;
74 	bool			v2;
75 	bool			nesting;
76 	bool			dirty_page_tracking;
77 	struct list_head	emulated_iommu_groups;
78 };
79 
80 struct vfio_domain {
81 	struct iommu_domain	*domain;
82 	struct list_head	next;
83 	struct list_head	group_list;
84 	bool			fgsp : 1;	/* Fine-grained super pages */
85 	bool			enforce_cache_coherency : 1;
86 };
87 
88 struct vfio_dma {
89 	struct rb_node		node;
90 	dma_addr_t		iova;		/* Device address */
91 	unsigned long		vaddr;		/* Process virtual addr */
92 	size_t			size;		/* Map size (bytes) */
93 	int			prot;		/* IOMMU_READ/WRITE */
94 	bool			iommu_mapped;
95 	bool			lock_cap;	/* capable(CAP_IPC_LOCK) */
96 	bool			vaddr_invalid;
97 	struct task_struct	*task;
98 	struct rb_root		pfn_list;	/* Ex-user pinned pfn list */
99 	unsigned long		*bitmap;
100 	struct mm_struct	*mm;
101 	size_t			locked_vm;
102 };
103 
104 struct vfio_batch {
105 	struct page		**pages;	/* for pin_user_pages_remote */
106 	struct page		*fallback_page; /* if pages alloc fails */
107 	int			capacity;	/* length of pages array */
108 	int			size;		/* of batch currently */
109 	int			offset;		/* of next entry in pages */
110 };
111 
112 struct vfio_iommu_group {
113 	struct iommu_group	*iommu_group;
114 	struct list_head	next;
115 	bool			pinned_page_dirty_scope;
116 };
117 
118 struct vfio_iova {
119 	struct list_head	list;
120 	dma_addr_t		start;
121 	dma_addr_t		end;
122 };
123 
124 /*
125  * Guest RAM pinning working set or DMA target
126  */
127 struct vfio_pfn {
128 	struct rb_node		node;
129 	dma_addr_t		iova;		/* Device address */
130 	unsigned long		pfn;		/* Host pfn */
131 	unsigned int		ref_count;
132 };
133 
134 struct vfio_regions {
135 	struct list_head list;
136 	dma_addr_t iova;
137 	phys_addr_t phys;
138 	size_t len;
139 };
140 
141 #define DIRTY_BITMAP_BYTES(n)	(ALIGN(n, BITS_PER_TYPE(u64)) / BITS_PER_BYTE)
142 
143 /*
144  * Input argument of number of bits to bitmap_set() is unsigned integer, which
145  * further casts to signed integer for unaligned multi-bit operation,
146  * __bitmap_set().
147  * Then maximum bitmap size supported is 2^31 bits divided by 2^3 bits/byte,
148  * that is 2^28 (256 MB) which maps to 2^31 * 2^12 = 2^43 (8TB) on 4K page
149  * system.
150  */
151 #define DIRTY_BITMAP_PAGES_MAX	 ((u64)INT_MAX)
152 #define DIRTY_BITMAP_SIZE_MAX	 DIRTY_BITMAP_BYTES(DIRTY_BITMAP_PAGES_MAX)
153 
154 static int put_pfn(unsigned long pfn, int prot);
155 
156 static struct vfio_iommu_group*
157 vfio_iommu_find_iommu_group(struct vfio_iommu *iommu,
158 			    struct iommu_group *iommu_group);
159 
160 /*
161  * This code handles mapping and unmapping of user data buffers
162  * into DMA'ble space using the IOMMU
163  */
164 
vfio_find_dma(struct vfio_iommu * iommu,dma_addr_t start,size_t size)165 static struct vfio_dma *vfio_find_dma(struct vfio_iommu *iommu,
166 				      dma_addr_t start, size_t size)
167 {
168 	struct rb_node *node = iommu->dma_list.rb_node;
169 
170 	while (node) {
171 		struct vfio_dma *dma = rb_entry(node, struct vfio_dma, node);
172 
173 		if (start + size <= dma->iova)
174 			node = node->rb_left;
175 		else if (start >= dma->iova + dma->size)
176 			node = node->rb_right;
177 		else
178 			return dma;
179 	}
180 
181 	return NULL;
182 }
183 
vfio_find_dma_first_node(struct vfio_iommu * iommu,dma_addr_t start,u64 size)184 static struct rb_node *vfio_find_dma_first_node(struct vfio_iommu *iommu,
185 						dma_addr_t start, u64 size)
186 {
187 	struct rb_node *res = NULL;
188 	struct rb_node *node = iommu->dma_list.rb_node;
189 	struct vfio_dma *dma_res = NULL;
190 
191 	while (node) {
192 		struct vfio_dma *dma = rb_entry(node, struct vfio_dma, node);
193 
194 		if (start < dma->iova + dma->size) {
195 			res = node;
196 			dma_res = dma;
197 			if (start >= dma->iova)
198 				break;
199 			node = node->rb_left;
200 		} else {
201 			node = node->rb_right;
202 		}
203 	}
204 	if (res && size && dma_res->iova >= start + size)
205 		res = NULL;
206 	return res;
207 }
208 
vfio_link_dma(struct vfio_iommu * iommu,struct vfio_dma * new)209 static void vfio_link_dma(struct vfio_iommu *iommu, struct vfio_dma *new)
210 {
211 	struct rb_node **link = &iommu->dma_list.rb_node, *parent = NULL;
212 	struct vfio_dma *dma;
213 
214 	while (*link) {
215 		parent = *link;
216 		dma = rb_entry(parent, struct vfio_dma, node);
217 
218 		if (new->iova + new->size <= dma->iova)
219 			link = &(*link)->rb_left;
220 		else
221 			link = &(*link)->rb_right;
222 	}
223 
224 	rb_link_node(&new->node, parent, link);
225 	rb_insert_color(&new->node, &iommu->dma_list);
226 }
227 
vfio_unlink_dma(struct vfio_iommu * iommu,struct vfio_dma * old)228 static void vfio_unlink_dma(struct vfio_iommu *iommu, struct vfio_dma *old)
229 {
230 	rb_erase(&old->node, &iommu->dma_list);
231 }
232 
233 
vfio_dma_bitmap_alloc(struct vfio_dma * dma,size_t pgsize)234 static int vfio_dma_bitmap_alloc(struct vfio_dma *dma, size_t pgsize)
235 {
236 	uint64_t npages = dma->size / pgsize;
237 
238 	if (npages > DIRTY_BITMAP_PAGES_MAX)
239 		return -EINVAL;
240 
241 	/*
242 	 * Allocate extra 64 bits that are used to calculate shift required for
243 	 * bitmap_shift_left() to manipulate and club unaligned number of pages
244 	 * in adjacent vfio_dma ranges.
245 	 */
246 	dma->bitmap = kvzalloc(DIRTY_BITMAP_BYTES(npages) + sizeof(u64),
247 			       GFP_KERNEL);
248 	if (!dma->bitmap)
249 		return -ENOMEM;
250 
251 	return 0;
252 }
253 
vfio_dma_bitmap_free(struct vfio_dma * dma)254 static void vfio_dma_bitmap_free(struct vfio_dma *dma)
255 {
256 	kvfree(dma->bitmap);
257 	dma->bitmap = NULL;
258 }
259 
vfio_dma_populate_bitmap(struct vfio_dma * dma,size_t pgsize)260 static void vfio_dma_populate_bitmap(struct vfio_dma *dma, size_t pgsize)
261 {
262 	struct rb_node *p;
263 	unsigned long pgshift = __ffs(pgsize);
264 
265 	for (p = rb_first(&dma->pfn_list); p; p = rb_next(p)) {
266 		struct vfio_pfn *vpfn = rb_entry(p, struct vfio_pfn, node);
267 
268 		bitmap_set(dma->bitmap, (vpfn->iova - dma->iova) >> pgshift, 1);
269 	}
270 }
271 
vfio_iommu_populate_bitmap_full(struct vfio_iommu * iommu)272 static void vfio_iommu_populate_bitmap_full(struct vfio_iommu *iommu)
273 {
274 	struct rb_node *n;
275 	unsigned long pgshift = __ffs(iommu->pgsize_bitmap);
276 
277 	for (n = rb_first(&iommu->dma_list); n; n = rb_next(n)) {
278 		struct vfio_dma *dma = rb_entry(n, struct vfio_dma, node);
279 
280 		bitmap_set(dma->bitmap, 0, dma->size >> pgshift);
281 	}
282 }
283 
vfio_dma_bitmap_alloc_all(struct vfio_iommu * iommu,size_t pgsize)284 static int vfio_dma_bitmap_alloc_all(struct vfio_iommu *iommu, size_t pgsize)
285 {
286 	struct rb_node *n;
287 
288 	for (n = rb_first(&iommu->dma_list); n; n = rb_next(n)) {
289 		struct vfio_dma *dma = rb_entry(n, struct vfio_dma, node);
290 		int ret;
291 
292 		ret = vfio_dma_bitmap_alloc(dma, pgsize);
293 		if (ret) {
294 			struct rb_node *p;
295 
296 			for (p = rb_prev(n); p; p = rb_prev(p)) {
297 				struct vfio_dma *dma = rb_entry(n,
298 							struct vfio_dma, node);
299 
300 				vfio_dma_bitmap_free(dma);
301 			}
302 			return ret;
303 		}
304 		vfio_dma_populate_bitmap(dma, pgsize);
305 	}
306 	return 0;
307 }
308 
vfio_dma_bitmap_free_all(struct vfio_iommu * iommu)309 static void vfio_dma_bitmap_free_all(struct vfio_iommu *iommu)
310 {
311 	struct rb_node *n;
312 
313 	for (n = rb_first(&iommu->dma_list); n; n = rb_next(n)) {
314 		struct vfio_dma *dma = rb_entry(n, struct vfio_dma, node);
315 
316 		vfio_dma_bitmap_free(dma);
317 	}
318 }
319 
320 /*
321  * Helper Functions for host iova-pfn list
322  */
vfio_find_vpfn(struct vfio_dma * dma,dma_addr_t iova)323 static struct vfio_pfn *vfio_find_vpfn(struct vfio_dma *dma, dma_addr_t iova)
324 {
325 	struct vfio_pfn *vpfn;
326 	struct rb_node *node = dma->pfn_list.rb_node;
327 
328 	while (node) {
329 		vpfn = rb_entry(node, struct vfio_pfn, node);
330 
331 		if (iova < vpfn->iova)
332 			node = node->rb_left;
333 		else if (iova > vpfn->iova)
334 			node = node->rb_right;
335 		else
336 			return vpfn;
337 	}
338 	return NULL;
339 }
340 
vfio_link_pfn(struct vfio_dma * dma,struct vfio_pfn * new)341 static void vfio_link_pfn(struct vfio_dma *dma,
342 			  struct vfio_pfn *new)
343 {
344 	struct rb_node **link, *parent = NULL;
345 	struct vfio_pfn *vpfn;
346 
347 	link = &dma->pfn_list.rb_node;
348 	while (*link) {
349 		parent = *link;
350 		vpfn = rb_entry(parent, struct vfio_pfn, node);
351 
352 		if (new->iova < vpfn->iova)
353 			link = &(*link)->rb_left;
354 		else
355 			link = &(*link)->rb_right;
356 	}
357 
358 	rb_link_node(&new->node, parent, link);
359 	rb_insert_color(&new->node, &dma->pfn_list);
360 }
361 
vfio_unlink_pfn(struct vfio_dma * dma,struct vfio_pfn * old)362 static void vfio_unlink_pfn(struct vfio_dma *dma, struct vfio_pfn *old)
363 {
364 	rb_erase(&old->node, &dma->pfn_list);
365 }
366 
vfio_add_to_pfn_list(struct vfio_dma * dma,dma_addr_t iova,unsigned long pfn)367 static int vfio_add_to_pfn_list(struct vfio_dma *dma, dma_addr_t iova,
368 				unsigned long pfn)
369 {
370 	struct vfio_pfn *vpfn;
371 
372 	vpfn = kzalloc(sizeof(*vpfn), GFP_KERNEL);
373 	if (!vpfn)
374 		return -ENOMEM;
375 
376 	vpfn->iova = iova;
377 	vpfn->pfn = pfn;
378 	vpfn->ref_count = 1;
379 	vfio_link_pfn(dma, vpfn);
380 	return 0;
381 }
382 
vfio_remove_from_pfn_list(struct vfio_dma * dma,struct vfio_pfn * vpfn)383 static void vfio_remove_from_pfn_list(struct vfio_dma *dma,
384 				      struct vfio_pfn *vpfn)
385 {
386 	vfio_unlink_pfn(dma, vpfn);
387 	kfree(vpfn);
388 }
389 
vfio_iova_get_vfio_pfn(struct vfio_dma * dma,unsigned long iova)390 static struct vfio_pfn *vfio_iova_get_vfio_pfn(struct vfio_dma *dma,
391 					       unsigned long iova)
392 {
393 	struct vfio_pfn *vpfn = vfio_find_vpfn(dma, iova);
394 
395 	if (vpfn)
396 		vpfn->ref_count++;
397 	return vpfn;
398 }
399 
vfio_iova_put_vfio_pfn(struct vfio_dma * dma,struct vfio_pfn * vpfn)400 static int vfio_iova_put_vfio_pfn(struct vfio_dma *dma, struct vfio_pfn *vpfn)
401 {
402 	int ret = 0;
403 
404 	vpfn->ref_count--;
405 	if (!vpfn->ref_count) {
406 		ret = put_pfn(vpfn->pfn, dma->prot);
407 		vfio_remove_from_pfn_list(dma, vpfn);
408 	}
409 	return ret;
410 }
411 
mm_lock_acct(struct task_struct * task,struct mm_struct * mm,bool lock_cap,long npage)412 static int mm_lock_acct(struct task_struct *task, struct mm_struct *mm,
413 			bool lock_cap, long npage)
414 {
415 	int ret = mmap_write_lock_killable(mm);
416 
417 	if (ret)
418 		return ret;
419 
420 	ret = __account_locked_vm(mm, abs(npage), npage > 0, task, lock_cap);
421 	mmap_write_unlock(mm);
422 	return ret;
423 }
424 
vfio_lock_acct(struct vfio_dma * dma,long npage,bool async)425 static int vfio_lock_acct(struct vfio_dma *dma, long npage, bool async)
426 {
427 	struct mm_struct *mm;
428 	int ret;
429 
430 	if (!npage)
431 		return 0;
432 
433 	mm = dma->mm;
434 	if (async && !mmget_not_zero(mm))
435 		return -ESRCH; /* process exited */
436 
437 	ret = mm_lock_acct(dma->task, mm, dma->lock_cap, npage);
438 	if (!ret)
439 		dma->locked_vm += npage;
440 
441 	if (async)
442 		mmput(mm);
443 
444 	return ret;
445 }
446 
447 /*
448  * Some mappings aren't backed by a struct page, for example an mmap'd
449  * MMIO range for our own or another device.  These use a different
450  * pfn conversion and shouldn't be tracked as locked pages.
451  * For compound pages, any driver that sets the reserved bit in head
452  * page needs to set the reserved bit in all subpages to be safe.
453  */
is_invalid_reserved_pfn(unsigned long pfn)454 static bool is_invalid_reserved_pfn(unsigned long pfn)
455 {
456 	if (pfn_valid(pfn))
457 		return PageReserved(pfn_to_page(pfn));
458 
459 	return true;
460 }
461 
put_pfn(unsigned long pfn,int prot)462 static int put_pfn(unsigned long pfn, int prot)
463 {
464 	if (!is_invalid_reserved_pfn(pfn)) {
465 		struct page *page = pfn_to_page(pfn);
466 
467 		unpin_user_pages_dirty_lock(&page, 1, prot & IOMMU_WRITE);
468 		return 1;
469 	}
470 	return 0;
471 }
472 
473 #define VFIO_BATCH_MAX_CAPACITY (PAGE_SIZE / sizeof(struct page *))
474 
vfio_batch_init(struct vfio_batch * batch)475 static void vfio_batch_init(struct vfio_batch *batch)
476 {
477 	batch->size = 0;
478 	batch->offset = 0;
479 
480 	if (unlikely(disable_hugepages))
481 		goto fallback;
482 
483 	batch->pages = (struct page **) __get_free_page(GFP_KERNEL);
484 	if (!batch->pages)
485 		goto fallback;
486 
487 	batch->capacity = VFIO_BATCH_MAX_CAPACITY;
488 	return;
489 
490 fallback:
491 	batch->pages = &batch->fallback_page;
492 	batch->capacity = 1;
493 }
494 
vfio_batch_unpin(struct vfio_batch * batch,struct vfio_dma * dma)495 static void vfio_batch_unpin(struct vfio_batch *batch, struct vfio_dma *dma)
496 {
497 	while (batch->size) {
498 		unsigned long pfn = page_to_pfn(batch->pages[batch->offset]);
499 
500 		put_pfn(pfn, dma->prot);
501 		batch->offset++;
502 		batch->size--;
503 	}
504 }
505 
vfio_batch_fini(struct vfio_batch * batch)506 static void vfio_batch_fini(struct vfio_batch *batch)
507 {
508 	if (batch->capacity == VFIO_BATCH_MAX_CAPACITY)
509 		free_page((unsigned long)batch->pages);
510 }
511 
follow_fault_pfn(struct vm_area_struct * vma,struct mm_struct * mm,unsigned long vaddr,unsigned long * pfn,bool write_fault)512 static int follow_fault_pfn(struct vm_area_struct *vma, struct mm_struct *mm,
513 			    unsigned long vaddr, unsigned long *pfn,
514 			    bool write_fault)
515 {
516 	struct follow_pfnmap_args args = { .vma = vma, .address = vaddr };
517 	int ret;
518 
519 	ret = follow_pfnmap_start(&args);
520 	if (ret) {
521 		bool unlocked = false;
522 
523 		ret = fixup_user_fault(mm, vaddr,
524 				       FAULT_FLAG_REMOTE |
525 				       (write_fault ?  FAULT_FLAG_WRITE : 0),
526 				       &unlocked);
527 		if (unlocked)
528 			return -EAGAIN;
529 
530 		if (ret)
531 			return ret;
532 
533 		ret = follow_pfnmap_start(&args);
534 		if (ret)
535 			return ret;
536 	}
537 
538 	if (write_fault && !args.writable)
539 		ret = -EFAULT;
540 	else
541 		*pfn = args.pfn;
542 
543 	follow_pfnmap_end(&args);
544 	return ret;
545 }
546 
547 /*
548  * Returns the positive number of pfns successfully obtained or a negative
549  * error code.
550  */
vaddr_get_pfns(struct mm_struct * mm,unsigned long vaddr,long npages,int prot,unsigned long * pfn,struct page ** pages)551 static int vaddr_get_pfns(struct mm_struct *mm, unsigned long vaddr,
552 			  long npages, int prot, unsigned long *pfn,
553 			  struct page **pages)
554 {
555 	struct vm_area_struct *vma;
556 	unsigned int flags = 0;
557 	int ret;
558 
559 	if (prot & IOMMU_WRITE)
560 		flags |= FOLL_WRITE;
561 
562 	mmap_read_lock(mm);
563 	ret = pin_user_pages_remote(mm, vaddr, npages, flags | FOLL_LONGTERM,
564 				    pages, NULL);
565 	if (ret > 0) {
566 		*pfn = page_to_pfn(pages[0]);
567 		goto done;
568 	}
569 
570 	vaddr = untagged_addr_remote(mm, vaddr);
571 
572 retry:
573 	vma = vma_lookup(mm, vaddr);
574 
575 	if (vma && vma->vm_flags & VM_PFNMAP) {
576 		ret = follow_fault_pfn(vma, mm, vaddr, pfn, prot & IOMMU_WRITE);
577 		if (ret == -EAGAIN)
578 			goto retry;
579 
580 		if (!ret) {
581 			if (is_invalid_reserved_pfn(*pfn))
582 				ret = 1;
583 			else
584 				ret = -EFAULT;
585 		}
586 	}
587 done:
588 	mmap_read_unlock(mm);
589 	return ret;
590 }
591 
592 /*
593  * Attempt to pin pages.  We really don't want to track all the pfns and
594  * the iommu can only map chunks of consecutive pfns anyway, so get the
595  * first page and all consecutive pages with the same locking.
596  */
vfio_pin_pages_remote(struct vfio_dma * dma,unsigned long vaddr,long npage,unsigned long * pfn_base,unsigned long limit,struct vfio_batch * batch)597 static long vfio_pin_pages_remote(struct vfio_dma *dma, unsigned long vaddr,
598 				  long npage, unsigned long *pfn_base,
599 				  unsigned long limit, struct vfio_batch *batch)
600 {
601 	unsigned long pfn;
602 	struct mm_struct *mm = current->mm;
603 	long ret, pinned = 0, lock_acct = 0;
604 	bool rsvd;
605 	dma_addr_t iova = vaddr - dma->vaddr + dma->iova;
606 
607 	/* This code path is only user initiated */
608 	if (!mm)
609 		return -ENODEV;
610 
611 	if (batch->size) {
612 		/* Leftover pages in batch from an earlier call. */
613 		*pfn_base = page_to_pfn(batch->pages[batch->offset]);
614 		pfn = *pfn_base;
615 		rsvd = is_invalid_reserved_pfn(*pfn_base);
616 	} else {
617 		*pfn_base = 0;
618 	}
619 
620 	while (npage) {
621 		if (!batch->size) {
622 			/* Empty batch, so refill it. */
623 			long req_pages = min_t(long, npage, batch->capacity);
624 
625 			ret = vaddr_get_pfns(mm, vaddr, req_pages, dma->prot,
626 					     &pfn, batch->pages);
627 			if (ret < 0)
628 				goto unpin_out;
629 
630 			batch->size = ret;
631 			batch->offset = 0;
632 
633 			if (!*pfn_base) {
634 				*pfn_base = pfn;
635 				rsvd = is_invalid_reserved_pfn(*pfn_base);
636 			}
637 		}
638 
639 		/*
640 		 * pfn is preset for the first iteration of this inner loop and
641 		 * updated at the end to handle a VM_PFNMAP pfn.  In that case,
642 		 * batch->pages isn't valid (there's no struct page), so allow
643 		 * batch->pages to be touched only when there's more than one
644 		 * pfn to check, which guarantees the pfns are from a
645 		 * !VM_PFNMAP vma.
646 		 */
647 		while (true) {
648 			if (pfn != *pfn_base + pinned ||
649 			    rsvd != is_invalid_reserved_pfn(pfn))
650 				goto out;
651 
652 			/*
653 			 * Reserved pages aren't counted against the user,
654 			 * externally pinned pages are already counted against
655 			 * the user.
656 			 */
657 			if (!rsvd && !vfio_find_vpfn(dma, iova)) {
658 				if (!dma->lock_cap &&
659 				    mm->locked_vm + lock_acct + 1 > limit) {
660 					pr_warn("%s: RLIMIT_MEMLOCK (%ld) exceeded\n",
661 						__func__, limit << PAGE_SHIFT);
662 					ret = -ENOMEM;
663 					goto unpin_out;
664 				}
665 				lock_acct++;
666 			}
667 
668 			pinned++;
669 			npage--;
670 			vaddr += PAGE_SIZE;
671 			iova += PAGE_SIZE;
672 			batch->offset++;
673 			batch->size--;
674 
675 			if (!batch->size)
676 				break;
677 
678 			pfn = page_to_pfn(batch->pages[batch->offset]);
679 		}
680 
681 		if (unlikely(disable_hugepages))
682 			break;
683 	}
684 
685 out:
686 	ret = vfio_lock_acct(dma, lock_acct, false);
687 
688 unpin_out:
689 	if (batch->size == 1 && !batch->offset) {
690 		/* May be a VM_PFNMAP pfn, which the batch can't remember. */
691 		put_pfn(pfn, dma->prot);
692 		batch->size = 0;
693 	}
694 
695 	if (ret < 0) {
696 		if (pinned && !rsvd) {
697 			for (pfn = *pfn_base ; pinned ; pfn++, pinned--)
698 				put_pfn(pfn, dma->prot);
699 		}
700 		vfio_batch_unpin(batch, dma);
701 
702 		return ret;
703 	}
704 
705 	return pinned;
706 }
707 
vfio_unpin_pages_remote(struct vfio_dma * dma,dma_addr_t iova,unsigned long pfn,long npage,bool do_accounting)708 static long vfio_unpin_pages_remote(struct vfio_dma *dma, dma_addr_t iova,
709 				    unsigned long pfn, long npage,
710 				    bool do_accounting)
711 {
712 	long unlocked = 0, locked = 0;
713 	long i;
714 
715 	for (i = 0; i < npage; i++, iova += PAGE_SIZE) {
716 		if (put_pfn(pfn++, dma->prot)) {
717 			unlocked++;
718 			if (vfio_find_vpfn(dma, iova))
719 				locked++;
720 		}
721 	}
722 
723 	if (do_accounting)
724 		vfio_lock_acct(dma, locked - unlocked, true);
725 
726 	return unlocked;
727 }
728 
vfio_pin_page_external(struct vfio_dma * dma,unsigned long vaddr,unsigned long * pfn_base,bool do_accounting)729 static int vfio_pin_page_external(struct vfio_dma *dma, unsigned long vaddr,
730 				  unsigned long *pfn_base, bool do_accounting)
731 {
732 	struct page *pages[1];
733 	struct mm_struct *mm;
734 	int ret;
735 
736 	mm = dma->mm;
737 	if (!mmget_not_zero(mm))
738 		return -ENODEV;
739 
740 	ret = vaddr_get_pfns(mm, vaddr, 1, dma->prot, pfn_base, pages);
741 	if (ret != 1)
742 		goto out;
743 
744 	ret = 0;
745 
746 	if (do_accounting && !is_invalid_reserved_pfn(*pfn_base)) {
747 		ret = vfio_lock_acct(dma, 1, false);
748 		if (ret) {
749 			put_pfn(*pfn_base, dma->prot);
750 			if (ret == -ENOMEM)
751 				pr_warn("%s: Task %s (%d) RLIMIT_MEMLOCK "
752 					"(%ld) exceeded\n", __func__,
753 					dma->task->comm, task_pid_nr(dma->task),
754 					task_rlimit(dma->task, RLIMIT_MEMLOCK));
755 		}
756 	}
757 
758 out:
759 	mmput(mm);
760 	return ret;
761 }
762 
vfio_unpin_page_external(struct vfio_dma * dma,dma_addr_t iova,bool do_accounting)763 static int vfio_unpin_page_external(struct vfio_dma *dma, dma_addr_t iova,
764 				    bool do_accounting)
765 {
766 	int unlocked;
767 	struct vfio_pfn *vpfn = vfio_find_vpfn(dma, iova);
768 
769 	if (!vpfn)
770 		return 0;
771 
772 	unlocked = vfio_iova_put_vfio_pfn(dma, vpfn);
773 
774 	if (do_accounting)
775 		vfio_lock_acct(dma, -unlocked, true);
776 
777 	return unlocked;
778 }
779 
vfio_iommu_type1_pin_pages(void * iommu_data,struct iommu_group * iommu_group,dma_addr_t user_iova,int npage,int prot,struct page ** pages)780 static int vfio_iommu_type1_pin_pages(void *iommu_data,
781 				      struct iommu_group *iommu_group,
782 				      dma_addr_t user_iova,
783 				      int npage, int prot,
784 				      struct page **pages)
785 {
786 	struct vfio_iommu *iommu = iommu_data;
787 	struct vfio_iommu_group *group;
788 	int i, j, ret;
789 	unsigned long remote_vaddr;
790 	struct vfio_dma *dma;
791 	bool do_accounting;
792 
793 	if (!iommu || !pages)
794 		return -EINVAL;
795 
796 	/* Supported for v2 version only */
797 	if (!iommu->v2)
798 		return -EACCES;
799 
800 	mutex_lock(&iommu->lock);
801 
802 	if (WARN_ONCE(iommu->vaddr_invalid_count,
803 		      "vfio_pin_pages not allowed with VFIO_UPDATE_VADDR\n")) {
804 		ret = -EBUSY;
805 		goto pin_done;
806 	}
807 
808 	/* Fail if no dma_umap notifier is registered */
809 	if (list_empty(&iommu->device_list)) {
810 		ret = -EINVAL;
811 		goto pin_done;
812 	}
813 
814 	/*
815 	 * If iommu capable domain exist in the container then all pages are
816 	 * already pinned and accounted. Accounting should be done if there is no
817 	 * iommu capable domain in the container.
818 	 */
819 	do_accounting = list_empty(&iommu->domain_list);
820 
821 	for (i = 0; i < npage; i++) {
822 		unsigned long phys_pfn;
823 		dma_addr_t iova;
824 		struct vfio_pfn *vpfn;
825 
826 		iova = user_iova + PAGE_SIZE * i;
827 		dma = vfio_find_dma(iommu, iova, PAGE_SIZE);
828 		if (!dma) {
829 			ret = -EINVAL;
830 			goto pin_unwind;
831 		}
832 
833 		if ((dma->prot & prot) != prot) {
834 			ret = -EPERM;
835 			goto pin_unwind;
836 		}
837 
838 		vpfn = vfio_iova_get_vfio_pfn(dma, iova);
839 		if (vpfn) {
840 			pages[i] = pfn_to_page(vpfn->pfn);
841 			continue;
842 		}
843 
844 		remote_vaddr = dma->vaddr + (iova - dma->iova);
845 		ret = vfio_pin_page_external(dma, remote_vaddr, &phys_pfn,
846 					     do_accounting);
847 		if (ret)
848 			goto pin_unwind;
849 
850 		if (!pfn_valid(phys_pfn)) {
851 			ret = -EINVAL;
852 			goto pin_unwind;
853 		}
854 
855 		ret = vfio_add_to_pfn_list(dma, iova, phys_pfn);
856 		if (ret) {
857 			if (put_pfn(phys_pfn, dma->prot) && do_accounting)
858 				vfio_lock_acct(dma, -1, true);
859 			goto pin_unwind;
860 		}
861 
862 		pages[i] = pfn_to_page(phys_pfn);
863 
864 		if (iommu->dirty_page_tracking) {
865 			unsigned long pgshift = __ffs(iommu->pgsize_bitmap);
866 
867 			/*
868 			 * Bitmap populated with the smallest supported page
869 			 * size
870 			 */
871 			bitmap_set(dma->bitmap,
872 				   (iova - dma->iova) >> pgshift, 1);
873 		}
874 	}
875 	ret = i;
876 
877 	group = vfio_iommu_find_iommu_group(iommu, iommu_group);
878 	if (!group->pinned_page_dirty_scope) {
879 		group->pinned_page_dirty_scope = true;
880 		iommu->num_non_pinned_groups--;
881 	}
882 
883 	goto pin_done;
884 
885 pin_unwind:
886 	pages[i] = NULL;
887 	for (j = 0; j < i; j++) {
888 		dma_addr_t iova;
889 
890 		iova = user_iova + PAGE_SIZE * j;
891 		dma = vfio_find_dma(iommu, iova, PAGE_SIZE);
892 		vfio_unpin_page_external(dma, iova, do_accounting);
893 		pages[j] = NULL;
894 	}
895 pin_done:
896 	mutex_unlock(&iommu->lock);
897 	return ret;
898 }
899 
vfio_iommu_type1_unpin_pages(void * iommu_data,dma_addr_t user_iova,int npage)900 static void vfio_iommu_type1_unpin_pages(void *iommu_data,
901 					 dma_addr_t user_iova, int npage)
902 {
903 	struct vfio_iommu *iommu = iommu_data;
904 	bool do_accounting;
905 	int i;
906 
907 	/* Supported for v2 version only */
908 	if (WARN_ON(!iommu->v2))
909 		return;
910 
911 	mutex_lock(&iommu->lock);
912 
913 	do_accounting = list_empty(&iommu->domain_list);
914 	for (i = 0; i < npage; i++) {
915 		dma_addr_t iova = user_iova + PAGE_SIZE * i;
916 		struct vfio_dma *dma;
917 
918 		dma = vfio_find_dma(iommu, iova, PAGE_SIZE);
919 		if (!dma)
920 			break;
921 
922 		vfio_unpin_page_external(dma, iova, do_accounting);
923 	}
924 
925 	mutex_unlock(&iommu->lock);
926 
927 	WARN_ON(i != npage);
928 }
929 
vfio_sync_unpin(struct vfio_dma * dma,struct vfio_domain * domain,struct list_head * regions,struct iommu_iotlb_gather * iotlb_gather)930 static long vfio_sync_unpin(struct vfio_dma *dma, struct vfio_domain *domain,
931 			    struct list_head *regions,
932 			    struct iommu_iotlb_gather *iotlb_gather)
933 {
934 	long unlocked = 0;
935 	struct vfio_regions *entry, *next;
936 
937 	iommu_iotlb_sync(domain->domain, iotlb_gather);
938 
939 	list_for_each_entry_safe(entry, next, regions, list) {
940 		unlocked += vfio_unpin_pages_remote(dma,
941 						    entry->iova,
942 						    entry->phys >> PAGE_SHIFT,
943 						    entry->len >> PAGE_SHIFT,
944 						    false);
945 		list_del(&entry->list);
946 		kfree(entry);
947 	}
948 
949 	cond_resched();
950 
951 	return unlocked;
952 }
953 
954 /*
955  * Generally, VFIO needs to unpin remote pages after each IOTLB flush.
956  * Therefore, when using IOTLB flush sync interface, VFIO need to keep track
957  * of these regions (currently using a list).
958  *
959  * This value specifies maximum number of regions for each IOTLB flush sync.
960  */
961 #define VFIO_IOMMU_TLB_SYNC_MAX		512
962 
unmap_unpin_fast(struct vfio_domain * domain,struct vfio_dma * dma,dma_addr_t * iova,size_t len,phys_addr_t phys,long * unlocked,struct list_head * unmapped_list,int * unmapped_cnt,struct iommu_iotlb_gather * iotlb_gather)963 static size_t unmap_unpin_fast(struct vfio_domain *domain,
964 			       struct vfio_dma *dma, dma_addr_t *iova,
965 			       size_t len, phys_addr_t phys, long *unlocked,
966 			       struct list_head *unmapped_list,
967 			       int *unmapped_cnt,
968 			       struct iommu_iotlb_gather *iotlb_gather)
969 {
970 	size_t unmapped = 0;
971 	struct vfio_regions *entry = kzalloc(sizeof(*entry), GFP_KERNEL);
972 
973 	if (entry) {
974 		unmapped = iommu_unmap_fast(domain->domain, *iova, len,
975 					    iotlb_gather);
976 
977 		if (!unmapped) {
978 			kfree(entry);
979 		} else {
980 			entry->iova = *iova;
981 			entry->phys = phys;
982 			entry->len  = unmapped;
983 			list_add_tail(&entry->list, unmapped_list);
984 
985 			*iova += unmapped;
986 			(*unmapped_cnt)++;
987 		}
988 	}
989 
990 	/*
991 	 * Sync if the number of fast-unmap regions hits the limit
992 	 * or in case of errors.
993 	 */
994 	if (*unmapped_cnt >= VFIO_IOMMU_TLB_SYNC_MAX || !unmapped) {
995 		*unlocked += vfio_sync_unpin(dma, domain, unmapped_list,
996 					     iotlb_gather);
997 		*unmapped_cnt = 0;
998 	}
999 
1000 	return unmapped;
1001 }
1002 
unmap_unpin_slow(struct vfio_domain * domain,struct vfio_dma * dma,dma_addr_t * iova,size_t len,phys_addr_t phys,long * unlocked)1003 static size_t unmap_unpin_slow(struct vfio_domain *domain,
1004 			       struct vfio_dma *dma, dma_addr_t *iova,
1005 			       size_t len, phys_addr_t phys,
1006 			       long *unlocked)
1007 {
1008 	size_t unmapped = iommu_unmap(domain->domain, *iova, len);
1009 
1010 	if (unmapped) {
1011 		*unlocked += vfio_unpin_pages_remote(dma, *iova,
1012 						     phys >> PAGE_SHIFT,
1013 						     unmapped >> PAGE_SHIFT,
1014 						     false);
1015 		*iova += unmapped;
1016 		cond_resched();
1017 	}
1018 	return unmapped;
1019 }
1020 
vfio_unmap_unpin(struct vfio_iommu * iommu,struct vfio_dma * dma,bool do_accounting)1021 static long vfio_unmap_unpin(struct vfio_iommu *iommu, struct vfio_dma *dma,
1022 			     bool do_accounting)
1023 {
1024 	dma_addr_t iova = dma->iova, end = dma->iova + dma->size;
1025 	struct vfio_domain *domain, *d;
1026 	LIST_HEAD(unmapped_region_list);
1027 	struct iommu_iotlb_gather iotlb_gather;
1028 	int unmapped_region_cnt = 0;
1029 	long unlocked = 0;
1030 
1031 	if (!dma->size)
1032 		return 0;
1033 
1034 	if (list_empty(&iommu->domain_list))
1035 		return 0;
1036 
1037 	/*
1038 	 * We use the IOMMU to track the physical addresses, otherwise we'd
1039 	 * need a much more complicated tracking system.  Unfortunately that
1040 	 * means we need to use one of the iommu domains to figure out the
1041 	 * pfns to unpin.  The rest need to be unmapped in advance so we have
1042 	 * no iommu translations remaining when the pages are unpinned.
1043 	 */
1044 	domain = d = list_first_entry(&iommu->domain_list,
1045 				      struct vfio_domain, next);
1046 
1047 	list_for_each_entry_continue(d, &iommu->domain_list, next) {
1048 		iommu_unmap(d->domain, dma->iova, dma->size);
1049 		cond_resched();
1050 	}
1051 
1052 	iommu_iotlb_gather_init(&iotlb_gather);
1053 	while (iova < end) {
1054 		size_t unmapped, len;
1055 		phys_addr_t phys, next;
1056 
1057 		phys = iommu_iova_to_phys(domain->domain, iova);
1058 		if (WARN_ON(!phys)) {
1059 			iova += PAGE_SIZE;
1060 			continue;
1061 		}
1062 
1063 		/*
1064 		 * To optimize for fewer iommu_unmap() calls, each of which
1065 		 * may require hardware cache flushing, try to find the
1066 		 * largest contiguous physical memory chunk to unmap.
1067 		 */
1068 		for (len = PAGE_SIZE;
1069 		     !domain->fgsp && iova + len < end; len += PAGE_SIZE) {
1070 			next = iommu_iova_to_phys(domain->domain, iova + len);
1071 			if (next != phys + len)
1072 				break;
1073 		}
1074 
1075 		/*
1076 		 * First, try to use fast unmap/unpin. In case of failure,
1077 		 * switch to slow unmap/unpin path.
1078 		 */
1079 		unmapped = unmap_unpin_fast(domain, dma, &iova, len, phys,
1080 					    &unlocked, &unmapped_region_list,
1081 					    &unmapped_region_cnt,
1082 					    &iotlb_gather);
1083 		if (!unmapped) {
1084 			unmapped = unmap_unpin_slow(domain, dma, &iova, len,
1085 						    phys, &unlocked);
1086 			if (WARN_ON(!unmapped))
1087 				break;
1088 		}
1089 	}
1090 
1091 	dma->iommu_mapped = false;
1092 
1093 	if (unmapped_region_cnt) {
1094 		unlocked += vfio_sync_unpin(dma, domain, &unmapped_region_list,
1095 					    &iotlb_gather);
1096 	}
1097 
1098 	if (do_accounting) {
1099 		vfio_lock_acct(dma, -unlocked, true);
1100 		return 0;
1101 	}
1102 	return unlocked;
1103 }
1104 
vfio_remove_dma(struct vfio_iommu * iommu,struct vfio_dma * dma)1105 static void vfio_remove_dma(struct vfio_iommu *iommu, struct vfio_dma *dma)
1106 {
1107 	WARN_ON(!RB_EMPTY_ROOT(&dma->pfn_list));
1108 	vfio_unmap_unpin(iommu, dma, true);
1109 	vfio_unlink_dma(iommu, dma);
1110 	put_task_struct(dma->task);
1111 	mmdrop(dma->mm);
1112 	vfio_dma_bitmap_free(dma);
1113 	if (dma->vaddr_invalid)
1114 		iommu->vaddr_invalid_count--;
1115 	kfree(dma);
1116 	iommu->dma_avail++;
1117 }
1118 
vfio_update_pgsize_bitmap(struct vfio_iommu * iommu)1119 static void vfio_update_pgsize_bitmap(struct vfio_iommu *iommu)
1120 {
1121 	struct vfio_domain *domain;
1122 
1123 	iommu->pgsize_bitmap = ULONG_MAX;
1124 
1125 	list_for_each_entry(domain, &iommu->domain_list, next)
1126 		iommu->pgsize_bitmap &= domain->domain->pgsize_bitmap;
1127 
1128 	/*
1129 	 * In case the IOMMU supports page sizes smaller than PAGE_SIZE
1130 	 * we pretend PAGE_SIZE is supported and hide sub-PAGE_SIZE sizes.
1131 	 * That way the user will be able to map/unmap buffers whose size/
1132 	 * start address is aligned with PAGE_SIZE. Pinning code uses that
1133 	 * granularity while iommu driver can use the sub-PAGE_SIZE size
1134 	 * to map the buffer.
1135 	 */
1136 	if (iommu->pgsize_bitmap & ~PAGE_MASK) {
1137 		iommu->pgsize_bitmap &= PAGE_MASK;
1138 		iommu->pgsize_bitmap |= PAGE_SIZE;
1139 	}
1140 }
1141 
update_user_bitmap(u64 __user * bitmap,struct vfio_iommu * iommu,struct vfio_dma * dma,dma_addr_t base_iova,size_t pgsize)1142 static int update_user_bitmap(u64 __user *bitmap, struct vfio_iommu *iommu,
1143 			      struct vfio_dma *dma, dma_addr_t base_iova,
1144 			      size_t pgsize)
1145 {
1146 	unsigned long pgshift = __ffs(pgsize);
1147 	unsigned long nbits = dma->size >> pgshift;
1148 	unsigned long bit_offset = (dma->iova - base_iova) >> pgshift;
1149 	unsigned long copy_offset = bit_offset / BITS_PER_LONG;
1150 	unsigned long shift = bit_offset % BITS_PER_LONG;
1151 	unsigned long leftover;
1152 
1153 	/*
1154 	 * mark all pages dirty if any IOMMU capable device is not able
1155 	 * to report dirty pages and all pages are pinned and mapped.
1156 	 */
1157 	if (iommu->num_non_pinned_groups && dma->iommu_mapped)
1158 		bitmap_set(dma->bitmap, 0, nbits);
1159 
1160 	if (shift) {
1161 		bitmap_shift_left(dma->bitmap, dma->bitmap, shift,
1162 				  nbits + shift);
1163 
1164 		if (copy_from_user(&leftover,
1165 				   (void __user *)(bitmap + copy_offset),
1166 				   sizeof(leftover)))
1167 			return -EFAULT;
1168 
1169 		bitmap_or(dma->bitmap, dma->bitmap, &leftover, shift);
1170 	}
1171 
1172 	if (copy_to_user((void __user *)(bitmap + copy_offset), dma->bitmap,
1173 			 DIRTY_BITMAP_BYTES(nbits + shift)))
1174 		return -EFAULT;
1175 
1176 	return 0;
1177 }
1178 
vfio_iova_dirty_bitmap(u64 __user * bitmap,struct vfio_iommu * iommu,dma_addr_t iova,size_t size,size_t pgsize)1179 static int vfio_iova_dirty_bitmap(u64 __user *bitmap, struct vfio_iommu *iommu,
1180 				  dma_addr_t iova, size_t size, size_t pgsize)
1181 {
1182 	struct vfio_dma *dma;
1183 	struct rb_node *n;
1184 	unsigned long pgshift = __ffs(pgsize);
1185 	int ret;
1186 
1187 	/*
1188 	 * GET_BITMAP request must fully cover vfio_dma mappings.  Multiple
1189 	 * vfio_dma mappings may be clubbed by specifying large ranges, but
1190 	 * there must not be any previous mappings bisected by the range.
1191 	 * An error will be returned if these conditions are not met.
1192 	 */
1193 	dma = vfio_find_dma(iommu, iova, 1);
1194 	if (dma && dma->iova != iova)
1195 		return -EINVAL;
1196 
1197 	dma = vfio_find_dma(iommu, iova + size - 1, 0);
1198 	if (dma && dma->iova + dma->size != iova + size)
1199 		return -EINVAL;
1200 
1201 	for (n = rb_first(&iommu->dma_list); n; n = rb_next(n)) {
1202 		struct vfio_dma *dma = rb_entry(n, struct vfio_dma, node);
1203 
1204 		if (dma->iova < iova)
1205 			continue;
1206 
1207 		if (dma->iova > iova + size - 1)
1208 			break;
1209 
1210 		ret = update_user_bitmap(bitmap, iommu, dma, iova, pgsize);
1211 		if (ret)
1212 			return ret;
1213 
1214 		/*
1215 		 * Re-populate bitmap to include all pinned pages which are
1216 		 * considered as dirty but exclude pages which are unpinned and
1217 		 * pages which are marked dirty by vfio_dma_rw()
1218 		 */
1219 		bitmap_clear(dma->bitmap, 0, dma->size >> pgshift);
1220 		vfio_dma_populate_bitmap(dma, pgsize);
1221 	}
1222 	return 0;
1223 }
1224 
verify_bitmap_size(uint64_t npages,uint64_t bitmap_size)1225 static int verify_bitmap_size(uint64_t npages, uint64_t bitmap_size)
1226 {
1227 	if (!npages || !bitmap_size || (bitmap_size > DIRTY_BITMAP_SIZE_MAX) ||
1228 	    (bitmap_size < DIRTY_BITMAP_BYTES(npages)))
1229 		return -EINVAL;
1230 
1231 	return 0;
1232 }
1233 
1234 /*
1235  * Notify VFIO drivers using vfio_register_emulated_iommu_dev() to invalidate
1236  * and unmap iovas within the range we're about to unmap. Drivers MUST unpin
1237  * pages in response to an invalidation.
1238  */
vfio_notify_dma_unmap(struct vfio_iommu * iommu,struct vfio_dma * dma)1239 static void vfio_notify_dma_unmap(struct vfio_iommu *iommu,
1240 				  struct vfio_dma *dma)
1241 {
1242 	struct vfio_device *device;
1243 
1244 	if (list_empty(&iommu->device_list))
1245 		return;
1246 
1247 	/*
1248 	 * The device is expected to call vfio_unpin_pages() for any IOVA it has
1249 	 * pinned within the range. Since vfio_unpin_pages() will eventually
1250 	 * call back down to this code and try to obtain the iommu->lock we must
1251 	 * drop it.
1252 	 */
1253 	mutex_lock(&iommu->device_list_lock);
1254 	mutex_unlock(&iommu->lock);
1255 
1256 	list_for_each_entry(device, &iommu->device_list, iommu_entry)
1257 		device->ops->dma_unmap(device, dma->iova, dma->size);
1258 
1259 	mutex_unlock(&iommu->device_list_lock);
1260 	mutex_lock(&iommu->lock);
1261 }
1262 
vfio_dma_do_unmap(struct vfio_iommu * iommu,struct vfio_iommu_type1_dma_unmap * unmap,struct vfio_bitmap * bitmap)1263 static int vfio_dma_do_unmap(struct vfio_iommu *iommu,
1264 			     struct vfio_iommu_type1_dma_unmap *unmap,
1265 			     struct vfio_bitmap *bitmap)
1266 {
1267 	struct vfio_dma *dma, *dma_last = NULL;
1268 	size_t unmapped = 0, pgsize;
1269 	int ret = -EINVAL, retries = 0;
1270 	unsigned long pgshift;
1271 	dma_addr_t iova = unmap->iova;
1272 	u64 size = unmap->size;
1273 	bool unmap_all = unmap->flags & VFIO_DMA_UNMAP_FLAG_ALL;
1274 	bool invalidate_vaddr = unmap->flags & VFIO_DMA_UNMAP_FLAG_VADDR;
1275 	struct rb_node *n, *first_n;
1276 
1277 	mutex_lock(&iommu->lock);
1278 
1279 	/* Cannot update vaddr if mdev is present. */
1280 	if (invalidate_vaddr && !list_empty(&iommu->emulated_iommu_groups)) {
1281 		ret = -EBUSY;
1282 		goto unlock;
1283 	}
1284 
1285 	pgshift = __ffs(iommu->pgsize_bitmap);
1286 	pgsize = (size_t)1 << pgshift;
1287 
1288 	if (iova & (pgsize - 1))
1289 		goto unlock;
1290 
1291 	if (unmap_all) {
1292 		if (iova || size)
1293 			goto unlock;
1294 		size = U64_MAX;
1295 	} else if (!size || size & (pgsize - 1) ||
1296 		   iova + size - 1 < iova || size > SIZE_MAX) {
1297 		goto unlock;
1298 	}
1299 
1300 	/* When dirty tracking is enabled, allow only min supported pgsize */
1301 	if ((unmap->flags & VFIO_DMA_UNMAP_FLAG_GET_DIRTY_BITMAP) &&
1302 	    (!iommu->dirty_page_tracking || (bitmap->pgsize != pgsize))) {
1303 		goto unlock;
1304 	}
1305 
1306 	WARN_ON((pgsize - 1) & PAGE_MASK);
1307 again:
1308 	/*
1309 	 * vfio-iommu-type1 (v1) - User mappings were coalesced together to
1310 	 * avoid tracking individual mappings.  This means that the granularity
1311 	 * of the original mapping was lost and the user was allowed to attempt
1312 	 * to unmap any range.  Depending on the contiguousness of physical
1313 	 * memory and page sizes supported by the IOMMU, arbitrary unmaps may
1314 	 * or may not have worked.  We only guaranteed unmap granularity
1315 	 * matching the original mapping; even though it was untracked here,
1316 	 * the original mappings are reflected in IOMMU mappings.  This
1317 	 * resulted in a couple unusual behaviors.  First, if a range is not
1318 	 * able to be unmapped, ex. a set of 4k pages that was mapped as a
1319 	 * 2M hugepage into the IOMMU, the unmap ioctl returns success but with
1320 	 * a zero sized unmap.  Also, if an unmap request overlaps the first
1321 	 * address of a hugepage, the IOMMU will unmap the entire hugepage.
1322 	 * This also returns success and the returned unmap size reflects the
1323 	 * actual size unmapped.
1324 	 *
1325 	 * We attempt to maintain compatibility with this "v1" interface, but
1326 	 * we take control out of the hands of the IOMMU.  Therefore, an unmap
1327 	 * request offset from the beginning of the original mapping will
1328 	 * return success with zero sized unmap.  And an unmap request covering
1329 	 * the first iova of mapping will unmap the entire range.
1330 	 *
1331 	 * The v2 version of this interface intends to be more deterministic.
1332 	 * Unmap requests must fully cover previous mappings.  Multiple
1333 	 * mappings may still be unmaped by specifying large ranges, but there
1334 	 * must not be any previous mappings bisected by the range.  An error
1335 	 * will be returned if these conditions are not met.  The v2 interface
1336 	 * will only return success and a size of zero if there were no
1337 	 * mappings within the range.
1338 	 */
1339 	if (iommu->v2 && !unmap_all) {
1340 		dma = vfio_find_dma(iommu, iova, 1);
1341 		if (dma && dma->iova != iova)
1342 			goto unlock;
1343 
1344 		dma = vfio_find_dma(iommu, iova + size - 1, 0);
1345 		if (dma && dma->iova + dma->size != iova + size)
1346 			goto unlock;
1347 	}
1348 
1349 	ret = 0;
1350 	n = first_n = vfio_find_dma_first_node(iommu, iova, size);
1351 
1352 	while (n) {
1353 		dma = rb_entry(n, struct vfio_dma, node);
1354 		if (dma->iova >= iova + size)
1355 			break;
1356 
1357 		if (!iommu->v2 && iova > dma->iova)
1358 			break;
1359 
1360 		if (invalidate_vaddr) {
1361 			if (dma->vaddr_invalid) {
1362 				struct rb_node *last_n = n;
1363 
1364 				for (n = first_n; n != last_n; n = rb_next(n)) {
1365 					dma = rb_entry(n,
1366 						       struct vfio_dma, node);
1367 					dma->vaddr_invalid = false;
1368 					iommu->vaddr_invalid_count--;
1369 				}
1370 				ret = -EINVAL;
1371 				unmapped = 0;
1372 				break;
1373 			}
1374 			dma->vaddr_invalid = true;
1375 			iommu->vaddr_invalid_count++;
1376 			unmapped += dma->size;
1377 			n = rb_next(n);
1378 			continue;
1379 		}
1380 
1381 		if (!RB_EMPTY_ROOT(&dma->pfn_list)) {
1382 			if (dma_last == dma) {
1383 				BUG_ON(++retries > 10);
1384 			} else {
1385 				dma_last = dma;
1386 				retries = 0;
1387 			}
1388 
1389 			vfio_notify_dma_unmap(iommu, dma);
1390 			goto again;
1391 		}
1392 
1393 		if (unmap->flags & VFIO_DMA_UNMAP_FLAG_GET_DIRTY_BITMAP) {
1394 			ret = update_user_bitmap(bitmap->data, iommu, dma,
1395 						 iova, pgsize);
1396 			if (ret)
1397 				break;
1398 		}
1399 
1400 		unmapped += dma->size;
1401 		n = rb_next(n);
1402 		vfio_remove_dma(iommu, dma);
1403 	}
1404 
1405 unlock:
1406 	mutex_unlock(&iommu->lock);
1407 
1408 	/* Report how much was unmapped */
1409 	unmap->size = unmapped;
1410 
1411 	return ret;
1412 }
1413 
vfio_iommu_map(struct vfio_iommu * iommu,dma_addr_t iova,unsigned long pfn,long npage,int prot)1414 static int vfio_iommu_map(struct vfio_iommu *iommu, dma_addr_t iova,
1415 			  unsigned long pfn, long npage, int prot)
1416 {
1417 	struct vfio_domain *d;
1418 	int ret;
1419 
1420 	list_for_each_entry(d, &iommu->domain_list, next) {
1421 		ret = iommu_map(d->domain, iova, (phys_addr_t)pfn << PAGE_SHIFT,
1422 				npage << PAGE_SHIFT, prot | IOMMU_CACHE,
1423 				GFP_KERNEL_ACCOUNT);
1424 		if (ret)
1425 			goto unwind;
1426 
1427 		cond_resched();
1428 	}
1429 
1430 	return 0;
1431 
1432 unwind:
1433 	list_for_each_entry_continue_reverse(d, &iommu->domain_list, next) {
1434 		iommu_unmap(d->domain, iova, npage << PAGE_SHIFT);
1435 		cond_resched();
1436 	}
1437 
1438 	return ret;
1439 }
1440 
vfio_pin_map_dma(struct vfio_iommu * iommu,struct vfio_dma * dma,size_t map_size)1441 static int vfio_pin_map_dma(struct vfio_iommu *iommu, struct vfio_dma *dma,
1442 			    size_t map_size)
1443 {
1444 	dma_addr_t iova = dma->iova;
1445 	unsigned long vaddr = dma->vaddr;
1446 	struct vfio_batch batch;
1447 	size_t size = map_size;
1448 	long npage;
1449 	unsigned long pfn, limit = rlimit(RLIMIT_MEMLOCK) >> PAGE_SHIFT;
1450 	int ret = 0;
1451 
1452 	vfio_batch_init(&batch);
1453 
1454 	while (size) {
1455 		/* Pin a contiguous chunk of memory */
1456 		npage = vfio_pin_pages_remote(dma, vaddr + dma->size,
1457 					      size >> PAGE_SHIFT, &pfn, limit,
1458 					      &batch);
1459 		if (npage <= 0) {
1460 			WARN_ON(!npage);
1461 			ret = (int)npage;
1462 			break;
1463 		}
1464 
1465 		/* Map it! */
1466 		ret = vfio_iommu_map(iommu, iova + dma->size, pfn, npage,
1467 				     dma->prot);
1468 		if (ret) {
1469 			vfio_unpin_pages_remote(dma, iova + dma->size, pfn,
1470 						npage, true);
1471 			vfio_batch_unpin(&batch, dma);
1472 			break;
1473 		}
1474 
1475 		size -= npage << PAGE_SHIFT;
1476 		dma->size += npage << PAGE_SHIFT;
1477 	}
1478 
1479 	vfio_batch_fini(&batch);
1480 	dma->iommu_mapped = true;
1481 
1482 	if (ret)
1483 		vfio_remove_dma(iommu, dma);
1484 
1485 	return ret;
1486 }
1487 
1488 /*
1489  * Check dma map request is within a valid iova range
1490  */
vfio_iommu_iova_dma_valid(struct vfio_iommu * iommu,dma_addr_t start,dma_addr_t end)1491 static bool vfio_iommu_iova_dma_valid(struct vfio_iommu *iommu,
1492 				      dma_addr_t start, dma_addr_t end)
1493 {
1494 	struct list_head *iova = &iommu->iova_list;
1495 	struct vfio_iova *node;
1496 
1497 	list_for_each_entry(node, iova, list) {
1498 		if (start >= node->start && end <= node->end)
1499 			return true;
1500 	}
1501 
1502 	/*
1503 	 * Check for list_empty() as well since a container with
1504 	 * a single mdev device will have an empty list.
1505 	 */
1506 	return list_empty(iova);
1507 }
1508 
vfio_change_dma_owner(struct vfio_dma * dma)1509 static int vfio_change_dma_owner(struct vfio_dma *dma)
1510 {
1511 	struct task_struct *task = current->group_leader;
1512 	struct mm_struct *mm = current->mm;
1513 	long npage = dma->locked_vm;
1514 	bool lock_cap;
1515 	int ret;
1516 
1517 	if (mm == dma->mm)
1518 		return 0;
1519 
1520 	lock_cap = capable(CAP_IPC_LOCK);
1521 	ret = mm_lock_acct(task, mm, lock_cap, npage);
1522 	if (ret)
1523 		return ret;
1524 
1525 	if (mmget_not_zero(dma->mm)) {
1526 		mm_lock_acct(dma->task, dma->mm, dma->lock_cap, -npage);
1527 		mmput(dma->mm);
1528 	}
1529 
1530 	if (dma->task != task) {
1531 		put_task_struct(dma->task);
1532 		dma->task = get_task_struct(task);
1533 	}
1534 	mmdrop(dma->mm);
1535 	dma->mm = mm;
1536 	mmgrab(dma->mm);
1537 	dma->lock_cap = lock_cap;
1538 	return 0;
1539 }
1540 
vfio_dma_do_map(struct vfio_iommu * iommu,struct vfio_iommu_type1_dma_map * map)1541 static int vfio_dma_do_map(struct vfio_iommu *iommu,
1542 			   struct vfio_iommu_type1_dma_map *map)
1543 {
1544 	bool set_vaddr = map->flags & VFIO_DMA_MAP_FLAG_VADDR;
1545 	dma_addr_t iova = map->iova;
1546 	unsigned long vaddr = map->vaddr;
1547 	size_t size = map->size;
1548 	int ret = 0, prot = 0;
1549 	size_t pgsize;
1550 	struct vfio_dma *dma;
1551 
1552 	/* Verify that none of our __u64 fields overflow */
1553 	if (map->size != size || map->vaddr != vaddr || map->iova != iova)
1554 		return -EINVAL;
1555 
1556 	/* READ/WRITE from device perspective */
1557 	if (map->flags & VFIO_DMA_MAP_FLAG_WRITE)
1558 		prot |= IOMMU_WRITE;
1559 	if (map->flags & VFIO_DMA_MAP_FLAG_READ)
1560 		prot |= IOMMU_READ;
1561 
1562 	if ((prot && set_vaddr) || (!prot && !set_vaddr))
1563 		return -EINVAL;
1564 
1565 	mutex_lock(&iommu->lock);
1566 
1567 	pgsize = (size_t)1 << __ffs(iommu->pgsize_bitmap);
1568 
1569 	WARN_ON((pgsize - 1) & PAGE_MASK);
1570 
1571 	if (!size || (size | iova | vaddr) & (pgsize - 1)) {
1572 		ret = -EINVAL;
1573 		goto out_unlock;
1574 	}
1575 
1576 	/* Don't allow IOVA or virtual address wrap */
1577 	if (iova + size - 1 < iova || vaddr + size - 1 < vaddr) {
1578 		ret = -EINVAL;
1579 		goto out_unlock;
1580 	}
1581 
1582 	dma = vfio_find_dma(iommu, iova, size);
1583 	if (set_vaddr) {
1584 		if (!dma) {
1585 			ret = -ENOENT;
1586 		} else if (!dma->vaddr_invalid || dma->iova != iova ||
1587 			   dma->size != size) {
1588 			ret = -EINVAL;
1589 		} else {
1590 			ret = vfio_change_dma_owner(dma);
1591 			if (ret)
1592 				goto out_unlock;
1593 			dma->vaddr = vaddr;
1594 			dma->vaddr_invalid = false;
1595 			iommu->vaddr_invalid_count--;
1596 		}
1597 		goto out_unlock;
1598 	} else if (dma) {
1599 		ret = -EEXIST;
1600 		goto out_unlock;
1601 	}
1602 
1603 	if (!iommu->dma_avail) {
1604 		ret = -ENOSPC;
1605 		goto out_unlock;
1606 	}
1607 
1608 	if (!vfio_iommu_iova_dma_valid(iommu, iova, iova + size - 1)) {
1609 		ret = -EINVAL;
1610 		goto out_unlock;
1611 	}
1612 
1613 	dma = kzalloc(sizeof(*dma), GFP_KERNEL);
1614 	if (!dma) {
1615 		ret = -ENOMEM;
1616 		goto out_unlock;
1617 	}
1618 
1619 	iommu->dma_avail--;
1620 	dma->iova = iova;
1621 	dma->vaddr = vaddr;
1622 	dma->prot = prot;
1623 
1624 	/*
1625 	 * We need to be able to both add to a task's locked memory and test
1626 	 * against the locked memory limit and we need to be able to do both
1627 	 * outside of this call path as pinning can be asynchronous via the
1628 	 * external interfaces for mdev devices.  RLIMIT_MEMLOCK requires a
1629 	 * task_struct. Save the group_leader so that all DMA tracking uses
1630 	 * the same task, to make debugging easier.  VM locked pages requires
1631 	 * an mm_struct, so grab the mm in case the task dies.
1632 	 */
1633 	get_task_struct(current->group_leader);
1634 	dma->task = current->group_leader;
1635 	dma->lock_cap = capable(CAP_IPC_LOCK);
1636 	dma->mm = current->mm;
1637 	mmgrab(dma->mm);
1638 
1639 	dma->pfn_list = RB_ROOT;
1640 
1641 	/* Insert zero-sized and grow as we map chunks of it */
1642 	vfio_link_dma(iommu, dma);
1643 
1644 	/* Don't pin and map if container doesn't contain IOMMU capable domain*/
1645 	if (list_empty(&iommu->domain_list))
1646 		dma->size = size;
1647 	else
1648 		ret = vfio_pin_map_dma(iommu, dma, size);
1649 
1650 	if (!ret && iommu->dirty_page_tracking) {
1651 		ret = vfio_dma_bitmap_alloc(dma, pgsize);
1652 		if (ret)
1653 			vfio_remove_dma(iommu, dma);
1654 	}
1655 
1656 out_unlock:
1657 	mutex_unlock(&iommu->lock);
1658 	return ret;
1659 }
1660 
vfio_iommu_replay(struct vfio_iommu * iommu,struct vfio_domain * domain)1661 static int vfio_iommu_replay(struct vfio_iommu *iommu,
1662 			     struct vfio_domain *domain)
1663 {
1664 	struct vfio_batch batch;
1665 	struct vfio_domain *d = NULL;
1666 	struct rb_node *n;
1667 	unsigned long limit = rlimit(RLIMIT_MEMLOCK) >> PAGE_SHIFT;
1668 	int ret;
1669 
1670 	/* Arbitrarily pick the first domain in the list for lookups */
1671 	if (!list_empty(&iommu->domain_list))
1672 		d = list_first_entry(&iommu->domain_list,
1673 				     struct vfio_domain, next);
1674 
1675 	vfio_batch_init(&batch);
1676 
1677 	n = rb_first(&iommu->dma_list);
1678 
1679 	for (; n; n = rb_next(n)) {
1680 		struct vfio_dma *dma;
1681 		dma_addr_t iova;
1682 
1683 		dma = rb_entry(n, struct vfio_dma, node);
1684 		iova = dma->iova;
1685 
1686 		while (iova < dma->iova + dma->size) {
1687 			phys_addr_t phys;
1688 			size_t size;
1689 
1690 			if (dma->iommu_mapped) {
1691 				phys_addr_t p;
1692 				dma_addr_t i;
1693 
1694 				if (WARN_ON(!d)) { /* mapped w/o a domain?! */
1695 					ret = -EINVAL;
1696 					goto unwind;
1697 				}
1698 
1699 				phys = iommu_iova_to_phys(d->domain, iova);
1700 
1701 				if (WARN_ON(!phys)) {
1702 					iova += PAGE_SIZE;
1703 					continue;
1704 				}
1705 
1706 				size = PAGE_SIZE;
1707 				p = phys + size;
1708 				i = iova + size;
1709 				while (i < dma->iova + dma->size &&
1710 				       p == iommu_iova_to_phys(d->domain, i)) {
1711 					size += PAGE_SIZE;
1712 					p += PAGE_SIZE;
1713 					i += PAGE_SIZE;
1714 				}
1715 			} else {
1716 				unsigned long pfn;
1717 				unsigned long vaddr = dma->vaddr +
1718 						     (iova - dma->iova);
1719 				size_t n = dma->iova + dma->size - iova;
1720 				long npage;
1721 
1722 				npage = vfio_pin_pages_remote(dma, vaddr,
1723 							      n >> PAGE_SHIFT,
1724 							      &pfn, limit,
1725 							      &batch);
1726 				if (npage <= 0) {
1727 					WARN_ON(!npage);
1728 					ret = (int)npage;
1729 					goto unwind;
1730 				}
1731 
1732 				phys = pfn << PAGE_SHIFT;
1733 				size = npage << PAGE_SHIFT;
1734 			}
1735 
1736 			ret = iommu_map(domain->domain, iova, phys, size,
1737 					dma->prot | IOMMU_CACHE,
1738 					GFP_KERNEL_ACCOUNT);
1739 			if (ret) {
1740 				if (!dma->iommu_mapped) {
1741 					vfio_unpin_pages_remote(dma, iova,
1742 							phys >> PAGE_SHIFT,
1743 							size >> PAGE_SHIFT,
1744 							true);
1745 					vfio_batch_unpin(&batch, dma);
1746 				}
1747 				goto unwind;
1748 			}
1749 
1750 			iova += size;
1751 		}
1752 	}
1753 
1754 	/* All dmas are now mapped, defer to second tree walk for unwind */
1755 	for (n = rb_first(&iommu->dma_list); n; n = rb_next(n)) {
1756 		struct vfio_dma *dma = rb_entry(n, struct vfio_dma, node);
1757 
1758 		dma->iommu_mapped = true;
1759 	}
1760 
1761 	vfio_batch_fini(&batch);
1762 	return 0;
1763 
1764 unwind:
1765 	for (; n; n = rb_prev(n)) {
1766 		struct vfio_dma *dma = rb_entry(n, struct vfio_dma, node);
1767 		dma_addr_t iova;
1768 
1769 		if (dma->iommu_mapped) {
1770 			iommu_unmap(domain->domain, dma->iova, dma->size);
1771 			continue;
1772 		}
1773 
1774 		iova = dma->iova;
1775 		while (iova < dma->iova + dma->size) {
1776 			phys_addr_t phys, p;
1777 			size_t size;
1778 			dma_addr_t i;
1779 
1780 			phys = iommu_iova_to_phys(domain->domain, iova);
1781 			if (!phys) {
1782 				iova += PAGE_SIZE;
1783 				continue;
1784 			}
1785 
1786 			size = PAGE_SIZE;
1787 			p = phys + size;
1788 			i = iova + size;
1789 			while (i < dma->iova + dma->size &&
1790 			       p == iommu_iova_to_phys(domain->domain, i)) {
1791 				size += PAGE_SIZE;
1792 				p += PAGE_SIZE;
1793 				i += PAGE_SIZE;
1794 			}
1795 
1796 			iommu_unmap(domain->domain, iova, size);
1797 			vfio_unpin_pages_remote(dma, iova, phys >> PAGE_SHIFT,
1798 						size >> PAGE_SHIFT, true);
1799 		}
1800 	}
1801 
1802 	vfio_batch_fini(&batch);
1803 	return ret;
1804 }
1805 
1806 /*
1807  * We change our unmap behavior slightly depending on whether the IOMMU
1808  * supports fine-grained superpages.  IOMMUs like AMD-Vi will use a superpage
1809  * for practically any contiguous power-of-two mapping we give it.  This means
1810  * we don't need to look for contiguous chunks ourselves to make unmapping
1811  * more efficient.  On IOMMUs with coarse-grained super pages, like Intel VT-d
1812  * with discrete 2M/1G/512G/1T superpages, identifying contiguous chunks
1813  * significantly boosts non-hugetlbfs mappings and doesn't seem to hurt when
1814  * hugetlbfs is in use.
1815  */
vfio_test_domain_fgsp(struct vfio_domain * domain,struct list_head * regions)1816 static void vfio_test_domain_fgsp(struct vfio_domain *domain, struct list_head *regions)
1817 {
1818 	int ret, order = get_order(PAGE_SIZE * 2);
1819 	struct vfio_iova *region;
1820 	struct page *pages;
1821 	dma_addr_t start;
1822 
1823 	pages = alloc_pages(GFP_KERNEL | __GFP_ZERO, order);
1824 	if (!pages)
1825 		return;
1826 
1827 	list_for_each_entry(region, regions, list) {
1828 		start = ALIGN(region->start, PAGE_SIZE * 2);
1829 		if (start >= region->end || (region->end - start < PAGE_SIZE * 2))
1830 			continue;
1831 
1832 		ret = iommu_map(domain->domain, start, page_to_phys(pages), PAGE_SIZE * 2,
1833 				IOMMU_READ | IOMMU_WRITE | IOMMU_CACHE,
1834 				GFP_KERNEL_ACCOUNT);
1835 		if (!ret) {
1836 			size_t unmapped = iommu_unmap(domain->domain, start, PAGE_SIZE);
1837 
1838 			if (unmapped == PAGE_SIZE)
1839 				iommu_unmap(domain->domain, start + PAGE_SIZE, PAGE_SIZE);
1840 			else
1841 				domain->fgsp = true;
1842 		}
1843 		break;
1844 	}
1845 
1846 	__free_pages(pages, order);
1847 }
1848 
find_iommu_group(struct vfio_domain * domain,struct iommu_group * iommu_group)1849 static struct vfio_iommu_group *find_iommu_group(struct vfio_domain *domain,
1850 						 struct iommu_group *iommu_group)
1851 {
1852 	struct vfio_iommu_group *g;
1853 
1854 	list_for_each_entry(g, &domain->group_list, next) {
1855 		if (g->iommu_group == iommu_group)
1856 			return g;
1857 	}
1858 
1859 	return NULL;
1860 }
1861 
1862 static struct vfio_iommu_group*
vfio_iommu_find_iommu_group(struct vfio_iommu * iommu,struct iommu_group * iommu_group)1863 vfio_iommu_find_iommu_group(struct vfio_iommu *iommu,
1864 			    struct iommu_group *iommu_group)
1865 {
1866 	struct vfio_iommu_group *group;
1867 	struct vfio_domain *domain;
1868 
1869 	list_for_each_entry(domain, &iommu->domain_list, next) {
1870 		group = find_iommu_group(domain, iommu_group);
1871 		if (group)
1872 			return group;
1873 	}
1874 
1875 	list_for_each_entry(group, &iommu->emulated_iommu_groups, next)
1876 		if (group->iommu_group == iommu_group)
1877 			return group;
1878 	return NULL;
1879 }
1880 
vfio_iommu_has_sw_msi(struct list_head * group_resv_regions,phys_addr_t * base)1881 static bool vfio_iommu_has_sw_msi(struct list_head *group_resv_regions,
1882 				  phys_addr_t *base)
1883 {
1884 	struct iommu_resv_region *region;
1885 	bool ret = false;
1886 
1887 	list_for_each_entry(region, group_resv_regions, list) {
1888 		/*
1889 		 * The presence of any 'real' MSI regions should take
1890 		 * precedence over the software-managed one if the
1891 		 * IOMMU driver happens to advertise both types.
1892 		 */
1893 		if (region->type == IOMMU_RESV_MSI) {
1894 			ret = false;
1895 			break;
1896 		}
1897 
1898 		if (region->type == IOMMU_RESV_SW_MSI) {
1899 			*base = region->start;
1900 			ret = true;
1901 		}
1902 	}
1903 
1904 	return ret;
1905 }
1906 
1907 /*
1908  * This is a helper function to insert an address range to iova list.
1909  * The list is initially created with a single entry corresponding to
1910  * the IOMMU domain geometry to which the device group is attached.
1911  * The list aperture gets modified when a new domain is added to the
1912  * container if the new aperture doesn't conflict with the current one
1913  * or with any existing dma mappings. The list is also modified to
1914  * exclude any reserved regions associated with the device group.
1915  */
vfio_iommu_iova_insert(struct list_head * head,dma_addr_t start,dma_addr_t end)1916 static int vfio_iommu_iova_insert(struct list_head *head,
1917 				  dma_addr_t start, dma_addr_t end)
1918 {
1919 	struct vfio_iova *region;
1920 
1921 	region = kmalloc(sizeof(*region), GFP_KERNEL);
1922 	if (!region)
1923 		return -ENOMEM;
1924 
1925 	INIT_LIST_HEAD(&region->list);
1926 	region->start = start;
1927 	region->end = end;
1928 
1929 	list_add_tail(&region->list, head);
1930 	return 0;
1931 }
1932 
1933 /*
1934  * Check the new iommu aperture conflicts with existing aper or with any
1935  * existing dma mappings.
1936  */
vfio_iommu_aper_conflict(struct vfio_iommu * iommu,dma_addr_t start,dma_addr_t end)1937 static bool vfio_iommu_aper_conflict(struct vfio_iommu *iommu,
1938 				     dma_addr_t start, dma_addr_t end)
1939 {
1940 	struct vfio_iova *first, *last;
1941 	struct list_head *iova = &iommu->iova_list;
1942 
1943 	if (list_empty(iova))
1944 		return false;
1945 
1946 	/* Disjoint sets, return conflict */
1947 	first = list_first_entry(iova, struct vfio_iova, list);
1948 	last = list_last_entry(iova, struct vfio_iova, list);
1949 	if (start > last->end || end < first->start)
1950 		return true;
1951 
1952 	/* Check for any existing dma mappings below the new start */
1953 	if (start > first->start) {
1954 		if (vfio_find_dma(iommu, first->start, start - first->start))
1955 			return true;
1956 	}
1957 
1958 	/* Check for any existing dma mappings beyond the new end */
1959 	if (end < last->end) {
1960 		if (vfio_find_dma(iommu, end + 1, last->end - end))
1961 			return true;
1962 	}
1963 
1964 	return false;
1965 }
1966 
1967 /*
1968  * Resize iommu iova aperture window. This is called only if the new
1969  * aperture has no conflict with existing aperture and dma mappings.
1970  */
vfio_iommu_aper_resize(struct list_head * iova,dma_addr_t start,dma_addr_t end)1971 static int vfio_iommu_aper_resize(struct list_head *iova,
1972 				  dma_addr_t start, dma_addr_t end)
1973 {
1974 	struct vfio_iova *node, *next;
1975 
1976 	if (list_empty(iova))
1977 		return vfio_iommu_iova_insert(iova, start, end);
1978 
1979 	/* Adjust iova list start */
1980 	list_for_each_entry_safe(node, next, iova, list) {
1981 		if (start < node->start)
1982 			break;
1983 		if (start >= node->start && start < node->end) {
1984 			node->start = start;
1985 			break;
1986 		}
1987 		/* Delete nodes before new start */
1988 		list_del(&node->list);
1989 		kfree(node);
1990 	}
1991 
1992 	/* Adjust iova list end */
1993 	list_for_each_entry_safe(node, next, iova, list) {
1994 		if (end > node->end)
1995 			continue;
1996 		if (end > node->start && end <= node->end) {
1997 			node->end = end;
1998 			continue;
1999 		}
2000 		/* Delete nodes after new end */
2001 		list_del(&node->list);
2002 		kfree(node);
2003 	}
2004 
2005 	return 0;
2006 }
2007 
2008 /*
2009  * Check reserved region conflicts with existing dma mappings
2010  */
vfio_iommu_resv_conflict(struct vfio_iommu * iommu,struct list_head * resv_regions)2011 static bool vfio_iommu_resv_conflict(struct vfio_iommu *iommu,
2012 				     struct list_head *resv_regions)
2013 {
2014 	struct iommu_resv_region *region;
2015 
2016 	/* Check for conflict with existing dma mappings */
2017 	list_for_each_entry(region, resv_regions, list) {
2018 		if (region->type == IOMMU_RESV_DIRECT_RELAXABLE)
2019 			continue;
2020 
2021 		if (vfio_find_dma(iommu, region->start, region->length))
2022 			return true;
2023 	}
2024 
2025 	return false;
2026 }
2027 
2028 /*
2029  * Check iova region overlap with  reserved regions and
2030  * exclude them from the iommu iova range
2031  */
vfio_iommu_resv_exclude(struct list_head * iova,struct list_head * resv_regions)2032 static int vfio_iommu_resv_exclude(struct list_head *iova,
2033 				   struct list_head *resv_regions)
2034 {
2035 	struct iommu_resv_region *resv;
2036 	struct vfio_iova *n, *next;
2037 
2038 	list_for_each_entry(resv, resv_regions, list) {
2039 		phys_addr_t start, end;
2040 
2041 		if (resv->type == IOMMU_RESV_DIRECT_RELAXABLE)
2042 			continue;
2043 
2044 		start = resv->start;
2045 		end = resv->start + resv->length - 1;
2046 
2047 		list_for_each_entry_safe(n, next, iova, list) {
2048 			int ret = 0;
2049 
2050 			/* No overlap */
2051 			if (start > n->end || end < n->start)
2052 				continue;
2053 			/*
2054 			 * Insert a new node if current node overlaps with the
2055 			 * reserve region to exclude that from valid iova range.
2056 			 * Note that, new node is inserted before the current
2057 			 * node and finally the current node is deleted keeping
2058 			 * the list updated and sorted.
2059 			 */
2060 			if (start > n->start)
2061 				ret = vfio_iommu_iova_insert(&n->list, n->start,
2062 							     start - 1);
2063 			if (!ret && end < n->end)
2064 				ret = vfio_iommu_iova_insert(&n->list, end + 1,
2065 							     n->end);
2066 			if (ret)
2067 				return ret;
2068 
2069 			list_del(&n->list);
2070 			kfree(n);
2071 		}
2072 	}
2073 
2074 	if (list_empty(iova))
2075 		return -EINVAL;
2076 
2077 	return 0;
2078 }
2079 
vfio_iommu_resv_free(struct list_head * resv_regions)2080 static void vfio_iommu_resv_free(struct list_head *resv_regions)
2081 {
2082 	struct iommu_resv_region *n, *next;
2083 
2084 	list_for_each_entry_safe(n, next, resv_regions, list) {
2085 		list_del(&n->list);
2086 		kfree(n);
2087 	}
2088 }
2089 
vfio_iommu_iova_free(struct list_head * iova)2090 static void vfio_iommu_iova_free(struct list_head *iova)
2091 {
2092 	struct vfio_iova *n, *next;
2093 
2094 	list_for_each_entry_safe(n, next, iova, list) {
2095 		list_del(&n->list);
2096 		kfree(n);
2097 	}
2098 }
2099 
vfio_iommu_iova_get_copy(struct vfio_iommu * iommu,struct list_head * iova_copy)2100 static int vfio_iommu_iova_get_copy(struct vfio_iommu *iommu,
2101 				    struct list_head *iova_copy)
2102 {
2103 	struct list_head *iova = &iommu->iova_list;
2104 	struct vfio_iova *n;
2105 	int ret;
2106 
2107 	list_for_each_entry(n, iova, list) {
2108 		ret = vfio_iommu_iova_insert(iova_copy, n->start, n->end);
2109 		if (ret)
2110 			goto out_free;
2111 	}
2112 
2113 	return 0;
2114 
2115 out_free:
2116 	vfio_iommu_iova_free(iova_copy);
2117 	return ret;
2118 }
2119 
vfio_iommu_iova_insert_copy(struct vfio_iommu * iommu,struct list_head * iova_copy)2120 static void vfio_iommu_iova_insert_copy(struct vfio_iommu *iommu,
2121 					struct list_head *iova_copy)
2122 {
2123 	struct list_head *iova = &iommu->iova_list;
2124 
2125 	vfio_iommu_iova_free(iova);
2126 
2127 	list_splice_tail(iova_copy, iova);
2128 }
2129 
vfio_iommu_domain_alloc(struct device * dev,void * data)2130 static int vfio_iommu_domain_alloc(struct device *dev, void *data)
2131 {
2132 	struct iommu_domain **domain = data;
2133 
2134 	*domain = iommu_paging_domain_alloc(dev);
2135 	return 1; /* Don't iterate */
2136 }
2137 
vfio_iommu_type1_attach_group(void * iommu_data,struct iommu_group * iommu_group,enum vfio_group_type type)2138 static int vfio_iommu_type1_attach_group(void *iommu_data,
2139 		struct iommu_group *iommu_group, enum vfio_group_type type)
2140 {
2141 	struct vfio_iommu *iommu = iommu_data;
2142 	struct vfio_iommu_group *group;
2143 	struct vfio_domain *domain, *d;
2144 	bool resv_msi;
2145 	phys_addr_t resv_msi_base = 0;
2146 	struct iommu_domain_geometry *geo;
2147 	LIST_HEAD(iova_copy);
2148 	LIST_HEAD(group_resv_regions);
2149 	int ret = -EBUSY;
2150 
2151 	mutex_lock(&iommu->lock);
2152 
2153 	/* Attach could require pinning, so disallow while vaddr is invalid. */
2154 	if (iommu->vaddr_invalid_count)
2155 		goto out_unlock;
2156 
2157 	/* Check for duplicates */
2158 	ret = -EINVAL;
2159 	if (vfio_iommu_find_iommu_group(iommu, iommu_group))
2160 		goto out_unlock;
2161 
2162 	ret = -ENOMEM;
2163 	group = kzalloc(sizeof(*group), GFP_KERNEL);
2164 	if (!group)
2165 		goto out_unlock;
2166 	group->iommu_group = iommu_group;
2167 
2168 	if (type == VFIO_EMULATED_IOMMU) {
2169 		list_add(&group->next, &iommu->emulated_iommu_groups);
2170 		/*
2171 		 * An emulated IOMMU group cannot dirty memory directly, it can
2172 		 * only use interfaces that provide dirty tracking.
2173 		 * The iommu scope can only be promoted with the addition of a
2174 		 * dirty tracking group.
2175 		 */
2176 		group->pinned_page_dirty_scope = true;
2177 		ret = 0;
2178 		goto out_unlock;
2179 	}
2180 
2181 	ret = -ENOMEM;
2182 	domain = kzalloc(sizeof(*domain), GFP_KERNEL);
2183 	if (!domain)
2184 		goto out_free_group;
2185 
2186 	/*
2187 	 * Going via the iommu_group iterator avoids races, and trivially gives
2188 	 * us a representative device for the IOMMU API call. We don't actually
2189 	 * want to iterate beyond the first device (if any).
2190 	 */
2191 	iommu_group_for_each_dev(iommu_group, &domain->domain,
2192 				 vfio_iommu_domain_alloc);
2193 	if (IS_ERR(domain->domain)) {
2194 		ret = PTR_ERR(domain->domain);
2195 		goto out_free_domain;
2196 	}
2197 
2198 	if (iommu->nesting) {
2199 		ret = iommu_enable_nesting(domain->domain);
2200 		if (ret)
2201 			goto out_domain;
2202 	}
2203 
2204 	ret = iommu_attach_group(domain->domain, group->iommu_group);
2205 	if (ret)
2206 		goto out_domain;
2207 
2208 	/* Get aperture info */
2209 	geo = &domain->domain->geometry;
2210 	if (vfio_iommu_aper_conflict(iommu, geo->aperture_start,
2211 				     geo->aperture_end)) {
2212 		ret = -EINVAL;
2213 		goto out_detach;
2214 	}
2215 
2216 	ret = iommu_get_group_resv_regions(iommu_group, &group_resv_regions);
2217 	if (ret)
2218 		goto out_detach;
2219 
2220 	if (vfio_iommu_resv_conflict(iommu, &group_resv_regions)) {
2221 		ret = -EINVAL;
2222 		goto out_detach;
2223 	}
2224 
2225 	/*
2226 	 * We don't want to work on the original iova list as the list
2227 	 * gets modified and in case of failure we have to retain the
2228 	 * original list. Get a copy here.
2229 	 */
2230 	ret = vfio_iommu_iova_get_copy(iommu, &iova_copy);
2231 	if (ret)
2232 		goto out_detach;
2233 
2234 	ret = vfio_iommu_aper_resize(&iova_copy, geo->aperture_start,
2235 				     geo->aperture_end);
2236 	if (ret)
2237 		goto out_detach;
2238 
2239 	ret = vfio_iommu_resv_exclude(&iova_copy, &group_resv_regions);
2240 	if (ret)
2241 		goto out_detach;
2242 
2243 	resv_msi = vfio_iommu_has_sw_msi(&group_resv_regions, &resv_msi_base);
2244 
2245 	INIT_LIST_HEAD(&domain->group_list);
2246 	list_add(&group->next, &domain->group_list);
2247 
2248 	if (!allow_unsafe_interrupts &&
2249 	    !iommu_group_has_isolated_msi(iommu_group)) {
2250 		pr_warn("%s: No interrupt remapping support.  Use the module param \"allow_unsafe_interrupts\" to enable VFIO IOMMU support on this platform\n",
2251 		       __func__);
2252 		ret = -EPERM;
2253 		goto out_detach;
2254 	}
2255 
2256 	/*
2257 	 * If the IOMMU can block non-coherent operations (ie PCIe TLPs with
2258 	 * no-snoop set) then VFIO always turns this feature on because on Intel
2259 	 * platforms it optimizes KVM to disable wbinvd emulation.
2260 	 */
2261 	if (domain->domain->ops->enforce_cache_coherency)
2262 		domain->enforce_cache_coherency =
2263 			domain->domain->ops->enforce_cache_coherency(
2264 				domain->domain);
2265 
2266 	/*
2267 	 * Try to match an existing compatible domain.  We don't want to
2268 	 * preclude an IOMMU driver supporting multiple bus_types and being
2269 	 * able to include different bus_types in the same IOMMU domain, so
2270 	 * we test whether the domains use the same iommu_ops rather than
2271 	 * testing if they're on the same bus_type.
2272 	 */
2273 	list_for_each_entry(d, &iommu->domain_list, next) {
2274 		if (d->domain->ops == domain->domain->ops &&
2275 		    d->enforce_cache_coherency ==
2276 			    domain->enforce_cache_coherency) {
2277 			iommu_detach_group(domain->domain, group->iommu_group);
2278 			if (!iommu_attach_group(d->domain,
2279 						group->iommu_group)) {
2280 				list_add(&group->next, &d->group_list);
2281 				iommu_domain_free(domain->domain);
2282 				kfree(domain);
2283 				goto done;
2284 			}
2285 
2286 			ret = iommu_attach_group(domain->domain,
2287 						 group->iommu_group);
2288 			if (ret)
2289 				goto out_domain;
2290 		}
2291 	}
2292 
2293 	vfio_test_domain_fgsp(domain, &iova_copy);
2294 
2295 	/* replay mappings on new domains */
2296 	ret = vfio_iommu_replay(iommu, domain);
2297 	if (ret)
2298 		goto out_detach;
2299 
2300 	if (resv_msi) {
2301 		ret = iommu_get_msi_cookie(domain->domain, resv_msi_base);
2302 		if (ret && ret != -ENODEV)
2303 			goto out_detach;
2304 	}
2305 
2306 	list_add(&domain->next, &iommu->domain_list);
2307 	vfio_update_pgsize_bitmap(iommu);
2308 done:
2309 	/* Delete the old one and insert new iova list */
2310 	vfio_iommu_iova_insert_copy(iommu, &iova_copy);
2311 
2312 	/*
2313 	 * An iommu backed group can dirty memory directly and therefore
2314 	 * demotes the iommu scope until it declares itself dirty tracking
2315 	 * capable via the page pinning interface.
2316 	 */
2317 	iommu->num_non_pinned_groups++;
2318 	mutex_unlock(&iommu->lock);
2319 	vfio_iommu_resv_free(&group_resv_regions);
2320 
2321 	return 0;
2322 
2323 out_detach:
2324 	iommu_detach_group(domain->domain, group->iommu_group);
2325 out_domain:
2326 	iommu_domain_free(domain->domain);
2327 	vfio_iommu_iova_free(&iova_copy);
2328 	vfio_iommu_resv_free(&group_resv_regions);
2329 out_free_domain:
2330 	kfree(domain);
2331 out_free_group:
2332 	kfree(group);
2333 out_unlock:
2334 	mutex_unlock(&iommu->lock);
2335 	return ret;
2336 }
2337 
vfio_iommu_unmap_unpin_all(struct vfio_iommu * iommu)2338 static void vfio_iommu_unmap_unpin_all(struct vfio_iommu *iommu)
2339 {
2340 	struct rb_node *node;
2341 
2342 	while ((node = rb_first(&iommu->dma_list)))
2343 		vfio_remove_dma(iommu, rb_entry(node, struct vfio_dma, node));
2344 }
2345 
vfio_iommu_unmap_unpin_reaccount(struct vfio_iommu * iommu)2346 static void vfio_iommu_unmap_unpin_reaccount(struct vfio_iommu *iommu)
2347 {
2348 	struct rb_node *n, *p;
2349 
2350 	n = rb_first(&iommu->dma_list);
2351 	for (; n; n = rb_next(n)) {
2352 		struct vfio_dma *dma;
2353 		long locked = 0, unlocked = 0;
2354 
2355 		dma = rb_entry(n, struct vfio_dma, node);
2356 		unlocked += vfio_unmap_unpin(iommu, dma, false);
2357 		p = rb_first(&dma->pfn_list);
2358 		for (; p; p = rb_next(p)) {
2359 			struct vfio_pfn *vpfn = rb_entry(p, struct vfio_pfn,
2360 							 node);
2361 
2362 			if (!is_invalid_reserved_pfn(vpfn->pfn))
2363 				locked++;
2364 		}
2365 		vfio_lock_acct(dma, locked - unlocked, true);
2366 	}
2367 }
2368 
2369 /*
2370  * Called when a domain is removed in detach. It is possible that
2371  * the removed domain decided the iova aperture window. Modify the
2372  * iova aperture with the smallest window among existing domains.
2373  */
vfio_iommu_aper_expand(struct vfio_iommu * iommu,struct list_head * iova_copy)2374 static void vfio_iommu_aper_expand(struct vfio_iommu *iommu,
2375 				   struct list_head *iova_copy)
2376 {
2377 	struct vfio_domain *domain;
2378 	struct vfio_iova *node;
2379 	dma_addr_t start = 0;
2380 	dma_addr_t end = (dma_addr_t)~0;
2381 
2382 	if (list_empty(iova_copy))
2383 		return;
2384 
2385 	list_for_each_entry(domain, &iommu->domain_list, next) {
2386 		struct iommu_domain_geometry *geo = &domain->domain->geometry;
2387 
2388 		if (geo->aperture_start > start)
2389 			start = geo->aperture_start;
2390 		if (geo->aperture_end < end)
2391 			end = geo->aperture_end;
2392 	}
2393 
2394 	/* Modify aperture limits. The new aper is either same or bigger */
2395 	node = list_first_entry(iova_copy, struct vfio_iova, list);
2396 	node->start = start;
2397 	node = list_last_entry(iova_copy, struct vfio_iova, list);
2398 	node->end = end;
2399 }
2400 
2401 /*
2402  * Called when a group is detached. The reserved regions for that
2403  * group can be part of valid iova now. But since reserved regions
2404  * may be duplicated among groups, populate the iova valid regions
2405  * list again.
2406  */
vfio_iommu_resv_refresh(struct vfio_iommu * iommu,struct list_head * iova_copy)2407 static int vfio_iommu_resv_refresh(struct vfio_iommu *iommu,
2408 				   struct list_head *iova_copy)
2409 {
2410 	struct vfio_domain *d;
2411 	struct vfio_iommu_group *g;
2412 	struct vfio_iova *node;
2413 	dma_addr_t start, end;
2414 	LIST_HEAD(resv_regions);
2415 	int ret;
2416 
2417 	if (list_empty(iova_copy))
2418 		return -EINVAL;
2419 
2420 	list_for_each_entry(d, &iommu->domain_list, next) {
2421 		list_for_each_entry(g, &d->group_list, next) {
2422 			ret = iommu_get_group_resv_regions(g->iommu_group,
2423 							   &resv_regions);
2424 			if (ret)
2425 				goto done;
2426 		}
2427 	}
2428 
2429 	node = list_first_entry(iova_copy, struct vfio_iova, list);
2430 	start = node->start;
2431 	node = list_last_entry(iova_copy, struct vfio_iova, list);
2432 	end = node->end;
2433 
2434 	/* purge the iova list and create new one */
2435 	vfio_iommu_iova_free(iova_copy);
2436 
2437 	ret = vfio_iommu_aper_resize(iova_copy, start, end);
2438 	if (ret)
2439 		goto done;
2440 
2441 	/* Exclude current reserved regions from iova ranges */
2442 	ret = vfio_iommu_resv_exclude(iova_copy, &resv_regions);
2443 done:
2444 	vfio_iommu_resv_free(&resv_regions);
2445 	return ret;
2446 }
2447 
vfio_iommu_type1_detach_group(void * iommu_data,struct iommu_group * iommu_group)2448 static void vfio_iommu_type1_detach_group(void *iommu_data,
2449 					  struct iommu_group *iommu_group)
2450 {
2451 	struct vfio_iommu *iommu = iommu_data;
2452 	struct vfio_domain *domain;
2453 	struct vfio_iommu_group *group;
2454 	bool update_dirty_scope = false;
2455 	LIST_HEAD(iova_copy);
2456 
2457 	mutex_lock(&iommu->lock);
2458 	list_for_each_entry(group, &iommu->emulated_iommu_groups, next) {
2459 		if (group->iommu_group != iommu_group)
2460 			continue;
2461 		update_dirty_scope = !group->pinned_page_dirty_scope;
2462 		list_del(&group->next);
2463 		kfree(group);
2464 
2465 		if (list_empty(&iommu->emulated_iommu_groups) &&
2466 		    list_empty(&iommu->domain_list)) {
2467 			WARN_ON(!list_empty(&iommu->device_list));
2468 			vfio_iommu_unmap_unpin_all(iommu);
2469 		}
2470 		goto detach_group_done;
2471 	}
2472 
2473 	/*
2474 	 * Get a copy of iova list. This will be used to update
2475 	 * and to replace the current one later. Please note that
2476 	 * we will leave the original list as it is if update fails.
2477 	 */
2478 	vfio_iommu_iova_get_copy(iommu, &iova_copy);
2479 
2480 	list_for_each_entry(domain, &iommu->domain_list, next) {
2481 		group = find_iommu_group(domain, iommu_group);
2482 		if (!group)
2483 			continue;
2484 
2485 		iommu_detach_group(domain->domain, group->iommu_group);
2486 		update_dirty_scope = !group->pinned_page_dirty_scope;
2487 		list_del(&group->next);
2488 		kfree(group);
2489 		/*
2490 		 * Group ownership provides privilege, if the group list is
2491 		 * empty, the domain goes away. If it's the last domain with
2492 		 * iommu and external domain doesn't exist, then all the
2493 		 * mappings go away too. If it's the last domain with iommu and
2494 		 * external domain exist, update accounting
2495 		 */
2496 		if (list_empty(&domain->group_list)) {
2497 			if (list_is_singular(&iommu->domain_list)) {
2498 				if (list_empty(&iommu->emulated_iommu_groups)) {
2499 					WARN_ON(!list_empty(
2500 						&iommu->device_list));
2501 					vfio_iommu_unmap_unpin_all(iommu);
2502 				} else {
2503 					vfio_iommu_unmap_unpin_reaccount(iommu);
2504 				}
2505 			}
2506 			iommu_domain_free(domain->domain);
2507 			list_del(&domain->next);
2508 			kfree(domain);
2509 			vfio_iommu_aper_expand(iommu, &iova_copy);
2510 			vfio_update_pgsize_bitmap(iommu);
2511 		}
2512 		break;
2513 	}
2514 
2515 	if (!vfio_iommu_resv_refresh(iommu, &iova_copy))
2516 		vfio_iommu_iova_insert_copy(iommu, &iova_copy);
2517 	else
2518 		vfio_iommu_iova_free(&iova_copy);
2519 
2520 detach_group_done:
2521 	/*
2522 	 * Removal of a group without dirty tracking may allow the iommu scope
2523 	 * to be promoted.
2524 	 */
2525 	if (update_dirty_scope) {
2526 		iommu->num_non_pinned_groups--;
2527 		if (iommu->dirty_page_tracking)
2528 			vfio_iommu_populate_bitmap_full(iommu);
2529 	}
2530 	mutex_unlock(&iommu->lock);
2531 }
2532 
vfio_iommu_type1_open(unsigned long arg)2533 static void *vfio_iommu_type1_open(unsigned long arg)
2534 {
2535 	struct vfio_iommu *iommu;
2536 
2537 	iommu = kzalloc(sizeof(*iommu), GFP_KERNEL);
2538 	if (!iommu)
2539 		return ERR_PTR(-ENOMEM);
2540 
2541 	switch (arg) {
2542 	case VFIO_TYPE1_IOMMU:
2543 		break;
2544 	case VFIO_TYPE1_NESTING_IOMMU:
2545 		iommu->nesting = true;
2546 		fallthrough;
2547 	case VFIO_TYPE1v2_IOMMU:
2548 		iommu->v2 = true;
2549 		break;
2550 	default:
2551 		kfree(iommu);
2552 		return ERR_PTR(-EINVAL);
2553 	}
2554 
2555 	INIT_LIST_HEAD(&iommu->domain_list);
2556 	INIT_LIST_HEAD(&iommu->iova_list);
2557 	iommu->dma_list = RB_ROOT;
2558 	iommu->dma_avail = dma_entry_limit;
2559 	mutex_init(&iommu->lock);
2560 	mutex_init(&iommu->device_list_lock);
2561 	INIT_LIST_HEAD(&iommu->device_list);
2562 	iommu->pgsize_bitmap = PAGE_MASK;
2563 	INIT_LIST_HEAD(&iommu->emulated_iommu_groups);
2564 
2565 	return iommu;
2566 }
2567 
vfio_release_domain(struct vfio_domain * domain)2568 static void vfio_release_domain(struct vfio_domain *domain)
2569 {
2570 	struct vfio_iommu_group *group, *group_tmp;
2571 
2572 	list_for_each_entry_safe(group, group_tmp,
2573 				 &domain->group_list, next) {
2574 		iommu_detach_group(domain->domain, group->iommu_group);
2575 		list_del(&group->next);
2576 		kfree(group);
2577 	}
2578 
2579 	iommu_domain_free(domain->domain);
2580 }
2581 
vfio_iommu_type1_release(void * iommu_data)2582 static void vfio_iommu_type1_release(void *iommu_data)
2583 {
2584 	struct vfio_iommu *iommu = iommu_data;
2585 	struct vfio_domain *domain, *domain_tmp;
2586 	struct vfio_iommu_group *group, *next_group;
2587 
2588 	list_for_each_entry_safe(group, next_group,
2589 			&iommu->emulated_iommu_groups, next) {
2590 		list_del(&group->next);
2591 		kfree(group);
2592 	}
2593 
2594 	vfio_iommu_unmap_unpin_all(iommu);
2595 
2596 	list_for_each_entry_safe(domain, domain_tmp,
2597 				 &iommu->domain_list, next) {
2598 		vfio_release_domain(domain);
2599 		list_del(&domain->next);
2600 		kfree(domain);
2601 	}
2602 
2603 	vfio_iommu_iova_free(&iommu->iova_list);
2604 
2605 	kfree(iommu);
2606 }
2607 
vfio_domains_have_enforce_cache_coherency(struct vfio_iommu * iommu)2608 static int vfio_domains_have_enforce_cache_coherency(struct vfio_iommu *iommu)
2609 {
2610 	struct vfio_domain *domain;
2611 	int ret = 1;
2612 
2613 	mutex_lock(&iommu->lock);
2614 	list_for_each_entry(domain, &iommu->domain_list, next) {
2615 		if (!(domain->enforce_cache_coherency)) {
2616 			ret = 0;
2617 			break;
2618 		}
2619 	}
2620 	mutex_unlock(&iommu->lock);
2621 
2622 	return ret;
2623 }
2624 
vfio_iommu_has_emulated(struct vfio_iommu * iommu)2625 static bool vfio_iommu_has_emulated(struct vfio_iommu *iommu)
2626 {
2627 	bool ret;
2628 
2629 	mutex_lock(&iommu->lock);
2630 	ret = !list_empty(&iommu->emulated_iommu_groups);
2631 	mutex_unlock(&iommu->lock);
2632 	return ret;
2633 }
2634 
vfio_iommu_type1_check_extension(struct vfio_iommu * iommu,unsigned long arg)2635 static int vfio_iommu_type1_check_extension(struct vfio_iommu *iommu,
2636 					    unsigned long arg)
2637 {
2638 	switch (arg) {
2639 	case VFIO_TYPE1_IOMMU:
2640 	case VFIO_TYPE1v2_IOMMU:
2641 	case VFIO_TYPE1_NESTING_IOMMU:
2642 	case VFIO_UNMAP_ALL:
2643 		return 1;
2644 	case VFIO_UPDATE_VADDR:
2645 		/*
2646 		 * Disable this feature if mdevs are present.  They cannot
2647 		 * safely pin/unpin/rw while vaddrs are being updated.
2648 		 */
2649 		return iommu && !vfio_iommu_has_emulated(iommu);
2650 	case VFIO_DMA_CC_IOMMU:
2651 		if (!iommu)
2652 			return 0;
2653 		return vfio_domains_have_enforce_cache_coherency(iommu);
2654 	default:
2655 		return 0;
2656 	}
2657 }
2658 
vfio_iommu_iova_add_cap(struct vfio_info_cap * caps,struct vfio_iommu_type1_info_cap_iova_range * cap_iovas,size_t size)2659 static int vfio_iommu_iova_add_cap(struct vfio_info_cap *caps,
2660 		 struct vfio_iommu_type1_info_cap_iova_range *cap_iovas,
2661 		 size_t size)
2662 {
2663 	struct vfio_info_cap_header *header;
2664 	struct vfio_iommu_type1_info_cap_iova_range *iova_cap;
2665 
2666 	header = vfio_info_cap_add(caps, size,
2667 				   VFIO_IOMMU_TYPE1_INFO_CAP_IOVA_RANGE, 1);
2668 	if (IS_ERR(header))
2669 		return PTR_ERR(header);
2670 
2671 	iova_cap = container_of(header,
2672 				struct vfio_iommu_type1_info_cap_iova_range,
2673 				header);
2674 	iova_cap->nr_iovas = cap_iovas->nr_iovas;
2675 	memcpy(iova_cap->iova_ranges, cap_iovas->iova_ranges,
2676 	       cap_iovas->nr_iovas * sizeof(*cap_iovas->iova_ranges));
2677 	return 0;
2678 }
2679 
vfio_iommu_iova_build_caps(struct vfio_iommu * iommu,struct vfio_info_cap * caps)2680 static int vfio_iommu_iova_build_caps(struct vfio_iommu *iommu,
2681 				      struct vfio_info_cap *caps)
2682 {
2683 	struct vfio_iommu_type1_info_cap_iova_range *cap_iovas;
2684 	struct vfio_iova *iova;
2685 	size_t size;
2686 	int iovas = 0, i = 0, ret;
2687 
2688 	list_for_each_entry(iova, &iommu->iova_list, list)
2689 		iovas++;
2690 
2691 	if (!iovas) {
2692 		/*
2693 		 * Return 0 as a container with a single mdev device
2694 		 * will have an empty list
2695 		 */
2696 		return 0;
2697 	}
2698 
2699 	size = struct_size(cap_iovas, iova_ranges, iovas);
2700 
2701 	cap_iovas = kzalloc(size, GFP_KERNEL);
2702 	if (!cap_iovas)
2703 		return -ENOMEM;
2704 
2705 	cap_iovas->nr_iovas = iovas;
2706 
2707 	list_for_each_entry(iova, &iommu->iova_list, list) {
2708 		cap_iovas->iova_ranges[i].start = iova->start;
2709 		cap_iovas->iova_ranges[i].end = iova->end;
2710 		i++;
2711 	}
2712 
2713 	ret = vfio_iommu_iova_add_cap(caps, cap_iovas, size);
2714 
2715 	kfree(cap_iovas);
2716 	return ret;
2717 }
2718 
vfio_iommu_migration_build_caps(struct vfio_iommu * iommu,struct vfio_info_cap * caps)2719 static int vfio_iommu_migration_build_caps(struct vfio_iommu *iommu,
2720 					   struct vfio_info_cap *caps)
2721 {
2722 	struct vfio_iommu_type1_info_cap_migration cap_mig = {};
2723 
2724 	cap_mig.header.id = VFIO_IOMMU_TYPE1_INFO_CAP_MIGRATION;
2725 	cap_mig.header.version = 1;
2726 
2727 	cap_mig.flags = 0;
2728 	/* support minimum pgsize */
2729 	cap_mig.pgsize_bitmap = (size_t)1 << __ffs(iommu->pgsize_bitmap);
2730 	cap_mig.max_dirty_bitmap_size = DIRTY_BITMAP_SIZE_MAX;
2731 
2732 	return vfio_info_add_capability(caps, &cap_mig.header, sizeof(cap_mig));
2733 }
2734 
vfio_iommu_dma_avail_build_caps(struct vfio_iommu * iommu,struct vfio_info_cap * caps)2735 static int vfio_iommu_dma_avail_build_caps(struct vfio_iommu *iommu,
2736 					   struct vfio_info_cap *caps)
2737 {
2738 	struct vfio_iommu_type1_info_dma_avail cap_dma_avail;
2739 
2740 	cap_dma_avail.header.id = VFIO_IOMMU_TYPE1_INFO_DMA_AVAIL;
2741 	cap_dma_avail.header.version = 1;
2742 
2743 	cap_dma_avail.avail = iommu->dma_avail;
2744 
2745 	return vfio_info_add_capability(caps, &cap_dma_avail.header,
2746 					sizeof(cap_dma_avail));
2747 }
2748 
vfio_iommu_type1_get_info(struct vfio_iommu * iommu,unsigned long arg)2749 static int vfio_iommu_type1_get_info(struct vfio_iommu *iommu,
2750 				     unsigned long arg)
2751 {
2752 	struct vfio_iommu_type1_info info = {};
2753 	unsigned long minsz;
2754 	struct vfio_info_cap caps = { .buf = NULL, .size = 0 };
2755 	int ret;
2756 
2757 	minsz = offsetofend(struct vfio_iommu_type1_info, iova_pgsizes);
2758 
2759 	if (copy_from_user(&info, (void __user *)arg, minsz))
2760 		return -EFAULT;
2761 
2762 	if (info.argsz < minsz)
2763 		return -EINVAL;
2764 
2765 	minsz = min_t(size_t, info.argsz, sizeof(info));
2766 
2767 	mutex_lock(&iommu->lock);
2768 	info.flags = VFIO_IOMMU_INFO_PGSIZES;
2769 
2770 	info.iova_pgsizes = iommu->pgsize_bitmap;
2771 
2772 	ret = vfio_iommu_migration_build_caps(iommu, &caps);
2773 
2774 	if (!ret)
2775 		ret = vfio_iommu_dma_avail_build_caps(iommu, &caps);
2776 
2777 	if (!ret)
2778 		ret = vfio_iommu_iova_build_caps(iommu, &caps);
2779 
2780 	mutex_unlock(&iommu->lock);
2781 
2782 	if (ret)
2783 		return ret;
2784 
2785 	if (caps.size) {
2786 		info.flags |= VFIO_IOMMU_INFO_CAPS;
2787 
2788 		if (info.argsz < sizeof(info) + caps.size) {
2789 			info.argsz = sizeof(info) + caps.size;
2790 		} else {
2791 			vfio_info_cap_shift(&caps, sizeof(info));
2792 			if (copy_to_user((void __user *)arg +
2793 					sizeof(info), caps.buf,
2794 					caps.size)) {
2795 				kfree(caps.buf);
2796 				return -EFAULT;
2797 			}
2798 			info.cap_offset = sizeof(info);
2799 		}
2800 
2801 		kfree(caps.buf);
2802 	}
2803 
2804 	return copy_to_user((void __user *)arg, &info, minsz) ?
2805 			-EFAULT : 0;
2806 }
2807 
vfio_iommu_type1_map_dma(struct vfio_iommu * iommu,unsigned long arg)2808 static int vfio_iommu_type1_map_dma(struct vfio_iommu *iommu,
2809 				    unsigned long arg)
2810 {
2811 	struct vfio_iommu_type1_dma_map map;
2812 	unsigned long minsz;
2813 	uint32_t mask = VFIO_DMA_MAP_FLAG_READ | VFIO_DMA_MAP_FLAG_WRITE |
2814 			VFIO_DMA_MAP_FLAG_VADDR;
2815 
2816 	minsz = offsetofend(struct vfio_iommu_type1_dma_map, size);
2817 
2818 	if (copy_from_user(&map, (void __user *)arg, minsz))
2819 		return -EFAULT;
2820 
2821 	if (map.argsz < minsz || map.flags & ~mask)
2822 		return -EINVAL;
2823 
2824 	return vfio_dma_do_map(iommu, &map);
2825 }
2826 
vfio_iommu_type1_unmap_dma(struct vfio_iommu * iommu,unsigned long arg)2827 static int vfio_iommu_type1_unmap_dma(struct vfio_iommu *iommu,
2828 				      unsigned long arg)
2829 {
2830 	struct vfio_iommu_type1_dma_unmap unmap;
2831 	struct vfio_bitmap bitmap = { 0 };
2832 	uint32_t mask = VFIO_DMA_UNMAP_FLAG_GET_DIRTY_BITMAP |
2833 			VFIO_DMA_UNMAP_FLAG_VADDR |
2834 			VFIO_DMA_UNMAP_FLAG_ALL;
2835 	unsigned long minsz;
2836 	int ret;
2837 
2838 	minsz = offsetofend(struct vfio_iommu_type1_dma_unmap, size);
2839 
2840 	if (copy_from_user(&unmap, (void __user *)arg, minsz))
2841 		return -EFAULT;
2842 
2843 	if (unmap.argsz < minsz || unmap.flags & ~mask)
2844 		return -EINVAL;
2845 
2846 	if ((unmap.flags & VFIO_DMA_UNMAP_FLAG_GET_DIRTY_BITMAP) &&
2847 	    (unmap.flags & (VFIO_DMA_UNMAP_FLAG_ALL |
2848 			    VFIO_DMA_UNMAP_FLAG_VADDR)))
2849 		return -EINVAL;
2850 
2851 	if (unmap.flags & VFIO_DMA_UNMAP_FLAG_GET_DIRTY_BITMAP) {
2852 		unsigned long pgshift;
2853 
2854 		if (unmap.argsz < (minsz + sizeof(bitmap)))
2855 			return -EINVAL;
2856 
2857 		if (copy_from_user(&bitmap,
2858 				   (void __user *)(arg + minsz),
2859 				   sizeof(bitmap)))
2860 			return -EFAULT;
2861 
2862 		if (!access_ok((void __user *)bitmap.data, bitmap.size))
2863 			return -EINVAL;
2864 
2865 		pgshift = __ffs(bitmap.pgsize);
2866 		ret = verify_bitmap_size(unmap.size >> pgshift,
2867 					 bitmap.size);
2868 		if (ret)
2869 			return ret;
2870 	}
2871 
2872 	ret = vfio_dma_do_unmap(iommu, &unmap, &bitmap);
2873 	if (ret)
2874 		return ret;
2875 
2876 	return copy_to_user((void __user *)arg, &unmap, minsz) ?
2877 			-EFAULT : 0;
2878 }
2879 
vfio_iommu_type1_dirty_pages(struct vfio_iommu * iommu,unsigned long arg)2880 static int vfio_iommu_type1_dirty_pages(struct vfio_iommu *iommu,
2881 					unsigned long arg)
2882 {
2883 	struct vfio_iommu_type1_dirty_bitmap dirty;
2884 	uint32_t mask = VFIO_IOMMU_DIRTY_PAGES_FLAG_START |
2885 			VFIO_IOMMU_DIRTY_PAGES_FLAG_STOP |
2886 			VFIO_IOMMU_DIRTY_PAGES_FLAG_GET_BITMAP;
2887 	unsigned long minsz;
2888 	int ret = 0;
2889 
2890 	if (!iommu->v2)
2891 		return -EACCES;
2892 
2893 	minsz = offsetofend(struct vfio_iommu_type1_dirty_bitmap, flags);
2894 
2895 	if (copy_from_user(&dirty, (void __user *)arg, minsz))
2896 		return -EFAULT;
2897 
2898 	if (dirty.argsz < minsz || dirty.flags & ~mask)
2899 		return -EINVAL;
2900 
2901 	/* only one flag should be set at a time */
2902 	if (__ffs(dirty.flags) != __fls(dirty.flags))
2903 		return -EINVAL;
2904 
2905 	if (dirty.flags & VFIO_IOMMU_DIRTY_PAGES_FLAG_START) {
2906 		size_t pgsize;
2907 
2908 		mutex_lock(&iommu->lock);
2909 		pgsize = 1 << __ffs(iommu->pgsize_bitmap);
2910 		if (!iommu->dirty_page_tracking) {
2911 			ret = vfio_dma_bitmap_alloc_all(iommu, pgsize);
2912 			if (!ret)
2913 				iommu->dirty_page_tracking = true;
2914 		}
2915 		mutex_unlock(&iommu->lock);
2916 		return ret;
2917 	} else if (dirty.flags & VFIO_IOMMU_DIRTY_PAGES_FLAG_STOP) {
2918 		mutex_lock(&iommu->lock);
2919 		if (iommu->dirty_page_tracking) {
2920 			iommu->dirty_page_tracking = false;
2921 			vfio_dma_bitmap_free_all(iommu);
2922 		}
2923 		mutex_unlock(&iommu->lock);
2924 		return 0;
2925 	} else if (dirty.flags & VFIO_IOMMU_DIRTY_PAGES_FLAG_GET_BITMAP) {
2926 		struct vfio_iommu_type1_dirty_bitmap_get range;
2927 		unsigned long pgshift;
2928 		size_t data_size = dirty.argsz - minsz;
2929 		size_t iommu_pgsize;
2930 
2931 		if (!data_size || data_size < sizeof(range))
2932 			return -EINVAL;
2933 
2934 		if (copy_from_user(&range, (void __user *)(arg + minsz),
2935 				   sizeof(range)))
2936 			return -EFAULT;
2937 
2938 		if (range.iova + range.size < range.iova)
2939 			return -EINVAL;
2940 		if (!access_ok((void __user *)range.bitmap.data,
2941 			       range.bitmap.size))
2942 			return -EINVAL;
2943 
2944 		pgshift = __ffs(range.bitmap.pgsize);
2945 		ret = verify_bitmap_size(range.size >> pgshift,
2946 					 range.bitmap.size);
2947 		if (ret)
2948 			return ret;
2949 
2950 		mutex_lock(&iommu->lock);
2951 
2952 		iommu_pgsize = (size_t)1 << __ffs(iommu->pgsize_bitmap);
2953 
2954 		/* allow only smallest supported pgsize */
2955 		if (range.bitmap.pgsize != iommu_pgsize) {
2956 			ret = -EINVAL;
2957 			goto out_unlock;
2958 		}
2959 		if (range.iova & (iommu_pgsize - 1)) {
2960 			ret = -EINVAL;
2961 			goto out_unlock;
2962 		}
2963 		if (!range.size || range.size & (iommu_pgsize - 1)) {
2964 			ret = -EINVAL;
2965 			goto out_unlock;
2966 		}
2967 
2968 		if (iommu->dirty_page_tracking)
2969 			ret = vfio_iova_dirty_bitmap(range.bitmap.data,
2970 						     iommu, range.iova,
2971 						     range.size,
2972 						     range.bitmap.pgsize);
2973 		else
2974 			ret = -EINVAL;
2975 out_unlock:
2976 		mutex_unlock(&iommu->lock);
2977 
2978 		return ret;
2979 	}
2980 
2981 	return -EINVAL;
2982 }
2983 
vfio_iommu_type1_ioctl(void * iommu_data,unsigned int cmd,unsigned long arg)2984 static long vfio_iommu_type1_ioctl(void *iommu_data,
2985 				   unsigned int cmd, unsigned long arg)
2986 {
2987 	struct vfio_iommu *iommu = iommu_data;
2988 
2989 	switch (cmd) {
2990 	case VFIO_CHECK_EXTENSION:
2991 		return vfio_iommu_type1_check_extension(iommu, arg);
2992 	case VFIO_IOMMU_GET_INFO:
2993 		return vfio_iommu_type1_get_info(iommu, arg);
2994 	case VFIO_IOMMU_MAP_DMA:
2995 		return vfio_iommu_type1_map_dma(iommu, arg);
2996 	case VFIO_IOMMU_UNMAP_DMA:
2997 		return vfio_iommu_type1_unmap_dma(iommu, arg);
2998 	case VFIO_IOMMU_DIRTY_PAGES:
2999 		return vfio_iommu_type1_dirty_pages(iommu, arg);
3000 	default:
3001 		return -ENOTTY;
3002 	}
3003 }
3004 
vfio_iommu_type1_register_device(void * iommu_data,struct vfio_device * vdev)3005 static void vfio_iommu_type1_register_device(void *iommu_data,
3006 					     struct vfio_device *vdev)
3007 {
3008 	struct vfio_iommu *iommu = iommu_data;
3009 
3010 	if (!vdev->ops->dma_unmap)
3011 		return;
3012 
3013 	/*
3014 	 * list_empty(&iommu->device_list) is tested under the iommu->lock while
3015 	 * iteration for dma_unmap must be done under the device_list_lock.
3016 	 * Holding both locks here allows avoiding the device_list_lock in
3017 	 * several fast paths. See vfio_notify_dma_unmap()
3018 	 */
3019 	mutex_lock(&iommu->lock);
3020 	mutex_lock(&iommu->device_list_lock);
3021 	list_add(&vdev->iommu_entry, &iommu->device_list);
3022 	mutex_unlock(&iommu->device_list_lock);
3023 	mutex_unlock(&iommu->lock);
3024 }
3025 
vfio_iommu_type1_unregister_device(void * iommu_data,struct vfio_device * vdev)3026 static void vfio_iommu_type1_unregister_device(void *iommu_data,
3027 					       struct vfio_device *vdev)
3028 {
3029 	struct vfio_iommu *iommu = iommu_data;
3030 
3031 	if (!vdev->ops->dma_unmap)
3032 		return;
3033 
3034 	mutex_lock(&iommu->lock);
3035 	mutex_lock(&iommu->device_list_lock);
3036 	list_del(&vdev->iommu_entry);
3037 	mutex_unlock(&iommu->device_list_lock);
3038 	mutex_unlock(&iommu->lock);
3039 }
3040 
vfio_iommu_type1_dma_rw_chunk(struct vfio_iommu * iommu,dma_addr_t user_iova,void * data,size_t count,bool write,size_t * copied)3041 static int vfio_iommu_type1_dma_rw_chunk(struct vfio_iommu *iommu,
3042 					 dma_addr_t user_iova, void *data,
3043 					 size_t count, bool write,
3044 					 size_t *copied)
3045 {
3046 	struct mm_struct *mm;
3047 	unsigned long vaddr;
3048 	struct vfio_dma *dma;
3049 	bool kthread = current->mm == NULL;
3050 	size_t offset;
3051 
3052 	*copied = 0;
3053 
3054 	dma = vfio_find_dma(iommu, user_iova, 1);
3055 	if (!dma)
3056 		return -EINVAL;
3057 
3058 	if ((write && !(dma->prot & IOMMU_WRITE)) ||
3059 			!(dma->prot & IOMMU_READ))
3060 		return -EPERM;
3061 
3062 	mm = dma->mm;
3063 	if (!mmget_not_zero(mm))
3064 		return -EPERM;
3065 
3066 	if (kthread)
3067 		kthread_use_mm(mm);
3068 	else if (current->mm != mm)
3069 		goto out;
3070 
3071 	offset = user_iova - dma->iova;
3072 
3073 	if (count > dma->size - offset)
3074 		count = dma->size - offset;
3075 
3076 	vaddr = dma->vaddr + offset;
3077 
3078 	if (write) {
3079 		*copied = copy_to_user((void __user *)vaddr, data,
3080 					 count) ? 0 : count;
3081 		if (*copied && iommu->dirty_page_tracking) {
3082 			unsigned long pgshift = __ffs(iommu->pgsize_bitmap);
3083 			/*
3084 			 * Bitmap populated with the smallest supported page
3085 			 * size
3086 			 */
3087 			bitmap_set(dma->bitmap, offset >> pgshift,
3088 				   ((offset + *copied - 1) >> pgshift) -
3089 				   (offset >> pgshift) + 1);
3090 		}
3091 	} else
3092 		*copied = copy_from_user(data, (void __user *)vaddr,
3093 					   count) ? 0 : count;
3094 	if (kthread)
3095 		kthread_unuse_mm(mm);
3096 out:
3097 	mmput(mm);
3098 	return *copied ? 0 : -EFAULT;
3099 }
3100 
vfio_iommu_type1_dma_rw(void * iommu_data,dma_addr_t user_iova,void * data,size_t count,bool write)3101 static int vfio_iommu_type1_dma_rw(void *iommu_data, dma_addr_t user_iova,
3102 				   void *data, size_t count, bool write)
3103 {
3104 	struct vfio_iommu *iommu = iommu_data;
3105 	int ret = 0;
3106 	size_t done;
3107 
3108 	mutex_lock(&iommu->lock);
3109 
3110 	if (WARN_ONCE(iommu->vaddr_invalid_count,
3111 		      "vfio_dma_rw not allowed with VFIO_UPDATE_VADDR\n")) {
3112 		ret = -EBUSY;
3113 		goto out;
3114 	}
3115 
3116 	while (count > 0) {
3117 		ret = vfio_iommu_type1_dma_rw_chunk(iommu, user_iova, data,
3118 						    count, write, &done);
3119 		if (ret)
3120 			break;
3121 
3122 		count -= done;
3123 		data += done;
3124 		user_iova += done;
3125 	}
3126 
3127 out:
3128 	mutex_unlock(&iommu->lock);
3129 	return ret;
3130 }
3131 
3132 static struct iommu_domain *
vfio_iommu_type1_group_iommu_domain(void * iommu_data,struct iommu_group * iommu_group)3133 vfio_iommu_type1_group_iommu_domain(void *iommu_data,
3134 				    struct iommu_group *iommu_group)
3135 {
3136 	struct iommu_domain *domain = ERR_PTR(-ENODEV);
3137 	struct vfio_iommu *iommu = iommu_data;
3138 	struct vfio_domain *d;
3139 
3140 	if (!iommu || !iommu_group)
3141 		return ERR_PTR(-EINVAL);
3142 
3143 	mutex_lock(&iommu->lock);
3144 	list_for_each_entry(d, &iommu->domain_list, next) {
3145 		if (find_iommu_group(d, iommu_group)) {
3146 			domain = d->domain;
3147 			break;
3148 		}
3149 	}
3150 	mutex_unlock(&iommu->lock);
3151 
3152 	return domain;
3153 }
3154 
3155 static const struct vfio_iommu_driver_ops vfio_iommu_driver_ops_type1 = {
3156 	.name			= "vfio-iommu-type1",
3157 	.owner			= THIS_MODULE,
3158 	.open			= vfio_iommu_type1_open,
3159 	.release		= vfio_iommu_type1_release,
3160 	.ioctl			= vfio_iommu_type1_ioctl,
3161 	.attach_group		= vfio_iommu_type1_attach_group,
3162 	.detach_group		= vfio_iommu_type1_detach_group,
3163 	.pin_pages		= vfio_iommu_type1_pin_pages,
3164 	.unpin_pages		= vfio_iommu_type1_unpin_pages,
3165 	.register_device	= vfio_iommu_type1_register_device,
3166 	.unregister_device	= vfio_iommu_type1_unregister_device,
3167 	.dma_rw			= vfio_iommu_type1_dma_rw,
3168 	.group_iommu_domain	= vfio_iommu_type1_group_iommu_domain,
3169 };
3170 
vfio_iommu_type1_init(void)3171 static int __init vfio_iommu_type1_init(void)
3172 {
3173 	return vfio_register_iommu_driver(&vfio_iommu_driver_ops_type1);
3174 }
3175 
vfio_iommu_type1_cleanup(void)3176 static void __exit vfio_iommu_type1_cleanup(void)
3177 {
3178 	vfio_unregister_iommu_driver(&vfio_iommu_driver_ops_type1);
3179 }
3180 
3181 module_init(vfio_iommu_type1_init);
3182 module_exit(vfio_iommu_type1_cleanup);
3183 
3184 MODULE_VERSION(DRIVER_VERSION);
3185 MODULE_LICENSE("GPL v2");
3186 MODULE_AUTHOR(DRIVER_AUTHOR);
3187 MODULE_DESCRIPTION(DRIVER_DESC);
3188