1 // SPDX-License-Identifier: GPL-2.0-only
5  * Copyright (C) 2015-2023  Red Hat, Inc.
8 #include "uffd-common.h"
15 int uffd = -1, uffd_flags, finished, *pipefd, test_type;
30 	mem_fd = memfd_create("uffd-test", memfd_flags);  in uffd_mem_fd_create()
52 			   MAP_ANONYMOUS | MAP_PRIVATE, -1, 0);  in anon_allocate_area()
55 		return -errno;  in anon_allocate_area()
89 		return -errno;  in hugetlb_allocate_area()
96 			return -errno;  in hugetlb_allocate_area()
107 	close(mem_fd);  in hugetlb_allocate_area()
146 		return -errno;  in shmem_allocate_area()
156 		return -errno;  in shmem_allocate_area()
166 	close(mem_fd);  in shmem_allocate_area()
243 		return -1;  in userfaultfd_open()
250 		return -1;  in userfaultfd_open()
273 			if (close(pipefd[i]))  in uffd_test_ctx_clear()
274 				err("close pipefd");  in uffd_test_ctx_clear()
285 	if (uffd != -1) {  in uffd_test_ctx_clear()
286 		if (close(uffd))  in uffd_test_ctx_clear()
287 			err("close uffd");  in uffd_test_ctx_clear()
288 		uffd = -1;  in uffd_test_ctx_clear()
303 	if (uffd_test_case_ops && uffd_test_case_ops->pre_alloc) {  in uffd_test_ctx_init()
304 		ret = uffd_test_case_ops->pre_alloc(errmsg);  in uffd_test_ctx_init()
309 	ret = uffd_test_ops->allocate_area((void **)&area_src, true);  in uffd_test_ctx_init()
310 	ret |= uffd_test_ops->allocate_area((void **)&area_dst, false);  in uffd_test_ctx_init()
317 	if (uffd_test_case_ops && uffd_test_case_ops->post_alloc) {  in uffd_test_ctx_init()
318 		ret = uffd_test_case_ops->post_alloc(errmsg);  in uffd_test_ctx_init()
341 		 * zero, so leave a placeholder below always non-zero  in uffd_test_ctx_init()
366 	uffd_test_ops->release_pages(area_dst);  in uffd_test_ctx_init()
383 	prms.range.start = start;  in wp_range()
384 	prms.range.len = len;  in wp_range()
385 	/* Undo write-protect, do wakeup after that */  in wp_range()
397 	req.range.start = start;  in continue_range()
398 	req.range.len = len;  in continue_range()
410 	 * error (-EEXIST) on purpose, to verify doing so doesn't cause a BUG.  in continue_range()
414 	if (ret >= 0 || req.mapped != -EEXIST)  in continue_range()
440 	if (msg->event != UFFD_EVENT_PAGEFAULT)  in uffd_handle_page_fault()
441 		err("unexpected msg event %u", msg->event);  in uffd_handle_page_fault()
443 	if (msg->arg.pagefault.flags & UFFD_PAGEFAULT_FLAG_WP) {  in uffd_handle_page_fault()
445 		wp_range(uffd, msg->arg.pagefault.address, page_size, false);  in uffd_handle_page_fault()
446 		args->wp_faults++;  in uffd_handle_page_fault()
447 	} else if (msg->arg.pagefault.flags & UFFD_PAGEFAULT_FLAG_MINOR) {  in uffd_handle_page_fault()
454 		 * To prove we can modify the original range for testing  in uffd_handle_page_fault()
455 		 * purposes, we're going to bit flip this range before  in uffd_handle_page_fault()
459 		 * area_dst (non-UFFD-registered) and area_dst_alias  in uffd_handle_page_fault()
460 		 * (UFFD-registered).  in uffd_handle_page_fault()
464 				   ((char *)msg->arg.pagefault.address -  in uffd_handle_page_fault()
468 		continue_range(uffd, msg->arg.pagefault.address, page_size,  in uffd_handle_page_fault()
469 			       args->apply_wp);  in uffd_handle_page_fault()
470 		args->minor_faults++;  in uffd_handle_page_fault()
486 		 * logically a pthread-compatible lib can implement the  in uffd_handle_page_fault()
493 		if (msg->arg.pagefault.flags & UFFD_PAGEFAULT_FLAG_WRITE)  in uffd_handle_page_fault()
496 		offset = (char *)(unsigned long)msg->arg.pagefault.address - area_dst;  in uffd_handle_page_fault()
497 		offset &= ~(page_size-1);  in uffd_handle_page_fault()
499 		if (copy_page(uffd, offset, args->apply_wp))  in uffd_handle_page_fault()
500 			args->missing_faults++;  in uffd_handle_page_fault()
507 	unsigned long cpu = args->cpu;  in uffd_poll_thread()
514 	if (!args->handle_fault)  in uffd_poll_thread()
515 		args->handle_fault = uffd_handle_page_fault;  in uffd_poll_thread()
525 		ret = poll(pollfd, 2, -1);  in uffd_poll_thread()
547 			args->handle_fault(&msg, args);  in uffd_poll_thread()
550 			close(uffd);  in uffd_poll_thread()
555 			uffd_reg.range.start = msg.arg.remove.start;  in uffd_poll_thread()
556 			uffd_reg.range.len = msg.arg.remove.end -  in uffd_poll_thread()
558 			if (ioctl(uffd, UFFDIO_UNREGISTER, &uffd_reg.range))  in uffd_poll_thread()
574 	uffd_test_ops->alias_mapping(&uffdio_copy->dst,  in retry_copy_page()
575 				     uffdio_copy->len,  in retry_copy_page()
579 		if (uffdio_copy->copy != -EEXIST)  in retry_copy_page()
581 			    (int64_t)uffdio_copy->copy);  in retry_copy_page()
584 		    (int64_t)uffdio_copy->copy);  in retry_copy_page()
616 		if (uffdio_copy.copy != -EEXIST)  in __copy_page()
650 		if (uffdio_move.move != -EEXIST)  in move_page()
669 	close(fd);  in uffd_open_dev()
699 		/* Maybe the kernel is older than user-only mode? */  in uffd_get_features()
706 		close(fd);  in uffd_get_features()
707 		return -errno;  in uffd_get_features()
711 	close(fd);  in uffd_get_features()