1 // SPDX-License-Identifier: GPL-2.0-only
5  *  Copyright (C) 2015-2023  Red Hat, Inc.
8 #include <asm-generic/unistd.h>
9 #include "uffd-common.h"
26 	((__typeof__(x))((((unsigned long)(x)) + ((align_to)-1)) & ~((align_to)-1)))
50 		.name = "shmem-private",
62 		.name = "hugetlb-private",
189 	map_shared = mem_type->shared;  in uffd_setup_environment()
190 	uffd_test_ops = mem_type->mem_ops;  in uffd_setup_environment()
191 	uffd_test_case_ops = test->test_case_ops;  in uffd_setup_environment()
193 	if (mem_type->mem_flag & (MEM_HUGETLB_PRIVATE | MEM_HUGETLB))  in uffd_setup_environment()
203 	args->mem_type = mem_type;  in uffd_setup_environment()
205 	return uffd_test_ctx_init(test->uffd_feature_required, errmsg);  in uffd_setup_environment()
215 	return (features & test->uffd_feature_required) ==  in uffd_feature_supported()
216 	    test->uffd_feature_required;  in uffd_feature_supported()
230 #define  pagemap_check_wp(value, wp) do {				\  argument
231 		if (!!(value & PM_UFFD_WP) != wp)			\
232 			err("pagemap uffd-wp bit error: 0x%"PRIx64, value); \
247 	while (uffd_read_msg(args->parent_uffd, &msg));  in fork_event_consumer()
253 	args->child_uffd = msg.arg.fork.ufd;  in fork_event_consumer()
273 		/* Read-only pins */  in pin_pages()
277 	if (args->pinned)  in pin_pages()
280 	args->gup_fd = open("/sys/kernel/debug/gup_test", O_RDWR);  in pin_pages()
281 	if (args->gup_fd < 0)  in pin_pages()
282 		return -errno;  in pin_pages()
284 	if (ioctl(args->gup_fd, PIN_LONGTERM_TEST_START, &test)) {  in pin_pages()
286 		close(args->gup_fd);  in pin_pages()
287 		return -errno;  in pin_pages()
289 	args->pinned = true;  in pin_pages()
295 	if (!args->pinned)  in unpin_pages()
297 	if (ioctl(args->gup_fd, PIN_LONGTERM_TEST_STOP))  in unpin_pages()
299 	close(args->gup_fd);  in unpin_pages()
300 	args->pinned = false;  in unpin_pages()
305 	fork_event_args args = { .parent_uffd = uffd, .child_uffd = -1 };  in pagemap_test_fork()
336 		 * After fork(), we should handle uffd-wp bit differently:  in pagemap_test_fork()
396 	/* Test read-zero-page upon pte marker */  in uffd_wp_unpopulated_test()
424 		uffd_test_fail("Detected %s uffd-wp bit in child in present pte",  in uffd_wp_fork_test_common()
440 	if (args->mem_type->shared) {  in uffd_wp_fork_test_common()
445 		 * NOTE: ignore retval because private-hugetlb doesn't yet  in uffd_wp_fork_test_common()
451 	/* Uffd-wp should persist even swapped out */  in uffd_wp_fork_test_common()
455 		uffd_test_fail("Detected %s uffd-wp bit in child in zapped pte",  in uffd_wp_fork_test_common()
514 		uffd_test_fail("Detected %s uffd-wp bit in early CoW of fork()",  in uffd_wp_fork_pin_test_common()
527 		uffd_test_fail("Detected %s uffd-wp bit when RO pin",  in uffd_wp_fork_pin_test_common()
554 		expected_byte = ~((uint8_t)(i % ((uint8_t)-1)));  in check_memory_contents()
571 	 * NOTE: MADV_COLLAPSE is not yet compatible with WP, so testing  in uffd_minor_test_common()
577 			  /* NOTE! MADV_COLLAPSE may not work with uffd-wp */  in uffd_minor_test_common()
582 	 * After registering with UFFD, populate the non-UFFD-registered side of  in uffd_minor_test_common()
586 		memset(area_dst + (p * page_size), p % ((uint8_t)-1),  in uffd_minor_test_common()
594 	 * Read each of the pages back using the UFFD-registered mapping. We  in uffd_minor_test_common()
596 	 * fault. uffd_poll_thread will resolve the fault by bit-flipping the  in uffd_minor_test_common()
614 		uffd_test_ops->check_pmd_mapping(area_dst,  in uffd_minor_test_common()
618 		 * This won't cause uffd-fault - it purely just makes sure there  in uffd_minor_test_common()
657  * For non-cooperative userfaultfd test we fork() a process that will
672  * test robustness use case - we release monitored area, fork a process
677 static int faulting_process(int signal_test, bool wp)  in faulting_process()  argument
695 		lastnr = (unsigned long)-1;  in faulting_process()
712 						if (copy_page(uffd, offset, wp))  in faulting_process()
715 						/* This is a WP request */  in faulting_process()
763 	uffd_test_ops->release_pages(area_dst);  in faulting_process()
773 static void uffd_sigbus_test_common(bool wp)  in uffd_sigbus_test_common()  argument
787 			  true, wp, false))  in uffd_sigbus_test_common()
790 	if (faulting_process(1, wp))  in uffd_sigbus_test_common()
793 	uffd_test_ops->release_pages(area_dst);  in uffd_sigbus_test_common()
795 	args.apply_wp = wp;  in uffd_sigbus_test_common()
807 		exit(faulting_process(2, wp));  in uffd_sigbus_test_common()
833 static void uffd_events_test_common(bool wp)  in uffd_events_test_common()  argument
845 			  true, wp, false))  in uffd_events_test_common()
848 	args.apply_wp = wp;  in uffd_events_test_common()
860 		exit(faulting_process(0, wp));  in uffd_events_test_common()
889 	uffd_test_ops->alias_mapping(&uffdio_zeropage->range.start,  in retry_uffdio_zeropage()
890 				     uffdio_zeropage->range.len,  in retry_uffdio_zeropage()
893 		if (uffdio_zeropage->zeropage != -EEXIST)  in retry_uffdio_zeropage()
895 			    (int64_t)uffdio_zeropage->zeropage);  in retry_uffdio_zeropage()
898 		    (int64_t)uffdio_zeropage->zeropage);  in retry_uffdio_zeropage()
917 		else if (res != -EINVAL)  in do_uffdio_zeropage()
918 			err("UFFDIO_ZEROPAGE not -EINVAL");  in do_uffdio_zeropage()
962 				err("data non-zero at offset %d\n", i);  in uffd_zeropage_test()
1009 	if (msg->event != UFFD_EVENT_PAGEFAULT)  in uffd_poison_handle_fault()
1010 		err("unexpected msg event %u", msg->event);  in uffd_poison_handle_fault()
1012 	if (msg->arg.pagefault.flags &  in uffd_poison_handle_fault()
1014 		err("unexpected fault type %llu", msg->arg.pagefault.flags);  in uffd_poison_handle_fault()
1016 	offset = (char *)(unsigned long)msg->arg.pagefault.address - area_dst;  in uffd_poison_handle_fault()
1017 	offset &= ~(page_size-1);  in uffd_poison_handle_fault()
1019 	/* Odd pages -> copy zeroed page; even pages -> poison. */  in uffd_poison_handle_fault()
1090 	if (msg->event != UFFD_EVENT_PAGEFAULT)  in uffd_move_handle_fault_common()
1091 		err("unexpected msg event %u", msg->event);  in uffd_move_handle_fault_common()
1093 	if (msg->arg.pagefault.flags &  in uffd_move_handle_fault_common()
1095 		err("unexpected fault type %llu", msg->arg.pagefault.flags);  in uffd_move_handle_fault_common()
1097 	offset = (char *)(unsigned long)msg->arg.pagefault.address - area_dst;  in uffd_move_handle_fault_common()
1098 	offset &= ~(len-1);  in uffd_move_handle_fault_common()
1101 		args->missing_faults++;  in uffd_move_handle_fault_common()
1150 			src_offs = (aligned_src - area_src) / page_size;  in uffd_move_test_common()
1151 			dst_offs = (aligned_dst - area_dst) / page_size;  in uffd_move_test_common()
1152 			step_count--;  in uffd_move_test_common()
1161 	 * Read each of the pages back using the UFFD-registered mapping. We  in uffd_move_test_common()
1169 		/* Check area_src content */  in uffd_move_test_common()
1185 		/* Re-check area_src content which should be empty */  in uffd_move_test_common()
1238 			return -errno;  in prevent_hugepages()
1253 		return -errno;  in request_hugepages()
1271 do_register_ioctls_test(uffd_test_args_t *args, bool miss, bool wp, bool minor)  in do_register_ioctls_test()  argument
1274 	mem_type_t *mem_type = args->mem_type;  in do_register_ioctls_test()
1278 					miss, wp, minor, &ioctls);  in do_register_ioctls_test()
1282 	 * just fail with -EINVAL first..  in do_register_ioctls_test()
1287 	if ((minor && (mem_type->mem_flag == MEM_ANON)) ||  in do_register_ioctls_test()
1288 	    (!miss && !wp && !minor)) {  in do_register_ioctls_test()
1289 		if (ret != -EINVAL)  in do_register_ioctls_test()
1290 			err("register (miss=%d, wp=%d, minor=%d) failed "  in do_register_ioctls_test()
1291 			    "with wrong errno=%d", miss, wp, minor, ret);  in do_register_ioctls_test()
1298 	if (wp)  in do_register_ioctls_test()
1305 		    "(miss=%d, wp=%d, minor=%d): expected=0x%"PRIx64", "  in do_register_ioctls_test()
1306 		    "returned=0x%"PRIx64, miss, wp, minor, expected, ioctls);  in do_register_ioctls_test()
1314 	int miss, wp, minor;  in uffd_register_ioctls_test()  local
1317 		for (wp = 0; wp <= 1; wp++)  in uffd_register_ioctls_test()
1319 				do_register_ioctls_test(args, miss, wp, minor);  in uffd_register_ioctls_test()
1327 		.name = "register-ioctls",
1351 		.name = "move-pmd",
1358 		.name = "move-pmd-split",
1365 		.name = "wp-fork",
1372 		.name = "wp-fork-with-event",
1377 		/* when set, child process should inherit uffd-wp bits */
1381 		.name = "wp-fork-pin",
1388 		.name = "wp-fork-pin-with-event",
1393 		/* when set, child process should inherit uffd-wp bits */
1397 		.name = "wp-unpopulated",
1411 		.name = "minor-wp",
1419 		 * minor mode supports wr-protect.  There's no feature flag
1425 		.name = "minor-collapse",
1440 		.name = "sigbus-wp",
1455 		.name = "events-wp",
1473 	printf("usage: %s [-f TESTNAME]\n", prog);  in usage()
1475 	puts(" -f: test name to filter (e.g., event)");  in usage()
1476 	puts(" -h: show the help msg");  in usage()
1477 	puts(" -l: list tests only");  in usage()
1495 	while ((opt = getopt(argc, argv, "f:hl")) != -1) {  in main()
1523 		if (test_filter && !strstr(test->name, test_filter))  in main()
1526 			printf("%s\n", test->name);  in main()
1531 			if (!(test->mem_targets & mem_type->mem_flag))  in main()
1534 			uffd_test_start("%s on %s", test->name, mem_type->name);  in main()
1535 			if ((mem_type->mem_flag == MEM_HUGETLB ||  in main()
1536 			    mem_type->mem_flag == MEM_HUGETLB_PRIVATE) &&  in main()
1550 			test->uffd_fn(&args);  in main()