1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3 * Userfaultfd unit tests.
4 *
5 * Copyright (C) 2015-2023 Red Hat, Inc.
6 */
7
8 #include <asm-generic/unistd.h>
9 #include "uffd-common.h"
10
11 #include "../../../../mm/gup_test.h"
12
13 /* The unit test doesn't need a large or random size, make it 32MB for now */
14 #define UFFD_TEST_MEM_SIZE (32UL << 20)
15
16 #define MEM_ANON BIT_ULL(0)
17 #define MEM_SHMEM BIT_ULL(1)
18 #define MEM_SHMEM_PRIVATE BIT_ULL(2)
19 #define MEM_HUGETLB BIT_ULL(3)
20 #define MEM_HUGETLB_PRIVATE BIT_ULL(4)
21
22 #define MEM_ALL (MEM_ANON | MEM_SHMEM | MEM_SHMEM_PRIVATE | \
23 MEM_HUGETLB | MEM_HUGETLB_PRIVATE)
24
25 #define ALIGN_UP(x, align_to) \
26 ((__typeof__(x))((((unsigned long)(x)) + ((align_to)-1)) & ~((align_to)-1)))
27
28 struct mem_type {
29 const char *name;
30 unsigned int mem_flag;
31 uffd_test_ops_t *mem_ops;
32 bool shared;
33 };
34 typedef struct mem_type mem_type_t;
35
36 mem_type_t mem_types[] = {
37 {
38 .name = "anon",
39 .mem_flag = MEM_ANON,
40 .mem_ops = &anon_uffd_test_ops,
41 .shared = false,
42 },
43 {
44 .name = "shmem",
45 .mem_flag = MEM_SHMEM,
46 .mem_ops = &shmem_uffd_test_ops,
47 .shared = true,
48 },
49 {
50 .name = "shmem-private",
51 .mem_flag = MEM_SHMEM_PRIVATE,
52 .mem_ops = &shmem_uffd_test_ops,
53 .shared = false,
54 },
55 {
56 .name = "hugetlb",
57 .mem_flag = MEM_HUGETLB,
58 .mem_ops = &hugetlb_uffd_test_ops,
59 .shared = true,
60 },
61 {
62 .name = "hugetlb-private",
63 .mem_flag = MEM_HUGETLB_PRIVATE,
64 .mem_ops = &hugetlb_uffd_test_ops,
65 .shared = false,
66 },
67 };
68
69 /* Arguments to be passed over to each uffd unit test */
70 struct uffd_test_args {
71 mem_type_t *mem_type;
72 };
73 typedef struct uffd_test_args uffd_test_args_t;
74
75 /* Returns: UFFD_TEST_* */
76 typedef void (*uffd_test_fn)(uffd_test_args_t *);
77
78 typedef struct {
79 const char *name;
80 uffd_test_fn uffd_fn;
81 unsigned int mem_targets;
82 uint64_t uffd_feature_required;
83 uffd_test_case_ops_t *test_case_ops;
84 } uffd_test_case_t;
85
uffd_test_report(void)86 static void uffd_test_report(void)
87 {
88 printf("Userfaults unit tests: pass=%u, skip=%u, fail=%u (total=%u)\n",
89 ksft_get_pass_cnt(),
90 ksft_get_xskip_cnt(),
91 ksft_get_fail_cnt(),
92 ksft_test_num());
93 }
94
uffd_test_pass(void)95 static void uffd_test_pass(void)
96 {
97 printf("done\n");
98 ksft_inc_pass_cnt();
99 }
100
101 #define uffd_test_start(...) do { \
102 printf("Testing "); \
103 printf(__VA_ARGS__); \
104 printf("... "); \
105 fflush(stdout); \
106 } while (0)
107
108 #define uffd_test_fail(...) do { \
109 printf("failed [reason: "); \
110 printf(__VA_ARGS__); \
111 printf("]\n"); \
112 ksft_inc_fail_cnt(); \
113 } while (0)
114
uffd_test_skip(const char * message)115 static void uffd_test_skip(const char *message)
116 {
117 printf("skipped [reason: %s]\n", message);
118 ksft_inc_xskip_cnt();
119 }
120
121 /*
122 * Returns 1 if specific userfaultfd supported, 0 otherwise. Note, we'll
123 * return 1 even if some test failed as long as uffd supported, because in
124 * that case we still want to proceed with the rest uffd unit tests.
125 */
test_uffd_api(bool use_dev)126 static int test_uffd_api(bool use_dev)
127 {
128 struct uffdio_api uffdio_api;
129 int uffd;
130
131 uffd_test_start("UFFDIO_API (with %s)",
132 use_dev ? "/dev/userfaultfd" : "syscall");
133
134 if (use_dev)
135 uffd = uffd_open_dev(UFFD_FLAGS);
136 else
137 uffd = uffd_open_sys(UFFD_FLAGS);
138 if (uffd < 0) {
139 uffd_test_skip("cannot open userfaultfd handle");
140 return 0;
141 }
142
143 /* Test wrong UFFD_API */
144 uffdio_api.api = 0xab;
145 uffdio_api.features = 0;
146 if (ioctl(uffd, UFFDIO_API, &uffdio_api) == 0) {
147 uffd_test_fail("UFFDIO_API should fail with wrong api but didn't");
148 goto out;
149 }
150
151 /* Test wrong feature bit */
152 uffdio_api.api = UFFD_API;
153 uffdio_api.features = BIT_ULL(63);
154 if (ioctl(uffd, UFFDIO_API, &uffdio_api) == 0) {
155 uffd_test_fail("UFFDIO_API should fail with wrong feature but didn't");
156 goto out;
157 }
158
159 /* Test normal UFFDIO_API */
160 uffdio_api.api = UFFD_API;
161 uffdio_api.features = 0;
162 if (ioctl(uffd, UFFDIO_API, &uffdio_api)) {
163 uffd_test_fail("UFFDIO_API should succeed but failed");
164 goto out;
165 }
166
167 /* Test double requests of UFFDIO_API with a random feature set */
168 uffdio_api.features = BIT_ULL(0);
169 if (ioctl(uffd, UFFDIO_API, &uffdio_api) == 0) {
170 uffd_test_fail("UFFDIO_API should reject initialized uffd");
171 goto out;
172 }
173
174 uffd_test_pass();
175 out:
176 close(uffd);
177 /* We have a valid uffd handle */
178 return 1;
179 }
180
181 /*
182 * This function initializes the global variables. TODO: remove global
183 * vars and then remove this.
184 */
185 static int
uffd_setup_environment(uffd_test_args_t * args,uffd_test_case_t * test,mem_type_t * mem_type,const char ** errmsg)186 uffd_setup_environment(uffd_test_args_t *args, uffd_test_case_t *test,
187 mem_type_t *mem_type, const char **errmsg)
188 {
189 map_shared = mem_type->shared;
190 uffd_test_ops = mem_type->mem_ops;
191 uffd_test_case_ops = test->test_case_ops;
192
193 if (mem_type->mem_flag & (MEM_HUGETLB_PRIVATE | MEM_HUGETLB))
194 page_size = default_huge_page_size();
195 else
196 page_size = psize();
197
198 nr_pages = UFFD_TEST_MEM_SIZE / page_size;
199 /* TODO: remove this global var.. it's so ugly */
200 nr_cpus = 1;
201
202 /* Initialize test arguments */
203 args->mem_type = mem_type;
204
205 return uffd_test_ctx_init(test->uffd_feature_required, errmsg);
206 }
207
uffd_feature_supported(uffd_test_case_t * test)208 static bool uffd_feature_supported(uffd_test_case_t *test)
209 {
210 uint64_t features;
211
212 if (uffd_get_features(&features))
213 return false;
214
215 return (features & test->uffd_feature_required) ==
216 test->uffd_feature_required;
217 }
218
pagemap_open(void)219 static int pagemap_open(void)
220 {
221 int fd = open("/proc/self/pagemap", O_RDONLY);
222
223 if (fd < 0)
224 err("open pagemap");
225
226 return fd;
227 }
228
229 /* This macro let __LINE__ works in err() */
230 #define pagemap_check_wp(value, wp) do { \
231 if (!!(value & PM_UFFD_WP) != wp) \
232 err("pagemap uffd-wp bit error: 0x%"PRIx64, value); \
233 } while (0)
234
235 typedef struct {
236 int parent_uffd, child_uffd;
237 } fork_event_args;
238
fork_event_consumer(void * data)239 static void *fork_event_consumer(void *data)
240 {
241 fork_event_args *args = data;
242 struct uffd_msg msg = { 0 };
243
244 ready_for_fork = true;
245
246 /* Read until a full msg received */
247 while (uffd_read_msg(args->parent_uffd, &msg));
248
249 if (msg.event != UFFD_EVENT_FORK)
250 err("wrong message: %u\n", msg.event);
251
252 /* Just to be properly freed later */
253 args->child_uffd = msg.arg.fork.ufd;
254 return NULL;
255 }
256
257 typedef struct {
258 int gup_fd;
259 bool pinned;
260 } pin_args;
261
262 /*
263 * Returns 0 if succeed, <0 for errors. pin_pages() needs to be paired
264 * with unpin_pages(). Currently it needs to be RO longterm pin to satisfy
265 * all needs of the test cases (e.g., trigger unshare, trigger fork() early
266 * CoW, etc.).
267 */
pin_pages(pin_args * args,void * buffer,size_t size)268 static int pin_pages(pin_args *args, void *buffer, size_t size)
269 {
270 struct pin_longterm_test test = {
271 .addr = (uintptr_t)buffer,
272 .size = size,
273 /* Read-only pins */
274 .flags = 0,
275 };
276
277 if (args->pinned)
278 err("already pinned");
279
280 args->gup_fd = open("/sys/kernel/debug/gup_test", O_RDWR);
281 if (args->gup_fd < 0)
282 return -errno;
283
284 if (ioctl(args->gup_fd, PIN_LONGTERM_TEST_START, &test)) {
285 /* Even if gup_test existed, can be an old gup_test / kernel */
286 close(args->gup_fd);
287 return -errno;
288 }
289 args->pinned = true;
290 return 0;
291 }
292
unpin_pages(pin_args * args)293 static void unpin_pages(pin_args *args)
294 {
295 if (!args->pinned)
296 err("unpin without pin first");
297 if (ioctl(args->gup_fd, PIN_LONGTERM_TEST_STOP))
298 err("PIN_LONGTERM_TEST_STOP");
299 close(args->gup_fd);
300 args->pinned = false;
301 }
302
pagemap_test_fork(int uffd,bool with_event,bool test_pin)303 static int pagemap_test_fork(int uffd, bool with_event, bool test_pin)
304 {
305 fork_event_args args = { .parent_uffd = uffd, .child_uffd = -1 };
306 pthread_t thread;
307 pid_t child;
308 uint64_t value;
309 int fd, result;
310
311 /* Prepare a thread to resolve EVENT_FORK */
312 if (with_event) {
313 ready_for_fork = false;
314 if (pthread_create(&thread, NULL, fork_event_consumer, &args))
315 err("pthread_create()");
316 while (!ready_for_fork)
317 ; /* Wait for the poll_thread to start executing before forking */
318 }
319
320 child = fork();
321 if (!child) {
322 /* Open the pagemap fd of the child itself */
323 pin_args args = {};
324
325 fd = pagemap_open();
326
327 if (test_pin && pin_pages(&args, area_dst, page_size))
328 /*
329 * Normally when reach here we have pinned in
330 * previous tests, so shouldn't fail anymore
331 */
332 err("pin page failed in child");
333
334 value = pagemap_get_entry(fd, area_dst);
335 /*
336 * After fork(), we should handle uffd-wp bit differently:
337 *
338 * (1) when with EVENT_FORK, it should persist
339 * (2) when without EVENT_FORK, it should be dropped
340 */
341 pagemap_check_wp(value, with_event);
342 if (test_pin)
343 unpin_pages(&args);
344 /* Succeed */
345 exit(0);
346 }
347 waitpid(child, &result, 0);
348
349 if (with_event) {
350 if (pthread_join(thread, NULL))
351 err("pthread_join()");
352 if (args.child_uffd < 0)
353 err("Didn't receive child uffd");
354 close(args.child_uffd);
355 }
356
357 return result;
358 }
359
uffd_wp_unpopulated_test(uffd_test_args_t * args)360 static void uffd_wp_unpopulated_test(uffd_test_args_t *args)
361 {
362 uint64_t value;
363 int pagemap_fd;
364
365 if (uffd_register(uffd, area_dst, nr_pages * page_size,
366 false, true, false))
367 err("register failed");
368
369 pagemap_fd = pagemap_open();
370
371 /* Test applying pte marker to anon unpopulated */
372 wp_range(uffd, (uint64_t)area_dst, page_size, true);
373 value = pagemap_get_entry(pagemap_fd, area_dst);
374 pagemap_check_wp(value, true);
375
376 /* Test unprotect on anon pte marker */
377 wp_range(uffd, (uint64_t)area_dst, page_size, false);
378 value = pagemap_get_entry(pagemap_fd, area_dst);
379 pagemap_check_wp(value, false);
380
381 /* Test zap on anon marker */
382 wp_range(uffd, (uint64_t)area_dst, page_size, true);
383 if (madvise(area_dst, page_size, MADV_DONTNEED))
384 err("madvise(MADV_DONTNEED) failed");
385 value = pagemap_get_entry(pagemap_fd, area_dst);
386 pagemap_check_wp(value, false);
387
388 /* Test fault in after marker removed */
389 *area_dst = 1;
390 value = pagemap_get_entry(pagemap_fd, area_dst);
391 pagemap_check_wp(value, false);
392 /* Drop it to make pte none again */
393 if (madvise(area_dst, page_size, MADV_DONTNEED))
394 err("madvise(MADV_DONTNEED) failed");
395
396 /* Test read-zero-page upon pte marker */
397 wp_range(uffd, (uint64_t)area_dst, page_size, true);
398 *(volatile char *)area_dst;
399 /* Drop it to make pte none again */
400 if (madvise(area_dst, page_size, MADV_DONTNEED))
401 err("madvise(MADV_DONTNEED) failed");
402
403 uffd_test_pass();
404 }
405
uffd_wp_fork_test_common(uffd_test_args_t * args,bool with_event)406 static void uffd_wp_fork_test_common(uffd_test_args_t *args,
407 bool with_event)
408 {
409 int pagemap_fd;
410 uint64_t value;
411
412 if (uffd_register(uffd, area_dst, nr_pages * page_size,
413 false, true, false))
414 err("register failed");
415
416 pagemap_fd = pagemap_open();
417
418 /* Touch the page */
419 *area_dst = 1;
420 wp_range(uffd, (uint64_t)area_dst, page_size, true);
421 value = pagemap_get_entry(pagemap_fd, area_dst);
422 pagemap_check_wp(value, true);
423 if (pagemap_test_fork(uffd, with_event, false)) {
424 uffd_test_fail("Detected %s uffd-wp bit in child in present pte",
425 with_event ? "missing" : "stall");
426 goto out;
427 }
428
429 /*
430 * This is an attempt for zapping the pgtable so as to test the
431 * markers.
432 *
433 * For private mappings, PAGEOUT will only work on exclusive ptes
434 * (PM_MMAP_EXCLUSIVE) which we should satisfy.
435 *
436 * For shared, PAGEOUT may not work. Use DONTNEED instead which
437 * plays a similar role of zapping (rather than freeing the page)
438 * to expose pte markers.
439 */
440 if (args->mem_type->shared) {
441 if (madvise(area_dst, page_size, MADV_DONTNEED))
442 err("MADV_DONTNEED");
443 } else {
444 /*
445 * NOTE: ignore retval because private-hugetlb doesn't yet
446 * support swapping, so it could fail.
447 */
448 madvise(area_dst, page_size, MADV_PAGEOUT);
449 }
450
451 /* Uffd-wp should persist even swapped out */
452 value = pagemap_get_entry(pagemap_fd, area_dst);
453 pagemap_check_wp(value, true);
454 if (pagemap_test_fork(uffd, with_event, false)) {
455 uffd_test_fail("Detected %s uffd-wp bit in child in zapped pte",
456 with_event ? "missing" : "stall");
457 goto out;
458 }
459
460 /* Unprotect; this tests swap pte modifications */
461 wp_range(uffd, (uint64_t)area_dst, page_size, false);
462 value = pagemap_get_entry(pagemap_fd, area_dst);
463 pagemap_check_wp(value, false);
464
465 /* Fault in the page from disk */
466 *area_dst = 2;
467 value = pagemap_get_entry(pagemap_fd, area_dst);
468 pagemap_check_wp(value, false);
469 uffd_test_pass();
470 out:
471 if (uffd_unregister(uffd, area_dst, nr_pages * page_size))
472 err("unregister failed");
473 close(pagemap_fd);
474 }
475
uffd_wp_fork_test(uffd_test_args_t * args)476 static void uffd_wp_fork_test(uffd_test_args_t *args)
477 {
478 uffd_wp_fork_test_common(args, false);
479 }
480
uffd_wp_fork_with_event_test(uffd_test_args_t * args)481 static void uffd_wp_fork_with_event_test(uffd_test_args_t *args)
482 {
483 uffd_wp_fork_test_common(args, true);
484 }
485
uffd_wp_fork_pin_test_common(uffd_test_args_t * args,bool with_event)486 static void uffd_wp_fork_pin_test_common(uffd_test_args_t *args,
487 bool with_event)
488 {
489 int pagemap_fd;
490 pin_args pin_args = {};
491
492 if (uffd_register(uffd, area_dst, page_size, false, true, false))
493 err("register failed");
494
495 pagemap_fd = pagemap_open();
496
497 /* Touch the page */
498 *area_dst = 1;
499 wp_range(uffd, (uint64_t)area_dst, page_size, true);
500
501 /*
502 * 1. First pin, then fork(). This tests fork() special path when
503 * doing early CoW if the page is private.
504 */
505 if (pin_pages(&pin_args, area_dst, page_size)) {
506 uffd_test_skip("Possibly CONFIG_GUP_TEST missing "
507 "or unprivileged");
508 close(pagemap_fd);
509 uffd_unregister(uffd, area_dst, page_size);
510 return;
511 }
512
513 if (pagemap_test_fork(uffd, with_event, false)) {
514 uffd_test_fail("Detected %s uffd-wp bit in early CoW of fork()",
515 with_event ? "missing" : "stall");
516 unpin_pages(&pin_args);
517 goto out;
518 }
519
520 unpin_pages(&pin_args);
521
522 /*
523 * 2. First fork(), then pin (in the child, where test_pin==true).
524 * This tests COR, aka, page unsharing on private memories.
525 */
526 if (pagemap_test_fork(uffd, with_event, true)) {
527 uffd_test_fail("Detected %s uffd-wp bit when RO pin",
528 with_event ? "missing" : "stall");
529 goto out;
530 }
531 uffd_test_pass();
532 out:
533 if (uffd_unregister(uffd, area_dst, page_size))
534 err("register failed");
535 close(pagemap_fd);
536 }
537
uffd_wp_fork_pin_test(uffd_test_args_t * args)538 static void uffd_wp_fork_pin_test(uffd_test_args_t *args)
539 {
540 uffd_wp_fork_pin_test_common(args, false);
541 }
542
uffd_wp_fork_pin_with_event_test(uffd_test_args_t * args)543 static void uffd_wp_fork_pin_with_event_test(uffd_test_args_t *args)
544 {
545 uffd_wp_fork_pin_test_common(args, true);
546 }
547
check_memory_contents(char * p)548 static void check_memory_contents(char *p)
549 {
550 unsigned long i, j;
551 uint8_t expected_byte;
552
553 for (i = 0; i < nr_pages; ++i) {
554 expected_byte = ~((uint8_t)(i % ((uint8_t)-1)));
555 for (j = 0; j < page_size; j++) {
556 uint8_t v = *(uint8_t *)(p + (i * page_size) + j);
557 if (v != expected_byte)
558 err("unexpected page contents");
559 }
560 }
561 }
562
uffd_minor_test_common(bool test_collapse,bool test_wp)563 static void uffd_minor_test_common(bool test_collapse, bool test_wp)
564 {
565 unsigned long p;
566 pthread_t uffd_mon;
567 char c;
568 struct uffd_args args = { 0 };
569
570 /*
571 * NOTE: MADV_COLLAPSE is not yet compatible with WP, so testing
572 * both do not make much sense.
573 */
574 assert(!(test_collapse && test_wp));
575
576 if (uffd_register(uffd, area_dst_alias, nr_pages * page_size,
577 /* NOTE! MADV_COLLAPSE may not work with uffd-wp */
578 false, test_wp, true))
579 err("register failure");
580
581 /*
582 * After registering with UFFD, populate the non-UFFD-registered side of
583 * the shared mapping. This should *not* trigger any UFFD minor faults.
584 */
585 for (p = 0; p < nr_pages; ++p)
586 memset(area_dst + (p * page_size), p % ((uint8_t)-1),
587 page_size);
588
589 args.apply_wp = test_wp;
590 if (pthread_create(&uffd_mon, NULL, uffd_poll_thread, &args))
591 err("uffd_poll_thread create");
592
593 /*
594 * Read each of the pages back using the UFFD-registered mapping. We
595 * expect that the first time we touch a page, it will result in a minor
596 * fault. uffd_poll_thread will resolve the fault by bit-flipping the
597 * page's contents, and then issuing a CONTINUE ioctl.
598 */
599 check_memory_contents(area_dst_alias);
600
601 if (write(pipefd[1], &c, sizeof(c)) != sizeof(c))
602 err("pipe write");
603 if (pthread_join(uffd_mon, NULL))
604 err("join() failed");
605
606 if (test_collapse) {
607 if (madvise(area_dst_alias, nr_pages * page_size,
608 MADV_COLLAPSE)) {
609 /* It's fine to fail for this one... */
610 uffd_test_skip("MADV_COLLAPSE failed");
611 return;
612 }
613
614 uffd_test_ops->check_pmd_mapping(area_dst,
615 nr_pages * page_size /
616 read_pmd_pagesize());
617 /*
618 * This won't cause uffd-fault - it purely just makes sure there
619 * was no corruption.
620 */
621 check_memory_contents(area_dst_alias);
622 }
623
624 if (args.missing_faults != 0 || args.minor_faults != nr_pages)
625 uffd_test_fail("stats check error");
626 else
627 uffd_test_pass();
628 }
629
uffd_minor_test(uffd_test_args_t * args)630 void uffd_minor_test(uffd_test_args_t *args)
631 {
632 uffd_minor_test_common(false, false);
633 }
634
uffd_minor_wp_test(uffd_test_args_t * args)635 void uffd_minor_wp_test(uffd_test_args_t *args)
636 {
637 uffd_minor_test_common(false, true);
638 }
639
uffd_minor_collapse_test(uffd_test_args_t * args)640 void uffd_minor_collapse_test(uffd_test_args_t *args)
641 {
642 uffd_minor_test_common(true, false);
643 }
644
645 static sigjmp_buf jbuf, *sigbuf;
646
sighndl(int sig,siginfo_t * siginfo,void * ptr)647 static void sighndl(int sig, siginfo_t *siginfo, void *ptr)
648 {
649 if (sig == SIGBUS) {
650 if (sigbuf)
651 siglongjmp(*sigbuf, 1);
652 abort();
653 }
654 }
655
656 /*
657 * For non-cooperative userfaultfd test we fork() a process that will
658 * generate pagefaults, will mremap the area monitored by the
659 * userfaultfd and at last this process will release the monitored
660 * area.
661 * For the anonymous and shared memory the area is divided into two
662 * parts, the first part is accessed before mremap, and the second
663 * part is accessed after mremap. Since hugetlbfs does not support
664 * mremap, the entire monitored area is accessed in a single pass for
665 * HUGETLB_TEST.
666 * The release of the pages currently generates event for shmem and
667 * anonymous memory (UFFD_EVENT_REMOVE), hence it is not checked
668 * for hugetlb.
669 * For signal test(UFFD_FEATURE_SIGBUS), signal_test = 1, we register
670 * monitored area, generate pagefaults and test that signal is delivered.
671 * Use UFFDIO_COPY to allocate missing page and retry. For signal_test = 2
672 * test robustness use case - we release monitored area, fork a process
673 * that will generate pagefaults and verify signal is generated.
674 * This also tests UFFD_FEATURE_EVENT_FORK event along with the signal
675 * feature. Using monitor thread, verify no userfault events are generated.
676 */
faulting_process(int signal_test,bool wp)677 static int faulting_process(int signal_test, bool wp)
678 {
679 unsigned long nr, i;
680 unsigned long long count;
681 unsigned long split_nr_pages;
682 unsigned long lastnr;
683 struct sigaction act;
684 volatile unsigned long signalled = 0;
685
686 split_nr_pages = (nr_pages + 1) / 2;
687
688 if (signal_test) {
689 sigbuf = &jbuf;
690 memset(&act, 0, sizeof(act));
691 act.sa_sigaction = sighndl;
692 act.sa_flags = SA_SIGINFO;
693 if (sigaction(SIGBUS, &act, 0))
694 err("sigaction");
695 lastnr = (unsigned long)-1;
696 }
697
698 for (nr = 0; nr < split_nr_pages; nr++) {
699 volatile int steps = 1;
700 unsigned long offset = nr * page_size;
701
702 if (signal_test) {
703 if (sigsetjmp(*sigbuf, 1) != 0) {
704 if (steps == 1 && nr == lastnr)
705 err("Signal repeated");
706
707 lastnr = nr;
708 if (signal_test == 1) {
709 if (steps == 1) {
710 /* This is a MISSING request */
711 steps++;
712 if (copy_page(uffd, offset, wp))
713 signalled++;
714 } else {
715 /* This is a WP request */
716 assert(steps == 2);
717 wp_range(uffd,
718 (__u64)area_dst +
719 offset,
720 page_size, false);
721 }
722 } else {
723 signalled++;
724 continue;
725 }
726 }
727 }
728
729 count = *area_count(area_dst, nr);
730 if (count != count_verify[nr])
731 err("nr %lu memory corruption %llu %llu\n",
732 nr, count, count_verify[nr]);
733 /*
734 * Trigger write protection if there is by writing
735 * the same value back.
736 */
737 *area_count(area_dst, nr) = count;
738 }
739
740 if (signal_test)
741 return signalled != split_nr_pages;
742
743 area_dst = mremap(area_dst, nr_pages * page_size, nr_pages * page_size,
744 MREMAP_MAYMOVE | MREMAP_FIXED, area_src);
745 if (area_dst == MAP_FAILED)
746 err("mremap");
747 /* Reset area_src since we just clobbered it */
748 area_src = NULL;
749
750 for (; nr < nr_pages; nr++) {
751 count = *area_count(area_dst, nr);
752 if (count != count_verify[nr]) {
753 err("nr %lu memory corruption %llu %llu\n",
754 nr, count, count_verify[nr]);
755 }
756 /*
757 * Trigger write protection if there is by writing
758 * the same value back.
759 */
760 *area_count(area_dst, nr) = count;
761 }
762
763 uffd_test_ops->release_pages(area_dst);
764
765 for (nr = 0; nr < nr_pages; nr++)
766 for (i = 0; i < page_size; i++)
767 if (*(area_dst + nr * page_size + i) != 0)
768 err("page %lu offset %lu is not zero", nr, i);
769
770 return 0;
771 }
772
uffd_sigbus_test_common(bool wp)773 static void uffd_sigbus_test_common(bool wp)
774 {
775 unsigned long userfaults;
776 pthread_t uffd_mon;
777 pid_t pid;
778 int err;
779 char c;
780 struct uffd_args args = { 0 };
781
782 ready_for_fork = false;
783
784 fcntl(uffd, F_SETFL, uffd_flags | O_NONBLOCK);
785
786 if (uffd_register(uffd, area_dst, nr_pages * page_size,
787 true, wp, false))
788 err("register failure");
789
790 if (faulting_process(1, wp))
791 err("faulting process failed");
792
793 uffd_test_ops->release_pages(area_dst);
794
795 args.apply_wp = wp;
796 if (pthread_create(&uffd_mon, NULL, uffd_poll_thread, &args))
797 err("uffd_poll_thread create");
798
799 while (!ready_for_fork)
800 ; /* Wait for the poll_thread to start executing before forking */
801
802 pid = fork();
803 if (pid < 0)
804 err("fork");
805
806 if (!pid)
807 exit(faulting_process(2, wp));
808
809 waitpid(pid, &err, 0);
810 if (err)
811 err("faulting process failed");
812 if (write(pipefd[1], &c, sizeof(c)) != sizeof(c))
813 err("pipe write");
814 if (pthread_join(uffd_mon, (void **)&userfaults))
815 err("pthread_join()");
816
817 if (userfaults)
818 uffd_test_fail("Signal test failed, userfaults: %ld", userfaults);
819 else
820 uffd_test_pass();
821 }
822
uffd_sigbus_test(uffd_test_args_t * args)823 static void uffd_sigbus_test(uffd_test_args_t *args)
824 {
825 uffd_sigbus_test_common(false);
826 }
827
uffd_sigbus_wp_test(uffd_test_args_t * args)828 static void uffd_sigbus_wp_test(uffd_test_args_t *args)
829 {
830 uffd_sigbus_test_common(true);
831 }
832
uffd_events_test_common(bool wp)833 static void uffd_events_test_common(bool wp)
834 {
835 pthread_t uffd_mon;
836 pid_t pid;
837 int err;
838 char c;
839 struct uffd_args args = { 0 };
840
841 ready_for_fork = false;
842
843 fcntl(uffd, F_SETFL, uffd_flags | O_NONBLOCK);
844 if (uffd_register(uffd, area_dst, nr_pages * page_size,
845 true, wp, false))
846 err("register failure");
847
848 args.apply_wp = wp;
849 if (pthread_create(&uffd_mon, NULL, uffd_poll_thread, &args))
850 err("uffd_poll_thread create");
851
852 while (!ready_for_fork)
853 ; /* Wait for the poll_thread to start executing before forking */
854
855 pid = fork();
856 if (pid < 0)
857 err("fork");
858
859 if (!pid)
860 exit(faulting_process(0, wp));
861
862 waitpid(pid, &err, 0);
863 if (err)
864 err("faulting process failed");
865 if (write(pipefd[1], &c, sizeof(c)) != sizeof(c))
866 err("pipe write");
867 if (pthread_join(uffd_mon, NULL))
868 err("pthread_join()");
869
870 if (args.missing_faults != nr_pages)
871 uffd_test_fail("Fault counts wrong");
872 else
873 uffd_test_pass();
874 }
875
uffd_events_test(uffd_test_args_t * args)876 static void uffd_events_test(uffd_test_args_t *args)
877 {
878 uffd_events_test_common(false);
879 }
880
uffd_events_wp_test(uffd_test_args_t * args)881 static void uffd_events_wp_test(uffd_test_args_t *args)
882 {
883 uffd_events_test_common(true);
884 }
885
retry_uffdio_zeropage(int ufd,struct uffdio_zeropage * uffdio_zeropage)886 static void retry_uffdio_zeropage(int ufd,
887 struct uffdio_zeropage *uffdio_zeropage)
888 {
889 uffd_test_ops->alias_mapping(&uffdio_zeropage->range.start,
890 uffdio_zeropage->range.len,
891 0);
892 if (ioctl(ufd, UFFDIO_ZEROPAGE, uffdio_zeropage)) {
893 if (uffdio_zeropage->zeropage != -EEXIST)
894 err("UFFDIO_ZEROPAGE error: %"PRId64,
895 (int64_t)uffdio_zeropage->zeropage);
896 } else {
897 err("UFFDIO_ZEROPAGE error: %"PRId64,
898 (int64_t)uffdio_zeropage->zeropage);
899 }
900 }
901
do_uffdio_zeropage(int ufd,bool has_zeropage)902 static bool do_uffdio_zeropage(int ufd, bool has_zeropage)
903 {
904 struct uffdio_zeropage uffdio_zeropage = { 0 };
905 int ret;
906 __s64 res;
907
908 uffdio_zeropage.range.start = (unsigned long) area_dst;
909 uffdio_zeropage.range.len = page_size;
910 uffdio_zeropage.mode = 0;
911 ret = ioctl(ufd, UFFDIO_ZEROPAGE, &uffdio_zeropage);
912 res = uffdio_zeropage.zeropage;
913 if (ret) {
914 /* real retval in ufdio_zeropage.zeropage */
915 if (has_zeropage)
916 err("UFFDIO_ZEROPAGE error: %"PRId64, (int64_t)res);
917 else if (res != -EINVAL)
918 err("UFFDIO_ZEROPAGE not -EINVAL");
919 } else if (has_zeropage) {
920 if (res != page_size)
921 err("UFFDIO_ZEROPAGE unexpected size");
922 else
923 retry_uffdio_zeropage(ufd, &uffdio_zeropage);
924 return true;
925 } else
926 err("UFFDIO_ZEROPAGE succeeded");
927
928 return false;
929 }
930
931 /*
932 * Registers a range with MISSING mode only for zeropage test. Return true
933 * if UFFDIO_ZEROPAGE supported, false otherwise. Can't use uffd_register()
934 * because we want to detect .ioctls along the way.
935 */
936 static bool
uffd_register_detect_zeropage(int uffd,void * addr,uint64_t len)937 uffd_register_detect_zeropage(int uffd, void *addr, uint64_t len)
938 {
939 uint64_t ioctls = 0;
940
941 if (uffd_register_with_ioctls(uffd, addr, len, true,
942 false, false, &ioctls))
943 err("zeropage register fail");
944
945 return ioctls & (1 << _UFFDIO_ZEROPAGE);
946 }
947
948 /* exercise UFFDIO_ZEROPAGE */
uffd_zeropage_test(uffd_test_args_t * args)949 static void uffd_zeropage_test(uffd_test_args_t *args)
950 {
951 bool has_zeropage;
952 int i;
953
954 has_zeropage = uffd_register_detect_zeropage(uffd, area_dst, page_size);
955 if (area_dst_alias)
956 /* Ignore the retval; we already have it */
957 uffd_register_detect_zeropage(uffd, area_dst_alias, page_size);
958
959 if (do_uffdio_zeropage(uffd, has_zeropage))
960 for (i = 0; i < page_size; i++)
961 if (area_dst[i] != 0)
962 err("data non-zero at offset %d\n", i);
963
964 if (uffd_unregister(uffd, area_dst, page_size))
965 err("unregister");
966
967 if (area_dst_alias && uffd_unregister(uffd, area_dst_alias, page_size))
968 err("unregister");
969
970 uffd_test_pass();
971 }
972
uffd_register_poison(int uffd,void * addr,uint64_t len)973 static void uffd_register_poison(int uffd, void *addr, uint64_t len)
974 {
975 uint64_t ioctls = 0;
976 uint64_t expected = (1 << _UFFDIO_COPY) | (1 << _UFFDIO_POISON);
977
978 if (uffd_register_with_ioctls(uffd, addr, len, true,
979 false, false, &ioctls))
980 err("poison register fail");
981
982 if ((ioctls & expected) != expected)
983 err("registered area doesn't support COPY and POISON ioctls");
984 }
985
do_uffdio_poison(int uffd,unsigned long offset)986 static void do_uffdio_poison(int uffd, unsigned long offset)
987 {
988 struct uffdio_poison uffdio_poison = { 0 };
989 int ret;
990 __s64 res;
991
992 uffdio_poison.range.start = (unsigned long) area_dst + offset;
993 uffdio_poison.range.len = page_size;
994 uffdio_poison.mode = 0;
995 ret = ioctl(uffd, UFFDIO_POISON, &uffdio_poison);
996 res = uffdio_poison.updated;
997
998 if (ret)
999 err("UFFDIO_POISON error: %"PRId64, (int64_t)res);
1000 else if (res != page_size)
1001 err("UFFDIO_POISON unexpected size: %"PRId64, (int64_t)res);
1002 }
1003
uffd_poison_handle_fault(struct uffd_msg * msg,struct uffd_args * args)1004 static void uffd_poison_handle_fault(
1005 struct uffd_msg *msg, struct uffd_args *args)
1006 {
1007 unsigned long offset;
1008
1009 if (msg->event != UFFD_EVENT_PAGEFAULT)
1010 err("unexpected msg event %u", msg->event);
1011
1012 if (msg->arg.pagefault.flags &
1013 (UFFD_PAGEFAULT_FLAG_WP | UFFD_PAGEFAULT_FLAG_MINOR))
1014 err("unexpected fault type %llu", msg->arg.pagefault.flags);
1015
1016 offset = (char *)(unsigned long)msg->arg.pagefault.address - area_dst;
1017 offset &= ~(page_size-1);
1018
1019 /* Odd pages -> copy zeroed page; even pages -> poison. */
1020 if (offset & page_size)
1021 copy_page(uffd, offset, false);
1022 else
1023 do_uffdio_poison(uffd, offset);
1024 }
1025
uffd_poison_test(uffd_test_args_t * targs)1026 static void uffd_poison_test(uffd_test_args_t *targs)
1027 {
1028 pthread_t uffd_mon;
1029 char c;
1030 struct uffd_args args = { 0 };
1031 struct sigaction act = { 0 };
1032 unsigned long nr_sigbus = 0;
1033 unsigned long nr;
1034
1035 fcntl(uffd, F_SETFL, uffd_flags | O_NONBLOCK);
1036
1037 uffd_register_poison(uffd, area_dst, nr_pages * page_size);
1038 memset(area_src, 0, nr_pages * page_size);
1039
1040 args.handle_fault = uffd_poison_handle_fault;
1041 if (pthread_create(&uffd_mon, NULL, uffd_poll_thread, &args))
1042 err("uffd_poll_thread create");
1043
1044 sigbuf = &jbuf;
1045 act.sa_sigaction = sighndl;
1046 act.sa_flags = SA_SIGINFO;
1047 if (sigaction(SIGBUS, &act, 0))
1048 err("sigaction");
1049
1050 for (nr = 0; nr < nr_pages; ++nr) {
1051 unsigned long offset = nr * page_size;
1052 const char *bytes = (const char *) area_dst + offset;
1053 const char *i;
1054
1055 if (sigsetjmp(*sigbuf, 1)) {
1056 /*
1057 * Access below triggered a SIGBUS, which was caught by
1058 * sighndl, which then jumped here. Count this SIGBUS,
1059 * and move on to next page.
1060 */
1061 ++nr_sigbus;
1062 continue;
1063 }
1064
1065 for (i = bytes; i < bytes + page_size; ++i) {
1066 if (*i)
1067 err("nonzero byte in area_dst (%p) at %p: %u",
1068 area_dst, i, *i);
1069 }
1070 }
1071
1072 if (write(pipefd[1], &c, sizeof(c)) != sizeof(c))
1073 err("pipe write");
1074 if (pthread_join(uffd_mon, NULL))
1075 err("pthread_join()");
1076
1077 if (nr_sigbus != nr_pages / 2)
1078 err("expected to receive %lu SIGBUS, actually received %lu",
1079 nr_pages / 2, nr_sigbus);
1080
1081 uffd_test_pass();
1082 }
1083
1084 static void
uffd_move_handle_fault_common(struct uffd_msg * msg,struct uffd_args * args,unsigned long len)1085 uffd_move_handle_fault_common(struct uffd_msg *msg, struct uffd_args *args,
1086 unsigned long len)
1087 {
1088 unsigned long offset;
1089
1090 if (msg->event != UFFD_EVENT_PAGEFAULT)
1091 err("unexpected msg event %u", msg->event);
1092
1093 if (msg->arg.pagefault.flags &
1094 (UFFD_PAGEFAULT_FLAG_WP | UFFD_PAGEFAULT_FLAG_MINOR | UFFD_PAGEFAULT_FLAG_WRITE))
1095 err("unexpected fault type %llu", msg->arg.pagefault.flags);
1096
1097 offset = (char *)(unsigned long)msg->arg.pagefault.address - area_dst;
1098 offset &= ~(len-1);
1099
1100 if (move_page(uffd, offset, len))
1101 args->missing_faults++;
1102 }
1103
uffd_move_handle_fault(struct uffd_msg * msg,struct uffd_args * args)1104 static void uffd_move_handle_fault(struct uffd_msg *msg,
1105 struct uffd_args *args)
1106 {
1107 uffd_move_handle_fault_common(msg, args, page_size);
1108 }
1109
uffd_move_pmd_handle_fault(struct uffd_msg * msg,struct uffd_args * args)1110 static void uffd_move_pmd_handle_fault(struct uffd_msg *msg,
1111 struct uffd_args *args)
1112 {
1113 uffd_move_handle_fault_common(msg, args, read_pmd_pagesize());
1114 }
1115
1116 static void
uffd_move_test_common(uffd_test_args_t * targs,unsigned long chunk_size,void (* handle_fault)(struct uffd_msg * msg,struct uffd_args * args))1117 uffd_move_test_common(uffd_test_args_t *targs, unsigned long chunk_size,
1118 void (*handle_fault)(struct uffd_msg *msg, struct uffd_args *args))
1119 {
1120 unsigned long nr;
1121 pthread_t uffd_mon;
1122 char c;
1123 unsigned long long count;
1124 struct uffd_args args = { 0 };
1125 char *orig_area_src, *orig_area_dst;
1126 unsigned long step_size, step_count;
1127 unsigned long src_offs = 0;
1128 unsigned long dst_offs = 0;
1129
1130 /* Prevent source pages from being mapped more than once */
1131 if (madvise(area_src, nr_pages * page_size, MADV_DONTFORK))
1132 err("madvise(MADV_DONTFORK) failure");
1133
1134 if (uffd_register(uffd, area_dst, nr_pages * page_size,
1135 true, false, false))
1136 err("register failure");
1137
1138 args.handle_fault = handle_fault;
1139 if (pthread_create(&uffd_mon, NULL, uffd_poll_thread, &args))
1140 err("uffd_poll_thread create");
1141
1142 step_size = chunk_size / page_size;
1143 step_count = nr_pages / step_size;
1144
1145 if (chunk_size > page_size) {
1146 char *aligned_src = ALIGN_UP(area_src, chunk_size);
1147 char *aligned_dst = ALIGN_UP(area_dst, chunk_size);
1148
1149 if (aligned_src != area_src || aligned_dst != area_dst) {
1150 src_offs = (aligned_src - area_src) / page_size;
1151 dst_offs = (aligned_dst - area_dst) / page_size;
1152 step_count--;
1153 }
1154 orig_area_src = area_src;
1155 orig_area_dst = area_dst;
1156 area_src = aligned_src;
1157 area_dst = aligned_dst;
1158 }
1159
1160 /*
1161 * Read each of the pages back using the UFFD-registered mapping. We
1162 * expect that the first time we touch a page, it will result in a missing
1163 * fault. uffd_poll_thread will resolve the fault by moving source
1164 * page to destination.
1165 */
1166 for (nr = 0; nr < step_count * step_size; nr += step_size) {
1167 unsigned long i;
1168
1169 /* Check area_src content */
1170 for (i = 0; i < step_size; i++) {
1171 count = *area_count(area_src, nr + i);
1172 if (count != count_verify[src_offs + nr + i])
1173 err("nr %lu source memory invalid %llu %llu\n",
1174 nr + i, count, count_verify[src_offs + nr + i]);
1175 }
1176
1177 /* Faulting into area_dst should move the page or the huge page */
1178 for (i = 0; i < step_size; i++) {
1179 count = *area_count(area_dst, nr + i);
1180 if (count != count_verify[dst_offs + nr + i])
1181 err("nr %lu memory corruption %llu %llu\n",
1182 nr, count, count_verify[dst_offs + nr + i]);
1183 }
1184
1185 /* Re-check area_src content which should be empty */
1186 for (i = 0; i < step_size; i++) {
1187 count = *area_count(area_src, nr + i);
1188 if (count != 0)
1189 err("nr %lu move failed %llu %llu\n",
1190 nr, count, count_verify[src_offs + nr + i]);
1191 }
1192 }
1193 if (step_size > page_size) {
1194 area_src = orig_area_src;
1195 area_dst = orig_area_dst;
1196 }
1197
1198 if (write(pipefd[1], &c, sizeof(c)) != sizeof(c))
1199 err("pipe write");
1200 if (pthread_join(uffd_mon, NULL))
1201 err("join() failed");
1202
1203 if (args.missing_faults != step_count || args.minor_faults != 0)
1204 uffd_test_fail("stats check error");
1205 else
1206 uffd_test_pass();
1207 }
1208
uffd_move_test(uffd_test_args_t * targs)1209 static void uffd_move_test(uffd_test_args_t *targs)
1210 {
1211 uffd_move_test_common(targs, page_size, uffd_move_handle_fault);
1212 }
1213
uffd_move_pmd_test(uffd_test_args_t * targs)1214 static void uffd_move_pmd_test(uffd_test_args_t *targs)
1215 {
1216 if (madvise(area_dst, nr_pages * page_size, MADV_HUGEPAGE))
1217 err("madvise(MADV_HUGEPAGE) failure");
1218 uffd_move_test_common(targs, read_pmd_pagesize(),
1219 uffd_move_pmd_handle_fault);
1220 }
1221
uffd_move_pmd_split_test(uffd_test_args_t * targs)1222 static void uffd_move_pmd_split_test(uffd_test_args_t *targs)
1223 {
1224 if (madvise(area_dst, nr_pages * page_size, MADV_NOHUGEPAGE))
1225 err("madvise(MADV_NOHUGEPAGE) failure");
1226 uffd_move_test_common(targs, read_pmd_pagesize(),
1227 uffd_move_pmd_handle_fault);
1228 }
1229
prevent_hugepages(const char ** errmsg)1230 static int prevent_hugepages(const char **errmsg)
1231 {
1232 /* This should be done before source area is populated */
1233 if (madvise(area_src, nr_pages * page_size, MADV_NOHUGEPAGE)) {
1234 /* Ignore only if CONFIG_TRANSPARENT_HUGEPAGE=n */
1235 if (errno != EINVAL) {
1236 if (errmsg)
1237 *errmsg = "madvise(MADV_NOHUGEPAGE) failed";
1238 return -errno;
1239 }
1240 }
1241 return 0;
1242 }
1243
request_hugepages(const char ** errmsg)1244 static int request_hugepages(const char **errmsg)
1245 {
1246 /* This should be done before source area is populated */
1247 if (madvise(area_src, nr_pages * page_size, MADV_HUGEPAGE)) {
1248 if (errmsg) {
1249 *errmsg = (errno == EINVAL) ?
1250 "CONFIG_TRANSPARENT_HUGEPAGE is not set" :
1251 "madvise(MADV_HUGEPAGE) failed";
1252 }
1253 return -errno;
1254 }
1255 return 0;
1256 }
1257
1258 struct uffd_test_case_ops uffd_move_test_case_ops = {
1259 .post_alloc = prevent_hugepages,
1260 };
1261
1262 struct uffd_test_case_ops uffd_move_test_pmd_case_ops = {
1263 .post_alloc = request_hugepages,
1264 };
1265
1266 /*
1267 * Test the returned uffdio_register.ioctls with different register modes.
1268 * Note that _UFFDIO_ZEROPAGE is tested separately in the zeropage test.
1269 */
1270 static void
do_register_ioctls_test(uffd_test_args_t * args,bool miss,bool wp,bool minor)1271 do_register_ioctls_test(uffd_test_args_t *args, bool miss, bool wp, bool minor)
1272 {
1273 uint64_t ioctls = 0, expected = BIT_ULL(_UFFDIO_WAKE);
1274 mem_type_t *mem_type = args->mem_type;
1275 int ret;
1276
1277 ret = uffd_register_with_ioctls(uffd, area_dst, page_size,
1278 miss, wp, minor, &ioctls);
1279
1280 /*
1281 * Handle special cases of UFFDIO_REGISTER here where it should
1282 * just fail with -EINVAL first..
1283 *
1284 * Case 1: register MINOR on anon
1285 * Case 2: register with no mode selected
1286 */
1287 if ((minor && (mem_type->mem_flag == MEM_ANON)) ||
1288 (!miss && !wp && !minor)) {
1289 if (ret != -EINVAL)
1290 err("register (miss=%d, wp=%d, minor=%d) failed "
1291 "with wrong errno=%d", miss, wp, minor, ret);
1292 return;
1293 }
1294
1295 /* UFFDIO_REGISTER should succeed, then check ioctls returned */
1296 if (miss)
1297 expected |= BIT_ULL(_UFFDIO_COPY);
1298 if (wp)
1299 expected |= BIT_ULL(_UFFDIO_WRITEPROTECT);
1300 if (minor)
1301 expected |= BIT_ULL(_UFFDIO_CONTINUE);
1302
1303 if ((ioctls & expected) != expected)
1304 err("unexpected uffdio_register.ioctls "
1305 "(miss=%d, wp=%d, minor=%d): expected=0x%"PRIx64", "
1306 "returned=0x%"PRIx64, miss, wp, minor, expected, ioctls);
1307
1308 if (uffd_unregister(uffd, area_dst, page_size))
1309 err("unregister");
1310 }
1311
uffd_register_ioctls_test(uffd_test_args_t * args)1312 static void uffd_register_ioctls_test(uffd_test_args_t *args)
1313 {
1314 int miss, wp, minor;
1315
1316 for (miss = 0; miss <= 1; miss++)
1317 for (wp = 0; wp <= 1; wp++)
1318 for (minor = 0; minor <= 1; minor++)
1319 do_register_ioctls_test(args, miss, wp, minor);
1320
1321 uffd_test_pass();
1322 }
1323
1324 uffd_test_case_t uffd_tests[] = {
1325 {
1326 /* Test returned uffdio_register.ioctls. */
1327 .name = "register-ioctls",
1328 .uffd_fn = uffd_register_ioctls_test,
1329 .mem_targets = MEM_ALL,
1330 .uffd_feature_required = UFFD_FEATURE_MISSING_HUGETLBFS |
1331 UFFD_FEATURE_MISSING_SHMEM |
1332 UFFD_FEATURE_PAGEFAULT_FLAG_WP |
1333 UFFD_FEATURE_WP_HUGETLBFS_SHMEM |
1334 UFFD_FEATURE_MINOR_HUGETLBFS |
1335 UFFD_FEATURE_MINOR_SHMEM,
1336 },
1337 {
1338 .name = "zeropage",
1339 .uffd_fn = uffd_zeropage_test,
1340 .mem_targets = MEM_ALL,
1341 .uffd_feature_required = 0,
1342 },
1343 {
1344 .name = "move",
1345 .uffd_fn = uffd_move_test,
1346 .mem_targets = MEM_ANON,
1347 .uffd_feature_required = UFFD_FEATURE_MOVE,
1348 .test_case_ops = &uffd_move_test_case_ops,
1349 },
1350 {
1351 .name = "move-pmd",
1352 .uffd_fn = uffd_move_pmd_test,
1353 .mem_targets = MEM_ANON,
1354 .uffd_feature_required = UFFD_FEATURE_MOVE,
1355 .test_case_ops = &uffd_move_test_pmd_case_ops,
1356 },
1357 {
1358 .name = "move-pmd-split",
1359 .uffd_fn = uffd_move_pmd_split_test,
1360 .mem_targets = MEM_ANON,
1361 .uffd_feature_required = UFFD_FEATURE_MOVE,
1362 .test_case_ops = &uffd_move_test_pmd_case_ops,
1363 },
1364 {
1365 .name = "wp-fork",
1366 .uffd_fn = uffd_wp_fork_test,
1367 .mem_targets = MEM_ALL,
1368 .uffd_feature_required = UFFD_FEATURE_PAGEFAULT_FLAG_WP |
1369 UFFD_FEATURE_WP_HUGETLBFS_SHMEM,
1370 },
1371 {
1372 .name = "wp-fork-with-event",
1373 .uffd_fn = uffd_wp_fork_with_event_test,
1374 .mem_targets = MEM_ALL,
1375 .uffd_feature_required = UFFD_FEATURE_PAGEFAULT_FLAG_WP |
1376 UFFD_FEATURE_WP_HUGETLBFS_SHMEM |
1377 /* when set, child process should inherit uffd-wp bits */
1378 UFFD_FEATURE_EVENT_FORK,
1379 },
1380 {
1381 .name = "wp-fork-pin",
1382 .uffd_fn = uffd_wp_fork_pin_test,
1383 .mem_targets = MEM_ALL,
1384 .uffd_feature_required = UFFD_FEATURE_PAGEFAULT_FLAG_WP |
1385 UFFD_FEATURE_WP_HUGETLBFS_SHMEM,
1386 },
1387 {
1388 .name = "wp-fork-pin-with-event",
1389 .uffd_fn = uffd_wp_fork_pin_with_event_test,
1390 .mem_targets = MEM_ALL,
1391 .uffd_feature_required = UFFD_FEATURE_PAGEFAULT_FLAG_WP |
1392 UFFD_FEATURE_WP_HUGETLBFS_SHMEM |
1393 /* when set, child process should inherit uffd-wp bits */
1394 UFFD_FEATURE_EVENT_FORK,
1395 },
1396 {
1397 .name = "wp-unpopulated",
1398 .uffd_fn = uffd_wp_unpopulated_test,
1399 .mem_targets = MEM_ANON,
1400 .uffd_feature_required =
1401 UFFD_FEATURE_PAGEFAULT_FLAG_WP | UFFD_FEATURE_WP_UNPOPULATED,
1402 },
1403 {
1404 .name = "minor",
1405 .uffd_fn = uffd_minor_test,
1406 .mem_targets = MEM_SHMEM | MEM_HUGETLB,
1407 .uffd_feature_required =
1408 UFFD_FEATURE_MINOR_HUGETLBFS | UFFD_FEATURE_MINOR_SHMEM,
1409 },
1410 {
1411 .name = "minor-wp",
1412 .uffd_fn = uffd_minor_wp_test,
1413 .mem_targets = MEM_SHMEM | MEM_HUGETLB,
1414 .uffd_feature_required =
1415 UFFD_FEATURE_MINOR_HUGETLBFS | UFFD_FEATURE_MINOR_SHMEM |
1416 UFFD_FEATURE_PAGEFAULT_FLAG_WP |
1417 /*
1418 * HACK: here we leveraged WP_UNPOPULATED to detect whether
1419 * minor mode supports wr-protect. There's no feature flag
1420 * for it so this is the best we can test against.
1421 */
1422 UFFD_FEATURE_WP_UNPOPULATED,
1423 },
1424 {
1425 .name = "minor-collapse",
1426 .uffd_fn = uffd_minor_collapse_test,
1427 /* MADV_COLLAPSE only works with shmem */
1428 .mem_targets = MEM_SHMEM,
1429 /* We can't test MADV_COLLAPSE, so try our luck */
1430 .uffd_feature_required = UFFD_FEATURE_MINOR_SHMEM,
1431 },
1432 {
1433 .name = "sigbus",
1434 .uffd_fn = uffd_sigbus_test,
1435 .mem_targets = MEM_ALL,
1436 .uffd_feature_required = UFFD_FEATURE_SIGBUS |
1437 UFFD_FEATURE_EVENT_FORK,
1438 },
1439 {
1440 .name = "sigbus-wp",
1441 .uffd_fn = uffd_sigbus_wp_test,
1442 .mem_targets = MEM_ALL,
1443 .uffd_feature_required = UFFD_FEATURE_SIGBUS |
1444 UFFD_FEATURE_EVENT_FORK | UFFD_FEATURE_PAGEFAULT_FLAG_WP |
1445 UFFD_FEATURE_WP_HUGETLBFS_SHMEM,
1446 },
1447 {
1448 .name = "events",
1449 .uffd_fn = uffd_events_test,
1450 .mem_targets = MEM_ALL,
1451 .uffd_feature_required = UFFD_FEATURE_EVENT_FORK |
1452 UFFD_FEATURE_EVENT_REMAP | UFFD_FEATURE_EVENT_REMOVE,
1453 },
1454 {
1455 .name = "events-wp",
1456 .uffd_fn = uffd_events_wp_test,
1457 .mem_targets = MEM_ALL,
1458 .uffd_feature_required = UFFD_FEATURE_EVENT_FORK |
1459 UFFD_FEATURE_EVENT_REMAP | UFFD_FEATURE_EVENT_REMOVE |
1460 UFFD_FEATURE_PAGEFAULT_FLAG_WP |
1461 UFFD_FEATURE_WP_HUGETLBFS_SHMEM,
1462 },
1463 {
1464 .name = "poison",
1465 .uffd_fn = uffd_poison_test,
1466 .mem_targets = MEM_ALL,
1467 .uffd_feature_required = UFFD_FEATURE_POISON,
1468 },
1469 };
1470
usage(const char * prog)1471 static void usage(const char *prog)
1472 {
1473 printf("usage: %s [-f TESTNAME]\n", prog);
1474 puts("");
1475 puts(" -f: test name to filter (e.g., event)");
1476 puts(" -h: show the help msg");
1477 puts(" -l: list tests only");
1478 puts("");
1479 exit(KSFT_FAIL);
1480 }
1481
main(int argc,char * argv[])1482 int main(int argc, char *argv[])
1483 {
1484 int n_tests = sizeof(uffd_tests) / sizeof(uffd_test_case_t);
1485 int n_mems = sizeof(mem_types) / sizeof(mem_type_t);
1486 const char *test_filter = NULL;
1487 bool list_only = false;
1488 uffd_test_case_t *test;
1489 mem_type_t *mem_type;
1490 uffd_test_args_t args;
1491 const char *errmsg;
1492 int has_uffd, opt;
1493 int i, j;
1494
1495 while ((opt = getopt(argc, argv, "f:hl")) != -1) {
1496 switch (opt) {
1497 case 'f':
1498 test_filter = optarg;
1499 break;
1500 case 'l':
1501 list_only = true;
1502 break;
1503 case 'h':
1504 default:
1505 /* Unknown */
1506 usage(argv[0]);
1507 break;
1508 }
1509 }
1510
1511 if (!test_filter && !list_only) {
1512 has_uffd = test_uffd_api(false);
1513 has_uffd |= test_uffd_api(true);
1514
1515 if (!has_uffd) {
1516 printf("Userfaultfd not supported or unprivileged, skip all tests\n");
1517 exit(KSFT_SKIP);
1518 }
1519 }
1520
1521 for (i = 0; i < n_tests; i++) {
1522 test = &uffd_tests[i];
1523 if (test_filter && !strstr(test->name, test_filter))
1524 continue;
1525 if (list_only) {
1526 printf("%s\n", test->name);
1527 continue;
1528 }
1529 for (j = 0; j < n_mems; j++) {
1530 mem_type = &mem_types[j];
1531 if (!(test->mem_targets & mem_type->mem_flag))
1532 continue;
1533
1534 uffd_test_start("%s on %s", test->name, mem_type->name);
1535 if ((mem_type->mem_flag == MEM_HUGETLB ||
1536 mem_type->mem_flag == MEM_HUGETLB_PRIVATE) &&
1537 (default_huge_page_size() == 0)) {
1538 uffd_test_skip("huge page size is 0, feature missing?");
1539 continue;
1540 }
1541 if (!uffd_feature_supported(test)) {
1542 uffd_test_skip("feature missing");
1543 continue;
1544 }
1545 if (uffd_setup_environment(&args, test, mem_type,
1546 &errmsg)) {
1547 uffd_test_skip(errmsg);
1548 continue;
1549 }
1550 test->uffd_fn(&args);
1551 uffd_test_ctx_clear();
1552 }
1553 }
1554
1555 if (!list_only)
1556 uffd_test_report();
1557
1558 return ksft_get_fail_cnt() ? KSFT_FAIL : KSFT_PASS;
1559 }
1560
1561