Lines Matching +full:1 +full:p
27 #define BASE_ADDR ((void *)(1UL << 30))
46 void (*cleanup_area)(void *p, unsigned long size);
47 void (*fault)(void *p, unsigned long start, unsigned long end);
57 void (*collapse)(const char *msg, char *p, int nr_hpages,
132 char buf[1 << 10]; in get_finfo()
260 void *p; in alloc_mapping() local
262 p = mmap(BASE_ADDR, nr * hpage_pmd_size, PROT_READ | PROT_WRITE, in alloc_mapping()
263 MAP_ANONYMOUS | MAP_PRIVATE, -1, 0); in alloc_mapping()
264 if (p != BASE_ADDR) { in alloc_mapping()
265 printf("Failed to allocate VMA at %p\n", BASE_ADDR); in alloc_mapping()
269 return p; in alloc_mapping()
272 static void fill_memory(int *p, unsigned long start, unsigned long end) in fill_memory() argument
277 p[i * page_size / sizeof(*p)] = i + 0xdead0000; in fill_memory()
286 static int madvise_collapse_retry(void *p, unsigned long size) in madvise_collapse_retry() argument
292 ret = madvise(p, size, MADV_COLLAPSE); in madvise_collapse_retry()
306 void *p = ops->setup_area(1); in alloc_hpage() local
308 ops->fault(p, 0, hpage_pmd_size); in alloc_hpage()
318 if (madvise_collapse_retry(p, hpage_pmd_size)) { in alloc_hpage()
322 if (!ops->check_huge(p, 1)) { in alloc_hpage()
326 if (madvise(p, hpage_pmd_size, MADV_HUGEPAGE)) { in alloc_hpage()
331 return p; in alloc_hpage()
334 static void validate_memory(int *p, unsigned long start, unsigned long end) in validate_memory() argument
339 if (p[i * page_size / sizeof(*p)] != i + 0xdead0000) { in validate_memory()
341 i, p[i * page_size / sizeof(*p)]); in validate_memory()
352 static void anon_cleanup_area(void *p, unsigned long size) in anon_cleanup_area() argument
354 munmap(p, size); in anon_cleanup_area()
357 static void anon_fault(void *p, unsigned long start, unsigned long end) in anon_fault() argument
359 fill_memory(p, start, end); in anon_fault()
370 void *p; in file_setup_area() local
384 p = alloc_mapping(nr_hpages); in file_setup_area()
385 fill_memory(p, 0, size); in file_setup_area()
386 write(fd, p, size); in file_setup_area()
388 munmap(p, size); in file_setup_area()
397 p = mmap(BASE_ADDR, size, PROT_READ | PROT_EXEC, in file_setup_area()
399 if (p == MAP_FAILED || p != BASE_ADDR) { in file_setup_area()
407 return p; in file_setup_area()
410 static void file_cleanup_area(void *p, unsigned long size) in file_cleanup_area() argument
412 munmap(p, size); in file_cleanup_area()
417 static void file_fault(void *p, unsigned long start, unsigned long end) in file_fault() argument
419 if (madvise(((char *)p) + start, end - start, MADV_POPULATE_READ)) { in file_fault()
440 void *p; in shmem_setup_area() local
452 p = mmap(BASE_ADDR, size, PROT_READ | PROT_WRITE, MAP_SHARED, finfo.fd, in shmem_setup_area()
454 if (p != BASE_ADDR) { in shmem_setup_area()
458 return p; in shmem_setup_area()
461 static void shmem_cleanup_area(void *p, unsigned long size) in shmem_cleanup_area() argument
463 munmap(p, size); in shmem_cleanup_area()
496 static void __madvise_collapse(const char *msg, char *p, int nr_hpages, in __madvise_collapse() argument
513 madvise(p, nr_hpages * hpage_pmd_size, MADV_HUGEPAGE); in __madvise_collapse()
514 ret = madvise_collapse_retry(p, nr_hpages * hpage_pmd_size); in __madvise_collapse()
517 else if (!ops->check_huge(p, expect ? nr_hpages : 0)) in __madvise_collapse()
525 static void madvise_collapse(const char *msg, char *p, int nr_hpages, in madvise_collapse() argument
529 if (!ops->check_huge(p, 0)) { in madvise_collapse()
533 __madvise_collapse(msg, p, nr_hpages, ops, expect); in madvise_collapse()
537 static bool wait_for_scan(const char *msg, char *p, int nr_hpages, in wait_for_scan() argument
544 if (!ops->check_huge(p, 0)) { in wait_for_scan()
549 madvise(p, nr_hpages * hpage_pmd_size, MADV_HUGEPAGE); in wait_for_scan()
556 if (ops->check_huge(p, nr_hpages)) in wait_for_scan()
564 madvise(p, nr_hpages * hpage_pmd_size, MADV_NOHUGEPAGE); in wait_for_scan()
566 return timeout == -1; in wait_for_scan()
569 static void khugepaged_collapse(const char *msg, char *p, int nr_hpages, in khugepaged_collapse() argument
572 if (wait_for_scan(msg, p, nr_hpages, ops)) { in khugepaged_collapse()
586 ops->fault(p, 0, nr_hpages * hpage_pmd_size); in khugepaged_collapse()
588 if (ops->check_huge(p, expect ? nr_hpages : 0)) in khugepaged_collapse()
619 char *p; in alloc_at_fault() local
624 p = alloc_mapping(1); in alloc_at_fault()
625 *p = 1; in alloc_at_fault()
627 if (check_huge_anon(p, 1, hpage_pmd_size)) in alloc_at_fault()
634 madvise(p, page_size, MADV_DONTNEED); in alloc_at_fault()
636 if (check_huge_anon(p, 0, hpage_pmd_size)) in alloc_at_fault()
640 munmap(p, hpage_pmd_size); in alloc_at_fault()
645 void *p; in collapse_full() local
649 p = ops->setup_area(nr_hpages); in collapse_full()
650 ops->fault(p, 0, size); in collapse_full()
651 c->collapse("Collapse multiple fully populated PTE table", p, nr_hpages, in collapse_full()
653 validate_memory(p, 0, size); in collapse_full()
654 ops->cleanup_area(p, size); in collapse_full()
659 void *p; in collapse_empty() local
661 p = ops->setup_area(1); in collapse_empty()
662 c->collapse("Do not collapse empty PTE table", p, 1, ops, false); in collapse_empty()
663 ops->cleanup_area(p, hpage_pmd_size); in collapse_empty()
668 void *p; in collapse_single_pte_entry() local
670 p = ops->setup_area(1); in collapse_single_pte_entry()
671 ops->fault(p, 0, page_size); in collapse_single_pte_entry()
672 c->collapse("Collapse PTE table with single PTE entry present", p, in collapse_single_pte_entry()
673 1, ops, true); in collapse_single_pte_entry()
674 ops->cleanup_area(p, hpage_pmd_size); in collapse_single_pte_entry()
681 void *p; in collapse_max_ptes_none() local
682 int fault_nr_pages = is_anon(ops) ? 1 << anon_order : 1; in collapse_max_ptes_none()
687 p = ops->setup_area(1); in collapse_max_ptes_none()
696 ops->fault(p, 0, (hpage_pmd_nr - max_ptes_none - fault_nr_pages) * page_size); in collapse_max_ptes_none()
697 c->collapse("Maybe collapse with max_ptes_none exceeded", p, 1, in collapse_max_ptes_none()
699 validate_memory(p, 0, (hpage_pmd_nr - max_ptes_none - fault_nr_pages) * page_size); in collapse_max_ptes_none()
702 ops->fault(p, 0, (hpage_pmd_nr - max_ptes_none) * page_size); in collapse_max_ptes_none()
703 c->collapse("Collapse with max_ptes_none PTEs empty", p, 1, ops, in collapse_max_ptes_none()
705 validate_memory(p, 0, in collapse_max_ptes_none()
709 ops->cleanup_area(p, hpage_pmd_size); in collapse_max_ptes_none()
715 void *p; in collapse_swapin_single_pte() local
717 p = ops->setup_area(1); in collapse_swapin_single_pte()
718 ops->fault(p, 0, hpage_pmd_size); in collapse_swapin_single_pte()
721 if (madvise(p, page_size, MADV_PAGEOUT)) { in collapse_swapin_single_pte()
725 if (check_swap(p, page_size)) { in collapse_swapin_single_pte()
732 c->collapse("Collapse with swapping in single PTE entry", p, 1, ops, in collapse_swapin_single_pte()
734 validate_memory(p, 0, hpage_pmd_size); in collapse_swapin_single_pte()
736 ops->cleanup_area(p, hpage_pmd_size); in collapse_swapin_single_pte()
742 void *p; in collapse_max_ptes_swap() local
744 p = ops->setup_area(1); in collapse_max_ptes_swap()
745 ops->fault(p, 0, hpage_pmd_size); in collapse_max_ptes_swap()
747 printf("Swapout %d of %d pages...", max_ptes_swap + 1, hpage_pmd_nr); in collapse_max_ptes_swap()
748 if (madvise(p, (max_ptes_swap + 1) * page_size, MADV_PAGEOUT)) { in collapse_max_ptes_swap()
752 if (check_swap(p, (max_ptes_swap + 1) * page_size)) { in collapse_max_ptes_swap()
759 c->collapse("Maybe collapse with max_ptes_swap exceeded", p, 1, ops, in collapse_max_ptes_swap()
761 validate_memory(p, 0, hpage_pmd_size); in collapse_max_ptes_swap()
764 ops->fault(p, 0, hpage_pmd_size); in collapse_max_ptes_swap()
767 if (madvise(p, max_ptes_swap * page_size, MADV_PAGEOUT)) { in collapse_max_ptes_swap()
771 if (check_swap(p, max_ptes_swap * page_size)) { in collapse_max_ptes_swap()
778 c->collapse("Collapse with max_ptes_swap pages swapped out", p, in collapse_max_ptes_swap()
779 1, ops, true); in collapse_max_ptes_swap()
780 validate_memory(p, 0, hpage_pmd_size); in collapse_max_ptes_swap()
783 ops->cleanup_area(p, hpage_pmd_size); in collapse_max_ptes_swap()
788 void *p; in collapse_single_pte_entry_compound() local
790 p = alloc_hpage(ops); in collapse_single_pte_entry_compound()
799 madvise(p, hpage_pmd_size, MADV_NOHUGEPAGE); in collapse_single_pte_entry_compound()
801 madvise(p + page_size, hpage_pmd_size - page_size, MADV_DONTNEED); in collapse_single_pte_entry_compound()
802 if (ops->check_huge(p, 0)) in collapse_single_pte_entry_compound()
808 p, 1, ops, true); in collapse_single_pte_entry_compound()
809 validate_memory(p, 0, page_size); in collapse_single_pte_entry_compound()
811 ops->cleanup_area(p, hpage_pmd_size); in collapse_single_pte_entry_compound()
816 void *p; in collapse_full_of_compound() local
818 p = alloc_hpage(ops); in collapse_full_of_compound()
820 madvise(p, page_size, MADV_NOHUGEPAGE); in collapse_full_of_compound()
821 madvise(p, hpage_pmd_size, MADV_NOHUGEPAGE); in collapse_full_of_compound()
822 if (ops->check_huge(p, 0)) in collapse_full_of_compound()
827 c->collapse("Collapse PTE table full of compound pages", p, 1, ops, in collapse_full_of_compound()
829 validate_memory(p, 0, hpage_pmd_size); in collapse_full_of_compound()
830 ops->cleanup_area(p, hpage_pmd_size); in collapse_full_of_compound()
835 void *p; in collapse_compound_extreme() local
838 p = ops->setup_area(1); in collapse_compound_extreme()
841 i + 1, hpage_pmd_nr); in collapse_compound_extreme()
845 if (!ops->check_huge(BASE_ADDR, 1)) { in collapse_compound_extreme()
851 p = mremap(BASE_ADDR - i * page_size, in collapse_compound_extreme()
853 (i + 1) * page_size, in collapse_compound_extreme()
856 if (p == MAP_FAILED) { in collapse_compound_extreme()
861 p = mremap(BASE_ADDR + 2 * hpage_pmd_size, in collapse_compound_extreme()
862 (i + 1) * page_size, in collapse_compound_extreme()
863 (i + 1) * page_size + hpage_pmd_size, in collapse_compound_extreme()
865 BASE_ADDR - (i + 1) * page_size); in collapse_compound_extreme()
866 if (p == MAP_FAILED) { in collapse_compound_extreme()
873 ops->fault(p, 0, hpage_pmd_size); in collapse_compound_extreme()
874 if (!ops->check_huge(p, 1)) in collapse_compound_extreme()
879 c->collapse("Collapse PTE table full of different compound pages", p, 1, in collapse_compound_extreme()
882 validate_memory(p, 0, hpage_pmd_size); in collapse_compound_extreme()
883 ops->cleanup_area(p, hpage_pmd_size); in collapse_compound_extreme()
889 void *p; in collapse_fork() local
891 p = ops->setup_area(1); in collapse_fork()
894 ops->fault(p, 0, page_size); in collapse_fork()
895 if (ops->check_huge(p, 0)) in collapse_fork()
906 if (ops->check_huge(p, 0)) in collapse_fork()
911 ops->fault(p, page_size, 2 * page_size); in collapse_fork()
913 p, 1, ops, true); in collapse_fork()
915 validate_memory(p, 0, page_size); in collapse_fork()
916 ops->cleanup_area(p, hpage_pmd_size); in collapse_fork()
924 if (ops->check_huge(p, 0)) in collapse_fork()
928 validate_memory(p, 0, page_size); in collapse_fork()
929 ops->cleanup_area(p, hpage_pmd_size); in collapse_fork()
935 void *p; in collapse_fork_compound() local
937 p = alloc_hpage(ops); in collapse_fork_compound()
944 if (ops->check_huge(p, 1)) in collapse_fork_compound()
950 madvise(p, page_size, MADV_NOHUGEPAGE); in collapse_fork_compound()
951 madvise(p, hpage_pmd_size, MADV_NOHUGEPAGE); in collapse_fork_compound()
952 if (ops->check_huge(p, 0)) in collapse_fork_compound()
956 ops->fault(p, 0, page_size); in collapse_fork_compound()
958 thp_write_num("khugepaged/max_ptes_shared", hpage_pmd_nr - 1); in collapse_fork_compound()
960 p, 1, ops, true); in collapse_fork_compound()
964 validate_memory(p, 0, hpage_pmd_size); in collapse_fork_compound()
965 ops->cleanup_area(p, hpage_pmd_size); in collapse_fork_compound()
973 if (ops->check_huge(p, 1)) in collapse_fork_compound()
977 validate_memory(p, 0, hpage_pmd_size); in collapse_fork_compound()
978 ops->cleanup_area(p, hpage_pmd_size); in collapse_fork_compound()
985 void *p; in collapse_max_ptes_shared() local
987 p = alloc_hpage(ops); in collapse_max_ptes_shared()
994 if (ops->check_huge(p, 1)) in collapse_max_ptes_shared()
1000 hpage_pmd_nr - max_ptes_shared - 1, hpage_pmd_nr); in collapse_max_ptes_shared()
1001 ops->fault(p, 0, (hpage_pmd_nr - max_ptes_shared - 1) * page_size); in collapse_max_ptes_shared()
1002 if (ops->check_huge(p, 0)) in collapse_max_ptes_shared()
1007 c->collapse("Maybe collapse with max_ptes_shared exceeded", p, in collapse_max_ptes_shared()
1008 1, ops, !c->enforce_pte_scan_limits); in collapse_max_ptes_shared()
1013 ops->fault(p, 0, (hpage_pmd_nr - max_ptes_shared) * in collapse_max_ptes_shared()
1015 if (ops->check_huge(p, 0)) in collapse_max_ptes_shared()
1021 p, 1, ops, true); in collapse_max_ptes_shared()
1024 validate_memory(p, 0, hpage_pmd_size); in collapse_max_ptes_shared()
1025 ops->cleanup_area(p, hpage_pmd_size); in collapse_max_ptes_shared()
1033 if (ops->check_huge(p, 1)) in collapse_max_ptes_shared()
1037 validate_memory(p, 0, hpage_pmd_size); in collapse_max_ptes_shared()
1038 ops->cleanup_area(p, hpage_pmd_size); in collapse_max_ptes_shared()
1044 void *p; in madvise_collapse_existing_thps() local
1046 p = ops->setup_area(1); in madvise_collapse_existing_thps()
1047 ops->fault(p, 0, hpage_pmd_size); in madvise_collapse_existing_thps()
1048 c->collapse("Collapse fully populated PTE table...", p, 1, ops, true); in madvise_collapse_existing_thps()
1049 validate_memory(p, 0, hpage_pmd_size); in madvise_collapse_existing_thps()
1052 __madvise_collapse("Re-collapse PMD-mapped hugepage", p, 1, ops, true); in madvise_collapse_existing_thps()
1053 validate_memory(p, 0, hpage_pmd_size); in madvise_collapse_existing_thps()
1054 ops->cleanup_area(p, hpage_pmd_size); in madvise_collapse_existing_thps()
1064 void *p; in madvise_retracted_page_tables() local
1065 int nr_hpages = 1; in madvise_retracted_page_tables()
1068 p = ops->setup_area(nr_hpages); in madvise_retracted_page_tables()
1069 ops->fault(p, 0, size); in madvise_retracted_page_tables()
1072 if (wait_for_scan("Collapse and leave PMD cleared", p, nr_hpages, in madvise_retracted_page_tables()
1078 c->collapse("Install huge PMD from page cache", p, nr_hpages, ops, in madvise_retracted_page_tables()
1080 validate_memory(p, 0, size); in madvise_retracted_page_tables()
1081 ops->cleanup_area(p, size); in madvise_retracted_page_tables()
1099 exit(1); in usage()
1108 while ((opt = getopt(argc, argv, "s:h")) != -1) { in parse_test_type()
1167 get_finfo(argv[1]); in parse_test_type()
1179 .defrag = 1, in main()
1206 default_settings.khugepaged.max_ptes_none = hpage_pmd_nr - 1; in main()