1 // SPDX-License-Identifier: GPL-2.0 2 #include <string.h> 3 #include <fcntl.h> 4 #include <dirent.h> 5 #include <sys/ioctl.h> 6 #include <linux/userfaultfd.h> 7 #include <linux/fs.h> 8 #include <sys/syscall.h> 9 #include <unistd.h> 10 #include "../kselftest.h" 11 #include "vm_util.h" 12 13 #define PMD_SIZE_FILE_PATH "/sys/kernel/mm/transparent_hugepage/hpage_pmd_size" 14 #define SMAP_FILE_PATH "/proc/self/smaps" 15 #define STATUS_FILE_PATH "/proc/self/status" 16 #define MAX_LINE_LENGTH 500 17 18 unsigned int __page_size; 19 unsigned int __page_shift; 20 pagemap_get_entry(int fd,char * start)21 uint64_t pagemap_get_entry(int fd, char *start) 22 { 23 const unsigned long pfn = (unsigned long)start / getpagesize(); 24 uint64_t entry; 25 int ret; 26 27 ret = pread(fd, &entry, sizeof(entry), pfn * sizeof(entry)); 28 if (ret != sizeof(entry)) 29 ksft_exit_fail_msg("reading pagemap failed\n"); 30 return entry; 31 } 32 __pagemap_scan_get_categories(int fd,char * start,struct page_region * r)33 static uint64_t __pagemap_scan_get_categories(int fd, char *start, struct page_region *r) 34 { 35 struct pm_scan_arg arg; 36 37 arg.start = (uintptr_t)start; 38 arg.end = (uintptr_t)(start + psize()); 39 arg.vec = (uintptr_t)r; 40 arg.vec_len = 1; 41 arg.flags = 0; 42 arg.size = sizeof(struct pm_scan_arg); 43 arg.max_pages = 0; 44 arg.category_inverted = 0; 45 arg.category_mask = 0; 46 arg.category_anyof_mask = PAGE_IS_WPALLOWED | PAGE_IS_WRITTEN | PAGE_IS_FILE | 47 PAGE_IS_PRESENT | PAGE_IS_SWAPPED | PAGE_IS_PFNZERO | 48 PAGE_IS_HUGE | PAGE_IS_SOFT_DIRTY; 49 arg.return_mask = arg.category_anyof_mask; 50 51 return ioctl(fd, PAGEMAP_SCAN, &arg); 52 } 53 pagemap_scan_get_categories(int fd,char * start)54 static uint64_t pagemap_scan_get_categories(int fd, char *start) 55 { 56 struct page_region r; 57 long ret; 58 59 ret = __pagemap_scan_get_categories(fd, start, &r); 60 if (ret < 0) 61 ksft_exit_fail_msg("PAGEMAP_SCAN failed: %s\n", strerror(errno)); 62 if (ret == 0) 63 return 0; 64 return r.categories; 65 } 66 67 /* `start` is any valid address. */ pagemap_scan_supported(int fd,char * start)68 static bool pagemap_scan_supported(int fd, char *start) 69 { 70 static int supported = -1; 71 int ret; 72 73 if (supported != -1) 74 return supported; 75 76 /* Provide an invalid address in order to trigger EFAULT. */ 77 ret = __pagemap_scan_get_categories(fd, start, (struct page_region *) ~0UL); 78 if (ret == 0) 79 ksft_exit_fail_msg("PAGEMAP_SCAN succeeded unexpectedly\n"); 80 81 supported = errno == EFAULT; 82 83 return supported; 84 } 85 page_entry_is(int fd,char * start,char * desc,uint64_t pagemap_flags,uint64_t pagescan_flags)86 static bool page_entry_is(int fd, char *start, char *desc, 87 uint64_t pagemap_flags, uint64_t pagescan_flags) 88 { 89 bool m = pagemap_get_entry(fd, start) & pagemap_flags; 90 91 if (pagemap_scan_supported(fd, start)) { 92 bool s = pagemap_scan_get_categories(fd, start) & pagescan_flags; 93 94 if (m == s) 95 return m; 96 97 ksft_exit_fail_msg( 98 "read and ioctl return unmatched results for %s: %d %d", desc, m, s); 99 } 100 return m; 101 } 102 pagemap_is_softdirty(int fd,char * start)103 bool pagemap_is_softdirty(int fd, char *start) 104 { 105 return page_entry_is(fd, start, "soft-dirty", 106 PM_SOFT_DIRTY, PAGE_IS_SOFT_DIRTY); 107 } 108 pagemap_is_swapped(int fd,char * start)109 bool pagemap_is_swapped(int fd, char *start) 110 { 111 return page_entry_is(fd, start, "swap", PM_SWAP, PAGE_IS_SWAPPED); 112 } 113 pagemap_is_populated(int fd,char * start)114 bool pagemap_is_populated(int fd, char *start) 115 { 116 return page_entry_is(fd, start, "populated", 117 PM_PRESENT | PM_SWAP, 118 PAGE_IS_PRESENT | PAGE_IS_SWAPPED); 119 } 120 pagemap_get_pfn(int fd,char * start)121 unsigned long pagemap_get_pfn(int fd, char *start) 122 { 123 uint64_t entry = pagemap_get_entry(fd, start); 124 125 /* If present (63th bit), PFN is at bit 0 -- 54. */ 126 if (entry & PM_PRESENT) 127 return entry & 0x007fffffffffffffull; 128 return -1ul; 129 } 130 clear_softdirty(void)131 void clear_softdirty(void) 132 { 133 int ret; 134 const char *ctrl = "4"; 135 int fd = open("/proc/self/clear_refs", O_WRONLY); 136 137 if (fd < 0) 138 ksft_exit_fail_msg("opening clear_refs failed\n"); 139 ret = write(fd, ctrl, strlen(ctrl)); 140 close(fd); 141 if (ret != strlen(ctrl)) 142 ksft_exit_fail_msg("writing clear_refs failed\n"); 143 } 144 check_for_pattern(FILE * fp,const char * pattern,char * buf,size_t len)145 bool check_for_pattern(FILE *fp, const char *pattern, char *buf, size_t len) 146 { 147 while (fgets(buf, len, fp)) { 148 if (!strncmp(buf, pattern, strlen(pattern))) 149 return true; 150 } 151 return false; 152 } 153 read_pmd_pagesize(void)154 uint64_t read_pmd_pagesize(void) 155 { 156 int fd; 157 char buf[20]; 158 ssize_t num_read; 159 160 fd = open(PMD_SIZE_FILE_PATH, O_RDONLY); 161 if (fd == -1) 162 return 0; 163 164 num_read = read(fd, buf, 19); 165 if (num_read < 1) { 166 close(fd); 167 return 0; 168 } 169 buf[num_read] = '\0'; 170 close(fd); 171 172 return strtoul(buf, NULL, 10); 173 } 174 rss_anon(void)175 unsigned long rss_anon(void) 176 { 177 unsigned long rss_anon = 0; 178 FILE *fp; 179 char buffer[MAX_LINE_LENGTH]; 180 181 fp = fopen(STATUS_FILE_PATH, "r"); 182 if (!fp) 183 ksft_exit_fail_msg("%s: Failed to open file %s\n", __func__, STATUS_FILE_PATH); 184 185 if (!check_for_pattern(fp, "RssAnon:", buffer, sizeof(buffer))) 186 goto err_out; 187 188 if (sscanf(buffer, "RssAnon:%10lu kB", &rss_anon) != 1) 189 ksft_exit_fail_msg("Reading status error\n"); 190 191 err_out: 192 fclose(fp); 193 return rss_anon; 194 } 195 __check_huge(void * addr,char * pattern,int nr_hpages,uint64_t hpage_size)196 bool __check_huge(void *addr, char *pattern, int nr_hpages, 197 uint64_t hpage_size) 198 { 199 uint64_t thp = -1; 200 int ret; 201 FILE *fp; 202 char buffer[MAX_LINE_LENGTH]; 203 char addr_pattern[MAX_LINE_LENGTH]; 204 205 ret = snprintf(addr_pattern, MAX_LINE_LENGTH, "%08lx-", 206 (unsigned long) addr); 207 if (ret >= MAX_LINE_LENGTH) 208 ksft_exit_fail_msg("%s: Pattern is too long\n", __func__); 209 210 fp = fopen(SMAP_FILE_PATH, "r"); 211 if (!fp) 212 ksft_exit_fail_msg("%s: Failed to open file %s\n", __func__, SMAP_FILE_PATH); 213 214 if (!check_for_pattern(fp, addr_pattern, buffer, sizeof(buffer))) 215 goto err_out; 216 217 /* 218 * Fetch the pattern in the same block and check the number of 219 * hugepages. 220 */ 221 if (!check_for_pattern(fp, pattern, buffer, sizeof(buffer))) 222 goto err_out; 223 224 snprintf(addr_pattern, MAX_LINE_LENGTH, "%s%%9ld kB", pattern); 225 226 if (sscanf(buffer, addr_pattern, &thp) != 1) 227 ksft_exit_fail_msg("Reading smap error\n"); 228 229 err_out: 230 fclose(fp); 231 return thp == (nr_hpages * (hpage_size >> 10)); 232 } 233 check_huge_anon(void * addr,int nr_hpages,uint64_t hpage_size)234 bool check_huge_anon(void *addr, int nr_hpages, uint64_t hpage_size) 235 { 236 return __check_huge(addr, "AnonHugePages: ", nr_hpages, hpage_size); 237 } 238 check_huge_file(void * addr,int nr_hpages,uint64_t hpage_size)239 bool check_huge_file(void *addr, int nr_hpages, uint64_t hpage_size) 240 { 241 return __check_huge(addr, "FilePmdMapped:", nr_hpages, hpage_size); 242 } 243 check_huge_shmem(void * addr,int nr_hpages,uint64_t hpage_size)244 bool check_huge_shmem(void *addr, int nr_hpages, uint64_t hpage_size) 245 { 246 return __check_huge(addr, "ShmemPmdMapped:", nr_hpages, hpage_size); 247 } 248 allocate_transhuge(void * ptr,int pagemap_fd)249 int64_t allocate_transhuge(void *ptr, int pagemap_fd) 250 { 251 uint64_t ent[2]; 252 253 /* drop pmd */ 254 if (mmap(ptr, HPAGE_SIZE, PROT_READ | PROT_WRITE, 255 MAP_FIXED | MAP_ANONYMOUS | 256 MAP_NORESERVE | MAP_PRIVATE, -1, 0) != ptr) 257 ksft_exit_fail_msg("mmap transhuge\n"); 258 259 if (madvise(ptr, HPAGE_SIZE, MADV_HUGEPAGE)) 260 ksft_exit_fail_msg("MADV_HUGEPAGE\n"); 261 262 /* allocate transparent huge page */ 263 *(volatile void **)ptr = ptr; 264 265 if (pread(pagemap_fd, ent, sizeof(ent), 266 (uintptr_t)ptr >> (pshift() - 3)) != sizeof(ent)) 267 ksft_exit_fail_msg("read pagemap\n"); 268 269 if (PAGEMAP_PRESENT(ent[0]) && PAGEMAP_PRESENT(ent[1]) && 270 PAGEMAP_PFN(ent[0]) + 1 == PAGEMAP_PFN(ent[1]) && 271 !(PAGEMAP_PFN(ent[0]) & ((1 << (HPAGE_SHIFT - pshift())) - 1))) 272 return PAGEMAP_PFN(ent[0]); 273 274 return -1; 275 } 276 default_huge_page_size(void)277 unsigned long default_huge_page_size(void) 278 { 279 unsigned long hps = 0; 280 char *line = NULL; 281 size_t linelen = 0; 282 FILE *f = fopen("/proc/meminfo", "r"); 283 284 if (!f) 285 return 0; 286 while (getline(&line, &linelen, f) > 0) { 287 if (sscanf(line, "Hugepagesize: %lu kB", &hps) == 1) { 288 hps <<= 10; 289 break; 290 } 291 } 292 293 free(line); 294 fclose(f); 295 return hps; 296 } 297 detect_hugetlb_page_sizes(size_t sizes[],int max)298 int detect_hugetlb_page_sizes(size_t sizes[], int max) 299 { 300 DIR *dir = opendir("/sys/kernel/mm/hugepages/"); 301 int count = 0; 302 303 if (!dir) 304 return 0; 305 306 while (count < max) { 307 struct dirent *entry = readdir(dir); 308 size_t kb; 309 310 if (!entry) 311 break; 312 if (entry->d_type != DT_DIR) 313 continue; 314 if (sscanf(entry->d_name, "hugepages-%zukB", &kb) != 1) 315 continue; 316 sizes[count++] = kb * 1024; 317 ksft_print_msg("[INFO] detected hugetlb page size: %zu KiB\n", 318 kb); 319 } 320 closedir(dir); 321 return count; 322 } 323 324 /* If `ioctls' non-NULL, the allowed ioctls will be returned into the var */ uffd_register_with_ioctls(int uffd,void * addr,uint64_t len,bool miss,bool wp,bool minor,uint64_t * ioctls)325 int uffd_register_with_ioctls(int uffd, void *addr, uint64_t len, 326 bool miss, bool wp, bool minor, uint64_t *ioctls) 327 { 328 struct uffdio_register uffdio_register = { 0 }; 329 uint64_t mode = 0; 330 int ret = 0; 331 332 if (miss) 333 mode |= UFFDIO_REGISTER_MODE_MISSING; 334 if (wp) 335 mode |= UFFDIO_REGISTER_MODE_WP; 336 if (minor) 337 mode |= UFFDIO_REGISTER_MODE_MINOR; 338 339 uffdio_register.range.start = (unsigned long)addr; 340 uffdio_register.range.len = len; 341 uffdio_register.mode = mode; 342 343 if (ioctl(uffd, UFFDIO_REGISTER, &uffdio_register) == -1) 344 ret = -errno; 345 else if (ioctls) 346 *ioctls = uffdio_register.ioctls; 347 348 return ret; 349 } 350 uffd_register(int uffd,void * addr,uint64_t len,bool miss,bool wp,bool minor)351 int uffd_register(int uffd, void *addr, uint64_t len, 352 bool miss, bool wp, bool minor) 353 { 354 return uffd_register_with_ioctls(uffd, addr, len, 355 miss, wp, minor, NULL); 356 } 357 uffd_unregister(int uffd,void * addr,uint64_t len)358 int uffd_unregister(int uffd, void *addr, uint64_t len) 359 { 360 struct uffdio_range range = { .start = (uintptr_t)addr, .len = len }; 361 int ret = 0; 362 363 if (ioctl(uffd, UFFDIO_UNREGISTER, &range) == -1) 364 ret = -errno; 365 366 return ret; 367 } 368 get_free_hugepages(void)369 unsigned long get_free_hugepages(void) 370 { 371 unsigned long fhp = 0; 372 char *line = NULL; 373 size_t linelen = 0; 374 FILE *f = fopen("/proc/meminfo", "r"); 375 376 if (!f) 377 return fhp; 378 while (getline(&line, &linelen, f) > 0) { 379 if (sscanf(line, "HugePages_Free: %lu", &fhp) == 1) 380 break; 381 } 382 383 free(line); 384 fclose(f); 385 return fhp; 386 } 387