1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Test cases for SL[AOU]B/page initialization at alloc/free time. 4 */ 5 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt 6 7 #include <linux/init.h> 8 #include <linux/kernel.h> 9 #include <linux/mm.h> 10 #include <linux/module.h> 11 #include <linux/slab.h> 12 #include <linux/string.h> 13 #include <linux/vmalloc.h> 14 15 #define GARBAGE_INT (0x09A7BA9E) 16 #define GARBAGE_BYTE (0x9E) 17 18 #define REPORT_FAILURES_IN_FN() \ 19 do { \ 20 if (failures) \ 21 pr_info("%s failed %d out of %d times\n", \ 22 __func__, failures, num_tests); \ 23 else \ 24 pr_info("all %d tests in %s passed\n", \ 25 num_tests, __func__); \ 26 } while (0) 27 28 /* Calculate the number of uninitialized bytes in the buffer. */ count_nonzero_bytes(void * ptr,size_t size)29 static int __init count_nonzero_bytes(void *ptr, size_t size) 30 { 31 int i, ret = 0; 32 unsigned char *p = (unsigned char *)ptr; 33 34 for (i = 0; i < size; i++) 35 if (p[i]) 36 ret++; 37 return ret; 38 } 39 40 /* Fill a buffer with garbage, skipping |skip| first bytes. */ fill_with_garbage_skip(void * ptr,int size,size_t skip)41 static void __init fill_with_garbage_skip(void *ptr, int size, size_t skip) 42 { 43 unsigned int *p = (unsigned int *)((char *)ptr + skip); 44 int i = 0; 45 46 WARN_ON(skip > size); 47 size -= skip; 48 49 while (size >= sizeof(*p)) { 50 p[i] = GARBAGE_INT; 51 i++; 52 size -= sizeof(*p); 53 } 54 if (size) 55 memset(&p[i], GARBAGE_BYTE, size); 56 } 57 fill_with_garbage(void * ptr,size_t size)58 static void __init fill_with_garbage(void *ptr, size_t size) 59 { 60 fill_with_garbage_skip(ptr, size, 0); 61 } 62 do_alloc_pages_order(int order,int * total_failures)63 static int __init do_alloc_pages_order(int order, int *total_failures) 64 { 65 struct page *page; 66 void *buf; 67 size_t size = PAGE_SIZE << order; 68 69 page = alloc_pages(GFP_KERNEL, order); 70 if (!page) 71 goto err; 72 buf = page_address(page); 73 fill_with_garbage(buf, size); 74 __free_pages(page, order); 75 76 page = alloc_pages(GFP_KERNEL, order); 77 if (!page) 78 goto err; 79 buf = page_address(page); 80 if (count_nonzero_bytes(buf, size)) 81 (*total_failures)++; 82 fill_with_garbage(buf, size); 83 __free_pages(page, order); 84 return 1; 85 err: 86 (*total_failures)++; 87 return 1; 88 } 89 90 /* Test the page allocator by calling alloc_pages with different orders. */ test_pages(int * total_failures)91 static int __init test_pages(int *total_failures) 92 { 93 int failures = 0, num_tests = 0; 94 int i; 95 96 for (i = 0; i < NR_PAGE_ORDERS; i++) 97 num_tests += do_alloc_pages_order(i, &failures); 98 99 REPORT_FAILURES_IN_FN(); 100 *total_failures += failures; 101 return num_tests; 102 } 103 104 /* Test kmalloc() with given parameters. */ do_kmalloc_size(size_t size,int * total_failures)105 static int __init do_kmalloc_size(size_t size, int *total_failures) 106 { 107 void *buf; 108 109 buf = kmalloc(size, GFP_KERNEL); 110 if (!buf) 111 goto err; 112 fill_with_garbage(buf, size); 113 kfree(buf); 114 115 buf = kmalloc(size, GFP_KERNEL); 116 if (!buf) 117 goto err; 118 if (count_nonzero_bytes(buf, size)) 119 (*total_failures)++; 120 fill_with_garbage(buf, size); 121 kfree(buf); 122 return 1; 123 err: 124 (*total_failures)++; 125 return 1; 126 } 127 128 /* Test vmalloc() with given parameters. */ do_vmalloc_size(size_t size,int * total_failures)129 static int __init do_vmalloc_size(size_t size, int *total_failures) 130 { 131 void *buf; 132 133 buf = vmalloc(size); 134 if (!buf) 135 goto err; 136 fill_with_garbage(buf, size); 137 vfree(buf); 138 139 buf = vmalloc(size); 140 if (!buf) 141 goto err; 142 if (count_nonzero_bytes(buf, size)) 143 (*total_failures)++; 144 fill_with_garbage(buf, size); 145 vfree(buf); 146 return 1; 147 err: 148 (*total_failures)++; 149 return 1; 150 } 151 152 /* Test kmalloc()/vmalloc() by allocating objects of different sizes. */ test_kvmalloc(int * total_failures)153 static int __init test_kvmalloc(int *total_failures) 154 { 155 int failures = 0, num_tests = 0; 156 int i, size; 157 158 for (i = 0; i < 20; i++) { 159 size = 1 << i; 160 num_tests += do_kmalloc_size(size, &failures); 161 num_tests += do_vmalloc_size(size, &failures); 162 } 163 164 REPORT_FAILURES_IN_FN(); 165 *total_failures += failures; 166 return num_tests; 167 } 168 169 #define CTOR_BYTES (sizeof(unsigned int)) 170 #define CTOR_PATTERN (0x41414141) 171 /* Initialize the first 4 bytes of the object. */ test_ctor(void * obj)172 static void test_ctor(void *obj) 173 { 174 *(unsigned int *)obj = CTOR_PATTERN; 175 } 176 177 /* 178 * Check the invariants for the buffer allocated from a slab cache. 179 * If the cache has a test constructor, the first 4 bytes of the object must 180 * always remain equal to CTOR_PATTERN. 181 * If the cache isn't an RCU-typesafe one, or if the allocation is done with 182 * __GFP_ZERO, then the object contents must be zeroed after allocation. 183 * If the cache is an RCU-typesafe one, the object contents must never be 184 * zeroed after the first use. This is checked by memcmp() in 185 * do_kmem_cache_size(). 186 */ check_buf(void * buf,int size,bool want_ctor,bool want_rcu,bool want_zero)187 static bool __init check_buf(void *buf, int size, bool want_ctor, 188 bool want_rcu, bool want_zero) 189 { 190 int bytes; 191 bool fail = false; 192 193 bytes = count_nonzero_bytes(buf, size); 194 WARN_ON(want_ctor && want_zero); 195 if (want_zero) 196 return bytes; 197 if (want_ctor) { 198 if (*(unsigned int *)buf != CTOR_PATTERN) 199 fail = 1; 200 } else { 201 if (bytes) 202 fail = !want_rcu; 203 } 204 return fail; 205 } 206 207 #define BULK_SIZE 100 208 static void *bulk_array[BULK_SIZE]; 209 210 /* 211 * Test kmem_cache with given parameters: 212 * want_ctor - use a constructor; 213 * want_rcu - use SLAB_TYPESAFE_BY_RCU; 214 * want_zero - use __GFP_ZERO. 215 */ do_kmem_cache_size(size_t size,bool want_ctor,bool want_rcu,bool want_zero,int * total_failures)216 static int __init do_kmem_cache_size(size_t size, bool want_ctor, 217 bool want_rcu, bool want_zero, 218 int *total_failures) 219 { 220 struct kmem_cache *c; 221 int iter; 222 bool fail = false; 223 gfp_t alloc_mask = GFP_KERNEL | (want_zero ? __GFP_ZERO : 0); 224 void *buf, *buf_copy; 225 226 c = kmem_cache_create("test_cache", size, 1, 227 want_rcu ? SLAB_TYPESAFE_BY_RCU : 0, 228 want_ctor ? test_ctor : NULL); 229 for (iter = 0; iter < 10; iter++) { 230 /* Do a test of bulk allocations */ 231 if (!want_rcu && !want_ctor) { 232 int ret; 233 234 ret = kmem_cache_alloc_bulk(c, alloc_mask, BULK_SIZE, bulk_array); 235 if (!ret) { 236 fail = true; 237 } else { 238 int i; 239 for (i = 0; i < ret; i++) 240 fail |= check_buf(bulk_array[i], size, want_ctor, want_rcu, want_zero); 241 kmem_cache_free_bulk(c, ret, bulk_array); 242 } 243 } 244 245 buf = kmem_cache_alloc(c, alloc_mask); 246 /* Check that buf is zeroed, if it must be. */ 247 fail |= check_buf(buf, size, want_ctor, want_rcu, want_zero); 248 fill_with_garbage_skip(buf, size, want_ctor ? CTOR_BYTES : 0); 249 250 if (!want_rcu) { 251 kmem_cache_free(c, buf); 252 continue; 253 } 254 255 /* 256 * If this is an RCU cache, use a critical section to ensure we 257 * can touch objects after they're freed. 258 */ 259 rcu_read_lock(); 260 /* 261 * Copy the buffer to check that it's not wiped on 262 * free(). 263 */ 264 buf_copy = kmalloc(size, GFP_ATOMIC); 265 if (buf_copy) 266 memcpy(buf_copy, buf, size); 267 268 kmem_cache_free(c, buf); 269 /* 270 * Check that |buf| is intact after kmem_cache_free(). 271 * |want_zero| is false, because we wrote garbage to 272 * the buffer already. 273 */ 274 fail |= check_buf(buf, size, want_ctor, want_rcu, 275 false); 276 if (buf_copy) { 277 fail |= (bool)memcmp(buf, buf_copy, size); 278 kfree(buf_copy); 279 } 280 rcu_read_unlock(); 281 } 282 kmem_cache_destroy(c); 283 284 *total_failures += fail; 285 return 1; 286 } 287 288 /* 289 * Check that the data written to an RCU-allocated object survives 290 * reallocation. 291 */ do_kmem_cache_rcu_persistent(int size,int * total_failures)292 static int __init do_kmem_cache_rcu_persistent(int size, int *total_failures) 293 { 294 struct kmem_cache *c; 295 void *buf, *buf_contents, *saved_ptr; 296 void **used_objects; 297 int i, iter, maxiter = 1024; 298 bool fail = false; 299 300 c = kmem_cache_create("test_cache", size, size, SLAB_TYPESAFE_BY_RCU, 301 NULL); 302 buf = kmem_cache_alloc(c, GFP_KERNEL); 303 if (!buf) 304 goto out; 305 saved_ptr = buf; 306 fill_with_garbage(buf, size); 307 buf_contents = kmalloc(size, GFP_KERNEL); 308 if (!buf_contents) { 309 kmem_cache_free(c, buf); 310 goto out; 311 } 312 used_objects = kmalloc_array(maxiter, sizeof(void *), GFP_KERNEL); 313 if (!used_objects) { 314 kmem_cache_free(c, buf); 315 kfree(buf_contents); 316 goto out; 317 } 318 memcpy(buf_contents, buf, size); 319 kmem_cache_free(c, buf); 320 /* 321 * Run for a fixed number of iterations. If we never hit saved_ptr, 322 * assume the test passes. 323 */ 324 for (iter = 0; iter < maxiter; iter++) { 325 buf = kmem_cache_alloc(c, GFP_KERNEL); 326 used_objects[iter] = buf; 327 if (buf == saved_ptr) { 328 fail = memcmp(buf_contents, buf, size); 329 for (i = 0; i <= iter; i++) 330 kmem_cache_free(c, used_objects[i]); 331 goto free_out; 332 } 333 } 334 335 for (iter = 0; iter < maxiter; iter++) 336 kmem_cache_free(c, used_objects[iter]); 337 338 free_out: 339 kfree(buf_contents); 340 kfree(used_objects); 341 out: 342 kmem_cache_destroy(c); 343 *total_failures += fail; 344 return 1; 345 } 346 do_kmem_cache_size_bulk(int size,int * total_failures)347 static int __init do_kmem_cache_size_bulk(int size, int *total_failures) 348 { 349 struct kmem_cache *c; 350 int i, iter, maxiter = 1024; 351 int num, bytes; 352 bool fail = false; 353 void *objects[10]; 354 355 c = kmem_cache_create("test_cache", size, size, 0, NULL); 356 for (iter = 0; (iter < maxiter) && !fail; iter++) { 357 num = kmem_cache_alloc_bulk(c, GFP_KERNEL, ARRAY_SIZE(objects), 358 objects); 359 for (i = 0; i < num; i++) { 360 bytes = count_nonzero_bytes(objects[i], size); 361 if (bytes) 362 fail = true; 363 fill_with_garbage(objects[i], size); 364 } 365 366 if (num) 367 kmem_cache_free_bulk(c, num, objects); 368 } 369 kmem_cache_destroy(c); 370 *total_failures += fail; 371 return 1; 372 } 373 374 /* 375 * Test kmem_cache allocation by creating caches of different sizes, with and 376 * without constructors, with and without SLAB_TYPESAFE_BY_RCU. 377 */ test_kmemcache(int * total_failures)378 static int __init test_kmemcache(int *total_failures) 379 { 380 int failures = 0, num_tests = 0; 381 int i, flags, size; 382 bool ctor, rcu, zero; 383 384 for (i = 0; i < 10; i++) { 385 size = 8 << i; 386 for (flags = 0; flags < 8; flags++) { 387 ctor = flags & 1; 388 rcu = flags & 2; 389 zero = flags & 4; 390 if (ctor & zero) 391 continue; 392 num_tests += do_kmem_cache_size(size, ctor, rcu, zero, 393 &failures); 394 } 395 num_tests += do_kmem_cache_size_bulk(size, &failures); 396 } 397 REPORT_FAILURES_IN_FN(); 398 *total_failures += failures; 399 return num_tests; 400 } 401 402 /* Test the behavior of SLAB_TYPESAFE_BY_RCU caches of different sizes. */ test_rcu_persistent(int * total_failures)403 static int __init test_rcu_persistent(int *total_failures) 404 { 405 int failures = 0, num_tests = 0; 406 int i, size; 407 408 for (i = 0; i < 10; i++) { 409 size = 8 << i; 410 num_tests += do_kmem_cache_rcu_persistent(size, &failures); 411 } 412 REPORT_FAILURES_IN_FN(); 413 *total_failures += failures; 414 return num_tests; 415 } 416 417 /* 418 * Run the tests. Each test function returns the number of executed tests and 419 * updates |failures| with the number of failed tests. 420 */ test_meminit_init(void)421 static int __init test_meminit_init(void) 422 { 423 int failures = 0, num_tests = 0; 424 425 num_tests += test_pages(&failures); 426 num_tests += test_kvmalloc(&failures); 427 num_tests += test_kmemcache(&failures); 428 num_tests += test_rcu_persistent(&failures); 429 430 if (failures == 0) 431 pr_info("all %d tests passed!\n", num_tests); 432 else 433 pr_info("failures: %d out of %d\n", failures, num_tests); 434 435 return failures ? -EINVAL : 0; 436 } 437 module_init(test_meminit_init); 438 439 MODULE_DESCRIPTION("Test cases for SL[AOU]B/page initialization at alloc/free time"); 440 MODULE_LICENSE("GPL"); 441