1 // SPDX-License-Identifier: GPL-2.0+ 2 /* 3 * Common functions for in-kernel torture tests. 4 * 5 * Copyright (C) IBM Corporation, 2014 6 * 7 * Author: Paul E. McKenney <paulmck@linux.ibm.com> 8 * Based on kernel/rcu/torture.c. 9 */ 10 11 #define pr_fmt(fmt) fmt 12 13 #include <linux/types.h> 14 #include <linux/kernel.h> 15 #include <linux/init.h> 16 #include <linux/module.h> 17 #include <linux/kthread.h> 18 #include <linux/err.h> 19 #include <linux/spinlock.h> 20 #include <linux/smp.h> 21 #include <linux/interrupt.h> 22 #include <linux/sched.h> 23 #include <linux/sched/clock.h> 24 #include <linux/atomic.h> 25 #include <linux/bitops.h> 26 #include <linux/completion.h> 27 #include <linux/moduleparam.h> 28 #include <linux/percpu.h> 29 #include <linux/notifier.h> 30 #include <linux/reboot.h> 31 #include <linux/freezer.h> 32 #include <linux/cpu.h> 33 #include <linux/delay.h> 34 #include <linux/stat.h> 35 #include <linux/slab.h> 36 #include <linux/trace_clock.h> 37 #include <linux/ktime.h> 38 #include <asm/byteorder.h> 39 #include <linux/torture.h> 40 #include <linux/sched/rt.h> 41 #include "rcu/rcu.h" 42 43 MODULE_DESCRIPTION("Common functions for in-kernel torture tests"); 44 MODULE_LICENSE("GPL"); 45 MODULE_AUTHOR("Paul E. McKenney <paulmck@linux.ibm.com>"); 46 47 static bool disable_onoff_at_boot; 48 module_param(disable_onoff_at_boot, bool, 0444); 49 50 static bool ftrace_dump_at_shutdown; 51 module_param(ftrace_dump_at_shutdown, bool, 0444); 52 53 static int verbose_sleep_frequency; 54 module_param(verbose_sleep_frequency, int, 0444); 55 56 static int verbose_sleep_duration = 1; 57 module_param(verbose_sleep_duration, int, 0444); 58 59 static int random_shuffle; 60 module_param(random_shuffle, int, 0444); 61 62 static char *torture_type; 63 static int verbose; 64 65 /* Mediate rmmod and system shutdown. Concurrent rmmod & shutdown illegal! */ 66 #define FULLSTOP_DONTSTOP 0 /* Normal operation. */ 67 #define FULLSTOP_SHUTDOWN 1 /* System shutdown with torture running. */ 68 #define FULLSTOP_RMMOD 2 /* Normal rmmod of torture. */ 69 static int fullstop = FULLSTOP_RMMOD; 70 static DEFINE_MUTEX(fullstop_mutex); 71 72 static atomic_t verbose_sleep_counter; 73 74 /* 75 * Sleep if needed from VERBOSE_TOROUT*(). 76 */ verbose_torout_sleep(void)77 void verbose_torout_sleep(void) 78 { 79 if (verbose_sleep_frequency > 0 && 80 verbose_sleep_duration > 0 && 81 !(atomic_inc_return(&verbose_sleep_counter) % verbose_sleep_frequency)) 82 schedule_timeout_uninterruptible(verbose_sleep_duration); 83 } 84 EXPORT_SYMBOL_GPL(verbose_torout_sleep); 85 86 /* 87 * Schedule a high-resolution-timer sleep in nanoseconds, with a 32-bit 88 * nanosecond random fuzz. This function and its friends desynchronize 89 * testing from the timer wheel. 90 */ torture_hrtimeout_ns(ktime_t baset_ns,u32 fuzzt_ns,const enum hrtimer_mode mode,struct torture_random_state * trsp)91 int torture_hrtimeout_ns(ktime_t baset_ns, u32 fuzzt_ns, const enum hrtimer_mode mode, 92 struct torture_random_state *trsp) 93 { 94 ktime_t hto = baset_ns; 95 96 if (trsp) 97 hto += torture_random(trsp) % fuzzt_ns; 98 set_current_state(TASK_IDLE); 99 return schedule_hrtimeout(&hto, mode); 100 } 101 EXPORT_SYMBOL_GPL(torture_hrtimeout_ns); 102 103 /* 104 * Schedule a high-resolution-timer sleep in microseconds, with a 32-bit 105 * nanosecond (not microsecond!) random fuzz. 106 */ torture_hrtimeout_us(u32 baset_us,u32 fuzzt_ns,struct torture_random_state * trsp)107 int torture_hrtimeout_us(u32 baset_us, u32 fuzzt_ns, struct torture_random_state *trsp) 108 { 109 ktime_t baset_ns = baset_us * NSEC_PER_USEC; 110 111 return torture_hrtimeout_ns(baset_ns, fuzzt_ns, HRTIMER_MODE_REL, trsp); 112 } 113 EXPORT_SYMBOL_GPL(torture_hrtimeout_us); 114 115 /* 116 * Schedule a high-resolution-timer sleep in milliseconds, with a 32-bit 117 * microsecond (not millisecond!) random fuzz. 118 */ torture_hrtimeout_ms(u32 baset_ms,u32 fuzzt_us,struct torture_random_state * trsp)119 int torture_hrtimeout_ms(u32 baset_ms, u32 fuzzt_us, struct torture_random_state *trsp) 120 { 121 ktime_t baset_ns = baset_ms * NSEC_PER_MSEC; 122 u32 fuzzt_ns; 123 124 if ((u32)~0U / NSEC_PER_USEC < fuzzt_us) 125 fuzzt_ns = (u32)~0U; 126 else 127 fuzzt_ns = fuzzt_us * NSEC_PER_USEC; 128 return torture_hrtimeout_ns(baset_ns, fuzzt_ns, HRTIMER_MODE_REL, trsp); 129 } 130 EXPORT_SYMBOL_GPL(torture_hrtimeout_ms); 131 132 /* 133 * Schedule a high-resolution-timer sleep in jiffies, with an 134 * implied one-jiffy random fuzz. This is intended to replace calls to 135 * schedule_timeout_interruptible() and friends. 136 */ torture_hrtimeout_jiffies(u32 baset_j,struct torture_random_state * trsp)137 int torture_hrtimeout_jiffies(u32 baset_j, struct torture_random_state *trsp) 138 { 139 ktime_t baset_ns = jiffies_to_nsecs(baset_j); 140 141 return torture_hrtimeout_ns(baset_ns, jiffies_to_nsecs(1), HRTIMER_MODE_REL, trsp); 142 } 143 EXPORT_SYMBOL_GPL(torture_hrtimeout_jiffies); 144 145 /* 146 * Schedule a high-resolution-timer sleep in milliseconds, with a 32-bit 147 * millisecond (not second!) random fuzz. 148 */ torture_hrtimeout_s(u32 baset_s,u32 fuzzt_ms,struct torture_random_state * trsp)149 int torture_hrtimeout_s(u32 baset_s, u32 fuzzt_ms, struct torture_random_state *trsp) 150 { 151 ktime_t baset_ns = baset_s * NSEC_PER_SEC; 152 u32 fuzzt_ns; 153 154 if ((u32)~0U / NSEC_PER_MSEC < fuzzt_ms) 155 fuzzt_ns = (u32)~0U; 156 else 157 fuzzt_ns = fuzzt_ms * NSEC_PER_MSEC; 158 return torture_hrtimeout_ns(baset_ns, fuzzt_ns, HRTIMER_MODE_REL, trsp); 159 } 160 EXPORT_SYMBOL_GPL(torture_hrtimeout_s); 161 162 #ifdef CONFIG_HOTPLUG_CPU 163 164 /* 165 * Variables for online-offline handling. Only present if CPU hotplug 166 * is enabled, otherwise does nothing. 167 */ 168 169 static struct task_struct *onoff_task; 170 static long onoff_holdoff; 171 static long onoff_interval; 172 static torture_ofl_func *onoff_f; 173 static long n_offline_attempts; 174 static long n_offline_successes; 175 static unsigned long sum_offline; 176 static int min_offline = -1; 177 static int max_offline; 178 static long n_online_attempts; 179 static long n_online_successes; 180 static unsigned long sum_online; 181 static int min_online = -1; 182 static int max_online; 183 184 static int torture_online_cpus = NR_CPUS; 185 186 /* 187 * Some torture testing leverages confusion as to the number of online 188 * CPUs. This function returns the torture-testing view of this number, 189 * which allows torture tests to load-balance appropriately. 190 */ torture_num_online_cpus(void)191 int torture_num_online_cpus(void) 192 { 193 return READ_ONCE(torture_online_cpus); 194 } 195 EXPORT_SYMBOL_GPL(torture_num_online_cpus); 196 197 /* 198 * Attempt to take a CPU offline. Return false if the CPU is already 199 * offline or if it is not subject to CPU-hotplug operations. The 200 * caller can detect other failures by looking at the statistics. 201 */ torture_offline(int cpu,long * n_offl_attempts,long * n_offl_successes,unsigned long * sum_offl,int * min_offl,int * max_offl)202 bool torture_offline(int cpu, long *n_offl_attempts, long *n_offl_successes, 203 unsigned long *sum_offl, int *min_offl, int *max_offl) 204 { 205 unsigned long delta; 206 int ret; 207 char *s; 208 unsigned long starttime; 209 210 if (!cpu_online(cpu) || !cpu_is_hotpluggable(cpu)) 211 return false; 212 if (num_online_cpus() <= 1) 213 return false; /* Can't offline the last CPU. */ 214 215 if (verbose > 1) 216 pr_alert("%s" TORTURE_FLAG 217 "torture_onoff task: offlining %d\n", 218 torture_type, cpu); 219 starttime = jiffies; 220 (*n_offl_attempts)++; 221 ret = remove_cpu(cpu); 222 if (ret) { 223 s = ""; 224 if (!rcu_inkernel_boot_has_ended() && ret == -EBUSY) { 225 // PCI probe frequently disables hotplug during boot. 226 (*n_offl_attempts)--; 227 s = " (-EBUSY forgiven during boot)"; 228 } 229 if (verbose) 230 pr_alert("%s" TORTURE_FLAG 231 "torture_onoff task: offline %d failed%s: errno %d\n", 232 torture_type, cpu, s, ret); 233 } else { 234 if (verbose > 1) 235 pr_alert("%s" TORTURE_FLAG 236 "torture_onoff task: offlined %d\n", 237 torture_type, cpu); 238 if (onoff_f) 239 onoff_f(); 240 (*n_offl_successes)++; 241 delta = jiffies - starttime; 242 *sum_offl += delta; 243 if (*min_offl < 0) { 244 *min_offl = delta; 245 *max_offl = delta; 246 } 247 if (*min_offl > delta) 248 *min_offl = delta; 249 if (*max_offl < delta) 250 *max_offl = delta; 251 WRITE_ONCE(torture_online_cpus, torture_online_cpus - 1); 252 WARN_ON_ONCE(torture_online_cpus <= 0); 253 } 254 255 return true; 256 } 257 EXPORT_SYMBOL_GPL(torture_offline); 258 259 /* 260 * Attempt to bring a CPU online. Return false if the CPU is already 261 * online or if it is not subject to CPU-hotplug operations. The 262 * caller can detect other failures by looking at the statistics. 263 */ torture_online(int cpu,long * n_onl_attempts,long * n_onl_successes,unsigned long * sum_onl,int * min_onl,int * max_onl)264 bool torture_online(int cpu, long *n_onl_attempts, long *n_onl_successes, 265 unsigned long *sum_onl, int *min_onl, int *max_onl) 266 { 267 unsigned long delta; 268 int ret; 269 char *s; 270 unsigned long starttime; 271 272 if (cpu_online(cpu) || !cpu_is_hotpluggable(cpu)) 273 return false; 274 275 if (verbose > 1) 276 pr_alert("%s" TORTURE_FLAG 277 "torture_onoff task: onlining %d\n", 278 torture_type, cpu); 279 starttime = jiffies; 280 (*n_onl_attempts)++; 281 ret = add_cpu(cpu); 282 if (ret) { 283 s = ""; 284 if (!rcu_inkernel_boot_has_ended() && ret == -EBUSY) { 285 // PCI probe frequently disables hotplug during boot. 286 (*n_onl_attempts)--; 287 s = " (-EBUSY forgiven during boot)"; 288 } 289 if (verbose) 290 pr_alert("%s" TORTURE_FLAG 291 "torture_onoff task: online %d failed%s: errno %d\n", 292 torture_type, cpu, s, ret); 293 } else { 294 if (verbose > 1) 295 pr_alert("%s" TORTURE_FLAG 296 "torture_onoff task: onlined %d\n", 297 torture_type, cpu); 298 (*n_onl_successes)++; 299 delta = jiffies - starttime; 300 *sum_onl += delta; 301 if (*min_onl < 0) { 302 *min_onl = delta; 303 *max_onl = delta; 304 } 305 if (*min_onl > delta) 306 *min_onl = delta; 307 if (*max_onl < delta) 308 *max_onl = delta; 309 WRITE_ONCE(torture_online_cpus, torture_online_cpus + 1); 310 } 311 312 return true; 313 } 314 EXPORT_SYMBOL_GPL(torture_online); 315 316 /* 317 * Get everything online at the beginning and ends of tests. 318 */ torture_online_all(char * phase)319 static void torture_online_all(char *phase) 320 { 321 int cpu; 322 int ret; 323 324 for_each_possible_cpu(cpu) { 325 if (cpu_online(cpu)) 326 continue; 327 ret = add_cpu(cpu); 328 if (ret && verbose) { 329 pr_alert("%s" TORTURE_FLAG 330 "%s: %s online %d: errno %d\n", 331 __func__, phase, torture_type, cpu, ret); 332 } 333 } 334 } 335 336 /* 337 * Execute random CPU-hotplug operations at the interval specified 338 * by the onoff_interval. 339 */ 340 static int torture_onoff(void * arg)341 torture_onoff(void *arg) 342 { 343 int cpu; 344 int maxcpu = -1; 345 DEFINE_TORTURE_RANDOM(rand); 346 347 VERBOSE_TOROUT_STRING("torture_onoff task started"); 348 for_each_online_cpu(cpu) 349 maxcpu = cpu; 350 WARN_ON(maxcpu < 0); 351 torture_online_all("Initial"); 352 if (maxcpu == 0) { 353 VERBOSE_TOROUT_STRING("Only one CPU, so CPU-hotplug testing is disabled"); 354 goto stop; 355 } 356 357 if (onoff_holdoff > 0) { 358 VERBOSE_TOROUT_STRING("torture_onoff begin holdoff"); 359 torture_hrtimeout_jiffies(onoff_holdoff, &rand); 360 VERBOSE_TOROUT_STRING("torture_onoff end holdoff"); 361 } 362 while (!torture_must_stop()) { 363 if (disable_onoff_at_boot && !rcu_inkernel_boot_has_ended()) { 364 torture_hrtimeout_jiffies(HZ / 10, &rand); 365 continue; 366 } 367 cpu = torture_random(&rand) % (maxcpu + 1); 368 if (!torture_offline(cpu, 369 &n_offline_attempts, &n_offline_successes, 370 &sum_offline, &min_offline, &max_offline)) 371 torture_online(cpu, 372 &n_online_attempts, &n_online_successes, 373 &sum_online, &min_online, &max_online); 374 torture_hrtimeout_jiffies(onoff_interval, &rand); 375 } 376 377 stop: 378 torture_kthread_stopping("torture_onoff"); 379 torture_online_all("Final"); 380 return 0; 381 } 382 383 #endif /* #ifdef CONFIG_HOTPLUG_CPU */ 384 385 /* 386 * Initiate online-offline handling. 387 */ torture_onoff_init(long ooholdoff,long oointerval,torture_ofl_func * f)388 int torture_onoff_init(long ooholdoff, long oointerval, torture_ofl_func *f) 389 { 390 #ifdef CONFIG_HOTPLUG_CPU 391 onoff_holdoff = ooholdoff; 392 onoff_interval = oointerval; 393 onoff_f = f; 394 if (onoff_interval <= 0) 395 return 0; 396 return torture_create_kthread(torture_onoff, NULL, onoff_task); 397 #else /* #ifdef CONFIG_HOTPLUG_CPU */ 398 return 0; 399 #endif /* #else #ifdef CONFIG_HOTPLUG_CPU */ 400 } 401 EXPORT_SYMBOL_GPL(torture_onoff_init); 402 403 /* 404 * Clean up after online/offline testing. 405 */ torture_onoff_cleanup(void)406 static void torture_onoff_cleanup(void) 407 { 408 #ifdef CONFIG_HOTPLUG_CPU 409 if (onoff_task == NULL) 410 return; 411 VERBOSE_TOROUT_STRING("Stopping torture_onoff task"); 412 kthread_stop(onoff_task); 413 onoff_task = NULL; 414 #endif /* #ifdef CONFIG_HOTPLUG_CPU */ 415 } 416 417 /* 418 * Print online/offline testing statistics. 419 */ torture_onoff_stats(void)420 void torture_onoff_stats(void) 421 { 422 #ifdef CONFIG_HOTPLUG_CPU 423 pr_cont("onoff: %ld/%ld:%ld/%ld %d,%d:%d,%d %lu:%lu (HZ=%d) ", 424 n_online_successes, n_online_attempts, 425 n_offline_successes, n_offline_attempts, 426 min_online, max_online, 427 min_offline, max_offline, 428 sum_online, sum_offline, HZ); 429 #endif /* #ifdef CONFIG_HOTPLUG_CPU */ 430 } 431 EXPORT_SYMBOL_GPL(torture_onoff_stats); 432 433 /* 434 * Were all the online/offline operations successful? 435 */ torture_onoff_failures(void)436 bool torture_onoff_failures(void) 437 { 438 #ifdef CONFIG_HOTPLUG_CPU 439 return n_online_successes != n_online_attempts || 440 n_offline_successes != n_offline_attempts; 441 #else /* #ifdef CONFIG_HOTPLUG_CPU */ 442 return false; 443 #endif /* #else #ifdef CONFIG_HOTPLUG_CPU */ 444 } 445 EXPORT_SYMBOL_GPL(torture_onoff_failures); 446 447 #define TORTURE_RANDOM_MULT 39916801 /* prime */ 448 #define TORTURE_RANDOM_ADD 479001701 /* prime */ 449 #define TORTURE_RANDOM_REFRESH 10000 450 451 /* 452 * Crude but fast random-number generator. Uses a linear congruential 453 * generator, with occasional help from cpu_clock(). 454 */ 455 unsigned long torture_random(struct torture_random_state * trsp)456 torture_random(struct torture_random_state *trsp) 457 { 458 if (--trsp->trs_count < 0) { 459 trsp->trs_state += (unsigned long)local_clock() + raw_smp_processor_id(); 460 trsp->trs_count = TORTURE_RANDOM_REFRESH; 461 } 462 trsp->trs_state = trsp->trs_state * TORTURE_RANDOM_MULT + 463 TORTURE_RANDOM_ADD; 464 return swahw32(trsp->trs_state); 465 } 466 EXPORT_SYMBOL_GPL(torture_random); 467 468 /* 469 * Variables for shuffling. The idea is to ensure that each CPU stays 470 * idle for an extended period to test interactions with dyntick idle, 471 * as well as interactions with any per-CPU variables. 472 */ 473 struct shuffle_task { 474 struct list_head st_l; 475 struct task_struct *st_t; 476 }; 477 478 static long shuffle_interval; /* In jiffies. */ 479 static struct task_struct *shuffler_task; 480 static cpumask_var_t shuffle_tmp_mask; 481 static int shuffle_idle_cpu; /* Force all torture tasks off this CPU */ 482 static struct list_head shuffle_task_list = LIST_HEAD_INIT(shuffle_task_list); 483 static DEFINE_MUTEX(shuffle_task_mutex); 484 485 /* 486 * Register a task to be shuffled. If there is no memory, just splat 487 * and don't bother registering. 488 */ torture_shuffle_task_register(struct task_struct * tp)489 void torture_shuffle_task_register(struct task_struct *tp) 490 { 491 struct shuffle_task *stp; 492 493 if (WARN_ON_ONCE(tp == NULL)) 494 return; 495 stp = kmalloc(sizeof(*stp), GFP_KERNEL); 496 if (WARN_ON_ONCE(stp == NULL)) 497 return; 498 stp->st_t = tp; 499 mutex_lock(&shuffle_task_mutex); 500 list_add(&stp->st_l, &shuffle_task_list); 501 mutex_unlock(&shuffle_task_mutex); 502 } 503 EXPORT_SYMBOL_GPL(torture_shuffle_task_register); 504 505 /* 506 * Unregister all tasks, for example, at the end of the torture run. 507 */ torture_shuffle_task_unregister_all(void)508 static void torture_shuffle_task_unregister_all(void) 509 { 510 struct shuffle_task *stp; 511 struct shuffle_task *p; 512 513 mutex_lock(&shuffle_task_mutex); 514 list_for_each_entry_safe(stp, p, &shuffle_task_list, st_l) { 515 list_del(&stp->st_l); 516 kfree(stp); 517 } 518 mutex_unlock(&shuffle_task_mutex); 519 } 520 521 /* Shuffle tasks such that we allow shuffle_idle_cpu to become idle. 522 * A special case is when shuffle_idle_cpu = -1, in which case we allow 523 * the tasks to run on all CPUs. 524 */ torture_shuffle_tasks(struct torture_random_state * trp)525 static void torture_shuffle_tasks(struct torture_random_state *trp) 526 { 527 struct shuffle_task *stp; 528 529 cpumask_setall(shuffle_tmp_mask); 530 cpus_read_lock(); 531 532 /* No point in shuffling if there is only one online CPU (ex: UP) */ 533 if (num_online_cpus() == 1) { 534 cpus_read_unlock(); 535 return; 536 } 537 538 /* Advance to the next CPU. Upon overflow, don't idle any CPUs. */ 539 shuffle_idle_cpu = cpumask_next(shuffle_idle_cpu, shuffle_tmp_mask); 540 if (shuffle_idle_cpu >= nr_cpu_ids) 541 shuffle_idle_cpu = -1; 542 else 543 cpumask_clear_cpu(shuffle_idle_cpu, shuffle_tmp_mask); 544 545 mutex_lock(&shuffle_task_mutex); 546 list_for_each_entry(stp, &shuffle_task_list, st_l) { 547 if (!random_shuffle || torture_random(trp) & 0x1) 548 set_cpus_allowed_ptr(stp->st_t, shuffle_tmp_mask); 549 } 550 mutex_unlock(&shuffle_task_mutex); 551 552 cpus_read_unlock(); 553 } 554 555 /* Shuffle tasks across CPUs, with the intent of allowing each CPU in the 556 * system to become idle at a time and cut off its timer ticks. This is meant 557 * to test the support for such tickless idle CPU in RCU. 558 */ torture_shuffle(void * arg)559 static int torture_shuffle(void *arg) 560 { 561 DEFINE_TORTURE_RANDOM(rand); 562 563 VERBOSE_TOROUT_STRING("torture_shuffle task started"); 564 do { 565 torture_hrtimeout_jiffies(shuffle_interval, &rand); 566 torture_shuffle_tasks(&rand); 567 torture_shutdown_absorb("torture_shuffle"); 568 } while (!torture_must_stop()); 569 torture_kthread_stopping("torture_shuffle"); 570 return 0; 571 } 572 573 /* 574 * Start the shuffler, with shuffint in jiffies. 575 */ torture_shuffle_init(long shuffint)576 int torture_shuffle_init(long shuffint) 577 { 578 shuffle_interval = shuffint; 579 580 shuffle_idle_cpu = -1; 581 582 if (!alloc_cpumask_var(&shuffle_tmp_mask, GFP_KERNEL)) { 583 TOROUT_ERRSTRING("Failed to alloc mask"); 584 return -ENOMEM; 585 } 586 587 /* Create the shuffler thread */ 588 return torture_create_kthread(torture_shuffle, NULL, shuffler_task); 589 } 590 EXPORT_SYMBOL_GPL(torture_shuffle_init); 591 592 /* 593 * Stop the shuffling. 594 */ torture_shuffle_cleanup(void)595 static void torture_shuffle_cleanup(void) 596 { 597 torture_shuffle_task_unregister_all(); 598 if (shuffler_task) { 599 VERBOSE_TOROUT_STRING("Stopping torture_shuffle task"); 600 kthread_stop(shuffler_task); 601 free_cpumask_var(shuffle_tmp_mask); 602 } 603 shuffler_task = NULL; 604 } 605 606 /* 607 * Variables for auto-shutdown. This allows "lights out" torture runs 608 * to be fully scripted. 609 */ 610 static struct task_struct *shutdown_task; 611 static ktime_t shutdown_time; /* time to system shutdown. */ 612 static void (*torture_shutdown_hook)(void); 613 614 /* 615 * Absorb kthreads into a kernel function that won't return, so that 616 * they won't ever access module text or data again. 617 */ torture_shutdown_absorb(const char * title)618 void torture_shutdown_absorb(const char *title) 619 { 620 while (READ_ONCE(fullstop) == FULLSTOP_SHUTDOWN) { 621 pr_notice("torture thread %s parking due to system shutdown\n", 622 title); 623 schedule_timeout_uninterruptible(MAX_SCHEDULE_TIMEOUT); 624 } 625 } 626 EXPORT_SYMBOL_GPL(torture_shutdown_absorb); 627 628 /* 629 * Cause the torture test to shutdown the system after the test has 630 * run for the time specified by the shutdown_secs parameter. 631 */ torture_shutdown(void * arg)632 static int torture_shutdown(void *arg) 633 { 634 ktime_t ktime_snap; 635 636 VERBOSE_TOROUT_STRING("torture_shutdown task started"); 637 ktime_snap = ktime_get(); 638 while (ktime_before(ktime_snap, shutdown_time) && 639 !torture_must_stop()) { 640 if (verbose) 641 pr_alert("%s" TORTURE_FLAG 642 "torture_shutdown task: %llu ms remaining\n", 643 torture_type, 644 ktime_ms_delta(shutdown_time, ktime_snap)); 645 set_current_state(TASK_INTERRUPTIBLE); 646 schedule_hrtimeout(&shutdown_time, HRTIMER_MODE_ABS); 647 ktime_snap = ktime_get(); 648 } 649 if (torture_must_stop()) { 650 torture_kthread_stopping("torture_shutdown"); 651 return 0; 652 } 653 654 /* OK, shut down the system. */ 655 656 VERBOSE_TOROUT_STRING("torture_shutdown task shutting down system"); 657 shutdown_task = NULL; /* Avoid self-kill deadlock. */ 658 if (torture_shutdown_hook) 659 torture_shutdown_hook(); 660 else 661 VERBOSE_TOROUT_STRING("No torture_shutdown_hook(), skipping."); 662 if (ftrace_dump_at_shutdown) 663 rcu_ftrace_dump(DUMP_ALL); 664 kernel_power_off(); /* Shut down the system. */ 665 return 0; 666 } 667 668 /* 669 * Start up the shutdown task. 670 */ torture_shutdown_init(int ssecs,void (* cleanup)(void))671 int torture_shutdown_init(int ssecs, void (*cleanup)(void)) 672 { 673 torture_shutdown_hook = cleanup; 674 if (ssecs > 0) { 675 shutdown_time = ktime_add(ktime_get(), ktime_set(ssecs, 0)); 676 return torture_create_kthread(torture_shutdown, NULL, 677 shutdown_task); 678 } 679 return 0; 680 } 681 EXPORT_SYMBOL_GPL(torture_shutdown_init); 682 683 /* 684 * Detect and respond to a system shutdown. 685 */ torture_shutdown_notify(struct notifier_block * unused1,unsigned long unused2,void * unused3)686 static int torture_shutdown_notify(struct notifier_block *unused1, 687 unsigned long unused2, void *unused3) 688 { 689 mutex_lock(&fullstop_mutex); 690 if (READ_ONCE(fullstop) == FULLSTOP_DONTSTOP) { 691 VERBOSE_TOROUT_STRING("Unscheduled system shutdown detected"); 692 WRITE_ONCE(fullstop, FULLSTOP_SHUTDOWN); 693 } else { 694 pr_warn("Concurrent rmmod and shutdown illegal!\n"); 695 } 696 mutex_unlock(&fullstop_mutex); 697 return NOTIFY_DONE; 698 } 699 700 static struct notifier_block torture_shutdown_nb = { 701 .notifier_call = torture_shutdown_notify, 702 }; 703 704 /* 705 * Shut down the shutdown task. Say what??? Heh! This can happen if 706 * the torture module gets an rmmod before the shutdown time arrives. ;-) 707 */ torture_shutdown_cleanup(void)708 static void torture_shutdown_cleanup(void) 709 { 710 unregister_reboot_notifier(&torture_shutdown_nb); 711 if (shutdown_task != NULL) { 712 VERBOSE_TOROUT_STRING("Stopping torture_shutdown task"); 713 kthread_stop(shutdown_task); 714 } 715 shutdown_task = NULL; 716 } 717 718 /* 719 * Variables for stuttering, which means to periodically pause and 720 * restart testing in order to catch bugs that appear when load is 721 * suddenly applied to or removed from the system. 722 */ 723 static struct task_struct *stutter_task; 724 static ktime_t stutter_till_abs_time; 725 static int stutter; 726 static int stutter_gap; 727 728 /* 729 * Block until the stutter interval ends. This must be called periodically 730 * by all running kthreads that need to be subject to stuttering. 731 */ stutter_wait(const char * title)732 bool stutter_wait(const char *title) 733 { 734 bool ret = false; 735 ktime_t till_ns; 736 737 cond_resched_tasks_rcu_qs(); 738 till_ns = READ_ONCE(stutter_till_abs_time); 739 if (till_ns && ktime_before(ktime_get(), till_ns)) { 740 torture_hrtimeout_ns(till_ns, 0, HRTIMER_MODE_ABS, NULL); 741 ret = true; 742 } 743 torture_shutdown_absorb(title); 744 return ret; 745 } 746 EXPORT_SYMBOL_GPL(stutter_wait); 747 748 /* 749 * Cause the torture test to "stutter", starting and stopping all 750 * threads periodically. 751 */ torture_stutter(void * arg)752 static int torture_stutter(void *arg) 753 { 754 ktime_t till_ns; 755 756 VERBOSE_TOROUT_STRING("torture_stutter task started"); 757 do { 758 if (!torture_must_stop() && stutter > 1) { 759 till_ns = ktime_add_ns(ktime_get(), 760 jiffies_to_nsecs(stutter)); 761 WRITE_ONCE(stutter_till_abs_time, till_ns); 762 torture_hrtimeout_jiffies(stutter - 1, NULL); 763 } 764 if (!torture_must_stop()) 765 torture_hrtimeout_jiffies(stutter_gap, NULL); 766 torture_shutdown_absorb("torture_stutter"); 767 } while (!torture_must_stop()); 768 torture_kthread_stopping("torture_stutter"); 769 return 0; 770 } 771 772 /* 773 * Initialize and kick off the torture_stutter kthread. 774 */ torture_stutter_init(const int s,const int sgap)775 int torture_stutter_init(const int s, const int sgap) 776 { 777 stutter = s; 778 stutter_gap = sgap; 779 return torture_create_kthread(torture_stutter, NULL, stutter_task); 780 } 781 EXPORT_SYMBOL_GPL(torture_stutter_init); 782 783 /* 784 * Cleanup after the torture_stutter kthread. 785 */ torture_stutter_cleanup(void)786 static void torture_stutter_cleanup(void) 787 { 788 if (!stutter_task) 789 return; 790 VERBOSE_TOROUT_STRING("Stopping torture_stutter task"); 791 kthread_stop(stutter_task); 792 stutter_task = NULL; 793 } 794 795 static void torture_print_module_parms(void)796 torture_print_module_parms(void) 797 { 798 pr_alert("torture module --- %s: disable_onoff_at_boot=%d ftrace_dump_at_shutdown=%d verbose_sleep_frequency=%d verbose_sleep_duration=%d random_shuffle=%d\n", 799 torture_type, disable_onoff_at_boot, ftrace_dump_at_shutdown, verbose_sleep_frequency, verbose_sleep_duration, random_shuffle); 800 } 801 802 /* 803 * Initialize torture module. Please note that this is -not- invoked via 804 * the usual module_init() mechanism, but rather by an explicit call from 805 * the client torture module. This call must be paired with a later 806 * torture_init_end(). 807 * 808 * The runnable parameter points to a flag that controls whether or not 809 * the test is currently runnable. If there is no such flag, pass in NULL. 810 */ torture_init_begin(char * ttype,int v)811 bool torture_init_begin(char *ttype, int v) 812 { 813 mutex_lock(&fullstop_mutex); 814 if (torture_type != NULL) { 815 pr_alert("%s: Refusing %s init: %s running.\n", 816 __func__, ttype, torture_type); 817 pr_alert("%s: One torture test at a time!\n", __func__); 818 mutex_unlock(&fullstop_mutex); 819 return false; 820 } 821 torture_type = ttype; 822 verbose = v; 823 fullstop = FULLSTOP_DONTSTOP; 824 torture_print_module_parms(); 825 return true; 826 } 827 EXPORT_SYMBOL_GPL(torture_init_begin); 828 829 /* 830 * Tell the torture module that initialization is complete. 831 */ torture_init_end(void)832 void torture_init_end(void) 833 { 834 mutex_unlock(&fullstop_mutex); 835 register_reboot_notifier(&torture_shutdown_nb); 836 } 837 EXPORT_SYMBOL_GPL(torture_init_end); 838 839 /* 840 * Clean up torture module. Please note that this is -not- invoked via 841 * the usual module_exit() mechanism, but rather by an explicit call from 842 * the client torture module. Returns true if a race with system shutdown 843 * is detected, otherwise, all kthreads started by functions in this file 844 * will be shut down. 845 * 846 * This must be called before the caller starts shutting down its own 847 * kthreads. 848 * 849 * Both torture_cleanup_begin() and torture_cleanup_end() must be paired, 850 * in order to correctly perform the cleanup. They are separated because 851 * threads can still need to reference the torture_type type, thus nullify 852 * only after completing all other relevant calls. 853 */ torture_cleanup_begin(void)854 bool torture_cleanup_begin(void) 855 { 856 mutex_lock(&fullstop_mutex); 857 if (READ_ONCE(fullstop) == FULLSTOP_SHUTDOWN) { 858 pr_warn("Concurrent rmmod and shutdown illegal!\n"); 859 mutex_unlock(&fullstop_mutex); 860 schedule_timeout_uninterruptible(10); 861 return true; 862 } 863 WRITE_ONCE(fullstop, FULLSTOP_RMMOD); 864 mutex_unlock(&fullstop_mutex); 865 torture_shutdown_cleanup(); 866 torture_shuffle_cleanup(); 867 torture_stutter_cleanup(); 868 torture_onoff_cleanup(); 869 return false; 870 } 871 EXPORT_SYMBOL_GPL(torture_cleanup_begin); 872 torture_cleanup_end(void)873 void torture_cleanup_end(void) 874 { 875 mutex_lock(&fullstop_mutex); 876 torture_type = NULL; 877 mutex_unlock(&fullstop_mutex); 878 } 879 EXPORT_SYMBOL_GPL(torture_cleanup_end); 880 881 /* 882 * Is it time for the current torture test to stop? 883 */ torture_must_stop(void)884 bool torture_must_stop(void) 885 { 886 return torture_must_stop_irq() || kthread_should_stop(); 887 } 888 EXPORT_SYMBOL_GPL(torture_must_stop); 889 890 /* 891 * Is it time for the current torture test to stop? This is the irq-safe 892 * version, hence no check for kthread_should_stop(). 893 */ torture_must_stop_irq(void)894 bool torture_must_stop_irq(void) 895 { 896 return READ_ONCE(fullstop) != FULLSTOP_DONTSTOP; 897 } 898 EXPORT_SYMBOL_GPL(torture_must_stop_irq); 899 900 /* 901 * Each kthread must wait for kthread_should_stop() before returning from 902 * its top-level function, otherwise segfaults ensue. This function 903 * prints a "stopping" message and waits for kthread_should_stop(), and 904 * should be called from all torture kthreads immediately prior to 905 * returning. 906 */ torture_kthread_stopping(char * title)907 void torture_kthread_stopping(char *title) 908 { 909 char buf[128]; 910 911 snprintf(buf, sizeof(buf), "%s is stopping", title); 912 VERBOSE_TOROUT_STRING(buf); 913 while (!kthread_should_stop()) { 914 torture_shutdown_absorb(title); 915 schedule_timeout_uninterruptible(HZ / 20); 916 } 917 } 918 EXPORT_SYMBOL_GPL(torture_kthread_stopping); 919 920 /* 921 * Create a generic torture kthread that is immediately runnable. If you 922 * need the kthread to be stopped so that you can do something to it before 923 * it starts, you will need to open-code your own. 924 */ _torture_create_kthread(int (* fn)(void * arg),void * arg,char * s,char * m,char * f,struct task_struct ** tp,void (* cbf)(struct task_struct * tp))925 int _torture_create_kthread(int (*fn)(void *arg), void *arg, char *s, char *m, 926 char *f, struct task_struct **tp, void (*cbf)(struct task_struct *tp)) 927 { 928 int ret = 0; 929 930 VERBOSE_TOROUT_STRING(m); 931 *tp = kthread_create(fn, arg, "%s", s); 932 if (IS_ERR(*tp)) { 933 ret = PTR_ERR(*tp); 934 TOROUT_ERRSTRING(f); 935 *tp = NULL; 936 return ret; 937 } 938 939 if (cbf) 940 cbf(*tp); 941 942 wake_up_process(*tp); // Process is sleeping, so ordering provided. 943 torture_shuffle_task_register(*tp); 944 return ret; 945 } 946 EXPORT_SYMBOL_GPL(_torture_create_kthread); 947 948 /* 949 * Stop a generic kthread, emitting a message. 950 */ _torture_stop_kthread(char * m,struct task_struct ** tp)951 void _torture_stop_kthread(char *m, struct task_struct **tp) 952 { 953 if (*tp == NULL) 954 return; 955 VERBOSE_TOROUT_STRING(m); 956 kthread_stop(*tp); 957 *tp = NULL; 958 } 959 EXPORT_SYMBOL_GPL(_torture_stop_kthread); 960