1  // SPDX-License-Identifier: GPL-2.0+
2  /*
3   * Common functions for in-kernel torture tests.
4   *
5   * Copyright (C) IBM Corporation, 2014
6   *
7   * Author: Paul E. McKenney <paulmck@linux.ibm.com>
8   *	Based on kernel/rcu/torture.c.
9   */
10  
11  #define pr_fmt(fmt) fmt
12  
13  #include <linux/types.h>
14  #include <linux/kernel.h>
15  #include <linux/init.h>
16  #include <linux/module.h>
17  #include <linux/kthread.h>
18  #include <linux/err.h>
19  #include <linux/spinlock.h>
20  #include <linux/smp.h>
21  #include <linux/interrupt.h>
22  #include <linux/sched.h>
23  #include <linux/sched/clock.h>
24  #include <linux/atomic.h>
25  #include <linux/bitops.h>
26  #include <linux/completion.h>
27  #include <linux/moduleparam.h>
28  #include <linux/percpu.h>
29  #include <linux/notifier.h>
30  #include <linux/reboot.h>
31  #include <linux/freezer.h>
32  #include <linux/cpu.h>
33  #include <linux/delay.h>
34  #include <linux/stat.h>
35  #include <linux/slab.h>
36  #include <linux/trace_clock.h>
37  #include <linux/ktime.h>
38  #include <asm/byteorder.h>
39  #include <linux/torture.h>
40  #include <linux/sched/rt.h>
41  #include "rcu/rcu.h"
42  
43  MODULE_DESCRIPTION("Common functions for in-kernel torture tests");
44  MODULE_LICENSE("GPL");
45  MODULE_AUTHOR("Paul E. McKenney <paulmck@linux.ibm.com>");
46  
47  static bool disable_onoff_at_boot;
48  module_param(disable_onoff_at_boot, bool, 0444);
49  
50  static bool ftrace_dump_at_shutdown;
51  module_param(ftrace_dump_at_shutdown, bool, 0444);
52  
53  static int verbose_sleep_frequency;
54  module_param(verbose_sleep_frequency, int, 0444);
55  
56  static int verbose_sleep_duration = 1;
57  module_param(verbose_sleep_duration, int, 0444);
58  
59  static int random_shuffle;
60  module_param(random_shuffle, int, 0444);
61  
62  static char *torture_type;
63  static int verbose;
64  
65  /* Mediate rmmod and system shutdown.  Concurrent rmmod & shutdown illegal! */
66  #define FULLSTOP_DONTSTOP 0	/* Normal operation. */
67  #define FULLSTOP_SHUTDOWN 1	/* System shutdown with torture running. */
68  #define FULLSTOP_RMMOD    2	/* Normal rmmod of torture. */
69  static int fullstop = FULLSTOP_RMMOD;
70  static DEFINE_MUTEX(fullstop_mutex);
71  
72  static atomic_t verbose_sleep_counter;
73  
74  /*
75   * Sleep if needed from VERBOSE_TOROUT*().
76   */
verbose_torout_sleep(void)77  void verbose_torout_sleep(void)
78  {
79  	if (verbose_sleep_frequency > 0 &&
80  	    verbose_sleep_duration > 0 &&
81  	    !(atomic_inc_return(&verbose_sleep_counter) % verbose_sleep_frequency))
82  		schedule_timeout_uninterruptible(verbose_sleep_duration);
83  }
84  EXPORT_SYMBOL_GPL(verbose_torout_sleep);
85  
86  /*
87   * Schedule a high-resolution-timer sleep in nanoseconds, with a 32-bit
88   * nanosecond random fuzz.  This function and its friends desynchronize
89   * testing from the timer wheel.
90   */
torture_hrtimeout_ns(ktime_t baset_ns,u32 fuzzt_ns,const enum hrtimer_mode mode,struct torture_random_state * trsp)91  int torture_hrtimeout_ns(ktime_t baset_ns, u32 fuzzt_ns, const enum hrtimer_mode mode,
92  			 struct torture_random_state *trsp)
93  {
94  	ktime_t hto = baset_ns;
95  
96  	if (trsp)
97  		hto += torture_random(trsp) % fuzzt_ns;
98  	set_current_state(TASK_IDLE);
99  	return schedule_hrtimeout(&hto, mode);
100  }
101  EXPORT_SYMBOL_GPL(torture_hrtimeout_ns);
102  
103  /*
104   * Schedule a high-resolution-timer sleep in microseconds, with a 32-bit
105   * nanosecond (not microsecond!) random fuzz.
106   */
torture_hrtimeout_us(u32 baset_us,u32 fuzzt_ns,struct torture_random_state * trsp)107  int torture_hrtimeout_us(u32 baset_us, u32 fuzzt_ns, struct torture_random_state *trsp)
108  {
109  	ktime_t baset_ns = baset_us * NSEC_PER_USEC;
110  
111  	return torture_hrtimeout_ns(baset_ns, fuzzt_ns, HRTIMER_MODE_REL, trsp);
112  }
113  EXPORT_SYMBOL_GPL(torture_hrtimeout_us);
114  
115  /*
116   * Schedule a high-resolution-timer sleep in milliseconds, with a 32-bit
117   * microsecond (not millisecond!) random fuzz.
118   */
torture_hrtimeout_ms(u32 baset_ms,u32 fuzzt_us,struct torture_random_state * trsp)119  int torture_hrtimeout_ms(u32 baset_ms, u32 fuzzt_us, struct torture_random_state *trsp)
120  {
121  	ktime_t baset_ns = baset_ms * NSEC_PER_MSEC;
122  	u32 fuzzt_ns;
123  
124  	if ((u32)~0U / NSEC_PER_USEC < fuzzt_us)
125  		fuzzt_ns = (u32)~0U;
126  	else
127  		fuzzt_ns = fuzzt_us * NSEC_PER_USEC;
128  	return torture_hrtimeout_ns(baset_ns, fuzzt_ns, HRTIMER_MODE_REL, trsp);
129  }
130  EXPORT_SYMBOL_GPL(torture_hrtimeout_ms);
131  
132  /*
133   * Schedule a high-resolution-timer sleep in jiffies, with an
134   * implied one-jiffy random fuzz.  This is intended to replace calls to
135   * schedule_timeout_interruptible() and friends.
136   */
torture_hrtimeout_jiffies(u32 baset_j,struct torture_random_state * trsp)137  int torture_hrtimeout_jiffies(u32 baset_j, struct torture_random_state *trsp)
138  {
139  	ktime_t baset_ns = jiffies_to_nsecs(baset_j);
140  
141  	return torture_hrtimeout_ns(baset_ns, jiffies_to_nsecs(1), HRTIMER_MODE_REL, trsp);
142  }
143  EXPORT_SYMBOL_GPL(torture_hrtimeout_jiffies);
144  
145  /*
146   * Schedule a high-resolution-timer sleep in milliseconds, with a 32-bit
147   * millisecond (not second!) random fuzz.
148   */
torture_hrtimeout_s(u32 baset_s,u32 fuzzt_ms,struct torture_random_state * trsp)149  int torture_hrtimeout_s(u32 baset_s, u32 fuzzt_ms, struct torture_random_state *trsp)
150  {
151  	ktime_t baset_ns = baset_s * NSEC_PER_SEC;
152  	u32 fuzzt_ns;
153  
154  	if ((u32)~0U / NSEC_PER_MSEC < fuzzt_ms)
155  		fuzzt_ns = (u32)~0U;
156  	else
157  		fuzzt_ns = fuzzt_ms * NSEC_PER_MSEC;
158  	return torture_hrtimeout_ns(baset_ns, fuzzt_ns, HRTIMER_MODE_REL, trsp);
159  }
160  EXPORT_SYMBOL_GPL(torture_hrtimeout_s);
161  
162  #ifdef CONFIG_HOTPLUG_CPU
163  
164  /*
165   * Variables for online-offline handling.  Only present if CPU hotplug
166   * is enabled, otherwise does nothing.
167   */
168  
169  static struct task_struct *onoff_task;
170  static long onoff_holdoff;
171  static long onoff_interval;
172  static torture_ofl_func *onoff_f;
173  static long n_offline_attempts;
174  static long n_offline_successes;
175  static unsigned long sum_offline;
176  static int min_offline = -1;
177  static int max_offline;
178  static long n_online_attempts;
179  static long n_online_successes;
180  static unsigned long sum_online;
181  static int min_online = -1;
182  static int max_online;
183  
184  static int torture_online_cpus = NR_CPUS;
185  
186  /*
187   * Some torture testing leverages confusion as to the number of online
188   * CPUs.  This function returns the torture-testing view of this number,
189   * which allows torture tests to load-balance appropriately.
190   */
torture_num_online_cpus(void)191  int torture_num_online_cpus(void)
192  {
193  	return READ_ONCE(torture_online_cpus);
194  }
195  EXPORT_SYMBOL_GPL(torture_num_online_cpus);
196  
197  /*
198   * Attempt to take a CPU offline.  Return false if the CPU is already
199   * offline or if it is not subject to CPU-hotplug operations.  The
200   * caller can detect other failures by looking at the statistics.
201   */
torture_offline(int cpu,long * n_offl_attempts,long * n_offl_successes,unsigned long * sum_offl,int * min_offl,int * max_offl)202  bool torture_offline(int cpu, long *n_offl_attempts, long *n_offl_successes,
203  		     unsigned long *sum_offl, int *min_offl, int *max_offl)
204  {
205  	unsigned long delta;
206  	int ret;
207  	char *s;
208  	unsigned long starttime;
209  
210  	if (!cpu_online(cpu) || !cpu_is_hotpluggable(cpu))
211  		return false;
212  	if (num_online_cpus() <= 1)
213  		return false;  /* Can't offline the last CPU. */
214  
215  	if (verbose > 1)
216  		pr_alert("%s" TORTURE_FLAG
217  			 "torture_onoff task: offlining %d\n",
218  			 torture_type, cpu);
219  	starttime = jiffies;
220  	(*n_offl_attempts)++;
221  	ret = remove_cpu(cpu);
222  	if (ret) {
223  		s = "";
224  		if (!rcu_inkernel_boot_has_ended() && ret == -EBUSY) {
225  			// PCI probe frequently disables hotplug during boot.
226  			(*n_offl_attempts)--;
227  			s = " (-EBUSY forgiven during boot)";
228  		}
229  		if (verbose)
230  			pr_alert("%s" TORTURE_FLAG
231  				 "torture_onoff task: offline %d failed%s: errno %d\n",
232  				 torture_type, cpu, s, ret);
233  	} else {
234  		if (verbose > 1)
235  			pr_alert("%s" TORTURE_FLAG
236  				 "torture_onoff task: offlined %d\n",
237  				 torture_type, cpu);
238  		if (onoff_f)
239  			onoff_f();
240  		(*n_offl_successes)++;
241  		delta = jiffies - starttime;
242  		*sum_offl += delta;
243  		if (*min_offl < 0) {
244  			*min_offl = delta;
245  			*max_offl = delta;
246  		}
247  		if (*min_offl > delta)
248  			*min_offl = delta;
249  		if (*max_offl < delta)
250  			*max_offl = delta;
251  		WRITE_ONCE(torture_online_cpus, torture_online_cpus - 1);
252  		WARN_ON_ONCE(torture_online_cpus <= 0);
253  	}
254  
255  	return true;
256  }
257  EXPORT_SYMBOL_GPL(torture_offline);
258  
259  /*
260   * Attempt to bring a CPU online.  Return false if the CPU is already
261   * online or if it is not subject to CPU-hotplug operations.  The
262   * caller can detect other failures by looking at the statistics.
263   */
torture_online(int cpu,long * n_onl_attempts,long * n_onl_successes,unsigned long * sum_onl,int * min_onl,int * max_onl)264  bool torture_online(int cpu, long *n_onl_attempts, long *n_onl_successes,
265  		    unsigned long *sum_onl, int *min_onl, int *max_onl)
266  {
267  	unsigned long delta;
268  	int ret;
269  	char *s;
270  	unsigned long starttime;
271  
272  	if (cpu_online(cpu) || !cpu_is_hotpluggable(cpu))
273  		return false;
274  
275  	if (verbose > 1)
276  		pr_alert("%s" TORTURE_FLAG
277  			 "torture_onoff task: onlining %d\n",
278  			 torture_type, cpu);
279  	starttime = jiffies;
280  	(*n_onl_attempts)++;
281  	ret = add_cpu(cpu);
282  	if (ret) {
283  		s = "";
284  		if (!rcu_inkernel_boot_has_ended() && ret == -EBUSY) {
285  			// PCI probe frequently disables hotplug during boot.
286  			(*n_onl_attempts)--;
287  			s = " (-EBUSY forgiven during boot)";
288  		}
289  		if (verbose)
290  			pr_alert("%s" TORTURE_FLAG
291  				 "torture_onoff task: online %d failed%s: errno %d\n",
292  				 torture_type, cpu, s, ret);
293  	} else {
294  		if (verbose > 1)
295  			pr_alert("%s" TORTURE_FLAG
296  				 "torture_onoff task: onlined %d\n",
297  				 torture_type, cpu);
298  		(*n_onl_successes)++;
299  		delta = jiffies - starttime;
300  		*sum_onl += delta;
301  		if (*min_onl < 0) {
302  			*min_onl = delta;
303  			*max_onl = delta;
304  		}
305  		if (*min_onl > delta)
306  			*min_onl = delta;
307  		if (*max_onl < delta)
308  			*max_onl = delta;
309  		WRITE_ONCE(torture_online_cpus, torture_online_cpus + 1);
310  	}
311  
312  	return true;
313  }
314  EXPORT_SYMBOL_GPL(torture_online);
315  
316  /*
317   * Get everything online at the beginning and ends of tests.
318   */
torture_online_all(char * phase)319  static void torture_online_all(char *phase)
320  {
321  	int cpu;
322  	int ret;
323  
324  	for_each_possible_cpu(cpu) {
325  		if (cpu_online(cpu))
326  			continue;
327  		ret = add_cpu(cpu);
328  		if (ret && verbose) {
329  			pr_alert("%s" TORTURE_FLAG
330  				 "%s: %s online %d: errno %d\n",
331  				 __func__, phase, torture_type, cpu, ret);
332  		}
333  	}
334  }
335  
336  /*
337   * Execute random CPU-hotplug operations at the interval specified
338   * by the onoff_interval.
339   */
340  static int
torture_onoff(void * arg)341  torture_onoff(void *arg)
342  {
343  	int cpu;
344  	int maxcpu = -1;
345  	DEFINE_TORTURE_RANDOM(rand);
346  
347  	VERBOSE_TOROUT_STRING("torture_onoff task started");
348  	for_each_online_cpu(cpu)
349  		maxcpu = cpu;
350  	WARN_ON(maxcpu < 0);
351  	torture_online_all("Initial");
352  	if (maxcpu == 0) {
353  		VERBOSE_TOROUT_STRING("Only one CPU, so CPU-hotplug testing is disabled");
354  		goto stop;
355  	}
356  
357  	if (onoff_holdoff > 0) {
358  		VERBOSE_TOROUT_STRING("torture_onoff begin holdoff");
359  		torture_hrtimeout_jiffies(onoff_holdoff, &rand);
360  		VERBOSE_TOROUT_STRING("torture_onoff end holdoff");
361  	}
362  	while (!torture_must_stop()) {
363  		if (disable_onoff_at_boot && !rcu_inkernel_boot_has_ended()) {
364  			torture_hrtimeout_jiffies(HZ / 10, &rand);
365  			continue;
366  		}
367  		cpu = torture_random(&rand) % (maxcpu + 1);
368  		if (!torture_offline(cpu,
369  				     &n_offline_attempts, &n_offline_successes,
370  				     &sum_offline, &min_offline, &max_offline))
371  			torture_online(cpu,
372  				       &n_online_attempts, &n_online_successes,
373  				       &sum_online, &min_online, &max_online);
374  		torture_hrtimeout_jiffies(onoff_interval, &rand);
375  	}
376  
377  stop:
378  	torture_kthread_stopping("torture_onoff");
379  	torture_online_all("Final");
380  	return 0;
381  }
382  
383  #endif /* #ifdef CONFIG_HOTPLUG_CPU */
384  
385  /*
386   * Initiate online-offline handling.
387   */
torture_onoff_init(long ooholdoff,long oointerval,torture_ofl_func * f)388  int torture_onoff_init(long ooholdoff, long oointerval, torture_ofl_func *f)
389  {
390  #ifdef CONFIG_HOTPLUG_CPU
391  	onoff_holdoff = ooholdoff;
392  	onoff_interval = oointerval;
393  	onoff_f = f;
394  	if (onoff_interval <= 0)
395  		return 0;
396  	return torture_create_kthread(torture_onoff, NULL, onoff_task);
397  #else /* #ifdef CONFIG_HOTPLUG_CPU */
398  	return 0;
399  #endif /* #else #ifdef CONFIG_HOTPLUG_CPU */
400  }
401  EXPORT_SYMBOL_GPL(torture_onoff_init);
402  
403  /*
404   * Clean up after online/offline testing.
405   */
torture_onoff_cleanup(void)406  static void torture_onoff_cleanup(void)
407  {
408  #ifdef CONFIG_HOTPLUG_CPU
409  	if (onoff_task == NULL)
410  		return;
411  	VERBOSE_TOROUT_STRING("Stopping torture_onoff task");
412  	kthread_stop(onoff_task);
413  	onoff_task = NULL;
414  #endif /* #ifdef CONFIG_HOTPLUG_CPU */
415  }
416  
417  /*
418   * Print online/offline testing statistics.
419   */
torture_onoff_stats(void)420  void torture_onoff_stats(void)
421  {
422  #ifdef CONFIG_HOTPLUG_CPU
423  	pr_cont("onoff: %ld/%ld:%ld/%ld %d,%d:%d,%d %lu:%lu (HZ=%d) ",
424  		n_online_successes, n_online_attempts,
425  		n_offline_successes, n_offline_attempts,
426  		min_online, max_online,
427  		min_offline, max_offline,
428  		sum_online, sum_offline, HZ);
429  #endif /* #ifdef CONFIG_HOTPLUG_CPU */
430  }
431  EXPORT_SYMBOL_GPL(torture_onoff_stats);
432  
433  /*
434   * Were all the online/offline operations successful?
435   */
torture_onoff_failures(void)436  bool torture_onoff_failures(void)
437  {
438  #ifdef CONFIG_HOTPLUG_CPU
439  	return n_online_successes != n_online_attempts ||
440  	       n_offline_successes != n_offline_attempts;
441  #else /* #ifdef CONFIG_HOTPLUG_CPU */
442  	return false;
443  #endif /* #else #ifdef CONFIG_HOTPLUG_CPU */
444  }
445  EXPORT_SYMBOL_GPL(torture_onoff_failures);
446  
447  #define TORTURE_RANDOM_MULT	39916801  /* prime */
448  #define TORTURE_RANDOM_ADD	479001701 /* prime */
449  #define TORTURE_RANDOM_REFRESH	10000
450  
451  /*
452   * Crude but fast random-number generator.  Uses a linear congruential
453   * generator, with occasional help from cpu_clock().
454   */
455  unsigned long
torture_random(struct torture_random_state * trsp)456  torture_random(struct torture_random_state *trsp)
457  {
458  	if (--trsp->trs_count < 0) {
459  		trsp->trs_state += (unsigned long)local_clock() + raw_smp_processor_id();
460  		trsp->trs_count = TORTURE_RANDOM_REFRESH;
461  	}
462  	trsp->trs_state = trsp->trs_state * TORTURE_RANDOM_MULT +
463  		TORTURE_RANDOM_ADD;
464  	return swahw32(trsp->trs_state);
465  }
466  EXPORT_SYMBOL_GPL(torture_random);
467  
468  /*
469   * Variables for shuffling.  The idea is to ensure that each CPU stays
470   * idle for an extended period to test interactions with dyntick idle,
471   * as well as interactions with any per-CPU variables.
472   */
473  struct shuffle_task {
474  	struct list_head st_l;
475  	struct task_struct *st_t;
476  };
477  
478  static long shuffle_interval;	/* In jiffies. */
479  static struct task_struct *shuffler_task;
480  static cpumask_var_t shuffle_tmp_mask;
481  static int shuffle_idle_cpu;	/* Force all torture tasks off this CPU */
482  static struct list_head shuffle_task_list = LIST_HEAD_INIT(shuffle_task_list);
483  static DEFINE_MUTEX(shuffle_task_mutex);
484  
485  /*
486   * Register a task to be shuffled.  If there is no memory, just splat
487   * and don't bother registering.
488   */
torture_shuffle_task_register(struct task_struct * tp)489  void torture_shuffle_task_register(struct task_struct *tp)
490  {
491  	struct shuffle_task *stp;
492  
493  	if (WARN_ON_ONCE(tp == NULL))
494  		return;
495  	stp = kmalloc(sizeof(*stp), GFP_KERNEL);
496  	if (WARN_ON_ONCE(stp == NULL))
497  		return;
498  	stp->st_t = tp;
499  	mutex_lock(&shuffle_task_mutex);
500  	list_add(&stp->st_l, &shuffle_task_list);
501  	mutex_unlock(&shuffle_task_mutex);
502  }
503  EXPORT_SYMBOL_GPL(torture_shuffle_task_register);
504  
505  /*
506   * Unregister all tasks, for example, at the end of the torture run.
507   */
torture_shuffle_task_unregister_all(void)508  static void torture_shuffle_task_unregister_all(void)
509  {
510  	struct shuffle_task *stp;
511  	struct shuffle_task *p;
512  
513  	mutex_lock(&shuffle_task_mutex);
514  	list_for_each_entry_safe(stp, p, &shuffle_task_list, st_l) {
515  		list_del(&stp->st_l);
516  		kfree(stp);
517  	}
518  	mutex_unlock(&shuffle_task_mutex);
519  }
520  
521  /* Shuffle tasks such that we allow shuffle_idle_cpu to become idle.
522   * A special case is when shuffle_idle_cpu = -1, in which case we allow
523   * the tasks to run on all CPUs.
524   */
torture_shuffle_tasks(struct torture_random_state * trp)525  static void torture_shuffle_tasks(struct torture_random_state *trp)
526  {
527  	struct shuffle_task *stp;
528  
529  	cpumask_setall(shuffle_tmp_mask);
530  	cpus_read_lock();
531  
532  	/* No point in shuffling if there is only one online CPU (ex: UP) */
533  	if (num_online_cpus() == 1) {
534  		cpus_read_unlock();
535  		return;
536  	}
537  
538  	/* Advance to the next CPU.  Upon overflow, don't idle any CPUs. */
539  	shuffle_idle_cpu = cpumask_next(shuffle_idle_cpu, shuffle_tmp_mask);
540  	if (shuffle_idle_cpu >= nr_cpu_ids)
541  		shuffle_idle_cpu = -1;
542  	else
543  		cpumask_clear_cpu(shuffle_idle_cpu, shuffle_tmp_mask);
544  
545  	mutex_lock(&shuffle_task_mutex);
546  	list_for_each_entry(stp, &shuffle_task_list, st_l) {
547  		if (!random_shuffle || torture_random(trp) & 0x1)
548  			set_cpus_allowed_ptr(stp->st_t, shuffle_tmp_mask);
549  	}
550  	mutex_unlock(&shuffle_task_mutex);
551  
552  	cpus_read_unlock();
553  }
554  
555  /* Shuffle tasks across CPUs, with the intent of allowing each CPU in the
556   * system to become idle at a time and cut off its timer ticks. This is meant
557   * to test the support for such tickless idle CPU in RCU.
558   */
torture_shuffle(void * arg)559  static int torture_shuffle(void *arg)
560  {
561  	DEFINE_TORTURE_RANDOM(rand);
562  
563  	VERBOSE_TOROUT_STRING("torture_shuffle task started");
564  	do {
565  		torture_hrtimeout_jiffies(shuffle_interval, &rand);
566  		torture_shuffle_tasks(&rand);
567  		torture_shutdown_absorb("torture_shuffle");
568  	} while (!torture_must_stop());
569  	torture_kthread_stopping("torture_shuffle");
570  	return 0;
571  }
572  
573  /*
574   * Start the shuffler, with shuffint in jiffies.
575   */
torture_shuffle_init(long shuffint)576  int torture_shuffle_init(long shuffint)
577  {
578  	shuffle_interval = shuffint;
579  
580  	shuffle_idle_cpu = -1;
581  
582  	if (!alloc_cpumask_var(&shuffle_tmp_mask, GFP_KERNEL)) {
583  		TOROUT_ERRSTRING("Failed to alloc mask");
584  		return -ENOMEM;
585  	}
586  
587  	/* Create the shuffler thread */
588  	return torture_create_kthread(torture_shuffle, NULL, shuffler_task);
589  }
590  EXPORT_SYMBOL_GPL(torture_shuffle_init);
591  
592  /*
593   * Stop the shuffling.
594   */
torture_shuffle_cleanup(void)595  static void torture_shuffle_cleanup(void)
596  {
597  	torture_shuffle_task_unregister_all();
598  	if (shuffler_task) {
599  		VERBOSE_TOROUT_STRING("Stopping torture_shuffle task");
600  		kthread_stop(shuffler_task);
601  		free_cpumask_var(shuffle_tmp_mask);
602  	}
603  	shuffler_task = NULL;
604  }
605  
606  /*
607   * Variables for auto-shutdown.  This allows "lights out" torture runs
608   * to be fully scripted.
609   */
610  static struct task_struct *shutdown_task;
611  static ktime_t shutdown_time;		/* time to system shutdown. */
612  static void (*torture_shutdown_hook)(void);
613  
614  /*
615   * Absorb kthreads into a kernel function that won't return, so that
616   * they won't ever access module text or data again.
617   */
torture_shutdown_absorb(const char * title)618  void torture_shutdown_absorb(const char *title)
619  {
620  	while (READ_ONCE(fullstop) == FULLSTOP_SHUTDOWN) {
621  		pr_notice("torture thread %s parking due to system shutdown\n",
622  			  title);
623  		schedule_timeout_uninterruptible(MAX_SCHEDULE_TIMEOUT);
624  	}
625  }
626  EXPORT_SYMBOL_GPL(torture_shutdown_absorb);
627  
628  /*
629   * Cause the torture test to shutdown the system after the test has
630   * run for the time specified by the shutdown_secs parameter.
631   */
torture_shutdown(void * arg)632  static int torture_shutdown(void *arg)
633  {
634  	ktime_t ktime_snap;
635  
636  	VERBOSE_TOROUT_STRING("torture_shutdown task started");
637  	ktime_snap = ktime_get();
638  	while (ktime_before(ktime_snap, shutdown_time) &&
639  	       !torture_must_stop()) {
640  		if (verbose)
641  			pr_alert("%s" TORTURE_FLAG
642  				 "torture_shutdown task: %llu ms remaining\n",
643  				 torture_type,
644  				 ktime_ms_delta(shutdown_time, ktime_snap));
645  		set_current_state(TASK_INTERRUPTIBLE);
646  		schedule_hrtimeout(&shutdown_time, HRTIMER_MODE_ABS);
647  		ktime_snap = ktime_get();
648  	}
649  	if (torture_must_stop()) {
650  		torture_kthread_stopping("torture_shutdown");
651  		return 0;
652  	}
653  
654  	/* OK, shut down the system. */
655  
656  	VERBOSE_TOROUT_STRING("torture_shutdown task shutting down system");
657  	shutdown_task = NULL;	/* Avoid self-kill deadlock. */
658  	if (torture_shutdown_hook)
659  		torture_shutdown_hook();
660  	else
661  		VERBOSE_TOROUT_STRING("No torture_shutdown_hook(), skipping.");
662  	if (ftrace_dump_at_shutdown)
663  		rcu_ftrace_dump(DUMP_ALL);
664  	kernel_power_off();	/* Shut down the system. */
665  	return 0;
666  }
667  
668  /*
669   * Start up the shutdown task.
670   */
torture_shutdown_init(int ssecs,void (* cleanup)(void))671  int torture_shutdown_init(int ssecs, void (*cleanup)(void))
672  {
673  	torture_shutdown_hook = cleanup;
674  	if (ssecs > 0) {
675  		shutdown_time = ktime_add(ktime_get(), ktime_set(ssecs, 0));
676  		return torture_create_kthread(torture_shutdown, NULL,
677  					      shutdown_task);
678  	}
679  	return 0;
680  }
681  EXPORT_SYMBOL_GPL(torture_shutdown_init);
682  
683  /*
684   * Detect and respond to a system shutdown.
685   */
torture_shutdown_notify(struct notifier_block * unused1,unsigned long unused2,void * unused3)686  static int torture_shutdown_notify(struct notifier_block *unused1,
687  				   unsigned long unused2, void *unused3)
688  {
689  	mutex_lock(&fullstop_mutex);
690  	if (READ_ONCE(fullstop) == FULLSTOP_DONTSTOP) {
691  		VERBOSE_TOROUT_STRING("Unscheduled system shutdown detected");
692  		WRITE_ONCE(fullstop, FULLSTOP_SHUTDOWN);
693  	} else {
694  		pr_warn("Concurrent rmmod and shutdown illegal!\n");
695  	}
696  	mutex_unlock(&fullstop_mutex);
697  	return NOTIFY_DONE;
698  }
699  
700  static struct notifier_block torture_shutdown_nb = {
701  	.notifier_call = torture_shutdown_notify,
702  };
703  
704  /*
705   * Shut down the shutdown task.  Say what???  Heh!  This can happen if
706   * the torture module gets an rmmod before the shutdown time arrives.  ;-)
707   */
torture_shutdown_cleanup(void)708  static void torture_shutdown_cleanup(void)
709  {
710  	unregister_reboot_notifier(&torture_shutdown_nb);
711  	if (shutdown_task != NULL) {
712  		VERBOSE_TOROUT_STRING("Stopping torture_shutdown task");
713  		kthread_stop(shutdown_task);
714  	}
715  	shutdown_task = NULL;
716  }
717  
718  /*
719   * Variables for stuttering, which means to periodically pause and
720   * restart testing in order to catch bugs that appear when load is
721   * suddenly applied to or removed from the system.
722   */
723  static struct task_struct *stutter_task;
724  static ktime_t stutter_till_abs_time;
725  static int stutter;
726  static int stutter_gap;
727  
728  /*
729   * Block until the stutter interval ends.  This must be called periodically
730   * by all running kthreads that need to be subject to stuttering.
731   */
stutter_wait(const char * title)732  bool stutter_wait(const char *title)
733  {
734  	bool ret = false;
735  	ktime_t till_ns;
736  
737  	cond_resched_tasks_rcu_qs();
738  	till_ns = READ_ONCE(stutter_till_abs_time);
739  	if (till_ns && ktime_before(ktime_get(), till_ns)) {
740  		torture_hrtimeout_ns(till_ns, 0, HRTIMER_MODE_ABS, NULL);
741  		ret = true;
742  	}
743  	torture_shutdown_absorb(title);
744  	return ret;
745  }
746  EXPORT_SYMBOL_GPL(stutter_wait);
747  
748  /*
749   * Cause the torture test to "stutter", starting and stopping all
750   * threads periodically.
751   */
torture_stutter(void * arg)752  static int torture_stutter(void *arg)
753  {
754  	ktime_t till_ns;
755  
756  	VERBOSE_TOROUT_STRING("torture_stutter task started");
757  	do {
758  		if (!torture_must_stop() && stutter > 1) {
759  			till_ns = ktime_add_ns(ktime_get(),
760  					       jiffies_to_nsecs(stutter));
761  			WRITE_ONCE(stutter_till_abs_time, till_ns);
762  			torture_hrtimeout_jiffies(stutter - 1, NULL);
763  		}
764  		if (!torture_must_stop())
765  			torture_hrtimeout_jiffies(stutter_gap, NULL);
766  		torture_shutdown_absorb("torture_stutter");
767  	} while (!torture_must_stop());
768  	torture_kthread_stopping("torture_stutter");
769  	return 0;
770  }
771  
772  /*
773   * Initialize and kick off the torture_stutter kthread.
774   */
torture_stutter_init(const int s,const int sgap)775  int torture_stutter_init(const int s, const int sgap)
776  {
777  	stutter = s;
778  	stutter_gap = sgap;
779  	return torture_create_kthread(torture_stutter, NULL, stutter_task);
780  }
781  EXPORT_SYMBOL_GPL(torture_stutter_init);
782  
783  /*
784   * Cleanup after the torture_stutter kthread.
785   */
torture_stutter_cleanup(void)786  static void torture_stutter_cleanup(void)
787  {
788  	if (!stutter_task)
789  		return;
790  	VERBOSE_TOROUT_STRING("Stopping torture_stutter task");
791  	kthread_stop(stutter_task);
792  	stutter_task = NULL;
793  }
794  
795  static void
torture_print_module_parms(void)796  torture_print_module_parms(void)
797  {
798  	pr_alert("torture module --- %s:  disable_onoff_at_boot=%d ftrace_dump_at_shutdown=%d verbose_sleep_frequency=%d verbose_sleep_duration=%d random_shuffle=%d\n",
799  		 torture_type, disable_onoff_at_boot, ftrace_dump_at_shutdown, verbose_sleep_frequency, verbose_sleep_duration, random_shuffle);
800  }
801  
802  /*
803   * Initialize torture module.  Please note that this is -not- invoked via
804   * the usual module_init() mechanism, but rather by an explicit call from
805   * the client torture module.  This call must be paired with a later
806   * torture_init_end().
807   *
808   * The runnable parameter points to a flag that controls whether or not
809   * the test is currently runnable.  If there is no such flag, pass in NULL.
810   */
torture_init_begin(char * ttype,int v)811  bool torture_init_begin(char *ttype, int v)
812  {
813  	mutex_lock(&fullstop_mutex);
814  	if (torture_type != NULL) {
815  		pr_alert("%s: Refusing %s init: %s running.\n",
816  			  __func__, ttype, torture_type);
817  		pr_alert("%s: One torture test at a time!\n", __func__);
818  		mutex_unlock(&fullstop_mutex);
819  		return false;
820  	}
821  	torture_type = ttype;
822  	verbose = v;
823  	fullstop = FULLSTOP_DONTSTOP;
824  	torture_print_module_parms();
825  	return true;
826  }
827  EXPORT_SYMBOL_GPL(torture_init_begin);
828  
829  /*
830   * Tell the torture module that initialization is complete.
831   */
torture_init_end(void)832  void torture_init_end(void)
833  {
834  	mutex_unlock(&fullstop_mutex);
835  	register_reboot_notifier(&torture_shutdown_nb);
836  }
837  EXPORT_SYMBOL_GPL(torture_init_end);
838  
839  /*
840   * Clean up torture module.  Please note that this is -not- invoked via
841   * the usual module_exit() mechanism, but rather by an explicit call from
842   * the client torture module.  Returns true if a race with system shutdown
843   * is detected, otherwise, all kthreads started by functions in this file
844   * will be shut down.
845   *
846   * This must be called before the caller starts shutting down its own
847   * kthreads.
848   *
849   * Both torture_cleanup_begin() and torture_cleanup_end() must be paired,
850   * in order to correctly perform the cleanup. They are separated because
851   * threads can still need to reference the torture_type type, thus nullify
852   * only after completing all other relevant calls.
853   */
torture_cleanup_begin(void)854  bool torture_cleanup_begin(void)
855  {
856  	mutex_lock(&fullstop_mutex);
857  	if (READ_ONCE(fullstop) == FULLSTOP_SHUTDOWN) {
858  		pr_warn("Concurrent rmmod and shutdown illegal!\n");
859  		mutex_unlock(&fullstop_mutex);
860  		schedule_timeout_uninterruptible(10);
861  		return true;
862  	}
863  	WRITE_ONCE(fullstop, FULLSTOP_RMMOD);
864  	mutex_unlock(&fullstop_mutex);
865  	torture_shutdown_cleanup();
866  	torture_shuffle_cleanup();
867  	torture_stutter_cleanup();
868  	torture_onoff_cleanup();
869  	return false;
870  }
871  EXPORT_SYMBOL_GPL(torture_cleanup_begin);
872  
torture_cleanup_end(void)873  void torture_cleanup_end(void)
874  {
875  	mutex_lock(&fullstop_mutex);
876  	torture_type = NULL;
877  	mutex_unlock(&fullstop_mutex);
878  }
879  EXPORT_SYMBOL_GPL(torture_cleanup_end);
880  
881  /*
882   * Is it time for the current torture test to stop?
883   */
torture_must_stop(void)884  bool torture_must_stop(void)
885  {
886  	return torture_must_stop_irq() || kthread_should_stop();
887  }
888  EXPORT_SYMBOL_GPL(torture_must_stop);
889  
890  /*
891   * Is it time for the current torture test to stop?  This is the irq-safe
892   * version, hence no check for kthread_should_stop().
893   */
torture_must_stop_irq(void)894  bool torture_must_stop_irq(void)
895  {
896  	return READ_ONCE(fullstop) != FULLSTOP_DONTSTOP;
897  }
898  EXPORT_SYMBOL_GPL(torture_must_stop_irq);
899  
900  /*
901   * Each kthread must wait for kthread_should_stop() before returning from
902   * its top-level function, otherwise segfaults ensue.  This function
903   * prints a "stopping" message and waits for kthread_should_stop(), and
904   * should be called from all torture kthreads immediately prior to
905   * returning.
906   */
torture_kthread_stopping(char * title)907  void torture_kthread_stopping(char *title)
908  {
909  	char buf[128];
910  
911  	snprintf(buf, sizeof(buf), "%s is stopping", title);
912  	VERBOSE_TOROUT_STRING(buf);
913  	while (!kthread_should_stop()) {
914  		torture_shutdown_absorb(title);
915  		schedule_timeout_uninterruptible(HZ / 20);
916  	}
917  }
918  EXPORT_SYMBOL_GPL(torture_kthread_stopping);
919  
920  /*
921   * Create a generic torture kthread that is immediately runnable.  If you
922   * need the kthread to be stopped so that you can do something to it before
923   * it starts, you will need to open-code your own.
924   */
_torture_create_kthread(int (* fn)(void * arg),void * arg,char * s,char * m,char * f,struct task_struct ** tp,void (* cbf)(struct task_struct * tp))925  int _torture_create_kthread(int (*fn)(void *arg), void *arg, char *s, char *m,
926  			    char *f, struct task_struct **tp, void (*cbf)(struct task_struct *tp))
927  {
928  	int ret = 0;
929  
930  	VERBOSE_TOROUT_STRING(m);
931  	*tp = kthread_create(fn, arg, "%s", s);
932  	if (IS_ERR(*tp)) {
933  		ret = PTR_ERR(*tp);
934  		TOROUT_ERRSTRING(f);
935  		*tp = NULL;
936  		return ret;
937  	}
938  
939  	if (cbf)
940  		cbf(*tp);
941  
942  	wake_up_process(*tp);  // Process is sleeping, so ordering provided.
943  	torture_shuffle_task_register(*tp);
944  	return ret;
945  }
946  EXPORT_SYMBOL_GPL(_torture_create_kthread);
947  
948  /*
949   * Stop a generic kthread, emitting a message.
950   */
_torture_stop_kthread(char * m,struct task_struct ** tp)951  void _torture_stop_kthread(char *m, struct task_struct **tp)
952  {
953  	if (*tp == NULL)
954  		return;
955  	VERBOSE_TOROUT_STRING(m);
956  	kthread_stop(*tp);
957  	*tp = NULL;
958  }
959  EXPORT_SYMBOL_GPL(_torture_stop_kthread);
960