1 // SPDX-License-Identifier: GPL-2.0
8  * states for a specified fraction of time over a specified period.
13  * All of the kthreads used for idle injection are created at init time.
19  * The idle + run duration is specified via separate helpers and that allows
34  * It is up to the user of this framework to provide a lock for higher-level
51  * struct idle_inject_thread - task on/off switch structure
53  * @should_run: whether or not to run the task (for the smpboot kthread API)
61  * struct idle_inject_device - idle injection data
63  * @idle_duration_us: duration of CPU idle time to inject
64  * @run_duration_us: duration of CPU run time to allow
71  * instance has an idle duration, a run duration and mask of CPUs to inject
74  * Actual CPU idle time is injected by calling kernel scheduler interface
78  * update() - This callback is invoked just before waking up CPUs to inject
80  * cycle. It also allows the caller to readjust the idle and run duration by
96  * idle_inject_wakeup - Wake up idle injection threads
107 	for_each_cpu_and(cpu, to_cpumask(ii_dev->cpumask), cpu_online_mask) {  in idle_inject_wakeup()
109 		iit->should_run = 1;  in idle_inject_wakeup()
110 		wake_up_process(iit->tsk);  in idle_inject_wakeup()
115  * idle_inject_timer_fn - idle injection timer function
120  * play_idle_precise() to inject a specified amount of CPU idle time.
130 	if (!ii_dev->update || ii_dev->update())  in idle_inject_timer_fn()
133 	duration_us = READ_ONCE(ii_dev->run_duration_us);  in idle_inject_timer_fn()
134 	duration_us += READ_ONCE(ii_dev->idle_duration_us);  in idle_inject_timer_fn()
142  * idle_inject_fn - idle injection work function
146  * idle time.
157 	 * Let the smpboot main loop know that the task should not run again.  in idle_inject_fn()
159 	iit->should_run = 0;  in idle_inject_fn()
161 	play_idle_precise(READ_ONCE(ii_dev->idle_duration_us) * NSEC_PER_USEC,  in idle_inject_fn()
162 			  READ_ONCE(ii_dev->latency_us) * NSEC_PER_USEC);  in idle_inject_fn()
166  * idle_inject_set_duration - idle and run duration update helper
168  * @run_duration_us: CPU run time to allow in microseconds
169  * @idle_duration_us: CPU idle time to inject in microseconds
176 		WRITE_ONCE(ii_dev->run_duration_us, run_duration_us);  in idle_inject_set_duration()
177 		WRITE_ONCE(ii_dev->idle_duration_us, idle_duration_us);  in idle_inject_set_duration()
185  * idle_inject_get_duration - idle and run duration retrieval helper
187  * @run_duration_us: memory location to store the current CPU run time
188  * @idle_duration_us: memory location to store the current CPU idle time
194 	*run_duration_us = READ_ONCE(ii_dev->run_duration_us);  in idle_inject_get_duration()
195 	*idle_duration_us = READ_ONCE(ii_dev->idle_duration_us);  in idle_inject_get_duration()
200  * idle_inject_set_latency - set the maximum latency allowed
207 	WRITE_ONCE(ii_dev->latency_us, latency_us);  in idle_inject_set_latency()
212  * idle_inject_start - start idle injections
216  * injection kthreads associated with @ii_dev to let them inject CPU idle time
219  * Return: -EINVAL if the CPU idle or CPU run time is not set or 0 on success.
223 	unsigned int idle_duration_us = READ_ONCE(ii_dev->idle_duration_us);  in idle_inject_start()
224 	unsigned int run_duration_us = READ_ONCE(ii_dev->run_duration_us);  in idle_inject_start()
227 		return -EINVAL;  in idle_inject_start()
230 		 cpumask_pr_args(to_cpumask(ii_dev->cpumask)));  in idle_inject_start()
234 	hrtimer_start(&ii_dev->timer,  in idle_inject_start()
244  * idle_inject_stop - stops idle injections
248  * If CPU idle time is being injected when this function runs, then it will
260 		 cpumask_pr_args(to_cpumask(ii_dev->cpumask)));  in idle_inject_stop()
262 	hrtimer_cancel(&ii_dev->timer);  in idle_inject_stop()
279 	for_each_cpu(cpu, to_cpumask(ii_dev->cpumask)) {  in idle_inject_stop()
281 		iit->should_run = 0;  in idle_inject_stop()
283 		wait_task_inactive(iit->tsk, TASK_ANY);  in idle_inject_stop()
291  * idle_inject_setup - prepare the current task for idle injection
303  * idle_inject_should_run - function helper for the smpboot API
306  * Return: whether or not the thread can run.
313 	return iit->should_run;  in idle_inject_should_run()
317  * idle_inject_register_full - initialize idle injection on a set of CPUs
341 	cpumask_copy(to_cpumask(ii_dev->cpumask), cpumask);  in idle_inject_register_full()
342 	hrtimer_init(&ii_dev->timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);  in idle_inject_register_full()
343 	ii_dev->timer.function = idle_inject_timer_fn;  in idle_inject_register_full()
344 	ii_dev->latency_us = UINT_MAX;  in idle_inject_register_full()
345 	ii_dev->update = update;  in idle_inject_register_full()
347 	for_each_cpu(cpu, to_cpumask(ii_dev->cpumask)) {  in idle_inject_register_full()
360 	for_each_cpu(cpu_rb, to_cpumask(ii_dev->cpumask)) {  in idle_inject_register_full()
373  * idle_inject_register - initialize idle injection on a set of CPUs
390  * idle_inject_unregister - unregister idle injection control device
403 	for_each_cpu(cpu, to_cpumask(ii_dev->cpumask))  in idle_inject_unregister()