1 // SPDX-License-Identifier: GPL-2.0
3  * Timer events oriented CPU idle governor
5  * Copyright (C) 2018 - 2021 Intel Corporation
10  * DOC: teo-description
15  * wakeups from idle states.  Moreover, information about what happened in the
17  * idle state with target residency within the (known) time till the closest
19  * the upcoming CPU idle period and, if not, then which of the shallower idle
20  * states to choose instead of it.
22  * Of course, non-timer wakeup sources are more important in some use cases
23  * which can be covered by taking a few most recent idle time intervals of the
25  * consider idle duration values greater than the sleep length, because the
29  * Thus this governor estimates whether or not the prospective idle duration of
31  * an idle state for it accordingly.
35  * idle states provided by the %CPUIdle driver in the ascending order.  That is,
37  * the second idle state (idle state 1), the second bin spans from the target
38  * residency of idle state 1 up to, but not including, the target residency of
39  * idle state 2, the third bin spans from the target residency of idle state 2
40  * up to, but not including, the target residency of idle state 3 and so on.
41  * The last bin spans from the target residency of the deepest idle state
45  * They are updated every time before selecting an idle state for the given CPU
49  * sleep length and the idle duration measured after CPU wakeup fall into the
52  * situations in which the measured idle duration is so much shorter than the
53  * sleep length that the bin it falls into corresponds to an idle state
57  * In order to select an idle state for a CPU, the governor takes the following
61  * 1. Find the deepest CPU idle state whose target residency does not exceed
62  *    the current sleep length (the candidate idle state) and compute 2 sums as
65  *    - The sum of the "hits" and "intercepts" metrics for the candidate state
66  *      and all of the deeper idle states (it represents the cases in which the
67  *      CPU was idle long enough to avoid being intercepted if the sleep length
70  *    - The sum of the "intercepts" metrics for all of the idle states shallower
72  *      idle long enough to avoid being intercepted if the sleep length had been
76  *    up early, so look for an alternative idle state to select.
78  *    - Traverse the idle states shallower than the candidate one in the
81  *    - For each of them compute the sum of the "intercepts" metrics over all
82  *      of the idle states between it and the candidate one (including the
85  *    - If each of these sums that needs to be taken into account (because the
89  *      not exceeded the idle duration in over a half of the relevant cases),
90  *      select the given idle state instead of the candidate one.
111  * struct teo_bin - Metrics used by the TEO cpuidle governor.
121  * struct teo_cpu - CPU data used by the TEO cpuidle governor.
122  * @time_span_ns: Time between idle state selection and post-wakeup update.
124  * @state_bins: Idle state data bins for this CPU.
139  * teo_update - Update CPU metrics after wakeup.
145 	struct teo_cpu *cpu_data = per_cpu_ptr(&teo_cpus, dev->cpu);  in teo_update()
150 	if (cpu_data->time_span_ns >= cpu_data->sleep_length_ns) {  in teo_update()
153 		 * enough to the closest timer event expected at the idle state  in teo_update()
158 		u64 lat_ns = drv->states[dev->last_state_idx].exit_latency_ns;  in teo_update()
162 		 * (saved) time till the next timer event and the measured idle  in teo_update()
167 		measured_ns = dev->last_residency_ns;  in teo_update()
170 		 * executed by the CPU is not likely to be worst-case every  in teo_update()
175 			measured_ns -= lat_ns / 2;  in teo_update()
180 	cpu_data->total = 0;  in teo_update()
184 	 * find the bins that the sleep length and the measured idle duration  in teo_update()
187 	for (i = 0; i < drv->state_count; i++) {  in teo_update()
188 		struct teo_bin *bin = &cpu_data->state_bins[i];  in teo_update()
190 		bin->hits -= bin->hits >> DECAY_SHIFT;  in teo_update()
191 		bin->intercepts -= bin->intercepts >> DECAY_SHIFT;  in teo_update()
193 		cpu_data->total += bin->hits + bin->intercepts;  in teo_update()
195 		target_residency_ns = drv->states[i].target_residency_ns;  in teo_update()
197 		if (target_residency_ns <= cpu_data->sleep_length_ns) {  in teo_update()
207 	 * to stop the tick.  This effectively adds an extra hits-only bin  in teo_update()
208 	 * beyond the last state-related one.  in teo_update()
211 		cpu_data->tick_hits -= cpu_data->tick_hits >> DECAY_SHIFT;  in teo_update()
213 		cpu_data->total += cpu_data->tick_hits;  in teo_update()
215 		if (TICK_NSEC <= cpu_data->sleep_length_ns) {  in teo_update()
216 			idx_timer = drv->state_count;  in teo_update()
218 				cpu_data->tick_hits += PULSE;  in teo_update()
225 	 * If the measured idle duration falls into the same bin as the sleep  in teo_update()
228 	 * the measured idle duration.  in teo_update()
231 		cpu_data->state_bins[idx_timer].hits += PULSE;  in teo_update()
233 		cpu_data->state_bins[idx_duration].intercepts += PULSE;  in teo_update()
236 	cpu_data->total += PULSE;  in teo_update()
242 		drv->states[i].target_residency_ns >= TICK_NSEC;  in teo_state_ok()
246  * teo_find_shallower_state - Find shallower idle state matching given duration.
249  * @state_idx: Index of the capping idle state.
250  * @duration_ns: Idle duration value to match.
251  * @no_poll: Don't consider polling states.
259 	for (i = state_idx - 1; i >= 0; i--) {  in teo_find_shallower_state()
260 		if (dev->states_usage[i].disable ||  in teo_find_shallower_state()
261 				(no_poll && drv->states[i].flags & CPUIDLE_FLAG_POLLING))  in teo_find_shallower_state()
265 		if (drv->states[i].target_residency_ns <= duration_ns)  in teo_find_shallower_state()
272  * teo_select - Selects the next idle state to enter.
280 	struct teo_cpu *cpu_data = per_cpu_ptr(&teo_cpus, dev->cpu);  in teo_select()
281 	s64 latency_req = cpuidle_governor_latency_req(dev->cpu);  in teo_select()
289 	int idx0 = 0, idx = -1;  in teo_select()
294 	if (dev->last_state_idx >= 0) {  in teo_select()
296 		dev->last_state_idx = -1;  in teo_select()
299 	cpu_data->time_span_ns = local_clock();  in teo_select()
304 	cpu_data->sleep_length_ns = KTIME_MAX;  in teo_select()
307 	if (drv->state_count < 2) {  in teo_select()
312 	if (!dev->states_usage[0].disable)  in teo_select()
316 	for (i = 1; i < drv->state_count; i++) {  in teo_select()
317 		struct teo_bin *prev_bin = &cpu_data->state_bins[i-1];  in teo_select()
318 		struct cpuidle_state *s = &drv->states[i];  in teo_select()
321 		 * Update the sums of idle state mertics for all of the states  in teo_select()
324 		intercept_sum += prev_bin->intercepts;  in teo_select()
325 		hit_sum += prev_bin->hits;  in teo_select()
327 		if (dev->states_usage[i].disable)  in teo_select()
335 		if (s->exit_latency_ns <= latency_req)  in teo_select()
345 		idx = 0; /* No states enabled, must use 0. */  in teo_select()
351 		 * Only one idle state is enabled, so use it, but do not  in teo_select()
354 		duration_ns = drv->states[idx].target_residency_ns;  in teo_select()
359 			cpu_data->state_bins[drv->state_count-1].intercepts;  in teo_select()
362 	 * If the sum of the intercepts metric for all of the idle states  in teo_select()
365 	 * all of the deeper states a shallower idle state is likely to be a  in teo_select()
369 	if (2 * idx_intercept_sum > cpu_data->total - idx_hit_sum) {  in teo_select()
373 		 * Look for the deepest idle state whose target residency had  in teo_select()
374 		 * not exceeded the idle duration in over a half of the relevant  in teo_select()
382 		for (i = idx - 1; i >= 0; i--) {  in teo_select()
383 			struct teo_bin *bin = &cpu_data->state_bins[i];  in teo_select()
385 			intercept_sum += bin->intercepts;  in teo_select()
394 				    !dev->states_usage[i].disable)  in teo_select()
402 			if (dev->states_usage[i].disable)  in teo_select()
426 		cpu_data->sleep_length_ns = duration_ns;  in teo_select()
432 	 * idle state shallower than the current candidate one.  in teo_select()
439 	 * because an immediate non-timer wakeup is expected in that case.  in teo_select()
449 	if ((drv->states[0].flags & CPUIDLE_FLAG_POLLING) &&  in teo_select()
450 	    drv->states[idx].target_residency_ns < RESIDENCY_THRESHOLD_NS)  in teo_select()
454 	cpu_data->sleep_length_ns = duration_ns;  in teo_select()
460 	if (drv->states[idx].target_residency_ns > duration_ns) {  in teo_select()
471 	if (drv->states[idx].target_residency_ns < TICK_NSEC &&  in teo_select()
472 	    tick_intercept_sum > cpu_data->total / 2 + cpu_data->total / 8)  in teo_select()
478 	 * one or the expected idle duration is shorter than the tick period  in teo_select()
481 	if ((!(drv->states[idx].flags & CPUIDLE_FLAG_POLLING) &&  in teo_select()
491 	    drv->states[idx].target_residency_ns > delta_tick)  in teo_select()
500  * teo_reflect - Note that governor data for the CPU need to be updated.
506 	struct teo_cpu *cpu_data = per_cpu_ptr(&teo_cpus, dev->cpu);  in teo_reflect()
508 	dev->last_state_idx = state;  in teo_reflect()
511 	 * nets, assume that the CPU might have been idle for the entire sleep  in teo_reflect()
514 	if (dev->poll_time_limit ||  in teo_reflect()
515 	    (tick_nohz_idle_got_tick() && cpu_data->sleep_length_ns > TICK_NSEC)) {  in teo_reflect()
516 		dev->poll_time_limit = false;  in teo_reflect()
517 		cpu_data->time_span_ns = cpu_data->sleep_length_ns;  in teo_reflect()
519 		cpu_data->time_span_ns = local_clock() - cpu_data->time_span_ns;  in teo_reflect()
524  * teo_enable_device - Initialize the governor's data for the target CPU.
531 	struct teo_cpu *cpu_data = per_cpu_ptr(&teo_cpus, dev->cpu);  in teo_enable_device()