1 // SPDX-License-Identifier: GPL-2.0
3  * CPUFreq governor based on scheduler-provided CPU utilization data.
52 	/* The field below is for single-CPU policies only: */
67 	 * Since cpufreq_update_util() is called with rq->lock held for  in sugov_should_update_freq()
68 	 * the @target_cpu, our per-CPU data is fully serialized.  in sugov_should_update_freq()
70 	 * However, drivers cannot in general deal with cross-CPU  in sugov_should_update_freq()
81 	if (!cpufreq_this_cpu_can_update(sg_policy->policy))  in sugov_should_update_freq()
84 	if (unlikely(sg_policy->limits_changed)) {  in sugov_should_update_freq()
85 		sg_policy->limits_changed = false;  in sugov_should_update_freq()
86 		sg_policy->need_freq_update = true;  in sugov_should_update_freq()
90 	delta_ns = time - sg_policy->last_freq_update_time;  in sugov_should_update_freq()
92 	return delta_ns >= sg_policy->freq_update_delay_ns;  in sugov_should_update_freq()
98 	if (sg_policy->need_freq_update)  in sugov_update_next_freq()
99 		sg_policy->need_freq_update = cpufreq_driver_test_flags(CPUFREQ_NEED_UPDATE_LIMITS);  in sugov_update_next_freq()
100 	else if (sg_policy->next_freq == next_freq)  in sugov_update_next_freq()
103 	sg_policy->next_freq = next_freq;  in sugov_update_next_freq()
104 	sg_policy->last_freq_update_time = time;  in sugov_update_next_freq()
111 	if (!sg_policy->work_in_progress) {  in sugov_deferred_update()
112 		sg_policy->work_in_progress = true;  in sugov_deferred_update()
113 		irq_work_queue(&sg_policy->irq_work);  in sugov_deferred_update()
118  * get_capacity_ref_freq - get the reference frequency that has been used to
128 	unsigned int freq = arch_scale_freq_ref(policy->cpu);  in get_capacity_ref_freq()  local
130 	if (freq)  in get_capacity_ref_freq()
131 		return freq;  in get_capacity_ref_freq()
134 		return policy->cpuinfo.max_freq;  in get_capacity_ref_freq()
140 	return policy->cur + (policy->cur >> 2);  in get_capacity_ref_freq()
144  * get_next_freq - Compute a new frequency for a given cpufreq policy.
147  * @max: CPU capacity.
149  * If the utilization is frequency-invariant, choose the new frequency to be
152  * next_freq = C * max_freq * util / max
154  * Otherwise, approximate the would-be frequency-invariant utilization by
157  * next_freq = C * curr_freq * util_raw / max
159  * Take C = 1.25 for the frequency tipping point at (util / max) = 0.8.
161  * The lowest driver-supported frequency which is equal or greater than the raw
162  * next_freq (as calculated above) is returned, subject to policy min/max and
166 				  unsigned long util, unsigned long max)  in get_next_freq()  argument
168 	struct cpufreq_policy *policy = sg_policy->policy;  in get_next_freq()
169 	unsigned int freq;  in get_next_freq()  local
171 	freq = get_capacity_ref_freq(policy);  in get_next_freq()
172 	freq = map_util_freq(util, freq, max);  in get_next_freq()
174 	if (freq == sg_policy->cached_raw_freq && !sg_policy->need_freq_update)  in get_next_freq()
175 		return sg_policy->next_freq;  in get_next_freq()
177 	sg_policy->cached_raw_freq = freq;  in get_next_freq()
178 	return cpufreq_driver_resolve_freq(policy, freq);  in get_next_freq()
183 				 unsigned long max)  in sugov_effective_cpu_perf()  argument
187 	/* Actually we don't need to target the max performance */  in sugov_effective_cpu_perf()
188 	if (actual < max)  in sugov_effective_cpu_perf()
189 		max = actual;  in sugov_effective_cpu_perf()
195 	return max(min, max);  in sugov_effective_cpu_perf()
200 	unsigned long min, max, util = scx_cpuperf_target(sg_cpu->cpu);  in sugov_get_util()  local
203 		util += cpu_util_cfs_boost(sg_cpu->cpu);  in sugov_get_util()
204 	util = effective_cpu_util(sg_cpu->cpu, util, &min, &max);  in sugov_get_util()
205 	util = max(util, boost);  in sugov_get_util()
206 	sg_cpu->bw_min = min;  in sugov_get_util()
207 	sg_cpu->util = sugov_effective_cpu_perf(sg_cpu->cpu, util, min, max);  in sugov_get_util()
211  * sugov_iowait_reset() - Reset the IO boost status of a CPU.
224 	s64 delta_ns = time - sg_cpu->last_update;  in sugov_iowait_reset()
230 	sg_cpu->iowait_boost = set_iowait_boost ? IOWAIT_BOOST_MIN : 0;  in sugov_iowait_reset()
231 	sg_cpu->iowait_boost_pending = set_iowait_boost;  in sugov_iowait_reset()
237  * sugov_iowait_boost() - Updates the IO boost status of a CPU.
256 	if (sg_cpu->iowait_boost &&  in sugov_iowait_boost()
265 	if (sg_cpu->iowait_boost_pending)  in sugov_iowait_boost()
267 	sg_cpu->iowait_boost_pending = true;  in sugov_iowait_boost()
270 	if (sg_cpu->iowait_boost) {  in sugov_iowait_boost()
271 		sg_cpu->iowait_boost =  in sugov_iowait_boost()
272 			min_t(unsigned int, sg_cpu->iowait_boost << 1, SCHED_CAPACITY_SCALE);  in sugov_iowait_boost()
277 	sg_cpu->iowait_boost = IOWAIT_BOOST_MIN;  in sugov_iowait_boost()
281  * sugov_iowait_apply() - Apply the IO boost to a CPU.
284  * @max_cap: the max CPU capacity
302 	if (!sg_cpu->iowait_boost)  in sugov_iowait_apply()
309 	if (!sg_cpu->iowait_boost_pending) {  in sugov_iowait_apply()
313 		sg_cpu->iowait_boost >>= 1;  in sugov_iowait_apply()
314 		if (sg_cpu->iowait_boost < IOWAIT_BOOST_MIN) {  in sugov_iowait_apply()
315 			sg_cpu->iowait_boost = 0;  in sugov_iowait_apply()
320 	sg_cpu->iowait_boost_pending = false;  in sugov_iowait_apply()
323 	 * sg_cpu->util is already in capacity scale; convert iowait_boost  in sugov_iowait_apply()
326 	return (sg_cpu->iowait_boost * max_cap) >> SCHED_CAPACITY_SHIFT;  in sugov_iowait_apply()
344 	if (uclamp_rq_is_capped(cpu_rq(sg_cpu->cpu)))  in sugov_hold_freq()
351 	idle_calls = tick_nohz_get_idle_calls_cpu(sg_cpu->cpu);  in sugov_hold_freq()
352 	ret = idle_calls == sg_cpu->saved_idle_calls;  in sugov_hold_freq()
354 	sg_cpu->saved_idle_calls = idle_calls;  in sugov_hold_freq()
367 	if (cpu_bw_dl(cpu_rq(sg_cpu->cpu)) > sg_cpu->bw_min)  in ignore_dl_rate_limit()
368 		sg_cpu->sg_policy->limits_changed = true;  in ignore_dl_rate_limit()
378 	sg_cpu->last_update = time;  in sugov_update_single_common()
382 	if (!sugov_should_update_freq(sg_cpu->sg_policy, time))  in sugov_update_single_common()
395 	struct sugov_policy *sg_policy = sg_cpu->sg_policy;  in sugov_update_single_freq()
396 	unsigned int cached_freq = sg_policy->cached_raw_freq;  in sugov_update_single_freq()
400 	max_cap = arch_scale_cpu_capacity(sg_cpu->cpu);  in sugov_update_single_freq()
405 	next_f = get_next_freq(sg_policy, sg_cpu->util, max_cap);  in sugov_update_single_freq()
407 	if (sugov_hold_freq(sg_cpu) && next_f < sg_policy->next_freq &&  in sugov_update_single_freq()
408 	    !sg_policy->need_freq_update) {  in sugov_update_single_freq()
409 		next_f = sg_policy->next_freq;  in sugov_update_single_freq()
411 		/* Restore cached freq as next_freq has changed */  in sugov_update_single_freq()
412 		sg_policy->cached_raw_freq = cached_freq;  in sugov_update_single_freq()
419 	 * This code runs under rq->lock for the target CPU, so it won't run  in sugov_update_single_freq()
423 	if (sg_policy->policy->fast_switch_enabled) {  in sugov_update_single_freq()
424 		cpufreq_driver_fast_switch(sg_policy->policy, next_f);  in sugov_update_single_freq()
426 		raw_spin_lock(&sg_policy->update_lock);  in sugov_update_single_freq()
428 		raw_spin_unlock(&sg_policy->update_lock);  in sugov_update_single_freq()
436 	unsigned long prev_util = sg_cpu->util;  in sugov_update_single_perf()
449 	max_cap = arch_scale_cpu_capacity(sg_cpu->cpu);  in sugov_update_single_perf()
454 	if (sugov_hold_freq(sg_cpu) && sg_cpu->util < prev_util)  in sugov_update_single_perf()
455 		sg_cpu->util = prev_util;  in sugov_update_single_perf()
457 	cpufreq_driver_adjust_perf(sg_cpu->cpu, sg_cpu->bw_min,  in sugov_update_single_perf()
458 				   sg_cpu->util, max_cap);  in sugov_update_single_perf()
460 	sg_cpu->sg_policy->last_freq_update_time = time;  in sugov_update_single_perf()
465 	struct sugov_policy *sg_policy = sg_cpu->sg_policy;  in sugov_next_freq_shared()
466 	struct cpufreq_policy *policy = sg_policy->policy;  in sugov_next_freq_shared()
470 	max_cap = arch_scale_cpu_capacity(sg_cpu->cpu);  in sugov_next_freq_shared()
472 	for_each_cpu(j, policy->cpus) {  in sugov_next_freq_shared()
479 		util = max(j_sg_cpu->util, util);  in sugov_next_freq_shared()
489 	struct sugov_policy *sg_policy = sg_cpu->sg_policy;  in sugov_update_shared()
492 	raw_spin_lock(&sg_policy->update_lock);  in sugov_update_shared()
495 	sg_cpu->last_update = time;  in sugov_update_shared()
505 		if (sg_policy->policy->fast_switch_enabled)  in sugov_update_shared()
506 			cpufreq_driver_fast_switch(sg_policy->policy, next_f);  in sugov_update_shared()
511 	raw_spin_unlock(&sg_policy->update_lock);  in sugov_update_shared()
517 	unsigned int freq;  in sugov_work()  local
521 	 * Hold sg_policy->update_lock shortly to handle the case where:  in sugov_work()
522 	 * in case sg_policy->next_freq is read here, and then updated by  in sugov_work()
530 	raw_spin_lock_irqsave(&sg_policy->update_lock, flags);  in sugov_work()
531 	freq = sg_policy->next_freq;  in sugov_work()
532 	sg_policy->work_in_progress = false;  in sugov_work()
533 	raw_spin_unlock_irqrestore(&sg_policy->update_lock, flags);  in sugov_work()
535 	mutex_lock(&sg_policy->work_lock);  in sugov_work()
536 	__cpufreq_driver_target(sg_policy->policy, freq, CPUFREQ_RELATION_L);  in sugov_work()
537 	mutex_unlock(&sg_policy->work_lock);  in sugov_work()
546 	kthread_queue_work(&sg_policy->worker, &sg_policy->work);  in sugov_irq_work()
563 	return sprintf(buf, "%u\n", tunables->rate_limit_us);  in rate_limit_us_show()
574 		return -EINVAL;  in rate_limit_us_store()
576 	tunables->rate_limit_us = rate_limit_us;  in rate_limit_us_store()
578 	list_for_each_entry(sg_policy, &attr_set->policy_list, tunables_hook)  in rate_limit_us_store()
579 		sg_policy->freq_update_delay_ns = rate_limit_us * NSEC_PER_USEC;  in rate_limit_us_store()
642 	sg_policy->policy = policy;  in sugov_policy_alloc()
643 	raw_spin_lock_init(&sg_policy->update_lock);  in sugov_policy_alloc()
669 	struct cpufreq_policy *policy = sg_policy->policy;  in sugov_kthread_create()
673 	if (policy->fast_switch_enabled)  in sugov_kthread_create()
676 	kthread_init_work(&sg_policy->work, sugov_work);  in sugov_kthread_create()
677 	kthread_init_worker(&sg_policy->worker);  in sugov_kthread_create()
678 	thread = kthread_create(kthread_worker_fn, &sg_policy->worker,  in sugov_kthread_create()
680 				cpumask_first(policy->related_cpus));  in sugov_kthread_create()
693 	sg_policy->thread = thread;  in sugov_kthread_create()
694 	kthread_bind_mask(thread, policy->related_cpus);  in sugov_kthread_create()
695 	init_irq_work(&sg_policy->irq_work, sugov_irq_work);  in sugov_kthread_create()
696 	mutex_init(&sg_policy->work_lock);  in sugov_kthread_create()
706 	if (sg_policy->policy->fast_switch_enabled)  in sugov_kthread_stop()
709 	kthread_flush_worker(&sg_policy->worker);  in sugov_kthread_stop()
710 	kthread_stop(sg_policy->thread);  in sugov_kthread_stop()
711 	mutex_destroy(&sg_policy->work_lock);  in sugov_kthread_stop()
720 		gov_attr_set_init(&tunables->attr_set, &sg_policy->tunables_hook);  in sugov_tunables_alloc()
740 	if (policy->governor_data)  in sugov_init()
741 		return -EBUSY;  in sugov_init()
747 		ret = -ENOMEM;  in sugov_init()
759 			ret = -EINVAL;  in sugov_init()
762 		policy->governor_data = sg_policy;  in sugov_init()
763 		sg_policy->tunables = global_tunables;  in sugov_init()
765 		gov_attr_set_get(&global_tunables->attr_set, &sg_policy->tunables_hook);  in sugov_init()
771 		ret = -ENOMEM;  in sugov_init()
775 	tunables->rate_limit_us = cpufreq_policy_transition_delay_us(policy);  in sugov_init()
777 	policy->governor_data = sg_policy;  in sugov_init()
778 	sg_policy->tunables = tunables;  in sugov_init()
780 	ret = kobject_init_and_add(&tunables->attr_set.kobj, &sugov_tunables_ktype,  in sugov_init()
793 	kobject_put(&tunables->attr_set.kobj);  in sugov_init()
794 	policy->governor_data = NULL;  in sugov_init()
813 	struct sugov_policy *sg_policy = policy->governor_data;  in sugov_exit()
814 	struct sugov_tunables *tunables = sg_policy->tunables;  in sugov_exit()
819 	count = gov_attr_set_put(&tunables->attr_set, &sg_policy->tunables_hook);  in sugov_exit()
820 	policy->governor_data = NULL;  in sugov_exit()
835 	struct sugov_policy *sg_policy = policy->governor_data;  in sugov_start()
839 	sg_policy->freq_update_delay_ns	= sg_policy->tunables->rate_limit_us * NSEC_PER_USEC;  in sugov_start()
840 	sg_policy->last_freq_update_time	= 0;  in sugov_start()
841 	sg_policy->next_freq			= 0;  in sugov_start()
842 	sg_policy->work_in_progress		= false;  in sugov_start()
843 	sg_policy->limits_changed		= false;  in sugov_start()
844 	sg_policy->cached_raw_freq		= 0;  in sugov_start()
846 	sg_policy->need_freq_update = cpufreq_driver_test_flags(CPUFREQ_NEED_UPDATE_LIMITS);  in sugov_start()
850 	else if (policy->fast_switch_enabled && cpufreq_driver_has_adjust_perf())  in sugov_start()
855 	for_each_cpu(cpu, policy->cpus) {  in sugov_start()
859 		sg_cpu->cpu = cpu;  in sugov_start()
860 		sg_cpu->sg_policy = sg_policy;  in sugov_start()
861 		cpufreq_add_update_util_hook(cpu, &sg_cpu->update_util, uu);  in sugov_start()
868 	struct sugov_policy *sg_policy = policy->governor_data;  in sugov_stop()
871 	for_each_cpu(cpu, policy->cpus)  in sugov_stop()
876 	if (!policy->fast_switch_enabled) {  in sugov_stop()
877 		irq_work_sync(&sg_policy->irq_work);  in sugov_stop()
878 		kthread_cancel_work_sync(&sg_policy->work);  in sugov_stop()
884 	struct sugov_policy *sg_policy = policy->governor_data;  in sugov_limits()
886 	if (!policy->fast_switch_enabled) {  in sugov_limits()
887 		mutex_lock(&sg_policy->work_lock);  in sugov_limits()
889 		mutex_unlock(&sg_policy->work_lock);  in sugov_limits()
892 	sg_policy->limits_changed = true;  in sugov_limits()