Lines Matching full:coupled
3 * coupled.c - helper functions to enter the same idle state on multiple cpus
21 * DOC: Coupled cpuidle states
30 * WFI), and one or more "coupled" power states that affect blocks
32 * sometimes the whole SoC). Entering a coupled power state must
36 * WFI state until all cpus are ready to enter a coupled state, at
37 * which point the coupled state function will be called on all
46 * ready counter matches the number of online coupled cpus. If any
50 * requested_state stores the deepest coupled idle state each cpu
56 * and only read after all the cpus are ready for the coupled idle
60 * of cpus in the coupled set that are currently or soon will be
62 * the waiting loop, in the ready loop, or in the coupled idle state.
64 * or in the coupled idle state.
66 * To use coupled cpuidle states, a cpuidle driver must:
69 * coupled cpus, usually the same as cpu_possible_mask if all cpus
74 * coupled state. This is usually WFI.
88 * struct cpuidle_coupled - data for set of cpus that share a coupled idle state
89 * @coupled_cpus: mask of cpus that are part of the coupled set
90 * @requested_state: array of requested states for cpus in the coupled set
95 * @prevent: flag to prevent coupled idle while a cpu is hotplugging
132 * cpuidle_coupled_parallel_barrier - synchronize all online coupled cpus
137 * cpus in the same coupled group have called this function. Once any caller
144 * Must only be called from within a coupled idle state handler
151 int n = dev->coupled->online_count; in cpuidle_coupled_parallel_barrier()
169 * cpuidle_state_is_coupled - check if a state is part of a coupled set
173 * Returns true if the target state is coupled with cpus besides this one
181 * cpuidle_coupled_state_verify - check if the coupled states are correctly set.
185 * * -EINVAL if any coupled state(safe_state_index) is wrongly set.
204 * @coupled: the struct coupled that contains the current cpu
206 static inline void cpuidle_coupled_set_ready(struct cpuidle_coupled *coupled) in cpuidle_coupled_set_ready() argument
208 atomic_add(MAX_WAITING_CPUS, &coupled->ready_waiting_counts); in cpuidle_coupled_set_ready()
213 * @coupled: the struct coupled that contains the current cpu
219 * down from the number of online cpus without going through the coupled idle
226 inline int cpuidle_coupled_set_not_ready(struct cpuidle_coupled *coupled) in cpuidle_coupled_set_not_ready() argument
231 all = coupled->online_count | (coupled->online_count << WAITING_BITS); in cpuidle_coupled_set_not_ready()
232 ret = atomic_add_unless(&coupled->ready_waiting_counts, in cpuidle_coupled_set_not_ready()
239 * cpuidle_coupled_no_cpus_ready - check if no cpus in a coupled set are ready
240 * @coupled: the struct coupled that contains the current cpu
242 * Returns true if all of the cpus in a coupled set are out of the ready loop.
244 static inline int cpuidle_coupled_no_cpus_ready(struct cpuidle_coupled *coupled) in cpuidle_coupled_no_cpus_ready() argument
246 int r = atomic_read(&coupled->ready_waiting_counts) >> WAITING_BITS; in cpuidle_coupled_no_cpus_ready()
251 * cpuidle_coupled_cpus_ready - check if all cpus in a coupled set are ready
252 * @coupled: the struct coupled that contains the current cpu
254 * Returns true if all cpus coupled to this target state are in the ready loop
256 static inline bool cpuidle_coupled_cpus_ready(struct cpuidle_coupled *coupled) in cpuidle_coupled_cpus_ready() argument
258 int r = atomic_read(&coupled->ready_waiting_counts) >> WAITING_BITS; in cpuidle_coupled_cpus_ready()
259 return r == coupled->online_count; in cpuidle_coupled_cpus_ready()
263 * cpuidle_coupled_cpus_waiting - check if all cpus in a coupled set are waiting
264 * @coupled: the struct coupled that contains the current cpu
266 * Returns true if all cpus coupled to this target state are in the wait loop
268 static inline bool cpuidle_coupled_cpus_waiting(struct cpuidle_coupled *coupled) in cpuidle_coupled_cpus_waiting() argument
270 int w = atomic_read(&coupled->ready_waiting_counts) & WAITING_MASK; in cpuidle_coupled_cpus_waiting()
271 return w == coupled->online_count; in cpuidle_coupled_cpus_waiting()
275 * cpuidle_coupled_no_cpus_waiting - check if no cpus in coupled set are waiting
276 * @coupled: the struct coupled that contains the current cpu
278 * Returns true if all of the cpus in a coupled set are out of the waiting loop.
280 static inline int cpuidle_coupled_no_cpus_waiting(struct cpuidle_coupled *coupled) in cpuidle_coupled_no_cpus_waiting() argument
282 int w = atomic_read(&coupled->ready_waiting_counts) & WAITING_MASK; in cpuidle_coupled_no_cpus_waiting()
289 * @coupled: the struct coupled that contains the current cpu
291 * Returns the deepest idle state that all coupled cpus can enter
294 struct cpuidle_coupled *coupled) in cpuidle_coupled_get_state() argument
306 for_each_cpu(i, &coupled->coupled_cpus) in cpuidle_coupled_get_state()
307 if (cpu_online(i) && coupled->requested_state[i] < state) in cpuidle_coupled_get_state()
308 state = coupled->requested_state[i]; in cpuidle_coupled_get_state()
343 * @coupled: the struct coupled that contains the current cpu
348 struct cpuidle_coupled *coupled) in cpuidle_coupled_poke_others() argument
352 for_each_cpu(cpu, &coupled->coupled_cpus) in cpuidle_coupled_poke_others()
360 * @coupled: the struct coupled that contains the current cpu
367 struct cpuidle_coupled *coupled, int next_state) in cpuidle_coupled_set_waiting() argument
369 coupled->requested_state[cpu] = next_state; in cpuidle_coupled_set_waiting()
375 return atomic_inc_return(&coupled->ready_waiting_counts) & WAITING_MASK; in cpuidle_coupled_set_waiting()
381 * @coupled: the struct coupled that contains the current cpu
386 struct cpuidle_coupled *coupled) in cpuidle_coupled_set_not_waiting() argument
394 atomic_dec(&coupled->ready_waiting_counts); in cpuidle_coupled_set_not_waiting()
396 coupled->requested_state[cpu] = CPUIDLE_COUPLED_NOT_IDLE; in cpuidle_coupled_set_not_waiting()
402 * @coupled: the struct coupled that contains the current cpu
408 static void cpuidle_coupled_set_done(int cpu, struct cpuidle_coupled *coupled) in cpuidle_coupled_set_done() argument
410 cpuidle_coupled_set_not_waiting(cpu, coupled); in cpuidle_coupled_set_done()
411 atomic_sub(MAX_WAITING_CPUS, &coupled->ready_waiting_counts); in cpuidle_coupled_set_done()
440 static bool cpuidle_coupled_any_pokes_pending(struct cpuidle_coupled *coupled) in cpuidle_coupled_any_pokes_pending() argument
442 return cpumask_first_and_and(cpu_online_mask, &coupled->coupled_cpus, in cpuidle_coupled_any_pokes_pending()
447 * cpuidle_enter_state_coupled - attempt to enter a state with coupled cpus
452 * Coordinate with coupled cpus to enter the target state. This is a two
457 * all the other cpus to call this function. Once all coupled cpus are idle,
458 * the second stage will start. Each coupled cpu will spin until all cpus have
469 struct cpuidle_coupled *coupled = dev->coupled; in cpuidle_enter_state_coupled() local
472 if (!coupled) in cpuidle_enter_state_coupled()
475 while (coupled->prevent) { in cpuidle_enter_state_coupled()
492 w = cpuidle_coupled_set_waiting(dev->cpu, coupled, next_state); in cpuidle_enter_state_coupled()
500 if (w == coupled->online_count) { in cpuidle_enter_state_coupled()
502 cpuidle_coupled_poke_others(dev->cpu, coupled); in cpuidle_enter_state_coupled()
507 * Wait for all coupled cpus to be idle, using the deepest state in cpuidle_enter_state_coupled()
514 while (!cpuidle_coupled_cpus_waiting(coupled) || in cpuidle_enter_state_coupled()
520 cpuidle_coupled_set_not_waiting(dev->cpu, coupled); in cpuidle_enter_state_coupled()
524 if (coupled->prevent) { in cpuidle_enter_state_coupled()
525 cpuidle_coupled_set_not_waiting(dev->cpu, coupled); in cpuidle_enter_state_coupled()
536 cpuidle_coupled_set_not_waiting(dev->cpu, coupled); in cpuidle_enter_state_coupled()
547 * All coupled cpus are probably idle. There is a small chance that in cpuidle_enter_state_coupled()
549 * and spin until all coupled cpus have incremented the counter. Once a in cpuidle_enter_state_coupled()
555 cpuidle_coupled_set_ready(coupled); in cpuidle_enter_state_coupled()
556 while (!cpuidle_coupled_cpus_ready(coupled)) { in cpuidle_enter_state_coupled()
558 if (!cpuidle_coupled_cpus_waiting(coupled)) in cpuidle_enter_state_coupled()
559 if (!cpuidle_coupled_set_not_ready(coupled)) in cpuidle_enter_state_coupled()
578 * coupled idle state of all cpus and retry. in cpuidle_enter_state_coupled()
580 if (cpuidle_coupled_any_pokes_pending(coupled)) { in cpuidle_enter_state_coupled()
581 cpuidle_coupled_set_done(dev->cpu, coupled); in cpuidle_enter_state_coupled()
583 cpuidle_coupled_parallel_barrier(dev, &coupled->abort_barrier); in cpuidle_enter_state_coupled()
587 /* all cpus have acked the coupled state */ in cpuidle_enter_state_coupled()
588 next_state = cpuidle_coupled_get_state(dev, coupled); in cpuidle_enter_state_coupled()
592 cpuidle_coupled_set_done(dev->cpu, coupled); in cpuidle_enter_state_coupled()
605 * Calling local_irq_enable here allows coupled states to return with in cpuidle_enter_state_coupled()
612 * Wait until all coupled cpus have exited idle. There is no risk that in cpuidle_enter_state_coupled()
616 while (!cpuidle_coupled_no_cpus_ready(coupled)) in cpuidle_enter_state_coupled()
622 static void cpuidle_coupled_update_online_cpus(struct cpuidle_coupled *coupled) in cpuidle_coupled_update_online_cpus() argument
624 coupled->online_count = cpumask_weight_and(cpu_online_mask, &coupled->coupled_cpus); in cpuidle_coupled_update_online_cpus()
628 * cpuidle_coupled_register_device - register a coupled cpuidle device
631 * Called from cpuidle_register_device to handle coupled idle init. Finds the
632 * cpuidle_coupled struct for this set of coupled cpus, or creates one if none
640 struct cpuidle_coupled *coupled; in cpuidle_coupled_register_device() local
647 if (other_dev && other_dev->coupled) { in cpuidle_coupled_register_device()
648 coupled = other_dev->coupled; in cpuidle_coupled_register_device()
653 /* No existing coupled info found, create a new one */ in cpuidle_coupled_register_device()
654 coupled = kzalloc(sizeof(struct cpuidle_coupled), GFP_KERNEL); in cpuidle_coupled_register_device()
655 if (!coupled) in cpuidle_coupled_register_device()
658 coupled->coupled_cpus = dev->coupled_cpus; in cpuidle_coupled_register_device()
661 dev->coupled = coupled; in cpuidle_coupled_register_device()
662 if (WARN_ON(!cpumask_equal(&dev->coupled_cpus, &coupled->coupled_cpus))) in cpuidle_coupled_register_device()
663 coupled->prevent++; in cpuidle_coupled_register_device()
665 cpuidle_coupled_update_online_cpus(coupled); in cpuidle_coupled_register_device()
667 coupled->refcnt++; in cpuidle_coupled_register_device()
676 * cpuidle_coupled_unregister_device - unregister a coupled cpuidle device
679 * Called from cpuidle_unregister_device to tear down coupled idle. Removes the
680 * cpu from the coupled idle set, and frees the cpuidle_coupled_info struct if
685 struct cpuidle_coupled *coupled = dev->coupled; in cpuidle_coupled_unregister_device() local
690 if (--coupled->refcnt) in cpuidle_coupled_unregister_device()
691 kfree(coupled); in cpuidle_coupled_unregister_device()
692 dev->coupled = NULL; in cpuidle_coupled_unregister_device()
696 * cpuidle_coupled_prevent_idle - prevent cpus from entering a coupled state
697 * @coupled: the struct coupled that contains the cpu that is changing state
699 * Disables coupled cpuidle on a coupled set of cpus. Used to ensure that
700 * cpu_online_mask doesn't change while cpus are coordinating coupled idle.
702 static void cpuidle_coupled_prevent_idle(struct cpuidle_coupled *coupled) in cpuidle_coupled_prevent_idle() argument
707 coupled->prevent++; in cpuidle_coupled_prevent_idle()
708 cpuidle_coupled_poke_others(cpu, coupled); in cpuidle_coupled_prevent_idle()
710 while (!cpuidle_coupled_no_cpus_waiting(coupled)) in cpuidle_coupled_prevent_idle()
715 * cpuidle_coupled_allow_idle - allows cpus to enter a coupled state
716 * @coupled: the struct coupled that contains the cpu that is changing state
718 * Enables coupled cpuidle on a coupled set of cpus. Used to ensure that
719 * cpu_online_mask doesn't change while cpus are coordinating coupled idle.
721 static void cpuidle_coupled_allow_idle(struct cpuidle_coupled *coupled) in cpuidle_coupled_allow_idle() argument
730 coupled->prevent--; in cpuidle_coupled_allow_idle()
732 cpuidle_coupled_poke_others(cpu, coupled); in cpuidle_coupled_allow_idle()
743 if (dev && dev->coupled) { in coupled_cpu_online()
744 cpuidle_coupled_update_online_cpus(dev->coupled); in coupled_cpu_online()
745 cpuidle_coupled_allow_idle(dev->coupled); in coupled_cpu_online()
759 if (dev && dev->coupled) in coupled_cpu_up_prepare()
760 cpuidle_coupled_prevent_idle(dev->coupled); in coupled_cpu_up_prepare()
771 "cpuidle/coupled:prepare", in cpuidle_coupled_init()
777 "cpuidle/coupled:online", in cpuidle_coupled_init()