Lines Matching +full:timer +full:- +full:cannot +full:- +full:wake +full:- +full:cpu
1 // SPDX-License-Identifier: GPL-2.0
3 * This file contains functions which emulate a local clock-event
6 * Copyright(C) 2005-2006, Thomas Gleixner <tglx@linutronix.de>
7 * Copyright(C) 2005-2007, Red Hat, Inc., Ingo Molnar
8 * Copyright(C) 2006-2007, Timesys Corp., Thomas Gleixner
10 #include <linux/cpu.h>
20 #include "tick-internal.h"
24 * timer stops in C3 state.
39 static void tick_broadcast_clear_oneshot(int cpu);
42 static void tick_broadcast_oneshot_offline(unsigned int cpu);
47 static inline void tick_broadcast_clear_oneshot(int cpu) { } in tick_broadcast_clear_oneshot() argument
50 static inline void tick_broadcast_oneshot_offline(unsigned int cpu) { } in tick_broadcast_oneshot_offline() argument
67 static struct clock_event_device *tick_get_oneshot_wakeup_device(int cpu);
69 const struct clock_event_device *tick_get_wakeup_device(int cpu) in tick_get_wakeup_device() argument
71 return tick_get_oneshot_wakeup_device(cpu); in tick_get_wakeup_device()
89 if ((newdev->features & CLOCK_EVT_FEAT_DUMMY) || in tick_check_broadcast_device()
90 (newdev->features & CLOCK_EVT_FEAT_PERCPU) || in tick_check_broadcast_device()
91 (newdev->features & CLOCK_EVT_FEAT_C3STOP)) in tick_check_broadcast_device()
95 !(newdev->features & CLOCK_EVT_FEAT_ONESHOT)) in tick_check_broadcast_device()
98 return !curdev || newdev->rating > curdev->rating; in tick_check_broadcast_device()
102 static struct clock_event_device *tick_get_oneshot_wakeup_device(int cpu) in tick_get_oneshot_wakeup_device() argument
104 return per_cpu(tick_oneshot_wakeup_device, cpu); in tick_get_oneshot_wakeup_device()
117 int cpu) in tick_set_oneshot_wakeup_device() argument
119 struct clock_event_device *curdev = tick_get_oneshot_wakeup_device(cpu); in tick_set_oneshot_wakeup_device()
124 if ((newdev->features & CLOCK_EVT_FEAT_DUMMY) || in tick_set_oneshot_wakeup_device()
125 (newdev->features & CLOCK_EVT_FEAT_C3STOP)) in tick_set_oneshot_wakeup_device()
128 if (!(newdev->features & CLOCK_EVT_FEAT_PERCPU) || in tick_set_oneshot_wakeup_device()
129 !(newdev->features & CLOCK_EVT_FEAT_ONESHOT)) in tick_set_oneshot_wakeup_device()
132 if (!cpumask_equal(newdev->cpumask, cpumask_of(cpu))) in tick_set_oneshot_wakeup_device()
135 if (curdev && newdev->rating <= curdev->rating) in tick_set_oneshot_wakeup_device()
138 if (!try_module_get(newdev->owner)) in tick_set_oneshot_wakeup_device()
141 newdev->event_handler = tick_oneshot_wakeup_handler; in tick_set_oneshot_wakeup_device()
144 per_cpu(tick_oneshot_wakeup_device, cpu) = newdev; in tick_set_oneshot_wakeup_device()
148 static struct clock_event_device *tick_get_oneshot_wakeup_device(int cpu) in tick_get_oneshot_wakeup_device() argument
154 int cpu) in tick_set_oneshot_wakeup_device() argument
163 void tick_install_broadcast_device(struct clock_event_device *dev, int cpu) in tick_install_broadcast_device() argument
167 if (tick_set_oneshot_wakeup_device(dev, cpu)) in tick_install_broadcast_device()
173 if (!try_module_get(dev->owner)) in tick_install_broadcast_device()
178 cur->event_handler = clockevents_handle_noop; in tick_install_broadcast_device()
183 if (!(dev->features & CLOCK_EVT_FEAT_ONESHOT)) in tick_install_broadcast_device()
197 * where we did not switch to oneshot mode because the per cpu in tick_install_broadcast_device()
216 int ret = -ENODEV; in tick_broadcast_update_freq()
229 pr_crit_once("Failed to broadcast timer tick. Some CPUs may be unresponsive.\n"); in err_broadcast()
234 if (!dev->broadcast) in tick_device_setup_broadcast_func()
235 dev->broadcast = tick_broadcast; in tick_device_setup_broadcast_func()
236 if (!dev->broadcast) { in tick_device_setup_broadcast_func()
238 dev->name); in tick_device_setup_broadcast_func()
239 dev->broadcast = err_broadcast; in tick_device_setup_broadcast_func()
247 int tick_device_uses_broadcast(struct clock_event_device *dev, int cpu) in tick_device_uses_broadcast() argument
259 * the cpu local device. in tick_device_uses_broadcast()
262 dev->event_handler = tick_handle_periodic; in tick_device_uses_broadcast()
264 cpumask_set_cpu(cpu, tick_broadcast_mask); in tick_device_uses_broadcast()
272 * Clear the broadcast bit for this cpu if the in tick_device_uses_broadcast()
275 if (!(dev->features & CLOCK_EVT_FEAT_C3STOP)) in tick_device_uses_broadcast()
276 cpumask_clear_cpu(cpu, tick_broadcast_mask); in tick_device_uses_broadcast()
281 * Clear the broadcast bit if the CPU is not in in tick_device_uses_broadcast()
284 if (!cpumask_test_cpu(cpu, tick_broadcast_on)) in tick_device_uses_broadcast()
285 cpumask_clear_cpu(cpu, tick_broadcast_mask); in tick_device_uses_broadcast()
292 * because the CPU is running and therefore in tick_device_uses_broadcast()
297 tick_broadcast_clear_oneshot(cpu); in tick_device_uses_broadcast()
310 * If we kept the cpu in the broadcast mask, in tick_device_uses_broadcast()
311 * tell the caller to leave the per cpu device in tick_device_uses_broadcast()
317 if (bc && !(bc->features & CLOCK_EVT_FEAT_HRTIMER)) in tick_device_uses_broadcast()
318 ret = cpumask_test_cpu(cpu, tick_broadcast_mask); in tick_device_uses_broadcast()
331 struct clock_event_device *evt = td->evtdev; in tick_receive_broadcast()
334 return -ENODEV; in tick_receive_broadcast()
336 if (!evt->event_handler) in tick_receive_broadcast()
337 return -EINVAL; in tick_receive_broadcast()
339 evt->event_handler(evt); in tick_receive_broadcast()
348 int cpu = smp_processor_id(); in tick_do_broadcast() local
353 * Check, if the current cpu is in the mask in tick_do_broadcast()
355 if (cpumask_test_cpu(cpu, mask)) { in tick_do_broadcast()
358 cpumask_clear_cpu(cpu, mask); in tick_do_broadcast()
371 local = !(bc->features & CLOCK_EVT_FEAT_HRTIMER); in tick_do_broadcast()
382 td->evtdev->broadcast(mask); in tick_do_broadcast()
389 * - invoke the broadcast handlers
416 ktime_t next = ktime_add_ns(dev->next_event, TICK_NSEC); in tick_handle_periodic_broadcast()
423 * We run the handler of the local cpu after dropping in tick_handle_periodic_broadcast()
428 td->evtdev->event_handler(td->evtdev); in tick_handle_periodic_broadcast()
432 * tick_broadcast_control - Enable/disable or force broadcast mode
436 * might stop. Note: TICK_BROADCAST_FORCE cannot be undone.
442 int cpu, bc_stopped; in tick_broadcast_control() local
448 dev = td->evtdev; in tick_broadcast_control()
453 if (!dev || !(dev->features & CLOCK_EVT_FEAT_C3STOP)) in tick_broadcast_control()
459 cpu = smp_processor_id(); in tick_broadcast_control()
468 cpumask_set_cpu(cpu, tick_broadcast_on); in tick_broadcast_control()
469 if (!cpumask_test_and_set_cpu(cpu, tick_broadcast_mask)) { in tick_broadcast_control()
471 * Only shutdown the cpu local device, if: in tick_broadcast_control()
473 * - the broadcast device exists in tick_broadcast_control()
474 * - the broadcast device is not a hrtimer based one in tick_broadcast_control()
475 * - the broadcast device is in periodic mode to in tick_broadcast_control()
478 if (bc && !(bc->features & CLOCK_EVT_FEAT_HRTIMER) && in tick_broadcast_control()
487 cpumask_clear_cpu(cpu, tick_broadcast_on); in tick_broadcast_control()
488 if (cpumask_test_and_clear_cpu(cpu, tick_broadcast_mask)) { in tick_broadcast_control()
518 dev->event_handler = tick_handle_periodic; in tick_set_periodic_handler()
520 dev->event_handler = tick_handle_periodic_broadcast; in tick_set_periodic_handler()
535 * Remove a CPU from broadcasting
537 void tick_broadcast_offline(unsigned int cpu) in tick_broadcast_offline() argument
540 cpumask_clear_cpu(cpu, tick_broadcast_mask); in tick_broadcast_offline()
541 cpumask_clear_cpu(cpu, tick_broadcast_on); in tick_broadcast_offline()
542 tick_broadcast_oneshot_offline(cpu); in tick_broadcast_offline()
564 * This is called from tick_resume_local() on a resuming CPU. That's
569 * bit of the resuming CPU in the broadcast mask is safe as well.
641 if (!(bc->features & CLOCK_EVT_FEAT_DYNIRQ)) in tick_broadcast_set_affinity()
644 if (cpumask_equal(bc->cpumask, cpumask)) in tick_broadcast_set_affinity()
647 bc->cpumask = cpumask; in tick_broadcast_set_affinity()
648 irq_set_affinity(bc->irq, bc->cpumask); in tick_broadcast_set_affinity()
651 static void tick_broadcast_set_event(struct clock_event_device *bc, int cpu, in tick_broadcast_set_event() argument
658 tick_broadcast_set_affinity(bc, cpumask_of(cpu)); in tick_broadcast_set_event()
668 * per cpu device.
677 * periodic to oneshot. If the CPU has not yet in tick_check_oneshot_broadcast_this_cpu()
680 if (td->mode == TICKDEV_MODE_ONESHOT) { in tick_check_oneshot_broadcast_this_cpu()
681 clockevents_switch_state(td->evtdev, in tick_check_oneshot_broadcast_this_cpu()
694 int cpu, next_cpu = 0; in tick_handle_oneshot_broadcast() local
698 dev->next_event = KTIME_MAX; in tick_handle_oneshot_broadcast()
703 for_each_cpu(cpu, tick_broadcast_oneshot_mask) { in tick_handle_oneshot_broadcast()
712 td = &per_cpu(tick_cpu_device, cpu); in tick_handle_oneshot_broadcast()
713 if (td->evtdev->next_event <= now) { in tick_handle_oneshot_broadcast()
714 cpumask_set_cpu(cpu, tmpmask); in tick_handle_oneshot_broadcast()
716 * Mark the remote cpu in the pending mask, so in tick_handle_oneshot_broadcast()
717 * it can avoid reprogramming the cpu local in tick_handle_oneshot_broadcast()
718 * timer in tick_broadcast_oneshot_control(). in tick_handle_oneshot_broadcast()
720 cpumask_set_cpu(cpu, tick_broadcast_pending_mask); in tick_handle_oneshot_broadcast()
721 } else if (td->evtdev->next_event < next_event) { in tick_handle_oneshot_broadcast()
722 next_event = td->evtdev->next_event; in tick_handle_oneshot_broadcast()
723 next_cpu = cpu; in tick_handle_oneshot_broadcast()
728 * Remove the current cpu from the pending mask. The event is in tick_handle_oneshot_broadcast()
752 * - The global event did not expire any CPU local in tick_handle_oneshot_broadcast()
756 * - There are pending events on sleeping CPUs which were not in tick_handle_oneshot_broadcast()
766 td->evtdev->event_handler(td->evtdev); in tick_handle_oneshot_broadcast()
770 static int broadcast_needs_cpu(struct clock_event_device *bc, int cpu) in broadcast_needs_cpu() argument
772 if (!(bc->features & CLOCK_EVT_FEAT_HRTIMER)) in broadcast_needs_cpu()
774 if (bc->next_event == KTIME_MAX) in broadcast_needs_cpu()
776 return bc->bound_on == cpu ? -EBUSY : 0; in broadcast_needs_cpu()
783 * For hrtimer based broadcasting we cannot shutdown the cpu in broadcast_shutdown_local()
785 * if we own the broadcast timer. in broadcast_shutdown_local()
787 if (bc->features & CLOCK_EVT_FEAT_HRTIMER) { in broadcast_shutdown_local()
790 if (dev->next_event < bc->next_event) in broadcast_shutdown_local()
798 int cpu) in ___tick_broadcast_oneshot_control() argument
800 struct clock_event_device *bc, *dev = td->evtdev; in ___tick_broadcast_oneshot_control()
809 * If the current CPU owns the hrtimer broadcast in ___tick_broadcast_oneshot_control()
810 * mechanism, it cannot go deep idle and we do not add in ___tick_broadcast_oneshot_control()
811 * the CPU to the broadcast mask. We don't have to go in ___tick_broadcast_oneshot_control()
812 * through the EXIT path as the local timer is not in ___tick_broadcast_oneshot_control()
815 ret = broadcast_needs_cpu(bc, cpu); in ___tick_broadcast_oneshot_control()
825 if (bc->features & CLOCK_EVT_FEAT_HRTIMER) in ___tick_broadcast_oneshot_control()
826 ret = -EBUSY; in ___tick_broadcast_oneshot_control()
830 if (!cpumask_test_and_set_cpu(cpu, tick_broadcast_oneshot_mask)) { in ___tick_broadcast_oneshot_control()
831 WARN_ON_ONCE(cpumask_test_cpu(cpu, tick_broadcast_pending_mask)); in ___tick_broadcast_oneshot_control()
833 /* Conditionally shut down the local timer. */ in ___tick_broadcast_oneshot_control()
837 * We only reprogram the broadcast timer if we in ___tick_broadcast_oneshot_control()
839 * if the cpu local event is earlier than the in ___tick_broadcast_oneshot_control()
840 * broadcast event. If the current CPU is in in ___tick_broadcast_oneshot_control()
843 * busy, so the CPU does not try to go deep in ___tick_broadcast_oneshot_control()
846 if (cpumask_test_cpu(cpu, tick_broadcast_force_mask)) { in ___tick_broadcast_oneshot_control()
847 ret = -EBUSY; in ___tick_broadcast_oneshot_control()
848 } else if (dev->next_event < bc->next_event) { in ___tick_broadcast_oneshot_control()
849 tick_broadcast_set_event(bc, cpu, dev->next_event); in ___tick_broadcast_oneshot_control()
853 * timer to this cpu. If yes, remove in ___tick_broadcast_oneshot_control()
857 ret = broadcast_needs_cpu(bc, cpu); in ___tick_broadcast_oneshot_control()
859 cpumask_clear_cpu(cpu, in ___tick_broadcast_oneshot_control()
865 if (cpumask_test_and_clear_cpu(cpu, tick_broadcast_oneshot_mask)) { in ___tick_broadcast_oneshot_control()
868 * The cpu which was handling the broadcast in ___tick_broadcast_oneshot_control()
869 * timer marked this cpu in the broadcast in ___tick_broadcast_oneshot_control()
873 * handler. No need to reprogram the timer in ___tick_broadcast_oneshot_control()
876 if (cpumask_test_and_clear_cpu(cpu, in ___tick_broadcast_oneshot_control()
883 if (dev->next_event == KTIME_MAX) in ___tick_broadcast_oneshot_control()
887 * either the CPU handling the broadcast in ___tick_broadcast_oneshot_control()
891 * if the cpu local expiry time is already in ___tick_broadcast_oneshot_control()
892 * reached, we would reprogram the cpu local in ___tick_broadcast_oneshot_control()
893 * timer with an already expired event. in ___tick_broadcast_oneshot_control()
895 * This can lead to a ping-pong when we return in ___tick_broadcast_oneshot_control()
897 * timer before the cpu local timer was able in ___tick_broadcast_oneshot_control()
902 * enough out that the ping-pong starts. in ___tick_broadcast_oneshot_control()
904 * If the cpu local next_event has expired in ___tick_broadcast_oneshot_control()
905 * then we know that the broadcast timer in ___tick_broadcast_oneshot_control()
910 * will invoke the cpu local handler. in ___tick_broadcast_oneshot_control()
912 * We cannot call the handler directly from in ___tick_broadcast_oneshot_control()
918 if (dev->next_event <= now) { in ___tick_broadcast_oneshot_control()
919 cpumask_set_cpu(cpu, tick_broadcast_force_mask); in ___tick_broadcast_oneshot_control()
924 * the cpu local timer device. in ___tick_broadcast_oneshot_control()
926 tick_program_event(dev->next_event, 1); in ___tick_broadcast_oneshot_control()
936 int cpu) in tick_oneshot_wakeup_control() argument
940 dev = td->evtdev; in tick_oneshot_wakeup_control()
941 if (td->mode != TICKDEV_MODE_ONESHOT) in tick_oneshot_wakeup_control()
942 return -EINVAL; in tick_oneshot_wakeup_control()
944 wd = tick_get_oneshot_wakeup_device(cpu); in tick_oneshot_wakeup_control()
946 return -ENODEV; in tick_oneshot_wakeup_control()
952 clockevents_program_event(wd, dev->next_event, 1); in tick_oneshot_wakeup_control()
957 return -ENODEV; in tick_oneshot_wakeup_control()
966 int cpu = smp_processor_id(); in __tick_broadcast_oneshot_control() local
968 if (!tick_oneshot_wakeup_control(state, td, cpu)) in __tick_broadcast_oneshot_control()
972 return ___tick_broadcast_oneshot_control(state, td, cpu); in __tick_broadcast_oneshot_control()
978 return -EBUSY; in __tick_broadcast_oneshot_control()
982 * Reset the one shot broadcast for a cpu
986 static void tick_broadcast_clear_oneshot(int cpu) in tick_broadcast_clear_oneshot() argument
988 cpumask_clear_cpu(cpu, tick_broadcast_oneshot_mask); in tick_broadcast_clear_oneshot()
989 cpumask_clear_cpu(cpu, tick_broadcast_pending_mask); in tick_broadcast_clear_oneshot()
996 int cpu; in tick_broadcast_init_next_event() local
998 for_each_cpu(cpu, mask) { in tick_broadcast_init_next_event()
999 td = &per_cpu(tick_cpu_device, cpu); in tick_broadcast_init_next_event()
1000 if (td->evtdev) in tick_broadcast_init_next_event()
1001 td->evtdev->next_event = expires; in tick_broadcast_init_next_event()
1022 * tick_broadcast_setup_oneshot - setup the broadcast device
1027 int cpu = smp_processor_id(); in tick_broadcast_setup_oneshot() local
1035 * CPU handling the NOHZ change, the other CPUs will reach this in tick_broadcast_setup_oneshot()
1036 * code via hrtimer_run_queues() -> tick_check_oneshot_change() in tick_broadcast_setup_oneshot()
1039 if (bc->event_handler == tick_handle_oneshot_broadcast) { in tick_broadcast_setup_oneshot()
1041 * The CPU which switched from periodic to oneshot mode in tick_broadcast_setup_oneshot()
1047 * Clear the bit for the local CPU as the set bit would in tick_broadcast_setup_oneshot()
1048 * prevent the first tick_broadcast_enter() after this CPU in tick_broadcast_setup_oneshot()
1053 * but this cannot avoid the tick_broadcast_clear_oneshot() in tick_broadcast_setup_oneshot()
1058 tick_broadcast_clear_oneshot(cpu); in tick_broadcast_setup_oneshot()
1063 bc->event_handler = tick_handle_oneshot_broadcast; in tick_broadcast_setup_oneshot()
1064 bc->next_event = KTIME_MAX; in tick_broadcast_setup_oneshot()
1069 * get their wake-up at the next tick. This is achieved by ORing in tick_broadcast_setup_oneshot()
1080 /* Remove the local CPU as it is obviously not idle */ in tick_broadcast_setup_oneshot()
1081 cpumask_clear_cpu(cpu, tmpmask); in tick_broadcast_setup_oneshot()
1085 * Ensure that the oneshot broadcast handler will wake the in tick_broadcast_setup_oneshot()
1107 * immediately in order to reevaluate the next expiring timer. in tick_broadcast_setup_oneshot()
1118 tick_broadcast_set_event(bc, cpu, nexttick); in tick_broadcast_setup_oneshot()
1152 * If the broadcast force bit of the current CPU is set, in hotplug_cpu__broadcast_tick_pull()
1153 * then the current CPU has not yet reprogrammed the local in hotplug_cpu__broadcast_tick_pull()
1154 * timer device to avoid a ping-pong race. See in hotplug_cpu__broadcast_tick_pull()
1165 * Detect this condition and reprogram the cpu local timer in hotplug_cpu__broadcast_tick_pull()
1172 tick_program_event(td->evtdev->next_event, 1); in hotplug_cpu__broadcast_tick_pull()
1175 /* This moves the broadcast assignment to this CPU: */ in hotplug_cpu__broadcast_tick_pull()
1176 clockevents_program_event(bc, bc->next_event, 1); in hotplug_cpu__broadcast_tick_pull()
1182 * Remove a dying CPU from broadcasting
1184 static void tick_broadcast_oneshot_offline(unsigned int cpu) in tick_broadcast_oneshot_offline() argument
1186 if (tick_get_oneshot_wakeup_device(cpu)) in tick_broadcast_oneshot_offline()
1187 tick_set_oneshot_wakeup_device(NULL, cpu); in tick_broadcast_oneshot_offline()
1190 * Clear the broadcast masks for the dead cpu, but do not stop in tick_broadcast_oneshot_offline()
1193 cpumask_clear_cpu(cpu, tick_broadcast_oneshot_mask); in tick_broadcast_oneshot_offline()
1194 cpumask_clear_cpu(cpu, tick_broadcast_pending_mask); in tick_broadcast_oneshot_offline()
1195 cpumask_clear_cpu(cpu, tick_broadcast_force_mask); in tick_broadcast_oneshot_offline()
1214 return bc ? bc->features & CLOCK_EVT_FEAT_ONESHOT : false; in tick_broadcast_oneshot_available()
1222 if (!bc || (bc->features & CLOCK_EVT_FEAT_HRTIMER)) in __tick_broadcast_oneshot_control()
1223 return -EBUSY; in __tick_broadcast_oneshot_control()