1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * hrtimers - High-resolution kernel timers 4 * 5 * Copyright(C) 2005, Thomas Gleixner <tglx@linutronix.de> 6 * Copyright(C) 2005, Red Hat, Inc., Ingo Molnar 7 * 8 * data type definitions, declarations, prototypes 9 * 10 * Started by: Thomas Gleixner and Ingo Molnar 11 */ 12 #ifndef _LINUX_HRTIMER_H 13 #define _LINUX_HRTIMER_H 14 15 #include <linux/hrtimer_defs.h> 16 #include <linux/hrtimer_types.h> 17 #include <linux/init.h> 18 #include <linux/list.h> 19 #include <linux/percpu-defs.h> 20 #include <linux/rbtree.h> 21 #include <linux/timer.h> 22 23 /* 24 * Mode arguments of xxx_hrtimer functions: 25 * 26 * HRTIMER_MODE_ABS - Time value is absolute 27 * HRTIMER_MODE_REL - Time value is relative to now 28 * HRTIMER_MODE_PINNED - Timer is bound to CPU (is only considered 29 * when starting the timer) 30 * HRTIMER_MODE_SOFT - Timer callback function will be executed in 31 * soft irq context 32 * HRTIMER_MODE_HARD - Timer callback function will be executed in 33 * hard irq context even on PREEMPT_RT. 34 */ 35 enum hrtimer_mode { 36 HRTIMER_MODE_ABS = 0x00, 37 HRTIMER_MODE_REL = 0x01, 38 HRTIMER_MODE_PINNED = 0x02, 39 HRTIMER_MODE_SOFT = 0x04, 40 HRTIMER_MODE_HARD = 0x08, 41 42 HRTIMER_MODE_ABS_PINNED = HRTIMER_MODE_ABS | HRTIMER_MODE_PINNED, 43 HRTIMER_MODE_REL_PINNED = HRTIMER_MODE_REL | HRTIMER_MODE_PINNED, 44 45 HRTIMER_MODE_ABS_SOFT = HRTIMER_MODE_ABS | HRTIMER_MODE_SOFT, 46 HRTIMER_MODE_REL_SOFT = HRTIMER_MODE_REL | HRTIMER_MODE_SOFT, 47 48 HRTIMER_MODE_ABS_PINNED_SOFT = HRTIMER_MODE_ABS_PINNED | HRTIMER_MODE_SOFT, 49 HRTIMER_MODE_REL_PINNED_SOFT = HRTIMER_MODE_REL_PINNED | HRTIMER_MODE_SOFT, 50 51 HRTIMER_MODE_ABS_HARD = HRTIMER_MODE_ABS | HRTIMER_MODE_HARD, 52 HRTIMER_MODE_REL_HARD = HRTIMER_MODE_REL | HRTIMER_MODE_HARD, 53 54 HRTIMER_MODE_ABS_PINNED_HARD = HRTIMER_MODE_ABS_PINNED | HRTIMER_MODE_HARD, 55 HRTIMER_MODE_REL_PINNED_HARD = HRTIMER_MODE_REL_PINNED | HRTIMER_MODE_HARD, 56 }; 57 58 /* 59 * Values to track state of the timer 60 * 61 * Possible states: 62 * 63 * 0x00 inactive 64 * 0x01 enqueued into rbtree 65 * 66 * The callback state is not part of the timer->state because clearing it would 67 * mean touching the timer after the callback, this makes it impossible to free 68 * the timer from the callback function. 69 * 70 * Therefore we track the callback state in: 71 * 72 * timer->base->cpu_base->running == timer 73 * 74 * On SMP it is possible to have a "callback function running and enqueued" 75 * status. It happens for example when a posix timer expired and the callback 76 * queued a signal. Between dropping the lock which protects the posix timer 77 * and reacquiring the base lock of the hrtimer, another CPU can deliver the 78 * signal and rearm the timer. 79 * 80 * All state transitions are protected by cpu_base->lock. 81 */ 82 #define HRTIMER_STATE_INACTIVE 0x00 83 #define HRTIMER_STATE_ENQUEUED 0x01 84 85 /** 86 * struct hrtimer_sleeper - simple sleeper structure 87 * @timer: embedded timer structure 88 * @task: task to wake up 89 * 90 * task is set to NULL, when the timer expires. 91 */ 92 struct hrtimer_sleeper { 93 struct hrtimer timer; 94 struct task_struct *task; 95 }; 96 hrtimer_set_expires(struct hrtimer * timer,ktime_t time)97 static inline void hrtimer_set_expires(struct hrtimer *timer, ktime_t time) 98 { 99 timer->node.expires = time; 100 timer->_softexpires = time; 101 } 102 hrtimer_set_expires_range(struct hrtimer * timer,ktime_t time,ktime_t delta)103 static inline void hrtimer_set_expires_range(struct hrtimer *timer, ktime_t time, ktime_t delta) 104 { 105 timer->_softexpires = time; 106 timer->node.expires = ktime_add_safe(time, delta); 107 } 108 hrtimer_set_expires_range_ns(struct hrtimer * timer,ktime_t time,u64 delta)109 static inline void hrtimer_set_expires_range_ns(struct hrtimer *timer, ktime_t time, u64 delta) 110 { 111 timer->_softexpires = time; 112 timer->node.expires = ktime_add_safe(time, ns_to_ktime(delta)); 113 } 114 hrtimer_set_expires_tv64(struct hrtimer * timer,s64 tv64)115 static inline void hrtimer_set_expires_tv64(struct hrtimer *timer, s64 tv64) 116 { 117 timer->node.expires = tv64; 118 timer->_softexpires = tv64; 119 } 120 hrtimer_add_expires(struct hrtimer * timer,ktime_t time)121 static inline void hrtimer_add_expires(struct hrtimer *timer, ktime_t time) 122 { 123 timer->node.expires = ktime_add_safe(timer->node.expires, time); 124 timer->_softexpires = ktime_add_safe(timer->_softexpires, time); 125 } 126 hrtimer_add_expires_ns(struct hrtimer * timer,u64 ns)127 static inline void hrtimer_add_expires_ns(struct hrtimer *timer, u64 ns) 128 { 129 timer->node.expires = ktime_add_ns(timer->node.expires, ns); 130 timer->_softexpires = ktime_add_ns(timer->_softexpires, ns); 131 } 132 hrtimer_get_expires(const struct hrtimer * timer)133 static inline ktime_t hrtimer_get_expires(const struct hrtimer *timer) 134 { 135 return timer->node.expires; 136 } 137 hrtimer_get_softexpires(const struct hrtimer * timer)138 static inline ktime_t hrtimer_get_softexpires(const struct hrtimer *timer) 139 { 140 return timer->_softexpires; 141 } 142 hrtimer_get_expires_tv64(const struct hrtimer * timer)143 static inline s64 hrtimer_get_expires_tv64(const struct hrtimer *timer) 144 { 145 return timer->node.expires; 146 } hrtimer_get_softexpires_tv64(const struct hrtimer * timer)147 static inline s64 hrtimer_get_softexpires_tv64(const struct hrtimer *timer) 148 { 149 return timer->_softexpires; 150 } 151 hrtimer_get_expires_ns(const struct hrtimer * timer)152 static inline s64 hrtimer_get_expires_ns(const struct hrtimer *timer) 153 { 154 return ktime_to_ns(timer->node.expires); 155 } 156 hrtimer_expires_remaining(const struct hrtimer * timer)157 static inline ktime_t hrtimer_expires_remaining(const struct hrtimer *timer) 158 { 159 return ktime_sub(timer->node.expires, timer->base->get_time()); 160 } 161 hrtimer_cb_get_time(struct hrtimer * timer)162 static inline ktime_t hrtimer_cb_get_time(struct hrtimer *timer) 163 { 164 return timer->base->get_time(); 165 } 166 hrtimer_is_hres_active(struct hrtimer * timer)167 static inline int hrtimer_is_hres_active(struct hrtimer *timer) 168 { 169 return IS_ENABLED(CONFIG_HIGH_RES_TIMERS) ? 170 timer->base->cpu_base->hres_active : 0; 171 } 172 173 #ifdef CONFIG_HIGH_RES_TIMERS 174 struct clock_event_device; 175 176 extern void hrtimer_interrupt(struct clock_event_device *dev); 177 178 extern unsigned int hrtimer_resolution; 179 180 #else 181 182 #define hrtimer_resolution (unsigned int)LOW_RES_NSEC 183 184 #endif 185 186 static inline ktime_t __hrtimer_expires_remaining_adjusted(const struct hrtimer * timer,ktime_t now)187 __hrtimer_expires_remaining_adjusted(const struct hrtimer *timer, ktime_t now) 188 { 189 ktime_t rem = ktime_sub(timer->node.expires, now); 190 191 /* 192 * Adjust relative timers for the extra we added in 193 * hrtimer_start_range_ns() to prevent short timeouts. 194 */ 195 if (IS_ENABLED(CONFIG_TIME_LOW_RES) && timer->is_rel) 196 rem -= hrtimer_resolution; 197 return rem; 198 } 199 200 static inline ktime_t hrtimer_expires_remaining_adjusted(const struct hrtimer * timer)201 hrtimer_expires_remaining_adjusted(const struct hrtimer *timer) 202 { 203 return __hrtimer_expires_remaining_adjusted(timer, 204 timer->base->get_time()); 205 } 206 207 #ifdef CONFIG_TIMERFD 208 extern void timerfd_clock_was_set(void); 209 extern void timerfd_resume(void); 210 #else timerfd_clock_was_set(void)211 static inline void timerfd_clock_was_set(void) { } timerfd_resume(void)212 static inline void timerfd_resume(void) { } 213 #endif 214 215 DECLARE_PER_CPU(struct tick_device, tick_cpu_device); 216 217 #ifdef CONFIG_PREEMPT_RT 218 void hrtimer_cancel_wait_running(const struct hrtimer *timer); 219 #else hrtimer_cancel_wait_running(struct hrtimer * timer)220 static inline void hrtimer_cancel_wait_running(struct hrtimer *timer) 221 { 222 cpu_relax(); 223 } 224 #endif 225 226 /* Exported timer functions: */ 227 228 /* Initialize timers: */ 229 extern void hrtimer_init(struct hrtimer *timer, clockid_t which_clock, 230 enum hrtimer_mode mode); 231 extern void hrtimer_init_sleeper(struct hrtimer_sleeper *sl, clockid_t clock_id, 232 enum hrtimer_mode mode); 233 234 #ifdef CONFIG_DEBUG_OBJECTS_TIMERS 235 extern void hrtimer_init_on_stack(struct hrtimer *timer, clockid_t which_clock, 236 enum hrtimer_mode mode); 237 extern void hrtimer_init_sleeper_on_stack(struct hrtimer_sleeper *sl, 238 clockid_t clock_id, 239 enum hrtimer_mode mode); 240 241 extern void destroy_hrtimer_on_stack(struct hrtimer *timer); 242 #else hrtimer_init_on_stack(struct hrtimer * timer,clockid_t which_clock,enum hrtimer_mode mode)243 static inline void hrtimer_init_on_stack(struct hrtimer *timer, 244 clockid_t which_clock, 245 enum hrtimer_mode mode) 246 { 247 hrtimer_init(timer, which_clock, mode); 248 } 249 hrtimer_init_sleeper_on_stack(struct hrtimer_sleeper * sl,clockid_t clock_id,enum hrtimer_mode mode)250 static inline void hrtimer_init_sleeper_on_stack(struct hrtimer_sleeper *sl, 251 clockid_t clock_id, 252 enum hrtimer_mode mode) 253 { 254 hrtimer_init_sleeper(sl, clock_id, mode); 255 } 256 destroy_hrtimer_on_stack(struct hrtimer * timer)257 static inline void destroy_hrtimer_on_stack(struct hrtimer *timer) { } 258 #endif 259 260 /* Basic timer operations: */ 261 extern void hrtimer_start_range_ns(struct hrtimer *timer, ktime_t tim, 262 u64 range_ns, const enum hrtimer_mode mode); 263 264 /** 265 * hrtimer_start - (re)start an hrtimer 266 * @timer: the timer to be added 267 * @tim: expiry time 268 * @mode: timer mode: absolute (HRTIMER_MODE_ABS) or 269 * relative (HRTIMER_MODE_REL), and pinned (HRTIMER_MODE_PINNED); 270 * softirq based mode is considered for debug purpose only! 271 */ hrtimer_start(struct hrtimer * timer,ktime_t tim,const enum hrtimer_mode mode)272 static inline void hrtimer_start(struct hrtimer *timer, ktime_t tim, 273 const enum hrtimer_mode mode) 274 { 275 hrtimer_start_range_ns(timer, tim, 0, mode); 276 } 277 278 extern int hrtimer_cancel(struct hrtimer *timer); 279 extern int hrtimer_try_to_cancel(struct hrtimer *timer); 280 hrtimer_start_expires(struct hrtimer * timer,enum hrtimer_mode mode)281 static inline void hrtimer_start_expires(struct hrtimer *timer, 282 enum hrtimer_mode mode) 283 { 284 u64 delta; 285 ktime_t soft, hard; 286 soft = hrtimer_get_softexpires(timer); 287 hard = hrtimer_get_expires(timer); 288 delta = ktime_to_ns(ktime_sub(hard, soft)); 289 hrtimer_start_range_ns(timer, soft, delta, mode); 290 } 291 292 void hrtimer_sleeper_start_expires(struct hrtimer_sleeper *sl, 293 enum hrtimer_mode mode); 294 hrtimer_restart(struct hrtimer * timer)295 static inline void hrtimer_restart(struct hrtimer *timer) 296 { 297 hrtimer_start_expires(timer, HRTIMER_MODE_ABS); 298 } 299 300 /* Query timers: */ 301 extern ktime_t __hrtimer_get_remaining(const struct hrtimer *timer, bool adjust); 302 303 /** 304 * hrtimer_get_remaining - get remaining time for the timer 305 * @timer: the timer to read 306 */ hrtimer_get_remaining(const struct hrtimer * timer)307 static inline ktime_t hrtimer_get_remaining(const struct hrtimer *timer) 308 { 309 return __hrtimer_get_remaining(timer, false); 310 } 311 312 extern u64 hrtimer_get_next_event(void); 313 extern u64 hrtimer_next_event_without(const struct hrtimer *exclude); 314 315 extern bool hrtimer_active(const struct hrtimer *timer); 316 317 /** 318 * hrtimer_is_queued - check, whether the timer is on one of the queues 319 * @timer: Timer to check 320 * 321 * Returns: True if the timer is queued, false otherwise 322 * 323 * The function can be used lockless, but it gives only a current snapshot. 324 */ hrtimer_is_queued(struct hrtimer * timer)325 static inline bool hrtimer_is_queued(struct hrtimer *timer) 326 { 327 /* The READ_ONCE pairs with the update functions of timer->state */ 328 return !!(READ_ONCE(timer->state) & HRTIMER_STATE_ENQUEUED); 329 } 330 331 /* 332 * Helper function to check, whether the timer is running the callback 333 * function 334 */ hrtimer_callback_running(struct hrtimer * timer)335 static inline int hrtimer_callback_running(struct hrtimer *timer) 336 { 337 return timer->base->running == timer; 338 } 339 340 /* Forward a hrtimer so it expires after now: */ 341 extern u64 342 hrtimer_forward(struct hrtimer *timer, ktime_t now, ktime_t interval); 343 344 /** 345 * hrtimer_forward_now() - forward the timer expiry so it expires after now 346 * @timer: hrtimer to forward 347 * @interval: the interval to forward 348 * 349 * It is a variant of hrtimer_forward(). The timer will expire after the current 350 * time of the hrtimer clock base. See hrtimer_forward() for details. 351 */ hrtimer_forward_now(struct hrtimer * timer,ktime_t interval)352 static inline u64 hrtimer_forward_now(struct hrtimer *timer, 353 ktime_t interval) 354 { 355 return hrtimer_forward(timer, timer->base->get_time(), interval); 356 } 357 358 /* Precise sleep: */ 359 360 extern int nanosleep_copyout(struct restart_block *, struct timespec64 *); 361 extern long hrtimer_nanosleep(ktime_t rqtp, const enum hrtimer_mode mode, 362 const clockid_t clockid); 363 364 extern int schedule_hrtimeout_range(ktime_t *expires, u64 delta, 365 const enum hrtimer_mode mode); 366 extern int schedule_hrtimeout_range_clock(ktime_t *expires, 367 u64 delta, 368 const enum hrtimer_mode mode, 369 clockid_t clock_id); 370 extern int schedule_hrtimeout(ktime_t *expires, const enum hrtimer_mode mode); 371 372 /* Soft interrupt function to run the hrtimer queues: */ 373 extern void hrtimer_run_queues(void); 374 375 /* Bootup initialization: */ 376 extern void __init hrtimers_init(void); 377 378 /* Show pending timers: */ 379 extern void sysrq_timer_list_show(void); 380 381 int hrtimers_prepare_cpu(unsigned int cpu); 382 #ifdef CONFIG_HOTPLUG_CPU 383 int hrtimers_cpu_dying(unsigned int cpu); 384 #else 385 #define hrtimers_cpu_dying NULL 386 #endif 387 388 #endif 389