Lines Matching +full:up +full:- +full:samples
1 // SPDX-License-Identifier: GPL-2.0
8 #include <linux/posix-timers.h>
20 #include "posix-timers.h"
28 pct->bases[CPUCLOCK_PROF].nextevt = cpu_limit * NSEC_PER_SEC; in posix_cputimers_group_init()
29 pct->timers_active = true; in posix_cputimers_group_init()
35 * tsk->signal->posix_cputimers.bases[clock].nextevt expiration cache if
39 * Returns 0 on success, -ESRCH on failure. Can fail if the task is exiting and
48 return -ESRCH; in update_rlimit_cpu()
102 ret = pid_for_clock(clock, false) ? 0 : -EINVAL; in validate_clock_permissions()
115 return pid_task(timer->it.cpu.pid, clock_pid_type(timer->it_clock)); in cpu_timer_task_rcu()
124 u64 delta, incr, expires = timer->it.cpu.node.expires; in bump_cpu_timer()
127 if (!timer->it_interval) in bump_cpu_timer()
133 incr = timer->it_interval; in bump_cpu_timer()
134 delta = now + incr - expires; in bump_cpu_timer()
137 for (i = 0; incr < delta - incr; i++) in bump_cpu_timer()
140 for (; i >= 0; incr >>= 1, i--) { in bump_cpu_timer()
144 timer->it.cpu.node.expires += incr; in bump_cpu_timer()
145 timer->it_overrun += 1LL << i; in bump_cpu_timer()
146 delta -= incr; in bump_cpu_timer()
148 return timer->it.cpu.node.expires; in bump_cpu_timer()
154 return !(~pct->bases[CPUCLOCK_PROF].nextevt | in expiry_cache_is_inactive()
155 ~pct->bases[CPUCLOCK_VIRT].nextevt | in expiry_cache_is_inactive()
156 ~pct->bases[CPUCLOCK_SCHED].nextevt); in expiry_cache_is_inactive()
165 tp->tv_sec = 0; in posix_cpu_clock_getres()
166 tp->tv_nsec = ((NSEC_PER_SEC + HZ - 1) / HZ); in posix_cpu_clock_getres()
173 tp->tv_nsec = 1; in posix_cpu_clock_getres()
188 return error ? : -EPERM; in posix_cpu_clock_set()
192 * Sample a per-thread clock for the given task. clkid is validated.
214 static inline void store_samples(u64 *samples, u64 stime, u64 utime, u64 rtime) in store_samples() argument
216 samples[CPUCLOCK_PROF] = stime + utime; in store_samples()
217 samples[CPUCLOCK_VIRT] = utime; in store_samples()
218 samples[CPUCLOCK_SCHED] = rtime; in store_samples()
221 static void task_sample_cputime(struct task_struct *p, u64 *samples) in task_sample_cputime() argument
226 store_samples(samples, stime, utime, p->se.sum_exec_runtime); in task_sample_cputime()
230 u64 *samples) in proc_sample_cputime_atomic() argument
234 utime = atomic64_read(&at->utime); in proc_sample_cputime_atomic()
235 stime = atomic64_read(&at->stime); in proc_sample_cputime_atomic()
236 rtime = atomic64_read(&at->sum_exec_runtime); in proc_sample_cputime_atomic()
237 store_samples(samples, stime, utime, rtime); in proc_sample_cputime_atomic()
257 __update_gt_cputime(&cputime_atomic->utime, sum->utime); in update_gt_cputime()
258 __update_gt_cputime(&cputime_atomic->stime, sum->stime); in update_gt_cputime()
259 __update_gt_cputime(&cputime_atomic->sum_exec_runtime, sum->sum_exec_runtime); in update_gt_cputime()
263 * thread_group_sample_cputime - Sample cputime for a given task
265 * @samples: Storage for time samples
273 void thread_group_sample_cputime(struct task_struct *tsk, u64 *samples) in thread_group_sample_cputime() argument
275 struct thread_group_cputimer *cputimer = &tsk->signal->cputimer; in thread_group_sample_cputime()
276 struct posix_cputimers *pct = &tsk->signal->posix_cputimers; in thread_group_sample_cputime()
278 WARN_ON_ONCE(!pct->timers_active); in thread_group_sample_cputime()
280 proc_sample_cputime_atomic(&cputimer->cputime_atomic, samples); in thread_group_sample_cputime()
284 * thread_group_start_cputime - Start cputime and return a sample
286 * @samples: Storage for time samples
295 static void thread_group_start_cputime(struct task_struct *tsk, u64 *samples) in thread_group_start_cputime() argument
297 struct thread_group_cputimer *cputimer = &tsk->signal->cputimer; in thread_group_start_cputime()
298 struct posix_cputimers *pct = &tsk->signal->posix_cputimers; in thread_group_start_cputime()
303 if (!READ_ONCE(pct->timers_active)) { in thread_group_start_cputime()
312 update_gt_cputime(&cputimer->cputime_atomic, &sum); in thread_group_start_cputime()
321 WRITE_ONCE(pct->timers_active, true); in thread_group_start_cputime()
323 proc_sample_cputime_atomic(&cputimer->cputime_atomic, samples); in thread_group_start_cputime()
326 static void __thread_group_cputime(struct task_struct *tsk, u64 *samples) in __thread_group_cputime() argument
331 store_samples(samples, ct.stime, ct.utime, ct.sum_exec_runtime); in __thread_group_cputime()
342 struct thread_group_cputimer *cputimer = &p->signal->cputimer; in cpu_clock_sample_group()
343 struct posix_cputimers *pct = &p->signal->posix_cputimers; in cpu_clock_sample_group()
344 u64 samples[CPUCLOCK_MAX]; in cpu_clock_sample_group() local
346 if (!READ_ONCE(pct->timers_active)) { in cpu_clock_sample_group()
348 thread_group_start_cputime(p, samples); in cpu_clock_sample_group()
350 __thread_group_cputime(p, samples); in cpu_clock_sample_group()
352 proc_sample_cputime_atomic(&cputimer->cputime_atomic, samples); in cpu_clock_sample_group()
355 return samples[clkid]; in cpu_clock_sample_group()
368 return -EINVAL; in posix_cpu_clock_get()
382 * Validate the clockid_t for a new CPU-clock timer, and initialize the timer.
384 * new timer already all-zeros initialized.
392 pid = pid_for_clock(new_timer->it_clock, false); in posix_cpu_timer_create()
395 return -EINVAL; in posix_cpu_timer_create()
407 lockdep_set_class(&new_timer->it_lock, &posix_cpu_timers_key); in posix_cpu_timer_create()
409 new_timer->kclock = &clock_posix_cpu; in posix_cpu_timer_create()
410 timerqueue_init(&new_timer->it.cpu.node); in posix_cpu_timer_create()
411 new_timer->it.cpu.pid = get_pid(pid); in posix_cpu_timer_create()
419 int clkidx = CPUCLOCK_WHICH(timer->it_clock); in timer_base()
421 if (CPUCLOCK_PERTHREAD(timer->it_clock)) in timer_base()
422 return tsk->posix_cputimers.bases + clkidx; in timer_base()
424 return tsk->signal->posix_cputimers.bases + clkidx; in timer_base()
429 * This will also re-evaluate the need to keep around the process wide
438 base->nextevt = 0; in trigger_base_recalc_expires()
453 struct cpu_timer *ctmr = &timer->it.cpu; in disarm_timer()
456 timer->it_active = 0; in disarm_timer()
461 if (cpu_timer_getexpires(ctmr) == base->nextevt) in disarm_timer()
467 * Clean up a CPU-clock timer that is about to be destroyed.
474 struct cpu_timer *ctmr = &timer->it.cpu; in posix_cpu_timer_del()
495 WARN_ON_ONCE(ctmr->head || timerqueue_node_queued(&ctmr->node)); in posix_cpu_timer_del()
497 if (timer->it.cpu.firing) in posix_cpu_timer_del()
508 put_pid(ctmr->pid); in posix_cpu_timer_del()
521 ctmr->head = NULL; in cleanup_timerqueue()
534 cleanup_timerqueue(&pct->bases[CPUCLOCK_PROF].tqhead); in cleanup_timers()
535 cleanup_timerqueue(&pct->bases[CPUCLOCK_VIRT].tqhead); in cleanup_timers()
536 cleanup_timerqueue(&pct->bases[CPUCLOCK_SCHED].tqhead); in cleanup_timers()
546 cleanup_timers(&tsk->posix_cputimers); in posix_cpu_timers_exit()
550 cleanup_timers(&tsk->signal->posix_cputimers); in posix_cpu_timers_exit_group()
560 struct cpu_timer *ctmr = &timer->it.cpu; in arm_timer()
563 timer->it_active = 1; in arm_timer()
564 if (!cpu_timer_enqueue(&base->tqhead, ctmr)) in arm_timer()
568 * We are the new earliest-expiring POSIX 1.b timer, hence in arm_timer()
573 if (newexp < base->nextevt) in arm_timer()
574 base->nextevt = newexp; in arm_timer()
576 if (CPUCLOCK_PERTHREAD(timer->it_clock)) in arm_timer()
587 struct cpu_timer *ctmr = &timer->it.cpu; in cpu_timer_fire()
589 timer->it_active = 0; in cpu_timer_fire()
590 if (unlikely(timer->sigq == NULL)) { in cpu_timer_fire()
595 wake_up_process(timer->it_process); in cpu_timer_fire()
597 } else if (!timer->it_interval) { in cpu_timer_fire()
599 * One-shot timer. Clear it as soon as it's fired. in cpu_timer_fire()
611 ++timer->it_requeue_pending; in cpu_timer_fire()
626 bool sigev_none = timer->it_sigev_notify == SIGEV_NONE; in posix_cpu_timer_set()
627 clockid_t clkid = CPUCLOCK_WHICH(timer->it_clock); in posix_cpu_timer_set()
628 struct cpu_timer *ctmr = &timer->it.cpu; in posix_cpu_timer_set()
643 return -ESRCH; in posix_cpu_timer_set()
650 new_expires = ktime_to_ns(timespec64_to_ktime(new->it_value)); in posix_cpu_timer_set()
653 * Protect against sighand release/switch in exit/exec and p->cpu_timers in posix_cpu_timer_set()
654 * and p->signal->cpu_timers read/write in arm_timer() in posix_cpu_timer_set()
663 return -ESRCH; in posix_cpu_timer_set()
669 if (unlikely(timer->it.cpu.firing)) { in posix_cpu_timer_set()
670 timer->it.cpu.firing = -1; in posix_cpu_timer_set()
674 timer->it_active = 0; in posix_cpu_timer_set()
681 if (CPUCLOCK_PERTHREAD(timer->it_clock)) in posix_cpu_timer_set()
688 old->it_value = (struct timespec64){ }; in posix_cpu_timer_set()
738 bool sigev_none = timer->it_sigev_notify == SIGEV_NONE; in __posix_cpu_timer_get()
739 u64 expires, iv = timer->it_interval; in __posix_cpu_timer_get()
744 * - SIGEV_NONE timers which are never armed in __posix_cpu_timer_get()
745 * - Timers which expired, but the signal has not yet been in __posix_cpu_timer_get()
748 if (iv && ((timer->it_requeue_pending & REQUEUE_PENDING) || sigev_none)) in __posix_cpu_timer_get()
751 expires = cpu_timer_getexpires(&timer->it.cpu); in __posix_cpu_timer_get()
759 itp->it_value = ns_to_timespec64(expires - now); in __posix_cpu_timer_get()
768 itp->it_value.tv_nsec = 1; in __posix_cpu_timer_get()
774 clockid_t clkid = CPUCLOCK_WHICH(timer->it_clock); in posix_cpu_timer_get()
780 if (p && cpu_timer_getexpires(&timer->it.cpu)) { in posix_cpu_timer_get()
781 itp->it_interval = ktime_to_timespec64(timer->it_interval); in posix_cpu_timer_get()
783 if (CPUCLOCK_PERTHREAD(timer->it_clock)) in posix_cpu_timer_get()
811 ctmr->firing = 1; in collect_timerqueue()
813 rcu_assign_pointer(ctmr->handling, current); in collect_timerqueue()
815 list_add_tail(&ctmr->elist, firing); in collect_timerqueue()
821 static void collect_posix_cputimers(struct posix_cputimers *pct, u64 *samples, in collect_posix_cputimers() argument
824 struct posix_cputimer_base *base = pct->bases; in collect_posix_cputimers()
828 base->nextevt = collect_timerqueue(&base->tqhead, firing, in collect_posix_cputimers()
829 samples[i]); in collect_posix_cputimers()
835 if (tsk->dl.dl_overrun) { in check_dl_overrun()
836 tsk->dl.dl_overrun = 0; in check_dl_overrun()
849 current->comm, task_pid_nr(current)); in check_rlimit()
856 * Check for any per-thread CPU timers that have fired and move them off
857 * the tsk->cpu_timers[N] list onto the firing list. Here we update the
858 * tsk->it_*_expires values to reflect the remaining thread CPU timers.
863 struct posix_cputimers *pct = &tsk->posix_cputimers; in check_thread_timers()
864 u64 samples[CPUCLOCK_MAX]; in check_thread_timers() local
873 task_sample_cputime(tsk, samples); in check_thread_timers()
874 collect_posix_cputimers(pct, samples, firing); in check_thread_timers()
882 unsigned long rttime = tsk->rt.timeout * (USEC_PER_SEC / HZ); in check_thread_timers()
893 tsk->signal->rlim[RLIMIT_RTTIME].rlim_cur = soft; in check_thread_timers()
903 struct posix_cputimers *pct = &sig->posix_cputimers; in stop_process_timers()
906 WRITE_ONCE(pct->timers_active, false); in stop_process_timers()
913 if (!it->expires) in check_cpu_itimer()
916 if (cur_time >= it->expires) { in check_cpu_itimer()
917 if (it->incr) in check_cpu_itimer()
918 it->expires += it->incr; in check_cpu_itimer()
920 it->expires = 0; in check_cpu_itimer()
928 if (it->expires && it->expires < *expires) in check_cpu_itimer()
929 *expires = it->expires; in check_cpu_itimer()
933 * Check for any per-thread CPU timers that have fired and move them
934 * off the tsk->*_timers list onto the firing list. Per-thread timers
940 struct signal_struct *const sig = tsk->signal; in check_process_timers()
941 struct posix_cputimers *pct = &sig->posix_cputimers; in check_process_timers()
942 u64 samples[CPUCLOCK_MAX]; in check_process_timers() local
950 if (!READ_ONCE(pct->timers_active) || pct->expiry_active) in check_process_timers()
957 pct->expiry_active = true; in check_process_timers()
963 proc_sample_cputime_atomic(&sig->cputimer.cputime_atomic, samples); in check_process_timers()
964 collect_posix_cputimers(pct, samples, firing); in check_process_timers()
969 check_cpu_itimer(tsk, &sig->it[CPUCLOCK_PROF], in check_process_timers()
970 &pct->bases[CPUCLOCK_PROF].nextevt, in check_process_timers()
971 samples[CPUCLOCK_PROF], SIGPROF); in check_process_timers()
972 check_cpu_itimer(tsk, &sig->it[CPUCLOCK_VIRT], in check_process_timers()
973 &pct->bases[CPUCLOCK_VIRT].nextevt, in check_process_timers()
974 samples[CPUCLOCK_VIRT], SIGVTALRM); in check_process_timers()
978 /* RLIMIT_CPU is in seconds. Samples are nanoseconds */ in check_process_timers()
980 u64 ptime = samples[CPUCLOCK_PROF]; in check_process_timers()
991 sig->rlim[RLIMIT_CPU].rlim_cur = soft + 1; in check_process_timers()
996 if (softns < pct->bases[CPUCLOCK_PROF].nextevt) in check_process_timers()
997 pct->bases[CPUCLOCK_PROF].nextevt = softns; in check_process_timers()
1003 pct->expiry_active = false; in check_process_timers()
1012 clockid_t clkid = CPUCLOCK_WHICH(timer->it_clock); in posix_cpu_timer_rearm()
1031 if (CPUCLOCK_PERTHREAD(timer->it_clock)) in posix_cpu_timer_rearm()
1039 * Now re-arm for the new expiry time. in posix_cpu_timer_rearm()
1048 * task_cputimers_expired - Check whether posix CPU timers are expired
1050 * @samples: Array of current samples for the CPUCLOCK clocks
1053 * Returns true if any member of @samples is greater than the corresponding
1054 * member of @pct->bases[CLK].nextevt. False otherwise
1057 task_cputimers_expired(const u64 *samples, struct posix_cputimers *pct) in task_cputimers_expired() argument
1062 if (samples[i] >= pct->bases[i].nextevt) in task_cputimers_expired()
1069 * fastpath_timer_check - POSIX CPU timers fast path.
1080 struct posix_cputimers *pct = &tsk->posix_cputimers; in fastpath_timer_check()
1084 u64 samples[CPUCLOCK_MAX]; in fastpath_timer_check() local
1086 task_sample_cputime(tsk, samples); in fastpath_timer_check()
1087 if (task_cputimers_expired(samples, pct)) in fastpath_timer_check()
1091 sig = tsk->signal; in fastpath_timer_check()
1092 pct = &sig->posix_cputimers; in fastpath_timer_check()
1108 if (READ_ONCE(pct->timers_active) && !READ_ONCE(pct->expiry_active)) { in fastpath_timer_check()
1109 u64 samples[CPUCLOCK_MAX]; in fastpath_timer_check() local
1111 proc_sample_cputime_atomic(&sig->cputimer.cputime_atomic, in fastpath_timer_check()
1112 samples); in fastpath_timer_check()
1114 if (task_cputimers_expired(samples, pct)) in fastpath_timer_check()
1118 if (dl_task(tsk) && tsk->dl.dl_overrun) in fastpath_timer_check()
1131 mutex_lock(&cw->mutex); in posix_cpu_timers_work()
1133 mutex_unlock(&cw->mutex); in posix_cpu_timers_work()
1137 * Invoked from the posix-timer core when a cancel operation failed because
1143 struct task_struct *tsk = rcu_dereference(timr->it.cpu.handling); in posix_cpu_timer_wait_running()
1154 mutex_lock(&tsk->posix_cputimers_work.mutex); in posix_cpu_timer_wait_running()
1156 mutex_unlock(&tsk->posix_cputimers_work.mutex); in posix_cpu_timer_wait_running()
1165 /* Ensure that timr->it.cpu.handling task cannot go away */ in posix_cpu_timer_wait_running_nsleep()
1167 spin_unlock_irq(&timr->it_lock); in posix_cpu_timer_wait_running_nsleep()
1171 spin_lock_irq(&timr->it_lock); in posix_cpu_timer_wait_running_nsleep()
1183 memset(&p->posix_cputimers_work.work, 0, in clear_posix_cputimers_work()
1184 sizeof(p->posix_cputimers_work.work)); in clear_posix_cputimers_work()
1185 init_task_work(&p->posix_cputimers_work.work, in clear_posix_cputimers_work()
1187 mutex_init(&p->posix_cputimers_work.mutex); in clear_posix_cputimers_work()
1188 p->posix_cputimers_work.scheduled = false; in clear_posix_cputimers_work()
1201 * Note: All operations on tsk->posix_cputimer_work.scheduled happen either
1208 return tsk->posix_cputimers_work.scheduled; in posix_cpu_timers_work_scheduled()
1213 if (WARN_ON_ONCE(tsk->posix_cputimers_work.scheduled)) in __run_posix_cpu_timers()
1217 tsk->posix_cputimers_work.scheduled = true; in __run_posix_cpu_timers()
1218 task_work_add(tsk, &tsk->posix_cputimers_work.work, TWA_RESUME); in __run_posix_cpu_timers()
1232 tsk->posix_cputimers_work.scheduled = false; in posix_cpu_timers_enable_work()
1253 tsk->posix_cputimers_work.scheduled = false; in posix_cpu_timers_enable_work()
1273 spin_unlock_irq(&timr->it_lock); in posix_cpu_timer_wait_running_nsleep()
1275 spin_lock_irq(&timr->it_lock); in posix_cpu_timer_wait_running_nsleep()
1309 * Here we take off tsk->signal->cpu_timers[N] and in handle_posix_cpu_timers()
1310 * tsk->cpu_timers[N] all the timers that are firing, and in handle_posix_cpu_timers()
1333 * - On !RT kernels no tick can have happened on this CPU in handle_posix_cpu_timers()
1338 * - On RT kernels ticks might have happened but the tick in handle_posix_cpu_timers()
1354 * that gets the timer lock before we do will give it up and in handle_posix_cpu_timers()
1375 spin_lock(&timer->it_lock); in handle_posix_cpu_timers()
1376 list_del_init(&timer->it.cpu.elist); in handle_posix_cpu_timers()
1377 cpu_firing = timer->it.cpu.firing; in handle_posix_cpu_timers()
1378 timer->it.cpu.firing = 0; in handle_posix_cpu_timers()
1380 * The firing flag is -1 if we collided with a reset in handle_posix_cpu_timers()
1382 * almost-firing as an overrun. So don't generate an event. in handle_posix_cpu_timers()
1387 rcu_assign_pointer(timer->it.cpu.handling, NULL); in handle_posix_cpu_timers()
1388 spin_unlock(&timer->it_lock); in handle_posix_cpu_timers()
1421 * Set one of the process-wide special case CPU timers or RLIMIT_CPU.
1422 * The tsk->sighand->siglock must be held by the caller.
1432 nextevt = &tsk->signal->posix_cputimers.bases[clkid].nextevt; in set_process_cpu_timer()
1446 *oldval -= now; in set_process_cpu_timer()
1473 * Set up a temporary timer and then wait for it to go off. in do_cpu_nanosleep()
1478 timer.it_overrun = -1; in do_cpu_nanosleep()
1540 error = -ERESTART_RESTARTBLOCK; in do_cpu_nanosleep()
1544 restart = ¤t->restart_block; in do_cpu_nanosleep()
1545 restart->nanosleep.expires = expires; in do_cpu_nanosleep()
1546 if (restart->nanosleep.type != TT_NONE) in do_cpu_nanosleep()
1558 struct restart_block *restart_block = ¤t->restart_block; in posix_cpu_nsleep()
1567 return -EINVAL; in posix_cpu_nsleep()
1571 if (error == -ERESTART_RESTARTBLOCK) { in posix_cpu_nsleep()
1574 return -ERESTARTNOHAND; in posix_cpu_nsleep()
1576 restart_block->nanosleep.clockid = which_clock; in posix_cpu_nsleep()
1584 clockid_t which_clock = restart_block->nanosleep.clockid; in posix_cpu_nsleep_restart()
1587 t = ns_to_timespec64(restart_block->nanosleep.expires); in posix_cpu_nsleep_restart()
1607 timer->it_clock = PROCESS_CLOCK; in process_cpu_timer_create()
1627 timer->it_clock = THREAD_CLOCK; in thread_cpu_timer_create()