Lines Matching +full:timer +full:- +full:cannot +full:- +full:wake +full:- +full:cpu
1 // SPDX-License-Identifier: GPL-2.0
24 * guard against timer DoS.
58 return dl_se->dl_server; in dl_server()
74 struct rq *rq = dl_se->rq; in rq_of_dl_se()
84 return &rq_of_dl_se(dl_se)->dl; in dl_rq_of_se()
89 return !RB_EMPTY_NODE(&dl_se->rb_node); in on_dl_rq()
95 return dl_se->pi_se; in pi_of()
119 return &cpu_rq(i)->rd->dl_bw; in dl_bw_of()
124 struct root_domain *rd = cpu_rq(i)->rd; in dl_bw_cpus()
130 if (cpumask_subset(rd->span, cpu_active_mask)) in dl_bw_cpus()
131 return cpumask_weight(rd->span); in dl_bw_cpus()
135 for_each_cpu_and(i, rd->span, cpu_active_mask) in dl_bw_cpus()
153 * XXX Fix: If 'rq->rd == def_root_domain' perform AC against capacity
154 * of the CPU the task is running on rather rd's \Sum CPU capacity.
165 return __dl_bw_capacity(cpu_rq(i)->rd->span); in dl_bw_capacity()
169 static inline bool dl_bw_visited(int cpu, u64 gen) in dl_bw_visited() argument
171 struct root_domain *rd = cpu_rq(cpu)->rd; in dl_bw_visited()
173 if (rd->visit_gen == gen) in dl_bw_visited()
176 rd->visit_gen = gen; in dl_bw_visited()
188 for_each_cpu_and(i, rd->span, cpu_active_mask) { in __dl_update()
191 rq->dl.extra_bw += bw; in __dl_update()
197 return &cpu_rq(i)->dl.dl_bw; in dl_bw_of()
210 static inline bool dl_bw_visited(int cpu, u64 gen) in dl_bw_visited() argument
220 dl->extra_bw += bw; in __dl_update()
227 dl_b->total_bw -= tsk_bw; in __dl_sub()
234 dl_b->total_bw += tsk_bw; in __dl_add()
235 __dl_update(dl_b, -((s32)tsk_bw / cpus)); in __dl_add()
241 return dl_b->bw != -1 && in __dl_overflow()
242 cap_scale(dl_b->bw, cap) < dl_b->total_bw - old_bw + new_bw; in __dl_overflow()
248 u64 old = dl_rq->running_bw; in __add_running_bw()
251 dl_rq->running_bw += dl_bw; in __add_running_bw()
252 SCHED_WARN_ON(dl_rq->running_bw < old); /* overflow */ in __add_running_bw()
253 SCHED_WARN_ON(dl_rq->running_bw > dl_rq->this_bw); in __add_running_bw()
261 u64 old = dl_rq->running_bw; in __sub_running_bw()
264 dl_rq->running_bw -= dl_bw; in __sub_running_bw()
265 SCHED_WARN_ON(dl_rq->running_bw > old); /* underflow */ in __sub_running_bw()
266 if (dl_rq->running_bw > old) in __sub_running_bw()
267 dl_rq->running_bw = 0; in __sub_running_bw()
275 u64 old = dl_rq->this_bw; in __add_rq_bw()
278 dl_rq->this_bw += dl_bw; in __add_rq_bw()
279 SCHED_WARN_ON(dl_rq->this_bw < old); /* overflow */ in __add_rq_bw()
285 u64 old = dl_rq->this_bw; in __sub_rq_bw()
288 dl_rq->this_bw -= dl_bw; in __sub_rq_bw()
289 SCHED_WARN_ON(dl_rq->this_bw > old); /* underflow */ in __sub_rq_bw()
290 if (dl_rq->this_bw > old) in __sub_rq_bw()
291 dl_rq->this_bw = 0; in __sub_rq_bw()
292 SCHED_WARN_ON(dl_rq->running_bw > dl_rq->this_bw); in __sub_rq_bw()
299 __add_rq_bw(dl_se->dl_bw, dl_rq); in add_rq_bw()
306 __sub_rq_bw(dl_se->dl_bw, dl_rq); in sub_rq_bw()
313 __add_running_bw(dl_se->dl_bw, dl_rq); in add_running_bw()
320 __sub_running_bw(dl_se->dl_bw, dl_rq); in sub_running_bw()
325 if (dl_se->dl_non_contending) { in dl_rq_change_utilization()
326 sub_running_bw(dl_se, &rq->dl); in dl_rq_change_utilization()
327 dl_se->dl_non_contending = 0; in dl_rq_change_utilization()
330 * If the timer handler is currently running and the in dl_rq_change_utilization()
331 * timer cannot be canceled, inactive_task_timer() in dl_rq_change_utilization()
336 if (hrtimer_try_to_cancel(&dl_se->inactive_timer) == 1) { in dl_rq_change_utilization()
341 __sub_rq_bw(dl_se->dl_bw, &rq->dl); in dl_rq_change_utilization()
342 __add_rq_bw(new_bw, &rq->dl); in dl_rq_change_utilization()
347 WARN_ON_ONCE(p->dl.flags & SCHED_FLAG_SUGOV); in dl_change_utilization()
352 dl_rq_change_utilization(task_rq(p), &p->dl, new_bw); in dl_change_utilization()
358 * The utilization of a task cannot be immediately removed from
360 * Instead, we have to wait for the so called "0-lag time".
362 * If a task blocks before the "0-lag time", a timer (the inactive
363 * timer) is armed, and running_bw is decreased when the timer
366 * If the task wakes up again before the inactive timer fires,
367 * the timer is canceled, whereas if the task wakes up after the
368 * inactive timer fired (and running_bw has been decreased) the
371 * is used to avoid race conditions between the inactive timer handler
377 * "ACTIVE non contending" task is a blocked task for which the "0-lag time"
378 * has not passed yet. An "INACTIVE" task is a task for which the "0-lag"
380 * +------------------+
382 * +------------------>+ contending |
384 * | +----+------+------+
387 * +--------+-------+ | |
388 * | | t >= 0-lag | | wakeup
389 * | INACTIVE |<---------------+ |
391 * +--------+-------+ | |
393 * | t < 0-lag | |
396 * | +----+------+------+
398 * +-------------------+ |
399 * inactive timer | non contending |
400 * fired +------------------+
403 * blocks, and checks if the 0-lag time already passed or
405 * in the second case, it arms the inactive timer).
413 struct hrtimer *timer = &dl_se->inactive_timer; in task_non_contending() local
415 struct dl_rq *dl_rq = &rq->dl; in task_non_contending()
419 * If this is a non-deadline task that has been boosted, in task_non_contending()
422 if (dl_se->dl_runtime == 0) in task_non_contending()
428 WARN_ON(dl_se->dl_non_contending); in task_non_contending()
430 zerolag_time = dl_se->deadline - in task_non_contending()
431 div64_long((dl_se->runtime * dl_se->dl_period), in task_non_contending()
432 dl_se->dl_runtime); in task_non_contending()
435 * Using relative times instead of the absolute "0-lag time" in task_non_contending()
438 zerolag_time -= rq_clock(rq); in task_non_contending()
441 * If the "0-lag time" already passed, decrease the active in task_non_contending()
442 * utilization now, instead of starting a timer in task_non_contending()
444 if ((zerolag_time < 0) || hrtimer_active(&dl_se->inactive_timer)) { in task_non_contending()
453 if (!dl_task(p) || READ_ONCE(p->__state) == TASK_DEAD) { in task_non_contending()
456 if (READ_ONCE(p->__state) == TASK_DEAD) in task_non_contending()
457 sub_rq_bw(dl_se, &rq->dl); in task_non_contending()
458 raw_spin_lock(&dl_b->lock); in task_non_contending()
459 __dl_sub(dl_b, dl_se->dl_bw, dl_bw_cpus(task_cpu(p))); in task_non_contending()
460 raw_spin_unlock(&dl_b->lock); in task_non_contending()
468 dl_se->dl_non_contending = 1; in task_non_contending()
472 hrtimer_start(timer, ns_to_ktime(zerolag_time), HRTIMER_MODE_REL_HARD); in task_non_contending()
480 * If this is a non-deadline task that has been boosted, in task_contending()
483 if (dl_se->dl_runtime == 0) in task_contending()
489 if (dl_se->dl_non_contending) { in task_contending()
490 dl_se->dl_non_contending = 0; in task_contending()
492 * If the timer handler is currently running and the in task_contending()
493 * timer cannot be canceled, inactive_task_timer() in task_contending()
498 if (hrtimer_try_to_cancel(&dl_se->inactive_timer) == 1) { in task_contending()
507 * when the "inactive timer" fired). in task_contending()
516 return rb_first_cached(&dl_rq->root) == &dl_se->rb_node; in is_leftmost()
523 raw_spin_lock_init(&dl_b->lock); in init_dl_bw()
525 dl_b->bw = -1; in init_dl_bw()
527 dl_b->bw = to_ratio(global_rt_period(), global_rt_runtime()); in init_dl_bw()
528 dl_b->total_bw = 0; in init_dl_bw()
533 dl_rq->root = RB_ROOT_CACHED; in init_dl_rq()
536 /* zero means no -deadline tasks */ in init_dl_rq()
537 dl_rq->earliest_dl.curr = dl_rq->earliest_dl.next = 0; in init_dl_rq()
539 dl_rq->overloaded = 0; in init_dl_rq()
540 dl_rq->pushable_dl_tasks_root = RB_ROOT_CACHED; in init_dl_rq()
542 init_dl_bw(&dl_rq->dl_bw); in init_dl_rq()
545 dl_rq->running_bw = 0; in init_dl_rq()
546 dl_rq->this_bw = 0; in init_dl_rq()
554 return atomic_read(&rq->rd->dlo_count); in dl_overloaded()
559 if (!rq->online) in dl_set_overload()
562 cpumask_set_cpu(rq->cpu, rq->rd->dlo_mask); in dl_set_overload()
570 atomic_inc(&rq->rd->dlo_count); in dl_set_overload()
575 if (!rq->online) in dl_clear_overload()
578 atomic_dec(&rq->rd->dlo_count); in dl_clear_overload()
579 cpumask_clear_cpu(rq->cpu, rq->rd->dlo_mask); in dl_clear_overload()
587 return dl_entity_preempt(&__node_2_pdl(a)->dl, &__node_2_pdl(b)->dl); in __pushable_less()
592 return !RB_EMPTY_ROOT(&rq->dl.pushable_dl_tasks_root.rb_root); in has_pushable_dl_tasks()
596 * The list of pushable -deadline task is not a plist, like in
597 * sched_rt.c, it is an rb-tree with tasks ordered by deadline.
603 WARN_ON_ONCE(!RB_EMPTY_NODE(&p->pushable_dl_tasks)); in enqueue_pushable_dl_task()
605 leftmost = rb_add_cached(&p->pushable_dl_tasks, in enqueue_pushable_dl_task()
606 &rq->dl.pushable_dl_tasks_root, in enqueue_pushable_dl_task()
609 rq->dl.earliest_dl.next = p->dl.deadline; in enqueue_pushable_dl_task()
611 if (!rq->dl.overloaded) { in enqueue_pushable_dl_task()
613 rq->dl.overloaded = 1; in enqueue_pushable_dl_task()
619 struct dl_rq *dl_rq = &rq->dl; in dequeue_pushable_dl_task()
620 struct rb_root_cached *root = &dl_rq->pushable_dl_tasks_root; in dequeue_pushable_dl_task()
623 if (RB_EMPTY_NODE(&p->pushable_dl_tasks)) in dequeue_pushable_dl_task()
626 leftmost = rb_erase_cached(&p->pushable_dl_tasks, root); in dequeue_pushable_dl_task()
628 dl_rq->earliest_dl.next = __node_2_pdl(leftmost)->dl.deadline; in dequeue_pushable_dl_task()
630 RB_CLEAR_NODE(&p->pushable_dl_tasks); in dequeue_pushable_dl_task()
632 if (!has_pushable_dl_tasks(rq) && rq->dl.overloaded) { in dequeue_pushable_dl_task()
634 rq->dl.overloaded = 0; in dequeue_pushable_dl_task()
642 return rq->online && dl_task(prev); in need_pull_dl_task()
656 queue_balance_callback(rq, &per_cpu(dl_push_head, rq->cpu), push_dl_tasks); in deadline_queue_push_tasks()
661 queue_balance_callback(rq, &per_cpu(dl_pull_head, rq->cpu), pull_dl_task); in deadline_queue_pull_task()
673 int cpu; in dl_task_offline_migration() local
676 * If we cannot preempt any rq, fall back to pick any in dl_task_offline_migration()
677 * online CPU: in dl_task_offline_migration()
679 cpu = cpumask_any_and(cpu_active_mask, p->cpus_ptr); in dl_task_offline_migration()
680 if (cpu >= nr_cpu_ids) { in dl_task_offline_migration()
682 * Failed to find any suitable CPU. in dl_task_offline_migration()
692 cpu = cpumask_any(cpu_active_mask); in dl_task_offline_migration()
694 later_rq = cpu_rq(cpu); in dl_task_offline_migration()
698 if (p->dl.dl_non_contending || p->dl.dl_throttled) { in dl_task_offline_migration()
700 * Inactive timer is armed (or callback is running, but in dl_task_offline_migration()
705 sub_running_bw(&p->dl, &rq->dl); in dl_task_offline_migration()
706 sub_rq_bw(&p->dl, &rq->dl); in dl_task_offline_migration()
708 add_rq_bw(&p->dl, &later_rq->dl); in dl_task_offline_migration()
709 add_running_bw(&p->dl, &later_rq->dl); in dl_task_offline_migration()
711 sub_rq_bw(&p->dl, &rq->dl); in dl_task_offline_migration()
712 add_rq_bw(&p->dl, &later_rq->dl); in dl_task_offline_migration()
720 dl_b = &rq->rd->dl_bw; in dl_task_offline_migration()
721 raw_spin_lock(&dl_b->lock); in dl_task_offline_migration()
722 __dl_sub(dl_b, p->dl.dl_bw, cpumask_weight(rq->rd->span)); in dl_task_offline_migration()
723 raw_spin_unlock(&dl_b->lock); in dl_task_offline_migration()
725 dl_b = &later_rq->rd->dl_bw; in dl_task_offline_migration()
726 raw_spin_lock(&dl_b->lock); in dl_task_offline_migration()
727 __dl_add(dl_b, p->dl.dl_bw, cpumask_weight(later_rq->rd->span)); in dl_task_offline_migration()
728 raw_spin_unlock(&dl_b->lock); in dl_task_offline_migration()
730 set_task_cpu(p, later_rq->cpu); in dl_task_offline_migration()
776 /* for non-boosted task, pi_of(dl_se) == dl_se */ in replenish_dl_new_period()
777 dl_se->deadline = rq_clock(rq) + pi_of(dl_se)->dl_deadline; in replenish_dl_new_period()
778 dl_se->runtime = pi_of(dl_se)->dl_runtime; in replenish_dl_new_period()
784 if (dl_se->dl_defer & !dl_se->dl_defer_running) { in replenish_dl_new_period()
785 dl_se->dl_throttled = 1; in replenish_dl_new_period()
786 dl_se->dl_defer_armed = 1; in replenish_dl_new_period()
793 * - the absolute deadline of the entity has to be placed at
795 * - the runtime of the entity has to be set to the maximum value.
797 * The capability of specifying such event is useful whenever a -deadline
808 WARN_ON(dl_time_before(rq_clock(rq), dl_se->deadline)); in setup_new_dl_entity()
811 * We are racing with the deadline timer. So, do nothing because in setup_new_dl_entity()
812 * the deadline timer handler will take care of properly recharging in setup_new_dl_entity()
815 if (dl_se->dl_throttled) in setup_new_dl_entity()
852 WARN_ON_ONCE(pi_of(dl_se)->dl_runtime <= 0); in replenish_dl_entity()
855 * This could be the case for a !-dl task that is boosted. in replenish_dl_entity()
864 if (dl_se->dl_deadline == 0 || in replenish_dl_entity()
865 (dl_se->dl_defer_armed && dl_entity_overflow(dl_se, rq_clock(rq)))) { in replenish_dl_entity()
866 dl_se->deadline = rq_clock(rq) + pi_of(dl_se)->dl_deadline; in replenish_dl_entity()
867 dl_se->runtime = pi_of(dl_se)->dl_runtime; in replenish_dl_entity()
870 if (dl_se->dl_yielded && dl_se->runtime > 0) in replenish_dl_entity()
871 dl_se->runtime = 0; in replenish_dl_entity()
879 while (dl_se->runtime <= 0) { in replenish_dl_entity()
880 dl_se->deadline += pi_of(dl_se)->dl_period; in replenish_dl_entity()
881 dl_se->runtime += pi_of(dl_se)->dl_runtime; in replenish_dl_entity()
886 * the future" with respect to rq->clock. If it's in replenish_dl_entity()
893 if (dl_time_before(dl_se->deadline, rq_clock(rq))) { in replenish_dl_entity()
898 if (dl_se->dl_yielded) in replenish_dl_entity()
899 dl_se->dl_yielded = 0; in replenish_dl_entity()
900 if (dl_se->dl_throttled) in replenish_dl_entity()
901 dl_se->dl_throttled = 0; in replenish_dl_entity()
907 if (dl_se->dl_defer_armed) { in replenish_dl_entity()
908 dl_se->dl_defer_armed = 0; in replenish_dl_entity()
915 * and arm the defer timer. in replenish_dl_entity()
917 if (dl_se->dl_defer && !dl_se->dl_defer_running && in replenish_dl_entity()
918 dl_time_before(rq_clock(dl_se->rq), dl_se->deadline - dl_se->runtime)) { in replenish_dl_entity()
919 if (!is_dl_boosted(dl_se) && dl_se->server_has_tasks(dl_se)) { in replenish_dl_entity()
922 * Set dl_se->dl_defer_armed and dl_throttled variables to in replenish_dl_entity()
926 dl_se->dl_defer_armed = 1; in replenish_dl_entity()
927 dl_se->dl_throttled = 1; in replenish_dl_entity()
930 * If for whatever reason (delays), a previous timer was in replenish_dl_entity()
934 hrtimer_try_to_cancel(&dl_se->dl_timer); in replenish_dl_entity()
935 dl_se->dl_defer_armed = 0; in replenish_dl_entity()
936 dl_se->dl_throttled = 0; in replenish_dl_entity()
943 * Here we check if --at time t-- an entity (which is probably being
954 * Documentation/scheduler/sched-deadline.rst for more information).
958 * runtime / (deadline - t) > dl_runtime / dl_deadline ,
979 * and (deadline - t), since t is rq->clock, is the time left in dl_entity_overflow()
988 left = (pi_of(dl_se)->dl_deadline >> DL_SCALE) * (dl_se->runtime >> DL_SCALE); in dl_entity_overflow()
989 right = ((dl_se->deadline - t) >> DL_SCALE) * in dl_entity_overflow()
990 (pi_of(dl_se)->dl_runtime >> DL_SCALE); in dl_entity_overflow()
996 * Revised wakeup rule [1]: For self-suspending tasks, rather then
997 * re-initializing task's runtime and deadline, the revised wakeup
1002 * runtime / (deadline - t) > dl_runtime / dl_deadline
1005 * runtime = (dl_runtime / dl_deadline) * (deadline - t)
1011 * bandwidth server revisited. SIGBED Rev. 11, 4 (January 2015), 19-24.
1016 u64 laxity = dl_se->deadline - rq_clock(rq); in update_dl_revised_wakeup()
1024 WARN_ON(dl_time_before(dl_se->deadline, rq_clock(rq))); in update_dl_revised_wakeup()
1026 dl_se->runtime = (dl_se->dl_density * laxity) >> BW_SHIFT; in update_dl_revised_wakeup()
1042 return dl_se->dl_deadline == dl_se->dl_period; in dl_is_implicit()
1047 * might need to be updated. This is done by a CBS wake up rule. There are two
1079 if (dl_time_before(dl_se->deadline, rq_clock(rq)) || in update_dl_entity()
1083 !dl_time_before(dl_se->deadline, rq_clock(rq)) && in update_dl_entity()
1090 } else if (dl_server(dl_se) && dl_se->dl_defer) { in update_dl_entity()
1095 if (!dl_se->dl_defer_running) { in update_dl_entity()
1096 dl_se->dl_defer_armed = 1; in update_dl_entity()
1097 dl_se->dl_throttled = 1; in update_dl_entity()
1104 return dl_se->deadline - dl_se->dl_deadline + dl_se->dl_period; in dl_next_period()
1110 * set the bandwidth replenishment timer to the replenishment instant
1113 * Notice that it is important for the caller to know if the timer
1119 struct hrtimer *timer = &dl_se->dl_timer; in start_dl_timer() local
1128 * We want the timer to fire at the deadline, but considering in start_dl_timer()
1129 * that it is actually coming from rq->clock and not from in start_dl_timer()
1132 * The deferred reservation will have its timer set to in start_dl_timer()
1133 * (deadline - runtime). At that point, the CBS rule will decide in start_dl_timer()
1138 if (dl_se->dl_defer_armed) { in start_dl_timer()
1139 WARN_ON_ONCE(!dl_se->dl_throttled); in start_dl_timer()
1140 act = ns_to_ktime(dl_se->deadline - dl_se->runtime); in start_dl_timer()
1142 /* act = deadline - rel-deadline + period */ in start_dl_timer()
1146 now = hrtimer_cb_get_time(timer); in start_dl_timer()
1147 delta = ktime_to_ns(now) - rq_clock(rq); in start_dl_timer()
1153 * start the timer in the past! in start_dl_timer()
1163 * harmless because we're holding task_rq()->lock, therefore the timer in start_dl_timer()
1167 if (!hrtimer_is_queued(timer)) { in start_dl_timer()
1170 hrtimer_start(timer, act, HRTIMER_MODE_ABS_HARD); in start_dl_timer()
1185 * Nothing relies on rq->lock after this, so its safe to drop in __push_dl_task()
1186 * rq->lock. in __push_dl_task()
1195 /* a defer timer will not be reset if the runtime consumed was < dl_server_min_res */
1198 static enum hrtimer_restart dl_server_timer(struct hrtimer *timer, struct sched_dl_entity *dl_se) in dl_server_timer() argument
1206 if (!dl_se->dl_throttled || !dl_se->dl_runtime) in dl_server_timer()
1212 if (!dl_se->dl_runtime) in dl_server_timer()
1215 if (!dl_se->server_has_tasks(dl_se)) { in dl_server_timer()
1220 if (dl_se->dl_defer_armed) { in dl_server_timer()
1223 * If so, it is possible to push the defer timer for this amount in dl_server_timer()
1225 * forwarding the timer for a too small amount of time. in dl_server_timer()
1227 if (dl_time_before(rq_clock(dl_se->rq), in dl_server_timer()
1228 (dl_se->deadline - dl_se->runtime - dl_server_min_res))) { in dl_server_timer()
1230 /* reset the defer timer */ in dl_server_timer()
1231 fw = dl_se->deadline - rq_clock(dl_se->rq) - dl_se->runtime; in dl_server_timer()
1233 hrtimer_forward_now(timer, ns_to_ktime(fw)); in dl_server_timer()
1237 dl_se->dl_defer_running = 1; in dl_server_timer()
1242 if (!dl_task(dl_se->rq->curr) || dl_entity_preempt(dl_se, &dl_se->rq->curr->dl)) in dl_server_timer()
1252 * This is the bandwidth enforcement timer callback. If here, we know
1253 * a task is not on its dl_rq, since the fact that the timer was running
1264 static enum hrtimer_restart dl_task_timer(struct hrtimer *timer) in dl_task_timer() argument
1266 struct sched_dl_entity *dl_se = container_of(timer, in dl_task_timer()
1274 return dl_server_timer(timer, dl_se); in dl_task_timer()
1294 * Spurious timer due to start_dl_timer() race; or we already received in dl_task_timer()
1297 if (!dl_se->dl_throttled) in dl_task_timer()
1304 * If the throttle happened during sched-out; like: in dl_task_timer()
1312 * prev->on_rq = 0; in dl_task_timer()
1315 * but do not enqueue -- wait for our wakeup to do that. in dl_task_timer()
1323 if (unlikely(!rq->online)) { in dl_task_timer()
1342 if (dl_task(rq->curr)) in dl_task_timer()
1363 struct hrtimer *timer = &dl_se->dl_timer; in init_dl_task_timer() local
1365 hrtimer_init(timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL_HARD); in init_dl_task_timer()
1366 timer->function = dl_task_timer; in init_dl_task_timer()
1372 * cannot use the runtime, and so it replenishes the task. This rule
1384 * task and set the replenishing timer to the begin of the next period,
1391 if (dl_time_before(dl_se->deadline, rq_clock(rq)) && in dl_check_constrained_dl()
1395 dl_se->dl_throttled = 1; in dl_check_constrained_dl()
1396 if (dl_se->runtime > 0) in dl_check_constrained_dl()
1397 dl_se->runtime = 0; in dl_check_constrained_dl()
1404 return (dl_se->runtime <= 0); in dl_runtime_exceeded()
1409 * GRUB reclaiming algorithm, the runtime is not decreased as "dq = -dt",
1410 * but as "dq = -(max{u, (Umax - Uinact - Uextra)} / Umax) dt",
1412 * utilization, Uinact is the (per-runqueue) inactive utilization, computed
1416 * Since rq->dl.running_bw and rq->dl.this_bw contain utilizations multiplied
1418 * Since rq->dl.bw_ratio contains 1 / Umax multiplied by 2^RATIO_SHIFT, dl_bw
1419 * is multiplied by rq->dl.bw_ratio and shifted right by RATIO_SHIFT.
1421 * larger than 2^(64 - 20 - 8), which is more than 64 seconds. So, overflow is
1427 u64 u_inact = rq->dl.this_bw - rq->dl.running_bw; /* Utot - Uact */ in grub_reclaim()
1430 * Instead of computing max{u, (u_max - u_inact - u_extra)}, we in grub_reclaim()
1431 * compare u_inact + u_extra with u_max - u, because u_inact + u_extra in grub_reclaim()
1432 * can be larger than u_max. So, u_max - u_inact - u_extra would be in grub_reclaim()
1435 if (u_inact + rq->dl.extra_bw > rq->dl.max_bw - dl_se->dl_bw) in grub_reclaim()
1436 u_act = dl_se->dl_bw; in grub_reclaim()
1438 u_act = rq->dl.max_bw - u_inact - rq->dl.extra_bw; in grub_reclaim()
1440 u_act = (u_act * rq->dl.bw_ratio) >> RATIO_SHIFT; in grub_reclaim()
1449 * For tasks that participate in GRUB, we implement GRUB-PA: the in dl_scaled_delta_exec()
1453 * according to current frequency and CPU maximum capacity. in dl_scaled_delta_exec()
1455 if (unlikely(dl_se->flags & SCHED_FLAG_RECLAIM)) { in dl_scaled_delta_exec()
1458 int cpu = cpu_of(rq); in dl_scaled_delta_exec() local
1459 unsigned long scale_freq = arch_scale_freq_capacity(cpu); in dl_scaled_delta_exec()
1460 unsigned long scale_cpu = arch_scale_cpu_capacity(cpu); in dl_scaled_delta_exec()
1477 if (unlikely(dl_se->dl_yielded)) in update_curr_dl_se()
1482 if (dl_server(dl_se) && dl_se->dl_throttled && !dl_se->dl_defer) in update_curr_dl_se()
1490 dl_se->runtime -= scaled_delta_exec; in update_curr_dl_se()
1500 if (dl_se->dl_defer && dl_se->dl_throttled && dl_runtime_exceeded(dl_se)) { in update_curr_dl_se()
1502 * If the server was previously activated - the starving condition in update_curr_dl_se()
1507 dl_se->dl_defer_running = 0; in update_curr_dl_se()
1509 hrtimer_try_to_cancel(&dl_se->dl_timer); in update_curr_dl_se()
1511 replenish_dl_new_period(dl_se, dl_se->rq); in update_curr_dl_se()
1514 * Not being able to start the timer seems problematic. If it could not in update_curr_dl_se()
1525 if (dl_runtime_exceeded(dl_se) || dl_se->dl_yielded) { in update_curr_dl_se()
1526 dl_se->dl_throttled = 1; in update_curr_dl_se()
1530 (dl_se->flags & SCHED_FLAG_DL_OVERRUN)) in update_curr_dl_se()
1531 dl_se->dl_overrun = 1; in update_curr_dl_se()
1535 update_stats_dequeue_dl(&rq->dl, dl_se, 0); in update_curr_dl_se()
1546 if (!is_leftmost(dl_se, &rq->dl)) in update_curr_dl_se()
1551 * The fair server (sole dl_server) does not account for real-time in update_curr_dl_se()
1554 if (dl_se == &rq->fair_server) in update_curr_dl_se()
1559 * Because -- for now -- we share the rt bandwidth, we need to in update_curr_dl_se()
1566 * using deadline servers -- however there's a few nasties to figure in update_curr_dl_se()
1570 struct rt_rq *rt_rq = &rq->rt; in update_curr_dl_se()
1572 raw_spin_lock(&rt_rq->rt_runtime_lock); in update_curr_dl_se()
1579 rt_rq->rt_time += delta_exec; in update_curr_dl_se()
1580 raw_spin_unlock(&rt_rq->rt_runtime_lock); in update_curr_dl_se()
1586 * In the non-defer mode, the idle time is not accounted, as the
1597 if (!rq->fair_server.dl_defer) in dl_server_update_idle_time()
1601 if (rq->fair_server.runtime < 0) in dl_server_update_idle_time()
1604 delta_exec = rq_clock_task(rq) - p->se.exec_start; in dl_server_update_idle_time()
1608 scaled_delta_exec = dl_scaled_delta_exec(rq, &rq->fair_server, delta_exec); in dl_server_update_idle_time()
1610 rq->fair_server.runtime -= scaled_delta_exec; in dl_server_update_idle_time()
1612 if (rq->fair_server.runtime < 0) { in dl_server_update_idle_time()
1613 rq->fair_server.dl_defer_running = 0; in dl_server_update_idle_time()
1614 rq->fair_server.runtime = 0; in dl_server_update_idle_time()
1617 p->se.exec_start = rq_clock_task(rq); in dl_server_update_idle_time()
1623 if (dl_se->dl_runtime) in dl_server_update()
1624 update_curr_dl_se(dl_se->rq, dl_se, delta_exec); in dl_server_update()
1629 struct rq *rq = dl_se->rq; in dl_server_start()
1642 dl_se->dl_server = 1; in dl_server_start()
1643 dl_se->dl_defer = 1; in dl_server_start()
1647 if (!dl_se->dl_runtime) in dl_server_start()
1651 if (!dl_task(dl_se->rq->curr) || dl_entity_preempt(dl_se, &rq->curr->dl)) in dl_server_start()
1652 resched_curr(dl_se->rq); in dl_server_start()
1657 if (!dl_se->dl_runtime) in dl_server_stop()
1661 hrtimer_try_to_cancel(&dl_se->dl_timer); in dl_server_stop()
1662 dl_se->dl_defer_armed = 0; in dl_server_stop()
1663 dl_se->dl_throttled = 0; in dl_server_stop()
1670 dl_se->rq = rq; in dl_server_init()
1671 dl_se->server_has_tasks = has_tasks; in dl_server_init()
1672 dl_se->server_pick_task = pick_task; in dl_server_init()
1677 u64 new_bw = dl_se->dl_bw; in __dl_server_attach_root()
1678 int cpu = cpu_of(rq); in __dl_server_attach_root() local
1682 guard(raw_spinlock)(&dl_b->lock); in __dl_server_attach_root()
1684 if (!dl_bw_cpus(cpu)) in __dl_server_attach_root()
1687 __dl_add(dl_b, new_bw, dl_bw_cpus(cpu)); in __dl_server_attach_root()
1692 u64 old_bw = init ? 0 : to_ratio(dl_se->dl_period, dl_se->dl_runtime); in dl_server_apply_params()
1694 struct rq *rq = dl_se->rq; in dl_server_apply_params()
1695 int cpu = cpu_of(rq); in dl_server_apply_params() local
1701 dl_b = dl_bw_of(cpu); in dl_server_apply_params()
1702 guard(raw_spinlock)(&dl_b->lock); in dl_server_apply_params()
1704 cpus = dl_bw_cpus(cpu); in dl_server_apply_params()
1705 cap = dl_bw_capacity(cpu); in dl_server_apply_params()
1708 return -EBUSY; in dl_server_apply_params()
1711 __add_rq_bw(new_bw, &rq->dl); in dl_server_apply_params()
1714 __dl_sub(dl_b, dl_se->dl_bw, cpus); in dl_server_apply_params()
1720 dl_se->dl_runtime = runtime; in dl_server_apply_params()
1721 dl_se->dl_deadline = period; in dl_server_apply_params()
1722 dl_se->dl_period = period; in dl_server_apply_params()
1724 dl_se->runtime = 0; in dl_server_apply_params()
1725 dl_se->deadline = 0; in dl_server_apply_params()
1727 dl_se->dl_bw = to_ratio(dl_se->dl_period, dl_se->dl_runtime); in dl_server_apply_params()
1728 dl_se->dl_density = to_ratio(dl_se->dl_deadline, dl_se->dl_runtime); in dl_server_apply_params()
1735 * a -deadline task and has not been removed from the dl_rq).
1739 struct task_struct *curr = rq->curr; in update_curr_dl()
1740 struct sched_dl_entity *dl_se = &curr->dl; in update_curr_dl()
1758 static enum hrtimer_restart inactive_task_timer(struct hrtimer *timer) in inactive_task_timer() argument
1760 struct sched_dl_entity *dl_se = container_of(timer, in inactive_task_timer()
1771 rq = dl_se->rq; in inactive_task_timer()
1781 if (!dl_task(p) || READ_ONCE(p->__state) == TASK_DEAD) { in inactive_task_timer()
1784 if (READ_ONCE(p->__state) == TASK_DEAD && dl_se->dl_non_contending) { in inactive_task_timer()
1785 sub_running_bw(&p->dl, dl_rq_of_se(&p->dl)); in inactive_task_timer()
1786 sub_rq_bw(&p->dl, dl_rq_of_se(&p->dl)); in inactive_task_timer()
1787 dl_se->dl_non_contending = 0; in inactive_task_timer()
1790 raw_spin_lock(&dl_b->lock); in inactive_task_timer()
1791 __dl_sub(dl_b, p->dl.dl_bw, dl_bw_cpus(task_cpu(p))); in inactive_task_timer()
1792 raw_spin_unlock(&dl_b->lock); in inactive_task_timer()
1799 if (dl_se->dl_non_contending == 0) in inactive_task_timer()
1802 sub_running_bw(dl_se, &rq->dl); in inactive_task_timer()
1803 dl_se->dl_non_contending = 0; in inactive_task_timer()
1818 struct hrtimer *timer = &dl_se->inactive_timer; in init_dl_inactive_task_timer() local
1820 hrtimer_init(timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL_HARD); in init_dl_inactive_task_timer()
1821 timer->function = inactive_task_timer; in init_dl_inactive_task_timer()
1833 if (dl_rq->earliest_dl.curr == 0 || in inc_dl_deadline()
1834 dl_time_before(deadline, dl_rq->earliest_dl.curr)) { in inc_dl_deadline()
1835 if (dl_rq->earliest_dl.curr == 0) in inc_dl_deadline()
1836 cpupri_set(&rq->rd->cpupri, rq->cpu, CPUPRI_HIGHER); in inc_dl_deadline()
1837 dl_rq->earliest_dl.curr = deadline; in inc_dl_deadline()
1838 cpudl_set(&rq->rd->cpudl, rq->cpu, deadline); in inc_dl_deadline()
1850 if (!dl_rq->dl_nr_running) { in dec_dl_deadline()
1851 dl_rq->earliest_dl.curr = 0; in dec_dl_deadline()
1852 dl_rq->earliest_dl.next = 0; in dec_dl_deadline()
1853 cpudl_clear(&rq->rd->cpudl, rq->cpu); in dec_dl_deadline()
1854 cpupri_set(&rq->rd->cpupri, rq->cpu, rq->rt.highest_prio.curr); in dec_dl_deadline()
1856 struct rb_node *leftmost = rb_first_cached(&dl_rq->root); in dec_dl_deadline()
1859 dl_rq->earliest_dl.curr = entry->deadline; in dec_dl_deadline()
1860 cpudl_set(&rq->rd->cpudl, rq->cpu, entry->deadline); in dec_dl_deadline()
1874 u64 deadline = dl_se->deadline; in inc_dl_tasks()
1876 dl_rq->dl_nr_running++; in inc_dl_tasks()
1885 WARN_ON(!dl_rq->dl_nr_running); in dec_dl_tasks()
1886 dl_rq->dl_nr_running--; in dec_dl_tasks()
1889 dec_dl_deadline(dl_rq, dl_se->deadline); in dec_dl_tasks()
1894 return dl_time_before(__node_2_dle(a)->deadline, __node_2_dle(b)->deadline); in __dl_less()
1906 return &dl_task_of(dl_se)->stats; in __schedstats_from_dl_se()
1956 state = READ_ONCE(p->__state); in update_stats_dequeue_dl()
1958 __schedstat_set(p->stats.sleep_start, in update_stats_dequeue_dl()
1962 __schedstat_set(p->stats.block_start, in update_stats_dequeue_dl()
1971 WARN_ON_ONCE(!RB_EMPTY_NODE(&dl_se->rb_node)); in __enqueue_dl_entity()
1973 rb_add_cached(&dl_se->rb_node, &dl_rq->root, __dl_less); in __enqueue_dl_entity()
1982 if (RB_EMPTY_NODE(&dl_se->rb_node)) in __dequeue_dl_entity()
1985 rb_erase_cached(&dl_se->rb_node, &dl_rq->root); in __dequeue_dl_entity()
1987 RB_CLEAR_NODE(&dl_se->rb_node); in __dequeue_dl_entity()
2003 * the replenishment timer will be set to the next period. in enqueue_dl_entity()
2005 if (!dl_se->dl_throttled && !dl_is_implicit(dl_se)) in enqueue_dl_entity()
2018 * its rq, the bandwidth timer callback (which clearly has not in enqueue_dl_entity()
2022 * task's state - in GRUB parlance, "inactive" vs "active contending"). in enqueue_dl_entity()
2027 if (!dl_se->dl_defer && dl_se->dl_throttled && !(flags & ENQUEUE_REPLENISH)) { in enqueue_dl_entity()
2045 dl_time_before(dl_se->deadline, rq_clock(rq_of_dl_se(dl_se)))) { in enqueue_dl_entity()
2053 if (dl_se->dl_throttled && start_dl_timer(dl_se)) in enqueue_dl_entity()
2057 * We're about to enqueue, make sure we're not ->dl_throttled! in enqueue_dl_entity()
2058 * In case the timer was not started, say because the defer time in enqueue_dl_entity()
2062 if (dl_se->dl_throttled) { in enqueue_dl_entity()
2063 hrtimer_try_to_cancel(&dl_se->dl_timer); in enqueue_dl_entity()
2064 dl_se->dl_defer_armed = 0; in enqueue_dl_entity()
2065 dl_se->dl_throttled = 0; in enqueue_dl_entity()
2083 * This check allows to start the inactive timer (or to immediately in dequeue_dl_entity()
2086 * (p->state == TASK_DEAD). We can handle the two cases in the same in dequeue_dl_entity()
2097 if (is_dl_boosted(&p->dl)) { in enqueue_task_dl()
2110 if (p->dl.dl_throttled) { in enqueue_task_dl()
2112 * The replenish timer needs to be canceled. No in enqueue_task_dl()
2116 * If the timer callback was running (hrtimer_try_to_cancel == -1), in enqueue_task_dl()
2119 if (hrtimer_try_to_cancel(&p->dl.dl_timer) == 1 && in enqueue_task_dl()
2120 !dl_server(&p->dl)) in enqueue_task_dl()
2122 p->dl.dl_throttled = 0; in enqueue_task_dl()
2124 } else if (!dl_prio(p->normal_prio)) { in enqueue_task_dl()
2130 * clear the flag, otherwise the task may wake up as throttled after in enqueue_task_dl()
2134 p->dl.dl_throttled = 0; in enqueue_task_dl()
2136 printk_deferred_once("sched: DL de-boosted task PID %d: REPLENISH flag missing\n", in enqueue_task_dl()
2143 update_stats_wait_start_dl(dl_rq_of_se(&p->dl), &p->dl); in enqueue_task_dl()
2145 if (p->on_rq == TASK_ON_RQ_MIGRATING) in enqueue_task_dl()
2148 enqueue_dl_entity(&p->dl, flags); in enqueue_task_dl()
2150 if (dl_server(&p->dl)) in enqueue_task_dl()
2153 if (!task_current(rq, p) && !p->dl.dl_throttled && p->nr_cpus_allowed > 1) in enqueue_task_dl()
2161 if (p->on_rq == TASK_ON_RQ_MIGRATING) in dequeue_task_dl()
2164 dequeue_dl_entity(&p->dl, flags); in dequeue_task_dl()
2165 if (!p->dl.dl_throttled && !dl_server(&p->dl)) in dequeue_task_dl()
2172 * Yield task semantic for -deadline tasks is:
2174 * get off from the CPU until our next instance, with
2186 * it and the bandwidth timer will wake it up and will give it in yield_task_dl()
2189 rq->curr->dl.dl_yielded = 1; in yield_task_dl()
2206 return (!rq->dl.dl_nr_running || in dl_task_is_earliest_deadline()
2207 dl_time_before(p->dl.deadline, in dl_task_is_earliest_deadline()
2208 rq->dl.earliest_dl.curr)); in dl_task_is_earliest_deadline()
2214 select_task_rq_dl(struct task_struct *p, int cpu, int flags) in select_task_rq_dl() argument
2223 rq = cpu_rq(cpu); in select_task_rq_dl()
2226 curr = READ_ONCE(rq->curr); /* unlocked access */ in select_task_rq_dl()
2229 * If we are dealing with a -deadline task, we must in select_task_rq_dl()
2230 * decide where to wake it up. in select_task_rq_dl()
2238 (curr->nr_cpus_allowed < 2 || in select_task_rq_dl()
2239 !dl_entity_preempt(&p->dl, &curr->dl)) && in select_task_rq_dl()
2240 p->nr_cpus_allowed > 1; in select_task_rq_dl()
2243 * Take the capacity of the CPU into account to in select_task_rq_dl()
2247 select_rq |= !dl_task_fits_capacity(p, cpu); in select_task_rq_dl()
2252 if (target != -1 && in select_task_rq_dl()
2254 cpu = target; in select_task_rq_dl()
2259 return cpu; in select_task_rq_dl()
2267 if (READ_ONCE(p->__state) != TASK_WAKING) in migrate_task_rq_dl()
2272 * Since p->state == TASK_WAKING, set_task_cpu() has been called in migrate_task_rq_dl()
2273 * from try_to_wake_up(). Hence, p->pi_lock is locked, but in migrate_task_rq_dl()
2274 * rq->lock is not... So, lock it in migrate_task_rq_dl()
2277 if (p->dl.dl_non_contending) { in migrate_task_rq_dl()
2279 sub_running_bw(&p->dl, &rq->dl); in migrate_task_rq_dl()
2280 p->dl.dl_non_contending = 0; in migrate_task_rq_dl()
2282 * If the timer handler is currently running and the in migrate_task_rq_dl()
2283 * timer cannot be canceled, inactive_task_timer() in migrate_task_rq_dl()
2288 if (hrtimer_try_to_cancel(&p->dl.inactive_timer) == 1) in migrate_task_rq_dl()
2291 sub_rq_bw(&p->dl, &rq->dl); in migrate_task_rq_dl()
2301 if (rq->curr->nr_cpus_allowed == 1 || in check_preempt_equal_dl()
2302 !cpudl_find(&rq->rd->cpudl, rq->curr, NULL)) in check_preempt_equal_dl()
2309 if (p->nr_cpus_allowed != 1 && in check_preempt_equal_dl()
2310 cpudl_find(&rq->rd->cpudl, p, NULL)) in check_preempt_equal_dl()
2318 if (!on_dl_rq(&p->dl) && need_pull_dl_task(rq, p)) { in balance_dl()
2321 * picked for load-balance and preemption/IRQs are still in balance_dl()
2335 * Only called when both the current and waking task are -deadline
2341 if (dl_entity_preempt(&p->dl, &rq->curr->dl)) { in wakeup_preempt_dl()
2351 if ((p->dl.deadline == rq->curr->dl.deadline) && in wakeup_preempt_dl()
2352 !test_tsk_need_resched(rq->curr)) in wakeup_preempt_dl()
2360 hrtick_start(rq, dl_se->runtime); in start_hrtick_dl()
2370 struct sched_dl_entity *dl_se = &p->dl; in set_next_task_dl()
2371 struct dl_rq *dl_rq = &rq->dl; in set_next_task_dl()
2373 p->se.exec_start = rq_clock_task(rq); in set_next_task_dl()
2374 if (on_dl_rq(&p->dl)) in set_next_task_dl()
2383 if (rq->curr->sched_class != &dl_sched_class) in set_next_task_dl()
2389 start_hrtick_dl(rq, &p->dl); in set_next_task_dl()
2394 struct rb_node *left = rb_first_cached(&dl_rq->root); in pick_next_dl_entity()
2403 * __pick_next_task_dl - Helper to pick the next -deadline task to run.
2409 struct dl_rq *dl_rq = &rq->dl; in __pick_task_dl()
2420 p = dl_se->server_pick_task(dl_se); in __pick_task_dl()
2422 dl_se->dl_yielded = 1; in __pick_task_dl()
2426 rq->dl_server = dl_se; in __pick_task_dl()
2441 struct sched_dl_entity *dl_se = &p->dl; in put_prev_task_dl()
2442 struct dl_rq *dl_rq = &rq->dl; in put_prev_task_dl()
2444 if (on_dl_rq(&p->dl)) in put_prev_task_dl()
2450 if (on_dl_rq(&p->dl) && p->nr_cpus_allowed > 1) in put_prev_task_dl()
2472 if (hrtick_enabled_dl(rq) && queued && p->dl.runtime > 0 && in task_tick_dl()
2473 is_leftmost(&p->dl, &rq->dl)) in task_tick_dl()
2474 start_hrtick_dl(rq, &p->dl); in task_tick_dl()
2480 * SCHED_DEADLINE tasks cannot fork and this is achieved through in task_fork_dl()
2490 static int pick_dl_task(struct rq *rq, struct task_struct *p, int cpu) in pick_dl_task() argument
2493 cpumask_test_cpu(cpu, &p->cpus_mask)) in pick_dl_task()
2500 * on the CPU, NULL otherwise:
2502 static struct task_struct *pick_earliest_pushable_dl_task(struct rq *rq, int cpu) in pick_earliest_pushable_dl_task() argument
2510 next_node = rb_first_cached(&rq->dl.pushable_dl_tasks_root); in pick_earliest_pushable_dl_task()
2516 if (pick_dl_task(rq, p, cpu)) in pick_earliest_pushable_dl_task()
2533 int cpu = task_cpu(task); in find_later_rq() local
2537 return -1; in find_later_rq()
2539 if (task->nr_cpus_allowed == 1) in find_later_rq()
2540 return -1; in find_later_rq()
2544 * first, then we can look for a suitable CPU. in find_later_rq()
2546 if (!cpudl_find(&task_rq(task)->rd->cpudl, task, later_mask)) in find_later_rq()
2547 return -1; in find_later_rq()
2558 * The last CPU where the task run is our first in find_later_rq()
2559 * guess, since it is most likely cache-hot there. in find_later_rq()
2561 if (cpumask_test_cpu(cpu, later_mask)) in find_later_rq()
2562 return cpu; in find_later_rq()
2568 this_cpu = -1; in find_later_rq()
2571 for_each_domain(cpu, sd) { in find_later_rq()
2572 if (sd->flags & SD_WAKE_AFFINE) { in find_later_rq()
2579 if (this_cpu != -1 && in find_later_rq()
2588 * Last chance: if a CPU being in both later_mask in find_later_rq()
2590 * choice. Of course, the latest possible CPU is in find_later_rq()
2605 if (this_cpu != -1) in find_later_rq()
2608 cpu = cpumask_any_distribute(later_mask); in find_later_rq()
2609 if (cpu < nr_cpu_ids) in find_later_rq()
2610 return cpu; in find_later_rq()
2612 return -1; in find_later_rq()
2620 int cpu; in find_lock_later_rq() local
2623 cpu = find_later_rq(task); in find_lock_later_rq()
2625 if ((cpu == -1) || (cpu == rq->cpu)) in find_lock_later_rq()
2628 later_rq = cpu_rq(cpu); in find_lock_later_rq()
2643 !cpumask_test_cpu(later_rq->cpu, &task->cpus_mask) || in find_lock_later_rq()
2655 * If the rq we found has no -deadline task, or in find_lock_later_rq()
2677 p = __node_2_pdl(rb_first_cached(&rq->dl.pushable_dl_tasks_root)); in pick_next_pushable_dl_task()
2679 WARN_ON_ONCE(rq->cpu != task_cpu(p)); in pick_next_pushable_dl_task()
2681 WARN_ON_ONCE(p->nr_cpus_allowed <= 1); in pick_next_pushable_dl_task()
2690 * See if the non running -deadline tasks on this rq
2691 * can be sent to some other CPU where they can preempt
2706 * If next_task preempts rq->curr, and rq->curr in push_dl_task()
2710 if (dl_task(rq->curr) && in push_dl_task()
2711 dl_time_before(next_task->dl.deadline, rq->curr->dl.deadline) && in push_dl_task()
2712 rq->curr->nr_cpus_allowed > 1) { in push_dl_task()
2720 if (WARN_ON(next_task == rq->curr)) in push_dl_task()
2733 * find_lock_later_rq releases rq->lock and it is in push_dl_task()
2740 * again, some other CPU will pull it when ready. in push_dl_task()
2755 set_task_cpu(next_task, later_rq->cpu); in push_dl_task()
2771 /* push_dl_task() will return true if it moved a -deadline task */ in push_dl_tasks()
2778 int this_cpu = this_rq->cpu, cpu; in pull_dl_task() local
2793 for_each_cpu(cpu, this_rq->rd->dlo_mask) { in pull_dl_task()
2794 if (this_cpu == cpu) in pull_dl_task()
2797 src_rq = cpu_rq(cpu); in pull_dl_task()
2803 if (this_rq->dl.dl_nr_running && in pull_dl_task()
2804 dl_time_before(this_rq->dl.earliest_dl.curr, in pull_dl_task()
2805 src_rq->dl.earliest_dl.next)) in pull_dl_task()
2808 /* Might drop this_rq->lock */ in pull_dl_task()
2816 if (src_rq->dl.dl_nr_running <= 1) in pull_dl_task()
2823 * - it preempts our current (if there's one), in pull_dl_task()
2824 * - it will preempt the last one we pulled (if any). in pull_dl_task()
2826 if (p && dl_time_before(p->dl.deadline, dmin) && in pull_dl_task()
2828 WARN_ON(p == src_rq->curr); in pull_dl_task()
2835 if (dl_time_before(p->dl.deadline, in pull_dl_task()
2836 src_rq->curr->dl.deadline)) in pull_dl_task()
2845 dmin = p->dl.deadline; in pull_dl_task()
2857 stop_one_cpu_nowait(src_rq->cpu, push_cpu_stop, in pull_dl_task()
2858 push_task, &src_rq->push_work); in pull_dl_task()
2875 !test_tsk_need_resched(rq->curr) && in task_woken_dl()
2876 p->nr_cpus_allowed > 1 && in task_woken_dl()
2877 dl_task(rq->curr) && in task_woken_dl()
2878 (rq->curr->nr_cpus_allowed < 2 || in task_woken_dl()
2879 !dl_entity_preempt(&p->dl, &rq->curr->dl))) { in task_woken_dl()
2893 src_rd = rq->rd; in set_cpus_allowed_dl()
2900 if (!cpumask_intersects(src_rd->span, ctx->new_mask)) { in set_cpus_allowed_dl()
2909 raw_spin_lock(&src_dl_b->lock); in set_cpus_allowed_dl()
2910 __dl_sub(src_dl_b, p->dl.dl_bw, dl_bw_cpus(task_cpu(p))); in set_cpus_allowed_dl()
2911 raw_spin_unlock(&src_dl_b->lock); in set_cpus_allowed_dl()
2917 /* Assumes rq->lock is held */
2920 if (rq->dl.overloaded) in rq_online_dl()
2923 cpudl_set_freecpu(&rq->rd->cpudl, rq->cpu); in rq_online_dl()
2924 if (rq->dl.dl_nr_running > 0) in rq_online_dl()
2925 cpudl_set(&rq->rd->cpudl, rq->cpu, rq->dl.earliest_dl.curr); in rq_online_dl()
2928 /* Assumes rq->lock is held */
2931 if (rq->dl.overloaded) in rq_offline_dl()
2934 cpudl_clear(&rq->rd->cpudl, rq->cpu); in rq_offline_dl()
2935 cpudl_clear_freecpu(&rq->rd->cpudl, rq->cpu); in rq_offline_dl()
2953 raw_spin_lock_irqsave(&p->pi_lock, rf.flags); in dl_add_task_root_domain()
2955 raw_spin_unlock_irqrestore(&p->pi_lock, rf.flags); in dl_add_task_root_domain()
2961 dl_b = &rq->rd->dl_bw; in dl_add_task_root_domain()
2962 raw_spin_lock(&dl_b->lock); in dl_add_task_root_domain()
2964 __dl_add(dl_b, p->dl.dl_bw, cpumask_weight(rq->rd->span)); in dl_add_task_root_domain()
2966 raw_spin_unlock(&dl_b->lock); in dl_add_task_root_domain()
2975 raw_spin_lock_irqsave(&rd->dl_bw.lock, flags); in dl_clear_root_domain()
2976 rd->dl_bw.total_bw = 0; in dl_clear_root_domain()
2977 raw_spin_unlock_irqrestore(&rd->dl_bw.lock, flags); in dl_clear_root_domain()
2985 * task_non_contending() can start the "inactive timer" (if the 0-lag in switched_from_dl()
2987 * the "inactive timer" fires, it can continue to consume its current in switched_from_dl()
2989 * SCHED_DEADLINE until the 0-lag time passes, inactive_task_timer() in switched_from_dl()
2992 if (task_on_rq_queued(p) && p->dl.dl_runtime) in switched_from_dl()
2993 task_non_contending(&p->dl); in switched_from_dl()
3003 * Inactive timer is armed. However, p is leaving DEADLINE and in switched_from_dl()
3008 if (p->dl.dl_non_contending) in switched_from_dl()
3009 sub_running_bw(&p->dl, &rq->dl); in switched_from_dl()
3010 sub_rq_bw(&p->dl, &rq->dl); in switched_from_dl()
3014 * We cannot use inactive_task_timer() to invoke sub_running_bw() in switched_from_dl()
3015 * at the 0-lag time, because the task could have been migrated in switched_from_dl()
3018 if (p->dl.dl_non_contending) in switched_from_dl()
3019 p->dl.dl_non_contending = 0; in switched_from_dl()
3022 * Since this might be the only -deadline task on the rq, in switched_from_dl()
3024 * from an overloaded CPU, if any. in switched_from_dl()
3026 if (!task_on_rq_queued(p) || rq->dl.dl_nr_running) in switched_from_dl()
3033 * When switching to -deadline, we may overload the rq, then
3038 if (hrtimer_try_to_cancel(&p->dl.inactive_timer) == 1) in switched_to_dl()
3049 add_rq_bw(&p->dl, &rq->dl); in switched_to_dl()
3054 if (rq->curr != p) { in switched_to_dl()
3056 if (p->nr_cpus_allowed > 1 && rq->dl.overloaded) in switched_to_dl()
3059 if (dl_task(rq->curr)) in switched_to_dl()
3069 * If the scheduling parameters of a -deadline task changed,
3085 if (!rq->dl.overloaded) in prio_changed_dl()
3094 if (dl_time_before(rq->dl.earliest_dl.curr, p->dl.deadline)) in prio_changed_dl()
3103 if (!dl_task(rq->curr) || in prio_changed_dl()
3104 dl_time_before(p->dl.deadline, rq->curr->dl.deadline)) in prio_changed_dl()
3117 static int task_is_throttled_dl(struct task_struct *p, int cpu) in task_is_throttled_dl() argument
3119 return p->dl.dl_throttled; in task_is_throttled_dl()
3169 int cpu, cpus, ret = 0; in sched_dl_global_validate() local
3177 for_each_possible_cpu(cpu) { in sched_dl_global_validate()
3180 if (dl_bw_visited(cpu, gen)) in sched_dl_global_validate()
3183 dl_b = dl_bw_of(cpu); in sched_dl_global_validate()
3184 cpus = dl_bw_cpus(cpu); in sched_dl_global_validate()
3186 raw_spin_lock_irqsave(&dl_b->lock, flags); in sched_dl_global_validate()
3187 if (new_bw * cpus < dl_b->total_bw) in sched_dl_global_validate()
3188 ret = -EBUSY; in sched_dl_global_validate()
3189 raw_spin_unlock_irqrestore(&dl_b->lock, flags); in sched_dl_global_validate()
3204 dl_rq->bw_ratio = 1 << RATIO_SHIFT; in init_dl_rq_bw_ratio()
3205 dl_rq->max_bw = dl_rq->extra_bw = 1 << BW_SHIFT; in init_dl_rq_bw_ratio()
3207 dl_rq->bw_ratio = to_ratio(global_rt_runtime(), in init_dl_rq_bw_ratio()
3208 global_rt_period()) >> (BW_SHIFT - RATIO_SHIFT); in init_dl_rq_bw_ratio()
3209 dl_rq->max_bw = dl_rq->extra_bw = in init_dl_rq_bw_ratio()
3216 u64 new_bw = -1; in sched_dl_do_global()
3219 int cpu; in sched_dl_do_global() local
3225 for_each_possible_cpu(cpu) { in sched_dl_do_global()
3228 if (dl_bw_visited(cpu, gen)) { in sched_dl_do_global()
3233 dl_b = dl_bw_of(cpu); in sched_dl_do_global()
3235 raw_spin_lock_irqsave(&dl_b->lock, flags); in sched_dl_do_global()
3236 dl_b->bw = new_bw; in sched_dl_do_global()
3237 raw_spin_unlock_irqrestore(&dl_b->lock, flags); in sched_dl_do_global()
3240 init_dl_rq_bw_ratio(&cpu_rq(cpu)->dl); in sched_dl_do_global()
3250 * This function is called while holding p's rq->lock.
3255 u64 period = attr->sched_period ?: attr->sched_deadline; in sched_dl_overflow()
3256 u64 runtime = attr->sched_runtime; in sched_dl_overflow()
3258 int cpus, err = -1, cpu = task_cpu(p); in sched_dl_overflow() local
3259 struct dl_bw *dl_b = dl_bw_of(cpu); in sched_dl_overflow()
3262 if (attr->sched_flags & SCHED_FLAG_SUGOV) in sched_dl_overflow()
3266 if (new_bw == p->dl.dl_bw && task_has_dl_policy(p)) in sched_dl_overflow()
3270 * Either if a task, enters, leave, or stays -deadline but changes in sched_dl_overflow()
3274 raw_spin_lock(&dl_b->lock); in sched_dl_overflow()
3275 cpus = dl_bw_cpus(cpu); in sched_dl_overflow()
3276 cap = dl_bw_capacity(cpu); in sched_dl_overflow()
3280 if (hrtimer_active(&p->dl.inactive_timer)) in sched_dl_overflow()
3281 __dl_sub(dl_b, p->dl.dl_bw, cpus); in sched_dl_overflow()
3285 !__dl_overflow(dl_b, cap, p->dl.dl_bw, new_bw)) { in sched_dl_overflow()
3289 * utilization change until the task's 0-lag point. in sched_dl_overflow()
3291 * timer" when the task is not inactive. in sched_dl_overflow()
3293 __dl_sub(dl_b, p->dl.dl_bw, cpus); in sched_dl_overflow()
3301 * (0-lag) time. in sched_dl_overflow()
3305 raw_spin_unlock(&dl_b->lock); in sched_dl_overflow()
3320 struct sched_dl_entity *dl_se = &p->dl; in __setparam_dl()
3322 dl_se->dl_runtime = attr->sched_runtime; in __setparam_dl()
3323 dl_se->dl_deadline = attr->sched_deadline; in __setparam_dl()
3324 dl_se->dl_period = attr->sched_period ?: dl_se->dl_deadline; in __setparam_dl()
3325 dl_se->flags = attr->sched_flags & SCHED_DL_FLAGS; in __setparam_dl()
3326 dl_se->dl_bw = to_ratio(dl_se->dl_period, dl_se->dl_runtime); in __setparam_dl()
3327 dl_se->dl_density = to_ratio(dl_se->dl_deadline, dl_se->dl_runtime); in __setparam_dl()
3332 struct sched_dl_entity *dl_se = &p->dl; in __getparam_dl()
3334 attr->sched_priority = p->rt_priority; in __getparam_dl()
3335 attr->sched_runtime = dl_se->dl_runtime; in __getparam_dl()
3336 attr->sched_deadline = dl_se->dl_deadline; in __getparam_dl()
3337 attr->sched_period = dl_se->dl_period; in __getparam_dl()
3338 attr->sched_flags &= ~SCHED_DL_FLAGS; in __getparam_dl()
3339 attr->sched_flags |= dl_se->flags; in __getparam_dl()
3343 * This function validates the new parameters of a -deadline task.
3357 if (attr->sched_flags & SCHED_FLAG_SUGOV) in __checkparam_dl()
3361 if (attr->sched_deadline == 0) in __checkparam_dl()
3368 if (attr->sched_runtime < (1ULL << DL_SCALE)) in __checkparam_dl()
3372 * Since we use the MSB for wrap-around and sign issues, make in __checkparam_dl()
3375 if (attr->sched_deadline & (1ULL << 63) || in __checkparam_dl()
3376 attr->sched_period & (1ULL << 63)) in __checkparam_dl()
3379 period = attr->sched_period; in __checkparam_dl()
3381 period = attr->sched_deadline; in __checkparam_dl()
3384 if (period < attr->sched_deadline || in __checkparam_dl()
3385 attr->sched_deadline < attr->sched_runtime) in __checkparam_dl()
3402 dl_se->dl_runtime = 0; in __dl_clear_params()
3403 dl_se->dl_deadline = 0; in __dl_clear_params()
3404 dl_se->dl_period = 0; in __dl_clear_params()
3405 dl_se->flags = 0; in __dl_clear_params()
3406 dl_se->dl_bw = 0; in __dl_clear_params()
3407 dl_se->dl_density = 0; in __dl_clear_params()
3409 dl_se->dl_throttled = 0; in __dl_clear_params()
3410 dl_se->dl_yielded = 0; in __dl_clear_params()
3411 dl_se->dl_non_contending = 0; in __dl_clear_params()
3412 dl_se->dl_overrun = 0; in __dl_clear_params()
3413 dl_se->dl_server = 0; in __dl_clear_params()
3416 dl_se->pi_se = dl_se; in __dl_clear_params()
3422 RB_CLEAR_NODE(&dl_se->rb_node); in init_dl_entity()
3430 struct sched_dl_entity *dl_se = &p->dl; in dl_param_changed()
3432 if (dl_se->dl_runtime != attr->sched_runtime || in dl_param_changed()
3433 dl_se->dl_deadline != attr->sched_deadline || in dl_param_changed()
3434 dl_se->dl_period != attr->sched_period || in dl_param_changed()
3435 dl_se->flags != (attr->sched_flags & SCHED_DL_FLAGS)) in dl_param_changed()
3452 raw_spin_lock_irqsave(&cur_dl_b->lock, flags); in dl_cpuset_cpumask_can_shrink()
3455 raw_spin_unlock_irqrestore(&cur_dl_b->lock, flags); in dl_cpuset_cpumask_can_shrink()
3467 static int dl_bw_manage(enum dl_bw_request req, int cpu, u64 dl_bw) in dl_bw_manage() argument
3474 dl_b = dl_bw_of(cpu); in dl_bw_manage()
3475 raw_spin_lock_irqsave(&dl_b->lock, flags); in dl_bw_manage()
3478 __dl_sub(dl_b, dl_bw, dl_bw_cpus(cpu)); in dl_bw_manage()
3480 unsigned long cap = dl_bw_capacity(cpu); in dl_bw_manage()
3491 __dl_add(dl_b, dl_bw, dl_bw_cpus(cpu)); in dl_bw_manage()
3495 raw_spin_unlock_irqrestore(&dl_b->lock, flags); in dl_bw_manage()
3498 return overflow ? -EBUSY : 0; in dl_bw_manage()
3501 int dl_bw_check_overflow(int cpu) in dl_bw_check_overflow() argument
3503 return dl_bw_manage(dl_bw_req_check_overflow, cpu, 0); in dl_bw_check_overflow()
3506 int dl_bw_alloc(int cpu, u64 dl_bw) in dl_bw_alloc() argument
3508 return dl_bw_manage(dl_bw_req_alloc, cpu, dl_bw); in dl_bw_alloc()
3511 void dl_bw_free(int cpu, u64 dl_bw) in dl_bw_free() argument
3513 dl_bw_manage(dl_bw_req_free, cpu, dl_bw); in dl_bw_free()
3518 void print_dl_stats(struct seq_file *m, int cpu) in print_dl_stats() argument
3520 print_dl_rq(m, cpu, &cpu_rq(cpu)->dl); in print_dl_stats()