15 static void cgroup_base_stat_flush(struct cgroup *cgrp, int cpu);
17 static struct cgroup_rstat_cpu *cgroup_rstat_cpu(struct cgroup *cgrp, int cpu)  in cgroup_rstat_cpu()  argument
19 	return per_cpu_ptr(cgrp->rstat_cpu, cpu);  in cgroup_rstat_cpu()
32 				     struct cgroup *cgrp, const bool fast_path)  in _cgroup_rstat_cpu_lock()  argument
48 			trace_cgroup_rstat_cpu_lock_contended_fastpath(cgrp, cpu, contended);  in _cgroup_rstat_cpu_lock()
50 			trace_cgroup_rstat_cpu_lock_contended(cgrp, cpu, contended);  in _cgroup_rstat_cpu_lock()
56 		trace_cgroup_rstat_cpu_locked_fastpath(cgrp, cpu, contended);  in _cgroup_rstat_cpu_lock()
58 		trace_cgroup_rstat_cpu_locked(cgrp, cpu, contended);  in _cgroup_rstat_cpu_lock()
65 			      struct cgroup *cgrp, unsigned long flags,  in _cgroup_rstat_cpu_unlock()  argument
69 		trace_cgroup_rstat_cpu_unlock_fastpath(cgrp, cpu, false);  in _cgroup_rstat_cpu_unlock()
71 		trace_cgroup_rstat_cpu_unlock(cgrp, cpu, false);  in _cgroup_rstat_cpu_unlock()
85 __bpf_kfunc void cgroup_rstat_updated(struct cgroup *cgrp, int cpu)  in cgroup_rstat_updated()  argument
98 	if (data_race(cgroup_rstat_cpu(cgrp, cpu)->updated_next))  in cgroup_rstat_updated()
101 	flags = _cgroup_rstat_cpu_lock(cpu_lock, cpu, cgrp, true);  in cgroup_rstat_updated()
105 		struct cgroup_rstat_cpu *rstatc = cgroup_rstat_cpu(cgrp, cpu);  in cgroup_rstat_updated()
106 		struct cgroup *parent = cgroup_parent(cgrp);  in cgroup_rstat_updated()
118 			rstatc->updated_next = cgrp;  in cgroup_rstat_updated()
124 		prstatc->updated_children = cgrp;  in cgroup_rstat_updated()
126 		cgrp = parent;  in cgroup_rstat_updated()
129 	_cgroup_rstat_cpu_unlock(cpu_lock, cpu, cgrp, flags, true);  in cgroup_rstat_updated()
266 __weak noinline void bpf_rstat_flush(struct cgroup *cgrp,  in bpf_rstat_flush()  argument
282 static inline void __cgroup_rstat_lock(struct cgroup *cgrp, int cpu_in_loop)  in __cgroup_rstat_lock()  argument
289 		trace_cgroup_rstat_lock_contended(cgrp, cpu_in_loop, contended);  in __cgroup_rstat_lock()
292 	trace_cgroup_rstat_locked(cgrp, cpu_in_loop, contended);  in __cgroup_rstat_lock()
295 static inline void __cgroup_rstat_unlock(struct cgroup *cgrp, int cpu_in_loop)  in __cgroup_rstat_unlock()  argument
298 	trace_cgroup_rstat_unlock(cgrp, cpu_in_loop, false);  in __cgroup_rstat_unlock()
303 static void cgroup_rstat_flush_locked(struct cgroup *cgrp)  in cgroup_rstat_flush_locked()  argument
311 		struct cgroup *pos = cgroup_rstat_updated_list(cgrp, cpu);  in cgroup_rstat_flush_locked()
328 			__cgroup_rstat_unlock(cgrp, cpu);  in cgroup_rstat_flush_locked()
331 			__cgroup_rstat_lock(cgrp, cpu);  in cgroup_rstat_flush_locked()
349 __bpf_kfunc void cgroup_rstat_flush(struct cgroup *cgrp)  in cgroup_rstat_flush()  argument
353 	__cgroup_rstat_lock(cgrp, -1);  in cgroup_rstat_flush()
354 	cgroup_rstat_flush_locked(cgrp);  in cgroup_rstat_flush()
355 	__cgroup_rstat_unlock(cgrp, -1);  in cgroup_rstat_flush()
367 void cgroup_rstat_flush_hold(struct cgroup *cgrp)  in cgroup_rstat_flush_hold()  argument
371 	__cgroup_rstat_lock(cgrp, -1);  in cgroup_rstat_flush_hold()
372 	cgroup_rstat_flush_locked(cgrp);  in cgroup_rstat_flush_hold()
379 void cgroup_rstat_flush_release(struct cgroup *cgrp)  in cgroup_rstat_flush_release()  argument
382 	__cgroup_rstat_unlock(cgrp, -1);  in cgroup_rstat_flush_release()
385 int cgroup_rstat_init(struct cgroup *cgrp)  in cgroup_rstat_init()  argument
390 	if (!cgrp->rstat_cpu) {  in cgroup_rstat_init()
391 		cgrp->rstat_cpu = alloc_percpu(struct cgroup_rstat_cpu);  in cgroup_rstat_init()
392 		if (!cgrp->rstat_cpu)  in cgroup_rstat_init()
398 		struct cgroup_rstat_cpu *rstatc = cgroup_rstat_cpu(cgrp, cpu);  in cgroup_rstat_init()
400 		rstatc->updated_children = cgrp;  in cgroup_rstat_init()
407 void cgroup_rstat_exit(struct cgroup *cgrp)  in cgroup_rstat_exit()  argument
411 	cgroup_rstat_flush(cgrp);  in cgroup_rstat_exit()
415 		struct cgroup_rstat_cpu *rstatc = cgroup_rstat_cpu(cgrp, cpu);  in cgroup_rstat_exit()
417 		if (WARN_ON_ONCE(rstatc->updated_children != cgrp) ||  in cgroup_rstat_exit()
422 	free_percpu(cgrp->rstat_cpu);  in cgroup_rstat_exit()
423 	cgrp->rstat_cpu = NULL;  in cgroup_rstat_exit()
460 static void cgroup_base_stat_flush(struct cgroup *cgrp, int cpu)  in cgroup_base_stat_flush()  argument
462 	struct cgroup_rstat_cpu *rstatc = cgroup_rstat_cpu(cgrp, cpu);  in cgroup_base_stat_flush()
463 	struct cgroup *parent = cgroup_parent(cgrp);  in cgroup_base_stat_flush()
480 	cgroup_base_stat_add(&cgrp->bstat, &delta);  in cgroup_base_stat_flush()
486 		delta = cgrp->bstat;  in cgroup_base_stat_flush()
487 		cgroup_base_stat_sub(&delta, &cgrp->last_bstat);  in cgroup_base_stat_flush()
489 		cgroup_base_stat_add(&cgrp->last_bstat, &delta);  in cgroup_base_stat_flush()
500 cgroup_base_stat_cputime_account_begin(struct cgroup *cgrp, unsigned long *flags)  in cgroup_base_stat_cputime_account_begin()  argument
504 	rstatc = get_cpu_ptr(cgrp->rstat_cpu);  in cgroup_base_stat_cputime_account_begin()
509 static void cgroup_base_stat_cputime_account_end(struct cgroup *cgrp,  in cgroup_base_stat_cputime_account_end()  argument
514 	cgroup_rstat_updated(cgrp, smp_processor_id());  in cgroup_base_stat_cputime_account_end()
518 void __cgroup_account_cputime(struct cgroup *cgrp, u64 delta_exec)  in __cgroup_account_cputime()  argument
523 	rstatc = cgroup_base_stat_cputime_account_begin(cgrp, &flags);  in __cgroup_account_cputime()
525 	cgroup_base_stat_cputime_account_end(cgrp, rstatc, flags);  in __cgroup_account_cputime()
528 void __cgroup_account_cputime_field(struct cgroup *cgrp,  in __cgroup_account_cputime_field()  argument
534 	rstatc = cgroup_base_stat_cputime_account_begin(cgrp, &flags);  in __cgroup_account_cputime_field()
555 	cgroup_base_stat_cputime_account_end(cgrp, rstatc, flags);  in __cgroup_account_cputime_field()
610 	struct cgroup *cgrp = seq_css(seq)->cgroup;  in cgroup_base_stat_cputime_show()  local
613 	if (cgroup_parent(cgrp)) {  in cgroup_base_stat_cputime_show()
614 		cgroup_rstat_flush_hold(cgrp);  in cgroup_base_stat_cputime_show()
615 		usage = cgrp->bstat.cputime.sum_exec_runtime;  in cgroup_base_stat_cputime_show()
616 		cputime_adjust(&cgrp->bstat.cputime, &cgrp->prev_cputime,  in cgroup_base_stat_cputime_show()
618 		cgroup_rstat_flush_release(cgrp);  in cgroup_base_stat_cputime_show()
621 		root_cgroup_cputime(&cgrp->bstat);  in cgroup_base_stat_cputime_show()
622 		usage = cgrp->bstat.cputime.sum_exec_runtime;  in cgroup_base_stat_cputime_show()
623 		utime = cgrp->bstat.cputime.utime;  in cgroup_base_stat_cputime_show()
624 		stime = cgrp->bstat.cputime.stime;  in cgroup_base_stat_cputime_show()
636 	cgroup_force_idle_show(seq, &cgrp->bstat);  in cgroup_base_stat_cputime_show()