1 // SPDX-License-Identifier: GPL-2.0-or-later
3 #include "cpuset-internal.h"
10 	struct cpuset *cs;  member
14  * Frequency meter - How fast is some event occurring?
18  *   fmeter_init() - initialize a frequency meter.
19  *   fmeter_markevent() - called each time the event happens.
20  *   fmeter_getrate() - returns the recent rate of such events.
21  *   fmeter_update() - internal routine used to update fmeter.
28  * The filter is single-pole low-pass recursive (IIR).  The time unit
29  * is 1 second.  Arithmetic is done using 32-bit integers scaled to
33  * has a half-life of 10 seconds, meaning that if the events quit
58 #define FM_COEF 933		/* coefficient for half-life of 10 secs */
66 	fmp->cnt = 0;  in fmeter_init()
67 	fmp->val = 0;  in fmeter_init()
68 	fmp->time = 0;  in fmeter_init()
69 	spin_lock_init(&fmp->lock);  in fmeter_init()
72 /* Internal meter update - process cnt events and update value */
79 	ticks = now - fmp->time;  in fmeter_update()
81 	if (ticks == 0)  in fmeter_update()
85 	while (ticks-- > 0)  in fmeter_update()
86 		fmp->val = (FM_COEF * fmp->val) / FM_SCALE;  in fmeter_update()
87 	fmp->time = now;  in fmeter_update()
89 	fmp->val += ((FM_SCALE - FM_COEF) * fmp->cnt) / FM_SCALE;  in fmeter_update()
90 	fmp->cnt = 0;  in fmeter_update()
96 	spin_lock(&fmp->lock);  in fmeter_markevent()
98 	fmp->cnt = min(FM_MAXCNT, fmp->cnt + FM_SCALE);  in fmeter_markevent()
99 	spin_unlock(&fmp->lock);  in fmeter_markevent()
107 	spin_lock(&fmp->lock);  in fmeter_getrate()
109 	val = fmp->val;  in fmeter_getrate()
110 	spin_unlock(&fmp->lock);  in fmeter_getrate()
123  * __cpuset_memory_pressure_bump - keep stats of per-cpuset reclaims.
134  * Display to user space in the per-cpuset read-only file
143 	fmeter_markevent(&task_cs(current)->fmeter);  in __cpuset_memory_pressure_bump()
147 static int update_relax_domain_level(struct cpuset *cs, s64 val)  in update_relax_domain_level()  argument
150 	if (val < -1 || val > sched_domain_level_max + 1)  in update_relax_domain_level()
151 		return -EINVAL;  in update_relax_domain_level()
154 	if (val != cs->relax_domain_level) {  in update_relax_domain_level()
155 		cs->relax_domain_level = val;  in update_relax_domain_level()
156 		if (!cpumask_empty(cs->cpus_allowed) &&  in update_relax_domain_level()
157 		    is_sched_load_balance(cs))  in update_relax_domain_level()
161 	return 0;  in update_relax_domain_level()
167 	struct cpuset *cs = css_cs(css);  in cpuset_write_s64()  local
168 	cpuset_filetype_t type = cft->private;  in cpuset_write_s64()
169 	int retval = -ENODEV;  in cpuset_write_s64()
173 	if (!is_cpuset_online(cs))  in cpuset_write_s64()
178 		retval = update_relax_domain_level(cs, val);  in cpuset_write_s64()
181 		retval = -EINVAL;  in cpuset_write_s64()
192 	struct cpuset *cs = css_cs(css);  in cpuset_read_s64()  local
193 	cpuset_filetype_t type = cft->private;  in cpuset_read_s64()
197 		return cs->relax_domain_level;  in cpuset_read_s64()
203 	return 0;  in cpuset_read_s64()
212 void cpuset1_update_task_spread_flags(struct cpuset *cs,  in cpuset1_update_task_spread_flags()  argument
218 	if (is_spread_page(cs))  in cpuset1_update_task_spread_flags()
223 	if (is_spread_slab(cs))  in cpuset1_update_task_spread_flags()
230  * cpuset1_update_tasks_flags - update the spread flags of tasks in the cpuset.
231  * @cs: the cpuset in which each task's spread flags needs to be changed
233  * Iterate through each task of @cs updating its spread flags.  As this
237 void cpuset1_update_tasks_flags(struct cpuset *cs)  in cpuset1_update_tasks_flags()  argument
242 	css_task_iter_start(&cs->css, 0, &it);  in cpuset1_update_tasks_flags()
244 		cpuset1_update_task_spread_flags(cs, task);  in cpuset1_update_tasks_flags()
253  * cpuset to its next-highest non-empty parent.
255 static void remove_tasks_in_empty_cpuset(struct cpuset *cs)  in remove_tasks_in_empty_cpuset()  argument
260 	 * Find its next-highest non-empty parent, (top cpuset  in remove_tasks_in_empty_cpuset()
263 	parent = parent_cs(cs);  in remove_tasks_in_empty_cpuset()
264 	while (cpumask_empty(parent->cpus_allowed) ||  in remove_tasks_in_empty_cpuset()
265 			nodes_empty(parent->mems_allowed))  in remove_tasks_in_empty_cpuset()
268 	if (cgroup_transfer_tasks(parent->css.cgroup, cs->css.cgroup)) {  in remove_tasks_in_empty_cpuset()
270 		pr_cont_cgroup_name(cs->css.cgroup);  in remove_tasks_in_empty_cpuset()
280 	remove_tasks_in_empty_cpuset(s->cs);  in cpuset_migrate_tasks_workfn()
281 	css_put(&s->cs->css);  in cpuset_migrate_tasks_workfn()
285 void cpuset1_hotplug_update_tasks(struct cpuset *cs,  in cpuset1_hotplug_update_tasks()  argument
292 	cpumask_copy(cs->cpus_allowed, new_cpus);  in cpuset1_hotplug_update_tasks()
293 	cpumask_copy(cs->effective_cpus, new_cpus);  in cpuset1_hotplug_update_tasks()
294 	cs->mems_allowed = *new_mems;  in cpuset1_hotplug_update_tasks()
295 	cs->effective_mems = *new_mems;  in cpuset1_hotplug_update_tasks()
302 	if (cpus_updated && !cpumask_empty(cs->cpus_allowed))  in cpuset1_hotplug_update_tasks()
303 		cpuset_update_tasks_cpumask(cs, new_cpus);  in cpuset1_hotplug_update_tasks()
304 	if (mems_updated && !nodes_empty(cs->mems_allowed))  in cpuset1_hotplug_update_tasks()
305 		cpuset_update_tasks_nodemask(cs);  in cpuset1_hotplug_update_tasks()
307 	is_empty = cpumask_empty(cs->cpus_allowed) ||  in cpuset1_hotplug_update_tasks()
308 		   nodes_empty(cs->mems_allowed);  in cpuset1_hotplug_update_tasks()
315 	if (is_empty && cs->css.cgroup->nr_populated_csets &&  in cpuset1_hotplug_update_tasks()
316 	    css_tryget_online(&cs->css)) {  in cpuset1_hotplug_update_tasks()
321 			css_put(&cs->css);  in cpuset1_hotplug_update_tasks()
325 		s->cs = cs;  in cpuset1_hotplug_update_tasks()
326 		INIT_WORK(&s->work, cpuset_migrate_tasks_workfn);  in cpuset1_hotplug_update_tasks()
327 		schedule_work(&s->work);  in cpuset1_hotplug_update_tasks()
332  * is_cpuset_subset(p, q) - Is cpuset p a subset of cpuset q?
341 	return	cpumask_subset(p->cpus_allowed, q->cpus_allowed) &&  in is_cpuset_subset()
342 		nodes_subset(p->mems_allowed, q->mems_allowed) &&  in is_cpuset_subset()
348  * cpuset1_validate_change() - Validate conditions specific to legacy (v1)
360 	ret = -EBUSY;  in cpuset1_validate_change()
366 	ret = -EACCES;  in cpuset1_validate_change()
371 	ret = 0;  in cpuset1_validate_change()
378 	struct cpuset *cs = css_cs(css);  in cpuset_read_u64()  local
379 	cpuset_filetype_t type = cft->private;  in cpuset_read_u64()
383 		return is_cpu_exclusive(cs);  in cpuset_read_u64()
385 		return is_mem_exclusive(cs);  in cpuset_read_u64()
387 		return is_mem_hardwall(cs);  in cpuset_read_u64()
389 		return is_sched_load_balance(cs);  in cpuset_read_u64()
391 		return is_memory_migrate(cs);  in cpuset_read_u64()
395 		return fmeter_getrate(&cs->fmeter);  in cpuset_read_u64()
397 		return is_spread_page(cs);  in cpuset_read_u64()
399 		return is_spread_slab(cs);  in cpuset_read_u64()
405 	return 0;  in cpuset_read_u64()
411 	struct cpuset *cs = css_cs(css);  in cpuset_write_u64()  local
412 	cpuset_filetype_t type = cft->private;  in cpuset_write_u64()
413 	int retval = 0;  in cpuset_write_u64()
417 	if (!is_cpuset_online(cs)) {  in cpuset_write_u64()
418 		retval = -ENODEV;  in cpuset_write_u64()
424 		retval = cpuset_update_flag(CS_CPU_EXCLUSIVE, cs, val);  in cpuset_write_u64()
427 		retval = cpuset_update_flag(CS_MEM_EXCLUSIVE, cs, val);  in cpuset_write_u64()
430 		retval = cpuset_update_flag(CS_MEM_HARDWALL, cs, val);  in cpuset_write_u64()
433 		retval = cpuset_update_flag(CS_SCHED_LOAD_BALANCE, cs, val);  in cpuset_write_u64()
436 		retval = cpuset_update_flag(CS_MEMORY_MIGRATE, cs, val);  in cpuset_write_u64()
442 		retval = cpuset_update_flag(CS_SPREAD_PAGE, cs, val);  in cpuset_write_u64()
445 		retval = cpuset_update_flag(CS_SPREAD_SLAB, cs, val);  in cpuset_write_u64()
448 		retval = -EINVAL;  in cpuset_write_u64()