616 		count += link->cset->nr_tasks;  in __cgroup_task_count()
752 static bool css_set_threaded(struct css_set *cset)  in css_set_threaded()  argument
754 	return cset->dom_cset != cset;  in css_set_threaded()
766 static bool css_set_populated(struct css_set *cset)  in css_set_populated()  argument
770 	return !list_empty(&cset->tasks) || !list_empty(&cset->mg_tasks);  in css_set_populated()
830 static void css_set_update_populated(struct css_set *cset, bool populated)  in css_set_update_populated()  argument
836 	list_for_each_entry(link, &cset->cgrp_links, cgrp_link)  in css_set_update_populated()
846 static void css_set_skip_task_iters(struct css_set *cset,  in css_set_skip_task_iters()  argument
851 	list_for_each_entry_safe(it, pos, &cset->task_iters, iters_node)  in css_set_skip_task_iters()
925 void put_css_set_locked(struct css_set *cset)  in put_css_set_locked()  argument
933 	if (!refcount_dec_and_test(&cset->refcount))  in put_css_set_locked()
936 	WARN_ON_ONCE(!list_empty(&cset->threaded_csets));  in put_css_set_locked()
940 		list_del(&cset->e_cset_node[ssid]);  in put_css_set_locked()
941 		css_put(cset->subsys[ssid]);  in put_css_set_locked()
943 	hash_del(&cset->hlist);  in put_css_set_locked()
946 	list_for_each_entry_safe(link, tmp_link, &cset->cgrp_links, cgrp_link) {  in put_css_set_locked()
954 	if (css_set_threaded(cset)) {  in put_css_set_locked()
955 		list_del(&cset->threaded_csets_node);  in put_css_set_locked()
956 		put_css_set_locked(cset->dom_cset);  in put_css_set_locked()
959 	kfree_rcu(cset, rcu_head);  in put_css_set_locked()
972 static bool compare_css_sets(struct css_set *cset,  in compare_css_sets()  argument
985 	if (memcmp(template, cset->subsys, sizeof(cset->subsys)))  in compare_css_sets()
995 	if (new_dfl_cgrp->dom_cgrp != cset->dom_cset->dfl_cgrp)  in compare_css_sets()
1004 	l1 = &cset->cgrp_links;  in compare_css_sets()
1013 		if (l1 == &cset->cgrp_links) {  in compare_css_sets()
1057 	struct css_set *cset;  in find_existing_css_set()  local
1083 	hash_for_each_possible(css_set_table, cset, hlist, key) {  in find_existing_css_set()
1084 		if (!compare_css_sets(cset, old_cset, cgrp, template))  in find_existing_css_set()
1088 		return cset;  in find_existing_css_set()
1137 static void link_css_set(struct list_head *tmp_links, struct css_set *cset,  in link_css_set()  argument
1145 		cset->dfl_cgrp = cgrp;  in link_css_set()
1148 	link->cset = cset;  in link_css_set()
1156 	list_add_tail(&link->cgrp_link, &cset->cgrp_links);  in link_css_set()
1174 	struct css_set *cset;  in find_css_set()  local
1186 	cset = find_existing_css_set(old_cset, cgrp, template);  in find_css_set()
1187 	if (cset)  in find_css_set()
1188 		get_css_set(cset);  in find_css_set()
1191 	if (cset)  in find_css_set()
1192 		return cset;  in find_css_set()
1194 	cset = kzalloc(sizeof(*cset), GFP_KERNEL);  in find_css_set()
1195 	if (!cset)  in find_css_set()
1200 		kfree(cset);  in find_css_set()
1204 	refcount_set(&cset->refcount, 1);  in find_css_set()
1205 	cset->dom_cset = cset;  in find_css_set()
1206 	INIT_LIST_HEAD(&cset->tasks);  in find_css_set()
1207 	INIT_LIST_HEAD(&cset->mg_tasks);  in find_css_set()
1208 	INIT_LIST_HEAD(&cset->dying_tasks);  in find_css_set()
1209 	INIT_LIST_HEAD(&cset->task_iters);  in find_css_set()
1210 	INIT_LIST_HEAD(&cset->threaded_csets);  in find_css_set()
1211 	INIT_HLIST_NODE(&cset->hlist);  in find_css_set()
1212 	INIT_LIST_HEAD(&cset->cgrp_links);  in find_css_set()
1213 	INIT_LIST_HEAD(&cset->mg_src_preload_node);  in find_css_set()
1214 	INIT_LIST_HEAD(&cset->mg_dst_preload_node);  in find_css_set()
1215 	INIT_LIST_HEAD(&cset->mg_node);  in find_css_set()
1219 	memcpy(cset->subsys, template, sizeof(cset->subsys));  in find_css_set()
1228 		link_css_set(&tmp_links, cset, c);  in find_css_set()
1236 	key = css_set_hash(cset->subsys);  in find_css_set()
1237 	hash_add(css_set_table, &cset->hlist, key);  in find_css_set()
1240 		struct cgroup_subsys_state *css = cset->subsys[ssid];  in find_css_set()
1242 		list_add_tail(&cset->e_cset_node[ssid],  in find_css_set()
1255 	if (cgroup_is_threaded(cset->dfl_cgrp)) {  in find_css_set()
1258 		dcset = find_css_set(cset, cset->dfl_cgrp->dom_cgrp);  in find_css_set()
1260 			put_css_set(cset);  in find_css_set()
1265 		cset->dom_cset = dcset;  in find_css_set()
1266 		list_add_tail(&cset->threaded_csets_node,  in find_css_set()
1271 	return cset;  in find_css_set()
1369 static inline struct cgroup *__cset_cgroup_from_root(struct css_set *cset,  in __cset_cgroup_from_root()  argument
1374 	if (cset == &init_css_set) {  in __cset_cgroup_from_root()
1377 		res_cgroup = cset->dfl_cgrp;  in __cset_cgroup_from_root()
1382 		list_for_each_entry(link, &cset->cgrp_links, cgrp_link) {  in __cset_cgroup_from_root()
1412 	struct css_set *cset;  in current_cgns_cgroup_from_root()  local
1418 	cset = current->nsproxy->cgroup_ns->root_cset;  in current_cgns_cgroup_from_root()
1419 	res = __cset_cgroup_from_root(cset, root);  in current_cgns_cgroup_from_root()
1444 	struct css_set *cset;  in current_cgns_cgroup_dfl()  local
1447 		cset = current->nsproxy->cgroup_ns->root_cset;  in current_cgns_cgroup_dfl()
1448 		return __cset_cgroup_from_root(cset, &cgrp_dfl_root);  in current_cgns_cgroup_dfl()
1461 static struct cgroup *cset_cgroup_from_root(struct css_set *cset,  in cset_cgroup_from_root()  argument
1466 	return __cset_cgroup_from_root(cset, root);  in cset_cgroup_from_root()
1829 		struct css_set *cset, *cset_pos;  in rebind_subsystems()  local
1849 		list_for_each_entry_safe(cset, cset_pos, &scgrp->e_csets[ss->id],  in rebind_subsystems()
1851 			list_move_tail(&cset->e_cset_node[ss->id],  in rebind_subsystems()
1860 			list_for_each_entry(it, &cset->task_iters, iters_node)  in rebind_subsystems()
2090 	struct css_set *cset;  in cgroup_setup_root()  local
2161 	hash_for_each(css_set_table, i, cset, hlist) {  in cgroup_setup_root()
2162 		link_css_set(&tmp_links, cset, root_cgrp);  in cgroup_setup_root()
2163 		if (css_set_populated(cset))  in cgroup_setup_root()
2464 	struct css_set *cset;  in cgroup_migrate_add_task()  local
2475 	cset = task_css_set(task);  in cgroup_migrate_add_task()
2476 	if (!cset->mg_src_cgrp)  in cgroup_migrate_add_task()
2481 	list_move_tail(&task->cg_list, &cset->mg_tasks);  in cgroup_migrate_add_task()
2482 	if (list_empty(&cset->mg_node))  in cgroup_migrate_add_task()
2483 		list_add_tail(&cset->mg_node,  in cgroup_migrate_add_task()
2485 	if (list_empty(&cset->mg_dst_cset->mg_node))  in cgroup_migrate_add_task()
2486 		list_add_tail(&cset->mg_dst_cset->mg_node,  in cgroup_migrate_add_task()
2517 	struct css_set *cset = tset->cur_cset;  in cgroup_taskset_next()  local
2520 	while (CGROUP_HAS_SUBSYS_CONFIG && &cset->mg_node != tset->csets) {  in cgroup_taskset_next()
2522 			task = list_first_entry(&cset->mg_tasks,  in cgroup_taskset_next()
2527 		if (&task->cg_list != &cset->mg_tasks) {  in cgroup_taskset_next()
2528 			tset->cur_cset = cset;  in cgroup_taskset_next()
2537 			if (cset->mg_dst_cset)  in cgroup_taskset_next()
2538 				*dst_cssp = cset->mg_dst_cset->subsys[tset->ssid];  in cgroup_taskset_next()
2540 				*dst_cssp = cset->subsys[tset->ssid];  in cgroup_taskset_next()
2545 		cset = list_next_entry(cset, mg_node);  in cgroup_taskset_next()
2566 	struct css_set *cset, *tmp_cset;  in cgroup_migrate_execute()  local
2589 	list_for_each_entry(cset, &tset->src_csets, mg_node) {  in cgroup_migrate_execute()
2590 		list_for_each_entry_safe(task, tmp_task, &cset->mg_tasks, cg_list) {  in cgroup_migrate_execute()
2592 			struct css_set *to_cset = cset->mg_dst_cset;  in cgroup_migrate_execute()
2643 	list_for_each_entry_safe(cset, tmp_cset, &tset->src_csets, mg_node) {  in cgroup_migrate_execute()
2644 		list_splice_tail_init(&cset->mg_tasks, &cset->tasks);  in cgroup_migrate_execute()
2645 		list_del_init(&cset->mg_node);  in cgroup_migrate_execute()
2701 	struct css_set *cset, *tmp_cset;  in cgroup_migrate_finish()  local
2707 	list_for_each_entry_safe(cset, tmp_cset, &mgctx->preloaded_src_csets,  in cgroup_migrate_finish()
2709 		cset->mg_src_cgrp = NULL;  in cgroup_migrate_finish()
2710 		cset->mg_dst_cgrp = NULL;  in cgroup_migrate_finish()
2711 		cset->mg_dst_cset = NULL;  in cgroup_migrate_finish()
2712 		list_del_init(&cset->mg_src_preload_node);  in cgroup_migrate_finish()
2713 		put_css_set_locked(cset);  in cgroup_migrate_finish()
2716 	list_for_each_entry_safe(cset, tmp_cset, &mgctx->preloaded_dst_csets,  in cgroup_migrate_finish()
2718 		cset->mg_src_cgrp = NULL;  in cgroup_migrate_finish()
2719 		cset->mg_dst_cgrp = NULL;  in cgroup_migrate_finish()
2720 		cset->mg_dst_cset = NULL;  in cgroup_migrate_finish()
2721 		list_del_init(&cset->mg_dst_preload_node);  in cgroup_migrate_finish()
2722 		put_css_set_locked(cset);  in cgroup_migrate_finish()
3061 			cgroup_migrate_add_src(link->cset, dsct, &mgctx);  in cgroup_update_dfl_csses()
4793 	struct css_set *cset;  in css_task_iter_next_css_set()  local
4819 		cset = container_of(l, struct css_set, e_cset_node[it->ss->id]);  in css_task_iter_next_css_set()
4822 		cset = link->cset;  in css_task_iter_next_css_set()
4831 		it->cur_dcset = cset;  in css_task_iter_next_css_set()
4832 		get_css_set(cset);  in css_task_iter_next_css_set()
4834 		it->tcset_head = &cset->threaded_csets;  in css_task_iter_next_css_set()
4835 		it->tcset_pos = &cset->threaded_csets;  in css_task_iter_next_css_set()
4838 	return cset;  in css_task_iter_next_css_set()
4849 	struct css_set *cset;  in css_task_iter_advance_css_set()  local
4854 	while ((cset = css_task_iter_next_css_set(it))) {  in css_task_iter_advance_css_set()
4855 		if (!list_empty(&cset->tasks)) {  in css_task_iter_advance_css_set()
4856 			it->cur_tasks_head = &cset->tasks;  in css_task_iter_advance_css_set()
4858 		} else if (!list_empty(&cset->mg_tasks)) {  in css_task_iter_advance_css_set()
4859 			it->cur_tasks_head = &cset->mg_tasks;  in css_task_iter_advance_css_set()
4861 		} else if (!list_empty(&cset->dying_tasks)) {  in css_task_iter_advance_css_set()
4862 			it->cur_tasks_head = &cset->dying_tasks;  in css_task_iter_advance_css_set()
4866 	if (!cset) {  in css_task_iter_advance_css_set()
4891 	get_css_set(cset);  in css_task_iter_advance_css_set()
4892 	it->cur_cset = cset;  in css_task_iter_advance_css_set()
4893 	list_add(&it->iters_node, &cset->task_iters);  in css_task_iter_advance_css_set()
6000 		link->cset->dead = true;  in cgroup_destroy_locked()
6477 	struct css_set *cset;  in cgroup_css_set_fork()  local
6487 	cset = task_css_set(current);  in cgroup_css_set_fork()
6488 	get_css_set(cset);  in cgroup_css_set_fork()
6492 		kargs->cset = cset;  in cgroup_css_set_fork()
6538 	ret = cgroup_attach_permissions(cset->dfl_cgrp, dst_cgrp, sb,  in cgroup_css_set_fork()
6544 	kargs->cset = find_css_set(cset, dst_cgrp);  in cgroup_css_set_fork()
6545 	if (!kargs->cset) {  in cgroup_css_set_fork()
6550 	put_css_set(cset);  in cgroup_css_set_fork()
6562 	put_css_set(cset);  in cgroup_css_set_fork()
6563 	if (kargs->cset)  in cgroup_css_set_fork()
6564 		put_css_set(kargs->cset);  in cgroup_css_set_fork()
6579 	struct css_set *cset = kargs->cset;  in cgroup_css_set_put_fork()  local
6583 	if (cset) {  in cgroup_css_set_put_fork()
6584 		put_css_set(cset);  in cgroup_css_set_put_fork()
6585 		kargs->cset = NULL;  in cgroup_css_set_put_fork()
6618 		ret = ss->can_fork(child, kargs->cset);  in cgroup_can_fork()
6630 			ss->cancel_fork(child, kargs->cset);  in cgroup_can_fork()
6655 			ss->cancel_fork(child, kargs->cset);  in cgroup_cancel_fork()
6675 	struct css_set *cset;  in cgroup_post_fork()  local
6678 	cset = kargs->cset;  in cgroup_post_fork()
6679 	kargs->cset = NULL;  in cgroup_post_fork()
6688 			cgrp_flags = cset->dfl_cgrp->flags;  in cgroup_post_fork()
6691 		cset->nr_tasks++;  in cgroup_post_fork()
6692 		css_set_move_task(child, NULL, cset, false);  in cgroup_post_fork()
6694 		put_css_set(cset);  in cgroup_post_fork()
6695 		cset = NULL;  in cgroup_post_fork()
6741 		get_css_set(cset);  in cgroup_post_fork()
6742 		child->nsproxy->cgroup_ns->root_cset = cset;  in cgroup_post_fork()
6763 	struct css_set *cset;  in cgroup_exit()  local
6769 	cset = task_css_set(tsk);  in cgroup_exit()
6770 	css_set_move_task(tsk, cset, NULL, false);  in cgroup_exit()
6771 	cset->nr_tasks--;  in cgroup_exit()
6774 		list_add_tail(&tsk->cg_list, &cset->dying_tasks);  in cgroup_exit()
6811 	struct css_set *cset = task_css_set(task);  in cgroup_free()  local
6812 	put_css_set(cset);  in cgroup_free()
7060 		struct css_set *cset;  in cgroup_sk_alloc()  local
7062 		cset = task_css_set(current);  in cgroup_sk_alloc()
7063 		if (likely(cgroup_tryget(cset->dfl_cgrp))) {  in cgroup_sk_alloc()
7064 			cgroup = cset->dfl_cgrp;  in cgroup_sk_alloc()