Lines Matching refs:cgrp
167 struct cgroup_root cgrp_dfl_root = { .cgrp.rstat_cpu = &cgrp_dfl_root_rstat_cpu };
241 static int cgroup_apply_control(struct cgroup *cgrp);
242 static void cgroup_finalize_control(struct cgroup *cgrp, int ret);
245 static int cgroup_destroy_locked(struct cgroup *cgrp);
246 static struct cgroup_subsys_state *css_create(struct cgroup *cgrp,
251 struct cgroup *cgrp, struct cftype cfts[],
319 bool cgroup_on_dfl(const struct cgroup *cgrp) in cgroup_on_dfl() argument
321 return cgrp->root == &cgrp_dfl_root; in cgroup_on_dfl()
355 static bool cgroup_has_tasks(struct cgroup *cgrp) in cgroup_has_tasks() argument
357 return cgrp->nr_populated_csets; in cgroup_has_tasks()
360 static bool cgroup_is_threaded(struct cgroup *cgrp) in cgroup_is_threaded() argument
362 return cgrp->dom_cgrp != cgrp; in cgroup_is_threaded()
366 static bool cgroup_is_mixable(struct cgroup *cgrp) in cgroup_is_mixable() argument
373 return !cgroup_parent(cgrp); in cgroup_is_mixable()
377 static bool cgroup_can_be_thread_root(struct cgroup *cgrp) in cgroup_can_be_thread_root() argument
380 if (cgroup_is_mixable(cgrp)) in cgroup_can_be_thread_root()
384 if (cgroup_is_threaded(cgrp)) in cgroup_can_be_thread_root()
388 if (cgrp->nr_populated_domain_children) in cgroup_can_be_thread_root()
392 if (cgrp->subtree_control & ~cgrp_dfl_threaded_ss_mask) in cgroup_can_be_thread_root()
399 static bool cgroup_is_thread_root(struct cgroup *cgrp) in cgroup_is_thread_root() argument
402 if (cgroup_is_threaded(cgrp)) in cgroup_is_thread_root()
406 if (cgrp->nr_threaded_children) in cgroup_is_thread_root()
413 if (cgroup_has_tasks(cgrp) && in cgroup_is_thread_root()
414 (cgrp->subtree_control & cgrp_dfl_threaded_ss_mask)) in cgroup_is_thread_root()
421 static bool cgroup_is_valid_domain(struct cgroup *cgrp) in cgroup_is_valid_domain() argument
424 if (cgroup_is_threaded(cgrp)) in cgroup_is_valid_domain()
428 while ((cgrp = cgroup_parent(cgrp))) { in cgroup_is_valid_domain()
429 if (!cgroup_is_mixable(cgrp) && cgroup_is_thread_root(cgrp)) in cgroup_is_valid_domain()
431 if (cgroup_is_threaded(cgrp)) in cgroup_is_valid_domain()
439 static u16 cgroup_control(struct cgroup *cgrp) in cgroup_control() argument
441 struct cgroup *parent = cgroup_parent(cgrp); in cgroup_control()
442 u16 root_ss_mask = cgrp->root->subsys_mask; in cgroup_control()
448 if (cgroup_is_threaded(cgrp)) in cgroup_control()
453 if (cgroup_on_dfl(cgrp)) in cgroup_control()
460 static u16 cgroup_ss_mask(struct cgroup *cgrp) in cgroup_ss_mask() argument
462 struct cgroup *parent = cgroup_parent(cgrp); in cgroup_ss_mask()
468 if (cgroup_is_threaded(cgrp)) in cgroup_ss_mask()
473 return cgrp->root->subsys_mask; in cgroup_ss_mask()
487 static struct cgroup_subsys_state *cgroup_css(struct cgroup *cgrp, in cgroup_css() argument
491 return rcu_dereference_check(cgrp->subsys[ss->id], in cgroup_css()
494 return &cgrp->self; in cgroup_css()
507 static struct cgroup_subsys_state *cgroup_e_css_by_mask(struct cgroup *cgrp, in cgroup_e_css_by_mask() argument
513 return &cgrp->self; in cgroup_e_css_by_mask()
519 while (!(cgroup_ss_mask(cgrp) & (1 << ss->id))) { in cgroup_e_css_by_mask()
520 cgrp = cgroup_parent(cgrp); in cgroup_e_css_by_mask()
521 if (!cgrp) in cgroup_e_css_by_mask()
525 return cgroup_css(cgrp, ss); in cgroup_e_css_by_mask()
541 struct cgroup_subsys_state *cgroup_e_css(struct cgroup *cgrp, in cgroup_e_css() argument
550 css = cgroup_css(cgrp, ss); in cgroup_e_css()
554 cgrp = cgroup_parent(cgrp); in cgroup_e_css()
555 } while (cgrp); in cgroup_e_css()
571 struct cgroup_subsys_state *cgroup_get_e_css(struct cgroup *cgrp, in cgroup_get_e_css() argument
582 css = cgroup_css(cgrp, ss); in cgroup_get_e_css()
586 cgrp = cgroup_parent(cgrp); in cgroup_get_e_css()
587 } while (cgrp); in cgroup_get_e_css()
597 static void cgroup_get_live(struct cgroup *cgrp) in cgroup_get_live() argument
599 WARN_ON_ONCE(cgroup_is_dead(cgrp)); in cgroup_get_live()
600 cgroup_get(cgrp); in cgroup_get_live()
608 int __cgroup_task_count(const struct cgroup *cgrp) in __cgroup_task_count() argument
615 list_for_each_entry(link, &cgrp->cset_links, cset_link) in __cgroup_task_count()
625 int cgroup_task_count(const struct cgroup *cgrp) in cgroup_task_count() argument
630 count = __cgroup_task_count(cgrp); in cgroup_task_count()
638 struct cgroup *cgrp = of->kn->parent->priv; in of_css() local
650 return rcu_dereference_raw(cgrp->subsys[cft->ss->id]); in of_css()
652 return &cgrp->self; in of_css()
664 #define for_each_css(css, ssid, cgrp) \ argument
667 (cgrp)->subsys[(ssid)], \
696 #define cgroup_for_each_live_child(child, cgrp) \ argument
697 list_for_each_entry((child), &(cgrp)->self.children, self.sibling) \
704 #define cgroup_for_each_live_descendant_pre(dsct, d_css, cgrp) \ argument
705 css_for_each_descendant_pre((d_css), cgroup_css((cgrp), NULL)) \
713 #define cgroup_for_each_live_descendant_post(dsct, d_css, cgrp) \ argument
714 css_for_each_descendant_post((d_css), cgroup_css((cgrp), NULL)) \
747 .dfl_cgrp = &cgrp_dfl_root.cgrp,
790 static void cgroup_update_populated(struct cgroup *cgrp, bool populated) in cgroup_update_populated() argument
798 bool was_populated = cgroup_is_populated(cgrp); in cgroup_update_populated()
801 cgrp->nr_populated_csets += adj; in cgroup_update_populated()
804 cgrp->nr_populated_threaded_children += adj; in cgroup_update_populated()
806 cgrp->nr_populated_domain_children += adj; in cgroup_update_populated()
809 if (was_populated == cgroup_is_populated(cgrp)) in cgroup_update_populated()
812 cgroup1_check_for_release(cgrp); in cgroup_update_populated()
813 TRACE_CGROUP_PATH(notify_populated, cgrp, in cgroup_update_populated()
814 cgroup_is_populated(cgrp)); in cgroup_update_populated()
815 cgroup_file_notify(&cgrp->events_file); in cgroup_update_populated()
817 child = cgrp; in cgroup_update_populated()
818 cgrp = cgroup_parent(cgrp); in cgroup_update_populated()
819 } while (cgrp); in cgroup_update_populated()
837 cgroup_update_populated(link->cgrp, populated); in css_set_update_populated()
949 if (cgroup_parent(link->cgrp)) in put_css_set_locked()
950 cgroup_put(link->cgrp); in put_css_set_locked()
1022 cgrp1 = link1->cgrp; in compare_css_sets()
1023 cgrp2 = link2->cgrp; in compare_css_sets()
1052 struct cgroup *cgrp, in find_existing_css_set() argument
1055 struct cgroup_root *root = cgrp->root; in find_existing_css_set()
1072 template[i] = cgroup_e_css_by_mask(cgrp, ss); in find_existing_css_set()
1084 if (!compare_css_sets(cset, old_cset, cgrp, template)) in find_existing_css_set()
1138 struct cgroup *cgrp) in link_css_set() argument
1144 if (cgroup_on_dfl(cgrp)) in link_css_set()
1145 cset->dfl_cgrp = cgrp; in link_css_set()
1149 link->cgrp = cgrp; in link_css_set()
1155 list_move_tail(&link->cset_link, &cgrp->cset_links); in link_css_set()
1158 if (cgroup_parent(cgrp)) in link_css_set()
1159 cgroup_get_live(cgrp); in link_css_set()
1171 struct cgroup *cgrp) in find_css_set() argument
1186 cset = find_existing_css_set(old_cset, cgrp, template); in find_css_set()
1224 struct cgroup *c = link->cgrp; in find_css_set()
1226 if (c->root == cgrp->root) in find_css_set()
1227 c = cgrp; in find_css_set()
1323 struct cgroup *cgrp = &root->cgrp; in cgroup_destroy_root() local
1328 cgroup_lock_and_drain_offline(&cgrp_dfl_root.cgrp); in cgroup_destroy_root()
1331 BUG_ON(!list_empty(&cgrp->self.children)); in cgroup_destroy_root()
1342 list_for_each_entry_safe(link, tmp_link, &cgrp->cset_links, cset_link) { in cgroup_destroy_root()
1361 cgroup_rstat_exit(cgrp); in cgroup_destroy_root()
1375 res_cgroup = &root->cgrp; in __cset_cgroup_from_root()
1383 struct cgroup *c = link->cgrp; in __cset_cgroup_from_root()
1456 return &cgrp_dfl_root.cgrp; in current_cgns_cgroup_dfl()
1513 static char *cgroup_file_name(struct cgroup *cgrp, const struct cftype *cft, in cgroup_file_name() argument
1519 !(cgrp->root->flags & CGRP_ROOT_NOPREFIX)) { in cgroup_file_name()
1523 dbg, cgroup_on_dfl(cgrp) ? ss->name : ss->legacy_name, in cgroup_file_name()
1610 struct cgroup *cgrp; in cgroup_kn_unlock() local
1613 cgrp = kn->priv; in cgroup_kn_unlock()
1615 cgrp = kn->parent->priv; in cgroup_kn_unlock()
1620 cgroup_put(cgrp); in cgroup_kn_unlock()
1642 struct cgroup *cgrp; in cgroup_kn_lock_live() local
1645 cgrp = kn->priv; in cgroup_kn_lock_live()
1647 cgrp = kn->parent->priv; in cgroup_kn_lock_live()
1655 if (!cgroup_tryget(cgrp)) in cgroup_kn_lock_live()
1660 cgroup_lock_and_drain_offline(cgrp); in cgroup_kn_lock_live()
1664 if (!cgroup_is_dead(cgrp)) in cgroup_kn_lock_live()
1665 return cgrp; in cgroup_kn_lock_live()
1671 static void cgroup_rm_file(struct cgroup *cgrp, const struct cftype *cft) in cgroup_rm_file() argument
1678 struct cgroup_subsys_state *css = cgroup_css(cgrp, cft->ss); in cgroup_rm_file()
1688 kernfs_remove_by_name(cgrp->kn, cgroup_file_name(cgrp, cft, name)); in cgroup_rm_file()
1697 struct cgroup *cgrp = css->cgroup; in css_clear_dir() local
1706 if (cgroup_on_dfl(cgrp)) { in css_clear_dir()
1707 cgroup_addrm_files(css, cgrp, in css_clear_dir()
1710 cgroup_addrm_files(css, cgrp, in css_clear_dir()
1713 cgroup_addrm_files(css, cgrp, in css_clear_dir()
1718 cgroup_addrm_files(css, cgrp, cfts, false); in css_clear_dir()
1730 struct cgroup *cgrp = css->cgroup; in css_populate_dir() local
1738 if (cgroup_on_dfl(cgrp)) { in css_populate_dir()
1739 ret = cgroup_addrm_files(css, cgrp, in css_populate_dir()
1745 ret = cgroup_addrm_files(css, cgrp, in css_populate_dir()
1748 cgroup_addrm_files(css, cgrp, in css_populate_dir()
1754 ret = cgroup_addrm_files(css, cgrp, in css_populate_dir()
1761 ret = cgroup_addrm_files(css, cgrp, cfts, true); in css_populate_dir()
1776 cgroup_addrm_files(css, cgrp, cfts, false); in css_populate_dir()
1783 struct cgroup *dcgrp = &dst_root->cgrp; in rebind_subsystems()
1796 if (css_next_child(NULL, cgroup_css(&ss->root->cgrp, ss)) && in rebind_subsystems()
1814 struct cgroup *scgrp = &cgrp_dfl_root.cgrp; in rebind_subsystems()
1827 struct cgroup *scgrp = &src_root->cgrp; in rebind_subsystems()
2040 static void init_cgroup_housekeeping(struct cgroup *cgrp) in init_cgroup_housekeeping() argument
2045 INIT_LIST_HEAD(&cgrp->self.sibling); in init_cgroup_housekeeping()
2046 INIT_LIST_HEAD(&cgrp->self.children); in init_cgroup_housekeeping()
2047 INIT_LIST_HEAD(&cgrp->cset_links); in init_cgroup_housekeeping()
2048 INIT_LIST_HEAD(&cgrp->pidlists); in init_cgroup_housekeeping()
2049 mutex_init(&cgrp->pidlist_mutex); in init_cgroup_housekeeping()
2050 cgrp->self.cgroup = cgrp; in init_cgroup_housekeeping()
2051 cgrp->self.flags |= CSS_ONLINE; in init_cgroup_housekeeping()
2052 cgrp->dom_cgrp = cgrp; in init_cgroup_housekeeping()
2053 cgrp->max_descendants = INT_MAX; in init_cgroup_housekeeping()
2054 cgrp->max_depth = INT_MAX; in init_cgroup_housekeeping()
2055 INIT_LIST_HEAD(&cgrp->rstat_css_list); in init_cgroup_housekeeping()
2056 prev_cputime_init(&cgrp->prev_cputime); in init_cgroup_housekeeping()
2059 INIT_LIST_HEAD(&cgrp->e_csets[ssid]); in init_cgroup_housekeeping()
2061 init_waitqueue_head(&cgrp->offline_waitq); in init_cgroup_housekeeping()
2062 INIT_WORK(&cgrp->release_agent_work, cgroup1_release_agent); in init_cgroup_housekeeping()
2068 struct cgroup *cgrp = &root->cgrp; in init_cgroup_root() local
2072 cgrp->root = root; in init_cgroup_root()
2073 init_cgroup_housekeeping(cgrp); in init_cgroup_root()
2082 set_bit(CGRP_CPUSET_CLONE_CHILDREN, &root->cgrp.flags); in init_cgroup_root()
2088 struct cgroup *root_cgrp = &root->cgrp; in cgroup_setup_root()
2207 struct cgroup *cgrp; in cgroup_do_get_tree() local
2212 cgrp = cset_cgroup_from_root(ctx->ns->root_cset, ctx->root); in cgroup_do_get_tree()
2217 nsdentry = kernfs_node_dentry(cgrp->kn, sb); in cgroup_do_get_tree()
2228 cgroup_put(&ctx->root->cgrp); in cgroup_do_get_tree()
2253 cgroup_get_live(&cgrp_dfl_root.cgrp); in cgroup_get_tree()
2316 if (list_empty(&root->cgrp.self.children) && root != &cgrp_dfl_root && in cgroup_kill_sb()
2317 !percpu_ref_is_dying(&root->cgrp.self.refcnt)) { in cgroup_kill_sb()
2318 cgroup_bpf_offline(&root->cgrp); in cgroup_kill_sb()
2319 percpu_ref_kill(&root->cgrp.self.refcnt); in cgroup_kill_sb()
2321 cgroup_put(&root->cgrp); in cgroup_kill_sb()
2385 int cgroup_path_ns_locked(struct cgroup *cgrp, char *buf, size_t buflen, in cgroup_path_ns_locked() argument
2388 struct cgroup *root = cset_cgroup_from_root(ns->root_cset, cgrp->root); in cgroup_path_ns_locked()
2390 return kernfs_path_from_node(cgrp->kn, root->kn, buf, buflen); in cgroup_path_ns_locked()
2393 int cgroup_path_ns(struct cgroup *cgrp, char *buf, size_t buflen, in cgroup_path_ns() argument
2401 ret = cgroup_path_ns_locked(cgrp, buf, buflen, ns); in cgroup_path_ns()
3011 struct cgroup *cgrp = seq_css(seq)->cgroup; in cgroup_controllers_show() local
3013 cgroup_print_ss_mask(seq, cgroup_control(cgrp)); in cgroup_controllers_show()
3020 struct cgroup *cgrp = seq_css(seq)->cgroup; in cgroup_subtree_control_show() local
3022 cgroup_print_ss_mask(seq, cgrp->subtree_control); in cgroup_subtree_control_show()
3035 static int cgroup_update_dfl_csses(struct cgroup *cgrp) in cgroup_update_dfl_csses() argument
3048 cgroup_for_each_live_descendant_pre(dsct, d_css, cgrp) { in cgroup_update_dfl_csses()
3057 if (dsct == cgrp) in cgroup_update_dfl_csses()
3105 void cgroup_lock_and_drain_offline(struct cgroup *cgrp) in cgroup_lock_and_drain_offline() argument
3116 cgroup_for_each_live_descendant_post(dsct, d_css, cgrp) { in cgroup_lock_and_drain_offline()
3146 static void cgroup_save_control(struct cgroup *cgrp) in cgroup_save_control() argument
3151 cgroup_for_each_live_descendant_pre(dsct, d_css, cgrp) { in cgroup_save_control()
3166 static void cgroup_propagate_control(struct cgroup *cgrp) in cgroup_propagate_control() argument
3171 cgroup_for_each_live_descendant_pre(dsct, d_css, cgrp) { in cgroup_propagate_control()
3187 static void cgroup_restore_control(struct cgroup *cgrp) in cgroup_restore_control() argument
3192 cgroup_for_each_live_descendant_post(dsct, d_css, cgrp) { in cgroup_restore_control()
3202 struct cgroup *cgrp = css->cgroup; in css_visible() local
3204 if (cgroup_control(cgrp) & (1 << ss->id)) in css_visible()
3206 if (!(cgroup_ss_mask(cgrp) & (1 << ss->id))) in css_visible()
3208 return cgroup_on_dfl(cgrp) && ss->implicit_on_dfl; in css_visible()
3224 static int cgroup_apply_control_enable(struct cgroup *cgrp) in cgroup_apply_control_enable() argument
3231 cgroup_for_each_live_descendant_pre(dsct, d_css, cgrp) { in cgroup_apply_control_enable()
3270 static void cgroup_apply_control_disable(struct cgroup *cgrp) in cgroup_apply_control_disable() argument
3277 cgroup_for_each_live_descendant_post(dsct, d_css, cgrp) { in cgroup_apply_control_disable()
3315 static int cgroup_apply_control(struct cgroup *cgrp) in cgroup_apply_control() argument
3319 cgroup_propagate_control(cgrp); in cgroup_apply_control()
3321 ret = cgroup_apply_control_enable(cgrp); in cgroup_apply_control()
3330 return cgroup_update_dfl_csses(cgrp); in cgroup_apply_control()
3340 static void cgroup_finalize_control(struct cgroup *cgrp, int ret) in cgroup_finalize_control() argument
3343 cgroup_restore_control(cgrp); in cgroup_finalize_control()
3344 cgroup_propagate_control(cgrp); in cgroup_finalize_control()
3347 cgroup_apply_control_disable(cgrp); in cgroup_finalize_control()
3350 static int cgroup_vet_subtree_control_enable(struct cgroup *cgrp, u16 enable) in cgroup_vet_subtree_control_enable() argument
3359 if (!cgroup_is_valid_domain(cgrp->dom_cgrp)) in cgroup_vet_subtree_control_enable()
3363 if (cgroup_is_mixable(cgrp)) in cgroup_vet_subtree_control_enable()
3368 if (cgroup_is_thread_root(cgrp) || cgroup_is_threaded(cgrp)) in cgroup_vet_subtree_control_enable()
3376 if (cgroup_can_be_thread_root(cgrp) || cgroup_is_threaded(cgrp)) in cgroup_vet_subtree_control_enable()
3384 if (cgroup_has_tasks(cgrp)) in cgroup_vet_subtree_control_enable()
3396 struct cgroup *cgrp, *child; in cgroup_subtree_control_write() local
3429 cgrp = cgroup_kn_lock_live(of->kn, true); in cgroup_subtree_control_write()
3430 if (!cgrp) in cgroup_subtree_control_write()
3435 if (cgrp->subtree_control & (1 << ssid)) { in cgroup_subtree_control_write()
3440 if (!(cgroup_control(cgrp) & (1 << ssid))) { in cgroup_subtree_control_write()
3445 if (!(cgrp->subtree_control & (1 << ssid))) { in cgroup_subtree_control_write()
3451 cgroup_for_each_live_child(child, cgrp) { in cgroup_subtree_control_write()
3465 ret = cgroup_vet_subtree_control_enable(cgrp, enable); in cgroup_subtree_control_write()
3470 cgroup_save_control(cgrp); in cgroup_subtree_control_write()
3472 cgrp->subtree_control |= enable; in cgroup_subtree_control_write()
3473 cgrp->subtree_control &= ~disable; in cgroup_subtree_control_write()
3475 ret = cgroup_apply_control(cgrp); in cgroup_subtree_control_write()
3476 cgroup_finalize_control(cgrp, ret); in cgroup_subtree_control_write()
3480 kernfs_activate(cgrp->kn); in cgroup_subtree_control_write()
3495 static int cgroup_enable_threaded(struct cgroup *cgrp) in cgroup_enable_threaded() argument
3497 struct cgroup *parent = cgroup_parent(cgrp); in cgroup_enable_threaded()
3506 if (cgroup_is_threaded(cgrp)) in cgroup_enable_threaded()
3515 if (cgroup_is_populated(cgrp) || in cgroup_enable_threaded()
3516 cgrp->subtree_control & ~cgrp_dfl_threaded_ss_mask) in cgroup_enable_threaded()
3528 cgroup_save_control(cgrp); in cgroup_enable_threaded()
3530 cgroup_for_each_live_descendant_pre(dsct, d_css, cgrp) in cgroup_enable_threaded()
3531 if (dsct == cgrp || cgroup_is_threaded(dsct)) in cgroup_enable_threaded()
3534 ret = cgroup_apply_control(cgrp); in cgroup_enable_threaded()
3538 cgroup_finalize_control(cgrp, ret); in cgroup_enable_threaded()
3544 struct cgroup *cgrp = seq_css(seq)->cgroup; in cgroup_type_show() local
3546 if (cgroup_is_threaded(cgrp)) in cgroup_type_show()
3548 else if (!cgroup_is_valid_domain(cgrp)) in cgroup_type_show()
3550 else if (cgroup_is_thread_root(cgrp)) in cgroup_type_show()
3561 struct cgroup *cgrp; in cgroup_type_write() local
3569 cgrp = cgroup_kn_lock_live(of->kn, true); in cgroup_type_write()
3570 if (!cgrp) in cgroup_type_write()
3574 ret = cgroup_enable_threaded(cgrp); in cgroup_type_write()
3582 struct cgroup *cgrp = seq_css(seq)->cgroup; in cgroup_max_descendants_show() local
3583 int descendants = READ_ONCE(cgrp->max_descendants); in cgroup_max_descendants_show()
3596 struct cgroup *cgrp; in cgroup_max_descendants_write() local
3612 cgrp = cgroup_kn_lock_live(of->kn, false); in cgroup_max_descendants_write()
3613 if (!cgrp) in cgroup_max_descendants_write()
3616 cgrp->max_descendants = descendants; in cgroup_max_descendants_write()
3625 struct cgroup *cgrp = seq_css(seq)->cgroup; in cgroup_max_depth_show() local
3626 int depth = READ_ONCE(cgrp->max_depth); in cgroup_max_depth_show()
3639 struct cgroup *cgrp; in cgroup_max_depth_write() local
3655 cgrp = cgroup_kn_lock_live(of->kn, false); in cgroup_max_depth_write()
3656 if (!cgrp) in cgroup_max_depth_write()
3659 cgrp->max_depth = depth; in cgroup_max_depth_write()
3668 struct cgroup *cgrp = seq_css(seq)->cgroup; in cgroup_events_show() local
3670 seq_printf(seq, "populated %d\n", cgroup_is_populated(cgrp)); in cgroup_events_show()
3671 seq_printf(seq, "frozen %d\n", test_bit(CGRP_FROZEN, &cgrp->flags)); in cgroup_events_show()
3725 static struct cgroup_subsys_state *cgroup_tryget_css(struct cgroup *cgrp, in cgroup_tryget_css() argument
3731 css = cgroup_css(cgrp, ss); in cgroup_tryget_css()
3741 struct cgroup *cgrp = seq_css(seq)->cgroup; in cgroup_extra_stat_show() local
3749 css = cgroup_tryget_css(cgrp, ss); in cgroup_extra_stat_show()
3759 struct cgroup *cgrp, int ssid) in cgroup_local_stat_show() argument
3768 css = cgroup_tryget_css(cgrp, ss); in cgroup_local_stat_show()
3791 struct cgroup __maybe_unused *cgrp = seq_css(seq)->cgroup; in cpu_local_stat_show() local
3795 ret = cgroup_local_stat_show(seq, cgrp, cpu_cgrp_id); in cpu_local_stat_show()
3803 struct cgroup *cgrp = seq_css(seq)->cgroup; in cgroup_io_pressure_show() local
3804 struct psi_group *psi = cgroup_psi(cgrp); in cgroup_io_pressure_show()
3810 struct cgroup *cgrp = seq_css(seq)->cgroup; in cgroup_memory_pressure_show() local
3811 struct psi_group *psi = cgroup_psi(cgrp); in cgroup_memory_pressure_show()
3817 struct cgroup *cgrp = seq_css(seq)->cgroup; in cgroup_cpu_pressure_show() local
3818 struct psi_group *psi = cgroup_psi(cgrp); in cgroup_cpu_pressure_show()
3828 struct cgroup *cgrp; in pressure_write() local
3831 cgrp = cgroup_kn_lock_live(of->kn, false); in pressure_write()
3832 if (!cgrp) in pressure_write()
3835 cgroup_get(cgrp); in pressure_write()
3840 cgroup_put(cgrp); in pressure_write()
3844 psi = cgroup_psi(cgrp); in pressure_write()
3847 cgroup_put(cgrp); in pressure_write()
3852 cgroup_put(cgrp); in pressure_write()
3881 struct cgroup *cgrp = seq_css(seq)->cgroup; in cgroup_irq_pressure_show() local
3882 struct psi_group *psi = cgroup_psi(cgrp); in cgroup_irq_pressure_show()
3897 struct cgroup *cgrp = seq_css(seq)->cgroup; in cgroup_pressure_show() local
3898 struct psi_group *psi = cgroup_psi(cgrp); in cgroup_pressure_show()
3911 struct cgroup *cgrp; in cgroup_pressure_write() local
3921 cgrp = cgroup_kn_lock_live(of->kn, false); in cgroup_pressure_write()
3922 if (!cgrp) in cgroup_pressure_write()
3925 psi = cgroup_psi(cgrp); in cgroup_pressure_write()
3931 cgroup_file_show(&cgrp->psi_files[i], enable); in cgroup_pressure_write()
3976 struct cgroup *cgrp = seq_css(seq)->cgroup; in cgroup_freeze_show() local
3978 seq_printf(seq, "%d\n", cgrp->freezer.freeze); in cgroup_freeze_show()
3986 struct cgroup *cgrp; in cgroup_freeze_write() local
3997 cgrp = cgroup_kn_lock_live(of->kn, false); in cgroup_freeze_write()
3998 if (!cgrp) in cgroup_freeze_write()
4001 cgroup_freeze(cgrp, freeze); in cgroup_freeze_write()
4008 static void __cgroup_kill(struct cgroup *cgrp) in __cgroup_kill() argument
4016 set_bit(CGRP_KILL, &cgrp->flags); in __cgroup_kill()
4019 css_task_iter_start(&cgrp->self, CSS_TASK_ITER_PROCS | CSS_TASK_ITER_THREADED, &it); in __cgroup_kill()
4034 clear_bit(CGRP_KILL, &cgrp->flags); in __cgroup_kill()
4038 static void cgroup_kill(struct cgroup *cgrp) in cgroup_kill() argument
4045 cgroup_for_each_live_descendant_pre(dsct, css, cgrp) in cgroup_kill()
4054 struct cgroup *cgrp; in cgroup_kill_write() local
4063 cgrp = cgroup_kn_lock_live(of->kn, false); in cgroup_kill_write()
4064 if (!cgrp) in cgroup_kill_write()
4072 if (cgroup_is_threaded(cgrp)) in cgroup_kill_write()
4075 cgroup_kill(cgrp); in cgroup_kill_write()
4122 struct cgroup *cgrp = of->kn->parent->priv; in cgroup_file_write() local
4136 if ((cgrp->root->flags & CGRP_ROOT_NS_DELEGATE) && in cgroup_file_write()
4138 ctx->ns != &init_cgroup_ns && ctx->ns->root_cset->dfl_cgrp == cgrp) in cgroup_file_write()
4151 css = cgroup_css(cgrp, cft->ss); in cgroup_file_write()
4241 static int cgroup_add_file(struct cgroup_subsys_state *css, struct cgroup *cgrp, in cgroup_add_file() argument
4251 kn = __kernfs_create_file(cgrp->kn, cgroup_file_name(cgrp, cft, name), in cgroup_add_file()
4283 struct cgroup *cgrp, struct cftype cfts[], in cgroup_addrm_files() argument
4294 if ((cft->flags & __CFTYPE_ONLY_ON_DFL) && !cgroup_on_dfl(cgrp)) in cgroup_addrm_files()
4296 if ((cft->flags & __CFTYPE_NOT_ON_DFL) && cgroup_on_dfl(cgrp)) in cgroup_addrm_files()
4298 if ((cft->flags & CFTYPE_NOT_ON_ROOT) && !cgroup_parent(cgrp)) in cgroup_addrm_files()
4300 if ((cft->flags & CFTYPE_ONLY_ON_ROOT) && cgroup_parent(cgrp)) in cgroup_addrm_files()
4305 ret = cgroup_add_file(css, cgrp, cft); in cgroup_addrm_files()
4314 cgroup_rm_file(cgrp, cft); in cgroup_addrm_files()
4323 struct cgroup *root = &ss->root->cgrp; in cgroup_apply_cftypes()
4331 struct cgroup *cgrp = css->cgroup; in cgroup_apply_cftypes() local
4336 ret = cgroup_addrm_files(css, cgrp, cfts, is_add); in cgroup_apply_cftypes()
5077 struct cgroup *cgrp = seq_css(s)->cgroup; in __cgroup_procs_start() local
5088 css_task_iter_start(&cgrp->self, iter_flags, it); in __cgroup_procs_start()
5092 css_task_iter_start(&cgrp->self, iter_flags, it); in __cgroup_procs_start()
5101 struct cgroup *cgrp = seq_css(s)->cgroup; in cgroup_procs_start() local
5109 if (cgroup_is_threaded(cgrp)) in cgroup_procs_start()
5122 static int cgroup_may_write(const struct cgroup *cgrp, struct super_block *sb) in cgroup_may_write() argument
5129 inode = kernfs_get_inode(sb, cgrp->procs_file.kn); in cgroup_may_write()
5405 struct cgroup *cgrp = css->cgroup; in css_free_rwork_fn() local
5416 cgroup_put(cgrp); in css_free_rwork_fn()
5422 atomic_dec(&cgrp->root->nr_cgrps); in css_free_rwork_fn()
5423 if (!cgroup_on_dfl(cgrp)) in css_free_rwork_fn()
5424 cgroup1_pidlist_destroy_all(cgrp); in css_free_rwork_fn()
5425 cancel_work_sync(&cgrp->release_agent_work); in css_free_rwork_fn()
5426 bpf_cgrp_storage_free(cgrp); in css_free_rwork_fn()
5428 if (cgroup_parent(cgrp)) { in css_free_rwork_fn()
5435 cgroup_put(cgroup_parent(cgrp)); in css_free_rwork_fn()
5436 kernfs_put(cgrp->kn); in css_free_rwork_fn()
5437 psi_cgroup_free(cgrp); in css_free_rwork_fn()
5438 cgroup_rstat_exit(cgrp); in css_free_rwork_fn()
5439 kfree(cgrp); in css_free_rwork_fn()
5446 cgroup_destroy_root(cgrp->root); in css_free_rwork_fn()
5456 struct cgroup *cgrp = css->cgroup; in css_release_work_fn() local
5468 cgroup_rstat_flush(cgrp); in css_release_work_fn()
5476 cgrp->nr_dying_subsys[ss->id]--; in css_release_work_fn()
5485 parent_cgrp = cgroup_parent(cgrp); in css_release_work_fn()
5494 TRACE_CGROUP_PATH(release, cgrp); in css_release_work_fn()
5496 cgroup_rstat_flush(cgrp); in css_release_work_fn()
5499 for (tcgrp = cgroup_parent(cgrp); tcgrp; in css_release_work_fn()
5511 if (cgrp->kn) in css_release_work_fn()
5512 RCU_INIT_POINTER(*(void __rcu __force **)&cgrp->kn->priv, in css_release_work_fn()
5532 struct cgroup_subsys *ss, struct cgroup *cgrp) in init_and_link_css() argument
5536 cgroup_get_live(cgrp); in init_and_link_css()
5539 css->cgroup = cgrp; in init_and_link_css()
5548 if (cgroup_parent(cgrp)) { in init_and_link_css()
5549 css->parent = cgroup_css(cgroup_parent(cgrp), ss); in init_and_link_css()
5554 list_add_rcu(&css->rstat_css_node, &cgrp->rstat_css_list); in init_and_link_css()
5556 BUG_ON(cgroup_css(cgrp, ss)); in init_and_link_css()
5621 static struct cgroup_subsys_state *css_create(struct cgroup *cgrp, in css_create() argument
5624 struct cgroup *parent = cgroup_parent(cgrp); in css_create()
5637 init_and_link_css(css, ss, cgrp); in css_create()
5675 struct cgroup *cgrp, *tcgrp; in cgroup_create() local
5681 cgrp = kzalloc(struct_size(cgrp, ancestors, (level + 1)), GFP_KERNEL); in cgroup_create()
5682 if (!cgrp) in cgroup_create()
5685 ret = percpu_ref_init(&cgrp->self.refcnt, css_release, 0, GFP_KERNEL); in cgroup_create()
5689 ret = cgroup_rstat_init(cgrp); in cgroup_create()
5696 cgrp, NULL); in cgroup_create()
5701 cgrp->kn = kn; in cgroup_create()
5703 init_cgroup_housekeeping(cgrp); in cgroup_create()
5705 cgrp->self.parent = &parent->self; in cgroup_create()
5706 cgrp->root = root; in cgroup_create()
5707 cgrp->level = level; in cgroup_create()
5709 ret = psi_cgroup_alloc(cgrp); in cgroup_create()
5713 ret = cgroup_bpf_inherit(cgrp); in cgroup_create()
5721 cgrp->freezer.e_freeze = parent->freezer.e_freeze; in cgroup_create()
5722 if (cgrp->freezer.e_freeze) { in cgroup_create()
5729 set_bit(CGRP_FREEZE, &cgrp->flags); in cgroup_create()
5730 set_bit(CGRP_FROZEN, &cgrp->flags); in cgroup_create()
5734 for (tcgrp = cgrp; tcgrp; tcgrp = cgroup_parent(tcgrp)) { in cgroup_create()
5735 cgrp->ancestors[tcgrp->level] = tcgrp; in cgroup_create()
5737 if (tcgrp != cgrp) { in cgroup_create()
5745 if (cgrp->freezer.e_freeze) in cgroup_create()
5752 set_bit(CGRP_NOTIFY_ON_RELEASE, &cgrp->flags); in cgroup_create()
5755 set_bit(CGRP_CPUSET_CLONE_CHILDREN, &cgrp->flags); in cgroup_create()
5757 cgrp->self.serial_nr = css_serial_nr_next++; in cgroup_create()
5760 list_add_tail_rcu(&cgrp->self.sibling, &cgroup_parent(cgrp)->self.children); in cgroup_create()
5768 if (!cgroup_on_dfl(cgrp)) in cgroup_create()
5769 cgrp->subtree_control = cgroup_control(cgrp); in cgroup_create()
5771 cgroup_propagate_control(cgrp); in cgroup_create()
5773 return cgrp; in cgroup_create()
5776 psi_cgroup_free(cgrp); in cgroup_create()
5778 kernfs_remove(cgrp->kn); in cgroup_create()
5780 cgroup_rstat_exit(cgrp); in cgroup_create()
5782 percpu_ref_exit(&cgrp->self.refcnt); in cgroup_create()
5784 kfree(cgrp); in cgroup_create()
5813 struct cgroup *parent, *cgrp; in cgroup_mkdir() local
5829 cgrp = cgroup_create(parent, name, mode); in cgroup_mkdir()
5830 if (IS_ERR(cgrp)) { in cgroup_mkdir()
5831 ret = PTR_ERR(cgrp); in cgroup_mkdir()
5839 kernfs_get(cgrp->kn); in cgroup_mkdir()
5841 ret = css_populate_dir(&cgrp->self); in cgroup_mkdir()
5845 ret = cgroup_apply_control_enable(cgrp); in cgroup_mkdir()
5849 TRACE_CGROUP_PATH(mkdir, cgrp); in cgroup_mkdir()
5852 kernfs_activate(cgrp->kn); in cgroup_mkdir()
5858 cgroup_destroy_locked(cgrp); in cgroup_mkdir()
5965 static int cgroup_destroy_locked(struct cgroup *cgrp) in cgroup_destroy_locked() argument
5968 struct cgroup *tcgrp, *parent = cgroup_parent(cgrp); in cgroup_destroy_locked()
5979 if (cgroup_is_populated(cgrp)) in cgroup_destroy_locked()
5987 if (css_has_online_children(&cgrp->self)) in cgroup_destroy_locked()
5996 cgrp->self.flags &= ~CSS_ONLINE; in cgroup_destroy_locked()
5999 list_for_each_entry(link, &cgrp->cset_links, cset_link) in cgroup_destroy_locked()
6004 for_each_css(css, ssid, cgrp) in cgroup_destroy_locked()
6008 css_clear_dir(&cgrp->self); in cgroup_destroy_locked()
6009 kernfs_remove(cgrp->kn); in cgroup_destroy_locked()
6011 if (cgroup_is_threaded(cgrp)) in cgroup_destroy_locked()
6022 if (test_bit(CGRP_FROZEN, &cgrp->flags)) in cgroup_destroy_locked()
6029 cgroup_bpf_offline(cgrp); in cgroup_destroy_locked()
6032 percpu_ref_kill(&cgrp->self.refcnt); in cgroup_destroy_locked()
6039 struct cgroup *cgrp; in cgroup_rmdir() local
6042 cgrp = cgroup_kn_lock_live(kn, false); in cgroup_rmdir()
6043 if (!cgrp) in cgroup_rmdir()
6046 ret = cgroup_destroy_locked(cgrp); in cgroup_rmdir()
6048 TRACE_CGROUP_PATH(rmdir, cgrp); in cgroup_rmdir()
6077 init_and_link_css(css, ss, &cgrp_dfl_root.cgrp); in cgroup_init_subsys()
6128 cgrp_dfl_root.cgrp.self.flags |= CSS_NO_REF; in cgroup_init_early()
6197 &cgrp_dfl_root.cgrp.e_csets[ssid]); in cgroup_init()
6291 struct cgroup *cgrp, *root_cgrp; in cgroup_get_from_id() local
6304 cgrp = rcu_dereference(*(void __rcu __force **)&kn->priv); in cgroup_get_from_id()
6305 if (cgrp && !cgroup_tryget(cgrp)) in cgroup_get_from_id()
6306 cgrp = NULL; in cgroup_get_from_id()
6311 if (!cgrp) in cgroup_get_from_id()
6315 if (!cgroup_is_descendant(cgrp, root_cgrp)) { in cgroup_get_from_id()
6316 cgroup_put(cgrp); in cgroup_get_from_id()
6320 return cgrp; in cgroup_get_from_id()
6346 struct cgroup *cgrp; in proc_cgroup_show() local
6352 cgrp = task_cgroup_from_root(tsk, root); in proc_cgroup_show()
6354 if (!cgrp) in proc_cgroup_show()
6376 if (cgroup_on_dfl(cgrp) || !(tsk->flags & PF_EXITING)) { in proc_cgroup_show()
6377 retval = cgroup_path_ns_locked(cgrp, buf, PATH_MAX, in proc_cgroup_show()
6389 if (cgroup_on_dfl(cgrp) && cgroup_is_dead(cgrp)) in proc_cgroup_show()
6443 struct cgroup *cgrp = cgroup_v1v2_get_from_file(f); in cgroup_get_from_file() local
6445 if (IS_ERR(cgrp)) in cgroup_get_from_file()
6446 return ERR_CAST(cgrp); in cgroup_get_from_file()
6448 if (!cgroup_on_dfl(cgrp)) { in cgroup_get_from_file()
6449 cgroup_put(cgrp); in cgroup_get_from_file()
6453 return cgrp; in cgroup_get_from_file()
6552 kargs->cgrp = dst_cgrp; in cgroup_css_set_fork()
6578 struct cgroup *cgrp = kargs->cgrp; in cgroup_css_set_put_fork() local
6590 if (cgrp) { in cgroup_css_set_put_fork()
6591 cgroup_put(cgrp); in cgroup_css_set_put_fork()
6592 kargs->cgrp = NULL; in cgroup_css_set_put_fork()
6685 if (kargs->cgrp) in cgroup_post_fork()
6686 cgrp_flags = kargs->cgrp->flags; in cgroup_post_fork()
6879 struct cgroup *cgrp; in css_tryget_online_from_dir() local
6893 cgrp = rcu_dereference(*(void __rcu __force **)&kn->priv); in css_tryget_online_from_dir()
6894 if (cgrp) in css_tryget_online_from_dir()
6895 css = cgroup_css(cgrp, ss); in css_tryget_online_from_dir()
6930 struct cgroup *cgrp = ERR_PTR(-ENOENT); in cgroup_get_from_path() local
6939 cgrp = ERR_PTR(-ENOTDIR); in cgroup_get_from_path()
6945 cgrp = rcu_dereference(*(void __rcu __force **)&kn->priv); in cgroup_get_from_path()
6946 if (!cgrp || !cgroup_tryget(cgrp)) in cgroup_get_from_path()
6947 cgrp = ERR_PTR(-ENOENT); in cgroup_get_from_path()
6954 return cgrp; in cgroup_get_from_path()
6969 struct cgroup *cgrp; in cgroup_v1v2_get_from_fd() local
6974 cgrp = cgroup_v1v2_get_from_file(fd_file(f)); in cgroup_v1v2_get_from_fd()
6976 return cgrp; in cgroup_v1v2_get_from_fd()
6986 struct cgroup *cgrp = cgroup_v1v2_get_from_fd(fd); in cgroup_get_from_fd() local
6988 if (IS_ERR(cgrp)) in cgroup_get_from_fd()
6989 return ERR_CAST(cgrp); in cgroup_get_from_fd()
6991 if (!cgroup_on_dfl(cgrp)) { in cgroup_get_from_fd()
6992 cgroup_put(cgrp); in cgroup_get_from_fd()
6995 return cgrp; in cgroup_get_from_fd()
7054 cgroup = &cgrp_dfl_root.cgrp; in cgroup_sk_alloc()
7077 struct cgroup *cgrp = sock_cgroup_ptr(skcd); in cgroup_sk_clone() local
7084 cgroup_get(cgrp); in cgroup_sk_clone()
7085 cgroup_bpf_get(cgrp); in cgroup_sk_clone()
7090 struct cgroup *cgrp = sock_cgroup_ptr(skcd); in cgroup_sk_free() local
7092 cgroup_bpf_put(cgrp); in cgroup_sk_free()
7093 cgroup_put(cgrp); in cgroup_sk_free()