43 __hugetlb_cgroup_counter_from_cgroup(struct hugetlb_cgroup *h_cg, int idx,  in __hugetlb_cgroup_counter_from_cgroup()  argument
47 		return &h_cg->rsvd_hugepage[idx];  in __hugetlb_cgroup_counter_from_cgroup()
48 	return &h_cg->hugepage[idx];  in __hugetlb_cgroup_counter_from_cgroup()
52 hugetlb_cgroup_counter_from_cgroup(struct hugetlb_cgroup *h_cg, int idx)  in hugetlb_cgroup_counter_from_cgroup()  argument
54 	return __hugetlb_cgroup_counter_from_cgroup(h_cg, idx, false);  in hugetlb_cgroup_counter_from_cgroup()
58 hugetlb_cgroup_counter_from_cgroup_rsvd(struct hugetlb_cgroup *h_cg, int idx)  in hugetlb_cgroup_counter_from_cgroup_rsvd()  argument
60 	return __hugetlb_cgroup_counter_from_cgroup(h_cg, idx, true);  in hugetlb_cgroup_counter_from_cgroup_rsvd()
75 static inline bool hugetlb_cgroup_is_root(struct hugetlb_cgroup *h_cg)  in hugetlb_cgroup_is_root()  argument
77 	return (h_cg == root_h_cgroup);  in hugetlb_cgroup_is_root()
81 parent_hugetlb_cgroup(struct hugetlb_cgroup *h_cg)  in parent_hugetlb_cgroup()  argument
83 	return hugetlb_cgroup_from_css(h_cg->css.parent);  in parent_hugetlb_cgroup()
86 static inline bool hugetlb_cgroup_have_usage(struct hugetlb_cgroup *h_cg)  in hugetlb_cgroup_have_usage()  argument
92 		    hugetlb_cgroup_counter_from_cgroup(h_cg, hstate_index(h))))  in hugetlb_cgroup_have_usage()
197 static void hugetlb_cgroup_move_parent(int idx, struct hugetlb_cgroup *h_cg,  in hugetlb_cgroup_move_parent()  argument
203 	struct hugetlb_cgroup *parent = parent_hugetlb_cgroup(h_cg);  in hugetlb_cgroup_move_parent()
212 	if (!page_hcg || page_hcg != h_cg)  in hugetlb_cgroup_move_parent()
221 	counter = &h_cg->hugepage[idx];  in hugetlb_cgroup_move_parent()
236 	struct hugetlb_cgroup *h_cg = hugetlb_cgroup_from_css(css);  in hugetlb_cgroup_css_offline()  local
244 				hugetlb_cgroup_move_parent(hstate_index(h), h_cg, page);  in hugetlb_cgroup_css_offline()
249 	} while (hugetlb_cgroup_have_usage(h_cg));  in hugetlb_cgroup_css_offline()
271 	struct hugetlb_cgroup *h_cg = NULL;  in __hugetlb_cgroup_charge_cgroup()  local
277 	h_cg = hugetlb_cgroup_from_task(current);  in __hugetlb_cgroup_charge_cgroup()
278 	if (!css_tryget(&h_cg->css)) {  in __hugetlb_cgroup_charge_cgroup()
285 		    __hugetlb_cgroup_counter_from_cgroup(h_cg, idx, rsvd),  in __hugetlb_cgroup_charge_cgroup()
288 		hugetlb_event(h_cg, idx, HUGETLB_MAX);  in __hugetlb_cgroup_charge_cgroup()
289 		css_put(&h_cg->css);  in __hugetlb_cgroup_charge_cgroup()
296 		css_put(&h_cg->css);  in __hugetlb_cgroup_charge_cgroup()
298 	*ptr = h_cg;  in __hugetlb_cgroup_charge_cgroup()
316 					   struct hugetlb_cgroup *h_cg,  in __hugetlb_cgroup_commit_charge()  argument
319 	if (hugetlb_cgroup_disabled() || !h_cg)  in __hugetlb_cgroup_commit_charge()
322 	__set_hugetlb_cgroup(folio, h_cg, rsvd);  in __hugetlb_cgroup_commit_charge()
325 			h_cg->nodeinfo[folio_nid(folio)]->usage[idx];  in __hugetlb_cgroup_commit_charge()
331 		WRITE_ONCE(h_cg->nodeinfo[folio_nid(folio)]->usage[idx],  in __hugetlb_cgroup_commit_charge()
337 				  struct hugetlb_cgroup *h_cg,  in hugetlb_cgroup_commit_charge()  argument
340 	__hugetlb_cgroup_commit_charge(idx, nr_pages, h_cg, folio, false);  in hugetlb_cgroup_commit_charge()
344 				       struct hugetlb_cgroup *h_cg,  in hugetlb_cgroup_commit_charge_rsvd()  argument
347 	__hugetlb_cgroup_commit_charge(idx, nr_pages, h_cg, folio, true);  in hugetlb_cgroup_commit_charge_rsvd()
356 	struct hugetlb_cgroup *h_cg;  in __hugetlb_cgroup_uncharge_folio()  local
361 	h_cg = __hugetlb_cgroup_from_folio(folio, rsvd);  in __hugetlb_cgroup_uncharge_folio()
362 	if (unlikely(!h_cg))  in __hugetlb_cgroup_uncharge_folio()
366 	page_counter_uncharge(__hugetlb_cgroup_counter_from_cgroup(h_cg, idx,  in __hugetlb_cgroup_uncharge_folio()
371 		css_put(&h_cg->css);  in __hugetlb_cgroup_uncharge_folio()
374 			h_cg->nodeinfo[folio_nid(folio)]->usage[idx];  in __hugetlb_cgroup_uncharge_folio()
380 		WRITE_ONCE(h_cg->nodeinfo[folio_nid(folio)]->usage[idx],  in __hugetlb_cgroup_uncharge_folio()
398 					     struct hugetlb_cgroup *h_cg,  in __hugetlb_cgroup_uncharge_cgroup()  argument
401 	if (hugetlb_cgroup_disabled() || !h_cg)  in __hugetlb_cgroup_uncharge_cgroup()
404 	page_counter_uncharge(__hugetlb_cgroup_counter_from_cgroup(h_cg, idx,  in __hugetlb_cgroup_uncharge_cgroup()
409 		css_put(&h_cg->css);  in __hugetlb_cgroup_uncharge_cgroup()
413 				    struct hugetlb_cgroup *h_cg)  in hugetlb_cgroup_uncharge_cgroup()  argument
415 	__hugetlb_cgroup_uncharge_cgroup(idx, nr_pages, h_cg, false);  in hugetlb_cgroup_uncharge_cgroup()
419 					 struct hugetlb_cgroup *h_cg)  in hugetlb_cgroup_uncharge_cgroup_rsvd()  argument
421 	__hugetlb_cgroup_uncharge_cgroup(idx, nr_pages, h_cg, true);  in hugetlb_cgroup_uncharge_cgroup_rsvd()
474 	struct hugetlb_cgroup *h_cg = hugetlb_cgroup_from_css(seq_css(seq));  in hugetlb_cgroup_read_numa_stat()  local
482 			usage += READ_ONCE(h_cg->nodeinfo[nid]->usage[idx]);  in hugetlb_cgroup_read_numa_stat()
488 				   READ_ONCE(h_cg->nodeinfo[nid]->usage[idx]) *  in hugetlb_cgroup_read_numa_stat()
498 		   page_counter_read(&h_cg->hugepage[idx]) * PAGE_SIZE);  in hugetlb_cgroup_read_numa_stat()
507 		css_for_each_descendant_pre(css, &h_cg->css) {  in hugetlb_cgroup_read_numa_stat()
526 	struct hugetlb_cgroup *h_cg = hugetlb_cgroup_from_css(css);  in hugetlb_cgroup_read_u64()  local
528 	counter = &h_cg->hugepage[MEMFILE_IDX(cft->private)];  in hugetlb_cgroup_read_u64()
529 	rsvd_counter = &h_cg->rsvd_hugepage[MEMFILE_IDX(cft->private)];  in hugetlb_cgroup_read_u64()
560 	struct hugetlb_cgroup *h_cg = hugetlb_cgroup_from_css(seq_css(seq));  in hugetlb_cgroup_read_u64_max()  local
563 	counter = &h_cg->hugepage[idx];  in hugetlb_cgroup_read_u64_max()
570 		counter = &h_cg->rsvd_hugepage[idx];  in hugetlb_cgroup_read_u64_max()
577 		counter = &h_cg->rsvd_hugepage[idx];  in hugetlb_cgroup_read_u64_max()
601 	struct hugetlb_cgroup *h_cg = hugetlb_cgroup_from_css(of_css(of));  in hugetlb_cgroup_write()  local
604 	if (hugetlb_cgroup_is_root(h_cg)) /* Can't set limit on root */  in hugetlb_cgroup_write()
622 			__hugetlb_cgroup_counter_from_cgroup(h_cg, idx, rsvd),  in hugetlb_cgroup_write()
650 	struct hugetlb_cgroup *h_cg = hugetlb_cgroup_from_css(of_css(of));  in hugetlb_cgroup_reset()  local
652 	counter = &h_cg->hugepage[MEMFILE_IDX(of_cft(of)->private)];  in hugetlb_cgroup_reset()
653 	rsvd_counter = &h_cg->rsvd_hugepage[MEMFILE_IDX(of_cft(of)->private)];  in hugetlb_cgroup_reset()
691 	struct hugetlb_cgroup *h_cg = hugetlb_cgroup_from_css(seq_css(seq));  in __hugetlb_events_show()  local
696 		max = atomic_long_read(&h_cg->events_local[idx][HUGETLB_MAX]);  in __hugetlb_events_show()
698 		max = atomic_long_read(&h_cg->events[idx][HUGETLB_MAX]);  in __hugetlb_events_show()
902 	struct hugetlb_cgroup *h_cg;  in hugetlb_cgroup_migrate()  local
910 	h_cg = hugetlb_cgroup_from_folio(old_folio);  in hugetlb_cgroup_migrate()
916 	set_hugetlb_cgroup(new_folio, h_cg);  in hugetlb_cgroup_migrate()