Lines Matching +full:precharge +full:- +full:current
1 // SPDX-License-Identifier: GPL-2.0-or-later
7 #include <linux/backing-dev.h>
17 #include "memcontrol-v1.h"
20 * Cgroups above their limits are maintained in a RB-Tree, independent of
58 unsigned long precharge; member
138 struct rb_node **p = &mctz->rb_root.rb_node; in __mem_cgroup_insert_exceeded()
143 if (mz->on_tree) in __mem_cgroup_insert_exceeded()
146 mz->usage_in_excess = new_usage_in_excess; in __mem_cgroup_insert_exceeded()
147 if (!mz->usage_in_excess) in __mem_cgroup_insert_exceeded()
153 if (mz->usage_in_excess < mz_node->usage_in_excess) { in __mem_cgroup_insert_exceeded()
154 p = &(*p)->rb_left; in __mem_cgroup_insert_exceeded()
157 p = &(*p)->rb_right; in __mem_cgroup_insert_exceeded()
162 mctz->rb_rightmost = &mz->tree_node; in __mem_cgroup_insert_exceeded()
164 rb_link_node(&mz->tree_node, parent, p); in __mem_cgroup_insert_exceeded()
165 rb_insert_color(&mz->tree_node, &mctz->rb_root); in __mem_cgroup_insert_exceeded()
166 mz->on_tree = true; in __mem_cgroup_insert_exceeded()
172 if (!mz->on_tree) in __mem_cgroup_remove_exceeded()
175 if (&mz->tree_node == mctz->rb_rightmost) in __mem_cgroup_remove_exceeded()
176 mctz->rb_rightmost = rb_prev(&mz->tree_node); in __mem_cgroup_remove_exceeded()
178 rb_erase(&mz->tree_node, &mctz->rb_root); in __mem_cgroup_remove_exceeded()
179 mz->on_tree = false; in __mem_cgroup_remove_exceeded()
187 spin_lock_irqsave(&mctz->lock, flags); in mem_cgroup_remove_exceeded()
189 spin_unlock_irqrestore(&mctz->lock, flags); in mem_cgroup_remove_exceeded()
194 unsigned long nr_pages = page_counter_read(&memcg->memory); in soft_limit_excess()
195 unsigned long soft_limit = READ_ONCE(memcg->soft_limit); in soft_limit_excess()
199 excess = nr_pages - soft_limit; in soft_limit_excess()
224 mz = memcg->nodeinfo[nid]; in memcg1_update_tree()
227 * We have to update the tree if mz is on RB-tree or in memcg1_update_tree()
230 if (excess || mz->on_tree) { in memcg1_update_tree()
233 spin_lock_irqsave(&mctz->lock, flags); in memcg1_update_tree()
234 /* if on-tree, remove it */ in memcg1_update_tree()
235 if (mz->on_tree) in memcg1_update_tree()
238 * Insert again. mz->usage_in_excess will be updated. in memcg1_update_tree()
242 spin_unlock_irqrestore(&mctz->lock, flags); in memcg1_update_tree()
254 mz = memcg->nodeinfo[nid]; in memcg1_remove_from_trees()
268 if (!mctz->rb_rightmost) in __mem_cgroup_largest_soft_limit_node()
271 mz = rb_entry(mctz->rb_rightmost, in __mem_cgroup_largest_soft_limit_node()
279 if (!soft_limit_excess(mz->memcg) || in __mem_cgroup_largest_soft_limit_node()
280 !css_tryget(&mz->memcg->css)) in __mem_cgroup_largest_soft_limit_node()
291 spin_lock_irq(&mctz->lock); in mem_cgroup_largest_soft_limit_node()
293 spin_unlock_irq(&mctz->lock); in mem_cgroup_largest_soft_limit_node()
364 mctz = soft_limit_tree.rb_tree_per_node[pgdat->node_id]; in memcg1_soft_limit_reclaim()
371 if (!mctz || RB_EMPTY_ROOT(&mctz->rb_root)) in memcg1_soft_limit_reclaim()
387 reclaimed = mem_cgroup_soft_reclaim(mz->memcg, pgdat, in memcg1_soft_limit_reclaim()
390 spin_lock_irq(&mctz->lock); in memcg1_soft_limit_reclaim()
400 excess = soft_limit_excess(mz->memcg); in memcg1_soft_limit_reclaim()
411 spin_unlock_irq(&mctz->lock); in memcg1_soft_limit_reclaim()
412 css_put(&mz->memcg->css); in memcg1_soft_limit_reclaim()
425 css_put(&next_mz->memcg->css); in memcg1_soft_limit_reclaim()
433 * moving cgroups. This is for waiting at high-memory pressure
460 if (mc.moving_task && current != mc.moving_task) { in memcg1_wait_acct_move()
475 * folio_memcg_lock - Bind a folio to its memcg.
491 * path can get away without acquiring the memcg->move_lock in folio_memcg_lock()
505 might_lock(&memcg->move_lock); in folio_memcg_lock()
509 if (atomic_read(&memcg->moving_account) <= 0) in folio_memcg_lock()
512 spin_lock_irqsave(&memcg->move_lock, flags); in folio_memcg_lock()
514 spin_unlock_irqrestore(&memcg->move_lock, flags); in folio_memcg_lock()
520 * critical sections holding the fast-path RCU lock and one in folio_memcg_lock()
524 memcg->move_lock_task = current; in folio_memcg_lock()
525 memcg->move_lock_flags = flags; in folio_memcg_lock()
530 if (memcg && memcg->move_lock_task == current) { in __folio_memcg_unlock()
531 unsigned long flags = memcg->move_lock_flags; in __folio_memcg_unlock()
533 memcg->move_lock_task = NULL; in __folio_memcg_unlock()
534 memcg->move_lock_flags = 0; in __folio_memcg_unlock()
536 spin_unlock_irqrestore(&memcg->move_lock, flags); in __folio_memcg_unlock()
543 * folio_memcg_unlock - Release the binding between a folio and its memcg.
557 * mem_cgroup_move_swap_account - move swap charge and swap_cgroup's record.
565 * Returns 0 on success, -EINVAL on failure.
579 mod_memcg_state(from, MEMCG_SWAP, -1); in mem_cgroup_move_swap_account()
583 return -EINVAL; in mem_cgroup_move_swap_account()
589 return -EINVAL; in mem_cgroup_move_swap_account()
596 return mem_cgroup_from_css(css)->move_charge_at_immigrate; in mem_cgroup_move_charge_read()
606 "Please report your usecase to linux-mm@kvack.org if you " in mem_cgroup_move_charge_write()
610 return -EINVAL; in mem_cgroup_move_charge_write()
613 * No kind of locking is needed in here, because ->can_attach() will in mem_cgroup_move_charge_write()
618 memcg->move_charge_at_immigrate = val; in mem_cgroup_move_charge_write()
625 return -ENOSYS; in mem_cgroup_move_charge_write()
638 mc.precharge += count; in mem_cgroup_do_precharge()
643 while (count--) { in mem_cgroup_do_precharge()
647 mc.precharge++; in mem_cgroup_do_precharge()
713 entry->val = ent.val; in mc_handle_swap_pte()
731 if (!vma->vm_file) /* anonymous vma */ in mc_handle_file_pte()
736 /* folio is moved even if it's not RSS of this task(page-faulted). */ in mc_handle_file_pte()
739 folio = filemap_get_incore_folio(vma->vm_file->f_mapping, index); in mc_handle_file_pte()
749 * mem_cgroup_move_account - move account of the folio
775 ret = -EINVAL; in mem_cgroup_move_account()
787 __mod_lruvec_state(from_vec, NR_ANON_MAPPED, -nr_pages); in mem_cgroup_move_account()
791 -nr_pages); in mem_cgroup_move_account()
797 __mod_lruvec_state(from_vec, NR_FILE_PAGES, -nr_pages); in mem_cgroup_move_account()
801 __mod_lruvec_state(from_vec, NR_SHMEM, -nr_pages); in mem_cgroup_move_account()
806 __mod_lruvec_state(from_vec, NR_FILE_MAPPED, -nr_pages); in mem_cgroup_move_account()
815 -nr_pages); in mem_cgroup_move_account()
824 __mod_lruvec_state(from_vec, NR_SWAPCACHE, -nr_pages); in mem_cgroup_move_account()
829 __mod_lruvec_state(from_vec, NR_WRITEBACK, -nr_pages); in mem_cgroup_move_account()
848 css_get(&to->css); in mem_cgroup_move_account()
849 css_put(&from->css); in mem_cgroup_move_account()
851 /* Warning should never happen, so don't worry about refcount non-0 */ in mem_cgroup_move_account()
853 folio->memcg_data = (unsigned long)to; in mem_cgroup_move_account()
863 memcg1_charge_statistics(from, -nr_pages); in mem_cgroup_move_account()
871 * get_mctgt_type - get target type of moving charge
879 * * MC_TARGET_NONE - If the pte is not a target for move charge.
880 * * MC_TARGET_PAGE - If the page corresponding to this pte is a target for
881 * move charge. If @target is not NULL, the folio is stored in target->folio
883 * * MC_TARGET_SWAP - If the swap entry corresponding to this pte is a
885 * stored in target->ent.
886 * * MC_TARGET_DEVICE - Like MC_TARGET_PAGE but page is device memory and
949 target->folio = folio; in get_mctgt_type()
959 * But we cannot move a tail-page in a THP. in get_mctgt_type()
965 target->ent = ent; in get_mctgt_type()
1001 target->folio = folio; in get_mctgt_type_thp()
1018 struct vm_area_struct *vma = walk->vma; in mem_cgroup_count_precharge_pte_range()
1030 mc.precharge += HPAGE_PMD_NR; in mem_cgroup_count_precharge_pte_range()
1035 pte = pte_offset_map_lock(vma->vm_mm, pmd, addr, &ptl); in mem_cgroup_count_precharge_pte_range()
1040 mc.precharge++; /* increment precharge temporarily */ in mem_cgroup_count_precharge_pte_range()
1041 pte_unmap_unlock(pte - 1, ptl); in mem_cgroup_count_precharge_pte_range()
1054 unsigned long precharge; in mem_cgroup_count_precharge() local
1060 precharge = mc.precharge; in mem_cgroup_count_precharge()
1061 mc.precharge = 0; in mem_cgroup_count_precharge()
1063 return precharge; in mem_cgroup_count_precharge()
1068 unsigned long precharge = mem_cgroup_count_precharge(mm); in mem_cgroup_precharge_mc() local
1071 mc.moving_task = current; in mem_cgroup_precharge_mc()
1072 return mem_cgroup_do_precharge(precharge); in mem_cgroup_precharge_mc()
1082 if (mc.precharge) { in __mem_cgroup_clear_mc()
1083 mem_cgroup_cancel_charge(mc.to, mc.precharge); in __mem_cgroup_clear_mc()
1084 mc.precharge = 0; in __mem_cgroup_clear_mc()
1098 page_counter_uncharge(&mc.from->memsw, mc.moved_swap); in __mem_cgroup_clear_mc()
1103 * we charged both to->memory and to->memsw, so we in __mem_cgroup_clear_mc()
1104 * should uncharge to->memory. in __mem_cgroup_clear_mc()
1107 page_counter_uncharge(&mc.to->memory, mc.moved_swap); in __mem_cgroup_clear_mc()
1150 * Multi-process migrations only happen on the default hierarchy in memcg1_can_attach()
1166 * tunable will only affect upcoming migrations, not the current one. in memcg1_can_attach()
1169 move_flags = READ_ONCE(memcg->move_charge_at_immigrate); in memcg1_can_attach()
1181 if (mm->owner == p) { in memcg1_can_attach()
1184 VM_BUG_ON(mc.precharge); in memcg1_can_attach()
1216 struct vm_area_struct *vma = walk->vma; in mem_cgroup_move_charge_pte_range()
1227 if (mc.precharge < HPAGE_PMD_NR) { in mem_cgroup_move_charge_pte_range()
1239 if (!list_empty(&folio->_deferred_list)) { in mem_cgroup_move_charge_pte_range()
1258 mc.precharge -= HPAGE_PMD_NR; in mem_cgroup_move_charge_pte_range()
1269 mc.precharge -= HPAGE_PMD_NR; in mem_cgroup_move_charge_pte_range()
1280 pte = pte_offset_map_lock(vma->vm_mm, pmd, addr, &ptl); in mem_cgroup_move_charge_pte_range()
1288 if (!mc.precharge) in mem_cgroup_move_charge_pte_range()
1309 mc.precharge--; in mem_cgroup_move_charge_pte_range()
1322 mc.precharge--; in mem_cgroup_move_charge_pte_range()
1332 pte_unmap_unlock(pte - 1, ptl); in mem_cgroup_move_charge_pte_range()
1361 * for already started RCU-only updates to finish. in mem_cgroup_move_charge()
1363 atomic_inc(&mc.from->moving_account); in mem_cgroup_move_charge()
1371 * to move enough charges, but moving charge is a best-effort in mem_cgroup_move_charge()
1384 atomic_dec(&mc.from->moving_account); in mem_cgroup_move_charge()
1416 t = rcu_dereference(memcg->thresholds.primary); in __mem_cgroup_threshold()
1418 t = rcu_dereference(memcg->memsw_thresholds.primary); in __mem_cgroup_threshold()
1430 i = t->current_threshold; in __mem_cgroup_threshold()
1438 for (; i >= 0 && unlikely(t->entries[i].threshold > usage); i--) in __mem_cgroup_threshold()
1439 eventfd_signal(t->entries[i].eventfd); in __mem_cgroup_threshold()
1450 for (; i < t->size && unlikely(t->entries[i].threshold <= usage); i++) in __mem_cgroup_threshold()
1451 eventfd_signal(t->entries[i].eventfd); in __mem_cgroup_threshold()
1454 t->current_threshold = i - 1; in __mem_cgroup_threshold()
1483 nr_pages = -nr_pages; /* for event */ in memcg1_charge_statistics()
1486 __this_cpu_add(memcg->events_percpu->nr_page_events, nr_pages); in memcg1_charge_statistics()
1497 val = __this_cpu_read(memcg->events_percpu->nr_page_events); in memcg1_event_ratelimit()
1498 next = __this_cpu_read(memcg->events_percpu->targets[target]); in memcg1_event_ratelimit()
1500 if ((long)(next - val) < 0) { in memcg1_event_ratelimit()
1511 __this_cpu_write(memcg->events_percpu->targets[target], next); in memcg1_event_ratelimit()
1553 * i_pages lock which is taken with interrupts-off. It is in memcg1_swapout()
1555 * only synchronisation we have for updating the per-CPU variables. in memcg1_swapout()
1559 memcg1_charge_statistics(memcg, -folio_nr_pages(folio)); in memcg1_swapout()
1571 __this_cpu_add(memcg->events_percpu->nr_page_events, nr_memory); in memcg1_uncharge_batch()
1581 if (_a->threshold > _b->threshold) in compare_thresholds()
1584 if (_a->threshold < _b->threshold) in compare_thresholds()
1585 return -1; in compare_thresholds()
1596 list_for_each_entry(ev, &memcg->oom_notify, list) in mem_cgroup_oom_notify_cb()
1597 eventfd_signal(ev->eventfd); in mem_cgroup_oom_notify_cb()
1620 ret = page_counter_memparse(args, "-1", &threshold); in __mem_cgroup_usage_register_event()
1624 mutex_lock(&memcg->thresholds_lock); in __mem_cgroup_usage_register_event()
1627 thresholds = &memcg->thresholds; in __mem_cgroup_usage_register_event()
1630 thresholds = &memcg->memsw_thresholds; in __mem_cgroup_usage_register_event()
1636 if (thresholds->primary) in __mem_cgroup_usage_register_event()
1639 size = thresholds->primary ? thresholds->primary->size + 1 : 1; in __mem_cgroup_usage_register_event()
1644 ret = -ENOMEM; in __mem_cgroup_usage_register_event()
1647 new->size = size; in __mem_cgroup_usage_register_event()
1650 if (thresholds->primary) in __mem_cgroup_usage_register_event()
1651 memcpy(new->entries, thresholds->primary->entries, in __mem_cgroup_usage_register_event()
1652 flex_array_size(new, entries, size - 1)); in __mem_cgroup_usage_register_event()
1655 new->entries[size - 1].eventfd = eventfd; in __mem_cgroup_usage_register_event()
1656 new->entries[size - 1].threshold = threshold; in __mem_cgroup_usage_register_event()
1658 /* Sort thresholds. Registering of new threshold isn't time-critical */ in __mem_cgroup_usage_register_event()
1659 sort(new->entries, size, sizeof(*new->entries), in __mem_cgroup_usage_register_event()
1662 /* Find current threshold */ in __mem_cgroup_usage_register_event()
1663 new->current_threshold = -1; in __mem_cgroup_usage_register_event()
1665 if (new->entries[i].threshold <= usage) { in __mem_cgroup_usage_register_event()
1667 * new->current_threshold will not be used until in __mem_cgroup_usage_register_event()
1671 ++new->current_threshold; in __mem_cgroup_usage_register_event()
1677 kfree(thresholds->spare); in __mem_cgroup_usage_register_event()
1678 thresholds->spare = thresholds->primary; in __mem_cgroup_usage_register_event()
1680 rcu_assign_pointer(thresholds->primary, new); in __mem_cgroup_usage_register_event()
1686 mutex_unlock(&memcg->thresholds_lock); in __mem_cgroup_usage_register_event()
1711 mutex_lock(&memcg->thresholds_lock); in __mem_cgroup_usage_unregister_event()
1714 thresholds = &memcg->thresholds; in __mem_cgroup_usage_unregister_event()
1717 thresholds = &memcg->memsw_thresholds; in __mem_cgroup_usage_unregister_event()
1722 if (!thresholds->primary) in __mem_cgroup_usage_unregister_event()
1730 for (i = 0; i < thresholds->primary->size; i++) { in __mem_cgroup_usage_unregister_event()
1731 if (thresholds->primary->entries[i].eventfd != eventfd) in __mem_cgroup_usage_unregister_event()
1737 new = thresholds->spare; in __mem_cgroup_usage_unregister_event()
1750 new->size = size; in __mem_cgroup_usage_unregister_event()
1752 /* Copy thresholds and find current threshold */ in __mem_cgroup_usage_unregister_event()
1753 new->current_threshold = -1; in __mem_cgroup_usage_unregister_event()
1754 for (i = 0, j = 0; i < thresholds->primary->size; i++) { in __mem_cgroup_usage_unregister_event()
1755 if (thresholds->primary->entries[i].eventfd == eventfd) in __mem_cgroup_usage_unregister_event()
1758 new->entries[j] = thresholds->primary->entries[i]; in __mem_cgroup_usage_unregister_event()
1759 if (new->entries[j].threshold <= usage) { in __mem_cgroup_usage_unregister_event()
1761 * new->current_threshold will not be used in __mem_cgroup_usage_unregister_event()
1765 ++new->current_threshold; in __mem_cgroup_usage_unregister_event()
1772 thresholds->spare = thresholds->primary; in __mem_cgroup_usage_unregister_event()
1774 rcu_assign_pointer(thresholds->primary, new); in __mem_cgroup_usage_unregister_event()
1781 kfree(thresholds->spare); in __mem_cgroup_usage_unregister_event()
1782 thresholds->spare = NULL; in __mem_cgroup_usage_unregister_event()
1785 mutex_unlock(&memcg->thresholds_lock); in __mem_cgroup_usage_unregister_event()
1807 return -ENOMEM; in mem_cgroup_oom_register_event()
1811 event->eventfd = eventfd; in mem_cgroup_oom_register_event()
1812 list_add(&event->list, &memcg->oom_notify); in mem_cgroup_oom_register_event()
1815 if (memcg->under_oom) in mem_cgroup_oom_register_event()
1829 list_for_each_entry_safe(ev, tmp, &memcg->oom_notify, list) { in mem_cgroup_oom_unregister_event()
1830 if (ev->eventfd == eventfd) { in mem_cgroup_oom_unregister_event()
1831 list_del(&ev->list); in mem_cgroup_oom_unregister_event()
1844 * This is way over-engineered. It tries to support fully configurable
1861 struct mem_cgroup *memcg = event->memcg; in memcg_event_remove()
1863 remove_wait_queue(event->wqh, &event->wait); in memcg_event_remove()
1865 event->unregister_event(memcg, event->eventfd); in memcg_event_remove()
1868 eventfd_signal(event->eventfd); in memcg_event_remove()
1870 eventfd_ctx_put(event->eventfd); in memcg_event_remove()
1872 css_put(&memcg->css); in memcg_event_remove()
1878 * Called with wqh->lock held and interrupts disabled.
1885 struct mem_cgroup *memcg = event->memcg; in memcg_event_wake()
1895 * side will require wqh->lock via remove_wait_queue(), in memcg_event_wake()
1898 spin_lock(&memcg->event_list_lock); in memcg_event_wake()
1899 if (!list_empty(&event->list)) { in memcg_event_wake()
1900 list_del_init(&event->list); in memcg_event_wake()
1905 schedule_work(&event->remove); in memcg_event_wake()
1907 spin_unlock(&memcg->event_list_lock); in memcg_event_wake()
1919 event->wqh = wqh; in memcg_event_ptable_queue_proc()
1920 add_wait_queue(wqh, &event->wait); in memcg_event_ptable_queue_proc()
1947 return -EOPNOTSUPP; in memcg_write_event_control()
1953 return -EINVAL; in memcg_write_event_control()
1962 return -EINVAL; in memcg_write_event_control()
1966 return -ENOMEM; in memcg_write_event_control()
1968 event->memcg = memcg; in memcg_write_event_control()
1969 INIT_LIST_HEAD(&event->list); in memcg_write_event_control()
1970 init_poll_funcptr(&event->pt, memcg_event_ptable_queue_proc); in memcg_write_event_control()
1971 init_waitqueue_func_entry(&event->wait, memcg_event_wake); in memcg_write_event_control()
1972 INIT_WORK(&event->remove, memcg_event_remove); in memcg_write_event_control()
1976 ret = -EBADF; in memcg_write_event_control()
1980 event->eventfd = eventfd_ctx_fileget(fd_file(efile)); in memcg_write_event_control()
1981 if (IS_ERR(event->eventfd)) { in memcg_write_event_control()
1982 ret = PTR_ERR(event->eventfd); in memcg_write_event_control()
1988 ret = -EBADF; in memcg_write_event_control()
2002 cdentry = fd_file(cfile)->f_path.dentry; in memcg_write_event_control()
2003 if (cdentry->d_sb->s_type != &cgroup_fs_type || !d_is_reg(cdentry)) { in memcg_write_event_control()
2004 ret = -EINVAL; in memcg_write_event_control()
2016 name = cdentry->d_name.name; in memcg_write_event_control()
2019 event->register_event = mem_cgroup_usage_register_event; in memcg_write_event_control()
2020 event->unregister_event = mem_cgroup_usage_unregister_event; in memcg_write_event_control()
2023 "Please report your usecase to linux-mm-@kvack.org" in memcg_write_event_control()
2025 event->register_event = mem_cgroup_oom_register_event; in memcg_write_event_control()
2026 event->unregister_event = mem_cgroup_oom_unregister_event; in memcg_write_event_control()
2029 "Please report your usecase to linux-mm-@kvack.org " in memcg_write_event_control()
2031 event->register_event = vmpressure_register_event; in memcg_write_event_control()
2032 event->unregister_event = vmpressure_unregister_event; in memcg_write_event_control()
2034 event->register_event = memsw_cgroup_usage_register_event; in memcg_write_event_control()
2035 event->unregister_event = memsw_cgroup_usage_unregister_event; in memcg_write_event_control()
2037 ret = -EINVAL; in memcg_write_event_control()
2046 cfile_css = css_tryget_online_from_dir(cdentry->d_parent, in memcg_write_event_control()
2048 ret = -EINVAL; in memcg_write_event_control()
2056 ret = event->register_event(memcg, event->eventfd, buf); in memcg_write_event_control()
2060 vfs_poll(fd_file(efile), &event->pt); in memcg_write_event_control()
2062 spin_lock_irq(&memcg->event_list_lock); in memcg_write_event_control()
2063 list_add(&event->list, &memcg->event_list); in memcg_write_event_control()
2064 spin_unlock_irq(&memcg->event_list_lock); in memcg_write_event_control()
2076 eventfd_ctx_put(event->eventfd); in memcg_write_event_control()
2087 INIT_LIST_HEAD(&memcg->oom_notify); in memcg1_memcg_init()
2088 mutex_init(&memcg->thresholds_lock); in memcg1_memcg_init()
2089 spin_lock_init(&memcg->move_lock); in memcg1_memcg_init()
2090 INIT_LIST_HEAD(&memcg->event_list); in memcg1_memcg_init()
2091 spin_lock_init(&memcg->event_list_lock); in memcg1_memcg_init()
2103 spin_lock_irq(&memcg->event_list_lock); in memcg1_css_offline()
2104 list_for_each_entry_safe(event, tmp, &memcg->event_list, list) { in memcg1_css_offline()
2105 list_del_init(&event->list); in memcg1_css_offline()
2106 schedule_work(&event->remove); in memcg1_css_offline()
2108 spin_unlock_irq(&memcg->event_list_lock); in memcg1_css_offline()
2112 * Check OOM-Killer is already running under our hierarchy.
2122 if (iter->oom_lock) { in mem_cgroup_oom_trylock()
2131 iter->oom_lock = true; in mem_cgroup_oom_trylock()
2144 iter->oom_lock = false; in mem_cgroup_oom_trylock()
2161 iter->oom_lock = false; in mem_cgroup_oom_unlock()
2171 iter->under_oom++; in mem_cgroup_mark_under_oom()
2185 if (iter->under_oom > 0) in mem_cgroup_unmark_under_oom()
2186 iter->under_oom--; in mem_cgroup_unmark_under_oom()
2205 oom_wait_memcg = oom_wait_info->memcg; in memcg_oom_wake_function()
2216 * For the following lockless ->under_oom test, the only required in memcg1_oom_recover()
2223 if (memcg && memcg->under_oom) in memcg1_oom_recover()
2228 * mem_cgroup_oom_synchronize - complete memcg OOM handling
2246 struct mem_cgroup *memcg = current->memcg_in_oom; in mem_cgroup_oom_synchronize()
2260 owait.wait.private = current; in mem_cgroup_oom_synchronize()
2278 current->memcg_in_oom = NULL; in mem_cgroup_oom_synchronize()
2279 css_put(&memcg->css); in mem_cgroup_oom_synchronize()
2296 * On the other hand, in-kernel OOM killer allows for an async victim in memcg1_oom_prepare()
2304 if (READ_ONCE(memcg->oom_kill_disable)) { in memcg1_oom_prepare()
2305 if (current->in_user_fault) { in memcg1_oom_prepare()
2306 css_get(&memcg->css); in memcg1_oom_prepare()
2307 current->memcg_in_oom = memcg; in memcg1_oom_prepare()
2339 struct page_counter *counter = memsw ? &memcg->memsw : &memcg->memory; in mem_cgroup_resize_max()
2342 if (signal_pending(current)) { in mem_cgroup_resize_max()
2343 ret = -EINTR; in mem_cgroup_resize_max()
2352 limits_invariant = memsw ? max >= READ_ONCE(memcg->memory.max) : in mem_cgroup_resize_max()
2353 max <= memcg->memsw.max; in mem_cgroup_resize_max()
2356 ret = -EINVAL; in mem_cgroup_resize_max()
2359 if (max > counter->max) in mem_cgroup_resize_max()
2375 ret = -EBUSY; in mem_cgroup_resize_max()
2395 /* we call try-to-free pages for make this cgroup empty */ in mem_cgroup_force_empty()
2401 while (nr_retries && page_counter_read(&memcg->memory)) { in mem_cgroup_force_empty()
2402 if (signal_pending(current)) in mem_cgroup_force_empty()
2403 return -EINTR; in mem_cgroup_force_empty()
2407 nr_retries--; in mem_cgroup_force_empty()
2420 return -EINVAL; in mem_cgroup_force_empty_write()
2436 pr_warn_once("Non-hierarchical mode is deprecated. " in mem_cgroup_hierarchy_write()
2437 "Please report your usecase to linux-mm@kvack.org if you " in mem_cgroup_hierarchy_write()
2440 return -EINVAL; in mem_cgroup_hierarchy_write()
2449 switch (MEMFILE_TYPE(cft->private)) { in mem_cgroup_read_u64()
2451 counter = &memcg->memory; in mem_cgroup_read_u64()
2454 counter = &memcg->memsw; in mem_cgroup_read_u64()
2457 counter = &memcg->kmem; in mem_cgroup_read_u64()
2460 counter = &memcg->tcpmem; in mem_cgroup_read_u64()
2466 switch (MEMFILE_ATTR(cft->private)) { in mem_cgroup_read_u64()
2468 if (counter == &memcg->memory) in mem_cgroup_read_u64()
2470 if (counter == &memcg->memsw) in mem_cgroup_read_u64()
2474 return (u64)counter->max * PAGE_SIZE; in mem_cgroup_read_u64()
2476 return (u64)counter->watermark * PAGE_SIZE; in mem_cgroup_read_u64()
2478 return counter->failcnt; in mem_cgroup_read_u64()
2480 return (u64)READ_ONCE(memcg->soft_limit) * PAGE_SIZE; in mem_cgroup_read_u64()
2493 return -EINVAL; in mem_cgroup_dummy_seq_show()
2502 ret = page_counter_set_max(&memcg->tcpmem, max); in memcg_update_tcp_max()
2506 if (!memcg->tcpmem_active) { in memcg_update_tcp_max()
2524 memcg->tcpmem_active = true; in memcg_update_tcp_max()
2543 ret = page_counter_memparse(buf, "-1", &nr_pages); in mem_cgroup_write()
2547 switch (MEMFILE_ATTR(of_cft(of)->private)) { in mem_cgroup_write()
2550 ret = -EINVAL; in mem_cgroup_write()
2553 switch (MEMFILE_TYPE(of_cft(of)->private)) { in mem_cgroup_write()
2563 "Please report your usecase to linux-mm@kvack.org if you " in mem_cgroup_write()
2569 "Please report your usecase to linux-mm@kvack.org if you " in mem_cgroup_write()
2577 ret = -EOPNOTSUPP; in mem_cgroup_write()
2580 "Please report your usecase to linux-mm@kvack.org if you " in mem_cgroup_write()
2582 WRITE_ONCE(memcg->soft_limit, nr_pages); in mem_cgroup_write()
2596 switch (MEMFILE_TYPE(of_cft(of)->private)) { in mem_cgroup_reset()
2598 counter = &memcg->memory; in mem_cgroup_reset()
2601 counter = &memcg->memsw; in mem_cgroup_reset()
2604 counter = &memcg->kmem; in mem_cgroup_reset()
2607 counter = &memcg->tcpmem; in mem_cgroup_reset()
2613 switch (MEMFILE_ATTR(of_cft(of)->private)) { in mem_cgroup_reset()
2618 counter->failcnt = 0; in mem_cgroup_reset()
2631 #define LRU_ALL ((1 << NR_LRU_LISTS) - 1)
2691 seq_printf(m, "%s=%lu", stat->name, in memcg_numa_stat_show()
2692 mem_cgroup_nr_lru_pages(memcg, stat->lru_mask, in memcg_numa_stat_show()
2697 stat->lru_mask, false)); in memcg_numa_stat_show()
2703 seq_printf(m, "hierarchical_%s=%lu", stat->name, in memcg_numa_stat_show()
2704 mem_cgroup_nr_lru_pages(memcg, stat->lru_mask, in memcg_numa_stat_show()
2709 stat->lru_mask, true)); in memcg_numa_stat_show()
2790 memory = min(memory, READ_ONCE(mi->memory.max)); in memcg1_stat_format()
2791 memsw = min(memsw, READ_ONCE(mi->memsw.max)); in memcg1_stat_format()
2824 mz = memcg->nodeinfo[pgdat->node_id]; in memcg1_stat_format()
2826 anon_cost += mz->lruvec.anon_cost; in memcg1_stat_format()
2827 file_cost += mz->lruvec.file_cost; in memcg1_stat_format()
2849 return -EINVAL; in mem_cgroup_swappiness_write()
2852 WRITE_ONCE(memcg->swappiness, val); in mem_cgroup_swappiness_write()
2863 seq_printf(sf, "oom_kill_disable %d\n", READ_ONCE(memcg->oom_kill_disable)); in mem_cgroup_oom_control_read()
2864 seq_printf(sf, "under_oom %d\n", (bool)memcg->under_oom); in mem_cgroup_oom_control_read()
2866 atomic_long_read(&memcg->memory_events[MEMCG_OOM_KILL])); in mem_cgroup_oom_control_read()
2876 "Please report your usecase to linux-mm-@kvack.org if you " in mem_cgroup_oom_control_write()
2881 return -EINVAL; in mem_cgroup_oom_control_write()
2883 WRITE_ONCE(memcg->oom_kill_disable, val); in mem_cgroup_oom_control_write()
3060 page_counter_charge(&memcg->kmem, nr_pages); in memcg1_account_kmem()
3062 page_counter_uncharge(&memcg->kmem, -nr_pages); in memcg1_account_kmem()
3071 if (page_counter_try_charge(&memcg->tcpmem, nr_pages, &fail)) { in memcg1_charge_skmem()
3072 memcg->tcpmem_pressure = 0; in memcg1_charge_skmem()
3075 memcg->tcpmem_pressure = 1; in memcg1_charge_skmem()
3077 page_counter_charge(&memcg->tcpmem, nr_pages); in memcg1_charge_skmem()
3085 memcg->events_percpu = alloc_percpu_gfp(struct memcg1_events_percpu, in memcg1_alloc_events()
3087 return !!memcg->events_percpu; in memcg1_alloc_events()
3092 if (memcg->events_percpu) in memcg1_free_events()
3093 free_percpu(memcg->events_percpu); in memcg1_free_events()
3105 rtpn->rb_root = RB_ROOT; in memcg1_init()
3106 rtpn->rb_rightmost = NULL; in memcg1_init()
3107 spin_lock_init(&rtpn->lock); in memcg1_init()