289 static bool cond_accept_memory(struct zone *zone, unsigned int order);
314 _deferred_grow_zone(struct zone *zone, unsigned int order)  in _deferred_grow_zone()  argument
316 	return deferred_grow_zone(zone, order);  in _deferred_grow_zone()
324 static inline bool _deferred_grow_zone(struct zone *zone, unsigned int order)  in _deferred_grow_zone()  argument
429 static int page_outside_zone_boundaries(struct zone *zone, struct page *page)  in page_outside_zone_boundaries()  argument
437 		seq = zone_span_seqbegin(zone);  in page_outside_zone_boundaries()
438 		start_pfn = zone->zone_start_pfn;  in page_outside_zone_boundaries()
439 		sp = zone->spanned_pages;  in page_outside_zone_boundaries()
440 		ret = !zone_spans_pfn(zone, pfn);  in page_outside_zone_boundaries()
441 	} while (zone_span_seqretry(zone, seq));  in page_outside_zone_boundaries()
445 			pfn, zone_to_nid(zone), zone->name,  in page_outside_zone_boundaries()
454 static bool __maybe_unused bad_range(struct zone *zone, struct page *page)  in bad_range()  argument
456 	if (page_outside_zone_boundaries(zone, page))  in bad_range()
458 	if (zone != page_zone(page))  in bad_range()
464 static inline bool __maybe_unused bad_range(struct zone *zone, struct page *page)  in bad_range()  argument
584 static inline struct capture_control *task_capc(struct zone *zone)  in task_capc()  argument
591 		capc->cc->zone == zone ? capc : NULL;  in task_capc()
622 static inline struct capture_control *task_capc(struct zone *zone)  in task_capc()  argument
635 static inline void account_freepages(struct zone *zone, int nr_pages,  in account_freepages()  argument
638 	lockdep_assert_held(&zone->lock);  in account_freepages()
643 	__mod_zone_page_state(zone, NR_FREE_PAGES, nr_pages);  in account_freepages()
646 		__mod_zone_page_state(zone, NR_FREE_CMA_PAGES, nr_pages);  in account_freepages()
648 		WRITE_ONCE(zone->nr_free_highatomic,  in account_freepages()
649 			   zone->nr_free_highatomic + nr_pages);  in account_freepages()
653 static inline void __add_to_free_list(struct page *page, struct zone *zone,  in __add_to_free_list()  argument
657 	struct free_area *area = &zone->free_area[order];  in __add_to_free_list()
675 static inline void move_to_free_list(struct page *page, struct zone *zone,  in move_to_free_list()  argument
678 	struct free_area *area = &zone->free_area[order];  in move_to_free_list()
687 	account_freepages(zone, -(1 << order), old_mt);  in move_to_free_list()
688 	account_freepages(zone, 1 << order, new_mt);  in move_to_free_list()
691 static inline void __del_page_from_free_list(struct page *page, struct zone *zone,  in __del_page_from_free_list()  argument
705 	zone->free_area[order].nr_free--;  in __del_page_from_free_list()
708 static inline void del_page_from_free_list(struct page *page, struct zone *zone,  in del_page_from_free_list()  argument
711 	__del_page_from_free_list(page, zone, order, migratetype);  in del_page_from_free_list()
712 	account_freepages(zone, -(1 << order), migratetype);  in del_page_from_free_list()
773 		struct zone *zone, unsigned int order,  in __free_one_page()  argument
776 	struct capture_control *capc = task_capc(zone);  in __free_one_page()
782 	VM_BUG_ON(!zone_is_initialized(zone));  in __free_one_page()
787 	VM_BUG_ON_PAGE(bad_range(zone, page), page);  in __free_one_page()
789 	account_freepages(zone, 1 << order, migratetype);  in __free_one_page()
795 			account_freepages(zone, -(1 << order), migratetype);  in __free_one_page()
823 			clear_page_guard(zone, buddy, order);  in __free_one_page()
825 			__del_page_from_free_list(buddy, zone, order, buddy_mt);  in __free_one_page()
852 	__add_to_free_list(page, zone, order, migratetype, to_tail);  in __free_one_page()
1175 static void free_pcppages_bulk(struct zone *zone, int count,  in free_pcppages_bulk()  argument
1192 	spin_lock_irqsave(&zone->lock, flags);  in free_pcppages_bulk()
1220 			__free_one_page(page, pfn, zone, order, mt, FPI_NONE);  in free_pcppages_bulk()
1225 	spin_unlock_irqrestore(&zone->lock, flags);  in free_pcppages_bulk()
1229 static void split_large_buddy(struct zone *zone, struct page *page,  in split_large_buddy()  argument
1244 		__free_one_page(page, pfn, zone, order, mt, fpi);  in split_large_buddy()
1250 static void free_one_page(struct zone *zone, struct page *page,  in free_one_page()  argument
1256 	spin_lock_irqsave(&zone->lock, flags);  in free_one_page()
1257 	split_large_buddy(zone, page, pfn, order, fpi_flags);  in free_one_page()
1258 	spin_unlock_irqrestore(&zone->lock, flags);  in free_one_page()
1267 	struct zone *zone = page_zone(page);  in __free_pages_ok()  local
1270 		free_one_page(zone, page, pfn, order, fpi_flags);  in __free_pages_ok()
1352 				     unsigned long end_pfn, struct zone *zone)  in __pageblock_pfn_to_page()  argument
1367 	if (page_zone(start_page) != zone)  in __pageblock_pfn_to_page()
1393 static inline unsigned int expand(struct zone *zone, struct page *page, int low,  in expand()  argument
1402 		VM_BUG_ON_PAGE(bad_range(zone, &page[size]), &page[size]);  in expand()
1410 		if (set_page_guard(zone, &page[size], high))  in expand()
1413 		__add_to_free_list(&page[size], zone, high, migratetype, false);  in expand()
1421 static __always_inline void page_del_and_expand(struct zone *zone,  in page_del_and_expand()  argument
1427 	__del_page_from_free_list(page, zone, high, migratetype);  in page_del_and_expand()
1428 	nr_pages -= expand(zone, page, low, high, migratetype);  in page_del_and_expand()
1429 	account_freepages(zone, -nr_pages, migratetype);  in page_del_and_expand()
1586 struct page *__rmqueue_smallest(struct zone *zone, unsigned int order,  in __rmqueue_smallest()  argument
1595 		area = &(zone->free_area[current_order]);  in __rmqueue_smallest()
1600 		page_del_and_expand(zone, page, order, current_order,  in __rmqueue_smallest()
1625 static __always_inline struct page *__rmqueue_cma_fallback(struct zone *zone,  in __rmqueue_cma_fallback()  argument
1628 	return __rmqueue_smallest(zone, order, MIGRATE_CMA);  in __rmqueue_cma_fallback()
1631 static inline struct page *__rmqueue_cma_fallback(struct zone *zone,  in __rmqueue_cma_fallback()  argument
1639 static int __move_freepages_block(struct zone *zone, unsigned long start_pfn,  in __move_freepages_block()  argument
1658 		VM_BUG_ON_PAGE(page_to_nid(page) != zone_to_nid(zone), page);  in __move_freepages_block()
1659 		VM_BUG_ON_PAGE(page_zone(page) != zone, page);  in __move_freepages_block()
1663 		move_to_free_list(page, zone, order, old_mt, new_mt);  in __move_freepages_block()
1674 static bool prep_move_freepages_block(struct zone *zone, struct page *page,  in prep_move_freepages_block()  argument
1691 	if (!zone_spans_pfn(zone, start))  in prep_move_freepages_block()
1693 	if (!zone_spans_pfn(zone, end - 1))  in prep_move_freepages_block()
1724 static int move_freepages_block(struct zone *zone, struct page *page,  in move_freepages_block()  argument
1729 	if (!prep_move_freepages_block(zone, page, &start_pfn, NULL, NULL))  in move_freepages_block()
1732 	return __move_freepages_block(zone, start_pfn, old_mt, new_mt);  in move_freepages_block()
1779 bool move_freepages_block_isolate(struct zone *zone, struct page *page,  in move_freepages_block_isolate()  argument
1784 	if (!prep_move_freepages_block(zone, page, &start_pfn, NULL, NULL))  in move_freepages_block_isolate()
1797 		del_page_from_free_list(buddy, zone, order,  in move_freepages_block_isolate()
1800 		split_large_buddy(zone, buddy, pfn, order, FPI_NONE);  in move_freepages_block_isolate()
1808 		del_page_from_free_list(page, zone, order,  in move_freepages_block_isolate()
1811 		split_large_buddy(zone, page, pfn, order, FPI_NONE);  in move_freepages_block_isolate()
1815 	__move_freepages_block(zone, start_pfn,  in move_freepages_block_isolate()
1866 static inline bool boost_watermark(struct zone *zone)  in boost_watermark()  argument
1878 	if ((pageblock_nr_pages * 4) > zone_managed_pages(zone))  in boost_watermark()
1881 	max_boost = mult_frac(zone->_watermark[WMARK_HIGH],  in boost_watermark()
1897 	zone->watermark_boost = min(zone->watermark_boost + pageblock_nr_pages,  in boost_watermark()
1912 steal_suitable_fallback(struct zone *zone, struct page *page,  in steal_suitable_fallback()  argument
1933 		del_page_from_free_list(page, zone, current_order, block_type);  in steal_suitable_fallback()
1935 		nr_added = expand(zone, page, order, current_order, start_type);  in steal_suitable_fallback()
1936 		account_freepages(zone, nr_added, start_type);  in steal_suitable_fallback()
1945 	if (boost_watermark(zone) && (alloc_flags & ALLOC_KSWAPD))  in steal_suitable_fallback()
1946 		set_bit(ZONE_BOOSTED_WATERMARK, &zone->flags);  in steal_suitable_fallback()
1953 	if (!prep_move_freepages_block(zone, page, &start_pfn, &free_pages,  in steal_suitable_fallback()
1984 		__move_freepages_block(zone, start_pfn, block_type, start_type);  in steal_suitable_fallback()
1985 		return __rmqueue_smallest(zone, order, start_type);  in steal_suitable_fallback()
1989 	page_del_and_expand(zone, page, order, current_order, block_type);  in steal_suitable_fallback()
2033 					 struct zone *zone)  in reserve_highatomic_pageblock()  argument
2044 	if ((zone_managed_pages(zone) / 100) < pageblock_nr_pages)  in reserve_highatomic_pageblock()
2046 	max_managed = ALIGN((zone_managed_pages(zone) / 100), pageblock_nr_pages);  in reserve_highatomic_pageblock()
2047 	if (zone->nr_reserved_highatomic >= max_managed)  in reserve_highatomic_pageblock()
2050 	spin_lock_irqsave(&zone->lock, flags);  in reserve_highatomic_pageblock()
2053 	if (zone->nr_reserved_highatomic >= max_managed)  in reserve_highatomic_pageblock()
2063 		if (move_freepages_block(zone, page, mt, MIGRATE_HIGHATOMIC) == -1)  in reserve_highatomic_pageblock()
2065 		zone->nr_reserved_highatomic += pageblock_nr_pages;  in reserve_highatomic_pageblock()
2068 		zone->nr_reserved_highatomic += 1 << order;  in reserve_highatomic_pageblock()
2072 	spin_unlock_irqrestore(&zone->lock, flags);  in reserve_highatomic_pageblock()
2090 	struct zone *zone;  in unreserve_highatomic_pageblock()  local
2095 	for_each_zone_zonelist_nodemask(zone, z, zonelist, ac->highest_zoneidx,  in unreserve_highatomic_pageblock()
2101 		if (!force && zone->nr_reserved_highatomic <=  in unreserve_highatomic_pageblock()
2105 		spin_lock_irqsave(&zone->lock, flags);  in unreserve_highatomic_pageblock()
2107 			struct free_area *area = &(zone->free_area[order]);  in unreserve_highatomic_pageblock()
2132 				size = min(size, zone->nr_reserved_highatomic);  in unreserve_highatomic_pageblock()
2133 				zone->nr_reserved_highatomic -= size;  in unreserve_highatomic_pageblock()
2146 				ret = move_freepages_block(zone, page, mt,  in unreserve_highatomic_pageblock()
2149 				move_to_free_list(page, zone, order, mt,  in unreserve_highatomic_pageblock()
2161 				spin_unlock_irqrestore(&zone->lock, flags);  in unreserve_highatomic_pageblock()
2165 		spin_unlock_irqrestore(&zone->lock, flags);  in unreserve_highatomic_pageblock()
2182 __rmqueue_fallback(struct zone *zone, int order, int start_migratetype,  in __rmqueue_fallback()  argument
2207 		area = &(zone->free_area[current_order]);  in __rmqueue_fallback()
2232 		area = &(zone->free_area[current_order]);  in __rmqueue_fallback()
2249 	page = steal_suitable_fallback(zone, page, current_order, order,  in __rmqueue_fallback()
2263 __rmqueue(struct zone *zone, unsigned int order, int migratetype,  in __rmqueue()  argument
2275 		    zone_page_state(zone, NR_FREE_CMA_PAGES) >  in __rmqueue()
2276 		    zone_page_state(zone, NR_FREE_PAGES) / 2) {  in __rmqueue()
2277 			page = __rmqueue_cma_fallback(zone, order);  in __rmqueue()
2283 	page = __rmqueue_smallest(zone, order, migratetype);  in __rmqueue()
2286 			page = __rmqueue_cma_fallback(zone, order);  in __rmqueue()
2289 			page = __rmqueue_fallback(zone, order, migratetype,  in __rmqueue()
2300 static int rmqueue_bulk(struct zone *zone, unsigned int order,  in rmqueue_bulk()  argument
2307 	spin_lock_irqsave(&zone->lock, flags);  in rmqueue_bulk()
2309 		struct page *page = __rmqueue(zone, order, migratetype,  in rmqueue_bulk()
2326 	spin_unlock_irqrestore(&zone->lock, flags);  in rmqueue_bulk()
2335 int decay_pcp_high(struct zone *zone, struct per_cpu_pages *pcp)  in decay_pcp_high()  argument
2357 		free_pcppages_bulk(zone, to_drain, pcp, 0);  in decay_pcp_high()
2371 void drain_zone_pages(struct zone *zone, struct per_cpu_pages *pcp)  in drain_zone_pages()  argument
2379 		free_pcppages_bulk(zone, to_drain, pcp, 0);  in drain_zone_pages()
2388 static void drain_pages_zone(unsigned int cpu, struct zone *zone)  in drain_pages_zone()  argument
2390 	struct per_cpu_pages *pcp = per_cpu_ptr(zone->per_cpu_pageset, cpu);  in drain_pages_zone()
2400 			free_pcppages_bulk(zone, to_drain, pcp, 0);  in drain_pages_zone()
2412 	struct zone *zone;  in drain_pages()  local
2414 	for_each_populated_zone(zone) {  in drain_pages()
2415 		drain_pages_zone(cpu, zone);  in drain_pages()
2422 void drain_local_pages(struct zone *zone)  in drain_local_pages()  argument
2426 	if (zone)  in drain_local_pages()
2427 		drain_pages_zone(cpu, zone);  in drain_local_pages()
2442 static void __drain_all_pages(struct zone *zone, bool force_all_cpus)  in __drain_all_pages()  argument
2458 		if (!zone)  in __drain_all_pages()
2471 		struct zone *z;  in __drain_all_pages()
2480 		} else if (zone) {  in __drain_all_pages()
2481 			pcp = per_cpu_ptr(zone->per_cpu_pageset, cpu);  in __drain_all_pages()
2501 		if (zone)  in __drain_all_pages()
2502 			drain_pages_zone(cpu, zone);  in __drain_all_pages()
2515 void drain_all_pages(struct zone *zone)  in drain_all_pages()  argument
2517 	__drain_all_pages(zone, false);  in drain_all_pages()
2545 static int nr_pcp_high(struct per_cpu_pages *pcp, struct zone *zone,  in nr_pcp_high()  argument
2567 	if (test_bit(ZONE_RECLAIM_ACTIVE, &zone->flags)) {  in nr_pcp_high()
2577 	if (test_bit(ZONE_BELOW_HIGH, &zone->flags)) {  in nr_pcp_high()
2593 static void free_unref_page_commit(struct zone *zone, struct per_cpu_pages *pcp,  in free_unref_page_commit()  argument
2630 	high = nr_pcp_high(pcp, zone, batch, free_high);  in free_unref_page_commit()
2632 		free_pcppages_bulk(zone, nr_pcp_free(pcp, batch, high, free_high),  in free_unref_page_commit()
2634 		if (test_bit(ZONE_BELOW_HIGH, &zone->flags) &&  in free_unref_page_commit()
2635 		    zone_watermark_ok(zone, 0, high_wmark_pages(zone),  in free_unref_page_commit()
2637 			clear_bit(ZONE_BELOW_HIGH, &zone->flags);  in free_unref_page_commit()
2648 	struct zone *zone;  in free_unref_page()  local
2676 	zone = page_zone(page);  in free_unref_page()
2678 	pcp = pcp_spin_trylock(zone->per_cpu_pageset);  in free_unref_page()
2680 		free_unref_page_commit(zone, pcp, page, migratetype, order);  in free_unref_page()
2683 		free_one_page(zone, page, pfn, order, FPI_NONE);  in free_unref_page()
2695 	struct zone *locked_zone = NULL;  in free_unref_folios()
2724 		struct zone *zone = folio_zone(folio);  in free_unref_folios()  local
2733 		if (zone != locked_zone ||  in free_unref_folios()
2747 				free_one_page(zone, &folio->page, pfn,  in free_unref_folios()
2757 			pcp = pcp_spin_trylock(zone->per_cpu_pageset);  in free_unref_folios()
2760 				free_one_page(zone, &folio->page, pfn,  in free_unref_folios()
2764 			locked_zone = zone;  in free_unref_folios()
2775 		free_unref_page_commit(zone, pcp, &folio->page, migratetype,  in free_unref_folios()
2811 	struct zone *zone = page_zone(page);  in __isolate_free_page()  local
2822 		watermark = zone->_watermark[WMARK_MIN] + (1UL << order);  in __isolate_free_page()
2823 		if (!zone_watermark_ok(zone, 0, watermark, 0, ALLOC_CMA))  in __isolate_free_page()
2827 	del_page_from_free_list(page, zone, order, mt);  in __isolate_free_page()
2842 				move_freepages_block(zone, page, mt,  in __isolate_free_page()
2861 	struct zone *zone = page_zone(page);  in __putback_isolated_page()  local
2864 	lockdep_assert_held(&zone->lock);  in __putback_isolated_page()
2867 	__free_one_page(page, page_to_pfn(page), zone, order, mt,  in __putback_isolated_page()
2874 static inline void zone_statistics(struct zone *preferred_zone, struct zone *z,  in zone_statistics()
2898 struct page *rmqueue_buddy(struct zone *preferred_zone, struct zone *zone,  in rmqueue_buddy()  argument
2907 		spin_lock_irqsave(&zone->lock, flags);  in rmqueue_buddy()
2909 			page = __rmqueue_smallest(zone, order, MIGRATE_HIGHATOMIC);  in rmqueue_buddy()
2911 			page = __rmqueue(zone, order, migratetype, alloc_flags);  in rmqueue_buddy()
2920 				page = __rmqueue_smallest(zone, order, MIGRATE_HIGHATOMIC);  in rmqueue_buddy()
2923 				spin_unlock_irqrestore(&zone->lock, flags);  in rmqueue_buddy()
2927 		spin_unlock_irqrestore(&zone->lock, flags);  in rmqueue_buddy()
2931 	zone_statistics(preferred_zone, zone, 1);  in rmqueue_buddy()
2936 static int nr_pcp_alloc(struct per_cpu_pages *pcp, struct zone *zone, int order)  in nr_pcp_alloc()  argument
2959 	if (high_min != high_max && !test_bit(ZONE_BELOW_HIGH, &zone->flags))  in nr_pcp_alloc()
2988 struct page *__rmqueue_pcplist(struct zone *zone, unsigned int order,  in __rmqueue_pcplist()  argument
2998 			int batch = nr_pcp_alloc(pcp, zone, order);  in __rmqueue_pcplist()
3001 			alloced = rmqueue_bulk(zone, order,  in __rmqueue_pcplist()
3019 static struct page *rmqueue_pcplist(struct zone *preferred_zone,  in rmqueue_pcplist()
3020 			struct zone *zone, unsigned int order,  in rmqueue_pcplist()  argument
3030 	pcp = pcp_spin_trylock(zone->per_cpu_pageset);  in rmqueue_pcplist()
3043 	page = __rmqueue_pcplist(zone, order, migratetype, alloc_flags, pcp, list);  in rmqueue_pcplist()
3048 		zone_statistics(preferred_zone, zone, 1);  in rmqueue_pcplist()
3066 struct page *rmqueue(struct zone *preferred_zone,  in rmqueue()
3067 			struct zone *zone, unsigned int order,  in rmqueue()  argument
3074 		page = rmqueue_pcplist(preferred_zone, zone, order,  in rmqueue()
3080 	page = rmqueue_buddy(preferred_zone, zone, order, alloc_flags,  in rmqueue()
3086 	    unlikely(test_bit(ZONE_BOOSTED_WATERMARK, &zone->flags))) {  in rmqueue()
3087 		clear_bit(ZONE_BOOSTED_WATERMARK, &zone->flags);  in rmqueue()
3088 		wakeup_kswapd(zone, 0, 0, zone_idx(zone));  in rmqueue()
3091 	VM_BUG_ON_PAGE(page && bad_range(zone, page), page);  in rmqueue()
3095 static inline long __zone_watermark_unusable_free(struct zone *z,  in __zone_watermark_unusable_free()
3122 bool __zone_watermark_ok(struct zone *z, unsigned int order, unsigned long mark,  in __zone_watermark_ok()
3200 bool zone_watermark_ok(struct zone *z, unsigned int order, unsigned long mark,  in zone_watermark_ok()
3207 static inline bool zone_watermark_fast(struct zone *z, unsigned int order,  in zone_watermark_fast()
3252 bool zone_watermark_ok_safe(struct zone *z, unsigned int order,  in zone_watermark_ok_safe()
3267 static bool zone_allows_reclaim(struct zone *local_zone, struct zone *zone)  in zone_allows_reclaim()  argument
3269 	return node_distance(zone_to_nid(local_zone), zone_to_nid(zone)) <=  in zone_allows_reclaim()
3273 static bool zone_allows_reclaim(struct zone *local_zone, struct zone *zone)  in zone_allows_reclaim()  argument
3288 alloc_flags_nofragment(struct zone *zone, gfp_t gfp_mask)  in alloc_flags_nofragment()  argument
3299 	if (!zone)  in alloc_flags_nofragment()
3302 	if (zone_idx(zone) != ZONE_NORMAL)  in alloc_flags_nofragment()
3311 	if (nr_online_nodes > 1 && !populated_zone(--zone))  in alloc_flags_nofragment()
3339 	struct zone *zone;  in get_page_from_freelist()  local
3351 	for_next_zone_zonelist_nodemask(zone, z, ac->highest_zoneidx,  in get_page_from_freelist()
3358 			!__cpuset_zone_allowed(zone, gfp_mask))  in get_page_from_freelist()
3380 			if (last_pgdat != zone->zone_pgdat) {  in get_page_from_freelist()
3381 				last_pgdat = zone->zone_pgdat;  in get_page_from_freelist()
3382 				last_pgdat_dirty_ok = node_dirty_ok(zone->zone_pgdat);  in get_page_from_freelist()
3390 		    zone != zonelist_zone(ac->preferred_zoneref)) {  in get_page_from_freelist()
3399 			if (zone_to_nid(zone) != local_nid) {  in get_page_from_freelist()
3405 		cond_accept_memory(zone, order);  in get_page_from_freelist()
3414 		if (test_bit(ZONE_BELOW_HIGH, &zone->flags))  in get_page_from_freelist()
3417 		mark = high_wmark_pages(zone);  in get_page_from_freelist()
3418 		if (zone_watermark_fast(zone, order, mark,  in get_page_from_freelist()
3423 			set_bit(ZONE_BELOW_HIGH, &zone->flags);  in get_page_from_freelist()
3426 		mark = wmark_pages(zone, alloc_flags & ALLOC_WMARK_MASK);  in get_page_from_freelist()
3427 		if (!zone_watermark_fast(zone, order, mark,  in get_page_from_freelist()
3432 			if (cond_accept_memory(zone, order))  in get_page_from_freelist()
3440 				if (_deferred_grow_zone(zone, order))  in get_page_from_freelist()
3449 			    !zone_allows_reclaim(zonelist_zone(ac->preferred_zoneref), zone))  in get_page_from_freelist()
3452 			ret = node_reclaim(zone->zone_pgdat, gfp_mask, order);  in get_page_from_freelist()
3462 				if (zone_watermark_ok(zone, order, mark,  in get_page_from_freelist()
3471 		page = rmqueue(zonelist_zone(ac->preferred_zoneref), zone, order,  in get_page_from_freelist()
3481 				reserve_highatomic_pageblock(page, order, zone);  in get_page_from_freelist()
3485 			if (cond_accept_memory(zone, order))  in get_page_from_freelist()
3490 				if (_deferred_grow_zone(zone, order))  in get_page_from_freelist()
3707 		struct zone *zone = page_zone(page);  in __alloc_pages_direct_compact()  local
3709 		zone->compact_blockskip_flush = false;  in __alloc_pages_direct_compact()
3710 		compaction_defer_reset(zone, order, true);  in __alloc_pages_direct_compact()
3807 	struct zone *zone;  in should_compact_retry()  local
3819 	for_each_zone_zonelist_nodemask(zone, z, ac->zonelist,  in should_compact_retry()
3821 		if (zone_watermark_ok(zone, 0, min_wmark_pages(zone),  in should_compact_retry()
3977 	struct zone *zone;  in wake_all_kswapds()  local
3981 	for_each_zone_zonelist_nodemask(zone, z, ac->zonelist, highest_zoneidx,  in wake_all_kswapds()
3983 		if (!managed_zone(zone))  in wake_all_kswapds()
3985 		if (last_pgdat != zone->zone_pgdat) {  in wake_all_kswapds()
3986 			wakeup_kswapd(zone, gfp_mask, order, highest_zoneidx);  in wake_all_kswapds()
3987 			last_pgdat = zone->zone_pgdat;  in wake_all_kswapds()
4098 	struct zone *zone;  in should_reclaim_retry()  local
4122 	for_each_zone_zonelist_nodemask(zone, z, ac->zonelist,  in should_reclaim_retry()
4126 		unsigned long min_wmark = min_wmark_pages(zone);  in should_reclaim_retry()
4131 			!__cpuset_zone_allowed(zone, gfp_mask))  in should_reclaim_retry()
4134 		available = reclaimable = zone_reclaimable_pages(zone);  in should_reclaim_retry()
4135 		available += zone_page_state_snapshot(zone, NR_FREE_PAGES);  in should_reclaim_retry()
4141 		wmark = __zone_watermark_ok(zone, order, min_wmark,  in should_reclaim_retry()
4558 	struct zone *zone;  in alloc_pages_bulk_noprof()  local
4611 	for_next_zone_zonelist_nodemask(zone, z, ac.highest_zoneidx, ac.nodemask) {  in alloc_pages_bulk_noprof()
4615 		    !__cpuset_zone_allowed(zone, gfp)) {  in alloc_pages_bulk_noprof()
4619 		if (nr_online_nodes > 1 && zone != zonelist_zone(ac.preferred_zoneref) &&  in alloc_pages_bulk_noprof()
4620 		    zone_to_nid(zone) != zonelist_node_idx(ac.preferred_zoneref)) {  in alloc_pages_bulk_noprof()
4624 		cond_accept_memory(zone, 0);  in alloc_pages_bulk_noprof()
4626 		mark = wmark_pages(zone, alloc_flags & ALLOC_WMARK_MASK) + nr_pages;  in alloc_pages_bulk_noprof()
4627 		if (zone_watermark_fast(zone, 0,  mark,  in alloc_pages_bulk_noprof()
4633 		if (cond_accept_memory(zone, 0))  in alloc_pages_bulk_noprof()
4638 			if (_deferred_grow_zone(zone, 0))  in alloc_pages_bulk_noprof()
4647 	if (unlikely(!zone))  in alloc_pages_bulk_noprof()
4652 	pcp = pcp_spin_trylock(zone->per_cpu_pageset);  in alloc_pages_bulk_noprof()
4666 		page = __rmqueue_pcplist(zone, 0, ac.migratetype, alloc_flags,  in alloc_pages_bulk_noprof()
4689 	__count_zid_vm_events(PGALLOC, zone_idx(zone), nr_account);  in alloc_pages_bulk_noprof()
4690 	zone_statistics(zonelist_zone(ac.preferred_zoneref), zone, nr_account);  in alloc_pages_bulk_noprof()
5102 	struct zone *zone;  in nr_free_zone_pages()  local
5109 	for_each_zone_zonelist(zone, z, zonelist, offset) {  in nr_free_zone_pages()
5110 		unsigned long size = zone_managed_pages(zone);  in nr_free_zone_pages()
5111 		unsigned long high = high_wmark_pages(zone);  in nr_free_zone_pages()
5134 static void zoneref_set_zone(struct zone *zone, struct zoneref *zoneref)  in zoneref_set_zone()  argument
5136 	zoneref->zone = zone;  in zoneref_set_zone()
5137 	zoneref->zone_idx = zone_idx(zone);  in zoneref_set_zone()
5147 	struct zone *zone;  in build_zonerefs_node()  local
5153 		zone = pgdat->node_zones + zone_type;  in build_zonerefs_node()
5154 		if (populated_zone(zone)) {  in build_zonerefs_node()
5155 			zoneref_set_zone(zone, &zonerefs[nr_zones++]);  in build_zonerefs_node()
5279 	zonerefs->zone = NULL;  in build_zonelists_in_node_order()
5294 	zonerefs->zone = NULL;  in build_thisnode_zonelists()
5370 	zonerefs->zone = NULL;  in build_zonelists()
5523 static int zone_batchsize(struct zone *zone)  in zone_batchsize()  argument
5534 	batch = min(zone_managed_pages(zone) >> 10, SZ_1M / PAGE_SIZE);  in zone_batchsize()
5572 static int zone_highsize(struct zone *zone, int batch, int cpu_online,  in zone_highsize()  argument
5586 		total_pages = low_wmark_pages(zone);  in zone_highsize()
5593 		total_pages = zone_managed_pages(zone) / high_fraction;  in zone_highsize()
5604 	nr_split_cpus = cpumask_weight(cpumask_of_node(zone_to_nid(zone))) + cpu_online;  in zone_highsize()
5669 static void __zone_set_pageset_high_and_batch(struct zone *zone, unsigned long high_min,  in __zone_set_pageset_high_and_batch()  argument
5676 		pcp = per_cpu_ptr(zone->per_cpu_pageset, cpu);  in __zone_set_pageset_high_and_batch()
5685 static void zone_set_pageset_high_and_batch(struct zone *zone, int cpu_online)  in zone_set_pageset_high_and_batch()  argument
5689 	new_batch = max(1, zone_batchsize(zone));  in zone_set_pageset_high_and_batch()
5691 		new_high_min = zone_highsize(zone, new_batch, cpu_online,  in zone_set_pageset_high_and_batch()
5699 		new_high_min = zone_highsize(zone, new_batch, cpu_online, 0);  in zone_set_pageset_high_and_batch()
5700 		new_high_max = zone_highsize(zone, new_batch, cpu_online,  in zone_set_pageset_high_and_batch()
5704 	if (zone->pageset_high_min == new_high_min &&  in zone_set_pageset_high_and_batch()
5705 	    zone->pageset_high_max == new_high_max &&  in zone_set_pageset_high_and_batch()
5706 	    zone->pageset_batch == new_batch)  in zone_set_pageset_high_and_batch()
5709 	zone->pageset_high_min = new_high_min;  in zone_set_pageset_high_and_batch()
5710 	zone->pageset_high_max = new_high_max;  in zone_set_pageset_high_and_batch()
5711 	zone->pageset_batch = new_batch;  in zone_set_pageset_high_and_batch()
5713 	__zone_set_pageset_high_and_batch(zone, new_high_min, new_high_max,  in zone_set_pageset_high_and_batch()
5717 void __meminit setup_zone_pageset(struct zone *zone)  in setup_zone_pageset()  argument
5723 		zone->per_cpu_zonestats = alloc_percpu(struct per_cpu_zonestat);  in setup_zone_pageset()
5725 	zone->per_cpu_pageset = alloc_percpu(struct per_cpu_pages);  in setup_zone_pageset()
5730 		pcp = per_cpu_ptr(zone->per_cpu_pageset, cpu);  in setup_zone_pageset()
5731 		pzstats = per_cpu_ptr(zone->per_cpu_zonestats, cpu);  in setup_zone_pageset()
5735 	zone_set_pageset_high_and_batch(zone, 0);  in setup_zone_pageset()
5742 static void zone_pcp_update(struct zone *zone, int cpu_online)  in zone_pcp_update()  argument
5745 	zone_set_pageset_high_and_batch(zone, cpu_online);  in zone_pcp_update()
5749 static void zone_pcp_update_cacheinfo(struct zone *zone, unsigned int cpu)  in zone_pcp_update_cacheinfo()  argument
5754 	pcp = per_cpu_ptr(zone->per_cpu_pageset, cpu);  in zone_pcp_update_cacheinfo()
5773 	struct zone *zone;  in setup_pcp_cacheinfo()  local
5775 	for_each_populated_zone(zone)  in setup_pcp_cacheinfo()
5776 		zone_pcp_update_cacheinfo(zone, cpu);  in setup_pcp_cacheinfo()
5786 	struct zone *zone;  in setup_per_cpu_pageset()  local
5789 	for_each_populated_zone(zone)  in setup_per_cpu_pageset()
5790 		setup_zone_pageset(zone);  in setup_per_cpu_pageset()
5811 __meminit void zone_pcp_init(struct zone *zone)  in zone_pcp_init()  argument
5818 	zone->per_cpu_pageset = &boot_pageset;  in zone_pcp_init()
5819 	zone->per_cpu_zonestats = &boot_zonestats;  in zone_pcp_init()
5820 	zone->pageset_high_min = BOOT_PAGESET_HIGH;  in zone_pcp_init()
5821 	zone->pageset_high_max = BOOT_PAGESET_HIGH;  in zone_pcp_init()
5822 	zone->pageset_batch = BOOT_PAGESET_BATCH;  in zone_pcp_init()
5824 	if (populated_zone(zone))  in zone_pcp_init()
5825 		pr_debug("  %s zone: %lu pages, LIFO batch:%u\n", zone->name,  in zone_pcp_init()
5826 			 zone->present_pages, zone_batchsize(zone));  in zone_pcp_init()
5884 	struct zone *zone;  in page_alloc_cpu_dead()  local
5907 	for_each_populated_zone(zone)  in page_alloc_cpu_dead()
5908 		zone_pcp_update(zone, 0);  in page_alloc_cpu_dead()
5915 	struct zone *zone;  in page_alloc_cpu_online()  local
5917 	for_each_populated_zone(zone)  in page_alloc_cpu_online()
5918 		zone_pcp_update(zone, 1);  in page_alloc_cpu_online()
5948 			struct zone *zone = pgdat->node_zones + i;  in calculate_totalreserve_pages()  local
5950 			unsigned long managed_pages = zone_managed_pages(zone);  in calculate_totalreserve_pages()
5954 				if (zone->lowmem_reserve[j] > max)  in calculate_totalreserve_pages()
5955 					max = zone->lowmem_reserve[j];  in calculate_totalreserve_pages()
5959 			max += high_wmark_pages(zone);  in calculate_totalreserve_pages()
5985 			struct zone *zone = &pgdat->node_zones[i];  in setup_per_zone_lowmem_reserve()  local
5987 			bool clear = !ratio || !zone_managed_pages(zone);  in setup_per_zone_lowmem_reserve()
5991 				struct zone *upper_zone = &pgdat->node_zones[j];  in setup_per_zone_lowmem_reserve()
5997 					zone->lowmem_reserve[j] = 0;  in setup_per_zone_lowmem_reserve()
5999 					zone->lowmem_reserve[j] = managed_pages / ratio;  in setup_per_zone_lowmem_reserve()
6012 	struct zone *zone;  in __setup_per_zone_wmarks()  local
6016 	for_each_zone(zone) {  in __setup_per_zone_wmarks()
6017 		if (!is_highmem(zone) && zone_idx(zone) != ZONE_MOVABLE)  in __setup_per_zone_wmarks()
6018 			lowmem_pages += zone_managed_pages(zone);  in __setup_per_zone_wmarks()
6021 	for_each_zone(zone) {  in __setup_per_zone_wmarks()
6024 		spin_lock_irqsave(&zone->lock, flags);  in __setup_per_zone_wmarks()
6025 		tmp = (u64)pages_min * zone_managed_pages(zone);  in __setup_per_zone_wmarks()
6027 		if (is_highmem(zone) || zone_idx(zone) == ZONE_MOVABLE) {  in __setup_per_zone_wmarks()
6039 			min_pages = zone_managed_pages(zone) / 1024;  in __setup_per_zone_wmarks()
6041 			zone->_watermark[WMARK_MIN] = min_pages;  in __setup_per_zone_wmarks()
6047 			zone->_watermark[WMARK_MIN] = tmp;  in __setup_per_zone_wmarks()
6056 			    mult_frac(zone_managed_pages(zone),  in __setup_per_zone_wmarks()
6059 		zone->watermark_boost = 0;  in __setup_per_zone_wmarks()
6060 		zone->_watermark[WMARK_LOW]  = min_wmark_pages(zone) + tmp;  in __setup_per_zone_wmarks()
6061 		zone->_watermark[WMARK_HIGH] = low_wmark_pages(zone) + tmp;  in __setup_per_zone_wmarks()
6062 		zone->_watermark[WMARK_PROMO] = high_wmark_pages(zone) + tmp;  in __setup_per_zone_wmarks()
6064 		spin_unlock_irqrestore(&zone->lock, flags);  in __setup_per_zone_wmarks()
6080 	struct zone *zone;  in setup_per_zone_wmarks()  local
6091 	for_each_zone(zone)  in setup_per_zone_wmarks()
6092 		zone_pcp_update(zone, 0);  in setup_per_zone_wmarks()
6193 	struct zone *zone;  in setup_min_unmapped_ratio()  local
6198 	for_each_zone(zone)  in setup_min_unmapped_ratio()
6199 		zone->zone_pgdat->min_unmapped_pages += (zone_managed_pages(zone) *  in setup_min_unmapped_ratio()
6221 	struct zone *zone;  in setup_min_slab_ratio()  local
6226 	for_each_zone(zone)  in setup_min_slab_ratio()
6227 		zone->zone_pgdat->min_slab_pages += (zone_managed_pages(zone) *  in setup_min_slab_ratio()
6279 	struct zone *zone;  in percpu_pagelist_high_fraction_sysctl_handler()  local
6302 	for_each_populated_zone(zone)  in percpu_pagelist_high_fraction_sysctl_handler()
6303 		zone_set_pageset_high_and_batch(zone, 0);  in percpu_pagelist_high_fraction_sysctl_handler()
6414 		.nid = zone_to_nid(cc->zone),  in __alloc_contig_migrate_range()
6443 		nr_reclaimed = reclaim_clean_pages_from_list(cc->zone,  in __alloc_contig_migrate_range()
6540 		.zone = page_zone(pfn_to_page(start)),  in alloc_contig_range_noprof()
6574 	drain_all_pages(cc.zone);  in alloc_contig_range_noprof()
6656 static bool pfn_range_valid_contig(struct zone *z, unsigned long start_pfn,  in pfn_range_valid_contig()
6679 static bool zone_spans_last_pfn(const struct zone *zone,  in zone_spans_last_pfn()  argument
6684 	return zone_spans_pfn(zone, last_pfn);  in zone_spans_last_pfn()
6713 	struct zone *zone;  in alloc_contig_pages_noprof()  local
6717 	for_each_zone_zonelist_nodemask(zone, z, zonelist,  in alloc_contig_pages_noprof()
6719 		spin_lock_irqsave(&zone->lock, flags);  in alloc_contig_pages_noprof()
6721 		pfn = ALIGN(zone->zone_start_pfn, nr_pages);  in alloc_contig_pages_noprof()
6722 		while (zone_spans_last_pfn(zone, pfn, nr_pages)) {  in alloc_contig_pages_noprof()
6723 			if (pfn_range_valid_contig(zone, pfn, nr_pages)) {  in alloc_contig_pages_noprof()
6731 				spin_unlock_irqrestore(&zone->lock, flags);  in alloc_contig_pages_noprof()
6736 				spin_lock_irqsave(&zone->lock, flags);  in alloc_contig_pages_noprof()
6740 		spin_unlock_irqrestore(&zone->lock, flags);  in alloc_contig_pages_noprof()
6780 void zone_pcp_disable(struct zone *zone)  in zone_pcp_disable()  argument
6783 	__zone_set_pageset_high_and_batch(zone, 0, 0, 1);  in zone_pcp_disable()
6784 	__drain_all_pages(zone, true);  in zone_pcp_disable()
6787 void zone_pcp_enable(struct zone *zone)  in zone_pcp_enable()  argument
6789 	__zone_set_pageset_high_and_batch(zone, zone->pageset_high_min,  in zone_pcp_enable()
6790 		zone->pageset_high_max, zone->pageset_batch);  in zone_pcp_enable()
6794 void zone_pcp_reset(struct zone *zone)  in zone_pcp_reset()  argument
6799 	if (zone->per_cpu_pageset != &boot_pageset) {  in zone_pcp_reset()
6801 			pzstats = per_cpu_ptr(zone->per_cpu_zonestats, cpu);  in zone_pcp_reset()
6802 			drain_zonestat(zone, pzstats);  in zone_pcp_reset()
6804 		free_percpu(zone->per_cpu_pageset);  in zone_pcp_reset()
6805 		zone->per_cpu_pageset = &boot_pageset;  in zone_pcp_reset()
6806 		if (zone->per_cpu_zonestats != &boot_zonestats) {  in zone_pcp_reset()
6807 			free_percpu(zone->per_cpu_zonestats);  in zone_pcp_reset()
6808 			zone->per_cpu_zonestats = &boot_zonestats;  in zone_pcp_reset()
6828 	struct zone *zone;  in __offline_isolated_pages()  local
6832 	zone = page_zone(pfn_to_page(pfn));  in __offline_isolated_pages()
6833 	spin_lock_irqsave(&zone->lock, flags);  in __offline_isolated_pages()
6860 		del_page_from_free_list(page, zone, order, MIGRATE_ISOLATE);  in __offline_isolated_pages()
6863 	spin_unlock_irqrestore(&zone->lock, flags);  in __offline_isolated_pages()
6890 static inline void add_to_free_list(struct page *page, struct zone *zone,  in add_to_free_list()  argument
6894 	__add_to_free_list(page, zone, order, migratetype, tail);  in add_to_free_list()
6895 	account_freepages(zone, 1 << order, migratetype);  in add_to_free_list()
6902 static void break_down_buddy_pages(struct zone *zone, struct page *page,  in break_down_buddy_pages()  argument
6920 		if (set_page_guard(zone, current_buddy, high))  in break_down_buddy_pages()
6923 		add_to_free_list(current_buddy, zone, high, migratetype, false);  in break_down_buddy_pages()
6933 	struct zone *zone = page_zone(page);  in take_page_off_buddy()  local
6939 	spin_lock_irqsave(&zone->lock, flags);  in take_page_off_buddy()
6949 			del_page_from_free_list(page_head, zone, page_order,  in take_page_off_buddy()
6951 			break_down_buddy_pages(zone, page_head, page, 0,  in take_page_off_buddy()
6960 	spin_unlock_irqrestore(&zone->lock, flags);  in take_page_off_buddy()
6969 	struct zone *zone = page_zone(page);  in put_page_back_buddy()  local
6973 	spin_lock_irqsave(&zone->lock, flags);  in put_page_back_buddy()
6979 		__free_one_page(page, pfn, zone, 0, migratetype, FPI_NONE);  in put_page_back_buddy()
6984 	spin_unlock_irqrestore(&zone->lock, flags);  in put_page_back_buddy()
6996 		struct zone *zone = &pgdat->node_zones[ZONE_DMA];  in has_managed_dma()  local
6998 		if (managed_zone(zone))  in has_managed_dma()
7033 static void __accept_page(struct zone *zone, unsigned long *flags,  in __accept_page()  argument
7039 	last = list_empty(&zone->unaccepted_pages);  in __accept_page()
7041 	account_freepages(zone, -MAX_ORDER_NR_PAGES, MIGRATE_MOVABLE);  in __accept_page()
7042 	__mod_zone_page_state(zone, NR_UNACCEPTED, -MAX_ORDER_NR_PAGES);  in __accept_page()
7044 	spin_unlock_irqrestore(&zone->lock, *flags);  in __accept_page()
7056 	struct zone *zone = page_zone(page);  in accept_page()  local
7059 	spin_lock_irqsave(&zone->lock, flags);  in accept_page()
7061 		spin_unlock_irqrestore(&zone->lock, flags);  in accept_page()
7066 	__accept_page(zone, &flags, page);  in accept_page()
7069 static bool try_to_accept_memory_one(struct zone *zone)  in try_to_accept_memory_one()  argument
7074 	spin_lock_irqsave(&zone->lock, flags);  in try_to_accept_memory_one()
7075 	page = list_first_entry_or_null(&zone->unaccepted_pages,  in try_to_accept_memory_one()
7078 		spin_unlock_irqrestore(&zone->lock, flags);  in try_to_accept_memory_one()
7083 	__accept_page(zone, &flags, page);  in try_to_accept_memory_one()
7093 static bool cond_accept_memory(struct zone *zone, unsigned int order)  in cond_accept_memory()  argument
7101 	if (list_empty(&zone->unaccepted_pages))  in cond_accept_memory()
7105 	to_accept = promo_wmark_pages(zone) -  in cond_accept_memory()
7106 		    (zone_page_state(zone, NR_FREE_PAGES) -  in cond_accept_memory()
7107 		    __zone_watermark_unusable_free(zone, order, 0) -  in cond_accept_memory()
7108 		    zone_page_state(zone, NR_UNACCEPTED));  in cond_accept_memory()
7111 		if (!try_to_accept_memory_one(zone))  in cond_accept_memory()
7122 	struct zone *zone = page_zone(page);  in __free_unaccepted()  local
7129 	spin_lock_irqsave(&zone->lock, flags);  in __free_unaccepted()
7130 	first = list_empty(&zone->unaccepted_pages);  in __free_unaccepted()
7131 	list_add_tail(&page->lru, &zone->unaccepted_pages);  in __free_unaccepted()
7132 	account_freepages(zone, MAX_ORDER_NR_PAGES, MIGRATE_MOVABLE);  in __free_unaccepted()
7133 	__mod_zone_page_state(zone, NR_UNACCEPTED, MAX_ORDER_NR_PAGES);  in __free_unaccepted()
7135 	spin_unlock_irqrestore(&zone->lock, flags);  in __free_unaccepted()
7150 static bool cond_accept_memory(struct zone *zone, unsigned int order)  in cond_accept_memory()  argument