38 static void zero_zone_numa_counters(struct zone *zone)  in zero_zone_numa_counters()  argument
43 		atomic_long_set(&zone->vm_numa_event[item], 0);  in zero_zone_numa_counters()
45 			per_cpu_ptr(zone->per_cpu_zonestats, cpu)->vm_numa_event[item]  in zero_zone_numa_counters()
54 	struct zone *zone;  in zero_zones_numa_counters()  local
56 	for_each_populated_zone(zone)  in zero_zones_numa_counters()
57 		zero_zone_numa_counters(zone);  in zero_zones_numa_counters()
169 static void fold_vm_zone_numa_events(struct zone *zone)  in fold_vm_zone_numa_events()  argument
178 		pzstats = per_cpu_ptr(zone->per_cpu_zonestats, cpu);  in fold_vm_zone_numa_events()
184 		zone_numa_event_add(zone_numa_events[item], zone, item);  in fold_vm_zone_numa_events()
189 	struct zone *zone;  in fold_vm_numa_events()  local
191 	for_each_populated_zone(zone)  in fold_vm_numa_events()
192 		fold_vm_zone_numa_events(zone);  in fold_vm_numa_events()
198 int calculate_pressure_threshold(struct zone *zone)  in calculate_pressure_threshold()  argument
211 	watermark_distance = low_wmark_pages(zone) - min_wmark_pages(zone);  in calculate_pressure_threshold()
222 int calculate_normal_threshold(struct zone *zone)  in calculate_normal_threshold()  argument
257 	mem = zone_managed_pages(zone) >> (27 - PAGE_SHIFT);  in calculate_normal_threshold()
275 	struct zone *zone;  in refresh_zone_stat_thresholds()  local
286 	for_each_populated_zone(zone) {  in refresh_zone_stat_thresholds()
287 		struct pglist_data *pgdat = zone->zone_pgdat;  in refresh_zone_stat_thresholds()
290 		threshold = calculate_normal_threshold(zone);  in refresh_zone_stat_thresholds()
295 			per_cpu_ptr(zone->per_cpu_zonestats, cpu)->stat_threshold  in refresh_zone_stat_thresholds()
309 		tolerate_drift = low_wmark_pages(zone) - min_wmark_pages(zone);  in refresh_zone_stat_thresholds()
312 			zone->percpu_drift_mark = high_wmark_pages(zone) +  in refresh_zone_stat_thresholds()
318 				int (*calculate_pressure)(struct zone *))  in set_pgdat_percpu_threshold()  argument
320 	struct zone *zone;  in set_pgdat_percpu_threshold()  local
326 		zone = &pgdat->node_zones[i];  in set_pgdat_percpu_threshold()
327 		if (!zone->percpu_drift_mark)  in set_pgdat_percpu_threshold()
330 		threshold = (*calculate_pressure)(zone);  in set_pgdat_percpu_threshold()
332 			per_cpu_ptr(zone->per_cpu_zonestats, cpu)->stat_threshold  in set_pgdat_percpu_threshold()
342 void __mod_zone_page_state(struct zone *zone, enum zone_stat_item item,  in __mod_zone_page_state()  argument
345 	struct per_cpu_zonestat __percpu *pcp = zone->per_cpu_zonestats;  in __mod_zone_page_state()
364 		zone_page_state_add(x, zone, item);  in __mod_zone_page_state()
432 void __inc_zone_state(struct zone *zone, enum zone_stat_item item)  in __inc_zone_state()  argument
434 	struct per_cpu_zonestat __percpu *pcp = zone->per_cpu_zonestats;  in __inc_zone_state()
446 		zone_page_state_add(v + overstep, zone, item);  in __inc_zone_state()
488 void __dec_zone_state(struct zone *zone, enum zone_stat_item item)  in __dec_zone_state()  argument
490 	struct per_cpu_zonestat __percpu *pcp = zone->per_cpu_zonestats;  in __dec_zone_state()
502 		zone_page_state_add(v - overstep, zone, item);  in __dec_zone_state()
557 static inline void mod_zone_state(struct zone *zone,  in mod_zone_state()  argument
560 	struct per_cpu_zonestat __percpu *pcp = zone->per_cpu_zonestats;  in mod_zone_state()
593 		zone_page_state_add(z, zone, item);  in mod_zone_state()
596 void mod_zone_page_state(struct zone *zone, enum zone_stat_item item,  in mod_zone_page_state()  argument
599 	mod_zone_state(zone, item, delta, 0);  in mod_zone_page_state()
692 void mod_zone_page_state(struct zone *zone, enum zone_stat_item item,  in mod_zone_page_state()  argument
698 	__mod_zone_page_state(zone, item, delta);  in mod_zone_page_state()
706 	struct zone *zone;  in inc_zone_page_state()  local
708 	zone = page_zone(page);  in inc_zone_page_state()
710 	__inc_zone_state(zone, item);  in inc_zone_page_state()
811 	struct zone *zone;  in refresh_cpu_vm_stats()  local
817 	for_each_populated_zone(zone) {  in refresh_cpu_vm_stats()
818 		struct per_cpu_zonestat __percpu *pzstats = zone->per_cpu_zonestats;  in refresh_cpu_vm_stats()
819 		struct per_cpu_pages __percpu *pcp = zone->per_cpu_pageset;  in refresh_cpu_vm_stats()
827 				atomic_long_add(v, &zone->vm_stat[i]);  in refresh_cpu_vm_stats()
839 			changes += decay_pcp_high(zone, this_cpu_ptr(pcp));  in refresh_cpu_vm_stats()
855 			if (zone_to_nid(zone) == numa_node_id()) {  in refresh_cpu_vm_stats()
866 				drain_zone_pages(zone, this_cpu_ptr(pcp));  in refresh_cpu_vm_stats()
899 	struct zone *zone;  in cpu_vm_stats_fold()  local
904 	for_each_populated_zone(zone) {  in cpu_vm_stats_fold()
907 		pzstats = per_cpu_ptr(zone->per_cpu_zonestats, cpu);  in cpu_vm_stats_fold()
915 				atomic_long_add(v, &zone->vm_stat[i]);  in cpu_vm_stats_fold()
926 				zone_numa_event_add(v, zone, i);  in cpu_vm_stats_fold()
955 void drain_zonestat(struct zone *zone, struct per_cpu_zonestat *pzstats)  in drain_zonestat()  argument
964 			zone_page_state_add(v, zone, i);  in drain_zonestat()
973 			zone_numa_event_add(v, zone, i);  in drain_zonestat()
989 	struct zone *zones = NODE_DATA(node)->node_zones;  in sum_zone_node_page_state()
1003 	struct zone *zones = NODE_DATA(node)->node_zones;  in sum_zone_numa_event_state()
1070 static void fill_contig_page_info(struct zone *zone,  in fill_contig_page_info()  argument
1089 		blocks = data_race(zone->free_area[order].nr_free);  in fill_contig_page_info()
1137 unsigned int extfrag_for_order(struct zone *zone, unsigned int order)  in extfrag_for_order()  argument
1141 	fill_contig_page_info(zone, order, &info);  in extfrag_for_order()
1151 int fragmentation_index(struct zone *zone, unsigned int order)  in fragmentation_index()  argument
1155 	fill_contig_page_info(zone, order, &info);  in fragmentation_index()
1503 		void (*print)(struct seq_file *m, pg_data_t *, struct zone *))  in walk_zones_in_node()  argument
1505 	struct zone *zone;  in walk_zones_in_node()  local
1506 	struct zone *node_zones = pgdat->node_zones;  in walk_zones_in_node()
1509 	for (zone = node_zones; zone - node_zones < MAX_NR_ZONES; ++zone) {  in walk_zones_in_node()
1510 		if (assert_populated && !populated_zone(zone))  in walk_zones_in_node()
1514 			spin_lock_irqsave(&zone->lock, flags);  in walk_zones_in_node()
1515 		print(m, pgdat, zone);  in walk_zones_in_node()
1517 			spin_unlock_irqrestore(&zone->lock, flags);  in walk_zones_in_node()
1524 						struct zone *zone)  in frag_show_print()  argument
1528 	seq_printf(m, "Node %d, zone %8s ", pgdat->node_id, zone->name);  in frag_show_print()
1534 		seq_printf(m, "%6lu ", data_race(zone->free_area[order].nr_free));  in frag_show_print()
1549 					pg_data_t *pgdat, struct zone *zone)  in pagetypeinfo_showfree_print()  argument
1556 					zone->name,  in pagetypeinfo_showfree_print()
1564 			area = &(zone->free_area[order]);  in pagetypeinfo_showfree_print()
1582 			spin_unlock_irq(&zone->lock);  in pagetypeinfo_showfree_print()
1584 			spin_lock_irq(&zone->lock);  in pagetypeinfo_showfree_print()
1606 					pg_data_t *pgdat, struct zone *zone)  in pagetypeinfo_showblockcount_print()  argument
1610 	unsigned long start_pfn = zone->zone_start_pfn;  in pagetypeinfo_showblockcount_print()
1611 	unsigned long end_pfn = zone_end_pfn(zone);  in pagetypeinfo_showblockcount_print()
1621 		if (page_zone(page) != zone)  in pagetypeinfo_showblockcount_print()
1631 	seq_printf(m, "Node %d, zone %8s ", pgdat->node_id, zone->name);  in pagetypeinfo_showblockcount_print()
1713 static bool is_zone_first_populated(pg_data_t *pgdat, struct zone *zone)  in is_zone_first_populated()  argument
1718 		struct zone *compare = &pgdat->node_zones[zid];  in is_zone_first_populated()
1721 			return zone == compare;  in is_zone_first_populated()
1728 							struct zone *zone)  in zoneinfo_show_print()  argument
1731 	seq_printf(m, "Node %d, zone %8s", pgdat->node_id, zone->name);  in zoneinfo_show_print()
1732 	if (is_zone_first_populated(pgdat, zone)) {  in zoneinfo_show_print()
1754 		   zone_page_state(zone, NR_FREE_PAGES),  in zoneinfo_show_print()
1755 		   zone->watermark_boost,  in zoneinfo_show_print()
1756 		   min_wmark_pages(zone),  in zoneinfo_show_print()
1757 		   low_wmark_pages(zone),  in zoneinfo_show_print()
1758 		   high_wmark_pages(zone),  in zoneinfo_show_print()
1759 		   promo_wmark_pages(zone),  in zoneinfo_show_print()
1760 		   zone->spanned_pages,  in zoneinfo_show_print()
1761 		   zone->present_pages,  in zoneinfo_show_print()
1762 		   zone_managed_pages(zone),  in zoneinfo_show_print()
1763 		   zone_cma_pages(zone));  in zoneinfo_show_print()
1767 		   zone->lowmem_reserve[0]);  in zoneinfo_show_print()
1768 	for (i = 1; i < ARRAY_SIZE(zone->lowmem_reserve); i++)  in zoneinfo_show_print()
1769 		seq_printf(m, ", %ld", zone->lowmem_reserve[i]);  in zoneinfo_show_print()
1773 	if (!populated_zone(zone)) {  in zoneinfo_show_print()
1780 			   zone_page_state(zone, i));  in zoneinfo_show_print()
1785 			   zone_numa_event_state(zone, i));  in zoneinfo_show_print()
1793 		pcp = per_cpu_ptr(zone->per_cpu_pageset, i);  in zoneinfo_show_print()
1804 		pzstats = per_cpu_ptr(zone->per_cpu_zonestats, i);  in zoneinfo_show_print()
1813 		   zone->zone_start_pfn);  in zoneinfo_show_print()
2021 	struct zone *zone;  in need_update()  local
2023 	for_each_populated_zone(zone) {  in need_update()
2024 		struct per_cpu_zonestat *pzstats = per_cpu_ptr(zone->per_cpu_zonestats, cpu);  in need_update()
2033 		if (last_pgdat == zone->zone_pgdat)  in need_update()
2035 		last_pgdat = zone->zone_pgdat;  in need_update()
2036 		n = per_cpu_ptr(zone->zone_pgdat->per_cpu_nodestats, cpu);  in need_update()
2229 					pg_data_t *pgdat, struct zone *zone)  in unusable_show_print()  argument
2237 				zone->name);  in unusable_show_print()
2239 		fill_contig_page_info(zone, order, &info);  in unusable_show_print()
2279 					pg_data_t *pgdat, struct zone *zone)  in extfrag_show_print()  argument
2289 				zone->name);  in extfrag_show_print()
2291 		fill_contig_page_info(zone, order, &info);  in extfrag_show_print()