158 static void defer_compaction(struct zone *zone, int order)  in defer_compaction()  argument
160 	zone->compact_considered = 0;  in defer_compaction()
161 	zone->compact_defer_shift++;  in defer_compaction()
163 	if (order < zone->compact_order_failed)  in defer_compaction()
164 		zone->compact_order_failed = order;  in defer_compaction()
166 	if (zone->compact_defer_shift > COMPACT_MAX_DEFER_SHIFT)  in defer_compaction()
167 		zone->compact_defer_shift = COMPACT_MAX_DEFER_SHIFT;  in defer_compaction()
169 	trace_mm_compaction_defer_compaction(zone, order);  in defer_compaction()
173 static bool compaction_deferred(struct zone *zone, int order)  in compaction_deferred()  argument
175 	unsigned long defer_limit = 1UL << zone->compact_defer_shift;  in compaction_deferred()
177 	if (order < zone->compact_order_failed)  in compaction_deferred()
181 	if (++zone->compact_considered >= defer_limit) {  in compaction_deferred()
182 		zone->compact_considered = defer_limit;  in compaction_deferred()
186 	trace_mm_compaction_deferred(zone, order);  in compaction_deferred()
196 void compaction_defer_reset(struct zone *zone, int order,  in compaction_defer_reset()  argument
200 		zone->compact_considered = 0;  in compaction_defer_reset()
201 		zone->compact_defer_shift = 0;  in compaction_defer_reset()
203 	if (order >= zone->compact_order_failed)  in compaction_defer_reset()
204 		zone->compact_order_failed = order + 1;  in compaction_defer_reset()
206 	trace_mm_compaction_defer_reset(zone, order);  in compaction_defer_reset()
210 static bool compaction_restarting(struct zone *zone, int order)  in compaction_restarting()  argument
212 	if (order < zone->compact_order_failed)  in compaction_restarting()
215 	return zone->compact_defer_shift == COMPACT_MAX_DEFER_SHIFT &&  in compaction_restarting()
216 		zone->compact_considered >= 1UL << zone->compact_defer_shift;  in compaction_restarting()
229 static void reset_cached_positions(struct zone *zone)  in reset_cached_positions()  argument
231 	zone->compact_cached_migrate_pfn[0] = zone->zone_start_pfn;  in reset_cached_positions()
232 	zone->compact_cached_migrate_pfn[1] = zone->zone_start_pfn;  in reset_cached_positions()
233 	zone->compact_cached_free_pfn =  in reset_cached_positions()
234 				pageblock_start_pfn(zone_end_pfn(zone) - 1);  in reset_cached_positions()
308 __reset_isolation_pfn(struct zone *zone, unsigned long pfn, bool check_source,  in __reset_isolation_pfn()  argument
318 	if (zone != page_zone(page))  in __reset_isolation_pfn()
340 	block_pfn = max(block_pfn, zone->zone_start_pfn);  in __reset_isolation_pfn()
349 	block_pfn = min(block_pfn, zone_end_pfn(zone) - 1);  in __reset_isolation_pfn()
381 static void __reset_isolation_suitable(struct zone *zone)  in __reset_isolation_suitable()  argument
383 	unsigned long migrate_pfn = zone->zone_start_pfn;  in __reset_isolation_suitable()
384 	unsigned long free_pfn = zone_end_pfn(zone) - 1;  in __reset_isolation_suitable()
391 	if (!zone->compact_blockskip_flush)  in __reset_isolation_suitable()
394 	zone->compact_blockskip_flush = false;  in __reset_isolation_suitable()
407 		if (__reset_isolation_pfn(zone, migrate_pfn, true, source_set) &&  in __reset_isolation_suitable()
411 			zone->compact_init_migrate_pfn = reset_migrate;  in __reset_isolation_suitable()
412 			zone->compact_cached_migrate_pfn[0] = reset_migrate;  in __reset_isolation_suitable()
413 			zone->compact_cached_migrate_pfn[1] = reset_migrate;  in __reset_isolation_suitable()
417 		if (__reset_isolation_pfn(zone, free_pfn, free_set, true) &&  in __reset_isolation_suitable()
421 			zone->compact_init_free_pfn = reset_free;  in __reset_isolation_suitable()
422 			zone->compact_cached_free_pfn = reset_free;  in __reset_isolation_suitable()
428 		zone->compact_cached_migrate_pfn[0] = migrate_pfn;  in __reset_isolation_suitable()
429 		zone->compact_cached_migrate_pfn[1] = migrate_pfn;  in __reset_isolation_suitable()
430 		zone->compact_cached_free_pfn = free_pfn;  in __reset_isolation_suitable()
439 		struct zone *zone = &pgdat->node_zones[zoneid];  in reset_isolation_suitable()  local
440 		if (!populated_zone(zone))  in reset_isolation_suitable()
443 		__reset_isolation_suitable(zone);  in reset_isolation_suitable()
468 	struct zone *zone = cc->zone;  in update_cached_migrate()  local
477 	if (pfn > zone->compact_cached_migrate_pfn[0])  in update_cached_migrate()
478 		zone->compact_cached_migrate_pfn[0] = pfn;  in update_cached_migrate()
480 	    pfn > zone->compact_cached_migrate_pfn[1])  in update_cached_migrate()
481 		zone->compact_cached_migrate_pfn[1] = pfn;  in update_cached_migrate()
491 	struct zone *zone = cc->zone;  in update_pageblock_skip()  local
498 	if (pfn < zone->compact_cached_free_pfn)  in update_pageblock_skip()
499 		zone->compact_cached_free_pfn = pfn;  in update_pageblock_skip()
618 		    && compact_unlock_should_abort(&cc->zone->lock, flags,  in isolate_freepages_block()
647 			locked = compact_lock_irqsave(&cc->zone->lock,  in isolate_freepages_block()
683 		spin_unlock_irqrestore(&cc->zone->lock, flags);  in isolate_freepages_block()
737 	if (block_start_pfn < cc->zone->zone_start_pfn)  in isolate_freepages_range()
738 		block_start_pfn = cc->zone->zone_start_pfn;  in isolate_freepages_range()
760 					block_end_pfn, cc->zone))  in isolate_freepages_range()
794 	pg_data_t *pgdat = cc->zone->zone_pgdat;  in too_many_isolated()
871 	pg_data_t *pgdat = cc->zone->zone_pgdat;  in isolate_migratepages_block()
972 				    low_pfn == cc->zone->zone_start_pfn)) {  in isolate_migratepages_block()
1363 	if (block_start_pfn < cc->zone->zone_start_pfn)  in isolate_migratepages_range()
1364 		block_start_pfn = cc->zone->zone_start_pfn;  in isolate_migratepages_range()
1374 					block_end_pfn, cc->zone))  in isolate_migratepages_range()
1506 	start_pfn = max(pageblock_start_pfn(pfn), cc->zone->zone_start_pfn);  in fast_isolate_around()
1507 	end_pfn = min(pageblock_end_pfn(pfn), zone_end_pfn(cc->zone));  in fast_isolate_around()
1509 	page = pageblock_pfn_to_page(start_pfn, end_pfn, cc->zone);  in fast_isolate_around()
1557 	if (cc->free_pfn >= cc->zone->compact_init_free_pfn) {  in fast_isolate_freepages()
1582 		struct free_area *area = &cc->zone->free_area[order];  in fast_isolate_freepages()
1592 		spin_lock_irqsave(&cc->zone->lock, flags);  in fast_isolate_freepages()
1603 					      cc->zone->zone_start_pfn);  in fast_isolate_freepages()
1651 		spin_unlock_irqrestore(&cc->zone->lock, flags);  in fast_isolate_freepages()
1683 						    zone_end_pfn(cc->zone)),  in fast_isolate_freepages()
1684 						cc->zone);  in fast_isolate_freepages()
1694 	if (highest && highest >= cc->zone->compact_cached_free_pfn) {  in fast_isolate_freepages()
1696 		cc->zone->compact_cached_free_pfn = highest;  in fast_isolate_freepages()
1713 	struct zone *zone = cc->zone;  in isolate_freepages()  local
1740 						zone_end_pfn(zone));  in isolate_freepages()
1763 									zone);  in isolate_freepages()
1982 	if (pfn != cc->zone->zone_start_pfn && pfn != pageblock_start_pfn(pfn))  in fast_find_migrateblock()
2009 	if (cc->migrate_pfn != cc->zone->zone_start_pfn)  in fast_find_migrateblock()
2016 		struct free_area *area = &cc->zone->free_area[order];  in fast_find_migrateblock()
2024 		spin_lock_irqsave(&cc->zone->lock, flags);  in fast_find_migrateblock()
2050 				if (pfn < cc->zone->zone_start_pfn)  in fast_find_migrateblock()
2051 					pfn = cc->zone->zone_start_pfn;  in fast_find_migrateblock()
2057 		spin_unlock_irqrestore(&cc->zone->lock, flags);  in fast_find_migrateblock()
2096 	if (block_start_pfn < cc->zone->zone_start_pfn)  in isolate_migratepages()
2097 		block_start_pfn = cc->zone->zone_start_pfn;  in isolate_migratepages()
2128 						block_end_pfn, cc->zone);  in isolate_migratepages()
2146 		     low_pfn == cc->zone->zone_start_pfn) &&  in isolate_migratepages()
2200 static unsigned int fragmentation_score_zone(struct zone *zone)  in fragmentation_score_zone()  argument
2202 	return extfrag_for_order(zone, COMPACTION_HPAGE_ORDER);  in fragmentation_score_zone()
2215 static unsigned int fragmentation_score_zone_weighted(struct zone *zone)  in fragmentation_score_zone_weighted()  argument
2219 	score = zone->present_pages * fragmentation_score_zone(zone);  in fragmentation_score_zone_weighted()
2220 	return div64_ul(score, zone->zone_pgdat->node_present_pages + 1);  in fragmentation_score_zone_weighted()
2236 		struct zone *zone;  in fragmentation_score_node()  local
2238 		zone = &pgdat->node_zones[zoneid];  in fragmentation_score_node()
2239 		if (!populated_zone(zone))  in fragmentation_score_node()
2241 		score += fragmentation_score_zone_weighted(zone);  in fragmentation_score_node()
2280 		reset_cached_positions(cc->zone);  in __compact_finished()
2289 			cc->zone->compact_blockskip_flush = true;  in __compact_finished()
2301 		pgdat = cc->zone->zone_pgdat;  in __compact_finished()
2305 		score = fragmentation_score_zone(cc->zone);  in __compact_finished()
2331 		struct free_area *area = &cc->zone->free_area[order];  in __compact_finished()
2373 	trace_mm_compaction_finished(cc->zone, cc->order, ret);  in compact_finished()
2380 static bool __compaction_suitable(struct zone *zone, int order,  in __compaction_suitable()  argument
2400 				low_wmark_pages(zone) : min_wmark_pages(zone);  in __compaction_suitable()
2402 	return __zone_watermark_ok(zone, 0, watermark, highest_zoneidx,  in __compaction_suitable()
2409 bool compaction_suitable(struct zone *zone, int order, int highest_zoneidx)  in compaction_suitable()  argument
2414 	suitable = __compaction_suitable(zone, order, highest_zoneidx,  in compaction_suitable()
2415 					 zone_page_state(zone, NR_FREE_PAGES));  in compaction_suitable()
2435 			int fragindex = fragmentation_index(zone, order);  in compaction_suitable()
2447 	trace_mm_compaction_suitable(zone, order, compact_result);  in compaction_suitable()
2455 	struct zone *zone;  in compaction_zonelist_suitable()  local
2462 	for_each_zone_zonelist_nodemask(zone, z, ac->zonelist,  in compaction_zonelist_suitable()
2472 		available = zone_reclaimable_pages(zone) / order;  in compaction_zonelist_suitable()
2473 		available += zone_page_state_snapshot(zone, NR_FREE_PAGES);  in compaction_zonelist_suitable()
2474 		if (__compaction_suitable(zone, order, ac->highest_zoneidx,  in compaction_zonelist_suitable()
2490 compaction_suit_allocation_order(struct zone *zone, unsigned int order,  in compaction_suit_allocation_order()  argument
2495 	watermark = wmark_pages(zone, alloc_flags & ALLOC_WMARK_MASK);  in compaction_suit_allocation_order()
2496 	if (zone_watermark_ok(zone, order, watermark, highest_zoneidx,  in compaction_suit_allocation_order()
2500 	if (!compaction_suitable(zone, order, highest_zoneidx))  in compaction_suit_allocation_order()
2510 	unsigned long start_pfn = cc->zone->zone_start_pfn;  in compact_zone()
2511 	unsigned long end_pfn = zone_end_pfn(cc->zone);  in compact_zone()
2533 		ret = compaction_suit_allocation_order(cc->zone, cc->order,  in compact_zone()
2544 	if (compaction_restarting(cc->zone, cc->order))  in compact_zone()
2545 		__reset_isolation_suitable(cc->zone);  in compact_zone()
2558 		cc->migrate_pfn = cc->zone->compact_cached_migrate_pfn[sync];  in compact_zone()
2559 		cc->free_pfn = cc->zone->compact_cached_free_pfn;  in compact_zone()
2562 			cc->zone->compact_cached_free_pfn = cc->free_pfn;  in compact_zone()
2566 			cc->zone->compact_cached_migrate_pfn[0] = cc->migrate_pfn;  in compact_zone()
2567 			cc->zone->compact_cached_migrate_pfn[1] = cc->migrate_pfn;  in compact_zone()
2570 		if (cc->migrate_pfn <= cc->zone->compact_init_migrate_pfn)  in compact_zone()
2585 		cc->zone->compact_cached_migrate_pfn[0] == cc->zone->compact_cached_migrate_pfn[1];  in compact_zone()
2619 				cc->zone->compact_cached_migrate_pfn[1] =  in compact_zone()
2620 					cc->zone->compact_cached_migrate_pfn[0];  in compact_zone()
2631 			last_migrated_pfn = max(cc->zone->zone_start_pfn,  in compact_zone()
2706 				lru_add_drain_cpu_zone(cc->zone);  in compact_zone()
2729 		if (free_pfn > cc->zone->compact_cached_free_pfn)  in compact_zone()
2730 			cc->zone->compact_cached_free_pfn = free_pfn;  in compact_zone()
2743 static enum compact_result compact_zone_order(struct zone *zone, int order,  in compact_zone_order()  argument
2753 		.zone = zone,  in compact_zone_order()
2813 	struct zone *zone;  in try_to_compact_pages()  local
2822 	for_each_zone_zonelist_nodemask(zone, z, ac->zonelist,  in try_to_compact_pages()
2828 			!__cpuset_zone_allowed(zone, gfp_mask))  in try_to_compact_pages()
2832 					&& compaction_deferred(zone, order)) {  in try_to_compact_pages()
2837 		status = compact_zone_order(zone, order, gfp_mask, prio,  in try_to_compact_pages()
2849 			compaction_defer_reset(zone, order, false);  in try_to_compact_pages()
2861 			defer_compaction(zone, order);  in try_to_compact_pages()
2890 	struct zone *zone;  in compact_node()  local
2901 		zone = &pgdat->node_zones[zoneid];  in compact_node()
2902 		if (!populated_zone(zone))  in compact_node()
2908 		cc.zone = zone;  in compact_node()
3026 	struct zone *zone;  in kcompactd_node_suitable()  local
3031 		zone = &pgdat->node_zones[zoneid];  in kcompactd_node_suitable()
3033 		if (!populated_zone(zone))  in kcompactd_node_suitable()
3036 		ret = compaction_suit_allocation_order(zone,  in kcompactd_node_suitable()
3053 	struct zone *zone;  in kcompactd_do_work()  local
3071 		zone = &pgdat->node_zones[zoneid];  in kcompactd_do_work()
3072 		if (!populated_zone(zone))  in kcompactd_do_work()
3075 		if (compaction_deferred(zone, cc.order))  in kcompactd_do_work()
3078 		ret = compaction_suit_allocation_order(zone,  in kcompactd_do_work()
3086 		cc.zone = zone;  in kcompactd_do_work()
3090 			compaction_defer_reset(zone, cc.order, false);  in kcompactd_do_work()
3098 			drain_all_pages(zone);  in kcompactd_do_work()
3104 			defer_compaction(zone, cc.order);  in kcompactd_do_work()