7 #include <linux/page-isolation.h>
28  * Returns a page without holding a reference. If the caller wants to
29  * dereference that page (e.g., dumping), it has to make sure that it
33 static struct page *has_unmovable_pages(unsigned long start_pfn, unsigned long end_pfn,  in has_unmovable_pages()
36 	struct page *page = pfn_to_page(start_pfn);  in has_unmovable_pages()  local
37 	struct zone *zone = page_zone(page);  in has_unmovable_pages()
43 	if (is_migrate_cma_page(page)) {  in has_unmovable_pages()
52 		return page;  in has_unmovable_pages()
56 		page = pfn_to_page(pfn);  in has_unmovable_pages()
64 		if (PageReserved(page))  in has_unmovable_pages()
65 			return page;  in has_unmovable_pages()
79 		 * handle each tail page individually in migration.  in has_unmovable_pages()
81 		if (PageHuge(page) || PageTransCompound(page)) {  in has_unmovable_pages()
82 			struct folio *folio = page_folio(page);  in has_unmovable_pages()
85 			if (PageHuge(page)) {  in has_unmovable_pages()
87 					return page;  in has_unmovable_pages()
89 				return page;  in has_unmovable_pages()
92 			skip_pages = folio_nr_pages(folio) - folio_page_idx(folio, page);  in has_unmovable_pages()
98 		 * We can't use page_count without pin a page  in has_unmovable_pages()
99 		 * because another CPU can free compound page.  in has_unmovable_pages()
101 		 * because their page->_refcount is zero at all time.  in has_unmovable_pages()
103 		if (!page_ref_count(page)) {  in has_unmovable_pages()
104 			if (PageBuddy(page))  in has_unmovable_pages()
105 				pfn += (1 << buddy_order(page)) - 1;  in has_unmovable_pages()
110 		 * The HWPoisoned page may be not in buddy system, and  in has_unmovable_pages()
113 		if ((flags & MEMORY_OFFLINE) && PageHWPoison(page))  in has_unmovable_pages()
126 		if ((flags & MEMORY_OFFLINE) && PageOffline(page))  in has_unmovable_pages()
129 		if (__PageMovable(page) || PageLRU(page))  in has_unmovable_pages()
137 		return page;  in has_unmovable_pages()
143  * This function set pageblock migratetype to isolate if no unmovable page is
147 static int set_migratetype_isolate(struct page *page, int migratetype, int isol_flags,  in set_migratetype_isolate()  argument
150 	struct zone *zone = page_zone(page);  in set_migratetype_isolate()
151 	struct page *unmovable;  in set_migratetype_isolate()
155 	if (PageUnaccepted(page))  in set_migratetype_isolate()
156 		accept_page(page);  in set_migratetype_isolate()
165 	if (is_migrate_isolate_page(page)) {  in set_migratetype_isolate()
174 	 * Pass the intersection of [start_pfn, end_pfn) and the page's pageblock  in set_migratetype_isolate()
177 	check_unmovable_start = max(page_to_pfn(page), start_pfn);  in set_migratetype_isolate()
178 	check_unmovable_end = min(pageblock_end_pfn(page_to_pfn(page)),  in set_migratetype_isolate()
184 		if (!move_freepages_block_isolate(zone, page, MIGRATE_ISOLATE)) {  in set_migratetype_isolate()
199 		dump_page(unmovable, "unmovable page");  in set_migratetype_isolate()
205 static void unset_migratetype_isolate(struct page *page, int migratetype)  in unset_migratetype_isolate()  argument
211 	struct page *buddy;  in unset_migratetype_isolate()
213 	zone = page_zone(page);  in unset_migratetype_isolate()
215 	if (!is_migrate_isolate_page(page))  in unset_migratetype_isolate()
221 	 * it is possible that there is free buddy page.  in unset_migratetype_isolate()
226 	if (PageBuddy(page)) {  in unset_migratetype_isolate()
227 		order = buddy_order(page);  in unset_migratetype_isolate()
229 			buddy = find_buddy_page_pfn(page, page_to_pfn(page),  in unset_migratetype_isolate()
232 				isolated_page = !!__isolate_free_page(page, order);  in unset_migratetype_isolate()
234 				 * Isolating a free page in an isolated pageblock  in unset_migratetype_isolate()
258 		WARN_ON_ONCE(!move_freepages_block_isolate(zone, page, migratetype));  in unset_migratetype_isolate()
260 		set_pageblock_migratetype(page, migratetype);  in unset_migratetype_isolate()
261 		__putback_isolated_page(page, order, migratetype);  in unset_migratetype_isolate()
268 static inline struct page *
274 		struct page *page;  in __first_valid_page()  local
276 		page = pfn_to_online_page(pfn + i);  in __first_valid_page()
277 		if (!page)  in __first_valid_page()
279 		return page;  in __first_valid_page()
286  * within a free or in-use page.
287  * @boundary_pfn:		pageblock-aligned pfn that a page might cross
296  * pageblock. When not all pageblocks within a page are isolated at the same
297  * time, free page accounting can go wrong. For example, in the case of
298  * MAX_PAGE_ORDER = pageblock_order + 1, a MAX_PAGE_ORDER page has two
302  * When either pageblock is isolated, if it is a free page, the page is not
304  * in-use page and freed later, __free_one_page() does not split the free page
305  * either. The function handles this by splitting the free page or migrating
306  * the in-use page then splitting the free page.
328 	 * free or in-use page. Also make sure all to-be-isolated pageblocks  in isolate_single_pageblock()
349 	 * a free or in-use page across boundary_pfn:  in isolate_single_pageblock()
351 	 * 1. isolate before boundary_pfn: the page after is not online  in isolate_single_pageblock()
352 	 * 2. isolate after boundary_pfn: the page before is not online  in isolate_single_pageblock()
368 		struct page *page = __first_valid_page(pfn, boundary_pfn - pfn);  in isolate_single_pageblock()  local
370 		VM_BUG_ON(!page);  in isolate_single_pageblock()
371 		pfn = page_to_pfn(page);  in isolate_single_pageblock()
373 		if (PageUnaccepted(page)) {  in isolate_single_pageblock()
378 		if (PageBuddy(page)) {  in isolate_single_pageblock()
379 			int order = buddy_order(page);  in isolate_single_pageblock()
389 		 * If a compound page is straddling our block, attempt  in isolate_single_pageblock()
393 		 * free page that straddles into our block: gigantic  in isolate_single_pageblock()
401 		if (PageCompound(page)) {  in isolate_single_pageblock()
402 			struct page *head = compound_head(page);  in isolate_single_pageblock()
407 			    PageHuge(page)) {  in isolate_single_pageblock()
419 			VM_WARN_ON_ONCE_PAGE(PageLRU(page), page);  in isolate_single_pageblock()
420 			VM_WARN_ON_ONCE_PAGE(__PageMovable(page), page);  in isolate_single_pageblock()
436  * start_isolate_page_range() - mark page range MIGRATE_ISOLATE
450  * Making page-allocation-type to be MIGRATE_ISOLATE means free pages in
469  * Please note that there is no strong synchronization with the page allocator
470  * either. Pages might be freed while their page blocks are marked ISOLATED.
484 	struct page *page;  in start_isolate_page_range()  local
485 	/* isolation is done at page block granularity */  in start_isolate_page_range()
512 		page = __first_valid_page(pfn, pageblock_nr_pages);  in start_isolate_page_range()
513 		if (page && set_migratetype_isolate(page, migratetype, flags,  in start_isolate_page_range()
531  * This finds every MIGRATE_ISOLATE page block in the given range
538 	struct page *page;  in undo_isolate_page_range()  local
545 		page = __first_valid_page(pfn, pageblock_nr_pages);  in undo_isolate_page_range()
546 		if (!page || !is_migrate_isolate_page(page))  in undo_isolate_page_range()
548 		unset_migratetype_isolate(page, migratetype);  in undo_isolate_page_range()
562 	struct page *page;  in __test_page_isolated_in_pageblock()  local
565 		page = pfn_to_page(pfn);  in __test_page_isolated_in_pageblock()
566 		if (PageBuddy(page))  in __test_page_isolated_in_pageblock()
568 			 * If the page is on a free list, it has to be on  in __test_page_isolated_in_pageblock()
572 			pfn += 1 << buddy_order(page);  in __test_page_isolated_in_pageblock()
573 		else if ((flags & MEMORY_OFFLINE) && PageHWPoison(page))  in __test_page_isolated_in_pageblock()
574 			/* A HWPoisoned page cannot be also PageBuddy */  in __test_page_isolated_in_pageblock()
576 		else if ((flags & MEMORY_OFFLINE) && PageOffline(page) &&  in __test_page_isolated_in_pageblock()
577 			 !page_count(page))  in __test_page_isolated_in_pageblock()
610 	struct page *page;  in test_pages_isolated()  local
620 		page = __first_valid_page(pfn, pageblock_nr_pages);  in test_pages_isolated()
621 		if (page && !is_migrate_isolate_page(page))  in test_pages_isolated()
624 	page = __first_valid_page(start_pfn, end_pfn - start_pfn);  in test_pages_isolated()
625 	if ((pfn < end_pfn) || !page) {  in test_pages_isolated()
631 	zone = page_zone(page);  in test_pages_isolated()