406 	struct mem_zone_bm_rtree *zone;  member
468 static int add_rtree_block(struct mem_zone_bm_rtree *zone, gfp_t gfp_mask,  in add_rtree_block()  argument
475 	block_nr = zone->blocks;  in add_rtree_block()
485 	for (i = zone->levels; i < levels_needed; i++) {  in add_rtree_block()
487 					&zone->nodes);  in add_rtree_block()
491 		node->data[0] = (unsigned long)zone->rtree;  in add_rtree_block()
492 		zone->rtree = node;  in add_rtree_block()
493 		zone->levels += 1;  in add_rtree_block()
497 	block = alloc_rtree_node(gfp_mask, safe_needed, ca, &zone->leaves);  in add_rtree_block()
502 	node = zone->rtree;  in add_rtree_block()
503 	dst = &zone->rtree;  in add_rtree_block()
504 	block_nr = zone->blocks;  in add_rtree_block()
505 	for (i = zone->levels; i > 0; i--) {  in add_rtree_block()
510 						&zone->nodes);  in add_rtree_block()
522 	zone->blocks += 1;  in add_rtree_block()
528 static void free_zone_bm_rtree(struct mem_zone_bm_rtree *zone,
544 	struct mem_zone_bm_rtree *zone;  in create_zone_bm_rtree()  local
549 	zone  = chain_alloc(ca, sizeof(struct mem_zone_bm_rtree));  in create_zone_bm_rtree()
550 	if (!zone)  in create_zone_bm_rtree()
553 	INIT_LIST_HEAD(&zone->nodes);  in create_zone_bm_rtree()
554 	INIT_LIST_HEAD(&zone->leaves);  in create_zone_bm_rtree()
555 	zone->start_pfn = start;  in create_zone_bm_rtree()
556 	zone->end_pfn = end;  in create_zone_bm_rtree()
560 		if (add_rtree_block(zone, gfp_mask, safe_needed, ca)) {  in create_zone_bm_rtree()
561 			free_zone_bm_rtree(zone, PG_UNSAFE_CLEAR);  in create_zone_bm_rtree()
566 	return zone;  in create_zone_bm_rtree()
576 static void free_zone_bm_rtree(struct mem_zone_bm_rtree *zone,  in free_zone_bm_rtree()  argument
581 	list_for_each_entry(node, &zone->nodes, list)  in free_zone_bm_rtree()
584 	list_for_each_entry(node, &zone->leaves, list)  in free_zone_bm_rtree()
590 	bm->cur.zone = list_entry(bm->zones.next, struct mem_zone_bm_rtree,  in memory_bm_position_reset()
592 	bm->cur.node = list_entry(bm->cur.zone->leaves.next,  in memory_bm_position_reset()
630 	struct zone *zone;  in create_mem_extents()  local
634 	for_each_populated_zone(zone) {  in create_mem_extents()
638 		zone_start = zone->zone_start_pfn;  in create_mem_extents()
639 		zone_end = zone_end_pfn(zone);  in create_mem_extents()
700 		struct mem_zone_bm_rtree *zone;  in memory_bm_create()  local
702 		zone = create_zone_bm_rtree(gfp_mask, safe_needed, &ca,  in memory_bm_create()
704 		if (!zone) {  in memory_bm_create()
708 		list_add_tail(&zone->list, &bm->zones);  in memory_bm_create()
729 	struct mem_zone_bm_rtree *zone;  in memory_bm_free()  local
731 	list_for_each_entry(zone, &bm->zones, list)  in memory_bm_free()
732 		free_zone_bm_rtree(zone, clear_nosave_free);  in memory_bm_free()
751 	struct mem_zone_bm_rtree *curr, *zone;  in memory_bm_find_bit()  local
755 	zone = bm->cur.zone;  in memory_bm_find_bit()
757 	if (pfn >= zone->start_pfn && pfn < zone->end_pfn)  in memory_bm_find_bit()
760 	zone = NULL;  in memory_bm_find_bit()
765 			zone = curr;  in memory_bm_find_bit()
770 	if (!zone)  in memory_bm_find_bit()
785 	if (zone == bm->cur.zone &&  in memory_bm_find_bit()
786 	    ((pfn - zone->start_pfn) & ~BM_BLOCK_MASK) == bm->cur.node_pfn)  in memory_bm_find_bit()
789 	node      = zone->rtree;  in memory_bm_find_bit()
790 	block_nr  = (pfn - zone->start_pfn) >> BM_BLOCK_SHIFT;  in memory_bm_find_bit()
792 	for (i = zone->levels; i > 0; i--) {  in memory_bm_find_bit()
803 	bm->cur.zone = zone;  in memory_bm_find_bit()
805 	bm->cur.node_pfn = (pfn - zone->start_pfn) & ~BM_BLOCK_MASK;  in memory_bm_find_bit()
810 	*bit_nr = (pfn - zone->start_pfn) & BM_BLOCK_MASK;  in memory_bm_find_bit()
894 	if (!list_is_last(&bm->cur.node->list, &bm->cur.zone->leaves)) {  in rtree_next_node()
904 	if (!list_is_last(&bm->cur.zone->list, &bm->zones)) {  in rtree_next_node()
905 		bm->cur.zone = list_entry(bm->cur.zone->list.next,  in rtree_next_node()
907 		bm->cur.node = list_entry(bm->cur.zone->leaves.next,  in rtree_next_node()
935 		pages	  = bm->cur.zone->end_pfn - bm->cur.zone->start_pfn;  in memory_bm_next_pfn()
940 			pfn = bm->cur.zone->start_pfn + bm->cur.node_pfn + bit;  in memory_bm_next_pfn()
963 static void recycle_zone_bm_rtree(struct mem_zone_bm_rtree *zone)  in recycle_zone_bm_rtree()  argument
967 	list_for_each_entry(node, &zone->nodes, list)  in recycle_zone_bm_rtree()
970 	list_for_each_entry(node, &zone->leaves, list)  in recycle_zone_bm_rtree()
976 	struct mem_zone_bm_rtree *zone;  in memory_bm_recycle()  local
979 	list_for_each_entry(zone, &bm->zones, list)  in memory_bm_recycle()
980 		recycle_zone_bm_rtree(zone);  in memory_bm_recycle()
1228 unsigned int snapshot_additional_pages(struct zone *zone)  in snapshot_additional_pages()  argument
1232 	rtree = nodes = DIV_ROUND_UP(zone->spanned_pages, BM_BITS_PER_BLOCK);  in snapshot_additional_pages()
1248 static void mark_free_pages(struct zone *zone)  in mark_free_pages()  argument
1255 	if (zone_is_empty(zone))  in mark_free_pages()
1258 	spin_lock_irqsave(&zone->lock, flags);  in mark_free_pages()
1260 	max_zone_pfn = zone_end_pfn(zone);  in mark_free_pages()
1261 	for (pfn = zone->zone_start_pfn; pfn < max_zone_pfn; pfn++)  in mark_free_pages()
1270 			if (page_zone(page) != zone)  in mark_free_pages()
1279 				&zone->free_area[order].free_list[t], buddy_list) {  in mark_free_pages()
1292 	spin_unlock_irqrestore(&zone->lock, flags);  in mark_free_pages()
1303 	struct zone *zone;  in count_free_highmem_pages()  local
1306 	for_each_populated_zone(zone)  in count_free_highmem_pages()
1307 		if (is_highmem(zone))  in count_free_highmem_pages()
1308 			cnt += zone_page_state(zone, NR_FREE_PAGES);  in count_free_highmem_pages()
1321 static struct page *saveable_highmem_page(struct zone *zone, unsigned long pfn)  in saveable_highmem_page()  argument
1329 	if (!page || page_zone(page) != zone)  in saveable_highmem_page()
1351 	struct zone *zone;  in count_highmem_pages()  local
1354 	for_each_populated_zone(zone) {  in count_highmem_pages()
1357 		if (!is_highmem(zone))  in count_highmem_pages()
1360 		mark_free_pages(zone);  in count_highmem_pages()
1361 		max_zone_pfn = zone_end_pfn(zone);  in count_highmem_pages()
1362 		for (pfn = zone->zone_start_pfn; pfn < max_zone_pfn; pfn++)  in count_highmem_pages()
1363 			if (saveable_highmem_page(zone, pfn))  in count_highmem_pages()
1380 static struct page *saveable_page(struct zone *zone, unsigned long pfn)  in saveable_page()  argument
1388 	if (!page || page_zone(page) != zone)  in saveable_page()
1414 	struct zone *zone;  in count_data_pages()  local
1418 	for_each_populated_zone(zone) {  in count_data_pages()
1419 		if (is_highmem(zone))  in count_data_pages()
1422 		mark_free_pages(zone);  in count_data_pages()
1423 		max_zone_pfn = zone_end_pfn(zone);  in count_data_pages()
1424 		for (pfn = zone->zone_start_pfn; pfn < max_zone_pfn; pfn++)  in count_data_pages()
1425 			if (saveable_page(zone, pfn))  in count_data_pages()
1472 static inline struct page *page_is_saveable(struct zone *zone, unsigned long pfn)  in page_is_saveable()  argument
1474 	return is_highmem(zone) ?  in page_is_saveable()
1475 		saveable_highmem_page(zone, pfn) : saveable_page(zone, pfn);  in page_is_saveable()
1509 #define page_is_saveable(zone, pfn)	saveable_page(zone, pfn)  argument
1529 	struct zone *zone;  in copy_data_pages()  local
1532 	for_each_populated_zone(zone) {  in copy_data_pages()
1535 		mark_free_pages(zone);  in copy_data_pages()
1536 		max_zone_pfn = zone_end_pfn(zone);  in copy_data_pages()
1537 		for (pfn = zone->zone_start_pfn; pfn < max_zone_pfn; pfn++)  in copy_data_pages()
1538 			if (page_is_saveable(zone, pfn))  in copy_data_pages()
1831 	struct zone *zone;  in hibernate_preallocate_memory()  local
1874 	for_each_populated_zone(zone) {  in hibernate_preallocate_memory()
1875 		size += snapshot_additional_pages(zone);  in hibernate_preallocate_memory()
1876 		if (is_highmem(zone))  in hibernate_preallocate_memory()
1877 			highmem += zone_page_state(zone, NR_FREE_PAGES);  in hibernate_preallocate_memory()
1879 			count += zone_page_state(zone, NR_FREE_PAGES);  in hibernate_preallocate_memory()
2016 	struct zone *zone;  in enough_free_mem()  local
2019 	for_each_populated_zone(zone)  in enough_free_mem()
2020 		if (!is_highmem(zone))  in enough_free_mem()
2021 			free += zone_page_state(zone, NR_FREE_PAGES);  in enough_free_mem()