320 	int bit_off = ALIGN(block->contig_hint_start, align) -  in pcpu_check_block_hint()  local
323 	return bit_off + bits <= block->contig_hint;  in pcpu_check_block_hint()
365 static void pcpu_next_md_free_region(struct pcpu_chunk *chunk, int *bit_off,  in pcpu_next_md_free_region()  argument
368 	int i = pcpu_off_to_block_index(*bit_off);  in pcpu_next_md_free_region()
369 	int block_off = pcpu_off_to_block_off(*bit_off);  in pcpu_next_md_free_region()
394 			*bit_off = pcpu_block_off_to_off(i,  in pcpu_next_md_free_region()
402 		*bit_off = (i + 1) * PCPU_BITMAP_BLOCK_BITS - block->right_free;  in pcpu_next_md_free_region()
421 				 int align, int *bit_off, int *bits)  in pcpu_next_fit_region()  argument
423 	int i = pcpu_off_to_block_index(*bit_off);  in pcpu_next_fit_region()
424 	int block_off = pcpu_off_to_block_off(*bit_off);  in pcpu_next_fit_region()
453 			*bit_off = pcpu_block_off_to_off(i, start);  in pcpu_next_fit_region()
459 		*bit_off = ALIGN(PCPU_BITMAP_BLOCK_BITS - block->right_free,  in pcpu_next_fit_region()
461 		*bits = PCPU_BITMAP_BLOCK_BITS - *bit_off;  in pcpu_next_fit_region()
462 		*bit_off = pcpu_block_off_to_off(i, *bit_off);  in pcpu_next_fit_region()
468 	*bit_off = pcpu_chunk_map_bits(chunk);  in pcpu_next_fit_region()
477 #define pcpu_for_each_md_free_region(chunk, bit_off, bits)		\  argument
478 	for (pcpu_next_md_free_region((chunk), &(bit_off), &(bits));	\
479 	     (bit_off) < pcpu_chunk_map_bits((chunk));			\
480 	     (bit_off) += (bits) + 1,					\
481 	     pcpu_next_md_free_region((chunk), &(bit_off), &(bits)))
483 #define pcpu_for_each_fit_region(chunk, alloc_bits, align, bit_off, bits)     \  argument
484 	for (pcpu_next_fit_region((chunk), (alloc_bits), (align), &(bit_off), \
486 	     (bit_off) < pcpu_chunk_map_bits((chunk));			      \
487 	     (bit_off) += (bits),					      \
488 	     pcpu_next_fit_region((chunk), (alloc_bits), (align), &(bit_off), \
712 static void pcpu_block_update_scan(struct pcpu_chunk *chunk, int bit_off,  in pcpu_block_update_scan()  argument
715 	int s_off = pcpu_off_to_block_off(bit_off);  in pcpu_block_update_scan()
723 	s_index = pcpu_off_to_block_index(bit_off);  in pcpu_block_update_scan()
748 	int bit_off, bits;  in pcpu_chunk_refresh_hint()  local
752 		bit_off = chunk_md->scan_hint_start + chunk_md->scan_hint;  in pcpu_chunk_refresh_hint()
757 		bit_off = chunk_md->first_free;  in pcpu_chunk_refresh_hint()
762 	pcpu_for_each_md_free_region(chunk, bit_off, bits)  in pcpu_chunk_refresh_hint()
763 		pcpu_block_update(chunk_md, bit_off, bit_off + bits);  in pcpu_chunk_refresh_hint()
808 static void pcpu_block_update_hint_alloc(struct pcpu_chunk *chunk, int bit_off,  in pcpu_block_update_hint_alloc()  argument
823 	s_index = pcpu_off_to_block_index(bit_off);  in pcpu_block_update_hint_alloc()
824 	e_index = pcpu_off_to_block_index(bit_off + bits - 1);  in pcpu_block_update_hint_alloc()
825 	s_off = pcpu_off_to_block_off(bit_off);  in pcpu_block_update_hint_alloc()
826 	e_off = pcpu_off_to_block_off(bit_off + bits - 1) + 1;  in pcpu_block_update_hint_alloc()
928 				bit_off,  in pcpu_block_update_hint_alloc()
929 				bit_off + bits))  in pcpu_block_update_hint_alloc()
940 				bit_off,  in pcpu_block_update_hint_alloc()
941 				bit_off + bits))  in pcpu_block_update_hint_alloc()
963 static void pcpu_block_update_hint_free(struct pcpu_chunk *chunk, int bit_off,  in pcpu_block_update_hint_free()  argument
978 	s_index = pcpu_off_to_block_index(bit_off);  in pcpu_block_update_hint_free()
979 	e_index = pcpu_off_to_block_index(bit_off + bits - 1);  in pcpu_block_update_hint_free()
980 	s_off = pcpu_off_to_block_off(bit_off);  in pcpu_block_update_hint_free()
981 	e_off = pcpu_off_to_block_off(bit_off + bits - 1) + 1;  in pcpu_block_update_hint_free()
1073 static bool pcpu_is_populated(struct pcpu_chunk *chunk, int bit_off, int bits,  in pcpu_is_populated()  argument
1078 	start = PFN_DOWN(bit_off * PCPU_MIN_ALLOC_SIZE);  in pcpu_is_populated()
1079 	end = PFN_UP((bit_off + bits) * PCPU_MIN_ALLOC_SIZE);  in pcpu_is_populated()
1114 	int bit_off, bits, next_off;  in pcpu_find_block_fit()  local
1124 	bit_off = pcpu_next_hint(chunk_md, alloc_bits);  in pcpu_find_block_fit()
1126 	pcpu_for_each_fit_region(chunk, alloc_bits, align, bit_off, bits) {  in pcpu_find_block_fit()
1127 		if (!pop_only || pcpu_is_populated(chunk, bit_off, bits,  in pcpu_find_block_fit()
1131 		bit_off = next_off;  in pcpu_find_block_fit()
1135 	if (bit_off == pcpu_chunk_map_bits(chunk))  in pcpu_find_block_fit()
1138 	return bit_off;  in pcpu_find_block_fit()
1222 	int bit_off, end, oslot;  in pcpu_alloc_area()  local
1233 	bit_off = pcpu_find_zero_area(chunk->alloc_map, end, start, alloc_bits,  in pcpu_alloc_area()
1235 	if (bit_off >= end)  in pcpu_alloc_area()
1242 	bitmap_set(chunk->alloc_map, bit_off, alloc_bits);  in pcpu_alloc_area()
1245 	set_bit(bit_off, chunk->bound_map);  in pcpu_alloc_area()
1246 	bitmap_clear(chunk->bound_map, bit_off + 1, alloc_bits - 1);  in pcpu_alloc_area()
1247 	set_bit(bit_off + alloc_bits, chunk->bound_map);  in pcpu_alloc_area()
1252 	if (bit_off == chunk_md->first_free)  in pcpu_alloc_area()
1256 					bit_off + alloc_bits);  in pcpu_alloc_area()
1258 	pcpu_block_update_hint_alloc(chunk, bit_off, alloc_bits);  in pcpu_alloc_area()
1262 	return bit_off * PCPU_MIN_ALLOC_SIZE;  in pcpu_alloc_area()
1279 	int bit_off, bits, end, oslot, freed;  in pcpu_free_area()  local
1286 	bit_off = off / PCPU_MIN_ALLOC_SIZE;  in pcpu_free_area()
1290 			    bit_off + 1);  in pcpu_free_area()
1291 	bits = end - bit_off;  in pcpu_free_area()
1292 	bitmap_clear(chunk->alloc_map, bit_off, bits);  in pcpu_free_area()
1300 	chunk_md->first_free = min(chunk_md->first_free, bit_off);  in pcpu_free_area()
1302 	pcpu_block_update_hint_free(chunk, bit_off, bits);  in pcpu_free_area()