3  * mm/percpu-vm.c - vmalloc area based chunk allocation
9  * This is the default chunk allocator.
13 static struct page *pcpu_chunk_page(struct pcpu_chunk *chunk,  in pcpu_chunk_page()  argument
16 	/* must not be used on pre-mapped chunk */  in pcpu_chunk_page()
17 	WARN_ON(chunk->immutable);  in pcpu_chunk_page()
19 	return vmalloc_to_page((void *)pcpu_chunk_addr(chunk, cpu, page_idx));  in pcpu_chunk_page()
45  * pcpu_free_pages - free pages which were allocated for @chunk
46  * @chunk: chunk pages were allocated for
52  * The pages were allocated for @chunk.
54 static void pcpu_free_pages(struct pcpu_chunk *chunk,  in pcpu_free_pages()  argument
71  * pcpu_alloc_pages - allocates pages for @chunk
72  * @chunk: target chunk
79  * The allocation is for @chunk.  Percpu core doesn't care about the
82 static int pcpu_alloc_pages(struct pcpu_chunk *chunk,  in pcpu_alloc_pages()  argument
117  * @chunk: chunk the regions to be flushed belongs to
121  * Pages in [@page_start,@page_end) of @chunk are about to be
127 static void pcpu_pre_unmap_flush(struct pcpu_chunk *chunk,  in pcpu_pre_unmap_flush()  argument
131 		pcpu_chunk_addr(chunk, pcpu_low_unit_cpu, page_start),  in pcpu_pre_unmap_flush()
132 		pcpu_chunk_addr(chunk, pcpu_high_unit_cpu, page_end));  in pcpu_pre_unmap_flush()
142  * @chunk: chunk of interest
147  * For each cpu, unmap pages [@page_start,@page_end) out of @chunk.
153 static void pcpu_unmap_pages(struct pcpu_chunk *chunk,  in pcpu_unmap_pages()  argument
163 			page = pcpu_chunk_page(chunk, cpu, i);  in pcpu_unmap_pages()
167 		__pcpu_unmap_pages(pcpu_chunk_addr(chunk, cpu, page_start),  in pcpu_unmap_pages()
174  * @chunk: pcpu_chunk the regions to be flushed belong to
178  * Pages [@page_start,@page_end) of @chunk have been unmapped.  Flush
185 static void pcpu_post_unmap_tlb_flush(struct pcpu_chunk *chunk,  in pcpu_post_unmap_tlb_flush()  argument
189 		pcpu_chunk_addr(chunk, pcpu_low_unit_cpu, page_start),  in pcpu_post_unmap_tlb_flush()
190 		pcpu_chunk_addr(chunk, pcpu_high_unit_cpu, page_end));  in pcpu_post_unmap_tlb_flush()
202  * @chunk: chunk of interest
207  * For each cpu, map pages [@page_start,@page_end) into @chunk.  The
212  * reverse lookup (addr -> chunk).
214 static int pcpu_map_pages(struct pcpu_chunk *chunk,  in pcpu_map_pages()  argument
221 		err = __pcpu_map_pages(pcpu_chunk_addr(chunk, cpu, page_start),  in pcpu_map_pages()
229 					    chunk);  in pcpu_map_pages()
234 		__pcpu_unmap_pages(pcpu_chunk_addr(chunk, tcpu, page_start),  in pcpu_map_pages()
239 	pcpu_post_unmap_tlb_flush(chunk, page_start, page_end);  in pcpu_map_pages()
245  * @chunk: pcpu_chunk the regions to be flushed belong to
249  * Pages [@page_start,@page_end) of @chunk have been mapped.  Flush
255 static void pcpu_post_map_flush(struct pcpu_chunk *chunk,  in pcpu_post_map_flush()  argument
259 		pcpu_chunk_addr(chunk, pcpu_low_unit_cpu, page_start),  in pcpu_post_map_flush()
260 		pcpu_chunk_addr(chunk, pcpu_high_unit_cpu, page_end));  in pcpu_post_map_flush()
265  * @chunk: chunk of interest
271  * @chunk.
276 static int pcpu_populate_chunk(struct pcpu_chunk *chunk,  in pcpu_populate_chunk()  argument
285 	if (pcpu_alloc_pages(chunk, pages, page_start, page_end, gfp))  in pcpu_populate_chunk()
288 	if (pcpu_map_pages(chunk, pages, page_start, page_end)) {  in pcpu_populate_chunk()
289 		pcpu_free_pages(chunk, pages, page_start, page_end);  in pcpu_populate_chunk()
292 	pcpu_post_map_flush(chunk, page_start, page_end);  in pcpu_populate_chunk()
299  * @chunk: chunk to depopulate
304  * from @chunk.
312 static void pcpu_depopulate_chunk(struct pcpu_chunk *chunk,  in pcpu_depopulate_chunk()  argument
326 	pcpu_pre_unmap_flush(chunk, page_start, page_end);  in pcpu_depopulate_chunk()
328 	pcpu_unmap_pages(chunk, pages, page_start, page_end);  in pcpu_depopulate_chunk()
330 	pcpu_free_pages(chunk, pages, page_start, page_end);  in pcpu_depopulate_chunk()
335 	struct pcpu_chunk *chunk;  in pcpu_create_chunk()  local
338 	chunk = pcpu_alloc_chunk(gfp);  in pcpu_create_chunk()
339 	if (!chunk)  in pcpu_create_chunk()
345 		pcpu_free_chunk(chunk);  in pcpu_create_chunk()
349 	chunk->data = vms;  in pcpu_create_chunk()
350 	chunk->base_addr = vms[0]->addr - pcpu_group_offsets[0];  in pcpu_create_chunk()
353 	trace_percpu_create_chunk(chunk->base_addr);  in pcpu_create_chunk()
355 	return chunk;  in pcpu_create_chunk()
358 static void pcpu_destroy_chunk(struct pcpu_chunk *chunk)  in pcpu_destroy_chunk()  argument
360 	if (!chunk)  in pcpu_destroy_chunk()
364 	trace_percpu_destroy_chunk(chunk->base_addr);  in pcpu_destroy_chunk()
366 	if (chunk->data)  in pcpu_destroy_chunk()
367 		pcpu_free_vm_areas(chunk->data, pcpu_nr_groups);  in pcpu_destroy_chunk()
368 	pcpu_free_chunk(chunk);  in pcpu_destroy_chunk()
383  * pcpu_should_reclaim_chunk - determine if a chunk should go into reclaim
384  * @chunk: chunk of interest
386  * This is the entry point for percpu reclaim.  If a chunk qualifies, it is then
391  * list which enables them to be pulled back in for allocation if no other chunk
394 static bool pcpu_should_reclaim_chunk(struct pcpu_chunk *chunk)  in pcpu_should_reclaim_chunk()  argument
396 	/* do not reclaim either the first chunk or reserved chunk */  in pcpu_should_reclaim_chunk()
397 	if (chunk == pcpu_first_chunk || chunk == pcpu_reserved_chunk)  in pcpu_should_reclaim_chunk()
404 	 * chunk, move it to the to_depopulate list.  in pcpu_should_reclaim_chunk()
406 	return ((chunk->isolated && chunk->nr_empty_pop_pages) ||  in pcpu_should_reclaim_chunk()
408 		 (PCPU_EMPTY_POP_PAGES_HIGH + chunk->nr_empty_pop_pages) &&  in pcpu_should_reclaim_chunk()
409 		 chunk->nr_empty_pop_pages >= chunk->nr_pages / 4));  in pcpu_should_reclaim_chunk()