3  *  Page table allocation functions
47 	unsigned long *table;  in crst_table_alloc()  local
51 	table = ptdesc_to_virt(ptdesc);  in crst_table_alloc()
52 	__arch_set_page_dat(table, 1UL << CRST_ALLOC_ORDER);  in crst_table_alloc()
53 	return table;  in crst_table_alloc()
56 void crst_table_free(struct mm_struct *mm, unsigned long *table)  in crst_table_free()  argument
58 	if (!table)  in crst_table_free()
60 	pagetable_free(virt_to_ptdesc(table));  in crst_table_free()
143 	u64 *table;  in page_table_alloc_pgste()  local
147 		table = (u64 *)ptdesc_to_virt(ptdesc);  in page_table_alloc_pgste()
148 		__arch_set_page_dat(table, 1);  in page_table_alloc_pgste()
149 		memset64(table, _PAGE_INVALID, PTRS_PER_PTE);  in page_table_alloc_pgste()
150 		memset64(table + PTRS_PER_PTE, 0, PTRS_PER_PTE);  in page_table_alloc_pgste()
165 	unsigned long *table;  in page_table_alloc()  local
174 	table = ptdesc_to_virt(ptdesc);  in page_table_alloc()
175 	__arch_set_page_dat(table, 1);  in page_table_alloc()
178 	memset64((u64 *)table, _PAGE_INVALID, PTRS_PER_PTE);  in page_table_alloc()
179 	memset64((u64 *)table + PTRS_PER_PTE, 0, PTRS_PER_PTE);  in page_table_alloc()
180 	return table;  in page_table_alloc()
189 void page_table_free(struct mm_struct *mm, unsigned long *table)  in page_table_free()  argument
191 	struct ptdesc *ptdesc = virt_to_ptdesc(table);  in page_table_free()
196 void __tlb_remove_table(void *table)  in __tlb_remove_table()  argument
198 	struct ptdesc *ptdesc = virt_to_ptdesc(table);  in __tlb_remove_table()
239 	unsigned long *table;  in base_pgt_alloc()  local
241 	table = kmem_cache_alloc(base_pgt_cache, GFP_KERNEL);  in base_pgt_alloc()
242 	if (table)  in base_pgt_alloc()
243 		memset64((u64 *)table, _PAGE_INVALID, PTRS_PER_PTE);  in base_pgt_alloc()
244 	return table;  in base_pgt_alloc()
247 static void base_pgt_free(unsigned long *table)  in base_pgt_free()  argument
249 	kmem_cache_free(base_pgt_cache, table);  in base_pgt_free()
254 	unsigned long *table;  in base_crst_alloc()  local
260 	table = ptdesc_address(ptdesc);  in base_crst_alloc()
261 	crst_table_init(table, val);  in base_crst_alloc()
262 	return table;  in base_crst_alloc()
265 static void base_crst_free(unsigned long *table)  in base_crst_free()  argument
267 	if (!table)  in base_crst_free()
269 	pagetable_free(virt_to_ptdesc(table));  in base_crst_free()
316 	unsigned long *ste, next, *table;  in base_segment_walk()  local
326 			table = base_pgt_alloc();  in base_segment_walk()
327 			if (!table)  in base_segment_walk()
329 			*ste = __pa(table) | _SEGMENT_ENTRY;  in base_segment_walk()
331 		table = __va(*ste & _SEGMENT_ENTRY_ORIGIN);  in base_segment_walk()
332 		rc = base_page_walk(table, addr, next, alloc);  in base_segment_walk()
336 			base_pgt_free(table);  in base_segment_walk()
345 	unsigned long *rtte, next, *table;  in base_region3_walk()  local
355 			table = base_crst_alloc(_SEGMENT_ENTRY_EMPTY);  in base_region3_walk()
356 			if (!table)  in base_region3_walk()
358 			*rtte = __pa(table) | _REGION3_ENTRY;  in base_region3_walk()
360 		table = __va(*rtte & _REGION_ENTRY_ORIGIN);  in base_region3_walk()
361 		rc = base_segment_walk(table, addr, next, alloc);  in base_region3_walk()
365 			base_crst_free(table);  in base_region3_walk()
373 	unsigned long *rste, next, *table;  in base_region2_walk()  local
383 			table = base_crst_alloc(_REGION3_ENTRY_EMPTY);  in base_region2_walk()
384 			if (!table)  in base_region2_walk()
386 			*rste = __pa(table) | _REGION2_ENTRY;  in base_region2_walk()
388 		table = __va(*rste & _REGION_ENTRY_ORIGIN);  in base_region2_walk()
389 		rc = base_region3_walk(table, addr, next, alloc);  in base_region2_walk()
393 			base_crst_free(table);  in base_region2_walk()
401 	unsigned long *rfte, next, *table;  in base_region1_walk()  local
411 			table = base_crst_alloc(_REGION2_ENTRY_EMPTY);  in base_region1_walk()
412 			if (!table)  in base_region1_walk()
414 			*rfte = __pa(table) | _REGION1_ENTRY;  in base_region1_walk()
416 		table = __va(*rfte & _REGION_ENTRY_ORIGIN);  in base_region1_walk()
417 		rc = base_region2_walk(table, addr, next, alloc);  in base_region1_walk()
421 			base_crst_free(table);  in base_region1_walk()
435 	unsigned long *table = __va(asce & _ASCE_ORIGIN);  in base_asce_free()  local
441 		base_segment_walk(table, 0, _REGION3_SIZE, 0);  in base_asce_free()
444 		base_region3_walk(table, 0, _REGION2_SIZE, 0);  in base_asce_free()
447 		base_region2_walk(table, 0, _REGION1_SIZE, 0);  in base_asce_free()
450 		base_region1_walk(table, 0, TASK_SIZE_MAX, 0);  in base_asce_free()
453 	base_crst_free(table);  in base_asce_free()
487 	unsigned long asce, *table, end;  in base_asce_alloc()  local
494 		table = base_crst_alloc(_SEGMENT_ENTRY_EMPTY);  in base_asce_alloc()
495 		if (!table)  in base_asce_alloc()
497 		rc = base_segment_walk(table, addr, end, 1);  in base_asce_alloc()
498 		asce = __pa(table) | _ASCE_TYPE_SEGMENT | _ASCE_TABLE_LENGTH;  in base_asce_alloc()
500 		table = base_crst_alloc(_REGION3_ENTRY_EMPTY);  in base_asce_alloc()
501 		if (!table)  in base_asce_alloc()
503 		rc = base_region3_walk(table, addr, end, 1);  in base_asce_alloc()
504 		asce = __pa(table) | _ASCE_TYPE_REGION3 | _ASCE_TABLE_LENGTH;  in base_asce_alloc()
506 		table = base_crst_alloc(_REGION2_ENTRY_EMPTY);  in base_asce_alloc()
507 		if (!table)  in base_asce_alloc()
509 		rc = base_region2_walk(table, addr, end, 1);  in base_asce_alloc()
510 		asce = __pa(table) | _ASCE_TYPE_REGION2 | _ASCE_TABLE_LENGTH;  in base_asce_alloc()
512 		table = base_crst_alloc(_REGION1_ENTRY_EMPTY);  in base_asce_alloc()
513 		if (!table)  in base_asce_alloc()
515 		rc = base_region1_walk(table, addr, end, 1);  in base_asce_alloc()
516 		asce = __pa(table) | _ASCE_TYPE_REGION1 | _ASCE_TABLE_LENGTH;  in base_asce_alloc()