378 static inline phys_addr_t __init alloc_pte_early(uintptr_t va)  in alloc_pte_early()  argument
387 static inline phys_addr_t __init alloc_pte_fixmap(uintptr_t va)  in alloc_pte_fixmap()  argument
392 static phys_addr_t __meminit alloc_pte_late(uintptr_t va)  in alloc_pte_late()  argument
400 static void __meminit create_pte_mapping(pte_t *ptep, uintptr_t va, phys_addr_t pa, phys_addr_t sz,  in create_pte_mapping()  argument
403 	uintptr_t pte_idx = pte_index(va);  in create_pte_mapping()
460 static phys_addr_t __init alloc_pmd_early(uintptr_t va)  in alloc_pmd_early()  argument
462 	BUG_ON((va - kernel_map.virt_addr) >> PUD_SHIFT);  in alloc_pmd_early()
467 static phys_addr_t __init alloc_pmd_fixmap(uintptr_t va)  in alloc_pmd_fixmap()  argument
472 static phys_addr_t __meminit alloc_pmd_late(uintptr_t va)  in alloc_pmd_late()  argument
481 					 uintptr_t va, phys_addr_t pa,  in create_pmd_mapping()  argument
486 	uintptr_t pmd_idx = pmd_index(va);  in create_pmd_mapping()
495 		pte_phys = pt_ops.alloc_pte(va);  in create_pmd_mapping()
504 	create_pte_mapping(ptep, va, pa, sz, prot);  in create_pmd_mapping()
523 static phys_addr_t __init alloc_pud_early(uintptr_t va)  in alloc_pud_early()  argument
526 	BUG_ON((va - kernel_map.virt_addr) >> PGDIR_SHIFT);  in alloc_pud_early()
531 static phys_addr_t __init alloc_pud_fixmap(uintptr_t va)  in alloc_pud_fixmap()  argument
536 static phys_addr_t __meminit alloc_pud_late(uintptr_t va)  in alloc_pud_late()  argument
561 static phys_addr_t __init alloc_p4d_early(uintptr_t va)  in alloc_p4d_early()  argument
564 	BUG_ON((va - kernel_map.virt_addr) >> PGDIR_SHIFT);  in alloc_p4d_early()
569 static phys_addr_t __init alloc_p4d_fixmap(uintptr_t va)  in alloc_p4d_fixmap()  argument
574 static phys_addr_t __meminit alloc_p4d_late(uintptr_t va)  in alloc_p4d_late()  argument
583 static void __meminit create_pud_mapping(pud_t *pudp, uintptr_t va, phys_addr_t pa, phys_addr_t sz,  in create_pud_mapping()  argument
588 	uintptr_t pud_index = pud_index(va);  in create_pud_mapping()
597 		next_phys = pt_ops.alloc_pmd(va);  in create_pud_mapping()
606 	create_pmd_mapping(nextp, va, pa, sz, prot);  in create_pud_mapping()
609 static void __meminit create_p4d_mapping(p4d_t *p4dp, uintptr_t va, phys_addr_t pa, phys_addr_t sz,  in create_p4d_mapping()  argument
614 	uintptr_t p4d_index = p4d_index(va);  in create_p4d_mapping()
623 		next_phys = pt_ops.alloc_pud(va);  in create_p4d_mapping()
632 	create_pud_mapping(nextp, va, pa, sz, prot);  in create_p4d_mapping()
666 void __meminit create_pgd_mapping(pgd_t *pgdp, uintptr_t va, phys_addr_t pa, phys_addr_t sz,  in create_pgd_mapping()  argument
671 	uintptr_t pgd_idx = pgd_index(va);  in create_pgd_mapping()
680 		next_phys = alloc_pgd_next(va);  in create_pgd_mapping()
689 	create_pgd_next_mapping(nextp, va, pa, sz, prot);  in create_pgd_mapping()
692 static uintptr_t __meminit best_map_size(phys_addr_t pa, uintptr_t va, phys_addr_t size)  in best_map_size()  argument
698 	    !(pa & (P4D_SIZE - 1)) && !(va & (P4D_SIZE - 1)) && size >= P4D_SIZE)  in best_map_size()
702 	    !(pa & (PUD_SIZE - 1)) && !(va & (PUD_SIZE - 1)) && size >= PUD_SIZE)  in best_map_size()
706 	    !(pa & (PMD_SIZE - 1)) && !(va & (PMD_SIZE - 1)) && size >= PMD_SIZE)  in best_map_size()
728 static __meminit pgprot_t pgprot_from_va(uintptr_t va)  in pgprot_from_va()  argument
730 	if (is_va_kernel_text(va))  in pgprot_from_va()
739 	if (IS_ENABLED(CONFIG_64BIT) && is_va_kernel_lm_alias_text(va))  in pgprot_from_va()
753 static __meminit pgprot_t pgprot_from_va(uintptr_t va)  in pgprot_from_va()  argument
755 	if (IS_ENABLED(CONFIG_64BIT) && !is_kernel_mapping(va))  in pgprot_from_va()
922 	uintptr_t va, start_va, end_va;  in create_kernel_page_table()  local
926 	for (va = kernel_map.virt_addr; va < end_va; va += PMD_SIZE)  in create_kernel_page_table()
927 		create_pgd_mapping(pgdir, va,  in create_kernel_page_table()
928 				   kernel_map.xiprom + (va - kernel_map.virt_addr),  in create_kernel_page_table()
934 	for (va = start_va; va < end_va; va += PMD_SIZE)  in create_kernel_page_table()
935 		create_pgd_mapping(pgdir, va,  in create_kernel_page_table()
936 				   kernel_map.phys_addr + (va - start_va),  in create_kernel_page_table()
942 	uintptr_t va, end_va;  in create_kernel_page_table()  local
945 	for (va = kernel_map.virt_addr; va < end_va; va += PMD_SIZE)  in create_kernel_page_table()
946 		create_pgd_mapping(pgdir, va,  in create_kernel_page_table()
947 				   kernel_map.phys_addr + (va - kernel_map.virt_addr),  in create_kernel_page_table()
950 					PAGE_KERNEL_EXEC : pgprot_from_va(va));  in create_kernel_page_table()
1249 	uintptr_t va, map_size;  in create_linear_mapping_range()  local
1252 		va = (uintptr_t)__va(pa);  in create_linear_mapping_range()
1254 					    best_map_size(pa, va, end - pa);  in create_linear_mapping_range()
1256 		create_pgd_mapping(swapper_pg_dir, va, pa, map_size,  in create_linear_mapping_range()
1257 				   pgprot ? *pgprot : pgprot_from_va(va));  in create_linear_mapping_range()
1741 static void __meminit remove_pgd_mapping(unsigned long va, unsigned long end, bool is_vmemmap,  in remove_pgd_mapping()  argument
1748 	for (addr = va; addr < end; addr = next) {  in remove_pgd_mapping()
1767 	unsigned long va = (unsigned long)__va(start);  in remove_linear_mapping()  local
1770 	remove_pgd_mapping(va, end, false, NULL);  in remove_linear_mapping()