92 static void __ref zero_pte_populate(pmd_t *pmd, unsigned long addr,  in zero_pte_populate()  argument
95 	pte_t *pte = pte_offset_kernel(pmd, addr);  in zero_pte_populate()
102 	while (addr + PAGE_SIZE <= end) {  in zero_pte_populate()
103 		set_pte_at(&init_mm, addr, pte, zero_pte);  in zero_pte_populate()
104 		addr += PAGE_SIZE;  in zero_pte_populate()
105 		pte = pte_offset_kernel(pmd, addr);  in zero_pte_populate()
109 void __weak __meminit kernel_pte_init(void *addr)  in kernel_pte_init()  argument
113 static int __ref zero_pmd_populate(pud_t *pud, unsigned long addr,  in zero_pmd_populate()  argument
116 	pmd_t *pmd = pmd_offset(pud, addr);  in zero_pmd_populate()
120 		next = pmd_addr_end(addr, end);  in zero_pmd_populate()
122 		if (IS_ALIGNED(addr, PMD_SIZE) && end - addr >= PMD_SIZE) {  in zero_pmd_populate()
142 		zero_pte_populate(pmd, addr, next);  in zero_pmd_populate()
143 	} while (pmd++, addr = next, addr != end);  in zero_pmd_populate()
148 void __weak __meminit pmd_init(void *addr)  in pmd_init()  argument
152 static int __ref zero_pud_populate(p4d_t *p4d, unsigned long addr,  in zero_pud_populate()  argument
155 	pud_t *pud = pud_offset(p4d, addr);  in zero_pud_populate()
159 		next = pud_addr_end(addr, end);  in zero_pud_populate()
160 		if (IS_ALIGNED(addr, PUD_SIZE) && end - addr >= PUD_SIZE) {  in zero_pud_populate()
165 			pmd = pmd_offset(pud, addr);  in zero_pud_populate()
175 				p = pmd_alloc(&init_mm, pud, addr);  in zero_pud_populate()
184 		zero_pmd_populate(pud, addr, next);  in zero_pud_populate()
185 	} while (pud++, addr = next, addr != end);  in zero_pud_populate()
190 void __weak __meminit pud_init(void *addr)  in pud_init()  argument
194 static int __ref zero_p4d_populate(pgd_t *pgd, unsigned long addr,  in zero_p4d_populate()  argument
197 	p4d_t *p4d = p4d_offset(pgd, addr);  in zero_p4d_populate()
201 		next = p4d_addr_end(addr, end);  in zero_p4d_populate()
202 		if (IS_ALIGNED(addr, P4D_SIZE) && end - addr >= P4D_SIZE) {  in zero_p4d_populate()
208 			pud = pud_offset(p4d, addr);  in zero_p4d_populate()
211 			pmd = pmd_offset(pud, addr);  in zero_p4d_populate()
221 				p = pud_alloc(&init_mm, p4d, addr);  in zero_p4d_populate()
230 		zero_pud_populate(p4d, addr, next);  in zero_p4d_populate()
231 	} while (p4d++, addr = next, addr != end);  in zero_p4d_populate()
245 	unsigned long addr = (unsigned long)shadow_start;  in kasan_populate_early_shadow()  local
247 	pgd_t *pgd = pgd_offset_k(addr);  in kasan_populate_early_shadow()
251 		next = pgd_addr_end(addr, end);  in kasan_populate_early_shadow()
253 		if (IS_ALIGNED(addr, PGDIR_SIZE) && end - addr >= PGDIR_SIZE) {  in kasan_populate_early_shadow()
268 			p4d = p4d_offset(pgd, addr);  in kasan_populate_early_shadow()
271 			pud = pud_offset(p4d, addr);  in kasan_populate_early_shadow()
274 			pmd = pmd_offset(pud, addr);  in kasan_populate_early_shadow()
284 				p = p4d_alloc(&init_mm, pgd, addr);  in kasan_populate_early_shadow()
292 		zero_p4d_populate(pgd, addr, next);  in kasan_populate_early_shadow()
293 	} while (pgd++, addr = next, addr != end);  in kasan_populate_early_shadow()
358 static void kasan_remove_pte_table(pte_t *pte, unsigned long addr,  in kasan_remove_pte_table()  argument
364 	for (; addr < end; addr = next, pte++) {  in kasan_remove_pte_table()
365 		next = (addr + PAGE_SIZE) & PAGE_MASK;  in kasan_remove_pte_table()
376 		pte_clear(&init_mm, addr, pte);  in kasan_remove_pte_table()
380 static void kasan_remove_pmd_table(pmd_t *pmd, unsigned long addr,  in kasan_remove_pmd_table()  argument
385 	for (; addr < end; addr = next, pmd++) {  in kasan_remove_pmd_table()
388 		next = pmd_addr_end(addr, end);  in kasan_remove_pmd_table()
394 			if (IS_ALIGNED(addr, PMD_SIZE) &&  in kasan_remove_pmd_table()
400 		pte = pte_offset_kernel(pmd, addr);  in kasan_remove_pmd_table()
401 		kasan_remove_pte_table(pte, addr, next);  in kasan_remove_pmd_table()
406 static void kasan_remove_pud_table(pud_t *pud, unsigned long addr,  in kasan_remove_pud_table()  argument
411 	for (; addr < end; addr = next, pud++) {  in kasan_remove_pud_table()
414 		next = pud_addr_end(addr, end);  in kasan_remove_pud_table()
420 			if (IS_ALIGNED(addr, PUD_SIZE) &&  in kasan_remove_pud_table()
426 		pmd = pmd_offset(pud, addr);  in kasan_remove_pud_table()
428 		kasan_remove_pmd_table(pmd, addr, next);  in kasan_remove_pud_table()
433 static void kasan_remove_p4d_table(p4d_t *p4d, unsigned long addr,  in kasan_remove_p4d_table()  argument
438 	for (; addr < end; addr = next, p4d++) {  in kasan_remove_p4d_table()
441 		next = p4d_addr_end(addr, end);  in kasan_remove_p4d_table()
447 			if (IS_ALIGNED(addr, P4D_SIZE) &&  in kasan_remove_p4d_table()
453 		pud = pud_offset(p4d, addr);  in kasan_remove_p4d_table()
454 		kasan_remove_pud_table(pud, addr, next);  in kasan_remove_p4d_table()
461 	unsigned long addr, end, next;  in kasan_remove_zero_shadow()  local
464 	addr = (unsigned long)kasan_mem_to_shadow(start);  in kasan_remove_zero_shadow()
465 	end = addr + (size >> KASAN_SHADOW_SCALE_SHIFT);  in kasan_remove_zero_shadow()
471 	for (; addr < end; addr = next) {  in kasan_remove_zero_shadow()
474 		next = pgd_addr_end(addr, end);  in kasan_remove_zero_shadow()
476 		pgd = pgd_offset_k(addr);  in kasan_remove_zero_shadow()
481 			if (IS_ALIGNED(addr, PGDIR_SIZE) &&  in kasan_remove_zero_shadow()
488 		p4d = p4d_offset(pgd, addr);  in kasan_remove_zero_shadow()
489 		kasan_remove_p4d_table(p4d, addr, next);  in kasan_remove_zero_shadow()