40 static void pgtable_populate(unsigned long addr, unsigned long end, enum populate_mode mode);
53 static inline void kasan_populate(unsigned long start, unsigned long end, enum populate_mode mode)  in kasan_populate()  argument
57 	pgtable_populate(start, end, mode);  in kasan_populate()
110 					   unsigned long end, enum populate_mode mode)  in kasan_pgd_populate_zero_shadow()  argument
112 	if (mode == POPULATE_KASAN_ZERO_SHADOW &&  in kasan_pgd_populate_zero_shadow()
121 					   unsigned long end, enum populate_mode mode)  in kasan_p4d_populate_zero_shadow()  argument
123 	if (mode == POPULATE_KASAN_ZERO_SHADOW &&  in kasan_p4d_populate_zero_shadow()
132 					   unsigned long end, enum populate_mode mode)  in kasan_pud_populate_zero_shadow()  argument
134 	if (mode == POPULATE_KASAN_ZERO_SHADOW &&  in kasan_pud_populate_zero_shadow()
143 					   unsigned long end, enum populate_mode mode)  in kasan_pmd_populate_zero_shadow()  argument
145 	if (mode == POPULATE_KASAN_ZERO_SHADOW &&  in kasan_pmd_populate_zero_shadow()
153 static bool kasan_pte_populate_zero_shadow(pte_t *pte, enum populate_mode mode)  in kasan_pte_populate_zero_shadow()  argument
155 	if (mode == POPULATE_KASAN_ZERO_SHADOW) {  in kasan_pte_populate_zero_shadow()
168 						  unsigned long end, enum populate_mode mode)  in kasan_pgd_populate_zero_shadow()  argument
174 						  unsigned long end, enum populate_mode mode)  in kasan_p4d_populate_zero_shadow()  argument
180 						  unsigned long end, enum populate_mode mode)  in kasan_pud_populate_zero_shadow()  argument
186 						  unsigned long end, enum populate_mode mode)  in kasan_pmd_populate_zero_shadow()  argument
191 static bool kasan_pte_populate_zero_shadow(pte_t *pte, enum populate_mode mode)  in kasan_pte_populate_zero_shadow()  argument
239 static unsigned long _pa(unsigned long addr, unsigned long size, enum populate_mode mode)  in _pa()  argument
241 	switch (mode) {  in _pa()
265 static bool large_allowed(enum populate_mode mode)  in large_allowed()  argument
267 	return (mode == POPULATE_DIRECT) || (mode == POPULATE_IDENTITY) || (mode == POPULATE_KERNEL);  in large_allowed()
271 			  enum populate_mode mode)  in can_large_pud()  argument
275 	return machine.has_edat2 && large_allowed(mode) &&  in can_large_pud()
277 	       IS_ALIGNED(_pa(addr, size, mode), PUD_SIZE);  in can_large_pud()
281 			  enum populate_mode mode)  in can_large_pmd()  argument
285 	return machine.has_edat1 && large_allowed(mode) &&  in can_large_pmd()
287 	       IS_ALIGNED(_pa(addr, size, mode), PMD_SIZE);  in can_large_pmd()
291 				 enum populate_mode mode)  in pgtable_pte_populate()  argument
299 			if (kasan_pte_populate_zero_shadow(pte, mode))  in pgtable_pte_populate()
301 			entry = __pte(_pa(addr, PAGE_SIZE, mode));  in pgtable_pte_populate()
309 	if (mode == POPULATE_DIRECT)  in pgtable_pte_populate()
314 				 enum populate_mode mode)  in pgtable_pmd_populate()  argument
324 			if (kasan_pmd_populate_zero_shadow(pmd, addr, next, mode))  in pgtable_pmd_populate()
326 			if (can_large_pmd(pmd, addr, next, mode)) {  in pgtable_pmd_populate()
327 				entry = __pmd(_pa(addr, _SEGMENT_SIZE, mode));  in pgtable_pmd_populate()
340 		pgtable_pte_populate(pmd, addr, next, mode);  in pgtable_pmd_populate()
342 	if (mode == POPULATE_DIRECT)  in pgtable_pmd_populate()
347 				 enum populate_mode mode)  in pgtable_pud_populate()  argument
357 			if (kasan_pud_populate_zero_shadow(pud, addr, next, mode))  in pgtable_pud_populate()
359 			if (can_large_pud(pud, addr, next, mode)) {  in pgtable_pud_populate()
360 				entry = __pud(_pa(addr, _REGION3_SIZE, mode));  in pgtable_pud_populate()
373 		pgtable_pmd_populate(pud, addr, next, mode);  in pgtable_pud_populate()
375 	if (mode == POPULATE_DIRECT)  in pgtable_pud_populate()
380 				 enum populate_mode mode)  in pgtable_p4d_populate()  argument
390 			if (kasan_p4d_populate_zero_shadow(p4d, addr, next, mode))  in pgtable_p4d_populate()
395 		pgtable_pud_populate(p4d, addr, next, mode);  in pgtable_p4d_populate()
399 static void pgtable_populate(unsigned long addr, unsigned long end, enum populate_mode mode)  in pgtable_populate()  argument
409 			if (kasan_pgd_populate_zero_shadow(pgd, addr, next, mode))  in pgtable_populate()
415 		if (mode == POPULATE_KASAN_SHALLOW)  in pgtable_populate()
418 		pgtable_p4d_populate(pgd, addr, next, mode);  in pgtable_populate()