| /linux-6.12.1/arch/riscv/mm/ |
| D | kasan_init.c | 115 p4d_t *p4dp, *p; in kasan_populate_p4d() local
123 p4dp = p4d_offset(pgd, vaddr); in kasan_populate_p4d()
128 if (p4d_none(p4dp_get(p4dp)) && IS_ALIGNED(vaddr, P4D_SIZE) && in kasan_populate_p4d()
132 set_p4d(p4dp, pfn_p4d(PFN_DOWN(phys_addr), PAGE_KERNEL)); in kasan_populate_p4d()
138 kasan_populate_pud(p4dp, vaddr, next); in kasan_populate_p4d()
139 } while (p4dp++, vaddr = next, vaddr != end); in kasan_populate_p4d()
165 static void __init kasan_early_clear_pud(p4d_t *p4dp, in kasan_early_clear_pud() argument
172 pudp = (pud_t *)p4dp; in kasan_early_clear_pud()
174 base_pud = pt_ops.get_pud_virt(pfn_to_phys(_p4d_pfn(p4dp_get(p4dp)))); in kasan_early_clear_pud()
193 p4d_t *p4dp, *base_p4d; in kasan_early_clear_p4d() local
[all …]
|
| D | pageattr.c | 135 static int __split_linear_mapping_pud(p4d_t *p4dp, in __split_linear_mapping_pud() argument
142 pudp = pud_offset(p4dp, vaddr); in __split_linear_mapping_pud()
183 p4d_t *p4dp; in __split_linear_mapping_p4d() local
187 p4dp = p4d_offset(pgdp, vaddr); in __split_linear_mapping_p4d()
200 if (p4d_leaf(p4dp_get(p4dp))) { in __split_linear_mapping_p4d()
202 unsigned long pfn = _p4d_pfn(p4dp_get(p4dp)); in __split_linear_mapping_p4d()
203 pgprot_t prot = __pgprot(p4d_val(p4dp_get(p4dp)) & ~_PAGE_PFN_MASK); in __split_linear_mapping_p4d()
227 set_p4d(p4dp, pfn_p4d(page_to_pfn(pud_page), PAGE_TABLE)); in __split_linear_mapping_p4d()
230 ret = __split_linear_mapping_pud(p4dp, vaddr, next); in __split_linear_mapping_p4d()
233 } while (p4dp++, vaddr = next, vaddr != end); in __split_linear_mapping_p4d()
|
| /linux-6.12.1/arch/arm64/include/asm/ |
| D | pgalloc.h | 45 static inline void __p4d_populate(p4d_t *p4dp, phys_addr_t pudp, p4dval_t prot) in __p4d_populate() argument
48 set_p4d(p4dp, __p4d(__phys_to_p4d_val(pudp) | prot)); in __p4d_populate()
51 static inline void p4d_populate(struct mm_struct *mm, p4d_t *p4dp, pud_t *pudp) in p4d_populate() argument
56 __p4d_populate(p4dp, __pa(pudp), p4dval); in p4d_populate()
66 static inline void __p4d_populate(p4d_t *p4dp, phys_addr_t pudp, p4dval_t prot) in __p4d_populate() argument
74 static inline void __pgd_populate(pgd_t *pgdp, phys_addr_t p4dp, pgdval_t prot) in __pgd_populate() argument
77 set_pgd(pgdp, __pgd(__phys_to_pgd_val(p4dp) | prot)); in __pgd_populate()
80 static inline void pgd_populate(struct mm_struct *mm, pgd_t *pgdp, p4d_t *p4dp) in pgd_populate() argument
85 __pgd_populate(pgdp, __pa(p4dp), pgdval); in pgd_populate()
107 static inline void __pgd_populate(pgd_t *pgdp, phys_addr_t p4dp, pgdval_t prot) in __pgd_populate() argument
|
| D | pgtable.h | 903 static inline void set_p4d(p4d_t *p4dp, p4d_t p4d) in set_p4d() argument
905 if (in_swapper_pgdir(p4dp)) { in set_p4d()
906 set_swapper_pgd((pgd_t *)p4dp, __pgd(p4d_val(p4d))); in set_p4d()
910 WRITE_ONCE(*p4dp, p4d); in set_p4d()
915 static inline void p4d_clear(p4d_t *p4dp) in p4d_clear() argument
918 set_p4d(p4dp, __p4d(0)); in p4d_clear()
928 static inline pud_t *p4d_to_folded_pud(p4d_t *p4dp, unsigned long addr) in p4d_to_folded_pud() argument
930 return (pud_t *)PTR_ALIGN_DOWN(p4dp, PAGE_SIZE) + pud_index(addr); in p4d_to_folded_pud()
938 static inline phys_addr_t pud_offset_phys(p4d_t *p4dp, unsigned long addr) in pud_offset_phys() argument
942 return p4d_page_paddr(READ_ONCE(*p4dp)) + pud_index(addr) * sizeof(pud_t); in pud_offset_phys()
[all …]
|
| /linux-6.12.1/arch/powerpc/mm/nohash/ |
| D | book3e_pgtable.c | 77 p4d_t *p4dp; in map_kernel_page() local
85 p4dp = p4d_offset(pgdp, ea); in map_kernel_page()
86 pudp = pud_alloc(&init_mm, p4dp, ea); in map_kernel_page()
97 p4dp = p4d_offset(pgdp, ea); in map_kernel_page()
98 if (p4d_none(*p4dp)) { in map_kernel_page()
100 p4d_populate(&init_mm, p4dp, pudp); in map_kernel_page()
102 pudp = pud_offset(p4dp, ea); in map_kernel_page()
|
| /linux-6.12.1/arch/arm64/mm/ |
| D | fixmap.c | 72 static void __init early_fixmap_init_pud(p4d_t *p4dp, unsigned long addr, in early_fixmap_init_pud() argument
75 p4d_t p4d = READ_ONCE(*p4dp); in early_fixmap_init_pud()
89 __p4d_populate(p4dp, __pa_symbol(bm_pud), P4D_TYPE_TABLE); in early_fixmap_init_pud()
91 pudp = pud_offset_kimg(p4dp, addr); in early_fixmap_init_pud()
107 p4d_t *p4dp = p4d_offset_kimg(pgdp, addr); in early_fixmap_init() local
109 early_fixmap_init_pud(p4dp, addr, end); in early_fixmap_init()
|
| D | kasan_init.c | 89 static pud_t *__init kasan_pud_offset(p4d_t *p4dp, unsigned long addr, int node, in kasan_pud_offset() argument
92 if (p4d_none(READ_ONCE(*p4dp))) { in kasan_pud_offset()
96 __p4d_populate(p4dp, pud_phys, P4D_TYPE_TABLE); in kasan_pud_offset()
99 return early ? pud_offset_kimg(p4dp, addr) : pud_offset(p4dp, addr); in kasan_pud_offset()
144 static void __init kasan_pud_populate(p4d_t *p4dp, unsigned long addr, in kasan_pud_populate() argument
148 pud_t *pudp = kasan_pud_offset(p4dp, addr, node, early); in kasan_pud_populate()
160 p4d_t *p4dp = kasan_p4d_offset(pgdp, addr, node, early); in kasan_p4d_populate() local
164 kasan_pud_populate(p4dp, addr, next, node, early); in kasan_p4d_populate()
165 } while (p4dp++, addr = next, addr != end && p4d_none(READ_ONCE(*p4dp))); in kasan_p4d_populate()
|
| D | mmu.c | 326 static void alloc_init_pud(p4d_t *p4dp, unsigned long addr, unsigned long end, in alloc_init_pud() argument
332 p4d_t p4d = READ_ONCE(*p4dp); in alloc_init_pud()
346 __p4d_populate(p4dp, pud_phys, p4dval); in alloc_init_pud()
349 pudp = pud_set_fixmap_offset(p4dp, addr); in alloc_init_pud()
391 p4d_t *p4dp; in alloc_init_p4d() local
401 p4dp = p4d_set_fixmap(p4d_phys); in alloc_init_p4d()
402 init_clear_pgtable(p4dp); in alloc_init_p4d()
403 p4dp += p4d_index(addr); in alloc_init_p4d()
407 p4dp = p4d_set_fixmap_offset(pgdp, addr); in alloc_init_p4d()
411 p4d_t old_p4d = READ_ONCE(*p4dp); in alloc_init_p4d()
[all …]
|
| D | hugetlbpage.c | 86 p4d_t *p4dp; in find_num_contig() local
91 p4dp = p4d_offset(pgdp, addr); in find_num_contig()
92 pudp = pud_offset(p4dp, addr); in find_num_contig()
259 p4d_t *p4dp; in huge_pte_alloc() local
265 p4dp = p4d_alloc(mm, pgdp, addr); in huge_pte_alloc()
266 if (!p4dp) in huge_pte_alloc()
269 pudp = pud_alloc(mm, p4dp, addr); in huge_pte_alloc()
300 p4d_t *p4dp; in huge_pte_offset() local
308 p4dp = p4d_offset(pgdp, addr); in huge_pte_offset()
309 if (!p4d_present(READ_ONCE(*p4dp))) in huge_pte_offset()
[all …]
|
| D | pageattr.c | 216 p4d_t *p4dp; in kernel_page_present() local
226 p4dp = p4d_offset(pgdp, addr); in kernel_page_present()
227 if (p4d_none(READ_ONCE(*p4dp))) in kernel_page_present()
230 pudp = pud_offset(p4dp, addr); in kernel_page_present()
|
| /linux-6.12.1/arch/loongarch/mm/ |
| D | kasan_init.c | 144 static pud_t *__init kasan_pud_offset(p4d_t *p4dp, unsigned long addr, int node, bool early) in kasan_pud_offset() argument
146 if (__p4d_none(early, p4dp_get(p4dp))) { in kasan_pud_offset()
151 p4d_populate(&init_mm, p4dp, (pud_t *)__va(pud_phys)); in kasan_pud_offset()
154 return pud_offset(p4dp, addr); in kasan_pud_offset()
197 static void __init kasan_pud_populate(p4d_t *p4dp, unsigned long addr, in kasan_pud_populate() argument
201 pud_t *pudp = kasan_pud_offset(p4dp, addr, node, early); in kasan_pud_populate()
213 p4d_t *p4dp = kasan_p4d_offset(pgdp, addr, node, early); in kasan_p4d_populate() local
217 kasan_pud_populate(p4dp, addr, next, node, early); in kasan_p4d_populate()
218 } while (p4dp++, addr = next, addr != end && __p4d_none(early, READ_ONCE(*p4dp))); in kasan_p4d_populate()
|
| /linux-6.12.1/arch/s390/mm/ |
| D | hugetlbpage.c | 195 p4d_t *p4dp; in huge_pte_alloc() local
200 p4dp = p4d_alloc(mm, pgdp, addr); in huge_pte_alloc()
201 if (p4dp) { in huge_pte_alloc()
202 pudp = pud_alloc(mm, p4dp, addr); in huge_pte_alloc()
217 p4d_t *p4dp; in huge_pte_offset() local
223 p4dp = p4d_offset(pgdp, addr); in huge_pte_offset()
224 if (p4d_present(*p4dp)) { in huge_pte_offset()
225 pudp = pud_offset(p4dp, addr); in huge_pte_offset()
|
| D | pageattr.c | 302 p4d_t *p4dp; in walk_p4d_level() local
305 p4dp = p4d_offset(pgd, addr); in walk_p4d_level()
307 if (p4d_none(*p4dp)) in walk_p4d_level()
310 rc = walk_pud_level(p4dp, addr, next, flags); in walk_p4d_level()
311 p4dp++; in walk_p4d_level()
|
| /linux-6.12.1/arch/powerpc/mm/kasan/ |
| D | init_book3e_64.c | 35 p4d_t *p4dp; in kasan_map_kernel_page() local
41 p4dp = p4d_offset(pgdp, ea); in kasan_map_kernel_page()
42 if (kasan_pud_table(*p4dp)) { in kasan_map_kernel_page()
45 p4d_populate(&init_mm, p4dp, pudp); in kasan_map_kernel_page()
47 pudp = pud_offset(p4dp, ea); in kasan_map_kernel_page()
|
| /linux-6.12.1/arch/x86/include/asm/ |
| D | pgtable_64.h | 142 static inline void native_set_p4d(p4d_t *p4dp, p4d_t p4d) in native_set_p4d() argument
148 WRITE_ONCE(*p4dp, p4d); in native_set_p4d()
153 pgd = pti_set_user_pgtbl((pgd_t *)p4dp, pgd); in native_set_p4d()
154 WRITE_ONCE(*p4dp, native_make_p4d(native_pgd_val(pgd))); in native_set_p4d()
|
| /linux-6.12.1/arch/powerpc/mm/book3s64/ |
| D | radix_pgtable.c | 79 p4d_t *p4dp; in early_map_kernel_page() local
85 p4dp = p4d_offset(pgdp, ea); in early_map_kernel_page()
86 if (p4d_none(*p4dp)) { in early_map_kernel_page()
89 p4d_populate(&init_mm, p4dp, pudp); in early_map_kernel_page()
91 pudp = pud_offset(p4dp, ea); in early_map_kernel_page()
131 p4d_t *p4dp; in __map_kernel_page() local
154 p4dp = p4d_offset(pgdp, ea); in __map_kernel_page()
155 pudp = pud_alloc(&init_mm, p4dp, ea); in __map_kernel_page()
192 p4d_t *p4dp; in radix__change_memory_range() local
205 p4dp = p4d_offset(pgdp, idx); in radix__change_memory_range()
[all …]
|
| /linux-6.12.1/arch/sparc/mm/ |
| D | srmmu.c | 498 p4d_t *p4dp; in srmmu_mapioaddr() local
506 p4dp = p4d_offset(pgdp, virt_addr); in srmmu_mapioaddr()
507 pudp = pud_offset(p4dp, virt_addr); in srmmu_mapioaddr()
537 p4d_t *p4dp; in srmmu_unmapioaddr() local
544 p4dp = p4d_offset(pgdp, virt_addr); in srmmu_unmapioaddr()
545 pudp = pud_offset(p4dp, virt_addr); in srmmu_unmapioaddr()
684 p4d_t *p4dp; in srmmu_early_allocate_ptable_skeleton() local
691 p4dp = p4d_offset(pgdp, start); in srmmu_early_allocate_ptable_skeleton()
692 pudp = pud_offset(p4dp, start); in srmmu_early_allocate_ptable_skeleton()
719 p4d_t *p4dp; in srmmu_allocate_ptable_skeleton() local
[all …]
|
| D | fault_64.c | 82 p4d_t *p4dp; in get_user_insn() local
91 p4dp = p4d_offset(pgdp, tpc); in get_user_insn()
92 if (p4d_none(*p4dp) || unlikely(p4d_bad(*p4dp))) in get_user_insn()
94 pudp = pud_offset(p4dp, tpc); in get_user_insn()
|
| /linux-6.12.1/arch/powerpc/include/asm/nohash/64/ |
| D | pgtable-4k.h | 67 static inline void p4d_clear(p4d_t *p4dp) in p4d_clear() argument
69 *p4dp = __p4d(0); in p4d_clear()
|
| /linux-6.12.1/arch/arc/include/asm/ |
| D | pgalloc.h | 75 static inline void p4d_populate(struct mm_struct *mm, p4d_t *p4dp, pud_t *pudp) in p4d_populate() argument
77 set_p4d(p4dp, __p4d((unsigned long)pudp)); in p4d_populate()
|
| /linux-6.12.1/include/trace/events/ |
| D | xen.h | 224 TP_PROTO(p4d_t *p4dp, p4d_t *user_p4dp, p4d_t p4dval),
225 TP_ARGS(p4dp, user_p4dp, p4dval),
227 __field(p4d_t *, p4dp)
231 TP_fast_assign(__entry->p4dp = p4dp;
235 __entry->p4dp, __entry->user_p4dp,
|
| /linux-6.12.1/arch/arm/mm/ |
| D | kasan_init.c | 128 p4d_t *p4dp; in kasan_pgd_populate() local
155 p4dp = p4d_offset(pgdp, addr); in kasan_pgd_populate()
156 pudp = pud_offset(p4dp, addr); in kasan_pgd_populate()
|
| /linux-6.12.1/mm/ |
| D | debug_vm_pgtable.c | 51 p4d_t *p4dp; member
529 p4d_t p4d = p4dp_get(args->p4dp); in p4d_clear_tests()
536 p4d_clear(args->p4dp); in p4d_clear_tests()
537 p4d = p4dp_get(args->p4dp); in p4d_clear_tests()
554 p4d_clear(args->p4dp); in p4d_populate_tests()
555 p4d_populate(args->mm, args->p4dp, args->start_pudp); in p4d_populate_tests()
556 p4d = p4dp_get(args->p4dp); in p4d_populate_tests()
586 p4d_clear(args->p4dp); in pgd_populate_tests()
1222 args->p4dp = p4d_alloc(args->mm, args->pgdp, args->vaddr); in init_args()
1223 if (!args->p4dp) { in init_args()
[all …]
|
| /linux-6.12.1/arch/x86/boot/compressed/ |
| D | ident_map_64.c | 260 p4d_t *p4dp; in set_clr_page_flags() local
283 p4dp = p4d_offset(pgdp, address); in set_clr_page_flags()
284 pudp = pud_offset(p4dp, address); in set_clr_page_flags()
|
| /linux-6.12.1/arch/powerpc/mm/ |
| D | pgtable.c | 433 p4d_t p4d, *p4dp; in __find_linux_pte() local
458 p4dp = p4d_offset(pgdp, ea); in __find_linux_pte()
459 p4d = READ_ONCE(*p4dp); in __find_linux_pte()
466 ret_pte = (pte_t *)p4dp; in __find_linux_pte()
|