Home
last modified time | relevance | path

Searched refs:ptdesc (Results 1 – 24 of 24) sorted by relevance

/linux-6.12.1/arch/powerpc/mm/
Dpgtable-frag.c21 struct ptdesc *ptdesc; in pte_frag_destroy() local
23 ptdesc = virt_to_ptdesc(pte_frag); in pte_frag_destroy()
27 if (atomic_sub_and_test(PTE_FRAG_NR - count, &ptdesc->pt_frag_refcount)) { in pte_frag_destroy()
28 pagetable_pte_dtor(ptdesc); in pte_frag_destroy()
29 pagetable_free(ptdesc); in pte_frag_destroy()
58 struct ptdesc *ptdesc; in __alloc_for_ptecache() local
61 ptdesc = pagetable_alloc(PGALLOC_GFP | __GFP_ACCOUNT, 0); in __alloc_for_ptecache()
62 if (!ptdesc) in __alloc_for_ptecache()
64 if (!pagetable_pte_ctor(ptdesc)) { in __alloc_for_ptecache()
65 pagetable_free(ptdesc); in __alloc_for_ptecache()
[all …]
/linux-6.12.1/include/asm-generic/
Dpgalloc.h21 struct ptdesc *ptdesc = pagetable_alloc_noprof(GFP_PGTABLE_KERNEL & in __pte_alloc_one_kernel_noprof() local
24 if (!ptdesc) in __pte_alloc_one_kernel_noprof()
26 return ptdesc_address(ptdesc); in __pte_alloc_one_kernel_noprof()
68 struct ptdesc *ptdesc; in __pte_alloc_one_noprof() local
70 ptdesc = pagetable_alloc_noprof(gfp, 0); in __pte_alloc_one_noprof()
71 if (!ptdesc) in __pte_alloc_one_noprof()
73 if (!pagetable_pte_ctor(ptdesc)) { in __pte_alloc_one_noprof()
74 pagetable_free(ptdesc); in __pte_alloc_one_noprof()
78 return ptdesc_page(ptdesc); in __pte_alloc_one_noprof()
110 struct ptdesc *ptdesc = page_ptdesc(pte_page); in pte_free() local
[all …]
Dtlb.h497 static inline void tlb_remove_page_ptdesc(struct mmu_gather *tlb, struct ptdesc *pt) in tlb_remove_page_ptdesc()
/linux-6.12.1/arch/m68k/include/asm/
Dmcf_pgalloc.h17 struct ptdesc *ptdesc = pagetable_alloc((GFP_DMA | __GFP_ZERO) & in pte_alloc_one_kernel() local
20 if (!ptdesc) in pte_alloc_one_kernel()
23 return ptdesc_address(ptdesc); in pte_alloc_one_kernel()
38 struct ptdesc *ptdesc = virt_to_ptdesc(pgtable); in __pte_free_tlb() local
40 pagetable_pte_dtor(ptdesc); in __pte_free_tlb()
41 pagetable_free(ptdesc); in __pte_free_tlb()
46 struct ptdesc *ptdesc = pagetable_alloc(GFP_DMA | __GFP_ZERO, 0); in pte_alloc_one() local
49 if (!ptdesc) in pte_alloc_one()
51 if (!pagetable_pte_ctor(ptdesc)) { in pte_alloc_one()
52 pagetable_free(ptdesc); in pte_alloc_one()
[all …]
/linux-6.12.1/arch/s390/mm/
Dpgalloc.c46 struct ptdesc *ptdesc = pagetable_alloc(GFP_KERNEL, CRST_ALLOC_ORDER); in crst_table_alloc() local
49 if (!ptdesc) in crst_table_alloc()
51 table = ptdesc_to_virt(ptdesc); in crst_table_alloc()
140 struct ptdesc *page_table_alloc_pgste(struct mm_struct *mm) in page_table_alloc_pgste()
142 struct ptdesc *ptdesc; in page_table_alloc_pgste() local
145 ptdesc = pagetable_alloc(GFP_KERNEL, 0); in page_table_alloc_pgste()
146 if (ptdesc) { in page_table_alloc_pgste()
147 table = (u64 *)ptdesc_to_virt(ptdesc); in page_table_alloc_pgste()
152 return ptdesc; in page_table_alloc_pgste()
155 void page_table_free_pgste(struct ptdesc *ptdesc) in page_table_free_pgste() argument
[all …]
Dgmap.c209 struct ptdesc *ptdesc, *n; in gmap_free() local
212 list_for_each_entry_safe(ptdesc, n, &gmap->pt_list, pt_list) in gmap_free()
213 page_table_free_pgste(ptdesc); in gmap_free()
1353 struct ptdesc *ptdesc; in gmap_unshadow_pgt() local
1366 ptdesc = page_ptdesc(phys_to_page(pgt)); in gmap_unshadow_pgt()
1367 list_del(&ptdesc->pt_list); in gmap_unshadow_pgt()
1368 page_table_free_pgste(ptdesc); in gmap_unshadow_pgt()
1382 struct ptdesc *ptdesc; in __gmap_unshadow_sgt() local
1394 ptdesc = page_ptdesc(phys_to_page(pgt)); in __gmap_unshadow_sgt()
1395 list_del(&ptdesc->pt_list); in __gmap_unshadow_sgt()
[all …]
/linux-6.12.1/arch/arm64/include/asm/
Dtlb.h83 struct ptdesc *ptdesc = page_ptdesc(pte); in __pte_free_tlb() local
85 pagetable_pte_dtor(ptdesc); in __pte_free_tlb()
86 tlb_remove_ptdesc(tlb, ptdesc); in __pte_free_tlb()
93 struct ptdesc *ptdesc = virt_to_ptdesc(pmdp); in __pmd_free_tlb() local
95 pagetable_pmd_dtor(ptdesc); in __pmd_free_tlb()
96 tlb_remove_ptdesc(tlb, ptdesc); in __pmd_free_tlb()
104 struct ptdesc *ptdesc = virt_to_ptdesc(pudp); in __pud_free_tlb() local
109 pagetable_pud_dtor(ptdesc); in __pud_free_tlb()
110 tlb_remove_ptdesc(tlb, ptdesc); in __pud_free_tlb()
/linux-6.12.1/arch/loongarch/include/asm/
Dpgalloc.h69 struct ptdesc *ptdesc; in pmd_alloc_one() local
71 ptdesc = pagetable_alloc(GFP_KERNEL_ACCOUNT, 0); in pmd_alloc_one()
72 if (!ptdesc) in pmd_alloc_one()
75 if (!pagetable_pmd_ctor(ptdesc)) { in pmd_alloc_one()
76 pagetable_free(ptdesc); in pmd_alloc_one()
80 pmd = ptdesc_address(ptdesc); in pmd_alloc_one()
94 struct ptdesc *ptdesc = pagetable_alloc(GFP_KERNEL & ~__GFP_HIGHMEM, 0); in pud_alloc_one() local
96 if (!ptdesc) in pud_alloc_one()
98 pagetable_pud_ctor(ptdesc); in pud_alloc_one()
99 pud = ptdesc_address(ptdesc); in pud_alloc_one()
/linux-6.12.1/arch/mips/include/asm/
Dpgalloc.h68 struct ptdesc *ptdesc; in pmd_alloc_one() local
70 ptdesc = pagetable_alloc(GFP_KERNEL_ACCOUNT, PMD_TABLE_ORDER); in pmd_alloc_one()
71 if (!ptdesc) in pmd_alloc_one()
74 if (!pagetable_pmd_ctor(ptdesc)) { in pmd_alloc_one()
75 pagetable_free(ptdesc); in pmd_alloc_one()
79 pmd = ptdesc_address(ptdesc); in pmd_alloc_one()
93 struct ptdesc *ptdesc = pagetable_alloc(GFP_KERNEL & ~__GFP_HIGHMEM, in pud_alloc_one() local
96 if (!ptdesc) in pud_alloc_one()
98 pagetable_pud_ctor(ptdesc); in pud_alloc_one()
99 pud = ptdesc_address(ptdesc); in pud_alloc_one()
/linux-6.12.1/arch/arm/include/asm/
Dtlb.h42 struct ptdesc *ptdesc = page_ptdesc(pte); in __pte_free_tlb() local
44 pagetable_pte_dtor(ptdesc); in __pte_free_tlb()
55 tlb_remove_ptdesc(tlb, ptdesc); in __pte_free_tlb()
62 struct ptdesc *ptdesc = virt_to_ptdesc(pmdp); in __pmd_free_tlb() local
64 pagetable_pmd_dtor(ptdesc); in __pmd_free_tlb()
65 tlb_remove_ptdesc(tlb, ptdesc); in __pmd_free_tlb()
/linux-6.12.1/arch/riscv/include/asm/
Dpgalloc.h111 struct ptdesc *ptdesc = virt_to_ptdesc(pud); in __pud_free_tlb() local
113 pagetable_pud_dtor(ptdesc); in __pud_free_tlb()
114 riscv_tlb_remove_ptdesc(tlb, ptdesc); in __pud_free_tlb()
178 struct ptdesc *ptdesc = virt_to_ptdesc(pmd); in __pmd_free_tlb() local
180 pagetable_pmd_dtor(ptdesc); in __pmd_free_tlb()
181 riscv_tlb_remove_ptdesc(tlb, ptdesc); in __pmd_free_tlb()
189 struct ptdesc *ptdesc = page_ptdesc(pte); in __pte_free_tlb() local
191 pagetable_pte_dtor(ptdesc); in __pte_free_tlb()
192 riscv_tlb_remove_ptdesc(tlb, ptdesc); in __pte_free_tlb()
/linux-6.12.1/arch/x86/mm/
Dpgtable.c63 struct ptdesc *ptdesc = virt_to_ptdesc(pmd); in ___pmd_free_tlb() local
72 pagetable_pmd_dtor(ptdesc); in ___pmd_free_tlb()
73 paravirt_tlb_remove_table(tlb, ptdesc_page(ptdesc)); in ___pmd_free_tlb()
79 struct ptdesc *ptdesc = virt_to_ptdesc(pud); in ___pud_free_tlb() local
81 pagetable_pud_dtor(ptdesc); in ___pud_free_tlb()
98 struct ptdesc *ptdesc = virt_to_ptdesc(pgd); in pgd_list_add() local
100 list_add(&ptdesc->pt_list, &pgd_list); in pgd_list_add()
105 struct ptdesc *ptdesc = virt_to_ptdesc(pgd); in pgd_list_del() local
107 list_del(&ptdesc->pt_list); in pgd_list_del()
219 struct ptdesc *ptdesc; in free_pmds() local
[all …]
/linux-6.12.1/arch/powerpc/mm/book3s64/
Dpgtable.c409 struct ptdesc *ptdesc; in __alloc_for_pmdcache() local
414 ptdesc = pagetable_alloc(gfp, 0); in __alloc_for_pmdcache()
415 if (!ptdesc) in __alloc_for_pmdcache()
417 if (!pagetable_pmd_ctor(ptdesc)) { in __alloc_for_pmdcache()
418 pagetable_free(ptdesc); in __alloc_for_pmdcache()
422 atomic_set(&ptdesc->pt_frag_refcount, 1); in __alloc_for_pmdcache()
424 ret = ptdesc_address(ptdesc); in __alloc_for_pmdcache()
439 atomic_set(&ptdesc->pt_frag_refcount, PMD_FRAG_NR); in __alloc_for_pmdcache()
460 struct ptdesc *ptdesc = virt_to_ptdesc(pmd); in pmd_fragment_free() local
462 if (pagetable_is_reserved(ptdesc)) in pmd_fragment_free()
[all …]
Dmmu_context.c249 struct ptdesc *ptdesc; in pmd_frag_destroy() local
251 ptdesc = virt_to_ptdesc(pmd_frag); in pmd_frag_destroy()
255 if (atomic_sub_and_test(PMD_FRAG_NR - count, &ptdesc->pt_frag_refcount)) { in pmd_frag_destroy()
256 pagetable_pmd_dtor(ptdesc); in pmd_frag_destroy()
257 pagetable_free(ptdesc); in pmd_frag_destroy()
/linux-6.12.1/arch/mips/mm/
Dpgtable.c14 struct ptdesc *ptdesc = pagetable_alloc(GFP_KERNEL & ~__GFP_HIGHMEM, in pgd_alloc() local
17 if (ptdesc) { in pgd_alloc()
18 ret = ptdesc_address(ptdesc); in pgd_alloc()
/linux-6.12.1/include/linux/
Dmm.h2854 static inline struct ptdesc *virt_to_ptdesc(const void *x) in virt_to_ptdesc()
2859 static inline void *ptdesc_to_virt(const struct ptdesc *pt) in ptdesc_to_virt()
2864 static inline void *ptdesc_address(const struct ptdesc *pt) in ptdesc_address()
2869 static inline bool pagetable_is_reserved(struct ptdesc *pt) in pagetable_is_reserved()
2884 static inline struct ptdesc *pagetable_alloc_noprof(gfp_t gfp, unsigned int order) in pagetable_alloc_noprof()
2899 static inline void pagetable_free(struct ptdesc *pt) in pagetable_free()
2909 bool ptlock_alloc(struct ptdesc *ptdesc);
2910 void ptlock_free(struct ptdesc *ptdesc);
2912 static inline spinlock_t *ptlock_ptr(struct ptdesc *ptdesc) in ptlock_ptr() argument
2914 return ptdesc->ptl; in ptlock_ptr()
[all …]
Dmm_types.h457 struct ptdesc { struct
492 static_assert(offsetof(struct page, pg) == offsetof(struct ptdesc, pt)) argument
505 static_assert(sizeof(struct ptdesc) <= sizeof(struct page));
508 const struct ptdesc *: (const struct page *)(pt), \
509 struct ptdesc *: (struct page *)(pt)))
512 const struct ptdesc *: (const struct folio *)(pt), \
513 struct ptdesc *: (struct folio *)(pt)))
516 const struct page *: (const struct ptdesc *)(p), \
517 struct page *: (struct ptdesc *)(p)))
/linux-6.12.1/arch/loongarch/mm/
Dpgtable.c27 struct ptdesc *ptdesc = pagetable_alloc(GFP_KERNEL & ~__GFP_HIGHMEM, 0); in pgd_alloc() local
29 if (ptdesc) { in pgd_alloc()
30 ret = (pgd_t *)ptdesc_address(ptdesc); in pgd_alloc()
/linux-6.12.1/arch/s390/include/asm/
Dpgalloc.h26 struct ptdesc *page_table_alloc_pgste(struct mm_struct *mm);
28 void page_table_free_pgste(struct ptdesc *ptdesc);
/linux-6.12.1/arch/riscv/mm/
Dinit.c394 struct ptdesc *ptdesc = pagetable_alloc(GFP_KERNEL & ~__GFP_HIGHMEM, 0); in alloc_pte_late() local
396 BUG_ON(!ptdesc || !pagetable_pte_ctor(ptdesc)); in alloc_pte_late()
397 return __pa((pte_t *)ptdesc_address(ptdesc)); in alloc_pte_late()
474 struct ptdesc *ptdesc = pagetable_alloc(GFP_KERNEL & ~__GFP_HIGHMEM, 0); in alloc_pmd_late() local
476 BUG_ON(!ptdesc || !pagetable_pmd_ctor(ptdesc)); in alloc_pmd_late()
477 return __pa((pmd_t *)ptdesc_address(ptdesc)); in alloc_pmd_late()
1551 struct ptdesc *ptdesc = page_ptdesc(page); in free_pte_table() local
1561 pagetable_pte_dtor(ptdesc); in free_pte_table()
1565 pagetable_free(ptdesc); in free_pte_table()
1572 struct ptdesc *ptdesc = page_ptdesc(page); in free_pmd_table() local
[all …]
/linux-6.12.1/arch/sparc/mm/
Dinit_64.c2898 struct ptdesc *ptdesc = pagetable_alloc(GFP_KERNEL | __GFP_ZERO, 0); in pte_alloc_one() local
2900 if (!ptdesc) in pte_alloc_one()
2902 if (!pagetable_pte_ctor(ptdesc)) { in pte_alloc_one()
2903 pagetable_free(ptdesc); in pte_alloc_one()
2906 return ptdesc_address(ptdesc); in pte_alloc_one()
2916 struct ptdesc *ptdesc = virt_to_ptdesc(pte); in __pte_free() local
2918 pagetable_pte_dtor(ptdesc); in __pte_free()
2919 pagetable_free(ptdesc); in __pte_free()
/linux-6.12.1/arch/arm/mm/
Dmmu.c740 void *ptdesc = pagetable_alloc(GFP_PGTABLE_KERNEL & ~__GFP_HIGHMEM, in late_alloc() local
743 if (!ptdesc || !pagetable_pte_ctor(ptdesc)) in late_alloc()
745 return ptdesc_to_virt(ptdesc); in late_alloc()
/linux-6.12.1/arch/arm64/mm/
Dmmu.c486 struct ptdesc *ptdesc = page_ptdesc(phys_to_page(pa)); in pgd_pgtable_alloc() local
497 BUG_ON(!pagetable_pte_ctor(ptdesc)); in pgd_pgtable_alloc()
499 BUG_ON(!pagetable_pmd_ctor(ptdesc)); in pgd_pgtable_alloc()
/linux-6.12.1/mm/
Dmemory.c6914 bool ptlock_alloc(struct ptdesc *ptdesc) in ptlock_alloc() argument
6921 ptdesc->ptl = ptl; in ptlock_alloc()
6925 void ptlock_free(struct ptdesc *ptdesc) in ptlock_free() argument
6927 kmem_cache_free(page_ptl_cachep, ptdesc->ptl); in ptlock_free()