Home
last modified time | relevance | path

Searched refs:tlb (Results 1 – 25 of 171) sorted by relevance

1234567

/linux-6.12.1/include/asm-generic/
Dtlb.h210 extern void tlb_remove_table(struct mmu_gather *tlb, void *table);
218 #define tlb_remove_table(tlb, page) tlb_remove_page((tlb), (page)) argument
269 extern bool __tlb_remove_page_size(struct mmu_gather *tlb, struct page *page,
271 bool __tlb_remove_folio_pages(struct mmu_gather *tlb, struct page *page,
279 #define tlb_delay_rmap(tlb) (((tlb)->delayed_rmap = 1), true) argument
280 extern void tlb_flush_rmaps(struct mmu_gather *tlb, struct vm_area_struct *vma);
293 #define tlb_delay_rmap(tlb) (false) argument
294 static inline void tlb_flush_rmaps(struct mmu_gather *tlb, struct vm_area_struct *vma) { } in tlb_flush_rmaps() argument
360 void tlb_flush_mmu(struct mmu_gather *tlb);
362 static inline void __tlb_adjust_range(struct mmu_gather *tlb, in __tlb_adjust_range() argument
[all …]
/linux-6.12.1/mm/
Dmmu_gather.c18 static bool tlb_next_batch(struct mmu_gather *tlb) in tlb_next_batch() argument
23 if (tlb->delayed_rmap && tlb->active != &tlb->local) in tlb_next_batch()
26 batch = tlb->active; in tlb_next_batch()
28 tlb->active = batch->next; in tlb_next_batch()
32 if (tlb->batch_count == MAX_GATHER_BATCH_COUNT) in tlb_next_batch()
39 tlb->batch_count++; in tlb_next_batch()
44 tlb->active->next = batch; in tlb_next_batch()
45 tlb->active = batch; in tlb_next_batch()
82 void tlb_flush_rmaps(struct mmu_gather *tlb, struct vm_area_struct *vma) in tlb_flush_rmaps() argument
84 if (!tlb->delayed_rmap) in tlb_flush_rmaps()
[all …]
Dmadvise.c41 struct mmu_gather *tlb; member
341 struct mmu_gather *tlb = private->tlb; in madvise_cold_or_pageout_pte_range() local
343 struct mm_struct *mm = tlb->mm; in madvise_cold_or_pageout_pte_range()
364 tlb_change_page_size(tlb, HPAGE_PMD_SIZE); in madvise_cold_or_pageout_pte_range()
407 tlb_remove_pmd_tlb_entry(tlb, pmd, addr); in madvise_cold_or_pageout_pte_range()
432 tlb_change_page_size(tlb, PAGE_SIZE); in madvise_cold_or_pageout_pte_range()
521 tlb_remove_tlb_entries(tlb, pte, nr, addr); in madvise_cold_or_pageout_pte_range()
561 static void madvise_cold_page_range(struct mmu_gather *tlb, in madvise_cold_page_range() argument
567 .tlb = tlb, in madvise_cold_page_range()
570 tlb_start_vma(tlb, vma); in madvise_cold_page_range()
[all …]
Dmprotect.c86 static long change_pte_range(struct mmu_gather *tlb, in change_pte_range() argument
98 tlb_change_page_size(tlb, PAGE_SIZE); in change_pte_range()
197 tlb_flush_pte_range(tlb, addr, PAGE_SIZE); in change_pte_range()
359 static inline long change_pmd_range(struct mmu_gather *tlb, in change_pmd_range() argument
400 ret = change_huge_pmd(tlb, vma, pmd, in change_pmd_range()
415 ret = change_pte_range(tlb, vma, pmd, addr, next, newprot, in change_pmd_range()
429 static inline long change_pud_range(struct mmu_gather *tlb, in change_pud_range() argument
467 ret = change_huge_pud(tlb, vma, pudp, in change_pud_range()
478 pages += change_pmd_range(tlb, vma, pudp, addr, next, newprot, in change_pud_range()
488 static inline long change_p4d_range(struct mmu_gather *tlb, in change_p4d_range() argument
[all …]
/linux-6.12.1/arch/arm64/include/asm/
Dtlb.h20 static void tlb_flush(struct mmu_gather *tlb);
29 static inline int tlb_get_level(struct mmu_gather *tlb) in tlb_get_level() argument
32 if (tlb->freed_tables) in tlb_get_level()
35 if (tlb->cleared_ptes && !(tlb->cleared_pmds || in tlb_get_level()
36 tlb->cleared_puds || in tlb_get_level()
37 tlb->cleared_p4ds)) in tlb_get_level()
40 if (tlb->cleared_pmds && !(tlb->cleared_ptes || in tlb_get_level()
41 tlb->cleared_puds || in tlb_get_level()
42 tlb->cleared_p4ds)) in tlb_get_level()
45 if (tlb->cleared_puds && !(tlb->cleared_ptes || in tlb_get_level()
[all …]
/linux-6.12.1/arch/s390/include/asm/
Dtlb.h26 static inline void tlb_flush(struct mmu_gather *tlb);
27 static inline bool __tlb_remove_page_size(struct mmu_gather *tlb,
29 static inline bool __tlb_remove_folio_pages(struct mmu_gather *tlb,
48 static inline bool __tlb_remove_page_size(struct mmu_gather *tlb, in __tlb_remove_page_size() argument
57 static inline bool __tlb_remove_folio_pages(struct mmu_gather *tlb, in __tlb_remove_folio_pages() argument
72 static inline void tlb_flush(struct mmu_gather *tlb) in tlb_flush() argument
74 __tlb_flush_mm_lazy(tlb->mm); in tlb_flush()
81 static inline void pte_free_tlb(struct mmu_gather *tlb, pgtable_t pte, in pte_free_tlb() argument
84 __tlb_adjust_range(tlb, address, PAGE_SIZE); in pte_free_tlb()
85 tlb->mm->context.flush_mm = 1; in pte_free_tlb()
[all …]
/linux-6.12.1/arch/riscv/include/asm/
Dtlb.h11 static void tlb_flush(struct mmu_gather *tlb);
34 static inline void tlb_flush(struct mmu_gather *tlb) in tlb_flush() argument
37 if (tlb->fullmm || tlb->need_flush_all || tlb->freed_tables) in tlb_flush()
38 flush_tlb_mm(tlb->mm); in tlb_flush()
40 flush_tlb_mm_range(tlb->mm, tlb->start, tlb->end, in tlb_flush()
41 tlb_get_unmap_size(tlb)); in tlb_flush()
Dpgalloc.h19 static inline void riscv_tlb_remove_ptdesc(struct mmu_gather *tlb, void *pt) in riscv_tlb_remove_ptdesc() argument
22 tlb_remove_ptdesc(tlb, pt); in riscv_tlb_remove_ptdesc()
24 tlb_remove_page_ptdesc(tlb, pt); in riscv_tlb_remove_ptdesc()
107 static inline void __pud_free_tlb(struct mmu_gather *tlb, pud_t *pud, in __pud_free_tlb() argument
114 riscv_tlb_remove_ptdesc(tlb, ptdesc); in __pud_free_tlb()
145 static inline void __p4d_free_tlb(struct mmu_gather *tlb, p4d_t *p4d, in __p4d_free_tlb() argument
149 riscv_tlb_remove_ptdesc(tlb, virt_to_ptdesc(p4d)); in __p4d_free_tlb()
175 static inline void __pmd_free_tlb(struct mmu_gather *tlb, pmd_t *pmd, in __pmd_free_tlb() argument
181 riscv_tlb_remove_ptdesc(tlb, ptdesc); in __pmd_free_tlb()
186 static inline void __pte_free_tlb(struct mmu_gather *tlb, pgtable_t pte, in __pte_free_tlb() argument
[all …]
/linux-6.12.1/arch/x86/include/asm/
Dtlb.h6 static inline void tlb_flush(struct mmu_gather *tlb);
10 static inline void tlb_flush(struct mmu_gather *tlb) in tlb_flush() argument
13 unsigned int stride_shift = tlb_get_unmap_shift(tlb); in tlb_flush()
15 if (!tlb->fullmm && !tlb->need_flush_all) { in tlb_flush()
16 start = tlb->start; in tlb_flush()
17 end = tlb->end; in tlb_flush()
20 flush_tlb_mm_range(tlb->mm, start, end, stride_shift, tlb->freed_tables); in tlb_flush()
Dpgalloc.h56 extern void ___pte_free_tlb(struct mmu_gather *tlb, struct page *pte);
58 static inline void __pte_free_tlb(struct mmu_gather *tlb, struct page *pte, in __pte_free_tlb() argument
61 ___pte_free_tlb(tlb, pte); in __pte_free_tlb()
88 extern void ___pmd_free_tlb(struct mmu_gather *tlb, pmd_t *pmd);
90 static inline void __pmd_free_tlb(struct mmu_gather *tlb, pmd_t *pmd, in __pmd_free_tlb() argument
93 ___pmd_free_tlb(tlb, pmd); in __pmd_free_tlb()
125 extern void ___pud_free_tlb(struct mmu_gather *tlb, pud_t *pud);
127 static inline void __pud_free_tlb(struct mmu_gather *tlb, pud_t *pud, in __pud_free_tlb() argument
130 ___pud_free_tlb(tlb, pud); in __pud_free_tlb()
168 extern void ___p4d_free_tlb(struct mmu_gather *tlb, p4d_t *p4d);
[all …]
/linux-6.12.1/arch/sh/mm/
DMakefile30 debugfs-$(CONFIG_CPU_SH4) += tlb-debugfs.o
31 tlb-$(CONFIG_CPU_SH3) := tlb-sh3.o
32 tlb-$(CONFIG_CPU_SH4) := tlb-sh4.o tlb-urb.o
33 tlb-$(CONFIG_CPU_HAS_PTEAEX) := tlb-pteaex.o tlb-urb.o
34 obj-y += $(tlb-y)
/linux-6.12.1/arch/arm/include/asm/
Dtlb.h23 #define tlb_flush(tlb) ((void) tlb) argument
40 __pte_free_tlb(struct mmu_gather *tlb, pgtable_t pte, unsigned long addr) in __pte_free_tlb() argument
52 __tlb_adjust_range(tlb, addr - PAGE_SIZE, 2 * PAGE_SIZE); in __pte_free_tlb()
55 tlb_remove_ptdesc(tlb, ptdesc); in __pte_free_tlb()
59 __pmd_free_tlb(struct mmu_gather *tlb, pmd_t *pmdp, unsigned long addr) in __pmd_free_tlb() argument
65 tlb_remove_ptdesc(tlb, ptdesc); in __pmd_free_tlb()
/linux-6.12.1/arch/powerpc/include/asm/nohash/
Dpgalloc.h8 extern void tlb_remove_table(struct mmu_gather *tlb, void *table);
10 extern void tlb_flush_pgtable(struct mmu_gather *tlb, unsigned long address);
13 static inline void tlb_flush_pgtable(struct mmu_gather *tlb, in tlb_flush_pgtable() argument
53 static inline void pgtable_free_tlb(struct mmu_gather *tlb, void *table, int shift) in pgtable_free_tlb() argument
59 tlb_remove_table(tlb, (void *)pgf); in pgtable_free_tlb()
70 static inline void __pte_free_tlb(struct mmu_gather *tlb, pgtable_t table, in __pte_free_tlb() argument
73 tlb_flush_pgtable(tlb, address); in __pte_free_tlb()
74 pgtable_free_tlb(tlb, table, 0); in __pte_free_tlb()
/linux-6.12.1/arch/sparc/include/asm/
Dpgalloc_64.h82 static inline void pgtable_free_tlb(struct mmu_gather *tlb, void *table, bool is_page) in pgtable_free_tlb() argument
87 tlb_remove_table(tlb, (void *)pgf); in pgtable_free_tlb()
100 static inline void pgtable_free_tlb(struct mmu_gather *tlb, void *table, bool is_page) in pgtable_free_tlb() argument
106 static inline void __pte_free_tlb(struct mmu_gather *tlb, pte_t *pte, in __pte_free_tlb() argument
109 pgtable_free_tlb(tlb, pte, true); in __pte_free_tlb()
112 #define __pmd_free_tlb(tlb, pmd, addr) \ argument
113 pgtable_free_tlb(tlb, pmd, false)
115 #define __pud_free_tlb(tlb, pud, addr) \ argument
116 pgtable_free_tlb(tlb, pud, false)
/linux-6.12.1/include/linux/
Dio-pgtable.h103 const struct iommu_flush_ops *tlb; member
254 if (iop->cfg.tlb && iop->cfg.tlb->tlb_flush_all) in io_pgtable_tlb_flush_all()
255 iop->cfg.tlb->tlb_flush_all(iop->cookie); in io_pgtable_tlb_flush_all()
262 if (iop->cfg.tlb && iop->cfg.tlb->tlb_flush_walk) in io_pgtable_tlb_flush_walk()
263 iop->cfg.tlb->tlb_flush_walk(iova, size, granule, iop->cookie); in io_pgtable_tlb_flush_walk()
271 if (iop->cfg.tlb && iop->cfg.tlb->tlb_add_page) in io_pgtable_tlb_add_page()
272 iop->cfg.tlb->tlb_add_page(gather, iova, granule, iop->cookie); in io_pgtable_tlb_add_page()
/linux-6.12.1/arch/loongarch/include/asm/
Dtlb.h135 static void tlb_flush(struct mmu_gather *tlb);
140 static inline void tlb_flush(struct mmu_gather *tlb) in tlb_flush() argument
144 vma.vm_mm = tlb->mm; in tlb_flush()
146 if (tlb->fullmm) { in tlb_flush()
147 flush_tlb_mm(tlb->mm); in tlb_flush()
151 flush_tlb_range(&vma, tlb->start, tlb->end); in tlb_flush()
Dpgalloc.h58 #define __pte_free_tlb(tlb, pte, address) \ argument
61 tlb_remove_page_ptdesc((tlb), page_ptdesc(pte)); \
85 #define __pmd_free_tlb(tlb, x, addr) pmd_free((tlb)->mm, x) argument
105 #define __pud_free_tlb(tlb, x, addr) pud_free((tlb)->mm, x) argument
/linux-6.12.1/arch/riscv/boot/dts/sifive/
Dfu540-c000.dtsi48 d-tlb-sets = <1>;
49 d-tlb-size = <32>;
54 i-tlb-sets = <1>;
55 i-tlb-size = <32>;
62 tlb-split;
75 d-tlb-sets = <1>;
76 d-tlb-size = <32>;
81 i-tlb-sets = <1>;
82 i-tlb-size = <32>;
89 tlb-split;
[all …]
Dfu740-c000.dtsi49 d-tlb-sets = <1>;
50 d-tlb-size = <40>;
55 i-tlb-sets = <1>;
56 i-tlb-size = <40>;
64 tlb-split;
76 d-tlb-sets = <1>;
77 d-tlb-size = <40>;
82 i-tlb-sets = <1>;
83 i-tlb-size = <40>;
91 tlb-split;
[all …]
/linux-6.12.1/arch/arc/include/asm/
Dpgalloc.h80 #define __pud_free_tlb(tlb, pmd, addr) pud_free((tlb)->mm, pmd) argument
91 #define __pmd_free_tlb(tlb, pmd, addr) pmd_free((tlb)->mm, pmd) argument
95 #define __pte_free_tlb(tlb, pte, addr) pte_free((tlb)->mm, pte) argument
/linux-6.12.1/arch/alpha/include/asm/
Dtlb.h7 #define __pte_free_tlb(tlb, pte, address) pte_free((tlb)->mm, pte) argument
8 #define __pmd_free_tlb(tlb, pmd, address) pmd_free((tlb)->mm, pmd) argument
/linux-6.12.1/arch/mips/include/asm/
Dpgalloc.h57 #define __pte_free_tlb(tlb, pte, address) \ argument
60 tlb_remove_page_ptdesc((tlb), page_ptdesc(pte)); \
84 #define __pmd_free_tlb(tlb, x, addr) pmd_free((tlb)->mm, x) argument
110 #define __pud_free_tlb(tlb, x, addr) pud_free((tlb)->mm, x) argument
/linux-6.12.1/arch/parisc/include/asm/
Dtlb.h8 #define __pmd_free_tlb(tlb, pmd, addr) pmd_free((tlb)->mm, pmd) argument
10 #define __pte_free_tlb(tlb, pte, addr) pte_free((tlb)->mm, pte) argument
/linux-6.12.1/drivers/gpu/drm/i915/gt/
Dintel_tlb.c137 mutex_lock(&gt->tlb.invalidate_lock); in intel_gt_invalidate_tlb_full()
154 write_seqcount_invalidate(&gt->tlb.seqno); in intel_gt_invalidate_tlb_full()
156 mutex_unlock(&gt->tlb.invalidate_lock); in intel_gt_invalidate_tlb_full()
162 mutex_init(&gt->tlb.invalidate_lock); in intel_gt_init_tlb()
163 seqcount_mutex_init(&gt->tlb.seqno, &gt->tlb.invalidate_lock); in intel_gt_init_tlb()
168 mutex_destroy(&gt->tlb.invalidate_lock); in intel_gt_fini_tlb()
/linux-6.12.1/arch/powerpc/include/asm/book3s/32/
Dpgalloc.h25 #define __pmd_free_tlb(tlb,x,a) do { } while (0) argument
50 static inline void pgtable_free_tlb(struct mmu_gather *tlb, in pgtable_free_tlb() argument
56 tlb_remove_table(tlb, (void *)pgf); in pgtable_free_tlb()
67 static inline void __pte_free_tlb(struct mmu_gather *tlb, pgtable_t table, in __pte_free_tlb() argument
70 pgtable_free_tlb(tlb, table, 0); in __pte_free_tlb()

1234567