| /linux-6.12.1/mm/ |
| D | page_vma_mapped.c | 10 static inline bool not_found(struct page_vma_mapped_walk *pvmw) in not_found() argument
12 page_vma_mapped_walk_done(pvmw); in not_found()
16 static bool map_pte(struct page_vma_mapped_walk *pvmw, spinlock_t **ptlp) in map_pte() argument
20 if (pvmw->flags & PVMW_SYNC) { in map_pte()
22 pvmw->pte = pte_offset_map_lock(pvmw->vma->vm_mm, pvmw->pmd, in map_pte()
23 pvmw->address, &pvmw->ptl); in map_pte()
24 *ptlp = pvmw->ptl; in map_pte()
25 return !!pvmw->pte; in map_pte()
35 pvmw->pte = pte_offset_map_nolock(pvmw->vma->vm_mm, pvmw->pmd, in map_pte()
36 pvmw->address, ptlp); in map_pte()
[all …]
|
| D | rmap.c | 843 DEFINE_FOLIO_VMA_WALK(pvmw, folio, vma, address, 0); in folio_referenced_one()
847 while (page_vma_mapped_walk(&pvmw)) { in folio_referenced_one()
848 address = pvmw.address; in folio_referenced_one()
851 if (!folio_test_large(folio) || !pvmw.pte) { in folio_referenced_one()
854 page_vma_mapped_walk_done(&pvmw); in folio_referenced_one()
884 page_vma_mapped_walk_done(&pvmw); in folio_referenced_one()
888 if (lru_gen_enabled() && pvmw.pte) { in folio_referenced_one()
889 if (lru_gen_look_around(&pvmw)) in folio_referenced_one()
891 } else if (pvmw.pte) { in folio_referenced_one()
893 pvmw.pte)) in folio_referenced_one()
[all …]
|
| D | page_idle.c | 56 DEFINE_FOLIO_VMA_WALK(pvmw, folio, vma, addr, 0); in page_idle_clear_pte_refs_one()
59 while (page_vma_mapped_walk(&pvmw)) { in page_idle_clear_pte_refs_one()
60 addr = pvmw.address; in page_idle_clear_pte_refs_one()
61 if (pvmw.pte) { in page_idle_clear_pte_refs_one()
66 if (ptep_clear_young_notify(vma, addr, pvmw.pte)) in page_idle_clear_pte_refs_one()
69 if (pmdp_clear_young_notify(vma, addr, pvmw.pmd)) in page_idle_clear_pte_refs_one()
|
| D | migrate.c | 200 static bool try_to_map_unused_to_zeropage(struct page_vma_mapped_walk *pvmw, in try_to_map_unused_to_zeropage() argument
213 VM_BUG_ON_PAGE(pte_present(*pvmw->pte), page); in try_to_map_unused_to_zeropage()
215 if (folio_test_mlocked(folio) || (pvmw->vma->vm_flags & VM_LOCKED) || in try_to_map_unused_to_zeropage()
216 mm_forbids_zeropage(pvmw->vma->vm_mm)) in try_to_map_unused_to_zeropage()
231 newpte = pte_mkspecial(pfn_pte(my_zero_pfn(pvmw->address), in try_to_map_unused_to_zeropage()
232 pvmw->vma->vm_page_prot)); in try_to_map_unused_to_zeropage()
233 set_pte_at(pvmw->vma->vm_mm, pvmw->address, pvmw->pte, newpte); in try_to_map_unused_to_zeropage()
235 dec_mm_counter(pvmw->vma->vm_mm, mm_counter(folio)); in try_to_map_unused_to_zeropage()
251 DEFINE_FOLIO_VMA_WALK(pvmw, rmap_walk_arg->folio, vma, addr, PVMW_SYNC | PVMW_MIGRATION); in remove_migration_pte()
253 while (page_vma_mapped_walk(&pvmw)) { in remove_migration_pte()
[all …]
|
| D | ksm.c | 1250 DEFINE_FOLIO_VMA_WALK(pvmw, folio, vma, 0, 0); in write_protect_page()
1260 pvmw.address = page_address_in_vma(&folio->page, vma); in write_protect_page()
1261 if (pvmw.address == -EFAULT) in write_protect_page()
1264 mmu_notifier_range_init(&range, MMU_NOTIFY_CLEAR, 0, mm, pvmw.address, in write_protect_page()
1265 pvmw.address + PAGE_SIZE); in write_protect_page()
1268 if (!page_vma_mapped_walk(&pvmw)) in write_protect_page()
1270 if (WARN_ONCE(!pvmw.pte, "Unexpected PMD mapping?")) in write_protect_page()
1274 entry = ptep_get(pvmw.pte); in write_protect_page()
1278 flush_cache_page(vma, pvmw.address, folio_pfn(folio)); in write_protect_page()
1293 entry = ptep_clear_flush(vma, pvmw.address, pvmw.pte); in write_protect_page()
[all …]
|
| D | internal.h | 986 static inline unsigned long vma_address_end(struct page_vma_mapped_walk *pvmw) in vma_address_end() argument
988 struct vm_area_struct *vma = pvmw->vma; in vma_address_end()
993 if (pvmw->nr_pages == 1) in vma_address_end()
994 return pvmw->address + PAGE_SIZE; in vma_address_end()
996 pgoff = pvmw->pgoff + pvmw->nr_pages; in vma_address_end()
|
| D | huge_memory.c | 4151 int set_pmd_migration_entry(struct page_vma_mapped_walk *pvmw, in set_pmd_migration_entry() argument
4155 struct vm_area_struct *vma = pvmw->vma; in set_pmd_migration_entry()
4157 unsigned long address = pvmw->address; in set_pmd_migration_entry()
4163 if (!(pvmw->pmd && !pvmw->pte)) in set_pmd_migration_entry()
4167 pmdval = pmdp_invalidate(vma, address, pvmw->pmd); in set_pmd_migration_entry()
4172 set_pmd_at(mm, address, pvmw->pmd, pmdval); in set_pmd_migration_entry()
4193 set_pmd_at(mm, address, pvmw->pmd, pmdswp); in set_pmd_migration_entry()
4201 void remove_migration_pmd(struct page_vma_mapped_walk *pvmw, struct page *new) in remove_migration_pmd() argument
4204 struct vm_area_struct *vma = pvmw->vma; in remove_migration_pmd()
4206 unsigned long address = pvmw->address; in remove_migration_pmd()
[all …]
|
| D | vmscan.c | 4044 bool lru_gen_look_around(struct page_vma_mapped_walk *pvmw) in lru_gen_look_around() argument
4051 pte_t *pte = pvmw->pte; in lru_gen_look_around()
4052 unsigned long addr = pvmw->address; in lru_gen_look_around()
4053 struct vm_area_struct *vma = pvmw->vma; in lru_gen_look_around()
4054 struct folio *folio = pfn_folio(pvmw->pfn); in lru_gen_look_around()
4063 lockdep_assert_held(pvmw->ptl); in lru_gen_look_around()
4069 if (spin_is_contended(pvmw->ptl)) in lru_gen_look_around()
4146 update_bloom_filter(mm_state, max_seq, pvmw->pmd); in lru_gen_look_around()
|
| /linux-6.12.1/include/linux/ |
| D | rmap.h | 697 static inline void page_vma_mapped_walk_done(struct page_vma_mapped_walk *pvmw) in page_vma_mapped_walk_done() argument
700 if (pvmw->pte && !is_vm_hugetlb_page(pvmw->vma)) in page_vma_mapped_walk_done()
701 pte_unmap(pvmw->pte); in page_vma_mapped_walk_done()
702 if (pvmw->ptl) in page_vma_mapped_walk_done()
703 spin_unlock(pvmw->ptl); in page_vma_mapped_walk_done()
716 page_vma_mapped_walk_restart(struct page_vma_mapped_walk *pvmw) in page_vma_mapped_walk_restart() argument
718 WARN_ON_ONCE(!pvmw->pmd && !pvmw->pte); in page_vma_mapped_walk_restart()
720 if (likely(pvmw->ptl)) in page_vma_mapped_walk_restart()
721 spin_unlock(pvmw->ptl); in page_vma_mapped_walk_restart()
725 pvmw->ptl = NULL; in page_vma_mapped_walk_restart()
[all …]
|
| D | swapops.h | 531 extern int set_pmd_migration_entry(struct page_vma_mapped_walk *pvmw,
534 extern void remove_migration_pmd(struct page_vma_mapped_walk *pvmw,
564 static inline int set_pmd_migration_entry(struct page_vma_mapped_walk *pvmw, in set_pmd_migration_entry() argument
570 static inline void remove_migration_pmd(struct page_vma_mapped_walk *pvmw, in remove_migration_pmd() argument
|
| D | mmzone.h | 558 bool lru_gen_look_around(struct page_vma_mapped_walk *pvmw);
577 static inline bool lru_gen_look_around(struct page_vma_mapped_walk *pvmw) in lru_gen_look_around() argument
|
| /linux-6.12.1/mm/damon/ |
| D | paddr.c | 25 DEFINE_FOLIO_VMA_WALK(pvmw, folio, vma, addr, 0); in damon_folio_mkold_one()
27 while (page_vma_mapped_walk(&pvmw)) { in damon_folio_mkold_one()
28 addr = pvmw.address; in damon_folio_mkold_one()
29 if (pvmw.pte) in damon_folio_mkold_one()
30 damon_ptep_mkold(pvmw.pte, vma, addr); in damon_folio_mkold_one()
32 damon_pmdp_mkold(pvmw.pmd, vma, addr); in damon_folio_mkold_one()
94 DEFINE_FOLIO_VMA_WALK(pvmw, folio, vma, addr, 0); in damon_folio_young_one()
97 while (page_vma_mapped_walk(&pvmw)) { in damon_folio_young_one()
98 addr = pvmw.address; in damon_folio_young_one()
99 if (pvmw.pte) { in damon_folio_young_one()
[all …]
|
| /linux-6.12.1/kernel/events/ |
| D | uprobes.c | 167 DEFINE_FOLIO_VMA_WALK(pvmw, old_folio, vma, addr, 0); in __replace_page()
186 if (!page_vma_mapped_walk(&pvmw)) in __replace_page()
188 VM_BUG_ON_PAGE(addr != pvmw.address, old_page); in __replace_page()
203 flush_cache_page(vma, addr, pte_pfn(ptep_get(pvmw.pte))); in __replace_page()
204 ptep_clear_flush(vma, addr, pvmw.pte); in __replace_page()
206 set_pte_at(mm, addr, pvmw.pte, in __replace_page()
212 page_vma_mapped_walk_done(&pvmw); in __replace_page()
|