Lines Matching refs:pvmw
843 DEFINE_FOLIO_VMA_WALK(pvmw, folio, vma, address, 0); in folio_referenced_one()
847 while (page_vma_mapped_walk(&pvmw)) { in folio_referenced_one()
848 address = pvmw.address; in folio_referenced_one()
851 if (!folio_test_large(folio) || !pvmw.pte) { in folio_referenced_one()
854 page_vma_mapped_walk_done(&pvmw); in folio_referenced_one()
884 page_vma_mapped_walk_done(&pvmw); in folio_referenced_one()
888 if (lru_gen_enabled() && pvmw.pte) { in folio_referenced_one()
889 if (lru_gen_look_around(&pvmw)) in folio_referenced_one()
891 } else if (pvmw.pte) { in folio_referenced_one()
893 pvmw.pte)) in folio_referenced_one()
897 pvmw.pmd)) in folio_referenced_one()
1014 static int page_vma_mkclean_one(struct page_vma_mapped_walk *pvmw) in page_vma_mkclean_one() argument
1017 struct vm_area_struct *vma = pvmw->vma; in page_vma_mkclean_one()
1019 unsigned long address = pvmw->address; in page_vma_mkclean_one()
1026 vma->vm_mm, address, vma_address_end(pvmw)); in page_vma_mkclean_one()
1029 while (page_vma_mapped_walk(pvmw)) { in page_vma_mkclean_one()
1032 address = pvmw->address; in page_vma_mkclean_one()
1033 if (pvmw->pte) { in page_vma_mkclean_one()
1034 pte_t *pte = pvmw->pte; in page_vma_mkclean_one()
1048 pmd_t *pmd = pvmw->pmd; in page_vma_mkclean_one()
1079 DEFINE_FOLIO_VMA_WALK(pvmw, folio, vma, address, PVMW_SYNC); in page_mkclean_one()
1082 *cleaned += page_vma_mkclean_one(&pvmw); in page_mkclean_one()
1135 struct page_vma_mapped_walk pvmw = { in pfn_mkclean_range() local
1146 pvmw.address = vma_address(vma, pgoff, nr_pages); in pfn_mkclean_range()
1147 VM_BUG_ON_VMA(pvmw.address == -EFAULT, vma); in pfn_mkclean_range()
1149 return page_vma_mkclean_one(&pvmw); in pfn_mkclean_range()
1639 DEFINE_FOLIO_VMA_WALK(pvmw, folio, vma, address, 0); in try_to_unmap_one()
1655 pvmw.flags = PVMW_SYNC; in try_to_unmap_one()
1665 range.end = vma_address_end(&pvmw); in try_to_unmap_one()
1681 while (page_vma_mapped_walk(&pvmw)) { in try_to_unmap_one()
1693 if (!pvmw.pte) { in try_to_unmap_one()
1694 if (unmap_huge_pmd_locked(vma, pvmw.address, pvmw.pmd, in try_to_unmap_one()
1703 split_huge_pmd_locked(vma, pvmw.address, in try_to_unmap_one()
1704 pvmw.pmd, false, folio); in try_to_unmap_one()
1706 page_vma_mapped_walk_restart(&pvmw); in try_to_unmap_one()
1712 VM_BUG_ON_FOLIO(!pvmw.pte, folio); in try_to_unmap_one()
1714 pfn = pte_pfn(ptep_get(pvmw.pte)); in try_to_unmap_one()
1716 address = pvmw.address; in try_to_unmap_one()
1751 if (huge_pmd_unshare(mm, vma, address, pvmw.pte)) { in try_to_unmap_one()
1769 pteval = huge_ptep_clear_flush(vma, address, pvmw.pte); in try_to_unmap_one()
1782 pteval = ptep_get_and_clear(mm, address, pvmw.pte); in try_to_unmap_one()
1786 pteval = ptep_clear_flush(vma, address, pvmw.pte); in try_to_unmap_one()
1795 pte_install_uffd_wp_if_needed(vma, address, pvmw.pte, pteval); in try_to_unmap_one()
1808 set_huge_pte_at(mm, address, pvmw.pte, pteval, in try_to_unmap_one()
1812 set_pte_at(mm, address, pvmw.pte, pteval); in try_to_unmap_one()
1880 set_pte_at(mm, address, pvmw.pte, pteval); in try_to_unmap_one()
1891 set_pte_at(mm, address, pvmw.pte, pteval); in try_to_unmap_one()
1896 set_pte_at(mm, address, pvmw.pte, pteval); in try_to_unmap_one()
1904 set_pte_at(mm, address, pvmw.pte, pteval); in try_to_unmap_one()
1922 set_pte_at(mm, address, pvmw.pte, swp_pte); in try_to_unmap_one()
1949 page_vma_mapped_walk_done(&pvmw); in try_to_unmap_one()
2004 DEFINE_FOLIO_VMA_WALK(pvmw, folio, vma, address, 0); in try_to_migrate_one()
2020 pvmw.flags = PVMW_SYNC; in try_to_migrate_one()
2037 range.end = vma_address_end(&pvmw); in try_to_migrate_one()
2053 while (page_vma_mapped_walk(&pvmw)) { in try_to_migrate_one()
2056 if (!pvmw.pte) { in try_to_migrate_one()
2058 pmd_pfn(*pvmw.pmd) - folio_pfn(folio)); in try_to_migrate_one()
2062 if (set_pmd_migration_entry(&pvmw, subpage)) { in try_to_migrate_one()
2064 page_vma_mapped_walk_done(&pvmw); in try_to_migrate_one()
2072 VM_BUG_ON_FOLIO(!pvmw.pte, folio); in try_to_migrate_one()
2074 pfn = pte_pfn(ptep_get(pvmw.pte)); in try_to_migrate_one()
2092 address = pvmw.address; in try_to_migrate_one()
2121 page_vma_mapped_walk_done(&pvmw); in try_to_migrate_one()
2125 if (huge_pmd_unshare(mm, vma, address, pvmw.pte)) { in try_to_migrate_one()
2140 page_vma_mapped_walk_done(&pvmw); in try_to_migrate_one()
2146 pteval = huge_ptep_clear_flush(vma, address, pvmw.pte); in try_to_migrate_one()
2159 pteval = ptep_get_and_clear(mm, address, pvmw.pte); in try_to_migrate_one()
2163 pteval = ptep_clear_flush(vma, address, pvmw.pte); in try_to_migrate_one()
2205 set_pte_at(mm, pvmw.address, pvmw.pte, swp_pte); in try_to_migrate_one()
2206 trace_set_migration_pte(pvmw.address, pte_val(swp_pte), in try_to_migrate_one()
2216 set_huge_pte_at(mm, address, pvmw.pte, pteval, in try_to_migrate_one()
2220 set_pte_at(mm, address, pvmw.pte, pteval); in try_to_migrate_one()
2241 set_huge_pte_at(mm, address, pvmw.pte, in try_to_migrate_one()
2244 set_pte_at(mm, address, pvmw.pte, pteval); in try_to_migrate_one()
2246 page_vma_mapped_walk_done(&pvmw); in try_to_migrate_one()
2256 set_huge_pte_at(mm, address, pvmw.pte, in try_to_migrate_one()
2259 page_vma_mapped_walk_done(&pvmw); in try_to_migrate_one()
2264 set_pte_at(mm, address, pvmw.pte, pteval); in try_to_migrate_one()
2266 page_vma_mapped_walk_done(&pvmw); in try_to_migrate_one()
2294 set_huge_pte_at(mm, address, pvmw.pte, swp_pte, in try_to_migrate_one()
2297 set_pte_at(mm, address, pvmw.pte, swp_pte); in try_to_migrate_one()
2378 DEFINE_FOLIO_VMA_WALK(pvmw, folio, vma, address, 0); in page_make_device_exclusive_one()
2394 while (page_vma_mapped_walk(&pvmw)) { in page_make_device_exclusive_one()
2396 VM_BUG_ON_FOLIO(!pvmw.pte, folio); in page_make_device_exclusive_one()
2398 ptent = ptep_get(pvmw.pte); in page_make_device_exclusive_one()
2401 page_vma_mapped_walk_done(&pvmw); in page_make_device_exclusive_one()
2407 address = pvmw.address; in page_make_device_exclusive_one()
2411 pteval = ptep_clear_flush(vma, address, pvmw.pte); in page_make_device_exclusive_one()
2442 set_pte_at(mm, address, pvmw.pte, swp_pte); in page_make_device_exclusive_one()