Lines Matching refs:pvmw
811 DEFINE_FOLIO_VMA_WALK(pvmw, folio, vma, address, 0); in folio_referenced_one()
814 while (page_vma_mapped_walk(&pvmw)) { in folio_referenced_one()
815 address = pvmw.address; in folio_referenced_one()
818 (!folio_test_large(folio) || !pvmw.pte)) { in folio_referenced_one()
820 mlock_vma_folio(folio, vma, !pvmw.pte); in folio_referenced_one()
821 page_vma_mapped_walk_done(&pvmw); in folio_referenced_one()
826 if (pvmw.pte) { in folio_referenced_one()
827 if (lru_gen_enabled() && pte_young(*pvmw.pte)) { in folio_referenced_one()
828 lru_gen_look_around(&pvmw); in folio_referenced_one()
833 pvmw.pte)) in folio_referenced_one()
837 pvmw.pmd)) in folio_referenced_one()
937 static int page_vma_mkclean_one(struct page_vma_mapped_walk *pvmw) in page_vma_mkclean_one() argument
940 struct vm_area_struct *vma = pvmw->vma; in page_vma_mkclean_one()
942 unsigned long address = pvmw->address; in page_vma_mkclean_one()
949 vma->vm_mm, address, vma_address_end(pvmw)); in page_vma_mkclean_one()
952 while (page_vma_mapped_walk(pvmw)) { in page_vma_mkclean_one()
955 address = pvmw->address; in page_vma_mkclean_one()
956 if (pvmw->pte) { in page_vma_mkclean_one()
958 pte_t *pte = pvmw->pte; in page_vma_mkclean_one()
971 pmd_t *pmd = pvmw->pmd; in page_vma_mkclean_one()
1009 DEFINE_FOLIO_VMA_WALK(pvmw, folio, vma, address, PVMW_SYNC); in page_mkclean_one()
1012 *cleaned += page_vma_mkclean_one(&pvmw); in page_mkclean_one()
1065 struct page_vma_mapped_walk pvmw = { in pfn_mkclean_range() local
1076 pvmw.address = vma_pgoff_address(pgoff, nr_pages, vma); in pfn_mkclean_range()
1077 VM_BUG_ON_VMA(pvmw.address == -EFAULT, vma); in pfn_mkclean_range()
1079 return page_vma_mkclean_one(&pvmw); in pfn_mkclean_range()
1453 DEFINE_FOLIO_VMA_WALK(pvmw, folio, vma, address, 0); in try_to_unmap_one()
1467 pvmw.flags = PVMW_SYNC; in try_to_unmap_one()
1480 range.end = vma_address_end(&pvmw); in try_to_unmap_one()
1493 while (page_vma_mapped_walk(&pvmw)) { in try_to_unmap_one()
1495 VM_BUG_ON_FOLIO(!pvmw.pte, folio); in try_to_unmap_one()
1504 page_vma_mapped_walk_done(&pvmw); in try_to_unmap_one()
1510 pte_pfn(*pvmw.pte) - folio_pfn(folio)); in try_to_unmap_one()
1511 address = pvmw.address; in try_to_unmap_one()
1545 page_vma_mapped_walk_done(&pvmw); in try_to_unmap_one()
1549 if (huge_pmd_unshare(mm, vma, address, pvmw.pte)) { in try_to_unmap_one()
1565 page_vma_mapped_walk_done(&pvmw); in try_to_unmap_one()
1570 pteval = huge_ptep_clear_flush(vma, address, pvmw.pte); in try_to_unmap_one()
1572 flush_cache_page(vma, address, pte_pfn(*pvmw.pte)); in try_to_unmap_one()
1583 pteval = ptep_get_and_clear(mm, address, pvmw.pte); in try_to_unmap_one()
1587 pteval = ptep_clear_flush(vma, address, pvmw.pte); in try_to_unmap_one()
1596 pte_install_uffd_wp_if_needed(vma, address, pvmw.pte, pteval); in try_to_unmap_one()
1609 set_huge_pte_at(mm, address, pvmw.pte, pteval); in try_to_unmap_one()
1612 set_pte_at(mm, address, pvmw.pte, pteval); in try_to_unmap_one()
1644 page_vma_mapped_walk_done(&pvmw); in try_to_unmap_one()
1685 set_pte_at(mm, address, pvmw.pte, pteval); in try_to_unmap_one()
1688 page_vma_mapped_walk_done(&pvmw); in try_to_unmap_one()
1693 set_pte_at(mm, address, pvmw.pte, pteval); in try_to_unmap_one()
1695 page_vma_mapped_walk_done(&pvmw); in try_to_unmap_one()
1700 set_pte_at(mm, address, pvmw.pte, pteval); in try_to_unmap_one()
1702 page_vma_mapped_walk_done(&pvmw); in try_to_unmap_one()
1710 set_pte_at(mm, address, pvmw.pte, pteval); in try_to_unmap_one()
1712 page_vma_mapped_walk_done(&pvmw); in try_to_unmap_one()
1730 set_pte_at(mm, address, pvmw.pte, swp_pte); in try_to_unmap_one()
1813 DEFINE_FOLIO_VMA_WALK(pvmw, folio, vma, address, 0); in try_to_migrate_one()
1827 pvmw.flags = PVMW_SYNC; in try_to_migrate_one()
1844 range.end = vma_address_end(&pvmw); in try_to_migrate_one()
1857 while (page_vma_mapped_walk(&pvmw)) { in try_to_migrate_one()
1860 if (!pvmw.pte) { in try_to_migrate_one()
1862 pmd_pfn(*pvmw.pmd) - folio_pfn(folio)); in try_to_migrate_one()
1866 if (set_pmd_migration_entry(&pvmw, subpage)) { in try_to_migrate_one()
1868 page_vma_mapped_walk_done(&pvmw); in try_to_migrate_one()
1876 VM_BUG_ON_FOLIO(!pvmw.pte, folio); in try_to_migrate_one()
1893 pte_pfn(*pvmw.pte) - folio_pfn(folio)); in try_to_migrate_one()
1895 address = pvmw.address; in try_to_migrate_one()
1924 page_vma_mapped_walk_done(&pvmw); in try_to_migrate_one()
1928 if (huge_pmd_unshare(mm, vma, address, pvmw.pte)) { in try_to_migrate_one()
1945 page_vma_mapped_walk_done(&pvmw); in try_to_migrate_one()
1951 pteval = huge_ptep_clear_flush(vma, address, pvmw.pte); in try_to_migrate_one()
1953 flush_cache_page(vma, address, pte_pfn(*pvmw.pte)); in try_to_migrate_one()
1964 pteval = ptep_get_and_clear(mm, address, pvmw.pte); in try_to_migrate_one()
1968 pteval = ptep_clear_flush(vma, address, pvmw.pte); in try_to_migrate_one()
2009 set_pte_at(mm, pvmw.address, pvmw.pte, swp_pte); in try_to_migrate_one()
2010 trace_set_migration_pte(pvmw.address, pte_val(swp_pte), in try_to_migrate_one()
2020 set_huge_pte_at(mm, address, pvmw.pte, pteval); in try_to_migrate_one()
2023 set_pte_at(mm, address, pvmw.pte, pteval); in try_to_migrate_one()
2047 set_huge_pte_at(mm, address, pvmw.pte, pteval); in try_to_migrate_one()
2049 set_pte_at(mm, address, pvmw.pte, pteval); in try_to_migrate_one()
2051 page_vma_mapped_walk_done(&pvmw); in try_to_migrate_one()
2061 set_huge_pte_at(mm, address, pvmw.pte, pteval); in try_to_migrate_one()
2063 set_pte_at(mm, address, pvmw.pte, pteval); in try_to_migrate_one()
2065 page_vma_mapped_walk_done(&pvmw); in try_to_migrate_one()
2093 set_huge_pte_at(mm, address, pvmw.pte, swp_pte); in try_to_migrate_one()
2095 set_pte_at(mm, address, pvmw.pte, swp_pte); in try_to_migrate_one()
2180 DEFINE_FOLIO_VMA_WALK(pvmw, folio, vma, address, 0); in page_make_device_exclusive_one()
2195 while (page_vma_mapped_walk(&pvmw)) { in page_make_device_exclusive_one()
2197 VM_BUG_ON_FOLIO(!pvmw.pte, folio); in page_make_device_exclusive_one()
2199 if (!pte_present(*pvmw.pte)) { in page_make_device_exclusive_one()
2201 page_vma_mapped_walk_done(&pvmw); in page_make_device_exclusive_one()
2206 pte_pfn(*pvmw.pte) - folio_pfn(folio)); in page_make_device_exclusive_one()
2207 address = pvmw.address; in page_make_device_exclusive_one()
2210 flush_cache_page(vma, address, pte_pfn(*pvmw.pte)); in page_make_device_exclusive_one()
2211 pteval = ptep_clear_flush(vma, address, pvmw.pte); in page_make_device_exclusive_one()
2242 set_pte_at(mm, address, pvmw.pte, swp_pte); in page_make_device_exclusive_one()