Lines Matching refs:folio
63 struct folio *fault_folio = migrate->fault_page ? in migrate_vma_collect_pmd()
76 struct folio *folio; in migrate_vma_collect_pmd() local
84 folio = pmd_folio(*pmdp); in migrate_vma_collect_pmd()
85 if (is_huge_zero_folio(folio)) { in migrate_vma_collect_pmd()
91 folio_get(folio); in migrate_vma_collect_pmd()
94 if (WARN_ON_ONCE(fault_folio == folio)) in migrate_vma_collect_pmd()
97 if (unlikely(!folio_trylock(folio))) in migrate_vma_collect_pmd()
100 ret = split_folio(folio); in migrate_vma_collect_pmd()
101 if (fault_folio != folio) in migrate_vma_collect_pmd()
102 folio_unlock(folio); in migrate_vma_collect_pmd()
103 folio_put(folio); in migrate_vma_collect_pmd()
118 struct folio *folio; in migrate_vma_collect_pmd() local
193 folio = page_folio(page); in migrate_vma_collect_pmd()
194 folio_get(folio); in migrate_vma_collect_pmd()
208 if (fault_folio == folio || folio_trylock(folio)) { in migrate_vma_collect_pmd()
213 anon_exclusive = folio_test_anon(folio) && in migrate_vma_collect_pmd()
218 if (folio_try_share_anon_rmap_pte(folio, page)) { in migrate_vma_collect_pmd()
220 if (fault_folio != folio) in migrate_vma_collect_pmd()
221 folio_unlock(folio); in migrate_vma_collect_pmd()
222 folio_put(folio); in migrate_vma_collect_pmd()
234 folio_mark_dirty(folio); in migrate_vma_collect_pmd()
271 folio_remove_rmap_pte(folio, page, vma); in migrate_vma_collect_pmd()
272 folio_put(folio); in migrate_vma_collect_pmd()
277 folio_put(folio); in migrate_vma_collect_pmd()
341 struct folio *folio = page_folio(page); in migrate_vma_check_page() local
355 if (folio_test_large(folio)) in migrate_vma_check_page()
359 if (folio_is_zone_device(folio)) in migrate_vma_check_page()
363 if (folio_mapping(folio)) in migrate_vma_check_page()
364 extra += 1 + folio_has_private(folio); in migrate_vma_check_page()
366 if ((folio_ref_count(folio) - extra) > folio_mapcount(folio)) in migrate_vma_check_page()
380 struct folio *fault_folio = fault_page ? in migrate_device_unmap()
390 struct folio *folio; in migrate_device_unmap() local
398 folio = page_folio(page); in migrate_device_unmap()
400 if (!folio_is_zone_device(folio)) { in migrate_device_unmap()
401 if (!folio_test_lru(folio) && allow_drain) { in migrate_device_unmap()
407 if (!folio_isolate_lru(folio)) { in migrate_device_unmap()
414 folio_put(folio); in migrate_device_unmap()
417 if (folio_mapped(folio)) in migrate_device_unmap()
418 try_to_migrate(folio, 0); in migrate_device_unmap()
420 if (folio_mapped(folio) || in migrate_device_unmap()
422 if (!folio_is_zone_device(folio)) { in migrate_device_unmap()
423 folio_get(folio); in migrate_device_unmap()
424 folio_putback_lru(folio); in migrate_device_unmap()
437 struct folio *folio; in migrate_device_unmap() local
442 folio = page_folio(page); in migrate_device_unmap()
443 remove_migration_ptes(folio, folio, 0); in migrate_device_unmap()
446 if (fault_folio != folio) in migrate_device_unmap()
447 folio_unlock(folio); in migrate_device_unmap()
448 folio_put(folio); in migrate_device_unmap()
591 struct folio *folio = page_folio(page); in migrate_vma_insert_page() local
624 if (mem_cgroup_charge(folio, vma->vm_mm, GFP_KERNEL)) in migrate_vma_insert_page()
632 __folio_mark_uptodate(folio); in migrate_vma_insert_page()
634 if (folio_is_device_private(folio)) { in migrate_vma_insert_page()
645 if (folio_is_zone_device(folio) && in migrate_vma_insert_page()
646 !folio_is_device_coherent(folio)) { in migrate_vma_insert_page()
680 folio_add_new_anon_rmap(folio, vma, addr, RMAP_EXCLUSIVE); in migrate_vma_insert_page()
681 if (!folio_is_zone_device(folio)) in migrate_vma_insert_page()
682 folio_add_lru_vma(folio, vma); in migrate_vma_insert_page()
683 folio_get(folio); in migrate_vma_insert_page()
714 struct folio *newfolio, *folio; in __migrate_device_pages() local
750 folio = page_folio(page); in __migrate_device_pages()
751 mapping = folio_mapping(folio); in __migrate_device_pages()
762 if (!folio_test_anon(folio) || in __migrate_device_pages()
763 !folio_free_swap(folio)) { in __migrate_device_pages()
776 BUG_ON(folio_test_writeback(folio)); in __migrate_device_pages()
780 r = folio_migrate_mapping(mapping, newfolio, folio, extra_cnt); in __migrate_device_pages()
784 folio_migrate_flags(newfolio, folio); in __migrate_device_pages()
826 struct folio *fault_folio = fault_page ? in __migrate_device_finalize()
831 struct folio *dst = NULL, *src = NULL; in __migrate_device_finalize()
910 struct folio *folio; in migrate_device_pfn_lock() local
912 folio = folio_get_nontail_page(pfn_to_page(pfn)); in migrate_device_pfn_lock()
913 if (!folio) in migrate_device_pfn_lock()
916 if (!folio_trylock(folio)) { in migrate_device_pfn_lock()
917 folio_put(folio); in migrate_device_pfn_lock()
983 int migrate_device_coherent_folio(struct folio *folio) in migrate_device_coherent_folio() argument
986 struct folio *dfolio; in migrate_device_coherent_folio()
988 WARN_ON_ONCE(folio_test_large(folio)); in migrate_device_coherent_folio()
990 folio_lock(folio); in migrate_device_coherent_folio()
991 src_pfn = migrate_pfn(folio_pfn(folio)) | MIGRATE_PFN_MIGRATE; in migrate_device_coherent_folio()
1010 folio_copy(dfolio, folio); in migrate_device_coherent_folio()