Lines Matching refs:hpage
810 static bool hpage_collapse_alloc_page(struct page **hpage, gfp_t gfp, int node, in hpage_collapse_alloc_page() argument
813 *hpage = __alloc_pages(gfp, HPAGE_PMD_ORDER, node, nmask); in hpage_collapse_alloc_page()
814 if (unlikely(!*hpage)) { in hpage_collapse_alloc_page()
819 prep_transhuge_page(*hpage); in hpage_collapse_alloc_page()
969 static int alloc_charge_hpage(struct page **hpage, struct mm_struct *mm, in alloc_charge_hpage() argument
976 if (!hpage_collapse_alloc_page(hpage, gfp, node, &cc->alloc_nmask)) in alloc_charge_hpage()
978 if (unlikely(mem_cgroup_charge(page_folio(*hpage), mm, gfp))) in alloc_charge_hpage()
980 count_memcg_page_event(*hpage, THP_COLLAPSE_ALLOC); in alloc_charge_hpage()
992 struct page *hpage; in collapse_huge_page() local
1008 result = alloc_charge_hpage(&hpage, mm, cc); in collapse_huge_page()
1101 __collapse_huge_page_copy(pte, hpage, vma, address, pte_ptl, in collapse_huge_page()
1110 __SetPageUptodate(hpage); in collapse_huge_page()
1113 _pmd = mk_huge_pmd(hpage, vma->vm_page_prot); in collapse_huge_page()
1118 page_add_new_anon_rmap(hpage, vma, address); in collapse_huge_page()
1119 lru_cache_add_inactive_or_unevictable(hpage, vma); in collapse_huge_page()
1125 hpage = NULL; in collapse_huge_page()
1131 if (hpage) { in collapse_huge_page()
1132 mem_cgroup_uncharge(page_folio(hpage)); in collapse_huge_page()
1133 put_page(hpage); in collapse_huge_page()
1376 pmd_t *pmdp, struct page *hpage) in set_huge_pmd() argument
1385 VM_BUG_ON(!PageTransHuge(hpage)); in set_huge_pmd()
1388 if (do_set_pmd(&vmf, hpage)) in set_huge_pmd()
1391 get_page(hpage); in set_huge_pmd()
1454 struct page *hpage; in collapse_pte_mapped_thp() local
1486 hpage = find_lock_page(vma->vm_file->f_mapping, in collapse_pte_mapped_thp()
1488 if (!hpage) in collapse_pte_mapped_thp()
1491 if (!PageHead(hpage)) { in collapse_pte_mapped_thp()
1496 if (compound_order(hpage) != HPAGE_PMD_ORDER) { in collapse_pte_mapped_thp()
1554 if (hpage + i != page) in collapse_pte_mapped_thp()
1576 page_ref_sub(hpage, count); in collapse_pte_mapped_thp()
1577 add_mm_counter(vma->vm_mm, mm_counter_file(hpage), -count); in collapse_pte_mapped_thp()
1594 ? set_huge_pmd(vma, haddr, pmd, hpage) in collapse_pte_mapped_thp()
1598 unlock_page(hpage); in collapse_pte_mapped_thp()
1599 put_page(hpage); in collapse_pte_mapped_thp()
1633 unsigned long target_addr, struct page *hpage, in retract_page_tables() argument
1722 result = set_huge_pmd(vma, addr, pmd, hpage); in retract_page_tables()
1775 struct page *hpage; in collapse_file() local
1786 result = alloc_charge_hpage(&hpage, mm, cc); in collapse_file()
1806 __SetPageLocked(hpage); in collapse_file()
1808 __SetPageSwapBacked(hpage); in collapse_file()
1809 hpage->index = start; in collapse_file()
1810 hpage->mapping = mapping; in collapse_file()
1842 xas_store(&xas, hpage); in collapse_file()
1994 xas_store(&xas, hpage); in collapse_file()
2001 nr = thp_nr_pages(hpage); in collapse_file()
2004 __mod_lruvec_page_state(hpage, NR_SHMEM_THPS, nr); in collapse_file()
2006 __mod_lruvec_page_state(hpage, NR_FILE_THPS, nr); in collapse_file()
2017 __mod_lruvec_page_state(hpage, NR_FILE_THPS, -nr); in collapse_file()
2024 __mod_lruvec_page_state(hpage, NR_FILE_PAGES, nr_none); in collapse_file()
2026 __mod_lruvec_page_state(hpage, NR_SHMEM, nr_none); in collapse_file()
2031 xas_store(&xas, hpage); in collapse_file()
2054 clear_highpage(hpage + (index % HPAGE_PMD_NR)); in collapse_file()
2057 copy_highpage(hpage + (page->index % HPAGE_PMD_NR), in collapse_file()
2069 clear_highpage(hpage + (index % HPAGE_PMD_NR)); in collapse_file()
2073 folio = page_folio(hpage); in collapse_file()
2084 result = retract_page_tables(mapping, start, mm, addr, hpage, in collapse_file()
2086 unlock_page(hpage); in collapse_file()
2087 hpage = NULL; in collapse_file()
2126 hpage->mapping = NULL; in collapse_file()
2129 if (hpage) in collapse_file()
2130 unlock_page(hpage); in collapse_file()
2133 if (hpage) { in collapse_file()
2134 mem_cgroup_uncharge(page_folio(hpage)); in collapse_file()
2135 put_page(hpage); in collapse_file()
2138 trace_mm_khugepaged_collapse_file(mm, hpage, index, is_shmem, addr, file, nr, result); in collapse_file()