Lines Matching refs:vmf
1217 static vm_fault_t __do_huge_pmd_anonymous_page(struct vm_fault *vmf) in __do_huge_pmd_anonymous_page() argument
1219 unsigned long haddr = vmf->address & HPAGE_PMD_MASK; in __do_huge_pmd_anonymous_page()
1220 struct vm_area_struct *vma = vmf->vma; in __do_huge_pmd_anonymous_page()
1225 folio = vma_alloc_anon_folio_pmd(vma, vmf->address); in __do_huge_pmd_anonymous_page()
1235 vmf->ptl = pmd_lock(vma->vm_mm, vmf->pmd); in __do_huge_pmd_anonymous_page()
1236 if (unlikely(!pmd_none(*vmf->pmd))) { in __do_huge_pmd_anonymous_page()
1245 spin_unlock(vmf->ptl); in __do_huge_pmd_anonymous_page()
1248 ret = handle_userfault(vmf, VM_UFFD_MISSING); in __do_huge_pmd_anonymous_page()
1252 pgtable_trans_huge_deposit(vma->vm_mm, vmf->pmd, pgtable); in __do_huge_pmd_anonymous_page()
1253 map_anon_folio_pmd(folio, vmf->pmd, vma, haddr); in __do_huge_pmd_anonymous_page()
1256 spin_unlock(vmf->ptl); in __do_huge_pmd_anonymous_page()
1261 spin_unlock(vmf->ptl); in __do_huge_pmd_anonymous_page()
1317 vm_fault_t do_huge_pmd_anonymous_page(struct vm_fault *vmf) in do_huge_pmd_anonymous_page() argument
1319 struct vm_area_struct *vma = vmf->vma; in do_huge_pmd_anonymous_page()
1320 unsigned long haddr = vmf->address & HPAGE_PMD_MASK; in do_huge_pmd_anonymous_page()
1325 ret = vmf_anon_prepare(vmf); in do_huge_pmd_anonymous_page()
1330 if (!(vmf->flags & FAULT_FLAG_WRITE) && in do_huge_pmd_anonymous_page()
1346 vmf->ptl = pmd_lock(vma->vm_mm, vmf->pmd); in do_huge_pmd_anonymous_page()
1348 if (pmd_none(*vmf->pmd)) { in do_huge_pmd_anonymous_page()
1351 spin_unlock(vmf->ptl); in do_huge_pmd_anonymous_page()
1354 spin_unlock(vmf->ptl); in do_huge_pmd_anonymous_page()
1356 ret = handle_userfault(vmf, VM_UFFD_MISSING); in do_huge_pmd_anonymous_page()
1360 haddr, vmf->pmd, zero_folio); in do_huge_pmd_anonymous_page()
1361 update_mmu_cache_pmd(vma, vmf->address, vmf->pmd); in do_huge_pmd_anonymous_page()
1362 spin_unlock(vmf->ptl); in do_huge_pmd_anonymous_page()
1365 spin_unlock(vmf->ptl); in do_huge_pmd_anonymous_page()
1371 return __do_huge_pmd_anonymous_page(vmf); in do_huge_pmd_anonymous_page()
1444 vm_fault_t vmf_insert_pfn_pmd(struct vm_fault *vmf, unsigned long pfn, in vmf_insert_pfn_pmd() argument
1447 unsigned long addr = vmf->address & PMD_MASK; in vmf_insert_pfn_pmd()
1448 struct vm_area_struct *vma = vmf->vma; in vmf_insert_pfn_pmd()
1478 ptl = pmd_lock(vma->vm_mm, vmf->pmd); in vmf_insert_pfn_pmd()
1479 error = insert_pmd(vma, addr, vmf->pmd, fop, pgprot, write, in vmf_insert_pfn_pmd()
1489 vm_fault_t vmf_insert_folio_pmd(struct vm_fault *vmf, struct folio *folio, in vmf_insert_folio_pmd() argument
1492 struct vm_area_struct *vma = vmf->vma; in vmf_insert_folio_pmd()
1493 unsigned long addr = vmf->address & PMD_MASK; in vmf_insert_folio_pmd()
1515 ptl = pmd_lock(mm, vmf->pmd); in vmf_insert_folio_pmd()
1516 error = insert_pmd(vma, addr, vmf->pmd, fop, vma->vm_page_prot, in vmf_insert_folio_pmd()
1583 vm_fault_t vmf_insert_pfn_pud(struct vm_fault *vmf, unsigned long pfn, in vmf_insert_pfn_pud() argument
1586 unsigned long addr = vmf->address & PUD_MASK; in vmf_insert_pfn_pud()
1587 struct vm_area_struct *vma = vmf->vma; in vmf_insert_pfn_pud()
1609 ptl = pud_lock(vma->vm_mm, vmf->pud); in vmf_insert_pfn_pud()
1610 insert_pud(vma, addr, vmf->pud, fop, pgprot, write); in vmf_insert_pfn_pud()
1625 vm_fault_t vmf_insert_folio_pud(struct vm_fault *vmf, struct folio *folio, in vmf_insert_folio_pud() argument
1628 struct vm_area_struct *vma = vmf->vma; in vmf_insert_folio_pud()
1629 unsigned long addr = vmf->address & PUD_MASK; in vmf_insert_folio_pud()
1630 pud_t *pud = vmf->pud; in vmf_insert_folio_pud()
1645 insert_pud(vma, addr, vmf->pud, fop, vma->vm_page_prot, write); in vmf_insert_folio_pud()
1838 void huge_pud_set_accessed(struct vm_fault *vmf, pud_t orig_pud) in huge_pud_set_accessed() argument
1840 bool write = vmf->flags & FAULT_FLAG_WRITE; in huge_pud_set_accessed()
1842 vmf->ptl = pud_lock(vmf->vma->vm_mm, vmf->pud); in huge_pud_set_accessed()
1843 if (unlikely(!pud_same(*vmf->pud, orig_pud))) in huge_pud_set_accessed()
1846 touch_pud(vmf->vma, vmf->address, vmf->pud, write); in huge_pud_set_accessed()
1848 spin_unlock(vmf->ptl); in huge_pud_set_accessed()
1852 void huge_pmd_set_accessed(struct vm_fault *vmf) in huge_pmd_set_accessed() argument
1854 bool write = vmf->flags & FAULT_FLAG_WRITE; in huge_pmd_set_accessed()
1856 vmf->ptl = pmd_lock(vmf->vma->vm_mm, vmf->pmd); in huge_pmd_set_accessed()
1857 if (unlikely(!pmd_same(*vmf->pmd, vmf->orig_pmd))) in huge_pmd_set_accessed()
1860 touch_pmd(vmf->vma, vmf->address, vmf->pmd, write); in huge_pmd_set_accessed()
1863 spin_unlock(vmf->ptl); in huge_pmd_set_accessed()
1866 static vm_fault_t do_huge_zero_wp_pmd(struct vm_fault *vmf) in do_huge_zero_wp_pmd() argument
1868 unsigned long haddr = vmf->address & HPAGE_PMD_MASK; in do_huge_zero_wp_pmd()
1869 struct vm_area_struct *vma = vmf->vma; in do_huge_zero_wp_pmd()
1874 folio = vma_alloc_anon_folio_pmd(vma, vmf->address); in do_huge_zero_wp_pmd()
1881 vmf->ptl = pmd_lock(vma->vm_mm, vmf->pmd); in do_huge_zero_wp_pmd()
1882 if (unlikely(!pmd_same(pmdp_get(vmf->pmd), vmf->orig_pmd))) in do_huge_zero_wp_pmd()
1887 (void)pmdp_huge_clear_flush(vma, haddr, vmf->pmd); in do_huge_zero_wp_pmd()
1888 map_anon_folio_pmd(folio, vmf->pmd, vma, haddr); in do_huge_zero_wp_pmd()
1893 spin_unlock(vmf->ptl); in do_huge_zero_wp_pmd()
1898 vm_fault_t do_huge_pmd_wp_page(struct vm_fault *vmf) in do_huge_pmd_wp_page() argument
1900 const bool unshare = vmf->flags & FAULT_FLAG_UNSHARE; in do_huge_pmd_wp_page()
1901 struct vm_area_struct *vma = vmf->vma; in do_huge_pmd_wp_page()
1904 unsigned long haddr = vmf->address & HPAGE_PMD_MASK; in do_huge_pmd_wp_page()
1905 pmd_t orig_pmd = vmf->orig_pmd; in do_huge_pmd_wp_page()
1907 vmf->ptl = pmd_lockptr(vma->vm_mm, vmf->pmd); in do_huge_pmd_wp_page()
1911 vm_fault_t ret = do_huge_zero_wp_pmd(vmf); in do_huge_pmd_wp_page()
1920 spin_lock(vmf->ptl); in do_huge_pmd_wp_page()
1922 if (unlikely(!pmd_same(*vmf->pmd, orig_pmd))) { in do_huge_pmd_wp_page()
1923 spin_unlock(vmf->ptl); in do_huge_pmd_wp_page()
1937 spin_unlock(vmf->ptl); in do_huge_pmd_wp_page()
1939 spin_lock(vmf->ptl); in do_huge_pmd_wp_page()
1940 if (unlikely(!pmd_same(*vmf->pmd, orig_pmd))) { in do_huge_pmd_wp_page()
1941 spin_unlock(vmf->ptl); in do_huge_pmd_wp_page()
1973 spin_unlock(vmf->ptl); in do_huge_pmd_wp_page()
1978 if (pmdp_set_access_flags(vma, haddr, vmf->pmd, entry, 1)) in do_huge_pmd_wp_page()
1979 update_mmu_cache_pmd(vma, vmf->address, vmf->pmd); in do_huge_pmd_wp_page()
1980 spin_unlock(vmf->ptl); in do_huge_pmd_wp_page()
1986 spin_unlock(vmf->ptl); in do_huge_pmd_wp_page()
1988 __split_huge_pmd(vma, vmf->pmd, vmf->address, false); in do_huge_pmd_wp_page()
2023 vm_fault_t do_huge_pmd_numa_page(struct vm_fault *vmf) in do_huge_pmd_numa_page() argument
2025 struct vm_area_struct *vma = vmf->vma; in do_huge_pmd_numa_page()
2027 unsigned long haddr = vmf->address & HPAGE_PMD_MASK; in do_huge_pmd_numa_page()
2034 vmf->ptl = pmd_lock(vma->vm_mm, vmf->pmd); in do_huge_pmd_numa_page()
2035 old_pmd = pmdp_get(vmf->pmd); in do_huge_pmd_numa_page()
2037 if (unlikely(!pmd_same(old_pmd, vmf->orig_pmd))) { in do_huge_pmd_numa_page()
2038 spin_unlock(vmf->ptl); in do_huge_pmd_numa_page()
2050 can_change_pmd_writable(vma, vmf->address, pmd)) in do_huge_pmd_numa_page()
2059 target_nid = numa_migrate_check(folio, vmf, haddr, &flags, writable, in do_huge_pmd_numa_page()
2068 spin_unlock(vmf->ptl); in do_huge_pmd_numa_page()
2079 vmf->ptl = pmd_lock(vma->vm_mm, vmf->pmd); in do_huge_pmd_numa_page()
2080 if (unlikely(!pmd_same(pmdp_get(vmf->pmd), vmf->orig_pmd))) { in do_huge_pmd_numa_page()
2081 spin_unlock(vmf->ptl); in do_huge_pmd_numa_page()
2086 pmd = pmd_modify(pmdp_get(vmf->pmd), vma->vm_page_prot); in do_huge_pmd_numa_page()
2090 set_pmd_at(vma->vm_mm, haddr, vmf->pmd, pmd); in do_huge_pmd_numa_page()
2091 update_mmu_cache_pmd(vma, vmf->address, vmf->pmd); in do_huge_pmd_numa_page()
2092 spin_unlock(vmf->ptl); in do_huge_pmd_numa_page()