Lines Matching refs:dst_pmd
168 int mfill_atomic_install_pte(pmd_t *dst_pmd, in mfill_atomic_install_pte() argument
192 dst_pte = pte_offset_map_lock(dst_mm, dst_pmd, dst_addr, &ptl); in mfill_atomic_install_pte()
237 static int mfill_atomic_pte_copy(pmd_t *dst_pmd, in mfill_atomic_pte_copy() argument
302 ret = mfill_atomic_install_pte(dst_pmd, dst_vma, dst_addr, in mfill_atomic_pte_copy()
313 static int mfill_atomic_pte_zeroed_folio(pmd_t *dst_pmd, in mfill_atomic_pte_zeroed_folio() argument
334 ret = mfill_atomic_install_pte(dst_pmd, dst_vma, dst_addr, in mfill_atomic_pte_zeroed_folio()
345 static int mfill_atomic_pte_zeropage(pmd_t *dst_pmd, in mfill_atomic_pte_zeropage() argument
354 return mfill_atomic_pte_zeroed_folio(dst_pmd, dst_vma, dst_addr); in mfill_atomic_pte_zeropage()
359 dst_pte = pte_offset_map_lock(dst_vma->vm_mm, dst_pmd, dst_addr, &ptl); in mfill_atomic_pte_zeropage()
380 static int mfill_atomic_pte_continue(pmd_t *dst_pmd, in mfill_atomic_pte_continue() argument
408 ret = mfill_atomic_install_pte(dst_pmd, dst_vma, dst_addr, in mfill_atomic_pte_continue()
424 static int mfill_atomic_pte_poison(pmd_t *dst_pmd, in mfill_atomic_pte_poison() argument
436 dst_pte = pte_offset_map_lock(dst_mm, dst_pmd, dst_addr, &ptl); in mfill_atomic_pte_poison()
653 static __always_inline ssize_t mfill_atomic_pte(pmd_t *dst_pmd, in mfill_atomic_pte() argument
663 return mfill_atomic_pte_continue(dst_pmd, dst_vma, in mfill_atomic_pte()
666 return mfill_atomic_pte_poison(dst_pmd, dst_vma, in mfill_atomic_pte()
682 err = mfill_atomic_pte_copy(dst_pmd, dst_vma, in mfill_atomic_pte()
686 err = mfill_atomic_pte_zeropage(dst_pmd, in mfill_atomic_pte()
689 err = shmem_mfill_atomic_pte(dst_pmd, dst_vma, in mfill_atomic_pte()
706 pmd_t *dst_pmd; in mfill_atomic() local
780 dst_pmd = mm_alloc_pmd(dst_mm, dst_addr); in mfill_atomic()
781 if (unlikely(!dst_pmd)) { in mfill_atomic()
786 dst_pmdval = pmdp_get_lockless(dst_pmd); in mfill_atomic()
788 unlikely(__pte_alloc(dst_mm, dst_pmd))) { in mfill_atomic()
792 dst_pmdval = pmdp_get_lockless(dst_pmd); in mfill_atomic()
812 err = mfill_atomic_pte(dst_pmd, dst_vma, dst_addr, in mfill_atomic()
1022 pmd_t *dst_pmd, pmd_t dst_pmdval) in is_pte_pages_stable() argument
1026 pmd_same(dst_pmdval, pmdp_get_lockless(dst_pmd)); in is_pte_pages_stable()
1035 pmd_t *dst_pmd, pmd_t dst_pmdval, in move_present_pte() argument
1044 dst_pmd, dst_pmdval)) { in move_present_pte()
1085 pmd_t *dst_pmd, pmd_t dst_pmdval, in move_swap_pte() argument
1102 dst_pmd, dst_pmdval)) { in move_swap_pte()
1152 pmd_t *dst_pmd, pmd_t dst_pmdval, in move_zeropage_pte() argument
1159 dst_pmd, dst_pmdval)) { in move_zeropage_pte()
1179 static int move_pages_pte(struct mm_struct *mm, pmd_t *dst_pmd, pmd_t *src_pmd, in move_pages_pte() argument
1210 dst_pte = pte_offset_map_rw_nolock(mm, dst_pmd, dst_addr, &dst_pmdval, in move_pages_pte()
1239 if (pmd_none(*dst_pmd) || pmd_none(*src_pmd) || in move_pages_pte()
1240 pmd_trans_huge(*dst_pmd) || pmd_trans_huge(*src_pmd)) { in move_pages_pte()
1275 dst_pmd, dst_pmdval, dst_ptl, src_ptl); in move_pages_pte()
1380 orig_dst_pte, orig_src_pte, dst_pmd, in move_pages_pte()
1443 orig_dst_pte, orig_src_pte, dst_pmd, dst_pmdval, in move_pages_pte()
1736 pmd_t *src_pmd, *dst_pmd; in move_pages() local
1801 dst_pmd = mm_alloc_pmd(mm, dst_addr); in move_pages()
1802 if (unlikely(!dst_pmd)) { in move_pages()
1807 dst_pmdval = pmdp_get_lockless(dst_pmd); in move_pages()
1842 err = move_pages_huge_pmd(mm, dst_pmd, src_pmd, in move_pages()
1858 if (unlikely(pte_alloc(mm, dst_pmd))) { in move_pages()
1863 err = move_pages_pte(mm, dst_pmd, src_pmd, in move_pages()