Lines Matching refs:dst_pte
175 pte_t _dst_pte, *dst_pte; in mfill_atomic_install_pte() local
192 dst_pte = pte_offset_map_lock(dst_mm, dst_pmd, dst_addr, &ptl); in mfill_atomic_install_pte()
193 if (!dst_pte) in mfill_atomic_install_pte()
207 if (!pte_none_mostly(ptep_get(dst_pte))) in mfill_atomic_install_pte()
226 set_pte_at(dst_mm, dst_addr, dst_pte, _dst_pte); in mfill_atomic_install_pte()
229 update_mmu_cache(dst_vma, dst_addr, dst_pte); in mfill_atomic_install_pte()
232 pte_unmap_unlock(dst_pte, ptl); in mfill_atomic_install_pte()
349 pte_t _dst_pte, *dst_pte; in mfill_atomic_pte_zeropage() local
359 dst_pte = pte_offset_map_lock(dst_vma->vm_mm, dst_pmd, dst_addr, &ptl); in mfill_atomic_pte_zeropage()
360 if (!dst_pte) in mfill_atomic_pte_zeropage()
367 if (!pte_none(ptep_get(dst_pte))) in mfill_atomic_pte_zeropage()
369 set_pte_at(dst_vma->vm_mm, dst_addr, dst_pte, _dst_pte); in mfill_atomic_pte_zeropage()
371 update_mmu_cache(dst_vma, dst_addr, dst_pte); in mfill_atomic_pte_zeropage()
374 pte_unmap_unlock(dst_pte, ptl); in mfill_atomic_pte_zeropage()
431 pte_t _dst_pte, *dst_pte; in mfill_atomic_pte_poison() local
436 dst_pte = pte_offset_map_lock(dst_mm, dst_pmd, dst_addr, &ptl); in mfill_atomic_pte_poison()
437 if (!dst_pte) in mfill_atomic_pte_poison()
447 if (!pte_none(ptep_get(dst_pte))) in mfill_atomic_pte_poison()
450 set_pte_at(dst_mm, dst_addr, dst_pte, _dst_pte); in mfill_atomic_pte_poison()
453 update_mmu_cache(dst_vma, dst_addr, dst_pte); in mfill_atomic_pte_poison()
456 pte_unmap_unlock(dst_pte, ptl); in mfill_atomic_pte_poison()
498 pte_t *dst_pte; in mfill_atomic_hugetlb() local
579 dst_pte = huge_pte_alloc(dst_mm, dst_vma, dst_addr, vma_hpagesize); in mfill_atomic_hugetlb()
580 if (!dst_pte) { in mfill_atomic_hugetlb()
587 !huge_pte_none_mostly(huge_ptep_get(dst_mm, dst_addr, dst_pte))) { in mfill_atomic_hugetlb()
594 err = hugetlb_mfill_atomic_pte(dst_pte, dst_vma, dst_addr, in mfill_atomic_hugetlb()
1020 static inline bool is_pte_pages_stable(pte_t *dst_pte, pte_t *src_pte, in is_pte_pages_stable() argument
1025 pte_same(ptep_get(dst_pte), orig_dst_pte) && in is_pte_pages_stable()
1033 pte_t *dst_pte, pte_t *src_pte, in move_present_pte() argument
1043 if (!is_pte_pages_stable(dst_pte, src_pte, orig_dst_pte, orig_src_pte, in move_present_pte()
1075 set_pte_at(mm, dst_addr, dst_pte, orig_dst_pte); in move_present_pte()
1083 pte_t *dst_pte, pte_t *src_pte, in move_swap_pte() argument
1101 if (!is_pte_pages_stable(dst_pte, src_pte, orig_dst_pte, orig_src_pte, in move_swap_pte()
1140 set_pte_at(mm, dst_addr, dst_pte, orig_src_pte); in move_swap_pte()
1150 pte_t *dst_pte, pte_t *src_pte, in move_zeropage_pte() argument
1158 if (!is_pte_pages_stable(dst_pte, src_pte, orig_dst_pte, orig_src_pte, in move_zeropage_pte()
1167 set_pte_at(mm, dst_addr, dst_pte, zero_pte); in move_zeropage_pte()
1191 pte_t *dst_pte = NULL; in move_pages_pte() local
1210 dst_pte = pte_offset_map_rw_nolock(mm, dst_pmd, dst_addr, &dst_pmdval, in move_pages_pte()
1214 if (unlikely(!dst_pte)) { in move_pages_pte()
1246 orig_dst_pte = ptep_get(dst_pte); in move_pages_pte()
1273 dst_addr, src_addr, dst_pte, src_pte, in move_pages_pte()
1327 pte_unmap(dst_pte); in move_pages_pte()
1328 src_pte = dst_pte = NULL; in move_pages_pte()
1344 pte_unmap(dst_pte); in move_pages_pte()
1345 src_pte = dst_pte = NULL; in move_pages_pte()
1370 pte_unmap(dst_pte); in move_pages_pte()
1371 src_pte = dst_pte = NULL; in move_pages_pte()
1379 dst_addr, src_addr, dst_pte, src_pte, in move_pages_pte()
1389 pte_unmap(dst_pte); in move_pages_pte()
1390 src_pte = dst_pte = NULL; in move_pages_pte()
1433 pte_unmap(dst_pte); in move_pages_pte()
1434 src_pte = dst_pte = NULL; in move_pages_pte()
1442 err = move_swap_pte(mm, dst_vma, dst_addr, src_addr, dst_pte, src_pte, in move_pages_pte()
1456 if (dst_pte) in move_pages_pte()
1457 pte_unmap(dst_pte); in move_pages_pte()