Lines Matching refs:dst_addr
148 unsigned long dst_addr) in mfill_file_over_size() argument
157 offset = linear_page_index(dst_vma, dst_addr); in mfill_file_over_size()
170 unsigned long dst_addr, struct page *page, in mfill_atomic_install_pte() argument
192 dst_pte = pte_offset_map_lock(dst_mm, dst_pmd, dst_addr, &ptl); in mfill_atomic_install_pte()
196 if (mfill_file_over_size(dst_vma, dst_addr)) { in mfill_atomic_install_pte()
216 folio_add_new_anon_rmap(folio, dst_vma, dst_addr, RMAP_EXCLUSIVE); in mfill_atomic_install_pte()
226 set_pte_at(dst_mm, dst_addr, dst_pte, _dst_pte); in mfill_atomic_install_pte()
229 update_mmu_cache(dst_vma, dst_addr, dst_pte); in mfill_atomic_install_pte()
239 unsigned long dst_addr, in mfill_atomic_pte_copy() argument
251 dst_addr); in mfill_atomic_pte_copy()
302 ret = mfill_atomic_install_pte(dst_pmd, dst_vma, dst_addr, in mfill_atomic_pte_copy()
315 unsigned long dst_addr) in mfill_atomic_pte_zeroed_folio() argument
320 folio = vma_alloc_zeroed_movable_folio(dst_vma, dst_addr); in mfill_atomic_pte_zeroed_folio()
334 ret = mfill_atomic_install_pte(dst_pmd, dst_vma, dst_addr, in mfill_atomic_pte_zeroed_folio()
347 unsigned long dst_addr) in mfill_atomic_pte_zeropage() argument
354 return mfill_atomic_pte_zeroed_folio(dst_pmd, dst_vma, dst_addr); in mfill_atomic_pte_zeropage()
356 _dst_pte = pte_mkspecial(pfn_pte(my_zero_pfn(dst_addr), in mfill_atomic_pte_zeropage()
359 dst_pte = pte_offset_map_lock(dst_vma->vm_mm, dst_pmd, dst_addr, &ptl); in mfill_atomic_pte_zeropage()
362 if (mfill_file_over_size(dst_vma, dst_addr)) { in mfill_atomic_pte_zeropage()
369 set_pte_at(dst_vma->vm_mm, dst_addr, dst_pte, _dst_pte); in mfill_atomic_pte_zeropage()
371 update_mmu_cache(dst_vma, dst_addr, dst_pte); in mfill_atomic_pte_zeropage()
382 unsigned long dst_addr, in mfill_atomic_pte_continue() argument
386 pgoff_t pgoff = linear_page_index(dst_vma, dst_addr); in mfill_atomic_pte_continue()
408 ret = mfill_atomic_install_pte(dst_pmd, dst_vma, dst_addr, in mfill_atomic_pte_continue()
426 unsigned long dst_addr, in mfill_atomic_pte_poison() argument
436 dst_pte = pte_offset_map_lock(dst_mm, dst_pmd, dst_addr, &ptl); in mfill_atomic_pte_poison()
440 if (mfill_file_over_size(dst_vma, dst_addr)) { in mfill_atomic_pte_poison()
450 set_pte_at(dst_mm, dst_addr, dst_pte, _dst_pte); in mfill_atomic_pte_poison()
453 update_mmu_cache(dst_vma, dst_addr, dst_pte); in mfill_atomic_pte_poison()
499 unsigned long src_addr, dst_addr; in mfill_atomic_hugetlb() local
520 dst_addr = dst_start; in mfill_atomic_hugetlb()
564 VM_WARN_ON_ONCE(dst_addr >= dst_start + len); in mfill_atomic_hugetlb()
572 idx = linear_page_index(dst_vma, dst_addr); in mfill_atomic_hugetlb()
579 dst_pte = huge_pte_alloc(dst_mm, dst_vma, dst_addr, vma_hpagesize); in mfill_atomic_hugetlb()
587 !huge_pte_none_mostly(huge_ptep_get(dst_mm, dst_addr, dst_pte))) { in mfill_atomic_hugetlb()
594 err = hugetlb_mfill_atomic_pte(dst_pte, dst_vma, dst_addr, in mfill_atomic_hugetlb()
620 dst_addr += vma_hpagesize; in mfill_atomic_hugetlb()
655 unsigned long dst_addr, in mfill_atomic_pte() argument
664 dst_addr, flags); in mfill_atomic_pte()
667 dst_addr, flags); in mfill_atomic_pte()
683 dst_addr, src_addr, in mfill_atomic_pte()
687 dst_vma, dst_addr); in mfill_atomic_pte()
690 dst_addr, src_addr, in mfill_atomic_pte()
707 unsigned long src_addr, dst_addr; in mfill_atomic() local
722 dst_addr = dst_start; in mfill_atomic()
778 VM_WARN_ON_ONCE(dst_addr >= dst_start + len); in mfill_atomic()
780 dst_pmd = mm_alloc_pmd(dst_mm, dst_addr); in mfill_atomic()
812 err = mfill_atomic_pte(dst_pmd, dst_vma, dst_addr, in mfill_atomic()
838 dst_addr += PAGE_SIZE; in mfill_atomic()
1032 unsigned long dst_addr, unsigned long src_addr, in move_present_pte() argument
1064 src_folio->index = linear_page_index(dst_vma, dst_addr); in move_present_pte()
1075 set_pte_at(mm, dst_addr, dst_pte, orig_dst_pte); in move_present_pte()
1082 unsigned long dst_addr, unsigned long src_addr, in move_swap_pte() argument
1114 src_folio->index = linear_page_index(dst_vma, dst_addr); in move_swap_pte()
1140 set_pte_at(mm, dst_addr, dst_pte, orig_src_pte); in move_swap_pte()
1149 unsigned long dst_addr, unsigned long src_addr, in move_zeropage_pte() argument
1164 zero_pte = pte_mkspecial(pfn_pte(my_zero_pfn(dst_addr), in move_zeropage_pte()
1167 set_pte_at(mm, dst_addr, dst_pte, zero_pte); in move_zeropage_pte()
1182 unsigned long dst_addr, unsigned long src_addr, in move_pages_pte() argument
1210 dst_pte = pte_offset_map_rw_nolock(mm, dst_pmd, dst_addr, &dst_pmdval, in move_pages_pte()
1273 dst_addr, src_addr, dst_pte, src_pte, in move_pages_pte()
1379 dst_addr, src_addr, dst_pte, src_pte, in move_pages_pte()
1442 err = move_swap_pte(mm, dst_vma, dst_addr, src_addr, dst_pte, src_pte, in move_pages_pte()
1468 static inline bool move_splits_huge_pmd(unsigned long dst_addr, in move_splits_huge_pmd() argument
1472 return (src_addr & ~HPAGE_PMD_MASK) || (dst_addr & ~HPAGE_PMD_MASK) || in move_splits_huge_pmd()
1476 static inline bool move_splits_huge_pmd(unsigned long dst_addr, in move_splits_huge_pmd() argument
1735 unsigned long src_addr, dst_addr; in move_pages() local
1778 for (src_addr = src_start, dst_addr = dst_start; in move_pages()
1801 dst_pmd = mm_alloc_pmd(mm, dst_addr); in move_pages()
1822 if (move_splits_huge_pmd(dst_addr, src_addr, src_start + len) || in move_pages()
1844 dst_addr, src_addr); in move_pages()
1865 dst_addr, src_addr, mode); in move_pages()
1885 dst_addr += step_size; in move_pages()