Lines Matching refs:dst_mm
23 struct vm_area_struct *find_dst_vma(struct mm_struct *dst_mm, in find_dst_vma() argument
33 dst_vma = find_vma(dst_mm, dst_start); in find_dst_vma()
58 int mfill_atomic_install_pte(struct mm_struct *dst_mm, pmd_t *dst_pmd, in mfill_atomic_install_pte() argument
82 dst_pte = pte_offset_map_lock(dst_mm, dst_pmd, dst_addr, &ptl); in mfill_atomic_install_pte()
118 inc_mm_counter(dst_mm, mm_counter(page)); in mfill_atomic_install_pte()
120 set_pte_at(dst_mm, dst_addr, dst_pte, _dst_pte); in mfill_atomic_install_pte()
130 static int mcopy_atomic_pte(struct mm_struct *dst_mm, in mcopy_atomic_pte() argument
193 if (mem_cgroup_charge(page_folio(page), dst_mm, GFP_KERNEL)) in mcopy_atomic_pte()
196 ret = mfill_atomic_install_pte(dst_mm, dst_pmd, dst_vma, dst_addr, in mcopy_atomic_pte()
207 static int mfill_zeropage_pte(struct mm_struct *dst_mm, in mfill_zeropage_pte() argument
220 dst_pte = pte_offset_map_lock(dst_mm, dst_pmd, dst_addr, &ptl); in mfill_zeropage_pte()
233 set_pte_at(dst_mm, dst_addr, dst_pte, _dst_pte); in mfill_zeropage_pte()
243 static int mcontinue_atomic_pte(struct mm_struct *dst_mm, in mcontinue_atomic_pte() argument
272 ret = mfill_atomic_install_pte(dst_mm, dst_pmd, dst_vma, dst_addr, in mcontinue_atomic_pte()
313 static __always_inline ssize_t __mcopy_atomic_hugetlb(struct mm_struct *dst_mm, in __mcopy_atomic_hugetlb() argument
339 mmap_read_unlock(dst_mm); in __mcopy_atomic_hugetlb()
363 dst_vma = find_dst_vma(dst_mm, dst_start, len); in __mcopy_atomic_hugetlb()
399 dst_pte = huge_pte_alloc(dst_mm, dst_vma, dst_addr, vma_hpagesize); in __mcopy_atomic_hugetlb()
414 err = hugetlb_mcopy_atomic_pte(dst_mm, dst_pte, dst_vma, in __mcopy_atomic_hugetlb()
424 mmap_read_unlock(dst_mm); in __mcopy_atomic_hugetlb()
435 mmap_read_lock(dst_mm); in __mcopy_atomic_hugetlb()
455 mmap_read_unlock(dst_mm); in __mcopy_atomic_hugetlb()
466 extern ssize_t __mcopy_atomic_hugetlb(struct mm_struct *dst_mm,
475 static __always_inline ssize_t mfill_atomic_pte(struct mm_struct *dst_mm, in mfill_atomic_pte() argument
487 return mcontinue_atomic_pte(dst_mm, dst_pmd, dst_vma, dst_addr, in mfill_atomic_pte()
503 err = mcopy_atomic_pte(dst_mm, dst_pmd, dst_vma, in mfill_atomic_pte()
507 err = mfill_zeropage_pte(dst_mm, dst_pmd, in mfill_atomic_pte()
510 err = shmem_mfill_atomic_pte(dst_mm, dst_pmd, dst_vma, in mfill_atomic_pte()
519 static __always_inline ssize_t __mcopy_atomic(struct mm_struct *dst_mm, in __mcopy_atomic() argument
550 mmap_read_lock(dst_mm); in __mcopy_atomic()
566 dst_vma = find_dst_vma(dst_mm, dst_start, len); in __mcopy_atomic()
591 return __mcopy_atomic_hugetlb(dst_mm, dst_vma, dst_start, in __mcopy_atomic()
615 dst_pmd = mm_alloc_pmd(dst_mm, dst_addr); in __mcopy_atomic()
631 unlikely(__pte_alloc(dst_mm, dst_pmd))) { in __mcopy_atomic()
644 err = mfill_atomic_pte(dst_mm, dst_pmd, dst_vma, dst_addr, in __mcopy_atomic()
651 mmap_read_unlock(dst_mm); in __mcopy_atomic()
681 mmap_read_unlock(dst_mm); in __mcopy_atomic()
691 ssize_t mcopy_atomic(struct mm_struct *dst_mm, unsigned long dst_start, in mcopy_atomic() argument
695 return __mcopy_atomic(dst_mm, dst_start, src_start, len, in mcopy_atomic()
699 ssize_t mfill_zeropage(struct mm_struct *dst_mm, unsigned long start, in mfill_zeropage() argument
702 return __mcopy_atomic(dst_mm, start, 0, len, MCOPY_ATOMIC_ZEROPAGE, in mfill_zeropage()
706 ssize_t mcopy_continue(struct mm_struct *dst_mm, unsigned long start, in mcopy_continue() argument
709 return __mcopy_atomic(dst_mm, start, 0, len, MCOPY_ATOMIC_CONTINUE, in mcopy_continue()
713 long uffd_wp_range(struct mm_struct *dst_mm, struct vm_area_struct *dst_vma, in uffd_wp_range() argument
733 tlb_gather_mmu(&tlb, dst_mm); in uffd_wp_range()
740 int mwriteprotect_range(struct mm_struct *dst_mm, unsigned long start, in mwriteprotect_range() argument
757 mmap_read_lock(dst_mm); in mwriteprotect_range()
769 dst_vma = find_dst_vma(dst_mm, start, len); in mwriteprotect_range()
785 err = uffd_wp_range(dst_mm, dst_vma, start, len, enable_wp); in mwriteprotect_range()
792 mmap_read_unlock(dst_mm); in mwriteprotect_range()