Lines Matching refs:mm
42 struct vm_area_struct *find_vma_and_prepare_anon(struct mm_struct *mm, in find_vma_and_prepare_anon() argument
47 mmap_assert_locked(mm); in find_vma_and_prepare_anon()
48 vma = vma_lookup(mm, addr); in find_vma_and_prepare_anon()
69 static struct vm_area_struct *uffd_lock_vma(struct mm_struct *mm, in uffd_lock_vma() argument
74 vma = lock_vma_under_rcu(mm, address); in uffd_lock_vma()
86 mmap_read_lock(mm); in uffd_lock_vma()
87 vma = find_vma_and_prepare_anon(mm, address); in uffd_lock_vma()
95 mmap_read_unlock(mm); in uffd_lock_vma()
461 static pmd_t *mm_alloc_pmd(struct mm_struct *mm, unsigned long address) in mm_alloc_pmd() argument
467 pgd = pgd_offset(mm, address); in mm_alloc_pmd()
468 p4d = p4d_alloc(mm, pgd, address); in mm_alloc_pmd()
471 pud = pud_alloc(mm, p4d, address); in mm_alloc_pmd()
479 return pmd_alloc(mm, pud, address); in mm_alloc_pmd()
703 struct mm_struct *dst_mm = ctx->mm; in mfill_atomic()
932 struct mm_struct *dst_mm = ctx->mm; in mwriteprotect_range()
1029 static int move_present_pte(struct mm_struct *mm, in move_present_pte() argument
1058 set_pte_at(mm, src_addr, src_pte, orig_src_pte); in move_present_pte()
1075 set_pte_at(mm, dst_addr, dst_pte, orig_dst_pte); in move_present_pte()
1081 static int move_swap_pte(struct mm_struct *mm, struct vm_area_struct *dst_vma, in move_swap_pte() argument
1136 orig_src_pte = ptep_get_and_clear(mm, src_addr, src_pte); in move_swap_pte()
1140 set_pte_at(mm, dst_addr, dst_pte, orig_src_pte); in move_swap_pte()
1146 static int move_zeropage_pte(struct mm_struct *mm, in move_zeropage_pte() argument
1167 set_pte_at(mm, dst_addr, dst_pte, zero_pte); in move_zeropage_pte()
1179 static int move_pages_pte(struct mm_struct *mm, pmd_t *dst_pmd, pmd_t *src_pmd, in move_pages_pte() argument
1200 mmu_notifier_range_init(&range, MMU_NOTIFY_CLEAR, 0, mm, in move_pages_pte()
1210 dst_pte = pte_offset_map_rw_nolock(mm, dst_pmd, dst_addr, &dst_pmdval, in move_pages_pte()
1224 src_pte = pte_offset_map_rw_nolock(mm, src_pmd, src_addr, &dummy_pmdval, in move_pages_pte()
1272 err = move_zeropage_pte(mm, dst_vma, src_vma, in move_pages_pte()
1378 err = move_present_pte(mm, dst_vma, src_vma, in move_pages_pte()
1391 migration_entry_wait(mm, src_pmd, src_addr); in move_pages_pte()
1442 err = move_swap_pte(mm, dst_vma, dst_addr, src_addr, dst_pte, src_pte, in move_pages_pte()
1528 int find_vmas_mm_locked(struct mm_struct *mm, in find_vmas_mm_locked() argument
1536 mmap_assert_locked(mm); in find_vmas_mm_locked()
1537 vma = find_vma_and_prepare_anon(mm, dst_start); in find_vmas_mm_locked()
1546 vma = vma_lookup(mm, src_start); in find_vmas_mm_locked()
1555 static int uffd_move_lock(struct mm_struct *mm, in uffd_move_lock() argument
1564 vma = uffd_lock_vma(mm, dst_start); in uffd_move_lock()
1590 *src_vmap = lock_vma_under_rcu(mm, src_start); in uffd_move_lock()
1597 mmap_read_lock(mm); in uffd_move_lock()
1598 err = find_vmas_mm_locked(mm, dst_start, src_start, dst_vmap, src_vmap); in uffd_move_lock()
1617 mmap_read_unlock(mm); in uffd_move_lock()
1631 static int uffd_move_lock(struct mm_struct *mm, in uffd_move_lock() argument
1639 mmap_read_lock(mm); in uffd_move_lock()
1640 err = find_vmas_mm_locked(mm, dst_start, src_start, dst_vmap, src_vmap); in uffd_move_lock()
1642 mmap_read_unlock(mm); in uffd_move_lock()
1733 struct mm_struct *mm = ctx->mm; in move_pages() local
1749 err = uffd_move_lock(mm, dst_start, src_start, &dst_vma, &src_vma); in move_pages()
1789 src_pmd = mm_find_pmd(mm, src_addr); in move_pages()
1795 src_pmd = mm_alloc_pmd(mm, src_addr); in move_pages()
1801 dst_pmd = mm_alloc_pmd(mm, dst_addr); in move_pages()
1842 err = move_pages_huge_pmd(mm, dst_pmd, src_pmd, in move_pages()
1852 if (unlikely(__pte_alloc(mm, src_pmd))) { in move_pages()
1858 if (unlikely(pte_alloc(mm, dst_pmd))) { in move_pages()
1863 err = move_pages_pte(mm, dst_pmd, src_pmd, in move_pages()
1972 VMA_ITERATOR(vmi, ctx->mm, start); in userfaultfd_register_range()
2028 struct mm_struct *mm = ctx->mm; in userfaultfd_release_new() local
2030 VMA_ITERATOR(vmi, mm, 0); in userfaultfd_release_new()
2033 mmap_write_lock(mm); in userfaultfd_release_new()
2038 mmap_write_unlock(mm); in userfaultfd_release_new()
2041 void userfaultfd_release_all(struct mm_struct *mm, in userfaultfd_release_all() argument
2045 VMA_ITERATOR(vmi, mm, 0); in userfaultfd_release_all()
2047 if (!mmget_not_zero(mm)) in userfaultfd_release_all()
2058 mmap_write_lock(mm); in userfaultfd_release_all()
2073 mmap_write_unlock(mm); in userfaultfd_release_all()
2074 mmput(mm); in userfaultfd_release_all()