Lines Matching refs:flags
107 static void gup_put_folio(struct folio *folio, int refs, unsigned int flags) in gup_put_folio() argument
109 if (flags & FOLL_PIN) { in gup_put_folio()
146 unsigned int flags) in try_grab_folio() argument
151 if (unlikely(!(flags & FOLL_PCI_P2PDMA) && is_pci_p2pdma_page(&folio->page))) in try_grab_folio()
154 if (flags & FOLL_GET) in try_grab_folio()
156 else if (flags & FOLL_PIN) { in try_grab_folio()
531 unsigned int flags) in try_grab_folio_fast() argument
538 if (WARN_ON_ONCE((flags & (FOLL_GET | FOLL_PIN)) == 0)) in try_grab_folio_fast()
541 if (unlikely(!(flags & FOLL_PCI_P2PDMA) && is_pci_p2pdma_page(page))) in try_grab_folio_fast()
544 if (flags & FOLL_GET) in try_grab_folio_fast()
565 if (unlikely((flags & FOLL_LONGTERM) && in try_grab_folio_fast()
599 struct vm_area_struct *vma, unsigned int flags) in can_follow_write_common() argument
602 if (!(flags & FOLL_FORCE)) in can_follow_write_common()
625 unsigned int flags, unsigned long address) in no_page_table() argument
627 if (!(flags & FOLL_DUMP)) in no_page_table()
653 unsigned int flags) in can_follow_write_pud() argument
659 return can_follow_write_common(page, vma, flags); in can_follow_write_pud()
664 int flags, struct follow_page_context *ctx) in follow_huge_pud() argument
677 if ((flags & FOLL_WRITE) && in follow_huge_pud()
678 !can_follow_write_pud(pud, pfn_to_page(pfn), vma, flags)) in follow_huge_pud()
684 if (!pud_write(pud) && gup_must_unshare(vma, flags, page)) in follow_huge_pud()
687 ret = try_grab_folio(page_folio(page), 1, flags); in follow_huge_pud()
699 unsigned int flags) in can_follow_write_pmd() argument
705 if (!can_follow_write_common(page, vma, flags)) in can_follow_write_pmd()
716 unsigned int flags, in follow_huge_pmd() argument
727 if ((flags & FOLL_WRITE) && in follow_huge_pmd()
728 !can_follow_write_pmd(pmdval, page, vma, flags)) in follow_huge_pmd()
732 if ((flags & FOLL_DUMP) && is_huge_zero_pmd(pmdval)) in follow_huge_pmd()
735 if (pmd_protnone(*pmd) && !gup_can_follow_protnone(vma, flags)) in follow_huge_pmd()
738 if (!pmd_write(pmdval) && gup_must_unshare(vma, flags, page)) in follow_huge_pmd()
741 VM_WARN_ON_ONCE_PAGE((flags & FOLL_PIN) && PageAnon(page) && in follow_huge_pmd()
744 ret = try_grab_folio(page_folio(page), 1, flags); in follow_huge_pmd()
749 if (pmd_trans_huge(pmdval) && (flags & FOLL_TOUCH)) in follow_huge_pmd()
750 touch_pmd(vma, addr, pmd, flags & FOLL_WRITE); in follow_huge_pmd()
762 int flags, struct follow_page_context *ctx) in follow_huge_pud() argument
769 unsigned int flags, in follow_huge_pmd() argument
777 pte_t *pte, unsigned int flags) in follow_pfn_pte() argument
779 if (flags & FOLL_TOUCH) { in follow_pfn_pte()
783 if (flags & FOLL_WRITE) in follow_pfn_pte()
800 unsigned int flags) in can_follow_write_pte() argument
806 if (!can_follow_write_common(page, vma, flags)) in can_follow_write_pte()
816 unsigned long address, pmd_t *pmd, unsigned int flags, in follow_page_pte() argument
828 return no_page_table(vma, flags, address); in follow_page_pte()
832 if (pte_protnone(pte) && !gup_can_follow_protnone(vma, flags)) in follow_page_pte()
840 if ((flags & FOLL_WRITE) && in follow_page_pte()
841 !can_follow_write_pte(pte, page, vma, flags)) { in follow_page_pte()
847 if (flags & FOLL_DUMP) { in follow_page_pte()
856 ret = follow_pfn_pte(vma, address, ptep, flags); in follow_page_pte()
863 if (!pte_write(pte) && gup_must_unshare(vma, flags, page)) { in follow_page_pte()
868 VM_WARN_ON_ONCE_PAGE((flags & FOLL_PIN) && PageAnon(page) && in follow_page_pte()
872 ret = try_grab_folio(folio, 1, flags); in follow_page_pte()
883 if (flags & FOLL_PIN) { in follow_page_pte()
891 if (flags & FOLL_TOUCH) { in follow_page_pte()
892 if ((flags & FOLL_WRITE) && in follow_page_pte()
909 return no_page_table(vma, flags, address); in follow_page_pte()
914 unsigned int flags, in follow_pmd_mask() argument
925 return no_page_table(vma, flags, address); in follow_pmd_mask()
927 return no_page_table(vma, flags, address); in follow_pmd_mask()
929 return follow_page_pte(vma, address, pmd, flags, &ctx->pgmap); in follow_pmd_mask()
931 if (pmd_protnone(pmdval) && !gup_can_follow_protnone(vma, flags)) in follow_pmd_mask()
932 return no_page_table(vma, flags, address); in follow_pmd_mask()
938 return no_page_table(vma, flags, address); in follow_pmd_mask()
942 return follow_page_pte(vma, address, pmd, flags, &ctx->pgmap); in follow_pmd_mask()
944 if (pmd_trans_huge(pmdval) && (flags & FOLL_SPLIT_PMD)) { in follow_pmd_mask()
949 follow_page_pte(vma, address, pmd, flags, &ctx->pgmap); in follow_pmd_mask()
951 page = follow_huge_pmd(vma, address, pmd, flags, ctx); in follow_pmd_mask()
958 unsigned int flags, in follow_pud_mask() argument
969 return no_page_table(vma, flags, address); in follow_pud_mask()
972 page = follow_huge_pud(vma, address, pudp, flags, ctx); in follow_pud_mask()
976 return no_page_table(vma, flags, address); in follow_pud_mask()
979 return no_page_table(vma, flags, address); in follow_pud_mask()
981 return follow_pmd_mask(vma, address, pudp, flags, ctx); in follow_pud_mask()
986 unsigned int flags, in follow_p4d_mask() argument
996 return no_page_table(vma, flags, address); in follow_p4d_mask()
998 return follow_pud_mask(vma, address, p4dp, flags, ctx); in follow_p4d_mask()
1026 unsigned long address, unsigned int flags, in follow_page_mask() argument
1039 page = no_page_table(vma, flags, address); in follow_page_mask()
1041 page = follow_p4d_mask(vma, address, pgd, flags, ctx); in follow_page_mask()
1106 unsigned long address, unsigned int flags, bool unshare, in faultin_page() argument
1112 if (flags & FOLL_NOFAULT) in faultin_page()
1114 if (flags & FOLL_WRITE) in faultin_page()
1116 if (flags & FOLL_REMOTE) in faultin_page()
1118 if (flags & FOLL_UNLOCKABLE) { in faultin_page()
1126 if (flags & FOLL_INTERRUPTIBLE) in faultin_page()
1129 if (flags & FOLL_NOWAIT) in faultin_page()
1131 if (flags & FOLL_TRIED) { in faultin_page()
1165 int err = vm_fault_to_errno(ret, flags); in faultin_page()
1645 static bool gup_signal_pending(unsigned int flags) in gup_signal_pending() argument
1650 if (!(flags & FOLL_INTERRUPTIBLE)) in gup_signal_pending()
1674 unsigned int flags) in __get_user_pages_locked() argument
1695 if (flags & FOLL_PIN) in __get_user_pages_locked()
1696 mm_set_has_pinned_flag(&mm->flags); in __get_user_pages_locked()
1707 if (pages && !(flags & FOLL_PIN)) in __get_user_pages_locked()
1708 flags |= FOLL_GET; in __get_user_pages_locked()
1712 ret = __get_user_pages(mm, start, nr_pages, flags, pages, in __get_user_pages_locked()
1714 if (!(flags & FOLL_UNLOCKABLE)) { in __get_user_pages_locked()
1758 if (gup_signal_pending(flags)) { in __get_user_pages_locked()
1772 ret = __get_user_pages(mm, start, 1, flags | FOLL_TRIED, in __get_user_pages_locked()
1807 if (WARN_ON_ONCE(pages_done == 0 && !(flags & FOLL_NOWAIT))) in __get_user_pages_locked()
2484 unsigned int flags; in __gup_longterm_locked() local
2491 flags = memalloc_pin_save(); in __gup_longterm_locked()
2504 memalloc_pin_restore(flags); in __gup_longterm_locked()
2750 static bool gup_fast_folio_allowed(struct folio *folio, unsigned int flags) in gup_fast_folio_allowed() argument
2761 if ((flags & (FOLL_PIN | FOLL_LONGTERM | FOLL_WRITE)) == in gup_fast_folio_allowed()
2822 unsigned int flags, struct page **pages) in gup_fast_undo_dev_pagemap() argument
2828 gup_put_folio(folio, 1, flags); in gup_fast_undo_dev_pagemap()
2853 unsigned long end, unsigned int flags, struct page **pages, in gup_fast_pte_range() argument
2878 if (!pte_access_permitted(pte, flags & FOLL_WRITE)) in gup_fast_pte_range()
2888 folio = try_grab_folio_fast(page, 1, flags); in gup_fast_pte_range()
2894 gup_put_folio(folio, 1, flags); in gup_fast_pte_range()
2898 if (!gup_fast_folio_allowed(folio, flags)) { in gup_fast_pte_range()
2899 gup_put_folio(folio, 1, flags); in gup_fast_pte_range()
2903 if (!pte_write(pte) && gup_must_unshare(NULL, flags, page)) { in gup_fast_pte_range()
2904 gup_put_folio(folio, 1, flags); in gup_fast_pte_range()
2914 if (flags & FOLL_PIN) { in gup_fast_pte_range()
2917 gup_put_folio(folio, 1, flags); in gup_fast_pte_range()
2946 unsigned long end, unsigned int flags, struct page **pages, in gup_fast_pte_range() argument
2954 unsigned long end, unsigned int flags, struct page **pages, in gup_fast_pmd_leaf() argument
2961 if (!pmd_access_permitted(orig, flags & FOLL_WRITE)) in gup_fast_pmd_leaf()
2970 folio = try_grab_folio_fast(page, refs, flags); in gup_fast_pmd_leaf()
2975 gup_put_folio(folio, refs, flags); in gup_fast_pmd_leaf()
2979 if (!gup_fast_folio_allowed(folio, flags)) { in gup_fast_pmd_leaf()
2980 gup_put_folio(folio, refs, flags); in gup_fast_pmd_leaf()
2983 if (!pmd_write(orig) && gup_must_unshare(NULL, flags, &folio->page)) { in gup_fast_pmd_leaf()
2984 gup_put_folio(folio, refs, flags); in gup_fast_pmd_leaf()
2994 unsigned long end, unsigned int flags, struct page **pages, in gup_fast_pud_leaf() argument
3001 if (!pud_access_permitted(orig, flags & FOLL_WRITE)) in gup_fast_pud_leaf()
3010 folio = try_grab_folio_fast(page, refs, flags); in gup_fast_pud_leaf()
3015 gup_put_folio(folio, refs, flags); in gup_fast_pud_leaf()
3019 if (!gup_fast_folio_allowed(folio, flags)) { in gup_fast_pud_leaf()
3020 gup_put_folio(folio, refs, flags); in gup_fast_pud_leaf()
3024 if (!pud_write(orig) && gup_must_unshare(NULL, flags, &folio->page)) { in gup_fast_pud_leaf()
3025 gup_put_folio(folio, refs, flags); in gup_fast_pud_leaf()
3035 unsigned long end, unsigned int flags, struct page **pages, in gup_fast_pmd_range() argument
3054 if (!gup_fast_pmd_leaf(pmd, pmdp, addr, next, flags, in gup_fast_pmd_range()
3058 } else if (!gup_fast_pte_range(pmd, pmdp, addr, next, flags, in gup_fast_pmd_range()
3067 unsigned long end, unsigned int flags, struct page **pages, in gup_fast_pud_range() argument
3081 if (!gup_fast_pud_leaf(pud, pudp, addr, next, flags, in gup_fast_pud_range()
3084 } else if (!gup_fast_pmd_range(pudp, pud, addr, next, flags, in gup_fast_pud_range()
3093 unsigned long end, unsigned int flags, struct page **pages, in gup_fast_p4d_range() argument
3107 if (!gup_fast_pud_range(p4dp, p4d, addr, next, flags, in gup_fast_p4d_range()
3116 unsigned int flags, struct page **pages, int *nr) in gup_fast_pgd_range() argument
3129 if (!gup_fast_p4d_range(pgdp, pgd, addr, next, flags, in gup_fast_pgd_range()
3136 unsigned int flags, struct page **pages, int *nr) in gup_fast_pgd_range() argument
3155 unsigned long flags; in gup_fast() local
3179 local_irq_save(flags); in gup_fast()
3181 local_irq_restore(flags); in gup_fast()
3213 mm_set_has_pinned_flag(¤t->mm->flags); in gup_fast_fallback()
3463 unsigned int flags, nr_folios, nr_found; in memfd_pin_folios() local
3488 flags = memalloc_pin_save(); in memfd_pin_folios()
3548 memalloc_pin_restore(flags); in memfd_pin_folios()
3551 memalloc_pin_restore(flags); in memfd_pin_folios()