Lines Matching refs:gup_flags
1089 unsigned int gup_flags, struct vm_area_struct **vma, in get_gate_page() argument
1101 if (gup_flags & FOLL_WRITE) in get_gate_page()
1129 if ((gup_flags & FOLL_DUMP) || !is_zero_pfn(pte_pfn(entry))) in get_gate_page()
1133 ret = try_grab_folio(page_folio(*page), 1, gup_flags); in get_gate_page()
1244 unsigned long gup_flags) in writable_file_mapping_allowed() argument
1250 if ((gup_flags & (FOLL_PIN | FOLL_LONGTERM)) != in writable_file_mapping_allowed()
1261 static int check_vma_flags(struct vm_area_struct *vma, unsigned long gup_flags) in check_vma_flags() argument
1264 int write = (gup_flags & FOLL_WRITE); in check_vma_flags()
1265 int foreign = (gup_flags & FOLL_REMOTE); in check_vma_flags()
1271 if ((gup_flags & FOLL_ANON) && !vma_anon) in check_vma_flags()
1274 if ((gup_flags & FOLL_LONGTERM) && vma_is_fsdax(vma)) in check_vma_flags()
1282 !writable_file_mapping_allowed(vma, gup_flags)) in check_vma_flags()
1286 if (!(gup_flags & FOLL_FORCE)) in check_vma_flags()
1304 if (!(gup_flags & FOLL_FORCE)) in check_vma_flags()
1417 unsigned int gup_flags, struct page **pages, in __get_user_pages() argument
1429 VM_BUG_ON(!!pages != !!(gup_flags & (FOLL_GET | FOLL_PIN))); in __get_user_pages()
1441 if (gup_flags & FOLL_MADV_POPULATE) { in __get_user_pages()
1447 if (check_vma_flags(vma, gup_flags)) { in __get_user_pages()
1456 gup_flags, &vma, in __get_user_pages()
1468 ret = check_vma_flags(vma, gup_flags); in __get_user_pages()
1483 page = follow_page_mask(vma, start, gup_flags, &ctx); in __get_user_pages()
1485 ret = faultin_page(vma, start, gup_flags, in __get_user_pages()
1542 gup_flags)) { in __get_user_pages()
1547 gup_put_folio(folio, 1, gup_flags); in __get_user_pages()
1882 int gup_flags; in populate_vma_page_range() local
1902 gup_flags = FOLL_TOUCH; in populate_vma_page_range()
1912 gup_flags |= FOLL_WRITE; in populate_vma_page_range()
1914 gup_flags |= FOLL_FORCE; in populate_vma_page_range()
1917 gup_flags |= FOLL_UNLOCKABLE; in populate_vma_page_range()
1923 ret = __get_user_pages(mm, start, nr_pages, gup_flags, in populate_vma_page_range()
1954 int gup_flags; in faultin_page_range() local
1970 gup_flags = FOLL_TOUCH | FOLL_HWPOISON | FOLL_UNLOCKABLE | in faultin_page_range()
1973 gup_flags |= FOLL_WRITE; in faultin_page_range()
1976 gup_flags); in faultin_page_range()
2517 unsigned int gup_flags) in __gup_longterm_locked() argument
2522 if (!(gup_flags & FOLL_LONGTERM)) in __gup_longterm_locked()
2524 locked, gup_flags); in __gup_longterm_locked()
2530 gup_flags); in __gup_longterm_locked()
2550 unsigned int gup_flags = *gup_flags_p; in is_valid_gup_args() local
2559 if (WARN_ON_ONCE(gup_flags & INTERNAL_GUP_FLAGS)) in is_valid_gup_args()
2562 gup_flags |= to_set; in is_valid_gup_args()
2568 gup_flags |= FOLL_UNLOCKABLE; in is_valid_gup_args()
2572 if (WARN_ON_ONCE((gup_flags & (FOLL_PIN | FOLL_GET)) == in is_valid_gup_args()
2577 if (WARN_ON_ONCE(!(gup_flags & FOLL_PIN) && (gup_flags & FOLL_LONGTERM))) in is_valid_gup_args()
2581 if (WARN_ON_ONCE((gup_flags & (FOLL_GET | FOLL_PIN)) && !pages)) in is_valid_gup_args()
2585 if (WARN_ON_ONCE((gup_flags & FOLL_LONGTERM) && in is_valid_gup_args()
2586 (gup_flags & FOLL_PCI_P2PDMA))) in is_valid_gup_args()
2589 *gup_flags_p = gup_flags; in is_valid_gup_args()
2652 unsigned int gup_flags, struct page **pages, in get_user_pages_remote() argument
2657 if (!is_valid_gup_args(pages, locked, &gup_flags, in get_user_pages_remote()
2663 gup_flags); in get_user_pages_remote()
2670 unsigned int gup_flags, struct page **pages, in get_user_pages_remote() argument
2692 unsigned int gup_flags, struct page **pages) in get_user_pages() argument
2696 if (!is_valid_gup_args(pages, NULL, &gup_flags, FOLL_TOUCH)) in get_user_pages()
2700 &locked, gup_flags); in get_user_pages()
2720 struct page **pages, unsigned int gup_flags) in get_user_pages_unlocked() argument
2724 if (!is_valid_gup_args(pages, NULL, &gup_flags, in get_user_pages_unlocked()
2729 &locked, gup_flags); in get_user_pages_unlocked()
3343 unsigned int gup_flags, struct page **pages) in gup_fast() argument
3353 if (gup_flags & FOLL_PIN) { in gup_fast()
3371 gup_fast_pgd_range(start, end, gup_flags, pages, &nr_pinned); in gup_fast()
3378 if (gup_flags & FOLL_PIN) { in gup_fast()
3390 unsigned int gup_flags, struct page **pages) in gup_fast_fallback() argument
3397 if (WARN_ON_ONCE(gup_flags & ~(FOLL_WRITE | FOLL_LONGTERM | in gup_fast_fallback()
3403 if (gup_flags & FOLL_PIN) in gup_fast_fallback()
3406 if (!(gup_flags & FOLL_FAST_ONLY)) in gup_fast_fallback()
3418 nr_pinned = gup_fast(start, end, gup_flags, pages); in gup_fast_fallback()
3419 if (nr_pinned == nr_pages || gup_flags & FOLL_FAST_ONLY) in gup_fast_fallback()
3427 gup_flags | FOLL_TOUCH | FOLL_UNLOCKABLE); in gup_fast_fallback()
3459 unsigned int gup_flags, struct page **pages) in get_user_pages_fast_only() argument
3468 if (!is_valid_gup_args(pages, NULL, &gup_flags, in get_user_pages_fast_only()
3472 return gup_fast_fallback(start, nr_pages, gup_flags, pages); in get_user_pages_fast_only()
3493 unsigned int gup_flags, struct page **pages) in get_user_pages_fast() argument
3501 if (!is_valid_gup_args(pages, NULL, &gup_flags, FOLL_GET)) in get_user_pages_fast()
3503 return gup_fast_fallback(start, nr_pages, gup_flags, pages); in get_user_pages_fast()
3527 unsigned int gup_flags, struct page **pages) in pin_user_pages_fast() argument
3529 if (!is_valid_gup_args(pages, NULL, &gup_flags, FOLL_PIN)) in pin_user_pages_fast()
3531 return gup_fast_fallback(start, nr_pages, gup_flags, pages); in pin_user_pages_fast()
3560 unsigned int gup_flags, struct page **pages, in pin_user_pages_remote() argument
3565 if (!is_valid_gup_args(pages, locked, &gup_flags, in pin_user_pages_remote()
3570 gup_flags); in pin_user_pages_remote()
3593 unsigned int gup_flags, struct page **pages) in pin_user_pages() argument
3597 if (!is_valid_gup_args(pages, NULL, &gup_flags, FOLL_PIN)) in pin_user_pages()
3600 pages, &locked, gup_flags); in pin_user_pages()
3613 struct page **pages, unsigned int gup_flags) in pin_user_pages_unlocked() argument
3617 if (!is_valid_gup_args(pages, NULL, &gup_flags, in pin_user_pages_unlocked()
3622 &locked, gup_flags); in pin_user_pages_unlocked()