Lines Matching refs:map
2326 static void update_ksm_flags(struct mmap_state *map) in update_ksm_flags() argument
2328 map->vm_flags = ksm_vma_flags(map->mm, map->file, map->vm_flags); in update_ksm_flags()
2341 static int __mmap_prepare(struct mmap_state *map, struct list_head *uf) in __mmap_prepare() argument
2344 struct vma_iterator *vmi = map->vmi; in __mmap_prepare()
2345 struct vma_munmap_struct *vms = &map->vms; in __mmap_prepare()
2348 vms->vma = vma_find(vmi, map->end); in __mmap_prepare()
2349 init_vma_munmap(vms, vmi, vms->vma, map->addr, map->end, uf, in __mmap_prepare()
2354 mt_init_flags(&map->mt_detach, in __mmap_prepare()
2356 mt_on_stack(map->mt_detach); in __mmap_prepare()
2357 mas_init(&map->mas_detach, &map->mt_detach, /* addr = */ 0); in __mmap_prepare()
2359 error = vms_gather_munmap_vmas(vms, &map->mas_detach); in __mmap_prepare()
2366 map->next = vms->next; in __mmap_prepare()
2367 map->prev = vms->prev; in __mmap_prepare()
2369 map->next = vma_iter_next_rewind(vmi, &map->prev); in __mmap_prepare()
2373 if (!may_expand_vm(map->mm, map->vm_flags, map->pglen - vms->nr_pages)) in __mmap_prepare()
2377 if (accountable_mapping(map->file, map->vm_flags)) { in __mmap_prepare()
2378 map->charged = map->pglen; in __mmap_prepare()
2379 map->charged -= vms->nr_accounted; in __mmap_prepare()
2380 if (map->charged) { in __mmap_prepare()
2381 error = security_vm_enough_memory_mm(map->mm, map->charged); in __mmap_prepare()
2387 map->vm_flags |= VM_ACCOUNT; in __mmap_prepare()
2396 vms_clean_up_area(vms, &map->mas_detach); in __mmap_prepare()
2402 static int __mmap_new_file_vma(struct mmap_state *map, in __mmap_new_file_vma() argument
2405 struct vma_iterator *vmi = map->vmi; in __mmap_new_file_vma()
2408 vma->vm_file = get_file(map->file); in __mmap_new_file_vma()
2410 if (!map->file->f_op->mmap) in __mmap_new_file_vma()
2420 unmap_region(&vmi->mas, vma, map->prev, map->next); in __mmap_new_file_vma()
2426 WARN_ON_ONCE(map->addr != vma->vm_start); in __mmap_new_file_vma()
2431 VM_WARN_ON_ONCE(map->vm_flags != vma->vm_flags && in __mmap_new_file_vma()
2432 !(map->vm_flags & VM_MAYWRITE) && in __mmap_new_file_vma()
2435 map->file = vma->vm_file; in __mmap_new_file_vma()
2436 map->vm_flags = vma->vm_flags; in __mmap_new_file_vma()
2450 static int __mmap_new_vma(struct mmap_state *map, struct vm_area_struct **vmap) in __mmap_new_vma() argument
2452 struct vma_iterator *vmi = map->vmi; in __mmap_new_vma()
2461 vma = vm_area_alloc(map->mm); in __mmap_new_vma()
2465 vma_iter_config(vmi, map->addr, map->end); in __mmap_new_vma()
2466 vma_set_range(vma, map->addr, map->end, map->pgoff); in __mmap_new_vma()
2467 vm_flags_init(vma, map->vm_flags); in __mmap_new_vma()
2468 vma->vm_page_prot = map->page_prot; in __mmap_new_vma()
2475 if (map->file) in __mmap_new_vma()
2476 error = __mmap_new_file_vma(map, vma); in __mmap_new_vma()
2477 else if (map->vm_flags & VM_SHARED) in __mmap_new_vma()
2485 if (!map->check_ksm_early) { in __mmap_new_vma()
2486 update_ksm_flags(map); in __mmap_new_vma()
2487 vm_flags_init(vma, map->vm_flags); in __mmap_new_vma()
2492 WARN_ON_ONCE(!arch_validate_flags(map->vm_flags)); in __mmap_new_vma()
2498 map->mm->map_count++; in __mmap_new_vma()
2506 khugepaged_enter_vma(vma, map->vm_flags); in __mmap_new_vma()
2524 static void __mmap_complete(struct mmap_state *map, struct vm_area_struct *vma) in __mmap_complete() argument
2526 struct mm_struct *mm = map->mm; in __mmap_complete()
2532 vms_complete_munmap_vmas(&map->vms, &map->mas_detach); in __mmap_complete()
2534 vm_stat_account(mm, vma->vm_flags, map->pglen); in __mmap_complete()
2541 mm->locked_vm += map->pglen; in __mmap_complete()
2570 static int call_mmap_prepare(struct mmap_state *map) in call_mmap_prepare() argument
2574 .mm = map->mm, in call_mmap_prepare()
2575 .start = map->addr, in call_mmap_prepare()
2576 .end = map->end, in call_mmap_prepare()
2578 .pgoff = map->pgoff, in call_mmap_prepare()
2579 .file = map->file, in call_mmap_prepare()
2580 .vm_flags = map->vm_flags, in call_mmap_prepare()
2581 .page_prot = map->page_prot, in call_mmap_prepare()
2585 err = vfs_mmap_prepare(map->file, &desc); in call_mmap_prepare()
2590 map->pgoff = desc.pgoff; in call_mmap_prepare()
2591 map->file = desc.file; in call_mmap_prepare()
2592 map->vm_flags = desc.vm_flags; in call_mmap_prepare()
2593 map->page_prot = desc.page_prot; in call_mmap_prepare()
2595 map->vm_ops = desc.vm_ops; in call_mmap_prepare()
2596 map->vm_private_data = desc.private_data; in call_mmap_prepare()
2602 struct mmap_state *map) in set_vma_user_defined_fields() argument
2604 if (map->vm_ops) in set_vma_user_defined_fields()
2605 vma->vm_ops = map->vm_ops; in set_vma_user_defined_fields()
2606 vma->vm_private_data = map->vm_private_data; in set_vma_user_defined_fields()
2613 static bool can_set_ksm_flags_early(struct mmap_state *map) in can_set_ksm_flags_early() argument
2615 struct file *file = map->file; in can_set_ksm_flags_early()
2647 MMAP_STATE(map, mm, &vmi, addr, len, pgoff, vm_flags, file); in __mmap_region()
2649 map.check_ksm_early = can_set_ksm_flags_early(&map); in __mmap_region()
2651 error = __mmap_prepare(&map, uf); in __mmap_region()
2653 error = call_mmap_prepare(&map); in __mmap_region()
2657 if (map.check_ksm_early) in __mmap_region()
2658 update_ksm_flags(&map); in __mmap_region()
2661 if (map.prev || map.next) { in __mmap_region()
2662 VMG_MMAP_STATE(vmg, &map, /* vma = */ NULL); in __mmap_region()
2669 error = __mmap_new_vma(&map, &vma); in __mmap_region()
2675 set_vma_user_defined_fields(vma, &map); in __mmap_region()
2677 __mmap_complete(&map, vma); in __mmap_region()
2683 if (map.charged) in __mmap_region()
2684 vm_unacct_memory(map.charged); in __mmap_region()
2686 vms_abort_munmap_vmas(&map.vms, &map.mas_detach); in __mmap_region()