Lines Matching refs:vma
3245 static bool vma_is_accessed(struct mm_struct *mm, struct vm_area_struct *vma) in vma_is_accessed() argument
3254 if ((READ_ONCE(current->mm->numa_scan_seq) - vma->numab_state->start_scan_seq) < 2) in vma_is_accessed()
3257 pids = vma->numab_state->pids_active[0] | vma->numab_state->pids_active[1]; in vma_is_accessed()
3265 if (mm->numa_scan_offset > vma->vm_start) { in vma_is_accessed()
3266 trace_sched_skip_vma_numa(mm, vma, NUMAB_SKIP_IGNORE_PID); in vma_is_accessed()
3276 (vma->numab_state->prev_scan_seq + get_nr_threads(current))) in vma_is_accessed()
3294 struct vm_area_struct *vma; in task_numa_work() local
3372 vma = vma_next(&vmi); in task_numa_work()
3373 if (!vma) { in task_numa_work()
3377 vma = vma_next(&vmi); in task_numa_work()
3380 for (; vma; vma = vma_next(&vmi)) { in task_numa_work()
3381 if (!vma_migratable(vma) || !vma_policy_mof(vma) || in task_numa_work()
3382 is_vm_hugetlb_page(vma) || (vma->vm_flags & VM_MIXEDMAP)) { in task_numa_work()
3383 trace_sched_skip_vma_numa(mm, vma, NUMAB_SKIP_UNSUITABLE); in task_numa_work()
3393 if (!vma->vm_mm || in task_numa_work()
3394 (vma->vm_file && (vma->vm_flags & (VM_READ|VM_WRITE)) == (VM_READ))) { in task_numa_work()
3395 trace_sched_skip_vma_numa(mm, vma, NUMAB_SKIP_SHARED_RO); in task_numa_work()
3403 if (!vma_is_accessible(vma)) { in task_numa_work()
3404 trace_sched_skip_vma_numa(mm, vma, NUMAB_SKIP_INACCESSIBLE); in task_numa_work()
3409 if (!vma->numab_state) { in task_numa_work()
3416 if (cmpxchg(&vma->numab_state, NULL, ptr)) { in task_numa_work()
3421 vma->numab_state->start_scan_seq = mm->numa_scan_seq; in task_numa_work()
3423 vma->numab_state->next_scan = now + in task_numa_work()
3427 vma->numab_state->pids_active_reset = vma->numab_state->next_scan + in task_numa_work()
3435 vma->numab_state->prev_scan_seq = mm->numa_scan_seq - 1; in task_numa_work()
3443 vma->numab_state->next_scan)) { in task_numa_work()
3444 trace_sched_skip_vma_numa(mm, vma, NUMAB_SKIP_SCAN_DELAY); in task_numa_work()
3450 time_after(jiffies, vma->numab_state->pids_active_reset)) { in task_numa_work()
3451 vma->numab_state->pids_active_reset = vma->numab_state->pids_active_reset + in task_numa_work()
3453 vma->numab_state->pids_active[0] = READ_ONCE(vma->numab_state->pids_active[1]); in task_numa_work()
3454 vma->numab_state->pids_active[1] = 0; in task_numa_work()
3458 if (vma->numab_state->prev_scan_seq == mm->numa_scan_seq) { in task_numa_work()
3459 mm->numa_scan_offset = vma->vm_end; in task_numa_work()
3460 trace_sched_skip_vma_numa(mm, vma, NUMAB_SKIP_SEQ_COMPLETED); in task_numa_work()
3468 if (!vma_pids_forced && !vma_is_accessed(mm, vma)) { in task_numa_work()
3470 trace_sched_skip_vma_numa(mm, vma, NUMAB_SKIP_PID_INACTIVE); in task_numa_work()
3475 start = max(start, vma->vm_start); in task_numa_work()
3477 end = min(end, vma->vm_end); in task_numa_work()
3478 nr_pte_updates = change_prot_numa(vma, start, end); in task_numa_work()
3497 } while (end != vma->vm_end); in task_numa_work()
3500 vma->numab_state->prev_scan_seq = mm->numa_scan_seq; in task_numa_work()
3515 if (!vma && !vma_pids_forced && vma_pids_skipped) { in task_numa_work()
3527 if (vma) in task_numa_work()