Lines Matching refs:mm_state
3289 filter = READ_ONCE(lruvec->mm_state.filters[gen]); in test_bloom_filter()
3304 filter = READ_ONCE(lruvec->mm_state.filters[gen]); in update_bloom_filter()
3321 filter = lruvec->mm_state.filters[gen]; in reset_bloom_filter()
3329 WRITE_ONCE(lruvec->mm_state.filters[gen], filter); in reset_bloom_filter()
3369 if (lruvec->mm_state.tail == &mm_list->fifo) in lru_gen_add_mm()
3370 lruvec->mm_state.tail = &mm->lru_gen.list; in lru_gen_add_mm()
3398 if (lruvec->mm_state.tail == &mm->lru_gen.list) in lru_gen_del_mm()
3399 lruvec->mm_state.tail = lruvec->mm_state.tail->next; in lru_gen_del_mm()
3402 if (lruvec->mm_state.head != &mm->lru_gen.list) in lru_gen_del_mm()
3405 lruvec->mm_state.head = lruvec->mm_state.head->next; in lru_gen_del_mm()
3407 if (lruvec->mm_state.head == &mm_list->fifo) in lru_gen_del_mm()
3408 WRITE_ONCE(lruvec->mm_state.seq, lruvec->mm_state.seq + 1); in lru_gen_del_mm()
3462 WRITE_ONCE(lruvec->mm_state.stats[hist][i], in reset_mm_stats()
3463 lruvec->mm_state.stats[hist][i] + walk->mm_stats[i]); in reset_mm_stats()
3469 hist = lru_hist_from_seq(lruvec->mm_state.seq + 1); in reset_mm_stats()
3472 WRITE_ONCE(lruvec->mm_state.stats[hist][i], 0); in reset_mm_stats()
3508 struct lru_gen_mm_state *mm_state = &lruvec->mm_state; in iterate_mm_list() local
3523 VM_WARN_ON_ONCE(mm_state->seq + 1 < walk->max_seq); in iterate_mm_list()
3524 VM_WARN_ON_ONCE(*iter && mm_state->seq > walk->max_seq); in iterate_mm_list()
3525 VM_WARN_ON_ONCE(*iter && !mm_state->nr_walkers); in iterate_mm_list()
3527 if (walk->max_seq <= mm_state->seq) { in iterate_mm_list()
3533 if (!mm_state->nr_walkers) { in iterate_mm_list()
3534 VM_WARN_ON_ONCE(mm_state->head && mm_state->head != &mm_list->fifo); in iterate_mm_list()
3536 mm_state->head = mm_list->fifo.next; in iterate_mm_list()
3540 while (!mm && mm_state->head != &mm_list->fifo) { in iterate_mm_list()
3541 mm = list_entry(mm_state->head, struct mm_struct, lru_gen.list); in iterate_mm_list()
3543 mm_state->head = mm_state->head->next; in iterate_mm_list()
3546 if (!mm_state->tail || mm_state->tail == &mm->lru_gen.list) { in iterate_mm_list()
3547 mm_state->tail = mm_state->head; in iterate_mm_list()
3555 if (mm_state->head == &mm_list->fifo) in iterate_mm_list()
3556 WRITE_ONCE(mm_state->seq, mm_state->seq + 1); in iterate_mm_list()
3559 mm_state->nr_walkers--; in iterate_mm_list()
3561 mm_state->nr_walkers++; in iterate_mm_list()
3563 if (mm_state->nr_walkers) in iterate_mm_list()
3587 struct lru_gen_mm_state *mm_state = &lruvec->mm_state; in iterate_mm_list_nowalk() local
3591 VM_WARN_ON_ONCE(mm_state->seq + 1 < max_seq); in iterate_mm_list_nowalk()
3593 if (max_seq > mm_state->seq && !mm_state->nr_walkers) { in iterate_mm_list_nowalk()
3594 VM_WARN_ON_ONCE(mm_state->head && mm_state->head != &mm_list->fifo); in iterate_mm_list_nowalk()
3596 WRITE_ONCE(mm_state->seq, mm_state->seq + 1); in iterate_mm_list_nowalk()
4196 if (wq_has_sleeper(&walk->lruvec->mm_state.wait)) in walk_pud_range()
4434 if (max_seq <= READ_ONCE(lruvec->mm_state.seq)) { in try_to_inc_max_seq()
4471 wait_event_killable(lruvec->mm_state.wait, in try_to_inc_max_seq()
4480 if (wq_has_sleeper(&lruvec->mm_state.wait)) in try_to_inc_max_seq()
4481 wake_up_all(&lruvec->mm_state.wait); in try_to_inc_max_seq()
5842 n = READ_ONCE(lruvec->mm_state.stats[hist][i]); in lru_gen_seq_show_full()
5845 n = READ_ONCE(lruvec->mm_state.stats[hist][i]); in lru_gen_seq_show_full()
6124 lruvec->mm_state.seq = MIN_NR_GENS; in lru_gen_init_lruvec()
6125 init_waitqueue_head(&lruvec->mm_state.wait); in lru_gen_init_lruvec()
6158 VM_WARN_ON_ONCE(lruvec->mm_state.nr_walkers); in lru_gen_exit_memcg()
6165 bitmap_free(lruvec->mm_state.filters[i]); in lru_gen_exit_memcg()
6166 lruvec->mm_state.filters[i] = NULL; in lru_gen_exit_memcg()