Lines Matching refs:type
464 #define CHECK_RECLAIMER_OFFSET(type) \ argument
466 BUILD_BUG_ON(PGSTEAL_##type - PGSTEAL_KSWAPD != \
467 PGDEMOTE_##type - PGDEMOTE_KSWAPD); \
468 BUILD_BUG_ON(PGSTEAL_##type - PGSTEAL_KSWAPD != \
469 PGSCAN_##type - PGSCAN_KSWAPD); \
1588 int type = folio_is_file_lru(folio); in shrink_folio_list() local
1590 stat->nr_activate[type] += nr_pages; in shrink_folio_list()
2727 #define for_each_gen_type_zone(gen, type, zone) \ argument
2729 for ((type) = 0; (type) < ANON_AND_FILE; (type)++) \
2732 #define for_each_evictable_type(type, swappiness) \ argument
2733 for ((type) = min_type(swappiness); (type) <= max_type(swappiness); (type)++)
2773 static int get_nr_gens(struct lruvec *lruvec, int type) in get_nr_gens() argument
2775 return lruvec->lrugen.max_seq - lruvec->lrugen.min_seq[type] + 1; in get_nr_gens()
2780 int type; in seq_is_valid() local
2782 for (type = 0; type < ANON_AND_FILE; type++) { in seq_is_valid()
2783 int n = get_nr_gens(lruvec, type); in seq_is_valid()
3187 static void read_ctrl_pos(struct lruvec *lruvec, int type, int tier, int gain, in read_ctrl_pos() argument
3192 int hist = lru_hist_from_seq(lrugen->min_seq[type]); in read_ctrl_pos()
3198 pos->refaulted += lrugen->avg_refaulted[type][i] + in read_ctrl_pos()
3199 atomic_long_read(&lrugen->refaulted[hist][type][i]); in read_ctrl_pos()
3200 pos->total += lrugen->avg_total[type][i] + in read_ctrl_pos()
3201 lrugen->protected[hist][type][i] + in read_ctrl_pos()
3202 atomic_long_read(&lrugen->evicted[hist][type][i]); in read_ctrl_pos()
3206 static void reset_ctrl_pos(struct lruvec *lruvec, int type, bool carryover) in reset_ctrl_pos() argument
3211 unsigned long seq = carryover ? lrugen->min_seq[type] : lrugen->max_seq + 1; in reset_ctrl_pos()
3224 sum = lrugen->avg_refaulted[type][tier] + in reset_ctrl_pos()
3225 atomic_long_read(&lrugen->refaulted[hist][type][tier]); in reset_ctrl_pos()
3226 WRITE_ONCE(lrugen->avg_refaulted[type][tier], sum / 2); in reset_ctrl_pos()
3228 sum = lrugen->avg_total[type][tier] + in reset_ctrl_pos()
3229 lrugen->protected[hist][type][tier] + in reset_ctrl_pos()
3230 atomic_long_read(&lrugen->evicted[hist][type][tier]); in reset_ctrl_pos()
3231 WRITE_ONCE(lrugen->avg_total[type][tier], sum / 2); in reset_ctrl_pos()
3235 atomic_long_set(&lrugen->refaulted[hist][type][tier], 0); in reset_ctrl_pos()
3236 atomic_long_set(&lrugen->evicted[hist][type][tier], 0); in reset_ctrl_pos()
3237 WRITE_ONCE(lrugen->protected[hist][type][tier], 0); in reset_ctrl_pos()
3285 int type = folio_is_file_lru(folio); in folio_inc_gen() local
3287 int new_gen, old_gen = lru_gen_from_seq(lrugen->min_seq[type]); in folio_inc_gen()
3315 int type = folio_is_file_lru(folio); in update_batch_size() local
3324 walk->nr_pages[old_gen][type][zone] -= delta; in update_batch_size()
3325 walk->nr_pages[new_gen][type][zone] += delta; in update_batch_size()
3330 int gen, type, zone; in reset_batch_size() local
3336 for_each_gen_type_zone(gen, type, zone) { in reset_batch_size()
3337 enum lru_list lru = type * LRU_INACTIVE_FILE; in reset_batch_size()
3338 int delta = walk->nr_pages[gen][type][zone]; in reset_batch_size()
3343 walk->nr_pages[gen][type][zone] = 0; in reset_batch_size()
3344 WRITE_ONCE(lrugen->nr_pages[gen][type][zone], in reset_batch_size()
3345 lrugen->nr_pages[gen][type][zone] + delta); in reset_batch_size()
3873 static bool inc_min_seq(struct lruvec *lruvec, int type, int swappiness) in inc_min_seq() argument
3878 int hist = lru_hist_from_seq(lrugen->min_seq[type]); in inc_min_seq()
3879 int new_gen, old_gen = lru_gen_from_seq(lrugen->min_seq[type]); in inc_min_seq()
3882 if (type && (swappiness == SWAPPINESS_ANON_ONLY)) in inc_min_seq()
3886 if (!type && !swappiness) in inc_min_seq()
3891 struct list_head *head = &lrugen->folios[old_gen][type][zone]; in inc_min_seq()
3900 VM_WARN_ON_ONCE_FOLIO(folio_is_file_lru(folio) != type, folio); in inc_min_seq()
3904 list_move_tail(&folio->lru, &lrugen->folios[new_gen][type][zone]); in inc_min_seq()
3911 WRITE_ONCE(lrugen->protected[hist][type][tier], in inc_min_seq()
3912 lrugen->protected[hist][type][tier] + delta); in inc_min_seq()
3920 reset_ctrl_pos(lruvec, type, true); in inc_min_seq()
3921 WRITE_ONCE(lrugen->min_seq[type], lrugen->min_seq[type] + 1); in inc_min_seq()
3928 int gen, type, zone; in try_to_inc_min_seq() local
3937 for_each_evictable_type(type, swappiness) { in try_to_inc_min_seq()
3938 while (min_seq[type] + MIN_NR_GENS <= lrugen->max_seq) { in try_to_inc_min_seq()
3939 gen = lru_gen_from_seq(min_seq[type]); in try_to_inc_min_seq()
3942 if (!list_empty(&lrugen->folios[gen][type][zone])) in try_to_inc_min_seq()
3946 min_seq[type]++; in try_to_inc_min_seq()
3971 for_each_evictable_type(type, swappiness) { in try_to_inc_min_seq()
3972 if (min_seq[type] <= lrugen->min_seq[type]) in try_to_inc_min_seq()
3975 reset_ctrl_pos(lruvec, type, true); in try_to_inc_min_seq()
3976 WRITE_ONCE(lrugen->min_seq[type], min_seq[type]); in try_to_inc_min_seq()
3987 int type, zone; in inc_max_seq() local
4001 for (type = 0; type < ANON_AND_FILE; type++) { in inc_max_seq()
4002 if (get_nr_gens(lruvec, type) != MAX_NR_GENS) in inc_max_seq()
4005 if (inc_min_seq(lruvec, type, swappiness)) in inc_max_seq()
4022 for (type = 0; type < ANON_AND_FILE; type++) { in inc_max_seq()
4024 enum lru_list lru = type * LRU_INACTIVE_FILE; in inc_max_seq()
4025 long delta = lrugen->nr_pages[prev][type][zone] - in inc_max_seq()
4026 lrugen->nr_pages[next][type][zone]; in inc_max_seq()
4036 for (type = 0; type < ANON_AND_FILE; type++) in inc_max_seq()
4037 reset_ctrl_pos(lruvec, type, false); in inc_max_seq()
4134 int gen, type, zone; in lruvec_is_sizable() local
4142 for_each_evictable_type(type, swappiness) { in lruvec_is_sizable()
4145 for (seq = min_seq[type]; seq <= max_seq; seq++) { in lruvec_is_sizable()
4149 total += max(READ_ONCE(lrugen->nr_pages[gen][type][zone]), 0L); in lruvec_is_sizable()
4468 int type = folio_is_file_lru(folio); in sort_folio() local
4489 if (gen != lru_gen_from_seq(lrugen->min_seq[type])) { in sort_folio()
4490 list_move(&folio->lru, &lrugen->folios[gen][type][zone]); in sort_folio()
4497 list_move(&folio->lru, &lrugen->folios[gen][type][zone]); in sort_folio()
4501 int hist = lru_hist_from_seq(lrugen->min_seq[type]); in sort_folio()
4503 WRITE_ONCE(lrugen->protected[hist][type][tier], in sort_folio()
4504 lrugen->protected[hist][type][tier] + delta); in sort_folio()
4512 list_move_tail(&folio->lru, &lrugen->folios[gen][type][zone]); in sort_folio()
4518 if (type == LRU_GEN_FILE && dirty) { in sort_folio()
4525 if (writeback || (type == LRU_GEN_FILE && dirty)) { in sort_folio()
4527 list_move(&folio->lru, &lrugen->folios[gen][type][zone]); in sort_folio()
4568 struct scan_control *sc, int type, int tier, in scan_folios() argument
4584 if (get_nr_gens(lruvec, type) == MIN_NR_GENS) in scan_folios()
4587 gen = lru_gen_from_seq(lrugen->min_seq[type]); in scan_folios()
4593 struct list_head *head = &lrugen->folios[gen][type][zone]; in scan_folios()
4601 VM_WARN_ON_ONCE_FOLIO(folio_is_file_lru(folio) != type, folio); in scan_folios()
4637 __count_vm_events(PGSCAN_ANON + type, isolated); in scan_folios()
4640 type ? LRU_INACTIVE_FILE : LRU_INACTIVE_ANON); in scan_folios()
4641 if (type == LRU_GEN_FILE) in scan_folios()
4650 static int get_tier_idx(struct lruvec *lruvec, int type) in get_tier_idx() argument
4660 read_ctrl_pos(lruvec, type, 0, 2, &sp); in get_tier_idx()
4662 read_ctrl_pos(lruvec, type, tier, 3, &pv); in get_tier_idx()
4694 int type = get_type_to_scan(lruvec, swappiness); in isolate_folios() local
4698 int tier = get_tier_idx(lruvec, type); in isolate_folios()
4700 *type_scanned = type; in isolate_folios()
4702 scanned = scan_folios(nr_to_scan, lruvec, sc, type, tier, list); in isolate_folios()
4706 type = !type; in isolate_folios()
4715 int type; in evict_folios() local
4732 scanned = isolate_folios(nr_to_scan, lruvec, sc, swappiness, &type, &list); in evict_folios()
4749 type ? LRU_INACTIVE_FILE : LRU_INACTIVE_ANON); in evict_folios()
4768 if (lru_gen_folio_seq(lruvec, folio, false) == min_seq[type]) in evict_folios()
4789 __count_vm_events(PGSTEAL_ANON + type, reclaimed); in evict_folios()
4806 int gen, type, zone; in should_run_aging() local
4816 for_each_evictable_type(type, swappiness) { in should_run_aging()
4819 for (seq = min_seq[type]; seq <= max_seq; seq++) { in should_run_aging()
4823 size += max(READ_ONCE(lrugen->nr_pages[gen][type][zone]), 0L); in should_run_aging()
5122 int gen, type, zone; in state_is_valid() local
5124 for_each_gen_type_zone(gen, type, zone) { in state_is_valid()
5125 if (!list_empty(&lrugen->folios[gen][type][zone])) in state_is_valid()
5139 int type = is_file_lru(lru); in fill_evictable() local
5149 VM_WARN_ON_ONCE_FOLIO(folio_is_file_lru(folio) != type, folio); in fill_evictable()
5166 int gen, type, zone; in drain_evictable() local
5169 for_each_gen_type_zone(gen, type, zone) { in drain_evictable()
5170 struct list_head *head = &lruvec->lrugen.folios[gen][type][zone]; in drain_evictable()
5178 VM_WARN_ON_ONCE_FOLIO(folio_is_file_lru(folio) != type, folio); in drain_evictable()
5385 int type, tier; in lru_gen_seq_show_full() local
5392 for (type = 0; type < ANON_AND_FILE; type++) { in lru_gen_seq_show_full()
5398 n[0] = READ_ONCE(lrugen->avg_refaulted[type][tier]); in lru_gen_seq_show_full()
5399 n[1] = READ_ONCE(lrugen->avg_total[type][tier]); in lru_gen_seq_show_full()
5400 } else if (seq == min_seq[type] || NR_HIST_GENS > 1) { in lru_gen_seq_show_full()
5402 n[0] = atomic_long_read(&lrugen->refaulted[hist][type][tier]); in lru_gen_seq_show_full()
5403 n[1] = atomic_long_read(&lrugen->evicted[hist][type][tier]); in lru_gen_seq_show_full()
5404 n[2] = READ_ONCE(lrugen->protected[hist][type][tier]); in lru_gen_seq_show_full()
5466 int type, zone; in lru_gen_seq_show() local
5472 for (type = 0; type < ANON_AND_FILE; type++) { in lru_gen_seq_show()
5474 char mark = full && seq < min_seq[type] ? 'x' : ' '; in lru_gen_seq_show()
5477 size += max(READ_ONCE(lrugen->nr_pages[gen][type][zone]), 0L); in lru_gen_seq_show()
5708 int gen, type, zone; in lru_gen_init_lruvec() local
5718 for_each_gen_type_zone(gen, type, zone) in lru_gen_init_lruvec()
5719 INIT_LIST_HEAD(&lrugen->folios[gen][type][zone]); in lru_gen_init_lruvec()