Lines Matching refs:folio
27 static inline int folio_is_file_lru(struct folio *folio) in folio_is_file_lru() argument
29 return !folio_test_swapbacked(folio); in folio_is_file_lru()
65 static __always_inline void __folio_clear_lru_flags(struct folio *folio) in __folio_clear_lru_flags() argument
67 VM_BUG_ON_FOLIO(!folio_test_lru(folio), folio); in __folio_clear_lru_flags()
69 __folio_clear_lru(folio); in __folio_clear_lru_flags()
72 if (folio_test_active(folio) && folio_test_unevictable(folio)) in __folio_clear_lru_flags()
75 __folio_clear_active(folio); in __folio_clear_lru_flags()
76 __folio_clear_unevictable(folio); in __folio_clear_lru_flags()
86 static __always_inline enum lru_list folio_lru_list(struct folio *folio) in folio_lru_list() argument
90 VM_BUG_ON_FOLIO(folio_test_active(folio) && folio_test_unevictable(folio), folio); in folio_lru_list()
92 if (folio_test_unevictable(folio)) in folio_lru_list()
95 lru = folio_is_file_lru(folio) ? LRU_INACTIVE_FILE : LRU_INACTIVE_ANON; in folio_lru_list()
96 if (folio_test_active(folio)) in folio_lru_list()
143 static inline int folio_lru_refs(struct folio *folio) in folio_lru_refs() argument
145 unsigned long flags = READ_ONCE(folio->flags); in folio_lru_refs()
157 static inline int folio_lru_gen(struct folio *folio) in folio_lru_gen() argument
159 unsigned long flags = READ_ONCE(folio->flags); in folio_lru_gen()
174 static inline void lru_gen_update_size(struct lruvec *lruvec, struct folio *folio, in lru_gen_update_size() argument
177 int type = folio_is_file_lru(folio); in lru_gen_update_size()
178 int zone = folio_zonenum(folio); in lru_gen_update_size()
179 int delta = folio_nr_pages(folio); in lru_gen_update_size()
220 static inline bool lru_gen_add_folio(struct lruvec *lruvec, struct folio *folio, bool reclaiming) in lru_gen_add_folio() argument
224 int gen = folio_lru_gen(folio); in lru_gen_add_folio()
225 int type = folio_is_file_lru(folio); in lru_gen_add_folio()
226 int zone = folio_zonenum(folio); in lru_gen_add_folio()
229 VM_WARN_ON_ONCE_FOLIO(gen != -1, folio); in lru_gen_add_folio()
231 if (folio_test_unevictable(folio) || !lrugen->enabled) in lru_gen_add_folio()
242 if (folio_test_active(folio)) in lru_gen_add_folio()
244 else if ((type == LRU_GEN_ANON && !folio_test_swapcache(folio)) || in lru_gen_add_folio()
245 (folio_test_reclaim(folio) && in lru_gen_add_folio()
246 (folio_test_dirty(folio) || folio_test_writeback(folio)))) in lru_gen_add_folio()
254 set_mask_bits(&folio->flags, LRU_GEN_MASK | BIT(PG_active), flags); in lru_gen_add_folio()
256 lru_gen_update_size(lruvec, folio, -1, gen); in lru_gen_add_folio()
259 list_add_tail(&folio->lru, &lrugen->folios[gen][type][zone]); in lru_gen_add_folio()
261 list_add(&folio->lru, &lrugen->folios[gen][type][zone]); in lru_gen_add_folio()
266 static inline bool lru_gen_del_folio(struct lruvec *lruvec, struct folio *folio, bool reclaiming) in lru_gen_del_folio() argument
269 int gen = folio_lru_gen(folio); in lru_gen_del_folio()
274 VM_WARN_ON_ONCE_FOLIO(folio_test_active(folio), folio); in lru_gen_del_folio()
275 VM_WARN_ON_ONCE_FOLIO(folio_test_unevictable(folio), folio); in lru_gen_del_folio()
279 flags = set_mask_bits(&folio->flags, LRU_GEN_MASK, flags); in lru_gen_del_folio()
282 lru_gen_update_size(lruvec, folio, gen, -1); in lru_gen_del_folio()
283 list_del(&folio->lru); in lru_gen_del_folio()
300 static inline bool lru_gen_add_folio(struct lruvec *lruvec, struct folio *folio, bool reclaiming) in lru_gen_add_folio() argument
305 static inline bool lru_gen_del_folio(struct lruvec *lruvec, struct folio *folio, bool reclaiming) in lru_gen_del_folio() argument
313 void lruvec_add_folio(struct lruvec *lruvec, struct folio *folio) in lruvec_add_folio() argument
315 enum lru_list lru = folio_lru_list(folio); in lruvec_add_folio()
317 if (lru_gen_add_folio(lruvec, folio, false)) in lruvec_add_folio()
320 update_lru_size(lruvec, lru, folio_zonenum(folio), in lruvec_add_folio()
321 folio_nr_pages(folio)); in lruvec_add_folio()
323 list_add(&folio->lru, &lruvec->lists[lru]); in lruvec_add_folio()
333 void lruvec_add_folio_tail(struct lruvec *lruvec, struct folio *folio) in lruvec_add_folio_tail() argument
335 enum lru_list lru = folio_lru_list(folio); in lruvec_add_folio_tail()
337 if (lru_gen_add_folio(lruvec, folio, true)) in lruvec_add_folio_tail()
340 update_lru_size(lruvec, lru, folio_zonenum(folio), in lruvec_add_folio_tail()
341 folio_nr_pages(folio)); in lruvec_add_folio_tail()
343 list_add_tail(&folio->lru, &lruvec->lists[lru]); in lruvec_add_folio_tail()
347 void lruvec_del_folio(struct lruvec *lruvec, struct folio *folio) in lruvec_del_folio() argument
349 enum lru_list lru = folio_lru_list(folio); in lruvec_del_folio()
351 if (lru_gen_del_folio(lruvec, folio, false)) in lruvec_del_folio()
355 list_del(&folio->lru); in lruvec_del_folio()
356 update_lru_size(lruvec, lru, folio_zonenum(folio), in lruvec_del_folio()
357 -folio_nr_pages(folio)); in lruvec_del_folio()