Lines Matching refs:folio
351 static inline bool folio_memcg_kmem(struct folio *folio);
375 static inline struct mem_cgroup *__folio_memcg(struct folio *folio) in __folio_memcg() argument
377 unsigned long memcg_data = folio->memcg_data; in __folio_memcg()
379 VM_BUG_ON_FOLIO(folio_test_slab(folio), folio); in __folio_memcg()
380 VM_BUG_ON_FOLIO(memcg_data & MEMCG_DATA_OBJCGS, folio); in __folio_memcg()
381 VM_BUG_ON_FOLIO(memcg_data & MEMCG_DATA_KMEM, folio); in __folio_memcg()
396 static inline struct obj_cgroup *__folio_objcg(struct folio *folio) in __folio_objcg() argument
398 unsigned long memcg_data = folio->memcg_data; in __folio_objcg()
400 VM_BUG_ON_FOLIO(folio_test_slab(folio), folio); in __folio_objcg()
401 VM_BUG_ON_FOLIO(memcg_data & MEMCG_DATA_OBJCGS, folio); in __folio_objcg()
402 VM_BUG_ON_FOLIO(!(memcg_data & MEMCG_DATA_KMEM), folio); in __folio_objcg()
428 static inline struct mem_cgroup *folio_memcg(struct folio *folio) in folio_memcg() argument
430 if (folio_memcg_kmem(folio)) in folio_memcg()
431 return obj_cgroup_memcg(__folio_objcg(folio)); in folio_memcg()
432 return __folio_memcg(folio); in folio_memcg()
451 static inline struct mem_cgroup *folio_memcg_rcu(struct folio *folio) in folio_memcg_rcu() argument
453 unsigned long memcg_data = READ_ONCE(folio->memcg_data); in folio_memcg_rcu()
455 VM_BUG_ON_FOLIO(folio_test_slab(folio), folio); in folio_memcg_rcu()
490 static inline struct mem_cgroup *folio_memcg_check(struct folio *folio) in folio_memcg_check() argument
496 unsigned long memcg_data = READ_ONCE(folio->memcg_data); in folio_memcg_check()
515 return folio_memcg_check((struct folio *)page); in page_memcg_check()
541 static inline bool folio_memcg_kmem(struct folio *folio) in folio_memcg_kmem() argument
543 VM_BUG_ON_PGFLAGS(PageTail(&folio->page), &folio->page); in folio_memcg_kmem()
544 VM_BUG_ON_FOLIO(folio->memcg_data & MEMCG_DATA_OBJCGS, folio); in folio_memcg_kmem()
545 return folio->memcg_data & MEMCG_DATA_KMEM; in folio_memcg_kmem()
550 static inline bool folio_memcg_kmem(struct folio *folio) in folio_memcg_kmem() argument
657 int __mem_cgroup_charge(struct folio *folio, struct mm_struct *mm, gfp_t gfp);
673 static inline int mem_cgroup_charge(struct folio *folio, struct mm_struct *mm, in mem_cgroup_charge() argument
678 return __mem_cgroup_charge(folio, mm, gfp); in mem_cgroup_charge()
681 int mem_cgroup_swapin_charge_folio(struct folio *folio, struct mm_struct *mm,
685 void __mem_cgroup_uncharge(struct folio *folio);
693 static inline void mem_cgroup_uncharge(struct folio *folio) in mem_cgroup_uncharge() argument
697 __mem_cgroup_uncharge(folio); in mem_cgroup_uncharge()
708 void mem_cgroup_migrate(struct folio *old, struct folio *new);
752 static inline struct lruvec *folio_lruvec(struct folio *folio) in folio_lruvec() argument
754 struct mem_cgroup *memcg = folio_memcg(folio); in folio_lruvec()
756 VM_WARN_ON_ONCE_FOLIO(!memcg && !mem_cgroup_disabled(), folio); in folio_lruvec()
757 return mem_cgroup_lruvec(memcg, folio_pgdat(folio)); in folio_lruvec()
764 struct lruvec *folio_lruvec_lock(struct folio *folio);
765 struct lruvec *folio_lruvec_lock_irq(struct folio *folio);
766 struct lruvec *folio_lruvec_lock_irqsave(struct folio *folio,
770 void lruvec_memcg_debug(struct lruvec *lruvec, struct folio *folio);
773 void lruvec_memcg_debug(struct lruvec *lruvec, struct folio *folio) in lruvec_memcg_debug() argument
893 struct cgroup_subsys_state *mem_cgroup_css_from_folio(struct folio *folio);
949 void folio_memcg_lock(struct folio *folio);
950 void folio_memcg_unlock(struct folio *folio);
1089 static inline void count_memcg_folio_events(struct folio *folio, in count_memcg_folio_events() argument
1092 struct mem_cgroup *memcg = folio_memcg(folio); in count_memcg_folio_events()
1164 static inline struct mem_cgroup *folio_memcg(struct folio *folio) in folio_memcg() argument
1174 static inline struct mem_cgroup *folio_memcg_rcu(struct folio *folio) in folio_memcg_rcu() argument
1180 static inline struct mem_cgroup *folio_memcg_check(struct folio *folio) in folio_memcg_check() argument
1190 static inline bool folio_memcg_kmem(struct folio *folio) in folio_memcg_kmem() argument
1250 static inline int mem_cgroup_charge(struct folio *folio, in mem_cgroup_charge() argument
1256 static inline int mem_cgroup_swapin_charge_folio(struct folio *folio, in mem_cgroup_swapin_charge_folio() argument
1266 static inline void mem_cgroup_uncharge(struct folio *folio) in mem_cgroup_uncharge() argument
1274 static inline void mem_cgroup_migrate(struct folio *old, struct folio *new) in mem_cgroup_migrate()
1284 static inline struct lruvec *folio_lruvec(struct folio *folio) in folio_lruvec() argument
1286 struct pglist_data *pgdat = folio_pgdat(folio); in folio_lruvec()
1291 void lruvec_memcg_debug(struct lruvec *lruvec, struct folio *folio) in lruvec_memcg_debug() argument
1330 static inline struct lruvec *folio_lruvec_lock(struct folio *folio) in folio_lruvec_lock() argument
1332 struct pglist_data *pgdat = folio_pgdat(folio); in folio_lruvec_lock()
1338 static inline struct lruvec *folio_lruvec_lock_irq(struct folio *folio) in folio_lruvec_lock_irq() argument
1340 struct pglist_data *pgdat = folio_pgdat(folio); in folio_lruvec_lock_irq()
1346 static inline struct lruvec *folio_lruvec_lock_irqsave(struct folio *folio, in folio_lruvec_lock_irqsave() argument
1349 struct pglist_data *pgdat = folio_pgdat(folio); in folio_lruvec_lock_irqsave()
1448 static inline void folio_memcg_lock(struct folio *folio) in folio_memcg_lock() argument
1452 static inline void folio_memcg_unlock(struct folio *folio) in folio_memcg_unlock() argument
1580 static inline void count_memcg_folio_events(struct folio *folio, in count_memcg_folio_events() argument
1643 static inline bool folio_matches_lruvec(struct folio *folio, in folio_matches_lruvec() argument
1646 return lruvec_pgdat(lruvec) == folio_pgdat(folio) && in folio_matches_lruvec()
1647 lruvec_memcg(lruvec) == folio_memcg(folio); in folio_matches_lruvec()
1651 static inline struct lruvec *folio_lruvec_relock_irq(struct folio *folio, in folio_lruvec_relock_irq() argument
1655 if (folio_matches_lruvec(folio, locked_lruvec)) in folio_lruvec_relock_irq()
1661 return folio_lruvec_lock_irq(folio); in folio_lruvec_relock_irq()
1665 static inline struct lruvec *folio_lruvec_relock_irqsave(struct folio *folio, in folio_lruvec_relock_irqsave() argument
1669 if (folio_matches_lruvec(folio, locked_lruvec)) in folio_lruvec_relock_irqsave()
1675 return folio_lruvec_lock_irqsave(folio, flags); in folio_lruvec_relock_irqsave()
1685 void mem_cgroup_track_foreign_dirty_slowpath(struct folio *folio,
1688 static inline void mem_cgroup_track_foreign_dirty(struct folio *folio, in mem_cgroup_track_foreign_dirty() argument
1696 memcg = folio_memcg(folio); in mem_cgroup_track_foreign_dirty()
1698 mem_cgroup_track_foreign_dirty_slowpath(folio, wb); in mem_cgroup_track_foreign_dirty()
1718 static inline void mem_cgroup_track_foreign_dirty(struct folio *folio, in mem_cgroup_track_foreign_dirty() argument