Lines Matching refs:objcg
127 static void obj_cgroup_uncharge_pages(struct obj_cgroup *objcg,
132 struct obj_cgroup *objcg = container_of(ref, struct obj_cgroup, refcnt); in obj_cgroup_release() local
157 nr_bytes = atomic_read(&objcg->nr_charged_bytes); in obj_cgroup_release()
162 obj_cgroup_uncharge_pages(objcg, nr_pages); in obj_cgroup_release()
165 list_del(&objcg->list); in obj_cgroup_release()
169 kfree_rcu(objcg, rcu); in obj_cgroup_release()
174 struct obj_cgroup *objcg; in obj_cgroup_alloc() local
177 objcg = kzalloc(sizeof(struct obj_cgroup), GFP_KERNEL); in obj_cgroup_alloc()
178 if (!objcg) in obj_cgroup_alloc()
181 ret = percpu_ref_init(&objcg->refcnt, obj_cgroup_release, 0, in obj_cgroup_alloc()
184 kfree(objcg); in obj_cgroup_alloc()
187 INIT_LIST_HEAD(&objcg->list); in obj_cgroup_alloc()
188 return objcg; in obj_cgroup_alloc()
194 struct obj_cgroup *objcg, *iter; in memcg_reparent_objcgs() local
196 objcg = rcu_replace_pointer(memcg->objcg, NULL, true); in memcg_reparent_objcgs()
201 list_add(&objcg->list, &memcg->objcg_list); in memcg_reparent_objcgs()
210 percpu_ref_kill(&objcg->refcnt); in memcg_reparent_objcgs()
2395 static inline void __mod_objcg_mlstate(struct obj_cgroup *objcg, in __mod_objcg_mlstate() argument
2403 memcg = obj_cgroup_memcg(objcg); in __mod_objcg_mlstate()
2428 if (obj_exts[off].objcg) in mem_cgroup_from_obj_folio()
2429 return obj_cgroup_memcg(obj_exts[off].objcg); in mem_cgroup_from_obj_folio()
2463 struct obj_cgroup *objcg = NULL; in __get_obj_cgroup_from_memcg() local
2466 objcg = rcu_dereference(memcg->objcg); in __get_obj_cgroup_from_memcg()
2467 if (likely(objcg && obj_cgroup_tryget(objcg))) in __get_obj_cgroup_from_memcg()
2469 objcg = NULL; in __get_obj_cgroup_from_memcg()
2471 return objcg; in __get_obj_cgroup_from_memcg()
2477 struct obj_cgroup *old, *objcg = NULL; in current_objcg_update() local
2481 old = xchg(¤t->objcg, NULL); in current_objcg_update()
2498 if (unlikely(objcg)) { in current_objcg_update()
2499 obj_cgroup_put(objcg); in current_objcg_update()
2500 objcg = NULL; in current_objcg_update()
2512 objcg = __get_obj_cgroup_from_memcg(memcg); in current_objcg_update()
2520 } while (!try_cmpxchg(¤t->objcg, &old, objcg)); in current_objcg_update()
2522 return objcg; in current_objcg_update()
2528 struct obj_cgroup *objcg; in current_obj_cgroup() local
2535 objcg = READ_ONCE(current->objcg); in current_obj_cgroup()
2536 if (unlikely((unsigned long)objcg & CURRENT_OBJCG_UPDATE_FLAG)) in current_obj_cgroup()
2537 objcg = current_objcg_update(); in current_obj_cgroup()
2542 return objcg; in current_obj_cgroup()
2552 objcg = NULL; in current_obj_cgroup()
2560 objcg = rcu_dereference_check(memcg->objcg, 1); in current_obj_cgroup()
2561 if (likely(objcg)) in current_obj_cgroup()
2565 return objcg; in current_obj_cgroup()
2570 struct obj_cgroup *objcg; in get_obj_cgroup_from_folio() local
2576 objcg = __folio_objcg(folio); in get_obj_cgroup_from_folio()
2577 obj_cgroup_get(objcg); in get_obj_cgroup_from_folio()
2584 objcg = __get_obj_cgroup_from_memcg(memcg); in get_obj_cgroup_from_folio()
2586 objcg = NULL; in get_obj_cgroup_from_folio()
2589 return objcg; in get_obj_cgroup_from_folio()
2597 static void obj_cgroup_uncharge_pages(struct obj_cgroup *objcg, in obj_cgroup_uncharge_pages() argument
2602 memcg = get_mem_cgroup_from_objcg(objcg); in obj_cgroup_uncharge_pages()
2619 static int obj_cgroup_charge_pages(struct obj_cgroup *objcg, gfp_t gfp, in obj_cgroup_charge_pages() argument
2625 memcg = get_mem_cgroup_from_objcg(objcg); in obj_cgroup_charge_pages()
2649 struct obj_cgroup *objcg; in __memcg_kmem_charge_page() local
2652 objcg = current_obj_cgroup(); in __memcg_kmem_charge_page()
2653 if (objcg) { in __memcg_kmem_charge_page()
2654 ret = obj_cgroup_charge_pages(objcg, gfp, 1 << order); in __memcg_kmem_charge_page()
2656 obj_cgroup_get(objcg); in __memcg_kmem_charge_page()
2657 page->memcg_data = (unsigned long)objcg | in __memcg_kmem_charge_page()
2673 struct obj_cgroup *objcg; in __memcg_kmem_uncharge_page() local
2679 objcg = __folio_objcg(folio); in __memcg_kmem_uncharge_page()
2680 obj_cgroup_uncharge_pages(objcg, nr_pages); in __memcg_kmem_uncharge_page()
2682 obj_cgroup_put(objcg); in __memcg_kmem_uncharge_page()
2685 static void mod_objcg_state(struct obj_cgroup *objcg, struct pglist_data *pgdat, in mod_objcg_state() argument
2701 if (READ_ONCE(stock->cached_objcg) != objcg) { in mod_objcg_state()
2703 obj_cgroup_get(objcg); in mod_objcg_state()
2704 stock->nr_bytes = atomic_read(&objcg->nr_charged_bytes) in mod_objcg_state()
2705 ? atomic_xchg(&objcg->nr_charged_bytes, 0) : 0; in mod_objcg_state()
2706 WRITE_ONCE(stock->cached_objcg, objcg); in mod_objcg_state()
2713 __mod_objcg_mlstate(objcg, oldpg, NR_SLAB_RECLAIMABLE_B, in mod_objcg_state()
2718 __mod_objcg_mlstate(objcg, oldpg, NR_SLAB_UNRECLAIMABLE_B, in mod_objcg_state()
2744 __mod_objcg_mlstate(objcg, pgdat, idx, nr); in mod_objcg_state()
2750 static bool consume_obj_stock(struct obj_cgroup *objcg, unsigned int nr_bytes) in consume_obj_stock() argument
2759 if (objcg == READ_ONCE(stock->cached_objcg) && stock->nr_bytes >= nr_bytes) { in consume_obj_stock()
2836 struct obj_cgroup *objcg = READ_ONCE(stock->cached_objcg); in obj_stock_flush_required() local
2839 if (objcg) { in obj_stock_flush_required()
2840 memcg = obj_cgroup_memcg(objcg); in obj_stock_flush_required()
2848 static void refill_obj_stock(struct obj_cgroup *objcg, unsigned int nr_bytes, in refill_obj_stock() argument
2859 if (READ_ONCE(stock->cached_objcg) != objcg) { /* reset if necessary */ in refill_obj_stock()
2861 obj_cgroup_get(objcg); in refill_obj_stock()
2862 WRITE_ONCE(stock->cached_objcg, objcg); in refill_obj_stock()
2863 stock->nr_bytes = atomic_read(&objcg->nr_charged_bytes) in refill_obj_stock()
2864 ? atomic_xchg(&objcg->nr_charged_bytes, 0) : 0; in refill_obj_stock()
2878 obj_cgroup_uncharge_pages(objcg, nr_pages); in refill_obj_stock()
2881 int obj_cgroup_charge(struct obj_cgroup *objcg, gfp_t gfp, size_t size) in obj_cgroup_charge() argument
2886 if (consume_obj_stock(objcg, size)) in obj_cgroup_charge()
2918 ret = obj_cgroup_charge_pages(objcg, gfp, nr_pages); in obj_cgroup_charge()
2920 refill_obj_stock(objcg, PAGE_SIZE - nr_bytes, false); in obj_cgroup_charge()
2925 void obj_cgroup_uncharge(struct obj_cgroup *objcg, size_t size) in obj_cgroup_uncharge() argument
2927 refill_obj_stock(objcg, size, true); in obj_cgroup_uncharge()
2942 struct obj_cgroup *objcg; in __memcg_slab_post_alloc_hook() local
2952 objcg = current_obj_cgroup(); in __memcg_slab_post_alloc_hook()
2953 if (!objcg) in __memcg_slab_post_alloc_hook()
2971 memcg = get_mem_cgroup_from_objcg(objcg); in __memcg_slab_post_alloc_hook()
2979 if (obj_cgroup_charge(objcg, flags, size * obj_full_size(s))) in __memcg_slab_post_alloc_hook()
2987 obj_cgroup_uncharge(objcg, obj_full_size(s)); in __memcg_slab_post_alloc_hook()
2992 obj_cgroup_get(objcg); in __memcg_slab_post_alloc_hook()
2993 slab_obj_exts(slab)[off].objcg = objcg; in __memcg_slab_post_alloc_hook()
2994 mod_objcg_state(objcg, slab_pgdat(slab), in __memcg_slab_post_alloc_hook()
3005 struct obj_cgroup *objcg; in __memcg_slab_free_hook() local
3009 objcg = obj_exts[off].objcg; in __memcg_slab_free_hook()
3010 if (!objcg) in __memcg_slab_free_hook()
3013 obj_exts[off].objcg = NULL; in __memcg_slab_free_hook()
3014 obj_cgroup_uncharge(objcg, obj_full_size(s)); in __memcg_slab_free_hook()
3015 mod_objcg_state(objcg, slab_pgdat(slab), cache_vmstat_idx(s), in __memcg_slab_free_hook()
3017 obj_cgroup_put(objcg); in __memcg_slab_free_hook()
3067 struct obj_cgroup *objcg; in memcg_online_kmem() local
3075 objcg = obj_cgroup_alloc(); in memcg_online_kmem()
3076 if (!objcg) in memcg_online_kmem()
3079 objcg->memcg = memcg; in memcg_online_kmem()
3080 rcu_assign_pointer(memcg->objcg, objcg); in memcg_online_kmem()
3081 obj_cgroup_get(objcg); in memcg_online_kmem()
3082 memcg->orig_objcg = objcg; in memcg_online_kmem()
3847 task->objcg = (struct obj_cgroup *)CURRENT_OBJCG_UPDATE_FLAG; in mem_cgroup_fork()
3852 struct obj_cgroup *objcg = task->objcg; in mem_cgroup_exit() local
3854 objcg = (struct obj_cgroup *) in mem_cgroup_exit()
3855 ((unsigned long)objcg & ~CURRENT_OBJCG_UPDATE_FLAG); in mem_cgroup_exit()
3856 obj_cgroup_put(objcg); in mem_cgroup_exit()
3864 task->objcg = NULL; in mem_cgroup_exit()
3896 set_bit(CURRENT_OBJCG_UPDATE_BIT, (unsigned long *)&task->objcg); in mem_cgroup_kmem_attach()
4633 struct obj_cgroup *objcg; in uncharge_folio() local
4643 objcg = __folio_objcg(folio); in uncharge_folio()
4648 memcg = get_mem_cgroup_from_objcg(objcg); in uncharge_folio()
4675 obj_cgroup_put(objcg); in uncharge_folio()
5259 bool obj_cgroup_may_zswap(struct obj_cgroup *objcg) in obj_cgroup_may_zswap() argument
5267 original_memcg = get_mem_cgroup_from_objcg(objcg); in obj_cgroup_may_zswap()
5303 void obj_cgroup_charge_zswap(struct obj_cgroup *objcg, size_t size) in obj_cgroup_charge_zswap() argument
5313 if (obj_cgroup_charge(objcg, GFP_KERNEL, size)) in obj_cgroup_charge_zswap()
5317 memcg = obj_cgroup_memcg(objcg); in obj_cgroup_charge_zswap()
5330 void obj_cgroup_uncharge_zswap(struct obj_cgroup *objcg, size_t size) in obj_cgroup_uncharge_zswap() argument
5337 obj_cgroup_uncharge(objcg, size); in obj_cgroup_uncharge_zswap()
5340 memcg = obj_cgroup_memcg(objcg); in obj_cgroup_uncharge_zswap()