Lines Matching refs:stock
2206 static struct obj_cgroup *drain_obj_stock(struct memcg_stock_pcp *stock);
2207 static bool obj_stock_flush_required(struct memcg_stock_pcp *stock,
2212 static inline struct obj_cgroup *drain_obj_stock(struct memcg_stock_pcp *stock) in drain_obj_stock() argument
2216 static bool obj_stock_flush_required(struct memcg_stock_pcp *stock, in obj_stock_flush_required() argument
2239 struct memcg_stock_pcp *stock; in consume_stock() local
2248 stock = this_cpu_ptr(&memcg_stock); in consume_stock()
2249 if (memcg == stock->cached && stock->nr_pages >= nr_pages) { in consume_stock()
2250 stock->nr_pages -= nr_pages; in consume_stock()
2262 static void drain_stock(struct memcg_stock_pcp *stock) in drain_stock() argument
2264 struct mem_cgroup *old = stock->cached; in drain_stock()
2269 if (stock->nr_pages) { in drain_stock()
2270 page_counter_uncharge(&old->memory, stock->nr_pages); in drain_stock()
2272 page_counter_uncharge(&old->memsw, stock->nr_pages); in drain_stock()
2273 stock->nr_pages = 0; in drain_stock()
2277 stock->cached = NULL; in drain_stock()
2282 struct memcg_stock_pcp *stock; in drain_local_stock() local
2293 stock = this_cpu_ptr(&memcg_stock); in drain_local_stock()
2294 old = drain_obj_stock(stock); in drain_local_stock()
2295 drain_stock(stock); in drain_local_stock()
2296 clear_bit(FLUSHING_CACHED_CHARGE, &stock->flags); in drain_local_stock()
2309 struct memcg_stock_pcp *stock; in __refill_stock() local
2311 stock = this_cpu_ptr(&memcg_stock); in __refill_stock()
2312 if (stock->cached != memcg) { /* reset if necessary */ in __refill_stock()
2313 drain_stock(stock); in __refill_stock()
2315 stock->cached = memcg; in __refill_stock()
2317 stock->nr_pages += nr_pages; in __refill_stock()
2319 if (stock->nr_pages > MEMCG_CHARGE_BATCH) in __refill_stock()
2320 drain_stock(stock); in __refill_stock()
2352 struct memcg_stock_pcp *stock = &per_cpu(memcg_stock, cpu); in drain_all_stock() local
2357 memcg = stock->cached; in drain_all_stock()
2358 if (memcg && stock->nr_pages && in drain_all_stock()
2361 else if (obj_stock_flush_required(stock, root_memcg)) in drain_all_stock()
2366 !test_and_set_bit(FLUSHING_CACHED_CHARGE, &stock->flags)) { in drain_all_stock()
2368 drain_local_stock(&stock->work); in drain_all_stock()
2370 schedule_work_on(cpu, &stock->work); in drain_all_stock()
2379 struct memcg_stock_pcp *stock; in memcg_hotplug_cpu_dead() local
2381 stock = &per_cpu(memcg_stock, cpu); in memcg_hotplug_cpu_dead()
2382 drain_stock(stock); in memcg_hotplug_cpu_dead()
3169 struct memcg_stock_pcp *stock; in mod_objcg_state() local
3175 stock = this_cpu_ptr(&memcg_stock); in mod_objcg_state()
3182 if (stock->cached_objcg != objcg) { in mod_objcg_state()
3183 old = drain_obj_stock(stock); in mod_objcg_state()
3185 stock->nr_bytes = atomic_read(&objcg->nr_charged_bytes) in mod_objcg_state()
3187 stock->cached_objcg = objcg; in mod_objcg_state()
3188 stock->cached_pgdat = pgdat; in mod_objcg_state()
3189 } else if (stock->cached_pgdat != pgdat) { in mod_objcg_state()
3191 struct pglist_data *oldpg = stock->cached_pgdat; in mod_objcg_state()
3193 if (stock->nr_slab_reclaimable_b) { in mod_objcg_state()
3195 stock->nr_slab_reclaimable_b); in mod_objcg_state()
3196 stock->nr_slab_reclaimable_b = 0; in mod_objcg_state()
3198 if (stock->nr_slab_unreclaimable_b) { in mod_objcg_state()
3200 stock->nr_slab_unreclaimable_b); in mod_objcg_state()
3201 stock->nr_slab_unreclaimable_b = 0; in mod_objcg_state()
3203 stock->cached_pgdat = pgdat; in mod_objcg_state()
3206 bytes = (idx == NR_SLAB_RECLAIMABLE_B) ? &stock->nr_slab_reclaimable_b in mod_objcg_state()
3207 : &stock->nr_slab_unreclaimable_b; in mod_objcg_state()
3234 struct memcg_stock_pcp *stock; in consume_obj_stock() local
3240 stock = this_cpu_ptr(&memcg_stock); in consume_obj_stock()
3241 if (objcg == stock->cached_objcg && stock->nr_bytes >= nr_bytes) { in consume_obj_stock()
3242 stock->nr_bytes -= nr_bytes; in consume_obj_stock()
3251 static struct obj_cgroup *drain_obj_stock(struct memcg_stock_pcp *stock) in drain_obj_stock() argument
3253 struct obj_cgroup *old = stock->cached_objcg; in drain_obj_stock()
3258 if (stock->nr_bytes) { in drain_obj_stock()
3259 unsigned int nr_pages = stock->nr_bytes >> PAGE_SHIFT; in drain_obj_stock()
3260 unsigned int nr_bytes = stock->nr_bytes & (PAGE_SIZE - 1); in drain_obj_stock()
3284 stock->nr_bytes = 0; in drain_obj_stock()
3290 if (stock->nr_slab_reclaimable_b || stock->nr_slab_unreclaimable_b) { in drain_obj_stock()
3291 if (stock->nr_slab_reclaimable_b) { in drain_obj_stock()
3292 mod_objcg_mlstate(old, stock->cached_pgdat, in drain_obj_stock()
3294 stock->nr_slab_reclaimable_b); in drain_obj_stock()
3295 stock->nr_slab_reclaimable_b = 0; in drain_obj_stock()
3297 if (stock->nr_slab_unreclaimable_b) { in drain_obj_stock()
3298 mod_objcg_mlstate(old, stock->cached_pgdat, in drain_obj_stock()
3300 stock->nr_slab_unreclaimable_b); in drain_obj_stock()
3301 stock->nr_slab_unreclaimable_b = 0; in drain_obj_stock()
3303 stock->cached_pgdat = NULL; in drain_obj_stock()
3306 stock->cached_objcg = NULL; in drain_obj_stock()
3314 static bool obj_stock_flush_required(struct memcg_stock_pcp *stock, in obj_stock_flush_required() argument
3319 if (stock->cached_objcg) { in obj_stock_flush_required()
3320 memcg = obj_cgroup_memcg(stock->cached_objcg); in obj_stock_flush_required()
3331 struct memcg_stock_pcp *stock; in refill_obj_stock() local
3338 stock = this_cpu_ptr(&memcg_stock); in refill_obj_stock()
3339 if (stock->cached_objcg != objcg) { /* reset if necessary */ in refill_obj_stock()
3340 old = drain_obj_stock(stock); in refill_obj_stock()
3342 stock->cached_objcg = objcg; in refill_obj_stock()
3343 stock->nr_bytes = atomic_read(&objcg->nr_charged_bytes) in refill_obj_stock()
3347 stock->nr_bytes += nr_bytes; in refill_obj_stock()
3349 if (allow_uncharge && (stock->nr_bytes > PAGE_SIZE)) { in refill_obj_stock()
3350 nr_pages = stock->nr_bytes >> PAGE_SHIFT; in refill_obj_stock()
3351 stock->nr_bytes &= (PAGE_SIZE - 1); in refill_obj_stock()