Lines Matching refs:h_cg
33 __hugetlb_cgroup_counter_from_cgroup(struct hugetlb_cgroup *h_cg, int idx, in __hugetlb_cgroup_counter_from_cgroup() argument
37 return &h_cg->rsvd_hugepage[idx]; in __hugetlb_cgroup_counter_from_cgroup()
38 return &h_cg->hugepage[idx]; in __hugetlb_cgroup_counter_from_cgroup()
42 hugetlb_cgroup_counter_from_cgroup(struct hugetlb_cgroup *h_cg, int idx) in hugetlb_cgroup_counter_from_cgroup() argument
44 return __hugetlb_cgroup_counter_from_cgroup(h_cg, idx, false); in hugetlb_cgroup_counter_from_cgroup()
48 hugetlb_cgroup_counter_from_cgroup_rsvd(struct hugetlb_cgroup *h_cg, int idx) in hugetlb_cgroup_counter_from_cgroup_rsvd() argument
50 return __hugetlb_cgroup_counter_from_cgroup(h_cg, idx, true); in hugetlb_cgroup_counter_from_cgroup_rsvd()
65 static inline bool hugetlb_cgroup_is_root(struct hugetlb_cgroup *h_cg) in hugetlb_cgroup_is_root() argument
67 return (h_cg == root_h_cgroup); in hugetlb_cgroup_is_root()
71 parent_hugetlb_cgroup(struct hugetlb_cgroup *h_cg) in parent_hugetlb_cgroup() argument
73 return hugetlb_cgroup_from_css(h_cg->css.parent); in parent_hugetlb_cgroup()
76 static inline bool hugetlb_cgroup_have_usage(struct hugetlb_cgroup *h_cg) in hugetlb_cgroup_have_usage() argument
82 hugetlb_cgroup_counter_from_cgroup(h_cg, hstate_index(h)))) in hugetlb_cgroup_have_usage()
187 static void hugetlb_cgroup_move_parent(int idx, struct hugetlb_cgroup *h_cg, in hugetlb_cgroup_move_parent() argument
193 struct hugetlb_cgroup *parent = parent_hugetlb_cgroup(h_cg); in hugetlb_cgroup_move_parent()
202 if (!page_hcg || page_hcg != h_cg) in hugetlb_cgroup_move_parent()
211 counter = &h_cg->hugepage[idx]; in hugetlb_cgroup_move_parent()
226 struct hugetlb_cgroup *h_cg = hugetlb_cgroup_from_css(css); in hugetlb_cgroup_css_offline() local
234 hugetlb_cgroup_move_parent(hstate_index(h), h_cg, page); in hugetlb_cgroup_css_offline()
239 } while (hugetlb_cgroup_have_usage(h_cg)); in hugetlb_cgroup_css_offline()
261 struct hugetlb_cgroup *h_cg = NULL; in __hugetlb_cgroup_charge_cgroup() local
273 h_cg = hugetlb_cgroup_from_task(current); in __hugetlb_cgroup_charge_cgroup()
274 if (!css_tryget(&h_cg->css)) { in __hugetlb_cgroup_charge_cgroup()
281 __hugetlb_cgroup_counter_from_cgroup(h_cg, idx, rsvd), in __hugetlb_cgroup_charge_cgroup()
284 hugetlb_event(h_cg, idx, HUGETLB_MAX); in __hugetlb_cgroup_charge_cgroup()
285 css_put(&h_cg->css); in __hugetlb_cgroup_charge_cgroup()
292 css_put(&h_cg->css); in __hugetlb_cgroup_charge_cgroup()
294 *ptr = h_cg; in __hugetlb_cgroup_charge_cgroup()
312 struct hugetlb_cgroup *h_cg, in __hugetlb_cgroup_commit_charge() argument
315 if (hugetlb_cgroup_disabled() || !h_cg) in __hugetlb_cgroup_commit_charge()
318 __set_hugetlb_cgroup(folio, h_cg, rsvd); in __hugetlb_cgroup_commit_charge()
321 h_cg->nodeinfo[folio_nid(folio)]->usage[idx]; in __hugetlb_cgroup_commit_charge()
327 WRITE_ONCE(h_cg->nodeinfo[folio_nid(folio)]->usage[idx], in __hugetlb_cgroup_commit_charge()
333 struct hugetlb_cgroup *h_cg, in hugetlb_cgroup_commit_charge() argument
336 __hugetlb_cgroup_commit_charge(idx, nr_pages, h_cg, folio, false); in hugetlb_cgroup_commit_charge()
340 struct hugetlb_cgroup *h_cg, in hugetlb_cgroup_commit_charge_rsvd() argument
343 __hugetlb_cgroup_commit_charge(idx, nr_pages, h_cg, folio, true); in hugetlb_cgroup_commit_charge_rsvd()
352 struct hugetlb_cgroup *h_cg; in __hugetlb_cgroup_uncharge_folio() local
357 h_cg = __hugetlb_cgroup_from_folio(folio, rsvd); in __hugetlb_cgroup_uncharge_folio()
358 if (unlikely(!h_cg)) in __hugetlb_cgroup_uncharge_folio()
362 page_counter_uncharge(__hugetlb_cgroup_counter_from_cgroup(h_cg, idx, in __hugetlb_cgroup_uncharge_folio()
367 css_put(&h_cg->css); in __hugetlb_cgroup_uncharge_folio()
370 h_cg->nodeinfo[folio_nid(folio)]->usage[idx]; in __hugetlb_cgroup_uncharge_folio()
376 WRITE_ONCE(h_cg->nodeinfo[folio_nid(folio)]->usage[idx], in __hugetlb_cgroup_uncharge_folio()
394 struct hugetlb_cgroup *h_cg, in __hugetlb_cgroup_uncharge_cgroup() argument
397 if (hugetlb_cgroup_disabled() || !h_cg) in __hugetlb_cgroup_uncharge_cgroup()
403 page_counter_uncharge(__hugetlb_cgroup_counter_from_cgroup(h_cg, idx, in __hugetlb_cgroup_uncharge_cgroup()
408 css_put(&h_cg->css); in __hugetlb_cgroup_uncharge_cgroup()
412 struct hugetlb_cgroup *h_cg) in hugetlb_cgroup_uncharge_cgroup() argument
414 __hugetlb_cgroup_uncharge_cgroup(idx, nr_pages, h_cg, false); in hugetlb_cgroup_uncharge_cgroup()
418 struct hugetlb_cgroup *h_cg) in hugetlb_cgroup_uncharge_cgroup_rsvd() argument
420 __hugetlb_cgroup_uncharge_cgroup(idx, nr_pages, h_cg, true); in hugetlb_cgroup_uncharge_cgroup_rsvd()
473 struct hugetlb_cgroup *h_cg = hugetlb_cgroup_from_css(seq_css(seq)); in hugetlb_cgroup_read_numa_stat() local
481 usage += READ_ONCE(h_cg->nodeinfo[nid]->usage[idx]); in hugetlb_cgroup_read_numa_stat()
487 READ_ONCE(h_cg->nodeinfo[nid]->usage[idx]) * in hugetlb_cgroup_read_numa_stat()
497 page_counter_read(&h_cg->hugepage[idx]) * PAGE_SIZE); in hugetlb_cgroup_read_numa_stat()
506 css_for_each_descendant_pre(css, &h_cg->css) { in hugetlb_cgroup_read_numa_stat()
525 struct hugetlb_cgroup *h_cg = hugetlb_cgroup_from_css(css); in hugetlb_cgroup_read_u64() local
527 counter = &h_cg->hugepage[MEMFILE_IDX(cft->private)]; in hugetlb_cgroup_read_u64()
528 rsvd_counter = &h_cg->rsvd_hugepage[MEMFILE_IDX(cft->private)]; in hugetlb_cgroup_read_u64()
559 struct hugetlb_cgroup *h_cg = hugetlb_cgroup_from_css(seq_css(seq)); in hugetlb_cgroup_read_u64_max() local
562 counter = &h_cg->hugepage[idx]; in hugetlb_cgroup_read_u64_max()
569 counter = &h_cg->rsvd_hugepage[idx]; in hugetlb_cgroup_read_u64_max()
576 counter = &h_cg->rsvd_hugepage[idx]; in hugetlb_cgroup_read_u64_max()
600 struct hugetlb_cgroup *h_cg = hugetlb_cgroup_from_css(of_css(of)); in hugetlb_cgroup_write() local
603 if (hugetlb_cgroup_is_root(h_cg)) /* Can't set limit on root */ in hugetlb_cgroup_write()
621 __hugetlb_cgroup_counter_from_cgroup(h_cg, idx, rsvd), in hugetlb_cgroup_write()
649 struct hugetlb_cgroup *h_cg = hugetlb_cgroup_from_css(of_css(of)); in hugetlb_cgroup_reset() local
651 counter = &h_cg->hugepage[MEMFILE_IDX(of_cft(of)->private)]; in hugetlb_cgroup_reset()
652 rsvd_counter = &h_cg->rsvd_hugepage[MEMFILE_IDX(of_cft(of)->private)]; in hugetlb_cgroup_reset()
690 struct hugetlb_cgroup *h_cg = hugetlb_cgroup_from_css(seq_css(seq)); in __hugetlb_events_show() local
695 max = atomic_long_read(&h_cg->events_local[idx][HUGETLB_MAX]); in __hugetlb_events_show()
697 max = atomic_long_read(&h_cg->events[idx][HUGETLB_MAX]); in __hugetlb_events_show()
889 struct hugetlb_cgroup *h_cg; in hugetlb_cgroup_migrate() local
897 h_cg = hugetlb_cgroup_from_folio(old_folio); in hugetlb_cgroup_migrate()
903 set_hugetlb_cgroup(new_folio, h_cg); in hugetlb_cgroup_migrate()