Lines Matching refs:slab
9 struct slab { struct
34 struct slab *next; argument
73 static_assert(offsetof(struct page, pg) == offsetof(struct slab, sl)) argument
85 static_assert(sizeof(struct slab) <= sizeof(struct page));
87 static_assert(IS_ALIGNED(offsetof(struct slab, freelist), 2*sizeof(void *)));
100 const struct folio *: (const struct slab *)(folio), \
101 struct folio *: (struct slab *)(folio)))
115 const struct slab *: (const struct folio *)s, \
116 struct slab *: (struct folio *)s))
131 const struct page *: (const struct slab *)(p), \
132 struct page *: (struct slab *)(p)))
148 static inline bool slab_test_pfmemalloc(const struct slab *slab) in slab_test_pfmemalloc() argument
150 return folio_test_active((struct folio *)slab_folio(slab)); in slab_test_pfmemalloc()
153 static inline void slab_set_pfmemalloc(struct slab *slab) in slab_set_pfmemalloc() argument
155 folio_set_active(slab_folio(slab)); in slab_set_pfmemalloc()
158 static inline void slab_clear_pfmemalloc(struct slab *slab) in slab_clear_pfmemalloc() argument
160 folio_clear_active(slab_folio(slab)); in slab_clear_pfmemalloc()
163 static inline void __slab_clear_pfmemalloc(struct slab *slab) in __slab_clear_pfmemalloc() argument
165 __folio_clear_active(slab_folio(slab)); in __slab_clear_pfmemalloc()
168 static inline void *slab_address(const struct slab *slab) in slab_address() argument
170 return folio_address(slab_folio(slab)); in slab_address()
173 static inline int slab_nid(const struct slab *slab) in slab_nid() argument
175 return folio_nid(slab_folio(slab)); in slab_nid()
178 static inline pg_data_t *slab_pgdat(const struct slab *slab) in slab_pgdat() argument
180 return folio_pgdat(slab_folio(slab)); in slab_pgdat()
183 static inline struct slab *virt_to_slab(const void *addr) in virt_to_slab()
193 static inline int slab_order(const struct slab *slab) in slab_order() argument
195 return folio_order((struct folio *)slab_folio(slab)); in slab_order()
198 static inline size_t slab_size(const struct slab *slab) in slab_size() argument
200 return PAGE_SIZE << slab_order(slab); in slab_size()
456 static inline struct obj_cgroup **slab_objcgs(struct slab *slab) in slab_objcgs() argument
458 unsigned long memcg_data = READ_ONCE(slab->memcg_data); in slab_objcgs()
461 slab_page(slab)); in slab_objcgs()
462 VM_BUG_ON_PAGE(memcg_data & MEMCG_DATA_KMEM, slab_page(slab)); in slab_objcgs()
467 int memcg_alloc_slab_cgroups(struct slab *slab, struct kmem_cache *s,
472 static inline void memcg_free_slab_cgroups(struct slab *slab) in memcg_free_slab_cgroups() argument
474 kfree(slab_objcgs(slab)); in memcg_free_slab_cgroups()
475 slab->memcg_data = 0; in memcg_free_slab_cgroups()
534 struct slab *slab; in memcg_slab_post_alloc_hook() local
543 slab = virt_to_slab(p[i]); in memcg_slab_post_alloc_hook()
545 if (!slab_objcgs(slab) && in memcg_slab_post_alloc_hook()
546 memcg_alloc_slab_cgroups(slab, s, flags, in memcg_slab_post_alloc_hook()
552 off = obj_to_index(s, slab, p[i]); in memcg_slab_post_alloc_hook()
554 slab_objcgs(slab)[off] = objcg; in memcg_slab_post_alloc_hook() local
555 mod_objcg_state(objcg, slab_pgdat(slab), in memcg_slab_post_alloc_hook()
564 static inline void memcg_slab_free_hook(struct kmem_cache *s, struct slab *slab, in memcg_slab_free_hook() argument
573 objcgs = slab_objcgs(slab); in memcg_slab_free_hook()
581 off = obj_to_index(s, slab, p[i]); in memcg_slab_free_hook()
588 mod_objcg_state(objcg, slab_pgdat(slab), cache_vmstat_idx(s), in memcg_slab_free_hook()
595 static inline struct obj_cgroup **slab_objcgs(struct slab *slab) in slab_objcgs() argument
605 static inline int memcg_alloc_slab_cgroups(struct slab *slab, in memcg_alloc_slab_cgroups() argument
612 static inline void memcg_free_slab_cgroups(struct slab *slab) in memcg_free_slab_cgroups() argument
631 static inline void memcg_slab_free_hook(struct kmem_cache *s, struct slab *slab, in memcg_slab_free_hook() argument
640 struct slab *slab; in virt_to_cache() local
642 slab = virt_to_slab(obj); in virt_to_cache()
643 if (WARN_ONCE(!slab, "%s: Object is not a Slab page!\n", in virt_to_cache()
646 return slab->slab_cache; in virt_to_cache()
649 static __always_inline void account_slab(struct slab *slab, int order, in account_slab() argument
653 memcg_alloc_slab_cgroups(slab, s, gfp, true); in account_slab()
655 mod_node_page_state(slab_pgdat(slab), cache_vmstat_idx(s), in account_slab()
659 static __always_inline void unaccount_slab(struct slab *slab, int order, in unaccount_slab() argument
663 memcg_free_slab_cgroups(slab); in unaccount_slab()
665 mod_node_page_state(slab_pgdat(slab), cache_vmstat_idx(s), in unaccount_slab()
884 struct slab *kp_slab;
892 void __kmem_obj_info(struct kmem_obj_info *kpp, void *object, struct slab *slab);
897 const struct slab *slab, bool to_user);
901 const struct slab *slab, bool to_user) in __check_heap_object() argument