Lines Matching refs:class
280 unsigned int class:CLASS_BITS + 1; member
556 *class_idx = zspage->class; in get_zspage_mapping()
562 return pool->size_class[zspage->class]; in zspage_class()
569 zspage->class = class_idx; in set_zspage_mapping()
592 static inline void class_stat_inc(struct size_class *class, in class_stat_inc() argument
595 class->stats.objs[type] += cnt; in class_stat_inc()
599 static inline void class_stat_dec(struct size_class *class, in class_stat_dec() argument
602 class->stats.objs[type] -= cnt; in class_stat_dec()
606 static inline unsigned long zs_stat_get(struct size_class *class, in zs_stat_get() argument
609 return class->stats.objs[type]; in zs_stat_get()
629 static unsigned long zs_can_compact(struct size_class *class);
635 struct size_class *class; in zs_stats_size_show() local
649 class = pool->size_class[i]; in zs_stats_size_show()
651 if (class->index != i) in zs_stats_size_show()
655 class_almost_full = zs_stat_get(class, CLASS_ALMOST_FULL); in zs_stats_size_show()
656 class_almost_empty = zs_stat_get(class, CLASS_ALMOST_EMPTY); in zs_stats_size_show()
657 obj_allocated = zs_stat_get(class, OBJ_ALLOCATED); in zs_stats_size_show()
658 obj_used = zs_stat_get(class, OBJ_USED); in zs_stats_size_show()
659 freeable = zs_can_compact(class); in zs_stats_size_show()
662 objs_per_zspage = class->objs_per_zspage; in zs_stats_size_show()
664 class->pages_per_zspage; in zs_stats_size_show()
668 i, class->size, class_almost_full, class_almost_empty, in zs_stats_size_show()
670 class->pages_per_zspage, freeable); in zs_stats_size_show()
734 static enum fullness_group get_fullness_group(struct size_class *class, in get_fullness_group() argument
741 objs_per_zspage = class->objs_per_zspage; in get_fullness_group()
761 static void insert_zspage(struct size_class *class, in insert_zspage() argument
767 class_stat_inc(class, fullness, 1); in insert_zspage()
768 head = list_first_entry_or_null(&class->fullness_list[fullness], in insert_zspage()
777 list_add(&zspage->list, &class->fullness_list[fullness]); in insert_zspage()
784 static void remove_zspage(struct size_class *class, in remove_zspage() argument
788 VM_BUG_ON(list_empty(&class->fullness_list[fullness])); in remove_zspage()
791 class_stat_dec(class, fullness, 1); in remove_zspage()
803 static enum fullness_group fix_fullness_group(struct size_class *class, in fix_fullness_group() argument
810 newfg = get_fullness_group(class, zspage); in fix_fullness_group()
814 remove_zspage(class, zspage, currfg); in fix_fullness_group()
815 insert_zspage(class, zspage, newfg); in fix_fullness_group()
944 static unsigned long find_deferred_handle_obj(struct size_class *class,
950 static void free_handles(struct zs_pool *pool, struct size_class *class, in free_handles() argument
958 handle = find_deferred_handle_obj(class, page, &obj_idx); in free_handles()
972 static inline void free_handles(struct zs_pool *pool, struct size_class *class, in free_handles() argument
976 static void __free_zspage(struct zs_pool *pool, struct size_class *class, in __free_zspage() argument
991 free_handles(pool, class, zspage); in __free_zspage()
1006 class_stat_dec(class, OBJ_ALLOCATED, class->objs_per_zspage); in __free_zspage()
1007 atomic_long_sub(class->pages_per_zspage, in __free_zspage()
1011 static void free_zspage(struct zs_pool *pool, struct size_class *class, in free_zspage() argument
1027 remove_zspage(class, zspage, ZS_EMPTY); in free_zspage()
1031 __free_zspage(pool, class, zspage); in free_zspage()
1035 static void init_zspage(struct size_class *class, struct zspage *zspage) in init_zspage() argument
1051 while ((off += class->size) < PAGE_SIZE) { in init_zspage()
1053 link += class->size / sizeof(*link); in init_zspage()
1084 static void create_page_chain(struct size_class *class, struct zspage *zspage, in create_page_chain() argument
1090 int nr_pages = class->pages_per_zspage; in create_page_chain()
1107 if (unlikely(class->objs_per_zspage == 1 && in create_page_chain()
1108 class->pages_per_zspage == 1)) in create_page_chain()
1121 struct size_class *class, in alloc_zspage() argument
1134 for (i = 0; i < class->pages_per_zspage; i++) { in alloc_zspage()
1151 create_page_chain(class, zspage, pages); in alloc_zspage()
1152 init_zspage(class, zspage); in alloc_zspage()
1158 static struct zspage *find_get_zspage(struct size_class *class) in find_get_zspage() argument
1164 zspage = list_first_entry_or_null(&class->fullness_list[i], in find_get_zspage()
1280 static bool zspage_full(struct size_class *class, struct zspage *zspage) in zspage_full() argument
1282 return get_zspage_inuse(zspage) == class->objs_per_zspage; in zspage_full()
1298 struct size_class *class; in zs_lookup_class_index() local
1300 class = pool->size_class[get_size_class_index(size)]; in zs_lookup_class_index()
1302 return class->index; in zs_lookup_class_index()
1335 struct size_class *class; in zs_map_object() local
1387 class = zspage_class(pool, zspage); in zs_map_object()
1388 off = (class->size * obj_idx) & ~PAGE_MASK; in zs_map_object()
1393 if (off + class->size <= PAGE_SIZE) { in zs_map_object()
1405 ret = __zs_map_object(area, pages, off, class->size); in zs_map_object()
1421 struct size_class *class; in zs_unmap_object() local
1427 class = zspage_class(pool, zspage); in zs_unmap_object()
1428 off = (class->size * obj_idx) & ~PAGE_MASK; in zs_unmap_object()
1431 if (off + class->size <= PAGE_SIZE) in zs_unmap_object()
1440 __zs_unmap_object(area, pages, off, class->size); in zs_unmap_object()
1473 struct size_class *class; in obj_malloc() local
1479 class = pool->size_class[zspage->class]; in obj_malloc()
1483 offset = obj * class->size; in obj_malloc()
1523 struct size_class *class; in zs_malloc() local
1536 class = pool->size_class[get_size_class_index(size)]; in zs_malloc()
1540 zspage = find_get_zspage(class); in zs_malloc()
1544 fix_fullness_group(class, zspage); in zs_malloc()
1546 class_stat_inc(class, OBJ_USED, 1); in zs_malloc()
1554 zspage = alloc_zspage(pool, class, gfp); in zs_malloc()
1562 newfg = get_fullness_group(class, zspage); in zs_malloc()
1563 insert_zspage(class, zspage, newfg); in zs_malloc()
1564 set_zspage_mapping(zspage, class->index, newfg); in zs_malloc()
1566 atomic_long_add(class->pages_per_zspage, in zs_malloc()
1568 class_stat_inc(class, OBJ_ALLOCATED, class->objs_per_zspage); in zs_malloc()
1569 class_stat_inc(class, OBJ_USED, 1); in zs_malloc()
1624 struct size_class *class; in zs_free() local
1638 class = zspage_class(pool, zspage); in zs_free()
1640 class_stat_dec(class, OBJ_USED, 1); in zs_free()
1650 obj_free(class->size, obj, &handle); in zs_free()
1655 obj_free(class->size, obj, NULL); in zs_free()
1657 fullness = fix_fullness_group(class, zspage); in zs_free()
1659 free_zspage(pool, class, zspage); in zs_free()
1666 static void zs_object_copy(struct size_class *class, unsigned long dst, in zs_object_copy() argument
1676 s_size = d_size = class->size; in zs_object_copy()
1681 s_off = (class->size * s_objidx) & ~PAGE_MASK; in zs_object_copy()
1682 d_off = (class->size * d_objidx) & ~PAGE_MASK; in zs_object_copy()
1684 if (s_off + class->size > PAGE_SIZE) in zs_object_copy()
1687 if (d_off + class->size > PAGE_SIZE) in zs_object_copy()
1698 if (written == class->size) in zs_object_copy()
1719 s_size = class->size - written; in zs_object_copy()
1727 d_size = class->size - written; in zs_object_copy()
1740 static unsigned long find_tagged_obj(struct size_class *class, in find_tagged_obj() argument
1749 offset += class->size * index; in find_tagged_obj()
1755 offset += class->size; in find_tagged_obj()
1770 static unsigned long find_alloced_obj(struct size_class *class, in find_alloced_obj() argument
1773 return find_tagged_obj(class, page, obj_idx, OBJ_ALLOCATED_TAG); in find_alloced_obj()
1781 static unsigned long find_deferred_handle_obj(struct size_class *class, in find_deferred_handle_obj() argument
1784 return find_tagged_obj(class, page, obj_idx, OBJ_DEFERRED_HANDLE_TAG); in find_deferred_handle_obj()
1799 static int migrate_zspage(struct zs_pool *pool, struct size_class *class, in migrate_zspage() argument
1810 handle = find_alloced_obj(class, s_page, &obj_idx); in migrate_zspage()
1820 if (zspage_full(class, get_zspage(d_page))) { in migrate_zspage()
1827 zs_object_copy(class, free_obj, used_obj); in migrate_zspage()
1830 obj_free(class->size, used_obj, NULL); in migrate_zspage()
1840 static struct zspage *isolate_zspage(struct size_class *class, bool source) in isolate_zspage() argument
1852 zspage = list_first_entry_or_null(&class->fullness_list[fg[i]], in isolate_zspage()
1855 remove_zspage(class, zspage, fg[i]); in isolate_zspage()
1870 static enum fullness_group putback_zspage(struct size_class *class, in putback_zspage() argument
1875 fullness = get_fullness_group(class, zspage); in putback_zspage()
1876 insert_zspage(class, zspage, fullness); in putback_zspage()
1877 set_zspage_mapping(zspage, class->index, fullness); in putback_zspage()
1988 static void replace_sub_page(struct size_class *class, struct zspage *zspage, in replace_sub_page() argument
2004 create_page_chain(class, zspage, pages); in replace_sub_page()
2033 struct size_class *class; in zs_page_migrate() local
2061 class = zspage_class(pool, zspage); in zs_page_migrate()
2077 addr += class->size) { in zs_page_migrate()
2089 replace_sub_page(class, zspage, newpage, page); in zs_page_migrate()
2135 struct size_class *class; in async_free_zspage() local
2144 class = pool->size_class[i]; in async_free_zspage()
2145 if (class->index != i) in async_free_zspage()
2149 list_splice_init(&class->fullness_list[ZS_EMPTY], &free_pages); in async_free_zspage()
2159 class = pool->size_class[class_idx]; in async_free_zspage()
2164 __free_zspage(pool, class, zspage); in async_free_zspage()
2203 static unsigned long zs_can_compact(struct size_class *class) in zs_can_compact() argument
2206 unsigned long obj_allocated = zs_stat_get(class, OBJ_ALLOCATED); in zs_can_compact()
2207 unsigned long obj_used = zs_stat_get(class, OBJ_USED); in zs_can_compact()
2213 obj_wasted /= class->objs_per_zspage; in zs_can_compact()
2215 return obj_wasted * class->pages_per_zspage; in zs_can_compact()
2219 struct size_class *class) in __zs_compact() argument
2231 while ((src_zspage = isolate_zspage(class, true))) { in __zs_compact()
2235 if (!zs_can_compact(class)) in __zs_compact()
2241 while ((dst_zspage = isolate_zspage(class, false))) { in __zs_compact()
2249 if (!migrate_zspage(pool, class, &cc)) in __zs_compact()
2252 putback_zspage(class, dst_zspage); in __zs_compact()
2263 putback_zspage(class, dst_zspage); in __zs_compact()
2266 if (putback_zspage(class, src_zspage) == ZS_EMPTY) { in __zs_compact()
2268 free_zspage(pool, class, src_zspage); in __zs_compact()
2269 pages_freed += class->pages_per_zspage; in __zs_compact()
2278 putback_zspage(class, src_zspage); in __zs_compact()
2290 struct size_class *class; in zs_compact() local
2294 class = pool->size_class[i]; in zs_compact()
2295 if (class->index != i) in zs_compact()
2297 pages_freed += __zs_compact(pool, class); in zs_compact()
2332 struct size_class *class; in zs_shrinker_count() local
2338 class = pool->size_class[i]; in zs_shrinker_count()
2339 if (class->index != i) in zs_shrinker_count()
2342 pages_to_free += zs_can_compact(class); in zs_shrinker_count()
2423 struct size_class *class; in zs_create_pool() local
2469 class = kzalloc(sizeof(struct size_class), GFP_KERNEL); in zs_create_pool()
2470 if (!class) in zs_create_pool()
2473 class->size = size; in zs_create_pool()
2474 class->index = i; in zs_create_pool()
2475 class->pages_per_zspage = pages_per_zspage; in zs_create_pool()
2476 class->objs_per_zspage = objs_per_zspage; in zs_create_pool()
2477 pool->size_class[i] = class; in zs_create_pool()
2480 INIT_LIST_HEAD(&class->fullness_list[fullness]); in zs_create_pool()
2482 prev_class = class; in zs_create_pool()
2518 struct size_class *class = pool->size_class[i]; in zs_destroy_pool() local
2520 if (!class) in zs_destroy_pool()
2523 if (class->index != i) in zs_destroy_pool()
2527 if (!list_empty(&class->fullness_list[fg])) { in zs_destroy_pool()
2529 class->size, fg); in zs_destroy_pool()
2532 kfree(class); in zs_destroy_pool()
2542 static void restore_freelist(struct zs_pool *pool, struct size_class *class, in restore_freelist() argument
2564 off += class->size; in restore_freelist()
2585 off += class->size; in restore_freelist()
2642 struct size_class *class; in zs_reclaim_page() local
2650 class = zspage_class(pool, zspage); in zs_reclaim_page()
2651 fullness = get_fullness_group(class, zspage); in zs_reclaim_page()
2654 remove_zspage(class, zspage, fullness); in zs_reclaim_page()
2665 handle = find_alloced_obj(class, page, &obj_idx); in zs_reclaim_page()
2704 __free_zspage(pool, class, zspage); in zs_reclaim_page()
2714 restore_freelist(pool, class, zspage); in zs_reclaim_page()
2715 putback_zspage(class, zspage); in zs_reclaim_page()