Lines Matching refs:shrinker
216 static int shrinker_memcg_alloc(struct shrinker *shrinker) in shrinker_memcg_alloc() argument
224 id = idr_alloc(&shrinker_idr, shrinker, 0, 0, GFP_KERNEL); in shrinker_memcg_alloc()
234 shrinker->id = id; in shrinker_memcg_alloc()
241 static void shrinker_memcg_remove(struct shrinker *shrinker) in shrinker_memcg_remove() argument
243 int id = shrinker->id; in shrinker_memcg_remove()
252 static long xchg_nr_deferred_memcg(int nid, struct shrinker *shrinker, in xchg_nr_deferred_memcg() argument
261 unit = info->unit[shrinker_id_to_index(shrinker->id)]; in xchg_nr_deferred_memcg()
262 nr_deferred = atomic_long_xchg(&unit->nr_deferred[shrinker_id_to_offset(shrinker->id)], 0); in xchg_nr_deferred_memcg()
268 static long add_nr_deferred_memcg(long nr, int nid, struct shrinker *shrinker, in add_nr_deferred_memcg() argument
277 unit = info->unit[shrinker_id_to_index(shrinker->id)]; in add_nr_deferred_memcg()
279 atomic_long_add_return(nr, &unit->nr_deferred[shrinker_id_to_offset(shrinker->id)]); in add_nr_deferred_memcg()
314 static int shrinker_memcg_alloc(struct shrinker *shrinker) in shrinker_memcg_alloc() argument
319 static void shrinker_memcg_remove(struct shrinker *shrinker) in shrinker_memcg_remove() argument
323 static long xchg_nr_deferred_memcg(int nid, struct shrinker *shrinker, in xchg_nr_deferred_memcg() argument
329 static long add_nr_deferred_memcg(long nr, int nid, struct shrinker *shrinker, in add_nr_deferred_memcg() argument
336 static long xchg_nr_deferred(struct shrinker *shrinker, in xchg_nr_deferred() argument
341 if (!(shrinker->flags & SHRINKER_NUMA_AWARE)) in xchg_nr_deferred()
345 (shrinker->flags & SHRINKER_MEMCG_AWARE)) in xchg_nr_deferred()
346 return xchg_nr_deferred_memcg(nid, shrinker, in xchg_nr_deferred()
349 return atomic_long_xchg(&shrinker->nr_deferred[nid], 0); in xchg_nr_deferred()
353 static long add_nr_deferred(long nr, struct shrinker *shrinker, in add_nr_deferred() argument
358 if (!(shrinker->flags & SHRINKER_NUMA_AWARE)) in add_nr_deferred()
362 (shrinker->flags & SHRINKER_MEMCG_AWARE)) in add_nr_deferred()
363 return add_nr_deferred_memcg(nr, nid, shrinker, in add_nr_deferred()
366 return atomic_long_add_return(nr, &shrinker->nr_deferred[nid]); in add_nr_deferred()
372 struct shrinker *shrinker, int priority) in do_shrink_slab() argument
380 long batch_size = shrinker->batch ? shrinker->batch in do_shrink_slab()
384 freeable = shrinker->count_objects(shrinker, shrinkctl); in do_shrink_slab()
393 nr = xchg_nr_deferred(shrinker, shrinkctl); in do_shrink_slab()
395 if (shrinker->seeks) { in do_shrink_slab()
398 do_div(delta, shrinker->seeks); in do_shrink_slab()
412 trace_mm_shrink_slab_start(shrinker, shrinkctl, nr, in do_shrink_slab()
437 ret = shrinker->scan_objects(shrinker, shrinkctl); in do_shrink_slab()
462 new_nr = add_nr_deferred(next_deferred, shrinker, shrinkctl); in do_shrink_slab()
464 trace_mm_shrink_slab_end(shrinker, shrinkctl->nid, freed, nr, new_nr, total_scan); in do_shrink_slab()
533 struct shrinker *shrinker; in shrink_slab_memcg() local
537 shrinker = idr_find(&shrinker_idr, shrinker_id); in shrink_slab_memcg()
538 if (unlikely(!shrinker || !shrinker_try_get(shrinker))) { in shrink_slab_memcg()
547 !(shrinker->flags & SHRINKER_NONSLAB)) in shrink_slab_memcg()
550 ret = do_shrink_slab(&sc, shrinker, priority); in shrink_slab_memcg()
569 ret = do_shrink_slab(&sc, shrinker, priority); in shrink_slab_memcg()
576 shrinker_put(shrinker); in shrink_slab_memcg()
618 struct shrinker *shrinker; in shrink_slab() local
652 list_for_each_entry_rcu(shrinker, &shrinker_list, list) { in shrink_slab()
659 if (!shrinker_try_get(shrinker)) in shrink_slab()
664 ret = do_shrink_slab(&sc, shrinker, priority); in shrink_slab()
670 shrinker_put(shrinker); in shrink_slab()
678 struct shrinker *shrinker_alloc(unsigned int flags, const char *fmt, ...) in shrinker_alloc()
680 struct shrinker *shrinker; in shrinker_alloc() local
685 shrinker = kzalloc(sizeof(struct shrinker), GFP_KERNEL); in shrinker_alloc()
686 if (!shrinker) in shrinker_alloc()
690 err = shrinker_debugfs_name_alloc(shrinker, fmt, ap); in shrinker_alloc()
695 shrinker->flags = flags | SHRINKER_ALLOCATED; in shrinker_alloc()
696 shrinker->seeks = DEFAULT_SEEKS; in shrinker_alloc()
699 err = shrinker_memcg_alloc(shrinker); in shrinker_alloc()
702 shrinker->flags &= ~SHRINKER_MEMCG_AWARE; in shrinker_alloc()
709 return shrinker; in shrinker_alloc()
720 size = sizeof(*shrinker->nr_deferred); in shrinker_alloc()
724 shrinker->nr_deferred = kzalloc(size, GFP_KERNEL); in shrinker_alloc()
725 if (!shrinker->nr_deferred) in shrinker_alloc()
728 return shrinker; in shrinker_alloc()
731 shrinker_debugfs_name_free(shrinker); in shrinker_alloc()
733 kfree(shrinker); in shrinker_alloc()
738 void shrinker_register(struct shrinker *shrinker) in shrinker_register() argument
740 if (unlikely(!(shrinker->flags & SHRINKER_ALLOCATED))) { in shrinker_register()
746 list_add_tail_rcu(&shrinker->list, &shrinker_list); in shrinker_register()
747 shrinker->flags |= SHRINKER_REGISTERED; in shrinker_register()
748 shrinker_debugfs_add(shrinker); in shrinker_register()
751 init_completion(&shrinker->done); in shrinker_register()
757 refcount_set(&shrinker->refcount, 1); in shrinker_register()
763 struct shrinker *shrinker = container_of(head, struct shrinker, rcu); in shrinker_free_rcu_cb() local
765 kfree(shrinker->nr_deferred); in shrinker_free_rcu_cb()
766 kfree(shrinker); in shrinker_free_rcu_cb()
769 void shrinker_free(struct shrinker *shrinker) in shrinker_free() argument
774 if (!shrinker) in shrinker_free()
777 if (shrinker->flags & SHRINKER_REGISTERED) { in shrinker_free()
779 shrinker_put(shrinker); in shrinker_free()
786 wait_for_completion(&shrinker->done); in shrinker_free()
790 if (shrinker->flags & SHRINKER_REGISTERED) { in shrinker_free()
795 list_del_rcu(&shrinker->list); in shrinker_free()
796 debugfs_entry = shrinker_debugfs_detach(shrinker, &debugfs_id); in shrinker_free()
797 shrinker->flags &= ~SHRINKER_REGISTERED; in shrinker_free()
800 shrinker_debugfs_name_free(shrinker); in shrinker_free()
802 if (shrinker->flags & SHRINKER_MEMCG_AWARE) in shrinker_free()
803 shrinker_memcg_remove(shrinker); in shrinker_free()
809 call_rcu(&shrinker->rcu, shrinker_free_rcu_cb); in shrinker_free()