Lines Matching refs:fbc
24 struct percpu_counter *fbc = addr; in percpu_counter_fixup_free() local
28 percpu_counter_destroy(fbc); in percpu_counter_fixup_free()
29 debug_object_free(fbc, &percpu_counter_debug_descr); in percpu_counter_fixup_free()
41 static inline void debug_percpu_counter_activate(struct percpu_counter *fbc) in debug_percpu_counter_activate() argument
43 debug_object_init(fbc, &percpu_counter_debug_descr); in debug_percpu_counter_activate()
44 debug_object_activate(fbc, &percpu_counter_debug_descr); in debug_percpu_counter_activate()
47 static inline void debug_percpu_counter_deactivate(struct percpu_counter *fbc) in debug_percpu_counter_deactivate() argument
49 debug_object_deactivate(fbc, &percpu_counter_debug_descr); in debug_percpu_counter_deactivate()
50 debug_object_free(fbc, &percpu_counter_debug_descr); in debug_percpu_counter_deactivate()
54 static inline void debug_percpu_counter_activate(struct percpu_counter *fbc) in debug_percpu_counter_activate() argument
56 static inline void debug_percpu_counter_deactivate(struct percpu_counter *fbc) in debug_percpu_counter_deactivate() argument
60 void percpu_counter_set(struct percpu_counter *fbc, s64 amount) in percpu_counter_set() argument
65 raw_spin_lock_irqsave(&fbc->lock, flags); in percpu_counter_set()
67 s32 *pcount = per_cpu_ptr(fbc->counters, cpu); in percpu_counter_set()
70 fbc->count = amount; in percpu_counter_set()
71 raw_spin_unlock_irqrestore(&fbc->lock, flags); in percpu_counter_set()
93 void percpu_counter_add_batch(struct percpu_counter *fbc, s64 amount, s32 batch) in percpu_counter_add_batch() argument
98 count = this_cpu_read(*fbc->counters); in percpu_counter_add_batch()
101 raw_spin_lock_irqsave(&fbc->lock, flags); in percpu_counter_add_batch()
106 count = __this_cpu_read(*fbc->counters); in percpu_counter_add_batch()
107 fbc->count += count + amount; in percpu_counter_add_batch()
108 __this_cpu_sub(*fbc->counters, count); in percpu_counter_add_batch()
109 raw_spin_unlock_irqrestore(&fbc->lock, flags); in percpu_counter_add_batch()
112 } while (!this_cpu_try_cmpxchg(*fbc->counters, &count, count + amount)); in percpu_counter_add_batch()
120 void percpu_counter_add_batch(struct percpu_counter *fbc, s64 amount, s32 batch) in percpu_counter_add_batch() argument
126 count = __this_cpu_read(*fbc->counters) + amount; in percpu_counter_add_batch()
128 raw_spin_lock(&fbc->lock); in percpu_counter_add_batch()
129 fbc->count += count; in percpu_counter_add_batch()
130 __this_cpu_sub(*fbc->counters, count - amount); in percpu_counter_add_batch()
131 raw_spin_unlock(&fbc->lock); in percpu_counter_add_batch()
133 this_cpu_add(*fbc->counters, amount); in percpu_counter_add_batch()
146 void percpu_counter_sync(struct percpu_counter *fbc) in percpu_counter_sync() argument
151 raw_spin_lock_irqsave(&fbc->lock, flags); in percpu_counter_sync()
152 count = __this_cpu_read(*fbc->counters); in percpu_counter_sync()
153 fbc->count += count; in percpu_counter_sync()
154 __this_cpu_sub(*fbc->counters, count); in percpu_counter_sync()
155 raw_spin_unlock_irqrestore(&fbc->lock, flags); in percpu_counter_sync()
171 s64 __percpu_counter_sum(struct percpu_counter *fbc) in __percpu_counter_sum() argument
177 raw_spin_lock_irqsave(&fbc->lock, flags); in __percpu_counter_sum()
178 ret = fbc->count; in __percpu_counter_sum()
180 s32 *pcount = per_cpu_ptr(fbc->counters, cpu); in __percpu_counter_sum()
183 raw_spin_unlock_irqrestore(&fbc->lock, flags); in __percpu_counter_sum()
188 int __percpu_counter_init_many(struct percpu_counter *fbc, s64 amount, in __percpu_counter_init_many() argument
201 fbc[0].counters = NULL; in __percpu_counter_init_many()
206 raw_spin_lock_init(&fbc[i].lock); in __percpu_counter_init_many()
207 lockdep_set_class(&fbc[i].lock, key); in __percpu_counter_init_many()
209 INIT_LIST_HEAD(&fbc[i].list); in __percpu_counter_init_many()
211 fbc[i].count = amount; in __percpu_counter_init_many()
212 fbc[i].counters = (void __percpu *)counters + i * counter_size; in __percpu_counter_init_many()
214 debug_percpu_counter_activate(&fbc[i]); in __percpu_counter_init_many()
220 list_add(&fbc[i].list, &percpu_counters); in __percpu_counter_init_many()
227 void percpu_counter_destroy_many(struct percpu_counter *fbc, u32 nr_counters) in percpu_counter_destroy_many() argument
232 if (WARN_ON_ONCE(!fbc)) in percpu_counter_destroy_many()
235 if (!fbc[0].counters) in percpu_counter_destroy_many()
239 debug_percpu_counter_deactivate(&fbc[i]); in percpu_counter_destroy_many()
244 list_del(&fbc[i].list); in percpu_counter_destroy_many()
248 free_percpu(fbc[0].counters); in percpu_counter_destroy_many()
251 fbc[i].counters = NULL; in percpu_counter_destroy_many()
269 struct percpu_counter *fbc; in percpu_counter_cpu_dead() local
274 list_for_each_entry(fbc, &percpu_counters, list) { in percpu_counter_cpu_dead()
277 raw_spin_lock(&fbc->lock); in percpu_counter_cpu_dead()
278 pcount = per_cpu_ptr(fbc->counters, cpu); in percpu_counter_cpu_dead()
279 fbc->count += *pcount; in percpu_counter_cpu_dead()
281 raw_spin_unlock(&fbc->lock); in percpu_counter_cpu_dead()
292 int __percpu_counter_compare(struct percpu_counter *fbc, s64 rhs, s32 batch) in __percpu_counter_compare() argument
296 count = percpu_counter_read(fbc); in __percpu_counter_compare()
305 count = percpu_counter_sum(fbc); in __percpu_counter_compare()
327 bool __percpu_counter_limited_add(struct percpu_counter *fbc, in __percpu_counter_limited_add() argument
340 count = __this_cpu_read(*fbc->counters); in __percpu_counter_limited_add()
344 ((amount > 0 && fbc->count + unknown <= limit) || in __percpu_counter_limited_add()
345 (amount < 0 && fbc->count - unknown >= limit))) { in __percpu_counter_limited_add()
346 this_cpu_add(*fbc->counters, amount); in __percpu_counter_limited_add()
351 raw_spin_lock(&fbc->lock); in __percpu_counter_limited_add()
352 count = fbc->count + amount; in __percpu_counter_limited_add()
372 pcount = per_cpu_ptr(fbc->counters, cpu); in __percpu_counter_limited_add()
385 count = __this_cpu_read(*fbc->counters); in __percpu_counter_limited_add()
386 fbc->count += count + amount; in __percpu_counter_limited_add()
387 __this_cpu_sub(*fbc->counters, count); in __percpu_counter_limited_add()
389 raw_spin_unlock(&fbc->lock); in __percpu_counter_limited_add()