Lines Matching refs:key
72 jea->key = jeb->key - delta; in jump_label_swap()
76 jeb->key = tmp.key + delta; in jump_label_swap()
93 static void jump_label_update(struct static_key *key);
104 int static_key_count(struct static_key *key) in static_key_count() argument
110 int n = atomic_read(&key->enabled); in static_key_count()
127 bool static_key_fast_inc_not_disabled(struct static_key *key) in static_key_fast_inc_not_disabled() argument
131 STATIC_KEY_CHECK_USE(key); in static_key_fast_inc_not_disabled()
138 v = atomic_read(&key->enabled); in static_key_fast_inc_not_disabled()
142 } while (!likely(atomic_try_cmpxchg(&key->enabled, &v, v + 1))); in static_key_fast_inc_not_disabled()
148 bool static_key_slow_inc_cpuslocked(struct static_key *key) in static_key_slow_inc_cpuslocked() argument
159 if (static_key_fast_inc_not_disabled(key)) in static_key_slow_inc_cpuslocked()
163 if (atomic_read(&key->enabled) == 0) { in static_key_slow_inc_cpuslocked()
164 atomic_set(&key->enabled, -1); in static_key_slow_inc_cpuslocked()
165 jump_label_update(key); in static_key_slow_inc_cpuslocked()
170 atomic_set_release(&key->enabled, 1); in static_key_slow_inc_cpuslocked()
172 if (WARN_ON_ONCE(!static_key_fast_inc_not_disabled(key))) { in static_key_slow_inc_cpuslocked()
181 bool static_key_slow_inc(struct static_key *key) in static_key_slow_inc() argument
186 ret = static_key_slow_inc_cpuslocked(key); in static_key_slow_inc()
192 void static_key_enable_cpuslocked(struct static_key *key) in static_key_enable_cpuslocked() argument
194 STATIC_KEY_CHECK_USE(key); in static_key_enable_cpuslocked()
197 if (atomic_read(&key->enabled) > 0) { in static_key_enable_cpuslocked()
198 WARN_ON_ONCE(atomic_read(&key->enabled) != 1); in static_key_enable_cpuslocked()
203 if (atomic_read(&key->enabled) == 0) { in static_key_enable_cpuslocked()
204 atomic_set(&key->enabled, -1); in static_key_enable_cpuslocked()
205 jump_label_update(key); in static_key_enable_cpuslocked()
209 atomic_set_release(&key->enabled, 1); in static_key_enable_cpuslocked()
215 void static_key_enable(struct static_key *key) in static_key_enable() argument
218 static_key_enable_cpuslocked(key); in static_key_enable()
223 void static_key_disable_cpuslocked(struct static_key *key) in static_key_disable_cpuslocked() argument
225 STATIC_KEY_CHECK_USE(key); in static_key_disable_cpuslocked()
228 if (atomic_read(&key->enabled) != 1) { in static_key_disable_cpuslocked()
229 WARN_ON_ONCE(atomic_read(&key->enabled) != 0); in static_key_disable_cpuslocked()
234 if (atomic_cmpxchg(&key->enabled, 1, 0)) in static_key_disable_cpuslocked()
235 jump_label_update(key); in static_key_disable_cpuslocked()
240 void static_key_disable(struct static_key *key) in static_key_disable() argument
243 static_key_disable_cpuslocked(key); in static_key_disable()
248 static bool static_key_slow_try_dec(struct static_key *key) in static_key_slow_try_dec() argument
252 val = atomic_fetch_add_unless(&key->enabled, -1, 1); in static_key_slow_try_dec()
267 static void __static_key_slow_dec_cpuslocked(struct static_key *key) in __static_key_slow_dec_cpuslocked() argument
271 if (static_key_slow_try_dec(key)) in __static_key_slow_dec_cpuslocked()
275 if (atomic_dec_and_test(&key->enabled)) in __static_key_slow_dec_cpuslocked()
276 jump_label_update(key); in __static_key_slow_dec_cpuslocked()
280 static void __static_key_slow_dec(struct static_key *key) in __static_key_slow_dec() argument
283 __static_key_slow_dec_cpuslocked(key); in __static_key_slow_dec()
289 struct static_key_deferred *key = in jump_label_update_timeout() local
291 __static_key_slow_dec(&key->key); in jump_label_update_timeout()
295 void static_key_slow_dec(struct static_key *key) in static_key_slow_dec() argument
297 STATIC_KEY_CHECK_USE(key); in static_key_slow_dec()
298 __static_key_slow_dec(key); in static_key_slow_dec()
302 void static_key_slow_dec_cpuslocked(struct static_key *key) in static_key_slow_dec_cpuslocked() argument
304 STATIC_KEY_CHECK_USE(key); in static_key_slow_dec_cpuslocked()
305 __static_key_slow_dec_cpuslocked(key); in static_key_slow_dec_cpuslocked()
308 void __static_key_slow_dec_deferred(struct static_key *key, in __static_key_slow_dec_deferred() argument
312 STATIC_KEY_CHECK_USE(key); in __static_key_slow_dec_deferred()
314 if (static_key_slow_try_dec(key)) in __static_key_slow_dec_deferred()
321 void __static_key_deferred_flush(void *key, struct delayed_work *work) in __static_key_deferred_flush() argument
323 STATIC_KEY_CHECK_USE(key); in __static_key_deferred_flush()
328 void jump_label_rate_limit(struct static_key_deferred *key, in jump_label_rate_limit() argument
331 STATIC_KEY_CHECK_USE(key); in jump_label_rate_limit()
332 key->timeout = rl; in jump_label_rate_limit()
333 INIT_DELAYED_WORK(&key->work, jump_label_update_timeout); in jump_label_rate_limit()
371 static inline struct jump_entry *static_key_entries(struct static_key *key) in static_key_entries() argument
373 WARN_ON_ONCE(key->type & JUMP_TYPE_LINKED); in static_key_entries()
374 return (struct jump_entry *)(key->type & ~JUMP_TYPE_MASK); in static_key_entries()
377 static inline bool static_key_type(struct static_key *key) in static_key_type() argument
379 return key->type & JUMP_TYPE_TRUE; in static_key_type()
382 static inline bool static_key_linked(struct static_key *key) in static_key_linked() argument
384 return key->type & JUMP_TYPE_LINKED; in static_key_linked()
387 static inline void static_key_clear_linked(struct static_key *key) in static_key_clear_linked() argument
389 key->type &= ~JUMP_TYPE_LINKED; in static_key_clear_linked()
392 static inline void static_key_set_linked(struct static_key *key) in static_key_set_linked() argument
394 key->type |= JUMP_TYPE_LINKED; in static_key_set_linked()
406 static void static_key_set_entries(struct static_key *key, in static_key_set_entries() argument
412 type = key->type & JUMP_TYPE_MASK; in static_key_set_entries()
413 key->entries = entries; in static_key_set_entries()
414 key->type |= type; in static_key_set_entries()
419 struct static_key *key = jump_entry_key(entry); in jump_label_type() local
420 bool enabled = static_key_enabled(key); in jump_label_type()
454 static void __jump_label_update(struct static_key *key, in __jump_label_update() argument
459 for (; (entry < stop) && (jump_entry_key(entry) == key); entry++) { in __jump_label_update()
465 static void __jump_label_update(struct static_key *key, in __jump_label_update() argument
470 for (; (entry < stop) && (jump_entry_key(entry) == key); entry++) { in __jump_label_update()
491 struct static_key *key = NULL; in jump_label_init() local
522 if (iterk == key) in jump_label_init()
525 key = iterk; in jump_label_init()
526 static_key_set_entries(key, iter); in jump_label_init()
537 struct static_key *key = jump_entry_key(entry); in jump_label_init_type() local
538 bool type = static_key_type(key); in jump_label_init_type()
551 static inline struct static_key_mod *static_key_mod(struct static_key *key) in static_key_mod() argument
553 WARN_ON_ONCE(!static_key_linked(key)); in static_key_mod()
554 return (struct static_key_mod *)(key->type & ~JUMP_TYPE_MASK); in static_key_mod()
563 static void static_key_set_mod(struct static_key *key, in static_key_set_mod() argument
569 type = key->type & JUMP_TYPE_MASK; in static_key_set_mod()
570 key->next = mod; in static_key_set_mod()
571 key->type |= type; in static_key_set_mod()
598 static void __jump_label_mod_update(struct static_key *key) in __jump_label_mod_update() argument
602 for (mod = static_key_mod(key); mod; mod = mod->next) { in __jump_label_mod_update()
618 __jump_label_update(key, mod->entries, stop, in __jump_label_mod_update()
628 struct static_key *key = NULL; in jump_label_add_module() local
645 if (iterk == key) in jump_label_add_module()
648 key = iterk; in jump_label_add_module()
649 if (within_module((unsigned long)key, mod)) { in jump_label_add_module()
650 static_key_set_entries(key, iter); in jump_label_add_module()
656 if (!static_key_linked(key)) { in jump_label_add_module()
664 jlm2->mod = __module_address((unsigned long)key); in jump_label_add_module()
666 jlm2->entries = static_key_entries(key); in jump_label_add_module()
668 static_key_set_mod(key, jlm2); in jump_label_add_module()
669 static_key_set_linked(key); in jump_label_add_module()
673 jlm->next = static_key_mod(key); in jump_label_add_module()
674 static_key_set_mod(key, jlm); in jump_label_add_module()
675 static_key_set_linked(key); in jump_label_add_module()
679 __jump_label_update(key, iter, iter_stop, true); in jump_label_add_module()
690 struct static_key *key = NULL; in jump_label_del_module() local
694 if (jump_entry_key(iter) == key) in jump_label_del_module()
697 key = jump_entry_key(iter); in jump_label_del_module()
699 if (within_module((unsigned long)key, mod)) in jump_label_del_module()
703 if (WARN_ON(!static_key_linked(key))) in jump_label_del_module()
706 prev = &key->next; in jump_label_del_module()
707 jlm = static_key_mod(key); in jump_label_del_module()
718 if (prev == &key->next) in jump_label_del_module()
719 static_key_set_mod(key, jlm->next); in jump_label_del_module()
725 jlm = static_key_mod(key); in jump_label_del_module()
728 static_key_set_entries(key, jlm->entries); in jump_label_del_module()
729 static_key_clear_linked(key); in jump_label_del_module()
805 static void jump_label_update(struct static_key *key) in jump_label_update() argument
813 if (static_key_linked(key)) { in jump_label_update()
814 __jump_label_mod_update(key); in jump_label_update()
819 mod = __module_address((unsigned long)key); in jump_label_update()
826 entry = static_key_entries(key); in jump_label_update()
829 __jump_label_update(key, entry, stop, init); in jump_label_update()
841 WARN_ON(static_key_enabled(&sk_true.key) != true); in jump_label_test()
842 WARN_ON(static_key_enabled(&sk_false.key) != false); in jump_label_test()
852 WARN_ON(static_key_enabled(&sk_true.key) == true); in jump_label_test()
853 WARN_ON(static_key_enabled(&sk_false.key) == false); in jump_label_test()