Lines Matching refs:lock
46 __mutex_init(struct mutex *lock, const char *name, struct lock_class_key *key) in __mutex_init() argument
48 atomic_long_set(&lock->owner, 0); in __mutex_init()
49 raw_spin_lock_init(&lock->wait_lock); in __mutex_init()
50 INIT_LIST_HEAD(&lock->wait_list); in __mutex_init()
52 osq_lock_init(&lock->osq); in __mutex_init()
55 debug_mutex_init(lock, name, key); in __mutex_init()
79 static inline struct task_struct *__mutex_owner(struct mutex *lock) in __mutex_owner() argument
81 return (struct task_struct *)(atomic_long_read(&lock->owner) & ~MUTEX_FLAGS); in __mutex_owner()
89 bool mutex_is_locked(struct mutex *lock) in mutex_is_locked() argument
91 return __mutex_owner(lock) != NULL; in mutex_is_locked()
103 static inline struct task_struct *__mutex_trylock_common(struct mutex *lock, bool handoff) in __mutex_trylock_common() argument
107 owner = atomic_long_read(&lock->owner); in __mutex_trylock_common()
129 if (atomic_long_try_cmpxchg_acquire(&lock->owner, &owner, task | flags)) { in __mutex_trylock_common()
142 static inline bool __mutex_trylock_or_handoff(struct mutex *lock, bool handoff) in __mutex_trylock_or_handoff() argument
144 return !__mutex_trylock_common(lock, handoff); in __mutex_trylock_or_handoff()
150 static inline bool __mutex_trylock(struct mutex *lock) in __mutex_trylock() argument
152 return !__mutex_trylock_common(lock, false); in __mutex_trylock()
166 static __always_inline bool __mutex_trylock_fast(struct mutex *lock) in __mutex_trylock_fast() argument
171 if (atomic_long_try_cmpxchg_acquire(&lock->owner, &zero, curr)) in __mutex_trylock_fast()
177 static __always_inline bool __mutex_unlock_fast(struct mutex *lock) in __mutex_unlock_fast() argument
181 return atomic_long_try_cmpxchg_release(&lock->owner, &curr, 0UL); in __mutex_unlock_fast()
185 static inline void __mutex_set_flag(struct mutex *lock, unsigned long flag) in __mutex_set_flag() argument
187 atomic_long_or(flag, &lock->owner); in __mutex_set_flag()
190 static inline void __mutex_clear_flag(struct mutex *lock, unsigned long flag) in __mutex_clear_flag() argument
192 atomic_long_andnot(flag, &lock->owner); in __mutex_clear_flag()
195 static inline bool __mutex_waiter_is_first(struct mutex *lock, struct mutex_waiter *waiter) in __mutex_waiter_is_first() argument
197 return list_first_entry(&lock->wait_list, struct mutex_waiter, list) == waiter; in __mutex_waiter_is_first()
205 __mutex_add_waiter(struct mutex *lock, struct mutex_waiter *waiter, in __mutex_add_waiter() argument
208 debug_mutex_add_waiter(lock, waiter, current); in __mutex_add_waiter()
211 if (__mutex_waiter_is_first(lock, waiter)) in __mutex_add_waiter()
212 __mutex_set_flag(lock, MUTEX_FLAG_WAITERS); in __mutex_add_waiter()
216 __mutex_remove_waiter(struct mutex *lock, struct mutex_waiter *waiter) in __mutex_remove_waiter() argument
219 if (likely(list_empty(&lock->wait_list))) in __mutex_remove_waiter()
220 __mutex_clear_flag(lock, MUTEX_FLAGS); in __mutex_remove_waiter()
222 debug_mutex_remove_waiter(lock, waiter, current); in __mutex_remove_waiter()
231 static void __mutex_handoff(struct mutex *lock, struct task_struct *task) in __mutex_handoff() argument
233 unsigned long owner = atomic_long_read(&lock->owner); in __mutex_handoff()
246 if (atomic_long_try_cmpxchg_release(&lock->owner, &owner, new)) in __mutex_handoff()
258 static void __sched __mutex_lock_slowpath(struct mutex *lock);
281 void __sched mutex_lock(struct mutex *lock) in mutex_lock() argument
285 if (!__mutex_trylock_fast(lock)) in mutex_lock()
286 __mutex_lock_slowpath(lock); in mutex_lock()
298 static inline struct task_struct *__mutex_trylock_or_owner(struct mutex *lock) in __mutex_trylock_or_owner() argument
300 return __mutex_trylock_common(lock, false); in __mutex_trylock_or_owner()
304 bool ww_mutex_spin_on_owner(struct mutex *lock, struct ww_acquire_ctx *ww_ctx, in ww_mutex_spin_on_owner() argument
309 ww = container_of(lock, struct ww_mutex, base); in ww_mutex_spin_on_owner()
332 if (!waiter && (atomic_long_read(&lock->owner) & MUTEX_FLAG_WAITERS)) in ww_mutex_spin_on_owner()
339 if (waiter && !__mutex_waiter_is_first(lock, waiter)) in ww_mutex_spin_on_owner()
352 bool mutex_spin_on_owner(struct mutex *lock, struct task_struct *owner, in mutex_spin_on_owner() argument
359 while (__mutex_owner(lock) == owner) { in mutex_spin_on_owner()
378 if (ww_ctx && !ww_mutex_spin_on_owner(lock, ww_ctx, waiter)) { in mutex_spin_on_owner()
392 static inline int mutex_can_spin_on_owner(struct mutex *lock) in mutex_can_spin_on_owner() argument
407 owner = __mutex_owner(lock); in mutex_can_spin_on_owner()
441 mutex_optimistic_spin(struct mutex *lock, struct ww_acquire_ctx *ww_ctx, in mutex_optimistic_spin() argument
452 if (!mutex_can_spin_on_owner(lock)) in mutex_optimistic_spin()
460 if (!osq_lock(&lock->osq)) in mutex_optimistic_spin()
468 owner = __mutex_trylock_or_owner(lock); in mutex_optimistic_spin()
476 if (!mutex_spin_on_owner(lock, owner, ww_ctx, waiter)) in mutex_optimistic_spin()
489 osq_unlock(&lock->osq); in mutex_optimistic_spin()
496 osq_unlock(&lock->osq); in mutex_optimistic_spin()
517 mutex_optimistic_spin(struct mutex *lock, struct ww_acquire_ctx *ww_ctx, in mutex_optimistic_spin() argument
524 static noinline void __sched __mutex_unlock_slowpath(struct mutex *lock, unsigned long ip);
542 void __sched mutex_unlock(struct mutex *lock) in mutex_unlock() argument
545 if (__mutex_unlock_fast(lock)) in mutex_unlock()
548 __mutex_unlock_slowpath(lock, _RET_IP_); in mutex_unlock()
563 void __sched ww_mutex_unlock(struct ww_mutex *lock) in ww_mutex_unlock() argument
565 __ww_mutex_unlock(lock); in ww_mutex_unlock()
566 mutex_unlock(&lock->base); in ww_mutex_unlock()
574 __mutex_lock_common(struct mutex *lock, unsigned int state, unsigned int subclass, in __mutex_lock_common() argument
587 MUTEX_WARN_ON(lock->magic != lock); in __mutex_lock_common()
589 ww = container_of(lock, struct ww_mutex, base); in __mutex_lock_common()
608 mutex_acquire_nest(&lock->dep_map, subclass, 0, nest_lock, ip); in __mutex_lock_common()
610 trace_contention_begin(lock, LCB_F_MUTEX | LCB_F_SPIN); in __mutex_lock_common()
611 if (__mutex_trylock(lock) || in __mutex_lock_common()
612 mutex_optimistic_spin(lock, ww_ctx, NULL)) { in __mutex_lock_common()
614 lock_acquired(&lock->dep_map, ip); in __mutex_lock_common()
617 trace_contention_end(lock, 0); in __mutex_lock_common()
622 raw_spin_lock(&lock->wait_lock); in __mutex_lock_common()
626 if (__mutex_trylock(lock)) { in __mutex_lock_common()
628 __ww_mutex_check_waiters(lock, ww_ctx); in __mutex_lock_common()
633 debug_mutex_lock_common(lock, &waiter); in __mutex_lock_common()
638 lock_contended(&lock->dep_map, ip); in __mutex_lock_common()
642 __mutex_add_waiter(lock, &waiter, &lock->wait_list); in __mutex_lock_common()
648 ret = __ww_mutex_add_waiter(&waiter, lock, ww_ctx); in __mutex_lock_common()
654 trace_contention_begin(lock, LCB_F_MUTEX); in __mutex_lock_common()
664 if (__mutex_trylock(lock)) in __mutex_lock_common()
678 ret = __ww_mutex_check_kill(lock, &waiter, ww_ctx); in __mutex_lock_common()
683 raw_spin_unlock(&lock->wait_lock); in __mutex_lock_common()
686 first = __mutex_waiter_is_first(lock, &waiter); in __mutex_lock_common()
694 if (__mutex_trylock_or_handoff(lock, first)) in __mutex_lock_common()
698 trace_contention_begin(lock, LCB_F_MUTEX | LCB_F_SPIN); in __mutex_lock_common()
699 if (mutex_optimistic_spin(lock, ww_ctx, &waiter)) in __mutex_lock_common()
701 trace_contention_begin(lock, LCB_F_MUTEX); in __mutex_lock_common()
704 raw_spin_lock(&lock->wait_lock); in __mutex_lock_common()
706 raw_spin_lock(&lock->wait_lock); in __mutex_lock_common()
716 !__mutex_waiter_is_first(lock, &waiter)) in __mutex_lock_common()
717 __ww_mutex_check_waiters(lock, ww_ctx); in __mutex_lock_common()
720 __mutex_remove_waiter(lock, &waiter); in __mutex_lock_common()
726 lock_acquired(&lock->dep_map, ip); in __mutex_lock_common()
727 trace_contention_end(lock, 0); in __mutex_lock_common()
732 raw_spin_unlock(&lock->wait_lock); in __mutex_lock_common()
738 __mutex_remove_waiter(lock, &waiter); in __mutex_lock_common()
740 trace_contention_end(lock, ret); in __mutex_lock_common()
741 raw_spin_unlock(&lock->wait_lock); in __mutex_lock_common()
743 mutex_release(&lock->dep_map, ip); in __mutex_lock_common()
749 __mutex_lock(struct mutex *lock, unsigned int state, unsigned int subclass, in __mutex_lock() argument
752 return __mutex_lock_common(lock, state, subclass, nest_lock, ip, NULL, false); in __mutex_lock()
756 __ww_mutex_lock(struct mutex *lock, unsigned int state, unsigned int subclass, in __ww_mutex_lock() argument
759 return __mutex_lock_common(lock, state, subclass, NULL, ip, ww_ctx, true); in __ww_mutex_lock()
802 mutex_lock_nested(struct mutex *lock, unsigned int subclass) in mutex_lock_nested() argument
804 __mutex_lock(lock, TASK_UNINTERRUPTIBLE, subclass, NULL, _RET_IP_); in mutex_lock_nested()
810 _mutex_lock_nest_lock(struct mutex *lock, struct lockdep_map *nest) in _mutex_lock_nest_lock() argument
812 __mutex_lock(lock, TASK_UNINTERRUPTIBLE, 0, nest, _RET_IP_); in _mutex_lock_nest_lock()
817 mutex_lock_killable_nested(struct mutex *lock, unsigned int subclass) in mutex_lock_killable_nested() argument
819 return __mutex_lock(lock, TASK_KILLABLE, subclass, NULL, _RET_IP_); in mutex_lock_killable_nested()
824 mutex_lock_interruptible_nested(struct mutex *lock, unsigned int subclass) in mutex_lock_interruptible_nested() argument
826 return __mutex_lock(lock, TASK_INTERRUPTIBLE, subclass, NULL, _RET_IP_); in mutex_lock_interruptible_nested()
831 mutex_lock_io_nested(struct mutex *lock, unsigned int subclass) in mutex_lock_io_nested() argument
838 __mutex_lock_common(lock, TASK_UNINTERRUPTIBLE, in mutex_lock_io_nested()
845 ww_mutex_deadlock_injection(struct ww_mutex *lock, struct ww_acquire_ctx *ctx) in ww_mutex_deadlock_injection() argument
859 ctx->contending_lock = lock; in ww_mutex_deadlock_injection()
861 ww_mutex_unlock(lock); in ww_mutex_deadlock_injection()
871 ww_mutex_lock(struct ww_mutex *lock, struct ww_acquire_ctx *ctx) in ww_mutex_lock() argument
876 ret = __ww_mutex_lock(&lock->base, TASK_UNINTERRUPTIBLE, in ww_mutex_lock()
879 return ww_mutex_deadlock_injection(lock, ctx); in ww_mutex_lock()
886 ww_mutex_lock_interruptible(struct ww_mutex *lock, struct ww_acquire_ctx *ctx) in ww_mutex_lock_interruptible() argument
891 ret = __ww_mutex_lock(&lock->base, TASK_INTERRUPTIBLE, in ww_mutex_lock_interruptible()
895 return ww_mutex_deadlock_injection(lock, ctx); in ww_mutex_lock_interruptible()
906 static noinline void __sched __mutex_unlock_slowpath(struct mutex *lock, unsigned long ip) in __mutex_unlock_slowpath() argument
912 mutex_release(&lock->dep_map, ip); in __mutex_unlock_slowpath()
921 owner = atomic_long_read(&lock->owner); in __mutex_unlock_slowpath()
929 if (atomic_long_try_cmpxchg_release(&lock->owner, &owner, __owner_flags(owner))) { in __mutex_unlock_slowpath()
937 raw_spin_lock(&lock->wait_lock); in __mutex_unlock_slowpath()
938 debug_mutex_unlock(lock); in __mutex_unlock_slowpath()
939 if (!list_empty(&lock->wait_list)) { in __mutex_unlock_slowpath()
942 list_first_entry(&lock->wait_list, in __mutex_unlock_slowpath()
947 debug_mutex_wake_waiter(lock, waiter); in __mutex_unlock_slowpath()
952 __mutex_handoff(lock, next); in __mutex_unlock_slowpath()
954 raw_spin_unlock(&lock->wait_lock); in __mutex_unlock_slowpath()
965 __mutex_lock_killable_slowpath(struct mutex *lock);
968 __mutex_lock_interruptible_slowpath(struct mutex *lock);
982 int __sched mutex_lock_interruptible(struct mutex *lock) in mutex_lock_interruptible() argument
986 if (__mutex_trylock_fast(lock)) in mutex_lock_interruptible()
989 return __mutex_lock_interruptible_slowpath(lock); in mutex_lock_interruptible()
1006 int __sched mutex_lock_killable(struct mutex *lock) in mutex_lock_killable() argument
1010 if (__mutex_trylock_fast(lock)) in mutex_lock_killable()
1013 return __mutex_lock_killable_slowpath(lock); in mutex_lock_killable()
1027 void __sched mutex_lock_io(struct mutex *lock) in mutex_lock_io() argument
1032 mutex_lock(lock); in mutex_lock_io()
1038 __mutex_lock_slowpath(struct mutex *lock) in __mutex_lock_slowpath() argument
1040 __mutex_lock(lock, TASK_UNINTERRUPTIBLE, 0, NULL, _RET_IP_); in __mutex_lock_slowpath()
1044 __mutex_lock_killable_slowpath(struct mutex *lock) in __mutex_lock_killable_slowpath() argument
1046 return __mutex_lock(lock, TASK_KILLABLE, 0, NULL, _RET_IP_); in __mutex_lock_killable_slowpath()
1050 __mutex_lock_interruptible_slowpath(struct mutex *lock) in __mutex_lock_interruptible_slowpath() argument
1052 return __mutex_lock(lock, TASK_INTERRUPTIBLE, 0, NULL, _RET_IP_); in __mutex_lock_interruptible_slowpath()
1056 __ww_mutex_lock_slowpath(struct ww_mutex *lock, struct ww_acquire_ctx *ctx) in __ww_mutex_lock_slowpath() argument
1058 return __ww_mutex_lock(&lock->base, TASK_UNINTERRUPTIBLE, 0, in __ww_mutex_lock_slowpath()
1063 __ww_mutex_lock_interruptible_slowpath(struct ww_mutex *lock, in __ww_mutex_lock_interruptible_slowpath() argument
1066 return __ww_mutex_lock(&lock->base, TASK_INTERRUPTIBLE, 0, in __ww_mutex_lock_interruptible_slowpath()
1086 int __sched mutex_trylock(struct mutex *lock) in mutex_trylock() argument
1090 MUTEX_WARN_ON(lock->magic != lock); in mutex_trylock()
1092 locked = __mutex_trylock(lock); in mutex_trylock()
1094 mutex_acquire(&lock->dep_map, 0, 1, _RET_IP_); in mutex_trylock()
1102 ww_mutex_lock(struct ww_mutex *lock, struct ww_acquire_ctx *ctx) in ww_mutex_lock() argument
1106 if (__mutex_trylock_fast(&lock->base)) { in ww_mutex_lock()
1108 ww_mutex_set_context_fastpath(lock, ctx); in ww_mutex_lock()
1112 return __ww_mutex_lock_slowpath(lock, ctx); in ww_mutex_lock()
1117 ww_mutex_lock_interruptible(struct ww_mutex *lock, struct ww_acquire_ctx *ctx) in ww_mutex_lock_interruptible() argument
1121 if (__mutex_trylock_fast(&lock->base)) { in ww_mutex_lock_interruptible()
1123 ww_mutex_set_context_fastpath(lock, ctx); in ww_mutex_lock_interruptible()
1127 return __ww_mutex_lock_interruptible_slowpath(lock, ctx); in ww_mutex_lock_interruptible()
1144 int atomic_dec_and_mutex_lock(atomic_t *cnt, struct mutex *lock) in atomic_dec_and_mutex_lock() argument
1150 mutex_lock(lock); in atomic_dec_and_mutex_lock()
1153 mutex_unlock(lock); in atomic_dec_and_mutex_lock()