Lines Matching refs:lock
17 struct qspinlock *lock; member
122 static __always_inline u32 trylock_clean_tail(struct qspinlock *lock, u32 tail) in trylock_clean_tail() argument
145 : "r" (&lock->val), "r"(tail), "r" (newval), in trylock_clean_tail()
161 static __always_inline u32 publish_tail_cpu(struct qspinlock *lock, u32 tail) in publish_tail_cpu() argument
175 : "r" (&lock->val), "r" (tail), "r"(_Q_TAIL_CPU_MASK) in publish_tail_cpu()
181 static __always_inline u32 set_mustq(struct qspinlock *lock) in set_mustq() argument
191 : "r" (&lock->val), "r" (_Q_MUST_Q_VAL) in set_mustq()
197 static __always_inline u32 clear_mustq(struct qspinlock *lock) in clear_mustq() argument
207 : "r" (&lock->val), "r" (_Q_MUST_Q_VAL) in clear_mustq()
213 static __always_inline bool try_set_sleepy(struct qspinlock *lock, u32 old) in try_set_sleepy() argument
229 : "r" (&lock->val), "r"(old), "r" (new) in try_set_sleepy()
235 static __always_inline void seen_sleepy_owner(struct qspinlock *lock, u32 val) in seen_sleepy_owner() argument
241 try_set_sleepy(lock, val); in seen_sleepy_owner()
260 static struct qnode *get_tail_qnode(struct qspinlock *lock, int prev_cpu) in get_tail_qnode() argument
276 if (qnode->lock == lock) in get_tail_qnode()
284 static __always_inline bool __yield_to_locked_owner(struct qspinlock *lock, u32 val, bool paravirt,… in __yield_to_locked_owner() argument
306 seen_sleepy_owner(lock, val); in __yield_to_locked_owner()
318 if (READ_ONCE(lock->val) == val) { in __yield_to_locked_owner()
320 clear_mustq(lock); in __yield_to_locked_owner()
323 set_mustq(lock); in __yield_to_locked_owner()
337 static __always_inline bool yield_to_locked_owner(struct qspinlock *lock, u32 val, bool paravirt) in yield_to_locked_owner() argument
339 return __yield_to_locked_owner(lock, val, paravirt, false); in yield_to_locked_owner()
343 static __always_inline bool yield_head_to_locked_owner(struct qspinlock *lock, u32 val, bool paravi… in yield_head_to_locked_owner() argument
350 return __yield_to_locked_owner(lock, val, paravirt, mustq); in yield_head_to_locked_owner()
376 static __always_inline bool yield_to_prev(struct qspinlock *lock, struct qnode *node, int prev_cpu,… in yield_to_prev() argument
392 u32 val = READ_ONCE(lock->val); in yield_to_prev()
407 preempted = yield_to_locked_owner(lock, val, paravirt); in yield_to_prev()
456 static __always_inline bool try_to_steal_lock(struct qspinlock *lock, bool paravirt) in try_to_steal_lock() argument
473 val = READ_ONCE(lock->val); in try_to_steal_lock()
480 if (__queued_spin_trylock_steal(lock)) in try_to_steal_lock()
484 preempted = yield_to_locked_owner(lock, val, paravirt); in try_to_steal_lock()
498 if (try_set_sleepy(lock, val)) in try_to_steal_lock()
526 static __always_inline void queued_spin_lock_mcs_queue(struct qspinlock *lock, bool paravirt) in queued_spin_lock_mcs_queue() argument
542 while (!queued_spin_trylock(lock)) in queued_spin_lock_mcs_queue()
556 node->lock = lock; in queued_spin_lock_mcs_queue()
568 old = publish_tail_cpu(lock, tail); in queued_spin_lock_mcs_queue()
576 struct qnode *prev = get_tail_qnode(lock, prev_cpu); in queued_spin_lock_mcs_queue()
586 if (yield_to_prev(lock, node, prev_cpu, paravirt)) in queued_spin_lock_mcs_queue()
612 val = READ_ONCE(lock->val); in queued_spin_lock_mcs_queue()
628 if (try_set_sleepy(lock, val)) in queued_spin_lock_mcs_queue()
634 preempted = yield_head_to_locked_owner(lock, val, paravirt); in queued_spin_lock_mcs_queue()
652 set_mustq(lock); in queued_spin_lock_mcs_queue()
660 old = trylock_clean_tail(lock, tail); in queued_spin_lock_mcs_queue()
706 node->lock = NULL; in queued_spin_lock_mcs_queue()
711 void queued_spin_lock_slowpath(struct qspinlock *lock) in queued_spin_lock_slowpath() argument
719 if (try_to_steal_lock(lock, true)) { in queued_spin_lock_slowpath()
723 queued_spin_lock_mcs_queue(lock, true); in queued_spin_lock_slowpath()
725 if (try_to_steal_lock(lock, false)) { in queued_spin_lock_slowpath()
729 queued_spin_lock_mcs_queue(lock, false); in queued_spin_lock_slowpath()
747 static DEFINE_MUTEX(lock); in steal_spins_set()
756 mutex_lock(&lock); in steal_spins_set()
770 mutex_unlock(&lock); in steal_spins_set()