Lines Matching refs:rwb
53 static __always_inline int rwbase_read_trylock(struct rwbase_rt *rwb) in rwbase_read_trylock() argument
61 for (r = atomic_read(&rwb->readers); r < 0;) { in rwbase_read_trylock()
62 if (likely(atomic_try_cmpxchg_acquire(&rwb->readers, &r, r + 1))) in rwbase_read_trylock()
68 static int __sched __rwbase_read_lock(struct rwbase_rt *rwb, in __rwbase_read_lock() argument
71 struct rt_mutex_base *rtm = &rwb->rtmutex; in __rwbase_read_lock()
79 if (atomic_read(&rwb->readers) != WRITER_BIAS) { in __rwbase_read_lock()
80 atomic_inc(&rwb->readers); in __rwbase_read_lock()
115 trace_contention_begin(rwb, LCB_F_RT | LCB_F_READ); in __rwbase_read_lock()
131 atomic_inc(&rwb->readers); in __rwbase_read_lock()
136 trace_contention_end(rwb, ret); in __rwbase_read_lock()
140 static __always_inline int rwbase_read_lock(struct rwbase_rt *rwb, in rwbase_read_lock() argument
143 if (rwbase_read_trylock(rwb)) in rwbase_read_lock()
146 return __rwbase_read_lock(rwb, state); in rwbase_read_lock()
149 static void __sched __rwbase_read_unlock(struct rwbase_rt *rwb, in __rwbase_read_unlock() argument
152 struct rt_mutex_base *rtm = &rwb->rtmutex; in __rwbase_read_unlock()
173 static __always_inline void rwbase_read_unlock(struct rwbase_rt *rwb, in rwbase_read_unlock() argument
182 if (unlikely(atomic_dec_and_test(&rwb->readers))) in rwbase_read_unlock()
183 __rwbase_read_unlock(rwb, state); in rwbase_read_unlock()
186 static inline void __rwbase_write_unlock(struct rwbase_rt *rwb, int bias, in __rwbase_write_unlock() argument
189 struct rt_mutex_base *rtm = &rwb->rtmutex; in __rwbase_write_unlock()
195 (void)atomic_add_return_release(READER_BIAS - bias, &rwb->readers); in __rwbase_write_unlock()
200 static inline void rwbase_write_unlock(struct rwbase_rt *rwb) in rwbase_write_unlock() argument
202 struct rt_mutex_base *rtm = &rwb->rtmutex; in rwbase_write_unlock()
206 __rwbase_write_unlock(rwb, WRITER_BIAS, flags); in rwbase_write_unlock()
209 static inline void rwbase_write_downgrade(struct rwbase_rt *rwb) in rwbase_write_downgrade() argument
211 struct rt_mutex_base *rtm = &rwb->rtmutex; in rwbase_write_downgrade()
216 __rwbase_write_unlock(rwb, WRITER_BIAS - 1, flags); in rwbase_write_downgrade()
219 static inline bool __rwbase_write_trylock(struct rwbase_rt *rwb) in __rwbase_write_trylock() argument
222 lockdep_assert_held(&rwb->rtmutex.wait_lock); in __rwbase_write_trylock()
228 if (!atomic_read_acquire(&rwb->readers)) { in __rwbase_write_trylock()
229 atomic_set(&rwb->readers, WRITER_BIAS); in __rwbase_write_trylock()
236 static int __sched rwbase_write_lock(struct rwbase_rt *rwb, in rwbase_write_lock() argument
239 struct rt_mutex_base *rtm = &rwb->rtmutex; in rwbase_write_lock()
247 atomic_sub(READER_BIAS, &rwb->readers); in rwbase_write_lock()
250 if (__rwbase_write_trylock(rwb)) in rwbase_write_lock()
254 trace_contention_begin(rwb, LCB_F_RT | LCB_F_WRITE); in rwbase_write_lock()
259 __rwbase_write_unlock(rwb, 0, flags); in rwbase_write_lock()
260 trace_contention_end(rwb, -EINTR); in rwbase_write_lock()
264 if (__rwbase_write_trylock(rwb)) in rwbase_write_lock()
274 trace_contention_end(rwb, 0); in rwbase_write_lock()
281 static inline int rwbase_write_trylock(struct rwbase_rt *rwb) in rwbase_write_trylock() argument
283 struct rt_mutex_base *rtm = &rwb->rtmutex; in rwbase_write_trylock()
289 atomic_sub(READER_BIAS, &rwb->readers); in rwbase_write_trylock()
292 if (__rwbase_write_trylock(rwb)) { in rwbase_write_trylock()
296 __rwbase_write_unlock(rwb, 0, flags); in rwbase_write_trylock()