Lines Matching refs:sem
14 int __percpu_init_rwsem(struct percpu_rw_semaphore *sem, in __percpu_init_rwsem() argument
17 sem->read_count = alloc_percpu(int); in __percpu_init_rwsem()
18 if (unlikely(!sem->read_count)) in __percpu_init_rwsem()
21 rcu_sync_init(&sem->rss); in __percpu_init_rwsem()
22 rcuwait_init(&sem->writer); in __percpu_init_rwsem()
23 init_waitqueue_head(&sem->waiters); in __percpu_init_rwsem()
24 atomic_set(&sem->block, 0); in __percpu_init_rwsem()
26 debug_check_no_locks_freed((void *)sem, sizeof(*sem)); in __percpu_init_rwsem()
27 lockdep_init_map(&sem->dep_map, name, key, 0); in __percpu_init_rwsem()
33 void percpu_free_rwsem(struct percpu_rw_semaphore *sem) in percpu_free_rwsem() argument
39 if (!sem->read_count) in percpu_free_rwsem()
42 rcu_sync_dtor(&sem->rss); in percpu_free_rwsem()
43 free_percpu(sem->read_count); in percpu_free_rwsem()
44 sem->read_count = NULL; /* catch use after free bugs */ in percpu_free_rwsem()
48 static bool __percpu_down_read_trylock(struct percpu_rw_semaphore *sem) in __percpu_down_read_trylock() argument
50 this_cpu_inc(*sem->read_count); in __percpu_down_read_trylock()
73 if (likely(!atomic_read_acquire(&sem->block))) in __percpu_down_read_trylock()
76 this_cpu_dec(*sem->read_count); in __percpu_down_read_trylock()
79 rcuwait_wake_up(&sem->writer); in __percpu_down_read_trylock()
84 static inline bool __percpu_down_write_trylock(struct percpu_rw_semaphore *sem) in __percpu_down_write_trylock() argument
86 if (atomic_read(&sem->block)) in __percpu_down_write_trylock()
89 return atomic_xchg(&sem->block, 1) == 0; in __percpu_down_write_trylock()
92 static bool __percpu_rwsem_trylock(struct percpu_rw_semaphore *sem, bool reader) in __percpu_rwsem_trylock() argument
98 ret = __percpu_down_read_trylock(sem); in __percpu_rwsem_trylock()
103 return __percpu_down_write_trylock(sem); in __percpu_rwsem_trylock()
124 struct percpu_rw_semaphore *sem = key; in percpu_rwsem_wake_function() local
128 if (!__percpu_rwsem_trylock(sem, reader)) in percpu_rwsem_wake_function()
141 static void percpu_rwsem_wait(struct percpu_rw_semaphore *sem, bool reader) in percpu_rwsem_wait() argument
146 spin_lock_irq(&sem->waiters.lock); in percpu_rwsem_wait()
151 wait = !__percpu_rwsem_trylock(sem, reader); in percpu_rwsem_wait()
154 __add_wait_queue_entry_tail(&sem->waiters, &wq_entry); in percpu_rwsem_wait()
156 spin_unlock_irq(&sem->waiters.lock); in percpu_rwsem_wait()
167 bool __sched __percpu_down_read(struct percpu_rw_semaphore *sem, bool try) in __percpu_down_read() argument
169 if (__percpu_down_read_trylock(sem)) in __percpu_down_read()
175 trace_contention_begin(sem, LCB_F_PERCPU | LCB_F_READ); in __percpu_down_read()
177 percpu_rwsem_wait(sem, /* .reader = */ true); in __percpu_down_read()
179 trace_contention_end(sem, 0); in __percpu_down_read()
195 bool percpu_is_read_locked(struct percpu_rw_semaphore *sem) in percpu_is_read_locked() argument
197 return per_cpu_sum(*sem->read_count) != 0 && !atomic_read(&sem->block); in percpu_is_read_locked()
209 static bool readers_active_check(struct percpu_rw_semaphore *sem) in readers_active_check() argument
211 if (per_cpu_sum(*sem->read_count) != 0) in readers_active_check()
224 void __sched percpu_down_write(struct percpu_rw_semaphore *sem) in percpu_down_write() argument
229 rwsem_acquire(&sem->dep_map, 0, 0, _RET_IP_); in percpu_down_write()
232 rcu_sync_enter(&sem->rss); in percpu_down_write()
238 if (!__percpu_down_write_trylock(sem)) { in percpu_down_write()
239 trace_contention_begin(sem, LCB_F_PERCPU | LCB_F_WRITE); in percpu_down_write()
240 percpu_rwsem_wait(sem, /* .reader = */ false); in percpu_down_write()
253 rcuwait_wait_event(&sem->writer, readers_active_check(sem), TASK_UNINTERRUPTIBLE); in percpu_down_write()
255 trace_contention_end(sem, 0); in percpu_down_write()
259 void percpu_up_write(struct percpu_rw_semaphore *sem) in percpu_up_write() argument
261 rwsem_release(&sem->dep_map, _RET_IP_); in percpu_up_write()
273 atomic_set_release(&sem->block, 0); in percpu_up_write()
278 __wake_up(&sem->waiters, TASK_NORMAL, 1, sem); in percpu_up_write()
285 rcu_sync_exit(&sem->rss); in percpu_up_write()