Lines Matching refs:rsp
21 void rcu_sync_init(struct rcu_sync *rsp) in rcu_sync_init() argument
23 memset(rsp, 0, sizeof(*rsp)); in rcu_sync_init()
24 init_waitqueue_head(&rsp->gp_wait); in rcu_sync_init()
29 static void rcu_sync_call(struct rcu_sync *rsp) in rcu_sync_call() argument
31 call_rcu_hurry(&rsp->cb_head, rcu_sync_func); in rcu_sync_call()
59 struct rcu_sync *rsp = container_of(rhp, struct rcu_sync, cb_head); in rcu_sync_func() local
62 WARN_ON_ONCE(READ_ONCE(rsp->gp_state) == GP_IDLE); in rcu_sync_func()
63 WARN_ON_ONCE(READ_ONCE(rsp->gp_state) == GP_PASSED); in rcu_sync_func()
65 spin_lock_irqsave(&rsp->rss_lock, flags); in rcu_sync_func()
66 if (rsp->gp_count) { in rcu_sync_func()
70 WRITE_ONCE(rsp->gp_state, GP_PASSED); in rcu_sync_func()
71 wake_up_locked(&rsp->gp_wait); in rcu_sync_func()
72 } else if (rsp->gp_state == GP_REPLAY) { in rcu_sync_func()
77 WRITE_ONCE(rsp->gp_state, GP_EXIT); in rcu_sync_func()
78 rcu_sync_call(rsp); in rcu_sync_func()
85 WRITE_ONCE(rsp->gp_state, GP_IDLE); in rcu_sync_func()
87 spin_unlock_irqrestore(&rsp->rss_lock, flags); in rcu_sync_func()
105 void rcu_sync_enter(struct rcu_sync *rsp) in rcu_sync_enter() argument
109 spin_lock_irq(&rsp->rss_lock); in rcu_sync_enter()
110 gp_state = rsp->gp_state; in rcu_sync_enter()
112 WRITE_ONCE(rsp->gp_state, GP_ENTER); in rcu_sync_enter()
113 WARN_ON_ONCE(rsp->gp_count); in rcu_sync_enter()
125 rsp->gp_count++; in rcu_sync_enter()
126 spin_unlock_irq(&rsp->rss_lock); in rcu_sync_enter()
134 rcu_sync_func(&rsp->cb_head); in rcu_sync_enter()
139 wait_event(rsp->gp_wait, READ_ONCE(rsp->gp_state) >= GP_PASSED); in rcu_sync_enter()
152 void rcu_sync_exit(struct rcu_sync *rsp) in rcu_sync_exit() argument
154 WARN_ON_ONCE(READ_ONCE(rsp->gp_state) == GP_IDLE); in rcu_sync_exit()
156 spin_lock_irq(&rsp->rss_lock); in rcu_sync_exit()
157 WARN_ON_ONCE(rsp->gp_count == 0); in rcu_sync_exit()
158 if (!--rsp->gp_count) { in rcu_sync_exit()
159 if (rsp->gp_state == GP_PASSED) { in rcu_sync_exit()
160 WRITE_ONCE(rsp->gp_state, GP_EXIT); in rcu_sync_exit()
161 rcu_sync_call(rsp); in rcu_sync_exit()
162 } else if (rsp->gp_state == GP_EXIT) { in rcu_sync_exit()
163 WRITE_ONCE(rsp->gp_state, GP_REPLAY); in rcu_sync_exit()
166 spin_unlock_irq(&rsp->rss_lock); in rcu_sync_exit()
173 void rcu_sync_dtor(struct rcu_sync *rsp) in rcu_sync_dtor() argument
177 WARN_ON_ONCE(READ_ONCE(rsp->gp_state) == GP_PASSED); in rcu_sync_dtor()
179 spin_lock_irq(&rsp->rss_lock); in rcu_sync_dtor()
180 WARN_ON_ONCE(rsp->gp_count); in rcu_sync_dtor()
181 if (rsp->gp_state == GP_REPLAY) in rcu_sync_dtor()
182 WRITE_ONCE(rsp->gp_state, GP_EXIT); in rcu_sync_dtor()
183 gp_state = rsp->gp_state; in rcu_sync_dtor()
184 spin_unlock_irq(&rsp->rss_lock); in rcu_sync_dtor()
188 WARN_ON_ONCE(rsp->gp_state != GP_IDLE); in rcu_sync_dtor()