Lines Matching refs:rsp
21 void rcu_sync_init(struct rcu_sync *rsp) in rcu_sync_init() argument
23 memset(rsp, 0, sizeof(*rsp)); in rcu_sync_init()
24 init_waitqueue_head(&rsp->gp_wait); in rcu_sync_init()
36 void rcu_sync_enter_start(struct rcu_sync *rsp) in rcu_sync_enter_start() argument
38 rsp->gp_count++; in rcu_sync_enter_start()
39 rsp->gp_state = GP_PASSED; in rcu_sync_enter_start()
45 static void rcu_sync_call(struct rcu_sync *rsp) in rcu_sync_call() argument
47 call_rcu_hurry(&rsp->cb_head, rcu_sync_func); in rcu_sync_call()
75 struct rcu_sync *rsp = container_of(rhp, struct rcu_sync, cb_head); in rcu_sync_func() local
78 WARN_ON_ONCE(READ_ONCE(rsp->gp_state) == GP_IDLE); in rcu_sync_func()
79 WARN_ON_ONCE(READ_ONCE(rsp->gp_state) == GP_PASSED); in rcu_sync_func()
81 spin_lock_irqsave(&rsp->rss_lock, flags); in rcu_sync_func()
82 if (rsp->gp_count) { in rcu_sync_func()
86 WRITE_ONCE(rsp->gp_state, GP_PASSED); in rcu_sync_func()
87 wake_up_locked(&rsp->gp_wait); in rcu_sync_func()
88 } else if (rsp->gp_state == GP_REPLAY) { in rcu_sync_func()
93 WRITE_ONCE(rsp->gp_state, GP_EXIT); in rcu_sync_func()
94 rcu_sync_call(rsp); in rcu_sync_func()
101 WRITE_ONCE(rsp->gp_state, GP_IDLE); in rcu_sync_func()
103 spin_unlock_irqrestore(&rsp->rss_lock, flags); in rcu_sync_func()
121 void rcu_sync_enter(struct rcu_sync *rsp) in rcu_sync_enter() argument
125 spin_lock_irq(&rsp->rss_lock); in rcu_sync_enter()
126 gp_state = rsp->gp_state; in rcu_sync_enter()
128 WRITE_ONCE(rsp->gp_state, GP_ENTER); in rcu_sync_enter()
129 WARN_ON_ONCE(rsp->gp_count); in rcu_sync_enter()
141 rsp->gp_count++; in rcu_sync_enter()
142 spin_unlock_irq(&rsp->rss_lock); in rcu_sync_enter()
150 rcu_sync_func(&rsp->cb_head); in rcu_sync_enter()
155 wait_event(rsp->gp_wait, READ_ONCE(rsp->gp_state) >= GP_PASSED); in rcu_sync_enter()
168 void rcu_sync_exit(struct rcu_sync *rsp) in rcu_sync_exit() argument
170 WARN_ON_ONCE(READ_ONCE(rsp->gp_state) == GP_IDLE); in rcu_sync_exit()
171 WARN_ON_ONCE(READ_ONCE(rsp->gp_count) == 0); in rcu_sync_exit()
173 spin_lock_irq(&rsp->rss_lock); in rcu_sync_exit()
174 if (!--rsp->gp_count) { in rcu_sync_exit()
175 if (rsp->gp_state == GP_PASSED) { in rcu_sync_exit()
176 WRITE_ONCE(rsp->gp_state, GP_EXIT); in rcu_sync_exit()
177 rcu_sync_call(rsp); in rcu_sync_exit()
178 } else if (rsp->gp_state == GP_EXIT) { in rcu_sync_exit()
179 WRITE_ONCE(rsp->gp_state, GP_REPLAY); in rcu_sync_exit()
182 spin_unlock_irq(&rsp->rss_lock); in rcu_sync_exit()
189 void rcu_sync_dtor(struct rcu_sync *rsp) in rcu_sync_dtor() argument
193 WARN_ON_ONCE(READ_ONCE(rsp->gp_count)); in rcu_sync_dtor()
194 WARN_ON_ONCE(READ_ONCE(rsp->gp_state) == GP_PASSED); in rcu_sync_dtor()
196 spin_lock_irq(&rsp->rss_lock); in rcu_sync_dtor()
197 if (rsp->gp_state == GP_REPLAY) in rcu_sync_dtor()
198 WRITE_ONCE(rsp->gp_state, GP_EXIT); in rcu_sync_dtor()
199 gp_state = rsp->gp_state; in rcu_sync_dtor()
200 spin_unlock_irq(&rsp->rss_lock); in rcu_sync_dtor()
204 WARN_ON_ONCE(rsp->gp_state != GP_IDLE); in rcu_sync_dtor()