Lines Matching refs:ssp
76 static void srcu_reschedule(struct srcu_struct *ssp, unsigned long delay);
122 static void init_srcu_struct_data(struct srcu_struct *ssp) in init_srcu_struct_data() argument
134 sdp = per_cpu_ptr(ssp->sda, cpu); in init_srcu_struct_data()
138 sdp->srcu_gp_seq_needed = ssp->srcu_gp_seq; in init_srcu_struct_data()
139 sdp->srcu_gp_seq_needed_exp = ssp->srcu_gp_seq; in init_srcu_struct_data()
144 sdp->ssp = ssp; in init_srcu_struct_data()
164 static bool init_srcu_struct_nodes(struct srcu_struct *ssp, gfp_t gfp_flags) in init_srcu_struct_nodes() argument
176 ssp->node = kcalloc(rcu_num_nodes, sizeof(*ssp->node), gfp_flags); in init_srcu_struct_nodes()
177 if (!ssp->node) in init_srcu_struct_nodes()
181 ssp->level[0] = &ssp->node[0]; in init_srcu_struct_nodes()
183 ssp->level[i] = ssp->level[i - 1] + num_rcu_lvl[i - 1]; in init_srcu_struct_nodes()
187 srcu_for_each_node_breadth_first(ssp, snp) { in init_srcu_struct_nodes()
198 if (snp == &ssp->node[0]) { in init_srcu_struct_nodes()
205 if (snp == ssp->level[level + 1]) in init_srcu_struct_nodes()
207 snp->srcu_parent = ssp->level[level - 1] + in init_srcu_struct_nodes()
208 (snp - ssp->level[level]) / in init_srcu_struct_nodes()
217 snp_first = ssp->level[level]; in init_srcu_struct_nodes()
219 sdp = per_cpu_ptr(ssp->sda, cpu); in init_srcu_struct_nodes()
228 smp_store_release(&ssp->srcu_size_state, SRCU_SIZE_WAIT_BARRIER); in init_srcu_struct_nodes()
237 static int init_srcu_struct_fields(struct srcu_struct *ssp, bool is_static) in init_srcu_struct_fields() argument
239 ssp->srcu_size_state = SRCU_SIZE_SMALL; in init_srcu_struct_fields()
240 ssp->node = NULL; in init_srcu_struct_fields()
241 mutex_init(&ssp->srcu_cb_mutex); in init_srcu_struct_fields()
242 mutex_init(&ssp->srcu_gp_mutex); in init_srcu_struct_fields()
243 ssp->srcu_idx = 0; in init_srcu_struct_fields()
244 ssp->srcu_gp_seq = 0; in init_srcu_struct_fields()
245 ssp->srcu_barrier_seq = 0; in init_srcu_struct_fields()
246 mutex_init(&ssp->srcu_barrier_mutex); in init_srcu_struct_fields()
247 atomic_set(&ssp->srcu_barrier_cpu_cnt, 0); in init_srcu_struct_fields()
248 INIT_DELAYED_WORK(&ssp->work, process_srcu); in init_srcu_struct_fields()
249 ssp->sda_is_static = is_static; in init_srcu_struct_fields()
251 ssp->sda = alloc_percpu(struct srcu_data); in init_srcu_struct_fields()
252 if (!ssp->sda) in init_srcu_struct_fields()
254 init_srcu_struct_data(ssp); in init_srcu_struct_fields()
255 ssp->srcu_gp_seq_needed_exp = 0; in init_srcu_struct_fields()
256 ssp->srcu_last_gp_end = ktime_get_mono_fast_ns(); in init_srcu_struct_fields()
257 if (READ_ONCE(ssp->srcu_size_state) == SRCU_SIZE_SMALL && SRCU_SIZING_IS_INIT()) { in init_srcu_struct_fields()
258 if (!init_srcu_struct_nodes(ssp, GFP_ATOMIC)) { in init_srcu_struct_fields()
259 if (!ssp->sda_is_static) { in init_srcu_struct_fields()
260 free_percpu(ssp->sda); in init_srcu_struct_fields()
261 ssp->sda = NULL; in init_srcu_struct_fields()
265 WRITE_ONCE(ssp->srcu_size_state, SRCU_SIZE_BIG); in init_srcu_struct_fields()
268 smp_store_release(&ssp->srcu_gp_seq_needed, 0); /* Init done. */ in init_srcu_struct_fields()
274 int __init_srcu_struct(struct srcu_struct *ssp, const char *name, in __init_srcu_struct() argument
278 debug_check_no_locks_freed((void *)ssp, sizeof(*ssp)); in __init_srcu_struct()
279 lockdep_init_map(&ssp->dep_map, name, key, 0); in __init_srcu_struct()
280 spin_lock_init(&ACCESS_PRIVATE(ssp, lock)); in __init_srcu_struct()
281 return init_srcu_struct_fields(ssp, false); in __init_srcu_struct()
295 int init_srcu_struct(struct srcu_struct *ssp) in init_srcu_struct() argument
297 spin_lock_init(&ACCESS_PRIVATE(ssp, lock)); in init_srcu_struct()
298 return init_srcu_struct_fields(ssp, false); in init_srcu_struct()
307 static void __srcu_transition_to_big(struct srcu_struct *ssp) in __srcu_transition_to_big() argument
309 lockdep_assert_held(&ACCESS_PRIVATE(ssp, lock)); in __srcu_transition_to_big()
310 smp_store_release(&ssp->srcu_size_state, SRCU_SIZE_ALLOC); in __srcu_transition_to_big()
316 static void srcu_transition_to_big(struct srcu_struct *ssp) in srcu_transition_to_big() argument
321 if (smp_load_acquire(&ssp->srcu_size_state) != SRCU_SIZE_SMALL) in srcu_transition_to_big()
323 spin_lock_irqsave_rcu_node(ssp, flags); in srcu_transition_to_big()
324 if (smp_load_acquire(&ssp->srcu_size_state) != SRCU_SIZE_SMALL) { in srcu_transition_to_big()
325 spin_unlock_irqrestore_rcu_node(ssp, flags); in srcu_transition_to_big()
328 __srcu_transition_to_big(ssp); in srcu_transition_to_big()
329 spin_unlock_irqrestore_rcu_node(ssp, flags); in srcu_transition_to_big()
336 static void spin_lock_irqsave_check_contention(struct srcu_struct *ssp) in spin_lock_irqsave_check_contention() argument
340 if (!SRCU_SIZING_IS_CONTEND() || ssp->srcu_size_state) in spin_lock_irqsave_check_contention()
343 if (ssp->srcu_size_jiffies != j) { in spin_lock_irqsave_check_contention()
344 ssp->srcu_size_jiffies = j; in spin_lock_irqsave_check_contention()
345 ssp->srcu_n_lock_retries = 0; in spin_lock_irqsave_check_contention()
347 if (++ssp->srcu_n_lock_retries <= small_contention_lim) in spin_lock_irqsave_check_contention()
349 __srcu_transition_to_big(ssp); in spin_lock_irqsave_check_contention()
360 struct srcu_struct *ssp = sdp->ssp; in spin_lock_irqsave_sdp_contention() local
364 spin_lock_irqsave_rcu_node(ssp, *flags); in spin_lock_irqsave_sdp_contention()
365 spin_lock_irqsave_check_contention(ssp); in spin_lock_irqsave_sdp_contention()
366 spin_unlock_irqrestore_rcu_node(ssp, *flags); in spin_lock_irqsave_sdp_contention()
376 static void spin_lock_irqsave_ssp_contention(struct srcu_struct *ssp, unsigned long *flags) in spin_lock_irqsave_ssp_contention() argument
378 if (spin_trylock_irqsave_rcu_node(ssp, *flags)) in spin_lock_irqsave_ssp_contention()
380 spin_lock_irqsave_rcu_node(ssp, *flags); in spin_lock_irqsave_ssp_contention()
381 spin_lock_irqsave_check_contention(ssp); in spin_lock_irqsave_ssp_contention()
392 static void check_init_srcu_struct(struct srcu_struct *ssp) in check_init_srcu_struct() argument
397 if (!rcu_seq_state(smp_load_acquire(&ssp->srcu_gp_seq_needed))) /*^^^*/ in check_init_srcu_struct()
399 spin_lock_irqsave_rcu_node(ssp, flags); in check_init_srcu_struct()
400 if (!rcu_seq_state(ssp->srcu_gp_seq_needed)) { in check_init_srcu_struct()
401 spin_unlock_irqrestore_rcu_node(ssp, flags); in check_init_srcu_struct()
404 init_srcu_struct_fields(ssp, true); in check_init_srcu_struct()
405 spin_unlock_irqrestore_rcu_node(ssp, flags); in check_init_srcu_struct()
412 static unsigned long srcu_readers_lock_idx(struct srcu_struct *ssp, int idx) in srcu_readers_lock_idx() argument
418 struct srcu_data *cpuc = per_cpu_ptr(ssp->sda, cpu); in srcu_readers_lock_idx()
429 static unsigned long srcu_readers_unlock_idx(struct srcu_struct *ssp, int idx) in srcu_readers_unlock_idx() argument
436 struct srcu_data *cpuc = per_cpu_ptr(ssp->sda, cpu); in srcu_readers_unlock_idx()
443 "Mixed NMI-safe readers for srcu_struct at %ps.\n", ssp); in srcu_readers_unlock_idx()
451 static bool srcu_readers_active_idx_check(struct srcu_struct *ssp, int idx) in srcu_readers_active_idx_check() argument
455 unlocks = srcu_readers_unlock_idx(ssp, idx); in srcu_readers_active_idx_check()
526 return srcu_readers_lock_idx(ssp, idx) == unlocks; in srcu_readers_active_idx_check()
538 static bool srcu_readers_active(struct srcu_struct *ssp) in srcu_readers_active() argument
544 struct srcu_data *cpuc = per_cpu_ptr(ssp->sda, cpu); in srcu_readers_active()
605 static unsigned long srcu_get_delay(struct srcu_struct *ssp) in srcu_get_delay() argument
611 if (ULONG_CMP_LT(READ_ONCE(ssp->srcu_gp_seq), READ_ONCE(ssp->srcu_gp_seq_needed_exp))) in srcu_get_delay()
613 if (rcu_seq_state(READ_ONCE(ssp->srcu_gp_seq))) { in srcu_get_delay()
615 gpstart = READ_ONCE(ssp->srcu_gp_start); in srcu_get_delay()
619 WRITE_ONCE(ssp->srcu_n_exp_nodelay, READ_ONCE(ssp->srcu_n_exp_nodelay) + 1); in srcu_get_delay()
620 if (READ_ONCE(ssp->srcu_n_exp_nodelay) > srcu_max_nodelay_phase) in srcu_get_delay()
634 void cleanup_srcu_struct(struct srcu_struct *ssp) in cleanup_srcu_struct() argument
638 if (WARN_ON(!srcu_get_delay(ssp))) in cleanup_srcu_struct()
640 if (WARN_ON(srcu_readers_active(ssp))) in cleanup_srcu_struct()
642 flush_delayed_work(&ssp->work); in cleanup_srcu_struct()
644 struct srcu_data *sdp = per_cpu_ptr(ssp->sda, cpu); in cleanup_srcu_struct()
651 if (WARN_ON(rcu_seq_state(READ_ONCE(ssp->srcu_gp_seq)) != SRCU_STATE_IDLE) || in cleanup_srcu_struct()
652 WARN_ON(rcu_seq_current(&ssp->srcu_gp_seq) != ssp->srcu_gp_seq_needed) || in cleanup_srcu_struct()
653 WARN_ON(srcu_readers_active(ssp))) { in cleanup_srcu_struct()
655 __func__, ssp, rcu_seq_state(READ_ONCE(ssp->srcu_gp_seq)), in cleanup_srcu_struct()
656 rcu_seq_current(&ssp->srcu_gp_seq), ssp->srcu_gp_seq_needed); in cleanup_srcu_struct()
659 if (!ssp->sda_is_static) { in cleanup_srcu_struct()
660 free_percpu(ssp->sda); in cleanup_srcu_struct()
661 ssp->sda = NULL; in cleanup_srcu_struct()
663 kfree(ssp->node); in cleanup_srcu_struct()
664 ssp->node = NULL; in cleanup_srcu_struct()
665 ssp->srcu_size_state = SRCU_SIZE_SMALL; in cleanup_srcu_struct()
673 void srcu_check_nmi_safety(struct srcu_struct *ssp, bool nmi_safe) in srcu_check_nmi_safety() argument
681 sdp = raw_cpu_ptr(ssp->sda); in srcu_check_nmi_safety()
697 int __srcu_read_lock(struct srcu_struct *ssp) in __srcu_read_lock() argument
701 idx = READ_ONCE(ssp->srcu_idx) & 0x1; in __srcu_read_lock()
702 this_cpu_inc(ssp->sda->srcu_lock_count[idx].counter); in __srcu_read_lock()
713 void __srcu_read_unlock(struct srcu_struct *ssp, int idx) in __srcu_read_unlock() argument
716 this_cpu_inc(ssp->sda->srcu_unlock_count[idx].counter); in __srcu_read_unlock()
727 int __srcu_read_lock_nmisafe(struct srcu_struct *ssp) in __srcu_read_lock_nmisafe() argument
730 struct srcu_data *sdp = raw_cpu_ptr(ssp->sda); in __srcu_read_lock_nmisafe()
732 idx = READ_ONCE(ssp->srcu_idx) & 0x1; in __srcu_read_lock_nmisafe()
744 void __srcu_read_unlock_nmisafe(struct srcu_struct *ssp, int idx) in __srcu_read_unlock_nmisafe() argument
746 struct srcu_data *sdp = raw_cpu_ptr(ssp->sda); in __srcu_read_unlock_nmisafe()
758 static void srcu_gp_start(struct srcu_struct *ssp) in srcu_gp_start() argument
763 if (smp_load_acquire(&ssp->srcu_size_state) < SRCU_SIZE_WAIT_BARRIER) in srcu_gp_start()
764 sdp = per_cpu_ptr(ssp->sda, get_boot_cpu_id()); in srcu_gp_start()
766 sdp = this_cpu_ptr(ssp->sda); in srcu_gp_start()
767 lockdep_assert_held(&ACCESS_PRIVATE(ssp, lock)); in srcu_gp_start()
768 WARN_ON_ONCE(ULONG_CMP_GE(ssp->srcu_gp_seq, ssp->srcu_gp_seq_needed)); in srcu_gp_start()
771 rcu_seq_current(&ssp->srcu_gp_seq)); in srcu_gp_start()
773 rcu_seq_snap(&ssp->srcu_gp_seq)); in srcu_gp_start()
775 WRITE_ONCE(ssp->srcu_gp_start, jiffies); in srcu_gp_start()
776 WRITE_ONCE(ssp->srcu_n_exp_nodelay, 0); in srcu_gp_start()
778 rcu_seq_start(&ssp->srcu_gp_seq); in srcu_gp_start()
779 state = rcu_seq_state(ssp->srcu_gp_seq); in srcu_gp_start()
817 static void srcu_schedule_cbs_snp(struct srcu_struct *ssp, struct srcu_node *snp, in srcu_schedule_cbs_snp() argument
825 srcu_schedule_cbs_sdp(per_cpu_ptr(ssp->sda, cpu), delay); in srcu_schedule_cbs_snp()
838 static void srcu_gp_end(struct srcu_struct *ssp) in srcu_gp_end() argument
854 mutex_lock(&ssp->srcu_cb_mutex); in srcu_gp_end()
857 spin_lock_irq_rcu_node(ssp); in srcu_gp_end()
858 idx = rcu_seq_state(ssp->srcu_gp_seq); in srcu_gp_end()
860 if (ULONG_CMP_LT(READ_ONCE(ssp->srcu_gp_seq), READ_ONCE(ssp->srcu_gp_seq_needed_exp))) in srcu_gp_end()
863 WRITE_ONCE(ssp->srcu_last_gp_end, ktime_get_mono_fast_ns()); in srcu_gp_end()
864 rcu_seq_end(&ssp->srcu_gp_seq); in srcu_gp_end()
865 gpseq = rcu_seq_current(&ssp->srcu_gp_seq); in srcu_gp_end()
866 if (ULONG_CMP_LT(ssp->srcu_gp_seq_needed_exp, gpseq)) in srcu_gp_end()
867 WRITE_ONCE(ssp->srcu_gp_seq_needed_exp, gpseq); in srcu_gp_end()
868 spin_unlock_irq_rcu_node(ssp); in srcu_gp_end()
869 mutex_unlock(&ssp->srcu_gp_mutex); in srcu_gp_end()
873 ss_state = smp_load_acquire(&ssp->srcu_size_state); in srcu_gp_end()
875 srcu_schedule_cbs_sdp(per_cpu_ptr(ssp->sda, get_boot_cpu_id()), in srcu_gp_end()
879 srcu_for_each_node_breadth_first(ssp, snp) { in srcu_gp_end()
882 last_lvl = snp >= ssp->level[rcu_num_lvls - 1]; in srcu_gp_end()
897 srcu_schedule_cbs_snp(ssp, snp, mask, cbdelay); in srcu_gp_end()
904 sdp = per_cpu_ptr(ssp->sda, cpu); in srcu_gp_end()
914 mutex_unlock(&ssp->srcu_cb_mutex); in srcu_gp_end()
917 spin_lock_irq_rcu_node(ssp); in srcu_gp_end()
918 gpseq = rcu_seq_current(&ssp->srcu_gp_seq); in srcu_gp_end()
920 ULONG_CMP_LT(gpseq, ssp->srcu_gp_seq_needed)) { in srcu_gp_end()
921 srcu_gp_start(ssp); in srcu_gp_end()
922 spin_unlock_irq_rcu_node(ssp); in srcu_gp_end()
923 srcu_reschedule(ssp, 0); in srcu_gp_end()
925 spin_unlock_irq_rcu_node(ssp); in srcu_gp_end()
931 init_srcu_struct_nodes(ssp, GFP_KERNEL); in srcu_gp_end()
933 smp_store_release(&ssp->srcu_size_state, ss_state + 1); in srcu_gp_end()
944 static void srcu_funnel_exp_start(struct srcu_struct *ssp, struct srcu_node *snp, in srcu_funnel_exp_start() argument
953 if (WARN_ON_ONCE(rcu_seq_done(&ssp->srcu_gp_seq, s)) || in srcu_funnel_exp_start()
965 spin_lock_irqsave_ssp_contention(ssp, &flags); in srcu_funnel_exp_start()
966 if (ULONG_CMP_LT(ssp->srcu_gp_seq_needed_exp, s)) in srcu_funnel_exp_start()
967 WRITE_ONCE(ssp->srcu_gp_seq_needed_exp, s); in srcu_funnel_exp_start()
968 spin_unlock_irqrestore_rcu_node(ssp, flags); in srcu_funnel_exp_start()
984 static void srcu_funnel_gp_start(struct srcu_struct *ssp, struct srcu_data *sdp, in srcu_funnel_gp_start() argument
995 if (smp_load_acquire(&ssp->srcu_size_state) < SRCU_SIZE_WAIT_BARRIER) in srcu_funnel_gp_start()
1003 if (WARN_ON_ONCE(rcu_seq_done(&ssp->srcu_gp_seq, s)) && snp != snp_leaf) in srcu_funnel_gp_start()
1016 srcu_funnel_exp_start(ssp, snp, s); in srcu_funnel_gp_start()
1029 spin_lock_irqsave_ssp_contention(ssp, &flags); in srcu_funnel_gp_start()
1030 if (ULONG_CMP_LT(ssp->srcu_gp_seq_needed, s)) { in srcu_funnel_gp_start()
1035 smp_store_release(&ssp->srcu_gp_seq_needed, s); /*^^^*/ in srcu_funnel_gp_start()
1037 if (!do_norm && ULONG_CMP_LT(ssp->srcu_gp_seq_needed_exp, s)) in srcu_funnel_gp_start()
1038 WRITE_ONCE(ssp->srcu_gp_seq_needed_exp, s); in srcu_funnel_gp_start()
1041 if (!WARN_ON_ONCE(rcu_seq_done(&ssp->srcu_gp_seq, s)) && in srcu_funnel_gp_start()
1042 rcu_seq_state(ssp->srcu_gp_seq) == SRCU_STATE_IDLE) { in srcu_funnel_gp_start()
1043 WARN_ON_ONCE(ULONG_CMP_GE(ssp->srcu_gp_seq, ssp->srcu_gp_seq_needed)); in srcu_funnel_gp_start()
1044 srcu_gp_start(ssp); in srcu_funnel_gp_start()
1052 queue_delayed_work(rcu_gp_wq, &ssp->work, in srcu_funnel_gp_start()
1053 !!srcu_get_delay(ssp)); in srcu_funnel_gp_start()
1054 else if (list_empty(&ssp->work.work.entry)) in srcu_funnel_gp_start()
1055 list_add(&ssp->work.work.entry, &srcu_boot_list); in srcu_funnel_gp_start()
1057 spin_unlock_irqrestore_rcu_node(ssp, flags); in srcu_funnel_gp_start()
1065 static bool try_check_zero(struct srcu_struct *ssp, int idx, int trycount) in try_check_zero() argument
1069 curdelay = !srcu_get_delay(ssp); in try_check_zero()
1072 if (srcu_readers_active_idx_check(ssp, idx)) in try_check_zero()
1085 static void srcu_flip(struct srcu_struct *ssp) in srcu_flip() argument
1097 WRITE_ONCE(ssp->srcu_idx, ssp->srcu_idx + 1); in srcu_flip()
1131 static bool srcu_might_be_idle(struct srcu_struct *ssp) in srcu_might_be_idle() argument
1139 check_init_srcu_struct(ssp); in srcu_might_be_idle()
1141 sdp = raw_cpu_ptr(ssp->sda); in srcu_might_be_idle()
1157 tlast = READ_ONCE(ssp->srcu_last_gp_end); in srcu_might_be_idle()
1163 curseq = rcu_seq_current(&ssp->srcu_gp_seq); in srcu_might_be_idle()
1165 if (ULONG_CMP_LT(curseq, READ_ONCE(ssp->srcu_gp_seq_needed))) in srcu_might_be_idle()
1168 if (curseq != rcu_seq_current(&ssp->srcu_gp_seq)) in srcu_might_be_idle()
1183 static unsigned long srcu_gp_start_if_needed(struct srcu_struct *ssp, in srcu_gp_start_if_needed() argument
1195 check_init_srcu_struct(ssp); in srcu_gp_start_if_needed()
1201 idx = __srcu_read_lock_nmisafe(ssp); in srcu_gp_start_if_needed()
1202 ss_state = smp_load_acquire(&ssp->srcu_size_state); in srcu_gp_start_if_needed()
1204 sdp = per_cpu_ptr(ssp->sda, get_boot_cpu_id()); in srcu_gp_start_if_needed()
1206 sdp = raw_cpu_ptr(ssp->sda); in srcu_gp_start_if_needed()
1211 rcu_seq_current(&ssp->srcu_gp_seq)); in srcu_gp_start_if_needed()
1212 s = rcu_seq_snap(&ssp->srcu_gp_seq); in srcu_gp_start_if_needed()
1231 srcu_funnel_gp_start(ssp, sdp, s, do_norm); in srcu_gp_start_if_needed()
1233 srcu_funnel_exp_start(ssp, sdp_mynode, s); in srcu_gp_start_if_needed()
1234 __srcu_read_unlock_nmisafe(ssp, idx); in srcu_gp_start_if_needed()
1266 static void __call_srcu(struct srcu_struct *ssp, struct rcu_head *rhp, in __call_srcu() argument
1276 (void)srcu_gp_start_if_needed(ssp, rhp, do_norm); in __call_srcu()
1296 void call_srcu(struct srcu_struct *ssp, struct rcu_head *rhp, in call_srcu() argument
1299 __call_srcu(ssp, rhp, func, true); in call_srcu()
1306 static void __synchronize_srcu(struct srcu_struct *ssp, bool do_norm) in __synchronize_srcu() argument
1310 RCU_LOCKDEP_WARN(lockdep_is_held(ssp) || in __synchronize_srcu()
1319 check_init_srcu_struct(ssp); in __synchronize_srcu()
1322 __call_srcu(ssp, &rcu.head, wakeme_after_rcu, do_norm); in __synchronize_srcu()
1346 void synchronize_srcu_expedited(struct srcu_struct *ssp) in synchronize_srcu_expedited() argument
1348 __synchronize_srcu(ssp, rcu_gp_is_normal()); in synchronize_srcu_expedited()
1399 void synchronize_srcu(struct srcu_struct *ssp) in synchronize_srcu() argument
1401 if (srcu_might_be_idle(ssp) || rcu_gp_is_expedited()) in synchronize_srcu()
1402 synchronize_srcu_expedited(ssp); in synchronize_srcu()
1404 __synchronize_srcu(ssp, true); in synchronize_srcu()
1418 unsigned long get_state_synchronize_srcu(struct srcu_struct *ssp) in get_state_synchronize_srcu() argument
1423 return rcu_seq_snap(&ssp->srcu_gp_seq); in get_state_synchronize_srcu()
1437 unsigned long start_poll_synchronize_srcu(struct srcu_struct *ssp) in start_poll_synchronize_srcu() argument
1439 return srcu_gp_start_if_needed(ssp, NULL, true); in start_poll_synchronize_srcu()
1468 bool poll_state_synchronize_srcu(struct srcu_struct *ssp, unsigned long cookie) in poll_state_synchronize_srcu() argument
1470 if (!rcu_seq_done(&ssp->srcu_gp_seq, cookie)) in poll_state_synchronize_srcu()
1485 struct srcu_struct *ssp; in srcu_barrier_cb() local
1488 ssp = sdp->ssp; in srcu_barrier_cb()
1489 if (atomic_dec_and_test(&ssp->srcu_barrier_cpu_cnt)) in srcu_barrier_cb()
1490 complete(&ssp->srcu_barrier_completion); in srcu_barrier_cb()
1501 static void srcu_barrier_one_cpu(struct srcu_struct *ssp, struct srcu_data *sdp) in srcu_barrier_one_cpu() argument
1504 atomic_inc(&ssp->srcu_barrier_cpu_cnt); in srcu_barrier_one_cpu()
1510 atomic_dec(&ssp->srcu_barrier_cpu_cnt); in srcu_barrier_one_cpu()
1519 void srcu_barrier(struct srcu_struct *ssp) in srcu_barrier() argument
1523 unsigned long s = rcu_seq_snap(&ssp->srcu_barrier_seq); in srcu_barrier()
1525 check_init_srcu_struct(ssp); in srcu_barrier()
1526 mutex_lock(&ssp->srcu_barrier_mutex); in srcu_barrier()
1527 if (rcu_seq_done(&ssp->srcu_barrier_seq, s)) { in srcu_barrier()
1529 mutex_unlock(&ssp->srcu_barrier_mutex); in srcu_barrier()
1532 rcu_seq_start(&ssp->srcu_barrier_seq); in srcu_barrier()
1533 init_completion(&ssp->srcu_barrier_completion); in srcu_barrier()
1536 atomic_set(&ssp->srcu_barrier_cpu_cnt, 1); in srcu_barrier()
1538 idx = __srcu_read_lock_nmisafe(ssp); in srcu_barrier()
1539 if (smp_load_acquire(&ssp->srcu_size_state) < SRCU_SIZE_WAIT_BARRIER) in srcu_barrier()
1540 srcu_barrier_one_cpu(ssp, per_cpu_ptr(ssp->sda, get_boot_cpu_id())); in srcu_barrier()
1543 srcu_barrier_one_cpu(ssp, per_cpu_ptr(ssp->sda, cpu)); in srcu_barrier()
1544 __srcu_read_unlock_nmisafe(ssp, idx); in srcu_barrier()
1547 if (atomic_dec_and_test(&ssp->srcu_barrier_cpu_cnt)) in srcu_barrier()
1548 complete(&ssp->srcu_barrier_completion); in srcu_barrier()
1549 wait_for_completion(&ssp->srcu_barrier_completion); in srcu_barrier()
1551 rcu_seq_end(&ssp->srcu_barrier_seq); in srcu_barrier()
1552 mutex_unlock(&ssp->srcu_barrier_mutex); in srcu_barrier()
1563 unsigned long srcu_batches_completed(struct srcu_struct *ssp) in srcu_batches_completed() argument
1565 return READ_ONCE(ssp->srcu_idx); in srcu_batches_completed()
1574 static void srcu_advance_state(struct srcu_struct *ssp) in srcu_advance_state() argument
1578 mutex_lock(&ssp->srcu_gp_mutex); in srcu_advance_state()
1590 idx = rcu_seq_state(smp_load_acquire(&ssp->srcu_gp_seq)); /* ^^^ */ in srcu_advance_state()
1592 spin_lock_irq_rcu_node(ssp); in srcu_advance_state()
1593 if (ULONG_CMP_GE(ssp->srcu_gp_seq, ssp->srcu_gp_seq_needed)) { in srcu_advance_state()
1594 WARN_ON_ONCE(rcu_seq_state(ssp->srcu_gp_seq)); in srcu_advance_state()
1595 spin_unlock_irq_rcu_node(ssp); in srcu_advance_state()
1596 mutex_unlock(&ssp->srcu_gp_mutex); in srcu_advance_state()
1599 idx = rcu_seq_state(READ_ONCE(ssp->srcu_gp_seq)); in srcu_advance_state()
1601 srcu_gp_start(ssp); in srcu_advance_state()
1602 spin_unlock_irq_rcu_node(ssp); in srcu_advance_state()
1604 mutex_unlock(&ssp->srcu_gp_mutex); in srcu_advance_state()
1609 if (rcu_seq_state(READ_ONCE(ssp->srcu_gp_seq)) == SRCU_STATE_SCAN1) { in srcu_advance_state()
1610 idx = 1 ^ (ssp->srcu_idx & 1); in srcu_advance_state()
1611 if (!try_check_zero(ssp, idx, 1)) { in srcu_advance_state()
1612 mutex_unlock(&ssp->srcu_gp_mutex); in srcu_advance_state()
1615 srcu_flip(ssp); in srcu_advance_state()
1616 spin_lock_irq_rcu_node(ssp); in srcu_advance_state()
1617 rcu_seq_set_state(&ssp->srcu_gp_seq, SRCU_STATE_SCAN2); in srcu_advance_state()
1618 ssp->srcu_n_exp_nodelay = 0; in srcu_advance_state()
1619 spin_unlock_irq_rcu_node(ssp); in srcu_advance_state()
1622 if (rcu_seq_state(READ_ONCE(ssp->srcu_gp_seq)) == SRCU_STATE_SCAN2) { in srcu_advance_state()
1628 idx = 1 ^ (ssp->srcu_idx & 1); in srcu_advance_state()
1629 if (!try_check_zero(ssp, idx, 2)) { in srcu_advance_state()
1630 mutex_unlock(&ssp->srcu_gp_mutex); in srcu_advance_state()
1633 ssp->srcu_n_exp_nodelay = 0; in srcu_advance_state()
1634 srcu_gp_end(ssp); /* Releases ->srcu_gp_mutex. */ in srcu_advance_state()
1651 struct srcu_struct *ssp; in srcu_invoke_callbacks() local
1655 ssp = sdp->ssp; in srcu_invoke_callbacks()
1659 rcu_seq_current(&ssp->srcu_gp_seq)); in srcu_invoke_callbacks()
1687 rcu_seq_snap(&ssp->srcu_gp_seq)); in srcu_invoke_callbacks()
1699 static void srcu_reschedule(struct srcu_struct *ssp, unsigned long delay) in srcu_reschedule() argument
1703 spin_lock_irq_rcu_node(ssp); in srcu_reschedule()
1704 if (ULONG_CMP_GE(ssp->srcu_gp_seq, ssp->srcu_gp_seq_needed)) { in srcu_reschedule()
1705 if (!WARN_ON_ONCE(rcu_seq_state(ssp->srcu_gp_seq))) { in srcu_reschedule()
1709 } else if (!rcu_seq_state(ssp->srcu_gp_seq)) { in srcu_reschedule()
1711 srcu_gp_start(ssp); in srcu_reschedule()
1713 spin_unlock_irq_rcu_node(ssp); in srcu_reschedule()
1716 queue_delayed_work(rcu_gp_wq, &ssp->work, delay); in srcu_reschedule()
1726 struct srcu_struct *ssp; in process_srcu() local
1728 ssp = container_of(work, struct srcu_struct, work.work); in process_srcu()
1730 srcu_advance_state(ssp); in process_srcu()
1731 curdelay = srcu_get_delay(ssp); in process_srcu()
1733 WRITE_ONCE(ssp->reschedule_count, 0); in process_srcu()
1736 if (READ_ONCE(ssp->reschedule_jiffies) == j) { in process_srcu()
1737 WRITE_ONCE(ssp->reschedule_count, READ_ONCE(ssp->reschedule_count) + 1); in process_srcu()
1738 if (READ_ONCE(ssp->reschedule_count) > srcu_max_nodelay) in process_srcu()
1741 WRITE_ONCE(ssp->reschedule_count, 1); in process_srcu()
1742 WRITE_ONCE(ssp->reschedule_jiffies, j); in process_srcu()
1745 srcu_reschedule(ssp, curdelay); in process_srcu()
1749 struct srcu_struct *ssp, int *flags, in srcutorture_get_gp_data() argument
1755 *gp_seq = rcu_seq_current(&ssp->srcu_gp_seq); in srcutorture_get_gp_data()
1772 void srcu_torture_stats_print(struct srcu_struct *ssp, char *tt, char *tf) in srcu_torture_stats_print() argument
1777 int ss_state = READ_ONCE(ssp->srcu_size_state); in srcu_torture_stats_print()
1780 idx = ssp->srcu_idx & 0x1; in srcu_torture_stats_print()
1784 tt, tf, rcu_seq_current(&ssp->srcu_gp_seq), ss_state, in srcu_torture_stats_print()
1786 if (!ssp->sda) { in srcu_torture_stats_print()
1797 sdp = per_cpu_ptr(ssp->sda, cpu); in srcu_torture_stats_print()
1821 srcu_transition_to_big(ssp); in srcu_torture_stats_print()
1841 struct srcu_struct *ssp; in srcu_init() local
1861 ssp = list_first_entry(&srcu_boot_list, struct srcu_struct, in srcu_init()
1863 list_del_init(&ssp->work.work.entry); in srcu_init()
1864 if (SRCU_SIZING_IS(SRCU_SIZING_INIT) && ssp->srcu_size_state == SRCU_SIZE_SMALL) in srcu_init()
1865 ssp->srcu_size_state = SRCU_SIZE_ALLOC; in srcu_init()
1866 queue_work(rcu_gp_wq, &ssp->work.work); in srcu_init()