Lines Matching refs:srcu_gp_seq
138 sdp->srcu_gp_seq_needed = ssp->srcu_gp_seq; in init_srcu_struct_data()
139 sdp->srcu_gp_seq_needed_exp = ssp->srcu_gp_seq; in init_srcu_struct_data()
244 ssp->srcu_gp_seq = 0; in init_srcu_struct_fields()
611 if (ULONG_CMP_LT(READ_ONCE(ssp->srcu_gp_seq), READ_ONCE(ssp->srcu_gp_seq_needed_exp))) in srcu_get_delay()
613 if (rcu_seq_state(READ_ONCE(ssp->srcu_gp_seq))) { in srcu_get_delay()
651 if (WARN_ON(rcu_seq_state(READ_ONCE(ssp->srcu_gp_seq)) != SRCU_STATE_IDLE) || in cleanup_srcu_struct()
652 WARN_ON(rcu_seq_current(&ssp->srcu_gp_seq) != ssp->srcu_gp_seq_needed) || in cleanup_srcu_struct()
655 __func__, ssp, rcu_seq_state(READ_ONCE(ssp->srcu_gp_seq)), in cleanup_srcu_struct()
656 rcu_seq_current(&ssp->srcu_gp_seq), ssp->srcu_gp_seq_needed); in cleanup_srcu_struct()
768 WARN_ON_ONCE(ULONG_CMP_GE(ssp->srcu_gp_seq, ssp->srcu_gp_seq_needed)); in srcu_gp_start()
771 rcu_seq_current(&ssp->srcu_gp_seq)); in srcu_gp_start()
773 rcu_seq_snap(&ssp->srcu_gp_seq)); in srcu_gp_start()
778 rcu_seq_start(&ssp->srcu_gp_seq); in srcu_gp_start()
779 state = rcu_seq_state(ssp->srcu_gp_seq); in srcu_gp_start()
858 idx = rcu_seq_state(ssp->srcu_gp_seq); in srcu_gp_end()
860 if (ULONG_CMP_LT(READ_ONCE(ssp->srcu_gp_seq), READ_ONCE(ssp->srcu_gp_seq_needed_exp))) in srcu_gp_end()
864 rcu_seq_end(&ssp->srcu_gp_seq); in srcu_gp_end()
865 gpseq = rcu_seq_current(&ssp->srcu_gp_seq); in srcu_gp_end()
918 gpseq = rcu_seq_current(&ssp->srcu_gp_seq); in srcu_gp_end()
953 if (WARN_ON_ONCE(rcu_seq_done(&ssp->srcu_gp_seq, s)) || in srcu_funnel_exp_start()
1003 if (WARN_ON_ONCE(rcu_seq_done(&ssp->srcu_gp_seq, s)) && snp != snp_leaf) in srcu_funnel_gp_start()
1041 if (!WARN_ON_ONCE(rcu_seq_done(&ssp->srcu_gp_seq, s)) && in srcu_funnel_gp_start()
1042 rcu_seq_state(ssp->srcu_gp_seq) == SRCU_STATE_IDLE) { in srcu_funnel_gp_start()
1043 WARN_ON_ONCE(ULONG_CMP_GE(ssp->srcu_gp_seq, ssp->srcu_gp_seq_needed)); in srcu_funnel_gp_start()
1163 curseq = rcu_seq_current(&ssp->srcu_gp_seq); in srcu_might_be_idle()
1168 if (curseq != rcu_seq_current(&ssp->srcu_gp_seq)) in srcu_might_be_idle()
1211 rcu_seq_current(&ssp->srcu_gp_seq)); in srcu_gp_start_if_needed()
1212 s = rcu_seq_snap(&ssp->srcu_gp_seq); in srcu_gp_start_if_needed()
1423 return rcu_seq_snap(&ssp->srcu_gp_seq); in get_state_synchronize_srcu()
1470 if (!rcu_seq_done(&ssp->srcu_gp_seq, cookie)) in poll_state_synchronize_srcu()
1590 idx = rcu_seq_state(smp_load_acquire(&ssp->srcu_gp_seq)); /* ^^^ */ in srcu_advance_state()
1593 if (ULONG_CMP_GE(ssp->srcu_gp_seq, ssp->srcu_gp_seq_needed)) { in srcu_advance_state()
1594 WARN_ON_ONCE(rcu_seq_state(ssp->srcu_gp_seq)); in srcu_advance_state()
1599 idx = rcu_seq_state(READ_ONCE(ssp->srcu_gp_seq)); in srcu_advance_state()
1609 if (rcu_seq_state(READ_ONCE(ssp->srcu_gp_seq)) == SRCU_STATE_SCAN1) { in srcu_advance_state()
1617 rcu_seq_set_state(&ssp->srcu_gp_seq, SRCU_STATE_SCAN2); in srcu_advance_state()
1622 if (rcu_seq_state(READ_ONCE(ssp->srcu_gp_seq)) == SRCU_STATE_SCAN2) { in srcu_advance_state()
1659 rcu_seq_current(&ssp->srcu_gp_seq)); in srcu_invoke_callbacks()
1687 rcu_seq_snap(&ssp->srcu_gp_seq)); in srcu_invoke_callbacks()
1704 if (ULONG_CMP_GE(ssp->srcu_gp_seq, ssp->srcu_gp_seq_needed)) { in srcu_reschedule()
1705 if (!WARN_ON_ONCE(rcu_seq_state(ssp->srcu_gp_seq))) { in srcu_reschedule()
1709 } else if (!rcu_seq_state(ssp->srcu_gp_seq)) { in srcu_reschedule()
1755 *gp_seq = rcu_seq_current(&ssp->srcu_gp_seq); in srcutorture_get_gp_data()
1784 tt, tf, rcu_seq_current(&ssp->srcu_gp_seq), ss_state, in srcu_torture_stats_print()