Lines Matching refs:gp_seq
87 .gp_seq = (0UL - 300UL) << RCU_SEQ_CTR_SHIFT,
226 return rcu_seq_state(rcu_seq_current(&rcu_state.gp_seq)); in rcu_gp_in_progress()
483 return READ_ONCE(rcu_state.gp_seq); in rcu_get_gp_seq()
511 unsigned long *gp_seq) in rcutorture_get_gp_data() argument
516 *gp_seq = rcu_seq_current(&rcu_state.gp_seq); in rcutorture_get_gp_data()
728 if (ULONG_CMP_LT(rcu_seq_current(&rdp->gp_seq) + ULONG_MAX / 4, in rcu_gpnum_ovf()
729 rnp->gp_seq)) in rcu_gpnum_ovf()
731 if (ULONG_CMP_LT(rdp->rcu_iw_gp_seq + ULONG_MAX / 4, rnp->gp_seq)) in rcu_gpnum_ovf()
732 rdp->rcu_iw_gp_seq = rnp->gp_seq + ULONG_MAX / 4; in rcu_gpnum_ovf()
744 trace_rcu_fqs(rcu_state.name, rdp->gp_seq, rdp->cpu, TPS("dti")); in dyntick_save_progress_counter()
771 trace_rcu_fqs(rcu_state.name, rdp->gp_seq, rdp->cpu, TPS("dti")); in rcu_implicit_dynticks_qs()
799 (long)rnp->gp_seq, (long)rnp->completedqs); in rcu_implicit_dynticks_qs()
863 !rdp->rcu_iw_pending && rdp->rcu_iw_gp_seq != rnp->gp_seq && in rcu_implicit_dynticks_qs()
866 rdp->rcu_iw_gp_seq = rnp->gp_seq; in rcu_implicit_dynticks_qs()
870 if (rcu_cpu_stall_cputime && rdp->snap_record.gp_seq != rdp->gp_seq) { in rcu_implicit_dynticks_qs()
885 rsrp->gp_seq = rdp->gp_seq; in rcu_implicit_dynticks_qs()
896 trace_rcu_future_grace_period(rcu_state.name, READ_ONCE(rnp->gp_seq), in trace_rcu_this_gp()
938 rcu_seq_started(&rnp->gp_seq, gp_seq_req) || in rcu_start_this_gp()
940 rcu_seq_state(rcu_seq_current(&rnp->gp_seq)))) { in rcu_start_this_gp()
946 if (rcu_seq_state(rcu_seq_current(&rnp->gp_seq))) { in rcu_start_this_gp()
975 trace_rcu_grace_period(rcu_state.name, data_race(rcu_state.gp_seq), TPS("newreq")); in rcu_start_this_gp()
997 needmore = ULONG_CMP_LT(rnp->gp_seq, rnp->gp_seq_needed); in rcu_future_gp_cleanup()
999 rnp->gp_seq_needed = rnp->gp_seq; /* Avoid counter wrap. */ in rcu_future_gp_cleanup()
1000 trace_rcu_this_gp(rnp, rdp, rnp->gp_seq, in rcu_future_gp_cleanup()
1028 WRITE_ONCE(rcu_state.gp_wake_seq, READ_ONCE(rcu_state.gp_seq)); in rcu_gp_kthread_wake()
1068 gp_seq_req = rcu_seq_snap(&rcu_state.gp_seq); in rcu_accelerate_cbs()
1097 c = rcu_seq_snap(&rcu_state.gp_seq); in rcu_accelerate_cbs_unlocked()
1133 rcu_segcblist_advance(&rdp->cblist, rnp->gp_seq); in rcu_advance_cbs()
1147 if (!rcu_seq_state(rcu_seq_current(&rnp->gp_seq)) || !raw_spin_trylock_rcu_node(rnp)) in rcu_advance_cbs_nowake()
1150 if (rcu_seq_state(rcu_seq_current(&rnp->gp_seq))) in rcu_advance_cbs_nowake()
1182 if (rdp->gp_seq == rnp->gp_seq) in __note_gp_changes()
1186 if (rcu_seq_completed_gp(rdp->gp_seq, rnp->gp_seq) || in __note_gp_changes()
1191 trace_rcu_grace_period(rcu_state.name, rdp->gp_seq, TPS("cpuend")); in __note_gp_changes()
1200 if (rcu_seq_new_gp(rdp->gp_seq, rnp->gp_seq) || in __note_gp_changes()
1207 trace_rcu_grace_period(rcu_state.name, rnp->gp_seq, TPS("cpustart")); in __note_gp_changes()
1213 rdp->gp_seq = rnp->gp_seq; /* Remember new grace-period state. */ in __note_gp_changes()
1231 if ((rdp->gp_seq == rcu_seq_current(&rnp->gp_seq) && in note_gp_changes()
1274 !(rcu_seq_ctr(rcu_state.gp_seq) % (rcu_num_nodes * PER_RCU_NODE_PERIOD * delay))) in rcu_gp_slow()
1414 rcu_seq_start(&rcu_state.gp_seq); in rcu_gp_init()
1415 ASSERT_EXCLUSIVE_WRITER(rcu_state.gp_seq); in rcu_gp_init()
1416 trace_rcu_grace_period(rcu_state.name, rcu_state.gp_seq, TPS("start")); in rcu_gp_init()
1500 WRITE_ONCE(rnp->gp_seq, rcu_state.gp_seq); in rcu_gp_init()
1504 trace_rcu_grace_period_init(rcu_state.name, rnp->gp_seq, in rcu_gp_init()
1511 rcu_report_qs_rnp(mask, rnp, rnp->gp_seq, flags); in rcu_gp_init()
1605 trace_rcu_grace_period(rcu_state.name, rcu_state.gp_seq, in rcu_gp_fqs_loop()
1628 trace_rcu_grace_period(rcu_state.name, rcu_state.gp_seq, in rcu_gp_fqs_loop()
1636 trace_rcu_grace_period(rcu_state.name, rcu_state.gp_seq, in rcu_gp_fqs_loop()
1647 trace_rcu_grace_period(rcu_state.name, rcu_state.gp_seq, in rcu_gp_fqs_loop()
1701 new_gp_seq = rcu_state.gp_seq; in rcu_gp_cleanup()
1708 WRITE_ONCE(rnp->gp_seq, new_gp_seq); in rcu_gp_cleanup()
1733 trace_rcu_grace_period(rcu_state.name, rcu_state.gp_seq, TPS("end")); in rcu_gp_cleanup()
1734 rcu_seq_end(&rcu_state.gp_seq); in rcu_gp_cleanup()
1735 ASSERT_EXCLUSIVE_WRITER(rcu_state.gp_seq); in rcu_gp_cleanup()
1739 if (!needgp && ULONG_CMP_LT(rnp->gp_seq, rnp->gp_seq_needed)) { in rcu_gp_cleanup()
1760 trace_rcu_grace_period(rcu_state.name, rcu_state.gp_seq, TPS("newreq")); in rcu_gp_cleanup()
1788 trace_rcu_grace_period(rcu_state.name, rcu_state.gp_seq, in rcu_gp_kthread()
1802 trace_rcu_grace_period(rcu_state.name, rcu_state.gp_seq, in rcu_gp_kthread()
1861 if ((!(rnp->qsmask & mask) && mask) || rnp->gp_seq != gps) { in rcu_report_qs_rnp()
1874 trace_rcu_quiescent_state_report(rcu_state.name, rnp->gp_seq, in rcu_report_qs_rnp()
1884 rnp->completedqs = rnp->gp_seq; in rcu_report_qs_rnp()
1930 rnp->completedqs = rnp->gp_seq; in rcu_report_unblock_qs_rnp()
1942 gps = rnp->gp_seq; in rcu_report_unblock_qs_rnp()
1965 if (rdp->cpu_no_qs.b.norm || rdp->gp_seq != rnp->gp_seq || in rcu_report_qs_rdp()
2000 rcu_report_qs_rnp(mask, rnp, rnp->gp_seq, flags); in rcu_report_qs_rdp()
2263 rcu_report_qs_rnp(mask, rnp, rnp->gp_seq, flags); in force_qs_rnp()
3504 rcu_state.gp_seq += (1 << RCU_SEQ_CTR_SHIFT); in synchronize_rcu()
3506 rnp->gp_seq_needed = rnp->gp_seq = rcu_state.gp_seq; in synchronize_rcu()
3569 rgosp->rgos_norm = rcu_seq_snap(&rnp->gp_seq); in get_state_synchronize_rcu_full()
3596 needwake = rcu_start_this_gp(rnp, rdp, rcu_seq_snap(&rcu_state.gp_seq)); in start_poll_synchronize_rcu_common()
3616 unsigned long gp_seq = get_state_synchronize_rcu(); in start_poll_synchronize_rcu() local
3619 return gp_seq; in start_poll_synchronize_rcu()
3728 rcu_seq_done_exact(&rnp->gp_seq, rgosp->rgos_norm) || in poll_state_synchronize_rcu_full()
3833 if (rcu_seq_current(&rnp->gp_seq) != rdp->gp_seq || in rcu_pending()
4112 trace_rcu_grace_period(rcu_state.name, READ_ONCE(rnp->gp_seq), in rcutree_dying_cpu()
4222 rdp->rcu_ofl_gp_seq = rcu_state.gp_seq; in rcu_boot_init_percpu_data()
4224 rdp->rcu_onl_gp_seq = rcu_state.gp_seq; in rcu_boot_init_percpu_data()
4271 rdp->gp_seq = READ_ONCE(rnp->gp_seq); in rcutree_prepare_cpu()
4272 rdp->gp_seq_needed = rdp->gp_seq; in rcutree_prepare_cpu()
4277 rdp->rcu_iw_gp_seq = rdp->gp_seq - 1; in rcutree_prepare_cpu()
4278 trace_rcu_grace_period(rcu_state.name, rdp->gp_seq, TPS("cpuonl")); in rcutree_prepare_cpu()
4384 rdp->rcu_onl_gp_seq = READ_ONCE(rcu_state.gp_seq); in rcu_cpu_starting()
4395 rcu_report_qs_rnp(mask, rnp, rnp->gp_seq, flags2); in rcu_cpu_starting()
4429 rdp->rcu_ofl_gp_seq = READ_ONCE(rcu_state.gp_seq); in rcu_report_dead()
4434 rcu_report_qs_rnp(mask, rnp, rnp->gp_seq, flags); in rcu_report_dead()
4631 rnp->gp_seq_needed = rnp->gp_seq = rcu_state.gp_seq; in rcu_scheduler_starting()
4680 rnp->gp_seq = rcu_state.gp_seq; in rcu_init_one()
4681 rnp->gp_seq_needed = rcu_state.gp_seq; in rcu_init_one()
4682 rnp->completedqs = rcu_state.gp_seq; in rcu_init_one()