Lines Matching refs:rcu_state
21 rcu_seq_start(&rcu_state.expedited_sequence); in rcu_exp_gp_seq_start()
22 rcu_poll_gp_seq_start_unlocked(&rcu_state.gp_seq_polled_exp_snap); in rcu_exp_gp_seq_start()
31 return rcu_seq_endval(&rcu_state.expedited_sequence); in rcu_exp_gp_seq_endval()
39 rcu_poll_gp_seq_end_unlocked(&rcu_state.gp_seq_polled_exp_snap); in rcu_exp_gp_seq_end()
40 rcu_seq_end(&rcu_state.expedited_sequence); in rcu_exp_gp_seq_end()
54 s = rcu_seq_snap(&rcu_state.expedited_sequence); in rcu_exp_gp_seq_snap()
55 trace_rcu_exp_grace_period(rcu_state.name, s, TPS("snap")); in rcu_exp_gp_seq_snap()
66 return rcu_seq_done(&rcu_state.expedited_sequence, s); in rcu_exp_gp_seq_done()
82 int ncpus = smp_load_acquire(&rcu_state.ncpus); /* Order vs. locking. */ in sync_exp_reset_tree_hotplug()
87 if (likely(ncpus == rcu_state.ncpus_snap)) in sync_exp_reset_tree_hotplug()
89 rcu_state.ncpus_snap = ncpus; in sync_exp_reset_tree_hotplug()
204 swake_up_one(&rcu_state.expedited_wq); in __rcu_report_exp_rnp()
269 trace_rcu_exp_grace_period(rcu_state.name, s, TPS("done")); in sync_exp_work_done()
293 mutex_trylock(&rcu_state.exp_mutex)) in exp_funnel_lock()
313 trace_rcu_exp_funnel_lock(rcu_state.name, rnp->level, in exp_funnel_lock()
322 trace_rcu_exp_funnel_lock(rcu_state.name, rnp->level, in exp_funnel_lock()
325 mutex_lock(&rcu_state.exp_mutex); in exp_funnel_lock()
328 mutex_unlock(&rcu_state.exp_mutex); in exp_funnel_lock()
332 trace_rcu_exp_grace_period(rcu_state.name, s, TPS("start")); in exp_funnel_lock()
407 trace_rcu_exp_grace_period(rcu_state.name, rcu_exp_gp_seq_endval(), TPS("selectofl")); in __sync_rcu_exp_select_node_cpus()
536 trace_rcu_exp_grace_period(rcu_state.name, rcu_exp_gp_seq_endval(), TPS("reset")); in sync_rcu_exp_select_cpus()
538 trace_rcu_exp_grace_period(rcu_state.name, rcu_exp_gp_seq_endval(), TPS("select")); in sync_rcu_exp_select_cpus()
572 t = swait_event_timeout_exclusive(rcu_state.expedited_wq, in synchronize_rcu_expedited_wait_once()
598 trace_rcu_exp_grace_period(rcu_state.name, rcu_exp_gp_seq_endval(), TPS("startwait")); in synchronize_rcu_expedited_wait()
627 trace_rcu_stall_warning(rcu_state.name, TPS("ExpeditedStall")); in synchronize_rcu_expedited_wait()
629 rcu_state.name); in synchronize_rcu_expedited_wait()
649 jiffies - jiffies_start, rcu_state.expedited_sequence, in synchronize_rcu_expedited_wait()
697 mutex_lock(&rcu_state.exp_wake_mutex); in rcu_exp_wait_wake()
699 trace_rcu_exp_grace_period(rcu_state.name, s, TPS("end")); in rcu_exp_wait_wake()
712 trace_rcu_exp_grace_period(rcu_state.name, s, TPS("endwake")); in rcu_exp_wait_wake()
713 mutex_unlock(&rcu_state.exp_wake_mutex); in rcu_exp_wait_wake()
971 rcu_poll_gp_seq_start_unlocked(&rcu_state.gp_seq_polled_exp_snap); in synchronize_rcu_expedited()
972 rcu_poll_gp_seq_end_unlocked(&rcu_state.gp_seq_polled_exp_snap); in synchronize_rcu_expedited()
976 rcu_state.expedited_sequence += (1 << RCU_SEQ_CTR_SHIFT); in synchronize_rcu_expedited()
1009 mutex_unlock(&rcu_state.exp_mutex); in synchronize_rcu_expedited()
1036 …pr_info("%s: i = %d s = %lx gp_seq_polled = %lx\n", __func__, i, s, READ_ONCE(rcu_state.gp_seq_pol… in sync_rcu_do_polled_gp()