Lines Matching refs:rcu_state
22 rcu_seq_start(&rcu_state.expedited_sequence); in rcu_exp_gp_seq_start()
23 rcu_poll_gp_seq_start_unlocked(&rcu_state.gp_seq_polled_exp_snap); in rcu_exp_gp_seq_start()
32 return rcu_seq_endval(&rcu_state.expedited_sequence); in rcu_exp_gp_seq_endval()
40 rcu_poll_gp_seq_end_unlocked(&rcu_state.gp_seq_polled_exp_snap); in rcu_exp_gp_seq_end()
41 rcu_seq_end(&rcu_state.expedited_sequence); in rcu_exp_gp_seq_end()
55 s = rcu_seq_snap(&rcu_state.expedited_sequence); in rcu_exp_gp_seq_snap()
56 trace_rcu_exp_grace_period(rcu_state.name, s, TPS("snap")); in rcu_exp_gp_seq_snap()
67 return rcu_seq_done(&rcu_state.expedited_sequence, s); in rcu_exp_gp_seq_done()
83 int ncpus = smp_load_acquire(&rcu_state.ncpus); /* Order vs. locking. */ in sync_exp_reset_tree_hotplug()
88 if (likely(ncpus == rcu_state.ncpus_snap)) in sync_exp_reset_tree_hotplug()
90 rcu_state.ncpus_snap = ncpus; in sync_exp_reset_tree_hotplug()
203 swake_up_one_online(&rcu_state.expedited_wq); in __rcu_report_exp_rnp()
268 trace_rcu_exp_grace_period(rcu_state.name, s, TPS("done")); in sync_exp_work_done()
297 mutex_trylock(&rcu_state.exp_mutex)) in exp_funnel_lock()
317 trace_rcu_exp_funnel_lock(rcu_state.name, rnp->level, in exp_funnel_lock()
326 trace_rcu_exp_funnel_lock(rcu_state.name, rnp->level, in exp_funnel_lock()
329 mutex_lock(&rcu_state.exp_mutex); in exp_funnel_lock()
332 mutex_unlock(&rcu_state.exp_mutex); in exp_funnel_lock()
336 trace_rcu_exp_grace_period(rcu_state.name, s, TPS("start")); in exp_funnel_lock()
425 trace_rcu_exp_grace_period(rcu_state.name, rcu_exp_gp_seq_endval(), TPS("selectofl")); in __sync_rcu_exp_select_node_cpus()
500 trace_rcu_exp_grace_period(rcu_state.name, rcu_exp_gp_seq_endval(), TPS("reset")); in sync_rcu_exp_select_cpus()
502 trace_rcu_exp_grace_period(rcu_state.name, rcu_exp_gp_seq_endval(), TPS("select")); in sync_rcu_exp_select_cpus()
536 t = swait_event_timeout_exclusive(rcu_state.expedited_wq, in synchronize_rcu_expedited_wait_once()
558 …detected expedited stalls, but suppressed full report due to a stuck CSD-lock.\n", rcu_state.name); in synchronize_rcu_expedited_stall()
561 pr_err("INFO: %s detected expedited stalls on CPUs/tasks: {", rcu_state.name); in synchronize_rcu_expedited_stall()
581 j - jiffies_start, rcu_state.expedited_sequence, data_race(rnp_root->expmask), in synchronize_rcu_expedited_stall()
622 trace_rcu_exp_grace_period(rcu_state.name, rcu_exp_gp_seq_endval(), TPS("startwait")); in synchronize_rcu_expedited_wait()
658 trace_rcu_stall_warning(rcu_state.name, TPS("ExpeditedStall")); in synchronize_rcu_expedited_wait()
683 mutex_lock(&rcu_state.exp_wake_mutex); in rcu_exp_wait_wake()
685 trace_rcu_exp_grace_period(rcu_state.name, s, TPS("end")); in rcu_exp_wait_wake()
698 trace_rcu_exp_grace_period(rcu_state.name, s, TPS("endwake")); in rcu_exp_wait_wake()
699 mutex_unlock(&rcu_state.exp_wake_mutex); in rcu_exp_wait_wake()
958 rcu_poll_gp_seq_start_unlocked(&rcu_state.gp_seq_polled_exp_snap); in synchronize_rcu_expedited()
959 rcu_poll_gp_seq_end_unlocked(&rcu_state.gp_seq_polled_exp_snap); in synchronize_rcu_expedited()
963 rcu_state.expedited_sequence += (1 << RCU_SEQ_CTR_SHIFT); in synchronize_rcu_expedited()
995 mutex_unlock(&rcu_state.exp_mutex); in synchronize_rcu_expedited()
1019 …pr_info("%s: i = %d s = %lx gp_seq_polled = %lx\n", __func__, i, s, READ_ONCE(rcu_state.gp_seq_pol… in sync_rcu_do_polled_gp()