Lines Matching refs:rdp
237 struct rcu_data *rdp; in rcu_report_exp_cpu_mult() local
246 rdp = per_cpu_ptr(&rcu_data, cpu); in rcu_report_exp_cpu_mult()
247 if (!IS_ENABLED(CONFIG_NO_HZ_FULL) || !rdp->rcu_forced_tick_exp) in rcu_report_exp_cpu_mult()
249 rdp->rcu_forced_tick_exp = false; in rcu_report_exp_cpu_mult()
258 static void rcu_report_exp_rdp(struct rcu_data *rdp) in rcu_report_exp_rdp() argument
260 WRITE_ONCE(rdp->cpu_no_qs.b.exp, false); in rcu_report_exp_rdp()
261 rcu_report_exp_cpu_mult(rdp->mynode, rdp->grpmask, true); in rcu_report_exp_rdp()
289 struct rcu_data *rdp = per_cpu_ptr(&rcu_data, raw_smp_processor_id()); in exp_funnel_lock() local
290 struct rcu_node *rnp = rdp->mynode; in exp_funnel_lock()
358 struct rcu_data *rdp = per_cpu_ptr(&rcu_data, cpu); in __sync_rcu_exp_select_node_cpus() local
359 unsigned long mask = rdp->grpmask; in __sync_rcu_exp_select_node_cpus()
384 rdp->exp_watching_snap = snap; in __sync_rcu_exp_select_node_cpus()
400 struct rcu_data *rdp = per_cpu_ptr(&rcu_data, cpu); in __sync_rcu_exp_select_node_cpus() local
401 unsigned long mask = rdp->grpmask; in __sync_rcu_exp_select_node_cpus()
404 if (rcu_watching_snap_stopped_since(rdp, rdp->exp_watching_snap)) { in __sync_rcu_exp_select_node_cpus()
566 struct rcu_data *rdp; in synchronize_rcu_expedited_stall() local
572 rdp = per_cpu_ptr(&rcu_data, cpu); in synchronize_rcu_expedited_stall()
575 "o."[!!(rdp->grpmask & rnp->expmaskinit)], in synchronize_rcu_expedited_stall()
576 "N."[!!(rdp->grpmask & rnp->expmaskinitnext)], in synchronize_rcu_expedited_stall()
577 "D."[!!data_race(rdp->cpu_no_qs.b.exp)]); in synchronize_rcu_expedited_stall()
618 struct rcu_data *rdp; in synchronize_rcu_expedited_wait() local
632 rdp = per_cpu_ptr(&rcu_data, cpu); in synchronize_rcu_expedited_wait()
633 if (rdp->rcu_forced_tick_exp) in synchronize_rcu_expedited_wait()
635 rdp->rcu_forced_tick_exp = true; in synchronize_rcu_expedited_wait()
728 struct rcu_data *rdp = this_cpu_ptr(&rcu_data); in rcu_exp_handler() local
729 struct rcu_node *rnp = rdp->mynode; in rcu_exp_handler()
740 rcu_report_exp_rdp(rdp); in rcu_exp_handler()
742 WRITE_ONCE(rdp->cpu_no_qs.b.exp, true); in rcu_exp_handler()
763 if (rnp->expmask & rdp->grpmask) { in rcu_exp_handler()
764 WRITE_ONCE(rdp->cpu_no_qs.b.exp, true); in rcu_exp_handler()
851 struct rcu_data *rdp = this_cpu_ptr(&rcu_data); in rcu_exp_handler() local
852 struct rcu_node *rnp = rdp->mynode; in rcu_exp_handler()
855 if (!(READ_ONCE(rnp->expmask) & rdp->grpmask) || in rcu_exp_handler()
871 struct rcu_data *rdp; in sync_sched_exp_online_cleanup() local
875 rdp = per_cpu_ptr(&rcu_data, cpu); in sync_sched_exp_online_cleanup()
876 rnp = rdp->mynode; in sync_sched_exp_online_cleanup()
879 if (!(READ_ONCE(rnp->expmask) & rdp->grpmask) || in sync_sched_exp_online_cleanup()
880 READ_ONCE(rdp->cpu_no_qs.b.exp)) { in sync_sched_exp_online_cleanup()
1041 struct rcu_data *rdp; in start_poll_synchronize_rcu_expedited() local
1046 rdp = per_cpu_ptr(&rcu_data, raw_smp_processor_id()); in start_poll_synchronize_rcu_expedited()
1047 rnp = rdp->mynode; in start_poll_synchronize_rcu_expedited()