Lines Matching refs:rcu_data
78 static DEFINE_PER_CPU_SHARED_ALIGNED(struct rcu_data, rcu_data) = {
149 static void rcu_report_exp_rdp(struct rcu_data *rdp);
151 static void check_cb_ovld_locked(struct rcu_data *rdp, struct rcu_node *rnp);
152 static bool rcu_rdp_is_offloaded(struct rcu_data *rdp);
153 static bool rcu_rdp_cpu_online(struct rcu_data *rdp);
235 struct rcu_data *rdp = per_cpu_ptr(&rcu_data, cpu); in rcu_get_n_cbs_cpu()
290 static bool rcu_dynticks_in_eqs_since(struct rcu_data *rdp, int snap) in rcu_dynticks_in_eqs_since()
329 raw_cpu_write(rcu_data.rcu_need_heavy_qs, false); in rcu_momentary_dyntick_idle()
475 static void force_qs_rnp(int (*f)(struct rcu_data *rdp));
548 struct rcu_data *rdp = this_cpu_ptr(&rcu_data); in rcu_irq_work_resched()
611 struct rcu_data *rdp = this_cpu_ptr(&rcu_data); in __rcu_irq_enter_check_tick()
658 return !rcu_segcblist_empty(&this_cpu_ptr(&rcu_data)->cblist) && in rcu_needs_cpu()
659 !rcu_rdp_is_offloaded(this_cpu_ptr(&rcu_data)); in rcu_needs_cpu()
667 static void rcu_disable_urgency_upon_qs(struct rcu_data *rdp) in rcu_disable_urgency_upon_qs()
715 smp_store_release(per_cpu_ptr(&rcu_data.rcu_urgent_qs, cpu), true); in rcu_request_urgent_qs_task()
725 static void rcu_gpnum_ovf(struct rcu_node *rnp, struct rcu_data *rdp) in rcu_gpnum_ovf()
740 static int dyntick_save_progress_counter(struct rcu_data *rdp) in dyntick_save_progress_counter()
757 static int rcu_implicit_dynticks_qs(struct rcu_data *rdp) in rcu_implicit_dynticks_qs()
893 static void trace_rcu_this_gp(struct rcu_node *rnp, struct rcu_data *rdp, in trace_rcu_this_gp()
917 static bool rcu_start_this_gp(struct rcu_node *rnp_start, struct rcu_data *rdp, in rcu_start_this_gp()
995 struct rcu_data *rdp = this_cpu_ptr(&rcu_data); in rcu_future_gp_cleanup()
1044 static bool rcu_accelerate_cbs(struct rcu_node *rnp, struct rcu_data *rdp) in rcu_accelerate_cbs()
1091 struct rcu_data *rdp) in rcu_accelerate_cbs_unlocked()
1120 static bool rcu_advance_cbs(struct rcu_node *rnp, struct rcu_data *rdp) in rcu_advance_cbs()
1144 struct rcu_data *rdp) in rcu_advance_cbs_nowake()
1174 static bool __note_gp_changes(struct rcu_node *rnp, struct rcu_data *rdp) in __note_gp_changes()
1223 static void note_gp_changes(struct rcu_data *rdp) in note_gp_changes()
1390 struct rcu_data *rdp; in rcu_gp_init()
1497 rdp = this_cpu_ptr(&rcu_data); in rcu_gp_init()
1670 struct rcu_data *rdp; in rcu_gp_cleanup()
1711 rdp = this_cpu_ptr(&rcu_data); in rcu_gp_cleanup()
1719 rdp = per_cpu_ptr(&rcu_data, cpu); in rcu_gp_cleanup()
1738 rdp = this_cpu_ptr(&rcu_data); in rcu_gp_cleanup()
1954 rcu_report_qs_rdp(struct rcu_data *rdp) in rcu_report_qs_rdp()
2020 rcu_check_quiescent_state(struct rcu_data *rdp) in rcu_check_quiescent_state()
2050 static void rcu_do_batch(struct rcu_data *rdp) in rcu_do_batch()
2193 WARN_ON_ONCE(time_before(j, __this_cpu_read(rcu_data.last_sched_clock))); in rcu_sched_clock_irq()
2194 __this_cpu_write(rcu_data.last_sched_clock, j); in rcu_sched_clock_irq()
2198 raw_cpu_inc(rcu_data.ticks_this_gp); in rcu_sched_clock_irq()
2200 if (smp_load_acquire(this_cpu_ptr(&rcu_data.rcu_urgent_qs))) { in rcu_sched_clock_irq()
2206 __this_cpu_write(rcu_data.rcu_urgent_qs, false); in rcu_sched_clock_irq()
2225 static void force_qs_rnp(int (*f)(struct rcu_data *rdp)) in force_qs_rnp()
2230 struct rcu_data *rdp; in force_qs_rnp()
2255 rdp = per_cpu_ptr(&rcu_data, cpu); in force_qs_rnp()
2283 rnp = raw_cpu_read(rcu_data.mynode); in rcu_force_quiescent_state()
2321 struct rcu_data *rdp = raw_cpu_ptr(&rcu_data); in rcu_core()
2408 __this_cpu_write(rcu_data.rcu_cpu_has_work, 1); in invoke_rcu_core_kthread()
2409 t = __this_cpu_read(rcu_data.rcu_cpu_kthread_task); in invoke_rcu_core_kthread()
2411 rcu_wake_cond(t, __this_cpu_read(rcu_data.rcu_cpu_kthread_status)); in invoke_rcu_core_kthread()
2430 per_cpu(rcu_data.rcu_cpu_kthread_status, cpu) = RCU_KTHREAD_OFFCPU; in rcu_cpu_kthread_park()
2435 return __this_cpu_read(rcu_data.rcu_cpu_has_work); in rcu_cpu_kthread_should_run()
2445 unsigned int *statusp = this_cpu_ptr(&rcu_data.rcu_cpu_kthread_status); in rcu_cpu_kthread()
2446 char work, *workp = this_cpu_ptr(&rcu_data.rcu_cpu_has_work); in rcu_cpu_kthread()
2447 unsigned long *j = this_cpu_ptr(&rcu_data.rcuc_activity); in rcu_cpu_kthread()
2477 .store = &rcu_data.rcu_cpu_kthread_task,
2493 per_cpu(rcu_data.rcu_cpu_has_work, cpu) = 0; in rcu_spawn_core_kthreads()
2504 static void __call_rcu_core(struct rcu_data *rdp, struct rcu_head *head, in __call_rcu_core()
2559 static void check_cb_ovld_locked(struct rcu_data *rdp, struct rcu_node *rnp) in check_cb_ovld_locked()
2582 static void check_cb_ovld(struct rcu_data *rdp) in check_cb_ovld()
2601 struct rcu_data *rdp; in __call_rcu_common()
2624 rdp = this_cpu_ptr(&rcu_data); in __call_rcu_common()
3505 for (rnp = this_cpu_ptr(&rcu_data)->mynode; rnp; rnp = rnp->parent) in synchronize_rcu()
3582 struct rcu_data *rdp; in start_poll_synchronize_rcu_common()
3587 rdp = this_cpu_ptr(&rcu_data); in start_poll_synchronize_rcu_common()
3800 struct rcu_data *rdp = this_cpu_ptr(&rcu_data); in rcu_pending()
3876 static void rcu_barrier_entrain(struct rcu_data *rdp) in rcu_barrier_entrain()
3916 struct rcu_data *rdp = per_cpu_ptr(&rcu_data, cpu); in rcu_barrier_handler()
3939 struct rcu_data *rdp; in rcu_barrier()
3978 rdp = per_cpu_ptr(&rcu_data, cpu); in rcu_barrier()
4020 rdp = per_cpu_ptr(&rcu_data, cpu); in rcu_barrier()
4046 static bool rcu_rdp_cpu_online(struct rcu_data *rdp) in rcu_rdp_cpu_online()
4068 struct rcu_data *rdp; in rcu_lockdep_current_cpu_online()
4074 rdp = this_cpu_ptr(&rcu_data); in rcu_lockdep_current_cpu_online()
4105 struct rcu_data *rdp = per_cpu_ptr(&rcu_data, cpu); in rcutree_dying_cpu()
4214 struct rcu_data *rdp = per_cpu_ptr(&rcu_data, cpu); in rcu_boot_init_percpu_data()
4245 struct rcu_data *rdp = per_cpu_ptr(&rcu_data, cpu); in rcutree_prepare_cpu()
4292 struct rcu_data *rdp = per_cpu_ptr(&rcu_data, cpu); in rcutree_affinity_setting()
4304 struct rcu_data *rdp; in rcutree_online_cpu()
4307 rdp = per_cpu_ptr(&rcu_data, cpu); in rcutree_online_cpu()
4329 struct rcu_data *rdp; in rcutree_offline_cpu()
4332 rdp = per_cpu_ptr(&rcu_data, cpu); in rcutree_offline_cpu()
4360 struct rcu_data *rdp; in rcu_cpu_starting()
4364 rdp = per_cpu_ptr(&rcu_data, cpu); in rcu_cpu_starting()
4416 struct rcu_data *rdp = per_cpu_ptr(&rcu_data, cpu); in rcu_report_dead()
4454 struct rcu_data *my_rdp; in rcutree_migrate_callbacks()
4456 struct rcu_data *rdp = per_cpu_ptr(&rcu_data, cpu); in rcutree_migrate_callbacks()
4466 my_rdp = this_cpu_ptr(&rcu_data); in rcutree_migrate_callbacks()
4576 struct rcu_data *rdp = this_cpu_ptr(&rcu_data); in rcu_spawn_gp_kthread()
4720 per_cpu_ptr(&rcu_data, i)->mynode = rnp; in rcu_init_one()
4944 if (!(per_cpu_ptr(&rcu_data, cpu)->mynode->exp_seq_poll_rq & 0x1)) in rcu_init()