Lines Matching refs:cpu
102 WARN_ON_ONCE(smp_processor_id() != rdp->cpu); in rcu_nocb_bypass_lock()
202 trace_rcu_nocb_wake(rcu_state.name, rdp->cpu, in __wake_nocb_gp()
218 trace_rcu_nocb_wake(rcu_state.name, rdp->cpu, TPS("DoWake")); in __wake_nocb_gp()
294 trace_rcu_nocb_wake(rcu_state.name, rdp->cpu, reason); in wake_nocb_gp_defer()
443 trace_rcu_nocb_wake(rcu_state.name, rdp->cpu, in rcu_nocb_try_bypass()
462 trace_rcu_nocb_wake(rcu_state.name, rdp->cpu, in rcu_nocb_try_bypass()
493 trace_rcu_nocb_wake(rcu_state.name, rdp->cpu, TPS("FirstBQ")); in rcu_nocb_try_bypass()
508 trace_rcu_nocb_wake(rcu_state.name, rdp->cpu, in rcu_nocb_try_bypass()
512 trace_rcu_nocb_wake(rcu_state.name, rdp->cpu, in rcu_nocb_try_bypass()
542 trace_rcu_nocb_wake(rcu_state.name, rdp->cpu, in __call_rcu_nocb_wake()
561 trace_rcu_nocb_wake(rcu_state.name, rdp->cpu, in __call_rcu_nocb_wake()
587 trace_rcu_nocb_wake(rcu_state.name, rdp->cpu, TPS("WakeNot")); in __call_rcu_nocb_wake()
591 trace_rcu_nocb_wake(rcu_state.name, rdp->cpu, TPS("WakeNot")); in __call_rcu_nocb_wake()
636 static void nocb_gp_sleep(struct rcu_data *my_rdp, int cpu) in nocb_gp_sleep() argument
638 trace_rcu_nocb_wake(rcu_state.name, cpu, TPS("Sleep")); in nocb_gp_sleep()
641 trace_rcu_nocb_wake(rcu_state.name, cpu, TPS("EndSleep")); in nocb_gp_sleep()
651 int __maybe_unused cpu = my_rdp->cpu; in nocb_gp_wait() local
690 trace_rcu_nocb_wake(rcu_state.name, rdp->cpu, TPS("Check")); in nocb_gp_wait()
717 trace_rcu_nocb_wake(rcu_state.name, rdp->cpu, in nocb_gp_wait()
747 trace_rcu_nocb_wake(rcu_state.name, rdp->cpu, in nocb_gp_wait()
786 trace_rcu_nocb_wake(rcu_state.name, cpu, TPS("Poll")); in nocb_gp_wait()
793 nocb_gp_sleep(my_rdp, cpu); in nocb_gp_wait()
799 nocb_gp_sleep(my_rdp, cpu); in nocb_gp_wait()
902 trace_rcu_nocb_wake(rcu_state.name, rdp->cpu, TPS("WokeEmpty")); in nocb_cb_wait()
930 trace_rcu_nocb_wake(rcu_state.name, rdp->cpu, TPS("CBSleep")); in nocb_cb_wait()
979 trace_rcu_nocb_wake(rcu_state.name, rdp->cpu, TPS("DeferredWake")); in do_nocb_deferred_wakeup_common()
991 trace_rcu_nocb_wake(rcu_state.name, rdp->cpu, TPS("Timer")); in do_nocb_deferred_wakeup_timer()
1063 WARN_ON_ONCE(cpu_online(rdp->cpu) && rdp->cpu != raw_smp_processor_id()); in rcu_nocb_rdp_deoffload()
1065 pr_info("De-offloading %d\n", rdp->cpu); in rcu_nocb_rdp_deoffload()
1110 int rcu_nocb_cpu_deoffload(int cpu) in rcu_nocb_cpu_deoffload() argument
1112 struct rcu_data *rdp = per_cpu_ptr(&rcu_data, cpu); in rcu_nocb_cpu_deoffload()
1118 if (!cpu_online(cpu)) { in rcu_nocb_cpu_deoffload()
1121 cpumask_clear_cpu(cpu, rcu_nocb_mask); in rcu_nocb_cpu_deoffload()
1123 pr_info("NOCB: Cannot CB-deoffload online CPU %d\n", rdp->cpu); in rcu_nocb_cpu_deoffload()
1150 WARN_ON_ONCE(cpu_online(rdp->cpu)); in rcu_nocb_rdp_offload()
1161 pr_info("Offloading %d\n", rdp->cpu); in rcu_nocb_rdp_offload()
1178 int rcu_nocb_cpu_offload(int cpu) in rcu_nocb_cpu_offload() argument
1180 struct rcu_data *rdp = per_cpu_ptr(&rcu_data, cpu); in rcu_nocb_cpu_offload()
1186 if (!cpu_online(cpu)) { in rcu_nocb_cpu_offload()
1189 cpumask_set_cpu(cpu, rcu_nocb_mask); in rcu_nocb_cpu_offload()
1191 pr_info("NOCB: Cannot CB-offload online CPU %d\n", rdp->cpu); in rcu_nocb_cpu_offload()
1206 int cpu; in lazy_rcu_shrink_count() local
1217 for_each_cpu(cpu, rcu_nocb_mask) { in lazy_rcu_shrink_count()
1218 struct rcu_data *rdp = per_cpu_ptr(&rcu_data, cpu); in lazy_rcu_shrink_count()
1231 int cpu; in lazy_rcu_shrink_scan() local
1252 for_each_cpu(cpu, rcu_nocb_mask) { in lazy_rcu_shrink_scan()
1253 struct rcu_data *rdp = per_cpu_ptr(&rcu_data, cpu); in lazy_rcu_shrink_scan()
1290 int cpu; in rcu_init_nohz() local
1344 for_each_cpu(cpu, rcu_nocb_mask) { in rcu_init_nohz()
1345 rdp = per_cpu_ptr(&rcu_data, cpu); in rcu_init_nohz()
1373 static void rcu_spawn_cpu_nocb_kthread(int cpu) in rcu_spawn_cpu_nocb_kthread() argument
1375 struct rcu_data *rdp = per_cpu_ptr(&rcu_data, cpu); in rcu_spawn_cpu_nocb_kthread()
1393 "rcuog/%d", rdp_gp->cpu); in rcu_spawn_cpu_nocb_kthread()
1406 "rcuo%c/%d", rcu_state.abbr, cpu); in rcu_spawn_cpu_nocb_kthread()
1433 cpumask_clear_cpu(cpu, rcu_nocb_mask); in rcu_spawn_cpu_nocb_kthread()
1447 int cpu; in rcu_organize_nocb_kthreads() local
1468 for_each_possible_cpu(cpu) { in rcu_organize_nocb_kthreads()
1469 rdp = per_cpu_ptr(&rcu_data, cpu); in rcu_organize_nocb_kthreads()
1470 if (rdp->cpu >= nl) { in rcu_organize_nocb_kthreads()
1473 nl = DIV_ROUND_UP(rdp->cpu + 1, ls) * ls; in rcu_organize_nocb_kthreads()
1483 __func__, cpu); in rcu_organize_nocb_kthreads()
1489 pr_cont(" %d", cpu); in rcu_organize_nocb_kthreads()
1492 if (cpumask_test_cpu(cpu, rcu_nocb_mask)) in rcu_organize_nocb_kthreads()
1532 rdp->cpu, in show_rcu_nocb_gp_state()
1581 rdp->cpu, rdp->nocb_gp_rdp->cpu, in show_rcu_nocb_state()
1582 nocb_next_rdp ? nocb_next_rdp->cpu : -1, in show_rcu_nocb_state()
1697 static void rcu_spawn_cpu_nocb_kthread(int cpu) in rcu_spawn_cpu_nocb_kthread() argument