Lines Matching refs:rcu_state
103 return !time_before(j, READ_ONCE(rcu_state.gp_start) + d); in rcu_gp_might_be_stalled()
156 WRITE_ONCE(rcu_state.jiffies_stall, in rcu_cpu_stall_reset()
170 WRITE_ONCE(rcu_state.gp_start, j); in record_gp_stall_check_time()
173 WRITE_ONCE(rcu_state.jiffies_stall, j + j1); in record_gp_stall_check_time()
174 rcu_state.jiffies_resched = j + j1 / 2; in record_gp_stall_check_time()
175 rcu_state.n_force_qs_gpstart = READ_ONCE(rcu_state.n_force_qs); in record_gp_stall_check_time()
196 j = READ_ONCE(rcu_state.jiffies_kick_kthreads); in rcu_stall_kick_kthreads()
197 if (time_after(jiffies, j) && rcu_state.gp_kthread && in rcu_stall_kick_kthreads()
198 (rcu_gp_in_progress() || READ_ONCE(rcu_state.gp_flags))) { in rcu_stall_kick_kthreads()
200 rcu_state.name); in rcu_stall_kick_kthreads()
202 wake_up_process(rcu_state.gp_kthread); in rcu_stall_kick_kthreads()
203 WRITE_ONCE(rcu_state.jiffies_kick_kthreads, j + HZ); in rcu_stall_kick_kthreads()
403 unsigned long j = jiffies - READ_ONCE(rcu_state.gp_activity); in rcu_is_gp_kthread_starving()
488 ticks_value = rcu_seq_ctr(rcu_state.gp_seq - rdp->gp_seq); in print_cpu_stall_info()
513 data_race(rcu_state.n_force_qs) - rcu_state.n_force_qs_gpstart, in print_cpu_stall_info()
524 struct task_struct *gpk = rcu_state.gp_kthread; in rcu_check_gp_kthread_starvation()
530 rcu_state.name, j, in rcu_check_gp_kthread_starvation()
531 (long)rcu_seq_current(&rcu_state.gp_seq), in rcu_check_gp_kthread_starvation()
532 data_race(READ_ONCE(rcu_state.gp_flags)), in rcu_check_gp_kthread_starvation()
533 gp_state_getname(rcu_state.gp_state), in rcu_check_gp_kthread_starvation()
534 data_race(READ_ONCE(rcu_state.gp_state)), in rcu_check_gp_kthread_starvation()
537 …r("\tUnless %s kthread gets sufficient CPU time, OOM is now expected behavior.\n", rcu_state.name); in rcu_check_gp_kthread_starvation()
556 struct task_struct *gpk = rcu_state.gp_kthread; in rcu_check_gp_kthread_expired_fqs_timer()
565 gp_state = smp_load_acquire(&rcu_state.gp_state); in rcu_check_gp_kthread_expired_fqs_timer()
566 jiffies_fqs = READ_ONCE(rcu_state.jiffies_force_qs); in rcu_check_gp_kthread_expired_fqs_timer()
573 rcu_state.name, (jiffies - jiffies_fqs), in rcu_check_gp_kthread_expired_fqs_timer()
574 (long)rcu_seq_current(&rcu_state.gp_seq), in rcu_check_gp_kthread_expired_fqs_timer()
575 data_race(rcu_state.gp_flags), in rcu_check_gp_kthread_expired_fqs_timer()
605 trace_rcu_stall_warning(rcu_state.name, TPS("StallDetected")); in print_other_cpu_stall()
606 pr_err("INFO: %s detected stalls on CPUs/tasks:\n", rcu_state.name); in print_other_cpu_stall()
624 (long)rcu_seq_current(&rcu_state.gp_seq), totqlen, rcu_state.n_online_cpus); in print_other_cpu_stall()
632 if (rcu_seq_current(&rcu_state.gp_seq) != gp_seq) { in print_other_cpu_stall()
636 gpa = data_race(READ_ONCE(rcu_state.gp_activity)); in print_other_cpu_stall()
638 rcu_state.name, j - gpa, j, gpa, in print_other_cpu_stall()
644 if (ULONG_CMP_GE(jiffies, READ_ONCE(rcu_state.jiffies_stall))) in print_other_cpu_stall()
645 WRITE_ONCE(rcu_state.jiffies_stall, in print_other_cpu_stall()
676 trace_rcu_stall_warning(rcu_state.name, TPS("SelfDetected")); in print_cpu_stall()
677 pr_err("INFO: %s self-detected stall on CPU\n", rcu_state.name); in print_cpu_stall()
685 (long)rcu_seq_current(&rcu_state.gp_seq), totqlen, rcu_state.n_online_cpus); in print_cpu_stall()
694 if (ULONG_CMP_GE(jiffies, READ_ONCE(rcu_state.jiffies_stall))) in print_cpu_stall()
695 WRITE_ONCE(rcu_state.jiffies_stall, in print_cpu_stall()
748 gs1 = READ_ONCE(rcu_state.gp_seq); in check_cpu_stall()
750 js = READ_ONCE(rcu_state.jiffies_stall); in check_cpu_stall()
752 gps = READ_ONCE(rcu_state.gp_start); in check_cpu_stall()
754 gs2 = READ_ONCE(rcu_state.gp_seq); in check_cpu_stall()
763 cmpxchg(&rcu_state.jiffies_stall, js, jn) == js) { in check_cpu_stall()
781 cmpxchg(&rcu_state.jiffies_stall, js, jn) == js) { in check_cpu_stall()
797 if (didstall && READ_ONCE(rcu_state.jiffies_stall) == jn) { in check_cpu_stall()
799 WRITE_ONCE(rcu_state.jiffies_stall, jn); in check_cpu_stall()
879 struct task_struct *t = READ_ONCE(rcu_state.gp_kthread); in show_rcu_gp_kthreads()
882 ja = j - data_race(READ_ONCE(rcu_state.gp_activity)); in show_rcu_gp_kthreads()
883 jr = j - data_race(READ_ONCE(rcu_state.gp_req_activity)); in show_rcu_gp_kthreads()
884 js = j - data_race(READ_ONCE(rcu_state.gp_start)); in show_rcu_gp_kthreads()
885 jw = j - data_race(READ_ONCE(rcu_state.gp_wake_time)); in show_rcu_gp_kthreads()
887 rcu_state.name, gp_state_getname(rcu_state.gp_state), in show_rcu_gp_kthreads()
888 data_race(READ_ONCE(rcu_state.gp_state)), in show_rcu_gp_kthreads()
890 js, ja, jr, jw, (long)data_race(READ_ONCE(rcu_state.gp_wake_seq)), in show_rcu_gp_kthreads()
891 (long)data_race(READ_ONCE(rcu_state.gp_seq)), in show_rcu_gp_kthreads()
893 data_race(READ_ONCE(rcu_state.gp_max)), in show_rcu_gp_kthreads()
894 data_race(READ_ONCE(rcu_state.gp_flags))); in show_rcu_gp_kthreads()
896 if (ULONG_CMP_GE(READ_ONCE(rcu_state.gp_seq), READ_ONCE(rnp->gp_seq_needed)) && in show_rcu_gp_kthreads()
915 ULONG_CMP_GE(READ_ONCE(rcu_state.gp_seq), in show_rcu_gp_kthreads()
948 !smp_load_acquire(&rcu_state.gp_kthread)) // Get stable kthread. in rcu_check_gp_start_stall()
951 if (time_before(j, READ_ONCE(rcu_state.gp_req_activity) + gpssdelay) || in rcu_check_gp_start_stall()
952 time_before(j, READ_ONCE(rcu_state.gp_activity) + gpssdelay) || in rcu_check_gp_start_stall()
961 time_before(j, READ_ONCE(rcu_state.gp_req_activity) + gpssdelay) || in rcu_check_gp_start_stall()
962 time_before(j, READ_ONCE(rcu_state.gp_activity) + gpssdelay) || in rcu_check_gp_start_stall()
975 time_before(j, READ_ONCE(rcu_state.gp_req_activity) + gpssdelay) || in rcu_check_gp_start_stall()
976 time_before(j, READ_ONCE(rcu_state.gp_activity) + gpssdelay) || in rcu_check_gp_start_stall()
1006 __func__, jiffies - data_race(READ_ONCE(rcu_state.gp_start))); in rcu_fwd_progress_check()
1010 __func__, jiffies - data_race(READ_ONCE(rcu_state.gp_end))); in rcu_fwd_progress_check()