Lines Matching refs:cblist
345 rcu_segcblist_inc_len(&rdp->cblist); /* Must precede enqueue. */ in rcu_nocb_do_flush_bypass()
360 rcu_segcblist_insert_pend_cbs(&rdp->cblist, &rcl); in rcu_nocb_do_flush_bypass()
430 *was_alldone = !rcu_segcblist_pend_cbs(&rdp->cblist); in rcu_nocb_try_bypass()
436 if (!rcu_segcblist_completely_offloaded(&rdp->cblist)) { in rcu_nocb_try_bypass()
438 *was_alldone = !rcu_segcblist_pend_cbs(&rdp->cblist); in rcu_nocb_try_bypass()
446 *was_alldone = !rcu_segcblist_pend_cbs(&rdp->cblist); in rcu_nocb_try_bypass()
471 *was_alldone = !rcu_segcblist_pend_cbs(&rdp->cblist); in rcu_nocb_try_bypass()
488 *was_alldone = !rcu_segcblist_pend_cbs(&rdp->cblist); in rcu_nocb_try_bypass()
498 rcu_segcblist_nextgp(&rdp->cblist, &cur_gp_seq) && in rcu_nocb_try_bypass()
516 rcu_segcblist_inc_len(&rdp->cblist); /* Must precede enqueue. */ in rcu_nocb_try_bypass()
540 if (!rcu_segcblist_pend_cbs(&rdp->cblist)) { in rcu_nocb_try_bypass()
579 len = rcu_segcblist_n_cbs(&rdp->cblist); in __call_rcu_nocb_wake()
605 rcu_segcblist_nextgp(&rdp->cblist, &cur_gp_seq) && in __call_rcu_nocb_wake()
612 !rcu_segcblist_ready_cbs(&rdp->cblist)) && in __call_rcu_nocb_wake()
630 struct rcu_segcblist *cblist = &rdp->cblist; in nocb_gp_toggle_rdp() local
635 if (rcu_segcblist_test_flags(cblist, SEGCBLIST_OFFLOADED) && in nocb_gp_toggle_rdp()
636 !rcu_segcblist_test_flags(cblist, SEGCBLIST_KTHREAD_GP)) { in nocb_gp_toggle_rdp()
641 rcu_segcblist_set_flags(cblist, SEGCBLIST_KTHREAD_GP); in nocb_gp_toggle_rdp()
642 if (rcu_segcblist_test_flags(cblist, SEGCBLIST_KTHREAD_CB)) in nocb_gp_toggle_rdp()
645 } else if (!rcu_segcblist_test_flags(cblist, SEGCBLIST_OFFLOADED) && in nocb_gp_toggle_rdp()
646 rcu_segcblist_test_flags(cblist, SEGCBLIST_KTHREAD_GP)) { in nocb_gp_toggle_rdp()
651 rcu_segcblist_clear_flags(cblist, SEGCBLIST_KTHREAD_GP); in nocb_gp_toggle_rdp()
652 if (!rcu_segcblist_test_flags(cblist, SEGCBLIST_KTHREAD_CB)) in nocb_gp_toggle_rdp()
733 } else if (!bypass_ncbs && rcu_segcblist_empty(&rdp->cblist)) { in nocb_gp_wait()
757 if (!rcu_segcblist_restempty(&rdp->cblist, in nocb_gp_wait()
759 (rcu_segcblist_nextgp(&rdp->cblist, &cur_gp_seq) && in nocb_gp_wait()
763 wasempty = rcu_segcblist_restempty(&rdp->cblist, in nocb_gp_wait()
769 !rcu_segcblist_restempty(&rdp->cblist, in nocb_gp_wait()
771 if (rcu_segcblist_nextgp(&rdp->cblist, &cur_gp_seq)) { in nocb_gp_wait()
779 if (rcu_segcblist_ready_cbs(&rdp->cblist)) { in nocb_gp_wait()
910 return rcu_segcblist_test_flags(&rdp->cblist, flags); in nocb_cb_can_run()
924 struct rcu_segcblist *cblist = &rdp->cblist; in nocb_cb_wait() local
958 if (rcu_segcblist_nextgp(cblist, &cur_gp_seq) && in nocb_cb_wait()
965 if (rcu_segcblist_test_flags(cblist, SEGCBLIST_OFFLOADED)) { in nocb_cb_wait()
966 if (!rcu_segcblist_test_flags(cblist, SEGCBLIST_KTHREAD_CB)) { in nocb_cb_wait()
967 rcu_segcblist_set_flags(cblist, SEGCBLIST_KTHREAD_CB); in nocb_cb_wait()
968 if (rcu_segcblist_test_flags(cblist, SEGCBLIST_KTHREAD_GP)) in nocb_cb_wait()
971 if (rcu_segcblist_ready_cbs(cblist)) in nocb_cb_wait()
979 WARN_ON_ONCE(!rcu_segcblist_test_flags(cblist, SEGCBLIST_KTHREAD_CB)); in nocb_cb_wait()
980 rcu_segcblist_clear_flags(cblist, SEGCBLIST_KTHREAD_CB); in nocb_cb_wait()
981 if (!rcu_segcblist_test_flags(cblist, SEGCBLIST_KTHREAD_GP)) in nocb_cb_wait()
1083 struct rcu_segcblist *cblist = &rdp->cblist; in rdp_offload_toggle() local
1087 rcu_segcblist_offload(cblist, offload); in rdp_offload_toggle()
1114 struct rcu_segcblist *cblist = &rdp->cblist; in rcu_nocb_rdp_deoffload() local
1146 rcu_segcblist_set_flags(cblist, SEGCBLIST_RCU_CORE); in rcu_nocb_rdp_deoffload()
1161 rcu_segcblist_clear_flags(&rdp->cblist, SEGCBLIST_KTHREAD_CB); in rcu_nocb_rdp_deoffload()
1166 !rcu_segcblist_test_flags(cblist, in rcu_nocb_rdp_deoffload()
1175 rcu_segcblist_clear_flags(&rdp->cblist, in rcu_nocb_rdp_deoffload()
1192 rcu_segcblist_clear_flags(cblist, SEGCBLIST_LOCKING); in rcu_nocb_rdp_deoffload()
1233 struct rcu_segcblist *cblist = &rdp->cblist; in rcu_nocb_rdp_offload() local
1277 rcu_segcblist_test_flags(cblist, SEGCBLIST_KTHREAD_CB) && in rcu_nocb_rdp_offload()
1278 rcu_segcblist_test_flags(cblist, SEGCBLIST_KTHREAD_GP)); in rcu_nocb_rdp_offload()
1285 rcu_segcblist_clear_flags(cblist, SEGCBLIST_RCU_CORE); in rcu_nocb_rdp_offload()
1412 if (rcu_segcblist_empty(&rdp->cblist)) in rcu_init_nohz()
1413 rcu_segcblist_init(&rdp->cblist); in rcu_init_nohz()
1414 rcu_segcblist_offload(&rdp->cblist, true); in rcu_init_nohz()
1415 rcu_segcblist_set_flags(&rdp->cblist, SEGCBLIST_KTHREAD_CB | SEGCBLIST_KTHREAD_GP); in rcu_init_nohz()
1416 rcu_segcblist_clear_flags(&rdp->cblist, SEGCBLIST_RCU_CORE); in rcu_init_nohz()
1611 struct rcu_segcblist *rsclp = &rdp->cblist; in show_rcu_nocb_state()
1644 rcu_segcblist_n_cbs(&rdp->cblist), in show_rcu_nocb_state()