Lines Matching refs:rdp
16 static bool rcu_rdp_is_offloaded(struct rcu_data *rdp) in rcu_rdp_is_offloaded() argument
30 lockdep_is_held(&rdp->nocb_lock) || in rcu_rdp_is_offloaded()
33 rdp == this_cpu_ptr(&rcu_data)) || in rcu_rdp_is_offloaded()
34 rcu_current_is_nocb_kthread(rdp)), in rcu_rdp_is_offloaded()
38 return rcu_segcblist_is_offloaded(&rdp->cblist); in rcu_rdp_is_offloaded()
162 static void rcu_preempt_ctxt_queue(struct rcu_node *rnp, struct rcu_data *rdp) in rcu_preempt_ctxt_queue() argument
167 (rnp->qsmask & rdp->grpmask ? RCU_GP_BLKD : 0) + in rcu_preempt_ctxt_queue()
168 (rnp->expmask & rdp->grpmask ? RCU_EXP_BLKD : 0); in rcu_preempt_ctxt_queue()
172 WARN_ON_ONCE(rdp->mynode != rnp); in rcu_preempt_ctxt_queue()
176 rdp->grpmask); in rcu_preempt_ctxt_queue()
261 !(rnp->qsmask & rdp->grpmask)); in rcu_preempt_ctxt_queue()
263 !(rnp->expmask & rdp->grpmask)); in rcu_preempt_ctxt_queue()
274 if (blkd_state & RCU_EXP_BLKD && rdp->cpu_no_qs.b.exp) in rcu_preempt_ctxt_queue()
275 rcu_report_exp_rdp(rdp); in rcu_preempt_ctxt_queue()
277 WARN_ON_ONCE(rdp->cpu_no_qs.b.exp); in rcu_preempt_ctxt_queue()
326 struct rcu_data *rdp = this_cpu_ptr(&rcu_data); in rcu_note_context_switch() local
336 rnp = rdp->mynode; in rcu_note_context_switch()
346 WARN_ON_ONCE(!rcu_rdp_cpu_online(rdp)); in rcu_note_context_switch()
350 (rnp->qsmask & rdp->grpmask) in rcu_note_context_switch()
353 rcu_preempt_ctxt_queue(rnp, rdp); in rcu_note_context_switch()
368 if (rdp->cpu_no_qs.b.exp) in rcu_note_context_switch()
369 rcu_report_exp_rdp(rdp); in rcu_note_context_switch()
484 struct rcu_data *rdp; in rcu_preempt_deferred_qs_irqrestore() local
494 rdp = this_cpu_ptr(&rcu_data); in rcu_preempt_deferred_qs_irqrestore()
495 if (!special.s && !rdp->cpu_no_qs.b.exp) { in rcu_preempt_deferred_qs_irqrestore()
502 rdp->cpu_no_qs.b.norm = false; in rcu_preempt_deferred_qs_irqrestore()
503 rcu_report_qs_rdp(rdp); in rcu_preempt_deferred_qs_irqrestore()
516 if (rdp->cpu_no_qs.b.exp) in rcu_preempt_deferred_qs_irqrestore()
517 rcu_report_exp_rdp(rdp); in rcu_preempt_deferred_qs_irqrestore()
626 struct rcu_data *rdp; in rcu_preempt_deferred_qs_handler() local
628 rdp = container_of(iwp, struct rcu_data, defer_qs_iw); in rcu_preempt_deferred_qs_handler()
629 rdp->defer_qs_iw_pending = false; in rcu_preempt_deferred_qs_handler()
652 struct rcu_data *rdp = this_cpu_ptr(&rcu_data); in rcu_read_unlock_special() local
653 struct rcu_node *rnp = rdp->mynode; in rcu_read_unlock_special()
656 (rdp->grpmask & READ_ONCE(rnp->expmask)) || in rcu_read_unlock_special()
658 ((rdp->grpmask & READ_ONCE(rnp->qsmask)) || t->rcu_blocked_node)) || in rcu_read_unlock_special()
675 expboost && !rdp->defer_qs_iw_pending && cpu_online(rdp->cpu)) { in rcu_read_unlock_special()
680 rdp->defer_qs_iw = IRQ_WORK_INIT_HARD( in rcu_read_unlock_special()
683 init_irq_work(&rdp->defer_qs_iw, in rcu_read_unlock_special()
685 rdp->defer_qs_iw_pending = true; in rcu_read_unlock_special()
686 irq_work_queue_on(&rdp->defer_qs_iw, rdp->cpu); in rcu_read_unlock_special()
794 struct rcu_data *rdp; in dump_blkd_tasks() local
816 rdp = per_cpu_ptr(&rcu_data, cpu); in dump_blkd_tasks()
818 cpu, ".o"[rcu_rdp_cpu_online(rdp)], in dump_blkd_tasks()
819 (long)rdp->rcu_onl_gp_seq, rdp->rcu_onl_gp_state, in dump_blkd_tasks()
820 (long)rdp->rcu_ofl_gp_seq, rdp->rcu_ofl_gp_state); in dump_blkd_tasks()
833 struct rcu_data *rdp; in rcu_read_unlock_strict() local
837 rdp = this_cpu_ptr(&rcu_data); in rcu_read_unlock_strict()
838 rdp->cpu_no_qs.b.norm = false; in rcu_read_unlock_strict()
839 rcu_report_qs_rdp(rdp); in rcu_read_unlock_strict()
955 struct rcu_data *rdp = this_cpu_ptr(&rcu_data); in rcu_preempt_deferred_qs() local
957 if (READ_ONCE(rdp->cpu_no_qs.b.exp)) in rcu_preempt_deferred_qs()
958 rcu_report_exp_rdp(rdp); in rcu_preempt_deferred_qs()
1018 struct rcu_data *rdp = per_cpu_ptr(&rcu_data, cpu); in rcu_cpu_kthread_setup() local
1026 WRITE_ONCE(rdp->rcuc_activity, jiffies); in rcu_cpu_kthread_setup()
1029 static bool rcu_is_callbacks_nocb_kthread(struct rcu_data *rdp) in rcu_is_callbacks_nocb_kthread() argument
1032 return rdp->nocb_cb_kthread == current; in rcu_is_callbacks_nocb_kthread()
1042 static bool rcu_is_callbacks_kthread(struct rcu_data *rdp) in rcu_is_callbacks_kthread() argument
1044 return rdp->rcu_cpu_kthread_task == current || in rcu_is_callbacks_kthread()
1045 rcu_is_callbacks_nocb_kthread(rdp); in rcu_is_callbacks_kthread()