Lines Matching refs:sdp
89 struct srcu_data *sdp; in init_srcu_struct_nodes() local
132 WARN_ON_ONCE(ARRAY_SIZE(sdp->srcu_lock_count) != in init_srcu_struct_nodes()
133 ARRAY_SIZE(sdp->srcu_unlock_count)); in init_srcu_struct_nodes()
137 sdp = per_cpu_ptr(ssp->sda, cpu); in init_srcu_struct_nodes()
138 spin_lock_init(&ACCESS_PRIVATE(sdp, lock)); in init_srcu_struct_nodes()
139 rcu_segcblist_init(&sdp->srcu_cblist); in init_srcu_struct_nodes()
140 sdp->srcu_cblist_invoking = false; in init_srcu_struct_nodes()
141 sdp->srcu_gp_seq_needed = ssp->srcu_gp_seq; in init_srcu_struct_nodes()
142 sdp->srcu_gp_seq_needed_exp = ssp->srcu_gp_seq; in init_srcu_struct_nodes()
143 sdp->mynode = &snp_first[cpu / levelspread[level]]; in init_srcu_struct_nodes()
144 for (snp = sdp->mynode; snp != NULL; snp = snp->srcu_parent) { in init_srcu_struct_nodes()
149 sdp->cpu = cpu; in init_srcu_struct_nodes()
150 INIT_WORK(&sdp->work, srcu_invoke_callbacks); in init_srcu_struct_nodes()
151 timer_setup(&sdp->delay_work, srcu_delay_timer, 0); in init_srcu_struct_nodes()
152 sdp->ssp = ssp; in init_srcu_struct_nodes()
153 sdp->grpmask = 1 << (cpu - sdp->mynode->grplo); in init_srcu_struct_nodes()
377 struct srcu_data *sdp = per_cpu_ptr(ssp->sda, cpu); in cleanup_srcu_struct() local
379 del_timer_sync(&sdp->delay_work); in cleanup_srcu_struct()
380 flush_work(&sdp->work); in cleanup_srcu_struct()
381 if (WARN_ON(rcu_segcblist_n_cbs(&sdp->srcu_cblist))) in cleanup_srcu_struct()
437 struct srcu_data *sdp = this_cpu_ptr(ssp->sda); in srcu_gp_start() local
442 spin_lock_rcu_node(sdp); /* Interrupts already disabled. */ in srcu_gp_start()
443 rcu_segcblist_advance(&sdp->srcu_cblist, in srcu_gp_start()
445 (void)rcu_segcblist_accelerate(&sdp->srcu_cblist, in srcu_gp_start()
447 spin_unlock_rcu_node(sdp); /* Interrupts remain disabled. */ in srcu_gp_start()
457 struct srcu_data *sdp = container_of(t, struct srcu_data, delay_work); in srcu_delay_timer() local
459 queue_work_on(sdp->cpu, rcu_gp_wq, &sdp->work); in srcu_delay_timer()
462 static void srcu_queue_delayed_work_on(struct srcu_data *sdp, in srcu_queue_delayed_work_on() argument
466 queue_work_on(sdp->cpu, rcu_gp_wq, &sdp->work); in srcu_queue_delayed_work_on()
470 timer_reduce(&sdp->delay_work, jiffies + delay); in srcu_queue_delayed_work_on()
477 static void srcu_schedule_cbs_sdp(struct srcu_data *sdp, unsigned long delay) in srcu_schedule_cbs_sdp() argument
479 srcu_queue_delayed_work_on(sdp, delay); in srcu_schedule_cbs_sdp()
519 struct srcu_data *sdp; in srcu_gp_end() local
560 sdp = per_cpu_ptr(ssp->sda, cpu); in srcu_gp_end()
561 spin_lock_irqsave_rcu_node(sdp, flags); in srcu_gp_end()
563 sdp->srcu_gp_seq_needed + 100)) in srcu_gp_end()
564 sdp->srcu_gp_seq_needed = gpseq; in srcu_gp_end()
566 sdp->srcu_gp_seq_needed_exp + 100)) in srcu_gp_end()
567 sdp->srcu_gp_seq_needed_exp = gpseq; in srcu_gp_end()
568 spin_unlock_irqrestore_rcu_node(sdp, flags); in srcu_gp_end()
628 static void srcu_funnel_gp_start(struct srcu_struct *ssp, struct srcu_data *sdp, in srcu_funnel_gp_start() argument
632 int idx = rcu_seq_ctr(s) % ARRAY_SIZE(sdp->mynode->srcu_have_cbs); in srcu_funnel_gp_start()
633 struct srcu_node *snp = sdp->mynode; in srcu_funnel_gp_start()
638 if (rcu_seq_done(&ssp->srcu_gp_seq, s) && snp != sdp->mynode) in srcu_funnel_gp_start()
643 if (snp == sdp->mynode && snp_seq == s) in srcu_funnel_gp_start()
644 snp->srcu_data_have_cbs[idx] |= sdp->grpmask; in srcu_funnel_gp_start()
646 if (snp == sdp->mynode && snp_seq != s) { in srcu_funnel_gp_start()
647 srcu_schedule_cbs_sdp(sdp, do_norm in srcu_funnel_gp_start()
657 if (snp == sdp->mynode) in srcu_funnel_gp_start()
658 snp->srcu_data_have_cbs[idx] |= sdp->grpmask; in srcu_funnel_gp_start()
760 struct srcu_data *sdp; in srcu_might_be_idle() local
766 sdp = raw_cpu_ptr(ssp->sda); in srcu_might_be_idle()
767 spin_lock_irqsave_rcu_node(sdp, flags); in srcu_might_be_idle()
768 if (rcu_segcblist_pend_cbs(&sdp->srcu_cblist)) { in srcu_might_be_idle()
769 spin_unlock_irqrestore_rcu_node(sdp, flags); in srcu_might_be_idle()
772 spin_unlock_irqrestore_rcu_node(sdp, flags); in srcu_might_be_idle()
816 struct srcu_data *sdp; in srcu_gp_start_if_needed() local
820 sdp = raw_cpu_ptr(ssp->sda); in srcu_gp_start_if_needed()
821 spin_lock_irqsave_rcu_node(sdp, flags); in srcu_gp_start_if_needed()
823 rcu_segcblist_enqueue(&sdp->srcu_cblist, rhp); in srcu_gp_start_if_needed()
824 rcu_segcblist_advance(&sdp->srcu_cblist, in srcu_gp_start_if_needed()
827 (void)rcu_segcblist_accelerate(&sdp->srcu_cblist, s); in srcu_gp_start_if_needed()
828 if (ULONG_CMP_LT(sdp->srcu_gp_seq_needed, s)) { in srcu_gp_start_if_needed()
829 sdp->srcu_gp_seq_needed = s; in srcu_gp_start_if_needed()
832 if (!do_norm && ULONG_CMP_LT(sdp->srcu_gp_seq_needed_exp, s)) { in srcu_gp_start_if_needed()
833 sdp->srcu_gp_seq_needed_exp = s; in srcu_gp_start_if_needed()
836 spin_unlock_irqrestore_rcu_node(sdp, flags); in srcu_gp_start_if_needed()
838 srcu_funnel_gp_start(ssp, sdp, s, do_norm); in srcu_gp_start_if_needed()
840 srcu_funnel_exp_start(ssp, sdp->mynode, s); in srcu_gp_start_if_needed()
1091 struct srcu_data *sdp; in srcu_barrier_cb() local
1094 sdp = container_of(rhp, struct srcu_data, srcu_barrier_head); in srcu_barrier_cb()
1095 ssp = sdp->ssp; in srcu_barrier_cb()
1107 struct srcu_data *sdp; in srcu_barrier() local
1132 sdp = per_cpu_ptr(ssp->sda, cpu); in srcu_barrier()
1133 spin_lock_irq_rcu_node(sdp); in srcu_barrier()
1135 sdp->srcu_barrier_head.func = srcu_barrier_cb; in srcu_barrier()
1136 debug_rcu_head_queue(&sdp->srcu_barrier_head); in srcu_barrier()
1137 if (!rcu_segcblist_entrain(&sdp->srcu_cblist, in srcu_barrier()
1138 &sdp->srcu_barrier_head)) { in srcu_barrier()
1139 debug_rcu_head_unqueue(&sdp->srcu_barrier_head); in srcu_barrier()
1142 spin_unlock_irq_rcu_node(sdp); in srcu_barrier()
1247 struct srcu_data *sdp; in srcu_invoke_callbacks() local
1250 sdp = container_of(work, struct srcu_data, work); in srcu_invoke_callbacks()
1252 ssp = sdp->ssp; in srcu_invoke_callbacks()
1254 spin_lock_irq_rcu_node(sdp); in srcu_invoke_callbacks()
1255 rcu_segcblist_advance(&sdp->srcu_cblist, in srcu_invoke_callbacks()
1257 if (sdp->srcu_cblist_invoking || in srcu_invoke_callbacks()
1258 !rcu_segcblist_ready_cbs(&sdp->srcu_cblist)) { in srcu_invoke_callbacks()
1259 spin_unlock_irq_rcu_node(sdp); in srcu_invoke_callbacks()
1264 sdp->srcu_cblist_invoking = true; in srcu_invoke_callbacks()
1265 rcu_segcblist_extract_done_cbs(&sdp->srcu_cblist, &ready_cbs); in srcu_invoke_callbacks()
1267 spin_unlock_irq_rcu_node(sdp); in srcu_invoke_callbacks()
1281 spin_lock_irq_rcu_node(sdp); in srcu_invoke_callbacks()
1282 rcu_segcblist_add_len(&sdp->srcu_cblist, -len); in srcu_invoke_callbacks()
1283 (void)rcu_segcblist_accelerate(&sdp->srcu_cblist, in srcu_invoke_callbacks()
1285 sdp->srcu_cblist_invoking = false; in srcu_invoke_callbacks()
1286 more = rcu_segcblist_ready_cbs(&sdp->srcu_cblist); in srcu_invoke_callbacks()
1287 spin_unlock_irq_rcu_node(sdp); in srcu_invoke_callbacks()
1289 srcu_schedule_cbs_sdp(sdp, 0); in srcu_invoke_callbacks()
1353 struct srcu_data *sdp; in srcu_torture_stats_print() local
1355 sdp = per_cpu_ptr(ssp->sda, cpu); in srcu_torture_stats_print()
1356 u0 = data_race(sdp->srcu_unlock_count[!idx]); in srcu_torture_stats_print()
1357 u1 = data_race(sdp->srcu_unlock_count[idx]); in srcu_torture_stats_print()
1365 l0 = data_race(sdp->srcu_lock_count[!idx]); in srcu_torture_stats_print()
1366 l1 = data_race(sdp->srcu_lock_count[idx]); in srcu_torture_stats_print()
1372 "C."[rcu_segcblist_empty(&sdp->srcu_cblist)]); in srcu_torture_stats_print()