Lines Matching refs:cur_ops
352 static struct rcu_torture_ops *cur_ops; variable
384 started = cur_ops->get_gp_seq(); in rcu_read_delay()
390 completed = cur_ops->get_gp_seq(); in rcu_read_delay()
391 do_trace_rcu_torture_read(cur_ops->name, NULL, ts, in rcu_read_delay()
468 cur_ops->deferred_free(rp); in rcu_torture_cb()
883 if (!cur_ops->gp_diff) in rcutorture_seq_diff()
885 return cur_ops->gp_diff(new, old); in rcutorture_seq_diff()
935 if (cur_ops->poll_gp_state(gp_state)) in rcu_torture_boost_failed()
937 if (cur_ops->check_boost_failed && !cur_ops->check_boost_failed(gp_state, &cpu)) { in rcu_torture_boost_failed()
947 if (!xchg(&dbg_done, 1) && cur_ops->gp_kthread_dbg) { in rcu_torture_boost_failed()
950 cur_ops->gp_kthread_dbg(); in rcu_torture_boost_failed()
952 gp_done = cur_ops->poll_gp_state(gp_state); in rcu_torture_boost_failed()
959 } else if (cur_ops->check_boost_failed && !cur_ops->check_boost_failed(gp_state, NULL)) { in rcu_torture_boost_failed()
1000 if (gp_initiated && !failed && !cur_ops->poll_gp_state(gp_state)) in rcu_torture_boost()
1003 if (!gp_initiated || cur_ops->poll_gp_state(gp_state)) { in rcu_torture_boost()
1004 gp_state = cur_ops->start_gp_poll(); in rcu_torture_boost()
1013 if (cur_ops->poll_gp_state(gp_state)) in rcu_torture_boost()
1021 if (gp_initiated && !failed && !cur_ops->poll_gp_state(gp_state)) in rcu_torture_boost()
1079 cur_ops->fqs(); in rcu_torture_fqs()
1106 if (gp_cond1 && cur_ops->get_gp_state && cur_ops->cond_sync) { in rcu_torture_write_types()
1109 } else if (gp_cond && (!cur_ops->get_gp_state || !cur_ops->cond_sync)) { in rcu_torture_write_types()
1112 if (gp_exp1 && cur_ops->exp_sync) { in rcu_torture_write_types()
1115 } else if (gp_exp && !cur_ops->exp_sync) { in rcu_torture_write_types()
1118 if (gp_normal1 && cur_ops->deferred_free) { in rcu_torture_write_types()
1121 } else if (gp_normal && !cur_ops->deferred_free) { in rcu_torture_write_types()
1124 if (gp_poll1 && cur_ops->start_gp_poll && cur_ops->poll_gp_state) { in rcu_torture_write_types()
1127 } else if (gp_poll && (!cur_ops->start_gp_poll || !cur_ops->poll_gp_state)) { in rcu_torture_write_types()
1130 if (gp_sync1 && cur_ops->sync) { in rcu_torture_write_types()
1133 } else if (gp_sync && !cur_ops->sync) { in rcu_torture_write_types()
1163 torture_type, cur_ops->name); in rcu_torture_writer()
1197 if (cur_ops->get_gp_state && cur_ops->poll_gp_state) { in rcu_torture_writer()
1198 idx = cur_ops->readlock(); in rcu_torture_writer()
1199 cookie = cur_ops->get_gp_state(); in rcu_torture_writer()
1201 cur_ops->poll_gp_state(cookie), in rcu_torture_writer()
1206 cookie, cur_ops->get_gp_state()); in rcu_torture_writer()
1207 cur_ops->readunlock(idx); in rcu_torture_writer()
1212 cur_ops->deferred_free(old_rp); in rcu_torture_writer()
1216 cur_ops->exp_sync(); in rcu_torture_writer()
1221 gp_snap = cur_ops->get_gp_state(); in rcu_torture_writer()
1224 cur_ops->cond_sync(gp_snap); in rcu_torture_writer()
1229 gp_snap = cur_ops->start_gp_poll(); in rcu_torture_writer()
1231 while (!cur_ops->poll_gp_state(gp_snap)) in rcu_torture_writer()
1238 cur_ops->sync(); in rcu_torture_writer()
1267 !cur_ops->slow_gps && in rcu_torture_writer()
1311 if (cur_ops->cb_barrier != NULL && in rcu_torture_fakewriter()
1313 cur_ops->cb_barrier(); in rcu_torture_fakewriter()
1319 cur_ops->exp_sync(); in rcu_torture_fakewriter()
1322 gp_snap = cur_ops->get_gp_state(); in rcu_torture_fakewriter()
1324 cur_ops->cond_sync(gp_snap); in rcu_torture_fakewriter()
1327 gp_snap = cur_ops->start_gp_poll(); in rcu_torture_fakewriter()
1328 while (!cur_ops->poll_gp_state(gp_snap)) { in rcu_torture_fakewriter()
1334 cur_ops->sync(); in rcu_torture_fakewriter()
1444 idxnew = cur_ops->readlock() << RCUTORTURE_RDR_SHIFT; in rcutorture_one_extend()
1468 cur_ops->readunlock(idxold >> RCUTORTURE_RDR_SHIFT); in rcutorture_one_extend()
1475 cur_ops->read_delay(trsp, rtrsp); in rcutorture_one_extend()
1493 mask = extendables & RCUTORTURE_MAX_EXTEND & cur_ops->extendables; in rcutorture_extend_mask_max()
1586 if (cur_ops->get_gp_state && cur_ops->poll_gp_state) in rcu_torture_one_read()
1587 cookie = cur_ops->get_gp_state(); in rcu_torture_one_read()
1588 started = cur_ops->get_gp_seq(); in rcu_torture_one_read()
1591 !cur_ops->readlock_held || cur_ops->readlock_held()); in rcu_torture_one_read()
1607 completed = cur_ops->get_gp_seq(); in rcu_torture_one_read()
1609 do_trace_rcu_torture_read(cur_ops->name, &p->rtort_rcu, in rcu_torture_one_read()
1621 if (cur_ops->get_gp_state && cur_ops->poll_gp_state) in rcu_torture_one_read()
1622 WARN_ONCE(cur_ops->poll_gp_state(cookie), in rcu_torture_one_read()
1627 cookie, cur_ops->get_gp_state()); in rcu_torture_one_read()
1659 if (cur_ops->call) { in rcu_torture_timer()
1663 cur_ops->call(rhp, rcu_torture_timer_cb); in rcu_torture_timer()
1684 if (irqreader && cur_ops->irq_capable) in rcu_torture_reader()
1688 if (irqreader && cur_ops->irq_capable) { in rcu_torture_reader()
1702 if (irqreader && cur_ops->irq_capable) { in rcu_torture_reader()
1852 if (cur_ops->stats) in rcu_torture_stats_print()
1853 cur_ops->stats(); in rcu_torture_stats_print()
1860 rcutorture_get_gp_data(cur_ops->ttype, in rcu_torture_stats_print()
1862 srcutorture_get_gp_data(cur_ops->ttype, srcu_ctlp, in rcu_torture_stats_print()
1874 if (cur_ops->gp_kthread_dbg) in rcu_torture_stats_print()
1875 cur_ops->gp_kthread_dbg(); in rcu_torture_stats_print()
1939 rcu_torture_print_module_parms(struct rcu_torture_ops *cur_ops, const char *tag) in rcu_torture_print_module_parms() argument
1957 test_boost, cur_ops->can_boost, in rcu_torture_print_module_parms()
2040 idx = cur_ops->readlock(); in rcu_torture_stall()
2060 cur_ops->readunlock(idx); in rcu_torture_stall()
2096 cur_ops->call(&fcsp->rh, rcu_torture_fwd_prog_cb); in rcu_torture_fwd_prog_cb()
2176 rfp->n_launders_hist[i].launder_gp_seq = cur_ops->get_gp_seq(); in rcu_torture_fwd_cb_cr()
2241 if (!cur_ops->sync) in rcu_torture_fwd_prog_nr()
2243 if (cur_ops->call && cur_ops->cb_barrier) { in rcu_torture_fwd_prog_nr()
2250 cur_ops->sync(); /* Later readers see above write. */ in rcu_torture_fwd_prog_nr()
2253 cur_ops->call(&fcs.rh, rcu_torture_fwd_prog_cb); in rcu_torture_fwd_prog_nr()
2256 gps = cur_ops->get_gp_seq(); in rcu_torture_fwd_prog_nr()
2257 sd = cur_ops->stall_dur() + 1; in rcu_torture_fwd_prog_nr()
2265 idx = cur_ops->readlock(); in rcu_torture_fwd_prog_nr()
2267 cur_ops->readunlock(idx); in rcu_torture_fwd_prog_nr()
2277 gps = rcutorture_seq_diff(cur_ops->get_gp_seq(), gps); in rcu_torture_fwd_prog_nr()
2283 cur_ops->sync(); /* Wait for running CB to complete. */ in rcu_torture_fwd_prog_nr()
2284 cur_ops->cb_barrier(); /* Wait for queued callbacks. */ in rcu_torture_fwd_prog_nr()
2314 if (!cur_ops->call) in rcu_torture_fwd_prog_cr()
2319 cur_ops->sync(); /* Later readers see above write. */ in rcu_torture_fwd_prog_cr()
2330 gps = cur_ops->get_gp_seq(); in rcu_torture_fwd_prog_cr()
2358 cur_ops->call(&rfcp->rh, rcu_torture_fwd_cb_cr); in rcu_torture_fwd_prog_cr()
2369 gps = rcutorture_seq_diff(cur_ops->get_gp_seq(), gps); in rcu_torture_fwd_prog_cr()
2370 cur_ops->cb_barrier(); /* Wait for callbacks to be invoked. */ in rcu_torture_fwd_prog_cr()
2469 if ((!cur_ops->sync && !cur_ops->call) || in rcu_torture_fwd_prog_init()
2470 !cur_ops->stall_dur || cur_ops->stall_dur() <= 0 || cur_ops == &rcu_busted_ops) { in rcu_torture_fwd_prog_init()
2521 cur_ops->call(rhp, rcu_torture_barrier_cbf); in rcu_torture_barrier1cb()
2550 cur_ops->call(&rcu, rcu_torture_barrier_cbf); in rcu_torture_barrier_cbs()
2555 if (cur_ops->cb_barrier != NULL) in rcu_torture_barrier_cbs()
2556 cur_ops->cb_barrier(); in rcu_torture_barrier_cbs()
2581 cur_ops->cb_barrier(); /* Implies smp_mb() for wait_event(). */ in rcu_torture_barrier()
2594 cur_ops->cb_barrier(); in rcu_torture_barrier()
2619 if (cur_ops->call == NULL || cur_ops->cb_barrier == NULL) { in rcu_torture_barrier_init()
2622 torture_type, cur_ops->name); in rcu_torture_barrier_init()
2672 if (!(test_boost == 1 && cur_ops->can_boost) && test_boost != 2) in rcu_torture_can_boost()
2674 if (!cur_ops->start_gp_poll || !cur_ops->poll_gp_state) in rcu_torture_can_boost()
2797 if (cur_ops->cb_barrier != NULL) in rcu_torture_cleanup()
2798 cur_ops->cb_barrier(); in rcu_torture_cleanup()
2801 if (!cur_ops) { in rcu_torture_cleanup()
2806 if (cur_ops->gp_kthread_dbg) in rcu_torture_cleanup()
2807 cur_ops->gp_kthread_dbg(); in rcu_torture_cleanup()
2839 rcutorture_get_gp_data(cur_ops->ttype, &flags, &gp_seq); in rcu_torture_cleanup()
2840 srcutorture_get_gp_data(cur_ops->ttype, srcu_ctlp, &flags, &gp_seq); in rcu_torture_cleanup()
2842 cur_ops->name, (long)gp_seq, flags, in rcu_torture_cleanup()
2853 if (cur_ops->cb_barrier != NULL) in rcu_torture_cleanup()
2854 cur_ops->cb_barrier(); in rcu_torture_cleanup()
2855 if (cur_ops->cleanup != NULL) in rcu_torture_cleanup()
2856 cur_ops->cleanup(); in rcu_torture_cleanup()
2890 rcu_torture_print_module_parms(cur_ops, "End of test: FAILURE"); in rcu_torture_cleanup()
2892 rcu_torture_print_module_parms(cur_ops, in rcu_torture_cleanup()
2895 rcu_torture_print_module_parms(cur_ops, "End of test: SUCCESS"); in rcu_torture_cleanup()
2964 if (cur_ops->sync && !(++n & 0xfff)) in rcutorture_sync()
2965 cur_ops->sync(); in rcutorture_sync()
2987 cur_ops = torture_ops[i]; in rcu_torture_init()
2988 if (strcmp(torture_type, cur_ops->name) == 0) in rcu_torture_init()
2999 cur_ops = NULL; in rcu_torture_init()
3002 if (cur_ops->fqs == NULL && fqs_duration != 0) { in rcu_torture_init()
3006 if (cur_ops->init) in rcu_torture_init()
3007 cur_ops->init(); in rcu_torture_init()
3016 rcu_torture_print_module_parms(cur_ops, "Start of test"); in rcu_torture_init()
3017 rcutorture_get_gp_data(cur_ops->ttype, &flags, &gp_seq); in rcu_torture_init()
3018 srcutorture_get_gp_data(cur_ops->ttype, srcu_ctlp, &flags, &gp_seq); in rcu_torture_init()
3021 cur_ops->name, (long)gp_seq, flags); in rcu_torture_init()
3134 t = cur_ops->stall_dur ? cur_ops->stall_dur() : stutter * HZ; in rcu_torture_init()