Lines Matching refs:now
558 u64 now; member
944 static void ioc_refresh_vrate(struct ioc *ioc, struct ioc_now *now) in ioc_refresh_vrate() argument
946 s64 pleft = ioc->period_at + ioc->period_us - now->now; in ioc_refresh_vrate()
1023 static void ioc_now(struct ioc *ioc, struct ioc_now *now) in ioc_now() argument
1027 now->now_ns = ktime_get(); in ioc_now()
1028 now->now = ktime_to_us(now->now_ns); in ioc_now()
1029 now->vrate = atomic64_read(&ioc->vtime_rate); in ioc_now()
1041 now->vnow = ioc->period_at_vtime + in ioc_now()
1042 (now->now - ioc->period_at) * now->vrate; in ioc_now()
1046 static void ioc_start_period(struct ioc *ioc, struct ioc_now *now) in ioc_start_period() argument
1051 ioc->period_at = now->now; in ioc_start_period()
1052 ioc->period_at_vtime = now->vnow; in ioc_start_period()
1065 bool save, struct ioc_now *now) in __propagate_weights() argument
1086 iocg->saved_margin = now->vnow - atomic64_read(&iocg->vtime); in __propagate_weights()
1140 bool save, struct ioc_now *now) in propagate_weights() argument
1142 __propagate_weights(iocg, active, inuse, save, now); in propagate_weights()
1226 static void weight_updated(struct ioc_gq *iocg, struct ioc_now *now) in weight_updated() argument
1237 propagate_weights(iocg, weight, iocg->inuse, true, now); in weight_updated()
1241 static bool iocg_activate(struct ioc_gq *iocg, struct ioc_now *now) in iocg_activate() argument
1253 ioc_now(ioc, now); in iocg_activate()
1266 ioc_now(ioc, now); in iocg_activate()
1287 vtarget = now->vnow - ioc->margins.target; in iocg_activate()
1303 iocg->last_inuse ?: iocg->weight, true, now); in iocg_activate()
1305 TRACE_IOCG_PATH(iocg_activate, iocg, now, in iocg_activate()
1308 iocg->activated_at = now->now; in iocg_activate()
1312 ioc->dfgv_period_at = now->now; in iocg_activate()
1314 ioc_start_period(ioc, now); in iocg_activate()
1326 static bool iocg_kick_delay(struct ioc_gq *iocg, struct ioc_now *now) in iocg_kick_delay() argument
1337 tdelta = now->now - iocg->delay_at; in iocg_kick_delay()
1346 abs_cost_to_cost(iocg->abs_vdebt, hwa) - now->vnow; in iocg_kick_delay()
1363 iocg->delay_at = now->now; in iocg_kick_delay()
1369 iocg->indelay_since = now->now; in iocg_kick_delay()
1374 iocg->local_stat.indelay_us += now->now - iocg->indelay_since; in iocg_kick_delay()
1384 struct ioc_now *now) in iocg_incur_debt() argument
1397 iocg->indebt_since = now->now; in iocg_incur_debt()
1398 propagate_weights(iocg, iocg->active, 0, false, now); in iocg_incur_debt()
1409 struct ioc_now *now) in iocg_pay_debt() argument
1422 iocg->local_stat.indebt_us += now->now - iocg->indebt_since; in iocg_pay_debt()
1426 false, now); in iocg_pay_debt()
1463 struct ioc_now *now) in iocg_kick_waitq() argument
1474 vbudget = now->vnow - atomic64_read(&iocg->vtime); in iocg_kick_waitq()
1486 iocg_pay_debt(iocg, abs_vpay, now); in iocg_kick_waitq()
1491 iocg_kick_delay(iocg, now); in iocg_kick_waitq()
1516 iocg->local_stat.wait_us += now->now - iocg->wait_since; in iocg_kick_waitq()
1523 iocg->wait_since = now->now; in iocg_kick_waitq()
1530 expires = now->now_ns + in iocg_kick_waitq()
1549 struct ioc_now now; in iocg_waitq_timer_fn() local
1552 ioc_now(iocg->ioc, &now); in iocg_waitq_timer_fn()
1555 iocg_kick_waitq(iocg, pay_debt, &now); in iocg_waitq_timer_fn()
1645 static void iocg_flush_stat_one(struct ioc_gq *iocg, struct ioc_now *now) in iocg_flush_stat_one() argument
1695 static void iocg_flush_stat(struct list_head *target_iocgs, struct ioc_now *now) in iocg_flush_stat() argument
1702 iocg_flush_stat_one(iocg, now); in iocg_flush_stat()
1708 iocg_flush_stat_one(iocg, now); in iocg_flush_stat()
1719 u32 usage, struct ioc_now *now) in hweight_after_donation() argument
1731 time_after64(vtime, now->vnow - ioc->margins.min)) in hweight_after_donation()
1735 excess = now->vnow - vtime - ioc->margins.target; in hweight_after_donation()
1759 delta = div64_s64(WEIGHT_ONE * (now->vnow - vtime), in hweight_after_donation()
1760 now->vnow - ioc->period_at_vtime); in hweight_after_donation()
1824 static void transfer_surpluses(struct list_head *surpluses, struct ioc_now *now) in transfer_surpluses() argument
2026 TRACE_IOCG_PATH(inuse_transfer, iocg, now, in transfer_surpluses()
2031 __propagate_weights(iocg, iocg->active, inuse, true, now); in transfer_surpluses()
2051 struct ioc_now *now) in ioc_forgive_debts() argument
2058 ioc->dfgv_period_at = now->now; in ioc_forgive_debts()
2074 if (time_before64(now->now, ioc->dfgv_period_at + DFGV_PERIOD)) in ioc_forgive_debts()
2081 dur = now->now - ioc->dfgv_period_at; in ioc_forgive_debts()
2084 ioc->dfgv_period_at = now->now; in ioc_forgive_debts()
2121 iocg_kick_waitq(iocg, true, now); in ioc_forgive_debts()
2123 TRACE_IOCG_PATH(iocg_forgive_debt, iocg, now, usage_pct, in ioc_forgive_debts()
2141 static int ioc_check_iocgs(struct ioc *ioc, struct ioc_now *now) in ioc_check_iocgs() argument
2155 iocg->local_stat.wait_us += now->now - iocg->wait_since; in ioc_check_iocgs()
2156 iocg->wait_since = now->now; in ioc_check_iocgs()
2160 now->now - iocg->indebt_since; in ioc_check_iocgs()
2161 iocg->indebt_since = now->now; in ioc_check_iocgs()
2165 now->now - iocg->indelay_since; in ioc_check_iocgs()
2166 iocg->indelay_since = now->now; in ioc_check_iocgs()
2172 iocg_kick_waitq(iocg, true, now); in ioc_check_iocgs()
2186 excess = now->vnow - vtime - ioc->margins.target; in ioc_check_iocgs()
2195 TRACE_IOCG_PATH(iocg_idle, iocg, now, in ioc_check_iocgs()
2198 __propagate_weights(iocg, 0, 0, false, now); in ioc_check_iocgs()
2213 struct ioc_now now; in ioc_timer_fn() local
2229 ioc_now(ioc, &now); in ioc_timer_fn()
2231 period_vtime = now.vnow - ioc->period_at_vtime; in ioc_timer_fn()
2237 nr_debtors = ioc_check_iocgs(ioc, &now); in ioc_timer_fn()
2243 iocg_flush_stat(&ioc->active_iocgs, &now); in ioc_timer_fn()
2267 time_after64(vtime, now.vnow - in ioc_timer_fn()
2269 time_before64(vdone, now.vnow - period_vtime)) in ioc_timer_fn()
2283 time_before64(vtime, now.vnow - ioc->margins.low))) { in ioc_timer_fn()
2297 usage_dur = max_t(u64, now.now - iocg->activated_at, 1); in ioc_timer_fn()
2299 usage_dur = max_t(u64, now.now - ioc->period_at, 1); in ioc_timer_fn()
2313 usage, &now); in ioc_timer_fn()
2326 TRACE_IOCG_PATH(inuse_shortage, iocg, &now, in ioc_timer_fn()
2331 iocg->active, true, &now); in ioc_timer_fn()
2341 transfer_surpluses(&surpluses, &now); in ioc_timer_fn()
2400 ioc_forgive_debts(ioc, usage_us_sum, nr_debtors, &now); in ioc_timer_fn()
2410 ioc_start_period(ioc, &now); in ioc_timer_fn()
2417 ioc_refresh_vrate(ioc, &now); in ioc_timer_fn()
2424 u64 abs_cost, struct ioc_now *now) in adjust_inuse_and_calc_cost() argument
2436 margin = now->vnow - vtime - cost; in adjust_inuse_and_calc_cost()
2469 propagate_weights(iocg, iocg->active, new_inuse, true, now); in adjust_inuse_and_calc_cost()
2472 } while (time_after64(vtime + cost, now->vnow) && in adjust_inuse_and_calc_cost()
2477 TRACE_IOCG_PATH(inuse_adjust, iocg, now, in adjust_inuse_and_calc_cost()
2562 struct ioc_now now; in ioc_rqos_throttle() local
2577 if (!iocg_activate(iocg, &now)) in ioc_rqos_throttle()
2582 cost = adjust_inuse_and_calc_cost(iocg, vtime, abs_cost, &now); in ioc_rqos_throttle()
2590 time_before_eq64(vtime + cost, now.vnow)) { in ioc_rqos_throttle()
2638 iocg_incur_debt(iocg, abs_cost, &now); in ioc_rqos_throttle()
2639 if (iocg_kick_delay(iocg, &now)) in ioc_rqos_throttle()
2654 &now); in ioc_rqos_throttle()
2677 iocg_kick_waitq(iocg, ioc_locked, &now); in ioc_rqos_throttle()
2698 struct ioc_now now; in ioc_rqos_merge() local
2710 ioc_now(ioc, &now); in ioc_rqos_merge()
2713 cost = adjust_inuse_and_calc_cost(iocg, vtime, abs_cost, &now); in ioc_rqos_merge()
2725 time_before_eq64(atomic64_read(&iocg->vtime) + cost, now.vnow)) { in ioc_rqos_merge()
2739 iocg_incur_debt(iocg, abs_cost, &now); in ioc_rqos_merge()
2740 if (iocg_kick_delay(iocg, &now)) in ioc_rqos_merge()
2938 struct ioc_now now; in ioc_pd_init() local
2942 ioc_now(ioc, &now); in ioc_pd_init()
2945 atomic64_set(&iocg->vtime, now.vnow); in ioc_pd_init()
2946 atomic64_set(&iocg->done_vtime, now.vnow); in ioc_pd_init()
2966 weight_updated(iocg, &now); in ioc_pd_init()
2980 struct ioc_now now; in ioc_pd_free() local
2982 ioc_now(ioc, &now); in ioc_pd_free()
2983 propagate_weights(iocg, 0, 0, false, &now); in ioc_pd_free()
3052 struct ioc_now now; in ioc_weight_write() local
3073 ioc_now(iocg->ioc, &now); in ioc_weight_write()
3074 weight_updated(iocg, &now); in ioc_weight_write()
3100 ioc_now(iocg->ioc, &now); in ioc_weight_write()
3101 weight_updated(iocg, &now); in ioc_weight_write()