Lines Matching refs:rwb
146 static inline bool rwb_enabled(struct rq_wb *rwb) in rwb_enabled() argument
148 return rwb && rwb->enable_state != WBT_STATE_OFF_DEFAULT && in rwb_enabled()
149 rwb->wb_normal != 0; in rwb_enabled()
152 static void wb_timestamp(struct rq_wb *rwb, unsigned long *var) in wb_timestamp() argument
154 if (rwb_enabled(rwb)) { in wb_timestamp()
166 static bool wb_recent_wait(struct rq_wb *rwb) in wb_recent_wait() argument
168 struct bdi_writeback *wb = &rwb->rqos.disk->bdi->wb; in wb_recent_wait()
173 static inline struct rq_wait *get_rq_wait(struct rq_wb *rwb, in get_rq_wait() argument
177 return &rwb->rq_wait[WBT_RWQ_KSWAPD]; in get_rq_wait()
179 return &rwb->rq_wait[WBT_RWQ_DISCARD]; in get_rq_wait()
181 return &rwb->rq_wait[WBT_RWQ_BG]; in get_rq_wait()
184 static void rwb_wake_all(struct rq_wb *rwb) in rwb_wake_all() argument
189 struct rq_wait *rqw = &rwb->rq_wait[i]; in rwb_wake_all()
196 static void wbt_rqw_done(struct rq_wb *rwb, struct rq_wait *rqw, in wbt_rqw_done() argument
207 if (unlikely(!rwb_enabled(rwb))) { in wbt_rqw_done()
208 rwb_wake_all(rwb); in wbt_rqw_done()
218 limit = rwb->wb_background; in wbt_rqw_done()
219 else if (rwb->wc && !wb_recent_wait(rwb)) in wbt_rqw_done()
222 limit = rwb->wb_normal; in wbt_rqw_done()
233 if (!inflight || diff >= rwb->wb_background / 2) in wbt_rqw_done()
240 struct rq_wb *rwb = RQWB(rqos); in __wbt_done() local
246 rqw = get_rq_wait(rwb, wb_acct); in __wbt_done()
247 wbt_rqw_done(rwb, rqw, wb_acct); in __wbt_done()
256 struct rq_wb *rwb = RQWB(rqos); in wbt_done() local
259 if (rwb->sync_cookie == rq) { in wbt_done()
260 rwb->sync_issue = 0; in wbt_done()
261 rwb->sync_cookie = NULL; in wbt_done()
265 wb_timestamp(rwb, &rwb->last_comp); in wbt_done()
267 WARN_ON_ONCE(rq == rwb->sync_cookie); in wbt_done()
285 static u64 rwb_sync_issue_lat(struct rq_wb *rwb) in rwb_sync_issue_lat() argument
287 u64 now, issue = READ_ONCE(rwb->sync_issue); in rwb_sync_issue_lat()
289 if (!issue || !rwb->sync_cookie) in rwb_sync_issue_lat()
296 static inline unsigned int wbt_inflight(struct rq_wb *rwb) in wbt_inflight() argument
301 ret += atomic_read(&rwb->rq_wait[i].inflight); in wbt_inflight()
313 static int latency_exceeded(struct rq_wb *rwb, struct blk_rq_stat *stat) in latency_exceeded() argument
315 struct backing_dev_info *bdi = rwb->rqos.disk->bdi; in latency_exceeded()
316 struct rq_depth *rqd = &rwb->rq_depth; in latency_exceeded()
328 thislat = rwb_sync_issue_lat(rwb); in latency_exceeded()
329 if (thislat > rwb->cur_win_nsec || in latency_exceeded()
330 (thislat > rwb->min_lat_nsec && !stat[READ].nr_samples)) { in latency_exceeded()
345 if (stat[WRITE].nr_samples || wb_recent_wait(rwb) || in latency_exceeded()
346 wbt_inflight(rwb)) in latency_exceeded()
354 if (stat[READ].min > rwb->min_lat_nsec) { in latency_exceeded()
366 static void rwb_trace_step(struct rq_wb *rwb, const char *msg) in rwb_trace_step() argument
368 struct backing_dev_info *bdi = rwb->rqos.disk->bdi; in rwb_trace_step()
369 struct rq_depth *rqd = &rwb->rq_depth; in rwb_trace_step()
371 trace_wbt_step(bdi, msg, rqd->scale_step, rwb->cur_win_nsec, in rwb_trace_step()
372 rwb->wb_background, rwb->wb_normal, rqd->max_depth); in rwb_trace_step()
375 static void calc_wb_limits(struct rq_wb *rwb) in calc_wb_limits() argument
377 if (rwb->min_lat_nsec == 0) { in calc_wb_limits()
378 rwb->wb_normal = rwb->wb_background = 0; in calc_wb_limits()
379 } else if (rwb->rq_depth.max_depth <= 2) { in calc_wb_limits()
380 rwb->wb_normal = rwb->rq_depth.max_depth; in calc_wb_limits()
381 rwb->wb_background = 1; in calc_wb_limits()
383 rwb->wb_normal = (rwb->rq_depth.max_depth + 1) / 2; in calc_wb_limits()
384 rwb->wb_background = (rwb->rq_depth.max_depth + 3) / 4; in calc_wb_limits()
388 static void scale_up(struct rq_wb *rwb) in scale_up() argument
390 if (!rq_depth_scale_up(&rwb->rq_depth)) in scale_up()
392 calc_wb_limits(rwb); in scale_up()
393 rwb->unknown_cnt = 0; in scale_up()
394 rwb_wake_all(rwb); in scale_up()
395 rwb_trace_step(rwb, tracepoint_string("scale up")); in scale_up()
398 static void scale_down(struct rq_wb *rwb, bool hard_throttle) in scale_down() argument
400 if (!rq_depth_scale_down(&rwb->rq_depth, hard_throttle)) in scale_down()
402 calc_wb_limits(rwb); in scale_down()
403 rwb->unknown_cnt = 0; in scale_down()
404 rwb_trace_step(rwb, tracepoint_string("scale down")); in scale_down()
407 static void rwb_arm_timer(struct rq_wb *rwb) in rwb_arm_timer() argument
409 struct rq_depth *rqd = &rwb->rq_depth; in rwb_arm_timer()
418 rwb->cur_win_nsec = div_u64(rwb->win_nsec << 4, in rwb_arm_timer()
425 rwb->cur_win_nsec = rwb->win_nsec; in rwb_arm_timer()
428 blk_stat_activate_nsecs(rwb->cb, rwb->cur_win_nsec); in rwb_arm_timer()
433 struct rq_wb *rwb = cb->data; in wb_timer_fn() local
434 struct rq_depth *rqd = &rwb->rq_depth; in wb_timer_fn()
435 unsigned int inflight = wbt_inflight(rwb); in wb_timer_fn()
438 if (!rwb->rqos.disk) in wb_timer_fn()
441 status = latency_exceeded(rwb, cb->stat); in wb_timer_fn()
443 trace_wbt_timer(rwb->rqos.disk->bdi, status, rqd->scale_step, inflight); in wb_timer_fn()
452 scale_down(rwb, true); in wb_timer_fn()
455 scale_up(rwb); in wb_timer_fn()
463 scale_up(rwb); in wb_timer_fn()
466 if (++rwb->unknown_cnt < RWB_UNKNOWN_BUMP) in wb_timer_fn()
474 scale_up(rwb); in wb_timer_fn()
476 scale_down(rwb, false); in wb_timer_fn()
486 rwb_arm_timer(rwb); in wb_timer_fn()
489 static void wbt_update_limits(struct rq_wb *rwb) in wbt_update_limits() argument
491 struct rq_depth *rqd = &rwb->rq_depth; in wbt_update_limits()
497 calc_wb_limits(rwb); in wbt_update_limits()
499 rwb_wake_all(rwb); in wbt_update_limits()
534 static bool close_io(struct rq_wb *rwb) in close_io() argument
538 return time_before(now, rwb->last_issue + HZ / 10) || in close_io()
539 time_before(now, rwb->last_comp + HZ / 10); in close_io()
544 static inline unsigned int get_limit(struct rq_wb *rwb, blk_opf_t opf) in get_limit() argument
552 if (!rwb_enabled(rwb)) in get_limit()
556 return rwb->wb_background; in get_limit()
566 if ((opf & REQ_HIPRIO) || wb_recent_wait(rwb) || current_is_kswapd()) in get_limit()
567 limit = rwb->rq_depth.max_depth; in get_limit()
568 else if ((opf & REQ_BACKGROUND) || close_io(rwb)) { in get_limit()
573 limit = rwb->wb_background; in get_limit()
575 limit = rwb->wb_normal; in get_limit()
581 struct rq_wb *rwb; member
589 return rq_wait_inc_below(rqw, get_limit(data->rwb, data->opf)); in wbt_inflight_cb()
595 wbt_rqw_done(data->rwb, rqw, data->wb_acct); in wbt_cleanup_cb()
602 static void __wbt_wait(struct rq_wb *rwb, enum wbt_flags wb_acct, in __wbt_wait() argument
605 struct rq_wait *rqw = get_rq_wait(rwb, wb_acct); in __wbt_wait()
607 .rwb = rwb, in __wbt_wait()
633 static enum wbt_flags bio_to_wbt_flags(struct rq_wb *rwb, struct bio *bio) in bio_to_wbt_flags() argument
637 if (!rwb_enabled(rwb)) in bio_to_wbt_flags()
654 struct rq_wb *rwb = RQWB(rqos); in wbt_cleanup() local
655 enum wbt_flags flags = bio_to_wbt_flags(rwb, bio); in wbt_cleanup()
666 struct rq_wb *rwb = RQWB(rqos); in wbt_wait() local
669 flags = bio_to_wbt_flags(rwb, bio); in wbt_wait()
672 wb_timestamp(rwb, &rwb->last_issue); in wbt_wait()
676 __wbt_wait(rwb, flags, bio->bi_opf); in wbt_wait()
678 if (!blk_stat_is_active(rwb->cb)) in wbt_wait()
679 rwb_arm_timer(rwb); in wbt_wait()
684 struct rq_wb *rwb = RQWB(rqos); in wbt_track() local
685 rq->wbt_flags |= bio_to_wbt_flags(rwb, bio); in wbt_track()
690 struct rq_wb *rwb = RQWB(rqos); in wbt_issue() local
692 if (!rwb_enabled(rwb)) in wbt_issue()
702 if (wbt_is_read(rq) && !rwb->sync_issue) { in wbt_issue()
703 rwb->sync_cookie = rq; in wbt_issue()
704 rwb->sync_issue = rq->io_start_time_ns; in wbt_issue()
710 struct rq_wb *rwb = RQWB(rqos); in wbt_requeue() local
711 if (!rwb_enabled(rwb)) in wbt_requeue()
713 if (rq == rwb->sync_cookie) { in wbt_requeue()
714 rwb->sync_issue = 0; in wbt_requeue()
715 rwb->sync_cookie = NULL; in wbt_requeue()
787 struct rq_wb *rwb = RQWB(rqos); in wbt_exit() local
789 blk_stat_remove_callback(rqos->disk->queue, rwb->cb); in wbt_exit()
790 blk_stat_free_callback(rwb->cb); in wbt_exit()
791 kfree(rwb); in wbt_exit()
800 struct rq_wb *rwb; in wbt_disable_default() local
803 rwb = RQWB(rqos); in wbt_disable_default()
804 if (rwb->enable_state == WBT_STATE_ON_DEFAULT) { in wbt_disable_default()
805 blk_stat_deactivate(rwb->cb); in wbt_disable_default()
806 rwb->enable_state = WBT_STATE_OFF_DEFAULT; in wbt_disable_default()
815 struct rq_wb *rwb = RQWB(rqos); in wbt_curr_win_nsec_show() local
817 seq_printf(m, "%llu\n", rwb->cur_win_nsec); in wbt_curr_win_nsec_show()
824 struct rq_wb *rwb = RQWB(rqos); in wbt_enabled_show() local
826 seq_printf(m, "%d\n", rwb->enable_state); in wbt_enabled_show()
841 struct rq_wb *rwb = RQWB(rqos); in wbt_inflight_show() local
846 atomic_read(&rwb->rq_wait[i].inflight)); in wbt_inflight_show()
853 struct rq_wb *rwb = RQWB(rqos); in wbt_min_lat_nsec_show() local
855 seq_printf(m, "%lu\n", rwb->min_lat_nsec); in wbt_min_lat_nsec_show()
862 struct rq_wb *rwb = RQWB(rqos); in wbt_unknown_cnt_show() local
864 seq_printf(m, "%u\n", rwb->unknown_cnt); in wbt_unknown_cnt_show()
871 struct rq_wb *rwb = RQWB(rqos); in wbt_normal_show() local
873 seq_printf(m, "%u\n", rwb->wb_normal); in wbt_normal_show()
880 struct rq_wb *rwb = RQWB(rqos); in wbt_background_show() local
882 seq_printf(m, "%u\n", rwb->wb_background); in wbt_background_show()
916 struct rq_wb *rwb; in wbt_init() local
920 rwb = kzalloc(sizeof(*rwb), GFP_KERNEL); in wbt_init()
921 if (!rwb) in wbt_init()
924 rwb->cb = blk_stat_alloc_callback(wb_timer_fn, wbt_data_dir, 2, rwb); in wbt_init()
925 if (!rwb->cb) { in wbt_init()
926 kfree(rwb); in wbt_init()
931 rq_wait_init(&rwb->rq_wait[i]); in wbt_init()
933 rwb->last_comp = rwb->last_issue = jiffies; in wbt_init()
934 rwb->win_nsec = RWB_WINDOW_NSEC; in wbt_init()
935 rwb->enable_state = WBT_STATE_ON_DEFAULT; in wbt_init()
936 rwb->wc = test_bit(QUEUE_FLAG_WC, &q->queue_flags); in wbt_init()
937 rwb->rq_depth.default_depth = RWB_DEF_DEPTH; in wbt_init()
938 rwb->min_lat_nsec = wbt_default_latency_nsec(q); in wbt_init()
939 rwb->rq_depth.queue_depth = blk_queue_depth(q); in wbt_init()
940 wbt_update_limits(rwb); in wbt_init()
945 ret = rq_qos_add(&rwb->rqos, disk, RQ_QOS_WBT, &wbt_rqos_ops); in wbt_init()
949 blk_stat_add_callback(q, rwb->cb); in wbt_init()
954 blk_stat_free_callback(rwb->cb); in wbt_init()
955 kfree(rwb); in wbt_init()