Lines Matching refs:rwb
145 static inline bool rwb_enabled(struct rq_wb *rwb) in rwb_enabled() argument
147 return rwb && rwb->enable_state != WBT_STATE_OFF_DEFAULT && in rwb_enabled()
148 rwb->enable_state != WBT_STATE_OFF_MANUAL; in rwb_enabled()
151 static void wb_timestamp(struct rq_wb *rwb, unsigned long *var) in wb_timestamp() argument
153 if (rwb_enabled(rwb)) { in wb_timestamp()
165 static bool wb_recent_wait(struct rq_wb *rwb) in wb_recent_wait() argument
167 struct backing_dev_info *bdi = rwb->rqos.disk->bdi; in wb_recent_wait()
172 static inline struct rq_wait *get_rq_wait(struct rq_wb *rwb, in get_rq_wait() argument
176 return &rwb->rq_wait[WBT_RWQ_SWAP]; in get_rq_wait()
178 return &rwb->rq_wait[WBT_RWQ_DISCARD]; in get_rq_wait()
180 return &rwb->rq_wait[WBT_RWQ_BG]; in get_rq_wait()
183 static void rwb_wake_all(struct rq_wb *rwb) in rwb_wake_all() argument
188 struct rq_wait *rqw = &rwb->rq_wait[i]; in rwb_wake_all()
195 static void wbt_rqw_done(struct rq_wb *rwb, struct rq_wait *rqw, in wbt_rqw_done() argument
208 limit = rwb->wb_background; in wbt_rqw_done()
209 else if (blk_queue_write_cache(rwb->rqos.disk->queue) && in wbt_rqw_done()
210 !wb_recent_wait(rwb)) in wbt_rqw_done()
213 limit = rwb->wb_normal; in wbt_rqw_done()
224 if (!inflight || diff >= rwb->wb_background / 2) in wbt_rqw_done()
231 struct rq_wb *rwb = RQWB(rqos); in __wbt_done() local
237 rqw = get_rq_wait(rwb, wb_acct); in __wbt_done()
238 wbt_rqw_done(rwb, rqw, wb_acct); in __wbt_done()
247 struct rq_wb *rwb = RQWB(rqos); in wbt_done() local
250 if (rwb->sync_cookie == rq) { in wbt_done()
251 rwb->sync_issue = 0; in wbt_done()
252 rwb->sync_cookie = NULL; in wbt_done()
256 wb_timestamp(rwb, &rwb->last_comp); in wbt_done()
258 WARN_ON_ONCE(rq == rwb->sync_cookie); in wbt_done()
276 static u64 rwb_sync_issue_lat(struct rq_wb *rwb) in rwb_sync_issue_lat() argument
278 u64 issue = READ_ONCE(rwb->sync_issue); in rwb_sync_issue_lat()
280 if (!issue || !rwb->sync_cookie) in rwb_sync_issue_lat()
286 static inline unsigned int wbt_inflight(struct rq_wb *rwb) in wbt_inflight() argument
291 ret += atomic_read(&rwb->rq_wait[i].inflight); in wbt_inflight()
303 static int latency_exceeded(struct rq_wb *rwb, struct blk_rq_stat *stat) in latency_exceeded() argument
305 struct backing_dev_info *bdi = rwb->rqos.disk->bdi; in latency_exceeded()
306 struct rq_depth *rqd = &rwb->rq_depth; in latency_exceeded()
318 thislat = rwb_sync_issue_lat(rwb); in latency_exceeded()
319 if (thislat > rwb->cur_win_nsec || in latency_exceeded()
320 (thislat > rwb->min_lat_nsec && !stat[READ].nr_samples)) { in latency_exceeded()
335 if (stat[WRITE].nr_samples || wb_recent_wait(rwb) || in latency_exceeded()
336 wbt_inflight(rwb)) in latency_exceeded()
344 if (stat[READ].min > rwb->min_lat_nsec) { in latency_exceeded()
356 static void rwb_trace_step(struct rq_wb *rwb, const char *msg) in rwb_trace_step() argument
358 struct backing_dev_info *bdi = rwb->rqos.disk->bdi; in rwb_trace_step()
359 struct rq_depth *rqd = &rwb->rq_depth; in rwb_trace_step()
361 trace_wbt_step(bdi, msg, rqd->scale_step, rwb->cur_win_nsec, in rwb_trace_step()
362 rwb->wb_background, rwb->wb_normal, rqd->max_depth); in rwb_trace_step()
365 static void calc_wb_limits(struct rq_wb *rwb) in calc_wb_limits() argument
367 if (rwb->min_lat_nsec == 0) { in calc_wb_limits()
368 rwb->wb_normal = rwb->wb_background = 0; in calc_wb_limits()
369 } else if (rwb->rq_depth.max_depth <= 2) { in calc_wb_limits()
370 rwb->wb_normal = rwb->rq_depth.max_depth; in calc_wb_limits()
371 rwb->wb_background = 1; in calc_wb_limits()
373 rwb->wb_normal = (rwb->rq_depth.max_depth + 1) / 2; in calc_wb_limits()
374 rwb->wb_background = (rwb->rq_depth.max_depth + 3) / 4; in calc_wb_limits()
378 static void scale_up(struct rq_wb *rwb) in scale_up() argument
380 if (!rq_depth_scale_up(&rwb->rq_depth)) in scale_up()
382 calc_wb_limits(rwb); in scale_up()
383 rwb->unknown_cnt = 0; in scale_up()
384 rwb_wake_all(rwb); in scale_up()
385 rwb_trace_step(rwb, tracepoint_string("scale up")); in scale_up()
388 static void scale_down(struct rq_wb *rwb, bool hard_throttle) in scale_down() argument
390 if (!rq_depth_scale_down(&rwb->rq_depth, hard_throttle)) in scale_down()
392 calc_wb_limits(rwb); in scale_down()
393 rwb->unknown_cnt = 0; in scale_down()
394 rwb_trace_step(rwb, tracepoint_string("scale down")); in scale_down()
397 static void rwb_arm_timer(struct rq_wb *rwb) in rwb_arm_timer() argument
399 struct rq_depth *rqd = &rwb->rq_depth; in rwb_arm_timer()
408 rwb->cur_win_nsec = div_u64(rwb->win_nsec << 4, in rwb_arm_timer()
415 rwb->cur_win_nsec = rwb->win_nsec; in rwb_arm_timer()
418 blk_stat_activate_nsecs(rwb->cb, rwb->cur_win_nsec); in rwb_arm_timer()
423 struct rq_wb *rwb = cb->data; in wb_timer_fn() local
424 struct rq_depth *rqd = &rwb->rq_depth; in wb_timer_fn()
425 unsigned int inflight = wbt_inflight(rwb); in wb_timer_fn()
428 if (!rwb->rqos.disk) in wb_timer_fn()
431 status = latency_exceeded(rwb, cb->stat); in wb_timer_fn()
433 trace_wbt_timer(rwb->rqos.disk->bdi, status, rqd->scale_step, inflight); in wb_timer_fn()
442 scale_down(rwb, true); in wb_timer_fn()
445 scale_up(rwb); in wb_timer_fn()
453 scale_up(rwb); in wb_timer_fn()
456 if (++rwb->unknown_cnt < RWB_UNKNOWN_BUMP) in wb_timer_fn()
464 scale_up(rwb); in wb_timer_fn()
466 scale_down(rwb, false); in wb_timer_fn()
476 rwb_arm_timer(rwb); in wb_timer_fn()
479 static void wbt_update_limits(struct rq_wb *rwb) in wbt_update_limits() argument
481 struct rq_depth *rqd = &rwb->rq_depth; in wbt_update_limits()
487 calc_wb_limits(rwb); in wbt_update_limits()
489 rwb_wake_all(rwb); in wbt_update_limits()
523 static bool close_io(struct rq_wb *rwb) in close_io() argument
527 return time_before(now, rwb->last_issue + HZ / 10) || in close_io()
528 time_before(now, rwb->last_comp + HZ / 10); in close_io()
533 static inline unsigned int get_limit(struct rq_wb *rwb, blk_opf_t opf) in get_limit() argument
538 return rwb->wb_background; in get_limit()
548 if ((opf & REQ_HIPRIO) || wb_recent_wait(rwb)) in get_limit()
549 limit = rwb->rq_depth.max_depth; in get_limit()
550 else if ((opf & REQ_BACKGROUND) || close_io(rwb)) { in get_limit()
555 limit = rwb->wb_background; in get_limit()
557 limit = rwb->wb_normal; in get_limit()
563 struct rq_wb *rwb; member
571 return rq_wait_inc_below(rqw, get_limit(data->rwb, data->opf)); in wbt_inflight_cb()
577 wbt_rqw_done(data->rwb, rqw, data->wb_acct); in wbt_cleanup_cb()
584 static void __wbt_wait(struct rq_wb *rwb, enum wbt_flags wb_acct, in __wbt_wait() argument
587 struct rq_wait *rqw = get_rq_wait(rwb, wb_acct); in __wbt_wait()
589 .rwb = rwb, in __wbt_wait()
615 static enum wbt_flags bio_to_wbt_flags(struct rq_wb *rwb, struct bio *bio) in bio_to_wbt_flags() argument
619 if (!rwb_enabled(rwb)) in bio_to_wbt_flags()
636 struct rq_wb *rwb = RQWB(rqos); in wbt_cleanup() local
637 enum wbt_flags flags = bio_to_wbt_flags(rwb, bio); in wbt_cleanup()
648 struct rq_wb *rwb = RQWB(rqos); in wbt_wait() local
651 flags = bio_to_wbt_flags(rwb, bio); in wbt_wait()
654 wb_timestamp(rwb, &rwb->last_issue); in wbt_wait()
658 __wbt_wait(rwb, flags, bio->bi_opf); in wbt_wait()
660 if (!blk_stat_is_active(rwb->cb)) in wbt_wait()
661 rwb_arm_timer(rwb); in wbt_wait()
666 struct rq_wb *rwb = RQWB(rqos); in wbt_track() local
667 rq->wbt_flags |= bio_to_wbt_flags(rwb, bio); in wbt_track()
672 struct rq_wb *rwb = RQWB(rqos); in wbt_issue() local
674 if (!rwb_enabled(rwb)) in wbt_issue()
684 if (wbt_is_read(rq) && !rwb->sync_issue) { in wbt_issue()
685 rwb->sync_cookie = rq; in wbt_issue()
686 rwb->sync_issue = rq->io_start_time_ns; in wbt_issue()
692 struct rq_wb *rwb = RQWB(rqos); in wbt_requeue() local
693 if (!rwb_enabled(rwb)) in wbt_requeue()
695 if (rq == rwb->sync_cookie) { in wbt_requeue()
696 rwb->sync_issue = 0; in wbt_requeue()
697 rwb->sync_cookie = NULL; in wbt_requeue()
764 struct rq_wb *rwb = RQWB(rqos); in wbt_exit() local
766 blk_stat_remove_callback(rqos->disk->queue, rwb->cb); in wbt_exit()
767 blk_stat_free_callback(rwb->cb); in wbt_exit()
768 kfree(rwb); in wbt_exit()
777 struct rq_wb *rwb; in wbt_disable_default() local
780 rwb = RQWB(rqos); in wbt_disable_default()
781 if (rwb->enable_state == WBT_STATE_ON_DEFAULT) { in wbt_disable_default()
782 blk_stat_deactivate(rwb->cb); in wbt_disable_default()
783 rwb->enable_state = WBT_STATE_OFF_DEFAULT; in wbt_disable_default()
792 struct rq_wb *rwb = RQWB(rqos); in wbt_curr_win_nsec_show() local
794 seq_printf(m, "%llu\n", rwb->cur_win_nsec); in wbt_curr_win_nsec_show()
801 struct rq_wb *rwb = RQWB(rqos); in wbt_enabled_show() local
803 seq_printf(m, "%d\n", rwb->enable_state); in wbt_enabled_show()
818 struct rq_wb *rwb = RQWB(rqos); in wbt_inflight_show() local
823 atomic_read(&rwb->rq_wait[i].inflight)); in wbt_inflight_show()
830 struct rq_wb *rwb = RQWB(rqos); in wbt_min_lat_nsec_show() local
832 seq_printf(m, "%lu\n", rwb->min_lat_nsec); in wbt_min_lat_nsec_show()
839 struct rq_wb *rwb = RQWB(rqos); in wbt_unknown_cnt_show() local
841 seq_printf(m, "%u\n", rwb->unknown_cnt); in wbt_unknown_cnt_show()
848 struct rq_wb *rwb = RQWB(rqos); in wbt_normal_show() local
850 seq_printf(m, "%u\n", rwb->wb_normal); in wbt_normal_show()
857 struct rq_wb *rwb = RQWB(rqos); in wbt_background_show() local
859 seq_printf(m, "%u\n", rwb->wb_background); in wbt_background_show()
893 struct rq_wb *rwb; in wbt_init() local
897 rwb = kzalloc(sizeof(*rwb), GFP_KERNEL); in wbt_init()
898 if (!rwb) in wbt_init()
901 rwb->cb = blk_stat_alloc_callback(wb_timer_fn, wbt_data_dir, 2, rwb); in wbt_init()
902 if (!rwb->cb) { in wbt_init()
903 kfree(rwb); in wbt_init()
908 rq_wait_init(&rwb->rq_wait[i]); in wbt_init()
910 rwb->last_comp = rwb->last_issue = jiffies; in wbt_init()
911 rwb->win_nsec = RWB_WINDOW_NSEC; in wbt_init()
912 rwb->enable_state = WBT_STATE_ON_DEFAULT; in wbt_init()
913 rwb->rq_depth.default_depth = RWB_DEF_DEPTH; in wbt_init()
914 rwb->min_lat_nsec = wbt_default_latency_nsec(q); in wbt_init()
915 rwb->rq_depth.queue_depth = blk_queue_depth(q); in wbt_init()
916 wbt_update_limits(rwb); in wbt_init()
922 ret = rq_qos_add(&rwb->rqos, disk, RQ_QOS_WBT, &wbt_rqos_ops); in wbt_init()
927 blk_stat_add_callback(q, rwb->cb); in wbt_init()
932 blk_stat_free_callback(rwb->cb); in wbt_init()
933 kfree(rwb); in wbt_init()