3  * IO cost model based controller.
10  * observable cost metric.  This is distinguished from CPU and memory where
22  * While there is no cost metric we can trivially observe, it isn't a
23  * complete mystery.  For example, on a rotational device, seek cost
30  * 1. IO Cost Model
32  * IO cost model estimates the cost of an IO given its basic parameters and
33  * history (e.g. the end sector of the last IO).  The cost is measured in
34  * device time.  If a given IO is estimated to cost 10ms, the device should
37  * Currently, there's only one builtin cost model - linear.  Each IO is
38  * classified as sequential or random and given a base cost accordingly.
39  * On top of that, a size cost proportional to the length of the IO is
44  * /sys/fs/cgroup/io.cost.model.
86  * It's unrealistic to expect the cost model to be perfect.  There are too
123  * and can be set through /sys/fs/cgroup/io.cost.qos.
130  * cost per second, i.e., 10% of the device capacity.  The naive
133  * compared to free-for-all competition.  This is too high a cost to pay
170  * - inflt	: The percentage of in-flight IO cost at the end of last period
239 	 * As vtime is used to calculate the cost of each IO, it needs to
241 	 * represent the cost of a single page worth of discard with
306 	 * size-proportional components of cost calculation in closer
307 	 * numbers of digits to per-IO cost components.
323 /* io.cost.qos controls including per-dev enable of the whole controller */
330 /* io.cost.qos params */
341 /* io.cost.model controls */
348 /* builtin linear cost model coefficients */
501 	 * than issue.  The delta behind `vtime` represents the cost of
712 static u64 cost_to_abs_cost(u64 cost, u32 hw_inuse)  in cost_to_abs_cost()  argument
714 	return DIV64_U64_ROUND_UP(cost * hw_inuse, WEIGHT_ONE);  in cost_to_abs_cost()
718 			    u64 abs_cost, u64 cost)  in iocg_commit_bio()  argument
722 	bio->bi_iocost_cost = cost;  in iocg_commit_bio()
723 	atomic64_add(cost, &iocg->vtime);  in iocg_commit_bio()
862  * and calculate the linear model cost coefficients.
864  *  *@page	per-page cost		1s / (@bps / 4096)
865  *  *@seqio	base cost of a seq IO	max((1s / @seqiops) - *@page, 0)
866  *  @randiops	base cost of a rand IO	max((1s / @randiops) - *@page, 0)
1466 	u64 cost = abs_cost_to_cost(wait->abs_cost, ctx->hw_inuse);  in iocg_wake_fn()  local
1468 	ctx->vbudget -= cost;  in iocg_wake_fn()
1473 	iocg_commit_bio(ctx->iocg, wait->bio, wait->abs_cost, cost);  in iocg_wake_fn()
2470 	u64 cost, new_inuse;  in adjust_inuse_and_calc_cost()  local
2475 	cost = abs_cost_to_cost(abs_cost, hwi);  in adjust_inuse_and_calc_cost()
2476 	margin = now->vnow - vtime - cost;  in adjust_inuse_and_calc_cost()
2480 		return cost;  in adjust_inuse_and_calc_cost()
2488 		return cost;  in adjust_inuse_and_calc_cost()
2495 		return cost;  in adjust_inuse_and_calc_cost()
2511 		cost = abs_cost_to_cost(abs_cost, hwi);  in adjust_inuse_and_calc_cost()
2512 	} while (time_after64(vtime + cost, now->vnow) &&  in adjust_inuse_and_calc_cost()
2520 	return cost;  in adjust_inuse_and_calc_cost()
2530 	u64 cost = 0;  in calc_vtime_cost_builtin()  local
2532 	/* Can't calculate cost for empty bio */  in calc_vtime_cost_builtin()
2558 			cost += coef_randio;  in calc_vtime_cost_builtin()
2560 			cost += coef_seqio;  in calc_vtime_cost_builtin()
2563 	cost += pages * coef_page;  in calc_vtime_cost_builtin()
2565 	*costp = cost;  in calc_vtime_cost_builtin()
2570 	u64 cost;  in calc_vtime_cost()  local
2572 	calc_vtime_cost_builtin(bio, iocg, is_merge, &cost);  in calc_vtime_cost()
2573 	return cost;  in calc_vtime_cost()
2595 	u64 cost;  in calc_size_vtime_cost()  local
2597 	calc_size_vtime_cost_builtin(rq, ioc, &cost);  in calc_size_vtime_cost()
2598 	return cost;  in calc_size_vtime_cost()
2608 	u64 abs_cost, cost, vtime;  in ioc_rqos_throttle()  local
2616 	/* calculate the absolute vtime cost */  in ioc_rqos_throttle()
2626 	cost = adjust_inuse_and_calc_cost(iocg, vtime, abs_cost, &now);  in ioc_rqos_throttle()
2634 	    time_before_eq64(vtime + cost, now.vnow)) {  in ioc_rqos_throttle()
2635 		iocg_commit_bio(iocg, bio, abs_cost, cost);  in ioc_rqos_throttle()
2660 		iocg_commit_bio(iocg, bio, abs_cost, cost);  in ioc_rqos_throttle()
2705 	 * or too long.  Each wait entry records the absolute cost it's  in ioc_rqos_throttle()
2743 	u64 vtime, abs_cost, cost;  in ioc_rqos_merge()  local
2757 	cost = adjust_inuse_and_calc_cost(iocg, vtime, abs_cost, &now);  in ioc_rqos_merge()
2766 	 * cost assigned.  in ioc_rqos_merge()
2769 	    time_before_eq64(atomic64_read(&iocg->vtime) + cost, now.vnow)) {  in ioc_rqos_merge()
2770 		iocg_commit_bio(iocg, bio, abs_cost, cost);  in ioc_rqos_merge()
2788 		iocg_commit_bio(iocg, bio, abs_cost, cost);  in ioc_rqos_merge()
3055 		seq_printf(s, " cost.vrate=%u.%02u", vp10k / 100, vp10k % 100);  in ioc_pd_stat()
3058 	seq_printf(s, " cost.usage=%llu", iocg->last_stat.usage_us);  in ioc_pd_stat()
3061 		seq_printf(s, " cost.wait=%llu cost.indebt=%llu cost.indelay=%llu",  in ioc_pd_stat()
3514 		.name = "cost.qos",
3520 		.name = "cost.model",