1 // SPDX-License-Identifier: GPL-2.0
24 #include <linux/sched/clock.h>
46 		return -EINVAL;  in parse_u64()
50 			return -ERANGE;  in parse_u64()
52 		if (v > U64_MAX - (*cp - '0'))  in parse_u64()
53 			return -ERANGE;  in parse_u64()
54 		v += *cp - '0';  in parse_u64()
59 	return cp - start;  in parse_u64()
66 	while (p--) {  in bch2_pow()
68 			return -ERANGE;  in bch2_pow()
110 	return cp - start;  in parse_unit_suffix()
144 		return -ERANGE;  in __bch2_strtou64_h()
148 		return -ERANGE;  in __bch2_strtou64_h()
152 		return -ERANGE;  in __bch2_strtou64_h()
156 	return cp - start;  in __bch2_strtou64_h()
162 	bool positive = *cp != '-';  in __bch2_strtoh()
165 	if (*cp == '+' || *cp == '-')  in __bch2_strtoh()
173 		return -EINVAL;  in __bch2_strtoh()
177 			return -ERANGE;  in __bch2_strtoh()
180 			return -ERANGE;  in __bch2_strtoh()
183 			return -ERANGE;  in __bch2_strtoh()
184 		v = -v;  in __bch2_strtoh()
213 		return -ENOMEM;  in STRTO_H()
218 		int flag = match_string(list, -1, p);  in STRTO_H()
221 			ret = -1;  in STRTO_H()
247 		prt_char(out, '0' + ((v >> --nr_bits) & 1));  in bch2_prt_u64_base2_nbits()
275 		printk("%s%.*s\n", prefix, (int) (p - lines), lines);  in __bch2_print_string_as_lines()
300 	stack->nr = 0;  in bch2_save_backtrace()
305 	if (!down_read_trylock(&task->signal->exec_update_lock))  in bch2_save_backtrace()
306 		return -1;  in bch2_save_backtrace()
309 		nr_entries = stack_trace_save_tsk(task, stack->data, stack->size, skipnr + 1);  in bch2_save_backtrace()
310 	} while (nr_entries == stack->size &&  in bch2_save_backtrace()
311 		 !(ret = darray_make_room_gfp(stack, stack->size * 2, gfp)));  in bch2_save_backtrace()
313 	stack->nr = nr_entries;  in bch2_save_backtrace()
314 	up_read(&task->signal->exec_update_lock);  in bch2_save_backtrace()
359 void bch2_pr_time_units(struct printbuf *out, u64 ns)  in bch2_pr_time_units()  argument
361 	const struct time_unit *u = bch2_pick_time_units(ns);  in bch2_pr_time_units()
363 	prt_printf(out, "%llu %s", div64_u64(ns, u->nsecs), u->name);  in bch2_pr_time_units()
366 static void bch2_pr_time_units_aligned(struct printbuf *out, u64 ns)  in bch2_pr_time_units_aligned()  argument
368 	const struct time_unit *u = bch2_pick_time_units(ns);  in bch2_pr_time_units_aligned()
370 	prt_printf(out, "%llu \r%s", div64_u64(ns, u->nsecs), u->name);  in bch2_pr_time_units_aligned()
373 static inline void pr_name_and_units(struct printbuf *out, const char *name, u64 ns)  in pr_name_and_units()  argument
376 	bch2_pr_time_units_aligned(out, ns);  in pr_name_and_units()
388 	if (stats->buffer) {  in bch2_time_stats_to_text()
391 		spin_lock_irq(&stats->lock);  in bch2_time_stats_to_text()
393 			__bch2_time_stats_clear_buffer(stats, per_cpu_ptr(stats->buffer, cpu));  in bch2_time_stats_to_text()
394 		spin_unlock_irq(&stats->lock);  in bch2_time_stats_to_text()
400 	if (stats->freq_stats.n) {  in bch2_time_stats_to_text()
401 		f_mean = mean_and_variance_get_mean(stats->freq_stats);  in bch2_time_stats_to_text()
402 		f_stddev = mean_and_variance_get_stddev(stats->freq_stats);  in bch2_time_stats_to_text()
403 		d_mean = mean_and_variance_get_mean(stats->duration_stats);  in bch2_time_stats_to_text()
404 		d_stddev = mean_and_variance_get_stddev(stats->duration_stats);  in bch2_time_stats_to_text()
407 	printbuf_tabstop_push(out, out->indent + TABSTOP_SIZE);  in bch2_time_stats_to_text()
408 	prt_printf(out, "count:\t%llu\n", stats->duration_stats.n);  in bch2_time_stats_to_text()
413 	printbuf_tabstop_push(out, out->indent + 20);  in bch2_time_stats_to_text()
421 	printbuf_tabstop_push(out, out->indent + 20);  in bch2_time_stats_to_text()
426 	prt_printf(out, "duration of events\n");  in bch2_time_stats_to_text()
429 	pr_name_and_units(out, "min:", stats->min_duration);  in bch2_time_stats_to_text()
430 	pr_name_and_units(out, "max:", stats->max_duration);  in bch2_time_stats_to_text()
431 	pr_name_and_units(out, "total:", stats->total_duration);  in bch2_time_stats_to_text()
436 …bch2_pr_time_units_aligned(out, mean_and_variance_weighted_get_mean(stats->duration_stats_weighted…  in bch2_time_stats_to_text()
442 …bch2_pr_time_units_aligned(out, mean_and_variance_weighted_get_stddev(stats->duration_stats_weight…  in bch2_time_stats_to_text()
450 	pr_name_and_units(out, "min:", stats->min_freq);  in bch2_time_stats_to_text()
451 	pr_name_and_units(out, "max:", stats->max_freq);  in bch2_time_stats_to_text()
456 …bch2_pr_time_units_aligned(out, mean_and_variance_weighted_get_mean(stats->freq_stats_weighted, TI…  in bch2_time_stats_to_text()
462 …bch2_pr_time_units_aligned(out, mean_and_variance_weighted_get_stddev(stats->freq_stats_weighted, …  in bch2_time_stats_to_text()
472 			bch2_pick_time_units(quantiles->entries[i].m);  in bch2_time_stats_to_text()
475 		prt_printf(out, "quantiles (%s):\t", u->name);  in bch2_time_stats_to_text()
477 			bool is_last = eytzinger0_next(i, NR_QUANTILES) == -1;  in bch2_time_stats_to_text()
479 			u64 q = max(quantiles->entries[i].m, last_q);  in bch2_time_stats_to_text()
480 			prt_printf(out, "%llu ", div64_u64(q, u->nsecs));  in bch2_time_stats_to_text()
491  * bch2_ratelimit_delay() - return how long to delay until the next time to do
500 	return time_after64(d->next, now)  in bch2_ratelimit_delay()
501 		? nsecs_to_jiffies(d->next - now)  in bch2_ratelimit_delay()
506  * bch2_ratelimit_increment() - increment @d by the amount of work done
514 	d->next += div_u64(done * NSEC_PER_SEC, d->rate);  in bch2_ratelimit_increment()
516 	if (time_before64(now + NSEC_PER_SEC, d->next))  in bch2_ratelimit_increment()
517 		d->next = now + NSEC_PER_SEC;  in bch2_ratelimit_increment()
519 	if (time_after64(now - NSEC_PER_SEC * 2, d->next))  in bch2_ratelimit_increment()
520 		d->next = now - NSEC_PER_SEC * 2;  in bch2_ratelimit_increment()
530  * @sign: 1 or -1; 1 if increasing the rate makes actual go up, -1 if increasing
538 	unsigned long seconds_since_update = (jiffies - pd->last_update) / HZ;  in bch2_pd_controller_update()
543 	pd->last_update = jiffies;  in bch2_pd_controller_update()
545 	proportional = actual - target;  in bch2_pd_controller_update()
547 	proportional = div_s64(proportional, pd->p_term_inverse);  in bch2_pd_controller_update()
549 	derivative = actual - pd->last_actual;  in bch2_pd_controller_update()
551 	derivative = ewma_add(pd->smoothed_derivative, derivative,  in bch2_pd_controller_update()
552 			      (pd->d_term / seconds_since_update) ?: 1);  in bch2_pd_controller_update()
553 	derivative = derivative * pd->d_term;  in bch2_pd_controller_update()
554 	derivative = div_s64(derivative, pd->p_term_inverse);  in bch2_pd_controller_update()
560 	    pd->backpressure &&  in bch2_pd_controller_update()
562 			 pd->rate.next + NSEC_PER_MSEC))  in bch2_pd_controller_update()
565 	change *= (sign * -1);  in bch2_pd_controller_update()
567 	pd->rate.rate = clamp_t(s64, (s64) pd->rate.rate + change,  in bch2_pd_controller_update()
570 	pd->last_actual		= actual;  in bch2_pd_controller_update()
571 	pd->last_derivative	= derivative;  in bch2_pd_controller_update()
572 	pd->last_proportional	= proportional;  in bch2_pd_controller_update()
573 	pd->last_change		= change;  in bch2_pd_controller_update()
574 	pd->last_target		= target;  in bch2_pd_controller_update()
579 	pd->rate.rate		= 1024;  in bch2_pd_controller_init()
580 	pd->last_update		= jiffies;  in bch2_pd_controller_init()
581 	pd->p_term_inverse	= 6000;  in bch2_pd_controller_init()
582 	pd->d_term		= 30;  in bch2_pd_controller_init()
583 	pd->d_smooth		= pd->d_term;  in bch2_pd_controller_init()
584 	pd->backpressure	= 1;  in bch2_pd_controller_init()
589 	if (!out->nr_tabstops)  in bch2_pd_controller_debug_to_text()
593 	prt_human_readable_s64(out, pd->rate.rate);  in bch2_pd_controller_debug_to_text()
597 	prt_human_readable_u64(out, pd->last_target);  in bch2_pd_controller_debug_to_text()
601 	prt_human_readable_u64(out, pd->last_actual);  in bch2_pd_controller_debug_to_text()
605 	prt_human_readable_s64(out, pd->last_proportional);  in bch2_pd_controller_debug_to_text()
609 	prt_human_readable_s64(out, pd->last_derivative);  in bch2_pd_controller_debug_to_text()
613 	prt_human_readable_s64(out, pd->last_change);  in bch2_pd_controller_debug_to_text()
616 	prt_printf(out, "next io:\t%llims\n", div64_s64(pd->rate.next - local_clock(), NSEC_PER_MSEC));  in bch2_pd_controller_debug_to_text()
628 		unsigned len = min_t(size_t, PAGE_SIZE - offset, size);  in bch2_bio_map()
631 		size -= len;  in bch2_bio_map()
643 			return -ENOMEM;  in bch2_bio_alloc_pages()
650 		size -= len;  in bch2_bio_alloc_pages()
665 		rand &= roundup_pow_of_two(max) - 1;  in bch2_rand_range()
749 		BUG_ON(eytzinger0_prev(-1, size) != eytzinger0_last(size));
750 		BUG_ON(eytzinger0_next(-1, size) != eytzinger0_first(size));
752 		BUG_ON(eytzinger0_prev(eytzinger0_first(size), size)	!= -1);
753 		BUG_ON(eytzinger0_next(eytzinger0_last(size), size)	!= -1);
771 	return (*l > *r) - (*r - *l);
776 	int i, c1 = -1, c2 = -1;
792 		pr_info("find_le(%2u) -> [%2zi] = %2i should be %2i",
817 			eytzinger0_find_test_val(test_array, nr, test_array[i] - 1);
828  * Accumulate percpu counters onto one cpu's copy - only valid when access
868 		return -ENOMEM;  in bch2_split_devs()
886 	return -ENOMEM;  in bch2_split_devs()