1 // SPDX-License-Identifier: GPL-2.0
3  * trace_hwlat.c - A simple Hardware Latency detector.
20  * Although certain hardware-inducing latencies are necessary (for example,
22  * and remote management) they can wreak havoc upon any OS-level performance
23  * guarantees toward low-latency, especially when the OS is not even made
27  * sampling the built-in CPU timer, looking for discontiguous readings.
31  *          environment requiring any kind of low-latency performance
34  * Copyright (C) 2008-2009 Jon Masters, Red Hat, Inc. <jcm@redhat.com>
35  * Copyright (C) 2013-2016 Steven Rostedt, Red Hat, Inc. <srostedt@redhat.com>
67 static char *thread_mode_str[] = { "none", "round-robin", "per-cpu" };
91 /* Individual latency samples are stored here when detected. */
134 	struct trace_buffer *buffer = tr->array_buffer.buffer;  in trace_hwlat_sample()
143 	entry->seqnum			= sample->seqnum;  in trace_hwlat_sample()
144 	entry->duration			= sample->duration;  in trace_hwlat_sample()
145 	entry->outer_duration		= sample->outer_duration;  in trace_hwlat_sample()
146 	entry->timestamp		= sample->timestamp;  in trace_hwlat_sample()
147 	entry->nmi_total_ts		= sample->nmi_total_ts;  in trace_hwlat_sample()
148 	entry->nmi_count		= sample->nmi_count;  in trace_hwlat_sample()
149 	entry->count			= sample->count;  in trace_hwlat_sample()
159 #define time_sub(a, b)	((a) - (b))
167 	if (!kdata->kthread)  in trace_hwlat_callback()
176 			kdata->nmi_ts_start = time_get();  in trace_hwlat_callback()
178 			kdata->nmi_total_ts += time_get() - kdata->nmi_ts_start;  in trace_hwlat_callback()
182 		kdata->nmi_count++;  in trace_hwlat_callback()
186  * hwlat_err - report a hwlat error.
191 	trace_array_printk_buf(tr->array_buffer.buffer, _THIS_IP_, msg);	\
195  * get_sample - sample the CPU TSC and look for likely hardware latencies
198  * hardware-induced latency. Called with interrupts disabled and with
211 	int ret = -1;  in get_sample()
216 	kdata->nmi_total_ts = 0;  in get_sample()
217 	kdata->nmi_count = 0;  in get_sample()
280 	/* If we exceed the threshold value, we have found a hardware latency */  in get_sample()
282 		u64 latency;  in get_sample()  local
287 		if (kdata->nmi_total_ts)  in get_sample()
288 			do_div(kdata->nmi_total_ts, NSEC_PER_USEC);  in get_sample()
294 		s.nmi_total_ts = kdata->nmi_total_ts;  in get_sample()
295 		s.nmi_count = kdata->nmi_count;  in get_sample()
299 		latency = max(sample, outer_sample);  in get_sample()
301 		/* Keep a running maximum ever recorded hardware latency */  in get_sample()
302 		if (latency > tr->max_latency) {  in get_sample()
303 			tr->max_latency = latency;  in get_sample()
325 	if (!cpumask_equal(current_mask, current->cpus_ptr))  in move_to_next_cpu()
329 	cpumask_and(current_mask, cpu_online_mask, tr->tracing_cpumask);  in move_to_next_cpu()
347 	pr_info(BANNER "cpumask changed while in round-robin mode, switching to mode none\n");  in move_to_next_cpu()
351  * kthread_fn - The CPU time sampling/hardware latency detection kernel thread
354  * disable interrupts, which does (intentionally) introduce latency since we
374 		interval = hwlat_data.sample_window - hwlat_data.sample_width;  in kthread_fn()
391  * stop_stop_kthread - Inform the hardware latency sampling/detector kthread to stop
393  * This kicks the running hardware latency sampling/detector kernel thread and
402 	kthread = kdata->kthread;  in stop_single_kthread()
408 	kdata->kthread = NULL;  in stop_single_kthread()
416  * start_single_kthread - Kick off the hardware latency sampling/detector kthread
429 	if (kdata->kthread)  in start_single_kthread()
436 		return -ENOMEM;  in start_single_kthread()
440 	cpumask_and(current_mask, cpu_online_mask, tr->tracing_cpumask);  in start_single_kthread()
451 	kdata->kthread = kthread;  in start_single_kthread()
460  * stop_cpu_kthread - Stop a hwlat cpu kthread
473  * stop_per_cpu_kthreads - Inform the hardware latency sampling/detector kthread to stop
475  * This kicks the running hardware latency sampling/detector kernel threads and
489  * start_cpu_kthread - Start a hwlat cpu kthread
502 		return -ENOMEM;  in start_cpu_kthread()
525 	if (!cpumask_test_cpu(cpu, tr->tracing_cpumask))  in hwlat_hotplug_workfn()
539  * hwlat_cpu_init - CPU hotplug online callback function
548  * hwlat_cpu_die - CPU hotplug offline callback function
575  * start_per_cpu_kthreads - Kick off the hardware latency sampling/detector kthreads
591 	cpumask_and(current_mask, cpu_online_mask, tr->tracing_cpumask);  in start_per_cpu_kthreads()
640 	if (mode < MODE_MAX - 1) /* if mode is any but last */  in s_mode_show()
668  * hwlat_mode_write - Write function for "mode" entry
675  * to the hardware latency detector. hwlatd has different operation modes.
678  * the "round-robin" one, in which a single hwlatd thread runs, migrating
679  * among the allowed CPUs in a round-robin fashion. The "per-cpu" mode
691 		return -EINVAL;  in hwlat_mode_write()
694 		return -EFAULT;  in hwlat_mode_write()
700 	ret = -EINVAL;  in hwlat_mode_write()
764  * init_tracefs - A function to initialize the tracefs interface files
778 		return -ENOMEM;  in init_tracefs()
782 		return -ENOMEM;  in init_tracefs()
809 	return -ENOMEM;  in init_tracefs()
836 		return -EBUSY;  in hwlat_tracer_init()
841 	tr->max_latency = 0;  in hwlat_tracer_init()