291 	struct osnoise_variables *osn_var;  in osn_var_reset()  local
299 		osn_var = per_cpu_ptr(&per_cpu_osnoise_var, cpu);  in osn_var_reset()
300 		memset(osn_var, 0, sizeof(*osn_var));  in osn_var_reset()
377 static inline int timerlat_softirq_exit(struct osnoise_variables *osn_var)  in timerlat_softirq_exit()  argument
385 		osn_var->softirq.arrival_time = 0;  in timerlat_softirq_exit()
386 		osn_var->softirq.delta_start = 0;  in timerlat_softirq_exit()
392 static inline int timerlat_thread_exit(struct osnoise_variables *osn_var)  in timerlat_thread_exit()  argument
400 		osn_var->thread.delta_start = 0;  in timerlat_thread_exit()
401 		osn_var->thread.arrival_time = 0;  in timerlat_thread_exit()
412 static inline int timerlat_softirq_exit(struct osnoise_variables *osn_var)  in timerlat_softirq_exit()  argument
416 static inline int timerlat_thread_exit(struct osnoise_variables *osn_var)  in timerlat_thread_exit()  argument
726 cond_move_irq_delta_start(struct osnoise_variables *osn_var, u64 duration)  in cond_move_irq_delta_start()  argument
728 	if (osn_var->irq.delta_start)  in cond_move_irq_delta_start()
729 		osn_var->irq.delta_start += duration;  in cond_move_irq_delta_start()
742 cond_move_softirq_delta_start(struct osnoise_variables *osn_var, u64 duration)  in cond_move_softirq_delta_start()  argument
744 	if (osn_var->softirq.delta_start)  in cond_move_softirq_delta_start()
745 		osn_var->softirq.delta_start += duration;  in cond_move_softirq_delta_start()
748 #define cond_move_softirq_delta_start(osn_var, duration) do {} while (0)  argument
760 cond_move_thread_delta_start(struct osnoise_variables *osn_var, u64 duration)  in cond_move_thread_delta_start()  argument
762 	if (osn_var->thread.delta_start)  in cond_move_thread_delta_start()
763 		osn_var->thread.delta_start += duration;  in cond_move_thread_delta_start()
794 get_int_safe_duration(struct osnoise_variables *osn_var, u64 *delta_start)  in get_int_safe_duration()  argument
800 		int_counter = local_read(&osn_var->int_counter);  in get_int_safe_duration()
809 	} while (int_counter != local_read(&osn_var->int_counter));  in get_int_safe_duration()
833 set_int_safe_time(struct osnoise_variables *osn_var, u64 *time)  in set_int_safe_time()  argument
838 		int_counter = local_read(&osn_var->int_counter);  in set_int_safe_time()
846 	} while (int_counter != local_read(&osn_var->int_counter));  in set_int_safe_time()
856 copy_int_safe_time(struct osnoise_variables *osn_var, u64 *dst, u64 *src)  in copy_int_safe_time()  argument
861 		int_counter = local_read(&osn_var->int_counter);  in copy_int_safe_time()
869 	} while (int_counter != local_read(&osn_var->int_counter));  in copy_int_safe_time()
885 	struct osnoise_variables *osn_var = this_cpu_osn_var();  in trace_osnoise_callback()  local
888 	if (!osn_var->sampling)  in trace_osnoise_callback()
897 			osn_var->nmi.delta_start = time_get();  in trace_osnoise_callback()
898 			local_inc(&osn_var->int_counter);  in trace_osnoise_callback()
900 			duration = time_get() - osn_var->nmi.delta_start;  in trace_osnoise_callback()
902 			trace_nmi_noise(osn_var->nmi.delta_start, duration);  in trace_osnoise_callback()
904 			cond_move_irq_delta_start(osn_var, duration);  in trace_osnoise_callback()
905 			cond_move_softirq_delta_start(osn_var, duration);  in trace_osnoise_callback()
906 			cond_move_thread_delta_start(osn_var, duration);  in trace_osnoise_callback()
911 		osn_var->nmi.count++;  in trace_osnoise_callback()
925 	struct osnoise_variables *osn_var = this_cpu_osn_var();  in osnoise_trace_irq_entry()  local
927 	if (!osn_var->sampling)  in osnoise_trace_irq_entry()
933 	osn_var->irq.arrival_time = time_get();  in osnoise_trace_irq_entry()
934 	set_int_safe_time(osn_var, &osn_var->irq.delta_start);  in osnoise_trace_irq_entry()
935 	osn_var->irq.count++;  in osnoise_trace_irq_entry()
937 	local_inc(&osn_var->int_counter);  in osnoise_trace_irq_entry()
948 	struct osnoise_variables *osn_var = this_cpu_osn_var();  in osnoise_trace_irq_exit()  local
951 	if (!osn_var->sampling)  in osnoise_trace_irq_exit()
954 	duration = get_int_safe_duration(osn_var, &osn_var->irq.delta_start);  in osnoise_trace_irq_exit()
955 	trace_irq_noise(id, desc, osn_var->irq.arrival_time, duration);  in osnoise_trace_irq_exit()
956 	osn_var->irq.arrival_time = 0;  in osnoise_trace_irq_exit()
957 	cond_move_softirq_delta_start(osn_var, duration);  in osnoise_trace_irq_exit()
958 	cond_move_thread_delta_start(osn_var, duration);  in osnoise_trace_irq_exit()
1056 	struct osnoise_variables *osn_var = this_cpu_osn_var();  in trace_softirq_entry_callback()  local
1058 	if (!osn_var->sampling)  in trace_softirq_entry_callback()
1064 	osn_var->softirq.arrival_time = time_get();  in trace_softirq_entry_callback()
1065 	set_int_safe_time(osn_var, &osn_var->softirq.delta_start);  in trace_softirq_entry_callback()
1066 	osn_var->softirq.count++;  in trace_softirq_entry_callback()
1068 	local_inc(&osn_var->int_counter);  in trace_softirq_entry_callback()
1079 	struct osnoise_variables *osn_var = this_cpu_osn_var();  in trace_softirq_exit_callback()  local
1082 	if (!osn_var->sampling)  in trace_softirq_exit_callback()
1086 		if (!timerlat_softirq_exit(osn_var))  in trace_softirq_exit_callback()
1089 	duration = get_int_safe_duration(osn_var, &osn_var->softirq.delta_start);  in trace_softirq_exit_callback()
1090 	trace_softirq_noise(vec_nr, osn_var->softirq.arrival_time, duration);  in trace_softirq_exit_callback()
1091 	cond_move_thread_delta_start(osn_var, duration);  in trace_softirq_exit_callback()
1092 	osn_var->softirq.arrival_time = 0;  in trace_softirq_exit_callback()
1152 thread_entry(struct osnoise_variables *osn_var, struct task_struct *t)  in thread_entry()  argument
1154 	if (!osn_var->sampling)  in thread_entry()
1160 	osn_var->thread.arrival_time = time_get();  in thread_entry()
1162 	set_int_safe_time(osn_var, &osn_var->thread.delta_start);  in thread_entry()
1164 	osn_var->thread.count++;  in thread_entry()
1165 	local_inc(&osn_var->int_counter);  in thread_entry()
1174 thread_exit(struct osnoise_variables *osn_var, struct task_struct *t)  in thread_exit()  argument
1178 	if (!osn_var->sampling)  in thread_exit()
1182 		if (!timerlat_thread_exit(osn_var))  in thread_exit()
1185 	duration = get_int_safe_duration(osn_var, &osn_var->thread.delta_start);  in thread_exit()
1187 	trace_thread_noise(t, osn_var->thread.arrival_time, duration);  in thread_exit()
1189 	osn_var->thread.arrival_time = 0;  in thread_exit()
1227 	struct osnoise_variables *osn_var;  in trace_sched_migrate_callback()  local
1230 	osn_var = per_cpu_ptr(&per_cpu_osnoise_var, cpu);  in trace_sched_migrate_callback()
1231 	if (osn_var->pid == p->pid && dest_cpu != cpu) {  in trace_sched_migrate_callback()
1276 	struct osnoise_variables *osn_var = this_cpu_osn_var();  in trace_sched_switch_callback()  local
1279 	if ((p->pid != osn_var->pid) || !workload)  in trace_sched_switch_callback()
1280 		thread_exit(osn_var, p);  in trace_sched_switch_callback()
1282 	if ((n->pid != osn_var->pid) || !workload)  in trace_sched_switch_callback()
1283 		thread_entry(osn_var, n);  in trace_sched_switch_callback()
1331 save_osn_sample_stats(struct osnoise_variables *osn_var, struct osnoise_sample *s)  in save_osn_sample_stats()  argument
1333 	s->nmi_count = osn_var->nmi.count;  in save_osn_sample_stats()
1334 	s->irq_count = osn_var->irq.count;  in save_osn_sample_stats()
1335 	s->softirq_count = osn_var->softirq.count;  in save_osn_sample_stats()
1336 	s->thread_count = osn_var->thread.count;  in save_osn_sample_stats()
1347 diff_osn_sample_stats(struct osnoise_variables *osn_var, struct osnoise_sample *s)  in diff_osn_sample_stats()  argument
1349 	s->nmi_count = osn_var->nmi.count - s->nmi_count;  in diff_osn_sample_stats()
1350 	s->irq_count = osn_var->irq.count - s->irq_count;  in diff_osn_sample_stats()
1351 	s->softirq_count = osn_var->softirq.count - s->softirq_count;  in diff_osn_sample_stats()
1352 	s->thread_count = osn_var->thread.count - s->thread_count;  in diff_osn_sample_stats()
1423 	struct osnoise_variables *osn_var = this_cpu_osn_var();  in run_osnoise()  local
1445 	osn_var->pid = current->pid;  in run_osnoise()
1450 	save_osn_sample_stats(osn_var, &s);  in run_osnoise()
1469 	osn_var->sampling = true;  in run_osnoise()
1487 	last_int_count = set_int_safe_time(osn_var, &last_sample);  in run_osnoise()
1493 		int_count = set_int_safe_time(osn_var, &sample);  in run_osnoise()
1579 	osn_var->sampling = false;  in run_osnoise()
1604 	diff_osn_sample_stats(osn_var, &s);  in run_osnoise()
1737 	struct osnoise_variables *osn_var = this_cpu_osn_var();  in timerlat_irq()  local
1757 	osn_var->thread.arrival_time = time_get();  in timerlat_irq()
1780 	if (!IS_ENABLED(CONFIG_PREEMPT_RT) && osn_var->softirq.delta_start) {  in timerlat_irq()
1781 		copy_int_safe_time(osn_var, &osn_var->thread.delta_start,  in timerlat_irq()
1782 				   &osn_var->softirq.delta_start);  in timerlat_irq()
1784 		copy_int_safe_time(osn_var, &osn_var->softirq.delta_start,  in timerlat_irq()
1785 				    &osn_var->irq.delta_start);  in timerlat_irq()
1787 		copy_int_safe_time(osn_var, &osn_var->thread.delta_start,  in timerlat_irq()
1788 				    &osn_var->irq.delta_start);  in timerlat_irq()
1871 	struct osnoise_variables *osn_var = this_cpu_osn_var();  in timerlat_main()  local
1902 	osn_var->pid = current->pid;  in timerlat_main()
1910 	osn_var->sampling = 1;  in timerlat_main()
2409 	struct osnoise_variables *osn_var;  in timerlat_fd_open()  local
2426 	osn_var = this_cpu_osn_var();  in timerlat_fd_open()
2431 	if (osn_var->pid) {  in timerlat_fd_open()
2457 	osn_var->kthread = current;  in timerlat_fd_open()
2458 	osn_var->pid = current->pid;  in timerlat_fd_open()
2489 	struct osnoise_variables *osn_var;  in timerlat_fd_read()  local
2517 	osn_var = this_cpu_osn_var();  in timerlat_fd_read()
2527 	if (likely(osn_var->sampling)) {  in timerlat_fd_read()
2556 		osn_var->sampling = 1;  in timerlat_fd_read()
2593 	struct osnoise_variables *osn_var;  in timerlat_fd_release()  local
2600 	osn_var = per_cpu_ptr(&per_cpu_osnoise_var, cpu);  in timerlat_fd_release()
2607 	osn_var->sampling = 0;  in timerlat_fd_release()
2608 	osn_var->pid = 0;  in timerlat_fd_release()
2613 	if (osn_var->kthread) {  in timerlat_fd_release()
2614 		put_task_struct(osn_var->kthread);  in timerlat_fd_release()
2615 		osn_var->kthread = NULL;  in timerlat_fd_release()