1 // SPDX-License-Identifier: GPL-2.0
8  * print a warning if not and turn off the TSC clock-source.
10  * The warp-check is point-to-point between two CPUs, the CPU
11  * initiating the bootup is the 'source CPU', the freshly booting
12  * CPU is the 'target CPU'.
14  * Only two CPUs may participate - they can enter in any order.
15  * ( The serial nature of the boot logic and the CPU hotplug lock
63 	if (!resume && time_before(jiffies, adj->nextcheck))  in tsc_verify_tsc_adjust()
66 	adj->nextcheck = jiffies + HZ;  in tsc_verify_tsc_adjust()
69 	if (adj->adjusted == curval)  in tsc_verify_tsc_adjust()
73 	wrmsrl(MSR_IA32_TSC_ADJUST, adj->adjusted);  in tsc_verify_tsc_adjust()
75 	if (!adj->warned || resume) {  in tsc_verify_tsc_adjust()
76 		pr_warn(FW_BUG "TSC ADJUST differs: CPU%u %lld --> %lld. Restoring\n",  in tsc_verify_tsc_adjust()
77 			smp_processor_id(), adj->adjusted, curval);  in tsc_verify_tsc_adjust()
78 		adj->warned = true;  in tsc_verify_tsc_adjust()
123 				   unsigned int cpu, bool bootcpu)  in tsc_sanitize_first_cpu()  argument
126 	 * First online CPU in a package stores the boot value in the  in tsc_sanitize_first_cpu()
131 	 * On the boot cpu we just force set the ADJUST value to 0 if it's  in tsc_sanitize_first_cpu()
143 			pr_warn(FW_BUG "TSC ADJUST: CPU%u: %lld force to 0\n",  in tsc_sanitize_first_cpu()
144 				cpu, bootval);  in tsc_sanitize_first_cpu()
148 			pr_info("TSC ADJUST: CPU%u: %lld NOT forced to 0\n",  in tsc_sanitize_first_cpu()
149 				cpu, bootval);  in tsc_sanitize_first_cpu()
152 	cur->adjusted = bootval;  in tsc_sanitize_first_cpu()
169 	cur->bootval = bootval;  in tsc_store_and_check_tsc_adjust()
170 	cur->nextcheck = jiffies + HZ;  in tsc_store_and_check_tsc_adjust()
183 	unsigned int refcpu, cpu = smp_processor_id();  in tsc_store_and_check_tsc_adjust()  local
191 	cur->bootval = bootval;  in tsc_store_and_check_tsc_adjust()
192 	cur->nextcheck = jiffies + HZ;  in tsc_store_and_check_tsc_adjust()
193 	cur->warned = false;  in tsc_store_and_check_tsc_adjust()
198 	cur->adjusted = bootval;  in tsc_store_and_check_tsc_adjust()
201 	 * Check whether this CPU is the first in a package to come up. In  in tsc_store_and_check_tsc_adjust()
205 	 * boot CPU topology_core_cpumask() might not be available yet.  in tsc_store_and_check_tsc_adjust()
207 	mask = topology_core_cpumask(cpu);  in tsc_store_and_check_tsc_adjust()
208 	refcpu = mask ? cpumask_any_but(mask, cpu) : nr_cpu_ids;  in tsc_store_and_check_tsc_adjust()
221 	if (bootval != ref->bootval)  in tsc_store_and_check_tsc_adjust()
226 	 * value on this newly upcoming CPU differs from the adjustment  in tsc_store_and_check_tsc_adjust()
227 	 * value of the already online CPU in this package, set it to that  in tsc_store_and_check_tsc_adjust()
230 	if (bootval != ref->adjusted) {  in tsc_store_and_check_tsc_adjust()
231 		cur->adjusted = ref->adjusted;  in tsc_store_and_check_tsc_adjust()
232 		wrmsrl(MSR_IA32_TSC_ADJUST, ref->adjusted);  in tsc_store_and_check_tsc_adjust()
236 	 * test:  in tsc_store_and_check_tsc_adjust()
251  * we want to have the fastest, inlined, non-debug version
252  * of a critical section, to be able to prove TSC time-warps:
262  * TSC-warp measurement loop running on both CPUs.  This is not called
280 		 * another CPU) and update the previous TSC timestamp.  in check_tsc_warp()
302 		 * we saw a time-warp of the TSC going backwards:  in check_tsc_warp()
306 			max_warp = max(max_warp, prev - now);  in check_tsc_warp()
310 			 * one CPU should observe time going backwards.  in check_tsc_warp()
319 	WARN(!(now-start),  in check_tsc_warp()
321 			now-start, end-start);  in check_tsc_warp()
326  * If the target CPU coming online doesn't have any of its core-siblings
327  * online, a timeout of 20msec will be used for the TSC-warp measurement
330  * have more and more logical-siblings in that socket).
333  * core-siblings, if the first logical CPU in a socket passed the sync test.
334  * But as the TSC is per-logical CPU and can potentially be modified wrongly
335  * by the bios, TSC sync test for smaller duration should be able
339 static inline unsigned int loop_timeout(int cpu)  in loop_timeout()  argument
341 	return (cpumask_weight(topology_core_cpumask(cpu)) > 1) ? 2 : 20;  in loop_timeout()
352  * The freshly booted CPU initiates this via an async SMP function call.
356 	unsigned int cpu = (unsigned long)__cpu;  in check_tsc_sync_source()  local
360 	 * Set the maximum number of test runs to  in check_tsc_sync_source()
361 	 *  1 if the CPU does not provide the TSC_ADJUST MSR  in check_tsc_sync_source()
370 	while (atomic_read(&start_count) != cpus - 1)  in check_tsc_sync_source()
378 	check_tsc_warp(loop_timeout(cpu));  in check_tsc_sync_source()
380 	while (atomic_read(&stop_count) != cpus-1)  in check_tsc_sync_source()
384 	 * If the test was successful set the number of runs to zero and  in check_tsc_sync_source()
391 		pr_debug("TSC synchronization [CPU#%d -> CPU#%u]: passed\n",  in check_tsc_sync_source()
392 			smp_processor_id(), cpu);  in check_tsc_sync_source()
398 		pr_warn("TSC synchronization [CPU#%d -> CPU#%u]:\n",  in check_tsc_sync_source()
399 			smp_processor_id(), cpu);  in check_tsc_sync_source()
408 	 * Reset it - just in case we boot another CPU later:  in check_tsc_sync_source()
434 	unsigned int cpu = smp_processor_id();  in check_tsc_sync_target()  local
444 	 * successful skip the test.  in check_tsc_sync_target()
446 	 * The test is also skipped when the TSC is marked reliable. This  in check_tsc_sync_target()
454 	/* Kick the control CPU into the TSC synchronization function */  in check_tsc_sync_target()
456 				 (unsigned long *)(unsigned long)cpu, 0);  in check_tsc_sync_target()
459 	 * Register this CPU's participation and wait for the  in check_tsc_sync_target()
460 	 * source CPU to start the measurement:  in check_tsc_sync_target()
466 	cur_max_warp = check_tsc_warp(loop_timeout(cpu));  in check_tsc_sync_target()
479 	 * Wait for the source CPU to print stuff:  in check_tsc_sync_target()
485 	 * Reset it for the next sync test:  in check_tsc_sync_target()
490 	 * Check the number of remaining test runs. If not zero, the test  in check_tsc_sync_target()
492 	 * test was either successful or failed terminally.  in check_tsc_sync_target()
498 	 * If the warp value of this CPU is 0, then the other CPU  in check_tsc_sync_target()
503 		cur_max_warp = -gbl_max_warp;  in check_tsc_sync_target()
510 	 * really depends on CPU, node distance and frequency. So  in check_tsc_sync_target()
516 	cur->adjusted += cur_max_warp;  in check_tsc_sync_target()
518 	pr_warn("TSC ADJUST compensate: CPU%u observed %lld warp. Adjust: %lld\n",  in check_tsc_sync_target()
519 		cpu, cur_max_warp, cur->adjusted);  in check_tsc_sync_target()
521 	wrmsrl(MSR_IA32_TSC_ADJUST, cur->adjusted);  in check_tsc_sync_target()