1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3  * X86 specific Hyper-V initialization code.
4  *
5  * Copyright (C) 2016, Microsoft, Inc.
6  *
7  * Author : K. Y. Srinivasan <kys@microsoft.com>
8  */
9 
10 #define pr_fmt(fmt)  "Hyper-V: " fmt
11 
12 #include <linux/efi.h>
13 #include <linux/types.h>
14 #include <linux/bitfield.h>
15 #include <linux/io.h>
16 #include <asm/apic.h>
17 #include <asm/desc.h>
18 #include <asm/e820/api.h>
19 #include <asm/sev.h>
20 #include <asm/ibt.h>
21 #include <asm/hypervisor.h>
22 #include <asm/hyperv-tlfs.h>
23 #include <asm/mshyperv.h>
24 #include <asm/idtentry.h>
25 #include <asm/set_memory.h>
26 #include <linux/kexec.h>
27 #include <linux/version.h>
28 #include <linux/vmalloc.h>
29 #include <linux/mm.h>
30 #include <linux/hyperv.h>
31 #include <linux/slab.h>
32 #include <linux/kernel.h>
33 #include <linux/cpuhotplug.h>
34 #include <linux/syscore_ops.h>
35 #include <clocksource/hyperv_timer.h>
36 #include <linux/highmem.h>
37 
38 u64 hv_current_partition_id = ~0ull;
39 EXPORT_SYMBOL_GPL(hv_current_partition_id);
40 
41 void *hv_hypercall_pg;
42 EXPORT_SYMBOL_GPL(hv_hypercall_pg);
43 
44 union hv_ghcb * __percpu *hv_ghcb_pg;
45 
46 /* Storage to save the hypercall page temporarily for hibernation */
47 static void *hv_hypercall_pg_saved;
48 
49 struct hv_vp_assist_page **hv_vp_assist_page;
50 EXPORT_SYMBOL_GPL(hv_vp_assist_page);
51 
hyperv_init_ghcb(void)52 static int hyperv_init_ghcb(void)
53 {
54 	u64 ghcb_gpa;
55 	void *ghcb_va;
56 	void **ghcb_base;
57 
58 	if (!ms_hyperv.paravisor_present || !hv_isolation_type_snp())
59 		return 0;
60 
61 	if (!hv_ghcb_pg)
62 		return -EINVAL;
63 
64 	/*
65 	 * GHCB page is allocated by paravisor. The address
66 	 * returned by MSR_AMD64_SEV_ES_GHCB is above shared
67 	 * memory boundary and map it here.
68 	 */
69 	rdmsrl(MSR_AMD64_SEV_ES_GHCB, ghcb_gpa);
70 
71 	/* Mask out vTOM bit. ioremap_cache() maps decrypted */
72 	ghcb_gpa &= ~ms_hyperv.shared_gpa_boundary;
73 	ghcb_va = (void *)ioremap_cache(ghcb_gpa, HV_HYP_PAGE_SIZE);
74 	if (!ghcb_va)
75 		return -ENOMEM;
76 
77 	ghcb_base = (void **)this_cpu_ptr(hv_ghcb_pg);
78 	*ghcb_base = ghcb_va;
79 
80 	return 0;
81 }
82 
hv_cpu_init(unsigned int cpu)83 static int hv_cpu_init(unsigned int cpu)
84 {
85 	union hv_vp_assist_msr_contents msr = { 0 };
86 	struct hv_vp_assist_page **hvp;
87 	int ret;
88 
89 	ret = hv_common_cpu_init(cpu);
90 	if (ret)
91 		return ret;
92 
93 	if (!hv_vp_assist_page)
94 		return 0;
95 
96 	hvp = &hv_vp_assist_page[cpu];
97 	if (hv_root_partition) {
98 		/*
99 		 * For root partition we get the hypervisor provided VP assist
100 		 * page, instead of allocating a new page.
101 		 */
102 		rdmsrl(HV_X64_MSR_VP_ASSIST_PAGE, msr.as_uint64);
103 		*hvp = memremap(msr.pfn << HV_X64_MSR_VP_ASSIST_PAGE_ADDRESS_SHIFT,
104 				PAGE_SIZE, MEMREMAP_WB);
105 	} else {
106 		/*
107 		 * The VP assist page is an "overlay" page (see Hyper-V TLFS's
108 		 * Section 5.2.1 "GPA Overlay Pages"). Here it must be zeroed
109 		 * out to make sure we always write the EOI MSR in
110 		 * hv_apic_eoi_write() *after* the EOI optimization is disabled
111 		 * in hv_cpu_die(), otherwise a CPU may not be stopped in the
112 		 * case of CPU offlining and the VM will hang.
113 		 */
114 		if (!*hvp) {
115 			*hvp = __vmalloc(PAGE_SIZE, GFP_KERNEL | __GFP_ZERO);
116 
117 			/*
118 			 * Hyper-V should never specify a VM that is a Confidential
119 			 * VM and also running in the root partition. Root partition
120 			 * is blocked to run in Confidential VM. So only decrypt assist
121 			 * page in non-root partition here.
122 			 */
123 			if (*hvp && !ms_hyperv.paravisor_present && hv_isolation_type_snp()) {
124 				WARN_ON_ONCE(set_memory_decrypted((unsigned long)(*hvp), 1));
125 				memset(*hvp, 0, PAGE_SIZE);
126 			}
127 		}
128 
129 		if (*hvp)
130 			msr.pfn = vmalloc_to_pfn(*hvp);
131 
132 	}
133 	if (!WARN_ON(!(*hvp))) {
134 		msr.enable = 1;
135 		wrmsrl(HV_X64_MSR_VP_ASSIST_PAGE, msr.as_uint64);
136 	}
137 
138 	return hyperv_init_ghcb();
139 }
140 
141 static void (*hv_reenlightenment_cb)(void);
142 
hv_reenlightenment_notify(struct work_struct * dummy)143 static void hv_reenlightenment_notify(struct work_struct *dummy)
144 {
145 	struct hv_tsc_emulation_status emu_status;
146 
147 	rdmsrl(HV_X64_MSR_TSC_EMULATION_STATUS, *(u64 *)&emu_status);
148 
149 	/* Don't issue the callback if TSC accesses are not emulated */
150 	if (hv_reenlightenment_cb && emu_status.inprogress)
151 		hv_reenlightenment_cb();
152 }
153 static DECLARE_DELAYED_WORK(hv_reenlightenment_work, hv_reenlightenment_notify);
154 
hyperv_stop_tsc_emulation(void)155 void hyperv_stop_tsc_emulation(void)
156 {
157 	u64 freq;
158 	struct hv_tsc_emulation_status emu_status;
159 
160 	rdmsrl(HV_X64_MSR_TSC_EMULATION_STATUS, *(u64 *)&emu_status);
161 	emu_status.inprogress = 0;
162 	wrmsrl(HV_X64_MSR_TSC_EMULATION_STATUS, *(u64 *)&emu_status);
163 
164 	rdmsrl(HV_X64_MSR_TSC_FREQUENCY, freq);
165 	tsc_khz = div64_u64(freq, 1000);
166 }
167 EXPORT_SYMBOL_GPL(hyperv_stop_tsc_emulation);
168 
hv_reenlightenment_available(void)169 static inline bool hv_reenlightenment_available(void)
170 {
171 	/*
172 	 * Check for required features and privileges to make TSC frequency
173 	 * change notifications work.
174 	 */
175 	return ms_hyperv.features & HV_ACCESS_FREQUENCY_MSRS &&
176 		ms_hyperv.misc_features & HV_FEATURE_FREQUENCY_MSRS_AVAILABLE &&
177 		ms_hyperv.features & HV_ACCESS_REENLIGHTENMENT;
178 }
179 
DEFINE_IDTENTRY_SYSVEC(sysvec_hyperv_reenlightenment)180 DEFINE_IDTENTRY_SYSVEC(sysvec_hyperv_reenlightenment)
181 {
182 	apic_eoi();
183 	inc_irq_stat(irq_hv_reenlightenment_count);
184 	schedule_delayed_work(&hv_reenlightenment_work, HZ/10);
185 }
186 
set_hv_tscchange_cb(void (* cb)(void))187 void set_hv_tscchange_cb(void (*cb)(void))
188 {
189 	struct hv_reenlightenment_control re_ctrl = {
190 		.vector = HYPERV_REENLIGHTENMENT_VECTOR,
191 		.enabled = 1,
192 	};
193 	struct hv_tsc_emulation_control emu_ctrl = {.enabled = 1};
194 
195 	if (!hv_reenlightenment_available()) {
196 		pr_warn("reenlightenment support is unavailable\n");
197 		return;
198 	}
199 
200 	if (!hv_vp_index)
201 		return;
202 
203 	hv_reenlightenment_cb = cb;
204 
205 	/* Make sure callback is registered before we write to MSRs */
206 	wmb();
207 
208 	re_ctrl.target_vp = hv_vp_index[get_cpu()];
209 
210 	wrmsrl(HV_X64_MSR_REENLIGHTENMENT_CONTROL, *((u64 *)&re_ctrl));
211 	wrmsrl(HV_X64_MSR_TSC_EMULATION_CONTROL, *((u64 *)&emu_ctrl));
212 
213 	put_cpu();
214 }
215 EXPORT_SYMBOL_GPL(set_hv_tscchange_cb);
216 
clear_hv_tscchange_cb(void)217 void clear_hv_tscchange_cb(void)
218 {
219 	struct hv_reenlightenment_control re_ctrl;
220 
221 	if (!hv_reenlightenment_available())
222 		return;
223 
224 	rdmsrl(HV_X64_MSR_REENLIGHTENMENT_CONTROL, *(u64 *)&re_ctrl);
225 	re_ctrl.enabled = 0;
226 	wrmsrl(HV_X64_MSR_REENLIGHTENMENT_CONTROL, *(u64 *)&re_ctrl);
227 
228 	hv_reenlightenment_cb = NULL;
229 }
230 EXPORT_SYMBOL_GPL(clear_hv_tscchange_cb);
231 
hv_cpu_die(unsigned int cpu)232 static int hv_cpu_die(unsigned int cpu)
233 {
234 	struct hv_reenlightenment_control re_ctrl;
235 	unsigned int new_cpu;
236 	void **ghcb_va;
237 
238 	if (hv_ghcb_pg) {
239 		ghcb_va = (void **)this_cpu_ptr(hv_ghcb_pg);
240 		if (*ghcb_va)
241 			iounmap(*ghcb_va);
242 		*ghcb_va = NULL;
243 	}
244 
245 	hv_common_cpu_die(cpu);
246 
247 	if (hv_vp_assist_page && hv_vp_assist_page[cpu]) {
248 		union hv_vp_assist_msr_contents msr = { 0 };
249 		if (hv_root_partition) {
250 			/*
251 			 * For root partition the VP assist page is mapped to
252 			 * hypervisor provided page, and thus we unmap the
253 			 * page here and nullify it, so that in future we have
254 			 * correct page address mapped in hv_cpu_init.
255 			 */
256 			memunmap(hv_vp_assist_page[cpu]);
257 			hv_vp_assist_page[cpu] = NULL;
258 			rdmsrl(HV_X64_MSR_VP_ASSIST_PAGE, msr.as_uint64);
259 			msr.enable = 0;
260 		}
261 		wrmsrl(HV_X64_MSR_VP_ASSIST_PAGE, msr.as_uint64);
262 	}
263 
264 	if (hv_reenlightenment_cb == NULL)
265 		return 0;
266 
267 	rdmsrl(HV_X64_MSR_REENLIGHTENMENT_CONTROL, *((u64 *)&re_ctrl));
268 	if (re_ctrl.target_vp == hv_vp_index[cpu]) {
269 		/*
270 		 * Reassign reenlightenment notifications to some other online
271 		 * CPU or just disable the feature if there are no online CPUs
272 		 * left (happens on hibernation).
273 		 */
274 		new_cpu = cpumask_any_but(cpu_online_mask, cpu);
275 
276 		if (new_cpu < nr_cpu_ids)
277 			re_ctrl.target_vp = hv_vp_index[new_cpu];
278 		else
279 			re_ctrl.enabled = 0;
280 
281 		wrmsrl(HV_X64_MSR_REENLIGHTENMENT_CONTROL, *((u64 *)&re_ctrl));
282 	}
283 
284 	return 0;
285 }
286 
hv_pci_init(void)287 static int __init hv_pci_init(void)
288 {
289 	bool gen2vm = efi_enabled(EFI_BOOT);
290 
291 	/*
292 	 * A Generation-2 VM doesn't support legacy PCI/PCIe, so both
293 	 * raw_pci_ops and raw_pci_ext_ops are NULL, and pci_subsys_init() ->
294 	 * pcibios_init() doesn't call pcibios_resource_survey() ->
295 	 * e820__reserve_resources_late(); as a result, any emulated persistent
296 	 * memory of E820_TYPE_PRAM (12) via the kernel parameter
297 	 * memmap=nn[KMG]!ss is not added into iomem_resource and hence can't be
298 	 * detected by register_e820_pmem(). Fix this by directly calling
299 	 * e820__reserve_resources_late() here: e820__reserve_resources_late()
300 	 * depends on e820__reserve_resources(), which has been called earlier
301 	 * from setup_arch(). Note: e820__reserve_resources_late() also adds
302 	 * any memory of E820_TYPE_PMEM (7) into iomem_resource, and
303 	 * acpi_nfit_register_region() -> acpi_nfit_insert_resource() ->
304 	 * region_intersects() returns REGION_INTERSECTS, so the memory of
305 	 * E820_TYPE_PMEM won't get added twice.
306 	 *
307 	 * We return 0 here so that pci_arch_init() won't print the warning:
308 	 * "PCI: Fatal: No config space access function found"
309 	 */
310 	if (gen2vm) {
311 		e820__reserve_resources_late();
312 		return 0;
313 	}
314 
315 	/* For Generation-1 VM, we'll proceed in pci_arch_init().  */
316 	return 1;
317 }
318 
hv_suspend(void)319 static int hv_suspend(void)
320 {
321 	union hv_x64_msr_hypercall_contents hypercall_msr;
322 	int ret;
323 
324 	if (hv_root_partition)
325 		return -EPERM;
326 
327 	/*
328 	 * Reset the hypercall page as it is going to be invalidated
329 	 * across hibernation. Setting hv_hypercall_pg to NULL ensures
330 	 * that any subsequent hypercall operation fails safely instead of
331 	 * crashing due to an access of an invalid page. The hypercall page
332 	 * pointer is restored on resume.
333 	 */
334 	hv_hypercall_pg_saved = hv_hypercall_pg;
335 	hv_hypercall_pg = NULL;
336 
337 	/* Disable the hypercall page in the hypervisor */
338 	rdmsrl(HV_X64_MSR_HYPERCALL, hypercall_msr.as_uint64);
339 	hypercall_msr.enable = 0;
340 	wrmsrl(HV_X64_MSR_HYPERCALL, hypercall_msr.as_uint64);
341 
342 	ret = hv_cpu_die(0);
343 	return ret;
344 }
345 
hv_resume(void)346 static void hv_resume(void)
347 {
348 	union hv_x64_msr_hypercall_contents hypercall_msr;
349 	int ret;
350 
351 	ret = hv_cpu_init(0);
352 	WARN_ON(ret);
353 
354 	/* Re-enable the hypercall page */
355 	rdmsrl(HV_X64_MSR_HYPERCALL, hypercall_msr.as_uint64);
356 	hypercall_msr.enable = 1;
357 	hypercall_msr.guest_physical_address =
358 		vmalloc_to_pfn(hv_hypercall_pg_saved);
359 	wrmsrl(HV_X64_MSR_HYPERCALL, hypercall_msr.as_uint64);
360 
361 	hv_hypercall_pg = hv_hypercall_pg_saved;
362 	hv_hypercall_pg_saved = NULL;
363 
364 	/*
365 	 * Reenlightenment notifications are disabled by hv_cpu_die(0),
366 	 * reenable them here if hv_reenlightenment_cb was previously set.
367 	 */
368 	if (hv_reenlightenment_cb)
369 		set_hv_tscchange_cb(hv_reenlightenment_cb);
370 }
371 
372 /* Note: when the ops are called, only CPU0 is online and IRQs are disabled. */
373 static struct syscore_ops hv_syscore_ops = {
374 	.suspend	= hv_suspend,
375 	.resume		= hv_resume,
376 };
377 
378 static void (* __initdata old_setup_percpu_clockev)(void);
379 
hv_stimer_setup_percpu_clockev(void)380 static void __init hv_stimer_setup_percpu_clockev(void)
381 {
382 	/*
383 	 * Ignore any errors in setting up stimer clockevents
384 	 * as we can run with the LAPIC timer as a fallback.
385 	 */
386 	(void)hv_stimer_alloc(false);
387 
388 	/*
389 	 * Still register the LAPIC timer, because the direct-mode STIMER is
390 	 * not supported by old versions of Hyper-V. This also allows users
391 	 * to switch to LAPIC timer via /sys, if they want to.
392 	 */
393 	if (old_setup_percpu_clockev)
394 		old_setup_percpu_clockev();
395 }
396 
hv_get_partition_id(void)397 static void __init hv_get_partition_id(void)
398 {
399 	struct hv_get_partition_id *output_page;
400 	u64 status;
401 	unsigned long flags;
402 
403 	local_irq_save(flags);
404 	output_page = *this_cpu_ptr(hyperv_pcpu_output_arg);
405 	status = hv_do_hypercall(HVCALL_GET_PARTITION_ID, NULL, output_page);
406 	if (!hv_result_success(status)) {
407 		/* No point in proceeding if this failed */
408 		pr_err("Failed to get partition ID: %lld\n", status);
409 		BUG();
410 	}
411 	hv_current_partition_id = output_page->partition_id;
412 	local_irq_restore(flags);
413 }
414 
415 #if IS_ENABLED(CONFIG_HYPERV_VTL_MODE)
get_vtl(void)416 static u8 __init get_vtl(void)
417 {
418 	u64 control = HV_HYPERCALL_REP_COMP_1 | HVCALL_GET_VP_REGISTERS;
419 	struct hv_get_vp_registers_input *input;
420 	struct hv_get_vp_registers_output *output;
421 	unsigned long flags;
422 	u64 ret;
423 
424 	local_irq_save(flags);
425 	input = *this_cpu_ptr(hyperv_pcpu_input_arg);
426 	output = (struct hv_get_vp_registers_output *)input;
427 
428 	memset(input, 0, struct_size(input, element, 1));
429 	input->header.partitionid = HV_PARTITION_ID_SELF;
430 	input->header.vpindex = HV_VP_INDEX_SELF;
431 	input->header.inputvtl = 0;
432 	input->element[0].name0 = HV_X64_REGISTER_VSM_VP_STATUS;
433 
434 	ret = hv_do_hypercall(control, input, output);
435 	if (hv_result_success(ret)) {
436 		ret = output->as64.low & HV_X64_VTL_MASK;
437 	} else {
438 		pr_err("Failed to get VTL(error: %lld) exiting...\n", ret);
439 		BUG();
440 	}
441 
442 	local_irq_restore(flags);
443 	return ret;
444 }
445 #else
get_vtl(void)446 static inline u8 get_vtl(void) { return 0; }
447 #endif
448 
449 /*
450  * This function is to be invoked early in the boot sequence after the
451  * hypervisor has been detected.
452  *
453  * 1. Setup the hypercall page.
454  * 2. Register Hyper-V specific clocksource.
455  * 3. Setup Hyper-V specific APIC entry points.
456  */
hyperv_init(void)457 void __init hyperv_init(void)
458 {
459 	u64 guest_id;
460 	union hv_x64_msr_hypercall_contents hypercall_msr;
461 	int cpuhp;
462 
463 	if (x86_hyper_type != X86_HYPER_MS_HYPERV)
464 		return;
465 
466 	if (hv_common_init())
467 		return;
468 
469 	/*
470 	 * The VP assist page is useless to a TDX guest: the only use we
471 	 * would have for it is lazy EOI, which can not be used with TDX.
472 	 */
473 	if (hv_isolation_type_tdx())
474 		hv_vp_assist_page = NULL;
475 	else
476 		hv_vp_assist_page = kcalloc(num_possible_cpus(),
477 					    sizeof(*hv_vp_assist_page),
478 					    GFP_KERNEL);
479 	if (!hv_vp_assist_page) {
480 		ms_hyperv.hints &= ~HV_X64_ENLIGHTENED_VMCS_RECOMMENDED;
481 
482 		if (!hv_isolation_type_tdx())
483 			goto common_free;
484 	}
485 
486 	if (ms_hyperv.paravisor_present && hv_isolation_type_snp()) {
487 		/* Negotiate GHCB Version. */
488 		if (!hv_ghcb_negotiate_protocol())
489 			hv_ghcb_terminate(SEV_TERM_SET_GEN,
490 					  GHCB_SEV_ES_PROT_UNSUPPORTED);
491 
492 		hv_ghcb_pg = alloc_percpu(union hv_ghcb *);
493 		if (!hv_ghcb_pg)
494 			goto free_vp_assist_page;
495 	}
496 
497 	cpuhp = cpuhp_setup_state(CPUHP_AP_HYPERV_ONLINE, "x86/hyperv_init:online",
498 				  hv_cpu_init, hv_cpu_die);
499 	if (cpuhp < 0)
500 		goto free_ghcb_page;
501 
502 	/*
503 	 * Setup the hypercall page and enable hypercalls.
504 	 * 1. Register the guest ID
505 	 * 2. Enable the hypercall and register the hypercall page
506 	 *
507 	 * A TDX VM with no paravisor only uses TDX GHCI rather than hv_hypercall_pg:
508 	 * when the hypercall input is a page, such a VM must pass a decrypted
509 	 * page to Hyper-V, e.g. hv_post_message() uses the per-CPU page
510 	 * hyperv_pcpu_input_arg, which is decrypted if no paravisor is present.
511 	 *
512 	 * A TDX VM with the paravisor uses hv_hypercall_pg for most hypercalls,
513 	 * which are handled by the paravisor and the VM must use an encrypted
514 	 * input page: in such a VM, the hyperv_pcpu_input_arg is encrypted and
515 	 * used in the hypercalls, e.g. see hv_mark_gpa_visibility() and
516 	 * hv_arch_irq_unmask(). Such a VM uses TDX GHCI for two hypercalls:
517 	 * 1. HVCALL_SIGNAL_EVENT: see vmbus_set_event() and _hv_do_fast_hypercall8().
518 	 * 2. HVCALL_POST_MESSAGE: the input page must be a decrypted page, i.e.
519 	 * hv_post_message() in such a VM can't use the encrypted hyperv_pcpu_input_arg;
520 	 * instead, hv_post_message() uses the post_msg_page, which is decrypted
521 	 * in such a VM and is only used in such a VM.
522 	 */
523 	guest_id = hv_generate_guest_id(LINUX_VERSION_CODE);
524 	wrmsrl(HV_X64_MSR_GUEST_OS_ID, guest_id);
525 
526 	/* With the paravisor, the VM must also write the ID via GHCB/GHCI */
527 	hv_ivm_msr_write(HV_X64_MSR_GUEST_OS_ID, guest_id);
528 
529 	/* A TDX VM with no paravisor only uses TDX GHCI rather than hv_hypercall_pg */
530 	if (hv_isolation_type_tdx() && !ms_hyperv.paravisor_present)
531 		goto skip_hypercall_pg_init;
532 
533 	hv_hypercall_pg = __vmalloc_node_range(PAGE_SIZE, 1, VMALLOC_START,
534 			VMALLOC_END, GFP_KERNEL, PAGE_KERNEL_ROX,
535 			VM_FLUSH_RESET_PERMS, NUMA_NO_NODE,
536 			__builtin_return_address(0));
537 	if (hv_hypercall_pg == NULL)
538 		goto clean_guest_os_id;
539 
540 	rdmsrl(HV_X64_MSR_HYPERCALL, hypercall_msr.as_uint64);
541 	hypercall_msr.enable = 1;
542 
543 	if (hv_root_partition) {
544 		struct page *pg;
545 		void *src;
546 
547 		/*
548 		 * For the root partition, the hypervisor will set up its
549 		 * hypercall page. The hypervisor guarantees it will not show
550 		 * up in the root's address space. The root can't change the
551 		 * location of the hypercall page.
552 		 *
553 		 * Order is important here. We must enable the hypercall page
554 		 * so it is populated with code, then copy the code to an
555 		 * executable page.
556 		 */
557 		wrmsrl(HV_X64_MSR_HYPERCALL, hypercall_msr.as_uint64);
558 
559 		pg = vmalloc_to_page(hv_hypercall_pg);
560 		src = memremap(hypercall_msr.guest_physical_address << PAGE_SHIFT, PAGE_SIZE,
561 				MEMREMAP_WB);
562 		BUG_ON(!src);
563 		memcpy_to_page(pg, 0, src, HV_HYP_PAGE_SIZE);
564 		memunmap(src);
565 
566 		hv_remap_tsc_clocksource();
567 	} else {
568 		hypercall_msr.guest_physical_address = vmalloc_to_pfn(hv_hypercall_pg);
569 		wrmsrl(HV_X64_MSR_HYPERCALL, hypercall_msr.as_uint64);
570 	}
571 
572 skip_hypercall_pg_init:
573 	/*
574 	 * Some versions of Hyper-V that provide IBT in guest VMs have a bug
575 	 * in that there's no ENDBR64 instruction at the entry to the
576 	 * hypercall page. Because hypercalls are invoked via an indirect call
577 	 * to the hypercall page, all hypercall attempts fail when IBT is
578 	 * enabled, and Linux panics. For such buggy versions, disable IBT.
579 	 *
580 	 * Fixed versions of Hyper-V always provide ENDBR64 on the hypercall
581 	 * page, so if future Linux kernel versions enable IBT for 32-bit
582 	 * builds, additional hypercall page hackery will be required here
583 	 * to provide an ENDBR32.
584 	 */
585 #ifdef CONFIG_X86_KERNEL_IBT
586 	if (cpu_feature_enabled(X86_FEATURE_IBT) &&
587 	    *(u32 *)hv_hypercall_pg != gen_endbr()) {
588 		setup_clear_cpu_cap(X86_FEATURE_IBT);
589 		pr_warn("Disabling IBT because of Hyper-V bug\n");
590 	}
591 #endif
592 
593 	/*
594 	 * hyperv_init() is called before LAPIC is initialized: see
595 	 * apic_intr_mode_init() -> x86_platform.apic_post_init() and
596 	 * apic_bsp_setup() -> setup_local_APIC(). The direct-mode STIMER
597 	 * depends on LAPIC, so hv_stimer_alloc() should be called from
598 	 * x86_init.timers.setup_percpu_clockev.
599 	 */
600 	old_setup_percpu_clockev = x86_init.timers.setup_percpu_clockev;
601 	x86_init.timers.setup_percpu_clockev = hv_stimer_setup_percpu_clockev;
602 
603 	hv_apic_init();
604 
605 	x86_init.pci.arch_init = hv_pci_init;
606 
607 	register_syscore_ops(&hv_syscore_ops);
608 
609 	if (cpuid_ebx(HYPERV_CPUID_FEATURES) & HV_ACCESS_PARTITION_ID)
610 		hv_get_partition_id();
611 
612 	BUG_ON(hv_root_partition && hv_current_partition_id == ~0ull);
613 
614 #ifdef CONFIG_PCI_MSI
615 	/*
616 	 * If we're running as root, we want to create our own PCI MSI domain.
617 	 * We can't set this in hv_pci_init because that would be too late.
618 	 */
619 	if (hv_root_partition)
620 		x86_init.irqs.create_pci_msi_domain = hv_create_pci_msi_domain;
621 #endif
622 
623 	/* Query the VMs extended capability once, so that it can be cached. */
624 	hv_query_ext_cap(0);
625 
626 	/* Find the VTL */
627 	ms_hyperv.vtl = get_vtl();
628 
629 	if (ms_hyperv.vtl > 0) /* non default VTL */
630 		hv_vtl_early_init();
631 
632 	return;
633 
634 clean_guest_os_id:
635 	wrmsrl(HV_X64_MSR_GUEST_OS_ID, 0);
636 	hv_ivm_msr_write(HV_X64_MSR_GUEST_OS_ID, 0);
637 	cpuhp_remove_state(CPUHP_AP_HYPERV_ONLINE);
638 free_ghcb_page:
639 	free_percpu(hv_ghcb_pg);
640 free_vp_assist_page:
641 	kfree(hv_vp_assist_page);
642 	hv_vp_assist_page = NULL;
643 common_free:
644 	hv_common_free();
645 }
646 
647 /*
648  * This routine is called before kexec/kdump, it does the required cleanup.
649  */
hyperv_cleanup(void)650 void hyperv_cleanup(void)
651 {
652 	union hv_x64_msr_hypercall_contents hypercall_msr;
653 	union hv_reference_tsc_msr tsc_msr;
654 
655 	/* Reset our OS id */
656 	wrmsrl(HV_X64_MSR_GUEST_OS_ID, 0);
657 	hv_ivm_msr_write(HV_X64_MSR_GUEST_OS_ID, 0);
658 
659 	/*
660 	 * Reset hypercall page reference before reset the page,
661 	 * let hypercall operations fail safely rather than
662 	 * panic the kernel for using invalid hypercall page
663 	 */
664 	hv_hypercall_pg = NULL;
665 
666 	/* Reset the hypercall page */
667 	hypercall_msr.as_uint64 = hv_get_msr(HV_X64_MSR_HYPERCALL);
668 	hypercall_msr.enable = 0;
669 	hv_set_msr(HV_X64_MSR_HYPERCALL, hypercall_msr.as_uint64);
670 
671 	/* Reset the TSC page */
672 	tsc_msr.as_uint64 = hv_get_msr(HV_X64_MSR_REFERENCE_TSC);
673 	tsc_msr.enable = 0;
674 	hv_set_msr(HV_X64_MSR_REFERENCE_TSC, tsc_msr.as_uint64);
675 }
676 
hyperv_report_panic(struct pt_regs * regs,long err,bool in_die)677 void hyperv_report_panic(struct pt_regs *regs, long err, bool in_die)
678 {
679 	static bool panic_reported;
680 	u64 guest_id;
681 
682 	if (in_die && !panic_on_oops)
683 		return;
684 
685 	/*
686 	 * We prefer to report panic on 'die' chain as we have proper
687 	 * registers to report, but if we miss it (e.g. on BUG()) we need
688 	 * to report it on 'panic'.
689 	 */
690 	if (panic_reported)
691 		return;
692 	panic_reported = true;
693 
694 	rdmsrl(HV_X64_MSR_GUEST_OS_ID, guest_id);
695 
696 	wrmsrl(HV_X64_MSR_CRASH_P0, err);
697 	wrmsrl(HV_X64_MSR_CRASH_P1, guest_id);
698 	wrmsrl(HV_X64_MSR_CRASH_P2, regs->ip);
699 	wrmsrl(HV_X64_MSR_CRASH_P3, regs->ax);
700 	wrmsrl(HV_X64_MSR_CRASH_P4, regs->sp);
701 
702 	/*
703 	 * Let Hyper-V know there is crash data available
704 	 */
705 	wrmsrl(HV_X64_MSR_CRASH_CTL, HV_CRASH_CTL_CRASH_NOTIFY);
706 }
707 EXPORT_SYMBOL_GPL(hyperv_report_panic);
708 
hv_is_hyperv_initialized(void)709 bool hv_is_hyperv_initialized(void)
710 {
711 	union hv_x64_msr_hypercall_contents hypercall_msr;
712 
713 	/*
714 	 * Ensure that we're really on Hyper-V, and not a KVM or Xen
715 	 * emulation of Hyper-V
716 	 */
717 	if (x86_hyper_type != X86_HYPER_MS_HYPERV)
718 		return false;
719 
720 	/* A TDX VM with no paravisor uses TDX GHCI call rather than hv_hypercall_pg */
721 	if (hv_isolation_type_tdx() && !ms_hyperv.paravisor_present)
722 		return true;
723 	/*
724 	 * Verify that earlier initialization succeeded by checking
725 	 * that the hypercall page is setup
726 	 */
727 	hypercall_msr.as_uint64 = 0;
728 	rdmsrl(HV_X64_MSR_HYPERCALL, hypercall_msr.as_uint64);
729 
730 	return hypercall_msr.enable;
731 }
732 EXPORT_SYMBOL_GPL(hv_is_hyperv_initialized);
733