197 	 * The kernel can not handle #VE's when accessing normal kernel  in tdx_parse_tdinfo()
198 	 * memory.  Ensure that no #VE will be delivered for accesses to  in tdx_parse_tdinfo()
199 	 * TD-private memory.  Only VMM-shared memory (MMIO) will #VE.  in tdx_parse_tdinfo()
214  * The TDX module spec states that #VE may be injected for a limited set of
217  *  - Emulation of the architectural #VE injection on EPT violation;
231  * information if #VE occurred due to instruction execution, but not for EPT
234 static int ve_instr_len(struct ve_info *ve)  in ve_instr_len()  argument
236 	switch (ve->exit_reason) {  in ve_instr_len()
242 		/* It is safe to use ve->instr_len for #VE due instructions */  in ve_instr_len()
243 		return ve->instr_len;  in ve_instr_len()
246 		 * For EPT violations, ve->insn_len is not defined. For those,  in ve_instr_len()
250 		WARN_ONCE(1, "ve->instr_len is not defined for EPT violations");  in ve_instr_len()
253 		WARN_ONCE(1, "Unexpected #VE-type: %lld\n", ve->exit_reason);  in ve_instr_len()
254 		return ve->instr_len;  in ve_instr_len()
281 static int handle_halt(struct ve_info *ve)  in handle_halt()  argument
288 	return ve_instr_len(ve);  in handle_halt()
302 static int read_msr(struct pt_regs *regs, struct ve_info *ve)  in read_msr()  argument
320 	return ve_instr_len(ve);  in read_msr()
323 static int write_msr(struct pt_regs *regs, struct ve_info *ve)  in write_msr()  argument
340 	return ve_instr_len(ve);  in write_msr()
343 static int handle_cpuid(struct pt_regs *regs, struct ve_info *ve)  in handle_cpuid()  argument
361 		return ve_instr_len(ve);  in handle_cpuid()
382 	return ve_instr_len(ve);  in handle_cpuid()
408 static int handle_mmio(struct pt_regs *regs, struct ve_info *ve)  in handle_mmio()  argument
437 	if (!fault_in_kernel_space(ve->gla)) {  in handle_mmio()
459 		if (!mmio_write(size, ve->gpa, val))  in handle_mmio()
464 		if (!mmio_write(size, ve->gpa, val))  in handle_mmio()
486 	if (!mmio_read(size, ve->gpa, &val))  in handle_mmio()
566 static int handle_io(struct pt_regs *regs, struct ve_info *ve)  in handle_io()  argument
568 	u32 exit_qual = ve->exit_qual;  in handle_io()
587 	return ve_instr_len(ve);  in handle_io()
591  * Early #VE exception handler. Only handles a subset of port I/O.
596 	struct ve_info ve;  in tdx_early_handle_ve()  local
599 	tdx_get_ve_info(&ve);  in tdx_early_handle_ve()
601 	if (ve.exit_reason != EXIT_REASON_IO_INSTRUCTION)  in tdx_early_handle_ve()
604 	insn_len = handle_io(regs, &ve);  in tdx_early_handle_ve()
612 void tdx_get_ve_info(struct ve_info *ve)  in tdx_get_ve_info()  argument
617 	 * Called during #VE handling to retrieve the #VE info from the  in tdx_get_ve_info()
620 	 * This has to be called early in #VE handling.  A "nested" #VE which  in tdx_get_ve_info()
623 	 * The call retrieves the #VE info from the TDX module, which also  in tdx_get_ve_info()
624 	 * clears the "#VE valid" flag. This must be done before anything else  in tdx_get_ve_info()
625 	 * because any #VE that occurs while the valid flag is set will lead to  in tdx_get_ve_info()
628 	 * Note, the TDX module treats virtual NMIs as inhibited if the #VE  in tdx_get_ve_info()
629 	 * valid flag is set. It means that NMI=>#VE will not result in a #DF.  in tdx_get_ve_info()
634 	ve->exit_reason = args.rcx;  in tdx_get_ve_info()
635 	ve->exit_qual   = args.rdx;  in tdx_get_ve_info()
636 	ve->gla         = args.r8;  in tdx_get_ve_info()
637 	ve->gpa         = args.r9;  in tdx_get_ve_info()
638 	ve->instr_len   = lower_32_bits(args.r10);  in tdx_get_ve_info()
639 	ve->instr_info  = upper_32_bits(args.r10);  in tdx_get_ve_info()
643  * Handle the user initiated #VE.
648 static int virt_exception_user(struct pt_regs *regs, struct ve_info *ve)  in virt_exception_user()  argument
650 	switch (ve->exit_reason) {  in virt_exception_user()
652 		return handle_cpuid(regs, ve);  in virt_exception_user()
654 		pr_warn("Unexpected #VE: %lld\n", ve->exit_reason);  in virt_exception_user()
665  * Handle the kernel #VE.
670 static int virt_exception_kernel(struct pt_regs *regs, struct ve_info *ve)  in virt_exception_kernel()  argument
672 	switch (ve->exit_reason) {  in virt_exception_kernel()
674 		return handle_halt(ve);  in virt_exception_kernel()
676 		return read_msr(regs, ve);  in virt_exception_kernel()
678 		return write_msr(regs, ve);  in virt_exception_kernel()
680 		return handle_cpuid(regs, ve);  in virt_exception_kernel()
682 		if (is_private_gpa(ve->gpa))  in virt_exception_kernel()
684 		return handle_mmio(regs, ve);  in virt_exception_kernel()
686 		return handle_io(regs, ve);  in virt_exception_kernel()
688 		pr_warn("Unexpected #VE: %lld\n", ve->exit_reason);  in virt_exception_kernel()
693 bool tdx_handle_virt_exception(struct pt_regs *regs, struct ve_info *ve)  in tdx_handle_virt_exception()  argument
698 		insn_len = virt_exception_user(regs, ve);  in tdx_handle_virt_exception()
700 		insn_len = virt_exception_kernel(regs, ve);  in tdx_handle_virt_exception()
704 	/* After successful #VE handling, move the IP */  in tdx_handle_virt_exception()
971          *   - Shared mapping  => Private Page == Recoverable #VE  in tdx_early_init()
980 	 * which can result in a #VE.  But, there is never a private mapping to  in tdx_early_init()
994 	 * bringup low level code. That raises #VE which cannot be handled  in tdx_early_init()