Lines Matching full:vcpu
109 ((struct kvm_vcpu *)(ctxt)->vcpu)
131 static void update_cr8_intercept(struct kvm_vcpu *vcpu);
132 static void process_nmi(struct kvm_vcpu *vcpu);
133 static void __kvm_set_rflags(struct kvm_vcpu *vcpu, unsigned long rflags);
134 static void store_regs(struct kvm_vcpu *vcpu);
135 static int sync_regs(struct kvm_vcpu *vcpu);
136 static int kvm_vcpu_do_singlestep(struct kvm_vcpu *vcpu);
138 static int __set_sregs2(struct kvm_vcpu *vcpu, struct kvm_sregs2 *sregs2);
139 static void __get_sregs2(struct kvm_vcpu *vcpu, struct kvm_sregs2 *sregs2);
260 STATS_DESC_COUNTER(VCPU, pf_taken),
261 STATS_DESC_COUNTER(VCPU, pf_fixed),
262 STATS_DESC_COUNTER(VCPU, pf_emulate),
263 STATS_DESC_COUNTER(VCPU, pf_spurious),
264 STATS_DESC_COUNTER(VCPU, pf_fast),
265 STATS_DESC_COUNTER(VCPU, pf_mmio_spte_created),
266 STATS_DESC_COUNTER(VCPU, pf_guest),
267 STATS_DESC_COUNTER(VCPU, tlb_flush),
268 STATS_DESC_COUNTER(VCPU, invlpg),
269 STATS_DESC_COUNTER(VCPU, exits),
270 STATS_DESC_COUNTER(VCPU, io_exits),
271 STATS_DESC_COUNTER(VCPU, mmio_exits),
272 STATS_DESC_COUNTER(VCPU, signal_exits),
273 STATS_DESC_COUNTER(VCPU, irq_window_exits),
274 STATS_DESC_COUNTER(VCPU, nmi_window_exits),
275 STATS_DESC_COUNTER(VCPU, l1d_flush),
276 STATS_DESC_COUNTER(VCPU, halt_exits),
277 STATS_DESC_COUNTER(VCPU, request_irq_exits),
278 STATS_DESC_COUNTER(VCPU, irq_exits),
279 STATS_DESC_COUNTER(VCPU, host_state_reload),
280 STATS_DESC_COUNTER(VCPU, fpu_reload),
281 STATS_DESC_COUNTER(VCPU, insn_emulation),
282 STATS_DESC_COUNTER(VCPU, insn_emulation_fail),
283 STATS_DESC_COUNTER(VCPU, hypercalls),
284 STATS_DESC_COUNTER(VCPU, irq_injections),
285 STATS_DESC_COUNTER(VCPU, nmi_injections),
286 STATS_DESC_COUNTER(VCPU, req_event),
287 STATS_DESC_COUNTER(VCPU, nested_run),
288 STATS_DESC_COUNTER(VCPU, directed_yield_attempted),
289 STATS_DESC_COUNTER(VCPU, directed_yield_successful),
290 STATS_DESC_COUNTER(VCPU, preemption_reported),
291 STATS_DESC_COUNTER(VCPU, preemption_other),
292 STATS_DESC_IBOOLEAN(VCPU, guest_mode),
293 STATS_DESC_COUNTER(VCPU, notify_window_exits),
462 * patch, are immutable once the vCPU model is defined.
496 typedef int (*msr_access_t)(struct kvm_vcpu *vcpu, u32 index, u64 *data,
499 static __always_inline int kvm_do_msr_access(struct kvm_vcpu *vcpu, u32 msr, in kvm_do_msr_access() argument
513 ret = msr_access_fn(vcpu, msr, data, host_initiated); in kvm_do_msr_access()
554 static inline void kvm_async_pf_hash_reset(struct kvm_vcpu *vcpu) in kvm_async_pf_hash_reset() argument
558 vcpu->arch.apf.gfns[i] = ~0; in kvm_async_pf_hash_reset()
670 u64 kvm_get_apic_base(struct kvm_vcpu *vcpu) in kvm_get_apic_base() argument
672 return vcpu->arch.apic_base; in kvm_get_apic_base()
675 enum lapic_mode kvm_get_apic_mode(struct kvm_vcpu *vcpu) in kvm_get_apic_mode() argument
677 return kvm_apic_mode(kvm_get_apic_base(vcpu)); in kvm_get_apic_mode()
681 int kvm_set_apic_base(struct kvm_vcpu *vcpu, struct msr_data *msr_info) in kvm_set_apic_base() argument
683 enum lapic_mode old_mode = kvm_get_apic_mode(vcpu); in kvm_set_apic_base()
685 u64 reserved_bits = kvm_vcpu_reserved_gpa_bits_raw(vcpu) | 0x2ff | in kvm_set_apic_base()
686 (guest_cpuid_has(vcpu, X86_FEATURE_X2APIC) ? 0 : X2APIC_ENABLE); in kvm_set_apic_base()
697 kvm_lapic_set_base(vcpu, msr_info->data); in kvm_set_apic_base()
698 kvm_recalculate_apic_map(vcpu->kvm); in kvm_set_apic_base()
769 void kvm_deliver_exception_payload(struct kvm_vcpu *vcpu, in kvm_deliver_exception_payload() argument
782 vcpu->arch.dr6 &= ~DR_TRAP_BITS; in kvm_deliver_exception_payload()
799 vcpu->arch.dr6 |= DR6_ACTIVE_LOW; in kvm_deliver_exception_payload()
800 vcpu->arch.dr6 |= ex->payload; in kvm_deliver_exception_payload()
801 vcpu->arch.dr6 ^= ex->payload & DR6_ACTIVE_LOW; in kvm_deliver_exception_payload()
809 vcpu->arch.dr6 &= ~BIT(12); in kvm_deliver_exception_payload()
812 vcpu->arch.cr2 = ex->payload; in kvm_deliver_exception_payload()
821 static void kvm_queue_exception_vmexit(struct kvm_vcpu *vcpu, unsigned int vector, in kvm_queue_exception_vmexit() argument
825 struct kvm_queued_exception *ex = &vcpu->arch.exception_vmexit; in kvm_queue_exception_vmexit()
836 static void kvm_multiple_exception(struct kvm_vcpu *vcpu, in kvm_multiple_exception() argument
843 kvm_make_request(KVM_REQ_EVENT, vcpu); in kvm_multiple_exception()
852 if (!reinject && is_guest_mode(vcpu) && in kvm_multiple_exception()
853 kvm_x86_ops.nested_ops->is_exception_vmexit(vcpu, nr, error_code)) { in kvm_multiple_exception()
854 kvm_queue_exception_vmexit(vcpu, nr, has_error, error_code, in kvm_multiple_exception()
859 if (!vcpu->arch.exception.pending && !vcpu->arch.exception.injected) { in kvm_multiple_exception()
869 WARN_ON_ONCE(kvm_is_exception_pending(vcpu)); in kvm_multiple_exception()
870 vcpu->arch.exception.injected = true; in kvm_multiple_exception()
880 vcpu->arch.exception.pending = true; in kvm_multiple_exception()
881 vcpu->arch.exception.injected = false; in kvm_multiple_exception()
883 vcpu->arch.exception.has_error_code = has_error; in kvm_multiple_exception()
884 vcpu->arch.exception.vector = nr; in kvm_multiple_exception()
885 vcpu->arch.exception.error_code = error_code; in kvm_multiple_exception()
886 vcpu->arch.exception.has_payload = has_payload; in kvm_multiple_exception()
887 vcpu->arch.exception.payload = payload; in kvm_multiple_exception()
888 if (!is_guest_mode(vcpu)) in kvm_multiple_exception()
889 kvm_deliver_exception_payload(vcpu, in kvm_multiple_exception()
890 &vcpu->arch.exception); in kvm_multiple_exception()
895 prev_nr = vcpu->arch.exception.vector; in kvm_multiple_exception()
898 kvm_make_request(KVM_REQ_TRIPLE_FAULT, vcpu); in kvm_multiple_exception()
909 vcpu->arch.exception.injected = false; in kvm_multiple_exception()
910 vcpu->arch.exception.pending = false; in kvm_multiple_exception()
912 kvm_queue_exception_e(vcpu, DF_VECTOR, 0); in kvm_multiple_exception()
921 void kvm_queue_exception(struct kvm_vcpu *vcpu, unsigned nr) in kvm_queue_exception() argument
923 kvm_multiple_exception(vcpu, nr, false, 0, false, 0, false); in kvm_queue_exception()
927 void kvm_requeue_exception(struct kvm_vcpu *vcpu, unsigned nr) in kvm_requeue_exception() argument
929 kvm_multiple_exception(vcpu, nr, false, 0, false, 0, true); in kvm_requeue_exception()
933 void kvm_queue_exception_p(struct kvm_vcpu *vcpu, unsigned nr, in kvm_queue_exception_p() argument
936 kvm_multiple_exception(vcpu, nr, false, 0, true, payload, false); in kvm_queue_exception_p()
940 static void kvm_queue_exception_e_p(struct kvm_vcpu *vcpu, unsigned nr, in kvm_queue_exception_e_p() argument
943 kvm_multiple_exception(vcpu, nr, true, error_code, in kvm_queue_exception_e_p()
947 int kvm_complete_insn_gp(struct kvm_vcpu *vcpu, int err) in kvm_complete_insn_gp() argument
950 kvm_inject_gp(vcpu, 0); in kvm_complete_insn_gp()
952 return kvm_skip_emulated_instruction(vcpu); in kvm_complete_insn_gp()
958 static int complete_emulated_insn_gp(struct kvm_vcpu *vcpu, int err) in complete_emulated_insn_gp() argument
961 kvm_inject_gp(vcpu, 0); in complete_emulated_insn_gp()
965 return kvm_emulate_instruction(vcpu, EMULTYPE_NO_DECODE | EMULTYPE_SKIP | in complete_emulated_insn_gp()
969 void kvm_inject_page_fault(struct kvm_vcpu *vcpu, struct x86_exception *fault) in kvm_inject_page_fault() argument
971 ++vcpu->stat.pf_guest; in kvm_inject_page_fault()
977 if (is_guest_mode(vcpu) && fault->async_page_fault) in kvm_inject_page_fault()
978 kvm_queue_exception_vmexit(vcpu, PF_VECTOR, in kvm_inject_page_fault()
982 kvm_queue_exception_e_p(vcpu, PF_VECTOR, fault->error_code, in kvm_inject_page_fault()
986 void kvm_inject_emulated_page_fault(struct kvm_vcpu *vcpu, in kvm_inject_emulated_page_fault() argument
992 fault_mmu = fault->nested_page_fault ? vcpu->arch.mmu : in kvm_inject_emulated_page_fault()
993 vcpu->arch.walk_mmu; in kvm_inject_emulated_page_fault()
1001 kvm_mmu_invalidate_addr(vcpu, fault_mmu, fault->address, in kvm_inject_emulated_page_fault()
1004 fault_mmu->inject_page_fault(vcpu, fault); in kvm_inject_emulated_page_fault()
1008 void kvm_inject_nmi(struct kvm_vcpu *vcpu) in kvm_inject_nmi() argument
1010 atomic_inc(&vcpu->arch.nmi_queued); in kvm_inject_nmi()
1011 kvm_make_request(KVM_REQ_NMI, vcpu); in kvm_inject_nmi()
1014 void kvm_queue_exception_e(struct kvm_vcpu *vcpu, unsigned nr, u32 error_code) in kvm_queue_exception_e() argument
1016 kvm_multiple_exception(vcpu, nr, true, error_code, false, 0, false); in kvm_queue_exception_e()
1020 void kvm_requeue_exception_e(struct kvm_vcpu *vcpu, unsigned nr, u32 error_code) in kvm_requeue_exception_e() argument
1022 kvm_multiple_exception(vcpu, nr, true, error_code, false, 0, true); in kvm_requeue_exception_e()
1030 bool kvm_require_cpl(struct kvm_vcpu *vcpu, int required_cpl) in kvm_require_cpl() argument
1032 if (kvm_x86_call(get_cpl)(vcpu) <= required_cpl) in kvm_require_cpl()
1034 kvm_queue_exception_e(vcpu, GP_VECTOR, 0); in kvm_require_cpl()
1038 bool kvm_require_dr(struct kvm_vcpu *vcpu, int dr) in kvm_require_dr() argument
1040 if ((dr != 4 && dr != 5) || !kvm_is_cr4_bit_set(vcpu, X86_CR4_DE)) in kvm_require_dr()
1043 kvm_queue_exception(vcpu, UD_VECTOR); in kvm_require_dr()
1048 static inline u64 pdptr_rsvd_bits(struct kvm_vcpu *vcpu) in pdptr_rsvd_bits() argument
1050 return vcpu->arch.reserved_gpa_bits | rsvd_bits(5, 8) | rsvd_bits(1, 2); in pdptr_rsvd_bits()
1056 int load_pdptrs(struct kvm_vcpu *vcpu, unsigned long cr3) in load_pdptrs() argument
1058 struct kvm_mmu *mmu = vcpu->arch.walk_mmu; in load_pdptrs()
1069 real_gpa = kvm_translate_gpa(vcpu, mmu, gfn_to_gpa(pdpt_gfn), in load_pdptrs()
1075 ret = kvm_vcpu_read_guest_page(vcpu, gpa_to_gfn(real_gpa), pdpte, in load_pdptrs()
1082 (pdpte[i] & pdptr_rsvd_bits(vcpu))) { in load_pdptrs()
1092 kvm_mmu_free_roots(vcpu->kvm, mmu, KVM_MMU_ROOT_CURRENT); in load_pdptrs()
1095 kvm_register_mark_dirty(vcpu, VCPU_EXREG_PDPTR); in load_pdptrs()
1096 kvm_make_request(KVM_REQ_LOAD_MMU_PGD, vcpu); in load_pdptrs()
1097 vcpu->arch.pdptrs_from_userspace = false; in load_pdptrs()
1103 static bool kvm_is_valid_cr0(struct kvm_vcpu *vcpu, unsigned long cr0) in kvm_is_valid_cr0() argument
1116 return kvm_x86_call(is_valid_cr0)(vcpu, cr0); in kvm_is_valid_cr0()
1119 void kvm_post_set_cr0(struct kvm_vcpu *vcpu, unsigned long old_cr0, unsigned long cr0) in kvm_post_set_cr0() argument
1134 kvm_init_mmu(vcpu); in kvm_post_set_cr0()
1140 kvm_clear_async_pf_completion_queue(vcpu); in kvm_post_set_cr0()
1141 kvm_async_pf_hash_reset(vcpu); in kvm_post_set_cr0()
1148 kvm_make_request(KVM_REQ_TLB_FLUSH_GUEST, vcpu); in kvm_post_set_cr0()
1152 kvm_mmu_reset_context(vcpu); in kvm_post_set_cr0()
1156 int kvm_set_cr0(struct kvm_vcpu *vcpu, unsigned long cr0) in kvm_set_cr0() argument
1158 unsigned long old_cr0 = kvm_read_cr0(vcpu); in kvm_set_cr0()
1160 if (!kvm_is_valid_cr0(vcpu, cr0)) in kvm_set_cr0()
1169 if ((vcpu->arch.efer & EFER_LME) && !is_paging(vcpu) && in kvm_set_cr0()
1173 if (!is_pae(vcpu)) in kvm_set_cr0()
1175 kvm_x86_call(get_cs_db_l_bits)(vcpu, &cs_db, &cs_l); in kvm_set_cr0()
1180 if (!(vcpu->arch.efer & EFER_LME) && (cr0 & X86_CR0_PG) && in kvm_set_cr0()
1181 is_pae(vcpu) && ((cr0 ^ old_cr0) & X86_CR0_PDPTR_BITS) && in kvm_set_cr0()
1182 !load_pdptrs(vcpu, kvm_read_cr3(vcpu))) in kvm_set_cr0()
1186 (is_64_bit_mode(vcpu) || kvm_is_cr4_bit_set(vcpu, X86_CR4_PCIDE))) in kvm_set_cr0()
1189 kvm_x86_call(set_cr0)(vcpu, cr0); in kvm_set_cr0()
1191 kvm_post_set_cr0(vcpu, old_cr0, cr0); in kvm_set_cr0()
1197 void kvm_lmsw(struct kvm_vcpu *vcpu, unsigned long msw) in kvm_lmsw() argument
1199 (void)kvm_set_cr0(vcpu, kvm_read_cr0_bits(vcpu, ~0x0eul) | (msw & 0x0f)); in kvm_lmsw()
1203 void kvm_load_guest_xsave_state(struct kvm_vcpu *vcpu) in kvm_load_guest_xsave_state() argument
1205 if (vcpu->arch.guest_state_protected) in kvm_load_guest_xsave_state()
1208 if (kvm_is_cr4_bit_set(vcpu, X86_CR4_OSXSAVE)) { in kvm_load_guest_xsave_state()
1210 if (vcpu->arch.xcr0 != kvm_host.xcr0) in kvm_load_guest_xsave_state()
1211 xsetbv(XCR_XFEATURE_ENABLED_MASK, vcpu->arch.xcr0); in kvm_load_guest_xsave_state()
1213 if (guest_can_use(vcpu, X86_FEATURE_XSAVES) && in kvm_load_guest_xsave_state()
1214 vcpu->arch.ia32_xss != kvm_host.xss) in kvm_load_guest_xsave_state()
1215 wrmsrl(MSR_IA32_XSS, vcpu->arch.ia32_xss); in kvm_load_guest_xsave_state()
1219 vcpu->arch.pkru != vcpu->arch.host_pkru && in kvm_load_guest_xsave_state()
1220 ((vcpu->arch.xcr0 & XFEATURE_MASK_PKRU) || in kvm_load_guest_xsave_state()
1221 kvm_is_cr4_bit_set(vcpu, X86_CR4_PKE))) in kvm_load_guest_xsave_state()
1222 write_pkru(vcpu->arch.pkru); in kvm_load_guest_xsave_state()
1226 void kvm_load_host_xsave_state(struct kvm_vcpu *vcpu) in kvm_load_host_xsave_state() argument
1228 if (vcpu->arch.guest_state_protected) in kvm_load_host_xsave_state()
1232 ((vcpu->arch.xcr0 & XFEATURE_MASK_PKRU) || in kvm_load_host_xsave_state()
1233 kvm_is_cr4_bit_set(vcpu, X86_CR4_PKE))) { in kvm_load_host_xsave_state()
1234 vcpu->arch.pkru = rdpkru(); in kvm_load_host_xsave_state()
1235 if (vcpu->arch.pkru != vcpu->arch.host_pkru) in kvm_load_host_xsave_state()
1236 write_pkru(vcpu->arch.host_pkru); in kvm_load_host_xsave_state()
1239 if (kvm_is_cr4_bit_set(vcpu, X86_CR4_OSXSAVE)) { in kvm_load_host_xsave_state()
1241 if (vcpu->arch.xcr0 != kvm_host.xcr0) in kvm_load_host_xsave_state()
1244 if (guest_can_use(vcpu, X86_FEATURE_XSAVES) && in kvm_load_host_xsave_state()
1245 vcpu->arch.ia32_xss != kvm_host.xss) in kvm_load_host_xsave_state()
1253 static inline u64 kvm_guest_supported_xfd(struct kvm_vcpu *vcpu) in kvm_guest_supported_xfd() argument
1255 return vcpu->arch.guest_supported_xcr0 & XFEATURE_MASK_USER_DYNAMIC; in kvm_guest_supported_xfd()
1259 static int __kvm_set_xcr(struct kvm_vcpu *vcpu, u32 index, u64 xcr) in __kvm_set_xcr() argument
1262 u64 old_xcr0 = vcpu->arch.xcr0; in __kvm_set_xcr()
1278 valid_bits = vcpu->arch.guest_supported_xcr0 | XFEATURE_MASK_FP; in __kvm_set_xcr()
1297 vcpu->arch.xcr0 = xcr0; in __kvm_set_xcr()
1300 kvm_update_cpuid_runtime(vcpu); in __kvm_set_xcr()
1304 int kvm_emulate_xsetbv(struct kvm_vcpu *vcpu) in kvm_emulate_xsetbv() argument
1307 if (kvm_x86_call(get_cpl)(vcpu) != 0 || in kvm_emulate_xsetbv()
1308 __kvm_set_xcr(vcpu, kvm_rcx_read(vcpu), kvm_read_edx_eax(vcpu))) { in kvm_emulate_xsetbv()
1309 kvm_inject_gp(vcpu, 0); in kvm_emulate_xsetbv()
1313 return kvm_skip_emulated_instruction(vcpu); in kvm_emulate_xsetbv()
1317 bool __kvm_is_valid_cr4(struct kvm_vcpu *vcpu, unsigned long cr4) in __kvm_is_valid_cr4() argument
1322 if (cr4 & vcpu->arch.cr4_guest_rsvd_bits) in __kvm_is_valid_cr4()
1329 static bool kvm_is_valid_cr4(struct kvm_vcpu *vcpu, unsigned long cr4) in kvm_is_valid_cr4() argument
1331 return __kvm_is_valid_cr4(vcpu, cr4) && in kvm_is_valid_cr4()
1332 kvm_x86_call(is_valid_cr4)(vcpu, cr4); in kvm_is_valid_cr4()
1335 void kvm_post_set_cr4(struct kvm_vcpu *vcpu, unsigned long old_cr4, unsigned long cr4) in kvm_post_set_cr4() argument
1338 kvm_mmu_reset_context(vcpu); in kvm_post_set_cr4()
1350 kvm_mmu_unload(vcpu); in kvm_post_set_cr4()
1362 kvm_make_request(KVM_REQ_TLB_FLUSH_GUEST, vcpu); in kvm_post_set_cr4()
1372 kvm_make_request(KVM_REQ_TLB_FLUSH_CURRENT, vcpu); in kvm_post_set_cr4()
1377 int kvm_set_cr4(struct kvm_vcpu *vcpu, unsigned long cr4) in kvm_set_cr4() argument
1379 unsigned long old_cr4 = kvm_read_cr4(vcpu); in kvm_set_cr4()
1381 if (!kvm_is_valid_cr4(vcpu, cr4)) in kvm_set_cr4()
1384 if (is_long_mode(vcpu)) { in kvm_set_cr4()
1389 } else if (is_paging(vcpu) && (cr4 & X86_CR4_PAE) in kvm_set_cr4()
1391 && !load_pdptrs(vcpu, kvm_read_cr3(vcpu))) in kvm_set_cr4()
1396 if ((kvm_read_cr3(vcpu) & X86_CR3_PCID_MASK) || !is_long_mode(vcpu)) in kvm_set_cr4()
1400 kvm_x86_call(set_cr4)(vcpu, cr4); in kvm_set_cr4()
1402 kvm_post_set_cr4(vcpu, old_cr4, cr4); in kvm_set_cr4()
1408 static void kvm_invalidate_pcid(struct kvm_vcpu *vcpu, unsigned long pcid) in kvm_invalidate_pcid() argument
1410 struct kvm_mmu *mmu = vcpu->arch.mmu; in kvm_invalidate_pcid()
1422 kvm_make_request(KVM_REQ_TLB_FLUSH_GUEST, vcpu); in kvm_invalidate_pcid()
1431 if (kvm_get_active_pcid(vcpu) == pcid) { in kvm_invalidate_pcid()
1432 kvm_make_request(KVM_REQ_MMU_SYNC, vcpu); in kvm_invalidate_pcid()
1433 kvm_make_request(KVM_REQ_TLB_FLUSH_CURRENT, vcpu); in kvm_invalidate_pcid()
1441 if (!kvm_is_cr4_bit_set(vcpu, X86_CR4_PCIDE)) in kvm_invalidate_pcid()
1445 if (kvm_get_pcid(vcpu, mmu->prev_roots[i].pgd) == pcid) in kvm_invalidate_pcid()
1448 kvm_mmu_free_roots(vcpu->kvm, mmu, roots_to_free); in kvm_invalidate_pcid()
1451 int kvm_set_cr3(struct kvm_vcpu *vcpu, unsigned long cr3) in kvm_set_cr3() argument
1456 if (kvm_is_cr4_bit_set(vcpu, X86_CR4_PCIDE)) { in kvm_set_cr3()
1464 if (cr3 == kvm_read_cr3(vcpu) && !is_pae_paging(vcpu)) in kvm_set_cr3()
1470 * the current vCPU mode is accurate. in kvm_set_cr3()
1472 if (!kvm_vcpu_is_legal_cr3(vcpu, cr3)) in kvm_set_cr3()
1475 if (is_pae_paging(vcpu) && !load_pdptrs(vcpu, cr3)) in kvm_set_cr3()
1478 if (cr3 != kvm_read_cr3(vcpu)) in kvm_set_cr3()
1479 kvm_mmu_new_pgd(vcpu, cr3); in kvm_set_cr3()
1481 vcpu->arch.cr3 = cr3; in kvm_set_cr3()
1482 kvm_register_mark_dirty(vcpu, VCPU_EXREG_CR3); in kvm_set_cr3()
1494 kvm_invalidate_pcid(vcpu, pcid); in kvm_set_cr3()
1500 int kvm_set_cr8(struct kvm_vcpu *vcpu, unsigned long cr8) in kvm_set_cr8() argument
1504 if (lapic_in_kernel(vcpu)) in kvm_set_cr8()
1505 kvm_lapic_set_tpr(vcpu, cr8); in kvm_set_cr8()
1507 vcpu->arch.cr8 = cr8; in kvm_set_cr8()
1512 unsigned long kvm_get_cr8(struct kvm_vcpu *vcpu) in kvm_get_cr8() argument
1514 if (lapic_in_kernel(vcpu)) in kvm_get_cr8()
1515 return kvm_lapic_get_cr8(vcpu); in kvm_get_cr8()
1517 return vcpu->arch.cr8; in kvm_get_cr8()
1521 static void kvm_update_dr0123(struct kvm_vcpu *vcpu) in kvm_update_dr0123() argument
1525 if (!(vcpu->guest_debug & KVM_GUESTDBG_USE_HW_BP)) { in kvm_update_dr0123()
1527 vcpu->arch.eff_db[i] = vcpu->arch.db[i]; in kvm_update_dr0123()
1531 void kvm_update_dr7(struct kvm_vcpu *vcpu) in kvm_update_dr7() argument
1535 if (vcpu->guest_debug & KVM_GUESTDBG_USE_HW_BP) in kvm_update_dr7()
1536 dr7 = vcpu->arch.guest_debug_dr7; in kvm_update_dr7()
1538 dr7 = vcpu->arch.dr7; in kvm_update_dr7()
1539 kvm_x86_call(set_dr7)(vcpu, dr7); in kvm_update_dr7()
1540 vcpu->arch.switch_db_regs &= ~KVM_DEBUGREG_BP_ENABLED; in kvm_update_dr7()
1542 vcpu->arch.switch_db_regs |= KVM_DEBUGREG_BP_ENABLED; in kvm_update_dr7()
1546 static u64 kvm_dr6_fixed(struct kvm_vcpu *vcpu) in kvm_dr6_fixed() argument
1550 if (!guest_cpuid_has(vcpu, X86_FEATURE_RTM)) in kvm_dr6_fixed()
1553 if (!guest_cpuid_has(vcpu, X86_FEATURE_BUS_LOCK_DETECT)) in kvm_dr6_fixed()
1558 int kvm_set_dr(struct kvm_vcpu *vcpu, int dr, unsigned long val) in kvm_set_dr() argument
1560 size_t size = ARRAY_SIZE(vcpu->arch.db); in kvm_set_dr()
1564 vcpu->arch.db[array_index_nospec(dr, size)] = val; in kvm_set_dr()
1565 if (!(vcpu->guest_debug & KVM_GUESTDBG_USE_HW_BP)) in kvm_set_dr()
1566 vcpu->arch.eff_db[dr] = val; in kvm_set_dr()
1572 vcpu->arch.dr6 = (val & DR6_VOLATILE) | kvm_dr6_fixed(vcpu); in kvm_set_dr()
1578 vcpu->arch.dr7 = (val & DR7_VOLATILE) | DR7_FIXED_1; in kvm_set_dr()
1579 kvm_update_dr7(vcpu); in kvm_set_dr()
1587 unsigned long kvm_get_dr(struct kvm_vcpu *vcpu, int dr) in kvm_get_dr() argument
1589 size_t size = ARRAY_SIZE(vcpu->arch.db); in kvm_get_dr()
1593 return vcpu->arch.db[array_index_nospec(dr, size)]; in kvm_get_dr()
1596 return vcpu->arch.dr6; in kvm_get_dr()
1599 return vcpu->arch.dr7; in kvm_get_dr()
1604 int kvm_emulate_rdpmc(struct kvm_vcpu *vcpu) in kvm_emulate_rdpmc() argument
1606 u32 ecx = kvm_rcx_read(vcpu); in kvm_emulate_rdpmc()
1609 if (kvm_pmu_rdpmc(vcpu, ecx, &data)) { in kvm_emulate_rdpmc()
1610 kvm_inject_gp(vcpu, 0); in kvm_emulate_rdpmc()
1614 kvm_rax_write(vcpu, (u32)data); in kvm_emulate_rdpmc()
1615 kvm_rdx_write(vcpu, data >> 32); in kvm_emulate_rdpmc()
1616 return kvm_skip_emulated_instruction(vcpu); in kvm_emulate_rdpmc()
1697 static int kvm_get_feature_msr(struct kvm_vcpu *vcpu, u32 index, u64 *data, in kvm_get_feature_msr() argument
1718 static int do_get_feature_msr(struct kvm_vcpu *vcpu, unsigned index, u64 *data) in do_get_feature_msr() argument
1720 return kvm_do_msr_access(vcpu, index, data, true, MSR_TYPE_R, in do_get_feature_msr()
1724 static bool __kvm_valid_efer(struct kvm_vcpu *vcpu, u64 efer) in __kvm_valid_efer() argument
1726 if (efer & EFER_AUTOIBRS && !guest_cpuid_has(vcpu, X86_FEATURE_AUTOIBRS)) in __kvm_valid_efer()
1729 if (efer & EFER_FFXSR && !guest_cpuid_has(vcpu, X86_FEATURE_FXSR_OPT)) in __kvm_valid_efer()
1732 if (efer & EFER_SVME && !guest_cpuid_has(vcpu, X86_FEATURE_SVM)) in __kvm_valid_efer()
1736 !guest_cpuid_has(vcpu, X86_FEATURE_LM)) in __kvm_valid_efer()
1739 if (efer & EFER_NX && !guest_cpuid_has(vcpu, X86_FEATURE_NX)) in __kvm_valid_efer()
1745 bool kvm_valid_efer(struct kvm_vcpu *vcpu, u64 efer) in kvm_valid_efer() argument
1750 return __kvm_valid_efer(vcpu, efer); in kvm_valid_efer()
1754 static int set_efer(struct kvm_vcpu *vcpu, struct msr_data *msr_info) in set_efer() argument
1756 u64 old_efer = vcpu->arch.efer; in set_efer()
1764 if (!__kvm_valid_efer(vcpu, efer)) in set_efer()
1767 if (is_paging(vcpu) && in set_efer()
1768 (vcpu->arch.efer & EFER_LME) != (efer & EFER_LME)) in set_efer()
1773 efer |= vcpu->arch.efer & EFER_LMA; in set_efer()
1775 r = kvm_x86_call(set_efer)(vcpu, efer); in set_efer()
1782 kvm_mmu_reset_context(vcpu); in set_efer()
1786 kvm_hv_xsaves_xsavec_maybe_warn(vcpu); in set_efer()
1797 bool kvm_msr_allowed(struct kvm_vcpu *vcpu, u32 index, u32 type) in kvm_msr_allowed() argument
1801 struct kvm *kvm = vcpu->kvm; in kvm_msr_allowed()
1846 static int __kvm_set_msr(struct kvm_vcpu *vcpu, u32 index, u64 data, in __kvm_set_msr() argument
1857 if (is_noncanonical_address(data, vcpu)) in __kvm_set_msr()
1874 data = __canonical_address(data, vcpu_virt_addr_bits(vcpu)); in __kvm_set_msr()
1881 !guest_cpuid_has(vcpu, X86_FEATURE_RDTSCP) && in __kvm_set_msr()
1882 !guest_cpuid_has(vcpu, X86_FEATURE_RDPID)) in __kvm_set_msr()
1894 if (guest_cpuid_is_intel_compatible(vcpu) && (data >> 32) != 0) in __kvm_set_msr()
1905 return kvm_x86_call(set_msr)(vcpu, &msr); in __kvm_set_msr()
1908 static int _kvm_set_msr(struct kvm_vcpu *vcpu, u32 index, u64 *data, in _kvm_set_msr() argument
1911 return __kvm_set_msr(vcpu, index, *data, host_initiated); in _kvm_set_msr()
1914 static int kvm_set_msr_ignored_check(struct kvm_vcpu *vcpu, in kvm_set_msr_ignored_check() argument
1917 return kvm_do_msr_access(vcpu, index, &data, host_initiated, MSR_TYPE_W, in kvm_set_msr_ignored_check()
1927 int __kvm_get_msr(struct kvm_vcpu *vcpu, u32 index, u64 *data, in __kvm_get_msr() argument
1939 !guest_cpuid_has(vcpu, X86_FEATURE_RDTSCP) && in __kvm_get_msr()
1940 !guest_cpuid_has(vcpu, X86_FEATURE_RDPID)) in __kvm_get_msr()
1948 ret = kvm_x86_call(get_msr)(vcpu, &msr); in __kvm_get_msr()
1954 static int kvm_get_msr_ignored_check(struct kvm_vcpu *vcpu, in kvm_get_msr_ignored_check() argument
1957 return kvm_do_msr_access(vcpu, index, data, host_initiated, MSR_TYPE_R, in kvm_get_msr_ignored_check()
1961 int kvm_get_msr_with_filter(struct kvm_vcpu *vcpu, u32 index, u64 *data) in kvm_get_msr_with_filter() argument
1963 if (!kvm_msr_allowed(vcpu, index, KVM_MSR_FILTER_READ)) in kvm_get_msr_with_filter()
1965 return kvm_get_msr_ignored_check(vcpu, index, data, false); in kvm_get_msr_with_filter()
1969 int kvm_set_msr_with_filter(struct kvm_vcpu *vcpu, u32 index, u64 data) in kvm_set_msr_with_filter() argument
1971 if (!kvm_msr_allowed(vcpu, index, KVM_MSR_FILTER_WRITE)) in kvm_set_msr_with_filter()
1973 return kvm_set_msr_ignored_check(vcpu, index, data, false); in kvm_set_msr_with_filter()
1977 int kvm_get_msr(struct kvm_vcpu *vcpu, u32 index, u64 *data) in kvm_get_msr() argument
1979 return kvm_get_msr_ignored_check(vcpu, index, data, false); in kvm_get_msr()
1983 int kvm_set_msr(struct kvm_vcpu *vcpu, u32 index, u64 data) in kvm_set_msr() argument
1985 return kvm_set_msr_ignored_check(vcpu, index, data, false); in kvm_set_msr()
1989 static void complete_userspace_rdmsr(struct kvm_vcpu *vcpu) in complete_userspace_rdmsr() argument
1991 if (!vcpu->run->msr.error) { in complete_userspace_rdmsr()
1992 kvm_rax_write(vcpu, (u32)vcpu->run->msr.data); in complete_userspace_rdmsr()
1993 kvm_rdx_write(vcpu, vcpu->run->msr.data >> 32); in complete_userspace_rdmsr()
1997 static int complete_emulated_msr_access(struct kvm_vcpu *vcpu) in complete_emulated_msr_access() argument
1999 return complete_emulated_insn_gp(vcpu, vcpu->run->msr.error); in complete_emulated_msr_access()
2002 static int complete_emulated_rdmsr(struct kvm_vcpu *vcpu) in complete_emulated_rdmsr() argument
2004 complete_userspace_rdmsr(vcpu); in complete_emulated_rdmsr()
2005 return complete_emulated_msr_access(vcpu); in complete_emulated_rdmsr()
2008 static int complete_fast_msr_access(struct kvm_vcpu *vcpu) in complete_fast_msr_access() argument
2010 return kvm_x86_call(complete_emulated_msr)(vcpu, vcpu->run->msr.error); in complete_fast_msr_access()
2013 static int complete_fast_rdmsr(struct kvm_vcpu *vcpu) in complete_fast_rdmsr() argument
2015 complete_userspace_rdmsr(vcpu); in complete_fast_rdmsr()
2016 return complete_fast_msr_access(vcpu); in complete_fast_rdmsr()
2031 static int kvm_msr_user_space(struct kvm_vcpu *vcpu, u32 index, in kvm_msr_user_space() argument
2033 int (*completion)(struct kvm_vcpu *vcpu), in kvm_msr_user_space() argument
2039 if (!(vcpu->kvm->arch.user_space_msr_mask & msr_reason)) in kvm_msr_user_space()
2042 vcpu->run->exit_reason = exit_reason; in kvm_msr_user_space()
2043 vcpu->run->msr.error = 0; in kvm_msr_user_space()
2044 memset(vcpu->run->msr.pad, 0, sizeof(vcpu->run->msr.pad)); in kvm_msr_user_space()
2045 vcpu->run->msr.reason = msr_reason; in kvm_msr_user_space()
2046 vcpu->run->msr.index = index; in kvm_msr_user_space()
2047 vcpu->run->msr.data = data; in kvm_msr_user_space()
2048 vcpu->arch.complete_userspace_io = completion; in kvm_msr_user_space()
2053 int kvm_emulate_rdmsr(struct kvm_vcpu *vcpu) in kvm_emulate_rdmsr() argument
2055 u32 ecx = kvm_rcx_read(vcpu); in kvm_emulate_rdmsr()
2059 r = kvm_get_msr_with_filter(vcpu, ecx, &data); in kvm_emulate_rdmsr()
2064 kvm_rax_write(vcpu, data & -1u); in kvm_emulate_rdmsr()
2065 kvm_rdx_write(vcpu, (data >> 32) & -1u); in kvm_emulate_rdmsr()
2068 if (kvm_msr_user_space(vcpu, ecx, KVM_EXIT_X86_RDMSR, 0, in kvm_emulate_rdmsr()
2074 return kvm_x86_call(complete_emulated_msr)(vcpu, r); in kvm_emulate_rdmsr()
2078 int kvm_emulate_wrmsr(struct kvm_vcpu *vcpu) in kvm_emulate_wrmsr() argument
2080 u32 ecx = kvm_rcx_read(vcpu); in kvm_emulate_wrmsr()
2081 u64 data = kvm_read_edx_eax(vcpu); in kvm_emulate_wrmsr()
2084 r = kvm_set_msr_with_filter(vcpu, ecx, data); in kvm_emulate_wrmsr()
2090 if (kvm_msr_user_space(vcpu, ecx, KVM_EXIT_X86_WRMSR, data, in kvm_emulate_wrmsr()
2099 return kvm_x86_call(complete_emulated_msr)(vcpu, r); in kvm_emulate_wrmsr()
2103 int kvm_emulate_as_nop(struct kvm_vcpu *vcpu) in kvm_emulate_as_nop() argument
2105 return kvm_skip_emulated_instruction(vcpu); in kvm_emulate_as_nop()
2108 int kvm_emulate_invd(struct kvm_vcpu *vcpu) in kvm_emulate_invd() argument
2111 return kvm_emulate_as_nop(vcpu); in kvm_emulate_invd()
2115 int kvm_handle_invalid_op(struct kvm_vcpu *vcpu) in kvm_handle_invalid_op() argument
2117 kvm_queue_exception(vcpu, UD_VECTOR); in kvm_handle_invalid_op()
2123 static int kvm_emulate_monitor_mwait(struct kvm_vcpu *vcpu, const char *insn) in kvm_emulate_monitor_mwait() argument
2125 if (!kvm_check_has_quirk(vcpu->kvm, KVM_X86_QUIRK_MWAIT_NEVER_UD_FAULTS) && in kvm_emulate_monitor_mwait()
2126 !guest_cpuid_has(vcpu, X86_FEATURE_MWAIT)) in kvm_emulate_monitor_mwait()
2127 return kvm_handle_invalid_op(vcpu); in kvm_emulate_monitor_mwait()
2130 return kvm_emulate_as_nop(vcpu); in kvm_emulate_monitor_mwait()
2132 int kvm_emulate_mwait(struct kvm_vcpu *vcpu) in kvm_emulate_mwait() argument
2134 return kvm_emulate_monitor_mwait(vcpu, "MWAIT"); in kvm_emulate_mwait()
2138 int kvm_emulate_monitor(struct kvm_vcpu *vcpu) in kvm_emulate_monitor() argument
2140 return kvm_emulate_monitor_mwait(vcpu, "MONITOR"); in kvm_emulate_monitor()
2144 static inline bool kvm_vcpu_exit_request(struct kvm_vcpu *vcpu) in kvm_vcpu_exit_request() argument
2147 return vcpu->mode == EXITING_GUEST_MODE || kvm_request_pending(vcpu) || in kvm_vcpu_exit_request()
2158 static int handle_fastpath_set_x2apic_icr_irqoff(struct kvm_vcpu *vcpu, u64 data) in handle_fastpath_set_x2apic_icr_irqoff() argument
2160 if (!lapic_in_kernel(vcpu) || !apic_x2apic_mode(vcpu->arch.apic)) in handle_fastpath_set_x2apic_icr_irqoff()
2167 return kvm_x2apic_icr_write(vcpu->arch.apic, data); in handle_fastpath_set_x2apic_icr_irqoff()
2172 static int handle_fastpath_set_tscdeadline(struct kvm_vcpu *vcpu, u64 data) in handle_fastpath_set_tscdeadline() argument
2174 if (!kvm_can_use_hv_timer(vcpu)) in handle_fastpath_set_tscdeadline()
2177 kvm_set_lapic_tscdeadline_msr(vcpu, data); in handle_fastpath_set_tscdeadline()
2181 fastpath_t handle_fastpath_set_msr_irqoff(struct kvm_vcpu *vcpu) in handle_fastpath_set_msr_irqoff() argument
2183 u32 msr = kvm_rcx_read(vcpu); in handle_fastpath_set_msr_irqoff()
2188 kvm_vcpu_srcu_read_lock(vcpu); in handle_fastpath_set_msr_irqoff()
2192 data = kvm_read_edx_eax(vcpu); in handle_fastpath_set_msr_irqoff()
2193 handled = !handle_fastpath_set_x2apic_icr_irqoff(vcpu, data); in handle_fastpath_set_msr_irqoff()
2196 data = kvm_read_edx_eax(vcpu); in handle_fastpath_set_msr_irqoff()
2197 handled = !handle_fastpath_set_tscdeadline(vcpu, data); in handle_fastpath_set_msr_irqoff()
2205 if (!kvm_skip_emulated_instruction(vcpu)) in handle_fastpath_set_msr_irqoff()
2214 kvm_vcpu_srcu_read_unlock(vcpu); in handle_fastpath_set_msr_irqoff()
2223 static int do_get_msr(struct kvm_vcpu *vcpu, unsigned index, u64 *data) in do_get_msr() argument
2225 return kvm_get_msr_ignored_check(vcpu, index, data, true); in do_get_msr()
2228 static int do_set_msr(struct kvm_vcpu *vcpu, unsigned index, u64 *data) in do_set_msr() argument
2234 * not support modifying the guest vCPU model on the fly, e.g. changing in do_set_msr()
2239 if (kvm_vcpu_has_run(vcpu) && kvm_is_immutable_feature_msr(index) && in do_set_msr()
2240 (do_get_msr(vcpu, index, &val) || *data != val)) in do_set_msr()
2243 return kvm_set_msr_ignored_check(vcpu, index, *data, true); in do_set_msr()
2353 static void kvm_write_system_time(struct kvm_vcpu *vcpu, gpa_t system_time, in kvm_write_system_time() argument
2356 struct kvm_arch *ka = &vcpu->kvm->arch; in kvm_write_system_time()
2358 if (vcpu->vcpu_id == 0 && !host_initiated) { in kvm_write_system_time()
2360 kvm_make_request(KVM_REQ_MASTERCLOCK_UPDATE, vcpu); in kvm_write_system_time()
2365 vcpu->arch.time = system_time; in kvm_write_system_time()
2366 kvm_make_request(KVM_REQ_GLOBAL_CLOCK_UPDATE, vcpu); in kvm_write_system_time()
2370 kvm_gpc_activate(&vcpu->arch.pv_time, system_time & ~1ULL, in kvm_write_system_time()
2373 kvm_gpc_deactivate(&vcpu->arch.pv_time); in kvm_write_system_time()
2426 static void kvm_vcpu_write_tsc_multiplier(struct kvm_vcpu *vcpu, u64 l1_multiplier);
2428 static int set_tsc_khz(struct kvm_vcpu *vcpu, u32 user_tsc_khz, bool scale) in set_tsc_khz() argument
2434 kvm_vcpu_write_tsc_multiplier(vcpu, kvm_caps.default_tsc_scaling_ratio); in set_tsc_khz()
2441 vcpu->arch.tsc_catchup = 1; in set_tsc_khz()
2442 vcpu->arch.tsc_always_catchup = 1; in set_tsc_khz()
2460 kvm_vcpu_write_tsc_multiplier(vcpu, ratio); in set_tsc_khz()
2464 static int kvm_set_tsc_khz(struct kvm_vcpu *vcpu, u32 user_tsc_khz) in kvm_set_tsc_khz() argument
2472 kvm_vcpu_write_tsc_multiplier(vcpu, kvm_caps.default_tsc_scaling_ratio); in kvm_set_tsc_khz()
2478 &vcpu->arch.virtual_tsc_shift, in kvm_set_tsc_khz()
2479 &vcpu->arch.virtual_tsc_mult); in kvm_set_tsc_khz()
2480 vcpu->arch.virtual_tsc_khz = user_tsc_khz; in kvm_set_tsc_khz()
2495 return set_tsc_khz(vcpu, user_tsc_khz, use_scaling); in kvm_set_tsc_khz()
2498 static u64 compute_guest_tsc(struct kvm_vcpu *vcpu, s64 kernel_ns) in compute_guest_tsc() argument
2500 u64 tsc = pvclock_scale_delta(kernel_ns-vcpu->arch.this_tsc_nsec, in compute_guest_tsc()
2501 vcpu->arch.virtual_tsc_mult, in compute_guest_tsc()
2502 vcpu->arch.virtual_tsc_shift); in compute_guest_tsc()
2503 tsc += vcpu->arch.this_tsc_write; in compute_guest_tsc()
2514 static void kvm_track_tsc_matching(struct kvm_vcpu *vcpu, bool new_generation) in kvm_track_tsc_matching() argument
2517 struct kvm_arch *ka = &vcpu->kvm->arch; in kvm_track_tsc_matching()
2523 * vCPUs doesn't include the reference vCPU, hence "+1". in kvm_track_tsc_matching()
2526 atomic_read(&vcpu->kvm->online_vcpus)) && in kvm_track_tsc_matching()
2537 kvm_make_request(KVM_REQ_MASTERCLOCK_UPDATE, vcpu); in kvm_track_tsc_matching()
2539 trace_kvm_track_tsc(vcpu->vcpu_id, ka->nr_vcpus_matched_tsc, in kvm_track_tsc_matching()
2540 atomic_read(&vcpu->kvm->online_vcpus), in kvm_track_tsc_matching()
2570 static u64 kvm_compute_l1_tsc_offset(struct kvm_vcpu *vcpu, u64 target_tsc) in kvm_compute_l1_tsc_offset() argument
2574 tsc = kvm_scale_tsc(rdtsc(), vcpu->arch.l1_tsc_scaling_ratio); in kvm_compute_l1_tsc_offset()
2579 u64 kvm_read_l1_tsc(struct kvm_vcpu *vcpu, u64 host_tsc) in kvm_read_l1_tsc() argument
2581 return vcpu->arch.l1_tsc_offset + in kvm_read_l1_tsc()
2582 kvm_scale_tsc(host_tsc, vcpu->arch.l1_tsc_scaling_ratio); in kvm_read_l1_tsc()
2611 static void kvm_vcpu_write_tsc_offset(struct kvm_vcpu *vcpu, u64 l1_offset) in kvm_vcpu_write_tsc_offset() argument
2613 trace_kvm_write_tsc_offset(vcpu->vcpu_id, in kvm_vcpu_write_tsc_offset()
2614 vcpu->arch.l1_tsc_offset, in kvm_vcpu_write_tsc_offset()
2617 vcpu->arch.l1_tsc_offset = l1_offset; in kvm_vcpu_write_tsc_offset()
2624 if (is_guest_mode(vcpu)) in kvm_vcpu_write_tsc_offset()
2625 vcpu->arch.tsc_offset = kvm_calc_nested_tsc_offset( in kvm_vcpu_write_tsc_offset()
2627 kvm_x86_call(get_l2_tsc_offset)(vcpu), in kvm_vcpu_write_tsc_offset()
2628 kvm_x86_call(get_l2_tsc_multiplier)(vcpu)); in kvm_vcpu_write_tsc_offset()
2630 vcpu->arch.tsc_offset = l1_offset; in kvm_vcpu_write_tsc_offset()
2632 kvm_x86_call(write_tsc_offset)(vcpu); in kvm_vcpu_write_tsc_offset()
2635 static void kvm_vcpu_write_tsc_multiplier(struct kvm_vcpu *vcpu, u64 l1_multiplier) in kvm_vcpu_write_tsc_multiplier() argument
2637 vcpu->arch.l1_tsc_scaling_ratio = l1_multiplier; in kvm_vcpu_write_tsc_multiplier()
2640 if (is_guest_mode(vcpu)) in kvm_vcpu_write_tsc_multiplier()
2641 vcpu->arch.tsc_scaling_ratio = kvm_calc_nested_tsc_multiplier( in kvm_vcpu_write_tsc_multiplier()
2643 kvm_x86_call(get_l2_tsc_multiplier)(vcpu)); in kvm_vcpu_write_tsc_multiplier()
2645 vcpu->arch.tsc_scaling_ratio = l1_multiplier; in kvm_vcpu_write_tsc_multiplier()
2648 kvm_x86_call(write_tsc_multiplier)(vcpu); in kvm_vcpu_write_tsc_multiplier()
2666 * offset for the vcpu and tracks the TSC matching generation that the vcpu
2669 static void __kvm_synchronize_tsc(struct kvm_vcpu *vcpu, u64 offset, u64 tsc, in __kvm_synchronize_tsc() argument
2672 struct kvm *kvm = vcpu->kvm; in __kvm_synchronize_tsc()
2682 kvm->arch.last_tsc_khz = vcpu->arch.virtual_tsc_khz; in __kvm_synchronize_tsc()
2685 vcpu->arch.last_guest_tsc = tsc; in __kvm_synchronize_tsc()
2687 kvm_vcpu_write_tsc_offset(vcpu, offset); in __kvm_synchronize_tsc()
2704 } else if (vcpu->arch.this_tsc_generation != kvm->arch.cur_tsc_generation) { in __kvm_synchronize_tsc()
2708 /* Keep track of which generation this VCPU has synchronized to */ in __kvm_synchronize_tsc()
2709 vcpu->arch.this_tsc_generation = kvm->arch.cur_tsc_generation; in __kvm_synchronize_tsc()
2710 vcpu->arch.this_tsc_nsec = kvm->arch.cur_tsc_nsec; in __kvm_synchronize_tsc()
2711 vcpu->arch.this_tsc_write = kvm->arch.cur_tsc_write; in __kvm_synchronize_tsc()
2713 kvm_track_tsc_matching(vcpu, !matched); in __kvm_synchronize_tsc()
2716 static void kvm_synchronize_tsc(struct kvm_vcpu *vcpu, u64 *user_value) in kvm_synchronize_tsc() argument
2719 struct kvm *kvm = vcpu->kvm; in kvm_synchronize_tsc()
2726 offset = kvm_compute_l1_tsc_offset(vcpu, data); in kvm_synchronize_tsc()
2730 if (vcpu->arch.virtual_tsc_khz) { in kvm_synchronize_tsc()
2733 * Force synchronization when creating a vCPU, or when in kvm_synchronize_tsc()
2739 nsec_to_cycles(vcpu, elapsed); in kvm_synchronize_tsc()
2740 u64 tsc_hz = vcpu->arch.virtual_tsc_khz * 1000LL; in kvm_synchronize_tsc()
2744 * previously set vCPU, we assume that they were intended to be in kvm_synchronize_tsc()
2751 * value written by userspace (on any vCPU) should not be subject in kvm_synchronize_tsc()
2753 * come from the kernel's default vCPU creation. Make the 1-second in kvm_synchronize_tsc()
2771 vcpu->arch.virtual_tsc_khz == kvm->arch.last_tsc_khz) { in kvm_synchronize_tsc()
2775 u64 delta = nsec_to_cycles(vcpu, elapsed); in kvm_synchronize_tsc()
2777 offset = kvm_compute_l1_tsc_offset(vcpu, data); in kvm_synchronize_tsc()
2782 __kvm_synchronize_tsc(vcpu, offset, data, ns, matched); in kvm_synchronize_tsc()
2786 static inline void adjust_tsc_offset_guest(struct kvm_vcpu *vcpu, in adjust_tsc_offset_guest() argument
2789 u64 tsc_offset = vcpu->arch.l1_tsc_offset; in adjust_tsc_offset_guest()
2790 kvm_vcpu_write_tsc_offset(vcpu, tsc_offset + adjustment); in adjust_tsc_offset_guest()
2793 static inline void adjust_tsc_offset_host(struct kvm_vcpu *vcpu, s64 adjustment) in adjust_tsc_offset_host() argument
2795 if (vcpu->arch.l1_tsc_scaling_ratio != kvm_caps.default_tsc_scaling_ratio) in adjust_tsc_offset_host()
2798 vcpu->arch.l1_tsc_scaling_ratio); in adjust_tsc_offset_host()
2799 adjust_tsc_offset_guest(vcpu, adjustment); in adjust_tsc_offset_host()
3068 struct kvm_vcpu *vcpu; in kvm_end_pvclock_update() local
3073 kvm_for_each_vcpu(i, vcpu, kvm) in kvm_end_pvclock_update()
3074 kvm_make_request(KVM_REQ_CLOCK_UPDATE, vcpu); in kvm_end_pvclock_update()
3077 kvm_for_each_vcpu(i, vcpu, kvm) in kvm_end_pvclock_update()
3078 kvm_clear_request(KVM_REQ_MCLOCK_INPROGRESS, vcpu); in kvm_end_pvclock_update()
3165 struct kvm_vcpu_arch *vcpu = &v->arch; in kvm_setup_guest_pvclock() local
3182 * This VCPU is paused, but it's legal for a guest to read another in kvm_setup_guest_pvclock()
3183 * VCPU's kvmclock, so we really have to follow the specification where in kvm_setup_guest_pvclock()
3188 guest_hv_clock->version = vcpu->hv_clock.version = (guest_hv_clock->version + 1) | 1; in kvm_setup_guest_pvclock()
3192 vcpu->hv_clock.flags |= (guest_hv_clock->flags & PVCLOCK_GUEST_STOPPED); in kvm_setup_guest_pvclock()
3194 if (vcpu->pvclock_set_guest_stopped_request) { in kvm_setup_guest_pvclock()
3195 vcpu->hv_clock.flags |= PVCLOCK_GUEST_STOPPED; in kvm_setup_guest_pvclock()
3196 vcpu->pvclock_set_guest_stopped_request = false; in kvm_setup_guest_pvclock()
3199 memcpy(guest_hv_clock, &vcpu->hv_clock, sizeof(*guest_hv_clock)); in kvm_setup_guest_pvclock()
3206 guest_hv_clock->version = ++vcpu->hv_clock.version; in kvm_setup_guest_pvclock()
3211 trace_kvm_pvclock_update(v->vcpu_id, &vcpu->hv_clock); in kvm_setup_guest_pvclock()
3218 struct kvm_vcpu_arch *vcpu = &v->arch; in kvm_guest_time_update() local
3270 * 2) Broken TSC compensation resets the base at each VCPU in kvm_guest_time_update()
3276 if (vcpu->tsc_catchup) { in kvm_guest_time_update()
3292 if (unlikely(vcpu->hw_tsc_khz != tgt_tsc_khz)) { in kvm_guest_time_update()
3294 &vcpu->hv_clock.tsc_shift, in kvm_guest_time_update()
3295 &vcpu->hv_clock.tsc_to_system_mul); in kvm_guest_time_update()
3296 vcpu->hw_tsc_khz = tgt_tsc_khz; in kvm_guest_time_update()
3300 vcpu->hv_clock.tsc_timestamp = tsc_timestamp; in kvm_guest_time_update()
3301 vcpu->hv_clock.system_time = kernel_ns + v->kvm->arch.kvmclock_offset; in kvm_guest_time_update()
3302 vcpu->last_guest_tsc = tsc_timestamp; in kvm_guest_time_update()
3309 vcpu->hv_clock.flags = pvclock_flags; in kvm_guest_time_update()
3311 if (vcpu->pv_time.active) in kvm_guest_time_update()
3312 kvm_setup_guest_pvclock(v, &vcpu->pv_time, 0, false); in kvm_guest_time_update()
3314 if (vcpu->xen.vcpu_info_cache.active) in kvm_guest_time_update()
3315 kvm_setup_guest_pvclock(v, &vcpu->xen.vcpu_info_cache, in kvm_guest_time_update()
3318 if (vcpu->xen.vcpu_time_info_cache.active) in kvm_guest_time_update()
3319 kvm_setup_guest_pvclock(v, &vcpu->xen.vcpu_time_info_cache, 0, in kvm_guest_time_update()
3322 kvm_hv_setup_tsc_page(v->kvm, &vcpu->hv_clock); in kvm_guest_time_update()
3403 * kvmclock updates which are isolated to a given vcpu, such as
3404 * vcpu->cpu migration, should not allow system_timestamp from
3406 * correction applies to one vcpu's system_timestamp but not
3412 * The time for a remote vcpu to update its kvmclock is bound
3425 struct kvm_vcpu *vcpu; in kvmclock_update_fn() local
3427 kvm_for_each_vcpu(i, vcpu, kvm) { in kvmclock_update_fn()
3428 kvm_make_request(KVM_REQ_CLOCK_UPDATE, vcpu); in kvmclock_update_fn()
3429 kvm_vcpu_kick(vcpu); in kvmclock_update_fn()
3469 static bool can_set_mci_status(struct kvm_vcpu *vcpu) in can_set_mci_status() argument
3472 if (guest_cpuid_is_amd_compatible(vcpu)) in can_set_mci_status()
3473 return !!(vcpu->arch.msr_hwcr & BIT_ULL(18)); in can_set_mci_status()
3478 static int set_msr_mce(struct kvm_vcpu *vcpu, struct msr_data *msr_info) in set_msr_mce() argument
3480 u64 mcg_cap = vcpu->arch.mcg_cap; in set_msr_mce()
3488 vcpu->arch.mcg_status = data; in set_msr_mce()
3496 vcpu->arch.mcg_ctl = data; in set_msr_mce()
3510 vcpu->arch.mci_ctl2_banks[offset] = data; in set_msr_mce()
3537 data != 0 && !can_set_mci_status(vcpu)) in set_msr_mce()
3542 vcpu->arch.mce_banks[offset] = data; in set_msr_mce()
3550 static inline bool kvm_pv_async_pf_enabled(struct kvm_vcpu *vcpu) in kvm_pv_async_pf_enabled() argument
3554 return (vcpu->arch.apf.msr_en_val & mask) == mask; in kvm_pv_async_pf_enabled()
3557 static int kvm_pv_enable_async_pf(struct kvm_vcpu *vcpu, u64 data) in kvm_pv_enable_async_pf() argument
3565 if (!guest_pv_has(vcpu, KVM_FEATURE_ASYNC_PF_VMEXIT) && in kvm_pv_enable_async_pf()
3569 if (!guest_pv_has(vcpu, KVM_FEATURE_ASYNC_PF_INT) && in kvm_pv_enable_async_pf()
3573 if (!lapic_in_kernel(vcpu)) in kvm_pv_enable_async_pf()
3576 vcpu->arch.apf.msr_en_val = data; in kvm_pv_enable_async_pf()
3578 if (!kvm_pv_async_pf_enabled(vcpu)) { in kvm_pv_enable_async_pf()
3579 kvm_clear_async_pf_completion_queue(vcpu); in kvm_pv_enable_async_pf()
3580 kvm_async_pf_hash_reset(vcpu); in kvm_pv_enable_async_pf()
3584 if (kvm_gfn_to_hva_cache_init(vcpu->kvm, &vcpu->arch.apf.data, gpa, in kvm_pv_enable_async_pf()
3588 vcpu->arch.apf.send_user_only = !(data & KVM_ASYNC_PF_SEND_ALWAYS); in kvm_pv_enable_async_pf()
3589 vcpu->arch.apf.delivery_as_pf_vmexit = data & KVM_ASYNC_PF_DELIVERY_AS_PF_VMEXIT; in kvm_pv_enable_async_pf()
3591 kvm_async_pf_wakeup_all(vcpu); in kvm_pv_enable_async_pf()
3596 static int kvm_pv_enable_async_pf_int(struct kvm_vcpu *vcpu, u64 data) in kvm_pv_enable_async_pf_int() argument
3602 if (!lapic_in_kernel(vcpu)) in kvm_pv_enable_async_pf_int()
3605 vcpu->arch.apf.msr_int_val = data; in kvm_pv_enable_async_pf_int()
3607 vcpu->arch.apf.vec = data & KVM_ASYNC_PF_VEC_MASK; in kvm_pv_enable_async_pf_int()
3612 static void kvmclock_reset(struct kvm_vcpu *vcpu) in kvmclock_reset() argument
3614 kvm_gpc_deactivate(&vcpu->arch.pv_time); in kvmclock_reset()
3615 vcpu->arch.time = 0; in kvmclock_reset()
3618 static void kvm_vcpu_flush_tlb_all(struct kvm_vcpu *vcpu) in kvm_vcpu_flush_tlb_all() argument
3620 ++vcpu->stat.tlb_flush; in kvm_vcpu_flush_tlb_all()
3621 kvm_x86_call(flush_tlb_all)(vcpu); in kvm_vcpu_flush_tlb_all()
3624 kvm_clear_request(KVM_REQ_TLB_FLUSH_CURRENT, vcpu); in kvm_vcpu_flush_tlb_all()
3627 static void kvm_vcpu_flush_tlb_guest(struct kvm_vcpu *vcpu) in kvm_vcpu_flush_tlb_guest() argument
3629 ++vcpu->stat.tlb_flush; in kvm_vcpu_flush_tlb_guest()
3638 kvm_mmu_sync_roots(vcpu); in kvm_vcpu_flush_tlb_guest()
3639 kvm_mmu_sync_prev_roots(vcpu); in kvm_vcpu_flush_tlb_guest()
3642 kvm_x86_call(flush_tlb_guest)(vcpu); in kvm_vcpu_flush_tlb_guest()
3648 kvm_hv_vcpu_purge_flush_tlb(vcpu); in kvm_vcpu_flush_tlb_guest()
3652 static inline void kvm_vcpu_flush_tlb_current(struct kvm_vcpu *vcpu) in kvm_vcpu_flush_tlb_current() argument
3654 ++vcpu->stat.tlb_flush; in kvm_vcpu_flush_tlb_current()
3655 kvm_x86_call(flush_tlb_current)(vcpu); in kvm_vcpu_flush_tlb_current()
3664 void kvm_service_local_tlb_flush_requests(struct kvm_vcpu *vcpu) in kvm_service_local_tlb_flush_requests() argument
3666 if (kvm_check_request(KVM_REQ_TLB_FLUSH_CURRENT, vcpu)) in kvm_service_local_tlb_flush_requests()
3667 kvm_vcpu_flush_tlb_current(vcpu); in kvm_service_local_tlb_flush_requests()
3669 if (kvm_check_request(KVM_REQ_TLB_FLUSH_GUEST, vcpu)) in kvm_service_local_tlb_flush_requests()
3670 kvm_vcpu_flush_tlb_guest(vcpu); in kvm_service_local_tlb_flush_requests()
3674 static void record_steal_time(struct kvm_vcpu *vcpu) in record_steal_time() argument
3676 struct gfn_to_hva_cache *ghc = &vcpu->arch.st.cache; in record_steal_time()
3679 gpa_t gpa = vcpu->arch.st.msr_val & KVM_STEAL_VALID_BITS; in record_steal_time()
3683 if (kvm_xen_msr_enabled(vcpu->kvm)) { in record_steal_time()
3684 kvm_xen_runstate_set_running(vcpu); in record_steal_time()
3688 if (!(vcpu->arch.st.msr_val & KVM_MSR_ENABLED)) in record_steal_time()
3691 if (WARN_ON_ONCE(current->mm != vcpu->kvm->mm)) in record_steal_time()
3694 slots = kvm_memslots(vcpu->kvm); in record_steal_time()
3702 if (kvm_gfn_to_hva_cache_init(vcpu->kvm, ghc, gpa, sizeof(*st)) || in record_steal_time()
3712 if (guest_pv_has(vcpu, KVM_FEATURE_PV_TLB_FLUSH)) { in record_steal_time()
3731 vcpu->arch.st.preempted = 0; in record_steal_time()
3733 trace_kvm_pv_tlb_flush(vcpu->vcpu_id, in record_steal_time()
3736 kvm_vcpu_flush_tlb_guest(vcpu); in record_steal_time()
3745 vcpu->arch.st.preempted = 0; in record_steal_time()
3759 vcpu->arch.st.last_steal; in record_steal_time()
3760 vcpu->arch.st.last_steal = current->sched_info.run_delay; in record_steal_time()
3769 mark_page_dirty_in_slot(vcpu->kvm, ghc->memslot, gpa_to_gfn(ghc->gpa)); in record_steal_time()
3772 int kvm_set_msr_common(struct kvm_vcpu *vcpu, struct msr_data *msr_info) in kvm_set_msr_common() argument
3777 if (msr && msr == vcpu->kvm->arch.xen_hvm_config.msr) in kvm_set_msr_common()
3778 return kvm_xen_write_hypercall_page(vcpu, data); in kvm_set_msr_common()
3793 vcpu->arch.microcode_version = data; in kvm_set_msr_common()
3798 vcpu->arch.arch_capabilities = data; in kvm_set_msr_common()
3808 * disallows changing feature MSRs after the vCPU has run; PMU in kvm_set_msr_common()
3809 * refresh will bug the VM if called after the vCPU has run. in kvm_set_msr_common()
3811 if (vcpu->arch.perf_capabilities == data) in kvm_set_msr_common()
3814 vcpu->arch.perf_capabilities = data; in kvm_set_msr_common()
3815 kvm_pmu_refresh(vcpu); in kvm_set_msr_common()
3821 if ((!guest_has_pred_cmd_msr(vcpu))) in kvm_set_msr_common()
3824 if (!guest_cpuid_has(vcpu, X86_FEATURE_SPEC_CTRL) && in kvm_set_msr_common()
3825 !guest_cpuid_has(vcpu, X86_FEATURE_AMD_IBPB)) in kvm_set_msr_common()
3828 if (!guest_cpuid_has(vcpu, X86_FEATURE_SBPB)) in kvm_set_msr_common()
3849 !guest_cpuid_has(vcpu, X86_FEATURE_FLUSH_L1D)) in kvm_set_msr_common()
3860 return set_efer(vcpu, msr_info); in kvm_set_msr_common()
3872 kvm_pr_unimpl_wrmsr(vcpu, msr, data); in kvm_set_msr_common()
3875 vcpu->arch.msr_hwcr = data; in kvm_set_msr_common()
3879 kvm_pr_unimpl_wrmsr(vcpu, msr, data); in kvm_set_msr_common()
3887 vcpu->arch.pat = data; in kvm_set_msr_common()
3891 return kvm_mtrr_set_msr(vcpu, msr, data); in kvm_set_msr_common()
3893 return kvm_set_apic_base(vcpu, msr_info); in kvm_set_msr_common()
3895 return kvm_x2apic_msr_write(vcpu, msr, data); in kvm_set_msr_common()
3897 kvm_set_lapic_tscdeadline_msr(vcpu, data); in kvm_set_msr_common()
3900 if (guest_cpuid_has(vcpu, X86_FEATURE_TSC_ADJUST)) { in kvm_set_msr_common()
3902 s64 adj = data - vcpu->arch.ia32_tsc_adjust_msr; in kvm_set_msr_common()
3903 adjust_tsc_offset_guest(vcpu, adj); in kvm_set_msr_common()
3907 kvm_make_request(KVM_REQ_CLOCK_UPDATE, vcpu); in kvm_set_msr_common()
3909 vcpu->arch.ia32_tsc_adjust_msr = data; in kvm_set_msr_common()
3913 u64 old_val = vcpu->arch.ia32_misc_enable_msr; in kvm_set_msr_common()
3925 if (!kvm_check_has_quirk(vcpu->kvm, KVM_X86_QUIRK_MISC_ENABLE_NO_MWAIT) && in kvm_set_msr_common()
3927 if (!guest_cpuid_has(vcpu, X86_FEATURE_XMM3)) in kvm_set_msr_common()
3929 vcpu->arch.ia32_misc_enable_msr = data; in kvm_set_msr_common()
3930 kvm_update_cpuid_runtime(vcpu); in kvm_set_msr_common()
3932 vcpu->arch.ia32_misc_enable_msr = data; in kvm_set_msr_common()
3939 vcpu->arch.smbase = data; in kvm_set_msr_common()
3942 vcpu->arch.msr_ia32_power_ctl = data; in kvm_set_msr_common()
3946 kvm_synchronize_tsc(vcpu, &data); in kvm_set_msr_common()
3948 u64 adj = kvm_compute_l1_tsc_offset(vcpu, data) - vcpu->arch.l1_tsc_offset; in kvm_set_msr_common()
3949 adjust_tsc_offset_guest(vcpu, adj); in kvm_set_msr_common()
3950 vcpu->arch.ia32_tsc_adjust_msr += adj; in kvm_set_msr_common()
3955 !guest_cpuid_has(vcpu, X86_FEATURE_XSAVES)) in kvm_set_msr_common()
3964 vcpu->arch.ia32_xss = data; in kvm_set_msr_common()
3965 kvm_update_cpuid_runtime(vcpu); in kvm_set_msr_common()
3970 vcpu->arch.smi_count = data; in kvm_set_msr_common()
3973 if (!guest_pv_has(vcpu, KVM_FEATURE_CLOCKSOURCE2)) in kvm_set_msr_common()
3976 vcpu->kvm->arch.wall_clock = data; in kvm_set_msr_common()
3977 kvm_write_wall_clock(vcpu->kvm, data, 0); in kvm_set_msr_common()
3980 if (!guest_pv_has(vcpu, KVM_FEATURE_CLOCKSOURCE)) in kvm_set_msr_common()
3983 vcpu->kvm->arch.wall_clock = data; in kvm_set_msr_common()
3984 kvm_write_wall_clock(vcpu->kvm, data, 0); in kvm_set_msr_common()
3987 if (!guest_pv_has(vcpu, KVM_FEATURE_CLOCKSOURCE2)) in kvm_set_msr_common()
3990 kvm_write_system_time(vcpu, data, false, msr_info->host_initiated); in kvm_set_msr_common()
3993 if (!guest_pv_has(vcpu, KVM_FEATURE_CLOCKSOURCE)) in kvm_set_msr_common()
3996 kvm_write_system_time(vcpu, data, true, msr_info->host_initiated); in kvm_set_msr_common()
3999 if (!guest_pv_has(vcpu, KVM_FEATURE_ASYNC_PF)) in kvm_set_msr_common()
4002 if (kvm_pv_enable_async_pf(vcpu, data)) in kvm_set_msr_common()
4006 if (!guest_pv_has(vcpu, KVM_FEATURE_ASYNC_PF_INT)) in kvm_set_msr_common()
4009 if (kvm_pv_enable_async_pf_int(vcpu, data)) in kvm_set_msr_common()
4013 if (!guest_pv_has(vcpu, KVM_FEATURE_ASYNC_PF_INT)) in kvm_set_msr_common()
4016 vcpu->arch.apf.pageready_pending = false; in kvm_set_msr_common()
4017 kvm_check_async_pf_completion(vcpu); in kvm_set_msr_common()
4021 if (!guest_pv_has(vcpu, KVM_FEATURE_STEAL_TIME)) in kvm_set_msr_common()
4030 vcpu->arch.st.msr_val = data; in kvm_set_msr_common()
4035 kvm_make_request(KVM_REQ_STEAL_UPDATE, vcpu); in kvm_set_msr_common()
4039 if (!guest_pv_has(vcpu, KVM_FEATURE_PV_EOI)) in kvm_set_msr_common()
4042 if (kvm_lapic_set_pv_eoi(vcpu, data, sizeof(u8))) in kvm_set_msr_common()
4047 if (!guest_pv_has(vcpu, KVM_FEATURE_POLL_CONTROL)) in kvm_set_msr_common()
4054 vcpu->arch.msr_kvm_poll_control = data; in kvm_set_msr_common()
4061 return set_msr_mce(vcpu, msr_info); in kvm_set_msr_common()
4067 if (kvm_pmu_is_valid_msr(vcpu, msr)) in kvm_set_msr_common()
4068 return kvm_pmu_set_msr(vcpu, msr_info); in kvm_set_msr_common()
4071 kvm_pr_unimpl_wrmsr(vcpu, msr, data); in kvm_set_msr_common()
4094 return kvm_hv_set_msr_common(vcpu, msr, data, in kvm_set_msr_common()
4101 kvm_pr_unimpl_wrmsr(vcpu, msr, data); in kvm_set_msr_common()
4104 if (!guest_cpuid_has(vcpu, X86_FEATURE_OSVW)) in kvm_set_msr_common()
4106 vcpu->arch.osvw.length = data; in kvm_set_msr_common()
4109 if (!guest_cpuid_has(vcpu, X86_FEATURE_OSVW)) in kvm_set_msr_common()
4111 vcpu->arch.osvw.status = data; in kvm_set_msr_common()
4116 cpuid_fault_enabled(vcpu))) in kvm_set_msr_common()
4118 vcpu->arch.msr_platform_info = data; in kvm_set_msr_common()
4123 !supports_cpuid_fault(vcpu))) in kvm_set_msr_common()
4125 vcpu->arch.msr_misc_features_enables = data; in kvm_set_msr_common()
4130 !guest_cpuid_has(vcpu, X86_FEATURE_XFD)) in kvm_set_msr_common()
4133 if (data & ~kvm_guest_supported_xfd(vcpu)) in kvm_set_msr_common()
4136 fpu_update_guest_xfd(&vcpu->arch.guest_fpu, data); in kvm_set_msr_common()
4140 !guest_cpuid_has(vcpu, X86_FEATURE_XFD)) in kvm_set_msr_common()
4143 if (data & ~kvm_guest_supported_xfd(vcpu)) in kvm_set_msr_common()
4146 vcpu->arch.guest_fpu.xfd_err = data; in kvm_set_msr_common()
4150 if (kvm_pmu_is_valid_msr(vcpu, msr)) in kvm_set_msr_common()
4151 return kvm_pmu_set_msr(vcpu, msr_info); in kvm_set_msr_common()
4159 static int get_msr_mce(struct kvm_vcpu *vcpu, u32 msr, u64 *pdata, bool host) in get_msr_mce() argument
4162 u64 mcg_cap = vcpu->arch.mcg_cap; in get_msr_mce()
4172 data = vcpu->arch.mcg_cap; in get_msr_mce()
4177 data = vcpu->arch.mcg_ctl; in get_msr_mce()
4180 data = vcpu->arch.mcg_status; in get_msr_mce()
4191 data = vcpu->arch.mci_ctl2_banks[offset]; in get_msr_mce()
4200 data = vcpu->arch.mce_banks[offset]; in get_msr_mce()
4209 int kvm_get_msr_common(struct kvm_vcpu *vcpu, struct msr_data *msr_info) in kvm_get_msr_common() argument
4247 if (kvm_pmu_is_valid_msr(vcpu, msr_info->index)) in kvm_get_msr_common()
4248 return kvm_pmu_get_msr(vcpu, msr_info); in kvm_get_msr_common()
4252 msr_info->data = vcpu->arch.microcode_version; in kvm_get_msr_common()
4256 !guest_cpuid_has(vcpu, X86_FEATURE_ARCH_CAPABILITIES)) in kvm_get_msr_common()
4258 msr_info->data = vcpu->arch.arch_capabilities; in kvm_get_msr_common()
4262 !guest_cpuid_has(vcpu, X86_FEATURE_PDCM)) in kvm_get_msr_common()
4264 msr_info->data = vcpu->arch.perf_capabilities; in kvm_get_msr_common()
4267 msr_info->data = vcpu->arch.msr_ia32_power_ctl; in kvm_get_msr_common()
4282 offset = vcpu->arch.l1_tsc_offset; in kvm_get_msr_common()
4283 ratio = vcpu->arch.l1_tsc_scaling_ratio; in kvm_get_msr_common()
4285 offset = vcpu->arch.tsc_offset; in kvm_get_msr_common()
4286 ratio = vcpu->arch.tsc_scaling_ratio; in kvm_get_msr_common()
4293 msr_info->data = vcpu->arch.pat; in kvm_get_msr_common()
4298 return kvm_mtrr_get_msr(vcpu, msr_info->index, &msr_info->data); in kvm_get_msr_common()
4317 msr_info->data = kvm_get_apic_base(vcpu); in kvm_get_msr_common()
4320 return kvm_x2apic_msr_read(vcpu, msr_info->index, &msr_info->data); in kvm_get_msr_common()
4322 msr_info->data = kvm_get_lapic_tscdeadline_msr(vcpu); in kvm_get_msr_common()
4325 msr_info->data = (u64)vcpu->arch.ia32_tsc_adjust_msr; in kvm_get_msr_common()
4328 msr_info->data = vcpu->arch.ia32_misc_enable_msr; in kvm_get_msr_common()
4333 msr_info->data = vcpu->arch.smbase; in kvm_get_msr_common()
4336 msr_info->data = vcpu->arch.smi_count; in kvm_get_msr_common()
4345 msr_info->data = vcpu->arch.efer; in kvm_get_msr_common()
4348 if (!guest_pv_has(vcpu, KVM_FEATURE_CLOCKSOURCE)) in kvm_get_msr_common()
4351 msr_info->data = vcpu->kvm->arch.wall_clock; in kvm_get_msr_common()
4354 if (!guest_pv_has(vcpu, KVM_FEATURE_CLOCKSOURCE2)) in kvm_get_msr_common()
4357 msr_info->data = vcpu->kvm->arch.wall_clock; in kvm_get_msr_common()
4360 if (!guest_pv_has(vcpu, KVM_FEATURE_CLOCKSOURCE)) in kvm_get_msr_common()
4363 msr_info->data = vcpu->arch.time; in kvm_get_msr_common()
4366 if (!guest_pv_has(vcpu, KVM_FEATURE_CLOCKSOURCE2)) in kvm_get_msr_common()
4369 msr_info->data = vcpu->arch.time; in kvm_get_msr_common()
4372 if (!guest_pv_has(vcpu, KVM_FEATURE_ASYNC_PF)) in kvm_get_msr_common()
4375 msr_info->data = vcpu->arch.apf.msr_en_val; in kvm_get_msr_common()
4378 if (!guest_pv_has(vcpu, KVM_FEATURE_ASYNC_PF_INT)) in kvm_get_msr_common()
4381 msr_info->data = vcpu->arch.apf.msr_int_val; in kvm_get_msr_common()
4384 if (!guest_pv_has(vcpu, KVM_FEATURE_ASYNC_PF_INT)) in kvm_get_msr_common()
4390 if (!guest_pv_has(vcpu, KVM_FEATURE_STEAL_TIME)) in kvm_get_msr_common()
4393 msr_info->data = vcpu->arch.st.msr_val; in kvm_get_msr_common()
4396 if (!guest_pv_has(vcpu, KVM_FEATURE_PV_EOI)) in kvm_get_msr_common()
4399 msr_info->data = vcpu->arch.pv_eoi.msr_val; in kvm_get_msr_common()
4402 if (!guest_pv_has(vcpu, KVM_FEATURE_POLL_CONTROL)) in kvm_get_msr_common()
4405 msr_info->data = vcpu->arch.msr_kvm_poll_control; in kvm_get_msr_common()
4414 return get_msr_mce(vcpu, msr_info->index, &msr_info->data, in kvm_get_msr_common()
4418 !guest_cpuid_has(vcpu, X86_FEATURE_XSAVES)) in kvm_get_msr_common()
4420 msr_info->data = vcpu->arch.ia32_xss; in kvm_get_msr_common()
4445 return kvm_hv_get_msr_common(vcpu, in kvm_get_msr_common()
4463 if (!guest_cpuid_has(vcpu, X86_FEATURE_OSVW)) in kvm_get_msr_common()
4465 msr_info->data = vcpu->arch.osvw.length; in kvm_get_msr_common()
4468 if (!guest_cpuid_has(vcpu, X86_FEATURE_OSVW)) in kvm_get_msr_common()
4470 msr_info->data = vcpu->arch.osvw.status; in kvm_get_msr_common()
4474 !vcpu->kvm->arch.guest_can_read_msr_platform_info) in kvm_get_msr_common()
4476 msr_info->data = vcpu->arch.msr_platform_info; in kvm_get_msr_common()
4479 msr_info->data = vcpu->arch.msr_misc_features_enables; in kvm_get_msr_common()
4482 msr_info->data = vcpu->arch.msr_hwcr; in kvm_get_msr_common()
4487 !guest_cpuid_has(vcpu, X86_FEATURE_XFD)) in kvm_get_msr_common()
4490 msr_info->data = vcpu->arch.guest_fpu.fpstate->xfd; in kvm_get_msr_common()
4494 !guest_cpuid_has(vcpu, X86_FEATURE_XFD)) in kvm_get_msr_common()
4497 msr_info->data = vcpu->arch.guest_fpu.xfd_err; in kvm_get_msr_common()
4501 if (kvm_pmu_is_valid_msr(vcpu, msr_info->index)) in kvm_get_msr_common()
4502 return kvm_pmu_get_msr(vcpu, msr_info); in kvm_get_msr_common()
4515 static int __msr_io(struct kvm_vcpu *vcpu, struct kvm_msrs *msrs, in __msr_io() argument
4517 int (*do_msr)(struct kvm_vcpu *vcpu, in __msr_io() argument
4523 if (do_msr(vcpu, entries[i].index, &entries[i].data)) in __msr_io()
4534 static int msr_io(struct kvm_vcpu *vcpu, struct kvm_msrs __user *user_msrs, in msr_io() argument
4535 int (*do_msr)(struct kvm_vcpu *vcpu, in msr_io() argument
4559 r = __msr_io(vcpu, &msrs, entries, do_msr); in msr_io()
4577 static int kvm_ioctl_get_supported_hv_cpuid(struct kvm_vcpu *vcpu, in kvm_ioctl_get_supported_hv_cpuid() argument
4587 r = kvm_get_hv_cpuid(vcpu, &cpuid, cpuid_arg->entries); in kvm_ioctl_get_supported_hv_cpuid()
4978 static bool need_emulate_wbinvd(struct kvm_vcpu *vcpu) in need_emulate_wbinvd() argument
4980 return kvm_arch_has_noncoherent_dma(vcpu->kvm); in need_emulate_wbinvd()
4983 void kvm_arch_vcpu_load(struct kvm_vcpu *vcpu, int cpu) in kvm_arch_vcpu_load() argument
4985 struct kvm_pmu *pmu = vcpu_to_pmu(vcpu); in kvm_arch_vcpu_load()
4987 vcpu->arch.l1tf_flush_l1d = true; in kvm_arch_vcpu_load()
4989 if (vcpu->scheduled_out && pmu->version && pmu->event_count) { in kvm_arch_vcpu_load()
4991 kvm_make_request(KVM_REQ_PMU, vcpu); in kvm_arch_vcpu_load()
4995 if (need_emulate_wbinvd(vcpu)) { in kvm_arch_vcpu_load()
4997 cpumask_set_cpu(cpu, vcpu->arch.wbinvd_dirty_mask); in kvm_arch_vcpu_load()
4998 else if (vcpu->cpu != -1 && vcpu->cpu != cpu) in kvm_arch_vcpu_load()
4999 smp_call_function_single(vcpu->cpu, in kvm_arch_vcpu_load()
5003 kvm_x86_call(vcpu_load)(vcpu, cpu); in kvm_arch_vcpu_load()
5006 vcpu->arch.host_pkru = read_pkru(); in kvm_arch_vcpu_load()
5009 if (unlikely(vcpu->arch.tsc_offset_adjustment)) { in kvm_arch_vcpu_load()
5010 adjust_tsc_offset_host(vcpu, vcpu->arch.tsc_offset_adjustment); in kvm_arch_vcpu_load()
5011 vcpu->arch.tsc_offset_adjustment = 0; in kvm_arch_vcpu_load()
5012 kvm_make_request(KVM_REQ_CLOCK_UPDATE, vcpu); in kvm_arch_vcpu_load()
5015 if (unlikely(vcpu->cpu != cpu) || kvm_check_tsc_unstable()) { in kvm_arch_vcpu_load()
5016 s64 tsc_delta = !vcpu->arch.last_host_tsc ? 0 : in kvm_arch_vcpu_load()
5017 rdtsc() - vcpu->arch.last_host_tsc; in kvm_arch_vcpu_load()
5022 u64 offset = kvm_compute_l1_tsc_offset(vcpu, in kvm_arch_vcpu_load()
5023 vcpu->arch.last_guest_tsc); in kvm_arch_vcpu_load()
5024 kvm_vcpu_write_tsc_offset(vcpu, offset); in kvm_arch_vcpu_load()
5025 vcpu->arch.tsc_catchup = 1; in kvm_arch_vcpu_load()
5028 if (kvm_lapic_hv_timer_in_use(vcpu)) in kvm_arch_vcpu_load()
5029 kvm_lapic_restart_hv_timer(vcpu); in kvm_arch_vcpu_load()
5033 * kvmclock on vcpu->cpu migration in kvm_arch_vcpu_load()
5035 if (!vcpu->kvm->arch.use_master_clock || vcpu->cpu == -1) in kvm_arch_vcpu_load()
5036 kvm_make_request(KVM_REQ_GLOBAL_CLOCK_UPDATE, vcpu); in kvm_arch_vcpu_load()
5037 if (vcpu->cpu != cpu) in kvm_arch_vcpu_load()
5038 kvm_make_request(KVM_REQ_MIGRATE_TIMER, vcpu); in kvm_arch_vcpu_load()
5039 vcpu->cpu = cpu; in kvm_arch_vcpu_load()
5042 kvm_make_request(KVM_REQ_STEAL_UPDATE, vcpu); in kvm_arch_vcpu_load()
5045 static void kvm_steal_time_set_preempted(struct kvm_vcpu *vcpu) in kvm_steal_time_set_preempted() argument
5047 struct gfn_to_hva_cache *ghc = &vcpu->arch.st.cache; in kvm_steal_time_set_preempted()
5051 gpa_t gpa = vcpu->arch.st.msr_val & KVM_STEAL_VALID_BITS; in kvm_steal_time_set_preempted()
5054 * The vCPU can be marked preempted if and only if the VM-Exit was on in kvm_steal_time_set_preempted()
5057 * when this is true, for example allowing the vCPU to be marked in kvm_steal_time_set_preempted()
5060 if (!vcpu->arch.at_instruction_boundary) { in kvm_steal_time_set_preempted()
5061 vcpu->stat.preemption_other++; in kvm_steal_time_set_preempted()
5065 vcpu->stat.preemption_reported++; in kvm_steal_time_set_preempted()
5066 if (!(vcpu->arch.st.msr_val & KVM_MSR_ENABLED)) in kvm_steal_time_set_preempted()
5069 if (vcpu->arch.st.preempted) in kvm_steal_time_set_preempted()
5073 if (unlikely(current->mm != vcpu->kvm->mm)) in kvm_steal_time_set_preempted()
5076 slots = kvm_memslots(vcpu->kvm); in kvm_steal_time_set_preempted()
5087 vcpu->arch.st.preempted = KVM_VCPU_PREEMPTED; in kvm_steal_time_set_preempted()
5089 mark_page_dirty_in_slot(vcpu->kvm, ghc->memslot, gpa_to_gfn(ghc->gpa)); in kvm_steal_time_set_preempted()
5092 void kvm_arch_vcpu_put(struct kvm_vcpu *vcpu) in kvm_arch_vcpu_put() argument
5096 if (vcpu->preempted) { in kvm_arch_vcpu_put()
5097 vcpu->arch.preempted_in_kernel = kvm_arch_vcpu_in_kernel(vcpu); in kvm_arch_vcpu_put()
5103 idx = srcu_read_lock(&vcpu->kvm->srcu); in kvm_arch_vcpu_put()
5104 if (kvm_xen_msr_enabled(vcpu->kvm)) in kvm_arch_vcpu_put()
5105 kvm_xen_runstate_set_preempted(vcpu); in kvm_arch_vcpu_put()
5107 kvm_steal_time_set_preempted(vcpu); in kvm_arch_vcpu_put()
5108 srcu_read_unlock(&vcpu->kvm->srcu, idx); in kvm_arch_vcpu_put()
5111 kvm_x86_call(vcpu_put)(vcpu); in kvm_arch_vcpu_put()
5112 vcpu->arch.last_host_tsc = rdtsc(); in kvm_arch_vcpu_put()
5115 static int kvm_vcpu_ioctl_get_lapic(struct kvm_vcpu *vcpu, in kvm_vcpu_ioctl_get_lapic() argument
5118 kvm_x86_call(sync_pir_to_irr)(vcpu); in kvm_vcpu_ioctl_get_lapic()
5120 return kvm_apic_get_state(vcpu, s); in kvm_vcpu_ioctl_get_lapic()
5123 static int kvm_vcpu_ioctl_set_lapic(struct kvm_vcpu *vcpu, in kvm_vcpu_ioctl_set_lapic() argument
5128 r = kvm_apic_set_state(vcpu, s); in kvm_vcpu_ioctl_set_lapic()
5131 update_cr8_intercept(vcpu); in kvm_vcpu_ioctl_set_lapic()
5136 static int kvm_cpu_accept_dm_intr(struct kvm_vcpu *vcpu) in kvm_cpu_accept_dm_intr() argument
5144 if (kvm_cpu_has_extint(vcpu)) in kvm_cpu_accept_dm_intr()
5148 return (!lapic_in_kernel(vcpu) || in kvm_cpu_accept_dm_intr()
5149 kvm_apic_accept_pic_intr(vcpu)); in kvm_cpu_accept_dm_intr()
5152 static int kvm_vcpu_ready_for_interrupt_injection(struct kvm_vcpu *vcpu) in kvm_vcpu_ready_for_interrupt_injection() argument
5161 return (kvm_arch_interrupt_allowed(vcpu) && in kvm_vcpu_ready_for_interrupt_injection()
5162 kvm_cpu_accept_dm_intr(vcpu) && in kvm_vcpu_ready_for_interrupt_injection()
5163 !kvm_event_needs_reinjection(vcpu) && in kvm_vcpu_ready_for_interrupt_injection()
5164 !kvm_is_exception_pending(vcpu)); in kvm_vcpu_ready_for_interrupt_injection()
5167 static int kvm_vcpu_ioctl_interrupt(struct kvm_vcpu *vcpu, in kvm_vcpu_ioctl_interrupt() argument
5173 if (!irqchip_in_kernel(vcpu->kvm)) { in kvm_vcpu_ioctl_interrupt()
5174 kvm_queue_interrupt(vcpu, irq->irq, false); in kvm_vcpu_ioctl_interrupt()
5175 kvm_make_request(KVM_REQ_EVENT, vcpu); in kvm_vcpu_ioctl_interrupt()
5183 if (pic_in_kernel(vcpu->kvm)) in kvm_vcpu_ioctl_interrupt()
5186 if (vcpu->arch.pending_external_vector != -1) in kvm_vcpu_ioctl_interrupt()
5189 vcpu->arch.pending_external_vector = irq->irq; in kvm_vcpu_ioctl_interrupt()
5190 kvm_make_request(KVM_REQ_EVENT, vcpu); in kvm_vcpu_ioctl_interrupt()
5194 static int kvm_vcpu_ioctl_nmi(struct kvm_vcpu *vcpu) in kvm_vcpu_ioctl_nmi() argument
5196 kvm_inject_nmi(vcpu); in kvm_vcpu_ioctl_nmi()
5201 static int vcpu_ioctl_tpr_access_reporting(struct kvm_vcpu *vcpu, in vcpu_ioctl_tpr_access_reporting() argument
5206 vcpu->arch.tpr_access_reporting = !!tac->enabled; in vcpu_ioctl_tpr_access_reporting()
5210 static int kvm_vcpu_ioctl_x86_setup_mce(struct kvm_vcpu *vcpu, in kvm_vcpu_ioctl_x86_setup_mce() argument
5222 vcpu->arch.mcg_cap = mcg_cap; in kvm_vcpu_ioctl_x86_setup_mce()
5225 vcpu->arch.mcg_ctl = ~(u64)0; in kvm_vcpu_ioctl_x86_setup_mce()
5228 vcpu->arch.mce_banks[bank*4] = ~(u64)0; in kvm_vcpu_ioctl_x86_setup_mce()
5230 vcpu->arch.mci_ctl2_banks[bank] = 0; in kvm_vcpu_ioctl_x86_setup_mce()
5233 kvm_apic_after_set_mcg_cap(vcpu); in kvm_vcpu_ioctl_x86_setup_mce()
5235 kvm_x86_call(setup_mce)(vcpu); in kvm_vcpu_ioctl_x86_setup_mce()
5257 static int kvm_vcpu_x86_set_ucna(struct kvm_vcpu *vcpu, struct kvm_x86_mce *mce, u64* banks) in kvm_vcpu_x86_set_ucna() argument
5259 u64 mcg_cap = vcpu->arch.mcg_cap; in kvm_vcpu_x86_set_ucna()
5264 vcpu->arch.mcg_status = mce->mcg_status; in kvm_vcpu_x86_set_ucna()
5267 !(vcpu->arch.mci_ctl2_banks[mce->bank] & MCI_CTL2_CMCI_EN)) in kvm_vcpu_x86_set_ucna()
5270 if (lapic_in_kernel(vcpu)) in kvm_vcpu_x86_set_ucna()
5271 kvm_apic_local_deliver(vcpu->arch.apic, APIC_LVTCMCI); in kvm_vcpu_x86_set_ucna()
5276 static int kvm_vcpu_ioctl_x86_set_mce(struct kvm_vcpu *vcpu, in kvm_vcpu_ioctl_x86_set_mce() argument
5279 u64 mcg_cap = vcpu->arch.mcg_cap; in kvm_vcpu_ioctl_x86_set_mce()
5281 u64 *banks = vcpu->arch.mce_banks; in kvm_vcpu_ioctl_x86_set_mce()
5289 return kvm_vcpu_x86_set_ucna(vcpu, mce, banks); in kvm_vcpu_ioctl_x86_set_mce()
5296 vcpu->arch.mcg_ctl != ~(u64)0) in kvm_vcpu_ioctl_x86_set_mce()
5305 if ((vcpu->arch.mcg_status & MCG_STATUS_MCIP) || in kvm_vcpu_ioctl_x86_set_mce()
5306 !kvm_is_cr4_bit_set(vcpu, X86_CR4_MCE)) { in kvm_vcpu_ioctl_x86_set_mce()
5307 kvm_make_request(KVM_REQ_TRIPLE_FAULT, vcpu); in kvm_vcpu_ioctl_x86_set_mce()
5314 vcpu->arch.mcg_status = mce->mcg_status; in kvm_vcpu_ioctl_x86_set_mce()
5316 kvm_queue_exception(vcpu, MC_VECTOR); in kvm_vcpu_ioctl_x86_set_mce()
5329 static void kvm_vcpu_ioctl_x86_get_vcpu_events(struct kvm_vcpu *vcpu, in kvm_vcpu_ioctl_x86_get_vcpu_events() argument
5334 process_nmi(vcpu); in kvm_vcpu_ioctl_x86_get_vcpu_events()
5337 if (kvm_check_request(KVM_REQ_SMI, vcpu)) in kvm_vcpu_ioctl_x86_get_vcpu_events()
5338 process_smi(vcpu); in kvm_vcpu_ioctl_x86_get_vcpu_events()
5348 if (vcpu->arch.exception_vmexit.pending && in kvm_vcpu_ioctl_x86_get_vcpu_events()
5349 !vcpu->arch.exception.pending && in kvm_vcpu_ioctl_x86_get_vcpu_events()
5350 !vcpu->arch.exception.injected) in kvm_vcpu_ioctl_x86_get_vcpu_events()
5351 ex = &vcpu->arch.exception_vmexit; in kvm_vcpu_ioctl_x86_get_vcpu_events()
5353 ex = &vcpu->arch.exception; in kvm_vcpu_ioctl_x86_get_vcpu_events()
5363 if (!vcpu->kvm->arch.exception_payload_enabled && in kvm_vcpu_ioctl_x86_get_vcpu_events()
5365 kvm_deliver_exception_payload(vcpu, ex); in kvm_vcpu_ioctl_x86_get_vcpu_events()
5383 if (!vcpu->kvm->arch.exception_payload_enabled) in kvm_vcpu_ioctl_x86_get_vcpu_events()
5393 vcpu->arch.interrupt.injected && !vcpu->arch.interrupt.soft; in kvm_vcpu_ioctl_x86_get_vcpu_events()
5394 events->interrupt.nr = vcpu->arch.interrupt.nr; in kvm_vcpu_ioctl_x86_get_vcpu_events()
5395 events->interrupt.shadow = kvm_x86_call(get_interrupt_shadow)(vcpu); in kvm_vcpu_ioctl_x86_get_vcpu_events()
5397 events->nmi.injected = vcpu->arch.nmi_injected; in kvm_vcpu_ioctl_x86_get_vcpu_events()
5398 events->nmi.pending = kvm_get_nr_pending_nmis(vcpu); in kvm_vcpu_ioctl_x86_get_vcpu_events()
5399 events->nmi.masked = kvm_x86_call(get_nmi_mask)(vcpu); in kvm_vcpu_ioctl_x86_get_vcpu_events()
5404 events->smi.smm = is_smm(vcpu); in kvm_vcpu_ioctl_x86_get_vcpu_events()
5405 events->smi.pending = vcpu->arch.smi_pending; in kvm_vcpu_ioctl_x86_get_vcpu_events()
5407 !!(vcpu->arch.hflags & HF_SMM_INSIDE_NMI_MASK); in kvm_vcpu_ioctl_x86_get_vcpu_events()
5409 events->smi.latched_init = kvm_lapic_latched_init(vcpu); in kvm_vcpu_ioctl_x86_get_vcpu_events()
5414 if (vcpu->kvm->arch.exception_payload_enabled) in kvm_vcpu_ioctl_x86_get_vcpu_events()
5416 if (vcpu->kvm->arch.triple_fault_event) { in kvm_vcpu_ioctl_x86_get_vcpu_events()
5417 events->triple_fault.pending = kvm_test_request(KVM_REQ_TRIPLE_FAULT, vcpu); in kvm_vcpu_ioctl_x86_get_vcpu_events()
5422 static int kvm_vcpu_ioctl_x86_set_vcpu_events(struct kvm_vcpu *vcpu, in kvm_vcpu_ioctl_x86_set_vcpu_events() argument
5434 if (!vcpu->kvm->arch.exception_payload_enabled) in kvm_vcpu_ioctl_x86_set_vcpu_events()
5452 vcpu->arch.mp_state == KVM_MP_STATE_INIT_RECEIVED) in kvm_vcpu_ioctl_x86_set_vcpu_events()
5455 process_nmi(vcpu); in kvm_vcpu_ioctl_x86_set_vcpu_events()
5465 vcpu->arch.exception_from_userspace = events->exception.pending; in kvm_vcpu_ioctl_x86_set_vcpu_events()
5467 vcpu->arch.exception_vmexit.pending = false; in kvm_vcpu_ioctl_x86_set_vcpu_events()
5469 vcpu->arch.exception.injected = events->exception.injected; in kvm_vcpu_ioctl_x86_set_vcpu_events()
5470 vcpu->arch.exception.pending = events->exception.pending; in kvm_vcpu_ioctl_x86_set_vcpu_events()
5471 vcpu->arch.exception.vector = events->exception.nr; in kvm_vcpu_ioctl_x86_set_vcpu_events()
5472 vcpu->arch.exception.has_error_code = events->exception.has_error_code; in kvm_vcpu_ioctl_x86_set_vcpu_events()
5473 vcpu->arch.exception.error_code = events->exception.error_code; in kvm_vcpu_ioctl_x86_set_vcpu_events()
5474 vcpu->arch.exception.has_payload = events->exception_has_payload; in kvm_vcpu_ioctl_x86_set_vcpu_events()
5475 vcpu->arch.exception.payload = events->exception_payload; in kvm_vcpu_ioctl_x86_set_vcpu_events()
5477 vcpu->arch.interrupt.injected = events->interrupt.injected; in kvm_vcpu_ioctl_x86_set_vcpu_events()
5478 vcpu->arch.interrupt.nr = events->interrupt.nr; in kvm_vcpu_ioctl_x86_set_vcpu_events()
5479 vcpu->arch.interrupt.soft = events->interrupt.soft; in kvm_vcpu_ioctl_x86_set_vcpu_events()
5481 kvm_x86_call(set_interrupt_shadow)(vcpu, in kvm_vcpu_ioctl_x86_set_vcpu_events()
5484 vcpu->arch.nmi_injected = events->nmi.injected; in kvm_vcpu_ioctl_x86_set_vcpu_events()
5486 vcpu->arch.nmi_pending = 0; in kvm_vcpu_ioctl_x86_set_vcpu_events()
5487 atomic_set(&vcpu->arch.nmi_queued, events->nmi.pending); in kvm_vcpu_ioctl_x86_set_vcpu_events()
5489 kvm_make_request(KVM_REQ_NMI, vcpu); in kvm_vcpu_ioctl_x86_set_vcpu_events()
5491 kvm_x86_call(set_nmi_mask)(vcpu, events->nmi.masked); in kvm_vcpu_ioctl_x86_set_vcpu_events()
5494 lapic_in_kernel(vcpu)) in kvm_vcpu_ioctl_x86_set_vcpu_events()
5495 vcpu->arch.apic->sipi_vector = events->sipi_vector; in kvm_vcpu_ioctl_x86_set_vcpu_events()
5499 if (!!(vcpu->arch.hflags & HF_SMM_MASK) != events->smi.smm) { in kvm_vcpu_ioctl_x86_set_vcpu_events()
5500 kvm_leave_nested(vcpu); in kvm_vcpu_ioctl_x86_set_vcpu_events()
5501 kvm_smm_changed(vcpu, events->smi.smm); in kvm_vcpu_ioctl_x86_set_vcpu_events()
5504 vcpu->arch.smi_pending = events->smi.pending; in kvm_vcpu_ioctl_x86_set_vcpu_events()
5508 vcpu->arch.hflags |= HF_SMM_INSIDE_NMI_MASK; in kvm_vcpu_ioctl_x86_set_vcpu_events()
5510 vcpu->arch.hflags &= ~HF_SMM_INSIDE_NMI_MASK; in kvm_vcpu_ioctl_x86_set_vcpu_events()
5519 if (lapic_in_kernel(vcpu)) { in kvm_vcpu_ioctl_x86_set_vcpu_events()
5521 set_bit(KVM_APIC_INIT, &vcpu->arch.apic->pending_events); in kvm_vcpu_ioctl_x86_set_vcpu_events()
5523 clear_bit(KVM_APIC_INIT, &vcpu->arch.apic->pending_events); in kvm_vcpu_ioctl_x86_set_vcpu_events()
5528 if (!vcpu->kvm->arch.triple_fault_event) in kvm_vcpu_ioctl_x86_set_vcpu_events()
5531 kvm_make_request(KVM_REQ_TRIPLE_FAULT, vcpu); in kvm_vcpu_ioctl_x86_set_vcpu_events()
5533 kvm_clear_request(KVM_REQ_TRIPLE_FAULT, vcpu); in kvm_vcpu_ioctl_x86_set_vcpu_events()
5536 kvm_make_request(KVM_REQ_EVENT, vcpu); in kvm_vcpu_ioctl_x86_set_vcpu_events()
5541 static int kvm_vcpu_ioctl_x86_get_debugregs(struct kvm_vcpu *vcpu, in kvm_vcpu_ioctl_x86_get_debugregs() argument
5546 if (vcpu->kvm->arch.has_protected_state && in kvm_vcpu_ioctl_x86_get_debugregs()
5547 vcpu->arch.guest_state_protected) in kvm_vcpu_ioctl_x86_get_debugregs()
5552 BUILD_BUG_ON(ARRAY_SIZE(vcpu->arch.db) != ARRAY_SIZE(dbgregs->db)); in kvm_vcpu_ioctl_x86_get_debugregs()
5553 for (i = 0; i < ARRAY_SIZE(vcpu->arch.db); i++) in kvm_vcpu_ioctl_x86_get_debugregs()
5554 dbgregs->db[i] = vcpu->arch.db[i]; in kvm_vcpu_ioctl_x86_get_debugregs()
5556 dbgregs->dr6 = vcpu->arch.dr6; in kvm_vcpu_ioctl_x86_get_debugregs()
5557 dbgregs->dr7 = vcpu->arch.dr7; in kvm_vcpu_ioctl_x86_get_debugregs()
5561 static int kvm_vcpu_ioctl_x86_set_debugregs(struct kvm_vcpu *vcpu, in kvm_vcpu_ioctl_x86_set_debugregs() argument
5566 if (vcpu->kvm->arch.has_protected_state && in kvm_vcpu_ioctl_x86_set_debugregs()
5567 vcpu->arch.guest_state_protected) in kvm_vcpu_ioctl_x86_set_debugregs()
5578 for (i = 0; i < ARRAY_SIZE(vcpu->arch.db); i++) in kvm_vcpu_ioctl_x86_set_debugregs()
5579 vcpu->arch.db[i] = dbgregs->db[i]; in kvm_vcpu_ioctl_x86_set_debugregs()
5581 kvm_update_dr0123(vcpu); in kvm_vcpu_ioctl_x86_set_debugregs()
5582 vcpu->arch.dr6 = dbgregs->dr6; in kvm_vcpu_ioctl_x86_set_debugregs()
5583 vcpu->arch.dr7 = dbgregs->dr7; in kvm_vcpu_ioctl_x86_set_debugregs()
5584 kvm_update_dr7(vcpu); in kvm_vcpu_ioctl_x86_set_debugregs()
5590 static int kvm_vcpu_ioctl_x86_get_xsave2(struct kvm_vcpu *vcpu, in kvm_vcpu_ioctl_x86_get_xsave2() argument
5605 u64 supported_xcr0 = vcpu->arch.guest_supported_xcr0 | in kvm_vcpu_ioctl_x86_get_xsave2()
5608 if (fpstate_is_confidential(&vcpu->arch.guest_fpu)) in kvm_vcpu_ioctl_x86_get_xsave2()
5609 return vcpu->kvm->arch.has_protected_state ? -EINVAL : 0; in kvm_vcpu_ioctl_x86_get_xsave2()
5611 fpu_copy_guest_fpstate_to_uabi(&vcpu->arch.guest_fpu, state, size, in kvm_vcpu_ioctl_x86_get_xsave2()
5612 supported_xcr0, vcpu->arch.pkru); in kvm_vcpu_ioctl_x86_get_xsave2()
5616 static int kvm_vcpu_ioctl_x86_get_xsave(struct kvm_vcpu *vcpu, in kvm_vcpu_ioctl_x86_get_xsave() argument
5619 return kvm_vcpu_ioctl_x86_get_xsave2(vcpu, (void *)guest_xsave->region, in kvm_vcpu_ioctl_x86_get_xsave()
5623 static int kvm_vcpu_ioctl_x86_set_xsave(struct kvm_vcpu *vcpu, in kvm_vcpu_ioctl_x86_set_xsave() argument
5626 if (fpstate_is_confidential(&vcpu->arch.guest_fpu)) in kvm_vcpu_ioctl_x86_set_xsave()
5627 return vcpu->kvm->arch.has_protected_state ? -EINVAL : 0; in kvm_vcpu_ioctl_x86_set_xsave()
5629 return fpu_copy_uabi_to_guest_fpstate(&vcpu->arch.guest_fpu, in kvm_vcpu_ioctl_x86_set_xsave()
5632 &vcpu->arch.pkru); in kvm_vcpu_ioctl_x86_set_xsave()
5635 static int kvm_vcpu_ioctl_x86_get_xcrs(struct kvm_vcpu *vcpu, in kvm_vcpu_ioctl_x86_get_xcrs() argument
5638 if (vcpu->kvm->arch.has_protected_state && in kvm_vcpu_ioctl_x86_get_xcrs()
5639 vcpu->arch.guest_state_protected) in kvm_vcpu_ioctl_x86_get_xcrs()
5650 guest_xcrs->xcrs[0].value = vcpu->arch.xcr0; in kvm_vcpu_ioctl_x86_get_xcrs()
5654 static int kvm_vcpu_ioctl_x86_set_xcrs(struct kvm_vcpu *vcpu, in kvm_vcpu_ioctl_x86_set_xcrs() argument
5659 if (vcpu->kvm->arch.has_protected_state && in kvm_vcpu_ioctl_x86_set_xcrs()
5660 vcpu->arch.guest_state_protected) in kvm_vcpu_ioctl_x86_set_xcrs()
5672 r = __kvm_set_xcr(vcpu, XCR_XFEATURE_ENABLED_MASK, in kvm_vcpu_ioctl_x86_set_xcrs()
5687 static int kvm_set_guest_paused(struct kvm_vcpu *vcpu) in kvm_set_guest_paused() argument
5689 if (!vcpu->arch.pv_time.active) in kvm_set_guest_paused()
5691 vcpu->arch.pvclock_set_guest_stopped_request = true; in kvm_set_guest_paused()
5692 kvm_make_request(KVM_REQ_CLOCK_UPDATE, vcpu); in kvm_set_guest_paused()
5696 static int kvm_arch_tsc_has_attr(struct kvm_vcpu *vcpu, in kvm_arch_tsc_has_attr() argument
5712 static int kvm_arch_tsc_get_attr(struct kvm_vcpu *vcpu, in kvm_arch_tsc_get_attr() argument
5721 if (put_user(vcpu->arch.l1_tsc_offset, uaddr)) in kvm_arch_tsc_get_attr()
5732 static int kvm_arch_tsc_set_attr(struct kvm_vcpu *vcpu, in kvm_arch_tsc_set_attr() argument
5736 struct kvm *kvm = vcpu->kvm; in kvm_arch_tsc_set_attr()
5751 matched = (vcpu->arch.virtual_tsc_khz && in kvm_arch_tsc_set_attr()
5752 kvm->arch.last_tsc_khz == vcpu->arch.virtual_tsc_khz && in kvm_arch_tsc_set_attr()
5755 tsc = kvm_scale_tsc(rdtsc(), vcpu->arch.l1_tsc_scaling_ratio) + offset; in kvm_arch_tsc_set_attr()
5759 __kvm_synchronize_tsc(vcpu, offset, tsc, ns, matched); in kvm_arch_tsc_set_attr()
5772 static int kvm_vcpu_ioctl_device_attr(struct kvm_vcpu *vcpu, in kvm_vcpu_ioctl_device_attr() argument
5787 r = kvm_arch_tsc_has_attr(vcpu, &attr); in kvm_vcpu_ioctl_device_attr()
5790 r = kvm_arch_tsc_get_attr(vcpu, &attr); in kvm_vcpu_ioctl_device_attr()
5793 r = kvm_arch_tsc_set_attr(vcpu, &attr); in kvm_vcpu_ioctl_device_attr()
5800 static int kvm_vcpu_ioctl_enable_cap(struct kvm_vcpu *vcpu, in kvm_vcpu_ioctl_enable_cap() argument
5814 if (!irqchip_in_kernel(vcpu->kvm)) in kvm_vcpu_ioctl_enable_cap()
5816 return kvm_hv_activate_synic(vcpu, cap->cap == in kvm_vcpu_ioctl_enable_cap()
5826 r = kvm_x86_ops.nested_ops->enable_evmcs(vcpu, &vmcs_version); in kvm_vcpu_ioctl_enable_cap()
5839 return kvm_x86_call(enable_l2_tlb_flush)(vcpu); in kvm_vcpu_ioctl_enable_cap()
5842 return kvm_hv_set_enforce_cpuid(vcpu, cap->args[0]); in kvm_vcpu_ioctl_enable_cap()
5846 vcpu->arch.pv_cpuid.enforce = cap->args[0]; in kvm_vcpu_ioctl_enable_cap()
5847 if (vcpu->arch.pv_cpuid.enforce) in kvm_vcpu_ioctl_enable_cap()
5848 kvm_update_pv_runtime(vcpu); in kvm_vcpu_ioctl_enable_cap()
5859 struct kvm_vcpu *vcpu = filp->private_data; in kvm_arch_vcpu_ioctl() local
5870 vcpu_load(vcpu); in kvm_arch_vcpu_ioctl()
5876 if (!lapic_in_kernel(vcpu)) in kvm_arch_vcpu_ioctl()
5883 r = kvm_vcpu_ioctl_get_lapic(vcpu, u.lapic); in kvm_arch_vcpu_ioctl()
5894 if (!lapic_in_kernel(vcpu)) in kvm_arch_vcpu_ioctl()
5902 r = kvm_vcpu_ioctl_set_lapic(vcpu, u.lapic); in kvm_arch_vcpu_ioctl()
5911 r = kvm_vcpu_ioctl_interrupt(vcpu, &irq); in kvm_arch_vcpu_ioctl()
5915 r = kvm_vcpu_ioctl_nmi(vcpu); in kvm_arch_vcpu_ioctl()
5919 r = kvm_inject_smi(vcpu); in kvm_arch_vcpu_ioctl()
5929 r = kvm_vcpu_ioctl_set_cpuid(vcpu, &cpuid, cpuid_arg->entries); in kvm_arch_vcpu_ioctl()
5939 r = kvm_vcpu_ioctl_set_cpuid2(vcpu, &cpuid, in kvm_arch_vcpu_ioctl()
5950 r = kvm_vcpu_ioctl_get_cpuid2(vcpu, &cpuid, in kvm_arch_vcpu_ioctl()
5961 int idx = srcu_read_lock(&vcpu->kvm->srcu); in kvm_arch_vcpu_ioctl()
5962 r = msr_io(vcpu, argp, do_get_msr, 1); in kvm_arch_vcpu_ioctl()
5963 srcu_read_unlock(&vcpu->kvm->srcu, idx); in kvm_arch_vcpu_ioctl()
5967 int idx = srcu_read_lock(&vcpu->kvm->srcu); in kvm_arch_vcpu_ioctl()
5968 r = msr_io(vcpu, argp, do_set_msr, 0); in kvm_arch_vcpu_ioctl()
5969 srcu_read_unlock(&vcpu->kvm->srcu, idx); in kvm_arch_vcpu_ioctl()
5978 r = vcpu_ioctl_tpr_access_reporting(vcpu, &tac); in kvm_arch_vcpu_ioctl()
5992 if (!lapic_in_kernel(vcpu)) in kvm_arch_vcpu_ioctl()
5997 idx = srcu_read_lock(&vcpu->kvm->srcu); in kvm_arch_vcpu_ioctl()
5998 r = kvm_lapic_set_vapic_addr(vcpu, va.vapic_addr); in kvm_arch_vcpu_ioctl()
5999 srcu_read_unlock(&vcpu->kvm->srcu, idx); in kvm_arch_vcpu_ioctl()
6008 r = kvm_vcpu_ioctl_x86_setup_mce(vcpu, mcg_cap); in kvm_arch_vcpu_ioctl()
6017 r = kvm_vcpu_ioctl_x86_set_mce(vcpu, &mce); in kvm_arch_vcpu_ioctl()
6023 kvm_vcpu_ioctl_x86_get_vcpu_events(vcpu, &events); in kvm_arch_vcpu_ioctl()
6038 kvm_vcpu_srcu_read_lock(vcpu); in kvm_arch_vcpu_ioctl()
6039 r = kvm_vcpu_ioctl_x86_set_vcpu_events(vcpu, &events); in kvm_arch_vcpu_ioctl()
6040 kvm_vcpu_srcu_read_unlock(vcpu); in kvm_arch_vcpu_ioctl()
6046 r = kvm_vcpu_ioctl_x86_get_debugregs(vcpu, &dbgregs); in kvm_arch_vcpu_ioctl()
6065 r = kvm_vcpu_ioctl_x86_set_debugregs(vcpu, &dbgregs); in kvm_arch_vcpu_ioctl()
6070 if (vcpu->arch.guest_fpu.uabi_size > sizeof(struct kvm_xsave)) in kvm_arch_vcpu_ioctl()
6078 r = kvm_vcpu_ioctl_x86_get_xsave(vcpu, u.xsave); in kvm_arch_vcpu_ioctl()
6089 int size = vcpu->arch.guest_fpu.uabi_size; in kvm_arch_vcpu_ioctl()
6097 r = kvm_vcpu_ioctl_x86_set_xsave(vcpu, u.xsave); in kvm_arch_vcpu_ioctl()
6102 int size = vcpu->arch.guest_fpu.uabi_size; in kvm_arch_vcpu_ioctl()
6109 r = kvm_vcpu_ioctl_x86_get_xsave2(vcpu, u.buffer, size); in kvm_arch_vcpu_ioctl()
6127 r = kvm_vcpu_ioctl_x86_get_xcrs(vcpu, u.xcrs); in kvm_arch_vcpu_ioctl()
6145 r = kvm_vcpu_ioctl_x86_set_xcrs(vcpu, u.xcrs); in kvm_arch_vcpu_ioctl()
6161 if (!kvm_set_tsc_khz(vcpu, user_tsc_khz)) in kvm_arch_vcpu_ioctl()
6167 r = vcpu->arch.virtual_tsc_khz; in kvm_arch_vcpu_ioctl()
6171 r = kvm_set_guest_paused(vcpu); in kvm_arch_vcpu_ioctl()
6180 r = kvm_vcpu_ioctl_enable_cap(vcpu, &cap); in kvm_arch_vcpu_ioctl()
6196 r = kvm_x86_ops.nested_ops->get_state(vcpu, user_kvm_nested_state, in kvm_arch_vcpu_ioctl()
6240 idx = srcu_read_lock(&vcpu->kvm->srcu); in kvm_arch_vcpu_ioctl()
6241 r = kvm_x86_ops.nested_ops->set_state(vcpu, user_kvm_nested_state, &kvm_state); in kvm_arch_vcpu_ioctl()
6242 srcu_read_unlock(&vcpu->kvm->srcu, idx); in kvm_arch_vcpu_ioctl()
6247 r = kvm_ioctl_get_supported_hv_cpuid(vcpu, argp); in kvm_arch_vcpu_ioctl()
6257 r = kvm_xen_vcpu_get_attr(vcpu, &xva); in kvm_arch_vcpu_ioctl()
6268 r = kvm_xen_vcpu_set_attr(vcpu, &xva); in kvm_arch_vcpu_ioctl()
6274 if (vcpu->kvm->arch.has_protected_state && in kvm_arch_vcpu_ioctl()
6275 vcpu->arch.guest_state_protected) in kvm_arch_vcpu_ioctl()
6282 __get_sregs2(vcpu, u.sregs2); in kvm_arch_vcpu_ioctl()
6291 if (vcpu->kvm->arch.has_protected_state && in kvm_arch_vcpu_ioctl()
6292 vcpu->arch.guest_state_protected) in kvm_arch_vcpu_ioctl()
6301 r = __set_sregs2(vcpu, u.sregs2); in kvm_arch_vcpu_ioctl()
6307 r = kvm_vcpu_ioctl_device_attr(vcpu, ioctl, argp); in kvm_arch_vcpu_ioctl()
6315 vcpu_put(vcpu); in kvm_arch_vcpu_ioctl()
6319 vm_fault_t kvm_arch_vcpu_fault(struct kvm_vcpu *vcpu, struct vm_fault *vmf) in kvm_arch_vcpu_fault() argument
6493 struct kvm_vcpu *vcpu; in kvm_arch_sync_dirty_log() local
6499 kvm_for_each_vcpu(i, vcpu, kvm) in kvm_arch_sync_dirty_log()
6500 kvm_vcpu_kick(vcpu); in kvm_arch_sync_dirty_log()
6937 struct kvm_vcpu *vcpu; in kvm_arch_suspend_notifier() local
6942 kvm_for_each_vcpu(i, vcpu, kvm) { in kvm_arch_suspend_notifier()
6943 if (!vcpu->arch.pv_time.active) in kvm_arch_suspend_notifier()
6946 ret = kvm_set_guest_paused(vcpu); in kvm_arch_suspend_notifier()
6948 kvm_err("Failed to pause guest VCPU%d: %d\n", in kvm_arch_suspend_notifier()
6949 vcpu->vcpu_id, ret); in kvm_arch_suspend_notifier()
7509 static int vcpu_mmio_write(struct kvm_vcpu *vcpu, gpa_t addr, int len, in vcpu_mmio_write() argument
7517 if (!(lapic_in_kernel(vcpu) && in vcpu_mmio_write()
7518 !kvm_iodevice_write(vcpu, &vcpu->arch.apic->dev, addr, n, v)) in vcpu_mmio_write()
7519 && kvm_io_bus_write(vcpu, KVM_MMIO_BUS, addr, n, v)) in vcpu_mmio_write()
7530 static int vcpu_mmio_read(struct kvm_vcpu *vcpu, gpa_t addr, int len, void *v) in vcpu_mmio_read() argument
7537 if (!(lapic_in_kernel(vcpu) && in vcpu_mmio_read()
7538 !kvm_iodevice_read(vcpu, &vcpu->arch.apic->dev, in vcpu_mmio_read()
7540 && kvm_io_bus_read(vcpu, KVM_MMIO_BUS, addr, n, v)) in vcpu_mmio_read()
7552 void kvm_set_segment(struct kvm_vcpu *vcpu, in kvm_set_segment() argument
7555 kvm_x86_call(set_segment)(vcpu, var, seg); in kvm_set_segment()
7558 void kvm_get_segment(struct kvm_vcpu *vcpu, in kvm_get_segment() argument
7561 kvm_x86_call(get_segment)(vcpu, var, seg); in kvm_get_segment()
7564 gpa_t translate_nested_gpa(struct kvm_vcpu *vcpu, gpa_t gpa, u64 access, in translate_nested_gpa() argument
7567 struct kvm_mmu *mmu = vcpu->arch.mmu; in translate_nested_gpa()
7570 BUG_ON(!mmu_is_nested(vcpu)); in translate_nested_gpa()
7574 t_gpa = mmu->gva_to_gpa(vcpu, mmu, gpa, access, exception); in translate_nested_gpa()
7579 gpa_t kvm_mmu_gva_to_gpa_read(struct kvm_vcpu *vcpu, gva_t gva, in kvm_mmu_gva_to_gpa_read() argument
7582 struct kvm_mmu *mmu = vcpu->arch.walk_mmu; in kvm_mmu_gva_to_gpa_read()
7584 u64 access = (kvm_x86_call(get_cpl)(vcpu) == 3) ? PFERR_USER_MASK : 0; in kvm_mmu_gva_to_gpa_read()
7585 return mmu->gva_to_gpa(vcpu, mmu, gva, access, exception); in kvm_mmu_gva_to_gpa_read()
7589 gpa_t kvm_mmu_gva_to_gpa_write(struct kvm_vcpu *vcpu, gva_t gva, in kvm_mmu_gva_to_gpa_write() argument
7592 struct kvm_mmu *mmu = vcpu->arch.walk_mmu; in kvm_mmu_gva_to_gpa_write()
7594 u64 access = (kvm_x86_call(get_cpl)(vcpu) == 3) ? PFERR_USER_MASK : 0; in kvm_mmu_gva_to_gpa_write()
7596 return mmu->gva_to_gpa(vcpu, mmu, gva, access, exception); in kvm_mmu_gva_to_gpa_write()
7601 gpa_t kvm_mmu_gva_to_gpa_system(struct kvm_vcpu *vcpu, gva_t gva, in kvm_mmu_gva_to_gpa_system() argument
7604 struct kvm_mmu *mmu = vcpu->arch.walk_mmu; in kvm_mmu_gva_to_gpa_system()
7606 return mmu->gva_to_gpa(vcpu, mmu, gva, 0, exception); in kvm_mmu_gva_to_gpa_system()
7610 struct kvm_vcpu *vcpu, u64 access, in kvm_read_guest_virt_helper() argument
7613 struct kvm_mmu *mmu = vcpu->arch.walk_mmu; in kvm_read_guest_virt_helper()
7618 gpa_t gpa = mmu->gva_to_gpa(vcpu, mmu, addr, access, exception); in kvm_read_guest_virt_helper()
7625 ret = kvm_vcpu_read_guest_page(vcpu, gpa >> PAGE_SHIFT, data, in kvm_read_guest_virt_helper()
7645 struct kvm_vcpu *vcpu = emul_to_vcpu(ctxt); in kvm_fetch_guest_virt() local
7646 struct kvm_mmu *mmu = vcpu->arch.walk_mmu; in kvm_fetch_guest_virt()
7647 u64 access = (kvm_x86_call(get_cpl)(vcpu) == 3) ? PFERR_USER_MASK : 0; in kvm_fetch_guest_virt()
7652 gpa_t gpa = mmu->gva_to_gpa(vcpu, mmu, addr, access|PFERR_FETCH_MASK, in kvm_fetch_guest_virt()
7660 ret = kvm_vcpu_read_guest_page(vcpu, gpa >> PAGE_SHIFT, val, in kvm_fetch_guest_virt()
7668 int kvm_read_guest_virt(struct kvm_vcpu *vcpu, in kvm_read_guest_virt() argument
7672 u64 access = (kvm_x86_call(get_cpl)(vcpu) == 3) ? PFERR_USER_MASK : 0; in kvm_read_guest_virt()
7681 return kvm_read_guest_virt_helper(addr, val, bytes, vcpu, access, in kvm_read_guest_virt()
7690 struct kvm_vcpu *vcpu = emul_to_vcpu(ctxt); in emulator_read_std() local
7695 else if (kvm_x86_call(get_cpl)(vcpu) == 3) in emulator_read_std()
7698 return kvm_read_guest_virt_helper(addr, val, bytes, vcpu, access, exception); in emulator_read_std()
7702 struct kvm_vcpu *vcpu, u64 access, in kvm_write_guest_virt_helper() argument
7705 struct kvm_mmu *mmu = vcpu->arch.walk_mmu; in kvm_write_guest_virt_helper()
7710 gpa_t gpa = mmu->gva_to_gpa(vcpu, mmu, addr, access, exception); in kvm_write_guest_virt_helper()
7717 ret = kvm_vcpu_write_guest(vcpu, gpa, data, towrite); in kvm_write_guest_virt_helper()
7735 struct kvm_vcpu *vcpu = emul_to_vcpu(ctxt); in emulator_write_std() local
7740 else if (kvm_x86_call(get_cpl)(vcpu) == 3) in emulator_write_std()
7743 return kvm_write_guest_virt_helper(addr, val, bytes, vcpu, in emulator_write_std()
7747 int kvm_write_guest_virt_system(struct kvm_vcpu *vcpu, gva_t addr, void *val, in kvm_write_guest_virt_system() argument
7751 vcpu->arch.l1tf_flush_l1d = true; in kvm_write_guest_virt_system()
7753 return kvm_write_guest_virt_helper(addr, val, bytes, vcpu, in kvm_write_guest_virt_system()
7758 static int kvm_check_emulate_insn(struct kvm_vcpu *vcpu, int emul_type, in kvm_check_emulate_insn() argument
7761 return kvm_x86_call(check_emulate_instruction)(vcpu, emul_type, in kvm_check_emulate_insn()
7765 int handle_ud(struct kvm_vcpu *vcpu) in handle_ud() argument
7774 r = kvm_check_emulate_insn(vcpu, emul_type, NULL, 0); in handle_ud()
7779 kvm_read_guest_virt(vcpu, kvm_get_linear_rip(vcpu), in handle_ud()
7783 kvm_set_rflags(vcpu, kvm_get_rflags(vcpu) & ~X86_EFLAGS_RF); in handle_ud()
7784 kvm_rip_write(vcpu, kvm_rip_read(vcpu) + sizeof(sig)); in handle_ud()
7788 return kvm_emulate_instruction(vcpu, emul_type); in handle_ud()
7792 static int vcpu_is_mmio_gpa(struct kvm_vcpu *vcpu, unsigned long gva, in vcpu_is_mmio_gpa() argument
7799 if (vcpu_match_mmio_gpa(vcpu, gpa)) { in vcpu_is_mmio_gpa()
7807 static int vcpu_mmio_gva_to_gpa(struct kvm_vcpu *vcpu, unsigned long gva, in vcpu_mmio_gva_to_gpa() argument
7811 struct kvm_mmu *mmu = vcpu->arch.walk_mmu; in vcpu_mmio_gva_to_gpa()
7812 u64 access = ((kvm_x86_call(get_cpl)(vcpu) == 3) ? PFERR_USER_MASK : 0) in vcpu_mmio_gva_to_gpa()
7820 if (vcpu_match_mmio_gva(vcpu, gva) && (!is_paging(vcpu) || in vcpu_mmio_gva_to_gpa()
7821 !permission_fault(vcpu, vcpu->arch.walk_mmu, in vcpu_mmio_gva_to_gpa()
7822 vcpu->arch.mmio_access, 0, access))) { in vcpu_mmio_gva_to_gpa()
7823 *gpa = vcpu->arch.mmio_gfn << PAGE_SHIFT | in vcpu_mmio_gva_to_gpa()
7829 *gpa = mmu->gva_to_gpa(vcpu, mmu, gva, access, exception); in vcpu_mmio_gva_to_gpa()
7834 return vcpu_is_mmio_gpa(vcpu, gva, *gpa, write); in vcpu_mmio_gva_to_gpa()
7837 int emulator_write_phys(struct kvm_vcpu *vcpu, gpa_t gpa, in emulator_write_phys() argument
7842 ret = kvm_vcpu_write_guest(vcpu, gpa, val, bytes); in emulator_write_phys()
7845 kvm_page_track_write(vcpu, gpa, val, bytes); in emulator_write_phys()
7850 int (*read_write_prepare)(struct kvm_vcpu *vcpu, void *val,
7852 int (*read_write_emulate)(struct kvm_vcpu *vcpu, gpa_t gpa,
7854 int (*read_write_mmio)(struct kvm_vcpu *vcpu, gpa_t gpa,
7856 int (*read_write_exit_mmio)(struct kvm_vcpu *vcpu, gpa_t gpa,
7861 static int read_prepare(struct kvm_vcpu *vcpu, void *val, int bytes) in read_prepare() argument
7863 if (vcpu->mmio_read_completed) { in read_prepare()
7865 vcpu->mmio_fragments[0].gpa, val); in read_prepare()
7866 vcpu->mmio_read_completed = 0; in read_prepare()
7873 static int read_emulate(struct kvm_vcpu *vcpu, gpa_t gpa, in read_emulate() argument
7876 return !kvm_vcpu_read_guest(vcpu, gpa, val, bytes); in read_emulate()
7879 static int write_emulate(struct kvm_vcpu *vcpu, gpa_t gpa, in write_emulate() argument
7882 return emulator_write_phys(vcpu, gpa, val, bytes); in write_emulate()
7885 static int write_mmio(struct kvm_vcpu *vcpu, gpa_t gpa, int bytes, void *val) in write_mmio() argument
7888 return vcpu_mmio_write(vcpu, gpa, bytes, val); in write_mmio()
7891 static int read_exit_mmio(struct kvm_vcpu *vcpu, gpa_t gpa, in read_exit_mmio() argument
7898 static int write_exit_mmio(struct kvm_vcpu *vcpu, gpa_t gpa, in write_exit_mmio() argument
7901 struct kvm_mmio_fragment *frag = &vcpu->mmio_fragments[0]; in write_exit_mmio()
7903 memcpy(vcpu->run->mmio.data, frag->data, min(8u, frag->len)); in write_exit_mmio()
7924 struct kvm_vcpu *vcpu, in emulator_read_write_onepage() argument
7931 struct x86_emulate_ctxt *ctxt = vcpu->arch.emulate_ctxt; in emulator_read_write_onepage()
7943 ret = vcpu_is_mmio_gpa(vcpu, addr, gpa, write); in emulator_read_write_onepage()
7945 ret = vcpu_mmio_gva_to_gpa(vcpu, addr, &gpa, exception, write); in emulator_read_write_onepage()
7950 if (!ret && ops->read_write_emulate(vcpu, gpa, val, bytes)) in emulator_read_write_onepage()
7956 handled = ops->read_write_mmio(vcpu, gpa, bytes, val); in emulator_read_write_onepage()
7964 WARN_ON(vcpu->mmio_nr_fragments >= KVM_MAX_MMIO_FRAGMENTS); in emulator_read_write_onepage()
7965 frag = &vcpu->mmio_fragments[vcpu->mmio_nr_fragments++]; in emulator_read_write_onepage()
7978 struct kvm_vcpu *vcpu = emul_to_vcpu(ctxt); in emulator_read_write() local
7983 ops->read_write_prepare(vcpu, val, bytes)) in emulator_read_write()
7986 vcpu->mmio_nr_fragments = 0; in emulator_read_write()
7994 vcpu, ops); in emulator_read_write()
8006 vcpu, ops); in emulator_read_write()
8010 if (!vcpu->mmio_nr_fragments) in emulator_read_write()
8013 gpa = vcpu->mmio_fragments[0].gpa; in emulator_read_write()
8015 vcpu->mmio_needed = 1; in emulator_read_write()
8016 vcpu->mmio_cur_fragment = 0; in emulator_read_write()
8018 vcpu->run->mmio.len = min(8u, vcpu->mmio_fragments[0].len); in emulator_read_write()
8019 vcpu->run->mmio.is_write = vcpu->mmio_is_write = ops->write; in emulator_read_write()
8020 vcpu->run->exit_reason = KVM_EXIT_MMIO; in emulator_read_write()
8021 vcpu->run->mmio.phys_addr = gpa; in emulator_read_write()
8023 return ops->read_write_exit_mmio(vcpu, gpa, val, bytes); in emulator_read_write()
8056 struct kvm_vcpu *vcpu = emul_to_vcpu(ctxt); in emulator_cmpxchg_emulated() local
8066 gpa = kvm_mmu_gva_to_gpa_write(vcpu, addr, NULL); in emulator_cmpxchg_emulated()
8084 hva = kvm_vcpu_gfn_to_hva(vcpu, gpa_to_gfn(gpa)); in emulator_cmpxchg_emulated()
8117 kvm_vcpu_mark_page_dirty(vcpu, gpa_to_gfn(gpa)); in emulator_cmpxchg_emulated()
8122 kvm_page_track_write(vcpu, gpa, new, bytes); in emulator_cmpxchg_emulated()
8132 static int emulator_pio_in_out(struct kvm_vcpu *vcpu, int size, in emulator_pio_in_out() argument
8139 WARN_ON_ONCE(vcpu->arch.pio.count); in emulator_pio_in_out()
8142 r = kvm_io_bus_read(vcpu, KVM_PIO_BUS, port, size, data); in emulator_pio_in_out()
8144 r = kvm_io_bus_write(vcpu, KVM_PIO_BUS, port, size, data); in emulator_pio_in_out()
8164 vcpu->arch.pio.port = port; in emulator_pio_in_out()
8165 vcpu->arch.pio.in = in; in emulator_pio_in_out()
8166 vcpu->arch.pio.count = count; in emulator_pio_in_out()
8167 vcpu->arch.pio.size = size; in emulator_pio_in_out()
8170 memset(vcpu->arch.pio_data, 0, size * count); in emulator_pio_in_out()
8172 memcpy(vcpu->arch.pio_data, data, size * count); in emulator_pio_in_out()
8174 vcpu->run->exit_reason = KVM_EXIT_IO; in emulator_pio_in_out()
8175 vcpu->run->io.direction = in ? KVM_EXIT_IO_IN : KVM_EXIT_IO_OUT; in emulator_pio_in_out()
8176 vcpu->run->io.size = size; in emulator_pio_in_out()
8177 vcpu->run->io.data_offset = KVM_PIO_PAGE_OFFSET * PAGE_SIZE; in emulator_pio_in_out()
8178 vcpu->run->io.count = count; in emulator_pio_in_out()
8179 vcpu->run->io.port = port; in emulator_pio_in_out()
8183 static int emulator_pio_in(struct kvm_vcpu *vcpu, int size, in emulator_pio_in() argument
8186 int r = emulator_pio_in_out(vcpu, size, port, val, count, true); in emulator_pio_in()
8193 static void complete_emulator_pio_in(struct kvm_vcpu *vcpu, void *val) in complete_emulator_pio_in() argument
8195 int size = vcpu->arch.pio.size; in complete_emulator_pio_in()
8196 unsigned int count = vcpu->arch.pio.count; in complete_emulator_pio_in()
8197 memcpy(val, vcpu->arch.pio_data, size * count); in complete_emulator_pio_in()
8198 trace_kvm_pio(KVM_PIO_IN, vcpu->arch.pio.port, size, count, vcpu->arch.pio_data); in complete_emulator_pio_in()
8199 vcpu->arch.pio.count = 0; in complete_emulator_pio_in()
8206 struct kvm_vcpu *vcpu = emul_to_vcpu(ctxt); in emulator_pio_in_emulated() local
8207 if (vcpu->arch.pio.count) { in emulator_pio_in_emulated()
8211 * can modify ECX before rerunning the vCPU. Ignore any such in emulator_pio_in_emulated()
8215 complete_emulator_pio_in(vcpu, val); in emulator_pio_in_emulated()
8219 return emulator_pio_in(vcpu, size, port, val, count); in emulator_pio_in_emulated()
8222 static int emulator_pio_out(struct kvm_vcpu *vcpu, int size, in emulator_pio_out() argument
8227 return emulator_pio_in_out(vcpu, size, port, (void *)val, count, false); in emulator_pio_out()
8237 static unsigned long get_segment_base(struct kvm_vcpu *vcpu, int seg) in get_segment_base() argument
8239 return kvm_x86_call(get_segment_base)(vcpu, seg); in get_segment_base()
8247 static int kvm_emulate_wbinvd_noskip(struct kvm_vcpu *vcpu) in kvm_emulate_wbinvd_noskip() argument
8249 if (!need_emulate_wbinvd(vcpu)) in kvm_emulate_wbinvd_noskip()
8255 cpumask_set_cpu(cpu, vcpu->arch.wbinvd_dirty_mask); in kvm_emulate_wbinvd_noskip()
8256 on_each_cpu_mask(vcpu->arch.wbinvd_dirty_mask, in kvm_emulate_wbinvd_noskip()
8259 cpumask_clear(vcpu->arch.wbinvd_dirty_mask); in kvm_emulate_wbinvd_noskip()
8265 int kvm_emulate_wbinvd(struct kvm_vcpu *vcpu) in kvm_emulate_wbinvd() argument
8267 kvm_emulate_wbinvd_noskip(vcpu); in kvm_emulate_wbinvd()
8268 return kvm_skip_emulated_instruction(vcpu); in kvm_emulate_wbinvd()
8298 struct kvm_vcpu *vcpu = emul_to_vcpu(ctxt); in emulator_get_cr() local
8303 value = kvm_read_cr0(vcpu); in emulator_get_cr()
8306 value = vcpu->arch.cr2; in emulator_get_cr()
8309 value = kvm_read_cr3(vcpu); in emulator_get_cr()
8312 value = kvm_read_cr4(vcpu); in emulator_get_cr()
8315 value = kvm_get_cr8(vcpu); in emulator_get_cr()
8327 struct kvm_vcpu *vcpu = emul_to_vcpu(ctxt); in emulator_set_cr() local
8332 res = kvm_set_cr0(vcpu, mk_cr_64(kvm_read_cr0(vcpu), val)); in emulator_set_cr()
8335 vcpu->arch.cr2 = val; in emulator_set_cr()
8338 res = kvm_set_cr3(vcpu, val); in emulator_set_cr()
8341 res = kvm_set_cr4(vcpu, mk_cr_64(kvm_read_cr4(vcpu), val)); in emulator_set_cr()
8344 res = kvm_set_cr8(vcpu, val); in emulator_set_cr()
8425 struct kvm_vcpu *vcpu = emul_to_vcpu(ctxt); in emulator_set_segment() local
8447 kvm_set_segment(vcpu, &var, seg); in emulator_set_segment()
8454 struct kvm_vcpu *vcpu = emul_to_vcpu(ctxt); in emulator_get_msr_with_filter() local
8457 r = kvm_get_msr_with_filter(vcpu, msr_index, pdata); in emulator_get_msr_with_filter()
8462 if (kvm_msr_user_space(vcpu, msr_index, KVM_EXIT_X86_RDMSR, 0, in emulator_get_msr_with_filter()
8477 struct kvm_vcpu *vcpu = emul_to_vcpu(ctxt); in emulator_set_msr_with_filter() local
8480 r = kvm_set_msr_with_filter(vcpu, msr_index, data); in emulator_set_msr_with_filter()
8485 if (kvm_msr_user_space(vcpu, msr_index, KVM_EXIT_X86_WRMSR, data, in emulator_set_msr_with_filter()
8663 static void toggle_interruptibility(struct kvm_vcpu *vcpu, u32 mask) in toggle_interruptibility() argument
8665 u32 int_shadow = kvm_x86_call(get_interrupt_shadow)(vcpu); in toggle_interruptibility()
8676 kvm_x86_call(set_interrupt_shadow)(vcpu, mask); in toggle_interruptibility()
8678 kvm_make_request(KVM_REQ_EVENT, vcpu); in toggle_interruptibility()
8682 static void inject_emulated_exception(struct kvm_vcpu *vcpu) in inject_emulated_exception() argument
8684 struct x86_emulate_ctxt *ctxt = vcpu->arch.emulate_ctxt; in inject_emulated_exception()
8687 kvm_inject_emulated_page_fault(vcpu, &ctxt->exception); in inject_emulated_exception()
8689 kvm_queue_exception_e(vcpu, ctxt->exception.vector, in inject_emulated_exception()
8692 kvm_queue_exception(vcpu, ctxt->exception.vector); in inject_emulated_exception()
8695 static struct x86_emulate_ctxt *alloc_emulate_ctxt(struct kvm_vcpu *vcpu) in alloc_emulate_ctxt() argument
8701 pr_err("failed to allocate vcpu's emulator\n"); in alloc_emulate_ctxt()
8705 ctxt->vcpu = vcpu; in alloc_emulate_ctxt()
8707 vcpu->arch.emulate_ctxt = ctxt; in alloc_emulate_ctxt()
8712 static void init_emulate_ctxt(struct kvm_vcpu *vcpu) in init_emulate_ctxt() argument
8714 struct x86_emulate_ctxt *ctxt = vcpu->arch.emulate_ctxt; in init_emulate_ctxt()
8717 kvm_x86_call(get_cs_db_l_bits)(vcpu, &cs_db, &cs_l); in init_emulate_ctxt()
8720 ctxt->eflags = kvm_get_rflags(vcpu); in init_emulate_ctxt()
8723 ctxt->eip = kvm_rip_read(vcpu); in init_emulate_ctxt()
8724 ctxt->mode = (!is_protmode(vcpu)) ? X86EMUL_MODE_REAL : in init_emulate_ctxt()
8726 (cs_l && is_long_mode(vcpu)) ? X86EMUL_MODE_PROT64 : in init_emulate_ctxt()
8735 vcpu->arch.emulate_regs_need_sync_from_vcpu = false; in init_emulate_ctxt()
8738 void kvm_inject_realmode_interrupt(struct kvm_vcpu *vcpu, int irq, int inc_eip) in kvm_inject_realmode_interrupt() argument
8740 struct x86_emulate_ctxt *ctxt = vcpu->arch.emulate_ctxt; in kvm_inject_realmode_interrupt()
8743 init_emulate_ctxt(vcpu); in kvm_inject_realmode_interrupt()
8751 kvm_make_request(KVM_REQ_TRIPLE_FAULT, vcpu); in kvm_inject_realmode_interrupt()
8754 kvm_rip_write(vcpu, ctxt->eip); in kvm_inject_realmode_interrupt()
8755 kvm_set_rflags(vcpu, ctxt->eflags); in kvm_inject_realmode_interrupt()
8760 static void prepare_emulation_failure_exit(struct kvm_vcpu *vcpu, u64 *data, in prepare_emulation_failure_exit() argument
8763 struct kvm_run *run = vcpu->run; in prepare_emulation_failure_exit()
8773 kvm_x86_call(get_exit_info)(vcpu, (u32 *)&info[0], &info[1], &info[2], in prepare_emulation_failure_exit()
8810 static void prepare_emulation_ctxt_failure_exit(struct kvm_vcpu *vcpu) in prepare_emulation_ctxt_failure_exit() argument
8812 struct x86_emulate_ctxt *ctxt = vcpu->arch.emulate_ctxt; in prepare_emulation_ctxt_failure_exit()
8814 prepare_emulation_failure_exit(vcpu, NULL, 0, ctxt->fetch.data, in prepare_emulation_ctxt_failure_exit()
8818 void __kvm_prepare_emulation_failure_exit(struct kvm_vcpu *vcpu, u64 *data, in __kvm_prepare_emulation_failure_exit() argument
8821 prepare_emulation_failure_exit(vcpu, data, ndata, NULL, 0); in __kvm_prepare_emulation_failure_exit()
8825 void kvm_prepare_emulation_failure_exit(struct kvm_vcpu *vcpu) in kvm_prepare_emulation_failure_exit() argument
8827 __kvm_prepare_emulation_failure_exit(vcpu, NULL, 0); in kvm_prepare_emulation_failure_exit()
8831 static int handle_emulation_failure(struct kvm_vcpu *vcpu, int emulation_type) in handle_emulation_failure() argument
8833 struct kvm *kvm = vcpu->kvm; in handle_emulation_failure()
8835 ++vcpu->stat.insn_emulation_fail; in handle_emulation_failure()
8836 trace_kvm_emulate_insn_failed(vcpu); in handle_emulation_failure()
8839 kvm_queue_exception_e(vcpu, GP_VECTOR, 0); in handle_emulation_failure()
8845 prepare_emulation_ctxt_failure_exit(vcpu); in handle_emulation_failure()
8849 kvm_queue_exception(vcpu, UD_VECTOR); in handle_emulation_failure()
8851 if (!is_guest_mode(vcpu) && kvm_x86_call(get_cpl)(vcpu) == 0) { in handle_emulation_failure()
8852 prepare_emulation_ctxt_failure_exit(vcpu); in handle_emulation_failure()
8859 static bool kvm_unprotect_and_retry_on_failure(struct kvm_vcpu *vcpu, in kvm_unprotect_and_retry_on_failure() argument
8871 * the vCPU into an infinite loop of page faults. E.g. KVM will create in kvm_unprotect_and_retry_on_failure()
8885 __kvm_mmu_unprotect_gfn_and_retry(vcpu, cr2_or_gpa, true); in kvm_unprotect_and_retry_on_failure()
8888 * Retry even if _this_ vCPU didn't unprotect the gfn, as it's possible in kvm_unprotect_and_retry_on_failure()
8897 static int complete_emulated_mmio(struct kvm_vcpu *vcpu);
8898 static int complete_emulated_pio(struct kvm_vcpu *vcpu);
8915 static int kvm_vcpu_do_singlestep(struct kvm_vcpu *vcpu) in kvm_vcpu_do_singlestep() argument
8917 struct kvm_run *kvm_run = vcpu->run; in kvm_vcpu_do_singlestep()
8919 if (vcpu->guest_debug & KVM_GUESTDBG_SINGLESTEP) { in kvm_vcpu_do_singlestep()
8921 kvm_run->debug.arch.pc = kvm_get_linear_rip(vcpu); in kvm_vcpu_do_singlestep()
8926 kvm_queue_exception_p(vcpu, DB_VECTOR, DR6_BS); in kvm_vcpu_do_singlestep()
8930 int kvm_skip_emulated_instruction(struct kvm_vcpu *vcpu) in kvm_skip_emulated_instruction() argument
8932 unsigned long rflags = kvm_x86_call(get_rflags)(vcpu); in kvm_skip_emulated_instruction()
8935 r = kvm_x86_call(skip_emulated_instruction)(vcpu); in kvm_skip_emulated_instruction()
8939 kvm_pmu_trigger_event(vcpu, kvm_pmu_eventsel.INSTRUCTIONS_RETIRED); in kvm_skip_emulated_instruction()
8950 r = kvm_vcpu_do_singlestep(vcpu); in kvm_skip_emulated_instruction()
8955 static bool kvm_is_code_breakpoint_inhibited(struct kvm_vcpu *vcpu) in kvm_is_code_breakpoint_inhibited() argument
8957 if (kvm_get_rflags(vcpu) & X86_EFLAGS_RF) in kvm_is_code_breakpoint_inhibited()
8964 if (!guest_cpuid_is_intel_compatible(vcpu)) in kvm_is_code_breakpoint_inhibited()
8967 return kvm_x86_call(get_interrupt_shadow)(vcpu) & KVM_X86_SHADOW_INT_MOV_SS; in kvm_is_code_breakpoint_inhibited()
8970 static bool kvm_vcpu_check_code_breakpoint(struct kvm_vcpu *vcpu, in kvm_vcpu_check_code_breakpoint() argument
8993 if (unlikely(vcpu->guest_debug & KVM_GUESTDBG_USE_HW_BP) && in kvm_vcpu_check_code_breakpoint()
8994 (vcpu->arch.guest_debug_dr7 & DR7_BP_EN_MASK)) { in kvm_vcpu_check_code_breakpoint()
8995 struct kvm_run *kvm_run = vcpu->run; in kvm_vcpu_check_code_breakpoint()
8996 unsigned long eip = kvm_get_linear_rip(vcpu); in kvm_vcpu_check_code_breakpoint()
8998 vcpu->arch.guest_debug_dr7, in kvm_vcpu_check_code_breakpoint()
8999 vcpu->arch.eff_db); in kvm_vcpu_check_code_breakpoint()
9011 if (unlikely(vcpu->arch.dr7 & DR7_BP_EN_MASK) && in kvm_vcpu_check_code_breakpoint()
9012 !kvm_is_code_breakpoint_inhibited(vcpu)) { in kvm_vcpu_check_code_breakpoint()
9013 unsigned long eip = kvm_get_linear_rip(vcpu); in kvm_vcpu_check_code_breakpoint()
9015 vcpu->arch.dr7, in kvm_vcpu_check_code_breakpoint()
9016 vcpu->arch.db); in kvm_vcpu_check_code_breakpoint()
9019 kvm_queue_exception_p(vcpu, DB_VECTOR, dr6); in kvm_vcpu_check_code_breakpoint()
9069 int x86_decode_emulated_instruction(struct kvm_vcpu *vcpu, int emulation_type, in x86_decode_emulated_instruction() argument
9072 struct x86_emulate_ctxt *ctxt = vcpu->arch.emulate_ctxt; in x86_decode_emulated_instruction()
9075 init_emulate_ctxt(vcpu); in x86_decode_emulated_instruction()
9079 trace_kvm_emulate_insn_start(vcpu); in x86_decode_emulated_instruction()
9080 ++vcpu->stat.insn_emulation; in x86_decode_emulated_instruction()
9086 int x86_emulate_instruction(struct kvm_vcpu *vcpu, gpa_t cr2_or_gpa, in x86_emulate_instruction() argument
9090 struct x86_emulate_ctxt *ctxt = vcpu->arch.emulate_ctxt; in x86_emulate_instruction()
9094 (WARN_ON_ONCE(is_guest_mode(vcpu)) || in x86_emulate_instruction()
9098 r = kvm_check_emulate_insn(vcpu, emulation_type, insn, insn_len); in x86_emulate_instruction()
9104 return handle_emulation_failure(vcpu, emulation_type); in x86_emulate_instruction()
9107 vcpu->arch.l1tf_flush_l1d = true; in x86_emulate_instruction()
9110 kvm_clear_exception_queue(vcpu); in x86_emulate_instruction()
9117 if (kvm_vcpu_check_code_breakpoint(vcpu, emulation_type, &r)) in x86_emulate_instruction()
9120 r = x86_decode_emulated_instruction(vcpu, emulation_type, in x86_emulate_instruction()
9125 kvm_queue_exception(vcpu, UD_VECTOR); in x86_emulate_instruction()
9128 if (kvm_unprotect_and_retry_on_failure(vcpu, cr2_or_gpa, in x86_emulate_instruction()
9140 inject_emulated_exception(vcpu); in x86_emulate_instruction()
9143 return handle_emulation_failure(vcpu, emulation_type); in x86_emulate_instruction()
9149 kvm_queue_exception_e(vcpu, GP_VECTOR, 0); in x86_emulate_instruction()
9170 kvm_rip_write(vcpu, ctxt->eip); in x86_emulate_instruction()
9172 kvm_set_rflags(vcpu, ctxt->eflags & ~X86_EFLAGS_RF); in x86_emulate_instruction()
9179 * and retry the instruction, as the vCPU is likely no longer using the in x86_emulate_instruction()
9184 kvm_mmu_unprotect_gfn_and_retry(vcpu, cr2_or_gpa)) in x86_emulate_instruction()
9189 if (vcpu->arch.emulate_regs_need_sync_from_vcpu) { in x86_emulate_instruction()
9190 vcpu->arch.emulate_regs_need_sync_from_vcpu = false; in x86_emulate_instruction()
9200 if (vcpu->arch.mmu->root_role.direct) { in x86_emulate_instruction()
9215 if (kvm_unprotect_and_retry_on_failure(vcpu, cr2_or_gpa, in x86_emulate_instruction()
9219 return handle_emulation_failure(vcpu, emulation_type); in x86_emulate_instruction()
9223 WARN_ON_ONCE(vcpu->mmio_needed && !vcpu->mmio_is_write); in x86_emulate_instruction()
9224 vcpu->mmio_needed = false; in x86_emulate_instruction()
9226 inject_emulated_exception(vcpu); in x86_emulate_instruction()
9227 } else if (vcpu->arch.pio.count) { in x86_emulate_instruction()
9228 if (!vcpu->arch.pio.in) { in x86_emulate_instruction()
9230 vcpu->arch.pio.count = 0; in x86_emulate_instruction()
9233 vcpu->arch.complete_userspace_io = complete_emulated_pio; in x86_emulate_instruction()
9236 } else if (vcpu->mmio_needed) { in x86_emulate_instruction()
9237 ++vcpu->stat.mmio_exits; in x86_emulate_instruction()
9239 if (!vcpu->mmio_is_write) in x86_emulate_instruction()
9242 vcpu->arch.complete_userspace_io = complete_emulated_mmio; in x86_emulate_instruction()
9243 } else if (vcpu->arch.complete_userspace_io) { in x86_emulate_instruction()
9253 unsigned long rflags = kvm_x86_call(get_rflags)(vcpu); in x86_emulate_instruction()
9254 toggle_interruptibility(vcpu, ctxt->interruptibility); in x86_emulate_instruction()
9255 vcpu->arch.emulate_regs_need_sync_to_vcpu = false; in x86_emulate_instruction()
9264 kvm_pmu_trigger_event(vcpu, kvm_pmu_eventsel.INSTRUCTIONS_RETIRED); in x86_emulate_instruction()
9266 kvm_pmu_trigger_event(vcpu, kvm_pmu_eventsel.BRANCH_INSTRUCTIONS_RETIRED); in x86_emulate_instruction()
9267 kvm_rip_write(vcpu, ctxt->eip); in x86_emulate_instruction()
9268 if (r && (ctxt->tf || (vcpu->guest_debug & KVM_GUESTDBG_SINGLESTEP))) in x86_emulate_instruction()
9269 r = kvm_vcpu_do_singlestep(vcpu); in x86_emulate_instruction()
9270 kvm_x86_call(update_emulated_instruction)(vcpu); in x86_emulate_instruction()
9271 __kvm_set_rflags(vcpu, ctxt->eflags); in x86_emulate_instruction()
9281 kvm_make_request(KVM_REQ_EVENT, vcpu); in x86_emulate_instruction()
9283 vcpu->arch.emulate_regs_need_sync_to_vcpu = true; in x86_emulate_instruction()
9288 int kvm_emulate_instruction(struct kvm_vcpu *vcpu, int emulation_type) in kvm_emulate_instruction() argument
9290 return x86_emulate_instruction(vcpu, 0, emulation_type, NULL, 0); in kvm_emulate_instruction()
9294 int kvm_emulate_instruction_from_buffer(struct kvm_vcpu *vcpu, in kvm_emulate_instruction_from_buffer() argument
9297 return x86_emulate_instruction(vcpu, 0, 0, insn, insn_len); in kvm_emulate_instruction_from_buffer()
9301 static int complete_fast_pio_out_port_0x7e(struct kvm_vcpu *vcpu) in complete_fast_pio_out_port_0x7e() argument
9303 vcpu->arch.pio.count = 0; in complete_fast_pio_out_port_0x7e()
9307 static int complete_fast_pio_out(struct kvm_vcpu *vcpu) in complete_fast_pio_out() argument
9309 vcpu->arch.pio.count = 0; in complete_fast_pio_out()
9311 if (unlikely(!kvm_is_linear_rip(vcpu, vcpu->arch.pio.linear_rip))) in complete_fast_pio_out()
9314 return kvm_skip_emulated_instruction(vcpu); in complete_fast_pio_out()
9317 static int kvm_fast_pio_out(struct kvm_vcpu *vcpu, int size, in kvm_fast_pio_out() argument
9320 unsigned long val = kvm_rax_read(vcpu); in kvm_fast_pio_out()
9321 int ret = emulator_pio_out(vcpu, size, port, &val, 1); in kvm_fast_pio_out()
9331 kvm_check_has_quirk(vcpu->kvm, KVM_X86_QUIRK_OUT_7E_INC_RIP)) { in kvm_fast_pio_out()
9332 vcpu->arch.complete_userspace_io = in kvm_fast_pio_out()
9334 kvm_skip_emulated_instruction(vcpu); in kvm_fast_pio_out()
9336 vcpu->arch.pio.linear_rip = kvm_get_linear_rip(vcpu); in kvm_fast_pio_out()
9337 vcpu->arch.complete_userspace_io = complete_fast_pio_out; in kvm_fast_pio_out()
9342 static int complete_fast_pio_in(struct kvm_vcpu *vcpu) in complete_fast_pio_in() argument
9347 BUG_ON(vcpu->arch.pio.count != 1); in complete_fast_pio_in()
9349 if (unlikely(!kvm_is_linear_rip(vcpu, vcpu->arch.pio.linear_rip))) { in complete_fast_pio_in()
9350 vcpu->arch.pio.count = 0; in complete_fast_pio_in()
9355 val = (vcpu->arch.pio.size < 4) ? kvm_rax_read(vcpu) : 0; in complete_fast_pio_in()
9357 complete_emulator_pio_in(vcpu, &val); in complete_fast_pio_in()
9358 kvm_rax_write(vcpu, val); in complete_fast_pio_in()
9360 return kvm_skip_emulated_instruction(vcpu); in complete_fast_pio_in()
9363 static int kvm_fast_pio_in(struct kvm_vcpu *vcpu, int size, in kvm_fast_pio_in() argument
9370 val = (size < 4) ? kvm_rax_read(vcpu) : 0; in kvm_fast_pio_in()
9372 ret = emulator_pio_in(vcpu, size, port, &val, 1); in kvm_fast_pio_in()
9374 kvm_rax_write(vcpu, val); in kvm_fast_pio_in()
9378 vcpu->arch.pio.linear_rip = kvm_get_linear_rip(vcpu); in kvm_fast_pio_in()
9379 vcpu->arch.complete_userspace_io = complete_fast_pio_in; in kvm_fast_pio_in()
9384 int kvm_fast_pio(struct kvm_vcpu *vcpu, int size, unsigned short port, int in) in kvm_fast_pio() argument
9389 ret = kvm_fast_pio_in(vcpu, size, port); in kvm_fast_pio()
9391 ret = kvm_fast_pio_out(vcpu, size, port); in kvm_fast_pio()
9392 return ret && kvm_skip_emulated_instruction(vcpu); in kvm_fast_pio()
9451 struct kvm_vcpu *vcpu; in __kvmclock_cpufreq_notifier() local
9466 * the TSC for each VCPU. We must flag these local variables in __kvmclock_cpufreq_notifier()
9485 * anytime after the setting of the VCPU's request bit, the in __kvmclock_cpufreq_notifier()
9498 kvm_for_each_vcpu(i, vcpu, kvm) { in __kvmclock_cpufreq_notifier()
9499 if (vcpu->cpu != cpu) in __kvmclock_cpufreq_notifier()
9501 kvm_make_request(KVM_REQ_CLOCK_UPDATE, vcpu); in __kvmclock_cpufreq_notifier()
9502 if (vcpu->cpu != raw_smp_processor_id()) in __kvmclock_cpufreq_notifier()
9582 struct kvm_vcpu *vcpu; in pvclock_gtod_update_fn() local
9587 kvm_for_each_vcpu(i, vcpu, kvm) in pvclock_gtod_update_fn()
9588 kvm_make_request(KVM_REQ_MASTERCLOCK_UPDATE, vcpu); in pvclock_gtod_update_fn()
9692 * vCPU's FPU state as a fxregs_state struct. in kvm_x86_vendor_init()
9858 static int kvm_pv_clock_pairing(struct kvm_vcpu *vcpu, gpa_t paddr, in kvm_pv_clock_pairing() argument
9873 if (vcpu->arch.tsc_always_catchup) in kvm_pv_clock_pairing()
9881 clock_pairing.tsc = kvm_read_l1_tsc(vcpu, cycle); in kvm_pv_clock_pairing()
9886 if (kvm_write_guest(vcpu->kvm, paddr, &clock_pairing, in kvm_pv_clock_pairing()
9895 * kvm_pv_kick_cpu_op: Kick a vcpu.
9897 * @apicid - apicid of vcpu to be kicked.
9921 bool kvm_vcpu_apicv_activated(struct kvm_vcpu *vcpu) in kvm_vcpu_apicv_activated() argument
9923 ulong vm_reasons = READ_ONCE(vcpu->kvm->arch.apicv_inhibit_reasons); in kvm_vcpu_apicv_activated()
9925 kvm_x86_call(vcpu_get_apicv_inhibit_reasons)(vcpu); in kvm_vcpu_apicv_activated()
9956 static void kvm_sched_yield(struct kvm_vcpu *vcpu, unsigned long dest_id) in kvm_sched_yield() argument
9961 vcpu->stat.directed_yield_attempted++; in kvm_sched_yield()
9967 map = rcu_dereference(vcpu->kvm->arch.apic_map); in kvm_sched_yield()
9970 target = map->phys_map[dest_id]->vcpu; in kvm_sched_yield()
9978 if (vcpu == target) in kvm_sched_yield()
9984 vcpu->stat.directed_yield_successful++; in kvm_sched_yield()
9990 static int complete_hypercall_exit(struct kvm_vcpu *vcpu) in complete_hypercall_exit() argument
9992 u64 ret = vcpu->run->hypercall.ret; in complete_hypercall_exit()
9994 if (!is_64_bit_mode(vcpu)) in complete_hypercall_exit()
9996 kvm_rax_write(vcpu, ret); in complete_hypercall_exit()
9997 ++vcpu->stat.hypercalls; in complete_hypercall_exit()
9998 return kvm_skip_emulated_instruction(vcpu); in complete_hypercall_exit()
10001 unsigned long __kvm_emulate_hypercall(struct kvm_vcpu *vcpu, unsigned long nr, in __kvm_emulate_hypercall() argument
10030 if (!guest_pv_has(vcpu, KVM_FEATURE_PV_UNHALT)) in __kvm_emulate_hypercall()
10033 kvm_pv_kick_cpu_op(vcpu->kvm, a1); in __kvm_emulate_hypercall()
10034 kvm_sched_yield(vcpu, a1); in __kvm_emulate_hypercall()
10039 ret = kvm_pv_clock_pairing(vcpu, a0, a1); in __kvm_emulate_hypercall()
10043 if (!guest_pv_has(vcpu, KVM_FEATURE_PV_SEND_IPI)) in __kvm_emulate_hypercall()
10046 ret = kvm_pv_send_ipi(vcpu->kvm, a0, a1, a2, a3, op_64_bit); in __kvm_emulate_hypercall()
10049 if (!guest_pv_has(vcpu, KVM_FEATURE_PV_SCHED_YIELD)) in __kvm_emulate_hypercall()
10052 kvm_sched_yield(vcpu, a0); in __kvm_emulate_hypercall()
10059 if (!(vcpu->kvm->arch.hypercall_exit_enabled & (1 << KVM_HC_MAP_GPA_RANGE))) in __kvm_emulate_hypercall()
10068 vcpu->run->exit_reason = KVM_EXIT_HYPERCALL; in __kvm_emulate_hypercall()
10069 vcpu->run->hypercall.nr = KVM_HC_MAP_GPA_RANGE; in __kvm_emulate_hypercall()
10070 vcpu->run->hypercall.args[0] = gpa; in __kvm_emulate_hypercall()
10071 vcpu->run->hypercall.args[1] = npages; in __kvm_emulate_hypercall()
10072 vcpu->run->hypercall.args[2] = attrs; in __kvm_emulate_hypercall()
10073 vcpu->run->hypercall.flags = 0; in __kvm_emulate_hypercall()
10075 vcpu->run->hypercall.flags |= KVM_EXIT_HYPERCALL_LONG_MODE; in __kvm_emulate_hypercall()
10077 WARN_ON_ONCE(vcpu->run->hypercall.flags & KVM_EXIT_HYPERCALL_MBZ); in __kvm_emulate_hypercall()
10078 vcpu->arch.complete_userspace_io = complete_hypercall_exit; in __kvm_emulate_hypercall()
10088 ++vcpu->stat.hypercalls; in __kvm_emulate_hypercall()
10093 int kvm_emulate_hypercall(struct kvm_vcpu *vcpu) in kvm_emulate_hypercall() argument
10099 if (kvm_xen_hypercall_enabled(vcpu->kvm)) in kvm_emulate_hypercall()
10100 return kvm_xen_hypercall(vcpu); in kvm_emulate_hypercall()
10102 if (kvm_hv_hypercall_enabled(vcpu)) in kvm_emulate_hypercall()
10103 return kvm_hv_hypercall(vcpu); in kvm_emulate_hypercall()
10105 nr = kvm_rax_read(vcpu); in kvm_emulate_hypercall()
10106 a0 = kvm_rbx_read(vcpu); in kvm_emulate_hypercall()
10107 a1 = kvm_rcx_read(vcpu); in kvm_emulate_hypercall()
10108 a2 = kvm_rdx_read(vcpu); in kvm_emulate_hypercall()
10109 a3 = kvm_rsi_read(vcpu); in kvm_emulate_hypercall()
10110 op_64_bit = is_64_bit_hypercall(vcpu); in kvm_emulate_hypercall()
10111 cpl = kvm_x86_call(get_cpl)(vcpu); in kvm_emulate_hypercall()
10113 ret = __kvm_emulate_hypercall(vcpu, nr, a0, a1, a2, a3, op_64_bit, cpl); in kvm_emulate_hypercall()
10120 kvm_rax_write(vcpu, ret); in kvm_emulate_hypercall()
10122 return kvm_skip_emulated_instruction(vcpu); in kvm_emulate_hypercall()
10128 struct kvm_vcpu *vcpu = emul_to_vcpu(ctxt); in emulator_fix_hypercall() local
10130 unsigned long rip = kvm_rip_read(vcpu); in emulator_fix_hypercall()
10136 if (!kvm_check_has_quirk(vcpu->kvm, KVM_X86_QUIRK_FIX_HYPERCALL_INSN)) { in emulator_fix_hypercall()
10143 kvm_x86_call(patch_hypercall)(vcpu, instruction); in emulator_fix_hypercall()
10149 static int dm_request_for_irq_injection(struct kvm_vcpu *vcpu) in dm_request_for_irq_injection() argument
10151 return vcpu->run->request_interrupt_window && in dm_request_for_irq_injection()
10152 likely(!pic_in_kernel(vcpu->kvm)); in dm_request_for_irq_injection()
10156 static void post_kvm_run_save(struct kvm_vcpu *vcpu) in post_kvm_run_save() argument
10158 struct kvm_run *kvm_run = vcpu->run; in post_kvm_run_save()
10160 kvm_run->if_flag = kvm_x86_call(get_if_flag)(vcpu); in post_kvm_run_save()
10161 kvm_run->cr8 = kvm_get_cr8(vcpu); in post_kvm_run_save()
10162 kvm_run->apic_base = kvm_get_apic_base(vcpu); in post_kvm_run_save()
10165 pic_in_kernel(vcpu->kvm) || in post_kvm_run_save()
10166 kvm_vcpu_ready_for_interrupt_injection(vcpu); in post_kvm_run_save()
10168 if (is_smm(vcpu)) in post_kvm_run_save()
10170 if (is_guest_mode(vcpu)) in post_kvm_run_save()
10174 static void update_cr8_intercept(struct kvm_vcpu *vcpu) in update_cr8_intercept() argument
10181 if (!lapic_in_kernel(vcpu)) in update_cr8_intercept()
10184 if (vcpu->arch.apic->apicv_active) in update_cr8_intercept()
10187 if (!vcpu->arch.apic->vapic_addr) in update_cr8_intercept()
10188 max_irr = kvm_lapic_find_highest_irr(vcpu); in update_cr8_intercept()
10195 tpr = kvm_lapic_get_cr8(vcpu); in update_cr8_intercept()
10197 kvm_x86_call(update_cr8_intercept)(vcpu, tpr, max_irr); in update_cr8_intercept()
10201 int kvm_check_nested_events(struct kvm_vcpu *vcpu) in kvm_check_nested_events() argument
10203 if (kvm_test_request(KVM_REQ_TRIPLE_FAULT, vcpu)) { in kvm_check_nested_events()
10204 kvm_x86_ops.nested_ops->triple_fault(vcpu); in kvm_check_nested_events()
10208 return kvm_x86_ops.nested_ops->check_events(vcpu); in kvm_check_nested_events()
10211 static void kvm_inject_exception(struct kvm_vcpu *vcpu) in kvm_inject_exception() argument
10214 * Suppress the error code if the vCPU is in Real Mode, as Real Mode in kvm_inject_exception()
10220 vcpu->arch.exception.has_error_code &= is_protmode(vcpu); in kvm_inject_exception()
10222 trace_kvm_inj_exception(vcpu->arch.exception.vector, in kvm_inject_exception()
10223 vcpu->arch.exception.has_error_code, in kvm_inject_exception()
10224 vcpu->arch.exception.error_code, in kvm_inject_exception()
10225 vcpu->arch.exception.injected); in kvm_inject_exception()
10227 kvm_x86_call(inject_exception)(vcpu); in kvm_inject_exception()
10269 static int kvm_check_and_inject_events(struct kvm_vcpu *vcpu, in kvm_check_and_inject_events() argument
10280 if (is_guest_mode(vcpu)) in kvm_check_and_inject_events()
10281 r = kvm_check_nested_events(vcpu); in kvm_check_and_inject_events()
10308 if (vcpu->arch.exception.injected) in kvm_check_and_inject_events()
10309 kvm_inject_exception(vcpu); in kvm_check_and_inject_events()
10310 else if (kvm_is_exception_pending(vcpu)) in kvm_check_and_inject_events()
10312 else if (vcpu->arch.nmi_injected) in kvm_check_and_inject_events()
10313 kvm_x86_call(inject_nmi)(vcpu); in kvm_check_and_inject_events()
10314 else if (vcpu->arch.interrupt.injected) in kvm_check_and_inject_events()
10315 kvm_x86_call(inject_irq)(vcpu, true); in kvm_check_and_inject_events()
10322 WARN_ON_ONCE(vcpu->arch.exception.injected && in kvm_check_and_inject_events()
10323 vcpu->arch.exception.pending); in kvm_check_and_inject_events()
10342 WARN_ON_ONCE(vcpu->arch.exception_vmexit.injected || in kvm_check_and_inject_events()
10343 vcpu->arch.exception_vmexit.pending); in kvm_check_and_inject_events()
10350 can_inject = !kvm_event_needs_reinjection(vcpu); in kvm_check_and_inject_events()
10352 if (vcpu->arch.exception.pending) { in kvm_check_and_inject_events()
10363 if (exception_type(vcpu->arch.exception.vector) == EXCPT_FAULT) in kvm_check_and_inject_events()
10364 __kvm_set_rflags(vcpu, kvm_get_rflags(vcpu) | in kvm_check_and_inject_events()
10367 if (vcpu->arch.exception.vector == DB_VECTOR) { in kvm_check_and_inject_events()
10368 kvm_deliver_exception_payload(vcpu, &vcpu->arch.exception); in kvm_check_and_inject_events()
10369 if (vcpu->arch.dr7 & DR7_GD) { in kvm_check_and_inject_events()
10370 vcpu->arch.dr7 &= ~DR7_GD; in kvm_check_and_inject_events()
10371 kvm_update_dr7(vcpu); in kvm_check_and_inject_events()
10375 kvm_inject_exception(vcpu); in kvm_check_and_inject_events()
10377 vcpu->arch.exception.pending = false; in kvm_check_and_inject_events()
10378 vcpu->arch.exception.injected = true; in kvm_check_and_inject_events()
10384 if (vcpu->guest_debug & KVM_GUESTDBG_BLOCKIRQ) in kvm_check_and_inject_events()
10399 if (vcpu->arch.smi_pending) { in kvm_check_and_inject_events()
10400 r = can_inject ? kvm_x86_call(smi_allowed)(vcpu, true) : in kvm_check_and_inject_events()
10405 vcpu->arch.smi_pending = false; in kvm_check_and_inject_events()
10406 ++vcpu->arch.smi_count; in kvm_check_and_inject_events()
10407 enter_smm(vcpu); in kvm_check_and_inject_events()
10410 kvm_x86_call(enable_smi_window)(vcpu); in kvm_check_and_inject_events()
10414 if (vcpu->arch.nmi_pending) { in kvm_check_and_inject_events()
10415 r = can_inject ? kvm_x86_call(nmi_allowed)(vcpu, true) : in kvm_check_and_inject_events()
10420 --vcpu->arch.nmi_pending; in kvm_check_and_inject_events()
10421 vcpu->arch.nmi_injected = true; in kvm_check_and_inject_events()
10422 kvm_x86_call(inject_nmi)(vcpu); in kvm_check_and_inject_events()
10424 WARN_ON(kvm_x86_call(nmi_allowed)(vcpu, true) < 0); in kvm_check_and_inject_events()
10426 if (vcpu->arch.nmi_pending) in kvm_check_and_inject_events()
10427 kvm_x86_call(enable_nmi_window)(vcpu); in kvm_check_and_inject_events()
10430 if (kvm_cpu_has_injectable_intr(vcpu)) { in kvm_check_and_inject_events()
10431 r = can_inject ? kvm_x86_call(interrupt_allowed)(vcpu, true) : in kvm_check_and_inject_events()
10436 int irq = kvm_cpu_get_interrupt(vcpu); in kvm_check_and_inject_events()
10439 kvm_queue_interrupt(vcpu, irq, false); in kvm_check_and_inject_events()
10440 kvm_x86_call(inject_irq)(vcpu, false); in kvm_check_and_inject_events()
10441 WARN_ON(kvm_x86_call(interrupt_allowed)(vcpu, true) < 0); in kvm_check_and_inject_events()
10444 if (kvm_cpu_has_injectable_intr(vcpu)) in kvm_check_and_inject_events()
10445 kvm_x86_call(enable_irq_window)(vcpu); in kvm_check_and_inject_events()
10448 if (is_guest_mode(vcpu) && in kvm_check_and_inject_events()
10450 kvm_x86_ops.nested_ops->has_events(vcpu, true)) in kvm_check_and_inject_events()
10456 * to the VMCS/VMCB. Queueing a new exception can put the vCPU into an in kvm_check_and_inject_events()
10461 * vCPU into an infinite loop. Triple fault can be queued when running in kvm_check_and_inject_events()
10465 WARN_ON_ONCE(vcpu->arch.exception.pending || in kvm_check_and_inject_events()
10466 vcpu->arch.exception_vmexit.pending); in kvm_check_and_inject_events()
10477 static void process_nmi(struct kvm_vcpu *vcpu) in process_nmi() argument
10483 * incoming NMIs as quickly as bare metal, e.g. if the vCPU is in process_nmi()
10491 if (kvm_x86_call(get_nmi_mask)(vcpu) || vcpu->arch.nmi_injected) in process_nmi()
10498 * tracked in vcpu->arch.nmi_pending. in process_nmi()
10500 if (kvm_x86_call(is_vnmi_pending)(vcpu)) in process_nmi()
10503 vcpu->arch.nmi_pending += atomic_xchg(&vcpu->arch.nmi_queued, 0); in process_nmi()
10504 vcpu->arch.nmi_pending = min(vcpu->arch.nmi_pending, limit); in process_nmi()
10506 if (vcpu->arch.nmi_pending && in process_nmi()
10507 (kvm_x86_call(set_vnmi_pending)(vcpu))) in process_nmi()
10508 vcpu->arch.nmi_pending--; in process_nmi()
10510 if (vcpu->arch.nmi_pending) in process_nmi()
10511 kvm_make_request(KVM_REQ_EVENT, vcpu); in process_nmi()
10515 int kvm_get_nr_pending_nmis(struct kvm_vcpu *vcpu) in kvm_get_nr_pending_nmis() argument
10517 return vcpu->arch.nmi_pending + in kvm_get_nr_pending_nmis()
10518 kvm_x86_call(is_vnmi_pending)(vcpu); in kvm_get_nr_pending_nmis()
10532 void __kvm_vcpu_update_apicv(struct kvm_vcpu *vcpu) in __kvm_vcpu_update_apicv() argument
10534 struct kvm_lapic *apic = vcpu->arch.apic; in __kvm_vcpu_update_apicv()
10537 if (!lapic_in_kernel(vcpu)) in __kvm_vcpu_update_apicv()
10540 down_read(&vcpu->kvm->arch.apicv_update_lock); in __kvm_vcpu_update_apicv()
10544 activate = kvm_vcpu_apicv_activated(vcpu) && in __kvm_vcpu_update_apicv()
10545 (kvm_get_apic_mode(vcpu) != LAPIC_MODE_DISABLED); in __kvm_vcpu_update_apicv()
10551 kvm_apic_update_apicv(vcpu); in __kvm_vcpu_update_apicv()
10552 kvm_x86_call(refresh_apicv_exec_ctrl)(vcpu); in __kvm_vcpu_update_apicv()
10561 kvm_make_request(KVM_REQ_EVENT, vcpu); in __kvm_vcpu_update_apicv()
10565 up_read(&vcpu->kvm->arch.apicv_update_lock); in __kvm_vcpu_update_apicv()
10569 static void kvm_vcpu_update_apicv(struct kvm_vcpu *vcpu) in kvm_vcpu_update_apicv() argument
10571 if (!lapic_in_kernel(vcpu)) in kvm_vcpu_update_apicv()
10576 * deleted if any vCPU has xAPIC virtualization and x2APIC enabled, but in kvm_vcpu_update_apicv()
10581 * the vCPU would incorrectly be able to access the vAPIC page via MMIO in kvm_vcpu_update_apicv()
10585 if (apic_x2apic_mode(vcpu->arch.apic) && in kvm_vcpu_update_apicv()
10587 kvm_inhibit_apic_access_page(vcpu); in kvm_vcpu_update_apicv()
10589 __kvm_vcpu_update_apicv(vcpu); in kvm_vcpu_update_apicv()
10645 static void vcpu_scan_ioapic(struct kvm_vcpu *vcpu) in vcpu_scan_ioapic() argument
10647 if (!kvm_apic_present(vcpu)) in vcpu_scan_ioapic()
10650 bitmap_zero(vcpu->arch.ioapic_handled_vectors, 256); in vcpu_scan_ioapic()
10652 kvm_x86_call(sync_pir_to_irr)(vcpu); in vcpu_scan_ioapic()
10654 if (irqchip_split(vcpu->kvm)) in vcpu_scan_ioapic()
10655 kvm_scan_ioapic_routes(vcpu, vcpu->arch.ioapic_handled_vectors); in vcpu_scan_ioapic()
10656 else if (ioapic_in_kernel(vcpu->kvm)) in vcpu_scan_ioapic()
10657 kvm_ioapic_scan_entry(vcpu, vcpu->arch.ioapic_handled_vectors); in vcpu_scan_ioapic()
10659 if (is_guest_mode(vcpu)) in vcpu_scan_ioapic()
10660 vcpu->arch.load_eoi_exitmap_pending = true; in vcpu_scan_ioapic()
10662 kvm_make_request(KVM_REQ_LOAD_EOI_EXITMAP, vcpu); in vcpu_scan_ioapic()
10665 static void vcpu_load_eoi_exitmap(struct kvm_vcpu *vcpu) in vcpu_load_eoi_exitmap() argument
10667 if (!kvm_apic_hw_enabled(vcpu->arch.apic)) in vcpu_load_eoi_exitmap()
10671 if (to_hv_vcpu(vcpu)) { in vcpu_load_eoi_exitmap()
10675 vcpu->arch.ioapic_handled_vectors, in vcpu_load_eoi_exitmap()
10676 to_hv_synic(vcpu)->vec_bitmap, 256); in vcpu_load_eoi_exitmap()
10677 kvm_x86_call(load_eoi_exitmap)(vcpu, eoi_exit_bitmap); in vcpu_load_eoi_exitmap()
10682 vcpu, (u64 *)vcpu->arch.ioapic_handled_vectors); in vcpu_load_eoi_exitmap()
10690 static void kvm_vcpu_reload_apic_access_page(struct kvm_vcpu *vcpu) in kvm_vcpu_reload_apic_access_page() argument
10692 if (!lapic_in_kernel(vcpu)) in kvm_vcpu_reload_apic_access_page()
10695 kvm_x86_call(set_apic_access_page_addr)(vcpu); in kvm_vcpu_reload_apic_access_page()
10704 static int vcpu_enter_guest(struct kvm_vcpu *vcpu) in vcpu_enter_guest() argument
10708 dm_request_for_irq_injection(vcpu) && in vcpu_enter_guest()
10709 kvm_cpu_accept_dm_intr(vcpu); in vcpu_enter_guest()
10714 if (kvm_request_pending(vcpu)) { in vcpu_enter_guest()
10715 if (kvm_check_request(KVM_REQ_VM_DEAD, vcpu)) { in vcpu_enter_guest()
10720 if (kvm_dirty_ring_check_request(vcpu)) { in vcpu_enter_guest()
10725 if (kvm_check_request(KVM_REQ_GET_NESTED_STATE_PAGES, vcpu)) { in vcpu_enter_guest()
10726 if (unlikely(!kvm_x86_ops.nested_ops->get_nested_state_pages(vcpu))) { in vcpu_enter_guest()
10731 if (kvm_check_request(KVM_REQ_MMU_FREE_OBSOLETE_ROOTS, vcpu)) in vcpu_enter_guest()
10732 kvm_mmu_free_obsolete_roots(vcpu); in vcpu_enter_guest()
10733 if (kvm_check_request(KVM_REQ_MIGRATE_TIMER, vcpu)) in vcpu_enter_guest()
10734 __kvm_migrate_timers(vcpu); in vcpu_enter_guest()
10735 if (kvm_check_request(KVM_REQ_MASTERCLOCK_UPDATE, vcpu)) in vcpu_enter_guest()
10736 kvm_update_masterclock(vcpu->kvm); in vcpu_enter_guest()
10737 if (kvm_check_request(KVM_REQ_GLOBAL_CLOCK_UPDATE, vcpu)) in vcpu_enter_guest()
10738 kvm_gen_kvmclock_update(vcpu); in vcpu_enter_guest()
10739 if (kvm_check_request(KVM_REQ_CLOCK_UPDATE, vcpu)) { in vcpu_enter_guest()
10740 r = kvm_guest_time_update(vcpu); in vcpu_enter_guest()
10744 if (kvm_check_request(KVM_REQ_MMU_SYNC, vcpu)) in vcpu_enter_guest()
10745 kvm_mmu_sync_roots(vcpu); in vcpu_enter_guest()
10746 if (kvm_check_request(KVM_REQ_LOAD_MMU_PGD, vcpu)) in vcpu_enter_guest()
10747 kvm_mmu_load_pgd(vcpu); in vcpu_enter_guest()
10754 if (kvm_check_request(KVM_REQ_TLB_FLUSH, vcpu)) in vcpu_enter_guest()
10755 kvm_vcpu_flush_tlb_all(vcpu); in vcpu_enter_guest()
10757 kvm_service_local_tlb_flush_requests(vcpu); in vcpu_enter_guest()
10761 * flushing fails. Note, Hyper-V's flushing is per-vCPU, but in vcpu_enter_guest()
10766 if (kvm_check_request(KVM_REQ_HV_TLB_FLUSH, vcpu) && in vcpu_enter_guest()
10767 kvm_hv_vcpu_flush_tlb(vcpu)) in vcpu_enter_guest()
10768 kvm_vcpu_flush_tlb_guest(vcpu); in vcpu_enter_guest()
10771 if (kvm_check_request(KVM_REQ_REPORT_TPR_ACCESS, vcpu)) { in vcpu_enter_guest()
10772 vcpu->run->exit_reason = KVM_EXIT_TPR_ACCESS; in vcpu_enter_guest()
10776 if (kvm_test_request(KVM_REQ_TRIPLE_FAULT, vcpu)) { in vcpu_enter_guest()
10777 if (is_guest_mode(vcpu)) in vcpu_enter_guest()
10778 kvm_x86_ops.nested_ops->triple_fault(vcpu); in vcpu_enter_guest()
10780 if (kvm_check_request(KVM_REQ_TRIPLE_FAULT, vcpu)) { in vcpu_enter_guest()
10781 vcpu->run->exit_reason = KVM_EXIT_SHUTDOWN; in vcpu_enter_guest()
10782 vcpu->mmio_needed = 0; in vcpu_enter_guest()
10787 if (kvm_check_request(KVM_REQ_APF_HALT, vcpu)) { in vcpu_enter_guest()
10789 vcpu->arch.apf.halted = true; in vcpu_enter_guest()
10793 if (kvm_check_request(KVM_REQ_STEAL_UPDATE, vcpu)) in vcpu_enter_guest()
10794 record_steal_time(vcpu); in vcpu_enter_guest()
10795 if (kvm_check_request(KVM_REQ_PMU, vcpu)) in vcpu_enter_guest()
10796 kvm_pmu_handle_event(vcpu); in vcpu_enter_guest()
10797 if (kvm_check_request(KVM_REQ_PMI, vcpu)) in vcpu_enter_guest()
10798 kvm_pmu_deliver_pmi(vcpu); in vcpu_enter_guest()
10800 if (kvm_check_request(KVM_REQ_SMI, vcpu)) in vcpu_enter_guest()
10801 process_smi(vcpu); in vcpu_enter_guest()
10803 if (kvm_check_request(KVM_REQ_NMI, vcpu)) in vcpu_enter_guest()
10804 process_nmi(vcpu); in vcpu_enter_guest()
10805 if (kvm_check_request(KVM_REQ_IOAPIC_EOI_EXIT, vcpu)) { in vcpu_enter_guest()
10806 BUG_ON(vcpu->arch.pending_ioapic_eoi > 255); in vcpu_enter_guest()
10807 if (test_bit(vcpu->arch.pending_ioapic_eoi, in vcpu_enter_guest()
10808 vcpu->arch.ioapic_handled_vectors)) { in vcpu_enter_guest()
10809 vcpu->run->exit_reason = KVM_EXIT_IOAPIC_EOI; in vcpu_enter_guest()
10810 vcpu->run->eoi.vector = in vcpu_enter_guest()
10811 vcpu->arch.pending_ioapic_eoi; in vcpu_enter_guest()
10816 if (kvm_check_request(KVM_REQ_SCAN_IOAPIC, vcpu)) in vcpu_enter_guest()
10817 vcpu_scan_ioapic(vcpu); in vcpu_enter_guest()
10818 if (kvm_check_request(KVM_REQ_LOAD_EOI_EXITMAP, vcpu)) in vcpu_enter_guest()
10819 vcpu_load_eoi_exitmap(vcpu); in vcpu_enter_guest()
10820 if (kvm_check_request(KVM_REQ_APIC_PAGE_RELOAD, vcpu)) in vcpu_enter_guest()
10821 kvm_vcpu_reload_apic_access_page(vcpu); in vcpu_enter_guest()
10823 if (kvm_check_request(KVM_REQ_HV_CRASH, vcpu)) { in vcpu_enter_guest()
10824 vcpu->run->exit_reason = KVM_EXIT_SYSTEM_EVENT; in vcpu_enter_guest()
10825 vcpu->run->system_event.type = KVM_SYSTEM_EVENT_CRASH; in vcpu_enter_guest()
10826 vcpu->run->system_event.ndata = 0; in vcpu_enter_guest()
10830 if (kvm_check_request(KVM_REQ_HV_RESET, vcpu)) { in vcpu_enter_guest()
10831 vcpu->run->exit_reason = KVM_EXIT_SYSTEM_EVENT; in vcpu_enter_guest()
10832 vcpu->run->system_event.type = KVM_SYSTEM_EVENT_RESET; in vcpu_enter_guest()
10833 vcpu->run->system_event.ndata = 0; in vcpu_enter_guest()
10837 if (kvm_check_request(KVM_REQ_HV_EXIT, vcpu)) { in vcpu_enter_guest()
10838 struct kvm_vcpu_hv *hv_vcpu = to_hv_vcpu(vcpu); in vcpu_enter_guest()
10840 vcpu->run->exit_reason = KVM_EXIT_HYPERV; in vcpu_enter_guest()
10841 vcpu->run->hyperv = hv_vcpu->exit; in vcpu_enter_guest()
10851 if (kvm_check_request(KVM_REQ_HV_STIMER, vcpu)) in vcpu_enter_guest()
10852 kvm_hv_process_stimers(vcpu); in vcpu_enter_guest()
10854 if (kvm_check_request(KVM_REQ_APICV_UPDATE, vcpu)) in vcpu_enter_guest()
10855 kvm_vcpu_update_apicv(vcpu); in vcpu_enter_guest()
10856 if (kvm_check_request(KVM_REQ_APF_READY, vcpu)) in vcpu_enter_guest()
10857 kvm_check_async_pf_completion(vcpu); in vcpu_enter_guest()
10858 if (kvm_check_request(KVM_REQ_MSR_FILTER_CHANGED, vcpu)) in vcpu_enter_guest()
10859 kvm_x86_call(msr_filter_changed)(vcpu); in vcpu_enter_guest()
10861 if (kvm_check_request(KVM_REQ_UPDATE_CPU_DIRTY_LOGGING, vcpu)) in vcpu_enter_guest()
10862 kvm_x86_call(update_cpu_dirty_logging)(vcpu); in vcpu_enter_guest()
10864 if (kvm_check_request(KVM_REQ_UPDATE_PROTECTED_GUEST_STATE, vcpu)) { in vcpu_enter_guest()
10865 kvm_vcpu_reset(vcpu, true); in vcpu_enter_guest()
10866 if (vcpu->arch.mp_state != KVM_MP_STATE_RUNNABLE) { in vcpu_enter_guest()
10873 if (kvm_check_request(KVM_REQ_EVENT, vcpu) || req_int_win || in vcpu_enter_guest()
10874 kvm_xen_has_interrupt(vcpu)) { in vcpu_enter_guest()
10875 ++vcpu->stat.req_event; in vcpu_enter_guest()
10876 r = kvm_apic_accept_events(vcpu); in vcpu_enter_guest()
10881 if (vcpu->arch.mp_state == KVM_MP_STATE_INIT_RECEIVED) { in vcpu_enter_guest()
10886 r = kvm_check_and_inject_events(vcpu, &req_immediate_exit); in vcpu_enter_guest()
10892 kvm_x86_call(enable_irq_window)(vcpu); in vcpu_enter_guest()
10894 if (kvm_lapic_enabled(vcpu)) { in vcpu_enter_guest()
10895 update_cr8_intercept(vcpu); in vcpu_enter_guest()
10896 kvm_lapic_sync_to_vapic(vcpu); in vcpu_enter_guest()
10900 r = kvm_mmu_reload(vcpu); in vcpu_enter_guest()
10907 kvm_x86_call(prepare_switch_to_guest)(vcpu); in vcpu_enter_guest()
10916 /* Store vcpu->apicv_active before vcpu->mode. */ in vcpu_enter_guest()
10917 smp_store_release(&vcpu->mode, IN_GUEST_MODE); in vcpu_enter_guest()
10919 kvm_vcpu_srcu_read_unlock(vcpu); in vcpu_enter_guest()
10930 * tables done while the VCPU is running. Please see the comment in vcpu_enter_guest()
10938 * target vCPU wasn't running). Do this regardless of the vCPU's APICv in vcpu_enter_guest()
10942 if (kvm_lapic_enabled(vcpu)) in vcpu_enter_guest()
10943 kvm_x86_call(sync_pir_to_irr)(vcpu); in vcpu_enter_guest()
10945 if (kvm_vcpu_exit_request(vcpu)) { in vcpu_enter_guest()
10946 vcpu->mode = OUTSIDE_GUEST_MODE; in vcpu_enter_guest()
10950 kvm_vcpu_srcu_read_lock(vcpu); in vcpu_enter_guest()
10956 kvm_make_request(KVM_REQ_EVENT, vcpu); in vcpu_enter_guest()
10962 if (vcpu->arch.guest_fpu.xfd_err) in vcpu_enter_guest()
10963 wrmsrl(MSR_IA32_XFD_ERR, vcpu->arch.guest_fpu.xfd_err); in vcpu_enter_guest()
10965 if (unlikely(vcpu->arch.switch_db_regs)) { in vcpu_enter_guest()
10967 set_debugreg(vcpu->arch.eff_db[0], 0); in vcpu_enter_guest()
10968 set_debugreg(vcpu->arch.eff_db[1], 1); in vcpu_enter_guest()
10969 set_debugreg(vcpu->arch.eff_db[2], 2); in vcpu_enter_guest()
10970 set_debugreg(vcpu->arch.eff_db[3], 3); in vcpu_enter_guest()
10979 * Assert that vCPU vs. VM APICv state is consistent. An APICv in vcpu_enter_guest()
10984 WARN_ON_ONCE((kvm_vcpu_apicv_activated(vcpu) != kvm_vcpu_apicv_active(vcpu)) && in vcpu_enter_guest()
10985 (kvm_get_apic_mode(vcpu) != LAPIC_MODE_DISABLED)); in vcpu_enter_guest()
10987 exit_fastpath = kvm_x86_call(vcpu_run)(vcpu, in vcpu_enter_guest()
10992 if (kvm_lapic_enabled(vcpu)) in vcpu_enter_guest()
10993 kvm_x86_call(sync_pir_to_irr)(vcpu); in vcpu_enter_guest()
10995 if (unlikely(kvm_vcpu_exit_request(vcpu))) { in vcpu_enter_guest()
11001 ++vcpu->stat.exits; in vcpu_enter_guest()
11010 if (unlikely(vcpu->arch.switch_db_regs & KVM_DEBUGREG_WONT_EXIT)) { in vcpu_enter_guest()
11011 WARN_ON(vcpu->guest_debug & KVM_GUESTDBG_USE_HW_BP); in vcpu_enter_guest()
11012 kvm_x86_call(sync_dirty_debug_regs)(vcpu); in vcpu_enter_guest()
11013 kvm_update_dr0123(vcpu); in vcpu_enter_guest()
11014 kvm_update_dr7(vcpu); in vcpu_enter_guest()
11027 vcpu->arch.last_vmentry_cpu = vcpu->cpu; in vcpu_enter_guest()
11028 vcpu->arch.last_guest_tsc = kvm_read_l1_tsc(vcpu, rdtsc()); in vcpu_enter_guest()
11030 vcpu->mode = OUTSIDE_GUEST_MODE; in vcpu_enter_guest()
11038 if (vcpu->arch.xfd_no_write_intercept) in vcpu_enter_guest()
11041 kvm_x86_call(handle_exit_irqoff)(vcpu); in vcpu_enter_guest()
11043 if (vcpu->arch.guest_fpu.xfd_err) in vcpu_enter_guest()
11053 kvm_before_interrupt(vcpu, KVM_HANDLING_IRQ); in vcpu_enter_guest()
11055 ++vcpu->stat.exits; in vcpu_enter_guest()
11057 kvm_after_interrupt(vcpu); in vcpu_enter_guest()
11071 kvm_vcpu_srcu_read_lock(vcpu); in vcpu_enter_guest()
11083 unsigned long rip = kvm_rip_read(vcpu); in vcpu_enter_guest()
11087 if (unlikely(vcpu->arch.tsc_always_catchup)) in vcpu_enter_guest()
11088 kvm_make_request(KVM_REQ_CLOCK_UPDATE, vcpu); in vcpu_enter_guest()
11090 if (vcpu->arch.apic_attention) in vcpu_enter_guest()
11091 kvm_lapic_sync_from_vapic(vcpu); in vcpu_enter_guest()
11096 r = kvm_x86_call(handle_exit)(vcpu, exit_fastpath); in vcpu_enter_guest()
11101 kvm_make_request(KVM_REQ_EVENT, vcpu); in vcpu_enter_guest()
11102 kvm_x86_call(cancel_injection)(vcpu); in vcpu_enter_guest()
11103 if (unlikely(vcpu->arch.apic_attention)) in vcpu_enter_guest()
11104 kvm_lapic_sync_from_vapic(vcpu); in vcpu_enter_guest()
11109 static bool kvm_vcpu_running(struct kvm_vcpu *vcpu) in kvm_vcpu_running() argument
11111 return (vcpu->arch.mp_state == KVM_MP_STATE_RUNNABLE && in kvm_vcpu_running()
11112 !vcpu->arch.apf.halted); in kvm_vcpu_running()
11115 static bool kvm_vcpu_has_events(struct kvm_vcpu *vcpu) in kvm_vcpu_has_events() argument
11117 if (!list_empty_careful(&vcpu->async_pf.done)) in kvm_vcpu_has_events()
11120 if (kvm_apic_has_pending_init_or_sipi(vcpu) && in kvm_vcpu_has_events()
11121 kvm_apic_init_sipi_allowed(vcpu)) in kvm_vcpu_has_events()
11124 if (vcpu->arch.pv.pv_unhalted) in kvm_vcpu_has_events()
11127 if (kvm_is_exception_pending(vcpu)) in kvm_vcpu_has_events()
11130 if (kvm_test_request(KVM_REQ_NMI, vcpu) || in kvm_vcpu_has_events()
11131 (vcpu->arch.nmi_pending && in kvm_vcpu_has_events()
11132 kvm_x86_call(nmi_allowed)(vcpu, false))) in kvm_vcpu_has_events()
11136 if (kvm_test_request(KVM_REQ_SMI, vcpu) || in kvm_vcpu_has_events()
11137 (vcpu->arch.smi_pending && in kvm_vcpu_has_events()
11138 kvm_x86_call(smi_allowed)(vcpu, false))) in kvm_vcpu_has_events()
11142 if (kvm_test_request(KVM_REQ_PMI, vcpu)) in kvm_vcpu_has_events()
11145 if (kvm_test_request(KVM_REQ_UPDATE_PROTECTED_GUEST_STATE, vcpu)) in kvm_vcpu_has_events()
11148 if (kvm_arch_interrupt_allowed(vcpu) && kvm_cpu_has_interrupt(vcpu)) in kvm_vcpu_has_events()
11151 if (kvm_hv_has_stimer_pending(vcpu)) in kvm_vcpu_has_events()
11154 if (is_guest_mode(vcpu) && in kvm_vcpu_has_events()
11156 kvm_x86_ops.nested_ops->has_events(vcpu, false)) in kvm_vcpu_has_events()
11159 if (kvm_xen_has_pending_events(vcpu)) in kvm_vcpu_has_events()
11165 int kvm_arch_vcpu_runnable(struct kvm_vcpu *vcpu) in kvm_arch_vcpu_runnable() argument
11167 return kvm_vcpu_running(vcpu) || kvm_vcpu_has_events(vcpu); in kvm_arch_vcpu_runnable()
11171 static inline int vcpu_block(struct kvm_vcpu *vcpu) in vcpu_block() argument
11175 if (!kvm_arch_vcpu_runnable(vcpu)) { in vcpu_block()
11178 * the guest's timer may be a break event for the vCPU, and the in vcpu_block()
11183 hv_timer = kvm_lapic_hv_timer_in_use(vcpu); in vcpu_block()
11185 kvm_lapic_switch_to_sw_timer(vcpu); in vcpu_block()
11187 kvm_vcpu_srcu_read_unlock(vcpu); in vcpu_block()
11188 if (vcpu->arch.mp_state == KVM_MP_STATE_HALTED) in vcpu_block()
11189 kvm_vcpu_halt(vcpu); in vcpu_block()
11191 kvm_vcpu_block(vcpu); in vcpu_block()
11192 kvm_vcpu_srcu_read_lock(vcpu); in vcpu_block()
11195 kvm_lapic_switch_to_hv_timer(vcpu); in vcpu_block()
11198 * If the vCPU is not runnable, a signal or another host event in vcpu_block()
11200 * vCPU's activity state. in vcpu_block()
11202 if (!kvm_arch_vcpu_runnable(vcpu)) in vcpu_block()
11212 if (is_guest_mode(vcpu)) { in vcpu_block()
11213 int r = kvm_check_nested_events(vcpu); in vcpu_block()
11220 if (kvm_apic_accept_events(vcpu) < 0) in vcpu_block()
11222 switch(vcpu->arch.mp_state) { in vcpu_block()
11225 vcpu->arch.pv.pv_unhalted = false; in vcpu_block()
11226 vcpu->arch.mp_state = in vcpu_block()
11230 vcpu->arch.apf.halted = false; in vcpu_block()
11242 static int vcpu_run(struct kvm_vcpu *vcpu) in vcpu_run() argument
11246 vcpu->run->exit_reason = KVM_EXIT_UNKNOWN; in vcpu_run()
11250 * If another guest vCPU requests a PV TLB flush in the middle in vcpu_run()
11255 vcpu->arch.at_instruction_boundary = false; in vcpu_run()
11256 if (kvm_vcpu_running(vcpu)) { in vcpu_run()
11257 r = vcpu_enter_guest(vcpu); in vcpu_run()
11259 r = vcpu_block(vcpu); in vcpu_run()
11265 kvm_clear_request(KVM_REQ_UNBLOCK, vcpu); in vcpu_run()
11266 if (kvm_xen_has_pending_events(vcpu)) in vcpu_run()
11267 kvm_xen_inject_pending_events(vcpu); in vcpu_run()
11269 if (kvm_cpu_has_pending_timer(vcpu)) in vcpu_run()
11270 kvm_inject_pending_timer_irqs(vcpu); in vcpu_run()
11272 if (dm_request_for_irq_injection(vcpu) && in vcpu_run()
11273 kvm_vcpu_ready_for_interrupt_injection(vcpu)) { in vcpu_run()
11275 vcpu->run->exit_reason = KVM_EXIT_IRQ_WINDOW_OPEN; in vcpu_run()
11276 ++vcpu->stat.request_irq_exits; in vcpu_run()
11281 kvm_vcpu_srcu_read_unlock(vcpu); in vcpu_run()
11282 r = xfer_to_guest_mode_handle_work(vcpu); in vcpu_run()
11283 kvm_vcpu_srcu_read_lock(vcpu); in vcpu_run()
11292 static int __kvm_emulate_halt(struct kvm_vcpu *vcpu, int state, int reason) in __kvm_emulate_halt() argument
11295 * The vCPU has halted, e.g. executed HLT. Update the run state if the in __kvm_emulate_halt()
11297 * state and halt the vCPU. Exit to userspace if the local APIC is in __kvm_emulate_halt()
11301 ++vcpu->stat.halt_exits; in __kvm_emulate_halt()
11302 if (lapic_in_kernel(vcpu)) { in __kvm_emulate_halt()
11303 if (kvm_vcpu_has_events(vcpu)) in __kvm_emulate_halt()
11304 vcpu->arch.pv.pv_unhalted = false; in __kvm_emulate_halt()
11306 vcpu->arch.mp_state = state; in __kvm_emulate_halt()
11309 vcpu->run->exit_reason = reason; in __kvm_emulate_halt()
11314 int kvm_emulate_halt_noskip(struct kvm_vcpu *vcpu) in kvm_emulate_halt_noskip() argument
11316 return __kvm_emulate_halt(vcpu, KVM_MP_STATE_HALTED, KVM_EXIT_HLT); in kvm_emulate_halt_noskip()
11320 int kvm_emulate_halt(struct kvm_vcpu *vcpu) in kvm_emulate_halt() argument
11322 int ret = kvm_skip_emulated_instruction(vcpu); in kvm_emulate_halt()
11327 return kvm_emulate_halt_noskip(vcpu) && ret; in kvm_emulate_halt()
11331 fastpath_t handle_fastpath_hlt(struct kvm_vcpu *vcpu) in handle_fastpath_hlt() argument
11335 kvm_vcpu_srcu_read_lock(vcpu); in handle_fastpath_hlt()
11336 ret = kvm_emulate_halt(vcpu); in handle_fastpath_hlt()
11337 kvm_vcpu_srcu_read_unlock(vcpu); in handle_fastpath_hlt()
11342 if (kvm_vcpu_running(vcpu)) in handle_fastpath_hlt()
11349 int kvm_emulate_ap_reset_hold(struct kvm_vcpu *vcpu) in kvm_emulate_ap_reset_hold() argument
11351 int ret = kvm_skip_emulated_instruction(vcpu); in kvm_emulate_ap_reset_hold()
11353 return __kvm_emulate_halt(vcpu, KVM_MP_STATE_AP_RESET_HOLD, in kvm_emulate_ap_reset_hold()
11358 bool kvm_arch_dy_has_pending_interrupt(struct kvm_vcpu *vcpu) in kvm_arch_dy_has_pending_interrupt() argument
11360 return kvm_vcpu_apicv_active(vcpu) && in kvm_arch_dy_has_pending_interrupt()
11361 kvm_x86_call(dy_apicv_has_pending_interrupt)(vcpu); in kvm_arch_dy_has_pending_interrupt()
11364 bool kvm_arch_vcpu_preempted_in_kernel(struct kvm_vcpu *vcpu) in kvm_arch_vcpu_preempted_in_kernel() argument
11366 return vcpu->arch.preempted_in_kernel; in kvm_arch_vcpu_preempted_in_kernel()
11369 bool kvm_arch_dy_runnable(struct kvm_vcpu *vcpu) in kvm_arch_dy_runnable() argument
11371 if (READ_ONCE(vcpu->arch.pv.pv_unhalted)) in kvm_arch_dy_runnable()
11374 if (kvm_test_request(KVM_REQ_NMI, vcpu) || in kvm_arch_dy_runnable()
11376 kvm_test_request(KVM_REQ_SMI, vcpu) || in kvm_arch_dy_runnable()
11378 kvm_test_request(KVM_REQ_EVENT, vcpu)) in kvm_arch_dy_runnable()
11381 return kvm_arch_dy_has_pending_interrupt(vcpu); in kvm_arch_dy_runnable()
11384 static inline int complete_emulated_io(struct kvm_vcpu *vcpu) in complete_emulated_io() argument
11386 return kvm_emulate_instruction(vcpu, EMULTYPE_NO_DECODE); in complete_emulated_io()
11389 static int complete_emulated_pio(struct kvm_vcpu *vcpu) in complete_emulated_pio() argument
11391 BUG_ON(!vcpu->arch.pio.count); in complete_emulated_pio()
11393 return complete_emulated_io(vcpu); in complete_emulated_pio()
11414 static int complete_emulated_mmio(struct kvm_vcpu *vcpu) in complete_emulated_mmio() argument
11416 struct kvm_run *run = vcpu->run; in complete_emulated_mmio()
11420 BUG_ON(!vcpu->mmio_needed); in complete_emulated_mmio()
11423 frag = &vcpu->mmio_fragments[vcpu->mmio_cur_fragment]; in complete_emulated_mmio()
11425 if (!vcpu->mmio_is_write) in complete_emulated_mmio()
11431 vcpu->mmio_cur_fragment++; in complete_emulated_mmio()
11439 if (vcpu->mmio_cur_fragment >= vcpu->mmio_nr_fragments) { in complete_emulated_mmio()
11440 vcpu->mmio_needed = 0; in complete_emulated_mmio()
11443 if (vcpu->mmio_is_write) in complete_emulated_mmio()
11445 vcpu->mmio_read_completed = 1; in complete_emulated_mmio()
11446 return complete_emulated_io(vcpu); in complete_emulated_mmio()
11451 if (vcpu->mmio_is_write) in complete_emulated_mmio()
11454 run->mmio.is_write = vcpu->mmio_is_write; in complete_emulated_mmio()
11455 vcpu->arch.complete_userspace_io = complete_emulated_mmio; in complete_emulated_mmio()
11460 static void kvm_load_guest_fpu(struct kvm_vcpu *vcpu) in kvm_load_guest_fpu() argument
11463 fpu_swap_kvm_fpstate(&vcpu->arch.guest_fpu, true); in kvm_load_guest_fpu()
11468 static void kvm_put_guest_fpu(struct kvm_vcpu *vcpu) in kvm_put_guest_fpu() argument
11470 fpu_swap_kvm_fpstate(&vcpu->arch.guest_fpu, false); in kvm_put_guest_fpu()
11471 ++vcpu->stat.fpu_reload; in kvm_put_guest_fpu()
11475 int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu) in kvm_arch_vcpu_ioctl_run() argument
11477 struct kvm_queued_exception *ex = &vcpu->arch.exception; in kvm_arch_vcpu_ioctl_run()
11478 struct kvm_run *kvm_run = vcpu->run; in kvm_arch_vcpu_ioctl_run()
11481 vcpu_load(vcpu); in kvm_arch_vcpu_ioctl_run()
11482 kvm_sigset_activate(vcpu); in kvm_arch_vcpu_ioctl_run()
11484 kvm_load_guest_fpu(vcpu); in kvm_arch_vcpu_ioctl_run()
11486 kvm_vcpu_srcu_read_lock(vcpu); in kvm_arch_vcpu_ioctl_run()
11487 if (unlikely(vcpu->arch.mp_state == KVM_MP_STATE_UNINITIALIZED)) { in kvm_arch_vcpu_ioctl_run()
11488 if (!vcpu->wants_to_run) { in kvm_arch_vcpu_ioctl_run()
11496 * APIC timer to be active is if userspace stuffed vCPU state, in kvm_arch_vcpu_ioctl_run()
11497 * i.e. put the vCPU into a nonsensical state. Only an INIT in kvm_arch_vcpu_ioctl_run()
11498 * will transition the vCPU out of UNINITIALIZED (without more in kvm_arch_vcpu_ioctl_run()
11503 kvm_vcpu_srcu_read_unlock(vcpu); in kvm_arch_vcpu_ioctl_run()
11504 kvm_vcpu_block(vcpu); in kvm_arch_vcpu_ioctl_run()
11505 kvm_vcpu_srcu_read_lock(vcpu); in kvm_arch_vcpu_ioctl_run()
11507 if (kvm_apic_accept_events(vcpu) < 0) { in kvm_arch_vcpu_ioctl_run()
11515 ++vcpu->stat.signal_exits; in kvm_arch_vcpu_ioctl_run()
11527 r = sync_regs(vcpu); in kvm_arch_vcpu_ioctl_run()
11533 if (!lapic_in_kernel(vcpu)) { in kvm_arch_vcpu_ioctl_run()
11534 if (kvm_set_cr8(vcpu, kvm_run->cr8) != 0) { in kvm_arch_vcpu_ioctl_run()
11544 if (vcpu->arch.exception_from_userspace && is_guest_mode(vcpu) && in kvm_arch_vcpu_ioctl_run()
11545 kvm_x86_ops.nested_ops->is_exception_vmexit(vcpu, ex->vector, in kvm_arch_vcpu_ioctl_run()
11547 kvm_queue_exception_vmexit(vcpu, ex->vector, in kvm_arch_vcpu_ioctl_run()
11553 vcpu->arch.exception_from_userspace = false; in kvm_arch_vcpu_ioctl_run()
11555 if (unlikely(vcpu->arch.complete_userspace_io)) { in kvm_arch_vcpu_ioctl_run()
11556 int (*cui)(struct kvm_vcpu *) = vcpu->arch.complete_userspace_io; in kvm_arch_vcpu_ioctl_run()
11557 vcpu->arch.complete_userspace_io = NULL; in kvm_arch_vcpu_ioctl_run()
11558 r = cui(vcpu); in kvm_arch_vcpu_ioctl_run()
11562 WARN_ON_ONCE(vcpu->arch.pio.count); in kvm_arch_vcpu_ioctl_run()
11563 WARN_ON_ONCE(vcpu->mmio_needed); in kvm_arch_vcpu_ioctl_run()
11566 if (!vcpu->wants_to_run) { in kvm_arch_vcpu_ioctl_run()
11571 r = kvm_x86_call(vcpu_pre_run)(vcpu); in kvm_arch_vcpu_ioctl_run()
11575 r = vcpu_run(vcpu); in kvm_arch_vcpu_ioctl_run()
11578 kvm_put_guest_fpu(vcpu); in kvm_arch_vcpu_ioctl_run()
11580 store_regs(vcpu); in kvm_arch_vcpu_ioctl_run()
11581 post_kvm_run_save(vcpu); in kvm_arch_vcpu_ioctl_run()
11582 kvm_vcpu_srcu_read_unlock(vcpu); in kvm_arch_vcpu_ioctl_run()
11584 kvm_sigset_deactivate(vcpu); in kvm_arch_vcpu_ioctl_run()
11585 vcpu_put(vcpu); in kvm_arch_vcpu_ioctl_run()
11589 static void __get_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs) in __get_regs() argument
11591 if (vcpu->arch.emulate_regs_need_sync_to_vcpu) { in __get_regs()
11595 * back from emulation context to vcpu. Userspace shouldn't do in __get_regs()
11599 emulator_writeback_register_cache(vcpu->arch.emulate_ctxt); in __get_regs()
11600 vcpu->arch.emulate_regs_need_sync_to_vcpu = false; in __get_regs()
11602 regs->rax = kvm_rax_read(vcpu); in __get_regs()
11603 regs->rbx = kvm_rbx_read(vcpu); in __get_regs()
11604 regs->rcx = kvm_rcx_read(vcpu); in __get_regs()
11605 regs->rdx = kvm_rdx_read(vcpu); in __get_regs()
11606 regs->rsi = kvm_rsi_read(vcpu); in __get_regs()
11607 regs->rdi = kvm_rdi_read(vcpu); in __get_regs()
11608 regs->rsp = kvm_rsp_read(vcpu); in __get_regs()
11609 regs->rbp = kvm_rbp_read(vcpu); in __get_regs()
11611 regs->r8 = kvm_r8_read(vcpu); in __get_regs()
11612 regs->r9 = kvm_r9_read(vcpu); in __get_regs()
11613 regs->r10 = kvm_r10_read(vcpu); in __get_regs()
11614 regs->r11 = kvm_r11_read(vcpu); in __get_regs()
11615 regs->r12 = kvm_r12_read(vcpu); in __get_regs()
11616 regs->r13 = kvm_r13_read(vcpu); in __get_regs()
11617 regs->r14 = kvm_r14_read(vcpu); in __get_regs()
11618 regs->r15 = kvm_r15_read(vcpu); in __get_regs()
11621 regs->rip = kvm_rip_read(vcpu); in __get_regs()
11622 regs->rflags = kvm_get_rflags(vcpu); in __get_regs()
11625 int kvm_arch_vcpu_ioctl_get_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs) in kvm_arch_vcpu_ioctl_get_regs() argument
11627 if (vcpu->kvm->arch.has_protected_state && in kvm_arch_vcpu_ioctl_get_regs()
11628 vcpu->arch.guest_state_protected) in kvm_arch_vcpu_ioctl_get_regs()
11631 vcpu_load(vcpu); in kvm_arch_vcpu_ioctl_get_regs()
11632 __get_regs(vcpu, regs); in kvm_arch_vcpu_ioctl_get_regs()
11633 vcpu_put(vcpu); in kvm_arch_vcpu_ioctl_get_regs()
11637 static void __set_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs) in __set_regs() argument
11639 vcpu->arch.emulate_regs_need_sync_from_vcpu = true; in __set_regs()
11640 vcpu->arch.emulate_regs_need_sync_to_vcpu = false; in __set_regs()
11642 kvm_rax_write(vcpu, regs->rax); in __set_regs()
11643 kvm_rbx_write(vcpu, regs->rbx); in __set_regs()
11644 kvm_rcx_write(vcpu, regs->rcx); in __set_regs()
11645 kvm_rdx_write(vcpu, regs->rdx); in __set_regs()
11646 kvm_rsi_write(vcpu, regs->rsi); in __set_regs()
11647 kvm_rdi_write(vcpu, regs->rdi); in __set_regs()
11648 kvm_rsp_write(vcpu, regs->rsp); in __set_regs()
11649 kvm_rbp_write(vcpu, regs->rbp); in __set_regs()
11651 kvm_r8_write(vcpu, regs->r8); in __set_regs()
11652 kvm_r9_write(vcpu, regs->r9); in __set_regs()
11653 kvm_r10_write(vcpu, regs->r10); in __set_regs()
11654 kvm_r11_write(vcpu, regs->r11); in __set_regs()
11655 kvm_r12_write(vcpu, regs->r12); in __set_regs()
11656 kvm_r13_write(vcpu, regs->r13); in __set_regs()
11657 kvm_r14_write(vcpu, regs->r14); in __set_regs()
11658 kvm_r15_write(vcpu, regs->r15); in __set_regs()
11661 kvm_rip_write(vcpu, regs->rip); in __set_regs()
11662 kvm_set_rflags(vcpu, regs->rflags | X86_EFLAGS_FIXED); in __set_regs()
11664 vcpu->arch.exception.pending = false; in __set_regs()
11665 vcpu->arch.exception_vmexit.pending = false; in __set_regs()
11667 kvm_make_request(KVM_REQ_EVENT, vcpu); in __set_regs()
11670 int kvm_arch_vcpu_ioctl_set_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs) in kvm_arch_vcpu_ioctl_set_regs() argument
11672 if (vcpu->kvm->arch.has_protected_state && in kvm_arch_vcpu_ioctl_set_regs()
11673 vcpu->arch.guest_state_protected) in kvm_arch_vcpu_ioctl_set_regs()
11676 vcpu_load(vcpu); in kvm_arch_vcpu_ioctl_set_regs()
11677 __set_regs(vcpu, regs); in kvm_arch_vcpu_ioctl_set_regs()
11678 vcpu_put(vcpu); in kvm_arch_vcpu_ioctl_set_regs()
11682 static void __get_sregs_common(struct kvm_vcpu *vcpu, struct kvm_sregs *sregs) in __get_sregs_common() argument
11686 if (vcpu->arch.guest_state_protected) in __get_sregs_common()
11689 kvm_get_segment(vcpu, &sregs->cs, VCPU_SREG_CS); in __get_sregs_common()
11690 kvm_get_segment(vcpu, &sregs->ds, VCPU_SREG_DS); in __get_sregs_common()
11691 kvm_get_segment(vcpu, &sregs->es, VCPU_SREG_ES); in __get_sregs_common()
11692 kvm_get_segment(vcpu, &sregs->fs, VCPU_SREG_FS); in __get_sregs_common()
11693 kvm_get_segment(vcpu, &sregs->gs, VCPU_SREG_GS); in __get_sregs_common()
11694 kvm_get_segment(vcpu, &sregs->ss, VCPU_SREG_SS); in __get_sregs_common()
11696 kvm_get_segment(vcpu, &sregs->tr, VCPU_SREG_TR); in __get_sregs_common()
11697 kvm_get_segment(vcpu, &sregs->ldt, VCPU_SREG_LDTR); in __get_sregs_common()
11699 kvm_x86_call(get_idt)(vcpu, &dt); in __get_sregs_common()
11702 kvm_x86_call(get_gdt)(vcpu, &dt); in __get_sregs_common()
11706 sregs->cr2 = vcpu->arch.cr2; in __get_sregs_common()
11707 sregs->cr3 = kvm_read_cr3(vcpu); in __get_sregs_common()
11710 sregs->cr0 = kvm_read_cr0(vcpu); in __get_sregs_common()
11711 sregs->cr4 = kvm_read_cr4(vcpu); in __get_sregs_common()
11712 sregs->cr8 = kvm_get_cr8(vcpu); in __get_sregs_common()
11713 sregs->efer = vcpu->arch.efer; in __get_sregs_common()
11714 sregs->apic_base = kvm_get_apic_base(vcpu); in __get_sregs_common()
11717 static void __get_sregs(struct kvm_vcpu *vcpu, struct kvm_sregs *sregs) in __get_sregs() argument
11719 __get_sregs_common(vcpu, sregs); in __get_sregs()
11721 if (vcpu->arch.guest_state_protected) in __get_sregs()
11724 if (vcpu->arch.interrupt.injected && !vcpu->arch.interrupt.soft) in __get_sregs()
11725 set_bit(vcpu->arch.interrupt.nr, in __get_sregs()
11729 static void __get_sregs2(struct kvm_vcpu *vcpu, struct kvm_sregs2 *sregs2) in __get_sregs2() argument
11733 __get_sregs_common(vcpu, (struct kvm_sregs *)sregs2); in __get_sregs2()
11735 if (vcpu->arch.guest_state_protected) in __get_sregs2()
11738 if (is_pae_paging(vcpu)) { in __get_sregs2()
11740 sregs2->pdptrs[i] = kvm_pdptr_read(vcpu, i); in __get_sregs2()
11745 int kvm_arch_vcpu_ioctl_get_sregs(struct kvm_vcpu *vcpu, in kvm_arch_vcpu_ioctl_get_sregs() argument
11748 if (vcpu->kvm->arch.has_protected_state && in kvm_arch_vcpu_ioctl_get_sregs()
11749 vcpu->arch.guest_state_protected) in kvm_arch_vcpu_ioctl_get_sregs()
11752 vcpu_load(vcpu); in kvm_arch_vcpu_ioctl_get_sregs()
11753 __get_sregs(vcpu, sregs); in kvm_arch_vcpu_ioctl_get_sregs()
11754 vcpu_put(vcpu); in kvm_arch_vcpu_ioctl_get_sregs()
11758 int kvm_arch_vcpu_ioctl_get_mpstate(struct kvm_vcpu *vcpu, in kvm_arch_vcpu_ioctl_get_mpstate() argument
11763 vcpu_load(vcpu); in kvm_arch_vcpu_ioctl_get_mpstate()
11765 kvm_load_guest_fpu(vcpu); in kvm_arch_vcpu_ioctl_get_mpstate()
11767 r = kvm_apic_accept_events(vcpu); in kvm_arch_vcpu_ioctl_get_mpstate()
11772 if ((vcpu->arch.mp_state == KVM_MP_STATE_HALTED || in kvm_arch_vcpu_ioctl_get_mpstate()
11773 vcpu->arch.mp_state == KVM_MP_STATE_AP_RESET_HOLD) && in kvm_arch_vcpu_ioctl_get_mpstate()
11774 vcpu->arch.pv.pv_unhalted) in kvm_arch_vcpu_ioctl_get_mpstate()
11777 mp_state->mp_state = vcpu->arch.mp_state; in kvm_arch_vcpu_ioctl_get_mpstate()
11781 kvm_put_guest_fpu(vcpu); in kvm_arch_vcpu_ioctl_get_mpstate()
11782 vcpu_put(vcpu); in kvm_arch_vcpu_ioctl_get_mpstate()
11786 int kvm_arch_vcpu_ioctl_set_mpstate(struct kvm_vcpu *vcpu, in kvm_arch_vcpu_ioctl_set_mpstate() argument
11791 vcpu_load(vcpu); in kvm_arch_vcpu_ioctl_set_mpstate()
11799 if (!lapic_in_kernel(vcpu)) in kvm_arch_vcpu_ioctl_set_mpstate()
11816 if ((!kvm_apic_init_sipi_allowed(vcpu) || vcpu->arch.smi_pending) && in kvm_arch_vcpu_ioctl_set_mpstate()
11822 vcpu->arch.mp_state = KVM_MP_STATE_INIT_RECEIVED; in kvm_arch_vcpu_ioctl_set_mpstate()
11823 set_bit(KVM_APIC_SIPI, &vcpu->arch.apic->pending_events); in kvm_arch_vcpu_ioctl_set_mpstate()
11825 vcpu->arch.mp_state = mp_state->mp_state; in kvm_arch_vcpu_ioctl_set_mpstate()
11826 kvm_make_request(KVM_REQ_EVENT, vcpu); in kvm_arch_vcpu_ioctl_set_mpstate()
11830 vcpu_put(vcpu); in kvm_arch_vcpu_ioctl_set_mpstate()
11834 int kvm_task_switch(struct kvm_vcpu *vcpu, u16 tss_selector, int idt_index, in kvm_task_switch() argument
11837 struct x86_emulate_ctxt *ctxt = vcpu->arch.emulate_ctxt; in kvm_task_switch()
11840 init_emulate_ctxt(vcpu); in kvm_task_switch()
11849 if (ret || vcpu->mmio_needed) { in kvm_task_switch()
11850 vcpu->mmio_needed = false; in kvm_task_switch()
11851 vcpu->run->exit_reason = KVM_EXIT_INTERNAL_ERROR; in kvm_task_switch()
11852 vcpu->run->internal.suberror = KVM_INTERNAL_ERROR_EMULATION; in kvm_task_switch()
11853 vcpu->run->internal.ndata = 0; in kvm_task_switch()
11857 kvm_rip_write(vcpu, ctxt->eip); in kvm_task_switch()
11858 kvm_set_rflags(vcpu, ctxt->eflags); in kvm_task_switch()
11863 static bool kvm_is_valid_sregs(struct kvm_vcpu *vcpu, struct kvm_sregs *sregs) in kvm_is_valid_sregs() argument
11873 if (!kvm_vcpu_is_legal_cr3(vcpu, sregs->cr3)) in kvm_is_valid_sregs()
11884 return kvm_is_valid_cr4(vcpu, sregs->cr4) && in kvm_is_valid_sregs()
11885 kvm_is_valid_cr0(vcpu, sregs->cr0); in kvm_is_valid_sregs()
11888 static int __set_sregs_common(struct kvm_vcpu *vcpu, struct kvm_sregs *sregs, in __set_sregs_common() argument
11895 if (!kvm_is_valid_sregs(vcpu, sregs)) in __set_sregs_common()
11900 if (kvm_set_apic_base(vcpu, &apic_base_msr)) in __set_sregs_common()
11903 if (vcpu->arch.guest_state_protected) in __set_sregs_common()
11908 kvm_x86_call(set_idt)(vcpu, &dt); in __set_sregs_common()
11911 kvm_x86_call(set_gdt)(vcpu, &dt); in __set_sregs_common()
11913 vcpu->arch.cr2 = sregs->cr2; in __set_sregs_common()
11914 *mmu_reset_needed |= kvm_read_cr3(vcpu) != sregs->cr3; in __set_sregs_common()
11915 vcpu->arch.cr3 = sregs->cr3; in __set_sregs_common()
11916 kvm_register_mark_dirty(vcpu, VCPU_EXREG_CR3); in __set_sregs_common()
11917 kvm_x86_call(post_set_cr3)(vcpu, sregs->cr3); in __set_sregs_common()
11919 kvm_set_cr8(vcpu, sregs->cr8); in __set_sregs_common()
11921 *mmu_reset_needed |= vcpu->arch.efer != sregs->efer; in __set_sregs_common()
11922 kvm_x86_call(set_efer)(vcpu, sregs->efer); in __set_sregs_common()
11924 *mmu_reset_needed |= kvm_read_cr0(vcpu) != sregs->cr0; in __set_sregs_common()
11925 kvm_x86_call(set_cr0)(vcpu, sregs->cr0); in __set_sregs_common()
11927 *mmu_reset_needed |= kvm_read_cr4(vcpu) != sregs->cr4; in __set_sregs_common()
11928 kvm_x86_call(set_cr4)(vcpu, sregs->cr4); in __set_sregs_common()
11931 idx = srcu_read_lock(&vcpu->kvm->srcu); in __set_sregs_common()
11932 if (is_pae_paging(vcpu)) { in __set_sregs_common()
11933 load_pdptrs(vcpu, kvm_read_cr3(vcpu)); in __set_sregs_common()
11936 srcu_read_unlock(&vcpu->kvm->srcu, idx); in __set_sregs_common()
11939 kvm_set_segment(vcpu, &sregs->cs, VCPU_SREG_CS); in __set_sregs_common()
11940 kvm_set_segment(vcpu, &sregs->ds, VCPU_SREG_DS); in __set_sregs_common()
11941 kvm_set_segment(vcpu, &sregs->es, VCPU_SREG_ES); in __set_sregs_common()
11942 kvm_set_segment(vcpu, &sregs->fs, VCPU_SREG_FS); in __set_sregs_common()
11943 kvm_set_segment(vcpu, &sregs->gs, VCPU_SREG_GS); in __set_sregs_common()
11944 kvm_set_segment(vcpu, &sregs->ss, VCPU_SREG_SS); in __set_sregs_common()
11946 kvm_set_segment(vcpu, &sregs->tr, VCPU_SREG_TR); in __set_sregs_common()
11947 kvm_set_segment(vcpu, &sregs->ldt, VCPU_SREG_LDTR); in __set_sregs_common()
11949 update_cr8_intercept(vcpu); in __set_sregs_common()
11951 /* Older userspace won't unhalt the vcpu on reset. */ in __set_sregs_common()
11952 if (kvm_vcpu_is_bsp(vcpu) && kvm_rip_read(vcpu) == 0xfff0 && in __set_sregs_common()
11954 !is_protmode(vcpu)) in __set_sregs_common()
11955 vcpu->arch.mp_state = KVM_MP_STATE_RUNNABLE; in __set_sregs_common()
11960 static int __set_sregs(struct kvm_vcpu *vcpu, struct kvm_sregs *sregs) in __set_sregs() argument
11964 int ret = __set_sregs_common(vcpu, sregs, &mmu_reset_needed, true); in __set_sregs()
11970 kvm_mmu_reset_context(vcpu); in __set_sregs()
11971 kvm_make_request(KVM_REQ_TLB_FLUSH_GUEST, vcpu); in __set_sregs()
11979 kvm_queue_interrupt(vcpu, pending_vec, false); in __set_sregs()
11981 kvm_make_request(KVM_REQ_EVENT, vcpu); in __set_sregs()
11986 static int __set_sregs2(struct kvm_vcpu *vcpu, struct kvm_sregs2 *sregs2) in __set_sregs2() argument
11997 if (valid_pdptrs && (!pae || vcpu->arch.guest_state_protected)) in __set_sregs2()
12000 ret = __set_sregs_common(vcpu, (struct kvm_sregs *)sregs2, in __set_sregs2()
12007 kvm_pdptr_write(vcpu, i, sregs2->pdptrs[i]); in __set_sregs2()
12009 kvm_register_mark_dirty(vcpu, VCPU_EXREG_PDPTR); in __set_sregs2()
12011 vcpu->arch.pdptrs_from_userspace = true; in __set_sregs2()
12014 kvm_mmu_reset_context(vcpu); in __set_sregs2()
12015 kvm_make_request(KVM_REQ_TLB_FLUSH_GUEST, vcpu); in __set_sregs2()
12020 int kvm_arch_vcpu_ioctl_set_sregs(struct kvm_vcpu *vcpu, in kvm_arch_vcpu_ioctl_set_sregs() argument
12025 if (vcpu->kvm->arch.has_protected_state && in kvm_arch_vcpu_ioctl_set_sregs()
12026 vcpu->arch.guest_state_protected) in kvm_arch_vcpu_ioctl_set_sregs()
12029 vcpu_load(vcpu); in kvm_arch_vcpu_ioctl_set_sregs()
12030 ret = __set_sregs(vcpu, sregs); in kvm_arch_vcpu_ioctl_set_sregs()
12031 vcpu_put(vcpu); in kvm_arch_vcpu_ioctl_set_sregs()
12038 struct kvm_vcpu *vcpu; in kvm_arch_vcpu_guestdbg_update_apicv_inhibit() local
12046 kvm_for_each_vcpu(i, vcpu, kvm) { in kvm_arch_vcpu_guestdbg_update_apicv_inhibit()
12047 if (vcpu->guest_debug & KVM_GUESTDBG_BLOCKIRQ) { in kvm_arch_vcpu_guestdbg_update_apicv_inhibit()
12056 int kvm_arch_vcpu_ioctl_set_guest_debug(struct kvm_vcpu *vcpu, in kvm_arch_vcpu_ioctl_set_guest_debug() argument
12062 if (vcpu->arch.guest_state_protected) in kvm_arch_vcpu_ioctl_set_guest_debug()
12065 vcpu_load(vcpu); in kvm_arch_vcpu_ioctl_set_guest_debug()
12069 if (kvm_is_exception_pending(vcpu)) in kvm_arch_vcpu_ioctl_set_guest_debug()
12072 kvm_queue_exception(vcpu, DB_VECTOR); in kvm_arch_vcpu_ioctl_set_guest_debug()
12074 kvm_queue_exception(vcpu, BP_VECTOR); in kvm_arch_vcpu_ioctl_set_guest_debug()
12081 rflags = kvm_get_rflags(vcpu); in kvm_arch_vcpu_ioctl_set_guest_debug()
12083 vcpu->guest_debug = dbg->control; in kvm_arch_vcpu_ioctl_set_guest_debug()
12084 if (!(vcpu->guest_debug & KVM_GUESTDBG_ENABLE)) in kvm_arch_vcpu_ioctl_set_guest_debug()
12085 vcpu->guest_debug = 0; in kvm_arch_vcpu_ioctl_set_guest_debug()
12087 if (vcpu->guest_debug & KVM_GUESTDBG_USE_HW_BP) { in kvm_arch_vcpu_ioctl_set_guest_debug()
12089 vcpu->arch.eff_db[i] = dbg->arch.debugreg[i]; in kvm_arch_vcpu_ioctl_set_guest_debug()
12090 vcpu->arch.guest_debug_dr7 = dbg->arch.debugreg[7]; in kvm_arch_vcpu_ioctl_set_guest_debug()
12093 vcpu->arch.eff_db[i] = vcpu->arch.db[i]; in kvm_arch_vcpu_ioctl_set_guest_debug()
12095 kvm_update_dr7(vcpu); in kvm_arch_vcpu_ioctl_set_guest_debug()
12097 if (vcpu->guest_debug & KVM_GUESTDBG_SINGLESTEP) in kvm_arch_vcpu_ioctl_set_guest_debug()
12098 vcpu->arch.singlestep_rip = kvm_get_linear_rip(vcpu); in kvm_arch_vcpu_ioctl_set_guest_debug()
12104 kvm_set_rflags(vcpu, rflags); in kvm_arch_vcpu_ioctl_set_guest_debug()
12106 kvm_x86_call(update_exception_bitmap)(vcpu); in kvm_arch_vcpu_ioctl_set_guest_debug()
12108 kvm_arch_vcpu_guestdbg_update_apicv_inhibit(vcpu->kvm); in kvm_arch_vcpu_ioctl_set_guest_debug()
12113 vcpu_put(vcpu); in kvm_arch_vcpu_ioctl_set_guest_debug()
12120 int kvm_arch_vcpu_ioctl_translate(struct kvm_vcpu *vcpu, in kvm_arch_vcpu_ioctl_translate() argument
12127 vcpu_load(vcpu); in kvm_arch_vcpu_ioctl_translate()
12129 idx = srcu_read_lock(&vcpu->kvm->srcu); in kvm_arch_vcpu_ioctl_translate()
12130 gpa = kvm_mmu_gva_to_gpa_system(vcpu, vaddr, NULL); in kvm_arch_vcpu_ioctl_translate()
12131 srcu_read_unlock(&vcpu->kvm->srcu, idx); in kvm_arch_vcpu_ioctl_translate()
12137 vcpu_put(vcpu); in kvm_arch_vcpu_ioctl_translate()
12141 int kvm_arch_vcpu_ioctl_get_fpu(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu) in kvm_arch_vcpu_ioctl_get_fpu() argument
12145 if (fpstate_is_confidential(&vcpu->arch.guest_fpu)) in kvm_arch_vcpu_ioctl_get_fpu()
12146 return vcpu->kvm->arch.has_protected_state ? -EINVAL : 0; in kvm_arch_vcpu_ioctl_get_fpu()
12148 vcpu_load(vcpu); in kvm_arch_vcpu_ioctl_get_fpu()
12150 fxsave = &vcpu->arch.guest_fpu.fpstate->regs.fxsave; in kvm_arch_vcpu_ioctl_get_fpu()
12160 vcpu_put(vcpu); in kvm_arch_vcpu_ioctl_get_fpu()
12164 int kvm_arch_vcpu_ioctl_set_fpu(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu) in kvm_arch_vcpu_ioctl_set_fpu() argument
12168 if (fpstate_is_confidential(&vcpu->arch.guest_fpu)) in kvm_arch_vcpu_ioctl_set_fpu()
12169 return vcpu->kvm->arch.has_protected_state ? -EINVAL : 0; in kvm_arch_vcpu_ioctl_set_fpu()
12171 vcpu_load(vcpu); in kvm_arch_vcpu_ioctl_set_fpu()
12173 fxsave = &vcpu->arch.guest_fpu.fpstate->regs.fxsave; in kvm_arch_vcpu_ioctl_set_fpu()
12184 vcpu_put(vcpu); in kvm_arch_vcpu_ioctl_set_fpu()
12188 static void store_regs(struct kvm_vcpu *vcpu) in store_regs() argument
12192 if (vcpu->run->kvm_valid_regs & KVM_SYNC_X86_REGS) in store_regs()
12193 __get_regs(vcpu, &vcpu->run->s.regs.regs); in store_regs()
12195 if (vcpu->run->kvm_valid_regs & KVM_SYNC_X86_SREGS) in store_regs()
12196 __get_sregs(vcpu, &vcpu->run->s.regs.sregs); in store_regs()
12198 if (vcpu->run->kvm_valid_regs & KVM_SYNC_X86_EVENTS) in store_regs()
12200 vcpu, &vcpu->run->s.regs.events); in store_regs()
12203 static int sync_regs(struct kvm_vcpu *vcpu) in sync_regs() argument
12205 if (vcpu->run->kvm_dirty_regs & KVM_SYNC_X86_REGS) { in sync_regs()
12206 __set_regs(vcpu, &vcpu->run->s.regs.regs); in sync_regs()
12207 vcpu->run->kvm_dirty_regs &= ~KVM_SYNC_X86_REGS; in sync_regs()
12210 if (vcpu->run->kvm_dirty_regs & KVM_SYNC_X86_SREGS) { in sync_regs()
12211 struct kvm_sregs sregs = vcpu->run->s.regs.sregs; in sync_regs()
12213 if (__set_sregs(vcpu, &sregs)) in sync_regs()
12216 vcpu->run->kvm_dirty_regs &= ~KVM_SYNC_X86_SREGS; in sync_regs()
12219 if (vcpu->run->kvm_dirty_regs & KVM_SYNC_X86_EVENTS) { in sync_regs()
12220 struct kvm_vcpu_events events = vcpu->run->s.regs.events; in sync_regs()
12222 if (kvm_vcpu_ioctl_x86_set_vcpu_events(vcpu, &events)) in sync_regs()
12225 vcpu->run->kvm_dirty_regs &= ~KVM_SYNC_X86_EVENTS; in sync_regs()
12246 int kvm_arch_vcpu_create(struct kvm_vcpu *vcpu) in kvm_arch_vcpu_create() argument
12251 vcpu->arch.last_vmentry_cpu = -1; in kvm_arch_vcpu_create()
12252 vcpu->arch.regs_avail = ~0; in kvm_arch_vcpu_create()
12253 vcpu->arch.regs_dirty = ~0; in kvm_arch_vcpu_create()
12255 kvm_gpc_init(&vcpu->arch.pv_time, vcpu->kvm); in kvm_arch_vcpu_create()
12257 if (!irqchip_in_kernel(vcpu->kvm) || kvm_vcpu_is_reset_bsp(vcpu)) in kvm_arch_vcpu_create()
12258 vcpu->arch.mp_state = KVM_MP_STATE_RUNNABLE; in kvm_arch_vcpu_create()
12260 vcpu->arch.mp_state = KVM_MP_STATE_UNINITIALIZED; in kvm_arch_vcpu_create()
12262 r = kvm_mmu_create(vcpu); in kvm_arch_vcpu_create()
12266 r = kvm_create_lapic(vcpu); in kvm_arch_vcpu_create()
12275 vcpu->arch.pio_data = page_address(page); in kvm_arch_vcpu_create()
12277 vcpu->arch.mce_banks = kcalloc(KVM_MAX_MCE_BANKS * 4, sizeof(u64), in kvm_arch_vcpu_create()
12279 vcpu->arch.mci_ctl2_banks = kcalloc(KVM_MAX_MCE_BANKS, sizeof(u64), in kvm_arch_vcpu_create()
12281 if (!vcpu->arch.mce_banks || !vcpu->arch.mci_ctl2_banks) in kvm_arch_vcpu_create()
12283 vcpu->arch.mcg_cap = KVM_MAX_MCE_BANKS; in kvm_arch_vcpu_create()
12285 if (!zalloc_cpumask_var(&vcpu->arch.wbinvd_dirty_mask, in kvm_arch_vcpu_create()
12289 if (!alloc_emulate_ctxt(vcpu)) in kvm_arch_vcpu_create()
12292 if (!fpu_alloc_guest_fpstate(&vcpu->arch.guest_fpu)) { in kvm_arch_vcpu_create()
12293 pr_err("failed to allocate vcpu's fpu\n"); in kvm_arch_vcpu_create()
12297 vcpu->arch.maxphyaddr = cpuid_query_maxphyaddr(vcpu); in kvm_arch_vcpu_create()
12298 vcpu->arch.reserved_gpa_bits = kvm_vcpu_reserved_gpa_bits_raw(vcpu); in kvm_arch_vcpu_create()
12300 kvm_async_pf_hash_reset(vcpu); in kvm_arch_vcpu_create()
12302 vcpu->arch.perf_capabilities = kvm_caps.supported_perf_cap; in kvm_arch_vcpu_create()
12303 kvm_pmu_init(vcpu); in kvm_arch_vcpu_create()
12305 vcpu->arch.pending_external_vector = -1; in kvm_arch_vcpu_create()
12306 vcpu->arch.preempted_in_kernel = false; in kvm_arch_vcpu_create()
12309 vcpu->arch.hv_root_tdp = INVALID_PAGE; in kvm_arch_vcpu_create()
12312 r = kvm_x86_call(vcpu_create)(vcpu); in kvm_arch_vcpu_create()
12316 vcpu->arch.arch_capabilities = kvm_get_arch_capabilities(); in kvm_arch_vcpu_create()
12317 vcpu->arch.msr_platform_info = MSR_PLATFORM_INFO_CPUID_FAULT; in kvm_arch_vcpu_create()
12318 kvm_xen_init_vcpu(vcpu); in kvm_arch_vcpu_create()
12319 vcpu_load(vcpu); in kvm_arch_vcpu_create()
12320 kvm_set_tsc_khz(vcpu, vcpu->kvm->arch.default_tsc_khz); in kvm_arch_vcpu_create()
12321 kvm_vcpu_reset(vcpu, false); in kvm_arch_vcpu_create()
12322 kvm_init_mmu(vcpu); in kvm_arch_vcpu_create()
12323 vcpu_put(vcpu); in kvm_arch_vcpu_create()
12327 fpu_free_guest_fpstate(&vcpu->arch.guest_fpu); in kvm_arch_vcpu_create()
12329 kmem_cache_free(x86_emulator_cache, vcpu->arch.emulate_ctxt); in kvm_arch_vcpu_create()
12331 free_cpumask_var(vcpu->arch.wbinvd_dirty_mask); in kvm_arch_vcpu_create()
12333 kfree(vcpu->arch.mce_banks); in kvm_arch_vcpu_create()
12334 kfree(vcpu->arch.mci_ctl2_banks); in kvm_arch_vcpu_create()
12335 free_page((unsigned long)vcpu->arch.pio_data); in kvm_arch_vcpu_create()
12337 kvm_free_lapic(vcpu); in kvm_arch_vcpu_create()
12339 kvm_mmu_destroy(vcpu); in kvm_arch_vcpu_create()
12343 void kvm_arch_vcpu_postcreate(struct kvm_vcpu *vcpu) in kvm_arch_vcpu_postcreate() argument
12345 struct kvm *kvm = vcpu->kvm; in kvm_arch_vcpu_postcreate()
12347 if (mutex_lock_killable(&vcpu->mutex)) in kvm_arch_vcpu_postcreate()
12349 vcpu_load(vcpu); in kvm_arch_vcpu_postcreate()
12350 kvm_synchronize_tsc(vcpu, NULL); in kvm_arch_vcpu_postcreate()
12351 vcpu_put(vcpu); in kvm_arch_vcpu_postcreate()
12354 vcpu->arch.msr_kvm_poll_control = 1; in kvm_arch_vcpu_postcreate()
12356 mutex_unlock(&vcpu->mutex); in kvm_arch_vcpu_postcreate()
12358 if (kvmclock_periodic_sync && vcpu->vcpu_idx == 0) in kvm_arch_vcpu_postcreate()
12363 void kvm_arch_vcpu_destroy(struct kvm_vcpu *vcpu) in kvm_arch_vcpu_destroy() argument
12367 kvmclock_reset(vcpu); in kvm_arch_vcpu_destroy()
12369 kvm_x86_call(vcpu_free)(vcpu); in kvm_arch_vcpu_destroy()
12371 kmem_cache_free(x86_emulator_cache, vcpu->arch.emulate_ctxt); in kvm_arch_vcpu_destroy()
12372 free_cpumask_var(vcpu->arch.wbinvd_dirty_mask); in kvm_arch_vcpu_destroy()
12373 fpu_free_guest_fpstate(&vcpu->arch.guest_fpu); in kvm_arch_vcpu_destroy()
12375 kvm_xen_destroy_vcpu(vcpu); in kvm_arch_vcpu_destroy()
12376 kvm_hv_vcpu_uninit(vcpu); in kvm_arch_vcpu_destroy()
12377 kvm_pmu_destroy(vcpu); in kvm_arch_vcpu_destroy()
12378 kfree(vcpu->arch.mce_banks); in kvm_arch_vcpu_destroy()
12379 kfree(vcpu->arch.mci_ctl2_banks); in kvm_arch_vcpu_destroy()
12380 kvm_free_lapic(vcpu); in kvm_arch_vcpu_destroy()
12381 idx = srcu_read_lock(&vcpu->kvm->srcu); in kvm_arch_vcpu_destroy()
12382 kvm_mmu_destroy(vcpu); in kvm_arch_vcpu_destroy()
12383 srcu_read_unlock(&vcpu->kvm->srcu, idx); in kvm_arch_vcpu_destroy()
12384 free_page((unsigned long)vcpu->arch.pio_data); in kvm_arch_vcpu_destroy()
12385 kvfree(vcpu->arch.cpuid_entries); in kvm_arch_vcpu_destroy()
12388 void kvm_vcpu_reset(struct kvm_vcpu *vcpu, bool init_event) in kvm_vcpu_reset() argument
12391 unsigned long old_cr0 = kvm_read_cr0(vcpu); in kvm_vcpu_reset()
12398 * into hardware, to be zeroed at vCPU creation. Use CRs as a sentinel in kvm_vcpu_reset()
12402 (old_cr0 || kvm_read_cr3(vcpu) || kvm_read_cr4(vcpu))); in kvm_vcpu_reset()
12406 * possible to INIT the vCPU while L2 is active. Force the vCPU back in kvm_vcpu_reset()
12410 if (is_guest_mode(vcpu)) in kvm_vcpu_reset()
12411 kvm_leave_nested(vcpu); in kvm_vcpu_reset()
12413 kvm_lapic_reset(vcpu, init_event); in kvm_vcpu_reset()
12415 WARN_ON_ONCE(is_guest_mode(vcpu) || is_smm(vcpu)); in kvm_vcpu_reset()
12416 vcpu->arch.hflags = 0; in kvm_vcpu_reset()
12418 vcpu->arch.smi_pending = 0; in kvm_vcpu_reset()
12419 vcpu->arch.smi_count = 0; in kvm_vcpu_reset()
12420 atomic_set(&vcpu->arch.nmi_queued, 0); in kvm_vcpu_reset()
12421 vcpu->arch.nmi_pending = 0; in kvm_vcpu_reset()
12422 vcpu->arch.nmi_injected = false; in kvm_vcpu_reset()
12423 kvm_clear_interrupt_queue(vcpu); in kvm_vcpu_reset()
12424 kvm_clear_exception_queue(vcpu); in kvm_vcpu_reset()
12426 memset(vcpu->arch.db, 0, sizeof(vcpu->arch.db)); in kvm_vcpu_reset()
12427 kvm_update_dr0123(vcpu); in kvm_vcpu_reset()
12428 vcpu->arch.dr6 = DR6_ACTIVE_LOW; in kvm_vcpu_reset()
12429 vcpu->arch.dr7 = DR7_FIXED_1; in kvm_vcpu_reset()
12430 kvm_update_dr7(vcpu); in kvm_vcpu_reset()
12432 vcpu->arch.cr2 = 0; in kvm_vcpu_reset()
12434 kvm_make_request(KVM_REQ_EVENT, vcpu); in kvm_vcpu_reset()
12435 vcpu->arch.apf.msr_en_val = 0; in kvm_vcpu_reset()
12436 vcpu->arch.apf.msr_int_val = 0; in kvm_vcpu_reset()
12437 vcpu->arch.st.msr_val = 0; in kvm_vcpu_reset()
12439 kvmclock_reset(vcpu); in kvm_vcpu_reset()
12441 kvm_clear_async_pf_completion_queue(vcpu); in kvm_vcpu_reset()
12442 kvm_async_pf_hash_reset(vcpu); in kvm_vcpu_reset()
12443 vcpu->arch.apf.halted = false; in kvm_vcpu_reset()
12445 if (vcpu->arch.guest_fpu.fpstate && kvm_mpx_supported()) { in kvm_vcpu_reset()
12446 struct fpstate *fpstate = vcpu->arch.guest_fpu.fpstate; in kvm_vcpu_reset()
12453 kvm_put_guest_fpu(vcpu); in kvm_vcpu_reset()
12459 kvm_load_guest_fpu(vcpu); in kvm_vcpu_reset()
12463 vcpu->arch.smbase = 0x30000; in kvm_vcpu_reset()
12465 vcpu->arch.pat = MSR_IA32_CR_PAT_DEFAULT; in kvm_vcpu_reset()
12467 vcpu->arch.msr_misc_features_enables = 0; in kvm_vcpu_reset()
12468 vcpu->arch.ia32_misc_enable_msr = MSR_IA32_MISC_ENABLE_PEBS_UNAVAIL | in kvm_vcpu_reset()
12471 __kvm_set_xcr(vcpu, 0, XFEATURE_MASK_FP); in kvm_vcpu_reset()
12472 __kvm_set_msr(vcpu, MSR_IA32_XSS, 0, true); in kvm_vcpu_reset()
12476 memset(vcpu->arch.regs, 0, sizeof(vcpu->arch.regs)); in kvm_vcpu_reset()
12477 kvm_register_mark_dirty(vcpu, VCPU_REGS_RSP); in kvm_vcpu_reset()
12482 * RESET since KVM emulates RESET before exposing the vCPU to userspace, in kvm_vcpu_reset()
12486 cpuid_0x1 = kvm_find_cpuid_entry(vcpu, 1); in kvm_vcpu_reset()
12487 kvm_rdx_write(vcpu, cpuid_0x1 ? cpuid_0x1->eax : 0x600); in kvm_vcpu_reset()
12489 kvm_x86_call(vcpu_reset)(vcpu, init_event); in kvm_vcpu_reset()
12491 kvm_set_rflags(vcpu, X86_EFLAGS_FIXED); in kvm_vcpu_reset()
12492 kvm_rip_write(vcpu, 0xfff0); in kvm_vcpu_reset()
12494 vcpu->arch.cr3 = 0; in kvm_vcpu_reset()
12495 kvm_register_mark_dirty(vcpu, VCPU_EXREG_CR3); in kvm_vcpu_reset()
12508 kvm_x86_call(set_cr0)(vcpu, new_cr0); in kvm_vcpu_reset()
12509 kvm_x86_call(set_cr4)(vcpu, 0); in kvm_vcpu_reset()
12510 kvm_x86_call(set_efer)(vcpu, 0); in kvm_vcpu_reset()
12511 kvm_x86_call(update_exception_bitmap)(vcpu); in kvm_vcpu_reset()
12522 kvm_make_request(KVM_REQ_TLB_FLUSH_GUEST, vcpu); in kvm_vcpu_reset()
12523 kvm_mmu_reset_context(vcpu); in kvm_vcpu_reset()
12536 kvm_make_request(KVM_REQ_TLB_FLUSH_GUEST, vcpu); in kvm_vcpu_reset()
12540 void kvm_vcpu_deliver_sipi_vector(struct kvm_vcpu *vcpu, u8 vector) in kvm_vcpu_deliver_sipi_vector() argument
12544 kvm_get_segment(vcpu, &cs, VCPU_SREG_CS); in kvm_vcpu_deliver_sipi_vector()
12547 kvm_set_segment(vcpu, &cs, VCPU_SREG_CS); in kvm_vcpu_deliver_sipi_vector()
12548 kvm_rip_write(vcpu, 0); in kvm_vcpu_deliver_sipi_vector()
12565 struct kvm_vcpu *vcpu; in kvm_arch_enable_virtualization_cpu() local
12585 kvm_for_each_vcpu(i, vcpu, kvm) { in kvm_arch_enable_virtualization_cpu()
12586 if (!stable && vcpu->cpu == smp_processor_id()) in kvm_arch_enable_virtualization_cpu()
12587 kvm_make_request(KVM_REQ_CLOCK_UPDATE, vcpu); in kvm_arch_enable_virtualization_cpu()
12588 if (stable && vcpu->arch.last_host_tsc > local_tsc) { in kvm_arch_enable_virtualization_cpu()
12590 if (vcpu->arch.last_host_tsc > max_tsc) in kvm_arch_enable_virtualization_cpu()
12591 max_tsc = vcpu->arch.last_host_tsc; in kvm_arch_enable_virtualization_cpu()
12608 * adjustment to TSC in each VCPU. When the VCPU later gets loaded, in kvm_arch_enable_virtualization_cpu()
12610 * adjustments, in case multiple suspend cycles happen before some VCPU in kvm_arch_enable_virtualization_cpu()
12638 kvm_for_each_vcpu(i, vcpu, kvm) { in kvm_arch_enable_virtualization_cpu()
12639 vcpu->arch.tsc_offset_adjustment += delta_cyc; in kvm_arch_enable_virtualization_cpu()
12640 vcpu->arch.last_host_tsc = local_tsc; in kvm_arch_enable_virtualization_cpu()
12641 kvm_make_request(KVM_REQ_MASTERCLOCK_UPDATE, vcpu); in kvm_arch_enable_virtualization_cpu()
12664 bool kvm_vcpu_is_reset_bsp(struct kvm_vcpu *vcpu) in kvm_vcpu_is_reset_bsp() argument
12666 return vcpu->kvm->arch.bsp_vcpu_id == vcpu->vcpu_id; in kvm_vcpu_is_reset_bsp()
12669 bool kvm_vcpu_is_bsp(struct kvm_vcpu *vcpu) in kvm_vcpu_is_bsp() argument
12671 return (vcpu->arch.apic_base & MSR_IA32_APICBASE_BSP) != 0; in kvm_vcpu_is_bsp()
12757 static void kvm_unload_vcpu_mmu(struct kvm_vcpu *vcpu) in kvm_unload_vcpu_mmu() argument
12759 vcpu_load(vcpu); in kvm_unload_vcpu_mmu()
12760 kvm_mmu_unload(vcpu); in kvm_unload_vcpu_mmu()
12761 vcpu_put(vcpu); in kvm_unload_vcpu_mmu()
12767 struct kvm_vcpu *vcpu; in kvm_unload_vcpu_mmus() local
12769 kvm_for_each_vcpu(i, vcpu, kvm) { in kvm_unload_vcpu_mmus()
12770 kvm_clear_async_pf_completion_queue(vcpu); in kvm_unload_vcpu_mmus()
12771 kvm_unload_vcpu_mmu(vcpu); in kvm_unload_vcpu_mmus()
13009 struct kvm_vcpu *vcpu; in kvm_arch_memslots_updated() local
13019 kvm_for_each_vcpu(i, vcpu, kvm) in kvm_arch_memslots_updated()
13020 kvm_vcpu_kick(vcpu); in kvm_arch_memslots_updated()
13208 bool kvm_arch_vcpu_in_kernel(struct kvm_vcpu *vcpu) in kvm_arch_vcpu_in_kernel() argument
13210 if (vcpu->arch.guest_state_protected) in kvm_arch_vcpu_in_kernel()
13213 return kvm_x86_call(get_cpl)(vcpu) == 0; in kvm_arch_vcpu_in_kernel()
13216 unsigned long kvm_arch_vcpu_get_ip(struct kvm_vcpu *vcpu) in kvm_arch_vcpu_get_ip() argument
13218 return kvm_rip_read(vcpu); in kvm_arch_vcpu_get_ip()
13221 int kvm_arch_vcpu_should_kick(struct kvm_vcpu *vcpu) in kvm_arch_vcpu_should_kick() argument
13223 return kvm_vcpu_exiting_guest_mode(vcpu) == IN_GUEST_MODE; in kvm_arch_vcpu_should_kick()
13226 int kvm_arch_interrupt_allowed(struct kvm_vcpu *vcpu) in kvm_arch_interrupt_allowed() argument
13228 return kvm_x86_call(interrupt_allowed)(vcpu, false); in kvm_arch_interrupt_allowed()
13231 unsigned long kvm_get_linear_rip(struct kvm_vcpu *vcpu) in kvm_get_linear_rip() argument
13234 if (vcpu->arch.guest_state_protected) in kvm_get_linear_rip()
13237 if (is_64_bit_mode(vcpu)) in kvm_get_linear_rip()
13238 return kvm_rip_read(vcpu); in kvm_get_linear_rip()
13239 return (u32)(get_segment_base(vcpu, VCPU_SREG_CS) + in kvm_get_linear_rip()
13240 kvm_rip_read(vcpu)); in kvm_get_linear_rip()
13244 bool kvm_is_linear_rip(struct kvm_vcpu *vcpu, unsigned long linear_rip) in kvm_is_linear_rip() argument
13246 return kvm_get_linear_rip(vcpu) == linear_rip; in kvm_is_linear_rip()
13250 unsigned long kvm_get_rflags(struct kvm_vcpu *vcpu) in kvm_get_rflags() argument
13254 rflags = kvm_x86_call(get_rflags)(vcpu); in kvm_get_rflags()
13255 if (vcpu->guest_debug & KVM_GUESTDBG_SINGLESTEP) in kvm_get_rflags()
13261 static void __kvm_set_rflags(struct kvm_vcpu *vcpu, unsigned long rflags) in __kvm_set_rflags() argument
13263 if (vcpu->guest_debug & KVM_GUESTDBG_SINGLESTEP && in __kvm_set_rflags()
13264 kvm_is_linear_rip(vcpu, vcpu->arch.singlestep_rip)) in __kvm_set_rflags()
13266 kvm_x86_call(set_rflags)(vcpu, rflags); in __kvm_set_rflags()
13269 void kvm_set_rflags(struct kvm_vcpu *vcpu, unsigned long rflags) in kvm_set_rflags() argument
13271 __kvm_set_rflags(vcpu, rflags); in kvm_set_rflags()
13272 kvm_make_request(KVM_REQ_EVENT, vcpu); in kvm_set_rflags()
13288 static void kvm_add_async_pf_gfn(struct kvm_vcpu *vcpu, gfn_t gfn) in kvm_add_async_pf_gfn() argument
13292 while (vcpu->arch.apf.gfns[key] != ~0) in kvm_add_async_pf_gfn()
13295 vcpu->arch.apf.gfns[key] = gfn; in kvm_add_async_pf_gfn()
13298 static u32 kvm_async_pf_gfn_slot(struct kvm_vcpu *vcpu, gfn_t gfn) in kvm_async_pf_gfn_slot() argument
13304 (vcpu->arch.apf.gfns[key] != gfn && in kvm_async_pf_gfn_slot()
13305 vcpu->arch.apf.gfns[key] != ~0); i++) in kvm_async_pf_gfn_slot()
13311 bool kvm_find_async_pf_gfn(struct kvm_vcpu *vcpu, gfn_t gfn) in kvm_find_async_pf_gfn() argument
13313 return vcpu->arch.apf.gfns[kvm_async_pf_gfn_slot(vcpu, gfn)] == gfn; in kvm_find_async_pf_gfn()
13316 static void kvm_del_async_pf_gfn(struct kvm_vcpu *vcpu, gfn_t gfn) in kvm_del_async_pf_gfn() argument
13320 i = j = kvm_async_pf_gfn_slot(vcpu, gfn); in kvm_del_async_pf_gfn()
13322 if (WARN_ON_ONCE(vcpu->arch.apf.gfns[i] != gfn)) in kvm_del_async_pf_gfn()
13326 vcpu->arch.apf.gfns[i] = ~0; in kvm_del_async_pf_gfn()
13329 if (vcpu->arch.apf.gfns[j] == ~0) in kvm_del_async_pf_gfn()
13331 k = kvm_async_pf_hash_fn(vcpu->arch.apf.gfns[j]); in kvm_del_async_pf_gfn()
13338 vcpu->arch.apf.gfns[i] = vcpu->arch.apf.gfns[j]; in kvm_del_async_pf_gfn()
13343 static inline int apf_put_user_notpresent(struct kvm_vcpu *vcpu) in apf_put_user_notpresent() argument
13347 return kvm_write_guest_cached(vcpu->kvm, &vcpu->arch.apf.data, &reason, in apf_put_user_notpresent()
13351 static inline int apf_put_user_ready(struct kvm_vcpu *vcpu, u32 token) in apf_put_user_ready() argument
13355 return kvm_write_guest_offset_cached(vcpu->kvm, &vcpu->arch.apf.data, in apf_put_user_ready()
13359 static inline bool apf_pageready_slot_free(struct kvm_vcpu *vcpu) in apf_pageready_slot_free() argument
13364 if (kvm_read_guest_offset_cached(vcpu->kvm, &vcpu->arch.apf.data, in apf_pageready_slot_free()
13371 static bool kvm_can_deliver_async_pf(struct kvm_vcpu *vcpu) in kvm_can_deliver_async_pf() argument
13374 if (!kvm_pv_async_pf_enabled(vcpu)) in kvm_can_deliver_async_pf()
13377 if (vcpu->arch.apf.send_user_only && in kvm_can_deliver_async_pf()
13378 kvm_x86_call(get_cpl)(vcpu) == 0) in kvm_can_deliver_async_pf()
13381 if (is_guest_mode(vcpu)) { in kvm_can_deliver_async_pf()
13386 return vcpu->arch.apf.delivery_as_pf_vmexit; in kvm_can_deliver_async_pf()
13393 return is_paging(vcpu); in kvm_can_deliver_async_pf()
13397 bool kvm_can_do_async_pf(struct kvm_vcpu *vcpu) in kvm_can_do_async_pf() argument
13399 if (unlikely(!lapic_in_kernel(vcpu) || in kvm_can_do_async_pf()
13400 kvm_event_needs_reinjection(vcpu) || in kvm_can_do_async_pf()
13401 kvm_is_exception_pending(vcpu))) in kvm_can_do_async_pf()
13404 if (kvm_hlt_in_guest(vcpu->kvm) && !kvm_can_deliver_async_pf(vcpu)) in kvm_can_do_async_pf()
13411 return kvm_arch_interrupt_allowed(vcpu); in kvm_can_do_async_pf()
13414 bool kvm_arch_async_page_not_present(struct kvm_vcpu *vcpu, in kvm_arch_async_page_not_present() argument
13420 kvm_add_async_pf_gfn(vcpu, work->arch.gfn); in kvm_arch_async_page_not_present()
13422 if (kvm_can_deliver_async_pf(vcpu) && in kvm_arch_async_page_not_present()
13423 !apf_put_user_notpresent(vcpu)) { in kvm_arch_async_page_not_present()
13430 kvm_inject_page_fault(vcpu, &fault); in kvm_arch_async_page_not_present()
13441 kvm_make_request(KVM_REQ_APF_HALT, vcpu); in kvm_arch_async_page_not_present()
13446 void kvm_arch_async_page_present(struct kvm_vcpu *vcpu, in kvm_arch_async_page_present() argument
13451 .vector = vcpu->arch.apf.vec in kvm_arch_async_page_present()
13457 kvm_del_async_pf_gfn(vcpu, work->arch.gfn); in kvm_arch_async_page_present()
13461 kvm_pv_async_pf_enabled(vcpu) && in kvm_arch_async_page_present()
13462 !apf_put_user_ready(vcpu, work->arch.token)) { in kvm_arch_async_page_present()
13463 vcpu->arch.apf.pageready_pending = true; in kvm_arch_async_page_present()
13464 kvm_apic_set_irq(vcpu, &irq, NULL); in kvm_arch_async_page_present()
13467 vcpu->arch.apf.halted = false; in kvm_arch_async_page_present()
13468 vcpu->arch.mp_state = KVM_MP_STATE_RUNNABLE; in kvm_arch_async_page_present()
13471 void kvm_arch_async_page_present_queued(struct kvm_vcpu *vcpu) in kvm_arch_async_page_present_queued() argument
13473 kvm_make_request(KVM_REQ_APF_READY, vcpu); in kvm_arch_async_page_present_queued()
13474 if (!vcpu->arch.apf.pageready_pending) in kvm_arch_async_page_present_queued()
13475 kvm_vcpu_kick(vcpu); in kvm_arch_async_page_present_queued()
13478 bool kvm_arch_can_dequeue_async_page_present(struct kvm_vcpu *vcpu) in kvm_arch_can_dequeue_async_page_present() argument
13480 if (!kvm_pv_async_pf_enabled(vcpu)) in kvm_arch_can_dequeue_async_page_present()
13483 return kvm_lapic_enabled(vcpu) && apf_pageready_slot_free(vcpu); in kvm_arch_can_dequeue_async_page_present()
13605 bool kvm_arch_no_poll(struct kvm_vcpu *vcpu) in kvm_arch_no_poll() argument
13607 return (vcpu->arch.msr_kvm_poll_control & 1) == 0; in kvm_arch_no_poll()
13651 void kvm_fixup_and_inject_pf_error(struct kvm_vcpu *vcpu, gva_t gva, u16 error_code) in kvm_fixup_and_inject_pf_error() argument
13653 struct kvm_mmu *mmu = vcpu->arch.walk_mmu; in kvm_fixup_and_inject_pf_error()
13659 mmu->gva_to_gpa(vcpu, mmu, gva, access, &fault) != INVALID_GPA) { in kvm_fixup_and_inject_pf_error()
13661 * If vcpu->arch.walk_mmu->gva_to_gpa succeeded, the page in kvm_fixup_and_inject_pf_error()
13672 vcpu->arch.walk_mmu->inject_page_fault(vcpu, &fault); in kvm_fixup_and_inject_pf_error()
13681 int kvm_handle_memory_failure(struct kvm_vcpu *vcpu, int r, in kvm_handle_memory_failure() argument
13685 if (KVM_BUG_ON(!e, vcpu->kvm)) in kvm_handle_memory_failure()
13688 kvm_inject_emulated_page_fault(vcpu, e); in kvm_handle_memory_failure()
13699 kvm_prepare_emulation_failure_exit(vcpu); in kvm_handle_memory_failure()
13705 int kvm_handle_invpcid(struct kvm_vcpu *vcpu, unsigned long type, gva_t gva) in kvm_handle_invpcid() argument
13715 r = kvm_read_guest_virt(vcpu, gva, &operand, sizeof(operand), &e); in kvm_handle_invpcid()
13717 return kvm_handle_memory_failure(vcpu, r, &e); in kvm_handle_invpcid()
13720 kvm_inject_gp(vcpu, 0); in kvm_handle_invpcid()
13724 pcid_enabled = kvm_is_cr4_bit_set(vcpu, X86_CR4_PCIDE); in kvm_handle_invpcid()
13733 is_noncanonical_address(operand.gla, vcpu)) { in kvm_handle_invpcid()
13734 kvm_inject_gp(vcpu, 0); in kvm_handle_invpcid()
13737 kvm_mmu_invpcid_gva(vcpu, operand.gla, operand.pcid); in kvm_handle_invpcid()
13738 return kvm_skip_emulated_instruction(vcpu); in kvm_handle_invpcid()
13742 kvm_inject_gp(vcpu, 0); in kvm_handle_invpcid()
13746 kvm_invalidate_pcid(vcpu, operand.pcid); in kvm_handle_invpcid()
13747 return kvm_skip_emulated_instruction(vcpu); in kvm_handle_invpcid()
13759 kvm_make_request(KVM_REQ_TLB_FLUSH_GUEST, vcpu); in kvm_handle_invpcid()
13760 return kvm_skip_emulated_instruction(vcpu); in kvm_handle_invpcid()
13763 kvm_inject_gp(vcpu, 0); in kvm_handle_invpcid()
13769 static int complete_sev_es_emulated_mmio(struct kvm_vcpu *vcpu) in complete_sev_es_emulated_mmio() argument
13771 struct kvm_run *run = vcpu->run; in complete_sev_es_emulated_mmio()
13775 BUG_ON(!vcpu->mmio_needed); in complete_sev_es_emulated_mmio()
13778 frag = &vcpu->mmio_fragments[vcpu->mmio_cur_fragment]; in complete_sev_es_emulated_mmio()
13780 if (!vcpu->mmio_is_write) in complete_sev_es_emulated_mmio()
13786 vcpu->mmio_cur_fragment++; in complete_sev_es_emulated_mmio()
13794 if (vcpu->mmio_cur_fragment >= vcpu->mmio_nr_fragments) { in complete_sev_es_emulated_mmio()
13795 vcpu->mmio_needed = 0; in complete_sev_es_emulated_mmio()
13805 run->mmio.is_write = vcpu->mmio_is_write; in complete_sev_es_emulated_mmio()
13810 vcpu->arch.complete_userspace_io = complete_sev_es_emulated_mmio; in complete_sev_es_emulated_mmio()
13815 int kvm_sev_es_mmio_write(struct kvm_vcpu *vcpu, gpa_t gpa, unsigned int bytes, in kvm_sev_es_mmio_write() argument
13824 handled = write_emultor.read_write_mmio(vcpu, gpa, bytes, data); in kvm_sev_es_mmio_write()
13833 frag = vcpu->mmio_fragments; in kvm_sev_es_mmio_write()
13834 vcpu->mmio_nr_fragments = 1; in kvm_sev_es_mmio_write()
13839 vcpu->mmio_needed = 1; in kvm_sev_es_mmio_write()
13840 vcpu->mmio_cur_fragment = 0; in kvm_sev_es_mmio_write()
13842 vcpu->run->mmio.phys_addr = gpa; in kvm_sev_es_mmio_write()
13843 vcpu->run->mmio.len = min(8u, frag->len); in kvm_sev_es_mmio_write()
13844 vcpu->run->mmio.is_write = 1; in kvm_sev_es_mmio_write()
13845 memcpy(vcpu->run->mmio.data, frag->data, min(8u, frag->len)); in kvm_sev_es_mmio_write()
13846 vcpu->run->exit_reason = KVM_EXIT_MMIO; in kvm_sev_es_mmio_write()
13848 vcpu->arch.complete_userspace_io = complete_sev_es_emulated_mmio; in kvm_sev_es_mmio_write()
13854 int kvm_sev_es_mmio_read(struct kvm_vcpu *vcpu, gpa_t gpa, unsigned int bytes, in kvm_sev_es_mmio_read() argument
13863 handled = read_emultor.read_write_mmio(vcpu, gpa, bytes, data); in kvm_sev_es_mmio_read()
13872 frag = vcpu->mmio_fragments; in kvm_sev_es_mmio_read()
13873 vcpu->mmio_nr_fragments = 1; in kvm_sev_es_mmio_read()
13878 vcpu->mmio_needed = 1; in kvm_sev_es_mmio_read()
13879 vcpu->mmio_cur_fragment = 0; in kvm_sev_es_mmio_read()
13881 vcpu->run->mmio.phys_addr = gpa; in kvm_sev_es_mmio_read()
13882 vcpu->run->mmio.len = min(8u, frag->len); in kvm_sev_es_mmio_read()
13883 vcpu->run->mmio.is_write = 0; in kvm_sev_es_mmio_read()
13884 vcpu->run->exit_reason = KVM_EXIT_MMIO; in kvm_sev_es_mmio_read()
13886 vcpu->arch.complete_userspace_io = complete_sev_es_emulated_mmio; in kvm_sev_es_mmio_read()
13892 static void advance_sev_es_emulated_pio(struct kvm_vcpu *vcpu, unsigned count, int size) in advance_sev_es_emulated_pio() argument
13894 vcpu->arch.sev_pio_count -= count; in advance_sev_es_emulated_pio()
13895 vcpu->arch.sev_pio_data += count * size; in advance_sev_es_emulated_pio()
13898 static int kvm_sev_es_outs(struct kvm_vcpu *vcpu, unsigned int size,
13901 static int complete_sev_es_emulated_outs(struct kvm_vcpu *vcpu) in complete_sev_es_emulated_outs() argument
13903 int size = vcpu->arch.pio.size; in complete_sev_es_emulated_outs()
13904 int port = vcpu->arch.pio.port; in complete_sev_es_emulated_outs()
13906 vcpu->arch.pio.count = 0; in complete_sev_es_emulated_outs()
13907 if (vcpu->arch.sev_pio_count) in complete_sev_es_emulated_outs()
13908 return kvm_sev_es_outs(vcpu, size, port); in complete_sev_es_emulated_outs()
13912 static int kvm_sev_es_outs(struct kvm_vcpu *vcpu, unsigned int size, in kvm_sev_es_outs() argument
13917 min_t(unsigned int, PAGE_SIZE / size, vcpu->arch.sev_pio_count); in kvm_sev_es_outs()
13918 int ret = emulator_pio_out(vcpu, size, port, vcpu->arch.sev_pio_data, count); in kvm_sev_es_outs()
13921 advance_sev_es_emulated_pio(vcpu, count, size); in kvm_sev_es_outs()
13926 if (!vcpu->arch.sev_pio_count) in kvm_sev_es_outs()
13930 vcpu->arch.complete_userspace_io = complete_sev_es_emulated_outs; in kvm_sev_es_outs()
13934 static int kvm_sev_es_ins(struct kvm_vcpu *vcpu, unsigned int size,
13937 static int complete_sev_es_emulated_ins(struct kvm_vcpu *vcpu) in complete_sev_es_emulated_ins() argument
13939 unsigned count = vcpu->arch.pio.count; in complete_sev_es_emulated_ins()
13940 int size = vcpu->arch.pio.size; in complete_sev_es_emulated_ins()
13941 int port = vcpu->arch.pio.port; in complete_sev_es_emulated_ins()
13943 complete_emulator_pio_in(vcpu, vcpu->arch.sev_pio_data); in complete_sev_es_emulated_ins()
13944 advance_sev_es_emulated_pio(vcpu, count, size); in complete_sev_es_emulated_ins()
13945 if (vcpu->arch.sev_pio_count) in complete_sev_es_emulated_ins()
13946 return kvm_sev_es_ins(vcpu, size, port); in complete_sev_es_emulated_ins()
13950 static int kvm_sev_es_ins(struct kvm_vcpu *vcpu, unsigned int size, in kvm_sev_es_ins() argument
13955 min_t(unsigned int, PAGE_SIZE / size, vcpu->arch.sev_pio_count); in kvm_sev_es_ins()
13956 if (!emulator_pio_in(vcpu, size, port, vcpu->arch.sev_pio_data, count)) in kvm_sev_es_ins()
13960 advance_sev_es_emulated_pio(vcpu, count, size); in kvm_sev_es_ins()
13961 if (!vcpu->arch.sev_pio_count) in kvm_sev_es_ins()
13965 vcpu->arch.complete_userspace_io = complete_sev_es_emulated_ins; in kvm_sev_es_ins()
13969 int kvm_sev_es_string_io(struct kvm_vcpu *vcpu, unsigned int size, in kvm_sev_es_string_io() argument
13973 vcpu->arch.sev_pio_data = data; in kvm_sev_es_string_io()
13974 vcpu->arch.sev_pio_count = count; in kvm_sev_es_string_io()
13975 return in ? kvm_sev_es_ins(vcpu, size, port) in kvm_sev_es_string_io()
13976 : kvm_sev_es_outs(vcpu, size, port); in kvm_sev_es_string_io()