Lines Matching +full:guest +full:- +full:index +full:- +full:bits
1 // SPDX-License-Identifier: GPL-2.0-only
3 * Kernel-based Virtual Machine -- Performance Monitoring Unit support
39 /* Instruction-Accurate PDIR (PDIR++) */
51 * - Each perf counter is defined as "struct kvm_pmc";
52 * - There are two types of perf counters: general purpose (gp) and fixed.
56 * - pmu.c understands the difference between gp counters and fixed counters.
57 * However AMD doesn't support fixed-counters;
58 * - There are three types of index to access perf counters (PMC):
61 * MSR_F15H_PERF_CTRn, where MSR_F15H_PERF_CTR[0-3] are
63 * 2. MSR Index (named idx): This normally is used by RDPMC instruction.
66 * that it also supports fixed counters. idx can be used to as index to
68 * 3. Global PMC Index (named pmc): pmc is an index specific to PMU
72 * * Intel: [0 .. KVM_MAX_NR_INTEL_GP_COUNTERS-1] <=> gp counters
74 * * AMD: [0 .. AMD64_NUM_COUNTERS-1] and, for families 15H
75 * and later, [0 .. AMD64_NUM_COUNTERS_CORE-1] <=> gp counters
82 *(((struct kvm_pmu_ops *)0)->func));
84 #include <asm/kvm-x86-pmu-ops.h>
95 #include <asm/kvm-x86-pmu-ops.h> in kvm_pmu_ops_update()
104 if (pmc->perf_event && pmc->perf_event->attr.precise_ip) { in __kvm_perf_overflow()
114 /* Indicate PEBS overflow PMI to guest. */ in __kvm_perf_overflow()
116 (unsigned long *)&pmu->global_status); in __kvm_perf_overflow()
119 __set_bit(pmc->idx, (unsigned long *)&pmu->global_status); in __kvm_perf_overflow()
122 if (pmc->intr && !skip_pmi) in __kvm_perf_overflow()
123 kvm_make_request(KVM_REQ_PMI, pmc->vcpu); in __kvm_perf_overflow()
130 struct kvm_pmc *pmc = perf_event->overflow_handler_context; in kvm_perf_overflow()
135 * KVM's handling of a related guest WRMSR. in kvm_perf_overflow()
137 if (test_and_set_bit(pmc->idx, pmc_to_pmu(pmc)->reprogram_pmi)) in kvm_perf_overflow()
142 kvm_make_request(KVM_REQ_PMU, pmc->vcpu); in kvm_perf_overflow()
154 if ((pmc->idx == 0 && x86_match_cpu(vmx_pebs_pdist_cpu)) || in pmc_get_pebs_precise_level()
155 (pmc->idx == 32 && x86_match_cpu(vmx_pebs_pdir_cpu))) in pmc_get_pebs_precise_level()
159 * The non-zero precision level of guest event makes the ordinary in pmc_get_pebs_precise_level()
160 * guest event becomes a guest PEBS event and triggers the host in pmc_get_pebs_precise_level()
162 * comes from the host counters or the guest. in pmc_get_pebs_precise_level()
169 u64 sample_period = (-counter_value) & pmc_bitmask(pmc); in get_sample_period()
192 bool pebs = test_bit(pmc->idx, (unsigned long *)&pmu->pebs_enable); in pmc_reprogram_counter()
194 attr.sample_period = get_sample_period(pmc, pmc->counter); in pmc_reprogram_counter()
208 * precision levels of guest and host PEBS events will not affect in pmc_reprogram_counter()
210 * in the PEBS record is calibrated on the guest side. in pmc_reprogram_counter()
215 event = perf_event_create_kernel_counter(&attr, -1, current, in pmc_reprogram_counter()
218 pr_debug_ratelimited("kvm_pmu: event creation failed %ld for pmc->idx = %d\n", in pmc_reprogram_counter()
219 PTR_ERR(event), pmc->idx); in pmc_reprogram_counter()
223 pmc->perf_event = event; in pmc_reprogram_counter()
224 pmc_to_pmu(pmc)->event_count++; in pmc_reprogram_counter()
225 pmc->is_paused = false; in pmc_reprogram_counter()
226 pmc->intr = intr || pebs; in pmc_reprogram_counter()
232 u64 counter = pmc->counter; in pmc_pause_counter()
236 if (pmc->perf_event && !pmc->is_paused) in pmc_pause_counter()
237 counter += perf_event_pause(pmc->perf_event, true); in pmc_pause_counter()
247 counter += pmc->emulated_counter; in pmc_pause_counter()
248 pmc->counter = counter & pmc_bitmask(pmc); in pmc_pause_counter()
250 pmc->emulated_counter = 0; in pmc_pause_counter()
251 pmc->is_paused = true; in pmc_pause_counter()
253 return pmc->counter < prev_counter; in pmc_pause_counter()
258 if (!pmc->perf_event) in pmc_resume_counter()
262 if (is_sampling_event(pmc->perf_event) && in pmc_resume_counter()
263 perf_event_period(pmc->perf_event, in pmc_resume_counter()
264 get_sample_period(pmc, pmc->counter))) in pmc_resume_counter()
267 if (test_bit(pmc->idx, (unsigned long *)&pmc_to_pmu(pmc)->pebs_enable) != in pmc_resume_counter()
268 (!!pmc->perf_event->attr.precise_ip)) in pmc_resume_counter()
272 perf_event_enable(pmc->perf_event); in pmc_resume_counter()
273 pmc->is_paused = false; in pmc_resume_counter()
280 if (pmc->perf_event) { in pmc_release_perf_event()
281 perf_event_release_kernel(pmc->perf_event); in pmc_release_perf_event()
282 pmc->perf_event = NULL; in pmc_release_perf_event()
283 pmc->current_config = 0; in pmc_release_perf_event()
284 pmc_to_pmu(pmc)->event_count--; in pmc_release_perf_event()
290 if (pmc->perf_event) { in pmc_stop_counter()
291 pmc->counter = pmc_read_counter(pmc); in pmc_stop_counter()
298 if (!pmc->perf_event || pmc->is_paused || in pmc_update_sample_period()
299 !is_sampling_event(pmc->perf_event)) in pmc_update_sample_period()
302 perf_event_period(pmc->perf_event, in pmc_update_sample_period()
303 get_sample_period(pmc, pmc->counter)); in pmc_update_sample_period()
310 * read-modify-write. Adjust the counter value so that its value is in pmc_write_counter()
316 pmc->emulated_counter = 0; in pmc_write_counter()
317 pmc->counter += val - pmc_read_counter(pmc); in pmc_write_counter()
318 pmc->counter &= pmc_bitmask(pmc); in pmc_write_counter()
328 return (a > b) - (a < b); in filter_cmp()
354 return -1; in find_filter_index()
356 return fe - events; in find_filter_index()
361 u64 mask = filter_event >> (KVM_PMU_MASKED_ENTRY_UMASK_MASK_SHIFT - 8); in is_filter_entry_match()
365 (KVM_PMU_MASKED_ENTRY_UMASK_MASK_SHIFT - 8)) != in is_filter_entry_match()
375 int i, index; in filter_contains_match() local
377 index = find_filter_index(events, nevents, event_select); in filter_contains_match()
378 if (index < 0) in filter_contains_match()
385 for (i = index; i < nevents; i++) { in filter_contains_match()
393 for (i = index - 1; i >= 0; i--) { in filter_contains_match()
407 if (filter_contains_match(f->includes, f->nr_includes, eventsel) && in is_gp_event_allowed()
408 !filter_contains_match(f->excludes, f->nr_excludes, eventsel)) in is_gp_event_allowed()
409 return f->action == KVM_PMU_EVENT_ALLOW; in is_gp_event_allowed()
411 return f->action == KVM_PMU_EVENT_DENY; in is_gp_event_allowed()
417 int fixed_idx = idx - KVM_FIXED_PMC_BASE_IDX; in is_fixed_event_allowed()
419 if (filter->action == KVM_PMU_EVENT_DENY && in is_fixed_event_allowed()
420 test_bit(fixed_idx, (ulong *)&filter->fixed_counter_bitmap)) in is_fixed_event_allowed()
422 if (filter->action == KVM_PMU_EVENT_ALLOW && in is_fixed_event_allowed()
423 !test_bit(fixed_idx, (ulong *)&filter->fixed_counter_bitmap)) in is_fixed_event_allowed()
432 struct kvm *kvm = pmc->vcpu->kvm; in check_pmu_event_filter()
434 filter = srcu_dereference(kvm->arch.pmu_event_filter, &kvm->srcu); in check_pmu_event_filter()
439 return is_gp_event_allowed(filter, pmc->eventsel); in check_pmu_event_filter()
441 return is_fixed_event_allowed(filter, pmc->idx); in check_pmu_event_filter()
453 u64 eventsel = pmc->eventsel; in reprogram_counter()
470 fixed_ctr_ctrl = fixed_ctrl_field(pmu->fixed_ctr_ctrl, in reprogram_counter()
471 pmc->idx - KVM_FIXED_PMC_BASE_IDX); in reprogram_counter()
481 if (pmc->current_config == new_config && pmc_resume_counter(pmc)) in reprogram_counter()
486 pmc->current_config = new_config; in reprogram_counter()
489 (eventsel & pmu->raw_event_mask), in reprogram_counter()
502 bitmap_copy(bitmap, pmu->reprogram_pmi, X86_PMC_IDX_MAX); in kvm_pmu_handle_event()
506 * other than the task that holds vcpu->mutex, take care to clear only in kvm_pmu_handle_event()
507 * the bits that will actually processed. in kvm_pmu_handle_event()
510 atomic64_andnot(*(s64 *)bitmap, &pmu->__reprogram_pmi); in kvm_pmu_handle_event()
514 * If reprogramming fails, e.g. due to contention, re-set the in kvm_pmu_handle_event()
517 * stall the guest if reprogramming repeatedly fails. in kvm_pmu_handle_event()
520 set_bit(pmc->idx, pmu->reprogram_pmi); in kvm_pmu_handle_event()
524 * Release unused perf_events if the corresponding guest MSRs weren't in kvm_pmu_handle_event()
528 if (unlikely(pmu->need_cleanup)) in kvm_pmu_handle_event()
572 vcpu->kvm->arch.kvmclock_offset; in kvm_pmu_rdpmc_vmware()
588 if (!pmu->version) in kvm_pmu_rdpmc()
611 kvm_apic_local_deliver(vcpu->arch.apic, APIC_LVTPC); in kvm_pmu_deliver_pmi()
635 __set_bit(pmc->idx, pmu->pmc_in_use); in kvm_pmu_mark_pmc_in_use()
641 u32 msr = msr_info->index; in kvm_pmu_get_msr()
646 msr_info->data = pmu->global_status; in kvm_pmu_get_msr()
650 msr_info->data = pmu->global_ctrl; in kvm_pmu_get_msr()
654 msr_info->data = 0; in kvm_pmu_get_msr()
666 u32 msr = msr_info->index; in kvm_pmu_set_msr()
667 u64 data = msr_info->data; in kvm_pmu_set_msr()
671 * Note, AMD ignores writes to reserved bits and read-only PMU MSRs, in kvm_pmu_set_msr()
676 if (!msr_info->host_initiated) in kvm_pmu_set_msr()
680 /* Per PPR, Read-only MSR. Writes are ignored. */ in kvm_pmu_set_msr()
681 if (!msr_info->host_initiated) in kvm_pmu_set_msr()
684 if (data & pmu->global_status_rsvd) in kvm_pmu_set_msr()
687 pmu->global_status = data; in kvm_pmu_set_msr()
690 data &= ~pmu->global_ctrl_rsvd; in kvm_pmu_set_msr()
696 if (pmu->global_ctrl != data) { in kvm_pmu_set_msr()
697 diff = pmu->global_ctrl ^ data; in kvm_pmu_set_msr()
698 pmu->global_ctrl = data; in kvm_pmu_set_msr()
704 * GLOBAL_OVF_CTRL, a.k.a. GLOBAL STATUS_RESET, clears bits in in kvm_pmu_set_msr()
705 * GLOBAL_STATUS, and so the set of reserved bits is the same. in kvm_pmu_set_msr()
707 if (data & pmu->global_status_rsvd) in kvm_pmu_set_msr()
711 if (!msr_info->host_initiated) in kvm_pmu_set_msr()
712 pmu->global_status &= ~data; in kvm_pmu_set_msr()
715 kvm_pmu_mark_pmc_in_use(vcpu, msr_info->index); in kvm_pmu_set_msr()
728 pmu->need_cleanup = false; in kvm_pmu_reset()
730 bitmap_zero(pmu->reprogram_pmi, X86_PMC_IDX_MAX); in kvm_pmu_reset()
732 kvm_for_each_pmc(pmu, pmc, i, pmu->all_valid_pmc_idx) { in kvm_pmu_reset()
734 pmc->counter = 0; in kvm_pmu_reset()
735 pmc->emulated_counter = 0; in kvm_pmu_reset()
738 pmc->eventsel = 0; in kvm_pmu_reset()
741 pmu->fixed_ctr_ctrl = pmu->global_ctrl = pmu->global_status = 0; in kvm_pmu_reset()
755 if (KVM_BUG_ON(kvm_vcpu_has_run(vcpu), vcpu->kvm)) in kvm_pmu_refresh()
764 pmu->version = 0; in kvm_pmu_refresh()
765 pmu->nr_arch_gp_counters = 0; in kvm_pmu_refresh()
766 pmu->nr_arch_fixed_counters = 0; in kvm_pmu_refresh()
767 pmu->counter_bitmask[KVM_PMC_GP] = 0; in kvm_pmu_refresh()
768 pmu->counter_bitmask[KVM_PMC_FIXED] = 0; in kvm_pmu_refresh()
769 pmu->reserved_bits = 0xffffffff00200000ull; in kvm_pmu_refresh()
770 pmu->raw_event_mask = X86_RAW_EVENT_MASK; in kvm_pmu_refresh()
771 pmu->global_ctrl_rsvd = ~0ull; in kvm_pmu_refresh()
772 pmu->global_status_rsvd = ~0ull; in kvm_pmu_refresh()
773 pmu->fixed_ctr_ctrl_rsvd = ~0ull; in kvm_pmu_refresh()
774 pmu->pebs_enable_rsvd = ~0ull; in kvm_pmu_refresh()
775 pmu->pebs_data_cfg_rsvd = ~0ull; in kvm_pmu_refresh()
776 bitmap_zero(pmu->all_valid_pmc_idx, X86_PMC_IDX_MAX); in kvm_pmu_refresh()
778 if (!vcpu->kvm->arch.enable_pmu) in kvm_pmu_refresh()
784 * At RESET, both Intel and AMD CPUs set all enable bits for general in kvm_pmu_refresh()
790 if (kvm_pmu_has_perf_global_ctrl(pmu) && pmu->nr_arch_gp_counters) in kvm_pmu_refresh()
791 pmu->global_ctrl = GENMASK_ULL(pmu->nr_arch_gp_counters - 1, 0); in kvm_pmu_refresh()
811 pmu->need_cleanup = false; in kvm_pmu_cleanup()
813 bitmap_andnot(bitmask, pmu->all_valid_pmc_idx, in kvm_pmu_cleanup()
814 pmu->pmc_in_use, X86_PMC_IDX_MAX); in kvm_pmu_cleanup()
817 if (pmc->perf_event && !pmc_speculative_in_use(pmc)) in kvm_pmu_cleanup()
823 bitmap_zero(pmu->pmc_in_use, X86_PMC_IDX_MAX); in kvm_pmu_cleanup()
833 pmc->emulated_counter++; in kvm_pmu_incr_counter()
843 config = pmc->eventsel; in cpl_is_matched()
847 config = fixed_ctrl_field(pmc_to_pmu(pmc)->fixed_ctr_ctrl, in cpl_is_matched()
848 pmc->idx - KVM_FIXED_PMC_BASE_IDX); in cpl_is_matched()
860 return (kvm_x86_call(get_cpl)(pmc->vcpu) == 0) ? select_os : in cpl_is_matched()
871 BUILD_BUG_ON(sizeof(pmu->global_ctrl) * BITS_PER_BYTE != X86_PMC_IDX_MAX); in kvm_pmu_trigger_event()
874 bitmap_copy(bitmap, pmu->all_valid_pmc_idx, X86_PMC_IDX_MAX); in kvm_pmu_trigger_event()
875 else if (!bitmap_and(bitmap, pmu->all_valid_pmc_idx, in kvm_pmu_trigger_event()
876 (unsigned long *)&pmu->global_ctrl, X86_PMC_IDX_MAX)) in kvm_pmu_trigger_event()
888 * bits (bits 35:34). Checking the "in HLE/RTM transaction" in kvm_pmu_trigger_event()
890 * KVM is emulating an instruction. Checking the reserved bits in kvm_pmu_trigger_event()
894 if (((pmc->eventsel ^ eventsel) & AMD64_RAW_EVENT_MASK_NB) || in kvm_pmu_trigger_event()
911 for (i = 0; i < filter->nevents; i++) { in is_masked_filter_valid()
912 if (filter->events[i] & ~mask) in is_masked_filter_valid()
923 for (i = 0, j = 0; i < filter->nevents; i++) { in convert_to_masked_filter()
925 * Skip events that are impossible to match against a guest in convert_to_masked_filter()
927 * of the guest event is used. To maintain backwards in convert_to_masked_filter()
928 * compatibility, impossible filters can't be rejected :-( in convert_to_masked_filter()
930 if (filter->events[i] & ~(kvm_pmu_ops.EVENTSEL_EVENT | in convert_to_masked_filter()
934 * Convert userspace events to a common in-kernel event so in convert_to_masked_filter()
936 * the in-kernel events use masked events because they are in convert_to_masked_filter()
941 filter->events[j++] = filter->events[i] | in convert_to_masked_filter()
945 filter->nevents = j; in convert_to_masked_filter()
952 if (!(filter->flags & KVM_PMU_EVENT_FLAG_MASKED_EVENTS)) in prepare_filter_lists()
955 return -EINVAL; in prepare_filter_lists()
962 * includes and excludes sub-lists. in prepare_filter_lists()
964 sort(&filter->events, filter->nevents, sizeof(filter->events[0]), in prepare_filter_lists()
967 i = filter->nevents; in prepare_filter_lists()
969 if (filter->flags & KVM_PMU_EVENT_FLAG_MASKED_EVENTS) { in prepare_filter_lists()
970 for (i = 0; i < filter->nevents; i++) { in prepare_filter_lists()
971 if (filter->events[i] & KVM_PMU_MASKED_ENTRY_EXCLUDE) in prepare_filter_lists()
976 filter->nr_includes = i; in prepare_filter_lists()
977 filter->nr_excludes = filter->nevents - filter->nr_includes; in prepare_filter_lists()
978 filter->includes = filter->events; in prepare_filter_lists()
979 filter->excludes = filter->events + filter->nr_includes; in prepare_filter_lists()
995 return -EFAULT; in kvm_vm_ioctl_set_pmu_event_filter()
999 return -EINVAL; in kvm_vm_ioctl_set_pmu_event_filter()
1002 return -EINVAL; in kvm_vm_ioctl_set_pmu_event_filter()
1005 return -E2BIG; in kvm_vm_ioctl_set_pmu_event_filter()
1010 return -ENOMEM; in kvm_vm_ioctl_set_pmu_event_filter()
1012 filter->action = tmp.action; in kvm_vm_ioctl_set_pmu_event_filter()
1013 filter->nevents = tmp.nevents; in kvm_vm_ioctl_set_pmu_event_filter()
1014 filter->fixed_counter_bitmap = tmp.fixed_counter_bitmap; in kvm_vm_ioctl_set_pmu_event_filter()
1015 filter->flags = tmp.flags; in kvm_vm_ioctl_set_pmu_event_filter()
1017 r = -EFAULT; in kvm_vm_ioctl_set_pmu_event_filter()
1018 if (copy_from_user(filter->events, user_filter->events, in kvm_vm_ioctl_set_pmu_event_filter()
1019 sizeof(filter->events[0]) * filter->nevents)) in kvm_vm_ioctl_set_pmu_event_filter()
1026 mutex_lock(&kvm->lock); in kvm_vm_ioctl_set_pmu_event_filter()
1027 filter = rcu_replace_pointer(kvm->arch.pmu_event_filter, filter, in kvm_vm_ioctl_set_pmu_event_filter()
1028 mutex_is_locked(&kvm->lock)); in kvm_vm_ioctl_set_pmu_event_filter()
1029 mutex_unlock(&kvm->lock); in kvm_vm_ioctl_set_pmu_event_filter()
1030 synchronize_srcu_expedited(&kvm->srcu); in kvm_vm_ioctl_set_pmu_event_filter()
1032 BUILD_BUG_ON(sizeof(((struct kvm_pmu *)0)->reprogram_pmi) > in kvm_vm_ioctl_set_pmu_event_filter()
1033 sizeof(((struct kvm_pmu *)0)->__reprogram_pmi)); in kvm_vm_ioctl_set_pmu_event_filter()
1036 atomic64_set(&vcpu_to_pmu(vcpu)->__reprogram_pmi, -1ull); in kvm_vm_ioctl_set_pmu_event_filter()