1 // SPDX-License-Identifier: GPL-2.0
2 /*
3 * Copyright 2019 Arm Limited
4 * Author: Andrew Murray <Andrew.Murray@arm.com>
5 */
6 #include <linux/kvm_host.h>
7 #include <linux/perf_event.h>
8 #include <linux/perf/arm_pmu.h>
9 #include <linux/perf/arm_pmuv3.h>
10
11 static DEFINE_PER_CPU(struct kvm_pmu_events, kvm_pmu_events);
12
13 /*
14 * Given the perf event attributes and system type, determine
15 * if we are going to need to switch counters at guest entry/exit.
16 */
kvm_pmu_switch_needed(struct perf_event_attr * attr)17 static bool kvm_pmu_switch_needed(struct perf_event_attr *attr)
18 {
19 /**
20 * With VHE the guest kernel runs at EL1 and the host at EL2,
21 * where user (EL0) is excluded then we have no reason to switch
22 * counters.
23 */
24 if (has_vhe() && attr->exclude_user)
25 return false;
26
27 /* Only switch if attributes are different */
28 return (attr->exclude_host != attr->exclude_guest);
29 }
30
kvm_get_pmu_events(void)31 struct kvm_pmu_events *kvm_get_pmu_events(void)
32 {
33 return this_cpu_ptr(&kvm_pmu_events);
34 }
35
36 /*
37 * Add events to track that we may want to switch at guest entry/exit
38 * time.
39 */
kvm_set_pmu_events(u64 set,struct perf_event_attr * attr)40 void kvm_set_pmu_events(u64 set, struct perf_event_attr *attr)
41 {
42 struct kvm_pmu_events *pmu = kvm_get_pmu_events();
43
44 if (!kvm_arm_support_pmu_v3() || !kvm_pmu_switch_needed(attr))
45 return;
46
47 if (!attr->exclude_host)
48 pmu->events_host |= set;
49 if (!attr->exclude_guest)
50 pmu->events_guest |= set;
51 }
52
53 /*
54 * Stop tracking events
55 */
kvm_clr_pmu_events(u64 clr)56 void kvm_clr_pmu_events(u64 clr)
57 {
58 struct kvm_pmu_events *pmu = kvm_get_pmu_events();
59
60 if (!kvm_arm_support_pmu_v3())
61 return;
62
63 pmu->events_host &= ~clr;
64 pmu->events_guest &= ~clr;
65 }
66
67 /*
68 * Read a value direct from PMEVTYPER<idx> where idx is 0-30
69 * or PMxCFILTR_EL0 where idx is 31-32.
70 */
kvm_vcpu_pmu_read_evtype_direct(int idx)71 static u64 kvm_vcpu_pmu_read_evtype_direct(int idx)
72 {
73 if (idx == ARMV8_PMU_CYCLE_IDX)
74 return read_pmccfiltr();
75 else if (idx == ARMV8_PMU_INSTR_IDX)
76 return read_pmicfiltr();
77
78 return read_pmevtypern(idx);
79 }
80
81 /*
82 * Write a value direct to PMEVTYPER<idx> where idx is 0-30
83 * or PMxCFILTR_EL0 where idx is 31-32.
84 */
kvm_vcpu_pmu_write_evtype_direct(int idx,u32 val)85 static void kvm_vcpu_pmu_write_evtype_direct(int idx, u32 val)
86 {
87 if (idx == ARMV8_PMU_CYCLE_IDX)
88 write_pmccfiltr(val);
89 else if (idx == ARMV8_PMU_INSTR_IDX)
90 write_pmicfiltr(val);
91 else
92 write_pmevtypern(idx, val);
93 }
94
95 /*
96 * Modify ARMv8 PMU events to include EL0 counting
97 */
kvm_vcpu_pmu_enable_el0(unsigned long events)98 static void kvm_vcpu_pmu_enable_el0(unsigned long events)
99 {
100 u64 typer;
101 u32 counter;
102
103 for_each_set_bit(counter, &events, ARMPMU_MAX_HWEVENTS) {
104 typer = kvm_vcpu_pmu_read_evtype_direct(counter);
105 typer &= ~ARMV8_PMU_EXCLUDE_EL0;
106 kvm_vcpu_pmu_write_evtype_direct(counter, typer);
107 }
108 }
109
110 /*
111 * Modify ARMv8 PMU events to exclude EL0 counting
112 */
kvm_vcpu_pmu_disable_el0(unsigned long events)113 static void kvm_vcpu_pmu_disable_el0(unsigned long events)
114 {
115 u64 typer;
116 u32 counter;
117
118 for_each_set_bit(counter, &events, ARMPMU_MAX_HWEVENTS) {
119 typer = kvm_vcpu_pmu_read_evtype_direct(counter);
120 typer |= ARMV8_PMU_EXCLUDE_EL0;
121 kvm_vcpu_pmu_write_evtype_direct(counter, typer);
122 }
123 }
124
125 /*
126 * On VHE ensure that only guest events have EL0 counting enabled.
127 * This is called from both vcpu_{load,put} and the sysreg handling.
128 * Since the latter is preemptible, special care must be taken to
129 * disable preemption.
130 */
kvm_vcpu_pmu_restore_guest(struct kvm_vcpu * vcpu)131 void kvm_vcpu_pmu_restore_guest(struct kvm_vcpu *vcpu)
132 {
133 struct kvm_pmu_events *pmu;
134 u64 events_guest, events_host;
135
136 if (!kvm_arm_support_pmu_v3() || !has_vhe())
137 return;
138
139 preempt_disable();
140 pmu = kvm_get_pmu_events();
141 events_guest = pmu->events_guest;
142 events_host = pmu->events_host;
143
144 kvm_vcpu_pmu_enable_el0(events_guest);
145 kvm_vcpu_pmu_disable_el0(events_host);
146 preempt_enable();
147 }
148
149 /*
150 * On VHE ensure that only host events have EL0 counting enabled
151 */
kvm_vcpu_pmu_restore_host(struct kvm_vcpu * vcpu)152 void kvm_vcpu_pmu_restore_host(struct kvm_vcpu *vcpu)
153 {
154 struct kvm_pmu_events *pmu;
155 u64 events_guest, events_host;
156
157 if (!kvm_arm_support_pmu_v3() || !has_vhe())
158 return;
159
160 pmu = kvm_get_pmu_events();
161 events_guest = pmu->events_guest;
162 events_host = pmu->events_host;
163
164 kvm_vcpu_pmu_enable_el0(events_host);
165 kvm_vcpu_pmu_disable_el0(events_guest);
166 }
167
168 /*
169 * With VHE, keep track of the PMUSERENR_EL0 value for the host EL0 on the pCPU
170 * where PMUSERENR_EL0 for the guest is loaded, since PMUSERENR_EL0 is switched
171 * to the value for the guest on vcpu_load(). The value for the host EL0
172 * will be restored on vcpu_put(), before returning to userspace.
173 * This isn't necessary for nVHE, as the register is context switched for
174 * every guest enter/exit.
175 *
176 * Return true if KVM takes care of the register. Otherwise return false.
177 */
kvm_set_pmuserenr(u64 val)178 bool kvm_set_pmuserenr(u64 val)
179 {
180 struct kvm_cpu_context *hctxt;
181 struct kvm_vcpu *vcpu;
182
183 if (!kvm_arm_support_pmu_v3() || !has_vhe())
184 return false;
185
186 vcpu = kvm_get_running_vcpu();
187 if (!vcpu || !vcpu_get_flag(vcpu, PMUSERENR_ON_CPU))
188 return false;
189
190 hctxt = host_data_ptr(host_ctxt);
191 ctxt_sys_reg(hctxt, PMUSERENR_EL0) = val;
192 return true;
193 }
194
195 /*
196 * If we interrupted the guest to update the host PMU context, make
197 * sure we re-apply the guest EL0 state.
198 */
kvm_vcpu_pmu_resync_el0(void)199 void kvm_vcpu_pmu_resync_el0(void)
200 {
201 struct kvm_vcpu *vcpu;
202
203 if (!has_vhe() || !in_interrupt())
204 return;
205
206 vcpu = kvm_get_running_vcpu();
207 if (!vcpu)
208 return;
209
210 kvm_make_request(KVM_REQ_RESYNC_PMU_EL0, vcpu);
211 }
212