1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3 * Copyright (C) 2016,2017 ARM Limited, All Rights Reserved.
4 * Author: Marc Zyngier <marc.zyngier@arm.com>
5 */
6
7 #include <linux/interrupt.h>
8 #include <linux/irq.h>
9 #include <linux/irqdomain.h>
10 #include <linux/msi.h>
11 #include <linux/pid.h>
12 #include <linux/sched.h>
13
14 #include <linux/irqchip/arm-gic-v4.h>
15
16 /*
17 * WARNING: The blurb below assumes that you understand the
18 * intricacies of GICv3, GICv4, and how a guest's view of a GICv3 gets
19 * translated into GICv4 commands. So it effectively targets at most
20 * two individuals. You know who you are.
21 *
22 * The core GICv4 code is designed to *avoid* exposing too much of the
23 * core GIC code (that would in turn leak into the hypervisor code),
24 * and instead provide a hypervisor agnostic interface to the HW (of
25 * course, the astute reader will quickly realize that hypervisor
26 * agnostic actually means KVM-specific - what were you thinking?).
27 *
28 * In order to achieve a modicum of isolation, we try to hide most of
29 * the GICv4 "stuff" behind normal irqchip operations:
30 *
31 * - Any guest-visible VLPI is backed by a Linux interrupt (and a
32 * physical LPI which gets unmapped when the guest maps the
33 * VLPI). This allows the same DevID/EventID pair to be either
34 * mapped to the LPI (host) or the VLPI (guest). Note that this is
35 * exclusive, and you cannot have both.
36 *
37 * - Enabling/disabling a VLPI is done by issuing mask/unmask calls.
38 *
39 * - Guest INT/CLEAR commands are implemented through
40 * irq_set_irqchip_state().
41 *
42 * - The *bizarre* stuff (mapping/unmapping an interrupt to a VLPI, or
43 * issuing an INV after changing a priority) gets shoved into the
44 * irq_set_vcpu_affinity() method. While this is quite horrible
45 * (let's face it, this is the irqchip version of an ioctl), it
46 * confines the crap to a single location. And map/unmap really is
47 * about setting the affinity of a VLPI to a vcpu, so only INV is
48 * majorly out of place. So there.
49 *
50 * A number of commands are simply not provided by this interface, as
51 * they do not make direct sense. For example, MAPD is purely local to
52 * the virtual ITS (because it references a virtual device, and the
53 * physical ITS is still very much in charge of the physical
54 * device). Same goes for things like MAPC (the physical ITS deals
55 * with the actual vPE affinity, and not the braindead concept of
56 * collection). SYNC is not provided either, as each and every command
57 * is followed by a VSYNC. This could be relaxed in the future, should
58 * this be seen as a bottleneck (yes, this means *never*).
59 *
60 * But handling VLPIs is only one side of the job of the GICv4
61 * code. The other (darker) side is to take care of the doorbell
62 * interrupts which are delivered when a VLPI targeting a non-running
63 * vcpu is being made pending.
64 *
65 * The choice made here is that each vcpu (VPE in old northern GICv4
66 * dialect) gets a single doorbell LPI, no matter how many interrupts
67 * are targeting it. This has a nice property, which is that the
68 * interrupt becomes a handle for the VPE, and that the hypervisor
69 * code can manipulate it through the normal interrupt API:
70 *
71 * - VMs (or rather the VM abstraction that matters to the GIC)
72 * contain an irq domain where each interrupt maps to a VPE. In
73 * turn, this domain sits on top of the normal LPI allocator, and a
74 * specially crafted irq_chip implementation.
75 *
76 * - mask/unmask do what is expected on the doorbell interrupt.
77 *
78 * - irq_set_affinity is used to move a VPE from one redistributor to
79 * another.
80 *
81 * - irq_set_vcpu_affinity once again gets hijacked for the purpose of
82 * creating a new sub-API, namely scheduling/descheduling a VPE
83 * (which involves programming GICR_V{PROP,PEND}BASER) and
84 * performing INVALL operations.
85 */
86
87 static struct irq_domain *gic_domain;
88 static const struct irq_domain_ops *vpe_domain_ops;
89 static const struct irq_domain_ops *sgi_domain_ops;
90
91 #ifdef CONFIG_ARM64
92 #include <asm/cpufeature.h>
93
gic_cpuif_has_vsgi(void)94 bool gic_cpuif_has_vsgi(void)
95 {
96 unsigned long fld, reg = read_sanitised_ftr_reg(SYS_ID_AA64PFR0_EL1);
97
98 fld = cpuid_feature_extract_unsigned_field(reg, ID_AA64PFR0_EL1_GIC_SHIFT);
99
100 return fld >= ID_AA64PFR0_EL1_GIC_V4P1;
101 }
102 #else
gic_cpuif_has_vsgi(void)103 bool gic_cpuif_has_vsgi(void)
104 {
105 return false;
106 }
107 #endif
108
has_v4_1(void)109 static bool has_v4_1(void)
110 {
111 return !!sgi_domain_ops;
112 }
113
has_v4_1_sgi(void)114 static bool has_v4_1_sgi(void)
115 {
116 return has_v4_1() && gic_cpuif_has_vsgi();
117 }
118
its_alloc_vcpu_sgis(struct its_vpe * vpe,int idx)119 static int its_alloc_vcpu_sgis(struct its_vpe *vpe, int idx)
120 {
121 char *name;
122 int sgi_base;
123
124 if (!has_v4_1_sgi())
125 return 0;
126
127 name = kasprintf(GFP_KERNEL, "GICv4-sgi-%d", task_pid_nr(current));
128 if (!name)
129 goto err;
130
131 vpe->fwnode = irq_domain_alloc_named_id_fwnode(name, idx);
132 if (!vpe->fwnode)
133 goto err;
134
135 kfree(name);
136 name = NULL;
137
138 vpe->sgi_domain = irq_domain_create_linear(vpe->fwnode, 16,
139 sgi_domain_ops, vpe);
140 if (!vpe->sgi_domain)
141 goto err;
142
143 sgi_base = irq_domain_alloc_irqs(vpe->sgi_domain, 16, NUMA_NO_NODE, vpe);
144 if (sgi_base <= 0)
145 goto err;
146
147 return 0;
148
149 err:
150 if (vpe->sgi_domain)
151 irq_domain_remove(vpe->sgi_domain);
152 if (vpe->fwnode)
153 irq_domain_free_fwnode(vpe->fwnode);
154 kfree(name);
155 return -ENOMEM;
156 }
157
its_alloc_vcpu_irqs(struct its_vm * vm)158 int its_alloc_vcpu_irqs(struct its_vm *vm)
159 {
160 int vpe_base_irq, i;
161
162 vm->fwnode = irq_domain_alloc_named_id_fwnode("GICv4-vpe",
163 task_pid_nr(current));
164 if (!vm->fwnode)
165 goto err;
166
167 vm->domain = irq_domain_create_hierarchy(gic_domain, 0, vm->nr_vpes,
168 vm->fwnode, vpe_domain_ops,
169 vm);
170 if (!vm->domain)
171 goto err;
172
173 for (i = 0; i < vm->nr_vpes; i++) {
174 vm->vpes[i]->its_vm = vm;
175 vm->vpes[i]->idai = true;
176 }
177
178 vpe_base_irq = irq_domain_alloc_irqs(vm->domain, vm->nr_vpes,
179 NUMA_NO_NODE, vm);
180 if (vpe_base_irq <= 0)
181 goto err;
182
183 for (i = 0; i < vm->nr_vpes; i++) {
184 int ret;
185 vm->vpes[i]->irq = vpe_base_irq + i;
186 ret = its_alloc_vcpu_sgis(vm->vpes[i], i);
187 if (ret)
188 goto err;
189 }
190
191 return 0;
192
193 err:
194 if (vm->domain)
195 irq_domain_remove(vm->domain);
196 if (vm->fwnode)
197 irq_domain_free_fwnode(vm->fwnode);
198
199 return -ENOMEM;
200 }
201
its_free_sgi_irqs(struct its_vm * vm)202 static void its_free_sgi_irqs(struct its_vm *vm)
203 {
204 int i;
205
206 if (!has_v4_1_sgi())
207 return;
208
209 for (i = 0; i < vm->nr_vpes; i++) {
210 unsigned int irq = irq_find_mapping(vm->vpes[i]->sgi_domain, 0);
211
212 if (WARN_ON(!irq))
213 continue;
214
215 irq_domain_free_irqs(irq, 16);
216 irq_domain_remove(vm->vpes[i]->sgi_domain);
217 irq_domain_free_fwnode(vm->vpes[i]->fwnode);
218 }
219 }
220
its_free_vcpu_irqs(struct its_vm * vm)221 void its_free_vcpu_irqs(struct its_vm *vm)
222 {
223 its_free_sgi_irqs(vm);
224 irq_domain_free_irqs(vm->vpes[0]->irq, vm->nr_vpes);
225 irq_domain_remove(vm->domain);
226 irq_domain_free_fwnode(vm->fwnode);
227 }
228
its_send_vpe_cmd(struct its_vpe * vpe,struct its_cmd_info * info)229 static int its_send_vpe_cmd(struct its_vpe *vpe, struct its_cmd_info *info)
230 {
231 return irq_set_vcpu_affinity(vpe->irq, info);
232 }
233
its_make_vpe_non_resident(struct its_vpe * vpe,bool db)234 int its_make_vpe_non_resident(struct its_vpe *vpe, bool db)
235 {
236 struct irq_desc *desc = irq_to_desc(vpe->irq);
237 struct its_cmd_info info = { };
238 int ret;
239
240 WARN_ON(preemptible());
241
242 info.cmd_type = DESCHEDULE_VPE;
243 if (has_v4_1()) {
244 /* GICv4.1 can directly deal with doorbells */
245 info.req_db = db;
246 } else {
247 /* Undo the nested disable_irq() calls... */
248 while (db && irqd_irq_disabled(&desc->irq_data))
249 enable_irq(vpe->irq);
250 }
251
252 ret = its_send_vpe_cmd(vpe, &info);
253 if (!ret)
254 vpe->resident = false;
255
256 vpe->ready = false;
257
258 return ret;
259 }
260
its_make_vpe_resident(struct its_vpe * vpe,bool g0en,bool g1en)261 int its_make_vpe_resident(struct its_vpe *vpe, bool g0en, bool g1en)
262 {
263 struct its_cmd_info info = { };
264 int ret;
265
266 WARN_ON(preemptible());
267
268 info.cmd_type = SCHEDULE_VPE;
269 if (has_v4_1()) {
270 info.g0en = g0en;
271 info.g1en = g1en;
272 } else {
273 /* Disabled the doorbell, as we're about to enter the guest */
274 disable_irq_nosync(vpe->irq);
275 }
276
277 ret = its_send_vpe_cmd(vpe, &info);
278 if (!ret)
279 vpe->resident = true;
280
281 return ret;
282 }
283
its_commit_vpe(struct its_vpe * vpe)284 int its_commit_vpe(struct its_vpe *vpe)
285 {
286 struct its_cmd_info info = {
287 .cmd_type = COMMIT_VPE,
288 };
289 int ret;
290
291 WARN_ON(preemptible());
292
293 ret = its_send_vpe_cmd(vpe, &info);
294 if (!ret)
295 vpe->ready = true;
296
297 return ret;
298 }
299
300
its_invall_vpe(struct its_vpe * vpe)301 int its_invall_vpe(struct its_vpe *vpe)
302 {
303 struct its_cmd_info info = {
304 .cmd_type = INVALL_VPE,
305 };
306
307 return its_send_vpe_cmd(vpe, &info);
308 }
309
its_map_vlpi(int irq,struct its_vlpi_map * map)310 int its_map_vlpi(int irq, struct its_vlpi_map *map)
311 {
312 struct its_cmd_info info = {
313 .cmd_type = MAP_VLPI,
314 {
315 .map = map,
316 },
317 };
318 int ret;
319
320 /*
321 * The host will never see that interrupt firing again, so it
322 * is vital that we don't do any lazy masking.
323 */
324 irq_set_status_flags(irq, IRQ_DISABLE_UNLAZY);
325
326 ret = irq_set_vcpu_affinity(irq, &info);
327 if (ret)
328 irq_clear_status_flags(irq, IRQ_DISABLE_UNLAZY);
329
330 return ret;
331 }
332
its_get_vlpi(int irq,struct its_vlpi_map * map)333 int its_get_vlpi(int irq, struct its_vlpi_map *map)
334 {
335 struct its_cmd_info info = {
336 .cmd_type = GET_VLPI,
337 {
338 .map = map,
339 },
340 };
341
342 return irq_set_vcpu_affinity(irq, &info);
343 }
344
its_unmap_vlpi(int irq)345 int its_unmap_vlpi(int irq)
346 {
347 irq_clear_status_flags(irq, IRQ_DISABLE_UNLAZY);
348 return irq_set_vcpu_affinity(irq, NULL);
349 }
350
its_prop_update_vlpi(int irq,u8 config,bool inv)351 int its_prop_update_vlpi(int irq, u8 config, bool inv)
352 {
353 struct its_cmd_info info = {
354 .cmd_type = inv ? PROP_UPDATE_AND_INV_VLPI : PROP_UPDATE_VLPI,
355 {
356 .config = config,
357 },
358 };
359
360 return irq_set_vcpu_affinity(irq, &info);
361 }
362
its_prop_update_vsgi(int irq,u8 priority,bool group)363 int its_prop_update_vsgi(int irq, u8 priority, bool group)
364 {
365 struct its_cmd_info info = {
366 .cmd_type = PROP_UPDATE_VSGI,
367 {
368 .priority = priority,
369 .group = group,
370 },
371 };
372
373 return irq_set_vcpu_affinity(irq, &info);
374 }
375
its_init_v4(struct irq_domain * domain,const struct irq_domain_ops * vpe_ops,const struct irq_domain_ops * sgi_ops)376 int its_init_v4(struct irq_domain *domain,
377 const struct irq_domain_ops *vpe_ops,
378 const struct irq_domain_ops *sgi_ops)
379 {
380 if (domain) {
381 pr_info("ITS: Enabling GICv4 support\n");
382 gic_domain = domain;
383 vpe_domain_ops = vpe_ops;
384 sgi_domain_ops = sgi_ops;
385 return 0;
386 }
387
388 pr_err("ITS: No GICv4 VPE domain allocated\n");
389 return -ENODEV;
390 }
391