1 // SPDX-License-Identifier: GPL-2.0
2 /*
3 * Copyright (C) 2021 Western Digital Corporation or its affiliates.
4 * Copyright (C) 2022 Ventana Micro Systems Inc.
5 */
6
7 #include <linux/acpi.h>
8 #include <linux/bitfield.h>
9 #include <linux/bitops.h>
10 #include <linux/cpu.h>
11 #include <linux/interrupt.h>
12 #include <linux/irqchip.h>
13 #include <linux/irqchip/chained_irq.h>
14 #include <linux/irqchip/riscv-aplic.h>
15 #include <linux/module.h>
16 #include <linux/of_address.h>
17 #include <linux/printk.h>
18 #include <linux/smp.h>
19
20 #include "irq-riscv-aplic-main.h"
21
22 #define APLIC_DISABLE_IDELIVERY 0
23 #define APLIC_ENABLE_IDELIVERY 1
24 #define APLIC_DISABLE_ITHRESHOLD 1
25 #define APLIC_ENABLE_ITHRESHOLD 0
26
27 struct aplic_direct {
28 struct aplic_priv priv;
29 struct irq_domain *irqdomain;
30 struct cpumask lmask;
31 };
32
33 struct aplic_idc {
34 unsigned int hart_index;
35 void __iomem *regs;
36 struct aplic_direct *direct;
37 };
38
39 static unsigned int aplic_direct_parent_irq;
40 static DEFINE_PER_CPU(struct aplic_idc, aplic_idcs);
41
aplic_direct_irq_eoi(struct irq_data * d)42 static void aplic_direct_irq_eoi(struct irq_data *d)
43 {
44 /*
45 * The fasteoi_handler requires irq_eoi() callback hence
46 * provide a dummy handler.
47 */
48 }
49
50 #ifdef CONFIG_SMP
aplic_direct_set_affinity(struct irq_data * d,const struct cpumask * mask_val,bool force)51 static int aplic_direct_set_affinity(struct irq_data *d, const struct cpumask *mask_val,
52 bool force)
53 {
54 struct aplic_priv *priv = irq_data_get_irq_chip_data(d);
55 struct aplic_direct *direct = container_of(priv, struct aplic_direct, priv);
56 struct aplic_idc *idc;
57 unsigned int cpu, val;
58 void __iomem *target;
59
60 if (force)
61 cpu = cpumask_first_and(&direct->lmask, mask_val);
62 else
63 cpu = cpumask_first_and_and(&direct->lmask, mask_val, cpu_online_mask);
64
65 if (cpu >= nr_cpu_ids)
66 return -EINVAL;
67
68 idc = per_cpu_ptr(&aplic_idcs, cpu);
69 target = priv->regs + APLIC_TARGET_BASE + (d->hwirq - 1) * sizeof(u32);
70 val = FIELD_PREP(APLIC_TARGET_HART_IDX, idc->hart_index);
71 val |= FIELD_PREP(APLIC_TARGET_IPRIO, APLIC_DEFAULT_PRIORITY);
72 writel(val, target);
73
74 irq_data_update_effective_affinity(d, cpumask_of(cpu));
75
76 return IRQ_SET_MASK_OK_DONE;
77 }
78 #endif
79
80 static struct irq_chip aplic_direct_chip = {
81 .name = "APLIC-DIRECT",
82 .irq_mask = aplic_irq_mask,
83 .irq_unmask = aplic_irq_unmask,
84 .irq_set_type = aplic_irq_set_type,
85 .irq_eoi = aplic_direct_irq_eoi,
86 #ifdef CONFIG_SMP
87 .irq_set_affinity = aplic_direct_set_affinity,
88 #endif
89 .flags = IRQCHIP_SET_TYPE_MASKED |
90 IRQCHIP_SKIP_SET_WAKE |
91 IRQCHIP_MASK_ON_SUSPEND,
92 };
93
aplic_direct_irqdomain_translate(struct irq_domain * d,struct irq_fwspec * fwspec,unsigned long * hwirq,unsigned int * type)94 static int aplic_direct_irqdomain_translate(struct irq_domain *d, struct irq_fwspec *fwspec,
95 unsigned long *hwirq, unsigned int *type)
96 {
97 struct aplic_priv *priv = d->host_data;
98
99 return aplic_irqdomain_translate(fwspec, priv->gsi_base, hwirq, type);
100 }
101
aplic_direct_irqdomain_alloc(struct irq_domain * domain,unsigned int virq,unsigned int nr_irqs,void * arg)102 static int aplic_direct_irqdomain_alloc(struct irq_domain *domain, unsigned int virq,
103 unsigned int nr_irqs, void *arg)
104 {
105 struct aplic_priv *priv = domain->host_data;
106 struct aplic_direct *direct = container_of(priv, struct aplic_direct, priv);
107 struct irq_fwspec *fwspec = arg;
108 irq_hw_number_t hwirq;
109 unsigned int type;
110 int i, ret;
111
112 ret = aplic_irqdomain_translate(fwspec, priv->gsi_base, &hwirq, &type);
113 if (ret)
114 return ret;
115
116 for (i = 0; i < nr_irqs; i++) {
117 irq_domain_set_info(domain, virq + i, hwirq + i, &aplic_direct_chip,
118 priv, handle_fasteoi_irq, NULL, NULL);
119 irq_set_affinity(virq + i, &direct->lmask);
120 }
121
122 return 0;
123 }
124
125 static const struct irq_domain_ops aplic_direct_irqdomain_ops = {
126 .translate = aplic_direct_irqdomain_translate,
127 .alloc = aplic_direct_irqdomain_alloc,
128 .free = irq_domain_free_irqs_top,
129 };
130
131 /*
132 * To handle an APLIC direct interrupts, we just read the CLAIMI register
133 * which will return highest priority pending interrupt and clear the
134 * pending bit of the interrupt. This process is repeated until CLAIMI
135 * register return zero value.
136 */
aplic_direct_handle_irq(struct irq_desc * desc)137 static void aplic_direct_handle_irq(struct irq_desc *desc)
138 {
139 struct aplic_idc *idc = this_cpu_ptr(&aplic_idcs);
140 struct irq_domain *irqdomain = idc->direct->irqdomain;
141 struct irq_chip *chip = irq_desc_get_chip(desc);
142 irq_hw_number_t hw_irq;
143 int irq;
144
145 chained_irq_enter(chip, desc);
146
147 while ((hw_irq = readl(idc->regs + APLIC_IDC_CLAIMI))) {
148 hw_irq = hw_irq >> APLIC_IDC_TOPI_ID_SHIFT;
149 irq = irq_find_mapping(irqdomain, hw_irq);
150
151 if (unlikely(irq <= 0)) {
152 dev_warn_ratelimited(idc->direct->priv.dev,
153 "hw_irq %lu mapping not found\n", hw_irq);
154 } else {
155 generic_handle_irq(irq);
156 }
157 }
158
159 chained_irq_exit(chip, desc);
160 }
161
aplic_idc_set_delivery(struct aplic_idc * idc,bool en)162 static void aplic_idc_set_delivery(struct aplic_idc *idc, bool en)
163 {
164 u32 de = (en) ? APLIC_ENABLE_IDELIVERY : APLIC_DISABLE_IDELIVERY;
165 u32 th = (en) ? APLIC_ENABLE_ITHRESHOLD : APLIC_DISABLE_ITHRESHOLD;
166
167 /* Priority must be less than threshold for interrupt triggering */
168 writel(th, idc->regs + APLIC_IDC_ITHRESHOLD);
169
170 /* Delivery must be set to 1 for interrupt triggering */
171 writel(de, idc->regs + APLIC_IDC_IDELIVERY);
172 }
173
aplic_direct_dying_cpu(unsigned int cpu)174 static int aplic_direct_dying_cpu(unsigned int cpu)
175 {
176 if (aplic_direct_parent_irq)
177 disable_percpu_irq(aplic_direct_parent_irq);
178
179 return 0;
180 }
181
aplic_direct_starting_cpu(unsigned int cpu)182 static int aplic_direct_starting_cpu(unsigned int cpu)
183 {
184 if (aplic_direct_parent_irq) {
185 enable_percpu_irq(aplic_direct_parent_irq,
186 irq_get_trigger_type(aplic_direct_parent_irq));
187 }
188
189 return 0;
190 }
191
aplic_direct_parse_parent_hwirq(struct device * dev,u32 index,u32 * parent_hwirq,unsigned long * parent_hartid,struct aplic_priv * priv)192 static int aplic_direct_parse_parent_hwirq(struct device *dev, u32 index,
193 u32 *parent_hwirq, unsigned long *parent_hartid,
194 struct aplic_priv *priv)
195 {
196 struct of_phandle_args parent;
197 unsigned long hartid;
198 int rc;
199
200 if (!is_of_node(dev->fwnode)) {
201 hartid = acpi_rintc_ext_parent_to_hartid(priv->acpi_aplic_id, index);
202 if (hartid == INVALID_HARTID)
203 return -ENODEV;
204
205 *parent_hartid = hartid;
206 *parent_hwirq = RV_IRQ_EXT;
207 return 0;
208 }
209
210 rc = of_irq_parse_one(to_of_node(dev->fwnode), index, &parent);
211 if (rc)
212 return rc;
213
214 rc = riscv_of_parent_hartid(parent.np, parent_hartid);
215 if (rc)
216 return rc;
217
218 *parent_hwirq = parent.args[0];
219 return 0;
220 }
221
aplic_direct_setup(struct device * dev,void __iomem * regs)222 int aplic_direct_setup(struct device *dev, void __iomem *regs)
223 {
224 int i, j, rc, cpu, current_cpu, setup_count = 0;
225 struct aplic_direct *direct;
226 struct irq_domain *domain;
227 struct aplic_priv *priv;
228 struct aplic_idc *idc;
229 unsigned long hartid;
230 u32 v, hwirq;
231
232 direct = devm_kzalloc(dev, sizeof(*direct), GFP_KERNEL);
233 if (!direct)
234 return -ENOMEM;
235 priv = &direct->priv;
236
237 rc = aplic_setup_priv(priv, dev, regs);
238 if (rc) {
239 dev_err(dev, "failed to create APLIC context\n");
240 return rc;
241 }
242
243 /* Setup per-CPU IDC and target CPU mask */
244 current_cpu = get_cpu();
245 for (i = 0; i < priv->nr_idcs; i++) {
246 rc = aplic_direct_parse_parent_hwirq(dev, i, &hwirq, &hartid, priv);
247 if (rc) {
248 dev_warn(dev, "parent irq for IDC%d not found\n", i);
249 continue;
250 }
251
252 /*
253 * Skip interrupts other than external interrupts for
254 * current privilege level.
255 */
256 if (hwirq != RV_IRQ_EXT)
257 continue;
258
259 cpu = riscv_hartid_to_cpuid(hartid);
260 if (cpu < 0) {
261 dev_warn(dev, "invalid cpuid for IDC%d\n", i);
262 continue;
263 }
264
265 cpumask_set_cpu(cpu, &direct->lmask);
266
267 idc = per_cpu_ptr(&aplic_idcs, cpu);
268 idc->hart_index = i;
269 idc->regs = priv->regs + APLIC_IDC_BASE + i * APLIC_IDC_SIZE;
270 idc->direct = direct;
271
272 aplic_idc_set_delivery(idc, true);
273
274 /*
275 * Boot cpu might not have APLIC hart_index = 0 so check
276 * and update target registers of all interrupts.
277 */
278 if (cpu == current_cpu && idc->hart_index) {
279 v = FIELD_PREP(APLIC_TARGET_HART_IDX, idc->hart_index);
280 v |= FIELD_PREP(APLIC_TARGET_IPRIO, APLIC_DEFAULT_PRIORITY);
281 for (j = 1; j <= priv->nr_irqs; j++)
282 writel(v, priv->regs + APLIC_TARGET_BASE + (j - 1) * sizeof(u32));
283 }
284
285 setup_count++;
286 }
287 put_cpu();
288
289 /* Find parent domain and register chained handler */
290 domain = irq_find_matching_fwnode(riscv_get_intc_hwnode(),
291 DOMAIN_BUS_ANY);
292 if (!aplic_direct_parent_irq && domain) {
293 aplic_direct_parent_irq = irq_create_mapping(domain, RV_IRQ_EXT);
294 if (aplic_direct_parent_irq) {
295 irq_set_chained_handler(aplic_direct_parent_irq,
296 aplic_direct_handle_irq);
297
298 /*
299 * Setup CPUHP notifier to enable parent
300 * interrupt on all CPUs
301 */
302 cpuhp_setup_state(CPUHP_AP_ONLINE_DYN,
303 "irqchip/riscv/aplic:starting",
304 aplic_direct_starting_cpu,
305 aplic_direct_dying_cpu);
306 }
307 }
308
309 /* Fail if we were not able to setup IDC for any CPU */
310 if (!setup_count)
311 return -ENODEV;
312
313 /* Setup global config and interrupt delivery */
314 aplic_init_hw_global(priv, false);
315
316 /* Create irq domain instance for the APLIC */
317 direct->irqdomain = irq_domain_create_linear(dev->fwnode, priv->nr_irqs + 1,
318 &aplic_direct_irqdomain_ops, priv);
319 if (!direct->irqdomain) {
320 dev_err(dev, "failed to create direct irq domain\n");
321 return -ENOMEM;
322 }
323
324 /* Advertise the interrupt controller */
325 dev_info(dev, "%d interrupts directly connected to %d CPUs\n",
326 priv->nr_irqs, priv->nr_idcs);
327
328 return 0;
329 }
330