1 // SPDX-License-Identifier: GPL-2.0
2 /*
3 * Copyright (C) 2021 Western Digital Corporation or its affiliates.
4 * Copyright (C) 2022 Ventana Micro Systems Inc.
5 */
6
7 #include <linux/bitfield.h>
8 #include <linux/bitops.h>
9 #include <linux/cpu.h>
10 #include <linux/interrupt.h>
11 #include <linux/irqchip.h>
12 #include <linux/irqchip/riscv-aplic.h>
13 #include <linux/irqchip/riscv-imsic.h>
14 #include <linux/module.h>
15 #include <linux/msi.h>
16 #include <linux/of_irq.h>
17 #include <linux/platform_device.h>
18 #include <linux/printk.h>
19 #include <linux/smp.h>
20
21 #include "irq-riscv-aplic-main.h"
22
aplic_msi_irq_mask(struct irq_data * d)23 static void aplic_msi_irq_mask(struct irq_data *d)
24 {
25 aplic_irq_mask(d);
26 irq_chip_mask_parent(d);
27 }
28
aplic_msi_irq_unmask(struct irq_data * d)29 static void aplic_msi_irq_unmask(struct irq_data *d)
30 {
31 irq_chip_unmask_parent(d);
32 aplic_irq_unmask(d);
33 }
34
aplic_msi_irq_retrigger_level(struct irq_data * d)35 static void aplic_msi_irq_retrigger_level(struct irq_data *d)
36 {
37 struct aplic_priv *priv = irq_data_get_irq_chip_data(d);
38
39 switch (irqd_get_trigger_type(d)) {
40 case IRQ_TYPE_LEVEL_LOW:
41 case IRQ_TYPE_LEVEL_HIGH:
42 /*
43 * The section "4.9.2 Special consideration for level-sensitive interrupt
44 * sources" of the RISC-V AIA specification says:
45 *
46 * A second option is for the interrupt service routine to write the
47 * APLIC’s source identity number for the interrupt to the domain’s
48 * setipnum register just before exiting. This will cause the interrupt’s
49 * pending bit to be set to one again if the source is still asserting
50 * an interrupt, but not if the source is not asserting an interrupt.
51 */
52 writel(d->hwirq, priv->regs + APLIC_SETIPNUM_LE);
53 break;
54 }
55 }
56
aplic_msi_irq_eoi(struct irq_data * d)57 static void aplic_msi_irq_eoi(struct irq_data *d)
58 {
59 /*
60 * EOI handling is required only for level-triggered interrupts
61 * when APLIC is in MSI mode.
62 */
63 aplic_msi_irq_retrigger_level(d);
64 }
65
aplic_msi_irq_set_type(struct irq_data * d,unsigned int type)66 static int aplic_msi_irq_set_type(struct irq_data *d, unsigned int type)
67 {
68 int rc = aplic_irq_set_type(d, type);
69
70 if (rc)
71 return rc;
72 /*
73 * Updating sourcecfg register for level-triggered interrupts
74 * requires interrupt retriggering when APLIC is in MSI mode.
75 */
76 aplic_msi_irq_retrigger_level(d);
77 return 0;
78 }
79
aplic_msi_write_msg(struct irq_data * d,struct msi_msg * msg)80 static void aplic_msi_write_msg(struct irq_data *d, struct msi_msg *msg)
81 {
82 unsigned int group_index, hart_index, guest_index, val;
83 struct aplic_priv *priv = irq_data_get_irq_chip_data(d);
84 struct aplic_msicfg *mc = &priv->msicfg;
85 phys_addr_t tppn, tbppn, msg_addr;
86 void __iomem *target;
87
88 /* For zeroed MSI, simply write zero into the target register */
89 if (!msg->address_hi && !msg->address_lo && !msg->data) {
90 target = priv->regs + APLIC_TARGET_BASE;
91 target += (d->hwirq - 1) * sizeof(u32);
92 writel(0, target);
93 return;
94 }
95
96 /* Sanity check on message data */
97 WARN_ON(msg->data > APLIC_TARGET_EIID_MASK);
98
99 /* Compute target MSI address */
100 msg_addr = (((u64)msg->address_hi) << 32) | msg->address_lo;
101 tppn = msg_addr >> APLIC_xMSICFGADDR_PPN_SHIFT;
102
103 /* Compute target HART Base PPN */
104 tbppn = tppn;
105 tbppn &= ~APLIC_xMSICFGADDR_PPN_HART(mc->lhxs);
106 tbppn &= ~APLIC_xMSICFGADDR_PPN_LHX(mc->lhxw, mc->lhxs);
107 tbppn &= ~APLIC_xMSICFGADDR_PPN_HHX(mc->hhxw, mc->hhxs);
108 WARN_ON(tbppn != mc->base_ppn);
109
110 /* Compute target group and hart indexes */
111 group_index = (tppn >> APLIC_xMSICFGADDR_PPN_HHX_SHIFT(mc->hhxs)) &
112 APLIC_xMSICFGADDR_PPN_HHX_MASK(mc->hhxw);
113 hart_index = (tppn >> APLIC_xMSICFGADDR_PPN_LHX_SHIFT(mc->lhxs)) &
114 APLIC_xMSICFGADDR_PPN_LHX_MASK(mc->lhxw);
115 hart_index |= (group_index << mc->lhxw);
116 WARN_ON(hart_index > APLIC_TARGET_HART_IDX_MASK);
117
118 /* Compute target guest index */
119 guest_index = tppn & APLIC_xMSICFGADDR_PPN_HART(mc->lhxs);
120 WARN_ON(guest_index > APLIC_TARGET_GUEST_IDX_MASK);
121
122 /* Update IRQ TARGET register */
123 target = priv->regs + APLIC_TARGET_BASE;
124 target += (d->hwirq - 1) * sizeof(u32);
125 val = FIELD_PREP(APLIC_TARGET_HART_IDX, hart_index);
126 val |= FIELD_PREP(APLIC_TARGET_GUEST_IDX, guest_index);
127 val |= FIELD_PREP(APLIC_TARGET_EIID, msg->data);
128 writel(val, target);
129 }
130
aplic_msi_set_desc(msi_alloc_info_t * arg,struct msi_desc * desc)131 static void aplic_msi_set_desc(msi_alloc_info_t *arg, struct msi_desc *desc)
132 {
133 arg->desc = desc;
134 arg->hwirq = (u32)desc->data.icookie.value;
135 }
136
aplic_msi_translate(struct irq_domain * d,struct irq_fwspec * fwspec,unsigned long * hwirq,unsigned int * type)137 static int aplic_msi_translate(struct irq_domain *d, struct irq_fwspec *fwspec,
138 unsigned long *hwirq, unsigned int *type)
139 {
140 struct msi_domain_info *info = d->host_data;
141 struct aplic_priv *priv = info->data;
142
143 return aplic_irqdomain_translate(fwspec, priv->gsi_base, hwirq, type);
144 }
145
146 static const struct msi_domain_template aplic_msi_template = {
147 .chip = {
148 .name = "APLIC-MSI",
149 .irq_mask = aplic_msi_irq_mask,
150 .irq_unmask = aplic_msi_irq_unmask,
151 .irq_set_type = aplic_msi_irq_set_type,
152 .irq_eoi = aplic_msi_irq_eoi,
153 #ifdef CONFIG_SMP
154 .irq_set_affinity = irq_chip_set_affinity_parent,
155 #endif
156 .irq_write_msi_msg = aplic_msi_write_msg,
157 .flags = IRQCHIP_SET_TYPE_MASKED |
158 IRQCHIP_SKIP_SET_WAKE |
159 IRQCHIP_MASK_ON_SUSPEND,
160 },
161
162 .ops = {
163 .set_desc = aplic_msi_set_desc,
164 .msi_translate = aplic_msi_translate,
165 },
166
167 .info = {
168 .bus_token = DOMAIN_BUS_WIRED_TO_MSI,
169 .flags = MSI_FLAG_USE_DEV_FWNODE,
170 .handler = handle_fasteoi_irq,
171 .handler_name = "fasteoi",
172 },
173 };
174
aplic_msi_setup(struct device * dev,void __iomem * regs)175 int aplic_msi_setup(struct device *dev, void __iomem *regs)
176 {
177 const struct imsic_global_config *imsic_global;
178 struct irq_domain *msi_domain;
179 struct aplic_priv *priv;
180 struct aplic_msicfg *mc;
181 phys_addr_t pa;
182 int rc;
183
184 priv = devm_kzalloc(dev, sizeof(*priv), GFP_KERNEL);
185 if (!priv)
186 return -ENOMEM;
187
188 rc = aplic_setup_priv(priv, dev, regs);
189 if (rc) {
190 dev_err(dev, "failed to create APLIC context\n");
191 return rc;
192 }
193 mc = &priv->msicfg;
194
195 /*
196 * The APLIC outgoing MSI config registers assume target MSI
197 * controller to be RISC-V AIA IMSIC controller.
198 */
199 imsic_global = imsic_get_global_config();
200 if (!imsic_global) {
201 dev_err(dev, "IMSIC global config not found\n");
202 return -ENODEV;
203 }
204
205 /* Find number of guest index bits (LHXS) */
206 mc->lhxs = imsic_global->guest_index_bits;
207 if (APLIC_xMSICFGADDRH_LHXS_MASK < mc->lhxs) {
208 dev_err(dev, "IMSIC guest index bits big for APLIC LHXS\n");
209 return -EINVAL;
210 }
211
212 /* Find number of HART index bits (LHXW) */
213 mc->lhxw = imsic_global->hart_index_bits;
214 if (APLIC_xMSICFGADDRH_LHXW_MASK < mc->lhxw) {
215 dev_err(dev, "IMSIC hart index bits big for APLIC LHXW\n");
216 return -EINVAL;
217 }
218
219 /* Find number of group index bits (HHXW) */
220 mc->hhxw = imsic_global->group_index_bits;
221 if (APLIC_xMSICFGADDRH_HHXW_MASK < mc->hhxw) {
222 dev_err(dev, "IMSIC group index bits big for APLIC HHXW\n");
223 return -EINVAL;
224 }
225
226 /* Find first bit position of group index (HHXS) */
227 mc->hhxs = imsic_global->group_index_shift;
228 if (mc->hhxs < (2 * APLIC_xMSICFGADDR_PPN_SHIFT)) {
229 dev_err(dev, "IMSIC group index shift should be >= %d\n",
230 (2 * APLIC_xMSICFGADDR_PPN_SHIFT));
231 return -EINVAL;
232 }
233 mc->hhxs -= (2 * APLIC_xMSICFGADDR_PPN_SHIFT);
234 if (APLIC_xMSICFGADDRH_HHXS_MASK < mc->hhxs) {
235 dev_err(dev, "IMSIC group index shift big for APLIC HHXS\n");
236 return -EINVAL;
237 }
238
239 /* Compute PPN base */
240 mc->base_ppn = imsic_global->base_addr >> APLIC_xMSICFGADDR_PPN_SHIFT;
241 mc->base_ppn &= ~APLIC_xMSICFGADDR_PPN_HART(mc->lhxs);
242 mc->base_ppn &= ~APLIC_xMSICFGADDR_PPN_LHX(mc->lhxw, mc->lhxs);
243 mc->base_ppn &= ~APLIC_xMSICFGADDR_PPN_HHX(mc->hhxw, mc->hhxs);
244
245 /* Setup global config and interrupt delivery */
246 aplic_init_hw_global(priv, true);
247
248 /* Set the APLIC device MSI domain if not available */
249 if (!dev_get_msi_domain(dev)) {
250 /*
251 * The device MSI domain for OF devices is only set at the
252 * time of populating/creating OF device. If the device MSI
253 * domain is discovered later after the OF device is created
254 * then we need to set it explicitly before using any platform
255 * MSI functions.
256 *
257 * In case of APLIC device, the parent MSI domain is always
258 * IMSIC and the IMSIC MSI domains are created later through
259 * the platform driver probing so we set it explicitly here.
260 */
261 if (is_of_node(dev->fwnode)) {
262 of_msi_configure(dev, to_of_node(dev->fwnode));
263 } else {
264 msi_domain = irq_find_matching_fwnode(imsic_acpi_get_fwnode(dev),
265 DOMAIN_BUS_PLATFORM_MSI);
266 if (msi_domain)
267 dev_set_msi_domain(dev, msi_domain);
268 }
269 }
270
271 if (!msi_create_device_irq_domain(dev, MSI_DEFAULT_DOMAIN, &aplic_msi_template,
272 priv->nr_irqs + 1, priv, priv)) {
273 dev_err(dev, "failed to create MSI irq domain\n");
274 return -ENOMEM;
275 }
276
277 /* Advertise the interrupt controller */
278 pa = priv->msicfg.base_ppn << APLIC_xMSICFGADDR_PPN_SHIFT;
279 dev_info(dev, "%d interrupts forwarded to MSI base %pa\n", priv->nr_irqs, &pa);
280
281 return 0;
282 }
283