1 // SPDX-License-Identifier: GPL-2.0
2 /*
3  * Copyright (C) 2021 Western Digital Corporation or its affiliates.
4  * Copyright (C) 2022 Ventana Micro Systems Inc.
5  */
6 
7 #define pr_fmt(fmt) "riscv-imsic: " fmt
8 #include <linux/acpi.h>
9 #include <linux/cpu.h>
10 #include <linux/interrupt.h>
11 #include <linux/io.h>
12 #include <linux/irq.h>
13 #include <linux/irqchip.h>
14 #include <linux/irqchip/chained_irq.h>
15 #include <linux/irqchip/riscv-imsic.h>
16 #include <linux/module.h>
17 #include <linux/pci.h>
18 #include <linux/spinlock.h>
19 #include <linux/smp.h>
20 
21 #include "irq-riscv-imsic-state.h"
22 
23 static int imsic_parent_irq;
24 
25 #ifdef CONFIG_SMP
imsic_ipi_send(unsigned int cpu)26 static void imsic_ipi_send(unsigned int cpu)
27 {
28 	struct imsic_local_config *local = per_cpu_ptr(imsic->global.local, cpu);
29 
30 	writel_relaxed(IMSIC_IPI_ID, local->msi_va);
31 }
32 
imsic_ipi_starting_cpu(void)33 static void imsic_ipi_starting_cpu(void)
34 {
35 	/* Enable IPIs for current CPU. */
36 	__imsic_id_set_enable(IMSIC_IPI_ID);
37 }
38 
imsic_ipi_dying_cpu(void)39 static void imsic_ipi_dying_cpu(void)
40 {
41 	/* Disable IPIs for current CPU. */
42 	__imsic_id_clear_enable(IMSIC_IPI_ID);
43 }
44 
imsic_ipi_domain_init(void)45 static int __init imsic_ipi_domain_init(void)
46 {
47 	int virq;
48 
49 	/* Create IMSIC IPI multiplexing */
50 	virq = ipi_mux_create(IMSIC_NR_IPI, imsic_ipi_send);
51 	if (virq <= 0)
52 		return virq < 0 ? virq : -ENOMEM;
53 
54 	/* Set vIRQ range */
55 	riscv_ipi_set_virq_range(virq, IMSIC_NR_IPI);
56 
57 	/* Announce that IMSIC is providing IPIs */
58 	pr_info("%pfwP: providing IPIs using interrupt %d\n", imsic->fwnode, IMSIC_IPI_ID);
59 
60 	return 0;
61 }
62 #else
imsic_ipi_starting_cpu(void)63 static void imsic_ipi_starting_cpu(void) { }
imsic_ipi_dying_cpu(void)64 static void imsic_ipi_dying_cpu(void) { }
imsic_ipi_domain_init(void)65 static int __init imsic_ipi_domain_init(void) { return 0; }
66 #endif
67 
68 /*
69  * To handle an interrupt, we read the TOPEI CSR and write zero in one
70  * instruction. If TOPEI CSR is non-zero then we translate TOPEI.ID to
71  * Linux interrupt number and let Linux IRQ subsystem handle it.
72  */
imsic_handle_irq(struct irq_desc * desc)73 static void imsic_handle_irq(struct irq_desc *desc)
74 {
75 	struct irq_chip *chip = irq_desc_get_chip(desc);
76 	int err, cpu = smp_processor_id();
77 	struct imsic_vector *vec;
78 	unsigned long local_id;
79 
80 	chained_irq_enter(chip, desc);
81 
82 	while ((local_id = csr_swap(CSR_TOPEI, 0))) {
83 		local_id >>= TOPEI_ID_SHIFT;
84 
85 		if (local_id == IMSIC_IPI_ID) {
86 			if (IS_ENABLED(CONFIG_SMP))
87 				ipi_mux_process();
88 			continue;
89 		}
90 
91 		if (unlikely(!imsic->base_domain))
92 			continue;
93 
94 		vec = imsic_vector_from_local_id(cpu, local_id);
95 		if (!vec) {
96 			pr_warn_ratelimited("vector not found for local ID 0x%lx\n", local_id);
97 			continue;
98 		}
99 
100 		err = generic_handle_domain_irq(imsic->base_domain, vec->hwirq);
101 		if (unlikely(err))
102 			pr_warn_ratelimited("hwirq 0x%x mapping not found\n", vec->hwirq);
103 	}
104 
105 	chained_irq_exit(chip, desc);
106 }
107 
imsic_starting_cpu(unsigned int cpu)108 static int imsic_starting_cpu(unsigned int cpu)
109 {
110 	/* Mark per-CPU IMSIC state as online */
111 	imsic_state_online();
112 
113 	/* Enable per-CPU parent interrupt */
114 	enable_percpu_irq(imsic_parent_irq, irq_get_trigger_type(imsic_parent_irq));
115 
116 	/* Setup IPIs */
117 	imsic_ipi_starting_cpu();
118 
119 	/*
120 	 * Interrupts identities might have been enabled/disabled while
121 	 * this CPU was not running so sync-up local enable/disable state.
122 	 */
123 	imsic_local_sync_all();
124 
125 	/* Enable local interrupt delivery */
126 	imsic_local_delivery(true);
127 
128 	return 0;
129 }
130 
imsic_dying_cpu(unsigned int cpu)131 static int imsic_dying_cpu(unsigned int cpu)
132 {
133 	/* Cleanup IPIs */
134 	imsic_ipi_dying_cpu();
135 
136 	/* Mark per-CPU IMSIC state as offline */
137 	imsic_state_offline();
138 
139 	return 0;
140 }
141 
imsic_early_probe(struct fwnode_handle * fwnode)142 static int __init imsic_early_probe(struct fwnode_handle *fwnode)
143 {
144 	struct irq_domain *domain;
145 	int rc;
146 
147 	/* Find parent domain and register chained handler */
148 	domain = irq_find_matching_fwnode(riscv_get_intc_hwnode(), DOMAIN_BUS_ANY);
149 	if (!domain) {
150 		pr_err("%pfwP: Failed to find INTC domain\n", fwnode);
151 		return -ENOENT;
152 	}
153 	imsic_parent_irq = irq_create_mapping(domain, RV_IRQ_EXT);
154 	if (!imsic_parent_irq) {
155 		pr_err("%pfwP: Failed to create INTC mapping\n", fwnode);
156 		return -ENOENT;
157 	}
158 
159 	/* Initialize IPI domain */
160 	rc = imsic_ipi_domain_init();
161 	if (rc) {
162 		pr_err("%pfwP: Failed to initialize IPI domain\n", fwnode);
163 		return rc;
164 	}
165 
166 	/* Setup chained handler to the parent domain interrupt */
167 	irq_set_chained_handler(imsic_parent_irq, imsic_handle_irq);
168 
169 	/*
170 	 * Setup cpuhp state (must be done after setting imsic_parent_irq)
171 	 *
172 	 * Don't disable per-CPU IMSIC file when CPU goes offline
173 	 * because this affects IPI and the masking/unmasking of
174 	 * virtual IPIs is done via generic IPI-Mux
175 	 */
176 	cpuhp_setup_state(CPUHP_AP_IRQ_RISCV_IMSIC_STARTING, "irqchip/riscv/imsic:starting",
177 			  imsic_starting_cpu, imsic_dying_cpu);
178 
179 	return 0;
180 }
181 
imsic_early_dt_init(struct device_node * node,struct device_node * parent)182 static int __init imsic_early_dt_init(struct device_node *node, struct device_node *parent)
183 {
184 	struct fwnode_handle *fwnode = &node->fwnode;
185 	int rc;
186 
187 	/* Setup IMSIC state */
188 	rc = imsic_setup_state(fwnode, NULL);
189 	if (rc) {
190 		pr_err("%pfwP: failed to setup state (error %d)\n", fwnode, rc);
191 		return rc;
192 	}
193 
194 	/* Do early setup of IPIs */
195 	rc = imsic_early_probe(fwnode);
196 	if (rc)
197 		return rc;
198 
199 	/* Ensure that OF platform device gets probed */
200 	of_node_clear_flag(node, OF_POPULATED);
201 	return 0;
202 }
203 
204 IRQCHIP_DECLARE(riscv_imsic, "riscv,imsics", imsic_early_dt_init);
205 
206 #ifdef CONFIG_ACPI
207 
208 static struct fwnode_handle *imsic_acpi_fwnode;
209 
imsic_acpi_get_fwnode(struct device * dev)210 struct fwnode_handle *imsic_acpi_get_fwnode(struct device *dev)
211 {
212 	return imsic_acpi_fwnode;
213 }
214 
imsic_early_acpi_init(union acpi_subtable_headers * header,const unsigned long end)215 static int __init imsic_early_acpi_init(union acpi_subtable_headers *header,
216 					const unsigned long end)
217 {
218 	struct acpi_madt_imsic *imsic = (struct acpi_madt_imsic *)header;
219 	int rc;
220 
221 	imsic_acpi_fwnode = irq_domain_alloc_named_fwnode("imsic");
222 	if (!imsic_acpi_fwnode) {
223 		pr_err("unable to allocate IMSIC FW node\n");
224 		return -ENOMEM;
225 	}
226 
227 	/* Setup IMSIC state */
228 	rc = imsic_setup_state(imsic_acpi_fwnode, imsic);
229 	if (rc) {
230 		pr_err("%pfwP: failed to setup state (error %d)\n", imsic_acpi_fwnode, rc);
231 		return rc;
232 	}
233 
234 	/* Do early setup of IMSIC state and IPIs */
235 	rc = imsic_early_probe(imsic_acpi_fwnode);
236 	if (rc) {
237 		irq_domain_free_fwnode(imsic_acpi_fwnode);
238 		imsic_acpi_fwnode = NULL;
239 		return rc;
240 	}
241 
242 	rc = imsic_platform_acpi_probe(imsic_acpi_fwnode);
243 
244 #ifdef CONFIG_PCI
245 	if (!rc)
246 		pci_msi_register_fwnode_provider(&imsic_acpi_get_fwnode);
247 #endif
248 
249 	if (rc)
250 		pr_err("%pfwP: failed to register IMSIC for MSI functionality (error %d)\n",
251 		       imsic_acpi_fwnode, rc);
252 
253 	/*
254 	 * Even if imsic_platform_acpi_probe() fails, the IPI part of IMSIC can
255 	 * continue to work. So, no need to return failure. This is similar to
256 	 * DT where IPI works but MSI probe fails for some reason.
257 	 */
258 	return 0;
259 }
260 
261 IRQCHIP_ACPI_DECLARE(riscv_imsic, ACPI_MADT_TYPE_IMSIC, NULL,
262 		     1, imsic_early_acpi_init);
263 #endif
264