1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3 *
4 * Copyright (C) 2013 Freescale Semiconductor, Inc.
5 * Author: Varun Sethi <varun.sethi@freescale.com>
6 */
7
8 #define pr_fmt(fmt) "fsl-pamu-domain: %s: " fmt, __func__
9
10 #include "fsl_pamu_domain.h"
11
12 #include <linux/platform_device.h>
13 #include <sysdev/fsl_pci.h>
14
15 /*
16 * Global spinlock that needs to be held while
17 * configuring PAMU.
18 */
19 static DEFINE_SPINLOCK(iommu_lock);
20
21 static struct kmem_cache *fsl_pamu_domain_cache;
22 static struct kmem_cache *iommu_devinfo_cache;
23 static DEFINE_SPINLOCK(device_domain_lock);
24
25 struct iommu_device pamu_iommu; /* IOMMU core code handle */
26
to_fsl_dma_domain(struct iommu_domain * dom)27 static struct fsl_dma_domain *to_fsl_dma_domain(struct iommu_domain *dom)
28 {
29 return container_of(dom, struct fsl_dma_domain, iommu_domain);
30 }
31
iommu_init_mempool(void)32 static int __init iommu_init_mempool(void)
33 {
34 fsl_pamu_domain_cache = kmem_cache_create("fsl_pamu_domain",
35 sizeof(struct fsl_dma_domain),
36 0,
37 SLAB_HWCACHE_ALIGN,
38 NULL);
39 if (!fsl_pamu_domain_cache) {
40 pr_debug("Couldn't create fsl iommu_domain cache\n");
41 return -ENOMEM;
42 }
43
44 iommu_devinfo_cache = kmem_cache_create("iommu_devinfo",
45 sizeof(struct device_domain_info),
46 0,
47 SLAB_HWCACHE_ALIGN,
48 NULL);
49 if (!iommu_devinfo_cache) {
50 pr_debug("Couldn't create devinfo cache\n");
51 kmem_cache_destroy(fsl_pamu_domain_cache);
52 return -ENOMEM;
53 }
54
55 return 0;
56 }
57
update_liodn_stash(int liodn,struct fsl_dma_domain * dma_domain,u32 val)58 static int update_liodn_stash(int liodn, struct fsl_dma_domain *dma_domain,
59 u32 val)
60 {
61 int ret = 0;
62 unsigned long flags;
63
64 spin_lock_irqsave(&iommu_lock, flags);
65 ret = pamu_update_paace_stash(liodn, val);
66 if (ret) {
67 pr_debug("Failed to update SPAACE for liodn %d\n ", liodn);
68 spin_unlock_irqrestore(&iommu_lock, flags);
69 return ret;
70 }
71
72 spin_unlock_irqrestore(&iommu_lock, flags);
73
74 return ret;
75 }
76
77 /* Set the geometry parameters for a LIODN */
pamu_set_liodn(struct fsl_dma_domain * dma_domain,struct device * dev,int liodn)78 static int pamu_set_liodn(struct fsl_dma_domain *dma_domain, struct device *dev,
79 int liodn)
80 {
81 u32 omi_index = ~(u32)0;
82 unsigned long flags;
83 int ret;
84
85 /*
86 * Configure the omi_index at the geometry setup time.
87 * This is a static value which depends on the type of
88 * device and would not change thereafter.
89 */
90 get_ome_index(&omi_index, dev);
91
92 spin_lock_irqsave(&iommu_lock, flags);
93 ret = pamu_disable_liodn(liodn);
94 if (ret)
95 goto out_unlock;
96 ret = pamu_config_ppaace(liodn, omi_index, dma_domain->stash_id, 0);
97 if (ret)
98 goto out_unlock;
99 ret = pamu_config_ppaace(liodn, ~(u32)0, dma_domain->stash_id,
100 PAACE_AP_PERMS_QUERY | PAACE_AP_PERMS_UPDATE);
101 out_unlock:
102 spin_unlock_irqrestore(&iommu_lock, flags);
103 if (ret) {
104 pr_debug("PAACE configuration failed for liodn %d\n",
105 liodn);
106 }
107 return ret;
108 }
109
remove_device_ref(struct device_domain_info * info)110 static void remove_device_ref(struct device_domain_info *info)
111 {
112 unsigned long flags;
113
114 list_del(&info->link);
115 spin_lock_irqsave(&iommu_lock, flags);
116 pamu_disable_liodn(info->liodn);
117 spin_unlock_irqrestore(&iommu_lock, flags);
118 spin_lock_irqsave(&device_domain_lock, flags);
119 dev_iommu_priv_set(info->dev, NULL);
120 kmem_cache_free(iommu_devinfo_cache, info);
121 spin_unlock_irqrestore(&device_domain_lock, flags);
122 }
123
detach_device(struct device * dev,struct fsl_dma_domain * dma_domain)124 static void detach_device(struct device *dev, struct fsl_dma_domain *dma_domain)
125 {
126 struct device_domain_info *info, *tmp;
127 unsigned long flags;
128
129 spin_lock_irqsave(&dma_domain->domain_lock, flags);
130 /* Remove the device from the domain device list */
131 list_for_each_entry_safe(info, tmp, &dma_domain->devices, link) {
132 if (!dev || (info->dev == dev))
133 remove_device_ref(info);
134 }
135 spin_unlock_irqrestore(&dma_domain->domain_lock, flags);
136 }
137
attach_device(struct fsl_dma_domain * dma_domain,int liodn,struct device * dev)138 static void attach_device(struct fsl_dma_domain *dma_domain, int liodn, struct device *dev)
139 {
140 struct device_domain_info *info, *old_domain_info;
141 unsigned long flags;
142
143 spin_lock_irqsave(&device_domain_lock, flags);
144 /*
145 * Check here if the device is already attached to domain or not.
146 * If the device is already attached to a domain detach it.
147 */
148 old_domain_info = dev_iommu_priv_get(dev);
149 if (old_domain_info && old_domain_info->domain != dma_domain) {
150 spin_unlock_irqrestore(&device_domain_lock, flags);
151 detach_device(dev, old_domain_info->domain);
152 spin_lock_irqsave(&device_domain_lock, flags);
153 }
154
155 info = kmem_cache_zalloc(iommu_devinfo_cache, GFP_ATOMIC);
156
157 info->dev = dev;
158 info->liodn = liodn;
159 info->domain = dma_domain;
160
161 list_add(&info->link, &dma_domain->devices);
162 /*
163 * In case of devices with multiple LIODNs just store
164 * the info for the first LIODN as all
165 * LIODNs share the same domain
166 */
167 if (!dev_iommu_priv_get(dev))
168 dev_iommu_priv_set(dev, info);
169 spin_unlock_irqrestore(&device_domain_lock, flags);
170 }
171
fsl_pamu_iova_to_phys(struct iommu_domain * domain,dma_addr_t iova)172 static phys_addr_t fsl_pamu_iova_to_phys(struct iommu_domain *domain,
173 dma_addr_t iova)
174 {
175 if (iova < domain->geometry.aperture_start ||
176 iova > domain->geometry.aperture_end)
177 return 0;
178 return iova;
179 }
180
fsl_pamu_capable(struct device * dev,enum iommu_cap cap)181 static bool fsl_pamu_capable(struct device *dev, enum iommu_cap cap)
182 {
183 return cap == IOMMU_CAP_CACHE_COHERENCY;
184 }
185
fsl_pamu_domain_free(struct iommu_domain * domain)186 static void fsl_pamu_domain_free(struct iommu_domain *domain)
187 {
188 struct fsl_dma_domain *dma_domain = to_fsl_dma_domain(domain);
189
190 /* remove all the devices from the device list */
191 detach_device(NULL, dma_domain);
192 kmem_cache_free(fsl_pamu_domain_cache, dma_domain);
193 }
194
fsl_pamu_domain_alloc(unsigned type)195 static struct iommu_domain *fsl_pamu_domain_alloc(unsigned type)
196 {
197 struct fsl_dma_domain *dma_domain;
198
199 /*
200 * FIXME: This isn't creating an unmanaged domain since the
201 * default_domain_ops do not have any map/unmap function it doesn't meet
202 * the requirements for __IOMMU_DOMAIN_PAGING. The only purpose seems to
203 * allow drivers/soc/fsl/qbman/qman_portal.c to do
204 * fsl_pamu_configure_l1_stash()
205 */
206 if (type != IOMMU_DOMAIN_UNMANAGED)
207 return NULL;
208
209 dma_domain = kmem_cache_zalloc(fsl_pamu_domain_cache, GFP_KERNEL);
210 if (!dma_domain)
211 return NULL;
212
213 dma_domain->stash_id = ~(u32)0;
214 INIT_LIST_HEAD(&dma_domain->devices);
215 spin_lock_init(&dma_domain->domain_lock);
216
217 /* default geometry 64 GB i.e. maximum system address */
218 dma_domain->iommu_domain. geometry.aperture_start = 0;
219 dma_domain->iommu_domain.geometry.aperture_end = (1ULL << 36) - 1;
220 dma_domain->iommu_domain.geometry.force_aperture = true;
221
222 return &dma_domain->iommu_domain;
223 }
224
225 /* Update stash destination for all LIODNs associated with the domain */
update_domain_stash(struct fsl_dma_domain * dma_domain,u32 val)226 static int update_domain_stash(struct fsl_dma_domain *dma_domain, u32 val)
227 {
228 struct device_domain_info *info;
229 int ret = 0;
230
231 list_for_each_entry(info, &dma_domain->devices, link) {
232 ret = update_liodn_stash(info->liodn, dma_domain, val);
233 if (ret)
234 break;
235 }
236
237 return ret;
238 }
239
fsl_pamu_attach_device(struct iommu_domain * domain,struct device * dev)240 static int fsl_pamu_attach_device(struct iommu_domain *domain,
241 struct device *dev)
242 {
243 struct fsl_dma_domain *dma_domain = to_fsl_dma_domain(domain);
244 unsigned long flags;
245 int len, ret = 0, i;
246 const u32 *liodn;
247 struct pci_dev *pdev = NULL;
248 struct pci_controller *pci_ctl;
249
250 /*
251 * Use LIODN of the PCI controller while attaching a
252 * PCI device.
253 */
254 if (dev_is_pci(dev)) {
255 pdev = to_pci_dev(dev);
256 pci_ctl = pci_bus_to_host(pdev->bus);
257 /*
258 * make dev point to pci controller device
259 * so we can get the LIODN programmed by
260 * u-boot.
261 */
262 dev = pci_ctl->parent;
263 }
264
265 liodn = of_get_property(dev->of_node, "fsl,liodn", &len);
266 if (!liodn) {
267 pr_debug("missing fsl,liodn property at %pOF\n", dev->of_node);
268 return -ENODEV;
269 }
270
271 spin_lock_irqsave(&dma_domain->domain_lock, flags);
272 for (i = 0; i < len / sizeof(u32); i++) {
273 /* Ensure that LIODN value is valid */
274 if (liodn[i] >= PAACE_NUMBER_ENTRIES) {
275 pr_debug("Invalid liodn %d, attach device failed for %pOF\n",
276 liodn[i], dev->of_node);
277 ret = -ENODEV;
278 break;
279 }
280
281 attach_device(dma_domain, liodn[i], dev);
282 ret = pamu_set_liodn(dma_domain, dev, liodn[i]);
283 if (ret)
284 break;
285 ret = pamu_enable_liodn(liodn[i]);
286 if (ret)
287 break;
288 }
289 spin_unlock_irqrestore(&dma_domain->domain_lock, flags);
290 return ret;
291 }
292
293 /*
294 * FIXME: fsl/pamu is completely broken in terms of how it works with the iommu
295 * API. Immediately after probe the HW is left in an IDENTITY translation and
296 * the driver provides a non-working UNMANAGED domain that it can switch over
297 * to. However it cannot switch back to an IDENTITY translation, instead it
298 * switches to what looks like BLOCKING.
299 */
fsl_pamu_platform_attach(struct iommu_domain * platform_domain,struct device * dev)300 static int fsl_pamu_platform_attach(struct iommu_domain *platform_domain,
301 struct device *dev)
302 {
303 struct iommu_domain *domain = iommu_get_domain_for_dev(dev);
304 struct fsl_dma_domain *dma_domain;
305 const u32 *prop;
306 int len;
307 struct pci_dev *pdev = NULL;
308 struct pci_controller *pci_ctl;
309
310 /*
311 * Hack to keep things working as they always have, only leaving an
312 * UNMANAGED domain makes it BLOCKING.
313 */
314 if (domain == platform_domain || !domain ||
315 domain->type != IOMMU_DOMAIN_UNMANAGED)
316 return 0;
317
318 dma_domain = to_fsl_dma_domain(domain);
319
320 /*
321 * Use LIODN of the PCI controller while detaching a
322 * PCI device.
323 */
324 if (dev_is_pci(dev)) {
325 pdev = to_pci_dev(dev);
326 pci_ctl = pci_bus_to_host(pdev->bus);
327 /*
328 * make dev point to pci controller device
329 * so we can get the LIODN programmed by
330 * u-boot.
331 */
332 dev = pci_ctl->parent;
333 }
334
335 prop = of_get_property(dev->of_node, "fsl,liodn", &len);
336 if (prop)
337 detach_device(dev, dma_domain);
338 else
339 pr_debug("missing fsl,liodn property at %pOF\n", dev->of_node);
340 return 0;
341 }
342
343 static struct iommu_domain_ops fsl_pamu_platform_ops = {
344 .attach_dev = fsl_pamu_platform_attach,
345 };
346
347 static struct iommu_domain fsl_pamu_platform_domain = {
348 .type = IOMMU_DOMAIN_PLATFORM,
349 .ops = &fsl_pamu_platform_ops,
350 };
351
352 /* Set the domain stash attribute */
fsl_pamu_configure_l1_stash(struct iommu_domain * domain,u32 cpu)353 int fsl_pamu_configure_l1_stash(struct iommu_domain *domain, u32 cpu)
354 {
355 struct fsl_dma_domain *dma_domain = to_fsl_dma_domain(domain);
356 unsigned long flags;
357 int ret;
358
359 spin_lock_irqsave(&dma_domain->domain_lock, flags);
360 dma_domain->stash_id = get_stash_id(PAMU_ATTR_CACHE_L1, cpu);
361 if (dma_domain->stash_id == ~(u32)0) {
362 pr_debug("Invalid stash attributes\n");
363 spin_unlock_irqrestore(&dma_domain->domain_lock, flags);
364 return -EINVAL;
365 }
366 ret = update_domain_stash(dma_domain, dma_domain->stash_id);
367 spin_unlock_irqrestore(&dma_domain->domain_lock, flags);
368
369 return ret;
370 }
371
check_pci_ctl_endpt_part(struct pci_controller * pci_ctl)372 static bool check_pci_ctl_endpt_part(struct pci_controller *pci_ctl)
373 {
374 u32 version;
375
376 /* Check the PCI controller version number by readding BRR1 register */
377 version = in_be32(pci_ctl->cfg_addr + (PCI_FSL_BRR1 >> 2));
378 version &= PCI_FSL_BRR1_VER;
379 /* If PCI controller version is >= 0x204 we can partition endpoints */
380 return version >= 0x204;
381 }
382
fsl_pamu_device_group(struct device * dev)383 static struct iommu_group *fsl_pamu_device_group(struct device *dev)
384 {
385 struct iommu_group *group;
386 struct pci_dev *pdev;
387
388 /*
389 * For platform devices we allocate a separate group for each of the
390 * devices.
391 */
392 if (!dev_is_pci(dev))
393 return generic_device_group(dev);
394
395 /*
396 * We can partition PCIe devices so assign device group to the device
397 */
398 pdev = to_pci_dev(dev);
399 if (check_pci_ctl_endpt_part(pci_bus_to_host(pdev->bus)))
400 return pci_device_group(&pdev->dev);
401
402 /*
403 * All devices connected to the controller will share the same device
404 * group.
405 *
406 * Due to ordering between fsl_pamu_init() and fsl_pci_init() it is
407 * guaranteed that the pci_ctl->parent platform_device will have the
408 * iommu driver bound and will already have a group set. So we just
409 * re-use this group as the group for every device in the hose.
410 */
411 group = iommu_group_get(pci_bus_to_host(pdev->bus)->parent);
412 if (WARN_ON(!group))
413 return ERR_PTR(-EINVAL);
414 return group;
415 }
416
fsl_pamu_probe_device(struct device * dev)417 static struct iommu_device *fsl_pamu_probe_device(struct device *dev)
418 {
419 /*
420 * uboot must fill the fsl,liodn for platform devices to be supported by
421 * the iommu.
422 */
423 if (!dev_is_pci(dev) &&
424 !of_property_present(dev->of_node, "fsl,liodn"))
425 return ERR_PTR(-ENODEV);
426
427 return &pamu_iommu;
428 }
429
430 static const struct iommu_ops fsl_pamu_ops = {
431 .default_domain = &fsl_pamu_platform_domain,
432 .capable = fsl_pamu_capable,
433 .domain_alloc = fsl_pamu_domain_alloc,
434 .probe_device = fsl_pamu_probe_device,
435 .device_group = fsl_pamu_device_group,
436 .default_domain_ops = &(const struct iommu_domain_ops) {
437 .attach_dev = fsl_pamu_attach_device,
438 .iova_to_phys = fsl_pamu_iova_to_phys,
439 .free = fsl_pamu_domain_free,
440 }
441 };
442
pamu_domain_init(void)443 int __init pamu_domain_init(void)
444 {
445 int ret = 0;
446
447 ret = iommu_init_mempool();
448 if (ret)
449 return ret;
450
451 ret = iommu_device_sysfs_add(&pamu_iommu, NULL, NULL, "iommu0");
452 if (ret)
453 return ret;
454
455 ret = iommu_device_register(&pamu_iommu, &fsl_pamu_ops, NULL);
456 if (ret) {
457 iommu_device_sysfs_remove(&pamu_iommu);
458 pr_err("Can't register iommu device\n");
459 }
460
461 return ret;
462 }
463