1 // SPDX-License-Identifier: GPL-2.0 or MIT
2 /* Copyright 2018 Marty E. Plummer <hanetzer@startmail.com> */
3 /* Copyright 2019 Linaro, Ltd, Rob Herring <robh@kernel.org> */
4 /* Copyright 2023 Collabora ltd. */
5
6 #include <linux/clk.h>
7 #include <linux/mm.h>
8 #include <linux/platform_device.h>
9 #include <linux/pm_domain.h>
10 #include <linux/pm_runtime.h>
11 #include <linux/regulator/consumer.h>
12 #include <linux/reset.h>
13
14 #include <drm/drm_drv.h>
15 #include <drm/drm_managed.h>
16
17 #include "panthor_devfreq.h"
18 #include "panthor_device.h"
19 #include "panthor_fw.h"
20 #include "panthor_gpu.h"
21 #include "panthor_mmu.h"
22 #include "panthor_regs.h"
23 #include "panthor_sched.h"
24
panthor_clk_init(struct panthor_device * ptdev)25 static int panthor_clk_init(struct panthor_device *ptdev)
26 {
27 ptdev->clks.core = devm_clk_get(ptdev->base.dev, NULL);
28 if (IS_ERR(ptdev->clks.core))
29 return dev_err_probe(ptdev->base.dev,
30 PTR_ERR(ptdev->clks.core),
31 "get 'core' clock failed");
32
33 ptdev->clks.stacks = devm_clk_get_optional(ptdev->base.dev, "stacks");
34 if (IS_ERR(ptdev->clks.stacks))
35 return dev_err_probe(ptdev->base.dev,
36 PTR_ERR(ptdev->clks.stacks),
37 "get 'stacks' clock failed");
38
39 ptdev->clks.coregroup = devm_clk_get_optional(ptdev->base.dev, "coregroup");
40 if (IS_ERR(ptdev->clks.coregroup))
41 return dev_err_probe(ptdev->base.dev,
42 PTR_ERR(ptdev->clks.coregroup),
43 "get 'coregroup' clock failed");
44
45 drm_info(&ptdev->base, "clock rate = %lu\n", clk_get_rate(ptdev->clks.core));
46 return 0;
47 }
48
panthor_device_unplug(struct panthor_device * ptdev)49 void panthor_device_unplug(struct panthor_device *ptdev)
50 {
51 /* This function can be called from two different path: the reset work
52 * and the platform device remove callback. drm_dev_unplug() doesn't
53 * deal with concurrent callers, so we have to protect drm_dev_unplug()
54 * calls with our own lock, and bail out if the device is already
55 * unplugged.
56 */
57 mutex_lock(&ptdev->unplug.lock);
58 if (drm_dev_is_unplugged(&ptdev->base)) {
59 /* Someone beat us, release the lock and wait for the unplug
60 * operation to be reported as done.
61 **/
62 mutex_unlock(&ptdev->unplug.lock);
63 wait_for_completion(&ptdev->unplug.done);
64 return;
65 }
66
67 /* Call drm_dev_unplug() so any access to HW blocks happening after
68 * that point get rejected.
69 */
70 drm_dev_unplug(&ptdev->base);
71
72 /* We do the rest of the unplug with the unplug lock released,
73 * future callers will wait on ptdev->unplug.done anyway.
74 */
75 mutex_unlock(&ptdev->unplug.lock);
76
77 drm_WARN_ON(&ptdev->base, pm_runtime_get_sync(ptdev->base.dev) < 0);
78
79 /* Now, try to cleanly shutdown the GPU before the device resources
80 * get reclaimed.
81 */
82 panthor_sched_unplug(ptdev);
83 panthor_fw_unplug(ptdev);
84 panthor_mmu_unplug(ptdev);
85 panthor_gpu_unplug(ptdev);
86
87 pm_runtime_dont_use_autosuspend(ptdev->base.dev);
88 pm_runtime_put_sync_suspend(ptdev->base.dev);
89
90 /* If PM is disabled, we need to call the suspend handler manually. */
91 if (!IS_ENABLED(CONFIG_PM))
92 panthor_device_suspend(ptdev->base.dev);
93
94 /* Report the unplug operation as done to unblock concurrent
95 * panthor_device_unplug() callers.
96 */
97 complete_all(&ptdev->unplug.done);
98 }
99
panthor_device_reset_cleanup(struct drm_device * ddev,void * data)100 static void panthor_device_reset_cleanup(struct drm_device *ddev, void *data)
101 {
102 struct panthor_device *ptdev = container_of(ddev, struct panthor_device, base);
103
104 cancel_work_sync(&ptdev->reset.work);
105 destroy_workqueue(ptdev->reset.wq);
106 }
107
panthor_device_reset_work(struct work_struct * work)108 static void panthor_device_reset_work(struct work_struct *work)
109 {
110 struct panthor_device *ptdev = container_of(work, struct panthor_device, reset.work);
111 int ret = 0, cookie;
112
113 if (atomic_read(&ptdev->pm.state) != PANTHOR_DEVICE_PM_STATE_ACTIVE) {
114 /*
115 * No need for a reset as the device has been (or will be)
116 * powered down
117 */
118 atomic_set(&ptdev->reset.pending, 0);
119 return;
120 }
121
122 if (!drm_dev_enter(&ptdev->base, &cookie))
123 return;
124
125 panthor_sched_pre_reset(ptdev);
126 panthor_fw_pre_reset(ptdev, true);
127 panthor_mmu_pre_reset(ptdev);
128 panthor_gpu_soft_reset(ptdev);
129 panthor_gpu_l2_power_on(ptdev);
130 panthor_mmu_post_reset(ptdev);
131 ret = panthor_fw_post_reset(ptdev);
132 atomic_set(&ptdev->reset.pending, 0);
133 panthor_sched_post_reset(ptdev, ret != 0);
134 drm_dev_exit(cookie);
135
136 if (ret) {
137 panthor_device_unplug(ptdev);
138 drm_err(&ptdev->base, "Failed to boot MCU after reset, making device unusable.");
139 }
140 }
141
panthor_device_is_initialized(struct panthor_device * ptdev)142 static bool panthor_device_is_initialized(struct panthor_device *ptdev)
143 {
144 return !!ptdev->scheduler;
145 }
146
panthor_device_free_page(struct drm_device * ddev,void * data)147 static void panthor_device_free_page(struct drm_device *ddev, void *data)
148 {
149 __free_page(data);
150 }
151
panthor_device_init(struct panthor_device * ptdev)152 int panthor_device_init(struct panthor_device *ptdev)
153 {
154 u32 *dummy_page_virt;
155 struct resource *res;
156 struct page *p;
157 int ret;
158
159 ptdev->coherent = device_get_dma_attr(ptdev->base.dev) == DEV_DMA_COHERENT;
160
161 init_completion(&ptdev->unplug.done);
162 ret = drmm_mutex_init(&ptdev->base, &ptdev->unplug.lock);
163 if (ret)
164 return ret;
165
166 ret = drmm_mutex_init(&ptdev->base, &ptdev->pm.mmio_lock);
167 if (ret)
168 return ret;
169
170 atomic_set(&ptdev->pm.state, PANTHOR_DEVICE_PM_STATE_SUSPENDED);
171 p = alloc_page(GFP_KERNEL | __GFP_ZERO);
172 if (!p)
173 return -ENOMEM;
174
175 ptdev->pm.dummy_latest_flush = p;
176 dummy_page_virt = page_address(p);
177 ret = drmm_add_action_or_reset(&ptdev->base, panthor_device_free_page,
178 ptdev->pm.dummy_latest_flush);
179 if (ret)
180 return ret;
181
182 /*
183 * Set the dummy page holding the latest flush to 1. This will cause the
184 * flush to avoided as we know it isn't necessary if the submission
185 * happens while the dummy page is mapped. Zero cannot be used because
186 * that means 'always flush'.
187 */
188 *dummy_page_virt = 1;
189
190 INIT_WORK(&ptdev->reset.work, panthor_device_reset_work);
191 ptdev->reset.wq = alloc_ordered_workqueue("panthor-reset-wq", 0);
192 if (!ptdev->reset.wq)
193 return -ENOMEM;
194
195 ret = drmm_add_action_or_reset(&ptdev->base, panthor_device_reset_cleanup, NULL);
196 if (ret)
197 return ret;
198
199 ret = panthor_clk_init(ptdev);
200 if (ret)
201 return ret;
202
203 ret = panthor_devfreq_init(ptdev);
204 if (ret)
205 return ret;
206
207 ptdev->iomem = devm_platform_get_and_ioremap_resource(to_platform_device(ptdev->base.dev),
208 0, &res);
209 if (IS_ERR(ptdev->iomem))
210 return PTR_ERR(ptdev->iomem);
211
212 ptdev->phys_addr = res->start;
213
214 ret = devm_pm_runtime_enable(ptdev->base.dev);
215 if (ret)
216 return ret;
217
218 ret = pm_runtime_resume_and_get(ptdev->base.dev);
219 if (ret)
220 return ret;
221
222 /* If PM is disabled, we need to call panthor_device_resume() manually. */
223 if (!IS_ENABLED(CONFIG_PM)) {
224 ret = panthor_device_resume(ptdev->base.dev);
225 if (ret)
226 return ret;
227 }
228
229 ret = panthor_gpu_init(ptdev);
230 if (ret)
231 goto err_rpm_put;
232
233 ret = panthor_mmu_init(ptdev);
234 if (ret)
235 goto err_unplug_gpu;
236
237 ret = panthor_fw_init(ptdev);
238 if (ret)
239 goto err_unplug_mmu;
240
241 ret = panthor_sched_init(ptdev);
242 if (ret)
243 goto err_unplug_fw;
244
245 /* ~3 frames */
246 pm_runtime_set_autosuspend_delay(ptdev->base.dev, 50);
247 pm_runtime_use_autosuspend(ptdev->base.dev);
248
249 ret = drm_dev_register(&ptdev->base, 0);
250 if (ret)
251 goto err_disable_autosuspend;
252
253 pm_runtime_put_autosuspend(ptdev->base.dev);
254 return 0;
255
256 err_disable_autosuspend:
257 pm_runtime_dont_use_autosuspend(ptdev->base.dev);
258 panthor_sched_unplug(ptdev);
259
260 err_unplug_fw:
261 panthor_fw_unplug(ptdev);
262
263 err_unplug_mmu:
264 panthor_mmu_unplug(ptdev);
265
266 err_unplug_gpu:
267 panthor_gpu_unplug(ptdev);
268
269 err_rpm_put:
270 pm_runtime_put_sync_suspend(ptdev->base.dev);
271 return ret;
272 }
273
274 #define PANTHOR_EXCEPTION(id) \
275 [DRM_PANTHOR_EXCEPTION_ ## id] = { \
276 .name = #id, \
277 }
278
279 struct panthor_exception_info {
280 const char *name;
281 };
282
283 static const struct panthor_exception_info panthor_exception_infos[] = {
284 PANTHOR_EXCEPTION(OK),
285 PANTHOR_EXCEPTION(TERMINATED),
286 PANTHOR_EXCEPTION(KABOOM),
287 PANTHOR_EXCEPTION(EUREKA),
288 PANTHOR_EXCEPTION(ACTIVE),
289 PANTHOR_EXCEPTION(CS_RES_TERM),
290 PANTHOR_EXCEPTION(CS_CONFIG_FAULT),
291 PANTHOR_EXCEPTION(CS_UNRECOVERABLE),
292 PANTHOR_EXCEPTION(CS_ENDPOINT_FAULT),
293 PANTHOR_EXCEPTION(CS_BUS_FAULT),
294 PANTHOR_EXCEPTION(CS_INSTR_INVALID),
295 PANTHOR_EXCEPTION(CS_CALL_STACK_OVERFLOW),
296 PANTHOR_EXCEPTION(CS_INHERIT_FAULT),
297 PANTHOR_EXCEPTION(INSTR_INVALID_PC),
298 PANTHOR_EXCEPTION(INSTR_INVALID_ENC),
299 PANTHOR_EXCEPTION(INSTR_BARRIER_FAULT),
300 PANTHOR_EXCEPTION(DATA_INVALID_FAULT),
301 PANTHOR_EXCEPTION(TILE_RANGE_FAULT),
302 PANTHOR_EXCEPTION(ADDR_RANGE_FAULT),
303 PANTHOR_EXCEPTION(IMPRECISE_FAULT),
304 PANTHOR_EXCEPTION(OOM),
305 PANTHOR_EXCEPTION(CSF_FW_INTERNAL_ERROR),
306 PANTHOR_EXCEPTION(CSF_RES_EVICTION_TIMEOUT),
307 PANTHOR_EXCEPTION(GPU_BUS_FAULT),
308 PANTHOR_EXCEPTION(GPU_SHAREABILITY_FAULT),
309 PANTHOR_EXCEPTION(SYS_SHAREABILITY_FAULT),
310 PANTHOR_EXCEPTION(GPU_CACHEABILITY_FAULT),
311 PANTHOR_EXCEPTION(TRANSLATION_FAULT_0),
312 PANTHOR_EXCEPTION(TRANSLATION_FAULT_1),
313 PANTHOR_EXCEPTION(TRANSLATION_FAULT_2),
314 PANTHOR_EXCEPTION(TRANSLATION_FAULT_3),
315 PANTHOR_EXCEPTION(TRANSLATION_FAULT_4),
316 PANTHOR_EXCEPTION(PERM_FAULT_0),
317 PANTHOR_EXCEPTION(PERM_FAULT_1),
318 PANTHOR_EXCEPTION(PERM_FAULT_2),
319 PANTHOR_EXCEPTION(PERM_FAULT_3),
320 PANTHOR_EXCEPTION(ACCESS_FLAG_1),
321 PANTHOR_EXCEPTION(ACCESS_FLAG_2),
322 PANTHOR_EXCEPTION(ACCESS_FLAG_3),
323 PANTHOR_EXCEPTION(ADDR_SIZE_FAULT_IN),
324 PANTHOR_EXCEPTION(ADDR_SIZE_FAULT_OUT0),
325 PANTHOR_EXCEPTION(ADDR_SIZE_FAULT_OUT1),
326 PANTHOR_EXCEPTION(ADDR_SIZE_FAULT_OUT2),
327 PANTHOR_EXCEPTION(ADDR_SIZE_FAULT_OUT3),
328 PANTHOR_EXCEPTION(MEM_ATTR_FAULT_0),
329 PANTHOR_EXCEPTION(MEM_ATTR_FAULT_1),
330 PANTHOR_EXCEPTION(MEM_ATTR_FAULT_2),
331 PANTHOR_EXCEPTION(MEM_ATTR_FAULT_3),
332 };
333
panthor_exception_name(struct panthor_device * ptdev,u32 exception_code)334 const char *panthor_exception_name(struct panthor_device *ptdev, u32 exception_code)
335 {
336 if (exception_code >= ARRAY_SIZE(panthor_exception_infos) ||
337 !panthor_exception_infos[exception_code].name)
338 return "Unknown exception type";
339
340 return panthor_exception_infos[exception_code].name;
341 }
342
panthor_mmio_vm_fault(struct vm_fault * vmf)343 static vm_fault_t panthor_mmio_vm_fault(struct vm_fault *vmf)
344 {
345 struct vm_area_struct *vma = vmf->vma;
346 struct panthor_device *ptdev = vma->vm_private_data;
347 u64 offset = (u64)vma->vm_pgoff << PAGE_SHIFT;
348 unsigned long pfn;
349 pgprot_t pgprot;
350 vm_fault_t ret;
351 bool active;
352 int cookie;
353
354 if (!drm_dev_enter(&ptdev->base, &cookie))
355 return VM_FAULT_SIGBUS;
356
357 mutex_lock(&ptdev->pm.mmio_lock);
358 active = atomic_read(&ptdev->pm.state) == PANTHOR_DEVICE_PM_STATE_ACTIVE;
359
360 switch (offset) {
361 case DRM_PANTHOR_USER_FLUSH_ID_MMIO_OFFSET:
362 if (active)
363 pfn = __phys_to_pfn(ptdev->phys_addr + CSF_GPU_LATEST_FLUSH_ID);
364 else
365 pfn = page_to_pfn(ptdev->pm.dummy_latest_flush);
366 break;
367
368 default:
369 ret = VM_FAULT_SIGBUS;
370 goto out_unlock;
371 }
372
373 pgprot = vma->vm_page_prot;
374 if (active)
375 pgprot = pgprot_noncached(pgprot);
376
377 ret = vmf_insert_pfn_prot(vma, vmf->address, pfn, pgprot);
378
379 out_unlock:
380 mutex_unlock(&ptdev->pm.mmio_lock);
381 drm_dev_exit(cookie);
382 return ret;
383 }
384
385 static const struct vm_operations_struct panthor_mmio_vm_ops = {
386 .fault = panthor_mmio_vm_fault,
387 };
388
panthor_device_mmap_io(struct panthor_device * ptdev,struct vm_area_struct * vma)389 int panthor_device_mmap_io(struct panthor_device *ptdev, struct vm_area_struct *vma)
390 {
391 u64 offset = (u64)vma->vm_pgoff << PAGE_SHIFT;
392
393 if ((vma->vm_flags & VM_SHARED) == 0)
394 return -EINVAL;
395
396 switch (offset) {
397 case DRM_PANTHOR_USER_FLUSH_ID_MMIO_OFFSET:
398 if (vma->vm_end - vma->vm_start != PAGE_SIZE ||
399 (vma->vm_flags & (VM_WRITE | VM_EXEC)))
400 return -EINVAL;
401 vm_flags_clear(vma, VM_MAYWRITE);
402
403 break;
404
405 default:
406 return -EINVAL;
407 }
408
409 /* Defer actual mapping to the fault handler. */
410 vma->vm_private_data = ptdev;
411 vma->vm_ops = &panthor_mmio_vm_ops;
412 vm_flags_set(vma,
413 VM_IO | VM_DONTCOPY | VM_DONTEXPAND |
414 VM_NORESERVE | VM_DONTDUMP | VM_PFNMAP);
415 return 0;
416 }
417
panthor_device_resume(struct device * dev)418 int panthor_device_resume(struct device *dev)
419 {
420 struct panthor_device *ptdev = dev_get_drvdata(dev);
421 int ret, cookie;
422
423 if (atomic_read(&ptdev->pm.state) != PANTHOR_DEVICE_PM_STATE_SUSPENDED)
424 return -EINVAL;
425
426 atomic_set(&ptdev->pm.state, PANTHOR_DEVICE_PM_STATE_RESUMING);
427
428 ret = clk_prepare_enable(ptdev->clks.core);
429 if (ret)
430 goto err_set_suspended;
431
432 ret = clk_prepare_enable(ptdev->clks.stacks);
433 if (ret)
434 goto err_disable_core_clk;
435
436 ret = clk_prepare_enable(ptdev->clks.coregroup);
437 if (ret)
438 goto err_disable_stacks_clk;
439
440 ret = panthor_devfreq_resume(ptdev);
441 if (ret)
442 goto err_disable_coregroup_clk;
443
444 if (panthor_device_is_initialized(ptdev) &&
445 drm_dev_enter(&ptdev->base, &cookie)) {
446 panthor_gpu_resume(ptdev);
447 panthor_mmu_resume(ptdev);
448 ret = drm_WARN_ON(&ptdev->base, panthor_fw_resume(ptdev));
449 if (!ret) {
450 panthor_sched_resume(ptdev);
451 } else {
452 panthor_mmu_suspend(ptdev);
453 panthor_gpu_suspend(ptdev);
454 }
455
456 drm_dev_exit(cookie);
457
458 if (ret)
459 goto err_suspend_devfreq;
460 }
461
462 if (atomic_read(&ptdev->reset.pending))
463 queue_work(ptdev->reset.wq, &ptdev->reset.work);
464
465 /* Clear all IOMEM mappings pointing to this device after we've
466 * resumed. This way the fake mappings pointing to the dummy pages
467 * are removed and the real iomem mapping will be restored on next
468 * access.
469 */
470 mutex_lock(&ptdev->pm.mmio_lock);
471 unmap_mapping_range(ptdev->base.anon_inode->i_mapping,
472 DRM_PANTHOR_USER_MMIO_OFFSET, 0, 1);
473 atomic_set(&ptdev->pm.state, PANTHOR_DEVICE_PM_STATE_ACTIVE);
474 mutex_unlock(&ptdev->pm.mmio_lock);
475 return 0;
476
477 err_suspend_devfreq:
478 panthor_devfreq_suspend(ptdev);
479
480 err_disable_coregroup_clk:
481 clk_disable_unprepare(ptdev->clks.coregroup);
482
483 err_disable_stacks_clk:
484 clk_disable_unprepare(ptdev->clks.stacks);
485
486 err_disable_core_clk:
487 clk_disable_unprepare(ptdev->clks.core);
488
489 err_set_suspended:
490 atomic_set(&ptdev->pm.state, PANTHOR_DEVICE_PM_STATE_SUSPENDED);
491 return ret;
492 }
493
panthor_device_suspend(struct device * dev)494 int panthor_device_suspend(struct device *dev)
495 {
496 struct panthor_device *ptdev = dev_get_drvdata(dev);
497 int ret, cookie;
498
499 if (atomic_read(&ptdev->pm.state) != PANTHOR_DEVICE_PM_STATE_ACTIVE)
500 return -EINVAL;
501
502 /* Clear all IOMEM mappings pointing to this device before we
503 * shutdown the power-domain and clocks. Failing to do that results
504 * in external aborts when the process accesses the iomem region.
505 * We change the state and call unmap_mapping_range() with the
506 * mmio_lock held to make sure the vm_fault handler won't set up
507 * invalid mappings.
508 */
509 mutex_lock(&ptdev->pm.mmio_lock);
510 atomic_set(&ptdev->pm.state, PANTHOR_DEVICE_PM_STATE_SUSPENDING);
511 unmap_mapping_range(ptdev->base.anon_inode->i_mapping,
512 DRM_PANTHOR_USER_MMIO_OFFSET, 0, 1);
513 mutex_unlock(&ptdev->pm.mmio_lock);
514
515 if (panthor_device_is_initialized(ptdev) &&
516 drm_dev_enter(&ptdev->base, &cookie)) {
517 cancel_work_sync(&ptdev->reset.work);
518
519 /* We prepare everything as if we were resetting the GPU.
520 * The end of the reset will happen in the resume path though.
521 */
522 panthor_sched_suspend(ptdev);
523 panthor_fw_suspend(ptdev);
524 panthor_mmu_suspend(ptdev);
525 panthor_gpu_suspend(ptdev);
526 drm_dev_exit(cookie);
527 }
528
529 ret = panthor_devfreq_suspend(ptdev);
530 if (ret) {
531 if (panthor_device_is_initialized(ptdev) &&
532 drm_dev_enter(&ptdev->base, &cookie)) {
533 panthor_gpu_resume(ptdev);
534 panthor_mmu_resume(ptdev);
535 drm_WARN_ON(&ptdev->base, panthor_fw_resume(ptdev));
536 panthor_sched_resume(ptdev);
537 drm_dev_exit(cookie);
538 }
539
540 goto err_set_active;
541 }
542
543 clk_disable_unprepare(ptdev->clks.coregroup);
544 clk_disable_unprepare(ptdev->clks.stacks);
545 clk_disable_unprepare(ptdev->clks.core);
546 atomic_set(&ptdev->pm.state, PANTHOR_DEVICE_PM_STATE_SUSPENDED);
547 return 0;
548
549 err_set_active:
550 /* If something failed and we have to revert back to an
551 * active state, we also need to clear the MMIO userspace
552 * mappings, so any dumb pages that were mapped while we
553 * were trying to suspend gets invalidated.
554 */
555 mutex_lock(&ptdev->pm.mmio_lock);
556 atomic_set(&ptdev->pm.state, PANTHOR_DEVICE_PM_STATE_ACTIVE);
557 unmap_mapping_range(ptdev->base.anon_inode->i_mapping,
558 DRM_PANTHOR_USER_MMIO_OFFSET, 0, 1);
559 mutex_unlock(&ptdev->pm.mmio_lock);
560 return ret;
561 }
562