1 // SPDX-License-Identifier: GPL-2.0
8 #include <linux/dma-mapping.h>
27 	if (test_bit(PANFROST_COMP_BIT_GPU, pfdev->is_suspended))  in panfrost_gpu_irq_handler()
39 		dev_warn(pfdev->dev, "GPU Fault 0x%08x (%s) at 0x%016llx\n",  in panfrost_gpu_irq_handler()
44 			dev_warn(pfdev->dev, "There were multiple GPU faults - some have not been reported\n");  in panfrost_gpu_irq_handler()
68 	clear_bit(PANFROST_COMP_BIT_GPU, pfdev->is_suspended);  in panfrost_gpu_soft_reset()
71 	ret = readl_relaxed_poll_timeout(pfdev->iomem + GPU_INT_RAWSTAT,  in panfrost_gpu_soft_reset()
75 		dev_err(pfdev->dev, "gpu soft reset timed out, attempting hard reset\n");  in panfrost_gpu_soft_reset()
78 		ret = readl_relaxed_poll_timeout(pfdev->iomem + GPU_INT_RAWSTAT, val,  in panfrost_gpu_soft_reset()
81 			dev_err(pfdev->dev, "gpu hard reset timed out\n");  in panfrost_gpu_soft_reset()
95 	 * All in-flight jobs should have released their cycle  in panfrost_gpu_soft_reset()
98 	if (drm_WARN_ON(pfdev->ddev, atomic_read(&pfdev->cycle_counter.use_count) != 0))  in panfrost_gpu_soft_reset()
99 		atomic_set(&pfdev->cycle_counter.use_count, 0);  in panfrost_gpu_soft_reset()
107 	 * The Amlogic integrated Mali-T820, Mali-G31 & Mali-G52 needs  in panfrost_gpu_amlogic_quirk()
157 	    pfdev->features.revision >= 0x2000)  in panfrost_gpu_init_quirks()
160 		 pfdev->features.coherency_features == COHERENCY_ACE)  in panfrost_gpu_init_quirks()
171 	if (pfdev->comp->vendor_quirk)  in panfrost_gpu_init_quirks()
172 		pfdev->comp->vendor_quirk(pfdev);  in panfrost_gpu_init_quirks()
179 	u32 id;  member
184 		u32 revision;  member
192 	.id = _id,						\
200 	.revision = (_rev) << 12 | (_p) << 4 | (_s),		\
233 	/* MediaTek MT8192 has a Mali-G57 with a different GPU ID from the
236 	 * standard Mali-G57 for now.
251 	pfdev->features.l2_features = gpu_read(pfdev, GPU_L2_FEATURES);  in panfrost_gpu_init_features()
252 	pfdev->features.core_features = gpu_read(pfdev, GPU_CORE_FEATURES);  in panfrost_gpu_init_features()
253 	pfdev->features.tiler_features = gpu_read(pfdev, GPU_TILER_FEATURES);  in panfrost_gpu_init_features()
254 	pfdev->features.mem_features = gpu_read(pfdev, GPU_MEM_FEATURES);  in panfrost_gpu_init_features()
255 	pfdev->features.mmu_features = gpu_read(pfdev, GPU_MMU_FEATURES);  in panfrost_gpu_init_features()
256 	pfdev->features.thread_features = gpu_read(pfdev, GPU_THREAD_FEATURES);  in panfrost_gpu_init_features()
257 	pfdev->features.max_threads = gpu_read(pfdev, GPU_THREAD_MAX_THREADS);  in panfrost_gpu_init_features()
258 	pfdev->features.thread_max_workgroup_sz = gpu_read(pfdev, GPU_THREAD_MAX_WORKGROUP_SIZE);  in panfrost_gpu_init_features()
259 	pfdev->features.thread_max_barrier_sz = gpu_read(pfdev, GPU_THREAD_MAX_BARRIER_SIZE);  in panfrost_gpu_init_features()
260 	pfdev->features.coherency_features = gpu_read(pfdev, GPU_COHERENCY_FEATURES);  in panfrost_gpu_init_features()
261 	pfdev->features.afbc_features = gpu_read(pfdev, GPU_AFBC_FEATURES);  in panfrost_gpu_init_features()
263 		pfdev->features.texture_features[i] = gpu_read(pfdev, GPU_TEXTURE_FEATURES(i));  in panfrost_gpu_init_features()
265 	pfdev->features.as_present = gpu_read(pfdev, GPU_AS_PRESENT);  in panfrost_gpu_init_features()
267 	pfdev->features.js_present = gpu_read(pfdev, GPU_JS_PRESENT);  in panfrost_gpu_init_features()
268 	num_js = hweight32(pfdev->features.js_present);  in panfrost_gpu_init_features()
270 		pfdev->features.js_features[i] = gpu_read(pfdev, GPU_JS_FEATURES(i));  in panfrost_gpu_init_features()
272 	pfdev->features.shader_present = gpu_read(pfdev, GPU_SHADER_PRESENT_LO);  in panfrost_gpu_init_features()
273 	pfdev->features.shader_present |= (u64)gpu_read(pfdev, GPU_SHADER_PRESENT_HI) << 32;  in panfrost_gpu_init_features()
275 	pfdev->features.tiler_present = gpu_read(pfdev, GPU_TILER_PRESENT_LO);  in panfrost_gpu_init_features()
276 	pfdev->features.tiler_present |= (u64)gpu_read(pfdev, GPU_TILER_PRESENT_HI) << 32;  in panfrost_gpu_init_features()
278 	pfdev->features.l2_present = gpu_read(pfdev, GPU_L2_PRESENT_LO);  in panfrost_gpu_init_features()
279 	pfdev->features.l2_present |= (u64)gpu_read(pfdev, GPU_L2_PRESENT_HI) << 32;  in panfrost_gpu_init_features()
280 	pfdev->features.nr_core_groups = hweight64(pfdev->features.l2_present);  in panfrost_gpu_init_features()
282 	pfdev->features.stack_present = gpu_read(pfdev, GPU_STACK_PRESENT_LO);  in panfrost_gpu_init_features()
283 	pfdev->features.stack_present |= (u64)gpu_read(pfdev, GPU_STACK_PRESENT_HI) << 32;  in panfrost_gpu_init_features()
285 	pfdev->features.thread_tls_alloc = gpu_read(pfdev, GPU_THREAD_TLS_ALLOC);  in panfrost_gpu_init_features()
288 	pfdev->features.revision = gpu_id & 0xffff;  in panfrost_gpu_init_features()
289 	pfdev->features.id = gpu_id >> 16;  in panfrost_gpu_init_features()
291 	/* The T60x has an oddball ID value. Fix it up to the standard Midgard  in panfrost_gpu_init_features()
294 	if (pfdev->features.id == 0x6956)  in panfrost_gpu_init_features()
295 		pfdev->features.id = 0x0600;  in panfrost_gpu_init_features()
297 	major = (pfdev->features.revision >> 12) & 0xf;  in panfrost_gpu_init_features()
298 	minor = (pfdev->features.revision >> 4) & 0xff;  in panfrost_gpu_init_features()
299 	status = pfdev->features.revision & 0xf;  in panfrost_gpu_init_features()
300 	rev = pfdev->features.revision;  in panfrost_gpu_init_features()
302 	gpu_id = pfdev->features.id;  in panfrost_gpu_init_features()
304 	for (model = gpu_models; model->name; model++) {  in panfrost_gpu_init_features()
305 		int best = -1;  in panfrost_gpu_init_features()
307 		if (!panfrost_model_eq(pfdev, model->id))  in panfrost_gpu_init_features()
310 		name = model->name;  in panfrost_gpu_init_features()
311 		hw_feat = model->features;  in panfrost_gpu_init_features()
312 		hw_issues |= model->issues;  in panfrost_gpu_init_features()
314 			if (model->revs[i].revision == rev) {  in panfrost_gpu_init_features()
317 			} else if (model->revs[i].revision == (rev & ~0xf))  in panfrost_gpu_init_features()
322 			hw_issues |= model->revs[best].issues;  in panfrost_gpu_init_features()
327 	bitmap_from_u64(pfdev->features.hw_features, hw_feat);  in panfrost_gpu_init_features()
328 	bitmap_from_u64(pfdev->features.hw_issues, hw_issues);  in panfrost_gpu_init_features()
330 	dev_info(pfdev->dev, "mali-%s id 0x%x major 0x%x minor 0x%x status 0x%x",  in panfrost_gpu_init_features()
332 	dev_info(pfdev->dev, "features: %64pb, issues: %64pb",  in panfrost_gpu_init_features()
333 		 pfdev->features.hw_features,  in panfrost_gpu_init_features()
334 		 pfdev->features.hw_issues);  in panfrost_gpu_init_features()
336 …dev_info(pfdev->dev, "Features: L2:0x%08x Shader:0x%08x Tiler:0x%08x Mem:0x%0x MMU:0x%08x AS:0x%x …  in panfrost_gpu_init_features()
337 		 pfdev->features.l2_features,  in panfrost_gpu_init_features()
338 		 pfdev->features.core_features,  in panfrost_gpu_init_features()
339 		 pfdev->features.tiler_features,  in panfrost_gpu_init_features()
340 		 pfdev->features.mem_features,  in panfrost_gpu_init_features()
341 		 pfdev->features.mmu_features,  in panfrost_gpu_init_features()
342 		 pfdev->features.as_present,  in panfrost_gpu_init_features()
343 		 pfdev->features.js_present);  in panfrost_gpu_init_features()
345 	dev_info(pfdev->dev, "shader_present=0x%0llx l2_present=0x%0llx",  in panfrost_gpu_init_features()
346 		 pfdev->features.shader_present, pfdev->features.l2_present);  in panfrost_gpu_init_features()
351 	if (atomic_inc_not_zero(&pfdev->cycle_counter.use_count))  in panfrost_cycle_counter_get()
354 	spin_lock(&pfdev->cycle_counter.lock);  in panfrost_cycle_counter_get()
355 	if (atomic_inc_return(&pfdev->cycle_counter.use_count) == 1)  in panfrost_cycle_counter_get()
357 	spin_unlock(&pfdev->cycle_counter.lock);  in panfrost_cycle_counter_get()
362 	if (atomic_add_unless(&pfdev->cycle_counter.use_count, -1, 1))  in panfrost_cycle_counter_put()
365 	spin_lock(&pfdev->cycle_counter.lock);  in panfrost_cycle_counter_put()
366 	if (atomic_dec_return(&pfdev->cycle_counter.use_count) == 0)  in panfrost_cycle_counter_put()
368 	spin_unlock(&pfdev->cycle_counter.lock);  in panfrost_cycle_counter_put()
387 	if (pfdev->features.l2_present == 1)  in panfrost_get_core_mask()
392 	 * ~(l2_present - 1) unsets all bits in l2_present except  in panfrost_get_core_mask()
393 	 * the bottom bit. (l2_present - 2) has all the bits in  in panfrost_get_core_mask()
397 	core_mask = ~(pfdev->features.l2_present - 1) &  in panfrost_get_core_mask()
398 		     (pfdev->features.l2_present - 2);  in panfrost_get_core_mask()
399 	dev_info_once(pfdev->dev, "using only 1st core group (%lu cores from %lu)\n",  in panfrost_get_core_mask()
401 		      hweight64(pfdev->features.shader_present));  in panfrost_get_core_mask()
415 	gpu_write(pfdev, L2_PWRON_LO, pfdev->features.l2_present & core_mask);  in panfrost_gpu_power_on()
416 	ret = readl_relaxed_poll_timeout(pfdev->iomem + L2_READY_LO,  in panfrost_gpu_power_on()
417 		val, val == (pfdev->features.l2_present & core_mask),  in panfrost_gpu_power_on()
420 		dev_err(pfdev->dev, "error powering up gpu L2");  in panfrost_gpu_power_on()
423 		  pfdev->features.shader_present & core_mask);  in panfrost_gpu_power_on()
424 	ret = readl_relaxed_poll_timeout(pfdev->iomem + SHADER_READY_LO,  in panfrost_gpu_power_on()
425 		val, val == (pfdev->features.shader_present & core_mask),  in panfrost_gpu_power_on()
428 		dev_err(pfdev->dev, "error powering up gpu shader");  in panfrost_gpu_power_on()
430 	gpu_write(pfdev, TILER_PWRON_LO, pfdev->features.tiler_present);  in panfrost_gpu_power_on()
431 	ret = readl_relaxed_poll_timeout(pfdev->iomem + TILER_READY_LO,  in panfrost_gpu_power_on()
432 		val, val == pfdev->features.tiler_present, 10, 1000);  in panfrost_gpu_power_on()
434 		dev_err(pfdev->dev, "error powering up gpu tiler");  in panfrost_gpu_power_on()
442 	gpu_write(pfdev, SHADER_PWROFF_LO, pfdev->features.shader_present);  in panfrost_gpu_power_off()
443 	ret = readl_relaxed_poll_timeout(pfdev->iomem + SHADER_PWRTRANS_LO,  in panfrost_gpu_power_off()
446 		dev_err(pfdev->dev, "shader power transition timeout");  in panfrost_gpu_power_off()
448 	gpu_write(pfdev, TILER_PWROFF_LO, pfdev->features.tiler_present);  in panfrost_gpu_power_off()
449 	ret = readl_relaxed_poll_timeout(pfdev->iomem + TILER_PWRTRANS_LO,  in panfrost_gpu_power_off()
452 		dev_err(pfdev->dev, "tiler power transition timeout");  in panfrost_gpu_power_off()
454 	gpu_write(pfdev, L2_PWROFF_LO, pfdev->features.l2_present);  in panfrost_gpu_power_off()
455 	ret = readl_poll_timeout(pfdev->iomem + L2_PWRTRANS_LO,  in panfrost_gpu_power_off()
458 		dev_err(pfdev->dev, "l2 power transition timeout");  in panfrost_gpu_power_off()
463 	set_bit(PANFROST_COMP_BIT_GPU, pfdev->is_suspended);  in panfrost_gpu_suspend_irq()
466 	synchronize_irq(pfdev->gpu_irq);  in panfrost_gpu_suspend_irq()
479 	err = dma_set_mask_and_coherent(pfdev->dev,  in panfrost_gpu_init()
480 		DMA_BIT_MASK(FIELD_GET(0xff00, pfdev->features.mmu_features)));  in panfrost_gpu_init()
484 	dma_set_max_seg_size(pfdev->dev, UINT_MAX);  in panfrost_gpu_init()
486 	pfdev->gpu_irq = platform_get_irq_byname(to_platform_device(pfdev->dev), "gpu");  in panfrost_gpu_init()
487 	if (pfdev->gpu_irq < 0)  in panfrost_gpu_init()
488 		return pfdev->gpu_irq;  in panfrost_gpu_init()
490 	err = devm_request_irq(pfdev->dev, pfdev->gpu_irq, panfrost_gpu_irq_handler,  in panfrost_gpu_init()
491 			       IRQF_SHARED, KBUILD_MODNAME "-gpu", pfdev);  in panfrost_gpu_init()
493 		dev_err(pfdev->dev, "failed to request gpu irq");  in panfrost_gpu_init()
513 		if (pm_runtime_get_if_in_use(pfdev->dev)) {  in panfrost_gpu_get_latest_flush_id()
515 			pm_runtime_put(pfdev->dev);  in panfrost_gpu_get_latest_flush_id()