30  * DOC: Interrupt Handling
32  * Interrupts generated within GPU hardware raise interrupt requests that are
34  * type of the interrupt and dispatching matching handlers. If handling an
35  * interrupt requires calling kernel functions that may sleep processing is
41  * For GPU interrupt sources that may be driven by another driver, IRQ domain
118  * amdgpu_irq_disable_all - disable *all* interrupts
130 	spin_lock_irqsave(&adev->irq.lock, irqflags);  in amdgpu_irq_disable_all()
132 		if (!adev->irq.client[i].sources)  in amdgpu_irq_disable_all()
136 			struct amdgpu_irq_src *src = adev->irq.client[i].sources[j];  in amdgpu_irq_disable_all()  local
138 			if (!src || !src->funcs->set || !src->num_types)  in amdgpu_irq_disable_all()
141 			for (k = 0; k < src->num_types; ++k) {  in amdgpu_irq_disable_all()
142 				r = src->funcs->set(adev, src, k,  in amdgpu_irq_disable_all()
145 					DRM_ERROR("error disabling interrupt (%d)\n",  in amdgpu_irq_disable_all()
150 	spin_unlock_irqrestore(&adev->irq.lock, irqflags);  in amdgpu_irq_disable_all()
154  * amdgpu_irq_handler - IRQ handler
170 	ret = amdgpu_ih_process(adev, &adev->irq.ih);  in amdgpu_irq_handler()
172 		pm_runtime_mark_last_busy(dev->dev);  in amdgpu_irq_handler()
180  * amdgpu_irq_handle_ih1 - kick of processing for IH1
191 	amdgpu_ih_process(adev, &adev->irq.ih1);  in amdgpu_irq_handle_ih1()
195  * amdgpu_irq_handle_ih2 - kick of processing for IH2
206 	amdgpu_ih_process(adev, &adev->irq.ih2);  in amdgpu_irq_handle_ih2()
210  * amdgpu_irq_handle_ih_soft - kick of processing for ih_soft
221 	amdgpu_ih_process(adev, &adev->irq.ih_soft);  in amdgpu_irq_handle_ih_soft()
225  * amdgpu_msi_ok - check whether MSI functionality is enabled
249 	pci_read_config_word(adev->pdev, adev->pdev->msix_cap + PCI_MSIX_FLAGS, &ctrl);  in amdgpu_restore_msix()
255 	pci_write_config_word(adev->pdev, adev->pdev->msix_cap + PCI_MSIX_FLAGS, ctrl);  in amdgpu_restore_msix()
257 	pci_write_config_word(adev->pdev, adev->pdev->msix_cap + PCI_MSIX_FLAGS, ctrl);  in amdgpu_restore_msix()
261  * amdgpu_irq_init - initialize interrupt handling
266  * functionality, initializes vblank, hotplug and reset interrupt handling.
276 	spin_lock_init(&adev->irq.lock);  in amdgpu_irq_init()
279 	adev->irq.msi_enabled = false;  in amdgpu_irq_init()
287 	r = pci_alloc_irq_vectors(adev->pdev, 1, 1, flags);  in amdgpu_irq_init()
289 		dev_err(adev->dev, "Failed to alloc msi vectors\n");  in amdgpu_irq_init()
294 		adev->irq.msi_enabled = true;  in amdgpu_irq_init()
295 		dev_dbg(adev->dev, "using MSI/MSI-X.\n");  in amdgpu_irq_init()
298 	INIT_WORK(&adev->irq.ih1_work, amdgpu_irq_handle_ih1);  in amdgpu_irq_init()
299 	INIT_WORK(&adev->irq.ih2_work, amdgpu_irq_handle_ih2);  in amdgpu_irq_init()
300 	INIT_WORK(&adev->irq.ih_soft_work, amdgpu_irq_handle_ih_soft);  in amdgpu_irq_init()
302 	/* Use vector 0 for MSI-X. */  in amdgpu_irq_init()
303 	r = pci_irq_vector(adev->pdev, 0);  in amdgpu_irq_init()
309 	r = request_irq(irq, amdgpu_irq_handler, IRQF_SHARED, adev_to_drm(adev)->driver->name,  in amdgpu_irq_init()
314 	adev->irq.installed = true;  in amdgpu_irq_init()
315 	adev->irq.irq = irq;  in amdgpu_irq_init()
316 	adev_to_drm(adev)->max_vblank_count = 0x00ffffff;  in amdgpu_irq_init()
322 	if (adev->irq.msi_enabled)  in amdgpu_irq_init()
323 		pci_free_irq_vectors(adev->pdev);  in amdgpu_irq_init()
325 	adev->irq.msi_enabled = false;  in amdgpu_irq_init()
331 	if (adev->irq.installed) {  in amdgpu_irq_fini_hw()
332 		free_irq(adev->irq.irq, adev_to_drm(adev));  in amdgpu_irq_fini_hw()
333 		adev->irq.installed = false;  in amdgpu_irq_fini_hw()
334 		if (adev->irq.msi_enabled)  in amdgpu_irq_fini_hw()
335 			pci_free_irq_vectors(adev->pdev);  in amdgpu_irq_fini_hw()
338 	amdgpu_ih_ring_fini(adev, &adev->irq.ih_soft);  in amdgpu_irq_fini_hw()
339 	amdgpu_ih_ring_fini(adev, &adev->irq.ih);  in amdgpu_irq_fini_hw()
340 	amdgpu_ih_ring_fini(adev, &adev->irq.ih1);  in amdgpu_irq_fini_hw()
341 	amdgpu_ih_ring_fini(adev, &adev->irq.ih2);  in amdgpu_irq_fini_hw()
345  * amdgpu_irq_fini_sw - shut down interrupt handling
350  * functionality, shuts down vblank, hotplug and reset interrupt handling,
358 		if (!adev->irq.client[i].sources)  in amdgpu_irq_fini_sw()
362 			struct amdgpu_irq_src *src = adev->irq.client[i].sources[j];  in amdgpu_irq_fini_sw()  local
364 			if (!src)  in amdgpu_irq_fini_sw()
367 			kfree(src->enabled_types);  in amdgpu_irq_fini_sw()
368 			src->enabled_types = NULL;  in amdgpu_irq_fini_sw()
370 		kfree(adev->irq.client[i].sources);  in amdgpu_irq_fini_sw()
371 		adev->irq.client[i].sources = NULL;  in amdgpu_irq_fini_sw()
376  * amdgpu_irq_add_id - register IRQ source
393 		return -EINVAL;  in amdgpu_irq_add_id()
396 		return -EINVAL;  in amdgpu_irq_add_id()
398 	if (!source->funcs)  in amdgpu_irq_add_id()
399 		return -EINVAL;  in amdgpu_irq_add_id()
401 	if (!adev->irq.client[client_id].sources) {  in amdgpu_irq_add_id()
402 		adev->irq.client[client_id].sources =  in amdgpu_irq_add_id()
406 		if (!adev->irq.client[client_id].sources)  in amdgpu_irq_add_id()
407 			return -ENOMEM;  in amdgpu_irq_add_id()
410 	if (adev->irq.client[client_id].sources[src_id] != NULL)  in amdgpu_irq_add_id()
411 		return -EINVAL;  in amdgpu_irq_add_id()
413 	if (source->num_types && !source->enabled_types) {  in amdgpu_irq_add_id()
416 		types = kcalloc(source->num_types, sizeof(atomic_t),  in amdgpu_irq_add_id()
419 			return -ENOMEM;  in amdgpu_irq_add_id()
421 		source->enabled_types = types;  in amdgpu_irq_add_id()
424 	adev->irq.client[client_id].sources[src_id] = source;  in amdgpu_irq_add_id()
429  * amdgpu_irq_dispatch - dispatch IRQ to IP blocks
432  * @ih: interrupt ring instance
439 	u32 ring_index = ih->rptr >> 2;  in amdgpu_irq_dispatch()
442 	struct amdgpu_irq_src *src;  in amdgpu_irq_dispatch()  local
447 	entry.iv_entry = (const uint32_t *)&ih->ring[ring_index];  in amdgpu_irq_dispatch()
458 	trace_amdgpu_iv(ih - &adev->irq.ih, &entry);  in amdgpu_irq_dispatch()
471 		   adev->irq.virq[src_id]) {  in amdgpu_irq_dispatch()
472 		generic_handle_domain_irq(adev->irq.domain, src_id);  in amdgpu_irq_dispatch()
474 	} else if (!adev->irq.client[client_id].sources) {  in amdgpu_irq_dispatch()
475 		DRM_DEBUG("Unregistered interrupt client_id: %d src_id: %d\n",  in amdgpu_irq_dispatch()
478 	} else if ((src = adev->irq.client[client_id].sources[src_id])) {  in amdgpu_irq_dispatch()
479 		r = src->funcs->process(adev, src, &entry);  in amdgpu_irq_dispatch()
481 			DRM_ERROR("error processing interrupt (%d)\n", r);  in amdgpu_irq_dispatch()
486 		DRM_DEBUG("Unregistered interrupt src_id: %d of client_id:%d\n",  in amdgpu_irq_dispatch()
494 	if (amdgpu_ih_ts_after(ih->processed_timestamp, entry.timestamp))  in amdgpu_irq_dispatch()
495 		ih->processed_timestamp = entry.timestamp;  in amdgpu_irq_dispatch()
499  * amdgpu_irq_delegate - delegate IV to soft IH ring
512 	amdgpu_ih_ring_write(adev, &adev->irq.ih_soft, entry->iv_entry, num_dw);  in amdgpu_irq_delegate()
513 	schedule_work(&adev->irq.ih_soft_work);  in amdgpu_irq_delegate()
517  * amdgpu_irq_update - update hardware interrupt state
520  * @src: interrupt source pointer
521  * @type: type of interrupt
523  * Updates interrupt state for the specific source (all ASICs).
526 			     struct amdgpu_irq_src *src, unsigned int type)  in amdgpu_irq_update()  argument
532 	spin_lock_irqsave(&adev->irq.lock, irqflags);  in amdgpu_irq_update()
537 	if (amdgpu_irq_enabled(adev, src, type))  in amdgpu_irq_update()
542 	r = src->funcs->set(adev, src, type, state);  in amdgpu_irq_update()
543 	spin_unlock_irqrestore(&adev->irq.lock, irqflags);  in amdgpu_irq_update()
548  * amdgpu_irq_gpu_reset_resume_helper - update interrupt states on all sources
563 		if (!adev->irq.client[i].sources)  in amdgpu_irq_gpu_reset_resume_helper()
567 			struct amdgpu_irq_src *src = adev->irq.client[i].sources[j];  in amdgpu_irq_gpu_reset_resume_helper()  local
569 			if (!src || !src->funcs || !src->funcs->set)  in amdgpu_irq_gpu_reset_resume_helper()
571 			for (k = 0; k < src->num_types; k++)  in amdgpu_irq_gpu_reset_resume_helper()
572 				amdgpu_irq_update(adev, src, k);  in amdgpu_irq_gpu_reset_resume_helper()
578  * amdgpu_irq_get - enable interrupt
581  * @src: interrupt source pointer
582  * @type: type of interrupt
584  * Enables specified type of interrupt on the specified source (all ASICs).
589 int amdgpu_irq_get(struct amdgpu_device *adev, struct amdgpu_irq_src *src,  in amdgpu_irq_get()  argument
592 	if (!adev->irq.installed)  in amdgpu_irq_get()
593 		return -ENOENT;  in amdgpu_irq_get()
595 	if (type >= src->num_types)  in amdgpu_irq_get()
596 		return -EINVAL;  in amdgpu_irq_get()
598 	if (!src->enabled_types || !src->funcs->set)  in amdgpu_irq_get()
599 		return -EINVAL;  in amdgpu_irq_get()
601 	if (atomic_inc_return(&src->enabled_types[type]) == 1)  in amdgpu_irq_get()
602 		return amdgpu_irq_update(adev, src, type);  in amdgpu_irq_get()
608  * amdgpu_irq_put - disable interrupt
611  * @src: interrupt source pointer
612  * @type: type of interrupt
614  * Enables specified type of interrupt on the specified source (all ASICs).
619 int amdgpu_irq_put(struct amdgpu_device *adev, struct amdgpu_irq_src *src,  in amdgpu_irq_put()  argument
622 	if (!adev->irq.installed)  in amdgpu_irq_put()
623 		return -ENOENT;  in amdgpu_irq_put()
625 	if (type >= src->num_types)  in amdgpu_irq_put()
626 		return -EINVAL;  in amdgpu_irq_put()
628 	if (!src->enabled_types || !src->funcs->set)  in amdgpu_irq_put()
629 		return -EINVAL;  in amdgpu_irq_put()
631 	if (WARN_ON(!amdgpu_irq_enabled(adev, src, type)))  in amdgpu_irq_put()
632 		return -EINVAL;  in amdgpu_irq_put()
634 	if (atomic_dec_and_test(&src->enabled_types[type]))  in amdgpu_irq_put()
635 		return amdgpu_irq_update(adev, src, type);  in amdgpu_irq_put()
641  * amdgpu_irq_enabled - check whether interrupt is enabled or not
644  * @src: interrupt source pointer
645  * @type: type of interrupt
647  * Checks whether the given type of interrupt is enabled on the given source.
650  * *true* if interrupt is enabled, *false* if interrupt is disabled or on
653 bool amdgpu_irq_enabled(struct amdgpu_device *adev, struct amdgpu_irq_src *src,  in amdgpu_irq_enabled()  argument
656 	if (!adev->irq.installed)  in amdgpu_irq_enabled()
659 	if (type >= src->num_types)  in amdgpu_irq_enabled()
662 	if (!src->enabled_types || !src->funcs->set)  in amdgpu_irq_enabled()
665 	return !!atomic_read(&src->enabled_types[type]);  in amdgpu_irq_enabled()
679 /* amdgpu hardware interrupt chip descriptor */
681 	.name = "amdgpu-ih",
687  * amdgpu_irqdomain_map - create mapping between virtual and hardware IRQ numbers
693  * Current implementation assigns simple interrupt handler to the given virtual
703 		return -EPERM;  in amdgpu_irqdomain_map()
716  * amdgpu_irq_add_domain - create a linear IRQ domain
720  * Creates an IRQ domain for GPU interrupt sources
728 	adev->irq.domain = irq_domain_add_linear(NULL, AMDGPU_MAX_IRQ_SRC_ID,  in amdgpu_irq_add_domain()
730 	if (!adev->irq.domain) {  in amdgpu_irq_add_domain()
732 		return -ENODEV;  in amdgpu_irq_add_domain()
739  * amdgpu_irq_remove_domain - remove the IRQ domain
743  * Removes the IRQ domain for GPU interrupt sources
748 	if (adev->irq.domain) {  in amdgpu_irq_remove_domain()
749 		irq_domain_remove(adev->irq.domain);  in amdgpu_irq_remove_domain()
750 		adev->irq.domain = NULL;  in amdgpu_irq_remove_domain()
755  * amdgpu_irq_create_mapping - create mapping between domain Linux IRQs
760  * Creates mapping between a domain IRQ (GPU IH src id) and a Linux IRQ
761  * Use this for components that generate a GPU interrupt, but are driven
769 	adev->irq.virq[src_id] = irq_create_mapping(adev->irq.domain, src_id);  in amdgpu_irq_create_mapping()
771 	return adev->irq.virq[src_id];  in amdgpu_irq_create_mapping()