1 // SPDX-License-Identifier: GPL-2.0-or-later
3  * PCI Error Recovery Driver for RPA-compliant PPC64 platform.
17 #include <asm/ppc-pci.h>
18 #include <asm/pci-bridge.h>
55 		return "need reset";  in pci_ers_result_name()
78 	return !edev || (edev->mode & EEH_DEV_REMOVED);  in eeh_dev_removed()
83 	if (!edev->pdev)  in eeh_edev_actionable()
85 	if (edev->pdev->error_state == pci_channel_io_perm_failure)  in eeh_edev_actionable()
89 	if (eeh_pe_passed(edev->pe))  in eeh_edev_actionable()
96  * eeh_pcid_get - Get the PCI device driver
101  * of the PCI device driver to prevent that being unloaded on
106 	if (!pdev || !pdev->dev.driver)  in eeh_pcid_get()
109 	if (!try_module_get(pdev->dev.driver->owner))  in eeh_pcid_get()
112 	return to_pci_driver(pdev->dev.driver);  in eeh_pcid_get()
116  * eeh_pcid_put - Dereference on the PCI device driver
119  * The function is called to do dereference on the PCI device
124 	if (!pdev || !pdev->dev.driver)  in eeh_pcid_put()
127 	module_put(pdev->dev.driver->owner);  in eeh_pcid_put()
131  * eeh_disable_irq - Disable interrupt for the recovering device
136  * device. If the device has enabled MSI or MSI-X interrupt, we needn't
138  * devices encountering EEH errors, which includes MSI or MSI-X.
142 	/* Don't disable MSI and MSI-X interrupts. They are  in eeh_disable_irq()
146 	if (edev->pdev->msi_enabled || edev->pdev->msix_enabled)  in eeh_disable_irq()
149 	if (!irq_has_action(edev->pdev->irq))  in eeh_disable_irq()
152 	edev->mode |= EEH_DEV_IRQ_DISABLED;  in eeh_disable_irq()
153 	disable_irq_nosync(edev->pdev->irq);  in eeh_disable_irq()
157  * eeh_enable_irq - Enable interrupt for the recovering device
165 	if ((edev->mode) & EEH_DEV_IRQ_DISABLED) {  in eeh_enable_irq()
166 		edev->mode &= ~EEH_DEV_IRQ_DISABLED;  in eeh_enable_irq()
187 		if (irqd_irq_disabled(irq_get_irq_data(edev->pdev->irq)))  in eeh_enable_irq()
188 			enable_irq(edev->pdev->irq);  in eeh_enable_irq()
200 	 * We cannot access the config space on some adapters.  in eeh_dev_save_state()
206 	if (edev->pe && (edev->pe->state & EEH_PE_CFG_RESTRICTED))  in eeh_dev_save_state()
224 				edev->pdev->error_state = s;  in eeh_set_channel_state()
237 			if (!eeh_pcid_get(edev->pdev))  in eeh_set_irq_state()
245 			eeh_pcid_put(edev->pdev);  in eeh_set_irq_state()
261 	pdev = edev->pdev;  in eeh_pe_report_edev()
263 		get_device(&pdev->dev);  in eeh_pe_report_edev()
269 	device_lock(&pdev->dev);  in eeh_pe_report_edev()
275 		else if (!driver->err_handler)  in eeh_pe_report_edev()
277 		else if (edev->mode & EEH_DEV_NO_HANDLER)  in eeh_pe_report_edev()
282 				      driver->name,  in eeh_pe_report_edev()
292 			      !eeh_dev_removed(edev), !eeh_pe_passed(edev->pe));  in eeh_pe_report_edev()
294 	device_unlock(&pdev->dev);  in eeh_pe_report_edev()
295 	if (edev->pdev != pdev)  in eeh_pe_report_edev()
297 	put_device(&pdev->dev);  in eeh_pe_report_edev()
317  * eeh_report_error - Report pci error to each device driver
329 	if (!driver->err_handler->error_detected)  in eeh_report_error()
332 	eeh_edev_info(edev, "Invoking %s->error_detected(IO frozen)",  in eeh_report_error()
333 		      driver->name);  in eeh_report_error()
334 	rc = driver->err_handler->error_detected(pdev, pci_channel_io_frozen);  in eeh_report_error()
336 	edev->in_error = true;  in eeh_report_error()
342  * eeh_report_mmio_enabled - Tell drivers that MMIO has been enabled
353 	if (!driver->err_handler->mmio_enabled)  in eeh_report_mmio_enabled()
355 	eeh_edev_info(edev, "Invoking %s->mmio_enabled()", driver->name);  in eeh_report_mmio_enabled()
356 	return driver->err_handler->mmio_enabled(pdev);  in eeh_report_mmio_enabled()
360  * eeh_report_reset - Tell device that slot has been reset
364  * This routine must be called while EEH tries to reset particular
366  * some actions, usually to save data the driver needs so that the
373 	if (!driver->err_handler->slot_reset || !edev->in_error)  in eeh_report_reset()
375 	eeh_edev_info(edev, "Invoking %s->slot_reset()", driver->name);  in eeh_report_reset()
376 	return driver->err_handler->slot_reset(pdev);  in eeh_report_reset()
388 	 * the blocked config space on some adapters. We have  in eeh_dev_restore_state()
392 	if (edev->pe && (edev->pe->state & EEH_PE_CFG_RESTRICTED)) {  in eeh_dev_restore_state()
393 		if (list_is_last(&edev->entry, &edev->pe->edevs))  in eeh_dev_restore_state()
394 			eeh_pe_restore_bars(edev->pe);  in eeh_dev_restore_state()
407  * eeh_report_resume - Tell device to resume normal operations
412  * could resume so that the device driver can do some initialization
419 	if (!driver->err_handler->resume || !edev->in_error)  in eeh_report_resume()
422 	eeh_edev_info(edev, "Invoking %s->resume()", driver->name);  in eeh_report_resume()
423 	driver->err_handler->resume(pdev);  in eeh_report_resume()
425 	pci_uevent_ers(edev->pdev, PCI_ERS_RESULT_RECOVERED);  in eeh_report_resume()
427 	if (eeh_ops->notify_resume)  in eeh_report_resume()
428 		eeh_ops->notify_resume(edev);  in eeh_report_resume()
434  * eeh_report_failure - Tell device driver that device is dead.
439  * dead, and that no further recovery attempts will be made on it.
447 	if (!driver->err_handler->error_detected)  in eeh_report_failure()
450 	eeh_edev_info(edev, "Invoking %s->error_detected(permanent failure)",  in eeh_report_failure()
451 		      driver->name);  in eeh_report_failure()
452 	rc = driver->err_handler->error_detected(pdev,  in eeh_report_failure()
464 	if (!(edev->physfn)) {  in eeh_add_virt_device()
471 		if (driver->err_handler) {  in eeh_add_virt_device()
479 	pci_iov_add_virtfn(edev->physfn, edev->vf_index);  in eeh_add_virt_device()
498 	    (dev->hdr_type == PCI_HEADER_TYPE_BRIDGE))  in eeh_rmv_device()
504 			if (driver->err_handler &&  in eeh_rmv_device()
505 			    driver->err_handler->error_detected &&  in eeh_rmv_device()
506 			    driver->err_handler->slot_reset) {  in eeh_rmv_device()
517 	edev->mode |= EEH_DEV_DISCONNECTED;  in eeh_rmv_device()
519 		rmv_data->removed_dev_count++;  in eeh_rmv_device()
521 	if (edev->physfn) {  in eeh_rmv_device()
523 		pci_iov_remove_virtfn(edev->physfn, edev->vf_index);  in eeh_rmv_device()
524 		edev->pdev = NULL;  in eeh_rmv_device()
527 			list_add(&edev->rmv_entry, &rmv_data->removed_vf_list);  in eeh_rmv_device()
540 		if (!(edev->mode & EEH_DEV_DISCONNECTED))  in eeh_pe_detach_dev()
543 		edev->mode &= ~(EEH_DEV_DISCONNECTED | EEH_DEV_IRQ_DISABLED);  in eeh_pe_detach_dev()
554  * PE reset (for 3 times), we try to clear the frozen state
568 				return -EIO;  in eeh_clear_pe_frozen_state()
580 	if (pe->state & EEH_PE_RECOVERING)  in eeh_pe_reset_and_recover()
589 	/* Issue reset */  in eeh_pe_reset_and_recover()
613  * eeh_reset_device - Perform actual reset of a pci slot
619  * This routine must be called to do reset on the indicated PE.
620  * During the reset, udev might be invoked because those affected
637 	cnt = pe->freeze_count;  in eeh_reset_device()
638 	tstamp = pe->tstamp;  in eeh_reset_device()
647 	if (any_passed || driver_eeh_aware || (pe->type & EEH_PE_VF)) {  in eeh_reset_device()
656 	 * Reset the pci controller. (Asserts RST#; resets config space).  in eeh_reset_device()
658 	 * up if the reset failed for some reason.  in eeh_reset_device()
660 	 * During the reset, it's very dangerous to have uncontrolled PCI  in eeh_reset_device()
671 	eeh_ops->configure_bridge(pe);  in eeh_reset_device()
681 	/* Give the system 5 seconds to finish running the user-space  in eeh_reset_device()
687 	if (!driver_eeh_aware || rmv_data->removed_dev_count) {  in eeh_reset_device()
697 		edev = list_first_entry(&pe->edevs, struct eeh_dev, entry);  in eeh_reset_device()
699 		if (pe->type & EEH_PE_VF) {  in eeh_reset_device()
709 	pe->tstamp = tstamp;  in eeh_reset_device()
710 	pe->freeze_count = cnt;  in eeh_reset_device()
717  * to come back on line, in seconds.
724  * NB: This needs to be recursive to ensure the leaf PEs get removed
733 	list_for_each_entry_safe(child_pe, tmp, &pe->child_list, child)  in eeh_pe_cleanup()
736 	if (pe->state & EEH_PE_KEEP)  in eeh_pe_cleanup()
739 	if (!(pe->state & EEH_PE_INVALID))  in eeh_pe_cleanup()
742 	if (list_empty(&pe->edevs) && list_empty(&pe->child_list)) {  in eeh_pe_cleanup()
743 		list_del(&pe->child);  in eeh_pe_cleanup()
749  * eeh_check_slot_presence - Check if a device is still present in a slot
770 	if (pdev->error_state == pci_channel_io_perm_failure)  in eeh_slot_presence_check()
773 	slot = pdev->slot;  in eeh_slot_presence_check()
774 	if (!slot || !slot->hotplug)  in eeh_slot_presence_check()
777 	ops = slot->hotplug->ops;  in eeh_slot_presence_check()
778 	if (!ops || !ops->get_adapter_status)  in eeh_slot_presence_check()
782 	if (ops->set_attention_status)  in eeh_slot_presence_check()
783 		ops->set_attention_status(slot->hotplug, 1);  in eeh_slot_presence_check()
785 	rc = ops->get_adapter_status(slot->hotplug, &state);  in eeh_slot_presence_check()
800 	if (pdev->error_state == pci_channel_io_perm_failure)  in eeh_clear_slot_attention()
803 	slot = pdev->slot;  in eeh_clear_slot_attention()
804 	if (!slot || !slot->hotplug)  in eeh_clear_slot_attention()
807 	ops = slot->hotplug->ops;  in eeh_clear_slot_attention()
808 	if (!ops || !ops->set_attention_status)  in eeh_clear_slot_attention()
811 	ops->set_attention_status(slot->hotplug, 0);  in eeh_clear_slot_attention()
815  * eeh_handle_normal_event - Handle EEH events on a specific PE
816  * @pe: EEH PE - which should not be used after we return, as it may
822  * While PHB detects address or data parity errors on particular PCI
826  * #PERR or other misc PCI-related errors also can trigger EEH errors.
848 		pr_err("%s: Cannot find PCI bus for PHB#%x-PE#%x\n",  in eeh_handle_normal_event()
849 			__func__, pe->phb->global_number, pe->addr);  in eeh_handle_normal_event()
854 	 * When devices are hot-removed we might get an EEH due to  in eeh_handle_normal_event()
864 			if (eeh_slot_presence_check(edev->pdev))  in eeh_handle_normal_event()
868 		pr_warn("EEH: Frozen PHB#%x-PE#%x is empty!\n",  in eeh_handle_normal_event()
869 			pe->phb->global_number, pe->addr);  in eeh_handle_normal_event()
871 		 * The device is removed, tear down its state, on powernv  in eeh_handle_normal_event()
872 		 * hotplug driver would take care of it but not on pseries,  in eeh_handle_normal_event()
883 	if (pe->type & EEH_PE_PHB) {  in eeh_handle_normal_event()
885 			pe->phb->global_number, eeh_pe_loc_get(pe));  in eeh_handle_normal_event()
887 		struct eeh_pe *phb_pe = eeh_phb_pe_get(pe->phb);  in eeh_handle_normal_event()
889 		pr_err("EEH: Recovering PHB#%x-PE#%x\n",  in eeh_handle_normal_event()
890 		       pe->phb->global_number, pe->addr);  in eeh_handle_normal_event()
900 	if (pe->trace_entries) {  in eeh_handle_normal_event()
901 		void **ptrs = (void **) pe->stack_trace;  in eeh_handle_normal_event()
904 		pr_err("EEH: Frozen PHB#%x-PE#%x detected\n",  in eeh_handle_normal_event()
905 		       pe->phb->global_number, pe->addr);  in eeh_handle_normal_event()
909 		for (i = 0; i < pe->trace_entries; i++)  in eeh_handle_normal_event()
912 		pe->trace_entries = 0;  in eeh_handle_normal_event()
918 			edev->mode &= ~EEH_DEV_NO_HANDLER;  in eeh_handle_normal_event()
921 	pe->freeze_count++;  in eeh_handle_normal_event()
922 	if (pe->freeze_count > eeh_max_freezes) {  in eeh_handle_normal_event()
923 …pr_err("EEH: PHB#%x-PE#%x has failed %d times in the last hour and has been permanently disabled.\…  in eeh_handle_normal_event()
924 		       pe->phb->global_number, pe->addr,  in eeh_handle_normal_event()
925 		       pe->freeze_count);  in eeh_handle_normal_event()
931 	 * a reset sequence, giving each an opportunity to do what it needs  in eeh_handle_normal_event()
932 	 * to accomplish the reset.  Each child gets a report of the  in eeh_handle_normal_event()
933 	 * status ... if any child can't handle the reset, then the entire  in eeh_handle_normal_event()
936 	 * When the PHB is fenced, we have to issue a reset to recover from  in eeh_handle_normal_event()
941 		pe->freeze_count, eeh_max_freezes);  in eeh_handle_normal_event()
951 	 * Error logged on a PHB are always fences which need a full  in eeh_handle_normal_event()
952 	 * PHB reset to clear so force that to happen.  in eeh_handle_normal_event()
954 	if ((pe->type & EEH_PE_PHB) && result != PCI_ERS_RESULT_NONE)  in eeh_handle_normal_event()
973 	/* If all device drivers were EEH-unaware, then shut  in eeh_handle_normal_event()
978 		pr_info("EEH: Reset with hotplug activity\n");  in eeh_handle_normal_event()
981 			pr_warn("%s: Unable to reset, err=%d\n", __func__, rc);  in eeh_handle_normal_event()
986 	/* If all devices reported they can proceed, then re-enable MMIO */  in eeh_handle_normal_event()
996 			pr_info("EEH: Notify device drivers to resume I/O\n");  in eeh_handle_normal_event()
1011 			 * We didn't do PE reset for the case. The PE  in eeh_handle_normal_event()
1020 	/* If any device called out for a reset, then reset the slot */  in eeh_handle_normal_event()
1022 		pr_info("EEH: Reset without hotplug activity\n");  in eeh_handle_normal_event()
1025 			pr_warn("%s: Cannot reset, err=%d\n", __func__, rc);  in eeh_handle_normal_event()
1045 			list_del(&edev->rmv_entry);  in eeh_handle_normal_event()
1048 		/* Tell all device drivers that they can resume operations */  in eeh_handle_normal_event()
1049 		pr_info("EEH: Notify device driver to resume\n");  in eeh_handle_normal_event()
1052 		eeh_pe_report("resume", pe, eeh_report_resume, NULL);  in eeh_handle_normal_event()
1055 				edev->mode &= ~EEH_DEV_NO_HANDLER;  in eeh_handle_normal_event()
1056 				edev->in_error = false;  in eeh_handle_normal_event()
1066 	 * About 90% of all real-life EEH failures in the field  in eeh_handle_normal_event()
1070 	pr_err("EEH: Unable to recover from failure from PHB#%x-PE#%x.\n"  in eeh_handle_normal_event()
1072 		pe->phb->global_number, pe->addr);  in eeh_handle_normal_event()
1090 	if (pe->type & EEH_PE_VF) {  in eeh_handle_normal_event()
1114 			eeh_clear_slot_attention(edev->pdev);  in eeh_handle_normal_event()
1120  * eeh_handle_special_event - Handle EEH events without a specific failing PE
1137 		rc = eeh_ops->next_error(&pe);  in eeh_handle_special_event()
1193 					edev->mode &= ~EEH_DEV_NO_HANDLER;  in eeh_handle_special_event()
1206 				    !(phb_pe->state & EEH_PE_ISOLATED) ||  in eeh_handle_special_event()
1207 				    (phb_pe->state & EEH_PE_RECOVERING))  in eeh_handle_special_event()
1213 					       "PHB#%x-PE#%x\n",  in eeh_handle_special_event()
1215 					       pe->phb->global_number,  in eeh_handle_special_event()
1216 					       pe->addr);  in eeh_handle_special_event()