1 // SPDX-License-Identifier: GPL-2.0-or-later 2 /* 3 * Copyright IBM Corporation 2001, 2005, 2006 4 * Copyright Dave Engebretsen & Todd Inglett 2001 5 * Copyright Linas Vepstas 2005, 2006 6 * Copyright 2001-2012 IBM Corporation. 7 * 8 * Please address comments and feedback to Linas Vepstas <linas@austin.ibm.com> 9 */ 10 11 #include <linux/delay.h> 12 #include <linux/sched.h> 13 #include <linux/init.h> 14 #include <linux/list.h> 15 #include <linux/pci.h> 16 #include <linux/iommu.h> 17 #include <linux/proc_fs.h> 18 #include <linux/rbtree.h> 19 #include <linux/reboot.h> 20 #include <linux/seq_file.h> 21 #include <linux/spinlock.h> 22 #include <linux/export.h> 23 #include <linux/of.h> 24 #include <linux/debugfs.h> 25 26 #include <linux/atomic.h> 27 #include <asm/eeh.h> 28 #include <asm/eeh_event.h> 29 #include <asm/io.h> 30 #include <asm/iommu.h> 31 #include <asm/machdep.h> 32 #include <asm/ppc-pci.h> 33 #include <asm/rtas.h> 34 #include <asm/pte-walk.h> 35 36 37 /** Overview: 38 * EEH, or "Enhanced Error Handling" is a PCI bridge technology for 39 * dealing with PCI bus errors that can't be dealt with within the 40 * usual PCI framework, except by check-stopping the CPU. Systems 41 * that are designed for high-availability/reliability cannot afford 42 * to crash due to a "mere" PCI error, thus the need for EEH. 43 * An EEH-capable bridge operates by converting a detected error 44 * into a "slot freeze", taking the PCI adapter off-line, making 45 * the slot behave, from the OS'es point of view, as if the slot 46 * were "empty": all reads return 0xff's and all writes are silently 47 * ignored. EEH slot isolation events can be triggered by parity 48 * errors on the address or data busses (e.g. during posted writes), 49 * which in turn might be caused by low voltage on the bus, dust, 50 * vibration, humidity, radioactivity or plain-old failed hardware. 51 * 52 * Note, however, that one of the leading causes of EEH slot 53 * freeze events are buggy device drivers, buggy device microcode, 54 * or buggy device hardware. This is because any attempt by the 55 * device to bus-master data to a memory address that is not 56 * assigned to the device will trigger a slot freeze. (The idea 57 * is to prevent devices-gone-wild from corrupting system memory). 58 * Buggy hardware/drivers will have a miserable time co-existing 59 * with EEH. 60 * 61 * Ideally, a PCI device driver, when suspecting that an isolation 62 * event has occurred (e.g. by reading 0xff's), will then ask EEH 63 * whether this is the case, and then take appropriate steps to 64 * reset the PCI slot, the PCI device, and then resume operations. 65 * However, until that day, the checking is done here, with the 66 * eeh_check_failure() routine embedded in the MMIO macros. If 67 * the slot is found to be isolated, an "EEH Event" is synthesized 68 * and sent out for processing. 69 */ 70 71 /* If a device driver keeps reading an MMIO register in an interrupt 72 * handler after a slot isolation event, it might be broken. 73 * This sets the threshold for how many read attempts we allow 74 * before printing an error message. 75 */ 76 #define EEH_MAX_FAILS 2100000 77 78 /* Time to wait for a PCI slot to report status, in milliseconds */ 79 #define PCI_BUS_RESET_WAIT_MSEC (5*60*1000) 80 81 /* 82 * EEH probe mode support, which is part of the flags, 83 * is to support multiple platforms for EEH. Some platforms 84 * like pSeries do PCI emunation based on device tree. 85 * However, other platforms like powernv probe PCI devices 86 * from hardware. The flag is used to distinguish that. 87 * In addition, struct eeh_ops::probe would be invoked for 88 * particular OF node or PCI device so that the corresponding 89 * PE would be created there. 90 */ 91 int eeh_subsystem_flags; 92 EXPORT_SYMBOL(eeh_subsystem_flags); 93 94 /* 95 * EEH allowed maximal frozen times. If one particular PE's 96 * frozen count in last hour exceeds this limit, the PE will 97 * be forced to be offline permanently. 98 */ 99 u32 eeh_max_freezes = 5; 100 101 /* 102 * Controls whether a recovery event should be scheduled when an 103 * isolated device is discovered. This is only really useful for 104 * debugging problems with the EEH core. 105 */ 106 bool eeh_debugfs_no_recover; 107 108 /* Platform dependent EEH operations */ 109 struct eeh_ops *eeh_ops = NULL; 110 111 /* Lock to avoid races due to multiple reports of an error */ 112 DEFINE_RAW_SPINLOCK(confirm_error_lock); 113 EXPORT_SYMBOL_GPL(confirm_error_lock); 114 115 /* Lock to protect passed flags */ 116 static DEFINE_MUTEX(eeh_dev_mutex); 117 118 /* Buffer for reporting pci register dumps. Its here in BSS, and 119 * not dynamically alloced, so that it ends up in RMO where RTAS 120 * can access it. 121 */ 122 #define EEH_PCI_REGS_LOG_LEN 8192 123 static unsigned char pci_regs_buf[EEH_PCI_REGS_LOG_LEN]; 124 125 /* 126 * The struct is used to maintain the EEH global statistic 127 * information. Besides, the EEH global statistics will be 128 * exported to user space through procfs 129 */ 130 struct eeh_stats { 131 u64 no_device; /* PCI device not found */ 132 u64 no_dn; /* OF node not found */ 133 u64 no_cfg_addr; /* Config address not found */ 134 u64 ignored_check; /* EEH check skipped */ 135 u64 total_mmio_ffs; /* Total EEH checks */ 136 u64 false_positives; /* Unnecessary EEH checks */ 137 u64 slot_resets; /* PE reset */ 138 }; 139 140 static struct eeh_stats eeh_stats; 141 eeh_setup(char * str)142 static int __init eeh_setup(char *str) 143 { 144 if (!strcmp(str, "off")) 145 eeh_add_flag(EEH_FORCE_DISABLED); 146 else if (!strcmp(str, "early_log")) 147 eeh_add_flag(EEH_EARLY_DUMP_LOG); 148 149 return 1; 150 } 151 __setup("eeh=", eeh_setup); 152 eeh_show_enabled(void)153 void eeh_show_enabled(void) 154 { 155 if (eeh_has_flag(EEH_FORCE_DISABLED)) 156 pr_info("EEH: Recovery disabled by kernel parameter.\n"); 157 else if (eeh_has_flag(EEH_ENABLED)) 158 pr_info("EEH: Capable adapter found: recovery enabled.\n"); 159 else 160 pr_info("EEH: No capable adapters found: recovery disabled.\n"); 161 } 162 163 /* 164 * This routine captures assorted PCI configuration space data 165 * for the indicated PCI device, and puts them into a buffer 166 * for RTAS error logging. 167 */ eeh_dump_dev_log(struct eeh_dev * edev,char * buf,size_t len)168 static size_t eeh_dump_dev_log(struct eeh_dev *edev, char *buf, size_t len) 169 { 170 u32 cfg; 171 int cap, i; 172 int n = 0, l = 0; 173 char buffer[128]; 174 175 n += scnprintf(buf+n, len-n, "%04x:%02x:%02x.%01x\n", 176 edev->pe->phb->global_number, edev->bdfn >> 8, 177 PCI_SLOT(edev->bdfn), PCI_FUNC(edev->bdfn)); 178 pr_warn("EEH: of node=%04x:%02x:%02x.%01x\n", 179 edev->pe->phb->global_number, edev->bdfn >> 8, 180 PCI_SLOT(edev->bdfn), PCI_FUNC(edev->bdfn)); 181 182 eeh_ops->read_config(edev, PCI_VENDOR_ID, 4, &cfg); 183 n += scnprintf(buf+n, len-n, "dev/vend:%08x\n", cfg); 184 pr_warn("EEH: PCI device/vendor: %08x\n", cfg); 185 186 eeh_ops->read_config(edev, PCI_COMMAND, 4, &cfg); 187 n += scnprintf(buf+n, len-n, "cmd/stat:%x\n", cfg); 188 pr_warn("EEH: PCI cmd/status register: %08x\n", cfg); 189 190 /* Gather bridge-specific registers */ 191 if (edev->mode & EEH_DEV_BRIDGE) { 192 eeh_ops->read_config(edev, PCI_SEC_STATUS, 2, &cfg); 193 n += scnprintf(buf+n, len-n, "sec stat:%x\n", cfg); 194 pr_warn("EEH: Bridge secondary status: %04x\n", cfg); 195 196 eeh_ops->read_config(edev, PCI_BRIDGE_CONTROL, 2, &cfg); 197 n += scnprintf(buf+n, len-n, "brdg ctl:%x\n", cfg); 198 pr_warn("EEH: Bridge control: %04x\n", cfg); 199 } 200 201 /* Dump out the PCI-X command and status regs */ 202 cap = edev->pcix_cap; 203 if (cap) { 204 eeh_ops->read_config(edev, cap, 4, &cfg); 205 n += scnprintf(buf+n, len-n, "pcix-cmd:%x\n", cfg); 206 pr_warn("EEH: PCI-X cmd: %08x\n", cfg); 207 208 eeh_ops->read_config(edev, cap+4, 4, &cfg); 209 n += scnprintf(buf+n, len-n, "pcix-stat:%x\n", cfg); 210 pr_warn("EEH: PCI-X status: %08x\n", cfg); 211 } 212 213 /* If PCI-E capable, dump PCI-E cap 10 */ 214 cap = edev->pcie_cap; 215 if (cap) { 216 n += scnprintf(buf+n, len-n, "pci-e cap10:\n"); 217 pr_warn("EEH: PCI-E capabilities and status follow:\n"); 218 219 for (i=0; i<=8; i++) { 220 eeh_ops->read_config(edev, cap+4*i, 4, &cfg); 221 n += scnprintf(buf+n, len-n, "%02x:%x\n", 4*i, cfg); 222 223 if ((i % 4) == 0) { 224 if (i != 0) 225 pr_warn("%s\n", buffer); 226 227 l = scnprintf(buffer, sizeof(buffer), 228 "EEH: PCI-E %02x: %08x ", 229 4*i, cfg); 230 } else { 231 l += scnprintf(buffer+l, sizeof(buffer)-l, 232 "%08x ", cfg); 233 } 234 235 } 236 237 pr_warn("%s\n", buffer); 238 } 239 240 /* If AER capable, dump it */ 241 cap = edev->aer_cap; 242 if (cap) { 243 n += scnprintf(buf+n, len-n, "pci-e AER:\n"); 244 pr_warn("EEH: PCI-E AER capability register set follows:\n"); 245 246 for (i=0; i<=13; i++) { 247 eeh_ops->read_config(edev, cap+4*i, 4, &cfg); 248 n += scnprintf(buf+n, len-n, "%02x:%x\n", 4*i, cfg); 249 250 if ((i % 4) == 0) { 251 if (i != 0) 252 pr_warn("%s\n", buffer); 253 254 l = scnprintf(buffer, sizeof(buffer), 255 "EEH: PCI-E AER %02x: %08x ", 256 4*i, cfg); 257 } else { 258 l += scnprintf(buffer+l, sizeof(buffer)-l, 259 "%08x ", cfg); 260 } 261 } 262 263 pr_warn("%s\n", buffer); 264 } 265 266 return n; 267 } 268 eeh_dump_pe_log(struct eeh_pe * pe,void * flag)269 static void *eeh_dump_pe_log(struct eeh_pe *pe, void *flag) 270 { 271 struct eeh_dev *edev, *tmp; 272 size_t *plen = flag; 273 274 eeh_pe_for_each_dev(pe, edev, tmp) 275 *plen += eeh_dump_dev_log(edev, pci_regs_buf + *plen, 276 EEH_PCI_REGS_LOG_LEN - *plen); 277 278 return NULL; 279 } 280 281 /** 282 * eeh_slot_error_detail - Generate combined log including driver log and error log 283 * @pe: EEH PE 284 * @severity: temporary or permanent error log 285 * 286 * This routine should be called to generate the combined log, which 287 * is comprised of driver log and error log. The driver log is figured 288 * out from the config space of the corresponding PCI device, while 289 * the error log is fetched through platform dependent function call. 290 */ eeh_slot_error_detail(struct eeh_pe * pe,int severity)291 void eeh_slot_error_detail(struct eeh_pe *pe, int severity) 292 { 293 size_t loglen = 0; 294 295 /* 296 * When the PHB is fenced or dead, it's pointless to collect 297 * the data from PCI config space because it should return 298 * 0xFF's. For ER, we still retrieve the data from the PCI 299 * config space. 300 * 301 * For pHyp, we have to enable IO for log retrieval. Otherwise, 302 * 0xFF's is always returned from PCI config space. 303 * 304 * When the @severity is EEH_LOG_PERM, the PE is going to be 305 * removed. Prior to that, the drivers for devices included in 306 * the PE will be closed. The drivers rely on working IO path 307 * to bring the devices to quiet state. Otherwise, PCI traffic 308 * from those devices after they are removed is like to cause 309 * another unexpected EEH error. 310 */ 311 if (!(pe->type & EEH_PE_PHB)) { 312 if (eeh_has_flag(EEH_ENABLE_IO_FOR_LOG) || 313 severity == EEH_LOG_PERM) 314 eeh_pci_enable(pe, EEH_OPT_THAW_MMIO); 315 316 /* 317 * The config space of some PCI devices can't be accessed 318 * when their PEs are in frozen state. Otherwise, fenced 319 * PHB might be seen. Those PEs are identified with flag 320 * EEH_PE_CFG_RESTRICTED, indicating EEH_PE_CFG_BLOCKED 321 * is set automatically when the PE is put to EEH_PE_ISOLATED. 322 * 323 * Restoring BARs possibly triggers PCI config access in 324 * (OPAL) firmware and then causes fenced PHB. If the 325 * PCI config is blocked with flag EEH_PE_CFG_BLOCKED, it's 326 * pointless to restore BARs and dump config space. 327 */ 328 eeh_ops->configure_bridge(pe); 329 if (!(pe->state & EEH_PE_CFG_BLOCKED)) { 330 eeh_pe_restore_bars(pe); 331 332 pci_regs_buf[0] = 0; 333 eeh_pe_traverse(pe, eeh_dump_pe_log, &loglen); 334 } 335 } 336 337 eeh_ops->get_log(pe, severity, pci_regs_buf, loglen); 338 } 339 340 /** 341 * eeh_token_to_phys - Convert EEH address token to phys address 342 * @token: I/O token, should be address in the form 0xA.... 343 * 344 * This routine should be called to convert virtual I/O address 345 * to physical one. 346 */ eeh_token_to_phys(unsigned long token)347 static inline unsigned long eeh_token_to_phys(unsigned long token) 348 { 349 return ppc_find_vmap_phys(token); 350 } 351 352 /* 353 * On PowerNV platform, we might already have fenced PHB there. 354 * For that case, it's meaningless to recover frozen PE. Intead, 355 * We have to handle fenced PHB firstly. 356 */ eeh_phb_check_failure(struct eeh_pe * pe)357 static int eeh_phb_check_failure(struct eeh_pe *pe) 358 { 359 struct eeh_pe *phb_pe; 360 unsigned long flags; 361 int ret; 362 363 if (!eeh_has_flag(EEH_PROBE_MODE_DEV)) 364 return -EPERM; 365 366 /* Find the PHB PE */ 367 phb_pe = eeh_phb_pe_get(pe->phb); 368 if (!phb_pe) { 369 pr_warn("%s Can't find PE for PHB#%x\n", 370 __func__, pe->phb->global_number); 371 return -EEXIST; 372 } 373 374 /* If the PHB has been in problematic state */ 375 eeh_serialize_lock(&flags); 376 if (phb_pe->state & EEH_PE_ISOLATED) { 377 ret = 0; 378 goto out; 379 } 380 381 /* Check PHB state */ 382 ret = eeh_ops->get_state(phb_pe, NULL); 383 if ((ret < 0) || 384 (ret == EEH_STATE_NOT_SUPPORT) || eeh_state_active(ret)) { 385 ret = 0; 386 goto out; 387 } 388 389 /* Isolate the PHB and send event */ 390 eeh_pe_mark_isolated(phb_pe); 391 eeh_serialize_unlock(flags); 392 393 pr_debug("EEH: PHB#%x failure detected, location: %s\n", 394 phb_pe->phb->global_number, eeh_pe_loc_get(phb_pe)); 395 eeh_send_failure_event(phb_pe); 396 return 1; 397 out: 398 eeh_serialize_unlock(flags); 399 return ret; 400 } 401 eeh_driver_name(struct pci_dev * pdev)402 static inline const char *eeh_driver_name(struct pci_dev *pdev) 403 { 404 if (pdev) 405 return dev_driver_string(&pdev->dev); 406 407 return "<null>"; 408 } 409 410 /** 411 * eeh_dev_check_failure - Check if all 1's data is due to EEH slot freeze 412 * @edev: eeh device 413 * 414 * Check for an EEH failure for the given device node. Call this 415 * routine if the result of a read was all 0xff's and you want to 416 * find out if this is due to an EEH slot freeze. This routine 417 * will query firmware for the EEH status. 418 * 419 * Returns 0 if there has not been an EEH error; otherwise returns 420 * a non-zero value and queues up a slot isolation event notification. 421 * 422 * It is safe to call this routine in an interrupt context. 423 */ eeh_dev_check_failure(struct eeh_dev * edev)424 int eeh_dev_check_failure(struct eeh_dev *edev) 425 { 426 int ret; 427 unsigned long flags; 428 struct device_node *dn; 429 struct pci_dev *dev; 430 struct eeh_pe *pe, *parent_pe; 431 int rc = 0; 432 const char *location = NULL; 433 434 eeh_stats.total_mmio_ffs++; 435 436 if (!eeh_enabled()) 437 return 0; 438 439 if (!edev) { 440 eeh_stats.no_dn++; 441 return 0; 442 } 443 dev = eeh_dev_to_pci_dev(edev); 444 pe = eeh_dev_to_pe(edev); 445 446 /* Access to IO BARs might get this far and still not want checking. */ 447 if (!pe) { 448 eeh_stats.ignored_check++; 449 eeh_edev_dbg(edev, "Ignored check\n"); 450 return 0; 451 } 452 453 /* 454 * On PowerNV platform, we might already have fenced PHB 455 * there and we need take care of that firstly. 456 */ 457 ret = eeh_phb_check_failure(pe); 458 if (ret > 0) 459 return ret; 460 461 /* 462 * If the PE isn't owned by us, we shouldn't check the 463 * state. Instead, let the owner handle it if the PE has 464 * been frozen. 465 */ 466 if (eeh_pe_passed(pe)) 467 return 0; 468 469 /* If we already have a pending isolation event for this 470 * slot, we know it's bad already, we don't need to check. 471 * Do this checking under a lock; as multiple PCI devices 472 * in one slot might report errors simultaneously, and we 473 * only want one error recovery routine running. 474 */ 475 eeh_serialize_lock(&flags); 476 rc = 1; 477 if (pe->state & EEH_PE_ISOLATED) { 478 pe->check_count++; 479 if (pe->check_count == EEH_MAX_FAILS) { 480 dn = pci_device_to_OF_node(dev); 481 if (dn) 482 location = of_get_property(dn, "ibm,loc-code", 483 NULL); 484 eeh_edev_err(edev, "%d reads ignored for recovering device at location=%s driver=%s\n", 485 pe->check_count, 486 location ? location : "unknown", 487 eeh_driver_name(dev)); 488 eeh_edev_err(edev, "Might be infinite loop in %s driver\n", 489 eeh_driver_name(dev)); 490 dump_stack(); 491 } 492 goto dn_unlock; 493 } 494 495 /* 496 * Now test for an EEH failure. This is VERY expensive. 497 * Note that the eeh_config_addr may be a parent device 498 * in the case of a device behind a bridge, or it may be 499 * function zero of a multi-function device. 500 * In any case they must share a common PHB. 501 */ 502 ret = eeh_ops->get_state(pe, NULL); 503 504 /* Note that config-io to empty slots may fail; 505 * they are empty when they don't have children. 506 * We will punt with the following conditions: Failure to get 507 * PE's state, EEH not support and Permanently unavailable 508 * state, PE is in good state. 509 * 510 * On the pSeries, after reaching the threshold, get_state might 511 * return EEH_STATE_NOT_SUPPORT. However, it's possible that the 512 * device state remains uncleared if the device is not marked 513 * pci_channel_io_perm_failure. Therefore, consider logging the 514 * event to let device removal happen. 515 * 516 */ 517 if ((ret < 0) || 518 (ret == EEH_STATE_NOT_SUPPORT && 519 dev->error_state == pci_channel_io_perm_failure) || 520 eeh_state_active(ret)) { 521 eeh_stats.false_positives++; 522 pe->false_positives++; 523 rc = 0; 524 goto dn_unlock; 525 } 526 527 /* 528 * It should be corner case that the parent PE has been 529 * put into frozen state as well. We should take care 530 * that at first. 531 */ 532 parent_pe = pe->parent; 533 while (parent_pe) { 534 /* Hit the ceiling ? */ 535 if (parent_pe->type & EEH_PE_PHB) 536 break; 537 538 /* Frozen parent PE ? */ 539 ret = eeh_ops->get_state(parent_pe, NULL); 540 if (ret > 0 && !eeh_state_active(ret)) { 541 pe = parent_pe; 542 pr_err("EEH: Failure of PHB#%x-PE#%x will be handled at parent PHB#%x-PE#%x.\n", 543 pe->phb->global_number, pe->addr, 544 pe->phb->global_number, parent_pe->addr); 545 } 546 547 /* Next parent level */ 548 parent_pe = parent_pe->parent; 549 } 550 551 eeh_stats.slot_resets++; 552 553 /* Avoid repeated reports of this failure, including problems 554 * with other functions on this device, and functions under 555 * bridges. 556 */ 557 eeh_pe_mark_isolated(pe); 558 eeh_serialize_unlock(flags); 559 560 /* Most EEH events are due to device driver bugs. Having 561 * a stack trace will help the device-driver authors figure 562 * out what happened. So print that out. 563 */ 564 pr_debug("EEH: %s: Frozen PHB#%x-PE#%x detected\n", 565 __func__, pe->phb->global_number, pe->addr); 566 eeh_send_failure_event(pe); 567 568 return 1; 569 570 dn_unlock: 571 eeh_serialize_unlock(flags); 572 return rc; 573 } 574 575 EXPORT_SYMBOL_GPL(eeh_dev_check_failure); 576 577 /** 578 * eeh_check_failure - Check if all 1's data is due to EEH slot freeze 579 * @token: I/O address 580 * 581 * Check for an EEH failure at the given I/O address. Call this 582 * routine if the result of a read was all 0xff's and you want to 583 * find out if this is due to an EEH slot freeze event. This routine 584 * will query firmware for the EEH status. 585 * 586 * Note this routine is safe to call in an interrupt context. 587 */ eeh_check_failure(const volatile void __iomem * token)588 int eeh_check_failure(const volatile void __iomem *token) 589 { 590 unsigned long addr; 591 struct eeh_dev *edev; 592 593 /* Finding the phys addr + pci device; this is pretty quick. */ 594 addr = eeh_token_to_phys((unsigned long __force) token); 595 edev = eeh_addr_cache_get_dev(addr); 596 if (!edev) { 597 eeh_stats.no_device++; 598 return 0; 599 } 600 601 return eeh_dev_check_failure(edev); 602 } 603 EXPORT_SYMBOL(eeh_check_failure); 604 605 606 /** 607 * eeh_pci_enable - Enable MMIO or DMA transfers for this slot 608 * @pe: EEH PE 609 * @function: EEH option 610 * 611 * This routine should be called to reenable frozen MMIO or DMA 612 * so that it would work correctly again. It's useful while doing 613 * recovery or log collection on the indicated device. 614 */ eeh_pci_enable(struct eeh_pe * pe,int function)615 int eeh_pci_enable(struct eeh_pe *pe, int function) 616 { 617 int active_flag, rc; 618 619 /* 620 * pHyp doesn't allow to enable IO or DMA on unfrozen PE. 621 * Also, it's pointless to enable them on unfrozen PE. So 622 * we have to check before enabling IO or DMA. 623 */ 624 switch (function) { 625 case EEH_OPT_THAW_MMIO: 626 active_flag = EEH_STATE_MMIO_ACTIVE | EEH_STATE_MMIO_ENABLED; 627 break; 628 case EEH_OPT_THAW_DMA: 629 active_flag = EEH_STATE_DMA_ACTIVE; 630 break; 631 case EEH_OPT_DISABLE: 632 case EEH_OPT_ENABLE: 633 case EEH_OPT_FREEZE_PE: 634 active_flag = 0; 635 break; 636 default: 637 pr_warn("%s: Invalid function %d\n", 638 __func__, function); 639 return -EINVAL; 640 } 641 642 /* 643 * Check if IO or DMA has been enabled before 644 * enabling them. 645 */ 646 if (active_flag) { 647 rc = eeh_ops->get_state(pe, NULL); 648 if (rc < 0) 649 return rc; 650 651 /* Needn't enable it at all */ 652 if (rc == EEH_STATE_NOT_SUPPORT) 653 return 0; 654 655 /* It's already enabled */ 656 if (rc & active_flag) 657 return 0; 658 } 659 660 661 /* Issue the request */ 662 rc = eeh_ops->set_option(pe, function); 663 if (rc) 664 pr_warn("%s: Unexpected state change %d on " 665 "PHB#%x-PE#%x, err=%d\n", 666 __func__, function, pe->phb->global_number, 667 pe->addr, rc); 668 669 /* Check if the request is finished successfully */ 670 if (active_flag) { 671 rc = eeh_wait_state(pe, PCI_BUS_RESET_WAIT_MSEC); 672 if (rc < 0) 673 return rc; 674 675 if (rc & active_flag) 676 return 0; 677 678 return -EIO; 679 } 680 681 return rc; 682 } 683 eeh_disable_and_save_dev_state(struct eeh_dev * edev,void * userdata)684 static void eeh_disable_and_save_dev_state(struct eeh_dev *edev, 685 void *userdata) 686 { 687 struct pci_dev *pdev = eeh_dev_to_pci_dev(edev); 688 struct pci_dev *dev = userdata; 689 690 /* 691 * The caller should have disabled and saved the 692 * state for the specified device 693 */ 694 if (!pdev || pdev == dev) 695 return; 696 697 /* Ensure we have D0 power state */ 698 pci_set_power_state(pdev, PCI_D0); 699 700 /* Save device state */ 701 pci_save_state(pdev); 702 703 /* 704 * Disable device to avoid any DMA traffic and 705 * interrupt from the device 706 */ 707 pci_write_config_word(pdev, PCI_COMMAND, PCI_COMMAND_INTX_DISABLE); 708 } 709 eeh_restore_dev_state(struct eeh_dev * edev,void * userdata)710 static void eeh_restore_dev_state(struct eeh_dev *edev, void *userdata) 711 { 712 struct pci_dev *pdev = eeh_dev_to_pci_dev(edev); 713 struct pci_dev *dev = userdata; 714 715 if (!pdev) 716 return; 717 718 /* Apply customization from firmware */ 719 if (eeh_ops->restore_config) 720 eeh_ops->restore_config(edev); 721 722 /* The caller should restore state for the specified device */ 723 if (pdev != dev) 724 pci_restore_state(pdev); 725 } 726 727 /** 728 * pcibios_set_pcie_reset_state - Set PCI-E reset state 729 * @dev: pci device struct 730 * @state: reset state to enter 731 * 732 * Return value: 733 * 0 if success 734 */ pcibios_set_pcie_reset_state(struct pci_dev * dev,enum pcie_reset_state state)735 int pcibios_set_pcie_reset_state(struct pci_dev *dev, enum pcie_reset_state state) 736 { 737 struct eeh_dev *edev = pci_dev_to_eeh_dev(dev); 738 struct eeh_pe *pe = eeh_dev_to_pe(edev); 739 740 if (!pe) { 741 pr_err("%s: No PE found on PCI device %s\n", 742 __func__, pci_name(dev)); 743 return -EINVAL; 744 } 745 746 switch (state) { 747 case pcie_deassert_reset: 748 eeh_ops->reset(pe, EEH_RESET_DEACTIVATE); 749 eeh_unfreeze_pe(pe); 750 if (!(pe->type & EEH_PE_VF)) 751 eeh_pe_state_clear(pe, EEH_PE_CFG_BLOCKED, true); 752 eeh_pe_dev_traverse(pe, eeh_restore_dev_state, dev); 753 eeh_pe_state_clear(pe, EEH_PE_ISOLATED, true); 754 break; 755 case pcie_hot_reset: 756 eeh_pe_mark_isolated(pe); 757 eeh_pe_state_clear(pe, EEH_PE_CFG_BLOCKED, true); 758 eeh_ops->set_option(pe, EEH_OPT_FREEZE_PE); 759 eeh_pe_dev_traverse(pe, eeh_disable_and_save_dev_state, dev); 760 if (!(pe->type & EEH_PE_VF)) 761 eeh_pe_state_mark(pe, EEH_PE_CFG_BLOCKED); 762 eeh_ops->reset(pe, EEH_RESET_HOT); 763 break; 764 case pcie_warm_reset: 765 eeh_pe_mark_isolated(pe); 766 eeh_pe_state_clear(pe, EEH_PE_CFG_BLOCKED, true); 767 eeh_ops->set_option(pe, EEH_OPT_FREEZE_PE); 768 eeh_pe_dev_traverse(pe, eeh_disable_and_save_dev_state, dev); 769 if (!(pe->type & EEH_PE_VF)) 770 eeh_pe_state_mark(pe, EEH_PE_CFG_BLOCKED); 771 eeh_ops->reset(pe, EEH_RESET_FUNDAMENTAL); 772 break; 773 default: 774 eeh_pe_state_clear(pe, EEH_PE_ISOLATED | EEH_PE_CFG_BLOCKED, true); 775 return -EINVAL; 776 } 777 778 return 0; 779 } 780 781 /** 782 * eeh_set_dev_freset - Check the required reset for the indicated device 783 * @edev: EEH device 784 * @flag: return value 785 * 786 * Each device might have its preferred reset type: fundamental or 787 * hot reset. The routine is used to collected the information for 788 * the indicated device and its children so that the bunch of the 789 * devices could be reset properly. 790 */ eeh_set_dev_freset(struct eeh_dev * edev,void * flag)791 static void eeh_set_dev_freset(struct eeh_dev *edev, void *flag) 792 { 793 struct pci_dev *dev; 794 unsigned int *freset = (unsigned int *)flag; 795 796 dev = eeh_dev_to_pci_dev(edev); 797 if (dev) 798 *freset |= dev->needs_freset; 799 } 800 eeh_pe_refreeze_passed(struct eeh_pe * root)801 static void eeh_pe_refreeze_passed(struct eeh_pe *root) 802 { 803 struct eeh_pe *pe; 804 int state; 805 806 eeh_for_each_pe(root, pe) { 807 if (eeh_pe_passed(pe)) { 808 state = eeh_ops->get_state(pe, NULL); 809 if (state & 810 (EEH_STATE_MMIO_ACTIVE | EEH_STATE_MMIO_ENABLED)) { 811 pr_info("EEH: Passed-through PE PHB#%x-PE#%x was thawed by reset, re-freezing for safety.\n", 812 pe->phb->global_number, pe->addr); 813 eeh_pe_set_option(pe, EEH_OPT_FREEZE_PE); 814 } 815 } 816 } 817 } 818 819 /** 820 * eeh_pe_reset_full - Complete a full reset process on the indicated PE 821 * @pe: EEH PE 822 * @include_passed: include passed-through devices? 823 * 824 * This function executes a full reset procedure on a PE, including setting 825 * the appropriate flags, performing a fundamental or hot reset, and then 826 * deactivating the reset status. It is designed to be used within the EEH 827 * subsystem, as opposed to eeh_pe_reset which is exported to drivers and 828 * only performs a single operation at a time. 829 * 830 * This function will attempt to reset a PE three times before failing. 831 */ eeh_pe_reset_full(struct eeh_pe * pe,bool include_passed)832 int eeh_pe_reset_full(struct eeh_pe *pe, bool include_passed) 833 { 834 int reset_state = (EEH_PE_RESET | EEH_PE_CFG_BLOCKED); 835 int type = EEH_RESET_HOT; 836 unsigned int freset = 0; 837 int i, state = 0, ret; 838 839 /* 840 * Determine the type of reset to perform - hot or fundamental. 841 * Hot reset is the default operation, unless any device under the 842 * PE requires a fundamental reset. 843 */ 844 eeh_pe_dev_traverse(pe, eeh_set_dev_freset, &freset); 845 846 if (freset) 847 type = EEH_RESET_FUNDAMENTAL; 848 849 /* Mark the PE as in reset state and block config space accesses */ 850 eeh_pe_state_mark(pe, reset_state); 851 852 /* Make three attempts at resetting the bus */ 853 for (i = 0; i < 3; i++) { 854 ret = eeh_pe_reset(pe, type, include_passed); 855 if (!ret) 856 ret = eeh_pe_reset(pe, EEH_RESET_DEACTIVATE, 857 include_passed); 858 if (ret) { 859 ret = -EIO; 860 pr_warn("EEH: Failure %d resetting PHB#%x-PE#%x (attempt %d)\n\n", 861 state, pe->phb->global_number, pe->addr, i + 1); 862 continue; 863 } 864 if (i) 865 pr_warn("EEH: PHB#%x-PE#%x: Successful reset (attempt %d)\n", 866 pe->phb->global_number, pe->addr, i + 1); 867 868 /* Wait until the PE is in a functioning state */ 869 state = eeh_wait_state(pe, PCI_BUS_RESET_WAIT_MSEC); 870 if (state < 0) { 871 pr_warn("EEH: Unrecoverable slot failure on PHB#%x-PE#%x", 872 pe->phb->global_number, pe->addr); 873 ret = -ENOTRECOVERABLE; 874 break; 875 } 876 if (eeh_state_active(state)) 877 break; 878 else 879 pr_warn("EEH: PHB#%x-PE#%x: Slot inactive after reset: 0x%x (attempt %d)\n", 880 pe->phb->global_number, pe->addr, state, i + 1); 881 } 882 883 /* Resetting the PE may have unfrozen child PEs. If those PEs have been 884 * (potentially) passed through to a guest, re-freeze them: 885 */ 886 if (!include_passed) 887 eeh_pe_refreeze_passed(pe); 888 889 eeh_pe_state_clear(pe, reset_state, true); 890 return ret; 891 } 892 893 /** 894 * eeh_save_bars - Save device bars 895 * @edev: PCI device associated EEH device 896 * 897 * Save the values of the device bars. Unlike the restore 898 * routine, this routine is *not* recursive. This is because 899 * PCI devices are added individually; but, for the restore, 900 * an entire slot is reset at a time. 901 */ eeh_save_bars(struct eeh_dev * edev)902 void eeh_save_bars(struct eeh_dev *edev) 903 { 904 int i; 905 906 if (!edev) 907 return; 908 909 for (i = 0; i < 16; i++) 910 eeh_ops->read_config(edev, i * 4, 4, &edev->config_space[i]); 911 912 /* 913 * For PCI bridges including root port, we need enable bus 914 * master explicitly. Otherwise, it can't fetch IODA table 915 * entries correctly. So we cache the bit in advance so that 916 * we can restore it after reset, either PHB range or PE range. 917 */ 918 if (edev->mode & EEH_DEV_BRIDGE) 919 edev->config_space[1] |= PCI_COMMAND_MASTER; 920 } 921 eeh_reboot_notifier(struct notifier_block * nb,unsigned long action,void * unused)922 static int eeh_reboot_notifier(struct notifier_block *nb, 923 unsigned long action, void *unused) 924 { 925 eeh_clear_flag(EEH_ENABLED); 926 return NOTIFY_DONE; 927 } 928 929 static struct notifier_block eeh_reboot_nb = { 930 .notifier_call = eeh_reboot_notifier, 931 }; 932 eeh_device_notifier(struct notifier_block * nb,unsigned long action,void * data)933 static int eeh_device_notifier(struct notifier_block *nb, 934 unsigned long action, void *data) 935 { 936 struct device *dev = data; 937 938 switch (action) { 939 /* 940 * Note: It's not possible to perform EEH device addition (i.e. 941 * {pseries,pnv}_pcibios_bus_add_device()) here because it depends on 942 * the device's resources, which have not yet been set up. 943 */ 944 case BUS_NOTIFY_DEL_DEVICE: 945 eeh_remove_device(to_pci_dev(dev)); 946 break; 947 default: 948 break; 949 } 950 return NOTIFY_DONE; 951 } 952 953 static struct notifier_block eeh_device_nb = { 954 .notifier_call = eeh_device_notifier, 955 }; 956 957 /** 958 * eeh_init - System wide EEH initialization 959 * @ops: struct to trace EEH operation callback functions 960 * 961 * It's the platform's job to call this from an arch_initcall(). 962 */ eeh_init(struct eeh_ops * ops)963 int eeh_init(struct eeh_ops *ops) 964 { 965 struct pci_controller *hose, *tmp; 966 int ret = 0; 967 968 /* the platform should only initialise EEH once */ 969 if (WARN_ON(eeh_ops)) 970 return -EEXIST; 971 if (WARN_ON(!ops)) 972 return -ENOENT; 973 eeh_ops = ops; 974 975 /* Register reboot notifier */ 976 ret = register_reboot_notifier(&eeh_reboot_nb); 977 if (ret) { 978 pr_warn("%s: Failed to register reboot notifier (%d)\n", 979 __func__, ret); 980 return ret; 981 } 982 983 ret = bus_register_notifier(&pci_bus_type, &eeh_device_nb); 984 if (ret) { 985 pr_warn("%s: Failed to register bus notifier (%d)\n", 986 __func__, ret); 987 return ret; 988 } 989 990 /* Initialize PHB PEs */ 991 list_for_each_entry_safe(hose, tmp, &hose_list, list_node) 992 eeh_phb_pe_create(hose); 993 994 eeh_addr_cache_init(); 995 996 /* Initialize EEH event */ 997 return eeh_event_init(); 998 } 999 1000 /** 1001 * eeh_probe_device() - Perform EEH initialization for the indicated pci device 1002 * @dev: pci device for which to set up EEH 1003 * 1004 * This routine must be used to complete EEH initialization for PCI 1005 * devices that were added after system boot (e.g. hotplug, dlpar). 1006 */ eeh_probe_device(struct pci_dev * dev)1007 void eeh_probe_device(struct pci_dev *dev) 1008 { 1009 struct eeh_dev *edev; 1010 1011 pr_debug("EEH: Adding device %s\n", pci_name(dev)); 1012 1013 /* 1014 * pci_dev_to_eeh_dev() can only work if eeh_probe_dev() was 1015 * already called for this device. 1016 */ 1017 if (WARN_ON_ONCE(pci_dev_to_eeh_dev(dev))) { 1018 pci_dbg(dev, "Already bound to an eeh_dev!\n"); 1019 return; 1020 } 1021 1022 edev = eeh_ops->probe(dev); 1023 if (!edev) { 1024 pr_debug("EEH: Adding device failed\n"); 1025 return; 1026 } 1027 1028 /* 1029 * FIXME: We rely on pcibios_release_device() to remove the 1030 * existing EEH state. The release function is only called if 1031 * the pci_dev's refcount drops to zero so if something is 1032 * keeping a ref to a device (e.g. a filesystem) we need to 1033 * remove the old EEH state. 1034 * 1035 * FIXME: HEY MA, LOOK AT ME, NO LOCKING! 1036 */ 1037 if (edev->pdev && edev->pdev != dev) { 1038 eeh_pe_tree_remove(edev); 1039 eeh_addr_cache_rmv_dev(edev->pdev); 1040 eeh_sysfs_remove_device(edev->pdev); 1041 1042 /* 1043 * We definitely should have the PCI device removed 1044 * though it wasn't correctly. So we needn't call 1045 * into error handler afterwards. 1046 */ 1047 edev->mode |= EEH_DEV_NO_HANDLER; 1048 } 1049 1050 /* bind the pdev and the edev together */ 1051 edev->pdev = dev; 1052 dev->dev.archdata.edev = edev; 1053 eeh_addr_cache_insert_dev(dev); 1054 eeh_sysfs_add_device(dev); 1055 } 1056 1057 /** 1058 * eeh_remove_device - Undo EEH setup for the indicated pci device 1059 * @dev: pci device to be removed 1060 * 1061 * This routine should be called when a device is removed from 1062 * a running system (e.g. by hotplug or dlpar). It unregisters 1063 * the PCI device from the EEH subsystem. I/O errors affecting 1064 * this device will no longer be detected after this call; thus, 1065 * i/o errors affecting this slot may leave this device unusable. 1066 */ eeh_remove_device(struct pci_dev * dev)1067 void eeh_remove_device(struct pci_dev *dev) 1068 { 1069 struct eeh_dev *edev; 1070 1071 if (!dev || !eeh_enabled()) 1072 return; 1073 edev = pci_dev_to_eeh_dev(dev); 1074 1075 /* Unregister the device with the EEH/PCI address search system */ 1076 dev_dbg(&dev->dev, "EEH: Removing device\n"); 1077 1078 if (!edev || !edev->pdev || !edev->pe) { 1079 dev_dbg(&dev->dev, "EEH: Device not referenced!\n"); 1080 return; 1081 } 1082 1083 /* 1084 * During the hotplug for EEH error recovery, we need the EEH 1085 * device attached to the parent PE in order for BAR restore 1086 * a bit later. So we keep it for BAR restore and remove it 1087 * from the parent PE during the BAR resotre. 1088 */ 1089 edev->pdev = NULL; 1090 1091 /* 1092 * eeh_sysfs_remove_device() uses pci_dev_to_eeh_dev() so we need to 1093 * remove the sysfs files before clearing dev.archdata.edev 1094 */ 1095 if (edev->mode & EEH_DEV_SYSFS) 1096 eeh_sysfs_remove_device(dev); 1097 1098 /* 1099 * We're removing from the PCI subsystem, that means 1100 * the PCI device driver can't support EEH or not 1101 * well. So we rely on hotplug completely to do recovery 1102 * for the specific PCI device. 1103 */ 1104 edev->mode |= EEH_DEV_NO_HANDLER; 1105 1106 eeh_addr_cache_rmv_dev(dev); 1107 1108 /* 1109 * The flag "in_error" is used to trace EEH devices for VFs 1110 * in error state or not. It's set in eeh_report_error(). If 1111 * it's not set, eeh_report_{reset,resume}() won't be called 1112 * for the VF EEH device. 1113 */ 1114 edev->in_error = false; 1115 dev->dev.archdata.edev = NULL; 1116 if (!(edev->pe->state & EEH_PE_KEEP)) 1117 eeh_pe_tree_remove(edev); 1118 else 1119 edev->mode |= EEH_DEV_DISCONNECTED; 1120 } 1121 eeh_unfreeze_pe(struct eeh_pe * pe)1122 int eeh_unfreeze_pe(struct eeh_pe *pe) 1123 { 1124 int ret; 1125 1126 ret = eeh_pci_enable(pe, EEH_OPT_THAW_MMIO); 1127 if (ret) { 1128 pr_warn("%s: Failure %d enabling IO on PHB#%x-PE#%x\n", 1129 __func__, ret, pe->phb->global_number, pe->addr); 1130 return ret; 1131 } 1132 1133 ret = eeh_pci_enable(pe, EEH_OPT_THAW_DMA); 1134 if (ret) { 1135 pr_warn("%s: Failure %d enabling DMA on PHB#%x-PE#%x\n", 1136 __func__, ret, pe->phb->global_number, pe->addr); 1137 return ret; 1138 } 1139 1140 return ret; 1141 } 1142 1143 1144 static struct pci_device_id eeh_reset_ids[] = { 1145 { PCI_DEVICE(0x19a2, 0x0710) }, /* Emulex, BE */ 1146 { PCI_DEVICE(0x10df, 0xe220) }, /* Emulex, Lancer */ 1147 { PCI_DEVICE(0x14e4, 0x1657) }, /* Broadcom BCM5719 */ 1148 { 0 } 1149 }; 1150 eeh_pe_change_owner(struct eeh_pe * pe)1151 static int eeh_pe_change_owner(struct eeh_pe *pe) 1152 { 1153 struct eeh_dev *edev, *tmp; 1154 struct pci_dev *pdev; 1155 struct pci_device_id *id; 1156 int ret; 1157 1158 /* Check PE state */ 1159 ret = eeh_ops->get_state(pe, NULL); 1160 if (ret < 0 || ret == EEH_STATE_NOT_SUPPORT) 1161 return 0; 1162 1163 /* Unfrozen PE, nothing to do */ 1164 if (eeh_state_active(ret)) 1165 return 0; 1166 1167 /* Frozen PE, check if it needs PE level reset */ 1168 eeh_pe_for_each_dev(pe, edev, tmp) { 1169 pdev = eeh_dev_to_pci_dev(edev); 1170 if (!pdev) 1171 continue; 1172 1173 for (id = &eeh_reset_ids[0]; id->vendor != 0; id++) { 1174 if (id->vendor != PCI_ANY_ID && 1175 id->vendor != pdev->vendor) 1176 continue; 1177 if (id->device != PCI_ANY_ID && 1178 id->device != pdev->device) 1179 continue; 1180 if (id->subvendor != PCI_ANY_ID && 1181 id->subvendor != pdev->subsystem_vendor) 1182 continue; 1183 if (id->subdevice != PCI_ANY_ID && 1184 id->subdevice != pdev->subsystem_device) 1185 continue; 1186 1187 return eeh_pe_reset_and_recover(pe); 1188 } 1189 } 1190 1191 ret = eeh_unfreeze_pe(pe); 1192 if (!ret) 1193 eeh_pe_state_clear(pe, EEH_PE_ISOLATED, true); 1194 return ret; 1195 } 1196 1197 /** 1198 * eeh_dev_open - Increase count of pass through devices for PE 1199 * @pdev: PCI device 1200 * 1201 * Increase count of passed through devices for the indicated 1202 * PE. In the result, the EEH errors detected on the PE won't be 1203 * reported. The PE owner will be responsible for detection 1204 * and recovery. 1205 */ eeh_dev_open(struct pci_dev * pdev)1206 int eeh_dev_open(struct pci_dev *pdev) 1207 { 1208 struct eeh_dev *edev; 1209 int ret = -ENODEV; 1210 1211 mutex_lock(&eeh_dev_mutex); 1212 1213 /* No PCI device ? */ 1214 if (!pdev) 1215 goto out; 1216 1217 /* No EEH device or PE ? */ 1218 edev = pci_dev_to_eeh_dev(pdev); 1219 if (!edev || !edev->pe) 1220 goto out; 1221 1222 /* 1223 * The PE might have been put into frozen state, but we 1224 * didn't detect that yet. The passed through PCI devices 1225 * in frozen PE won't work properly. Clear the frozen state 1226 * in advance. 1227 */ 1228 ret = eeh_pe_change_owner(edev->pe); 1229 if (ret) 1230 goto out; 1231 1232 /* Increase PE's pass through count */ 1233 atomic_inc(&edev->pe->pass_dev_cnt); 1234 mutex_unlock(&eeh_dev_mutex); 1235 1236 return 0; 1237 out: 1238 mutex_unlock(&eeh_dev_mutex); 1239 return ret; 1240 } 1241 EXPORT_SYMBOL_GPL(eeh_dev_open); 1242 1243 /** 1244 * eeh_dev_release - Decrease count of pass through devices for PE 1245 * @pdev: PCI device 1246 * 1247 * Decrease count of pass through devices for the indicated PE. If 1248 * there is no passed through device in PE, the EEH errors detected 1249 * on the PE will be reported and handled as usual. 1250 */ eeh_dev_release(struct pci_dev * pdev)1251 void eeh_dev_release(struct pci_dev *pdev) 1252 { 1253 struct eeh_dev *edev; 1254 1255 mutex_lock(&eeh_dev_mutex); 1256 1257 /* No PCI device ? */ 1258 if (!pdev) 1259 goto out; 1260 1261 /* No EEH device ? */ 1262 edev = pci_dev_to_eeh_dev(pdev); 1263 if (!edev || !edev->pe || !eeh_pe_passed(edev->pe)) 1264 goto out; 1265 1266 /* Decrease PE's pass through count */ 1267 WARN_ON(atomic_dec_if_positive(&edev->pe->pass_dev_cnt) < 0); 1268 eeh_pe_change_owner(edev->pe); 1269 out: 1270 mutex_unlock(&eeh_dev_mutex); 1271 } 1272 EXPORT_SYMBOL(eeh_dev_release); 1273 1274 #ifdef CONFIG_IOMMU_API 1275 1276 /** 1277 * eeh_iommu_group_to_pe - Convert IOMMU group to EEH PE 1278 * @group: IOMMU group 1279 * 1280 * The routine is called to convert IOMMU group to EEH PE. 1281 */ eeh_iommu_group_to_pe(struct iommu_group * group)1282 struct eeh_pe *eeh_iommu_group_to_pe(struct iommu_group *group) 1283 { 1284 struct pci_dev *pdev = NULL; 1285 struct eeh_dev *edev; 1286 int ret; 1287 1288 /* No IOMMU group ? */ 1289 if (!group) 1290 return NULL; 1291 1292 ret = iommu_group_for_each_dev(group, &pdev, dev_has_iommu_table); 1293 if (!ret || !pdev) 1294 return NULL; 1295 1296 /* No EEH device or PE ? */ 1297 edev = pci_dev_to_eeh_dev(pdev); 1298 if (!edev || !edev->pe) 1299 return NULL; 1300 1301 return edev->pe; 1302 } 1303 EXPORT_SYMBOL_GPL(eeh_iommu_group_to_pe); 1304 1305 #endif /* CONFIG_IOMMU_API */ 1306 1307 /** 1308 * eeh_pe_set_option - Set options for the indicated PE 1309 * @pe: EEH PE 1310 * @option: requested option 1311 * 1312 * The routine is called to enable or disable EEH functionality 1313 * on the indicated PE, to enable IO or DMA for the frozen PE. 1314 */ eeh_pe_set_option(struct eeh_pe * pe,int option)1315 int eeh_pe_set_option(struct eeh_pe *pe, int option) 1316 { 1317 int ret = 0; 1318 1319 /* Invalid PE ? */ 1320 if (!pe) 1321 return -ENODEV; 1322 1323 /* 1324 * EEH functionality could possibly be disabled, just 1325 * return error for the case. And the EEH functionality 1326 * isn't expected to be disabled on one specific PE. 1327 */ 1328 switch (option) { 1329 case EEH_OPT_ENABLE: 1330 if (eeh_enabled()) { 1331 ret = eeh_pe_change_owner(pe); 1332 break; 1333 } 1334 ret = -EIO; 1335 break; 1336 case EEH_OPT_DISABLE: 1337 break; 1338 case EEH_OPT_THAW_MMIO: 1339 case EEH_OPT_THAW_DMA: 1340 case EEH_OPT_FREEZE_PE: 1341 if (!eeh_ops || !eeh_ops->set_option) { 1342 ret = -ENOENT; 1343 break; 1344 } 1345 1346 ret = eeh_pci_enable(pe, option); 1347 break; 1348 default: 1349 pr_debug("%s: Option %d out of range (%d, %d)\n", 1350 __func__, option, EEH_OPT_DISABLE, EEH_OPT_THAW_DMA); 1351 ret = -EINVAL; 1352 } 1353 1354 return ret; 1355 } 1356 EXPORT_SYMBOL_GPL(eeh_pe_set_option); 1357 1358 /** 1359 * eeh_pe_get_state - Retrieve PE's state 1360 * @pe: EEH PE 1361 * 1362 * Retrieve the PE's state, which includes 3 aspects: enabled 1363 * DMA, enabled IO and asserted reset. 1364 */ eeh_pe_get_state(struct eeh_pe * pe)1365 int eeh_pe_get_state(struct eeh_pe *pe) 1366 { 1367 int result, ret = 0; 1368 bool rst_active, dma_en, mmio_en; 1369 1370 /* Existing PE ? */ 1371 if (!pe) 1372 return -ENODEV; 1373 1374 if (!eeh_ops || !eeh_ops->get_state) 1375 return -ENOENT; 1376 1377 /* 1378 * If the parent PE is owned by the host kernel and is undergoing 1379 * error recovery, we should return the PE state as temporarily 1380 * unavailable so that the error recovery on the guest is suspended 1381 * until the recovery completes on the host. 1382 */ 1383 if (pe->parent && 1384 !(pe->state & EEH_PE_REMOVED) && 1385 (pe->parent->state & (EEH_PE_ISOLATED | EEH_PE_RECOVERING))) 1386 return EEH_PE_STATE_UNAVAIL; 1387 1388 result = eeh_ops->get_state(pe, NULL); 1389 rst_active = !!(result & EEH_STATE_RESET_ACTIVE); 1390 dma_en = !!(result & EEH_STATE_DMA_ENABLED); 1391 mmio_en = !!(result & EEH_STATE_MMIO_ENABLED); 1392 1393 if (rst_active) 1394 ret = EEH_PE_STATE_RESET; 1395 else if (dma_en && mmio_en) 1396 ret = EEH_PE_STATE_NORMAL; 1397 else if (!dma_en && !mmio_en) 1398 ret = EEH_PE_STATE_STOPPED_IO_DMA; 1399 else if (!dma_en && mmio_en) 1400 ret = EEH_PE_STATE_STOPPED_DMA; 1401 else 1402 ret = EEH_PE_STATE_UNAVAIL; 1403 1404 return ret; 1405 } 1406 EXPORT_SYMBOL_GPL(eeh_pe_get_state); 1407 eeh_pe_reenable_devices(struct eeh_pe * pe,bool include_passed)1408 static int eeh_pe_reenable_devices(struct eeh_pe *pe, bool include_passed) 1409 { 1410 struct eeh_dev *edev, *tmp; 1411 struct pci_dev *pdev; 1412 int ret = 0; 1413 1414 eeh_pe_restore_bars(pe); 1415 1416 /* 1417 * Reenable PCI devices as the devices passed 1418 * through are always enabled before the reset. 1419 */ 1420 eeh_pe_for_each_dev(pe, edev, tmp) { 1421 pdev = eeh_dev_to_pci_dev(edev); 1422 if (!pdev) 1423 continue; 1424 1425 ret = pci_reenable_device(pdev); 1426 if (ret) { 1427 pr_warn("%s: Failure %d reenabling %s\n", 1428 __func__, ret, pci_name(pdev)); 1429 return ret; 1430 } 1431 } 1432 1433 /* The PE is still in frozen state */ 1434 if (include_passed || !eeh_pe_passed(pe)) { 1435 ret = eeh_unfreeze_pe(pe); 1436 } else 1437 pr_info("EEH: Note: Leaving passthrough PHB#%x-PE#%x frozen.\n", 1438 pe->phb->global_number, pe->addr); 1439 if (!ret) 1440 eeh_pe_state_clear(pe, EEH_PE_ISOLATED, include_passed); 1441 return ret; 1442 } 1443 1444 1445 /** 1446 * eeh_pe_reset - Issue PE reset according to specified type 1447 * @pe: EEH PE 1448 * @option: reset type 1449 * @include_passed: include passed-through devices? 1450 * 1451 * The routine is called to reset the specified PE with the 1452 * indicated type, either fundamental reset or hot reset. 1453 * PE reset is the most important part for error recovery. 1454 */ eeh_pe_reset(struct eeh_pe * pe,int option,bool include_passed)1455 int eeh_pe_reset(struct eeh_pe *pe, int option, bool include_passed) 1456 { 1457 int ret = 0; 1458 1459 /* Invalid PE ? */ 1460 if (!pe) 1461 return -ENODEV; 1462 1463 if (!eeh_ops || !eeh_ops->set_option || !eeh_ops->reset) 1464 return -ENOENT; 1465 1466 switch (option) { 1467 case EEH_RESET_DEACTIVATE: 1468 ret = eeh_ops->reset(pe, option); 1469 eeh_pe_state_clear(pe, EEH_PE_CFG_BLOCKED, include_passed); 1470 if (ret) 1471 break; 1472 1473 ret = eeh_pe_reenable_devices(pe, include_passed); 1474 break; 1475 case EEH_RESET_HOT: 1476 case EEH_RESET_FUNDAMENTAL: 1477 /* 1478 * Proactively freeze the PE to drop all MMIO access 1479 * during reset, which should be banned as it's always 1480 * cause recursive EEH error. 1481 */ 1482 eeh_ops->set_option(pe, EEH_OPT_FREEZE_PE); 1483 1484 eeh_pe_state_mark(pe, EEH_PE_CFG_BLOCKED); 1485 ret = eeh_ops->reset(pe, option); 1486 break; 1487 default: 1488 pr_debug("%s: Unsupported option %d\n", 1489 __func__, option); 1490 ret = -EINVAL; 1491 } 1492 1493 return ret; 1494 } 1495 EXPORT_SYMBOL_GPL(eeh_pe_reset); 1496 1497 /** 1498 * eeh_pe_configure - Configure PCI bridges after PE reset 1499 * @pe: EEH PE 1500 * 1501 * The routine is called to restore the PCI config space for 1502 * those PCI devices, especially PCI bridges affected by PE 1503 * reset issued previously. 1504 */ eeh_pe_configure(struct eeh_pe * pe)1505 int eeh_pe_configure(struct eeh_pe *pe) 1506 { 1507 int ret = 0; 1508 1509 /* Invalid PE ? */ 1510 if (!pe) 1511 return -ENODEV; 1512 1513 return ret; 1514 } 1515 EXPORT_SYMBOL_GPL(eeh_pe_configure); 1516 1517 /** 1518 * eeh_pe_inject_err - Injecting the specified PCI error to the indicated PE 1519 * @pe: the indicated PE 1520 * @type: error type 1521 * @func: error function 1522 * @addr: address 1523 * @mask: address mask 1524 * 1525 * The routine is called to inject the specified PCI error, which 1526 * is determined by @type and @func, to the indicated PE for 1527 * testing purpose. 1528 */ eeh_pe_inject_err(struct eeh_pe * pe,int type,int func,unsigned long addr,unsigned long mask)1529 int eeh_pe_inject_err(struct eeh_pe *pe, int type, int func, 1530 unsigned long addr, unsigned long mask) 1531 { 1532 /* Invalid PE ? */ 1533 if (!pe) 1534 return -ENODEV; 1535 1536 /* Unsupported operation ? */ 1537 if (!eeh_ops || !eeh_ops->err_inject) 1538 return -ENOENT; 1539 1540 /* Check on PCI error function */ 1541 if (func < EEH_ERR_FUNC_MIN || func > EEH_ERR_FUNC_MAX) 1542 return -EINVAL; 1543 1544 return eeh_ops->err_inject(pe, type, func, addr, mask); 1545 } 1546 EXPORT_SYMBOL_GPL(eeh_pe_inject_err); 1547 1548 #ifdef CONFIG_PROC_FS proc_eeh_show(struct seq_file * m,void * v)1549 static int proc_eeh_show(struct seq_file *m, void *v) 1550 { 1551 if (!eeh_enabled()) { 1552 seq_printf(m, "EEH Subsystem is globally disabled\n"); 1553 seq_printf(m, "eeh_total_mmio_ffs=%llu\n", eeh_stats.total_mmio_ffs); 1554 } else { 1555 seq_printf(m, "EEH Subsystem is enabled\n"); 1556 seq_printf(m, 1557 "no device=%llu\n" 1558 "no device node=%llu\n" 1559 "no config address=%llu\n" 1560 "check not wanted=%llu\n" 1561 "eeh_total_mmio_ffs=%llu\n" 1562 "eeh_false_positives=%llu\n" 1563 "eeh_slot_resets=%llu\n", 1564 eeh_stats.no_device, 1565 eeh_stats.no_dn, 1566 eeh_stats.no_cfg_addr, 1567 eeh_stats.ignored_check, 1568 eeh_stats.total_mmio_ffs, 1569 eeh_stats.false_positives, 1570 eeh_stats.slot_resets); 1571 } 1572 1573 return 0; 1574 } 1575 #endif /* CONFIG_PROC_FS */ 1576 eeh_break_device(struct pci_dev * pdev)1577 static int eeh_break_device(struct pci_dev *pdev) 1578 { 1579 struct resource *bar = NULL; 1580 void __iomem *mapped; 1581 u16 old, bit; 1582 int i, pos; 1583 1584 /* Do we have an MMIO BAR to disable? */ 1585 for (i = 0; i <= PCI_STD_RESOURCE_END; i++) { 1586 struct resource *r = &pdev->resource[i]; 1587 1588 if (!r->flags || !r->start) 1589 continue; 1590 if (r->flags & IORESOURCE_IO) 1591 continue; 1592 if (r->flags & IORESOURCE_UNSET) 1593 continue; 1594 1595 bar = r; 1596 break; 1597 } 1598 1599 if (!bar) { 1600 pci_err(pdev, "Unable to find Memory BAR to cause EEH with\n"); 1601 return -ENXIO; 1602 } 1603 1604 pci_err(pdev, "Going to break: %pR\n", bar); 1605 1606 if (pdev->is_virtfn) { 1607 #ifndef CONFIG_PCI_IOV 1608 return -ENXIO; 1609 #else 1610 /* 1611 * VFs don't have a per-function COMMAND register, so the best 1612 * we can do is clear the Memory Space Enable bit in the PF's 1613 * SRIOV control reg. 1614 * 1615 * Unfortunately, this requires that we have a PF (i.e doesn't 1616 * work for a passed-through VF) and it has the potential side 1617 * effect of also causing an EEH on every other VF under the 1618 * PF. Oh well. 1619 */ 1620 pdev = pdev->physfn; 1621 if (!pdev) 1622 return -ENXIO; /* passed through VFs have no PF */ 1623 1624 pos = pci_find_ext_capability(pdev, PCI_EXT_CAP_ID_SRIOV); 1625 pos += PCI_SRIOV_CTRL; 1626 bit = PCI_SRIOV_CTRL_MSE; 1627 #endif /* !CONFIG_PCI_IOV */ 1628 } else { 1629 bit = PCI_COMMAND_MEMORY; 1630 pos = PCI_COMMAND; 1631 } 1632 1633 /* 1634 * Process here is: 1635 * 1636 * 1. Disable Memory space. 1637 * 1638 * 2. Perform an MMIO to the device. This should result in an error 1639 * (CA / UR) being raised by the device which results in an EEH 1640 * PE freeze. Using the in_8() accessor skips the eeh detection hook 1641 * so the freeze hook so the EEH Detection machinery won't be 1642 * triggered here. This is to match the usual behaviour of EEH 1643 * where the HW will asynchronously freeze a PE and it's up to 1644 * the kernel to notice and deal with it. 1645 * 1646 * 3. Turn Memory space back on. This is more important for VFs 1647 * since recovery will probably fail if we don't. For normal 1648 * the COMMAND register is reset as a part of re-initialising 1649 * the device. 1650 * 1651 * Breaking stuff is the point so who cares if it's racy ;) 1652 */ 1653 pci_read_config_word(pdev, pos, &old); 1654 1655 mapped = ioremap(bar->start, PAGE_SIZE); 1656 if (!mapped) { 1657 pci_err(pdev, "Unable to map MMIO BAR %pR\n", bar); 1658 return -ENXIO; 1659 } 1660 1661 pci_write_config_word(pdev, pos, old & ~bit); 1662 in_8(mapped); 1663 pci_write_config_word(pdev, pos, old); 1664 1665 iounmap(mapped); 1666 1667 return 0; 1668 } 1669 eeh_pe_inject_mmio_error(struct pci_dev * pdev)1670 int eeh_pe_inject_mmio_error(struct pci_dev *pdev) 1671 { 1672 return eeh_break_device(pdev); 1673 } 1674 1675 #ifdef CONFIG_DEBUG_FS 1676 1677 eeh_debug_lookup_pdev(struct file * filp,const char __user * user_buf,size_t count,loff_t * ppos)1678 static struct pci_dev *eeh_debug_lookup_pdev(struct file *filp, 1679 const char __user *user_buf, 1680 size_t count, loff_t *ppos) 1681 { 1682 uint32_t domain, bus, dev, fn; 1683 struct pci_dev *pdev; 1684 char buf[20]; 1685 int ret; 1686 1687 memset(buf, 0, sizeof(buf)); 1688 ret = simple_write_to_buffer(buf, sizeof(buf)-1, ppos, user_buf, count); 1689 if (!ret) 1690 return ERR_PTR(-EFAULT); 1691 1692 ret = sscanf(buf, "%x:%x:%x.%x", &domain, &bus, &dev, &fn); 1693 if (ret != 4) { 1694 pr_err("%s: expected 4 args, got %d\n", __func__, ret); 1695 return ERR_PTR(-EINVAL); 1696 } 1697 1698 pdev = pci_get_domain_bus_and_slot(domain, bus, (dev << 3) | fn); 1699 if (!pdev) 1700 return ERR_PTR(-ENODEV); 1701 1702 return pdev; 1703 } 1704 eeh_enable_dbgfs_set(void * data,u64 val)1705 static int eeh_enable_dbgfs_set(void *data, u64 val) 1706 { 1707 if (val) 1708 eeh_clear_flag(EEH_FORCE_DISABLED); 1709 else 1710 eeh_add_flag(EEH_FORCE_DISABLED); 1711 1712 return 0; 1713 } 1714 eeh_enable_dbgfs_get(void * data,u64 * val)1715 static int eeh_enable_dbgfs_get(void *data, u64 *val) 1716 { 1717 if (eeh_enabled()) 1718 *val = 0x1ul; 1719 else 1720 *val = 0x0ul; 1721 return 0; 1722 } 1723 1724 DEFINE_DEBUGFS_ATTRIBUTE(eeh_enable_dbgfs_ops, eeh_enable_dbgfs_get, 1725 eeh_enable_dbgfs_set, "0x%llx\n"); 1726 eeh_force_recover_write(struct file * filp,const char __user * user_buf,size_t count,loff_t * ppos)1727 static ssize_t eeh_force_recover_write(struct file *filp, 1728 const char __user *user_buf, 1729 size_t count, loff_t *ppos) 1730 { 1731 struct pci_controller *hose; 1732 uint32_t phbid, pe_no; 1733 struct eeh_pe *pe; 1734 char buf[20]; 1735 int ret; 1736 1737 ret = simple_write_to_buffer(buf, sizeof(buf), ppos, user_buf, count); 1738 if (!ret) 1739 return -EFAULT; 1740 1741 /* 1742 * When PE is NULL the event is a "special" event. Rather than 1743 * recovering a specific PE it forces the EEH core to scan for failed 1744 * PHBs and recovers each. This needs to be done before any device 1745 * recoveries can occur. 1746 */ 1747 if (!strncmp(buf, "hwcheck", 7)) { 1748 __eeh_send_failure_event(NULL); 1749 return count; 1750 } 1751 1752 ret = sscanf(buf, "%x:%x", &phbid, &pe_no); 1753 if (ret != 2) 1754 return -EINVAL; 1755 1756 hose = pci_find_controller_for_domain(phbid); 1757 if (!hose) 1758 return -ENODEV; 1759 1760 /* Retrieve PE */ 1761 pe = eeh_pe_get(hose, pe_no); 1762 if (!pe) 1763 return -ENODEV; 1764 1765 /* 1766 * We don't do any state checking here since the detection 1767 * process is async to the recovery process. The recovery 1768 * thread *should* not break even if we schedule a recovery 1769 * from an odd state (e.g. PE removed, or recovery of a 1770 * non-isolated PE) 1771 */ 1772 __eeh_send_failure_event(pe); 1773 1774 return ret < 0 ? ret : count; 1775 } 1776 1777 static const struct file_operations eeh_force_recover_fops = { 1778 .open = simple_open, 1779 .write = eeh_force_recover_write, 1780 }; 1781 eeh_debugfs_dev_usage(struct file * filp,char __user * user_buf,size_t count,loff_t * ppos)1782 static ssize_t eeh_debugfs_dev_usage(struct file *filp, 1783 char __user *user_buf, 1784 size_t count, loff_t *ppos) 1785 { 1786 static const char usage[] = "input format: <domain>:<bus>:<dev>.<fn>\n"; 1787 1788 return simple_read_from_buffer(user_buf, count, ppos, 1789 usage, sizeof(usage) - 1); 1790 } 1791 eeh_dev_check_write(struct file * filp,const char __user * user_buf,size_t count,loff_t * ppos)1792 static ssize_t eeh_dev_check_write(struct file *filp, 1793 const char __user *user_buf, 1794 size_t count, loff_t *ppos) 1795 { 1796 struct pci_dev *pdev; 1797 struct eeh_dev *edev; 1798 int ret; 1799 1800 pdev = eeh_debug_lookup_pdev(filp, user_buf, count, ppos); 1801 if (IS_ERR(pdev)) 1802 return PTR_ERR(pdev); 1803 1804 edev = pci_dev_to_eeh_dev(pdev); 1805 if (!edev) { 1806 pci_err(pdev, "No eeh_dev for this device!\n"); 1807 pci_dev_put(pdev); 1808 return -ENODEV; 1809 } 1810 1811 ret = eeh_dev_check_failure(edev); 1812 pci_info(pdev, "eeh_dev_check_failure(%s) = %d\n", 1813 pci_name(pdev), ret); 1814 1815 pci_dev_put(pdev); 1816 1817 return count; 1818 } 1819 1820 static const struct file_operations eeh_dev_check_fops = { 1821 .open = simple_open, 1822 .write = eeh_dev_check_write, 1823 .read = eeh_debugfs_dev_usage, 1824 }; 1825 eeh_dev_break_write(struct file * filp,const char __user * user_buf,size_t count,loff_t * ppos)1826 static ssize_t eeh_dev_break_write(struct file *filp, 1827 const char __user *user_buf, 1828 size_t count, loff_t *ppos) 1829 { 1830 struct pci_dev *pdev; 1831 int ret; 1832 1833 pdev = eeh_debug_lookup_pdev(filp, user_buf, count, ppos); 1834 if (IS_ERR(pdev)) 1835 return PTR_ERR(pdev); 1836 1837 ret = eeh_break_device(pdev); 1838 pci_dev_put(pdev); 1839 1840 if (ret < 0) 1841 return ret; 1842 1843 return count; 1844 } 1845 1846 static const struct file_operations eeh_dev_break_fops = { 1847 .open = simple_open, 1848 .write = eeh_dev_break_write, 1849 .read = eeh_debugfs_dev_usage, 1850 }; 1851 eeh_dev_can_recover(struct file * filp,const char __user * user_buf,size_t count,loff_t * ppos)1852 static ssize_t eeh_dev_can_recover(struct file *filp, 1853 const char __user *user_buf, 1854 size_t count, loff_t *ppos) 1855 { 1856 struct pci_driver *drv; 1857 struct pci_dev *pdev; 1858 size_t ret; 1859 1860 pdev = eeh_debug_lookup_pdev(filp, user_buf, count, ppos); 1861 if (IS_ERR(pdev)) 1862 return PTR_ERR(pdev); 1863 1864 /* 1865 * In order for error recovery to work the driver needs to implement 1866 * .error_detected(), so it can quiesce IO to the device, and 1867 * .slot_reset() so it can re-initialise the device after a reset. 1868 * 1869 * Ideally they'd implement .resume() too, but some drivers which 1870 * we need to support (notably IPR) don't so I guess we can tolerate 1871 * that. 1872 * 1873 * .mmio_enabled() is mostly there as a work-around for devices which 1874 * take forever to re-init after a hot reset. Implementing that is 1875 * strictly optional. 1876 */ 1877 drv = pci_dev_driver(pdev); 1878 if (drv && 1879 drv->err_handler && 1880 drv->err_handler->error_detected && 1881 drv->err_handler->slot_reset) { 1882 ret = count; 1883 } else { 1884 ret = -EOPNOTSUPP; 1885 } 1886 1887 pci_dev_put(pdev); 1888 1889 return ret; 1890 } 1891 1892 static const struct file_operations eeh_dev_can_recover_fops = { 1893 .open = simple_open, 1894 .write = eeh_dev_can_recover, 1895 .read = eeh_debugfs_dev_usage, 1896 }; 1897 1898 #endif 1899 eeh_init_proc(void)1900 static int __init eeh_init_proc(void) 1901 { 1902 if (machine_is(pseries) || machine_is(powernv)) { 1903 proc_create_single("powerpc/eeh", 0, NULL, proc_eeh_show); 1904 #ifdef CONFIG_DEBUG_FS 1905 debugfs_create_file_unsafe("eeh_enable", 0600, 1906 arch_debugfs_dir, NULL, 1907 &eeh_enable_dbgfs_ops); 1908 debugfs_create_u32("eeh_max_freezes", 0600, 1909 arch_debugfs_dir, &eeh_max_freezes); 1910 debugfs_create_bool("eeh_disable_recovery", 0600, 1911 arch_debugfs_dir, 1912 &eeh_debugfs_no_recover); 1913 debugfs_create_file_unsafe("eeh_dev_check", 0600, 1914 arch_debugfs_dir, NULL, 1915 &eeh_dev_check_fops); 1916 debugfs_create_file_unsafe("eeh_dev_break", 0600, 1917 arch_debugfs_dir, NULL, 1918 &eeh_dev_break_fops); 1919 debugfs_create_file_unsafe("eeh_force_recover", 0600, 1920 arch_debugfs_dir, NULL, 1921 &eeh_force_recover_fops); 1922 debugfs_create_file_unsafe("eeh_dev_can_recover", 0600, 1923 arch_debugfs_dir, NULL, 1924 &eeh_dev_can_recover_fops); 1925 eeh_cache_debugfs_init(); 1926 #endif 1927 } 1928 1929 return 0; 1930 } 1931 __initcall(eeh_init_proc); 1932