Lines Matching +full:pcie +full:- +full:mirror
35 #include <linux/pci-p2pdma.h>
36 #include <linux/apple-gmux.h>
86 #include <asm/intel-family.h>
99 #define AMDGPU_RETRY_SRIOV_RESET(r) ((r) == -EBUSY || (r) == -ETIMEDOUT || (r) == -EINVAL)
153 * of PCIe replays (NAKs)
202 return -EINVAL; in amdgpu_sysfs_reg_state_get()
218 ret = sysfs_create_bin_file(&adev->dev->kobj, &bin_attr_reg_state); in amdgpu_reg_state_sysfs_init()
227 sysfs_remove_bin_file(&adev->dev->kobj, &bin_attr_reg_state); in amdgpu_reg_state_sysfs_fini()
240 * - "cem" - PCIE CEM card
241 * - "oam" - Open Compute Accelerator Module
242 * - "unknown" - Not known
255 if (adev->smuio.funcs && adev->smuio.funcs->get_pkg_type) in amdgpu_device_get_board_info()
256 pkg_type = adev->smuio.funcs->get_pkg_type(adev); in amdgpu_device_get_board_info()
287 if (adev->flags & AMD_IS_APU) in amdgpu_board_attrs_is_visible()
290 return attr->mode; in amdgpu_board_attrs_is_visible()
302 * amdgpu_device_supports_px - Is the device a dGPU with ATPX power control
313 if ((adev->flags & AMD_IS_PX) && !amdgpu_is_atpx_hybrid()) in amdgpu_device_supports_px()
319 * amdgpu_device_supports_boco - Is the device a dGPU with ACPI power resources
330 if (adev->has_pr3 || in amdgpu_device_supports_boco()
331 ((adev->flags & AMD_IS_PX) && amdgpu_is_atpx_hybrid())) in amdgpu_device_supports_boco()
337 * amdgpu_device_supports_baco - Does the device support BACO
360 adev->pm.rpm_mode = AMDGPU_RUNPM_NONE; in amdgpu_device_detect_runtime_pm_mode()
366 adev->pm.rpm_mode = AMDGPU_RUNPM_BAMACO; in amdgpu_device_detect_runtime_pm_mode()
367 dev_info(adev->dev, "Forcing BAMACO for runtime pm\n"); in amdgpu_device_detect_runtime_pm_mode()
369 adev->pm.rpm_mode = AMDGPU_RUNPM_BACO; in amdgpu_device_detect_runtime_pm_mode()
370 dev_info(adev->dev, "Requested mode BAMACO not available,fallback to use BACO\n"); in amdgpu_device_detect_runtime_pm_mode()
375 adev->pm.rpm_mode = AMDGPU_RUNPM_BACO; in amdgpu_device_detect_runtime_pm_mode()
376 dev_info(adev->dev, "Forcing BACO for runtime pm\n"); in amdgpu_device_detect_runtime_pm_mode()
379 case -1: in amdgpu_device_detect_runtime_pm_mode()
380 case -2: in amdgpu_device_detect_runtime_pm_mode()
382 adev->pm.rpm_mode = AMDGPU_RUNPM_PX; in amdgpu_device_detect_runtime_pm_mode()
383 dev_info(adev->dev, "Using ATPX for runtime pm\n"); in amdgpu_device_detect_runtime_pm_mode()
385 adev->pm.rpm_mode = AMDGPU_RUNPM_BOCO; in amdgpu_device_detect_runtime_pm_mode()
386 dev_info(adev->dev, "Using BOCO for runtime pm\n"); in amdgpu_device_detect_runtime_pm_mode()
391 switch (adev->asic_type) { in amdgpu_device_detect_runtime_pm_mode()
398 if (!adev->gmc.noretry) in amdgpu_device_detect_runtime_pm_mode()
399 adev->pm.rpm_mode = AMDGPU_RUNPM_BACO; in amdgpu_device_detect_runtime_pm_mode()
403 adev->pm.rpm_mode = AMDGPU_RUNPM_BACO; in amdgpu_device_detect_runtime_pm_mode()
407 if (adev->pm.rpm_mode == AMDGPU_RUNPM_BACO) { in amdgpu_device_detect_runtime_pm_mode()
409 adev->pm.rpm_mode = AMDGPU_RUNPM_BAMACO; in amdgpu_device_detect_runtime_pm_mode()
410 dev_info(adev->dev, "Using BAMACO for runtime pm\n"); in amdgpu_device_detect_runtime_pm_mode()
412 dev_info(adev->dev, "Using BACO for runtime pm\n"); in amdgpu_device_detect_runtime_pm_mode()
418 dev_info(adev->dev, "runtime pm is manually disabled\n"); in amdgpu_device_detect_runtime_pm_mode()
425 if (adev->pm.rpm_mode == AMDGPU_RUNPM_NONE) in amdgpu_device_detect_runtime_pm_mode()
426 dev_info(adev->dev, "Runtime PM not available\n"); in amdgpu_device_detect_runtime_pm_mode()
429 * amdgpu_device_supports_smart_shift - Is the device dGPU with
448 * amdgpu_device_mm_access - access vram by MM_INDEX/MM_DATA
454 * @write: true - write to vram, otherwise - read from vram
470 spin_lock_irqsave(&adev->mmio_idx_lock, flags); in amdgpu_device_mm_access()
485 spin_unlock_irqrestore(&adev->mmio_idx_lock, flags); in amdgpu_device_mm_access()
490 * amdgpu_device_aper_access - access vram by vram aperature
496 * @write: true - write to vram, otherwise - read from vram
508 if (!adev->mman.aper_base_kaddr) in amdgpu_device_aper_access()
511 last = min(pos + size, adev->gmc.visible_vram_size); in amdgpu_device_aper_access()
513 addr = adev->mman.aper_base_kaddr + pos; in amdgpu_device_aper_access()
514 count = last - pos; in amdgpu_device_aper_access()
519 * after the system memory contents are sent over PCIe device in amdgpu_device_aper_access()
526 * to the PCIe device in amdgpu_device_aper_access()
541 * amdgpu_device_vram_access - read/write a buffer in vram
547 * @write: true - write to vram, otherwise - read from vram
556 size -= count; in amdgpu_device_vram_access()
572 if (adev->no_hw_access) in amdgpu_device_skip_hw_access()
588 if (down_read_trylock(&adev->reset_domain->sem)) in amdgpu_device_skip_hw_access()
589 up_read(&adev->reset_domain->sem); in amdgpu_device_skip_hw_access()
591 lockdep_assert_held(&adev->reset_domain->sem); in amdgpu_device_skip_hw_access()
598 * amdgpu_device_rreg - read a memory mapped IO or indirect register
614 if ((reg * 4) < adev->rmmio_size) { in amdgpu_device_rreg()
617 down_read_trylock(&adev->reset_domain->sem)) { in amdgpu_device_rreg()
619 up_read(&adev->reset_domain->sem); in amdgpu_device_rreg()
621 ret = readl(((void __iomem *)adev->rmmio) + (reg * 4)); in amdgpu_device_rreg()
624 ret = adev->pcie_rreg(adev, reg * 4); in amdgpu_device_rreg()
627 trace_amdgpu_device_rreg(adev->pdev->device, reg, ret); in amdgpu_device_rreg()
638 * amdgpu_mm_rreg8 - read a memory mapped IO register
650 if (offset < adev->rmmio_size) in amdgpu_mm_rreg8()
651 return (readb(adev->rmmio + offset)); in amdgpu_mm_rreg8()
657 * amdgpu_device_xcc_rreg - read a memory mapped IO or indirect register with specific XCC
675 if ((reg * 4) < adev->rmmio_size) { in amdgpu_device_xcc_rreg()
678 adev->gfx.rlc.rlcg_reg_access_supported && in amdgpu_device_xcc_rreg()
685 down_read_trylock(&adev->reset_domain->sem)) { in amdgpu_device_xcc_rreg()
687 up_read(&adev->reset_domain->sem); in amdgpu_device_xcc_rreg()
689 ret = readl(((void __iomem *)adev->rmmio) + (reg * 4)); in amdgpu_device_xcc_rreg()
692 ret = adev->pcie_rreg(adev, reg * 4); in amdgpu_device_xcc_rreg()
705 * amdgpu_mm_wreg8 - read a memory mapped IO register
718 if (offset < adev->rmmio_size) in amdgpu_mm_wreg8()
719 writeb(value, adev->rmmio + offset); in amdgpu_mm_wreg8()
725 * amdgpu_device_wreg - write to a memory mapped IO or indirect register
741 if ((reg * 4) < adev->rmmio_size) { in amdgpu_device_wreg()
744 down_read_trylock(&adev->reset_domain->sem)) { in amdgpu_device_wreg()
746 up_read(&adev->reset_domain->sem); in amdgpu_device_wreg()
748 writel(v, ((void __iomem *)adev->rmmio) + (reg * 4)); in amdgpu_device_wreg()
751 adev->pcie_wreg(adev, reg * 4, v); in amdgpu_device_wreg()
754 trace_amdgpu_device_wreg(adev->pdev->device, reg, v); in amdgpu_device_wreg()
758 …* amdgpu_mm_wreg_mmio_rlc - write register either with direct/indirect mmio or with RLC path if i…
775 adev->gfx.rlc.funcs && in amdgpu_mm_wreg_mmio_rlc()
776 adev->gfx.rlc.funcs->is_rlcg_access_range) { in amdgpu_mm_wreg_mmio_rlc()
777 if (adev->gfx.rlc.funcs->is_rlcg_access_range(adev, reg)) in amdgpu_mm_wreg_mmio_rlc()
779 } else if ((reg * 4) >= adev->rmmio_size) { in amdgpu_mm_wreg_mmio_rlc()
780 adev->pcie_wreg(adev, reg * 4, v); in amdgpu_mm_wreg_mmio_rlc()
782 writel(v, ((void __iomem *)adev->rmmio) + (reg * 4)); in amdgpu_mm_wreg_mmio_rlc()
787 * amdgpu_device_xcc_wreg - write to a memory mapped IO or indirect register with specific XCC
806 if ((reg * 4) < adev->rmmio_size) { in amdgpu_device_xcc_wreg()
809 adev->gfx.rlc.rlcg_reg_access_supported && in amdgpu_device_xcc_wreg()
816 down_read_trylock(&adev->reset_domain->sem)) { in amdgpu_device_xcc_wreg()
818 up_read(&adev->reset_domain->sem); in amdgpu_device_xcc_wreg()
820 writel(v, ((void __iomem *)adev->rmmio) + (reg * 4)); in amdgpu_device_xcc_wreg()
823 adev->pcie_wreg(adev, reg * 4, v); in amdgpu_device_xcc_wreg()
828 * amdgpu_device_indirect_rreg - read an indirect register
843 pcie_index = adev->nbio.funcs->get_pcie_index_offset(adev); in amdgpu_device_indirect_rreg()
844 pcie_data = adev->nbio.funcs->get_pcie_data_offset(adev); in amdgpu_device_indirect_rreg()
846 spin_lock_irqsave(&adev->pcie_idx_lock, flags); in amdgpu_device_indirect_rreg()
847 pcie_index_offset = (void __iomem *)adev->rmmio + pcie_index * 4; in amdgpu_device_indirect_rreg()
848 pcie_data_offset = (void __iomem *)adev->rmmio + pcie_data * 4; in amdgpu_device_indirect_rreg()
853 spin_unlock_irqrestore(&adev->pcie_idx_lock, flags); in amdgpu_device_indirect_rreg()
867 if (unlikely(!adev->nbio.funcs)) { in amdgpu_device_indirect_rreg_ext()
871 pcie_index = adev->nbio.funcs->get_pcie_index_offset(adev); in amdgpu_device_indirect_rreg_ext()
872 pcie_data = adev->nbio.funcs->get_pcie_data_offset(adev); in amdgpu_device_indirect_rreg_ext()
876 if (unlikely(!adev->nbio.funcs)) in amdgpu_device_indirect_rreg_ext()
879 pcie_index_hi = adev->nbio.funcs->get_pcie_index_hi_offset(adev); in amdgpu_device_indirect_rreg_ext()
884 spin_lock_irqsave(&adev->pcie_idx_lock, flags); in amdgpu_device_indirect_rreg_ext()
885 pcie_index_offset = (void __iomem *)adev->rmmio + pcie_index * 4; in amdgpu_device_indirect_rreg_ext()
886 pcie_data_offset = (void __iomem *)adev->rmmio + pcie_data * 4; in amdgpu_device_indirect_rreg_ext()
888 pcie_index_hi_offset = (void __iomem *)adev->rmmio + in amdgpu_device_indirect_rreg_ext()
905 spin_unlock_irqrestore(&adev->pcie_idx_lock, flags); in amdgpu_device_indirect_rreg_ext()
911 * amdgpu_device_indirect_rreg64 - read a 64bits indirect register
926 pcie_index = adev->nbio.funcs->get_pcie_index_offset(adev); in amdgpu_device_indirect_rreg64()
927 pcie_data = adev->nbio.funcs->get_pcie_data_offset(adev); in amdgpu_device_indirect_rreg64()
929 spin_lock_irqsave(&adev->pcie_idx_lock, flags); in amdgpu_device_indirect_rreg64()
930 pcie_index_offset = (void __iomem *)adev->rmmio + pcie_index * 4; in amdgpu_device_indirect_rreg64()
931 pcie_data_offset = (void __iomem *)adev->rmmio + pcie_data * 4; in amdgpu_device_indirect_rreg64()
941 spin_unlock_irqrestore(&adev->pcie_idx_lock, flags); in amdgpu_device_indirect_rreg64()
956 pcie_index = adev->nbio.funcs->get_pcie_index_offset(adev); in amdgpu_device_indirect_rreg64_ext()
957 pcie_data = adev->nbio.funcs->get_pcie_data_offset(adev); in amdgpu_device_indirect_rreg64_ext()
958 if ((reg_addr >> 32) && (adev->nbio.funcs->get_pcie_index_hi_offset)) in amdgpu_device_indirect_rreg64_ext()
959 pcie_index_hi = adev->nbio.funcs->get_pcie_index_hi_offset(adev); in amdgpu_device_indirect_rreg64_ext()
961 spin_lock_irqsave(&adev->pcie_idx_lock, flags); in amdgpu_device_indirect_rreg64_ext()
962 pcie_index_offset = (void __iomem *)adev->rmmio + pcie_index * 4; in amdgpu_device_indirect_rreg64_ext()
963 pcie_data_offset = (void __iomem *)adev->rmmio + pcie_data * 4; in amdgpu_device_indirect_rreg64_ext()
965 pcie_index_hi_offset = (void __iomem *)adev->rmmio + in amdgpu_device_indirect_rreg64_ext()
991 spin_unlock_irqrestore(&adev->pcie_idx_lock, flags); in amdgpu_device_indirect_rreg64_ext()
997 * amdgpu_device_indirect_wreg - write an indirect register address
1011 pcie_index = adev->nbio.funcs->get_pcie_index_offset(adev); in amdgpu_device_indirect_wreg()
1012 pcie_data = adev->nbio.funcs->get_pcie_data_offset(adev); in amdgpu_device_indirect_wreg()
1014 spin_lock_irqsave(&adev->pcie_idx_lock, flags); in amdgpu_device_indirect_wreg()
1015 pcie_index_offset = (void __iomem *)adev->rmmio + pcie_index * 4; in amdgpu_device_indirect_wreg()
1016 pcie_data_offset = (void __iomem *)adev->rmmio + pcie_data * 4; in amdgpu_device_indirect_wreg()
1022 spin_unlock_irqrestore(&adev->pcie_idx_lock, flags); in amdgpu_device_indirect_wreg()
1033 pcie_index = adev->nbio.funcs->get_pcie_index_offset(adev); in amdgpu_device_indirect_wreg_ext()
1034 pcie_data = adev->nbio.funcs->get_pcie_data_offset(adev); in amdgpu_device_indirect_wreg_ext()
1035 if ((reg_addr >> 32) && (adev->nbio.funcs->get_pcie_index_hi_offset)) in amdgpu_device_indirect_wreg_ext()
1036 pcie_index_hi = adev->nbio.funcs->get_pcie_index_hi_offset(adev); in amdgpu_device_indirect_wreg_ext()
1040 spin_lock_irqsave(&adev->pcie_idx_lock, flags); in amdgpu_device_indirect_wreg_ext()
1041 pcie_index_offset = (void __iomem *)adev->rmmio + pcie_index * 4; in amdgpu_device_indirect_wreg_ext()
1042 pcie_data_offset = (void __iomem *)adev->rmmio + pcie_data * 4; in amdgpu_device_indirect_wreg_ext()
1044 pcie_index_hi_offset = (void __iomem *)adev->rmmio + in amdgpu_device_indirect_wreg_ext()
1062 spin_unlock_irqrestore(&adev->pcie_idx_lock, flags); in amdgpu_device_indirect_wreg_ext()
1066 * amdgpu_device_indirect_wreg64 - write a 64bits indirect register address
1080 pcie_index = adev->nbio.funcs->get_pcie_index_offset(adev); in amdgpu_device_indirect_wreg64()
1081 pcie_data = adev->nbio.funcs->get_pcie_data_offset(adev); in amdgpu_device_indirect_wreg64()
1083 spin_lock_irqsave(&adev->pcie_idx_lock, flags); in amdgpu_device_indirect_wreg64()
1084 pcie_index_offset = (void __iomem *)adev->rmmio + pcie_index * 4; in amdgpu_device_indirect_wreg64()
1085 pcie_data_offset = (void __iomem *)adev->rmmio + pcie_data * 4; in amdgpu_device_indirect_wreg64()
1097 spin_unlock_irqrestore(&adev->pcie_idx_lock, flags); in amdgpu_device_indirect_wreg64()
1109 pcie_index = adev->nbio.funcs->get_pcie_index_offset(adev); in amdgpu_device_indirect_wreg64_ext()
1110 pcie_data = adev->nbio.funcs->get_pcie_data_offset(adev); in amdgpu_device_indirect_wreg64_ext()
1111 if ((reg_addr >> 32) && (adev->nbio.funcs->get_pcie_index_hi_offset)) in amdgpu_device_indirect_wreg64_ext()
1112 pcie_index_hi = adev->nbio.funcs->get_pcie_index_hi_offset(adev); in amdgpu_device_indirect_wreg64_ext()
1114 spin_lock_irqsave(&adev->pcie_idx_lock, flags); in amdgpu_device_indirect_wreg64_ext()
1115 pcie_index_offset = (void __iomem *)adev->rmmio + pcie_index * 4; in amdgpu_device_indirect_wreg64_ext()
1116 pcie_data_offset = (void __iomem *)adev->rmmio + pcie_data * 4; in amdgpu_device_indirect_wreg64_ext()
1118 pcie_index_hi_offset = (void __iomem *)adev->rmmio + in amdgpu_device_indirect_wreg64_ext()
1146 spin_unlock_irqrestore(&adev->pcie_idx_lock, flags); in amdgpu_device_indirect_wreg64_ext()
1150 * amdgpu_device_get_rev_id - query device rev_id
1158 return adev->nbio.funcs->get_rev_id(adev); in amdgpu_device_get_rev_id()
1162 * amdgpu_invalid_rreg - dummy reg read function
1186 * amdgpu_invalid_wreg - dummy reg write function
1210 * amdgpu_invalid_rreg64 - dummy 64 bit reg read function
1234 * amdgpu_invalid_wreg64 - dummy reg write function
1258 * amdgpu_block_invalid_rreg - dummy reg read function
1278 * amdgpu_block_invalid_wreg - dummy reg write function
1298 * amdgpu_device_asic_init - Wrapper for atom asic_init
1317 return amdgpu_atom_asic_init(adev->mode_info.atom_context); in amdgpu_device_asic_init()
1324 * amdgpu_device_mem_scratch_init - allocate the VRAM scratch page
1336 &adev->mem_scratch.robj, in amdgpu_device_mem_scratch_init()
1337 &adev->mem_scratch.gpu_addr, in amdgpu_device_mem_scratch_init()
1338 (void **)&adev->mem_scratch.ptr); in amdgpu_device_mem_scratch_init()
1342 * amdgpu_device_mem_scratch_fini - Free the VRAM scratch page
1350 amdgpu_bo_free_kernel(&adev->mem_scratch.robj, NULL, NULL); in amdgpu_device_mem_scratch_fini()
1354 * amdgpu_device_program_register_sequence - program an array of registers.
1383 if (adev->family >= AMDGPU_FAMILY_AI) in amdgpu_device_program_register_sequence()
1393 * amdgpu_device_pci_config_reset - reset the GPU
1402 pci_write_config_dword(adev->pdev, 0x7c, AMDGPU_ASIC_RESET_DATA); in amdgpu_device_pci_config_reset()
1406 * amdgpu_device_pci_reset - reset the GPU using generic PCI means
1414 return pci_reset_function(adev->pdev); in amdgpu_device_pci_reset()
1424 * amdgpu_device_wb_fini - Disable Writeback and free memory
1433 if (adev->wb.wb_obj) { in amdgpu_device_wb_fini()
1434 amdgpu_bo_free_kernel(&adev->wb.wb_obj, in amdgpu_device_wb_fini()
1435 &adev->wb.gpu_addr, in amdgpu_device_wb_fini()
1436 (void **)&adev->wb.wb); in amdgpu_device_wb_fini()
1437 adev->wb.wb_obj = NULL; in amdgpu_device_wb_fini()
1442 * amdgpu_device_wb_init - Init Writeback driver info and allocate memory
1448 * Returns 0 on success or an -error on failure.
1454 if (adev->wb.wb_obj == NULL) { in amdgpu_device_wb_init()
1458 &adev->wb.wb_obj, &adev->wb.gpu_addr, in amdgpu_device_wb_init()
1459 (void **)&adev->wb.wb); in amdgpu_device_wb_init()
1461 dev_warn(adev->dev, "(%d) create WB bo failed\n", r); in amdgpu_device_wb_init()
1465 adev->wb.num_wb = AMDGPU_MAX_WB; in amdgpu_device_wb_init()
1466 memset(&adev->wb.used, 0, sizeof(adev->wb.used)); in amdgpu_device_wb_init()
1469 memset((char *)adev->wb.wb, 0, AMDGPU_MAX_WB * sizeof(uint32_t) * 8); in amdgpu_device_wb_init()
1476 * amdgpu_device_wb_get - Allocate a wb entry
1482 * Returns 0 on success or -EINVAL on failure.
1488 spin_lock_irqsave(&adev->wb.lock, flags); in amdgpu_device_wb_get()
1489 offset = find_first_zero_bit(adev->wb.used, adev->wb.num_wb); in amdgpu_device_wb_get()
1490 if (offset < adev->wb.num_wb) { in amdgpu_device_wb_get()
1491 __set_bit(offset, adev->wb.used); in amdgpu_device_wb_get()
1492 spin_unlock_irqrestore(&adev->wb.lock, flags); in amdgpu_device_wb_get()
1496 spin_unlock_irqrestore(&adev->wb.lock, flags); in amdgpu_device_wb_get()
1497 return -EINVAL; in amdgpu_device_wb_get()
1502 * amdgpu_device_wb_free - Free a wb entry
1514 spin_lock_irqsave(&adev->wb.lock, flags); in amdgpu_device_wb_free()
1515 if (wb < adev->wb.num_wb) in amdgpu_device_wb_free()
1516 __clear_bit(wb, adev->wb.used); in amdgpu_device_wb_free()
1517 spin_unlock_irqrestore(&adev->wb.lock, flags); in amdgpu_device_wb_free()
1521 * amdgpu_device_resize_fb_bar - try to resize FB BAR
1527 * driver loading by returning -ENODEV.
1531 int rbar_size = pci_rebar_bytes_to_size(adev->gmc.real_vram_size); in amdgpu_device_resize_fb_bar()
1546 if (!pci_find_ext_capability(adev->pdev, PCI_EXT_CAP_ID_VNDR)) in amdgpu_device_resize_fb_bar()
1550 if (adev->gmc.real_vram_size && in amdgpu_device_resize_fb_bar()
1551 (pci_resource_len(adev->pdev, 0) >= adev->gmc.real_vram_size)) in amdgpu_device_resize_fb_bar()
1555 root = adev->pdev->bus; in amdgpu_device_resize_fb_bar()
1556 while (root->parent) in amdgpu_device_resize_fb_bar()
1557 root = root->parent; in amdgpu_device_resize_fb_bar()
1560 if (res && res->flags & (IORESOURCE_MEM | IORESOURCE_MEM_64) && in amdgpu_device_resize_fb_bar()
1561 res->start > 0x100000000ull) in amdgpu_device_resize_fb_bar()
1570 rbar_size = min(fls(pci_rebar_get_possible_sizes(adev->pdev, 0)) - 1, in amdgpu_device_resize_fb_bar()
1574 pci_read_config_word(adev->pdev, PCI_COMMAND, &cmd); in amdgpu_device_resize_fb_bar()
1575 pci_write_config_word(adev->pdev, PCI_COMMAND, in amdgpu_device_resize_fb_bar()
1580 if (adev->asic_type >= CHIP_BONAIRE) in amdgpu_device_resize_fb_bar()
1581 pci_release_resource(adev->pdev, 2); in amdgpu_device_resize_fb_bar()
1583 pci_release_resource(adev->pdev, 0); in amdgpu_device_resize_fb_bar()
1585 r = pci_resize_resource(adev->pdev, 0, rbar_size); in amdgpu_device_resize_fb_bar()
1586 if (r == -ENOSPC) in amdgpu_device_resize_fb_bar()
1588 else if (r && r != -ENOTSUPP) in amdgpu_device_resize_fb_bar()
1591 pci_assign_unassigned_bus_resources(adev->pdev->bus); in amdgpu_device_resize_fb_bar()
1597 if (r || (pci_resource_flags(adev->pdev, 0) & IORESOURCE_UNSET)) in amdgpu_device_resize_fb_bar()
1598 return -ENODEV; in amdgpu_device_resize_fb_bar()
1600 pci_write_config_word(adev->pdev, PCI_COMMAND, cmd); in amdgpu_device_resize_fb_bar()
1607 if (hweight32(adev->aid_mask) && (adev->flags & AMD_IS_APU)) in amdgpu_device_read_bios()
1617 * amdgpu_device_need_post - check if the hw need post or not
1636 /* for FIJI: In whole GPU pass-through virtualization case, after VM reboot in amdgpu_device_need_post()
1641 if (adev->asic_type == CHIP_FIJI) { in amdgpu_device_need_post()
1645 err = request_firmware(&adev->pm.fw, "amdgpu/fiji_smc.bin", adev->dev); in amdgpu_device_need_post()
1650 fw_ver = *((uint32_t *)adev->pm.fw->data + 69); in amdgpu_device_need_post()
1651 release_firmware(adev->pm.fw); in amdgpu_device_need_post()
1658 if (adev->gmc.xgmi.pending_reset) in amdgpu_device_need_post()
1661 if (adev->has_hw_reset) { in amdgpu_device_need_post()
1662 adev->has_hw_reset = false; in amdgpu_device_need_post()
1667 if (adev->asic_type >= CHIP_BONAIRE) in amdgpu_device_need_post()
1689 case -1: in amdgpu_device_seamless_boot_supported()
1701 if (!(adev->flags & AMD_IS_APU)) in amdgpu_device_seamless_boot_supported()
1704 if (adev->mman.keep_stolen_vga_memory) in amdgpu_device_seamless_boot_supported()
1715 …gn/products/platforms/details/raptor-lake-s/13th-generation-core-processors-datasheet-volume-1-of-…
1716 * https://gitlab.freedesktop.org/drm/amd/-/issues/2663
1724 if (dev_is_removable(adev->dev)) in amdgpu_device_pcie_dynamic_switching_supported()
1727 if (c->x86_vendor == X86_VENDOR_INTEL) in amdgpu_device_pcie_dynamic_switching_supported()
1734 * amdgpu_device_should_use_aspm - check if the device should program ASPM
1738 * Confirm whether the module parameter and pcie bridge agree that ASPM should
1746 case -1: in amdgpu_device_should_use_aspm()
1755 if (adev->flags & AMD_IS_APU) in amdgpu_device_should_use_aspm()
1757 if (!(adev->pm.pp_feature & PP_PCIE_DPM_MASK)) in amdgpu_device_should_use_aspm()
1759 return pcie_aspm_enabled(adev->pdev); in amdgpu_device_should_use_aspm()
1764 * amdgpu_device_vga_set_decode - enable/disable vga decode
1786 * amdgpu_device_check_block_size - validate the vm block size
1801 if (amdgpu_vm_block_size == -1) in amdgpu_device_check_block_size()
1805 dev_warn(adev->dev, "VM page table size (%d) too small\n", in amdgpu_device_check_block_size()
1807 amdgpu_vm_block_size = -1; in amdgpu_device_check_block_size()
1812 * amdgpu_device_check_vm_size - validate the vm size
1822 if (amdgpu_vm_size == -1) in amdgpu_device_check_vm_size()
1826 dev_warn(adev->dev, "VM size (%d) too small, min is 1GB\n", in amdgpu_device_check_vm_size()
1828 amdgpu_vm_size = -1; in amdgpu_device_check_vm_size()
1844 DRM_WARN("Not 64-bit OS, feature not supported\n"); in amdgpu_device_check_smu_prv_buffer_size()
1862 adev->pm.smu_prv_buffer_size = amdgpu_smu_memory_pool_size << 28; in amdgpu_device_check_smu_prv_buffer_size()
1869 adev->pm.smu_prv_buffer_size = 0; in amdgpu_device_check_smu_prv_buffer_size()
1874 if (!(adev->flags & AMD_IS_APU) || in amdgpu_device_init_apu_flags()
1875 adev->asic_type < CHIP_RAVEN) in amdgpu_device_init_apu_flags()
1878 switch (adev->asic_type) { in amdgpu_device_init_apu_flags()
1880 if (adev->pdev->device == 0x15dd) in amdgpu_device_init_apu_flags()
1881 adev->apu_flags |= AMD_APU_IS_RAVEN; in amdgpu_device_init_apu_flags()
1882 if (adev->pdev->device == 0x15d8) in amdgpu_device_init_apu_flags()
1883 adev->apu_flags |= AMD_APU_IS_PICASSO; in amdgpu_device_init_apu_flags()
1886 if ((adev->pdev->device == 0x1636) || in amdgpu_device_init_apu_flags()
1887 (adev->pdev->device == 0x164c)) in amdgpu_device_init_apu_flags()
1888 adev->apu_flags |= AMD_APU_IS_RENOIR; in amdgpu_device_init_apu_flags()
1890 adev->apu_flags |= AMD_APU_IS_GREEN_SARDINE; in amdgpu_device_init_apu_flags()
1893 adev->apu_flags |= AMD_APU_IS_VANGOGH; in amdgpu_device_init_apu_flags()
1898 if ((adev->pdev->device == 0x13FE) || in amdgpu_device_init_apu_flags()
1899 (adev->pdev->device == 0x143F)) in amdgpu_device_init_apu_flags()
1900 adev->apu_flags |= AMD_APU_IS_CYAN_SKILLFISH2; in amdgpu_device_init_apu_flags()
1910 * amdgpu_device_check_arguments - validate module params
1922 dev_warn(adev->dev, "sched jobs (%d) must be at least 4\n", in amdgpu_device_check_arguments()
1926 dev_warn(adev->dev, "sched jobs (%d) must be a power of 2\n", in amdgpu_device_check_arguments()
1931 if (amdgpu_gart_size != -1 && amdgpu_gart_size < 32) { in amdgpu_device_check_arguments()
1933 dev_warn(adev->dev, "gart size (%d) too small\n", in amdgpu_device_check_arguments()
1935 amdgpu_gart_size = -1; in amdgpu_device_check_arguments()
1938 if (amdgpu_gtt_size != -1 && amdgpu_gtt_size < 32) { in amdgpu_device_check_arguments()
1940 dev_warn(adev->dev, "gtt size (%d) too small\n", in amdgpu_device_check_arguments()
1942 amdgpu_gtt_size = -1; in amdgpu_device_check_arguments()
1946 if (amdgpu_vm_fragment_size != -1 && in amdgpu_device_check_arguments()
1948 dev_warn(adev->dev, "valid range is between 4 and 9\n"); in amdgpu_device_check_arguments()
1949 amdgpu_vm_fragment_size = -1; in amdgpu_device_check_arguments()
1953 dev_warn(adev->dev, "sched hw submission jobs (%d) must be at least 2\n", in amdgpu_device_check_arguments()
1957 dev_warn(adev->dev, "sched hw submission jobs (%d) must be a power of 2\n", in amdgpu_device_check_arguments()
1962 if (amdgpu_reset_method < -1 || amdgpu_reset_method > 4) { in amdgpu_device_check_arguments()
1963 dev_warn(adev->dev, "invalid option for reset method, reverting to default\n"); in amdgpu_device_check_arguments()
1964 amdgpu_reset_method = -1; in amdgpu_device_check_arguments()
1973 adev->firmware.load_type = amdgpu_ucode_get_load_type(adev, amdgpu_fw_load_type); in amdgpu_device_check_arguments()
1976 adev->enforce_isolation[i] = !!enforce_isolation; in amdgpu_device_check_arguments()
1982 * amdgpu_switcheroo_set_state - set switcheroo state
2002 dev->switch_power_state = DRM_SWITCH_POWER_CHANGING; in amdgpu_switcheroo_set_state()
2011 dev->switch_power_state = DRM_SWITCH_POWER_ON; in amdgpu_switcheroo_set_state()
2014 dev->switch_power_state = DRM_SWITCH_POWER_CHANGING; in amdgpu_switcheroo_set_state()
2021 dev->switch_power_state = DRM_SWITCH_POWER_OFF; in amdgpu_switcheroo_set_state()
2026 * amdgpu_switcheroo_can_switch - see if switcheroo state can change
2043 return atomic_read(&dev->open_count) == 0; in amdgpu_switcheroo_can_switch()
2053 * amdgpu_device_ip_set_clockgating_state - set the CG state
2070 for (i = 0; i < adev->num_ip_blocks; i++) { in amdgpu_device_ip_set_clockgating_state()
2071 if (!adev->ip_blocks[i].status.valid) in amdgpu_device_ip_set_clockgating_state()
2073 if (adev->ip_blocks[i].version->type != block_type) in amdgpu_device_ip_set_clockgating_state()
2075 if (!adev->ip_blocks[i].version->funcs->set_clockgating_state) in amdgpu_device_ip_set_clockgating_state()
2077 r = adev->ip_blocks[i].version->funcs->set_clockgating_state( in amdgpu_device_ip_set_clockgating_state()
2081 adev->ip_blocks[i].version->funcs->name, r); in amdgpu_device_ip_set_clockgating_state()
2087 * amdgpu_device_ip_set_powergating_state - set the PG state
2104 for (i = 0; i < adev->num_ip_blocks; i++) { in amdgpu_device_ip_set_powergating_state()
2105 if (!adev->ip_blocks[i].status.valid) in amdgpu_device_ip_set_powergating_state()
2107 if (adev->ip_blocks[i].version->type != block_type) in amdgpu_device_ip_set_powergating_state()
2109 if (!adev->ip_blocks[i].version->funcs->set_powergating_state) in amdgpu_device_ip_set_powergating_state()
2111 r = adev->ip_blocks[i].version->funcs->set_powergating_state( in amdgpu_device_ip_set_powergating_state()
2115 adev->ip_blocks[i].version->funcs->name, r); in amdgpu_device_ip_set_powergating_state()
2121 * amdgpu_device_ip_get_clockgating_state - get the CG state
2136 for (i = 0; i < adev->num_ip_blocks; i++) { in amdgpu_device_ip_get_clockgating_state()
2137 if (!adev->ip_blocks[i].status.valid) in amdgpu_device_ip_get_clockgating_state()
2139 if (adev->ip_blocks[i].version->funcs->get_clockgating_state) in amdgpu_device_ip_get_clockgating_state()
2140 adev->ip_blocks[i].version->funcs->get_clockgating_state((void *)adev, flags); in amdgpu_device_ip_get_clockgating_state()
2145 * amdgpu_device_ip_wait_for_idle - wait for idle
2158 for (i = 0; i < adev->num_ip_blocks; i++) { in amdgpu_device_ip_wait_for_idle()
2159 if (!adev->ip_blocks[i].status.valid) in amdgpu_device_ip_wait_for_idle()
2161 if (adev->ip_blocks[i].version->type == block_type) { in amdgpu_device_ip_wait_for_idle()
2162 r = adev->ip_blocks[i].version->funcs->wait_for_idle((void *)adev); in amdgpu_device_ip_wait_for_idle()
2173 * amdgpu_device_ip_is_idle - is the hardware IP idle
2186 for (i = 0; i < adev->num_ip_blocks; i++) { in amdgpu_device_ip_is_idle()
2187 if (!adev->ip_blocks[i].status.valid) in amdgpu_device_ip_is_idle()
2189 if (adev->ip_blocks[i].version->type == block_type) in amdgpu_device_ip_is_idle()
2190 return adev->ip_blocks[i].version->funcs->is_idle((void *)adev); in amdgpu_device_ip_is_idle()
2197 * amdgpu_device_ip_get_ip_block - get a hw IP pointer
2211 for (i = 0; i < adev->num_ip_blocks; i++) in amdgpu_device_ip_get_ip_block()
2212 if (adev->ip_blocks[i].version->type == type) in amdgpu_device_ip_get_ip_block()
2213 return &adev->ip_blocks[i]; in amdgpu_device_ip_get_ip_block()
2235 if (ip_block && ((ip_block->version->major > major) || in amdgpu_device_ip_block_version_cmp()
2236 ((ip_block->version->major == major) && in amdgpu_device_ip_block_version_cmp()
2237 (ip_block->version->minor >= minor)))) in amdgpu_device_ip_block_version_cmp()
2256 return -EINVAL; in amdgpu_device_ip_block_add()
2258 switch (ip_block_version->type) { in amdgpu_device_ip_block_add()
2260 if (adev->harvest_ip_mask & AMD_HARVEST_IP_VCN_MASK) in amdgpu_device_ip_block_add()
2264 if (adev->harvest_ip_mask & AMD_HARVEST_IP_JPEG_MASK) in amdgpu_device_ip_block_add()
2271 DRM_INFO("add ip block number %d <%s>\n", adev->num_ip_blocks, in amdgpu_device_ip_block_add()
2272 ip_block_version->funcs->name); in amdgpu_device_ip_block_add()
2274 adev->ip_blocks[adev->num_ip_blocks++].version = ip_block_version; in amdgpu_device_ip_block_add()
2280 * amdgpu_device_enable_virtual_display - enable virtual display feature
2293 adev->enable_virtual_display = false; in amdgpu_device_enable_virtual_display()
2296 const char *pci_address_name = pci_name(adev->pdev); in amdgpu_device_enable_virtual_display()
2306 int res = -1; in amdgpu_device_enable_virtual_display()
2308 adev->enable_virtual_display = true; in amdgpu_device_enable_virtual_display()
2319 adev->mode_info.num_crtc = num_crtc; in amdgpu_device_enable_virtual_display()
2321 adev->mode_info.num_crtc = 1; in amdgpu_device_enable_virtual_display()
2329 adev->enable_virtual_display, adev->mode_info.num_crtc); in amdgpu_device_enable_virtual_display()
2337 if (amdgpu_sriov_vf(adev) && !adev->enable_virtual_display) { in amdgpu_device_set_sriov_virtual_display()
2338 adev->mode_info.num_crtc = 1; in amdgpu_device_set_sriov_virtual_display()
2339 adev->enable_virtual_display = true; in amdgpu_device_set_sriov_virtual_display()
2341 adev->enable_virtual_display, adev->mode_info.num_crtc); in amdgpu_device_set_sriov_virtual_display()
2346 * amdgpu_device_parse_gpu_info_fw - parse gpu info firmware
2353 * Returns 0 on success, -EINVAL on failure.
2361 adev->firmware.gpu_info_fw = NULL; in amdgpu_device_parse_gpu_info_fw()
2363 if (adev->mman.discovery_bin) in amdgpu_device_parse_gpu_info_fw()
2366 switch (adev->asic_type) { in amdgpu_device_parse_gpu_info_fw()
2376 if (adev->apu_flags & AMD_APU_IS_RAVEN2) in amdgpu_device_parse_gpu_info_fw()
2378 else if (adev->apu_flags & AMD_APU_IS_PICASSO) in amdgpu_device_parse_gpu_info_fw()
2391 err = amdgpu_ucode_request(adev, &adev->firmware.gpu_info_fw, in amdgpu_device_parse_gpu_info_fw()
2394 dev_err(adev->dev, in amdgpu_device_parse_gpu_info_fw()
2400 hdr = (const struct gpu_info_firmware_header_v1_0 *)adev->firmware.gpu_info_fw->data; in amdgpu_device_parse_gpu_info_fw()
2401 amdgpu_ucode_print_gpu_info_hdr(&hdr->header); in amdgpu_device_parse_gpu_info_fw()
2403 switch (hdr->version_major) { in amdgpu_device_parse_gpu_info_fw()
2407 (const struct gpu_info_firmware_v1_0 *)(adev->firmware.gpu_info_fw->data + in amdgpu_device_parse_gpu_info_fw()
2408 le32_to_cpu(hdr->header.ucode_array_offset_bytes)); in amdgpu_device_parse_gpu_info_fw()
2413 if (adev->asic_type == CHIP_NAVI12) in amdgpu_device_parse_gpu_info_fw()
2416 adev->gfx.config.max_shader_engines = le32_to_cpu(gpu_info_fw->gc_num_se); in amdgpu_device_parse_gpu_info_fw()
2417 adev->gfx.config.max_cu_per_sh = le32_to_cpu(gpu_info_fw->gc_num_cu_per_sh); in amdgpu_device_parse_gpu_info_fw()
2418 adev->gfx.config.max_sh_per_se = le32_to_cpu(gpu_info_fw->gc_num_sh_per_se); in amdgpu_device_parse_gpu_info_fw()
2419 adev->gfx.config.max_backends_per_se = le32_to_cpu(gpu_info_fw->gc_num_rb_per_se); in amdgpu_device_parse_gpu_info_fw()
2420 adev->gfx.config.max_texture_channel_caches = in amdgpu_device_parse_gpu_info_fw()
2421 le32_to_cpu(gpu_info_fw->gc_num_tccs); in amdgpu_device_parse_gpu_info_fw()
2422 adev->gfx.config.max_gprs = le32_to_cpu(gpu_info_fw->gc_num_gprs); in amdgpu_device_parse_gpu_info_fw()
2423 adev->gfx.config.max_gs_threads = le32_to_cpu(gpu_info_fw->gc_num_max_gs_thds); in amdgpu_device_parse_gpu_info_fw()
2424 adev->gfx.config.gs_vgt_table_depth = le32_to_cpu(gpu_info_fw->gc_gs_table_depth); in amdgpu_device_parse_gpu_info_fw()
2425 adev->gfx.config.gs_prim_buffer_depth = le32_to_cpu(gpu_info_fw->gc_gsprim_buff_depth); in amdgpu_device_parse_gpu_info_fw()
2426 adev->gfx.config.double_offchip_lds_buf = in amdgpu_device_parse_gpu_info_fw()
2427 le32_to_cpu(gpu_info_fw->gc_double_offchip_lds_buffer); in amdgpu_device_parse_gpu_info_fw()
2428 adev->gfx.cu_info.wave_front_size = le32_to_cpu(gpu_info_fw->gc_wave_size); in amdgpu_device_parse_gpu_info_fw()
2429 adev->gfx.cu_info.max_waves_per_simd = in amdgpu_device_parse_gpu_info_fw()
2430 le32_to_cpu(gpu_info_fw->gc_max_waves_per_simd); in amdgpu_device_parse_gpu_info_fw()
2431 adev->gfx.cu_info.max_scratch_slots_per_cu = in amdgpu_device_parse_gpu_info_fw()
2432 le32_to_cpu(gpu_info_fw->gc_max_scratch_slots_per_cu); in amdgpu_device_parse_gpu_info_fw()
2433 adev->gfx.cu_info.lds_size = le32_to_cpu(gpu_info_fw->gc_lds_size); in amdgpu_device_parse_gpu_info_fw()
2434 if (hdr->version_minor >= 1) { in amdgpu_device_parse_gpu_info_fw()
2436 (const struct gpu_info_firmware_v1_1 *)(adev->firmware.gpu_info_fw->data + in amdgpu_device_parse_gpu_info_fw()
2437 le32_to_cpu(hdr->header.ucode_array_offset_bytes)); in amdgpu_device_parse_gpu_info_fw()
2438 adev->gfx.config.num_sc_per_sh = in amdgpu_device_parse_gpu_info_fw()
2439 le32_to_cpu(gpu_info_fw->num_sc_per_sh); in amdgpu_device_parse_gpu_info_fw()
2440 adev->gfx.config.num_packer_per_sc = in amdgpu_device_parse_gpu_info_fw()
2441 le32_to_cpu(gpu_info_fw->num_packer_per_sc); in amdgpu_device_parse_gpu_info_fw()
2449 if (hdr->version_minor == 2) { in amdgpu_device_parse_gpu_info_fw()
2451 (const struct gpu_info_firmware_v1_2 *)(adev->firmware.gpu_info_fw->data + in amdgpu_device_parse_gpu_info_fw()
2452 le32_to_cpu(hdr->header.ucode_array_offset_bytes)); in amdgpu_device_parse_gpu_info_fw()
2453 adev->dm.soc_bounding_box = &gpu_info_fw->soc_bounding_box; in amdgpu_device_parse_gpu_info_fw()
2458 dev_err(adev->dev, in amdgpu_device_parse_gpu_info_fw()
2459 "Unsupported gpu_info table %d\n", hdr->header.ucode_version); in amdgpu_device_parse_gpu_info_fw()
2460 err = -EINVAL; in amdgpu_device_parse_gpu_info_fw()
2468 * amdgpu_device_ip_early_init - run early init for hardware IPs
2492 switch (adev->asic_type) { in amdgpu_device_ip_early_init()
2499 adev->family = AMDGPU_FAMILY_SI; in amdgpu_device_ip_early_init()
2511 if (adev->flags & AMD_IS_APU) in amdgpu_device_ip_early_init()
2512 adev->family = AMDGPU_FAMILY_KV; in amdgpu_device_ip_early_init()
2514 adev->family = AMDGPU_FAMILY_CI; in amdgpu_device_ip_early_init()
2530 if (adev->flags & AMD_IS_APU) in amdgpu_device_ip_early_init()
2531 adev->family = AMDGPU_FAMILY_CZ; in amdgpu_device_ip_early_init()
2533 adev->family = AMDGPU_FAMILY_VI; in amdgpu_device_ip_early_init()
2549 ((adev->flags & AMD_IS_APU) == 0) && in amdgpu_device_ip_early_init()
2550 !dev_is_removable(&adev->pdev->dev)) in amdgpu_device_ip_early_init()
2551 adev->flags |= AMD_IS_PX; in amdgpu_device_ip_early_init()
2553 if (!(adev->flags & AMD_IS_APU)) { in amdgpu_device_ip_early_init()
2554 parent = pcie_find_root_port(adev->pdev); in amdgpu_device_ip_early_init()
2555 adev->has_pr3 = parent ? pci_pr3_present(parent) : false; in amdgpu_device_ip_early_init()
2559 adev->pm.pp_feature = amdgpu_pp_feature_mask; in amdgpu_device_ip_early_init()
2561 adev->pm.pp_feature &= ~PP_GFXOFF_MASK; in amdgpu_device_ip_early_init()
2562 if (amdgpu_sriov_vf(adev) && adev->asic_type == CHIP_SIENNA_CICHLID) in amdgpu_device_ip_early_init()
2563 adev->pm.pp_feature &= ~PP_OVERDRIVE_MASK; in amdgpu_device_ip_early_init()
2565 adev->pm.pp_feature &= ~PP_PCIE_DPM_MASK; in amdgpu_device_ip_early_init()
2568 for (i = 0; i < adev->num_ip_blocks; i++) { in amdgpu_device_ip_early_init()
2571 i, adev->ip_blocks[i].version->funcs->name); in amdgpu_device_ip_early_init()
2572 adev->ip_blocks[i].status.valid = false; in amdgpu_device_ip_early_init()
2574 if (adev->ip_blocks[i].version->funcs->early_init) { in amdgpu_device_ip_early_init()
2575 r = adev->ip_blocks[i].version->funcs->early_init((void *)adev); in amdgpu_device_ip_early_init()
2576 if (r == -ENOENT) { in amdgpu_device_ip_early_init()
2577 adev->ip_blocks[i].status.valid = false; in amdgpu_device_ip_early_init()
2580 adev->ip_blocks[i].version->funcs->name, r); in amdgpu_device_ip_early_init()
2583 adev->ip_blocks[i].status.valid = true; in amdgpu_device_ip_early_init()
2586 adev->ip_blocks[i].status.valid = true; in amdgpu_device_ip_early_init()
2590 if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_COMMON) { in amdgpu_device_ip_early_init()
2598 return -EINVAL; in amdgpu_device_ip_early_init()
2602 dev_err(adev->dev, "amdgpu_atombios_init failed\n"); in amdgpu_device_ip_early_init()
2615 return -ENODEV; in amdgpu_device_ip_early_init()
2618 if (ip_block->status.valid != false) in amdgpu_device_ip_early_init()
2621 adev->cg_flags &= amdgpu_cg_mask; in amdgpu_device_ip_early_init()
2622 adev->pg_flags &= amdgpu_pg_mask; in amdgpu_device_ip_early_init()
2631 for (i = 0; i < adev->num_ip_blocks; i++) { in amdgpu_device_ip_hw_init_phase1()
2632 if (!adev->ip_blocks[i].status.sw) in amdgpu_device_ip_hw_init_phase1()
2634 if (adev->ip_blocks[i].status.hw) in amdgpu_device_ip_hw_init_phase1()
2636 if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_COMMON || in amdgpu_device_ip_hw_init_phase1()
2637 (amdgpu_sriov_vf(adev) && (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_PSP)) || in amdgpu_device_ip_hw_init_phase1()
2638 adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_IH) { in amdgpu_device_ip_hw_init_phase1()
2639 r = adev->ip_blocks[i].version->funcs->hw_init(adev); in amdgpu_device_ip_hw_init_phase1()
2642 adev->ip_blocks[i].version->funcs->name, r); in amdgpu_device_ip_hw_init_phase1()
2645 adev->ip_blocks[i].status.hw = true; in amdgpu_device_ip_hw_init_phase1()
2656 for (i = 0; i < adev->num_ip_blocks; i++) { in amdgpu_device_ip_hw_init_phase2()
2657 if (!adev->ip_blocks[i].status.sw) in amdgpu_device_ip_hw_init_phase2()
2659 if (adev->ip_blocks[i].status.hw) in amdgpu_device_ip_hw_init_phase2()
2661 r = adev->ip_blocks[i].version->funcs->hw_init(adev); in amdgpu_device_ip_hw_init_phase2()
2664 adev->ip_blocks[i].version->funcs->name, r); in amdgpu_device_ip_hw_init_phase2()
2667 adev->ip_blocks[i].status.hw = true; in amdgpu_device_ip_hw_init_phase2()
2679 if (adev->asic_type >= CHIP_VEGA10) { in amdgpu_device_fw_loading()
2680 for (i = 0; i < adev->num_ip_blocks; i++) { in amdgpu_device_fw_loading()
2681 if (adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_PSP) in amdgpu_device_fw_loading()
2684 if (!adev->ip_blocks[i].status.sw) in amdgpu_device_fw_loading()
2688 if (adev->ip_blocks[i].status.hw == true) in amdgpu_device_fw_loading()
2691 if (amdgpu_in_reset(adev) || adev->in_suspend) { in amdgpu_device_fw_loading()
2692 r = adev->ip_blocks[i].version->funcs->resume(adev); in amdgpu_device_fw_loading()
2695 adev->ip_blocks[i].version->funcs->name, r); in amdgpu_device_fw_loading()
2699 r = adev->ip_blocks[i].version->funcs->hw_init(adev); in amdgpu_device_fw_loading()
2702 adev->ip_blocks[i].version->funcs->name, r); in amdgpu_device_fw_loading()
2707 adev->ip_blocks[i].status.hw = true; in amdgpu_device_fw_loading()
2712 if (!amdgpu_sriov_vf(adev) || adev->asic_type == CHIP_TONGA) in amdgpu_device_fw_loading()
2724 struct amdgpu_ring *ring = adev->rings[i]; in amdgpu_device_init_schedulers()
2727 if (!ring || ring->no_scheduler) in amdgpu_device_init_schedulers()
2730 switch (ring->funcs->type) { in amdgpu_device_init_schedulers()
2732 timeout = adev->gfx_timeout; in amdgpu_device_init_schedulers()
2735 timeout = adev->compute_timeout; in amdgpu_device_init_schedulers()
2738 timeout = adev->sdma_timeout; in amdgpu_device_init_schedulers()
2741 timeout = adev->video_timeout; in amdgpu_device_init_schedulers()
2745 r = drm_sched_init(&ring->sched, &amdgpu_sched_ops, NULL, in amdgpu_device_init_schedulers()
2747 ring->num_hw_submission, 0, in amdgpu_device_init_schedulers()
2748 timeout, adev->reset_domain->wq, in amdgpu_device_init_schedulers()
2749 ring->sched_score, ring->name, in amdgpu_device_init_schedulers()
2750 adev->dev); in amdgpu_device_init_schedulers()
2753 ring->name); in amdgpu_device_init_schedulers()
2759 ring->name); in amdgpu_device_init_schedulers()
2765 ring->name); in amdgpu_device_init_schedulers()
2777 * amdgpu_device_ip_init - run init for hardware IPs
2795 for (i = 0; i < adev->num_ip_blocks; i++) { in amdgpu_device_ip_init()
2796 if (!adev->ip_blocks[i].status.valid) in amdgpu_device_ip_init()
2798 r = adev->ip_blocks[i].version->funcs->sw_init((void *)adev); in amdgpu_device_ip_init()
2801 adev->ip_blocks[i].version->funcs->name, r); in amdgpu_device_ip_init()
2804 adev->ip_blocks[i].status.sw = true; in amdgpu_device_ip_init()
2806 if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_COMMON) { in amdgpu_device_ip_init()
2808 r = adev->ip_blocks[i].version->funcs->hw_init((void *)adev); in amdgpu_device_ip_init()
2813 adev->ip_blocks[i].status.hw = true; in amdgpu_device_ip_init()
2814 } else if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GMC) { in amdgpu_device_ip_init()
2825 r = adev->ip_blocks[i].version->funcs->hw_init((void *)adev); in amdgpu_device_ip_init()
2835 adev->ip_blocks[i].status.hw = true; in amdgpu_device_ip_init()
2838 if (adev->gfx.mcbp) { in amdgpu_device_ip_init()
2839 r = amdgpu_allocate_static_csa(adev, &adev->virt.csa_obj, in amdgpu_device_ip_init()
2862 dev_err(adev->dev, "IB initialization failed (%d).\n", r); in amdgpu_device_ip_init()
2905 if (adev->gmc.xgmi.num_physical_nodes > 1) { in amdgpu_device_ip_init()
2911 r = -ENOENT; in amdgpu_device_ip_init()
2915 if (!hive->reset_domain || in amdgpu_device_ip_init()
2916 !amdgpu_reset_get_reset_domain(hive->reset_domain)) { in amdgpu_device_ip_init()
2917 r = -ENOENT; in amdgpu_device_ip_init()
2923 amdgpu_reset_put_reset_domain(adev->reset_domain); in amdgpu_device_ip_init()
2924 adev->reset_domain = hive->reset_domain; in amdgpu_device_ip_init()
2934 if (adev->mman.buffer_funcs_ring->sched.ready) in amdgpu_device_ip_init()
2938 if (!adev->gmc.xgmi.pending_reset) { in amdgpu_device_ip_init()
2951 * amdgpu_device_fill_reset_magic - writes reset magic to gart pointer
2961 memcpy(adev->reset_magic, adev->gart.ptr, AMDGPU_RESET_MAGIC_NUM); in amdgpu_device_fill_reset_magic()
2965 * amdgpu_device_check_vram_lost - check if vram is valid
2976 if (memcmp(adev->gart.ptr, adev->reset_magic, in amdgpu_device_check_vram_lost()
2997 * amdgpu_device_set_cg_state - set clockgating for amdgpu device
3017 for (j = 0; j < adev->num_ip_blocks; j++) { in amdgpu_device_set_cg_state()
3018 i = state == AMD_CG_STATE_GATE ? j : adev->num_ip_blocks - j - 1; in amdgpu_device_set_cg_state()
3019 if (!adev->ip_blocks[i].status.late_initialized) in amdgpu_device_set_cg_state()
3022 if (adev->in_s0ix && in amdgpu_device_set_cg_state()
3023 (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GFX || in amdgpu_device_set_cg_state()
3024 adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_SDMA)) in amdgpu_device_set_cg_state()
3027 if (adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_UVD && in amdgpu_device_set_cg_state()
3028 adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_VCE && in amdgpu_device_set_cg_state()
3029 adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_VCN && in amdgpu_device_set_cg_state()
3030 adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_JPEG && in amdgpu_device_set_cg_state()
3031 adev->ip_blocks[i].version->funcs->set_clockgating_state) { in amdgpu_device_set_cg_state()
3033 r = adev->ip_blocks[i].version->funcs->set_clockgating_state((void *)adev, in amdgpu_device_set_cg_state()
3037 adev->ip_blocks[i].version->funcs->name, r); in amdgpu_device_set_cg_state()
3054 for (j = 0; j < adev->num_ip_blocks; j++) { in amdgpu_device_set_pg_state()
3055 i = state == AMD_PG_STATE_GATE ? j : adev->num_ip_blocks - j - 1; in amdgpu_device_set_pg_state()
3056 if (!adev->ip_blocks[i].status.late_initialized) in amdgpu_device_set_pg_state()
3059 if (adev->in_s0ix && in amdgpu_device_set_pg_state()
3060 (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GFX || in amdgpu_device_set_pg_state()
3061 adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_SDMA)) in amdgpu_device_set_pg_state()
3064 if (adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_UVD && in amdgpu_device_set_pg_state()
3065 adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_VCE && in amdgpu_device_set_pg_state()
3066 adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_VCN && in amdgpu_device_set_pg_state()
3067 adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_JPEG && in amdgpu_device_set_pg_state()
3068 adev->ip_blocks[i].version->funcs->set_powergating_state) { in amdgpu_device_set_pg_state()
3070 r = adev->ip_blocks[i].version->funcs->set_powergating_state((void *)adev, in amdgpu_device_set_pg_state()
3074 adev->ip_blocks[i].version->funcs->name, r); in amdgpu_device_set_pg_state()
3100 adev = gpu_ins->adev; in amdgpu_device_enable_mgpu_fan_boost()
3101 if (!(adev->flags & AMD_IS_APU) && in amdgpu_device_enable_mgpu_fan_boost()
3102 !gpu_ins->mgpu_fan_enabled) { in amdgpu_device_enable_mgpu_fan_boost()
3107 gpu_ins->mgpu_fan_enabled = 1; in amdgpu_device_enable_mgpu_fan_boost()
3118 * amdgpu_device_ip_late_init - run late init for hardware IPs
3134 for (i = 0; i < adev->num_ip_blocks; i++) { in amdgpu_device_ip_late_init()
3135 if (!adev->ip_blocks[i].status.hw) in amdgpu_device_ip_late_init()
3137 if (adev->ip_blocks[i].version->funcs->late_init) { in amdgpu_device_ip_late_init()
3138 r = adev->ip_blocks[i].version->funcs->late_init((void *)adev); in amdgpu_device_ip_late_init()
3141 adev->ip_blocks[i].version->funcs->name, r); in amdgpu_device_ip_late_init()
3145 adev->ip_blocks[i].status.late_initialized = true; in amdgpu_device_ip_late_init()
3168 ((adev->asic_type == CHIP_ARCTURUS && adev->gmc.xgmi.num_physical_nodes > 1) || in amdgpu_device_ip_late_init()
3169 adev->asic_type == CHIP_ALDEBARAN)) in amdgpu_device_ip_late_init()
3172 if (adev->gmc.xgmi.num_physical_nodes > 1) { in amdgpu_device_ip_late_init()
3176 * Reset device p-state to low as this was booted with high. in amdgpu_device_ip_late_init()
3188 if (mgpu_info.num_dgpu == adev->gmc.xgmi.num_physical_nodes) { in amdgpu_device_ip_late_init()
3191 if (gpu_instance->adev->flags & AMD_IS_APU) in amdgpu_device_ip_late_init()
3194 r = amdgpu_xgmi_set_pstate(gpu_instance->adev, in amdgpu_device_ip_late_init()
3210 * amdgpu_device_smu_fini_early - smu hw_fini wrapper
3223 for (i = 0; i < adev->num_ip_blocks; i++) { in amdgpu_device_smu_fini_early()
3224 if (!adev->ip_blocks[i].status.hw) in amdgpu_device_smu_fini_early()
3226 if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_SMC) { in amdgpu_device_smu_fini_early()
3227 r = adev->ip_blocks[i].version->funcs->hw_fini((void *)adev); in amdgpu_device_smu_fini_early()
3231 adev->ip_blocks[i].version->funcs->name, r); in amdgpu_device_smu_fini_early()
3233 adev->ip_blocks[i].status.hw = false; in amdgpu_device_smu_fini_early()
3243 for (i = 0; i < adev->num_ip_blocks; i++) { in amdgpu_device_ip_fini_early()
3244 if (!adev->ip_blocks[i].version->funcs->early_fini) in amdgpu_device_ip_fini_early()
3247 r = adev->ip_blocks[i].version->funcs->early_fini((void *)adev); in amdgpu_device_ip_fini_early()
3250 adev->ip_blocks[i].version->funcs->name, r); in amdgpu_device_ip_fini_early()
3262 for (i = adev->num_ip_blocks - 1; i >= 0; i--) { in amdgpu_device_ip_fini_early()
3263 if (!adev->ip_blocks[i].status.hw) in amdgpu_device_ip_fini_early()
3266 r = adev->ip_blocks[i].version->funcs->hw_fini((void *)adev); in amdgpu_device_ip_fini_early()
3270 adev->ip_blocks[i].version->funcs->name, r); in amdgpu_device_ip_fini_early()
3273 adev->ip_blocks[i].status.hw = false; in amdgpu_device_ip_fini_early()
3285 * amdgpu_device_ip_fini - run fini for hardware IPs
3299 if (amdgpu_sriov_vf(adev) && adev->virt.ras_init_done) in amdgpu_device_ip_fini()
3302 if (adev->gmc.xgmi.num_physical_nodes > 1) in amdgpu_device_ip_fini()
3307 for (i = adev->num_ip_blocks - 1; i >= 0; i--) { in amdgpu_device_ip_fini()
3308 if (!adev->ip_blocks[i].status.sw) in amdgpu_device_ip_fini()
3311 if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GMC) { in amdgpu_device_ip_fini()
3313 amdgpu_free_static_csa(&adev->virt.csa_obj); in amdgpu_device_ip_fini()
3320 r = adev->ip_blocks[i].version->funcs->sw_fini((void *)adev); in amdgpu_device_ip_fini()
3324 adev->ip_blocks[i].version->funcs->name, r); in amdgpu_device_ip_fini()
3326 adev->ip_blocks[i].status.sw = false; in amdgpu_device_ip_fini()
3327 adev->ip_blocks[i].status.valid = false; in amdgpu_device_ip_fini()
3330 for (i = adev->num_ip_blocks - 1; i >= 0; i--) { in amdgpu_device_ip_fini()
3331 if (!adev->ip_blocks[i].status.late_initialized) in amdgpu_device_ip_fini()
3333 if (adev->ip_blocks[i].version->funcs->late_fini) in amdgpu_device_ip_fini()
3334 adev->ip_blocks[i].version->funcs->late_fini((void *)adev); in amdgpu_device_ip_fini()
3335 adev->ip_blocks[i].status.late_initialized = false; in amdgpu_device_ip_fini()
3344 * amdgpu_device_delayed_init_work_handler - work handler for IB tests
3364 WARN_ON_ONCE(adev->gfx.gfx_off_state); in amdgpu_device_delay_enable_gfx_off()
3365 WARN_ON_ONCE(adev->gfx.gfx_off_req_count); in amdgpu_device_delay_enable_gfx_off()
3368 adev->gfx.gfx_off_state = true; in amdgpu_device_delay_enable_gfx_off()
3372 * amdgpu_device_ip_suspend_phase1 - run suspend for hardware IPs (phase 1)
3395 dev_warn(adev->dev, "Failed to disallow df cstate"); in amdgpu_device_ip_suspend_phase1()
3397 for (i = adev->num_ip_blocks - 1; i >= 0; i--) { in amdgpu_device_ip_suspend_phase1()
3398 if (!adev->ip_blocks[i].status.valid) in amdgpu_device_ip_suspend_phase1()
3402 if (adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_DCE) in amdgpu_device_ip_suspend_phase1()
3406 r = adev->ip_blocks[i].version->funcs->suspend(adev); in amdgpu_device_ip_suspend_phase1()
3410 adev->ip_blocks[i].version->funcs->name, r); in amdgpu_device_ip_suspend_phase1()
3414 adev->ip_blocks[i].status.hw = false; in amdgpu_device_ip_suspend_phase1()
3421 * amdgpu_device_ip_suspend_phase2 - run suspend for hardware IPs (phase 2)
3435 if (adev->in_s0ix) in amdgpu_device_ip_suspend_phase2()
3438 for (i = adev->num_ip_blocks - 1; i >= 0; i--) { in amdgpu_device_ip_suspend_phase2()
3439 if (!adev->ip_blocks[i].status.valid) in amdgpu_device_ip_suspend_phase2()
3442 if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_DCE) in amdgpu_device_ip_suspend_phase2()
3446 adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_PSP) { in amdgpu_device_ip_suspend_phase2()
3447 adev->ip_blocks[i].status.hw = false; in amdgpu_device_ip_suspend_phase2()
3452 if (adev->gmc.xgmi.pending_reset && in amdgpu_device_ip_suspend_phase2()
3453 !(adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GMC || in amdgpu_device_ip_suspend_phase2()
3454 adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_SMC || in amdgpu_device_ip_suspend_phase2()
3455 adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_COMMON || in amdgpu_device_ip_suspend_phase2()
3456 adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_IH)) { in amdgpu_device_ip_suspend_phase2()
3457 adev->ip_blocks[i].status.hw = false; in amdgpu_device_ip_suspend_phase2()
3466 if (adev->in_s0ix && in amdgpu_device_ip_suspend_phase2()
3467 (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_PSP || in amdgpu_device_ip_suspend_phase2()
3468 adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GFX || in amdgpu_device_ip_suspend_phase2()
3469 adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_MES)) in amdgpu_device_ip_suspend_phase2()
3473 if (adev->in_s0ix && in amdgpu_device_ip_suspend_phase2()
3476 (adev->ip_blocks[i].version->type == in amdgpu_device_ip_suspend_phase2()
3480 /* Once swPSP provides the IMU, RLC FW binaries to TOS during cold-boot. in amdgpu_device_ip_suspend_phase2()
3481 * These are in TMR, hence are expected to be reused by PSP-TOS to reload in amdgpu_device_ip_suspend_phase2()
3483 * from here based on PMFW -> PSP message during re-init sequence. in amdgpu_device_ip_suspend_phase2()
3488 (adev->flags & AMD_IS_APU) && adev->gfx.imu.funcs && in amdgpu_device_ip_suspend_phase2()
3489 adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_PSP) in amdgpu_device_ip_suspend_phase2()
3493 r = adev->ip_blocks[i].version->funcs->suspend(adev); in amdgpu_device_ip_suspend_phase2()
3497 adev->ip_blocks[i].version->funcs->name, r); in amdgpu_device_ip_suspend_phase2()
3499 adev->ip_blocks[i].status.hw = false; in amdgpu_device_ip_suspend_phase2()
3502 if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_SMC) { in amdgpu_device_ip_suspend_phase2()
3503 r = amdgpu_dpm_set_mp1_state(adev, adev->mp1_state); in amdgpu_device_ip_suspend_phase2()
3506 adev->mp1_state, r); in amdgpu_device_ip_suspend_phase2()
3517 * amdgpu_device_ip_suspend - run suspend for hardware IPs
3560 for (i = 0; i < adev->num_ip_blocks; i++) { in amdgpu_device_ip_reinit_early_sriov()
3564 block = &adev->ip_blocks[i]; in amdgpu_device_ip_reinit_early_sriov()
3565 block->status.hw = false; in amdgpu_device_ip_reinit_early_sriov()
3569 if (block->version->type != ip_order[j] || in amdgpu_device_ip_reinit_early_sriov()
3570 !block->status.valid) in amdgpu_device_ip_reinit_early_sriov()
3573 r = block->version->funcs->hw_init(adev); in amdgpu_device_ip_reinit_early_sriov()
3574 DRM_INFO("RE-INIT-early: %s %s\n", block->version->funcs->name, r?"failed":"succeeded"); in amdgpu_device_ip_reinit_early_sriov()
3577 block->status.hw = true; in amdgpu_device_ip_reinit_early_sriov()
3604 for (j = 0; j < adev->num_ip_blocks; j++) { in amdgpu_device_ip_reinit_late_sriov()
3605 block = &adev->ip_blocks[j]; in amdgpu_device_ip_reinit_late_sriov()
3607 if (block->version->type != ip_order[i] || in amdgpu_device_ip_reinit_late_sriov()
3608 !block->status.valid || in amdgpu_device_ip_reinit_late_sriov()
3609 block->status.hw) in amdgpu_device_ip_reinit_late_sriov()
3612 if (block->version->type == AMD_IP_BLOCK_TYPE_SMC) in amdgpu_device_ip_reinit_late_sriov()
3613 r = block->version->funcs->resume(adev); in amdgpu_device_ip_reinit_late_sriov()
3615 r = block->version->funcs->hw_init(adev); in amdgpu_device_ip_reinit_late_sriov()
3617 DRM_INFO("RE-INIT-late: %s %s\n", block->version->funcs->name, r?"failed":"succeeded"); in amdgpu_device_ip_reinit_late_sriov()
3620 block->status.hw = true; in amdgpu_device_ip_reinit_late_sriov()
3628 * amdgpu_device_ip_resume_phase1 - run resume for hardware IPs
3643 for (i = 0; i < adev->num_ip_blocks; i++) { in amdgpu_device_ip_resume_phase1()
3644 if (!adev->ip_blocks[i].status.valid || adev->ip_blocks[i].status.hw) in amdgpu_device_ip_resume_phase1()
3646 if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_COMMON || in amdgpu_device_ip_resume_phase1()
3647 adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GMC || in amdgpu_device_ip_resume_phase1()
3648 adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_IH || in amdgpu_device_ip_resume_phase1()
3649 (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_PSP && amdgpu_sriov_vf(adev))) { in amdgpu_device_ip_resume_phase1()
3651 r = adev->ip_blocks[i].version->funcs->resume(adev); in amdgpu_device_ip_resume_phase1()
3654 adev->ip_blocks[i].version->funcs->name, r); in amdgpu_device_ip_resume_phase1()
3657 adev->ip_blocks[i].status.hw = true; in amdgpu_device_ip_resume_phase1()
3665 * amdgpu_device_ip_resume_phase2 - run resume for hardware IPs
3681 for (i = 0; i < adev->num_ip_blocks; i++) { in amdgpu_device_ip_resume_phase2()
3682 if (!adev->ip_blocks[i].status.valid || adev->ip_blocks[i].status.hw) in amdgpu_device_ip_resume_phase2()
3684 if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_COMMON || in amdgpu_device_ip_resume_phase2()
3685 adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GMC || in amdgpu_device_ip_resume_phase2()
3686 adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_IH || in amdgpu_device_ip_resume_phase2()
3687 adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_PSP) in amdgpu_device_ip_resume_phase2()
3689 r = adev->ip_blocks[i].version->funcs->resume(adev); in amdgpu_device_ip_resume_phase2()
3692 adev->ip_blocks[i].version->funcs->name, r); in amdgpu_device_ip_resume_phase2()
3695 adev->ip_blocks[i].status.hw = true; in amdgpu_device_ip_resume_phase2()
3702 * amdgpu_device_ip_resume - run resume for hardware IPs
3727 if (adev->mman.buffer_funcs_ring->sched.ready) in amdgpu_device_ip_resume()
3734 * amdgpu_device_detect_sriov_bios - determine if the board supports SR-IOV
3738 * Query the VBIOS data tables to determine if the board supports SR-IOV.
3743 if (adev->is_atom_fw) { in amdgpu_device_detect_sriov_bios()
3745 adev->virt.caps |= AMDGPU_SRIOV_CAPS_SRIOV_VBIOS; in amdgpu_device_detect_sriov_bios()
3748 adev->virt.caps |= AMDGPU_SRIOV_CAPS_SRIOV_VBIOS; in amdgpu_device_detect_sriov_bios()
3751 if (!(adev->virt.caps & AMDGPU_SRIOV_CAPS_SRIOV_VBIOS)) in amdgpu_device_detect_sriov_bios()
3757 * amdgpu_device_asic_has_dc_support - determine if DC supports the asic
3782 * Fallback to the non-DC driver here by default so as not to in amdgpu_device_asic_has_dc_support()
3798 * Fallback to the non-DC driver here by default so as not to in amdgpu_device_asic_has_dc_support()
3814 * amdgpu_device_has_dc_support - check if dc is supported
3822 if (adev->enable_virtual_display || in amdgpu_device_has_dc_support()
3823 (adev->harvest_ip_mask & AMD_HARVEST_IP_DMU_MASK)) in amdgpu_device_has_dc_support()
3826 return amdgpu_device_asic_has_dc_support(adev->asic_type); in amdgpu_device_has_dc_support()
3847 task_barrier_enter(&hive->tb); in amdgpu_device_xgmi_reset_func()
3848 adev->asic_reset_res = amdgpu_device_baco_enter(adev_to_drm(adev)); in amdgpu_device_xgmi_reset_func()
3850 if (adev->asic_reset_res) in amdgpu_device_xgmi_reset_func()
3853 task_barrier_exit(&hive->tb); in amdgpu_device_xgmi_reset_func()
3854 adev->asic_reset_res = amdgpu_device_baco_exit(adev_to_drm(adev)); in amdgpu_device_xgmi_reset_func()
3856 if (adev->asic_reset_res) in amdgpu_device_xgmi_reset_func()
3862 task_barrier_full(&hive->tb); in amdgpu_device_xgmi_reset_func()
3863 adev->asic_reset_res = amdgpu_asic_reset(adev); in amdgpu_device_xgmi_reset_func()
3867 if (adev->asic_reset_res) in amdgpu_device_xgmi_reset_func()
3869 adev->asic_reset_res, adev_to_drm(adev)->unique); in amdgpu_device_xgmi_reset_func()
3884 * In SR-IOV or passthrough mode, timeout for compute in amdgpu_device_get_job_timeout_settings()
3887 adev->gfx_timeout = msecs_to_jiffies(10000); in amdgpu_device_get_job_timeout_settings()
3888 adev->sdma_timeout = adev->video_timeout = adev->gfx_timeout; in amdgpu_device_get_job_timeout_settings()
3890 adev->compute_timeout = amdgpu_sriov_is_pp_one_vf(adev) ? in amdgpu_device_get_job_timeout_settings()
3893 adev->compute_timeout = msecs_to_jiffies(60000); in amdgpu_device_get_job_timeout_settings()
3907 dev_warn(adev->dev, "lockup timeout disabled"); in amdgpu_device_get_job_timeout_settings()
3915 adev->gfx_timeout = timeout; in amdgpu_device_get_job_timeout_settings()
3918 adev->compute_timeout = timeout; in amdgpu_device_get_job_timeout_settings()
3921 adev->sdma_timeout = timeout; in amdgpu_device_get_job_timeout_settings()
3924 adev->video_timeout = timeout; in amdgpu_device_get_job_timeout_settings()
3932 * it should apply to all non-compute jobs. in amdgpu_device_get_job_timeout_settings()
3935 adev->sdma_timeout = adev->video_timeout = adev->gfx_timeout; in amdgpu_device_get_job_timeout_settings()
3937 adev->compute_timeout = adev->gfx_timeout; in amdgpu_device_get_job_timeout_settings()
3945 * amdgpu_device_check_iommu_direct_map - check if RAM direct mapped to GPU
3955 domain = iommu_get_domain_for_dev(adev->dev); in amdgpu_device_check_iommu_direct_map()
3956 if (!domain || domain->type == IOMMU_DOMAIN_IDENTITY) in amdgpu_device_check_iommu_direct_map()
3957 adev->ram_is_direct_mapped = true; in amdgpu_device_check_iommu_direct_map()
3962 * amdgpu_device_check_iommu_remap - Check if DMA remapping is enabled.
3972 domain = iommu_get_domain_for_dev(adev->dev); in amdgpu_device_check_iommu_remap()
3973 if (domain && (domain->type == IOMMU_DOMAIN_DMA || in amdgpu_device_check_iommu_remap()
3974 domain->type == IOMMU_DOMAIN_DMA_FQ)) in amdgpu_device_check_iommu_remap()
3989 adev->gfx.mcbp = true; in amdgpu_device_set_mcbp()
3991 adev->gfx.mcbp = false; in amdgpu_device_set_mcbp()
3994 adev->gfx.mcbp = true; in amdgpu_device_set_mcbp()
3996 if (adev->gfx.mcbp) in amdgpu_device_set_mcbp()
4001 * amdgpu_device_init - initialize the driver
4014 struct pci_dev *pdev = adev->pdev; in amdgpu_device_init()
4020 adev->shutdown = false; in amdgpu_device_init()
4021 adev->flags = flags; in amdgpu_device_init()
4024 adev->asic_type = amdgpu_force_asic_type; in amdgpu_device_init()
4026 adev->asic_type = flags & AMD_ASIC_MASK; in amdgpu_device_init()
4028 adev->usec_timeout = AMDGPU_MAX_USEC_TIMEOUT; in amdgpu_device_init()
4030 adev->usec_timeout *= 10; in amdgpu_device_init()
4031 adev->gmc.gart_size = 512 * 1024 * 1024; in amdgpu_device_init()
4032 adev->accel_working = false; in amdgpu_device_init()
4033 adev->num_rings = 0; in amdgpu_device_init()
4034 RCU_INIT_POINTER(adev->gang_submit, dma_fence_get_stub()); in amdgpu_device_init()
4035 adev->mman.buffer_funcs = NULL; in amdgpu_device_init()
4036 adev->mman.buffer_funcs_ring = NULL; in amdgpu_device_init()
4037 adev->vm_manager.vm_pte_funcs = NULL; in amdgpu_device_init()
4038 adev->vm_manager.vm_pte_num_scheds = 0; in amdgpu_device_init()
4039 adev->gmc.gmc_funcs = NULL; in amdgpu_device_init()
4040 adev->harvest_ip_mask = 0x0; in amdgpu_device_init()
4041 adev->fence_context = dma_fence_context_alloc(AMDGPU_MAX_RINGS); in amdgpu_device_init()
4042 bitmap_zero(adev->gfx.pipe_reserve_bitmap, AMDGPU_MAX_COMPUTE_QUEUES); in amdgpu_device_init()
4044 adev->smc_rreg = &amdgpu_invalid_rreg; in amdgpu_device_init()
4045 adev->smc_wreg = &amdgpu_invalid_wreg; in amdgpu_device_init()
4046 adev->pcie_rreg = &amdgpu_invalid_rreg; in amdgpu_device_init()
4047 adev->pcie_wreg = &amdgpu_invalid_wreg; in amdgpu_device_init()
4048 adev->pcie_rreg_ext = &amdgpu_invalid_rreg_ext; in amdgpu_device_init()
4049 adev->pcie_wreg_ext = &amdgpu_invalid_wreg_ext; in amdgpu_device_init()
4050 adev->pciep_rreg = &amdgpu_invalid_rreg; in amdgpu_device_init()
4051 adev->pciep_wreg = &amdgpu_invalid_wreg; in amdgpu_device_init()
4052 adev->pcie_rreg64 = &amdgpu_invalid_rreg64; in amdgpu_device_init()
4053 adev->pcie_wreg64 = &amdgpu_invalid_wreg64; in amdgpu_device_init()
4054 adev->pcie_rreg64_ext = &amdgpu_invalid_rreg64_ext; in amdgpu_device_init()
4055 adev->pcie_wreg64_ext = &amdgpu_invalid_wreg64_ext; in amdgpu_device_init()
4056 adev->uvd_ctx_rreg = &amdgpu_invalid_rreg; in amdgpu_device_init()
4057 adev->uvd_ctx_wreg = &amdgpu_invalid_wreg; in amdgpu_device_init()
4058 adev->didt_rreg = &amdgpu_invalid_rreg; in amdgpu_device_init()
4059 adev->didt_wreg = &amdgpu_invalid_wreg; in amdgpu_device_init()
4060 adev->gc_cac_rreg = &amdgpu_invalid_rreg; in amdgpu_device_init()
4061 adev->gc_cac_wreg = &amdgpu_invalid_wreg; in amdgpu_device_init()
4062 adev->audio_endpt_rreg = &amdgpu_block_invalid_rreg; in amdgpu_device_init()
4063 adev->audio_endpt_wreg = &amdgpu_block_invalid_wreg; in amdgpu_device_init()
4066 amdgpu_asic_name[adev->asic_type], pdev->vendor, pdev->device, in amdgpu_device_init()
4067 pdev->subsystem_vendor, pdev->subsystem_device, pdev->revision); in amdgpu_device_init()
4072 mutex_init(&adev->firmware.mutex); in amdgpu_device_init()
4073 mutex_init(&adev->pm.mutex); in amdgpu_device_init()
4074 mutex_init(&adev->gfx.gpu_clock_mutex); in amdgpu_device_init()
4075 mutex_init(&adev->srbm_mutex); in amdgpu_device_init()
4076 mutex_init(&adev->gfx.pipe_reserve_mutex); in amdgpu_device_init()
4077 mutex_init(&adev->gfx.gfx_off_mutex); in amdgpu_device_init()
4078 mutex_init(&adev->gfx.partition_mutex); in amdgpu_device_init()
4079 mutex_init(&adev->grbm_idx_mutex); in amdgpu_device_init()
4080 mutex_init(&adev->mn_lock); in amdgpu_device_init()
4081 mutex_init(&adev->virt.vf_errors.lock); in amdgpu_device_init()
4082 mutex_init(&adev->virt.rlcg_reg_lock); in amdgpu_device_init()
4083 hash_init(adev->mn_hash); in amdgpu_device_init()
4084 mutex_init(&adev->psp.mutex); in amdgpu_device_init()
4085 mutex_init(&adev->notifier_lock); in amdgpu_device_init()
4086 mutex_init(&adev->pm.stable_pstate_ctx_lock); in amdgpu_device_init()
4087 mutex_init(&adev->benchmark_mutex); in amdgpu_device_init()
4088 mutex_init(&adev->gfx.reset_sem_mutex); in amdgpu_device_init()
4090 mutex_init(&adev->enforce_isolation_mutex); in amdgpu_device_init()
4091 mutex_init(&adev->gfx.kfd_sch_mutex); in amdgpu_device_init()
4099 spin_lock_init(&adev->mmio_idx_lock); in amdgpu_device_init()
4100 spin_lock_init(&adev->smc_idx_lock); in amdgpu_device_init()
4101 spin_lock_init(&adev->pcie_idx_lock); in amdgpu_device_init()
4102 spin_lock_init(&adev->uvd_ctx_idx_lock); in amdgpu_device_init()
4103 spin_lock_init(&adev->didt_idx_lock); in amdgpu_device_init()
4104 spin_lock_init(&adev->gc_cac_idx_lock); in amdgpu_device_init()
4105 spin_lock_init(&adev->se_cac_idx_lock); in amdgpu_device_init()
4106 spin_lock_init(&adev->audio_endpt_idx_lock); in amdgpu_device_init()
4107 spin_lock_init(&adev->mm_stats.lock); in amdgpu_device_init()
4108 spin_lock_init(&adev->wb.lock); in amdgpu_device_init()
4110 INIT_LIST_HEAD(&adev->reset_list); in amdgpu_device_init()
4112 INIT_LIST_HEAD(&adev->ras_list); in amdgpu_device_init()
4114 INIT_LIST_HEAD(&adev->pm.od_kobj_list); in amdgpu_device_init()
4116 INIT_DELAYED_WORK(&adev->delayed_init_work, in amdgpu_device_init()
4118 INIT_DELAYED_WORK(&adev->gfx.gfx_off_delay_work, in amdgpu_device_init()
4130 INIT_DELAYED_WORK(&adev->gfx.enforce_isolation[i].work, in amdgpu_device_init()
4132 adev->gfx.enforce_isolation[i].adev = adev; in amdgpu_device_init()
4133 adev->gfx.enforce_isolation[i].xcp_id = i; in amdgpu_device_init()
4136 INIT_WORK(&adev->xgmi_reset_work, amdgpu_device_xgmi_reset_func); in amdgpu_device_init()
4138 adev->gfx.gfx_off_req_count = 1; in amdgpu_device_init()
4139 adev->gfx.gfx_off_residency = 0; in amdgpu_device_init()
4140 adev->gfx.gfx_off_entrycount = 0; in amdgpu_device_init()
4141 adev->pm.ac_power = power_supply_is_system_supplied() > 0; in amdgpu_device_init()
4143 atomic_set(&adev->throttling_logging_enabled, 1); in amdgpu_device_init()
4146 * to avoid log flooding. "-1" is subtracted since the thermal in amdgpu_device_init()
4151 ratelimit_state_init(&adev->throttling_logging_rs, (60 - 1) * HZ, 1); in amdgpu_device_init()
4152 ratelimit_set_flags(&adev->throttling_logging_rs, RATELIMIT_MSG_ON_RELEASE); in amdgpu_device_init()
4156 if (adev->asic_type >= CHIP_BONAIRE) { in amdgpu_device_init()
4157 adev->rmmio_base = pci_resource_start(adev->pdev, 5); in amdgpu_device_init()
4158 adev->rmmio_size = pci_resource_len(adev->pdev, 5); in amdgpu_device_init()
4160 adev->rmmio_base = pci_resource_start(adev->pdev, 2); in amdgpu_device_init()
4161 adev->rmmio_size = pci_resource_len(adev->pdev, 2); in amdgpu_device_init()
4165 atomic_set(&adev->pm.pwr_state[i], POWER_STATE_UNKNOWN); in amdgpu_device_init()
4167 adev->rmmio = ioremap(adev->rmmio_base, adev->rmmio_size); in amdgpu_device_init()
4168 if (!adev->rmmio) in amdgpu_device_init()
4169 return -ENOMEM; in amdgpu_device_init()
4171 DRM_INFO("register mmio base: 0x%08X\n", (uint32_t)adev->rmmio_base); in amdgpu_device_init()
4172 DRM_INFO("register mmio size: %u\n", (unsigned int)adev->rmmio_size); in amdgpu_device_init()
4179 adev->reset_domain = amdgpu_reset_create_reset_domain(SINGLE_DEVICE, "amdgpu-reset-dev"); in amdgpu_device_init()
4180 if (!adev->reset_domain) in amdgpu_device_init()
4181 return -ENOMEM; in amdgpu_device_init()
4190 dev_err(adev->dev, "invalid lockup_timeout parameter syntax\n"); in amdgpu_device_init()
4202 r = drm_aperture_remove_conflicting_pci_framebuffers(adev->pdev, &amdgpu_kms_driver); in amdgpu_device_init()
4214 adev->virt.caps |= AMDGPU_VF_MMIO_ACCESS_PROTECT; in amdgpu_device_init()
4218 if (adev->gmc.xgmi.supported) { in amdgpu_device_init()
4219 r = adev->gfxhub.funcs->get_xgmi_info(adev); in amdgpu_device_init()
4224 /* enable PCIE atomic ops */ in amdgpu_device_init()
4226 if (adev->virt.fw_reserve.p_pf2vf) in amdgpu_device_init()
4227 adev->have_atomics_support = ((struct amd_sriov_msg_pf2vf_info *) in amdgpu_device_init()
4228 adev->virt.fw_reserve.p_pf2vf)->pcie_atomic_ops_support_flags == in amdgpu_device_init()
4230 /* APUs w/ gfx9 onwards doesn't reply on PCIe atomics, rather it is a in amdgpu_device_init()
4233 } else if ((adev->flags & AMD_IS_APU) && in amdgpu_device_init()
4236 adev->have_atomics_support = true; in amdgpu_device_init()
4238 adev->have_atomics_support = in amdgpu_device_init()
4239 !pci_enable_atomic_ops_to_root(adev->pdev, in amdgpu_device_init()
4244 if (!adev->have_atomics_support) in amdgpu_device_init()
4245 dev_info(adev->dev, "PCIE atomic ops is not supported\n"); in amdgpu_device_init()
4259 if (adev->bios) in amdgpu_device_init()
4266 if (adev->gmc.xgmi.num_physical_nodes) { in amdgpu_device_init()
4267 dev_info(adev->dev, "Pending hive reset.\n"); in amdgpu_device_init()
4268 adev->gmc.xgmi.pending_reset = true; in amdgpu_device_init()
4270 for (i = 0; i < adev->num_ip_blocks; i++) { in amdgpu_device_init()
4271 if (!adev->ip_blocks[i].status.valid) in amdgpu_device_init()
4273 if (!(adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GMC || in amdgpu_device_init()
4274 adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_COMMON || in amdgpu_device_init()
4275 adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_IH || in amdgpu_device_init()
4276 adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_SMC)) { in amdgpu_device_init()
4278 adev->ip_blocks[i].version->funcs->name); in amdgpu_device_init()
4279 adev->ip_blocks[i].status.hw = true; in amdgpu_device_init()
4296 dev_err(adev->dev, "asic reset on init failed\n"); in amdgpu_device_init()
4303 if (!adev->bios) { in amdgpu_device_init()
4304 dev_err(adev->dev, "no vBIOS found\n"); in amdgpu_device_init()
4305 r = -EINVAL; in amdgpu_device_init()
4311 dev_err(adev->dev, "gpu post error!\n"); in amdgpu_device_init()
4316 if (adev->bios) { in amdgpu_device_init()
4317 if (adev->is_atom_fw) { in amdgpu_device_init()
4321 dev_err(adev->dev, "amdgpu_atomfirmware_get_clock_info failed\n"); in amdgpu_device_init()
4329 dev_err(adev->dev, "amdgpu_atombios_get_clock_info failed\n"); in amdgpu_device_init()
4343 dev_err(adev->dev, "amdgpu_fence_driver_sw_init failed\n"); in amdgpu_device_init()
4353 dev_err(adev->dev, "amdgpu_device_ip_init failed\n"); in amdgpu_device_init()
4360 dev_info(adev->dev, in amdgpu_device_init()
4362 adev->gfx.config.max_shader_engines, in amdgpu_device_init()
4363 adev->gfx.config.max_sh_per_se, in amdgpu_device_init()
4364 adev->gfx.config.max_cu_per_sh, in amdgpu_device_init()
4365 adev->gfx.cu_info.number); in amdgpu_device_init()
4367 adev->accel_working = true; in amdgpu_device_init()
4377 adev->mm_stats.log2_max_MBps = ilog2(max(1u, max_MBps)); in amdgpu_device_init()
4389 if (!adev->gmc.xgmi.pending_reset) { in amdgpu_device_init()
4392 dev_err(adev->dev, "amdgpu_device_ip_late_init failed\n"); in amdgpu_device_init()
4398 queue_delayed_work(system_wq, &adev->delayed_init_work, in amdgpu_device_init()
4404 flush_delayed_work(&adev->delayed_init_work); in amdgpu_device_init()
4414 drm_err(&adev->ddev, in amdgpu_device_init()
4423 adev->ucode_sysfs_en = false; in amdgpu_device_init()
4426 adev->ucode_sysfs_en = true; in amdgpu_device_init()
4428 r = sysfs_create_files(&adev->dev->kobj, amdgpu_dev_attributes); in amdgpu_device_init()
4430 dev_err(adev->dev, "Could not create amdgpu device attr\n"); in amdgpu_device_init()
4432 r = devm_device_add_group(adev->dev, &amdgpu_board_attrs_group); in amdgpu_device_init()
4434 dev_err(adev->dev, in amdgpu_device_init()
4443 dev_err(adev->dev, "amdgpu_pmu_init failed\n"); in amdgpu_device_init()
4446 if (amdgpu_device_cache_pci_state(adev->pdev)) in amdgpu_device_init()
4453 if ((adev->pdev->class >> 8) == PCI_CLASS_DISPLAY_VGA) in amdgpu_device_init()
4454 vga_client_register(adev->pdev, amdgpu_device_vga_set_decode); in amdgpu_device_init()
4458 if (px || (!dev_is_removable(&adev->pdev->dev) && in amdgpu_device_init()
4460 vga_switcheroo_register_client(adev->pdev, in amdgpu_device_init()
4464 vga_switcheroo_init_domain_pm_ops(adev->dev, &adev->vga_pm_domain); in amdgpu_device_init()
4466 if (adev->gmc.xgmi.pending_reset) in amdgpu_device_init()
4483 dev_err(adev->dev, "VF exclusive mode timeout\n"); in amdgpu_device_init()
4485 adev->virt.caps &= ~AMDGPU_SRIOV_CAPS_RUNTIME; in amdgpu_device_init()
4486 adev->virt.ops = NULL; in amdgpu_device_init()
4487 r = -EAGAIN; in amdgpu_device_init()
4501 unmap_mapping_range(adev->ddev.anon_inode->i_mapping, 0, 0, 1); in amdgpu_device_unmap_mmio()
4503 /* Unmap all mapped bars - Doorbell, registers and VRAM */ in amdgpu_device_unmap_mmio()
4506 iounmap(adev->rmmio); in amdgpu_device_unmap_mmio()
4507 adev->rmmio = NULL; in amdgpu_device_unmap_mmio()
4508 if (adev->mman.aper_base_kaddr) in amdgpu_device_unmap_mmio()
4509 iounmap(adev->mman.aper_base_kaddr); in amdgpu_device_unmap_mmio()
4510 adev->mman.aper_base_kaddr = NULL; in amdgpu_device_unmap_mmio()
4513 if (!adev->gmc.xgmi.connected_to_cpu && !adev->gmc.is_app_apu) { in amdgpu_device_unmap_mmio()
4514 arch_phys_wc_del(adev->gmc.vram_mtrr); in amdgpu_device_unmap_mmio()
4515 arch_io_free_memtype_wc(adev->gmc.aper_base, adev->gmc.aper_size); in amdgpu_device_unmap_mmio()
4520 * amdgpu_device_fini_hw - tear down the driver
4529 dev_info(adev->dev, "amdgpu: finishing device.\n"); in amdgpu_device_fini_hw()
4530 flush_delayed_work(&adev->delayed_init_work); in amdgpu_device_fini_hw()
4532 if (adev->mman.initialized) in amdgpu_device_fini_hw()
4533 drain_workqueue(adev->mman.bdev.wq); in amdgpu_device_fini_hw()
4534 adev->shutdown = true; in amdgpu_device_fini_hw()
4546 if (adev->mode_info.mode_config_initialized) { in amdgpu_device_fini_hw()
4554 if (adev->pm.sysfs_initialized) in amdgpu_device_fini_hw()
4556 if (adev->ucode_sysfs_en) in amdgpu_device_fini_hw()
4558 sysfs_remove_files(&adev->dev->kobj, amdgpu_dev_attributes); in amdgpu_device_fini_hw()
4572 if (adev->mman.initialized) in amdgpu_device_fini_hw()
4573 ttm_device_clear_dma_mappings(&adev->mman.bdev); in amdgpu_device_fini_hw()
4589 amdgpu_ucode_release(&adev->firmware.gpu_info_fw); in amdgpu_device_fini_sw()
4590 adev->accel_working = false; in amdgpu_device_fini_sw()
4591 dma_fence_put(rcu_dereference_protected(adev->gang_submit, true)); in amdgpu_device_fini_sw()
4602 kfree(adev->bios); in amdgpu_device_fini_sw()
4603 adev->bios = NULL; in amdgpu_device_fini_sw()
4605 kfree(adev->fru_info); in amdgpu_device_fini_sw()
4606 adev->fru_info = NULL; in amdgpu_device_fini_sw()
4610 if (px || (!dev_is_removable(&adev->pdev->dev) && in amdgpu_device_fini_sw()
4612 vga_switcheroo_unregister_client(adev->pdev); in amdgpu_device_fini_sw()
4615 vga_switcheroo_fini_domain_pm_ops(adev->dev); in amdgpu_device_fini_sw()
4617 if ((adev->pdev->class >> 8) == PCI_CLASS_DISPLAY_VGA) in amdgpu_device_fini_sw()
4618 vga_client_unregister(adev->pdev); in amdgpu_device_fini_sw()
4622 iounmap(adev->rmmio); in amdgpu_device_fini_sw()
4623 adev->rmmio = NULL; in amdgpu_device_fini_sw()
4630 if (adev->mman.discovery_bin) in amdgpu_device_fini_sw()
4633 amdgpu_reset_put_reset_domain(adev->reset_domain); in amdgpu_device_fini_sw()
4634 adev->reset_domain = NULL; in amdgpu_device_fini_sw()
4636 kfree(adev->pci_state); in amdgpu_device_fini_sw()
4641 * amdgpu_device_evict_resources - evict device resources
4654 if ((adev->in_s3 || adev->in_s0ix) && (adev->flags & AMD_IS_APU)) in amdgpu_device_evict_resources()
4667 * amdgpu_device_prepare - prepare for device suspend
4682 if (dev->switch_power_state == DRM_SWITCH_POWER_OFF) in amdgpu_device_prepare()
4690 flush_delayed_work(&adev->gfx.gfx_off_delay_work); in amdgpu_device_prepare()
4692 for (i = 0; i < adev->num_ip_blocks; i++) { in amdgpu_device_prepare()
4693 if (!adev->ip_blocks[i].status.valid) in amdgpu_device_prepare()
4695 if (!adev->ip_blocks[i].version->funcs->prepare_suspend) in amdgpu_device_prepare()
4697 r = adev->ip_blocks[i].version->funcs->prepare_suspend((void *)adev); in amdgpu_device_prepare()
4705 adev->in_s0ix = adev->in_s3 = false; in amdgpu_device_prepare()
4711 * amdgpu_device_suspend - initiate device suspend
4725 if (dev->switch_power_state == DRM_SWITCH_POWER_OFF) in amdgpu_device_suspend()
4728 adev->in_suspend = true; in amdgpu_device_suspend()
4741 drm_fb_helper_set_suspend_unlocked(adev_to_drm(adev)->fb_helper, true); in amdgpu_device_suspend()
4743 cancel_delayed_work_sync(&adev->delayed_init_work); in amdgpu_device_suspend()
4749 if (!adev->in_s0ix) in amdgpu_device_suspend()
4750 amdgpu_amdkfd_suspend(adev, adev->in_runpm); in amdgpu_device_suspend()
4773 * amdgpu_device_resume - initiate device resume
4793 if (dev->switch_power_state == DRM_SWITCH_POWER_OFF) in amdgpu_device_resume()
4796 if (adev->in_s0ix) in amdgpu_device_resume()
4803 dev_err(adev->dev, "amdgpu asic init failed\n"); in amdgpu_device_resume()
4809 dev_err(adev->dev, "amdgpu_device_ip_resume failed (%d).\n", r); in amdgpu_device_resume()
4814 if (!adev->in_s0ix) { in amdgpu_device_resume()
4815 r = amdgpu_amdkfd_resume(adev, adev->in_runpm); in amdgpu_device_resume()
4824 queue_delayed_work(system_wq, &adev->delayed_init_work, in amdgpu_device_resume()
4836 flush_delayed_work(&adev->delayed_init_work); in amdgpu_device_resume()
4839 drm_fb_helper_set_suspend_unlocked(adev_to_drm(adev)->fb_helper, false); in amdgpu_device_resume()
4843 if (adev->mode_info.num_crtc) { in amdgpu_device_resume()
4854 dev->dev->power.disable_depth++; in amdgpu_device_resume()
4856 if (!adev->dc_enabled) in amdgpu_device_resume()
4861 dev->dev->power.disable_depth--; in amdgpu_device_resume()
4864 adev->in_suspend = false; in amdgpu_device_resume()
4866 if (adev->enable_mes) in amdgpu_device_resume()
4876 * amdgpu_device_ip_check_soft_reset - did soft reset succeed
4896 for (i = 0; i < adev->num_ip_blocks; i++) { in amdgpu_device_ip_check_soft_reset()
4897 if (!adev->ip_blocks[i].status.valid) in amdgpu_device_ip_check_soft_reset()
4899 if (adev->ip_blocks[i].version->funcs->check_soft_reset) in amdgpu_device_ip_check_soft_reset()
4900 adev->ip_blocks[i].status.hang = in amdgpu_device_ip_check_soft_reset()
4901 adev->ip_blocks[i].version->funcs->check_soft_reset(adev); in amdgpu_device_ip_check_soft_reset()
4902 if (adev->ip_blocks[i].status.hang) { in amdgpu_device_ip_check_soft_reset()
4903 dev_info(adev->dev, "IP block:%s is hung!\n", adev->ip_blocks[i].version->funcs->name); in amdgpu_device_ip_check_soft_reset()
4911 * amdgpu_device_ip_pre_soft_reset - prepare for soft reset
4925 for (i = 0; i < adev->num_ip_blocks; i++) { in amdgpu_device_ip_pre_soft_reset()
4926 if (!adev->ip_blocks[i].status.valid) in amdgpu_device_ip_pre_soft_reset()
4928 if (adev->ip_blocks[i].status.hang && in amdgpu_device_ip_pre_soft_reset()
4929 adev->ip_blocks[i].version->funcs->pre_soft_reset) { in amdgpu_device_ip_pre_soft_reset()
4930 r = adev->ip_blocks[i].version->funcs->pre_soft_reset(adev); in amdgpu_device_ip_pre_soft_reset()
4940 * amdgpu_device_ip_need_full_reset - check if a full asic reset is needed
4955 for (i = 0; i < adev->num_ip_blocks; i++) { in amdgpu_device_ip_need_full_reset()
4956 if (!adev->ip_blocks[i].status.valid) in amdgpu_device_ip_need_full_reset()
4958 if ((adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GMC) || in amdgpu_device_ip_need_full_reset()
4959 (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_SMC) || in amdgpu_device_ip_need_full_reset()
4960 (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_ACP) || in amdgpu_device_ip_need_full_reset()
4961 (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_DCE) || in amdgpu_device_ip_need_full_reset()
4962 adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_PSP) { in amdgpu_device_ip_need_full_reset()
4963 if (adev->ip_blocks[i].status.hang) { in amdgpu_device_ip_need_full_reset()
4964 dev_info(adev->dev, "Some block need full reset!\n"); in amdgpu_device_ip_need_full_reset()
4973 * amdgpu_device_ip_soft_reset - do a soft reset
4987 for (i = 0; i < adev->num_ip_blocks; i++) { in amdgpu_device_ip_soft_reset()
4988 if (!adev->ip_blocks[i].status.valid) in amdgpu_device_ip_soft_reset()
4990 if (adev->ip_blocks[i].status.hang && in amdgpu_device_ip_soft_reset()
4991 adev->ip_blocks[i].version->funcs->soft_reset) { in amdgpu_device_ip_soft_reset()
4992 r = adev->ip_blocks[i].version->funcs->soft_reset(adev); in amdgpu_device_ip_soft_reset()
5002 * amdgpu_device_ip_post_soft_reset - clean up from soft reset
5016 for (i = 0; i < adev->num_ip_blocks; i++) { in amdgpu_device_ip_post_soft_reset()
5017 if (!adev->ip_blocks[i].status.valid) in amdgpu_device_ip_post_soft_reset()
5019 if (adev->ip_blocks[i].status.hang && in amdgpu_device_ip_post_soft_reset()
5020 adev->ip_blocks[i].version->funcs->post_soft_reset) in amdgpu_device_ip_post_soft_reset()
5021 r = adev->ip_blocks[i].version->funcs->post_soft_reset(adev); in amdgpu_device_ip_post_soft_reset()
5030 * amdgpu_device_reset_sriov - reset ASIC for SR-IOV vf
5044 if (test_bit(AMDGPU_HOST_FLR, &reset_context->flags)) { in amdgpu_device_reset_sriov()
5048 clear_bit(AMDGPU_HOST_FLR, &reset_context->flags); in amdgpu_device_reset_sriov()
5080 if (hive && adev->gmc.xgmi.num_physical_nodes > 1) in amdgpu_device_reset_sriov()
5091 if (adev->virt.gim_feature & AMDGIM_FEATURE_GIM_FLR_VRAMLOST) in amdgpu_device_reset_sriov()
5095 * bare-metal does. in amdgpu_device_reset_sriov()
5110 * amdgpu_device_has_job_running - check if there is any job in mirror list
5114 * check if there is any job in mirror list
5122 struct amdgpu_ring *ring = adev->rings[i]; in amdgpu_device_has_job_running()
5127 spin_lock(&ring->sched.job_list_lock); in amdgpu_device_has_job_running()
5128 job = list_first_entry_or_null(&ring->sched.pending_list, in amdgpu_device_has_job_running()
5130 spin_unlock(&ring->sched.job_list_lock); in amdgpu_device_has_job_running()
5138 * amdgpu_device_should_recover_gpu - check if we should try GPU recovery
5158 if (amdgpu_gpu_recovery == -1) { in amdgpu_device_should_recover_gpu()
5159 switch (adev->asic_type) { in amdgpu_device_should_recover_gpu()
5184 dev_info(adev->dev, "GPU recovery disabled.\n"); in amdgpu_device_should_recover_gpu()
5195 dev_info(adev->dev, "GPU mode1 reset\n"); in amdgpu_device_mode1_reset()
5198 * values are used in other cases like restore after mode-2 reset. in amdgpu_device_mode1_reset()
5200 amdgpu_device_cache_pci_state(adev->pdev); in amdgpu_device_mode1_reset()
5203 pci_clear_master(adev->pdev); in amdgpu_device_mode1_reset()
5206 dev_info(adev->dev, "GPU smu mode1 reset\n"); in amdgpu_device_mode1_reset()
5209 dev_info(adev->dev, "GPU psp mode1 reset\n"); in amdgpu_device_mode1_reset()
5216 amdgpu_device_load_pci_state(adev->pdev); in amdgpu_device_mode1_reset()
5222 for (i = 0; i < adev->usec_timeout; i++) { in amdgpu_device_mode1_reset()
5223 u32 memsize = adev->nbio.funcs->get_memsize(adev); in amdgpu_device_mode1_reset()
5230 if (i >= adev->usec_timeout) { in amdgpu_device_mode1_reset()
5231 ret = -ETIMEDOUT; in amdgpu_device_mode1_reset()
5240 dev_err(adev->dev, "GPU mode1 reset failed\n"); in amdgpu_device_mode1_reset()
5249 struct amdgpu_device *tmp_adev = reset_context->reset_req_dev; in amdgpu_device_pre_asic_reset()
5251 test_bit(AMDGPU_NEED_FULL_RESET, &reset_context->flags); in amdgpu_device_pre_asic_reset()
5253 if (reset_context->reset_req_dev == adev) in amdgpu_device_pre_asic_reset()
5254 job = reset_context->job; in amdgpu_device_pre_asic_reset()
5263 struct amdgpu_ring *ring = adev->rings[i]; in amdgpu_device_pre_asic_reset()
5279 if (job && job->vm) in amdgpu_device_pre_asic_reset()
5280 drm_sched_increase_karma(&job->base); in amdgpu_device_pre_asic_reset()
5284 if (r == -EOPNOTSUPP) in amdgpu_device_pre_asic_reset()
5301 dev_info(adev->dev, "soft reset failed, will fallback to full reset!\n"); in amdgpu_device_pre_asic_reset()
5306 if (!test_bit(AMDGPU_SKIP_COREDUMP, &reset_context->flags)) { in amdgpu_device_pre_asic_reset()
5307 dev_info(tmp_adev->dev, "Dumping IP State\n"); in amdgpu_device_pre_asic_reset()
5309 for (i = 0; i < tmp_adev->num_ip_blocks; i++) in amdgpu_device_pre_asic_reset()
5310 if (tmp_adev->ip_blocks[i].version->funcs->dump_ip_state) in amdgpu_device_pre_asic_reset()
5311 tmp_adev->ip_blocks[i].version->funcs in amdgpu_device_pre_asic_reset()
5312 ->dump_ip_state((void *)tmp_adev); in amdgpu_device_pre_asic_reset()
5313 dev_info(tmp_adev->dev, "Dumping IP State Completed\n"); in amdgpu_device_pre_asic_reset()
5319 set_bit(AMDGPU_NEED_FULL_RESET, &reset_context->flags); in amdgpu_device_pre_asic_reset()
5322 &reset_context->flags); in amdgpu_device_pre_asic_reset()
5339 reset_context->reset_device_list = device_list_handle; in amdgpu_do_asic_reset()
5342 if (r == -EOPNOTSUPP) in amdgpu_do_asic_reset()
5349 test_bit(AMDGPU_NEED_FULL_RESET, &reset_context->flags); in amdgpu_do_asic_reset()
5350 skip_hw_reset = test_bit(AMDGPU_SKIP_HW_RESET, &reset_context->flags); in amdgpu_do_asic_reset()
5359 if (tmp_adev->gmc.xgmi.num_physical_nodes > 1) { in amdgpu_do_asic_reset()
5360 tmp_adev->gmc.xgmi.pending_reset = false; in amdgpu_do_asic_reset()
5361 if (!queue_work(system_unbound_wq, &tmp_adev->xgmi_reset_work)) in amdgpu_do_asic_reset()
5362 r = -EALREADY; in amdgpu_do_asic_reset()
5367 dev_err(tmp_adev->dev, "ASIC reset failed with error, %d for drm dev, %s", in amdgpu_do_asic_reset()
5368 r, adev_to_drm(tmp_adev)->unique); in amdgpu_do_asic_reset()
5376 if (tmp_adev->gmc.xgmi.num_physical_nodes > 1) { in amdgpu_do_asic_reset()
5377 flush_work(&tmp_adev->xgmi_reset_work); in amdgpu_do_asic_reset()
5378 r = tmp_adev->asic_reset_res; in amdgpu_do_asic_reset()
5400 dev_warn(tmp_adev->dev, "asic atom init failed!"); in amdgpu_do_asic_reset()
5402 dev_info(tmp_adev->dev, "GPU reset succeeded, trying to resume\n"); in amdgpu_do_asic_reset()
5410 if (!test_bit(AMDGPU_SKIP_COREDUMP, &reset_context->flags)) in amdgpu_do_asic_reset()
5411 amdgpu_coredump(tmp_adev, false, vram_lost, reset_context->job); in amdgpu_do_asic_reset()
5423 tmp_adev->xcp_mgr); in amdgpu_do_asic_reset()
5431 if (tmp_adev->mman.buffer_funcs_ring->sched.ready) in amdgpu_do_asic_reset()
5443 if (!reset_context->hive && in amdgpu_do_asic_reset()
5444 tmp_adev->gmc.xgmi.num_physical_nodes > 1) in amdgpu_do_asic_reset()
5451 drm_fb_helper_set_suspend_unlocked(adev_to_drm(tmp_adev)->fb_helper, false); in amdgpu_do_asic_reset()
5467 r = -EINVAL; in amdgpu_do_asic_reset()
5472 if (reset_context->hive && in amdgpu_do_asic_reset()
5473 tmp_adev->gmc.xgmi.num_physical_nodes > 1) in amdgpu_do_asic_reset()
5475 reset_context->hive, tmp_adev); in amdgpu_do_asic_reset()
5484 dev_err(tmp_adev->dev, "ib ring test failed (%d).\n", r); in amdgpu_do_asic_reset()
5486 r = -EAGAIN; in amdgpu_do_asic_reset()
5492 tmp_adev->asic_reset_res = r; in amdgpu_do_asic_reset()
5497 set_bit(AMDGPU_NEED_FULL_RESET, &reset_context->flags); in amdgpu_do_asic_reset()
5499 clear_bit(AMDGPU_NEED_FULL_RESET, &reset_context->flags); in amdgpu_do_asic_reset()
5508 adev->mp1_state = PP_MP1_STATE_SHUTDOWN; in amdgpu_device_set_mp1_state()
5511 adev->mp1_state = PP_MP1_STATE_RESET; in amdgpu_device_set_mp1_state()
5514 adev->mp1_state = PP_MP1_STATE_NONE; in amdgpu_device_set_mp1_state()
5522 adev->mp1_state = PP_MP1_STATE_NONE; in amdgpu_device_unset_mp1_state()
5529 p = pci_get_domain_bus_and_slot(pci_domain_nr(adev->pdev->bus), in amdgpu_device_resume_display_audio()
5530 adev->pdev->bus->number, 1); in amdgpu_device_resume_display_audio()
5532 pm_runtime_enable(&(p->dev)); in amdgpu_device_resume_display_audio()
5533 pm_runtime_resume(&(p->dev)); in amdgpu_device_resume_display_audio()
5552 return -EINVAL; in amdgpu_device_suspend_display_audio()
5554 p = pci_get_domain_bus_and_slot(pci_domain_nr(adev->pdev->bus), in amdgpu_device_suspend_display_audio()
5555 adev->pdev->bus->number, 1); in amdgpu_device_suspend_display_audio()
5557 return -ENODEV; in amdgpu_device_suspend_display_audio()
5559 expires = pm_runtime_autosuspend_expiration(&(p->dev)); in amdgpu_device_suspend_display_audio()
5569 while (!pm_runtime_status_suspended(&(p->dev))) { in amdgpu_device_suspend_display_audio()
5570 if (!pm_runtime_suspend(&(p->dev))) in amdgpu_device_suspend_display_audio()
5574 dev_warn(adev->dev, "failed to suspend display audio\n"); in amdgpu_device_suspend_display_audio()
5577 return -ETIMEDOUT; in amdgpu_device_suspend_display_audio()
5581 pm_runtime_disable(&(p->dev)); in amdgpu_device_suspend_display_audio()
5593 cancel_work(&adev->reset_work); in amdgpu_device_stop_pending_resets()
5596 if (adev->kfd.dev) in amdgpu_device_stop_pending_resets()
5597 cancel_work(&adev->kfd.reset_work); in amdgpu_device_stop_pending_resets()
5600 cancel_work(&adev->virt.flr_work); in amdgpu_device_stop_pending_resets()
5602 if (con && adev->ras_enabled) in amdgpu_device_stop_pending_resets()
5603 cancel_work(&con->recovery_work); in amdgpu_device_stop_pending_resets()
5614 pci_read_config_dword(tmp_adev->pdev, PCI_COMMAND, &status); in amdgpu_device_health_check()
5616 dev_err(tmp_adev->dev, "device lost from bus!"); in amdgpu_device_health_check()
5617 ret = -ENODEV; in amdgpu_device_health_check()
5625 * amdgpu_device_gpu_recover - reset the asic and recover scheduler
5632 * Attempt to do soft-reset or full-reset and reinitialize Asic
5659 amdgpu_ras_get_context(adev)->reboot) { in amdgpu_device_gpu_recover()
5666 dev_info(adev->dev, "GPU %s begin!\n", in amdgpu_device_gpu_recover()
5672 mutex_lock(&hive->hive_lock); in amdgpu_device_gpu_recover()
5674 reset_context->job = job; in amdgpu_device_gpu_recover()
5675 reset_context->hive = hive; in amdgpu_device_gpu_recover()
5682 if (!amdgpu_sriov_vf(adev) && (adev->gmc.xgmi.num_physical_nodes > 1) && hive) { in amdgpu_device_gpu_recover()
5683 list_for_each_entry(tmp_adev, &hive->device_list, gmc.xgmi.head) { in amdgpu_device_gpu_recover()
5684 list_add_tail(&tmp_adev->reset_list, &device_list); in amdgpu_device_gpu_recover()
5685 if (adev->shutdown) in amdgpu_device_gpu_recover()
5686 tmp_adev->shutdown = true; in amdgpu_device_gpu_recover()
5688 if (!list_is_first(&adev->reset_list, &device_list)) in amdgpu_device_gpu_recover()
5689 list_rotate_to_front(&adev->reset_list, &device_list); in amdgpu_device_gpu_recover()
5692 list_add_tail(&adev->reset_list, &device_list); in amdgpu_device_gpu_recover()
5705 amdgpu_device_lock_reset_domain(tmp_adev->reset_domain); in amdgpu_device_gpu_recover()
5727 cancel_delayed_work_sync(&tmp_adev->delayed_init_work); in amdgpu_device_gpu_recover()
5737 drm_fb_helper_set_suspend_unlocked(adev_to_drm(tmp_adev)->fb_helper, true); in amdgpu_device_gpu_recover()
5745 struct amdgpu_ring *ring = tmp_adev->rings[i]; in amdgpu_device_gpu_recover()
5750 drm_sched_stop(&ring->sched, job ? &job->base : NULL); in amdgpu_device_gpu_recover()
5753 amdgpu_job_stop_all_jobs_on_sched(&ring->sched); in amdgpu_device_gpu_recover()
5755 atomic_inc(&tmp_adev->gpu_reset_counter); in amdgpu_device_gpu_recover()
5765 * job->base holds a reference to parent fence in amdgpu_device_gpu_recover()
5767 if (job && dma_fence_is_signaled(&job->hw_fence)) { in amdgpu_device_gpu_recover()
5769 dev_info(adev->dev, "Guilty job already signaled, skipping HW reset"); in amdgpu_device_gpu_recover()
5778 dev_err(tmp_adev->dev, "GPU pre asic reset failed with err, %d for drm dev, %s ", in amdgpu_device_gpu_recover()
5779 r, adev_to_drm(tmp_adev)->unique); in amdgpu_device_gpu_recover()
5780 tmp_adev->asic_reset_res = r; in amdgpu_device_gpu_recover()
5788 dev_dbg(adev->dev, "Detected RAS error, wait for FLR completion\n"); in amdgpu_device_gpu_recover()
5790 set_bit(AMDGPU_HOST_FLR, &reset_context->flags); in amdgpu_device_gpu_recover()
5794 if (AMDGPU_RETRY_SRIOV_RESET(r) && (retry_limit--) > 0) { in amdgpu_device_gpu_recover()
5799 adev->asic_reset_res = r; in amdgpu_device_gpu_recover()
5802 if (r && r == -EAGAIN) in amdgpu_device_gpu_recover()
5822 struct amdgpu_ring *ring = tmp_adev->rings[i]; in amdgpu_device_gpu_recover()
5827 drm_sched_start(&ring->sched); in amdgpu_device_gpu_recover()
5833 if (tmp_adev->asic_reset_res) in amdgpu_device_gpu_recover()
5834 r = tmp_adev->asic_reset_res; in amdgpu_device_gpu_recover()
5836 tmp_adev->asic_reset_res = 0; in amdgpu_device_gpu_recover()
5843 if (reset_context->src != AMDGPU_RESET_SRC_RAS || in amdgpu_device_gpu_recover()
5845 dev_info(tmp_adev->dev, "GPU reset(%d) failed\n", in amdgpu_device_gpu_recover()
5846 atomic_read(&tmp_adev->gpu_reset_counter)); in amdgpu_device_gpu_recover()
5849 dev_info(tmp_adev->dev, "GPU reset(%d) succeeded!\n", atomic_read(&tmp_adev->gpu_reset_counter)); in amdgpu_device_gpu_recover()
5864 if (!adev->kfd.init_complete) in amdgpu_device_gpu_recover()
5877 amdgpu_device_unlock_reset_domain(tmp_adev->reset_domain); in amdgpu_device_gpu_recover()
5881 mutex_unlock(&hive->hive_lock); in amdgpu_device_gpu_recover()
5886 dev_info(adev->dev, "GPU reset end with ret = %d\n", r); in amdgpu_device_gpu_recover()
5888 atomic_set(&adev->reset_domain->reset_res, r); in amdgpu_device_gpu_recover()
5893 * amdgpu_device_partner_bandwidth - find the bandwidth of appropriate partner
5907 struct pci_dev *parent = adev->pdev; in amdgpu_device_partner_bandwidth()
5918 if (parent->vendor == PCI_VENDOR_ID_ATI) in amdgpu_device_partner_bandwidth()
5926 pcie_bandwidth_available(adev->pdev, NULL, speed, width); in amdgpu_device_partner_bandwidth()
5931 * amdgpu_device_get_pcie_info - fence pcie info about the PCIE slot
5935 * Fetchs and stores in the driver the PCIE capabilities (gen speed
5937 * virtualized environments where PCIE config space may not be available.
5946 adev->pm.pcie_gen_mask = amdgpu_pcie_gen_cap; in amdgpu_device_get_pcie_info()
5949 adev->pm.pcie_mlw_mask = amdgpu_pcie_lane_cap; in amdgpu_device_get_pcie_info()
5952 if (pci_is_root_bus(adev->pdev->bus) && !amdgpu_passthrough(adev)) { in amdgpu_device_get_pcie_info()
5953 if (adev->pm.pcie_gen_mask == 0) in amdgpu_device_get_pcie_info()
5954 adev->pm.pcie_gen_mask = AMDGPU_DEFAULT_PCIE_GEN_MASK; in amdgpu_device_get_pcie_info()
5955 if (adev->pm.pcie_mlw_mask == 0) in amdgpu_device_get_pcie_info()
5956 adev->pm.pcie_mlw_mask = AMDGPU_DEFAULT_PCIE_MLW_MASK; in amdgpu_device_get_pcie_info()
5960 if (adev->pm.pcie_gen_mask && adev->pm.pcie_mlw_mask) in amdgpu_device_get_pcie_info()
5966 if (adev->pm.pcie_gen_mask == 0) { in amdgpu_device_get_pcie_info()
5968 pdev = adev->pdev; in amdgpu_device_get_pcie_info()
5971 adev->pm.pcie_gen_mask |= (CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN1 | in amdgpu_device_get_pcie_info()
5976 adev->pm.pcie_gen_mask |= (CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN1 | in amdgpu_device_get_pcie_info()
5982 adev->pm.pcie_gen_mask |= (CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN1 | in amdgpu_device_get_pcie_info()
5987 adev->pm.pcie_gen_mask |= (CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN1 | in amdgpu_device_get_pcie_info()
5991 adev->pm.pcie_gen_mask |= (CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN1 | in amdgpu_device_get_pcie_info()
5994 adev->pm.pcie_gen_mask |= CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN1; in amdgpu_device_get_pcie_info()
5998 adev->pm.pcie_gen_mask |= (CAIL_PCIE_LINK_SPEED_SUPPORT_GEN1 | in amdgpu_device_get_pcie_info()
6002 adev->pm.pcie_gen_mask |= (CAIL_PCIE_LINK_SPEED_SUPPORT_GEN1 | in amdgpu_device_get_pcie_info()
6008 adev->pm.pcie_gen_mask |= (CAIL_PCIE_LINK_SPEED_SUPPORT_GEN1 | in amdgpu_device_get_pcie_info()
6013 adev->pm.pcie_gen_mask |= (CAIL_PCIE_LINK_SPEED_SUPPORT_GEN1 | in amdgpu_device_get_pcie_info()
6017 adev->pm.pcie_gen_mask |= (CAIL_PCIE_LINK_SPEED_SUPPORT_GEN1 | in amdgpu_device_get_pcie_info()
6020 adev->pm.pcie_gen_mask |= CAIL_PCIE_LINK_SPEED_SUPPORT_GEN1; in amdgpu_device_get_pcie_info()
6024 if (adev->pm.pcie_mlw_mask == 0) { in amdgpu_device_get_pcie_info()
6026 adev->pm.pcie_mlw_mask |= AMDGPU_DEFAULT_PCIE_MLW_MASK; in amdgpu_device_get_pcie_info()
6030 adev->pm.pcie_mlw_mask = (CAIL_PCIE_LINK_WIDTH_SUPPORT_X32 | in amdgpu_device_get_pcie_info()
6039 adev->pm.pcie_mlw_mask = (CAIL_PCIE_LINK_WIDTH_SUPPORT_X16 | in amdgpu_device_get_pcie_info()
6047 adev->pm.pcie_mlw_mask = (CAIL_PCIE_LINK_WIDTH_SUPPORT_X12 | in amdgpu_device_get_pcie_info()
6054 adev->pm.pcie_mlw_mask = (CAIL_PCIE_LINK_WIDTH_SUPPORT_X8 | in amdgpu_device_get_pcie_info()
6060 adev->pm.pcie_mlw_mask = (CAIL_PCIE_LINK_WIDTH_SUPPORT_X4 | in amdgpu_device_get_pcie_info()
6065 adev->pm.pcie_mlw_mask = (CAIL_PCIE_LINK_WIDTH_SUPPORT_X2 | in amdgpu_device_get_pcie_info()
6069 adev->pm.pcie_mlw_mask = CAIL_PCIE_LINK_WIDTH_SUPPORT_X1; in amdgpu_device_get_pcie_info()
6079 * amdgpu_device_is_peer_accessible - Check peer access through PCIe BAR
6084 * Return true if @peer_adev can access (DMA) @adev through the PCIe
6093 !adev->gmc.xgmi.connected_to_cpu && in amdgpu_device_is_peer_accessible()
6094 !(pci_p2pdma_distance(adev->pdev, peer_adev->dev, false) < 0); in amdgpu_device_is_peer_accessible()
6096 bool is_large_bar = adev->gmc.visible_vram_size && in amdgpu_device_is_peer_accessible()
6097 adev->gmc.real_vram_size == adev->gmc.visible_vram_size; in amdgpu_device_is_peer_accessible()
6101 uint64_t address_mask = peer_adev->dev->dma_mask ? in amdgpu_device_is_peer_accessible()
6102 ~*peer_adev->dev->dma_mask : ~((1ULL << 32) - 1); in amdgpu_device_is_peer_accessible()
6104 adev->gmc.aper_base + adev->gmc.aper_size - 1; in amdgpu_device_is_peer_accessible()
6106 p2p_addressable = !(adev->gmc.aper_base & address_mask || in amdgpu_device_is_peer_accessible()
6121 return -ENOTSUPP; in amdgpu_device_baco_enter()
6123 if (ras && adev->ras_enabled && in amdgpu_device_baco_enter()
6124 adev->nbio.funcs->enable_doorbell_interrupt) in amdgpu_device_baco_enter()
6125 adev->nbio.funcs->enable_doorbell_interrupt(adev, false); in amdgpu_device_baco_enter()
6137 return -ENOTSUPP; in amdgpu_device_baco_exit()
6143 if (ras && adev->ras_enabled && in amdgpu_device_baco_exit()
6144 adev->nbio.funcs->enable_doorbell_interrupt) in amdgpu_device_baco_exit()
6145 adev->nbio.funcs->enable_doorbell_interrupt(adev, true); in amdgpu_device_baco_exit()
6147 if (amdgpu_passthrough(adev) && adev->nbio.funcs && in amdgpu_device_baco_exit()
6148 adev->nbio.funcs->clear_doorbell_interrupt) in amdgpu_device_baco_exit()
6149 adev->nbio.funcs->clear_doorbell_interrupt(adev); in amdgpu_device_baco_exit()
6155 * amdgpu_pci_error_detected - Called when a PCI error is detected.
6171 if (adev->gmc.xgmi.num_physical_nodes > 1) { in amdgpu_pci_error_detected()
6176 adev->pci_channel_state = state; in amdgpu_pci_error_detected()
6184 * Locking adev->reset_domain->sem will prevent any external access in amdgpu_pci_error_detected()
6187 amdgpu_device_lock_reset_domain(adev->reset_domain); in amdgpu_pci_error_detected()
6195 struct amdgpu_ring *ring = adev->rings[i]; in amdgpu_pci_error_detected()
6200 drm_sched_stop(&ring->sched, NULL); in amdgpu_pci_error_detected()
6202 atomic_inc(&adev->gpu_reset_counter); in amdgpu_pci_error_detected()
6213 * amdgpu_pci_mmio_enabled - Enable MMIO and dump debug registers
6221 /* TODO - dump whatever for debugging purposes */ in amdgpu_pci_mmio_enabled()
6232 * amdgpu_pci_slot_reset - Called when PCI slot has been reset.
6259 list_add_tail(&adev->reset_list, &device_list); in amdgpu_pci_slot_reset()
6268 for (i = 0; i < adev->usec_timeout; i++) { in amdgpu_pci_slot_reset()
6276 r = -ETIME; in amdgpu_pci_slot_reset()
6285 adev->no_hw_access = true; in amdgpu_pci_slot_reset()
6287 adev->no_hw_access = false; in amdgpu_pci_slot_reset()
6295 if (amdgpu_device_cache_pci_state(adev->pdev)) in amdgpu_pci_slot_reset()
6296 pci_restore_state(adev->pdev); in amdgpu_pci_slot_reset()
6298 DRM_INFO("PCIe error recovery succeeded\n"); in amdgpu_pci_slot_reset()
6300 DRM_ERROR("PCIe error recovery failed, err:%d", r); in amdgpu_pci_slot_reset()
6302 amdgpu_device_unlock_reset_domain(adev->reset_domain); in amdgpu_pci_slot_reset()
6309 * amdgpu_pci_resume() - resume normal ops after PCI reset
6325 if (adev->pci_channel_state != pci_channel_io_frozen) in amdgpu_pci_resume()
6329 struct amdgpu_ring *ring = adev->rings[i]; in amdgpu_pci_resume()
6334 drm_sched_start(&ring->sched); in amdgpu_pci_resume()
6338 amdgpu_device_unlock_reset_domain(adev->reset_domain); in amdgpu_pci_resume()
6349 kfree(adev->pci_state); in amdgpu_device_cache_pci_state()
6351 adev->pci_state = pci_store_saved_state(pdev); in amdgpu_device_cache_pci_state()
6353 if (!adev->pci_state) { in amdgpu_device_cache_pci_state()
6371 if (!adev->pci_state) in amdgpu_device_load_pci_state()
6374 r = pci_load_saved_state(pdev, adev->pci_state); in amdgpu_device_load_pci_state()
6390 if ((adev->flags & AMD_IS_APU) && !amdgpu_passthrough(adev)) in amdgpu_device_flush_hdp()
6393 if (adev->gmc.xgmi.connected_to_cpu) in amdgpu_device_flush_hdp()
6396 if (ring && ring->funcs->emit_hdp_flush) in amdgpu_device_flush_hdp()
6406 if ((adev->flags & AMD_IS_APU) && !amdgpu_passthrough(adev)) in amdgpu_device_invalidate_hdp()
6409 if (adev->gmc.xgmi.connected_to_cpu) in amdgpu_device_invalidate_hdp()
6417 return atomic_read(&adev->reset_domain->in_gpu_reset); in amdgpu_in_reset()
6421 * amdgpu_device_halt() - bring hardware to some kind of halt state
6435 * 4. set adev->no_hw_access to avoid potential crashes after setp 5
6442 struct pci_dev *pdev = adev->pdev; in amdgpu_device_halt()
6452 adev->no_hw_access = true; in amdgpu_device_halt()
6466 address = adev->nbio.funcs->get_pcie_port_index_offset(adev); in amdgpu_device_pcie_port_rreg()
6467 data = adev->nbio.funcs->get_pcie_port_data_offset(adev); in amdgpu_device_pcie_port_rreg()
6469 spin_lock_irqsave(&adev->pcie_idx_lock, flags); in amdgpu_device_pcie_port_rreg()
6473 spin_unlock_irqrestore(&adev->pcie_idx_lock, flags); in amdgpu_device_pcie_port_rreg()
6482 address = adev->nbio.funcs->get_pcie_port_index_offset(adev); in amdgpu_device_pcie_port_wreg()
6483 data = adev->nbio.funcs->get_pcie_port_data_offset(adev); in amdgpu_device_pcie_port_wreg()
6485 spin_lock_irqsave(&adev->pcie_idx_lock, flags); in amdgpu_device_pcie_port_wreg()
6490 spin_unlock_irqrestore(&adev->pcie_idx_lock, flags); in amdgpu_device_pcie_port_wreg()
6494 * amdgpu_device_get_gang - return a reference to the current gang
6504 fence = dma_fence_get_rcu_safe(&adev->gang_submit); in amdgpu_device_get_gang()
6510 * amdgpu_device_switch_gang - switch to a new gang
6532 } while (cmpxchg((struct dma_fence __force **)&adev->gang_submit, in amdgpu_device_switch_gang()
6541 switch (adev->asic_type) { in amdgpu_device_has_display_hardware()
6574 (adev->harvest_ip_mask & AMD_HARVEST_IP_DMU_MASK)) in amdgpu_device_has_display_hardware()
6587 uint32_t loop = adev->usec_timeout; in amdgpu_device_wait_on_rreg()
6591 loop = adev->usec_timeout; in amdgpu_device_wait_on_rreg()
6596 loop--; in amdgpu_device_wait_on_rreg()
6601 ret = -ETIMEDOUT; in amdgpu_device_wait_on_rreg()