Lines Matching +full:9 +full:- +full:inch

379 	/* TCC (5 sub-ranges)*/
442 /* EA (3 sub-ranges)*/
854 mmRLC_SRM_INDEX_CNTL_ADDR_0 - mmRLC_SRM_INDEX_CNTL_ADDR_0,
855 mmRLC_SRM_INDEX_CNTL_ADDR_1 - mmRLC_SRM_INDEX_CNTL_ADDR_0,
856 mmRLC_SRM_INDEX_CNTL_ADDR_2 - mmRLC_SRM_INDEX_CNTL_ADDR_0,
857 mmRLC_SRM_INDEX_CNTL_ADDR_3 - mmRLC_SRM_INDEX_CNTL_ADDR_0,
858 mmRLC_SRM_INDEX_CNTL_ADDR_4 - mmRLC_SRM_INDEX_CNTL_ADDR_0,
859 mmRLC_SRM_INDEX_CNTL_ADDR_5 - mmRLC_SRM_INDEX_CNTL_ADDR_0,
860 mmRLC_SRM_INDEX_CNTL_ADDR_6 - mmRLC_SRM_INDEX_CNTL_ADDR_0,
861 mmRLC_SRM_INDEX_CNTL_ADDR_7 - mmRLC_SRM_INDEX_CNTL_ADDR_0,
866 mmRLC_SRM_INDEX_CNTL_DATA_0 - mmRLC_SRM_INDEX_CNTL_DATA_0,
867 mmRLC_SRM_INDEX_CNTL_DATA_1 - mmRLC_SRM_INDEX_CNTL_DATA_0,
868 mmRLC_SRM_INDEX_CNTL_DATA_2 - mmRLC_SRM_INDEX_CNTL_DATA_0,
869 mmRLC_SRM_INDEX_CNTL_DATA_3 - mmRLC_SRM_INDEX_CNTL_DATA_0,
870 mmRLC_SRM_INDEX_CNTL_DATA_4 - mmRLC_SRM_INDEX_CNTL_DATA_0,
871 mmRLC_SRM_INDEX_CNTL_DATA_5 - mmRLC_SRM_INDEX_CNTL_DATA_0,
872 mmRLC_SRM_INDEX_CNTL_DATA_6 - mmRLC_SRM_INDEX_CNTL_DATA_0,
873 mmRLC_SRM_INDEX_CNTL_DATA_7 - mmRLC_SRM_INDEX_CNTL_DATA_0,
903 struct amdgpu_device *adev = kiq_ring->adev; in gfx_v9_0_kiq_set_resources()
907 shader_mc_addr = adev->gfx.cleaner_shader_gpu_addr >> 8; in gfx_v9_0_kiq_set_resources()
927 uint64_t mqd_addr = amdgpu_bo_gpu_offset(ring->mqd_obj); in gfx_v9_0_kiq_map_queues()
928 uint64_t wptr_addr = ring->wptr_gpu_addr; in gfx_v9_0_kiq_map_queues()
929 uint32_t eng_sel = ring->funcs->type == AMDGPU_RING_TYPE_GFX ? 4 : 0; in gfx_v9_0_kiq_map_queues()
936 PACKET3_MAP_QUEUES_QUEUE(ring->queue) | in gfx_v9_0_kiq_map_queues()
937 PACKET3_MAP_QUEUES_PIPE(ring->pipe) | in gfx_v9_0_kiq_map_queues()
938 PACKET3_MAP_QUEUES_ME((ring->me == 1 ? 0 : 1)) | in gfx_v9_0_kiq_map_queues()
947 PACKET3_MAP_QUEUES_DOORBELL_OFFSET(ring->doorbell_index)); in gfx_v9_0_kiq_map_queues()
959 uint32_t eng_sel = ring->funcs->type == AMDGPU_RING_TYPE_GFX ? 4 : 0; in gfx_v9_0_kiq_unmap_queues()
968 PACKET3_UNMAP_QUEUES_DOORBELL_OFFSET0(ring->doorbell_index)); in gfx_v9_0_kiq_unmap_queues()
971 amdgpu_ring_write(kiq_ring, lower_32_bits(ring->wptr & ring->buf_mask)); in gfx_v9_0_kiq_unmap_queues()
987 uint32_t eng_sel = ring->funcs->type == AMDGPU_RING_TYPE_GFX ? 4 : 0; in gfx_v9_0_kiq_query_status()
996 PACKET3_QUERY_STATUS_DOORBELL_OFFSET(ring->doorbell_index) | in gfx_v9_0_kiq_query_status()
1021 struct amdgpu_device *adev = kiq_ring->adev; in gfx_v9_0_kiq_reset_hw_queue()
1026 mutex_lock(&adev->srbm_mutex); in gfx_v9_0_kiq_reset_hw_queue()
1033 for (i = 0; i < adev->usec_timeout; i++) { in gfx_v9_0_kiq_reset_hw_queue()
1038 if (i >= adev->usec_timeout) in gfx_v9_0_kiq_reset_hw_queue()
1039 dev_err(adev->dev, "fail to wait on hqd deactive\n"); in gfx_v9_0_kiq_reset_hw_queue()
1041 dev_err(adev->dev, "reset queue_type(%d) not supported\n", queue_type); in gfx_v9_0_kiq_reset_hw_queue()
1045 mutex_unlock(&adev->srbm_mutex); in gfx_v9_0_kiq_reset_hw_queue()
1066 adev->gfx.kiq[0].pmf = &gfx_v9_0_kiq_pm4_funcs; in gfx_v9_0_set_kiq_pm4_funcs()
1072 case IP_VERSION(9, 0, 1): in gfx_v9_0_init_golden_registers()
1080 case IP_VERSION(9, 2, 1): in gfx_v9_0_init_golden_registers()
1088 case IP_VERSION(9, 4, 0): in gfx_v9_0_init_golden_registers()
1096 case IP_VERSION(9, 4, 1): in gfx_v9_0_init_golden_registers()
1101 case IP_VERSION(9, 2, 2): in gfx_v9_0_init_golden_registers()
1102 case IP_VERSION(9, 1, 0): in gfx_v9_0_init_golden_registers()
1105 if (adev->apu_flags & AMD_APU_IS_RAVEN2) in gfx_v9_0_init_golden_registers()
1114 case IP_VERSION(9, 3, 0): in gfx_v9_0_init_golden_registers()
1119 case IP_VERSION(9, 4, 2): in gfx_v9_0_init_golden_registers()
1121 adev->smuio.funcs->get_die_id(adev)); in gfx_v9_0_init_golden_registers()
1127 if ((amdgpu_ip_version(adev, GC_HWIP, 0) != IP_VERSION(9, 4, 1)) && in gfx_v9_0_init_golden_registers()
1128 (amdgpu_ip_version(adev, GC_HWIP, 0) != IP_VERSION(9, 4, 2))) in gfx_v9_0_init_golden_registers()
1169 struct amdgpu_device *adev = ring->adev; in gfx_v9_0_ring_test_ring()
1181 amdgpu_ring_write(ring, scratch - PACKET3_SET_UCONFIG_REG_START); in gfx_v9_0_ring_test_ring()
1185 for (i = 0; i < adev->usec_timeout; i++) { in gfx_v9_0_ring_test_ring()
1192 if (i >= adev->usec_timeout) in gfx_v9_0_ring_test_ring()
1193 r = -ETIMEDOUT; in gfx_v9_0_ring_test_ring()
1199 struct amdgpu_device *adev = ring->adev; in gfx_v9_0_ring_test_ib()
1212 gpu_addr = adev->wb.gpu_addr + (index * 4); in gfx_v9_0_ring_test_ib()
1213 adev->wb.wb[index] = cpu_to_le32(0xCAFEDEAD); in gfx_v9_0_ring_test_ib()
1233 r = -ETIMEDOUT; in gfx_v9_0_ring_test_ib()
1239 tmp = adev->wb.wb[index]; in gfx_v9_0_ring_test_ib()
1243 r = -EINVAL; in gfx_v9_0_ring_test_ib()
1256 amdgpu_ucode_release(&adev->gfx.pfp_fw); in gfx_v9_0_free_microcode()
1257 amdgpu_ucode_release(&adev->gfx.me_fw); in gfx_v9_0_free_microcode()
1258 amdgpu_ucode_release(&adev->gfx.ce_fw); in gfx_v9_0_free_microcode()
1259 amdgpu_ucode_release(&adev->gfx.rlc_fw); in gfx_v9_0_free_microcode()
1260 amdgpu_ucode_release(&adev->gfx.mec_fw); in gfx_v9_0_free_microcode()
1261 amdgpu_ucode_release(&adev->gfx.mec2_fw); in gfx_v9_0_free_microcode()
1263 kfree(adev->gfx.rlc.register_list_format); in gfx_v9_0_free_microcode()
1268 adev->gfx.me_fw_write_wait = false; in gfx_v9_0_check_fw_write_wait()
1269 adev->gfx.mec_fw_write_wait = false; in gfx_v9_0_check_fw_write_wait()
1271 if ((amdgpu_ip_version(adev, GC_HWIP, 0) != IP_VERSION(9, 4, 1)) && in gfx_v9_0_check_fw_write_wait()
1272 ((adev->gfx.mec_fw_version < 0x000001a5) || in gfx_v9_0_check_fw_write_wait()
1273 (adev->gfx.mec_feature_version < 46) || in gfx_v9_0_check_fw_write_wait()
1274 (adev->gfx.pfp_fw_version < 0x000000b7) || in gfx_v9_0_check_fw_write_wait()
1275 (adev->gfx.pfp_feature_version < 46))) in gfx_v9_0_check_fw_write_wait()
1279 case IP_VERSION(9, 0, 1): in gfx_v9_0_check_fw_write_wait()
1280 if ((adev->gfx.me_fw_version >= 0x0000009c) && in gfx_v9_0_check_fw_write_wait()
1281 (adev->gfx.me_feature_version >= 42) && in gfx_v9_0_check_fw_write_wait()
1282 (adev->gfx.pfp_fw_version >= 0x000000b1) && in gfx_v9_0_check_fw_write_wait()
1283 (adev->gfx.pfp_feature_version >= 42)) in gfx_v9_0_check_fw_write_wait()
1284 adev->gfx.me_fw_write_wait = true; in gfx_v9_0_check_fw_write_wait()
1286 if ((adev->gfx.mec_fw_version >= 0x00000193) && in gfx_v9_0_check_fw_write_wait()
1287 (adev->gfx.mec_feature_version >= 42)) in gfx_v9_0_check_fw_write_wait()
1288 adev->gfx.mec_fw_write_wait = true; in gfx_v9_0_check_fw_write_wait()
1290 case IP_VERSION(9, 2, 1): in gfx_v9_0_check_fw_write_wait()
1291 if ((adev->gfx.me_fw_version >= 0x0000009c) && in gfx_v9_0_check_fw_write_wait()
1292 (adev->gfx.me_feature_version >= 44) && in gfx_v9_0_check_fw_write_wait()
1293 (adev->gfx.pfp_fw_version >= 0x000000b2) && in gfx_v9_0_check_fw_write_wait()
1294 (adev->gfx.pfp_feature_version >= 44)) in gfx_v9_0_check_fw_write_wait()
1295 adev->gfx.me_fw_write_wait = true; in gfx_v9_0_check_fw_write_wait()
1297 if ((adev->gfx.mec_fw_version >= 0x00000196) && in gfx_v9_0_check_fw_write_wait()
1298 (adev->gfx.mec_feature_version >= 44)) in gfx_v9_0_check_fw_write_wait()
1299 adev->gfx.mec_fw_write_wait = true; in gfx_v9_0_check_fw_write_wait()
1301 case IP_VERSION(9, 4, 0): in gfx_v9_0_check_fw_write_wait()
1302 if ((adev->gfx.me_fw_version >= 0x0000009c) && in gfx_v9_0_check_fw_write_wait()
1303 (adev->gfx.me_feature_version >= 44) && in gfx_v9_0_check_fw_write_wait()
1304 (adev->gfx.pfp_fw_version >= 0x000000b2) && in gfx_v9_0_check_fw_write_wait()
1305 (adev->gfx.pfp_feature_version >= 44)) in gfx_v9_0_check_fw_write_wait()
1306 adev->gfx.me_fw_write_wait = true; in gfx_v9_0_check_fw_write_wait()
1308 if ((adev->gfx.mec_fw_version >= 0x00000197) && in gfx_v9_0_check_fw_write_wait()
1309 (adev->gfx.mec_feature_version >= 44)) in gfx_v9_0_check_fw_write_wait()
1310 adev->gfx.mec_fw_write_wait = true; in gfx_v9_0_check_fw_write_wait()
1312 case IP_VERSION(9, 1, 0): in gfx_v9_0_check_fw_write_wait()
1313 case IP_VERSION(9, 2, 2): in gfx_v9_0_check_fw_write_wait()
1314 if ((adev->gfx.me_fw_version >= 0x0000009c) && in gfx_v9_0_check_fw_write_wait()
1315 (adev->gfx.me_feature_version >= 42) && in gfx_v9_0_check_fw_write_wait()
1316 (adev->gfx.pfp_fw_version >= 0x000000b1) && in gfx_v9_0_check_fw_write_wait()
1317 (adev->gfx.pfp_feature_version >= 42)) in gfx_v9_0_check_fw_write_wait()
1318 adev->gfx.me_fw_write_wait = true; in gfx_v9_0_check_fw_write_wait()
1320 if ((adev->gfx.mec_fw_version >= 0x00000192) && in gfx_v9_0_check_fw_write_wait()
1321 (adev->gfx.mec_feature_version >= 42)) in gfx_v9_0_check_fw_write_wait()
1322 adev->gfx.mec_fw_write_wait = true; in gfx_v9_0_check_fw_write_wait()
1325 adev->gfx.me_fw_write_wait = true; in gfx_v9_0_check_fw_write_wait()
1326 adev->gfx.mec_fw_write_wait = true; in gfx_v9_0_check_fw_write_wait()
1344 /* GFXOFF is unstable on C6 parts with a VBIOS 113-RAVEN-114 */
1346 /* Apple MacBook Pro (15-inch, 2019) Radeon Pro Vega 20 4 GB */
1359 while (p && p->chip_device != 0) { in gfx_v9_0_should_disable_gfxoff()
1360 if (pdev->vendor == p->chip_vendor && in gfx_v9_0_should_disable_gfxoff()
1361 pdev->device == p->chip_device && in gfx_v9_0_should_disable_gfxoff()
1362 pdev->subsystem_vendor == p->subsys_vendor && in gfx_v9_0_should_disable_gfxoff()
1363 pdev->subsystem_device == p->subsys_device && in gfx_v9_0_should_disable_gfxoff()
1364 pdev->revision == p->revision) { in gfx_v9_0_should_disable_gfxoff()
1374 if (adev->pm.fw_version >= 0x41e2b) in is_raven_kicker()
1382 if ((amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(9, 3, 0)) && in check_if_enlarge_doorbell_range()
1383 (adev->gfx.me_fw_version >= 0x000000a5) && in check_if_enlarge_doorbell_range()
1384 (adev->gfx.me_feature_version >= 52)) in check_if_enlarge_doorbell_range()
1392 if (gfx_v9_0_should_disable_gfxoff(adev->pdev)) in gfx_v9_0_check_if_need_gfxoff()
1393 adev->pm.pp_feature &= ~PP_GFXOFF_MASK; in gfx_v9_0_check_if_need_gfxoff()
1396 case IP_VERSION(9, 0, 1): in gfx_v9_0_check_if_need_gfxoff()
1397 case IP_VERSION(9, 2, 1): in gfx_v9_0_check_if_need_gfxoff()
1398 case IP_VERSION(9, 4, 0): in gfx_v9_0_check_if_need_gfxoff()
1400 case IP_VERSION(9, 2, 2): in gfx_v9_0_check_if_need_gfxoff()
1401 case IP_VERSION(9, 1, 0): in gfx_v9_0_check_if_need_gfxoff()
1402 if (!((adev->apu_flags & AMD_APU_IS_RAVEN2) || in gfx_v9_0_check_if_need_gfxoff()
1403 (adev->apu_flags & AMD_APU_IS_PICASSO)) && in gfx_v9_0_check_if_need_gfxoff()
1405 adev->gfx.rlc_fw_version < 531) || in gfx_v9_0_check_if_need_gfxoff()
1406 (adev->gfx.rlc_feature_version < 1) || in gfx_v9_0_check_if_need_gfxoff()
1407 !adev->gfx.rlc.is_rlc_v2_1)) in gfx_v9_0_check_if_need_gfxoff()
1408 adev->pm.pp_feature &= ~PP_GFXOFF_MASK; in gfx_v9_0_check_if_need_gfxoff()
1410 if (adev->pm.pp_feature & PP_GFXOFF_MASK) in gfx_v9_0_check_if_need_gfxoff()
1411 adev->pg_flags |= AMD_PG_SUPPORT_GFX_PG | in gfx_v9_0_check_if_need_gfxoff()
1415 case IP_VERSION(9, 3, 0): in gfx_v9_0_check_if_need_gfxoff()
1416 if (adev->pm.pp_feature & PP_GFXOFF_MASK) in gfx_v9_0_check_if_need_gfxoff()
1417 adev->pg_flags |= AMD_PG_SUPPORT_GFX_PG | in gfx_v9_0_check_if_need_gfxoff()
1431 err = amdgpu_ucode_request(adev, &adev->gfx.pfp_fw, in gfx_v9_0_init_cp_gfx_microcode()
1437 err = amdgpu_ucode_request(adev, &adev->gfx.me_fw, in gfx_v9_0_init_cp_gfx_microcode()
1443 err = amdgpu_ucode_request(adev, &adev->gfx.ce_fw, in gfx_v9_0_init_cp_gfx_microcode()
1451 amdgpu_ucode_release(&adev->gfx.pfp_fw); in gfx_v9_0_init_cp_gfx_microcode()
1452 amdgpu_ucode_release(&adev->gfx.me_fw); in gfx_v9_0_init_cp_gfx_microcode()
1453 amdgpu_ucode_release(&adev->gfx.ce_fw); in gfx_v9_0_init_cp_gfx_microcode()
1476 (((adev->pdev->revision >= 0xC8) && (adev->pdev->revision <= 0xCF)) || in gfx_v9_0_init_rlc_microcode()
1477 ((adev->pdev->revision >= 0xD8) && (adev->pdev->revision <= 0xDF)))) in gfx_v9_0_init_rlc_microcode()
1478 err = amdgpu_ucode_request(adev, &adev->gfx.rlc_fw, in gfx_v9_0_init_rlc_microcode()
1485 err = amdgpu_ucode_request(adev, &adev->gfx.rlc_fw, in gfx_v9_0_init_rlc_microcode()
1488 err = amdgpu_ucode_request(adev, &adev->gfx.rlc_fw, in gfx_v9_0_init_rlc_microcode()
1493 rlc_hdr = (const struct rlc_firmware_header_v2_0 *)adev->gfx.rlc_fw->data; in gfx_v9_0_init_rlc_microcode()
1494 version_major = le16_to_cpu(rlc_hdr->header.header_version_major); in gfx_v9_0_init_rlc_microcode()
1495 version_minor = le16_to_cpu(rlc_hdr->header.header_version_minor); in gfx_v9_0_init_rlc_microcode()
1499 amdgpu_ucode_release(&adev->gfx.rlc_fw); in gfx_v9_0_init_rlc_microcode()
1506 if (amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(9, 4, 2) || in gfx_v9_0_load_mec2_fw_bin_support()
1507 amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(9, 4, 1) || in gfx_v9_0_load_mec2_fw_bin_support()
1508 amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(9, 3, 0)) in gfx_v9_0_load_mec2_fw_bin_support()
1519 if (amdgpu_sriov_vf(adev) && (adev->asic_type == CHIP_ALDEBARAN)) in gfx_v9_0_init_cp_compute_microcode()
1520 err = amdgpu_ucode_request(adev, &adev->gfx.mec_fw, in gfx_v9_0_init_cp_compute_microcode()
1523 err = amdgpu_ucode_request(adev, &adev->gfx.mec_fw, in gfx_v9_0_init_cp_compute_microcode()
1532 if (amdgpu_sriov_vf(adev) && (adev->asic_type == CHIP_ALDEBARAN)) in gfx_v9_0_init_cp_compute_microcode()
1533 err = amdgpu_ucode_request(adev, &adev->gfx.mec2_fw, in gfx_v9_0_init_cp_compute_microcode()
1536 err = amdgpu_ucode_request(adev, &adev->gfx.mec2_fw, in gfx_v9_0_init_cp_compute_microcode()
1543 amdgpu_ucode_release(&adev->gfx.mec2_fw); in gfx_v9_0_init_cp_compute_microcode()
1546 adev->gfx.mec2_fw_version = adev->gfx.mec_fw_version; in gfx_v9_0_init_cp_compute_microcode()
1547 adev->gfx.mec2_feature_version = adev->gfx.mec_feature_version; in gfx_v9_0_init_cp_compute_microcode()
1555 amdgpu_ucode_release(&adev->gfx.mec_fw); in gfx_v9_0_init_cp_compute_microcode()
1568 if (adev->gfx.num_gfx_rings) { in gfx_v9_0_init_microcode()
1596 for (sect = gfx9_cs_data; sect->section != NULL; ++sect) { in gfx_v9_0_get_csb_size()
1597 for (ext = sect->section; ext->extent != NULL; ++ext) { in gfx_v9_0_get_csb_size()
1598 if (sect->id == SECT_CONTEXT) in gfx_v9_0_get_csb_size()
1599 count += 2 + ext->reg_count; in gfx_v9_0_get_csb_size()
1620 if (adev->gfx.rlc.cs_data == NULL) in gfx_v9_0_get_csb_buffer()
1632 for (sect = adev->gfx.rlc.cs_data; sect->section != NULL; ++sect) { in gfx_v9_0_get_csb_buffer()
1633 for (ext = sect->section; ext->extent != NULL; ++ext) { in gfx_v9_0_get_csb_buffer()
1634 if (sect->id == SECT_CONTEXT) { in gfx_v9_0_get_csb_buffer()
1636 cpu_to_le32(PACKET3(PACKET3_SET_CONTEXT_REG, ext->reg_count)); in gfx_v9_0_get_csb_buffer()
1637 buffer[count++] = cpu_to_le32(ext->reg_index - in gfx_v9_0_get_csb_buffer()
1639 for (i = 0; i < ext->reg_count; i++) in gfx_v9_0_get_csb_buffer()
1640 buffer[count++] = cpu_to_le32(ext->extent[i]); in gfx_v9_0_get_csb_buffer()
1656 struct amdgpu_cu_info *cu_info = &adev->gfx.cu_info; in gfx_v9_0_init_always_on_cu_mask()
1662 if (adev->flags & AMD_IS_APU) in gfx_v9_0_init_always_on_cu_mask()
1664 else if (amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(9, 2, 1)) in gfx_v9_0_init_always_on_cu_mask()
1669 mutex_lock(&adev->grbm_idx_mutex); in gfx_v9_0_init_always_on_cu_mask()
1670 for (i = 0; i < adev->gfx.config.max_shader_engines; i++) { in gfx_v9_0_init_always_on_cu_mask()
1671 for (j = 0; j < adev->gfx.config.max_sh_per_se; j++) { in gfx_v9_0_init_always_on_cu_mask()
1677 for (k = 0; k < adev->gfx.config.max_cu_per_sh; k ++) { in gfx_v9_0_init_always_on_cu_mask()
1678 if (cu_info->bitmap[0][i][j] & mask) { in gfx_v9_0_init_always_on_cu_mask()
1691 cu_info->ao_cu_bitmap[i][j] = cu_bitmap; in gfx_v9_0_init_always_on_cu_mask()
1695 mutex_unlock(&adev->grbm_idx_mutex); in gfx_v9_0_init_always_on_cu_mask()
1714 mutex_lock(&adev->grbm_idx_mutex); in gfx_v9_0_init_lbpw()
1725 /* set mmRLC_GPM_GENERAL_7[31-16] = 0x00C0 */ in gfx_v9_0_init_lbpw()
1742 mutex_unlock(&adev->grbm_idx_mutex); in gfx_v9_0_init_lbpw()
1763 mutex_lock(&adev->grbm_idx_mutex); in gfx_v9_4_init_lbpw()
1774 /* set mmRLC_GPM_GENERAL_7[31-16] = 0x00C0 */ in gfx_v9_4_init_lbpw()
1791 mutex_unlock(&adev->grbm_idx_mutex); in gfx_v9_4_init_lbpw()
1813 reg_access_ctrl = &adev->gfx.rlc.reg_access_ctrl[0]; in gfx_v9_0_init_rlcg_reg_access_ctrl()
1814 reg_access_ctrl->scratch_reg0 = SOC15_REG_OFFSET(GC, 0, mmSCRATCH_REG0); in gfx_v9_0_init_rlcg_reg_access_ctrl()
1815 reg_access_ctrl->scratch_reg1 = SOC15_REG_OFFSET(GC, 0, mmSCRATCH_REG1); in gfx_v9_0_init_rlcg_reg_access_ctrl()
1816 reg_access_ctrl->scratch_reg2 = SOC15_REG_OFFSET(GC, 0, mmSCRATCH_REG2); in gfx_v9_0_init_rlcg_reg_access_ctrl()
1817 reg_access_ctrl->scratch_reg3 = SOC15_REG_OFFSET(GC, 0, mmSCRATCH_REG3); in gfx_v9_0_init_rlcg_reg_access_ctrl()
1818 reg_access_ctrl->grbm_cntl = SOC15_REG_OFFSET(GC, 0, mmGRBM_GFX_CNTL); in gfx_v9_0_init_rlcg_reg_access_ctrl()
1819 reg_access_ctrl->grbm_idx = SOC15_REG_OFFSET(GC, 0, mmGRBM_GFX_INDEX); in gfx_v9_0_init_rlcg_reg_access_ctrl()
1820 reg_access_ctrl->spare_int = SOC15_REG_OFFSET(GC, 0, mmRLC_SPARE_INT); in gfx_v9_0_init_rlcg_reg_access_ctrl()
1821 adev->gfx.rlc.rlcg_reg_access_supported = true; in gfx_v9_0_init_rlcg_reg_access_ctrl()
1829 adev->gfx.rlc.cs_data = gfx9_cs_data; in gfx_v9_0_rlc_init()
1831 cs_data = adev->gfx.rlc.cs_data; in gfx_v9_0_rlc_init()
1840 if (adev->flags & AMD_IS_APU) { in gfx_v9_0_rlc_init()
1842 adev->gfx.rlc.cp_table_size = ALIGN(96 * 5 * 4, 2048) + (64 * 1024); /* JT + GDS */ in gfx_v9_0_rlc_init()
1853 amdgpu_bo_free_kernel(&adev->gfx.mec.hpd_eop_obj, NULL, NULL); in gfx_v9_0_mec_fini()
1854 amdgpu_bo_free_kernel(&adev->gfx.mec.mec_fw_obj, NULL, NULL); in gfx_v9_0_mec_fini()
1868 bitmap_zero(adev->gfx.mec_bitmap[0].queue_bitmap, AMDGPU_MAX_COMPUTE_QUEUES); in gfx_v9_0_mec_init()
1872 mec_hpd_size = adev->gfx.num_compute_rings * GFX9_MEC_HPD_SIZE; in gfx_v9_0_mec_init()
1877 &adev->gfx.mec.hpd_eop_obj, in gfx_v9_0_mec_init()
1878 &adev->gfx.mec.hpd_eop_gpu_addr, in gfx_v9_0_mec_init()
1881 dev_warn(adev->dev, "(%d) create HDP EOP bo failed\n", r); in gfx_v9_0_mec_init()
1888 amdgpu_bo_kunmap(adev->gfx.mec.hpd_eop_obj); in gfx_v9_0_mec_init()
1889 amdgpu_bo_unreserve(adev->gfx.mec.hpd_eop_obj); in gfx_v9_0_mec_init()
1892 mec_hdr = (const struct gfx_firmware_header_v1_0 *)adev->gfx.mec_fw->data; in gfx_v9_0_mec_init()
1895 (adev->gfx.mec_fw->data + in gfx_v9_0_mec_init()
1896 le32_to_cpu(mec_hdr->header.ucode_array_offset_bytes)); in gfx_v9_0_mec_init()
1897 fw_size = le32_to_cpu(mec_hdr->header.ucode_size_bytes); in gfx_v9_0_mec_init()
1899 r = amdgpu_bo_create_reserved(adev, mec_hdr->header.ucode_size_bytes, in gfx_v9_0_mec_init()
1901 &adev->gfx.mec.mec_fw_obj, in gfx_v9_0_mec_init()
1902 &adev->gfx.mec.mec_fw_gpu_addr, in gfx_v9_0_mec_init()
1905 dev_warn(adev->dev, "(%d) create mec firmware bo failed\n", r); in gfx_v9_0_mec_init()
1912 amdgpu_bo_kunmap(adev->gfx.mec.mec_fw_obj); in gfx_v9_0_mec_init()
1913 amdgpu_bo_unreserve(adev->gfx.mec.mec_fw_obj); in gfx_v9_0_mec_init()
1939 while (num--) in wave_read_regs()
2016 case IP_VERSION(9, 0, 1): in gfx_v9_0_gpu_early_init()
2017 adev->gfx.config.max_hw_contexts = 8; in gfx_v9_0_gpu_early_init()
2018 adev->gfx.config.sc_prim_fifo_size_frontend = 0x20; in gfx_v9_0_gpu_early_init()
2019 adev->gfx.config.sc_prim_fifo_size_backend = 0x100; in gfx_v9_0_gpu_early_init()
2020 adev->gfx.config.sc_hiz_tile_fifo_size = 0x30; in gfx_v9_0_gpu_early_init()
2021 adev->gfx.config.sc_earlyz_tile_fifo_size = 0x4C0; in gfx_v9_0_gpu_early_init()
2024 case IP_VERSION(9, 2, 1): in gfx_v9_0_gpu_early_init()
2025 adev->gfx.config.max_hw_contexts = 8; in gfx_v9_0_gpu_early_init()
2026 adev->gfx.config.sc_prim_fifo_size_frontend = 0x20; in gfx_v9_0_gpu_early_init()
2027 adev->gfx.config.sc_prim_fifo_size_backend = 0x100; in gfx_v9_0_gpu_early_init()
2028 adev->gfx.config.sc_hiz_tile_fifo_size = 0x30; in gfx_v9_0_gpu_early_init()
2029 adev->gfx.config.sc_earlyz_tile_fifo_size = 0x4C0; in gfx_v9_0_gpu_early_init()
2033 case IP_VERSION(9, 4, 0): in gfx_v9_0_gpu_early_init()
2034 adev->gfx.ras = &gfx_v9_0_ras; in gfx_v9_0_gpu_early_init()
2035 adev->gfx.config.max_hw_contexts = 8; in gfx_v9_0_gpu_early_init()
2036 adev->gfx.config.sc_prim_fifo_size_frontend = 0x20; in gfx_v9_0_gpu_early_init()
2037 adev->gfx.config.sc_prim_fifo_size_backend = 0x100; in gfx_v9_0_gpu_early_init()
2038 adev->gfx.config.sc_hiz_tile_fifo_size = 0x30; in gfx_v9_0_gpu_early_init()
2039 adev->gfx.config.sc_earlyz_tile_fifo_size = 0x4C0; in gfx_v9_0_gpu_early_init()
2048 case IP_VERSION(9, 2, 2): in gfx_v9_0_gpu_early_init()
2049 case IP_VERSION(9, 1, 0): in gfx_v9_0_gpu_early_init()
2050 adev->gfx.config.max_hw_contexts = 8; in gfx_v9_0_gpu_early_init()
2051 adev->gfx.config.sc_prim_fifo_size_frontend = 0x20; in gfx_v9_0_gpu_early_init()
2052 adev->gfx.config.sc_prim_fifo_size_backend = 0x100; in gfx_v9_0_gpu_early_init()
2053 adev->gfx.config.sc_hiz_tile_fifo_size = 0x30; in gfx_v9_0_gpu_early_init()
2054 adev->gfx.config.sc_earlyz_tile_fifo_size = 0x4C0; in gfx_v9_0_gpu_early_init()
2055 if (adev->apu_flags & AMD_APU_IS_RAVEN2) in gfx_v9_0_gpu_early_init()
2060 case IP_VERSION(9, 4, 1): in gfx_v9_0_gpu_early_init()
2061 adev->gfx.ras = &gfx_v9_4_ras; in gfx_v9_0_gpu_early_init()
2062 adev->gfx.config.max_hw_contexts = 8; in gfx_v9_0_gpu_early_init()
2063 adev->gfx.config.sc_prim_fifo_size_frontend = 0x20; in gfx_v9_0_gpu_early_init()
2064 adev->gfx.config.sc_prim_fifo_size_backend = 0x100; in gfx_v9_0_gpu_early_init()
2065 adev->gfx.config.sc_hiz_tile_fifo_size = 0x30; in gfx_v9_0_gpu_early_init()
2066 adev->gfx.config.sc_earlyz_tile_fifo_size = 0x4C0; in gfx_v9_0_gpu_early_init()
2071 case IP_VERSION(9, 3, 0): in gfx_v9_0_gpu_early_init()
2072 adev->gfx.config.max_hw_contexts = 8; in gfx_v9_0_gpu_early_init()
2073 adev->gfx.config.sc_prim_fifo_size_frontend = 0x20; in gfx_v9_0_gpu_early_init()
2074 adev->gfx.config.sc_prim_fifo_size_backend = 0x100; in gfx_v9_0_gpu_early_init()
2075 adev->gfx.config.sc_hiz_tile_fifo_size = 0x80; in gfx_v9_0_gpu_early_init()
2076 adev->gfx.config.sc_earlyz_tile_fifo_size = 0x4C0; in gfx_v9_0_gpu_early_init()
2081 case IP_VERSION(9, 4, 2): in gfx_v9_0_gpu_early_init()
2082 adev->gfx.ras = &gfx_v9_4_2_ras; in gfx_v9_0_gpu_early_init()
2083 adev->gfx.config.max_hw_contexts = 8; in gfx_v9_0_gpu_early_init()
2084 adev->gfx.config.sc_prim_fifo_size_frontend = 0x20; in gfx_v9_0_gpu_early_init()
2085 adev->gfx.config.sc_prim_fifo_size_backend = 0x100; in gfx_v9_0_gpu_early_init()
2086 adev->gfx.config.sc_hiz_tile_fifo_size = 0x30; in gfx_v9_0_gpu_early_init()
2087 adev->gfx.config.sc_earlyz_tile_fifo_size = 0x4C0; in gfx_v9_0_gpu_early_init()
2101 adev->gfx.config.gb_addr_config = gb_addr_config; in gfx_v9_0_gpu_early_init()
2103 adev->gfx.config.gb_addr_config_fields.num_pipes = 1 << in gfx_v9_0_gpu_early_init()
2105 adev->gfx.config.gb_addr_config, in gfx_v9_0_gpu_early_init()
2109 adev->gfx.config.max_tile_pipes = in gfx_v9_0_gpu_early_init()
2110 adev->gfx.config.gb_addr_config_fields.num_pipes; in gfx_v9_0_gpu_early_init()
2112 adev->gfx.config.gb_addr_config_fields.num_banks = 1 << in gfx_v9_0_gpu_early_init()
2114 adev->gfx.config.gb_addr_config, in gfx_v9_0_gpu_early_init()
2117 adev->gfx.config.gb_addr_config_fields.max_compress_frags = 1 << in gfx_v9_0_gpu_early_init()
2119 adev->gfx.config.gb_addr_config, in gfx_v9_0_gpu_early_init()
2122 adev->gfx.config.gb_addr_config_fields.num_rb_per_se = 1 << in gfx_v9_0_gpu_early_init()
2124 adev->gfx.config.gb_addr_config, in gfx_v9_0_gpu_early_init()
2127 adev->gfx.config.gb_addr_config_fields.num_se = 1 << in gfx_v9_0_gpu_early_init()
2129 adev->gfx.config.gb_addr_config, in gfx_v9_0_gpu_early_init()
2132 adev->gfx.config.gb_addr_config_fields.pipe_interleave_size = 1 << (8 + in gfx_v9_0_gpu_early_init()
2134 adev->gfx.config.gb_addr_config, in gfx_v9_0_gpu_early_init()
2145 struct amdgpu_ring *ring = &adev->gfx.compute_ring[ring_id]; in gfx_v9_0_compute_ring_init()
2148 ring = &adev->gfx.compute_ring[ring_id]; in gfx_v9_0_compute_ring_init()
2151 ring->me = mec + 1; in gfx_v9_0_compute_ring_init()
2152 ring->pipe = pipe; in gfx_v9_0_compute_ring_init()
2153 ring->queue = queue; in gfx_v9_0_compute_ring_init()
2155 ring->ring_obj = NULL; in gfx_v9_0_compute_ring_init()
2156 ring->use_doorbell = true; in gfx_v9_0_compute_ring_init()
2157 ring->doorbell_index = (adev->doorbell_index.mec_ring0 + ring_id) << 1; in gfx_v9_0_compute_ring_init()
2158 ring->eop_gpu_addr = adev->gfx.mec.hpd_eop_gpu_addr in gfx_v9_0_compute_ring_init()
2160 ring->vm_hub = AMDGPU_GFXHUB(0); in gfx_v9_0_compute_ring_init()
2161 sprintf(ring->name, "comp_%d.%d.%d", ring->me, ring->pipe, ring->queue); in gfx_v9_0_compute_ring_init()
2164 + ((ring->me - 1) * adev->gfx.mec.num_pipe_per_mec) in gfx_v9_0_compute_ring_init()
2165 + ring->pipe; in gfx_v9_0_compute_ring_init()
2168 /* type-2 packets are deprecated on MEC, use type-3 instead */ in gfx_v9_0_compute_ring_init()
2169 return amdgpu_ring_init(adev, ring, 1024, &adev->gfx.eop_irq, irq_type, in gfx_v9_0_compute_ring_init()
2182 adev->gfx.ip_dump_core = NULL; in gfx_v9_0_alloc_ip_dump()
2184 adev->gfx.ip_dump_core = ptr; in gfx_v9_0_alloc_ip_dump()
2189 inst = adev->gfx.mec.num_mec * adev->gfx.mec.num_pipe_per_mec * in gfx_v9_0_alloc_ip_dump()
2190 adev->gfx.mec.num_queue_per_pipe; in gfx_v9_0_alloc_ip_dump()
2195 adev->gfx.ip_dump_compute_queues = NULL; in gfx_v9_0_alloc_ip_dump()
2197 adev->gfx.ip_dump_compute_queues = ptr; in gfx_v9_0_alloc_ip_dump()
2210 case IP_VERSION(9, 0, 1): in gfx_v9_0_sw_init()
2211 case IP_VERSION(9, 2, 1): in gfx_v9_0_sw_init()
2212 case IP_VERSION(9, 4, 0): in gfx_v9_0_sw_init()
2213 case IP_VERSION(9, 2, 2): in gfx_v9_0_sw_init()
2214 case IP_VERSION(9, 1, 0): in gfx_v9_0_sw_init()
2215 case IP_VERSION(9, 4, 1): in gfx_v9_0_sw_init()
2216 case IP_VERSION(9, 3, 0): in gfx_v9_0_sw_init()
2217 case IP_VERSION(9, 4, 2): in gfx_v9_0_sw_init()
2218 adev->gfx.mec.num_mec = 2; in gfx_v9_0_sw_init()
2221 adev->gfx.mec.num_mec = 1; in gfx_v9_0_sw_init()
2227 adev->gfx.enable_cleaner_shader = false; in gfx_v9_0_sw_init()
2231 adev->gfx.mec.num_pipe_per_mec = 4; in gfx_v9_0_sw_init()
2232 adev->gfx.mec.num_queue_per_pipe = 8; in gfx_v9_0_sw_init()
2235 …r = amdgpu_irq_add_id(adev, SOC15_IH_CLIENTID_GRBM_CP, GFX_9_0__SRCID__CP_EOP_INTERRUPT, &adev->gf… in gfx_v9_0_sw_init()
2242 &adev->gfx.bad_op_irq); in gfx_v9_0_sw_init()
2248 &adev->gfx.priv_reg_irq); in gfx_v9_0_sw_init()
2254 &adev->gfx.priv_inst_irq); in gfx_v9_0_sw_init()
2260 &adev->gfx.cp_ecc_error_irq); in gfx_v9_0_sw_init()
2266 &adev->gfx.cp_ecc_error_irq); in gfx_v9_0_sw_init()
2270 adev->gfx.gfx_current_status = AMDGPU_GFX_NORMAL_MODE; in gfx_v9_0_sw_init()
2272 if (adev->gfx.rlc.funcs) { in gfx_v9_0_sw_init()
2273 if (adev->gfx.rlc.funcs->init) { in gfx_v9_0_sw_init()
2274 r = adev->gfx.rlc.funcs->init(adev); in gfx_v9_0_sw_init()
2276 dev_err(adev->dev, "Failed to init rlc BOs!\n"); in gfx_v9_0_sw_init()
2289 for (i = 0; i < adev->gfx.num_gfx_rings; i++) { in gfx_v9_0_sw_init()
2290 ring = &adev->gfx.gfx_ring[i]; in gfx_v9_0_sw_init()
2291 ring->ring_obj = NULL; in gfx_v9_0_sw_init()
2293 sprintf(ring->name, "gfx"); in gfx_v9_0_sw_init()
2295 sprintf(ring->name, "gfx_%d", i); in gfx_v9_0_sw_init()
2296 ring->use_doorbell = true; in gfx_v9_0_sw_init()
2297 ring->doorbell_index = adev->doorbell_index.gfx_ring0 << 1; in gfx_v9_0_sw_init()
2300 ring->no_scheduler = adev->gfx.mcbp; in gfx_v9_0_sw_init()
2301 ring->vm_hub = AMDGPU_GFXHUB(0); in gfx_v9_0_sw_init()
2302 r = amdgpu_ring_init(adev, ring, 1024, &adev->gfx.eop_irq, in gfx_v9_0_sw_init()
2310 if (adev->gfx.mcbp && adev->gfx.num_gfx_rings) { in gfx_v9_0_sw_init()
2312 ring = &adev->gfx.sw_gfx_ring[i]; in gfx_v9_0_sw_init()
2313 ring->ring_obj = NULL; in gfx_v9_0_sw_init()
2314 sprintf(ring->name, amdgpu_sw_ring_name(i)); in gfx_v9_0_sw_init()
2315 ring->use_doorbell = true; in gfx_v9_0_sw_init()
2316 ring->doorbell_index = adev->doorbell_index.gfx_ring0 << 1; in gfx_v9_0_sw_init()
2317 ring->is_sw_ring = true; in gfx_v9_0_sw_init()
2319 ring->vm_hub = AMDGPU_GFXHUB(0); in gfx_v9_0_sw_init()
2320 r = amdgpu_ring_init(adev, ring, 1024, &adev->gfx.eop_irq, in gfx_v9_0_sw_init()
2325 ring->wptr = 0; in gfx_v9_0_sw_init()
2329 r = amdgpu_ring_mux_init(&adev->gfx.muxer, &adev->gfx.gfx_ring[0], in gfx_v9_0_sw_init()
2336 r = amdgpu_ring_mux_add_sw_ring(&adev->gfx.muxer, in gfx_v9_0_sw_init()
2337 &adev->gfx.sw_gfx_ring[i]); in gfx_v9_0_sw_init()
2345 /* set up the compute queues - allocate horizontally across pipes */ in gfx_v9_0_sw_init()
2347 for (i = 0; i < adev->gfx.mec.num_mec; ++i) { in gfx_v9_0_sw_init()
2348 for (j = 0; j < adev->gfx.mec.num_queue_per_pipe; j++) { in gfx_v9_0_sw_init()
2349 for (k = 0; k < adev->gfx.mec.num_pipe_per_mec; k++) { in gfx_v9_0_sw_init()
2380 adev->gfx.ce_ram_size = 0x8000; in gfx_v9_0_sw_init()
2387 dev_err(adev->dev, "Failed to initialize gfx ras block!\n"); in gfx_v9_0_sw_init()
2388 return -EINVAL; in gfx_v9_0_sw_init()
2406 if (adev->gfx.mcbp && adev->gfx.num_gfx_rings) { in gfx_v9_0_sw_fini()
2408 amdgpu_ring_fini(&adev->gfx.sw_gfx_ring[i]); in gfx_v9_0_sw_fini()
2409 amdgpu_ring_mux_fini(&adev->gfx.muxer); in gfx_v9_0_sw_fini()
2412 for (i = 0; i < adev->gfx.num_gfx_rings; i++) in gfx_v9_0_sw_fini()
2413 amdgpu_ring_fini(&adev->gfx.gfx_ring[i]); in gfx_v9_0_sw_fini()
2414 for (i = 0; i < adev->gfx.num_compute_rings; i++) in gfx_v9_0_sw_fini()
2415 amdgpu_ring_fini(&adev->gfx.compute_ring[i]); in gfx_v9_0_sw_fini()
2418 amdgpu_gfx_kiq_free_ring(&adev->gfx.kiq[0].ring); in gfx_v9_0_sw_fini()
2422 amdgpu_bo_free_kernel(&adev->gfx.rlc.clear_state_obj, in gfx_v9_0_sw_fini()
2423 &adev->gfx.rlc.clear_state_gpu_addr, in gfx_v9_0_sw_fini()
2424 (void **)&adev->gfx.rlc.cs_ptr); in gfx_v9_0_sw_fini()
2425 if (adev->flags & AMD_IS_APU) { in gfx_v9_0_sw_fini()
2426 amdgpu_bo_free_kernel(&adev->gfx.rlc.cp_table_obj, in gfx_v9_0_sw_fini()
2427 &adev->gfx.rlc.cp_table_gpu_addr, in gfx_v9_0_sw_fini()
2428 (void **)&adev->gfx.rlc.cp_table_ptr); in gfx_v9_0_sw_fini()
2434 kfree(adev->gfx.ip_dump_core); in gfx_v9_0_sw_fini()
2435 kfree(adev->gfx.ip_dump_compute_queues); in gfx_v9_0_sw_fini()
2479 mask = amdgpu_gfx_create_bitmask(adev->gfx.config.max_backends_per_se / in gfx_v9_0_get_rb_active_bitmap()
2480 adev->gfx.config.max_sh_per_se); in gfx_v9_0_get_rb_active_bitmap()
2490 u32 rb_bitmap_width_per_sh = adev->gfx.config.max_backends_per_se / in gfx_v9_0_setup_rb()
2491 adev->gfx.config.max_sh_per_se; in gfx_v9_0_setup_rb()
2493 mutex_lock(&adev->grbm_idx_mutex); in gfx_v9_0_setup_rb()
2494 for (i = 0; i < adev->gfx.config.max_shader_engines; i++) { in gfx_v9_0_setup_rb()
2495 for (j = 0; j < adev->gfx.config.max_sh_per_se; j++) { in gfx_v9_0_setup_rb()
2498 active_rbs |= data << ((i * adev->gfx.config.max_sh_per_se + j) * in gfx_v9_0_setup_rb()
2503 mutex_unlock(&adev->grbm_idx_mutex); in gfx_v9_0_setup_rb()
2505 adev->gfx.config.backend_enable_mask = active_rbs; in gfx_v9_0_setup_rb()
2506 adev->gfx.config.num_rbs = hweight32(active_rbs); in gfx_v9_0_setup_rb()
2541 * LDS: 0x60000000'00000000 - 0x60000001'00000000 (4GB) in gfx_v9_0_init_compute_vmid()
2542 * Scratch: 0x60000001'00000000 - 0x60000002'00000000 (4GB) in gfx_v9_0_init_compute_vmid()
2543 * GPUVM: 0x60010000'00000000 - 0x60020000'00000000 (1TB) in gfx_v9_0_init_compute_vmid()
2551 mutex_lock(&adev->srbm_mutex); in gfx_v9_0_init_compute_vmid()
2552 for (i = adev->vm_manager.first_kfd_vmid; i < AMDGPU_NUM_VMID; i++) { in gfx_v9_0_init_compute_vmid()
2559 mutex_unlock(&adev->srbm_mutex); in gfx_v9_0_init_compute_vmid()
2563 for (i = adev->vm_manager.first_kfd_vmid; i < AMDGPU_NUM_VMID; i++) { in gfx_v9_0_init_compute_vmid()
2576 * Initialize all compute and user-gfx VMIDs to have no GDS, GWS, or OA in gfx_v9_0_init_gds_vmid()
2594 case IP_VERSION(9, 4, 1): in gfx_v9_0_init_sq_config()
2597 !READ_ONCE(adev->barrier_has_auto_waitcnt)); in gfx_v9_0_init_sq_config()
2614 if (adev->gfx.num_gfx_rings) in gfx_v9_0_constants_init()
2616 gfx_v9_0_get_cu_info(adev, &adev->gfx.cu_info); in gfx_v9_0_constants_init()
2617 adev->gfx.config.db_debug2 = RREG32_SOC15(GC, 0, mmDB_DEBUG2); in gfx_v9_0_constants_init()
2621 mutex_lock(&adev->srbm_mutex); in gfx_v9_0_constants_init()
2622 for (i = 0; i < adev->vm_manager.id_mgr[AMDGPU_GFXHUB(0)].num_ids; i++) { in gfx_v9_0_constants_init()
2629 !!adev->gmc.noretry); in gfx_v9_0_constants_init()
2636 !!adev->gmc.noretry); in gfx_v9_0_constants_init()
2639 (adev->gmc.private_aperture_start >> 48)); in gfx_v9_0_constants_init()
2641 (adev->gmc.shared_aperture_start >> 48)); in gfx_v9_0_constants_init()
2647 mutex_unlock(&adev->srbm_mutex); in gfx_v9_0_constants_init()
2659 mutex_lock(&adev->grbm_idx_mutex); in gfx_v9_0_wait_for_rlc_serdes()
2660 for (i = 0; i < adev->gfx.config.max_shader_engines; i++) { in gfx_v9_0_wait_for_rlc_serdes()
2661 for (j = 0; j < adev->gfx.config.max_sh_per_se; j++) { in gfx_v9_0_wait_for_rlc_serdes()
2663 for (k = 0; k < adev->usec_timeout; k++) { in gfx_v9_0_wait_for_rlc_serdes()
2668 if (k == adev->usec_timeout) { in gfx_v9_0_wait_for_rlc_serdes()
2671 mutex_unlock(&adev->grbm_idx_mutex); in gfx_v9_0_wait_for_rlc_serdes()
2679 mutex_unlock(&adev->grbm_idx_mutex); in gfx_v9_0_wait_for_rlc_serdes()
2685 for (k = 0; k < adev->usec_timeout; k++) { in gfx_v9_0_wait_for_rlc_serdes()
2704 if (adev->gfx.num_gfx_rings) in gfx_v9_0_enable_gui_idle_interrupt()
2712 adev->gfx.rlc.funcs->get_csb_buffer(adev, adev->gfx.rlc.cs_ptr); in gfx_v9_0_init_csb()
2715 adev->gfx.rlc.clear_state_gpu_addr >> 32); in gfx_v9_0_init_csb()
2717 adev->gfx.rlc.clear_state_gpu_addr & 0xfffffffc); in gfx_v9_0_init_csb()
2719 adev->gfx.rlc.clear_state_size); in gfx_v9_0_init_csb()
2772 kmemdup(adev->gfx.rlc.register_list_format, in gfx_v9_1_init_rlc_save_restore_list()
2773 adev->gfx.rlc.reg_list_format_size_bytes, GFP_KERNEL); in gfx_v9_1_init_rlc_save_restore_list()
2775 return -ENOMEM; in gfx_v9_1_init_rlc_save_restore_list()
2780 adev->gfx.rlc.reg_list_format_direct_reg_list_length, in gfx_v9_1_init_rlc_save_restore_list()
2781 adev->gfx.rlc.reg_list_format_size_bytes >> 2, in gfx_v9_1_init_rlc_save_restore_list()
2796 for (i = 0; i < adev->gfx.rlc.reg_list_size_bytes >> 2; i++) in gfx_v9_1_init_rlc_save_restore_list()
2798 adev->gfx.rlc.register_restore[i]); in gfx_v9_1_init_rlc_save_restore_list()
2802 adev->gfx.rlc.reg_list_format_start); in gfx_v9_1_init_rlc_save_restore_list()
2805 for (i = 0; i < adev->gfx.rlc.reg_list_format_direct_reg_list_length; i++) in gfx_v9_1_init_rlc_save_restore_list()
2810 while (i < (adev->gfx.rlc.reg_list_format_size_bytes >> 2)) { in gfx_v9_1_init_rlc_save_restore_list()
2832 list_size = adev->gfx.rlc.reg_list_size_bytes >> 2; in gfx_v9_1_init_rlc_save_restore_list()
2835 adev->gfx.rlc.reg_restore_list_size); in gfx_v9_1_init_rlc_save_restore_list()
2840 adev->gfx.rlc.starting_offsets_start); in gfx_v9_1_init_rlc_save_restore_list()
2897 if (adev->pg_flags & (AMD_PG_SUPPORT_GFX_PG | in gfx_v9_0_init_gfx_power_gating()
2930 if (amdgpu_ip_version(adev, GC_HWIP, 0) != IP_VERSION(9, 3, 0)) in gfx_v9_0_init_gfx_power_gating()
3041 if (adev->gfx.rlc.is_rlc_v2_1) { in gfx_v9_0_init_pg()
3043 IP_VERSION(9, 2, 1) || in gfx_v9_0_init_pg()
3044 (adev->apu_flags & AMD_APU_IS_RAVEN2)) in gfx_v9_0_init_pg()
3049 if (adev->pg_flags & (AMD_PG_SUPPORT_GFX_PG | in gfx_v9_0_init_pg()
3056 adev->gfx.rlc.cp_table_gpu_addr >> 8); in gfx_v9_0_init_pg()
3086 if (!(adev->flags & AMD_IS_APU)) { in gfx_v9_0_rlc_start()
3096 rlc_ucode_ver, adev->gfx.rlc_fw_version); in gfx_v9_0_rlc_start()
3114 if (!adev->gfx.rlc_fw) in gfx_v9_0_rlc_load_microcode()
3115 return -EINVAL; in gfx_v9_0_rlc_load_microcode()
3117 hdr = (const struct rlc_firmware_header_v2_0 *)adev->gfx.rlc_fw->data; in gfx_v9_0_rlc_load_microcode()
3118 amdgpu_ucode_print_rlc_hdr(&hdr->header); in gfx_v9_0_rlc_load_microcode()
3120 fw_data = (const __le32 *)(adev->gfx.rlc_fw->data + in gfx_v9_0_rlc_load_microcode()
3121 le32_to_cpu(hdr->header.ucode_array_offset_bytes)); in gfx_v9_0_rlc_load_microcode()
3122 fw_size = le32_to_cpu(hdr->header.ucode_size_bytes) / 4; in gfx_v9_0_rlc_load_microcode()
3128 WREG32_SOC15(GC, 0, mmRLC_GPM_UCODE_ADDR, adev->gfx.rlc_fw_version); in gfx_v9_0_rlc_load_microcode()
3142 adev->gfx.rlc.funcs->stop(adev); in gfx_v9_0_rlc_resume()
3149 if (adev->firmware.load_type != AMDGPU_FW_LOAD_PSP) { in gfx_v9_0_rlc_resume()
3157 case IP_VERSION(9, 2, 2): in gfx_v9_0_rlc_resume()
3158 case IP_VERSION(9, 1, 0): in gfx_v9_0_rlc_resume()
3165 case IP_VERSION(9, 4, 0): in gfx_v9_0_rlc_resume()
3178 adev->gfx.rlc.funcs->start(adev); in gfx_v9_0_rlc_resume()
3202 if (!adev->gfx.me_fw || !adev->gfx.pfp_fw || !adev->gfx.ce_fw) in gfx_v9_0_cp_gfx_load_microcode()
3203 return -EINVAL; in gfx_v9_0_cp_gfx_load_microcode()
3206 adev->gfx.pfp_fw->data; in gfx_v9_0_cp_gfx_load_microcode()
3208 adev->gfx.ce_fw->data; in gfx_v9_0_cp_gfx_load_microcode()
3210 adev->gfx.me_fw->data; in gfx_v9_0_cp_gfx_load_microcode()
3212 amdgpu_ucode_print_gfx_hdr(&pfp_hdr->header); in gfx_v9_0_cp_gfx_load_microcode()
3213 amdgpu_ucode_print_gfx_hdr(&ce_hdr->header); in gfx_v9_0_cp_gfx_load_microcode()
3214 amdgpu_ucode_print_gfx_hdr(&me_hdr->header); in gfx_v9_0_cp_gfx_load_microcode()
3220 (adev->gfx.pfp_fw->data + in gfx_v9_0_cp_gfx_load_microcode()
3221 le32_to_cpu(pfp_hdr->header.ucode_array_offset_bytes)); in gfx_v9_0_cp_gfx_load_microcode()
3222 fw_size = le32_to_cpu(pfp_hdr->header.ucode_size_bytes) / 4; in gfx_v9_0_cp_gfx_load_microcode()
3226 WREG32_SOC15(GC, 0, mmCP_PFP_UCODE_ADDR, adev->gfx.pfp_fw_version); in gfx_v9_0_cp_gfx_load_microcode()
3230 (adev->gfx.ce_fw->data + in gfx_v9_0_cp_gfx_load_microcode()
3231 le32_to_cpu(ce_hdr->header.ucode_array_offset_bytes)); in gfx_v9_0_cp_gfx_load_microcode()
3232 fw_size = le32_to_cpu(ce_hdr->header.ucode_size_bytes) / 4; in gfx_v9_0_cp_gfx_load_microcode()
3236 WREG32_SOC15(GC, 0, mmCP_CE_UCODE_ADDR, adev->gfx.ce_fw_version); in gfx_v9_0_cp_gfx_load_microcode()
3240 (adev->gfx.me_fw->data + in gfx_v9_0_cp_gfx_load_microcode()
3241 le32_to_cpu(me_hdr->header.ucode_array_offset_bytes)); in gfx_v9_0_cp_gfx_load_microcode()
3242 fw_size = le32_to_cpu(me_hdr->header.ucode_size_bytes) / 4; in gfx_v9_0_cp_gfx_load_microcode()
3246 WREG32_SOC15(GC, 0, mmCP_ME_RAM_WADDR, adev->gfx.me_fw_version); in gfx_v9_0_cp_gfx_load_microcode()
3253 struct amdgpu_ring *ring = &adev->gfx.gfx_ring[0]; in gfx_v9_0_cp_gfx_start()
3259 WREG32_SOC15(GC, 0, mmCP_MAX_CONTEXT, adev->gfx.config.max_hw_contexts - 1); in gfx_v9_0_cp_gfx_start()
3267 if (adev->flags & AMD_IS_APU && in gfx_v9_0_cp_gfx_start()
3268 adev->in_s3 && !adev->suspend_complete) { in gfx_v9_0_cp_gfx_start()
3285 for (sect = gfx9_cs_data; sect->section != NULL; ++sect) { in gfx_v9_0_cp_gfx_start()
3286 for (ext = sect->section; ext->extent != NULL; ++ext) { in gfx_v9_0_cp_gfx_start()
3287 if (sect->id == SECT_CONTEXT) { in gfx_v9_0_cp_gfx_start()
3290 ext->reg_count)); in gfx_v9_0_cp_gfx_start()
3292 ext->reg_index - PACKET3_SET_CONTEXT_REG_START); in gfx_v9_0_cp_gfx_start()
3293 for (i = 0; i < ext->reg_count; i++) in gfx_v9_0_cp_gfx_start()
3294 amdgpu_ring_write(ring, ext->extent[i]); in gfx_v9_0_cp_gfx_start()
3312 (SOC15_REG_OFFSET(GC, 0, mmVGT_INDEX_TYPE) - PACKET3_SET_UCONFIG_REG_START)); in gfx_v9_0_cp_gfx_start()
3335 ring = &adev->gfx.gfx_ring[0]; in gfx_v9_0_cp_gfx_resume()
3336 rb_bufsz = order_base_2(ring->ring_size / 8); in gfx_v9_0_cp_gfx_resume()
3338 tmp = REG_SET_FIELD(tmp, CP_RB0_CNTL, RB_BLKSZ, rb_bufsz - 2); in gfx_v9_0_cp_gfx_resume()
3345 ring->wptr = 0; in gfx_v9_0_cp_gfx_resume()
3346 WREG32_SOC15(GC, 0, mmCP_RB0_WPTR, lower_32_bits(ring->wptr)); in gfx_v9_0_cp_gfx_resume()
3347 WREG32_SOC15(GC, 0, mmCP_RB0_WPTR_HI, upper_32_bits(ring->wptr)); in gfx_v9_0_cp_gfx_resume()
3350 rptr_addr = ring->rptr_gpu_addr; in gfx_v9_0_cp_gfx_resume()
3354 wptr_gpu_addr = ring->wptr_gpu_addr; in gfx_v9_0_cp_gfx_resume()
3361 rb_addr = ring->gpu_addr >> 8; in gfx_v9_0_cp_gfx_resume()
3366 if (ring->use_doorbell) { in gfx_v9_0_cp_gfx_resume()
3368 DOORBELL_OFFSET, ring->doorbell_index); in gfx_v9_0_cp_gfx_resume()
3377 DOORBELL_RANGE_LOWER, ring->doorbell_index); in gfx_v9_0_cp_gfx_resume()
3397 adev->gfx.kiq[0].ring.sched.ready = false; in gfx_v9_0_cp_compute_enable()
3409 if (!adev->gfx.mec_fw) in gfx_v9_0_cp_compute_load_microcode()
3410 return -EINVAL; in gfx_v9_0_cp_compute_load_microcode()
3414 mec_hdr = (const struct gfx_firmware_header_v1_0 *)adev->gfx.mec_fw->data; in gfx_v9_0_cp_compute_load_microcode()
3415 amdgpu_ucode_print_gfx_hdr(&mec_hdr->header); in gfx_v9_0_cp_compute_load_microcode()
3418 (adev->gfx.mec_fw->data + in gfx_v9_0_cp_compute_load_microcode()
3419 le32_to_cpu(mec_hdr->header.ucode_array_offset_bytes)); in gfx_v9_0_cp_compute_load_microcode()
3426 adev->gfx.mec.mec_fw_gpu_addr & 0xFFFFF000); in gfx_v9_0_cp_compute_load_microcode()
3428 upper_32_bits(adev->gfx.mec.mec_fw_gpu_addr)); in gfx_v9_0_cp_compute_load_microcode()
3432 mec_hdr->jt_offset); in gfx_v9_0_cp_compute_load_microcode()
3433 for (i = 0; i < mec_hdr->jt_size; i++) in gfx_v9_0_cp_compute_load_microcode()
3435 le32_to_cpup(fw_data + mec_hdr->jt_offset + i)); in gfx_v9_0_cp_compute_load_microcode()
3438 adev->gfx.mec_fw_version); in gfx_v9_0_cp_compute_load_microcode()
3448 struct amdgpu_device *adev = ring->adev; in gfx_v9_0_kiq_setting()
3453 tmp |= (ring->me << 5) | (ring->pipe << 3) | (ring->queue); in gfx_v9_0_kiq_setting()
3461 struct amdgpu_device *adev = ring->adev; in gfx_v9_0_mqd_set_priority()
3463 if (ring->funcs->type == AMDGPU_RING_TYPE_COMPUTE) { in gfx_v9_0_mqd_set_priority()
3465 mqd->cp_hqd_pipe_priority = AMDGPU_GFX_PIPE_PRIO_HIGH; in gfx_v9_0_mqd_set_priority()
3466 mqd->cp_hqd_queue_priority = in gfx_v9_0_mqd_set_priority()
3474 struct amdgpu_device *adev = ring->adev; in gfx_v9_0_mqd_init()
3475 struct v9_mqd *mqd = ring->mqd_ptr; in gfx_v9_0_mqd_init()
3479 mqd->header = 0xC0310800; in gfx_v9_0_mqd_init()
3480 mqd->compute_pipelinestat_enable = 0x00000001; in gfx_v9_0_mqd_init()
3481 mqd->compute_static_thread_mgmt_se0 = 0xffffffff; in gfx_v9_0_mqd_init()
3482 mqd->compute_static_thread_mgmt_se1 = 0xffffffff; in gfx_v9_0_mqd_init()
3483 mqd->compute_static_thread_mgmt_se2 = 0xffffffff; in gfx_v9_0_mqd_init()
3484 mqd->compute_static_thread_mgmt_se3 = 0xffffffff; in gfx_v9_0_mqd_init()
3485 mqd->compute_static_thread_mgmt_se4 = 0xffffffff; in gfx_v9_0_mqd_init()
3486 mqd->compute_static_thread_mgmt_se5 = 0xffffffff; in gfx_v9_0_mqd_init()
3487 mqd->compute_static_thread_mgmt_se6 = 0xffffffff; in gfx_v9_0_mqd_init()
3488 mqd->compute_static_thread_mgmt_se7 = 0xffffffff; in gfx_v9_0_mqd_init()
3489 mqd->compute_misc_reserved = 0x00000003; in gfx_v9_0_mqd_init()
3491 mqd->dynamic_cu_mask_addr_lo = in gfx_v9_0_mqd_init()
3492 lower_32_bits(ring->mqd_gpu_addr in gfx_v9_0_mqd_init()
3494 mqd->dynamic_cu_mask_addr_hi = in gfx_v9_0_mqd_init()
3495 upper_32_bits(ring->mqd_gpu_addr in gfx_v9_0_mqd_init()
3498 eop_base_addr = ring->eop_gpu_addr >> 8; in gfx_v9_0_mqd_init()
3499 mqd->cp_hqd_eop_base_addr_lo = eop_base_addr; in gfx_v9_0_mqd_init()
3500 mqd->cp_hqd_eop_base_addr_hi = upper_32_bits(eop_base_addr); in gfx_v9_0_mqd_init()
3505 (order_base_2(GFX9_MEC_HPD_SIZE / 4) - 1)); in gfx_v9_0_mqd_init()
3507 mqd->cp_hqd_eop_control = tmp; in gfx_v9_0_mqd_init()
3512 if (ring->use_doorbell) { in gfx_v9_0_mqd_init()
3514 DOORBELL_OFFSET, ring->doorbell_index); in gfx_v9_0_mqd_init()
3526 mqd->cp_hqd_pq_doorbell_control = tmp; in gfx_v9_0_mqd_init()
3529 ring->wptr = 0; in gfx_v9_0_mqd_init()
3530 mqd->cp_hqd_dequeue_request = 0; in gfx_v9_0_mqd_init()
3531 mqd->cp_hqd_pq_rptr = 0; in gfx_v9_0_mqd_init()
3532 mqd->cp_hqd_pq_wptr_lo = 0; in gfx_v9_0_mqd_init()
3533 mqd->cp_hqd_pq_wptr_hi = 0; in gfx_v9_0_mqd_init()
3536 mqd->cp_mqd_base_addr_lo = ring->mqd_gpu_addr & 0xfffffffc; in gfx_v9_0_mqd_init()
3537 mqd->cp_mqd_base_addr_hi = upper_32_bits(ring->mqd_gpu_addr); in gfx_v9_0_mqd_init()
3542 mqd->cp_mqd_control = tmp; in gfx_v9_0_mqd_init()
3545 hqd_gpu_addr = ring->gpu_addr >> 8; in gfx_v9_0_mqd_init()
3546 mqd->cp_hqd_pq_base_lo = hqd_gpu_addr; in gfx_v9_0_mqd_init()
3547 mqd->cp_hqd_pq_base_hi = upper_32_bits(hqd_gpu_addr); in gfx_v9_0_mqd_init()
3552 (order_base_2(ring->ring_size / 4) - 1)); in gfx_v9_0_mqd_init()
3554 (order_base_2(AMDGPU_GPU_PAGE_SIZE / 4) - 1)); in gfx_v9_0_mqd_init()
3562 mqd->cp_hqd_pq_control = tmp; in gfx_v9_0_mqd_init()
3565 wb_gpu_addr = ring->rptr_gpu_addr; in gfx_v9_0_mqd_init()
3566 mqd->cp_hqd_pq_rptr_report_addr_lo = wb_gpu_addr & 0xfffffffc; in gfx_v9_0_mqd_init()
3567 mqd->cp_hqd_pq_rptr_report_addr_hi = in gfx_v9_0_mqd_init()
3571 wb_gpu_addr = ring->wptr_gpu_addr; in gfx_v9_0_mqd_init()
3572 mqd->cp_hqd_pq_wptr_poll_addr_lo = wb_gpu_addr & 0xfffffffc; in gfx_v9_0_mqd_init()
3573 mqd->cp_hqd_pq_wptr_poll_addr_hi = upper_32_bits(wb_gpu_addr) & 0xffff; in gfx_v9_0_mqd_init()
3576 ring->wptr = 0; in gfx_v9_0_mqd_init()
3577 mqd->cp_hqd_pq_rptr = RREG32_SOC15(GC, 0, mmCP_HQD_PQ_RPTR); in gfx_v9_0_mqd_init()
3580 mqd->cp_hqd_vmid = 0; in gfx_v9_0_mqd_init()
3584 mqd->cp_hqd_persistent_state = tmp; in gfx_v9_0_mqd_init()
3589 mqd->cp_hqd_ib_control = tmp; in gfx_v9_0_mqd_init()
3593 mqd->cp_hqd_quantum = RREG32_SOC15(GC, 0, mmCP_HQD_QUANTUM); in gfx_v9_0_mqd_init()
3598 if (ring->funcs->type == AMDGPU_RING_TYPE_KIQ) in gfx_v9_0_mqd_init()
3599 mqd->cp_hqd_active = 1; in gfx_v9_0_mqd_init()
3606 struct amdgpu_device *adev = ring->adev; in gfx_v9_0_kiq_init_register()
3607 struct v9_mqd *mqd = ring->mqd_ptr; in gfx_v9_0_kiq_init_register()
3614 mqd->cp_hqd_eop_base_addr_lo); in gfx_v9_0_kiq_init_register()
3616 mqd->cp_hqd_eop_base_addr_hi); in gfx_v9_0_kiq_init_register()
3620 mqd->cp_hqd_eop_control); in gfx_v9_0_kiq_init_register()
3624 mqd->cp_hqd_pq_doorbell_control); in gfx_v9_0_kiq_init_register()
3629 for (j = 0; j < adev->usec_timeout; j++) { in gfx_v9_0_kiq_init_register()
3635 mqd->cp_hqd_dequeue_request); in gfx_v9_0_kiq_init_register()
3637 mqd->cp_hqd_pq_rptr); in gfx_v9_0_kiq_init_register()
3639 mqd->cp_hqd_pq_wptr_lo); in gfx_v9_0_kiq_init_register()
3641 mqd->cp_hqd_pq_wptr_hi); in gfx_v9_0_kiq_init_register()
3646 mqd->cp_mqd_base_addr_lo); in gfx_v9_0_kiq_init_register()
3648 mqd->cp_mqd_base_addr_hi); in gfx_v9_0_kiq_init_register()
3652 mqd->cp_mqd_control); in gfx_v9_0_kiq_init_register()
3656 mqd->cp_hqd_pq_base_lo); in gfx_v9_0_kiq_init_register()
3658 mqd->cp_hqd_pq_base_hi); in gfx_v9_0_kiq_init_register()
3662 mqd->cp_hqd_pq_control); in gfx_v9_0_kiq_init_register()
3666 mqd->cp_hqd_pq_rptr_report_addr_lo); in gfx_v9_0_kiq_init_register()
3668 mqd->cp_hqd_pq_rptr_report_addr_hi); in gfx_v9_0_kiq_init_register()
3672 mqd->cp_hqd_pq_wptr_poll_addr_lo); in gfx_v9_0_kiq_init_register()
3674 mqd->cp_hqd_pq_wptr_poll_addr_hi); in gfx_v9_0_kiq_init_register()
3677 if (ring->use_doorbell) { in gfx_v9_0_kiq_init_register()
3679 (adev->doorbell_index.kiq * 2) << 2); in gfx_v9_0_kiq_init_register()
3687 (adev->doorbell.size - 4)); in gfx_v9_0_kiq_init_register()
3690 (adev->doorbell_index.userqueue_end * 2) << 2); in gfx_v9_0_kiq_init_register()
3694 mqd->cp_hqd_pq_doorbell_control); in gfx_v9_0_kiq_init_register()
3698 mqd->cp_hqd_pq_wptr_lo); in gfx_v9_0_kiq_init_register()
3700 mqd->cp_hqd_pq_wptr_hi); in gfx_v9_0_kiq_init_register()
3703 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_VMID, mqd->cp_hqd_vmid); in gfx_v9_0_kiq_init_register()
3706 mqd->cp_hqd_persistent_state); in gfx_v9_0_kiq_init_register()
3710 mqd->cp_hqd_active); in gfx_v9_0_kiq_init_register()
3712 if (ring->use_doorbell) in gfx_v9_0_kiq_init_register()
3720 struct amdgpu_device *adev = ring->adev; in gfx_v9_0_kiq_fini_register()
3728 for (j = 0; j < adev->usec_timeout; j++) { in gfx_v9_0_kiq_fini_register()
3759 struct amdgpu_device *adev = ring->adev; in gfx_v9_0_kiq_init_queue()
3760 struct v9_mqd *mqd = ring->mqd_ptr; in gfx_v9_0_kiq_init_queue()
3767 * driver need to re-init the mqd. in gfx_v9_0_kiq_init_queue()
3768 * check mqd->cp_hqd_pq_control since this value should not be 0 in gfx_v9_0_kiq_init_queue()
3770 tmp_mqd = (struct v9_mqd *)adev->gfx.kiq[0].mqd_backup; in gfx_v9_0_kiq_init_queue()
3771 if (amdgpu_in_reset(adev) && tmp_mqd->cp_hqd_pq_control){ in gfx_v9_0_kiq_init_queue()
3773 if (adev->gfx.kiq[0].mqd_backup) in gfx_v9_0_kiq_init_queue()
3774 memcpy(mqd, adev->gfx.kiq[0].mqd_backup, sizeof(struct v9_mqd_allocation)); in gfx_v9_0_kiq_init_queue()
3777 ring->wptr = 0; in gfx_v9_0_kiq_init_queue()
3780 mutex_lock(&adev->srbm_mutex); in gfx_v9_0_kiq_init_queue()
3781 soc15_grbm_select(adev, ring->me, ring->pipe, ring->queue, 0, 0); in gfx_v9_0_kiq_init_queue()
3784 mutex_unlock(&adev->srbm_mutex); in gfx_v9_0_kiq_init_queue()
3787 ((struct v9_mqd_allocation *)mqd)->dynamic_cu_mask = 0xFFFFFFFF; in gfx_v9_0_kiq_init_queue()
3788 ((struct v9_mqd_allocation *)mqd)->dynamic_rb_mask = 0xFFFFFFFF; in gfx_v9_0_kiq_init_queue()
3789 if (amdgpu_sriov_vf(adev) && adev->in_suspend) in gfx_v9_0_kiq_init_queue()
3791 mutex_lock(&adev->srbm_mutex); in gfx_v9_0_kiq_init_queue()
3792 soc15_grbm_select(adev, ring->me, ring->pipe, ring->queue, 0, 0); in gfx_v9_0_kiq_init_queue()
3796 mutex_unlock(&adev->srbm_mutex); in gfx_v9_0_kiq_init_queue()
3798 if (adev->gfx.kiq[0].mqd_backup) in gfx_v9_0_kiq_init_queue()
3799 memcpy(adev->gfx.kiq[0].mqd_backup, mqd, sizeof(struct v9_mqd_allocation)); in gfx_v9_0_kiq_init_queue()
3807 struct amdgpu_device *adev = ring->adev; in gfx_v9_0_kcq_init_queue()
3808 struct v9_mqd *mqd = ring->mqd_ptr; in gfx_v9_0_kcq_init_queue()
3809 int mqd_idx = ring - &adev->gfx.compute_ring[0]; in gfx_v9_0_kcq_init_queue()
3812 /* Same as above kiq init, driver need to re-init the mqd if mqd->cp_hqd_pq_control in gfx_v9_0_kcq_init_queue()
3815 tmp_mqd = (struct v9_mqd *)adev->gfx.mec.mqd_backup[mqd_idx]; in gfx_v9_0_kcq_init_queue()
3817 if (!restore && (!tmp_mqd->cp_hqd_pq_control || in gfx_v9_0_kcq_init_queue()
3818 (!amdgpu_in_reset(adev) && !adev->in_suspend))) { in gfx_v9_0_kcq_init_queue()
3820 ((struct v9_mqd_allocation *)mqd)->dynamic_cu_mask = 0xFFFFFFFF; in gfx_v9_0_kcq_init_queue()
3821 ((struct v9_mqd_allocation *)mqd)->dynamic_rb_mask = 0xFFFFFFFF; in gfx_v9_0_kcq_init_queue()
3822 mutex_lock(&adev->srbm_mutex); in gfx_v9_0_kcq_init_queue()
3823 soc15_grbm_select(adev, ring->me, ring->pipe, ring->queue, 0, 0); in gfx_v9_0_kcq_init_queue()
3826 mutex_unlock(&adev->srbm_mutex); in gfx_v9_0_kcq_init_queue()
3828 if (adev->gfx.mec.mqd_backup[mqd_idx]) in gfx_v9_0_kcq_init_queue()
3829 memcpy(adev->gfx.mec.mqd_backup[mqd_idx], mqd, sizeof(struct v9_mqd_allocation)); in gfx_v9_0_kcq_init_queue()
3832 if (adev->gfx.mec.mqd_backup[mqd_idx]) in gfx_v9_0_kcq_init_queue()
3833 memcpy(mqd, adev->gfx.mec.mqd_backup[mqd_idx], sizeof(struct v9_mqd_allocation)); in gfx_v9_0_kcq_init_queue()
3835 ring->wptr = 0; in gfx_v9_0_kcq_init_queue()
3836 atomic64_set((atomic64_t *)ring->wptr_cpu_addr, 0); in gfx_v9_0_kcq_init_queue()
3848 ring = &adev->gfx.kiq[0].ring; in gfx_v9_0_kiq_resume()
3850 r = amdgpu_bo_reserve(ring->mqd_obj, false); in gfx_v9_0_kiq_resume()
3854 r = amdgpu_bo_kmap(ring->mqd_obj, (void **)&ring->mqd_ptr); in gfx_v9_0_kiq_resume()
3856 amdgpu_bo_unreserve(ring->mqd_obj); in gfx_v9_0_kiq_resume()
3861 amdgpu_bo_kunmap(ring->mqd_obj); in gfx_v9_0_kiq_resume()
3862 ring->mqd_ptr = NULL; in gfx_v9_0_kiq_resume()
3863 amdgpu_bo_unreserve(ring->mqd_obj); in gfx_v9_0_kiq_resume()
3874 for (i = 0; i < adev->gfx.num_compute_rings; i++) { in gfx_v9_0_kcq_resume()
3875 ring = &adev->gfx.compute_ring[i]; in gfx_v9_0_kcq_resume()
3877 r = amdgpu_bo_reserve(ring->mqd_obj, false); in gfx_v9_0_kcq_resume()
3880 r = amdgpu_bo_kmap(ring->mqd_obj, (void **)&ring->mqd_ptr); in gfx_v9_0_kcq_resume()
3883 amdgpu_bo_kunmap(ring->mqd_obj); in gfx_v9_0_kcq_resume()
3884 ring->mqd_ptr = NULL; in gfx_v9_0_kcq_resume()
3886 amdgpu_bo_unreserve(ring->mqd_obj); in gfx_v9_0_kcq_resume()
3901 if (!(adev->flags & AMD_IS_APU)) in gfx_v9_0_cp_resume()
3904 if (adev->firmware.load_type != AMDGPU_FW_LOAD_PSP) { in gfx_v9_0_cp_resume()
3905 if (adev->gfx.num_gfx_rings) { in gfx_v9_0_cp_resume()
3921 if (adev->gfx.num_gfx_rings) { in gfx_v9_0_cp_resume()
3931 if (adev->gfx.num_gfx_rings) { in gfx_v9_0_cp_resume()
3932 ring = &adev->gfx.gfx_ring[0]; in gfx_v9_0_cp_resume()
3938 for (i = 0; i < adev->gfx.num_compute_rings; i++) { in gfx_v9_0_cp_resume()
3939 ring = &adev->gfx.compute_ring[i]; in gfx_v9_0_cp_resume()
3952 if (amdgpu_ip_version(adev, GC_HWIP, 0) != IP_VERSION(9, 4, 1) && in gfx_v9_0_init_tcp_config()
3953 amdgpu_ip_version(adev, GC_HWIP, 0) != IP_VERSION(9, 4, 2)) in gfx_v9_0_init_tcp_config()
3958 adev->df.hash_status.hash_64k); in gfx_v9_0_init_tcp_config()
3960 adev->df.hash_status.hash_2m); in gfx_v9_0_init_tcp_config()
3962 adev->df.hash_status.hash_1g); in gfx_v9_0_init_tcp_config()
3968 if (adev->gfx.num_gfx_rings) in gfx_v9_0_cp_enable()
3978 amdgpu_gfx_cleaner_shader_init(adev, adev->gfx.cleaner_shader_size, in gfx_v9_0_hw_init()
3979 adev->gfx.cleaner_shader_ptr); in gfx_v9_0_hw_init()
3988 r = adev->gfx.rlc.funcs->resume(adev); in gfx_v9_0_hw_init()
3996 if (amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(9, 4, 2)) in gfx_v9_0_hw_init()
4007 amdgpu_irq_put(adev, &adev->gfx.cp_ecc_error_irq, 0); in gfx_v9_0_hw_fini()
4008 amdgpu_irq_put(adev, &adev->gfx.priv_reg_irq, 0); in gfx_v9_0_hw_fini()
4009 amdgpu_irq_put(adev, &adev->gfx.priv_inst_irq, 0); in gfx_v9_0_hw_fini()
4010 amdgpu_irq_put(adev, &adev->gfx.bad_op_irq, 0); in gfx_v9_0_hw_fini()
4031 if (!amdgpu_in_reset(adev) && !adev->in_suspend) { in gfx_v9_0_hw_fini()
4032 mutex_lock(&adev->srbm_mutex); in gfx_v9_0_hw_fini()
4033 soc15_grbm_select(adev, adev->gfx.kiq[0].ring.me, in gfx_v9_0_hw_fini()
4034 adev->gfx.kiq[0].ring.pipe, in gfx_v9_0_hw_fini()
4035 adev->gfx.kiq[0].ring.queue, 0, 0); in gfx_v9_0_hw_fini()
4036 gfx_v9_0_kiq_fini_register(&adev->gfx.kiq[0].ring); in gfx_v9_0_hw_fini()
4038 mutex_unlock(&adev->srbm_mutex); in gfx_v9_0_hw_fini()
4044 if ((adev->gmc.xgmi.connected_to_cpu && amdgpu_in_reset(adev)) || in gfx_v9_0_hw_fini()
4045 (amdgpu_ip_version(adev, GC_HWIP, 0) >= IP_VERSION(9, 4, 2))) { in gfx_v9_0_hw_fini()
4046 dev_dbg(adev->dev, "Skipping RLC halt\n"); in gfx_v9_0_hw_fini()
4050 adev->gfx.rlc.funcs->stop(adev); in gfx_v9_0_hw_fini()
4080 for (i = 0; i < adev->usec_timeout; i++) { in gfx_v9_0_wait_for_idle()
4085 return -ETIMEDOUT; in gfx_v9_0_wait_for_idle()
4122 adev->gfx.rlc.funcs->stop(adev); in gfx_v9_0_soft_reset()
4124 if (adev->gfx.num_gfx_rings) in gfx_v9_0_soft_reset()
4134 dev_info(adev->dev, "GRBM_SOFT_RESET=0x%08X\n", tmp); in gfx_v9_0_soft_reset()
4157 struct amdgpu_kiq *kiq = &adev->gfx.kiq[0]; in gfx_v9_0_kiq_read_clock()
4158 struct amdgpu_ring *ring = &kiq->ring; in gfx_v9_0_kiq_read_clock()
4160 BUG_ON(!ring->funcs->emit_rreg); in gfx_v9_0_kiq_read_clock()
4162 spin_lock_irqsave(&kiq->ring_lock, flags); in gfx_v9_0_kiq_read_clock()
4169 amdgpu_ring_write(ring, 9 | /* src: register*/ in gfx_v9_0_kiq_read_clock()
4175 amdgpu_ring_write(ring, lower_32_bits(adev->wb.gpu_addr + in gfx_v9_0_kiq_read_clock()
4177 amdgpu_ring_write(ring, upper_32_bits(adev->wb.gpu_addr + in gfx_v9_0_kiq_read_clock()
4184 spin_unlock_irqrestore(&kiq->ring_lock, flags); in gfx_v9_0_kiq_read_clock()
4209 value = (uint64_t)adev->wb.wb[reg_val_offs] | in gfx_v9_0_kiq_read_clock()
4210 (uint64_t)adev->wb.wb[reg_val_offs + 1 ] << 32ULL; in gfx_v9_0_kiq_read_clock()
4217 spin_unlock_irqrestore(&kiq->ring_lock, flags); in gfx_v9_0_kiq_read_clock()
4230 case IP_VERSION(9, 3, 0): in gfx_v9_0_get_gpu_clock_counter()
4235 /* The SMUIO TSC clock frequency is 100MHz, which sets 32-bit carry over in gfx_v9_0_get_gpu_clock_counter()
4247 mutex_lock(&adev->gfx.gpu_clock_mutex); in gfx_v9_0_get_gpu_clock_counter()
4249 IP_VERSION(9, 0, 1) && in gfx_v9_0_get_gpu_clock_counter()
4257 mutex_unlock(&adev->gfx.gpu_clock_mutex); in gfx_v9_0_get_gpu_clock_counter()
4270 struct amdgpu_device *adev = ring->adev; in gfx_v9_0_ring_emit_gds_switch()
4290 (1 << (oa_size + oa_base)) - (1 << oa_base)); in gfx_v9_0_ring_emit_gds_switch()
4531 struct amdgpu_ring *ring = &adev->gfx.compute_ring[0]; in gfx_v9_0_do_edc_gds_workarounds()
4541 ring->name, r); in gfx_v9_0_do_edc_gds_workarounds()
4546 WREG32_SOC15(GC, 0, mmGDS_VMID0_SIZE, adev->gds.gds_size); in gfx_v9_0_do_edc_gds_workarounds()
4558 adev->gds.gds_size); in gfx_v9_0_do_edc_gds_workarounds()
4562 for (i = 0; i < adev->usec_timeout; i++) { in gfx_v9_0_do_edc_gds_workarounds()
4563 if (ring->wptr == gfx_v9_0_ring_get_rptr_compute(ring)) in gfx_v9_0_do_edc_gds_workarounds()
4568 if (i >= adev->usec_timeout) in gfx_v9_0_do_edc_gds_workarounds()
4569 r = -ETIMEDOUT; in gfx_v9_0_do_edc_gds_workarounds()
4578 struct amdgpu_ring *ring = &adev->gfx.compute_ring[0]; in gfx_v9_0_do_edc_gpr_workarounds()
4585 int compute_dim_x = adev->gfx.config.max_shader_engines * in gfx_v9_0_do_edc_gpr_workarounds()
4586 adev->gfx.config.max_cu_per_sh * in gfx_v9_0_do_edc_gpr_workarounds()
4587 adev->gfx.config.max_sh_per_se; in gfx_v9_0_do_edc_gpr_workarounds()
4589 int gpr_reg_size = adev->gfx.config.max_shader_engines + 6; in gfx_v9_0_do_edc_gpr_workarounds()
4599 if (!ring->sched.ready) in gfx_v9_0_do_edc_gpr_workarounds()
4602 if (amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(9, 4, 1)) { in gfx_v9_0_do_edc_gpr_workarounds()
4648 - PACKET3_SET_SH_REG_START; in gfx_v9_0_do_edc_gpr_workarounds()
4655 - PACKET3_SET_SH_REG_START; in gfx_v9_0_do_edc_gpr_workarounds()
4676 - PACKET3_SET_SH_REG_START; in gfx_v9_0_do_edc_gpr_workarounds()
4683 - PACKET3_SET_SH_REG_START; in gfx_v9_0_do_edc_gpr_workarounds()
4704 - PACKET3_SET_SH_REG_START; in gfx_v9_0_do_edc_gpr_workarounds()
4711 - PACKET3_SET_SH_REG_START; in gfx_v9_0_do_edc_gpr_workarounds()
4752 adev->gfx.funcs = &gfx_v9_0_gfx_funcs; in gfx_v9_0_early_init()
4754 if (amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(9, 4, 1) || in gfx_v9_0_early_init()
4755 amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(9, 4, 2)) in gfx_v9_0_early_init()
4756 adev->gfx.num_gfx_rings = 0; in gfx_v9_0_early_init()
4758 adev->gfx.num_gfx_rings = GFX9_NUM_GFX_RINGS; in gfx_v9_0_early_init()
4759 adev->gfx.xcc_mask = 1; in gfx_v9_0_early_init()
4760 adev->gfx.num_compute_rings = min(amdgpu_gfx_get_num_kcq(adev), in gfx_v9_0_early_init()
4785 if ((!adev->in_suspend) && in gfx_v9_0_ecc_late_init()
4786 (adev->gds.gds_size)) { in gfx_v9_0_ecc_late_init()
4793 if (amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(9, 4, 2)) in gfx_v9_0_ecc_late_init()
4801 if (adev->gfx.ras && in gfx_v9_0_ecc_late_init()
4802 adev->gfx.ras->enable_watchdog_timer) in gfx_v9_0_ecc_late_init()
4803 adev->gfx.ras->enable_watchdog_timer(adev); in gfx_v9_0_ecc_late_init()
4813 r = amdgpu_irq_get(adev, &adev->gfx.priv_reg_irq, 0); in gfx_v9_0_late_init()
4817 r = amdgpu_irq_get(adev, &adev->gfx.priv_inst_irq, 0); in gfx_v9_0_late_init()
4821 r = amdgpu_irq_get(adev, &adev->gfx.bad_op_irq, 0); in gfx_v9_0_late_init()
4829 if (amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(9, 4, 2)) in gfx_v9_0_late_init()
4831 adev->vm_manager.first_kfd_vmid, AMDGPU_NUM_VMID); in gfx_v9_0_late_init()
4834 adev->vm_manager.first_kfd_vmid, AMDGPU_NUM_VMID); in gfx_v9_0_late_init()
4861 for (i = 0; i < adev->usec_timeout; i++) { in gfx_v9_0_set_safe_mode()
4881 if ((adev->pg_flags & AMD_PG_SUPPORT_GFX_PG) && enable) { in gfx_v9_0_update_gfx_cg_power_gating()
4883 if (adev->pg_flags & AMD_PG_SUPPORT_GFX_PIPELINE) in gfx_v9_0_update_gfx_cg_power_gating()
4887 if (adev->pg_flags & AMD_PG_SUPPORT_GFX_PIPELINE) in gfx_v9_0_update_gfx_cg_power_gating()
4900 if ((adev->pg_flags & AMD_PG_SUPPORT_GFX_SMG) && enable) in gfx_v9_0_update_gfx_mg_power_gating()
4905 if ((adev->pg_flags & AMD_PG_SUPPORT_GFX_DMG) && enable) in gfx_v9_0_update_gfx_mg_power_gating()
4921 if (enable && (adev->cg_flags & AMD_CG_SUPPORT_GFX_MGCG)) { in gfx_v9_0_update_medium_grain_clock_gating()
4922 /* 1 - RLC_CGTT_MGCG_OVERRIDE */ in gfx_v9_0_update_medium_grain_clock_gating()
4925 if (amdgpu_ip_version(adev, GC_HWIP, 0) != IP_VERSION(9, 2, 1)) in gfx_v9_0_update_medium_grain_clock_gating()
4939 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_MGLS) { in gfx_v9_0_update_medium_grain_clock_gating()
4940 /* 2 - RLC memory Light sleep */ in gfx_v9_0_update_medium_grain_clock_gating()
4941 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_RLC_LS) { in gfx_v9_0_update_medium_grain_clock_gating()
4947 /* 3 - CP memory Light sleep */ in gfx_v9_0_update_medium_grain_clock_gating()
4948 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_CP_LS) { in gfx_v9_0_update_medium_grain_clock_gating()
4956 /* 1 - MGCG_OVERRIDE */ in gfx_v9_0_update_medium_grain_clock_gating()
4959 if (amdgpu_ip_version(adev, GC_HWIP, 0) != IP_VERSION(9, 2, 1)) in gfx_v9_0_update_medium_grain_clock_gating()
4970 /* 2 - disable MGLS in RLC */ in gfx_v9_0_update_medium_grain_clock_gating()
4977 /* 3 - disable MGLS in CP */ in gfx_v9_0_update_medium_grain_clock_gating()
4993 if (!adev->gfx.num_gfx_rings) in gfx_v9_0_update_3d_clock_gating()
5011 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_3D_CGCG) in gfx_v9_0_update_3d_clock_gating()
5017 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_3D_CGLS) in gfx_v9_0_update_3d_clock_gating()
5050 if (enable && (adev->cg_flags & AMD_CG_SUPPORT_GFX_CGCG)) { in gfx_v9_0_update_coarse_grain_clock_gating()
5054 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_CGLS) in gfx_v9_0_update_coarse_grain_clock_gating()
5065 if (amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(9, 4, 1)) in gfx_v9_0_update_coarse_grain_clock_gating()
5071 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_CGLS) in gfx_v9_0_update_coarse_grain_clock_gating()
5163 reg = adev->reg_offset[entry->hwip][entry->instance][entry->segment] + entry->reg; in gfx_v9_0_check_rlcg_range()
5201 case IP_VERSION(9, 2, 2): in gfx_v9_0_set_powergating_state()
5202 case IP_VERSION(9, 1, 0): in gfx_v9_0_set_powergating_state()
5203 case IP_VERSION(9, 3, 0): in gfx_v9_0_set_powergating_state()
5207 if (adev->pg_flags & AMD_PG_SUPPORT_RLC_SMU_HS) { in gfx_v9_0_set_powergating_state()
5215 if (adev->pg_flags & AMD_PG_SUPPORT_CP) in gfx_v9_0_set_powergating_state()
5229 case IP_VERSION(9, 2, 1): in gfx_v9_0_set_powergating_state()
5248 case IP_VERSION(9, 0, 1): in gfx_v9_0_set_clockgating_state()
5249 case IP_VERSION(9, 2, 1): in gfx_v9_0_set_clockgating_state()
5250 case IP_VERSION(9, 4, 0): in gfx_v9_0_set_clockgating_state()
5251 case IP_VERSION(9, 2, 2): in gfx_v9_0_set_clockgating_state()
5252 case IP_VERSION(9, 1, 0): in gfx_v9_0_set_clockgating_state()
5253 case IP_VERSION(9, 4, 1): in gfx_v9_0_set_clockgating_state()
5254 case IP_VERSION(9, 3, 0): in gfx_v9_0_set_clockgating_state()
5255 case IP_VERSION(9, 4, 2): in gfx_v9_0_set_clockgating_state()
5297 if (amdgpu_ip_version(adev, GC_HWIP, 0) != IP_VERSION(9, 4, 1)) { in gfx_v9_0_get_clockgating_state()
5311 return *ring->rptr_cpu_addr; /* gfx9 is 32bit rptr*/ in gfx_v9_0_ring_get_rptr_gfx()
5316 struct amdgpu_device *adev = ring->adev; in gfx_v9_0_ring_get_wptr_gfx()
5320 if (ring->use_doorbell) { in gfx_v9_0_ring_get_wptr_gfx()
5321 wptr = atomic64_read((atomic64_t *)ring->wptr_cpu_addr); in gfx_v9_0_ring_get_wptr_gfx()
5332 struct amdgpu_device *adev = ring->adev; in gfx_v9_0_ring_set_wptr_gfx()
5334 if (ring->use_doorbell) { in gfx_v9_0_ring_set_wptr_gfx()
5336 atomic64_set((atomic64_t *)ring->wptr_cpu_addr, ring->wptr); in gfx_v9_0_ring_set_wptr_gfx()
5337 WDOORBELL64(ring->doorbell_index, ring->wptr); in gfx_v9_0_ring_set_wptr_gfx()
5339 WREG32_SOC15(GC, 0, mmCP_RB0_WPTR, lower_32_bits(ring->wptr)); in gfx_v9_0_ring_set_wptr_gfx()
5340 WREG32_SOC15(GC, 0, mmCP_RB0_WPTR_HI, upper_32_bits(ring->wptr)); in gfx_v9_0_ring_set_wptr_gfx()
5346 struct amdgpu_device *adev = ring->adev; in gfx_v9_0_ring_emit_hdp_flush()
5348 const struct nbio_hdp_flush_reg *nbio_hf_reg = adev->nbio.hdp_flush_reg; in gfx_v9_0_ring_emit_hdp_flush()
5350 if (ring->funcs->type == AMDGPU_RING_TYPE_COMPUTE) { in gfx_v9_0_ring_emit_hdp_flush()
5351 switch (ring->me) { in gfx_v9_0_ring_emit_hdp_flush()
5353 ref_and_mask = nbio_hf_reg->ref_and_mask_cp2 << ring->pipe; in gfx_v9_0_ring_emit_hdp_flush()
5356 ref_and_mask = nbio_hf_reg->ref_and_mask_cp6 << ring->pipe; in gfx_v9_0_ring_emit_hdp_flush()
5363 ref_and_mask = nbio_hf_reg->ref_and_mask_cp0; in gfx_v9_0_ring_emit_hdp_flush()
5368 adev->nbio.funcs->get_hdp_flush_req_offset(adev), in gfx_v9_0_ring_emit_hdp_flush()
5369 adev->nbio.funcs->get_hdp_flush_done_offset(adev), in gfx_v9_0_ring_emit_hdp_flush()
5381 if (ib->flags & AMDGPU_IB_FLAG_CE) in gfx_v9_0_ring_emit_ib_gfx()
5386 control |= ib->length_dw | (vmid << 24); in gfx_v9_0_ring_emit_ib_gfx()
5388 if (ib->flags & AMDGPU_IB_FLAG_PREEMPT) { in gfx_v9_0_ring_emit_ib_gfx()
5394 if (!(ib->flags & AMDGPU_IB_FLAG_CE) && vmid) in gfx_v9_0_ring_emit_ib_gfx()
5396 (!amdgpu_sriov_vf(ring->adev) && in gfx_v9_0_ring_emit_ib_gfx()
5399 job->gds_size > 0 && job->gds_base != 0); in gfx_v9_0_ring_emit_ib_gfx()
5403 BUG_ON(ib->gpu_addr & 0x3); /* Dword align */ in gfx_v9_0_ring_emit_ib_gfx()
5408 lower_32_bits(ib->gpu_addr)); in gfx_v9_0_ring_emit_ib_gfx()
5409 amdgpu_ring_write(ring, upper_32_bits(ib->gpu_addr)); in gfx_v9_0_ring_emit_ib_gfx()
5417 u32 control = ring->ring[offset]; in gfx_v9_0_ring_patch_cntl()
5420 ring->ring[offset] = control; in gfx_v9_0_ring_patch_cntl()
5426 struct amdgpu_device *adev = ring->adev; in gfx_v9_0_ring_patch_ce_meta()
5432 if (ring->is_mes_queue) { in gfx_v9_0_ring_patch_ce_meta()
5440 ce_payload_cpu_addr = adev->virt.csa_cpu_addr + payload_offset; in gfx_v9_0_ring_patch_ce_meta()
5443 if (offset + (payload_size >> 2) <= ring->buf_mask + 1) { in gfx_v9_0_ring_patch_ce_meta()
5444 memcpy((void *)&ring->ring[offset], ce_payload_cpu_addr, payload_size); in gfx_v9_0_ring_patch_ce_meta()
5446 memcpy((void *)&ring->ring[offset], ce_payload_cpu_addr, in gfx_v9_0_ring_patch_ce_meta()
5447 (ring->buf_mask + 1 - offset) << 2); in gfx_v9_0_ring_patch_ce_meta()
5448 payload_size -= (ring->buf_mask + 1 - offset) << 2; in gfx_v9_0_ring_patch_ce_meta()
5449 memcpy((void *)&ring->ring[0], in gfx_v9_0_ring_patch_ce_meta()
5450 ce_payload_cpu_addr + ((ring->buf_mask + 1 - offset) << 2), in gfx_v9_0_ring_patch_ce_meta()
5458 struct amdgpu_device *adev = ring->adev; in gfx_v9_0_ring_patch_de_meta()
5464 if (ring->is_mes_queue) { in gfx_v9_0_ring_patch_de_meta()
5472 de_payload_cpu_addr = adev->virt.csa_cpu_addr + payload_offset; in gfx_v9_0_ring_patch_de_meta()
5475 ((struct v9_de_ib_state *)de_payload_cpu_addr)->ib_completion_status = in gfx_v9_0_ring_patch_de_meta()
5478 if (offset + (payload_size >> 2) <= ring->buf_mask + 1) { in gfx_v9_0_ring_patch_de_meta()
5479 memcpy((void *)&ring->ring[offset], de_payload_cpu_addr, payload_size); in gfx_v9_0_ring_patch_de_meta()
5481 memcpy((void *)&ring->ring[offset], de_payload_cpu_addr, in gfx_v9_0_ring_patch_de_meta()
5482 (ring->buf_mask + 1 - offset) << 2); in gfx_v9_0_ring_patch_de_meta()
5483 payload_size -= (ring->buf_mask + 1 - offset) << 2; in gfx_v9_0_ring_patch_de_meta()
5484 memcpy((void *)&ring->ring[0], in gfx_v9_0_ring_patch_de_meta()
5485 de_payload_cpu_addr + ((ring->buf_mask + 1 - offset) << 2), in gfx_v9_0_ring_patch_de_meta()
5496 u32 control = INDIRECT_BUFFER_VALID | ib->length_dw | (vmid << 24); in gfx_v9_0_ring_emit_ib_compute()
5508 if (ib->flags & AMDGPU_IB_FLAG_RESET_GDS_MAX_WAVE_ID) { in gfx_v9_0_ring_emit_ib_compute()
5511 amdgpu_ring_write(ring, ring->adev->gds.gds_compute_max_wave_id); in gfx_v9_0_ring_emit_ib_compute()
5515 BUG_ON(ib->gpu_addr & 0x3); /* Dword align */ in gfx_v9_0_ring_emit_ib_compute()
5520 lower_32_bits(ib->gpu_addr)); in gfx_v9_0_ring_emit_ib_compute()
5521 amdgpu_ring_write(ring, upper_32_bits(ib->gpu_addr)); in gfx_v9_0_ring_emit_ib_compute()
5534 /* RELEASE_MEM - flush caches, send int */ in gfx_v9_0_ring_emit_fence()
5568 int usepfp = (ring->funcs->type == AMDGPU_RING_TYPE_GFX); in gfx_v9_0_ring_emit_pipeline_sync()
5569 uint32_t seq = ring->fence_drv.sync_seq; in gfx_v9_0_ring_emit_pipeline_sync()
5570 uint64_t addr = ring->fence_drv.gpu_addr; in gfx_v9_0_ring_emit_pipeline_sync()
5583 if (ring->funcs->type == AMDGPU_RING_TYPE_GFX) { in gfx_v9_0_ring_emit_vm_flush()
5592 return *ring->rptr_cpu_addr; /* gfx9 hardware is 32bit rptr */ in gfx_v9_0_ring_get_rptr_compute()
5600 if (ring->use_doorbell) in gfx_v9_0_ring_get_wptr_compute()
5601 wptr = atomic64_read((atomic64_t *)ring->wptr_cpu_addr); in gfx_v9_0_ring_get_wptr_compute()
5609 struct amdgpu_device *adev = ring->adev; in gfx_v9_0_ring_set_wptr_compute()
5612 if (ring->use_doorbell) { in gfx_v9_0_ring_set_wptr_compute()
5613 atomic64_set((atomic64_t *)ring->wptr_cpu_addr, ring->wptr); in gfx_v9_0_ring_set_wptr_compute()
5614 WDOORBELL64(ring->doorbell_index, ring->wptr); in gfx_v9_0_ring_set_wptr_compute()
5623 struct amdgpu_device *adev = ring->adev; in gfx_v9_0_ring_emit_fence_kiq()
5655 struct amdgpu_device *adev = ring->adev; in gfx_v9_0_ring_emit_ce_meta()
5661 cnt = (sizeof(ce_payload) >> 2) + 4 - 2; in gfx_v9_0_ring_emit_ce_meta()
5663 if (ring->is_mes_queue) { in gfx_v9_0_ring_emit_ce_meta()
5673 ce_payload_gpu_addr = amdgpu_csa_vaddr(ring->adev) + offset; in gfx_v9_0_ring_emit_ce_meta()
5674 ce_payload_cpu_addr = adev->virt.csa_cpu_addr + offset; in gfx_v9_0_ring_emit_ce_meta()
5698 struct amdgpu_device *adev = ring->adev; in gfx_v9_0_ring_preempt_ib()
5699 struct amdgpu_kiq *kiq = &adev->gfx.kiq[0]; in gfx_v9_0_ring_preempt_ib()
5700 struct amdgpu_ring *kiq_ring = &kiq->ring; in gfx_v9_0_ring_preempt_ib()
5703 if (!kiq->pmf || !kiq->pmf->kiq_unmap_queues) in gfx_v9_0_ring_preempt_ib()
5704 return -EINVAL; in gfx_v9_0_ring_preempt_ib()
5706 spin_lock_irqsave(&kiq->ring_lock, flags); in gfx_v9_0_ring_preempt_ib()
5708 if (amdgpu_ring_alloc(kiq_ring, kiq->pmf->unmap_queues_size)) { in gfx_v9_0_ring_preempt_ib()
5709 spin_unlock_irqrestore(&kiq->ring_lock, flags); in gfx_v9_0_ring_preempt_ib()
5710 return -ENOMEM; in gfx_v9_0_ring_preempt_ib()
5716 ring->trail_seq += 1; in gfx_v9_0_ring_preempt_ib()
5718 gfx_v9_0_ring_emit_fence(ring, ring->trail_fence_gpu_addr, in gfx_v9_0_ring_preempt_ib()
5719 ring->trail_seq, AMDGPU_FENCE_FLAG_EXEC | AMDGPU_FENCE_FLAG_INT); in gfx_v9_0_ring_preempt_ib()
5722 kiq->pmf->kiq_unmap_queues(kiq_ring, ring, PREEMPT_QUEUES_NO_UNMAP, in gfx_v9_0_ring_preempt_ib()
5723 ring->trail_fence_gpu_addr, in gfx_v9_0_ring_preempt_ib()
5724 ring->trail_seq); in gfx_v9_0_ring_preempt_ib()
5727 spin_unlock_irqrestore(&kiq->ring_lock, flags); in gfx_v9_0_ring_preempt_ib()
5730 for (i = 0; i < adev->usec_timeout; i++) { in gfx_v9_0_ring_preempt_ib()
5731 if (ring->trail_seq == in gfx_v9_0_ring_preempt_ib()
5732 le32_to_cpu(*ring->trail_fence_cpu_addr)) in gfx_v9_0_ring_preempt_ib()
5737 if (i >= adev->usec_timeout) { in gfx_v9_0_ring_preempt_ib()
5738 r = -EINVAL; in gfx_v9_0_ring_preempt_ib()
5739 DRM_WARN("ring %d timeout to preempt ib\n", ring->idx); in gfx_v9_0_ring_preempt_ib()
5755 struct amdgpu_device *adev = ring->adev; in gfx_v9_0_ring_emit_de_meta()
5761 if (ring->is_mes_queue) { in gfx_v9_0_ring_emit_de_meta()
5776 de_payload_gpu_addr = amdgpu_csa_vaddr(ring->adev) + offset; in gfx_v9_0_ring_emit_de_meta()
5777 de_payload_cpu_addr = adev->virt.csa_cpu_addr + offset; in gfx_v9_0_ring_emit_de_meta()
5779 gds_addr = ALIGN(amdgpu_csa_vaddr(ring->adev) + in gfx_v9_0_ring_emit_de_meta()
5780 AMDGPU_CSA_SIZE - adev->gds.gds_size, in gfx_v9_0_ring_emit_de_meta()
5789 cnt = (sizeof(de_payload) >> 2) + 4 - 2; in gfx_v9_0_ring_emit_de_meta()
5821 (!amdgpu_sriov_vf(ring->adev) && in gfx_v9_ring_emit_cntxcntl()
5858 ret = ring->wptr & ring->buf_mask; in gfx_v9_0_ring_emit_init_cond_exec()
5867 struct amdgpu_device *adev = ring->adev; in gfx_v9_0_ring_emit_rreg()
5875 amdgpu_ring_write(ring, lower_32_bits(adev->wb.gpu_addr + in gfx_v9_0_ring_emit_rreg()
5877 amdgpu_ring_write(ring, upper_32_bits(adev->wb.gpu_addr + in gfx_v9_0_ring_emit_rreg()
5886 switch (ring->funcs->type) { in gfx_v9_0_ring_emit_wreg()
5914 int usepfp = (ring->funcs->type == AMDGPU_RING_TYPE_GFX); in gfx_v9_0_ring_emit_reg_write_reg_wait()
5915 struct amdgpu_device *adev = ring->adev; in gfx_v9_0_ring_emit_reg_write_reg_wait()
5916 bool fw_version_ok = (ring->funcs->type == AMDGPU_RING_TYPE_GFX) ? in gfx_v9_0_ring_emit_reg_write_reg_wait()
5917 adev->gfx.me_fw_write_wait : adev->gfx.mec_fw_write_wait; in gfx_v9_0_ring_emit_reg_write_reg_wait()
5929 struct amdgpu_device *adev = ring->adev; in gfx_v9_0_ring_soft_recovery()
6048 for (i = 0; i < adev->gfx.mec.num_mec; i++) { in gfx_v9_0_set_priv_reg_fault_state()
6049 for (j = 0; j < adev->gfx.mec.num_pipe_per_mec; j++) { in gfx_v9_0_set_priv_reg_fault_state()
6084 for (i = 0; i < adev->gfx.mec.num_mec; i++) { in gfx_v9_0_set_bad_op_fault_state()
6085 for (j = 0; j < adev->gfx.mec.num_pipe_per_mec; j++) { in gfx_v9_0_set_bad_op_fault_state()
6212 me_id = (entry->ring_id & 0x0c) >> 2; in gfx_v9_0_eop_irq()
6213 pipe_id = (entry->ring_id & 0x03) >> 0; in gfx_v9_0_eop_irq()
6214 queue_id = (entry->ring_id & 0x70) >> 4; in gfx_v9_0_eop_irq()
6218 if (adev->gfx.num_gfx_rings) { in gfx_v9_0_eop_irq()
6219 if (!adev->gfx.mcbp) { in gfx_v9_0_eop_irq()
6220 amdgpu_fence_process(&adev->gfx.gfx_ring[0]); in gfx_v9_0_eop_irq()
6221 } else if (!amdgpu_mcbp_handle_trailing_fence_irq(&adev->gfx.muxer)) { in gfx_v9_0_eop_irq()
6224 amdgpu_fence_process(&adev->gfx.sw_gfx_ring[i]); in gfx_v9_0_eop_irq()
6230 for (i = 0; i < adev->gfx.num_compute_rings; i++) { in gfx_v9_0_eop_irq()
6231 ring = &adev->gfx.compute_ring[i]; in gfx_v9_0_eop_irq()
6232 /* Per-queue interrupt is supported for MEC starting from VI. in gfx_v9_0_eop_irq()
6235 if ((ring->me == me_id) && (ring->pipe == pipe_id) && (ring->queue == queue_id)) in gfx_v9_0_eop_irq()
6250 me_id = (entry->ring_id & 0x0c) >> 2; in gfx_v9_0_fault()
6251 pipe_id = (entry->ring_id & 0x03) >> 0; in gfx_v9_0_fault()
6252 queue_id = (entry->ring_id & 0x70) >> 4; in gfx_v9_0_fault()
6256 drm_sched_fault(&adev->gfx.gfx_ring[0].sched); in gfx_v9_0_fault()
6260 for (i = 0; i < adev->gfx.num_compute_rings; i++) { in gfx_v9_0_fault()
6261 ring = &adev->gfx.compute_ring[i]; in gfx_v9_0_fault()
6262 if (ring->me == me_id && ring->pipe == pipe_id && in gfx_v9_0_fault()
6263 ring->queue == queue_id) in gfx_v9_0_fault()
6264 drm_sched_fault(&ring->sched); in gfx_v9_0_fault()
6747 return -EINVAL; in gfx_v9_0_ras_error_inject()
6749 if (info->head.sub_block_index >= ARRAY_SIZE(ras_gfx_subblocks)) in gfx_v9_0_ras_error_inject()
6750 return -EINVAL; in gfx_v9_0_ras_error_inject()
6752 if (!ras_gfx_subblocks[info->head.sub_block_index].name) in gfx_v9_0_ras_error_inject()
6753 return -EPERM; in gfx_v9_0_ras_error_inject()
6755 if (!(ras_gfx_subblocks[info->head.sub_block_index].hw_supported_error_type & in gfx_v9_0_ras_error_inject()
6756 info->head.type)) { in gfx_v9_0_ras_error_inject()
6758 ras_gfx_subblocks[info->head.sub_block_index].name, in gfx_v9_0_ras_error_inject()
6759 info->head.type); in gfx_v9_0_ras_error_inject()
6760 return -EPERM; in gfx_v9_0_ras_error_inject()
6763 if (!(ras_gfx_subblocks[info->head.sub_block_index].sw_supported_error_type & in gfx_v9_0_ras_error_inject()
6764 info->head.type)) { in gfx_v9_0_ras_error_inject()
6766 ras_gfx_subblocks[info->head.sub_block_index].name, in gfx_v9_0_ras_error_inject()
6767 info->head.type); in gfx_v9_0_ras_error_inject()
6768 return -EPERM; in gfx_v9_0_ras_error_inject()
6771 block_info.block_id = amdgpu_ras_block_to_ta(info->head.block); in gfx_v9_0_ras_error_inject()
6773 ras_gfx_subblocks[info->head.sub_block_index].ta_subblock; in gfx_v9_0_ras_error_inject()
6774 block_info.inject_error_type = amdgpu_ras_error_to_ta(info->head.type); in gfx_v9_0_ras_error_inject()
6775 block_info.address = info->address; in gfx_v9_0_ras_error_inject()
6776 block_info.value = info->value; in gfx_v9_0_ras_error_inject()
6778 mutex_lock(&adev->grbm_idx_mutex); in gfx_v9_0_ras_error_inject()
6779 ret = psp_ras_trigger_error(&adev->psp, &block_info, instance_mask); in gfx_v9_0_ras_error_inject()
6780 mutex_unlock(&adev->grbm_idx_mutex); in gfx_v9_0_ras_error_inject()
6877 dev_info(adev->dev, "Instance[%d]: SubBlock %s, " in gfx_v9_0_query_utc_edc_status()
6879 err_data->ce_count += sec_count; in gfx_v9_0_query_utc_edc_status()
6884 dev_info(adev->dev, "Instance[%d]: SubBlock %s, " in gfx_v9_0_query_utc_edc_status()
6886 err_data->ue_count += ded_count; in gfx_v9_0_query_utc_edc_status()
6897 dev_info(adev->dev, "Instance[%d]: SubBlock %s, " in gfx_v9_0_query_utc_edc_status()
6899 err_data->ce_count += sec_count; in gfx_v9_0_query_utc_edc_status()
6905 dev_info(adev->dev, "Instance[%d]: SubBlock %s, " in gfx_v9_0_query_utc_edc_status()
6907 err_data->ue_count += ded_count; in gfx_v9_0_query_utc_edc_status()
6917 dev_info(adev->dev, "Instance[%d]: SubBlock %s, " in gfx_v9_0_query_utc_edc_status()
6920 err_data->ce_count += sec_count; in gfx_v9_0_query_utc_edc_status()
6930 dev_info(adev->dev, "Instance[%d]: SubBlock %s, " in gfx_v9_0_query_utc_edc_status()
6933 err_data->ce_count += sec_count; in gfx_v9_0_query_utc_edc_status()
6938 dev_info(adev->dev, "Instance[%d]: SubBlock %s, " in gfx_v9_0_query_utc_edc_status()
6941 err_data->ue_count += ded_count; in gfx_v9_0_query_utc_edc_status()
6962 if(gfx_v9_0_ras_fields[i].reg_offset != reg->reg_offset || in gfx_v9_0_ras_error_count()
6963 gfx_v9_0_ras_fields[i].seg != reg->seg || in gfx_v9_0_ras_error_count()
6964 gfx_v9_0_ras_fields[i].inst != reg->inst) in gfx_v9_0_ras_error_count()
6971 dev_info(adev->dev, "GFX SubBlock %s, " in gfx_v9_0_ras_error_count()
6983 dev_info(adev->dev, "GFX SubBlock %s, " in gfx_v9_0_ras_error_count()
7003 mutex_lock(&adev->grbm_idx_mutex); in gfx_v9_0_reset_ras_error_count()
7013 mutex_unlock(&adev->grbm_idx_mutex); in gfx_v9_0_reset_ras_error_count()
7061 err_data->ue_count = 0; in gfx_v9_0_query_ras_error_count()
7062 err_data->ce_count = 0; in gfx_v9_0_query_ras_error_count()
7064 mutex_lock(&adev->grbm_idx_mutex); in gfx_v9_0_query_ras_error_count()
7081 err_data->ce_count += sec_count; in gfx_v9_0_query_ras_error_count()
7082 err_data->ue_count += ded_count; in gfx_v9_0_query_ras_error_count()
7085 mutex_unlock(&adev->grbm_idx_mutex); in gfx_v9_0_query_ras_error_count()
7099 /* ACQUIRE_MEM -make one or more surfaces valid for use by the subsequent operations */ in gfx_v9_0_emit_mem_sync()
7112 struct amdgpu_device *adev = ring->adev; in gfx_v9_0_emit_wave_limit_cs()
7116 /* mmSPI_WCL_PIPE_PERCENT_CS[0-7]_DEFAULT values are same */ in gfx_v9_0_emit_wave_limit_cs()
7142 struct amdgpu_device *adev = ring->adev; in gfx_v9_0_emit_wave_limit()
7159 * amdgpu controls only 1st ME(0-3 CS pipes). in gfx_v9_0_emit_wave_limit()
7161 for (i = 0; i < adev->gfx.mec.num_pipe_per_mec; i++) { in gfx_v9_0_emit_wave_limit()
7162 if (i != ring->pipe) in gfx_v9_0_emit_wave_limit()
7174 amdgpu_ring_write(ring, ring->funcs->nop); in gfx_v9_ring_insert_nop()
7179 amdgpu_ring_write(ring, PACKET3(PACKET3_NOP, min(num_nop - 2, 0x3ffe))); in gfx_v9_ring_insert_nop()
7181 /* Header is at index 0, followed by num_nops - 1 NOP packet's */ in gfx_v9_ring_insert_nop()
7183 amdgpu_ring_write(ring, ring->funcs->nop); in gfx_v9_ring_insert_nop()
7188 struct amdgpu_device *adev = ring->adev; in gfx_v9_0_reset_kgq()
7189 struct amdgpu_kiq *kiq = &adev->gfx.kiq[0]; in gfx_v9_0_reset_kgq()
7190 struct amdgpu_ring *kiq_ring = &kiq->ring; in gfx_v9_0_reset_kgq()
7196 return -EINVAL; in gfx_v9_0_reset_kgq()
7198 if (!kiq->pmf || !kiq->pmf->kiq_unmap_queues) in gfx_v9_0_reset_kgq()
7199 return -EINVAL; in gfx_v9_0_reset_kgq()
7201 spin_lock_irqsave(&kiq->ring_lock, flags); in gfx_v9_0_reset_kgq()
7204 spin_unlock_irqrestore(&kiq->ring_lock, flags); in gfx_v9_0_reset_kgq()
7205 return -ENOMEM; in gfx_v9_0_reset_kgq()
7213 spin_unlock_irqrestore(&kiq->ring_lock, flags); in gfx_v9_0_reset_kgq()
7220 return -ENOMEM; in gfx_v9_0_reset_kgq()
7221 gfx_v9_0_ring_emit_fence(ring, ring->fence_drv.gpu_addr, in gfx_v9_0_reset_kgq()
7222 ring->fence_drv.sync_seq, AMDGPU_FENCE_FLAG_EXEC); in gfx_v9_0_reset_kgq()
7234 struct amdgpu_device *adev = ring->adev; in gfx_v9_0_reset_kcq()
7235 struct amdgpu_kiq *kiq = &adev->gfx.kiq[0]; in gfx_v9_0_reset_kcq()
7236 struct amdgpu_ring *kiq_ring = &kiq->ring; in gfx_v9_0_reset_kcq()
7240 if (!adev->debug_exp_resets && in gfx_v9_0_reset_kcq()
7241 !adev->gfx.num_gfx_rings) in gfx_v9_0_reset_kcq()
7242 return -EINVAL; in gfx_v9_0_reset_kcq()
7245 return -EINVAL; in gfx_v9_0_reset_kcq()
7247 if (!kiq->pmf || !kiq->pmf->kiq_unmap_queues) in gfx_v9_0_reset_kcq()
7248 return -EINVAL; in gfx_v9_0_reset_kcq()
7250 spin_lock_irqsave(&kiq->ring_lock, flags); in gfx_v9_0_reset_kcq()
7252 if (amdgpu_ring_alloc(kiq_ring, kiq->pmf->unmap_queues_size)) { in gfx_v9_0_reset_kcq()
7253 spin_unlock_irqrestore(&kiq->ring_lock, flags); in gfx_v9_0_reset_kcq()
7254 return -ENOMEM; in gfx_v9_0_reset_kcq()
7257 kiq->pmf->kiq_unmap_queues(kiq_ring, ring, RESET_QUEUES, in gfx_v9_0_reset_kcq()
7261 spin_unlock_irqrestore(&kiq->ring_lock, flags); in gfx_v9_0_reset_kcq()
7269 mutex_lock(&adev->srbm_mutex); in gfx_v9_0_reset_kcq()
7270 soc15_grbm_select(adev, ring->me, ring->pipe, ring->queue, 0, 0); in gfx_v9_0_reset_kcq()
7271 for (i = 0; i < adev->usec_timeout; i++) { in gfx_v9_0_reset_kcq()
7276 if (i >= adev->usec_timeout) in gfx_v9_0_reset_kcq()
7277 r = -ETIMEDOUT; in gfx_v9_0_reset_kcq()
7279 mutex_unlock(&adev->srbm_mutex); in gfx_v9_0_reset_kcq()
7282 dev_err(adev->dev, "fail to wait on hqd deactive\n"); in gfx_v9_0_reset_kcq()
7286 r = amdgpu_bo_reserve(ring->mqd_obj, false); in gfx_v9_0_reset_kcq()
7288 dev_err(adev->dev, "fail to resv mqd_obj\n"); in gfx_v9_0_reset_kcq()
7291 r = amdgpu_bo_kmap(ring->mqd_obj, (void **)&ring->mqd_ptr); in gfx_v9_0_reset_kcq()
7294 amdgpu_bo_kunmap(ring->mqd_obj); in gfx_v9_0_reset_kcq()
7295 ring->mqd_ptr = NULL; in gfx_v9_0_reset_kcq()
7297 amdgpu_bo_unreserve(ring->mqd_obj); in gfx_v9_0_reset_kcq()
7299 dev_err(adev->dev, "fail to unresv mqd_obj\n"); in gfx_v9_0_reset_kcq()
7302 spin_lock_irqsave(&kiq->ring_lock, flags); in gfx_v9_0_reset_kcq()
7303 r = amdgpu_ring_alloc(kiq_ring, kiq->pmf->map_queues_size); in gfx_v9_0_reset_kcq()
7305 spin_unlock_irqrestore(&kiq->ring_lock, flags); in gfx_v9_0_reset_kcq()
7306 return -ENOMEM; in gfx_v9_0_reset_kcq()
7308 kiq->pmf->kiq_map_queues(kiq_ring, ring); in gfx_v9_0_reset_kcq()
7310 spin_unlock_irqrestore(&kiq->ring_lock, flags); in gfx_v9_0_reset_kcq()
7325 if (!adev->gfx.ip_dump_core) in gfx_v9_ip_print()
7329 drm_printf(p, "%-50s \t 0x%08x\n", in gfx_v9_ip_print()
7331 adev->gfx.ip_dump_core[i]); in gfx_v9_ip_print()
7334 if (!adev->gfx.ip_dump_compute_queues) in gfx_v9_ip_print()
7339 adev->gfx.mec.num_mec, in gfx_v9_ip_print()
7340 adev->gfx.mec.num_pipe_per_mec, in gfx_v9_ip_print()
7341 adev->gfx.mec.num_queue_per_pipe); in gfx_v9_ip_print()
7343 for (i = 0; i < adev->gfx.mec.num_mec; i++) { in gfx_v9_ip_print()
7344 for (j = 0; j < adev->gfx.mec.num_pipe_per_mec; j++) { in gfx_v9_ip_print()
7345 for (k = 0; k < adev->gfx.mec.num_queue_per_pipe; k++) { in gfx_v9_ip_print()
7348 drm_printf(p, "%-50s \t 0x%08x\n", in gfx_v9_ip_print()
7350 adev->gfx.ip_dump_compute_queues[index + reg]); in gfx_v9_ip_print()
7365 if (!adev->gfx.ip_dump_core || !adev->gfx.num_gfx_rings) in gfx_v9_ip_dump()
7370 adev->gfx.ip_dump_core[i] = RREG32(SOC15_REG_ENTRY_OFFSET(gc_reg_list_9[i])); in gfx_v9_ip_dump()
7374 if (!adev->gfx.ip_dump_compute_queues) in gfx_v9_ip_dump()
7379 mutex_lock(&adev->srbm_mutex); in gfx_v9_ip_dump()
7380 for (i = 0; i < adev->gfx.mec.num_mec; i++) { in gfx_v9_ip_dump()
7381 for (j = 0; j < adev->gfx.mec.num_pipe_per_mec; j++) { in gfx_v9_ip_dump()
7382 for (k = 0; k < adev->gfx.mec.num_queue_per_pipe; k++) { in gfx_v9_ip_dump()
7387 adev->gfx.ip_dump_compute_queues[index + reg] = in gfx_v9_ip_dump()
7396 mutex_unlock(&adev->srbm_mutex); in gfx_v9_ip_dump()
7619 adev->gfx.kiq[0].ring.funcs = &gfx_v9_0_ring_funcs_kiq; in gfx_v9_0_set_ring_funcs()
7621 for (i = 0; i < adev->gfx.num_gfx_rings; i++) in gfx_v9_0_set_ring_funcs()
7622 adev->gfx.gfx_ring[i].funcs = &gfx_v9_0_ring_funcs_gfx; in gfx_v9_0_set_ring_funcs()
7624 if (adev->gfx.mcbp && adev->gfx.num_gfx_rings) { in gfx_v9_0_set_ring_funcs()
7626 adev->gfx.sw_gfx_ring[i].funcs = &gfx_v9_0_sw_ring_funcs_gfx; in gfx_v9_0_set_ring_funcs()
7629 for (i = 0; i < adev->gfx.num_compute_rings; i++) in gfx_v9_0_set_ring_funcs()
7630 adev->gfx.compute_ring[i].funcs = &gfx_v9_0_ring_funcs_compute; in gfx_v9_0_set_ring_funcs()
7661 adev->gfx.eop_irq.num_types = AMDGPU_CP_IRQ_LAST; in gfx_v9_0_set_irq_funcs()
7662 adev->gfx.eop_irq.funcs = &gfx_v9_0_eop_irq_funcs; in gfx_v9_0_set_irq_funcs()
7664 adev->gfx.priv_reg_irq.num_types = 1; in gfx_v9_0_set_irq_funcs()
7665 adev->gfx.priv_reg_irq.funcs = &gfx_v9_0_priv_reg_irq_funcs; in gfx_v9_0_set_irq_funcs()
7667 adev->gfx.bad_op_irq.num_types = 1; in gfx_v9_0_set_irq_funcs()
7668 adev->gfx.bad_op_irq.funcs = &gfx_v9_0_bad_op_irq_funcs; in gfx_v9_0_set_irq_funcs()
7670 adev->gfx.priv_inst_irq.num_types = 1; in gfx_v9_0_set_irq_funcs()
7671 adev->gfx.priv_inst_irq.funcs = &gfx_v9_0_priv_inst_irq_funcs; in gfx_v9_0_set_irq_funcs()
7673 adev->gfx.cp_ecc_error_irq.num_types = 2; /*C5 ECC error and C9 FUE error*/ in gfx_v9_0_set_irq_funcs()
7674 adev->gfx.cp_ecc_error_irq.funcs = &gfx_v9_0_cp_ecc_error_irq_funcs; in gfx_v9_0_set_irq_funcs()
7680 case IP_VERSION(9, 0, 1): in gfx_v9_0_set_rlc_funcs()
7681 case IP_VERSION(9, 2, 1): in gfx_v9_0_set_rlc_funcs()
7682 case IP_VERSION(9, 4, 0): in gfx_v9_0_set_rlc_funcs()
7683 case IP_VERSION(9, 2, 2): in gfx_v9_0_set_rlc_funcs()
7684 case IP_VERSION(9, 1, 0): in gfx_v9_0_set_rlc_funcs()
7685 case IP_VERSION(9, 4, 1): in gfx_v9_0_set_rlc_funcs()
7686 case IP_VERSION(9, 3, 0): in gfx_v9_0_set_rlc_funcs()
7687 case IP_VERSION(9, 4, 2): in gfx_v9_0_set_rlc_funcs()
7688 adev->gfx.rlc.funcs = &gfx_v9_0_rlc_funcs; in gfx_v9_0_set_rlc_funcs()
7699 case IP_VERSION(9, 0, 1): in gfx_v9_0_set_gds_init()
7700 case IP_VERSION(9, 2, 1): in gfx_v9_0_set_gds_init()
7701 case IP_VERSION(9, 4, 0): in gfx_v9_0_set_gds_init()
7702 adev->gds.gds_size = 0x10000; in gfx_v9_0_set_gds_init()
7704 case IP_VERSION(9, 2, 2): in gfx_v9_0_set_gds_init()
7705 case IP_VERSION(9, 1, 0): in gfx_v9_0_set_gds_init()
7706 case IP_VERSION(9, 4, 1): in gfx_v9_0_set_gds_init()
7707 adev->gds.gds_size = 0x1000; in gfx_v9_0_set_gds_init()
7709 case IP_VERSION(9, 4, 2): in gfx_v9_0_set_gds_init()
7713 adev->gds.gds_size = 0; in gfx_v9_0_set_gds_init()
7716 adev->gds.gds_size = 0x10000; in gfx_v9_0_set_gds_init()
7721 case IP_VERSION(9, 0, 1): in gfx_v9_0_set_gds_init()
7722 case IP_VERSION(9, 4, 0): in gfx_v9_0_set_gds_init()
7723 adev->gds.gds_compute_max_wave_id = 0x7ff; in gfx_v9_0_set_gds_init()
7725 case IP_VERSION(9, 2, 1): in gfx_v9_0_set_gds_init()
7726 adev->gds.gds_compute_max_wave_id = 0x27f; in gfx_v9_0_set_gds_init()
7728 case IP_VERSION(9, 2, 2): in gfx_v9_0_set_gds_init()
7729 case IP_VERSION(9, 1, 0): in gfx_v9_0_set_gds_init()
7730 if (adev->apu_flags & AMD_APU_IS_RAVEN2) in gfx_v9_0_set_gds_init()
7731 adev->gds.gds_compute_max_wave_id = 0x77; /* raven2 */ in gfx_v9_0_set_gds_init()
7733 adev->gds.gds_compute_max_wave_id = 0x15f; /* raven1 */ in gfx_v9_0_set_gds_init()
7735 case IP_VERSION(9, 4, 1): in gfx_v9_0_set_gds_init()
7736 adev->gds.gds_compute_max_wave_id = 0xfff; in gfx_v9_0_set_gds_init()
7738 case IP_VERSION(9, 4, 2): in gfx_v9_0_set_gds_init()
7740 adev->gds.gds_compute_max_wave_id = 0; in gfx_v9_0_set_gds_init()
7744 adev->gds.gds_compute_max_wave_id = 0x7ff; in gfx_v9_0_set_gds_init()
7748 adev->gds.gws_size = 64; in gfx_v9_0_set_gds_init()
7749 adev->gds.oa_size = 16; in gfx_v9_0_set_gds_init()
7776 mask = amdgpu_gfx_create_bitmask(adev->gfx.config.max_cu_per_sh); in gfx_v9_0_get_cu_active_bitmap()
7789 return -EINVAL; in gfx_v9_0_get_cu_info()
7794 if (adev->gfx.config.max_shader_engines * in gfx_v9_0_get_cu_info()
7795 adev->gfx.config.max_sh_per_se > 16) in gfx_v9_0_get_cu_info()
7796 return -EINVAL; in gfx_v9_0_get_cu_info()
7799 adev->gfx.config.max_shader_engines, in gfx_v9_0_get_cu_info()
7800 adev->gfx.config.max_sh_per_se); in gfx_v9_0_get_cu_info()
7802 mutex_lock(&adev->grbm_idx_mutex); in gfx_v9_0_get_cu_info()
7803 for (i = 0; i < adev->gfx.config.max_shader_engines; i++) { in gfx_v9_0_get_cu_info()
7804 for (j = 0; j < adev->gfx.config.max_sh_per_se; j++) { in gfx_v9_0_get_cu_info()
7810 adev, disable_masks[i * adev->gfx.config.max_sh_per_se + j]); in gfx_v9_0_get_cu_info()
7820 * SE4,SH0 --> bitmap[0][1] in gfx_v9_0_get_cu_info()
7821 * SE5,SH0 --> bitmap[1][1] in gfx_v9_0_get_cu_info()
7822 * SE6,SH0 --> bitmap[2][1] in gfx_v9_0_get_cu_info()
7823 * SE7,SH0 --> bitmap[3][1] in gfx_v9_0_get_cu_info()
7825 cu_info->bitmap[0][i % 4][j + i / 4] = bitmap; in gfx_v9_0_get_cu_info()
7827 for (k = 0; k < adev->gfx.config.max_cu_per_sh; k ++) { in gfx_v9_0_get_cu_info()
7829 if (counter < adev->gfx.config.max_cu_per_sh) in gfx_v9_0_get_cu_info()
7838 cu_info->ao_cu_bitmap[i % 4][j + i / 4] = ao_bitmap; in gfx_v9_0_get_cu_info()
7842 mutex_unlock(&adev->grbm_idx_mutex); in gfx_v9_0_get_cu_info()
7844 cu_info->number = active_cu_number; in gfx_v9_0_get_cu_info()
7845 cu_info->ao_cu_mask = ao_cu_mask; in gfx_v9_0_get_cu_info()
7846 cu_info->simd_per_cu = NUM_SIMD_PER_CU; in gfx_v9_0_get_cu_info()
7854 .major = 9,