379 	/* TCC (5 sub-ranges)*/
416 	/* TCC range 4*/
442 	/* EA (3 sub-ranges)*/
479 	/* UTC ATC L2 4KB cache*/
854 	mmRLC_SRM_INDEX_CNTL_ADDR_0 - mmRLC_SRM_INDEX_CNTL_ADDR_0,
855 	mmRLC_SRM_INDEX_CNTL_ADDR_1 - mmRLC_SRM_INDEX_CNTL_ADDR_0,
856 	mmRLC_SRM_INDEX_CNTL_ADDR_2 - mmRLC_SRM_INDEX_CNTL_ADDR_0,
857 	mmRLC_SRM_INDEX_CNTL_ADDR_3 - mmRLC_SRM_INDEX_CNTL_ADDR_0,
858 	mmRLC_SRM_INDEX_CNTL_ADDR_4 - mmRLC_SRM_INDEX_CNTL_ADDR_0,
859 	mmRLC_SRM_INDEX_CNTL_ADDR_5 - mmRLC_SRM_INDEX_CNTL_ADDR_0,
860 	mmRLC_SRM_INDEX_CNTL_ADDR_6 - mmRLC_SRM_INDEX_CNTL_ADDR_0,
861 	mmRLC_SRM_INDEX_CNTL_ADDR_7 - mmRLC_SRM_INDEX_CNTL_ADDR_0,
866 	mmRLC_SRM_INDEX_CNTL_DATA_0 - mmRLC_SRM_INDEX_CNTL_DATA_0,
867 	mmRLC_SRM_INDEX_CNTL_DATA_1 - mmRLC_SRM_INDEX_CNTL_DATA_0,
868 	mmRLC_SRM_INDEX_CNTL_DATA_2 - mmRLC_SRM_INDEX_CNTL_DATA_0,
869 	mmRLC_SRM_INDEX_CNTL_DATA_3 - mmRLC_SRM_INDEX_CNTL_DATA_0,
870 	mmRLC_SRM_INDEX_CNTL_DATA_4 - mmRLC_SRM_INDEX_CNTL_DATA_0,
871 	mmRLC_SRM_INDEX_CNTL_DATA_5 - mmRLC_SRM_INDEX_CNTL_DATA_0,
872 	mmRLC_SRM_INDEX_CNTL_DATA_6 - mmRLC_SRM_INDEX_CNTL_DATA_0,
873 	mmRLC_SRM_INDEX_CNTL_DATA_7 - mmRLC_SRM_INDEX_CNTL_DATA_0,
888 static void gfx_v9_0_ring_emit_de_meta(struct amdgpu_ring *ring, bool resume, bool usegds);
889 static u64 gfx_v9_0_ring_get_rptr_compute(struct amdgpu_ring *ring);
903 	struct amdgpu_device *adev = kiq_ring->adev;  in gfx_v9_0_kiq_set_resources()
907 	shader_mc_addr = adev->gfx.cleaner_shader_gpu_addr >> 8;  in gfx_v9_0_kiq_set_resources()
925 				 struct amdgpu_ring *ring)  in gfx_v9_0_kiq_map_queues()  argument
927 	uint64_t mqd_addr = amdgpu_bo_gpu_offset(ring->mqd_obj);  in gfx_v9_0_kiq_map_queues()
928 	uint64_t wptr_addr = ring->wptr_gpu_addr;  in gfx_v9_0_kiq_map_queues()
929 	uint32_t eng_sel = ring->funcs->type == AMDGPU_RING_TYPE_GFX ? 4 : 0;  in gfx_v9_0_kiq_map_queues()
936 			 PACKET3_MAP_QUEUES_QUEUE(ring->queue) |  in gfx_v9_0_kiq_map_queues()
937 			 PACKET3_MAP_QUEUES_PIPE(ring->pipe) |  in gfx_v9_0_kiq_map_queues()
938 			 PACKET3_MAP_QUEUES_ME((ring->me == 1 ? 0 : 1)) |  in gfx_v9_0_kiq_map_queues()
947 			PACKET3_MAP_QUEUES_DOORBELL_OFFSET(ring->doorbell_index));  in gfx_v9_0_kiq_map_queues()
955 				   struct amdgpu_ring *ring,  in gfx_v9_0_kiq_unmap_queues()  argument
959 	uint32_t eng_sel = ring->funcs->type == AMDGPU_RING_TYPE_GFX ? 4 : 0;  in gfx_v9_0_kiq_unmap_queues()
961 	amdgpu_ring_write(kiq_ring, PACKET3(PACKET3_UNMAP_QUEUES, 4));  in gfx_v9_0_kiq_unmap_queues()
968 			PACKET3_UNMAP_QUEUES_DOORBELL_OFFSET0(ring->doorbell_index));  in gfx_v9_0_kiq_unmap_queues()
971 		amdgpu_ring_write(kiq_ring, lower_32_bits(ring->wptr & ring->buf_mask));  in gfx_v9_0_kiq_unmap_queues()
983 				   struct amdgpu_ring *ring,  in gfx_v9_0_kiq_query_status()  argument
987 	uint32_t eng_sel = ring->funcs->type == AMDGPU_RING_TYPE_GFX ? 4 : 0;  in gfx_v9_0_kiq_query_status()
996 			PACKET3_QUERY_STATUS_DOORBELL_OFFSET(ring->doorbell_index) |  in gfx_v9_0_kiq_query_status()
1021 	struct amdgpu_device *adev = kiq_ring->adev;  in gfx_v9_0_kiq_reset_hw_queue()
1026 	mutex_lock(&adev->srbm_mutex);  in gfx_v9_0_kiq_reset_hw_queue()
1033 		for (i = 0; i < adev->usec_timeout; i++) {  in gfx_v9_0_kiq_reset_hw_queue()
1038 		if (i >= adev->usec_timeout)  in gfx_v9_0_kiq_reset_hw_queue()
1039 			dev_err(adev->dev, "fail to wait on hqd deactive\n");  in gfx_v9_0_kiq_reset_hw_queue()
1041 		dev_err(adev->dev, "reset queue_type(%d) not supported\n", queue_type);  in gfx_v9_0_kiq_reset_hw_queue()
1045 	mutex_unlock(&adev->srbm_mutex);  in gfx_v9_0_kiq_reset_hw_queue()
1066 	adev->gfx.kiq[0].pmf = &gfx_v9_0_kiq_pm4_funcs;  in gfx_v9_0_set_kiq_pm4_funcs()
1088 	case IP_VERSION(9, 4, 0):  in gfx_v9_0_init_golden_registers()
1096 	case IP_VERSION(9, 4, 1):  in gfx_v9_0_init_golden_registers()
1105 		if (adev->apu_flags & AMD_APU_IS_RAVEN2)  in gfx_v9_0_init_golden_registers()
1119 	case IP_VERSION(9, 4, 2):  in gfx_v9_0_init_golden_registers()
1121 						 adev->smuio.funcs->get_die_id(adev));  in gfx_v9_0_init_golden_registers()
1127 	if ((amdgpu_ip_version(adev, GC_HWIP, 0) != IP_VERSION(9, 4, 1)) &&  in gfx_v9_0_init_golden_registers()
1128 	    (amdgpu_ip_version(adev, GC_HWIP, 0) != IP_VERSION(9, 4, 2)))  in gfx_v9_0_init_golden_registers()
1133 static void gfx_v9_0_write_data_to_reg(struct amdgpu_ring *ring, int eng_sel,  in gfx_v9_0_write_data_to_reg()  argument
1136 	amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));  in gfx_v9_0_write_data_to_reg()
1137 	amdgpu_ring_write(ring, WRITE_DATA_ENGINE_SEL(eng_sel) |  in gfx_v9_0_write_data_to_reg()
1140 	amdgpu_ring_write(ring, reg);  in gfx_v9_0_write_data_to_reg()
1141 	amdgpu_ring_write(ring, 0);  in gfx_v9_0_write_data_to_reg()
1142 	amdgpu_ring_write(ring, val);  in gfx_v9_0_write_data_to_reg()
1145 static void gfx_v9_0_wait_reg_mem(struct amdgpu_ring *ring, int eng_sel,  in gfx_v9_0_wait_reg_mem()  argument
1150 	amdgpu_ring_write(ring, PACKET3(PACKET3_WAIT_REG_MEM, 5));  in gfx_v9_0_wait_reg_mem()
1151 	amdgpu_ring_write(ring,  in gfx_v9_0_wait_reg_mem()
1160 	amdgpu_ring_write(ring, addr0);  in gfx_v9_0_wait_reg_mem()
1161 	amdgpu_ring_write(ring, addr1);  in gfx_v9_0_wait_reg_mem()
1162 	amdgpu_ring_write(ring, ref);  in gfx_v9_0_wait_reg_mem()
1163 	amdgpu_ring_write(ring, mask);  in gfx_v9_0_wait_reg_mem()
1164 	amdgpu_ring_write(ring, inv); /* poll interval */  in gfx_v9_0_wait_reg_mem()
1167 static int gfx_v9_0_ring_test_ring(struct amdgpu_ring *ring)  in gfx_v9_0_ring_test_ring()  argument
1169 	struct amdgpu_device *adev = ring->adev;  in gfx_v9_0_ring_test_ring()
1176 	r = amdgpu_ring_alloc(ring, 3);  in gfx_v9_0_ring_test_ring()
1180 	amdgpu_ring_write(ring, PACKET3(PACKET3_SET_UCONFIG_REG, 1));  in gfx_v9_0_ring_test_ring()
1181 	amdgpu_ring_write(ring, scratch - PACKET3_SET_UCONFIG_REG_START);  in gfx_v9_0_ring_test_ring()
1182 	amdgpu_ring_write(ring, 0xDEADBEEF);  in gfx_v9_0_ring_test_ring()
1183 	amdgpu_ring_commit(ring);  in gfx_v9_0_ring_test_ring()
1185 	for (i = 0; i < adev->usec_timeout; i++) {  in gfx_v9_0_ring_test_ring()
1192 	if (i >= adev->usec_timeout)  in gfx_v9_0_ring_test_ring()
1193 		r = -ETIMEDOUT;  in gfx_v9_0_ring_test_ring()
1197 static int gfx_v9_0_ring_test_ib(struct amdgpu_ring *ring, long timeout)  in gfx_v9_0_ring_test_ib()  argument
1199 	struct amdgpu_device *adev = ring->adev;  in gfx_v9_0_ring_test_ib()
1212 	gpu_addr = adev->wb.gpu_addr + (index * 4);  in gfx_v9_0_ring_test_ib()
1213 	adev->wb.wb[index] = cpu_to_le32(0xCAFEDEAD);  in gfx_v9_0_ring_test_ib()
1224 	ib.ptr[4] = 0xDEADBEEF;  in gfx_v9_0_ring_test_ib()
1227 	r = amdgpu_ib_schedule(ring, 1, &ib, NULL, &f);  in gfx_v9_0_ring_test_ib()
1233 		r = -ETIMEDOUT;  in gfx_v9_0_ring_test_ib()
1239 	tmp = adev->wb.wb[index];  in gfx_v9_0_ring_test_ib()
1243 		r = -EINVAL;  in gfx_v9_0_ring_test_ib()
1256 	amdgpu_ucode_release(&adev->gfx.pfp_fw);  in gfx_v9_0_free_microcode()
1257 	amdgpu_ucode_release(&adev->gfx.me_fw);  in gfx_v9_0_free_microcode()
1258 	amdgpu_ucode_release(&adev->gfx.ce_fw);  in gfx_v9_0_free_microcode()
1259 	amdgpu_ucode_release(&adev->gfx.rlc_fw);  in gfx_v9_0_free_microcode()
1260 	amdgpu_ucode_release(&adev->gfx.mec_fw);  in gfx_v9_0_free_microcode()
1261 	amdgpu_ucode_release(&adev->gfx.mec2_fw);  in gfx_v9_0_free_microcode()
1263 	kfree(adev->gfx.rlc.register_list_format);  in gfx_v9_0_free_microcode()
1268 	adev->gfx.me_fw_write_wait = false;  in gfx_v9_0_check_fw_write_wait()
1269 	adev->gfx.mec_fw_write_wait = false;  in gfx_v9_0_check_fw_write_wait()
1271 	if ((amdgpu_ip_version(adev, GC_HWIP, 0) != IP_VERSION(9, 4, 1)) &&  in gfx_v9_0_check_fw_write_wait()
1272 	    ((adev->gfx.mec_fw_version < 0x000001a5) ||  in gfx_v9_0_check_fw_write_wait()
1273 	     (adev->gfx.mec_feature_version < 46) ||  in gfx_v9_0_check_fw_write_wait()
1274 	     (adev->gfx.pfp_fw_version < 0x000000b7) ||  in gfx_v9_0_check_fw_write_wait()
1275 	     (adev->gfx.pfp_feature_version < 46)))  in gfx_v9_0_check_fw_write_wait()
1280 		if ((adev->gfx.me_fw_version >= 0x0000009c) &&  in gfx_v9_0_check_fw_write_wait()
1281 		    (adev->gfx.me_feature_version >= 42) &&  in gfx_v9_0_check_fw_write_wait()
1282 		    (adev->gfx.pfp_fw_version >=  0x000000b1) &&  in gfx_v9_0_check_fw_write_wait()
1283 		    (adev->gfx.pfp_feature_version >= 42))  in gfx_v9_0_check_fw_write_wait()
1284 			adev->gfx.me_fw_write_wait = true;  in gfx_v9_0_check_fw_write_wait()
1286 		if ((adev->gfx.mec_fw_version >=  0x00000193) &&  in gfx_v9_0_check_fw_write_wait()
1287 		    (adev->gfx.mec_feature_version >= 42))  in gfx_v9_0_check_fw_write_wait()
1288 			adev->gfx.mec_fw_write_wait = true;  in gfx_v9_0_check_fw_write_wait()
1291 		if ((adev->gfx.me_fw_version >= 0x0000009c) &&  in gfx_v9_0_check_fw_write_wait()
1292 		    (adev->gfx.me_feature_version >= 44) &&  in gfx_v9_0_check_fw_write_wait()
1293 		    (adev->gfx.pfp_fw_version >=  0x000000b2) &&  in gfx_v9_0_check_fw_write_wait()
1294 		    (adev->gfx.pfp_feature_version >= 44))  in gfx_v9_0_check_fw_write_wait()
1295 			adev->gfx.me_fw_write_wait = true;  in gfx_v9_0_check_fw_write_wait()
1297 		if ((adev->gfx.mec_fw_version >=  0x00000196) &&  in gfx_v9_0_check_fw_write_wait()
1298 		    (adev->gfx.mec_feature_version >= 44))  in gfx_v9_0_check_fw_write_wait()
1299 			adev->gfx.mec_fw_write_wait = true;  in gfx_v9_0_check_fw_write_wait()
1301 	case IP_VERSION(9, 4, 0):  in gfx_v9_0_check_fw_write_wait()
1302 		if ((adev->gfx.me_fw_version >= 0x0000009c) &&  in gfx_v9_0_check_fw_write_wait()
1303 		    (adev->gfx.me_feature_version >= 44) &&  in gfx_v9_0_check_fw_write_wait()
1304 		    (adev->gfx.pfp_fw_version >=  0x000000b2) &&  in gfx_v9_0_check_fw_write_wait()
1305 		    (adev->gfx.pfp_feature_version >= 44))  in gfx_v9_0_check_fw_write_wait()
1306 			adev->gfx.me_fw_write_wait = true;  in gfx_v9_0_check_fw_write_wait()
1308 		if ((adev->gfx.mec_fw_version >=  0x00000197) &&  in gfx_v9_0_check_fw_write_wait()
1309 		    (adev->gfx.mec_feature_version >= 44))  in gfx_v9_0_check_fw_write_wait()
1310 			adev->gfx.mec_fw_write_wait = true;  in gfx_v9_0_check_fw_write_wait()
1314 		if ((adev->gfx.me_fw_version >= 0x0000009c) &&  in gfx_v9_0_check_fw_write_wait()
1315 		    (adev->gfx.me_feature_version >= 42) &&  in gfx_v9_0_check_fw_write_wait()
1316 		    (adev->gfx.pfp_fw_version >=  0x000000b1) &&  in gfx_v9_0_check_fw_write_wait()
1317 		    (adev->gfx.pfp_feature_version >= 42))  in gfx_v9_0_check_fw_write_wait()
1318 			adev->gfx.me_fw_write_wait = true;  in gfx_v9_0_check_fw_write_wait()
1320 		if ((adev->gfx.mec_fw_version >=  0x00000192) &&  in gfx_v9_0_check_fw_write_wait()
1321 		    (adev->gfx.mec_feature_version >= 42))  in gfx_v9_0_check_fw_write_wait()
1322 			adev->gfx.mec_fw_write_wait = true;  in gfx_v9_0_check_fw_write_wait()
1325 		adev->gfx.me_fw_write_wait = true;  in gfx_v9_0_check_fw_write_wait()
1326 		adev->gfx.mec_fw_write_wait = true;  in gfx_v9_0_check_fw_write_wait()
1344 	/* GFXOFF is unstable on C6 parts with a VBIOS 113-RAVEN-114 */
1346 	/* Apple MacBook Pro (15-inch, 2019) Radeon Pro Vega 20 4 GB */
1359 	while (p && p->chip_device != 0) {  in gfx_v9_0_should_disable_gfxoff()
1360 		if (pdev->vendor == p->chip_vendor &&  in gfx_v9_0_should_disable_gfxoff()
1361 		    pdev->device == p->chip_device &&  in gfx_v9_0_should_disable_gfxoff()
1362 		    pdev->subsystem_vendor == p->subsys_vendor &&  in gfx_v9_0_should_disable_gfxoff()
1363 		    pdev->subsystem_device == p->subsys_device &&  in gfx_v9_0_should_disable_gfxoff()
1364 		    pdev->revision == p->revision) {  in gfx_v9_0_should_disable_gfxoff()
1374 	if (adev->pm.fw_version >= 0x41e2b)  in is_raven_kicker()
1383 	    (adev->gfx.me_fw_version >= 0x000000a5) &&  in check_if_enlarge_doorbell_range()
1384 	    (adev->gfx.me_feature_version >= 52))  in check_if_enlarge_doorbell_range()
1392 	if (gfx_v9_0_should_disable_gfxoff(adev->pdev))  in gfx_v9_0_check_if_need_gfxoff()
1393 		adev->pm.pp_feature &= ~PP_GFXOFF_MASK;  in gfx_v9_0_check_if_need_gfxoff()
1398 	case IP_VERSION(9, 4, 0):  in gfx_v9_0_check_if_need_gfxoff()
1402 		if (!((adev->apu_flags & AMD_APU_IS_RAVEN2) ||  in gfx_v9_0_check_if_need_gfxoff()
1403 		      (adev->apu_flags & AMD_APU_IS_PICASSO)) &&  in gfx_v9_0_check_if_need_gfxoff()
1405 		      adev->gfx.rlc_fw_version < 531) ||  in gfx_v9_0_check_if_need_gfxoff()
1406 		     (adev->gfx.rlc_feature_version < 1) ||  in gfx_v9_0_check_if_need_gfxoff()
1407 		     !adev->gfx.rlc.is_rlc_v2_1))  in gfx_v9_0_check_if_need_gfxoff()
1408 			adev->pm.pp_feature &= ~PP_GFXOFF_MASK;  in gfx_v9_0_check_if_need_gfxoff()
1410 		if (adev->pm.pp_feature & PP_GFXOFF_MASK)  in gfx_v9_0_check_if_need_gfxoff()
1411 			adev->pg_flags |= AMD_PG_SUPPORT_GFX_PG |  in gfx_v9_0_check_if_need_gfxoff()
1416 		if (adev->pm.pp_feature & PP_GFXOFF_MASK)  in gfx_v9_0_check_if_need_gfxoff()
1417 			adev->pg_flags |= AMD_PG_SUPPORT_GFX_PG |  in gfx_v9_0_check_if_need_gfxoff()
1431 	err = amdgpu_ucode_request(adev, &adev->gfx.pfp_fw,  in gfx_v9_0_init_cp_gfx_microcode()
1437 	err = amdgpu_ucode_request(adev, &adev->gfx.me_fw,  in gfx_v9_0_init_cp_gfx_microcode()
1443 	err = amdgpu_ucode_request(adev, &adev->gfx.ce_fw,  in gfx_v9_0_init_cp_gfx_microcode()
1451 		amdgpu_ucode_release(&adev->gfx.pfp_fw);  in gfx_v9_0_init_cp_gfx_microcode()
1452 		amdgpu_ucode_release(&adev->gfx.me_fw);  in gfx_v9_0_init_cp_gfx_microcode()
1453 		amdgpu_ucode_release(&adev->gfx.ce_fw);  in gfx_v9_0_init_cp_gfx_microcode()
1476 		(((adev->pdev->revision >= 0xC8) && (adev->pdev->revision <= 0xCF)) ||  in gfx_v9_0_init_rlc_microcode()
1477 		((adev->pdev->revision >= 0xD8) && (adev->pdev->revision <= 0xDF))))  in gfx_v9_0_init_rlc_microcode()
1478 		err = amdgpu_ucode_request(adev, &adev->gfx.rlc_fw,  in gfx_v9_0_init_rlc_microcode()
1485 		err = amdgpu_ucode_request(adev, &adev->gfx.rlc_fw,  in gfx_v9_0_init_rlc_microcode()
1488 		err = amdgpu_ucode_request(adev, &adev->gfx.rlc_fw,  in gfx_v9_0_init_rlc_microcode()
1493 	rlc_hdr = (const struct rlc_firmware_header_v2_0 *)adev->gfx.rlc_fw->data;  in gfx_v9_0_init_rlc_microcode()
1494 	version_major = le16_to_cpu(rlc_hdr->header.header_version_major);  in gfx_v9_0_init_rlc_microcode()
1495 	version_minor = le16_to_cpu(rlc_hdr->header.header_version_minor);  in gfx_v9_0_init_rlc_microcode()
1499 		amdgpu_ucode_release(&adev->gfx.rlc_fw);  in gfx_v9_0_init_rlc_microcode()
1506 	if (amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(9, 4, 2) ||  in gfx_v9_0_load_mec2_fw_bin_support()
1507 	    amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(9, 4, 1) ||  in gfx_v9_0_load_mec2_fw_bin_support()
1519 	if (amdgpu_sriov_vf(adev) && (adev->asic_type == CHIP_ALDEBARAN))  in gfx_v9_0_init_cp_compute_microcode()
1520 		err = amdgpu_ucode_request(adev, &adev->gfx.mec_fw,  in gfx_v9_0_init_cp_compute_microcode()
1523 		err = amdgpu_ucode_request(adev, &adev->gfx.mec_fw,  in gfx_v9_0_init_cp_compute_microcode()
1532 		if (amdgpu_sriov_vf(adev) && (adev->asic_type == CHIP_ALDEBARAN))  in gfx_v9_0_init_cp_compute_microcode()
1533 			err = amdgpu_ucode_request(adev, &adev->gfx.mec2_fw,  in gfx_v9_0_init_cp_compute_microcode()
1536 			err = amdgpu_ucode_request(adev, &adev->gfx.mec2_fw,  in gfx_v9_0_init_cp_compute_microcode()
1543 			amdgpu_ucode_release(&adev->gfx.mec2_fw);  in gfx_v9_0_init_cp_compute_microcode()
1546 		adev->gfx.mec2_fw_version = adev->gfx.mec_fw_version;  in gfx_v9_0_init_cp_compute_microcode()
1547 		adev->gfx.mec2_feature_version = adev->gfx.mec_feature_version;  in gfx_v9_0_init_cp_compute_microcode()
1555 		amdgpu_ucode_release(&adev->gfx.mec_fw);  in gfx_v9_0_init_cp_compute_microcode()
1568 	if (adev->gfx.num_gfx_rings) {  in gfx_v9_0_init_microcode()
1596 	for (sect = gfx9_cs_data; sect->section != NULL; ++sect) {  in gfx_v9_0_get_csb_size()
1597 		for (ext = sect->section; ext->extent != NULL; ++ext) {  in gfx_v9_0_get_csb_size()
1598 			if (sect->id == SECT_CONTEXT)  in gfx_v9_0_get_csb_size()
1599 				count += 2 + ext->reg_count;  in gfx_v9_0_get_csb_size()
1620 	if (adev->gfx.rlc.cs_data == NULL)  in gfx_v9_0_get_csb_buffer()
1632 	for (sect = adev->gfx.rlc.cs_data; sect->section != NULL; ++sect) {  in gfx_v9_0_get_csb_buffer()
1633 		for (ext = sect->section; ext->extent != NULL; ++ext) {  in gfx_v9_0_get_csb_buffer()
1634 			if (sect->id == SECT_CONTEXT) {  in gfx_v9_0_get_csb_buffer()
1636 					cpu_to_le32(PACKET3(PACKET3_SET_CONTEXT_REG, ext->reg_count));  in gfx_v9_0_get_csb_buffer()
1637 				buffer[count++] = cpu_to_le32(ext->reg_index -  in gfx_v9_0_get_csb_buffer()
1639 				for (i = 0; i < ext->reg_count; i++)  in gfx_v9_0_get_csb_buffer()
1640 					buffer[count++] = cpu_to_le32(ext->extent[i]);  in gfx_v9_0_get_csb_buffer()
1656 	struct amdgpu_cu_info *cu_info = &adev->gfx.cu_info;  in gfx_v9_0_init_always_on_cu_mask()
1662 	if (adev->flags & AMD_IS_APU)  in gfx_v9_0_init_always_on_cu_mask()
1663 		always_on_cu_num = 4;  in gfx_v9_0_init_always_on_cu_mask()
1669 	mutex_lock(&adev->grbm_idx_mutex);  in gfx_v9_0_init_always_on_cu_mask()
1670 	for (i = 0; i < adev->gfx.config.max_shader_engines; i++) {  in gfx_v9_0_init_always_on_cu_mask()
1671 		for (j = 0; j < adev->gfx.config.max_sh_per_se; j++) {  in gfx_v9_0_init_always_on_cu_mask()
1677 			for (k = 0; k < adev->gfx.config.max_cu_per_sh; k ++) {  in gfx_v9_0_init_always_on_cu_mask()
1678 				if (cu_info->bitmap[0][i][j] & mask) {  in gfx_v9_0_init_always_on_cu_mask()
1691 			cu_info->ao_cu_bitmap[i][j] = cu_bitmap;  in gfx_v9_0_init_always_on_cu_mask()
1695 	mutex_unlock(&adev->grbm_idx_mutex);  in gfx_v9_0_init_always_on_cu_mask()
1702 	/* set mmRLC_LB_THR_CONFIG_1/2/3/4 */  in gfx_v9_0_init_lbpw()
1714 	mutex_lock(&adev->grbm_idx_mutex);  in gfx_v9_0_init_lbpw()
1725 	/* set mmRLC_GPM_GENERAL_7[31-16] = 0x00C0 */  in gfx_v9_0_init_lbpw()
1732 	 * RLC_LB_ALWAYS_ACTIVE_CU_MASK = 0xF (4 CUs AON for Raven),  in gfx_v9_0_init_lbpw()
1742 	mutex_unlock(&adev->grbm_idx_mutex);  in gfx_v9_0_init_lbpw()
1751 	/* set mmRLC_LB_THR_CONFIG_1/2/3/4 */  in gfx_v9_4_init_lbpw()
1763 	mutex_lock(&adev->grbm_idx_mutex);  in gfx_v9_4_init_lbpw()
1774 	/* set mmRLC_GPM_GENERAL_7[31-16] = 0x00C0 */  in gfx_v9_4_init_lbpw()
1791 	mutex_unlock(&adev->grbm_idx_mutex);  in gfx_v9_4_init_lbpw()
1806 		return 4;  in gfx_v9_0_cp_jump_table_num()
1813 	reg_access_ctrl = &adev->gfx.rlc.reg_access_ctrl[0];  in gfx_v9_0_init_rlcg_reg_access_ctrl()
1814 	reg_access_ctrl->scratch_reg0 = SOC15_REG_OFFSET(GC, 0, mmSCRATCH_REG0);  in gfx_v9_0_init_rlcg_reg_access_ctrl()
1815 	reg_access_ctrl->scratch_reg1 = SOC15_REG_OFFSET(GC, 0, mmSCRATCH_REG1);  in gfx_v9_0_init_rlcg_reg_access_ctrl()
1816 	reg_access_ctrl->scratch_reg2 = SOC15_REG_OFFSET(GC, 0, mmSCRATCH_REG2);  in gfx_v9_0_init_rlcg_reg_access_ctrl()
1817 	reg_access_ctrl->scratch_reg3 = SOC15_REG_OFFSET(GC, 0, mmSCRATCH_REG3);  in gfx_v9_0_init_rlcg_reg_access_ctrl()
1818 	reg_access_ctrl->grbm_cntl = SOC15_REG_OFFSET(GC, 0, mmGRBM_GFX_CNTL);  in gfx_v9_0_init_rlcg_reg_access_ctrl()
1819 	reg_access_ctrl->grbm_idx = SOC15_REG_OFFSET(GC, 0, mmGRBM_GFX_INDEX);  in gfx_v9_0_init_rlcg_reg_access_ctrl()
1820 	reg_access_ctrl->spare_int = SOC15_REG_OFFSET(GC, 0, mmRLC_SPARE_INT);  in gfx_v9_0_init_rlcg_reg_access_ctrl()
1821 	adev->gfx.rlc.rlcg_reg_access_supported = true;  in gfx_v9_0_init_rlcg_reg_access_ctrl()
1829 	adev->gfx.rlc.cs_data = gfx9_cs_data;  in gfx_v9_0_rlc_init()
1831 	cs_data = adev->gfx.rlc.cs_data;  in gfx_v9_0_rlc_init()
1840 	if (adev->flags & AMD_IS_APU) {  in gfx_v9_0_rlc_init()
1842 		adev->gfx.rlc.cp_table_size = ALIGN(96 * 5 * 4, 2048) + (64 * 1024); /* JT + GDS */  in gfx_v9_0_rlc_init()
1853 	amdgpu_bo_free_kernel(&adev->gfx.mec.hpd_eop_obj, NULL, NULL);  in gfx_v9_0_mec_fini()
1854 	amdgpu_bo_free_kernel(&adev->gfx.mec.mec_fw_obj, NULL, NULL);  in gfx_v9_0_mec_fini()
1868 	bitmap_zero(adev->gfx.mec_bitmap[0].queue_bitmap, AMDGPU_MAX_COMPUTE_QUEUES);  in gfx_v9_0_mec_init()
1872 	mec_hpd_size = adev->gfx.num_compute_rings * GFX9_MEC_HPD_SIZE;  in gfx_v9_0_mec_init()
1877 					      &adev->gfx.mec.hpd_eop_obj,  in gfx_v9_0_mec_init()
1878 					      &adev->gfx.mec.hpd_eop_gpu_addr,  in gfx_v9_0_mec_init()
1881 			dev_warn(adev->dev, "(%d) create HDP EOP bo failed\n", r);  in gfx_v9_0_mec_init()
1888 		amdgpu_bo_kunmap(adev->gfx.mec.hpd_eop_obj);  in gfx_v9_0_mec_init()
1889 		amdgpu_bo_unreserve(adev->gfx.mec.hpd_eop_obj);  in gfx_v9_0_mec_init()
1892 	mec_hdr = (const struct gfx_firmware_header_v1_0 *)adev->gfx.mec_fw->data;  in gfx_v9_0_mec_init()
1895 		(adev->gfx.mec_fw->data +  in gfx_v9_0_mec_init()
1896 		 le32_to_cpu(mec_hdr->header.ucode_array_offset_bytes));  in gfx_v9_0_mec_init()
1897 	fw_size = le32_to_cpu(mec_hdr->header.ucode_size_bytes);  in gfx_v9_0_mec_init()
1899 	r = amdgpu_bo_create_reserved(adev, mec_hdr->header.ucode_size_bytes,  in gfx_v9_0_mec_init()
1901 				      &adev->gfx.mec.mec_fw_obj,  in gfx_v9_0_mec_init()
1902 				      &adev->gfx.mec.mec_fw_gpu_addr,  in gfx_v9_0_mec_init()
1905 		dev_warn(adev->dev, "(%d) create mec firmware bo failed\n", r);  in gfx_v9_0_mec_init()
1912 	amdgpu_bo_kunmap(adev->gfx.mec.mec_fw_obj);  in gfx_v9_0_mec_init()
1913 	amdgpu_bo_unreserve(adev->gfx.mec.mec_fw_obj);  in gfx_v9_0_mec_init()
1939 	while (num--)  in wave_read_regs()
2017 		adev->gfx.config.max_hw_contexts = 8;  in gfx_v9_0_gpu_early_init()
2018 		adev->gfx.config.sc_prim_fifo_size_frontend = 0x20;  in gfx_v9_0_gpu_early_init()
2019 		adev->gfx.config.sc_prim_fifo_size_backend = 0x100;  in gfx_v9_0_gpu_early_init()
2020 		adev->gfx.config.sc_hiz_tile_fifo_size = 0x30;  in gfx_v9_0_gpu_early_init()
2021 		adev->gfx.config.sc_earlyz_tile_fifo_size = 0x4C0;  in gfx_v9_0_gpu_early_init()
2025 		adev->gfx.config.max_hw_contexts = 8;  in gfx_v9_0_gpu_early_init()
2026 		adev->gfx.config.sc_prim_fifo_size_frontend = 0x20;  in gfx_v9_0_gpu_early_init()
2027 		adev->gfx.config.sc_prim_fifo_size_backend = 0x100;  in gfx_v9_0_gpu_early_init()
2028 		adev->gfx.config.sc_hiz_tile_fifo_size = 0x30;  in gfx_v9_0_gpu_early_init()
2029 		adev->gfx.config.sc_earlyz_tile_fifo_size = 0x4C0;  in gfx_v9_0_gpu_early_init()
2033 	case IP_VERSION(9, 4, 0):  in gfx_v9_0_gpu_early_init()
2034 		adev->gfx.ras = &gfx_v9_0_ras;  in gfx_v9_0_gpu_early_init()
2035 		adev->gfx.config.max_hw_contexts = 8;  in gfx_v9_0_gpu_early_init()
2036 		adev->gfx.config.sc_prim_fifo_size_frontend = 0x20;  in gfx_v9_0_gpu_early_init()
2037 		adev->gfx.config.sc_prim_fifo_size_backend = 0x100;  in gfx_v9_0_gpu_early_init()
2038 		adev->gfx.config.sc_hiz_tile_fifo_size = 0x30;  in gfx_v9_0_gpu_early_init()
2039 		adev->gfx.config.sc_earlyz_tile_fifo_size = 0x4C0;  in gfx_v9_0_gpu_early_init()
2050 		adev->gfx.config.max_hw_contexts = 8;  in gfx_v9_0_gpu_early_init()
2051 		adev->gfx.config.sc_prim_fifo_size_frontend = 0x20;  in gfx_v9_0_gpu_early_init()
2052 		adev->gfx.config.sc_prim_fifo_size_backend = 0x100;  in gfx_v9_0_gpu_early_init()
2053 		adev->gfx.config.sc_hiz_tile_fifo_size = 0x30;  in gfx_v9_0_gpu_early_init()
2054 		adev->gfx.config.sc_earlyz_tile_fifo_size = 0x4C0;  in gfx_v9_0_gpu_early_init()
2055 		if (adev->apu_flags & AMD_APU_IS_RAVEN2)  in gfx_v9_0_gpu_early_init()
2060 	case IP_VERSION(9, 4, 1):  in gfx_v9_0_gpu_early_init()
2061 		adev->gfx.ras = &gfx_v9_4_ras;  in gfx_v9_0_gpu_early_init()
2062 		adev->gfx.config.max_hw_contexts = 8;  in gfx_v9_0_gpu_early_init()
2063 		adev->gfx.config.sc_prim_fifo_size_frontend = 0x20;  in gfx_v9_0_gpu_early_init()
2064 		adev->gfx.config.sc_prim_fifo_size_backend = 0x100;  in gfx_v9_0_gpu_early_init()
2065 		adev->gfx.config.sc_hiz_tile_fifo_size = 0x30;  in gfx_v9_0_gpu_early_init()
2066 		adev->gfx.config.sc_earlyz_tile_fifo_size = 0x4C0;  in gfx_v9_0_gpu_early_init()
2072 		adev->gfx.config.max_hw_contexts = 8;  in gfx_v9_0_gpu_early_init()
2073 		adev->gfx.config.sc_prim_fifo_size_frontend = 0x20;  in gfx_v9_0_gpu_early_init()
2074 		adev->gfx.config.sc_prim_fifo_size_backend = 0x100;  in gfx_v9_0_gpu_early_init()
2075 		adev->gfx.config.sc_hiz_tile_fifo_size = 0x80;  in gfx_v9_0_gpu_early_init()
2076 		adev->gfx.config.sc_earlyz_tile_fifo_size = 0x4C0;  in gfx_v9_0_gpu_early_init()
2081 	case IP_VERSION(9, 4, 2):  in gfx_v9_0_gpu_early_init()
2082 		adev->gfx.ras = &gfx_v9_4_2_ras;  in gfx_v9_0_gpu_early_init()
2083 		adev->gfx.config.max_hw_contexts = 8;  in gfx_v9_0_gpu_early_init()
2084 		adev->gfx.config.sc_prim_fifo_size_frontend = 0x20;  in gfx_v9_0_gpu_early_init()
2085 		adev->gfx.config.sc_prim_fifo_size_backend = 0x100;  in gfx_v9_0_gpu_early_init()
2086 		adev->gfx.config.sc_hiz_tile_fifo_size = 0x30;  in gfx_v9_0_gpu_early_init()
2087 		adev->gfx.config.sc_earlyz_tile_fifo_size = 0x4C0;  in gfx_v9_0_gpu_early_init()
2101 	adev->gfx.config.gb_addr_config = gb_addr_config;  in gfx_v9_0_gpu_early_init()
2103 	adev->gfx.config.gb_addr_config_fields.num_pipes = 1 <<  in gfx_v9_0_gpu_early_init()
2105 					adev->gfx.config.gb_addr_config,  in gfx_v9_0_gpu_early_init()
2109 	adev->gfx.config.max_tile_pipes =  in gfx_v9_0_gpu_early_init()
2110 		adev->gfx.config.gb_addr_config_fields.num_pipes;  in gfx_v9_0_gpu_early_init()
2112 	adev->gfx.config.gb_addr_config_fields.num_banks = 1 <<  in gfx_v9_0_gpu_early_init()
2114 					adev->gfx.config.gb_addr_config,  in gfx_v9_0_gpu_early_init()
2117 	adev->gfx.config.gb_addr_config_fields.max_compress_frags = 1 <<  in gfx_v9_0_gpu_early_init()
2119 					adev->gfx.config.gb_addr_config,  in gfx_v9_0_gpu_early_init()
2122 	adev->gfx.config.gb_addr_config_fields.num_rb_per_se = 1 <<  in gfx_v9_0_gpu_early_init()
2124 					adev->gfx.config.gb_addr_config,  in gfx_v9_0_gpu_early_init()
2127 	adev->gfx.config.gb_addr_config_fields.num_se = 1 <<  in gfx_v9_0_gpu_early_init()
2129 					adev->gfx.config.gb_addr_config,  in gfx_v9_0_gpu_early_init()
2132 	adev->gfx.config.gb_addr_config_fields.pipe_interleave_size = 1 << (8 +  in gfx_v9_0_gpu_early_init()
2134 					adev->gfx.config.gb_addr_config,  in gfx_v9_0_gpu_early_init()
2145 	struct amdgpu_ring *ring = &adev->gfx.compute_ring[ring_id];  in gfx_v9_0_compute_ring_init()  local
2148 	ring = &adev->gfx.compute_ring[ring_id];  in gfx_v9_0_compute_ring_init()
2151 	ring->me = mec + 1;  in gfx_v9_0_compute_ring_init()
2152 	ring->pipe = pipe;  in gfx_v9_0_compute_ring_init()
2153 	ring->queue = queue;  in gfx_v9_0_compute_ring_init()
2155 	ring->ring_obj = NULL;  in gfx_v9_0_compute_ring_init()
2156 	ring->use_doorbell = true;  in gfx_v9_0_compute_ring_init()
2157 	ring->doorbell_index = (adev->doorbell_index.mec_ring0 + ring_id) << 1;  in gfx_v9_0_compute_ring_init()
2158 	ring->eop_gpu_addr = adev->gfx.mec.hpd_eop_gpu_addr  in gfx_v9_0_compute_ring_init()
2160 	ring->vm_hub = AMDGPU_GFXHUB(0);  in gfx_v9_0_compute_ring_init()
2161 	sprintf(ring->name, "comp_%d.%d.%d", ring->me, ring->pipe, ring->queue);  in gfx_v9_0_compute_ring_init()
2164 		+ ((ring->me - 1) * adev->gfx.mec.num_pipe_per_mec)  in gfx_v9_0_compute_ring_init()
2165 		+ ring->pipe;  in gfx_v9_0_compute_ring_init()
2166 	hw_prio = amdgpu_gfx_is_high_priority_compute_queue(adev, ring) ?  in gfx_v9_0_compute_ring_init()
2168 	/* type-2 packets are deprecated on MEC, use type-3 instead */  in gfx_v9_0_compute_ring_init()
2169 	return amdgpu_ring_init(adev, ring, 1024, &adev->gfx.eop_irq, irq_type,  in gfx_v9_0_compute_ring_init()
2182 		adev->gfx.ip_dump_core = NULL;  in gfx_v9_0_alloc_ip_dump()
2184 		adev->gfx.ip_dump_core = ptr;  in gfx_v9_0_alloc_ip_dump()
2189 	inst = adev->gfx.mec.num_mec * adev->gfx.mec.num_pipe_per_mec *  in gfx_v9_0_alloc_ip_dump()
2190 		adev->gfx.mec.num_queue_per_pipe;  in gfx_v9_0_alloc_ip_dump()
2195 		adev->gfx.ip_dump_compute_queues = NULL;  in gfx_v9_0_alloc_ip_dump()
2197 		adev->gfx.ip_dump_compute_queues = ptr;  in gfx_v9_0_alloc_ip_dump()
2205 	struct amdgpu_ring *ring;  in gfx_v9_0_sw_init()  local
2212 	case IP_VERSION(9, 4, 0):  in gfx_v9_0_sw_init()
2215 	case IP_VERSION(9, 4, 1):  in gfx_v9_0_sw_init()
2217 	case IP_VERSION(9, 4, 2):  in gfx_v9_0_sw_init()
2218 		adev->gfx.mec.num_mec = 2;  in gfx_v9_0_sw_init()
2221 		adev->gfx.mec.num_mec = 1;  in gfx_v9_0_sw_init()
2227 		adev->gfx.enable_cleaner_shader = false;  in gfx_v9_0_sw_init()
2231 	adev->gfx.mec.num_pipe_per_mec = 4;  in gfx_v9_0_sw_init()
2232 	adev->gfx.mec.num_queue_per_pipe = 8;  in gfx_v9_0_sw_init()
2235 …r = amdgpu_irq_add_id(adev, SOC15_IH_CLIENTID_GRBM_CP, GFX_9_0__SRCID__CP_EOP_INTERRUPT, &adev->gf…  in gfx_v9_0_sw_init()
2242 			      &adev->gfx.bad_op_irq);  in gfx_v9_0_sw_init()
2248 			      &adev->gfx.priv_reg_irq);  in gfx_v9_0_sw_init()
2254 			      &adev->gfx.priv_inst_irq);  in gfx_v9_0_sw_init()
2260 			      &adev->gfx.cp_ecc_error_irq);  in gfx_v9_0_sw_init()
2266 			      &adev->gfx.cp_ecc_error_irq);  in gfx_v9_0_sw_init()
2270 	adev->gfx.gfx_current_status = AMDGPU_GFX_NORMAL_MODE;  in gfx_v9_0_sw_init()
2272 	if (adev->gfx.rlc.funcs) {  in gfx_v9_0_sw_init()
2273 		if (adev->gfx.rlc.funcs->init) {  in gfx_v9_0_sw_init()
2274 			r = adev->gfx.rlc.funcs->init(adev);  in gfx_v9_0_sw_init()
2276 				dev_err(adev->dev, "Failed to init rlc BOs!\n");  in gfx_v9_0_sw_init()
2288 	/* set up the gfx ring */  in gfx_v9_0_sw_init()
2289 	for (i = 0; i < adev->gfx.num_gfx_rings; i++) {  in gfx_v9_0_sw_init()
2290 		ring = &adev->gfx.gfx_ring[i];  in gfx_v9_0_sw_init()
2291 		ring->ring_obj = NULL;  in gfx_v9_0_sw_init()
2293 			sprintf(ring->name, "gfx");  in gfx_v9_0_sw_init()
2295 			sprintf(ring->name, "gfx_%d", i);  in gfx_v9_0_sw_init()
2296 		ring->use_doorbell = true;  in gfx_v9_0_sw_init()
2297 		ring->doorbell_index = adev->doorbell_index.gfx_ring0 << 1;  in gfx_v9_0_sw_init()
2299 		/* disable scheduler on the real ring */  in gfx_v9_0_sw_init()
2300 		ring->no_scheduler = adev->gfx.mcbp;  in gfx_v9_0_sw_init()
2301 		ring->vm_hub = AMDGPU_GFXHUB(0);  in gfx_v9_0_sw_init()
2302 		r = amdgpu_ring_init(adev, ring, 1024, &adev->gfx.eop_irq,  in gfx_v9_0_sw_init()
2310 	if (adev->gfx.mcbp && adev->gfx.num_gfx_rings) {  in gfx_v9_0_sw_init()
2312 			ring = &adev->gfx.sw_gfx_ring[i];  in gfx_v9_0_sw_init()
2313 			ring->ring_obj = NULL;  in gfx_v9_0_sw_init()
2314 			sprintf(ring->name, amdgpu_sw_ring_name(i));  in gfx_v9_0_sw_init()
2315 			ring->use_doorbell = true;  in gfx_v9_0_sw_init()
2316 			ring->doorbell_index = adev->doorbell_index.gfx_ring0 << 1;  in gfx_v9_0_sw_init()
2317 			ring->is_sw_ring = true;  in gfx_v9_0_sw_init()
2319 			ring->vm_hub = AMDGPU_GFXHUB(0);  in gfx_v9_0_sw_init()
2320 			r = amdgpu_ring_init(adev, ring, 1024, &adev->gfx.eop_irq,  in gfx_v9_0_sw_init()
2325 			ring->wptr = 0;  in gfx_v9_0_sw_init()
2329 		r = amdgpu_ring_mux_init(&adev->gfx.muxer, &adev->gfx.gfx_ring[0],  in gfx_v9_0_sw_init()
2336 			r = amdgpu_ring_mux_add_sw_ring(&adev->gfx.muxer,  in gfx_v9_0_sw_init()
2337 							&adev->gfx.sw_gfx_ring[i]);  in gfx_v9_0_sw_init()
2345 	/* set up the compute queues - allocate horizontally across pipes */  in gfx_v9_0_sw_init()
2347 	for (i = 0; i < adev->gfx.mec.num_mec; ++i) {  in gfx_v9_0_sw_init()
2348 		for (j = 0; j < adev->gfx.mec.num_queue_per_pipe; j++) {  in gfx_v9_0_sw_init()
2349 			for (k = 0; k < adev->gfx.mec.num_pipe_per_mec; k++) {  in gfx_v9_0_sw_init()
2380 	adev->gfx.ce_ram_size = 0x8000;  in gfx_v9_0_sw_init()
2387 		dev_err(adev->dev, "Failed to initialize gfx ras block!\n");  in gfx_v9_0_sw_init()
2388 		return -EINVAL;  in gfx_v9_0_sw_init()
2406 	if (adev->gfx.mcbp && adev->gfx.num_gfx_rings) {  in gfx_v9_0_sw_fini()
2408 			amdgpu_ring_fini(&adev->gfx.sw_gfx_ring[i]);  in gfx_v9_0_sw_fini()
2409 		amdgpu_ring_mux_fini(&adev->gfx.muxer);  in gfx_v9_0_sw_fini()
2412 	for (i = 0; i < adev->gfx.num_gfx_rings; i++)  in gfx_v9_0_sw_fini()
2413 		amdgpu_ring_fini(&adev->gfx.gfx_ring[i]);  in gfx_v9_0_sw_fini()
2414 	for (i = 0; i < adev->gfx.num_compute_rings; i++)  in gfx_v9_0_sw_fini()
2415 		amdgpu_ring_fini(&adev->gfx.compute_ring[i]);  in gfx_v9_0_sw_fini()
2418 	amdgpu_gfx_kiq_free_ring(&adev->gfx.kiq[0].ring);  in gfx_v9_0_sw_fini()
2422 	amdgpu_bo_free_kernel(&adev->gfx.rlc.clear_state_obj,  in gfx_v9_0_sw_fini()
2423 				&adev->gfx.rlc.clear_state_gpu_addr,  in gfx_v9_0_sw_fini()
2424 				(void **)&adev->gfx.rlc.cs_ptr);  in gfx_v9_0_sw_fini()
2425 	if (adev->flags & AMD_IS_APU) {  in gfx_v9_0_sw_fini()
2426 		amdgpu_bo_free_kernel(&adev->gfx.rlc.cp_table_obj,  in gfx_v9_0_sw_fini()
2427 				&adev->gfx.rlc.cp_table_gpu_addr,  in gfx_v9_0_sw_fini()
2428 				(void **)&adev->gfx.rlc.cp_table_ptr);  in gfx_v9_0_sw_fini()
2434 	kfree(adev->gfx.ip_dump_core);  in gfx_v9_0_sw_fini()
2435 	kfree(adev->gfx.ip_dump_compute_queues);  in gfx_v9_0_sw_fini()
2479 	mask = amdgpu_gfx_create_bitmask(adev->gfx.config.max_backends_per_se /  in gfx_v9_0_get_rb_active_bitmap()
2480 					 adev->gfx.config.max_sh_per_se);  in gfx_v9_0_get_rb_active_bitmap()
2490 	u32 rb_bitmap_width_per_sh = adev->gfx.config.max_backends_per_se /  in gfx_v9_0_setup_rb()
2491 					adev->gfx.config.max_sh_per_se;  in gfx_v9_0_setup_rb()
2493 	mutex_lock(&adev->grbm_idx_mutex);  in gfx_v9_0_setup_rb()
2494 	for (i = 0; i < adev->gfx.config.max_shader_engines; i++) {  in gfx_v9_0_setup_rb()
2495 		for (j = 0; j < adev->gfx.config.max_sh_per_se; j++) {  in gfx_v9_0_setup_rb()
2498 			active_rbs |= data << ((i * adev->gfx.config.max_sh_per_se + j) *  in gfx_v9_0_setup_rb()
2503 	mutex_unlock(&adev->grbm_idx_mutex);  in gfx_v9_0_setup_rb()
2505 	adev->gfx.config.backend_enable_mask = active_rbs;  in gfx_v9_0_setup_rb()
2506 	adev->gfx.config.num_rbs = hweight32(active_rbs);  in gfx_v9_0_setup_rb()
2541 	 * LDS:         0x60000000'00000000 - 0x60000001'00000000 (4GB)  in gfx_v9_0_init_compute_vmid()
2542 	 * Scratch:     0x60000001'00000000 - 0x60000002'00000000 (4GB)  in gfx_v9_0_init_compute_vmid()
2543 	 * GPUVM:       0x60010000'00000000 - 0x60020000'00000000 (1TB)  in gfx_v9_0_init_compute_vmid()
2551 	mutex_lock(&adev->srbm_mutex);  in gfx_v9_0_init_compute_vmid()
2552 	for (i = adev->vm_manager.first_kfd_vmid; i < AMDGPU_NUM_VMID; i++) {  in gfx_v9_0_init_compute_vmid()
2559 	mutex_unlock(&adev->srbm_mutex);  in gfx_v9_0_init_compute_vmid()
2563 	for (i = adev->vm_manager.first_kfd_vmid; i < AMDGPU_NUM_VMID; i++) {  in gfx_v9_0_init_compute_vmid()
2576 	 * Initialize all compute and user-gfx VMIDs to have no GDS, GWS, or OA  in gfx_v9_0_init_gds_vmid()
2594 	case IP_VERSION(9, 4, 1):  in gfx_v9_0_init_sq_config()
2597 				!READ_ONCE(adev->barrier_has_auto_waitcnt));  in gfx_v9_0_init_sq_config()
2614 	if (adev->gfx.num_gfx_rings)  in gfx_v9_0_constants_init()
2616 	gfx_v9_0_get_cu_info(adev, &adev->gfx.cu_info);  in gfx_v9_0_constants_init()
2617 	adev->gfx.config.db_debug2 = RREG32_SOC15(GC, 0, mmDB_DEBUG2);  in gfx_v9_0_constants_init()
2621 	mutex_lock(&adev->srbm_mutex);  in gfx_v9_0_constants_init()
2622 	for (i = 0; i < adev->vm_manager.id_mgr[AMDGPU_GFXHUB(0)].num_ids; i++) {  in gfx_v9_0_constants_init()
2629 					    !!adev->gmc.noretry);  in gfx_v9_0_constants_init()
2636 					    !!adev->gmc.noretry);  in gfx_v9_0_constants_init()
2639 				(adev->gmc.private_aperture_start >> 48));  in gfx_v9_0_constants_init()
2641 				(adev->gmc.shared_aperture_start >> 48));  in gfx_v9_0_constants_init()
2647 	mutex_unlock(&adev->srbm_mutex);  in gfx_v9_0_constants_init()
2659 	mutex_lock(&adev->grbm_idx_mutex);  in gfx_v9_0_wait_for_rlc_serdes()
2660 	for (i = 0; i < adev->gfx.config.max_shader_engines; i++) {  in gfx_v9_0_wait_for_rlc_serdes()
2661 		for (j = 0; j < adev->gfx.config.max_sh_per_se; j++) {  in gfx_v9_0_wait_for_rlc_serdes()
2663 			for (k = 0; k < adev->usec_timeout; k++) {  in gfx_v9_0_wait_for_rlc_serdes()
2668 			if (k == adev->usec_timeout) {  in gfx_v9_0_wait_for_rlc_serdes()
2671 				mutex_unlock(&adev->grbm_idx_mutex);  in gfx_v9_0_wait_for_rlc_serdes()
2679 	mutex_unlock(&adev->grbm_idx_mutex);  in gfx_v9_0_wait_for_rlc_serdes()
2685 	for (k = 0; k < adev->usec_timeout; k++) {  in gfx_v9_0_wait_for_rlc_serdes()
2704 	if (adev->gfx.num_gfx_rings)  in gfx_v9_0_enable_gui_idle_interrupt()
2712 	adev->gfx.rlc.funcs->get_csb_buffer(adev, adev->gfx.rlc.cs_ptr);  in gfx_v9_0_init_csb()
2715 			adev->gfx.rlc.clear_state_gpu_addr >> 32);  in gfx_v9_0_init_csb()
2717 			adev->gfx.rlc.clear_state_gpu_addr & 0xfffffffc);  in gfx_v9_0_init_csb()
2719 			adev->gfx.rlc.clear_state_size);  in gfx_v9_0_init_csb()
2772 		kmemdup(adev->gfx.rlc.register_list_format,  in gfx_v9_1_init_rlc_save_restore_list()
2773 			adev->gfx.rlc.reg_list_format_size_bytes, GFP_KERNEL);  in gfx_v9_1_init_rlc_save_restore_list()
2775 		return -ENOMEM;  in gfx_v9_1_init_rlc_save_restore_list()
2780 				    adev->gfx.rlc.reg_list_format_direct_reg_list_length,  in gfx_v9_1_init_rlc_save_restore_list()
2781 				    adev->gfx.rlc.reg_list_format_size_bytes >> 2,  in gfx_v9_1_init_rlc_save_restore_list()
2796 	for (i = 0; i < adev->gfx.rlc.reg_list_size_bytes >> 2; i++)  in gfx_v9_1_init_rlc_save_restore_list()
2798 			adev->gfx.rlc.register_restore[i]);  in gfx_v9_1_init_rlc_save_restore_list()
2802 		adev->gfx.rlc.reg_list_format_start);  in gfx_v9_1_init_rlc_save_restore_list()
2805 	for (i = 0; i < adev->gfx.rlc.reg_list_format_direct_reg_list_length; i++)  in gfx_v9_1_init_rlc_save_restore_list()
2810 	while (i < (adev->gfx.rlc.reg_list_format_size_bytes >> 2)) {  in gfx_v9_1_init_rlc_save_restore_list()
2832 	list_size = adev->gfx.rlc.reg_list_size_bytes >> 2;  in gfx_v9_1_init_rlc_save_restore_list()
2835 		adev->gfx.rlc.reg_restore_list_size);  in gfx_v9_1_init_rlc_save_restore_list()
2840 		adev->gfx.rlc.starting_offsets_start);  in gfx_v9_1_init_rlc_save_restore_list()
2897 	if (adev->pg_flags & (AMD_PG_SUPPORT_GFX_PG |  in gfx_v9_0_init_gfx_power_gating()
3041 	if (adev->gfx.rlc.is_rlc_v2_1) {  in gfx_v9_0_init_pg()
3044 		    (adev->apu_flags & AMD_APU_IS_RAVEN2))  in gfx_v9_0_init_pg()
3049 	if (adev->pg_flags & (AMD_PG_SUPPORT_GFX_PG |  in gfx_v9_0_init_pg()
3056 			     adev->gfx.rlc.cp_table_gpu_addr >> 8);  in gfx_v9_0_init_pg()
3086 	if (!(adev->flags & AMD_IS_APU)) {  in gfx_v9_0_rlc_start()
3096 				rlc_ucode_ver, adev->gfx.rlc_fw_version);  in gfx_v9_0_rlc_start()
3114 	if (!adev->gfx.rlc_fw)  in gfx_v9_0_rlc_load_microcode()
3115 		return -EINVAL;  in gfx_v9_0_rlc_load_microcode()
3117 	hdr = (const struct rlc_firmware_header_v2_0 *)adev->gfx.rlc_fw->data;  in gfx_v9_0_rlc_load_microcode()
3118 	amdgpu_ucode_print_rlc_hdr(&hdr->header);  in gfx_v9_0_rlc_load_microcode()
3120 	fw_data = (const __le32 *)(adev->gfx.rlc_fw->data +  in gfx_v9_0_rlc_load_microcode()
3121 			   le32_to_cpu(hdr->header.ucode_array_offset_bytes));  in gfx_v9_0_rlc_load_microcode()
3122 	fw_size = le32_to_cpu(hdr->header.ucode_size_bytes) / 4;  in gfx_v9_0_rlc_load_microcode()
3128 	WREG32_SOC15(GC, 0, mmRLC_GPM_UCODE_ADDR, adev->gfx.rlc_fw_version);  in gfx_v9_0_rlc_load_microcode()
3142 	adev->gfx.rlc.funcs->stop(adev);  in gfx_v9_0_rlc_resume()
3149 	if (adev->firmware.load_type != AMDGPU_FW_LOAD_PSP) {  in gfx_v9_0_rlc_resume()
3165 	case IP_VERSION(9, 4, 0):  in gfx_v9_0_rlc_resume()
3178 	adev->gfx.rlc.funcs->start(adev);  in gfx_v9_0_rlc_resume()
3202 	if (!adev->gfx.me_fw || !adev->gfx.pfp_fw || !adev->gfx.ce_fw)  in gfx_v9_0_cp_gfx_load_microcode()
3203 		return -EINVAL;  in gfx_v9_0_cp_gfx_load_microcode()
3206 		adev->gfx.pfp_fw->data;  in gfx_v9_0_cp_gfx_load_microcode()
3208 		adev->gfx.ce_fw->data;  in gfx_v9_0_cp_gfx_load_microcode()
3210 		adev->gfx.me_fw->data;  in gfx_v9_0_cp_gfx_load_microcode()
3212 	amdgpu_ucode_print_gfx_hdr(&pfp_hdr->header);  in gfx_v9_0_cp_gfx_load_microcode()
3213 	amdgpu_ucode_print_gfx_hdr(&ce_hdr->header);  in gfx_v9_0_cp_gfx_load_microcode()
3214 	amdgpu_ucode_print_gfx_hdr(&me_hdr->header);  in gfx_v9_0_cp_gfx_load_microcode()
3220 		(adev->gfx.pfp_fw->data +  in gfx_v9_0_cp_gfx_load_microcode()
3221 		 le32_to_cpu(pfp_hdr->header.ucode_array_offset_bytes));  in gfx_v9_0_cp_gfx_load_microcode()
3222 	fw_size = le32_to_cpu(pfp_hdr->header.ucode_size_bytes) / 4;  in gfx_v9_0_cp_gfx_load_microcode()
3226 	WREG32_SOC15(GC, 0, mmCP_PFP_UCODE_ADDR, adev->gfx.pfp_fw_version);  in gfx_v9_0_cp_gfx_load_microcode()
3230 		(adev->gfx.ce_fw->data +  in gfx_v9_0_cp_gfx_load_microcode()
3231 		 le32_to_cpu(ce_hdr->header.ucode_array_offset_bytes));  in gfx_v9_0_cp_gfx_load_microcode()
3232 	fw_size = le32_to_cpu(ce_hdr->header.ucode_size_bytes) / 4;  in gfx_v9_0_cp_gfx_load_microcode()
3236 	WREG32_SOC15(GC, 0, mmCP_CE_UCODE_ADDR, adev->gfx.ce_fw_version);  in gfx_v9_0_cp_gfx_load_microcode()
3240 		(adev->gfx.me_fw->data +  in gfx_v9_0_cp_gfx_load_microcode()
3241 		 le32_to_cpu(me_hdr->header.ucode_array_offset_bytes));  in gfx_v9_0_cp_gfx_load_microcode()
3242 	fw_size = le32_to_cpu(me_hdr->header.ucode_size_bytes) / 4;  in gfx_v9_0_cp_gfx_load_microcode()
3246 	WREG32_SOC15(GC, 0, mmCP_ME_RAM_WADDR, adev->gfx.me_fw_version);  in gfx_v9_0_cp_gfx_load_microcode()
3253 	struct amdgpu_ring *ring = &adev->gfx.gfx_ring[0];  in gfx_v9_0_cp_gfx_start()  local
3259 	WREG32_SOC15(GC, 0, mmCP_MAX_CONTEXT, adev->gfx.config.max_hw_contexts - 1);  in gfx_v9_0_cp_gfx_start()
3267 	if (adev->flags & AMD_IS_APU &&  in gfx_v9_0_cp_gfx_start()
3268 			adev->in_s3 && !adev->suspend_complete) {  in gfx_v9_0_cp_gfx_start()
3272 	r = amdgpu_ring_alloc(ring, gfx_v9_0_get_csb_size(adev) + 4 + 3);  in gfx_v9_0_cp_gfx_start()
3274 		DRM_ERROR("amdgpu: cp failed to lock ring (%d).\n", r);  in gfx_v9_0_cp_gfx_start()
3278 	amdgpu_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));  in gfx_v9_0_cp_gfx_start()
3279 	amdgpu_ring_write(ring, PACKET3_PREAMBLE_BEGIN_CLEAR_STATE);  in gfx_v9_0_cp_gfx_start()
3281 	amdgpu_ring_write(ring, PACKET3(PACKET3_CONTEXT_CONTROL, 1));  in gfx_v9_0_cp_gfx_start()
3282 	amdgpu_ring_write(ring, 0x80000000);  in gfx_v9_0_cp_gfx_start()
3283 	amdgpu_ring_write(ring, 0x80000000);  in gfx_v9_0_cp_gfx_start()
3285 	for (sect = gfx9_cs_data; sect->section != NULL; ++sect) {  in gfx_v9_0_cp_gfx_start()
3286 		for (ext = sect->section; ext->extent != NULL; ++ext) {  in gfx_v9_0_cp_gfx_start()
3287 			if (sect->id == SECT_CONTEXT) {  in gfx_v9_0_cp_gfx_start()
3288 				amdgpu_ring_write(ring,  in gfx_v9_0_cp_gfx_start()
3290 					       ext->reg_count));  in gfx_v9_0_cp_gfx_start()
3291 				amdgpu_ring_write(ring,  in gfx_v9_0_cp_gfx_start()
3292 				       ext->reg_index - PACKET3_SET_CONTEXT_REG_START);  in gfx_v9_0_cp_gfx_start()
3293 				for (i = 0; i < ext->reg_count; i++)  in gfx_v9_0_cp_gfx_start()
3294 					amdgpu_ring_write(ring, ext->extent[i]);  in gfx_v9_0_cp_gfx_start()
3299 	amdgpu_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));  in gfx_v9_0_cp_gfx_start()
3300 	amdgpu_ring_write(ring, PACKET3_PREAMBLE_END_CLEAR_STATE);  in gfx_v9_0_cp_gfx_start()
3302 	amdgpu_ring_write(ring, PACKET3(PACKET3_CLEAR_STATE, 0));  in gfx_v9_0_cp_gfx_start()
3303 	amdgpu_ring_write(ring, 0);  in gfx_v9_0_cp_gfx_start()
3305 	amdgpu_ring_write(ring, PACKET3(PACKET3_SET_BASE, 2));  in gfx_v9_0_cp_gfx_start()
3306 	amdgpu_ring_write(ring, PACKET3_BASE_INDEX(CE_PARTITION_BASE));  in gfx_v9_0_cp_gfx_start()
3307 	amdgpu_ring_write(ring, 0x8000);  in gfx_v9_0_cp_gfx_start()
3308 	amdgpu_ring_write(ring, 0x8000);  in gfx_v9_0_cp_gfx_start()
3310 	amdgpu_ring_write(ring, PACKET3(PACKET3_SET_UCONFIG_REG,1));  in gfx_v9_0_cp_gfx_start()
3312 		(SOC15_REG_OFFSET(GC, 0, mmVGT_INDEX_TYPE) - PACKET3_SET_UCONFIG_REG_START));  in gfx_v9_0_cp_gfx_start()
3313 	amdgpu_ring_write(ring, tmp);  in gfx_v9_0_cp_gfx_start()
3314 	amdgpu_ring_write(ring, 0);  in gfx_v9_0_cp_gfx_start()
3316 	amdgpu_ring_commit(ring);  in gfx_v9_0_cp_gfx_start()
3323 	struct amdgpu_ring *ring;  in gfx_v9_0_cp_gfx_resume()  local
3334 	/* Set ring buffer size */  in gfx_v9_0_cp_gfx_resume()
3335 	ring = &adev->gfx.gfx_ring[0];  in gfx_v9_0_cp_gfx_resume()
3336 	rb_bufsz = order_base_2(ring->ring_size / 8);  in gfx_v9_0_cp_gfx_resume()
3338 	tmp = REG_SET_FIELD(tmp, CP_RB0_CNTL, RB_BLKSZ, rb_bufsz - 2);  in gfx_v9_0_cp_gfx_resume()
3344 	/* Initialize the ring buffer's write pointers */  in gfx_v9_0_cp_gfx_resume()
3345 	ring->wptr = 0;  in gfx_v9_0_cp_gfx_resume()
3346 	WREG32_SOC15(GC, 0, mmCP_RB0_WPTR, lower_32_bits(ring->wptr));  in gfx_v9_0_cp_gfx_resume()
3347 	WREG32_SOC15(GC, 0, mmCP_RB0_WPTR_HI, upper_32_bits(ring->wptr));  in gfx_v9_0_cp_gfx_resume()
3350 	rptr_addr = ring->rptr_gpu_addr;  in gfx_v9_0_cp_gfx_resume()
3354 	wptr_gpu_addr = ring->wptr_gpu_addr;  in gfx_v9_0_cp_gfx_resume()
3361 	rb_addr = ring->gpu_addr >> 8;  in gfx_v9_0_cp_gfx_resume()
3366 	if (ring->use_doorbell) {  in gfx_v9_0_cp_gfx_resume()
3368 				    DOORBELL_OFFSET, ring->doorbell_index);  in gfx_v9_0_cp_gfx_resume()
3377 			DOORBELL_RANGE_LOWER, ring->doorbell_index);  in gfx_v9_0_cp_gfx_resume()
3384 	/* start the ring */  in gfx_v9_0_cp_gfx_resume()
3397 		adev->gfx.kiq[0].ring.sched.ready = false;  in gfx_v9_0_cp_compute_enable()
3409 	if (!adev->gfx.mec_fw)  in gfx_v9_0_cp_compute_load_microcode()
3410 		return -EINVAL;  in gfx_v9_0_cp_compute_load_microcode()
3414 	mec_hdr = (const struct gfx_firmware_header_v1_0 *)adev->gfx.mec_fw->data;  in gfx_v9_0_cp_compute_load_microcode()
3415 	amdgpu_ucode_print_gfx_hdr(&mec_hdr->header);  in gfx_v9_0_cp_compute_load_microcode()
3418 		(adev->gfx.mec_fw->data +  in gfx_v9_0_cp_compute_load_microcode()
3419 		 le32_to_cpu(mec_hdr->header.ucode_array_offset_bytes));  in gfx_v9_0_cp_compute_load_microcode()
3426 		adev->gfx.mec.mec_fw_gpu_addr & 0xFFFFF000);  in gfx_v9_0_cp_compute_load_microcode()
3428 		upper_32_bits(adev->gfx.mec.mec_fw_gpu_addr));  in gfx_v9_0_cp_compute_load_microcode()
3432 			 mec_hdr->jt_offset);  in gfx_v9_0_cp_compute_load_microcode()
3433 	for (i = 0; i < mec_hdr->jt_size; i++)  in gfx_v9_0_cp_compute_load_microcode()
3435 			le32_to_cpup(fw_data + mec_hdr->jt_offset + i));  in gfx_v9_0_cp_compute_load_microcode()
3438 			adev->gfx.mec_fw_version);  in gfx_v9_0_cp_compute_load_microcode()
3445 static void gfx_v9_0_kiq_setting(struct amdgpu_ring *ring)  in gfx_v9_0_kiq_setting()  argument
3448 	struct amdgpu_device *adev = ring->adev;  in gfx_v9_0_kiq_setting()
3453 	tmp |= (ring->me << 5) | (ring->pipe << 3) | (ring->queue);  in gfx_v9_0_kiq_setting()
3459 static void gfx_v9_0_mqd_set_priority(struct amdgpu_ring *ring, struct v9_mqd *mqd)  in gfx_v9_0_mqd_set_priority()  argument
3461 	struct amdgpu_device *adev = ring->adev;  in gfx_v9_0_mqd_set_priority()
3463 	if (ring->funcs->type == AMDGPU_RING_TYPE_COMPUTE) {  in gfx_v9_0_mqd_set_priority()
3464 		if (amdgpu_gfx_is_high_priority_compute_queue(adev, ring)) {  in gfx_v9_0_mqd_set_priority()
3465 			mqd->cp_hqd_pipe_priority = AMDGPU_GFX_PIPE_PRIO_HIGH;  in gfx_v9_0_mqd_set_priority()
3466 			mqd->cp_hqd_queue_priority =  in gfx_v9_0_mqd_set_priority()
3472 static int gfx_v9_0_mqd_init(struct amdgpu_ring *ring)  in gfx_v9_0_mqd_init()  argument
3474 	struct amdgpu_device *adev = ring->adev;  in gfx_v9_0_mqd_init()
3475 	struct v9_mqd *mqd = ring->mqd_ptr;  in gfx_v9_0_mqd_init()
3479 	mqd->header = 0xC0310800;  in gfx_v9_0_mqd_init()
3480 	mqd->compute_pipelinestat_enable = 0x00000001;  in gfx_v9_0_mqd_init()
3481 	mqd->compute_static_thread_mgmt_se0 = 0xffffffff;  in gfx_v9_0_mqd_init()
3482 	mqd->compute_static_thread_mgmt_se1 = 0xffffffff;  in gfx_v9_0_mqd_init()
3483 	mqd->compute_static_thread_mgmt_se2 = 0xffffffff;  in gfx_v9_0_mqd_init()
3484 	mqd->compute_static_thread_mgmt_se3 = 0xffffffff;  in gfx_v9_0_mqd_init()
3485 	mqd->compute_static_thread_mgmt_se4 = 0xffffffff;  in gfx_v9_0_mqd_init()
3486 	mqd->compute_static_thread_mgmt_se5 = 0xffffffff;  in gfx_v9_0_mqd_init()
3487 	mqd->compute_static_thread_mgmt_se6 = 0xffffffff;  in gfx_v9_0_mqd_init()
3488 	mqd->compute_static_thread_mgmt_se7 = 0xffffffff;  in gfx_v9_0_mqd_init()
3489 	mqd->compute_misc_reserved = 0x00000003;  in gfx_v9_0_mqd_init()
3491 	mqd->dynamic_cu_mask_addr_lo =  in gfx_v9_0_mqd_init()
3492 		lower_32_bits(ring->mqd_gpu_addr  in gfx_v9_0_mqd_init()
3494 	mqd->dynamic_cu_mask_addr_hi =  in gfx_v9_0_mqd_init()
3495 		upper_32_bits(ring->mqd_gpu_addr  in gfx_v9_0_mqd_init()
3498 	eop_base_addr = ring->eop_gpu_addr >> 8;  in gfx_v9_0_mqd_init()
3499 	mqd->cp_hqd_eop_base_addr_lo = eop_base_addr;  in gfx_v9_0_mqd_init()
3500 	mqd->cp_hqd_eop_base_addr_hi = upper_32_bits(eop_base_addr);  in gfx_v9_0_mqd_init()
3505 			(order_base_2(GFX9_MEC_HPD_SIZE / 4) - 1));  in gfx_v9_0_mqd_init()
3507 	mqd->cp_hqd_eop_control = tmp;  in gfx_v9_0_mqd_init()
3512 	if (ring->use_doorbell) {  in gfx_v9_0_mqd_init()
3514 				    DOORBELL_OFFSET, ring->doorbell_index);  in gfx_v9_0_mqd_init()
3526 	mqd->cp_hqd_pq_doorbell_control = tmp;  in gfx_v9_0_mqd_init()
3529 	ring->wptr = 0;  in gfx_v9_0_mqd_init()
3530 	mqd->cp_hqd_dequeue_request = 0;  in gfx_v9_0_mqd_init()
3531 	mqd->cp_hqd_pq_rptr = 0;  in gfx_v9_0_mqd_init()
3532 	mqd->cp_hqd_pq_wptr_lo = 0;  in gfx_v9_0_mqd_init()
3533 	mqd->cp_hqd_pq_wptr_hi = 0;  in gfx_v9_0_mqd_init()
3536 	mqd->cp_mqd_base_addr_lo = ring->mqd_gpu_addr & 0xfffffffc;  in gfx_v9_0_mqd_init()
3537 	mqd->cp_mqd_base_addr_hi = upper_32_bits(ring->mqd_gpu_addr);  in gfx_v9_0_mqd_init()
3542 	mqd->cp_mqd_control = tmp;  in gfx_v9_0_mqd_init()
3545 	hqd_gpu_addr = ring->gpu_addr >> 8;  in gfx_v9_0_mqd_init()
3546 	mqd->cp_hqd_pq_base_lo = hqd_gpu_addr;  in gfx_v9_0_mqd_init()
3547 	mqd->cp_hqd_pq_base_hi = upper_32_bits(hqd_gpu_addr);  in gfx_v9_0_mqd_init()
3552 			    (order_base_2(ring->ring_size / 4) - 1));  in gfx_v9_0_mqd_init()
3554 			(order_base_2(AMDGPU_GPU_PAGE_SIZE / 4) - 1));  in gfx_v9_0_mqd_init()
3562 	mqd->cp_hqd_pq_control = tmp;  in gfx_v9_0_mqd_init()
3565 	wb_gpu_addr = ring->rptr_gpu_addr;  in gfx_v9_0_mqd_init()
3566 	mqd->cp_hqd_pq_rptr_report_addr_lo = wb_gpu_addr & 0xfffffffc;  in gfx_v9_0_mqd_init()
3567 	mqd->cp_hqd_pq_rptr_report_addr_hi =  in gfx_v9_0_mqd_init()
3571 	wb_gpu_addr = ring->wptr_gpu_addr;  in gfx_v9_0_mqd_init()
3572 	mqd->cp_hqd_pq_wptr_poll_addr_lo = wb_gpu_addr & 0xfffffffc;  in gfx_v9_0_mqd_init()
3573 	mqd->cp_hqd_pq_wptr_poll_addr_hi = upper_32_bits(wb_gpu_addr) & 0xffff;  in gfx_v9_0_mqd_init()
3576 	ring->wptr = 0;  in gfx_v9_0_mqd_init()
3577 	mqd->cp_hqd_pq_rptr = RREG32_SOC15(GC, 0, mmCP_HQD_PQ_RPTR);  in gfx_v9_0_mqd_init()
3580 	mqd->cp_hqd_vmid = 0;  in gfx_v9_0_mqd_init()
3584 	mqd->cp_hqd_persistent_state = tmp;  in gfx_v9_0_mqd_init()
3589 	mqd->cp_hqd_ib_control = tmp;  in gfx_v9_0_mqd_init()
3591 	/* set static priority for a queue/ring */  in gfx_v9_0_mqd_init()
3592 	gfx_v9_0_mqd_set_priority(ring, mqd);  in gfx_v9_0_mqd_init()
3593 	mqd->cp_hqd_quantum = RREG32_SOC15(GC, 0, mmCP_HQD_QUANTUM);  in gfx_v9_0_mqd_init()
3598 	if (ring->funcs->type == AMDGPU_RING_TYPE_KIQ)  in gfx_v9_0_mqd_init()
3599 		mqd->cp_hqd_active = 1;  in gfx_v9_0_mqd_init()
3604 static int gfx_v9_0_kiq_init_register(struct amdgpu_ring *ring)  in gfx_v9_0_kiq_init_register()  argument
3606 	struct amdgpu_device *adev = ring->adev;  in gfx_v9_0_kiq_init_register()
3607 	struct v9_mqd *mqd = ring->mqd_ptr;  in gfx_v9_0_kiq_init_register()
3614 	       mqd->cp_hqd_eop_base_addr_lo);  in gfx_v9_0_kiq_init_register()
3616 	       mqd->cp_hqd_eop_base_addr_hi);  in gfx_v9_0_kiq_init_register()
3620 	       mqd->cp_hqd_eop_control);  in gfx_v9_0_kiq_init_register()
3624 	       mqd->cp_hqd_pq_doorbell_control);  in gfx_v9_0_kiq_init_register()
3629 		for (j = 0; j < adev->usec_timeout; j++) {  in gfx_v9_0_kiq_init_register()
3635 		       mqd->cp_hqd_dequeue_request);  in gfx_v9_0_kiq_init_register()
3637 		       mqd->cp_hqd_pq_rptr);  in gfx_v9_0_kiq_init_register()
3639 		       mqd->cp_hqd_pq_wptr_lo);  in gfx_v9_0_kiq_init_register()
3641 		       mqd->cp_hqd_pq_wptr_hi);  in gfx_v9_0_kiq_init_register()
3646 	       mqd->cp_mqd_base_addr_lo);  in gfx_v9_0_kiq_init_register()
3648 	       mqd->cp_mqd_base_addr_hi);  in gfx_v9_0_kiq_init_register()
3652 	       mqd->cp_mqd_control);  in gfx_v9_0_kiq_init_register()
3656 	       mqd->cp_hqd_pq_base_lo);  in gfx_v9_0_kiq_init_register()
3658 	       mqd->cp_hqd_pq_base_hi);  in gfx_v9_0_kiq_init_register()
3662 	       mqd->cp_hqd_pq_control);  in gfx_v9_0_kiq_init_register()
3666 				mqd->cp_hqd_pq_rptr_report_addr_lo);  in gfx_v9_0_kiq_init_register()
3668 				mqd->cp_hqd_pq_rptr_report_addr_hi);  in gfx_v9_0_kiq_init_register()
3672 	       mqd->cp_hqd_pq_wptr_poll_addr_lo);  in gfx_v9_0_kiq_init_register()
3674 	       mqd->cp_hqd_pq_wptr_poll_addr_hi);  in gfx_v9_0_kiq_init_register()
3677 	if (ring->use_doorbell) {  in gfx_v9_0_kiq_init_register()
3679 					(adev->doorbell_index.kiq * 2) << 2);  in gfx_v9_0_kiq_init_register()
3687 					(adev->doorbell.size - 4));  in gfx_v9_0_kiq_init_register()
3690 					(adev->doorbell_index.userqueue_end * 2) << 2);  in gfx_v9_0_kiq_init_register()
3694 	       mqd->cp_hqd_pq_doorbell_control);  in gfx_v9_0_kiq_init_register()
3698 	       mqd->cp_hqd_pq_wptr_lo);  in gfx_v9_0_kiq_init_register()
3700 	       mqd->cp_hqd_pq_wptr_hi);  in gfx_v9_0_kiq_init_register()
3703 	WREG32_SOC15_RLC(GC, 0, mmCP_HQD_VMID, mqd->cp_hqd_vmid);  in gfx_v9_0_kiq_init_register()
3706 	       mqd->cp_hqd_persistent_state);  in gfx_v9_0_kiq_init_register()
3710 	       mqd->cp_hqd_active);  in gfx_v9_0_kiq_init_register()
3712 	if (ring->use_doorbell)  in gfx_v9_0_kiq_init_register()
3718 static int gfx_v9_0_kiq_fini_register(struct amdgpu_ring *ring)  in gfx_v9_0_kiq_fini_register()  argument
3720 	struct amdgpu_device *adev = ring->adev;  in gfx_v9_0_kiq_fini_register()
3728 		for (j = 0; j < adev->usec_timeout; j++) {  in gfx_v9_0_kiq_fini_register()
3757 static int gfx_v9_0_kiq_init_queue(struct amdgpu_ring *ring)  in gfx_v9_0_kiq_init_queue()  argument
3759 	struct amdgpu_device *adev = ring->adev;  in gfx_v9_0_kiq_init_queue()
3760 	struct v9_mqd *mqd = ring->mqd_ptr;  in gfx_v9_0_kiq_init_queue()
3763 	gfx_v9_0_kiq_setting(ring);  in gfx_v9_0_kiq_init_queue()
3767 	 * driver need to re-init the mqd.  in gfx_v9_0_kiq_init_queue()
3768 	 * check mqd->cp_hqd_pq_control since this value should not be 0  in gfx_v9_0_kiq_init_queue()
3770 	tmp_mqd = (struct v9_mqd *)adev->gfx.kiq[0].mqd_backup;  in gfx_v9_0_kiq_init_queue()
3771 	if (amdgpu_in_reset(adev) && tmp_mqd->cp_hqd_pq_control){  in gfx_v9_0_kiq_init_queue()
3773 		if (adev->gfx.kiq[0].mqd_backup)  in gfx_v9_0_kiq_init_queue()
3774 			memcpy(mqd, adev->gfx.kiq[0].mqd_backup, sizeof(struct v9_mqd_allocation));  in gfx_v9_0_kiq_init_queue()
3776 		/* reset ring buffer */  in gfx_v9_0_kiq_init_queue()
3777 		ring->wptr = 0;  in gfx_v9_0_kiq_init_queue()
3778 		amdgpu_ring_clear_ring(ring);  in gfx_v9_0_kiq_init_queue()
3780 		mutex_lock(&adev->srbm_mutex);  in gfx_v9_0_kiq_init_queue()
3781 		soc15_grbm_select(adev, ring->me, ring->pipe, ring->queue, 0, 0);  in gfx_v9_0_kiq_init_queue()
3782 		gfx_v9_0_kiq_init_register(ring);  in gfx_v9_0_kiq_init_queue()
3784 		mutex_unlock(&adev->srbm_mutex);  in gfx_v9_0_kiq_init_queue()
3787 		((struct v9_mqd_allocation *)mqd)->dynamic_cu_mask = 0xFFFFFFFF;  in gfx_v9_0_kiq_init_queue()
3788 		((struct v9_mqd_allocation *)mqd)->dynamic_rb_mask = 0xFFFFFFFF;  in gfx_v9_0_kiq_init_queue()
3789 		if (amdgpu_sriov_vf(adev) && adev->in_suspend)  in gfx_v9_0_kiq_init_queue()
3790 			amdgpu_ring_clear_ring(ring);  in gfx_v9_0_kiq_init_queue()
3791 		mutex_lock(&adev->srbm_mutex);  in gfx_v9_0_kiq_init_queue()
3792 		soc15_grbm_select(adev, ring->me, ring->pipe, ring->queue, 0, 0);  in gfx_v9_0_kiq_init_queue()
3793 		gfx_v9_0_mqd_init(ring);  in gfx_v9_0_kiq_init_queue()
3794 		gfx_v9_0_kiq_init_register(ring);  in gfx_v9_0_kiq_init_queue()
3796 		mutex_unlock(&adev->srbm_mutex);  in gfx_v9_0_kiq_init_queue()
3798 		if (adev->gfx.kiq[0].mqd_backup)  in gfx_v9_0_kiq_init_queue()
3799 			memcpy(adev->gfx.kiq[0].mqd_backup, mqd, sizeof(struct v9_mqd_allocation));  in gfx_v9_0_kiq_init_queue()
3805 static int gfx_v9_0_kcq_init_queue(struct amdgpu_ring *ring, bool restore)  in gfx_v9_0_kcq_init_queue()  argument
3807 	struct amdgpu_device *adev = ring->adev;  in gfx_v9_0_kcq_init_queue()
3808 	struct v9_mqd *mqd = ring->mqd_ptr;  in gfx_v9_0_kcq_init_queue()
3809 	int mqd_idx = ring - &adev->gfx.compute_ring[0];  in gfx_v9_0_kcq_init_queue()
3812 	/* Same as above kiq init, driver need to re-init the mqd if mqd->cp_hqd_pq_control  in gfx_v9_0_kcq_init_queue()
3815 	tmp_mqd = (struct v9_mqd *)adev->gfx.mec.mqd_backup[mqd_idx];  in gfx_v9_0_kcq_init_queue()
3817 	if (!restore && (!tmp_mqd->cp_hqd_pq_control ||  in gfx_v9_0_kcq_init_queue()
3818 	    (!amdgpu_in_reset(adev) && !adev->in_suspend))) {  in gfx_v9_0_kcq_init_queue()
3820 		((struct v9_mqd_allocation *)mqd)->dynamic_cu_mask = 0xFFFFFFFF;  in gfx_v9_0_kcq_init_queue()
3821 		((struct v9_mqd_allocation *)mqd)->dynamic_rb_mask = 0xFFFFFFFF;  in gfx_v9_0_kcq_init_queue()
3822 		mutex_lock(&adev->srbm_mutex);  in gfx_v9_0_kcq_init_queue()
3823 		soc15_grbm_select(adev, ring->me, ring->pipe, ring->queue, 0, 0);  in gfx_v9_0_kcq_init_queue()
3824 		gfx_v9_0_mqd_init(ring);  in gfx_v9_0_kcq_init_queue()
3826 		mutex_unlock(&adev->srbm_mutex);  in gfx_v9_0_kcq_init_queue()
3828 		if (adev->gfx.mec.mqd_backup[mqd_idx])  in gfx_v9_0_kcq_init_queue()
3829 			memcpy(adev->gfx.mec.mqd_backup[mqd_idx], mqd, sizeof(struct v9_mqd_allocation));  in gfx_v9_0_kcq_init_queue()
3832 		if (adev->gfx.mec.mqd_backup[mqd_idx])  in gfx_v9_0_kcq_init_queue()
3833 			memcpy(mqd, adev->gfx.mec.mqd_backup[mqd_idx], sizeof(struct v9_mqd_allocation));  in gfx_v9_0_kcq_init_queue()
3834 		/* reset ring buffer */  in gfx_v9_0_kcq_init_queue()
3835 		ring->wptr = 0;  in gfx_v9_0_kcq_init_queue()
3836 		atomic64_set((atomic64_t *)ring->wptr_cpu_addr, 0);  in gfx_v9_0_kcq_init_queue()
3837 		amdgpu_ring_clear_ring(ring);  in gfx_v9_0_kcq_init_queue()
3845 	struct amdgpu_ring *ring;  in gfx_v9_0_kiq_resume()  local
3848 	ring = &adev->gfx.kiq[0].ring;  in gfx_v9_0_kiq_resume()
3850 	r = amdgpu_bo_reserve(ring->mqd_obj, false);  in gfx_v9_0_kiq_resume()
3854 	r = amdgpu_bo_kmap(ring->mqd_obj, (void **)&ring->mqd_ptr);  in gfx_v9_0_kiq_resume()
3856 		amdgpu_bo_unreserve(ring->mqd_obj);  in gfx_v9_0_kiq_resume()
3860 	gfx_v9_0_kiq_init_queue(ring);  in gfx_v9_0_kiq_resume()
3861 	amdgpu_bo_kunmap(ring->mqd_obj);  in gfx_v9_0_kiq_resume()
3862 	ring->mqd_ptr = NULL;  in gfx_v9_0_kiq_resume()
3863 	amdgpu_bo_unreserve(ring->mqd_obj);  in gfx_v9_0_kiq_resume()
3869 	struct amdgpu_ring *ring = NULL;  in gfx_v9_0_kcq_resume()  local
3874 	for (i = 0; i < adev->gfx.num_compute_rings; i++) {  in gfx_v9_0_kcq_resume()
3875 		ring = &adev->gfx.compute_ring[i];  in gfx_v9_0_kcq_resume()
3877 		r = amdgpu_bo_reserve(ring->mqd_obj, false);  in gfx_v9_0_kcq_resume()
3880 		r = amdgpu_bo_kmap(ring->mqd_obj, (void **)&ring->mqd_ptr);  in gfx_v9_0_kcq_resume()
3882 			r = gfx_v9_0_kcq_init_queue(ring, false);  in gfx_v9_0_kcq_resume()
3883 			amdgpu_bo_kunmap(ring->mqd_obj);  in gfx_v9_0_kcq_resume()
3884 			ring->mqd_ptr = NULL;  in gfx_v9_0_kcq_resume()
3886 		amdgpu_bo_unreserve(ring->mqd_obj);  in gfx_v9_0_kcq_resume()
3899 	struct amdgpu_ring *ring;  in gfx_v9_0_cp_resume()  local
3901 	if (!(adev->flags & AMD_IS_APU))  in gfx_v9_0_cp_resume()
3904 	if (adev->firmware.load_type != AMDGPU_FW_LOAD_PSP) {  in gfx_v9_0_cp_resume()
3905 		if (adev->gfx.num_gfx_rings) {  in gfx_v9_0_cp_resume()
3921 	if (adev->gfx.num_gfx_rings) {  in gfx_v9_0_cp_resume()
3931 	if (adev->gfx.num_gfx_rings) {  in gfx_v9_0_cp_resume()
3932 		ring = &adev->gfx.gfx_ring[0];  in gfx_v9_0_cp_resume()
3933 		r = amdgpu_ring_test_helper(ring);  in gfx_v9_0_cp_resume()
3938 	for (i = 0; i < adev->gfx.num_compute_rings; i++) {  in gfx_v9_0_cp_resume()
3939 		ring = &adev->gfx.compute_ring[i];  in gfx_v9_0_cp_resume()
3940 		amdgpu_ring_test_helper(ring);  in gfx_v9_0_cp_resume()
3952 	if (amdgpu_ip_version(adev, GC_HWIP, 0) != IP_VERSION(9, 4, 1) &&  in gfx_v9_0_init_tcp_config()
3953 	    amdgpu_ip_version(adev, GC_HWIP, 0) != IP_VERSION(9, 4, 2))  in gfx_v9_0_init_tcp_config()
3958 				adev->df.hash_status.hash_64k);  in gfx_v9_0_init_tcp_config()
3960 				adev->df.hash_status.hash_2m);  in gfx_v9_0_init_tcp_config()
3962 				adev->df.hash_status.hash_1g);  in gfx_v9_0_init_tcp_config()
3968 	if (adev->gfx.num_gfx_rings)  in gfx_v9_0_cp_enable()
3978 	amdgpu_gfx_cleaner_shader_init(adev, adev->gfx.cleaner_shader_size,  in gfx_v9_0_hw_init()
3979 				       adev->gfx.cleaner_shader_ptr);  in gfx_v9_0_hw_init()
3988 	r = adev->gfx.rlc.funcs->resume(adev);  in gfx_v9_0_hw_init()
3996 	if (amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(9, 4, 2))  in gfx_v9_0_hw_init()
4007 		amdgpu_irq_put(adev, &adev->gfx.cp_ecc_error_irq, 0);  in gfx_v9_0_hw_fini()
4008 	amdgpu_irq_put(adev, &adev->gfx.priv_reg_irq, 0);  in gfx_v9_0_hw_fini()
4009 	amdgpu_irq_put(adev, &adev->gfx.priv_inst_irq, 0);  in gfx_v9_0_hw_fini()
4010 	amdgpu_irq_put(adev, &adev->gfx.bad_op_irq, 0);  in gfx_v9_0_hw_fini()
4031 	if (!amdgpu_in_reset(adev) && !adev->in_suspend) {  in gfx_v9_0_hw_fini()
4032 		mutex_lock(&adev->srbm_mutex);  in gfx_v9_0_hw_fini()
4033 		soc15_grbm_select(adev, adev->gfx.kiq[0].ring.me,  in gfx_v9_0_hw_fini()
4034 				adev->gfx.kiq[0].ring.pipe,  in gfx_v9_0_hw_fini()
4035 				adev->gfx.kiq[0].ring.queue, 0, 0);  in gfx_v9_0_hw_fini()
4036 		gfx_v9_0_kiq_fini_register(&adev->gfx.kiq[0].ring);  in gfx_v9_0_hw_fini()
4038 		mutex_unlock(&adev->srbm_mutex);  in gfx_v9_0_hw_fini()
4044 	if ((adev->gmc.xgmi.connected_to_cpu && amdgpu_in_reset(adev)) ||  in gfx_v9_0_hw_fini()
4045 	    (amdgpu_ip_version(adev, GC_HWIP, 0) >= IP_VERSION(9, 4, 2))) {  in gfx_v9_0_hw_fini()
4046 		dev_dbg(adev->dev, "Skipping RLC halt\n");  in gfx_v9_0_hw_fini()
4050 	adev->gfx.rlc.funcs->stop(adev);  in gfx_v9_0_hw_fini()
4080 	for (i = 0; i < adev->usec_timeout; i++) {  in gfx_v9_0_wait_for_idle()
4085 	return -ETIMEDOUT;  in gfx_v9_0_wait_for_idle()
4122 		adev->gfx.rlc.funcs->stop(adev);  in gfx_v9_0_soft_reset()
4124 		if (adev->gfx.num_gfx_rings)  in gfx_v9_0_soft_reset()
4134 			dev_info(adev->dev, "GRBM_SOFT_RESET=0x%08X\n", tmp);  in gfx_v9_0_soft_reset()
4157 	struct amdgpu_kiq *kiq = &adev->gfx.kiq[0];  in gfx_v9_0_kiq_read_clock()
4158 	struct amdgpu_ring *ring = &kiq->ring;  in gfx_v9_0_kiq_read_clock()  local
4160 	BUG_ON(!ring->funcs->emit_rreg);  in gfx_v9_0_kiq_read_clock()
4162 	spin_lock_irqsave(&kiq->ring_lock, flags);  in gfx_v9_0_kiq_read_clock()
4167 	amdgpu_ring_alloc(ring, 32);  in gfx_v9_0_kiq_read_clock()
4168 	amdgpu_ring_write(ring, PACKET3(PACKET3_COPY_DATA, 4));  in gfx_v9_0_kiq_read_clock()
4169 	amdgpu_ring_write(ring, 9 |	/* src: register*/  in gfx_v9_0_kiq_read_clock()
4173 	amdgpu_ring_write(ring, 0);  in gfx_v9_0_kiq_read_clock()
4174 	amdgpu_ring_write(ring, 0);  in gfx_v9_0_kiq_read_clock()
4175 	amdgpu_ring_write(ring, lower_32_bits(adev->wb.gpu_addr +  in gfx_v9_0_kiq_read_clock()
4176 				reg_val_offs * 4));  in gfx_v9_0_kiq_read_clock()
4177 	amdgpu_ring_write(ring, upper_32_bits(adev->wb.gpu_addr +  in gfx_v9_0_kiq_read_clock()
4178 				reg_val_offs * 4));  in gfx_v9_0_kiq_read_clock()
4179 	r = amdgpu_fence_emit_polling(ring, &seq, MAX_KIQ_REG_WAIT);  in gfx_v9_0_kiq_read_clock()
4183 	amdgpu_ring_commit(ring);  in gfx_v9_0_kiq_read_clock()
4184 	spin_unlock_irqrestore(&kiq->ring_lock, flags);  in gfx_v9_0_kiq_read_clock()
4186 	r = amdgpu_fence_wait_polling(ring, seq, MAX_KIQ_REG_WAIT);  in gfx_v9_0_kiq_read_clock()
4202 		r = amdgpu_fence_wait_polling(ring, seq, MAX_KIQ_REG_WAIT);  in gfx_v9_0_kiq_read_clock()
4209 	value = (uint64_t)adev->wb.wb[reg_val_offs] |  in gfx_v9_0_kiq_read_clock()
4210 		(uint64_t)adev->wb.wb[reg_val_offs + 1 ] << 32ULL;  in gfx_v9_0_kiq_read_clock()
4215 	amdgpu_ring_undo(ring);  in gfx_v9_0_kiq_read_clock()
4217 	spin_unlock_irqrestore(&kiq->ring_lock, flags);  in gfx_v9_0_kiq_read_clock()
4235 		/* The SMUIO TSC clock frequency is 100MHz, which sets 32-bit carry over  in gfx_v9_0_get_gpu_clock_counter()
4247 		mutex_lock(&adev->gfx.gpu_clock_mutex);  in gfx_v9_0_get_gpu_clock_counter()
4257 		mutex_unlock(&adev->gfx.gpu_clock_mutex);  in gfx_v9_0_get_gpu_clock_counter()
4264 static void gfx_v9_0_ring_emit_gds_switch(struct amdgpu_ring *ring,  in gfx_v9_0_ring_emit_gds_switch()  argument
4270 	struct amdgpu_device *adev = ring->adev;  in gfx_v9_0_ring_emit_gds_switch()
4273 	gfx_v9_0_write_data_to_reg(ring, 0, false,  in gfx_v9_0_ring_emit_gds_switch()
4278 	gfx_v9_0_write_data_to_reg(ring, 0, false,  in gfx_v9_0_ring_emit_gds_switch()
4283 	gfx_v9_0_write_data_to_reg(ring, 0, false,  in gfx_v9_0_ring_emit_gds_switch()
4288 	gfx_v9_0_write_data_to_reg(ring, 0, false,  in gfx_v9_0_ring_emit_gds_switch()
4290 				   (1 << (oa_size + oa_base)) - (1 << oa_base));  in gfx_v9_0_ring_emit_gds_switch()
4428    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_NUM_THREAD_Y), 4 },
4445    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_NUM_THREAD_Y), 4 },
4506    { SOC15_REG_ENTRY(GC, 0, mmSPI_EDC_CNT), 0, 4, 1},
4507    { SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT), 0, 4, 6},
4508    { SOC15_REG_ENTRY(GC, 0, mmSQ_EDC_DED_CNT), 0, 4, 16},
4509    { SOC15_REG_ENTRY(GC, 0, mmSQ_EDC_INFO), 0, 4, 16},
4510    { SOC15_REG_ENTRY(GC, 0, mmSQ_EDC_SEC_CNT), 0, 4, 16},
4512    { SOC15_REG_ENTRY(GC, 0, mmTCP_ATC_EDC_GATCL1_CNT), 0, 4, 16},
4513    { SOC15_REG_ENTRY(GC, 0, mmTCP_EDC_CNT), 0, 4, 16},
4514    { SOC15_REG_ENTRY(GC, 0, mmTCP_EDC_CNT_NEW), 0, 4, 16},
4515    { SOC15_REG_ENTRY(GC, 0, mmTD_EDC_CNT), 0, 4, 16},
4516    { SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT2), 0, 4, 6},
4517    { SOC15_REG_ENTRY(GC, 0, mmSQ_EDC_CNT), 0, 4, 16},
4518    { SOC15_REG_ENTRY(GC, 0, mmTA_EDC_CNT), 0, 4, 16},
4526    { SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT3), 0, 4, 6},
4531 	struct amdgpu_ring *ring = &adev->gfx.compute_ring[0];  in gfx_v9_0_do_edc_gds_workarounds()  local
4538 	r = amdgpu_ring_alloc(ring, 7);  in gfx_v9_0_do_edc_gds_workarounds()
4540 		DRM_ERROR("amdgpu: GDS workarounds failed to lock ring %s (%d).\n",  in gfx_v9_0_do_edc_gds_workarounds()
4541 			ring->name, r);  in gfx_v9_0_do_edc_gds_workarounds()
4546 	WREG32_SOC15(GC, 0, mmGDS_VMID0_SIZE, adev->gds.gds_size);  in gfx_v9_0_do_edc_gds_workarounds()
4548 	amdgpu_ring_write(ring, PACKET3(PACKET3_DMA_DATA, 5));  in gfx_v9_0_do_edc_gds_workarounds()
4549 	amdgpu_ring_write(ring, (PACKET3_DMA_DATA_CP_SYNC |  in gfx_v9_0_do_edc_gds_workarounds()
4553 	amdgpu_ring_write(ring, 0);  in gfx_v9_0_do_edc_gds_workarounds()
4554 	amdgpu_ring_write(ring, 0);  in gfx_v9_0_do_edc_gds_workarounds()
4555 	amdgpu_ring_write(ring, 0);  in gfx_v9_0_do_edc_gds_workarounds()
4556 	amdgpu_ring_write(ring, 0);  in gfx_v9_0_do_edc_gds_workarounds()
4557 	amdgpu_ring_write(ring, PACKET3_DMA_DATA_CMD_RAW_WAIT |  in gfx_v9_0_do_edc_gds_workarounds()
4558 				adev->gds.gds_size);  in gfx_v9_0_do_edc_gds_workarounds()
4560 	amdgpu_ring_commit(ring);  in gfx_v9_0_do_edc_gds_workarounds()
4562 	for (i = 0; i < adev->usec_timeout; i++) {  in gfx_v9_0_do_edc_gds_workarounds()
4563 		if (ring->wptr == gfx_v9_0_ring_get_rptr_compute(ring))  in gfx_v9_0_do_edc_gds_workarounds()
4568 	if (i >= adev->usec_timeout)  in gfx_v9_0_do_edc_gds_workarounds()
4569 		r = -ETIMEDOUT;  in gfx_v9_0_do_edc_gds_workarounds()
4578 	struct amdgpu_ring *ring = &adev->gfx.compute_ring[0];  in gfx_v9_0_do_edc_gpr_workarounds()  local
4585 	int compute_dim_x = adev->gfx.config.max_shader_engines *  in gfx_v9_0_do_edc_gpr_workarounds()
4586 						adev->gfx.config.max_cu_per_sh *  in gfx_v9_0_do_edc_gpr_workarounds()
4587 						adev->gfx.config.max_sh_per_se;  in gfx_v9_0_do_edc_gpr_workarounds()
4589 	int gpr_reg_size = adev->gfx.config.max_shader_engines + 6;  in gfx_v9_0_do_edc_gpr_workarounds()
4598 	/* bail if the compute ring is not ready */  in gfx_v9_0_do_edc_gpr_workarounds()
4599 	if (!ring->sched.ready)  in gfx_v9_0_do_edc_gpr_workarounds()
4602 	if (amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(9, 4, 1)) {  in gfx_v9_0_do_edc_gpr_workarounds()
4613 		(gpr_reg_size * 3 + 4 + 5 + 2) * 4; /* VGPRS */  in gfx_v9_0_do_edc_gpr_workarounds()
4615 		(gpr_reg_size * 3 + 4 + 5 + 2) * 4; /* SGPRS1 */  in gfx_v9_0_do_edc_gpr_workarounds()
4617 		(gpr_reg_size * 3 + 4 + 5 + 2) * 4; /* SGPRS2 */  in gfx_v9_0_do_edc_gpr_workarounds()
4635 		ib.ptr[i + (vgpr_offset / 4)] = vgpr_init_shader_ptr[i];  in gfx_v9_0_do_edc_gpr_workarounds()
4638 		ib.ptr[i + (sgpr_offset / 4)] = sgpr_init_compute_shader[i];  in gfx_v9_0_do_edc_gpr_workarounds()
4648 								- PACKET3_SET_SH_REG_START;  in gfx_v9_0_do_edc_gpr_workarounds()
4655 							- PACKET3_SET_SH_REG_START;  in gfx_v9_0_do_edc_gpr_workarounds()
4669 	ib.ptr[ib.length_dw++] = EVENT_TYPE(7) | EVENT_INDEX(4);  in gfx_v9_0_do_edc_gpr_workarounds()
4676 								- PACKET3_SET_SH_REG_START;  in gfx_v9_0_do_edc_gpr_workarounds()
4683 							- PACKET3_SET_SH_REG_START;  in gfx_v9_0_do_edc_gpr_workarounds()
4697 	ib.ptr[ib.length_dw++] = EVENT_TYPE(7) | EVENT_INDEX(4);  in gfx_v9_0_do_edc_gpr_workarounds()
4704 								- PACKET3_SET_SH_REG_START;  in gfx_v9_0_do_edc_gpr_workarounds()
4711 							- PACKET3_SET_SH_REG_START;  in gfx_v9_0_do_edc_gpr_workarounds()
4725 	ib.ptr[ib.length_dw++] = EVENT_TYPE(7) | EVENT_INDEX(4);  in gfx_v9_0_do_edc_gpr_workarounds()
4727 	/* shedule the ib on the ring */  in gfx_v9_0_do_edc_gpr_workarounds()
4728 	r = amdgpu_ib_schedule(ring, 1, &ib, NULL, &f);  in gfx_v9_0_do_edc_gpr_workarounds()
4752 	adev->gfx.funcs = &gfx_v9_0_gfx_funcs;  in gfx_v9_0_early_init()
4754 	if (amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(9, 4, 1) ||  in gfx_v9_0_early_init()
4755 	    amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(9, 4, 2))  in gfx_v9_0_early_init()
4756 		adev->gfx.num_gfx_rings = 0;  in gfx_v9_0_early_init()
4758 		adev->gfx.num_gfx_rings = GFX9_NUM_GFX_RINGS;  in gfx_v9_0_early_init()
4759 	adev->gfx.xcc_mask = 1;  in gfx_v9_0_early_init()
4760 	adev->gfx.num_compute_rings = min(amdgpu_gfx_get_num_kcq(adev),  in gfx_v9_0_early_init()
4785 	if ((!adev->in_suspend) &&  in gfx_v9_0_ecc_late_init()
4786 	    (adev->gds.gds_size)) {  in gfx_v9_0_ecc_late_init()
4793 	if (amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(9, 4, 2))  in gfx_v9_0_ecc_late_init()
4801 	if (adev->gfx.ras &&  in gfx_v9_0_ecc_late_init()
4802 	    adev->gfx.ras->enable_watchdog_timer)  in gfx_v9_0_ecc_late_init()
4803 		adev->gfx.ras->enable_watchdog_timer(adev);  in gfx_v9_0_ecc_late_init()
4813 	r = amdgpu_irq_get(adev, &adev->gfx.priv_reg_irq, 0);  in gfx_v9_0_late_init()
4817 	r = amdgpu_irq_get(adev, &adev->gfx.priv_inst_irq, 0);  in gfx_v9_0_late_init()
4821 	r = amdgpu_irq_get(adev, &adev->gfx.bad_op_irq, 0);  in gfx_v9_0_late_init()
4829 	if (amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(9, 4, 2))  in gfx_v9_0_late_init()
4831 			adev->vm_manager.first_kfd_vmid, AMDGPU_NUM_VMID);  in gfx_v9_0_late_init()
4834 			adev->vm_manager.first_kfd_vmid, AMDGPU_NUM_VMID);  in gfx_v9_0_late_init()
4861 	for (i = 0; i < adev->usec_timeout; i++) {  in gfx_v9_0_set_safe_mode()
4881 	if ((adev->pg_flags & AMD_PG_SUPPORT_GFX_PG) && enable) {  in gfx_v9_0_update_gfx_cg_power_gating()
4883 		if (adev->pg_flags & AMD_PG_SUPPORT_GFX_PIPELINE)  in gfx_v9_0_update_gfx_cg_power_gating()
4887 		if (adev->pg_flags & AMD_PG_SUPPORT_GFX_PIPELINE)  in gfx_v9_0_update_gfx_cg_power_gating()
4900 	if ((adev->pg_flags & AMD_PG_SUPPORT_GFX_SMG) && enable)  in gfx_v9_0_update_gfx_mg_power_gating()
4905 	if ((adev->pg_flags & AMD_PG_SUPPORT_GFX_DMG) && enable)  in gfx_v9_0_update_gfx_mg_power_gating()
4921 	if (enable && (adev->cg_flags & AMD_CG_SUPPORT_GFX_MGCG)) {  in gfx_v9_0_update_medium_grain_clock_gating()
4922 		/* 1 - RLC_CGTT_MGCG_OVERRIDE */  in gfx_v9_0_update_medium_grain_clock_gating()
4939 		if (adev->cg_flags & AMD_CG_SUPPORT_GFX_MGLS) {  in gfx_v9_0_update_medium_grain_clock_gating()
4940 			/* 2 - RLC memory Light sleep */  in gfx_v9_0_update_medium_grain_clock_gating()
4941 			if (adev->cg_flags & AMD_CG_SUPPORT_GFX_RLC_LS) {  in gfx_v9_0_update_medium_grain_clock_gating()
4947 			/* 3 - CP memory Light sleep */  in gfx_v9_0_update_medium_grain_clock_gating()
4948 			if (adev->cg_flags & AMD_CG_SUPPORT_GFX_CP_LS) {  in gfx_v9_0_update_medium_grain_clock_gating()
4956 		/* 1 - MGCG_OVERRIDE */  in gfx_v9_0_update_medium_grain_clock_gating()
4970 		/* 2 - disable MGLS in RLC */  in gfx_v9_0_update_medium_grain_clock_gating()
4977 		/* 3 - disable MGLS in CP */  in gfx_v9_0_update_medium_grain_clock_gating()
4993 	if (!adev->gfx.num_gfx_rings)  in gfx_v9_0_update_3d_clock_gating()
5011 		if (adev->cg_flags & AMD_CG_SUPPORT_GFX_3D_CGCG)  in gfx_v9_0_update_3d_clock_gating()
5017 		if (adev->cg_flags & AMD_CG_SUPPORT_GFX_3D_CGLS)  in gfx_v9_0_update_3d_clock_gating()
5050 	if (enable && (adev->cg_flags & AMD_CG_SUPPORT_GFX_CGCG)) {  in gfx_v9_0_update_coarse_grain_clock_gating()
5054 		if (adev->cg_flags & AMD_CG_SUPPORT_GFX_CGLS)  in gfx_v9_0_update_coarse_grain_clock_gating()
5065 		if (amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(9, 4, 1))  in gfx_v9_0_update_coarse_grain_clock_gating()
5071 		if (adev->cg_flags & AMD_CG_SUPPORT_GFX_CGLS)  in gfx_v9_0_update_coarse_grain_clock_gating()
5140 static void gfx_v9_0_update_spm_vmid(struct amdgpu_device *adev, struct amdgpu_ring *ring, unsigned…  in gfx_v9_0_update_spm_vmid()  argument
5163 		reg = adev->reg_offset[entry->hwip][entry->instance][entry->segment] + entry->reg;  in gfx_v9_0_check_rlcg_range()
5207 		if (adev->pg_flags & AMD_PG_SUPPORT_RLC_SMU_HS) {  in gfx_v9_0_set_powergating_state()
5215 		if (adev->pg_flags & AMD_PG_SUPPORT_CP)  in gfx_v9_0_set_powergating_state()
5250 	case IP_VERSION(9, 4, 0):  in gfx_v9_0_set_clockgating_state()
5253 	case IP_VERSION(9, 4, 1):  in gfx_v9_0_set_clockgating_state()
5255 	case IP_VERSION(9, 4, 2):  in gfx_v9_0_set_clockgating_state()
5297 	if (amdgpu_ip_version(adev, GC_HWIP, 0) != IP_VERSION(9, 4, 1)) {  in gfx_v9_0_get_clockgating_state()
5309 static u64 gfx_v9_0_ring_get_rptr_gfx(struct amdgpu_ring *ring)  in gfx_v9_0_ring_get_rptr_gfx()  argument
5311 	return *ring->rptr_cpu_addr; /* gfx9 is 32bit rptr*/  in gfx_v9_0_ring_get_rptr_gfx()
5314 static u64 gfx_v9_0_ring_get_wptr_gfx(struct amdgpu_ring *ring)  in gfx_v9_0_ring_get_wptr_gfx()  argument
5316 	struct amdgpu_device *adev = ring->adev;  in gfx_v9_0_ring_get_wptr_gfx()
5320 	if (ring->use_doorbell) {  in gfx_v9_0_ring_get_wptr_gfx()
5321 		wptr = atomic64_read((atomic64_t *)ring->wptr_cpu_addr);  in gfx_v9_0_ring_get_wptr_gfx()
5330 static void gfx_v9_0_ring_set_wptr_gfx(struct amdgpu_ring *ring)  in gfx_v9_0_ring_set_wptr_gfx()  argument
5332 	struct amdgpu_device *adev = ring->adev;  in gfx_v9_0_ring_set_wptr_gfx()
5334 	if (ring->use_doorbell) {  in gfx_v9_0_ring_set_wptr_gfx()
5336 		atomic64_set((atomic64_t *)ring->wptr_cpu_addr, ring->wptr);  in gfx_v9_0_ring_set_wptr_gfx()
5337 		WDOORBELL64(ring->doorbell_index, ring->wptr);  in gfx_v9_0_ring_set_wptr_gfx()
5339 		WREG32_SOC15(GC, 0, mmCP_RB0_WPTR, lower_32_bits(ring->wptr));  in gfx_v9_0_ring_set_wptr_gfx()
5340 		WREG32_SOC15(GC, 0, mmCP_RB0_WPTR_HI, upper_32_bits(ring->wptr));  in gfx_v9_0_ring_set_wptr_gfx()
5344 static void gfx_v9_0_ring_emit_hdp_flush(struct amdgpu_ring *ring)  in gfx_v9_0_ring_emit_hdp_flush()  argument
5346 	struct amdgpu_device *adev = ring->adev;  in gfx_v9_0_ring_emit_hdp_flush()
5348 	const struct nbio_hdp_flush_reg *nbio_hf_reg = adev->nbio.hdp_flush_reg;  in gfx_v9_0_ring_emit_hdp_flush()
5350 	if (ring->funcs->type == AMDGPU_RING_TYPE_COMPUTE) {  in gfx_v9_0_ring_emit_hdp_flush()
5351 		switch (ring->me) {  in gfx_v9_0_ring_emit_hdp_flush()
5353 			ref_and_mask = nbio_hf_reg->ref_and_mask_cp2 << ring->pipe;  in gfx_v9_0_ring_emit_hdp_flush()
5356 			ref_and_mask = nbio_hf_reg->ref_and_mask_cp6 << ring->pipe;  in gfx_v9_0_ring_emit_hdp_flush()
5363 		ref_and_mask = nbio_hf_reg->ref_and_mask_cp0;  in gfx_v9_0_ring_emit_hdp_flush()
5367 	gfx_v9_0_wait_reg_mem(ring, reg_mem_engine, 0, 1,  in gfx_v9_0_ring_emit_hdp_flush()
5368 			      adev->nbio.funcs->get_hdp_flush_req_offset(adev),  in gfx_v9_0_ring_emit_hdp_flush()
5369 			      adev->nbio.funcs->get_hdp_flush_done_offset(adev),  in gfx_v9_0_ring_emit_hdp_flush()
5373 static void gfx_v9_0_ring_emit_ib_gfx(struct amdgpu_ring *ring,  in gfx_v9_0_ring_emit_ib_gfx()  argument
5381 	if (ib->flags & AMDGPU_IB_FLAG_CE)  in gfx_v9_0_ring_emit_ib_gfx()
5386 	control |= ib->length_dw | (vmid << 24);  in gfx_v9_0_ring_emit_ib_gfx()
5388 	if (ib->flags & AMDGPU_IB_FLAG_PREEMPT) {  in gfx_v9_0_ring_emit_ib_gfx()
5394 		if (!(ib->flags & AMDGPU_IB_FLAG_CE) && vmid)  in gfx_v9_0_ring_emit_ib_gfx()
5395 			gfx_v9_0_ring_emit_de_meta(ring,  in gfx_v9_0_ring_emit_ib_gfx()
5396 						   (!amdgpu_sriov_vf(ring->adev) &&  in gfx_v9_0_ring_emit_ib_gfx()
5399 						   job->gds_size > 0 && job->gds_base != 0);  in gfx_v9_0_ring_emit_ib_gfx()
5402 	amdgpu_ring_write(ring, header);  in gfx_v9_0_ring_emit_ib_gfx()
5403 	BUG_ON(ib->gpu_addr & 0x3); /* Dword align */  in gfx_v9_0_ring_emit_ib_gfx()
5404 	amdgpu_ring_write(ring,  in gfx_v9_0_ring_emit_ib_gfx()
5408 		lower_32_bits(ib->gpu_addr));  in gfx_v9_0_ring_emit_ib_gfx()
5409 	amdgpu_ring_write(ring, upper_32_bits(ib->gpu_addr));  in gfx_v9_0_ring_emit_ib_gfx()
5410 	amdgpu_ring_ib_on_emit_cntl(ring);  in gfx_v9_0_ring_emit_ib_gfx()
5411 	amdgpu_ring_write(ring, control);  in gfx_v9_0_ring_emit_ib_gfx()
5414 static void gfx_v9_0_ring_patch_cntl(struct amdgpu_ring *ring,  in gfx_v9_0_ring_patch_cntl()  argument
5417 	u32 control = ring->ring[offset];  in gfx_v9_0_ring_patch_cntl()
5420 	ring->ring[offset] = control;  in gfx_v9_0_ring_patch_cntl()
5423 static void gfx_v9_0_ring_patch_ce_meta(struct amdgpu_ring *ring,  in gfx_v9_0_ring_patch_ce_meta()  argument
5426 	struct amdgpu_device *adev = ring->adev;  in gfx_v9_0_ring_patch_ce_meta()
5432 	if (ring->is_mes_queue) {  in gfx_v9_0_ring_patch_ce_meta()
5437 			amdgpu_mes_ctx_get_offs_cpu_addr(ring, payload_offset);  in gfx_v9_0_ring_patch_ce_meta()
5440 		ce_payload_cpu_addr = adev->virt.csa_cpu_addr + payload_offset;  in gfx_v9_0_ring_patch_ce_meta()
5443 	if (offset + (payload_size >> 2) <= ring->buf_mask + 1) {  in gfx_v9_0_ring_patch_ce_meta()
5444 		memcpy((void *)&ring->ring[offset], ce_payload_cpu_addr, payload_size);  in gfx_v9_0_ring_patch_ce_meta()
5446 		memcpy((void *)&ring->ring[offset], ce_payload_cpu_addr,  in gfx_v9_0_ring_patch_ce_meta()
5447 		       (ring->buf_mask + 1 - offset) << 2);  in gfx_v9_0_ring_patch_ce_meta()
5448 		payload_size -= (ring->buf_mask + 1 - offset) << 2;  in gfx_v9_0_ring_patch_ce_meta()
5449 		memcpy((void *)&ring->ring[0],  in gfx_v9_0_ring_patch_ce_meta()
5450 		       ce_payload_cpu_addr + ((ring->buf_mask + 1 - offset) << 2),  in gfx_v9_0_ring_patch_ce_meta()
5455 static void gfx_v9_0_ring_patch_de_meta(struct amdgpu_ring *ring,  in gfx_v9_0_ring_patch_de_meta()  argument
5458 	struct amdgpu_device *adev = ring->adev;  in gfx_v9_0_ring_patch_de_meta()
5464 	if (ring->is_mes_queue) {  in gfx_v9_0_ring_patch_de_meta()
5469 			amdgpu_mes_ctx_get_offs_cpu_addr(ring, payload_offset);  in gfx_v9_0_ring_patch_de_meta()
5472 		de_payload_cpu_addr = adev->virt.csa_cpu_addr + payload_offset;  in gfx_v9_0_ring_patch_de_meta()
5475 	((struct v9_de_ib_state *)de_payload_cpu_addr)->ib_completion_status =  in gfx_v9_0_ring_patch_de_meta()
5478 	if (offset + (payload_size >> 2) <= ring->buf_mask + 1) {  in gfx_v9_0_ring_patch_de_meta()
5479 		memcpy((void *)&ring->ring[offset], de_payload_cpu_addr, payload_size);  in gfx_v9_0_ring_patch_de_meta()
5481 		memcpy((void *)&ring->ring[offset], de_payload_cpu_addr,  in gfx_v9_0_ring_patch_de_meta()
5482 		       (ring->buf_mask + 1 - offset) << 2);  in gfx_v9_0_ring_patch_de_meta()
5483 		payload_size -= (ring->buf_mask + 1 - offset) << 2;  in gfx_v9_0_ring_patch_de_meta()
5484 		memcpy((void *)&ring->ring[0],  in gfx_v9_0_ring_patch_de_meta()
5485 		       de_payload_cpu_addr + ((ring->buf_mask + 1 - offset) << 2),  in gfx_v9_0_ring_patch_de_meta()
5490 static void gfx_v9_0_ring_emit_ib_compute(struct amdgpu_ring *ring,  in gfx_v9_0_ring_emit_ib_compute()  argument
5496 	u32 control = INDIRECT_BUFFER_VALID | ib->length_dw | (vmid << 24);  in gfx_v9_0_ring_emit_ib_compute()
5506 	 * GDS to 0 for this ring (me/pipe).  in gfx_v9_0_ring_emit_ib_compute()
5508 	if (ib->flags & AMDGPU_IB_FLAG_RESET_GDS_MAX_WAVE_ID) {  in gfx_v9_0_ring_emit_ib_compute()
5509 		amdgpu_ring_write(ring, PACKET3(PACKET3_SET_CONFIG_REG, 1));  in gfx_v9_0_ring_emit_ib_compute()
5510 		amdgpu_ring_write(ring, mmGDS_COMPUTE_MAX_WAVE_ID);  in gfx_v9_0_ring_emit_ib_compute()
5511 		amdgpu_ring_write(ring, ring->adev->gds.gds_compute_max_wave_id);  in gfx_v9_0_ring_emit_ib_compute()
5514 	amdgpu_ring_write(ring, PACKET3(PACKET3_INDIRECT_BUFFER, 2));  in gfx_v9_0_ring_emit_ib_compute()
5515 	BUG_ON(ib->gpu_addr & 0x3); /* Dword align */  in gfx_v9_0_ring_emit_ib_compute()
5516 	amdgpu_ring_write(ring,  in gfx_v9_0_ring_emit_ib_compute()
5520 				lower_32_bits(ib->gpu_addr));  in gfx_v9_0_ring_emit_ib_compute()
5521 	amdgpu_ring_write(ring, upper_32_bits(ib->gpu_addr));  in gfx_v9_0_ring_emit_ib_compute()
5522 	amdgpu_ring_write(ring, control);  in gfx_v9_0_ring_emit_ib_compute()
5525 static void gfx_v9_0_ring_emit_fence(struct amdgpu_ring *ring, u64 addr,  in gfx_v9_0_ring_emit_fence()  argument
5534 	/* RELEASE_MEM - flush caches, send int */  in gfx_v9_0_ring_emit_fence()
5535 	amdgpu_ring_write(ring, PACKET3(PACKET3_RELEASE_MEM, 6));  in gfx_v9_0_ring_emit_fence()
5548 	amdgpu_ring_write(ring, dw2);  in gfx_v9_0_ring_emit_fence()
5549 	amdgpu_ring_write(ring, DATA_SEL(write64bit ? 2 : 1) | INT_SEL(int_sel ? 2 : 0));  in gfx_v9_0_ring_emit_fence()
5559 	amdgpu_ring_write(ring, lower_32_bits(addr));  in gfx_v9_0_ring_emit_fence()
5560 	amdgpu_ring_write(ring, upper_32_bits(addr));  in gfx_v9_0_ring_emit_fence()
5561 	amdgpu_ring_write(ring, lower_32_bits(seq));  in gfx_v9_0_ring_emit_fence()
5562 	amdgpu_ring_write(ring, upper_32_bits(seq));  in gfx_v9_0_ring_emit_fence()
5563 	amdgpu_ring_write(ring, 0);  in gfx_v9_0_ring_emit_fence()
5566 static void gfx_v9_0_ring_emit_pipeline_sync(struct amdgpu_ring *ring)  in gfx_v9_0_ring_emit_pipeline_sync()  argument
5568 	int usepfp = (ring->funcs->type == AMDGPU_RING_TYPE_GFX);  in gfx_v9_0_ring_emit_pipeline_sync()
5569 	uint32_t seq = ring->fence_drv.sync_seq;  in gfx_v9_0_ring_emit_pipeline_sync()
5570 	uint64_t addr = ring->fence_drv.gpu_addr;  in gfx_v9_0_ring_emit_pipeline_sync()
5572 	gfx_v9_0_wait_reg_mem(ring, usepfp, 1, 0,  in gfx_v9_0_ring_emit_pipeline_sync()
5574 			      seq, 0xffffffff, 4);  in gfx_v9_0_ring_emit_pipeline_sync()
5577 static void gfx_v9_0_ring_emit_vm_flush(struct amdgpu_ring *ring,  in gfx_v9_0_ring_emit_vm_flush()  argument
5580 	amdgpu_gmc_emit_flush_gpu_tlb(ring, vmid, pd_addr);  in gfx_v9_0_ring_emit_vm_flush()
5583 	if (ring->funcs->type == AMDGPU_RING_TYPE_GFX) {  in gfx_v9_0_ring_emit_vm_flush()
5585 		amdgpu_ring_write(ring, PACKET3(PACKET3_PFP_SYNC_ME, 0));  in gfx_v9_0_ring_emit_vm_flush()
5586 		amdgpu_ring_write(ring, 0x0);  in gfx_v9_0_ring_emit_vm_flush()
5590 static u64 gfx_v9_0_ring_get_rptr_compute(struct amdgpu_ring *ring)  in gfx_v9_0_ring_get_rptr_compute()  argument
5592 	return *ring->rptr_cpu_addr; /* gfx9 hardware is 32bit rptr */  in gfx_v9_0_ring_get_rptr_compute()
5595 static u64 gfx_v9_0_ring_get_wptr_compute(struct amdgpu_ring *ring)  in gfx_v9_0_ring_get_wptr_compute()  argument
5600 	if (ring->use_doorbell)  in gfx_v9_0_ring_get_wptr_compute()
5601 		wptr = atomic64_read((atomic64_t *)ring->wptr_cpu_addr);  in gfx_v9_0_ring_get_wptr_compute()
5607 static void gfx_v9_0_ring_set_wptr_compute(struct amdgpu_ring *ring)  in gfx_v9_0_ring_set_wptr_compute()  argument
5609 	struct amdgpu_device *adev = ring->adev;  in gfx_v9_0_ring_set_wptr_compute()
5612 	if (ring->use_doorbell) {  in gfx_v9_0_ring_set_wptr_compute()
5613 		atomic64_set((atomic64_t *)ring->wptr_cpu_addr, ring->wptr);  in gfx_v9_0_ring_set_wptr_compute()
5614 		WDOORBELL64(ring->doorbell_index, ring->wptr);  in gfx_v9_0_ring_set_wptr_compute()
5620 static void gfx_v9_0_ring_emit_fence_kiq(struct amdgpu_ring *ring, u64 addr,  in gfx_v9_0_ring_emit_fence_kiq()  argument
5623 	struct amdgpu_device *adev = ring->adev;  in gfx_v9_0_ring_emit_fence_kiq()
5629 	amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));  in gfx_v9_0_ring_emit_fence_kiq()
5630 	amdgpu_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) |  in gfx_v9_0_ring_emit_fence_kiq()
5632 	amdgpu_ring_write(ring, lower_32_bits(addr));  in gfx_v9_0_ring_emit_fence_kiq()
5633 	amdgpu_ring_write(ring, upper_32_bits(addr));  in gfx_v9_0_ring_emit_fence_kiq()
5634 	amdgpu_ring_write(ring, lower_32_bits(seq));  in gfx_v9_0_ring_emit_fence_kiq()
5638 		amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));  in gfx_v9_0_ring_emit_fence_kiq()
5639 		amdgpu_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) |  in gfx_v9_0_ring_emit_fence_kiq()
5641 		amdgpu_ring_write(ring, SOC15_REG_OFFSET(GC, 0, mmCPC_INT_STATUS));  in gfx_v9_0_ring_emit_fence_kiq()
5642 		amdgpu_ring_write(ring, 0);  in gfx_v9_0_ring_emit_fence_kiq()
5643 		amdgpu_ring_write(ring, 0x20000000); /* src_id is 178 */  in gfx_v9_0_ring_emit_fence_kiq()
5647 static void gfx_v9_ring_emit_sb(struct amdgpu_ring *ring)  in gfx_v9_ring_emit_sb()  argument
5649 	amdgpu_ring_write(ring, PACKET3(PACKET3_SWITCH_BUFFER, 0));  in gfx_v9_ring_emit_sb()
5650 	amdgpu_ring_write(ring, 0);  in gfx_v9_ring_emit_sb()
5653 static void gfx_v9_0_ring_emit_ce_meta(struct amdgpu_ring *ring, bool resume)  in gfx_v9_0_ring_emit_ce_meta()  argument
5655 	struct amdgpu_device *adev = ring->adev;  in gfx_v9_0_ring_emit_ce_meta()
5661 	cnt = (sizeof(ce_payload) >> 2) + 4 - 2;  in gfx_v9_0_ring_emit_ce_meta()
5663 	if (ring->is_mes_queue) {  in gfx_v9_0_ring_emit_ce_meta()
5668 			amdgpu_mes_ctx_get_offs_gpu_addr(ring, offset);  in gfx_v9_0_ring_emit_ce_meta()
5670 			amdgpu_mes_ctx_get_offs_cpu_addr(ring, offset);  in gfx_v9_0_ring_emit_ce_meta()
5673 		ce_payload_gpu_addr = amdgpu_csa_vaddr(ring->adev) + offset;  in gfx_v9_0_ring_emit_ce_meta()
5674 		ce_payload_cpu_addr = adev->virt.csa_cpu_addr + offset;  in gfx_v9_0_ring_emit_ce_meta()
5677 	amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, cnt));  in gfx_v9_0_ring_emit_ce_meta()
5678 	amdgpu_ring_write(ring, (WRITE_DATA_ENGINE_SEL(2) |  in gfx_v9_0_ring_emit_ce_meta()
5682 	amdgpu_ring_write(ring, lower_32_bits(ce_payload_gpu_addr));  in gfx_v9_0_ring_emit_ce_meta()
5683 	amdgpu_ring_write(ring, upper_32_bits(ce_payload_gpu_addr));  in gfx_v9_0_ring_emit_ce_meta()
5685 	amdgpu_ring_ib_on_emit_ce(ring);  in gfx_v9_0_ring_emit_ce_meta()
5688 		amdgpu_ring_write_multiple(ring, ce_payload_cpu_addr,  in gfx_v9_0_ring_emit_ce_meta()
5691 		amdgpu_ring_write_multiple(ring, (void *)&ce_payload,  in gfx_v9_0_ring_emit_ce_meta()
5695 static int gfx_v9_0_ring_preempt_ib(struct amdgpu_ring *ring)  in gfx_v9_0_ring_preempt_ib()  argument
5698 	struct amdgpu_device *adev = ring->adev;  in gfx_v9_0_ring_preempt_ib()
5699 	struct amdgpu_kiq *kiq = &adev->gfx.kiq[0];  in gfx_v9_0_ring_preempt_ib()
5700 	struct amdgpu_ring *kiq_ring = &kiq->ring;  in gfx_v9_0_ring_preempt_ib()
5703 	if (!kiq->pmf || !kiq->pmf->kiq_unmap_queues)  in gfx_v9_0_ring_preempt_ib()
5704 		return -EINVAL;  in gfx_v9_0_ring_preempt_ib()
5706 	spin_lock_irqsave(&kiq->ring_lock, flags);  in gfx_v9_0_ring_preempt_ib()
5708 	if (amdgpu_ring_alloc(kiq_ring, kiq->pmf->unmap_queues_size)) {  in gfx_v9_0_ring_preempt_ib()
5709 		spin_unlock_irqrestore(&kiq->ring_lock, flags);  in gfx_v9_0_ring_preempt_ib()
5710 		return -ENOMEM;  in gfx_v9_0_ring_preempt_ib()
5714 	amdgpu_ring_set_preempt_cond_exec(ring, false);  in gfx_v9_0_ring_preempt_ib()
5716 	ring->trail_seq += 1;  in gfx_v9_0_ring_preempt_ib()
5717 	amdgpu_ring_alloc(ring, 13);  in gfx_v9_0_ring_preempt_ib()
5718 	gfx_v9_0_ring_emit_fence(ring, ring->trail_fence_gpu_addr,  in gfx_v9_0_ring_preempt_ib()
5719 				 ring->trail_seq, AMDGPU_FENCE_FLAG_EXEC | AMDGPU_FENCE_FLAG_INT);  in gfx_v9_0_ring_preempt_ib()
5722 	kiq->pmf->kiq_unmap_queues(kiq_ring, ring, PREEMPT_QUEUES_NO_UNMAP,  in gfx_v9_0_ring_preempt_ib()
5723 				   ring->trail_fence_gpu_addr,  in gfx_v9_0_ring_preempt_ib()
5724 				   ring->trail_seq);  in gfx_v9_0_ring_preempt_ib()
5727 	spin_unlock_irqrestore(&kiq->ring_lock, flags);  in gfx_v9_0_ring_preempt_ib()
5730 	for (i = 0; i < adev->usec_timeout; i++) {  in gfx_v9_0_ring_preempt_ib()
5731 		if (ring->trail_seq ==  in gfx_v9_0_ring_preempt_ib()
5732 			le32_to_cpu(*ring->trail_fence_cpu_addr))  in gfx_v9_0_ring_preempt_ib()
5737 	if (i >= adev->usec_timeout) {  in gfx_v9_0_ring_preempt_ib()
5738 		r = -EINVAL;  in gfx_v9_0_ring_preempt_ib()
5739 		DRM_WARN("ring %d timeout to preempt ib\n", ring->idx);  in gfx_v9_0_ring_preempt_ib()
5743 	amdgpu_ring_emit_wreg(ring,  in gfx_v9_0_ring_preempt_ib()
5746 	amdgpu_ring_commit(ring);  in gfx_v9_0_ring_preempt_ib()
5749 	amdgpu_ring_set_preempt_cond_exec(ring, true);  in gfx_v9_0_ring_preempt_ib()
5753 static void gfx_v9_0_ring_emit_de_meta(struct amdgpu_ring *ring, bool resume, bool usegds)  in gfx_v9_0_ring_emit_de_meta()  argument
5755 	struct amdgpu_device *adev = ring->adev;  in gfx_v9_0_ring_emit_de_meta()
5761 	if (ring->is_mes_queue) {  in gfx_v9_0_ring_emit_de_meta()
5766 			amdgpu_mes_ctx_get_offs_gpu_addr(ring, offset);  in gfx_v9_0_ring_emit_de_meta()
5768 			amdgpu_mes_ctx_get_offs_cpu_addr(ring, offset);  in gfx_v9_0_ring_emit_de_meta()
5773 		gds_addr = amdgpu_mes_ctx_get_offs_gpu_addr(ring, offset);  in gfx_v9_0_ring_emit_de_meta()
5776 		de_payload_gpu_addr = amdgpu_csa_vaddr(ring->adev) + offset;  in gfx_v9_0_ring_emit_de_meta()
5777 		de_payload_cpu_addr = adev->virt.csa_cpu_addr + offset;  in gfx_v9_0_ring_emit_de_meta()
5779 		gds_addr = ALIGN(amdgpu_csa_vaddr(ring->adev) +  in gfx_v9_0_ring_emit_de_meta()
5780 				 AMDGPU_CSA_SIZE - adev->gds.gds_size,  in gfx_v9_0_ring_emit_de_meta()
5789 	cnt = (sizeof(de_payload) >> 2) + 4 - 2;  in gfx_v9_0_ring_emit_de_meta()
5790 	amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, cnt));  in gfx_v9_0_ring_emit_de_meta()
5791 	amdgpu_ring_write(ring, (WRITE_DATA_ENGINE_SEL(1) |  in gfx_v9_0_ring_emit_de_meta()
5795 	amdgpu_ring_write(ring, lower_32_bits(de_payload_gpu_addr));  in gfx_v9_0_ring_emit_de_meta()
5796 	amdgpu_ring_write(ring, upper_32_bits(de_payload_gpu_addr));  in gfx_v9_0_ring_emit_de_meta()
5798 	amdgpu_ring_ib_on_emit_de(ring);  in gfx_v9_0_ring_emit_de_meta()
5800 		amdgpu_ring_write_multiple(ring, de_payload_cpu_addr,  in gfx_v9_0_ring_emit_de_meta()
5803 		amdgpu_ring_write_multiple(ring, (void *)&de_payload,  in gfx_v9_0_ring_emit_de_meta()
5807 static void gfx_v9_0_ring_emit_frame_cntl(struct amdgpu_ring *ring, bool start,  in gfx_v9_0_ring_emit_frame_cntl()  argument
5812 	amdgpu_ring_write(ring, PACKET3(PACKET3_FRAME_CONTROL, 0));  in gfx_v9_0_ring_emit_frame_cntl()
5813 	amdgpu_ring_write(ring, v | FRAME_CMD(start ? 0 : 1));  in gfx_v9_0_ring_emit_frame_cntl()
5816 static void gfx_v9_ring_emit_cntxcntl(struct amdgpu_ring *ring, uint32_t flags)  in gfx_v9_ring_emit_cntxcntl()  argument
5820 	gfx_v9_0_ring_emit_ce_meta(ring,  in gfx_v9_ring_emit_cntxcntl()
5821 				   (!amdgpu_sriov_vf(ring->adev) &&  in gfx_v9_ring_emit_cntxcntl()
5844 	amdgpu_ring_write(ring, PACKET3(PACKET3_CONTEXT_CONTROL, 1));  in gfx_v9_ring_emit_cntxcntl()
5845 	amdgpu_ring_write(ring, dw2);  in gfx_v9_ring_emit_cntxcntl()
5846 	amdgpu_ring_write(ring, 0);  in gfx_v9_ring_emit_cntxcntl()
5849 static unsigned gfx_v9_0_ring_emit_init_cond_exec(struct amdgpu_ring *ring,  in gfx_v9_0_ring_emit_init_cond_exec()  argument
5853 	amdgpu_ring_write(ring, PACKET3(PACKET3_COND_EXEC, 3));  in gfx_v9_0_ring_emit_init_cond_exec()
5854 	amdgpu_ring_write(ring, lower_32_bits(addr));  in gfx_v9_0_ring_emit_init_cond_exec()
5855 	amdgpu_ring_write(ring, upper_32_bits(addr));  in gfx_v9_0_ring_emit_init_cond_exec()
5857 	amdgpu_ring_write(ring, 0);  in gfx_v9_0_ring_emit_init_cond_exec()
5858 	ret = ring->wptr & ring->buf_mask;  in gfx_v9_0_ring_emit_init_cond_exec()
5860 	amdgpu_ring_write(ring, 0);  in gfx_v9_0_ring_emit_init_cond_exec()
5864 static void gfx_v9_0_ring_emit_rreg(struct amdgpu_ring *ring, uint32_t reg,  in gfx_v9_0_ring_emit_rreg()  argument
5867 	struct amdgpu_device *adev = ring->adev;  in gfx_v9_0_ring_emit_rreg()
5869 	amdgpu_ring_write(ring, PACKET3(PACKET3_COPY_DATA, 4));  in gfx_v9_0_ring_emit_rreg()
5870 	amdgpu_ring_write(ring, 0 |	/* src: register*/  in gfx_v9_0_ring_emit_rreg()
5873 	amdgpu_ring_write(ring, reg);  in gfx_v9_0_ring_emit_rreg()
5874 	amdgpu_ring_write(ring, 0);  in gfx_v9_0_ring_emit_rreg()
5875 	amdgpu_ring_write(ring, lower_32_bits(adev->wb.gpu_addr +  in gfx_v9_0_ring_emit_rreg()
5876 				reg_val_offs * 4));  in gfx_v9_0_ring_emit_rreg()
5877 	amdgpu_ring_write(ring, upper_32_bits(adev->wb.gpu_addr +  in gfx_v9_0_ring_emit_rreg()
5878 				reg_val_offs * 4));  in gfx_v9_0_ring_emit_rreg()
5881 static void gfx_v9_0_ring_emit_wreg(struct amdgpu_ring *ring, uint32_t reg,  in gfx_v9_0_ring_emit_wreg()  argument
5886 	switch (ring->funcs->type) {  in gfx_v9_0_ring_emit_wreg()
5897 	amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));  in gfx_v9_0_ring_emit_wreg()
5898 	amdgpu_ring_write(ring, cmd);  in gfx_v9_0_ring_emit_wreg()
5899 	amdgpu_ring_write(ring, reg);  in gfx_v9_0_ring_emit_wreg()
5900 	amdgpu_ring_write(ring, 0);  in gfx_v9_0_ring_emit_wreg()
5901 	amdgpu_ring_write(ring, val);  in gfx_v9_0_ring_emit_wreg()
5904 static void gfx_v9_0_ring_emit_reg_wait(struct amdgpu_ring *ring, uint32_t reg,  in gfx_v9_0_ring_emit_reg_wait()  argument
5907 	gfx_v9_0_wait_reg_mem(ring, 0, 0, 0, reg, 0, val, mask, 0x20);  in gfx_v9_0_ring_emit_reg_wait()
5910 static void gfx_v9_0_ring_emit_reg_write_reg_wait(struct amdgpu_ring *ring,  in gfx_v9_0_ring_emit_reg_write_reg_wait()  argument
5914 	int usepfp = (ring->funcs->type == AMDGPU_RING_TYPE_GFX);  in gfx_v9_0_ring_emit_reg_write_reg_wait()
5915 	struct amdgpu_device *adev = ring->adev;  in gfx_v9_0_ring_emit_reg_write_reg_wait()
5916 	bool fw_version_ok = (ring->funcs->type == AMDGPU_RING_TYPE_GFX) ?  in gfx_v9_0_ring_emit_reg_write_reg_wait()
5917 		adev->gfx.me_fw_write_wait : adev->gfx.mec_fw_write_wait;  in gfx_v9_0_ring_emit_reg_write_reg_wait()
5920 		gfx_v9_0_wait_reg_mem(ring, usepfp, 0, 1, reg0, reg1,  in gfx_v9_0_ring_emit_reg_write_reg_wait()
5923 		amdgpu_ring_emit_reg_write_reg_wait_helper(ring, reg0, reg1,  in gfx_v9_0_ring_emit_reg_write_reg_wait()
5927 static void gfx_v9_0_ring_soft_recovery(struct amdgpu_ring *ring, unsigned vmid)  in gfx_v9_0_ring_soft_recovery()  argument
5929 	struct amdgpu_device *adev = ring->adev;  in gfx_v9_0_ring_soft_recovery()
6048 		for (i = 0; i < adev->gfx.mec.num_mec; i++) {  in gfx_v9_0_set_priv_reg_fault_state()
6049 			for (j = 0; j < adev->gfx.mec.num_pipe_per_mec; j++) {  in gfx_v9_0_set_priv_reg_fault_state()
6084 		for (i = 0; i < adev->gfx.mec.num_mec; i++) {  in gfx_v9_0_set_bad_op_fault_state()
6085 			for (j = 0; j < adev->gfx.mec.num_pipe_per_mec; j++) {  in gfx_v9_0_set_bad_op_fault_state()
6209 	struct amdgpu_ring *ring;  in gfx_v9_0_eop_irq()  local
6212 	me_id = (entry->ring_id & 0x0c) >> 2;  in gfx_v9_0_eop_irq()
6213 	pipe_id = (entry->ring_id & 0x03) >> 0;  in gfx_v9_0_eop_irq()
6214 	queue_id = (entry->ring_id & 0x70) >> 4;  in gfx_v9_0_eop_irq()
6218 		if (adev->gfx.num_gfx_rings) {  in gfx_v9_0_eop_irq()
6219 			if (!adev->gfx.mcbp) {  in gfx_v9_0_eop_irq()
6220 				amdgpu_fence_process(&adev->gfx.gfx_ring[0]);  in gfx_v9_0_eop_irq()
6221 			} else if (!amdgpu_mcbp_handle_trailing_fence_irq(&adev->gfx.muxer)) {  in gfx_v9_0_eop_irq()
6224 					amdgpu_fence_process(&adev->gfx.sw_gfx_ring[i]);  in gfx_v9_0_eop_irq()
6230 		for (i = 0; i < adev->gfx.num_compute_rings; i++) {  in gfx_v9_0_eop_irq()
6231 			ring = &adev->gfx.compute_ring[i];  in gfx_v9_0_eop_irq()
6232 			/* Per-queue interrupt is supported for MEC starting from VI.  in gfx_v9_0_eop_irq()
6235 			if ((ring->me == me_id) && (ring->pipe == pipe_id) && (ring->queue == queue_id))  in gfx_v9_0_eop_irq()
6236 				amdgpu_fence_process(ring);  in gfx_v9_0_eop_irq()
6247 	struct amdgpu_ring *ring;  in gfx_v9_0_fault()  local
6250 	me_id = (entry->ring_id & 0x0c) >> 2;  in gfx_v9_0_fault()
6251 	pipe_id = (entry->ring_id & 0x03) >> 0;  in gfx_v9_0_fault()
6252 	queue_id = (entry->ring_id & 0x70) >> 4;  in gfx_v9_0_fault()
6256 		drm_sched_fault(&adev->gfx.gfx_ring[0].sched);  in gfx_v9_0_fault()
6260 		for (i = 0; i < adev->gfx.num_compute_rings; i++) {  in gfx_v9_0_fault()
6261 			ring = &adev->gfx.compute_ring[i];  in gfx_v9_0_fault()
6262 			if (ring->me == me_id && ring->pipe == pipe_id &&  in gfx_v9_0_fault()
6263 			    ring->queue == queue_id)  in gfx_v9_0_fault()
6264 				drm_sched_fault(&ring->sched);  in gfx_v9_0_fault()
6747 		return -EINVAL;  in gfx_v9_0_ras_error_inject()
6749 	if (info->head.sub_block_index >= ARRAY_SIZE(ras_gfx_subblocks))  in gfx_v9_0_ras_error_inject()
6750 		return -EINVAL;  in gfx_v9_0_ras_error_inject()
6752 	if (!ras_gfx_subblocks[info->head.sub_block_index].name)  in gfx_v9_0_ras_error_inject()
6753 		return -EPERM;  in gfx_v9_0_ras_error_inject()
6755 	if (!(ras_gfx_subblocks[info->head.sub_block_index].hw_supported_error_type &  in gfx_v9_0_ras_error_inject()
6756 	      info->head.type)) {  in gfx_v9_0_ras_error_inject()
6758 			ras_gfx_subblocks[info->head.sub_block_index].name,  in gfx_v9_0_ras_error_inject()
6759 			info->head.type);  in gfx_v9_0_ras_error_inject()
6760 		return -EPERM;  in gfx_v9_0_ras_error_inject()
6763 	if (!(ras_gfx_subblocks[info->head.sub_block_index].sw_supported_error_type &  in gfx_v9_0_ras_error_inject()
6764 	      info->head.type)) {  in gfx_v9_0_ras_error_inject()
6766 			ras_gfx_subblocks[info->head.sub_block_index].name,  in gfx_v9_0_ras_error_inject()
6767 			info->head.type);  in gfx_v9_0_ras_error_inject()
6768 		return -EPERM;  in gfx_v9_0_ras_error_inject()
6771 	block_info.block_id = amdgpu_ras_block_to_ta(info->head.block);  in gfx_v9_0_ras_error_inject()
6773 		ras_gfx_subblocks[info->head.sub_block_index].ta_subblock;  in gfx_v9_0_ras_error_inject()
6774 	block_info.inject_error_type = amdgpu_ras_error_to_ta(info->head.type);  in gfx_v9_0_ras_error_inject()
6775 	block_info.address = info->address;  in gfx_v9_0_ras_error_inject()
6776 	block_info.value = info->value;  in gfx_v9_0_ras_error_inject()
6778 	mutex_lock(&adev->grbm_idx_mutex);  in gfx_v9_0_ras_error_inject()
6779 	ret = psp_ras_trigger_error(&adev->psp, &block_info, instance_mask);  in gfx_v9_0_ras_error_inject()
6780 	mutex_unlock(&adev->grbm_idx_mutex);  in gfx_v9_0_ras_error_inject()
6877 			dev_info(adev->dev, "Instance[%d]: SubBlock %s, "  in gfx_v9_0_query_utc_edc_status()
6879 			err_data->ce_count += sec_count;  in gfx_v9_0_query_utc_edc_status()
6884 			dev_info(adev->dev, "Instance[%d]: SubBlock %s, "  in gfx_v9_0_query_utc_edc_status()
6886 			err_data->ue_count += ded_count;  in gfx_v9_0_query_utc_edc_status()
6897 			dev_info(adev->dev, "Instance[%d]: SubBlock %s, "  in gfx_v9_0_query_utc_edc_status()
6899 			err_data->ce_count += sec_count;  in gfx_v9_0_query_utc_edc_status()
6905 			dev_info(adev->dev, "Instance[%d]: SubBlock %s, "  in gfx_v9_0_query_utc_edc_status()
6907 			err_data->ue_count += ded_count;  in gfx_v9_0_query_utc_edc_status()
6917 			dev_info(adev->dev, "Instance[%d]: SubBlock %s, "  in gfx_v9_0_query_utc_edc_status()
6920 			err_data->ce_count += sec_count;  in gfx_v9_0_query_utc_edc_status()
6930 			dev_info(adev->dev, "Instance[%d]: SubBlock %s, "  in gfx_v9_0_query_utc_edc_status()
6933 			err_data->ce_count += sec_count;  in gfx_v9_0_query_utc_edc_status()
6938 			dev_info(adev->dev, "Instance[%d]: SubBlock %s, "  in gfx_v9_0_query_utc_edc_status()
6941 			err_data->ue_count += ded_count;  in gfx_v9_0_query_utc_edc_status()
6962 		if(gfx_v9_0_ras_fields[i].reg_offset != reg->reg_offset ||  in gfx_v9_0_ras_error_count()
6963 			gfx_v9_0_ras_fields[i].seg != reg->seg ||  in gfx_v9_0_ras_error_count()
6964 			gfx_v9_0_ras_fields[i].inst != reg->inst)  in gfx_v9_0_ras_error_count()
6971 			dev_info(adev->dev, "GFX SubBlock %s, "  in gfx_v9_0_ras_error_count()
6983 			dev_info(adev->dev, "GFX SubBlock %s, "  in gfx_v9_0_ras_error_count()
7003 	mutex_lock(&adev->grbm_idx_mutex);  in gfx_v9_0_reset_ras_error_count()
7013 	mutex_unlock(&adev->grbm_idx_mutex);  in gfx_v9_0_reset_ras_error_count()
7061 	err_data->ue_count = 0;  in gfx_v9_0_query_ras_error_count()
7062 	err_data->ce_count = 0;  in gfx_v9_0_query_ras_error_count()
7064 	mutex_lock(&adev->grbm_idx_mutex);  in gfx_v9_0_query_ras_error_count()
7081 	err_data->ce_count += sec_count;  in gfx_v9_0_query_ras_error_count()
7082 	err_data->ue_count += ded_count;  in gfx_v9_0_query_ras_error_count()
7085 	mutex_unlock(&adev->grbm_idx_mutex);  in gfx_v9_0_query_ras_error_count()
7090 static void gfx_v9_0_emit_mem_sync(struct amdgpu_ring *ring)  in gfx_v9_0_emit_mem_sync()  argument
7099 	/* ACQUIRE_MEM -make one or more surfaces valid for use by the subsequent operations */  in gfx_v9_0_emit_mem_sync()
7100 	amdgpu_ring_write(ring, PACKET3(PACKET3_ACQUIRE_MEM, 5));  in gfx_v9_0_emit_mem_sync()
7101 	amdgpu_ring_write(ring, cp_coher_cntl); /* CP_COHER_CNTL */  in gfx_v9_0_emit_mem_sync()
7102 	amdgpu_ring_write(ring, 0xffffffff);  /* CP_COHER_SIZE */  in gfx_v9_0_emit_mem_sync()
7103 	amdgpu_ring_write(ring, 0xffffff);  /* CP_COHER_SIZE_HI */  in gfx_v9_0_emit_mem_sync()
7104 	amdgpu_ring_write(ring, 0); /* CP_COHER_BASE */  in gfx_v9_0_emit_mem_sync()
7105 	amdgpu_ring_write(ring, 0);  /* CP_COHER_BASE_HI */  in gfx_v9_0_emit_mem_sync()
7106 	amdgpu_ring_write(ring, 0x0000000A); /* POLL_INTERVAL */  in gfx_v9_0_emit_mem_sync()
7109 static void gfx_v9_0_emit_wave_limit_cs(struct amdgpu_ring *ring,  in gfx_v9_0_emit_wave_limit_cs()  argument
7112 	struct amdgpu_device *adev = ring->adev;  in gfx_v9_0_emit_wave_limit_cs()
7116 	/* mmSPI_WCL_PIPE_PERCENT_CS[0-7]_DEFAULT values are same */  in gfx_v9_0_emit_wave_limit_cs()
7137 	amdgpu_ring_emit_wreg(ring, wcl_cs_reg, val);  in gfx_v9_0_emit_wave_limit_cs()
7140 static void gfx_v9_0_emit_wave_limit(struct amdgpu_ring *ring, bool enable)  in gfx_v9_0_emit_wave_limit()  argument
7142 	struct amdgpu_device *adev = ring->adev;  in gfx_v9_0_emit_wave_limit()
7152 	amdgpu_ring_emit_wreg(ring,  in gfx_v9_0_emit_wave_limit()
7159 	 * amdgpu controls only 1st ME(0-3 CS pipes).  in gfx_v9_0_emit_wave_limit()
7161 	for (i = 0; i < adev->gfx.mec.num_pipe_per_mec; i++) {  in gfx_v9_0_emit_wave_limit()
7162 		if (i != ring->pipe)  in gfx_v9_0_emit_wave_limit()
7163 			gfx_v9_0_emit_wave_limit_cs(ring, i, enable);  in gfx_v9_0_emit_wave_limit()
7168 static void gfx_v9_ring_insert_nop(struct amdgpu_ring *ring, uint32_t num_nop)  in gfx_v9_ring_insert_nop()  argument
7174 		amdgpu_ring_write(ring, ring->funcs->nop);  in gfx_v9_ring_insert_nop()
7179 	amdgpu_ring_write(ring, PACKET3(PACKET3_NOP, min(num_nop - 2, 0x3ffe)));  in gfx_v9_ring_insert_nop()
7181 	/* Header is at index 0, followed by num_nops - 1 NOP packet's */  in gfx_v9_ring_insert_nop()
7183 		amdgpu_ring_write(ring, ring->funcs->nop);  in gfx_v9_ring_insert_nop()
7186 static int gfx_v9_0_reset_kgq(struct amdgpu_ring *ring, unsigned int vmid)  in gfx_v9_0_reset_kgq()  argument
7188 	struct amdgpu_device *adev = ring->adev;  in gfx_v9_0_reset_kgq()
7189 	struct amdgpu_kiq *kiq = &adev->gfx.kiq[0];  in gfx_v9_0_reset_kgq()
7190 	struct amdgpu_ring *kiq_ring = &kiq->ring;  in gfx_v9_0_reset_kgq()
7196 		return -EINVAL;  in gfx_v9_0_reset_kgq()
7198 	if (!kiq->pmf || !kiq->pmf->kiq_unmap_queues)  in gfx_v9_0_reset_kgq()
7199 		return -EINVAL;  in gfx_v9_0_reset_kgq()
7201 	spin_lock_irqsave(&kiq->ring_lock, flags);  in gfx_v9_0_reset_kgq()
7204 		spin_unlock_irqrestore(&kiq->ring_lock, flags);  in gfx_v9_0_reset_kgq()
7205 		return -ENOMEM;  in gfx_v9_0_reset_kgq()
7213 	spin_unlock_irqrestore(&kiq->ring_lock, flags);  in gfx_v9_0_reset_kgq()
7219 	if (amdgpu_ring_alloc(ring, 7 + 7 + 5))  in gfx_v9_0_reset_kgq()
7220 		return -ENOMEM;  in gfx_v9_0_reset_kgq()
7221 	gfx_v9_0_ring_emit_fence(ring, ring->fence_drv.gpu_addr,  in gfx_v9_0_reset_kgq()
7222 				 ring->fence_drv.sync_seq, AMDGPU_FENCE_FLAG_EXEC);  in gfx_v9_0_reset_kgq()
7223 	gfx_v9_0_ring_emit_reg_wait(ring,  in gfx_v9_0_reset_kgq()
7225 	gfx_v9_0_ring_emit_wreg(ring,  in gfx_v9_0_reset_kgq()
7228 	return amdgpu_ring_test_ring(ring);  in gfx_v9_0_reset_kgq()
7231 static int gfx_v9_0_reset_kcq(struct amdgpu_ring *ring,  in gfx_v9_0_reset_kcq()  argument
7234 	struct amdgpu_device *adev = ring->adev;  in gfx_v9_0_reset_kcq()
7235 	struct amdgpu_kiq *kiq = &adev->gfx.kiq[0];  in gfx_v9_0_reset_kcq()
7236 	struct amdgpu_ring *kiq_ring = &kiq->ring;  in gfx_v9_0_reset_kcq()
7240 	if (!adev->debug_exp_resets &&  in gfx_v9_0_reset_kcq()
7241 	    !adev->gfx.num_gfx_rings)  in gfx_v9_0_reset_kcq()
7242 		return -EINVAL;  in gfx_v9_0_reset_kcq()
7245 		return -EINVAL;  in gfx_v9_0_reset_kcq()
7247 	if (!kiq->pmf || !kiq->pmf->kiq_unmap_queues)  in gfx_v9_0_reset_kcq()
7248 		return -EINVAL;  in gfx_v9_0_reset_kcq()
7250 	spin_lock_irqsave(&kiq->ring_lock, flags);  in gfx_v9_0_reset_kcq()
7252 	if (amdgpu_ring_alloc(kiq_ring, kiq->pmf->unmap_queues_size)) {  in gfx_v9_0_reset_kcq()
7253 		spin_unlock_irqrestore(&kiq->ring_lock, flags);  in gfx_v9_0_reset_kcq()
7254 		return -ENOMEM;  in gfx_v9_0_reset_kcq()
7257 	kiq->pmf->kiq_unmap_queues(kiq_ring, ring, RESET_QUEUES,  in gfx_v9_0_reset_kcq()
7261 	spin_unlock_irqrestore(&kiq->ring_lock, flags);  in gfx_v9_0_reset_kcq()
7269 	mutex_lock(&adev->srbm_mutex);  in gfx_v9_0_reset_kcq()
7270 	soc15_grbm_select(adev, ring->me, ring->pipe, ring->queue, 0, 0);  in gfx_v9_0_reset_kcq()
7271 	for (i = 0; i < adev->usec_timeout; i++) {  in gfx_v9_0_reset_kcq()
7276 	if (i >= adev->usec_timeout)  in gfx_v9_0_reset_kcq()
7277 		r = -ETIMEDOUT;  in gfx_v9_0_reset_kcq()
7279 	mutex_unlock(&adev->srbm_mutex);  in gfx_v9_0_reset_kcq()
7282 		dev_err(adev->dev, "fail to wait on hqd deactive\n");  in gfx_v9_0_reset_kcq()
7286 	r = amdgpu_bo_reserve(ring->mqd_obj, false);  in gfx_v9_0_reset_kcq()
7288 		dev_err(adev->dev, "fail to resv mqd_obj\n");  in gfx_v9_0_reset_kcq()
7291 	r = amdgpu_bo_kmap(ring->mqd_obj, (void **)&ring->mqd_ptr);  in gfx_v9_0_reset_kcq()
7293 		r = gfx_v9_0_kcq_init_queue(ring, true);  in gfx_v9_0_reset_kcq()
7294 		amdgpu_bo_kunmap(ring->mqd_obj);  in gfx_v9_0_reset_kcq()
7295 		ring->mqd_ptr = NULL;  in gfx_v9_0_reset_kcq()
7297 	amdgpu_bo_unreserve(ring->mqd_obj);  in gfx_v9_0_reset_kcq()
7299 		dev_err(adev->dev, "fail to unresv mqd_obj\n");  in gfx_v9_0_reset_kcq()
7302 	spin_lock_irqsave(&kiq->ring_lock, flags);  in gfx_v9_0_reset_kcq()
7303 	r = amdgpu_ring_alloc(kiq_ring, kiq->pmf->map_queues_size);  in gfx_v9_0_reset_kcq()
7305 		spin_unlock_irqrestore(&kiq->ring_lock, flags);  in gfx_v9_0_reset_kcq()
7306 		return -ENOMEM;  in gfx_v9_0_reset_kcq()
7308 	kiq->pmf->kiq_map_queues(kiq_ring, ring);  in gfx_v9_0_reset_kcq()
7310 	spin_unlock_irqrestore(&kiq->ring_lock, flags);  in gfx_v9_0_reset_kcq()
7316 	return amdgpu_ring_test_ring(ring);  in gfx_v9_0_reset_kcq()
7325 	if (!adev->gfx.ip_dump_core)  in gfx_v9_ip_print()
7329 		drm_printf(p, "%-50s \t 0x%08x\n",  in gfx_v9_ip_print()
7331 			   adev->gfx.ip_dump_core[i]);  in gfx_v9_ip_print()
7334 	if (!adev->gfx.ip_dump_compute_queues)  in gfx_v9_ip_print()
7339 		   adev->gfx.mec.num_mec,  in gfx_v9_ip_print()
7340 		   adev->gfx.mec.num_pipe_per_mec,  in gfx_v9_ip_print()
7341 		   adev->gfx.mec.num_queue_per_pipe);  in gfx_v9_ip_print()
7343 	for (i = 0; i < adev->gfx.mec.num_mec; i++) {  in gfx_v9_ip_print()
7344 		for (j = 0; j < adev->gfx.mec.num_pipe_per_mec; j++) {  in gfx_v9_ip_print()
7345 			for (k = 0; k < adev->gfx.mec.num_queue_per_pipe; k++) {  in gfx_v9_ip_print()
7348 					drm_printf(p, "%-50s \t 0x%08x\n",  in gfx_v9_ip_print()
7350 						   adev->gfx.ip_dump_compute_queues[index + reg]);  in gfx_v9_ip_print()
7365 	if (!adev->gfx.ip_dump_core || !adev->gfx.num_gfx_rings)  in gfx_v9_ip_dump()
7370 		adev->gfx.ip_dump_core[i] = RREG32(SOC15_REG_ENTRY_OFFSET(gc_reg_list_9[i]));  in gfx_v9_ip_dump()
7374 	if (!adev->gfx.ip_dump_compute_queues)  in gfx_v9_ip_dump()
7379 	mutex_lock(&adev->srbm_mutex);  in gfx_v9_ip_dump()
7380 	for (i = 0; i < adev->gfx.mec.num_mec; i++) {  in gfx_v9_ip_dump()
7381 		for (j = 0; j < adev->gfx.mec.num_pipe_per_mec; j++) {  in gfx_v9_ip_dump()
7382 			for (k = 0; k < adev->gfx.mec.num_queue_per_pipe; k++) {  in gfx_v9_ip_dump()
7387 					adev->gfx.ip_dump_compute_queues[index + reg] =  in gfx_v9_ip_dump()
7396 	mutex_unlock(&adev->srbm_mutex);  in gfx_v9_ip_dump()
7401 static void gfx_v9_0_ring_emit_cleaner_shader(struct amdgpu_ring *ring)  in gfx_v9_0_ring_emit_cleaner_shader()  argument
7404 	amdgpu_ring_write(ring, PACKET3(PACKET3_RUN_CLEANER_SHADER, 0));  in gfx_v9_0_ring_emit_cleaner_shader()
7405 	amdgpu_ring_write(ring, 0);  /* RESERVED field, programmed to zero */  in gfx_v9_0_ring_emit_cleaner_shader()
7445 		4 + /* double SWITCH_BUFFER,
7450 		4 +	 /*	VGT_flush */
7459 	.emit_ib_size =	4, /* gfx_v9_0_ring_emit_ib_gfx */
7502 		4 + /* double SWITCH_BUFFER,
7508 		4 +	 /*	VGT_flush */
7517 	.emit_ib_size =	4, /* gfx_v9_0_ring_emit_ib_gfx */
7619 	adev->gfx.kiq[0].ring.funcs = &gfx_v9_0_ring_funcs_kiq;  in gfx_v9_0_set_ring_funcs()
7621 	for (i = 0; i < adev->gfx.num_gfx_rings; i++)  in gfx_v9_0_set_ring_funcs()
7622 		adev->gfx.gfx_ring[i].funcs = &gfx_v9_0_ring_funcs_gfx;  in gfx_v9_0_set_ring_funcs()
7624 	if (adev->gfx.mcbp && adev->gfx.num_gfx_rings) {  in gfx_v9_0_set_ring_funcs()
7626 			adev->gfx.sw_gfx_ring[i].funcs = &gfx_v9_0_sw_ring_funcs_gfx;  in gfx_v9_0_set_ring_funcs()
7629 	for (i = 0; i < adev->gfx.num_compute_rings; i++)  in gfx_v9_0_set_ring_funcs()
7630 		adev->gfx.compute_ring[i].funcs = &gfx_v9_0_ring_funcs_compute;  in gfx_v9_0_set_ring_funcs()
7661 	adev->gfx.eop_irq.num_types = AMDGPU_CP_IRQ_LAST;  in gfx_v9_0_set_irq_funcs()
7662 	adev->gfx.eop_irq.funcs = &gfx_v9_0_eop_irq_funcs;  in gfx_v9_0_set_irq_funcs()
7664 	adev->gfx.priv_reg_irq.num_types = 1;  in gfx_v9_0_set_irq_funcs()
7665 	adev->gfx.priv_reg_irq.funcs = &gfx_v9_0_priv_reg_irq_funcs;  in gfx_v9_0_set_irq_funcs()
7667 	adev->gfx.bad_op_irq.num_types = 1;  in gfx_v9_0_set_irq_funcs()
7668 	adev->gfx.bad_op_irq.funcs = &gfx_v9_0_bad_op_irq_funcs;  in gfx_v9_0_set_irq_funcs()
7670 	adev->gfx.priv_inst_irq.num_types = 1;  in gfx_v9_0_set_irq_funcs()
7671 	adev->gfx.priv_inst_irq.funcs = &gfx_v9_0_priv_inst_irq_funcs;  in gfx_v9_0_set_irq_funcs()
7673 	adev->gfx.cp_ecc_error_irq.num_types = 2; /*C5 ECC error and C9 FUE error*/  in gfx_v9_0_set_irq_funcs()
7674 	adev->gfx.cp_ecc_error_irq.funcs = &gfx_v9_0_cp_ecc_error_irq_funcs;  in gfx_v9_0_set_irq_funcs()
7682 	case IP_VERSION(9, 4, 0):  in gfx_v9_0_set_rlc_funcs()
7685 	case IP_VERSION(9, 4, 1):  in gfx_v9_0_set_rlc_funcs()
7687 	case IP_VERSION(9, 4, 2):  in gfx_v9_0_set_rlc_funcs()
7688 		adev->gfx.rlc.funcs = &gfx_v9_0_rlc_funcs;  in gfx_v9_0_set_rlc_funcs()
7701 	case IP_VERSION(9, 4, 0):  in gfx_v9_0_set_gds_init()
7702 		adev->gds.gds_size = 0x10000;  in gfx_v9_0_set_gds_init()
7706 	case IP_VERSION(9, 4, 1):  in gfx_v9_0_set_gds_init()
7707 		adev->gds.gds_size = 0x1000;  in gfx_v9_0_set_gds_init()
7709 	case IP_VERSION(9, 4, 2):  in gfx_v9_0_set_gds_init()
7713 		adev->gds.gds_size = 0;  in gfx_v9_0_set_gds_init()
7716 		adev->gds.gds_size = 0x10000;  in gfx_v9_0_set_gds_init()
7722 	case IP_VERSION(9, 4, 0):  in gfx_v9_0_set_gds_init()
7723 		adev->gds.gds_compute_max_wave_id = 0x7ff;  in gfx_v9_0_set_gds_init()
7726 		adev->gds.gds_compute_max_wave_id = 0x27f;  in gfx_v9_0_set_gds_init()
7730 		if (adev->apu_flags & AMD_APU_IS_RAVEN2)  in gfx_v9_0_set_gds_init()
7731 			adev->gds.gds_compute_max_wave_id = 0x77; /* raven2 */  in gfx_v9_0_set_gds_init()
7733 			adev->gds.gds_compute_max_wave_id = 0x15f; /* raven1 */  in gfx_v9_0_set_gds_init()
7735 	case IP_VERSION(9, 4, 1):  in gfx_v9_0_set_gds_init()
7736 		adev->gds.gds_compute_max_wave_id = 0xfff;  in gfx_v9_0_set_gds_init()
7738 	case IP_VERSION(9, 4, 2):  in gfx_v9_0_set_gds_init()
7740 		adev->gds.gds_compute_max_wave_id = 0;  in gfx_v9_0_set_gds_init()
7744 		adev->gds.gds_compute_max_wave_id = 0x7ff;  in gfx_v9_0_set_gds_init()
7748 	adev->gds.gws_size = 64;  in gfx_v9_0_set_gds_init()
7749 	adev->gds.oa_size = 16;  in gfx_v9_0_set_gds_init()
7776 	mask = amdgpu_gfx_create_bitmask(adev->gfx.config.max_cu_per_sh);  in gfx_v9_0_get_cu_active_bitmap()
7786 	unsigned disable_masks[4 * 4];  in gfx_v9_0_get_cu_info()
7789 		return -EINVAL;  in gfx_v9_0_get_cu_info()
7792 	 * 16 comes from bitmap array size 4*4, and it can cover all gfx9 ASICs  in gfx_v9_0_get_cu_info()
7794 	if (adev->gfx.config.max_shader_engines *  in gfx_v9_0_get_cu_info()
7795 		adev->gfx.config.max_sh_per_se > 16)  in gfx_v9_0_get_cu_info()
7796 		return -EINVAL;  in gfx_v9_0_get_cu_info()
7799 				    adev->gfx.config.max_shader_engines,  in gfx_v9_0_get_cu_info()
7800 				    adev->gfx.config.max_sh_per_se);  in gfx_v9_0_get_cu_info()
7802 	mutex_lock(&adev->grbm_idx_mutex);  in gfx_v9_0_get_cu_info()
7803 	for (i = 0; i < adev->gfx.config.max_shader_engines; i++) {  in gfx_v9_0_get_cu_info()
7804 		for (j = 0; j < adev->gfx.config.max_sh_per_se; j++) {  in gfx_v9_0_get_cu_info()
7810 				adev, disable_masks[i * adev->gfx.config.max_sh_per_se + j]);  in gfx_v9_0_get_cu_info()
7815 			 * 4x4 size array, and it's usually suitable for Vega  in gfx_v9_0_get_cu_info()
7816 			 * ASICs which has 4*2 SE/SH layout.  in gfx_v9_0_get_cu_info()
7820 			 *    SE4,SH0 --> bitmap[0][1]  in gfx_v9_0_get_cu_info()
7821 			 *    SE5,SH0 --> bitmap[1][1]  in gfx_v9_0_get_cu_info()
7822 			 *    SE6,SH0 --> bitmap[2][1]  in gfx_v9_0_get_cu_info()
7823 			 *    SE7,SH0 --> bitmap[3][1]  in gfx_v9_0_get_cu_info()
7825 			cu_info->bitmap[0][i % 4][j + i / 4] = bitmap;  in gfx_v9_0_get_cu_info()
7827 			for (k = 0; k < adev->gfx.config.max_cu_per_sh; k ++) {  in gfx_v9_0_get_cu_info()
7829 					if (counter < adev->gfx.config.max_cu_per_sh)  in gfx_v9_0_get_cu_info()
7838 			cu_info->ao_cu_bitmap[i % 4][j + i / 4] = ao_bitmap;  in gfx_v9_0_get_cu_info()
7842 	mutex_unlock(&adev->grbm_idx_mutex);  in gfx_v9_0_get_cu_info()
7844 	cu_info->number = active_cu_number;  in gfx_v9_0_get_cu_info()
7845 	cu_info->ao_cu_mask = ao_cu_mask;  in gfx_v9_0_get_cu_info()
7846 	cu_info->simd_per_cu = NUM_SIMD_PER_CU;  in gfx_v9_0_get_cu_info()