Lines Matching +full:ts +full:- +full:inv
740 switch (adev->asic_type) { in gfx_v8_0_init_golden_registers()
803 if ((adev->pdev->device == 0x67DF) && (adev->pdev->revision == 0xc7) && in gfx_v8_0_init_golden_registers()
804 ((adev->pdev->subsystem_device == 0xb37 && adev->pdev->subsystem_vendor == 0x1002) || in gfx_v8_0_init_golden_registers()
805 (adev->pdev->subsystem_device == 0x4a8 && adev->pdev->subsystem_vendor == 0x1043) || in gfx_v8_0_init_golden_registers()
806 (adev->pdev->subsystem_device == 0x9480 && adev->pdev->subsystem_vendor == 0x1680))) { in gfx_v8_0_init_golden_registers()
840 struct amdgpu_device *adev = ring->adev; in gfx_v8_0_ring_test_ring()
851 amdgpu_ring_write(ring, mmSCRATCH_REG0 - PACKET3_SET_UCONFIG_REG_START); in gfx_v8_0_ring_test_ring()
855 for (i = 0; i < adev->usec_timeout; i++) { in gfx_v8_0_ring_test_ring()
862 if (i >= adev->usec_timeout) in gfx_v8_0_ring_test_ring()
863 r = -ETIMEDOUT; in gfx_v8_0_ring_test_ring()
870 struct amdgpu_device *adev = ring->adev; in gfx_v8_0_ring_test_ib()
883 gpu_addr = adev->wb.gpu_addr + (index * 4); in gfx_v8_0_ring_test_ib()
884 adev->wb.wb[index] = cpu_to_le32(0xCAFEDEAD); in gfx_v8_0_ring_test_ib()
904 r = -ETIMEDOUT; in gfx_v8_0_ring_test_ib()
910 tmp = adev->wb.wb[index]; in gfx_v8_0_ring_test_ib()
914 r = -EINVAL; in gfx_v8_0_ring_test_ib()
927 amdgpu_ucode_release(&adev->gfx.pfp_fw); in gfx_v8_0_free_microcode()
928 amdgpu_ucode_release(&adev->gfx.me_fw); in gfx_v8_0_free_microcode()
929 amdgpu_ucode_release(&adev->gfx.ce_fw); in gfx_v8_0_free_microcode()
930 amdgpu_ucode_release(&adev->gfx.rlc_fw); in gfx_v8_0_free_microcode()
931 amdgpu_ucode_release(&adev->gfx.mec_fw); in gfx_v8_0_free_microcode()
932 if ((adev->asic_type != CHIP_STONEY) && in gfx_v8_0_free_microcode()
933 (adev->asic_type != CHIP_TOPAZ)) in gfx_v8_0_free_microcode()
934 amdgpu_ucode_release(&adev->gfx.mec2_fw); in gfx_v8_0_free_microcode()
936 kfree(adev->gfx.rlc.register_list_format); in gfx_v8_0_free_microcode()
951 switch (adev->asic_type) { in gfx_v8_0_init_microcode()
983 if (adev->asic_type >= CHIP_POLARIS10 && adev->asic_type <= CHIP_POLARIS12) { in gfx_v8_0_init_microcode()
984 err = amdgpu_ucode_request(adev, &adev->gfx.pfp_fw, in gfx_v8_0_init_microcode()
986 if (err == -ENODEV) { in gfx_v8_0_init_microcode()
987 err = amdgpu_ucode_request(adev, &adev->gfx.pfp_fw, in gfx_v8_0_init_microcode()
991 err = amdgpu_ucode_request(adev, &adev->gfx.pfp_fw, in gfx_v8_0_init_microcode()
996 cp_hdr = (const struct gfx_firmware_header_v1_0 *)adev->gfx.pfp_fw->data; in gfx_v8_0_init_microcode()
997 adev->gfx.pfp_fw_version = le32_to_cpu(cp_hdr->header.ucode_version); in gfx_v8_0_init_microcode()
998 adev->gfx.pfp_feature_version = le32_to_cpu(cp_hdr->ucode_feature_version); in gfx_v8_0_init_microcode()
1000 if (adev->asic_type >= CHIP_POLARIS10 && adev->asic_type <= CHIP_POLARIS12) { in gfx_v8_0_init_microcode()
1001 err = amdgpu_ucode_request(adev, &adev->gfx.me_fw, in gfx_v8_0_init_microcode()
1003 if (err == -ENODEV) { in gfx_v8_0_init_microcode()
1004 err = amdgpu_ucode_request(adev, &adev->gfx.me_fw, in gfx_v8_0_init_microcode()
1008 err = amdgpu_ucode_request(adev, &adev->gfx.me_fw, in gfx_v8_0_init_microcode()
1013 cp_hdr = (const struct gfx_firmware_header_v1_0 *)adev->gfx.me_fw->data; in gfx_v8_0_init_microcode()
1014 adev->gfx.me_fw_version = le32_to_cpu(cp_hdr->header.ucode_version); in gfx_v8_0_init_microcode()
1016 adev->gfx.me_feature_version = le32_to_cpu(cp_hdr->ucode_feature_version); in gfx_v8_0_init_microcode()
1018 if (adev->asic_type >= CHIP_POLARIS10 && adev->asic_type <= CHIP_POLARIS12) { in gfx_v8_0_init_microcode()
1019 err = amdgpu_ucode_request(adev, &adev->gfx.ce_fw, in gfx_v8_0_init_microcode()
1021 if (err == -ENODEV) { in gfx_v8_0_init_microcode()
1022 err = amdgpu_ucode_request(adev, &adev->gfx.ce_fw, in gfx_v8_0_init_microcode()
1026 err = amdgpu_ucode_request(adev, &adev->gfx.ce_fw, in gfx_v8_0_init_microcode()
1031 cp_hdr = (const struct gfx_firmware_header_v1_0 *)adev->gfx.ce_fw->data; in gfx_v8_0_init_microcode()
1032 adev->gfx.ce_fw_version = le32_to_cpu(cp_hdr->header.ucode_version); in gfx_v8_0_init_microcode()
1033 adev->gfx.ce_feature_version = le32_to_cpu(cp_hdr->ucode_feature_version); in gfx_v8_0_init_microcode()
1039 if (adev->gfx.ce_feature_version >= 46 && in gfx_v8_0_init_microcode()
1040 adev->gfx.pfp_feature_version >= 46) { in gfx_v8_0_init_microcode()
1041 adev->virt.chained_ib_support = true; in gfx_v8_0_init_microcode()
1044 adev->virt.chained_ib_support = false; in gfx_v8_0_init_microcode()
1046 err = amdgpu_ucode_request(adev, &adev->gfx.rlc_fw, in gfx_v8_0_init_microcode()
1050 rlc_hdr = (const struct rlc_firmware_header_v2_0 *)adev->gfx.rlc_fw->data; in gfx_v8_0_init_microcode()
1051 adev->gfx.rlc_fw_version = le32_to_cpu(rlc_hdr->header.ucode_version); in gfx_v8_0_init_microcode()
1052 adev->gfx.rlc_feature_version = le32_to_cpu(rlc_hdr->ucode_feature_version); in gfx_v8_0_init_microcode()
1054 adev->gfx.rlc.save_and_restore_offset = in gfx_v8_0_init_microcode()
1055 le32_to_cpu(rlc_hdr->save_and_restore_offset); in gfx_v8_0_init_microcode()
1056 adev->gfx.rlc.clear_state_descriptor_offset = in gfx_v8_0_init_microcode()
1057 le32_to_cpu(rlc_hdr->clear_state_descriptor_offset); in gfx_v8_0_init_microcode()
1058 adev->gfx.rlc.avail_scratch_ram_locations = in gfx_v8_0_init_microcode()
1059 le32_to_cpu(rlc_hdr->avail_scratch_ram_locations); in gfx_v8_0_init_microcode()
1060 adev->gfx.rlc.reg_restore_list_size = in gfx_v8_0_init_microcode()
1061 le32_to_cpu(rlc_hdr->reg_restore_list_size); in gfx_v8_0_init_microcode()
1062 adev->gfx.rlc.reg_list_format_start = in gfx_v8_0_init_microcode()
1063 le32_to_cpu(rlc_hdr->reg_list_format_start); in gfx_v8_0_init_microcode()
1064 adev->gfx.rlc.reg_list_format_separate_start = in gfx_v8_0_init_microcode()
1065 le32_to_cpu(rlc_hdr->reg_list_format_separate_start); in gfx_v8_0_init_microcode()
1066 adev->gfx.rlc.starting_offsets_start = in gfx_v8_0_init_microcode()
1067 le32_to_cpu(rlc_hdr->starting_offsets_start); in gfx_v8_0_init_microcode()
1068 adev->gfx.rlc.reg_list_format_size_bytes = in gfx_v8_0_init_microcode()
1069 le32_to_cpu(rlc_hdr->reg_list_format_size_bytes); in gfx_v8_0_init_microcode()
1070 adev->gfx.rlc.reg_list_size_bytes = in gfx_v8_0_init_microcode()
1071 le32_to_cpu(rlc_hdr->reg_list_size_bytes); in gfx_v8_0_init_microcode()
1073 adev->gfx.rlc.register_list_format = in gfx_v8_0_init_microcode()
1074 kmalloc(adev->gfx.rlc.reg_list_format_size_bytes + in gfx_v8_0_init_microcode()
1075 adev->gfx.rlc.reg_list_size_bytes, GFP_KERNEL); in gfx_v8_0_init_microcode()
1077 if (!adev->gfx.rlc.register_list_format) { in gfx_v8_0_init_microcode()
1078 err = -ENOMEM; in gfx_v8_0_init_microcode()
1083 le32_to_cpu(rlc_hdr->reg_list_format_array_offset_bytes)); in gfx_v8_0_init_microcode()
1084 for (i = 0 ; i < (adev->gfx.rlc.reg_list_format_size_bytes >> 2); i++) in gfx_v8_0_init_microcode()
1085 adev->gfx.rlc.register_list_format[i] = le32_to_cpu(tmp[i]); in gfx_v8_0_init_microcode()
1087 adev->gfx.rlc.register_restore = adev->gfx.rlc.register_list_format + i; in gfx_v8_0_init_microcode()
1090 le32_to_cpu(rlc_hdr->reg_list_array_offset_bytes)); in gfx_v8_0_init_microcode()
1091 for (i = 0 ; i < (adev->gfx.rlc.reg_list_size_bytes >> 2); i++) in gfx_v8_0_init_microcode()
1092 adev->gfx.rlc.register_restore[i] = le32_to_cpu(tmp[i]); in gfx_v8_0_init_microcode()
1094 if (adev->asic_type >= CHIP_POLARIS10 && adev->asic_type <= CHIP_POLARIS12) { in gfx_v8_0_init_microcode()
1095 err = amdgpu_ucode_request(adev, &adev->gfx.mec_fw, in gfx_v8_0_init_microcode()
1097 if (err == -ENODEV) { in gfx_v8_0_init_microcode()
1098 err = amdgpu_ucode_request(adev, &adev->gfx.mec_fw, in gfx_v8_0_init_microcode()
1102 err = amdgpu_ucode_request(adev, &adev->gfx.mec_fw, in gfx_v8_0_init_microcode()
1107 cp_hdr = (const struct gfx_firmware_header_v1_0 *)adev->gfx.mec_fw->data; in gfx_v8_0_init_microcode()
1108 adev->gfx.mec_fw_version = le32_to_cpu(cp_hdr->header.ucode_version); in gfx_v8_0_init_microcode()
1109 adev->gfx.mec_feature_version = le32_to_cpu(cp_hdr->ucode_feature_version); in gfx_v8_0_init_microcode()
1111 if ((adev->asic_type != CHIP_STONEY) && in gfx_v8_0_init_microcode()
1112 (adev->asic_type != CHIP_TOPAZ)) { in gfx_v8_0_init_microcode()
1113 if (adev->asic_type >= CHIP_POLARIS10 && adev->asic_type <= CHIP_POLARIS12) { in gfx_v8_0_init_microcode()
1114 err = amdgpu_ucode_request(adev, &adev->gfx.mec2_fw, in gfx_v8_0_init_microcode()
1116 if (err == -ENODEV) { in gfx_v8_0_init_microcode()
1117 err = amdgpu_ucode_request(adev, &adev->gfx.mec2_fw, in gfx_v8_0_init_microcode()
1121 err = amdgpu_ucode_request(adev, &adev->gfx.mec2_fw, in gfx_v8_0_init_microcode()
1126 adev->gfx.mec2_fw->data; in gfx_v8_0_init_microcode()
1127 adev->gfx.mec2_fw_version = in gfx_v8_0_init_microcode()
1128 le32_to_cpu(cp_hdr->header.ucode_version); in gfx_v8_0_init_microcode()
1129 adev->gfx.mec2_feature_version = in gfx_v8_0_init_microcode()
1130 le32_to_cpu(cp_hdr->ucode_feature_version); in gfx_v8_0_init_microcode()
1133 adev->gfx.mec2_fw = NULL; in gfx_v8_0_init_microcode()
1137 info = &adev->firmware.ucode[AMDGPU_UCODE_ID_CP_PFP]; in gfx_v8_0_init_microcode()
1138 info->ucode_id = AMDGPU_UCODE_ID_CP_PFP; in gfx_v8_0_init_microcode()
1139 info->fw = adev->gfx.pfp_fw; in gfx_v8_0_init_microcode()
1140 header = (const struct common_firmware_header *)info->fw->data; in gfx_v8_0_init_microcode()
1141 adev->firmware.fw_size += in gfx_v8_0_init_microcode()
1142 ALIGN(le32_to_cpu(header->ucode_size_bytes), PAGE_SIZE); in gfx_v8_0_init_microcode()
1144 info = &adev->firmware.ucode[AMDGPU_UCODE_ID_CP_ME]; in gfx_v8_0_init_microcode()
1145 info->ucode_id = AMDGPU_UCODE_ID_CP_ME; in gfx_v8_0_init_microcode()
1146 info->fw = adev->gfx.me_fw; in gfx_v8_0_init_microcode()
1147 header = (const struct common_firmware_header *)info->fw->data; in gfx_v8_0_init_microcode()
1148 adev->firmware.fw_size += in gfx_v8_0_init_microcode()
1149 ALIGN(le32_to_cpu(header->ucode_size_bytes), PAGE_SIZE); in gfx_v8_0_init_microcode()
1151 info = &adev->firmware.ucode[AMDGPU_UCODE_ID_CP_CE]; in gfx_v8_0_init_microcode()
1152 info->ucode_id = AMDGPU_UCODE_ID_CP_CE; in gfx_v8_0_init_microcode()
1153 info->fw = adev->gfx.ce_fw; in gfx_v8_0_init_microcode()
1154 header = (const struct common_firmware_header *)info->fw->data; in gfx_v8_0_init_microcode()
1155 adev->firmware.fw_size += in gfx_v8_0_init_microcode()
1156 ALIGN(le32_to_cpu(header->ucode_size_bytes), PAGE_SIZE); in gfx_v8_0_init_microcode()
1158 info = &adev->firmware.ucode[AMDGPU_UCODE_ID_RLC_G]; in gfx_v8_0_init_microcode()
1159 info->ucode_id = AMDGPU_UCODE_ID_RLC_G; in gfx_v8_0_init_microcode()
1160 info->fw = adev->gfx.rlc_fw; in gfx_v8_0_init_microcode()
1161 header = (const struct common_firmware_header *)info->fw->data; in gfx_v8_0_init_microcode()
1162 adev->firmware.fw_size += in gfx_v8_0_init_microcode()
1163 ALIGN(le32_to_cpu(header->ucode_size_bytes), PAGE_SIZE); in gfx_v8_0_init_microcode()
1165 info = &adev->firmware.ucode[AMDGPU_UCODE_ID_CP_MEC1]; in gfx_v8_0_init_microcode()
1166 info->ucode_id = AMDGPU_UCODE_ID_CP_MEC1; in gfx_v8_0_init_microcode()
1167 info->fw = adev->gfx.mec_fw; in gfx_v8_0_init_microcode()
1168 header = (const struct common_firmware_header *)info->fw->data; in gfx_v8_0_init_microcode()
1169 adev->firmware.fw_size += in gfx_v8_0_init_microcode()
1170 ALIGN(le32_to_cpu(header->ucode_size_bytes), PAGE_SIZE); in gfx_v8_0_init_microcode()
1173 cp_hdr = (const struct gfx_firmware_header_v1_0 *)adev->gfx.mec_fw->data; in gfx_v8_0_init_microcode()
1174 adev->firmware.fw_size += in gfx_v8_0_init_microcode()
1175 ALIGN(le32_to_cpu(cp_hdr->jt_size) << 2, PAGE_SIZE); in gfx_v8_0_init_microcode()
1178 info = &adev->firmware.ucode[AMDGPU_UCODE_ID_STORAGE]; in gfx_v8_0_init_microcode()
1179 info->ucode_id = AMDGPU_UCODE_ID_STORAGE; in gfx_v8_0_init_microcode()
1180 info->fw = adev->gfx.mec_fw; in gfx_v8_0_init_microcode()
1181 adev->firmware.fw_size += in gfx_v8_0_init_microcode()
1185 if (adev->gfx.mec2_fw) { in gfx_v8_0_init_microcode()
1186 info = &adev->firmware.ucode[AMDGPU_UCODE_ID_CP_MEC2]; in gfx_v8_0_init_microcode()
1187 info->ucode_id = AMDGPU_UCODE_ID_CP_MEC2; in gfx_v8_0_init_microcode()
1188 info->fw = adev->gfx.mec2_fw; in gfx_v8_0_init_microcode()
1189 header = (const struct common_firmware_header *)info->fw->data; in gfx_v8_0_init_microcode()
1190 adev->firmware.fw_size += in gfx_v8_0_init_microcode()
1191 ALIGN(le32_to_cpu(header->ucode_size_bytes), PAGE_SIZE); in gfx_v8_0_init_microcode()
1196 dev_err(adev->dev, "gfx8: Failed to load firmware %s gfx firmware\n", chip_name); in gfx_v8_0_init_microcode()
1197 amdgpu_ucode_release(&adev->gfx.pfp_fw); in gfx_v8_0_init_microcode()
1198 amdgpu_ucode_release(&adev->gfx.me_fw); in gfx_v8_0_init_microcode()
1199 amdgpu_ucode_release(&adev->gfx.ce_fw); in gfx_v8_0_init_microcode()
1200 amdgpu_ucode_release(&adev->gfx.rlc_fw); in gfx_v8_0_init_microcode()
1201 amdgpu_ucode_release(&adev->gfx.mec_fw); in gfx_v8_0_init_microcode()
1202 amdgpu_ucode_release(&adev->gfx.mec2_fw); in gfx_v8_0_init_microcode()
1214 if (adev->gfx.rlc.cs_data == NULL) in gfx_v8_0_get_csb_buffer()
1226 for (sect = adev->gfx.rlc.cs_data; sect->section != NULL; ++sect) { in gfx_v8_0_get_csb_buffer()
1227 for (ext = sect->section; ext->extent != NULL; ++ext) { in gfx_v8_0_get_csb_buffer()
1228 if (sect->id == SECT_CONTEXT) { in gfx_v8_0_get_csb_buffer()
1230 cpu_to_le32(PACKET3(PACKET3_SET_CONTEXT_REG, ext->reg_count)); in gfx_v8_0_get_csb_buffer()
1231 buffer[count++] = cpu_to_le32(ext->reg_index - in gfx_v8_0_get_csb_buffer()
1233 for (i = 0; i < ext->reg_count; i++) in gfx_v8_0_get_csb_buffer()
1234 buffer[count++] = cpu_to_le32(ext->extent[i]); in gfx_v8_0_get_csb_buffer()
1242 buffer[count++] = cpu_to_le32(mmPA_SC_RASTER_CONFIG - in gfx_v8_0_get_csb_buffer()
1244 buffer[count++] = cpu_to_le32(adev->gfx.config.rb_config[0][0].raster_config); in gfx_v8_0_get_csb_buffer()
1245 buffer[count++] = cpu_to_le32(adev->gfx.config.rb_config[0][0].raster_config_1); in gfx_v8_0_get_csb_buffer()
1256 if (adev->asic_type == CHIP_CARRIZO) in gfx_v8_0_cp_jump_table_num()
1267 adev->gfx.rlc.cs_data = vi_cs_data; in gfx_v8_0_rlc_init()
1269 cs_data = adev->gfx.rlc.cs_data; in gfx_v8_0_rlc_init()
1278 if ((adev->asic_type == CHIP_CARRIZO) || in gfx_v8_0_rlc_init()
1279 (adev->asic_type == CHIP_STONEY)) { in gfx_v8_0_rlc_init()
1280 adev->gfx.rlc.cp_table_size = ALIGN(96 * 5 * 4, 2048) + (64 * 1024); /* JT + GDS */ in gfx_v8_0_rlc_init()
1287 if (adev->gfx.rlc.funcs->update_spm_vmid) in gfx_v8_0_rlc_init()
1288 adev->gfx.rlc.funcs->update_spm_vmid(adev, NULL, 0xf); in gfx_v8_0_rlc_init()
1295 amdgpu_bo_free_kernel(&adev->gfx.mec.hpd_eop_obj, NULL, NULL); in gfx_v8_0_mec_fini()
1304 bitmap_zero(adev->gfx.mec_bitmap[0].queue_bitmap, AMDGPU_MAX_COMPUTE_QUEUES); in gfx_v8_0_mec_init()
1309 mec_hpd_size = adev->gfx.num_compute_rings * GFX8_MEC_HPD_SIZE; in gfx_v8_0_mec_init()
1314 &adev->gfx.mec.hpd_eop_obj, in gfx_v8_0_mec_init()
1315 &adev->gfx.mec.hpd_eop_gpu_addr, in gfx_v8_0_mec_init()
1318 dev_warn(adev->dev, "(%d) create HDP EOP bo failed\n", r); in gfx_v8_0_mec_init()
1324 amdgpu_bo_kunmap(adev->gfx.mec.hpd_eop_obj); in gfx_v8_0_mec_init()
1325 amdgpu_bo_unreserve(adev->gfx.mec.hpd_eop_obj); in gfx_v8_0_mec_init()
1487 struct amdgpu_ring *ring = &adev->gfx.compute_ring[0]; in gfx_v8_0_do_edc_gpr_workarounds()
1496 if (adev->asic_type != CHIP_CARRIZO) in gfx_v8_0_do_edc_gpr_workarounds()
1500 if (!ring->sched.ready) in gfx_v8_0_do_edc_gpr_workarounds()
1541 ib.ptr[ib.length_dw++] = vgpr_init_regs[i] - PACKET3_SET_SH_REG_START; in gfx_v8_0_do_edc_gpr_workarounds()
1547 ib.ptr[ib.length_dw++] = mmCOMPUTE_PGM_LO - PACKET3_SET_SH_REG_START; in gfx_v8_0_do_edc_gpr_workarounds()
1567 ib.ptr[ib.length_dw++] = sgpr1_init_regs[i] - PACKET3_SET_SH_REG_START; in gfx_v8_0_do_edc_gpr_workarounds()
1573 ib.ptr[ib.length_dw++] = mmCOMPUTE_PGM_LO - PACKET3_SET_SH_REG_START; in gfx_v8_0_do_edc_gpr_workarounds()
1593 ib.ptr[ib.length_dw++] = sgpr2_init_regs[i] - PACKET3_SET_SH_REG_START; in gfx_v8_0_do_edc_gpr_workarounds()
1599 ib.ptr[ib.length_dw++] = mmCOMPUTE_PGM_LO - PACKET3_SET_SH_REG_START; in gfx_v8_0_do_edc_gpr_workarounds()
1657 switch (adev->asic_type) { in gfx_v8_0_gpu_early_init()
1659 adev->gfx.config.max_shader_engines = 1; in gfx_v8_0_gpu_early_init()
1660 adev->gfx.config.max_tile_pipes = 2; in gfx_v8_0_gpu_early_init()
1661 adev->gfx.config.max_cu_per_sh = 6; in gfx_v8_0_gpu_early_init()
1662 adev->gfx.config.max_sh_per_se = 1; in gfx_v8_0_gpu_early_init()
1663 adev->gfx.config.max_backends_per_se = 2; in gfx_v8_0_gpu_early_init()
1664 adev->gfx.config.max_texture_channel_caches = 2; in gfx_v8_0_gpu_early_init()
1665 adev->gfx.config.max_gprs = 256; in gfx_v8_0_gpu_early_init()
1666 adev->gfx.config.max_gs_threads = 32; in gfx_v8_0_gpu_early_init()
1667 adev->gfx.config.max_hw_contexts = 8; in gfx_v8_0_gpu_early_init()
1669 adev->gfx.config.sc_prim_fifo_size_frontend = 0x20; in gfx_v8_0_gpu_early_init()
1670 adev->gfx.config.sc_prim_fifo_size_backend = 0x100; in gfx_v8_0_gpu_early_init()
1671 adev->gfx.config.sc_hiz_tile_fifo_size = 0x30; in gfx_v8_0_gpu_early_init()
1672 adev->gfx.config.sc_earlyz_tile_fifo_size = 0x130; in gfx_v8_0_gpu_early_init()
1676 adev->gfx.config.max_shader_engines = 4; in gfx_v8_0_gpu_early_init()
1677 adev->gfx.config.max_tile_pipes = 16; in gfx_v8_0_gpu_early_init()
1678 adev->gfx.config.max_cu_per_sh = 16; in gfx_v8_0_gpu_early_init()
1679 adev->gfx.config.max_sh_per_se = 1; in gfx_v8_0_gpu_early_init()
1680 adev->gfx.config.max_backends_per_se = 4; in gfx_v8_0_gpu_early_init()
1681 adev->gfx.config.max_texture_channel_caches = 16; in gfx_v8_0_gpu_early_init()
1682 adev->gfx.config.max_gprs = 256; in gfx_v8_0_gpu_early_init()
1683 adev->gfx.config.max_gs_threads = 32; in gfx_v8_0_gpu_early_init()
1684 adev->gfx.config.max_hw_contexts = 8; in gfx_v8_0_gpu_early_init()
1686 adev->gfx.config.sc_prim_fifo_size_frontend = 0x20; in gfx_v8_0_gpu_early_init()
1687 adev->gfx.config.sc_prim_fifo_size_backend = 0x100; in gfx_v8_0_gpu_early_init()
1688 adev->gfx.config.sc_hiz_tile_fifo_size = 0x30; in gfx_v8_0_gpu_early_init()
1689 adev->gfx.config.sc_earlyz_tile_fifo_size = 0x130; in gfx_v8_0_gpu_early_init()
1697 adev->gfx.config.max_gprs = 256; in gfx_v8_0_gpu_early_init()
1698 adev->gfx.config.max_gs_threads = 32; in gfx_v8_0_gpu_early_init()
1699 adev->gfx.config.max_hw_contexts = 8; in gfx_v8_0_gpu_early_init()
1701 adev->gfx.config.sc_prim_fifo_size_frontend = 0x20; in gfx_v8_0_gpu_early_init()
1702 adev->gfx.config.sc_prim_fifo_size_backend = 0x100; in gfx_v8_0_gpu_early_init()
1703 adev->gfx.config.sc_hiz_tile_fifo_size = 0x30; in gfx_v8_0_gpu_early_init()
1704 adev->gfx.config.sc_earlyz_tile_fifo_size = 0x130; in gfx_v8_0_gpu_early_init()
1712 adev->gfx.config.max_gprs = 256; in gfx_v8_0_gpu_early_init()
1713 adev->gfx.config.max_gs_threads = 32; in gfx_v8_0_gpu_early_init()
1714 adev->gfx.config.max_hw_contexts = 8; in gfx_v8_0_gpu_early_init()
1716 adev->gfx.config.sc_prim_fifo_size_frontend = 0x20; in gfx_v8_0_gpu_early_init()
1717 adev->gfx.config.sc_prim_fifo_size_backend = 0x100; in gfx_v8_0_gpu_early_init()
1718 adev->gfx.config.sc_hiz_tile_fifo_size = 0x30; in gfx_v8_0_gpu_early_init()
1719 adev->gfx.config.sc_earlyz_tile_fifo_size = 0x130; in gfx_v8_0_gpu_early_init()
1723 adev->gfx.config.max_shader_engines = 4; in gfx_v8_0_gpu_early_init()
1724 adev->gfx.config.max_tile_pipes = 8; in gfx_v8_0_gpu_early_init()
1725 adev->gfx.config.max_cu_per_sh = 8; in gfx_v8_0_gpu_early_init()
1726 adev->gfx.config.max_sh_per_se = 1; in gfx_v8_0_gpu_early_init()
1727 adev->gfx.config.max_backends_per_se = 2; in gfx_v8_0_gpu_early_init()
1728 adev->gfx.config.max_texture_channel_caches = 8; in gfx_v8_0_gpu_early_init()
1729 adev->gfx.config.max_gprs = 256; in gfx_v8_0_gpu_early_init()
1730 adev->gfx.config.max_gs_threads = 32; in gfx_v8_0_gpu_early_init()
1731 adev->gfx.config.max_hw_contexts = 8; in gfx_v8_0_gpu_early_init()
1733 adev->gfx.config.sc_prim_fifo_size_frontend = 0x20; in gfx_v8_0_gpu_early_init()
1734 adev->gfx.config.sc_prim_fifo_size_backend = 0x100; in gfx_v8_0_gpu_early_init()
1735 adev->gfx.config.sc_hiz_tile_fifo_size = 0x30; in gfx_v8_0_gpu_early_init()
1736 adev->gfx.config.sc_earlyz_tile_fifo_size = 0x130; in gfx_v8_0_gpu_early_init()
1740 adev->gfx.config.max_shader_engines = 1; in gfx_v8_0_gpu_early_init()
1741 adev->gfx.config.max_tile_pipes = 2; in gfx_v8_0_gpu_early_init()
1742 adev->gfx.config.max_sh_per_se = 1; in gfx_v8_0_gpu_early_init()
1743 adev->gfx.config.max_backends_per_se = 2; in gfx_v8_0_gpu_early_init()
1744 adev->gfx.config.max_cu_per_sh = 8; in gfx_v8_0_gpu_early_init()
1745 adev->gfx.config.max_texture_channel_caches = 2; in gfx_v8_0_gpu_early_init()
1746 adev->gfx.config.max_gprs = 256; in gfx_v8_0_gpu_early_init()
1747 adev->gfx.config.max_gs_threads = 32; in gfx_v8_0_gpu_early_init()
1748 adev->gfx.config.max_hw_contexts = 8; in gfx_v8_0_gpu_early_init()
1750 adev->gfx.config.sc_prim_fifo_size_frontend = 0x20; in gfx_v8_0_gpu_early_init()
1751 adev->gfx.config.sc_prim_fifo_size_backend = 0x100; in gfx_v8_0_gpu_early_init()
1752 adev->gfx.config.sc_hiz_tile_fifo_size = 0x30; in gfx_v8_0_gpu_early_init()
1753 adev->gfx.config.sc_earlyz_tile_fifo_size = 0x130; in gfx_v8_0_gpu_early_init()
1757 adev->gfx.config.max_shader_engines = 1; in gfx_v8_0_gpu_early_init()
1758 adev->gfx.config.max_tile_pipes = 2; in gfx_v8_0_gpu_early_init()
1759 adev->gfx.config.max_sh_per_se = 1; in gfx_v8_0_gpu_early_init()
1760 adev->gfx.config.max_backends_per_se = 1; in gfx_v8_0_gpu_early_init()
1761 adev->gfx.config.max_cu_per_sh = 3; in gfx_v8_0_gpu_early_init()
1762 adev->gfx.config.max_texture_channel_caches = 2; in gfx_v8_0_gpu_early_init()
1763 adev->gfx.config.max_gprs = 256; in gfx_v8_0_gpu_early_init()
1764 adev->gfx.config.max_gs_threads = 16; in gfx_v8_0_gpu_early_init()
1765 adev->gfx.config.max_hw_contexts = 8; in gfx_v8_0_gpu_early_init()
1767 adev->gfx.config.sc_prim_fifo_size_frontend = 0x20; in gfx_v8_0_gpu_early_init()
1768 adev->gfx.config.sc_prim_fifo_size_backend = 0x100; in gfx_v8_0_gpu_early_init()
1769 adev->gfx.config.sc_hiz_tile_fifo_size = 0x30; in gfx_v8_0_gpu_early_init()
1770 adev->gfx.config.sc_earlyz_tile_fifo_size = 0x130; in gfx_v8_0_gpu_early_init()
1774 adev->gfx.config.max_shader_engines = 2; in gfx_v8_0_gpu_early_init()
1775 adev->gfx.config.max_tile_pipes = 4; in gfx_v8_0_gpu_early_init()
1776 adev->gfx.config.max_cu_per_sh = 2; in gfx_v8_0_gpu_early_init()
1777 adev->gfx.config.max_sh_per_se = 1; in gfx_v8_0_gpu_early_init()
1778 adev->gfx.config.max_backends_per_se = 2; in gfx_v8_0_gpu_early_init()
1779 adev->gfx.config.max_texture_channel_caches = 4; in gfx_v8_0_gpu_early_init()
1780 adev->gfx.config.max_gprs = 256; in gfx_v8_0_gpu_early_init()
1781 adev->gfx.config.max_gs_threads = 32; in gfx_v8_0_gpu_early_init()
1782 adev->gfx.config.max_hw_contexts = 8; in gfx_v8_0_gpu_early_init()
1784 adev->gfx.config.sc_prim_fifo_size_frontend = 0x20; in gfx_v8_0_gpu_early_init()
1785 adev->gfx.config.sc_prim_fifo_size_backend = 0x100; in gfx_v8_0_gpu_early_init()
1786 adev->gfx.config.sc_hiz_tile_fifo_size = 0x30; in gfx_v8_0_gpu_early_init()
1787 adev->gfx.config.sc_earlyz_tile_fifo_size = 0x130; in gfx_v8_0_gpu_early_init()
1792 adev->gfx.config.mc_arb_ramcfg = RREG32(mmMC_ARB_RAMCFG); in gfx_v8_0_gpu_early_init()
1793 mc_arb_ramcfg = adev->gfx.config.mc_arb_ramcfg; in gfx_v8_0_gpu_early_init()
1795 adev->gfx.config.num_banks = REG_GET_FIELD(mc_arb_ramcfg, in gfx_v8_0_gpu_early_init()
1797 adev->gfx.config.num_ranks = REG_GET_FIELD(mc_arb_ramcfg, in gfx_v8_0_gpu_early_init()
1800 adev->gfx.config.num_tile_pipes = adev->gfx.config.max_tile_pipes; in gfx_v8_0_gpu_early_init()
1801 adev->gfx.config.mem_max_burst_length_bytes = 256; in gfx_v8_0_gpu_early_init()
1802 if (adev->flags & AMD_IS_APU) { in gfx_v8_0_gpu_early_init()
1825 adev->gfx.config.mem_row_size_in_kb = 2; in gfx_v8_0_gpu_early_init()
1827 adev->gfx.config.mem_row_size_in_kb = 1; in gfx_v8_0_gpu_early_init()
1830 adev->gfx.config.mem_row_size_in_kb = (4 * (1 << (8 + tmp))) / 1024; in gfx_v8_0_gpu_early_init()
1831 if (adev->gfx.config.mem_row_size_in_kb > 4) in gfx_v8_0_gpu_early_init()
1832 adev->gfx.config.mem_row_size_in_kb = 4; in gfx_v8_0_gpu_early_init()
1835 adev->gfx.config.shader_engine_tile_size = 32; in gfx_v8_0_gpu_early_init()
1836 adev->gfx.config.num_gpus = 1; in gfx_v8_0_gpu_early_init()
1837 adev->gfx.config.multi_gpu_tile_size = 64; in gfx_v8_0_gpu_early_init()
1840 switch (adev->gfx.config.mem_row_size_in_kb) { in gfx_v8_0_gpu_early_init()
1852 adev->gfx.config.gb_addr_config = gb_addr_config; in gfx_v8_0_gpu_early_init()
1862 struct amdgpu_ring *ring = &adev->gfx.compute_ring[ring_id]; in gfx_v8_0_compute_ring_init()
1865 ring = &adev->gfx.compute_ring[ring_id]; in gfx_v8_0_compute_ring_init()
1868 ring->me = mec + 1; in gfx_v8_0_compute_ring_init()
1869 ring->pipe = pipe; in gfx_v8_0_compute_ring_init()
1870 ring->queue = queue; in gfx_v8_0_compute_ring_init()
1872 ring->ring_obj = NULL; in gfx_v8_0_compute_ring_init()
1873 ring->use_doorbell = true; in gfx_v8_0_compute_ring_init()
1874 ring->doorbell_index = adev->doorbell_index.mec_ring0 + ring_id; in gfx_v8_0_compute_ring_init()
1875 ring->eop_gpu_addr = adev->gfx.mec.hpd_eop_gpu_addr in gfx_v8_0_compute_ring_init()
1877 sprintf(ring->name, "comp_%d.%d.%d", ring->me, ring->pipe, ring->queue); in gfx_v8_0_compute_ring_init()
1880 + ((ring->me - 1) * adev->gfx.mec.num_pipe_per_mec) in gfx_v8_0_compute_ring_init()
1881 + ring->pipe; in gfx_v8_0_compute_ring_init()
1885 /* type-2 packets are deprecated on MEC, use type-3 instead */ in gfx_v8_0_compute_ring_init()
1886 r = amdgpu_ring_init(adev, ring, 1024, &adev->gfx.eop_irq, irq_type, in gfx_v8_0_compute_ring_init()
1904 switch (adev->asic_type) { in gfx_v8_0_sw_init()
1912 adev->gfx.mec.num_mec = 2; in gfx_v8_0_sw_init()
1917 adev->gfx.mec.num_mec = 1; in gfx_v8_0_sw_init()
1921 adev->gfx.mec.num_pipe_per_mec = 4; in gfx_v8_0_sw_init()
1922 adev->gfx.mec.num_queue_per_pipe = 8; in gfx_v8_0_sw_init()
1925 …r = amdgpu_irq_add_id(adev, AMDGPU_IRQ_CLIENTID_LEGACY, VISLANDS30_IV_SRCID_CP_END_OF_PIPE, &adev-… in gfx_v8_0_sw_init()
1931 &adev->gfx.priv_reg_irq); in gfx_v8_0_sw_init()
1937 &adev->gfx.priv_inst_irq); in gfx_v8_0_sw_init()
1943 &adev->gfx.cp_ecc_error_irq); in gfx_v8_0_sw_init()
1949 &adev->gfx.sq_irq); in gfx_v8_0_sw_init()
1955 INIT_WORK(&adev->gfx.sq_work.work, gfx_v8_0_sq_irq_work_func); in gfx_v8_0_sw_init()
1957 adev->gfx.gfx_current_status = AMDGPU_GFX_NORMAL_MODE; in gfx_v8_0_sw_init()
1965 r = adev->gfx.rlc.funcs->init(adev); in gfx_v8_0_sw_init()
1978 for (i = 0; i < adev->gfx.num_gfx_rings; i++) { in gfx_v8_0_sw_init()
1979 ring = &adev->gfx.gfx_ring[i]; in gfx_v8_0_sw_init()
1980 ring->ring_obj = NULL; in gfx_v8_0_sw_init()
1981 sprintf(ring->name, "gfx"); in gfx_v8_0_sw_init()
1983 if (adev->asic_type != CHIP_TOPAZ) { in gfx_v8_0_sw_init()
1984 ring->use_doorbell = true; in gfx_v8_0_sw_init()
1985 ring->doorbell_index = adev->doorbell_index.gfx_ring0; in gfx_v8_0_sw_init()
1988 r = amdgpu_ring_init(adev, ring, 1024, &adev->gfx.eop_irq, in gfx_v8_0_sw_init()
1996 /* set up the compute queues - allocate horizontally across pipes */ in gfx_v8_0_sw_init()
1998 for (i = 0; i < adev->gfx.mec.num_mec; ++i) { in gfx_v8_0_sw_init()
1999 for (j = 0; j < adev->gfx.mec.num_queue_per_pipe; j++) { in gfx_v8_0_sw_init()
2000 for (k = 0; k < adev->gfx.mec.num_pipe_per_mec; k++) { in gfx_v8_0_sw_init()
2031 adev->gfx.ce_ram_size = 0x8000; in gfx_v8_0_sw_init()
2045 for (i = 0; i < adev->gfx.num_gfx_rings; i++) in gfx_v8_0_sw_fini()
2046 amdgpu_ring_fini(&adev->gfx.gfx_ring[i]); in gfx_v8_0_sw_fini()
2047 for (i = 0; i < adev->gfx.num_compute_rings; i++) in gfx_v8_0_sw_fini()
2048 amdgpu_ring_fini(&adev->gfx.compute_ring[i]); in gfx_v8_0_sw_fini()
2051 amdgpu_gfx_kiq_free_ring(&adev->gfx.kiq[0].ring); in gfx_v8_0_sw_fini()
2056 amdgpu_bo_free_kernel(&adev->gfx.rlc.clear_state_obj, in gfx_v8_0_sw_fini()
2057 &adev->gfx.rlc.clear_state_gpu_addr, in gfx_v8_0_sw_fini()
2058 (void **)&adev->gfx.rlc.cs_ptr); in gfx_v8_0_sw_fini()
2059 if ((adev->asic_type == CHIP_CARRIZO) || in gfx_v8_0_sw_fini()
2060 (adev->asic_type == CHIP_STONEY)) { in gfx_v8_0_sw_fini()
2061 amdgpu_bo_free_kernel(&adev->gfx.rlc.cp_table_obj, in gfx_v8_0_sw_fini()
2062 &adev->gfx.rlc.cp_table_gpu_addr, in gfx_v8_0_sw_fini()
2063 (void **)&adev->gfx.rlc.cp_table_ptr); in gfx_v8_0_sw_fini()
2073 const u32 num_tile_mode_states = ARRAY_SIZE(adev->gfx.config.tile_mode_array); in gfx_v8_0_tiling_mode_table_init()
2074 const u32 num_secondary_tile_mode_states = ARRAY_SIZE(adev->gfx.config.macrotile_mode_array); in gfx_v8_0_tiling_mode_table_init()
2077 modearray = adev->gfx.config.tile_mode_array; in gfx_v8_0_tiling_mode_table_init()
2078 mod2array = adev->gfx.config.macrotile_mode_array; in gfx_v8_0_tiling_mode_table_init()
2086 switch (adev->asic_type) { in gfx_v8_0_tiling_mode_table_init()
3214 dev_warn(adev->dev, in gfx_v8_0_tiling_mode_table_init()
3216 adev->asic_type); in gfx_v8_0_tiling_mode_table_init()
3432 mask = amdgpu_gfx_create_bitmask(adev->gfx.config.max_backends_per_se / in gfx_v8_0_get_rb_active_bitmap()
3433 adev->gfx.config.max_sh_per_se); in gfx_v8_0_get_rb_active_bitmap()
3441 switch (adev->asic_type) { in gfx_v8_0_raster_config()
3474 DRM_ERROR("unknown asic: 0x%x\n", adev->asic_type); in gfx_v8_0_raster_config()
3484 unsigned sh_per_se = max_t(unsigned, adev->gfx.config.max_sh_per_se, 1); in gfx_v8_0_write_harvested_raster_configs()
3485 unsigned num_se = max_t(unsigned, adev->gfx.config.max_shader_engines, 1); in gfx_v8_0_write_harvested_raster_configs()
3491 se_mask[0] = ((1 << rb_per_se) - 1) & rb_mask; in gfx_v8_0_write_harvested_raster_configs()
3515 unsigned pkr0_mask = ((1 << rb_per_pkr) - 1) << (se * rb_per_se); in gfx_v8_0_write_harvested_raster_configs()
3594 u32 rb_bitmap_width_per_sh = adev->gfx.config.max_backends_per_se / in gfx_v8_0_setup_rb()
3595 adev->gfx.config.max_sh_per_se; in gfx_v8_0_setup_rb()
3598 mutex_lock(&adev->grbm_idx_mutex); in gfx_v8_0_setup_rb()
3599 for (i = 0; i < adev->gfx.config.max_shader_engines; i++) { in gfx_v8_0_setup_rb()
3600 for (j = 0; j < adev->gfx.config.max_sh_per_se; j++) { in gfx_v8_0_setup_rb()
3603 active_rbs |= data << ((i * adev->gfx.config.max_sh_per_se + j) * in gfx_v8_0_setup_rb()
3609 adev->gfx.config.backend_enable_mask = active_rbs; in gfx_v8_0_setup_rb()
3610 adev->gfx.config.num_rbs = hweight32(active_rbs); in gfx_v8_0_setup_rb()
3612 num_rb_pipes = min_t(unsigned, adev->gfx.config.max_backends_per_se * in gfx_v8_0_setup_rb()
3613 adev->gfx.config.max_shader_engines, 16); in gfx_v8_0_setup_rb()
3617 if (!adev->gfx.config.backend_enable_mask || in gfx_v8_0_setup_rb()
3618 adev->gfx.config.num_rbs >= num_rb_pipes) { in gfx_v8_0_setup_rb()
3623 adev->gfx.config.backend_enable_mask, in gfx_v8_0_setup_rb()
3628 for (i = 0; i < adev->gfx.config.max_shader_engines; i++) { in gfx_v8_0_setup_rb()
3629 for (j = 0; j < adev->gfx.config.max_sh_per_se; j++) { in gfx_v8_0_setup_rb()
3631 adev->gfx.config.rb_config[i][j].rb_backend_disable = in gfx_v8_0_setup_rb()
3633 adev->gfx.config.rb_config[i][j].user_rb_backend_disable = in gfx_v8_0_setup_rb()
3635 adev->gfx.config.rb_config[i][j].raster_config = in gfx_v8_0_setup_rb()
3637 adev->gfx.config.rb_config[i][j].raster_config_1 = in gfx_v8_0_setup_rb()
3642 mutex_unlock(&adev->grbm_idx_mutex); in gfx_v8_0_setup_rb()
3647 * gfx_v8_0_init_compute_vmid - gart enable
3662 * LDS: 0x60000000'00000000 - 0x60000001'00000000 (4GB) in gfx_v8_0_init_compute_vmid()
3663 * Scratch: 0x60000001'00000000 - 0x60000002'00000000 (4GB) in gfx_v8_0_init_compute_vmid()
3664 * GPUVM: 0x60010000'00000000 - 0x60020000'00000000 (1TB) in gfx_v8_0_init_compute_vmid()
3675 mutex_lock(&adev->srbm_mutex); in gfx_v8_0_init_compute_vmid()
3676 for (i = adev->vm_manager.first_kfd_vmid; i < AMDGPU_NUM_VMID; i++) { in gfx_v8_0_init_compute_vmid()
3685 mutex_unlock(&adev->srbm_mutex); in gfx_v8_0_init_compute_vmid()
3689 for (i = adev->vm_manager.first_kfd_vmid; i < AMDGPU_NUM_VMID; i++) { in gfx_v8_0_init_compute_vmid()
3702 * Initialize all compute and user-gfx VMIDs to have no GDS, GWS, or OA in gfx_v8_0_init_gds_vmid()
3717 switch (adev->asic_type) { in gfx_v8_0_config_init()
3719 adev->gfx.config.double_offchip_lds_buf = 1; in gfx_v8_0_config_init()
3723 adev->gfx.config.double_offchip_lds_buf = 0; in gfx_v8_0_config_init()
3734 WREG32(mmGB_ADDR_CONFIG, adev->gfx.config.gb_addr_config); in gfx_v8_0_constants_init()
3735 WREG32(mmHDP_ADDR_CONFIG, adev->gfx.config.gb_addr_config); in gfx_v8_0_constants_init()
3736 WREG32(mmDMIF_ADDR_CALC, adev->gfx.config.gb_addr_config); in gfx_v8_0_constants_init()
3753 mutex_lock(&adev->srbm_mutex); in gfx_v8_0_constants_init()
3754 for (i = 0; i < adev->vm_manager.id_mgr[0].num_ids; i++) { in gfx_v8_0_constants_init()
3770 tmp = adev->gmc.shared_aperture_start >> 48; in gfx_v8_0_constants_init()
3778 mutex_unlock(&adev->srbm_mutex); in gfx_v8_0_constants_init()
3783 mutex_lock(&adev->grbm_idx_mutex); in gfx_v8_0_constants_init()
3791 (adev->gfx.config.sc_prim_fifo_size_frontend << in gfx_v8_0_constants_init()
3793 (adev->gfx.config.sc_prim_fifo_size_backend << in gfx_v8_0_constants_init()
3795 (adev->gfx.config.sc_hiz_tile_fifo_size << in gfx_v8_0_constants_init()
3797 (adev->gfx.config.sc_earlyz_tile_fifo_size << in gfx_v8_0_constants_init()
3807 mutex_unlock(&adev->grbm_idx_mutex); in gfx_v8_0_constants_init()
3816 mutex_lock(&adev->grbm_idx_mutex); in gfx_v8_0_wait_for_rlc_serdes()
3817 for (i = 0; i < adev->gfx.config.max_shader_engines; i++) { in gfx_v8_0_wait_for_rlc_serdes()
3818 for (j = 0; j < adev->gfx.config.max_sh_per_se; j++) { in gfx_v8_0_wait_for_rlc_serdes()
3820 for (k = 0; k < adev->usec_timeout; k++) { in gfx_v8_0_wait_for_rlc_serdes()
3825 if (k == adev->usec_timeout) { in gfx_v8_0_wait_for_rlc_serdes()
3828 mutex_unlock(&adev->grbm_idx_mutex); in gfx_v8_0_wait_for_rlc_serdes()
3836 mutex_unlock(&adev->grbm_idx_mutex); in gfx_v8_0_wait_for_rlc_serdes()
3842 for (k = 0; k < adev->usec_timeout; k++) { in gfx_v8_0_wait_for_rlc_serdes()
3864 adev->gfx.rlc.funcs->get_csb_buffer(adev, adev->gfx.rlc.cs_ptr); in gfx_v8_0_init_csb()
3867 adev->gfx.rlc.clear_state_gpu_addr >> 32); in gfx_v8_0_init_csb()
3869 adev->gfx.rlc.clear_state_gpu_addr & 0xfffffffc); in gfx_v8_0_init_csb()
3871 adev->gfx.rlc.clear_state_size); in gfx_v8_0_init_csb()
3934 kmemdup(adev->gfx.rlc.register_list_format, in gfx_v8_0_init_save_restore_list()
3935 adev->gfx.rlc.reg_list_format_size_bytes, GFP_KERNEL); in gfx_v8_0_init_save_restore_list()
3937 return -ENOMEM; in gfx_v8_0_init_save_restore_list()
3941 adev->gfx.rlc.reg_list_format_size_bytes >> 2, in gfx_v8_0_init_save_restore_list()
3953 for (i = 0; i < adev->gfx.rlc.reg_list_size_bytes >> 2; i++) in gfx_v8_0_init_save_restore_list()
3954 WREG32(mmRLC_SRM_ARAM_DATA, adev->gfx.rlc.register_restore[i]); in gfx_v8_0_init_save_restore_list()
3957 WREG32(mmRLC_GPM_SCRATCH_ADDR, adev->gfx.rlc.reg_list_format_start); in gfx_v8_0_init_save_restore_list()
3958 for (i = 0; i < adev->gfx.rlc.reg_list_format_size_bytes >> 2; i++) in gfx_v8_0_init_save_restore_list()
3961 list_size = adev->gfx.rlc.reg_list_size_bytes >> 2; in gfx_v8_0_init_save_restore_list()
3963 WREG32(mmRLC_GPM_SCRATCH_ADDR, adev->gfx.rlc.reg_restore_list_size); in gfx_v8_0_init_save_restore_list()
3968 adev->gfx.rlc.starting_offsets_start); in gfx_v8_0_init_save_restore_list()
4028 if ((adev->asic_type == CHIP_CARRIZO) || in gfx_v8_0_init_pg()
4029 (adev->asic_type == CHIP_STONEY)) { in gfx_v8_0_init_pg()
4033 WREG32(mmRLC_JUMP_TABLE_RESTORE, adev->gfx.rlc.cp_table_gpu_addr >> 8); in gfx_v8_0_init_pg()
4035 WREG32(mmRLC_PG_ALWAYS_ON_CU_MASK, adev->gfx.cu_info.ao_cu_mask); in gfx_v8_0_init_pg()
4036 } else if ((adev->asic_type == CHIP_POLARIS11) || in gfx_v8_0_init_pg()
4037 (adev->asic_type == CHIP_POLARIS12) || in gfx_v8_0_init_pg()
4038 (adev->asic_type == CHIP_VEGAM)) { in gfx_v8_0_init_pg()
4069 if (!(adev->flags & AMD_IS_APU)) in gfx_v8_0_rlc_start()
4082 adev->gfx.rlc.funcs->stop(adev); in gfx_v8_0_rlc_resume()
4083 adev->gfx.rlc.funcs->reset(adev); in gfx_v8_0_rlc_resume()
4085 adev->gfx.rlc.funcs->start(adev); in gfx_v8_0_rlc_resume()
4118 for (sect = vi_cs_data; sect->section != NULL; ++sect) { in gfx_v8_0_get_csb_size()
4119 for (ext = sect->section; ext->extent != NULL; ++ext) { in gfx_v8_0_get_csb_size()
4120 if (sect->id == SECT_CONTEXT) in gfx_v8_0_get_csb_size()
4121 count += 2 + ext->reg_count; in gfx_v8_0_get_csb_size()
4138 struct amdgpu_ring *ring = &adev->gfx.gfx_ring[0]; in gfx_v8_0_cp_gfx_start()
4144 WREG32(mmCP_MAX_CONTEXT, adev->gfx.config.max_hw_contexts - 1); in gfx_v8_0_cp_gfx_start()
4164 for (sect = vi_cs_data; sect->section != NULL; ++sect) { in gfx_v8_0_cp_gfx_start()
4165 for (ext = sect->section; ext->extent != NULL; ++ext) { in gfx_v8_0_cp_gfx_start()
4166 if (sect->id == SECT_CONTEXT) { in gfx_v8_0_cp_gfx_start()
4169 ext->reg_count)); in gfx_v8_0_cp_gfx_start()
4171 ext->reg_index - PACKET3_SET_CONTEXT_REG_START); in gfx_v8_0_cp_gfx_start()
4172 for (i = 0; i < ext->reg_count; i++) in gfx_v8_0_cp_gfx_start()
4173 amdgpu_ring_write(ring, ext->extent[i]); in gfx_v8_0_cp_gfx_start()
4179 amdgpu_ring_write(ring, mmPA_SC_RASTER_CONFIG - PACKET3_SET_CONTEXT_REG_START); in gfx_v8_0_cp_gfx_start()
4180 amdgpu_ring_write(ring, adev->gfx.config.rb_config[0][0].raster_config); in gfx_v8_0_cp_gfx_start()
4181 amdgpu_ring_write(ring, adev->gfx.config.rb_config[0][0].raster_config_1); in gfx_v8_0_cp_gfx_start()
4203 if (adev->asic_type == CHIP_TOPAZ) in gfx_v8_0_set_cpg_door_bell()
4208 if (ring->use_doorbell) { in gfx_v8_0_set_cpg_door_bell()
4210 DOORBELL_OFFSET, ring->doorbell_index); in gfx_v8_0_set_cpg_door_bell()
4221 if (adev->flags & AMD_IS_APU) in gfx_v8_0_set_cpg_door_bell()
4226 adev->doorbell_index.gfx_ring0); in gfx_v8_0_set_cpg_door_bell()
4247 ring = &adev->gfx.gfx_ring[0]; in gfx_v8_0_cp_gfx_resume()
4248 rb_bufsz = order_base_2(ring->ring_size / 8); in gfx_v8_0_cp_gfx_resume()
4250 tmp = REG_SET_FIELD(tmp, CP_RB0_CNTL, RB_BLKSZ, rb_bufsz - 2); in gfx_v8_0_cp_gfx_resume()
4260 ring->wptr = 0; in gfx_v8_0_cp_gfx_resume()
4261 WREG32(mmCP_RB0_WPTR, lower_32_bits(ring->wptr)); in gfx_v8_0_cp_gfx_resume()
4264 rptr_addr = ring->rptr_gpu_addr; in gfx_v8_0_cp_gfx_resume()
4268 wptr_gpu_addr = ring->wptr_gpu_addr; in gfx_v8_0_cp_gfx_resume()
4274 rb_addr = ring->gpu_addr >> 8; in gfx_v8_0_cp_gfx_resume()
4292 adev->gfx.kiq[0].ring.sched.ready = false; in gfx_v8_0_cp_compute_enable()
4301 struct amdgpu_device *adev = ring->adev; in gfx_v8_0_kiq_setting()
4306 tmp |= (ring->me << 5) | (ring->pipe << 3) | (ring->queue); in gfx_v8_0_kiq_setting()
4314 struct amdgpu_ring *kiq_ring = &adev->gfx.kiq[0].ring; in gfx_v8_0_kiq_kcq_enable()
4319 if (!test_bit(i, adev->gfx.mec_bitmap[0].queue_bitmap)) in gfx_v8_0_kiq_kcq_enable()
4333 r = amdgpu_ring_alloc(kiq_ring, (8 * adev->gfx.num_compute_rings) + 8); in gfx_v8_0_kiq_kcq_enable()
4347 for (i = 0; i < adev->gfx.num_compute_rings; i++) { in gfx_v8_0_kiq_kcq_enable()
4348 struct amdgpu_ring *ring = &adev->gfx.compute_ring[i]; in gfx_v8_0_kiq_kcq_enable()
4349 uint64_t mqd_addr = amdgpu_bo_gpu_offset(ring->mqd_obj); in gfx_v8_0_kiq_kcq_enable()
4350 uint64_t wptr_addr = ring->wptr_gpu_addr; in gfx_v8_0_kiq_kcq_enable()
4358 PACKET3_MAP_QUEUES_DOORBELL_OFFSET(ring->doorbell_index) | in gfx_v8_0_kiq_kcq_enable()
4359 PACKET3_MAP_QUEUES_QUEUE(ring->queue) | in gfx_v8_0_kiq_kcq_enable()
4360 PACKET3_MAP_QUEUES_PIPE(ring->pipe) | in gfx_v8_0_kiq_kcq_enable()
4361 PACKET3_MAP_QUEUES_ME(ring->me == 1 ? 0 : 1)); /* doorbell */ in gfx_v8_0_kiq_kcq_enable()
4379 for (i = 0; i < adev->usec_timeout; i++) { in gfx_v8_0_deactivate_hqd()
4384 if (i == adev->usec_timeout) in gfx_v8_0_deactivate_hqd()
4385 r = -ETIMEDOUT; in gfx_v8_0_deactivate_hqd()
4396 struct amdgpu_device *adev = ring->adev; in gfx_v8_0_mqd_set_priority()
4398 if (ring->funcs->type == AMDGPU_RING_TYPE_COMPUTE) { in gfx_v8_0_mqd_set_priority()
4400 mqd->cp_hqd_pipe_priority = AMDGPU_GFX_PIPE_PRIO_HIGH; in gfx_v8_0_mqd_set_priority()
4401 mqd->cp_hqd_queue_priority = in gfx_v8_0_mqd_set_priority()
4409 struct amdgpu_device *adev = ring->adev; in gfx_v8_0_mqd_init()
4410 struct vi_mqd *mqd = ring->mqd_ptr; in gfx_v8_0_mqd_init()
4414 mqd->header = 0xC0310800; in gfx_v8_0_mqd_init()
4415 mqd->compute_pipelinestat_enable = 0x00000001; in gfx_v8_0_mqd_init()
4416 mqd->compute_static_thread_mgmt_se0 = 0xffffffff; in gfx_v8_0_mqd_init()
4417 mqd->compute_static_thread_mgmt_se1 = 0xffffffff; in gfx_v8_0_mqd_init()
4418 mqd->compute_static_thread_mgmt_se2 = 0xffffffff; in gfx_v8_0_mqd_init()
4419 mqd->compute_static_thread_mgmt_se3 = 0xffffffff; in gfx_v8_0_mqd_init()
4420 mqd->compute_misc_reserved = 0x00000003; in gfx_v8_0_mqd_init()
4421 mqd->dynamic_cu_mask_addr_lo = lower_32_bits(ring->mqd_gpu_addr in gfx_v8_0_mqd_init()
4423 mqd->dynamic_cu_mask_addr_hi = upper_32_bits(ring->mqd_gpu_addr in gfx_v8_0_mqd_init()
4425 eop_base_addr = ring->eop_gpu_addr >> 8; in gfx_v8_0_mqd_init()
4426 mqd->cp_hqd_eop_base_addr_lo = eop_base_addr; in gfx_v8_0_mqd_init()
4427 mqd->cp_hqd_eop_base_addr_hi = upper_32_bits(eop_base_addr); in gfx_v8_0_mqd_init()
4432 (order_base_2(GFX8_MEC_HPD_SIZE / 4) - 1)); in gfx_v8_0_mqd_init()
4434 mqd->cp_hqd_eop_control = tmp; in gfx_v8_0_mqd_init()
4440 ring->use_doorbell ? 1 : 0); in gfx_v8_0_mqd_init()
4442 mqd->cp_hqd_pq_doorbell_control = tmp; in gfx_v8_0_mqd_init()
4445 mqd->cp_mqd_base_addr_lo = ring->mqd_gpu_addr & 0xfffffffc; in gfx_v8_0_mqd_init()
4446 mqd->cp_mqd_base_addr_hi = upper_32_bits(ring->mqd_gpu_addr); in gfx_v8_0_mqd_init()
4451 mqd->cp_mqd_control = tmp; in gfx_v8_0_mqd_init()
4454 hqd_gpu_addr = ring->gpu_addr >> 8; in gfx_v8_0_mqd_init()
4455 mqd->cp_hqd_pq_base_lo = hqd_gpu_addr; in gfx_v8_0_mqd_init()
4456 mqd->cp_hqd_pq_base_hi = upper_32_bits(hqd_gpu_addr); in gfx_v8_0_mqd_init()
4461 (order_base_2(ring->ring_size / 4) - 1)); in gfx_v8_0_mqd_init()
4463 (order_base_2(AMDGPU_GPU_PAGE_SIZE / 4) - 1)); in gfx_v8_0_mqd_init()
4471 mqd->cp_hqd_pq_control = tmp; in gfx_v8_0_mqd_init()
4474 wb_gpu_addr = ring->rptr_gpu_addr; in gfx_v8_0_mqd_init()
4475 mqd->cp_hqd_pq_rptr_report_addr_lo = wb_gpu_addr & 0xfffffffc; in gfx_v8_0_mqd_init()
4476 mqd->cp_hqd_pq_rptr_report_addr_hi = in gfx_v8_0_mqd_init()
4480 wb_gpu_addr = ring->wptr_gpu_addr; in gfx_v8_0_mqd_init()
4481 mqd->cp_hqd_pq_wptr_poll_addr_lo = wb_gpu_addr & 0xfffffffc; in gfx_v8_0_mqd_init()
4482 mqd->cp_hqd_pq_wptr_poll_addr_hi = upper_32_bits(wb_gpu_addr) & 0xffff; in gfx_v8_0_mqd_init()
4486 if (ring->use_doorbell) { in gfx_v8_0_mqd_init()
4489 DOORBELL_OFFSET, ring->doorbell_index); in gfx_v8_0_mqd_init()
4499 mqd->cp_hqd_pq_doorbell_control = tmp; in gfx_v8_0_mqd_init()
4502 ring->wptr = 0; in gfx_v8_0_mqd_init()
4503 mqd->cp_hqd_pq_wptr = ring->wptr; in gfx_v8_0_mqd_init()
4504 mqd->cp_hqd_pq_rptr = RREG32(mmCP_HQD_PQ_RPTR); in gfx_v8_0_mqd_init()
4507 mqd->cp_hqd_vmid = 0; in gfx_v8_0_mqd_init()
4511 mqd->cp_hqd_persistent_state = tmp; in gfx_v8_0_mqd_init()
4517 mqd->cp_hqd_ib_control = tmp; in gfx_v8_0_mqd_init()
4521 mqd->cp_hqd_iq_timer = tmp; in gfx_v8_0_mqd_init()
4525 mqd->cp_hqd_ctx_save_control = tmp; in gfx_v8_0_mqd_init()
4528 mqd->cp_hqd_eop_rptr = RREG32(mmCP_HQD_EOP_RPTR); in gfx_v8_0_mqd_init()
4529 mqd->cp_hqd_eop_wptr = RREG32(mmCP_HQD_EOP_WPTR); in gfx_v8_0_mqd_init()
4530 mqd->cp_hqd_ctx_save_base_addr_lo = RREG32(mmCP_HQD_CTX_SAVE_BASE_ADDR_LO); in gfx_v8_0_mqd_init()
4531 mqd->cp_hqd_ctx_save_base_addr_hi = RREG32(mmCP_HQD_CTX_SAVE_BASE_ADDR_HI); in gfx_v8_0_mqd_init()
4532 mqd->cp_hqd_cntl_stack_offset = RREG32(mmCP_HQD_CNTL_STACK_OFFSET); in gfx_v8_0_mqd_init()
4533 mqd->cp_hqd_cntl_stack_size = RREG32(mmCP_HQD_CNTL_STACK_SIZE); in gfx_v8_0_mqd_init()
4534 mqd->cp_hqd_wg_state_offset = RREG32(mmCP_HQD_WG_STATE_OFFSET); in gfx_v8_0_mqd_init()
4535 mqd->cp_hqd_ctx_save_size = RREG32(mmCP_HQD_CTX_SAVE_SIZE); in gfx_v8_0_mqd_init()
4536 mqd->cp_hqd_eop_done_events = RREG32(mmCP_HQD_EOP_EVENTS); in gfx_v8_0_mqd_init()
4537 mqd->cp_hqd_error = RREG32(mmCP_HQD_ERROR); in gfx_v8_0_mqd_init()
4538 mqd->cp_hqd_eop_wptr_mem = RREG32(mmCP_HQD_EOP_WPTR_MEM); in gfx_v8_0_mqd_init()
4539 mqd->cp_hqd_eop_dones = RREG32(mmCP_HQD_EOP_DONES); in gfx_v8_0_mqd_init()
4543 mqd->cp_hqd_quantum = RREG32(mmCP_HQD_QUANTUM); in gfx_v8_0_mqd_init()
4548 if (ring->funcs->type == AMDGPU_RING_TYPE_KIQ) in gfx_v8_0_mqd_init()
4549 mqd->cp_hqd_active = 1; in gfx_v8_0_mqd_init()
4561 mqd_data = &mqd->cp_mqd_base_addr_lo; in gfx_v8_0_mqd_commit()
4568 WREG32(mqd_reg, mqd_data[mqd_reg - mmCP_MQD_BASE_ADDR]); in gfx_v8_0_mqd_commit()
4572 * on ASICs that do not support context-save. in gfx_v8_0_mqd_commit()
4575 if (adev->asic_type != CHIP_TONGA) { in gfx_v8_0_mqd_commit()
4576 WREG32(mmCP_HQD_EOP_RPTR, mqd->cp_hqd_eop_rptr); in gfx_v8_0_mqd_commit()
4577 WREG32(mmCP_HQD_EOP_WPTR, mqd->cp_hqd_eop_wptr); in gfx_v8_0_mqd_commit()
4578 WREG32(mmCP_HQD_EOP_WPTR_MEM, mqd->cp_hqd_eop_wptr_mem); in gfx_v8_0_mqd_commit()
4582 WREG32(mqd_reg, mqd_data[mqd_reg - mmCP_MQD_BASE_ADDR]); in gfx_v8_0_mqd_commit()
4586 WREG32(mqd_reg, mqd_data[mqd_reg - mmCP_MQD_BASE_ADDR]); in gfx_v8_0_mqd_commit()
4593 struct amdgpu_device *adev = ring->adev; in gfx_v8_0_kiq_init_queue()
4594 struct vi_mqd *mqd = ring->mqd_ptr; in gfx_v8_0_kiq_init_queue()
4600 if (adev->gfx.kiq[0].mqd_backup) in gfx_v8_0_kiq_init_queue()
4601 memcpy(mqd, adev->gfx.kiq[0].mqd_backup, sizeof(struct vi_mqd_allocation)); in gfx_v8_0_kiq_init_queue()
4604 ring->wptr = 0; in gfx_v8_0_kiq_init_queue()
4606 mutex_lock(&adev->srbm_mutex); in gfx_v8_0_kiq_init_queue()
4607 vi_srbm_select(adev, ring->me, ring->pipe, ring->queue, 0); in gfx_v8_0_kiq_init_queue()
4610 mutex_unlock(&adev->srbm_mutex); in gfx_v8_0_kiq_init_queue()
4613 ((struct vi_mqd_allocation *)mqd)->dynamic_cu_mask = 0xFFFFFFFF; in gfx_v8_0_kiq_init_queue()
4614 ((struct vi_mqd_allocation *)mqd)->dynamic_rb_mask = 0xFFFFFFFF; in gfx_v8_0_kiq_init_queue()
4615 if (amdgpu_sriov_vf(adev) && adev->in_suspend) in gfx_v8_0_kiq_init_queue()
4617 mutex_lock(&adev->srbm_mutex); in gfx_v8_0_kiq_init_queue()
4618 vi_srbm_select(adev, ring->me, ring->pipe, ring->queue, 0); in gfx_v8_0_kiq_init_queue()
4622 mutex_unlock(&adev->srbm_mutex); in gfx_v8_0_kiq_init_queue()
4624 if (adev->gfx.kiq[0].mqd_backup) in gfx_v8_0_kiq_init_queue()
4625 memcpy(adev->gfx.kiq[0].mqd_backup, mqd, sizeof(struct vi_mqd_allocation)); in gfx_v8_0_kiq_init_queue()
4633 struct amdgpu_device *adev = ring->adev; in gfx_v8_0_kcq_init_queue()
4634 struct vi_mqd *mqd = ring->mqd_ptr; in gfx_v8_0_kcq_init_queue()
4635 int mqd_idx = ring - &adev->gfx.compute_ring[0]; in gfx_v8_0_kcq_init_queue()
4637 if (!amdgpu_in_reset(adev) && !adev->in_suspend) { in gfx_v8_0_kcq_init_queue()
4639 ((struct vi_mqd_allocation *)mqd)->dynamic_cu_mask = 0xFFFFFFFF; in gfx_v8_0_kcq_init_queue()
4640 ((struct vi_mqd_allocation *)mqd)->dynamic_rb_mask = 0xFFFFFFFF; in gfx_v8_0_kcq_init_queue()
4641 mutex_lock(&adev->srbm_mutex); in gfx_v8_0_kcq_init_queue()
4642 vi_srbm_select(adev, ring->me, ring->pipe, ring->queue, 0); in gfx_v8_0_kcq_init_queue()
4645 mutex_unlock(&adev->srbm_mutex); in gfx_v8_0_kcq_init_queue()
4647 if (adev->gfx.mec.mqd_backup[mqd_idx]) in gfx_v8_0_kcq_init_queue()
4648 memcpy(adev->gfx.mec.mqd_backup[mqd_idx], mqd, sizeof(struct vi_mqd_allocation)); in gfx_v8_0_kcq_init_queue()
4651 if (adev->gfx.mec.mqd_backup[mqd_idx]) in gfx_v8_0_kcq_init_queue()
4652 memcpy(mqd, adev->gfx.mec.mqd_backup[mqd_idx], sizeof(struct vi_mqd_allocation)); in gfx_v8_0_kcq_init_queue()
4654 ring->wptr = 0; in gfx_v8_0_kcq_init_queue()
4662 if (adev->asic_type > CHIP_TONGA) { in gfx_v8_0_set_mec_doorbell_range()
4663 WREG32(mmCP_MEC_DOORBELL_RANGE_LOWER, adev->doorbell_index.kiq << 2); in gfx_v8_0_set_mec_doorbell_range()
4664 WREG32(mmCP_MEC_DOORBELL_RANGE_UPPER, adev->doorbell_index.mec_ring7 << 2); in gfx_v8_0_set_mec_doorbell_range()
4675 ring = &adev->gfx.kiq[0].ring; in gfx_v8_0_kiq_resume()
4677 r = amdgpu_bo_reserve(ring->mqd_obj, false); in gfx_v8_0_kiq_resume()
4681 r = amdgpu_bo_kmap(ring->mqd_obj, &ring->mqd_ptr); in gfx_v8_0_kiq_resume()
4683 amdgpu_bo_unreserve(ring->mqd_obj); in gfx_v8_0_kiq_resume()
4688 amdgpu_bo_kunmap(ring->mqd_obj); in gfx_v8_0_kiq_resume()
4689 ring->mqd_ptr = NULL; in gfx_v8_0_kiq_resume()
4690 amdgpu_bo_unreserve(ring->mqd_obj); in gfx_v8_0_kiq_resume()
4701 for (i = 0; i < adev->gfx.num_compute_rings; i++) { in gfx_v8_0_kcq_resume()
4702 ring = &adev->gfx.compute_ring[i]; in gfx_v8_0_kcq_resume()
4704 r = amdgpu_bo_reserve(ring->mqd_obj, false); in gfx_v8_0_kcq_resume()
4707 r = amdgpu_bo_kmap(ring->mqd_obj, &ring->mqd_ptr); in gfx_v8_0_kcq_resume()
4710 amdgpu_bo_kunmap(ring->mqd_obj); in gfx_v8_0_kcq_resume()
4711 ring->mqd_ptr = NULL; in gfx_v8_0_kcq_resume()
4713 amdgpu_bo_unreserve(ring->mqd_obj); in gfx_v8_0_kcq_resume()
4734 ring = &adev->gfx.gfx_ring[0]; in gfx_v8_0_cp_test_all_rings()
4739 ring = &adev->gfx.kiq[0].ring; in gfx_v8_0_cp_test_all_rings()
4744 for (i = 0; i < adev->gfx.num_compute_rings; i++) { in gfx_v8_0_cp_test_all_rings()
4745 ring = &adev->gfx.compute_ring[i]; in gfx_v8_0_cp_test_all_rings()
4756 if (!(adev->flags & AMD_IS_APU)) in gfx_v8_0_cp_resume()
4794 r = adev->gfx.rlc.funcs->resume(adev); in gfx_v8_0_hw_init()
4806 struct amdgpu_ring *kiq_ring = &adev->gfx.kiq[0].ring; in gfx_v8_0_kcq_disable()
4808 r = amdgpu_ring_alloc(kiq_ring, 6 * adev->gfx.num_compute_rings); in gfx_v8_0_kcq_disable()
4812 for (i = 0; i < adev->gfx.num_compute_rings; i++) { in gfx_v8_0_kcq_disable()
4813 struct amdgpu_ring *ring = &adev->gfx.compute_ring[i]; in gfx_v8_0_kcq_disable()
4821 amdgpu_ring_write(kiq_ring, PACKET3_UNMAP_QUEUES_DOORBELL_OFFSET0(ring->doorbell_index)); in gfx_v8_0_kcq_disable()
4859 for (i = 0; i < adev->usec_timeout; i++) { in gfx_v8_0_wait_for_rlc_idle()
4865 return -ETIMEDOUT; in gfx_v8_0_wait_for_rlc_idle()
4873 for (i = 0; i < adev->usec_timeout; i++) { in gfx_v8_0_wait_for_idle()
4879 return -ETIMEDOUT; in gfx_v8_0_wait_for_idle()
4886 amdgpu_irq_put(adev, &adev->gfx.priv_reg_irq, 0); in gfx_v8_0_hw_fini()
4887 amdgpu_irq_put(adev, &adev->gfx.priv_inst_irq, 0); in gfx_v8_0_hw_fini()
4889 amdgpu_irq_put(adev, &adev->gfx.cp_ecc_error_irq, 0); in gfx_v8_0_hw_fini()
4891 amdgpu_irq_put(adev, &adev->gfx.sq_irq, 0); in gfx_v8_0_hw_fini()
4906 adev->gfx.rlc.funcs->stop(adev); in gfx_v8_0_hw_fini()
4976 adev->gfx.grbm_soft_reset = grbm_soft_reset; in gfx_v8_0_check_soft_reset()
4977 adev->gfx.srbm_soft_reset = srbm_soft_reset; in gfx_v8_0_check_soft_reset()
4980 adev->gfx.grbm_soft_reset = 0; in gfx_v8_0_check_soft_reset()
4981 adev->gfx.srbm_soft_reset = 0; in gfx_v8_0_check_soft_reset()
4991 if ((!adev->gfx.grbm_soft_reset) && in gfx_v8_0_pre_soft_reset()
4992 (!adev->gfx.srbm_soft_reset)) in gfx_v8_0_pre_soft_reset()
4995 grbm_soft_reset = adev->gfx.grbm_soft_reset; in gfx_v8_0_pre_soft_reset()
4998 adev->gfx.rlc.funcs->stop(adev); in gfx_v8_0_pre_soft_reset()
5011 for (i = 0; i < adev->gfx.num_compute_rings; i++) { in gfx_v8_0_pre_soft_reset()
5012 struct amdgpu_ring *ring = &adev->gfx.compute_ring[i]; in gfx_v8_0_pre_soft_reset()
5014 mutex_lock(&adev->srbm_mutex); in gfx_v8_0_pre_soft_reset()
5015 vi_srbm_select(adev, ring->me, ring->pipe, ring->queue, 0); in gfx_v8_0_pre_soft_reset()
5018 mutex_unlock(&adev->srbm_mutex); in gfx_v8_0_pre_soft_reset()
5033 if ((!adev->gfx.grbm_soft_reset) && in gfx_v8_0_soft_reset()
5034 (!adev->gfx.srbm_soft_reset)) in gfx_v8_0_soft_reset()
5037 grbm_soft_reset = adev->gfx.grbm_soft_reset; in gfx_v8_0_soft_reset()
5038 srbm_soft_reset = adev->gfx.srbm_soft_reset; in gfx_v8_0_soft_reset()
5051 dev_info(adev->dev, "GRBM_SOFT_RESET=0x%08X\n", tmp); in gfx_v8_0_soft_reset()
5065 dev_info(adev->dev, "SRBM_SOFT_RESET=0x%08X\n", tmp); in gfx_v8_0_soft_reset()
5094 if ((!adev->gfx.grbm_soft_reset) && in gfx_v8_0_post_soft_reset()
5095 (!adev->gfx.srbm_soft_reset)) in gfx_v8_0_post_soft_reset()
5098 grbm_soft_reset = adev->gfx.grbm_soft_reset; in gfx_v8_0_post_soft_reset()
5106 for (i = 0; i < adev->gfx.num_compute_rings; i++) { in gfx_v8_0_post_soft_reset()
5107 struct amdgpu_ring *ring = &adev->gfx.compute_ring[i]; in gfx_v8_0_post_soft_reset()
5109 mutex_lock(&adev->srbm_mutex); in gfx_v8_0_post_soft_reset()
5110 vi_srbm_select(adev, ring->me, ring->pipe, ring->queue, 0); in gfx_v8_0_post_soft_reset()
5113 mutex_unlock(&adev->srbm_mutex); in gfx_v8_0_post_soft_reset()
5125 adev->gfx.rlc.funcs->start(adev); in gfx_v8_0_post_soft_reset()
5131 * gfx_v8_0_get_gpu_clock_counter - return GPU clock counter snapshot
5142 mutex_lock(&adev->gfx.gpu_clock_mutex); in gfx_v8_0_get_gpu_clock_counter()
5146 mutex_unlock(&adev->gfx.gpu_clock_mutex); in gfx_v8_0_get_gpu_clock_counter()
5186 amdgpu_ring_write(ring, (1 << (oa_size + oa_base)) - (1 << oa_base)); in gfx_v8_0_ring_emit_gds_switch()
5210 while (num--) in wave_read_regs()
5261 adev->gfx.xcc_mask = 1; in gfx_v8_0_early_init()
5262 adev->gfx.num_gfx_rings = GFX8_NUM_GFX_RINGS; in gfx_v8_0_early_init()
5263 adev->gfx.num_compute_rings = min(amdgpu_gfx_get_num_kcq(adev), in gfx_v8_0_early_init()
5265 adev->gfx.funcs = &gfx_v8_0_gfx_funcs; in gfx_v8_0_early_init()
5279 r = amdgpu_irq_get(adev, &adev->gfx.priv_reg_irq, 0); in gfx_v8_0_late_init()
5283 r = amdgpu_irq_get(adev, &adev->gfx.priv_inst_irq, 0); in gfx_v8_0_late_init()
5292 r = amdgpu_irq_get(adev, &adev->gfx.cp_ecc_error_irq, 0); in gfx_v8_0_late_init()
5298 r = amdgpu_irq_get(adev, &adev->gfx.sq_irq, 0); in gfx_v8_0_late_init()
5312 if ((adev->asic_type == CHIP_POLARIS11) || in gfx_v8_0_enable_gfx_static_mg_power_gating()
5313 (adev->asic_type == CHIP_POLARIS12) || in gfx_v8_0_enable_gfx_static_mg_power_gating()
5314 (adev->asic_type == CHIP_VEGAM)) in gfx_v8_0_enable_gfx_static_mg_power_gating()
5352 if ((adev->pg_flags & AMD_PG_SUPPORT_GFX_PG) && enable) { in cz_update_gfx_cg_power_gating()
5354 if (adev->pg_flags & AMD_PG_SUPPORT_GFX_PIPELINE) in cz_update_gfx_cg_power_gating()
5371 if (adev->pg_flags & (AMD_PG_SUPPORT_GFX_SMG | in gfx_v8_0_set_powergating_state()
5376 switch (adev->asic_type) { in gfx_v8_0_set_powergating_state()
5380 if (adev->pg_flags & AMD_PG_SUPPORT_RLC_SMU_HS) { in gfx_v8_0_set_powergating_state()
5387 if (adev->pg_flags & AMD_PG_SUPPORT_CP) in gfx_v8_0_set_powergating_state()
5394 if ((adev->pg_flags & AMD_PG_SUPPORT_GFX_SMG) && enable) in gfx_v8_0_set_powergating_state()
5399 if ((adev->pg_flags & AMD_PG_SUPPORT_GFX_DMG) && enable) in gfx_v8_0_set_powergating_state()
5407 if ((adev->pg_flags & AMD_PG_SUPPORT_GFX_SMG) && enable) in gfx_v8_0_set_powergating_state()
5412 if ((adev->pg_flags & AMD_PG_SUPPORT_GFX_DMG) && enable) in gfx_v8_0_set_powergating_state()
5417 if ((adev->pg_flags & AMD_PG_SUPPORT_GFX_QUICK_MG) && enable) in gfx_v8_0_set_powergating_state()
5425 if (adev->pg_flags & (AMD_PG_SUPPORT_GFX_SMG | in gfx_v8_0_set_powergating_state()
5486 if (adev->asic_type == CHIP_STONEY) in gfx_v8_0_send_serdes_cmd()
5545 for (i = 0; i < adev->usec_timeout; i++) { in gfx_v8_0_set_safe_mode()
5554 for (i = 0; i < adev->usec_timeout; i++) { in gfx_v8_0_set_safe_mode()
5571 for (i = 0; i < adev->usec_timeout; i++) { in gfx_v8_0_unset_safe_mode()
5623 if (enable && (adev->cg_flags & AMD_CG_SUPPORT_GFX_MGCG)) { in gfx_v8_0_update_medium_grain_clock_gating()
5624 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_MGLS) { in gfx_v8_0_update_medium_grain_clock_gating()
5625 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_RLC_LS) in gfx_v8_0_update_medium_grain_clock_gating()
5626 /* 1 - RLC memory Light sleep */ in gfx_v8_0_update_medium_grain_clock_gating()
5629 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_CP_LS) in gfx_v8_0_update_medium_grain_clock_gating()
5633 /* 3 - RLC_CGTT_MGCG_OVERRIDE */ in gfx_v8_0_update_medium_grain_clock_gating()
5635 if (adev->flags & AMD_IS_APU) in gfx_v8_0_update_medium_grain_clock_gating()
5648 /* 4 - wait for RLC_SERDES_CU_MASTER & RLC_SERDES_NONCU_MASTER idle */ in gfx_v8_0_update_medium_grain_clock_gating()
5651 /* 5 - clear mgcg override */ in gfx_v8_0_update_medium_grain_clock_gating()
5654 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_CGTS) { in gfx_v8_0_update_medium_grain_clock_gating()
5655 /* 6 - Enable CGTS(Tree Shade) MGCG /MGLS */ in gfx_v8_0_update_medium_grain_clock_gating()
5661 if ((adev->cg_flags & AMD_CG_SUPPORT_GFX_MGLS) && in gfx_v8_0_update_medium_grain_clock_gating()
5662 (adev->cg_flags & AMD_CG_SUPPORT_GFX_CGTS_LS)) in gfx_v8_0_update_medium_grain_clock_gating()
5671 /* 7 - wait for RLC_SERDES_CU_MASTER & RLC_SERDES_NONCU_MASTER idle */ in gfx_v8_0_update_medium_grain_clock_gating()
5674 /* 1 - MGCG_OVERRIDE[0] for CP and MGCG_OVERRIDE[1] for RLC */ in gfx_v8_0_update_medium_grain_clock_gating()
5683 /* 2 - disable MGLS in RLC */ in gfx_v8_0_update_medium_grain_clock_gating()
5690 /* 3 - disable MGLS in CP */ in gfx_v8_0_update_medium_grain_clock_gating()
5697 /* 4 - Disable CGTS(Tree Shade) MGCG and MGLS */ in gfx_v8_0_update_medium_grain_clock_gating()
5704 /* 5 - wait for RLC_SERDES_CU_MASTER & RLC_SERDES_NONCU_MASTER idle */ in gfx_v8_0_update_medium_grain_clock_gating()
5707 /* 6 - set mgcg override */ in gfx_v8_0_update_medium_grain_clock_gating()
5712 /* 7- wait for RLC_SERDES_CU_MASTER & RLC_SERDES_NONCU_MASTER idle */ in gfx_v8_0_update_medium_grain_clock_gating()
5728 if (enable && (adev->cg_flags & AMD_CG_SUPPORT_GFX_CGCG)) { in gfx_v8_0_update_coarse_grain_clock_gating()
5737 /* 2 - clear cgcg override */ in gfx_v8_0_update_coarse_grain_clock_gating()
5743 /* 3 - write cmd to set CGLS */ in gfx_v8_0_update_coarse_grain_clock_gating()
5746 /* 4 - enable cgcg */ in gfx_v8_0_update_coarse_grain_clock_gating()
5749 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_CGLS) { in gfx_v8_0_update_coarse_grain_clock_gating()
5815 /* CGCG/CGLS should be enabled after MGCG/MGLS/TS(CG/LS) in gfx_v8_0_update_gfx_clock_gating()
5816 * === MGCG + MGLS + TS(CG/LS) === in gfx_v8_0_update_gfx_clock_gating()
5821 /* CGCG/CGLS should be disabled before MGCG/MGLS/TS(CG/LS) in gfx_v8_0_update_gfx_clock_gating()
5836 if (adev->cg_flags & (AMD_CG_SUPPORT_GFX_CGCG | AMD_CG_SUPPORT_GFX_CGLS)) { in gfx_v8_0_tonga_update_gfx_clock_gating()
5837 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_CGLS) { in gfx_v8_0_tonga_update_gfx_clock_gating()
5841 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_CGCG) { in gfx_v8_0_tonga_update_gfx_clock_gating()
5855 if (adev->cg_flags & (AMD_CG_SUPPORT_GFX_MGCG | AMD_CG_SUPPORT_GFX_MGLS)) { in gfx_v8_0_tonga_update_gfx_clock_gating()
5856 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_MGLS) { in gfx_v8_0_tonga_update_gfx_clock_gating()
5861 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_MGCG) { in gfx_v8_0_tonga_update_gfx_clock_gating()
5886 if (adev->cg_flags & (AMD_CG_SUPPORT_GFX_CGCG | AMD_CG_SUPPORT_GFX_CGLS)) { in gfx_v8_0_polaris_update_gfx_clock_gating()
5887 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_CGLS) { in gfx_v8_0_polaris_update_gfx_clock_gating()
5891 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_CGCG) { in gfx_v8_0_polaris_update_gfx_clock_gating()
5905 if (adev->cg_flags & (AMD_CG_SUPPORT_GFX_3D_CGCG | AMD_CG_SUPPORT_GFX_3D_CGLS)) { in gfx_v8_0_polaris_update_gfx_clock_gating()
5906 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_3D_CGLS) { in gfx_v8_0_polaris_update_gfx_clock_gating()
5910 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_3D_CGCG) { in gfx_v8_0_polaris_update_gfx_clock_gating()
5924 if (adev->cg_flags & (AMD_CG_SUPPORT_GFX_MGCG | AMD_CG_SUPPORT_GFX_MGLS)) { in gfx_v8_0_polaris_update_gfx_clock_gating()
5925 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_MGLS) { in gfx_v8_0_polaris_update_gfx_clock_gating()
5930 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_MGCG) { in gfx_v8_0_polaris_update_gfx_clock_gating()
5945 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_RLC_LS) { in gfx_v8_0_polaris_update_gfx_clock_gating()
5960 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_CP_LS) { in gfx_v8_0_polaris_update_gfx_clock_gating()
5985 switch (adev->asic_type) { in gfx_v8_0_set_clockgating_state()
6009 return *ring->rptr_cpu_addr; in gfx_v8_0_ring_get_rptr()
6014 struct amdgpu_device *adev = ring->adev; in gfx_v8_0_ring_get_wptr_gfx()
6016 if (ring->use_doorbell) in gfx_v8_0_ring_get_wptr_gfx()
6018 return *ring->wptr_cpu_addr; in gfx_v8_0_ring_get_wptr_gfx()
6025 struct amdgpu_device *adev = ring->adev; in gfx_v8_0_ring_set_wptr_gfx()
6027 if (ring->use_doorbell) { in gfx_v8_0_ring_set_wptr_gfx()
6029 *ring->wptr_cpu_addr = lower_32_bits(ring->wptr); in gfx_v8_0_ring_set_wptr_gfx()
6030 WDOORBELL32(ring->doorbell_index, lower_32_bits(ring->wptr)); in gfx_v8_0_ring_set_wptr_gfx()
6032 WREG32(mmCP_RB0_WPTR, lower_32_bits(ring->wptr)); in gfx_v8_0_ring_set_wptr_gfx()
6041 if ((ring->funcs->type == AMDGPU_RING_TYPE_COMPUTE) || in gfx_v8_0_ring_emit_hdp_flush()
6042 (ring->funcs->type == AMDGPU_RING_TYPE_KIQ)) { in gfx_v8_0_ring_emit_hdp_flush()
6043 switch (ring->me) { in gfx_v8_0_ring_emit_hdp_flush()
6045 ref_and_mask = GPU_HDP_FLUSH_DONE__CP2_MASK << ring->pipe; in gfx_v8_0_ring_emit_hdp_flush()
6048 ref_and_mask = GPU_HDP_FLUSH_DONE__CP6_MASK << ring->pipe; in gfx_v8_0_ring_emit_hdp_flush()
6089 if (ib->flags & AMDGPU_IB_FLAG_CE) in gfx_v8_0_ring_emit_ib_gfx()
6094 control |= ib->length_dw | (vmid << 24); in gfx_v8_0_ring_emit_ib_gfx()
6096 if (amdgpu_sriov_vf(ring->adev) && (ib->flags & AMDGPU_IB_FLAG_PREEMPT)) { in gfx_v8_0_ring_emit_ib_gfx()
6099 if (!(ib->flags & AMDGPU_IB_FLAG_CE) && vmid) in gfx_v8_0_ring_emit_ib_gfx()
6108 (ib->gpu_addr & 0xFFFFFFFC)); in gfx_v8_0_ring_emit_ib_gfx()
6109 amdgpu_ring_write(ring, upper_32_bits(ib->gpu_addr) & 0xFFFF); in gfx_v8_0_ring_emit_ib_gfx()
6119 u32 control = INDIRECT_BUFFER_VALID | ib->length_dw | (vmid << 24); in gfx_v8_0_ring_emit_ib_compute()
6131 if (ib->flags & AMDGPU_IB_FLAG_RESET_GDS_MAX_WAVE_ID) { in gfx_v8_0_ring_emit_ib_compute()
6133 amdgpu_ring_write(ring, mmGDS_COMPUTE_MAX_WAVE_ID - PACKET3_SET_CONFIG_REG_START); in gfx_v8_0_ring_emit_ib_compute()
6134 amdgpu_ring_write(ring, ring->adev->gds.gds_compute_max_wave_id); in gfx_v8_0_ring_emit_ib_compute()
6142 (ib->gpu_addr & 0xFFFFFFFC)); in gfx_v8_0_ring_emit_ib_compute()
6143 amdgpu_ring_write(ring, upper_32_bits(ib->gpu_addr) & 0xFFFF); in gfx_v8_0_ring_emit_ib_compute()
6166 amdgpu_ring_write(ring, lower_32_bits(seq - 1)); in gfx_v8_0_ring_emit_fence_gfx()
6167 amdgpu_ring_write(ring, upper_32_bits(seq - 1)); in gfx_v8_0_ring_emit_fence_gfx()
6170 * EVENT_WRITE_EOP - flush caches, send int */ in gfx_v8_0_ring_emit_fence_gfx()
6188 int usepfp = (ring->funcs->type == AMDGPU_RING_TYPE_GFX); in gfx_v8_0_ring_emit_pipeline_sync()
6189 uint32_t seq = ring->fence_drv.sync_seq; in gfx_v8_0_ring_emit_pipeline_sync()
6190 uint64_t addr = ring->fence_drv.gpu_addr; in gfx_v8_0_ring_emit_pipeline_sync()
6206 int usepfp = (ring->funcs->type == AMDGPU_RING_TYPE_GFX); in gfx_v8_0_ring_emit_vm_flush()
6231 return *ring->wptr_cpu_addr; in gfx_v8_0_ring_get_wptr_compute()
6236 struct amdgpu_device *adev = ring->adev; in gfx_v8_0_ring_set_wptr_compute()
6239 *ring->wptr_cpu_addr = lower_32_bits(ring->wptr); in gfx_v8_0_ring_set_wptr_compute()
6240 WDOORBELL32(ring->doorbell_index, lower_32_bits(ring->wptr)); in gfx_v8_0_ring_set_wptr_compute()
6250 /* RELEASE_MEM - flush caches, send int */ in gfx_v8_0_ring_emit_fence_compute()
6299 if (amdgpu_sriov_vf(ring->adev)) in gfx_v8_ring_emit_cntxcntl()
6338 ret = ring->wptr & ring->buf_mask; in gfx_v8_0_ring_emit_init_cond_exec()
6347 struct amdgpu_device *adev = ring->adev; in gfx_v8_0_ring_emit_rreg()
6355 amdgpu_ring_write(ring, lower_32_bits(adev->wb.gpu_addr + in gfx_v8_0_ring_emit_rreg()
6357 amdgpu_ring_write(ring, upper_32_bits(adev->wb.gpu_addr + in gfx_v8_0_ring_emit_rreg()
6366 switch (ring->funcs->type) { in gfx_v8_0_ring_emit_wreg()
6388 uint32_t inv) in gfx_v8_0_wait_reg_mem() argument
6404 amdgpu_ring_write(ring, inv); /* poll interval */ in gfx_v8_0_wait_reg_mem()
6415 struct amdgpu_device *adev = ring->adev; in gfx_v8_0_ring_soft_recovery()
6561 return -EINVAL; in gfx_v8_0_set_cp_ecc_int_state()
6606 return -EINVAL; in gfx_v8_0_set_sq_int_state()
6624 me_id = (entry->ring_id & 0x0c) >> 2; in gfx_v8_0_eop_irq()
6625 pipe_id = (entry->ring_id & 0x03) >> 0; in gfx_v8_0_eop_irq()
6626 queue_id = (entry->ring_id & 0x70) >> 4; in gfx_v8_0_eop_irq()
6630 amdgpu_fence_process(&adev->gfx.gfx_ring[0]); in gfx_v8_0_eop_irq()
6634 for (i = 0; i < adev->gfx.num_compute_rings; i++) { in gfx_v8_0_eop_irq()
6635 ring = &adev->gfx.compute_ring[i]; in gfx_v8_0_eop_irq()
6636 /* Per-queue interrupt is supported for MEC starting from VI. in gfx_v8_0_eop_irq()
6639 if ((ring->me == me_id) && (ring->pipe == pipe_id) && (ring->queue == queue_id)) in gfx_v8_0_eop_irq()
6654 me_id = (entry->ring_id & 0x0c) >> 2; in gfx_v8_0_fault()
6655 pipe_id = (entry->ring_id & 0x03) >> 0; in gfx_v8_0_fault()
6656 queue_id = (entry->ring_id & 0x70) >> 4; in gfx_v8_0_fault()
6660 drm_sched_fault(&adev->gfx.gfx_ring[0].sched); in gfx_v8_0_fault()
6664 for (i = 0; i < adev->gfx.num_compute_rings; i++) { in gfx_v8_0_fault()
6665 ring = &adev->gfx.compute_ring[i]; in gfx_v8_0_fault()
6666 if (ring->me == me_id && ring->pipe == pipe_id && in gfx_v8_0_fault()
6667 ring->queue == queue_id) in gfx_v8_0_fault()
6668 drm_sched_fault(&ring->sched); in gfx_v8_0_fault()
6705 int sq_edc_source = -1; in gfx_v8_0_parse_sq_irq()
6740 mutex_lock(&adev->grbm_idx_mutex); in gfx_v8_0_parse_sq_irq()
6746 mutex_unlock(&adev->grbm_idx_mutex); in gfx_v8_0_parse_sq_irq()
6763 (sq_edc_source != -1) ? sq_edc_source_names[sq_edc_source] : "unavailable" in gfx_v8_0_parse_sq_irq()
6777 gfx_v8_0_parse_sq_irq(adev, sq_work->ih_data, true); in gfx_v8_0_sq_irq_work_func()
6784 unsigned ih_data = entry->src_data[0]; in gfx_v8_0_sq_irq()
6791 if (work_pending(&adev->gfx.sq_work.work)) { in gfx_v8_0_sq_irq()
6794 adev->gfx.sq_work.ih_data = ih_data; in gfx_v8_0_sq_irq()
6795 schedule_work(&adev->gfx.sq_work.work); in gfx_v8_0_sq_irq()
6830 /* mmSPI_WCL_PIPE_PERCENT_CS[0-7]_DEFAULT values are same */
6865 struct amdgpu_device *adev = ring->adev; in gfx_v8_0_emit_wave_limit()
6879 * amdgpu controls only 1st ME(0-3 CS pipes). in gfx_v8_0_emit_wave_limit()
6881 for (i = 0; i < adev->gfx.mec.num_pipe_per_mec; i++) { in gfx_v8_0_emit_wave_limit()
6882 if (i != ring->pipe) in gfx_v8_0_emit_wave_limit()
6891 struct amdgpu_device *adev = ring->adev; in gfx_v8_0_reset_kgq()
6892 struct amdgpu_kiq *kiq = &adev->gfx.kiq[0]; in gfx_v8_0_reset_kgq()
6893 struct amdgpu_ring *kiq_ring = &kiq->ring; in gfx_v8_0_reset_kgq()
6899 return -EINVAL; in gfx_v8_0_reset_kgq()
6901 if (!kiq->pmf || !kiq->pmf->kiq_unmap_queues) in gfx_v8_0_reset_kgq()
6902 return -EINVAL; in gfx_v8_0_reset_kgq()
6904 spin_lock_irqsave(&kiq->ring_lock, flags); in gfx_v8_0_reset_kgq()
6907 spin_unlock_irqrestore(&kiq->ring_lock, flags); in gfx_v8_0_reset_kgq()
6908 return -ENOMEM; in gfx_v8_0_reset_kgq()
6915 spin_unlock_irqrestore(&kiq->ring_lock, flags); in gfx_v8_0_reset_kgq()
6922 return -ENOMEM; in gfx_v8_0_reset_kgq()
6923 gfx_v8_0_ring_emit_fence_gfx(ring, ring->fence_drv.gpu_addr, in gfx_v8_0_reset_kgq()
6924 ring->fence_drv.sync_seq, AMDGPU_FENCE_FLAG_EXEC); in gfx_v8_0_reset_kgq()
7064 adev->gfx.kiq[0].ring.funcs = &gfx_v8_0_ring_funcs_kiq; in gfx_v8_0_set_ring_funcs()
7066 for (i = 0; i < adev->gfx.num_gfx_rings; i++) in gfx_v8_0_set_ring_funcs()
7067 adev->gfx.gfx_ring[i].funcs = &gfx_v8_0_ring_funcs_gfx; in gfx_v8_0_set_ring_funcs()
7069 for (i = 0; i < adev->gfx.num_compute_rings; i++) in gfx_v8_0_set_ring_funcs()
7070 adev->gfx.compute_ring[i].funcs = &gfx_v8_0_ring_funcs_compute; in gfx_v8_0_set_ring_funcs()
7100 adev->gfx.eop_irq.num_types = AMDGPU_CP_IRQ_LAST; in gfx_v8_0_set_irq_funcs()
7101 adev->gfx.eop_irq.funcs = &gfx_v8_0_eop_irq_funcs; in gfx_v8_0_set_irq_funcs()
7103 adev->gfx.priv_reg_irq.num_types = 1; in gfx_v8_0_set_irq_funcs()
7104 adev->gfx.priv_reg_irq.funcs = &gfx_v8_0_priv_reg_irq_funcs; in gfx_v8_0_set_irq_funcs()
7106 adev->gfx.priv_inst_irq.num_types = 1; in gfx_v8_0_set_irq_funcs()
7107 adev->gfx.priv_inst_irq.funcs = &gfx_v8_0_priv_inst_irq_funcs; in gfx_v8_0_set_irq_funcs()
7109 adev->gfx.cp_ecc_error_irq.num_types = 1; in gfx_v8_0_set_irq_funcs()
7110 adev->gfx.cp_ecc_error_irq.funcs = &gfx_v8_0_cp_ecc_error_irq_funcs; in gfx_v8_0_set_irq_funcs()
7112 adev->gfx.sq_irq.num_types = 1; in gfx_v8_0_set_irq_funcs()
7113 adev->gfx.sq_irq.funcs = &gfx_v8_0_sq_irq_funcs; in gfx_v8_0_set_irq_funcs()
7118 adev->gfx.rlc.funcs = &iceland_rlc_funcs; in gfx_v8_0_set_rlc_funcs()
7124 adev->gds.gds_size = RREG32(mmGDS_VMID0_SIZE); in gfx_v8_0_set_gds_init()
7125 adev->gds.gws_size = 64; in gfx_v8_0_set_gds_init()
7126 adev->gds.oa_size = 16; in gfx_v8_0_set_gds_init()
7127 adev->gds.gds_compute_max_wave_id = RREG32(mmGDS_COMPUTE_MAX_WAVE_ID); in gfx_v8_0_set_gds_init()
7151 mask = amdgpu_gfx_create_bitmask(adev->gfx.config.max_cu_per_sh); in gfx_v8_0_get_cu_active_bitmap()
7160 struct amdgpu_cu_info *cu_info = &adev->gfx.cu_info; in gfx_v8_0_get_cu_info()
7166 if (adev->flags & AMD_IS_APU) in gfx_v8_0_get_cu_info()
7169 ao_cu_num = adev->gfx.config.max_cu_per_sh; in gfx_v8_0_get_cu_info()
7173 mutex_lock(&adev->grbm_idx_mutex); in gfx_v8_0_get_cu_info()
7174 for (i = 0; i < adev->gfx.config.max_shader_engines; i++) { in gfx_v8_0_get_cu_info()
7175 for (j = 0; j < adev->gfx.config.max_sh_per_se; j++) { in gfx_v8_0_get_cu_info()
7184 cu_info->bitmap[0][i][j] = bitmap; in gfx_v8_0_get_cu_info()
7186 for (k = 0; k < adev->gfx.config.max_cu_per_sh; k ++) { in gfx_v8_0_get_cu_info()
7197 cu_info->ao_cu_bitmap[i][j] = ao_bitmap; in gfx_v8_0_get_cu_info()
7201 mutex_unlock(&adev->grbm_idx_mutex); in gfx_v8_0_get_cu_info()
7203 cu_info->number = active_cu_number; in gfx_v8_0_get_cu_info()
7204 cu_info->ao_cu_mask = ao_cu_mask; in gfx_v8_0_get_cu_info()
7205 cu_info->simd_per_cu = NUM_SIMD_PER_CU; in gfx_v8_0_get_cu_info()
7206 cu_info->max_waves_per_simd = 10; in gfx_v8_0_get_cu_info()
7207 cu_info->max_scratch_slots_per_cu = 32; in gfx_v8_0_get_cu_info()
7208 cu_info->wave_front_size = 64; in gfx_v8_0_get_cu_info()
7209 cu_info->lds_size = 64; in gfx_v8_0_get_cu_info()
7239 if (ring->adev->virt.chained_ib_support) { in gfx_v8_0_ring_emit_ce_meta()
7240 ce_payload_addr = amdgpu_csa_vaddr(ring->adev) + in gfx_v8_0_ring_emit_ce_meta()
7242 cnt_ce = (sizeof(ce_payload.chained) >> 2) + 4 - 2; in gfx_v8_0_ring_emit_ce_meta()
7244 ce_payload_addr = amdgpu_csa_vaddr(ring->adev) + in gfx_v8_0_ring_emit_ce_meta()
7246 cnt_ce = (sizeof(ce_payload.regular) >> 2) + 4 - 2; in gfx_v8_0_ring_emit_ce_meta()
7256 amdgpu_ring_write_multiple(ring, (void *)&ce_payload, cnt_ce - 2); in gfx_v8_0_ring_emit_ce_meta()
7268 csa_addr = amdgpu_csa_vaddr(ring->adev); in gfx_v8_0_ring_emit_de_meta()
7270 if (ring->adev->virt.chained_ib_support) { in gfx_v8_0_ring_emit_de_meta()
7274 cnt_de = (sizeof(de_payload.chained) >> 2) + 4 - 2; in gfx_v8_0_ring_emit_de_meta()
7279 cnt_de = (sizeof(de_payload.regular) >> 2) + 4 - 2; in gfx_v8_0_ring_emit_de_meta()
7289 amdgpu_ring_write_multiple(ring, (void *)&de_payload, cnt_de - 2); in gfx_v8_0_ring_emit_de_meta()