1 /* 2 * Copyright 2018 Advanced Micro Devices, Inc. 3 * 4 * Permission is hereby granted, free of charge, to any person obtaining a 5 * copy of this software and associated documentation files (the "Software"), 6 * to deal in the Software without restriction, including without limitation 7 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 8 * and/or sell copies of the Software, and to permit persons to whom the 9 * Software is furnished to do so, subject to the following conditions: 10 * 11 * The above copyright notice and this permission notice shall be included in 12 * all copies or substantial portions of the Software. 13 * 14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR 18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, 19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR 20 * OTHER DEALINGS IN THE SOFTWARE. 21 * 22 */ 23 24 #include <linux/firmware.h> 25 #include "amdgpu.h" 26 #include "amdgpu_sdma.h" 27 #include "amdgpu_ras.h" 28 29 #define AMDGPU_CSA_SDMA_SIZE 64 30 /* SDMA CSA reside in the 3rd page of CSA */ 31 #define AMDGPU_CSA_SDMA_OFFSET (4096 * 2) 32 33 /* 34 * GPU SDMA IP block helpers function. 35 */ 36 amdgpu_sdma_get_instance_from_ring(struct amdgpu_ring * ring)37 struct amdgpu_sdma_instance *amdgpu_sdma_get_instance_from_ring(struct amdgpu_ring *ring) 38 { 39 struct amdgpu_device *adev = ring->adev; 40 int i; 41 42 for (i = 0; i < adev->sdma.num_instances; i++) 43 if (ring == &adev->sdma.instance[i].ring || 44 ring == &adev->sdma.instance[i].page) 45 return &adev->sdma.instance[i]; 46 47 return NULL; 48 } 49 amdgpu_sdma_get_index_from_ring(struct amdgpu_ring * ring,uint32_t * index)50 int amdgpu_sdma_get_index_from_ring(struct amdgpu_ring *ring, uint32_t *index) 51 { 52 struct amdgpu_device *adev = ring->adev; 53 int i; 54 55 for (i = 0; i < adev->sdma.num_instances; i++) { 56 if (ring == &adev->sdma.instance[i].ring || 57 ring == &adev->sdma.instance[i].page) { 58 *index = i; 59 return 0; 60 } 61 } 62 63 return -EINVAL; 64 } 65 amdgpu_sdma_get_csa_mc_addr(struct amdgpu_ring * ring,unsigned int vmid)66 uint64_t amdgpu_sdma_get_csa_mc_addr(struct amdgpu_ring *ring, 67 unsigned int vmid) 68 { 69 struct amdgpu_device *adev = ring->adev; 70 uint64_t csa_mc_addr; 71 uint32_t index = 0; 72 int r; 73 74 /* don't enable OS preemption on SDMA under SRIOV */ 75 if (amdgpu_sriov_vf(adev) || vmid == 0 || !adev->gfx.mcbp) 76 return 0; 77 78 if (ring->is_mes_queue) { 79 uint32_t offset = 0; 80 81 offset = offsetof(struct amdgpu_mes_ctx_meta_data, 82 sdma[ring->idx].sdma_meta_data); 83 csa_mc_addr = amdgpu_mes_ctx_get_offs_gpu_addr(ring, offset); 84 } else { 85 r = amdgpu_sdma_get_index_from_ring(ring, &index); 86 87 if (r || index > 31) 88 csa_mc_addr = 0; 89 else 90 csa_mc_addr = amdgpu_csa_vaddr(adev) + 91 AMDGPU_CSA_SDMA_OFFSET + 92 index * AMDGPU_CSA_SDMA_SIZE; 93 } 94 95 return csa_mc_addr; 96 } 97 amdgpu_sdma_ras_late_init(struct amdgpu_device * adev,struct ras_common_if * ras_block)98 int amdgpu_sdma_ras_late_init(struct amdgpu_device *adev, 99 struct ras_common_if *ras_block) 100 { 101 int r, i; 102 103 r = amdgpu_ras_block_late_init(adev, ras_block); 104 if (r) 105 return r; 106 107 if (amdgpu_ras_is_supported(adev, ras_block->block)) { 108 for (i = 0; i < adev->sdma.num_instances; i++) { 109 r = amdgpu_irq_get(adev, &adev->sdma.ecc_irq, 110 AMDGPU_SDMA_IRQ_INSTANCE0 + i); 111 if (r) 112 goto late_fini; 113 } 114 } 115 116 return 0; 117 118 late_fini: 119 amdgpu_ras_block_late_fini(adev, ras_block); 120 return r; 121 } 122 amdgpu_sdma_process_ras_data_cb(struct amdgpu_device * adev,void * err_data,struct amdgpu_iv_entry * entry)123 int amdgpu_sdma_process_ras_data_cb(struct amdgpu_device *adev, 124 void *err_data, 125 struct amdgpu_iv_entry *entry) 126 { 127 kgd2kfd_set_sram_ecc_flag(adev->kfd.dev); 128 129 if (amdgpu_sriov_vf(adev)) 130 return AMDGPU_RAS_SUCCESS; 131 132 amdgpu_ras_reset_gpu(adev); 133 134 return AMDGPU_RAS_SUCCESS; 135 } 136 amdgpu_sdma_process_ecc_irq(struct amdgpu_device * adev,struct amdgpu_irq_src * source,struct amdgpu_iv_entry * entry)137 int amdgpu_sdma_process_ecc_irq(struct amdgpu_device *adev, 138 struct amdgpu_irq_src *source, 139 struct amdgpu_iv_entry *entry) 140 { 141 struct ras_common_if *ras_if = adev->sdma.ras_if; 142 struct ras_dispatch_if ih_data = { 143 .entry = entry, 144 }; 145 146 if (!ras_if) 147 return 0; 148 149 ih_data.head = *ras_if; 150 151 amdgpu_ras_interrupt_dispatch(adev, &ih_data); 152 return 0; 153 } 154 amdgpu_sdma_init_inst_ctx(struct amdgpu_sdma_instance * sdma_inst)155 static int amdgpu_sdma_init_inst_ctx(struct amdgpu_sdma_instance *sdma_inst) 156 { 157 uint16_t version_major; 158 const struct common_firmware_header *header = NULL; 159 const struct sdma_firmware_header_v1_0 *hdr; 160 const struct sdma_firmware_header_v2_0 *hdr_v2; 161 const struct sdma_firmware_header_v3_0 *hdr_v3; 162 163 header = (const struct common_firmware_header *) 164 sdma_inst->fw->data; 165 version_major = le16_to_cpu(header->header_version_major); 166 167 switch (version_major) { 168 case 1: 169 hdr = (const struct sdma_firmware_header_v1_0 *)sdma_inst->fw->data; 170 sdma_inst->fw_version = le32_to_cpu(hdr->header.ucode_version); 171 sdma_inst->feature_version = le32_to_cpu(hdr->ucode_feature_version); 172 break; 173 case 2: 174 hdr_v2 = (const struct sdma_firmware_header_v2_0 *)sdma_inst->fw->data; 175 sdma_inst->fw_version = le32_to_cpu(hdr_v2->header.ucode_version); 176 sdma_inst->feature_version = le32_to_cpu(hdr_v2->ucode_feature_version); 177 break; 178 case 3: 179 hdr_v3 = (const struct sdma_firmware_header_v3_0 *)sdma_inst->fw->data; 180 sdma_inst->fw_version = le32_to_cpu(hdr_v3->header.ucode_version); 181 sdma_inst->feature_version = le32_to_cpu(hdr_v3->ucode_feature_version); 182 break; 183 default: 184 return -EINVAL; 185 } 186 187 if (sdma_inst->feature_version >= 20) 188 sdma_inst->burst_nop = true; 189 190 return 0; 191 } 192 amdgpu_sdma_destroy_inst_ctx(struct amdgpu_device * adev,bool duplicate)193 void amdgpu_sdma_destroy_inst_ctx(struct amdgpu_device *adev, 194 bool duplicate) 195 { 196 int i; 197 198 for (i = 0; i < adev->sdma.num_instances; i++) { 199 amdgpu_ucode_release(&adev->sdma.instance[i].fw); 200 if (duplicate) 201 break; 202 } 203 204 memset((void *)adev->sdma.instance, 0, 205 sizeof(struct amdgpu_sdma_instance) * AMDGPU_MAX_SDMA_INSTANCES); 206 } 207 amdgpu_sdma_init_microcode(struct amdgpu_device * adev,u32 instance,bool duplicate)208 int amdgpu_sdma_init_microcode(struct amdgpu_device *adev, 209 u32 instance, bool duplicate) 210 { 211 struct amdgpu_firmware_info *info = NULL; 212 const struct common_firmware_header *header = NULL; 213 int err, i; 214 const struct sdma_firmware_header_v2_0 *sdma_hdr; 215 const struct sdma_firmware_header_v3_0 *sdma_hv3; 216 uint16_t version_major; 217 char ucode_prefix[30]; 218 219 amdgpu_ucode_ip_version_decode(adev, SDMA0_HWIP, ucode_prefix, sizeof(ucode_prefix)); 220 if (instance == 0) 221 err = amdgpu_ucode_request(adev, &adev->sdma.instance[instance].fw, 222 "amdgpu/%s.bin", ucode_prefix); 223 else 224 err = amdgpu_ucode_request(adev, &adev->sdma.instance[instance].fw, 225 "amdgpu/%s%d.bin", ucode_prefix, instance); 226 if (err) 227 goto out; 228 229 header = (const struct common_firmware_header *) 230 adev->sdma.instance[instance].fw->data; 231 version_major = le16_to_cpu(header->header_version_major); 232 233 if ((duplicate && instance) || (!duplicate && version_major > 1)) { 234 err = -EINVAL; 235 goto out; 236 } 237 238 err = amdgpu_sdma_init_inst_ctx(&adev->sdma.instance[instance]); 239 if (err) 240 goto out; 241 242 if (duplicate) { 243 for (i = 1; i < adev->sdma.num_instances; i++) 244 memcpy((void *)&adev->sdma.instance[i], 245 (void *)&adev->sdma.instance[0], 246 sizeof(struct amdgpu_sdma_instance)); 247 } 248 249 DRM_DEBUG("psp_load == '%s'\n", 250 adev->firmware.load_type == AMDGPU_FW_LOAD_PSP ? "true" : "false"); 251 252 if (adev->firmware.load_type == AMDGPU_FW_LOAD_PSP) { 253 switch (version_major) { 254 case 1: 255 for (i = 0; i < adev->sdma.num_instances; i++) { 256 if (!duplicate && (instance != i)) 257 continue; 258 else { 259 /* Use a single copy per SDMA firmware type. PSP uses the same instance for all 260 * groups of SDMAs */ 261 if ((amdgpu_ip_version(adev, SDMA0_HWIP, 0) == 262 IP_VERSION(4, 4, 2) || 263 amdgpu_ip_version(adev, SDMA0_HWIP, 0) == 264 IP_VERSION(4, 4, 5)) && 265 adev->firmware.load_type == 266 AMDGPU_FW_LOAD_PSP && 267 adev->sdma.num_inst_per_aid == i) { 268 break; 269 } 270 info = &adev->firmware.ucode[AMDGPU_UCODE_ID_SDMA0 + i]; 271 info->ucode_id = AMDGPU_UCODE_ID_SDMA0 + i; 272 info->fw = adev->sdma.instance[i].fw; 273 adev->firmware.fw_size += 274 ALIGN(le32_to_cpu(header->ucode_size_bytes), PAGE_SIZE); 275 } 276 } 277 break; 278 case 2: 279 sdma_hdr = (const struct sdma_firmware_header_v2_0 *) 280 adev->sdma.instance[0].fw->data; 281 info = &adev->firmware.ucode[AMDGPU_UCODE_ID_SDMA_UCODE_TH0]; 282 info->ucode_id = AMDGPU_UCODE_ID_SDMA_UCODE_TH0; 283 info->fw = adev->sdma.instance[0].fw; 284 adev->firmware.fw_size += 285 ALIGN(le32_to_cpu(sdma_hdr->ctx_ucode_size_bytes), PAGE_SIZE); 286 info = &adev->firmware.ucode[AMDGPU_UCODE_ID_SDMA_UCODE_TH1]; 287 info->ucode_id = AMDGPU_UCODE_ID_SDMA_UCODE_TH1; 288 info->fw = adev->sdma.instance[0].fw; 289 adev->firmware.fw_size += 290 ALIGN(le32_to_cpu(sdma_hdr->ctl_ucode_size_bytes), PAGE_SIZE); 291 break; 292 case 3: 293 sdma_hv3 = (const struct sdma_firmware_header_v3_0 *) 294 adev->sdma.instance[0].fw->data; 295 info = &adev->firmware.ucode[AMDGPU_UCODE_ID_SDMA_RS64]; 296 info->ucode_id = AMDGPU_UCODE_ID_SDMA_RS64; 297 info->fw = adev->sdma.instance[0].fw; 298 adev->firmware.fw_size += 299 ALIGN(le32_to_cpu(sdma_hv3->ucode_size_bytes), PAGE_SIZE); 300 break; 301 default: 302 err = -EINVAL; 303 } 304 } 305 306 out: 307 if (err) 308 amdgpu_sdma_destroy_inst_ctx(adev, duplicate); 309 return err; 310 } 311 amdgpu_sdma_ras_sw_init(struct amdgpu_device * adev)312 int amdgpu_sdma_ras_sw_init(struct amdgpu_device *adev) 313 { 314 int err = 0; 315 struct amdgpu_sdma_ras *ras = NULL; 316 317 /* adev->sdma.ras is NULL, which means sdma does not 318 * support ras function, then do nothing here. 319 */ 320 if (!adev->sdma.ras) 321 return 0; 322 323 ras = adev->sdma.ras; 324 325 err = amdgpu_ras_register_ras_block(adev, &ras->ras_block); 326 if (err) { 327 dev_err(adev->dev, "Failed to register sdma ras block!\n"); 328 return err; 329 } 330 331 strcpy(ras->ras_block.ras_comm.name, "sdma"); 332 ras->ras_block.ras_comm.block = AMDGPU_RAS_BLOCK__SDMA; 333 ras->ras_block.ras_comm.type = AMDGPU_RAS_ERROR__MULTI_UNCORRECTABLE; 334 adev->sdma.ras_if = &ras->ras_block.ras_comm; 335 336 /* If not define special ras_late_init function, use default ras_late_init */ 337 if (!ras->ras_block.ras_late_init) 338 ras->ras_block.ras_late_init = amdgpu_sdma_ras_late_init; 339 340 /* If not defined special ras_cb function, use default ras_cb */ 341 if (!ras->ras_block.ras_cb) 342 ras->ras_block.ras_cb = amdgpu_sdma_process_ras_data_cb; 343 344 return 0; 345 } 346