1 /*
2 * Copyright 2023 Advanced Micro Devices, Inc.
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
13 *
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20 * OTHER DEALINGS IN THE SOFTWARE.
21 *
22 */
23
24 #include "amdgpu.h"
25 #include "gfxhub_v12_0.h"
26
27 #include "gc/gc_12_0_0_offset.h"
28 #include "gc/gc_12_0_0_sh_mask.h"
29 #include "soc24_enum.h"
30 #include "soc15_common.h"
31
32 #define regGCVM_L2_CNTL3_DEFAULT 0x80120007
33 #define regGCVM_L2_CNTL4_DEFAULT 0x000000c1
34 #define regGCVM_L2_CNTL5_DEFAULT 0x00003fe0
35 #define regGRBM_GFX_INDEX_DEFAULT 0xe0000000
36
37 static const char *gfxhub_client_ids[] = {
38 "CB",
39 "DB",
40 "GE1",
41 "GE2",
42 "CPF",
43 "CPC",
44 "CPG",
45 "RLC",
46 "TCP",
47 "SQC (inst)",
48 "SQC (data)",
49 "SQG/PC/SC",
50 "Reserved",
51 "SDMA0",
52 "SDMA1",
53 "GCR",
54 "Reserved",
55 "Reserved",
56 "WGS",
57 "DSM",
58 "PA"
59 };
60
gfxhub_v12_0_get_invalidate_req(unsigned int vmid,uint32_t flush_type)61 static uint32_t gfxhub_v12_0_get_invalidate_req(unsigned int vmid,
62 uint32_t flush_type)
63 {
64 u32 req = 0;
65
66 /* invalidate using legacy mode on vmid*/
67 req = REG_SET_FIELD(req, GCVM_INVALIDATE_ENG0_REQ,
68 PER_VMID_INVALIDATE_REQ, 1 << vmid);
69 req = REG_SET_FIELD(req, GCVM_INVALIDATE_ENG0_REQ, FLUSH_TYPE, flush_type);
70 req = REG_SET_FIELD(req, GCVM_INVALIDATE_ENG0_REQ, INVALIDATE_L2_PTES, 1);
71 req = REG_SET_FIELD(req, GCVM_INVALIDATE_ENG0_REQ, INVALIDATE_L2_PDE0, 1);
72 req = REG_SET_FIELD(req, GCVM_INVALIDATE_ENG0_REQ, INVALIDATE_L2_PDE1, 1);
73 req = REG_SET_FIELD(req, GCVM_INVALIDATE_ENG0_REQ, INVALIDATE_L2_PDE2, 1);
74 req = REG_SET_FIELD(req, GCVM_INVALIDATE_ENG0_REQ, INVALIDATE_L1_PTES, 1);
75 req = REG_SET_FIELD(req, GCVM_INVALIDATE_ENG0_REQ,
76 CLEAR_PROTECTION_FAULT_STATUS_ADDR, 0);
77
78 return req;
79 }
80
81 static void
gfxhub_v12_0_print_l2_protection_fault_status(struct amdgpu_device * adev,uint32_t status)82 gfxhub_v12_0_print_l2_protection_fault_status(struct amdgpu_device *adev,
83 uint32_t status)
84 {
85 u32 cid = REG_GET_FIELD(status,
86 GCVM_L2_PROTECTION_FAULT_STATUS_LO32, CID);
87
88 dev_err(adev->dev,
89 "GCVM_L2_PROTECTION_FAULT_STATUS:0x%08X\n",
90 status);
91 dev_err(adev->dev, "\t Faulty UTCL2 client ID: %s (0x%x)\n",
92 cid >= ARRAY_SIZE(gfxhub_client_ids) ? "unknown" : gfxhub_client_ids[cid],
93 cid);
94 dev_err(adev->dev, "\t MORE_FAULTS: 0x%lx\n",
95 REG_GET_FIELD(status,
96 GCVM_L2_PROTECTION_FAULT_STATUS_LO32, MORE_FAULTS));
97 dev_err(adev->dev, "\t WALKER_ERROR: 0x%lx\n",
98 REG_GET_FIELD(status,
99 GCVM_L2_PROTECTION_FAULT_STATUS_LO32, WALKER_ERROR));
100 dev_err(adev->dev, "\t PERMISSION_FAULTS: 0x%lx\n",
101 REG_GET_FIELD(status,
102 GCVM_L2_PROTECTION_FAULT_STATUS_LO32, PERMISSION_FAULTS));
103 dev_err(adev->dev, "\t MAPPING_ERROR: 0x%lx\n",
104 REG_GET_FIELD(status,
105 GCVM_L2_PROTECTION_FAULT_STATUS_LO32, MAPPING_ERROR));
106 dev_err(adev->dev, "\t RW: 0x%lx\n",
107 REG_GET_FIELD(status,
108 GCVM_L2_PROTECTION_FAULT_STATUS_LO32, RW));
109 }
110
gfxhub_v12_0_get_fb_location(struct amdgpu_device * adev)111 static u64 gfxhub_v12_0_get_fb_location(struct amdgpu_device *adev)
112 {
113 u64 base = RREG32_SOC15(GC, 0, regGCMC_VM_FB_LOCATION_BASE);
114
115 base &= GCMC_VM_FB_LOCATION_BASE__FB_BASE_MASK;
116 base <<= 24;
117
118 return base;
119 }
120
gfxhub_v12_0_get_mc_fb_offset(struct amdgpu_device * adev)121 static u64 gfxhub_v12_0_get_mc_fb_offset(struct amdgpu_device *adev)
122 {
123 return (u64)RREG32_SOC15(GC, 0, regGCMC_VM_FB_OFFSET) << 24;
124 }
125
gfxhub_v12_0_setup_vm_pt_regs(struct amdgpu_device * adev,uint32_t vmid,uint64_t page_table_base)126 static void gfxhub_v12_0_setup_vm_pt_regs(struct amdgpu_device *adev,
127 uint32_t vmid,
128 uint64_t page_table_base)
129 {
130 struct amdgpu_vmhub *hub = &adev->vmhub[AMDGPU_GFXHUB(0)];
131
132 WREG32_SOC15_OFFSET(GC, 0, regGCVM_CONTEXT0_PAGE_TABLE_BASE_ADDR_LO32,
133 hub->ctx_addr_distance * vmid,
134 lower_32_bits(page_table_base));
135
136 WREG32_SOC15_OFFSET(GC, 0, regGCVM_CONTEXT0_PAGE_TABLE_BASE_ADDR_HI32,
137 hub->ctx_addr_distance * vmid,
138 upper_32_bits(page_table_base));
139 }
140
gfxhub_v12_0_init_gart_aperture_regs(struct amdgpu_device * adev)141 static void gfxhub_v12_0_init_gart_aperture_regs(struct amdgpu_device *adev)
142 {
143 uint64_t pt_base = amdgpu_gmc_pd_addr(adev->gart.bo);
144
145 gfxhub_v12_0_setup_vm_pt_regs(adev, 0, pt_base);
146
147 WREG32_SOC15(GC, 0, regGCVM_CONTEXT0_PAGE_TABLE_START_ADDR_LO32,
148 (u32)(adev->gmc.gart_start >> 12));
149 WREG32_SOC15(GC, 0, regGCVM_CONTEXT0_PAGE_TABLE_START_ADDR_HI32,
150 (u32)(adev->gmc.gart_start >> 44));
151
152 WREG32_SOC15(GC, 0, regGCVM_CONTEXT0_PAGE_TABLE_END_ADDR_LO32,
153 (u32)(adev->gmc.gart_end >> 12));
154 WREG32_SOC15(GC, 0, regGCVM_CONTEXT0_PAGE_TABLE_END_ADDR_HI32,
155 (u32)(adev->gmc.gart_end >> 44));
156 }
157
gfxhub_v12_0_init_system_aperture_regs(struct amdgpu_device * adev)158 static void gfxhub_v12_0_init_system_aperture_regs(struct amdgpu_device *adev)
159 {
160 uint64_t value;
161
162 /* Program the AGP BAR */
163 WREG32_SOC15(GC, 0, regGCMC_VM_AGP_BASE, 0);
164 WREG32_SOC15(GC, 0, regGCMC_VM_AGP_BOT, adev->gmc.agp_start >> 24);
165 WREG32_SOC15(GC, 0, regGCMC_VM_AGP_TOP, adev->gmc.agp_end >> 24);
166
167 /* Program the system aperture low logical page number. */
168 WREG32_SOC15(GC, 0, regGCMC_VM_SYSTEM_APERTURE_LOW_ADDR,
169 min(adev->gmc.fb_start, adev->gmc.agp_start) >> 18);
170 WREG32_SOC15(GC, 0, regGCMC_VM_SYSTEM_APERTURE_HIGH_ADDR,
171 max(adev->gmc.fb_end, adev->gmc.agp_end) >> 18);
172
173 /* Set default page address. */
174 value = adev->mem_scratch.gpu_addr - adev->gmc.vram_start
175 + adev->vm_manager.vram_base_offset;
176 WREG32_SOC15(GC, 0, regGCMC_VM_SYSTEM_APERTURE_DEFAULT_ADDR_LSB,
177 (u32)(value >> 12));
178 WREG32_SOC15(GC, 0, regGCMC_VM_SYSTEM_APERTURE_DEFAULT_ADDR_MSB,
179 (u32)(value >> 44));
180
181 /* Program "protection fault". */
182 WREG32_SOC15(GC, 0, regGCVM_L2_PROTECTION_FAULT_DEFAULT_ADDR_LO32,
183 (u32)(adev->dummy_page_addr >> 12));
184 WREG32_SOC15(GC, 0, regGCVM_L2_PROTECTION_FAULT_DEFAULT_ADDR_HI32,
185 (u32)((u64)adev->dummy_page_addr >> 44));
186
187 WREG32_FIELD15_PREREG(GC, 0, GCVM_L2_PROTECTION_FAULT_CNTL2,
188 ACTIVE_PAGE_MIGRATION_PTE_READ_RETRY, 1);
189 }
190
191
gfxhub_v12_0_init_tlb_regs(struct amdgpu_device * adev)192 static void gfxhub_v12_0_init_tlb_regs(struct amdgpu_device *adev)
193 {
194 uint32_t tmp;
195
196 /* Setup TLB control */
197 tmp = RREG32_SOC15(GC, 0, regGCMC_VM_MX_L1_TLB_CNTL);
198
199 tmp = REG_SET_FIELD(tmp, GCMC_VM_MX_L1_TLB_CNTL, ENABLE_L1_TLB, 1);
200 tmp = REG_SET_FIELD(tmp, GCMC_VM_MX_L1_TLB_CNTL, SYSTEM_ACCESS_MODE, 3);
201 tmp = REG_SET_FIELD(tmp, GCMC_VM_MX_L1_TLB_CNTL,
202 ENABLE_ADVANCED_DRIVER_MODEL, 1);
203 tmp = REG_SET_FIELD(tmp, GCMC_VM_MX_L1_TLB_CNTL,
204 SYSTEM_APERTURE_UNMAPPED_ACCESS, 0);
205 tmp = REG_SET_FIELD(tmp, GCMC_VM_MX_L1_TLB_CNTL, ECO_BITS, 0);
206 tmp = REG_SET_FIELD(tmp, GCMC_VM_MX_L1_TLB_CNTL,
207 MTYPE, MTYPE_UC); /* UC, uncached */
208
209 WREG32_SOC15(GC, 0, regGCMC_VM_MX_L1_TLB_CNTL, tmp);
210 }
211
gfxhub_v12_0_init_cache_regs(struct amdgpu_device * adev)212 static void gfxhub_v12_0_init_cache_regs(struct amdgpu_device *adev)
213 {
214 uint32_t tmp;
215
216 /* These registers are not accessible to VF-SRIOV.
217 * The PF will program them instead.
218 */
219 if (amdgpu_sriov_vf(adev))
220 return;
221
222 /* Setup L2 cache */
223 tmp = RREG32_SOC15(GC, 0, regGCVM_L2_CNTL);
224 tmp = REG_SET_FIELD(tmp, GCVM_L2_CNTL, ENABLE_L2_CACHE, 1);
225 tmp = REG_SET_FIELD(tmp, GCVM_L2_CNTL, ENABLE_L2_FRAGMENT_PROCESSING, 0);
226 tmp = REG_SET_FIELD(tmp, GCVM_L2_CNTL,
227 ENABLE_DEFAULT_PAGE_OUT_TO_SYSTEM_MEMORY, 1);
228 /* XXX for emulation, Refer to closed source code.*/
229 tmp = REG_SET_FIELD(tmp, GCVM_L2_CNTL,
230 L2_PDE0_CACHE_TAG_GENERATION_MODE, 0);
231 tmp = REG_SET_FIELD(tmp, GCVM_L2_CNTL, PDE_FAULT_CLASSIFICATION, 0);
232 tmp = REG_SET_FIELD(tmp, GCVM_L2_CNTL, CONTEXT1_IDENTITY_ACCESS_MODE, 1);
233 tmp = REG_SET_FIELD(tmp, GCVM_L2_CNTL, IDENTITY_MODE_FRAGMENT_SIZE, 0);
234 WREG32_SOC15(GC, 0, regGCVM_L2_CNTL, tmp);
235
236 tmp = RREG32_SOC15(GC, 0, regGCVM_L2_CNTL2);
237 tmp = REG_SET_FIELD(tmp, GCVM_L2_CNTL2, INVALIDATE_ALL_L1_TLBS, 1);
238 tmp = REG_SET_FIELD(tmp, GCVM_L2_CNTL2, INVALIDATE_L2_CACHE, 1);
239 WREG32_SOC15(GC, 0, regGCVM_L2_CNTL2, tmp);
240
241 tmp = regGCVM_L2_CNTL3_DEFAULT;
242 if (adev->gmc.translate_further) {
243 tmp = REG_SET_FIELD(tmp, GCVM_L2_CNTL3, BANK_SELECT, 12);
244 tmp = REG_SET_FIELD(tmp, GCVM_L2_CNTL3,
245 L2_CACHE_BIGK_FRAGMENT_SIZE, 9);
246 } else {
247 tmp = REG_SET_FIELD(tmp, GCVM_L2_CNTL3, BANK_SELECT, 9);
248 tmp = REG_SET_FIELD(tmp, GCVM_L2_CNTL3,
249 L2_CACHE_BIGK_FRAGMENT_SIZE, 6);
250 }
251 WREG32_SOC15(GC, 0, regGCVM_L2_CNTL3, tmp);
252
253 tmp = regGCVM_L2_CNTL4_DEFAULT;
254 tmp = REG_SET_FIELD(tmp, GCVM_L2_CNTL4, VMC_TAP_PDE_REQUEST_PHYSICAL, 0);
255 tmp = REG_SET_FIELD(tmp, GCVM_L2_CNTL4, VMC_TAP_PTE_REQUEST_PHYSICAL, 0);
256 WREG32_SOC15(GC, 0, regGCVM_L2_CNTL4, tmp);
257
258 tmp = regGCVM_L2_CNTL5_DEFAULT;
259 tmp = REG_SET_FIELD(tmp, GCVM_L2_CNTL5, L2_CACHE_SMALLK_FRAGMENT_SIZE, 0);
260 WREG32_SOC15(GC, 0, regGCVM_L2_CNTL5, tmp);
261 }
262
gfxhub_v12_0_enable_system_domain(struct amdgpu_device * adev)263 static void gfxhub_v12_0_enable_system_domain(struct amdgpu_device *adev)
264 {
265 uint32_t tmp;
266
267 tmp = RREG32_SOC15(GC, 0, regGCVM_CONTEXT0_CNTL);
268 tmp = REG_SET_FIELD(tmp, GCVM_CONTEXT0_CNTL, ENABLE_CONTEXT, 1);
269 tmp = REG_SET_FIELD(tmp, GCVM_CONTEXT0_CNTL, PAGE_TABLE_DEPTH, 0);
270 tmp = REG_SET_FIELD(tmp, GCVM_CONTEXT0_CNTL,
271 RETRY_PERMISSION_OR_INVALID_PAGE_FAULT, 0);
272 WREG32_SOC15(GC, 0, regGCVM_CONTEXT0_CNTL, tmp);
273 }
274
gfxhub_v12_0_disable_identity_aperture(struct amdgpu_device * adev)275 static void gfxhub_v12_0_disable_identity_aperture(struct amdgpu_device *adev)
276 {
277 /* These registers are not accessible to VF-SRIOV.
278 * The PF will program them instead.
279 */
280 if (amdgpu_sriov_vf(adev))
281 return;
282
283 WREG32_SOC15(GC, 0, regGCVM_L2_CONTEXT1_IDENTITY_APERTURE_LOW_ADDR_LO32,
284 0xFFFFFFFF);
285 WREG32_SOC15(GC, 0, regGCVM_L2_CONTEXT1_IDENTITY_APERTURE_LOW_ADDR_HI32,
286 0x0000000F);
287
288 WREG32_SOC15(GC, 0, regGCVM_L2_CONTEXT1_IDENTITY_APERTURE_HIGH_ADDR_LO32,
289 0);
290 WREG32_SOC15(GC, 0, regGCVM_L2_CONTEXT1_IDENTITY_APERTURE_HIGH_ADDR_HI32,
291 0);
292
293 WREG32_SOC15(GC, 0, regGCVM_L2_CONTEXT_IDENTITY_PHYSICAL_OFFSET_LO32, 0);
294 WREG32_SOC15(GC, 0, regGCVM_L2_CONTEXT_IDENTITY_PHYSICAL_OFFSET_HI32, 0);
295
296 }
297
gfxhub_v12_0_setup_vmid_config(struct amdgpu_device * adev)298 static void gfxhub_v12_0_setup_vmid_config(struct amdgpu_device *adev)
299 {
300 struct amdgpu_vmhub *hub = &adev->vmhub[AMDGPU_GFXHUB(0)];
301 int i;
302 uint32_t tmp;
303
304 for (i = 0; i <= 14; i++) {
305 tmp = RREG32_SOC15_OFFSET(GC, 0, regGCVM_CONTEXT1_CNTL, i);
306 tmp = REG_SET_FIELD(tmp, GCVM_CONTEXT1_CNTL, ENABLE_CONTEXT, 1);
307 tmp = REG_SET_FIELD(tmp, GCVM_CONTEXT1_CNTL, PAGE_TABLE_DEPTH,
308 adev->vm_manager.num_level);
309 tmp = REG_SET_FIELD(tmp, GCVM_CONTEXT1_CNTL,
310 RANGE_PROTECTION_FAULT_ENABLE_DEFAULT, 1);
311 tmp = REG_SET_FIELD(tmp, GCVM_CONTEXT1_CNTL,
312 DUMMY_PAGE_PROTECTION_FAULT_ENABLE_DEFAULT, 1);
313 tmp = REG_SET_FIELD(tmp, GCVM_CONTEXT1_CNTL,
314 PDE0_PROTECTION_FAULT_ENABLE_DEFAULT, 1);
315 tmp = REG_SET_FIELD(tmp, GCVM_CONTEXT1_CNTL,
316 VALID_PROTECTION_FAULT_ENABLE_DEFAULT, 1);
317 tmp = REG_SET_FIELD(tmp, GCVM_CONTEXT1_CNTL,
318 READ_PROTECTION_FAULT_ENABLE_DEFAULT, 1);
319 tmp = REG_SET_FIELD(tmp, GCVM_CONTEXT1_CNTL,
320 WRITE_PROTECTION_FAULT_ENABLE_DEFAULT, 1);
321 tmp = REG_SET_FIELD(tmp, GCVM_CONTEXT1_CNTL,
322 EXECUTE_PROTECTION_FAULT_ENABLE_DEFAULT, 1);
323 tmp = REG_SET_FIELD(tmp, GCVM_CONTEXT1_CNTL,
324 PAGE_TABLE_BLOCK_SIZE,
325 adev->vm_manager.block_size - 9);
326 /* Send no-retry XNACK on fault to suppress VM fault storm. */
327 tmp = REG_SET_FIELD(tmp, GCVM_CONTEXT1_CNTL,
328 RETRY_PERMISSION_OR_INVALID_PAGE_FAULT,
329 !amdgpu_noretry);
330 WREG32_SOC15_OFFSET(GC, 0, regGCVM_CONTEXT1_CNTL,
331 i * hub->ctx_distance, tmp);
332 WREG32_SOC15_OFFSET(GC, 0, regGCVM_CONTEXT1_PAGE_TABLE_START_ADDR_LO32,
333 i * hub->ctx_addr_distance, 0);
334 WREG32_SOC15_OFFSET(GC, 0, regGCVM_CONTEXT1_PAGE_TABLE_START_ADDR_HI32,
335 i * hub->ctx_addr_distance, 0);
336 WREG32_SOC15_OFFSET(GC, 0, regGCVM_CONTEXT1_PAGE_TABLE_END_ADDR_LO32,
337 i * hub->ctx_addr_distance,
338 lower_32_bits(adev->vm_manager.max_pfn - 1));
339 WREG32_SOC15_OFFSET(GC, 0, regGCVM_CONTEXT1_PAGE_TABLE_END_ADDR_HI32,
340 i * hub->ctx_addr_distance,
341 upper_32_bits(adev->vm_manager.max_pfn - 1));
342 }
343
344 hub->vm_cntx_cntl = tmp;
345 }
346
gfxhub_v12_0_program_invalidation(struct amdgpu_device * adev)347 static void gfxhub_v12_0_program_invalidation(struct amdgpu_device *adev)
348 {
349 struct amdgpu_vmhub *hub = &adev->vmhub[AMDGPU_GFXHUB(0)];
350 unsigned i;
351
352 for (i = 0 ; i < 18; ++i) {
353 WREG32_SOC15_OFFSET(GC, 0, regGCVM_INVALIDATE_ENG0_ADDR_RANGE_LO32,
354 i * hub->eng_addr_distance, 0xffffffff);
355 WREG32_SOC15_OFFSET(GC, 0, regGCVM_INVALIDATE_ENG0_ADDR_RANGE_HI32,
356 i * hub->eng_addr_distance, 0x1f);
357 }
358 }
359
gfxhub_v12_0_gart_enable(struct amdgpu_device * adev)360 static int gfxhub_v12_0_gart_enable(struct amdgpu_device *adev)
361 {
362 if (amdgpu_sriov_vf(adev)) {
363 /*
364 * GCMC_VM_FB_LOCATION_BASE/TOP is NULL for VF, becuase they are
365 * VF copy registers so vbios post doesn't program them, for
366 * SRIOV driver need to program them
367 */
368 WREG32_SOC15(GC, 0, regGCMC_VM_FB_LOCATION_BASE,
369 adev->gmc.vram_start >> 24);
370 WREG32_SOC15(GC, 0, regGCMC_VM_FB_LOCATION_TOP,
371 adev->gmc.vram_end >> 24);
372 }
373
374 /* GART Enable. */
375 gfxhub_v12_0_init_gart_aperture_regs(adev);
376 gfxhub_v12_0_init_system_aperture_regs(adev);
377 gfxhub_v12_0_init_tlb_regs(adev);
378 gfxhub_v12_0_init_cache_regs(adev);
379
380 gfxhub_v12_0_enable_system_domain(adev);
381 gfxhub_v12_0_disable_identity_aperture(adev);
382 gfxhub_v12_0_setup_vmid_config(adev);
383 gfxhub_v12_0_program_invalidation(adev);
384
385 return 0;
386 }
387
gfxhub_v12_0_gart_disable(struct amdgpu_device * adev)388 static void gfxhub_v12_0_gart_disable(struct amdgpu_device *adev)
389 {
390 struct amdgpu_vmhub *hub = &adev->vmhub[AMDGPU_GFXHUB(0)];
391 u32 tmp;
392 u32 i;
393
394 /* Disable all tables */
395 for (i = 0; i < 16; i++)
396 WREG32_SOC15_OFFSET(GC, 0, regGCVM_CONTEXT0_CNTL,
397 i * hub->ctx_distance, 0);
398
399 /* Setup TLB control */
400 tmp = RREG32_SOC15(GC, 0, regGCMC_VM_MX_L1_TLB_CNTL);
401 tmp = REG_SET_FIELD(tmp, GCMC_VM_MX_L1_TLB_CNTL, ENABLE_L1_TLB, 0);
402 tmp = REG_SET_FIELD(tmp, GCMC_VM_MX_L1_TLB_CNTL,
403 ENABLE_ADVANCED_DRIVER_MODEL, 0);
404 WREG32_SOC15(GC, 0, regGCMC_VM_MX_L1_TLB_CNTL, tmp);
405
406 /* Setup L2 cache */
407 WREG32_FIELD15_PREREG(GC, 0, GCVM_L2_CNTL, ENABLE_L2_CACHE, 0);
408 WREG32_SOC15(GC, 0, regGCVM_L2_CNTL3, 0);
409 }
410
411 /**
412 * gfxhub_v12_0_set_fault_enable_default - update GART/VM fault handling
413 *
414 * @adev: amdgpu_device pointer
415 * @value: true redirects VM faults to the default page
416 */
gfxhub_v12_0_set_fault_enable_default(struct amdgpu_device * adev,bool value)417 static void gfxhub_v12_0_set_fault_enable_default(struct amdgpu_device *adev,
418 bool value)
419 {
420 u32 tmp;
421
422 /* NO halt CP when page fault */
423 tmp = RREG32_SOC15(GC, 0, regCP_DEBUG);
424 tmp = REG_SET_FIELD(tmp, CP_DEBUG, CPG_UTCL1_ERROR_HALT_DISABLE, 1);
425 WREG32_SOC15(GC, 0, regCP_DEBUG, tmp);
426
427 /* These registers are not accessible to VF-SRIOV.
428 * The PF will program them instead.
429 */
430 if (amdgpu_sriov_vf(adev))
431 return;
432
433 tmp = RREG32_SOC15(GC, 0, regGCVM_L2_PROTECTION_FAULT_CNTL);
434 tmp = REG_SET_FIELD(tmp, GCVM_L2_PROTECTION_FAULT_CNTL,
435 RANGE_PROTECTION_FAULT_ENABLE_DEFAULT, value);
436 tmp = REG_SET_FIELD(tmp, GCVM_L2_PROTECTION_FAULT_CNTL,
437 PDE0_PROTECTION_FAULT_ENABLE_DEFAULT, value);
438 tmp = REG_SET_FIELD(tmp, GCVM_L2_PROTECTION_FAULT_CNTL,
439 PDE1_PROTECTION_FAULT_ENABLE_DEFAULT, value);
440 tmp = REG_SET_FIELD(tmp, GCVM_L2_PROTECTION_FAULT_CNTL,
441 PDE2_PROTECTION_FAULT_ENABLE_DEFAULT, value);
442 tmp = REG_SET_FIELD(tmp, GCVM_L2_PROTECTION_FAULT_CNTL,
443 TRANSLATE_FURTHER_PROTECTION_FAULT_ENABLE_DEFAULT,
444 value);
445 tmp = REG_SET_FIELD(tmp, GCVM_L2_PROTECTION_FAULT_CNTL,
446 NACK_PROTECTION_FAULT_ENABLE_DEFAULT, value);
447 tmp = REG_SET_FIELD(tmp, GCVM_L2_PROTECTION_FAULT_CNTL,
448 DUMMY_PAGE_PROTECTION_FAULT_ENABLE_DEFAULT, value);
449 tmp = REG_SET_FIELD(tmp, GCVM_L2_PROTECTION_FAULT_CNTL,
450 VALID_PROTECTION_FAULT_ENABLE_DEFAULT, value);
451 tmp = REG_SET_FIELD(tmp, GCVM_L2_PROTECTION_FAULT_CNTL,
452 READ_PROTECTION_FAULT_ENABLE_DEFAULT, value);
453 tmp = REG_SET_FIELD(tmp, GCVM_L2_PROTECTION_FAULT_CNTL,
454 WRITE_PROTECTION_FAULT_ENABLE_DEFAULT, value);
455 tmp = REG_SET_FIELD(tmp, GCVM_L2_PROTECTION_FAULT_CNTL,
456 EXECUTE_PROTECTION_FAULT_ENABLE_DEFAULT, value);
457 if (!value) {
458 tmp = REG_SET_FIELD(tmp, GCVM_L2_PROTECTION_FAULT_CNTL,
459 CRASH_ON_NO_RETRY_FAULT, 1);
460 tmp = REG_SET_FIELD(tmp, GCVM_L2_PROTECTION_FAULT_CNTL,
461 CRASH_ON_RETRY_FAULT, 1);
462 }
463 WREG32_SOC15(GC, 0, regGCVM_L2_PROTECTION_FAULT_CNTL, tmp);
464 }
465
466 static const struct amdgpu_vmhub_funcs gfxhub_v12_0_vmhub_funcs = {
467 .print_l2_protection_fault_status = gfxhub_v12_0_print_l2_protection_fault_status,
468 .get_invalidate_req = gfxhub_v12_0_get_invalidate_req,
469 };
470
gfxhub_v12_0_init(struct amdgpu_device * adev)471 static void gfxhub_v12_0_init(struct amdgpu_device *adev)
472 {
473 struct amdgpu_vmhub *hub = &adev->vmhub[AMDGPU_GFXHUB(0)];
474
475 hub->ctx0_ptb_addr_lo32 =
476 SOC15_REG_OFFSET(GC, 0,
477 regGCVM_CONTEXT0_PAGE_TABLE_BASE_ADDR_LO32);
478 hub->ctx0_ptb_addr_hi32 =
479 SOC15_REG_OFFSET(GC, 0,
480 regGCVM_CONTEXT0_PAGE_TABLE_BASE_ADDR_HI32);
481 hub->vm_inv_eng0_sem =
482 SOC15_REG_OFFSET(GC, 0, regGCVM_INVALIDATE_ENG0_SEM);
483 hub->vm_inv_eng0_req =
484 SOC15_REG_OFFSET(GC, 0, regGCVM_INVALIDATE_ENG0_REQ);
485 hub->vm_inv_eng0_ack =
486 SOC15_REG_OFFSET(GC, 0, regGCVM_INVALIDATE_ENG0_ACK);
487 hub->vm_context0_cntl =
488 SOC15_REG_OFFSET(GC, 0, regGCVM_CONTEXT0_CNTL);
489 hub->vm_l2_pro_fault_status =
490 SOC15_REG_OFFSET(GC, 0, regGCVM_L2_PROTECTION_FAULT_STATUS_LO32);
491 hub->vm_l2_pro_fault_cntl =
492 SOC15_REG_OFFSET(GC, 0, regGCVM_L2_PROTECTION_FAULT_CNTL);
493
494 hub->ctx_distance = regGCVM_CONTEXT1_CNTL - regGCVM_CONTEXT0_CNTL;
495 hub->ctx_addr_distance = regGCVM_CONTEXT1_PAGE_TABLE_BASE_ADDR_LO32 -
496 regGCVM_CONTEXT0_PAGE_TABLE_BASE_ADDR_LO32;
497 hub->eng_distance = regGCVM_INVALIDATE_ENG1_REQ -
498 regGCVM_INVALIDATE_ENG0_REQ;
499 hub->eng_addr_distance = regGCVM_INVALIDATE_ENG1_ADDR_RANGE_LO32 -
500 regGCVM_INVALIDATE_ENG0_ADDR_RANGE_LO32;
501
502 hub->vm_cntx_cntl_vm_fault = GCVM_CONTEXT1_CNTL__RANGE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK |
503 GCVM_CONTEXT1_CNTL__DUMMY_PAGE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK |
504 GCVM_CONTEXT1_CNTL__PDE0_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK |
505 GCVM_CONTEXT1_CNTL__VALID_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK |
506 GCVM_CONTEXT1_CNTL__READ_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK |
507 GCVM_CONTEXT1_CNTL__WRITE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK |
508 GCVM_CONTEXT1_CNTL__EXECUTE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK;
509
510 hub->vmhub_funcs = &gfxhub_v12_0_vmhub_funcs;
511 }
512
513 const struct amdgpu_gfxhub_funcs gfxhub_v12_0_funcs = {
514 .get_fb_location = gfxhub_v12_0_get_fb_location,
515 .get_mc_fb_offset = gfxhub_v12_0_get_mc_fb_offset,
516 .setup_vm_pt_regs = gfxhub_v12_0_setup_vm_pt_regs,
517 .gart_enable = gfxhub_v12_0_gart_enable,
518 .gart_disable = gfxhub_v12_0_gart_disable,
519 .set_fault_enable_default = gfxhub_v12_0_set_fault_enable_default,
520 .init = gfxhub_v12_0_init,
521 };
522