1 // SPDX-License-Identifier: MIT
2 /*
3 * Copyright © 2022 Intel Corporation
4 */
5
6 #include "xe_guc_ads.h"
7
8 #include <drm/drm_managed.h>
9
10 #include <generated/xe_wa_oob.h>
11
12 #include "abi/guc_actions_abi.h"
13 #include "regs/xe_engine_regs.h"
14 #include "regs/xe_gt_regs.h"
15 #include "regs/xe_guc_regs.h"
16 #include "xe_bo.h"
17 #include "xe_gt.h"
18 #include "xe_gt_ccs_mode.h"
19 #include "xe_gt_printk.h"
20 #include "xe_guc.h"
21 #include "xe_guc_ct.h"
22 #include "xe_hw_engine.h"
23 #include "xe_lrc.h"
24 #include "xe_map.h"
25 #include "xe_mmio.h"
26 #include "xe_platform_types.h"
27 #include "xe_uc_fw.h"
28 #include "xe_wa.h"
29
30 /* Slack of a few additional entries per engine */
31 #define ADS_REGSET_EXTRA_MAX 8
32
33 static struct xe_guc *
ads_to_guc(struct xe_guc_ads * ads)34 ads_to_guc(struct xe_guc_ads *ads)
35 {
36 return container_of(ads, struct xe_guc, ads);
37 }
38
39 static struct xe_gt *
ads_to_gt(struct xe_guc_ads * ads)40 ads_to_gt(struct xe_guc_ads *ads)
41 {
42 return container_of(ads, struct xe_gt, uc.guc.ads);
43 }
44
45 static struct xe_device *
ads_to_xe(struct xe_guc_ads * ads)46 ads_to_xe(struct xe_guc_ads *ads)
47 {
48 return gt_to_xe(ads_to_gt(ads));
49 }
50
51 static struct iosys_map *
ads_to_map(struct xe_guc_ads * ads)52 ads_to_map(struct xe_guc_ads *ads)
53 {
54 return &ads->bo->vmap;
55 }
56
57 /* UM Queue parameters: */
58 #define GUC_UM_QUEUE_SIZE (SZ_64K)
59 #define GUC_PAGE_RES_TIMEOUT_US (-1)
60
61 /*
62 * The Additional Data Struct (ADS) has pointers for different buffers used by
63 * the GuC. One single gem object contains the ADS struct itself (guc_ads) and
64 * all the extra buffers indirectly linked via the ADS struct's entries.
65 *
66 * Layout of the ADS blob allocated for the GuC:
67 *
68 * +---------------------------------------+ <== base
69 * | guc_ads |
70 * +---------------------------------------+
71 * | guc_policies |
72 * +---------------------------------------+
73 * | guc_gt_system_info |
74 * +---------------------------------------+
75 * | guc_engine_usage |
76 * +---------------------------------------+
77 * | guc_um_init_params |
78 * +---------------------------------------+ <== static
79 * | guc_mmio_reg[countA] (engine 0.0) |
80 * | guc_mmio_reg[countB] (engine 0.1) |
81 * | guc_mmio_reg[countC] (engine 1.0) |
82 * | ... |
83 * +---------------------------------------+ <== dynamic
84 * | padding |
85 * +---------------------------------------+ <== 4K aligned
86 * | golden contexts |
87 * +---------------------------------------+
88 * | padding |
89 * +---------------------------------------+ <== 4K aligned
90 * | w/a KLVs |
91 * +---------------------------------------+
92 * | padding |
93 * +---------------------------------------+ <== 4K aligned
94 * | capture lists |
95 * +---------------------------------------+
96 * | padding |
97 * +---------------------------------------+ <== 4K aligned
98 * | UM queues |
99 * +---------------------------------------+
100 * | padding |
101 * +---------------------------------------+ <== 4K aligned
102 * | private data |
103 * +---------------------------------------+
104 * | padding |
105 * +---------------------------------------+ <== 4K aligned
106 */
107 struct __guc_ads_blob {
108 struct guc_ads ads;
109 struct guc_policies policies;
110 struct guc_gt_system_info system_info;
111 struct guc_engine_usage engine_usage;
112 struct guc_um_init_params um_init_params;
113 /* From here on, location is dynamic! Refer to above diagram. */
114 struct guc_mmio_reg regset[];
115 } __packed;
116
117 #define ads_blob_read(ads_, field_) \
118 xe_map_rd_field(ads_to_xe(ads_), ads_to_map(ads_), 0, \
119 struct __guc_ads_blob, field_)
120
121 #define ads_blob_write(ads_, field_, val_) \
122 xe_map_wr_field(ads_to_xe(ads_), ads_to_map(ads_), 0, \
123 struct __guc_ads_blob, field_, val_)
124
125 #define info_map_write(xe_, map_, field_, val_) \
126 xe_map_wr_field(xe_, map_, 0, struct guc_gt_system_info, field_, val_)
127
128 #define info_map_read(xe_, map_, field_) \
129 xe_map_rd_field(xe_, map_, 0, struct guc_gt_system_info, field_)
130
guc_ads_regset_size(struct xe_guc_ads * ads)131 static size_t guc_ads_regset_size(struct xe_guc_ads *ads)
132 {
133 struct xe_device *xe = ads_to_xe(ads);
134
135 xe_assert(xe, ads->regset_size);
136
137 return ads->regset_size;
138 }
139
guc_ads_golden_lrc_size(struct xe_guc_ads * ads)140 static size_t guc_ads_golden_lrc_size(struct xe_guc_ads *ads)
141 {
142 return PAGE_ALIGN(ads->golden_lrc_size);
143 }
144
guc_ads_waklv_size(struct xe_guc_ads * ads)145 static u32 guc_ads_waklv_size(struct xe_guc_ads *ads)
146 {
147 return PAGE_ALIGN(ads->ads_waklv_size);
148 }
149
guc_ads_capture_size(struct xe_guc_ads * ads)150 static size_t guc_ads_capture_size(struct xe_guc_ads *ads)
151 {
152 /* FIXME: Allocate a proper capture list */
153 return PAGE_ALIGN(PAGE_SIZE);
154 }
155
guc_ads_um_queues_size(struct xe_guc_ads * ads)156 static size_t guc_ads_um_queues_size(struct xe_guc_ads *ads)
157 {
158 struct xe_device *xe = ads_to_xe(ads);
159
160 if (!xe->info.has_usm)
161 return 0;
162
163 return GUC_UM_QUEUE_SIZE * GUC_UM_HW_QUEUE_MAX;
164 }
165
guc_ads_private_data_size(struct xe_guc_ads * ads)166 static size_t guc_ads_private_data_size(struct xe_guc_ads *ads)
167 {
168 return PAGE_ALIGN(ads_to_guc(ads)->fw.private_data_size);
169 }
170
guc_ads_regset_offset(struct xe_guc_ads * ads)171 static size_t guc_ads_regset_offset(struct xe_guc_ads *ads)
172 {
173 return offsetof(struct __guc_ads_blob, regset);
174 }
175
guc_ads_golden_lrc_offset(struct xe_guc_ads * ads)176 static size_t guc_ads_golden_lrc_offset(struct xe_guc_ads *ads)
177 {
178 size_t offset;
179
180 offset = guc_ads_regset_offset(ads) +
181 guc_ads_regset_size(ads);
182
183 return PAGE_ALIGN(offset);
184 }
185
guc_ads_waklv_offset(struct xe_guc_ads * ads)186 static size_t guc_ads_waklv_offset(struct xe_guc_ads *ads)
187 {
188 u32 offset;
189
190 offset = guc_ads_golden_lrc_offset(ads) +
191 guc_ads_golden_lrc_size(ads);
192
193 return PAGE_ALIGN(offset);
194 }
195
guc_ads_capture_offset(struct xe_guc_ads * ads)196 static size_t guc_ads_capture_offset(struct xe_guc_ads *ads)
197 {
198 size_t offset;
199
200 offset = guc_ads_waklv_offset(ads) +
201 guc_ads_waklv_size(ads);
202
203 return PAGE_ALIGN(offset);
204 }
205
guc_ads_um_queues_offset(struct xe_guc_ads * ads)206 static size_t guc_ads_um_queues_offset(struct xe_guc_ads *ads)
207 {
208 u32 offset;
209
210 offset = guc_ads_capture_offset(ads) +
211 guc_ads_capture_size(ads);
212
213 return PAGE_ALIGN(offset);
214 }
215
guc_ads_private_data_offset(struct xe_guc_ads * ads)216 static size_t guc_ads_private_data_offset(struct xe_guc_ads *ads)
217 {
218 size_t offset;
219
220 offset = guc_ads_um_queues_offset(ads) +
221 guc_ads_um_queues_size(ads);
222
223 return PAGE_ALIGN(offset);
224 }
225
guc_ads_size(struct xe_guc_ads * ads)226 static size_t guc_ads_size(struct xe_guc_ads *ads)
227 {
228 return guc_ads_private_data_offset(ads) +
229 guc_ads_private_data_size(ads);
230 }
231
needs_wa_1607983814(struct xe_device * xe)232 static bool needs_wa_1607983814(struct xe_device *xe)
233 {
234 return GRAPHICS_VERx100(xe) < 1250;
235 }
236
calculate_regset_size(struct xe_gt * gt)237 static size_t calculate_regset_size(struct xe_gt *gt)
238 {
239 struct xe_reg_sr_entry *sr_entry;
240 unsigned long sr_idx;
241 struct xe_hw_engine *hwe;
242 enum xe_hw_engine_id id;
243 unsigned int count = 0;
244
245 for_each_hw_engine(hwe, gt, id)
246 xa_for_each(&hwe->reg_sr.xa, sr_idx, sr_entry)
247 count++;
248
249 count += ADS_REGSET_EXTRA_MAX * XE_NUM_HW_ENGINES;
250
251 if (needs_wa_1607983814(gt_to_xe(gt)))
252 count += LNCFCMOCS_REG_COUNT;
253
254 return count * sizeof(struct guc_mmio_reg);
255 }
256
engine_enable_mask(struct xe_gt * gt,enum xe_engine_class class)257 static u32 engine_enable_mask(struct xe_gt *gt, enum xe_engine_class class)
258 {
259 struct xe_hw_engine *hwe;
260 enum xe_hw_engine_id id;
261 u32 mask = 0;
262
263 for_each_hw_engine(hwe, gt, id)
264 if (hwe->class == class)
265 mask |= BIT(hwe->instance);
266
267 return mask;
268 }
269
calculate_golden_lrc_size(struct xe_guc_ads * ads)270 static size_t calculate_golden_lrc_size(struct xe_guc_ads *ads)
271 {
272 struct xe_gt *gt = ads_to_gt(ads);
273 size_t total_size = 0, alloc_size, real_size;
274 int class;
275
276 for (class = 0; class < XE_ENGINE_CLASS_MAX; ++class) {
277 if (!engine_enable_mask(gt, class))
278 continue;
279
280 real_size = xe_gt_lrc_size(gt, class);
281 alloc_size = PAGE_ALIGN(real_size);
282 total_size += alloc_size;
283 }
284
285 return total_size;
286 }
287
guc_waklv_enable_one_word(struct xe_guc_ads * ads,enum xe_guc_klv_ids klv_id,u32 value,u32 * offset,u32 * remain)288 static void guc_waklv_enable_one_word(struct xe_guc_ads *ads,
289 enum xe_guc_klv_ids klv_id,
290 u32 value,
291 u32 *offset, u32 *remain)
292 {
293 u32 size;
294 u32 klv_entry[] = {
295 /* 16:16 key/length */
296 FIELD_PREP(GUC_KLV_0_KEY, klv_id) |
297 FIELD_PREP(GUC_KLV_0_LEN, 1),
298 value,
299 /* 1 dword data */
300 };
301
302 size = sizeof(klv_entry);
303
304 if (*remain < size) {
305 drm_warn(&ads_to_xe(ads)->drm,
306 "w/a klv buffer too small to add klv id %d\n", klv_id);
307 } else {
308 xe_map_memcpy_to(ads_to_xe(ads), ads_to_map(ads), *offset,
309 klv_entry, size);
310 *offset += size;
311 *remain -= size;
312 }
313 }
314
guc_waklv_enable_simple(struct xe_guc_ads * ads,enum xe_guc_klv_ids klv_id,u32 * offset,u32 * remain)315 static void guc_waklv_enable_simple(struct xe_guc_ads *ads,
316 enum xe_guc_klv_ids klv_id, u32 *offset, u32 *remain)
317 {
318 u32 klv_entry[] = {
319 /* 16:16 key/length */
320 FIELD_PREP(GUC_KLV_0_KEY, klv_id) |
321 FIELD_PREP(GUC_KLV_0_LEN, 0),
322 /* 0 dwords data */
323 };
324 u32 size;
325
326 size = sizeof(klv_entry);
327
328 if (xe_gt_WARN(ads_to_gt(ads), *remain < size,
329 "w/a klv buffer too small to add klv id %d\n", klv_id))
330 return;
331
332 xe_map_memcpy_to(ads_to_xe(ads), ads_to_map(ads), *offset,
333 klv_entry, size);
334 *offset += size;
335 *remain -= size;
336 }
337
guc_waklv_init(struct xe_guc_ads * ads)338 static void guc_waklv_init(struct xe_guc_ads *ads)
339 {
340 struct xe_gt *gt = ads_to_gt(ads);
341 u64 addr_ggtt;
342 u32 offset, remain, size;
343
344 offset = guc_ads_waklv_offset(ads);
345 remain = guc_ads_waklv_size(ads);
346
347 if (XE_WA(gt, 14019882105))
348 guc_waklv_enable_simple(ads,
349 GUC_WORKAROUND_KLV_BLOCK_INTERRUPTS_WHEN_MGSR_BLOCKED,
350 &offset, &remain);
351 if (XE_WA(gt, 18024947630))
352 guc_waklv_enable_simple(ads,
353 GUC_WORKAROUND_KLV_ID_GAM_PFQ_SHADOW_TAIL_POLLING,
354 &offset, &remain);
355 if (XE_WA(gt, 16022287689))
356 guc_waklv_enable_simple(ads,
357 GUC_WORKAROUND_KLV_ID_DISABLE_MTP_DURING_ASYNC_COMPUTE,
358 &offset, &remain);
359
360 /*
361 * On RC6 exit, GuC will write register 0xB04 with the default value provided. As of now,
362 * the default value for this register is determined to be 0xC40. This could change in the
363 * future, so GuC depends on KMD to send it the correct value.
364 */
365 if (XE_WA(gt, 13011645652))
366 guc_waklv_enable_one_word(ads,
367 GUC_WA_KLV_NP_RD_WRITE_TO_CLEAR_RCSM_AT_CGP_LATE_RESTORE,
368 0xC40,
369 &offset, &remain);
370
371 if (XE_WA(gt, 14022293748) || XE_WA(gt, 22019794406))
372 guc_waklv_enable_simple(ads,
373 GUC_WORKAROUND_KLV_ID_BACK_TO_BACK_RCS_ENGINE_RESET,
374 &offset, &remain);
375
376 size = guc_ads_waklv_size(ads) - remain;
377 if (!size)
378 return;
379
380 offset = guc_ads_waklv_offset(ads);
381 addr_ggtt = xe_bo_ggtt_addr(ads->bo) + offset;
382
383 ads_blob_write(ads, ads.wa_klv_addr_lo, lower_32_bits(addr_ggtt));
384 ads_blob_write(ads, ads.wa_klv_addr_hi, upper_32_bits(addr_ggtt));
385 ads_blob_write(ads, ads.wa_klv_size, size);
386 }
387
calculate_waklv_size(struct xe_guc_ads * ads)388 static int calculate_waklv_size(struct xe_guc_ads *ads)
389 {
390 /*
391 * A single page is both the minimum size possible and
392 * is sufficiently large enough for all current platforms.
393 */
394 return SZ_4K;
395 }
396
397 #define MAX_GOLDEN_LRC_SIZE (SZ_4K * 64)
398
xe_guc_ads_init(struct xe_guc_ads * ads)399 int xe_guc_ads_init(struct xe_guc_ads *ads)
400 {
401 struct xe_device *xe = ads_to_xe(ads);
402 struct xe_gt *gt = ads_to_gt(ads);
403 struct xe_tile *tile = gt_to_tile(gt);
404 struct xe_bo *bo;
405
406 ads->golden_lrc_size = calculate_golden_lrc_size(ads);
407 ads->regset_size = calculate_regset_size(gt);
408 ads->ads_waklv_size = calculate_waklv_size(ads);
409
410 bo = xe_managed_bo_create_pin_map(xe, tile, guc_ads_size(ads) + MAX_GOLDEN_LRC_SIZE,
411 XE_BO_FLAG_SYSTEM |
412 XE_BO_FLAG_GGTT |
413 XE_BO_FLAG_GGTT_INVALIDATE);
414 if (IS_ERR(bo))
415 return PTR_ERR(bo);
416
417 ads->bo = bo;
418
419 return 0;
420 }
421
422 /**
423 * xe_guc_ads_init_post_hwconfig - initialize ADS post hwconfig load
424 * @ads: Additional data structures object
425 *
426 * Recalcuate golden_lrc_size & regset_size as the number hardware engines may
427 * have changed after the hwconfig was loaded. Also verify the new sizes fit in
428 * the already allocated ADS buffer object.
429 *
430 * Return: 0 on success, negative error code on error.
431 */
xe_guc_ads_init_post_hwconfig(struct xe_guc_ads * ads)432 int xe_guc_ads_init_post_hwconfig(struct xe_guc_ads *ads)
433 {
434 struct xe_gt *gt = ads_to_gt(ads);
435 u32 prev_regset_size = ads->regset_size;
436
437 xe_gt_assert(gt, ads->bo);
438
439 ads->golden_lrc_size = calculate_golden_lrc_size(ads);
440 ads->regset_size = calculate_regset_size(gt);
441
442 xe_gt_assert(gt, ads->golden_lrc_size +
443 (ads->regset_size - prev_regset_size) <=
444 MAX_GOLDEN_LRC_SIZE);
445
446 return 0;
447 }
448
guc_policies_init(struct xe_guc_ads * ads)449 static void guc_policies_init(struct xe_guc_ads *ads)
450 {
451 struct xe_device *xe = ads_to_xe(ads);
452 u32 global_flags = 0;
453
454 ads_blob_write(ads, policies.dpc_promote_time,
455 GLOBAL_POLICY_DEFAULT_DPC_PROMOTE_TIME_US);
456 ads_blob_write(ads, policies.max_num_work_items,
457 GLOBAL_POLICY_MAX_NUM_WI);
458
459 if (xe->wedged.mode == 2)
460 global_flags |= GLOBAL_POLICY_DISABLE_ENGINE_RESET;
461
462 ads_blob_write(ads, policies.global_flags, global_flags);
463 ads_blob_write(ads, policies.is_valid, 1);
464 }
465
fill_engine_enable_masks(struct xe_gt * gt,struct iosys_map * info_map)466 static void fill_engine_enable_masks(struct xe_gt *gt,
467 struct iosys_map *info_map)
468 {
469 struct xe_device *xe = gt_to_xe(gt);
470
471 info_map_write(xe, info_map, engine_enabled_masks[GUC_RENDER_CLASS],
472 engine_enable_mask(gt, XE_ENGINE_CLASS_RENDER));
473 info_map_write(xe, info_map, engine_enabled_masks[GUC_BLITTER_CLASS],
474 engine_enable_mask(gt, XE_ENGINE_CLASS_COPY));
475 info_map_write(xe, info_map, engine_enabled_masks[GUC_VIDEO_CLASS],
476 engine_enable_mask(gt, XE_ENGINE_CLASS_VIDEO_DECODE));
477 info_map_write(xe, info_map,
478 engine_enabled_masks[GUC_VIDEOENHANCE_CLASS],
479 engine_enable_mask(gt, XE_ENGINE_CLASS_VIDEO_ENHANCE));
480 info_map_write(xe, info_map, engine_enabled_masks[GUC_COMPUTE_CLASS],
481 engine_enable_mask(gt, XE_ENGINE_CLASS_COMPUTE));
482 info_map_write(xe, info_map, engine_enabled_masks[GUC_GSC_OTHER_CLASS],
483 engine_enable_mask(gt, XE_ENGINE_CLASS_OTHER));
484 }
485
guc_prep_golden_lrc_null(struct xe_guc_ads * ads)486 static void guc_prep_golden_lrc_null(struct xe_guc_ads *ads)
487 {
488 struct xe_device *xe = ads_to_xe(ads);
489 struct iosys_map info_map = IOSYS_MAP_INIT_OFFSET(ads_to_map(ads),
490 offsetof(struct __guc_ads_blob, system_info));
491 u8 guc_class;
492
493 for (guc_class = 0; guc_class <= GUC_MAX_ENGINE_CLASSES; ++guc_class) {
494 if (!info_map_read(xe, &info_map,
495 engine_enabled_masks[guc_class]))
496 continue;
497
498 ads_blob_write(ads, ads.eng_state_size[guc_class],
499 guc_ads_golden_lrc_size(ads) -
500 xe_lrc_skip_size(xe));
501 ads_blob_write(ads, ads.golden_context_lrca[guc_class],
502 xe_bo_ggtt_addr(ads->bo) +
503 guc_ads_golden_lrc_offset(ads));
504 }
505 }
506
guc_mapping_table_init_invalid(struct xe_gt * gt,struct iosys_map * info_map)507 static void guc_mapping_table_init_invalid(struct xe_gt *gt,
508 struct iosys_map *info_map)
509 {
510 struct xe_device *xe = gt_to_xe(gt);
511 unsigned int i, j;
512
513 /* Table must be set to invalid values for entries not used */
514 for (i = 0; i < GUC_MAX_ENGINE_CLASSES; ++i)
515 for (j = 0; j < GUC_MAX_INSTANCES_PER_CLASS; ++j)
516 info_map_write(xe, info_map, mapping_table[i][j],
517 GUC_MAX_INSTANCES_PER_CLASS);
518 }
519
guc_mapping_table_init(struct xe_gt * gt,struct iosys_map * info_map)520 static void guc_mapping_table_init(struct xe_gt *gt,
521 struct iosys_map *info_map)
522 {
523 struct xe_device *xe = gt_to_xe(gt);
524 struct xe_hw_engine *hwe;
525 enum xe_hw_engine_id id;
526
527 guc_mapping_table_init_invalid(gt, info_map);
528
529 for_each_hw_engine(hwe, gt, id) {
530 u8 guc_class;
531
532 guc_class = xe_engine_class_to_guc_class(hwe->class);
533 info_map_write(xe, info_map,
534 mapping_table[guc_class][hwe->logical_instance],
535 hwe->instance);
536 }
537 }
538
guc_capture_list_init(struct xe_guc_ads * ads)539 static void guc_capture_list_init(struct xe_guc_ads *ads)
540 {
541 int i, j;
542 u32 addr = xe_bo_ggtt_addr(ads->bo) + guc_ads_capture_offset(ads);
543
544 /* FIXME: Populate a proper capture list */
545 for (i = 0; i < GUC_CAPTURE_LIST_INDEX_MAX; i++) {
546 for (j = 0; j < GUC_MAX_ENGINE_CLASSES; j++) {
547 ads_blob_write(ads, ads.capture_instance[i][j], addr);
548 ads_blob_write(ads, ads.capture_class[i][j], addr);
549 }
550
551 ads_blob_write(ads, ads.capture_global[i], addr);
552 }
553 }
554
guc_mmio_regset_write_one(struct xe_guc_ads * ads,struct iosys_map * regset_map,struct xe_reg reg,unsigned int n_entry)555 static void guc_mmio_regset_write_one(struct xe_guc_ads *ads,
556 struct iosys_map *regset_map,
557 struct xe_reg reg,
558 unsigned int n_entry)
559 {
560 struct guc_mmio_reg entry = {
561 .offset = reg.addr,
562 .flags = reg.masked ? GUC_REGSET_MASKED : 0,
563 };
564
565 xe_map_memcpy_to(ads_to_xe(ads), regset_map, n_entry * sizeof(entry),
566 &entry, sizeof(entry));
567 }
568
guc_mmio_regset_write(struct xe_guc_ads * ads,struct iosys_map * regset_map,struct xe_hw_engine * hwe)569 static unsigned int guc_mmio_regset_write(struct xe_guc_ads *ads,
570 struct iosys_map *regset_map,
571 struct xe_hw_engine *hwe)
572 {
573 struct xe_device *xe = ads_to_xe(ads);
574 struct xe_hw_engine *hwe_rcs_reset_domain =
575 xe_gt_any_hw_engine_by_reset_domain(hwe->gt, XE_ENGINE_CLASS_RENDER);
576 struct xe_reg_sr_entry *entry;
577 unsigned long idx;
578 unsigned int count = 0;
579 const struct {
580 struct xe_reg reg;
581 bool skip;
582 } *e, extra_regs[] = {
583 { .reg = RING_MODE(hwe->mmio_base), },
584 { .reg = RING_HWS_PGA(hwe->mmio_base), },
585 { .reg = RING_IMR(hwe->mmio_base), },
586 { .reg = RCU_MODE, .skip = hwe != hwe_rcs_reset_domain },
587 { .reg = CCS_MODE,
588 .skip = hwe != hwe_rcs_reset_domain || !xe_gt_ccs_mode_enabled(hwe->gt) },
589 };
590 u32 i;
591
592 BUILD_BUG_ON(ARRAY_SIZE(extra_regs) > ADS_REGSET_EXTRA_MAX);
593
594 xa_for_each(&hwe->reg_sr.xa, idx, entry)
595 guc_mmio_regset_write_one(ads, regset_map, entry->reg, count++);
596
597 for (e = extra_regs; e < extra_regs + ARRAY_SIZE(extra_regs); e++) {
598 if (e->skip)
599 continue;
600
601 guc_mmio_regset_write_one(ads, regset_map, e->reg, count++);
602 }
603
604 /* Wa_1607983814 */
605 if (needs_wa_1607983814(xe) && hwe->class == XE_ENGINE_CLASS_RENDER) {
606 for (i = 0; i < LNCFCMOCS_REG_COUNT; i++) {
607 guc_mmio_regset_write_one(ads, regset_map,
608 XELP_LNCFCMOCS(i), count++);
609 }
610 }
611
612 return count;
613 }
614
guc_mmio_reg_state_init(struct xe_guc_ads * ads)615 static void guc_mmio_reg_state_init(struct xe_guc_ads *ads)
616 {
617 size_t regset_offset = guc_ads_regset_offset(ads);
618 struct xe_gt *gt = ads_to_gt(ads);
619 struct xe_hw_engine *hwe;
620 enum xe_hw_engine_id id;
621 u32 addr = xe_bo_ggtt_addr(ads->bo) + regset_offset;
622 struct iosys_map regset_map = IOSYS_MAP_INIT_OFFSET(ads_to_map(ads),
623 regset_offset);
624 unsigned int regset_used = 0;
625
626 for_each_hw_engine(hwe, gt, id) {
627 unsigned int count;
628 u8 gc;
629
630 /*
631 * 1. Write all MMIO entries for this exec queue to the table. No
632 * need to worry about fused-off engines and when there are
633 * entries in the regset: the reg_state_list has been zero'ed
634 * by xe_guc_ads_populate()
635 */
636 count = guc_mmio_regset_write(ads, ®set_map, hwe);
637 if (!count)
638 continue;
639
640 /*
641 * 2. Record in the header (ads.reg_state_list) the address
642 * location and number of entries
643 */
644 gc = xe_engine_class_to_guc_class(hwe->class);
645 ads_blob_write(ads, ads.reg_state_list[gc][hwe->instance].address, addr);
646 ads_blob_write(ads, ads.reg_state_list[gc][hwe->instance].count, count);
647
648 addr += count * sizeof(struct guc_mmio_reg);
649 iosys_map_incr(®set_map, count * sizeof(struct guc_mmio_reg));
650
651 regset_used += count * sizeof(struct guc_mmio_reg);
652 }
653
654 xe_gt_assert(gt, regset_used <= ads->regset_size);
655 }
656
guc_um_init_params(struct xe_guc_ads * ads)657 static void guc_um_init_params(struct xe_guc_ads *ads)
658 {
659 u32 um_queue_offset = guc_ads_um_queues_offset(ads);
660 u64 base_dpa;
661 u32 base_ggtt;
662 int i;
663
664 base_ggtt = xe_bo_ggtt_addr(ads->bo) + um_queue_offset;
665 base_dpa = xe_bo_main_addr(ads->bo, PAGE_SIZE) + um_queue_offset;
666
667 for (i = 0; i < GUC_UM_HW_QUEUE_MAX; ++i) {
668 ads_blob_write(ads, um_init_params.queue_params[i].base_dpa,
669 base_dpa + (i * GUC_UM_QUEUE_SIZE));
670 ads_blob_write(ads, um_init_params.queue_params[i].base_ggtt_address,
671 base_ggtt + (i * GUC_UM_QUEUE_SIZE));
672 ads_blob_write(ads, um_init_params.queue_params[i].size_in_bytes,
673 GUC_UM_QUEUE_SIZE);
674 }
675
676 ads_blob_write(ads, um_init_params.page_response_timeout_in_us,
677 GUC_PAGE_RES_TIMEOUT_US);
678 }
679
guc_doorbell_init(struct xe_guc_ads * ads)680 static void guc_doorbell_init(struct xe_guc_ads *ads)
681 {
682 struct xe_device *xe = ads_to_xe(ads);
683 struct xe_gt *gt = ads_to_gt(ads);
684
685 if (GRAPHICS_VER(xe) >= 12 && !IS_DGFX(xe)) {
686 u32 distdbreg =
687 xe_mmio_read32(gt, DIST_DBS_POPULATED);
688
689 ads_blob_write(ads,
690 system_info.generic_gt_sysinfo[GUC_GENERIC_GT_SYSINFO_DOORBELL_COUNT_PER_SQIDI],
691 REG_FIELD_GET(DOORBELLS_PER_SQIDI_MASK, distdbreg) + 1);
692 }
693 }
694
695 /**
696 * xe_guc_ads_populate_minimal - populate minimal ADS
697 * @ads: Additional data structures object
698 *
699 * This function populates a minimal ADS that does not support submissions but
700 * enough so the GuC can load and the hwconfig table can be read.
701 */
xe_guc_ads_populate_minimal(struct xe_guc_ads * ads)702 void xe_guc_ads_populate_minimal(struct xe_guc_ads *ads)
703 {
704 struct xe_gt *gt = ads_to_gt(ads);
705 struct iosys_map info_map = IOSYS_MAP_INIT_OFFSET(ads_to_map(ads),
706 offsetof(struct __guc_ads_blob, system_info));
707 u32 base = xe_bo_ggtt_addr(ads->bo);
708
709 xe_gt_assert(gt, ads->bo);
710
711 xe_map_memset(ads_to_xe(ads), ads_to_map(ads), 0, 0, ads->bo->size);
712 guc_policies_init(ads);
713 guc_prep_golden_lrc_null(ads);
714 guc_mapping_table_init_invalid(gt, &info_map);
715 guc_doorbell_init(ads);
716
717 ads_blob_write(ads, ads.scheduler_policies, base +
718 offsetof(struct __guc_ads_blob, policies));
719 ads_blob_write(ads, ads.gt_system_info, base +
720 offsetof(struct __guc_ads_blob, system_info));
721 ads_blob_write(ads, ads.private_data, base +
722 guc_ads_private_data_offset(ads));
723 }
724
xe_guc_ads_populate(struct xe_guc_ads * ads)725 void xe_guc_ads_populate(struct xe_guc_ads *ads)
726 {
727 struct xe_device *xe = ads_to_xe(ads);
728 struct xe_gt *gt = ads_to_gt(ads);
729 struct iosys_map info_map = IOSYS_MAP_INIT_OFFSET(ads_to_map(ads),
730 offsetof(struct __guc_ads_blob, system_info));
731 u32 base = xe_bo_ggtt_addr(ads->bo);
732
733 xe_gt_assert(gt, ads->bo);
734
735 xe_map_memset(ads_to_xe(ads), ads_to_map(ads), 0, 0, ads->bo->size);
736 guc_policies_init(ads);
737 fill_engine_enable_masks(gt, &info_map);
738 guc_mmio_reg_state_init(ads);
739 guc_prep_golden_lrc_null(ads);
740 guc_mapping_table_init(gt, &info_map);
741 guc_capture_list_init(ads);
742 guc_doorbell_init(ads);
743 guc_waklv_init(ads);
744
745 if (xe->info.has_usm) {
746 guc_um_init_params(ads);
747 ads_blob_write(ads, ads.um_init_data, base +
748 offsetof(struct __guc_ads_blob, um_init_params));
749 }
750
751 ads_blob_write(ads, ads.scheduler_policies, base +
752 offsetof(struct __guc_ads_blob, policies));
753 ads_blob_write(ads, ads.gt_system_info, base +
754 offsetof(struct __guc_ads_blob, system_info));
755 ads_blob_write(ads, ads.private_data, base +
756 guc_ads_private_data_offset(ads));
757 }
758
guc_populate_golden_lrc(struct xe_guc_ads * ads)759 static void guc_populate_golden_lrc(struct xe_guc_ads *ads)
760 {
761 struct xe_device *xe = ads_to_xe(ads);
762 struct xe_gt *gt = ads_to_gt(ads);
763 struct iosys_map info_map = IOSYS_MAP_INIT_OFFSET(ads_to_map(ads),
764 offsetof(struct __guc_ads_blob, system_info));
765 size_t total_size = 0, alloc_size, real_size;
766 u32 addr_ggtt, offset;
767 int class;
768
769 offset = guc_ads_golden_lrc_offset(ads);
770 addr_ggtt = xe_bo_ggtt_addr(ads->bo) + offset;
771
772 for (class = 0; class < XE_ENGINE_CLASS_MAX; ++class) {
773 u8 guc_class;
774
775 guc_class = xe_engine_class_to_guc_class(class);
776
777 if (!info_map_read(xe, &info_map,
778 engine_enabled_masks[guc_class]))
779 continue;
780
781 xe_gt_assert(gt, gt->default_lrc[class]);
782
783 real_size = xe_gt_lrc_size(gt, class);
784 alloc_size = PAGE_ALIGN(real_size);
785 total_size += alloc_size;
786
787 /*
788 * This interface is slightly confusing. We need to pass the
789 * base address of the full golden context and the size of just
790 * the engine state, which is the section of the context image
791 * that starts after the execlists LRC registers. This is
792 * required to allow the GuC to restore just the engine state
793 * when a watchdog reset occurs.
794 * We calculate the engine state size by removing the size of
795 * what comes before it in the context image (which is identical
796 * on all engines).
797 */
798 ads_blob_write(ads, ads.eng_state_size[guc_class],
799 real_size - xe_lrc_skip_size(xe));
800 ads_blob_write(ads, ads.golden_context_lrca[guc_class],
801 addr_ggtt);
802
803 xe_map_memcpy_to(xe, ads_to_map(ads), offset,
804 gt->default_lrc[class], real_size);
805
806 addr_ggtt += alloc_size;
807 offset += alloc_size;
808 }
809
810 xe_gt_assert(gt, total_size == ads->golden_lrc_size);
811 }
812
xe_guc_ads_populate_post_load(struct xe_guc_ads * ads)813 void xe_guc_ads_populate_post_load(struct xe_guc_ads *ads)
814 {
815 guc_populate_golden_lrc(ads);
816 }
817
guc_ads_action_update_policies(struct xe_guc_ads * ads,u32 policy_offset)818 static int guc_ads_action_update_policies(struct xe_guc_ads *ads, u32 policy_offset)
819 {
820 struct xe_guc_ct *ct = &ads_to_guc(ads)->ct;
821 u32 action[] = {
822 XE_GUC_ACTION_GLOBAL_SCHED_POLICY_CHANGE,
823 policy_offset
824 };
825
826 return xe_guc_ct_send(ct, action, ARRAY_SIZE(action), 0, 0);
827 }
828
829 /**
830 * xe_guc_ads_scheduler_policy_toggle_reset - Toggle reset policy
831 * @ads: Additional data structures object
832 *
833 * This function update the GuC's engine reset policy based on wedged.mode.
834 *
835 * Return: 0 on success, and negative error code otherwise.
836 */
xe_guc_ads_scheduler_policy_toggle_reset(struct xe_guc_ads * ads)837 int xe_guc_ads_scheduler_policy_toggle_reset(struct xe_guc_ads *ads)
838 {
839 struct xe_device *xe = ads_to_xe(ads);
840 struct xe_gt *gt = ads_to_gt(ads);
841 struct xe_tile *tile = gt_to_tile(gt);
842 struct guc_policies *policies;
843 struct xe_bo *bo;
844 int ret = 0;
845
846 policies = kmalloc(sizeof(*policies), GFP_KERNEL);
847 if (!policies)
848 return -ENOMEM;
849
850 policies->dpc_promote_time = ads_blob_read(ads, policies.dpc_promote_time);
851 policies->max_num_work_items = ads_blob_read(ads, policies.max_num_work_items);
852 policies->is_valid = 1;
853 if (xe->wedged.mode == 2)
854 policies->global_flags |= GLOBAL_POLICY_DISABLE_ENGINE_RESET;
855 else
856 policies->global_flags &= ~GLOBAL_POLICY_DISABLE_ENGINE_RESET;
857
858 bo = xe_managed_bo_create_from_data(xe, tile, policies, sizeof(struct guc_policies),
859 XE_BO_FLAG_VRAM_IF_DGFX(tile) |
860 XE_BO_FLAG_GGTT);
861 if (IS_ERR(bo)) {
862 ret = PTR_ERR(bo);
863 goto out;
864 }
865
866 ret = guc_ads_action_update_policies(ads, xe_bo_ggtt_addr(bo));
867 out:
868 kfree(policies);
869 return ret;
870 }
871