1 // SPDX-License-Identifier: MIT
2 /*
3 * Copyright © 2023 Intel Corporation
4 */
5
6 #include "xe_pat.h"
7
8 #include <uapi/drm/xe_drm.h>
9
10 #include <generated/xe_wa_oob.h>
11
12 #include "regs/xe_reg_defs.h"
13 #include "xe_assert.h"
14 #include "xe_device.h"
15 #include "xe_force_wake.h"
16 #include "xe_gt.h"
17 #include "xe_gt_mcr.h"
18 #include "xe_mmio.h"
19 #include "xe_sriov.h"
20 #include "xe_wa.h"
21
22 #define _PAT_ATS 0x47fc
23 #define _PAT_INDEX(index) _PICK_EVEN_2RANGES(index, 8, \
24 0x4800, 0x4804, \
25 0x4848, 0x484c)
26 #define _PAT_PTA 0x4820
27
28 #define XE2_NO_PROMOTE REG_BIT(10)
29 #define XE2_COMP_EN REG_BIT(9)
30 #define XE2_L3_CLOS REG_GENMASK(7, 6)
31 #define XE2_L3_POLICY REG_GENMASK(5, 4)
32 #define XE2_L4_POLICY REG_GENMASK(3, 2)
33 #define XE2_COH_MODE REG_GENMASK(1, 0)
34
35 #define XELPG_L4_POLICY_MASK REG_GENMASK(3, 2)
36 #define XELPG_PAT_3_UC REG_FIELD_PREP(XELPG_L4_POLICY_MASK, 3)
37 #define XELPG_PAT_1_WT REG_FIELD_PREP(XELPG_L4_POLICY_MASK, 1)
38 #define XELPG_PAT_0_WB REG_FIELD_PREP(XELPG_L4_POLICY_MASK, 0)
39 #define XELPG_INDEX_COH_MODE_MASK REG_GENMASK(1, 0)
40 #define XELPG_3_COH_2W REG_FIELD_PREP(XELPG_INDEX_COH_MODE_MASK, 3)
41 #define XELPG_2_COH_1W REG_FIELD_PREP(XELPG_INDEX_COH_MODE_MASK, 2)
42 #define XELPG_0_COH_NON REG_FIELD_PREP(XELPG_INDEX_COH_MODE_MASK, 0)
43
44 #define XEHPC_CLOS_LEVEL_MASK REG_GENMASK(3, 2)
45 #define XEHPC_PAT_CLOS(x) REG_FIELD_PREP(XEHPC_CLOS_LEVEL_MASK, x)
46
47 #define XELP_MEM_TYPE_MASK REG_GENMASK(1, 0)
48 #define XELP_PAT_WB REG_FIELD_PREP(XELP_MEM_TYPE_MASK, 3)
49 #define XELP_PAT_WT REG_FIELD_PREP(XELP_MEM_TYPE_MASK, 2)
50 #define XELP_PAT_WC REG_FIELD_PREP(XELP_MEM_TYPE_MASK, 1)
51 #define XELP_PAT_UC REG_FIELD_PREP(XELP_MEM_TYPE_MASK, 0)
52
53 static const char *XELP_MEM_TYPE_STR_MAP[] = { "UC", "WC", "WT", "WB" };
54
55 struct xe_pat_ops {
56 void (*program_graphics)(struct xe_gt *gt, const struct xe_pat_table_entry table[],
57 int n_entries);
58 void (*program_media)(struct xe_gt *gt, const struct xe_pat_table_entry table[],
59 int n_entries);
60 void (*dump)(struct xe_gt *gt, struct drm_printer *p);
61 };
62
63 static const struct xe_pat_table_entry xelp_pat_table[] = {
64 [0] = { XELP_PAT_WB, XE_COH_AT_LEAST_1WAY },
65 [1] = { XELP_PAT_WC, XE_COH_NONE },
66 [2] = { XELP_PAT_WT, XE_COH_NONE },
67 [3] = { XELP_PAT_UC, XE_COH_NONE },
68 };
69
70 static const struct xe_pat_table_entry xehpc_pat_table[] = {
71 [0] = { XELP_PAT_UC, XE_COH_NONE },
72 [1] = { XELP_PAT_WC, XE_COH_NONE },
73 [2] = { XELP_PAT_WT, XE_COH_NONE },
74 [3] = { XELP_PAT_WB, XE_COH_AT_LEAST_1WAY },
75 [4] = { XEHPC_PAT_CLOS(1) | XELP_PAT_WT, XE_COH_NONE },
76 [5] = { XEHPC_PAT_CLOS(1) | XELP_PAT_WB, XE_COH_AT_LEAST_1WAY },
77 [6] = { XEHPC_PAT_CLOS(2) | XELP_PAT_WT, XE_COH_NONE },
78 [7] = { XEHPC_PAT_CLOS(2) | XELP_PAT_WB, XE_COH_AT_LEAST_1WAY },
79 };
80
81 static const struct xe_pat_table_entry xelpg_pat_table[] = {
82 [0] = { XELPG_PAT_0_WB, XE_COH_NONE },
83 [1] = { XELPG_PAT_1_WT, XE_COH_NONE },
84 [2] = { XELPG_PAT_3_UC, XE_COH_NONE },
85 [3] = { XELPG_PAT_0_WB | XELPG_2_COH_1W, XE_COH_AT_LEAST_1WAY },
86 [4] = { XELPG_PAT_0_WB | XELPG_3_COH_2W, XE_COH_AT_LEAST_1WAY },
87 };
88
89 /*
90 * The Xe2 table is getting large/complicated so it's easier to review if
91 * provided in a form that exactly matches the bspec's formatting. The meaning
92 * of the fields here are:
93 * - no_promote: 0=promotable, 1=no promote
94 * - comp_en: 0=disable, 1=enable
95 * - l3clos: L3 class of service (0-3)
96 * - l3_policy: 0=WB, 1=XD ("WB - Transient Display"), 3=UC
97 * - l4_policy: 0=WB, 1=WT, 3=UC
98 * - coh_mode: 0=no snoop, 2=1-way coherent, 3=2-way coherent
99 *
100 * Reserved entries should be programmed with the maximum caching, minimum
101 * coherency (which matches an all-0's encoding), so we can just omit them
102 * in the table.
103 */
104 #define XE2_PAT(no_promote, comp_en, l3clos, l3_policy, l4_policy, __coh_mode) \
105 { \
106 .value = (no_promote ? XE2_NO_PROMOTE : 0) | \
107 (comp_en ? XE2_COMP_EN : 0) | \
108 REG_FIELD_PREP(XE2_L3_CLOS, l3clos) | \
109 REG_FIELD_PREP(XE2_L3_POLICY, l3_policy) | \
110 REG_FIELD_PREP(XE2_L4_POLICY, l4_policy) | \
111 REG_FIELD_PREP(XE2_COH_MODE, __coh_mode), \
112 .coh_mode = __coh_mode ? XE_COH_AT_LEAST_1WAY : XE_COH_NONE \
113 }
114
115 static const struct xe_pat_table_entry xe2_pat_table[] = {
116 [ 0] = XE2_PAT( 0, 0, 0, 0, 3, 0 ),
117 [ 1] = XE2_PAT( 0, 0, 0, 0, 3, 2 ),
118 [ 2] = XE2_PAT( 0, 0, 0, 0, 3, 3 ),
119 [ 3] = XE2_PAT( 0, 0, 0, 3, 3, 0 ),
120 [ 4] = XE2_PAT( 0, 0, 0, 3, 0, 2 ),
121 [ 5] = XE2_PAT( 0, 0, 0, 3, 3, 2 ),
122 [ 6] = XE2_PAT( 1, 0, 0, 1, 3, 0 ),
123 [ 7] = XE2_PAT( 0, 0, 0, 3, 0, 3 ),
124 [ 8] = XE2_PAT( 0, 0, 0, 3, 0, 0 ),
125 [ 9] = XE2_PAT( 0, 1, 0, 0, 3, 0 ),
126 [10] = XE2_PAT( 0, 1, 0, 3, 0, 0 ),
127 [11] = XE2_PAT( 1, 1, 0, 1, 3, 0 ),
128 [12] = XE2_PAT( 0, 1, 0, 3, 3, 0 ),
129 [13] = XE2_PAT( 0, 0, 0, 0, 0, 0 ),
130 [14] = XE2_PAT( 0, 1, 0, 0, 0, 0 ),
131 [15] = XE2_PAT( 1, 1, 0, 1, 1, 0 ),
132 /* 16..19 are reserved; leave set to all 0's */
133 [20] = XE2_PAT( 0, 0, 1, 0, 3, 0 ),
134 [21] = XE2_PAT( 0, 1, 1, 0, 3, 0 ),
135 [22] = XE2_PAT( 0, 0, 1, 0, 3, 2 ),
136 [23] = XE2_PAT( 0, 0, 1, 0, 3, 3 ),
137 [24] = XE2_PAT( 0, 0, 2, 0, 3, 0 ),
138 [25] = XE2_PAT( 0, 1, 2, 0, 3, 0 ),
139 [26] = XE2_PAT( 0, 0, 2, 0, 3, 2 ),
140 [27] = XE2_PAT( 0, 0, 2, 0, 3, 3 ),
141 [28] = XE2_PAT( 0, 0, 3, 0, 3, 0 ),
142 [29] = XE2_PAT( 0, 1, 3, 0, 3, 0 ),
143 [30] = XE2_PAT( 0, 0, 3, 0, 3, 2 ),
144 [31] = XE2_PAT( 0, 0, 3, 0, 3, 3 ),
145 };
146
147 /* Special PAT values programmed outside the main table */
148 static const struct xe_pat_table_entry xe2_pat_ats = XE2_PAT( 0, 0, 0, 0, 3, 3 );
149 static const struct xe_pat_table_entry xe2_pat_pta = XE2_PAT( 0, 0, 0, 0, 3, 0 );
150
xe_pat_index_get_coh_mode(struct xe_device * xe,u16 pat_index)151 u16 xe_pat_index_get_coh_mode(struct xe_device *xe, u16 pat_index)
152 {
153 WARN_ON(pat_index >= xe->pat.n_entries);
154 return xe->pat.table[pat_index].coh_mode;
155 }
156
program_pat(struct xe_gt * gt,const struct xe_pat_table_entry table[],int n_entries)157 static void program_pat(struct xe_gt *gt, const struct xe_pat_table_entry table[],
158 int n_entries)
159 {
160 for (int i = 0; i < n_entries; i++) {
161 struct xe_reg reg = XE_REG(_PAT_INDEX(i));
162
163 xe_mmio_write32(gt, reg, table[i].value);
164 }
165 }
166
program_pat_mcr(struct xe_gt * gt,const struct xe_pat_table_entry table[],int n_entries)167 static void program_pat_mcr(struct xe_gt *gt, const struct xe_pat_table_entry table[],
168 int n_entries)
169 {
170 for (int i = 0; i < n_entries; i++) {
171 struct xe_reg_mcr reg_mcr = XE_REG_MCR(_PAT_INDEX(i));
172
173 xe_gt_mcr_multicast_write(gt, reg_mcr, table[i].value);
174 }
175 }
176
xelp_dump(struct xe_gt * gt,struct drm_printer * p)177 static void xelp_dump(struct xe_gt *gt, struct drm_printer *p)
178 {
179 struct xe_device *xe = gt_to_xe(gt);
180 int i, err;
181
182 err = xe_force_wake_get(gt_to_fw(gt), XE_FW_GT);
183 if (err)
184 goto err_fw;
185
186 drm_printf(p, "PAT table:\n");
187
188 for (i = 0; i < xe->pat.n_entries; i++) {
189 u32 pat = xe_mmio_read32(gt, XE_REG(_PAT_INDEX(i)));
190 u8 mem_type = REG_FIELD_GET(XELP_MEM_TYPE_MASK, pat);
191
192 drm_printf(p, "PAT[%2d] = %s (%#8x)\n", i,
193 XELP_MEM_TYPE_STR_MAP[mem_type], pat);
194 }
195
196 err = xe_force_wake_put(gt_to_fw(gt), XE_FW_GT);
197 err_fw:
198 xe_assert(xe, !err);
199 }
200
201 static const struct xe_pat_ops xelp_pat_ops = {
202 .program_graphics = program_pat,
203 .dump = xelp_dump,
204 };
205
xehp_dump(struct xe_gt * gt,struct drm_printer * p)206 static void xehp_dump(struct xe_gt *gt, struct drm_printer *p)
207 {
208 struct xe_device *xe = gt_to_xe(gt);
209 int i, err;
210
211 err = xe_force_wake_get(gt_to_fw(gt), XE_FW_GT);
212 if (err)
213 goto err_fw;
214
215 drm_printf(p, "PAT table:\n");
216
217 for (i = 0; i < xe->pat.n_entries; i++) {
218 u32 pat = xe_gt_mcr_unicast_read_any(gt, XE_REG_MCR(_PAT_INDEX(i)));
219 u8 mem_type;
220
221 mem_type = REG_FIELD_GET(XELP_MEM_TYPE_MASK, pat);
222
223 drm_printf(p, "PAT[%2d] = %s (%#8x)\n", i,
224 XELP_MEM_TYPE_STR_MAP[mem_type], pat);
225 }
226
227 err = xe_force_wake_put(gt_to_fw(gt), XE_FW_GT);
228 err_fw:
229 xe_assert(xe, !err);
230 }
231
232 static const struct xe_pat_ops xehp_pat_ops = {
233 .program_graphics = program_pat_mcr,
234 .dump = xehp_dump,
235 };
236
xehpc_dump(struct xe_gt * gt,struct drm_printer * p)237 static void xehpc_dump(struct xe_gt *gt, struct drm_printer *p)
238 {
239 struct xe_device *xe = gt_to_xe(gt);
240 int i, err;
241
242 err = xe_force_wake_get(gt_to_fw(gt), XE_FW_GT);
243 if (err)
244 goto err_fw;
245
246 drm_printf(p, "PAT table:\n");
247
248 for (i = 0; i < xe->pat.n_entries; i++) {
249 u32 pat = xe_gt_mcr_unicast_read_any(gt, XE_REG_MCR(_PAT_INDEX(i)));
250
251 drm_printf(p, "PAT[%2d] = [ %u, %u ] (%#8x)\n", i,
252 REG_FIELD_GET(XELP_MEM_TYPE_MASK, pat),
253 REG_FIELD_GET(XEHPC_CLOS_LEVEL_MASK, pat), pat);
254 }
255
256 err = xe_force_wake_put(gt_to_fw(gt), XE_FW_GT);
257 err_fw:
258 xe_assert(xe, !err);
259 }
260
261 static const struct xe_pat_ops xehpc_pat_ops = {
262 .program_graphics = program_pat_mcr,
263 .dump = xehpc_dump,
264 };
265
xelpg_dump(struct xe_gt * gt,struct drm_printer * p)266 static void xelpg_dump(struct xe_gt *gt, struct drm_printer *p)
267 {
268 struct xe_device *xe = gt_to_xe(gt);
269 int i, err;
270
271 err = xe_force_wake_get(gt_to_fw(gt), XE_FW_GT);
272 if (err)
273 goto err_fw;
274
275 drm_printf(p, "PAT table:\n");
276
277 for (i = 0; i < xe->pat.n_entries; i++) {
278 u32 pat;
279
280 if (xe_gt_is_media_type(gt))
281 pat = xe_mmio_read32(gt, XE_REG(_PAT_INDEX(i)));
282 else
283 pat = xe_gt_mcr_unicast_read_any(gt, XE_REG_MCR(_PAT_INDEX(i)));
284
285 drm_printf(p, "PAT[%2d] = [ %u, %u ] (%#8x)\n", i,
286 REG_FIELD_GET(XELPG_L4_POLICY_MASK, pat),
287 REG_FIELD_GET(XELPG_INDEX_COH_MODE_MASK, pat), pat);
288 }
289
290 err = xe_force_wake_put(gt_to_fw(gt), XE_FW_GT);
291 err_fw:
292 xe_assert(xe, !err);
293 }
294
295 /*
296 * SAMedia register offsets are adjusted by the write methods and they target
297 * registers that are not MCR, while for normal GT they are MCR
298 */
299 static const struct xe_pat_ops xelpg_pat_ops = {
300 .program_graphics = program_pat,
301 .program_media = program_pat_mcr,
302 .dump = xelpg_dump,
303 };
304
xe2lpg_program_pat(struct xe_gt * gt,const struct xe_pat_table_entry table[],int n_entries)305 static void xe2lpg_program_pat(struct xe_gt *gt, const struct xe_pat_table_entry table[],
306 int n_entries)
307 {
308 program_pat_mcr(gt, table, n_entries);
309 xe_gt_mcr_multicast_write(gt, XE_REG_MCR(_PAT_ATS), xe2_pat_ats.value);
310
311 if (IS_DGFX(gt_to_xe(gt)))
312 xe_gt_mcr_multicast_write(gt, XE_REG_MCR(_PAT_PTA), xe2_pat_pta.value);
313 }
314
xe2lpm_program_pat(struct xe_gt * gt,const struct xe_pat_table_entry table[],int n_entries)315 static void xe2lpm_program_pat(struct xe_gt *gt, const struct xe_pat_table_entry table[],
316 int n_entries)
317 {
318 program_pat(gt, table, n_entries);
319 xe_mmio_write32(gt, XE_REG(_PAT_ATS), xe2_pat_ats.value);
320
321 if (IS_DGFX(gt_to_xe(gt)))
322 xe_mmio_write32(gt, XE_REG(_PAT_PTA), xe2_pat_pta.value);
323 }
324
xe2_dump(struct xe_gt * gt,struct drm_printer * p)325 static void xe2_dump(struct xe_gt *gt, struct drm_printer *p)
326 {
327 struct xe_device *xe = gt_to_xe(gt);
328 int i, err;
329 u32 pat;
330
331 err = xe_force_wake_get(gt_to_fw(gt), XE_FW_GT);
332 if (err)
333 goto err_fw;
334
335 drm_printf(p, "PAT table:\n");
336
337 for (i = 0; i < xe->pat.n_entries; i++) {
338 if (xe_gt_is_media_type(gt))
339 pat = xe_mmio_read32(gt, XE_REG(_PAT_INDEX(i)));
340 else
341 pat = xe_gt_mcr_unicast_read_any(gt, XE_REG_MCR(_PAT_INDEX(i)));
342
343 drm_printf(p, "PAT[%2d] = [ %u, %u, %u, %u, %u, %u ] (%#8x)\n", i,
344 !!(pat & XE2_NO_PROMOTE),
345 !!(pat & XE2_COMP_EN),
346 REG_FIELD_GET(XE2_L3_CLOS, pat),
347 REG_FIELD_GET(XE2_L3_POLICY, pat),
348 REG_FIELD_GET(XE2_L4_POLICY, pat),
349 REG_FIELD_GET(XE2_COH_MODE, pat),
350 pat);
351 }
352
353 /*
354 * Also print PTA_MODE, which describes how the hardware accesses
355 * PPGTT entries.
356 */
357 if (xe_gt_is_media_type(gt))
358 pat = xe_mmio_read32(gt, XE_REG(_PAT_PTA));
359 else
360 pat = xe_gt_mcr_unicast_read_any(gt, XE_REG_MCR(_PAT_PTA));
361
362 drm_printf(p, "Page Table Access:\n");
363 drm_printf(p, "PTA_MODE= [ %u, %u, %u, %u, %u, %u ] (%#8x)\n",
364 !!(pat & XE2_NO_PROMOTE),
365 !!(pat & XE2_COMP_EN),
366 REG_FIELD_GET(XE2_L3_CLOS, pat),
367 REG_FIELD_GET(XE2_L3_POLICY, pat),
368 REG_FIELD_GET(XE2_L4_POLICY, pat),
369 REG_FIELD_GET(XE2_COH_MODE, pat),
370 pat);
371
372 err = xe_force_wake_put(gt_to_fw(gt), XE_FW_GT);
373 err_fw:
374 xe_assert(xe, !err);
375 }
376
377 static const struct xe_pat_ops xe2_pat_ops = {
378 .program_graphics = xe2lpg_program_pat,
379 .program_media = xe2lpm_program_pat,
380 .dump = xe2_dump,
381 };
382
xe_pat_init_early(struct xe_device * xe)383 void xe_pat_init_early(struct xe_device *xe)
384 {
385 if (GRAPHICS_VER(xe) == 20) {
386 xe->pat.ops = &xe2_pat_ops;
387 xe->pat.table = xe2_pat_table;
388
389 /* Wa_16023588340. XXX: Should use XE_WA */
390 if (GRAPHICS_VERx100(xe) == 2001)
391 xe->pat.n_entries = 28; /* Disable CLOS3 */
392 else
393 xe->pat.n_entries = ARRAY_SIZE(xe2_pat_table);
394
395 xe->pat.idx[XE_CACHE_NONE] = 3;
396 xe->pat.idx[XE_CACHE_WT] = 15;
397 xe->pat.idx[XE_CACHE_WB] = 2;
398 xe->pat.idx[XE_CACHE_NONE_COMPRESSION] = 12; /*Applicable on xe2 and beyond */
399 } else if (xe->info.platform == XE_METEORLAKE) {
400 xe->pat.ops = &xelpg_pat_ops;
401 xe->pat.table = xelpg_pat_table;
402 xe->pat.n_entries = ARRAY_SIZE(xelpg_pat_table);
403 xe->pat.idx[XE_CACHE_NONE] = 2;
404 xe->pat.idx[XE_CACHE_WT] = 1;
405 xe->pat.idx[XE_CACHE_WB] = 3;
406 } else if (xe->info.platform == XE_PVC) {
407 xe->pat.ops = &xehpc_pat_ops;
408 xe->pat.table = xehpc_pat_table;
409 xe->pat.n_entries = ARRAY_SIZE(xehpc_pat_table);
410 xe->pat.idx[XE_CACHE_NONE] = 0;
411 xe->pat.idx[XE_CACHE_WT] = 2;
412 xe->pat.idx[XE_CACHE_WB] = 3;
413 } else if (xe->info.platform == XE_DG2) {
414 /*
415 * Table is the same as previous platforms, but programming
416 * method has changed.
417 */
418 xe->pat.ops = &xehp_pat_ops;
419 xe->pat.table = xelp_pat_table;
420 xe->pat.n_entries = ARRAY_SIZE(xelp_pat_table);
421 xe->pat.idx[XE_CACHE_NONE] = 3;
422 xe->pat.idx[XE_CACHE_WT] = 2;
423 xe->pat.idx[XE_CACHE_WB] = 0;
424 } else if (GRAPHICS_VERx100(xe) <= 1210) {
425 WARN_ON_ONCE(!IS_DGFX(xe) && !xe->info.has_llc);
426 xe->pat.ops = &xelp_pat_ops;
427 xe->pat.table = xelp_pat_table;
428 xe->pat.n_entries = ARRAY_SIZE(xelp_pat_table);
429 xe->pat.idx[XE_CACHE_NONE] = 3;
430 xe->pat.idx[XE_CACHE_WT] = 2;
431 xe->pat.idx[XE_CACHE_WB] = 0;
432 } else {
433 /*
434 * Going forward we expect to need new PAT settings for most
435 * new platforms; failure to provide a new table can easily
436 * lead to subtle, hard-to-debug problems. If none of the
437 * conditions above match the platform we're running on we'll
438 * raise an error rather than trying to silently inherit the
439 * most recent platform's behavior.
440 */
441 drm_err(&xe->drm, "Missing PAT table for platform with graphics version %d.%02d!\n",
442 GRAPHICS_VER(xe), GRAPHICS_VERx100(xe) % 100);
443 }
444
445 /* VFs can't program nor dump PAT settings */
446 if (IS_SRIOV_VF(xe))
447 xe->pat.ops = NULL;
448
449 xe_assert(xe, !xe->pat.ops || xe->pat.ops->dump);
450 xe_assert(xe, !xe->pat.ops || xe->pat.ops->program_graphics);
451 xe_assert(xe, !xe->pat.ops || MEDIA_VER(xe) < 13 || xe->pat.ops->program_media);
452 }
453
xe_pat_init(struct xe_gt * gt)454 void xe_pat_init(struct xe_gt *gt)
455 {
456 struct xe_device *xe = gt_to_xe(gt);
457
458 if (!xe->pat.ops)
459 return;
460
461 if (xe_gt_is_media_type(gt))
462 xe->pat.ops->program_media(gt, xe->pat.table, xe->pat.n_entries);
463 else
464 xe->pat.ops->program_graphics(gt, xe->pat.table, xe->pat.n_entries);
465 }
466
xe_pat_dump(struct xe_gt * gt,struct drm_printer * p)467 void xe_pat_dump(struct xe_gt *gt, struct drm_printer *p)
468 {
469 struct xe_device *xe = gt_to_xe(gt);
470
471 if (!xe->pat.ops)
472 return;
473
474 xe->pat.ops->dump(gt, p);
475 }
476