1 // SPDX-License-Identifier: MIT
2 /*
3  * Copyright © 2022 Intel Corporation
4  */
5 
6 #include "xe_reg_sr.h"
7 
8 #include <kunit/visibility.h>
9 #include <linux/align.h>
10 #include <linux/string_helpers.h>
11 #include <linux/xarray.h>
12 
13 #include <drm/drm_managed.h>
14 #include <drm/drm_print.h>
15 
16 #include "regs/xe_engine_regs.h"
17 #include "regs/xe_gt_regs.h"
18 #include "xe_device_types.h"
19 #include "xe_force_wake.h"
20 #include "xe_gt.h"
21 #include "xe_gt_mcr.h"
22 #include "xe_gt_printk.h"
23 #include "xe_hw_engine_types.h"
24 #include "xe_macros.h"
25 #include "xe_mmio.h"
26 #include "xe_reg_whitelist.h"
27 #include "xe_rtp_types.h"
28 
29 #define XE_REG_SR_GROW_STEP_DEFAULT	16
30 
reg_sr_fini(struct drm_device * drm,void * arg)31 static void reg_sr_fini(struct drm_device *drm, void *arg)
32 {
33 	struct xe_reg_sr *sr = arg;
34 
35 	xa_destroy(&sr->xa);
36 	kfree(sr->pool.arr);
37 	memset(&sr->pool, 0, sizeof(sr->pool));
38 }
39 
xe_reg_sr_init(struct xe_reg_sr * sr,const char * name,struct xe_device * xe)40 int xe_reg_sr_init(struct xe_reg_sr *sr, const char *name, struct xe_device *xe)
41 {
42 	xa_init(&sr->xa);
43 	memset(&sr->pool, 0, sizeof(sr->pool));
44 	sr->pool.grow_step = XE_REG_SR_GROW_STEP_DEFAULT;
45 	sr->name = name;
46 
47 	return drmm_add_action_or_reset(&xe->drm, reg_sr_fini, sr);
48 }
49 EXPORT_SYMBOL_IF_KUNIT(xe_reg_sr_init);
50 
alloc_entry(struct xe_reg_sr * sr)51 static struct xe_reg_sr_entry *alloc_entry(struct xe_reg_sr *sr)
52 {
53 	if (sr->pool.used == sr->pool.allocated) {
54 		struct xe_reg_sr_entry *arr;
55 
56 		arr = krealloc_array(sr->pool.arr,
57 				     ALIGN(sr->pool.allocated + 1, sr->pool.grow_step),
58 				     sizeof(*arr), GFP_KERNEL);
59 		if (!arr)
60 			return NULL;
61 
62 		sr->pool.arr = arr;
63 		sr->pool.allocated += sr->pool.grow_step;
64 	}
65 
66 	return &sr->pool.arr[sr->pool.used++];
67 }
68 
compatible_entries(const struct xe_reg_sr_entry * e1,const struct xe_reg_sr_entry * e2)69 static bool compatible_entries(const struct xe_reg_sr_entry *e1,
70 			       const struct xe_reg_sr_entry *e2)
71 {
72 	/*
73 	 * Don't allow overwriting values: clr_bits/set_bits should be disjoint
74 	 * when operating in the same register
75 	 */
76 	if (e1->clr_bits & e2->clr_bits || e1->set_bits & e2->set_bits ||
77 	    e1->clr_bits & e2->set_bits || e1->set_bits & e2->clr_bits)
78 		return false;
79 
80 	if (e1->reg.raw != e2->reg.raw)
81 		return false;
82 
83 	return true;
84 }
85 
reg_sr_inc_error(struct xe_reg_sr * sr)86 static void reg_sr_inc_error(struct xe_reg_sr *sr)
87 {
88 #if IS_ENABLED(CONFIG_DRM_XE_KUNIT_TEST)
89 	sr->errors++;
90 #endif
91 }
92 
xe_reg_sr_add(struct xe_reg_sr * sr,const struct xe_reg_sr_entry * e,struct xe_gt * gt)93 int xe_reg_sr_add(struct xe_reg_sr *sr,
94 		  const struct xe_reg_sr_entry *e,
95 		  struct xe_gt *gt)
96 {
97 	unsigned long idx = e->reg.addr;
98 	struct xe_reg_sr_entry *pentry = xa_load(&sr->xa, idx);
99 	int ret;
100 
101 	if (pentry) {
102 		if (!compatible_entries(pentry, e)) {
103 			ret = -EINVAL;
104 			goto fail;
105 		}
106 
107 		pentry->clr_bits |= e->clr_bits;
108 		pentry->set_bits |= e->set_bits;
109 		pentry->read_mask |= e->read_mask;
110 
111 		return 0;
112 	}
113 
114 	pentry = alloc_entry(sr);
115 	if (!pentry) {
116 		ret = -ENOMEM;
117 		goto fail;
118 	}
119 
120 	*pentry = *e;
121 	ret = xa_err(xa_store(&sr->xa, idx, pentry, GFP_KERNEL));
122 	if (ret)
123 		goto fail;
124 
125 	return 0;
126 
127 fail:
128 	xe_gt_err(gt,
129 		  "discarding save-restore reg %04lx (clear: %08x, set: %08x, masked: %s, mcr: %s): ret=%d\n",
130 		  idx, e->clr_bits, e->set_bits,
131 		  str_yes_no(e->reg.masked),
132 		  str_yes_no(e->reg.mcr),
133 		  ret);
134 	reg_sr_inc_error(sr);
135 
136 	return ret;
137 }
138 
139 /*
140  * Convert back from encoded value to type-safe, only to be used when reg.mcr
141  * is true
142  */
to_xe_reg_mcr(const struct xe_reg reg)143 static struct xe_reg_mcr to_xe_reg_mcr(const struct xe_reg reg)
144 {
145 	return (const struct xe_reg_mcr){.__reg.raw = reg.raw };
146 }
147 
apply_one_mmio(struct xe_gt * gt,struct xe_reg_sr_entry * entry)148 static void apply_one_mmio(struct xe_gt *gt, struct xe_reg_sr_entry *entry)
149 {
150 	struct xe_reg reg = entry->reg;
151 	struct xe_reg_mcr reg_mcr = to_xe_reg_mcr(reg);
152 	u32 val;
153 
154 	/*
155 	 * If this is a masked register, need to set the upper 16 bits.
156 	 * Set them to clr_bits since that is always a superset of the bits
157 	 * being modified.
158 	 *
159 	 * When it's not masked, we have to read it from hardware, unless we are
160 	 * supposed to set all bits.
161 	 */
162 	if (reg.masked)
163 		val = entry->clr_bits << 16;
164 	else if (entry->clr_bits + 1)
165 		val = (reg.mcr ?
166 		       xe_gt_mcr_unicast_read_any(gt, reg_mcr) :
167 		       xe_mmio_read32(gt, reg)) & (~entry->clr_bits);
168 	else
169 		val = 0;
170 
171 	/*
172 	 * TODO: add selftest to validate all tables, regardless of platform:
173 	 *   - Masked registers can't have set_bits with upper bits set
174 	 *   - set_bits must be contained in clr_bits
175 	 */
176 	val |= entry->set_bits;
177 
178 	xe_gt_dbg(gt, "REG[0x%x] = 0x%08x", reg.addr, val);
179 
180 	if (entry->reg.mcr)
181 		xe_gt_mcr_multicast_write(gt, reg_mcr, val);
182 	else
183 		xe_mmio_write32(gt, reg, val);
184 }
185 
xe_reg_sr_apply_mmio(struct xe_reg_sr * sr,struct xe_gt * gt)186 void xe_reg_sr_apply_mmio(struct xe_reg_sr *sr, struct xe_gt *gt)
187 {
188 	struct xe_reg_sr_entry *entry;
189 	unsigned long reg;
190 	int err;
191 
192 	if (xa_empty(&sr->xa))
193 		return;
194 
195 	xe_gt_dbg(gt, "Applying %s save-restore MMIOs\n", sr->name);
196 
197 	err = xe_force_wake_get(&gt->mmio.fw, XE_FORCEWAKE_ALL);
198 	if (err)
199 		goto err_force_wake;
200 
201 	xa_for_each(&sr->xa, reg, entry)
202 		apply_one_mmio(gt, entry);
203 
204 	err = xe_force_wake_put(&gt->mmio.fw, XE_FORCEWAKE_ALL);
205 	XE_WARN_ON(err);
206 
207 	return;
208 
209 err_force_wake:
210 	xe_gt_err(gt, "Failed to apply, err=%d\n", err);
211 }
212 
xe_reg_sr_apply_whitelist(struct xe_hw_engine * hwe)213 void xe_reg_sr_apply_whitelist(struct xe_hw_engine *hwe)
214 {
215 	struct xe_reg_sr *sr = &hwe->reg_whitelist;
216 	struct xe_gt *gt = hwe->gt;
217 	struct xe_device *xe = gt_to_xe(gt);
218 	struct xe_reg_sr_entry *entry;
219 	struct drm_printer p;
220 	u32 mmio_base = hwe->mmio_base;
221 	unsigned long reg;
222 	unsigned int slot = 0;
223 	int err;
224 
225 	if (xa_empty(&sr->xa))
226 		return;
227 
228 	drm_dbg(&xe->drm, "Whitelisting %s registers\n", sr->name);
229 
230 	err = xe_force_wake_get(&gt->mmio.fw, XE_FORCEWAKE_ALL);
231 	if (err)
232 		goto err_force_wake;
233 
234 	p = drm_dbg_printer(&xe->drm, DRM_UT_DRIVER, NULL);
235 	xa_for_each(&sr->xa, reg, entry) {
236 		if (slot == RING_MAX_NONPRIV_SLOTS) {
237 			xe_gt_err(gt,
238 				  "hwe %s: maximum register whitelist slots (%d) reached, refusing to add more\n",
239 				  hwe->name, RING_MAX_NONPRIV_SLOTS);
240 			break;
241 		}
242 
243 		xe_reg_whitelist_print_entry(&p, 0, reg, entry);
244 		xe_mmio_write32(gt, RING_FORCE_TO_NONPRIV(mmio_base, slot),
245 				reg | entry->set_bits);
246 		slot++;
247 	}
248 
249 	/* And clear the rest just in case of garbage */
250 	for (; slot < RING_MAX_NONPRIV_SLOTS; slot++) {
251 		u32 addr = RING_NOPID(mmio_base).addr;
252 
253 		xe_mmio_write32(gt, RING_FORCE_TO_NONPRIV(mmio_base, slot), addr);
254 	}
255 
256 	err = xe_force_wake_put(&gt->mmio.fw, XE_FORCEWAKE_ALL);
257 	XE_WARN_ON(err);
258 
259 	return;
260 
261 err_force_wake:
262 	drm_err(&xe->drm, "Failed to apply, err=%d\n", err);
263 }
264 
265 /**
266  * xe_reg_sr_dump - print all save/restore entries
267  * @sr: Save/restore entries
268  * @p: DRM printer
269  */
xe_reg_sr_dump(struct xe_reg_sr * sr,struct drm_printer * p)270 void xe_reg_sr_dump(struct xe_reg_sr *sr, struct drm_printer *p)
271 {
272 	struct xe_reg_sr_entry *entry;
273 	unsigned long reg;
274 
275 	if (!sr->name || xa_empty(&sr->xa))
276 		return;
277 
278 	drm_printf(p, "%s\n", sr->name);
279 	xa_for_each(&sr->xa, reg, entry)
280 		drm_printf(p, "\tREG[0x%lx] clr=0x%08x set=0x%08x masked=%s mcr=%s\n",
281 			   reg, entry->clr_bits, entry->set_bits,
282 			   str_yes_no(entry->reg.masked),
283 			   str_yes_no(entry->reg.mcr));
284 }
285