1 // SPDX-License-Identifier: MIT
2 /*
3  * Copyright © 2021 Intel Corporation
4  */
5 
6 /**
7  * DOC: display pinning helpers
8  */
9 
10 #include "gem/i915_gem_domain.h"
11 #include "gem/i915_gem_object.h"
12 
13 #include "i915_drv.h"
14 #include "intel_atomic_plane.h"
15 #include "intel_display_types.h"
16 #include "intel_dpt.h"
17 #include "intel_fb.h"
18 #include "intel_fb_pin.h"
19 
20 static struct i915_vma *
intel_fb_pin_to_dpt(const struct drm_framebuffer * fb,const struct i915_gtt_view * view,unsigned int alignment,unsigned long * out_flags,struct i915_address_space * vm)21 intel_fb_pin_to_dpt(const struct drm_framebuffer *fb,
22 		    const struct i915_gtt_view *view,
23 		    unsigned int alignment,
24 		    unsigned long *out_flags,
25 		    struct i915_address_space *vm)
26 {
27 	struct drm_device *dev = fb->dev;
28 	struct drm_i915_private *dev_priv = to_i915(dev);
29 	struct drm_i915_gem_object *obj = intel_fb_obj(fb);
30 	struct i915_gem_ww_ctx ww;
31 	struct i915_vma *vma;
32 	int ret;
33 
34 	/*
35 	 * We are not syncing against the binding (and potential migrations)
36 	 * below, so this vm must never be async.
37 	 */
38 	if (drm_WARN_ON(&dev_priv->drm, vm->bind_async_flags))
39 		return ERR_PTR(-EINVAL);
40 
41 	if (WARN_ON(!i915_gem_object_is_framebuffer(obj)))
42 		return ERR_PTR(-EINVAL);
43 
44 	atomic_inc(&dev_priv->gpu_error.pending_fb_pin);
45 
46 	for_i915_gem_ww(&ww, ret, true) {
47 		ret = i915_gem_object_lock(obj, &ww);
48 		if (ret)
49 			continue;
50 
51 		if (HAS_LMEM(dev_priv)) {
52 			unsigned int flags = obj->flags;
53 
54 			/*
55 			 * For this type of buffer we need to able to read from the CPU
56 			 * the clear color value found in the buffer, hence we need to
57 			 * ensure it is always in the mappable part of lmem, if this is
58 			 * a small-bar device.
59 			 */
60 			if (intel_fb_rc_ccs_cc_plane(fb) >= 0)
61 				flags &= ~I915_BO_ALLOC_GPU_ONLY;
62 			ret = __i915_gem_object_migrate(obj, &ww, INTEL_REGION_LMEM_0,
63 							flags);
64 			if (ret)
65 				continue;
66 		}
67 
68 		ret = i915_gem_object_set_cache_level(obj, I915_CACHE_NONE);
69 		if (ret)
70 			continue;
71 
72 		vma = i915_vma_instance(obj, vm, view);
73 		if (IS_ERR(vma)) {
74 			ret = PTR_ERR(vma);
75 			continue;
76 		}
77 
78 		if (i915_vma_misplaced(vma, 0, alignment, 0)) {
79 			ret = i915_vma_unbind(vma);
80 			if (ret)
81 				continue;
82 		}
83 
84 		ret = i915_vma_pin_ww(vma, &ww, 0, alignment, PIN_GLOBAL);
85 		if (ret)
86 			continue;
87 	}
88 	if (ret) {
89 		vma = ERR_PTR(ret);
90 		goto err;
91 	}
92 
93 	vma->display_alignment = max(vma->display_alignment, alignment);
94 
95 	i915_gem_object_flush_if_display(obj);
96 
97 	i915_vma_get(vma);
98 err:
99 	atomic_dec(&dev_priv->gpu_error.pending_fb_pin);
100 
101 	return vma;
102 }
103 
104 struct i915_vma *
intel_fb_pin_to_ggtt(const struct drm_framebuffer * fb,const struct i915_gtt_view * view,unsigned int alignment,unsigned int phys_alignment,bool uses_fence,unsigned long * out_flags)105 intel_fb_pin_to_ggtt(const struct drm_framebuffer *fb,
106 		     const struct i915_gtt_view *view,
107 		     unsigned int alignment,
108 		     unsigned int phys_alignment,
109 		     bool uses_fence,
110 		     unsigned long *out_flags)
111 {
112 	struct drm_device *dev = fb->dev;
113 	struct drm_i915_private *dev_priv = to_i915(dev);
114 	struct drm_i915_gem_object *obj = intel_fb_obj(fb);
115 	intel_wakeref_t wakeref;
116 	struct i915_gem_ww_ctx ww;
117 	struct i915_vma *vma;
118 	unsigned int pinctl;
119 	int ret;
120 
121 	if (drm_WARN_ON(dev, !i915_gem_object_is_framebuffer(obj)))
122 		return ERR_PTR(-EINVAL);
123 
124 	if (drm_WARN_ON(dev, alignment && !is_power_of_2(alignment)))
125 		return ERR_PTR(-EINVAL);
126 
127 	/* Note that the w/a also requires 64 PTE of padding following the
128 	 * bo. We currently fill all unused PTE with the shadow page and so
129 	 * we should always have valid PTE following the scanout preventing
130 	 * the VT-d warning.
131 	 */
132 	if (intel_scanout_needs_vtd_wa(dev_priv) && alignment < 256 * 1024)
133 		alignment = 256 * 1024;
134 
135 	/*
136 	 * Global gtt pte registers are special registers which actually forward
137 	 * writes to a chunk of system memory. Which means that there is no risk
138 	 * that the register values disappear as soon as we call
139 	 * intel_runtime_pm_put(), so it is correct to wrap only the
140 	 * pin/unpin/fence and not more.
141 	 */
142 	wakeref = intel_runtime_pm_get(&dev_priv->runtime_pm);
143 
144 	atomic_inc(&dev_priv->gpu_error.pending_fb_pin);
145 
146 	/*
147 	 * Valleyview is definitely limited to scanning out the first
148 	 * 512MiB. Lets presume this behaviour was inherited from the
149 	 * g4x display engine and that all earlier gen are similarly
150 	 * limited. Testing suggests that it is a little more
151 	 * complicated than this. For example, Cherryview appears quite
152 	 * happy to scanout from anywhere within its global aperture.
153 	 */
154 	pinctl = 0;
155 	if (HAS_GMCH(dev_priv))
156 		pinctl |= PIN_MAPPABLE;
157 
158 	i915_gem_ww_ctx_init(&ww, true);
159 retry:
160 	ret = i915_gem_object_lock(obj, &ww);
161 	if (!ret && phys_alignment)
162 		ret = i915_gem_object_attach_phys(obj, phys_alignment);
163 	else if (!ret && HAS_LMEM(dev_priv))
164 		ret = i915_gem_object_migrate(obj, &ww, INTEL_REGION_LMEM_0);
165 	if (!ret)
166 		ret = i915_gem_object_pin_pages(obj);
167 	if (ret)
168 		goto err;
169 
170 	vma = i915_gem_object_pin_to_display_plane(obj, &ww, alignment,
171 						   view, pinctl);
172 	if (IS_ERR(vma)) {
173 		ret = PTR_ERR(vma);
174 		goto err_unpin;
175 	}
176 
177 	if (uses_fence && i915_vma_is_map_and_fenceable(vma)) {
178 		/*
179 		 * Install a fence for tiled scan-out. Pre-i965 always needs a
180 		 * fence, whereas 965+ only requires a fence if using
181 		 * framebuffer compression.  For simplicity, we always, when
182 		 * possible, install a fence as the cost is not that onerous.
183 		 *
184 		 * If we fail to fence the tiled scanout, then either the
185 		 * modeset will reject the change (which is highly unlikely as
186 		 * the affected systems, all but one, do not have unmappable
187 		 * space) or we will not be able to enable full powersaving
188 		 * techniques (also likely not to apply due to various limits
189 		 * FBC and the like impose on the size of the buffer, which
190 		 * presumably we violated anyway with this unmappable buffer).
191 		 * Anyway, it is presumably better to stumble onwards with
192 		 * something and try to run the system in a "less than optimal"
193 		 * mode that matches the user configuration.
194 		 */
195 		ret = i915_vma_pin_fence(vma);
196 		if (ret != 0 && DISPLAY_VER(dev_priv) < 4) {
197 			i915_vma_unpin(vma);
198 			goto err_unpin;
199 		}
200 		ret = 0;
201 
202 		if (vma->fence)
203 			*out_flags |= PLANE_HAS_FENCE;
204 	}
205 
206 	i915_vma_get(vma);
207 
208 err_unpin:
209 	i915_gem_object_unpin_pages(obj);
210 err:
211 	if (ret == -EDEADLK) {
212 		ret = i915_gem_ww_ctx_backoff(&ww);
213 		if (!ret)
214 			goto retry;
215 	}
216 	i915_gem_ww_ctx_fini(&ww);
217 	if (ret)
218 		vma = ERR_PTR(ret);
219 
220 	atomic_dec(&dev_priv->gpu_error.pending_fb_pin);
221 	intel_runtime_pm_put(&dev_priv->runtime_pm, wakeref);
222 	return vma;
223 }
224 
intel_fb_unpin_vma(struct i915_vma * vma,unsigned long flags)225 void intel_fb_unpin_vma(struct i915_vma *vma, unsigned long flags)
226 {
227 	if (flags & PLANE_HAS_FENCE)
228 		i915_vma_unpin_fence(vma);
229 	i915_vma_unpin(vma);
230 	i915_vma_put(vma);
231 }
232 
233 static unsigned int
intel_plane_fb_min_alignment(const struct intel_plane_state * plane_state)234 intel_plane_fb_min_alignment(const struct intel_plane_state *plane_state)
235 {
236 	const struct intel_framebuffer *fb = to_intel_framebuffer(plane_state->hw.fb);
237 
238 	return fb->min_alignment;
239 }
240 
241 static unsigned int
intel_plane_fb_min_phys_alignment(const struct intel_plane_state * plane_state)242 intel_plane_fb_min_phys_alignment(const struct intel_plane_state *plane_state)
243 {
244 	struct intel_plane *plane = to_intel_plane(plane_state->uapi.plane);
245 	const struct drm_framebuffer *fb = plane_state->hw.fb;
246 
247 	if (!intel_plane_needs_physical(plane))
248 		return 0;
249 
250 	return plane->min_alignment(plane, fb, 0);
251 }
252 
intel_plane_pin_fb(struct intel_plane_state * plane_state)253 int intel_plane_pin_fb(struct intel_plane_state *plane_state)
254 {
255 	struct intel_plane *plane = to_intel_plane(plane_state->uapi.plane);
256 	const struct intel_framebuffer *fb =
257 		to_intel_framebuffer(plane_state->hw.fb);
258 	struct i915_vma *vma;
259 
260 	if (!intel_fb_uses_dpt(&fb->base)) {
261 		vma = intel_fb_pin_to_ggtt(&fb->base, &plane_state->view.gtt,
262 					   intel_plane_fb_min_alignment(plane_state),
263 					   intel_plane_fb_min_phys_alignment(plane_state),
264 					   intel_plane_uses_fence(plane_state),
265 					   &plane_state->flags);
266 		if (IS_ERR(vma))
267 			return PTR_ERR(vma);
268 
269 		plane_state->ggtt_vma = vma;
270 
271 		/*
272 		 * Pre-populate the dma address before we enter the vblank
273 		 * evade critical section as i915_gem_object_get_dma_address()
274 		 * will trigger might_sleep() even if it won't actually sleep,
275 		 * which is the case when the fb has already been pinned.
276 		 */
277 		if (intel_plane_needs_physical(plane))
278 			plane_state->phys_dma_addr =
279 				i915_gem_object_get_dma_address(intel_fb_obj(&fb->base), 0);
280 	} else {
281 		unsigned int alignment = intel_plane_fb_min_alignment(plane_state);
282 
283 		vma = intel_dpt_pin_to_ggtt(fb->dpt_vm, alignment / 512);
284 		if (IS_ERR(vma))
285 			return PTR_ERR(vma);
286 
287 		plane_state->ggtt_vma = vma;
288 
289 		vma = intel_fb_pin_to_dpt(&fb->base, &plane_state->view.gtt,
290 					  alignment, &plane_state->flags,
291 					  fb->dpt_vm);
292 		if (IS_ERR(vma)) {
293 			intel_dpt_unpin_from_ggtt(fb->dpt_vm);
294 			plane_state->ggtt_vma = NULL;
295 			return PTR_ERR(vma);
296 		}
297 
298 		plane_state->dpt_vma = vma;
299 
300 		WARN_ON(plane_state->ggtt_vma == plane_state->dpt_vma);
301 	}
302 
303 	return 0;
304 }
305 
intel_plane_unpin_fb(struct intel_plane_state * old_plane_state)306 void intel_plane_unpin_fb(struct intel_plane_state *old_plane_state)
307 {
308 	const struct intel_framebuffer *fb =
309 		to_intel_framebuffer(old_plane_state->hw.fb);
310 	struct i915_vma *vma;
311 
312 	if (!intel_fb_uses_dpt(&fb->base)) {
313 		vma = fetch_and_zero(&old_plane_state->ggtt_vma);
314 		if (vma)
315 			intel_fb_unpin_vma(vma, old_plane_state->flags);
316 	} else {
317 		vma = fetch_and_zero(&old_plane_state->dpt_vma);
318 		if (vma)
319 			intel_fb_unpin_vma(vma, old_plane_state->flags);
320 
321 		vma = fetch_and_zero(&old_plane_state->ggtt_vma);
322 		if (vma)
323 			intel_dpt_unpin_from_ggtt(fb->dpt_vm);
324 	}
325 }
326