1 // SPDX-License-Identifier: MIT
2 /*
3 * Copyright © 2020 Intel Corporation
4 */
5 #include <linux/kernel.h>
6 #include <linux/pm_qos.h>
7 #include <linux/slab.h>
8
9 #include <drm/drm_atomic_helper.h>
10 #include <drm/drm_fourcc.h>
11 #include <drm/drm_plane.h>
12 #include <drm/drm_vblank_work.h>
13
14 #include "i915_vgpu.h"
15 #include "i9xx_plane.h"
16 #include "icl_dsi.h"
17 #include "intel_atomic.h"
18 #include "intel_atomic_plane.h"
19 #include "intel_color.h"
20 #include "intel_crtc.h"
21 #include "intel_cursor.h"
22 #include "intel_display_debugfs.h"
23 #include "intel_display_irq.h"
24 #include "intel_display_trace.h"
25 #include "intel_display_types.h"
26 #include "intel_drrs.h"
27 #include "intel_dsi.h"
28 #include "intel_fifo_underrun.h"
29 #include "intel_pipe_crc.h"
30 #include "intel_psr.h"
31 #include "intel_sprite.h"
32 #include "intel_vblank.h"
33 #include "intel_vrr.h"
34 #include "skl_universal_plane.h"
35
assert_vblank_disabled(struct drm_crtc * crtc)36 static void assert_vblank_disabled(struct drm_crtc *crtc)
37 {
38 struct drm_i915_private *i915 = to_i915(crtc->dev);
39
40 if (I915_STATE_WARN(i915, drm_crtc_vblank_get(crtc) == 0,
41 "[CRTC:%d:%s] vblank assertion failure (expected off, current on)\n",
42 crtc->base.id, crtc->name))
43 drm_crtc_vblank_put(crtc);
44 }
45
intel_first_crtc(struct drm_i915_private * i915)46 struct intel_crtc *intel_first_crtc(struct drm_i915_private *i915)
47 {
48 return to_intel_crtc(drm_crtc_from_index(&i915->drm, 0));
49 }
50
intel_crtc_for_pipe(struct drm_i915_private * i915,enum pipe pipe)51 struct intel_crtc *intel_crtc_for_pipe(struct drm_i915_private *i915,
52 enum pipe pipe)
53 {
54 struct intel_crtc *crtc;
55
56 for_each_intel_crtc(&i915->drm, crtc) {
57 if (crtc->pipe == pipe)
58 return crtc;
59 }
60
61 return NULL;
62 }
63
intel_crtc_wait_for_next_vblank(struct intel_crtc * crtc)64 void intel_crtc_wait_for_next_vblank(struct intel_crtc *crtc)
65 {
66 drm_crtc_wait_one_vblank(&crtc->base);
67 }
68
intel_wait_for_vblank_if_active(struct drm_i915_private * i915,enum pipe pipe)69 void intel_wait_for_vblank_if_active(struct drm_i915_private *i915,
70 enum pipe pipe)
71 {
72 struct intel_crtc *crtc = intel_crtc_for_pipe(i915, pipe);
73
74 if (crtc->active)
75 intel_crtc_wait_for_next_vblank(crtc);
76 }
77
intel_crtc_get_vblank_counter(struct intel_crtc * crtc)78 u32 intel_crtc_get_vblank_counter(struct intel_crtc *crtc)
79 {
80 struct drm_vblank_crtc *vblank = drm_crtc_vblank_crtc(&crtc->base);
81
82 if (!crtc->active)
83 return 0;
84
85 if (!vblank->max_vblank_count)
86 return (u32)drm_crtc_accurate_vblank_count(&crtc->base);
87
88 return crtc->base.funcs->get_vblank_counter(&crtc->base);
89 }
90
intel_crtc_max_vblank_count(const struct intel_crtc_state * crtc_state)91 u32 intel_crtc_max_vblank_count(const struct intel_crtc_state *crtc_state)
92 {
93 struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
94
95 /*
96 * From Gen 11, In case of dsi cmd mode, frame counter wouldnt
97 * have updated at the beginning of TE, if we want to use
98 * the hw counter, then we would find it updated in only
99 * the next TE, hence switching to sw counter.
100 */
101 if (crtc_state->mode_flags & (I915_MODE_FLAG_DSI_USE_TE0 |
102 I915_MODE_FLAG_DSI_USE_TE1))
103 return 0;
104
105 /*
106 * On i965gm the hardware frame counter reads
107 * zero when the TV encoder is enabled :(
108 */
109 if (IS_I965GM(dev_priv) &&
110 (crtc_state->output_types & BIT(INTEL_OUTPUT_TVOUT)))
111 return 0;
112
113 if (DISPLAY_VER(dev_priv) >= 5 || IS_G4X(dev_priv))
114 return 0xffffffff; /* full 32 bit counter */
115 else if (DISPLAY_VER(dev_priv) >= 3)
116 return 0xffffff; /* only 24 bits of frame count */
117 else
118 return 0; /* Gen2 doesn't have a hardware frame counter */
119 }
120
intel_crtc_vblank_on(const struct intel_crtc_state * crtc_state)121 void intel_crtc_vblank_on(const struct intel_crtc_state *crtc_state)
122 {
123 struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
124
125 assert_vblank_disabled(&crtc->base);
126 drm_crtc_set_max_vblank_count(&crtc->base,
127 intel_crtc_max_vblank_count(crtc_state));
128 drm_crtc_vblank_on(&crtc->base);
129
130 /*
131 * Should really happen exactly when we enable the pipe
132 * but we want the frame counters in the trace, and that
133 * requires vblank support on some platforms/outputs.
134 */
135 trace_intel_pipe_enable(crtc);
136 }
137
intel_crtc_vblank_off(const struct intel_crtc_state * crtc_state)138 void intel_crtc_vblank_off(const struct intel_crtc_state *crtc_state)
139 {
140 struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
141
142 /*
143 * Should really happen exactly when we disable the pipe
144 * but we want the frame counters in the trace, and that
145 * requires vblank support on some platforms/outputs.
146 */
147 trace_intel_pipe_disable(crtc);
148
149 drm_crtc_vblank_off(&crtc->base);
150 assert_vblank_disabled(&crtc->base);
151 }
152
intel_crtc_state_alloc(struct intel_crtc * crtc)153 struct intel_crtc_state *intel_crtc_state_alloc(struct intel_crtc *crtc)
154 {
155 struct intel_crtc_state *crtc_state;
156
157 crtc_state = kmalloc(sizeof(*crtc_state), GFP_KERNEL);
158
159 if (crtc_state)
160 intel_crtc_state_reset(crtc_state, crtc);
161
162 return crtc_state;
163 }
164
intel_crtc_state_reset(struct intel_crtc_state * crtc_state,struct intel_crtc * crtc)165 void intel_crtc_state_reset(struct intel_crtc_state *crtc_state,
166 struct intel_crtc *crtc)
167 {
168 memset(crtc_state, 0, sizeof(*crtc_state));
169
170 __drm_atomic_helper_crtc_state_reset(&crtc_state->uapi, &crtc->base);
171
172 crtc_state->cpu_transcoder = INVALID_TRANSCODER;
173 crtc_state->master_transcoder = INVALID_TRANSCODER;
174 crtc_state->hsw_workaround_pipe = INVALID_PIPE;
175 crtc_state->scaler_state.scaler_id = -1;
176 crtc_state->mst_master_transcoder = INVALID_TRANSCODER;
177 crtc_state->max_link_bpp_x16 = INT_MAX;
178 }
179
intel_crtc_alloc(void)180 static struct intel_crtc *intel_crtc_alloc(void)
181 {
182 struct intel_crtc_state *crtc_state;
183 struct intel_crtc *crtc;
184
185 crtc = kzalloc(sizeof(*crtc), GFP_KERNEL);
186 if (!crtc)
187 return ERR_PTR(-ENOMEM);
188
189 crtc_state = intel_crtc_state_alloc(crtc);
190 if (!crtc_state) {
191 kfree(crtc);
192 return ERR_PTR(-ENOMEM);
193 }
194
195 crtc->base.state = &crtc_state->uapi;
196 crtc->config = crtc_state;
197
198 return crtc;
199 }
200
intel_crtc_free(struct intel_crtc * crtc)201 static void intel_crtc_free(struct intel_crtc *crtc)
202 {
203 intel_crtc_destroy_state(&crtc->base, crtc->base.state);
204 kfree(crtc);
205 }
206
intel_crtc_destroy(struct drm_crtc * _crtc)207 static void intel_crtc_destroy(struct drm_crtc *_crtc)
208 {
209 struct intel_crtc *crtc = to_intel_crtc(_crtc);
210
211 cpu_latency_qos_remove_request(&crtc->vblank_pm_qos);
212
213 drm_crtc_cleanup(&crtc->base);
214 kfree(crtc);
215 }
216
intel_crtc_late_register(struct drm_crtc * crtc)217 static int intel_crtc_late_register(struct drm_crtc *crtc)
218 {
219 intel_crtc_debugfs_add(to_intel_crtc(crtc));
220 return 0;
221 }
222
223 #define INTEL_CRTC_FUNCS \
224 .set_config = drm_atomic_helper_set_config, \
225 .destroy = intel_crtc_destroy, \
226 .page_flip = drm_atomic_helper_page_flip, \
227 .atomic_duplicate_state = intel_crtc_duplicate_state, \
228 .atomic_destroy_state = intel_crtc_destroy_state, \
229 .set_crc_source = intel_crtc_set_crc_source, \
230 .verify_crc_source = intel_crtc_verify_crc_source, \
231 .get_crc_sources = intel_crtc_get_crc_sources, \
232 .late_register = intel_crtc_late_register
233
234 static const struct drm_crtc_funcs bdw_crtc_funcs = {
235 INTEL_CRTC_FUNCS,
236
237 .get_vblank_counter = g4x_get_vblank_counter,
238 .enable_vblank = bdw_enable_vblank,
239 .disable_vblank = bdw_disable_vblank,
240 .get_vblank_timestamp = intel_crtc_get_vblank_timestamp,
241 };
242
243 static const struct drm_crtc_funcs ilk_crtc_funcs = {
244 INTEL_CRTC_FUNCS,
245
246 .get_vblank_counter = g4x_get_vblank_counter,
247 .enable_vblank = ilk_enable_vblank,
248 .disable_vblank = ilk_disable_vblank,
249 .get_vblank_timestamp = intel_crtc_get_vblank_timestamp,
250 };
251
252 static const struct drm_crtc_funcs g4x_crtc_funcs = {
253 INTEL_CRTC_FUNCS,
254
255 .get_vblank_counter = g4x_get_vblank_counter,
256 .enable_vblank = i965_enable_vblank,
257 .disable_vblank = i965_disable_vblank,
258 .get_vblank_timestamp = intel_crtc_get_vblank_timestamp,
259 };
260
261 static const struct drm_crtc_funcs i965_crtc_funcs = {
262 INTEL_CRTC_FUNCS,
263
264 .get_vblank_counter = i915_get_vblank_counter,
265 .enable_vblank = i965_enable_vblank,
266 .disable_vblank = i965_disable_vblank,
267 .get_vblank_timestamp = intel_crtc_get_vblank_timestamp,
268 };
269
270 static const struct drm_crtc_funcs i915gm_crtc_funcs = {
271 INTEL_CRTC_FUNCS,
272
273 .get_vblank_counter = i915_get_vblank_counter,
274 .enable_vblank = i915gm_enable_vblank,
275 .disable_vblank = i915gm_disable_vblank,
276 .get_vblank_timestamp = intel_crtc_get_vblank_timestamp,
277 };
278
279 static const struct drm_crtc_funcs i915_crtc_funcs = {
280 INTEL_CRTC_FUNCS,
281
282 .get_vblank_counter = i915_get_vblank_counter,
283 .enable_vblank = i8xx_enable_vblank,
284 .disable_vblank = i8xx_disable_vblank,
285 .get_vblank_timestamp = intel_crtc_get_vblank_timestamp,
286 };
287
288 static const struct drm_crtc_funcs i8xx_crtc_funcs = {
289 INTEL_CRTC_FUNCS,
290
291 /* no hw vblank counter */
292 .enable_vblank = i8xx_enable_vblank,
293 .disable_vblank = i8xx_disable_vblank,
294 .get_vblank_timestamp = intel_crtc_get_vblank_timestamp,
295 };
296
intel_crtc_init(struct drm_i915_private * dev_priv,enum pipe pipe)297 int intel_crtc_init(struct drm_i915_private *dev_priv, enum pipe pipe)
298 {
299 struct intel_plane *primary, *cursor;
300 const struct drm_crtc_funcs *funcs;
301 struct intel_crtc *crtc;
302 int sprite, ret;
303
304 crtc = intel_crtc_alloc();
305 if (IS_ERR(crtc))
306 return PTR_ERR(crtc);
307
308 crtc->pipe = pipe;
309 crtc->num_scalers = DISPLAY_RUNTIME_INFO(dev_priv)->num_scalers[pipe];
310
311 if (DISPLAY_VER(dev_priv) >= 9)
312 primary = skl_universal_plane_create(dev_priv, pipe, PLANE_1);
313 else
314 primary = intel_primary_plane_create(dev_priv, pipe);
315 if (IS_ERR(primary)) {
316 ret = PTR_ERR(primary);
317 goto fail;
318 }
319 crtc->plane_ids_mask |= BIT(primary->id);
320
321 intel_init_fifo_underrun_reporting(dev_priv, crtc, false);
322
323 for_each_sprite(dev_priv, pipe, sprite) {
324 struct intel_plane *plane;
325
326 if (DISPLAY_VER(dev_priv) >= 9)
327 plane = skl_universal_plane_create(dev_priv, pipe, PLANE_2 + sprite);
328 else
329 plane = intel_sprite_plane_create(dev_priv, pipe, sprite);
330 if (IS_ERR(plane)) {
331 ret = PTR_ERR(plane);
332 goto fail;
333 }
334 crtc->plane_ids_mask |= BIT(plane->id);
335 }
336
337 cursor = intel_cursor_plane_create(dev_priv, pipe);
338 if (IS_ERR(cursor)) {
339 ret = PTR_ERR(cursor);
340 goto fail;
341 }
342 crtc->plane_ids_mask |= BIT(cursor->id);
343
344 if (HAS_GMCH(dev_priv)) {
345 if (IS_CHERRYVIEW(dev_priv) ||
346 IS_VALLEYVIEW(dev_priv) || IS_G4X(dev_priv))
347 funcs = &g4x_crtc_funcs;
348 else if (DISPLAY_VER(dev_priv) == 4)
349 funcs = &i965_crtc_funcs;
350 else if (IS_I945GM(dev_priv) || IS_I915GM(dev_priv))
351 funcs = &i915gm_crtc_funcs;
352 else if (DISPLAY_VER(dev_priv) == 3)
353 funcs = &i915_crtc_funcs;
354 else
355 funcs = &i8xx_crtc_funcs;
356 } else {
357 if (DISPLAY_VER(dev_priv) >= 8)
358 funcs = &bdw_crtc_funcs;
359 else
360 funcs = &ilk_crtc_funcs;
361 }
362
363 ret = drm_crtc_init_with_planes(&dev_priv->drm, &crtc->base,
364 &primary->base, &cursor->base,
365 funcs, "pipe %c", pipe_name(pipe));
366 if (ret)
367 goto fail;
368
369 if (DISPLAY_VER(dev_priv) >= 11)
370 drm_crtc_create_scaling_filter_property(&crtc->base,
371 BIT(DRM_SCALING_FILTER_DEFAULT) |
372 BIT(DRM_SCALING_FILTER_NEAREST_NEIGHBOR));
373
374 intel_color_crtc_init(crtc);
375 intel_drrs_crtc_init(crtc);
376 intel_crtc_crc_init(crtc);
377
378 cpu_latency_qos_add_request(&crtc->vblank_pm_qos, PM_QOS_DEFAULT_VALUE);
379
380 drm_WARN_ON(&dev_priv->drm, drm_crtc_index(&crtc->base) != crtc->pipe);
381
382 return 0;
383
384 fail:
385 intel_crtc_free(crtc);
386
387 return ret;
388 }
389
intel_crtc_needs_vblank_work(const struct intel_crtc_state * crtc_state)390 static bool intel_crtc_needs_vblank_work(const struct intel_crtc_state *crtc_state)
391 {
392 return crtc_state->hw.active &&
393 !intel_crtc_needs_modeset(crtc_state) &&
394 !crtc_state->preload_luts &&
395 intel_crtc_needs_color_update(crtc_state) &&
396 !intel_color_uses_dsb(crtc_state);
397 }
398
intel_crtc_vblank_work(struct kthread_work * base)399 static void intel_crtc_vblank_work(struct kthread_work *base)
400 {
401 struct drm_vblank_work *work = to_drm_vblank_work(base);
402 struct intel_crtc_state *crtc_state =
403 container_of(work, typeof(*crtc_state), vblank_work);
404 struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
405
406 trace_intel_crtc_vblank_work_start(crtc);
407
408 intel_color_load_luts(crtc_state);
409
410 if (crtc_state->uapi.event) {
411 spin_lock_irq(&crtc->base.dev->event_lock);
412 drm_crtc_send_vblank_event(&crtc->base, crtc_state->uapi.event);
413 spin_unlock_irq(&crtc->base.dev->event_lock);
414 crtc_state->uapi.event = NULL;
415 }
416
417 trace_intel_crtc_vblank_work_end(crtc);
418 }
419
intel_crtc_vblank_work_init(struct intel_crtc_state * crtc_state)420 static void intel_crtc_vblank_work_init(struct intel_crtc_state *crtc_state)
421 {
422 struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
423
424 drm_vblank_work_init(&crtc_state->vblank_work, &crtc->base,
425 intel_crtc_vblank_work);
426 /*
427 * Interrupt latency is critical for getting the vblank
428 * work executed as early as possible during the vblank.
429 */
430 cpu_latency_qos_update_request(&crtc->vblank_pm_qos, 0);
431 }
432
intel_wait_for_vblank_workers(struct intel_atomic_state * state)433 void intel_wait_for_vblank_workers(struct intel_atomic_state *state)
434 {
435 struct intel_crtc_state *crtc_state;
436 struct intel_crtc *crtc;
437 int i;
438
439 for_each_new_intel_crtc_in_state(state, crtc, crtc_state, i) {
440 if (!intel_crtc_needs_vblank_work(crtc_state))
441 continue;
442
443 drm_vblank_work_flush(&crtc_state->vblank_work);
444 cpu_latency_qos_update_request(&crtc->vblank_pm_qos,
445 PM_QOS_DEFAULT_VALUE);
446 }
447 }
448
intel_usecs_to_scanlines(const struct drm_display_mode * adjusted_mode,int usecs)449 int intel_usecs_to_scanlines(const struct drm_display_mode *adjusted_mode,
450 int usecs)
451 {
452 /* paranoia */
453 if (!adjusted_mode->crtc_htotal)
454 return 1;
455
456 return DIV_ROUND_UP_ULL(mul_u32_u32(usecs, adjusted_mode->crtc_clock),
457 1000 * adjusted_mode->crtc_htotal);
458 }
459
460 /**
461 * intel_pipe_update_start() - start update of a set of display registers
462 * @state: the atomic state
463 * @crtc: the crtc
464 *
465 * Mark the start of an update to pipe registers that should be updated
466 * atomically regarding vblank. If the next vblank will happens within
467 * the next 100 us, this function waits until the vblank passes.
468 *
469 * After a successful call to this function, interrupts will be disabled
470 * until a subsequent call to intel_pipe_update_end(). That is done to
471 * avoid random delays.
472 */
intel_pipe_update_start(struct intel_atomic_state * state,struct intel_crtc * crtc)473 void intel_pipe_update_start(struct intel_atomic_state *state,
474 struct intel_crtc *crtc)
475 {
476 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
477 const struct intel_crtc_state *old_crtc_state =
478 intel_atomic_get_old_crtc_state(state, crtc);
479 struct intel_crtc_state *new_crtc_state =
480 intel_atomic_get_new_crtc_state(state, crtc);
481 struct intel_vblank_evade_ctx evade;
482 int scanline;
483
484 intel_psr_lock(new_crtc_state);
485
486 if (new_crtc_state->do_async_flip) {
487 spin_lock_irq(&crtc->base.dev->event_lock);
488 /* arm the event for the flip done irq handler */
489 crtc->flip_done_event = new_crtc_state->uapi.event;
490 spin_unlock_irq(&crtc->base.dev->event_lock);
491
492 new_crtc_state->uapi.event = NULL;
493 return;
494 }
495
496 if (intel_crtc_needs_vblank_work(new_crtc_state))
497 intel_crtc_vblank_work_init(new_crtc_state);
498
499 if (state->base.legacy_cursor_update) {
500 struct intel_plane *plane;
501 struct intel_plane_state *old_plane_state, *new_plane_state;
502 int i;
503
504 for_each_oldnew_intel_plane_in_state(state, plane, old_plane_state,
505 new_plane_state, i) {
506 if (old_plane_state->uapi.crtc == &crtc->base)
507 intel_plane_init_cursor_vblank_work(old_plane_state,
508 new_plane_state);
509 }
510 }
511
512 intel_vblank_evade_init(old_crtc_state, new_crtc_state, &evade);
513
514 if (drm_WARN_ON(&dev_priv->drm, drm_crtc_vblank_get(&crtc->base)))
515 goto irq_disable;
516
517 /*
518 * Wait for psr to idle out after enabling the VBL interrupts
519 * VBL interrupts will start the PSR exit and prevent a PSR
520 * re-entry as well.
521 */
522 intel_psr_wait_for_idle_locked(new_crtc_state);
523
524 local_irq_disable();
525
526 crtc->debug.min_vbl = evade.min;
527 crtc->debug.max_vbl = evade.max;
528 trace_intel_pipe_update_start(crtc);
529
530 scanline = intel_vblank_evade(&evade);
531
532 drm_crtc_vblank_put(&crtc->base);
533
534 crtc->debug.scanline_start = scanline;
535 crtc->debug.start_vbl_time = ktime_get();
536 crtc->debug.start_vbl_count = intel_crtc_get_vblank_counter(crtc);
537
538 trace_intel_pipe_update_vblank_evaded(crtc);
539 return;
540
541 irq_disable:
542 local_irq_disable();
543 }
544
545 #if IS_ENABLED(CONFIG_DRM_I915_DEBUG_VBLANK_EVADE)
dbg_vblank_evade(struct intel_crtc * crtc,ktime_t end)546 static void dbg_vblank_evade(struct intel_crtc *crtc, ktime_t end)
547 {
548 u64 delta = ktime_to_ns(ktime_sub(end, crtc->debug.start_vbl_time));
549 unsigned int h;
550
551 h = ilog2(delta >> 9);
552 if (h >= ARRAY_SIZE(crtc->debug.vbl.times))
553 h = ARRAY_SIZE(crtc->debug.vbl.times) - 1;
554 crtc->debug.vbl.times[h]++;
555
556 crtc->debug.vbl.sum += delta;
557 if (!crtc->debug.vbl.min || delta < crtc->debug.vbl.min)
558 crtc->debug.vbl.min = delta;
559 if (delta > crtc->debug.vbl.max)
560 crtc->debug.vbl.max = delta;
561
562 if (delta > 1000 * VBLANK_EVASION_TIME_US) {
563 drm_dbg_kms(crtc->base.dev,
564 "Atomic update on pipe (%c) took %lld us, max time under evasion is %u us\n",
565 pipe_name(crtc->pipe),
566 div_u64(delta, 1000),
567 VBLANK_EVASION_TIME_US);
568 crtc->debug.vbl.over++;
569 }
570 }
571 #else
dbg_vblank_evade(struct intel_crtc * crtc,ktime_t end)572 static void dbg_vblank_evade(struct intel_crtc *crtc, ktime_t end) {}
573 #endif
574
intel_crtc_arm_vblank_event(struct intel_crtc_state * crtc_state)575 void intel_crtc_arm_vblank_event(struct intel_crtc_state *crtc_state)
576 {
577 struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
578 unsigned long irqflags;
579
580 if (!crtc_state->uapi.event)
581 return;
582
583 drm_WARN_ON(crtc->base.dev, drm_crtc_vblank_get(&crtc->base) != 0);
584
585 spin_lock_irqsave(&crtc->base.dev->event_lock, irqflags);
586 drm_crtc_arm_vblank_event(&crtc->base, crtc_state->uapi.event);
587 spin_unlock_irqrestore(&crtc->base.dev->event_lock, irqflags);
588
589 crtc_state->uapi.event = NULL;
590 }
591
592 /**
593 * intel_pipe_update_end() - end update of a set of display registers
594 * @state: the atomic state
595 * @crtc: the crtc
596 *
597 * Mark the end of an update started with intel_pipe_update_start(). This
598 * re-enables interrupts and verifies the update was actually completed
599 * before a vblank.
600 */
intel_pipe_update_end(struct intel_atomic_state * state,struct intel_crtc * crtc)601 void intel_pipe_update_end(struct intel_atomic_state *state,
602 struct intel_crtc *crtc)
603 {
604 struct intel_crtc_state *new_crtc_state =
605 intel_atomic_get_new_crtc_state(state, crtc);
606 enum pipe pipe = crtc->pipe;
607 int scanline_end = intel_get_crtc_scanline(crtc);
608 u32 end_vbl_count = intel_crtc_get_vblank_counter(crtc);
609 ktime_t end_vbl_time = ktime_get();
610 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
611
612 if (new_crtc_state->do_async_flip)
613 goto out;
614
615 trace_intel_pipe_update_end(crtc, end_vbl_count, scanline_end);
616
617 /*
618 * Incase of mipi dsi command mode, we need to set frame update
619 * request for every commit.
620 */
621 if (DISPLAY_VER(dev_priv) >= 11 &&
622 intel_crtc_has_type(new_crtc_state, INTEL_OUTPUT_DSI))
623 icl_dsi_frame_update(new_crtc_state);
624
625 /* We're still in the vblank-evade critical section, this can't race.
626 * Would be slightly nice to just grab the vblank count and arm the
627 * event outside of the critical section - the spinlock might spin for a
628 * while ... */
629 if (intel_crtc_needs_vblank_work(new_crtc_state)) {
630 drm_vblank_work_schedule(&new_crtc_state->vblank_work,
631 drm_crtc_accurate_vblank_count(&crtc->base) + 1,
632 false);
633 } else {
634 intel_crtc_arm_vblank_event(new_crtc_state);
635 }
636
637 if (state->base.legacy_cursor_update) {
638 struct intel_plane *plane;
639 struct intel_plane_state *old_plane_state;
640 int i;
641
642 for_each_old_intel_plane_in_state(state, plane, old_plane_state, i) {
643 if (old_plane_state->uapi.crtc == &crtc->base &&
644 old_plane_state->unpin_work.vblank) {
645 drm_vblank_work_schedule(&old_plane_state->unpin_work,
646 drm_crtc_accurate_vblank_count(&crtc->base) + 1,
647 false);
648
649 /* Remove plane from atomic state, cleanup/free is done from vblank worker. */
650 memset(&state->base.planes[i], 0, sizeof(state->base.planes[i]));
651 }
652 }
653 }
654
655 /*
656 * Send VRR Push to terminate Vblank. If we are already in vblank
657 * this has to be done _after_ sampling the frame counter, as
658 * otherwise the push would immediately terminate the vblank and
659 * the sampled frame counter would correspond to the next frame
660 * instead of the current frame.
661 *
662 * There is a tiny race here (iff vblank evasion failed us) where
663 * we might sample the frame counter just before vmax vblank start
664 * but the push would be sent just after it. That would cause the
665 * push to affect the next frame instead of the current frame,
666 * which would cause the next frame to terminate already at vmin
667 * vblank start instead of vmax vblank start.
668 */
669 intel_vrr_send_push(new_crtc_state);
670
671 local_irq_enable();
672
673 if (intel_vgpu_active(dev_priv))
674 goto out;
675
676 if (crtc->debug.start_vbl_count &&
677 crtc->debug.start_vbl_count != end_vbl_count) {
678 drm_err(&dev_priv->drm,
679 "Atomic update failure on pipe %c (start=%u end=%u) time %lld us, min %d, max %d, scanline start %d, end %d\n",
680 pipe_name(pipe), crtc->debug.start_vbl_count,
681 end_vbl_count,
682 ktime_us_delta(end_vbl_time,
683 crtc->debug.start_vbl_time),
684 crtc->debug.min_vbl, crtc->debug.max_vbl,
685 crtc->debug.scanline_start, scanline_end);
686 }
687
688 dbg_vblank_evade(crtc, end_vbl_time);
689
690 out:
691 intel_psr_unlock(new_crtc_state);
692 }
693