1 /* i915_irq.c -- IRQ support for the I915 -*- linux-c -*-
2 */
3 /*
4 * Copyright 2003 Tungsten Graphics, Inc., Cedar Park, Texas.
5 * All Rights Reserved.
6 *
7 * Permission is hereby granted, free of charge, to any person obtaining a
8 * copy of this software and associated documentation files (the
9 * "Software"), to deal in the Software without restriction, including
10 * without limitation the rights to use, copy, modify, merge, publish,
11 * distribute, sub license, and/or sell copies of the Software, and to
12 * permit persons to whom the Software is furnished to do so, subject to
13 * the following conditions:
14 *
15 * The above copyright notice and this permission notice (including the
16 * next paragraph) shall be included in all copies or substantial portions
17 * of the Software.
18 *
19 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
20 * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
21 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT.
22 * IN NO EVENT SHALL TUNGSTEN GRAPHICS AND/OR ITS SUPPLIERS BE LIABLE FOR
23 * ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
24 * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
25 * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
26 *
27 */
28
29 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
30
31 #include <linux/slab.h>
32 #include <linux/sysrq.h>
33
34 #include <drm/drm_drv.h>
35
36 #include "display/intel_display_irq.h"
37 #include "display/intel_hotplug.h"
38 #include "display/intel_hotplug_irq.h"
39 #include "display/intel_lpe_audio.h"
40 #include "display/intel_psr_regs.h"
41
42 #include "gt/intel_breadcrumbs.h"
43 #include "gt/intel_gt.h"
44 #include "gt/intel_gt_irq.h"
45 #include "gt/intel_gt_pm_irq.h"
46 #include "gt/intel_gt_regs.h"
47 #include "gt/intel_rps.h"
48
49 #include "i915_driver.h"
50 #include "i915_drv.h"
51 #include "i915_irq.h"
52 #include "i915_reg.h"
53
54 /**
55 * DOC: interrupt handling
56 *
57 * These functions provide the basic support for enabling and disabling the
58 * interrupt handling support. There's a lot more functionality in i915_irq.c
59 * and related files, but that will be described in separate chapters.
60 */
61
62 /*
63 * Interrupt statistic for PMU. Increments the counter only if the
64 * interrupt originated from the GPU so interrupts from a device which
65 * shares the interrupt line are not accounted.
66 */
pmu_irq_stats(struct drm_i915_private * i915,irqreturn_t res)67 static inline void pmu_irq_stats(struct drm_i915_private *i915,
68 irqreturn_t res)
69 {
70 if (unlikely(res != IRQ_HANDLED))
71 return;
72
73 /*
74 * A clever compiler translates that into INC. A not so clever one
75 * should at least prevent store tearing.
76 */
77 WRITE_ONCE(i915->pmu.irq_count, i915->pmu.irq_count + 1);
78 }
79
gen3_irq_reset(struct intel_uncore * uncore,i915_reg_t imr,i915_reg_t iir,i915_reg_t ier)80 void gen3_irq_reset(struct intel_uncore *uncore, i915_reg_t imr,
81 i915_reg_t iir, i915_reg_t ier)
82 {
83 intel_uncore_write(uncore, imr, 0xffffffff);
84 intel_uncore_posting_read(uncore, imr);
85
86 intel_uncore_write(uncore, ier, 0);
87
88 /* IIR can theoretically queue up two events. Be paranoid. */
89 intel_uncore_write(uncore, iir, 0xffffffff);
90 intel_uncore_posting_read(uncore, iir);
91 intel_uncore_write(uncore, iir, 0xffffffff);
92 intel_uncore_posting_read(uncore, iir);
93 }
94
gen2_irq_reset(struct intel_uncore * uncore)95 static void gen2_irq_reset(struct intel_uncore *uncore)
96 {
97 intel_uncore_write16(uncore, GEN2_IMR, 0xffff);
98 intel_uncore_posting_read16(uncore, GEN2_IMR);
99
100 intel_uncore_write16(uncore, GEN2_IER, 0);
101
102 /* IIR can theoretically queue up two events. Be paranoid. */
103 intel_uncore_write16(uncore, GEN2_IIR, 0xffff);
104 intel_uncore_posting_read16(uncore, GEN2_IIR);
105 intel_uncore_write16(uncore, GEN2_IIR, 0xffff);
106 intel_uncore_posting_read16(uncore, GEN2_IIR);
107 }
108
109 /*
110 * We should clear IMR at preinstall/uninstall, and just check at postinstall.
111 */
gen3_assert_iir_is_zero(struct intel_uncore * uncore,i915_reg_t reg)112 void gen3_assert_iir_is_zero(struct intel_uncore *uncore, i915_reg_t reg)
113 {
114 u32 val = intel_uncore_read(uncore, reg);
115
116 if (val == 0)
117 return;
118
119 drm_WARN(&uncore->i915->drm, 1,
120 "Interrupt register 0x%x is not zero: 0x%08x\n",
121 i915_mmio_reg_offset(reg), val);
122 intel_uncore_write(uncore, reg, 0xffffffff);
123 intel_uncore_posting_read(uncore, reg);
124 intel_uncore_write(uncore, reg, 0xffffffff);
125 intel_uncore_posting_read(uncore, reg);
126 }
127
gen2_assert_iir_is_zero(struct intel_uncore * uncore)128 static void gen2_assert_iir_is_zero(struct intel_uncore *uncore)
129 {
130 u16 val = intel_uncore_read16(uncore, GEN2_IIR);
131
132 if (val == 0)
133 return;
134
135 drm_WARN(&uncore->i915->drm, 1,
136 "Interrupt register 0x%x is not zero: 0x%08x\n",
137 i915_mmio_reg_offset(GEN2_IIR), val);
138 intel_uncore_write16(uncore, GEN2_IIR, 0xffff);
139 intel_uncore_posting_read16(uncore, GEN2_IIR);
140 intel_uncore_write16(uncore, GEN2_IIR, 0xffff);
141 intel_uncore_posting_read16(uncore, GEN2_IIR);
142 }
143
gen3_irq_init(struct intel_uncore * uncore,i915_reg_t imr,u32 imr_val,i915_reg_t ier,u32 ier_val,i915_reg_t iir)144 void gen3_irq_init(struct intel_uncore *uncore,
145 i915_reg_t imr, u32 imr_val,
146 i915_reg_t ier, u32 ier_val,
147 i915_reg_t iir)
148 {
149 gen3_assert_iir_is_zero(uncore, iir);
150
151 intel_uncore_write(uncore, ier, ier_val);
152 intel_uncore_write(uncore, imr, imr_val);
153 intel_uncore_posting_read(uncore, imr);
154 }
155
gen2_irq_init(struct intel_uncore * uncore,u32 imr_val,u32 ier_val)156 static void gen2_irq_init(struct intel_uncore *uncore,
157 u32 imr_val, u32 ier_val)
158 {
159 gen2_assert_iir_is_zero(uncore);
160
161 intel_uncore_write16(uncore, GEN2_IER, ier_val);
162 intel_uncore_write16(uncore, GEN2_IMR, imr_val);
163 intel_uncore_posting_read16(uncore, GEN2_IMR);
164 }
165
166 /**
167 * ivb_parity_work - Workqueue called when a parity error interrupt
168 * occurred.
169 * @work: workqueue struct
170 *
171 * Doesn't actually do anything except notify userspace. As a consequence of
172 * this event, userspace should try to remap the bad rows since statistically
173 * it is likely the same row is more likely to go bad again.
174 */
ivb_parity_work(struct work_struct * work)175 static void ivb_parity_work(struct work_struct *work)
176 {
177 struct drm_i915_private *dev_priv =
178 container_of(work, typeof(*dev_priv), l3_parity.error_work);
179 struct intel_gt *gt = to_gt(dev_priv);
180 u32 error_status, row, bank, subbank;
181 char *parity_event[6];
182 u32 misccpctl;
183 u8 slice = 0;
184
185 /* We must turn off DOP level clock gating to access the L3 registers.
186 * In order to prevent a get/put style interface, acquire struct mutex
187 * any time we access those registers.
188 */
189 mutex_lock(&dev_priv->drm.struct_mutex);
190
191 /* If we've screwed up tracking, just let the interrupt fire again */
192 if (drm_WARN_ON(&dev_priv->drm, !dev_priv->l3_parity.which_slice))
193 goto out;
194
195 misccpctl = intel_uncore_rmw(&dev_priv->uncore, GEN7_MISCCPCTL,
196 GEN7_DOP_CLOCK_GATE_ENABLE, 0);
197 intel_uncore_posting_read(&dev_priv->uncore, GEN7_MISCCPCTL);
198
199 while ((slice = ffs(dev_priv->l3_parity.which_slice)) != 0) {
200 i915_reg_t reg;
201
202 slice--;
203 if (drm_WARN_ON_ONCE(&dev_priv->drm,
204 slice >= NUM_L3_SLICES(dev_priv)))
205 break;
206
207 dev_priv->l3_parity.which_slice &= ~(1<<slice);
208
209 reg = GEN7_L3CDERRST1(slice);
210
211 error_status = intel_uncore_read(&dev_priv->uncore, reg);
212 row = GEN7_PARITY_ERROR_ROW(error_status);
213 bank = GEN7_PARITY_ERROR_BANK(error_status);
214 subbank = GEN7_PARITY_ERROR_SUBBANK(error_status);
215
216 intel_uncore_write(&dev_priv->uncore, reg, GEN7_PARITY_ERROR_VALID | GEN7_L3CDERRST1_ENABLE);
217 intel_uncore_posting_read(&dev_priv->uncore, reg);
218
219 parity_event[0] = I915_L3_PARITY_UEVENT "=1";
220 parity_event[1] = kasprintf(GFP_KERNEL, "ROW=%d", row);
221 parity_event[2] = kasprintf(GFP_KERNEL, "BANK=%d", bank);
222 parity_event[3] = kasprintf(GFP_KERNEL, "SUBBANK=%d", subbank);
223 parity_event[4] = kasprintf(GFP_KERNEL, "SLICE=%d", slice);
224 parity_event[5] = NULL;
225
226 kobject_uevent_env(&dev_priv->drm.primary->kdev->kobj,
227 KOBJ_CHANGE, parity_event);
228
229 drm_dbg(&dev_priv->drm,
230 "Parity error: Slice = %d, Row = %d, Bank = %d, Sub bank = %d.\n",
231 slice, row, bank, subbank);
232
233 kfree(parity_event[4]);
234 kfree(parity_event[3]);
235 kfree(parity_event[2]);
236 kfree(parity_event[1]);
237 }
238
239 intel_uncore_write(&dev_priv->uncore, GEN7_MISCCPCTL, misccpctl);
240
241 out:
242 drm_WARN_ON(&dev_priv->drm, dev_priv->l3_parity.which_slice);
243 spin_lock_irq(gt->irq_lock);
244 gen5_gt_enable_irq(gt, GT_PARITY_ERROR(dev_priv));
245 spin_unlock_irq(gt->irq_lock);
246
247 mutex_unlock(&dev_priv->drm.struct_mutex);
248 }
249
valleyview_irq_handler(int irq,void * arg)250 static irqreturn_t valleyview_irq_handler(int irq, void *arg)
251 {
252 struct drm_i915_private *dev_priv = arg;
253 irqreturn_t ret = IRQ_NONE;
254
255 if (!intel_irqs_enabled(dev_priv))
256 return IRQ_NONE;
257
258 /* IRQs are synced during runtime_suspend, we don't require a wakeref */
259 disable_rpm_wakeref_asserts(&dev_priv->runtime_pm);
260
261 do {
262 u32 iir, gt_iir, pm_iir;
263 u32 pipe_stats[I915_MAX_PIPES] = {};
264 u32 hotplug_status = 0;
265 u32 ier = 0;
266
267 gt_iir = intel_uncore_read(&dev_priv->uncore, GTIIR);
268 pm_iir = intel_uncore_read(&dev_priv->uncore, GEN6_PMIIR);
269 iir = intel_uncore_read(&dev_priv->uncore, VLV_IIR);
270
271 if (gt_iir == 0 && pm_iir == 0 && iir == 0)
272 break;
273
274 ret = IRQ_HANDLED;
275
276 /*
277 * Theory on interrupt generation, based on empirical evidence:
278 *
279 * x = ((VLV_IIR & VLV_IER) ||
280 * (((GT_IIR & GT_IER) || (GEN6_PMIIR & GEN6_PMIER)) &&
281 * (VLV_MASTER_IER & MASTER_INTERRUPT_ENABLE)));
282 *
283 * A CPU interrupt will only be raised when 'x' has a 0->1 edge.
284 * Hence we clear MASTER_INTERRUPT_ENABLE and VLV_IER to
285 * guarantee the CPU interrupt will be raised again even if we
286 * don't end up clearing all the VLV_IIR, GT_IIR, GEN6_PMIIR
287 * bits this time around.
288 */
289 intel_uncore_write(&dev_priv->uncore, VLV_MASTER_IER, 0);
290 ier = intel_uncore_rmw(&dev_priv->uncore, VLV_IER, ~0, 0);
291
292 if (gt_iir)
293 intel_uncore_write(&dev_priv->uncore, GTIIR, gt_iir);
294 if (pm_iir)
295 intel_uncore_write(&dev_priv->uncore, GEN6_PMIIR, pm_iir);
296
297 if (iir & I915_DISPLAY_PORT_INTERRUPT)
298 hotplug_status = i9xx_hpd_irq_ack(dev_priv);
299
300 /* Call regardless, as some status bits might not be
301 * signalled in iir */
302 i9xx_pipestat_irq_ack(dev_priv, iir, pipe_stats);
303
304 if (iir & (I915_LPE_PIPE_A_INTERRUPT |
305 I915_LPE_PIPE_B_INTERRUPT))
306 intel_lpe_audio_irq_handler(dev_priv);
307
308 /*
309 * VLV_IIR is single buffered, and reflects the level
310 * from PIPESTAT/PORT_HOTPLUG_STAT, hence clear it last.
311 */
312 if (iir)
313 intel_uncore_write(&dev_priv->uncore, VLV_IIR, iir);
314
315 intel_uncore_write(&dev_priv->uncore, VLV_IER, ier);
316 intel_uncore_write(&dev_priv->uncore, VLV_MASTER_IER, MASTER_INTERRUPT_ENABLE);
317
318 if (gt_iir)
319 gen6_gt_irq_handler(to_gt(dev_priv), gt_iir);
320 if (pm_iir)
321 gen6_rps_irq_handler(&to_gt(dev_priv)->rps, pm_iir);
322
323 if (hotplug_status)
324 i9xx_hpd_irq_handler(dev_priv, hotplug_status);
325
326 valleyview_pipestat_irq_handler(dev_priv, pipe_stats);
327 } while (0);
328
329 pmu_irq_stats(dev_priv, ret);
330
331 enable_rpm_wakeref_asserts(&dev_priv->runtime_pm);
332
333 return ret;
334 }
335
cherryview_irq_handler(int irq,void * arg)336 static irqreturn_t cherryview_irq_handler(int irq, void *arg)
337 {
338 struct drm_i915_private *dev_priv = arg;
339 irqreturn_t ret = IRQ_NONE;
340
341 if (!intel_irqs_enabled(dev_priv))
342 return IRQ_NONE;
343
344 /* IRQs are synced during runtime_suspend, we don't require a wakeref */
345 disable_rpm_wakeref_asserts(&dev_priv->runtime_pm);
346
347 do {
348 u32 master_ctl, iir;
349 u32 pipe_stats[I915_MAX_PIPES] = {};
350 u32 hotplug_status = 0;
351 u32 ier = 0;
352
353 master_ctl = intel_uncore_read(&dev_priv->uncore, GEN8_MASTER_IRQ) & ~GEN8_MASTER_IRQ_CONTROL;
354 iir = intel_uncore_read(&dev_priv->uncore, VLV_IIR);
355
356 if (master_ctl == 0 && iir == 0)
357 break;
358
359 ret = IRQ_HANDLED;
360
361 /*
362 * Theory on interrupt generation, based on empirical evidence:
363 *
364 * x = ((VLV_IIR & VLV_IER) ||
365 * ((GEN8_MASTER_IRQ & ~GEN8_MASTER_IRQ_CONTROL) &&
366 * (GEN8_MASTER_IRQ & GEN8_MASTER_IRQ_CONTROL)));
367 *
368 * A CPU interrupt will only be raised when 'x' has a 0->1 edge.
369 * Hence we clear GEN8_MASTER_IRQ_CONTROL and VLV_IER to
370 * guarantee the CPU interrupt will be raised again even if we
371 * don't end up clearing all the VLV_IIR and GEN8_MASTER_IRQ_CONTROL
372 * bits this time around.
373 */
374 intel_uncore_write(&dev_priv->uncore, GEN8_MASTER_IRQ, 0);
375 ier = intel_uncore_rmw(&dev_priv->uncore, VLV_IER, ~0, 0);
376
377 gen8_gt_irq_handler(to_gt(dev_priv), master_ctl);
378
379 if (iir & I915_DISPLAY_PORT_INTERRUPT)
380 hotplug_status = i9xx_hpd_irq_ack(dev_priv);
381
382 /* Call regardless, as some status bits might not be
383 * signalled in iir */
384 i9xx_pipestat_irq_ack(dev_priv, iir, pipe_stats);
385
386 if (iir & (I915_LPE_PIPE_A_INTERRUPT |
387 I915_LPE_PIPE_B_INTERRUPT |
388 I915_LPE_PIPE_C_INTERRUPT))
389 intel_lpe_audio_irq_handler(dev_priv);
390
391 /*
392 * VLV_IIR is single buffered, and reflects the level
393 * from PIPESTAT/PORT_HOTPLUG_STAT, hence clear it last.
394 */
395 if (iir)
396 intel_uncore_write(&dev_priv->uncore, VLV_IIR, iir);
397
398 intel_uncore_write(&dev_priv->uncore, VLV_IER, ier);
399 intel_uncore_write(&dev_priv->uncore, GEN8_MASTER_IRQ, GEN8_MASTER_IRQ_CONTROL);
400
401 if (hotplug_status)
402 i9xx_hpd_irq_handler(dev_priv, hotplug_status);
403
404 valleyview_pipestat_irq_handler(dev_priv, pipe_stats);
405 } while (0);
406
407 pmu_irq_stats(dev_priv, ret);
408
409 enable_rpm_wakeref_asserts(&dev_priv->runtime_pm);
410
411 return ret;
412 }
413
414 /*
415 * To handle irqs with the minimum potential races with fresh interrupts, we:
416 * 1 - Disable Master Interrupt Control.
417 * 2 - Find the source(s) of the interrupt.
418 * 3 - Clear the Interrupt Identity bits (IIR).
419 * 4 - Process the interrupt(s) that had bits set in the IIRs.
420 * 5 - Re-enable Master Interrupt Control.
421 */
ilk_irq_handler(int irq,void * arg)422 static irqreturn_t ilk_irq_handler(int irq, void *arg)
423 {
424 struct drm_i915_private *i915 = arg;
425 void __iomem * const regs = intel_uncore_regs(&i915->uncore);
426 u32 de_iir, gt_iir, de_ier, sde_ier = 0;
427 irqreturn_t ret = IRQ_NONE;
428
429 if (unlikely(!intel_irqs_enabled(i915)))
430 return IRQ_NONE;
431
432 /* IRQs are synced during runtime_suspend, we don't require a wakeref */
433 disable_rpm_wakeref_asserts(&i915->runtime_pm);
434
435 /* disable master interrupt before clearing iir */
436 de_ier = raw_reg_read(regs, DEIER);
437 raw_reg_write(regs, DEIER, de_ier & ~DE_MASTER_IRQ_CONTROL);
438
439 /* Disable south interrupts. We'll only write to SDEIIR once, so further
440 * interrupts will will be stored on its back queue, and then we'll be
441 * able to process them after we restore SDEIER (as soon as we restore
442 * it, we'll get an interrupt if SDEIIR still has something to process
443 * due to its back queue). */
444 if (!HAS_PCH_NOP(i915)) {
445 sde_ier = raw_reg_read(regs, SDEIER);
446 raw_reg_write(regs, SDEIER, 0);
447 }
448
449 /* Find, clear, then process each source of interrupt */
450
451 gt_iir = raw_reg_read(regs, GTIIR);
452 if (gt_iir) {
453 raw_reg_write(regs, GTIIR, gt_iir);
454 if (GRAPHICS_VER(i915) >= 6)
455 gen6_gt_irq_handler(to_gt(i915), gt_iir);
456 else
457 gen5_gt_irq_handler(to_gt(i915), gt_iir);
458 ret = IRQ_HANDLED;
459 }
460
461 de_iir = raw_reg_read(regs, DEIIR);
462 if (de_iir) {
463 raw_reg_write(regs, DEIIR, de_iir);
464 if (DISPLAY_VER(i915) >= 7)
465 ivb_display_irq_handler(i915, de_iir);
466 else
467 ilk_display_irq_handler(i915, de_iir);
468 ret = IRQ_HANDLED;
469 }
470
471 if (GRAPHICS_VER(i915) >= 6) {
472 u32 pm_iir = raw_reg_read(regs, GEN6_PMIIR);
473 if (pm_iir) {
474 raw_reg_write(regs, GEN6_PMIIR, pm_iir);
475 gen6_rps_irq_handler(&to_gt(i915)->rps, pm_iir);
476 ret = IRQ_HANDLED;
477 }
478 }
479
480 raw_reg_write(regs, DEIER, de_ier);
481 if (sde_ier)
482 raw_reg_write(regs, SDEIER, sde_ier);
483
484 pmu_irq_stats(i915, ret);
485
486 /* IRQs are synced during runtime_suspend, we don't require a wakeref */
487 enable_rpm_wakeref_asserts(&i915->runtime_pm);
488
489 return ret;
490 }
491
gen8_master_intr_disable(void __iomem * const regs)492 static inline u32 gen8_master_intr_disable(void __iomem * const regs)
493 {
494 raw_reg_write(regs, GEN8_MASTER_IRQ, 0);
495
496 /*
497 * Now with master disabled, get a sample of level indications
498 * for this interrupt. Indications will be cleared on related acks.
499 * New indications can and will light up during processing,
500 * and will generate new interrupt after enabling master.
501 */
502 return raw_reg_read(regs, GEN8_MASTER_IRQ);
503 }
504
gen8_master_intr_enable(void __iomem * const regs)505 static inline void gen8_master_intr_enable(void __iomem * const regs)
506 {
507 raw_reg_write(regs, GEN8_MASTER_IRQ, GEN8_MASTER_IRQ_CONTROL);
508 }
509
gen8_irq_handler(int irq,void * arg)510 static irqreturn_t gen8_irq_handler(int irq, void *arg)
511 {
512 struct drm_i915_private *dev_priv = arg;
513 void __iomem * const regs = intel_uncore_regs(&dev_priv->uncore);
514 u32 master_ctl;
515
516 if (!intel_irqs_enabled(dev_priv))
517 return IRQ_NONE;
518
519 master_ctl = gen8_master_intr_disable(regs);
520 if (!master_ctl) {
521 gen8_master_intr_enable(regs);
522 return IRQ_NONE;
523 }
524
525 /* Find, queue (onto bottom-halves), then clear each source */
526 gen8_gt_irq_handler(to_gt(dev_priv), master_ctl);
527
528 /* IRQs are synced during runtime_suspend, we don't require a wakeref */
529 if (master_ctl & ~GEN8_GT_IRQS) {
530 disable_rpm_wakeref_asserts(&dev_priv->runtime_pm);
531 gen8_de_irq_handler(dev_priv, master_ctl);
532 enable_rpm_wakeref_asserts(&dev_priv->runtime_pm);
533 }
534
535 gen8_master_intr_enable(regs);
536
537 pmu_irq_stats(dev_priv, IRQ_HANDLED);
538
539 return IRQ_HANDLED;
540 }
541
gen11_master_intr_disable(void __iomem * const regs)542 static inline u32 gen11_master_intr_disable(void __iomem * const regs)
543 {
544 raw_reg_write(regs, GEN11_GFX_MSTR_IRQ, 0);
545
546 /*
547 * Now with master disabled, get a sample of level indications
548 * for this interrupt. Indications will be cleared on related acks.
549 * New indications can and will light up during processing,
550 * and will generate new interrupt after enabling master.
551 */
552 return raw_reg_read(regs, GEN11_GFX_MSTR_IRQ);
553 }
554
gen11_master_intr_enable(void __iomem * const regs)555 static inline void gen11_master_intr_enable(void __iomem * const regs)
556 {
557 raw_reg_write(regs, GEN11_GFX_MSTR_IRQ, GEN11_MASTER_IRQ);
558 }
559
gen11_irq_handler(int irq,void * arg)560 static irqreturn_t gen11_irq_handler(int irq, void *arg)
561 {
562 struct drm_i915_private *i915 = arg;
563 void __iomem * const regs = intel_uncore_regs(&i915->uncore);
564 struct intel_gt *gt = to_gt(i915);
565 u32 master_ctl;
566 u32 gu_misc_iir;
567
568 if (!intel_irqs_enabled(i915))
569 return IRQ_NONE;
570
571 master_ctl = gen11_master_intr_disable(regs);
572 if (!master_ctl) {
573 gen11_master_intr_enable(regs);
574 return IRQ_NONE;
575 }
576
577 /* Find, queue (onto bottom-halves), then clear each source */
578 gen11_gt_irq_handler(gt, master_ctl);
579
580 /* IRQs are synced during runtime_suspend, we don't require a wakeref */
581 if (master_ctl & GEN11_DISPLAY_IRQ)
582 gen11_display_irq_handler(i915);
583
584 gu_misc_iir = gen11_gu_misc_irq_ack(i915, master_ctl);
585
586 gen11_master_intr_enable(regs);
587
588 gen11_gu_misc_irq_handler(i915, gu_misc_iir);
589
590 pmu_irq_stats(i915, IRQ_HANDLED);
591
592 return IRQ_HANDLED;
593 }
594
dg1_master_intr_disable(void __iomem * const regs)595 static inline u32 dg1_master_intr_disable(void __iomem * const regs)
596 {
597 u32 val;
598
599 /* First disable interrupts */
600 raw_reg_write(regs, DG1_MSTR_TILE_INTR, 0);
601
602 /* Get the indication levels and ack the master unit */
603 val = raw_reg_read(regs, DG1_MSTR_TILE_INTR);
604 if (unlikely(!val))
605 return 0;
606
607 raw_reg_write(regs, DG1_MSTR_TILE_INTR, val);
608
609 return val;
610 }
611
dg1_master_intr_enable(void __iomem * const regs)612 static inline void dg1_master_intr_enable(void __iomem * const regs)
613 {
614 raw_reg_write(regs, DG1_MSTR_TILE_INTR, DG1_MSTR_IRQ);
615 }
616
dg1_irq_handler(int irq,void * arg)617 static irqreturn_t dg1_irq_handler(int irq, void *arg)
618 {
619 struct drm_i915_private * const i915 = arg;
620 struct intel_gt *gt = to_gt(i915);
621 void __iomem * const regs = intel_uncore_regs(gt->uncore);
622 u32 master_tile_ctl, master_ctl;
623 u32 gu_misc_iir;
624
625 if (!intel_irqs_enabled(i915))
626 return IRQ_NONE;
627
628 master_tile_ctl = dg1_master_intr_disable(regs);
629 if (!master_tile_ctl) {
630 dg1_master_intr_enable(regs);
631 return IRQ_NONE;
632 }
633
634 /* FIXME: we only support tile 0 for now. */
635 if (master_tile_ctl & DG1_MSTR_TILE(0)) {
636 master_ctl = raw_reg_read(regs, GEN11_GFX_MSTR_IRQ);
637 raw_reg_write(regs, GEN11_GFX_MSTR_IRQ, master_ctl);
638 } else {
639 drm_err(&i915->drm, "Tile not supported: 0x%08x\n",
640 master_tile_ctl);
641 dg1_master_intr_enable(regs);
642 return IRQ_NONE;
643 }
644
645 gen11_gt_irq_handler(gt, master_ctl);
646
647 if (master_ctl & GEN11_DISPLAY_IRQ)
648 gen11_display_irq_handler(i915);
649
650 gu_misc_iir = gen11_gu_misc_irq_ack(i915, master_ctl);
651
652 dg1_master_intr_enable(regs);
653
654 gen11_gu_misc_irq_handler(i915, gu_misc_iir);
655
656 pmu_irq_stats(i915, IRQ_HANDLED);
657
658 return IRQ_HANDLED;
659 }
660
ibx_irq_reset(struct drm_i915_private * dev_priv)661 static void ibx_irq_reset(struct drm_i915_private *dev_priv)
662 {
663 struct intel_uncore *uncore = &dev_priv->uncore;
664
665 if (HAS_PCH_NOP(dev_priv))
666 return;
667
668 GEN3_IRQ_RESET(uncore, SDE);
669
670 if (HAS_PCH_CPT(dev_priv) || HAS_PCH_LPT(dev_priv))
671 intel_uncore_write(&dev_priv->uncore, SERR_INT, 0xffffffff);
672 }
673
674 /* drm_dma.h hooks
675 */
ilk_irq_reset(struct drm_i915_private * dev_priv)676 static void ilk_irq_reset(struct drm_i915_private *dev_priv)
677 {
678 struct intel_uncore *uncore = &dev_priv->uncore;
679
680 GEN3_IRQ_RESET(uncore, DE);
681 dev_priv->irq_mask = ~0u;
682
683 if (GRAPHICS_VER(dev_priv) == 7)
684 intel_uncore_write(uncore, GEN7_ERR_INT, 0xffffffff);
685
686 if (IS_HASWELL(dev_priv)) {
687 intel_uncore_write(uncore, EDP_PSR_IMR, 0xffffffff);
688 intel_uncore_write(uncore, EDP_PSR_IIR, 0xffffffff);
689 }
690
691 gen5_gt_irq_reset(to_gt(dev_priv));
692
693 ibx_irq_reset(dev_priv);
694 }
695
valleyview_irq_reset(struct drm_i915_private * dev_priv)696 static void valleyview_irq_reset(struct drm_i915_private *dev_priv)
697 {
698 intel_uncore_write(&dev_priv->uncore, VLV_MASTER_IER, 0);
699 intel_uncore_posting_read(&dev_priv->uncore, VLV_MASTER_IER);
700
701 gen5_gt_irq_reset(to_gt(dev_priv));
702
703 spin_lock_irq(&dev_priv->irq_lock);
704 if (dev_priv->display.irq.display_irqs_enabled)
705 vlv_display_irq_reset(dev_priv);
706 spin_unlock_irq(&dev_priv->irq_lock);
707 }
708
gen8_irq_reset(struct drm_i915_private * dev_priv)709 static void gen8_irq_reset(struct drm_i915_private *dev_priv)
710 {
711 struct intel_uncore *uncore = &dev_priv->uncore;
712
713 gen8_master_intr_disable(intel_uncore_regs(uncore));
714
715 gen8_gt_irq_reset(to_gt(dev_priv));
716 gen8_display_irq_reset(dev_priv);
717 GEN3_IRQ_RESET(uncore, GEN8_PCU_);
718
719 if (HAS_PCH_SPLIT(dev_priv))
720 ibx_irq_reset(dev_priv);
721
722 }
723
gen11_irq_reset(struct drm_i915_private * dev_priv)724 static void gen11_irq_reset(struct drm_i915_private *dev_priv)
725 {
726 struct intel_gt *gt = to_gt(dev_priv);
727 struct intel_uncore *uncore = gt->uncore;
728
729 gen11_master_intr_disable(intel_uncore_regs(&dev_priv->uncore));
730
731 gen11_gt_irq_reset(gt);
732 gen11_display_irq_reset(dev_priv);
733
734 GEN3_IRQ_RESET(uncore, GEN11_GU_MISC_);
735 GEN3_IRQ_RESET(uncore, GEN8_PCU_);
736 }
737
dg1_irq_reset(struct drm_i915_private * dev_priv)738 static void dg1_irq_reset(struct drm_i915_private *dev_priv)
739 {
740 struct intel_uncore *uncore = &dev_priv->uncore;
741 struct intel_gt *gt;
742 unsigned int i;
743
744 dg1_master_intr_disable(intel_uncore_regs(&dev_priv->uncore));
745
746 for_each_gt(gt, dev_priv, i)
747 gen11_gt_irq_reset(gt);
748
749 gen11_display_irq_reset(dev_priv);
750
751 GEN3_IRQ_RESET(uncore, GEN11_GU_MISC_);
752 GEN3_IRQ_RESET(uncore, GEN8_PCU_);
753
754 intel_uncore_write(uncore, GEN11_GFX_MSTR_IRQ, ~0);
755 }
756
cherryview_irq_reset(struct drm_i915_private * dev_priv)757 static void cherryview_irq_reset(struct drm_i915_private *dev_priv)
758 {
759 struct intel_uncore *uncore = &dev_priv->uncore;
760
761 intel_uncore_write(uncore, GEN8_MASTER_IRQ, 0);
762 intel_uncore_posting_read(&dev_priv->uncore, GEN8_MASTER_IRQ);
763
764 gen8_gt_irq_reset(to_gt(dev_priv));
765
766 GEN3_IRQ_RESET(uncore, GEN8_PCU_);
767
768 spin_lock_irq(&dev_priv->irq_lock);
769 if (dev_priv->display.irq.display_irqs_enabled)
770 vlv_display_irq_reset(dev_priv);
771 spin_unlock_irq(&dev_priv->irq_lock);
772 }
773
ilk_irq_postinstall(struct drm_i915_private * dev_priv)774 static void ilk_irq_postinstall(struct drm_i915_private *dev_priv)
775 {
776 gen5_gt_irq_postinstall(to_gt(dev_priv));
777
778 ilk_de_irq_postinstall(dev_priv);
779 }
780
valleyview_irq_postinstall(struct drm_i915_private * dev_priv)781 static void valleyview_irq_postinstall(struct drm_i915_private *dev_priv)
782 {
783 gen5_gt_irq_postinstall(to_gt(dev_priv));
784
785 spin_lock_irq(&dev_priv->irq_lock);
786 if (dev_priv->display.irq.display_irqs_enabled)
787 vlv_display_irq_postinstall(dev_priv);
788 spin_unlock_irq(&dev_priv->irq_lock);
789
790 intel_uncore_write(&dev_priv->uncore, VLV_MASTER_IER, MASTER_INTERRUPT_ENABLE);
791 intel_uncore_posting_read(&dev_priv->uncore, VLV_MASTER_IER);
792 }
793
gen8_irq_postinstall(struct drm_i915_private * dev_priv)794 static void gen8_irq_postinstall(struct drm_i915_private *dev_priv)
795 {
796 gen8_gt_irq_postinstall(to_gt(dev_priv));
797 gen8_de_irq_postinstall(dev_priv);
798
799 gen8_master_intr_enable(intel_uncore_regs(&dev_priv->uncore));
800 }
801
gen11_irq_postinstall(struct drm_i915_private * dev_priv)802 static void gen11_irq_postinstall(struct drm_i915_private *dev_priv)
803 {
804 struct intel_gt *gt = to_gt(dev_priv);
805 struct intel_uncore *uncore = gt->uncore;
806 u32 gu_misc_masked = GEN11_GU_MISC_GSE;
807
808 gen11_gt_irq_postinstall(gt);
809 gen11_de_irq_postinstall(dev_priv);
810
811 GEN3_IRQ_INIT(uncore, GEN11_GU_MISC_, ~gu_misc_masked, gu_misc_masked);
812
813 gen11_master_intr_enable(intel_uncore_regs(uncore));
814 intel_uncore_posting_read(&dev_priv->uncore, GEN11_GFX_MSTR_IRQ);
815 }
816
dg1_irq_postinstall(struct drm_i915_private * dev_priv)817 static void dg1_irq_postinstall(struct drm_i915_private *dev_priv)
818 {
819 struct intel_uncore *uncore = &dev_priv->uncore;
820 u32 gu_misc_masked = GEN11_GU_MISC_GSE;
821 struct intel_gt *gt;
822 unsigned int i;
823
824 for_each_gt(gt, dev_priv, i)
825 gen11_gt_irq_postinstall(gt);
826
827 GEN3_IRQ_INIT(uncore, GEN11_GU_MISC_, ~gu_misc_masked, gu_misc_masked);
828
829 dg1_de_irq_postinstall(dev_priv);
830
831 dg1_master_intr_enable(intel_uncore_regs(uncore));
832 intel_uncore_posting_read(uncore, DG1_MSTR_TILE_INTR);
833 }
834
cherryview_irq_postinstall(struct drm_i915_private * dev_priv)835 static void cherryview_irq_postinstall(struct drm_i915_private *dev_priv)
836 {
837 gen8_gt_irq_postinstall(to_gt(dev_priv));
838
839 spin_lock_irq(&dev_priv->irq_lock);
840 if (dev_priv->display.irq.display_irqs_enabled)
841 vlv_display_irq_postinstall(dev_priv);
842 spin_unlock_irq(&dev_priv->irq_lock);
843
844 intel_uncore_write(&dev_priv->uncore, GEN8_MASTER_IRQ, GEN8_MASTER_IRQ_CONTROL);
845 intel_uncore_posting_read(&dev_priv->uncore, GEN8_MASTER_IRQ);
846 }
847
i8xx_irq_reset(struct drm_i915_private * dev_priv)848 static void i8xx_irq_reset(struct drm_i915_private *dev_priv)
849 {
850 struct intel_uncore *uncore = &dev_priv->uncore;
851
852 i9xx_pipestat_irq_reset(dev_priv);
853
854 gen2_irq_reset(uncore);
855 dev_priv->irq_mask = ~0u;
856 }
857
i9xx_error_mask(struct drm_i915_private * i915)858 static u32 i9xx_error_mask(struct drm_i915_private *i915)
859 {
860 /*
861 * On gen2/3 FBC generates (seemingly spurious)
862 * display INVALID_GTT/INVALID_GTT_PTE table errors.
863 *
864 * Also gen3 bspec has this to say:
865 * "DISPA_INVALID_GTT_PTE
866 " [DevNapa] : Reserved. This bit does not reflect the page
867 " table error for the display plane A."
868 *
869 * Unfortunately we can't mask off individual PGTBL_ER bits,
870 * so we just have to mask off all page table errors via EMR.
871 */
872 if (HAS_FBC(i915))
873 return ~I915_ERROR_MEMORY_REFRESH;
874 else
875 return ~(I915_ERROR_PAGE_TABLE |
876 I915_ERROR_MEMORY_REFRESH);
877 }
878
i8xx_irq_postinstall(struct drm_i915_private * dev_priv)879 static void i8xx_irq_postinstall(struct drm_i915_private *dev_priv)
880 {
881 struct intel_uncore *uncore = &dev_priv->uncore;
882 u16 enable_mask;
883
884 intel_uncore_write16(uncore, EMR, i9xx_error_mask(dev_priv));
885
886 /* Unmask the interrupts that we always want on. */
887 dev_priv->irq_mask =
888 ~(I915_DISPLAY_PIPE_A_EVENT_INTERRUPT |
889 I915_DISPLAY_PIPE_B_EVENT_INTERRUPT |
890 I915_MASTER_ERROR_INTERRUPT);
891
892 enable_mask =
893 I915_DISPLAY_PIPE_A_EVENT_INTERRUPT |
894 I915_DISPLAY_PIPE_B_EVENT_INTERRUPT |
895 I915_MASTER_ERROR_INTERRUPT |
896 I915_USER_INTERRUPT;
897
898 gen2_irq_init(uncore, dev_priv->irq_mask, enable_mask);
899
900 /* Interrupt setup is already guaranteed to be single-threaded, this is
901 * just to make the assert_spin_locked check happy. */
902 spin_lock_irq(&dev_priv->irq_lock);
903 i915_enable_pipestat(dev_priv, PIPE_A, PIPE_CRC_DONE_INTERRUPT_STATUS);
904 i915_enable_pipestat(dev_priv, PIPE_B, PIPE_CRC_DONE_INTERRUPT_STATUS);
905 spin_unlock_irq(&dev_priv->irq_lock);
906 }
907
i8xx_error_irq_ack(struct drm_i915_private * i915,u16 * eir,u16 * eir_stuck)908 static void i8xx_error_irq_ack(struct drm_i915_private *i915,
909 u16 *eir, u16 *eir_stuck)
910 {
911 struct intel_uncore *uncore = &i915->uncore;
912 u16 emr;
913
914 *eir = intel_uncore_read16(uncore, EIR);
915 intel_uncore_write16(uncore, EIR, *eir);
916
917 *eir_stuck = intel_uncore_read16(uncore, EIR);
918 if (*eir_stuck == 0)
919 return;
920
921 /*
922 * Toggle all EMR bits to make sure we get an edge
923 * in the ISR master error bit if we don't clear
924 * all the EIR bits. Otherwise the edge triggered
925 * IIR on i965/g4x wouldn't notice that an interrupt
926 * is still pending. Also some EIR bits can't be
927 * cleared except by handling the underlying error
928 * (or by a GPU reset) so we mask any bit that
929 * remains set.
930 */
931 emr = intel_uncore_read16(uncore, EMR);
932 intel_uncore_write16(uncore, EMR, 0xffff);
933 intel_uncore_write16(uncore, EMR, emr | *eir_stuck);
934 }
935
i8xx_error_irq_handler(struct drm_i915_private * dev_priv,u16 eir,u16 eir_stuck)936 static void i8xx_error_irq_handler(struct drm_i915_private *dev_priv,
937 u16 eir, u16 eir_stuck)
938 {
939 drm_dbg(&dev_priv->drm, "Master Error: EIR 0x%04x\n", eir);
940
941 if (eir_stuck)
942 drm_dbg(&dev_priv->drm, "EIR stuck: 0x%04x, masked\n",
943 eir_stuck);
944
945 drm_dbg(&dev_priv->drm, "PGTBL_ER: 0x%08x\n",
946 intel_uncore_read(&dev_priv->uncore, PGTBL_ER));
947 }
948
i9xx_error_irq_ack(struct drm_i915_private * dev_priv,u32 * eir,u32 * eir_stuck)949 static void i9xx_error_irq_ack(struct drm_i915_private *dev_priv,
950 u32 *eir, u32 *eir_stuck)
951 {
952 u32 emr;
953
954 *eir = intel_uncore_read(&dev_priv->uncore, EIR);
955 intel_uncore_write(&dev_priv->uncore, EIR, *eir);
956
957 *eir_stuck = intel_uncore_read(&dev_priv->uncore, EIR);
958 if (*eir_stuck == 0)
959 return;
960
961 /*
962 * Toggle all EMR bits to make sure we get an edge
963 * in the ISR master error bit if we don't clear
964 * all the EIR bits. Otherwise the edge triggered
965 * IIR on i965/g4x wouldn't notice that an interrupt
966 * is still pending. Also some EIR bits can't be
967 * cleared except by handling the underlying error
968 * (or by a GPU reset) so we mask any bit that
969 * remains set.
970 */
971 emr = intel_uncore_read(&dev_priv->uncore, EMR);
972 intel_uncore_write(&dev_priv->uncore, EMR, 0xffffffff);
973 intel_uncore_write(&dev_priv->uncore, EMR, emr | *eir_stuck);
974 }
975
i9xx_error_irq_handler(struct drm_i915_private * dev_priv,u32 eir,u32 eir_stuck)976 static void i9xx_error_irq_handler(struct drm_i915_private *dev_priv,
977 u32 eir, u32 eir_stuck)
978 {
979 drm_dbg(&dev_priv->drm, "Master Error, EIR 0x%08x\n", eir);
980
981 if (eir_stuck)
982 drm_dbg(&dev_priv->drm, "EIR stuck: 0x%08x, masked\n",
983 eir_stuck);
984
985 drm_dbg(&dev_priv->drm, "PGTBL_ER: 0x%08x\n",
986 intel_uncore_read(&dev_priv->uncore, PGTBL_ER));
987 }
988
i8xx_irq_handler(int irq,void * arg)989 static irqreturn_t i8xx_irq_handler(int irq, void *arg)
990 {
991 struct drm_i915_private *dev_priv = arg;
992 irqreturn_t ret = IRQ_NONE;
993
994 if (!intel_irqs_enabled(dev_priv))
995 return IRQ_NONE;
996
997 /* IRQs are synced during runtime_suspend, we don't require a wakeref */
998 disable_rpm_wakeref_asserts(&dev_priv->runtime_pm);
999
1000 do {
1001 u32 pipe_stats[I915_MAX_PIPES] = {};
1002 u16 eir = 0, eir_stuck = 0;
1003 u16 iir;
1004
1005 iir = intel_uncore_read16(&dev_priv->uncore, GEN2_IIR);
1006 if (iir == 0)
1007 break;
1008
1009 ret = IRQ_HANDLED;
1010
1011 /* Call regardless, as some status bits might not be
1012 * signalled in iir */
1013 i9xx_pipestat_irq_ack(dev_priv, iir, pipe_stats);
1014
1015 if (iir & I915_MASTER_ERROR_INTERRUPT)
1016 i8xx_error_irq_ack(dev_priv, &eir, &eir_stuck);
1017
1018 intel_uncore_write16(&dev_priv->uncore, GEN2_IIR, iir);
1019
1020 if (iir & I915_USER_INTERRUPT)
1021 intel_engine_cs_irq(to_gt(dev_priv)->engine[RCS0], iir);
1022
1023 if (iir & I915_MASTER_ERROR_INTERRUPT)
1024 i8xx_error_irq_handler(dev_priv, eir, eir_stuck);
1025
1026 i8xx_pipestat_irq_handler(dev_priv, iir, pipe_stats);
1027 } while (0);
1028
1029 pmu_irq_stats(dev_priv, ret);
1030
1031 enable_rpm_wakeref_asserts(&dev_priv->runtime_pm);
1032
1033 return ret;
1034 }
1035
i915_irq_reset(struct drm_i915_private * dev_priv)1036 static void i915_irq_reset(struct drm_i915_private *dev_priv)
1037 {
1038 struct intel_uncore *uncore = &dev_priv->uncore;
1039
1040 if (I915_HAS_HOTPLUG(dev_priv)) {
1041 i915_hotplug_interrupt_update(dev_priv, 0xffffffff, 0);
1042 intel_uncore_rmw(&dev_priv->uncore,
1043 PORT_HOTPLUG_STAT(dev_priv), 0, 0);
1044 }
1045
1046 i9xx_pipestat_irq_reset(dev_priv);
1047
1048 GEN3_IRQ_RESET(uncore, GEN2_);
1049 dev_priv->irq_mask = ~0u;
1050 }
1051
i915_irq_postinstall(struct drm_i915_private * dev_priv)1052 static void i915_irq_postinstall(struct drm_i915_private *dev_priv)
1053 {
1054 struct intel_uncore *uncore = &dev_priv->uncore;
1055 u32 enable_mask;
1056
1057 intel_uncore_write(uncore, EMR, i9xx_error_mask(dev_priv));
1058
1059 /* Unmask the interrupts that we always want on. */
1060 dev_priv->irq_mask =
1061 ~(I915_ASLE_INTERRUPT |
1062 I915_DISPLAY_PIPE_A_EVENT_INTERRUPT |
1063 I915_DISPLAY_PIPE_B_EVENT_INTERRUPT |
1064 I915_MASTER_ERROR_INTERRUPT);
1065
1066 enable_mask =
1067 I915_ASLE_INTERRUPT |
1068 I915_DISPLAY_PIPE_A_EVENT_INTERRUPT |
1069 I915_DISPLAY_PIPE_B_EVENT_INTERRUPT |
1070 I915_MASTER_ERROR_INTERRUPT |
1071 I915_USER_INTERRUPT;
1072
1073 if (I915_HAS_HOTPLUG(dev_priv)) {
1074 /* Enable in IER... */
1075 enable_mask |= I915_DISPLAY_PORT_INTERRUPT;
1076 /* and unmask in IMR */
1077 dev_priv->irq_mask &= ~I915_DISPLAY_PORT_INTERRUPT;
1078 }
1079
1080 GEN3_IRQ_INIT(uncore, GEN2_, dev_priv->irq_mask, enable_mask);
1081
1082 /* Interrupt setup is already guaranteed to be single-threaded, this is
1083 * just to make the assert_spin_locked check happy. */
1084 spin_lock_irq(&dev_priv->irq_lock);
1085 i915_enable_pipestat(dev_priv, PIPE_A, PIPE_CRC_DONE_INTERRUPT_STATUS);
1086 i915_enable_pipestat(dev_priv, PIPE_B, PIPE_CRC_DONE_INTERRUPT_STATUS);
1087 spin_unlock_irq(&dev_priv->irq_lock);
1088
1089 i915_enable_asle_pipestat(dev_priv);
1090 }
1091
i915_irq_handler(int irq,void * arg)1092 static irqreturn_t i915_irq_handler(int irq, void *arg)
1093 {
1094 struct drm_i915_private *dev_priv = arg;
1095 irqreturn_t ret = IRQ_NONE;
1096
1097 if (!intel_irqs_enabled(dev_priv))
1098 return IRQ_NONE;
1099
1100 /* IRQs are synced during runtime_suspend, we don't require a wakeref */
1101 disable_rpm_wakeref_asserts(&dev_priv->runtime_pm);
1102
1103 do {
1104 u32 pipe_stats[I915_MAX_PIPES] = {};
1105 u32 eir = 0, eir_stuck = 0;
1106 u32 hotplug_status = 0;
1107 u32 iir;
1108
1109 iir = intel_uncore_read(&dev_priv->uncore, GEN2_IIR);
1110 if (iir == 0)
1111 break;
1112
1113 ret = IRQ_HANDLED;
1114
1115 if (I915_HAS_HOTPLUG(dev_priv) &&
1116 iir & I915_DISPLAY_PORT_INTERRUPT)
1117 hotplug_status = i9xx_hpd_irq_ack(dev_priv);
1118
1119 /* Call regardless, as some status bits might not be
1120 * signalled in iir */
1121 i9xx_pipestat_irq_ack(dev_priv, iir, pipe_stats);
1122
1123 if (iir & I915_MASTER_ERROR_INTERRUPT)
1124 i9xx_error_irq_ack(dev_priv, &eir, &eir_stuck);
1125
1126 intel_uncore_write(&dev_priv->uncore, GEN2_IIR, iir);
1127
1128 if (iir & I915_USER_INTERRUPT)
1129 intel_engine_cs_irq(to_gt(dev_priv)->engine[RCS0], iir);
1130
1131 if (iir & I915_MASTER_ERROR_INTERRUPT)
1132 i9xx_error_irq_handler(dev_priv, eir, eir_stuck);
1133
1134 if (hotplug_status)
1135 i9xx_hpd_irq_handler(dev_priv, hotplug_status);
1136
1137 i915_pipestat_irq_handler(dev_priv, iir, pipe_stats);
1138 } while (0);
1139
1140 pmu_irq_stats(dev_priv, ret);
1141
1142 enable_rpm_wakeref_asserts(&dev_priv->runtime_pm);
1143
1144 return ret;
1145 }
1146
i965_irq_reset(struct drm_i915_private * dev_priv)1147 static void i965_irq_reset(struct drm_i915_private *dev_priv)
1148 {
1149 struct intel_uncore *uncore = &dev_priv->uncore;
1150
1151 i915_hotplug_interrupt_update(dev_priv, 0xffffffff, 0);
1152 intel_uncore_rmw(uncore, PORT_HOTPLUG_STAT(dev_priv), 0, 0);
1153
1154 i9xx_pipestat_irq_reset(dev_priv);
1155
1156 GEN3_IRQ_RESET(uncore, GEN2_);
1157 dev_priv->irq_mask = ~0u;
1158 }
1159
i965_error_mask(struct drm_i915_private * i915)1160 static u32 i965_error_mask(struct drm_i915_private *i915)
1161 {
1162 /*
1163 * Enable some error detection, note the instruction error mask
1164 * bit is reserved, so we leave it masked.
1165 *
1166 * i965 FBC no longer generates spurious GTT errors,
1167 * so we can always enable the page table errors.
1168 */
1169 if (IS_G4X(i915))
1170 return ~(GM45_ERROR_PAGE_TABLE |
1171 GM45_ERROR_MEM_PRIV |
1172 GM45_ERROR_CP_PRIV |
1173 I915_ERROR_MEMORY_REFRESH);
1174 else
1175 return ~(I915_ERROR_PAGE_TABLE |
1176 I915_ERROR_MEMORY_REFRESH);
1177 }
1178
i965_irq_postinstall(struct drm_i915_private * dev_priv)1179 static void i965_irq_postinstall(struct drm_i915_private *dev_priv)
1180 {
1181 struct intel_uncore *uncore = &dev_priv->uncore;
1182 u32 enable_mask;
1183
1184 intel_uncore_write(uncore, EMR, i965_error_mask(dev_priv));
1185
1186 /* Unmask the interrupts that we always want on. */
1187 dev_priv->irq_mask =
1188 ~(I915_ASLE_INTERRUPT |
1189 I915_DISPLAY_PORT_INTERRUPT |
1190 I915_DISPLAY_PIPE_A_EVENT_INTERRUPT |
1191 I915_DISPLAY_PIPE_B_EVENT_INTERRUPT |
1192 I915_MASTER_ERROR_INTERRUPT);
1193
1194 enable_mask =
1195 I915_ASLE_INTERRUPT |
1196 I915_DISPLAY_PORT_INTERRUPT |
1197 I915_DISPLAY_PIPE_A_EVENT_INTERRUPT |
1198 I915_DISPLAY_PIPE_B_EVENT_INTERRUPT |
1199 I915_MASTER_ERROR_INTERRUPT |
1200 I915_USER_INTERRUPT;
1201
1202 if (IS_G4X(dev_priv))
1203 enable_mask |= I915_BSD_USER_INTERRUPT;
1204
1205 GEN3_IRQ_INIT(uncore, GEN2_, dev_priv->irq_mask, enable_mask);
1206
1207 /* Interrupt setup is already guaranteed to be single-threaded, this is
1208 * just to make the assert_spin_locked check happy. */
1209 spin_lock_irq(&dev_priv->irq_lock);
1210 i915_enable_pipestat(dev_priv, PIPE_A, PIPE_GMBUS_INTERRUPT_STATUS);
1211 i915_enable_pipestat(dev_priv, PIPE_A, PIPE_CRC_DONE_INTERRUPT_STATUS);
1212 i915_enable_pipestat(dev_priv, PIPE_B, PIPE_CRC_DONE_INTERRUPT_STATUS);
1213 spin_unlock_irq(&dev_priv->irq_lock);
1214
1215 i915_enable_asle_pipestat(dev_priv);
1216 }
1217
i965_irq_handler(int irq,void * arg)1218 static irqreturn_t i965_irq_handler(int irq, void *arg)
1219 {
1220 struct drm_i915_private *dev_priv = arg;
1221 irqreturn_t ret = IRQ_NONE;
1222
1223 if (!intel_irqs_enabled(dev_priv))
1224 return IRQ_NONE;
1225
1226 /* IRQs are synced during runtime_suspend, we don't require a wakeref */
1227 disable_rpm_wakeref_asserts(&dev_priv->runtime_pm);
1228
1229 do {
1230 u32 pipe_stats[I915_MAX_PIPES] = {};
1231 u32 eir = 0, eir_stuck = 0;
1232 u32 hotplug_status = 0;
1233 u32 iir;
1234
1235 iir = intel_uncore_read(&dev_priv->uncore, GEN2_IIR);
1236 if (iir == 0)
1237 break;
1238
1239 ret = IRQ_HANDLED;
1240
1241 if (iir & I915_DISPLAY_PORT_INTERRUPT)
1242 hotplug_status = i9xx_hpd_irq_ack(dev_priv);
1243
1244 /* Call regardless, as some status bits might not be
1245 * signalled in iir */
1246 i9xx_pipestat_irq_ack(dev_priv, iir, pipe_stats);
1247
1248 if (iir & I915_MASTER_ERROR_INTERRUPT)
1249 i9xx_error_irq_ack(dev_priv, &eir, &eir_stuck);
1250
1251 intel_uncore_write(&dev_priv->uncore, GEN2_IIR, iir);
1252
1253 if (iir & I915_USER_INTERRUPT)
1254 intel_engine_cs_irq(to_gt(dev_priv)->engine[RCS0],
1255 iir);
1256
1257 if (iir & I915_BSD_USER_INTERRUPT)
1258 intel_engine_cs_irq(to_gt(dev_priv)->engine[VCS0],
1259 iir >> 25);
1260
1261 if (iir & I915_MASTER_ERROR_INTERRUPT)
1262 i9xx_error_irq_handler(dev_priv, eir, eir_stuck);
1263
1264 if (hotplug_status)
1265 i9xx_hpd_irq_handler(dev_priv, hotplug_status);
1266
1267 i965_pipestat_irq_handler(dev_priv, iir, pipe_stats);
1268 } while (0);
1269
1270 pmu_irq_stats(dev_priv, IRQ_HANDLED);
1271
1272 enable_rpm_wakeref_asserts(&dev_priv->runtime_pm);
1273
1274 return ret;
1275 }
1276
1277 /**
1278 * intel_irq_init - initializes irq support
1279 * @dev_priv: i915 device instance
1280 *
1281 * This function initializes all the irq support including work items, timers
1282 * and all the vtables. It does not setup the interrupt itself though.
1283 */
intel_irq_init(struct drm_i915_private * dev_priv)1284 void intel_irq_init(struct drm_i915_private *dev_priv)
1285 {
1286 int i;
1287
1288 INIT_WORK(&dev_priv->l3_parity.error_work, ivb_parity_work);
1289 for (i = 0; i < MAX_L3_SLICES; ++i)
1290 dev_priv->l3_parity.remap_info[i] = NULL;
1291
1292 /* pre-gen11 the guc irqs bits are in the upper 16 bits of the pm reg */
1293 if (HAS_GT_UC(dev_priv) && GRAPHICS_VER(dev_priv) < 11)
1294 to_gt(dev_priv)->pm_guc_events = GUC_INTR_GUC2HOST << 16;
1295 }
1296
1297 /**
1298 * intel_irq_fini - deinitializes IRQ support
1299 * @i915: i915 device instance
1300 *
1301 * This function deinitializes all the IRQ support.
1302 */
intel_irq_fini(struct drm_i915_private * i915)1303 void intel_irq_fini(struct drm_i915_private *i915)
1304 {
1305 int i;
1306
1307 for (i = 0; i < MAX_L3_SLICES; ++i)
1308 kfree(i915->l3_parity.remap_info[i]);
1309 }
1310
intel_irq_handler(struct drm_i915_private * dev_priv)1311 static irq_handler_t intel_irq_handler(struct drm_i915_private *dev_priv)
1312 {
1313 if (HAS_GMCH(dev_priv)) {
1314 if (IS_CHERRYVIEW(dev_priv))
1315 return cherryview_irq_handler;
1316 else if (IS_VALLEYVIEW(dev_priv))
1317 return valleyview_irq_handler;
1318 else if (GRAPHICS_VER(dev_priv) == 4)
1319 return i965_irq_handler;
1320 else if (GRAPHICS_VER(dev_priv) == 3)
1321 return i915_irq_handler;
1322 else
1323 return i8xx_irq_handler;
1324 } else {
1325 if (GRAPHICS_VER_FULL(dev_priv) >= IP_VER(12, 10))
1326 return dg1_irq_handler;
1327 else if (GRAPHICS_VER(dev_priv) >= 11)
1328 return gen11_irq_handler;
1329 else if (GRAPHICS_VER(dev_priv) >= 8)
1330 return gen8_irq_handler;
1331 else
1332 return ilk_irq_handler;
1333 }
1334 }
1335
intel_irq_reset(struct drm_i915_private * dev_priv)1336 static void intel_irq_reset(struct drm_i915_private *dev_priv)
1337 {
1338 if (HAS_GMCH(dev_priv)) {
1339 if (IS_CHERRYVIEW(dev_priv))
1340 cherryview_irq_reset(dev_priv);
1341 else if (IS_VALLEYVIEW(dev_priv))
1342 valleyview_irq_reset(dev_priv);
1343 else if (GRAPHICS_VER(dev_priv) == 4)
1344 i965_irq_reset(dev_priv);
1345 else if (GRAPHICS_VER(dev_priv) == 3)
1346 i915_irq_reset(dev_priv);
1347 else
1348 i8xx_irq_reset(dev_priv);
1349 } else {
1350 if (GRAPHICS_VER_FULL(dev_priv) >= IP_VER(12, 10))
1351 dg1_irq_reset(dev_priv);
1352 else if (GRAPHICS_VER(dev_priv) >= 11)
1353 gen11_irq_reset(dev_priv);
1354 else if (GRAPHICS_VER(dev_priv) >= 8)
1355 gen8_irq_reset(dev_priv);
1356 else
1357 ilk_irq_reset(dev_priv);
1358 }
1359 }
1360
intel_irq_postinstall(struct drm_i915_private * dev_priv)1361 static void intel_irq_postinstall(struct drm_i915_private *dev_priv)
1362 {
1363 if (HAS_GMCH(dev_priv)) {
1364 if (IS_CHERRYVIEW(dev_priv))
1365 cherryview_irq_postinstall(dev_priv);
1366 else if (IS_VALLEYVIEW(dev_priv))
1367 valleyview_irq_postinstall(dev_priv);
1368 else if (GRAPHICS_VER(dev_priv) == 4)
1369 i965_irq_postinstall(dev_priv);
1370 else if (GRAPHICS_VER(dev_priv) == 3)
1371 i915_irq_postinstall(dev_priv);
1372 else
1373 i8xx_irq_postinstall(dev_priv);
1374 } else {
1375 if (GRAPHICS_VER_FULL(dev_priv) >= IP_VER(12, 10))
1376 dg1_irq_postinstall(dev_priv);
1377 else if (GRAPHICS_VER(dev_priv) >= 11)
1378 gen11_irq_postinstall(dev_priv);
1379 else if (GRAPHICS_VER(dev_priv) >= 8)
1380 gen8_irq_postinstall(dev_priv);
1381 else
1382 ilk_irq_postinstall(dev_priv);
1383 }
1384 }
1385
1386 /**
1387 * intel_irq_install - enables the hardware interrupt
1388 * @dev_priv: i915 device instance
1389 *
1390 * This function enables the hardware interrupt handling, but leaves the hotplug
1391 * handling still disabled. It is called after intel_irq_init().
1392 *
1393 * In the driver load and resume code we need working interrupts in a few places
1394 * but don't want to deal with the hassle of concurrent probe and hotplug
1395 * workers. Hence the split into this two-stage approach.
1396 */
intel_irq_install(struct drm_i915_private * dev_priv)1397 int intel_irq_install(struct drm_i915_private *dev_priv)
1398 {
1399 int irq = to_pci_dev(dev_priv->drm.dev)->irq;
1400 int ret;
1401
1402 /*
1403 * We enable some interrupt sources in our postinstall hooks, so mark
1404 * interrupts as enabled _before_ actually enabling them to avoid
1405 * special cases in our ordering checks.
1406 */
1407 dev_priv->runtime_pm.irqs_enabled = true;
1408
1409 dev_priv->irq_enabled = true;
1410
1411 intel_irq_reset(dev_priv);
1412
1413 ret = request_irq(irq, intel_irq_handler(dev_priv),
1414 IRQF_SHARED, DRIVER_NAME, dev_priv);
1415 if (ret < 0) {
1416 dev_priv->irq_enabled = false;
1417 return ret;
1418 }
1419
1420 intel_irq_postinstall(dev_priv);
1421
1422 return ret;
1423 }
1424
1425 /**
1426 * intel_irq_uninstall - finilizes all irq handling
1427 * @dev_priv: i915 device instance
1428 *
1429 * This stops interrupt and hotplug handling and unregisters and frees all
1430 * resources acquired in the init functions.
1431 */
intel_irq_uninstall(struct drm_i915_private * dev_priv)1432 void intel_irq_uninstall(struct drm_i915_private *dev_priv)
1433 {
1434 int irq = to_pci_dev(dev_priv->drm.dev)->irq;
1435
1436 /*
1437 * FIXME we can get called twice during driver probe
1438 * error handling as well as during driver remove due to
1439 * intel_display_driver_remove() calling us out of sequence.
1440 * Would be nice if it didn't do that...
1441 */
1442 if (!dev_priv->irq_enabled)
1443 return;
1444
1445 dev_priv->irq_enabled = false;
1446
1447 intel_irq_reset(dev_priv);
1448
1449 free_irq(irq, dev_priv);
1450
1451 intel_hpd_cancel_work(dev_priv);
1452 dev_priv->runtime_pm.irqs_enabled = false;
1453 }
1454
1455 /**
1456 * intel_runtime_pm_disable_interrupts - runtime interrupt disabling
1457 * @dev_priv: i915 device instance
1458 *
1459 * This function is used to disable interrupts at runtime, both in the runtime
1460 * pm and the system suspend/resume code.
1461 */
intel_runtime_pm_disable_interrupts(struct drm_i915_private * dev_priv)1462 void intel_runtime_pm_disable_interrupts(struct drm_i915_private *dev_priv)
1463 {
1464 intel_irq_reset(dev_priv);
1465 dev_priv->runtime_pm.irqs_enabled = false;
1466 intel_synchronize_irq(dev_priv);
1467 }
1468
1469 /**
1470 * intel_runtime_pm_enable_interrupts - runtime interrupt enabling
1471 * @dev_priv: i915 device instance
1472 *
1473 * This function is used to enable interrupts at runtime, both in the runtime
1474 * pm and the system suspend/resume code.
1475 */
intel_runtime_pm_enable_interrupts(struct drm_i915_private * dev_priv)1476 void intel_runtime_pm_enable_interrupts(struct drm_i915_private *dev_priv)
1477 {
1478 dev_priv->runtime_pm.irqs_enabled = true;
1479 intel_irq_reset(dev_priv);
1480 intel_irq_postinstall(dev_priv);
1481 }
1482
intel_irqs_enabled(struct drm_i915_private * dev_priv)1483 bool intel_irqs_enabled(struct drm_i915_private *dev_priv)
1484 {
1485 return dev_priv->runtime_pm.irqs_enabled;
1486 }
1487
intel_synchronize_irq(struct drm_i915_private * i915)1488 void intel_synchronize_irq(struct drm_i915_private *i915)
1489 {
1490 synchronize_irq(to_pci_dev(i915->drm.dev)->irq);
1491 }
1492
intel_synchronize_hardirq(struct drm_i915_private * i915)1493 void intel_synchronize_hardirq(struct drm_i915_private *i915)
1494 {
1495 synchronize_hardirq(to_pci_dev(i915->drm.dev)->irq);
1496 }
1497