Lines Matching +full:cs +full:- +full:1
1 // SPDX-License-Identifier: MIT
26 #define CS_GPR(engine, n) ((engine)->mmio_base + 0x600 + (n) * 4)
35 return __vm_create_scratch_for_read_pinned(>->ggtt->vm, PAGE_SIZE); in create_scratch()
57 tasklet_hi_schedule(&engine->sched_engine->tasklet); in wait_for_submit()
68 if (!READ_ONCE(engine->execlists.pending[0]) && is_active(rq)) in wait_for_submit()
72 return -ETIME; in wait_for_submit()
75 } while (1); in wait_for_submit()
81 i915_ggtt_offset(ce->engine->status_page.vma) + in emit_semaphore_signal()
84 u32 *cs; in emit_semaphore_signal() local
90 cs = intel_ring_begin(rq, 4); in emit_semaphore_signal()
91 if (IS_ERR(cs)) { in emit_semaphore_signal()
93 return PTR_ERR(cs); in emit_semaphore_signal()
96 *cs++ = MI_STORE_DWORD_IMM_GEN4 | MI_USE_GGTT; in emit_semaphore_signal()
97 *cs++ = offset; in emit_semaphore_signal()
98 *cs++ = 0; in emit_semaphore_signal()
99 *cs++ = 1; in emit_semaphore_signal()
101 intel_ring_advance(rq, cs); in emit_semaphore_signal()
103 rq->sched.attr.priority = I915_PRIORITY_BARRIER; in emit_semaphore_signal()
114 rq = intel_engine_create_kernel_request(ce->engine); in context_flush()
118 fence = i915_active_fence_get(&ce->timeline->last_request); in context_flush()
127 err = -ETIME; in context_flush()
139 if (GRAPHICS_VER(engine->i915) < 12) in get_lri_mask()
142 switch (engine->class) { in get_lri_mask()
170 return -ENOMEM; in live_lrc_layout()
178 if (!engine->default_state) in live_lrc_layout()
181 hw = shmem_pin_map(engine->default_state); in live_lrc_layout()
183 err = -ENOMEM; in live_lrc_layout()
189 engine->kernel_context, engine, true); in live_lrc_layout()
203 engine->name, lri, dw); in live_lrc_layout()
210 engine->name, dw, lri); in live_lrc_layout()
211 err = -EINVAL; in live_lrc_layout()
217 engine->name, dw, lri, lrc[dw]); in live_lrc_layout()
218 err = -EINVAL; in live_lrc_layout()
228 * RCS && CCS: BITS(0 - 10) in live_lrc_layout()
229 * BCS: BITS(0 - 11) in live_lrc_layout()
230 * VECS && VCS: BITS(0 - 13) in live_lrc_layout()
243 engine->name, dw, offset, lrc[dw]); in live_lrc_layout()
244 err = -EINVAL; in live_lrc_layout()
253 lri -= 2; in live_lrc_layout()
258 pr_info("%s: HW register image:\n", engine->name); in live_lrc_layout()
261 pr_info("%s: SW register image:\n", engine->name); in live_lrc_layout()
265 shmem_unpin_map(engine->default_state, hw); in live_lrc_layout()
282 return -1; in find_offset()
304 i915_mmio_reg_offset(RING_START(engine->mmio_base)), in live_lrc_fixed()
305 CTX_RING_START - 1, in live_lrc_fixed()
309 i915_mmio_reg_offset(RING_CTL(engine->mmio_base)), in live_lrc_fixed()
310 CTX_RING_CTL - 1, in live_lrc_fixed()
314 i915_mmio_reg_offset(RING_HEAD(engine->mmio_base)), in live_lrc_fixed()
315 CTX_RING_HEAD - 1, in live_lrc_fixed()
319 i915_mmio_reg_offset(RING_TAIL(engine->mmio_base)), in live_lrc_fixed()
320 CTX_RING_TAIL - 1, in live_lrc_fixed()
324 i915_mmio_reg_offset(RING_MI_MODE(engine->mmio_base)), in live_lrc_fixed()
329 i915_mmio_reg_offset(RING_BBSTATE(engine->mmio_base)), in live_lrc_fixed()
330 CTX_BB_STATE - 1, in live_lrc_fixed()
334 i915_mmio_reg_offset(RING_BB_PER_CTX_PTR(engine->mmio_base)), in live_lrc_fixed()
339 i915_mmio_reg_offset(RING_INDIRECT_CTX(engine->mmio_base)), in live_lrc_fixed()
344 i915_mmio_reg_offset(RING_INDIRECT_CTX_OFFSET(engine->mmio_base)), in live_lrc_fixed()
349 i915_mmio_reg_offset(RING_CTX_TIMESTAMP(engine->mmio_base)), in live_lrc_fixed()
350 CTX_TIMESTAMP - 1, in live_lrc_fixed()
354 i915_mmio_reg_offset(GEN8_RING_CS_GPR(engine->mmio_base, 0)), in live_lrc_fixed()
359 i915_mmio_reg_offset(RING_CMD_BUF_CCTL(engine->mmio_base)), in live_lrc_fixed()
364 i915_mmio_reg_offset(RING_BB_OFFSET(engine->mmio_base)), in live_lrc_fixed()
372 if (!engine->default_state) in live_lrc_fixed()
375 hw = shmem_pin_map(engine->default_state); in live_lrc_fixed()
377 err = -ENOMEM; in live_lrc_fixed()
382 for (t = tbl; t->name; t++) { in live_lrc_fixed()
383 int dw = find_offset(hw, t->reg); in live_lrc_fixed()
385 if (dw != t->offset) { in live_lrc_fixed()
387 engine->name, in live_lrc_fixed()
388 t->name, in live_lrc_fixed()
389 t->reg, in live_lrc_fixed()
391 t->offset); in live_lrc_fixed()
392 err = -EINVAL; in live_lrc_fixed()
396 shmem_unpin_map(engine->default_state, hw); in live_lrc_fixed()
414 u32 *cs; in __live_lrc_state() local
424 err = i915_gem_object_lock(scratch->obj, &ww); in __live_lrc_state()
436 cs = intel_ring_begin(rq, 4 * MAX_IDX); in __live_lrc_state()
437 if (IS_ERR(cs)) { in __live_lrc_state()
438 err = PTR_ERR(cs); in __live_lrc_state()
443 *cs++ = MI_STORE_REGISTER_MEM_GEN8 | MI_USE_GGTT; in __live_lrc_state()
444 *cs++ = i915_mmio_reg_offset(RING_START(engine->mmio_base)); in __live_lrc_state()
445 *cs++ = i915_ggtt_offset(scratch) + RING_START_IDX * sizeof(u32); in __live_lrc_state()
446 *cs++ = 0; in __live_lrc_state()
448 expected[RING_START_IDX] = i915_ggtt_offset(ce->ring->vma); in __live_lrc_state()
450 *cs++ = MI_STORE_REGISTER_MEM_GEN8 | MI_USE_GGTT; in __live_lrc_state()
451 *cs++ = i915_mmio_reg_offset(RING_TAIL(engine->mmio_base)); in __live_lrc_state()
452 *cs++ = i915_ggtt_offset(scratch) + RING_TAIL_IDX * sizeof(u32); in __live_lrc_state()
453 *cs++ = 0; in __live_lrc_state()
463 expected[RING_TAIL_IDX] = ce->ring->tail; in __live_lrc_state()
466 err = -ETIME; in __live_lrc_state()
470 cs = i915_gem_object_pin_map(scratch->obj, I915_MAP_WB); in __live_lrc_state()
471 if (IS_ERR(cs)) { in __live_lrc_state()
472 err = PTR_ERR(cs); in __live_lrc_state()
477 if (cs[n] != expected[n]) { in __live_lrc_state()
479 engine->name, n, cs[n], expected[n]); in __live_lrc_state()
480 err = -EINVAL; in __live_lrc_state()
485 i915_gem_object_unpin_map(scratch->obj); in __live_lrc_state()
492 if (err == -EDEADLK) { in __live_lrc_state()
525 if (igt_flush_test(gt->i915)) in live_lrc_state()
526 err = -EIO; in live_lrc_state()
535 u32 *cs; in gpr_make_dirty() local
542 cs = intel_ring_begin(rq, 2 * NUM_GPR_DW + 2); in gpr_make_dirty()
543 if (IS_ERR(cs)) { in gpr_make_dirty()
545 return PTR_ERR(cs); in gpr_make_dirty()
548 *cs++ = MI_LOAD_REGISTER_IMM(NUM_GPR_DW); in gpr_make_dirty()
550 *cs++ = CS_GPR(ce->engine, n); in gpr_make_dirty()
551 *cs++ = STACK_MAGIC; in gpr_make_dirty()
553 *cs++ = MI_NOOP; in gpr_make_dirty()
555 intel_ring_advance(rq, cs); in gpr_make_dirty()
557 rq->sched.attr.priority = I915_PRIORITY_BARRIER; in gpr_make_dirty()
567 i915_ggtt_offset(ce->engine->status_page.vma) + in __gpr_read()
570 u32 *cs; in __gpr_read() local
578 cs = intel_ring_begin(rq, 6 + 4 * NUM_GPR_DW); in __gpr_read()
579 if (IS_ERR(cs)) { in __gpr_read()
581 return ERR_CAST(cs); in __gpr_read()
584 *cs++ = MI_ARB_ON_OFF | MI_ARB_ENABLE; in __gpr_read()
585 *cs++ = MI_NOOP; in __gpr_read()
587 *cs++ = MI_SEMAPHORE_WAIT | in __gpr_read()
591 *cs++ = 0; in __gpr_read()
592 *cs++ = offset; in __gpr_read()
593 *cs++ = 0; in __gpr_read()
596 *cs++ = MI_STORE_REGISTER_MEM_GEN8 | MI_USE_GGTT; in __gpr_read()
597 *cs++ = CS_GPR(ce->engine, n); in __gpr_read()
598 *cs++ = i915_ggtt_offset(scratch) + n * sizeof(u32); in __gpr_read()
599 *cs++ = 0; in __gpr_read()
618 u32 *slot = memset32(engine->status_page.addr + 1000, 0, 4); in __live_lrc_gpr()
621 u32 *cs; in __live_lrc_gpr() local
625 if (GRAPHICS_VER(engine->i915) < 9 && engine->class != RENDER_CLASS) in __live_lrc_gpr()
628 err = gpr_make_dirty(engine->kernel_context); in __live_lrc_gpr()
647 err = gpr_make_dirty(engine->kernel_context); in __live_lrc_gpr()
651 err = emit_semaphore_signal(engine->kernel_context, slot); in __live_lrc_gpr()
659 slot[0] = 1; in __live_lrc_gpr()
664 err = -ETIME; in __live_lrc_gpr()
668 cs = i915_gem_object_pin_map_unlocked(scratch->obj, I915_MAP_WB); in __live_lrc_gpr()
669 if (IS_ERR(cs)) { in __live_lrc_gpr()
670 err = PTR_ERR(cs); in __live_lrc_gpr()
675 if (cs[n]) { in __live_lrc_gpr()
677 engine->name, in __live_lrc_gpr()
678 n / 2, n & 1 ? "udw" : "ldw", in __live_lrc_gpr()
679 cs[n]); in __live_lrc_gpr()
680 err = -EINVAL; in __live_lrc_gpr()
685 i915_gem_object_unpin_map(scratch->obj); in __live_lrc_gpr()
688 memset32(&slot[0], -1, 4); in __live_lrc_gpr()
726 if (igt_flush_test(gt->i915)) in live_lrc_gpr()
727 err = -EIO; in live_lrc_gpr()
740 i915_ggtt_offset(ce->engine->status_page.vma) + in create_timestamp()
743 u32 *cs; in create_timestamp() local
750 cs = intel_ring_begin(rq, 10); in create_timestamp()
751 if (IS_ERR(cs)) { in create_timestamp()
752 err = PTR_ERR(cs); in create_timestamp()
756 *cs++ = MI_ARB_ON_OFF | MI_ARB_ENABLE; in create_timestamp()
757 *cs++ = MI_NOOP; in create_timestamp()
759 *cs++ = MI_SEMAPHORE_WAIT | in create_timestamp()
763 *cs++ = 0; in create_timestamp()
764 *cs++ = offset; in create_timestamp()
765 *cs++ = 0; in create_timestamp()
767 *cs++ = MI_STORE_REGISTER_MEM_GEN8 | MI_USE_GGTT; in create_timestamp()
768 *cs++ = i915_mmio_reg_offset(RING_CTX_TIMESTAMP(rq->engine->mmio_base)); in create_timestamp()
769 *cs++ = offset + idx * sizeof(u32); in create_timestamp()
770 *cs++ = 0; in create_timestamp()
772 intel_ring_advance(rq, cs); in create_timestamp()
794 return (s32)(end - start) > 0; in timestamp_advanced()
799 u32 *slot = memset32(arg->engine->status_page.addr + 1000, 0, 4); in __lrc_timestamp()
804 arg->ce[0]->lrc_reg_state[CTX_TIMESTAMP] = arg->poison; in __lrc_timestamp()
805 rq = create_timestamp(arg->ce[0], slot, 1); in __lrc_timestamp()
809 err = wait_for_submit(rq->engine, rq, HZ / 2); in __lrc_timestamp()
814 arg->ce[1]->lrc_reg_state[CTX_TIMESTAMP] = 0xdeadbeef; in __lrc_timestamp()
815 err = emit_semaphore_signal(arg->ce[1], slot); in __lrc_timestamp()
819 slot[0] = 1; in __lrc_timestamp()
824 err = context_flush(arg->ce[0], HZ / 2); in __lrc_timestamp()
828 if (!timestamp_advanced(arg->poison, slot[1])) { in __lrc_timestamp()
830 arg->engine->name, preempt ? "preempt" : "simple", in __lrc_timestamp()
831 arg->poison, slot[1]); in __lrc_timestamp()
832 err = -EINVAL; in __lrc_timestamp()
835 timestamp = READ_ONCE(arg->ce[0]->lrc_reg_state[CTX_TIMESTAMP]); in __lrc_timestamp()
836 if (!timestamp_advanced(slot[1], timestamp)) { in __lrc_timestamp()
838 arg->engine->name, preempt ? "preempt" : "simple", in __lrc_timestamp()
839 slot[1], timestamp); in __lrc_timestamp()
840 err = -EINVAL; in __lrc_timestamp()
844 memset32(slot, -1, 4); in __lrc_timestamp()
857 (u32)S32_MAX + 1, in live_lrc_timestamp()
915 if (igt_flush_test(gt->i915)) in live_lrc_timestamp()
916 err = -EIO; in live_lrc_timestamp()
931 obj = i915_gem_object_create_internal(vm->i915, size); in create_user_vma()
967 u32 dw, x, *cs, *hw; in store_context() local
970 batch = create_user_vma(ce->vm, SZ_64K); in store_context()
974 cs = i915_gem_object_pin_map_unlocked(batch->obj, I915_MAP_WC); in store_context()
975 if (IS_ERR(cs)) { in store_context()
977 return ERR_CAST(cs); in store_context()
980 defaults = shmem_pin_map(ce->engine->default_state); in store_context()
982 i915_gem_object_unpin_map(batch->obj); in store_context()
984 return ERR_PTR(-ENOMEM); in store_context()
1021 ce->engine->name); in store_context()
1027 len = (len + 1) / 2; in store_context()
1028 while (len--) { in store_context()
1029 *cs++ = MI_STORE_REGISTER_MEM_GEN8; in store_context()
1030 *cs++ = hw[dw]; in store_context()
1031 *cs++ = lower_32_bits(i915_vma_offset(scratch) + x); in store_context()
1032 *cs++ = upper_32_bits(i915_vma_offset(scratch) + x); in store_context()
1040 *cs++ = MI_BATCH_BUFFER_END; in store_context()
1042 shmem_unpin_map(ce->engine->default_state, defaults); in store_context()
1044 i915_gem_object_flush_map(batch->obj); in store_context()
1045 i915_gem_object_unpin_map(batch->obj); in store_context()
1058 u32 *cs; in record_registers() local
1091 cs = intel_ring_begin(rq, 14); in record_registers()
1092 if (IS_ERR(cs)) { in record_registers()
1093 err = PTR_ERR(cs); in record_registers()
1097 *cs++ = MI_ARB_ON_OFF | MI_ARB_DISABLE; in record_registers()
1098 *cs++ = MI_BATCH_BUFFER_START_GEN8 | BIT(8); in record_registers()
1099 *cs++ = lower_32_bits(i915_vma_offset(b_before)); in record_registers()
1100 *cs++ = upper_32_bits(i915_vma_offset(b_before)); in record_registers()
1102 *cs++ = MI_ARB_ON_OFF | MI_ARB_ENABLE; in record_registers()
1103 *cs++ = MI_SEMAPHORE_WAIT | in record_registers()
1107 *cs++ = 0; in record_registers()
1108 *cs++ = i915_ggtt_offset(ce->engine->status_page.vma) + in record_registers()
1110 *cs++ = 0; in record_registers()
1111 *cs++ = MI_NOOP; in record_registers()
1113 *cs++ = MI_ARB_ON_OFF | MI_ARB_DISABLE; in record_registers()
1114 *cs++ = MI_BATCH_BUFFER_START_GEN8 | BIT(8); in record_registers()
1115 *cs++ = lower_32_bits(i915_vma_offset(b_after)); in record_registers()
1116 *cs++ = upper_32_bits(i915_vma_offset(b_after)); in record_registers()
1118 intel_ring_advance(rq, cs); in record_registers()
1138 u32 dw, *cs, *hw; in load_context() local
1141 batch = create_user_vma(ce->vm, SZ_64K); in load_context()
1145 cs = i915_gem_object_pin_map_unlocked(batch->obj, I915_MAP_WC); in load_context()
1146 if (IS_ERR(cs)) { in load_context()
1148 return ERR_CAST(cs); in load_context()
1151 defaults = shmem_pin_map(ce->engine->default_state); in load_context()
1153 i915_gem_object_unpin_map(batch->obj); in load_context()
1155 return ERR_PTR(-ENOMEM); in load_context()
1180 ce->engine->name); in load_context()
1186 len = (len + 1) / 2; in load_context()
1187 *cs++ = MI_LOAD_REGISTER_IMM(len); in load_context()
1188 while (len--) { in load_context()
1189 *cs++ = hw[dw]; in load_context()
1190 *cs++ = safe_poison(hw[dw] & get_lri_mask(ce->engine, in load_context()
1198 *cs++ = MI_BATCH_BUFFER_END; in load_context()
1200 shmem_unpin_map(ce->engine->default_state, defaults); in load_context()
1202 i915_gem_object_flush_map(batch->obj); in load_context()
1203 i915_gem_object_unpin_map(batch->obj); in load_context()
1212 u32 *cs; in poison_registers() local
1229 cs = intel_ring_begin(rq, 8); in poison_registers()
1230 if (IS_ERR(cs)) { in poison_registers()
1231 err = PTR_ERR(cs); in poison_registers()
1235 *cs++ = MI_ARB_ON_OFF | MI_ARB_DISABLE; in poison_registers()
1236 *cs++ = MI_BATCH_BUFFER_START_GEN8 | BIT(8); in poison_registers()
1237 *cs++ = lower_32_bits(i915_vma_offset(batch)); in poison_registers()
1238 *cs++ = upper_32_bits(i915_vma_offset(batch)); in poison_registers()
1240 *cs++ = MI_STORE_DWORD_IMM_GEN4 | MI_USE_GGTT; in poison_registers()
1241 *cs++ = i915_ggtt_offset(ce->engine->status_page.vma) + in poison_registers()
1243 *cs++ = 0; in poison_registers()
1244 *cs++ = 1; in poison_registers()
1246 intel_ring_advance(rq, cs); in poison_registers()
1248 rq->sched.attr.priority = I915_PRIORITY_BARRIER; in poison_registers()
1272 A[0] = i915_gem_object_pin_map_unlocked(ref[0]->obj, I915_MAP_WC); in compare_isolation()
1276 A[1] = i915_gem_object_pin_map_unlocked(ref[1]->obj, I915_MAP_WC); in compare_isolation()
1277 if (IS_ERR(A[1])) { in compare_isolation()
1278 err = PTR_ERR(A[1]); in compare_isolation()
1282 B[0] = i915_gem_object_pin_map_unlocked(result[0]->obj, I915_MAP_WC); in compare_isolation()
1288 B[1] = i915_gem_object_pin_map_unlocked(result[1]->obj, I915_MAP_WC); in compare_isolation()
1289 if (IS_ERR(B[1])) { in compare_isolation()
1290 err = PTR_ERR(B[1]); in compare_isolation()
1294 lrc = i915_gem_object_pin_map_unlocked(ce->state->obj, in compare_isolation()
1295 intel_gt_coherent_map_type(engine->gt, in compare_isolation()
1296 ce->state->obj, in compare_isolation()
1304 defaults = shmem_pin_map(ce->engine->default_state); in compare_isolation()
1306 err = -ENOMEM; in compare_isolation()
1333 engine->name); in compare_isolation()
1339 len = (len + 1) / 2; in compare_isolation()
1340 while (len--) { in compare_isolation()
1341 if (!is_moving(A[0][x], A[1][x]) && in compare_isolation()
1342 (A[0][x] != B[0][x] || A[1][x] != B[1][x])) { in compare_isolation()
1350 engine->name, dw, in compare_isolation()
1351 hw[dw], hw[dw + 1], in compare_isolation()
1352 A[0][x], B[0][x], B[1][x], in compare_isolation()
1353 poison, lrc[dw + 1]); in compare_isolation()
1354 err = -EINVAL; in compare_isolation()
1363 shmem_unpin_map(ce->engine->default_state, defaults); in compare_isolation()
1365 i915_gem_object_unpin_map(ce->state->obj); in compare_isolation()
1367 i915_gem_object_unpin_map(result[1]->obj); in compare_isolation()
1369 i915_gem_object_unpin_map(result[0]->obj); in compare_isolation()
1371 i915_gem_object_unpin_map(ref[1]->obj); in compare_isolation()
1373 i915_gem_object_unpin_map(ref[0]->obj); in compare_isolation()
1388 ptr = i915_gem_object_pin_map_unlocked(vma->obj, I915_MAP_WC); in create_result_vma()
1394 memset(ptr, POISON_INUSE, vma->size); in create_result_vma()
1395 i915_gem_object_flush_map(vma->obj); in create_result_vma()
1396 i915_gem_object_unpin_map(vma->obj); in create_result_vma()
1403 u32 *sema = memset32(engine->status_page.addr + 1000, 0, 1); in __lrc_isolation()
1419 ref[0] = create_result_vma(A->vm, SZ_64K); in __lrc_isolation()
1425 ref[1] = create_result_vma(A->vm, SZ_64K); in __lrc_isolation()
1426 if (IS_ERR(ref[1])) { in __lrc_isolation()
1427 err = PTR_ERR(ref[1]); in __lrc_isolation()
1431 rq = record_registers(A, ref[0], ref[1], sema); in __lrc_isolation()
1437 WRITE_ONCE(*sema, 1); in __lrc_isolation()
1442 err = -ETIME; in __lrc_isolation()
1447 result[0] = create_result_vma(A->vm, SZ_64K); in __lrc_isolation()
1453 result[1] = create_result_vma(A->vm, SZ_64K); in __lrc_isolation()
1454 if (IS_ERR(result[1])) { in __lrc_isolation()
1455 err = PTR_ERR(result[1]); in __lrc_isolation()
1459 rq = record_registers(A, result[0], result[1], sema); in __lrc_isolation()
1468 __func__, engine->name); in __lrc_isolation()
1469 err = -ETIME; in __lrc_isolation()
1473 WRITE_ONCE(*sema, -1); in __lrc_isolation()
1481 i915_vma_put(result[1]); in __lrc_isolation()
1485 i915_vma_put(ref[1]); in __lrc_isolation()
1497 if (engine->class == COPY_ENGINE_CLASS && GRAPHICS_VER(engine->i915) == 9) in skip_isolation()
1500 if (engine->class == RENDER_CLASS && GRAPHICS_VER(engine->i915) == 11) in skip_isolation()
1521 * Our goal is try and verify that per-context state cannot be in live_lrc_isolation()
1522 * tampered with by another non-privileged client. in live_lrc_isolation()
1549 if (igt_flush_test(gt->i915)) { in live_lrc_isolation()
1550 err = -EIO; in live_lrc_isolation()
1571 err = -ETIME; in wabb_ctx_submit_req()
1583 u32 *cs, bool per_ctx) in emit_wabb_ctx_canary() argument
1585 *cs++ = MI_STORE_REGISTER_MEM_GEN8 | in emit_wabb_ctx_canary()
1588 *cs++ = i915_mmio_reg_offset(RING_START(0)); in emit_wabb_ctx_canary()
1589 *cs++ = i915_ggtt_offset(ce->state) + in emit_wabb_ctx_canary()
1593 *cs++ = 0; in emit_wabb_ctx_canary()
1595 return cs; in emit_wabb_ctx_canary()
1599 emit_indirect_ctx_bb_canary(const struct intel_context *ce, u32 *cs) in emit_indirect_ctx_bb_canary() argument
1601 return emit_wabb_ctx_canary(ce, cs, false); in emit_indirect_ctx_bb_canary()
1605 emit_per_ctx_bb_canary(const struct intel_context *ce, u32 *cs) in emit_per_ctx_bb_canary() argument
1607 return emit_wabb_ctx_canary(ce, cs, true); in emit_per_ctx_bb_canary()
1613 u32 *cs = context_wabb(ce, per_ctx); in wabb_ctx_setup() local
1615 cs[CTX_BB_CANARY_INDEX] = 0xdeadf00d; in wabb_ctx_setup()
1618 setup_per_ctx_bb(ce, ce->engine, emit_per_ctx_bb_canary); in wabb_ctx_setup()
1620 setup_indirect_ctx_bb(ce, ce->engine, emit_indirect_ctx_bb_canary); in wabb_ctx_setup()
1625 const u32 * const ctx_bb = (void *)(ce->lrc_reg_state) - in check_ring_start()
1629 if (ctx_bb[CTX_BB_CANARY_INDEX] == ce->lrc_reg_state[CTX_RING_START]) in check_ring_start()
1634 ce->lrc_reg_state[CTX_RING_START]); in check_ring_start()
1648 return -EINVAL; in wabb_ctx_check()
1675 if (!a->wa_bb_page) { in __lrc_wabb_ctx()
1676 GEM_BUG_ON(b->wa_bb_page); in __lrc_wabb_ctx()
1677 GEM_BUG_ON(GRAPHICS_VER(engine->i915) == 12); in __lrc_wabb_ctx()
1721 if (igt_flush_test(gt->i915)) in lrc_wabb_ctx()
1722 err = -EIO; in lrc_wabb_ctx()
1744 const unsigned int bit = I915_RESET_ENGINE + engine->id; in garbage_reset()
1745 unsigned long *lock = &engine->gt->reset.flags; in garbage_reset()
1749 tasklet_disable(&engine->sched_engine->tasklet); in garbage_reset()
1751 if (!rq->fence.error) in garbage_reset()
1754 tasklet_enable(&engine->sched_engine->tasklet); in garbage_reset()
1771 ce->lrc_reg_state, in garbage()
1772 ce->engine->context_size - in garbage()
1808 err = -ETIME; in __lrc_garbage()
1816 if (!hang->fence.error) { in __lrc_garbage()
1819 engine->name); in __lrc_garbage()
1820 err = -EINVAL; in __lrc_garbage()
1826 engine->name); in __lrc_garbage()
1828 err = -EIO; in __lrc_garbage()
1856 if (!intel_has_reset_engine(engine->gt)) in live_lrc_garbage()
1867 if (igt_flush_test(gt->i915)) in live_lrc_garbage()
1868 err = -EIO; in live_lrc_garbage()
1887 ce->stats.runtime.num_underflow = 0; in __live_pphwsp_runtime()
1888 ce->stats.runtime.max_underflow = 0; in __live_pphwsp_runtime()
1900 if (--loop == 0) in __live_pphwsp_runtime()
1910 } while (1); in __live_pphwsp_runtime()
1914 pr_err("%s: request not completed!\n", engine->name); in __live_pphwsp_runtime()
1918 igt_flush_test(engine->i915); in __live_pphwsp_runtime()
1921 engine->name, in __live_pphwsp_runtime()
1926 if (ce->stats.runtime.num_underflow) { in __live_pphwsp_runtime()
1928 engine->name, in __live_pphwsp_runtime()
1929 ce->stats.runtime.num_underflow, in __live_pphwsp_runtime()
1930 ce->stats.runtime.max_underflow); in __live_pphwsp_runtime()
1932 err = -EOVERFLOW; in __live_pphwsp_runtime()
1960 if (igt_flush_test(gt->i915)) in live_pphwsp_runtime()
1961 err = -EIO; in live_pphwsp_runtime()