Lines Matching +full:cs +full:- +full:0
1 // SPDX-License-Identifier: MIT
16 u32 *cs, flags = 0; in gen8_emit_flush_rcs() local
42 if (GRAPHICS_VER(rq->i915) == 9) in gen8_emit_flush_rcs()
46 if (IS_KABYLAKE(rq->i915) && IS_GRAPHICS_STEP(rq->i915, 0, STEP_C0)) in gen8_emit_flush_rcs()
58 cs = intel_ring_begin(rq, len); in gen8_emit_flush_rcs()
59 if (IS_ERR(cs)) in gen8_emit_flush_rcs()
60 return PTR_ERR(cs); in gen8_emit_flush_rcs()
63 cs = gen8_emit_pipe_control(cs, 0, 0); in gen8_emit_flush_rcs()
66 cs = gen8_emit_pipe_control(cs, PIPE_CONTROL_DC_FLUSH_ENABLE, in gen8_emit_flush_rcs()
67 0); in gen8_emit_flush_rcs()
69 cs = gen8_emit_pipe_control(cs, flags, LRC_PPHWSP_SCRATCH_ADDR); in gen8_emit_flush_rcs()
72 cs = gen8_emit_pipe_control(cs, PIPE_CONTROL_CS_STALL, 0); in gen8_emit_flush_rcs()
74 intel_ring_advance(rq, cs); in gen8_emit_flush_rcs()
76 return 0; in gen8_emit_flush_rcs()
81 u32 cmd, *cs; in gen8_emit_flush_xcs() local
83 cs = intel_ring_begin(rq, 4); in gen8_emit_flush_xcs()
84 if (IS_ERR(cs)) in gen8_emit_flush_xcs()
85 return PTR_ERR(cs); in gen8_emit_flush_xcs()
99 if (rq->engine->class == VIDEO_DECODE_CLASS) in gen8_emit_flush_xcs()
103 *cs++ = cmd; in gen8_emit_flush_xcs()
104 *cs++ = LRC_PPHWSP_SCRATCH_ADDR; in gen8_emit_flush_xcs()
105 *cs++ = 0; /* upper addr */ in gen8_emit_flush_xcs()
106 *cs++ = 0; /* value */ in gen8_emit_flush_xcs()
107 intel_ring_advance(rq, cs); in gen8_emit_flush_xcs()
109 return 0; in gen8_emit_flush_xcs()
115 u32 *cs; in gen11_emit_flush_rcs() local
116 u32 flags = 0; in gen11_emit_flush_rcs()
128 cs = intel_ring_begin(rq, 6); in gen11_emit_flush_rcs()
129 if (IS_ERR(cs)) in gen11_emit_flush_rcs()
130 return PTR_ERR(cs); in gen11_emit_flush_rcs()
132 cs = gen8_emit_pipe_control(cs, flags, LRC_PPHWSP_SCRATCH_ADDR); in gen11_emit_flush_rcs()
133 intel_ring_advance(rq, cs); in gen11_emit_flush_rcs()
137 u32 *cs; in gen11_emit_flush_rcs() local
138 u32 flags = 0; in gen11_emit_flush_rcs()
152 cs = intel_ring_begin(rq, 6); in gen11_emit_flush_rcs()
153 if (IS_ERR(cs)) in gen11_emit_flush_rcs()
154 return PTR_ERR(cs); in gen11_emit_flush_rcs()
156 cs = gen8_emit_pipe_control(cs, flags, LRC_PPHWSP_SCRATCH_ADDR); in gen11_emit_flush_rcs()
157 intel_ring_advance(rq, cs); in gen11_emit_flush_rcs()
160 return 0; in gen11_emit_flush_rcs()
170 switch (engine->id) { in gen12_get_aux_inv_reg()
196 return i915_mmio_reg_valid(reg) && !HAS_FLAT_CCS(engine->i915); in gen12_needs_ccs_aux_inv()
199 u32 *gen12_emit_aux_table_inv(struct intel_engine_cs *engine, u32 *cs) in gen12_emit_aux_table_inv() argument
202 u32 gsi_offset = engine->gt->uncore->gsi_offset; in gen12_emit_aux_table_inv()
205 return cs; in gen12_emit_aux_table_inv()
207 *cs++ = MI_LOAD_REGISTER_IMM(1) | MI_LRI_MMIO_REMAP_EN; in gen12_emit_aux_table_inv()
208 *cs++ = i915_mmio_reg_offset(inv_reg) + gsi_offset; in gen12_emit_aux_table_inv()
209 *cs++ = AUX_INV; in gen12_emit_aux_table_inv()
211 *cs++ = MI_SEMAPHORE_WAIT_TOKEN | in gen12_emit_aux_table_inv()
215 *cs++ = 0; in gen12_emit_aux_table_inv()
216 *cs++ = i915_mmio_reg_offset(inv_reg) + gsi_offset; in gen12_emit_aux_table_inv()
217 *cs++ = 0; in gen12_emit_aux_table_inv()
218 *cs++ = 0; in gen12_emit_aux_table_inv()
220 return cs; in gen12_emit_aux_table_inv()
226 if (IS_GFX_GT_IP_RANGE(rq->engine->gt, IP_VER(12, 70), IP_VER(12, 74)) || in mtl_dummy_pipe_control()
227 IS_DG2(rq->i915)) { in mtl_dummy_pipe_control()
228 u32 *cs; in mtl_dummy_pipe_control() local
231 cs = intel_ring_begin(rq, 6); in mtl_dummy_pipe_control()
232 if (IS_ERR(cs)) in mtl_dummy_pipe_control()
233 return PTR_ERR(cs); in mtl_dummy_pipe_control()
234 cs = gen12_emit_pipe_control(cs, in mtl_dummy_pipe_control()
235 0, in mtl_dummy_pipe_control()
238 intel_ring_advance(rq, cs); in mtl_dummy_pipe_control()
241 return 0; in mtl_dummy_pipe_control()
246 struct intel_engine_cs *engine = rq->engine; in gen12_emit_flush_rcs()
253 u32 bit_group_0 = 0; in gen12_emit_flush_rcs()
254 u32 bit_group_1 = 0; in gen12_emit_flush_rcs()
256 u32 *cs; in gen12_emit_flush_rcs() local
268 if (GRAPHICS_VER_FULL(rq->i915) >= IP_VER(12, 70)) in gen12_emit_flush_rcs()
273 * which happens as part of pipe-control so we can in gen12_emit_flush_rcs()
279 GRAPHICS_VER_FULL(rq->i915) < IP_VER(12, 70)) in gen12_emit_flush_rcs()
285 /* Wa_1409600907:tgl,adl-p */ in gen12_emit_flush_rcs()
295 if (!HAS_3D_PIPELINE(engine->i915)) in gen12_emit_flush_rcs()
297 else if (engine->class == COMPUTE_CLASS) in gen12_emit_flush_rcs()
300 cs = intel_ring_begin(rq, 6); in gen12_emit_flush_rcs()
301 if (IS_ERR(cs)) in gen12_emit_flush_rcs()
302 return PTR_ERR(cs); in gen12_emit_flush_rcs()
304 cs = gen12_emit_pipe_control(cs, bit_group_0, bit_group_1, in gen12_emit_flush_rcs()
306 intel_ring_advance(rq, cs); in gen12_emit_flush_rcs()
310 u32 flags = 0; in gen12_emit_flush_rcs()
311 u32 *cs, count; in gen12_emit_flush_rcs() local
331 if (!HAS_3D_PIPELINE(engine->i915)) in gen12_emit_flush_rcs()
333 else if (engine->class == COMPUTE_CLASS) in gen12_emit_flush_rcs()
337 if (gen12_needs_ccs_aux_inv(rq->engine)) in gen12_emit_flush_rcs()
340 cs = intel_ring_begin(rq, count); in gen12_emit_flush_rcs()
341 if (IS_ERR(cs)) in gen12_emit_flush_rcs()
342 return PTR_ERR(cs); in gen12_emit_flush_rcs()
345 * Prevent the pre-parser from skipping past the TLB in gen12_emit_flush_rcs()
349 *cs++ = preparser_disable(true); in gen12_emit_flush_rcs()
351 cs = gen8_emit_pipe_control(cs, flags, LRC_PPHWSP_SCRATCH_ADDR); in gen12_emit_flush_rcs()
353 cs = gen12_emit_aux_table_inv(engine, cs); in gen12_emit_flush_rcs()
355 *cs++ = preparser_disable(false); in gen12_emit_flush_rcs()
356 intel_ring_advance(rq, cs); in gen12_emit_flush_rcs()
359 return 0; in gen12_emit_flush_rcs()
365 u32 *cs; in gen12_emit_flush_xcs() local
370 if (gen12_needs_ccs_aux_inv(rq->engine)) in gen12_emit_flush_xcs()
374 cs = intel_ring_begin(rq, cmd); in gen12_emit_flush_xcs()
375 if (IS_ERR(cs)) in gen12_emit_flush_xcs()
376 return PTR_ERR(cs); in gen12_emit_flush_xcs()
379 *cs++ = preparser_disable(true); in gen12_emit_flush_xcs()
393 if (rq->engine->class == VIDEO_DECODE_CLASS) in gen12_emit_flush_xcs()
396 if (gen12_needs_ccs_aux_inv(rq->engine) && in gen12_emit_flush_xcs()
397 rq->engine->class == COPY_ENGINE_CLASS) in gen12_emit_flush_xcs()
401 *cs++ = cmd; in gen12_emit_flush_xcs()
402 *cs++ = LRC_PPHWSP_SCRATCH_ADDR; in gen12_emit_flush_xcs()
403 *cs++ = 0; /* upper addr */ in gen12_emit_flush_xcs()
404 *cs++ = 0; /* value */ in gen12_emit_flush_xcs()
406 cs = gen12_emit_aux_table_inv(rq->engine, cs); in gen12_emit_flush_xcs()
409 *cs++ = preparser_disable(false); in gen12_emit_flush_xcs()
411 intel_ring_advance(rq, cs); in gen12_emit_flush_xcs()
413 return 0; in gen12_emit_flush_xcs()
418 return (i915_ggtt_offset(engine->status_page.vma) + in preempt_address()
427 tl = rcu_dereference_protected(rq->timeline, in hwsp_offset()
431 return page_mask_bits(tl->hwsp_offset) + offset_in_page(rq->hwsp_seqno); in hwsp_offset()
436 u32 *cs; in gen8_emit_init_breadcrumb() local
439 if (!i915_request_timeline(rq)->has_initial_breadcrumb) in gen8_emit_init_breadcrumb()
440 return 0; in gen8_emit_init_breadcrumb()
442 cs = intel_ring_begin(rq, 6); in gen8_emit_init_breadcrumb()
443 if (IS_ERR(cs)) in gen8_emit_init_breadcrumb()
444 return PTR_ERR(cs); in gen8_emit_init_breadcrumb()
446 *cs++ = MI_STORE_DWORD_IMM_GEN4 | MI_USE_GGTT; in gen8_emit_init_breadcrumb()
447 *cs++ = hwsp_offset(rq); in gen8_emit_init_breadcrumb()
448 *cs++ = 0; in gen8_emit_init_breadcrumb()
449 *cs++ = rq->fence.seqno - 1; in gen8_emit_init_breadcrumb()
459 * or spinning on a kernel semaphore (or earlier). For no-preemption in gen8_emit_init_breadcrumb()
468 *cs++ = MI_NOOP; in gen8_emit_init_breadcrumb()
469 *cs++ = MI_ARB_CHECK; in gen8_emit_init_breadcrumb()
471 intel_ring_advance(rq, cs); in gen8_emit_init_breadcrumb()
474 rq->infix = intel_ring_offset(rq, cs); in gen8_emit_init_breadcrumb()
476 __set_bit(I915_FENCE_FLAG_INITIAL_BREADCRUMB, &rq->fence.flags); in gen8_emit_init_breadcrumb()
478 return 0; in gen8_emit_init_breadcrumb()
486 struct intel_context *ce = rq->context; in __xehp_emit_bb_start()
488 u32 *cs; in __xehp_emit_bb_start() local
490 GEM_BUG_ON(!ce->wa_bb_page); in __xehp_emit_bb_start()
492 cs = intel_ring_begin(rq, 12); in __xehp_emit_bb_start()
493 if (IS_ERR(cs)) in __xehp_emit_bb_start()
494 return PTR_ERR(cs); in __xehp_emit_bb_start()
496 *cs++ = MI_ARB_ON_OFF | arb; in __xehp_emit_bb_start()
498 *cs++ = MI_LOAD_REGISTER_MEM_GEN8 | in __xehp_emit_bb_start()
501 *cs++ = i915_mmio_reg_offset(RING_PREDICATE_RESULT(0)); in __xehp_emit_bb_start()
502 *cs++ = wa_offset + DG2_PREDICATE_RESULT_WA; in __xehp_emit_bb_start()
503 *cs++ = 0; in __xehp_emit_bb_start()
505 *cs++ = MI_BATCH_BUFFER_START_GEN8 | in __xehp_emit_bb_start()
506 (flags & I915_DISPATCH_SECURE ? 0 : BIT(8)); in __xehp_emit_bb_start()
507 *cs++ = lower_32_bits(offset); in __xehp_emit_bb_start()
508 *cs++ = upper_32_bits(offset); in __xehp_emit_bb_start()
511 *cs++ = MI_BATCH_BUFFER_START_GEN8; in __xehp_emit_bb_start()
512 *cs++ = wa_offset + DG2_PREDICATE_RESULT_BB; in __xehp_emit_bb_start()
513 *cs++ = 0; in __xehp_emit_bb_start()
515 *cs++ = MI_ARB_ON_OFF | MI_ARB_DISABLE; in __xehp_emit_bb_start()
517 intel_ring_advance(rq, cs); in __xehp_emit_bb_start()
519 return 0; in __xehp_emit_bb_start()
540 u32 *cs; in gen8_emit_bb_start_noarb() local
542 cs = intel_ring_begin(rq, 4); in gen8_emit_bb_start_noarb()
543 if (IS_ERR(cs)) in gen8_emit_bb_start_noarb()
544 return PTR_ERR(cs); in gen8_emit_bb_start_noarb()
556 * re-enabled before we close the request in gen8_emit_bb_start_noarb()
557 * (engine->emit_fini_breadcrumb). in gen8_emit_bb_start_noarb()
559 *cs++ = MI_ARB_ON_OFF | MI_ARB_DISABLE; in gen8_emit_bb_start_noarb()
562 *cs++ = MI_BATCH_BUFFER_START_GEN8 | in gen8_emit_bb_start_noarb()
563 (flags & I915_DISPATCH_SECURE ? 0 : BIT(8)); in gen8_emit_bb_start_noarb()
564 *cs++ = lower_32_bits(offset); in gen8_emit_bb_start_noarb()
565 *cs++ = upper_32_bits(offset); in gen8_emit_bb_start_noarb()
567 intel_ring_advance(rq, cs); in gen8_emit_bb_start_noarb()
569 return 0; in gen8_emit_bb_start_noarb()
576 u32 *cs; in gen8_emit_bb_start() local
581 cs = intel_ring_begin(rq, 6); in gen8_emit_bb_start()
582 if (IS_ERR(cs)) in gen8_emit_bb_start()
583 return PTR_ERR(cs); in gen8_emit_bb_start()
585 *cs++ = MI_ARB_ON_OFF | MI_ARB_ENABLE; in gen8_emit_bb_start()
587 *cs++ = MI_BATCH_BUFFER_START_GEN8 | in gen8_emit_bb_start()
588 (flags & I915_DISPATCH_SECURE ? 0 : BIT(8)); in gen8_emit_bb_start()
589 *cs++ = lower_32_bits(offset); in gen8_emit_bb_start()
590 *cs++ = upper_32_bits(offset); in gen8_emit_bb_start()
592 *cs++ = MI_ARB_ON_OFF | MI_ARB_DISABLE; in gen8_emit_bb_start()
593 *cs++ = MI_NOOP; in gen8_emit_bb_start()
595 intel_ring_advance(rq, cs); in gen8_emit_bb_start()
597 return 0; in gen8_emit_bb_start()
602 struct intel_ring *ring __maybe_unused = rq->ring; in assert_request_valid()
605 GEM_BUG_ON(intel_ring_direction(ring, rq->wa_tail, rq->head) <= 0); in assert_request_valid()
613 static u32 *gen8_emit_wa_tail(struct i915_request *rq, u32 *cs) in gen8_emit_wa_tail() argument
615 /* Ensure there's always at least one preemption point per-request. */ in gen8_emit_wa_tail()
616 *cs++ = MI_ARB_CHECK; in gen8_emit_wa_tail()
617 *cs++ = MI_NOOP; in gen8_emit_wa_tail()
618 rq->wa_tail = intel_ring_offset(rq, cs); in gen8_emit_wa_tail()
623 return cs; in gen8_emit_wa_tail()
626 static u32 *emit_preempt_busywait(struct i915_request *rq, u32 *cs) in emit_preempt_busywait() argument
628 *cs++ = MI_ARB_CHECK; /* trigger IDLE->ACTIVE first */ in emit_preempt_busywait()
629 *cs++ = MI_SEMAPHORE_WAIT | in emit_preempt_busywait()
633 *cs++ = 0; in emit_preempt_busywait()
634 *cs++ = preempt_address(rq->engine); in emit_preempt_busywait()
635 *cs++ = 0; in emit_preempt_busywait()
636 *cs++ = MI_NOOP; in emit_preempt_busywait()
638 return cs; in emit_preempt_busywait()
642 gen8_emit_fini_breadcrumb_tail(struct i915_request *rq, u32 *cs) in gen8_emit_fini_breadcrumb_tail() argument
644 *cs++ = MI_USER_INTERRUPT; in gen8_emit_fini_breadcrumb_tail()
646 *cs++ = MI_ARB_ON_OFF | MI_ARB_ENABLE; in gen8_emit_fini_breadcrumb_tail()
647 if (intel_engine_has_semaphores(rq->engine) && in gen8_emit_fini_breadcrumb_tail()
648 !intel_uc_uses_guc_submission(&rq->engine->gt->uc)) in gen8_emit_fini_breadcrumb_tail()
649 cs = emit_preempt_busywait(rq, cs); in gen8_emit_fini_breadcrumb_tail()
651 rq->tail = intel_ring_offset(rq, cs); in gen8_emit_fini_breadcrumb_tail()
652 assert_ring_tail_valid(rq->ring, rq->tail); in gen8_emit_fini_breadcrumb_tail()
654 return gen8_emit_wa_tail(rq, cs); in gen8_emit_fini_breadcrumb_tail()
657 static u32 *emit_xcs_breadcrumb(struct i915_request *rq, u32 *cs) in emit_xcs_breadcrumb() argument
659 return gen8_emit_ggtt_write(cs, rq->fence.seqno, hwsp_offset(rq), 0); in emit_xcs_breadcrumb()
662 u32 *gen8_emit_fini_breadcrumb_xcs(struct i915_request *rq, u32 *cs) in gen8_emit_fini_breadcrumb_xcs() argument
664 return gen8_emit_fini_breadcrumb_tail(rq, emit_xcs_breadcrumb(rq, cs)); in gen8_emit_fini_breadcrumb_xcs()
667 u32 *gen8_emit_fini_breadcrumb_rcs(struct i915_request *rq, u32 *cs) in gen8_emit_fini_breadcrumb_rcs() argument
669 cs = gen8_emit_pipe_control(cs, in gen8_emit_fini_breadcrumb_rcs()
675 0); in gen8_emit_fini_breadcrumb_rcs()
678 cs = gen8_emit_ggtt_write_rcs(cs, in gen8_emit_fini_breadcrumb_rcs()
679 rq->fence.seqno, in gen8_emit_fini_breadcrumb_rcs()
684 return gen8_emit_fini_breadcrumb_tail(rq, cs); in gen8_emit_fini_breadcrumb_rcs()
687 u32 *gen11_emit_fini_breadcrumb_rcs(struct i915_request *rq, u32 *cs) in gen11_emit_fini_breadcrumb_rcs() argument
689 cs = gen8_emit_pipe_control(cs, in gen11_emit_fini_breadcrumb_rcs()
696 0); in gen11_emit_fini_breadcrumb_rcs()
699 cs = gen8_emit_ggtt_write_rcs(cs, in gen11_emit_fini_breadcrumb_rcs()
700 rq->fence.seqno, in gen11_emit_fini_breadcrumb_rcs()
705 return gen8_emit_fini_breadcrumb_tail(rq, cs); in gen11_emit_fini_breadcrumb_rcs()
709 * Note that the CS instruction pre-parser will not stall on the breadcrumb
710 * flush and will continue pre-fetching the instructions after it before the
711 * memory sync is completed. On pre-gen12 HW, the pre-parser will stop at
712 * BB_START/END instructions, so, even though we might pre-fetch the pre-amble
715 * However, on gen12+ the parser can pre-fetch across the BB_START/END commands,
717 * the same intel_context, we might pre-fetch and then execute the pre-update
718 * instruction. To avoid this, the users of self-modifying code should either
721 * the in-kernel use-cases we've opted to use a separate context, see
723 * All the above applies only to the instructions themselves. Non-inline data
724 * used by the instructions is not pre-fetched.
727 static u32 *gen12_emit_preempt_busywait(struct i915_request *rq, u32 *cs) in gen12_emit_preempt_busywait() argument
729 *cs++ = MI_ARB_CHECK; /* trigger IDLE->ACTIVE first */ in gen12_emit_preempt_busywait()
730 *cs++ = MI_SEMAPHORE_WAIT_TOKEN | in gen12_emit_preempt_busywait()
734 *cs++ = 0; in gen12_emit_preempt_busywait()
735 *cs++ = preempt_address(rq->engine); in gen12_emit_preempt_busywait()
736 *cs++ = 0; in gen12_emit_preempt_busywait()
737 *cs++ = 0; in gen12_emit_preempt_busywait()
739 return cs; in gen12_emit_preempt_busywait()
745 #define HOLD_SWITCHOUT_SEMAPHORE_PPHWSP_OFFSET 0x540
748 return i915_ggtt_offset(rq->context->state) + in hold_switchout_semaphore_offset()
755 static u32 *hold_switchout_emit_wa_busywait(struct i915_request *rq, u32 *cs) in hold_switchout_emit_wa_busywait() argument
759 *cs++ = MI_ATOMIC_INLINE | MI_ATOMIC_GLOBAL_GTT | MI_ATOMIC_CS_STALL | in hold_switchout_emit_wa_busywait()
761 *cs++ = hold_switchout_semaphore_offset(rq); in hold_switchout_emit_wa_busywait()
762 *cs++ = 0; in hold_switchout_emit_wa_busywait()
763 *cs++ = 1; in hold_switchout_emit_wa_busywait()
769 for (i = 0; i < 8; ++i) in hold_switchout_emit_wa_busywait()
770 *cs++ = 0; in hold_switchout_emit_wa_busywait()
772 *cs++ = MI_SEMAPHORE_WAIT | in hold_switchout_emit_wa_busywait()
776 *cs++ = 0; in hold_switchout_emit_wa_busywait()
777 *cs++ = hold_switchout_semaphore_offset(rq); in hold_switchout_emit_wa_busywait()
778 *cs++ = 0; in hold_switchout_emit_wa_busywait()
780 return cs; in hold_switchout_emit_wa_busywait()
784 gen12_emit_fini_breadcrumb_tail(struct i915_request *rq, u32 *cs) in gen12_emit_fini_breadcrumb_tail() argument
786 *cs++ = MI_USER_INTERRUPT; in gen12_emit_fini_breadcrumb_tail()
788 *cs++ = MI_ARB_ON_OFF | MI_ARB_ENABLE; in gen12_emit_fini_breadcrumb_tail()
789 if (intel_engine_has_semaphores(rq->engine) && in gen12_emit_fini_breadcrumb_tail()
790 !intel_uc_uses_guc_submission(&rq->engine->gt->uc)) in gen12_emit_fini_breadcrumb_tail()
791 cs = gen12_emit_preempt_busywait(rq, cs); in gen12_emit_fini_breadcrumb_tail()
796 if (intel_engine_uses_wa_hold_switchout(rq->engine)) in gen12_emit_fini_breadcrumb_tail()
797 cs = hold_switchout_emit_wa_busywait(rq, cs); in gen12_emit_fini_breadcrumb_tail()
799 rq->tail = intel_ring_offset(rq, cs); in gen12_emit_fini_breadcrumb_tail()
800 assert_ring_tail_valid(rq->ring, rq->tail); in gen12_emit_fini_breadcrumb_tail()
802 return gen8_emit_wa_tail(rq, cs); in gen12_emit_fini_breadcrumb_tail()
805 u32 *gen12_emit_fini_breadcrumb_xcs(struct i915_request *rq, u32 *cs) in gen12_emit_fini_breadcrumb_xcs() argument
807 /* XXX Stalling flush before seqno write; post-sync not */ in gen12_emit_fini_breadcrumb_xcs()
808 cs = emit_xcs_breadcrumb(rq, __gen8_emit_flush_dw(cs, 0, 0, 0)); in gen12_emit_fini_breadcrumb_xcs()
809 return gen12_emit_fini_breadcrumb_tail(rq, cs); in gen12_emit_fini_breadcrumb_xcs()
812 u32 *gen12_emit_fini_breadcrumb_rcs(struct i915_request *rq, u32 *cs) in gen12_emit_fini_breadcrumb_rcs() argument
814 struct drm_i915_private *i915 = rq->i915; in gen12_emit_fini_breadcrumb_rcs()
815 struct intel_gt *gt = rq->engine->gt; in gen12_emit_fini_breadcrumb_rcs()
824 if (GRAPHICS_VER_FULL(rq->i915) < IP_VER(12, 70)) in gen12_emit_fini_breadcrumb_rcs()
830 cs = gen12_emit_pipe_control(cs, 0, in gen12_emit_fini_breadcrumb_rcs()
831 PIPE_CONTROL_DEPTH_CACHE_FLUSH, 0); in gen12_emit_fini_breadcrumb_rcs()
837 if (!HAS_3D_PIPELINE(rq->i915)) in gen12_emit_fini_breadcrumb_rcs()
839 else if (rq->engine->class == COMPUTE_CLASS) in gen12_emit_fini_breadcrumb_rcs()
842 cs = gen12_emit_pipe_control(cs, PIPE_CONTROL0_HDC_PIPELINE_FLUSH, flags, 0); in gen12_emit_fini_breadcrumb_rcs()
845 cs = gen12_emit_ggtt_write_rcs(cs, in gen12_emit_fini_breadcrumb_rcs()
846 rq->fence.seqno, in gen12_emit_fini_breadcrumb_rcs()
848 0, in gen12_emit_fini_breadcrumb_rcs()
852 return gen12_emit_fini_breadcrumb_tail(rq, cs); in gen12_emit_fini_breadcrumb_rcs()