16 	u32 *cs, flags = 0;  in gen8_emit_flush_rcs()  local
58 	cs = intel_ring_begin(rq, len);  in gen8_emit_flush_rcs()
59 	if (IS_ERR(cs))  in gen8_emit_flush_rcs()
60 		return PTR_ERR(cs);  in gen8_emit_flush_rcs()
63 		cs = gen8_emit_pipe_control(cs, 0, 0);  in gen8_emit_flush_rcs()
66 		cs = gen8_emit_pipe_control(cs, PIPE_CONTROL_DC_FLUSH_ENABLE,  in gen8_emit_flush_rcs()
69 	cs = gen8_emit_pipe_control(cs, flags, LRC_PPHWSP_SCRATCH_ADDR);  in gen8_emit_flush_rcs()
72 		cs = gen8_emit_pipe_control(cs, PIPE_CONTROL_CS_STALL, 0);  in gen8_emit_flush_rcs()
74 	intel_ring_advance(rq, cs);  in gen8_emit_flush_rcs()
81 	u32 cmd, *cs;  in gen8_emit_flush_xcs()  local
83 	cs = intel_ring_begin(rq, 4);  in gen8_emit_flush_xcs()
84 	if (IS_ERR(cs))  in gen8_emit_flush_xcs()
85 		return PTR_ERR(cs);  in gen8_emit_flush_xcs()
103 	*cs++ = cmd;  in gen8_emit_flush_xcs()
104 	*cs++ = LRC_PPHWSP_SCRATCH_ADDR;  in gen8_emit_flush_xcs()
105 	*cs++ = 0; /* upper addr */  in gen8_emit_flush_xcs()
106 	*cs++ = 0; /* value */  in gen8_emit_flush_xcs()
107 	intel_ring_advance(rq, cs);  in gen8_emit_flush_xcs()
115 		u32 *cs;  in gen11_emit_flush_rcs()  local
128 		cs = intel_ring_begin(rq, 6);  in gen11_emit_flush_rcs()
129 		if (IS_ERR(cs))  in gen11_emit_flush_rcs()
130 			return PTR_ERR(cs);  in gen11_emit_flush_rcs()
132 		cs = gen8_emit_pipe_control(cs, flags, LRC_PPHWSP_SCRATCH_ADDR);  in gen11_emit_flush_rcs()
133 		intel_ring_advance(rq, cs);  in gen11_emit_flush_rcs()
137 		u32 *cs;  in gen11_emit_flush_rcs()  local
152 		cs = intel_ring_begin(rq, 6);  in gen11_emit_flush_rcs()
153 		if (IS_ERR(cs))  in gen11_emit_flush_rcs()
154 			return PTR_ERR(cs);  in gen11_emit_flush_rcs()
156 		cs = gen8_emit_pipe_control(cs, flags, LRC_PPHWSP_SCRATCH_ADDR);  in gen11_emit_flush_rcs()
157 		intel_ring_advance(rq, cs);  in gen11_emit_flush_rcs()
199 u32 *gen12_emit_aux_table_inv(struct intel_engine_cs *engine, u32 *cs)  in gen12_emit_aux_table_inv()  argument
205 		return cs;  in gen12_emit_aux_table_inv()
207 	*cs++ = MI_LOAD_REGISTER_IMM(1) | MI_LRI_MMIO_REMAP_EN;  in gen12_emit_aux_table_inv()
208 	*cs++ = i915_mmio_reg_offset(inv_reg) + gsi_offset;  in gen12_emit_aux_table_inv()
209 	*cs++ = AUX_INV;  in gen12_emit_aux_table_inv()
211 	*cs++ = MI_SEMAPHORE_WAIT_TOKEN |  in gen12_emit_aux_table_inv()
215 	*cs++ = 0;  in gen12_emit_aux_table_inv()
216 	*cs++ = i915_mmio_reg_offset(inv_reg) + gsi_offset;  in gen12_emit_aux_table_inv()
217 	*cs++ = 0;  in gen12_emit_aux_table_inv()
218 	*cs++ = 0;  in gen12_emit_aux_table_inv()
220 	return cs;  in gen12_emit_aux_table_inv()
228 		u32 *cs;  in mtl_dummy_pipe_control()  local
231 		cs = intel_ring_begin(rq, 6);  in mtl_dummy_pipe_control()
232 		if (IS_ERR(cs))  in mtl_dummy_pipe_control()
233 			return PTR_ERR(cs);  in mtl_dummy_pipe_control()
234 		cs = gen12_emit_pipe_control(cs,  in mtl_dummy_pipe_control()
238 		intel_ring_advance(rq, cs);  in mtl_dummy_pipe_control()
256 		u32 *cs;  in gen12_emit_flush_rcs()  local
300 		cs = intel_ring_begin(rq, 6);  in gen12_emit_flush_rcs()
301 		if (IS_ERR(cs))  in gen12_emit_flush_rcs()
302 			return PTR_ERR(cs);  in gen12_emit_flush_rcs()
304 		cs = gen12_emit_pipe_control(cs, bit_group_0, bit_group_1,  in gen12_emit_flush_rcs()
306 		intel_ring_advance(rq, cs);  in gen12_emit_flush_rcs()
311 		u32 *cs, count;  in gen12_emit_flush_rcs()  local
340 		cs = intel_ring_begin(rq, count);  in gen12_emit_flush_rcs()
341 		if (IS_ERR(cs))  in gen12_emit_flush_rcs()
342 			return PTR_ERR(cs);  in gen12_emit_flush_rcs()
349 		*cs++ = preparser_disable(true);  in gen12_emit_flush_rcs()
351 		cs = gen8_emit_pipe_control(cs, flags, LRC_PPHWSP_SCRATCH_ADDR);  in gen12_emit_flush_rcs()
353 		cs = gen12_emit_aux_table_inv(engine, cs);  in gen12_emit_flush_rcs()
355 		*cs++ = preparser_disable(false);  in gen12_emit_flush_rcs()
356 		intel_ring_advance(rq, cs);  in gen12_emit_flush_rcs()
365 	u32 *cs;  in gen12_emit_flush_xcs()  local
374 	cs = intel_ring_begin(rq, cmd);  in gen12_emit_flush_xcs()
375 	if (IS_ERR(cs))  in gen12_emit_flush_xcs()
376 		return PTR_ERR(cs);  in gen12_emit_flush_xcs()
379 		*cs++ = preparser_disable(true);  in gen12_emit_flush_xcs()
401 	*cs++ = cmd;  in gen12_emit_flush_xcs()
402 	*cs++ = LRC_PPHWSP_SCRATCH_ADDR;  in gen12_emit_flush_xcs()
403 	*cs++ = 0; /* upper addr */  in gen12_emit_flush_xcs()
404 	*cs++ = 0; /* value */  in gen12_emit_flush_xcs()
406 	cs = gen12_emit_aux_table_inv(rq->engine, cs);  in gen12_emit_flush_xcs()
409 		*cs++ = preparser_disable(false);  in gen12_emit_flush_xcs()
411 	intel_ring_advance(rq, cs);  in gen12_emit_flush_xcs()
436 	u32 *cs;  in gen8_emit_init_breadcrumb()  local
442 	cs = intel_ring_begin(rq, 6);  in gen8_emit_init_breadcrumb()
443 	if (IS_ERR(cs))  in gen8_emit_init_breadcrumb()
444 		return PTR_ERR(cs);  in gen8_emit_init_breadcrumb()
446 	*cs++ = MI_STORE_DWORD_IMM_GEN4 | MI_USE_GGTT;  in gen8_emit_init_breadcrumb()
447 	*cs++ = hwsp_offset(rq);  in gen8_emit_init_breadcrumb()
448 	*cs++ = 0;  in gen8_emit_init_breadcrumb()
449 	*cs++ = rq->fence.seqno - 1;  in gen8_emit_init_breadcrumb()
468 	*cs++ = MI_NOOP;  in gen8_emit_init_breadcrumb()
469 	*cs++ = MI_ARB_CHECK;  in gen8_emit_init_breadcrumb()
471 	intel_ring_advance(rq, cs);  in gen8_emit_init_breadcrumb()
474 	rq->infix = intel_ring_offset(rq, cs);  in gen8_emit_init_breadcrumb()
488 	u32 *cs;  in __xehp_emit_bb_start()  local
492 	cs = intel_ring_begin(rq, 12);  in __xehp_emit_bb_start()
493 	if (IS_ERR(cs))  in __xehp_emit_bb_start()
494 		return PTR_ERR(cs);  in __xehp_emit_bb_start()
496 	*cs++ = MI_ARB_ON_OFF | arb;  in __xehp_emit_bb_start()
498 	*cs++ = MI_LOAD_REGISTER_MEM_GEN8 |  in __xehp_emit_bb_start()
501 	*cs++ = i915_mmio_reg_offset(RING_PREDICATE_RESULT(0));  in __xehp_emit_bb_start()
502 	*cs++ = wa_offset + DG2_PREDICATE_RESULT_WA;  in __xehp_emit_bb_start()
503 	*cs++ = 0;  in __xehp_emit_bb_start()
505 	*cs++ = MI_BATCH_BUFFER_START_GEN8 |  in __xehp_emit_bb_start()
507 	*cs++ = lower_32_bits(offset);  in __xehp_emit_bb_start()
508 	*cs++ = upper_32_bits(offset);  in __xehp_emit_bb_start()
511 	*cs++ = MI_BATCH_BUFFER_START_GEN8;  in __xehp_emit_bb_start()
512 	*cs++ = wa_offset + DG2_PREDICATE_RESULT_BB;  in __xehp_emit_bb_start()
513 	*cs++ = 0;  in __xehp_emit_bb_start()
515 	*cs++ = MI_ARB_ON_OFF | MI_ARB_DISABLE;  in __xehp_emit_bb_start()
517 	intel_ring_advance(rq, cs);  in __xehp_emit_bb_start()
540 	u32 *cs;  in gen8_emit_bb_start_noarb()  local
542 	cs = intel_ring_begin(rq, 4);  in gen8_emit_bb_start_noarb()
543 	if (IS_ERR(cs))  in gen8_emit_bb_start_noarb()
544 		return PTR_ERR(cs);  in gen8_emit_bb_start_noarb()
559 	*cs++ = MI_ARB_ON_OFF | MI_ARB_DISABLE;  in gen8_emit_bb_start_noarb()
562 	*cs++ = MI_BATCH_BUFFER_START_GEN8 |  in gen8_emit_bb_start_noarb()
564 	*cs++ = lower_32_bits(offset);  in gen8_emit_bb_start_noarb()
565 	*cs++ = upper_32_bits(offset);  in gen8_emit_bb_start_noarb()
567 	intel_ring_advance(rq, cs);  in gen8_emit_bb_start_noarb()
576 	u32 *cs;  in gen8_emit_bb_start()  local
581 	cs = intel_ring_begin(rq, 6);  in gen8_emit_bb_start()
582 	if (IS_ERR(cs))  in gen8_emit_bb_start()
583 		return PTR_ERR(cs);  in gen8_emit_bb_start()
585 	*cs++ = MI_ARB_ON_OFF | MI_ARB_ENABLE;  in gen8_emit_bb_start()
587 	*cs++ = MI_BATCH_BUFFER_START_GEN8 |  in gen8_emit_bb_start()
589 	*cs++ = lower_32_bits(offset);  in gen8_emit_bb_start()
590 	*cs++ = upper_32_bits(offset);  in gen8_emit_bb_start()
592 	*cs++ = MI_ARB_ON_OFF | MI_ARB_DISABLE;  in gen8_emit_bb_start()
593 	*cs++ = MI_NOOP;  in gen8_emit_bb_start()
595 	intel_ring_advance(rq, cs);  in gen8_emit_bb_start()
613 static u32 *gen8_emit_wa_tail(struct i915_request *rq, u32 *cs)  in gen8_emit_wa_tail()  argument
616 	*cs++ = MI_ARB_CHECK;  in gen8_emit_wa_tail()
617 	*cs++ = MI_NOOP;  in gen8_emit_wa_tail()
618 	rq->wa_tail = intel_ring_offset(rq, cs);  in gen8_emit_wa_tail()
623 	return cs;  in gen8_emit_wa_tail()
626 static u32 *emit_preempt_busywait(struct i915_request *rq, u32 *cs)  in emit_preempt_busywait()  argument
628 	*cs++ = MI_ARB_CHECK; /* trigger IDLE->ACTIVE first */  in emit_preempt_busywait()
629 	*cs++ = MI_SEMAPHORE_WAIT |  in emit_preempt_busywait()
633 	*cs++ = 0;  in emit_preempt_busywait()
634 	*cs++ = preempt_address(rq->engine);  in emit_preempt_busywait()
635 	*cs++ = 0;  in emit_preempt_busywait()
636 	*cs++ = MI_NOOP;  in emit_preempt_busywait()
638 	return cs;  in emit_preempt_busywait()
642 gen8_emit_fini_breadcrumb_tail(struct i915_request *rq, u32 *cs)  in gen8_emit_fini_breadcrumb_tail()  argument
644 	*cs++ = MI_USER_INTERRUPT;  in gen8_emit_fini_breadcrumb_tail()
646 	*cs++ = MI_ARB_ON_OFF | MI_ARB_ENABLE;  in gen8_emit_fini_breadcrumb_tail()
649 		cs = emit_preempt_busywait(rq, cs);  in gen8_emit_fini_breadcrumb_tail()
651 	rq->tail = intel_ring_offset(rq, cs);  in gen8_emit_fini_breadcrumb_tail()
654 	return gen8_emit_wa_tail(rq, cs);  in gen8_emit_fini_breadcrumb_tail()
657 static u32 *emit_xcs_breadcrumb(struct i915_request *rq, u32 *cs)  in emit_xcs_breadcrumb()  argument
659 	return gen8_emit_ggtt_write(cs, rq->fence.seqno, hwsp_offset(rq), 0);  in emit_xcs_breadcrumb()
662 u32 *gen8_emit_fini_breadcrumb_xcs(struct i915_request *rq, u32 *cs)  in gen8_emit_fini_breadcrumb_xcs()  argument
664 	return gen8_emit_fini_breadcrumb_tail(rq, emit_xcs_breadcrumb(rq, cs));  in gen8_emit_fini_breadcrumb_xcs()
667 u32 *gen8_emit_fini_breadcrumb_rcs(struct i915_request *rq, u32 *cs)  in gen8_emit_fini_breadcrumb_rcs()  argument
669 	cs = gen8_emit_pipe_control(cs,  in gen8_emit_fini_breadcrumb_rcs()
678 	cs = gen8_emit_ggtt_write_rcs(cs,  in gen8_emit_fini_breadcrumb_rcs()
684 	return gen8_emit_fini_breadcrumb_tail(rq, cs);  in gen8_emit_fini_breadcrumb_rcs()
687 u32 *gen11_emit_fini_breadcrumb_rcs(struct i915_request *rq, u32 *cs)  in gen11_emit_fini_breadcrumb_rcs()  argument
689 	cs = gen8_emit_pipe_control(cs,  in gen11_emit_fini_breadcrumb_rcs()
699 	cs = gen8_emit_ggtt_write_rcs(cs,  in gen11_emit_fini_breadcrumb_rcs()
705 	return gen8_emit_fini_breadcrumb_tail(rq, cs);  in gen11_emit_fini_breadcrumb_rcs()
709  * Note that the CS instruction pre-parser will not stall on the breadcrumb
727 static u32 *gen12_emit_preempt_busywait(struct i915_request *rq, u32 *cs)  in gen12_emit_preempt_busywait()  argument
729 	*cs++ = MI_ARB_CHECK; /* trigger IDLE->ACTIVE first */  in gen12_emit_preempt_busywait()
730 	*cs++ = MI_SEMAPHORE_WAIT_TOKEN |  in gen12_emit_preempt_busywait()
734 	*cs++ = 0;  in gen12_emit_preempt_busywait()
735 	*cs++ = preempt_address(rq->engine);  in gen12_emit_preempt_busywait()
736 	*cs++ = 0;  in gen12_emit_preempt_busywait()
737 	*cs++ = 0;  in gen12_emit_preempt_busywait()
739 	return cs;  in gen12_emit_preempt_busywait()
755 static u32 *hold_switchout_emit_wa_busywait(struct i915_request *rq, u32 *cs)  in hold_switchout_emit_wa_busywait()  argument
759 	*cs++ = MI_ATOMIC_INLINE | MI_ATOMIC_GLOBAL_GTT | MI_ATOMIC_CS_STALL |  in hold_switchout_emit_wa_busywait()
761 	*cs++ = hold_switchout_semaphore_offset(rq);  in hold_switchout_emit_wa_busywait()
762 	*cs++ = 0;  in hold_switchout_emit_wa_busywait()
763 	*cs++ = 1;  in hold_switchout_emit_wa_busywait()
770 		*cs++ = 0;  in hold_switchout_emit_wa_busywait()
772 	*cs++ = MI_SEMAPHORE_WAIT |  in hold_switchout_emit_wa_busywait()
776 	*cs++ = 0;  in hold_switchout_emit_wa_busywait()
777 	*cs++ = hold_switchout_semaphore_offset(rq);  in hold_switchout_emit_wa_busywait()
778 	*cs++ = 0;  in hold_switchout_emit_wa_busywait()
780 	return cs;  in hold_switchout_emit_wa_busywait()
784 gen12_emit_fini_breadcrumb_tail(struct i915_request *rq, u32 *cs)  in gen12_emit_fini_breadcrumb_tail()  argument
786 	*cs++ = MI_USER_INTERRUPT;  in gen12_emit_fini_breadcrumb_tail()
788 	*cs++ = MI_ARB_ON_OFF | MI_ARB_ENABLE;  in gen12_emit_fini_breadcrumb_tail()
791 		cs = gen12_emit_preempt_busywait(rq, cs);  in gen12_emit_fini_breadcrumb_tail()
797 		cs = hold_switchout_emit_wa_busywait(rq, cs);  in gen12_emit_fini_breadcrumb_tail()
799 	rq->tail = intel_ring_offset(rq, cs);  in gen12_emit_fini_breadcrumb_tail()
802 	return gen8_emit_wa_tail(rq, cs);  in gen12_emit_fini_breadcrumb_tail()
805 u32 *gen12_emit_fini_breadcrumb_xcs(struct i915_request *rq, u32 *cs)  in gen12_emit_fini_breadcrumb_xcs()  argument
808 	cs = emit_xcs_breadcrumb(rq, __gen8_emit_flush_dw(cs, 0, 0, 0));  in gen12_emit_fini_breadcrumb_xcs()
809 	return gen12_emit_fini_breadcrumb_tail(rq, cs);  in gen12_emit_fini_breadcrumb_xcs()
812 u32 *gen12_emit_fini_breadcrumb_rcs(struct i915_request *rq, u32 *cs)  in gen12_emit_fini_breadcrumb_rcs()  argument
830 		cs = gen12_emit_pipe_control(cs, 0,  in gen12_emit_fini_breadcrumb_rcs()
842 	cs = gen12_emit_pipe_control(cs, PIPE_CONTROL0_HDC_PIPELINE_FLUSH, flags, 0);  in gen12_emit_fini_breadcrumb_rcs()
845 	cs = gen12_emit_ggtt_write_rcs(cs,  in gen12_emit_fini_breadcrumb_rcs()
852 	return gen12_emit_fini_breadcrumb_tail(rq, cs);  in gen12_emit_fini_breadcrumb_rcs()