54 			      u32 *dw, int i)  in emit_aux_table_inv()  argument
56 	dw[i++] = MI_LOAD_REGISTER_IMM | MI_LRI_NUM_REGS(1) | MI_LRI_MMIO_REMAP_EN;  in emit_aux_table_inv()
57 	dw[i++] = reg.addr + gt->mmio.adj_offset;  in emit_aux_table_inv()
58 	dw[i++] = AUX_INV;  in emit_aux_table_inv()
59 	dw[i++] = MI_NOOP;  in emit_aux_table_inv()
61 	return i;  in emit_aux_table_inv()
64 static int emit_user_interrupt(u32 *dw, int i)  in emit_user_interrupt()  argument
66 	dw[i++] = MI_USER_INTERRUPT;  in emit_user_interrupt()
67 	dw[i++] = MI_ARB_ON_OFF | MI_ARB_ENABLE;  in emit_user_interrupt()
68 	dw[i++] = MI_ARB_CHECK;  in emit_user_interrupt()
70 	return i;  in emit_user_interrupt()
73 static int emit_store_imm_ggtt(u32 addr, u32 value, u32 *dw, int i)  in emit_store_imm_ggtt()  argument
75 	dw[i++] = MI_STORE_DATA_IMM | MI_SDI_GGTT | MI_SDI_NUM_DW(1);  in emit_store_imm_ggtt()
76 	dw[i++] = addr;  in emit_store_imm_ggtt()
77 	dw[i++] = 0;  in emit_store_imm_ggtt()
78 	dw[i++] = value;  in emit_store_imm_ggtt()
80 	return i;  in emit_store_imm_ggtt()
83 static int emit_flush_dw(u32 *dw, int i)  in emit_flush_dw()  argument
85 	dw[i++] = MI_FLUSH_DW | MI_FLUSH_IMM_DW;  in emit_flush_dw()
86 	dw[i++] = 0;  in emit_flush_dw()
87 	dw[i++] = 0;  in emit_flush_dw()
88 	dw[i++] = 0;  in emit_flush_dw()
90 	return i;  in emit_flush_dw()
94 			       u32 *dw, int i)  in emit_flush_imm_ggtt()  argument
96 	dw[i++] = MI_FLUSH_DW | MI_FLUSH_DW_OP_STOREDW | MI_FLUSH_IMM_DW |  in emit_flush_imm_ggtt()
98 	dw[i++] = addr | MI_FLUSH_DW_USE_GTT;  in emit_flush_imm_ggtt()
99 	dw[i++] = 0;  in emit_flush_imm_ggtt()
100 	dw[i++] = value;  in emit_flush_imm_ggtt()
102 	return i;  in emit_flush_imm_ggtt()
105 static int emit_bb_start(u64 batch_addr, u32 ppgtt_flag, u32 *dw, int i)  in emit_bb_start()  argument
107 	dw[i++] = MI_BATCH_BUFFER_START | ppgtt_flag | XE_INSTR_NUM_DW(3);  in emit_bb_start()
108 	dw[i++] = lower_32_bits(batch_addr);  in emit_bb_start()
109 	dw[i++] = upper_32_bits(batch_addr);  in emit_bb_start()
111 	return i;  in emit_bb_start()
114 static int emit_flush_invalidate(u32 flag, u32 *dw, int i)  in emit_flush_invalidate()  argument
116 	dw[i] = MI_FLUSH_DW;  in emit_flush_invalidate()
117 	dw[i] |= flag;  in emit_flush_invalidate()
118 	dw[i++] |= MI_INVALIDATE_TLB | MI_FLUSH_DW_OP_STOREDW | MI_FLUSH_IMM_DW |  in emit_flush_invalidate()
121 	dw[i++] = LRC_PPHWSP_SCRATCH_ADDR | MI_FLUSH_DW_USE_GTT;  in emit_flush_invalidate()
122 	dw[i++] = 0;  in emit_flush_invalidate()
123 	dw[i++] = ~0U;  in emit_flush_invalidate()
125 	return i;  in emit_flush_invalidate()
129 emit_pipe_control(u32 *dw, int i, u32 bit_group_0, u32 bit_group_1, u32 offset, u32 value)  in emit_pipe_control()  argument
131 	dw[i++] = GFX_OP_PIPE_CONTROL(6) | bit_group_0;  in emit_pipe_control()
132 	dw[i++] = bit_group_1;  in emit_pipe_control()
133 	dw[i++] = offset;  in emit_pipe_control()
134 	dw[i++] = 0;  in emit_pipe_control()
135 	dw[i++] = value;  in emit_pipe_control()
136 	dw[i++] = 0;  in emit_pipe_control()
138 	return i;  in emit_pipe_control()
142 				int i)  in emit_pipe_invalidate()  argument
159 	return emit_pipe_control(dw, i, 0, flags, LRC_PPHWSP_SCRATCH_ADDR, 0);  in emit_pipe_invalidate()
163 				       u32 *dw, int i)  in emit_store_imm_ppgtt_posted()  argument
165 	dw[i++] = MI_STORE_DATA_IMM | MI_SDI_NUM_QW(1);  in emit_store_imm_ppgtt_posted()
166 	dw[i++] = lower_32_bits(addr);  in emit_store_imm_ppgtt_posted()
167 	dw[i++] = upper_32_bits(addr);  in emit_store_imm_ppgtt_posted()
168 	dw[i++] = lower_32_bits(value);  in emit_store_imm_ppgtt_posted()
169 	dw[i++] = upper_32_bits(value);  in emit_store_imm_ppgtt_posted()
171 	return i;  in emit_store_imm_ppgtt_posted()
174 static int emit_render_cache_flush(struct xe_sched_job *job, u32 *dw, int i)  in emit_render_cache_flush()  argument
195 	return emit_pipe_control(dw, i, PIPE_CONTROL0_HDC_PIPELINE_FLUSH, flags, 0, 0);  in emit_render_cache_flush()
198 static int emit_pipe_control_to_ring_end(struct xe_hw_engine *hwe, u32 *dw, int i)  in emit_pipe_control_to_ring_end()  argument
201 		return i;  in emit_pipe_control_to_ring_end()
204 		i = emit_pipe_control(dw, i, 0, PIPE_CONTROL_LRI_POST_SYNC,  in emit_pipe_control_to_ring_end()
207 	return i;  in emit_pipe_control_to_ring_end()
211 			      int i)  in emit_pipe_imm_ggtt()  argument
219 	return emit_pipe_control(dw, i, 0, flags, addr, value);  in emit_pipe_imm_ggtt()
227 static int emit_copy_timestamp(struct xe_lrc *lrc, u32 *dw, int i)  in emit_copy_timestamp()  argument
229 	dw[i++] = MI_COPY_MEM_MEM | MI_COPY_MEM_MEM_SRC_GGTT |  in emit_copy_timestamp()
231 	dw[i++] = xe_lrc_ctx_job_timestamp_ggtt_addr(lrc);  in emit_copy_timestamp()
232 	dw[i++] = 0;  in emit_copy_timestamp()
233 	dw[i++] = xe_lrc_ctx_timestamp_ggtt_addr(lrc);  in emit_copy_timestamp()
234 	dw[i++] = 0;  in emit_copy_timestamp()
235 	dw[i++] = MI_NOOP;  in emit_copy_timestamp()
237 	return i;  in emit_copy_timestamp()
244 	u32 dw[MAX_JOB_SIZE_DW], i = 0;  in __emit_job_gen12_simple()  local
248 	i = emit_copy_timestamp(lrc, dw, i);  in __emit_job_gen12_simple()
251 		dw[i++] = preparser_disable(true);  in __emit_job_gen12_simple()
252 		i = emit_flush_imm_ggtt(xe_lrc_start_seqno_ggtt_addr(lrc),  in __emit_job_gen12_simple()
253 					seqno, true, dw, i);  in __emit_job_gen12_simple()
254 		dw[i++] = preparser_disable(false);  in __emit_job_gen12_simple()
256 		i = emit_store_imm_ggtt(xe_lrc_start_seqno_ggtt_addr(lrc),  in __emit_job_gen12_simple()
257 					seqno, dw, i);  in __emit_job_gen12_simple()
260 	i = emit_bb_start(batch_addr, ppgtt_flag, dw, i);  in __emit_job_gen12_simple()
263 		i = emit_flush_dw(dw, i);  in __emit_job_gen12_simple()
264 		i = emit_store_imm_ppgtt_posted(job->user_fence.addr,  in __emit_job_gen12_simple()
266 						dw, i);  in __emit_job_gen12_simple()
269 	i = emit_flush_imm_ggtt(xe_lrc_seqno_ggtt_addr(lrc), seqno, false, dw, i);  in __emit_job_gen12_simple()
271 	i = emit_user_interrupt(dw, i);  in __emit_job_gen12_simple()
273 	xe_gt_assert(gt, i <= MAX_JOB_SIZE_DW);  in __emit_job_gen12_simple()
275 	xe_lrc_write_ring(lrc, dw, i * sizeof(*dw));  in __emit_job_gen12_simple()
295 	u32 dw[MAX_JOB_SIZE_DW], i = 0;  in __emit_job_gen12_video()  local
301 	i = emit_copy_timestamp(lrc, dw, i);  in __emit_job_gen12_video()
303 	dw[i++] = preparser_disable(true);  in __emit_job_gen12_video()
308 			i = emit_aux_table_inv(gt, VD0_AUX_INV, dw, i);  in __emit_job_gen12_video()
310 			i = emit_aux_table_inv(gt, VE0_AUX_INV, dw, i);  in __emit_job_gen12_video()
314 		i = emit_flush_imm_ggtt(xe_lrc_start_seqno_ggtt_addr(lrc),  in __emit_job_gen12_video()
315 					seqno, true, dw, i);  in __emit_job_gen12_video()
317 	dw[i++] = preparser_disable(false);  in __emit_job_gen12_video()
320 		i = emit_store_imm_ggtt(xe_lrc_start_seqno_ggtt_addr(lrc),  in __emit_job_gen12_video()
321 					seqno, dw, i);  in __emit_job_gen12_video()
323 	i = emit_bb_start(batch_addr, ppgtt_flag, dw, i);  in __emit_job_gen12_video()
326 		i = emit_flush_dw(dw, i);  in __emit_job_gen12_video()
327 		i = emit_store_imm_ppgtt_posted(job->user_fence.addr,  in __emit_job_gen12_video()
329 						dw, i);  in __emit_job_gen12_video()
332 	i = emit_flush_imm_ggtt(xe_lrc_seqno_ggtt_addr(lrc), seqno, false, dw, i);  in __emit_job_gen12_video()
334 	i = emit_user_interrupt(dw, i);  in __emit_job_gen12_video()
336 	xe_gt_assert(gt, i <= MAX_JOB_SIZE_DW);  in __emit_job_gen12_video()
338 	xe_lrc_write_ring(lrc, dw, i * sizeof(*dw));  in __emit_job_gen12_video()
345 	u32 dw[MAX_JOB_SIZE_DW], i = 0;  in __emit_job_gen12_render_compute()  local
352 	i = emit_copy_timestamp(lrc, dw, i);  in __emit_job_gen12_render_compute()
354 	dw[i++] = preparser_disable(true);  in __emit_job_gen12_render_compute()
361 	i = emit_pipe_invalidate(mask_flags, job->ring_ops_flush_tlb, dw, i);  in __emit_job_gen12_render_compute()
365 		i = emit_aux_table_inv(gt, CCS_AUX_INV, dw, i);  in __emit_job_gen12_render_compute()
367 	dw[i++] = preparser_disable(false);  in __emit_job_gen12_render_compute()
369 	i = emit_store_imm_ggtt(xe_lrc_start_seqno_ggtt_addr(lrc),  in __emit_job_gen12_render_compute()
370 				seqno, dw, i);  in __emit_job_gen12_render_compute()
372 	i = emit_bb_start(batch_addr, ppgtt_flag, dw, i);  in __emit_job_gen12_render_compute()
374 	i = emit_render_cache_flush(job, dw, i);  in __emit_job_gen12_render_compute()
377 		i = emit_store_imm_ppgtt_posted(job->user_fence.addr,  in __emit_job_gen12_render_compute()
379 						dw, i);  in __emit_job_gen12_render_compute()
381 	i = emit_pipe_imm_ggtt(xe_lrc_seqno_ggtt_addr(lrc), seqno, lacks_render, dw, i);  in __emit_job_gen12_render_compute()
383 	i = emit_user_interrupt(dw, i);  in __emit_job_gen12_render_compute()
385 	i = emit_pipe_control_to_ring_end(job->q->hwe, dw, i);  in __emit_job_gen12_render_compute()
387 	xe_gt_assert(gt, i <= MAX_JOB_SIZE_DW);  in __emit_job_gen12_render_compute()
389 	xe_lrc_write_ring(lrc, dw, i * sizeof(*dw));  in __emit_job_gen12_render_compute()
395 	u32 dw[MAX_JOB_SIZE_DW], i = 0;  in emit_migration_job_gen12()  local
397 	i = emit_copy_timestamp(lrc, dw, i);  in emit_migration_job_gen12()
399 	i = emit_store_imm_ggtt(xe_lrc_start_seqno_ggtt_addr(lrc),  in emit_migration_job_gen12()
400 				seqno, dw, i);  in emit_migration_job_gen12()
402 	dw[i++] = MI_ARB_ON_OFF | MI_ARB_DISABLE; /* Enabled again below */  in emit_migration_job_gen12()
404 	i = emit_bb_start(job->ptrs[0].batch_addr, BIT(8), dw, i);  in emit_migration_job_gen12()
408 		dw[i++] = preparser_disable(true);  in emit_migration_job_gen12()
409 		i = emit_flush_invalidate(0, dw, i);  in emit_migration_job_gen12()
410 		dw[i++] = preparser_disable(false);  in emit_migration_job_gen12()
413 	i = emit_bb_start(job->ptrs[1].batch_addr, BIT(8), dw, i);  in emit_migration_job_gen12()
415 	dw[i++] = MI_FLUSH_DW | MI_INVALIDATE_TLB | job->migrate_flush_flags |  in emit_migration_job_gen12()
417 	dw[i++] = xe_lrc_seqno_ggtt_addr(lrc) | MI_FLUSH_DW_USE_GTT;  in emit_migration_job_gen12()
418 	dw[i++] = 0;  in emit_migration_job_gen12()
419 	dw[i++] = seqno; /* value */  in emit_migration_job_gen12()
421 	i = emit_user_interrupt(dw, i);  in emit_migration_job_gen12()
423 	xe_gt_assert(job->q->gt, i <= MAX_JOB_SIZE_DW);  in emit_migration_job_gen12()
425 	xe_lrc_write_ring(lrc, dw, i * sizeof(*dw));  in emit_migration_job_gen12()
441 	int i;  in emit_job_gen12_copy()  local
449 	for (i = 0; i < job->q->width; ++i)  in emit_job_gen12_copy()
450 		__emit_job_gen12_simple(job, job->q->lrc[i],  in emit_job_gen12_copy()
451 					job->ptrs[i].batch_addr,  in emit_job_gen12_copy()
457 	int i;  in emit_job_gen12_video()  local
460 	for (i = 0; i < job->q->width; ++i)  in emit_job_gen12_video()
461 		__emit_job_gen12_video(job, job->q->lrc[i],  in emit_job_gen12_video()
462 				       job->ptrs[i].batch_addr,  in emit_job_gen12_video()
468 	int i;  in emit_job_gen12_render_compute()  local
470 	for (i = 0; i < job->q->width; ++i)  in emit_job_gen12_render_compute()
471 		__emit_job_gen12_render_compute(job, job->q->lrc[i],  in emit_job_gen12_render_compute()
472 						job->ptrs[i].batch_addr,  in emit_job_gen12_render_compute()