Lines Matching refs:cs
16 u32 *cs, flags = 0; in gen8_emit_flush_rcs() local
58 cs = intel_ring_begin(rq, len); in gen8_emit_flush_rcs()
59 if (IS_ERR(cs)) in gen8_emit_flush_rcs()
60 return PTR_ERR(cs); in gen8_emit_flush_rcs()
63 cs = gen8_emit_pipe_control(cs, 0, 0); in gen8_emit_flush_rcs()
66 cs = gen8_emit_pipe_control(cs, PIPE_CONTROL_DC_FLUSH_ENABLE, in gen8_emit_flush_rcs()
69 cs = gen8_emit_pipe_control(cs, flags, LRC_PPHWSP_SCRATCH_ADDR); in gen8_emit_flush_rcs()
72 cs = gen8_emit_pipe_control(cs, PIPE_CONTROL_CS_STALL, 0); in gen8_emit_flush_rcs()
74 intel_ring_advance(rq, cs); in gen8_emit_flush_rcs()
81 u32 cmd, *cs; in gen8_emit_flush_xcs() local
83 cs = intel_ring_begin(rq, 4); in gen8_emit_flush_xcs()
84 if (IS_ERR(cs)) in gen8_emit_flush_xcs()
85 return PTR_ERR(cs); in gen8_emit_flush_xcs()
103 *cs++ = cmd; in gen8_emit_flush_xcs()
104 *cs++ = LRC_PPHWSP_SCRATCH_ADDR; in gen8_emit_flush_xcs()
105 *cs++ = 0; /* upper addr */ in gen8_emit_flush_xcs()
106 *cs++ = 0; /* value */ in gen8_emit_flush_xcs()
107 intel_ring_advance(rq, cs); in gen8_emit_flush_xcs()
115 u32 *cs; in gen11_emit_flush_rcs() local
128 cs = intel_ring_begin(rq, 6); in gen11_emit_flush_rcs()
129 if (IS_ERR(cs)) in gen11_emit_flush_rcs()
130 return PTR_ERR(cs); in gen11_emit_flush_rcs()
132 cs = gen8_emit_pipe_control(cs, flags, LRC_PPHWSP_SCRATCH_ADDR); in gen11_emit_flush_rcs()
133 intel_ring_advance(rq, cs); in gen11_emit_flush_rcs()
137 u32 *cs; in gen11_emit_flush_rcs() local
152 cs = intel_ring_begin(rq, 6); in gen11_emit_flush_rcs()
153 if (IS_ERR(cs)) in gen11_emit_flush_rcs()
154 return PTR_ERR(cs); in gen11_emit_flush_rcs()
156 cs = gen8_emit_pipe_control(cs, flags, LRC_PPHWSP_SCRATCH_ADDR); in gen11_emit_flush_rcs()
157 intel_ring_advance(rq, cs); in gen11_emit_flush_rcs()
168 u32 *gen12_emit_aux_table_inv(struct intel_gt *gt, u32 *cs, const i915_reg_t inv_reg) in gen12_emit_aux_table_inv() argument
172 *cs++ = MI_LOAD_REGISTER_IMM(1) | MI_LRI_MMIO_REMAP_EN; in gen12_emit_aux_table_inv()
173 *cs++ = i915_mmio_reg_offset(inv_reg) + gsi_offset; in gen12_emit_aux_table_inv()
174 *cs++ = AUX_INV; in gen12_emit_aux_table_inv()
175 *cs++ = MI_NOOP; in gen12_emit_aux_table_inv()
177 return cs; in gen12_emit_aux_table_inv()
186 u32 *cs; in gen12_emit_flush_rcs() local
207 cs = intel_ring_begin(rq, 6); in gen12_emit_flush_rcs()
208 if (IS_ERR(cs)) in gen12_emit_flush_rcs()
209 return PTR_ERR(cs); in gen12_emit_flush_rcs()
211 cs = gen12_emit_pipe_control(cs, in gen12_emit_flush_rcs()
214 intel_ring_advance(rq, cs); in gen12_emit_flush_rcs()
219 u32 *cs, count; in gen12_emit_flush_rcs() local
244 cs = intel_ring_begin(rq, count); in gen12_emit_flush_rcs()
245 if (IS_ERR(cs)) in gen12_emit_flush_rcs()
246 return PTR_ERR(cs); in gen12_emit_flush_rcs()
253 *cs++ = preparser_disable(true); in gen12_emit_flush_rcs()
255 cs = gen8_emit_pipe_control(cs, flags, LRC_PPHWSP_SCRATCH_ADDR); in gen12_emit_flush_rcs()
259 cs = gen12_emit_aux_table_inv(rq->engine->gt, in gen12_emit_flush_rcs()
260 cs, GEN12_GFX_CCS_AUX_NV); in gen12_emit_flush_rcs()
263 *cs++ = preparser_disable(false); in gen12_emit_flush_rcs()
264 intel_ring_advance(rq, cs); in gen12_emit_flush_rcs()
273 u32 cmd, *cs; in gen12_emit_flush_xcs() local
289 cs = intel_ring_begin(rq, cmd); in gen12_emit_flush_xcs()
290 if (IS_ERR(cs)) in gen12_emit_flush_xcs()
291 return PTR_ERR(cs); in gen12_emit_flush_xcs()
294 *cs++ = preparser_disable(true); in gen12_emit_flush_xcs()
312 *cs++ = cmd; in gen12_emit_flush_xcs()
313 *cs++ = LRC_PPHWSP_SCRATCH_ADDR; in gen12_emit_flush_xcs()
314 *cs++ = 0; /* upper addr */ in gen12_emit_flush_xcs()
315 *cs++ = 0; /* value */ in gen12_emit_flush_xcs()
319 cs = gen12_emit_aux_table_inv(rq->engine->gt, in gen12_emit_flush_xcs()
320 cs, GEN12_VD0_AUX_NV); in gen12_emit_flush_xcs()
322 cs = gen12_emit_aux_table_inv(rq->engine->gt, in gen12_emit_flush_xcs()
323 cs, GEN12_VE0_AUX_NV); in gen12_emit_flush_xcs()
327 *cs++ = preparser_disable(false); in gen12_emit_flush_xcs()
329 intel_ring_advance(rq, cs); in gen12_emit_flush_xcs()
354 u32 *cs; in gen8_emit_init_breadcrumb() local
360 cs = intel_ring_begin(rq, 6); in gen8_emit_init_breadcrumb()
361 if (IS_ERR(cs)) in gen8_emit_init_breadcrumb()
362 return PTR_ERR(cs); in gen8_emit_init_breadcrumb()
364 *cs++ = MI_STORE_DWORD_IMM_GEN4 | MI_USE_GGTT; in gen8_emit_init_breadcrumb()
365 *cs++ = hwsp_offset(rq); in gen8_emit_init_breadcrumb()
366 *cs++ = 0; in gen8_emit_init_breadcrumb()
367 *cs++ = rq->fence.seqno - 1; in gen8_emit_init_breadcrumb()
386 *cs++ = MI_NOOP; in gen8_emit_init_breadcrumb()
387 *cs++ = MI_ARB_CHECK; in gen8_emit_init_breadcrumb()
389 intel_ring_advance(rq, cs); in gen8_emit_init_breadcrumb()
392 rq->infix = intel_ring_offset(rq, cs); in gen8_emit_init_breadcrumb()
406 u32 *cs; in __xehp_emit_bb_start() local
410 cs = intel_ring_begin(rq, 12); in __xehp_emit_bb_start()
411 if (IS_ERR(cs)) in __xehp_emit_bb_start()
412 return PTR_ERR(cs); in __xehp_emit_bb_start()
414 *cs++ = MI_ARB_ON_OFF | arb; in __xehp_emit_bb_start()
416 *cs++ = MI_LOAD_REGISTER_MEM_GEN8 | in __xehp_emit_bb_start()
419 *cs++ = i915_mmio_reg_offset(RING_PREDICATE_RESULT(0)); in __xehp_emit_bb_start()
420 *cs++ = wa_offset + DG2_PREDICATE_RESULT_WA; in __xehp_emit_bb_start()
421 *cs++ = 0; in __xehp_emit_bb_start()
423 *cs++ = MI_BATCH_BUFFER_START_GEN8 | in __xehp_emit_bb_start()
425 *cs++ = lower_32_bits(offset); in __xehp_emit_bb_start()
426 *cs++ = upper_32_bits(offset); in __xehp_emit_bb_start()
429 *cs++ = MI_BATCH_BUFFER_START_GEN8; in __xehp_emit_bb_start()
430 *cs++ = wa_offset + DG2_PREDICATE_RESULT_BB; in __xehp_emit_bb_start()
431 *cs++ = 0; in __xehp_emit_bb_start()
433 *cs++ = MI_ARB_ON_OFF | MI_ARB_DISABLE; in __xehp_emit_bb_start()
435 intel_ring_advance(rq, cs); in __xehp_emit_bb_start()
458 u32 *cs; in gen8_emit_bb_start_noarb() local
460 cs = intel_ring_begin(rq, 4); in gen8_emit_bb_start_noarb()
461 if (IS_ERR(cs)) in gen8_emit_bb_start_noarb()
462 return PTR_ERR(cs); in gen8_emit_bb_start_noarb()
477 *cs++ = MI_ARB_ON_OFF | MI_ARB_DISABLE; in gen8_emit_bb_start_noarb()
480 *cs++ = MI_BATCH_BUFFER_START_GEN8 | in gen8_emit_bb_start_noarb()
482 *cs++ = lower_32_bits(offset); in gen8_emit_bb_start_noarb()
483 *cs++ = upper_32_bits(offset); in gen8_emit_bb_start_noarb()
485 intel_ring_advance(rq, cs); in gen8_emit_bb_start_noarb()
494 u32 *cs; in gen8_emit_bb_start() local
499 cs = intel_ring_begin(rq, 6); in gen8_emit_bb_start()
500 if (IS_ERR(cs)) in gen8_emit_bb_start()
501 return PTR_ERR(cs); in gen8_emit_bb_start()
503 *cs++ = MI_ARB_ON_OFF | MI_ARB_ENABLE; in gen8_emit_bb_start()
505 *cs++ = MI_BATCH_BUFFER_START_GEN8 | in gen8_emit_bb_start()
507 *cs++ = lower_32_bits(offset); in gen8_emit_bb_start()
508 *cs++ = upper_32_bits(offset); in gen8_emit_bb_start()
510 *cs++ = MI_ARB_ON_OFF | MI_ARB_DISABLE; in gen8_emit_bb_start()
511 *cs++ = MI_NOOP; in gen8_emit_bb_start()
513 intel_ring_advance(rq, cs); in gen8_emit_bb_start()
531 static u32 *gen8_emit_wa_tail(struct i915_request *rq, u32 *cs) in gen8_emit_wa_tail() argument
534 *cs++ = MI_ARB_CHECK; in gen8_emit_wa_tail()
535 *cs++ = MI_NOOP; in gen8_emit_wa_tail()
536 rq->wa_tail = intel_ring_offset(rq, cs); in gen8_emit_wa_tail()
541 return cs; in gen8_emit_wa_tail()
544 static u32 *emit_preempt_busywait(struct i915_request *rq, u32 *cs) in emit_preempt_busywait() argument
546 *cs++ = MI_ARB_CHECK; /* trigger IDLE->ACTIVE first */ in emit_preempt_busywait()
547 *cs++ = MI_SEMAPHORE_WAIT | in emit_preempt_busywait()
551 *cs++ = 0; in emit_preempt_busywait()
552 *cs++ = preempt_address(rq->engine); in emit_preempt_busywait()
553 *cs++ = 0; in emit_preempt_busywait()
554 *cs++ = MI_NOOP; in emit_preempt_busywait()
556 return cs; in emit_preempt_busywait()
560 gen8_emit_fini_breadcrumb_tail(struct i915_request *rq, u32 *cs) in gen8_emit_fini_breadcrumb_tail() argument
562 *cs++ = MI_USER_INTERRUPT; in gen8_emit_fini_breadcrumb_tail()
564 *cs++ = MI_ARB_ON_OFF | MI_ARB_ENABLE; in gen8_emit_fini_breadcrumb_tail()
567 cs = emit_preempt_busywait(rq, cs); in gen8_emit_fini_breadcrumb_tail()
569 rq->tail = intel_ring_offset(rq, cs); in gen8_emit_fini_breadcrumb_tail()
572 return gen8_emit_wa_tail(rq, cs); in gen8_emit_fini_breadcrumb_tail()
575 static u32 *emit_xcs_breadcrumb(struct i915_request *rq, u32 *cs) in emit_xcs_breadcrumb() argument
577 return gen8_emit_ggtt_write(cs, rq->fence.seqno, hwsp_offset(rq), 0); in emit_xcs_breadcrumb()
580 u32 *gen8_emit_fini_breadcrumb_xcs(struct i915_request *rq, u32 *cs) in gen8_emit_fini_breadcrumb_xcs() argument
582 return gen8_emit_fini_breadcrumb_tail(rq, emit_xcs_breadcrumb(rq, cs)); in gen8_emit_fini_breadcrumb_xcs()
585 u32 *gen8_emit_fini_breadcrumb_rcs(struct i915_request *rq, u32 *cs) in gen8_emit_fini_breadcrumb_rcs() argument
587 cs = gen8_emit_pipe_control(cs, in gen8_emit_fini_breadcrumb_rcs()
596 cs = gen8_emit_ggtt_write_rcs(cs, in gen8_emit_fini_breadcrumb_rcs()
602 return gen8_emit_fini_breadcrumb_tail(rq, cs); in gen8_emit_fini_breadcrumb_rcs()
605 u32 *gen11_emit_fini_breadcrumb_rcs(struct i915_request *rq, u32 *cs) in gen11_emit_fini_breadcrumb_rcs() argument
607 cs = gen8_emit_pipe_control(cs, in gen11_emit_fini_breadcrumb_rcs()
617 cs = gen8_emit_ggtt_write_rcs(cs, in gen11_emit_fini_breadcrumb_rcs()
623 return gen8_emit_fini_breadcrumb_tail(rq, cs); in gen11_emit_fini_breadcrumb_rcs()
645 static u32 *gen12_emit_preempt_busywait(struct i915_request *rq, u32 *cs) in gen12_emit_preempt_busywait() argument
647 *cs++ = MI_ARB_CHECK; /* trigger IDLE->ACTIVE first */ in gen12_emit_preempt_busywait()
648 *cs++ = MI_SEMAPHORE_WAIT_TOKEN | in gen12_emit_preempt_busywait()
652 *cs++ = 0; in gen12_emit_preempt_busywait()
653 *cs++ = preempt_address(rq->engine); in gen12_emit_preempt_busywait()
654 *cs++ = 0; in gen12_emit_preempt_busywait()
655 *cs++ = 0; in gen12_emit_preempt_busywait()
657 return cs; in gen12_emit_preempt_busywait()
669 static u32 *ccs_emit_wa_busywait(struct i915_request *rq, u32 *cs) in ccs_emit_wa_busywait() argument
673 *cs++ = MI_ATOMIC_INLINE | MI_ATOMIC_GLOBAL_GTT | MI_ATOMIC_CS_STALL | in ccs_emit_wa_busywait()
675 *cs++ = ccs_semaphore_offset(rq); in ccs_emit_wa_busywait()
676 *cs++ = 0; in ccs_emit_wa_busywait()
677 *cs++ = 1; in ccs_emit_wa_busywait()
684 *cs++ = 0; in ccs_emit_wa_busywait()
686 *cs++ = MI_SEMAPHORE_WAIT | in ccs_emit_wa_busywait()
690 *cs++ = 0; in ccs_emit_wa_busywait()
691 *cs++ = ccs_semaphore_offset(rq); in ccs_emit_wa_busywait()
692 *cs++ = 0; in ccs_emit_wa_busywait()
694 return cs; in ccs_emit_wa_busywait()
698 gen12_emit_fini_breadcrumb_tail(struct i915_request *rq, u32 *cs) in gen12_emit_fini_breadcrumb_tail() argument
700 *cs++ = MI_USER_INTERRUPT; in gen12_emit_fini_breadcrumb_tail()
702 *cs++ = MI_ARB_ON_OFF | MI_ARB_ENABLE; in gen12_emit_fini_breadcrumb_tail()
705 cs = gen12_emit_preempt_busywait(rq, cs); in gen12_emit_fini_breadcrumb_tail()
709 cs = ccs_emit_wa_busywait(rq, cs); in gen12_emit_fini_breadcrumb_tail()
711 rq->tail = intel_ring_offset(rq, cs); in gen12_emit_fini_breadcrumb_tail()
714 return gen8_emit_wa_tail(rq, cs); in gen12_emit_fini_breadcrumb_tail()
717 u32 *gen12_emit_fini_breadcrumb_xcs(struct i915_request *rq, u32 *cs) in gen12_emit_fini_breadcrumb_xcs() argument
720 cs = emit_xcs_breadcrumb(rq, __gen8_emit_flush_dw(cs, 0, 0, 0)); in gen12_emit_fini_breadcrumb_xcs()
721 return gen12_emit_fini_breadcrumb_tail(rq, cs); in gen12_emit_fini_breadcrumb_xcs()
724 u32 *gen12_emit_fini_breadcrumb_rcs(struct i915_request *rq, u32 *cs) in gen12_emit_fini_breadcrumb_rcs() argument
745 cs = gen12_emit_pipe_control(cs, PIPE_CONTROL0_HDC_PIPELINE_FLUSH, flags, 0); in gen12_emit_fini_breadcrumb_rcs()
748 cs = gen12_emit_ggtt_write_rcs(cs, in gen12_emit_fini_breadcrumb_rcs()
755 return gen12_emit_fini_breadcrumb_tail(rq, cs); in gen12_emit_fini_breadcrumb_rcs()