Lines Matching refs:rq
12 int gen8_emit_flush_rcs(struct i915_request *rq, u32 mode) in gen8_emit_flush_rcs() argument
41 if (GRAPHICS_VER(rq->engine->i915) == 9) in gen8_emit_flush_rcs()
45 if (IS_KBL_GT_STEP(rq->engine->i915, 0, STEP_C0)) in gen8_emit_flush_rcs()
57 cs = intel_ring_begin(rq, len); in gen8_emit_flush_rcs()
73 intel_ring_advance(rq, cs); in gen8_emit_flush_rcs()
78 int gen8_emit_flush_xcs(struct i915_request *rq, u32 mode) in gen8_emit_flush_xcs() argument
82 cs = intel_ring_begin(rq, 4); in gen8_emit_flush_xcs()
98 if (rq->engine->class == VIDEO_DECODE_CLASS) in gen8_emit_flush_xcs()
106 intel_ring_advance(rq, cs); in gen8_emit_flush_xcs()
111 int gen11_emit_flush_rcs(struct i915_request *rq, u32 mode) in gen11_emit_flush_rcs() argument
127 cs = intel_ring_begin(rq, 6); in gen11_emit_flush_rcs()
132 intel_ring_advance(rq, cs); in gen11_emit_flush_rcs()
151 cs = intel_ring_begin(rq, 6); in gen11_emit_flush_rcs()
156 intel_ring_advance(rq, cs); in gen11_emit_flush_rcs()
201 int gen12_emit_flush_rcs(struct i915_request *rq, u32 mode) in gen12_emit_flush_rcs() argument
221 cs = intel_ring_begin(rq, 6); in gen12_emit_flush_rcs()
228 intel_ring_advance(rq, cs); in gen12_emit_flush_rcs()
248 cs = intel_ring_begin(rq, 8 + 4); in gen12_emit_flush_rcs()
265 intel_ring_advance(rq, cs); in gen12_emit_flush_rcs()
271 int gen12_emit_flush_xcs(struct i915_request *rq, u32 mode) in gen12_emit_flush_xcs() argument
280 aux_inv = rq->engine->mask & ~BIT(BCS0); in gen12_emit_flush_xcs()
284 cs = intel_ring_begin(rq, cmd); in gen12_emit_flush_xcs()
303 if (rq->engine->class == VIDEO_DECODE_CLASS) in gen12_emit_flush_xcs()
317 for_each_engine_masked(engine, rq->engine->gt, aux_inv, tmp) { in gen12_emit_flush_xcs()
327 intel_ring_advance(rq, cs); in gen12_emit_flush_xcs()
338 static u32 hwsp_offset(const struct i915_request *rq) in hwsp_offset() argument
343 tl = rcu_dereference_protected(rq->timeline, in hwsp_offset()
344 !i915_request_signaled(rq)); in hwsp_offset()
347 return page_mask_bits(tl->hwsp_offset) + offset_in_page(rq->hwsp_seqno); in hwsp_offset()
350 int gen8_emit_init_breadcrumb(struct i915_request *rq) in gen8_emit_init_breadcrumb() argument
354 GEM_BUG_ON(i915_request_has_initial_breadcrumb(rq)); in gen8_emit_init_breadcrumb()
355 if (!i915_request_timeline(rq)->has_initial_breadcrumb) in gen8_emit_init_breadcrumb()
358 cs = intel_ring_begin(rq, 6); in gen8_emit_init_breadcrumb()
363 *cs++ = hwsp_offset(rq); in gen8_emit_init_breadcrumb()
365 *cs++ = rq->fence.seqno - 1; in gen8_emit_init_breadcrumb()
387 intel_ring_advance(rq, cs); in gen8_emit_init_breadcrumb()
390 rq->infix = intel_ring_offset(rq, cs); in gen8_emit_init_breadcrumb()
392 __set_bit(I915_FENCE_FLAG_INITIAL_BREADCRUMB, &rq->fence.flags); in gen8_emit_init_breadcrumb()
397 int gen8_emit_bb_start_noarb(struct i915_request *rq, in gen8_emit_bb_start_noarb() argument
403 cs = intel_ring_begin(rq, 4); in gen8_emit_bb_start_noarb()
428 intel_ring_advance(rq, cs); in gen8_emit_bb_start_noarb()
433 int gen8_emit_bb_start(struct i915_request *rq, in gen8_emit_bb_start() argument
439 if (unlikely(i915_request_has_nopreempt(rq))) in gen8_emit_bb_start()
440 return gen8_emit_bb_start_noarb(rq, offset, len, flags); in gen8_emit_bb_start()
442 cs = intel_ring_begin(rq, 6); in gen8_emit_bb_start()
456 intel_ring_advance(rq, cs); in gen8_emit_bb_start()
461 static void assert_request_valid(struct i915_request *rq) in assert_request_valid() argument
463 struct intel_ring *ring __maybe_unused = rq->ring; in assert_request_valid()
466 GEM_BUG_ON(intel_ring_direction(ring, rq->wa_tail, rq->head) <= 0); in assert_request_valid()
474 static u32 *gen8_emit_wa_tail(struct i915_request *rq, u32 *cs) in gen8_emit_wa_tail() argument
479 rq->wa_tail = intel_ring_offset(rq, cs); in gen8_emit_wa_tail()
482 assert_request_valid(rq); in gen8_emit_wa_tail()
487 static u32 *emit_preempt_busywait(struct i915_request *rq, u32 *cs) in emit_preempt_busywait() argument
495 *cs++ = preempt_address(rq->engine); in emit_preempt_busywait()
503 gen8_emit_fini_breadcrumb_tail(struct i915_request *rq, u32 *cs) in gen8_emit_fini_breadcrumb_tail() argument
508 if (intel_engine_has_semaphores(rq->engine) && in gen8_emit_fini_breadcrumb_tail()
509 !intel_uc_uses_guc_submission(&rq->engine->gt->uc)) in gen8_emit_fini_breadcrumb_tail()
510 cs = emit_preempt_busywait(rq, cs); in gen8_emit_fini_breadcrumb_tail()
512 rq->tail = intel_ring_offset(rq, cs); in gen8_emit_fini_breadcrumb_tail()
513 assert_ring_tail_valid(rq->ring, rq->tail); in gen8_emit_fini_breadcrumb_tail()
515 return gen8_emit_wa_tail(rq, cs); in gen8_emit_fini_breadcrumb_tail()
518 static u32 *emit_xcs_breadcrumb(struct i915_request *rq, u32 *cs) in emit_xcs_breadcrumb() argument
520 return gen8_emit_ggtt_write(cs, rq->fence.seqno, hwsp_offset(rq), 0); in emit_xcs_breadcrumb()
523 u32 *gen8_emit_fini_breadcrumb_xcs(struct i915_request *rq, u32 *cs) in gen8_emit_fini_breadcrumb_xcs() argument
525 return gen8_emit_fini_breadcrumb_tail(rq, emit_xcs_breadcrumb(rq, cs)); in gen8_emit_fini_breadcrumb_xcs()
528 u32 *gen8_emit_fini_breadcrumb_rcs(struct i915_request *rq, u32 *cs) in gen8_emit_fini_breadcrumb_rcs() argument
538 rq->fence.seqno, in gen8_emit_fini_breadcrumb_rcs()
539 hwsp_offset(rq), in gen8_emit_fini_breadcrumb_rcs()
543 return gen8_emit_fini_breadcrumb_tail(rq, cs); in gen8_emit_fini_breadcrumb_rcs()
546 u32 *gen11_emit_fini_breadcrumb_rcs(struct i915_request *rq, u32 *cs) in gen11_emit_fini_breadcrumb_rcs() argument
549 rq->fence.seqno, in gen11_emit_fini_breadcrumb_rcs()
550 hwsp_offset(rq), in gen11_emit_fini_breadcrumb_rcs()
558 return gen8_emit_fini_breadcrumb_tail(rq, cs); in gen11_emit_fini_breadcrumb_rcs()
580 static u32 *gen12_emit_preempt_busywait(struct i915_request *rq, u32 *cs) in gen12_emit_preempt_busywait() argument
588 *cs++ = preempt_address(rq->engine); in gen12_emit_preempt_busywait()
596 gen12_emit_fini_breadcrumb_tail(struct i915_request *rq, u32 *cs) in gen12_emit_fini_breadcrumb_tail() argument
601 if (intel_engine_has_semaphores(rq->engine) && in gen12_emit_fini_breadcrumb_tail()
602 !intel_uc_uses_guc_submission(&rq->engine->gt->uc)) in gen12_emit_fini_breadcrumb_tail()
603 cs = gen12_emit_preempt_busywait(rq, cs); in gen12_emit_fini_breadcrumb_tail()
605 rq->tail = intel_ring_offset(rq, cs); in gen12_emit_fini_breadcrumb_tail()
606 assert_ring_tail_valid(rq->ring, rq->tail); in gen12_emit_fini_breadcrumb_tail()
608 return gen8_emit_wa_tail(rq, cs); in gen12_emit_fini_breadcrumb_tail()
611 u32 *gen12_emit_fini_breadcrumb_xcs(struct i915_request *rq, u32 *cs) in gen12_emit_fini_breadcrumb_xcs() argument
614 cs = emit_xcs_breadcrumb(rq, __gen8_emit_flush_dw(cs, 0, 0, 0)); in gen12_emit_fini_breadcrumb_xcs()
615 return gen12_emit_fini_breadcrumb_tail(rq, cs); in gen12_emit_fini_breadcrumb_xcs()
618 u32 *gen12_emit_fini_breadcrumb_rcs(struct i915_request *rq, u32 *cs) in gen12_emit_fini_breadcrumb_rcs() argument
621 rq->fence.seqno, in gen12_emit_fini_breadcrumb_rcs()
622 hwsp_offset(rq), in gen12_emit_fini_breadcrumb_rcs()
634 return gen12_emit_fini_breadcrumb_tail(rq, cs); in gen12_emit_fini_breadcrumb_rcs()