Lines Matching refs:ring
57 struct amdgpu_ring *ring; member
102 static void amdgpu_fence_write(struct amdgpu_ring *ring, u32 seq) in amdgpu_fence_write() argument
104 struct amdgpu_fence_driver *drv = &ring->fence_drv; in amdgpu_fence_write()
118 static u32 amdgpu_fence_read(struct amdgpu_ring *ring) in amdgpu_fence_read() argument
120 struct amdgpu_fence_driver *drv = &ring->fence_drv; in amdgpu_fence_read()
142 int amdgpu_fence_emit(struct amdgpu_ring *ring, struct dma_fence **f, struct amdgpu_job *job, in amdgpu_fence_emit() argument
145 struct amdgpu_device *adev = ring->adev; in amdgpu_fence_emit()
158 am_fence->ring = ring; in amdgpu_fence_emit()
164 seq = ++ring->fence_drv.sync_seq; in amdgpu_fence_emit()
173 &ring->fence_drv.lock, in amdgpu_fence_emit()
174 adev->fence_context + ring->idx, seq); in amdgpu_fence_emit()
180 &ring->fence_drv.lock, in amdgpu_fence_emit()
181 adev->fence_context + ring->idx, seq); in amdgpu_fence_emit()
184 amdgpu_ring_emit_fence(ring, ring->fence_drv.gpu_addr, in amdgpu_fence_emit()
187 ptr = &ring->fence_drv.fences[seq & ring->fence_drv.num_fences_mask]; in amdgpu_fence_emit()
226 int amdgpu_fence_emit_polling(struct amdgpu_ring *ring, uint32_t *s, in amdgpu_fence_emit_polling() argument
235 seq = ++ring->fence_drv.sync_seq; in amdgpu_fence_emit_polling()
236 r = amdgpu_fence_wait_polling(ring, in amdgpu_fence_emit_polling()
237 seq - ring->fence_drv.num_fences_mask, in amdgpu_fence_emit_polling()
242 amdgpu_ring_emit_fence(ring, ring->fence_drv.gpu_addr, in amdgpu_fence_emit_polling()
257 static void amdgpu_fence_schedule_fallback(struct amdgpu_ring *ring) in amdgpu_fence_schedule_fallback() argument
259 mod_timer(&ring->fence_drv.fallback_timer, in amdgpu_fence_schedule_fallback()
274 bool amdgpu_fence_process(struct amdgpu_ring *ring) in amdgpu_fence_process() argument
276 struct amdgpu_fence_driver *drv = &ring->fence_drv; in amdgpu_fence_process()
277 struct amdgpu_device *adev = ring->adev; in amdgpu_fence_process()
281 last_seq = atomic_read(&ring->fence_drv.last_seq); in amdgpu_fence_process()
282 seq = amdgpu_fence_read(ring); in amdgpu_fence_process()
286 if (del_timer(&ring->fence_drv.fallback_timer) && in amdgpu_fence_process()
287 seq != ring->fence_drv.sync_seq) in amdgpu_fence_process()
288 amdgpu_fence_schedule_fallback(ring); in amdgpu_fence_process()
328 struct amdgpu_ring *ring = from_timer(ring, t, in amdgpu_fence_fallback() local
331 if (amdgpu_fence_process(ring)) in amdgpu_fence_fallback()
332 DRM_WARN("Fence fallback timer expired on ring %s\n", ring->name); in amdgpu_fence_fallback()
343 int amdgpu_fence_wait_empty(struct amdgpu_ring *ring) in amdgpu_fence_wait_empty() argument
345 uint64_t seq = READ_ONCE(ring->fence_drv.sync_seq); in amdgpu_fence_wait_empty()
352 ptr = &ring->fence_drv.fences[seq & ring->fence_drv.num_fences_mask]; in amdgpu_fence_wait_empty()
376 signed long amdgpu_fence_wait_polling(struct amdgpu_ring *ring, in amdgpu_fence_wait_polling() argument
383 seq = amdgpu_fence_read(ring); in amdgpu_fence_wait_polling()
399 unsigned amdgpu_fence_count_emitted(struct amdgpu_ring *ring) in amdgpu_fence_count_emitted() argument
407 emitted -= atomic_read(&ring->fence_drv.last_seq); in amdgpu_fence_count_emitted()
408 emitted += READ_ONCE(ring->fence_drv.sync_seq); in amdgpu_fence_count_emitted()
419 u64 amdgpu_fence_last_unsignaled_time_us(struct amdgpu_ring *ring) in amdgpu_fence_last_unsignaled_time_us() argument
421 struct amdgpu_fence_driver *drv = &ring->fence_drv; in amdgpu_fence_last_unsignaled_time_us()
425 last_seq = atomic_read(&ring->fence_drv.last_seq); in amdgpu_fence_last_unsignaled_time_us()
426 sync_seq = READ_ONCE(ring->fence_drv.sync_seq); in amdgpu_fence_last_unsignaled_time_us()
450 void amdgpu_fence_update_start_timestamp(struct amdgpu_ring *ring, uint32_t seq, ktime_t timestamp) in amdgpu_fence_update_start_timestamp() argument
452 struct amdgpu_fence_driver *drv = &ring->fence_drv; in amdgpu_fence_update_start_timestamp()
476 int amdgpu_fence_driver_start_ring(struct amdgpu_ring *ring, in amdgpu_fence_driver_start_ring() argument
480 struct amdgpu_device *adev = ring->adev; in amdgpu_fence_driver_start_ring()
483 if (ring->funcs->type != AMDGPU_RING_TYPE_UVD) { in amdgpu_fence_driver_start_ring()
484 ring->fence_drv.cpu_addr = ring->fence_cpu_addr; in amdgpu_fence_driver_start_ring()
485 ring->fence_drv.gpu_addr = ring->fence_gpu_addr; in amdgpu_fence_driver_start_ring()
489 ring->fence_drv.cpu_addr = adev->uvd.inst[ring->me].cpu_addr + index; in amdgpu_fence_driver_start_ring()
490 ring->fence_drv.gpu_addr = adev->uvd.inst[ring->me].gpu_addr + index; in amdgpu_fence_driver_start_ring()
492 amdgpu_fence_write(ring, atomic_read(&ring->fence_drv.last_seq)); in amdgpu_fence_driver_start_ring()
494 ring->fence_drv.irq_src = irq_src; in amdgpu_fence_driver_start_ring()
495 ring->fence_drv.irq_type = irq_type; in amdgpu_fence_driver_start_ring()
496 ring->fence_drv.initialized = true; in amdgpu_fence_driver_start_ring()
499 ring->name, ring->fence_drv.gpu_addr); in amdgpu_fence_driver_start_ring()
512 int amdgpu_fence_driver_init_ring(struct amdgpu_ring *ring) in amdgpu_fence_driver_init_ring() argument
514 struct amdgpu_device *adev = ring->adev; in amdgpu_fence_driver_init_ring()
519 if (!is_power_of_2(ring->num_hw_submission)) in amdgpu_fence_driver_init_ring()
522 ring->fence_drv.cpu_addr = NULL; in amdgpu_fence_driver_init_ring()
523 ring->fence_drv.gpu_addr = 0; in amdgpu_fence_driver_init_ring()
524 ring->fence_drv.sync_seq = 0; in amdgpu_fence_driver_init_ring()
525 atomic_set(&ring->fence_drv.last_seq, 0); in amdgpu_fence_driver_init_ring()
526 ring->fence_drv.initialized = false; in amdgpu_fence_driver_init_ring()
528 timer_setup(&ring->fence_drv.fallback_timer, amdgpu_fence_fallback, 0); in amdgpu_fence_driver_init_ring()
530 ring->fence_drv.num_fences_mask = ring->num_hw_submission * 2 - 1; in amdgpu_fence_driver_init_ring()
531 spin_lock_init(&ring->fence_drv.lock); in amdgpu_fence_driver_init_ring()
532 ring->fence_drv.fences = kcalloc(ring->num_hw_submission * 2, sizeof(void *), in amdgpu_fence_driver_init_ring()
535 if (!ring->fence_drv.fences) in amdgpu_fence_driver_init_ring()
571 struct amdgpu_ring *ring = adev->rings[i]; in amdgpu_fence_driver_hw_fini() local
573 if (!ring || !ring->fence_drv.initialized) in amdgpu_fence_driver_hw_fini()
578 r = amdgpu_fence_wait_empty(ring); in amdgpu_fence_driver_hw_fini()
583 amdgpu_fence_driver_force_completion(ring); in amdgpu_fence_driver_hw_fini()
585 if (ring->fence_drv.irq_src) in amdgpu_fence_driver_hw_fini()
586 amdgpu_irq_put(adev, ring->fence_drv.irq_src, in amdgpu_fence_driver_hw_fini()
587 ring->fence_drv.irq_type); in amdgpu_fence_driver_hw_fini()
589 del_timer_sync(&ring->fence_drv.fallback_timer); in amdgpu_fence_driver_hw_fini()
599 struct amdgpu_ring *ring = adev->rings[i]; in amdgpu_fence_driver_isr_toggle() local
601 if (!ring || !ring->fence_drv.initialized || !ring->fence_drv.irq_src) in amdgpu_fence_driver_isr_toggle()
616 struct amdgpu_ring *ring = adev->rings[i]; in amdgpu_fence_driver_sw_fini() local
618 if (!ring || !ring->fence_drv.initialized) in amdgpu_fence_driver_sw_fini()
627 if (ring->sched.ops) in amdgpu_fence_driver_sw_fini()
628 drm_sched_fini(&ring->sched); in amdgpu_fence_driver_sw_fini()
630 for (j = 0; j <= ring->fence_drv.num_fences_mask; ++j) in amdgpu_fence_driver_sw_fini()
631 dma_fence_put(ring->fence_drv.fences[j]); in amdgpu_fence_driver_sw_fini()
632 kfree(ring->fence_drv.fences); in amdgpu_fence_driver_sw_fini()
633 ring->fence_drv.fences = NULL; in amdgpu_fence_driver_sw_fini()
634 ring->fence_drv.initialized = false; in amdgpu_fence_driver_sw_fini()
655 struct amdgpu_ring *ring = adev->rings[i]; in amdgpu_fence_driver_hw_init() local
656 if (!ring || !ring->fence_drv.initialized) in amdgpu_fence_driver_hw_init()
660 if (ring->fence_drv.irq_src) in amdgpu_fence_driver_hw_init()
661 amdgpu_irq_get(adev, ring->fence_drv.irq_src, in amdgpu_fence_driver_hw_init()
662 ring->fence_drv.irq_type); in amdgpu_fence_driver_hw_init()
672 void amdgpu_fence_driver_clear_job_fences(struct amdgpu_ring *ring) in amdgpu_fence_driver_clear_job_fences() argument
677 for (i = 0; i <= ring->fence_drv.num_fences_mask; i++) { in amdgpu_fence_driver_clear_job_fences()
678 ptr = &ring->fence_drv.fences[i]; in amdgpu_fence_driver_clear_job_fences()
693 void amdgpu_fence_driver_force_completion(struct amdgpu_ring *ring) in amdgpu_fence_driver_force_completion() argument
695 amdgpu_fence_write(ring, ring->fence_drv.sync_seq); in amdgpu_fence_driver_force_completion()
696 amdgpu_fence_process(ring); in amdgpu_fence_driver_force_completion()
710 return (const char *)to_amdgpu_fence(f)->ring->name; in amdgpu_fence_get_timeline_name()
730 if (!timer_pending(&to_amdgpu_fence(f)->ring->fence_drv.fallback_timer)) in amdgpu_fence_enable_signaling()
731 amdgpu_fence_schedule_fallback(to_amdgpu_fence(f)->ring); in amdgpu_fence_enable_signaling()
833 struct amdgpu_ring *ring = adev->rings[i]; in amdgpu_debugfs_fence_info_show() local
834 if (!ring || !ring->fence_drv.initialized) in amdgpu_debugfs_fence_info_show()
837 amdgpu_fence_process(ring); in amdgpu_debugfs_fence_info_show()
839 seq_printf(m, "--- ring %d (%s) ---\n", i, ring->name); in amdgpu_debugfs_fence_info_show()
841 atomic_read(&ring->fence_drv.last_seq)); in amdgpu_debugfs_fence_info_show()
843 ring->fence_drv.sync_seq); in amdgpu_debugfs_fence_info_show()
845 if (ring->funcs->type == AMDGPU_RING_TYPE_GFX || in amdgpu_debugfs_fence_info_show()
846 ring->funcs->type == AMDGPU_RING_TYPE_SDMA) { in amdgpu_debugfs_fence_info_show()
848 le32_to_cpu(*ring->trail_fence_cpu_addr)); in amdgpu_debugfs_fence_info_show()
850 ring->trail_seq); in amdgpu_debugfs_fence_info_show()
853 if (ring->funcs->type != AMDGPU_RING_TYPE_GFX) in amdgpu_debugfs_fence_info_show()
858 le32_to_cpu(*(ring->fence_drv.cpu_addr + 2))); in amdgpu_debugfs_fence_info_show()
861 le32_to_cpu(*(ring->fence_drv.cpu_addr + 4))); in amdgpu_debugfs_fence_info_show()
864 le32_to_cpu(*(ring->fence_drv.cpu_addr + 6))); in amdgpu_debugfs_fence_info_show()