Lines Matching refs:rdev
67 static void radeon_fence_write(struct radeon_device *rdev, u32 seq, int ring) in radeon_fence_write() argument
69 struct radeon_fence_driver *drv = &rdev->fence_drv[ring]; in radeon_fence_write()
70 if (likely(rdev->wb.enabled || !drv->scratch_reg)) { in radeon_fence_write()
88 static u32 radeon_fence_read(struct radeon_device *rdev, int ring) in radeon_fence_read() argument
90 struct radeon_fence_driver *drv = &rdev->fence_drv[ring]; in radeon_fence_read()
93 if (likely(rdev->wb.enabled || !drv->scratch_reg)) { in radeon_fence_read()
113 static void radeon_fence_schedule_check(struct radeon_device *rdev, int ring) in radeon_fence_schedule_check() argument
120 &rdev->fence_drv[ring].lockup_work, in radeon_fence_schedule_check()
134 int radeon_fence_emit(struct radeon_device *rdev, in radeon_fence_emit() argument
145 (*fence)->rdev = rdev; in radeon_fence_emit()
146 (*fence)->seq = seq = ++rdev->fence_drv[ring].sync_seq[ring]; in radeon_fence_emit()
150 &rdev->fence_queue.lock, in radeon_fence_emit()
151 rdev->fence_context + ring, in radeon_fence_emit()
153 radeon_fence_ring_emit(rdev, ring, *fence); in radeon_fence_emit()
154 trace_radeon_fence_emit(rdev->ddev, ring, (*fence)->seq); in radeon_fence_emit()
155 radeon_fence_schedule_check(rdev, ring); in radeon_fence_emit()
177 seq = atomic64_read(&fence->rdev->fence_drv[fence->ring].last_seq); in radeon_fence_check_signaled()
180 radeon_irq_kms_sw_irq_put(fence->rdev, fence->ring); in radeon_fence_check_signaled()
181 __remove_wait_queue(&fence->rdev->fence_queue, &fence->fence_wake); in radeon_fence_check_signaled()
197 static bool radeon_fence_activity(struct radeon_device *rdev, int ring) in radeon_fence_activity() argument
224 last_seq = atomic64_read(&rdev->fence_drv[ring].last_seq); in radeon_fence_activity()
226 last_emitted = rdev->fence_drv[ring].sync_seq[ring]; in radeon_fence_activity()
227 seq = radeon_fence_read(rdev, ring); in radeon_fence_activity()
251 } while (atomic64_xchg(&rdev->fence_drv[ring].last_seq, seq) > seq); in radeon_fence_activity()
254 radeon_fence_schedule_check(rdev, ring); in radeon_fence_activity()
270 struct radeon_device *rdev; in radeon_fence_check_lockup() local
275 rdev = fence_drv->rdev; in radeon_fence_check_lockup()
276 ring = fence_drv - &rdev->fence_drv[0]; in radeon_fence_check_lockup()
278 if (!down_read_trylock(&rdev->exclusive_lock)) { in radeon_fence_check_lockup()
280 radeon_fence_schedule_check(rdev, ring); in radeon_fence_check_lockup()
284 if (fence_drv->delayed_irq && rdev->irq.installed) { in radeon_fence_check_lockup()
288 spin_lock_irqsave(&rdev->irq.lock, irqflags); in radeon_fence_check_lockup()
289 radeon_irq_set(rdev); in radeon_fence_check_lockup()
290 spin_unlock_irqrestore(&rdev->irq.lock, irqflags); in radeon_fence_check_lockup()
293 if (radeon_fence_activity(rdev, ring)) in radeon_fence_check_lockup()
294 wake_up_all(&rdev->fence_queue); in radeon_fence_check_lockup()
296 else if (radeon_ring_is_lockup(rdev, ring, &rdev->ring[ring])) { in radeon_fence_check_lockup()
299 dev_warn(rdev->dev, "GPU lockup (current fence id " in radeon_fence_check_lockup()
305 rdev->needs_reset = true; in radeon_fence_check_lockup()
306 wake_up_all(&rdev->fence_queue); in radeon_fence_check_lockup()
308 up_read(&rdev->exclusive_lock); in radeon_fence_check_lockup()
320 void radeon_fence_process(struct radeon_device *rdev, int ring) in radeon_fence_process() argument
322 if (radeon_fence_activity(rdev, ring)) in radeon_fence_process()
323 wake_up_all(&rdev->fence_queue); in radeon_fence_process()
340 static bool radeon_fence_seq_signaled(struct radeon_device *rdev, in radeon_fence_seq_signaled() argument
343 if (atomic64_read(&rdev->fence_drv[ring].last_seq) >= seq) { in radeon_fence_seq_signaled()
347 radeon_fence_process(rdev, ring); in radeon_fence_seq_signaled()
348 if (atomic64_read(&rdev->fence_drv[ring].last_seq) >= seq) { in radeon_fence_seq_signaled()
357 struct radeon_device *rdev = fence->rdev; in radeon_fence_is_signaled() local
361 if (atomic64_read(&rdev->fence_drv[ring].last_seq) >= seq) { in radeon_fence_is_signaled()
365 if (down_read_trylock(&rdev->exclusive_lock)) { in radeon_fence_is_signaled()
366 radeon_fence_process(rdev, ring); in radeon_fence_is_signaled()
367 up_read(&rdev->exclusive_lock); in radeon_fence_is_signaled()
369 if (atomic64_read(&rdev->fence_drv[ring].last_seq) >= seq) { in radeon_fence_is_signaled()
387 struct radeon_device *rdev = fence->rdev; in radeon_fence_enable_signaling() local
389 if (atomic64_read(&rdev->fence_drv[fence->ring].last_seq) >= fence->seq) in radeon_fence_enable_signaling()
392 if (down_read_trylock(&rdev->exclusive_lock)) { in radeon_fence_enable_signaling()
393 radeon_irq_kms_sw_irq_get(rdev, fence->ring); in radeon_fence_enable_signaling()
395 if (radeon_fence_activity(rdev, fence->ring)) in radeon_fence_enable_signaling()
396 wake_up_all_locked(&rdev->fence_queue); in radeon_fence_enable_signaling()
399 if (atomic64_read(&rdev->fence_drv[fence->ring].last_seq) >= fence->seq) { in radeon_fence_enable_signaling()
400 radeon_irq_kms_sw_irq_put(rdev, fence->ring); in radeon_fence_enable_signaling()
401 up_read(&rdev->exclusive_lock); in radeon_fence_enable_signaling()
405 up_read(&rdev->exclusive_lock); in radeon_fence_enable_signaling()
408 if (radeon_irq_kms_sw_irq_get_delayed(rdev, fence->ring)) in radeon_fence_enable_signaling()
409 rdev->fence_drv[fence->ring].delayed_irq = true; in radeon_fence_enable_signaling()
410 radeon_fence_schedule_check(rdev, fence->ring); in radeon_fence_enable_signaling()
416 __add_wait_queue(&rdev->fence_queue, &fence->fence_wake); in radeon_fence_enable_signaling()
434 if (radeon_fence_seq_signaled(fence->rdev, fence->seq, fence->ring)) { in radeon_fence_signaled()
452 static bool radeon_fence_any_seq_signaled(struct radeon_device *rdev, u64 *seq) in radeon_fence_any_seq_signaled() argument
457 if (seq[i] && radeon_fence_seq_signaled(rdev, seq[i], i)) in radeon_fence_any_seq_signaled()
480 static long radeon_fence_wait_seq_timeout(struct radeon_device *rdev, in radeon_fence_wait_seq_timeout() argument
487 if (radeon_fence_any_seq_signaled(rdev, target_seq)) in radeon_fence_wait_seq_timeout()
495 trace_radeon_fence_wait_begin(rdev->ddev, i, target_seq[i]); in radeon_fence_wait_seq_timeout()
496 radeon_irq_kms_sw_irq_get(rdev, i); in radeon_fence_wait_seq_timeout()
500 r = wait_event_interruptible_timeout(rdev->fence_queue, ( in radeon_fence_wait_seq_timeout()
501 radeon_fence_any_seq_signaled(rdev, target_seq) in radeon_fence_wait_seq_timeout()
502 || rdev->needs_reset), timeout); in radeon_fence_wait_seq_timeout()
504 r = wait_event_timeout(rdev->fence_queue, ( in radeon_fence_wait_seq_timeout()
505 radeon_fence_any_seq_signaled(rdev, target_seq) in radeon_fence_wait_seq_timeout()
506 || rdev->needs_reset), timeout); in radeon_fence_wait_seq_timeout()
509 if (rdev->needs_reset) in radeon_fence_wait_seq_timeout()
516 radeon_irq_kms_sw_irq_put(rdev, i); in radeon_fence_wait_seq_timeout()
517 trace_radeon_fence_wait_end(rdev->ddev, i, target_seq[i]); in radeon_fence_wait_seq_timeout()
551 r = radeon_fence_wait_seq_timeout(fence->rdev, seq, intr, timeout); in radeon_fence_wait_timeout()
594 int radeon_fence_wait_any(struct radeon_device *rdev, in radeon_fence_wait_any() argument
617 r = radeon_fence_wait_seq_timeout(rdev, seq, intr, MAX_SCHEDULE_TIMEOUT); in radeon_fence_wait_any()
634 int radeon_fence_wait_next(struct radeon_device *rdev, int ring) in radeon_fence_wait_next() argument
639 seq[ring] = atomic64_read(&rdev->fence_drv[ring].last_seq) + 1ULL; in radeon_fence_wait_next()
640 if (seq[ring] >= rdev->fence_drv[ring].sync_seq[ring]) { in radeon_fence_wait_next()
645 r = radeon_fence_wait_seq_timeout(rdev, seq, false, MAX_SCHEDULE_TIMEOUT); in radeon_fence_wait_next()
661 int radeon_fence_wait_empty(struct radeon_device *rdev, int ring) in radeon_fence_wait_empty() argument
666 seq[ring] = rdev->fence_drv[ring].sync_seq[ring]; in radeon_fence_wait_empty()
670 r = radeon_fence_wait_seq_timeout(rdev, seq, false, MAX_SCHEDULE_TIMEOUT); in radeon_fence_wait_empty()
675 dev_err(rdev->dev, "error waiting for ring[%d] to become idle (%ld)\n", in radeon_fence_wait_empty()
722 unsigned radeon_fence_count_emitted(struct radeon_device *rdev, int ring) in radeon_fence_count_emitted() argument
729 radeon_fence_process(rdev, ring); in radeon_fence_count_emitted()
730 emitted = rdev->fence_drv[ring].sync_seq[ring] in radeon_fence_count_emitted()
731 - atomic64_read(&rdev->fence_drv[ring].last_seq); in radeon_fence_count_emitted()
763 fdrv = &fence->rdev->fence_drv[dst_ring]; in radeon_fence_need_sync()
794 src = &fence->rdev->fence_drv[fence->ring]; in radeon_fence_note_sync()
795 dst = &fence->rdev->fence_drv[dst_ring]; in radeon_fence_note_sync()
816 int radeon_fence_driver_start_ring(struct radeon_device *rdev, int ring) in radeon_fence_driver_start_ring() argument
821 radeon_scratch_free(rdev, rdev->fence_drv[ring].scratch_reg); in radeon_fence_driver_start_ring()
822 if (rdev->wb.use_event || !radeon_ring_supports_scratch_reg(rdev, &rdev->ring[ring])) { in radeon_fence_driver_start_ring()
823 rdev->fence_drv[ring].scratch_reg = 0; in radeon_fence_driver_start_ring()
826 rdev->fence_drv[ring].cpu_addr = &rdev->wb.wb[index/4]; in radeon_fence_driver_start_ring()
827 rdev->fence_drv[ring].gpu_addr = rdev->wb.gpu_addr + in radeon_fence_driver_start_ring()
832 index = ALIGN(rdev->uvd_fw->size, 8); in radeon_fence_driver_start_ring()
833 rdev->fence_drv[ring].cpu_addr = rdev->uvd.cpu_addr + index; in radeon_fence_driver_start_ring()
834 rdev->fence_drv[ring].gpu_addr = rdev->uvd.gpu_addr + index; in radeon_fence_driver_start_ring()
838 r = radeon_scratch_get(rdev, &rdev->fence_drv[ring].scratch_reg); in radeon_fence_driver_start_ring()
840 dev_err(rdev->dev, "fence failed to get scratch register\n"); in radeon_fence_driver_start_ring()
844 rdev->fence_drv[ring].scratch_reg - in radeon_fence_driver_start_ring()
845 rdev->scratch.reg_base; in radeon_fence_driver_start_ring()
846 rdev->fence_drv[ring].cpu_addr = &rdev->wb.wb[index/4]; in radeon_fence_driver_start_ring()
847 rdev->fence_drv[ring].gpu_addr = rdev->wb.gpu_addr + index; in radeon_fence_driver_start_ring()
849 radeon_fence_write(rdev, atomic64_read(&rdev->fence_drv[ring].last_seq), ring); in radeon_fence_driver_start_ring()
850 rdev->fence_drv[ring].initialized = true; in radeon_fence_driver_start_ring()
851 dev_info(rdev->dev, "fence driver on ring %d use gpu addr 0x%016llx\n", in radeon_fence_driver_start_ring()
852 ring, rdev->fence_drv[ring].gpu_addr); in radeon_fence_driver_start_ring()
866 static void radeon_fence_driver_init_ring(struct radeon_device *rdev, int ring) in radeon_fence_driver_init_ring() argument
870 rdev->fence_drv[ring].scratch_reg = -1; in radeon_fence_driver_init_ring()
871 rdev->fence_drv[ring].cpu_addr = NULL; in radeon_fence_driver_init_ring()
872 rdev->fence_drv[ring].gpu_addr = 0; in radeon_fence_driver_init_ring()
874 rdev->fence_drv[ring].sync_seq[i] = 0; in radeon_fence_driver_init_ring()
875 atomic64_set(&rdev->fence_drv[ring].last_seq, 0); in radeon_fence_driver_init_ring()
876 rdev->fence_drv[ring].initialized = false; in radeon_fence_driver_init_ring()
877 INIT_DELAYED_WORK(&rdev->fence_drv[ring].lockup_work, in radeon_fence_driver_init_ring()
879 rdev->fence_drv[ring].rdev = rdev; in radeon_fence_driver_init_ring()
893 void radeon_fence_driver_init(struct radeon_device *rdev) in radeon_fence_driver_init() argument
897 init_waitqueue_head(&rdev->fence_queue); in radeon_fence_driver_init()
899 radeon_fence_driver_init_ring(rdev, ring); in radeon_fence_driver_init()
902 radeon_debugfs_fence_init(rdev); in radeon_fence_driver_init()
913 void radeon_fence_driver_fini(struct radeon_device *rdev) in radeon_fence_driver_fini() argument
917 mutex_lock(&rdev->ring_lock); in radeon_fence_driver_fini()
919 if (!rdev->fence_drv[ring].initialized) in radeon_fence_driver_fini()
921 r = radeon_fence_wait_empty(rdev, ring); in radeon_fence_driver_fini()
924 radeon_fence_driver_force_completion(rdev, ring); in radeon_fence_driver_fini()
926 cancel_delayed_work_sync(&rdev->fence_drv[ring].lockup_work); in radeon_fence_driver_fini()
927 wake_up_all(&rdev->fence_queue); in radeon_fence_driver_fini()
928 radeon_scratch_free(rdev, rdev->fence_drv[ring].scratch_reg); in radeon_fence_driver_fini()
929 rdev->fence_drv[ring].initialized = false; in radeon_fence_driver_fini()
931 mutex_unlock(&rdev->ring_lock); in radeon_fence_driver_fini()
943 void radeon_fence_driver_force_completion(struct radeon_device *rdev, int ring) in radeon_fence_driver_force_completion() argument
945 if (rdev->fence_drv[ring].initialized) { in radeon_fence_driver_force_completion()
946 radeon_fence_write(rdev, rdev->fence_drv[ring].sync_seq[ring], ring); in radeon_fence_driver_force_completion()
947 cancel_delayed_work_sync(&rdev->fence_drv[ring].lockup_work); in radeon_fence_driver_force_completion()
958 struct radeon_device *rdev = (struct radeon_device *)m->private; in radeon_debugfs_fence_info_show() local
962 if (!rdev->fence_drv[i].initialized) in radeon_debugfs_fence_info_show()
965 radeon_fence_process(rdev, i); in radeon_debugfs_fence_info_show()
969 (unsigned long long)atomic64_read(&rdev->fence_drv[i].last_seq)); in radeon_debugfs_fence_info_show()
971 rdev->fence_drv[i].sync_seq[i]); in radeon_debugfs_fence_info_show()
974 if (i != j && rdev->fence_drv[j].initialized) in radeon_debugfs_fence_info_show()
976 j, rdev->fence_drv[i].sync_seq[j]); in radeon_debugfs_fence_info_show()
989 struct radeon_device *rdev = (struct radeon_device *)data; in radeon_debugfs_gpu_reset() local
991 down_read(&rdev->exclusive_lock); in radeon_debugfs_gpu_reset()
992 *val = rdev->needs_reset; in radeon_debugfs_gpu_reset()
993 rdev->needs_reset = true; in radeon_debugfs_gpu_reset()
994 wake_up_all(&rdev->fence_queue); in radeon_debugfs_gpu_reset()
995 up_read(&rdev->exclusive_lock); in radeon_debugfs_gpu_reset()
1004 void radeon_debugfs_fence_init(struct radeon_device *rdev) in radeon_debugfs_fence_init() argument
1007 struct dentry *root = rdev->ddev->primary->debugfs_root; in radeon_debugfs_fence_init()
1009 debugfs_create_file("radeon_gpu_reset", 0444, root, rdev, in radeon_debugfs_fence_init()
1011 debugfs_create_file("radeon_fence_info", 0444, root, rdev, in radeon_debugfs_fence_init()
1062 struct radeon_device *rdev = fence->rdev; in radeon_fence_default_wait() local
1083 if (rdev->needs_reset) { in radeon_fence_default_wait()