Lines Matching refs:ring
61 int amdgpu_ring_alloc(struct amdgpu_ring *ring, unsigned ndw) in amdgpu_ring_alloc() argument
65 ndw = (ndw + ring->funcs->align_mask) & ~ring->funcs->align_mask; in amdgpu_ring_alloc()
70 if (WARN_ON_ONCE(ndw > ring->max_dw)) in amdgpu_ring_alloc()
73 ring->count_dw = ndw; in amdgpu_ring_alloc()
74 ring->wptr_old = ring->wptr; in amdgpu_ring_alloc()
76 if (ring->funcs->begin_use) in amdgpu_ring_alloc()
77 ring->funcs->begin_use(ring); in amdgpu_ring_alloc()
89 void amdgpu_ring_insert_nop(struct amdgpu_ring *ring, uint32_t count) in amdgpu_ring_insert_nop() argument
94 amdgpu_ring_write(ring, ring->funcs->nop); in amdgpu_ring_insert_nop()
105 void amdgpu_ring_generic_pad_ib(struct amdgpu_ring *ring, struct amdgpu_ib *ib) in amdgpu_ring_generic_pad_ib() argument
107 while (ib->length_dw & ring->funcs->align_mask) in amdgpu_ring_generic_pad_ib()
108 ib->ptr[ib->length_dw++] = ring->funcs->nop; in amdgpu_ring_generic_pad_ib()
120 void amdgpu_ring_commit(struct amdgpu_ring *ring) in amdgpu_ring_commit() argument
125 count = ring->funcs->align_mask + 1 - in amdgpu_ring_commit()
126 (ring->wptr & ring->funcs->align_mask); in amdgpu_ring_commit()
127 count %= ring->funcs->align_mask + 1; in amdgpu_ring_commit()
128 ring->funcs->insert_nop(ring, count); in amdgpu_ring_commit()
131 amdgpu_ring_set_wptr(ring); in amdgpu_ring_commit()
133 if (ring->funcs->end_use) in amdgpu_ring_commit()
134 ring->funcs->end_use(ring); in amdgpu_ring_commit()
144 void amdgpu_ring_undo(struct amdgpu_ring *ring) in amdgpu_ring_undo() argument
146 ring->wptr = ring->wptr_old; in amdgpu_ring_undo()
148 if (ring->funcs->end_use) in amdgpu_ring_undo()
149 ring->funcs->end_use(ring); in amdgpu_ring_undo()
152 #define amdgpu_ring_get_gpu_addr(ring, offset) \ argument
153 (ring->is_mes_queue ? \
154 (ring->mes_ctx->meta_data_gpu_addr + offset) : \
155 (ring->adev->wb.gpu_addr + offset * 4))
157 #define amdgpu_ring_get_cpu_addr(ring, offset) \ argument
158 (ring->is_mes_queue ? \
159 (void *)((uint8_t *)(ring->mes_ctx->meta_data_ptr) + offset) : \
160 (&ring->adev->wb.wb[offset]))
176 int amdgpu_ring_init(struct amdgpu_device *adev, struct amdgpu_ring *ring, in amdgpu_ring_init() argument
192 if (ring->funcs->type == AMDGPU_RING_TYPE_KIQ) in amdgpu_ring_init()
194 else if (ring == &adev->sdma.instance[0].page) in amdgpu_ring_init()
197 if (ring->adev == NULL) { in amdgpu_ring_init()
201 ring->adev = adev; in amdgpu_ring_init()
202 ring->num_hw_submission = sched_hw_submission; in amdgpu_ring_init()
203 ring->sched_score = sched_score; in amdgpu_ring_init()
204 ring->vmid_wait = dma_fence_get_stub(); in amdgpu_ring_init()
206 if (!ring->is_mes_queue) { in amdgpu_ring_init()
207 ring->idx = adev->num_rings++; in amdgpu_ring_init()
208 adev->rings[ring->idx] = ring; in amdgpu_ring_init()
211 r = amdgpu_fence_driver_init_ring(ring); in amdgpu_ring_init()
216 if (ring->is_mes_queue) { in amdgpu_ring_init()
217 ring->rptr_offs = amdgpu_mes_ctx_get_offs(ring, in amdgpu_ring_init()
219 ring->wptr_offs = amdgpu_mes_ctx_get_offs(ring, in amdgpu_ring_init()
221 ring->fence_offs = amdgpu_mes_ctx_get_offs(ring, in amdgpu_ring_init()
223 ring->trail_fence_offs = amdgpu_mes_ctx_get_offs(ring, in amdgpu_ring_init()
225 ring->cond_exe_offs = amdgpu_mes_ctx_get_offs(ring, in amdgpu_ring_init()
228 r = amdgpu_device_wb_get(adev, &ring->rptr_offs); in amdgpu_ring_init()
234 r = amdgpu_device_wb_get(adev, &ring->wptr_offs); in amdgpu_ring_init()
240 r = amdgpu_device_wb_get(adev, &ring->fence_offs); in amdgpu_ring_init()
246 r = amdgpu_device_wb_get(adev, &ring->trail_fence_offs); in amdgpu_ring_init()
252 r = amdgpu_device_wb_get(adev, &ring->cond_exe_offs); in amdgpu_ring_init()
259 ring->fence_gpu_addr = in amdgpu_ring_init()
260 amdgpu_ring_get_gpu_addr(ring, ring->fence_offs); in amdgpu_ring_init()
261 ring->fence_cpu_addr = in amdgpu_ring_init()
262 amdgpu_ring_get_cpu_addr(ring, ring->fence_offs); in amdgpu_ring_init()
264 ring->rptr_gpu_addr = in amdgpu_ring_init()
265 amdgpu_ring_get_gpu_addr(ring, ring->rptr_offs); in amdgpu_ring_init()
266 ring->rptr_cpu_addr = in amdgpu_ring_init()
267 amdgpu_ring_get_cpu_addr(ring, ring->rptr_offs); in amdgpu_ring_init()
269 ring->wptr_gpu_addr = in amdgpu_ring_init()
270 amdgpu_ring_get_gpu_addr(ring, ring->wptr_offs); in amdgpu_ring_init()
271 ring->wptr_cpu_addr = in amdgpu_ring_init()
272 amdgpu_ring_get_cpu_addr(ring, ring->wptr_offs); in amdgpu_ring_init()
274 ring->trail_fence_gpu_addr = in amdgpu_ring_init()
275 amdgpu_ring_get_gpu_addr(ring, ring->trail_fence_offs); in amdgpu_ring_init()
276 ring->trail_fence_cpu_addr = in amdgpu_ring_init()
277 amdgpu_ring_get_cpu_addr(ring, ring->trail_fence_offs); in amdgpu_ring_init()
279 ring->cond_exe_gpu_addr = in amdgpu_ring_init()
280 amdgpu_ring_get_gpu_addr(ring, ring->cond_exe_offs); in amdgpu_ring_init()
281 ring->cond_exe_cpu_addr = in amdgpu_ring_init()
282 amdgpu_ring_get_cpu_addr(ring, ring->cond_exe_offs); in amdgpu_ring_init()
285 *ring->cond_exe_cpu_addr = 1; in amdgpu_ring_init()
287 r = amdgpu_fence_driver_start_ring(ring, irq_src, irq_type); in amdgpu_ring_init()
293 ring->ring_size = roundup_pow_of_two(max_dw * 4 * sched_hw_submission); in amdgpu_ring_init()
295 ring->buf_mask = (ring->ring_size / 4) - 1; in amdgpu_ring_init()
296 ring->ptr_mask = ring->funcs->support_64bit_ptrs ? in amdgpu_ring_init()
297 0xffffffffffffffff : ring->buf_mask; in amdgpu_ring_init()
300 if (ring->is_mes_queue) { in amdgpu_ring_init()
303 BUG_ON(ring->ring_size > PAGE_SIZE*4); in amdgpu_ring_init()
305 offset = amdgpu_mes_ctx_get_offs(ring, in amdgpu_ring_init()
307 ring->gpu_addr = amdgpu_mes_ctx_get_offs_gpu_addr(ring, offset); in amdgpu_ring_init()
308 ring->ring = amdgpu_mes_ctx_get_offs_cpu_addr(ring, offset); in amdgpu_ring_init()
309 amdgpu_ring_clear_ring(ring); in amdgpu_ring_init()
311 } else if (ring->ring_obj == NULL) { in amdgpu_ring_init()
312 r = amdgpu_bo_create_kernel(adev, ring->ring_size + ring->funcs->extra_dw, PAGE_SIZE, in amdgpu_ring_init()
314 &ring->ring_obj, in amdgpu_ring_init()
315 &ring->gpu_addr, in amdgpu_ring_init()
316 (void **)&ring->ring); in amdgpu_ring_init()
321 amdgpu_ring_clear_ring(ring); in amdgpu_ring_init()
324 ring->max_dw = max_dw; in amdgpu_ring_init()
325 ring->hw_prio = hw_prio; in amdgpu_ring_init()
327 if (!ring->no_scheduler) { in amdgpu_ring_init()
328 hw_ip = ring->funcs->type; in amdgpu_ring_init()
331 &ring->sched; in amdgpu_ring_init()
344 void amdgpu_ring_fini(struct amdgpu_ring *ring) in amdgpu_ring_fini() argument
348 if (!(ring->adev) || in amdgpu_ring_fini()
349 (!ring->is_mes_queue && !(ring->adev->rings[ring->idx]))) in amdgpu_ring_fini()
352 ring->sched.ready = false; in amdgpu_ring_fini()
354 if (!ring->is_mes_queue) { in amdgpu_ring_fini()
355 amdgpu_device_wb_free(ring->adev, ring->rptr_offs); in amdgpu_ring_fini()
356 amdgpu_device_wb_free(ring->adev, ring->wptr_offs); in amdgpu_ring_fini()
358 amdgpu_device_wb_free(ring->adev, ring->cond_exe_offs); in amdgpu_ring_fini()
359 amdgpu_device_wb_free(ring->adev, ring->fence_offs); in amdgpu_ring_fini()
361 amdgpu_bo_free_kernel(&ring->ring_obj, in amdgpu_ring_fini()
362 &ring->gpu_addr, in amdgpu_ring_fini()
363 (void **)&ring->ring); in amdgpu_ring_fini()
366 dma_fence_put(ring->vmid_wait); in amdgpu_ring_fini()
367 ring->vmid_wait = NULL; in amdgpu_ring_fini()
368 ring->me = 0; in amdgpu_ring_fini()
370 if (!ring->is_mes_queue) in amdgpu_ring_fini()
371 ring->adev->rings[ring->idx] = NULL; in amdgpu_ring_fini()
386 void amdgpu_ring_emit_reg_write_reg_wait_helper(struct amdgpu_ring *ring, in amdgpu_ring_emit_reg_write_reg_wait_helper() argument
390 amdgpu_ring_emit_wreg(ring, reg0, ref); in amdgpu_ring_emit_reg_write_reg_wait_helper()
391 amdgpu_ring_emit_reg_wait(ring, reg1, mask, mask); in amdgpu_ring_emit_reg_write_reg_wait_helper()
403 bool amdgpu_ring_soft_recovery(struct amdgpu_ring *ring, unsigned int vmid, in amdgpu_ring_soft_recovery() argument
408 if (amdgpu_sriov_vf(ring->adev) || !ring->funcs->soft_recovery || !fence) in amdgpu_ring_soft_recovery()
411 atomic_inc(&ring->adev->gpu_reset_counter); in amdgpu_ring_soft_recovery()
414 ring->funcs->soft_recovery(ring, vmid); in amdgpu_ring_soft_recovery()
434 struct amdgpu_ring *ring = file_inode(f)->i_private; in amdgpu_debugfs_ring_read() local
444 early[0] = amdgpu_ring_get_rptr(ring) & ring->buf_mask; in amdgpu_debugfs_ring_read()
445 early[1] = amdgpu_ring_get_wptr(ring) & ring->buf_mask; in amdgpu_debugfs_ring_read()
446 early[2] = ring->wptr & ring->buf_mask; in amdgpu_debugfs_ring_read()
459 if (*pos >= (ring->ring_size + 12)) in amdgpu_debugfs_ring_read()
462 value = ring->ring[(*pos - 12)/4]; in amdgpu_debugfs_ring_read()
484 struct amdgpu_ring *ring) in amdgpu_debugfs_ring_init() argument
491 sprintf(name, "amdgpu_ring_%s", ring->name); in amdgpu_debugfs_ring_init()
492 debugfs_create_file_size(name, S_IFREG | S_IRUGO, root, ring, in amdgpu_debugfs_ring_init()
494 ring->ring_size + 12); in amdgpu_debugfs_ring_init()
508 int amdgpu_ring_test_helper(struct amdgpu_ring *ring) in amdgpu_ring_test_helper() argument
510 struct amdgpu_device *adev = ring->adev; in amdgpu_ring_test_helper()
513 r = amdgpu_ring_test_ring(ring); in amdgpu_ring_test_helper()
516 ring->name, r); in amdgpu_ring_test_helper()
519 ring->name); in amdgpu_ring_test_helper()
521 ring->sched.ready = !r; in amdgpu_ring_test_helper()
525 static void amdgpu_ring_to_mqd_prop(struct amdgpu_ring *ring, in amdgpu_ring_to_mqd_prop() argument
528 struct amdgpu_device *adev = ring->adev; in amdgpu_ring_to_mqd_prop()
532 prop->mqd_gpu_addr = ring->mqd_gpu_addr; in amdgpu_ring_to_mqd_prop()
533 prop->hqd_base_gpu_addr = ring->gpu_addr; in amdgpu_ring_to_mqd_prop()
534 prop->rptr_gpu_addr = ring->rptr_gpu_addr; in amdgpu_ring_to_mqd_prop()
535 prop->wptr_gpu_addr = ring->wptr_gpu_addr; in amdgpu_ring_to_mqd_prop()
536 prop->queue_size = ring->ring_size; in amdgpu_ring_to_mqd_prop()
537 prop->eop_gpu_addr = ring->eop_gpu_addr; in amdgpu_ring_to_mqd_prop()
538 prop->use_doorbell = ring->use_doorbell; in amdgpu_ring_to_mqd_prop()
539 prop->doorbell_index = ring->doorbell_index; in amdgpu_ring_to_mqd_prop()
544 prop->hqd_active = ring->funcs->type == AMDGPU_RING_TYPE_KIQ; in amdgpu_ring_to_mqd_prop()
546 if ((ring->funcs->type == AMDGPU_RING_TYPE_COMPUTE && in amdgpu_ring_to_mqd_prop()
547 amdgpu_gfx_is_high_priority_compute_queue(adev, ring)) || in amdgpu_ring_to_mqd_prop()
548 (ring->funcs->type == AMDGPU_RING_TYPE_GFX && in amdgpu_ring_to_mqd_prop()
549 amdgpu_gfx_is_high_priority_graphics_queue(adev, ring))) { in amdgpu_ring_to_mqd_prop()
555 int amdgpu_ring_init_mqd(struct amdgpu_ring *ring) in amdgpu_ring_init_mqd() argument
557 struct amdgpu_device *adev = ring->adev; in amdgpu_ring_init_mqd()
561 amdgpu_ring_to_mqd_prop(ring, &prop); in amdgpu_ring_init_mqd()
563 ring->wptr = 0; in amdgpu_ring_init_mqd()
565 if (ring->funcs->type == AMDGPU_RING_TYPE_KIQ) in amdgpu_ring_init_mqd()
568 mqd_mgr = &adev->mqds[ring->funcs->type]; in amdgpu_ring_init_mqd()
570 return mqd_mgr->init_mqd(adev, ring->mqd_ptr, &prop); in amdgpu_ring_init_mqd()
573 void amdgpu_ring_ib_begin(struct amdgpu_ring *ring) in amdgpu_ring_ib_begin() argument
575 if (ring->is_sw_ring) in amdgpu_ring_ib_begin()
576 amdgpu_sw_ring_ib_begin(ring); in amdgpu_ring_ib_begin()
579 void amdgpu_ring_ib_end(struct amdgpu_ring *ring) in amdgpu_ring_ib_end() argument
581 if (ring->is_sw_ring) in amdgpu_ring_ib_end()
582 amdgpu_sw_ring_ib_end(ring); in amdgpu_ring_ib_end()