Lines Matching refs:qpd
146 struct qcm_process_device *qpd) in program_sh_mem_settings() argument
153 dqm->dev->adev, qpd->vmid, qpd->sh_mem_config, in program_sh_mem_settings()
154 qpd->sh_mem_ape1_base, qpd->sh_mem_ape1_limit, in program_sh_mem_settings()
155 qpd->sh_mem_bases, xcc_id); in program_sh_mem_settings()
161 struct qcm_process_device *qpd; in kfd_hws_hang() local
166 qpd = cur->qpd; in kfd_hws_hang()
167 list_for_each_entry(q, &qpd->queues_list, list) { in kfd_hws_hang()
168 struct kfd_process_device *pdd = qpd_to_pdd(qpd); in kfd_hws_hang()
201 struct qcm_process_device *qpd) in add_queue_mes() argument
204 struct kfd_process_device *pdd = qpd_to_pdd(qpd); in add_queue_mes()
216 queue_input.page_table_base_addr = qpd->page_table_base; in add_queue_mes()
239 queue_input.tba_addr = qpd->tba_addr; in add_queue_mes()
240 queue_input.tma_addr = qpd->tma_addr; in add_queue_mes()
243 qpd->pqm->process->runtime_info.runtime_state == DEBUG_RUNTIME_STATE_ENABLED && in add_queue_mes()
244 (qpd->pqm->process->debug_trap_enabled || in add_queue_mes()
273 struct qcm_process_device *qpd) in remove_queue_mes() argument
307 struct qcm_process_device *qpd; in remove_all_kfd_queues_mes() local
312 qpd = cur->qpd; in remove_all_kfd_queues_mes()
313 list_for_each_entry(q, &qpd->queues_list, list) { in remove_all_kfd_queues_mes()
315 retval = remove_queue_mes(dqm, q, qpd); in remove_all_kfd_queues_mes()
334 struct qcm_process_device *qpd; in add_all_kfd_queues_mes() local
339 qpd = cur->qpd; in add_all_kfd_queues_mes()
340 list_for_each_entry(q, &qpd->queues_list, list) { in add_all_kfd_queues_mes()
343 retval = add_queue_mes(dqm, q, qpd); in add_all_kfd_queues_mes()
398 struct qcm_process_device *qpd, in increment_queue_count() argument
408 qpd->mapped_gws_queue = true; in increment_queue_count()
413 struct qcm_process_device *qpd, in decrement_queue_count() argument
423 qpd->mapped_gws_queue = false; in decrement_queue_count()
431 static int allocate_doorbell(struct qcm_process_device *qpd, in allocate_doorbell() argument
435 struct kfd_node *dev = qpd->dqm->dev; in allocate_doorbell()
462 uint32_t valid_id = idx_offset[qpd->dqm->dev->node_id * in allocate_doorbell()
463 get_num_all_sdma_engines(qpd->dqm) + in allocate_doorbell()
476 if (__test_and_set_bit(*restore_id, qpd->doorbell_bitmap)) in allocate_doorbell()
484 found = find_first_zero_bit(qpd->doorbell_bitmap, in allocate_doorbell()
490 set_bit(found, qpd->doorbell_bitmap); in allocate_doorbell()
496 qpd->proc_doorbells, in allocate_doorbell()
502 static void deallocate_doorbell(struct qcm_process_device *qpd, in deallocate_doorbell() argument
506 struct kfd_node *dev = qpd->dqm->dev; in deallocate_doorbell()
513 old = test_and_clear_bit(q->doorbell_id, qpd->doorbell_bitmap); in deallocate_doorbell()
518 struct qcm_process_device *qpd) in program_trap_handler_settings() argument
526 dqm->dev->adev, qpd->vmid, qpd->tba_addr, in program_trap_handler_settings()
527 qpd->tma_addr, xcc_id); in program_trap_handler_settings()
531 struct qcm_process_device *qpd, in allocate_vmid() argument
534 struct kfd_process_device *pdd = qpd_to_pdd(qpd); in allocate_vmid()
557 qpd->vmid = allocated_vmid; in allocate_vmid()
560 program_sh_mem_settings(dqm, qpd); in allocate_vmid()
563 program_trap_handler_settings(dqm, qpd); in allocate_vmid()
569 qpd->vmid, in allocate_vmid()
570 qpd->page_table_base); in allocate_vmid()
572 kfd_flush_tlb(qpd_to_pdd(qpd), TLB_FLUSH_LEGACY); in allocate_vmid()
576 qpd->sh_hidden_private_base, qpd->vmid); in allocate_vmid()
582 struct qcm_process_device *qpd) in flush_texture_cache_nocpsch() argument
584 const struct packet_manager_funcs *pmf = qpd->dqm->packet_mgr.pmf; in flush_texture_cache_nocpsch()
587 if (!qpd->ib_kaddr) in flush_texture_cache_nocpsch()
590 ret = pmf->release_mem(qpd->ib_base, (uint32_t *)qpd->ib_kaddr); in flush_texture_cache_nocpsch()
594 return amdgpu_amdkfd_submit_ib(kdev->adev, KGD_ENGINE_MEC1, qpd->vmid, in flush_texture_cache_nocpsch()
595 qpd->ib_base, (uint32_t *)qpd->ib_kaddr, in flush_texture_cache_nocpsch()
600 struct qcm_process_device *qpd, in deallocate_vmid() argument
607 if (flush_texture_cache_nocpsch(q->device, qpd)) in deallocate_vmid()
610 kfd_flush_tlb(qpd_to_pdd(qpd), TLB_FLUSH_LEGACY); in deallocate_vmid()
613 set_pasid_vmid_mapping(dqm, 0, qpd->vmid); in deallocate_vmid()
614 dqm->vmid_pasid[qpd->vmid] = 0; in deallocate_vmid()
616 qpd->vmid = 0; in deallocate_vmid()
622 struct qcm_process_device *qpd, in create_queue_nocpsch() argument
638 if (list_empty(&qpd->queues_list)) { in create_queue_nocpsch()
639 retval = allocate_vmid(dqm, qpd, q); in create_queue_nocpsch()
643 q->properties.vmid = qpd->vmid; in create_queue_nocpsch()
649 q->properties.is_evicted = !!qpd->evicted; in create_queue_nocpsch()
651 q->properties.tba_addr = qpd->tba_addr; in create_queue_nocpsch()
652 q->properties.tma_addr = qpd->tma_addr; in create_queue_nocpsch()
667 dqm->asic_ops.init_sdma_vm(dqm, q, qpd); in create_queue_nocpsch()
670 retval = allocate_doorbell(qpd, q, qd ? &qd->doorbell_id : NULL); in create_queue_nocpsch()
709 list_add(&q->list, &qpd->queues_list); in create_queue_nocpsch()
710 qpd->queue_count++; in create_queue_nocpsch()
712 increment_queue_count(dqm, qpd, q); in create_queue_nocpsch()
726 deallocate_doorbell(qpd, q); in create_queue_nocpsch()
734 if (list_empty(&qpd->queues_list)) in create_queue_nocpsch()
735 deallocate_vmid(dqm, qpd, q); in create_queue_nocpsch()
853 struct qcm_process_device *qpd, in destroy_queue_nocpsch_locked() argument
875 deallocate_doorbell(qpd, q); in destroy_queue_nocpsch_locked()
887 qpd->reset_wavefronts = true; in destroy_queue_nocpsch_locked()
890 if (list_empty(&qpd->queues_list)) { in destroy_queue_nocpsch_locked()
891 if (qpd->reset_wavefronts) { in destroy_queue_nocpsch_locked()
898 qpd->pqm->process); in destroy_queue_nocpsch_locked()
899 qpd->reset_wavefronts = false; in destroy_queue_nocpsch_locked()
902 deallocate_vmid(dqm, qpd, q); in destroy_queue_nocpsch_locked()
904 qpd->queue_count--; in destroy_queue_nocpsch_locked()
906 decrement_queue_count(dqm, qpd, q); in destroy_queue_nocpsch_locked()
912 struct qcm_process_device *qpd, in destroy_queue_nocpsch() argument
918 struct kfd_process_device *pdd = qpd_to_pdd(qpd); in destroy_queue_nocpsch()
933 retval = destroy_queue_nocpsch_locked(dqm, qpd, q); in destroy_queue_nocpsch()
970 retval = remove_queue_mes(dqm, q, &pdd->qpd); in update_queue()
1012 increment_queue_count(dqm, &pdd->qpd, q); in update_queue()
1014 decrement_queue_count(dqm, &pdd->qpd, q); in update_queue()
1018 pdd->qpd.mapped_gws_queue = true; in update_queue()
1024 pdd->qpd.mapped_gws_queue = false; in update_queue()
1033 retval = add_queue_mes(dqm, q, &pdd->qpd); in update_queue()
1085 int r = remove_queue_mes(dqm, q, &pdd->qpd); in suspend_single_queue()
1091 decrement_queue_count(dqm, &pdd->qpd, q); in suspend_single_queue()
1107 struct qcm_process_device *qpd, in resume_single_queue() argument
1115 pdd = qpd_to_pdd(qpd); in resume_single_queue()
1125 int r = add_queue_mes(dqm, q, &pdd->qpd); in resume_single_queue()
1132 increment_queue_count(dqm, qpd, q); in resume_single_queue()
1139 struct qcm_process_device *qpd) in evict_process_queues_nocpsch() argument
1147 if (qpd->evicted++ > 0) /* already evicted, do nothing */ in evict_process_queues_nocpsch()
1150 pdd = qpd_to_pdd(qpd); in evict_process_queues_nocpsch()
1158 list_for_each_entry(q, &qpd->queues_list, list) { in evict_process_queues_nocpsch()
1166 decrement_queue_count(dqm, qpd, q); in evict_process_queues_nocpsch()
1189 struct qcm_process_device *qpd) in evict_process_queues_cpsch() argument
1197 if (qpd->evicted++ > 0) /* already evicted, do nothing */ in evict_process_queues_cpsch()
1200 pdd = qpd_to_pdd(qpd); in evict_process_queues_cpsch()
1215 list_for_each_entry(q, &qpd->queues_list, list) { in evict_process_queues_cpsch()
1221 decrement_queue_count(dqm, qpd, q); in evict_process_queues_cpsch()
1226 err = remove_queue_mes(dqm, q, qpd); in evict_process_queues_cpsch()
1237 qpd->is_debug ? in evict_process_queues_cpsch()
1248 struct qcm_process_device *qpd) in restore_process_queues_nocpsch() argument
1258 pdd = qpd_to_pdd(qpd); in restore_process_queues_nocpsch()
1263 if (WARN_ON_ONCE(!qpd->evicted)) /* already restored, do nothing */ in restore_process_queues_nocpsch()
1265 if (qpd->evicted > 1) { /* ref count still > 0, decrement & quit */ in restore_process_queues_nocpsch()
1266 qpd->evicted--; in restore_process_queues_nocpsch()
1274 qpd->page_table_base = pd_base; in restore_process_queues_nocpsch()
1277 if (!list_empty(&qpd->queues_list)) { in restore_process_queues_nocpsch()
1280 qpd->vmid, in restore_process_queues_nocpsch()
1281 qpd->page_table_base); in restore_process_queues_nocpsch()
1297 list_for_each_entry(q, &qpd->queues_list, list) { in restore_process_queues_nocpsch()
1305 increment_queue_count(dqm, qpd, q); in restore_process_queues_nocpsch()
1318 qpd->evicted = 0; in restore_process_queues_nocpsch()
1329 struct qcm_process_device *qpd) in restore_process_queues_cpsch() argument
1337 pdd = qpd_to_pdd(qpd); in restore_process_queues_cpsch()
1340 if (WARN_ON_ONCE(!qpd->evicted)) /* already restored, do nothing */ in restore_process_queues_cpsch()
1342 if (qpd->evicted > 1) { /* ref count still > 0, decrement & quit */ in restore_process_queues_cpsch()
1343 qpd->evicted--; in restore_process_queues_cpsch()
1358 qpd->page_table_base = amdgpu_amdkfd_gpuvm_get_process_page_dir(pdd->drm_priv); in restore_process_queues_cpsch()
1359 pr_debug("Updated PD address to 0x%llx\n", qpd->page_table_base); in restore_process_queues_cpsch()
1362 list_for_each_entry(q, &qpd->queues_list, list) { in restore_process_queues_cpsch()
1368 increment_queue_count(dqm, &pdd->qpd, q); in restore_process_queues_cpsch()
1371 retval = add_queue_mes(dqm, q, qpd); in restore_process_queues_cpsch()
1385 qpd->evicted = 0; in restore_process_queues_cpsch()
1392 struct qcm_process_device *qpd) in register_process() argument
1403 n->qpd = qpd; in register_process()
1405 pdd = qpd_to_pdd(qpd); in register_process()
1413 qpd->page_table_base = pd_base; in register_process()
1416 retval = dqm->asic_ops.update_qpd(dqm, qpd); in register_process()
1431 struct qcm_process_device *qpd) in unregister_process() argument
1437 list_empty(&qpd->queues_list) ? "empty" : "not empty"); in unregister_process()
1443 if (qpd == cur->qpd) { in unregister_process()
1915 struct qcm_process_device *qpd) in create_kernel_queue_cpsch() argument
1933 list_add(&kq->list, &qpd->priv_queue_list); in create_kernel_queue_cpsch()
1934 increment_queue_count(dqm, qpd, kq->queue); in create_kernel_queue_cpsch()
1935 qpd->is_debug = true; in create_kernel_queue_cpsch()
1945 struct qcm_process_device *qpd) in destroy_kernel_queue_cpsch() argument
1949 decrement_queue_count(dqm, qpd, kq->queue); in destroy_kernel_queue_cpsch()
1950 qpd->is_debug = false; in destroy_kernel_queue_cpsch()
1964 struct qcm_process_device *qpd, in create_queue_cpsch() argument
1988 retval = allocate_doorbell(qpd, q, qd ? &qd->doorbell_id : NULL); in create_queue_cpsch()
1997 dqm->asic_ops.init_sdma_vm(dqm, q, qpd); in create_queue_cpsch()
1998 q->properties.tba_addr = qpd->tba_addr; in create_queue_cpsch()
1999 q->properties.tma_addr = qpd->tma_addr; in create_queue_cpsch()
2012 q->properties.is_evicted = !!qpd->evicted; in create_queue_cpsch()
2013 q->properties.is_dbg_wa = qpd->pqm->process->debug_trap_enabled && in create_queue_cpsch()
2024 list_add(&q->list, &qpd->queues_list); in create_queue_cpsch()
2025 qpd->queue_count++; in create_queue_cpsch()
2028 increment_queue_count(dqm, qpd, q); in create_queue_cpsch()
2034 retval = add_queue_mes(dqm, q, qpd); in create_queue_cpsch()
2052 qpd->queue_count--; in create_queue_cpsch()
2055 decrement_queue_count(dqm, qpd, q); in create_queue_cpsch()
2059 deallocate_doorbell(qpd, q); in create_queue_cpsch()
2126 struct qcm_process_device *qpd) in set_queue_as_reset() argument
2128 struct kfd_process_device *pdd = qpd_to_pdd(qpd); in set_queue_as_reset()
2136 decrement_queue_count(dqm, qpd, q); in set_queue_as_reset()
2186 struct qcm_process_device *qpd; in find_queue_by_address() local
2190 qpd = cur->qpd; in find_queue_by_address()
2191 list_for_each_entry(q, &qpd->queues_list, list) { in find_queue_by_address()
2238 set_queue_as_reset(dqm, q, &pdd->qpd); in reset_hung_queues()
2276 struct qcm_process_device *qpd; in set_sdma_queue_as_reset() local
2280 qpd = cur->qpd; in set_sdma_queue_as_reset()
2281 list_for_each_entry(q, &qpd->queues_list, list) { in set_sdma_queue_as_reset()
2285 set_queue_as_reset(dqm, q, qpd); in set_sdma_queue_as_reset()
2487 if (pdd->qpd.is_debug) in wait_on_destroy_queue()
2506 struct qcm_process_device *qpd, in destroy_queue_cpsch() argument
2512 struct kfd_process_device *pdd = qpd_to_pdd(qpd); in destroy_queue_cpsch()
2535 if (qpd->is_debug) { in destroy_queue_cpsch()
2548 deallocate_doorbell(qpd, q); in destroy_queue_cpsch()
2557 decrement_queue_count(dqm, qpd, q); in destroy_queue_cpsch()
2564 qpd->reset_wavefronts = true; in destroy_queue_cpsch()
2566 retval = remove_queue_mes(dqm, q, qpd); in destroy_queue_cpsch()
2570 qpd->queue_count--; in destroy_queue_cpsch()
2587 qpd->pqm->process, q->device, in destroy_queue_cpsch()
2601 struct qcm_process_device *qpd, in set_cache_memory_policy() argument
2617 qpd, in set_cache_memory_policy()
2627 if ((dqm->sched_policy == KFD_SCHED_POLICY_NO_HWS) && (qpd->vmid != 0)) in set_cache_memory_policy()
2628 program_sh_mem_settings(dqm, qpd); in set_cache_memory_policy()
2631 qpd->sh_mem_config, qpd->sh_mem_ape1_base, in set_cache_memory_policy()
2632 qpd->sh_mem_ape1_limit); in set_cache_memory_policy()
2640 struct qcm_process_device *qpd) in process_termination_nocpsch() argument
2650 while (!list_empty(&qpd->queues_list)) { in process_termination_nocpsch()
2654 q = list_first_entry(&qpd->queues_list, struct queue, list); in process_termination_nocpsch()
2657 ret = destroy_queue_nocpsch_locked(dqm, qpd, q); in process_termination_nocpsch()
2667 if (qpd == cur->qpd) { in process_termination_nocpsch()
2768 struct qcm_process_device *qpd) in process_termination_cpsch() argument
2785 list_for_each_entry_safe(kq, kq_next, &qpd->priv_queue_list, list) { in process_termination_cpsch()
2787 decrement_queue_count(dqm, qpd, kq->queue); in process_termination_cpsch()
2788 qpd->is_debug = false; in process_termination_cpsch()
2794 list_for_each_entry(q, &qpd->queues_list, list) { in process_termination_cpsch()
2801 decrement_queue_count(dqm, qpd, q); in process_termination_cpsch()
2804 retval = remove_queue_mes(dqm, q, qpd); in process_termination_cpsch()
2816 if (qpd == cur->qpd) { in process_termination_cpsch()
2828 if ((retval || qpd->reset_wavefronts) && in process_termination_cpsch()
2831 dbgdev_wave_reset_wavefronts(dqm->dev, qpd->pqm->process); in process_termination_cpsch()
2832 qpd->reset_wavefronts = false; in process_termination_cpsch()
2839 while (!list_empty(&qpd->queues_list)) { in process_termination_cpsch()
2840 q = list_first_entry(&qpd->queues_list, struct queue, list); in process_termination_cpsch()
2844 qpd->queue_count--; in process_termination_cpsch()
3056 struct qcm_process_device *qpd; in kfd_dqm_suspend_bad_queue_mes() local
3066 qpd = &pdd->qpd; in kfd_dqm_suspend_bad_queue_mes()
3068 list_for_each_entry(q, &qpd->queues_list, list) { in kfd_dqm_suspend_bad_queue_mes()
3078 decrement_queue_count(dqm, qpd, q); in kfd_dqm_suspend_bad_queue_mes()
3080 ret = remove_queue_mes(dqm, q, qpd); in kfd_dqm_suspend_bad_queue_mes()
3102 struct qcm_process_device *qpd) in kfd_dqm_evict_pasid_mes() argument
3109 if (qpd->evicted) { in kfd_dqm_evict_pasid_mes()
3113 qpd->evicted++; in kfd_dqm_evict_pasid_mes()
3125 ret = dqm->ops.evict_process_queues(dqm, qpd); in kfd_dqm_evict_pasid_mes()
3151 ret = kfd_dqm_evict_pasid_mes(dqm, &pdd->qpd); in kfd_evict_process_device()
3153 ret = dqm->ops.evict_process_queues(dqm, &pdd->qpd); in kfd_evict_process_device()
3159 struct qcm_process_device *qpd) in reserve_debug_trap_vmid() argument
3207 struct qcm_process_device *qpd) in release_debug_trap_vmid() argument
3305 struct qcm_process_device *qpd = &pdd->qpd; in copy_context_work_handler() local
3307 list_for_each_entry(q, &qpd->queues_list, list) { in copy_context_work_handler()
3363 struct qcm_process_device *qpd = &pdd->qpd; in resume_queues() local
3370 list_for_each_entry(q, &qpd->queues_list, list) { in resume_queues()
3380 int err = resume_single_queue(dqm, &pdd->qpd, q); in resume_queues()
3415 list_for_each_entry(q, &qpd->queues_list, list) { in resume_queues()
3466 struct qcm_process_device *qpd = &pdd->qpd; in suspend_queues() local
3474 list_for_each_entry(q, &qpd->queues_list, list) { in suspend_queues()
3518 list_for_each_entry(q, &qpd->queues_list, list) { in suspend_queues()
3652 struct qcm_process_device *qpd, in kfd_dqm_is_queue_in_process() argument
3663 list_for_each_entry(q, &qpd->queues_list, list) { in kfd_dqm_is_queue_in_process()