Lines Matching refs:pdd
76 static void kfd_process_device_destroy_cwsr_dgpu(struct kfd_process_device *pdd);
89 struct kfd_process_device *pdd; member
103 struct kfd_process_device *pdd; in kfd_sdma_activity_worker() local
116 pdd = workarea->pdd; in kfd_sdma_activity_worker()
117 if (!pdd) in kfd_sdma_activity_worker()
119 dqm = pdd->dev->dqm; in kfd_sdma_activity_worker()
120 qpd = &pdd->qpd; in kfd_sdma_activity_worker()
175 workarea->sdma_activity_counter = pdd->sdma_past_activity_counter; in kfd_sdma_activity_worker()
185 mm = get_task_mm(pdd->process->lead_thread); in kfd_sdma_activity_worker()
212 workarea->sdma_activity_counter += pdd->sdma_past_activity_counter; in kfd_sdma_activity_worker()
273 struct kfd_process_device *pdd = NULL; in kfd_get_cu_occupancy() local
278 pdd = container_of(attr, struct kfd_process_device, attr_cu_occupancy); in kfd_get_cu_occupancy()
279 dev = pdd->dev; in kfd_get_cu_occupancy()
284 proc = pdd->process; in kfd_get_cu_occupancy()
285 if (pdd->qpd.queue_count == 0) { in kfd_get_cu_occupancy()
311 kfd_dqm_is_queue_in_process(dev->dqm, &pdd->qpd, in kfd_get_cu_occupancy()
334 struct kfd_process_device *pdd = container_of(attr, struct kfd_process_device, in kfd_procfs_show() local
336 return snprintf(buffer, PAGE_SIZE, "%llu\n", atomic64_read(&pdd->vram_usage)); in kfd_procfs_show()
338 struct kfd_process_device *pdd = container_of(attr, struct kfd_process_device, in kfd_procfs_show() local
345 sdma_activity_work_handler.pdd = pdd; in kfd_procfs_show()
426 struct kfd_process_device *pdd = container_of(attr, in kfd_procfs_stats_show() local
431 evict_jiffies = atomic64_read(&pdd->evict_duration_counter); in kfd_procfs_stats_show()
451 struct kfd_process_device *pdd; in kfd_sysfs_counters_show() local
454 pdd = container_of(attr, struct kfd_process_device, in kfd_sysfs_counters_show()
456 return sysfs_emit(buf, "%llu\n", READ_ONCE(pdd->faults)); in kfd_sysfs_counters_show()
459 pdd = container_of(attr, struct kfd_process_device, in kfd_sysfs_counters_show()
461 return sysfs_emit(buf, "%llu\n", READ_ONCE(pdd->page_in)); in kfd_sysfs_counters_show()
464 pdd = container_of(attr, struct kfd_process_device, in kfd_sysfs_counters_show()
466 return sysfs_emit(buf, "%llu\n", READ_ONCE(pdd->page_out)); in kfd_sysfs_counters_show()
578 struct kfd_process_device *pdd = p->pdds[i]; in kfd_procfs_add_sysfs_stats() local
581 "stats_%u", pdd->dev->id); in kfd_procfs_add_sysfs_stats()
582 pdd->kobj_stats = kfd_alloc_struct(pdd->kobj_stats); in kfd_procfs_add_sysfs_stats()
583 if (!pdd->kobj_stats) in kfd_procfs_add_sysfs_stats()
586 ret = kobject_init_and_add(pdd->kobj_stats, in kfd_procfs_add_sysfs_stats()
594 kobject_put(pdd->kobj_stats); in kfd_procfs_add_sysfs_stats()
595 pdd->kobj_stats = NULL; in kfd_procfs_add_sysfs_stats()
599 kfd_sysfs_create_file(pdd->kobj_stats, &pdd->attr_evict, in kfd_procfs_add_sysfs_stats()
602 if (pdd->dev->kfd2kgd->get_cu_occupancy) in kfd_procfs_add_sysfs_stats()
603 kfd_sysfs_create_file(pdd->kobj_stats, in kfd_procfs_add_sysfs_stats()
604 &pdd->attr_cu_occupancy, in kfd_procfs_add_sysfs_stats()
626 struct kfd_process_device *pdd = p->pdds[i]; in kfd_procfs_add_sysfs_counters() local
630 "counters_%u", pdd->dev->id); in kfd_procfs_add_sysfs_counters()
644 pdd->kobj_counters = kobj_counters; in kfd_procfs_add_sysfs_counters()
645 kfd_sysfs_create_file(kobj_counters, &pdd->attr_faults, in kfd_procfs_add_sysfs_counters()
647 kfd_sysfs_create_file(kobj_counters, &pdd->attr_page_in, in kfd_procfs_add_sysfs_counters()
649 kfd_sysfs_create_file(kobj_counters, &pdd->attr_page_out, in kfd_procfs_add_sysfs_counters()
667 struct kfd_process_device *pdd = p->pdds[i]; in kfd_procfs_add_sysfs_files() local
669 snprintf(pdd->vram_filename, MAX_SYSFS_FILENAME_LEN, "vram_%u", in kfd_procfs_add_sysfs_files()
670 pdd->dev->id); in kfd_procfs_add_sysfs_files()
671 kfd_sysfs_create_file(p->kobj, &pdd->attr_vram, in kfd_procfs_add_sysfs_files()
672 pdd->vram_filename); in kfd_procfs_add_sysfs_files()
674 snprintf(pdd->sdma_filename, MAX_SYSFS_FILENAME_LEN, "sdma_%u", in kfd_procfs_add_sysfs_files()
675 pdd->dev->id); in kfd_procfs_add_sysfs_files()
676 kfd_sysfs_create_file(p->kobj, &pdd->attr_sdma, in kfd_procfs_add_sysfs_files()
677 pdd->sdma_filename); in kfd_procfs_add_sysfs_files()
719 struct kfd_process_device *pdd, void **kptr) in kfd_process_free_gpuvm() argument
721 struct kfd_node *dev = pdd->dev; in kfd_process_free_gpuvm()
728 amdgpu_amdkfd_gpuvm_unmap_memory_from_gpu(dev->adev, mem, pdd->drm_priv); in kfd_process_free_gpuvm()
729 amdgpu_amdkfd_gpuvm_free_memory_of_gpu(dev->adev, mem, pdd->drm_priv, in kfd_process_free_gpuvm()
739 static int kfd_process_alloc_gpuvm(struct kfd_process_device *pdd, in kfd_process_alloc_gpuvm() argument
743 struct kfd_node *kdev = pdd->dev; in kfd_process_alloc_gpuvm()
747 pdd->drm_priv, mem, NULL, in kfd_process_alloc_gpuvm()
753 pdd->drm_priv); in kfd_process_alloc_gpuvm()
775 amdgpu_amdkfd_gpuvm_unmap_memory_from_gpu(kdev->adev, *mem, pdd->drm_priv); in kfd_process_alloc_gpuvm()
778 amdgpu_amdkfd_gpuvm_free_memory_of_gpu(kdev->adev, *mem, pdd->drm_priv, in kfd_process_alloc_gpuvm()
792 static int kfd_process_device_reserve_ib_mem(struct kfd_process_device *pdd) in kfd_process_device_reserve_ib_mem() argument
794 struct qcm_process_device *qpd = &pdd->qpd; in kfd_process_device_reserve_ib_mem()
807 ret = kfd_process_alloc_gpuvm(pdd, qpd->ib_base, PAGE_SIZE, flags, in kfd_process_device_reserve_ib_mem()
818 static void kfd_process_device_destroy_ib_mem(struct kfd_process_device *pdd) in kfd_process_device_destroy_ib_mem() argument
820 struct qcm_process_device *qpd = &pdd->qpd; in kfd_process_device_destroy_ib_mem()
825 kfd_process_free_gpuvm(qpd->ib_mem, pdd, &qpd->ib_kaddr); in kfd_process_device_destroy_ib_mem()
985 static void kfd_process_device_free_bos(struct kfd_process_device *pdd) in kfd_process_device_free_bos() argument
987 struct kfd_process *p = pdd->process; in kfd_process_device_free_bos()
996 idr_for_each_entry(&pdd->alloc_idr, mem, id) { in kfd_process_device_free_bos()
1007 amdgpu_amdkfd_gpuvm_free_memory_of_gpu(pdd->dev->adev, mem, in kfd_process_device_free_bos()
1008 pdd->drm_priv, NULL); in kfd_process_device_free_bos()
1009 kfd_process_device_remove_obj_handle(pdd, id); in kfd_process_device_free_bos()
1019 struct kfd_process_device *pdd; in kfd_process_kunmap_signal_bo() local
1029 pdd = kfd_get_process_device_data(kdev, p); in kfd_process_kunmap_signal_bo()
1030 if (!pdd) in kfd_process_kunmap_signal_bo()
1034 pdd, GET_IDR_HANDLE(p->signal_handle)); in kfd_process_kunmap_signal_bo()
1057 struct kfd_process_device *pdd = p->pdds[i]; in kfd_process_destroy_pdds() local
1059 kfd_smi_event_process(pdd, false); in kfd_process_destroy_pdds()
1062 pdd->dev->id, p->lead_thread->pid); in kfd_process_destroy_pdds()
1063 kfd_process_device_destroy_cwsr_dgpu(pdd); in kfd_process_destroy_pdds()
1064 kfd_process_device_destroy_ib_mem(pdd); in kfd_process_destroy_pdds()
1066 if (pdd->drm_file) in kfd_process_destroy_pdds()
1067 fput(pdd->drm_file); in kfd_process_destroy_pdds()
1069 if (pdd->qpd.cwsr_kaddr && !pdd->qpd.cwsr_base) in kfd_process_destroy_pdds()
1070 free_pages((unsigned long)pdd->qpd.cwsr_kaddr, in kfd_process_destroy_pdds()
1073 idr_destroy(&pdd->alloc_idr); in kfd_process_destroy_pdds()
1075 kfd_free_process_doorbells(pdd->dev->kfd, pdd); in kfd_process_destroy_pdds()
1077 if (pdd->dev->kfd->shared_resources.enable_mes && in kfd_process_destroy_pdds()
1078 pdd->proc_ctx_cpu_ptr) in kfd_process_destroy_pdds()
1079 amdgpu_amdkfd_free_gtt_mem(pdd->dev->adev, in kfd_process_destroy_pdds()
1080 &pdd->proc_ctx_bo); in kfd_process_destroy_pdds()
1085 if (pdd->runtime_inuse) { in kfd_process_destroy_pdds()
1086 pm_runtime_mark_last_busy(adev_to_drm(pdd->dev->adev)->dev); in kfd_process_destroy_pdds()
1087 pm_runtime_put_autosuspend(adev_to_drm(pdd->dev->adev)->dev); in kfd_process_destroy_pdds()
1088 pdd->runtime_inuse = false; in kfd_process_destroy_pdds()
1091 kfree(pdd); in kfd_process_destroy_pdds()
1099 struct kfd_process_device *pdd; in kfd_process_remove_sysfs() local
1111 pdd = p->pdds[i]; in kfd_process_remove_sysfs()
1113 sysfs_remove_file(p->kobj, &pdd->attr_vram); in kfd_process_remove_sysfs()
1114 sysfs_remove_file(p->kobj, &pdd->attr_sdma); in kfd_process_remove_sysfs()
1116 sysfs_remove_file(pdd->kobj_stats, &pdd->attr_evict); in kfd_process_remove_sysfs()
1117 if (pdd->dev->kfd2kgd->get_cu_occupancy) in kfd_process_remove_sysfs()
1118 sysfs_remove_file(pdd->kobj_stats, in kfd_process_remove_sysfs()
1119 &pdd->attr_cu_occupancy); in kfd_process_remove_sysfs()
1120 kobject_del(pdd->kobj_stats); in kfd_process_remove_sysfs()
1121 kobject_put(pdd->kobj_stats); in kfd_process_remove_sysfs()
1122 pdd->kobj_stats = NULL; in kfd_process_remove_sysfs()
1126 pdd = p->pdds[i]; in kfd_process_remove_sysfs()
1128 sysfs_remove_file(pdd->kobj_counters, &pdd->attr_faults); in kfd_process_remove_sysfs()
1129 sysfs_remove_file(pdd->kobj_counters, &pdd->attr_page_in); in kfd_process_remove_sysfs()
1130 sysfs_remove_file(pdd->kobj_counters, &pdd->attr_page_out); in kfd_process_remove_sysfs()
1131 kobject_del(pdd->kobj_counters); in kfd_process_remove_sysfs()
1132 kobject_put(pdd->kobj_counters); in kfd_process_remove_sysfs()
1133 pdd->kobj_counters = NULL; in kfd_process_remove_sysfs()
1228 struct kfd_process_device *pdd = p->pdds[i]; in kfd_process_notifier_release_internal() local
1231 if (!kfd_dbg_is_rlc_restore_supported(pdd->dev) && p->runtime_info.ttmp_setup) in kfd_process_notifier_release_internal()
1232 amdgpu_gfx_off_ctrl(pdd->dev->adev, true); in kfd_process_notifier_release_internal()
1379 static int kfd_process_device_init_cwsr_dgpu(struct kfd_process_device *pdd) in kfd_process_device_init_cwsr_dgpu() argument
1381 struct kfd_node *dev = pdd->dev; in kfd_process_device_init_cwsr_dgpu()
1382 struct qcm_process_device *qpd = &pdd->qpd; in kfd_process_device_init_cwsr_dgpu()
1394 ret = kfd_process_alloc_gpuvm(pdd, qpd->cwsr_base, in kfd_process_device_init_cwsr_dgpu()
1405 kfd_process_set_trap_debug_flag(&pdd->qpd, in kfd_process_device_init_cwsr_dgpu()
1406 pdd->process->debug_trap_enabled); in kfd_process_device_init_cwsr_dgpu()
1415 static void kfd_process_device_destroy_cwsr_dgpu(struct kfd_process_device *pdd) in kfd_process_device_destroy_cwsr_dgpu() argument
1417 struct kfd_node *dev = pdd->dev; in kfd_process_device_destroy_cwsr_dgpu()
1418 struct qcm_process_device *qpd = &pdd->qpd; in kfd_process_device_destroy_cwsr_dgpu()
1423 kfd_process_free_gpuvm(qpd->cwsr_mem, pdd, &qpd->cwsr_kaddr); in kfd_process_device_destroy_cwsr_dgpu()
1618 struct kfd_process_device *pdd = NULL; in kfd_create_process_device_data() local
1622 pdd = kzalloc(sizeof(*pdd), GFP_KERNEL); in kfd_create_process_device_data()
1623 if (!pdd) in kfd_create_process_device_data()
1626 pdd->dev = dev; in kfd_create_process_device_data()
1627 INIT_LIST_HEAD(&pdd->qpd.queues_list); in kfd_create_process_device_data()
1628 INIT_LIST_HEAD(&pdd->qpd.priv_queue_list); in kfd_create_process_device_data()
1629 pdd->qpd.dqm = dev->dqm; in kfd_create_process_device_data()
1630 pdd->qpd.pqm = &p->pqm; in kfd_create_process_device_data()
1631 pdd->qpd.evicted = 0; in kfd_create_process_device_data()
1632 pdd->qpd.mapped_gws_queue = false; in kfd_create_process_device_data()
1633 pdd->process = p; in kfd_create_process_device_data()
1634 pdd->bound = PDD_UNBOUND; in kfd_create_process_device_data()
1635 pdd->already_dequeued = false; in kfd_create_process_device_data()
1636 pdd->runtime_inuse = false; in kfd_create_process_device_data()
1637 atomic64_set(&pdd->vram_usage, 0); in kfd_create_process_device_data()
1638 pdd->sdma_past_activity_counter = 0; in kfd_create_process_device_data()
1639 pdd->user_gpu_id = dev->id; in kfd_create_process_device_data()
1640 atomic64_set(&pdd->evict_duration_counter, 0); in kfd_create_process_device_data()
1642 p->pdds[p->n_pdds++] = pdd; in kfd_create_process_device_data()
1643 if (kfd_dbg_is_per_vmid_supported(pdd->dev)) in kfd_create_process_device_data()
1644 pdd->spi_dbg_override = pdd->dev->kfd2kgd->disable_debug_trap( in kfd_create_process_device_data()
1645 pdd->dev->adev, in kfd_create_process_device_data()
1650 idr_init(&pdd->alloc_idr); in kfd_create_process_device_data()
1652 return pdd; in kfd_create_process_device_data()
1669 int kfd_process_device_init_vm(struct kfd_process_device *pdd, in kfd_process_device_init_vm() argument
1682 if (pdd->drm_priv) in kfd_process_device_init_vm()
1690 p = pdd->process; in kfd_process_device_init_vm()
1691 dev = pdd->dev; in kfd_process_device_init_vm()
1704 pdd->drm_priv = drm_file->private_data; in kfd_process_device_init_vm()
1706 ret = kfd_process_device_reserve_ib_mem(pdd); in kfd_process_device_init_vm()
1709 ret = kfd_process_device_init_cwsr_dgpu(pdd); in kfd_process_device_init_vm()
1714 dev_warn(pdd->dev->adev->dev, "WARN: vm %p has no pasid associated", in kfd_process_device_init_vm()
1720 pdd->pasid = avm->pasid; in kfd_process_device_init_vm()
1721 pdd->drm_file = drm_file; in kfd_process_device_init_vm()
1723 kfd_smi_event_process(pdd, true); in kfd_process_device_init_vm()
1728 kfd_process_device_destroy_cwsr_dgpu(pdd); in kfd_process_device_init_vm()
1730 kfd_process_device_destroy_ib_mem(pdd); in kfd_process_device_init_vm()
1732 pdd->drm_priv = NULL; in kfd_process_device_init_vm()
1748 struct kfd_process_device *pdd; in kfd_bind_process_to_device() local
1751 pdd = kfd_get_process_device_data(dev, p); in kfd_bind_process_to_device()
1752 if (!pdd) { in kfd_bind_process_to_device()
1757 if (!pdd->drm_priv) in kfd_bind_process_to_device()
1765 if (!pdd->runtime_inuse) { in kfd_bind_process_to_device()
1777 pdd->runtime_inuse = true; in kfd_bind_process_to_device()
1779 return pdd; in kfd_bind_process_to_device()
1785 int kfd_process_device_create_obj_handle(struct kfd_process_device *pdd, in kfd_process_device_create_obj_handle() argument
1788 return idr_alloc(&pdd->alloc_idr, mem, 0, 0, GFP_KERNEL); in kfd_process_device_create_obj_handle()
1794 void *kfd_process_device_translate_handle(struct kfd_process_device *pdd, in kfd_process_device_translate_handle() argument
1800 return idr_find(&pdd->alloc_idr, handle); in kfd_process_device_translate_handle()
1806 void kfd_process_device_remove_obj_handle(struct kfd_process_device *pdd, in kfd_process_device_remove_obj_handle() argument
1810 idr_remove(&pdd->alloc_idr, handle); in kfd_process_device_remove_obj_handle()
1835 struct kfd_process_device **pdd) in kfd_lookup_process_by_pasid() argument
1843 if (pdd) in kfd_lookup_process_by_pasid()
1844 *pdd = ret_p; in kfd_lookup_process_by_pasid()
1853 if (pdd) in kfd_lookup_process_by_pasid()
1854 *pdd = NULL; in kfd_lookup_process_by_pasid()
1887 struct kfd_process_device *pdd = p->pdds[i]; in kfd_process_evict_queues() local
1888 struct device *dev = pdd->dev->adev->dev; in kfd_process_evict_queues()
1890 kfd_smi_event_queue_eviction(pdd->dev, p->lead_thread->pid, in kfd_process_evict_queues()
1893 r = pdd->dev->dqm->ops.evict_process_queues(pdd->dev->dqm, in kfd_process_evict_queues()
1894 &pdd->qpd); in kfd_process_evict_queues()
1905 pdd->dev->dqm->is_hws_hang = false; in kfd_process_evict_queues()
1915 struct kfd_process_device *pdd = p->pdds[i]; in kfd_process_evict_queues() local
1920 kfd_smi_event_queue_restore(pdd->dev, p->lead_thread->pid); in kfd_process_evict_queues()
1922 if (pdd->dev->dqm->ops.restore_process_queues(pdd->dev->dqm, in kfd_process_evict_queues()
1923 &pdd->qpd)) in kfd_process_evict_queues()
1924 dev_err(pdd->dev->adev->dev, in kfd_process_evict_queues()
1940 struct kfd_process_device *pdd = p->pdds[i]; in kfd_process_restore_queues() local
1941 struct device *dev = pdd->dev->adev->dev; in kfd_process_restore_queues()
1943 kfd_smi_event_queue_restore(pdd->dev, p->lead_thread->pid); in kfd_process_restore_queues()
1945 r = pdd->dev->dqm->ops.restore_process_queues(pdd->dev->dqm, in kfd_process_restore_queues()
1946 &pdd->qpd); in kfd_process_restore_queues()
2123 struct kfd_process_device *pdd; in kfd_reserved_mem_mmap() local
2131 pdd = kfd_get_process_device_data(dev, process); in kfd_reserved_mem_mmap()
2132 if (!pdd) in kfd_reserved_mem_mmap()
2134 qpd = &pdd->qpd; in kfd_reserved_mem_mmap()
2153 int kfd_process_drain_interrupts(struct kfd_process_device *pdd) in kfd_process_drain_interrupts() argument
2159 if (!KFD_IS_SOC15(pdd->dev)) in kfd_process_drain_interrupts()
2162 pdd->process->irq_drain_is_open = true; in kfd_process_drain_interrupts()
2167 irq_drain_fence[3] = pdd->pasid; in kfd_process_drain_interrupts()
2172 if (KFD_GC_VERSION(pdd->dev->kfd) == IP_VERSION(9, 4, 3) || in kfd_process_drain_interrupts()
2173 KFD_GC_VERSION(pdd->dev->kfd) == IP_VERSION(9, 4, 4) || in kfd_process_drain_interrupts()
2174 KFD_GC_VERSION(pdd->dev->kfd) == IP_VERSION(9, 5, 0)) { in kfd_process_drain_interrupts()
2175 node_id = ffs(pdd->dev->interrupt_bitmap) - 1; in kfd_process_drain_interrupts()
2180 if (amdgpu_amdkfd_send_close_event_drain_irq(pdd->dev->adev, in kfd_process_drain_interrupts()
2182 pdd->process->irq_drain_is_open = false; in kfd_process_drain_interrupts()
2186 r = wait_event_interruptible(pdd->process->wait_irq_drain, in kfd_process_drain_interrupts()
2187 !READ_ONCE(pdd->process->irq_drain_is_open)); in kfd_process_drain_interrupts()
2189 pdd->process->irq_drain_is_open = false; in kfd_process_drain_interrupts()
2283 struct kfd_process_device *pdd = p->pdds[i]; in kfd_process_device_data_by_id() local
2285 if (pdd->user_gpu_id == gpu_id) in kfd_process_device_data_by_id()
2286 return pdd; in kfd_process_device_data_by_id()
2300 struct kfd_process_device *pdd = p->pdds[i]; in kfd_process_get_user_gpu_id() local
2302 if (pdd->dev->id == actual_gpu_id) in kfd_process_get_user_gpu_id()
2303 return pdd->user_gpu_id; in kfd_process_get_user_gpu_id()