Lines Matching refs:pdd
75 static void kfd_process_device_destroy_cwsr_dgpu(struct kfd_process_device *pdd);
88 struct kfd_process_device *pdd; member
102 struct kfd_process_device *pdd; in kfd_sdma_activity_worker() local
115 pdd = workarea->pdd; in kfd_sdma_activity_worker()
116 if (!pdd) in kfd_sdma_activity_worker()
118 dqm = pdd->dev->dqm; in kfd_sdma_activity_worker()
119 qpd = &pdd->qpd; in kfd_sdma_activity_worker()
174 workarea->sdma_activity_counter = pdd->sdma_past_activity_counter; in kfd_sdma_activity_worker()
184 mm = get_task_mm(pdd->process->lead_thread); in kfd_sdma_activity_worker()
211 workarea->sdma_activity_counter += pdd->sdma_past_activity_counter; in kfd_sdma_activity_worker()
272 struct kfd_process_device *pdd = NULL; in kfd_get_cu_occupancy() local
279 pdd = container_of(attr, struct kfd_process_device, attr_cu_occupancy); in kfd_get_cu_occupancy()
280 dev = pdd->dev; in kfd_get_cu_occupancy()
285 proc = pdd->process; in kfd_get_cu_occupancy()
286 if (pdd->qpd.queue_count == 0) { in kfd_get_cu_occupancy()
308 kfd_dqm_is_queue_in_process(dev->dqm, &pdd->qpd, in kfd_get_cu_occupancy()
333 struct kfd_process_device *pdd = container_of(attr, struct kfd_process_device, in kfd_procfs_show() local
335 return snprintf(buffer, PAGE_SIZE, "%llu\n", atomic64_read(&pdd->vram_usage)); in kfd_procfs_show()
337 struct kfd_process_device *pdd = container_of(attr, struct kfd_process_device, in kfd_procfs_show() local
344 sdma_activity_work_handler.pdd = pdd; in kfd_procfs_show()
424 struct kfd_process_device *pdd = container_of(attr, in kfd_procfs_stats_show() local
429 evict_jiffies = atomic64_read(&pdd->evict_duration_counter); in kfd_procfs_stats_show()
449 struct kfd_process_device *pdd; in kfd_sysfs_counters_show() local
452 pdd = container_of(attr, struct kfd_process_device, in kfd_sysfs_counters_show()
454 return sysfs_emit(buf, "%llu\n", READ_ONCE(pdd->faults)); in kfd_sysfs_counters_show()
457 pdd = container_of(attr, struct kfd_process_device, in kfd_sysfs_counters_show()
459 return sysfs_emit(buf, "%llu\n", READ_ONCE(pdd->page_in)); in kfd_sysfs_counters_show()
462 pdd = container_of(attr, struct kfd_process_device, in kfd_sysfs_counters_show()
464 return sysfs_emit(buf, "%llu\n", READ_ONCE(pdd->page_out)); in kfd_sysfs_counters_show()
576 struct kfd_process_device *pdd = p->pdds[i]; in kfd_procfs_add_sysfs_stats() local
579 "stats_%u", pdd->dev->id); in kfd_procfs_add_sysfs_stats()
580 pdd->kobj_stats = kfd_alloc_struct(pdd->kobj_stats); in kfd_procfs_add_sysfs_stats()
581 if (!pdd->kobj_stats) in kfd_procfs_add_sysfs_stats()
584 ret = kobject_init_and_add(pdd->kobj_stats, in kfd_procfs_add_sysfs_stats()
592 kobject_put(pdd->kobj_stats); in kfd_procfs_add_sysfs_stats()
593 pdd->kobj_stats = NULL; in kfd_procfs_add_sysfs_stats()
597 kfd_sysfs_create_file(pdd->kobj_stats, &pdd->attr_evict, in kfd_procfs_add_sysfs_stats()
600 if (pdd->dev->kfd2kgd->get_cu_occupancy) in kfd_procfs_add_sysfs_stats()
601 kfd_sysfs_create_file(pdd->kobj_stats, in kfd_procfs_add_sysfs_stats()
602 &pdd->attr_cu_occupancy, in kfd_procfs_add_sysfs_stats()
624 struct kfd_process_device *pdd = p->pdds[i]; in kfd_procfs_add_sysfs_counters() local
628 "counters_%u", pdd->dev->id); in kfd_procfs_add_sysfs_counters()
642 pdd->kobj_counters = kobj_counters; in kfd_procfs_add_sysfs_counters()
643 kfd_sysfs_create_file(kobj_counters, &pdd->attr_faults, in kfd_procfs_add_sysfs_counters()
645 kfd_sysfs_create_file(kobj_counters, &pdd->attr_page_in, in kfd_procfs_add_sysfs_counters()
647 kfd_sysfs_create_file(kobj_counters, &pdd->attr_page_out, in kfd_procfs_add_sysfs_counters()
665 struct kfd_process_device *pdd = p->pdds[i]; in kfd_procfs_add_sysfs_files() local
667 snprintf(pdd->vram_filename, MAX_SYSFS_FILENAME_LEN, "vram_%u", in kfd_procfs_add_sysfs_files()
668 pdd->dev->id); in kfd_procfs_add_sysfs_files()
669 kfd_sysfs_create_file(p->kobj, &pdd->attr_vram, in kfd_procfs_add_sysfs_files()
670 pdd->vram_filename); in kfd_procfs_add_sysfs_files()
672 snprintf(pdd->sdma_filename, MAX_SYSFS_FILENAME_LEN, "sdma_%u", in kfd_procfs_add_sysfs_files()
673 pdd->dev->id); in kfd_procfs_add_sysfs_files()
674 kfd_sysfs_create_file(p->kobj, &pdd->attr_sdma, in kfd_procfs_add_sysfs_files()
675 pdd->sdma_filename); in kfd_procfs_add_sysfs_files()
717 struct kfd_process_device *pdd, void **kptr) in kfd_process_free_gpuvm() argument
719 struct kfd_node *dev = pdd->dev; in kfd_process_free_gpuvm()
726 amdgpu_amdkfd_gpuvm_unmap_memory_from_gpu(dev->adev, mem, pdd->drm_priv); in kfd_process_free_gpuvm()
727 amdgpu_amdkfd_gpuvm_free_memory_of_gpu(dev->adev, mem, pdd->drm_priv, in kfd_process_free_gpuvm()
737 static int kfd_process_alloc_gpuvm(struct kfd_process_device *pdd, in kfd_process_alloc_gpuvm() argument
741 struct kfd_node *kdev = pdd->dev; in kfd_process_alloc_gpuvm()
745 pdd->drm_priv, mem, NULL, in kfd_process_alloc_gpuvm()
751 pdd->drm_priv); in kfd_process_alloc_gpuvm()
773 amdgpu_amdkfd_gpuvm_unmap_memory_from_gpu(kdev->adev, *mem, pdd->drm_priv); in kfd_process_alloc_gpuvm()
776 amdgpu_amdkfd_gpuvm_free_memory_of_gpu(kdev->adev, *mem, pdd->drm_priv, in kfd_process_alloc_gpuvm()
790 static int kfd_process_device_reserve_ib_mem(struct kfd_process_device *pdd) in kfd_process_device_reserve_ib_mem() argument
792 struct qcm_process_device *qpd = &pdd->qpd; in kfd_process_device_reserve_ib_mem()
805 ret = kfd_process_alloc_gpuvm(pdd, qpd->ib_base, PAGE_SIZE, flags, in kfd_process_device_reserve_ib_mem()
816 static void kfd_process_device_destroy_ib_mem(struct kfd_process_device *pdd) in kfd_process_device_destroy_ib_mem() argument
818 struct qcm_process_device *qpd = &pdd->qpd; in kfd_process_device_destroy_ib_mem()
823 kfd_process_free_gpuvm(qpd->ib_mem, pdd, &qpd->ib_kaddr); in kfd_process_device_destroy_ib_mem()
981 static void kfd_process_device_free_bos(struct kfd_process_device *pdd) in kfd_process_device_free_bos() argument
983 struct kfd_process *p = pdd->process; in kfd_process_device_free_bos()
992 idr_for_each_entry(&pdd->alloc_idr, mem, id) { in kfd_process_device_free_bos()
1003 amdgpu_amdkfd_gpuvm_free_memory_of_gpu(pdd->dev->adev, mem, in kfd_process_device_free_bos()
1004 pdd->drm_priv, NULL); in kfd_process_device_free_bos()
1005 kfd_process_device_remove_obj_handle(pdd, id); in kfd_process_device_free_bos()
1015 struct kfd_process_device *pdd; in kfd_process_kunmap_signal_bo() local
1025 pdd = kfd_get_process_device_data(kdev, p); in kfd_process_kunmap_signal_bo()
1026 if (!pdd) in kfd_process_kunmap_signal_bo()
1030 pdd, GET_IDR_HANDLE(p->signal_handle)); in kfd_process_kunmap_signal_bo()
1053 struct kfd_process_device *pdd = p->pdds[i]; in kfd_process_destroy_pdds() local
1056 pdd->dev->id, p->pasid); in kfd_process_destroy_pdds()
1058 kfd_process_device_destroy_cwsr_dgpu(pdd); in kfd_process_destroy_pdds()
1059 kfd_process_device_destroy_ib_mem(pdd); in kfd_process_destroy_pdds()
1061 if (pdd->drm_file) { in kfd_process_destroy_pdds()
1063 pdd->dev->adev, pdd->drm_priv); in kfd_process_destroy_pdds()
1064 fput(pdd->drm_file); in kfd_process_destroy_pdds()
1067 if (pdd->qpd.cwsr_kaddr && !pdd->qpd.cwsr_base) in kfd_process_destroy_pdds()
1068 free_pages((unsigned long)pdd->qpd.cwsr_kaddr, in kfd_process_destroy_pdds()
1071 idr_destroy(&pdd->alloc_idr); in kfd_process_destroy_pdds()
1073 kfd_free_process_doorbells(pdd->dev->kfd, pdd); in kfd_process_destroy_pdds()
1075 if (pdd->dev->kfd->shared_resources.enable_mes) in kfd_process_destroy_pdds()
1076 amdgpu_amdkfd_free_gtt_mem(pdd->dev->adev, in kfd_process_destroy_pdds()
1077 &pdd->proc_ctx_bo); in kfd_process_destroy_pdds()
1082 if (pdd->runtime_inuse) { in kfd_process_destroy_pdds()
1083 pm_runtime_mark_last_busy(adev_to_drm(pdd->dev->adev)->dev); in kfd_process_destroy_pdds()
1084 pm_runtime_put_autosuspend(adev_to_drm(pdd->dev->adev)->dev); in kfd_process_destroy_pdds()
1085 pdd->runtime_inuse = false; in kfd_process_destroy_pdds()
1088 kfree(pdd); in kfd_process_destroy_pdds()
1096 struct kfd_process_device *pdd; in kfd_process_remove_sysfs() local
1108 pdd = p->pdds[i]; in kfd_process_remove_sysfs()
1110 sysfs_remove_file(p->kobj, &pdd->attr_vram); in kfd_process_remove_sysfs()
1111 sysfs_remove_file(p->kobj, &pdd->attr_sdma); in kfd_process_remove_sysfs()
1113 sysfs_remove_file(pdd->kobj_stats, &pdd->attr_evict); in kfd_process_remove_sysfs()
1114 if (pdd->dev->kfd2kgd->get_cu_occupancy) in kfd_process_remove_sysfs()
1115 sysfs_remove_file(pdd->kobj_stats, in kfd_process_remove_sysfs()
1116 &pdd->attr_cu_occupancy); in kfd_process_remove_sysfs()
1117 kobject_del(pdd->kobj_stats); in kfd_process_remove_sysfs()
1118 kobject_put(pdd->kobj_stats); in kfd_process_remove_sysfs()
1119 pdd->kobj_stats = NULL; in kfd_process_remove_sysfs()
1123 pdd = p->pdds[i]; in kfd_process_remove_sysfs()
1125 sysfs_remove_file(pdd->kobj_counters, &pdd->attr_faults); in kfd_process_remove_sysfs()
1126 sysfs_remove_file(pdd->kobj_counters, &pdd->attr_page_in); in kfd_process_remove_sysfs()
1127 sysfs_remove_file(pdd->kobj_counters, &pdd->attr_page_out); in kfd_process_remove_sysfs()
1128 kobject_del(pdd->kobj_counters); in kfd_process_remove_sysfs()
1129 kobject_put(pdd->kobj_counters); in kfd_process_remove_sysfs()
1130 pdd->kobj_counters = NULL; in kfd_process_remove_sysfs()
1210 struct kfd_process_device *pdd = p->pdds[i]; in kfd_process_notifier_release_internal() local
1213 if (!kfd_dbg_is_rlc_restore_supported(pdd->dev) && p->runtime_info.ttmp_setup) in kfd_process_notifier_release_internal()
1214 amdgpu_gfx_off_ctrl(pdd->dev->adev, true); in kfd_process_notifier_release_internal()
1361 static int kfd_process_device_init_cwsr_dgpu(struct kfd_process_device *pdd) in kfd_process_device_init_cwsr_dgpu() argument
1363 struct kfd_node *dev = pdd->dev; in kfd_process_device_init_cwsr_dgpu()
1364 struct qcm_process_device *qpd = &pdd->qpd; in kfd_process_device_init_cwsr_dgpu()
1376 ret = kfd_process_alloc_gpuvm(pdd, qpd->cwsr_base, in kfd_process_device_init_cwsr_dgpu()
1387 kfd_process_set_trap_debug_flag(&pdd->qpd, in kfd_process_device_init_cwsr_dgpu()
1388 pdd->process->debug_trap_enabled); in kfd_process_device_init_cwsr_dgpu()
1397 static void kfd_process_device_destroy_cwsr_dgpu(struct kfd_process_device *pdd) in kfd_process_device_destroy_cwsr_dgpu() argument
1399 struct kfd_node *dev = pdd->dev; in kfd_process_device_destroy_cwsr_dgpu()
1400 struct qcm_process_device *qpd = &pdd->qpd; in kfd_process_device_destroy_cwsr_dgpu()
1405 kfd_process_free_gpuvm(qpd->cwsr_mem, pdd, &qpd->cwsr_kaddr); in kfd_process_device_destroy_cwsr_dgpu()
1608 struct kfd_process_device *pdd = NULL; in kfd_create_process_device_data() local
1613 pdd = kzalloc(sizeof(*pdd), GFP_KERNEL); in kfd_create_process_device_data()
1614 if (!pdd) in kfd_create_process_device_data()
1617 pdd->dev = dev; in kfd_create_process_device_data()
1618 INIT_LIST_HEAD(&pdd->qpd.queues_list); in kfd_create_process_device_data()
1619 INIT_LIST_HEAD(&pdd->qpd.priv_queue_list); in kfd_create_process_device_data()
1620 pdd->qpd.dqm = dev->dqm; in kfd_create_process_device_data()
1621 pdd->qpd.pqm = &p->pqm; in kfd_create_process_device_data()
1622 pdd->qpd.evicted = 0; in kfd_create_process_device_data()
1623 pdd->qpd.mapped_gws_queue = false; in kfd_create_process_device_data()
1624 pdd->process = p; in kfd_create_process_device_data()
1625 pdd->bound = PDD_UNBOUND; in kfd_create_process_device_data()
1626 pdd->already_dequeued = false; in kfd_create_process_device_data()
1627 pdd->runtime_inuse = false; in kfd_create_process_device_data()
1628 atomic64_set(&pdd->vram_usage, 0); in kfd_create_process_device_data()
1629 pdd->sdma_past_activity_counter = 0; in kfd_create_process_device_data()
1630 pdd->user_gpu_id = dev->id; in kfd_create_process_device_data()
1631 atomic64_set(&pdd->evict_duration_counter, 0); in kfd_create_process_device_data()
1636 &pdd->proc_ctx_bo, in kfd_create_process_device_data()
1637 &pdd->proc_ctx_gpu_addr, in kfd_create_process_device_data()
1638 &pdd->proc_ctx_cpu_ptr, in kfd_create_process_device_data()
1645 memset(pdd->proc_ctx_cpu_ptr, 0, AMDGPU_MES_PROC_CTX_SIZE); in kfd_create_process_device_data()
1648 p->pdds[p->n_pdds++] = pdd; in kfd_create_process_device_data()
1649 if (kfd_dbg_is_per_vmid_supported(pdd->dev)) in kfd_create_process_device_data()
1650 pdd->spi_dbg_override = pdd->dev->kfd2kgd->disable_debug_trap( in kfd_create_process_device_data()
1651 pdd->dev->adev, in kfd_create_process_device_data()
1656 idr_init(&pdd->alloc_idr); in kfd_create_process_device_data()
1658 return pdd; in kfd_create_process_device_data()
1661 kfree(pdd); in kfd_create_process_device_data()
1679 int kfd_process_device_init_vm(struct kfd_process_device *pdd, in kfd_process_device_init_vm() argument
1692 if (pdd->drm_priv) in kfd_process_device_init_vm()
1700 p = pdd->process; in kfd_process_device_init_vm()
1701 dev = pdd->dev; in kfd_process_device_init_vm()
1714 pdd->drm_priv = drm_file->private_data; in kfd_process_device_init_vm()
1716 ret = kfd_process_device_reserve_ib_mem(pdd); in kfd_process_device_init_vm()
1719 ret = kfd_process_device_init_cwsr_dgpu(pdd); in kfd_process_device_init_vm()
1727 pdd->drm_file = drm_file; in kfd_process_device_init_vm()
1732 kfd_process_device_destroy_cwsr_dgpu(pdd); in kfd_process_device_init_vm()
1734 kfd_process_device_destroy_ib_mem(pdd); in kfd_process_device_init_vm()
1736 pdd->drm_priv = NULL; in kfd_process_device_init_vm()
1752 struct kfd_process_device *pdd; in kfd_bind_process_to_device() local
1755 pdd = kfd_get_process_device_data(dev, p); in kfd_bind_process_to_device()
1756 if (!pdd) { in kfd_bind_process_to_device()
1761 if (!pdd->drm_priv) in kfd_bind_process_to_device()
1769 if (!pdd->runtime_inuse) { in kfd_bind_process_to_device()
1781 pdd->runtime_inuse = true; in kfd_bind_process_to_device()
1783 return pdd; in kfd_bind_process_to_device()
1789 int kfd_process_device_create_obj_handle(struct kfd_process_device *pdd, in kfd_process_device_create_obj_handle() argument
1792 return idr_alloc(&pdd->alloc_idr, mem, 0, 0, GFP_KERNEL); in kfd_process_device_create_obj_handle()
1798 void *kfd_process_device_translate_handle(struct kfd_process_device *pdd, in kfd_process_device_translate_handle() argument
1804 return idr_find(&pdd->alloc_idr, handle); in kfd_process_device_translate_handle()
1810 void kfd_process_device_remove_obj_handle(struct kfd_process_device *pdd, in kfd_process_device_remove_obj_handle() argument
1814 idr_remove(&pdd->alloc_idr, handle); in kfd_process_device_remove_obj_handle()
1866 struct kfd_process_device *pdd = p->pdds[i]; in kfd_process_evict_queues() local
1867 struct device *dev = pdd->dev->adev->dev; in kfd_process_evict_queues()
1869 kfd_smi_event_queue_eviction(pdd->dev, p->lead_thread->pid, in kfd_process_evict_queues()
1872 r = pdd->dev->dqm->ops.evict_process_queues(pdd->dev->dqm, in kfd_process_evict_queues()
1873 &pdd->qpd); in kfd_process_evict_queues()
1884 pdd->dev->dqm->is_hws_hang = false; in kfd_process_evict_queues()
1894 struct kfd_process_device *pdd = p->pdds[i]; in kfd_process_evict_queues() local
1899 kfd_smi_event_queue_restore(pdd->dev, p->lead_thread->pid); in kfd_process_evict_queues()
1901 if (pdd->dev->dqm->ops.restore_process_queues(pdd->dev->dqm, in kfd_process_evict_queues()
1902 &pdd->qpd)) in kfd_process_evict_queues()
1903 dev_err(pdd->dev->adev->dev, in kfd_process_evict_queues()
1919 struct kfd_process_device *pdd = p->pdds[i]; in kfd_process_restore_queues() local
1920 struct device *dev = pdd->dev->adev->dev; in kfd_process_restore_queues()
1922 kfd_smi_event_queue_restore(pdd->dev, p->lead_thread->pid); in kfd_process_restore_queues()
1924 r = pdd->dev->dqm->ops.restore_process_queues(pdd->dev->dqm, in kfd_process_restore_queues()
1925 &pdd->qpd); in kfd_process_restore_queues()
2100 struct kfd_process_device *pdd; in kfd_reserved_mem_mmap() local
2108 pdd = kfd_get_process_device_data(dev, process); in kfd_reserved_mem_mmap()
2109 if (!pdd) in kfd_reserved_mem_mmap()
2111 qpd = &pdd->qpd; in kfd_reserved_mem_mmap()
2130 int kfd_process_drain_interrupts(struct kfd_process_device *pdd) in kfd_process_drain_interrupts() argument
2136 if (!KFD_IS_SOC15(pdd->dev)) in kfd_process_drain_interrupts()
2139 pdd->process->irq_drain_is_open = true; in kfd_process_drain_interrupts()
2144 irq_drain_fence[3] = pdd->process->pasid; in kfd_process_drain_interrupts()
2149 if (KFD_GC_VERSION(pdd->dev->kfd) == IP_VERSION(9, 4, 3) || in kfd_process_drain_interrupts()
2150 KFD_GC_VERSION(pdd->dev->kfd) == IP_VERSION(9, 4, 4)) { in kfd_process_drain_interrupts()
2151 node_id = ffs(pdd->dev->interrupt_bitmap) - 1; in kfd_process_drain_interrupts()
2156 if (amdgpu_amdkfd_send_close_event_drain_irq(pdd->dev->adev, in kfd_process_drain_interrupts()
2158 pdd->process->irq_drain_is_open = false; in kfd_process_drain_interrupts()
2162 r = wait_event_interruptible(pdd->process->wait_irq_drain, in kfd_process_drain_interrupts()
2163 !READ_ONCE(pdd->process->irq_drain_is_open)); in kfd_process_drain_interrupts()
2165 pdd->process->irq_drain_is_open = false; in kfd_process_drain_interrupts()
2259 struct kfd_process_device *pdd = p->pdds[i]; in kfd_process_device_data_by_id() local
2261 if (pdd->user_gpu_id == gpu_id) in kfd_process_device_data_by_id()
2262 return pdd; in kfd_process_device_data_by_id()
2276 struct kfd_process_device *pdd = p->pdds[i]; in kfd_process_get_user_gpu_id() local
2278 if (pdd->dev->id == actual_gpu_id) in kfd_process_get_user_gpu_id()
2279 return pdd->user_gpu_id; in kfd_process_get_user_gpu_id()