Lines Matching refs:pdd
77 static void kfd_process_device_destroy_cwsr_dgpu(struct kfd_process_device *pdd);
90 struct kfd_process_device *pdd; member
104 struct kfd_process_device *pdd; in kfd_sdma_activity_worker() local
117 pdd = workarea->pdd; in kfd_sdma_activity_worker()
118 if (!pdd) in kfd_sdma_activity_worker()
120 dqm = pdd->dev->dqm; in kfd_sdma_activity_worker()
121 qpd = &pdd->qpd; in kfd_sdma_activity_worker()
176 workarea->sdma_activity_counter = pdd->sdma_past_activity_counter; in kfd_sdma_activity_worker()
186 mm = get_task_mm(pdd->process->lead_thread); in kfd_sdma_activity_worker()
213 workarea->sdma_activity_counter += pdd->sdma_past_activity_counter; in kfd_sdma_activity_worker()
274 struct kfd_process_device *pdd = NULL; in kfd_get_cu_occupancy() local
276 pdd = container_of(attr, struct kfd_process_device, attr_cu_occupancy); in kfd_get_cu_occupancy()
277 dev = pdd->dev; in kfd_get_cu_occupancy()
282 proc = pdd->process; in kfd_get_cu_occupancy()
283 if (pdd->qpd.queue_count == 0) { in kfd_get_cu_occupancy()
309 struct kfd_process_device *pdd = container_of(attr, struct kfd_process_device, in kfd_procfs_show() local
311 return snprintf(buffer, PAGE_SIZE, "%llu\n", READ_ONCE(pdd->vram_usage)); in kfd_procfs_show()
313 struct kfd_process_device *pdd = container_of(attr, struct kfd_process_device, in kfd_procfs_show() local
320 sdma_activity_work_handler.pdd = pdd; in kfd_procfs_show()
400 struct kfd_process_device *pdd = container_of(attr, in kfd_procfs_stats_show() local
405 evict_jiffies = atomic64_read(&pdd->evict_duration_counter); in kfd_procfs_stats_show()
425 struct kfd_process_device *pdd; in kfd_sysfs_counters_show() local
428 pdd = container_of(attr, struct kfd_process_device, in kfd_sysfs_counters_show()
430 return sysfs_emit(buf, "%llu\n", READ_ONCE(pdd->faults)); in kfd_sysfs_counters_show()
433 pdd = container_of(attr, struct kfd_process_device, in kfd_sysfs_counters_show()
435 return sysfs_emit(buf, "%llu\n", READ_ONCE(pdd->page_in)); in kfd_sysfs_counters_show()
438 pdd = container_of(attr, struct kfd_process_device, in kfd_sysfs_counters_show()
440 return sysfs_emit(buf, "%llu\n", READ_ONCE(pdd->page_out)); in kfd_sysfs_counters_show()
552 struct kfd_process_device *pdd = p->pdds[i]; in kfd_procfs_add_sysfs_stats() local
555 "stats_%u", pdd->dev->id); in kfd_procfs_add_sysfs_stats()
556 pdd->kobj_stats = kfd_alloc_struct(pdd->kobj_stats); in kfd_procfs_add_sysfs_stats()
557 if (!pdd->kobj_stats) in kfd_procfs_add_sysfs_stats()
560 ret = kobject_init_and_add(pdd->kobj_stats, in kfd_procfs_add_sysfs_stats()
568 kobject_put(pdd->kobj_stats); in kfd_procfs_add_sysfs_stats()
569 pdd->kobj_stats = NULL; in kfd_procfs_add_sysfs_stats()
573 kfd_sysfs_create_file(pdd->kobj_stats, &pdd->attr_evict, in kfd_procfs_add_sysfs_stats()
576 if (pdd->dev->kfd2kgd->get_cu_occupancy) in kfd_procfs_add_sysfs_stats()
577 kfd_sysfs_create_file(pdd->kobj_stats, in kfd_procfs_add_sysfs_stats()
578 &pdd->attr_cu_occupancy, in kfd_procfs_add_sysfs_stats()
600 struct kfd_process_device *pdd = p->pdds[i]; in kfd_procfs_add_sysfs_counters() local
604 "counters_%u", pdd->dev->id); in kfd_procfs_add_sysfs_counters()
618 pdd->kobj_counters = kobj_counters; in kfd_procfs_add_sysfs_counters()
619 kfd_sysfs_create_file(kobj_counters, &pdd->attr_faults, in kfd_procfs_add_sysfs_counters()
621 kfd_sysfs_create_file(kobj_counters, &pdd->attr_page_in, in kfd_procfs_add_sysfs_counters()
623 kfd_sysfs_create_file(kobj_counters, &pdd->attr_page_out, in kfd_procfs_add_sysfs_counters()
641 struct kfd_process_device *pdd = p->pdds[i]; in kfd_procfs_add_sysfs_files() local
643 snprintf(pdd->vram_filename, MAX_SYSFS_FILENAME_LEN, "vram_%u", in kfd_procfs_add_sysfs_files()
644 pdd->dev->id); in kfd_procfs_add_sysfs_files()
645 kfd_sysfs_create_file(p->kobj, &pdd->attr_vram, in kfd_procfs_add_sysfs_files()
646 pdd->vram_filename); in kfd_procfs_add_sysfs_files()
648 snprintf(pdd->sdma_filename, MAX_SYSFS_FILENAME_LEN, "sdma_%u", in kfd_procfs_add_sysfs_files()
649 pdd->dev->id); in kfd_procfs_add_sysfs_files()
650 kfd_sysfs_create_file(p->kobj, &pdd->attr_sdma, in kfd_procfs_add_sysfs_files()
651 pdd->sdma_filename); in kfd_procfs_add_sysfs_files()
692 struct kfd_process_device *pdd, void **kptr) in kfd_process_free_gpuvm() argument
694 struct kfd_dev *dev = pdd->dev; in kfd_process_free_gpuvm()
701 amdgpu_amdkfd_gpuvm_unmap_memory_from_gpu(dev->adev, mem, pdd->drm_priv); in kfd_process_free_gpuvm()
702 amdgpu_amdkfd_gpuvm_free_memory_of_gpu(dev->adev, mem, pdd->drm_priv, in kfd_process_free_gpuvm()
712 static int kfd_process_alloc_gpuvm(struct kfd_process_device *pdd, in kfd_process_alloc_gpuvm() argument
716 struct kfd_dev *kdev = pdd->dev; in kfd_process_alloc_gpuvm()
720 pdd->drm_priv, mem, NULL, in kfd_process_alloc_gpuvm()
726 pdd->drm_priv); in kfd_process_alloc_gpuvm()
748 amdgpu_amdkfd_gpuvm_unmap_memory_from_gpu(kdev->adev, *mem, pdd->drm_priv); in kfd_process_alloc_gpuvm()
751 amdgpu_amdkfd_gpuvm_free_memory_of_gpu(kdev->adev, *mem, pdd->drm_priv, in kfd_process_alloc_gpuvm()
765 static int kfd_process_device_reserve_ib_mem(struct kfd_process_device *pdd) in kfd_process_device_reserve_ib_mem() argument
767 struct qcm_process_device *qpd = &pdd->qpd; in kfd_process_device_reserve_ib_mem()
780 ret = kfd_process_alloc_gpuvm(pdd, qpd->ib_base, PAGE_SIZE, flags, in kfd_process_device_reserve_ib_mem()
791 static void kfd_process_device_destroy_ib_mem(struct kfd_process_device *pdd) in kfd_process_device_destroy_ib_mem() argument
793 struct qcm_process_device *qpd = &pdd->qpd; in kfd_process_device_destroy_ib_mem()
798 kfd_process_free_gpuvm(qpd->ib_mem, pdd, &qpd->ib_kaddr); in kfd_process_device_destroy_ib_mem()
950 static void kfd_process_device_free_bos(struct kfd_process_device *pdd) in kfd_process_device_free_bos() argument
952 struct kfd_process *p = pdd->process; in kfd_process_device_free_bos()
961 idr_for_each_entry(&pdd->alloc_idr, mem, id) { in kfd_process_device_free_bos()
972 amdgpu_amdkfd_gpuvm_free_memory_of_gpu(pdd->dev->adev, mem, in kfd_process_device_free_bos()
973 pdd->drm_priv, NULL); in kfd_process_device_free_bos()
974 kfd_process_device_remove_obj_handle(pdd, id); in kfd_process_device_free_bos()
984 struct kfd_process_device *pdd; in kfd_process_kunmap_signal_bo() local
994 pdd = kfd_get_process_device_data(kdev, p); in kfd_process_kunmap_signal_bo()
995 if (!pdd) in kfd_process_kunmap_signal_bo()
999 pdd, GET_IDR_HANDLE(p->signal_handle)); in kfd_process_kunmap_signal_bo()
1022 struct kfd_process_device *pdd = p->pdds[i]; in kfd_process_destroy_pdds() local
1025 pdd->dev->id, p->pasid); in kfd_process_destroy_pdds()
1027 kfd_process_device_destroy_cwsr_dgpu(pdd); in kfd_process_destroy_pdds()
1028 kfd_process_device_destroy_ib_mem(pdd); in kfd_process_destroy_pdds()
1030 if (pdd->drm_file) { in kfd_process_destroy_pdds()
1032 pdd->dev->adev, pdd->drm_priv); in kfd_process_destroy_pdds()
1033 fput(pdd->drm_file); in kfd_process_destroy_pdds()
1036 if (pdd->qpd.cwsr_kaddr && !pdd->qpd.cwsr_base) in kfd_process_destroy_pdds()
1037 free_pages((unsigned long)pdd->qpd.cwsr_kaddr, in kfd_process_destroy_pdds()
1040 bitmap_free(pdd->qpd.doorbell_bitmap); in kfd_process_destroy_pdds()
1041 idr_destroy(&pdd->alloc_idr); in kfd_process_destroy_pdds()
1043 kfd_free_process_doorbells(pdd->dev, pdd->doorbell_index); in kfd_process_destroy_pdds()
1045 if (pdd->dev->shared_resources.enable_mes) in kfd_process_destroy_pdds()
1046 amdgpu_amdkfd_free_gtt_mem(pdd->dev->adev, in kfd_process_destroy_pdds()
1047 pdd->proc_ctx_bo); in kfd_process_destroy_pdds()
1052 if (pdd->runtime_inuse) { in kfd_process_destroy_pdds()
1053 pm_runtime_mark_last_busy(adev_to_drm(pdd->dev->adev)->dev); in kfd_process_destroy_pdds()
1054 pm_runtime_put_autosuspend(adev_to_drm(pdd->dev->adev)->dev); in kfd_process_destroy_pdds()
1055 pdd->runtime_inuse = false; in kfd_process_destroy_pdds()
1058 kfree(pdd); in kfd_process_destroy_pdds()
1066 struct kfd_process_device *pdd; in kfd_process_remove_sysfs() local
1078 pdd = p->pdds[i]; in kfd_process_remove_sysfs()
1080 sysfs_remove_file(p->kobj, &pdd->attr_vram); in kfd_process_remove_sysfs()
1081 sysfs_remove_file(p->kobj, &pdd->attr_sdma); in kfd_process_remove_sysfs()
1083 sysfs_remove_file(pdd->kobj_stats, &pdd->attr_evict); in kfd_process_remove_sysfs()
1084 if (pdd->dev->kfd2kgd->get_cu_occupancy) in kfd_process_remove_sysfs()
1085 sysfs_remove_file(pdd->kobj_stats, in kfd_process_remove_sysfs()
1086 &pdd->attr_cu_occupancy); in kfd_process_remove_sysfs()
1087 kobject_del(pdd->kobj_stats); in kfd_process_remove_sysfs()
1088 kobject_put(pdd->kobj_stats); in kfd_process_remove_sysfs()
1089 pdd->kobj_stats = NULL; in kfd_process_remove_sysfs()
1093 pdd = p->pdds[i]; in kfd_process_remove_sysfs()
1095 sysfs_remove_file(pdd->kobj_counters, &pdd->attr_faults); in kfd_process_remove_sysfs()
1096 sysfs_remove_file(pdd->kobj_counters, &pdd->attr_page_in); in kfd_process_remove_sysfs()
1097 sysfs_remove_file(pdd->kobj_counters, &pdd->attr_page_out); in kfd_process_remove_sysfs()
1098 kobject_del(pdd->kobj_counters); in kfd_process_remove_sysfs()
1099 kobject_put(pdd->kobj_counters); in kfd_process_remove_sysfs()
1100 pdd->kobj_counters = NULL; in kfd_process_remove_sysfs()
1239 static int kfd_process_device_init_cwsr_dgpu(struct kfd_process_device *pdd) in kfd_process_device_init_cwsr_dgpu() argument
1241 struct kfd_dev *dev = pdd->dev; in kfd_process_device_init_cwsr_dgpu()
1242 struct qcm_process_device *qpd = &pdd->qpd; in kfd_process_device_init_cwsr_dgpu()
1254 ret = kfd_process_alloc_gpuvm(pdd, qpd->cwsr_base, in kfd_process_device_init_cwsr_dgpu()
1272 static void kfd_process_device_destroy_cwsr_dgpu(struct kfd_process_device *pdd) in kfd_process_device_destroy_cwsr_dgpu() argument
1274 struct kfd_dev *dev = pdd->dev; in kfd_process_device_destroy_cwsr_dgpu()
1275 struct qcm_process_device *qpd = &pdd->qpd; in kfd_process_device_destroy_cwsr_dgpu()
1280 kfd_process_free_gpuvm(qpd->cwsr_mem, pdd, &qpd->cwsr_kaddr); in kfd_process_device_destroy_cwsr_dgpu()
1493 struct kfd_process_device *pdd = NULL; in kfd_create_process_device_data() local
1498 pdd = kzalloc(sizeof(*pdd), GFP_KERNEL); in kfd_create_process_device_data()
1499 if (!pdd) in kfd_create_process_device_data()
1502 if (init_doorbell_bitmap(&pdd->qpd, dev)) { in kfd_create_process_device_data()
1507 pdd->dev = dev; in kfd_create_process_device_data()
1508 INIT_LIST_HEAD(&pdd->qpd.queues_list); in kfd_create_process_device_data()
1509 INIT_LIST_HEAD(&pdd->qpd.priv_queue_list); in kfd_create_process_device_data()
1510 pdd->qpd.dqm = dev->dqm; in kfd_create_process_device_data()
1511 pdd->qpd.pqm = &p->pqm; in kfd_create_process_device_data()
1512 pdd->qpd.evicted = 0; in kfd_create_process_device_data()
1513 pdd->qpd.mapped_gws_queue = false; in kfd_create_process_device_data()
1514 pdd->process = p; in kfd_create_process_device_data()
1515 pdd->bound = PDD_UNBOUND; in kfd_create_process_device_data()
1516 pdd->already_dequeued = false; in kfd_create_process_device_data()
1517 pdd->runtime_inuse = false; in kfd_create_process_device_data()
1518 pdd->vram_usage = 0; in kfd_create_process_device_data()
1519 pdd->sdma_past_activity_counter = 0; in kfd_create_process_device_data()
1520 pdd->user_gpu_id = dev->id; in kfd_create_process_device_data()
1521 atomic64_set(&pdd->evict_duration_counter, 0); in kfd_create_process_device_data()
1526 &pdd->proc_ctx_bo, in kfd_create_process_device_data()
1527 &pdd->proc_ctx_gpu_addr, in kfd_create_process_device_data()
1528 &pdd->proc_ctx_cpu_ptr, in kfd_create_process_device_data()
1534 memset(pdd->proc_ctx_cpu_ptr, 0, AMDGPU_MES_PROC_CTX_SIZE); in kfd_create_process_device_data()
1537 p->pdds[p->n_pdds++] = pdd; in kfd_create_process_device_data()
1540 idr_init(&pdd->alloc_idr); in kfd_create_process_device_data()
1542 return pdd; in kfd_create_process_device_data()
1545 kfree(pdd); in kfd_create_process_device_data()
1563 int kfd_process_device_init_vm(struct kfd_process_device *pdd, in kfd_process_device_init_vm() argument
1575 if (pdd->drm_priv) in kfd_process_device_init_vm()
1583 p = pdd->process; in kfd_process_device_init_vm()
1584 dev = pdd->dev; in kfd_process_device_init_vm()
1593 pdd->drm_priv = drm_file->private_data; in kfd_process_device_init_vm()
1594 atomic64_set(&pdd->tlb_seq, 0); in kfd_process_device_init_vm()
1596 ret = kfd_process_device_reserve_ib_mem(pdd); in kfd_process_device_init_vm()
1599 ret = kfd_process_device_init_cwsr_dgpu(pdd); in kfd_process_device_init_vm()
1607 pdd->drm_file = drm_file; in kfd_process_device_init_vm()
1612 kfd_process_device_destroy_cwsr_dgpu(pdd); in kfd_process_device_init_vm()
1614 kfd_process_device_destroy_ib_mem(pdd); in kfd_process_device_init_vm()
1616 pdd->drm_priv = NULL; in kfd_process_device_init_vm()
1632 struct kfd_process_device *pdd; in kfd_bind_process_to_device() local
1635 pdd = kfd_get_process_device_data(dev, p); in kfd_bind_process_to_device()
1636 if (!pdd) { in kfd_bind_process_to_device()
1641 if (!pdd->drm_priv) in kfd_bind_process_to_device()
1649 if (!pdd->runtime_inuse) { in kfd_bind_process_to_device()
1657 err = kfd_iommu_bind_process_to_device(pdd); in kfd_bind_process_to_device()
1665 pdd->runtime_inuse = true; in kfd_bind_process_to_device()
1667 return pdd; in kfd_bind_process_to_device()
1671 if (!pdd->runtime_inuse) { in kfd_bind_process_to_device()
1682 int kfd_process_device_create_obj_handle(struct kfd_process_device *pdd, in kfd_process_device_create_obj_handle() argument
1685 return idr_alloc(&pdd->alloc_idr, mem, 0, 0, GFP_KERNEL); in kfd_process_device_create_obj_handle()
1691 void *kfd_process_device_translate_handle(struct kfd_process_device *pdd, in kfd_process_device_translate_handle() argument
1697 return idr_find(&pdd->alloc_idr, handle); in kfd_process_device_translate_handle()
1703 void kfd_process_device_remove_obj_handle(struct kfd_process_device *pdd, in kfd_process_device_remove_obj_handle() argument
1707 idr_remove(&pdd->alloc_idr, handle); in kfd_process_device_remove_obj_handle()
1759 struct kfd_process_device *pdd = p->pdds[i]; in kfd_process_evict_queues() local
1761 kfd_smi_event_queue_eviction(pdd->dev, p->lead_thread->pid, in kfd_process_evict_queues()
1764 r = pdd->dev->dqm->ops.evict_process_queues(pdd->dev->dqm, in kfd_process_evict_queues()
1765 &pdd->qpd); in kfd_process_evict_queues()
1784 struct kfd_process_device *pdd = p->pdds[i]; in kfd_process_evict_queues() local
1789 kfd_smi_event_queue_restore(pdd->dev, p->lead_thread->pid); in kfd_process_evict_queues()
1791 if (pdd->dev->dqm->ops.restore_process_queues(pdd->dev->dqm, in kfd_process_evict_queues()
1792 &pdd->qpd)) in kfd_process_evict_queues()
1808 struct kfd_process_device *pdd = p->pdds[i]; in kfd_process_restore_queues() local
1810 kfd_smi_event_queue_restore(pdd->dev, p->lead_thread->pid); in kfd_process_restore_queues()
1812 r = pdd->dev->dqm->ops.restore_process_queues(pdd->dev->dqm, in kfd_process_restore_queues()
1813 &pdd->qpd); in kfd_process_restore_queues()
1969 struct kfd_process_device *pdd; in kfd_reserved_mem_mmap() local
1977 pdd = kfd_get_process_device_data(dev, process); in kfd_reserved_mem_mmap()
1978 if (!pdd) in kfd_reserved_mem_mmap()
1980 qpd = &pdd->qpd; in kfd_reserved_mem_mmap()
1997 void kfd_flush_tlb(struct kfd_process_device *pdd, enum TLB_FLUSH_TYPE type) in kfd_flush_tlb() argument
1999 struct amdgpu_vm *vm = drm_priv_to_vm(pdd->drm_priv); in kfd_flush_tlb()
2001 struct kfd_dev *dev = pdd->dev; in kfd_flush_tlb()
2008 if (atomic64_xchg(&pdd->tlb_seq, tlb_seq) == tlb_seq) in kfd_flush_tlb()
2015 if (pdd->qpd.vmid) in kfd_flush_tlb()
2017 pdd->qpd.vmid); in kfd_flush_tlb()
2020 pdd->process->pasid, type); in kfd_flush_tlb()
2030 struct kfd_process_device *pdd = p->pdds[i]; in kfd_process_device_data_by_id() local
2032 if (pdd->user_gpu_id == gpu_id) in kfd_process_device_data_by_id()
2033 return pdd; in kfd_process_device_data_by_id()
2047 struct kfd_process_device *pdd = p->pdds[i]; in kfd_process_get_user_gpu_id() local
2049 if (pdd->dev->id == actual_gpu_id) in kfd_process_get_user_gpu_id()
2050 return pdd->user_gpu_id; in kfd_process_get_user_gpu_id()