Lines Matching refs:gpusvm
325 ¬ifier->gpusvm->notifier_list)) in __drm_gpusvm_notifier_next()
395 struct drm_gpusvm *gpusvm = notifier->gpusvm; in drm_gpusvm_notifier_invalidate() local
400 down_write(&gpusvm->notifier_lock); in drm_gpusvm_notifier_invalidate()
402 gpusvm->ops->invalidate(gpusvm, notifier, mmu_range); in drm_gpusvm_notifier_invalidate()
403 up_write(&gpusvm->notifier_lock); in drm_gpusvm_notifier_invalidate()
435 int drm_gpusvm_init(struct drm_gpusvm *gpusvm, in drm_gpusvm_init() argument
446 gpusvm->name = name; in drm_gpusvm_init()
447 gpusvm->drm = drm; in drm_gpusvm_init()
448 gpusvm->mm = mm; in drm_gpusvm_init()
449 gpusvm->device_private_page_owner = device_private_page_owner; in drm_gpusvm_init()
450 gpusvm->mm_start = mm_start; in drm_gpusvm_init()
451 gpusvm->mm_range = mm_range; in drm_gpusvm_init()
452 gpusvm->notifier_size = notifier_size; in drm_gpusvm_init()
453 gpusvm->ops = ops; in drm_gpusvm_init()
454 gpusvm->chunk_sizes = chunk_sizes; in drm_gpusvm_init()
455 gpusvm->num_chunks = num_chunks; in drm_gpusvm_init()
458 gpusvm->root = RB_ROOT_CACHED; in drm_gpusvm_init()
459 INIT_LIST_HEAD(&gpusvm->notifier_list); in drm_gpusvm_init()
461 init_rwsem(&gpusvm->notifier_lock); in drm_gpusvm_init()
464 might_lock(&gpusvm->notifier_lock); in drm_gpusvm_init()
468 gpusvm->lock_dep_map = NULL; in drm_gpusvm_init()
485 drm_gpusvm_notifier_find(struct drm_gpusvm *gpusvm, in drm_gpusvm_notifier_find() argument
488 return notifier_iter_first(&gpusvm->root, fault_addr, fault_addr + 1); in drm_gpusvm_notifier_find()
509 static void drm_gpusvm_notifier_insert(struct drm_gpusvm *gpusvm, in drm_gpusvm_notifier_insert() argument
515 interval_tree_insert(¬ifier->itree, &gpusvm->root); in drm_gpusvm_notifier_insert()
521 head = &gpusvm->notifier_list; in drm_gpusvm_notifier_insert()
533 static void drm_gpusvm_notifier_remove(struct drm_gpusvm *gpusvm, in drm_gpusvm_notifier_remove() argument
536 interval_tree_remove(¬ifier->itree, &gpusvm->root); in drm_gpusvm_notifier_remove()
547 void drm_gpusvm_fini(struct drm_gpusvm *gpusvm) in drm_gpusvm_fini() argument
551 drm_gpusvm_for_each_notifier_safe(notifier, next, gpusvm, 0, LONG_MAX) { in drm_gpusvm_fini()
562 drm_gpusvm_range_remove(gpusvm, range); in drm_gpusvm_fini()
565 mmdrop(gpusvm->mm); in drm_gpusvm_fini()
566 WARN_ON(!RB_EMPTY_ROOT(&gpusvm->root.rb_root)); in drm_gpusvm_fini()
580 drm_gpusvm_notifier_alloc(struct drm_gpusvm *gpusvm, unsigned long fault_addr) in drm_gpusvm_notifier_alloc() argument
584 if (gpusvm->ops->notifier_alloc) in drm_gpusvm_notifier_alloc()
585 notifier = gpusvm->ops->notifier_alloc(); in drm_gpusvm_notifier_alloc()
592 notifier->gpusvm = gpusvm; in drm_gpusvm_notifier_alloc()
593 notifier->itree.start = ALIGN_DOWN(fault_addr, gpusvm->notifier_size); in drm_gpusvm_notifier_alloc()
594 notifier->itree.last = ALIGN(fault_addr + 1, gpusvm->notifier_size) - 1; in drm_gpusvm_notifier_alloc()
609 static void drm_gpusvm_notifier_free(struct drm_gpusvm *gpusvm, in drm_gpusvm_notifier_free() argument
614 if (gpusvm->ops->notifier_free) in drm_gpusvm_notifier_free()
615 gpusvm->ops->notifier_free(notifier); in drm_gpusvm_notifier_free()
644 drm_gpusvm_notifier_lock(notifier->gpusvm); in drm_gpusvm_range_insert()
654 drm_gpusvm_notifier_unlock(notifier->gpusvm); in drm_gpusvm_range_insert()
684 drm_gpusvm_range_alloc(struct drm_gpusvm *gpusvm, in drm_gpusvm_range_alloc() argument
691 if (gpusvm->ops->range_alloc) in drm_gpusvm_range_alloc()
692 range = gpusvm->ops->range_alloc(gpusvm); in drm_gpusvm_range_alloc()
700 range->gpusvm = gpusvm; in drm_gpusvm_range_alloc()
723 static bool drm_gpusvm_check_pages(struct drm_gpusvm *gpusvm, in drm_gpusvm_check_pages() argument
732 .dev_private_owner = gpusvm->device_private_page_owner, in drm_gpusvm_check_pages()
740 mmap_assert_locked(gpusvm->mm); in drm_gpusvm_check_pages()
794 drm_gpusvm_range_chunk_size(struct drm_gpusvm *gpusvm, in drm_gpusvm_range_chunk_size() argument
806 for (; i < gpusvm->num_chunks; ++i) { in drm_gpusvm_range_chunk_size()
807 start = ALIGN_DOWN(fault_addr, gpusvm->chunk_sizes[i]); in drm_gpusvm_range_chunk_size()
808 end = ALIGN(fault_addr + 1, gpusvm->chunk_sizes[i]); in drm_gpusvm_range_chunk_size()
817 if (i == gpusvm->num_chunks) in drm_gpusvm_range_chunk_size()
847 !drm_gpusvm_check_pages(gpusvm, notifier, start, end)) { in drm_gpusvm_range_chunk_size()
863 static void drm_gpusvm_driver_lock_held(struct drm_gpusvm *gpusvm) in drm_gpusvm_driver_lock_held() argument
865 if ((gpusvm)->lock_dep_map) in drm_gpusvm_driver_lock_held()
866 lockdep_assert(lock_is_held_type((gpusvm)->lock_dep_map, 0)); in drm_gpusvm_driver_lock_held()
869 static void drm_gpusvm_driver_lock_held(struct drm_gpusvm *gpusvm) in drm_gpusvm_driver_lock_held() argument
884 drm_gpusvm_find_vma_start(struct drm_gpusvm *gpusvm, in drm_gpusvm_find_vma_start() argument
888 struct mm_struct *mm = gpusvm->mm; in drm_gpusvm_find_vma_start()
922 drm_gpusvm_range_find_or_insert(struct drm_gpusvm *gpusvm, in drm_gpusvm_range_find_or_insert() argument
930 struct mm_struct *mm = gpusvm->mm; in drm_gpusvm_range_find_or_insert()
937 drm_gpusvm_driver_lock_held(gpusvm); in drm_gpusvm_range_find_or_insert()
939 if (fault_addr < gpusvm->mm_start || in drm_gpusvm_range_find_or_insert()
940 fault_addr > gpusvm->mm_start + gpusvm->mm_range) in drm_gpusvm_range_find_or_insert()
946 notifier = drm_gpusvm_notifier_find(gpusvm, fault_addr); in drm_gpusvm_range_find_or_insert()
948 notifier = drm_gpusvm_notifier_alloc(gpusvm, fault_addr); in drm_gpusvm_range_find_or_insert()
987 chunk_size = drm_gpusvm_range_chunk_size(gpusvm, notifier, vas, in drm_gpusvm_range_find_or_insert()
996 range = drm_gpusvm_range_alloc(gpusvm, notifier, fault_addr, chunk_size, in drm_gpusvm_range_find_or_insert()
1005 drm_gpusvm_notifier_insert(gpusvm, notifier); in drm_gpusvm_range_find_or_insert()
1019 drm_gpusvm_notifier_free(gpusvm, notifier); in drm_gpusvm_range_find_or_insert()
1035 static void __drm_gpusvm_range_unmap_pages(struct drm_gpusvm *gpusvm, in __drm_gpusvm_range_unmap_pages() argument
1041 struct device *dev = gpusvm->drm->dev; in __drm_gpusvm_range_unmap_pages()
1043 lockdep_assert_held(&gpusvm->notifier_lock); in __drm_gpusvm_range_unmap_pages()
1080 static void drm_gpusvm_range_free_pages(struct drm_gpusvm *gpusvm, in drm_gpusvm_range_free_pages() argument
1083 lockdep_assert_held(&gpusvm->notifier_lock); in drm_gpusvm_range_free_pages()
1100 void drm_gpusvm_range_remove(struct drm_gpusvm *gpusvm, in drm_gpusvm_range_remove() argument
1107 drm_gpusvm_driver_lock_held(gpusvm); in drm_gpusvm_range_remove()
1109 notifier = drm_gpusvm_notifier_find(gpusvm, in drm_gpusvm_range_remove()
1114 drm_gpusvm_notifier_lock(gpusvm); in drm_gpusvm_range_remove()
1115 __drm_gpusvm_range_unmap_pages(gpusvm, range, npages); in drm_gpusvm_range_remove()
1116 drm_gpusvm_range_free_pages(gpusvm, range); in drm_gpusvm_range_remove()
1118 drm_gpusvm_notifier_unlock(gpusvm); in drm_gpusvm_range_remove()
1125 drm_gpusvm_notifier_remove(gpusvm, notifier); in drm_gpusvm_range_remove()
1126 drm_gpusvm_notifier_free(gpusvm, notifier); in drm_gpusvm_range_remove()
1160 struct drm_gpusvm *gpusvm = range->gpusvm; in drm_gpusvm_range_destroy() local
1162 if (gpusvm->ops->range_free) in drm_gpusvm_range_destroy()
1163 gpusvm->ops->range_free(range); in drm_gpusvm_range_destroy()
1195 bool drm_gpusvm_range_pages_valid(struct drm_gpusvm *gpusvm, in drm_gpusvm_range_pages_valid() argument
1198 lockdep_assert_held(&gpusvm->notifier_lock); in drm_gpusvm_range_pages_valid()
1215 drm_gpusvm_range_pages_valid_unlocked(struct drm_gpusvm *gpusvm, in drm_gpusvm_range_pages_valid_unlocked() argument
1223 drm_gpusvm_notifier_lock(gpusvm); in drm_gpusvm_range_pages_valid_unlocked()
1224 pages_valid = drm_gpusvm_range_pages_valid(gpusvm, range); in drm_gpusvm_range_pages_valid_unlocked()
1226 drm_gpusvm_range_free_pages(gpusvm, range); in drm_gpusvm_range_pages_valid_unlocked()
1227 drm_gpusvm_notifier_unlock(gpusvm); in drm_gpusvm_range_pages_valid_unlocked()
1243 int drm_gpusvm_range_get_pages(struct drm_gpusvm *gpusvm, in drm_gpusvm_range_get_pages() argument
1254 .dev_private_owner = gpusvm->device_private_page_owner, in drm_gpusvm_range_get_pages()
1256 struct mm_struct *mm = gpusvm->mm; in drm_gpusvm_range_get_pages()
1273 if (drm_gpusvm_range_pages_valid_unlocked(gpusvm, range)) in drm_gpusvm_range_get_pages()
1311 drm_gpusvm_notifier_lock(gpusvm); in drm_gpusvm_range_get_pages()
1315 drm_gpusvm_notifier_unlock(gpusvm); in drm_gpusvm_range_get_pages()
1321 drm_gpusvm_notifier_unlock(gpusvm); in drm_gpusvm_range_get_pages()
1328 drm_gpusvm_notifier_unlock(gpusvm); in drm_gpusvm_range_get_pages()
1361 if (drm_WARN_ON(gpusvm->drm, !dpagemap)) { in drm_gpusvm_range_get_pages()
1373 gpusvm->drm->dev, in drm_gpusvm_range_get_pages()
1376 if (dma_mapping_error(gpusvm->drm->dev, in drm_gpusvm_range_get_pages()
1394 addr = dma_map_page(gpusvm->drm->dev, in drm_gpusvm_range_get_pages()
1398 if (dma_mapping_error(gpusvm->drm->dev, addr)) { in drm_gpusvm_range_get_pages()
1420 drm_gpusvm_notifier_unlock(gpusvm); in drm_gpusvm_range_get_pages()
1428 __drm_gpusvm_range_unmap_pages(gpusvm, range, num_dma_mapped); in drm_gpusvm_range_get_pages()
1429 drm_gpusvm_notifier_unlock(gpusvm); in drm_gpusvm_range_get_pages()
1451 void drm_gpusvm_range_unmap_pages(struct drm_gpusvm *gpusvm, in drm_gpusvm_range_unmap_pages() argument
1459 lockdep_assert_held_write(&gpusvm->notifier_lock); in drm_gpusvm_range_unmap_pages()
1461 drm_gpusvm_notifier_lock(gpusvm); in drm_gpusvm_range_unmap_pages()
1463 __drm_gpusvm_range_unmap_pages(gpusvm, range, npages); in drm_gpusvm_range_unmap_pages()
1466 drm_gpusvm_notifier_unlock(gpusvm); in drm_gpusvm_range_unmap_pages()
1479 int drm_gpusvm_range_evict(struct drm_gpusvm *gpusvm, in drm_gpusvm_range_evict() argument
1496 struct mm_struct *mm = gpusvm->mm; in drm_gpusvm_range_evict()
1535 bool drm_gpusvm_has_mapping(struct drm_gpusvm *gpusvm, unsigned long start, in drm_gpusvm_has_mapping() argument
1540 drm_gpusvm_for_each_notifier(notifier, gpusvm, start, end) { in drm_gpusvm_has_mapping()
1562 lockdep_assert_held_write(&range->gpusvm->notifier_lock); in drm_gpusvm_range_set_unmapped()