Lines Matching refs:userptr
64 return mmu_interval_check_retry(&uvma->userptr.notifier, in xe_vma_userptr_check_repin()
65 uvma->userptr.notifier_seq) ? in xe_vma_userptr_check_repin()
252 down_read(&vm->userptr.notifier_lock); in xe_vm_add_compute_exec_queue()
266 up_read(&vm->userptr.notifier_lock); in xe_vm_add_compute_exec_queue()
315 lockdep_assert_held_read(&vm->userptr.notifier_lock); in __xe_vm_userptr_needs_repin()
317 return (list_empty(&vm->userptr.repin_list) && in __xe_vm_userptr_needs_repin()
318 list_empty(&vm->userptr.invalidated)) ? 0 : -EAGAIN; in __xe_vm_userptr_needs_repin()
549 down_read(&vm->userptr.notifier_lock); in preempt_rebind_work_func()
551 up_read(&vm->userptr.notifier_lock); in preempt_rebind_work_func()
565 up_read(&vm->userptr.notifier_lock); in preempt_rebind_work_func()
588 struct xe_userptr *userptr = &uvma->userptr; in __vma_userptr_invalidate() local
600 spin_lock(&vm->userptr.invalidated_lock); in __vma_userptr_invalidate()
601 list_move_tail(&userptr->invalidate_link, in __vma_userptr_invalidate()
602 &vm->userptr.invalidated); in __vma_userptr_invalidate()
603 spin_unlock(&vm->userptr.invalidated_lock); in __vma_userptr_invalidate()
623 if (xe_vm_in_fault_mode(vm) && userptr->initial_bind) { in __vma_userptr_invalidate()
635 struct xe_userptr_vma *uvma = container_of(mni, typeof(*uvma), userptr.notifier); in vma_userptr_invalidate()
649 down_write(&vm->userptr.notifier_lock); in vma_userptr_invalidate()
653 up_write(&vm->userptr.notifier_lock); in vma_userptr_invalidate()
677 lockdep_assert_held(&vm->userptr.notifier_lock); in xe_vma_userptr_force_invalidate()
684 if (!mmu_interval_read_retry(&uvma->userptr.notifier, in xe_vma_userptr_force_invalidate()
685 uvma->userptr.notifier_seq)) in xe_vma_userptr_force_invalidate()
686 uvma->userptr.notifier_seq -= 2; in xe_vma_userptr_force_invalidate()
700 spin_lock(&vm->userptr.invalidated_lock); in xe_vm_userptr_pin()
701 xe_assert(vm->xe, list_empty(&vm->userptr.repin_list)); in xe_vm_userptr_pin()
702 list_for_each_entry_safe(uvma, next, &vm->userptr.invalidated, in xe_vm_userptr_pin()
703 userptr.invalidate_link) { in xe_vm_userptr_pin()
704 list_del_init(&uvma->userptr.invalidate_link); in xe_vm_userptr_pin()
705 list_add_tail(&uvma->userptr.repin_link, in xe_vm_userptr_pin()
706 &vm->userptr.repin_list); in xe_vm_userptr_pin()
708 spin_unlock(&vm->userptr.invalidated_lock); in xe_vm_userptr_pin()
711 list_for_each_entry_safe(uvma, next, &vm->userptr.repin_list, in xe_vm_userptr_pin()
712 userptr.repin_link) { in xe_vm_userptr_pin()
715 list_del_init(&uvma->userptr.repin_link); in xe_vm_userptr_pin()
735 down_read(&vm->userptr.notifier_lock); in xe_vm_userptr_pin()
737 up_read(&vm->userptr.notifier_lock); in xe_vm_userptr_pin()
745 list_del_init(&uvma->userptr.repin_link); in xe_vm_userptr_pin()
752 down_write(&vm->userptr.notifier_lock); in xe_vm_userptr_pin()
753 spin_lock(&vm->userptr.invalidated_lock); in xe_vm_userptr_pin()
754 list_for_each_entry_safe(uvma, next, &vm->userptr.repin_list, in xe_vm_userptr_pin()
755 userptr.repin_link) { in xe_vm_userptr_pin()
756 list_del_init(&uvma->userptr.repin_link); in xe_vm_userptr_pin()
757 list_move_tail(&uvma->userptr.invalidate_link, in xe_vm_userptr_pin()
758 &vm->userptr.invalidated); in xe_vm_userptr_pin()
760 spin_unlock(&vm->userptr.invalidated_lock); in xe_vm_userptr_pin()
761 up_write(&vm->userptr.notifier_lock); in xe_vm_userptr_pin()
779 return (list_empty_careful(&vm->userptr.repin_list) && in xe_vm_userptr_check_repin()
780 list_empty_careful(&vm->userptr.invalidated)) ? 0 : -EAGAIN; in xe_vm_userptr_check_repin()
1246 struct xe_userptr *userptr = &to_userptr_vma(vma)->userptr; in xe_vma_create() local
1250 INIT_LIST_HEAD(&userptr->invalidate_link); in xe_vma_create()
1251 INIT_LIST_HEAD(&userptr->repin_link); in xe_vma_create()
1253 mutex_init(&userptr->unmap_mutex); in xe_vma_create()
1255 err = mmu_interval_notifier_insert(&userptr->notifier, in xe_vma_create()
1264 userptr->notifier_seq = LONG_MAX; in xe_vma_create()
1284 struct xe_userptr *userptr = &uvma->userptr; in xe_vma_destroy_late() local
1286 if (userptr->sg) in xe_vma_destroy_late()
1294 mmu_interval_notifier_remove(&userptr->notifier); in xe_vma_destroy_late()
1295 mutex_destroy(&userptr->unmap_mutex); in xe_vma_destroy_late()
1333 spin_lock(&vm->userptr.invalidated_lock); in xe_vma_destroy()
1334 xe_assert(vm->xe, list_empty(&to_userptr_vma(vma)->userptr.repin_link)); in xe_vma_destroy()
1335 list_del(&to_userptr_vma(vma)->userptr.invalidate_link); in xe_vma_destroy()
1336 spin_unlock(&vm->userptr.invalidated_lock); in xe_vma_destroy()
1686 INIT_LIST_HEAD(&vm->userptr.repin_list); in xe_vm_create()
1687 INIT_LIST_HEAD(&vm->userptr.invalidated); in xe_vm_create()
1688 init_rwsem(&vm->userptr.notifier_lock); in xe_vm_create()
1689 spin_lock_init(&vm->userptr.invalidated_lock); in xe_vm_create()
1903 down_read(&vm->userptr.notifier_lock); in xe_vm_close_and_put()
1905 up_read(&vm->userptr.notifier_lock); in xe_vm_close_and_put()
2210 down_read(&vm->userptr.notifier_lock); in prep_vma_destroy()
2212 up_read(&vm->userptr.notifier_lock); in prep_vma_destroy()
2794 down_read(&vm->userptr.notifier_lock); in xe_vma_op_unwind()
2796 up_read(&vm->userptr.notifier_lock); in xe_vma_op_unwind()
2815 down_read(&vm->userptr.notifier_lock); in xe_vma_op_unwind()
2817 up_read(&vm->userptr.notifier_lock); in xe_vma_op_unwind()
3940 lockdep_assert(lockdep_is_held_type(&vm->userptr.notifier_lock, 0) || in xe_vm_invalidate_vma()
3941 (lockdep_is_held_type(&vm->userptr.notifier_lock, 1) && in xe_vm_invalidate_vma()
3945 (&to_userptr_vma(vma)->userptr.notifier, in xe_vm_invalidate_vma()
3946 to_userptr_vma(vma)->userptr.notifier_seq)); in xe_vm_invalidate_vma()
4049 to_userptr_vma(vma)->userptr.notifier.mm; in xe_vm_snapshot_capture()
4090 void __user *userptr = (void __user *)(size_t)snap->snap[i].bo_ofs; in xe_vm_snapshot_capture_delayed() local
4093 if (!copy_from_user(snap->snap[i].data, userptr, snap->snap[i].len)) in xe_vm_snapshot_capture_delayed()