Lines Matching refs:userptr
62 return mmu_interval_check_retry(&uvma->userptr.notifier, in xe_vma_userptr_check_repin()
63 uvma->userptr.notifier_seq) ? in xe_vma_userptr_check_repin()
250 down_read(&vm->userptr.notifier_lock); in xe_vm_add_compute_exec_queue()
264 up_read(&vm->userptr.notifier_lock); in xe_vm_add_compute_exec_queue()
312 lockdep_assert_held_read(&vm->userptr.notifier_lock); in __xe_vm_userptr_needs_repin()
314 return (list_empty(&vm->userptr.repin_list) && in __xe_vm_userptr_needs_repin()
315 list_empty(&vm->userptr.invalidated)) ? 0 : -EAGAIN; in __xe_vm_userptr_needs_repin()
546 down_read(&vm->userptr.notifier_lock); in preempt_rebind_work_func()
548 up_read(&vm->userptr.notifier_lock); in preempt_rebind_work_func()
562 up_read(&vm->userptr.notifier_lock); in preempt_rebind_work_func()
587 struct xe_userptr *userptr = container_of(mni, typeof(*userptr), notifier); in vma_userptr_invalidate() local
588 struct xe_userptr_vma *uvma = container_of(userptr, typeof(*uvma), userptr); in vma_userptr_invalidate()
605 down_write(&vm->userptr.notifier_lock); in vma_userptr_invalidate()
609 if (!userptr->initial_bind) { in vma_userptr_invalidate()
610 up_write(&vm->userptr.notifier_lock); in vma_userptr_invalidate()
620 spin_lock(&vm->userptr.invalidated_lock); in vma_userptr_invalidate()
621 list_move_tail(&userptr->invalidate_link, in vma_userptr_invalidate()
622 &vm->userptr.invalidated); in vma_userptr_invalidate()
623 spin_unlock(&vm->userptr.invalidated_lock); in vma_userptr_invalidate()
626 up_write(&vm->userptr.notifier_lock); in vma_userptr_invalidate()
669 spin_lock(&vm->userptr.invalidated_lock); in xe_vm_userptr_pin()
670 list_for_each_entry_safe(uvma, next, &vm->userptr.invalidated, in xe_vm_userptr_pin()
671 userptr.invalidate_link) { in xe_vm_userptr_pin()
672 list_del_init(&uvma->userptr.invalidate_link); in xe_vm_userptr_pin()
673 list_move_tail(&uvma->userptr.repin_link, in xe_vm_userptr_pin()
674 &vm->userptr.repin_list); in xe_vm_userptr_pin()
676 spin_unlock(&vm->userptr.invalidated_lock); in xe_vm_userptr_pin()
679 list_for_each_entry_safe(uvma, next, &vm->userptr.repin_list, in xe_vm_userptr_pin()
680 userptr.repin_link) { in xe_vm_userptr_pin()
683 list_del_init(&uvma->userptr.repin_link); in xe_vm_userptr_pin()
699 list_del_init(&uvma->userptr.repin_link); in xe_vm_userptr_pin()
721 return (list_empty_careful(&vm->userptr.repin_list) && in xe_vm_userptr_check_repin()
722 list_empty_careful(&vm->userptr.invalidated)) ? 0 : -EAGAIN; in xe_vm_userptr_check_repin()
983 struct xe_userptr *userptr = &to_userptr_vma(vma)->userptr; in xe_vma_create() local
987 INIT_LIST_HEAD(&userptr->invalidate_link); in xe_vma_create()
988 INIT_LIST_HEAD(&userptr->repin_link); in xe_vma_create()
991 err = mmu_interval_notifier_insert(&userptr->notifier, in xe_vma_create()
1000 userptr->notifier_seq = LONG_MAX; in xe_vma_create()
1020 struct xe_userptr *userptr = &uvma->userptr; in xe_vma_destroy_late() local
1022 if (userptr->sg) in xe_vma_destroy_late()
1030 mmu_interval_notifier_remove(&userptr->notifier); in xe_vma_destroy_late()
1068 spin_lock(&vm->userptr.invalidated_lock); in xe_vma_destroy()
1069 list_del(&to_userptr_vma(vma)->userptr.invalidate_link); in xe_vma_destroy()
1070 spin_unlock(&vm->userptr.invalidated_lock); in xe_vma_destroy()
1399 INIT_LIST_HEAD(&vm->userptr.repin_list); in xe_vm_create()
1400 INIT_LIST_HEAD(&vm->userptr.invalidated); in xe_vm_create()
1401 init_rwsem(&vm->userptr.notifier_lock); in xe_vm_create()
1402 spin_lock_init(&vm->userptr.invalidated_lock); in xe_vm_create()
1568 down_read(&vm->userptr.notifier_lock); in xe_vm_close_and_put()
1570 up_read(&vm->userptr.notifier_lock); in xe_vm_close_and_put()
1845 down_read(&vm->userptr.notifier_lock); in prep_vma_destroy()
1847 up_read(&vm->userptr.notifier_lock); in prep_vma_destroy()
2295 down_read(&vm->userptr.notifier_lock); in xe_vma_op_unwind()
2297 up_read(&vm->userptr.notifier_lock); in xe_vma_op_unwind()
2316 down_read(&vm->userptr.notifier_lock); in xe_vma_op_unwind()
2318 up_read(&vm->userptr.notifier_lock); in xe_vma_op_unwind()
3183 (&to_userptr_vma(vma)->userptr.notifier, in xe_vm_invalidate_vma()
3184 to_userptr_vma(vma)->userptr.notifier_seq)); in xe_vm_invalidate_vma()
3280 to_userptr_vma(vma)->userptr.notifier.mm; in xe_vm_snapshot_capture()
3330 void __user *userptr = (void __user *)(size_t)snap->snap[i].bo_ofs; in xe_vm_snapshot_capture_delayed() local
3333 if (!copy_from_user(snap->snap[i].data, userptr, snap->snap[i].len)) in xe_vm_snapshot_capture_delayed()