Lines Matching refs:submit
30 struct msm_gem_submit *submit; in submit_create() local
34 sz = struct_size(submit, bos, nr_bos) + in submit_create()
35 ((u64)nr_cmds * sizeof(submit->cmd[0])); in submit_create()
40 submit = kzalloc(sz, GFP_KERNEL); in submit_create()
41 if (!submit) in submit_create()
44 ret = drm_sched_job_init(&submit->base, queue->entity, queue); in submit_create()
46 kfree(submit); in submit_create()
50 kref_init(&submit->ref); in submit_create()
51 submit->dev = dev; in submit_create()
52 submit->aspace = queue->ctx->aspace; in submit_create()
53 submit->gpu = gpu; in submit_create()
54 submit->cmd = (void *)&submit->bos[nr_bos]; in submit_create()
55 submit->queue = queue; in submit_create()
56 submit->pid = get_pid(task_pid(current)); in submit_create()
57 submit->ring = gpu->rb[queue->ring_nr]; in submit_create()
58 submit->fault_dumped = false; in submit_create()
61 submit->ident = atomic_inc_return(&ident) - 1; in submit_create()
63 INIT_LIST_HEAD(&submit->node); in submit_create()
65 return submit; in submit_create()
70 struct msm_gem_submit *submit = in __msm_gem_submit_destroy() local
74 if (submit->fence_id) { in __msm_gem_submit_destroy()
75 mutex_lock(&submit->queue->idr_lock); in __msm_gem_submit_destroy()
76 idr_remove(&submit->queue->fence_idr, submit->fence_id); in __msm_gem_submit_destroy()
77 mutex_unlock(&submit->queue->idr_lock); in __msm_gem_submit_destroy()
80 dma_fence_put(submit->user_fence); in __msm_gem_submit_destroy()
81 dma_fence_put(submit->hw_fence); in __msm_gem_submit_destroy()
83 put_pid(submit->pid); in __msm_gem_submit_destroy()
84 msm_submitqueue_put(submit->queue); in __msm_gem_submit_destroy()
86 for (i = 0; i < submit->nr_cmds; i++) in __msm_gem_submit_destroy()
87 kfree(submit->cmd[i].relocs); in __msm_gem_submit_destroy()
89 kfree(submit); in __msm_gem_submit_destroy()
92 static int submit_lookup_objects(struct msm_gem_submit *submit, in submit_lookup_objects() argument
106 submit->bos[i].flags = 0; in submit_lookup_objects()
125 submit->bos[i].handle = submit_bo.handle; in submit_lookup_objects()
126 submit->bos[i].flags = submit_bo.flags; in submit_lookup_objects()
128 submit->bos[i].iova = submit_bo.presumed; in submit_lookup_objects()
139 obj = idr_find(&file->object_idr, submit->bos[i].handle); in submit_lookup_objects()
141 DRM_ERROR("invalid handle %u at index %u\n", submit->bos[i].handle, i); in submit_lookup_objects()
148 submit->bos[i].obj = to_msm_bo(obj); in submit_lookup_objects()
155 submit->nr_bos = i; in submit_lookup_objects()
160 static int submit_lookup_cmds(struct msm_gem_submit *submit, in submit_lookup_cmds() argument
196 submit->cmd[i].type = submit_cmd.type; in submit_lookup_cmds()
197 submit->cmd[i].size = submit_cmd.size / 4; in submit_lookup_cmds()
198 submit->cmd[i].offset = submit_cmd.submit_offset / 4; in submit_lookup_cmds()
199 submit->cmd[i].idx = submit_cmd.submit_idx; in submit_lookup_cmds()
200 submit->cmd[i].nr_relocs = submit_cmd.nr_relocs; in submit_lookup_cmds()
211 submit->cmd[i].relocs = kmalloc(sz, GFP_KERNEL); in submit_lookup_cmds()
212 if (!submit->cmd[i].relocs) { in submit_lookup_cmds()
216 ret = copy_from_user(submit->cmd[i].relocs, userptr, sz); in submit_lookup_cmds()
231 static void submit_cleanup_bo(struct msm_gem_submit *submit, int i, in submit_cleanup_bo() argument
234 struct drm_gem_object *obj = &submit->bos[i].obj->base; in submit_cleanup_bo()
235 unsigned flags = submit->bos[i].flags & cleanup_flags; in submit_cleanup_bo()
242 submit->bos[i].flags &= ~cleanup_flags; in submit_cleanup_bo()
245 msm_gem_unpin_vma(submit->bos[i].vma); in submit_cleanup_bo()
254 static void submit_unlock_unpin_bo(struct msm_gem_submit *submit, int i) in submit_unlock_unpin_bo() argument
257 submit_cleanup_bo(submit, i, cleanup_flags); in submit_unlock_unpin_bo()
259 if (!(submit->bos[i].flags & BO_VALID)) in submit_unlock_unpin_bo()
260 submit->bos[i].iova = 0; in submit_unlock_unpin_bo()
264 static int submit_lock_objects(struct msm_gem_submit *submit) in submit_lock_objects() argument
269 for (i = 0; i < submit->nr_bos; i++) { in submit_lock_objects()
270 struct msm_gem_object *msm_obj = submit->bos[i].obj; in submit_lock_objects()
277 if (!(submit->bos[i].flags & BO_LOCKED)) { in submit_lock_objects()
279 &submit->ticket); in submit_lock_objects()
282 submit->bos[i].flags |= BO_LOCKED; in submit_lock_objects()
286 ww_acquire_done(&submit->ticket); in submit_lock_objects()
293 submit->bos[i].handle, i); in submit_lock_objects()
298 submit_unlock_unpin_bo(submit, i); in submit_lock_objects()
301 submit_unlock_unpin_bo(submit, slow_locked); in submit_lock_objects()
304 struct msm_gem_object *msm_obj = submit->bos[contended].obj; in submit_lock_objects()
307 &submit->ticket); in submit_lock_objects()
309 submit->bos[contended].flags |= BO_LOCKED; in submit_lock_objects()
324 static int submit_fence_sync(struct msm_gem_submit *submit, bool no_implicit) in submit_fence_sync() argument
328 for (i = 0; i < submit->nr_bos; i++) { in submit_fence_sync()
329 struct drm_gem_object *obj = &submit->bos[i].obj->base; in submit_fence_sync()
330 bool write = submit->bos[i].flags & MSM_SUBMIT_BO_WRITE; in submit_fence_sync()
352 if (submit->bos[i].flags & MSM_SUBMIT_BO_NO_IMPLICIT) in submit_fence_sync()
355 ret = drm_sched_job_add_implicit_dependencies(&submit->base, in submit_fence_sync()
365 static int submit_pin_objects(struct msm_gem_submit *submit) in submit_pin_objects() argument
369 submit->valid = true; in submit_pin_objects()
371 for (i = 0; i < submit->nr_bos; i++) { in submit_pin_objects()
372 struct drm_gem_object *obj = &submit->bos[i].obj->base; in submit_pin_objects()
376 vma = msm_gem_get_vma_locked(obj, submit->aspace); in submit_pin_objects()
386 submit->bos[i].flags |= BO_OBJ_PINNED | BO_VMA_PINNED; in submit_pin_objects()
387 submit->bos[i].vma = vma; in submit_pin_objects()
389 if (vma->iova == submit->bos[i].iova) { in submit_pin_objects()
390 submit->bos[i].flags |= BO_VALID; in submit_pin_objects()
392 submit->bos[i].iova = vma->iova; in submit_pin_objects()
394 submit->bos[i].flags &= ~BO_VALID; in submit_pin_objects()
395 submit->valid = false; in submit_pin_objects()
402 static void submit_attach_object_fences(struct msm_gem_submit *submit) in submit_attach_object_fences() argument
406 for (i = 0; i < submit->nr_bos; i++) { in submit_attach_object_fences()
407 struct drm_gem_object *obj = &submit->bos[i].obj->base; in submit_attach_object_fences()
409 if (submit->bos[i].flags & MSM_SUBMIT_BO_WRITE) in submit_attach_object_fences()
410 dma_resv_add_fence(obj->resv, submit->user_fence, in submit_attach_object_fences()
412 else if (submit->bos[i].flags & MSM_SUBMIT_BO_READ) in submit_attach_object_fences()
413 dma_resv_add_fence(obj->resv, submit->user_fence, in submit_attach_object_fences()
418 static int submit_bo(struct msm_gem_submit *submit, uint32_t idx, in submit_bo() argument
421 if (idx >= submit->nr_bos) { in submit_bo()
423 idx, submit->nr_bos); in submit_bo()
428 *obj = submit->bos[idx].obj; in submit_bo()
430 *iova = submit->bos[idx].iova; in submit_bo()
432 *valid = !!(submit->bos[idx].flags & BO_VALID); in submit_bo()
438 static int submit_reloc(struct msm_gem_submit *submit, struct msm_gem_object *obj, in submit_reloc() argument
487 ret = submit_bo(submit, submit_reloc.reloc_idx, NULL, &iova, &valid); in submit_reloc()
516 static void submit_cleanup(struct msm_gem_submit *submit, bool error) in submit_cleanup() argument
524 for (i = 0; i < submit->nr_bos; i++) { in submit_cleanup()
525 struct msm_gem_object *msm_obj = submit->bos[i].obj; in submit_cleanup()
526 submit_cleanup_bo(submit, i, cleanup_flags); in submit_cleanup()
532 void msm_submit_retire(struct msm_gem_submit *submit) in msm_submit_retire() argument
536 for (i = 0; i < submit->nr_bos; i++) { in msm_submit_retire()
537 struct drm_gem_object *obj = &submit->bos[i].obj->base; in msm_submit_retire()
549 static struct drm_syncobj **msm_parse_deps(struct msm_gem_submit *submit, in msm_parse_deps() argument
578 !drm_core_check_feature(submit->dev, DRIVER_SYNCOBJ_TIMELINE)) { in msm_parse_deps()
593 ret = drm_sched_job_add_dependency(&submit->base, fence); in msm_parse_deps()
722 struct msm_gem_submit *submit; in msm_ioctl_gem_submit() local
773 submit = submit_create(dev, gpu, queue, args->nr_bos, args->nr_cmds); in msm_ioctl_gem_submit()
774 if (IS_ERR(submit)) in msm_ioctl_gem_submit()
775 return PTR_ERR(submit); in msm_ioctl_gem_submit()
777 trace_msm_gpu_submit(pid_nr(submit->pid), ring->id, submit->ident, in msm_ioctl_gem_submit()
785 submit->in_rb = true; in msm_ioctl_gem_submit()
797 ret = drm_sched_job_add_dependency(&submit->base, in_fence); in msm_ioctl_gem_submit()
803 syncobjs_to_reset = msm_parse_deps(submit, file, in msm_ioctl_gem_submit()
824 ret = submit_lookup_objects(submit, args, file); in msm_ioctl_gem_submit()
828 ret = submit_lookup_cmds(submit, args, file); in msm_ioctl_gem_submit()
833 ww_acquire_init(&submit->ticket, &reservation_ww_class); in msm_ioctl_gem_submit()
835 ret = submit_lock_objects(submit); in msm_ioctl_gem_submit()
839 ret = submit_fence_sync(submit, !!(args->flags & MSM_SUBMIT_NO_IMPLICIT)); in msm_ioctl_gem_submit()
843 ret = submit_pin_objects(submit); in msm_ioctl_gem_submit()
851 ret = submit_bo(submit, submit->cmd[i].idx, in msm_ioctl_gem_submit()
856 if (!submit->cmd[i].size || in msm_ioctl_gem_submit()
857 ((submit->cmd[i].size + submit->cmd[i].offset) > in msm_ioctl_gem_submit()
859 DRM_ERROR("invalid cmdstream size: %u\n", submit->cmd[i].size * 4); in msm_ioctl_gem_submit()
864 submit->cmd[i].iova = iova + (submit->cmd[i].offset * 4); in msm_ioctl_gem_submit()
866 if (submit->valid) in msm_ioctl_gem_submit()
869 ret = submit_reloc(submit, msm_obj, submit->cmd[i].offset * 4, in msm_ioctl_gem_submit()
870 submit->cmd[i].nr_relocs, submit->cmd[i].relocs); in msm_ioctl_gem_submit()
875 submit->nr_cmds = i; in msm_ioctl_gem_submit()
892 drm_sched_job_arm(&submit->base); in msm_ioctl_gem_submit()
894 submit->user_fence = dma_fence_get(&submit->base.s_fence->finished); in msm_ioctl_gem_submit()
902 submit->fence_id = args->fence; in msm_ioctl_gem_submit()
903 ret = idr_alloc_u32(&queue->fence_idr, submit->user_fence, in msm_ioctl_gem_submit()
904 &submit->fence_id, submit->fence_id, in msm_ioctl_gem_submit()
916 submit->fence_id = idr_alloc_cyclic(&queue->fence_idr, in msm_ioctl_gem_submit()
917 submit->user_fence, 1, in msm_ioctl_gem_submit()
923 if (submit->fence_id < 0) { in msm_ioctl_gem_submit()
924 ret = submit->fence_id; in msm_ioctl_gem_submit()
925 submit->fence_id = 0; in msm_ioctl_gem_submit()
929 struct sync_file *sync_file = sync_file_create(submit->user_fence); in msm_ioctl_gem_submit()
938 submit_attach_object_fences(submit); in msm_ioctl_gem_submit()
941 msm_gem_submit_get(submit); in msm_ioctl_gem_submit()
943 drm_sched_entity_push_job(&submit->base); in msm_ioctl_gem_submit()
945 args->fence = submit->fence_id; in msm_ioctl_gem_submit()
946 queue->last_fence = submit->fence_id; in msm_ioctl_gem_submit()
950 submit->user_fence); in msm_ioctl_gem_submit()
954 submit_cleanup(submit, !!ret); in msm_ioctl_gem_submit()
956 ww_acquire_fini(&submit->ticket); in msm_ioctl_gem_submit()
962 msm_gem_submit_put(submit); in msm_ioctl_gem_submit()