Lines Matching refs:submit
35 struct msm_gem_submit *submit; in submit_create() local
39 sz = struct_size(submit, bos, nr_bos) + in submit_create()
40 ((u64)nr_cmds * sizeof(submit->cmd[0])); in submit_create()
45 submit = kzalloc(sz, GFP_KERNEL | __GFP_NOWARN | __GFP_NORETRY); in submit_create()
46 if (!submit) in submit_create()
49 ret = drm_sched_job_init(&submit->base, queue->entity, queue); in submit_create()
51 kfree(submit); in submit_create()
55 kref_init(&submit->ref); in submit_create()
56 submit->dev = dev; in submit_create()
57 submit->aspace = queue->ctx->aspace; in submit_create()
58 submit->gpu = gpu; in submit_create()
59 submit->cmd = (void *)&submit->bos[nr_bos]; in submit_create()
60 submit->queue = queue; in submit_create()
61 submit->ring = gpu->rb[queue->ring_nr]; in submit_create()
62 submit->fault_dumped = false; in submit_create()
64 INIT_LIST_HEAD(&submit->node); in submit_create()
66 return submit; in submit_create()
71 struct msm_gem_submit *submit = in __msm_gem_submit_destroy() local
75 if (submit->fence_id) { in __msm_gem_submit_destroy()
76 mutex_lock(&submit->queue->lock); in __msm_gem_submit_destroy()
77 idr_remove(&submit->queue->fence_idr, submit->fence_id); in __msm_gem_submit_destroy()
78 mutex_unlock(&submit->queue->lock); in __msm_gem_submit_destroy()
81 dma_fence_put(submit->user_fence); in __msm_gem_submit_destroy()
82 dma_fence_put(submit->hw_fence); in __msm_gem_submit_destroy()
84 put_pid(submit->pid); in __msm_gem_submit_destroy()
85 msm_submitqueue_put(submit->queue); in __msm_gem_submit_destroy()
87 for (i = 0; i < submit->nr_cmds; i++) in __msm_gem_submit_destroy()
88 kfree(submit->cmd[i].relocs); in __msm_gem_submit_destroy()
90 kfree(submit); in __msm_gem_submit_destroy()
93 static int submit_lookup_objects(struct msm_gem_submit *submit, in submit_lookup_objects() argument
107 submit->bos[i].flags = 0; in submit_lookup_objects()
126 submit->bos[i].handle = submit_bo.handle; in submit_lookup_objects()
127 submit->bos[i].flags = submit_bo.flags; in submit_lookup_objects()
129 submit->bos[i].iova = submit_bo.presumed; in submit_lookup_objects()
140 obj = idr_find(&file->object_idr, submit->bos[i].handle); in submit_lookup_objects()
142 DRM_ERROR("invalid handle %u at index %u\n", submit->bos[i].handle, i); in submit_lookup_objects()
149 submit->bos[i].obj = to_msm_bo(obj); in submit_lookup_objects()
156 submit->nr_bos = i; in submit_lookup_objects()
161 static int submit_lookup_cmds(struct msm_gem_submit *submit, in submit_lookup_cmds() argument
197 submit->cmd[i].type = submit_cmd.type; in submit_lookup_cmds()
198 submit->cmd[i].size = submit_cmd.size / 4; in submit_lookup_cmds()
199 submit->cmd[i].offset = submit_cmd.submit_offset / 4; in submit_lookup_cmds()
200 submit->cmd[i].idx = submit_cmd.submit_idx; in submit_lookup_cmds()
201 submit->cmd[i].nr_relocs = submit_cmd.nr_relocs; in submit_lookup_cmds()
212 submit->cmd[i].relocs = kmalloc(sz, GFP_KERNEL); in submit_lookup_cmds()
213 ret = copy_from_user(submit->cmd[i].relocs, userptr, sz); in submit_lookup_cmds()
228 static void submit_cleanup_bo(struct msm_gem_submit *submit, int i, in submit_cleanup_bo() argument
231 struct drm_gem_object *obj = &submit->bos[i].obj->base; in submit_cleanup_bo()
232 unsigned flags = submit->bos[i].flags & cleanup_flags; in submit_cleanup_bo()
235 msm_gem_unpin_iova_locked(obj, submit->aspace); in submit_cleanup_bo()
243 submit->bos[i].flags &= ~cleanup_flags; in submit_cleanup_bo()
246 static void submit_unlock_unpin_bo(struct msm_gem_submit *submit, int i) in submit_unlock_unpin_bo() argument
248 submit_cleanup_bo(submit, i, BO_PINNED | BO_ACTIVE | BO_LOCKED); in submit_unlock_unpin_bo()
250 if (!(submit->bos[i].flags & BO_VALID)) in submit_unlock_unpin_bo()
251 submit->bos[i].iova = 0; in submit_unlock_unpin_bo()
255 static int submit_lock_objects(struct msm_gem_submit *submit) in submit_lock_objects() argument
260 for (i = 0; i < submit->nr_bos; i++) { in submit_lock_objects()
261 struct msm_gem_object *msm_obj = submit->bos[i].obj; in submit_lock_objects()
268 if (!(submit->bos[i].flags & BO_LOCKED)) { in submit_lock_objects()
270 &submit->ticket); in submit_lock_objects()
273 submit->bos[i].flags |= BO_LOCKED; in submit_lock_objects()
277 ww_acquire_done(&submit->ticket); in submit_lock_objects()
284 submit->bos[i].handle, i); in submit_lock_objects()
289 submit_unlock_unpin_bo(submit, i); in submit_lock_objects()
292 submit_unlock_unpin_bo(submit, slow_locked); in submit_lock_objects()
295 struct msm_gem_object *msm_obj = submit->bos[contended].obj; in submit_lock_objects()
298 &submit->ticket); in submit_lock_objects()
300 submit->bos[contended].flags |= BO_LOCKED; in submit_lock_objects()
315 static int submit_fence_sync(struct msm_gem_submit *submit, bool no_implicit) in submit_fence_sync() argument
319 for (i = 0; i < submit->nr_bos; i++) { in submit_fence_sync()
320 struct drm_gem_object *obj = &submit->bos[i].obj->base; in submit_fence_sync()
321 bool write = submit->bos[i].flags & MSM_SUBMIT_BO_WRITE; in submit_fence_sync()
338 ret = drm_sched_job_add_implicit_dependencies(&submit->base, in submit_fence_sync()
348 static int submit_pin_objects(struct msm_gem_submit *submit) in submit_pin_objects() argument
352 submit->valid = true; in submit_pin_objects()
359 for (i = 0; i < submit->nr_bos; i++) { in submit_pin_objects()
360 struct drm_gem_object *obj = &submit->bos[i].obj->base; in submit_pin_objects()
362 msm_gem_active_get(obj, submit->gpu); in submit_pin_objects()
363 submit->bos[i].flags |= BO_ACTIVE; in submit_pin_objects()
366 for (i = 0; i < submit->nr_bos; i++) { in submit_pin_objects()
367 struct drm_gem_object *obj = &submit->bos[i].obj->base; in submit_pin_objects()
372 submit->aspace, &iova); in submit_pin_objects()
377 submit->bos[i].flags |= BO_PINNED; in submit_pin_objects()
379 if (iova == submit->bos[i].iova) { in submit_pin_objects()
380 submit->bos[i].flags |= BO_VALID; in submit_pin_objects()
382 submit->bos[i].iova = iova; in submit_pin_objects()
384 submit->bos[i].flags &= ~BO_VALID; in submit_pin_objects()
385 submit->valid = false; in submit_pin_objects()
392 static void submit_attach_object_fences(struct msm_gem_submit *submit) in submit_attach_object_fences() argument
396 for (i = 0; i < submit->nr_bos; i++) { in submit_attach_object_fences()
397 struct drm_gem_object *obj = &submit->bos[i].obj->base; in submit_attach_object_fences()
399 if (submit->bos[i].flags & MSM_SUBMIT_BO_WRITE) in submit_attach_object_fences()
400 dma_resv_add_excl_fence(obj->resv, submit->user_fence); in submit_attach_object_fences()
401 else if (submit->bos[i].flags & MSM_SUBMIT_BO_READ) in submit_attach_object_fences()
402 dma_resv_add_shared_fence(obj->resv, submit->user_fence); in submit_attach_object_fences()
406 static int submit_bo(struct msm_gem_submit *submit, uint32_t idx, in submit_bo() argument
409 if (idx >= submit->nr_bos) { in submit_bo()
411 idx, submit->nr_bos); in submit_bo()
416 *obj = submit->bos[idx].obj; in submit_bo()
418 *iova = submit->bos[idx].iova; in submit_bo()
420 *valid = !!(submit->bos[idx].flags & BO_VALID); in submit_bo()
426 static int submit_reloc(struct msm_gem_submit *submit, struct msm_gem_object *obj, in submit_reloc() argument
475 ret = submit_bo(submit, submit_reloc.reloc_idx, NULL, &iova, &valid); in submit_reloc()
504 static void submit_cleanup(struct msm_gem_submit *submit, bool error) in submit_cleanup() argument
512 for (i = 0; i < submit->nr_bos; i++) { in submit_cleanup()
513 struct msm_gem_object *msm_obj = submit->bos[i].obj; in submit_cleanup()
514 submit_cleanup_bo(submit, i, cleanup_flags); in submit_cleanup()
520 void msm_submit_retire(struct msm_gem_submit *submit) in msm_submit_retire() argument
524 for (i = 0; i < submit->nr_bos; i++) { in msm_submit_retire()
525 struct drm_gem_object *obj = &submit->bos[i].obj->base; in msm_submit_retire()
528 submit_cleanup_bo(submit, i, BO_PINNED | BO_ACTIVE); in msm_submit_retire()
540 static struct drm_syncobj **msm_parse_deps(struct msm_gem_submit *submit, in msm_parse_deps() argument
569 !drm_core_check_feature(submit->dev, DRIVER_SYNCOBJ_TIMELINE)) { in msm_parse_deps()
584 ret = drm_sched_job_add_dependency(&submit->base, fence); in msm_parse_deps()
715 struct msm_gem_submit *submit = NULL; in msm_ioctl_gem_submit() local
771 submit = submit_create(dev, gpu, queue, args->nr_bos, in msm_ioctl_gem_submit()
773 if (IS_ERR(submit)) { in msm_ioctl_gem_submit()
774 ret = PTR_ERR(submit); in msm_ioctl_gem_submit()
775 submit = NULL; in msm_ioctl_gem_submit()
779 submit->pid = pid; in msm_ioctl_gem_submit()
780 submit->ident = submitid; in msm_ioctl_gem_submit()
783 submit->in_rb = true; in msm_ioctl_gem_submit()
795 ret = drm_sched_job_add_dependency(&submit->base, in_fence); in msm_ioctl_gem_submit()
801 syncobjs_to_reset = msm_parse_deps(submit, file, in msm_ioctl_gem_submit()
822 ret = submit_lookup_objects(submit, args, file); in msm_ioctl_gem_submit()
826 ret = submit_lookup_cmds(submit, args, file); in msm_ioctl_gem_submit()
831 ww_acquire_init(&submit->ticket, &reservation_ww_class); in msm_ioctl_gem_submit()
833 ret = submit_lock_objects(submit); in msm_ioctl_gem_submit()
837 ret = submit_fence_sync(submit, !!(args->flags & MSM_SUBMIT_NO_IMPLICIT)); in msm_ioctl_gem_submit()
841 ret = submit_pin_objects(submit); in msm_ioctl_gem_submit()
849 ret = submit_bo(submit, submit->cmd[i].idx, in msm_ioctl_gem_submit()
854 if (!submit->cmd[i].size || in msm_ioctl_gem_submit()
855 ((submit->cmd[i].size + submit->cmd[i].offset) > in msm_ioctl_gem_submit()
857 DRM_ERROR("invalid cmdstream size: %u\n", submit->cmd[i].size * 4); in msm_ioctl_gem_submit()
862 submit->cmd[i].iova = iova + (submit->cmd[i].offset * 4); in msm_ioctl_gem_submit()
864 if (submit->valid) in msm_ioctl_gem_submit()
867 ret = submit_reloc(submit, msm_obj, submit->cmd[i].offset * 4, in msm_ioctl_gem_submit()
868 submit->cmd[i].nr_relocs, submit->cmd[i].relocs); in msm_ioctl_gem_submit()
873 submit->nr_cmds = i; in msm_ioctl_gem_submit()
875 drm_sched_job_arm(&submit->base); in msm_ioctl_gem_submit()
877 submit->user_fence = dma_fence_get(&submit->base.s_fence->finished); in msm_ioctl_gem_submit()
883 submit->fence_id = idr_alloc_cyclic(&queue->fence_idr, in msm_ioctl_gem_submit()
884 submit->user_fence, 0, INT_MAX, GFP_KERNEL); in msm_ioctl_gem_submit()
885 if (submit->fence_id < 0) { in msm_ioctl_gem_submit()
886 ret = submit->fence_id = 0; in msm_ioctl_gem_submit()
887 submit->fence_id = 0; in msm_ioctl_gem_submit()
891 struct sync_file *sync_file = sync_file_create(submit->user_fence); in msm_ioctl_gem_submit()
900 submit_attach_object_fences(submit); in msm_ioctl_gem_submit()
903 msm_gem_submit_get(submit); in msm_ioctl_gem_submit()
905 drm_sched_entity_push_job(&submit->base); in msm_ioctl_gem_submit()
907 args->fence = submit->fence_id; in msm_ioctl_gem_submit()
908 queue->last_fence = submit->fence_id; in msm_ioctl_gem_submit()
912 submit->user_fence); in msm_ioctl_gem_submit()
916 submit_cleanup(submit, !!ret); in msm_ioctl_gem_submit()
918 ww_acquire_fini(&submit->ticket); in msm_ioctl_gem_submit()
923 if (submit) in msm_ioctl_gem_submit()
924 msm_gem_submit_put(submit); in msm_ioctl_gem_submit()