Lines Matching refs:submit
25 #define SUBMIT_ERROR(err, submit, fmt, ...) \ argument
26 UERR(err, (submit)->dev, fmt, ##__VA_ARGS__)
38 struct msm_gem_submit *submit; in submit_create() local
42 sz = struct_size(submit, bos, nr_bos) + in submit_create()
43 ((u64)nr_cmds * sizeof(submit->cmd[0])); in submit_create()
48 submit = kzalloc(sz, GFP_KERNEL | __GFP_NOWARN); in submit_create()
49 if (!submit) in submit_create()
52 submit->hw_fence = msm_fence_alloc(); in submit_create()
53 if (IS_ERR(submit->hw_fence)) { in submit_create()
54 ret = PTR_ERR(submit->hw_fence); in submit_create()
55 kfree(submit); in submit_create()
59 ret = drm_sched_job_init(&submit->base, queue->entity, 1, queue, in submit_create()
62 kfree(submit->hw_fence); in submit_create()
63 kfree(submit); in submit_create()
67 kref_init(&submit->ref); in submit_create()
68 submit->dev = dev; in submit_create()
69 submit->vm = msm_context_vm(dev, queue->ctx); in submit_create()
70 submit->gpu = gpu; in submit_create()
71 submit->cmd = (void *)&submit->bos[nr_bos]; in submit_create()
72 submit->queue = queue; in submit_create()
73 submit->pid = get_pid(task_pid(current)); in submit_create()
74 submit->ring = gpu->rb[queue->ring_nr]; in submit_create()
75 submit->fault_dumped = false; in submit_create()
78 submit->ident = atomic_inc_return(&ident) - 1; in submit_create()
80 INIT_LIST_HEAD(&submit->node); in submit_create()
82 return submit; in submit_create()
87 struct msm_gem_submit *submit = in __msm_gem_submit_destroy() local
97 if (submit->base.s_fence) in __msm_gem_submit_destroy()
98 drm_sched_job_cleanup(&submit->base); in __msm_gem_submit_destroy()
100 if (submit->fence_id) { in __msm_gem_submit_destroy()
101 spin_lock(&submit->queue->idr_lock); in __msm_gem_submit_destroy()
102 idr_remove(&submit->queue->fence_idr, submit->fence_id); in __msm_gem_submit_destroy()
103 spin_unlock(&submit->queue->idr_lock); in __msm_gem_submit_destroy()
106 dma_fence_put(submit->user_fence); in __msm_gem_submit_destroy()
115 if (kref_read(&submit->hw_fence->refcount) == 0) { in __msm_gem_submit_destroy()
116 kfree(submit->hw_fence); in __msm_gem_submit_destroy()
118 dma_fence_put(submit->hw_fence); in __msm_gem_submit_destroy()
121 put_pid(submit->pid); in __msm_gem_submit_destroy()
122 msm_submitqueue_put(submit->queue); in __msm_gem_submit_destroy()
124 for (i = 0; i < submit->nr_cmds; i++) in __msm_gem_submit_destroy()
125 kfree(submit->cmd[i].relocs); in __msm_gem_submit_destroy()
127 kfree(submit); in __msm_gem_submit_destroy()
130 static int submit_lookup_objects(struct msm_gem_submit *submit, in submit_lookup_objects() argument
144 submit->bos[i].flags = 0; in submit_lookup_objects()
157 ret = SUBMIT_ERROR(EINVAL, submit, "invalid flags: %x\n", submit_bo.flags); in submit_lookup_objects()
162 submit->bos[i].handle = submit_bo.handle; in submit_lookup_objects()
163 submit->bos[i].flags = submit_bo.flags; in submit_lookup_objects()
174 obj = idr_find(&file->object_idr, submit->bos[i].handle); in submit_lookup_objects()
176 ret = SUBMIT_ERROR(EINVAL, submit, "invalid handle %u at index %u\n", submit->bos[i].handle, i); in submit_lookup_objects()
182 submit->bos[i].obj = obj; in submit_lookup_objects()
189 submit->nr_bos = i; in submit_lookup_objects()
194 static int submit_lookup_cmds(struct msm_gem_submit *submit, in submit_lookup_cmds() argument
220 return SUBMIT_ERROR(EINVAL, submit, "invalid type: %08x\n", submit_cmd.type); in submit_lookup_cmds()
224 ret = SUBMIT_ERROR(EINVAL, submit, "non-aligned cmdstream buffer size: %u\n", in submit_lookup_cmds()
231 ret = SUBMIT_ERROR(EINVAL, submit, "nr_relocs must be zero"); in submit_lookup_cmds()
236 ret = SUBMIT_ERROR(EINVAL, submit, "submit_idx/offset must be zero"); in submit_lookup_cmds()
240 submit->cmd[i].iova = submit_cmd.iova; in submit_lookup_cmds()
243 submit->cmd[i].type = submit_cmd.type; in submit_lookup_cmds()
244 submit->cmd[i].size = submit_cmd.size / 4; in submit_lookup_cmds()
245 submit->cmd[i].offset = submit_cmd.submit_offset / 4; in submit_lookup_cmds()
246 submit->cmd[i].idx = submit_cmd.submit_idx; in submit_lookup_cmds()
247 submit->cmd[i].nr_relocs = submit_cmd.nr_relocs; in submit_lookup_cmds()
258 submit->cmd[i].relocs = kmalloc(sz, GFP_KERNEL | __GFP_NOWARN); in submit_lookup_cmds()
259 if (!submit->cmd[i].relocs) { in submit_lookup_cmds()
263 ret = copy_from_user(submit->cmd[i].relocs, userptr, sz); in submit_lookup_cmds()
275 static int submit_lock_objects(struct msm_gem_submit *submit) in submit_lock_objects() argument
278 struct drm_exec *exec = &submit->exec; in submit_lock_objects()
281 if (msm_context_is_vmbind(submit->queue->ctx)) { in submit_lock_objects()
284 drm_exec_init(&submit->exec, flags, submit->nr_bos); in submit_lock_objects()
286 drm_exec_until_all_locked (&submit->exec) { in submit_lock_objects()
287 ret = drm_gpuvm_prepare_vm(submit->vm, exec, 1); in submit_lock_objects()
292 ret = drm_gpuvm_prepare_objects(submit->vm, exec, 1); in submit_lock_objects()
301 drm_exec_init(&submit->exec, flags, submit->nr_bos); in submit_lock_objects()
303 drm_exec_until_all_locked (&submit->exec) { in submit_lock_objects()
304 ret = drm_exec_lock_obj(&submit->exec, in submit_lock_objects()
305 drm_gpuvm_resv_obj(submit->vm)); in submit_lock_objects()
306 drm_exec_retry_on_contention(&submit->exec); in submit_lock_objects()
309 for (unsigned i = 0; i < submit->nr_bos; i++) { in submit_lock_objects()
310 struct drm_gem_object *obj = submit->bos[i].obj; in submit_lock_objects()
311 ret = drm_exec_prepare_obj(&submit->exec, obj, 1); in submit_lock_objects()
312 drm_exec_retry_on_contention(&submit->exec); in submit_lock_objects()
321 static int submit_fence_sync(struct msm_gem_submit *submit) in submit_fence_sync() argument
325 for (i = 0; i < submit->nr_bos; i++) { in submit_fence_sync()
326 struct drm_gem_object *obj = submit->bos[i].obj; in submit_fence_sync()
327 bool write = submit->bos[i].flags & MSM_SUBMIT_BO_WRITE; in submit_fence_sync()
333 if (submit->bos[i].flags & MSM_SUBMIT_BO_NO_IMPLICIT) in submit_fence_sync()
336 ret = drm_sched_job_add_implicit_dependencies(&submit->base, in submit_fence_sync()
346 static int submit_pin_objects(struct msm_gem_submit *submit) in submit_pin_objects() argument
348 struct msm_drm_private *priv = submit->dev->dev_private; in submit_pin_objects()
351 for (i = 0; i < submit->nr_bos; i++) { in submit_pin_objects()
352 struct drm_gem_object *obj = submit->bos[i].obj; in submit_pin_objects()
356 vma = msm_gem_get_vma_locked(obj, submit->vm); in submit_pin_objects()
366 submit->bos[i].vm_bo = drm_gpuvm_bo_get(vma->vm_bo); in submit_pin_objects()
367 submit->bos[i].iova = vma->va.addr; in submit_pin_objects()
378 for (i = 0; i < submit->nr_bos; i++) { in submit_pin_objects()
379 msm_gem_pin_obj_locked(submit->bos[i].obj); in submit_pin_objects()
383 submit->bos_pinned = true; in submit_pin_objects()
388 static void submit_unpin_objects(struct msm_gem_submit *submit) in submit_unpin_objects() argument
390 if (!submit->bos_pinned) in submit_unpin_objects()
393 for (int i = 0; i < submit->nr_bos; i++) { in submit_unpin_objects()
394 struct drm_gem_object *obj = submit->bos[i].obj; in submit_unpin_objects()
399 submit->bos_pinned = false; in submit_unpin_objects()
402 static void submit_attach_object_fences(struct msm_gem_submit *submit) in submit_attach_object_fences() argument
404 struct msm_gem_vm *vm = to_msm_vm(submit->vm); in submit_attach_object_fences()
407 if (msm_context_is_vmbind(submit->queue->ctx)) { in submit_attach_object_fences()
408 drm_gpuvm_resv_add_fence(submit->vm, &submit->exec, in submit_attach_object_fences()
409 submit->user_fence, in submit_attach_object_fences()
415 for (unsigned i = 0; i < submit->nr_bos; i++) { in submit_attach_object_fences()
416 struct drm_gem_object *obj = submit->bos[i].obj; in submit_attach_object_fences()
418 if (submit->bos[i].flags & MSM_SUBMIT_BO_WRITE) in submit_attach_object_fences()
419 dma_resv_add_fence(obj->resv, submit->user_fence, in submit_attach_object_fences()
421 else if (submit->bos[i].flags & MSM_SUBMIT_BO_READ) in submit_attach_object_fences()
422 dma_resv_add_fence(obj->resv, submit->user_fence, in submit_attach_object_fences()
427 vm->last_fence = dma_fence_unwrap_merge(submit->user_fence, last_fence); in submit_attach_object_fences()
431 static int submit_bo(struct msm_gem_submit *submit, uint32_t idx, in submit_bo() argument
434 if (idx >= submit->nr_bos) { in submit_bo()
435 return SUBMIT_ERROR(EINVAL, submit, "invalid buffer index: %u (out of %u)\n", in submit_bo()
436 idx, submit->nr_bos); in submit_bo()
440 *obj = submit->bos[idx].obj; in submit_bo()
442 *iova = submit->bos[idx].iova; in submit_bo()
448 static int submit_reloc(struct msm_gem_submit *submit, struct drm_gem_object *obj, in submit_reloc() argument
456 return SUBMIT_ERROR(EINVAL, submit, "non-aligned cmdstream buffer: %u\n", offset); in submit_reloc()
475 ret = SUBMIT_ERROR(EINVAL, submit, "non-aligned reloc offset: %u\n", in submit_reloc()
485 ret = SUBMIT_ERROR(EINVAL, submit, "invalid offset %u at reloc %u\n", off, i); in submit_reloc()
489 ret = submit_bo(submit, submit_reloc.reloc_idx, NULL, &iova); in submit_reloc()
515 static void submit_cleanup(struct msm_gem_submit *submit, bool error) in submit_cleanup() argument
517 if (submit->exec.objects) in submit_cleanup()
518 drm_exec_fini(&submit->exec); in submit_cleanup()
521 submit_unpin_objects(submit); in submit_cleanup()
523 msm_submit_retire(submit); in submit_cleanup()
527 void msm_submit_retire(struct msm_gem_submit *submit) in msm_submit_retire() argument
531 for (i = 0; i < submit->nr_bos; i++) { in msm_submit_retire()
532 struct drm_gem_object *obj = submit->bos[i].obj; in msm_submit_retire()
533 struct drm_gpuvm_bo *vm_bo = submit->bos[i].vm_bo; in msm_submit_retire()
548 struct msm_gem_submit *submit = NULL; in msm_ioctl_gem_submit() local
603 submit = submit_create(dev, gpu, queue, args->nr_bos, args->nr_cmds, in msm_ioctl_gem_submit()
605 if (IS_ERR(submit)) { in msm_ioctl_gem_submit()
606 ret = PTR_ERR(submit); in msm_ioctl_gem_submit()
610 trace_msm_gpu_submit(pid_nr(submit->pid), ring->id, submit->ident, in msm_ioctl_gem_submit()
618 submit->in_rb = true; in msm_ioctl_gem_submit()
630 ret = drm_sched_job_add_dependency(&submit->base, in_fence); in msm_ioctl_gem_submit()
636 syncobjs_to_reset = msm_syncobj_parse_deps(dev, &submit->base, in msm_ioctl_gem_submit()
657 ret = submit_lookup_objects(submit, args, file); in msm_ioctl_gem_submit()
661 ret = submit_lookup_cmds(submit, args, file); in msm_ioctl_gem_submit()
666 ret = submit_lock_objects(submit); in msm_ioctl_gem_submit()
671 ret = submit_fence_sync(submit); in msm_ioctl_gem_submit()
676 ret = submit_pin_objects(submit); in msm_ioctl_gem_submit()
686 ret = submit_bo(submit, submit->cmd[i].idx, &obj, &iova); in msm_ioctl_gem_submit()
690 if (!submit->cmd[i].size || in msm_ioctl_gem_submit()
691 (size_add(submit->cmd[i].size, submit->cmd[i].offset) > obj->size / 4)) { in msm_ioctl_gem_submit()
693 submit->cmd[i].size * 4); in msm_ioctl_gem_submit()
697 submit->cmd[i].iova = iova + (submit->cmd[i].offset * 4); in msm_ioctl_gem_submit()
699 if (likely(!submit->cmd[i].nr_relocs)) in msm_ioctl_gem_submit()
707 ret = submit_reloc(submit, obj, submit->cmd[i].offset * 4, in msm_ioctl_gem_submit()
708 submit->cmd[i].nr_relocs, submit->cmd[i].relocs); in msm_ioctl_gem_submit()
713 submit->nr_cmds = args->nr_cmds; in msm_ioctl_gem_submit()
733 drm_sched_job_arm(&submit->base); in msm_ioctl_gem_submit()
735 submit->user_fence = dma_fence_get(&submit->base.s_fence->finished); in msm_ioctl_gem_submit()
743 submit->fence_id = args->fence; in msm_ioctl_gem_submit()
744 ret = idr_alloc_u32(&queue->fence_idr, submit->user_fence, in msm_ioctl_gem_submit()
745 &submit->fence_id, submit->fence_id, in msm_ioctl_gem_submit()
757 submit->fence_id = idr_alloc_cyclic(&queue->fence_idr, in msm_ioctl_gem_submit()
758 submit->user_fence, 1, in msm_ioctl_gem_submit()
765 if (submit->fence_id < 0) { in msm_ioctl_gem_submit()
766 ret = submit->fence_id; in msm_ioctl_gem_submit()
767 submit->fence_id = 0; in msm_ioctl_gem_submit()
771 sync_file = sync_file_create(submit->user_fence); in msm_ioctl_gem_submit()
783 submit_attach_object_fences(submit); in msm_ioctl_gem_submit()
792 ret = drm_gpuvm_validate(submit->vm, &submit->exec); in msm_ioctl_gem_submit()
798 msm_gem_submit_get(submit); in msm_ioctl_gem_submit()
800 msm_rd_dump_submit(priv->rd, submit, NULL); in msm_ioctl_gem_submit()
802 drm_sched_entity_push_job(&submit->base); in msm_ioctl_gem_submit()
804 args->fence = submit->fence_id; in msm_ioctl_gem_submit()
805 queue->last_fence = submit->fence_id; in msm_ioctl_gem_submit()
808 msm_syncobj_process_post_deps(post_deps, args->nr_out_syncobjs, submit->user_fence); in msm_ioctl_gem_submit()
811 submit_cleanup(submit, !!ret); in msm_ioctl_gem_submit()
821 if (!IS_ERR_OR_NULL(submit)) { in msm_ioctl_gem_submit()
822 msm_gem_submit_put(submit); in msm_ioctl_gem_submit()