Lines Matching refs:submit
23 #define SUBMIT_ERROR(submit, fmt, ...) \ argument
24 DRM_DEV_DEBUG_DRIVER((submit)->dev->dev, fmt, ##__VA_ARGS__)
36 struct msm_gem_submit *submit; in submit_create() local
40 sz = struct_size(submit, bos, nr_bos) + in submit_create()
41 ((u64)nr_cmds * sizeof(submit->cmd[0])); in submit_create()
46 submit = kzalloc(sz, GFP_KERNEL | __GFP_NOWARN); in submit_create()
47 if (!submit) in submit_create()
50 submit->hw_fence = msm_fence_alloc(); in submit_create()
51 if (IS_ERR(submit->hw_fence)) { in submit_create()
52 ret = PTR_ERR(submit->hw_fence); in submit_create()
53 kfree(submit); in submit_create()
57 ret = drm_sched_job_init(&submit->base, queue->entity, 1, queue); in submit_create()
59 kfree(submit->hw_fence); in submit_create()
60 kfree(submit); in submit_create()
64 kref_init(&submit->ref); in submit_create()
65 submit->dev = dev; in submit_create()
66 submit->aspace = queue->ctx->aspace; in submit_create()
67 submit->gpu = gpu; in submit_create()
68 submit->cmd = (void *)&submit->bos[nr_bos]; in submit_create()
69 submit->queue = queue; in submit_create()
70 submit->pid = get_pid(task_pid(current)); in submit_create()
71 submit->ring = gpu->rb[queue->ring_nr]; in submit_create()
72 submit->fault_dumped = false; in submit_create()
75 submit->ident = atomic_inc_return(&ident) - 1; in submit_create()
77 INIT_LIST_HEAD(&submit->node); in submit_create()
79 return submit; in submit_create()
84 struct msm_gem_submit *submit = in __msm_gem_submit_destroy() local
88 if (submit->fence_id) { in __msm_gem_submit_destroy()
89 spin_lock(&submit->queue->idr_lock); in __msm_gem_submit_destroy()
90 idr_remove(&submit->queue->fence_idr, submit->fence_id); in __msm_gem_submit_destroy()
91 spin_unlock(&submit->queue->idr_lock); in __msm_gem_submit_destroy()
94 dma_fence_put(submit->user_fence); in __msm_gem_submit_destroy()
103 if (kref_read(&submit->hw_fence->refcount) == 0) { in __msm_gem_submit_destroy()
104 kfree(submit->hw_fence); in __msm_gem_submit_destroy()
106 dma_fence_put(submit->hw_fence); in __msm_gem_submit_destroy()
109 put_pid(submit->pid); in __msm_gem_submit_destroy()
110 msm_submitqueue_put(submit->queue); in __msm_gem_submit_destroy()
112 for (i = 0; i < submit->nr_cmds; i++) in __msm_gem_submit_destroy()
113 kfree(submit->cmd[i].relocs); in __msm_gem_submit_destroy()
115 kfree(submit); in __msm_gem_submit_destroy()
118 static int submit_lookup_objects(struct msm_gem_submit *submit, in submit_lookup_objects() argument
132 submit->bos[i].flags = 0; in submit_lookup_objects()
145 SUBMIT_ERROR(submit, "invalid flags: %x\n", submit_bo.flags); in submit_lookup_objects()
151 submit->bos[i].handle = submit_bo.handle; in submit_lookup_objects()
152 submit->bos[i].flags = submit_bo.flags; in submit_lookup_objects()
163 obj = idr_find(&file->object_idr, submit->bos[i].handle); in submit_lookup_objects()
165 SUBMIT_ERROR(submit, "invalid handle %u at index %u\n", submit->bos[i].handle, i); in submit_lookup_objects()
172 submit->bos[i].obj = obj; in submit_lookup_objects()
179 submit->nr_bos = i; in submit_lookup_objects()
184 static int submit_lookup_cmds(struct msm_gem_submit *submit, in submit_lookup_cmds() argument
209 SUBMIT_ERROR(submit, "invalid type: %08x\n", submit_cmd.type); in submit_lookup_cmds()
214 SUBMIT_ERROR(submit, "non-aligned cmdstream buffer size: %u\n", in submit_lookup_cmds()
220 submit->cmd[i].type = submit_cmd.type; in submit_lookup_cmds()
221 submit->cmd[i].size = submit_cmd.size / 4; in submit_lookup_cmds()
222 submit->cmd[i].offset = submit_cmd.submit_offset / 4; in submit_lookup_cmds()
223 submit->cmd[i].idx = submit_cmd.submit_idx; in submit_lookup_cmds()
224 submit->cmd[i].nr_relocs = submit_cmd.nr_relocs; in submit_lookup_cmds()
235 submit->cmd[i].relocs = kmalloc(sz, GFP_KERNEL | __GFP_NOWARN); in submit_lookup_cmds()
236 if (!submit->cmd[i].relocs) { in submit_lookup_cmds()
240 ret = copy_from_user(submit->cmd[i].relocs, userptr, sz); in submit_lookup_cmds()
252 static int submit_lock_objects(struct msm_gem_submit *submit) in submit_lock_objects() argument
256 drm_exec_init(&submit->exec, DRM_EXEC_INTERRUPTIBLE_WAIT, submit->nr_bos); in submit_lock_objects()
258 drm_exec_until_all_locked (&submit->exec) { in submit_lock_objects()
259 for (unsigned i = 0; i < submit->nr_bos; i++) { in submit_lock_objects()
260 struct drm_gem_object *obj = submit->bos[i].obj; in submit_lock_objects()
261 ret = drm_exec_prepare_obj(&submit->exec, obj, 1); in submit_lock_objects()
262 drm_exec_retry_on_contention(&submit->exec); in submit_lock_objects()
274 static int submit_fence_sync(struct msm_gem_submit *submit) in submit_fence_sync() argument
278 for (i = 0; i < submit->nr_bos; i++) { in submit_fence_sync()
279 struct drm_gem_object *obj = submit->bos[i].obj; in submit_fence_sync()
280 bool write = submit->bos[i].flags & MSM_SUBMIT_BO_WRITE; in submit_fence_sync()
286 if (submit->bos[i].flags & MSM_SUBMIT_BO_NO_IMPLICIT) in submit_fence_sync()
289 ret = drm_sched_job_add_implicit_dependencies(&submit->base, in submit_fence_sync()
299 static int submit_pin_objects(struct msm_gem_submit *submit) in submit_pin_objects() argument
301 struct msm_drm_private *priv = submit->dev->dev_private; in submit_pin_objects()
304 for (i = 0; i < submit->nr_bos; i++) { in submit_pin_objects()
305 struct drm_gem_object *obj = submit->bos[i].obj; in submit_pin_objects()
309 vma = msm_gem_get_vma_locked(obj, submit->aspace); in submit_pin_objects()
319 submit->bos[i].iova = vma->iova; in submit_pin_objects()
330 for (i = 0; i < submit->nr_bos; i++) { in submit_pin_objects()
331 msm_gem_pin_obj_locked(submit->bos[i].obj); in submit_pin_objects()
335 submit->bos_pinned = true; in submit_pin_objects()
340 static void submit_unpin_objects(struct msm_gem_submit *submit) in submit_unpin_objects() argument
342 if (!submit->bos_pinned) in submit_unpin_objects()
345 for (int i = 0; i < submit->nr_bos; i++) { in submit_unpin_objects()
346 struct drm_gem_object *obj = submit->bos[i].obj; in submit_unpin_objects()
351 submit->bos_pinned = false; in submit_unpin_objects()
354 static void submit_attach_object_fences(struct msm_gem_submit *submit) in submit_attach_object_fences() argument
358 for (i = 0; i < submit->nr_bos; i++) { in submit_attach_object_fences()
359 struct drm_gem_object *obj = submit->bos[i].obj; in submit_attach_object_fences()
361 if (submit->bos[i].flags & MSM_SUBMIT_BO_WRITE) in submit_attach_object_fences()
362 dma_resv_add_fence(obj->resv, submit->user_fence, in submit_attach_object_fences()
364 else if (submit->bos[i].flags & MSM_SUBMIT_BO_READ) in submit_attach_object_fences()
365 dma_resv_add_fence(obj->resv, submit->user_fence, in submit_attach_object_fences()
370 static int submit_bo(struct msm_gem_submit *submit, uint32_t idx, in submit_bo() argument
373 if (idx >= submit->nr_bos) { in submit_bo()
374 SUBMIT_ERROR(submit, "invalid buffer index: %u (out of %u)\n", in submit_bo()
375 idx, submit->nr_bos); in submit_bo()
380 *obj = submit->bos[idx].obj; in submit_bo()
382 *iova = submit->bos[idx].iova; in submit_bo()
388 static int submit_reloc(struct msm_gem_submit *submit, struct drm_gem_object *obj, in submit_reloc() argument
396 SUBMIT_ERROR(submit, "non-aligned cmdstream buffer: %u\n", offset); in submit_reloc()
417 SUBMIT_ERROR(submit, "non-aligned reloc offset: %u\n", in submit_reloc()
428 SUBMIT_ERROR(submit, "invalid offset %u at reloc %u\n", off, i); in submit_reloc()
433 ret = submit_bo(submit, submit_reloc.reloc_idx, NULL, &iova); in submit_reloc()
459 static void submit_cleanup(struct msm_gem_submit *submit, bool error) in submit_cleanup() argument
462 submit_unpin_objects(submit); in submit_cleanup()
464 msm_submit_retire(submit); in submit_cleanup()
467 if (submit->exec.objects) in submit_cleanup()
468 drm_exec_fini(&submit->exec); in submit_cleanup()
471 void msm_submit_retire(struct msm_gem_submit *submit) in msm_submit_retire() argument
475 for (i = 0; i < submit->nr_bos; i++) { in msm_submit_retire()
476 struct drm_gem_object *obj = submit->bos[i].obj; in msm_submit_retire()
488 static struct drm_syncobj **msm_parse_deps(struct msm_gem_submit *submit, in msm_parse_deps() argument
515 !drm_core_check_feature(submit->dev, DRIVER_SYNCOBJ_TIMELINE)) { in msm_parse_deps()
525 ret = drm_sched_job_add_syncobj_dependency(&submit->base, file, in msm_parse_deps()
655 struct msm_gem_submit *submit = NULL; in msm_ioctl_gem_submit() local
705 submit = submit_create(dev, gpu, queue, args->nr_bos, args->nr_cmds); in msm_ioctl_gem_submit()
706 if (IS_ERR(submit)) { in msm_ioctl_gem_submit()
707 ret = PTR_ERR(submit); in msm_ioctl_gem_submit()
711 trace_msm_gpu_submit(pid_nr(submit->pid), ring->id, submit->ident, in msm_ioctl_gem_submit()
719 submit->in_rb = true; in msm_ioctl_gem_submit()
731 ret = drm_sched_job_add_dependency(&submit->base, in_fence); in msm_ioctl_gem_submit()
737 syncobjs_to_reset = msm_parse_deps(submit, file, in msm_ioctl_gem_submit()
758 ret = submit_lookup_objects(submit, args, file); in msm_ioctl_gem_submit()
762 ret = submit_lookup_cmds(submit, args, file); in msm_ioctl_gem_submit()
767 ret = submit_lock_objects(submit); in msm_ioctl_gem_submit()
772 ret = submit_fence_sync(submit); in msm_ioctl_gem_submit()
777 ret = submit_pin_objects(submit); in msm_ioctl_gem_submit()
785 ret = submit_bo(submit, submit->cmd[i].idx, &obj, &iova); in msm_ioctl_gem_submit()
789 if (!submit->cmd[i].size || in msm_ioctl_gem_submit()
790 ((submit->cmd[i].size + submit->cmd[i].offset) > in msm_ioctl_gem_submit()
792 SUBMIT_ERROR(submit, "invalid cmdstream size: %u\n", submit->cmd[i].size * 4); in msm_ioctl_gem_submit()
797 submit->cmd[i].iova = iova + (submit->cmd[i].offset * 4); in msm_ioctl_gem_submit()
799 if (likely(!submit->cmd[i].nr_relocs)) in msm_ioctl_gem_submit()
803 SUBMIT_ERROR(submit, "relocs not allowed\n"); in msm_ioctl_gem_submit()
808 ret = submit_reloc(submit, obj, submit->cmd[i].offset * 4, in msm_ioctl_gem_submit()
809 submit->cmd[i].nr_relocs, submit->cmd[i].relocs); in msm_ioctl_gem_submit()
814 submit->nr_cmds = i; in msm_ioctl_gem_submit()
834 drm_sched_job_arm(&submit->base); in msm_ioctl_gem_submit()
836 submit->user_fence = dma_fence_get(&submit->base.s_fence->finished); in msm_ioctl_gem_submit()
844 submit->fence_id = args->fence; in msm_ioctl_gem_submit()
845 ret = idr_alloc_u32(&queue->fence_idr, submit->user_fence, in msm_ioctl_gem_submit()
846 &submit->fence_id, submit->fence_id, in msm_ioctl_gem_submit()
858 submit->fence_id = idr_alloc_cyclic(&queue->fence_idr, in msm_ioctl_gem_submit()
859 submit->user_fence, 1, in msm_ioctl_gem_submit()
866 if (submit->fence_id < 0) { in msm_ioctl_gem_submit()
867 ret = submit->fence_id; in msm_ioctl_gem_submit()
868 submit->fence_id = 0; in msm_ioctl_gem_submit()
872 struct sync_file *sync_file = sync_file_create(submit->user_fence); in msm_ioctl_gem_submit()
884 submit_attach_object_fences(submit); in msm_ioctl_gem_submit()
887 msm_gem_submit_get(submit); in msm_ioctl_gem_submit()
889 msm_rd_dump_submit(priv->rd, submit, NULL); in msm_ioctl_gem_submit()
891 drm_sched_entity_push_job(&submit->base); in msm_ioctl_gem_submit()
893 args->fence = submit->fence_id; in msm_ioctl_gem_submit()
894 queue->last_fence = submit->fence_id; in msm_ioctl_gem_submit()
898 submit->user_fence); in msm_ioctl_gem_submit()
902 submit_cleanup(submit, !!ret); in msm_ioctl_gem_submit()
909 if (!IS_ERR_OR_NULL(submit)) { in msm_ioctl_gem_submit()
910 msm_gem_submit_put(submit); in msm_ioctl_gem_submit()