Lines Matching +full:data +full:- +full:mapping
1 // SPDX-License-Identifier: GPL-2.0-only
4 #include <linux/dma-fence-array.h>
5 #include <linux/dma-mapping.h>
27 dev_err_ratelimited(context->client->base.dev, \
29 current->comm, ##__VA_ARGS__)
46 kref_get(&bo->ref); in gather_bo_get()
55 dma_free_attrs(bo->dev, bo->gather_data_words * 4, bo->gather_data, bo->gather_data_dma, in gather_bo_release()
64 kref_put(&bo->ref, gather_bo_release); in gather_bo_put()
76 return ERR_PTR(-ENOMEM); in gather_bo_pin()
78 kref_init(&map->ref); in gather_bo_pin()
79 map->bo = host1x_bo_get(bo); in gather_bo_pin()
80 map->direction = direction; in gather_bo_pin()
81 map->dev = dev; in gather_bo_pin()
83 map->sgt = kzalloc(sizeof(*map->sgt), GFP_KERNEL); in gather_bo_pin()
84 if (!map->sgt) { in gather_bo_pin()
85 err = -ENOMEM; in gather_bo_pin()
89 err = dma_get_sgtable(gather->dev, map->sgt, gather->gather_data, gather->gather_data_dma, in gather_bo_pin()
90 gather->gather_data_words * 4); in gather_bo_pin()
94 err = dma_map_sgtable(dev, map->sgt, direction, 0); in gather_bo_pin()
98 map->phys = sg_dma_address(map->sgt->sgl); in gather_bo_pin()
99 map->size = gather->gather_data_words * 4; in gather_bo_pin()
100 map->chunks = err; in gather_bo_pin()
105 sg_free_table(map->sgt); in gather_bo_pin()
106 kfree(map->sgt); in gather_bo_pin()
117 dma_unmap_sgtable(map->dev, map->sgt, map->direction, 0); in gather_bo_unpin()
118 sg_free_table(map->sgt); in gather_bo_unpin()
119 kfree(map->sgt); in gather_bo_unpin()
120 host1x_bo_put(map->bo); in gather_bo_unpin()
129 return bo->gather_data; in gather_bo_mmap()
148 struct tegra_drm_mapping *mapping; in tegra_drm_mapping_get() local
150 xa_lock(&context->mappings); in tegra_drm_mapping_get()
152 mapping = xa_load(&context->mappings, id); in tegra_drm_mapping_get()
153 if (mapping) in tegra_drm_mapping_get()
154 kref_get(&mapping->ref); in tegra_drm_mapping_get()
156 xa_unlock(&context->mappings); in tegra_drm_mapping_get()
158 return mapping; in tegra_drm_mapping_get()
164 void *data; in alloc_copy_user_array() local
167 return ERR_PTR(-EINVAL); in alloc_copy_user_array()
170 return ERR_PTR(-E2BIG); in alloc_copy_user_array()
172 data = vmemdup_user(from, copy_len); in alloc_copy_user_array()
173 if (IS_ERR(data)) in alloc_copy_user_array()
174 return ERR_CAST(data); in alloc_copy_user_array()
176 return data; in alloc_copy_user_array()
186 if (args->gather_data_words == 0) { in submit_copy_gather_data()
188 return -EINVAL; in submit_copy_gather_data()
191 if (check_mul_overflow((size_t)args->gather_data_words, (size_t)4, ©_len)) { in submit_copy_gather_data()
193 return -EINVAL; in submit_copy_gather_data()
199 return -ENOMEM; in submit_copy_gather_data()
202 host1x_bo_init(&bo->base, &gather_bo_ops); in submit_copy_gather_data()
203 kref_init(&bo->ref); in submit_copy_gather_data()
204 bo->dev = dev; in submit_copy_gather_data()
206 bo->gather_data = dma_alloc_attrs(dev, copy_len, &bo->gather_data_dma, in submit_copy_gather_data()
208 if (!bo->gather_data) { in submit_copy_gather_data()
209 SUBMIT_ERR(context, "failed to allocate memory for gather data"); in submit_copy_gather_data()
211 return -ENOMEM; in submit_copy_gather_data()
214 if (copy_from_user(bo->gather_data, u64_to_user_ptr(args->gather_data_ptr), copy_len)) { in submit_copy_gather_data()
215 SUBMIT_ERR(context, "failed to copy gather data from userspace"); in submit_copy_gather_data()
216 dma_free_attrs(dev, copy_len, bo->gather_data, bo->gather_data_dma, 0); in submit_copy_gather_data()
218 return -EFAULT; in submit_copy_gather_data()
221 bo->gather_data_words = args->gather_data_words; in submit_copy_gather_data()
229 struct drm_tegra_submit_buf *buf, struct tegra_drm_mapping *mapping) in submit_write_reloc() argument
232 dma_addr_t iova = mapping->iova + buf->reloc.target_offset; in submit_write_reloc()
236 if (buf->flags & DRM_TEGRA_SUBMIT_RELOC_SECTOR_LAYOUT) in submit_write_reloc()
240 written_ptr = iova >> buf->reloc.shift; in submit_write_reloc()
242 if (buf->reloc.gather_offset_words >= bo->gather_data_words) { in submit_write_reloc()
245 buf->reloc.gather_offset_words, bo->gather_data_words); in submit_write_reloc()
246 return -EINVAL; in submit_write_reloc()
249 buf->reloc.gather_offset_words = array_index_nospec(buf->reloc.gather_offset_words, in submit_write_reloc()
250 bo->gather_data_words); in submit_write_reloc()
252 bo->gather_data[buf->reloc.gather_offset_words] = written_ptr; in submit_write_reloc()
266 bufs = alloc_copy_user_array(u64_to_user_ptr(args->bufs_ptr), args->num_bufs, in submit_process_bufs()
273 mappings = kcalloc(args->num_bufs, sizeof(*mappings), GFP_KERNEL); in submit_process_bufs()
275 SUBMIT_ERR(context, "failed to allocate memory for mapping info"); in submit_process_bufs()
276 err = -ENOMEM; in submit_process_bufs()
280 for (i = 0; i < args->num_bufs; i++) { in submit_process_bufs()
282 struct tegra_drm_mapping *mapping; in submit_process_bufs() local
284 if (buf->flags & ~DRM_TEGRA_SUBMIT_RELOC_SECTOR_LAYOUT) { in submit_process_bufs()
286 err = -EINVAL; in submit_process_bufs()
290 mapping = tegra_drm_mapping_get(context, buf->mapping); in submit_process_bufs()
291 if (!mapping) { in submit_process_bufs()
292 SUBMIT_ERR(context, "invalid mapping ID '%u' for buffer", buf->mapping); in submit_process_bufs()
293 err = -EINVAL; in submit_process_bufs()
297 err = submit_write_reloc(context, bo, buf, mapping); in submit_process_bufs()
299 tegra_drm_mapping_put(mapping); in submit_process_bufs()
303 mappings[i].mapping = mapping; in submit_process_bufs()
304 mappings[i].flags = buf->flags; in submit_process_bufs()
307 job_data->used_mappings = mappings; in submit_process_bufs()
308 job_data->num_used_mappings = i; in submit_process_bufs()
315 while (i--) in submit_process_bufs()
316 tegra_drm_mapping_put(mappings[i].mapping); in submit_process_bufs()
319 job_data->used_mappings = NULL; in submit_process_bufs()
332 if (args->syncpt.flags) { in submit_get_syncpt()
334 return -EINVAL; in submit_get_syncpt()
338 sp = xa_load(syncpoints, args->syncpt.id); in submit_get_syncpt()
341 return -EINVAL; in submit_get_syncpt()
344 job->syncpt = host1x_syncpt_get(sp); in submit_get_syncpt()
345 job->syncpt_incrs = args->syncpt.increments; in submit_get_syncpt()
358 if (cmd->reserved[0] || cmd->reserved[1] || cmd->reserved[2]) { in submit_job_add_gather()
359 SUBMIT_ERR(context, "non-zero reserved field in GATHER_UPTR command"); in submit_job_add_gather()
360 return -EINVAL; in submit_job_add_gather()
364 if (cmd->words > 16383) { in submit_job_add_gather()
366 return -EINVAL; in submit_job_add_gather()
369 if (check_add_overflow(*offset, cmd->words, &next_offset)) { in submit_job_add_gather()
371 return -EINVAL; in submit_job_add_gather()
374 if (next_offset > bo->gather_data_words) { in submit_job_add_gather()
375 SUBMIT_ERR(context, "GATHER_UPTR command overflows gather data"); in submit_job_add_gather()
376 return -EINVAL; in submit_job_add_gather()
379 if (tegra_drm_fw_validate(context->client, bo->gather_data, *offset, in submit_job_add_gather()
380 cmd->words, job_data, class)) { in submit_job_add_gather()
382 return -EINVAL; in submit_job_add_gather()
385 host1x_job_add_gather(job, &bo->base, cmd->words, *offset * 4); in submit_job_add_gather()
403 class = context->client->base.class; in submit_create_job()
405 cmds = alloc_copy_user_array(u64_to_user_ptr(args->cmds_ptr), args->num_cmds, in submit_create_job()
412 job = host1x_job_alloc(context->channel, args->num_cmds, 0, true); in submit_create_job()
415 job = ERR_PTR(-ENOMEM); in submit_create_job()
423 job->client = &context->client->base; in submit_create_job()
424 job->class = context->client->base.class; in submit_create_job()
425 job->serialize = true; in submit_create_job()
427 for (i = 0; i < args->num_cmds; i++) { in submit_create_job()
430 if (cmd->flags) { in submit_create_job()
432 err = -EINVAL; in submit_create_job()
436 if (cmd->type == DRM_TEGRA_SUBMIT_CMD_GATHER_UPTR) { in submit_create_job()
437 err = submit_job_add_gather(job, context, &cmd->gather_uptr, bo, in submit_create_job()
441 } else if (cmd->type == DRM_TEGRA_SUBMIT_CMD_WAIT_SYNCPT) { in submit_create_job()
442 if (cmd->wait_syncpt.reserved[0] || cmd->wait_syncpt.reserved[1]) { in submit_create_job()
443 SUBMIT_ERR(context, "non-zero reserved value"); in submit_create_job()
444 err = -EINVAL; in submit_create_job()
448 host1x_job_add_wait(job, cmd->wait_syncpt.id, cmd->wait_syncpt.value, in submit_create_job()
450 } else if (cmd->type == DRM_TEGRA_SUBMIT_CMD_WAIT_SYNCPT_RELATIVE) { in submit_create_job()
451 if (cmd->wait_syncpt.reserved[0] || cmd->wait_syncpt.reserved[1]) { in submit_create_job()
452 SUBMIT_ERR(context, "non-zero reserved value"); in submit_create_job()
453 err = -EINVAL; in submit_create_job()
457 if (cmd->wait_syncpt.id != args->syncpt.id) { in submit_create_job()
459 err = -EINVAL; in submit_create_job()
463 host1x_job_add_wait(job, cmd->wait_syncpt.id, cmd->wait_syncpt.value, in submit_create_job()
467 err = -EINVAL; in submit_create_job()
474 err = -EINVAL; in submit_create_job()
492 struct tegra_drm_client *client = container_of(job->client, struct tegra_drm_client, base); in release_job()
493 struct tegra_drm_submit_data *job_data = job->user_data; in release_job()
496 if (job->memory_context) in release_job()
497 host1x_memory_context_put(job->memory_context); in release_job()
499 for (i = 0; i < job_data->num_used_mappings; i++) in release_job()
500 tegra_drm_mapping_put(job_data->used_mappings[i].mapping); in release_job()
502 kfree(job_data->used_mappings); in release_job()
505 pm_runtime_mark_last_busy(client->base.dev); in release_job()
506 pm_runtime_put_autosuspend(client->base.dev); in release_job()
509 int tegra_drm_ioctl_channel_submit(struct drm_device *drm, void *data, in tegra_drm_ioctl_channel_submit() argument
512 struct tegra_drm_file *fpriv = file->driver_priv; in tegra_drm_ioctl_channel_submit()
513 struct drm_tegra_channel_submit *args = data; in tegra_drm_ioctl_channel_submit()
522 mutex_lock(&fpriv->lock); in tegra_drm_ioctl_channel_submit()
524 context = xa_load(&fpriv->contexts, args->context); in tegra_drm_ioctl_channel_submit()
526 mutex_unlock(&fpriv->lock); in tegra_drm_ioctl_channel_submit()
528 current->comm, args->context); in tegra_drm_ioctl_channel_submit()
529 return -EINVAL; in tegra_drm_ioctl_channel_submit()
532 if (args->syncobj_in) { in tegra_drm_ioctl_channel_submit()
535 err = drm_syncobj_find_fence(file, args->syncobj_in, 0, 0, &fence); in tegra_drm_ioctl_channel_submit()
537 SUBMIT_ERR(context, "invalid syncobj_in '%#x'", args->syncobj_in); in tegra_drm_ioctl_channel_submit()
549 if (args->syncobj_out) { in tegra_drm_ioctl_channel_submit()
550 syncobj = drm_syncobj_find(file, args->syncobj_out); in tegra_drm_ioctl_channel_submit()
552 SUBMIT_ERR(context, "invalid syncobj_out '%#x'", args->syncobj_out); in tegra_drm_ioctl_channel_submit()
553 err = -ENOENT; in tegra_drm_ioctl_channel_submit()
559 err = submit_copy_gather_data(&bo, drm->dev, context, args); in tegra_drm_ioctl_channel_submit()
565 SUBMIT_ERR(context, "failed to allocate memory for job data"); in tegra_drm_ioctl_channel_submit()
566 err = -ENOMEM; in tegra_drm_ioctl_channel_submit()
570 /* Get data buffer mappings and do relocation patching. */ in tegra_drm_ioctl_channel_submit()
576 job = submit_create_job(context, bo, args, job_data, &fpriv->syncpoints); in tegra_drm_ioctl_channel_submit()
582 /* Map gather data for Host1x. */ in tegra_drm_ioctl_channel_submit()
583 err = host1x_job_pin(job, context->client->base.dev); in tegra_drm_ioctl_channel_submit()
589 if (context->client->ops->get_streamid_offset) { in tegra_drm_ioctl_channel_submit()
590 err = context->client->ops->get_streamid_offset( in tegra_drm_ioctl_channel_submit()
591 context->client, &job->engine_streamid_offset); in tegra_drm_ioctl_channel_submit()
598 if (context->memory_context && context->client->ops->can_use_memory_ctx) { in tegra_drm_ioctl_channel_submit()
601 err = context->client->ops->can_use_memory_ctx(context->client, &supported); in tegra_drm_ioctl_channel_submit()
608 job->memory_context = context->memory_context; in tegra_drm_ioctl_channel_submit()
609 host1x_memory_context_get(job->memory_context); in tegra_drm_ioctl_channel_submit()
611 } else if (context->client->ops->get_streamid_offset) { in tegra_drm_ioctl_channel_submit()
616 if (!tegra_dev_iommu_get_stream_id(context->client->base.dev, in tegra_drm_ioctl_channel_submit()
617 &job->engine_fallback_streamid)) in tegra_drm_ioctl_channel_submit()
618 job->engine_fallback_streamid = TEGRA_STREAM_ID_BYPASS; in tegra_drm_ioctl_channel_submit()
622 err = pm_runtime_resume_and_get(context->client->base.dev); in tegra_drm_ioctl_channel_submit()
628 job->user_data = job_data; in tegra_drm_ioctl_channel_submit()
629 job->release = release_job; in tegra_drm_ioctl_channel_submit()
630 job->timeout = 10000; in tegra_drm_ioctl_channel_submit()
646 args->syncpt.value = job->syncpt_end; in tegra_drm_ioctl_channel_submit()
649 struct dma_fence *fence = host1x_fence_create(job->syncpt, job->syncpt_end, true); in tegra_drm_ioctl_channel_submit()
661 if (job->memory_context) in tegra_drm_ioctl_channel_submit()
662 host1x_memory_context_put(job->memory_context); in tegra_drm_ioctl_channel_submit()
668 if (job_data && job_data->used_mappings) { in tegra_drm_ioctl_channel_submit()
669 for (i = 0; i < job_data->num_used_mappings; i++) in tegra_drm_ioctl_channel_submit()
670 tegra_drm_mapping_put(job_data->used_mappings[i].mapping); in tegra_drm_ioctl_channel_submit()
672 kfree(job_data->used_mappings); in tegra_drm_ioctl_channel_submit()
677 gather_bo_put(&bo->base); in tegra_drm_ioctl_channel_submit()
682 mutex_unlock(&fpriv->lock); in tegra_drm_ioctl_channel_submit()