Lines Matching full:vdev
28 static void ivpu_cmdq_ring_db(struct ivpu_device *vdev, struct ivpu_cmdq *cmdq) in ivpu_cmdq_ring_db() argument
30 ivpu_hw_db_set(vdev, cmdq->db_id); in ivpu_cmdq_ring_db()
33 static int ivpu_preemption_buffers_create(struct ivpu_device *vdev, in ivpu_preemption_buffers_create() argument
36 u64 primary_size = ALIGN(vdev->fw->primary_preempt_buf_size, PAGE_SIZE); in ivpu_preemption_buffers_create()
37 u64 secondary_size = ALIGN(vdev->fw->secondary_preempt_buf_size, PAGE_SIZE); in ivpu_preemption_buffers_create()
40 if (vdev->hw->sched_mode != VPU_SCHEDULING_MODE_HW) in ivpu_preemption_buffers_create()
43 range.start = vdev->hw->ranges.user.end - (primary_size * IVPU_NUM_CMDQS_PER_CTX); in ivpu_preemption_buffers_create()
44 range.end = vdev->hw->ranges.user.end; in ivpu_preemption_buffers_create()
45 cmdq->primary_preempt_buf = ivpu_bo_create(vdev, &file_priv->ctx, &range, primary_size, in ivpu_preemption_buffers_create()
48 ivpu_err(vdev, "Failed to create primary preemption buffer\n"); in ivpu_preemption_buffers_create()
52 range.start = vdev->hw->ranges.shave.end - (secondary_size * IVPU_NUM_CMDQS_PER_CTX); in ivpu_preemption_buffers_create()
53 range.end = vdev->hw->ranges.shave.end; in ivpu_preemption_buffers_create()
54 cmdq->secondary_preempt_buf = ivpu_bo_create(vdev, &file_priv->ctx, &range, secondary_size, in ivpu_preemption_buffers_create()
57 ivpu_err(vdev, "Failed to create secondary preemption buffer\n"); in ivpu_preemption_buffers_create()
68 static void ivpu_preemption_buffers_free(struct ivpu_device *vdev, in ivpu_preemption_buffers_free() argument
71 if (vdev->hw->sched_mode != VPU_SCHEDULING_MODE_HW) in ivpu_preemption_buffers_free()
74 drm_WARN_ON(&vdev->drm, !cmdq->primary_preempt_buf); in ivpu_preemption_buffers_free()
75 drm_WARN_ON(&vdev->drm, !cmdq->secondary_preempt_buf); in ivpu_preemption_buffers_free()
83 struct ivpu_device *vdev = file_priv->vdev; in ivpu_cmdq_alloc() local
91 ret = xa_alloc(&vdev->db_xa, &cmdq->db_id, NULL, db_xa_limit, GFP_KERNEL); in ivpu_cmdq_alloc()
93 ivpu_err(vdev, "Failed to allocate doorbell id: %d\n", ret); in ivpu_cmdq_alloc()
97 cmdq->mem = ivpu_bo_create_global(vdev, SZ_4K, DRM_IVPU_BO_WC | DRM_IVPU_BO_MAPPABLE); in ivpu_cmdq_alloc()
101 ret = ivpu_preemption_buffers_create(vdev, file_priv, cmdq); in ivpu_cmdq_alloc()
110 xa_erase(&vdev->db_xa, cmdq->db_id); in ivpu_cmdq_alloc()
121 ivpu_preemption_buffers_free(file_priv->vdev, file_priv, cmdq); in ivpu_cmdq_free()
123 xa_erase(&file_priv->vdev->db_xa, cmdq->db_id); in ivpu_cmdq_free()
130 struct ivpu_device *vdev = file_priv->vdev; in ivpu_hws_cmdq_init() local
133 ret = ivpu_jsm_hws_create_cmdq(vdev, file_priv->ctx.id, file_priv->ctx.id, cmdq->db_id, in ivpu_hws_cmdq_init()
139 ret = ivpu_jsm_hws_set_context_sched_properties(vdev, file_priv->ctx.id, cmdq->db_id, in ivpu_hws_cmdq_init()
149 struct ivpu_device *vdev = file_priv->vdev; in ivpu_register_db() local
152 if (vdev->hw->sched_mode == VPU_SCHEDULING_MODE_HW) in ivpu_register_db()
153 ret = ivpu_jsm_hws_register_db(vdev, file_priv->ctx.id, cmdq->db_id, cmdq->db_id, in ivpu_register_db()
156 ret = ivpu_jsm_register_db(vdev, file_priv->ctx.id, cmdq->db_id, in ivpu_register_db()
160 ivpu_dbg(vdev, JOB, "DB %d registered to ctx %d\n", cmdq->db_id, file_priv->ctx.id); in ivpu_register_db()
168 struct ivpu_device *vdev = file_priv->vdev; in ivpu_cmdq_init() local
187 if (vdev->hw->sched_mode == VPU_SCHEDULING_MODE_HW) { in ivpu_cmdq_init()
204 struct ivpu_device *vdev = file_priv->vdev; in ivpu_cmdq_fini() local
214 if (vdev->hw->sched_mode == VPU_SCHEDULING_MODE_HW) { in ivpu_cmdq_fini()
215 ret = ivpu_jsm_hws_destroy_cmdq(vdev, file_priv->ctx.id, cmdq->db_id); in ivpu_cmdq_fini()
217 ivpu_dbg(vdev, JOB, "Command queue %d destroyed\n", cmdq->db_id); in ivpu_cmdq_fini()
220 ret = ivpu_jsm_unregister_db(vdev, cmdq->db_id); in ivpu_cmdq_fini()
222 ivpu_dbg(vdev, JOB, "DB %d unregistered\n", cmdq->db_id); in ivpu_cmdq_fini()
302 void ivpu_cmdq_reset_all_contexts(struct ivpu_device *vdev) in ivpu_cmdq_reset_all_contexts() argument
307 mutex_lock(&vdev->context_list_lock); in ivpu_cmdq_reset_all_contexts()
309 xa_for_each(&vdev->context_xa, ctx_id, file_priv) in ivpu_cmdq_reset_all_contexts()
312 mutex_unlock(&vdev->context_list_lock); in ivpu_cmdq_reset_all_contexts()
332 struct ivpu_device *vdev = file_priv->vdev; in ivpu_context_abort_locked() local
338 if (vdev->hw->sched_mode == VPU_SCHEDULING_MODE_OS) in ivpu_context_abort_locked()
339 ivpu_jsm_context_release(vdev, file_priv->ctx.id); in ivpu_context_abort_locked()
344 struct ivpu_device *vdev = job->vdev; in ivpu_cmdq_push_job() local
352 ivpu_dbg(vdev, JOB, "Job queue full: ctx %d engine %d db %d head %d tail %d\n", in ivpu_cmdq_push_job()
364 if (vdev->hw->sched_mode == VPU_SCHEDULING_MODE_HW && in ivpu_cmdq_push_job()
382 struct ivpu_device *vdev; member
399 return dev_name(ivpu_fence->vdev->drm.dev); in ivpu_fence_get_timeline_name()
407 static struct dma_fence *ivpu_fence_create(struct ivpu_device *vdev) in ivpu_fence_create() argument
415 fence->vdev = vdev; in ivpu_fence_create()
424 struct ivpu_device *vdev = job->vdev; in ivpu_job_destroy() local
427 ivpu_dbg(vdev, JOB, "Job destroyed: id %3u ctx %2d engine %d", in ivpu_job_destroy()
442 struct ivpu_device *vdev = file_priv->vdev; in ivpu_job_create() local
449 job->vdev = vdev; in ivpu_job_create()
452 job->done_fence = ivpu_fence_create(vdev); in ivpu_job_create()
454 ivpu_warn_ratelimited(vdev, "Failed to create a fence\n"); in ivpu_job_create()
460 ivpu_dbg(vdev, JOB, "Job created: ctx %2d engine %d", file_priv->ctx.id, job->engine_idx); in ivpu_job_create()
468 static struct ivpu_job *ivpu_job_remove_from_submitted_jobs(struct ivpu_device *vdev, u32 job_id) in ivpu_job_remove_from_submitted_jobs() argument
472 xa_lock(&vdev->submitted_jobs_xa); in ivpu_job_remove_from_submitted_jobs()
473 job = __xa_erase(&vdev->submitted_jobs_xa, job_id); in ivpu_job_remove_from_submitted_jobs()
475 if (xa_empty(&vdev->submitted_jobs_xa) && job) { in ivpu_job_remove_from_submitted_jobs()
476 vdev->busy_time = ktime_add(ktime_sub(ktime_get(), vdev->busy_start_ts), in ivpu_job_remove_from_submitted_jobs()
477 vdev->busy_time); in ivpu_job_remove_from_submitted_jobs()
480 xa_unlock(&vdev->submitted_jobs_xa); in ivpu_job_remove_from_submitted_jobs()
485 static int ivpu_job_signal_and_destroy(struct ivpu_device *vdev, u32 job_id, u32 job_status) in ivpu_job_signal_and_destroy() argument
489 job = ivpu_job_remove_from_submitted_jobs(vdev, job_id); in ivpu_job_signal_and_destroy()
499 ivpu_dbg(vdev, JOB, "Job complete: id %3u ctx %2d engine %d status 0x%x\n", in ivpu_job_signal_and_destroy()
503 ivpu_stop_job_timeout_detection(vdev); in ivpu_job_signal_and_destroy()
505 ivpu_rpm_put(vdev); in ivpu_job_signal_and_destroy()
509 void ivpu_jobs_abort_all(struct ivpu_device *vdev) in ivpu_jobs_abort_all() argument
514 xa_for_each(&vdev->submitted_jobs_xa, id, job) in ivpu_jobs_abort_all()
515 ivpu_job_signal_and_destroy(vdev, id, DRM_IVPU_JOB_STATUS_ABORTED); in ivpu_jobs_abort_all()
521 struct ivpu_device *vdev = job->vdev; in ivpu_job_submit() local
527 ret = ivpu_rpm_get(vdev); in ivpu_job_submit()
535 ivpu_warn_ratelimited(vdev, "Failed to get job queue, ctx %d engine %d prio %d\n", in ivpu_job_submit()
544 xa_lock(&vdev->submitted_jobs_xa); in ivpu_job_submit()
545 is_first_job = xa_empty(&vdev->submitted_jobs_xa); in ivpu_job_submit()
546 ret = __xa_alloc(&vdev->submitted_jobs_xa, &job->job_id, job, job_id_range, GFP_KERNEL); in ivpu_job_submit()
548 ivpu_dbg(vdev, JOB, "Too many active jobs in ctx %d\n", in ivpu_job_submit()
558 ivpu_start_job_timeout_detection(vdev); in ivpu_job_submit()
564 ivpu_cmdq_ring_db(vdev, cmdq); in ivpu_job_submit()
566 vdev->busy_start_ts = ktime_get(); in ivpu_job_submit()
569 ivpu_dbg(vdev, JOB, "Job submitted: id %3u ctx %2d engine %d prio %d addr 0x%llx next %d\n", in ivpu_job_submit()
573 xa_unlock(&vdev->submitted_jobs_xa); in ivpu_job_submit()
578 ivpu_job_signal_and_destroy(vdev, job->job_id, VPU_JSM_STATUS_SUCCESS); in ivpu_job_submit()
583 __xa_erase(&vdev->submitted_jobs_xa, job->job_id); in ivpu_job_submit()
585 xa_unlock(&vdev->submitted_jobs_xa); in ivpu_job_submit()
588 ivpu_rpm_put(vdev); in ivpu_job_submit()
597 struct ivpu_device *vdev = file_priv->vdev; in ivpu_job_prepare_bos_for_submit() local
619 ivpu_warn(vdev, "Buffer is already in use\n"); in ivpu_job_prepare_bos_for_submit()
624 ivpu_warn(vdev, "Invalid command buffer offset %u\n", commands_offset); in ivpu_job_prepare_bos_for_submit()
633 ivpu_warn(vdev, "Failed to lock reservations: %d\n", ret); in ivpu_job_prepare_bos_for_submit()
640 ivpu_warn(vdev, "Failed to reserve fences: %d\n", ret); in ivpu_job_prepare_bos_for_submit()
669 struct ivpu_device *vdev = file_priv->vdev; in ivpu_submit_ioctl() local
706 if (!drm_dev_enter(&vdev->drm, &idx)) { in ivpu_submit_ioctl()
711 ivpu_dbg(vdev, JOB, "Submit ioctl: ctx %u buf_count %u\n", in ivpu_submit_ioctl()
716 ivpu_err(vdev, "Failed to create job\n"); in ivpu_submit_ioctl()
724 ivpu_err(vdev, "Failed to prepare job: %d\n", ret); in ivpu_submit_ioctl()
730 down_read(&vdev->pm->reset_lock); in ivpu_submit_ioctl()
732 up_read(&vdev->pm->reset_lock); in ivpu_submit_ioctl()
752 ivpu_job_done_callback(struct ivpu_device *vdev, struct ivpu_ipc_hdr *ipc_hdr, in ivpu_job_done_callback() argument
759 ivpu_err(vdev, "IPC message has no JSM payload\n"); in ivpu_job_done_callback()
764 ivpu_err(vdev, "Invalid JSM message result: %d\n", jsm_msg->result); in ivpu_job_done_callback()
769 ret = ivpu_job_signal_and_destroy(vdev, payload->job_id, payload->job_status); in ivpu_job_done_callback()
770 if (!ret && !xa_empty(&vdev->submitted_jobs_xa)) in ivpu_job_done_callback()
771 ivpu_start_job_timeout_detection(vdev); in ivpu_job_done_callback()
774 void ivpu_job_done_consumer_init(struct ivpu_device *vdev) in ivpu_job_done_consumer_init() argument
776 ivpu_ipc_consumer_add(vdev, &vdev->job_done_consumer, in ivpu_job_done_consumer_init()
780 void ivpu_job_done_consumer_fini(struct ivpu_device *vdev) in ivpu_job_done_consumer_fini() argument
782 ivpu_ipc_consumer_del(vdev, &vdev->job_done_consumer); in ivpu_job_done_consumer_fini()