Lines Matching +full:m +full:- +full:class
1 // SPDX-License-Identifier: MIT
40 * on HSW) - so the final size, including the extra state required for the
54 u8 class; member
65 .class = RENDER_CLASS,
72 .class = COPY_ENGINE_CLASS,
79 .class = COPY_ENGINE_CLASS,
86 .class = COPY_ENGINE_CLASS,
93 .class = COPY_ENGINE_CLASS,
100 .class = COPY_ENGINE_CLASS,
107 .class = COPY_ENGINE_CLASS,
114 .class = COPY_ENGINE_CLASS,
121 .class = COPY_ENGINE_CLASS,
128 .class = COPY_ENGINE_CLASS,
135 .class = VIDEO_DECODE_CLASS,
144 .class = VIDEO_DECODE_CLASS,
152 .class = VIDEO_DECODE_CLASS,
159 .class = VIDEO_DECODE_CLASS,
166 .class = VIDEO_DECODE_CLASS,
173 .class = VIDEO_DECODE_CLASS,
180 .class = VIDEO_DECODE_CLASS,
187 .class = VIDEO_DECODE_CLASS,
194 .class = VIDEO_ENHANCEMENT_CLASS,
202 .class = VIDEO_ENHANCEMENT_CLASS,
209 .class = VIDEO_ENHANCEMENT_CLASS,
216 .class = VIDEO_ENHANCEMENT_CLASS,
223 .class = COMPUTE_CLASS,
230 .class = COMPUTE_CLASS,
237 .class = COMPUTE_CLASS,
244 .class = COMPUTE_CLASS,
251 .class = OTHER_CLASS,
260 * intel_engine_context_size() - return the size of the context for an engine
262 * @class: engine class
264 * Each engine class may require a different amount of space for a context
267 * Return: size (in bytes) of an engine class specific context image
273 u32 intel_engine_context_size(struct intel_gt *gt, u8 class) in intel_engine_context_size() argument
275 struct intel_uncore *uncore = gt->uncore; in intel_engine_context_size()
280 switch (class) { in intel_engine_context_size()
284 switch (GRAPHICS_VER(gt->i915)) { in intel_engine_context_size()
286 MISSING_CASE(GRAPHICS_VER(gt->i915)); in intel_engine_context_size()
296 if (IS_HASWELL(gt->i915)) in intel_engine_context_size()
320 GRAPHICS_VER(gt->i915), cxt_size * 64, in intel_engine_context_size()
321 cxt_size - 1); in intel_engine_context_size()
331 MISSING_CASE(class); in intel_engine_context_size()
337 if (GRAPHICS_VER(gt->i915) < 8) in intel_engine_context_size()
365 GEM_WARN_ON(snprintf(engine->name, sizeof(engine->name), "%s'%u", in __sprint_engine_name()
366 intel_engine_class_repr(engine->class), in __sprint_engine_name()
367 engine->instance) >= sizeof(engine->name)); in __sprint_engine_name()
374 * per-engine HWSTAM until gen6. in intel_engine_set_hwsp_writemask()
376 if (GRAPHICS_VER(engine->i915) < 6 && engine->class != RENDER_CLASS) in intel_engine_set_hwsp_writemask()
379 if (GRAPHICS_VER(engine->i915) >= 3) in intel_engine_set_hwsp_writemask()
453 struct drm_i915_private *i915 = gt->i915; in intel_engine_setup()
462 if (GEM_DEBUG_WARN_ON(id >= ARRAY_SIZE(gt->engine))) in intel_engine_setup()
463 return -EINVAL; in intel_engine_setup()
465 if (GEM_DEBUG_WARN_ON(info->class > MAX_ENGINE_CLASS)) in intel_engine_setup()
466 return -EINVAL; in intel_engine_setup()
468 if (GEM_DEBUG_WARN_ON(info->instance > MAX_ENGINE_INSTANCE)) in intel_engine_setup()
469 return -EINVAL; in intel_engine_setup()
471 if (GEM_DEBUG_WARN_ON(gt->engine_class[info->class][info->instance])) in intel_engine_setup()
472 return -EINVAL; in intel_engine_setup()
476 return -ENOMEM; in intel_engine_setup()
478 BUILD_BUG_ON(BITS_PER_TYPE(engine->mask) < I915_NUM_ENGINES); in intel_engine_setup()
480 INIT_LIST_HEAD(&engine->pinned_contexts_list); in intel_engine_setup()
481 engine->id = id; in intel_engine_setup()
482 engine->legacy_idx = INVALID_ENGINE; in intel_engine_setup()
483 engine->mask = BIT(id); in intel_engine_setup()
484 engine->reset_domain = get_reset_domain(GRAPHICS_VER(gt->i915), in intel_engine_setup()
486 engine->i915 = i915; in intel_engine_setup()
487 engine->gt = gt; in intel_engine_setup()
488 engine->uncore = gt->uncore; in intel_engine_setup()
489 guc_class = engine_class_to_guc_class(info->class); in intel_engine_setup()
490 engine->guc_id = MAKE_GUC_ID(guc_class, info->instance); in intel_engine_setup()
491 engine->mmio_base = __engine_mmio_base(i915, info->mmio_bases); in intel_engine_setup()
493 engine->irq_handler = nop_irq_handler; in intel_engine_setup()
495 engine->class = info->class; in intel_engine_setup()
496 engine->instance = info->instance; in intel_engine_setup()
497 engine->logical_mask = BIT(logical_instance); in intel_engine_setup()
500 if ((engine->class == COMPUTE_CLASS || engine->class == RENDER_CLASS) && in intel_engine_setup()
501 __ffs(CCS_MASK(engine->gt) | RCS_MASK(engine->gt)) == engine->instance) in intel_engine_setup()
502 engine->flags |= I915_ENGINE_FIRST_RENDER_COMPUTE; in intel_engine_setup()
505 if (engine->class == RENDER_CLASS || engine->class == COMPUTE_CLASS) { in intel_engine_setup()
506 engine->flags |= I915_ENGINE_HAS_RCS_REG_STATE; in intel_engine_setup()
507 engine->flags |= I915_ENGINE_HAS_EU_PRIORITY; in intel_engine_setup()
510 engine->props.heartbeat_interval_ms = in intel_engine_setup()
512 engine->props.max_busywait_duration_ns = in intel_engine_setup()
514 engine->props.preempt_timeout_ms = in intel_engine_setup()
516 engine->props.stop_timeout_ms = in intel_engine_setup()
518 engine->props.timeslice_duration_ms = in intel_engine_setup()
522 * Mid-thread pre-emption is not available in Gen12. Unfortunately, in intel_engine_setup()
524 * reset due to not pre-empting in a timely manner. So, bump the in intel_engine_setup()
525 * pre-emption timeout value to be much higher for compute engines. in intel_engine_setup()
527 if (GRAPHICS_VER(i915) == 12 && (engine->flags & I915_ENGINE_HAS_RCS_REG_STATE)) in intel_engine_setup()
528 engine->props.preempt_timeout_ms = CONFIG_DRM_I915_PREEMPT_TIMEOUT_COMPUTE; in intel_engine_setup()
533 u64 clamp = intel_clamp_##field(engine, engine->props.field); \ in intel_engine_setup()
534 if (clamp != engine->props.field) { \ in intel_engine_setup()
535 drm_notice(&engine->i915->drm, \ in intel_engine_setup()
538 engine->props.field = clamp; \ in intel_engine_setup()
550 engine->defaults = engine->props; /* never to change again */ in intel_engine_setup()
552 engine->context_size = intel_engine_context_size(gt, engine->class); in intel_engine_setup()
553 if (WARN_ON(engine->context_size > BIT(20))) in intel_engine_setup()
554 engine->context_size = 0; in intel_engine_setup()
555 if (engine->context_size) in intel_engine_setup()
556 DRIVER_CAPS(i915)->has_logical_contexts = true; in intel_engine_setup()
558 ewma__engine_latency_init(&engine->latency); in intel_engine_setup()
560 ATOMIC_INIT_NOTIFIER_HEAD(&engine->context_status_notifier); in intel_engine_setup()
565 gt->engine_class[info->class][info->instance] = engine; in intel_engine_setup()
566 gt->engine[id] = engine; in intel_engine_setup()
591 if (intel_guc_submission_is_wanted(gt_to_guc(engine->gt))) in intel_clamp_preempt_timeout_ms()
612 if (intel_guc_submission_is_wanted(gt_to_guc(engine->gt))) in intel_clamp_timeslice_duration_ms()
622 struct drm_i915_private *i915 = engine->i915; in __setup_engine_capabilities()
624 if (engine->class == VIDEO_DECODE_CLASS) { in __setup_engine_capabilities()
630 (GRAPHICS_VER(i915) >= 9 && engine->instance == 0)) in __setup_engine_capabilities()
631 engine->uabi_capabilities |= in __setup_engine_capabilities()
639 (engine->gt->info.vdbox_sfc_access & in __setup_engine_capabilities()
640 BIT(engine->instance))) || in __setup_engine_capabilities()
641 (GRAPHICS_VER(i915) >= 9 && engine->instance == 0)) in __setup_engine_capabilities()
642 engine->uabi_capabilities |= in __setup_engine_capabilities()
644 } else if (engine->class == VIDEO_ENHANCEMENT_CLASS) { in __setup_engine_capabilities()
646 engine->gt->info.sfc_mask & BIT(engine->instance)) in __setup_engine_capabilities()
647 engine->uabi_capabilities |= in __setup_engine_capabilities()
662 * intel_engines_release() - free the resources allocated for Command Streamers
672 * that the HW is no longer accessing them -- having the GPU scribble in intel_engines_release()
680 if (!INTEL_INFO(gt->i915)->gpu_reset_clobbers_display) in intel_engines_release()
685 if (!engine->release) in intel_engines_release()
688 intel_wakeref_wait_for_idle(&engine->wakeref); in intel_engines_release()
691 engine->release(engine); in intel_engines_release()
692 engine->release = NULL; in intel_engines_release()
694 memset(&engine->reset, 0, sizeof(engine->reset)); in intel_engines_release()
697 llist_del_all(>->i915->uabi_engines_llist); in intel_engines_release()
702 if (!engine->request_pool) in intel_engine_free_request_pool()
705 kmem_cache_free(i915_request_slab_cache(), engine->request_pool); in intel_engine_free_request_pool()
713 /* Free the requests! dma-resv keeps fences around for an eternity */ in intel_engines_free()
719 gt->engine[id] = NULL; in intel_engines_free()
728 struct drm_i915_private *i915 = gt->i915; in gen11_vdbox_has_sfc()
740 if ((gt->info.sfc_mask & BIT(physical_vdbox / 2)) == 0) in gen11_vdbox_has_sfc()
744 !(BIT(physical_vdbox - 1) & vdbox_mask); in gen11_vdbox_has_sfc()
753 struct drm_i915_private *i915 = gt->i915; in engine_mask_apply_media_fuses()
760 if (MEDIA_VER(gt->i915) < 11) in engine_mask_apply_media_fuses()
768 media_fuse = intel_uncore_read(gt->uncore, GEN11_GT_VEBOX_VDBOX_DISABLE); in engine_mask_apply_media_fuses()
777 fuse1 = intel_uncore_read(gt->uncore, HSW_PAVP_FUSE1); in engine_mask_apply_media_fuses()
778 gt->info.sfc_mask = REG_FIELD_GET(XEHP_SFC_ENABLE_MASK, fuse1); in engine_mask_apply_media_fuses()
780 gt->info.sfc_mask = ~0; in engine_mask_apply_media_fuses()
790 gt->info.engine_mask &= ~BIT(_VCS(i)); in engine_mask_apply_media_fuses()
796 gt->info.vdbox_sfc_access |= BIT(i); in engine_mask_apply_media_fuses()
809 gt->info.engine_mask &= ~BIT(_VECS(i)); in engine_mask_apply_media_fuses()
819 struct drm_i915_private *i915 = gt->i915; in engine_mask_apply_compute_fuses()
820 struct intel_gt_info *info = >->info; in engine_mask_apply_compute_fuses()
821 int ss_per_ccs = info->sseu.max_subslices / I915_MAX_CCS; in engine_mask_apply_compute_fuses()
831 ccs_mask = intel_slicemask_from_xehp_dssmask(info->sseu.compute_subslice_mask, in engine_mask_apply_compute_fuses()
838 info->engine_mask &= ~BIT(_CCS(i)); in engine_mask_apply_compute_fuses()
845 * Note that we have a catch-22 situation where we need to be able to access
850 * calling this function and pruning the domains for fused-off engines
855 struct intel_gt_info *info = >->info; in init_engine_mask()
857 GEM_BUG_ON(!info->engine_mask); in init_engine_mask()
867 * catch-22 situation that breaks media C6 due to 2 requirements: in init_engine_mask()
874 if (__HAS_ENGINE(info->engine_mask, GSC0) && !intel_uc_wants_gsc_uc(>->uc)) { in init_engine_mask()
876 info->engine_mask &= ~BIT(GSC0); in init_engine_mask()
887 if (IS_DG2(gt->i915)) { in init_engine_mask()
894 gt->ccs.cslices = CCS_MASK(gt); in init_engine_mask()
897 info->engine_mask &= ~GENMASK(CCS3, CCS0); in init_engine_mask()
899 info->engine_mask |= BIT(_CCS(first_ccs)); in init_engine_mask()
902 return info->engine_mask; in init_engine_mask()
906 u8 class, const u8 *map, u8 num_instances) in populate_logical_ids() argument
914 intel_engines[i].class != class) in populate_logical_ids()
926 static void setup_logical_ids(struct intel_gt *gt, u8 *logical_ids, u8 class) in setup_logical_ids() argument
930 * to split-frame feature. in setup_logical_ids()
932 if (MEDIA_VER(gt->i915) >= 11 && class == VIDEO_DECODE_CLASS) { in setup_logical_ids()
935 populate_logical_ids(gt, logical_ids, class, in setup_logical_ids()
943 populate_logical_ids(gt, logical_ids, class, in setup_logical_ids()
949 * intel_engines_init_mmio() - allocate and prepare the Engine Command Streamers
952 * Return: non-zero if the initialization failed.
956 struct drm_i915_private *i915 = gt->i915; in intel_engines_init_mmio()
959 unsigned int i, class; in intel_engines_init_mmio() local
963 drm_WARN_ON(&i915->drm, engine_mask == 0); in intel_engines_init_mmio()
964 drm_WARN_ON(&i915->drm, engine_mask & in intel_engines_init_mmio()
965 GENMASK(BITS_PER_TYPE(mask) - 1, I915_NUM_ENGINES)); in intel_engines_init_mmio()
968 return -ENODEV; in intel_engines_init_mmio()
970 for (class = 0; class < MAX_ENGINE_CLASS + 1; ++class) { in intel_engines_init_mmio()
971 setup_logical_ids(gt, logical_ids, class); in intel_engines_init_mmio()
976 if (intel_engines[i].class != class || in intel_engines_init_mmio()
994 if (drm_WARN_ON(&i915->drm, mask != engine_mask)) in intel_engines_init_mmio()
995 gt->info.engine_mask = mask; in intel_engines_init_mmio()
997 gt->info.num_engines = hweight32(mask); in intel_engines_init_mmio()
1003 intel_uncore_prune_engine_fw_domains(gt->uncore, gt); in intel_engines_init_mmio()
1014 struct intel_engine_execlists * const execlists = &engine->execlists; in intel_engine_init_execlists()
1016 execlists->port_mask = 1; in intel_engine_init_execlists()
1020 memset(execlists->pending, 0, sizeof(execlists->pending)); in intel_engine_init_execlists()
1021 execlists->active = in intel_engine_init_execlists()
1022 memset(execlists->inflight, 0, sizeof(execlists->inflight)); in intel_engine_init_execlists()
1032 vma = fetch_and_zero(&engine->status_page.vma); in cleanup_status_page()
1036 if (!HWS_NEEDS_PHYSICAL(engine->i915)) in cleanup_status_page()
1039 i915_gem_object_unpin_map(vma->obj); in cleanup_status_page()
1040 i915_gem_object_put(vma->obj); in cleanup_status_page()
1049 if (!HAS_LLC(engine->i915) && i915_ggtt_has_aperture(engine->gt->ggtt)) in pin_ggtt_status_page()
1076 INIT_LIST_HEAD(&engine->status_page.timelines); in init_status_page()
1085 obj = i915_gem_object_create_internal(engine->i915, PAGE_SIZE); in init_status_page()
1087 gt_err(engine->gt, "Failed to allocate status page\n"); in init_status_page()
1093 vma = i915_vma_instance(obj, &engine->gt->ggtt->vm, NULL); in init_status_page()
1102 if (!ret && !HWS_NEEDS_PHYSICAL(engine->i915)) in init_status_page()
1113 engine->status_page.addr = memset(vaddr, 0, PAGE_SIZE); in init_status_page()
1114 engine->status_page.vma = vma; in init_status_page()
1120 if (ret == -EDEADLK) { in init_status_page()
1159 struct drm_i915_private *i915 = engine->i915; in intel_engine_init_tlb_invalidation()
1160 const unsigned int instance = engine->instance; in intel_engine_init_tlb_invalidation()
1161 const unsigned int class = engine->class; in intel_engine_init_tlb_invalidation() local
1168 * New platforms should not be added with catch-all-newer (>=) in intel_engine_init_tlb_invalidation()
1170 * and in turn mandates a human cross-check of whether the invalidation in intel_engine_init_tlb_invalidation()
1173 * For instance with the 11.00 -> 12.00 transition three out of five in intel_engine_init_tlb_invalidation()
1175 * 12.00 -> 12.50 transition multi cast handling is required too. in intel_engine_init_tlb_invalidation()
1178 if (engine->gt->type == GT_MEDIA) { in intel_engine_init_tlb_invalidation()
1202 if (gt_WARN_ONCE(engine->gt, !num, in intel_engine_init_tlb_invalidation()
1204 return -ENODEV; in intel_engine_init_tlb_invalidation()
1206 if (gt_WARN_ON_ONCE(engine->gt, in intel_engine_init_tlb_invalidation()
1207 class >= num || in intel_engine_init_tlb_invalidation()
1208 (!regs[class].reg.reg && in intel_engine_init_tlb_invalidation()
1209 !regs[class].mcr_reg.reg))) in intel_engine_init_tlb_invalidation()
1210 return -ERANGE; in intel_engine_init_tlb_invalidation()
1212 reg = regs[class]; in intel_engine_init_tlb_invalidation()
1214 if (regs == xelpmp_regs && class == OTHER_CLASS) { in intel_engine_init_tlb_invalidation()
1221 } else if (regs == gen8_regs && class == VIDEO_DECODE_CLASS && instance == 1) { in intel_engine_init_tlb_invalidation()
1230 engine->tlb_inv.mcr = regs == xehp_regs; in intel_engine_init_tlb_invalidation()
1231 engine->tlb_inv.reg = reg; in intel_engine_init_tlb_invalidation()
1232 engine->tlb_inv.done = val; in intel_engine_init_tlb_invalidation()
1235 (engine->class == VIDEO_DECODE_CLASS || in intel_engine_init_tlb_invalidation()
1236 engine->class == VIDEO_ENHANCEMENT_CLASS || in intel_engine_init_tlb_invalidation()
1237 engine->class == COMPUTE_CLASS || in intel_engine_init_tlb_invalidation()
1238 engine->class == OTHER_CLASS)) in intel_engine_init_tlb_invalidation()
1239 engine->tlb_inv.request = _MASKED_BIT_ENABLE(val); in intel_engine_init_tlb_invalidation()
1241 engine->tlb_inv.request = val; in intel_engine_init_tlb_invalidation()
1250 init_llist_head(&engine->barrier_tasks); in engine_setup_common()
1260 engine->breadcrumbs = intel_breadcrumbs_create(engine); in engine_setup_common()
1261 if (!engine->breadcrumbs) { in engine_setup_common()
1262 err = -ENOMEM; in engine_setup_common()
1266 engine->sched_engine = i915_sched_engine_create(ENGINE_PHYSICAL); in engine_setup_common()
1267 if (!engine->sched_engine) { in engine_setup_common()
1268 err = -ENOMEM; in engine_setup_common()
1271 engine->sched_engine->private_data = engine; in engine_setup_common()
1282 engine->sseu = in engine_setup_common()
1283 intel_sseu_from_device_info(&engine->gt->info.sseu); in engine_setup_common()
1289 if (GRAPHICS_VER(engine->i915) >= 12) in engine_setup_common()
1290 engine->flags |= I915_ENGINE_HAS_RELATIVE_MMIO; in engine_setup_common()
1295 i915_sched_engine_put(engine->sched_engine); in engine_setup_common()
1297 intel_breadcrumbs_put(engine->breadcrumbs); in engine_setup_common()
1311 struct intel_engine_cs *engine = ce->engine; in measure_breadcrumb_dw()
1315 GEM_BUG_ON(!engine->gt->scratch); in measure_breadcrumb_dw()
1319 return -ENOMEM; in measure_breadcrumb_dw()
1321 frame->rq.i915 = engine->i915; in measure_breadcrumb_dw()
1322 frame->rq.engine = engine; in measure_breadcrumb_dw()
1323 frame->rq.context = ce; in measure_breadcrumb_dw()
1324 rcu_assign_pointer(frame->rq.timeline, ce->timeline); in measure_breadcrumb_dw()
1325 frame->rq.hwsp_seqno = ce->timeline->hwsp_seqno; in measure_breadcrumb_dw()
1327 frame->ring.vaddr = frame->cs; in measure_breadcrumb_dw()
1328 frame->ring.size = sizeof(frame->cs); in measure_breadcrumb_dw()
1329 frame->ring.wrap = in measure_breadcrumb_dw()
1330 BITS_PER_TYPE(frame->ring.size) - ilog2(frame->ring.size); in measure_breadcrumb_dw()
1331 frame->ring.effective_size = frame->ring.size; in measure_breadcrumb_dw()
1332 intel_ring_update_space(&frame->ring); in measure_breadcrumb_dw()
1333 frame->rq.ring = &frame->ring; in measure_breadcrumb_dw()
1335 mutex_lock(&ce->timeline->mutex); in measure_breadcrumb_dw()
1336 spin_lock_irq(&engine->sched_engine->lock); in measure_breadcrumb_dw()
1338 dw = engine->emit_fini_breadcrumb(&frame->rq, frame->cs) - frame->cs; in measure_breadcrumb_dw()
1340 spin_unlock_irq(&engine->sched_engine->lock); in measure_breadcrumb_dw()
1341 mutex_unlock(&ce->timeline->mutex); in measure_breadcrumb_dw()
1364 __set_bit(CONTEXT_BARRIER_BIT, &ce->flags); in intel_engine_create_pinned_context()
1365 ce->timeline = page_pack_bits(NULL, hwsp); in intel_engine_create_pinned_context()
1366 ce->ring = NULL; in intel_engine_create_pinned_context()
1367 ce->ring_size = ring_size; in intel_engine_create_pinned_context()
1369 i915_vm_put(ce->vm); in intel_engine_create_pinned_context()
1370 ce->vm = i915_vm_get(vm); in intel_engine_create_pinned_context()
1372 err = intel_context_pin(ce); /* perma-pin so it is always available */ in intel_engine_create_pinned_context()
1378 list_add_tail(&ce->pinned_contexts_link, &engine->pinned_contexts_list); in intel_engine_create_pinned_context()
1381 * Give our perma-pinned kernel timelines a separate lockdep class, in intel_engine_create_pinned_context()
1386 lockdep_set_class_and_name(&ce->timeline->mutex, key, name); in intel_engine_create_pinned_context()
1393 struct intel_engine_cs *engine = ce->engine; in intel_engine_destroy_pinned_context()
1394 struct i915_vma *hwsp = engine->status_page.vma; in intel_engine_destroy_pinned_context()
1396 GEM_BUG_ON(ce->timeline->hwsp_ggtt != hwsp); in intel_engine_destroy_pinned_context()
1398 mutex_lock(&hwsp->vm->mutex); in intel_engine_destroy_pinned_context()
1399 list_del(&ce->timeline->engine_link); in intel_engine_destroy_pinned_context()
1400 mutex_unlock(&hwsp->vm->mutex); in intel_engine_destroy_pinned_context()
1402 list_del(&ce->pinned_contexts_link); in intel_engine_destroy_pinned_context()
1416 return intel_engine_create_pinned_context(engine, engine->gt->vm, SZ_512K, in create_ggtt_bind_context()
1426 return intel_engine_create_pinned_context(engine, engine->gt->vm, SZ_4K, in create_kernel_context()
1432 * engine_init_common - initialize engine state which might require hw access
1447 engine->set_default_submission(engine); in engine_init_common()
1466 if (i915_ggtt_require_binder(engine->i915) && engine->id == BCS0) { in engine_init_common()
1478 engine->emit_fini_breadcrumb_dw = ret; in engine_init_common()
1479 engine->kernel_context = ce; in engine_init_common()
1480 engine->bind_context = bce; in engine_init_common()
1499 if (intel_uc_uses_guc_submission(>->uc)) { in intel_engines_init()
1500 gt->submission_method = INTEL_SUBMISSION_GUC; in intel_engines_init()
1502 } else if (HAS_EXECLISTS(gt->i915)) { in intel_engines_init()
1503 gt->submission_method = INTEL_SUBMISSION_ELSP; in intel_engines_init()
1506 gt->submission_method = INTEL_SUBMISSION_RING; in intel_engines_init()
1522 GEM_BUG_ON(engine->release == NULL); in intel_engines_init()
1535 * intel_engine_cleanup_common - cleans up the engine state created by
1543 GEM_BUG_ON(!list_empty(&engine->sched_engine->requests)); in intel_engine_cleanup_common()
1545 i915_sched_engine_put(engine->sched_engine); in intel_engine_cleanup_common()
1546 intel_breadcrumbs_put(engine->breadcrumbs); in intel_engine_cleanup_common()
1551 if (engine->default_state) in intel_engine_cleanup_common()
1552 fput(engine->default_state); in intel_engine_cleanup_common()
1554 if (engine->kernel_context) in intel_engine_cleanup_common()
1555 intel_engine_destroy_pinned_context(engine->kernel_context); in intel_engine_cleanup_common()
1557 if (engine->bind_context) in intel_engine_cleanup_common()
1558 intel_engine_destroy_pinned_context(engine->bind_context); in intel_engine_cleanup_common()
1561 GEM_BUG_ON(!llist_empty(&engine->barrier_tasks)); in intel_engine_cleanup_common()
1564 intel_wa_list_free(&engine->ctx_wa_list); in intel_engine_cleanup_common()
1565 intel_wa_list_free(&engine->wa_list); in intel_engine_cleanup_common()
1566 intel_wa_list_free(&engine->whitelist); in intel_engine_cleanup_common()
1570 * intel_engine_resume - re-initializes the HW state of the engine
1580 return engine->resume(engine); in intel_engine_resume()
1585 struct drm_i915_private *i915 = engine->i915; in intel_engine_get_active_head()
1603 if (GRAPHICS_VER(engine->i915) >= 8) in intel_engine_get_last_batch_head()
1613 if (in_atomic() || irqs_disabled()) /* inside atomic preempt-reset? */ in stop_timeout()
1623 return READ_ONCE(engine->props.stop_timeout_ms); in stop_timeout()
1630 struct intel_uncore *uncore = engine->uncore; in __intel_engine_stop_cs()
1631 const i915_reg_t mode = RING_MI_MODE(engine->mmio_base); in __intel_engine_stop_cs()
1640 if (intel_engine_reset_needs_wa_22011802037(engine->gt)) in __intel_engine_stop_cs()
1641 intel_uncore_write_fw(uncore, RING_MODE_GEN7(engine->mmio_base), in __intel_engine_stop_cs()
1644 err = __intel_wait_for_register_fw(engine->uncore, mode, in __intel_engine_stop_cs()
1659 if (GRAPHICS_VER(engine->i915) < 3) in intel_engine_stop_cs()
1660 return -ENODEV; in intel_engine_stop_cs()
1677 "timed out on STOP_RING -> IDLE; HEAD:%04x, TAIL:%04x\n", in intel_engine_stop_cs()
1688 err = -ETIMEDOUT; in intel_engine_stop_cs()
1725 if (!_reg[engine->id].reg) in __cs_pending_mi_force_wakes()
1728 val = intel_uncore_read(engine->uncore, _reg[engine->id]); in __cs_pending_mi_force_wakes()
1742 ret = __intel_wait_for_register_fw(gt->uncore, in __gpm_wait_for_fw_complete()
1767 __gpm_wait_for_fw_complete(engine->gt, fw_pending); in intel_engine_wait_for_pending_mi_fw()
1774 struct drm_i915_private *i915 = engine->i915; in intel_engine_get_instdone()
1775 struct intel_uncore *uncore = engine->uncore; in intel_engine_get_instdone()
1776 u32 mmio_base = engine->mmio_base; in intel_engine_get_instdone()
1784 instdone->instdone = in intel_engine_get_instdone()
1787 if (engine->id != RCS0) in intel_engine_get_instdone()
1790 instdone->slice_common = in intel_engine_get_instdone()
1793 instdone->slice_common_extra[0] = in intel_engine_get_instdone()
1795 instdone->slice_common_extra[1] = in intel_engine_get_instdone()
1799 for_each_ss_steering(iter, engine->gt, slice, subslice) { in intel_engine_get_instdone()
1800 instdone->sampler[slice][subslice] = in intel_engine_get_instdone()
1801 intel_gt_mcr_read(engine->gt, in intel_engine_get_instdone()
1804 instdone->row[slice][subslice] = in intel_engine_get_instdone()
1805 intel_gt_mcr_read(engine->gt, in intel_engine_get_instdone()
1811 for_each_ss_steering(iter, engine->gt, slice, subslice) in intel_engine_get_instdone()
1812 instdone->geom_svg[slice][subslice] = in intel_engine_get_instdone()
1813 intel_gt_mcr_read(engine->gt, in intel_engine_get_instdone()
1818 instdone->instdone = in intel_engine_get_instdone()
1821 if (engine->id != RCS0) in intel_engine_get_instdone()
1824 instdone->slice_common = in intel_engine_get_instdone()
1826 instdone->sampler[0][0] = in intel_engine_get_instdone()
1828 instdone->row[0][0] = in intel_engine_get_instdone()
1831 instdone->instdone = in intel_engine_get_instdone()
1833 if (engine->id == RCS0) in intel_engine_get_instdone()
1835 instdone->slice_common = in intel_engine_get_instdone()
1838 instdone->instdone = intel_uncore_read(uncore, GEN2_INSTDONE); in intel_engine_get_instdone()
1846 if (I915_SELFTEST_ONLY(!engine->mmio_base)) in ring_is_idle()
1858 if (GRAPHICS_VER(engine->i915) > 2 && in ring_is_idle()
1869 struct tasklet_struct *t = &engine->sched_engine->tasklet; in __intel_engine_flush_submission()
1871 if (!t->callback) in __intel_engine_flush_submission()
1878 t->callback(t); in __intel_engine_flush_submission()
1889 * intel_engine_is_idle() - Report if the engine has finished process all work
1898 if (intel_gt_is_wedged(engine->gt)) in intel_engine_is_idle()
1905 intel_synchronize_hardirq(engine->i915); in intel_engine_is_idle()
1909 if (!i915_sched_engine_is_empty(engine->sched_engine)) in intel_engine_is_idle()
1929 if (!READ_ONCE(gt->awake)) in intel_engines_are_idle()
1942 if (!engine->irq_enable) in intel_engine_irq_enable()
1946 spin_lock(engine->gt->irq_lock); in intel_engine_irq_enable()
1947 engine->irq_enable(engine); in intel_engine_irq_enable()
1948 spin_unlock(engine->gt->irq_lock); in intel_engine_irq_enable()
1955 if (!engine->irq_disable) in intel_engine_irq_disable()
1959 spin_lock(engine->gt->irq_lock); in intel_engine_irq_disable()
1960 engine->irq_disable(engine); in intel_engine_irq_disable()
1961 spin_unlock(engine->gt->irq_lock); in intel_engine_irq_disable()
1970 if (engine->sanitize) in intel_engines_reset_default_submission()
1971 engine->sanitize(engine); in intel_engines_reset_default_submission()
1973 engine->set_default_submission(engine); in intel_engines_reset_default_submission()
1979 switch (GRAPHICS_VER(engine->i915)) { in intel_engine_can_store_dword()
1984 return !(IS_I915G(engine->i915) || IS_I915GM(engine->i915)); in intel_engine_can_store_dword()
1986 return !IS_I965G(engine->i915); /* who knows! */ in intel_engine_can_store_dword()
1988 return engine->class != VIDEO_DECODE_CLASS; /* b0rked */ in intel_engine_can_store_dword()
1999 * Even though we are holding the engine->sched_engine->lock here, there in get_timeline()
2000 * is no control over the submission queue per-se and we are in get_timeline()
2007 tl = rcu_dereference(rq->timeline); in get_timeline()
2008 if (!kref_get_unless_zero(&tl->kref)) in get_timeline()
2024 i915_ggtt_offset(rq->ring->vma), in print_ring()
2025 tl ? tl->hwsp_offset : 0, in print_ring()
2027 DIV_ROUND_CLOSEST_ULL(intel_context_get_total_runtime_ns(rq->context), in print_ring()
2037 static void hexdump(struct drm_printer *m, const void *buf, size_t len) in hexdump() argument
2049 drm_printf(m, "*\n"); in hexdump()
2055 WARN_ON_ONCE(hex_dump_to_buffer(buf + pos, len - pos, in hexdump()
2059 drm_printf(m, "[%04zx] %s\n", pos, line); in hexdump()
2068 if (!READ_ONCE(t->expires)) in repr_timer()
2078 struct drm_printer *m) in intel_engine_print_registers() argument
2080 struct drm_i915_private *i915 = engine->i915; in intel_engine_print_registers()
2081 struct intel_engine_execlists * const execlists = &engine->execlists; in intel_engine_print_registers()
2084 if (engine->id == RENDER_CLASS && IS_GRAPHICS_VER(i915, 4, 7)) in intel_engine_print_registers()
2085 drm_printf(m, "\tCCID: 0x%08x\n", ENGINE_READ(engine, CCID)); in intel_engine_print_registers()
2087 drm_printf(m, "\tEL_STAT_HI: 0x%08x\n", in intel_engine_print_registers()
2089 drm_printf(m, "\tEL_STAT_LO: 0x%08x\n", in intel_engine_print_registers()
2092 drm_printf(m, "\tRING_START: 0x%08x\n", in intel_engine_print_registers()
2094 drm_printf(m, "\tRING_HEAD: 0x%08x\n", in intel_engine_print_registers()
2096 drm_printf(m, "\tRING_TAIL: 0x%08x\n", in intel_engine_print_registers()
2098 drm_printf(m, "\tRING_CTL: 0x%08x%s\n", in intel_engine_print_registers()
2101 if (GRAPHICS_VER(engine->i915) > 2) { in intel_engine_print_registers()
2102 drm_printf(m, "\tRING_MODE: 0x%08x%s\n", in intel_engine_print_registers()
2108 drm_printf(m, "\tRING_IMR: 0x%08x\n", in intel_engine_print_registers()
2110 drm_printf(m, "\tRING_ESR: 0x%08x\n", in intel_engine_print_registers()
2112 drm_printf(m, "\tRING_EMR: 0x%08x\n", in intel_engine_print_registers()
2114 drm_printf(m, "\tRING_EIR: 0x%08x\n", in intel_engine_print_registers()
2119 drm_printf(m, "\tACTHD: 0x%08x_%08x\n", in intel_engine_print_registers()
2122 drm_printf(m, "\tBBADDR: 0x%08x_%08x\n", in intel_engine_print_registers()
2130 drm_printf(m, "\tDMA_FADDR: 0x%08x_%08x\n", in intel_engine_print_registers()
2133 drm_printf(m, "\tIPEIR: 0x%08x\n", in intel_engine_print_registers()
2135 drm_printf(m, "\tIPEHR: 0x%08x\n", in intel_engine_print_registers()
2138 drm_printf(m, "\tIPEIR: 0x%08x\n", ENGINE_READ(engine, IPEIR)); in intel_engine_print_registers()
2139 drm_printf(m, "\tIPEHR: 0x%08x\n", ENGINE_READ(engine, IPEHR)); in intel_engine_print_registers()
2145 &engine->status_page.addr[I915_HWS_CSB_BUF0_INDEX]; in intel_engine_print_registers()
2146 const u8 num_entries = execlists->csb_size; in intel_engine_print_registers()
2150 drm_printf(m, "\tExeclist tasklet queued? %s (%s), preempt? %s, timeslice? %s\n", in intel_engine_print_registers()
2151 str_yes_no(test_bit(TASKLET_STATE_SCHED, &engine->sched_engine->tasklet.state)), in intel_engine_print_registers()
2152 str_enabled_disabled(!atomic_read(&engine->sched_engine->tasklet.count)), in intel_engine_print_registers()
2153 repr_timer(&engine->execlists.preempt), in intel_engine_print_registers()
2154 repr_timer(&engine->execlists.timer)); in intel_engine_print_registers()
2156 read = execlists->csb_head; in intel_engine_print_registers()
2157 write = READ_ONCE(*execlists->csb_write); in intel_engine_print_registers()
2159 drm_printf(m, "\tExeclist status: 0x%08x %08x; CSB read:%d, write:%d, entries:%d\n", in intel_engine_print_registers()
2172 drm_printf(m, "\tExeclist CSB[%d]: 0x%08x, context: %d\n", in intel_engine_print_registers()
2176 i915_sched_engine_active_lock_bh(engine->sched_engine); in intel_engine_print_registers()
2178 for (port = execlists->active; (rq = *port); port++) { in intel_engine_print_registers()
2184 (int)(port - execlists->active), in intel_engine_print_registers()
2185 rq->context->lrc.ccid, in intel_engine_print_registers()
2186 intel_context_is_closed(rq->context) ? "!" : "", in intel_engine_print_registers()
2187 intel_context_is_banned(rq->context) ? "*" : ""); in intel_engine_print_registers()
2188 len += print_ring(hdr + len, sizeof(hdr) - len, rq); in intel_engine_print_registers()
2189 scnprintf(hdr + len, sizeof(hdr) - len, "rq: "); in intel_engine_print_registers()
2190 i915_request_show(m, rq, hdr, 0); in intel_engine_print_registers()
2192 for (port = execlists->pending; (rq = *port); port++) { in intel_engine_print_registers()
2198 (int)(port - execlists->pending), in intel_engine_print_registers()
2199 rq->context->lrc.ccid, in intel_engine_print_registers()
2200 intel_context_is_closed(rq->context) ? "!" : "", in intel_engine_print_registers()
2201 intel_context_is_banned(rq->context) ? "*" : ""); in intel_engine_print_registers()
2202 len += print_ring(hdr + len, sizeof(hdr) - len, rq); in intel_engine_print_registers()
2203 scnprintf(hdr + len, sizeof(hdr) - len, "rq: "); in intel_engine_print_registers()
2204 i915_request_show(m, rq, hdr, 0); in intel_engine_print_registers()
2207 i915_sched_engine_active_unlock_bh(engine->sched_engine); in intel_engine_print_registers()
2209 drm_printf(m, "\tPP_DIR_BASE: 0x%08x\n", in intel_engine_print_registers()
2211 drm_printf(m, "\tPP_DIR_BASE_READ: 0x%08x\n", in intel_engine_print_registers()
2213 drm_printf(m, "\tPP_DIR_DCLV: 0x%08x\n", in intel_engine_print_registers()
2218 static void print_request_ring(struct drm_printer *m, struct i915_request *rq) in print_request_ring() argument
2220 struct i915_vma_resource *vma_res = rq->batch_res; in print_request_ring()
2224 drm_printf(m, in print_request_ring()
2226 rq->head, rq->postfix, rq->tail, in print_request_ring()
2227 vma_res ? upper_32_bits(vma_res->start) : ~0u, in print_request_ring()
2228 vma_res ? lower_32_bits(vma_res->start) : ~0u); in print_request_ring()
2230 size = rq->tail - rq->head; in print_request_ring()
2231 if (rq->tail < rq->head) in print_request_ring()
2232 size += rq->ring->size; in print_request_ring()
2236 const void *vaddr = rq->ring->vaddr; in print_request_ring()
2237 unsigned int head = rq->head; in print_request_ring()
2240 if (rq->tail < head) { in print_request_ring()
2241 len = rq->ring->size - head; in print_request_ring()
2245 memcpy(ring + len, vaddr + head, size - len); in print_request_ring()
2247 hexdump(m, ring, size); in print_request_ring()
2258 struct drm_printer *m) in print_properties() argument
2265 .offset = offsetof(typeof(engine->props), x), \ in print_properties()
2279 drm_printf(m, "\tProperties:\n"); in print_properties()
2280 for (p = props; p->name; p++) in print_properties()
2281 drm_printf(m, "\t\t%s: %lu [default %lu]\n", in print_properties()
2282 p->name, in print_properties()
2283 read_ul(&engine->props, p->offset), in print_properties()
2284 read_ul(&engine->defaults, p->offset)); in print_properties()
2287 static void engine_dump_request(struct i915_request *rq, struct drm_printer *m, const char *msg) in engine_dump_request() argument
2291 i915_request_show(m, rq, msg, 0); in engine_dump_request()
2293 drm_printf(m, "\t\tring->start: 0x%08x\n", in engine_dump_request()
2294 i915_ggtt_offset(rq->ring->vma)); in engine_dump_request()
2295 drm_printf(m, "\t\tring->head: 0x%08x\n", in engine_dump_request()
2296 rq->ring->head); in engine_dump_request()
2297 drm_printf(m, "\t\tring->tail: 0x%08x\n", in engine_dump_request()
2298 rq->ring->tail); in engine_dump_request()
2299 drm_printf(m, "\t\tring->emit: 0x%08x\n", in engine_dump_request()
2300 rq->ring->emit); in engine_dump_request()
2301 drm_printf(m, "\t\tring->space: 0x%08x\n", in engine_dump_request()
2302 rq->ring->space); in engine_dump_request()
2305 drm_printf(m, "\t\tring->hwsp: 0x%08x\n", in engine_dump_request()
2306 tl->hwsp_offset); in engine_dump_request()
2310 print_request_ring(m, rq); in engine_dump_request()
2312 if (rq->context->lrc_reg_state) { in engine_dump_request()
2313 drm_printf(m, "Logical Ring Context:\n"); in engine_dump_request()
2314 hexdump(m, rq->context->lrc_reg_state, PAGE_SIZE); in engine_dump_request()
2320 struct drm_printer *m) in intel_engine_dump_active_requests() argument
2339 engine_dump_request(rq, m, msg); in intel_engine_dump_active_requests()
2344 struct drm_printer *m) in engine_dump_active_requests() argument
2350 * No need for an engine->irq_seqno_barrier() before the seqno reads. in engine_dump_active_requests()
2358 drm_printf(m, "\tRequests:\n"); in engine_dump_active_requests()
2361 engine_dump_request(hung_rq, m, "\t\thung"); in engine_dump_active_requests()
2363 drm_printf(m, "\t\tGot hung ce but no hung rq!\n"); in engine_dump_active_requests()
2365 if (intel_uc_uses_guc_submission(&engine->gt->uc)) in engine_dump_active_requests()
2366 intel_guc_dump_active_requests(engine, hung_rq, m); in engine_dump_active_requests()
2368 intel_execlists_dump_active_requests(engine, hung_rq, m); in engine_dump_active_requests()
2375 struct drm_printer *m, in intel_engine_dump() argument
2378 struct i915_gpu_error * const error = &engine->i915->gpu_error; in intel_engine_dump()
2387 drm_vprintf(m, header, &ap); in intel_engine_dump()
2391 if (intel_gt_is_wedged(engine->gt)) in intel_engine_dump()
2392 drm_printf(m, "*** WEDGED ***\n"); in intel_engine_dump()
2394 drm_printf(m, "\tAwake? %d\n", atomic_read(&engine->wakeref.count)); in intel_engine_dump()
2395 drm_printf(m, "\tBarriers?: %s\n", in intel_engine_dump()
2396 str_yes_no(!llist_empty(&engine->barrier_tasks))); in intel_engine_dump()
2397 drm_printf(m, "\tLatency: %luus\n", in intel_engine_dump()
2398 ewma__engine_latency_read(&engine->latency)); in intel_engine_dump()
2400 drm_printf(m, "\tRuntime: %llums\n", in intel_engine_dump()
2403 drm_printf(m, "\tForcewake: %x domains, %d active\n", in intel_engine_dump()
2404 engine->fw_domain, READ_ONCE(engine->fw_active)); in intel_engine_dump()
2407 rq = READ_ONCE(engine->heartbeat.systole); in intel_engine_dump()
2409 drm_printf(m, "\tHeartbeat: %d ms ago\n", in intel_engine_dump()
2410 jiffies_to_msecs(jiffies - rq->emitted_jiffies)); in intel_engine_dump()
2412 drm_printf(m, "\tReset count: %d (global %d)\n", in intel_engine_dump()
2415 print_properties(engine, m); in intel_engine_dump()
2417 engine_dump_active_requests(engine, m); in intel_engine_dump()
2419 drm_printf(m, "\tMMIO base: 0x%08x\n", engine->mmio_base); in intel_engine_dump()
2420 wakeref = intel_runtime_pm_get_if_in_use(engine->uncore->rpm); in intel_engine_dump()
2422 intel_engine_print_registers(engine, m); in intel_engine_dump()
2423 intel_runtime_pm_put(engine->uncore->rpm, wakeref); in intel_engine_dump()
2425 drm_printf(m, "\tDevice is asleep; skipping register dump\n"); in intel_engine_dump()
2428 intel_execlists_show_requests(engine, m, i915_request_show, 8); in intel_engine_dump()
2430 drm_printf(m, "HWSP:\n"); in intel_engine_dump()
2431 hexdump(m, engine->status_page.addr, PAGE_SIZE); in intel_engine_dump()
2433 drm_printf(m, "Idle? %s\n", str_yes_no(intel_engine_is_idle(engine))); in intel_engine_dump()
2435 intel_engine_print_breadcrumbs(engine, m); in intel_engine_dump()
2439 * intel_engine_get_busy_time() - Return current accumulated engine busyness
2447 return engine->busyness(engine, now); in intel_engine_get_busy_time()
2455 return ERR_PTR(-EINVAL); in intel_engine_create_virtual()
2460 GEM_BUG_ON(!siblings[0]->cops->create_virtual); in intel_engine_create_virtual()
2461 return siblings[0]->cops->create_virtual(siblings, count, flags); in intel_engine_create_virtual()
2473 GEM_BUG_ON(intel_uc_uses_guc_submission(&engine->gt->uc)); in engine_execlist_find_hung_request()
2480 * for all writes that were in transaction to be flushed) - adding an in engine_execlist_find_hung_request()
2482 * not need an engine->irq_seqno_barrier() before the seqno reads. in engine_execlist_find_hung_request()
2486 lockdep_assert_held(&engine->sched_engine->lock); in engine_execlist_find_hung_request()
2489 request = execlists_active(&engine->execlists); in engine_execlist_find_hung_request()
2491 struct intel_timeline *tl = request->context->timeline; in engine_execlist_find_hung_request()
2493 list_for_each_entry_from_reverse(request, &tl->requests, link) { in engine_execlist_find_hung_request()
2504 list_for_each_entry(request, &engine->sched_engine->requests, in engine_execlist_find_hung_request()
2533 if (intel_uc_uses_guc_submission(&engine->gt->uc)) in intel_engine_get_hung_entity()
2536 spin_lock_irqsave(&engine->sched_engine->lock, flags); in intel_engine_get_hung_entity()
2540 spin_unlock_irqrestore(&engine->sched_engine->lock, flags); in intel_engine_get_hung_entity()
2546 * If there are any non-fused-off CCS engines, we need to enable CCS in xehp_enable_ccs_engines()
2551 * of re-applying the setting after i915-triggered resets. in xehp_enable_ccs_engines()
2553 if (!CCS_MASK(engine->gt)) in xehp_enable_ccs_engines()
2556 intel_uncore_write(engine->uncore, GEN12_RCU_MODE, in xehp_enable_ccs_engines()