Lines Matching full:gt
65 struct xe_gt *gt = arg; in gt_fini() local
67 destroy_workqueue(gt->ordered_wq); in gt_fini()
72 struct xe_gt *gt; in xe_gt_alloc() local
75 gt = drmm_kzalloc(&tile_to_xe(tile)->drm, sizeof(*gt), GFP_KERNEL); in xe_gt_alloc()
76 if (!gt) in xe_gt_alloc()
79 gt->tile = tile; in xe_gt_alloc()
80 gt->ordered_wq = alloc_ordered_workqueue("gt-ordered-wq", 0); in xe_gt_alloc()
82 err = drmm_add_action_or_reset(>_to_xe(gt)->drm, gt_fini, gt); in xe_gt_alloc()
86 return gt; in xe_gt_alloc()
89 void xe_gt_sanitize(struct xe_gt *gt) in xe_gt_sanitize() argument
95 gt->uc.guc.submission_state.enabled = false; in xe_gt_sanitize()
98 static void xe_gt_enable_host_l2_vram(struct xe_gt *gt) in xe_gt_enable_host_l2_vram() argument
103 if (!XE_WA(gt, 16023588340)) in xe_gt_enable_host_l2_vram()
106 err = xe_force_wake_get(gt_to_fw(gt), XE_FW_GT); in xe_gt_enable_host_l2_vram()
110 if (!xe_gt_is_media_type(gt)) { in xe_gt_enable_host_l2_vram()
111 reg = xe_gt_mcr_unicast_read_any(gt, XE2_GAMREQSTRM_CTRL); in xe_gt_enable_host_l2_vram()
113 xe_gt_mcr_multicast_write(gt, XE2_GAMREQSTRM_CTRL, reg); in xe_gt_enable_host_l2_vram()
116 xe_gt_mcr_multicast_write(gt, XEHPC_L3CLOS_MASK(3), 0x3); in xe_gt_enable_host_l2_vram()
117 xe_force_wake_put(gt_to_fw(gt), XE_FW_GT); in xe_gt_enable_host_l2_vram()
120 static void xe_gt_disable_host_l2_vram(struct xe_gt *gt) in xe_gt_disable_host_l2_vram() argument
125 if (!XE_WA(gt, 16023588340)) in xe_gt_disable_host_l2_vram()
128 if (xe_gt_is_media_type(gt)) in xe_gt_disable_host_l2_vram()
131 err = xe_force_wake_get(gt_to_fw(gt), XE_FW_GT); in xe_gt_disable_host_l2_vram()
135 reg = xe_gt_mcr_unicast_read_any(gt, XE2_GAMREQSTRM_CTRL); in xe_gt_disable_host_l2_vram()
137 xe_gt_mcr_multicast_write(gt, XE2_GAMREQSTRM_CTRL, reg); in xe_gt_disable_host_l2_vram()
139 xe_force_wake_put(gt_to_fw(gt), XE_FW_GT); in xe_gt_disable_host_l2_vram()
143 * xe_gt_remove() - Clean up the GT structures before driver removal
144 * @gt: the GT object
150 void xe_gt_remove(struct xe_gt *gt) in xe_gt_remove() argument
154 xe_uc_remove(>->uc); in xe_gt_remove()
157 xe_hw_fence_irq_finish(>->fence_irq[i]); in xe_gt_remove()
159 xe_gt_disable_host_l2_vram(gt); in xe_gt_remove()
164 static int emit_nop_job(struct xe_gt *gt, struct xe_exec_queue *q) in emit_nop_job() argument
171 bb = xe_bb_new(gt, 4, false); in emit_nop_job()
205 static int emit_wa_job(struct xe_gt *gt, struct xe_exec_queue *q) in emit_wa_job() argument
218 bb = xe_bb_new(gt, xe_gt_lrc_size(gt, q->hwe->class), false); in emit_wa_job()
221 bb = xe_bb_new(gt, SZ_4K, false); in emit_wa_job()
230 xe_gt_dbg(gt, "LRC WA %s save-restore batch\n", sr->name); in emit_wa_job()
246 xe_gt_mcr_unicast_read_any(gt, reg_mcr) : in emit_wa_job()
247 xe_mmio_read32(gt, reg)) & (~entry->clr_bits); in emit_wa_job()
255 xe_gt_dbg(gt, "REG[0x%x] = 0x%08x", reg.addr, val); in emit_wa_job()
282 int xe_gt_record_default_lrcs(struct xe_gt *gt) in xe_gt_record_default_lrcs() argument
284 struct xe_device *xe = gt_to_xe(gt); in xe_gt_record_default_lrcs()
289 for_each_hw_engine(hwe, gt, id) { in xe_gt_record_default_lrcs()
293 if (gt->default_lrc[hwe->class]) in xe_gt_record_default_lrcs()
302 xe_gt_lrc_size(gt, hwe->class), in xe_gt_record_default_lrcs()
311 xe_gt_err(gt, "hwe %s: xe_exec_queue_create failed (%pe)\n", in xe_gt_record_default_lrcs()
317 err = emit_wa_job(gt, q); in xe_gt_record_default_lrcs()
319 xe_gt_err(gt, "hwe %s: emit_wa_job failed (%pe) guc_id=%u\n", in xe_gt_record_default_lrcs()
328 xe_gt_err(gt, "hwe %s: nop xe_exec_queue_create failed (%pe)\n", in xe_gt_record_default_lrcs()
334 err = emit_nop_job(gt, nop_q); in xe_gt_record_default_lrcs()
336 xe_gt_err(gt, "hwe %s: nop emit_nop_job failed (%pe) guc_id=%u\n", in xe_gt_record_default_lrcs()
342 err = emit_nop_job(gt, q); in xe_gt_record_default_lrcs()
344 xe_gt_err(gt, "hwe %s: emit_nop_job failed (%pe) guc_id=%u\n", in xe_gt_record_default_lrcs()
352 xe_gt_lrc_size(gt, hwe->class)); in xe_gt_record_default_lrcs()
354 gt->default_lrc[hwe->class] = default_lrc; in xe_gt_record_default_lrcs()
366 int xe_gt_init_early(struct xe_gt *gt) in xe_gt_init_early() argument
370 if (IS_SRIOV_PF(gt_to_xe(gt))) { in xe_gt_init_early()
371 err = xe_gt_sriov_pf_init_early(gt); in xe_gt_init_early()
376 xe_reg_sr_init(>->reg_sr, "GT", gt_to_xe(gt)); in xe_gt_init_early()
378 err = xe_wa_init(gt); in xe_gt_init_early()
382 xe_wa_process_gt(gt); in xe_gt_init_early()
383 xe_wa_process_oob(gt); in xe_gt_init_early()
384 xe_tuning_process_gt(gt); in xe_gt_init_early()
386 xe_force_wake_init_gt(gt, gt_to_fw(gt)); in xe_gt_init_early()
387 spin_lock_init(>->global_invl_lock); in xe_gt_init_early()
392 static void dump_pat_on_error(struct xe_gt *gt) in dump_pat_on_error() argument
397 snprintf(prefix, sizeof(prefix), "[GT%u Error]", gt->info.id); in dump_pat_on_error()
398 p = drm_dbg_printer(>_to_xe(gt)->drm, DRM_UT_DRIVER, prefix); in dump_pat_on_error()
400 xe_pat_dump(gt, &p); in dump_pat_on_error()
403 static int gt_fw_domain_init(struct xe_gt *gt) in gt_fw_domain_init() argument
407 err = xe_force_wake_get(gt_to_fw(gt), XE_FW_GT); in gt_fw_domain_init()
411 if (!xe_gt_is_media_type(gt)) { in gt_fw_domain_init()
412 err = xe_ggtt_init(gt_to_tile(gt)->mem.ggtt); in gt_fw_domain_init()
415 if (IS_SRIOV_PF(gt_to_xe(gt))) in gt_fw_domain_init()
416 xe_lmtt_init(>_to_tile(gt)->sriov.pf.lmtt); in gt_fw_domain_init()
420 xe_irq_enable_hwe(gt); in gt_fw_domain_init()
423 xe_gt_mcr_init(gt); in gt_fw_domain_init()
425 err = xe_hw_engines_init_early(gt); in gt_fw_domain_init()
429 err = xe_hw_engine_class_sysfs_init(gt); in gt_fw_domain_init()
434 err = xe_gt_ccs_mode_sysfs_init(gt); in gt_fw_domain_init()
442 gt->info.gmdid = xe_mmio_read32(gt, GMD_ID); in gt_fw_domain_init()
444 err = xe_force_wake_put(gt_to_fw(gt), XE_FW_GT); in gt_fw_domain_init()
450 dump_pat_on_error(gt); in gt_fw_domain_init()
451 xe_force_wake_put(gt_to_fw(gt), XE_FW_GT); in gt_fw_domain_init()
454 xe_hw_fence_irq_finish(>->fence_irq[i]); in gt_fw_domain_init()
459 static int all_fw_domain_init(struct xe_gt *gt) in all_fw_domain_init() argument
463 err = xe_force_wake_get(gt_to_fw(gt), XE_FORCEWAKE_ALL); in all_fw_domain_init()
467 xe_gt_mcr_set_implicit_defaults(gt); in all_fw_domain_init()
468 xe_reg_sr_apply_mmio(>->reg_sr, gt); in all_fw_domain_init()
470 err = xe_gt_clock_init(gt); in all_fw_domain_init()
474 xe_mocs_init(gt); in all_fw_domain_init()
475 err = xe_execlist_init(gt); in all_fw_domain_init()
479 err = xe_hw_engines_init(gt); in all_fw_domain_init()
483 err = xe_uc_init_post_hwconfig(>->uc); in all_fw_domain_init()
487 if (!xe_gt_is_media_type(gt)) { in all_fw_domain_init()
491 if (gt_to_xe(gt)->info.has_usm) { in all_fw_domain_init()
492 struct xe_device *xe = gt_to_xe(gt); in all_fw_domain_init()
494 gt->usm.bb_pool = xe_sa_bo_manager_init(gt_to_tile(gt), in all_fw_domain_init()
496 if (IS_ERR(gt->usm.bb_pool)) { in all_fw_domain_init()
497 err = PTR_ERR(gt->usm.bb_pool); in all_fw_domain_init()
503 if (!xe_gt_is_media_type(gt)) { in all_fw_domain_init()
504 struct xe_tile *tile = gt_to_tile(gt); in all_fw_domain_init()
513 err = xe_uc_init_hw(>->uc); in all_fw_domain_init()
518 if (xe_gt_ccs_mode_enabled(gt)) { in all_fw_domain_init()
519 gt->ccs_mode = 1; in all_fw_domain_init()
520 xe_gt_apply_ccs_mode(gt); in all_fw_domain_init()
523 if (IS_SRIOV_PF(gt_to_xe(gt)) && !xe_gt_is_media_type(gt)) in all_fw_domain_init()
524 xe_lmtt_init_hw(>_to_tile(gt)->sriov.pf.lmtt); in all_fw_domain_init()
526 if (IS_SRIOV_PF(gt_to_xe(gt))) in all_fw_domain_init()
527 xe_gt_sriov_pf_init_hw(gt); in all_fw_domain_init()
529 err = xe_force_wake_put(gt_to_fw(gt), XE_FORCEWAKE_ALL); in all_fw_domain_init()
535 xe_force_wake_put(gt_to_fw(gt), XE_FORCEWAKE_ALL); in all_fw_domain_init()
538 xe_hw_fence_irq_finish(>->fence_irq[i]); in all_fw_domain_init()
544 * Initialize enough GT to be able to load GuC in order to obtain hwconfig and
547 int xe_gt_init_hwconfig(struct xe_gt *gt) in xe_gt_init_hwconfig() argument
551 err = xe_force_wake_get(gt_to_fw(gt), XE_FW_GT); in xe_gt_init_hwconfig()
555 xe_gt_mcr_init_early(gt); in xe_gt_init_hwconfig()
556 xe_pat_init(gt); in xe_gt_init_hwconfig()
558 err = xe_uc_init(>->uc); in xe_gt_init_hwconfig()
562 err = xe_uc_init_hwconfig(>->uc); in xe_gt_init_hwconfig()
566 xe_gt_topology_init(gt); in xe_gt_init_hwconfig()
567 xe_gt_mcr_init(gt); in xe_gt_init_hwconfig()
568 xe_gt_enable_host_l2_vram(gt); in xe_gt_init_hwconfig()
571 xe_force_wake_put(gt_to_fw(gt), XE_FW_GT); in xe_gt_init_hwconfig()
576 int xe_gt_init(struct xe_gt *gt) in xe_gt_init() argument
581 INIT_WORK(>->reset.worker, gt_reset_worker); in xe_gt_init()
584 gt->ring_ops[i] = xe_ring_ops_get(gt, i); in xe_gt_init()
585 xe_hw_fence_irq_init(>->fence_irq[i]); in xe_gt_init()
588 err = xe_gt_tlb_invalidation_init(gt); in xe_gt_init()
592 err = xe_gt_pagefault_init(gt); in xe_gt_init()
596 xe_mocs_init_early(gt); in xe_gt_init()
598 err = xe_gt_sysfs_init(gt); in xe_gt_init()
602 err = gt_fw_domain_init(gt); in xe_gt_init()
606 err = xe_gt_idle_init(>->gtidle); in xe_gt_init()
610 err = xe_gt_freq_init(gt); in xe_gt_init()
614 xe_force_wake_init_engines(gt, gt_to_fw(gt)); in xe_gt_init()
616 err = all_fw_domain_init(gt); in xe_gt_init()
620 xe_gt_record_user_engines(gt); in xe_gt_init()
625 void xe_gt_record_user_engines(struct xe_gt *gt) in xe_gt_record_user_engines() argument
630 gt->user_engines.mask = 0; in xe_gt_record_user_engines()
631 memset(gt->user_engines.instances_per_class, 0, in xe_gt_record_user_engines()
632 sizeof(gt->user_engines.instances_per_class)); in xe_gt_record_user_engines()
634 for_each_hw_engine(hwe, gt, id) { in xe_gt_record_user_engines()
638 gt->user_engines.mask |= BIT_ULL(id); in xe_gt_record_user_engines()
639 gt->user_engines.instances_per_class[hwe->class]++; in xe_gt_record_user_engines()
642 xe_gt_assert(gt, (gt->user_engines.mask | gt->info.engine_mask) in xe_gt_record_user_engines()
643 == gt->info.engine_mask); in xe_gt_record_user_engines()
646 static int do_gt_reset(struct xe_gt *gt) in do_gt_reset() argument
650 xe_gsc_wa_14015076503(gt, true); in do_gt_reset()
652 xe_mmio_write32(gt, GDRST, GRDOM_FULL); in do_gt_reset()
653 err = xe_mmio_wait32(gt, GDRST, GRDOM_FULL, 0, 5000, NULL, false); in do_gt_reset()
655 xe_gt_err(gt, "failed to clear GRDOM_FULL (%pe)\n", in do_gt_reset()
658 xe_gsc_wa_14015076503(gt, false); in do_gt_reset()
663 static int vf_gt_restart(struct xe_gt *gt) in vf_gt_restart() argument
667 err = xe_uc_sanitize_reset(>->uc); in vf_gt_restart()
671 err = xe_uc_init_hw(>->uc); in vf_gt_restart()
675 err = xe_uc_start(>->uc); in vf_gt_restart()
682 static int do_gt_restart(struct xe_gt *gt) in do_gt_restart() argument
688 if (IS_SRIOV_VF(gt_to_xe(gt))) in do_gt_restart()
689 return vf_gt_restart(gt); in do_gt_restart()
691 xe_pat_init(gt); in do_gt_restart()
693 xe_gt_enable_host_l2_vram(gt); in do_gt_restart()
695 xe_gt_mcr_set_implicit_defaults(gt); in do_gt_restart()
696 xe_reg_sr_apply_mmio(>->reg_sr, gt); in do_gt_restart()
698 err = xe_wopcm_init(>->uc.wopcm); in do_gt_restart()
702 for_each_hw_engine(hwe, gt, id) in do_gt_restart()
705 err = xe_uc_sanitize_reset(>->uc); in do_gt_restart()
709 err = xe_uc_init_hw(>->uc); in do_gt_restart()
713 if (IS_SRIOV_PF(gt_to_xe(gt)) && !xe_gt_is_media_type(gt)) in do_gt_restart()
714 xe_lmtt_init_hw(>_to_tile(gt)->sriov.pf.lmtt); in do_gt_restart()
716 if (IS_SRIOV_PF(gt_to_xe(gt))) in do_gt_restart()
717 xe_gt_sriov_pf_init_hw(gt); in do_gt_restart()
719 xe_mocs_init(gt); in do_gt_restart()
720 err = xe_uc_start(>->uc); in do_gt_restart()
724 for_each_hw_engine(hwe, gt, id) { in do_gt_restart()
725 xe_reg_sr_apply_mmio(&hwe->reg_sr, gt); in do_gt_restart()
730 xe_gt_apply_ccs_mode(gt); in do_gt_restart()
732 /* Restore GT freq to expected values */ in do_gt_restart()
733 xe_gt_sanitize_freq(gt); in do_gt_restart()
735 if (IS_SRIOV_PF(gt_to_xe(gt))) in do_gt_restart()
736 xe_gt_sriov_pf_restart(gt); in do_gt_restart()
741 static int gt_reset(struct xe_gt *gt) in gt_reset() argument
745 if (xe_device_wedged(gt_to_xe(gt))) in gt_reset()
748 /* We only support GT resets with GuC submission */ in gt_reset()
749 if (!xe_device_uc_enabled(gt_to_xe(gt))) in gt_reset()
752 xe_gt_info(gt, "reset started\n"); in gt_reset()
754 xe_pm_runtime_get(gt_to_xe(gt)); in gt_reset()
761 xe_gt_sanitize(gt); in gt_reset()
763 err = xe_force_wake_get(gt_to_fw(gt), XE_FORCEWAKE_ALL); in gt_reset()
767 xe_uc_gucrc_disable(>->uc); in gt_reset()
768 xe_uc_stop_prepare(>->uc); in gt_reset()
769 xe_gt_pagefault_reset(gt); in gt_reset()
771 xe_uc_stop(>->uc); in gt_reset()
773 xe_gt_tlb_invalidation_reset(gt); in gt_reset()
775 err = do_gt_reset(gt); in gt_reset()
779 err = do_gt_restart(gt); in gt_reset()
783 err = xe_force_wake_put(gt_to_fw(gt), XE_FORCEWAKE_ALL); in gt_reset()
785 xe_pm_runtime_put(gt_to_xe(gt)); in gt_reset()
787 xe_gt_info(gt, "reset done\n"); in gt_reset()
792 XE_WARN_ON(xe_force_wake_put(gt_to_fw(gt), XE_FORCEWAKE_ALL)); in gt_reset()
794 XE_WARN_ON(xe_uc_start(>->uc)); in gt_reset()
796 xe_gt_err(gt, "reset failed (%pe)\n", ERR_PTR(err)); in gt_reset()
798 xe_device_declare_wedged(gt_to_xe(gt)); in gt_reset()
799 xe_pm_runtime_put(gt_to_xe(gt)); in gt_reset()
806 struct xe_gt *gt = container_of(w, typeof(*gt), reset.worker); in gt_reset_worker() local
808 gt_reset(gt); in gt_reset_worker()
811 void xe_gt_reset_async(struct xe_gt *gt) in xe_gt_reset_async() argument
813 xe_gt_info(gt, "trying reset\n"); in xe_gt_reset_async()
816 if (!xe_fault_inject_gt_reset() && xe_uc_reset_prepare(>->uc)) in xe_gt_reset_async()
819 xe_gt_info(gt, "reset queued\n"); in xe_gt_reset_async()
820 queue_work(gt->ordered_wq, >->reset.worker); in xe_gt_reset_async()
823 void xe_gt_suspend_prepare(struct xe_gt *gt) in xe_gt_suspend_prepare() argument
825 XE_WARN_ON(xe_force_wake_get(gt_to_fw(gt), XE_FORCEWAKE_ALL)); in xe_gt_suspend_prepare()
827 xe_uc_stop_prepare(>->uc); in xe_gt_suspend_prepare()
829 XE_WARN_ON(xe_force_wake_put(gt_to_fw(gt), XE_FORCEWAKE_ALL)); in xe_gt_suspend_prepare()
832 int xe_gt_suspend(struct xe_gt *gt) in xe_gt_suspend() argument
836 xe_gt_dbg(gt, "suspending\n"); in xe_gt_suspend()
837 xe_gt_sanitize(gt); in xe_gt_suspend()
839 err = xe_force_wake_get(gt_to_fw(gt), XE_FORCEWAKE_ALL); in xe_gt_suspend()
843 err = xe_uc_suspend(>->uc); in xe_gt_suspend()
847 xe_gt_idle_disable_pg(gt); in xe_gt_suspend()
849 xe_gt_disable_host_l2_vram(gt); in xe_gt_suspend()
851 XE_WARN_ON(xe_force_wake_put(gt_to_fw(gt), XE_FORCEWAKE_ALL)); in xe_gt_suspend()
852 xe_gt_dbg(gt, "suspended\n"); in xe_gt_suspend()
857 XE_WARN_ON(xe_force_wake_put(gt_to_fw(gt), XE_FORCEWAKE_ALL)); in xe_gt_suspend()
859 xe_gt_err(gt, "suspend failed (%pe)\n", ERR_PTR(err)); in xe_gt_suspend()
866 * @gt: the GT object
868 * Called after driver init/GSC load completes to restore GT frequencies if we
871 int xe_gt_sanitize_freq(struct xe_gt *gt) in xe_gt_sanitize_freq() argument
875 if ((!xe_uc_fw_is_available(>->uc.gsc.fw) || in xe_gt_sanitize_freq()
876 xe_uc_fw_is_loaded(>->uc.gsc.fw) || in xe_gt_sanitize_freq()
877 xe_uc_fw_is_in_error_state(>->uc.gsc.fw)) && in xe_gt_sanitize_freq()
878 XE_WA(gt, 22019338487)) in xe_gt_sanitize_freq()
879 ret = xe_guc_pc_restore_stashed_freq(>->uc.guc.pc); in xe_gt_sanitize_freq()
884 int xe_gt_resume(struct xe_gt *gt) in xe_gt_resume() argument
888 xe_gt_dbg(gt, "resuming\n"); in xe_gt_resume()
889 err = xe_force_wake_get(gt_to_fw(gt), XE_FORCEWAKE_ALL); in xe_gt_resume()
893 err = do_gt_restart(gt); in xe_gt_resume()
897 xe_gt_idle_enable_pg(gt); in xe_gt_resume()
899 XE_WARN_ON(xe_force_wake_put(gt_to_fw(gt), XE_FORCEWAKE_ALL)); in xe_gt_resume()
900 xe_gt_dbg(gt, "resumed\n"); in xe_gt_resume()
905 XE_WARN_ON(xe_force_wake_put(gt_to_fw(gt), XE_FORCEWAKE_ALL)); in xe_gt_resume()
907 xe_gt_err(gt, "resume failed (%pe)\n", ERR_PTR(err)); in xe_gt_resume()
912 struct xe_hw_engine *xe_gt_hw_engine(struct xe_gt *gt, in xe_gt_hw_engine() argument
919 for_each_hw_engine(hwe, gt, id) in xe_gt_hw_engine()
928 struct xe_hw_engine *xe_gt_any_hw_engine_by_reset_domain(struct xe_gt *gt, in xe_gt_any_hw_engine_by_reset_domain() argument
934 for_each_hw_engine(hwe, gt, id) { in xe_gt_any_hw_engine_by_reset_domain()
951 struct xe_hw_engine *xe_gt_any_hw_engine(struct xe_gt *gt) in xe_gt_any_hw_engine() argument
956 for_each_hw_engine(hwe, gt, id) in xe_gt_any_hw_engine()
963 * xe_gt_declare_wedged() - Declare GT wedged
964 * @gt: the GT object
966 * Wedge the GT which stops all submission, saves desired debug state, and
969 void xe_gt_declare_wedged(struct xe_gt *gt) in xe_gt_declare_wedged() argument
971 xe_gt_assert(gt, gt_to_xe(gt)->wedged.mode); in xe_gt_declare_wedged()
973 xe_uc_declare_wedged(>->uc); in xe_gt_declare_wedged()
974 xe_gt_tlb_invalidation_reset(gt); in xe_gt_declare_wedged()