Lines Matching refs:hl_device
2203 static int gaudi2_memset_device_memory(struct hl_device *hdev, u64 addr, u64 size, u64 val);
2204 static bool gaudi2_is_queue_enabled(struct hl_device *hdev, u32 hw_queue_id);
2205 static bool gaudi2_is_arc_enabled(struct hl_device *hdev, u64 arc_id);
2206 static void gaudi2_clr_arc_id_cap(struct hl_device *hdev, u64 arc_id);
2207 static void gaudi2_set_arc_id_cap(struct hl_device *hdev, u64 arc_id);
2208 static void gaudi2_memset_device_lbw(struct hl_device *hdev, u32 addr, u32 size, u32 val);
2209 static int gaudi2_send_job_to_kdma(struct hl_device *hdev, u64 src_addr, u64 dst_addr, u32 size,
2211 static bool gaudi2_get_tpc_idle_status(struct hl_device *hdev, u64 *mask_arr, u8 mask_len,
2213 static bool gaudi2_get_mme_idle_status(struct hl_device *hdev, u64 *mask_arr, u8 mask_len,
2215 static bool gaudi2_get_edma_idle_status(struct hl_device *hdev, u64 *mask_arr, u8 mask_len,
2217 static u64 gaudi2_mmu_scramble_addr(struct hl_device *hdev, u64 raw_addr);
2218 static u64 gaudi2_mmu_descramble_addr(struct hl_device *hdev, u64 scrambled_addr);
2220 static void gaudi2_init_scrambler_hbm(struct hl_device *hdev) in gaudi2_init_scrambler_hbm()
2225 static u32 gaudi2_get_signal_cb_size(struct hl_device *hdev) in gaudi2_get_signal_cb_size()
2230 static u32 gaudi2_get_wait_cb_size(struct hl_device *hdev) in gaudi2_get_wait_cb_size()
2235 void gaudi2_iterate_tpcs(struct hl_device *hdev, struct iterate_module_ctx *ctx) in gaudi2_iterate_tpcs()
2280 static int set_number_of_functional_hbms(struct hl_device *hdev) in set_number_of_functional_hbms()
2327 static int gaudi2_set_dram_properties(struct hl_device *hdev) in gaudi2_set_dram_properties()
2409 static int gaudi2_set_fixed_properties(struct hl_device *hdev) in gaudi2_set_fixed_properties()
2613 static int gaudi2_pci_bars_map(struct hl_device *hdev) in gaudi2_pci_bars_map()
2628 static u64 gaudi2_set_hbm_bar_base(struct hl_device *hdev, u64 addr) in gaudi2_set_hbm_bar_base()
2657 static int gaudi2_init_iatu(struct hl_device *hdev) in gaudi2_init_iatu()
2721 static enum hl_device_hw_state gaudi2_get_hw_state(struct hl_device *hdev) in gaudi2_get_hw_state()
2726 static int gaudi2_tpc_binning_init_prop(struct hl_device *hdev) in gaudi2_tpc_binning_init_prop()
2747 static int gaudi2_set_tpc_binning_masks(struct hl_device *hdev) in gaudi2_set_tpc_binning_masks()
2805 static int gaudi2_set_dec_binning_masks(struct hl_device *hdev) in gaudi2_set_dec_binning_masks()
2832 static void gaudi2_set_dram_binning_masks(struct hl_device *hdev) in gaudi2_set_dram_binning_masks()
2849 static int gaudi2_set_edma_binning_masks(struct hl_device *hdev) in gaudi2_set_edma_binning_masks()
2891 static int gaudi2_set_xbar_edge_enable_mask(struct hl_device *hdev, u32 xbar_edge_iso_mask) in gaudi2_set_xbar_edge_enable_mask()
2927 static int gaudi2_set_cluster_binning_masks_common(struct hl_device *hdev, u8 xbar_edge_iso_mask) in gaudi2_set_cluster_binning_masks_common()
2955 static int gaudi2_set_cluster_binning_masks(struct hl_device *hdev) in gaudi2_set_cluster_binning_masks()
2974 static int gaudi2_set_binning_masks(struct hl_device *hdev) in gaudi2_set_binning_masks()
2993 static int gaudi2_cpucp_info_get(struct hl_device *hdev) in gaudi2_cpucp_info_get()
3066 static int gaudi2_fetch_psoc_frequency(struct hl_device *hdev) in gaudi2_fetch_psoc_frequency()
3084 static int gaudi2_mmu_clear_pgt_range(struct hl_device *hdev) in gaudi2_mmu_clear_pgt_range()
3103 static int gaudi2_early_init(struct hl_device *hdev) in gaudi2_early_init()
3176 static int gaudi2_early_fini(struct hl_device *hdev) in gaudi2_early_fini()
3204 static void gaudi2_init_arcs(struct hl_device *hdev) in gaudi2_init_arcs()
3242 static int gaudi2_scrub_arc_dccm(struct hl_device *hdev, u32 cpu_id) in gaudi2_scrub_arc_dccm()
3291 static int gaudi2_scrub_arcs_dccm(struct hl_device *hdev) in gaudi2_scrub_arcs_dccm()
3308 static int gaudi2_late_init(struct hl_device *hdev) in gaudi2_late_init()
3348 static void gaudi2_late_fini(struct hl_device *hdev) in gaudi2_late_fini()
3369 static void gaudi2_user_mapped_blocks_init(struct hl_device *hdev) in gaudi2_user_mapped_blocks_init()
3441 static int gaudi2_alloc_cpu_accessible_dma_mem(struct hl_device *hdev) in gaudi2_alloc_cpu_accessible_dma_mem()
3483 static void gaudi2_set_pci_memory_regions(struct hl_device *hdev) in gaudi2_set_pci_memory_regions()
3516 static void gaudi2_user_interrupt_setup(struct hl_device *hdev) in gaudi2_user_interrupt_setup()
3561 static void gaudi2_special_blocks_free(struct hl_device *hdev) in gaudi2_special_blocks_free()
3572 static void gaudi2_special_blocks_iterator_free(struct hl_device *hdev) in gaudi2_special_blocks_iterator_free()
3577 static bool gaudi2_special_block_skip(struct hl_device *hdev, in gaudi2_special_block_skip()
3584 static int gaudi2_special_blocks_config(struct hl_device *hdev) in gaudi2_special_blocks_config()
3649 static int gaudi2_special_blocks_iterator_config(struct hl_device *hdev) in gaudi2_special_blocks_iterator_config()
3654 static void gaudi2_test_queues_msgs_free(struct hl_device *hdev) in gaudi2_test_queues_msgs_free()
3670 static int gaudi2_test_queues_msgs_alloc(struct hl_device *hdev) in gaudi2_test_queues_msgs_alloc()
3696 static int gaudi2_sw_init(struct hl_device *hdev) in gaudi2_sw_init()
3819 static int gaudi2_sw_fini(struct hl_device *hdev) in gaudi2_sw_fini()
3842 static void gaudi2_stop_qman_common(struct hl_device *hdev, u32 reg_base) in gaudi2_stop_qman_common()
3852 static void gaudi2_flush_qman_common(struct hl_device *hdev, u32 reg_base) in gaudi2_flush_qman_common()
3859 static void gaudi2_flush_qman_arc_common(struct hl_device *hdev, u32 reg_base) in gaudi2_flush_qman_arc_common()
3873 static void gaudi2_clear_qm_fence_counters_common(struct hl_device *hdev, u32 queue_id, in gaudi2_clear_qm_fence_counters_common()
3894 static void gaudi2_qman_manual_flush_common(struct hl_device *hdev, u32 queue_id) in gaudi2_qman_manual_flush_common()
3903 static void gaudi2_stop_dma_qmans(struct hl_device *hdev) in gaudi2_stop_dma_qmans()
3936 static void gaudi2_stop_mme_qmans(struct hl_device *hdev) in gaudi2_stop_mme_qmans()
3951 static void gaudi2_stop_tpc_qmans(struct hl_device *hdev) in gaudi2_stop_tpc_qmans()
3969 static void gaudi2_stop_rot_qmans(struct hl_device *hdev) in gaudi2_stop_rot_qmans()
3987 static void gaudi2_stop_nic_qmans(struct hl_device *hdev) in gaudi2_stop_nic_qmans()
4007 static void gaudi2_stall_dma_common(struct hl_device *hdev, u32 reg_base) in gaudi2_stall_dma_common()
4015 static void gaudi2_dma_stall(struct hl_device *hdev) in gaudi2_dma_stall()
4047 static void gaudi2_mme_stall(struct hl_device *hdev) in gaudi2_mme_stall()
4059 static void gaudi2_tpc_stall(struct hl_device *hdev) in gaudi2_tpc_stall()
4077 static void gaudi2_rotator_stall(struct hl_device *hdev) in gaudi2_rotator_stall()
4098 static void gaudi2_disable_qman_common(struct hl_device *hdev, u32 reg_base) in gaudi2_disable_qman_common()
4103 static void gaudi2_disable_dma_qmans(struct hl_device *hdev) in gaudi2_disable_dma_qmans()
4135 static void gaudi2_disable_mme_qmans(struct hl_device *hdev) in gaudi2_disable_mme_qmans()
4147 static void gaudi2_disable_tpc_qmans(struct hl_device *hdev) in gaudi2_disable_tpc_qmans()
4165 static void gaudi2_disable_rot_qmans(struct hl_device *hdev) in gaudi2_disable_rot_qmans()
4183 static void gaudi2_disable_nic_qmans(struct hl_device *hdev) in gaudi2_disable_nic_qmans()
4203 static void gaudi2_enable_timestamp(struct hl_device *hdev) in gaudi2_enable_timestamp()
4216 static void gaudi2_disable_timestamp(struct hl_device *hdev) in gaudi2_disable_timestamp()
4244 static void gaudi2_dec_disable_msix(struct hl_device *hdev, u32 max_irq_num) in gaudi2_dec_disable_msix()
4265 static int gaudi2_dec_enable_msix(struct hl_device *hdev) in gaudi2_dec_enable_msix()
4306 static int gaudi2_enable_msix(struct hl_device *hdev) in gaudi2_enable_msix()
4422 static void gaudi2_sync_irqs(struct hl_device *hdev) in gaudi2_sync_irqs()
4452 static void gaudi2_disable_msix(struct hl_device *hdev) in gaudi2_disable_msix()
4495 static void gaudi2_stop_dcore_dec(struct hl_device *hdev, int dcore_id) in gaudi2_stop_dcore_dec()
4535 static void gaudi2_stop_pcie_dec(struct hl_device *hdev) in gaudi2_stop_pcie_dec()
4575 static void gaudi2_stop_dec(struct hl_device *hdev) in gaudi2_stop_dec()
4589 static void gaudi2_set_arc_running_mode(struct hl_device *hdev, u32 cpu_id, u32 run_mode) in gaudi2_set_arc_running_mode()
4602 static void gaudi2_halt_arcs(struct hl_device *hdev) in gaudi2_halt_arcs()
4612 static int gaudi2_verify_arc_running_mode(struct hl_device *hdev, u32 cpu_id, u32 run_mode) in gaudi2_verify_arc_running_mode()
4639 static void gaudi2_reset_arcs(struct hl_device *hdev) in gaudi2_reset_arcs()
4652 static void gaudi2_nic_qmans_manual_flush(struct hl_device *hdev) in gaudi2_nic_qmans_manual_flush()
4671 static int gaudi2_set_engine_cores(struct hl_device *hdev, u32 *core_ids, in gaudi2_set_engine_cores()
4697 static int gaudi2_set_tpc_engine_mode(struct hl_device *hdev, u32 engine_id, u32 engine_command) in gaudi2_set_tpc_engine_mode()
4724 static int gaudi2_set_mme_engine_mode(struct hl_device *hdev, u32 engine_id, u32 engine_command) in gaudi2_set_mme_engine_mode()
4742 static int gaudi2_set_edma_engine_mode(struct hl_device *hdev, u32 engine_id, u32 engine_command) in gaudi2_set_edma_engine_mode()
4769 static int gaudi2_set_engine_modes(struct hl_device *hdev, in gaudi2_set_engine_modes()
4812 static int gaudi2_set_engines(struct hl_device *hdev, u32 *engine_ids, in gaudi2_set_engines()
4830 static void gaudi2_halt_engines(struct hl_device *hdev, bool hard_reset, bool fw_reset) in gaudi2_halt_engines()
4882 static void gaudi2_init_firmware_preload_params(struct hl_device *hdev) in gaudi2_init_firmware_preload_params()
4896 static void gaudi2_init_firmware_loader(struct hl_device *hdev) in gaudi2_init_firmware_loader()
4925 static int gaudi2_init_cpu(struct hl_device *hdev) in gaudi2_init_cpu()
4945 static int gaudi2_init_cpu_queues(struct hl_device *hdev, u32 cpu_timeout) in gaudi2_init_cpu_queues()
5014 static void gaudi2_init_qman_pq(struct hl_device *hdev, u32 reg_base, in gaudi2_init_qman_pq()
5041 static void gaudi2_init_qman_cp(struct hl_device *hdev, u32 reg_base) in gaudi2_init_qman_cp()
5063 static void gaudi2_init_qman_pqc(struct hl_device *hdev, u32 reg_base, in gaudi2_init_qman_pqc()
5093 static u32 gaudi2_get_dyn_sp_reg(struct hl_device *hdev, u32 queue_id_base) in gaudi2_get_dyn_sp_reg()
5142 static void gaudi2_init_qman_common(struct hl_device *hdev, u32 reg_base, in gaudi2_init_qman_common()
5178 static void gaudi2_init_qman(struct hl_device *hdev, u32 reg_base, in gaudi2_init_qman()
5192 static void gaudi2_init_dma_core(struct hl_device *hdev, u32 reg_base, in gaudi2_init_dma_core()
5222 static void gaudi2_init_kdma(struct hl_device *hdev) in gaudi2_init_kdma()
5237 static void gaudi2_init_pdma(struct hl_device *hdev) in gaudi2_init_pdma()
5260 static void gaudi2_init_edma_instance(struct hl_device *hdev, u8 seq) in gaudi2_init_edma_instance()
5274 static void gaudi2_init_edma(struct hl_device *hdev) in gaudi2_init_edma()
5312 static void gaudi2_arm_monitors_for_virt_msix_db(struct hl_device *hdev, u32 sob_id, in gaudi2_arm_monitors_for_virt_msix_db()
5378 static void gaudi2_prepare_sm_for_virt_msix_db(struct hl_device *hdev) in gaudi2_prepare_sm_for_virt_msix_db()
5400 static void gaudi2_init_sm(struct hl_device *hdev) in gaudi2_init_sm()
5443 static void gaudi2_init_mme_acc(struct hl_device *hdev, u32 reg_base) in gaudi2_init_mme_acc()
5465 static void gaudi2_init_dcore_mme(struct hl_device *hdev, int dcore_id, in gaudi2_init_dcore_mme()
5497 static void gaudi2_init_mme(struct hl_device *hdev) in gaudi2_init_mme()
5512 static void gaudi2_init_tpc_cfg(struct hl_device *hdev, u32 reg_base) in gaudi2_init_tpc_cfg()
5526 static void gaudi2_init_tpc_config(struct hl_device *hdev, int dcore, int inst, in gaudi2_init_tpc_config()
5548 static void gaudi2_init_tpc(struct hl_device *hdev) in gaudi2_init_tpc()
5569 static void gaudi2_init_rotator(struct hl_device *hdev) in gaudi2_init_rotator()
5584 static void gaudi2_init_vdec_brdg_ctrl(struct hl_device *hdev, u64 base_addr, u32 decoder_id) in gaudi2_init_vdec_brdg_ctrl()
5601 static void gaudi2_init_dec(struct hl_device *hdev) in gaudi2_init_dec()
5644 static int gaudi2_mmu_update_asid_hop0_addr(struct hl_device *hdev, in gaudi2_mmu_update_asid_hop0_addr()
5676 static void gaudi2_mmu_send_invalidate_cache_cmd(struct hl_device *hdev, u32 stlb_base, in gaudi2_mmu_send_invalidate_cache_cmd()
5690 static int gaudi2_mmu_invalidate_cache_status_poll(struct hl_device *hdev, u32 stlb_base, in gaudi2_mmu_invalidate_cache_status_poll()
5736 bool gaudi2_is_hmmu_enabled(struct hl_device *hdev, int dcore_id, int hmmu_id) in gaudi2_is_hmmu_enabled()
5758 static void gaudi2_mmu_invalidate_cache_trigger(struct hl_device *hdev, u32 stlb_base, in gaudi2_mmu_invalidate_cache_trigger()
5793 static inline void gaudi2_hmmu_invalidate_cache_trigger(struct hl_device *hdev, in gaudi2_hmmu_invalidate_cache_trigger()
5802 static inline int gaudi2_hmmu_invalidate_cache_status_poll(struct hl_device *hdev, in gaudi2_hmmu_invalidate_cache_status_poll()
5811 static int gaudi2_hmmus_invalidate_cache(struct hl_device *hdev, in gaudi2_hmmus_invalidate_cache()
5844 static int gaudi2_mmu_invalidate_cache(struct hl_device *hdev, bool is_hard, u32 flags) in gaudi2_mmu_invalidate_cache()
5869 static int gaudi2_mmu_invalidate_cache_range(struct hl_device *hdev, bool is_hard, in gaudi2_mmu_invalidate_cache_range()
5921 static int gaudi2_mmu_update_hop0_addr(struct hl_device *hdev, u32 stlb_base, in gaudi2_mmu_update_hop0_addr()
5949 static int gaudi2_mmu_init_common(struct hl_device *hdev, u32 mmu_base, u32 stlb_base, in gaudi2_mmu_init_common()
5995 static int gaudi2_pci_mmu_init(struct hl_device *hdev) in gaudi2_pci_mmu_init()
6046 static int gaudi2_dcore_hmmu_init(struct hl_device *hdev, int dcore_id, in gaudi2_dcore_hmmu_init()
6098 static int gaudi2_hbm_mmu_init(struct hl_device *hdev) in gaudi2_hbm_mmu_init()
6112 static int gaudi2_mmu_init(struct hl_device *hdev) in gaudi2_mmu_init()
6127 static int gaudi2_hw_init(struct hl_device *hdev) in gaudi2_hw_init()
6225 static void gaudi2_send_hard_reset_cmd(struct hl_device *hdev) in gaudi2_send_hard_reset_cmd()
6299 static void gaudi2_execute_hard_reset(struct hl_device *hdev) in gaudi2_execute_hard_reset()
6326 static int gaudi2_execute_soft_reset(struct hl_device *hdev, bool driver_performs_reset, in gaudi2_execute_soft_reset()
6347 static void gaudi2_poll_btm_indication(struct hl_device *hdev, u32 poll_timeout_us) in gaudi2_poll_btm_indication()
6368 static int gaudi2_hw_fini(struct hl_device *hdev, bool hard_reset, bool fw_reset) in gaudi2_hw_fini()
6468 static int gaudi2_suspend(struct hl_device *hdev) in gaudi2_suspend()
6473 static int gaudi2_resume(struct hl_device *hdev) in gaudi2_resume()
6478 static int gaudi2_mmap(struct hl_device *hdev, struct vm_area_struct *vma, in gaudi2_mmap()
6505 static bool gaudi2_is_queue_enabled(struct hl_device *hdev, u32 hw_queue_id) in gaudi2_is_queue_enabled()
6611 static bool gaudi2_is_arc_enabled(struct hl_device *hdev, u64 arc_id) in gaudi2_is_arc_enabled()
6631 static void gaudi2_clr_arc_id_cap(struct hl_device *hdev, u64 arc_id) in gaudi2_clr_arc_id_cap()
6654 static void gaudi2_set_arc_id_cap(struct hl_device *hdev, u64 arc_id) in gaudi2_set_arc_id_cap()
6677 static void gaudi2_ring_doorbell(struct hl_device *hdev, u32 hw_queue_id, u32 pi) in gaudi2_ring_doorbell()
6708 static void gaudi2_pqe_write(struct hl_device *hdev, __le64 *pqe, struct hl_bd *bd) in gaudi2_pqe_write()
6717 static void *gaudi2_dma_alloc_coherent(struct hl_device *hdev, size_t size, in gaudi2_dma_alloc_coherent()
6723 static void gaudi2_dma_free_coherent(struct hl_device *hdev, size_t size, in gaudi2_dma_free_coherent()
6729 static int gaudi2_send_cpu_message(struct hl_device *hdev, u32 *msg, u16 len, in gaudi2_send_cpu_message()
6746 static void *gaudi2_dma_pool_zalloc(struct hl_device *hdev, size_t size, in gaudi2_dma_pool_zalloc()
6755 static void gaudi2_dma_pool_free(struct hl_device *hdev, void *vaddr, dma_addr_t dma_addr) in gaudi2_dma_pool_free()
6760 static void *gaudi2_cpu_accessible_dma_pool_alloc(struct hl_device *hdev, size_t size, in gaudi2_cpu_accessible_dma_pool_alloc()
6766 static void gaudi2_cpu_accessible_dma_pool_free(struct hl_device *hdev, size_t size, void *vaddr) in gaudi2_cpu_accessible_dma_pool_free()
6771 static int gaudi2_validate_cb_address(struct hl_device *hdev, struct hl_cs_parser *parser) in gaudi2_validate_cb_address()
6828 static int gaudi2_cs_parser(struct hl_device *hdev, struct hl_cs_parser *parser) in gaudi2_cs_parser()
6843 static int gaudi2_send_heartbeat(struct hl_device *hdev) in gaudi2_send_heartbeat()
6856 static void gaudi2_kdma_set_mmbp_asid(struct hl_device *hdev, in gaudi2_kdma_set_mmbp_asid()
6871 static void gaudi2_arm_cq_monitor(struct hl_device *hdev, u32 sob_id, u32 mon_id, u32 cq_id, in gaudi2_arm_cq_monitor()
6901 static int gaudi2_send_job_to_kdma(struct hl_device *hdev, in gaudi2_send_job_to_kdma()
6972 static void gaudi2_memset_device_lbw(struct hl_device *hdev, u32 addr, u32 size, u32 val) in gaudi2_memset_device_lbw()
6980 static void gaudi2_qman_set_test_mode(struct hl_device *hdev, u32 hw_queue_id, bool enable) in gaudi2_qman_set_test_mode()
6993 static inline u32 gaudi2_test_queue_hw_queue_id_to_sob_id(struct hl_device *hdev, u32 hw_queue_id) in gaudi2_test_queue_hw_queue_id_to_sob_id()
6999 static void gaudi2_test_queue_clear(struct hl_device *hdev, u32 hw_queue_id) in gaudi2_test_queue_clear()
7008 static int gaudi2_test_queue_send_msg_short(struct hl_device *hdev, u32 hw_queue_id, u32 sob_val, in gaudi2_test_queue_send_msg_short()
7034 static int gaudi2_test_queue_wait_completion(struct hl_device *hdev, u32 hw_queue_id, u32 sob_val) in gaudi2_test_queue_wait_completion()
7063 static int gaudi2_test_cpu_queue(struct hl_device *hdev) in gaudi2_test_cpu_queue()
7077 static int gaudi2_test_queues(struct hl_device *hdev) in gaudi2_test_queues()
7119 static int gaudi2_compute_reset_late_init(struct hl_device *hdev) in gaudi2_compute_reset_late_init()
7140 static bool gaudi2_get_edma_idle_status(struct hl_device *hdev, u64 *mask_arr, u8 mask_len, in gaudi2_get_edma_idle_status()
7190 static bool gaudi2_get_pdma_idle_status(struct hl_device *hdev, u64 *mask_arr, u8 mask_len, in gaudi2_get_pdma_idle_status()
7230 static bool gaudi2_get_nic_idle_status(struct hl_device *hdev, u64 *mask_arr, u8 mask_len, in gaudi2_get_nic_idle_status()
7276 static bool gaudi2_get_mme_idle_status(struct hl_device *hdev, u64 *mask_arr, u8 mask_len, in gaudi2_get_mme_idle_status()
7319 static void gaudi2_is_tpc_engine_idle(struct hl_device *hdev, int dcore, int inst, u32 offset, in gaudi2_is_tpc_engine_idle()
7352 static bool gaudi2_get_tpc_idle_status(struct hl_device *hdev, u64 *mask_arr, u8 mask_len, in gaudi2_get_tpc_idle_status()
7380 static bool gaudi2_get_decoder_idle_status(struct hl_device *hdev, u64 *mask_arr, u8 mask_len, in gaudi2_get_decoder_idle_status()
7449 static bool gaudi2_get_rotator_idle_status(struct hl_device *hdev, u64 *mask_arr, u8 mask_len, in gaudi2_get_rotator_idle_status()
7487 static bool gaudi2_is_device_idle(struct hl_device *hdev, u64 *mask_arr, u8 mask_len, in gaudi2_is_device_idle()
7503 static void gaudi2_hw_queues_lock(struct hl_device *hdev) in gaudi2_hw_queues_lock()
7511 static void gaudi2_hw_queues_unlock(struct hl_device *hdev) in gaudi2_hw_queues_unlock()
7519 static u32 gaudi2_get_pci_id(struct hl_device *hdev) in gaudi2_get_pci_id()
7524 static int gaudi2_get_eeprom_data(struct hl_device *hdev, void *data, size_t max_size) in gaudi2_get_eeprom_data()
7534 static void gaudi2_update_eq_ci(struct hl_device *hdev, u32 val) in gaudi2_update_eq_ci()
7539 static void *gaudi2_get_events_stat(struct hl_device *hdev, bool aggregate, u32 *size) in gaudi2_get_events_stat()
7552 static void gaudi2_mmu_vdec_dcore_prepare(struct hl_device *hdev, int dcore_id, in gaudi2_mmu_vdec_dcore_prepare()
7574 static void gaudi2_mmu_dcore_prepare(struct hl_device *hdev, int dcore_id, u32 asid) in gaudi2_mmu_dcore_prepare()
7643 static void gudi2_mmu_vdec_shared_prepare(struct hl_device *hdev, in gudi2_mmu_vdec_shared_prepare()
7664 static void gudi2_mmu_arc_farm_arc_dup_eng_prepare(struct hl_device *hdev, int arc_farm_id, in gudi2_mmu_arc_farm_arc_dup_eng_prepare()
7673 static void gaudi2_arc_mmu_prepare(struct hl_device *hdev, u32 cpu_id, u32 asid) in gaudi2_arc_mmu_prepare()
7717 static int gaudi2_arc_mmu_prepare_all(struct hl_device *hdev, u32 asid) in gaudi2_arc_mmu_prepare_all()
7737 static int gaudi2_mmu_shared_prepare(struct hl_device *hdev, u32 asid) in gaudi2_mmu_shared_prepare()
7784 static void gaudi2_tpc_mmu_prepare(struct hl_device *hdev, int dcore, int inst, u32 offset, in gaudi2_tpc_mmu_prepare()
7796 static int gaudi2_mmu_prepare(struct hl_device *hdev, u32 asid) in gaudi2_mmu_prepare()
7846 static void gaudi2_print_event(struct hl_device *hdev, u16 event_type, in gaudi2_print_event()
7868 static bool gaudi2_handle_ecc_event(struct hl_device *hdev, u16 event_type, in gaudi2_handle_ecc_event()
7898 static void handle_lower_qman_data_on_err(struct hl_device *hdev, u64 qman_base, u32 engine_id) in handle_lower_qman_data_on_err()
7939 static int gaudi2_handle_qman_err_generic(struct hl_device *hdev, u16 event_type, in gaudi2_handle_qman_err_generic()
8001 static void gaudi2_razwi_rr_hbw_shared_printf_info(struct hl_device *hdev, in gaudi2_razwi_rr_hbw_shared_printf_info()
8029 static void gaudi2_razwi_rr_lbw_shared_printf_info(struct hl_device *hdev, in gaudi2_razwi_rr_lbw_shared_printf_info()
8055 static enum gaudi2_engine_id gaudi2_razwi_calc_engine_id(struct hl_device *hdev, in gaudi2_razwi_calc_engine_id()
8106 static void gaudi2_ack_module_razwi_event_handler(struct hl_device *hdev, in gaudi2_ack_module_razwi_event_handler()
8261 static void gaudi2_check_if_razwi_happened(struct hl_device *hdev) in gaudi2_check_if_razwi_happened()
8331 static bool gaudi2_handle_psoc_razwi_happened(struct hl_device *hdev, u32 razwi_reg, in gaudi2_handle_psoc_razwi_happened()
8420 static int gaudi2_ack_psoc_razwi_event_handler(struct hl_device *hdev, u64 *event_mask) in gaudi2_ack_psoc_razwi_event_handler()
8454 static int _gaudi2_handle_qm_sei_err(struct hl_device *hdev, u64 qman_base, u16 event_type) in _gaudi2_handle_qm_sei_err()
8474 static int gaudi2_handle_qm_sei_err(struct hl_device *hdev, u16 event_type, in gaudi2_handle_qm_sei_err()
8537 static int gaudi2_handle_qman_err(struct hl_device *hdev, u16 event_type, u64 *event_mask) in gaudi2_handle_qman_err()
8658 static int gaudi2_handle_arc_farm_sei_err(struct hl_device *hdev, u16 event_type, u64 *event_mask) in gaudi2_handle_arc_farm_sei_err()
8686 static int gaudi2_handle_cpu_sei_err(struct hl_device *hdev, u16 event_type) in gaudi2_handle_cpu_sei_err()
8708 static int gaudi2_handle_rot_err(struct hl_device *hdev, u8 rot_index, u16 event_type, in gaudi2_handle_rot_err()
8730 static int gaudi2_tpc_ack_interrupts(struct hl_device *hdev, u8 tpc_index, u16 event_type, in gaudi2_tpc_ack_interrupts()
8752 static int gaudi2_handle_dec_err(struct hl_device *hdev, u8 dec_index, u16 event_type, in gaudi2_handle_dec_err()
8789 static int gaudi2_handle_mme_err(struct hl_device *hdev, u8 mme_index, u16 event_type, in gaudi2_handle_mme_err()
8820 static int gaudi2_handle_mme_sbte_err(struct hl_device *hdev, u16 event_type) in gaudi2_handle_mme_sbte_err()
8832 static int gaudi2_handle_mme_wap_err(struct hl_device *hdev, u8 mme_index, u16 event_type, in gaudi2_handle_mme_wap_err()
8862 static int gaudi2_handle_kdma_core_event(struct hl_device *hdev, u16 event_type, in gaudi2_handle_kdma_core_event()
8885 static int gaudi2_handle_dma_core_event(struct hl_device *hdev, u16 event_type, u64 intr_cause) in gaudi2_handle_dma_core_event()
8902 static void gaudi2_print_pcie_mstr_rr_mstr_if_razwi_info(struct hl_device *hdev, u64 *event_mask) in gaudi2_print_pcie_mstr_rr_mstr_if_razwi_info()
8935 static int gaudi2_print_pcie_addr_dec_info(struct hl_device *hdev, u16 event_type, in gaudi2_print_pcie_addr_dec_info()
8962 static int gaudi2_handle_pif_fatal(struct hl_device *hdev, u16 event_type, in gaudi2_handle_pif_fatal()
8980 static int gaudi2_handle_hif_fatal(struct hl_device *hdev, u16 event_type, u64 intr_cause_data) in gaudi2_handle_hif_fatal()
8996 static void gaudi2_handle_page_error(struct hl_device *hdev, u64 mmu_base, bool is_pmmu, in gaudi2_handle_page_error()
9026 static void gaudi2_handle_access_error(struct hl_device *hdev, u64 mmu_base, bool is_pmmu) in gaudi2_handle_access_error()
9049 static int gaudi2_handle_mmu_spi_sei_generic(struct hl_device *hdev, u16 event_type, in gaudi2_handle_mmu_spi_sei_generic()
9083 static int gaudi2_handle_sm_err(struct hl_device *hdev, u16 event_type, u8 sm_index) in gaudi2_handle_sm_err()
9230 static int gaudi2_handle_mmu_spi_sei_err(struct hl_device *hdev, u16 event_type, u64 *event_mask) in gaudi2_handle_mmu_spi_sei_err()
9263 static bool gaudi2_hbm_sei_handle_read_err(struct hl_device *hdev, in gaudi2_hbm_sei_handle_read_err()
9325 static void gaudi2_hbm_sei_print_wr_par_info(struct hl_device *hdev, in gaudi2_hbm_sei_print_wr_par_info()
9350 static void gaudi2_hbm_sei_print_ca_par_info(struct hl_device *hdev, in gaudi2_hbm_sei_print_ca_par_info()
9367 static bool gaudi2_handle_hbm_mc_sei_err(struct hl_device *hdev, u16 event_type, in gaudi2_handle_hbm_mc_sei_err()
9432 static int gaudi2_handle_hbm_cattrip(struct hl_device *hdev, u16 event_type, in gaudi2_handle_hbm_cattrip()
9444 static int gaudi2_handle_hbm_mc_spi(struct hl_device *hdev, u64 intr_cause_data) in gaudi2_handle_hbm_mc_spi()
9458 static void gaudi2_print_clk_change_info(struct hl_device *hdev, u16 event_type, u64 *event_mask) in gaudi2_print_clk_change_info()
9503 static void gaudi2_print_out_of_sync_info(struct hl_device *hdev, u16 event_type, in gaudi2_print_out_of_sync_info()
9514 static int gaudi2_handle_pcie_p2p_msix(struct hl_device *hdev, u16 event_type) in gaudi2_handle_pcie_p2p_msix()
9542 static int gaudi2_handle_pcie_drain(struct hl_device *hdev, in gaudi2_handle_pcie_drain()
9562 static int gaudi2_handle_psoc_drain(struct hl_device *hdev, u64 intr_cause_data) in gaudi2_handle_psoc_drain()
9580 static void gaudi2_print_cpu_pkt_failure_info(struct hl_device *hdev, u16 event_type, in gaudi2_print_cpu_pkt_failure_info()
9590 static int hl_arc_event_handle(struct hl_device *hdev, u16 event_type, in hl_arc_event_handle()
9615 static u16 event_id_to_engine_id(struct hl_device *hdev, u16 event_type) in event_id_to_engine_id()
9780 static void gaudi2_handle_eqe(struct hl_device *hdev, struct hl_eq_entry *eq_entry) in gaudi2_handle_eqe()
10267 static int gaudi2_memset_memory_chunk_using_edma_qm(struct hl_device *hdev, in gaudi2_memset_memory_chunk_using_edma_qm()
10305 static int gaudi2_memset_device_memory(struct hl_device *hdev, u64 addr, u64 size, u64 val) in gaudi2_memset_device_memory()
10445 static int gaudi2_scrub_device_dram(struct hl_device *hdev, u64 val) in gaudi2_scrub_device_dram()
10459 static int gaudi2_scrub_device_mem(struct hl_device *hdev) in gaudi2_scrub_device_mem()
10489 static void gaudi2_restore_user_sm_registers(struct hl_device *hdev) in gaudi2_restore_user_sm_registers()
10581 static void gaudi2_restore_user_qm_registers(struct hl_device *hdev) in gaudi2_restore_user_qm_registers()
10600 static void gaudi2_restore_nic_qm_registers(struct hl_device *hdev) in gaudi2_restore_nic_qm_registers()
10619 static int gaudi2_context_switch(struct hl_device *hdev, u32 asid) in gaudi2_context_switch()
10624 static void gaudi2_restore_phase_topology(struct hl_device *hdev) in gaudi2_restore_phase_topology()
10628 static void gaudi2_init_block_instances(struct hl_device *hdev, u32 block_idx, in gaudi2_init_block_instances()
10647 static void gaudi2_init_blocks_with_mask(struct hl_device *hdev, struct dup_block_ctx *cfg_ctx, in gaudi2_init_blocks_with_mask()
10658 void gaudi2_init_blocks(struct hl_device *hdev, struct dup_block_ctx *cfg_ctx) in gaudi2_init_blocks()
10663 static int gaudi2_debugfs_read_dma(struct hl_device *hdev, u64 addr, u32 size, void *blob_addr) in gaudi2_debugfs_read_dma()
10763 static int gaudi2_internal_cb_pool_init(struct hl_device *hdev, struct hl_ctx *ctx) in gaudi2_internal_cb_pool_init()
10834 static void gaudi2_internal_cb_pool_fini(struct hl_device *hdev, struct hl_ctx *ctx) in gaudi2_internal_cb_pool_fini()
10853 static void gaudi2_restore_user_registers(struct hl_device *hdev) in gaudi2_restore_user_registers()
10861 struct hl_device *hdev = ctx->hdev; in gaudi2_map_virtual_msix_doorbell_memory()
10877 struct hl_device *hdev = ctx->hdev; in gaudi2_unmap_virtual_msix_doorbell_memory()
10930 struct hl_device *hdev = cs->ctx->hdev; in gaudi2_pre_schedule_cs()
10955 static u32 gaudi2_get_queue_id_for_cq(struct hl_device *hdev, u32 cq_idx) in gaudi2_get_queue_id_for_cq()
10960 static u32 gaudi2_gen_signal_cb(struct hl_device *hdev, void *data, u16 sob_id, u32 size, bool eb) in gaudi2_gen_signal_cb()
11003 static u32 gaudi2_add_arm_monitor_pkt(struct hl_device *hdev, struct packet_msg_short *pkt, in gaudi2_add_arm_monitor_pkt()
11053 static u32 gaudi2_gen_wait_cb(struct hl_device *hdev, struct hl_gen_wait_properties *prop) in gaudi2_gen_wait_cb()
11105 static void gaudi2_reset_sob(struct hl_device *hdev, void *data) in gaudi2_reset_sob()
11116 static void gaudi2_reset_sob_group(struct hl_device *hdev, u16 sob_group) in gaudi2_reset_sob_group()
11120 static u64 gaudi2_get_device_time(struct hl_device *hdev) in gaudi2_get_device_time()
11132 static int gaudi2_collective_wait_create_jobs(struct hl_device *hdev, struct hl_ctx *ctx, in gaudi2_collective_wait_create_jobs()
11157 static u64 gaudi2_mmu_scramble_addr(struct hl_device *hdev, u64 raw_addr) in gaudi2_mmu_scramble_addr()
11177 static u64 gaudi2_mmu_descramble_addr(struct hl_device *hdev, u64 scrambled_addr) in gaudi2_mmu_descramble_addr()
11198 static u32 gaudi2_get_dec_base_addr(struct hl_device *hdev, u32 core_id) in gaudi2_get_dec_base_addr()
11221 static int gaudi2_get_hw_block_id(struct hl_device *hdev, u64 block_addr, in gaudi2_get_hw_block_id()
11241 static int gaudi2_block_mmap(struct hl_device *hdev, struct vm_area_struct *vma, in gaudi2_block_mmap()
11275 static void gaudi2_enable_events_from_fw(struct hl_device *hdev) in gaudi2_enable_events_from_fw()
11287 static int gaudi2_get_mmu_base(struct hl_device *hdev, u64 mmu_id, u32 *mmu_base) in gaudi2_get_mmu_base()
11348 static void gaudi2_ack_mmu_error(struct hl_device *hdev, u64 mmu_id) in gaudi2_ack_mmu_error()
11364 static int gaudi2_ack_mmu_page_fault_or_access_error(struct hl_device *hdev, u64 mmu_cap_mask) in gaudi2_ack_mmu_page_fault_or_access_error()
11408 static int gaudi2_gen_sync_to_engine_map(struct hl_device *hdev, struct hl_sync_to_engine_map *map) in gaudi2_gen_sync_to_engine_map()
11421 struct hl_device *hdev, struct hl_mon_state_dump *mon) in gaudi2_print_single_monitor()
11428 static int gaudi2_print_fences_single_engine(struct hl_device *hdev, u64 base_offset, in gaudi2_print_fences_single_engine()
11444 static void gaudi2_state_dump_init(struct hl_device *hdev) in gaudi2_state_dump_init()
11451 static u32 gaudi2_get_sob_addr(struct hl_device *hdev, u32 sob_id) in gaudi2_get_sob_addr()
11461 static void gaudi2_add_device_attr(struct hl_device *hdev, struct attribute_group *dev_clk_attr_grp, in gaudi2_add_device_attr()
11468 static int gaudi2_mmu_get_real_page_size(struct hl_device *hdev, struct hl_mmu_properties *mmu_prop, in gaudi2_mmu_get_real_page_size()
11503 static int gaudi2_get_monitor_dump(struct hl_device *hdev, void *data) in gaudi2_get_monitor_dump()
11508 int gaudi2_send_device_activity(struct hl_device *hdev, bool open) in gaudi2_send_device_activity()
11518 static u64 gaudi2_read_pte(struct hl_device *hdev, u64 addr) in gaudi2_read_pte()
11531 static void gaudi2_write_pte(struct hl_device *hdev, u64 addr, u64 val) in gaudi2_write_pte()
11641 void gaudi2_set_asic_funcs(struct hl_device *hdev) in gaudi2_set_asic_funcs()