Lines Matching +full:max +full:- +full:outbound +full:- +full:regions
1 // SPDX-License-Identifier: GPL-2.0
4 * Copyright 2020-2022 HabanaLabs, Ltd.
45 * since the code already has built-in support for binning of up to MAX_FAULTY_TPCS TPCs
47 * for MAX faulty TPCs which reflects the cluster binning requirements
126 #define GAUDI2_PMMU_SPI_SEI_ENABLE_MASK GENMASK(GAUDI2_NUM_OF_MMU_SPI_SEI_CAUSE - 2, 0)
127 #define GAUDI2_HMMU_SPI_SEI_ENABLE_MASK GENMASK(GAUDI2_NUM_OF_MMU_SPI_SEI_CAUSE - 1, 0)
131 #define GAUDI2_VDEC_MSIX_ENTRIES (GAUDI2_IRQ_NUM_SHARED_DEC1_ABNRM - \
134 #define ENGINE_ID_DCORE_OFFSET (GAUDI2_DCORE1_ENGINE_ID_EDMA_0 - GAUDI2_DCORE0_ENGINE_ID_EDMA_0)
164 /* HW scrambles only bits 0-25 */
796 "FENCE 0 inc over max value and clipped",
797 "FENCE 1 inc over max value and clipped",
798 "FENCE 2 inc over max value and clipped",
799 "FENCE 3 inc over max value and clipped",
817 "FENCE 0 inc over max value and clipped",
818 "FENCE 1 inc over max value and clipped",
819 "FENCE 2 inc over max value and clipped",
820 "FENCE 3 inc over max value and clipped",
2114 * and read global errors. Most HW blocks are addressable and those who aren't (N/A)-
2137 {HBM_MC_SPI_THR_ENG_MASK, "temperature-based throttling engaged"},
2138 {HBM_MC_SPI_THR_DIS_ENG_MASK, "temperature-based throttling disengaged"},
2165 {"mmu rei0", -1}, /* no clear register bit */
2166 {"mmu rei1", -1}, /* no clear register bit */
2167 {"stlb rei0", -1}, /* no clear register bit */
2168 {"stlb rei1", -1}, /* no clear register bit */
2237 struct asic_fixed_properties *prop = &hdev->asic_prop; in gaudi2_iterate_tpcs()
2242 ctx->rc = 0; in gaudi2_iterate_tpcs()
2248 if (!(prop->tpc_enabled_mask & BIT(tpc_seq))) in gaudi2_iterate_tpcs()
2253 ctx->fn(hdev, dcore, inst, offset, ctx); in gaudi2_iterate_tpcs()
2254 if (ctx->rc) { in gaudi2_iterate_tpcs()
2255 dev_err(hdev->dev, "TPC iterator failed for DCORE%d TPC%d\n", in gaudi2_iterate_tpcs()
2262 if (!(prop->tpc_enabled_mask & BIT(TPC_ID_DCORE0_TPC6))) in gaudi2_iterate_tpcs()
2266 offset = DCORE_TPC_OFFSET * (NUM_DCORE0_TPC - 1); in gaudi2_iterate_tpcs()
2267 ctx->fn(hdev, 0, NUM_DCORE0_TPC - 1, offset, ctx); in gaudi2_iterate_tpcs()
2268 if (ctx->rc) in gaudi2_iterate_tpcs()
2269 dev_err(hdev->dev, "TPC iterator failed for DCORE0 TPC6\n"); in gaudi2_iterate_tpcs()
2282 struct asic_fixed_properties *prop = &hdev->asic_prop; in set_number_of_functional_hbms()
2283 u8 faulty_hbms = hweight64(hdev->dram_binning); in set_number_of_functional_hbms()
2287 dev_dbg(hdev->dev, "All HBM are in use (no binning)\n"); in set_number_of_functional_hbms()
2288 prop->num_functional_hbms = GAUDI2_HBM_NUM; in set_number_of_functional_hbms()
2299 dev_err(hdev->dev, in set_number_of_functional_hbms()
2300 "HBM binning supports max of %d faulty HBMs, supplied mask 0x%llx.\n", in set_number_of_functional_hbms()
2301 MAX_FAULTY_HBMS, hdev->dram_binning); in set_number_of_functional_hbms()
2302 return -EINVAL; in set_number_of_functional_hbms()
2307 * GAUDI2_HBM_NUM - 1. in set_number_of_functional_hbms()
2309 prop->num_functional_hbms = GAUDI2_HBM_NUM - faulty_hbms; in set_number_of_functional_hbms()
2329 struct asic_fixed_properties *prop = &hdev->asic_prop; in gaudi2_set_dram_properties()
2336 return -EINVAL; in gaudi2_set_dram_properties()
2343 basic_hbm_page_size = prop->num_functional_hbms * SZ_8M; in gaudi2_set_dram_properties()
2344 prop->dram_page_size = GAUDI2_COMPENSATE_TLB_PAGE_SIZE_FACTOR * basic_hbm_page_size; in gaudi2_set_dram_properties()
2345 prop->device_mem_alloc_default_page_size = prop->dram_page_size; in gaudi2_set_dram_properties()
2346 prop->dram_size = prop->num_functional_hbms * SZ_16G; in gaudi2_set_dram_properties()
2347 prop->dram_base_address = DRAM_PHYS_BASE; in gaudi2_set_dram_properties()
2348 prop->dram_end_address = prop->dram_base_address + prop->dram_size; in gaudi2_set_dram_properties()
2349 prop->dram_supports_virtual_memory = true; in gaudi2_set_dram_properties()
2351 prop->dram_user_base_address = DRAM_PHYS_BASE + prop->dram_page_size; in gaudi2_set_dram_properties()
2352 prop->dram_hints_align_mask = ~GAUDI2_HBM_MMU_SCRM_ADDRESS_MASK; in gaudi2_set_dram_properties()
2353 prop->hints_dram_reserved_va_range.start_addr = RESERVED_VA_RANGE_FOR_ARC_ON_HBM_START; in gaudi2_set_dram_properties()
2354 prop->hints_dram_reserved_va_range.end_addr = RESERVED_VA_RANGE_FOR_ARC_ON_HBM_END; in gaudi2_set_dram_properties()
2362 * 1. partition the virtual address space to DRAM-page (whole) pages. in gaudi2_set_dram_properties()
2373 prop->dmmu.start_addr = prop->dram_base_address + in gaudi2_set_dram_properties()
2374 (prop->dram_page_size * in gaudi2_set_dram_properties()
2375 DIV_ROUND_UP_SECTOR_T(prop->dram_size, prop->dram_page_size)); in gaudi2_set_dram_properties()
2376 prop->dmmu.end_addr = prop->dmmu.start_addr + prop->dram_page_size * in gaudi2_set_dram_properties()
2377 div_u64((VA_HBM_SPACE_END - prop->dmmu.start_addr), prop->dmmu.page_size); in gaudi2_set_dram_properties()
2382 hbm_drv_base_offset = roundup(CPU_FW_IMAGE_SIZE, prop->num_functional_hbms * SZ_8M); in gaudi2_set_dram_properties()
2391 prop->mmu_pgt_addr = DRAM_PHYS_BASE + hbm_drv_base_offset + in gaudi2_set_dram_properties()
2392 ((prop->dram_page_size - hbm_drv_base_offset) - in gaudi2_set_dram_properties()
2396 edma_pq_base_addr = prop->mmu_pgt_addr + HMMU_PAGE_TABLES_SIZE; in gaudi2_set_dram_properties()
2400 prop->hw_queues_props[i].q_dram_bd_address = edma_pq_base_addr + in gaudi2_set_dram_properties()
2411 struct asic_fixed_properties *prop = &hdev->asic_prop; in gaudi2_set_fixed_properties()
2416 prop->max_queues = GAUDI2_QUEUE_ID_SIZE; in gaudi2_set_fixed_properties()
2417 prop->hw_queues_props = kcalloc(prop->max_queues, sizeof(struct hw_queue_properties), in gaudi2_set_fixed_properties()
2420 if (!prop->hw_queues_props) in gaudi2_set_fixed_properties()
2421 return -ENOMEM; in gaudi2_set_fixed_properties()
2423 q_props = prop->hw_queues_props; in gaudi2_set_fixed_properties()
2446 prop->cache_line_size = DEVICE_CACHE_LINE_SIZE; in gaudi2_set_fixed_properties()
2447 prop->cfg_base_address = CFG_BASE; in gaudi2_set_fixed_properties()
2448 prop->device_dma_offset_for_host_access = HOST_PHYS_BASE_0; in gaudi2_set_fixed_properties()
2449 prop->host_base_address = HOST_PHYS_BASE_0; in gaudi2_set_fixed_properties()
2450 prop->host_end_address = prop->host_base_address + HOST_PHYS_SIZE_0; in gaudi2_set_fixed_properties()
2451 prop->max_pending_cs = GAUDI2_MAX_PENDING_CS; in gaudi2_set_fixed_properties()
2452 prop->completion_queues_count = GAUDI2_RESERVED_CQ_NUMBER; in gaudi2_set_fixed_properties()
2453 prop->user_dec_intr_count = NUMBER_OF_DEC; in gaudi2_set_fixed_properties()
2454 prop->user_interrupt_count = GAUDI2_IRQ_NUM_USER_LAST - GAUDI2_IRQ_NUM_USER_FIRST + 1; in gaudi2_set_fixed_properties()
2455 prop->completion_mode = HL_COMPLETION_MODE_CS; in gaudi2_set_fixed_properties()
2456 prop->sync_stream_first_sob = GAUDI2_RESERVED_SOB_NUMBER; in gaudi2_set_fixed_properties()
2457 prop->sync_stream_first_mon = GAUDI2_RESERVED_MON_NUMBER; in gaudi2_set_fixed_properties()
2459 prop->sram_base_address = SRAM_BASE_ADDR; in gaudi2_set_fixed_properties()
2460 prop->sram_size = SRAM_SIZE; in gaudi2_set_fixed_properties()
2461 prop->sram_end_address = prop->sram_base_address + prop->sram_size; in gaudi2_set_fixed_properties()
2462 prop->sram_user_base_address = prop->sram_base_address + SRAM_USER_BASE_OFFSET; in gaudi2_set_fixed_properties()
2464 prop->hints_range_reservation = true; in gaudi2_set_fixed_properties()
2466 prop->rotator_enabled_mask = BIT(NUM_OF_ROT) - 1; in gaudi2_set_fixed_properties()
2468 prop->max_asid = 2; in gaudi2_set_fixed_properties()
2470 prop->dmmu.pgt_size = HMMU_PAGE_TABLES_SIZE; in gaudi2_set_fixed_properties()
2471 prop->mmu_pte_size = HL_PTE_SIZE; in gaudi2_set_fixed_properties()
2473 prop->dmmu.hop_shifts[MMU_HOP0] = DHOP0_SHIFT; in gaudi2_set_fixed_properties()
2474 prop->dmmu.hop_shifts[MMU_HOP1] = DHOP1_SHIFT; in gaudi2_set_fixed_properties()
2475 prop->dmmu.hop_shifts[MMU_HOP2] = DHOP2_SHIFT; in gaudi2_set_fixed_properties()
2476 prop->dmmu.hop_shifts[MMU_HOP3] = DHOP3_SHIFT; in gaudi2_set_fixed_properties()
2477 prop->dmmu.hop_masks[MMU_HOP0] = DHOP0_MASK; in gaudi2_set_fixed_properties()
2478 prop->dmmu.hop_masks[MMU_HOP1] = DHOP1_MASK; in gaudi2_set_fixed_properties()
2479 prop->dmmu.hop_masks[MMU_HOP2] = DHOP2_MASK; in gaudi2_set_fixed_properties()
2480 prop->dmmu.hop_masks[MMU_HOP3] = DHOP3_MASK; in gaudi2_set_fixed_properties()
2481 prop->dmmu.page_size = PAGE_SIZE_1GB; in gaudi2_set_fixed_properties()
2482 prop->dmmu.num_hops = MMU_ARCH_4_HOPS; in gaudi2_set_fixed_properties()
2483 prop->dmmu.last_mask = LAST_MASK; in gaudi2_set_fixed_properties()
2484 prop->dmmu.host_resident = 0; in gaudi2_set_fixed_properties()
2485 prop->dmmu.hop_table_size = HOP_TABLE_SIZE_512_PTE; in gaudi2_set_fixed_properties()
2486 prop->dmmu.hop0_tables_total_size = HOP_TABLE_SIZE_512_PTE * prop->max_asid; in gaudi2_set_fixed_properties()
2492 rc = hdev->asic_funcs->set_dram_properties(hdev); in gaudi2_set_fixed_properties()
2496 prop->mmu_pgt_size = PMMU_PAGE_TABLES_SIZE; in gaudi2_set_fixed_properties()
2498 prop->pmmu.pgt_size = prop->mmu_pgt_size; in gaudi2_set_fixed_properties()
2499 hdev->pmmu_huge_range = true; in gaudi2_set_fixed_properties()
2500 prop->pmmu.host_resident = 1; in gaudi2_set_fixed_properties()
2501 prop->pmmu.num_hops = MMU_ARCH_6_HOPS; in gaudi2_set_fixed_properties()
2502 prop->pmmu.last_mask = LAST_MASK; in gaudi2_set_fixed_properties()
2503 prop->pmmu.hop_table_size = HOP_TABLE_SIZE_512_PTE; in gaudi2_set_fixed_properties()
2504 prop->pmmu.hop0_tables_total_size = HOP_TABLE_SIZE_512_PTE * prop->max_asid; in gaudi2_set_fixed_properties()
2506 prop->hints_host_reserved_va_range.start_addr = RESERVED_VA_FOR_VIRTUAL_MSIX_DOORBELL_START; in gaudi2_set_fixed_properties()
2507 prop->hints_host_reserved_va_range.end_addr = RESERVED_VA_RANGE_FOR_ARC_ON_HOST_END; in gaudi2_set_fixed_properties()
2508 prop->hints_host_hpage_reserved_va_range.start_addr = in gaudi2_set_fixed_properties()
2510 prop->hints_host_hpage_reserved_va_range.end_addr = in gaudi2_set_fixed_properties()
2514 prop->pmmu.hop_shifts[MMU_HOP0] = HOP0_SHIFT_64K; in gaudi2_set_fixed_properties()
2515 prop->pmmu.hop_shifts[MMU_HOP1] = HOP1_SHIFT_64K; in gaudi2_set_fixed_properties()
2516 prop->pmmu.hop_shifts[MMU_HOP2] = HOP2_SHIFT_64K; in gaudi2_set_fixed_properties()
2517 prop->pmmu.hop_shifts[MMU_HOP3] = HOP3_SHIFT_64K; in gaudi2_set_fixed_properties()
2518 prop->pmmu.hop_shifts[MMU_HOP4] = HOP4_SHIFT_64K; in gaudi2_set_fixed_properties()
2519 prop->pmmu.hop_shifts[MMU_HOP5] = HOP5_SHIFT_64K; in gaudi2_set_fixed_properties()
2520 prop->pmmu.hop_masks[MMU_HOP0] = HOP0_MASK_64K; in gaudi2_set_fixed_properties()
2521 prop->pmmu.hop_masks[MMU_HOP1] = HOP1_MASK_64K; in gaudi2_set_fixed_properties()
2522 prop->pmmu.hop_masks[MMU_HOP2] = HOP2_MASK_64K; in gaudi2_set_fixed_properties()
2523 prop->pmmu.hop_masks[MMU_HOP3] = HOP3_MASK_64K; in gaudi2_set_fixed_properties()
2524 prop->pmmu.hop_masks[MMU_HOP4] = HOP4_MASK_64K; in gaudi2_set_fixed_properties()
2525 prop->pmmu.hop_masks[MMU_HOP5] = HOP5_MASK_64K; in gaudi2_set_fixed_properties()
2526 prop->pmmu.start_addr = VA_HOST_SPACE_PAGE_START; in gaudi2_set_fixed_properties()
2527 prop->pmmu.end_addr = VA_HOST_SPACE_PAGE_END; in gaudi2_set_fixed_properties()
2528 prop->pmmu.page_size = PAGE_SIZE_64KB; in gaudi2_set_fixed_properties()
2531 memcpy(&prop->pmmu_huge, &prop->pmmu, sizeof(prop->pmmu)); in gaudi2_set_fixed_properties()
2532 prop->pmmu_huge.page_size = PAGE_SIZE_16MB; in gaudi2_set_fixed_properties()
2533 prop->pmmu_huge.start_addr = VA_HOST_SPACE_HPAGE_START; in gaudi2_set_fixed_properties()
2534 prop->pmmu_huge.end_addr = VA_HOST_SPACE_HPAGE_END; in gaudi2_set_fixed_properties()
2536 prop->pmmu.hop_shifts[MMU_HOP0] = HOP0_SHIFT_4K; in gaudi2_set_fixed_properties()
2537 prop->pmmu.hop_shifts[MMU_HOP1] = HOP1_SHIFT_4K; in gaudi2_set_fixed_properties()
2538 prop->pmmu.hop_shifts[MMU_HOP2] = HOP2_SHIFT_4K; in gaudi2_set_fixed_properties()
2539 prop->pmmu.hop_shifts[MMU_HOP3] = HOP3_SHIFT_4K; in gaudi2_set_fixed_properties()
2540 prop->pmmu.hop_shifts[MMU_HOP4] = HOP4_SHIFT_4K; in gaudi2_set_fixed_properties()
2541 prop->pmmu.hop_shifts[MMU_HOP5] = HOP5_SHIFT_4K; in gaudi2_set_fixed_properties()
2542 prop->pmmu.hop_masks[MMU_HOP0] = HOP0_MASK_4K; in gaudi2_set_fixed_properties()
2543 prop->pmmu.hop_masks[MMU_HOP1] = HOP1_MASK_4K; in gaudi2_set_fixed_properties()
2544 prop->pmmu.hop_masks[MMU_HOP2] = HOP2_MASK_4K; in gaudi2_set_fixed_properties()
2545 prop->pmmu.hop_masks[MMU_HOP3] = HOP3_MASK_4K; in gaudi2_set_fixed_properties()
2546 prop->pmmu.hop_masks[MMU_HOP4] = HOP4_MASK_4K; in gaudi2_set_fixed_properties()
2547 prop->pmmu.hop_masks[MMU_HOP5] = HOP5_MASK_4K; in gaudi2_set_fixed_properties()
2548 prop->pmmu.start_addr = VA_HOST_SPACE_PAGE_START; in gaudi2_set_fixed_properties()
2549 prop->pmmu.end_addr = VA_HOST_SPACE_PAGE_END; in gaudi2_set_fixed_properties()
2550 prop->pmmu.page_size = PAGE_SIZE_4KB; in gaudi2_set_fixed_properties()
2553 memcpy(&prop->pmmu_huge, &prop->pmmu, sizeof(prop->pmmu)); in gaudi2_set_fixed_properties()
2554 prop->pmmu_huge.page_size = PAGE_SIZE_2MB; in gaudi2_set_fixed_properties()
2555 prop->pmmu_huge.start_addr = VA_HOST_SPACE_HPAGE_START; in gaudi2_set_fixed_properties()
2556 prop->pmmu_huge.end_addr = VA_HOST_SPACE_HPAGE_END; in gaudi2_set_fixed_properties()
2559 prop->max_num_of_engines = GAUDI2_ENGINE_ID_SIZE; in gaudi2_set_fixed_properties()
2560 prop->num_engine_cores = CPU_ID_MAX; in gaudi2_set_fixed_properties()
2561 prop->cfg_size = CFG_SIZE; in gaudi2_set_fixed_properties()
2562 prop->num_of_events = GAUDI2_EVENT_SIZE; in gaudi2_set_fixed_properties()
2564 prop->supports_engine_modes = true; in gaudi2_set_fixed_properties()
2566 prop->dc_power_default = DC_POWER_DEFAULT; in gaudi2_set_fixed_properties()
2568 prop->cb_pool_cb_cnt = GAUDI2_CB_POOL_CB_CNT; in gaudi2_set_fixed_properties()
2569 prop->cb_pool_cb_size = GAUDI2_CB_POOL_CB_SIZE; in gaudi2_set_fixed_properties()
2570 prop->pcie_dbi_base_address = CFG_BASE + mmPCIE_DBI_BASE; in gaudi2_set_fixed_properties()
2571 prop->pcie_aux_dbi_reg_addr = CFG_BASE + mmPCIE_AUX_DBI; in gaudi2_set_fixed_properties()
2573 strscpy_pad(prop->cpucp_info.card_name, GAUDI2_DEFAULT_CARD_NAME, CARD_NAME_MAX_LEN); in gaudi2_set_fixed_properties()
2575 prop->mme_master_slave_mode = 1; in gaudi2_set_fixed_properties()
2577 prop->first_available_user_sob[0] = GAUDI2_RESERVED_SOB_NUMBER + in gaudi2_set_fixed_properties()
2580 prop->first_available_user_mon[0] = GAUDI2_RESERVED_MON_NUMBER + in gaudi2_set_fixed_properties()
2583 prop->first_available_user_interrupt = GAUDI2_IRQ_NUM_USER_FIRST; in gaudi2_set_fixed_properties()
2584 prop->tpc_interrupt_id = GAUDI2_IRQ_NUM_TPC_ASSERT; in gaudi2_set_fixed_properties()
2585 prop->eq_interrupt_id = GAUDI2_IRQ_NUM_EVENT_QUEUE; in gaudi2_set_fixed_properties()
2587 prop->first_available_cq[0] = GAUDI2_RESERVED_CQ_NUMBER; in gaudi2_set_fixed_properties()
2589 prop->fw_cpu_boot_dev_sts0_valid = false; in gaudi2_set_fixed_properties()
2590 prop->fw_cpu_boot_dev_sts1_valid = false; in gaudi2_set_fixed_properties()
2591 prop->hard_reset_done_by_fw = false; in gaudi2_set_fixed_properties()
2592 prop->gic_interrupts_enable = true; in gaudi2_set_fixed_properties()
2594 prop->server_type = HL_SERVER_TYPE_UNKNOWN; in gaudi2_set_fixed_properties()
2596 prop->max_dec = NUMBER_OF_DEC; in gaudi2_set_fixed_properties()
2598 prop->clk_pll_index = HL_GAUDI2_MME_PLL; in gaudi2_set_fixed_properties()
2600 prop->dma_mask = 64; in gaudi2_set_fixed_properties()
2602 prop->hbw_flush_reg = mmPCIE_WRAP_SPECIAL_GLBL_SPARE_0; in gaudi2_set_fixed_properties()
2604 prop->supports_advanced_cpucp_rc = true; in gaudi2_set_fixed_properties()
2609 kfree(prop->hw_queues_props); in gaudi2_set_fixed_properties()
2623 hdev->rmmio = hdev->pcie_bar[SRAM_CFG_BAR_ID] + (CFG_BASE - STM_FLASH_BASE_ADDR); in gaudi2_pci_bars_map()
2630 struct gaudi2_device *gaudi2 = hdev->asic_specific; in gaudi2_set_hbm_bar_base()
2635 if ((gaudi2) && (gaudi2->dram_bar_cur_addr == addr)) in gaudi2_set_hbm_bar_base()
2638 if (hdev->asic_prop.iatu_done_by_fw) in gaudi2_set_hbm_bar_base()
2641 /* Inbound Region 2 - Bar 4 - Point to DRAM */ in gaudi2_set_hbm_bar_base()
2650 old_addr = gaudi2->dram_bar_cur_addr; in gaudi2_set_hbm_bar_base()
2651 gaudi2->dram_bar_cur_addr = addr; in gaudi2_set_hbm_bar_base()
2664 if (hdev->asic_prop.iatu_done_by_fw) in gaudi2_init_iatu()
2667 /* Temporary inbound Region 0 - Bar 0 - Point to CFG in gaudi2_init_iatu()
2674 inbound_region.addr = STM_FLASH_BASE_ADDR - STM_FLASH_ALIGNED_OFF; in gaudi2_init_iatu()
2683 hdev->pcie_bar_phys[SRAM_CFG_BAR_ID] = (u64)bar_addr_high << 32 | bar_addr_low; in gaudi2_init_iatu()
2685 /* Inbound Region 0 - Bar 0 - Point to CFG */ in gaudi2_init_iatu()
2695 /* Inbound Region 1 - Bar 0 - Point to BAR0_RESERVED + SRAM */ in gaudi2_init_iatu()
2705 /* Inbound Region 2 - Bar 4 - Point to DRAM */ in gaudi2_init_iatu()
2713 /* Outbound Region 0 - Point to Host */ in gaudi2_init_iatu()
2728 struct asic_fixed_properties *prop = &hdev->asic_prop; in gaudi2_tpc_binning_init_prop()
2734 if (hweight64(hdev->tpc_binning) > MAX_CLUSTER_BINNING_FAULTY_TPCS) { in gaudi2_tpc_binning_init_prop()
2735 dev_err(hdev->dev, "TPC binning is supported for max of %d faulty TPCs, provided mask 0x%llx\n", in gaudi2_tpc_binning_init_prop()
2737 hdev->tpc_binning); in gaudi2_tpc_binning_init_prop()
2738 return -EINVAL; in gaudi2_tpc_binning_init_prop()
2741 prop->tpc_binning_mask = hdev->tpc_binning; in gaudi2_tpc_binning_init_prop()
2742 prop->tpc_enabled_mask = GAUDI2_TPC_FULL_MASK; in gaudi2_tpc_binning_init_prop()
2749 struct asic_fixed_properties *prop = &hdev->asic_prop; in gaudi2_set_tpc_binning_masks()
2750 struct hw_queue_properties *q_props = prop->hw_queues_props; in gaudi2_set_tpc_binning_masks()
2759 tpc_binning_mask = prop->tpc_binning_mask; in gaudi2_set_tpc_binning_masks()
2779 * Coverity complains about possible out-of-bound access in in gaudi2_set_tpc_binning_masks()
2783 dev_err(hdev->dev, in gaudi2_set_tpc_binning_masks()
2786 return -EINVAL; in gaudi2_set_tpc_binning_masks()
2791 clear_bit(subst_seq, (unsigned long *)&prop->tpc_enabled_mask); in gaudi2_set_tpc_binning_masks()
2807 struct asic_fixed_properties *prop = &hdev->asic_prop; in gaudi2_set_dec_binning_masks()
2810 num_faulty = hweight32(hdev->decoder_binning); in gaudi2_set_dec_binning_masks()
2817 …dev_err(hdev->dev, "decoder binning is supported for max of single faulty decoder, provided mask 0… in gaudi2_set_dec_binning_masks()
2818 hdev->decoder_binning); in gaudi2_set_dec_binning_masks()
2819 return -EINVAL; in gaudi2_set_dec_binning_masks()
2822 prop->decoder_binning_mask = (hdev->decoder_binning & GAUDI2_DECODER_FULL_MASK); in gaudi2_set_dec_binning_masks()
2824 if (prop->decoder_binning_mask) in gaudi2_set_dec_binning_masks()
2825 prop->decoder_enabled_mask = (GAUDI2_DECODER_FULL_MASK & ~BIT(DEC_ID_PCIE_VDEC1)); in gaudi2_set_dec_binning_masks()
2827 prop->decoder_enabled_mask = GAUDI2_DECODER_FULL_MASK; in gaudi2_set_dec_binning_masks()
2834 struct asic_fixed_properties *prop = &hdev->asic_prop; in gaudi2_set_dram_binning_masks()
2837 if (!hdev->dram_binning) { in gaudi2_set_dram_binning_masks()
2838 prop->dram_binning_mask = 0; in gaudi2_set_dram_binning_masks()
2839 prop->dram_enabled_mask = GAUDI2_DRAM_FULL_MASK; in gaudi2_set_dram_binning_masks()
2844 prop->faulty_dram_cluster_map |= hdev->dram_binning; in gaudi2_set_dram_binning_masks()
2845 prop->dram_binning_mask = hdev->dram_binning; in gaudi2_set_dram_binning_masks()
2846 prop->dram_enabled_mask = GAUDI2_DRAM_FULL_MASK & ~BIT(HBM_ID5); in gaudi2_set_dram_binning_masks()
2851 struct asic_fixed_properties *prop = &hdev->asic_prop; in gaudi2_set_edma_binning_masks()
2855 num_faulty = hweight32(hdev->edma_binning); in gaudi2_set_edma_binning_masks()
2862 dev_err(hdev->dev, in gaudi2_set_edma_binning_masks()
2863 "EDMA binning is supported for max of single faulty EDMA, provided mask 0x%x\n", in gaudi2_set_edma_binning_masks()
2864 hdev->edma_binning); in gaudi2_set_edma_binning_masks()
2865 return -EINVAL; in gaudi2_set_edma_binning_masks()
2868 if (!hdev->edma_binning) { in gaudi2_set_edma_binning_masks()
2869 prop->edma_binning_mask = 0; in gaudi2_set_edma_binning_masks()
2870 prop->edma_enabled_mask = GAUDI2_EDMA_FULL_MASK; in gaudi2_set_edma_binning_masks()
2874 seq = __ffs((unsigned long)hdev->edma_binning); in gaudi2_set_edma_binning_masks()
2877 prop->faulty_dram_cluster_map |= BIT(edma_to_hbm_cluster[seq]); in gaudi2_set_edma_binning_masks()
2878 prop->edma_binning_mask = hdev->edma_binning; in gaudi2_set_edma_binning_masks()
2879 prop->edma_enabled_mask = GAUDI2_EDMA_FULL_MASK & ~BIT(EDMA_ID_DCORE3_INSTANCE1); in gaudi2_set_edma_binning_masks()
2882 q_props = prop->hw_queues_props; in gaudi2_set_edma_binning_masks()
2893 struct asic_fixed_properties *prop = &hdev->asic_prop; in gaudi2_set_xbar_edge_enable_mask()
2898 prop->xbar_edge_enabled_mask = GAUDI2_XBAR_EDGE_FULL_MASK; in gaudi2_set_xbar_edge_enable_mask()
2913 dev_err(hdev->dev, "we cannot have more than %d faulty XBAR EDGE\n", in gaudi2_set_xbar_edge_enable_mask()
2915 return -EINVAL; in gaudi2_set_xbar_edge_enable_mask()
2921 prop->faulty_dram_cluster_map |= BIT(xbar_edge_to_hbm_cluster[seq]); in gaudi2_set_xbar_edge_enable_mask()
2922 prop->xbar_edge_enabled_mask = (~xbar_edge_iso_mask) & GAUDI2_XBAR_EDGE_FULL_MASK; in gaudi2_set_xbar_edge_enable_mask()
2934 * If more than single cluster is faulty- the chip is unusable in gaudi2_set_cluster_binning_masks_common()
2936 hdev->asic_prop.faulty_dram_cluster_map = 0; in gaudi2_set_cluster_binning_masks_common()
2950 hdev->asic_prop.hmmu_hif_enabled_mask = GAUDI2_HIF_HMMU_FULL_MASK; in gaudi2_set_cluster_binning_masks_common()
2957 struct asic_fixed_properties *prop = &hdev->asic_prop; in gaudi2_set_cluster_binning_masks()
2960 rc = gaudi2_set_cluster_binning_masks_common(hdev, prop->cpucp_info.xbar_binning_mask); in gaudi2_set_cluster_binning_masks()
2965 if (prop->faulty_dram_cluster_map) { in gaudi2_set_cluster_binning_masks()
2966 u8 cluster_seq = __ffs((unsigned long)prop->faulty_dram_cluster_map); in gaudi2_set_cluster_binning_masks()
2968 prop->hmmu_hif_enabled_mask = cluster_hmmu_hif_enabled_mask[cluster_seq]; in gaudi2_set_cluster_binning_masks()
2995 struct gaudi2_device *gaudi2 = hdev->asic_specific; in gaudi2_cpucp_info_get()
2996 struct asic_fixed_properties *prop = &hdev->asic_prop; in gaudi2_cpucp_info_get()
3001 if (!(gaudi2->hw_cap_initialized & HW_CAP_CPU_Q)) in gaudi2_cpucp_info_get()
3007 if (hdev->reset_info.in_compute_reset) in gaudi2_cpucp_info_get()
3015 dram_size = le64_to_cpu(prop->cpucp_info.dram_size); in gaudi2_cpucp_info_get()
3019 if ((dram_size != ((GAUDI2_HBM_NUM - 1) * SZ_16G)) && in gaudi2_cpucp_info_get()
3021 dev_err(hdev->dev, in gaudi2_cpucp_info_get()
3023 dram_size, prop->dram_size); in gaudi2_cpucp_info_get()
3024 dram_size = prop->dram_size; in gaudi2_cpucp_info_get()
3027 prop->dram_size = dram_size; in gaudi2_cpucp_info_get()
3028 prop->dram_end_address = prop->dram_base_address + dram_size; in gaudi2_cpucp_info_get()
3031 if (!strlen(prop->cpucp_info.card_name)) in gaudi2_cpucp_info_get()
3032 strscpy_pad(prop->cpucp_info.card_name, GAUDI2_DEFAULT_CARD_NAME, in gaudi2_cpucp_info_get()
3036 hdev->dram_binning = prop->cpucp_info.dram_binning_mask; in gaudi2_cpucp_info_get()
3037 hdev->edma_binning = prop->cpucp_info.edma_binning_mask; in gaudi2_cpucp_info_get()
3038 hdev->tpc_binning = le64_to_cpu(prop->cpucp_info.tpc_binning_mask); in gaudi2_cpucp_info_get()
3039 hdev->decoder_binning = lower_32_bits(le64_to_cpu(prop->cpucp_info.decoder_binning_mask)); in gaudi2_cpucp_info_get()
3041 dev_dbg(hdev->dev, "Read binning masks: tpc: 0x%llx, dram: 0x%llx, edma: 0x%x, dec: 0x%x\n", in gaudi2_cpucp_info_get()
3042 hdev->tpc_binning, hdev->dram_binning, hdev->edma_binning, in gaudi2_cpucp_info_get()
3043 hdev->decoder_binning); in gaudi2_cpucp_info_get()
3049 rc = hdev->asic_funcs->set_dram_properties(hdev); in gaudi2_cpucp_info_get()
3053 rc = hdev->asic_funcs->set_binning_masks(hdev); in gaudi2_cpucp_info_get()
3061 prop->max_power_default = (u64) max_power; in gaudi2_cpucp_info_get()
3068 struct gaudi2_device *gaudi2 = hdev->asic_specific; in gaudi2_fetch_psoc_frequency()
3072 if (!(gaudi2->hw_cap_initialized & HW_CAP_CPU_Q)) in gaudi2_fetch_psoc_frequency()
3079 hdev->asic_prop.psoc_timestamp_frequency = pll_freq_arr[3]; in gaudi2_fetch_psoc_frequency()
3086 struct gaudi2_device *gaudi2 = hdev->asic_specific; in gaudi2_mmu_clear_pgt_range()
3087 struct asic_fixed_properties *prop = &hdev->asic_prop; in gaudi2_mmu_clear_pgt_range()
3090 if (!(gaudi2->hw_cap_initialized & HW_CAP_MMU_MASK)) in gaudi2_mmu_clear_pgt_range()
3093 if (prop->dmmu.host_resident) in gaudi2_mmu_clear_pgt_range()
3096 rc = gaudi2_memset_device_memory(hdev, prop->mmu_pgt_addr, prop->dmmu.pgt_size, 0); in gaudi2_mmu_clear_pgt_range()
3098 dev_err(hdev->dev, "Failed to clear mmu pgt"); in gaudi2_mmu_clear_pgt_range()
3105 struct asic_fixed_properties *prop = &hdev->asic_prop; in gaudi2_early_init()
3106 struct pci_dev *pdev = hdev->pdev; in gaudi2_early_init()
3118 dev_err(hdev->dev, "Not " HL_NAME "? BAR %d size %pa, expecting %llu\n", in gaudi2_early_init()
3120 rc = -ENODEV; in gaudi2_early_init()
3126 dev_err(hdev->dev, "Not " HL_NAME "? BAR %d size %pa, expecting %llu\n", in gaudi2_early_init()
3128 rc = -ENODEV; in gaudi2_early_init()
3132 prop->dram_pci_bar_size = pci_resource_len(pdev, DRAM_BAR_ID); in gaudi2_early_init()
3133 hdev->dram_pci_bar_start = pci_resource_start(pdev, DRAM_BAR_ID); in gaudi2_early_init()
3138 if (hdev->pldm) in gaudi2_early_init()
3139 hdev->asic_prop.iatu_done_by_fw = false; in gaudi2_early_init()
3141 hdev->asic_prop.iatu_done_by_fw = true; in gaudi2_early_init()
3148 * version to determine whether we run with a security-enabled firmware in gaudi2_early_init()
3152 if (hdev->reset_on_preboot_fail) in gaudi2_early_init()
3154 hdev->asic_funcs->hw_fini(hdev, true, false); in gaudi2_early_init()
3159 dev_dbg(hdev->dev, "H/W state is dirty, must reset before initializing\n"); in gaudi2_early_init()
3160 rc = hdev->asic_funcs->hw_fini(hdev, true, false); in gaudi2_early_init()
3162 dev_err(hdev->dev, "failed to reset HW in dirty state (%d)\n", rc); in gaudi2_early_init()
3172 kfree(hdev->asic_prop.hw_queues_props); in gaudi2_early_init()
3178 kfree(hdev->asic_prop.hw_queues_props); in gaudi2_early_fini()
3206 struct cpu_dyn_regs *dyn_regs = &hdev->fw_loader.dynamic_loader.comm_desc.cpu_dyn_regs; in gaudi2_init_arcs()
3207 struct gaudi2_device *gaudi2 = hdev->asic_specific; in gaudi2_init_arcs()
3227 !(hdev->nic_ports_mask & BIT_ULL(arc_id - CPU_ID_NIC_QMAN_ARC0))) in gaudi2_init_arcs()
3230 if (gaudi2_is_arc_tpc_owned(arc_id) && !(gaudi2->tpc_hw_cap_initialized & in gaudi2_init_arcs()
3231 BIT_ULL(arc_id - CPU_ID_TPC_QMAN_ARC0))) in gaudi2_init_arcs()
3238 hdev->asic_prop.engine_core_interrupt_reg_addr = in gaudi2_init_arcs()
3239 CFG_BASE + le32_to_cpu(dyn_regs->eng_arc_irq_ctrl); in gaudi2_init_arcs()
3310 struct gaudi2_device *gaudi2 = hdev->asic_specific; in gaudi2_late_init()
3314 gaudi2->virt_msix_db_dma_addr); in gaudi2_late_init()
3320 dev_err(hdev->dev, "Failed to fetch psoc frequency\n"); in gaudi2_late_init()
3326 dev_err(hdev->dev, "Failed to clear MMU page tables range\n"); in gaudi2_late_init()
3334 dev_err(hdev->dev, "Failed to scrub arcs DCCM\n"); in gaudi2_late_init()
3355 struct user_mapped_block *blocks = gaudi2->mapped_blocks; in gaudi2_user_mapped_dec_init()
3371 struct gaudi2_device *gaudi2 = hdev->asic_specific; in gaudi2_user_mapped_blocks_init()
3372 struct user_mapped_block *blocks = gaudi2->mapped_blocks; in gaudi2_user_mapped_blocks_init()
3430 blocks[USR_MAPPED_BLK_SM_START_IDX + 2 * (i - 1)].size = SM_OBJS_BLOCK_SIZE; in gaudi2_user_mapped_blocks_init()
3431 blocks[USR_MAPPED_BLK_SM_START_IDX + 2 * (i - 1) + 1].size = HL_BLOCK_SIZE; in gaudi2_user_mapped_blocks_init()
3433 blocks[USR_MAPPED_BLK_SM_START_IDX + 2 * (i - 1)].address = in gaudi2_user_mapped_blocks_init()
3436 blocks[USR_MAPPED_BLK_SM_START_IDX + 2 * (i - 1) + 1].address = in gaudi2_user_mapped_blocks_init()
3447 /* The device ARC works with 32-bits addresses, and because there is a single HW register in gaudi2_alloc_cpu_accessible_dma_mem()
3456 rc = -ENOMEM; in gaudi2_alloc_cpu_accessible_dma_mem()
3460 end_addr = dma_addr_arr[i] + HL_CPU_ACCESSIBLE_MEM_SIZE - 1; in gaudi2_alloc_cpu_accessible_dma_mem()
3466 dev_err(hdev->dev, in gaudi2_alloc_cpu_accessible_dma_mem()
3468 rc = -EFAULT; in gaudi2_alloc_cpu_accessible_dma_mem()
3472 hdev->cpu_accessible_dma_mem = virt_addr_arr[i]; in gaudi2_alloc_cpu_accessible_dma_mem()
3473 hdev->cpu_accessible_dma_address = dma_addr_arr[i]; in gaudi2_alloc_cpu_accessible_dma_mem()
3485 struct asic_fixed_properties *prop = &hdev->asic_prop; in gaudi2_set_pci_memory_regions()
3489 region = &hdev->pci_mem_region[PCI_REGION_CFG]; in gaudi2_set_pci_memory_regions()
3490 region->region_base = CFG_BASE; in gaudi2_set_pci_memory_regions()
3491 region->region_size = CFG_SIZE; in gaudi2_set_pci_memory_regions()
3492 region->offset_in_bar = CFG_BASE - STM_FLASH_BASE_ADDR; in gaudi2_set_pci_memory_regions()
3493 region->bar_size = CFG_BAR_SIZE; in gaudi2_set_pci_memory_regions()
3494 region->bar_id = SRAM_CFG_BAR_ID; in gaudi2_set_pci_memory_regions()
3495 region->used = 1; in gaudi2_set_pci_memory_regions()
3498 region = &hdev->pci_mem_region[PCI_REGION_SRAM]; in gaudi2_set_pci_memory_regions()
3499 region->region_base = SRAM_BASE_ADDR; in gaudi2_set_pci_memory_regions()
3500 region->region_size = SRAM_SIZE; in gaudi2_set_pci_memory_regions()
3501 region->offset_in_bar = CFG_REGION_SIZE + BAR0_RSRVD_SIZE; in gaudi2_set_pci_memory_regions()
3502 region->bar_size = CFG_BAR_SIZE; in gaudi2_set_pci_memory_regions()
3503 region->bar_id = SRAM_CFG_BAR_ID; in gaudi2_set_pci_memory_regions()
3504 region->used = 1; in gaudi2_set_pci_memory_regions()
3507 region = &hdev->pci_mem_region[PCI_REGION_DRAM]; in gaudi2_set_pci_memory_regions()
3508 region->region_base = DRAM_PHYS_BASE; in gaudi2_set_pci_memory_regions()
3509 region->region_size = hdev->asic_prop.dram_size; in gaudi2_set_pci_memory_regions()
3510 region->offset_in_bar = 0; in gaudi2_set_pci_memory_regions()
3511 region->bar_size = prop->dram_pci_bar_size; in gaudi2_set_pci_memory_regions()
3512 region->bar_id = DRAM_BAR_ID; in gaudi2_set_pci_memory_regions()
3513 region->used = 1; in gaudi2_set_pci_memory_regions()
3518 struct asic_fixed_properties *prop = &hdev->asic_prop; in gaudi2_user_interrupt_setup()
3522 HL_USR_INTR_STRUCT_INIT(hdev->tpc_interrupt, hdev, 0, HL_USR_INTERRUPT_TPC); in gaudi2_user_interrupt_setup()
3525 HL_USR_INTR_STRUCT_INIT(hdev->unexpected_error_interrupt, hdev, 0, in gaudi2_user_interrupt_setup()
3529 HL_USR_INTR_STRUCT_INIT(hdev->common_user_cq_interrupt, hdev, in gaudi2_user_interrupt_setup()
3533 HL_USR_INTR_STRUCT_INIT(hdev->common_decoder_interrupt, hdev, in gaudi2_user_interrupt_setup()
3547 HL_USR_INTR_STRUCT_INIT(hdev->user_interrupt[j], hdev, i, in gaudi2_user_interrupt_setup()
3550 for (i = GAUDI2_IRQ_NUM_USER_FIRST, k = 0 ; k < prop->user_interrupt_count; i++, j++, k++) in gaudi2_user_interrupt_setup()
3551 HL_USR_INTR_STRUCT_INIT(hdev->user_interrupt[j], hdev, i, HL_USR_INTERRUPT_CQ); in gaudi2_user_interrupt_setup()
3563 struct asic_fixed_properties *prop = &hdev->asic_prop; in gaudi2_special_blocks_free()
3565 &prop->skip_special_blocks_cfg; in gaudi2_special_blocks_free()
3567 kfree(prop->special_blocks); in gaudi2_special_blocks_free()
3568 kfree(skip_special_blocks_cfg->block_types); in gaudi2_special_blocks_free()
3569 kfree(skip_special_blocks_cfg->block_ranges); in gaudi2_special_blocks_free()
3586 struct asic_fixed_properties *prop = &hdev->asic_prop; in gaudi2_special_blocks_config()
3590 prop->glbl_err_max_cause_num = GAUDI2_GLBL_ERR_MAX_CAUSE_NUM; in gaudi2_special_blocks_config()
3591 prop->num_of_special_blocks = ARRAY_SIZE(gaudi2_special_blocks); in gaudi2_special_blocks_config()
3592 prop->special_blocks = kmalloc_array(prop->num_of_special_blocks, in gaudi2_special_blocks_config()
3593 sizeof(*prop->special_blocks), GFP_KERNEL); in gaudi2_special_blocks_config()
3594 if (!prop->special_blocks) in gaudi2_special_blocks_config()
3595 return -ENOMEM; in gaudi2_special_blocks_config()
3597 for (i = 0 ; i < prop->num_of_special_blocks ; i++) in gaudi2_special_blocks_config()
3598 memcpy(&prop->special_blocks[i], &gaudi2_special_blocks[i], in gaudi2_special_blocks_config()
3599 sizeof(*prop->special_blocks)); in gaudi2_special_blocks_config()
3602 memset(&prop->skip_special_blocks_cfg, 0, sizeof(prop->skip_special_blocks_cfg)); in gaudi2_special_blocks_config()
3603 prop->skip_special_blocks_cfg.skip_block_hook = gaudi2_special_block_skip; in gaudi2_special_blocks_config()
3606 prop->skip_special_blocks_cfg.block_types = in gaudi2_special_blocks_config()
3609 if (!prop->skip_special_blocks_cfg.block_types) { in gaudi2_special_blocks_config()
3610 rc = -ENOMEM; in gaudi2_special_blocks_config()
3614 memcpy(prop->skip_special_blocks_cfg.block_types, gaudi2_iterator_skip_block_types, in gaudi2_special_blocks_config()
3617 prop->skip_special_blocks_cfg.block_types_len = in gaudi2_special_blocks_config()
3622 prop->skip_special_blocks_cfg.block_ranges = in gaudi2_special_blocks_config()
3625 if (!prop->skip_special_blocks_cfg.block_ranges) { in gaudi2_special_blocks_config()
3626 rc = -ENOMEM; in gaudi2_special_blocks_config()
3631 memcpy(&prop->skip_special_blocks_cfg.block_ranges[i], in gaudi2_special_blocks_config()
3635 prop->skip_special_blocks_cfg.block_ranges_len = in gaudi2_special_blocks_config()
3642 kfree(prop->skip_special_blocks_cfg.block_types); in gaudi2_special_blocks_config()
3644 kfree(prop->special_blocks); in gaudi2_special_blocks_config()
3656 struct gaudi2_device *gaudi2 = hdev->asic_specific; in gaudi2_test_queues_msgs_free()
3657 struct gaudi2_queues_test_info *msg_info = gaudi2->queues_test_info; in gaudi2_test_queues_msgs_free()
3661 /* bail-out if this is an allocation failure point */ in gaudi2_test_queues_msgs_free()
3672 struct gaudi2_device *gaudi2 = hdev->asic_specific; in gaudi2_test_queues_msgs_alloc()
3673 struct gaudi2_queues_test_info *msg_info = gaudi2->queues_test_info; in gaudi2_test_queues_msgs_alloc()
3676 /* allocate a message-short buf for each Q we intend to test */ in gaudi2_test_queues_msgs_alloc()
3682 dev_err(hdev->dev, in gaudi2_test_queues_msgs_alloc()
3684 rc = -ENOMEM; in gaudi2_test_queues_msgs_alloc()
3698 struct asic_fixed_properties *prop = &hdev->asic_prop; in gaudi2_sw_init()
3705 return -ENOMEM; in gaudi2_sw_init()
3711 if (gaudi2->num_of_valid_hw_events == GAUDI2_EVENT_SIZE) { in gaudi2_sw_init()
3712 dev_err(hdev->dev, "H/W events array exceeds the limit of %u events\n", in gaudi2_sw_init()
3714 rc = -EINVAL; in gaudi2_sw_init()
3718 gaudi2->hw_events[gaudi2->num_of_valid_hw_events++] = gaudi2_irq_map_table[i].fc_id; in gaudi2_sw_init()
3722 gaudi2->lfsr_rand_seeds[i] = gaudi2_get_non_zero_random_int(); in gaudi2_sw_init()
3724 gaudi2->cpucp_info_get = gaudi2_cpucp_info_get; in gaudi2_sw_init()
3726 hdev->asic_specific = gaudi2; in gaudi2_sw_init()
3729 * Use DEVICE_CACHE_LINE_SIZE for alignment since the NIC memory-mapped in gaudi2_sw_init()
3732 hdev->dma_pool = dma_pool_create(dev_name(hdev->dev), &hdev->pdev->dev, in gaudi2_sw_init()
3734 if (!hdev->dma_pool) { in gaudi2_sw_init()
3735 dev_err(hdev->dev, "failed to create DMA pool\n"); in gaudi2_sw_init()
3736 rc = -ENOMEM; in gaudi2_sw_init()
3744 hdev->cpu_accessible_dma_pool = gen_pool_create(ilog2(32), -1); in gaudi2_sw_init()
3745 if (!hdev->cpu_accessible_dma_pool) { in gaudi2_sw_init()
3746 dev_err(hdev->dev, "Failed to create CPU accessible DMA pool\n"); in gaudi2_sw_init()
3747 rc = -ENOMEM; in gaudi2_sw_init()
3751 rc = gen_pool_add(hdev->cpu_accessible_dma_pool, (uintptr_t) hdev->cpu_accessible_dma_mem, in gaudi2_sw_init()
3752 HL_CPU_ACCESSIBLE_MEM_SIZE, -1); in gaudi2_sw_init()
3754 dev_err(hdev->dev, "Failed to add memory to CPU accessible DMA pool\n"); in gaudi2_sw_init()
3755 rc = -EFAULT; in gaudi2_sw_init()
3759 gaudi2->virt_msix_db_cpu_addr = hl_cpu_accessible_dma_pool_alloc(hdev, prop->pmmu.page_size, in gaudi2_sw_init()
3760 &gaudi2->virt_msix_db_dma_addr); in gaudi2_sw_init()
3761 if (!gaudi2->virt_msix_db_cpu_addr) { in gaudi2_sw_init()
3762 dev_err(hdev->dev, "Failed to allocate DMA memory for virtual MSI-X doorbell\n"); in gaudi2_sw_init()
3763 rc = -ENOMEM; in gaudi2_sw_init()
3767 spin_lock_init(&gaudi2->hw_queues_lock); in gaudi2_sw_init()
3769 gaudi2->scratchpad_bus_address = prop->mmu_pgt_addr + HMMU_PAGE_TABLES_SIZE + EDMA_PQS_SIZE; in gaudi2_sw_init()
3776 hdev->supports_coresight = true; in gaudi2_sw_init()
3777 hdev->supports_sync_stream = true; in gaudi2_sw_init()
3778 hdev->supports_cb_mapping = true; in gaudi2_sw_init()
3779 hdev->supports_wait_for_multi_cs = false; in gaudi2_sw_init()
3781 prop->supports_compute_reset = true; in gaudi2_sw_init()
3785 hdev->event_queue.check_eqe_index = false; in gaudi2_sw_init()
3787 hdev->event_queue.check_eqe_index = true; in gaudi2_sw_init()
3789 hdev->asic_funcs->set_pci_memory_regions(hdev); in gaudi2_sw_init()
3799 hdev->heartbeat_debug_info.cpu_queue_id = GAUDI2_QUEUE_ID_CPU_PQ; in gaudi2_sw_init()
3806 hl_cpu_accessible_dma_pool_free(hdev, prop->pmmu.page_size, gaudi2->virt_msix_db_cpu_addr); in gaudi2_sw_init()
3808 gen_pool_destroy(hdev->cpu_accessible_dma_pool); in gaudi2_sw_init()
3810 hl_asic_dma_free_coherent(hdev, HL_CPU_ACCESSIBLE_MEM_SIZE, hdev->cpu_accessible_dma_mem, in gaudi2_sw_init()
3811 hdev->cpu_accessible_dma_address); in gaudi2_sw_init()
3813 dma_pool_destroy(hdev->dma_pool); in gaudi2_sw_init()
3821 struct asic_fixed_properties *prop = &hdev->asic_prop; in gaudi2_sw_fini()
3822 struct gaudi2_device *gaudi2 = hdev->asic_specific; in gaudi2_sw_fini()
3828 hl_cpu_accessible_dma_pool_free(hdev, prop->pmmu.page_size, gaudi2->virt_msix_db_cpu_addr); in gaudi2_sw_fini()
3830 gen_pool_destroy(hdev->cpu_accessible_dma_pool); in gaudi2_sw_fini()
3832 hl_asic_dma_free_coherent(hdev, HL_CPU_ACCESSIBLE_MEM_SIZE, hdev->cpu_accessible_dma_mem, in gaudi2_sw_fini()
3833 hdev->cpu_accessible_dma_address); in gaudi2_sw_fini()
3835 dma_pool_destroy(hdev->dma_pool); in gaudi2_sw_fini()
3865 * gaudi2_clear_qm_fence_counters_common - clear QM's fence counters
3882 size = mmPDMA0_QM_CP_BARRIER_CFG - mmPDMA0_QM_CP_FENCE0_CNT_0; in gaudi2_clear_qm_fence_counters_common()
3905 struct gaudi2_device *gaudi2 = hdev->asic_specific; in gaudi2_stop_dma_qmans()
3908 if (!(gaudi2->hw_cap_initialized & HW_CAP_PDMA_MASK)) in gaudi2_stop_dma_qmans()
3916 if (!(gaudi2->hw_cap_initialized & HW_CAP_EDMA_MASK)) in gaudi2_stop_dma_qmans()
3924 if (!(gaudi2->hw_cap_initialized & BIT_ULL(HW_CAP_EDMA_SHIFT + seq))) in gaudi2_stop_dma_qmans()
3938 struct gaudi2_device *gaudi2 = hdev->asic_specific; in gaudi2_stop_mme_qmans()
3941 offset = mmDCORE1_MME_QM_BASE - mmDCORE0_MME_QM_BASE; in gaudi2_stop_mme_qmans()
3944 if (!(gaudi2->hw_cap_initialized & BIT_ULL(HW_CAP_MME_SHIFT + i))) in gaudi2_stop_mme_qmans()
3953 struct gaudi2_device *gaudi2 = hdev->asic_specific; in gaudi2_stop_tpc_qmans()
3957 if (!(gaudi2->tpc_hw_cap_initialized & HW_CAP_TPC_MASK)) in gaudi2_stop_tpc_qmans()
3961 if (!(gaudi2->tpc_hw_cap_initialized & BIT_ULL(HW_CAP_TPC_SHIFT + i))) in gaudi2_stop_tpc_qmans()
3971 struct gaudi2_device *gaudi2 = hdev->asic_specific; in gaudi2_stop_rot_qmans()
3975 if (!(gaudi2->hw_cap_initialized & HW_CAP_ROT_MASK)) in gaudi2_stop_rot_qmans()
3979 if (!(gaudi2->hw_cap_initialized & BIT_ULL(HW_CAP_ROT_SHIFT + i))) in gaudi2_stop_rot_qmans()
3989 struct gaudi2_device *gaudi2 = hdev->asic_specific; in gaudi2_stop_nic_qmans()
3993 if (!(gaudi2->nic_hw_cap_initialized & HW_CAP_NIC_MASK)) in gaudi2_stop_nic_qmans()
3999 if (!(hdev->nic_ports_mask & BIT(i))) in gaudi2_stop_nic_qmans()
4017 struct gaudi2_device *gaudi2 = hdev->asic_specific; in gaudi2_dma_stall()
4020 if (!(gaudi2->hw_cap_initialized & HW_CAP_PDMA_MASK)) in gaudi2_dma_stall()
4027 if (!(gaudi2->hw_cap_initialized & HW_CAP_EDMA_MASK)) in gaudi2_dma_stall()
4035 if (!(gaudi2->hw_cap_initialized & BIT_ULL(HW_CAP_EDMA_SHIFT + seq))) in gaudi2_dma_stall()
4049 struct gaudi2_device *gaudi2 = hdev->asic_specific; in gaudi2_mme_stall()
4052 offset = mmDCORE1_MME_CTRL_LO_QM_STALL - mmDCORE0_MME_CTRL_LO_QM_STALL; in gaudi2_mme_stall()
4055 if (gaudi2->hw_cap_initialized & BIT_ULL(HW_CAP_MME_SHIFT + i)) in gaudi2_mme_stall()
4061 struct gaudi2_device *gaudi2 = hdev->asic_specific; in gaudi2_tpc_stall()
4065 if (!(gaudi2->tpc_hw_cap_initialized & HW_CAP_TPC_MASK)) in gaudi2_tpc_stall()
4069 if (!(gaudi2->tpc_hw_cap_initialized & BIT_ULL(HW_CAP_TPC_SHIFT + i))) in gaudi2_tpc_stall()
4079 struct gaudi2_device *gaudi2 = hdev->asic_specific; in gaudi2_rotator_stall()
4083 if (!(gaudi2->hw_cap_initialized & HW_CAP_ROT_MASK)) in gaudi2_rotator_stall()
4091 if (!(gaudi2->hw_cap_initialized & BIT_ULL(HW_CAP_ROT_SHIFT + i))) in gaudi2_rotator_stall()
4105 struct gaudi2_device *gaudi2 = hdev->asic_specific; in gaudi2_disable_dma_qmans()
4108 if (!(gaudi2->hw_cap_initialized & HW_CAP_PDMA_MASK)) in gaudi2_disable_dma_qmans()
4115 if (!(gaudi2->hw_cap_initialized & HW_CAP_EDMA_MASK)) in gaudi2_disable_dma_qmans()
4123 if (!(gaudi2->hw_cap_initialized & BIT_ULL(HW_CAP_EDMA_SHIFT + seq))) in gaudi2_disable_dma_qmans()
4137 struct gaudi2_device *gaudi2 = hdev->asic_specific; in gaudi2_disable_mme_qmans()
4140 offset = mmDCORE1_MME_QM_BASE - mmDCORE0_MME_QM_BASE; in gaudi2_disable_mme_qmans()
4143 if (gaudi2->hw_cap_initialized & BIT_ULL(HW_CAP_MME_SHIFT + i)) in gaudi2_disable_mme_qmans()
4149 struct gaudi2_device *gaudi2 = hdev->asic_specific; in gaudi2_disable_tpc_qmans()
4153 if (!(gaudi2->tpc_hw_cap_initialized & HW_CAP_TPC_MASK)) in gaudi2_disable_tpc_qmans()
4157 if (!(gaudi2->tpc_hw_cap_initialized & BIT_ULL(HW_CAP_TPC_SHIFT + i))) in gaudi2_disable_tpc_qmans()
4167 struct gaudi2_device *gaudi2 = hdev->asic_specific; in gaudi2_disable_rot_qmans()
4171 if (!(gaudi2->hw_cap_initialized & HW_CAP_ROT_MASK)) in gaudi2_disable_rot_qmans()
4175 if (!(gaudi2->hw_cap_initialized & BIT_ULL(HW_CAP_ROT_SHIFT + i))) in gaudi2_disable_rot_qmans()
4185 struct gaudi2_device *gaudi2 = hdev->asic_specific; in gaudi2_disable_nic_qmans()
4189 if (!(gaudi2->nic_hw_cap_initialized & HW_CAP_NIC_MASK)) in gaudi2_disable_nic_qmans()
4195 if (!(hdev->nic_ports_mask & BIT(i))) in gaudi2_disable_nic_qmans()
4208 /* Zero the lower/upper parts of the 64-bit counter */ in gaudi2_enable_timestamp()
4230 return gaudi2_vdec_irq_name[irq_number - GAUDI2_IRQ_NUM_DCORE0_DEC0_NRM]; in gaudi2_irq_name()
4250 irq = pci_irq_vector(hdev->pdev, i); in gaudi2_dec_disable_msix()
4251 relative_idx = i - GAUDI2_IRQ_NUM_DCORE0_DEC0_NRM; in gaudi2_dec_disable_msix()
4253 dec = hdev->dec + relative_idx / 2; in gaudi2_dec_disable_msix()
4261 (void *) &hdev->user_interrupt[dec->core_id])); in gaudi2_dec_disable_msix()
4274 irq = pci_irq_vector(hdev->pdev, i); in gaudi2_dec_enable_msix()
4275 relative_idx = i - GAUDI2_IRQ_NUM_DCORE0_DEC0_NRM; in gaudi2_dec_enable_msix()
4284 dec = hdev->dec + relative_idx / 2; in gaudi2_dec_enable_msix()
4290 (void *) &hdev->user_interrupt[dec->core_id]); in gaudi2_dec_enable_msix()
4294 dev_err(hdev->dev, "Failed to request IRQ %d", irq); in gaudi2_dec_enable_msix()
4308 struct asic_fixed_properties *prop = &hdev->asic_prop; in gaudi2_enable_msix()
4309 struct gaudi2_device *gaudi2 = hdev->asic_specific; in gaudi2_enable_msix()
4313 if (gaudi2->hw_cap_initialized & HW_CAP_MSIX) in gaudi2_enable_msix()
4318 rc = pci_alloc_irq_vectors(hdev->pdev, GAUDI2_MSIX_ENTRIES, GAUDI2_MSIX_ENTRIES, in gaudi2_enable_msix()
4321 dev_err(hdev->dev, "MSI-X: Failed to enable support -- %d/%d\n", in gaudi2_enable_msix()
4326 irq = pci_irq_vector(hdev->pdev, GAUDI2_IRQ_NUM_COMPLETION); in gaudi2_enable_msix()
4327 cq = &hdev->completion_queue[GAUDI2_RESERVED_CQ_CS_COMPLETION]; in gaudi2_enable_msix()
4330 dev_err(hdev->dev, "Failed to request IRQ %d", irq); in gaudi2_enable_msix()
4334 irq = pci_irq_vector(hdev->pdev, GAUDI2_IRQ_NUM_EVENT_QUEUE); in gaudi2_enable_msix()
4336 &hdev->event_queue); in gaudi2_enable_msix()
4338 dev_err(hdev->dev, "Failed to request IRQ %d", irq); in gaudi2_enable_msix()
4344 dev_err(hdev->dev, "Failed to enable decoder IRQ"); in gaudi2_enable_msix()
4348 irq = pci_irq_vector(hdev->pdev, GAUDI2_IRQ_NUM_TPC_ASSERT); in gaudi2_enable_msix()
4351 &hdev->tpc_interrupt); in gaudi2_enable_msix()
4353 dev_err(hdev->dev, "Failed to request IRQ %d", irq); in gaudi2_enable_msix()
4357 irq = pci_irq_vector(hdev->pdev, GAUDI2_IRQ_NUM_UNEXPECTED_ERROR); in gaudi2_enable_msix()
4360 &hdev->unexpected_error_interrupt); in gaudi2_enable_msix()
4362 dev_err(hdev->dev, "Failed to request IRQ %d", irq); in gaudi2_enable_msix()
4366 for (i = GAUDI2_IRQ_NUM_USER_FIRST, j = prop->user_dec_intr_count, user_irq_init_cnt = 0; in gaudi2_enable_msix()
4367 user_irq_init_cnt < prop->user_interrupt_count; in gaudi2_enable_msix()
4370 irq = pci_irq_vector(hdev->pdev, i); in gaudi2_enable_msix()
4373 &hdev->user_interrupt[j]); in gaudi2_enable_msix()
4375 dev_err(hdev->dev, "Failed to request IRQ %d", irq); in gaudi2_enable_msix()
4380 irq = pci_irq_vector(hdev->pdev, GAUDI2_IRQ_NUM_EQ_ERROR); in gaudi2_enable_msix()
4385 dev_err(hdev->dev, "Failed to request IRQ %d", irq); in gaudi2_enable_msix()
4389 gaudi2->hw_cap_initialized |= HW_CAP_MSIX; in gaudi2_enable_msix()
4394 for (i = GAUDI2_IRQ_NUM_USER_FIRST, j = prop->user_dec_intr_count; in gaudi2_enable_msix()
4397 irq = pci_irq_vector(hdev->pdev, i); in gaudi2_enable_msix()
4399 free_irq(irq, &hdev->user_interrupt[j]); in gaudi2_enable_msix()
4401 irq = pci_irq_vector(hdev->pdev, GAUDI2_IRQ_NUM_UNEXPECTED_ERROR); in gaudi2_enable_msix()
4402 free_irq(irq, &hdev->unexpected_error_interrupt); in gaudi2_enable_msix()
4404 irq = pci_irq_vector(hdev->pdev, GAUDI2_IRQ_NUM_TPC_ASSERT); in gaudi2_enable_msix()
4405 free_irq(irq, &hdev->tpc_interrupt); in gaudi2_enable_msix()
4409 irq = pci_irq_vector(hdev->pdev, GAUDI2_IRQ_NUM_EVENT_QUEUE); in gaudi2_enable_msix()
4413 irq = pci_irq_vector(hdev->pdev, GAUDI2_IRQ_NUM_COMPLETION); in gaudi2_enable_msix()
4417 pci_free_irq_vectors(hdev->pdev); in gaudi2_enable_msix()
4424 struct gaudi2_device *gaudi2 = hdev->asic_specific; in gaudi2_sync_irqs()
4428 if (!(gaudi2->hw_cap_initialized & HW_CAP_MSIX)) in gaudi2_sync_irqs()
4432 synchronize_irq(pci_irq_vector(hdev->pdev, GAUDI2_IRQ_NUM_COMPLETION)); in gaudi2_sync_irqs()
4435 irq = pci_irq_vector(hdev->pdev, i); in gaudi2_sync_irqs()
4439 synchronize_irq(pci_irq_vector(hdev->pdev, GAUDI2_IRQ_NUM_TPC_ASSERT)); in gaudi2_sync_irqs()
4440 synchronize_irq(pci_irq_vector(hdev->pdev, GAUDI2_IRQ_NUM_UNEXPECTED_ERROR)); in gaudi2_sync_irqs()
4442 for (i = GAUDI2_IRQ_NUM_USER_FIRST, j = 0 ; j < hdev->asic_prop.user_interrupt_count; in gaudi2_sync_irqs()
4444 irq = pci_irq_vector(hdev->pdev, i); in gaudi2_sync_irqs()
4448 synchronize_irq(pci_irq_vector(hdev->pdev, GAUDI2_IRQ_NUM_EVENT_QUEUE)); in gaudi2_sync_irqs()
4449 synchronize_irq(pci_irq_vector(hdev->pdev, GAUDI2_IRQ_NUM_EQ_ERROR)); in gaudi2_sync_irqs()
4454 struct asic_fixed_properties *prop = &hdev->asic_prop; in gaudi2_disable_msix()
4455 struct gaudi2_device *gaudi2 = hdev->asic_specific; in gaudi2_disable_msix()
4459 if (!(gaudi2->hw_cap_initialized & HW_CAP_MSIX)) in gaudi2_disable_msix()
4464 irq = pci_irq_vector(hdev->pdev, GAUDI2_IRQ_NUM_EVENT_QUEUE); in gaudi2_disable_msix()
4465 free_irq(irq, &hdev->event_queue); in gaudi2_disable_msix()
4469 irq = pci_irq_vector(hdev->pdev, GAUDI2_IRQ_NUM_TPC_ASSERT); in gaudi2_disable_msix()
4470 free_irq(irq, &hdev->tpc_interrupt); in gaudi2_disable_msix()
4472 irq = pci_irq_vector(hdev->pdev, GAUDI2_IRQ_NUM_UNEXPECTED_ERROR); in gaudi2_disable_msix()
4473 free_irq(irq, &hdev->unexpected_error_interrupt); in gaudi2_disable_msix()
4475 for (i = GAUDI2_IRQ_NUM_USER_FIRST, j = prop->user_dec_intr_count, k = 0; in gaudi2_disable_msix()
4476 k < hdev->asic_prop.user_interrupt_count ; i++, j++, k++) { in gaudi2_disable_msix()
4478 irq = pci_irq_vector(hdev->pdev, i); in gaudi2_disable_msix()
4480 free_irq(irq, &hdev->user_interrupt[j]); in gaudi2_disable_msix()
4483 irq = pci_irq_vector(hdev->pdev, GAUDI2_IRQ_NUM_COMPLETION); in gaudi2_disable_msix()
4484 cq = &hdev->completion_queue[GAUDI2_RESERVED_CQ_CS_COMPLETION]; in gaudi2_disable_msix()
4487 irq = pci_irq_vector(hdev->pdev, GAUDI2_IRQ_NUM_EQ_ERROR); in gaudi2_disable_msix()
4490 pci_free_irq_vectors(hdev->pdev); in gaudi2_disable_msix()
4492 gaudi2->hw_cap_initialized &= ~HW_CAP_MSIX; in gaudi2_disable_msix()
4502 if (hdev->pldm) in gaudi2_stop_dcore_dec()
4509 if (!(hdev->asic_prop.decoder_enabled_mask & BIT(dec_bit))) in gaudi2_stop_dcore_dec()
4529 dev_err(hdev->dev, in gaudi2_stop_dcore_dec()
4542 if (hdev->pldm) in gaudi2_stop_pcie_dec()
4549 if (!(hdev->asic_prop.decoder_enabled_mask & BIT(dec_bit))) in gaudi2_stop_pcie_dec()
4569 dev_err(hdev->dev, in gaudi2_stop_pcie_dec()
4577 struct gaudi2_device *gaudi2 = hdev->asic_specific; in gaudi2_stop_dec()
4580 if ((gaudi2->dec_hw_cap_initialized & HW_CAP_DEC_MASK) == 0) in gaudi2_stop_dec()
4617 if (hdev->pldm) in gaudi2_verify_arc_running_mode()
4641 struct gaudi2_device *gaudi2 = hdev->asic_specific; in gaudi2_reset_arcs()
4654 struct gaudi2_device *gaudi2 = hdev->asic_specific; in gaudi2_nic_qmans_manual_flush()
4658 if (!(gaudi2->nic_hw_cap_initialized & HW_CAP_NIC_MASK)) in gaudi2_nic_qmans_manual_flush()
4664 if (!(hdev->nic_ports_mask & BIT(i))) in gaudi2_nic_qmans_manual_flush()
4686 dev_err(hdev->dev, "failed to %s arc: %d\n", in gaudi2_set_engine_cores()
4689 return -1; in gaudi2_set_engine_cores()
4699 struct gaudi2_device *gaudi2 = hdev->asic_specific; in gaudi2_set_tpc_engine_mode()
4702 if (!(gaudi2->tpc_hw_cap_initialized & HW_CAP_TPC_MASK)) in gaudi2_set_tpc_engine_mode()
4706 if (!(gaudi2->tpc_hw_cap_initialized & BIT_ULL(HW_CAP_TPC_SHIFT + tpc_id))) in gaudi2_set_tpc_engine_mode()
4726 struct gaudi2_device *gaudi2 = hdev->asic_specific; in gaudi2_set_mme_engine_mode()
4730 if (!(gaudi2->hw_cap_initialized & BIT_ULL(HW_CAP_MME_SHIFT + mme_id))) in gaudi2_set_mme_engine_mode()
4744 struct gaudi2_device *gaudi2 = hdev->asic_specific; in gaudi2_set_edma_engine_mode()
4747 if (!(gaudi2->hw_cap_initialized & HW_CAP_EDMA_MASK)) in gaudi2_set_edma_engine_mode()
4751 if (!(gaudi2->hw_cap_initialized & BIT_ULL(HW_CAP_EDMA_SHIFT + edma_id))) in gaudi2_set_edma_engine_mode()
4804 dev_err(hdev->dev, "Invalid engine ID %u\n", engine_ids[i]); in gaudi2_set_engine_modes()
4805 return -EINVAL; in gaudi2_set_engine_modes()
4825 dev_err(hdev->dev, "failed to execute command id %u\n", engine_command); in gaudi2_set_engines()
4826 return -EINVAL; in gaudi2_set_engines()
4834 if (hdev->pldm) in gaudi2_halt_engines()
4884 struct pre_fw_load_props *pre_fw_load = &hdev->fw_loader.pre_fw_load; in gaudi2_init_firmware_preload_params()
4886 pre_fw_load->cpu_boot_status_reg = mmPSOC_GLOBAL_CONF_CPU_BOOT_STATUS; in gaudi2_init_firmware_preload_params()
4887 pre_fw_load->sts_boot_dev_sts0_reg = mmCPU_BOOT_DEV_STS0; in gaudi2_init_firmware_preload_params()
4888 pre_fw_load->sts_boot_dev_sts1_reg = mmCPU_BOOT_DEV_STS1; in gaudi2_init_firmware_preload_params()
4889 pre_fw_load->boot_err0_reg = mmCPU_BOOT_ERR0; in gaudi2_init_firmware_preload_params()
4890 pre_fw_load->boot_err1_reg = mmCPU_BOOT_ERR1; in gaudi2_init_firmware_preload_params()
4891 pre_fw_load->wait_for_preboot_timeout = GAUDI2_PREBOOT_REQ_TIMEOUT_USEC; in gaudi2_init_firmware_preload_params()
4892 pre_fw_load->wait_for_preboot_extended_timeout = in gaudi2_init_firmware_preload_params()
4898 struct fw_load_mgr *fw_loader = &hdev->fw_loader; in gaudi2_init_firmware_loader()
4903 fw_loader->fw_comp_loaded = FW_TYPE_NONE; in gaudi2_init_firmware_loader()
4904 fw_loader->boot_fit_img.image_name = GAUDI2_BOOT_FIT_FILE; in gaudi2_init_firmware_loader()
4905 fw_loader->linux_img.image_name = GAUDI2_LINUX_FW_FILE; in gaudi2_init_firmware_loader()
4906 fw_loader->boot_fit_timeout = GAUDI2_BOOT_FIT_REQ_TIMEOUT_USEC; in gaudi2_init_firmware_loader()
4907 fw_loader->skip_bmc = false; in gaudi2_init_firmware_loader()
4908 fw_loader->sram_bar_id = SRAM_CFG_BAR_ID; in gaudi2_init_firmware_loader()
4909 fw_loader->dram_bar_id = DRAM_BAR_ID; in gaudi2_init_firmware_loader()
4910 fw_loader->cpu_timeout = GAUDI2_CPU_TIMEOUT_USEC; in gaudi2_init_firmware_loader()
4914 * hard-coded). in later stages of the protocol those values will be in gaudi2_init_firmware_loader()
4916 * will always be up-to-date in gaudi2_init_firmware_loader()
4918 dynamic_loader = &hdev->fw_loader.dynamic_loader; in gaudi2_init_firmware_loader()
4919 dyn_regs = &dynamic_loader->comm_desc.cpu_dyn_regs; in gaudi2_init_firmware_loader()
4920 dyn_regs->kmd_msg_to_cpu = cpu_to_le32(mmPSOC_GLOBAL_CONF_KMD_MSG_TO_CPU); in gaudi2_init_firmware_loader()
4921 dyn_regs->cpu_cmd_status_to_host = cpu_to_le32(mmCPU_CMD_STATUS_TO_HOST); in gaudi2_init_firmware_loader()
4922 dynamic_loader->wait_for_bl_timeout = GAUDI2_WAIT_FOR_BL_TIMEOUT_USEC; in gaudi2_init_firmware_loader()
4927 struct gaudi2_device *gaudi2 = hdev->asic_specific; in gaudi2_init_cpu()
4930 if (!(hdev->fw_components & FW_TYPE_PREBOOT_CPU)) in gaudi2_init_cpu()
4933 if (gaudi2->hw_cap_initialized & HW_CAP_CPU) in gaudi2_init_cpu()
4940 gaudi2->hw_cap_initialized |= HW_CAP_CPU; in gaudi2_init_cpu()
4947 struct hl_hw_queue *cpu_pq = &hdev->kernel_queues[GAUDI2_QUEUE_ID_CPU_PQ]; in gaudi2_init_cpu_queues()
4948 struct asic_fixed_properties *prop = &hdev->asic_prop; in gaudi2_init_cpu_queues()
4949 struct gaudi2_device *gaudi2 = hdev->asic_specific; in gaudi2_init_cpu_queues()
4955 if (!hdev->cpu_queues_enable) in gaudi2_init_cpu_queues()
4958 if (gaudi2->hw_cap_initialized & HW_CAP_CPU_Q) in gaudi2_init_cpu_queues()
4961 eq = &hdev->event_queue; in gaudi2_init_cpu_queues()
4963 dyn_regs = &hdev->fw_loader.dynamic_loader.comm_desc.cpu_dyn_regs; in gaudi2_init_cpu_queues()
4965 WREG32(mmCPU_IF_PQ_BASE_ADDR_LOW, lower_32_bits(cpu_pq->bus_address)); in gaudi2_init_cpu_queues()
4966 WREG32(mmCPU_IF_PQ_BASE_ADDR_HIGH, upper_32_bits(cpu_pq->bus_address)); in gaudi2_init_cpu_queues()
4968 WREG32(mmCPU_IF_EQ_BASE_ADDR_LOW, lower_32_bits(eq->bus_address)); in gaudi2_init_cpu_queues()
4969 WREG32(mmCPU_IF_EQ_BASE_ADDR_HIGH, upper_32_bits(eq->bus_address)); in gaudi2_init_cpu_queues()
4971 WREG32(mmCPU_IF_CQ_BASE_ADDR_LOW, lower_32_bits(hdev->cpu_accessible_dma_address)); in gaudi2_init_cpu_queues()
4972 WREG32(mmCPU_IF_CQ_BASE_ADDR_HIGH, upper_32_bits(hdev->cpu_accessible_dma_address)); in gaudi2_init_cpu_queues()
4987 WREG32(le32_to_cpu(dyn_regs->gic_host_pi_upd_irq), in gaudi2_init_cpu_queues()
4999 dev_err(hdev->dev, "Failed to communicate with device CPU (timeout)\n"); in gaudi2_init_cpu_queues()
5000 return -EIO; in gaudi2_init_cpu_queues()
5004 if (prop->fw_cpu_boot_dev_sts0_valid) in gaudi2_init_cpu_queues()
5005 prop->fw_app_cpu_boot_dev_sts0 = RREG32(mmCPU_BOOT_DEV_STS0); in gaudi2_init_cpu_queues()
5007 if (prop->fw_cpu_boot_dev_sts1_valid) in gaudi2_init_cpu_queues()
5008 prop->fw_app_cpu_boot_dev_sts1 = RREG32(mmCPU_BOOT_DEV_STS1); in gaudi2_init_cpu_queues()
5010 gaudi2->hw_cap_initialized |= HW_CAP_CPU_Q; in gaudi2_init_cpu_queues()
5021 q = &hdev->kernel_queues[queue_id_base + pq_id]; in gaudi2_init_qman_pq()
5024 if (q->dram_bd) { in gaudi2_init_qman_pq()
5026 lower_32_bits(q->pq_dram_address)); in gaudi2_init_qman_pq()
5028 upper_32_bits(q->pq_dram_address)); in gaudi2_init_qman_pq()
5031 lower_32_bits(q->bus_address)); in gaudi2_init_qman_pq()
5033 upper_32_bits(q->bus_address)); in gaudi2_init_qman_pq()
5066 struct gaudi2_device *gaudi2 = hdev->asic_specific; in gaudi2_init_qman_pqc()
5077 lower_32_bits(gaudi2->scratchpad_bus_address)); in gaudi2_init_qman_pqc()
5079 upper_32_bits(gaudi2->scratchpad_bus_address)); in gaudi2_init_qman_pqc()
5095 struct cpu_dyn_regs *dyn_regs = &hdev->fw_loader.dynamic_loader.comm_desc.cpu_dyn_regs; in gaudi2_get_dyn_sp_reg()
5108 sp_reg_addr = le32_to_cpu(dyn_regs->gic_dma_qm_irq_ctrl); in gaudi2_get_dyn_sp_reg()
5117 sp_reg_addr = le32_to_cpu(dyn_regs->gic_mme_qm_irq_ctrl); in gaudi2_get_dyn_sp_reg()
5126 sp_reg_addr = le32_to_cpu(dyn_regs->gic_tpc_qm_irq_ctrl); in gaudi2_get_dyn_sp_reg()
5129 sp_reg_addr = le32_to_cpu(dyn_regs->gic_rot_qm_irq_ctrl); in gaudi2_get_dyn_sp_reg()
5132 sp_reg_addr = le32_to_cpu(dyn_regs->gic_nic_qm_irq_ctrl); in gaudi2_get_dyn_sp_reg()
5135 dev_err(hdev->dev, "Unexpected h/w queue %d\n", queue_id_base); in gaudi2_get_dyn_sp_reg()
5184 hdev->kernel_queues[queue_id_base + pq_id].cq_id = GAUDI2_RESERVED_CQ_CS_COMPLETION; in gaudi2_init_qman()
5205 dyn_regs = &hdev->fw_loader.dynamic_loader.comm_desc.cpu_dyn_regs; in gaudi2_init_dma_core()
5206 irq_handler_offset = le32_to_cpu(dyn_regs->gic_dma_core_irq_ctrl); in gaudi2_init_dma_core()
5224 struct gaudi2_device *gaudi2 = hdev->asic_specific; in gaudi2_init_kdma()
5227 if ((gaudi2->hw_cap_initialized & HW_CAP_KDMA) == HW_CAP_KDMA) in gaudi2_init_kdma()
5234 gaudi2->hw_cap_initialized |= HW_CAP_KDMA; in gaudi2_init_kdma()
5239 struct gaudi2_device *gaudi2 = hdev->asic_specific; in gaudi2_init_pdma()
5242 if ((gaudi2->hw_cap_initialized & HW_CAP_PDMA_MASK) == HW_CAP_PDMA_MASK) in gaudi2_init_pdma()
5257 gaudi2->hw_cap_initialized |= HW_CAP_PDMA_MASK; in gaudi2_init_pdma()
5276 struct asic_fixed_properties *prop = &hdev->asic_prop; in gaudi2_init_edma()
5277 struct gaudi2_device *gaudi2 = hdev->asic_specific; in gaudi2_init_edma()
5280 if ((gaudi2->hw_cap_initialized & HW_CAP_EDMA_MASK) == HW_CAP_EDMA_MASK) in gaudi2_init_edma()
5287 if (!(prop->edma_enabled_mask & BIT(seq))) in gaudi2_init_edma()
5292 gaudi2->hw_cap_initialized |= BIT_ULL(HW_CAP_EDMA_SHIFT + seq); in gaudi2_init_edma()
5298 * gaudi2_arm_monitors_for_virt_msix_db() - Arm monitors for writing to the virtual MSI-X doorbell.
5305 * write directly to the HBW host memory of the virtual MSI-X doorbell.
5310 * completion, by decrementing the sync object value and re-arming the monitor.
5316 struct gaudi2_device *gaudi2 = hdev->asic_specific; in gaudi2_arm_monitors_for_virt_msix_db()
5325 * 1. Write interrupt ID to the virtual MSI-X doorbell (master monitor) in gaudi2_arm_monitors_for_virt_msix_db()
5327 * 3. Re-arm the master monitor. in gaudi2_arm_monitors_for_virt_msix_db()
5339 payload = FIELD_PREP(DCORE0_SYNC_MNGR_OBJS_SOB_OBJ_VAL_MASK, 0x7FFF) | /* "-1" */ in gaudi2_arm_monitors_for_virt_msix_db()
5344 /* 3rd monitor: Re-arm the master monitor */ in gaudi2_arm_monitors_for_virt_msix_db()
5362 /* 1st monitor (master): Write interrupt ID to the virtual MSI-X doorbell */ in gaudi2_arm_monitors_for_virt_msix_db()
5368 addr = gaudi2->virt_msix_db_dma_addr; in gaudi2_arm_monitors_for_virt_msix_db()
5381 struct asic_fixed_properties *prop = &hdev->asic_prop; in gaudi2_prepare_sm_for_virt_msix_db()
5385 if (!(prop->decoder_enabled_mask & BIT(decoder_id))) in gaudi2_prepare_sm_for_virt_msix_db()
5402 struct gaudi2_device *gaudi2 = hdev->asic_specific; in gaudi2_init_sm()
5418 /* Init CQ0 DB - configure the monitor to trigger MSI-X interrupt */ in gaudi2_init_sm()
5419 WREG32(mmDCORE0_SYNC_MNGR_GLBL_LBW_ADDR_L_0, lower_32_bits(gaudi2->virt_msix_db_dma_addr)); in gaudi2_init_sm()
5420 WREG32(mmDCORE0_SYNC_MNGR_GLBL_LBW_ADDR_H_0, upper_32_bits(gaudi2->virt_msix_db_dma_addr)); in gaudi2_init_sm()
5425 hdev->completion_queue[i].bus_address; in gaudi2_init_sm()
5439 /* Initialize sync objects and monitors which are used for the virtual MSI-X doorbell */ in gaudi2_init_sm()
5445 struct gaudi2_device *gaudi2 = hdev->asic_specific; in gaudi2_init_mme_acc()
5461 WREG32(reg_base + MME_ACC_AP_LFSR_SEED_WDATA_OFFSET, gaudi2->lfsr_rand_seeds[i]); in gaudi2_init_mme_acc()
5484 dev_err(hdev->dev, "Invalid dcore id %u\n", dcore_id); in gaudi2_init_dcore_mme()
5499 struct gaudi2_device *gaudi2 = hdev->asic_specific; in gaudi2_init_mme()
5502 if ((gaudi2->hw_cap_initialized & HW_CAP_MME_MASK) == HW_CAP_MME_MASK) in gaudi2_init_mme()
5508 gaudi2->hw_cap_initialized |= BIT_ULL(HW_CAP_MME_SHIFT + i); in gaudi2_init_mme()
5529 struct gaudi2_device *gaudi2 = hdev->asic_specific; in gaudi2_init_tpc_config()
5530 struct gaudi2_tpc_init_cfg_data *cfg_data = ctx->data; in gaudi2_init_tpc_config()
5534 queue_id_base = cfg_data->dcore_tpc_qid_base[dcore] + (inst * NUM_OF_PQ_PER_QMAN); in gaudi2_init_tpc_config()
5536 if (dcore == 0 && inst == (NUM_DCORE0_TPC - 1)) in gaudi2_init_tpc_config()
5545 gaudi2->tpc_hw_cap_initialized |= BIT_ULL(HW_CAP_TPC_SHIFT + seq); in gaudi2_init_tpc_config()
5550 struct gaudi2_device *gaudi2 = hdev->asic_specific; in gaudi2_init_tpc()
5554 if (!hdev->asic_prop.tpc_enabled_mask) in gaudi2_init_tpc()
5557 if ((gaudi2->tpc_hw_cap_initialized & HW_CAP_TPC_MASK) == HW_CAP_TPC_MASK) in gaudi2_init_tpc()
5571 struct gaudi2_device *gaudi2 = hdev->asic_specific; in gaudi2_init_rotator()
5580 gaudi2->hw_cap_initialized |= BIT_ULL(HW_CAP_ROT_SHIFT + i); in gaudi2_init_rotator()
5603 struct gaudi2_device *gaudi2 = hdev->asic_specific; in gaudi2_init_dec()
5607 if (!hdev->asic_prop.decoder_enabled_mask) in gaudi2_init_dec()
5610 if ((gaudi2->dec_hw_cap_initialized & HW_CAP_DEC_MASK) == HW_CAP_DEC_MASK) in gaudi2_init_dec()
5617 if (!(hdev->asic_prop.decoder_enabled_mask & BIT(dec_bit))) in gaudi2_init_dec()
5627 gaudi2->dec_hw_cap_initialized |= BIT_ULL(HW_CAP_DEC_SHIFT + dec_bit); in gaudi2_init_dec()
5632 if (!(hdev->asic_prop.decoder_enabled_mask & BIT(dec_bit))) in gaudi2_init_dec()
5640 gaudi2->dec_hw_cap_initialized |= BIT_ULL(HW_CAP_DEC_SHIFT + dec_bit); in gaudi2_init_dec()
5650 if (hdev->pldm || !hdev->pdev) in gaudi2_mmu_update_asid_hop0_addr()
5669 dev_err(hdev->dev, "Timeout during MMU hop0 config of asid %d\n", asid); in gaudi2_mmu_update_asid_hop0_addr()
5696 timeout_usec = (hdev->pldm) ? GAUDI2_PLDM_MMU_TIMEOUT_USEC : in gaudi2_mmu_invalidate_cache_status_poll()
5700 if (inv_params->flags & MMU_OP_CLEAR_MEMCACHE) { in gaudi2_mmu_invalidate_cache_status_poll()
5719 if (inv_params->flags & MMU_OP_SKIP_LOW_CACHE_INV) in gaudi2_mmu_invalidate_cache_status_poll()
5722 start_offset = inv_params->range_invalidation ? in gaudi2_mmu_invalidate_cache_status_poll()
5738 struct gaudi2_device *gaudi2 = hdev->asic_specific; in gaudi2_is_hmmu_enabled()
5743 if (gaudi2->hw_cap_initialized & hw_cap) in gaudi2_is_hmmu_enabled()
5763 if (inv_params->range_invalidation) { in gaudi2_mmu_invalidate_cache_trigger()
5770 u64 start = inv_params->start_va - 1; in gaudi2_mmu_invalidate_cache_trigger()
5781 inv_params->end_va >> MMU_RANGE_INV_VA_LSB_SHIFT); in gaudi2_mmu_invalidate_cache_trigger()
5784 inv_params->end_va >> MMU_RANGE_INV_VA_MSB_SHIFT); in gaudi2_mmu_invalidate_cache_trigger()
5790 inv_params->inv_start_val, inv_params->flags); in gaudi2_mmu_invalidate_cache_trigger()
5846 struct gaudi2_device *gaudi2 = hdev->asic_specific; in gaudi2_mmu_invalidate_cache()
5850 if (hdev->reset_info.hard_reset_pending) in gaudi2_mmu_invalidate_cache()
5856 if ((flags & MMU_OP_USERPTR) && (gaudi2->hw_cap_initialized & HW_CAP_PMMU)) { in gaudi2_mmu_invalidate_cache()
5873 struct gaudi2_device *gaudi2 = hdev->asic_specific; in gaudi2_mmu_invalidate_cache_range()
5878 if (hdev->reset_info.hard_reset_pending) in gaudi2_mmu_invalidate_cache_range()
5887 if ((flags & MMU_OP_USERPTR) && (gaudi2->hw_cap_initialized & HW_CAP_PMMU)) { in gaudi2_mmu_invalidate_cache_range()
5924 struct asic_fixed_properties *prop = &hdev->asic_prop; in gaudi2_mmu_update_hop0_addr()
5926 u32 asid, max_asid = prop->max_asid; in gaudi2_mmu_update_hop0_addr()
5930 if (hdev->pldm) in gaudi2_mmu_update_hop0_addr()
5935 hop0_addr = hdev->mmu_priv.hr.mmu_asid_hop0[asid].phys_addr; in gaudi2_mmu_update_hop0_addr()
5937 hop0_addr = prop->mmu_pgt_addr + (asid * prop->dmmu.hop_table_size); in gaudi2_mmu_update_hop0_addr()
5941 dev_err(hdev->dev, "failed to set hop0 addr for asid %d\n", asid); in gaudi2_mmu_update_hop0_addr()
5955 if (hdev->pldm || !hdev->pdev) in gaudi2_mmu_init_common()
5971 dev_notice_ratelimited(hdev->dev, "Timeout when waiting for MMU SRAM init\n"); in gaudi2_mmu_init_common()
5988 dev_notice_ratelimited(hdev->dev, "Timeout when waiting for MMU invalidate all\n"); in gaudi2_mmu_init_common()
5997 struct asic_fixed_properties *prop = &hdev->asic_prop; in gaudi2_pci_mmu_init()
5998 struct gaudi2_device *gaudi2 = hdev->asic_specific; in gaudi2_pci_mmu_init()
6002 if (gaudi2->hw_cap_initialized & HW_CAP_PMMU) in gaudi2_pci_mmu_init()
6037 rc = gaudi2_mmu_init_common(hdev, mmu_base, stlb_base, prop->pmmu.host_resident); in gaudi2_pci_mmu_init()
6041 gaudi2->hw_cap_initialized |= HW_CAP_PMMU; in gaudi2_pci_mmu_init()
6049 struct asic_fixed_properties *prop = &hdev->asic_prop; in gaudi2_dcore_hmmu_init()
6050 struct gaudi2_device *gaudi2 = hdev->asic_specific; in gaudi2_dcore_hmmu_init()
6062 if ((gaudi2->hw_cap_initialized & hw_cap) || !(prop->hmmu_hif_enabled_mask & BIT(dmmu_seq))) in gaudi2_dcore_hmmu_init()
6089 rc = gaudi2_mmu_init_common(hdev, mmu_base, stlb_base, prop->dmmu.host_resident); in gaudi2_dcore_hmmu_init()
6093 gaudi2->hw_cap_initialized |= hw_cap; in gaudi2_dcore_hmmu_init()
6129 struct gaudi2_device *gaudi2 = hdev->asic_specific; in gaudi2_hw_init()
6146 if (hdev->asic_prop.iatu_done_by_fw) in gaudi2_hw_init()
6147 gaudi2->dram_bar_cur_addr = DRAM_PHYS_BASE; in gaudi2_hw_init()
6150 * Before pushing u-boot/linux to device, need to set the hbm bar to in gaudi2_hw_init()
6154 dev_err(hdev->dev, "failed to map HBM bar to DRAM base address\n"); in gaudi2_hw_init()
6155 return -EIO; in gaudi2_hw_init()
6160 dev_err(hdev->dev, "failed to initialize CPU\n"); in gaudi2_hw_init()
6169 dev_err(hdev->dev, "failed to initialize CPU H/W queues %d\n", rc); in gaudi2_hw_init()
6173 rc = gaudi2->cpucp_info_get(hdev); in gaudi2_hw_init()
6175 dev_err(hdev->dev, "Failed to get cpucp info\n"); in gaudi2_hw_init()
6218 * gaudi2_send_hard_reset_cmd - common function to handle reset
6227 struct cpu_dyn_regs *dyn_regs = &hdev->fw_loader.dynamic_loader.comm_desc.cpu_dyn_regs; in gaudi2_send_hard_reset_cmd()
6229 struct gaudi2_device *gaudi2 = hdev->asic_specific; in gaudi2_send_hard_reset_cmd()
6232 preboot_only = (hdev->fw_loader.fw_comp_loaded == FW_TYPE_PREBOOT_CPU); in gaudi2_send_hard_reset_cmd()
6233 heartbeat_reset = (hdev->reset_info.curr_reset_cause == HL_RESET_CAUSE_HEARTBEAT); in gaudi2_send_hard_reset_cmd()
6248 if (gaudi2 && (gaudi2->hw_cap_initialized & HW_CAP_CPU) && in gaudi2_send_hard_reset_cmd()
6261 WREG32(le32_to_cpu(dyn_regs->gic_host_halt_irq), in gaudi2_send_hard_reset_cmd()
6271 * For the case in which we are working with Linux/Bootfit this is a hail-mary in gaudi2_send_hard_reset_cmd()
6285 if (hdev->asic_prop.hard_reset_done_by_fw) in gaudi2_send_hard_reset_cmd()
6293 * gaudi2_execute_hard_reset - execute hard reset by driver/FW
6301 if (hdev->asic_prop.hard_reset_done_by_fw) { in gaudi2_execute_hard_reset()
6318 * gaudi2_execute_soft_reset - execute soft reset by driver/FW
6336 gaudi2_write_rr_to_all_lbw_rtrs(hdev, RR_TYPE_LONG, NUM_LONG_LBW_RR - 1, in gaudi2_execute_soft_reset()
6339 gaudi2_write_rr_to_all_lbw_rtrs(hdev, RR_TYPE_LONG, NUM_LONG_LBW_RR - 2, in gaudi2_execute_soft_reset()
6365 dev_err(hdev->dev, "Timeout while waiting for device to reset 0x%x\n", reg_val); in gaudi2_poll_btm_indication()
6370 struct gaudi2_device *gaudi2 = hdev->asic_specific; in gaudi2_hw_fini()
6375 if (hdev->pldm) { in gaudi2_hw_fini()
6390 driver_performs_reset = !hdev->asic_prop.hard_reset_done_by_fw; in gaudi2_hw_fini()
6399 driver_performs_reset = (hdev->fw_components == FW_TYPE_PREBOOT_CPU && in gaudi2_hw_fini()
6400 !hdev->asic_prop.fw_security_enabled); in gaudi2_hw_fini()
6412 * - setting the dirty bit in gaudi2_hw_fini()
6415 * - dirty bit cleared in gaudi2_hw_fini()
6416 * - BTM indication cleared in gaudi2_hw_fini()
6417 * - preboot ready indication cleared in gaudi2_hw_fini()
6419 * - BTM indication will be set in gaudi2_hw_fini()
6420 * - BIST test performed and another reset triggered in gaudi2_hw_fini()
6431 if (hdev->fw_components & FW_TYPE_PREBOOT_CPU) in gaudi2_hw_fini()
6440 gaudi2->dec_hw_cap_initialized &= ~(HW_CAP_DEC_MASK); in gaudi2_hw_fini()
6441 gaudi2->tpc_hw_cap_initialized &= ~(HW_CAP_TPC_MASK); in gaudi2_hw_fini()
6444 * Clear NIC capability mask in order for driver to re-configure in gaudi2_hw_fini()
6445 * NIC QMANs. NIC ports will not be re-configured during soft in gaudi2_hw_fini()
6448 gaudi2->nic_hw_cap_initialized &= ~(HW_CAP_NIC_MASK); in gaudi2_hw_fini()
6451 gaudi2->hw_cap_initialized &= in gaudi2_hw_fini()
6458 memset(gaudi2->events_stat, 0, sizeof(gaudi2->events_stat)); in gaudi2_hw_fini()
6460 gaudi2->hw_cap_initialized &= in gaudi2_hw_fini()
6488 rc = dma_mmap_coherent(hdev->dev, vma, cpu_addr, dma_addr, size); in gaudi2_mmap()
6490 dev_err(hdev->dev, "dma_mmap_coherent error %d", rc); in gaudi2_mmap()
6494 rc = remap_pfn_range(vma, vma->vm_start, in gaudi2_mmap()
6496 size, vma->vm_page_prot); in gaudi2_mmap()
6498 dev_err(hdev->dev, "remap_pfn_range error %d", rc); in gaudi2_mmap()
6507 struct gaudi2_device *gaudi2 = hdev->asic_specific; in gaudi2_is_queue_enabled()
6521 ((hw_queue_id - GAUDI2_QUEUE_ID_DCORE0_EDMA_0_0) >> 2); in gaudi2_is_queue_enabled()
6525 ((hw_queue_id - GAUDI2_QUEUE_ID_DCORE1_EDMA_0_0) >> 2); in gaudi2_is_queue_enabled()
6529 ((hw_queue_id - GAUDI2_QUEUE_ID_DCORE2_EDMA_0_0) >> 2); in gaudi2_is_queue_enabled()
6533 ((hw_queue_id - GAUDI2_QUEUE_ID_DCORE3_EDMA_0_0) >> 2); in gaudi2_is_queue_enabled()
6554 ((hw_queue_id - GAUDI2_QUEUE_ID_DCORE0_TPC_0_0) >> 2); in gaudi2_is_queue_enabled()
6558 return !!(gaudi2->tpc_hw_cap_initialized & BIT_ULL(0)); in gaudi2_is_queue_enabled()
6563 ((hw_queue_id - GAUDI2_QUEUE_ID_DCORE1_TPC_0_0) >> 2); in gaudi2_is_queue_enabled()
6568 ((hw_queue_id - GAUDI2_QUEUE_ID_DCORE2_TPC_0_0) >> 2); in gaudi2_is_queue_enabled()
6573 ((hw_queue_id - GAUDI2_QUEUE_ID_DCORE3_TPC_0_0) >> 2); in gaudi2_is_queue_enabled()
6581 hw_test_cap_bit = HW_CAP_ROT_SHIFT + ((hw_queue_id - GAUDI2_QUEUE_ID_ROT_0_0) >> 2); in gaudi2_is_queue_enabled()
6585 hw_nic_cap_bit = HW_CAP_NIC_SHIFT + ((hw_queue_id - GAUDI2_QUEUE_ID_NIC_0_0) >> 2); in gaudi2_is_queue_enabled()
6589 return !!(gaudi2->nic_hw_cap_initialized & BIT_ULL(0)); in gaudi2_is_queue_enabled()
6593 return !!(gaudi2->hw_cap_initialized & HW_CAP_CPU_Q); in gaudi2_is_queue_enabled()
6600 return !!(gaudi2->tpc_hw_cap_initialized & BIT_ULL(hw_tpc_cap_bit)); in gaudi2_is_queue_enabled()
6603 return !!(gaudi2->nic_hw_cap_initialized & BIT_ULL(hw_nic_cap_bit)); in gaudi2_is_queue_enabled()
6608 return !!(gaudi2->hw_cap_initialized & hw_cap_mask); in gaudi2_is_queue_enabled()
6613 struct gaudi2_device *gaudi2 = hdev->asic_specific; in gaudi2_is_arc_enabled()
6618 return !!(gaudi2->active_hw_arc & BIT_ULL(arc_id)); in gaudi2_is_arc_enabled()
6621 return !!(gaudi2->active_tpc_arc & BIT_ULL(arc_id - CPU_ID_TPC_QMAN_ARC0)); in gaudi2_is_arc_enabled()
6624 return !!(gaudi2->active_nic_arc & BIT_ULL(arc_id - CPU_ID_NIC_QMAN_ARC0)); in gaudi2_is_arc_enabled()
6633 struct gaudi2_device *gaudi2 = hdev->asic_specific; in gaudi2_clr_arc_id_cap()
6638 gaudi2->active_hw_arc &= ~(BIT_ULL(arc_id)); in gaudi2_clr_arc_id_cap()
6642 gaudi2->active_tpc_arc &= ~(BIT_ULL(arc_id - CPU_ID_TPC_QMAN_ARC0)); in gaudi2_clr_arc_id_cap()
6646 gaudi2->active_nic_arc &= ~(BIT_ULL(arc_id - CPU_ID_NIC_QMAN_ARC0)); in gaudi2_clr_arc_id_cap()
6656 struct gaudi2_device *gaudi2 = hdev->asic_specific; in gaudi2_set_arc_id_cap()
6661 gaudi2->active_hw_arc |= BIT_ULL(arc_id); in gaudi2_set_arc_id_cap()
6665 gaudi2->active_tpc_arc |= BIT_ULL(arc_id - CPU_ID_TPC_QMAN_ARC0); in gaudi2_set_arc_id_cap()
6669 gaudi2->active_nic_arc |= BIT_ULL(arc_id - CPU_ID_NIC_QMAN_ARC0); in gaudi2_set_arc_id_cap()
6679 struct cpu_dyn_regs *dyn_regs = &hdev->fw_loader.dynamic_loader.comm_desc.cpu_dyn_regs; in gaudi2_ring_doorbell()
6703 WREG32(le32_to_cpu(dyn_regs->gic_host_pi_upd_irq), in gaudi2_ring_doorbell()
6720 return dma_alloc_coherent(&hdev->pdev->dev, size, dma_handle, flags); in gaudi2_dma_alloc_coherent()
6726 dma_free_coherent(&hdev->pdev->dev, size, cpu_addr, dma_handle); in gaudi2_dma_free_coherent()
6732 struct gaudi2_device *gaudi2 = hdev->asic_specific; in gaudi2_send_cpu_message()
6734 if (!(gaudi2->hw_cap_initialized & HW_CAP_CPU_Q)) { in gaudi2_send_cpu_message()
6752 return dma_pool_zalloc(hdev->dma_pool, mem_flags, dma_handle); in gaudi2_dma_pool_zalloc()
6757 dma_pool_free(hdev->dma_pool, vaddr, dma_addr); in gaudi2_dma_pool_free()
6773 struct asic_fixed_properties *asic_prop = &hdev->asic_prop; in gaudi2_validate_cb_address()
6774 struct gaudi2_device *gaudi2 = hdev->asic_specific; in gaudi2_validate_cb_address()
6776 if (!gaudi2_is_queue_enabled(hdev, parser->hw_queue_id)) { in gaudi2_validate_cb_address()
6777 dev_err(hdev->dev, "h/w queue %d is disabled\n", parser->hw_queue_id); in gaudi2_validate_cb_address()
6778 return -EINVAL; in gaudi2_validate_cb_address()
6783 if (hl_mem_area_inside_range((u64) (uintptr_t) parser->user_cb, in gaudi2_validate_cb_address()
6784 parser->user_cb_size, in gaudi2_validate_cb_address()
6785 asic_prop->sram_user_base_address, in gaudi2_validate_cb_address()
6786 asic_prop->sram_end_address)) in gaudi2_validate_cb_address()
6789 if (hl_mem_area_inside_range((u64) (uintptr_t) parser->user_cb, in gaudi2_validate_cb_address()
6790 parser->user_cb_size, in gaudi2_validate_cb_address()
6791 asic_prop->dram_user_base_address, in gaudi2_validate_cb_address()
6792 asic_prop->dram_end_address)) in gaudi2_validate_cb_address()
6795 if ((gaudi2->hw_cap_initialized & HW_CAP_DMMU_MASK) && in gaudi2_validate_cb_address()
6796 hl_mem_area_inside_range((u64) (uintptr_t) parser->user_cb, in gaudi2_validate_cb_address()
6797 parser->user_cb_size, in gaudi2_validate_cb_address()
6798 asic_prop->dmmu.start_addr, in gaudi2_validate_cb_address()
6799 asic_prop->dmmu.end_addr)) in gaudi2_validate_cb_address()
6802 if (gaudi2->hw_cap_initialized & HW_CAP_PMMU) { in gaudi2_validate_cb_address()
6803 if (hl_mem_area_inside_range((u64) (uintptr_t) parser->user_cb, in gaudi2_validate_cb_address()
6804 parser->user_cb_size, in gaudi2_validate_cb_address()
6805 asic_prop->pmmu.start_addr, in gaudi2_validate_cb_address()
6806 asic_prop->pmmu.end_addr) || in gaudi2_validate_cb_address()
6808 (u64) (uintptr_t) parser->user_cb, in gaudi2_validate_cb_address()
6809 parser->user_cb_size, in gaudi2_validate_cb_address()
6810 asic_prop->pmmu_huge.start_addr, in gaudi2_validate_cb_address()
6811 asic_prop->pmmu_huge.end_addr)) in gaudi2_validate_cb_address()
6814 } else if (gaudi2_host_phys_addr_valid((u64) (uintptr_t) parser->user_cb)) { in gaudi2_validate_cb_address()
6815 if (!hdev->pdev) in gaudi2_validate_cb_address()
6818 if (!device_iommu_mapped(&hdev->pdev->dev)) in gaudi2_validate_cb_address()
6822 dev_err(hdev->dev, "CB address %p + 0x%x for internal QMAN is not valid\n", in gaudi2_validate_cb_address()
6823 parser->user_cb, parser->user_cb_size); in gaudi2_validate_cb_address()
6825 return -EFAULT; in gaudi2_validate_cb_address()
6830 struct gaudi2_device *gaudi2 = hdev->asic_specific; in gaudi2_cs_parser()
6832 if (!parser->is_kernel_allocated_cb) in gaudi2_cs_parser()
6835 if (!(gaudi2->hw_cap_initialized & HW_CAP_PMMU)) { in gaudi2_cs_parser()
6836 dev_err(hdev->dev, "PMMU not initialized - Unsupported mode in Gaudi2\n"); in gaudi2_cs_parser()
6837 return -EINVAL; in gaudi2_cs_parser()
6845 struct gaudi2_device *gaudi2 = hdev->asic_specific; in gaudi2_send_heartbeat()
6847 if (!(gaudi2->hw_cap_initialized & HW_CAP_CPU_Q)) in gaudi2_send_heartbeat()
6939 cq = &hdev->completion_queue[GAUDI2_RESERVED_CQ_KDMA_COMPLETION]; in gaudi2_send_job_to_kdma()
6940 cq_base = cq->kernel_address; in gaudi2_send_job_to_kdma()
6941 polling_addr = (u32 *)&cq_base[cq->ci]; in gaudi2_send_job_to_kdma()
6943 if (hdev->pldm) in gaudi2_send_job_to_kdma()
6962 dev_err(hdev->dev, "Timeout while waiting for KDMA to be idle\n"); in gaudi2_send_job_to_kdma()
6967 cq->ci = hl_cq_inc_ptr(cq->ci); in gaudi2_send_job_to_kdma()
6995 return hdev->asic_prop.first_available_user_sob[0] + in gaudi2_test_queue_hw_queue_id_to_sob_id()
6996 hw_queue_id - GAUDI2_QUEUE_ID_PDMA_0_0; in gaudi2_test_queue_hw_queue_id_to_sob_id()
7013 struct packet_msg_short *msg_short_pkt = msg_info->kern_addr; in gaudi2_test_queue_send_msg_short()
7023 msg_short_pkt->value = cpu_to_le32(sob_val); in gaudi2_test_queue_send_msg_short()
7024 msg_short_pkt->ctl = cpu_to_le32(tmp); in gaudi2_test_queue_send_msg_short()
7026 rc = hl_hw_queue_send_cb_no_cmpl(hdev, hw_queue_id, pkt_size, msg_info->dma_addr); in gaudi2_test_queue_send_msg_short()
7028 dev_err(hdev->dev, in gaudi2_test_queue_send_msg_short()
7041 if (hdev->pldm) in gaudi2_test_queue_wait_completion()
7054 if (rc == -ETIMEDOUT) { in gaudi2_test_queue_wait_completion()
7055 dev_err(hdev->dev, "H/W queue %d test failed (SOB_OBJ_0 == 0x%x)\n", in gaudi2_test_queue_wait_completion()
7057 rc = -EIO; in gaudi2_test_queue_wait_completion()
7065 struct gaudi2_device *gaudi2 = hdev->asic_specific; in gaudi2_test_cpu_queue()
7071 if (!(gaudi2->hw_cap_initialized & HW_CAP_CPU_Q)) in gaudi2_test_cpu_queue()
7079 struct gaudi2_device *gaudi2 = hdev->asic_specific; in gaudi2_test_queues()
7089 msg_info = &gaudi2->queues_test_info[i - GAUDI2_QUEUE_ID_PDMA_0_0]; in gaudi2_test_queues()
7108 /* chip is not usable, no need for cleanups, just bail-out with error */ in gaudi2_test_queues()
7121 struct gaudi2_device *gaudi2 = hdev->asic_specific; in gaudi2_compute_reset_late_init()
7129 dev_err(hdev->dev, "Failed to scrub arcs DCCM\n"); in gaudi2_compute_reset_late_init()
7136 irq_arr_size = gaudi2->num_of_valid_hw_events * sizeof(gaudi2->hw_events[0]); in gaudi2_compute_reset_late_init()
7137 return hl_fw_unmask_irq_arr(hdev, gaudi2->hw_events, irq_arr_size); in gaudi2_compute_reset_late_init()
7144 struct asic_fixed_properties *prop = &hdev->asic_prop; in gaudi2_get_edma_idle_status()
7146 const char *edma_fmt = "%-6d%-6d%-9s%#-14x%#-15x%#x\n"; in gaudi2_get_edma_idle_status()
7154 "---- ---- ------- ------------ ------------- -------------\n"); in gaudi2_get_edma_idle_status()
7160 if (!(prop->edma_enabled_mask & BIT(seq))) in gaudi2_get_edma_idle_status()
7195 const char *pdma_fmt = "%-6d%-9s%#-14x%#-15x%#x\n"; in gaudi2_get_pdma_idle_status()
7203 "---- ------- ------------ ------------- -------------\n"); in gaudi2_get_pdma_idle_status()
7234 const char *nic_fmt = "%-5d%-9s%#-14x%#-12x\n"; in gaudi2_get_nic_idle_status()
7241 if (e && hdev->nic_ports_mask) in gaudi2_get_nic_idle_status()
7244 "--- ------- ------------ ----------\n"); in gaudi2_get_nic_idle_status()
7252 if (!(hdev->nic_ports_mask & BIT(i))) in gaudi2_get_nic_idle_status()
7281 const char *mme_fmt = "%-5d%-6s%-9s%#-14x%#x\n"; in gaudi2_get_mme_idle_status()
7289 "--- ---- ------- ------------ ---------------\n"); in gaudi2_get_mme_idle_status()
7322 struct gaudi2_tpc_idle_data *idle_data = ctx->data; in gaudi2_is_tpc_engine_idle()
7327 if ((dcore == 0) && (inst == (NUM_DCORE0_TPC - 1))) in gaudi2_is_tpc_engine_idle()
7340 *(idle_data->is_idle) &= is_eng_idle; in gaudi2_is_tpc_engine_idle()
7342 if (idle_data->mask && !is_eng_idle) in gaudi2_is_tpc_engine_idle()
7343 set_bit(engine_idx, idle_data->mask); in gaudi2_is_tpc_engine_idle()
7345 if (idle_data->e) in gaudi2_is_tpc_engine_idle()
7346 hl_engine_data_sprintf(idle_data->e, in gaudi2_is_tpc_engine_idle()
7347 idle_data->tpc_fmt, dcore, inst, in gaudi2_is_tpc_engine_idle()
7355 struct asic_fixed_properties *prop = &hdev->asic_prop; in gaudi2_get_tpc_idle_status()
7360 .tpc_fmt = "%-6d%-5d%-9s%#-14x%#-12x%#x\n", in gaudi2_get_tpc_idle_status()
7370 if (e && prop->tpc_enabled_mask) in gaudi2_get_tpc_idle_status()
7373 "---- --- ------- ------------ ---------- ------\n"); in gaudi2_get_tpc_idle_status()
7383 struct asic_fixed_properties *prop = &hdev->asic_prop; in gaudi2_get_decoder_idle_status()
7385 const char *pcie_dec_fmt = "%-10d%-9s%#x\n"; in gaudi2_get_decoder_idle_status()
7386 const char *dec_fmt = "%-6d%-5d%-9s%#x\n"; in gaudi2_get_decoder_idle_status()
7393 if (e && (prop->decoder_enabled_mask & (~PCIE_DEC_EN_MASK))) in gaudi2_get_decoder_idle_status()
7396 "---- --- ------- ---------------\n"); in gaudi2_get_decoder_idle_status()
7401 if (!(prop->decoder_enabled_mask & dec_enabled_bit)) in gaudi2_get_decoder_idle_status()
7421 if (e && (prop->decoder_enabled_mask & PCIE_DEC_EN_MASK)) in gaudi2_get_decoder_idle_status()
7424 "-------- ------- ---------------\n"); in gaudi2_get_decoder_idle_status()
7429 if (!(prop->decoder_enabled_mask & BIT(dec_enabled_bit))) in gaudi2_get_decoder_idle_status()
7452 const char *rot_fmt = "%-6d%-5d%-9s%#-14x%#-14x%#x\n"; in gaudi2_get_rotator_idle_status()
7462 "---- --- ------- ------------ ------------ ----------\n"); in gaudi2_get_rotator_idle_status()
7504 __acquires(&gaudi2->hw_queues_lock) in gaudi2_hw_queues_lock()
7506 struct gaudi2_device *gaudi2 = hdev->asic_specific; in gaudi2_hw_queues_lock()
7508 spin_lock(&gaudi2->hw_queues_lock); in gaudi2_hw_queues_lock()
7512 __releases(&gaudi2->hw_queues_lock) in gaudi2_hw_queues_unlock()
7514 struct gaudi2_device *gaudi2 = hdev->asic_specific; in gaudi2_hw_queues_unlock()
7516 spin_unlock(&gaudi2->hw_queues_lock); in gaudi2_hw_queues_unlock()
7521 return hdev->pdev->device; in gaudi2_get_pci_id()
7526 struct gaudi2_device *gaudi2 = hdev->asic_specific; in gaudi2_get_eeprom_data()
7528 if (!(gaudi2->hw_cap_initialized & HW_CAP_CPU_Q)) in gaudi2_get_eeprom_data()
7541 struct gaudi2_device *gaudi2 = hdev->asic_specific; in gaudi2_get_events_stat()
7544 *size = (u32) sizeof(gaudi2->events_stat_aggregate); in gaudi2_get_events_stat()
7545 return gaudi2->events_stat_aggregate; in gaudi2_get_events_stat()
7548 *size = (u32) sizeof(gaudi2->events_stat); in gaudi2_get_events_stat()
7549 return gaudi2->events_stat; in gaudi2_get_events_stat()
7555 u32 offset = (mmDCORE0_VDEC1_BRDG_CTRL_BASE - mmDCORE0_VDEC0_BRDG_CTRL_BASE) * in gaudi2_mmu_vdec_dcore_prepare()
7578 struct asic_fixed_properties *prop = &hdev->asic_prop; in gaudi2_mmu_dcore_prepare()
7585 if (prop->edma_enabled_mask & BIT(edma_seq_base)) { in gaudi2_mmu_dcore_prepare()
7592 if (prop->edma_enabled_mask & BIT(edma_seq_base + 1)) { in gaudi2_mmu_dcore_prepare()
7602 * Sync Mngrs on dcores 1 - 3 are exposed to user, so must use user ASID in gaudi2_mmu_dcore_prepare()
7638 if (prop->decoder_enabled_mask & BIT(dcore_id * NUM_OF_DEC_PER_DCORE + vdec_id)) in gaudi2_mmu_dcore_prepare()
7646 u32 offset = (mmPCIE_VDEC1_BRDG_CTRL_BASE - mmPCIE_VDEC0_BRDG_CTRL_BASE) * shared_vdec_id; in gudi2_mmu_vdec_shared_prepare()
7667 u32 offset = (mmARC_FARM_ARC1_DUP_ENG_BASE - mmARC_FARM_ARC0_DUP_ENG_BASE) * arc_farm_id; in gudi2_mmu_arc_farm_arc_dup_eng_prepare()
7679 /* Enable MMU and configure asid for all relevant ARC regions */ in gaudi2_arc_mmu_prepare()
7721 if (hdev->fw_components & FW_TYPE_BOOT_CPU) in gaudi2_arc_mmu_prepare_all()
7739 struct asic_fixed_properties *prop = &hdev->asic_prop; in gaudi2_mmu_shared_prepare()
7767 if (prop->decoder_enabled_mask & BIT(NUM_OF_DCORES * NUM_OF_DEC_PER_DCORE + 0)) in gaudi2_mmu_shared_prepare()
7770 if (prop->decoder_enabled_mask & BIT(NUM_OF_DCORES * NUM_OF_DEC_PER_DCORE + 1)) in gaudi2_mmu_shared_prepare()
7787 struct gaudi2_tpc_mmu_data *mmu_data = ctx->data; in gaudi2_tpc_mmu_prepare()
7790 WREG32(mmDCORE0_TPC0_CFG_AXUSER_HB_ASID + offset, mmu_data->rw_asid); in gaudi2_tpc_mmu_prepare()
7792 WREG32(mmDCORE0_TPC0_QM_AXUSER_NONSECURED_HB_ASID + offset, mmu_data->rw_asid); in gaudi2_tpc_mmu_prepare()
7798 struct gaudi2_device *gaudi2 = hdev->asic_specific; in gaudi2_mmu_prepare()
7807 dev_crit(hdev->dev, "asid %u is too big\n", asid); in gaudi2_mmu_prepare()
7808 return -EINVAL; in gaudi2_mmu_prepare()
7811 if (!(gaudi2->hw_cap_initialized & HW_CAP_MMU_MASK)) in gaudi2_mmu_prepare()
7835 /* return in case of NIC status event - these events are received periodically and not as in is_info_event()
7857 dev_err_ratelimited(hdev->dev, "%s: %pV\n", in gaudi2_print_event()
7861 dev_err(hdev->dev, "%s: %pV\n", in gaudi2_print_event()
7879 ecc_address = le64_to_cpu(ecc_data->ecc_address); in gaudi2_handle_ecc_event()
7880 ecc_syndrome = le64_to_cpu(ecc_data->ecc_syndrom); in gaudi2_handle_ecc_event()
7881 memory_wrapper_idx = ecc_data->memory_wrapper_idx; in gaudi2_handle_ecc_event()
7884 block_id = le16_to_cpu(ecc_data->block_id); in gaudi2_handle_ecc_event()
7885 gaudi2_print_event(hdev, event_type, !ecc_data->is_critical, in gaudi2_handle_ecc_event()
7888 ecc_data->is_critical); in gaudi2_handle_ecc_event()
7890 gaudi2_print_event(hdev, event_type, !ecc_data->is_critical, in gaudi2_handle_ecc_event()
7892 ecc_address, ecc_syndrome, memory_wrapper_idx, ecc_data->is_critical); in gaudi2_handle_ecc_event()
7895 return !!ecc_data->is_critical; in gaudi2_handle_ecc_event()
7900 struct undefined_opcode_info *undef_opcode = &hdev->captured_err_info.undef_opcode; in handle_lower_qman_data_on_err()
7906 is_arc_cq = FIELD_GET(PDMA0_QM_CP_STS_CUR_CQ_MASK, cp_sts); /* 0 - legacy CQ, 1 - ARC_CQ */ in handle_lower_qman_data_on_err()
7924 dev_info(hdev->dev, in handle_lower_qman_data_on_err()
7928 if (undef_opcode->write_enable) { in handle_lower_qman_data_on_err()
7930 undef_opcode->timestamp = ktime_get(); in handle_lower_qman_data_on_err()
7931 undef_opcode->cq_addr = cq_ptr; in handle_lower_qman_data_on_err()
7932 undef_opcode->cq_size = cq_size; in handle_lower_qman_data_on_err()
7933 undef_opcode->engine_id = engine_id; in handle_lower_qman_data_on_err()
7934 undef_opcode->stream_id = QMAN_STREAMS; in handle_lower_qman_data_on_err()
7935 undef_opcode->write_enable = 0; in handle_lower_qman_data_on_err()
7946 glbl_sts_addr = qman_base + (mmDCORE0_TPC0_QM_GLBL_ERR_STS_0 - mmDCORE0_TPC0_QM_BASE); in gaudi2_handle_qman_err_generic()
7947 arb_err_addr = qman_base + (mmDCORE0_TPC0_QM_ARB_ERR_CAUSE - mmDCORE0_TPC0_QM_BASE); in gaudi2_handle_qman_err_generic()
8024 dev_err_ratelimited(hdev->dev, in gaudi2_razwi_rr_hbw_shared_printf_info()
8025 "%s-RAZWI SHARED RR HBW %s error, address %#llx, Initiator coordinates 0x%x\n", in gaudi2_razwi_rr_hbw_shared_printf_info()
8049 dev_err_ratelimited(hdev->dev, in gaudi2_razwi_rr_lbw_shared_printf_info()
8050 …"%s-RAZWI SHARED RR LBW %s error, mstr_if 0x%llx, captured address 0x%llX Initiator coordinates 0x… in gaudi2_razwi_rr_lbw_shared_printf_info()
8064 (GAUDI2_DCORE0_ENGINE_ID_TPC_0 - GAUDI2_DCORE0_ENGINE_ID_EDMA_0)); in gaudi2_razwi_calc_engine_id()
8067 return ((GAUDI2_DCORE0_ENGINE_ID_MME - GAUDI2_DCORE0_ENGINE_ID_EDMA_0) + in gaudi2_razwi_calc_engine_id()
8089 (GAUDI2_DCORE0_ENGINE_ID_DEC_0 - GAUDI2_DCORE0_ENGINE_ID_EDMA_0)); in gaudi2_razwi_calc_engine_id()
8120 if (hdev->tpc_binning) { in gaudi2_ack_module_razwi_event_handler()
8121 binned_idx = __ffs(hdev->tpc_binning); in gaudi2_ack_module_razwi_event_handler()
8187 if (hdev->decoder_binning) { in gaudi2_ack_module_razwi_event_handler()
8188 binned_idx = __ffs(hdev->decoder_binning); in gaudi2_ack_module_razwi_event_handler()
8218 (((s32)lbw_rtr_id - hbw_rtr_id) * DCORE_RTR_OFFSET); in gaudi2_ack_module_razwi_event_handler()
8263 struct asic_fixed_properties *prop = &hdev->asic_prop; in gaudi2_check_if_razwi_happened()
8268 if (prop->tpc_enabled_mask & BIT(mod_idx)) in gaudi2_check_if_razwi_happened()
8280 if (prop->edma_enabled_mask & BIT(mod_idx)) in gaudi2_check_if_razwi_happened()
8289 if (hdev->nic_ports_mask & BIT(mod_idx)) in gaudi2_check_if_razwi_happened()
8295 if (prop->decoder_enabled_mask & BIT(mod_idx)) in gaudi2_check_if_razwi_happened()
8319 PSOC_RAZWI_ENG_STR_SIZE - str_size, "%s", in gaudi2_psoc_razwi_get_engines()
8323 PSOC_RAZWI_ENG_STR_SIZE - str_size, " or %s", in gaudi2_psoc_razwi_get_engines()
8360 dev_err(hdev->dev, in gaudi2_handle_psoc_razwi_happened()
8374 dev_err(hdev->dev, in gaudi2_handle_psoc_razwi_happened()
8386 dev_err(hdev->dev, in gaudi2_handle_psoc_razwi_happened()
8398 dev_err(hdev->dev, in gaudi2_handle_psoc_razwi_happened()
8424 if (hdev->pldm || !(hdev->fw_components & FW_TYPE_LINUX)) { in gaudi2_ack_psoc_razwi_event_handler()
8432 dev_err_ratelimited(hdev->dev, in gaudi2_ack_psoc_razwi_event_handler()
8443 dev_err_ratelimited(hdev->dev, in gaudi2_ack_psoc_razwi_event_handler()
8448 if (hdev->pldm || !(hdev->fw_components & FW_TYPE_LINUX)) in gaudi2_ack_psoc_razwi_event_handler()
8484 index = event_type - GAUDI2_EVENT_TPC0_AXI_ERR_RSP; in gaudi2_handle_qm_sei_err()
8498 index = (event_type - GAUDI2_EVENT_MME0_CTRL_AXI_ERROR_RESPONSE) / in gaudi2_handle_qm_sei_err()
8499 (GAUDI2_EVENT_MME1_CTRL_AXI_ERROR_RESPONSE - in gaudi2_handle_qm_sei_err()
8506 index = event_type - GAUDI2_EVENT_PDMA_CH0_AXI_ERR_RSP; in gaudi2_handle_qm_sei_err()
8512 index = event_type - GAUDI2_EVENT_ROTATOR0_AXI_ERROR_RESPONSE; in gaudi2_handle_qm_sei_err()
8545 index = event_type - GAUDI2_EVENT_TPC0_QM; in gaudi2_handle_qman_err()
8550 index = event_type - GAUDI2_EVENT_TPC6_QM; in gaudi2_handle_qman_err()
8555 index = event_type - GAUDI2_EVENT_TPC12_QM; in gaudi2_handle_qman_err()
8560 index = event_type - GAUDI2_EVENT_TPC18_QM; in gaudi2_handle_qman_err()
8712 u64 intr_cause_data = le64_to_cpu(razwi_with_intr_cause->intr_cause.intr_cause_data); in gaudi2_handle_rot_err()
8734 u64 intr_cause_data = le64_to_cpu(razwi_with_intr_cause->intr_cause.intr_cause_data); in gaudi2_tpc_ack_interrupts()
8766 (dec_index - NUM_OF_VDEC_PER_DCORE * NUM_OF_DCORES); in gaudi2_handle_dec_err()
9013 dev_err_ratelimited(hdev->dev, "PMMU page fault on va 0x%llx\n", addr); in gaudi2_handle_page_error()
9017 dev_err_ratelimited(hdev->dev, "HMMU page fault on va range 0x%llx - 0x%llx\n", in gaudi2_handle_page_error()
9044 dev_err_ratelimited(hdev->dev, "%s access error on va 0x%llx\n", in gaudi2_handle_access_error()
9125 dev_err_ratelimited(hdev->dev, "SM%u err. err cause: CQ_INTR. queue index: %u\n", in gaudi2_handle_sm_err()
9269 dev_err_ratelimited(hdev->dev, in gaudi2_hbm_sei_handle_read_err()
9275 addr = le32_to_cpu(rd_err_data->dbg_rd_err_addr.rd_addr_val); in gaudi2_hbm_sei_handle_read_err()
9276 dev_err_ratelimited(hdev->dev, in gaudi2_hbm_sei_handle_read_err()
9286 if (le32_to_cpu(rd_err_data->dbg_rd_err_misc) & in gaudi2_hbm_sei_handle_read_err()
9288 dev_err_ratelimited(hdev->dev, "Beat%d ECC SERR: DM: %#x, Syndrome: %#x\n", in gaudi2_hbm_sei_handle_read_err()
9290 le32_to_cpu(rd_err_data->dbg_rd_err_dm), in gaudi2_hbm_sei_handle_read_err()
9291 le32_to_cpu(rd_err_data->dbg_rd_err_syndrome)); in gaudi2_hbm_sei_handle_read_err()
9293 if (le32_to_cpu(rd_err_data->dbg_rd_err_misc) & in gaudi2_hbm_sei_handle_read_err()
9295 dev_err_ratelimited(hdev->dev, "Beat%d ECC DERR: DM: %#x, Syndrome: %#x\n", in gaudi2_hbm_sei_handle_read_err()
9297 le32_to_cpu(rd_err_data->dbg_rd_err_dm), in gaudi2_hbm_sei_handle_read_err()
9298 le32_to_cpu(rd_err_data->dbg_rd_err_syndrome)); in gaudi2_hbm_sei_handle_read_err()
9303 if (le32_to_cpu(rd_err_data->dbg_rd_err_misc) & in gaudi2_hbm_sei_handle_read_err()
9305 dev_err_ratelimited(hdev->dev, in gaudi2_hbm_sei_handle_read_err()
9308 le32_to_cpu(rd_err_data->dbg_rd_err_dm), in gaudi2_hbm_sei_handle_read_err()
9309 (le32_to_cpu(rd_err_data->dbg_rd_err_misc) & in gaudi2_hbm_sei_handle_read_err()
9315 dev_err_ratelimited(hdev->dev, "Beat%d DQ data:\n", beat); in gaudi2_hbm_sei_handle_read_err()
9316 dev_err_ratelimited(hdev->dev, "\t0x%08x\n", in gaudi2_hbm_sei_handle_read_err()
9317 le32_to_cpu(rd_err_data->dbg_rd_err_data[beat * 2])); in gaudi2_hbm_sei_handle_read_err()
9318 dev_err_ratelimited(hdev->dev, "\t0x%08x\n", in gaudi2_hbm_sei_handle_read_err()
9319 le32_to_cpu(rd_err_data->dbg_rd_err_data[beat * 2 + 1])); in gaudi2_hbm_sei_handle_read_err()
9328 struct hbm_sei_wr_cmd_address *wr_cmd_addr = wr_par_err_data->dbg_last_wr_cmds; in gaudi2_hbm_sei_print_wr_par_info()
9329 u32 i, curr_addr, derr = wr_par_err_data->dbg_derr; in gaudi2_hbm_sei_print_wr_par_info()
9331 dev_err_ratelimited(hdev->dev, "WRITE PARITY ERROR count: %d\n", err_cnt); in gaudi2_hbm_sei_print_wr_par_info()
9333 dev_err_ratelimited(hdev->dev, "CK-0 DERR: 0x%02x, CK-1 DERR: 0x%02x\n", in gaudi2_hbm_sei_print_wr_par_info()
9336 /* JIRA H6-3286 - the following prints may not be valid */ in gaudi2_hbm_sei_print_wr_par_info()
9337 dev_err_ratelimited(hdev->dev, "Last latched write commands addresses:\n"); in gaudi2_hbm_sei_print_wr_par_info()
9340 dev_err_ratelimited(hdev->dev, in gaudi2_hbm_sei_print_wr_par_info()
9353 __le32 *col_cmd = ca_par_err_data->dbg_col; in gaudi2_hbm_sei_print_ca_par_info()
9354 __le16 *row_cmd = ca_par_err_data->dbg_row; in gaudi2_hbm_sei_print_ca_par_info()
9357 dev_err_ratelimited(hdev->dev, "CA ERROR count: %d\n", err_cnt); in gaudi2_hbm_sei_print_ca_par_info()
9359 dev_err_ratelimited(hdev->dev, "Last latched C&R bus commands:\n"); in gaudi2_hbm_sei_print_ca_par_info()
9361 dev_err_ratelimited(hdev->dev, "cmd%u: ROW(0x%04x) COL(0x%05x)\n", i, in gaudi2_hbm_sei_print_ca_par_info()
9373 hbm_id = (event_type - GAUDI2_EVENT_HBM0_MC0_SEI_SEVERE) / 4; in gaudi2_handle_hbm_mc_sei_err()
9374 mc_id = ((event_type - GAUDI2_EVENT_HBM0_MC0_SEI_SEVERE) / 2) % 2; in gaudi2_handle_hbm_mc_sei_err()
9376 cause_idx = sei_data->hdr.sei_cause; in gaudi2_handle_hbm_mc_sei_err()
9377 if (cause_idx > GAUDI2_NUM_OF_HBM_SEI_CAUSE - 1) { in gaudi2_handle_hbm_mc_sei_err()
9384 gaudi2_print_event(hdev, event_type, !sei_data->hdr.is_critical, in gaudi2_handle_hbm_mc_sei_err()
9385 "System %s Error Interrupt - HBM(%u) MC(%u) MC_CH(%u) MC_PC(%u). Error cause: %s", in gaudi2_handle_hbm_mc_sei_err()
9386 sei_data->hdr.is_critical ? "Critical" : "Non-critical", in gaudi2_handle_hbm_mc_sei_err()
9387 hbm_id, mc_id, sei_data->hdr.mc_channel, sei_data->hdr.mc_pseudo_channel, in gaudi2_handle_hbm_mc_sei_err()
9390 /* Print error-specific info */ in gaudi2_handle_hbm_mc_sei_err()
9397 gaudi2_hbm_sei_print_ca_par_info(hdev, &sei_data->ca_parity_even_info, in gaudi2_handle_hbm_mc_sei_err()
9398 le32_to_cpu(sei_data->hdr.cnt)); in gaudi2_handle_hbm_mc_sei_err()
9403 gaudi2_hbm_sei_print_ca_par_info(hdev, &sei_data->ca_parity_odd_info, in gaudi2_handle_hbm_mc_sei_err()
9404 le32_to_cpu(sei_data->hdr.cnt)); in gaudi2_handle_hbm_mc_sei_err()
9409 gaudi2_hbm_sei_print_wr_par_info(hdev, &sei_data->wr_parity_info, in gaudi2_handle_hbm_mc_sei_err()
9410 le32_to_cpu(sei_data->hdr.cnt)); in gaudi2_handle_hbm_mc_sei_err()
9419 &sei_data->read_err_info, in gaudi2_handle_hbm_mc_sei_err()
9420 le32_to_cpu(sei_data->hdr.cnt)); in gaudi2_handle_hbm_mc_sei_err()
9427 require_hard_reset |= !!sei_data->hdr.is_critical; in gaudi2_handle_hbm_mc_sei_err()
9450 dev_dbg(hdev->dev, "HBM spi event: notification cause(%s)\n", in gaudi2_handle_hbm_mc_spi()
9462 mutex_lock(&hdev->clk_throttling.lock); in gaudi2_print_clk_change_info()
9466 hdev->clk_throttling.current_reason |= HL_CLK_THROTTLE_POWER; in gaudi2_print_clk_change_info()
9467 hdev->clk_throttling.aggregated_reason |= HL_CLK_THROTTLE_POWER; in gaudi2_print_clk_change_info()
9468 hdev->clk_throttling.timestamp[HL_CLK_THROTTLE_TYPE_POWER].start = ktime_get(); in gaudi2_print_clk_change_info()
9469 hdev->clk_throttling.timestamp[HL_CLK_THROTTLE_TYPE_POWER].end = zero_time; in gaudi2_print_clk_change_info()
9470 dev_dbg_ratelimited(hdev->dev, "Clock throttling due to power consumption\n"); in gaudi2_print_clk_change_info()
9474 hdev->clk_throttling.current_reason &= ~HL_CLK_THROTTLE_POWER; in gaudi2_print_clk_change_info()
9475 hdev->clk_throttling.timestamp[HL_CLK_THROTTLE_TYPE_POWER].end = ktime_get(); in gaudi2_print_clk_change_info()
9476 dev_dbg_ratelimited(hdev->dev, "Power envelop is safe, back to optimal clock\n"); in gaudi2_print_clk_change_info()
9480 hdev->clk_throttling.current_reason |= HL_CLK_THROTTLE_THERMAL; in gaudi2_print_clk_change_info()
9481 hdev->clk_throttling.aggregated_reason |= HL_CLK_THROTTLE_THERMAL; in gaudi2_print_clk_change_info()
9482 hdev->clk_throttling.timestamp[HL_CLK_THROTTLE_TYPE_THERMAL].start = ktime_get(); in gaudi2_print_clk_change_info()
9483 hdev->clk_throttling.timestamp[HL_CLK_THROTTLE_TYPE_THERMAL].end = zero_time; in gaudi2_print_clk_change_info()
9485 dev_info_ratelimited(hdev->dev, "Clock throttling due to overheating\n"); in gaudi2_print_clk_change_info()
9489 hdev->clk_throttling.current_reason &= ~HL_CLK_THROTTLE_THERMAL; in gaudi2_print_clk_change_info()
9490 hdev->clk_throttling.timestamp[HL_CLK_THROTTLE_TYPE_THERMAL].end = ktime_get(); in gaudi2_print_clk_change_info()
9492 dev_info_ratelimited(hdev->dev, "Thermal envelop is safe, back to optimal clock\n"); in gaudi2_print_clk_change_info()
9496 dev_err(hdev->dev, "Received invalid clock change event %d\n", event_type); in gaudi2_print_clk_change_info()
9500 mutex_unlock(&hdev->clk_throttling.lock); in gaudi2_print_clk_change_info()
9506 struct hl_hw_queue *q = &hdev->kernel_queues[GAUDI2_QUEUE_ID_CPU_PQ]; in gaudi2_print_out_of_sync_info()
9510 le32_to_cpu(sync_err->pi), le32_to_cpu(sync_err->ci), in gaudi2_print_out_of_sync_info()
9511 q->pi, atomic_read(&q->ci)); in gaudi2_print_out_of_sync_info()
9532 "pcie msi-x gen denied due to vector num check failure, vec(0x%X)", in gaudi2_handle_pcie_p2p_msix()
9547 cause = le64_to_cpu(drain_data->intr_cause.intr_cause_data); in gaudi2_handle_pcie_drain()
9550 dev_err_ratelimited(hdev->dev, "PCIE AXI drain LBW completed\n"); in gaudi2_handle_pcie_drain()
9555 dev_err_ratelimited(hdev->dev, "PCIE AXI drain HBW completed\n"); in gaudi2_handle_pcie_drain()
9569 dev_err_ratelimited(hdev->dev, "PSOC %s completed\n", in gaudi2_handle_psoc_drain()
9583 struct hl_hw_queue *q = &hdev->kernel_queues[GAUDI2_QUEUE_ID_CPU_PQ]; in gaudi2_print_cpu_pkt_failure_info()
9587 le32_to_cpu(sync_err->pi), le32_to_cpu(sync_err->ci), q->pi, atomic_read(&q->ci)); in gaudi2_print_cpu_pkt_failure_info()
9597 intr_type = le32_to_cpu(data->intr_type); in hl_arc_event_handle()
9598 engine_id = le32_to_cpu(data->engine_id); in hl_arc_event_handle()
9599 payload = le64_to_cpu(data->payload); in hl_arc_event_handle()
9607 engine_id, intr_type, q->queue_index); in hl_arc_event_handle()
9622 index = event_type - GAUDI2_EVENT_TPC0_AXI_ERR_RSP; in event_id_to_engine_id()
9626 index = event_type - GAUDI2_EVENT_TPC0_QM; in event_id_to_engine_id()
9668 index = event_type - GAUDI2_EVENT_DEC0_AXI_ERR_RSPONSE; in event_id_to_engine_id()
9672 index = (event_type - GAUDI2_EVENT_DEC0_SPI) >> 1; in event_id_to_engine_id()
9676 index = event_type - GAUDI2_EVENT_NIC0_AXI_ERROR_RESPONSE; in event_id_to_engine_id()
9679 index = event_type - GAUDI2_EVENT_NIC0_QM0; in event_id_to_engine_id()
9682 index = event_type - GAUDI2_EVENT_NIC0_BMON_SPMU; in event_id_to_engine_id()
9685 index = (event_type - GAUDI2_EVENT_TPC0_BMON_SPMU) >> 1; in event_id_to_engine_id()
9738 return GAUDI2_DCORE1_ENGINE_ID_TPC_0 + index - TPC_ID_DCORE1_TPC0; in event_id_to_engine_id()
9740 return GAUDI2_DCORE2_ENGINE_ID_TPC_0 + index - TPC_ID_DCORE2_TPC0; in event_id_to_engine_id()
9742 return GAUDI2_DCORE3_ENGINE_ID_TPC_0 + index - TPC_ID_DCORE3_TPC0; in event_id_to_engine_id()
9782 struct gaudi2_device *gaudi2 = hdev->asic_specific; in gaudi2_handle_eqe()
9788 ctl = le32_to_cpu(eq_entry->hdr.ctl); in gaudi2_handle_eqe()
9792 dev_err(hdev->dev, "Event type %u exceeds maximum of %u", in gaudi2_handle_eqe()
9793 event_type, GAUDI2_EVENT_SIZE - 1); in gaudi2_handle_eqe()
9797 gaudi2->events_stat[event_type]++; in gaudi2_handle_eqe()
9798 gaudi2->events_stat_aggregate[event_type]++; in gaudi2_handle_eqe()
9806 reset_required = gaudi2_handle_ecc_event(hdev, event_type, &eq_entry->ecc_data); in gaudi2_handle_eqe()
9807 is_critical = eq_entry->ecc_data.is_critical; in gaudi2_handle_eqe()
9839 index = event_type - GAUDI2_EVENT_ROTATOR0_AXI_ERROR_RESPONSE; in gaudi2_handle_eqe()
9841 &eq_entry->razwi_with_intr_cause, &event_mask); in gaudi2_handle_eqe()
9847 index = event_type - GAUDI2_EVENT_TPC0_AXI_ERR_RSP; in gaudi2_handle_eqe()
9849 &eq_entry->razwi_with_intr_cause, &event_mask); in gaudi2_handle_eqe()
9855 index = event_type - GAUDI2_EVENT_DEC0_AXI_ERR_RSPONSE; in gaudi2_handle_eqe()
9885 index = (event_type - GAUDI2_EVENT_TPC0_KERNEL_ERR) / in gaudi2_handle_eqe()
9886 (GAUDI2_EVENT_TPC1_KERNEL_ERR - GAUDI2_EVENT_TPC0_KERNEL_ERR); in gaudi2_handle_eqe()
9888 &eq_entry->razwi_with_intr_cause, &event_mask); in gaudi2_handle_eqe()
9902 index = (event_type - GAUDI2_EVENT_DEC0_SPI) / in gaudi2_handle_eqe()
9903 (GAUDI2_EVENT_DEC1_SPI - GAUDI2_EVENT_DEC0_SPI); in gaudi2_handle_eqe()
9912 index = (event_type - GAUDI2_EVENT_MME0_CTRL_AXI_ERROR_RESPONSE) / in gaudi2_handle_eqe()
9913 (GAUDI2_EVENT_MME1_CTRL_AXI_ERROR_RESPONSE - in gaudi2_handle_eqe()
9924 index = (event_type - GAUDI2_EVENT_MME0_QMAN_SW_ERROR) / in gaudi2_handle_eqe()
9925 (GAUDI2_EVENT_MME1_QMAN_SW_ERROR - in gaudi2_handle_eqe()
9935 index = (event_type - GAUDI2_EVENT_MME0_WAP_SOURCE_RESULT_INVALID) / in gaudi2_handle_eqe()
9936 (GAUDI2_EVENT_MME1_WAP_SOURCE_RESULT_INVALID - in gaudi2_handle_eqe()
9945 le64_to_cpu(eq_entry->intr_cause.intr_cause_data)); in gaudi2_handle_eqe()
9951 le64_to_cpu(eq_entry->intr_cause.intr_cause_data)); in gaudi2_handle_eqe()
9957 le64_to_cpu(eq_entry->intr_cause.intr_cause_data)); in gaudi2_handle_eqe()
9963 le64_to_cpu(eq_entry->intr_cause.intr_cause_data), &event_mask); in gaudi2_handle_eqe()
9979 le64_to_cpu(eq_entry->intr_cause.intr_cause_data)); in gaudi2_handle_eqe()
9986 le64_to_cpu(eq_entry->intr_cause.intr_cause_data)); in gaudi2_handle_eqe()
9998 if (gaudi2_handle_hbm_mc_sei_err(hdev, event_type, &eq_entry->sei_data)) { in gaudi2_handle_eqe()
10001 is_critical = eq_entry->sei_data.hdr.is_critical; in gaudi2_handle_eqe()
10008 le64_to_cpu(eq_entry->intr_cause.intr_cause_data)); in gaudi2_handle_eqe()
10014 le64_to_cpu(eq_entry->intr_cause.intr_cause_data)); in gaudi2_handle_eqe()
10019 error_count = gaudi2_handle_pcie_drain(hdev, &eq_entry->pcie_drain_ind_data); in gaudi2_handle_eqe()
10028 le64_to_cpu(eq_entry->intr_cause.intr_cause_data)); in gaudi2_handle_eqe()
10137 gaudi2_print_out_of_sync_info(hdev, event_type, &eq_entry->pkt_sync_err); in gaudi2_handle_eqe()
10146 /* Do nothing- FW will handle it */ in gaudi2_handle_eqe()
10155 index = event_type - GAUDI2_EVENT_SM0_AXI_ERROR_RESPONSE; in gaudi2_handle_eqe()
10166 dev_info(hdev->dev, "CPLD shutdown cause, reset reason: 0x%llx\n", in gaudi2_handle_eqe()
10167 le64_to_cpu(eq_entry->data[0])); in gaudi2_handle_eqe()
10172 dev_err(hdev->dev, "CPLD shutdown event, reset reason: 0x%llx\n", in gaudi2_handle_eqe()
10173 le64_to_cpu(eq_entry->data[0])); in gaudi2_handle_eqe()
10179 gaudi2_print_cpu_pkt_failure_info(hdev, event_type, &eq_entry->pkt_sync_err); in gaudi2_handle_eqe()
10186 error_count = hl_arc_event_handle(hdev, event_type, &eq_entry->arc_data); in gaudi2_handle_eqe()
10204 dev_info_ratelimited(hdev->dev, "%s event received\n", in gaudi2_handle_eqe()
10214 dev_err_ratelimited(hdev->dev, "Cannot find handler for event %d\n", in gaudi2_handle_eqe()
10238 if (hdev->hard_reset_on_fw_events || in gaudi2_handle_eqe()
10239 (hdev->asic_prop.fw_security_enabled && is_critical)) in gaudi2_handle_eqe()
10253 if (hdev->asic_prop.fw_security_enabled && is_critical) { in gaudi2_handle_eqe()
10279 lin_dma_pkt->ctl = cpu_to_le32(ctl); in gaudi2_memset_memory_chunk_using_edma_qm()
10280 lin_dma_pkt->src_addr = cpu_to_le64(val); in gaudi2_memset_memory_chunk_using_edma_qm()
10281 lin_dma_pkt->dst_addr = cpu_to_le64(addr); in gaudi2_memset_memory_chunk_using_edma_qm()
10282 lin_dma_pkt->tsize = cpu_to_le32(size); in gaudi2_memset_memory_chunk_using_edma_qm()
10287 rc = hdev->asic_funcs->access_dev_mem(hdev, PCI_REGION_DRAM, in gaudi2_memset_memory_chunk_using_edma_qm()
10291 dev_err(hdev->dev, "Failed to copy lin_dma packet to HBM (%#llx)\n", in gaudi2_memset_memory_chunk_using_edma_qm()
10299 dev_err(hdev->dev, "Failed to send lin_dma packet to H/W queue %d\n", in gaudi2_memset_memory_chunk_using_edma_qm()
10314 struct asic_fixed_properties *prop = &hdev->asic_prop; in gaudi2_memset_device_memory()
10318 if (prop->edma_enabled_mask == 0) { in gaudi2_memset_device_memory()
10319 dev_info(hdev->dev, "non of the EDMA engines is enabled - skip dram scrubbing\n"); in gaudi2_memset_device_memory()
10320 return -EIO; in gaudi2_memset_device_memory()
10323 sob_offset = hdev->asic_prop.first_available_user_sob[0] * 4; in gaudi2_memset_device_memory()
10342 if ((addr >= prop->dram_user_base_address) && in gaudi2_memset_device_memory()
10343 (addr < prop->dram_user_base_address + cb_len)) in gaudi2_memset_device_memory()
10344 cur_addr += (prop->dram_user_base_address + cb_len) - addr; in gaudi2_memset_device_memory()
10348 return -ENOMEM; in gaudi2_memset_device_memory()
10351 * set mmu bypass for the scrubbing - all ddmas are configured the same so save in gaudi2_memset_device_memory()
10362 if (!(prop->edma_enabled_mask & BIT(edma_bit))) in gaudi2_memset_device_memory()
10385 if (!(prop->edma_enabled_mask & BIT(edma_bit))) in gaudi2_memset_device_memory()
10388 chunk_size = min_t(u64, SZ_2G, end_addr - cur_addr); in gaudi2_memset_device_memory()
10392 prop->dram_user_base_address + (dma_num * pkt_size), in gaudi2_memset_device_memory()
10409 dev_err(hdev->dev, "DMA Timeout during HBM scrubbing(sob: 0x%x, dma_num: 0x%x)\n", in gaudi2_memset_device_memory()
10419 if (!(prop->edma_enabled_mask & BIT(edma_bit))) in gaudi2_memset_device_memory()
10435 rc = hdev->asic_funcs->access_dev_mem(hdev, PCI_REGION_DRAM, in gaudi2_memset_device_memory()
10436 prop->dram_user_base_address + i, in gaudi2_memset_device_memory()
10448 struct asic_fixed_properties *prop = &hdev->asic_prop; in gaudi2_scrub_device_dram()
10449 u64 size = prop->dram_end_address - prop->dram_user_base_address; in gaudi2_scrub_device_dram()
10451 rc = gaudi2_memset_device_memory(hdev, prop->dram_user_base_address, size, val); in gaudi2_scrub_device_dram()
10454 dev_err(hdev->dev, "Failed to scrub dram, address: 0x%llx size: %llu\n", in gaudi2_scrub_device_dram()
10455 prop->dram_user_base_address, size); in gaudi2_scrub_device_dram()
10462 struct asic_fixed_properties *prop = &hdev->asic_prop; in gaudi2_scrub_device_mem()
10463 u64 val = hdev->memory_scrub_val; in gaudi2_scrub_device_mem()
10466 if (!hdev->memory_scrub) in gaudi2_scrub_device_mem()
10470 addr = prop->sram_user_base_address; in gaudi2_scrub_device_mem()
10471 size = hdev->pldm ? 0x10000 : (prop->sram_size - SRAM_USER_BASE_OFFSET); in gaudi2_scrub_device_mem()
10472 dev_dbg(hdev->dev, "Scrubbing SRAM: 0x%09llx - 0x%09llx, val: 0x%llx\n", in gaudi2_scrub_device_mem()
10476 dev_err(hdev->dev, "scrubbing SRAM failed (%d)\n", rc); in gaudi2_scrub_device_mem()
10483 dev_err(hdev->dev, "scrubbing DRAM failed (%d)\n", rc); in gaudi2_scrub_device_mem()
10496 offset = hdev->asic_prop.first_available_cq[0] * 4; in gaudi2_restore_user_sm_registers()
10503 size = mmDCORE0_SYNC_MNGR_GLBL_LBW_ADDR_H_0 - in gaudi2_restore_user_sm_registers()
10520 size = mmDCORE0_SYNC_MNGR_GLBL_LBW_ADDR_H_0 - mmDCORE0_SYNC_MNGR_GLBL_LBW_ADDR_L_0; in gaudi2_restore_user_sm_registers()
10538 offset = hdev->asic_prop.first_available_user_mon[0] * 4; in gaudi2_restore_user_sm_registers()
10541 size = mmDCORE0_SYNC_MNGR_OBJS_SM_SEC_0 - (mmDCORE0_SYNC_MNGR_OBJS_MON_STATUS_0 + offset); in gaudi2_restore_user_sm_registers()
10551 size = mmDCORE0_SYNC_MNGR_OBJS_SM_SEC_0 - mmDCORE0_SYNC_MNGR_OBJS_MON_STATUS_0; in gaudi2_restore_user_sm_registers()
10560 offset = hdev->asic_prop.first_available_user_sob[0] * 4; in gaudi2_restore_user_sm_registers()
10563 size = mmDCORE0_SYNC_MNGR_OBJS_MON_PAY_ADDRL_0 - in gaudi2_restore_user_sm_registers()
10570 size = mmDCORE0_SYNC_MNGR_OBJS_MON_PAY_ADDRL_0 - mmDCORE0_SYNC_MNGR_OBJS_SOB_OBJ_0; in gaudi2_restore_user_sm_registers()
10631 u64 block_base = cfg_ctx->base + block_idx * cfg_ctx->block_off; in gaudi2_init_block_instances()
10635 for (i = 0 ; i < cfg_ctx->instances ; i++) { in gaudi2_init_block_instances()
10636 seq = block_idx * cfg_ctx->instances + i; in gaudi2_init_block_instances()
10639 if (!(cfg_ctx->enabled_mask & BIT_ULL(seq))) in gaudi2_init_block_instances()
10642 cfg_ctx->instance_cfg_fn(hdev, block_base + i * cfg_ctx->instance_off, in gaudi2_init_block_instances()
10643 cfg_ctx->data); in gaudi2_init_block_instances()
10652 cfg_ctx->enabled_mask = mask; in gaudi2_init_blocks_with_mask()
10654 for (i = 0 ; i < cfg_ctx->blocks ; i++) in gaudi2_init_blocks_with_mask()
10675 dev_err(hdev->dev, "No ctx available\n"); in gaudi2_debugfs_read_dma()
10676 return -EINVAL; in gaudi2_debugfs_read_dma()
10683 dev_err(hdev->dev, "Failed to allocate memory for KDMA read\n"); in gaudi2_debugfs_read_dma()
10684 rc = -ENOMEM; in gaudi2_debugfs_read_dma()
10692 dev_err(hdev->dev, "Failed to reserve vmem on asic\n"); in gaudi2_debugfs_read_dma()
10693 rc = -ENOMEM; in gaudi2_debugfs_read_dma()
10698 mutex_lock(&hdev->mmu_lock); in gaudi2_debugfs_read_dma()
10702 dev_err(hdev->dev, "Failed to create mapping on asic mmu\n"); in gaudi2_debugfs_read_dma()
10708 ctx->asid, reserved_va_base, SZ_2M); in gaudi2_debugfs_read_dma()
10714 mutex_unlock(&hdev->mmu_lock); in gaudi2_debugfs_read_dma()
10717 gaudi2_kdma_set_mmbp_asid(hdev, false, ctx->asid); in gaudi2_debugfs_read_dma()
10738 size_left -= SZ_2M; in gaudi2_debugfs_read_dma()
10743 mutex_lock(&hdev->mmu_lock); in gaudi2_debugfs_read_dma()
10750 ctx->asid, reserved_va_base, SZ_2M); in gaudi2_debugfs_read_dma()
10753 mutex_unlock(&hdev->mmu_lock); in gaudi2_debugfs_read_dma()
10765 struct gaudi2_device *gaudi2 = hdev->asic_specific; in gaudi2_internal_cb_pool_init()
10768 if (!(gaudi2->hw_cap_initialized & HW_CAP_PMMU)) in gaudi2_internal_cb_pool_init()
10771 hdev->internal_cb_pool_virt_addr = hl_asic_dma_alloc_coherent(hdev, in gaudi2_internal_cb_pool_init()
10773 &hdev->internal_cb_pool_dma_addr, in gaudi2_internal_cb_pool_init()
10776 if (!hdev->internal_cb_pool_virt_addr) in gaudi2_internal_cb_pool_init()
10777 return -ENOMEM; in gaudi2_internal_cb_pool_init()
10782 hdev->internal_cb_pool = gen_pool_create(min_alloc_order, -1); in gaudi2_internal_cb_pool_init()
10783 if (!hdev->internal_cb_pool) { in gaudi2_internal_cb_pool_init()
10784 dev_err(hdev->dev, "Failed to create internal CB pool\n"); in gaudi2_internal_cb_pool_init()
10785 rc = -ENOMEM; in gaudi2_internal_cb_pool_init()
10789 rc = gen_pool_add(hdev->internal_cb_pool, (uintptr_t) hdev->internal_cb_pool_virt_addr, in gaudi2_internal_cb_pool_init()
10790 HOST_SPACE_INTERNAL_CB_SZ, -1); in gaudi2_internal_cb_pool_init()
10792 dev_err(hdev->dev, "Failed to add memory to internal CB pool\n"); in gaudi2_internal_cb_pool_init()
10793 rc = -EFAULT; in gaudi2_internal_cb_pool_init()
10797 hdev->internal_cb_va_base = hl_reserve_va_block(hdev, ctx, HL_VA_RANGE_TYPE_HOST, in gaudi2_internal_cb_pool_init()
10800 if (!hdev->internal_cb_va_base) { in gaudi2_internal_cb_pool_init()
10801 rc = -ENOMEM; in gaudi2_internal_cb_pool_init()
10805 mutex_lock(&hdev->mmu_lock); in gaudi2_internal_cb_pool_init()
10807 rc = hl_mmu_map_contiguous(ctx, hdev->internal_cb_va_base, hdev->internal_cb_pool_dma_addr, in gaudi2_internal_cb_pool_init()
10816 mutex_unlock(&hdev->mmu_lock); in gaudi2_internal_cb_pool_init()
10821 hl_mmu_unmap_contiguous(ctx, hdev->internal_cb_va_base, HOST_SPACE_INTERNAL_CB_SZ); in gaudi2_internal_cb_pool_init()
10823 mutex_unlock(&hdev->mmu_lock); in gaudi2_internal_cb_pool_init()
10824 hl_unreserve_va_block(hdev, ctx, hdev->internal_cb_va_base, HOST_SPACE_INTERNAL_CB_SZ); in gaudi2_internal_cb_pool_init()
10826 gen_pool_destroy(hdev->internal_cb_pool); in gaudi2_internal_cb_pool_init()
10828 hl_asic_dma_free_coherent(hdev, HOST_SPACE_INTERNAL_CB_SZ, hdev->internal_cb_pool_virt_addr, in gaudi2_internal_cb_pool_init()
10829 hdev->internal_cb_pool_dma_addr); in gaudi2_internal_cb_pool_init()
10836 struct gaudi2_device *gaudi2 = hdev->asic_specific; in gaudi2_internal_cb_pool_fini()
10838 if (!(gaudi2->hw_cap_initialized & HW_CAP_PMMU)) in gaudi2_internal_cb_pool_fini()
10841 mutex_lock(&hdev->mmu_lock); in gaudi2_internal_cb_pool_fini()
10842 hl_mmu_unmap_contiguous(ctx, hdev->internal_cb_va_base, HOST_SPACE_INTERNAL_CB_SZ); in gaudi2_internal_cb_pool_fini()
10843 hl_unreserve_va_block(hdev, ctx, hdev->internal_cb_va_base, HOST_SPACE_INTERNAL_CB_SZ); in gaudi2_internal_cb_pool_fini()
10845 mutex_unlock(&hdev->mmu_lock); in gaudi2_internal_cb_pool_fini()
10847 gen_pool_destroy(hdev->internal_cb_pool); in gaudi2_internal_cb_pool_fini()
10849 hl_asic_dma_free_coherent(hdev, HOST_SPACE_INTERNAL_CB_SZ, hdev->internal_cb_pool_virt_addr, in gaudi2_internal_cb_pool_fini()
10850 hdev->internal_cb_pool_dma_addr); in gaudi2_internal_cb_pool_fini()
10861 struct hl_device *hdev = ctx->hdev; in gaudi2_map_virtual_msix_doorbell_memory()
10862 struct asic_fixed_properties *prop = &hdev->asic_prop; in gaudi2_map_virtual_msix_doorbell_memory()
10863 struct gaudi2_device *gaudi2 = hdev->asic_specific; in gaudi2_map_virtual_msix_doorbell_memory()
10867 gaudi2->virt_msix_db_dma_addr, prop->pmmu.page_size, true); in gaudi2_map_virtual_msix_doorbell_memory()
10869 dev_err(hdev->dev, "Failed to map VA %#llx for virtual MSI-X doorbell memory\n", in gaudi2_map_virtual_msix_doorbell_memory()
10877 struct hl_device *hdev = ctx->hdev; in gaudi2_unmap_virtual_msix_doorbell_memory()
10878 struct asic_fixed_properties *prop = &hdev->asic_prop; in gaudi2_unmap_virtual_msix_doorbell_memory()
10882 prop->pmmu.page_size, true); in gaudi2_unmap_virtual_msix_doorbell_memory()
10884 dev_err(hdev->dev, "Failed to unmap VA %#llx of virtual MSI-X doorbell memory\n", in gaudi2_unmap_virtual_msix_doorbell_memory()
10892 if (ctx->asid == HL_KERNEL_ASID_ID) in gaudi2_ctx_init()
10895 rc = gaudi2_mmu_prepare(ctx->hdev, ctx->asid); in gaudi2_ctx_init()
10902 if (ctx->hdev->reset_upon_device_release) in gaudi2_ctx_init()
10903 gaudi2_restore_nic_qm_registers(ctx->hdev); in gaudi2_ctx_init()
10905 gaudi2_restore_user_registers(ctx->hdev); in gaudi2_ctx_init()
10907 rc = gaudi2_internal_cb_pool_init(ctx->hdev, ctx); in gaudi2_ctx_init()
10913 gaudi2_internal_cb_pool_fini(ctx->hdev, ctx); in gaudi2_ctx_init()
10920 if (ctx->asid == HL_KERNEL_ASID_ID) in gaudi2_ctx_fini()
10923 gaudi2_internal_cb_pool_fini(ctx->hdev, ctx); in gaudi2_ctx_fini()
10930 struct hl_device *hdev = cs->ctx->hdev; in gaudi2_pre_schedule_cs()
10931 int index = cs->sequence & (hdev->asic_prop.max_pending_cs - 1); in gaudi2_pre_schedule_cs()
10942 * generates MSI-X interrupt. in gaudi2_pre_schedule_cs()
10950 cs->jobs_cnt); in gaudi2_pre_schedule_cs()
10966 pkt = (struct packet_msg_short *) (uintptr_t) (cb->kernel_address + size); in gaudi2_gen_signal_cb()
10979 pkt->value = cpu_to_le32(value); in gaudi2_gen_signal_cb()
10980 pkt->ctl = cpu_to_le32(ctl); in gaudi2_gen_signal_cb()
10997 pkt->value = cpu_to_le32(value); in gaudi2_add_mon_msg_short()
10998 pkt->ctl = cpu_to_le32(ctl); in gaudi2_add_mon_msg_short()
11010 dev_err(hdev->dev, "sob_base %u (mask %#x) is not valid\n", sob_base, sob_mask); in gaudi2_add_arm_monitor_pkt()
11027 pkt->value = cpu_to_le32(value); in gaudi2_add_arm_monitor_pkt()
11028 pkt->ctl = cpu_to_le32(ctl); in gaudi2_add_arm_monitor_pkt()
11047 pkt->cfg = cpu_to_le32(cfg); in gaudi2_add_fence_pkt()
11048 pkt->ctl = cpu_to_le32(ctl); in gaudi2_add_fence_pkt()
11055 struct hl_cb *cb = prop->data; in gaudi2_gen_wait_cb()
11056 void *buf = (void *) (uintptr_t) (cb->kernel_address); in gaudi2_gen_wait_cb()
11059 u32 stream_index, size = prop->size; in gaudi2_gen_wait_cb()
11062 stream_index = prop->q_idx % 4; in gaudi2_gen_wait_cb()
11063 fence_addr = CFG_BASE + gaudi2_qm_blocks_bases[prop->q_idx] + in gaudi2_gen_wait_cb()
11073 msg_addr_offset = (mmDCORE0_SYNC_MNGR_OBJS_MON_PAY_ADDRL_0 + prop->mon_id * 4) - in gaudi2_gen_wait_cb()
11079 msg_addr_offset = (mmDCORE0_SYNC_MNGR_OBJS_MON_PAY_ADDRH_0 + prop->mon_id * 4) - in gaudi2_gen_wait_cb()
11088 msg_addr_offset = (mmDCORE0_SYNC_MNGR_OBJS_MON_PAY_DATA_0 + prop->mon_id * 4) - in gaudi2_gen_wait_cb()
11094 msg_addr_offset = (mmDCORE0_SYNC_MNGR_OBJS_MON_ARM_0 + prop->mon_id * 4) - monitor_base; in gaudi2_gen_wait_cb()
11096 size += gaudi2_add_arm_monitor_pkt(hdev, buf + size, prop->sob_base, prop->sob_mask, in gaudi2_gen_wait_cb()
11097 prop->sob_val, msg_addr_offset); in gaudi2_gen_wait_cb()
11109 dev_dbg(hdev->dev, "reset SOB, q_idx: %d, sob_id: %d\n", hw_sob->q_idx, hw_sob->sob_id); in gaudi2_reset_sob()
11111 WREG32(mmDCORE0_SYNC_MNGR_OBJS_SOB_OBJ_0 + hw_sob->sob_id * 4, 0); in gaudi2_reset_sob()
11113 kref_init(&hw_sob->kref); in gaudi2_reset_sob()
11136 return -EINVAL; in gaudi2_collective_wait_create_jobs()
11140 * hl_mmu_scramble - converts a dram (non power of 2) page-size aligned address
11141 * to DMMU page-size address (64MB) before mapping it in
11153 * PA1 0x3000000 VA1 0x9C000000 SVA1= (VA1/48M)*64M 0xD0000000 <- PA1/48M 0x1
11154 * PA2 0x9000000 VA2 0x9F000000 SVA2= (VA2/48M)*64M 0xD4000000 <- PA2/48M 0x3
11159 struct asic_fixed_properties *prop = &hdev->asic_prop; in gaudi2_mmu_scramble_addr()
11167 divisor = prop->num_functional_hbms * GAUDI2_HBM_MMU_SCRM_MEM_SIZE; in gaudi2_mmu_scramble_addr()
11179 struct asic_fixed_properties *prop = &hdev->asic_prop; in gaudi2_mmu_descramble_addr()
11187 divisor = prop->num_functional_hbms * GAUDI2_HBM_MMU_SCRM_MEM_SIZE; in gaudi2_mmu_descramble_addr()
11203 dev_err(hdev->dev, "Unexpected core number %d for DEC\n", core_id); in gaudi2_get_dec_base_addr()
11224 struct gaudi2_device *gaudi2 = hdev->asic_specific; in gaudi2_get_hw_block_id()
11228 if (block_addr == CFG_BASE + gaudi2->mapped_blocks[i].address) { in gaudi2_get_hw_block_id()
11231 *block_size = gaudi2->mapped_blocks[i].size; in gaudi2_get_hw_block_id()
11236 dev_err(hdev->dev, "Invalid block address %#llx", block_addr); in gaudi2_get_hw_block_id()
11238 return -EINVAL; in gaudi2_get_hw_block_id()
11244 struct gaudi2_device *gaudi2 = hdev->asic_specific; in gaudi2_block_mmap()
11250 dev_err(hdev->dev, "Invalid block id %u", block_id); in gaudi2_block_mmap()
11251 return -EINVAL; in gaudi2_block_mmap()
11255 if (block_size != gaudi2->mapped_blocks[block_id].size) { in gaudi2_block_mmap()
11256 dev_err(hdev->dev, "Invalid block size %u", block_size); in gaudi2_block_mmap()
11257 return -EINVAL; in gaudi2_block_mmap()
11260 offset_in_bar = CFG_BASE + gaudi2->mapped_blocks[block_id].address - STM_FLASH_BASE_ADDR; in gaudi2_block_mmap()
11262 address = pci_resource_start(hdev->pdev, SRAM_CFG_BAR_ID) + offset_in_bar; in gaudi2_block_mmap()
11267 rc = remap_pfn_range(vma, vma->vm_start, address >> PAGE_SHIFT, in gaudi2_block_mmap()
11268 block_size, vma->vm_page_prot); in gaudi2_block_mmap()
11270 dev_err(hdev->dev, "remap_pfn_range error %d", rc); in gaudi2_block_mmap()
11277 struct gaudi2_device *gaudi2 = hdev->asic_specific; in gaudi2_enable_events_from_fw()
11279 struct cpu_dyn_regs *dyn_regs = &hdev->fw_loader.dynamic_loader.comm_desc.cpu_dyn_regs; in gaudi2_enable_events_from_fw()
11280 u32 irq_handler_offset = le32_to_cpu(dyn_regs->gic_host_ints_irq); in gaudi2_enable_events_from_fw()
11282 if (gaudi2->hw_cap_initialized & HW_CAP_CPU_Q) in gaudi2_enable_events_from_fw()
11342 return -EINVAL; in gaudi2_get_mmu_base()
11351 struct gaudi2_device *gaudi2 = hdev->asic_specific; in gaudi2_ack_mmu_error()
11354 if (!(gaudi2->hw_cap_initialized & mmu_id)) in gaudi2_ack_mmu_error()
11404 default: return -EINVAL; in gaudi2_map_pll_idx_to_fw_idx()
11447 hdev->state_dump_specs.props = gaudi2_state_dump_specs_props; in gaudi2_state_dump_init()
11448 hdev->state_dump_specs.funcs = gaudi2_state_dump_funcs; in gaudi2_state_dump_init()
11471 struct asic_fixed_properties *prop = &hdev->asic_prop; in gaudi2_mmu_get_real_page_size()
11475 if (page_size % mmu_prop->page_size) in gaudi2_mmu_get_real_page_size()
11478 *real_page_size = mmu_prop->page_size; in gaudi2_mmu_get_real_page_size()
11482 if ((page_size % prop->dram_page_size) || (prop->dram_page_size > mmu_prop->page_size)) in gaudi2_mmu_get_real_page_size()
11493 *real_page_size = prop->dram_page_size; in gaudi2_mmu_get_real_page_size()
11498 dev_err(hdev->dev, "page size of 0x%X is not 0x%X aligned, can't map\n", in gaudi2_mmu_get_real_page_size()
11499 page_size, mmu_prop->page_size >> 10); in gaudi2_mmu_get_real_page_size()
11500 return -EFAULT; in gaudi2_mmu_get_real_page_size()
11505 return -EOPNOTSUPP; in gaudi2_get_monitor_dump()
11510 struct gaudi2_device *gaudi2 = hdev->asic_specific; in gaudi2_send_device_activity()
11512 if (!(gaudi2->hw_cap_initialized & HW_CAP_CPU_Q)) in gaudi2_send_device_activity()
11520 struct gaudi2_device *gaudi2 = hdev->asic_specific; in gaudi2_read_pte()
11523 if (hdev->reset_info.hard_reset_pending) in gaudi2_read_pte()
11526 val = readq(hdev->pcie_bar[DRAM_BAR_ID] + (addr - gaudi2->dram_bar_cur_addr)); in gaudi2_read_pte()
11533 struct gaudi2_device *gaudi2 = hdev->asic_specific; in gaudi2_write_pte()
11535 if (hdev->reset_info.hard_reset_pending) in gaudi2_write_pte()
11538 writeq(val, hdev->pcie_bar[DRAM_BAR_ID] + (addr - gaudi2->dram_bar_cur_addr)); in gaudi2_write_pte()
11643 hdev->asic_funcs = &gaudi2_funcs; in gaudi2_set_asic_funcs()