Lines Matching +full:prop +full:-
1 // SPDX-License-Identifier: GPL-2.0
4 * Copyright 2020-2022 HabanaLabs, Ltd.
17 hash_for_each_possible(ctx->hr_mmu_phys_hash, pgt_info, node, in hl_mmu_v2_hr_get_pgt_info()
19 if (phys_hop_addr == pgt_info->phys_addr) in hl_mmu_v2_hr_get_pgt_info()
28 hash_add(ctx->hr_mmu_phys_hash, &pgt_info->node, phys_addr); in hl_mmu_v2_hr_add_pgt_info()
33 return &ctx->hdev->mmu_priv.hr.mmu_asid_hop0[ctx->asid]; in hl_mmu_v2_hr_get_hop0_pgt_info()
37 * hl_mmu_v2_hr_init() - initialize the MMU module.
41 * - Create a pool of pages for pgt_infos.
42 * - Create a shadow table for pgt
44 * Return: 0 for success, non-zero for failure.
48 struct asic_fixed_properties *prop = &hdev->asic_prop; in hl_mmu_v2_hr_init() local
50 return hl_mmu_hr_init(hdev, &hdev->mmu_priv.hr, prop->pmmu.hop_table_size, in hl_mmu_v2_hr_init()
51 prop->mmu_pgt_size); in hl_mmu_v2_hr_init()
55 * hl_mmu_v2_hr_fini() - release the MMU module.
59 * - Disable MMU in H/W.
60 * - Free the pgt_infos pool.
66 struct asic_fixed_properties *prop = &hdev->asic_prop; in hl_mmu_v2_hr_fini() local
68 hl_mmu_hr_fini(hdev, &hdev->mmu_priv.hr, prop->pmmu.hop_table_size); in hl_mmu_v2_hr_fini()
72 * hl_mmu_v2_hr_ctx_init() - initialize a context for using the MMU module.
77 * Return: 0 on success, non-zero otherwise.
81 hash_init(ctx->hr_mmu_phys_hash); in hl_mmu_v2_hr_ctx_init()
86 * hl_mmu_v2_hr_ctx_fini - disable a ctx from using the mmu module
91 * - Free any pgts which were not freed yet
92 * - Free the mutex
93 * - Free DRAM default page mapping hops
97 struct hl_device *hdev = ctx->hdev; in hl_mmu_v2_hr_ctx_fini()
102 if (!hash_empty(ctx->hr_mmu_phys_hash)) in hl_mmu_v2_hr_ctx_fini()
103 dev_err(hdev->dev, "ctx %d is freed while it has pgts in use\n", in hl_mmu_v2_hr_ctx_fini()
104 ctx->asid); in hl_mmu_v2_hr_ctx_fini()
106 hash_for_each_safe(ctx->hr_mmu_phys_hash, i, tmp, pgt_info, node) { in hl_mmu_v2_hr_ctx_fini()
107 dev_err_ratelimited(hdev->dev, in hl_mmu_v2_hr_ctx_fini()
109 pgt_info->phys_addr, ctx->asid, pgt_info->num_of_ptes); in hl_mmu_v2_hr_ctx_fini()
110 hl_mmu_hr_free_hop_remove_pgt(pgt_info, &ctx->hdev->mmu_priv.hr, in hl_mmu_v2_hr_ctx_fini()
111 ctx->hdev->asic_prop.pmmu.hop_table_size); in hl_mmu_v2_hr_ctx_fini()
120 struct hl_device *hdev = ctx->hdev; in _hl_mmu_v2_hr_unmap()
121 struct asic_fixed_properties *prop; in _hl_mmu_v2_hr_unmap() local
126 prop = &hdev->asic_prop; in _hl_mmu_v2_hr_unmap()
129 mmu_prop = is_dram_addr ? &prop->dmmu : &prop->pmmu; in _hl_mmu_v2_hr_unmap()
130 hop_last = mmu_prop->num_hops - 1; in _hl_mmu_v2_hr_unmap()
132 scrambled_virt_addr = hdev->asic_funcs->scramble_addr(hdev, virt_addr); in _hl_mmu_v2_hr_unmap()
135 for (i = 0 ; i < mmu_prop->num_hops ; i++) { in _hl_mmu_v2_hr_unmap()
141 &ctx->hdev->mmu_func[MMU_HR_PGT].hr_funcs, curr_pte); in _hl_mmu_v2_hr_unmap()
146 hops_pgt_info[i]->phys_addr, in _hl_mmu_v2_hr_unmap()
149 return -EFAULT; in _hl_mmu_v2_hr_unmap()
153 ctx->hdev->asic_prop.pmmu.hop_table_size); in _hl_mmu_v2_hr_unmap()
155 if ((i < hop_last) && (curr_pte & mmu_prop->last_mask)) { in _hl_mmu_v2_hr_unmap()
163 dev_err(hdev->dev, "DRAM unmapping should use huge pages only\n"); in _hl_mmu_v2_hr_unmap()
164 return -EFAULT; in _hl_mmu_v2_hr_unmap()
170 for (i = hop_last ; i > 0 ; i--) { in _hl_mmu_v2_hr_unmap()
172 ctx->hdev->asic_prop.pmmu.hop_table_size); in _hl_mmu_v2_hr_unmap()
174 if (hl_mmu_hr_put_pte(ctx, hops_pgt_info[i], &ctx->hdev->mmu_priv.hr, in _hl_mmu_v2_hr_unmap()
175 ctx->hdev->asic_prop.pmmu.hop_table_size)) in _hl_mmu_v2_hr_unmap()
179 ctx->hdev->asic_prop.pmmu.hop_table_size); in _hl_mmu_v2_hr_unmap()
185 dev_err(hdev->dev, "virt addr 0x%llx is not mapped to phys addr\n", virt_addr); in _hl_mmu_v2_hr_unmap()
187 return -EINVAL; in _hl_mmu_v2_hr_unmap()
194 for (hop = (mmu_prop->num_hops - 1); hop; hop--) { in hl_mmu_v2_get_last_hop()
195 if (mmu_prop->hop_shifts[hop] == 0) in hl_mmu_v2_get_last_hop()
198 if (page_size <= (1 << mmu_prop->hop_shifts[hop])) in hl_mmu_v2_get_last_hop()
213 struct hl_device *hdev = ctx->hdev; in _hl_mmu_v2_hr_map()
214 struct asic_fixed_properties *prop = &hdev->asic_prop; in _hl_mmu_v2_hr_map() local
216 int i, hop_last, rc = -ENOMEM; in _hl_mmu_v2_hr_map()
226 mmu_prop = &prop->dmmu; in _hl_mmu_v2_hr_map()
227 else if (page_size == prop->pmmu_huge.page_size) in _hl_mmu_v2_hr_map()
228 mmu_prop = &prop->pmmu_huge; in _hl_mmu_v2_hr_map()
230 mmu_prop = &prop->pmmu; in _hl_mmu_v2_hr_map()
234 dev_err(ctx->hdev->dev, "Invalid last HOP %d\n", hop_last); in _hl_mmu_v2_hr_map()
235 return -EFAULT; in _hl_mmu_v2_hr_map()
238 scrambled_virt_addr = hdev->asic_funcs->scramble_addr(hdev, virt_addr); in _hl_mmu_v2_hr_map()
239 scrambled_phys_addr = hdev->asic_funcs->scramble_addr(hdev, phys_addr); in _hl_mmu_v2_hr_map()
247 &ctx->hdev->mmu_priv.hr, in _hl_mmu_v2_hr_map()
248 &ctx->hdev->mmu_func[MMU_HR_PGT].hr_funcs, in _hl_mmu_v2_hr_map()
254 hops_pgt_info[i]->phys_addr, in _hl_mmu_v2_hr_map()
258 ctx->hdev->asic_prop.pmmu.hop_table_size); in _hl_mmu_v2_hr_map()
262 dev_err(hdev->dev, "mapping already exists for virt_addr 0x%llx\n", in _hl_mmu_v2_hr_map()
266 dev_dbg(hdev->dev, "hop%d pte: 0x%llx (0x%llx)\n", in _hl_mmu_v2_hr_map()
271 ctx->hdev->asic_prop.pmmu.hop_table_size), in _hl_mmu_v2_hr_map()
273 rc = -EINVAL; in _hl_mmu_v2_hr_map()
277 curr_pte = (scrambled_phys_addr & HOP_PHYS_ADDR_MASK) | mmu_prop->last_mask in _hl_mmu_v2_hr_map()
282 ctx->hdev->asic_prop.pmmu.hop_table_size); in _hl_mmu_v2_hr_map()
284 /* for each new hop, add its address to the table of previous-hop */ in _hl_mmu_v2_hr_map()
287 curr_pte = (hops_pgt_info[i]->phys_addr & HOP_PHYS_ADDR_MASK) | in _hl_mmu_v2_hr_map()
289 hl_mmu_hr_write_pte(ctx, hops_pgt_info[i - 1], hop_pte_phys_addr[i - 1], in _hl_mmu_v2_hr_map()
290 curr_pte, ctx->hdev->asic_prop.pmmu.hop_table_size); in _hl_mmu_v2_hr_map()
291 if (i - 1) in _hl_mmu_v2_hr_map()
292 hl_mmu_hr_get_pte(ctx, &ctx->hdev->mmu_func[MMU_HR_PGT].hr_funcs, in _hl_mmu_v2_hr_map()
293 hops_pgt_info[i - 1]->phys_addr); in _hl_mmu_v2_hr_map()
297 hl_mmu_hr_get_pte(ctx, &ctx->hdev->mmu_func[MMU_HR_PGT].hr_funcs, in _hl_mmu_v2_hr_map()
298 hops_pgt_info[hop_last]->phys_addr); in _hl_mmu_v2_hr_map()
305 hl_mmu_hr_free_hop_remove_pgt(hops_pgt_info[i], &ctx->hdev->mmu_priv.hr, in _hl_mmu_v2_hr_map()
306 ctx->hdev->asic_prop.pmmu.hop_table_size); in _hl_mmu_v2_hr_map()
312 * hl_mmu_v2_swap_out - marks all mapping of the given ctx as swapped out
323 * hl_mmu_v2_swap_in - marks all mapping of the given ctx as swapped in
338 struct asic_fixed_properties *prop = &hdev->asic_prop; in hl_mmu_v2_hr_get_tlb_mapping_params() local
341 is_dram_addr = hl_mem_area_inside_range(virt_addr, prop->dmmu.page_size, in hl_mmu_v2_hr_get_tlb_mapping_params()
342 prop->dmmu.start_addr, in hl_mmu_v2_hr_get_tlb_mapping_params()
343 prop->dmmu.end_addr); in hl_mmu_v2_hr_get_tlb_mapping_params()
344 is_pmmu_addr = hl_mem_area_inside_range(virt_addr, prop->pmmu.page_size, in hl_mmu_v2_hr_get_tlb_mapping_params()
345 prop->pmmu.start_addr, in hl_mmu_v2_hr_get_tlb_mapping_params()
346 prop->pmmu.end_addr); in hl_mmu_v2_hr_get_tlb_mapping_params()
348 prop->pmmu_huge.page_size, in hl_mmu_v2_hr_get_tlb_mapping_params()
349 prop->pmmu_huge.start_addr, in hl_mmu_v2_hr_get_tlb_mapping_params()
350 prop->pmmu_huge.end_addr); in hl_mmu_v2_hr_get_tlb_mapping_params()
352 *mmu_prop = &prop->dmmu; in hl_mmu_v2_hr_get_tlb_mapping_params()
354 hops->range_type = HL_VA_RANGE_TYPE_DRAM; in hl_mmu_v2_hr_get_tlb_mapping_params()
356 *mmu_prop = &prop->pmmu; in hl_mmu_v2_hr_get_tlb_mapping_params()
358 hops->range_type = HL_VA_RANGE_TYPE_HOST; in hl_mmu_v2_hr_get_tlb_mapping_params()
360 *mmu_prop = &prop->pmmu_huge; in hl_mmu_v2_hr_get_tlb_mapping_params()
362 hops->range_type = HL_VA_RANGE_TYPE_HOST_HUGE; in hl_mmu_v2_hr_get_tlb_mapping_params()
364 return -EINVAL; in hl_mmu_v2_hr_get_tlb_mapping_params()
374 &ctx->hdev->mmu_func[MMU_HR_PGT].hr_funcs); in hl_mmu_v2_hr_get_tlb_info()
378 * hl_mmu_v2_prepare - prepare mmu_if for working with mmu v2
385 mmu->init = hl_mmu_v2_hr_init; in hl_mmu_v2_hr_set_funcs()
386 mmu->fini = hl_mmu_v2_hr_fini; in hl_mmu_v2_hr_set_funcs()
387 mmu->ctx_init = hl_mmu_v2_hr_ctx_init; in hl_mmu_v2_hr_set_funcs()
388 mmu->ctx_fini = hl_mmu_v2_hr_ctx_fini; in hl_mmu_v2_hr_set_funcs()
389 mmu->map = _hl_mmu_v2_hr_map; in hl_mmu_v2_hr_set_funcs()
390 mmu->unmap = _hl_mmu_v2_hr_unmap; in hl_mmu_v2_hr_set_funcs()
391 mmu->flush = hl_mmu_hr_flush; in hl_mmu_v2_hr_set_funcs()
392 mmu->swap_out = hl_mmu_v2_hr_swap_out; in hl_mmu_v2_hr_set_funcs()
393 mmu->swap_in = hl_mmu_v2_hr_swap_in; in hl_mmu_v2_hr_set_funcs()
394 mmu->get_tlb_info = hl_mmu_v2_hr_get_tlb_info; in hl_mmu_v2_hr_set_funcs()
395 mmu->hr_funcs.get_hop0_pgt_info = hl_mmu_v2_hr_get_hop0_pgt_info; in hl_mmu_v2_hr_set_funcs()
396 mmu->hr_funcs.get_pgt_info = hl_mmu_v2_hr_get_pgt_info; in hl_mmu_v2_hr_set_funcs()
397 mmu->hr_funcs.add_pgt_info = hl_mmu_v2_hr_add_pgt_info; in hl_mmu_v2_hr_set_funcs()
398 mmu->hr_funcs.get_tlb_mapping_params = hl_mmu_v2_hr_get_tlb_mapping_params; in hl_mmu_v2_hr_set_funcs()