108 	int pgd_idx, pud_idx, pmd_idx;  in ivpu_mmu_pgtables_free()  local
119 		for (pud_idx = 0; pud_idx < IVPU_MMU_PGTABLE_ENTRIES; ++pud_idx) {  in ivpu_mmu_pgtables_free()
120 			pmd_dma_ptr = pgtable->pmd_ptrs[pgd_idx][pud_idx];  in ivpu_mmu_pgtables_free()
121 			pmd_dma = pgtable->pud_ptrs[pgd_idx][pud_idx];  in ivpu_mmu_pgtables_free()
127 				pte_dma_ptr = pgtable->pte_ptrs[pgd_idx][pud_idx][pmd_idx];  in ivpu_mmu_pgtables_free()
128 				pte_dma = pgtable->pmd_ptrs[pgd_idx][pud_idx][pmd_idx];  in ivpu_mmu_pgtables_free()
133 			kfree(pgtable->pte_ptrs[pgd_idx][pud_idx]);  in ivpu_mmu_pgtables_free()
183 		    int pud_idx)  in ivpu_mmu_ensure_pmd()  argument
185 	u64 *pmd_dma_ptr = pgtable->pmd_ptrs[pgd_idx][pud_idx];  in ivpu_mmu_ensure_pmd()
195 	drm_WARN_ON(&vdev->drm, pgtable->pte_ptrs[pgd_idx][pud_idx]);  in ivpu_mmu_ensure_pmd()
196 	pgtable->pte_ptrs[pgd_idx][pud_idx] = kzalloc(IVPU_MMU_PGTABLE_SIZE, GFP_KERNEL);  in ivpu_mmu_ensure_pmd()
197 	if (!pgtable->pte_ptrs[pgd_idx][pud_idx])  in ivpu_mmu_ensure_pmd()
200 	pgtable->pmd_ptrs[pgd_idx][pud_idx] = pmd_dma_ptr;  in ivpu_mmu_ensure_pmd()
201 	pgtable->pud_ptrs[pgd_idx][pud_idx] = pmd_dma | IVPU_MMU_ENTRY_VALID;  in ivpu_mmu_ensure_pmd()
212 		    int pgd_idx, int pud_idx, int pmd_idx)  in ivpu_mmu_ensure_pte()  argument
214 	u64 *pte_dma_ptr = pgtable->pte_ptrs[pgd_idx][pud_idx][pmd_idx];  in ivpu_mmu_ensure_pte()
224 	pgtable->pte_ptrs[pgd_idx][pud_idx][pmd_idx] = pte_dma_ptr;  in ivpu_mmu_ensure_pte()
225 	pgtable->pmd_ptrs[pgd_idx][pud_idx][pmd_idx] = pte_dma | IVPU_MMU_ENTRY_VALID;  in ivpu_mmu_ensure_pte()
236 	int pud_idx = FIELD_GET(IVPU_MMU_PUD_INDEX_MASK, vpu_addr);  in ivpu_mmu_context_map_page()  local
245 	if (!ivpu_mmu_ensure_pmd(vdev, &ctx->pgtable, pgd_idx, pud_idx))  in ivpu_mmu_context_map_page()
249 	pte = ivpu_mmu_ensure_pte(vdev, &ctx->pgtable, pgd_idx, pud_idx, pmd_idx);  in ivpu_mmu_context_map_page()
287 	int pud_idx = FIELD_GET(IVPU_MMU_PUD_INDEX_MASK, vpu_addr);  in ivpu_mmu_context_unmap_page()  local
292 	ctx->pgtable.pte_ptrs[pgd_idx][pud_idx][pmd_idx][pte_idx] = IVPU_MMU_ENTRY_INVALID;  in ivpu_mmu_context_unmap_page()
327 	int pud_idx = FIELD_GET(IVPU_MMU_PUD_INDEX_MASK, vpu_addr);  in ivpu_mmu_context_set_page_ro()  local
331 	ctx->pgtable.pte_ptrs[pgd_idx][pud_idx][pmd_idx][pte_idx] |= IVPU_MMU_ENTRY_FLAG_RO;  in ivpu_mmu_context_set_page_ro()
338 	int pud_idx = FIELD_GET(IVPU_MMU_PUD_INDEX_MASK, vpu_addr);  in ivpu_mmu_context_split_page()  local
342 	ctx->pgtable.pte_ptrs[pgd_idx][pud_idx][pmd_idx][pte_idx] &= ~IVPU_MMU_ENTRY_FLAG_CONT;  in ivpu_mmu_context_split_page()