93 static int ivpu_mmu_pgtable_init(struct ivpu_device *vdev, struct ivpu_mmu_pgtable *pgtable)  in ivpu_mmu_pgtable_init()  argument
97 	pgtable->pgd_dma_ptr = ivpu_pgtable_alloc_page(vdev, &pgd_dma);  in ivpu_mmu_pgtable_init()
98 	if (!pgtable->pgd_dma_ptr)  in ivpu_mmu_pgtable_init()
101 	pgtable->pgd_dma = pgd_dma;  in ivpu_mmu_pgtable_init()
106 static void ivpu_mmu_pgtables_free(struct ivpu_device *vdev, struct ivpu_mmu_pgtable *pgtable)  in ivpu_mmu_pgtables_free()  argument
113 		pud_dma_ptr = pgtable->pud_ptrs[pgd_idx];  in ivpu_mmu_pgtables_free()
114 		pud_dma = pgtable->pgd_dma_ptr[pgd_idx];  in ivpu_mmu_pgtables_free()
120 			pmd_dma_ptr = pgtable->pmd_ptrs[pgd_idx][pud_idx];  in ivpu_mmu_pgtables_free()
121 			pmd_dma = pgtable->pud_ptrs[pgd_idx][pud_idx];  in ivpu_mmu_pgtables_free()
127 				pte_dma_ptr = pgtable->pte_ptrs[pgd_idx][pud_idx][pmd_idx];  in ivpu_mmu_pgtables_free()
128 				pte_dma = pgtable->pmd_ptrs[pgd_idx][pud_idx][pmd_idx];  in ivpu_mmu_pgtables_free()
133 			kfree(pgtable->pte_ptrs[pgd_idx][pud_idx]);  in ivpu_mmu_pgtables_free()
137 		kfree(pgtable->pmd_ptrs[pgd_idx]);  in ivpu_mmu_pgtables_free()
138 		kfree(pgtable->pte_ptrs[pgd_idx]);  in ivpu_mmu_pgtables_free()
142 	ivpu_pgtable_free_page(vdev, pgtable->pgd_dma_ptr, pgtable->pgd_dma);  in ivpu_mmu_pgtables_free()
146 ivpu_mmu_ensure_pud(struct ivpu_device *vdev, struct ivpu_mmu_pgtable *pgtable, int pgd_idx)  in ivpu_mmu_ensure_pud()  argument
148 	u64 *pud_dma_ptr = pgtable->pud_ptrs[pgd_idx];  in ivpu_mmu_ensure_pud()
158 	drm_WARN_ON(&vdev->drm, pgtable->pmd_ptrs[pgd_idx]);  in ivpu_mmu_ensure_pud()
159 	pgtable->pmd_ptrs[pgd_idx] = kzalloc(IVPU_MMU_PGTABLE_SIZE, GFP_KERNEL);  in ivpu_mmu_ensure_pud()
160 	if (!pgtable->pmd_ptrs[pgd_idx])  in ivpu_mmu_ensure_pud()
163 	drm_WARN_ON(&vdev->drm, pgtable->pte_ptrs[pgd_idx]);  in ivpu_mmu_ensure_pud()
164 	pgtable->pte_ptrs[pgd_idx] = kzalloc(IVPU_MMU_PGTABLE_SIZE, GFP_KERNEL);  in ivpu_mmu_ensure_pud()
165 	if (!pgtable->pte_ptrs[pgd_idx])  in ivpu_mmu_ensure_pud()
168 	pgtable->pud_ptrs[pgd_idx] = pud_dma_ptr;  in ivpu_mmu_ensure_pud()
169 	pgtable->pgd_dma_ptr[pgd_idx] = pud_dma | IVPU_MMU_ENTRY_VALID;  in ivpu_mmu_ensure_pud()
174 	kfree(pgtable->pmd_ptrs[pgd_idx]);  in ivpu_mmu_ensure_pud()
182 ivpu_mmu_ensure_pmd(struct ivpu_device *vdev, struct ivpu_mmu_pgtable *pgtable, int pgd_idx,  in ivpu_mmu_ensure_pmd()  argument
185 	u64 *pmd_dma_ptr = pgtable->pmd_ptrs[pgd_idx][pud_idx];  in ivpu_mmu_ensure_pmd()
195 	drm_WARN_ON(&vdev->drm, pgtable->pte_ptrs[pgd_idx][pud_idx]);  in ivpu_mmu_ensure_pmd()
196 	pgtable->pte_ptrs[pgd_idx][pud_idx] = kzalloc(IVPU_MMU_PGTABLE_SIZE, GFP_KERNEL);  in ivpu_mmu_ensure_pmd()
197 	if (!pgtable->pte_ptrs[pgd_idx][pud_idx])  in ivpu_mmu_ensure_pmd()
200 	pgtable->pmd_ptrs[pgd_idx][pud_idx] = pmd_dma_ptr;  in ivpu_mmu_ensure_pmd()
201 	pgtable->pud_ptrs[pgd_idx][pud_idx] = pmd_dma | IVPU_MMU_ENTRY_VALID;  in ivpu_mmu_ensure_pmd()
211 ivpu_mmu_ensure_pte(struct ivpu_device *vdev, struct ivpu_mmu_pgtable *pgtable,  in ivpu_mmu_ensure_pte()  argument
214 	u64 *pte_dma_ptr = pgtable->pte_ptrs[pgd_idx][pud_idx][pmd_idx];  in ivpu_mmu_ensure_pte()
224 	pgtable->pte_ptrs[pgd_idx][pud_idx][pmd_idx] = pte_dma_ptr;  in ivpu_mmu_ensure_pte()
225 	pgtable->pmd_ptrs[pgd_idx][pud_idx][pmd_idx] = pte_dma | IVPU_MMU_ENTRY_VALID;  in ivpu_mmu_ensure_pte()
241 	if (!ivpu_mmu_ensure_pud(vdev, &ctx->pgtable, pgd_idx))  in ivpu_mmu_context_map_page()
245 	if (!ivpu_mmu_ensure_pmd(vdev, &ctx->pgtable, pgd_idx, pud_idx))  in ivpu_mmu_context_map_page()
249 	pte = ivpu_mmu_ensure_pte(vdev, &ctx->pgtable, pgd_idx, pud_idx, pmd_idx);  in ivpu_mmu_context_map_page()
292 	ctx->pgtable.pte_ptrs[pgd_idx][pud_idx][pmd_idx][pte_idx] = IVPU_MMU_ENTRY_INVALID;  in ivpu_mmu_context_unmap_page()
331 	ctx->pgtable.pte_ptrs[pgd_idx][pud_idx][pmd_idx][pte_idx] |= IVPU_MMU_ENTRY_FLAG_RO;  in ivpu_mmu_context_set_page_ro()
342 	ctx->pgtable.pte_ptrs[pgd_idx][pud_idx][pmd_idx][pte_idx] &= ~IVPU_MMU_ENTRY_FLAG_CONT;  in ivpu_mmu_context_split_page()
541 	ret = ivpu_mmu_pgtable_init(vdev, &ctx->pgtable);  in ivpu_mmu_context_init()
563 	if (drm_WARN_ON(&vdev->drm, !ctx->pgtable.pgd_dma_ptr))  in ivpu_mmu_context_fini()
567 	ivpu_mmu_pgtables_free(vdev, &ctx->pgtable);  in ivpu_mmu_context_fini()
570 	ctx->pgtable.pgd_dma_ptr = NULL;  in ivpu_mmu_context_fini()
571 	ctx->pgtable.pgd_dma = 0;  in ivpu_mmu_context_fini()
619 	ret = ivpu_mmu_set_pgtable(vdev, ctx_id, &ctx->pgtable);  in ivpu_mmu_user_context_init()