119 static bool increase_address_space(struct amd_io_pgtable *pgtable,  in increase_address_space()  argument
123 	struct io_pgtable_cfg *cfg = &pgtable->pgtbl.cfg;  in increase_address_space()
125 		container_of(pgtable, struct protection_domain, iop);  in increase_address_space()
136 	if (address <= PM_LEVEL_SIZE(pgtable->mode))  in increase_address_space()
140 	if (WARN_ON_ONCE(pgtable->mode == PAGE_MODE_6_LEVEL))  in increase_address_space()
143 	*pte = PM_LEVEL_PDE(pgtable->mode, iommu_virt_to_phys(pgtable->root));  in increase_address_space()
145 	pgtable->root  = pte;  in increase_address_space()
146 	pgtable->mode += 1;  in increase_address_space()
159 static u64 *alloc_pte(struct amd_io_pgtable *pgtable,  in alloc_pte()  argument
166 	struct io_pgtable_cfg *cfg = &pgtable->pgtbl.cfg;  in alloc_pte()
172 	while (address > PM_LEVEL_SIZE(pgtable->mode)) {  in alloc_pte()
177 		if (!increase_address_space(pgtable, address, gfp))  in alloc_pte()
182 	level   = pgtable->mode - 1;  in alloc_pte()
183 	pte     = &pgtable->root[PM_LEVEL_INDEX(level, address)];  in alloc_pte()
255 static u64 *fetch_pte(struct amd_io_pgtable *pgtable,  in fetch_pte()  argument
264 	if (address > PM_LEVEL_SIZE(pgtable->mode))  in fetch_pte()
267 	level	   =  pgtable->mode - 1;  in fetch_pte()
268 	pte	   = &pgtable->root[PM_LEVEL_INDEX(level, address)];  in fetch_pte()
332 	struct amd_io_pgtable *pgtable = io_pgtable_ops_to_data(ops);  in iommu_v1_map_pages()  local
349 		pte   = alloc_pte(pgtable, iova, pgsize, NULL, gfp, &updated);  in iommu_v1_map_pages()
410 	struct amd_io_pgtable *pgtable = io_pgtable_ops_to_data(ops);  in iommu_v1_unmap_pages()  local
421 		pte = fetch_pte(pgtable, iova, &unmap_size);  in iommu_v1_unmap_pages()
441 	struct amd_io_pgtable *pgtable = io_pgtable_ops_to_data(ops);  in iommu_v1_iova_to_phys()  local
445 	pte = fetch_pte(pgtable, iova, &pte_pgsize);  in iommu_v1_iova_to_phys()
495 	struct amd_io_pgtable *pgtable = io_pgtable_ops_to_data(ops);  in iommu_v1_read_and_clear_dirty()  local
502 		ptep = fetch_pte(pgtable, iova, &pgsize);  in iommu_v1_read_and_clear_dirty()
528 	struct amd_io_pgtable *pgtable = container_of(iop, struct amd_io_pgtable, pgtbl);  in v1_free_pgtable()  local
531 	if (pgtable->mode == PAGE_MODE_NONE)  in v1_free_pgtable()
535 	BUG_ON(pgtable->mode < PAGE_MODE_NONE ||  in v1_free_pgtable()
536 	       pgtable->mode > PAGE_MODE_6_LEVEL);  in v1_free_pgtable()
538 	free_sub_pt(pgtable->root, pgtable->mode, &freelist);  in v1_free_pgtable()
544 	struct amd_io_pgtable *pgtable = io_pgtable_cfg_to_data(cfg);  in v1_alloc_pgtable()  local
546 	pgtable->root = iommu_alloc_page_node(cfg->amd.nid, GFP_KERNEL);  in v1_alloc_pgtable()
547 	if (!pgtable->root)  in v1_alloc_pgtable()
549 	pgtable->mode = PAGE_MODE_3_LEVEL;  in v1_alloc_pgtable()
555 	pgtable->pgtbl.ops.map_pages    = iommu_v1_map_pages;  in v1_alloc_pgtable()
556 	pgtable->pgtbl.ops.unmap_pages  = iommu_v1_unmap_pages;  in v1_alloc_pgtable()
557 	pgtable->pgtbl.ops.iova_to_phys = iommu_v1_iova_to_phys;  in v1_alloc_pgtable()
558 	pgtable->pgtbl.ops.read_and_clear_dirty = iommu_v1_read_and_clear_dirty;  in v1_alloc_pgtable()
560 	return &pgtable->pgtbl;  in v1_alloc_pgtable()