1 // SPDX-License-Identifier: GPL-2.0-only
6 #define pr_fmt(fmt)     "AMD-Vi: " fmt
16 	if (dev_data->pasid_enabled && dev_data->max_pasids &&  in is_pasid_enabled()
17 	    dev_data->gcr3_info.gcr3_tbl != NULL)  in is_pasid_enabled()
26 	if (pasid > 0 && pasid < dev_data->max_pasids)  in is_pasid_valid()
35 	amd_iommu_clear_gcr3(pdom_dev_data->dev_data, pdom_dev_data->pasid);  in remove_dev_pasid()
37 	list_del(&pdom_dev_data->list);  in remove_dev_pasid()
48 	lockdep_assert_held(&pdom->lock);  in remove_pdom_dev_pasid()
51 		if (pdom_dev_data->dev_data == dev_data &&  in remove_pdom_dev_pasid()
52 		    pdom_dev_data->pasid == pasid) {  in remove_pdom_dev_pasid()
69 	spin_lock_irqsave(&sva_pdom->lock, flags);  in sva_arch_invalidate_secondary_tlbs()
72 		amd_iommu_dev_flush_pasid_pages(pdom_dev_data->dev_data,  in sva_arch_invalidate_secondary_tlbs()
73 						pdom_dev_data->pasid,  in sva_arch_invalidate_secondary_tlbs()
74 						start, end - start);  in sva_arch_invalidate_secondary_tlbs()
77 	spin_unlock_irqrestore(&sva_pdom->lock, flags);  in sva_arch_invalidate_secondary_tlbs()
88 	spin_lock_irqsave(&sva_pdom->lock, flags);  in sva_mn_release()
94 	spin_unlock_irqrestore(&sva_pdom->lock, flags);  in sva_mn_release()
102 int iommu_sva_set_dev_pasid(struct iommu_domain *domain,  in iommu_sva_set_dev_pasid()  argument
106 	struct protection_domain *sva_pdom = to_pdomain(domain);  in iommu_sva_set_dev_pasid()
109 	int ret = -EINVAL;  in iommu_sva_set_dev_pasid()
119 	/* Add PASID to protection domain pasid list */  in iommu_sva_set_dev_pasid()
124 	pdom_dev_data->pasid = pasid;  in iommu_sva_set_dev_pasid()
125 	pdom_dev_data->dev_data = dev_data;  in iommu_sva_set_dev_pasid()
127 	spin_lock_irqsave(&sva_pdom->lock, flags);  in iommu_sva_set_dev_pasid()
131 				 iommu_virt_to_phys(domain->mm->pgd));  in iommu_sva_set_dev_pasid()
137 	list_add(&pdom_dev_data->list, &sva_pdom->dev_data_list);  in iommu_sva_set_dev_pasid()
140 	spin_unlock_irqrestore(&sva_pdom->lock, flags);  in iommu_sva_set_dev_pasid()
145 				struct iommu_domain *domain)  in amd_iommu_remove_dev_pasid()  argument
153 	sva_pdom = to_pdomain(domain);  in amd_iommu_remove_dev_pasid()
155 	spin_lock_irqsave(&sva_pdom->lock, flags);  in amd_iommu_remove_dev_pasid()
160 	spin_unlock_irqrestore(&sva_pdom->lock, flags);  in amd_iommu_remove_dev_pasid()
163 static void iommu_sva_domain_free(struct iommu_domain *domain)  in iommu_sva_domain_free()  argument
165 	struct protection_domain *sva_pdom = to_pdomain(domain);  in iommu_sva_domain_free()
167 	if (sva_pdom->mn.ops)  in iommu_sva_domain_free()
168 		mmu_notifier_unregister(&sva_pdom->mn, domain->mm);  in iommu_sva_domain_free()
170 	amd_iommu_domain_free(domain);  in iommu_sva_domain_free()
186 		return ERR_PTR(-ENOMEM);  in amd_iommu_domain_alloc_sva()
188 	pdom->domain.ops = &amd_sva_domain_ops;  in amd_iommu_domain_alloc_sva()
189 	pdom->mn.ops = &sva_mn;  in amd_iommu_domain_alloc_sva()
191 	ret = mmu_notifier_register(&pdom->mn, mm);  in amd_iommu_domain_alloc_sva()
197 	return &pdom->domain;  in amd_iommu_domain_alloc_sva()