@@ -339,10 +339,8 @@ arm_smmu_mmu_notifier_get(struct arm_smmu_domain *smmu_domain,
struct mm_struct *mm)
{
int ret;
- unsigned long flags;
struct arm_smmu_ctx_desc *cd;
struct arm_smmu_mmu_notifier *smmu_mn;
- struct arm_smmu_master *master;
list_for_each_entry(smmu_mn, &smmu_domain->mmu_notifiers, list) {
if (smmu_mn->mn.mm == mm) {
@@ -372,32 +370,9 @@ arm_smmu_mmu_notifier_get(struct arm_smmu_domain *smmu_domain,
goto err_free_cd;
}
- spin_lock_irqsave(&smmu_domain->devices_lock, flags);
- list_for_each_entry(master, &smmu_domain->devices, domain_head) {
- struct arm_smmu_cd target;
- struct arm_smmu_cd *cdptr;
-
- cdptr = arm_smmu_get_cd_ptr(master, mm->pasid);
- if (!cdptr) {
- ret = -ENOMEM;
- list_for_each_entry_from_reverse(master, &smmu_domain->devices, domain_head)
- arm_smmu_clear_cd(master, mm->pasid);
- break;
- }
-
- arm_smmu_make_sva_cd(&target, master, mm, cd->asid);
- arm_smmu_write_cd_entry(master, mm->pasid, cdptr, &target);
- }
- spin_unlock_irqrestore(&smmu_domain->devices_lock, flags);
- if (ret)
- goto err_put_notifier;
-
list_add(&smmu_mn->list, &smmu_domain->mmu_notifiers);
return smmu_mn;
-err_put_notifier:
- /* Frees smmu_mn */
- mmu_notifier_put(&smmu_mn->mn);
err_free_cd:
arm_smmu_free_shared_cd(cd);
return ERR_PTR(ret);
@@ -408,19 +383,12 @@ static void arm_smmu_mmu_notifier_put(struct arm_smmu_mmu_notifier *smmu_mn)
struct mm_struct *mm = smmu_mn->mn.mm;
struct arm_smmu_ctx_desc *cd = smmu_mn->cd;
struct arm_smmu_domain *smmu_domain = smmu_mn->domain;
- struct arm_smmu_master *master;
- unsigned long flags;
if (!refcount_dec_and_test(&smmu_mn->refs))
return;
list_del(&smmu_mn->list);
- spin_lock_irqsave(&smmu_domain->devices_lock, flags);
- list_for_each_entry(master, &smmu_domain->devices, domain_head)
- arm_smmu_clear_cd(master, mm->pasid);
- spin_unlock_irqrestore(&smmu_domain->devices_lock, flags);
-
/*
* If we went through clear(), we've already invalidated, and no
* new TLB entry can have been formed.
@@ -435,7 +403,8 @@ static void arm_smmu_mmu_notifier_put(struct arm_smmu_mmu_notifier *smmu_mn)
arm_smmu_free_shared_cd(cd);
}
-static int __arm_smmu_sva_bind(struct device *dev, struct mm_struct *mm)
+static int __arm_smmu_sva_bind(struct device *dev, struct mm_struct *mm,
+ struct arm_smmu_cd *target)
{
int ret;
struct arm_smmu_bond *bond;
@@ -462,6 +431,7 @@ static int __arm_smmu_sva_bind(struct device *dev, struct mm_struct *mm)
}
list_add(&bond->list, &master->bonds);
+ arm_smmu_make_sva_cd(target, master, mm, bond->smmu_mn->cd->asid);
return 0;
err_free_bond:
@@ -624,6 +594,8 @@ void arm_smmu_sva_remove_dev_pasid(struct iommu_domain *domain,
struct arm_smmu_bond *bond = NULL, *t;
struct arm_smmu_master *master = dev_iommu_priv_get(dev);
+ arm_smmu_remove_pasid(master, to_smmu_domain(domain), id);
+
mutex_lock(&sva_lock);
list_for_each_entry(t, &master->bonds, list) {
if (t->mm == mm) {
@@ -643,17 +615,26 @@ void arm_smmu_sva_remove_dev_pasid(struct iommu_domain *domain,
static int arm_smmu_sva_set_dev_pasid(struct iommu_domain *domain,
struct device *dev, ioasid_t id)
{
+ struct arm_smmu_master *master = dev_iommu_priv_get(dev);
int ret = 0;
struct mm_struct *mm = domain->mm;
+ struct arm_smmu_cd target;
if (mm->pasid != id)
return -EINVAL;
- mutex_lock(&sva_lock);
- ret = __arm_smmu_sva_bind(dev, mm);
- mutex_unlock(&sva_lock);
+ if (!arm_smmu_get_cd_ptr(master, id))
+ return -ENOMEM;
- return ret;
+ mutex_lock(&sva_lock);
+ ret = __arm_smmu_sva_bind(dev, mm, &target);
+ mutex_unlock(&sva_lock);
+ if (ret)
+ return ret;
+
+ /* This cannot fail since we preallocated the cdptr */
+ arm_smmu_set_pasid(master, to_smmu_domain(domain), id, &target);
+ return 0;
}
static void arm_smmu_sva_domain_free(struct iommu_domain *domain)
@@ -2601,6 +2601,30 @@ static int arm_smmu_attach_dev(struct iommu_domain *domain, struct device *dev)
return 0;
}
+int arm_smmu_set_pasid(struct arm_smmu_master *master,
+ struct arm_smmu_domain *smmu_domain, ioasid_t pasid,
+ const struct arm_smmu_cd *cd)
+{
+ struct arm_smmu_domain *sid_smmu_domain =
+ to_smmu_domain_safe(iommu_get_domain_for_dev(master->dev));
+ struct arm_smmu_cd *cdptr;
+
+ if (!sid_smmu_domain || sid_smmu_domain->stage != ARM_SMMU_DOMAIN_S1)
+ return -ENODEV;
+
+ cdptr = arm_smmu_get_cd_ptr(master, pasid);
+ if (!cdptr)
+ return -ENOMEM;
+ arm_smmu_write_cd_entry(master, pasid, cdptr, cd);
+ return 0;
+}
+
+void arm_smmu_remove_pasid(struct arm_smmu_master *master,
+ struct arm_smmu_domain *smmu_domain, ioasid_t pasid)
+{
+ arm_smmu_clear_cd(master, pasid);
+}
+
static int arm_smmu_attach_dev_ste(struct device *dev,
struct arm_smmu_ste *ste)
{
@@ -770,6 +770,12 @@ void arm_smmu_write_cd_entry(struct arm_smmu_master *master, int ssid,
struct arm_smmu_cd *cdptr,
const struct arm_smmu_cd *target);
+int arm_smmu_set_pasid(struct arm_smmu_master *master,
+ struct arm_smmu_domain *smmu_domain, ioasid_t pasid,
+ const struct arm_smmu_cd *cd);
+void arm_smmu_remove_pasid(struct arm_smmu_master *master,
+ struct arm_smmu_domain *smmu_domain, ioasid_t pasid);
+
void arm_smmu_tlb_inv_asid(struct arm_smmu_device *smmu, u16 asid);
void arm_smmu_tlb_inv_range_asid(unsigned long iova, size_t size, int asid,
size_t granule, bool leaf,
ops->set_dev_pasid()/ops->remove_dev_pasid() should work on a single CD table entry, the one that was actually passed in to the function. The current iterating over the master's list is a hold over from the prior design where the CD table was part of the S1 domain. Lift this code up and out so that we setup the CD only once for the correct thing. The SVA code "works" under a single configuration: - The RID domain is a S1 domain - The programmed PASID is the mm->pasid - Nothing changes while SVA is running (sva_enable) Invalidation will still iterate over the S1 domain's master list. That remains OK after this change, we may do harmless extra ATS invalidations for PASIDs that don't need it. Signed-off-by: Jason Gunthorpe <jgg@nvidia.com> --- .../iommu/arm/arm-smmu-v3/arm-smmu-v3-sva.c | 55 ++++++------------- drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c | 24 ++++++++ drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h | 6 ++ 3 files changed, 48 insertions(+), 37 deletions(-)