@@ -641,7 +641,7 @@ struct arm_smmu_domain {
struct iommu_domain domain;
struct list_head devices;
- spinlock_t devices_lock;
+ rwlock_t devices_lock;
};
struct arm_smmu_option_prop {
@@ -1536,10 +1536,10 @@ static int arm_smmu_atc_inv_domain(struct arm_smmu_domain *smmu_domain,
arm_smmu_atc_inv_to_cmd(ssid, iova, size, &cmd);
- spin_lock_irqsave(&smmu_domain->devices_lock, flags);
+ read_lock_irqsave(&smmu_domain->devices_lock, flags);
list_for_each_entry(master, &smmu_domain->devices, domain_head)
ret |= arm_smmu_atc_inv_master(master, &cmd);
- spin_unlock_irqrestore(&smmu_domain->devices_lock, flags);
+ read_unlock_irqrestore(&smmu_domain->devices_lock, flags);
return ret ? -ETIMEDOUT : 0;
}
@@ -1648,7 +1648,7 @@ static struct iommu_domain *arm_smmu_domain_alloc(unsigned type)
mutex_init(&smmu_domain->init_mutex);
INIT_LIST_HEAD(&smmu_domain->devices);
- spin_lock_init(&smmu_domain->devices_lock);
+ rwlock_init(&smmu_domain->devices_lock);
return &smmu_domain->domain;
}
@@ -1911,9 +1911,9 @@ static void arm_smmu_detach_dev(struct arm_smmu_master *master)
if (!smmu_domain)
return;
- spin_lock_irqsave(&smmu_domain->devices_lock, flags);
+ write_lock_irqsave(&smmu_domain->devices_lock, flags);
list_del(&master->domain_head);
- spin_unlock_irqrestore(&smmu_domain->devices_lock, flags);
+ write_unlock_irqrestore(&smmu_domain->devices_lock, flags);
master->domain = NULL;
arm_smmu_install_ste_for_dev(master);
@@ -1966,9 +1966,9 @@ static int arm_smmu_attach_dev(struct iommu_domain *domain, struct device *dev)
arm_smmu_install_ste_for_dev(master);
- spin_lock_irqsave(&smmu_domain->devices_lock, flags);
+ write_lock_irqsave(&smmu_domain->devices_lock, flags);
list_add(&master->domain_head, &smmu_domain->devices);
- spin_unlock_irqrestore(&smmu_domain->devices_lock, flags);
+ write_unlock_irqrestore(&smmu_domain->devices_lock, flags);
out_unlock:
mutex_unlock(&smmu_domain->init_mutex);
return ret;
A master add into or remove from smmu_domain->devices only happened in arm_smmu_attach_dev()/arm_smmu_detach_dev(), the frequency of these operations is very low. But we traverse smmu_domain->devices list in arm_smmu_atc_inv_domain() are frequent. So change the protection from spinlock to rwlock can improve concurrency, especially for the smmu domain without ATS masters. By the way, the cmdq has its own lock, so this change is safe. Here is the performance data tested on my board: Before: Jobs: 24 (f=24): [0.1% done] [9798M/0K /s] [2392K/0 iops] [09h:59m:13s] Jobs: 24 (f=24): [0.1% done] [9782M/0K /s] [2388K/0 iops] [09h:59m:12s] Jobs: 24 (f=24): [0.2% done] [9825M/0K /s] [2399K/0 iops] [09h:59m:11s] Jobs: 24 (f=24): [0.2% done] [9836M/0K /s] [2401K/0 iops] [09h:59m:10s] After: Jobs: 24 (f=24): [0.1% done] [10996M/0K /s] [2685K/0 iops] [09h:59m:13s] Jobs: 24 (f=24): [0.1% done] [10817M/0K /s] [2641K/0 iops] [09h:59m:12s] Jobs: 24 (f=24): [0.2% done] [11083M/0K /s] [2706K/0 iops] [09h:59m:11s] Jobs: 24 (f=24): [0.2% done] [10603M/0K /s] [2589K/0 iops] [09h:59m:10s] Signed-off-by: Zhen Lei <thunder.leizhen@huawei.com> Suggested-by: Will Deacon <will@kernel.org> --- drivers/iommu/arm-smmu-v3.c | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-)