@@ -367,18 +367,13 @@ static void arm_smmu_sva_domain_free(struct iommu_domain *domain)
{
struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
- /*
- * Ensure the ASID is empty in the iommu cache before allowing reuse.
- */
- arm_smmu_tlb_inv_asid(smmu_domain->smmu, smmu_domain->cd.asid);
-
/*
* Notice that the arm_smmu_mm_arch_invalidate_secondary_tlbs op can
* still be called/running at this point. We allow the ASID to be
* reused, and if there is a race then it just suffers harmless
* unnecessary invalidation.
*/
- xa_erase(&arm_smmu_asid_xa, smmu_domain->cd.asid);
+ arm_smmu_domain_free_id(smmu_domain);
/*
* Actual free is defered to the SRCU callback
@@ -423,7 +418,7 @@ struct iommu_domain *arm_smmu_sva_domain_alloc(struct device *dev,
return &smmu_domain->domain;
err_asid:
- xa_erase(&arm_smmu_asid_xa, smmu_domain->cd.asid);
+ arm_smmu_domain_free_id(smmu_domain);
err_free:
kfree(smmu_domain);
return ERR_PTR(ret);
@@ -2057,27 +2057,19 @@ int arm_smmu_atc_inv_domain(struct arm_smmu_domain *smmu_domain,
}
/* IO_PGTABLE API */
-static void arm_smmu_tlb_inv_context(void *cookie)
+static void arm_smmu_tlb_inv_context(struct arm_smmu_domain *smmu_domain)
{
- struct arm_smmu_domain *smmu_domain = cookie;
struct arm_smmu_device *smmu = smmu_domain->smmu;
struct arm_smmu_cmdq_ent cmd;
- /*
- * NOTE: when io-pgtable is in non-strict mode, we may get here with
- * PTEs previously cleared by unmaps on the current CPU not yet visible
- * to the SMMU. We are relying on the dma_wmb() implicit during cmd
- * insertion to guarantee those are observed before the TLBI. Do be
- * careful, 007.
- */
- if (smmu_domain->stage == ARM_SMMU_DOMAIN_S1) {
+ if ((smmu_domain->stage == ARM_SMMU_DOMAIN_S1 ||
+ smmu_domain->domain.type == IOMMU_DOMAIN_SVA)) {
arm_smmu_tlb_inv_asid(smmu, smmu_domain->cd.asid);
- } else {
- cmd.opcode = CMDQ_OP_TLBI_S12_VMALL;
- cmd.tlbi.vmid = smmu_domain->s2_cfg.vmid;
+ } else if (smmu_domain->stage == ARM_SMMU_DOMAIN_S2) {
+ cmd.opcode = CMDQ_OP_TLBI_S12_VMALL;
+ cmd.tlbi.vmid = smmu_domain->s2_cfg.vmid;
arm_smmu_cmdq_issue_cmd_with_sync(smmu, &cmd);
}
- arm_smmu_atc_inv_domain(smmu_domain, 0, 0);
}
static void __arm_smmu_tlb_inv_range(struct arm_smmu_cmdq_ent *cmd,
@@ -2211,7 +2203,6 @@ static void arm_smmu_tlb_inv_walk(unsigned long iova, size_t size,
}
static const struct iommu_flush_ops arm_smmu_flush_ops = {
- .tlb_flush_all = arm_smmu_tlb_inv_context,
.tlb_flush_walk = arm_smmu_tlb_inv_walk,
.tlb_add_page = arm_smmu_tlb_inv_page_nosync,
};
@@ -2275,25 +2266,42 @@ static struct iommu_domain *arm_smmu_domain_alloc_paging(struct device *dev)
return &smmu_domain->domain;
}
-static void arm_smmu_domain_free_paging(struct iommu_domain *domain)
+/*
+ * Return the domain's ASID or VMID back to the allocator. All IDs in the
+ * allocator do not have an IOTLB entries referencing them.
+ */
+void arm_smmu_domain_free_id(struct arm_smmu_domain *smmu_domain)
{
- struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
struct arm_smmu_device *smmu = smmu_domain->smmu;
- free_io_pgtable_ops(smmu_domain->pgtbl_ops);
+ arm_smmu_tlb_inv_context(smmu_domain);
- /* Free the ASID or VMID */
- if (smmu_domain->stage == ARM_SMMU_DOMAIN_S1) {
+ if ((smmu_domain->stage == ARM_SMMU_DOMAIN_S1 ||
+ smmu_domain->domain.type == IOMMU_DOMAIN_SVA) &&
+ smmu_domain->cd.asid) {
/* Prevent SVA from touching the CD while we're freeing it */
mutex_lock(&arm_smmu_asid_lock);
xa_erase(&arm_smmu_asid_xa, smmu_domain->cd.asid);
mutex_unlock(&arm_smmu_asid_lock);
- } else {
- struct arm_smmu_s2_cfg *cfg = &smmu_domain->s2_cfg;
- if (cfg->vmid)
- ida_free(&smmu->vmid_map, cfg->vmid);
+ } else if (smmu_domain->stage == ARM_SMMU_DOMAIN_S2 &&
+ smmu_domain->s2_cfg.vmid) {
+ ida_free(&smmu->vmid_map, smmu_domain->s2_cfg.vmid);
}
+}
+static void arm_smmu_domain_free_paging(struct iommu_domain *domain)
+{
+ struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
+
+ /*
+ * At this point the page table is not programmed into any STE/CD and
+ * there is no possible concurrent HW walker running due to prior STE/CD
+ * invalidations. However entries tagged with the ASID/VMID may still be
+ * in the IOTLB. Invalidating the IOTLB should fully serialize any
+ * concurrent dirty bit write back before freeing the PTE memory.
+ */
+ arm_smmu_domain_free_id(smmu_domain);
+ free_io_pgtable_ops(smmu_domain->pgtbl_ops);
kfree(smmu_domain);
}
@@ -2914,8 +2922,18 @@ static void arm_smmu_flush_iotlb_all(struct iommu_domain *domain)
{
struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
- if (smmu_domain->smmu)
+ /*
+ * NOTE: when io-pgtable is in non-strict mode, we may get here with
+ * PTEs previously cleared by unmaps on the current CPU not yet visible
+ * to the SMMU. We are relying on the dma_wmb() implicit during cmd
+ * insertion to guarantee those are observed before the TLBI. Do be
+ * careful, 007.
+ */
+
+ if (smmu_domain->smmu) {
arm_smmu_tlb_inv_context(smmu_domain);
+ arm_smmu_atc_inv_domain(smmu_domain, 0, 0);
+ }
}
static void arm_smmu_iotlb_sync(struct iommu_domain *domain,
@@ -772,6 +772,7 @@ int arm_smmu_set_pasid(struct arm_smmu_master *master,
void arm_smmu_remove_pasid(struct arm_smmu_master *master,
struct arm_smmu_domain *smmu_domain, ioasid_t pasid);
+void arm_smmu_domain_free_id(struct arm_smmu_domain *smmu_domain);
void arm_smmu_tlb_inv_asid(struct arm_smmu_device *smmu, u16 asid);
void arm_smmu_tlb_inv_range_asid(unsigned long iova, size_t size, int asid,
size_t granule, bool leaf,
The SMMUv3 IOTLB is tagged with a VMID/ASID cache tag. Any time the underlying translation is changed these need to be invalidated. At boot time the IOTLB starts out empty and all cache tags are available for allocation. When a tag is taken out of the allocator the code assumes the IOTLB doesn't reference it, and immediately programs it into a STE/CD. If the cache is referencing the tag then it will have stale data and IOMMU will become incoherent. Thus, whenever an ASID/VMID is freed back to the allocator we need to know that the IOTLB doesn't have any references to it. Invalidation is a bit inconsistent, the SVA code open codes an invalidation prior to freeing while the paging code runs through: arm_smmu_domain_free() free_io_pgtable_ops() io_pgtable_tlb_flush_all)() arm_smmu_tlb_inv_context() To do it. Make arm_smmu_tlb_inv_context() able to invalidate all the domain types and call it from a new arm_smmu_domain_free_id() which also puts back the ID. Lightly reorganize things so arm_smmu_domain_free_id() is the only place that does the final flush prior to ASID/VMID free and that arm_smmu_tlb_inv_context() provides full invalidation for both arm_smmu_flush_iotlb_all() and arm_smmu_domain_free_id(). Remove the iommu_flush_ops::tlb_flush_all to avoid duplicate invalidation on free. Nothing else calls this besides free_io_pgtable_ops(). Signed-off-by: Jason Gunthorpe <jgg@nvidia.com> --- .../iommu/arm/arm-smmu-v3/arm-smmu-v3-sva.c | 9 +-- drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c | 68 ++++++++++++------- drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h | 1 + 3 files changed, 46 insertions(+), 32 deletions(-)