@@ -2458,6 +2458,13 @@ static void arm_smmu_install_ste_for_dev(struct arm_smmu_master *master,
int i, j;
struct arm_smmu_device *smmu = master->smmu;
+ master->cd_table.in_ste =
+ FIELD_GET(STRTAB_STE_0_CFG, le64_to_cpu(target->data[0])) ==
+ STRTAB_STE_0_CFG_S1_TRANS;
+ master->ste_ats_enabled =
+ FIELD_GET(STRTAB_STE_1_EATS, le64_to_cpu(target->data[1])) ==
+ STRTAB_STE_1_EATS_TRANS;
+
for (i = 0; i < master->num_streams; ++i) {
u32 sid = master->streams[i].id;
struct arm_smmu_ste *step =
@@ -2784,27 +2791,47 @@ static int arm_smmu_attach_dev(struct iommu_domain *domain, struct device *dev)
return 0;
}
-static bool arm_smmu_is_s1_domain(struct iommu_domain *domain)
+static void arm_smmu_update_ste(struct arm_smmu_master *master,
+ struct iommu_domain *sid_domain,
+ bool want_ats)
{
- if (!domain || !(domain->type & __IOMMU_DOMAIN_PAGING))
- return false;
- return to_smmu_domain(domain)->stage == ARM_SMMU_DOMAIN_S1;
+ unsigned int s1dss = STRTAB_STE_1_S1DSS_TERMINATE;
+ struct arm_smmu_ste ste;
+
+ if (master->cd_table.in_ste && master->ste_ats_enabled == want_ats)
+ return;
+
+ if (sid_domain->type == IOMMU_DOMAIN_IDENTITY)
+ s1dss = STRTAB_STE_1_S1DSS_BYPASS;
+ else
+ WARN_ON(sid_domain->type != IOMMU_DOMAIN_BLOCKED);
+
+ /*
+ * Change the STE into a cdtable one with SID IDENTITY/BLOCKED behavior
+ * using s1dss if necessary. The cd_table is already installed then
+ * the S1DSS is correct and this will just update the EATS. Otherwise
+ * it installs the entire thing. This will be hitless.
+ */
+ arm_smmu_make_cdtable_ste(&ste, master, want_ats, s1dss);
+ arm_smmu_install_ste_for_dev(master, &ste);
}
int arm_smmu_set_pasid(struct arm_smmu_master *master,
struct arm_smmu_domain *smmu_domain, ioasid_t pasid,
const struct arm_smmu_cd *cd)
{
+ struct iommu_domain *sid_domain = iommu_get_domain_for_dev(master->dev);
struct attach_state state = {.ssid = pasid};
struct arm_smmu_cd *cdptr;
int ret;
- if (smmu_domain->smmu != master->smmu)
+ if (smmu_domain->smmu != master->smmu || pasid == IOMMU_NO_PASID)
return -EINVAL;
- if (!arm_smmu_is_s1_domain(iommu_get_domain_for_dev(master->dev)) ||
- !master->cd_table.used_sid)
- return -ENODEV;
+ if (!master->cd_table.in_ste &&
+ sid_domain->type != IOMMU_DOMAIN_IDENTITY &&
+ sid_domain->type != IOMMU_DOMAIN_BLOCKED)
+ return -EINVAL;
cdptr = arm_smmu_get_cd_ptr(master, pasid);
if (!cdptr)
@@ -2816,6 +2843,7 @@ int arm_smmu_set_pasid(struct arm_smmu_master *master,
goto out_unlock;
arm_smmu_write_cd_entry(master, pasid, cdptr, cd);
+ arm_smmu_update_ste(master, sid_domain, state.want_ats);
arm_smmu_attach_commit(master, &state);
@@ -2829,6 +2857,7 @@ static void arm_smmu_remove_dev_pasid(struct device *dev, ioasid_t pasid)
struct arm_smmu_master *master = dev_iommu_priv_get(dev);
struct arm_smmu_domain *smmu_domain;
struct iommu_domain *domain;
+ bool last_ssid = master->cd_table.used_ssids == 1;
domain = iommu_get_domain_for_dev_pasid(dev, pasid, IOMMU_DOMAIN_SVA);
if (WARN_ON(IS_ERR(domain)) || !domain)
@@ -2842,6 +2871,17 @@ static void arm_smmu_remove_dev_pasid(struct device *dev, ioasid_t pasid)
arm_smmu_atc_inv_master(master, pasid);
arm_smmu_remove_master_domain(master, &smmu_domain->domain, pasid);
mutex_unlock(&master->smmu->asid_lock);
+
+ /*
+ * When the last user of the CD table goes away downgrade the STE back
+ * to a non-cd_table one.
+ */
+ if (last_ssid && !master->cd_table.used_sid) {
+ struct iommu_domain *sid_domain =
+ iommu_get_domain_for_dev(master->dev);
+
+ sid_domain->ops->attach_dev(sid_domain, master->dev);
+ }
}
static void arm_smmu_attach_dev_ste(struct iommu_domain *domain,
@@ -600,7 +600,8 @@ struct arm_smmu_ctx_desc_cfg {
struct arm_smmu_l1_ctx_desc *l1_desc;
unsigned int num_l1_ents;
unsigned int used_ssids;
- bool used_sid;
+ u8 used_sid;
+ u8 in_ste;
u8 s1fmt;
/* log2 of the maximum number of CDs supported by this table */
u8 s1cdmax;
@@ -708,7 +709,8 @@ struct arm_smmu_master {
/* Locked by the iommu core using the group mutex */
struct arm_smmu_ctx_desc_cfg cd_table;
unsigned int num_streams;
- bool ats_enabled;
+ bool ats_enabled : 1;
+ bool ste_ats_enabled : 1;
bool stall_enabled;
bool sva_enabled;
bool iopf_enabled;