@@ -415,29 +415,27 @@ static void arm_smmu_mmu_notifier_put(struct arm_smmu_mmu_notifier *smmu_mn)
arm_smmu_free_shared_cd(cd);
}
-static int __arm_smmu_sva_bind(struct device *dev, ioasid_t pasid,
- struct mm_struct *mm)
+static struct arm_smmu_bond *__arm_smmu_sva_bind(struct device *dev,
+ struct mm_struct *mm)
{
int ret;
- struct arm_smmu_cd target;
- struct arm_smmu_cd *cdptr;
struct arm_smmu_bond *bond;
struct arm_smmu_master *master = dev_iommu_priv_get(dev);
struct iommu_domain *domain = iommu_get_domain_for_dev(dev);
struct arm_smmu_domain *smmu_domain;
if (!(domain->type & __IOMMU_DOMAIN_PAGING))
- return -ENODEV;
+ return ERR_PTR(-ENODEV);
smmu_domain = to_smmu_domain(domain);
if (smmu_domain->stage != ARM_SMMU_DOMAIN_S1)
- return -ENODEV;
+ return ERR_PTR(-ENODEV);
if (!master || !master->sva_enabled)
- return -ENODEV;
+ return ERR_PTR(-ENODEV);
bond = kzalloc(sizeof(*bond), GFP_KERNEL);
if (!bond)
- return -ENOMEM;
+ return ERR_PTR(-ENOMEM);
bond->mm = mm;
@@ -447,22 +445,12 @@ static int __arm_smmu_sva_bind(struct device *dev, ioasid_t pasid,
goto err_free_bond;
}
- cdptr = arm_smmu_alloc_cd_ptr(master, mm_get_enqcmd_pasid(mm));
- if (!cdptr) {
- ret = -ENOMEM;
- goto err_put_notifier;
- }
- arm_smmu_make_sva_cd(&target, master, mm, bond->smmu_mn->cd->asid);
- arm_smmu_write_cd_entry(master, pasid, cdptr, &target);
-
list_add(&bond->list, &master->bonds);
- return 0;
+ return bond;
-err_put_notifier:
- arm_smmu_mmu_notifier_put(bond->smmu_mn);
err_free_bond:
kfree(bond);
- return ret;
+ return ERR_PTR(ret);
}
bool arm_smmu_sva_supported(struct arm_smmu_device *smmu)
@@ -609,10 +597,9 @@ void arm_smmu_sva_remove_dev_pasid(struct iommu_domain *domain,
struct arm_smmu_bond *bond = NULL, *t;
struct arm_smmu_master *master = dev_iommu_priv_get(dev);
+ arm_smmu_remove_pasid(master, to_smmu_domain(domain), id);
+
mutex_lock(&sva_lock);
-
- arm_smmu_clear_cd(master, id);
-
list_for_each_entry(t, &master->bonds, list) {
if (t->mm == mm) {
bond = t;
@@ -631,17 +618,33 @@ void arm_smmu_sva_remove_dev_pasid(struct iommu_domain *domain,
static int arm_smmu_sva_set_dev_pasid(struct iommu_domain *domain,
struct device *dev, ioasid_t id)
{
- int ret = 0;
+ struct arm_smmu_master *master = dev_iommu_priv_get(dev);
struct mm_struct *mm = domain->mm;
+ struct arm_smmu_bond *bond;
+ struct arm_smmu_cd target;
+ int ret;
if (mm_get_enqcmd_pasid(mm) != id)
return -EINVAL;
mutex_lock(&sva_lock);
- ret = __arm_smmu_sva_bind(dev, id, mm);
- mutex_unlock(&sva_lock);
+ bond = __arm_smmu_sva_bind(dev, mm);
+ if (IS_ERR(bond)) {
+ mutex_unlock(&sva_lock);
+ return PTR_ERR(bond);
+ }
- return ret;
+ arm_smmu_make_sva_cd(&target, master, mm, bond->smmu_mn->cd->asid);
+ ret = arm_smmu_set_pasid(master, NULL, id, &target);
+ if (ret) {
+ list_del(&bond->list);
+ arm_smmu_mmu_notifier_put(bond->smmu_mn);
+ kfree(bond);
+ mutex_unlock(&sva_lock);
+ return ret;
+ }
+ mutex_unlock(&sva_lock);
+ return 0;
}
static void arm_smmu_sva_domain_free(struct iommu_domain *domain)
@@ -1223,8 +1223,8 @@ struct arm_smmu_cd *arm_smmu_get_cd_ptr(struct arm_smmu_master *master,
return &l1_desc->l2ptr[ssid % CTXDESC_L2_ENTRIES];
}
-struct arm_smmu_cd *arm_smmu_alloc_cd_ptr(struct arm_smmu_master *master,
- u32 ssid)
+static struct arm_smmu_cd *arm_smmu_alloc_cd_ptr(struct arm_smmu_master *master,
+ u32 ssid)
{
struct arm_smmu_ctx_desc_cfg *cd_table = &master->cd_table;
struct arm_smmu_device *smmu = master->smmu;
@@ -2417,6 +2417,10 @@ static void arm_smmu_install_ste_for_dev(struct arm_smmu_master *master,
int i, j;
struct arm_smmu_device *smmu = master->smmu;
+ master->cd_table.in_ste =
+ FIELD_GET(STRTAB_STE_0_CFG, le64_to_cpu(target->data[0])) ==
+ STRTAB_STE_0_CFG_S1_TRANS;
+
for (i = 0; i < master->num_streams; ++i) {
u32 sid = master->streams[i].id;
struct arm_smmu_ste *step =
@@ -2637,6 +2641,30 @@ static int arm_smmu_attach_dev(struct iommu_domain *domain, struct device *dev)
return 0;
}
+int arm_smmu_set_pasid(struct arm_smmu_master *master,
+ struct arm_smmu_domain *smmu_domain, ioasid_t pasid,
+ const struct arm_smmu_cd *cd)
+{
+ struct arm_smmu_cd *cdptr;
+
+ /* The core code validates pasid */
+
+ if (!master->cd_table.in_ste)
+ return -ENODEV;
+
+ cdptr = arm_smmu_alloc_cd_ptr(master, pasid);
+ if (!cdptr)
+ return -ENOMEM;
+ arm_smmu_write_cd_entry(master, pasid, cdptr, cd);
+ return 0;
+}
+
+void arm_smmu_remove_pasid(struct arm_smmu_master *master,
+ struct arm_smmu_domain *smmu_domain, ioasid_t pasid)
+{
+ arm_smmu_clear_cd(master, pasid);
+}
+
static int arm_smmu_attach_dev_ste(struct device *dev,
struct arm_smmu_ste *ste)
{
@@ -602,6 +602,7 @@ struct arm_smmu_ctx_desc_cfg {
dma_addr_t cdtab_dma;
struct arm_smmu_l1_ctx_desc *l1_desc;
unsigned int num_l1_ents;
+ u8 in_ste;
u8 s1fmt;
/* log2 of the maximum number of CDs supported by this table */
u8 s1cdmax;
@@ -746,8 +747,6 @@ extern struct mutex arm_smmu_asid_lock;
void arm_smmu_clear_cd(struct arm_smmu_master *master, ioasid_t ssid);
struct arm_smmu_cd *arm_smmu_get_cd_ptr(struct arm_smmu_master *master,
u32 ssid);
-struct arm_smmu_cd *arm_smmu_alloc_cd_ptr(struct arm_smmu_master *master,
- u32 ssid);
void arm_smmu_make_s1_cd(struct arm_smmu_cd *target,
struct arm_smmu_master *master,
struct arm_smmu_domain *smmu_domain);
@@ -755,6 +754,12 @@ void arm_smmu_write_cd_entry(struct arm_smmu_master *master, int ssid,
struct arm_smmu_cd *cdptr,
const struct arm_smmu_cd *target);
+int arm_smmu_set_pasid(struct arm_smmu_master *master,
+ struct arm_smmu_domain *smmu_domain, ioasid_t pasid,
+ const struct arm_smmu_cd *cd);
+void arm_smmu_remove_pasid(struct arm_smmu_master *master,
+ struct arm_smmu_domain *smmu_domain, ioasid_t pasid);
+
void arm_smmu_tlb_inv_asid(struct arm_smmu_device *smmu, u16 asid);
void arm_smmu_tlb_inv_range_asid(unsigned long iova, size_t size, int asid,
size_t granule, bool leaf,