@@ -434,6 +434,7 @@ config ARM_SMMU_V3
tristate "ARM Ltd. System MMU Version 3 (SMMUv3) Support"
depends on ARM64
select IOMMU_API
+ select IOMMU_SVA
select IOMMU_IO_PGTABLE_LPAE
select GENERIC_MSI_IRQ_DOMAIN
help
@@ -36,6 +36,7 @@
#include <linux/amba/bus.h>
#include "io-pgtable-arm.h"
+#include "iommu-sva.h"
/* MMIO registers */
#define ARM_SMMU_IDR0 0x0
@@ -1872,7 +1873,6 @@ static struct arm_smmu_ctx_desc *arm_smmu_share_asid(u16 asid)
return NULL;
}
-__maybe_unused
static struct arm_smmu_ctx_desc *arm_smmu_alloc_shared_cd(struct mm_struct *mm)
{
u16 asid;
@@ -1969,7 +1969,6 @@ static struct arm_smmu_ctx_desc *arm_smmu_alloc_shared_cd(struct mm_struct *mm)
return ERR_PTR(ret);
}
-__maybe_unused
static void arm_smmu_free_shared_cd(struct arm_smmu_ctx_desc *cd)
{
if (arm_smmu_free_asid(cd)) {
@@ -2958,6 +2957,12 @@ static int arm_smmu_attach_dev(struct iommu_domain *domain, struct device *dev)
smmu = master->smmu;
+ if (iommu_sva_enabled(dev)) {
+ /* Did the previous driver forget to release SVA handles? */
+ dev_err(dev, "cannot attach - SVA enabled\n");
+ return -EBUSY;
+ }
+
arm_smmu_detach_dev(master);
mutex_lock(&smmu_domain->init_mutex);
@@ -3057,6 +3062,81 @@ arm_smmu_iova_to_phys(struct iommu_domain *domain, dma_addr_t iova)
return ops->iova_to_phys(ops, iova);
}
+static void arm_smmu_mm_invalidate(struct device *dev, int pasid, void *entry,
+ unsigned long iova, size_t size)
+{
+ /* TODO: Invalidate ATC */
+}
+
+static int arm_smmu_mm_attach(struct device *dev, int pasid, void *entry,
+ bool attach_domain)
+{
+ struct arm_smmu_ctx_desc *cd = entry;
+ struct iommu_domain *domain = iommu_get_domain_for_dev(dev);
+ struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
+
+ /*
+ * If another device in the domain has already been attached, the
+ * context descriptor is already valid.
+ */
+ if (!attach_domain)
+ return 0;
+
+ return arm_smmu_write_ctx_desc(smmu_domain, pasid, cd);
+}
+
+static void arm_smmu_mm_detach(struct device *dev, int pasid, void *entry,
+ bool detach_domain)
+{
+ struct arm_smmu_ctx_desc *cd = entry;
+ struct iommu_domain *domain = iommu_get_domain_for_dev(dev);
+ struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
+
+ if (detach_domain) {
+ arm_smmu_write_ctx_desc(smmu_domain, pasid, NULL);
+
+ /*
+ * The ASID allocator won't broadcast the final TLB
+ * invalidations for this ASID, so we need to do it manually.
+ * For private contexts, freeing io-pgtable ops performs the
+ * invalidation.
+ */
+ arm_smmu_tlb_inv_asid(smmu_domain->smmu, cd->asid);
+ }
+
+ /* TODO: invalidate ATC */
+}
+
+static void *arm_smmu_mm_alloc(struct mm_struct *mm)
+{
+ return arm_smmu_alloc_shared_cd(mm);
+}
+
+static void arm_smmu_mm_free(void *entry)
+{
+ arm_smmu_free_shared_cd(entry);
+}
+
+static struct io_mm_ops arm_smmu_mm_ops = {
+ .alloc = arm_smmu_mm_alloc,
+ .invalidate = arm_smmu_mm_invalidate,
+ .attach = arm_smmu_mm_attach,
+ .detach = arm_smmu_mm_detach,
+ .release = arm_smmu_mm_free,
+};
+
+static struct iommu_sva *
+arm_smmu_sva_bind(struct device *dev, struct mm_struct *mm, void *drvdata)
+{
+ struct iommu_domain *domain = iommu_get_domain_for_dev(dev);
+ struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
+
+ if (smmu_domain->stage != ARM_SMMU_DOMAIN_S1)
+ return ERR_PTR(-EINVAL);
+
+ return iommu_sva_bind_generic(dev, mm, &arm_smmu_mm_ops, drvdata);
+}
+
static struct platform_driver arm_smmu_driver;
static
@@ -3175,6 +3255,7 @@ static void arm_smmu_remove_device(struct device *dev)
master = fwspec->iommu_priv;
smmu = master->smmu;
+ iommu_sva_disable(dev);
arm_smmu_detach_dev(master);
iommu_group_remove_device(dev);
iommu_device_unlink(&smmu->iommu, dev);
@@ -3294,6 +3375,90 @@ static void arm_smmu_get_resv_regions(struct device *dev,
iommu_dma_get_resv_regions(dev, head);
}
+static bool arm_smmu_iopf_supported(struct arm_smmu_master *master)
+{
+ return false;
+}
+
+static bool arm_smmu_dev_has_feature(struct device *dev,
+ enum iommu_dev_features feat)
+{
+ struct arm_smmu_master *master = dev_to_master(dev);
+
+ if (!master)
+ return false;
+
+ switch (feat) {
+ case IOMMU_DEV_FEAT_SVA:
+ if (!(master->smmu->features & ARM_SMMU_FEAT_SVA))
+ return false;
+
+ /* SSID and IOPF support are mandatory for the moment */
+ return master->ssid_bits && arm_smmu_iopf_supported(master);
+ default:
+ return false;
+ }
+}
+
+static bool arm_smmu_dev_feature_enabled(struct device *dev,
+ enum iommu_dev_features feat)
+{
+ struct arm_smmu_master *master = dev_to_master(dev);
+
+ if (!master)
+ return false;
+
+ switch (feat) {
+ case IOMMU_DEV_FEAT_SVA:
+ return iommu_sva_enabled(dev);
+ default:
+ return false;
+ }
+}
+
+static int arm_smmu_dev_enable_sva(struct device *dev)
+{
+ struct arm_smmu_master *master = dev_to_master(dev);
+ struct iommu_sva_param param = {
+ .min_pasid = 1,
+ .max_pasid = 0xfffffU,
+ };
+
+ param.max_pasid = min(param.max_pasid, (1U << master->ssid_bits) - 1);
+ return iommu_sva_enable(dev, ¶m);
+}
+
+static int arm_smmu_dev_enable_feature(struct device *dev,
+ enum iommu_dev_features feat)
+{
+ if (!arm_smmu_dev_has_feature(dev, feat))
+ return -ENODEV;
+
+ if (arm_smmu_dev_feature_enabled(dev, feat))
+ return -EBUSY;
+
+ switch (feat) {
+ case IOMMU_DEV_FEAT_SVA:
+ return arm_smmu_dev_enable_sva(dev);
+ default:
+ return -EINVAL;
+ }
+}
+
+static int arm_smmu_dev_disable_feature(struct device *dev,
+ enum iommu_dev_features feat)
+{
+ if (!arm_smmu_dev_feature_enabled(dev, feat))
+ return -EINVAL;
+
+ switch (feat) {
+ case IOMMU_DEV_FEAT_SVA:
+ return iommu_sva_disable(dev);
+ default:
+ return -EINVAL;
+ }
+}
+
static struct iommu_ops arm_smmu_ops = {
.capable = arm_smmu_capable,
.domain_alloc = arm_smmu_domain_alloc,
@@ -3312,6 +3477,13 @@ static struct iommu_ops arm_smmu_ops = {
.of_xlate = arm_smmu_of_xlate,
.get_resv_regions = arm_smmu_get_resv_regions,
.put_resv_regions = generic_iommu_put_resv_regions,
+ .dev_has_feat = arm_smmu_dev_has_feature,
+ .dev_feat_enabled = arm_smmu_dev_feature_enabled,
+ .dev_enable_feat = arm_smmu_dev_enable_feature,
+ .dev_disable_feat = arm_smmu_dev_disable_feature,
+ .sva_bind = arm_smmu_sva_bind,
+ .sva_unbind = iommu_sva_unbind_generic,
+ .sva_get_pasid = iommu_sva_get_pasid_generic,
.pgsize_bitmap = -1UL, /* Restricted during device attach */
};