@@ -602,10 +602,177 @@ static void smmu_free_domain(struct kvm_hyp_iommu_domain *domain)
hyp_free(smmu_domain);
}
+static void smmu_inv_domain(struct hyp_arm_smmu_v3_domain *smmu_domain)
+{
+ struct kvm_hyp_iommu_domain *domain = smmu_domain->domain;
+ struct hyp_arm_smmu_v3_device *smmu = smmu_domain->smmu;
+ struct arm_smmu_cmdq_ent cmd = {};
+
+ if (smmu_domain->pgtable->cfg.fmt == ARM_64_LPAE_S2) {
+ cmd.opcode = CMDQ_OP_TLBI_S12_VMALL;
+ cmd.tlbi.vmid = domain->domain_id;
+ } else {
+ cmd.opcode = CMDQ_OP_TLBI_NH_ASID;
+ cmd.tlbi.asid = domain->domain_id;
+ }
+
+ if (smmu->iommu.power_is_off)
+ return;
+
+ WARN_ON(smmu_send_cmd(smmu, &cmd));
+}
+
+static void smmu_tlb_flush_all(void *cookie)
+{
+ struct kvm_hyp_iommu_domain *domain = cookie;
+ struct hyp_arm_smmu_v3_domain *smmu_domain = domain->priv;
+ struct hyp_arm_smmu_v3_device *smmu = smmu_domain->smmu;
+
+ kvm_iommu_lock(&smmu->iommu);
+ smmu_inv_domain(smmu_domain);
+ kvm_iommu_unlock(&smmu->iommu);
+}
+
+static int smmu_tlb_inv_range_smmu(struct hyp_arm_smmu_v3_device *smmu,
+ struct kvm_hyp_iommu_domain *domain,
+ struct arm_smmu_cmdq_ent *cmd,
+ unsigned long iova, size_t size, size_t granule)
+{
+ int ret = 0;
+ unsigned long end = iova + size, num_pages = 0, tg = 0;
+ size_t inv_range = granule;
+ struct hyp_arm_smmu_v3_domain *smmu_domain = domain->priv;
+
+ kvm_iommu_lock(&smmu->iommu);
+ if (smmu->iommu.power_is_off)
+ goto out_ret;
+
+ /* Almost copy-paste from the kernel dirver. */
+ if (smmu->features & ARM_SMMU_FEAT_RANGE_INV) {
+ /* Get the leaf page size */
+ tg = __ffs(smmu_domain->pgtable->cfg.pgsize_bitmap);
+
+ num_pages = size >> tg;
+
+ /* Convert page size of 12,14,16 (log2) to 1,2,3 */
+ cmd->tlbi.tg = (tg - 10) / 2;
+
+ /*
+ * Determine what level the granule is at. For non-leaf, both
+ * io-pgtable and SVA pass a nominal last-level granule because
+ * they don't know what level(s) actually apply, so ignore that
+ * and leave TTL=0. However for various errata reasons we still
+ * want to use a range command, so avoid the SVA corner case
+ * where both scale and num could be 0 as well.
+ */
+ if (cmd->tlbi.leaf)
+ cmd->tlbi.ttl = 4 - ((ilog2(granule) - 3) / (tg - 3));
+ else if ((num_pages & CMDQ_TLBI_RANGE_NUM_MAX) == 1)
+ num_pages++;
+ }
+
+ while (iova < end) {
+ if (smmu->features & ARM_SMMU_FEAT_RANGE_INV) {
+ /*
+ * On each iteration of the loop, the range is 5 bits
+ * worth of the aligned size remaining.
+ * The range in pages is:
+ *
+ * range = (num_pages & (0x1f << __ffs(num_pages)))
+ */
+ unsigned long scale, num;
+
+ /* Determine the power of 2 multiple number of pages */
+ scale = __ffs(num_pages);
+ cmd->tlbi.scale = scale;
+
+ /* Determine how many chunks of 2^scale size we have */
+ num = (num_pages >> scale) & CMDQ_TLBI_RANGE_NUM_MAX;
+ cmd->tlbi.num = num - 1;
+
+ /* range is num * 2^scale * pgsize */
+ inv_range = num << (scale + tg);
+
+ /* Clear out the lower order bits for the next iteration */
+ num_pages -= num << scale;
+ }
+ cmd->tlbi.addr = iova;
+ WARN_ON(smmu_add_cmd(smmu, cmd));
+ BUG_ON(iova + inv_range < iova);
+ iova += inv_range;
+ }
+
+ ret = smmu_sync_cmd(smmu);
+out_ret:
+ kvm_iommu_unlock(&smmu->iommu);
+ return ret;
+}
+
+static void smmu_tlb_inv_range(struct kvm_hyp_iommu_domain *domain,
+ unsigned long iova, size_t size, size_t granule,
+ bool leaf)
+{
+ struct hyp_arm_smmu_v3_domain *smmu_domain = domain->priv;
+ unsigned long end = iova + size;
+ struct arm_smmu_cmdq_ent cmd;
+
+ cmd.tlbi.leaf = leaf;
+ if (smmu_domain->pgtable->cfg.fmt == ARM_64_LPAE_S2) {
+ cmd.opcode = CMDQ_OP_TLBI_S2_IPA;
+ cmd.tlbi.vmid = domain->domain_id;
+ } else {
+ cmd.opcode = CMDQ_OP_TLBI_NH_VA;
+ cmd.tlbi.asid = domain->domain_id;
+ cmd.tlbi.vmid = 0;
+ }
+ /*
+ * There are no mappings at high addresses since we don't use TTB1, so
+ * no overflow possible.
+ */
+ BUG_ON(end < iova);
+ WARN_ON(smmu_tlb_inv_range_smmu(smmu_domain->smmu, domain,
+ &cmd, iova, size, granule));
+}
+
+static void smmu_tlb_flush_walk(unsigned long iova, size_t size,
+ size_t granule, void *cookie)
+{
+ smmu_tlb_inv_range(cookie, iova, size, granule, false);
+}
+
+static void smmu_tlb_add_page(struct iommu_iotlb_gather *gather,
+ unsigned long iova, size_t granule,
+ void *cookie)
+{
+ if (gather)
+ kvm_iommu_iotlb_gather_add_page(cookie, gather, iova, granule);
+ else
+ smmu_tlb_inv_range(cookie, iova, granule, granule, true);
+}
+
+__maybe_unused
+static const struct iommu_flush_ops smmu_tlb_ops = {
+ .tlb_flush_all = smmu_tlb_flush_all,
+ .tlb_flush_walk = smmu_tlb_flush_walk,
+ .tlb_add_page = smmu_tlb_add_page,
+};
+
+static void smmu_iotlb_sync(struct kvm_hyp_iommu_domain *domain,
+ struct iommu_iotlb_gather *gather)
+{
+ size_t size;
+
+ if (!gather->pgsize)
+ return;
+ size = gather->end - gather->start + 1;
+ smmu_tlb_inv_range(domain, gather->start, size, gather->pgsize, true);
+}
+
/* Shared with the kernel driver in EL1 */
struct kvm_iommu_ops smmu_ops = {
.init = smmu_init,
.get_iommu_by_id = smmu_id_to_iommu,
.alloc_domain = smmu_alloc_domain,
.free_domain = smmu_free_domain,
+ .iotlb_sync = smmu_iotlb_sync,
};