@@ -115,6 +115,7 @@ enum __kvm_host_smccc_func {
__KVM_HOST_SMCCC_FUNC___pkvm_host_iommu_unmap_pages,
__KVM_HOST_SMCCC_FUNC___pkvm_host_iommu_iova_to_phys,
__KVM_HOST_SMCCC_FUNC___pkvm_host_hvc_pd,
+ __KVM_HOST_SMCCC_FUNC___pkvm_host_iommu_map_sg,
/*
* Start of the dynamically registered hypercalls. Start a bit
@@ -1655,4 +1655,23 @@ int kvm_iommu_register_driver(struct kvm_iommu_driver *kern_ops,
int kvm_iommu_init_driver(void);
void kvm_iommu_remove_driver(void);
+struct kvm_iommu_sg {
+ phys_addr_t phys;
+ size_t pgsize;
+ unsigned int pgcount;
+};
+
+static inline struct kvm_iommu_sg *kvm_iommu_sg_alloc(unsigned int nents, gfp_t gfp)
+{
+ return alloc_pages_exact(PAGE_ALIGN(nents * sizeof(struct kvm_iommu_sg)), gfp);
+}
+
+static inline void kvm_iommu_sg_free(struct kvm_iommu_sg *sg, unsigned int nents)
+{
+ free_pages_exact(sg, PAGE_ALIGN(nents * sizeof(struct kvm_iommu_sg)));
+}
+
+int kvm_iommu_share_hyp_sg(struct kvm_iommu_sg *sg, unsigned int nents);
+int kvm_iommu_unshare_hyp_sg(struct kvm_iommu_sg *sg, unsigned int nents);
+
#endif /* __ARM64_KVM_HOST_H__ */
@@ -22,6 +22,8 @@ size_t kvm_iommu_unmap_pages(pkvm_handle_t domain_id, unsigned long iova,
size_t pgsize, size_t pgcount);
phys_addr_t kvm_iommu_iova_to_phys(pkvm_handle_t domain_id, unsigned long iova);
bool kvm_iommu_host_dabt_handler(struct kvm_cpu_context *host_ctxt, u64 esr, u64 addr);
+size_t kvm_iommu_map_sg(pkvm_handle_t domain, unsigned long iova, struct kvm_iommu_sg *sg,
+ unsigned int nent, unsigned int prot);
/* Flags for memory allocation for IOMMU drivers */
#define IOMMU_PAGE_NOCACHE BIT(0)
@@ -1682,6 +1682,19 @@ static void handle___pkvm_host_hvc_pd(struct kvm_cpu_context *host_ctxt)
cpu_reg(host_ctxt, 1) = pkvm_host_hvc_pd(device_id, on);
}
+static void handle___pkvm_host_iommu_map_sg(struct kvm_cpu_context *host_ctxt)
+{
+ unsigned long ret;
+ DECLARE_REG(pkvm_handle_t, domain, host_ctxt, 1);
+ DECLARE_REG(unsigned long, iova, host_ctxt, 2);
+ DECLARE_REG(struct kvm_iommu_sg *, sg, host_ctxt, 3);
+ DECLARE_REG(unsigned int, nent, host_ctxt, 4);
+ DECLARE_REG(unsigned int, prot, host_ctxt, 5);
+
+ ret = kvm_iommu_map_sg(domain, iova, kern_hyp_va(sg), nent, prot);
+ hyp_reqs_smccc_encode(ret, host_ctxt, this_cpu_ptr(&host_hyp_reqs));
+}
+
typedef void (*hcall_t)(struct kvm_cpu_context *);
#define HANDLE_FUNC(x) [__KVM_HOST_SMCCC_FUNC_##x] = (hcall_t)handle_##x
@@ -1747,6 +1760,7 @@ static const hcall_t host_hcall[] = {
HANDLE_FUNC(__pkvm_host_iommu_unmap_pages),
HANDLE_FUNC(__pkvm_host_iommu_iova_to_phys),
HANDLE_FUNC(__pkvm_host_hvc_pd),
+ HANDLE_FUNC(__pkvm_host_iommu_map_sg),
};
static void handle_host_hcall(struct kvm_cpu_context *host_ctxt)
@@ -392,6 +392,64 @@ bool kvm_iommu_host_dabt_handler(struct kvm_cpu_context *host_ctxt, u64 esr, u64
return ret;
}
+size_t kvm_iommu_map_sg(pkvm_handle_t domain_id, unsigned long iova, struct kvm_iommu_sg *sg,
+ unsigned int nent, unsigned int prot)
+{
+ int ret;
+ size_t total_mapped = 0, mapped;
+ struct kvm_hyp_iommu_domain *domain;
+ phys_addr_t phys;
+ size_t size, pgsize, pgcount;
+ unsigned int orig_nent = nent;
+ struct kvm_iommu_sg *orig_sg = sg;
+
+ if (!kvm_iommu_ops || !kvm_iommu_ops->map_pages)
+ return 0;
+
+ if (prot & ~IOMMU_PROT_MASK)
+ return 0;
+
+ domain = handle_to_domain(domain_id);
+ if (!domain || domain_get(domain))
+ return 0;
+
+ ret = hyp_pin_shared_mem(sg, sg + nent);
+ if (ret)
+ goto out_put_domain;
+
+ while (nent--) {
+ phys = sg->phys;
+ pgsize = sg->pgsize;
+ pgcount = sg->pgcount;
+
+ if (__builtin_mul_overflow(pgsize, pgcount, &size) ||
+ iova + size < iova)
+ goto out_unpin_sg;
+
+ ret = __pkvm_host_use_dma(phys, size);
+ if (ret)
+ goto out_unpin_sg;
+
+ mapped = 0;
+ kvm_iommu_ops->map_pages(domain, iova, phys, pgsize, pgcount, prot, &mapped);
+ total_mapped += mapped;
+ phys += mapped;
+ iova += mapped;
+ /* Might need memory */
+ if (mapped != size) {
+ __pkvm_host_unuse_dma(phys, size - mapped);
+ break;
+ }
+ sg++;
+ }
+
+out_unpin_sg:
+ hyp_unpin_shared_mem(orig_sg, orig_sg + orig_nent);
+out_put_domain:
+ domain_put(domain);
+ return total_mapped;
+}
+
static int iommu_power_on(struct kvm_power_domain *pd)
{
struct kvm_hyp_iommu *iommu = container_of(pd, struct kvm_hyp_iommu,
@@ -55,3 +55,35 @@ void kvm_iommu_remove_driver(void)
if (smp_load_acquire(&iommu_driver))
iommu_driver->remove_driver();
}
+
+int kvm_iommu_share_hyp_sg(struct kvm_iommu_sg *sg, unsigned int nents)
+{
+ size_t nr_pages = PAGE_ALIGN(sizeof(*sg) * nents) >> PAGE_SHIFT;
+ phys_addr_t sg_pfn = virt_to_phys(sg) >> PAGE_SHIFT;
+ int i;
+ int ret;
+
+ for (i = 0 ; i < nr_pages ; ++i) {
+ ret = kvm_call_hyp_nvhe(__pkvm_host_share_hyp, sg_pfn + i);
+ if (ret)
+ return ret;
+ }
+
+ return 0;
+}
+
+int kvm_iommu_unshare_hyp_sg(struct kvm_iommu_sg *sg, unsigned int nents)
+{
+ size_t nr_pages = PAGE_ALIGN(sizeof(*sg) * nents) >> PAGE_SHIFT;
+ phys_addr_t sg_pfn = virt_to_phys(sg) >> PAGE_SHIFT;
+ int i;
+ int ret;
+
+ for (i = 0 ; i < nr_pages ; ++i) {
+ ret = kvm_call_hyp_nvhe(__pkvm_host_unshare_hyp, sg_pfn + i);
+ if (ret)
+ return ret;
+ }
+
+ return 0;
+}
Add a new type struct kvm_iommu_sg, that describes a simple sglist, and a hypercall that can consume it while calling the map_pages ops. Signed-off-by: Mostafa Saleh <smostafa@google.com> --- arch/arm64/include/asm/kvm_asm.h | 1 + arch/arm64/include/asm/kvm_host.h | 19 ++++++++ arch/arm64/kvm/hyp/include/nvhe/iommu.h | 2 + arch/arm64/kvm/hyp/nvhe/hyp-main.c | 14 ++++++ arch/arm64/kvm/hyp/nvhe/iommu/iommu.c | 58 +++++++++++++++++++++++++ arch/arm64/kvm/iommu.c | 32 ++++++++++++++ 6 files changed, 126 insertions(+)