@@ -24,6 +24,7 @@ enum {
GZVM_FUNC_INFORM_EXIT = 14,
GZVM_FUNC_MEMREGION_PURPOSE = 15,
GZVM_FUNC_SET_DTB_CONFIG = 16,
+ GZVM_FUNC_MAP_GUEST = 17,
NR_GZVM_FUNC,
};
@@ -48,6 +49,7 @@ enum {
#define MT_HVC_GZVM_INFORM_EXIT GZVM_HCALL_ID(GZVM_FUNC_INFORM_EXIT)
#define MT_HVC_GZVM_MEMREGION_PURPOSE GZVM_HCALL_ID(GZVM_FUNC_MEMREGION_PURPOSE)
#define MT_HVC_GZVM_SET_DTB_CONFIG GZVM_HCALL_ID(GZVM_FUNC_SET_DTB_CONFIG)
+#define MT_HVC_GZVM_MAP_GUEST GZVM_HCALL_ID(GZVM_FUNC_MAP_GUEST)
#define GIC_V3_NR_LRS 16
@@ -161,6 +161,122 @@ static int gzvm_vm_ioctl_get_pvmfw_size(struct gzvm *gzvm,
return 0;
}
+/**
+ * fill_constituents() - Populate pa to buffer until full
+ * @consti: Pointer to struct mem_region_addr_range.
+ * @consti_cnt: Constituent count.
+ * @max_nr_consti: Maximum number of constituent count.
+ * @gfn: Guest frame number.
+ * @total_pages: Total page numbers.
+ * @slot: Pointer to struct gzvm_memslot.
+ *
+ * Return: how many pages we've fill in, negative if error
+ */
+static int fill_constituents(struct mem_region_addr_range *consti,
+ int *consti_cnt, int max_nr_consti, u64 gfn,
+ u32 total_pages, struct gzvm_memslot *slot)
+{
+ u64 pfn, prev_pfn, gfn_end;
+ int nr_pages = 1;
+ int i = 0;
+
+ if (unlikely(total_pages == 0))
+ return -EINVAL;
+ gfn_end = gfn + total_pages;
+
+ /* entry 0 */
+ if (gzvm_gfn_to_pfn_memslot(slot, gfn, &pfn) != 0)
+ return -EFAULT;
+ consti[0].address = PFN_PHYS(pfn);
+ consti[0].pg_cnt = 1;
+ gfn++;
+ prev_pfn = pfn;
+
+ while (i < max_nr_consti && gfn < gfn_end) {
+ if (gzvm_gfn_to_pfn_memslot(slot, gfn, &pfn) != 0)
+ return -EFAULT;
+ if (pfn == (prev_pfn + 1)) {
+ consti[i].pg_cnt++;
+ } else {
+ i++;
+ if (i >= max_nr_consti)
+ break;
+ consti[i].address = PFN_PHYS(pfn);
+ consti[i].pg_cnt = 1;
+ }
+ prev_pfn = pfn;
+ gfn++;
+ nr_pages++;
+ }
+ if (i != max_nr_consti)
+ i++;
+ *consti_cnt = i;
+
+ return nr_pages;
+}
+
+/**
+ * populate_mem_region() - Iterate all mem slot and populate pa to buffer until it's full
+ * @gzvm: Pointer to struct gzvm.
+ *
+ * Return: 0 if it is successful, negative if error
+ */
+static int populate_mem_region(struct gzvm *gzvm)
+{
+ int slot_cnt = 0;
+
+ while (slot_cnt < GZVM_MAX_MEM_REGION && gzvm->memslot[slot_cnt].npages != 0) {
+ struct gzvm_memslot *memslot = &gzvm->memslot[slot_cnt];
+ struct gzvm_memory_region_ranges *region;
+ int max_nr_consti, remain_pages;
+ u64 gfn, gfn_end;
+ u32 buf_size;
+
+ buf_size = PAGE_SIZE * 2;
+ region = alloc_pages_exact(buf_size, GFP_KERNEL);
+ if (!region)
+ return -ENOMEM;
+
+ max_nr_consti = (buf_size - sizeof(*region)) /
+ sizeof(struct mem_region_addr_range);
+
+ region->slot = memslot->slot_id;
+ remain_pages = memslot->npages;
+ gfn = memslot->base_gfn;
+ gfn_end = gfn + remain_pages;
+
+ while (gfn < gfn_end) {
+ int nr_pages;
+
+ nr_pages = fill_constituents(region->constituents,
+ ®ion->constituent_cnt,
+ max_nr_consti, gfn,
+ remain_pages, memslot);
+
+ if (nr_pages < 0) {
+ pr_err("Failed to fill constituents\n");
+ free_pages_exact(region, buf_size);
+ return -EFAULT;
+ }
+
+ region->gpa = PFN_PHYS(gfn);
+ region->total_pages = nr_pages;
+ remain_pages -= nr_pages;
+ gfn += nr_pages;
+
+ if (gzvm_arch_set_memregion(gzvm->vm_id, buf_size,
+ virt_to_phys(region))) {
+ pr_err("Failed to register memregion to hypervisor\n");
+ free_pages_exact(region, buf_size);
+ return -EFAULT;
+ }
+ }
+ free_pages_exact(region, buf_size);
+ ++slot_cnt;
+ }
+ return 0;
+}
+
/**
* gzvm_vm_ioctl_cap_pvm() - Proceed GZVM_CAP_PROTECTED_VM's subcommands
* @gzvm: Pointer to struct gzvm.
@@ -182,6 +298,11 @@ static int gzvm_vm_ioctl_cap_pvm(struct gzvm *gzvm,
case GZVM_CAP_PVM_SET_PVMFW_GPA:
fallthrough;
case GZVM_CAP_PVM_SET_PROTECTED_VM:
+ /*
+ * To improve performance for protected VM, we have to populate VM's memory
+ * before VM booting
+ */
+ populate_mem_region(gzvm);
ret = gzvm_vm_arch_enable_cap(gzvm, cap, &res);
return ret;
case GZVM_CAP_PVM_GET_PVMFW_SIZE:
@@ -235,3 +356,12 @@ u64 gzvm_hva_to_pa_arch(u64 hva)
return GZVM_PA_ERR_BAD;
return par;
}
+
+int gzvm_arch_map_guest(u16 vm_id, int memslot_id, u64 pfn, u64 gfn,
+ u64 nr_pages)
+{
+ struct arm_smccc_res res;
+
+ return gzvm_hypcall_wrapper(MT_HVC_GZVM_MAP_GUEST, vm_id, memslot_id,
+ pfn, gfn, nr_pages, 0, 0, &res);
+}
@@ -8,4 +8,5 @@ GZVM_DIR ?= ../../../drivers/virt/geniezone
gzvm-y := $(GZVM_DIR)/gzvm_main.o $(GZVM_DIR)/gzvm_mmu.o \
$(GZVM_DIR)/gzvm_vm.o $(GZVM_DIR)/gzvm_vcpu.o \
- $(GZVM_DIR)/gzvm_irqfd.o $(GZVM_DIR)/gzvm_ioeventfd.o
+ $(GZVM_DIR)/gzvm_irqfd.o $(GZVM_DIR)/gzvm_ioeventfd.o \
+ $(GZVM_DIR)/gzvm_exception.o
new file mode 100644
@@ -0,0 +1,39 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright (c) 2023 MediaTek Inc.
+ */
+
+#include <linux/device.h>
+#include <linux/gzvm_drv.h>
+
+/**
+ * gzvm_handle_guest_exception() - Handle guest exception
+ * @vcpu: Pointer to struct gzvm_vcpu_run in userspace
+ * Return:
+ * * true - This exception has been processed, no need to back to VMM.
+ * * false - This exception has not been processed, require userspace.
+ */
+bool gzvm_handle_guest_exception(struct gzvm_vcpu *vcpu)
+{
+ int ret;
+
+ for (int i = 0; i < ARRAY_SIZE(vcpu->run->exception.reserved); i++) {
+ if (vcpu->run->exception.reserved[i])
+ return -EINVAL;
+ }
+
+ switch (vcpu->run->exception.exception) {
+ case GZVM_EXCEPTION_PAGE_FAULT:
+ ret = gzvm_handle_page_fault(vcpu);
+ break;
+ case GZVM_EXCEPTION_UNKNOWN:
+ fallthrough;
+ default:
+ ret = -EFAULT;
+ }
+
+ if (!ret)
+ return true;
+ else
+ return false;
+}
@@ -28,6 +28,8 @@ int gzvm_err_to_errno(unsigned long err)
return 0;
case ERR_NO_MEMORY:
return -ENOMEM;
+ case ERR_INVALID_ARGS:
+ return -EINVAL;
case ERR_NOT_SUPPORTED:
return -EOPNOTSUPP;
case ERR_NOT_IMPLEMENTED:
@@ -106,3 +106,35 @@ int gzvm_gfn_to_pfn_memslot(struct gzvm_memslot *memslot, u64 gfn,
return 0;
}
+
+/**
+ * gzvm_handle_page_fault() - Handle guest page fault, find corresponding page
+ * for the faulting gpa
+ * @vcpu: Pointer to struct gzvm_vcpu_run of the faulting vcpu
+ *
+ * Return:
+ * * 0 - Success to handle guest page fault
+ * * -EFAULT - Failed to map phys addr to guest's GPA
+ */
+int gzvm_handle_page_fault(struct gzvm_vcpu *vcpu)
+{
+ struct gzvm *vm = vcpu->gzvm;
+ u64 pfn, gfn;
+ int memslot_id;
+ int ret;
+
+ gfn = PHYS_PFN(vcpu->run->exception.fault_gpa);
+ memslot_id = gzvm_find_memslot(vm, gfn);
+ if (unlikely(memslot_id < 0))
+ return -EFAULT;
+
+ ret = gzvm_gfn_to_pfn_memslot(&vm->memslot[memslot_id], gfn, &pfn);
+ if (unlikely(ret))
+ return -EFAULT;
+
+ ret = gzvm_arch_map_guest(vm->vm_id, memslot_id, pfn, gfn, 1);
+ if (unlikely(ret))
+ return -EFAULT;
+
+ return 0;
+}
@@ -114,9 +114,11 @@ static long gzvm_vcpu_run(struct gzvm_vcpu *vcpu, void * __user argp)
* it's geniezone's responsibility to fill corresponding data
* structure
*/
- case GZVM_EXIT_HYPERCALL:
- fallthrough;
case GZVM_EXIT_EXCEPTION:
+ if (!gzvm_handle_guest_exception(vcpu))
+ need_userspace = true;
+ break;
+ case GZVM_EXIT_HYPERCALL:
fallthrough;
case GZVM_EXIT_DEBUG:
fallthrough;
@@ -24,57 +24,28 @@ u64 gzvm_gfn_to_hva_memslot(struct gzvm_memslot *memslot, u64 gfn)
}
/**
- * fill_constituents() - Populate pa to buffer until full
- * @consti: Pointer to struct mem_region_addr_range.
- * @consti_cnt: Constituent count.
- * @max_nr_consti: Maximum number of constituent count.
- * @gfn: Guest frame number.
- * @total_pages: Total page numbers.
- * @slot: Pointer to struct gzvm_memslot.
+ * gzvm_find_memslot() - Find memslot containing this @gpa
+ * @vm: Pointer to struct gzvm
+ * @gfn: Guest frame number
*
- * Return: how many pages we've fill in, negative if error
+ * Return:
+ * * >=0 - Index of memslot
+ * * -EFAULT - Not found
*/
-static int fill_constituents(struct mem_region_addr_range *consti,
- int *consti_cnt, int max_nr_consti, u64 gfn,
- u32 total_pages, struct gzvm_memslot *slot)
+int gzvm_find_memslot(struct gzvm *vm, u64 gfn)
{
- u64 pfn, prev_pfn, gfn_end;
- int nr_pages = 1;
- int i = 0;
+ int i;
- if (unlikely(total_pages == 0))
- return -EINVAL;
- gfn_end = gfn + total_pages;
+ for (i = 0; i < GZVM_MAX_MEM_REGION; i++) {
+ if (vm->memslot[i].npages == 0)
+ continue;
- /* entry 0 */
- if (gzvm_gfn_to_pfn_memslot(slot, gfn, &pfn) != 0)
- return -EFAULT;
- consti[0].address = PFN_PHYS(pfn);
- consti[0].pg_cnt = 1;
- gfn++;
- prev_pfn = pfn;
-
- while (i < max_nr_consti && gfn < gfn_end) {
- if (gzvm_gfn_to_pfn_memslot(slot, gfn, &pfn) != 0)
- return -EFAULT;
- if (pfn == (prev_pfn + 1)) {
- consti[i].pg_cnt++;
- } else {
- i++;
- if (i >= max_nr_consti)
- break;
- consti[i].address = PFN_PHYS(pfn);
- consti[i].pg_cnt = 1;
- }
- prev_pfn = pfn;
- gfn++;
- nr_pages++;
+ if (gfn >= vm->memslot[i].base_gfn &&
+ gfn < vm->memslot[i].base_gfn + vm->memslot[i].npages)
+ return i;
}
- if (i != max_nr_consti)
- i++;
- *consti_cnt = i;
- return nr_pages;
+ return -EFAULT;
}
/**
@@ -89,42 +60,24 @@ register_memslot_addr_range(struct gzvm *gzvm, struct gzvm_memslot *memslot)
{
struct gzvm_memory_region_ranges *region;
u32 buf_size = PAGE_SIZE * 2;
- int max_nr_consti, remain_pages;
- u64 gfn, gfn_end;
+ u64 gfn;
region = alloc_pages_exact(buf_size, GFP_KERNEL);
if (!region)
return -ENOMEM;
- max_nr_consti = (buf_size - sizeof(*region)) /
- sizeof(struct mem_region_addr_range);
region->slot = memslot->slot_id;
- remain_pages = memslot->npages;
+ region->total_pages = memslot->npages;
gfn = memslot->base_gfn;
- gfn_end = gfn + remain_pages;
- while (gfn < gfn_end) {
- int nr_pages = fill_constituents(region->constituents,
- ®ion->constituent_cnt,
- max_nr_consti, gfn,
- remain_pages, memslot);
- if (nr_pages < 0) {
- pr_err("Failed to fill constituents\n");
- free_pages_exact(region, buf_size);
- return nr_pages;
- }
- region->gpa = PFN_PHYS(gfn);
- region->total_pages = nr_pages;
+ region->gpa = PFN_PHYS(gfn);
- remain_pages -= nr_pages;
- gfn += nr_pages;
-
- if (gzvm_arch_set_memregion(gzvm->vm_id, buf_size,
- virt_to_phys(region))) {
- pr_err("Failed to register memregion to hypervisor\n");
- free_pages_exact(region, buf_size);
- return -EFAULT;
- }
+ if (gzvm_arch_set_memregion(gzvm->vm_id, buf_size,
+ virt_to_phys(region))) {
+ pr_err("Failed to register memregion to hypervisor\n");
+ free_pages_exact(region, buf_size);
+ return -EFAULT;
}
+
free_pages_exact(region, buf_size);
return 0;
}
@@ -29,6 +29,7 @@
*/
#define NO_ERROR (0)
#define ERR_NO_MEMORY (-5)
+#define ERR_INVALID_ARGS (-8)
#define ERR_NOT_SUPPORTED (-24)
#define ERR_NOT_IMPLEMENTED (-27)
#define ERR_FAULT (-40)
@@ -123,6 +124,8 @@ int gzvm_arch_set_memregion(u16 vm_id, size_t buf_size,
int gzvm_arch_check_extension(struct gzvm *gzvm, __u64 cap, void __user *argp);
int gzvm_arch_create_vm(unsigned long vm_type);
int gzvm_arch_destroy_vm(u16 vm_id);
+int gzvm_arch_map_guest(u16 vm_id, int memslot_id, u64 pfn, u64 gfn,
+ u64 nr_pages);
int gzvm_vm_ioctl_arch_enable_cap(struct gzvm *gzvm,
struct gzvm_enable_cap *cap,
void __user *argp);
@@ -141,6 +144,10 @@ int gzvm_arch_vcpu_run(struct gzvm_vcpu *vcpu, __u64 *exit_reason);
int gzvm_arch_destroy_vcpu(u16 vm_id, int vcpuid);
int gzvm_arch_inform_exit(u16 vm_id);
+int gzvm_find_memslot(struct gzvm *vm, u64 gpa);
+int gzvm_handle_page_fault(struct gzvm_vcpu *vcpu);
+bool gzvm_handle_guest_exception(struct gzvm_vcpu *vcpu);
+
int gzvm_arch_create_device(u16 vm_id, struct gzvm_create_device *gzvm_dev);
int gzvm_arch_inject_irq(struct gzvm *gzvm, unsigned int vcpu_idx,
u32 irq, bool level);
@@ -185,6 +185,12 @@ enum {
GZVM_EXIT_GZ = 0x9292000a,
};
+/* exception definitions of GZVM_EXIT_EXCEPTION */
+enum {
+ GZVM_EXCEPTION_UNKNOWN = 0x0,
+ GZVM_EXCEPTION_PAGE_FAULT = 0x1,
+};
+
/**
* struct gzvm_vcpu_run: Same purpose as kvm_run, this struct is
* shared between userspace, kernel and
@@ -209,6 +215,9 @@ enum {
* Handle exception occurred in VM
* @exception: Which exception vector
* @error_code: Exception error codes
+ * @fault_gpa: Fault GPA (guest physical address or IPA in ARM)
+ * @reserved: Future-proof reservation and reset to zero in hypervisor.
+ * Fill up to the union size, 256 bytes.
* @hypercall: The nested struct in anonymous union.
* Some hypercalls issued from VM must be handled
* @args: The hypercall's arguments
@@ -255,6 +264,8 @@ struct gzvm_vcpu_run {
struct {
__u32 exception;
__u32 error_code;
+ __u64 fault_gpa;
+ __u64 reserved[30];
} exception;
/* GZVM_EXIT_HYPERCALL */
struct {