@@ -66,7 +66,7 @@ EXPORT_SYMBOL(kvmm_kvm_arch_vcpu_run_map_fp);
*/
void kvm_arch_vcpu_load_fp(struct kvm_vcpu *vcpu)
{
- BUG_ON(!current->mm);
+ BUG_ON(!(vcpu->kvm->is_kvmm_vm ? current->active_mm : current->mm));
vcpu->arch.flags &= ~(KVM_ARM64_FP_ENABLED |
KVM_ARM64_HOST_SVE_IN_USE |
@@ -536,7 +536,8 @@ static inline void __fpregs_load_activate(void)
struct fpu *fpu = ¤t->thread.fpu;
int cpu = smp_processor_id();
- if (WARN_ON_ONCE(current->flags & PF_KTHREAD))
+ /* don't do anything if we're in the kernel */
+ if (current->flags & PF_KTHREAD)
return;
if (!fpregs_state_valid(fpu, cpu)) {
@@ -571,7 +572,8 @@ static inline void __fpregs_load_activate(void)
*/
static inline void switch_fpu_prepare(struct fpu *old_fpu, int cpu)
{
- if (static_cpu_has(X86_FEATURE_FPU) && !(current->flags & PF_KTHREAD)) {
+ if (static_cpu_has(X86_FEATURE_FPU) &&
+ !(current->flags & PF_KTHREAD)) {
if (!copy_fpregs_to_fpstate(old_fpu))
old_fpu->last_cpu = -1;
else
@@ -598,6 +600,10 @@ static inline void switch_fpu_finish(struct fpu *new_fpu)
if (!static_cpu_has(X86_FEATURE_FPU))
return;
+ /* don't do anything if we're in the kernel */
+ if (current->flags & PF_KTHREAD)
+ return;
+
set_thread_flag(TIF_NEED_FPU_LOAD);
if (!cpu_feature_enabled(X86_FEATURE_OSPKE))
@@ -1086,7 +1086,8 @@ static void emulator_get_fpu(void)
fpregs_lock();
fpregs_assert_state_consistent();
- if (test_thread_flag(TIF_NEED_FPU_LOAD))
+ if (test_thread_flag(TIF_NEED_FPU_LOAD) &&
+ !(current->flags & PF_KTHREAD))
switch_fpu_return();
}
@@ -1166,7 +1166,8 @@ void vmx_prepare_switch_to_guest(struct kvm_vcpu *vcpu)
savesegment(es, host_state->es_sel);
gs_base = cpu_kernelmode_gs_base(cpu);
- if (likely(is_64bit_mm(current->mm))) {
+ if (likely(is_64bit_mm(vcpu->kvm->is_kvmm_vm ?
+ current->active_mm : current->mm))) {
save_fsgs_for_kvm();
fs_sel = current->thread.fsindex;
gs_sel = current->thread.gsindex;
@@ -8390,7 +8390,8 @@ static int vcpu_enter_guest(struct kvm_vcpu *vcpu)
guest_enter_irqoff();
fpregs_assert_state_consistent();
- if (test_thread_flag(TIF_NEED_FPU_LOAD))
+ if (test_thread_flag(TIF_NEED_FPU_LOAD) &&
+ !(current->flags & PF_KTHREAD))
switch_fpu_return();
if (unlikely(vcpu->arch.switch_db_regs)) {
@@ -9903,11 +9904,13 @@ void kvm_arch_pre_destroy_vm(struct kvm *kvm)
void kvm_arch_destroy_vm(struct kvm *kvm)
{
- if (current->mm == kvm->mm) {
+ if (current->mm == kvm->mm || kvm->is_kvmm_vm) {
/*
* Free memory regions allocated on behalf of userspace,
* unless the the memory map has changed due to process exit
* or fd copying.
+ * In case it is a kvmm VM, then we need to invalidate the
+ * regions allocated.
*/
mutex_lock(&kvm->slots_lock);
__x86_set_memory_region(kvm, APIC_ACCESS_PAGE_PRIVATE_MEMSLOT,
@@ -121,8 +121,19 @@ static inline bool is_noslot_pfn(kvm_pfn_t pfn)
#define KVM_HVA_ERR_BAD (PAGE_OFFSET)
#define KVM_HVA_ERR_RO_BAD (PAGE_OFFSET + PAGE_SIZE)
+static inline bool kvmm_valid_addr(unsigned long addr)
+{
+ if (is_vmalloc_addr((void *)addr))
+ return true;
+ else
+ return false;
+}
+
static inline bool kvm_is_error_hva(unsigned long addr)
{
+ if (kvmm_valid_addr(addr))
+ return 0;
+
return addr >= PAGE_OFFSET;
}
@@ -503,6 +514,7 @@ struct kvm {
struct srcu_struct srcu;
struct srcu_struct irq_srcu;
pid_t userspace_pid;
+ bool is_kvmm_vm;
};
#define kvm_err(fmt, ...) \
@@ -926,6 +926,8 @@ static void stage2_unmap_memslot(struct kvm *kvm,
phys_addr_t addr = memslot->base_gfn << PAGE_SHIFT;
phys_addr_t size = PAGE_SIZE * memslot->npages;
hva_t reg_end = hva + size;
+ struct mm_struct *mm = kvm->is_kvmm_vm ?
+ current->active_mm : current->mm;
/*
* A memory region could potentially cover multiple VMAs, and any holes
@@ -940,7 +942,7 @@ static void stage2_unmap_memslot(struct kvm *kvm,
* +--------------------------------------------+
*/
do {
- struct vm_area_struct *vma = find_vma(current->mm, hva);
+ struct vm_area_struct *vma = find_vma(mm, hva);
hva_t vm_start, vm_end;
if (!vma || vma->vm_start >= reg_end)
@@ -972,9 +974,11 @@ void stage2_unmap_vm(struct kvm *kvm)
struct kvm_memslots *slots;
struct kvm_memory_slot *memslot;
int idx;
+ struct mm_struct *mm = kvm->is_kvmm_vm ?
+ current->active_mm : current->mm;
idx = srcu_read_lock(&kvm->srcu);
- down_read(¤t->mm->mmap_sem);
+ down_read(&mm->mmap_sem);
spin_lock(&kvm->mmu_lock);
slots = kvm_memslots(kvm);
@@ -982,7 +986,7 @@ void stage2_unmap_vm(struct kvm *kvm)
stage2_unmap_memslot(kvm, memslot);
spin_unlock(&kvm->mmu_lock);
- up_read(¤t->mm->mmap_sem);
+ up_read(&mm->mmap_sem);
srcu_read_unlock(&kvm->srcu, idx);
}
@@ -1673,6 +1677,8 @@ static int user_mem_abort(struct kvm_vcpu *vcpu, phys_addr_t fault_ipa,
pgprot_t mem_type = PAGE_S2;
bool logging_active = memslot_is_logging(memslot);
unsigned long vma_pagesize, flags = 0;
+ struct mm_struct *mm = kvm->is_kvmm_vm ?
+ current->active_mm : current->mm;
write_fault = kvm_is_write_fault(vcpu);
exec_fault = kvm_vcpu_trap_is_iabt(vcpu);
@@ -1684,15 +1690,17 @@ static int user_mem_abort(struct kvm_vcpu *vcpu, phys_addr_t fault_ipa,
}
/* Let's check if we will get back a huge page backed by hugetlbfs */
- down_read(¤t->mm->mmap_sem);
- vma = find_vma_intersection(current->mm, hva, hva + 1);
- if (unlikely(!vma)) {
+ down_read(&mm->mmap_sem);
+ vma = find_vma_intersection(mm, hva, hva + 1);
+ /* vma is invalid for a KVMM guest */
+ if (unlikely(!vma) && !kvm->is_kvmm_vm) {
kvm_err("Failed to find VMA for hva 0x%lx\n", hva);
- up_read(¤t->mm->mmap_sem);
+ up_read(&mm->mmap_sem);
return -EFAULT;
}
- if (is_vm_hugetlb_page(vma))
+ /* FIXME: not sure how to handle hugetlb in KVMM, skip for now */
+ if (is_vm_hugetlb_page(vma) && !kvm->is_kvmm_vm)
vma_shift = huge_page_shift(hstate_vma(vma));
else
vma_shift = PAGE_SHIFT;
@@ -1715,7 +1723,7 @@ static int user_mem_abort(struct kvm_vcpu *vcpu, phys_addr_t fault_ipa,
if (vma_pagesize == PMD_SIZE ||
(vma_pagesize == PUD_SIZE && kvm_stage2_has_pmd(kvm)))
gfn = (fault_ipa & huge_page_mask(hstate_vma(vma))) >> PAGE_SHIFT;
- up_read(¤t->mm->mmap_sem);
+ up_read(&mm->mmap_sem);
/* We need minimum second+third level pages */
ret = mmu_topup_memory_cache(memcache, kvm_mmu_cache_min_pages(kvm),
@@ -2278,6 +2286,8 @@ int kvm_arch_prepare_memory_region(struct kvm *kvm,
hva_t reg_end = hva + mem->memory_size;
bool writable = !(mem->flags & KVM_MEM_READONLY);
int ret = 0;
+ struct mm_struct *mm = kvm->is_kvmm_vm ?
+ current->active_mm : current->mm;
if (change != KVM_MR_CREATE && change != KVM_MR_MOVE &&
change != KVM_MR_FLAGS_ONLY)
@@ -2291,7 +2301,7 @@ int kvm_arch_prepare_memory_region(struct kvm *kvm,
(kvm_phys_size(kvm) >> PAGE_SHIFT))
return -EFAULT;
- down_read(¤t->mm->mmap_sem);
+ down_read(&mm->mmap_sem);
/*
* A memory region could potentially cover multiple VMAs, and any holes
* between them, so iterate over all of them to find out if we can map
@@ -2305,7 +2315,7 @@ int kvm_arch_prepare_memory_region(struct kvm *kvm,
* +--------------------------------------------+
*/
do {
- struct vm_area_struct *vma = find_vma(current->mm, hva);
+ struct vm_area_struct *vma = find_vma(mm, hva);
hva_t vm_start, vm_end;
if (!vma || vma->vm_start >= reg_end)
@@ -2350,7 +2360,7 @@ int kvm_arch_prepare_memory_region(struct kvm *kvm,
stage2_flush_memslot(kvm, memslot);
spin_unlock(&kvm->mmu_lock);
out:
- up_read(¤t->mm->mmap_sem);
+ up_read(&mm->mmap_sem);
return ret;
}
@@ -175,7 +175,9 @@ int kvm_setup_async_pf(struct kvm_vcpu *vcpu, gpa_t cr2_or_gpa,
work->cr2_or_gpa = cr2_or_gpa;
work->addr = hva;
work->arch = *arch;
- work->mm = current->mm;
+ /* if we're a KVMM VM, then current->mm is null */
+ work->mm = work->vcpu->kvm->is_kvmm_vm ?
+ current->active_mm : current->mm;
mmget(work->mm);
kvm_get_kvm(work->vcpu->kvm);
@@ -541,6 +541,9 @@ static const struct mmu_notifier_ops kvm_mmu_notifier_ops = {
static int kvm_init_mmu_notifier(struct kvm *kvm)
{
+ /* FIXME: ignore mmu_notifiers for KVMM guests, for now */
+ if (kvm->is_kvmm_vm)
+ return 0;
kvm->mmu_notifier.ops = &kvm_mmu_notifier_ops;
return mmu_notifier_register(&kvm->mmu_notifier, current->mm);
}
@@ -672,13 +675,22 @@ static struct kvm *kvm_create_vm(unsigned long type)
struct kvm *kvm = kvm_arch_alloc_vm();
int r = -ENOMEM;
int i;
+ /* if we're a KVMM guest, then current->mm is NULL */
+ struct mm_struct *mm = (type == 1) ? current->active_mm : current->mm;
if (!kvm)
return ERR_PTR(-ENOMEM);
+ if (type == 1) {
+ kvm->is_kvmm_vm = true;
+ type = 0;
+ } else {
+ kvm->is_kvmm_vm = false;
+ }
+
spin_lock_init(&kvm->mmu_lock);
- mmgrab(current->mm);
- kvm->mm = current->mm;
+ mmgrab(mm);
+ kvm->mm = mm;
kvm_eventfd_init(kvm);
mutex_init(&kvm->lock);
mutex_init(&kvm->irq_lock);
@@ -741,7 +753,7 @@ static struct kvm *kvm_create_vm(unsigned long type)
out_err:
#if defined(CONFIG_MMU_NOTIFIER) && defined(KVM_ARCH_WANT_MMU_NOTIFIER)
if (kvm->mmu_notifier.ops)
- mmu_notifier_unregister(&kvm->mmu_notifier, current->mm);
+ mmu_notifier_unregister(&kvm->mmu_notifier, mm);
#endif
out_err_no_mmu_notifier:
hardware_disable_all();
@@ -758,7 +770,7 @@ static struct kvm *kvm_create_vm(unsigned long type)
cleanup_srcu_struct(&kvm->srcu);
out_err_no_srcu:
kvm_arch_free_vm(kvm);
- mmdrop(current->mm);
+ mmdrop(mm);
return ERR_PTR(r);
}
struct kvm *kvmm_kvm_create_vm(unsigned long type)
@@ -1228,8 +1240,8 @@ int __kvm_set_memory_region(struct kvm *kvm,
/* We can read the guest memory with __xxx_user() later on. */
if ((id < KVM_USER_MEM_SLOTS) &&
((mem->userspace_addr & (PAGE_SIZE - 1)) ||
- !access_ok((void __user *)(unsigned long)mem->userspace_addr,
- mem->memory_size)))
+ !(access_ok((void __user *)(unsigned long)mem->userspace_addr,
+ mem->memory_size) || kvm->is_kvmm_vm)))
return -EINVAL;
if (as_id >= KVM_ADDRESS_SPACE_NUM || id >= KVM_MEM_SLOTS_NUM)
return -EINVAL;
@@ -1484,7 +1496,6 @@ static int kvm_get_dirty_log_protect(struct kvm *kvm, struct kvm_dirty_log *log)
return 0;
}
-
/**
* kvm_vm_ioctl_get_dirty_log - get and clear the log of dirty pages in a slot
* @kvm: kvm instance
@@ -1636,6 +1647,8 @@ unsigned long kvm_host_page_size(struct kvm_vcpu *vcpu, gfn_t gfn)
{
struct vm_area_struct *vma;
unsigned long addr, size;
+ struct mm_struct *mm = vcpu->kvm->is_kvmm_vm ?
+ current->active_mm : current->mm;
size = PAGE_SIZE;
@@ -1643,15 +1656,15 @@ unsigned long kvm_host_page_size(struct kvm_vcpu *vcpu, gfn_t gfn)
if (kvm_is_error_hva(addr))
return PAGE_SIZE;
- down_read(¤t->mm->mmap_sem);
- vma = find_vma(current->mm, addr);
+ down_read(&mm->mmap_sem);
+ vma = find_vma(mm, addr);
if (!vma)
goto out;
size = vma_kernel_pagesize(vma);
out:
- up_read(¤t->mm->mmap_sem);
+ up_read(&mm->mmap_sem);
return size;
}
@@ -1760,8 +1773,12 @@ static bool hva_to_pfn_fast(unsigned long addr, bool write_fault,
*/
if (!(write_fault || writable))
return false;
-
- npages = __get_user_pages_fast(addr, 1, 1, page);
+ if (kvmm_valid_addr(addr)) {
+ npages = 1;
+ page[0] = vmalloc_to_page((void *)addr);
+ } else {
+ npages = __get_user_pages_fast(addr, 1, 1, page);
+ }
if (npages == 1) {
*pfn = page_to_pfn(page[0]);
@@ -1794,15 +1811,28 @@ static int hva_to_pfn_slow(unsigned long addr, bool *async, bool write_fault,
if (async)
flags |= FOLL_NOWAIT;
- npages = get_user_pages_unlocked(addr, 1, &page, flags);
+ if (kvmm_valid_addr(addr)) {
+ npages = 1;
+ page = vmalloc_to_page((void *)addr);
+ } else {
+ npages = get_user_pages_unlocked(addr, 1, &page, flags);
+ }
if (npages != 1)
return npages;
/* map read fault as writable if possible */
if (unlikely(!write_fault) && writable) {
struct page *wpage;
+ int lnpages = 0;
+
+ if (kvmm_valid_addr(addr)) {
+ npages_local = 1;
+ wpage = vmalloc_to_page((void *)addr);
+ } else {
+ lnpages = __get_user_pages_fast(addr, 1, 1, &wpage);
+ }
- if (__get_user_pages_fast(addr, 1, 1, &wpage) == 1) {
+ if (lnpages == 1) {
*writable = true;
put_page(page);
page = wpage;
@@ -1830,6 +1860,8 @@ static int hva_to_pfn_remapped(struct vm_area_struct *vma,
{
unsigned long pfn;
int r;
+ struct mm_struct *mm = kvmm_valid_addr(addr) ?
+ current->active_mm : current->mm;
r = follow_pfn(vma, addr, &pfn);
if (r) {
@@ -1838,7 +1870,7 @@ static int hva_to_pfn_remapped(struct vm_area_struct *vma,
* not call the fault handler, so do it here.
*/
bool unlocked = false;
- r = fixup_user_fault(current, current->mm, addr,
+ r = fixup_user_fault(current, mm, addr,
(write_fault ? FAULT_FLAG_WRITE : 0),
&unlocked);
if (unlocked)
@@ -1892,6 +1924,8 @@ static kvm_pfn_t hva_to_pfn(unsigned long addr, bool atomic, bool *async,
struct vm_area_struct *vma;
kvm_pfn_t pfn = 0;
int npages, r;
+ struct mm_struct *mm = kvmm_valid_addr(addr) ?
+ current->active_mm : current->mm;
/* we can do it either atomically or asynchronously, not both */
BUG_ON(atomic && async);
@@ -1906,7 +1940,7 @@ static kvm_pfn_t hva_to_pfn(unsigned long addr, bool atomic, bool *async,
if (npages == 1)
return pfn;
- down_read(¤t->mm->mmap_sem);
+ down_read(&mm->mmap_sem);
if (npages == -EHWPOISON ||
(!async && check_user_page_hwpoison(addr))) {
pfn = KVM_PFN_ERR_HWPOISON;
@@ -1914,7 +1948,7 @@ static kvm_pfn_t hva_to_pfn(unsigned long addr, bool atomic, bool *async,
}
retry:
- vma = find_vma_intersection(current->mm, addr, addr + 1);
+ vma = find_vma_intersection(mm, addr, addr + 1);
if (vma == NULL)
pfn = KVM_PFN_ERR_FAULT;
@@ -1930,7 +1964,7 @@ static kvm_pfn_t hva_to_pfn(unsigned long addr, bool atomic, bool *async,
pfn = KVM_PFN_ERR_FAULT;
}
exit:
- up_read(¤t->mm->mmap_sem);
+ up_read(&mm->mmap_sem);
return pfn;
}
@@ -2208,6 +2242,9 @@ EXPORT_SYMBOL_GPL(kvm_release_page_clean);
void kvm_release_pfn_clean(kvm_pfn_t pfn)
{
+ /* rely on the fact that we're in the kernel */
+ if (current->flags & PF_KTHREAD)
+ return;
if (!is_error_noslot_pfn(pfn) && !kvm_is_reserved_pfn(pfn))
put_page(pfn_to_page(pfn));
}
@@ -2244,6 +2281,9 @@ EXPORT_SYMBOL_GPL(kvm_set_pfn_accessed);
void kvm_get_pfn(kvm_pfn_t pfn)
{
+ /* rely on the fact that we're in the kernel */
+ if (current->flags & PF_KTHREAD)
+ return;
if (!kvm_is_reserved_pfn(pfn))
get_page(pfn_to_page(pfn));
}
@@ -3120,8 +3160,10 @@ static long kvm_vcpu_ioctl(struct file *filp,
int r;
struct kvm_fpu *fpu = NULL;
struct kvm_sregs *kvm_sregs = NULL;
+ struct mm_struct *mm = vcpu->kvm->is_kvmm_vm ?
+ current->active_mm : current->mm;
- if (vcpu->kvm->mm != current->mm)
+ if (vcpu->kvm->mm != mm)
return -EIO;
if (unlikely(_IOC_TYPE(ioctl) != KVMIO))
@@ -3327,9 +3369,12 @@ static long kvm_vcpu_compat_ioctl(struct file *filp,
struct kvm_vcpu *vcpu = filp->private_data;
void __user *argp = compat_ptr(arg);
int r;
+ struct mm_struct *mm = vcpu->kvm->is_kvmm_vm ?
+ current->active_mm : current->mm;
- if (vcpu->kvm->mm != current->mm)
+ if (vcpu->kvm->mm != mm) {
return -EIO;
+ }
switch (ioctl) {
case KVM_SET_SIGNAL_MASK: {
@@ -3392,8 +3437,10 @@ static long kvm_device_ioctl(struct file *filp, unsigned int ioctl,
unsigned long arg)
{
struct kvm_device *dev = filp->private_data;
+ struct mm_struct *mm = dev->kvm->is_kvmm_vm ?
+ current->active_mm : current->mm;
- if (dev->kvm->mm != current->mm)
+ if (dev->kvm->mm != mm)
return -EIO;
switch (ioctl) {
@@ -3600,8 +3647,10 @@ static long kvm_vm_ioctl(struct file *filp,
struct kvm *kvm = filp->private_data;
void __user *argp = (void __user *)arg;
int r;
+ struct mm_struct *mm = kvm->is_kvmm_vm ?
+ current->active_mm : current->mm;
- if (kvm->mm != current->mm)
+ if (kvm->mm != mm)
return -EIO;
switch (ioctl) {
case KVM_CREATE_VCPU:
@@ -3797,9 +3846,11 @@ static long kvm_vm_compat_ioctl(struct file *filp,
unsigned int ioctl, unsigned long arg)
{
struct kvm *kvm = filp->private_data;
+ struct mm_struct *mm = kvm->is_kvmm_vm ?
+ current->active_mm : current->mm;
int r;
- if (kvm->mm != current->mm)
+ if (kvm->mm != mm)
return -EIO;
switch (ioctl) {
case KVM_GET_DIRTY_LOG: {