@@ -24,6 +24,7 @@
#define __ASM_KVM_HOST_H
#define KVM_MEMORY_SLOTS 32
+#define KVM_ALIAS_SLOTS 0
/* memory slots that does not exposed to userspace */
#define KVM_PRIVATE_MEM_SLOTS 4
@@ -1947,11 +1947,6 @@ int kvm_cpu_has_pending_timer(struct kvm_vcpu *vcpu)
return vcpu->arch.timer_fired;
}
-gfn_t unalias_gfn(struct kvm *kvm, gfn_t gfn)
-{
- return gfn;
-}
-
int kvm_arch_vcpu_runnable(struct kvm_vcpu *vcpu)
{
return (vcpu->arch.mp_state == KVM_MP_STATE_RUNNABLE) ||
@@ -35,11 +35,6 @@
#define CREATE_TRACE_POINTS
#include "trace.h"
-gfn_t unalias_gfn(struct kvm *kvm, gfn_t gfn)
-{
- return gfn;
-}
-
int kvm_arch_vcpu_runnable(struct kvm_vcpu *v)
{
return !(v->arch.msr & MSR_WE) || !!(v->arch.pending_exceptions);
@@ -21,6 +21,7 @@
#define KVM_MAX_VCPUS 64
#define KVM_MEMORY_SLOTS 32
+#define KVM_ALIAS_SLOTS 0
/* memory slots that does not exposed to userspace */
#define KVM_PRIVATE_MEM_SLOTS 4
@@ -741,11 +741,6 @@ void kvm_arch_flush_shadow(struct kvm *kvm)
{
}
-gfn_t unalias_gfn(struct kvm *kvm, gfn_t gfn)
-{
- return gfn;
-}
-
static int __init kvm_s390_init(void)
{
int ret;
@@ -367,24 +367,7 @@ struct kvm_vcpu_arch {
u64 hv_vapic;
};
-struct kvm_mem_alias {
- gfn_t base_gfn;
- unsigned long npages;
- gfn_t target_gfn;
-#define KVM_ALIAS_INVALID 1UL
- unsigned long flags;
-};
-
-#define KVM_ARCH_HAS_UNALIAS_INSTANTIATION
-
-struct kvm_mem_aliases {
- struct kvm_mem_alias aliases[KVM_ALIAS_SLOTS];
- int naliases;
-};
-
struct kvm_arch {
- struct kvm_mem_aliases *aliases;
-
unsigned int n_free_mmu_pages;
unsigned int n_requested_mmu_pages;
unsigned int n_alloc_mmu_pages;
@@ -674,8 +657,6 @@ void kvm_disable_tdp(void);
int load_pdptrs(struct kvm_vcpu *vcpu, unsigned long cr3);
int complete_pio(struct kvm_vcpu *vcpu);
-struct kvm_memory_slot *gfn_to_memslot_unaliased(struct kvm *kvm, gfn_t gfn);
-
static inline struct kvm_mmu_page *page_header(hpa_t shadow_page)
{
struct page *page = pfn_to_page(shadow_page >> PAGE_SHIFT);
@@ -347,9 +347,9 @@ enum vmcs_field {
#define AR_RESERVD_MASK 0xfffe0f00
-#define TSS_PRIVATE_MEMSLOT (KVM_MEMORY_SLOTS + 0)
-#define APIC_ACCESS_PAGE_PRIVATE_MEMSLOT (KVM_MEMORY_SLOTS + 1)
-#define IDENTITY_PAGETABLE_PRIVATE_MEMSLOT (KVM_MEMORY_SLOTS + 2)
+#define TSS_PRIVATE_MEMSLOT (KVM_MEMORY_SLOTS + KVM_ALIAS_SLOTS + 0)
+#define APIC_ACCESS_PAGE_PRIVATE_MEMSLOT (KVM_MEMORY_SLOTS + KVM_ALIAS_SLOTS + 1)
+#define IDENTITY_PAGETABLE_PRIVATE_MEMSLOT (KVM_MEMORY_SLOTS + KVM_ALIAS_SLOTS + 2)
#define VMX_NR_VPIDS (1 << 16)
#define VMX_VPID_EXTENT_SINGLE_CONTEXT 1
@@ -422,9 +422,7 @@ static void account_shadowed(struct kvm *kvm, gfn_t gfn)
int *write_count;
int i;
- gfn = unalias_gfn(kvm, gfn);
-
- slot = gfn_to_memslot_unaliased(kvm, gfn);
+ slot = gfn_to_memslot(kvm, gfn);
for (i = PT_DIRECTORY_LEVEL;
i < PT_PAGE_TABLE_LEVEL + KVM_NR_PAGE_SIZES; ++i) {
write_count = slot_largepage_idx(gfn, slot, i);
@@ -438,10 +436,9 @@ static void unaccount_shadowed(struct kvm *kvm, gfn_t gfn)
int *write_count;
int i;
- gfn = unalias_gfn(kvm, gfn);
for (i = PT_DIRECTORY_LEVEL;
i < PT_PAGE_TABLE_LEVEL + KVM_NR_PAGE_SIZES; ++i) {
- slot = gfn_to_memslot_unaliased(kvm, gfn);
+ slot = gfn_to_memslot(kvm, gfn);
write_count = slot_largepage_idx(gfn, slot, i);
*write_count -= 1;
WARN_ON(*write_count < 0);
@@ -455,8 +452,7 @@ static int has_wrprotected_page(struct kvm *kvm,
struct kvm_memory_slot *slot;
int *largepage_idx;
- gfn = unalias_gfn(kvm, gfn);
- slot = gfn_to_memslot_unaliased(kvm, gfn);
+ slot = gfn_to_memslot(kvm, gfn);
if (slot) {
largepage_idx = slot_largepage_idx(gfn, slot, level);
return *largepage_idx;
@@ -523,7 +519,6 @@ static int mapping_level(struct kvm_vcpu *vcpu, gfn_t large_gfn)
/*
* Take gfn and return the reverse mapping to it.
- * Note: gfn must be unaliased before this function get called
*/
static unsigned long *gfn_to_rmap(struct kvm *kvm, gfn_t gfn, int level)
@@ -563,7 +558,6 @@ static int rmap_add(struct kvm_vcpu *vcpu, u64 *spte, gfn_t gfn)
if (!is_rmap_spte(*spte))
return count;
- gfn = unalias_gfn(vcpu->kvm, gfn);
sp = page_header(__pa(spte));
sp->gfns[spte - sp->spt] = gfn;
rmapp = gfn_to_rmap(vcpu->kvm, gfn, sp->role.level);
@@ -700,7 +694,6 @@ static int rmap_write_protect(struct kvm *kvm, u64 gfn)
u64 *spte;
int i, write_protected = 0;
- gfn = unalias_gfn(kvm, gfn);
rmapp = gfn_to_rmap(kvm, gfn, PT_PAGE_TABLE_LEVEL);
spte = rmap_next(kvm, rmapp, NULL);
@@ -879,7 +872,6 @@ static void rmap_recycle(struct kvm_vcpu *vcpu, u64 *spte, gfn_t gfn)
sp = page_header(__pa(spte));
- gfn = unalias_gfn(vcpu->kvm, gfn);
rmapp = gfn_to_rmap(vcpu->kvm, gfn, sp->role.level);
kvm_unmap_rmapp(vcpu->kvm, rmapp, 0);
@@ -2896,7 +2888,6 @@ void kvm_mmu_slot_remove_write_access(struct kvm *kvm, int slot)
if (pt[i] & PT_WRITABLE_MASK)
pt[i] &= ~PT_WRITABLE_MASK;
}
- kvm_flush_remote_tlbs(kvm);
}
void kvm_mmu_zap_all(struct kvm *kvm)
@@ -3403,8 +3394,8 @@ static void audit_write_protection(struct kvm_vcpu *vcpu)
if (sp->unsync)
continue;
- gfn = unalias_gfn(vcpu->kvm, sp->gfn);
- slot = gfn_to_memslot_unaliased(vcpu->kvm, sp->gfn);
+ gfn = sp->gfn;
+ slot = gfn_to_memslot(vcpu->kvm, gfn);
rmapp = &slot->rmap[gfn - slot->base_gfn];
spte = rmap_next(vcpu->kvm, rmapp, NULL);
@@ -2396,42 +2396,6 @@ static int kvm_vm_ioctl_get_nr_mmu_pages(struct kvm *kvm)
return kvm->arch.n_alloc_mmu_pages;
}
-gfn_t unalias_gfn_instantiation(struct kvm *kvm, gfn_t gfn)
-{
- int i;
- struct kvm_mem_alias *alias;
- struct kvm_mem_aliases *aliases;
-
- aliases = rcu_dereference(kvm->arch.aliases);
-
- for (i = 0; i < aliases->naliases; ++i) {
- alias = &aliases->aliases[i];
- if (alias->flags & KVM_ALIAS_INVALID)
- continue;
- if (gfn >= alias->base_gfn
- && gfn < alias->base_gfn + alias->npages)
- return alias->target_gfn + gfn - alias->base_gfn;
- }
- return gfn;
-}
-
-gfn_t unalias_gfn(struct kvm *kvm, gfn_t gfn)
-{
- int i;
- struct kvm_mem_alias *alias;
- struct kvm_mem_aliases *aliases;
-
- aliases = rcu_dereference(kvm->arch.aliases);
-
- for (i = 0; i < aliases->naliases; ++i) {
- alias = &aliases->aliases[i];
- if (gfn >= alias->base_gfn
- && gfn < alias->base_gfn + alias->npages)
- return alias->target_gfn + gfn - alias->base_gfn;
- }
- return gfn;
-}
-
/*
* Set a new alias region. Aliases map a portion of physical memory into
* another portion. This is useful for memory windows, for example the PC
@@ -2440,9 +2404,11 @@ gfn_t unalias_gfn(struct kvm *kvm, gfn_t gfn)
static int kvm_vm_ioctl_set_memory_alias(struct kvm *kvm,
struct kvm_memory_alias *alias)
{
- int r, n;
- struct kvm_mem_alias *p;
- struct kvm_mem_aliases *aliases, *old_aliases;
+ int r;
+ struct kvm_userspace_memory_region alias_mem;
+ struct kvm_memory_slot *slot;
+ unsigned long offset_addr;
+ gfn_t target_base_gfn;
r = -EINVAL;
/* General sanity checks */
@@ -2459,45 +2425,27 @@ static int kvm_vm_ioctl_set_memory_alias(struct kvm *kvm,
< alias->target_phys_addr)
goto out;
- r = -ENOMEM;
- aliases = kzalloc(sizeof(struct kvm_mem_aliases), GFP_KERNEL);
- if (!aliases)
- goto out;
-
mutex_lock(&kvm->slots_lock);
- /* invalidate any gfn reference in case of deletion/shrinking */
- memcpy(aliases, kvm->arch.aliases, sizeof(struct kvm_mem_aliases));
- aliases->aliases[alias->slot].flags |= KVM_ALIAS_INVALID;
- old_aliases = kvm->arch.aliases;
- rcu_assign_pointer(kvm->arch.aliases, aliases);
- synchronize_srcu_expedited(&kvm->srcu);
- kvm_mmu_zap_all(kvm);
- kfree(old_aliases);
-
- r = -ENOMEM;
- aliases = kzalloc(sizeof(struct kvm_mem_aliases), GFP_KERNEL);
- if (!aliases)
+ target_base_gfn = alias->target_phys_addr >> PAGE_SHIFT;
+ slot = gfn_to_memslot(kvm, target_base_gfn);
+ if (!slot)
goto out_unlock;
- memcpy(aliases, kvm->arch.aliases, sizeof(struct kvm_mem_aliases));
-
- p = &aliases->aliases[alias->slot];
- p->base_gfn = alias->guest_phys_addr >> PAGE_SHIFT;
- p->npages = alias->memory_size >> PAGE_SHIFT;
- p->target_gfn = alias->target_phys_addr >> PAGE_SHIFT;
- p->flags &= ~(KVM_ALIAS_INVALID);
+ if (slot->npages < alias->memory_size >> PAGE_SHIFT)
+ goto out_unlock;
- for (n = KVM_ALIAS_SLOTS; n > 0; --n)
- if (aliases->aliases[n - 1].npages)
- break;
- aliases->naliases = n;
+ alias_mem.slot = alias->slot + KVM_MEMORY_SLOTS;
+ alias_mem.guest_phys_addr = alias->guest_phys_addr;
+ alias_mem.memory_size = alias->memory_size;
+ offset_addr = target_base_gfn - slot->base_gfn;
+ offset_addr = offset_addr << PAGE_SHIFT;
+ alias_mem.userspace_addr = slot->userspace_addr + offset_addr;
+ alias_mem.flags = 0;
- old_aliases = kvm->arch.aliases;
- rcu_assign_pointer(kvm->arch.aliases, aliases);
- synchronize_srcu_expedited(&kvm->srcu);
- kfree(old_aliases);
- r = 0;
+ r = __kvm_set_memory_region(kvm, &alias_mem, 1);
+ if (!r)
+ kvm_mmu_zap_all(kvm);
out_unlock:
mutex_unlock(&kvm->slots_lock);
@@ -2631,6 +2579,8 @@ int kvm_vm_ioctl_get_dirty_log(struct kvm *kvm,
{
int r, n, i;
struct kvm_memory_slot *memslot;
+ struct kvm_memory_slot *alias_memslot;
+ unsigned long size;
unsigned long is_dirty = 0;
unsigned long *dirty_bitmap = NULL;
@@ -2661,7 +2611,18 @@ int kvm_vm_ioctl_get_dirty_log(struct kvm *kvm,
struct kvm_memslots *slots, *old_slots;
spin_lock(&kvm->mmu_lock);
+ for (i = KVM_MEMORY_SLOTS; i < KVM_MEMORY_SLOTS +
+ KVM_ALIAS_SLOTS; ++i) {
+ alias_memslot = &kvm->memslots->memslots[i];
+ size = memslot->npages << PAGE_SHIFT;
+ if (alias_memslot->userspace_addr >=
+ memslot->userspace_addr &&
+ alias_memslot->userspace_addr <
+ memslot->userspace_addr + size)
+ kvm_mmu_slot_remove_write_access(kvm, i);
+ }
kvm_mmu_slot_remove_write_access(kvm, log->slot);
+ kvm_flush_remote_tlbs(kvm);
spin_unlock(&kvm->mmu_lock);
slots = kzalloc(sizeof(struct kvm_memslots), GFP_KERNEL);
@@ -5484,12 +5445,6 @@ struct kvm *kvm_arch_create_vm(void)
if (!kvm)
return ERR_PTR(-ENOMEM);
- kvm->arch.aliases = kzalloc(sizeof(struct kvm_mem_aliases), GFP_KERNEL);
- if (!kvm->arch.aliases) {
- kfree(kvm);
- return ERR_PTR(-ENOMEM);
- }
-
INIT_LIST_HEAD(&kvm->arch.active_mmu_pages);
INIT_LIST_HEAD(&kvm->arch.assigned_dev_head);
@@ -5547,8 +5502,6 @@ void kvm_arch_destroy_vm(struct kvm *kvm)
if (kvm->arch.ept_identity_pagetable)
put_page(kvm->arch.ept_identity_pagetable);
cleanup_srcu_struct(&kvm->srcu);
- kfree(kvm->arch.aliases);
- kfree(kvm);
}
int kvm_arch_prepare_memory_region(struct kvm *kvm,
@@ -5613,6 +5566,7 @@ void kvm_arch_commit_memory_region(struct kvm *kvm,
}
kvm_mmu_slot_remove_write_access(kvm, mem->slot);
+ kvm_flush_remote_tlbs(kvm);
spin_unlock(&kvm->mmu_lock);
}
@@ -117,6 +117,8 @@ struct kvm_memory_slot {
} *lpage_info[KVM_NR_PAGE_SIZES - 1];
unsigned long userspace_addr;
int user_alloc;
+ gfn_t real_base_gfn;
+ unsigned long real_npages;
};
struct kvm_kernel_irq_routing_entry {
@@ -156,7 +158,8 @@ struct kvm_irq_routing_table {};
struct kvm_memslots {
int nmemslots;
struct kvm_memory_slot memslots[KVM_MEMORY_SLOTS +
- KVM_PRIVATE_MEM_SLOTS];
+ KVM_PRIVATE_MEM_SLOTS +
+ KVM_ALIAS_SLOTS];
};
struct kvm {
@@ -267,8 +270,6 @@ void kvm_arch_commit_memory_region(struct kvm *kvm,
int user_alloc);
void kvm_disable_largepages(void);
void kvm_arch_flush_shadow(struct kvm *kvm);
-gfn_t unalias_gfn(struct kvm *kvm, gfn_t gfn);
-gfn_t unalias_gfn_instantiation(struct kvm *kvm, gfn_t gfn);
struct page *gfn_to_page(struct kvm *kvm, gfn_t gfn);
unsigned long gfn_to_hva(struct kvm *kvm, gfn_t gfn);
@@ -543,10 +544,6 @@ static inline int mmu_notifier_retry(struct kvm_vcpu *vcpu, unsigned long mmu_se
}
#endif
-#ifndef KVM_ARCH_HAS_UNALIAS_INSTANTIATION
-#define unalias_gfn_instantiation unalias_gfn
-#endif
-
#ifdef CONFIG_HAVE_KVM_IRQCHIP
#define KVM_MAX_IRQ_ROUTES 1024
@@ -450,10 +450,51 @@ out_err_nodisable:
return ERR_PTR(r);
}
+#ifdef CONFIG_X86
+
+static void update_alias_slots(struct kvm *kvm, struct kvm_memory_slot *slot)
+{
+ int i;
+
+ for (i = KVM_MEMORY_SLOTS; i < KVM_MEMORY_SLOTS + KVM_ALIAS_SLOTS;
+ ++i) {
+ struct kvm_memory_slot *alias_memslot =
+ &kvm->memslots->memslots[i];
+ unsigned long size = slot->npages << PAGE_SHIFT;
+
+ if (alias_memslot->real_base_gfn >= slot->base_gfn &&
+ alias_memslot->real_base_gfn < slot->base_gfn + size) {
+ if (slot->dirty_bitmap) {
+ unsigned long bitmap_addr;
+ unsigned long dirty_offset;
+ unsigned long offset_addr =
+ (alias_memslot->real_base_gfn -
+ slot->base_gfn) << PAGE_SHIFT;
+ alias_memslot->userspace_addr =
+ slot->userspace_addr + offset_addr;
+
+ dirty_offset =
+ ALIGN(offset_addr, BITS_PER_LONG) / 8;
+ bitmap_addr = (unsigned long) slot->dirty_bitmap;
+ bitmap_addr += dirty_offset;
+ alias_memslot->dirty_bitmap = (unsigned long *)bitmap_addr;
+ alias_memslot->base_gfn = alias_memslot->real_base_gfn;
+ alias_memslot->npages = alias_memslot->real_npages;
+ } else if (!slot->rmap) {
+ alias_memslot->base_gfn = 0;
+ alias_memslot->npages = 0;
+ }
+ }
+ }
+}
+
+#endif
+
/*
* Free any memory in @free but not in @dont.
*/
-static void kvm_free_physmem_slot(struct kvm_memory_slot *free,
+static void kvm_free_physmem_slot(struct kvm *kvm,
+ struct kvm_memory_slot *free,
struct kvm_memory_slot *dont)
{
int i;
@@ -472,9 +513,13 @@ static void kvm_free_physmem_slot(struct kvm_memory_slot *free,
}
}
- free->npages = 0;
free->dirty_bitmap = NULL;
free->rmap = NULL;
+
+#ifdef CONFIG_X86
+ update_alias_slots(kvm, free);
+#endif
+ free->npages = 0;
}
void kvm_free_physmem(struct kvm *kvm)
@@ -483,7 +528,7 @@ void kvm_free_physmem(struct kvm *kvm)
struct kvm_memslots *slots = kvm->memslots;
for (i = 0; i < slots->nmemslots; ++i)
- kvm_free_physmem_slot(&slots->memslots[i], NULL);
+ kvm_free_physmem_slot(kvm, &slots->memslots[i], NULL);
kfree(kvm->memslots);
}
@@ -563,7 +608,8 @@ int __kvm_set_memory_region(struct kvm *kvm,
goto out;
if (user_alloc && (mem->userspace_addr & (PAGE_SIZE - 1)))
goto out;
- if (mem->slot >= KVM_MEMORY_SLOTS + KVM_PRIVATE_MEM_SLOTS)
+ if (mem->slot >= KVM_MEMORY_SLOTS + KVM_PRIVATE_MEM_SLOTS +
+ KVM_ALIAS_SLOTS)
goto out;
if (mem->guest_phys_addr + mem->memory_size < mem->guest_phys_addr)
goto out;
@@ -616,6 +662,8 @@ int __kvm_set_memory_region(struct kvm *kvm,
new.user_alloc = user_alloc;
new.userspace_addr = mem->userspace_addr;
+ new.real_base_gfn = new.base_gfn;
+ new.real_npages = new.npages;
}
if (!npages)
goto skip_lpage;
@@ -740,7 +788,7 @@ skip_lpage:
kvm_arch_commit_memory_region(kvm, mem, old, user_alloc);
- kvm_free_physmem_slot(&old, &new);
+ kvm_free_physmem_slot(kvm, &old, &new);
kfree(old_memslots);
if (flush_shadow)
@@ -749,7 +797,7 @@ skip_lpage:
return 0;
out_free:
- kvm_free_physmem_slot(&new, &old);
+ kvm_free_physmem_slot(kvm, &new, &old);
out:
return r;
@@ -842,7 +890,7 @@ int kvm_is_error_hva(unsigned long addr)
}
EXPORT_SYMBOL_GPL(kvm_is_error_hva);
-struct kvm_memory_slot *gfn_to_memslot_unaliased(struct kvm *kvm, gfn_t gfn)
+struct kvm_memory_slot *gfn_to_memslot(struct kvm *kvm, gfn_t gfn)
{
int i;
struct kvm_memslots *slots = rcu_dereference(kvm->memslots);
@@ -856,21 +904,14 @@ struct kvm_memory_slot *gfn_to_memslot_unaliased(struct kvm *kvm, gfn_t gfn)
}
return NULL;
}
-EXPORT_SYMBOL_GPL(gfn_to_memslot_unaliased);
-
-struct kvm_memory_slot *gfn_to_memslot(struct kvm *kvm, gfn_t gfn)
-{
- gfn = unalias_gfn(kvm, gfn);
- return gfn_to_memslot_unaliased(kvm, gfn);
-}
+EXPORT_SYMBOL_GPL(gfn_to_memslot);
int kvm_is_visible_gfn(struct kvm *kvm, gfn_t gfn)
{
int i;
struct kvm_memslots *slots = rcu_dereference(kvm->memslots);
- gfn = unalias_gfn_instantiation(kvm, gfn);
- for (i = 0; i < KVM_MEMORY_SLOTS; ++i) {
+ for (i = 0; i < KVM_MEMORY_SLOTS + KVM_ALIAS_SLOTS; ++i) {
struct kvm_memory_slot *memslot = &slots->memslots[i];
if (memslot->flags & KVM_MEMSLOT_INVALID)
@@ -890,7 +931,6 @@ int memslot_id(struct kvm *kvm, gfn_t gfn)
struct kvm_memslots *slots = rcu_dereference(kvm->memslots);
struct kvm_memory_slot *memslot = NULL;
- gfn = unalias_gfn(kvm, gfn);
for (i = 0; i < slots->nmemslots; ++i) {
memslot = &slots->memslots[i];
@@ -906,8 +946,7 @@ unsigned long gfn_to_hva(struct kvm *kvm, gfn_t gfn)
{
struct kvm_memory_slot *slot;
- gfn = unalias_gfn_instantiation(kvm, gfn);
- slot = gfn_to_memslot_unaliased(kvm, gfn);
+ slot = gfn_to_memslot(kvm, gfn);
if (!slot || slot->flags & KVM_MEMSLOT_INVALID)
return bad_hva();
return (slot->userspace_addr + (gfn - slot->base_gfn) * PAGE_SIZE);
@@ -1174,8 +1213,7 @@ void mark_page_dirty(struct kvm *kvm, gfn_t gfn)
{
struct kvm_memory_slot *memslot;
- gfn = unalias_gfn(kvm, gfn);
- memslot = gfn_to_memslot_unaliased(kvm, gfn);
+ memslot = gfn_to_memslot(kvm, gfn);
if (memslot && memslot->dirty_bitmap) {
unsigned long rel_gfn = gfn - memslot->base_gfn;