@@ -215,6 +215,42 @@ static void sev_asid_free(struct kvm_sev_info *sev)
sev->misc_cg = NULL;
}
+static struct cpumask *sev_get_wbinvd_dirty_mask(struct kvm *kvm)
+{
+ struct kvm_sev_info *sev = &to_kvm_svm(kvm)->sev_info;
+
+ return sev->wbinvd_dirty_mask;
+}
+
+void sev_vcpu_load(struct kvm_vcpu *vcpu, int cpu)
+{
+ /*
+ * The per-VM wbinvd_dirty_mask should record all physical CPUs
+ * that are running a SEV guest and be used in memory reclamation.
+ *
+ * Migrating vCPUs between pCPUs is tricky. We cannot clear
+ * this mask each time reclamation finishes and record it again
+ * before VMRUN, because we cannot guarantee the pCPU will exit
+ * to VMM before the next reclamation happens.
+ *
+ * Thus we just keep stale pCPU numbers in the mask if vCPU
+ * migration happens.
+ */
+ cpumask_set_cpu(cpu, sev_get_wbinvd_dirty_mask(vcpu->kvm));
+}
+
+static void sev_do_wbinvd(struct kvm *kvm)
+{
+ struct cpumask *dirty_mask = sev_get_wbinvd_dirty_mask(kvm);
+
+ /*
+ * Although dirty_mask is not maintained perfectly and may lead
+ * to wbinvd on physical CPUs that are not running a SEV guest,
+ * it's still better than wbinvd_on_all_cpus().
+ */
+ wbinvd_on_many_cpus(dirty_mask);
+}
+
static void sev_decommission(unsigned int handle)
{
struct sev_data_decommission decommission;
@@ -265,6 +301,9 @@ static int sev_guest_init(struct kvm *kvm, struct kvm_sev_cmd *argp)
ret = sev_platform_init(&argp->error);
if (ret)
goto e_free;
+ if (!zalloc_cpumask_var(&sev->wbinvd_dirty_mask, GFP_KERNEL_ACCOUNT))
+ goto e_free;
+
INIT_LIST_HEAD(&sev->regions_list);
INIT_LIST_HEAD(&sev->mirror_vms);
@@ -2048,7 +2087,7 @@ int sev_mem_enc_unregister_region(struct kvm *kvm,
* releasing the pages back to the system for use. CLFLUSH will
* not do this, so issue a WBINVD.
*/
- wbinvd_on_all_cpus();
+ sev_do_wbinvd(kvm);
__unregister_enc_region_locked(kvm, region);
@@ -2152,7 +2191,7 @@ void sev_vm_destroy(struct kvm *kvm)
* releasing the pages back to the system for use. CLFLUSH will
* not do this, so issue a WBINVD.
*/
- wbinvd_on_all_cpus();
+ sev_do_wbinvd(kvm);
/*
* if userspace was terminated before unregistering the memory regions
@@ -2168,6 +2207,7 @@ void sev_vm_destroy(struct kvm *kvm)
sev_unbind_asid(kvm, sev->handle);
sev_asid_free(sev);
+ free_cpumask_var(sev->wbinvd_dirty_mask);
}
void __init sev_set_cpu_caps(void)
@@ -2343,7 +2383,7 @@ static void sev_flush_encrypted_page(struct kvm_vcpu *vcpu, void *va)
return;
do_wbinvd:
- wbinvd_on_all_cpus();
+ sev_do_wbinvd(vcpu->kvm);
}
void sev_guest_memory_reclaimed(struct kvm *kvm)
@@ -2351,7 +2391,7 @@ void sev_guest_memory_reclaimed(struct kvm *kvm)
if (!sev_guest(kvm))
return;
- wbinvd_on_all_cpus();
+ sev_do_wbinvd(kvm);
}
void sev_free_vcpu(struct kvm_vcpu *vcpu)
@@ -1560,6 +1560,8 @@ static void svm_vcpu_load(struct kvm_vcpu *vcpu, int cpu)
}
if (kvm_vcpu_apicv_active(vcpu))
avic_vcpu_load(vcpu, cpu);
+ if (sev_guest(vcpu->kvm))
+ sev_vcpu_load(vcpu, cpu);
}
static void svm_vcpu_put(struct kvm_vcpu *vcpu)
@@ -90,6 +90,9 @@ struct kvm_sev_info {
struct list_head mirror_entry; /* Use as a list entry of mirrors */
struct misc_cg *misc_cg; /* For misc cgroup accounting */
atomic_t migration_in_progress;
+
+ /* CPUs invoked VMRUN should do wbinvd after guest memory is reclaimed */
+ struct cpumask *wbinvd_dirty_mask;
};
struct kvm_svm {
@@ -694,6 +697,7 @@ void sev_es_vcpu_reset(struct vcpu_svm *svm);
void sev_vcpu_deliver_sipi_vector(struct kvm_vcpu *vcpu, u8 vector);
void sev_es_prepare_switch_to_guest(struct sev_es_save_area *hostsa);
void sev_es_unmap_ghcb(struct vcpu_svm *svm);
+void sev_vcpu_load(struct kvm_vcpu *vcpu, int cpu);
/* vmenter.S */
On AMD CPUs without ensuring cache consistency, each memory page reclamation in an SEV guest triggers a call to wbinvd_on_all_cpus(), thereby affecting the performance of other programs on the host. Typically, an AMD server may have 128 cores or more, while the SEV guest might only utilize 8 of these cores. Meanwhile, host can use qemu-affinity to bind these 8 vCPUs to specific physical CPUs. Therefore, keeping a record of the physical core numbers each time a vCPU runs can help avoid flushing the cache for all CPUs every time. Since the usage of sev_flush_asids() isn't tied to a single VM, we just replace all wbinvd_on_all_cpus() with sev_do_wbinvd() except for that in sev_flush_asids(). Signed-off-by: Zheyun Shen <szy0127@sjtu.edu.cn> --- arch/x86/kvm/svm/sev.c | 48 ++++++++++++++++++++++++++++++++++++++---- arch/x86/kvm/svm/svm.c | 2 ++ arch/x86/kvm/svm/svm.h | 4 ++++ 3 files changed, 50 insertions(+), 4 deletions(-)