@@ -18,6 +18,7 @@
*
* Copyright (c) 2019 Microsoft.
*/
+#include <xen/cpumask.h>
#include <xen/init.h>
#include <xen/types.h>
@@ -64,6 +65,14 @@ void hypervisor_resume(void)
ops->resume();
}
+int hypervisor_flush_tlb(const cpumask_t *mask, const void *va)
+{
+ if ( ops && ops->flush_tlb )
+ return ops->flush_tlb(mask, va);
+
+ return -ENOSYS;
+}
+
/*
* Local variables:
* mode: C
@@ -310,11 +310,17 @@ static void resume(void)
pv_console_init();
}
+static int flush_tlb(const cpumask_t *mask, const void *va)
+{
+ return xen_hypercall_hvm_op(HVMOP_flush_tlbs, NULL);
+}
+
static const struct hypervisor_ops ops = {
.name = "Xen",
.setup = setup,
.ap_setup = ap_setup,
.resume = resume,
+ .flush_tlb = flush_tlb,
};
const struct hypervisor_ops *__init xg_probe(void)
@@ -15,6 +15,7 @@
#include <xen/perfc.h>
#include <xen/spinlock.h>
#include <asm/current.h>
+#include <asm/guest.h>
#include <asm/smp.h>
#include <asm/mc146818rtc.h>
#include <asm/flushtlb.h>
@@ -262,6 +263,11 @@ void flush_area_mask(const cpumask_t *mask, const void *va, unsigned int flags)
if ( (flags & ~FLUSH_ORDER_MASK) &&
!cpumask_subset(mask, cpumask_of(cpu)) )
{
+ if ( cpu_has_hypervisor &&
+ !(flags & ~(FLUSH_TLB | FLUSH_TLB_GLOBAL | FLUSH_VA_VALID)) &&
+ !hypervisor_flush_tlb(mask, va) )
+ return;
+
spin_lock(&flush_lock);
cpumask_and(&flush_cpumask, mask, &cpu_online_map);
cpumask_clear_cpu(cpu, &flush_cpumask);
@@ -28,6 +28,8 @@ struct hypervisor_ops {
void (*ap_setup)(void);
/* Resume from suspension */
void (*resume)(void);
+ /* L0 assisted TLB flush */
+ int (*flush_tlb)(const cpumask_t *mask, const void *va);
};
#ifdef CONFIG_GUEST
@@ -36,9 +38,16 @@ const char *hypervisor_probe(void);
void hypervisor_setup(void);
void hypervisor_ap_setup(void);
void hypervisor_resume(void);
+/*
+ * L0 assisted TLB flush.
+ * mask: cpumask of the dirty vCPUs that should be flushed.
+ * va: linear address to flush, or NULL for global flushes.
+ */
+int hypervisor_flush_tlb(const cpumask_t *mask, const void *va);
#else
+#include <xen/cpumask.h>
#include <xen/lib.h>
#include <xen/types.h>
@@ -46,6 +55,10 @@ static inline const char *hypervisor_probe(void) { return NULL; }
static inline void hypervisor_setup(void) { ASSERT_UNREACHABLE(); }
static inline void hypervisor_ap_setup(void) { ASSERT_UNREACHABLE(); }
static inline void hypervisor_resume(void) { ASSERT_UNREACHABLE(); }
+static inline int hypervisor_flush_tlb(const cpumask_t *mask, const void *va)
+{
+ return -ENOSYS;
+}
#endif /* CONFIG_GUEST */
Use Xen's L0 HVMOP_flush_tlbs hypercall in order to perform flushes. This greatly increases the performance of TLB flushes when running with a high amount of vCPUs as a Xen guest, and is specially important when running in shim mode. The following figures are from a PV guest running `make -j32 xen` in shim mode with 32 vCPUs. Using x2APIC and ALLBUT shorthand: real 4m35.973s user 4m35.110s sys 36m24.117s Using L0 assisted flush: real 1m14.206s user 4m31.835s sys 5m45.013s The implementation adds a new hook to hypervisor_ops so other enlightenments can also implement such assisted flush just by filling the hook. Note that the Xen implementation completely ignores the dirty CPU mask and the linear address passed in, and always performs a global TLB flush on all vCPUs. Signed-off-by: Roger Pau Monné <roger.pau@citrix.com> --- Changes since v1: - Add a L0 assisted hook to hypervisor ops. --- xen/arch/x86/guest/hypervisor.c | 9 +++++++++ xen/arch/x86/guest/xen/xen.c | 6 ++++++ xen/arch/x86/smp.c | 6 ++++++ xen/include/asm-x86/guest/hypervisor.h | 13 +++++++++++++ 4 files changed, 34 insertions(+)