[v4] KVM: LAPIC: Tune lapic_timer_advance_ns smoothly
diff mbox series

Message ID 1568623199-5294-1-git-send-email-wanpengli@tencent.com
State New
Headers show
Series
  • [v4] KVM: LAPIC: Tune lapic_timer_advance_ns smoothly
Related show

Commit Message

Wanpeng Li Sept. 16, 2019, 8:39 a.m. UTC
From: Wanpeng Li <wanpengli@tencent.com>

Filter out drastic fluctuation and random fluctuation, remove
timer_advance_adjust_done altogether, the adjustment would be 
continuous.

Signed-off-by: Wanpeng Li <wanpengli@tencent.com>
---
 arch/x86/kvm/lapic.c | 26 ++++++++++++--------------
 arch/x86/kvm/lapic.h |  1 -
 arch/x86/kvm/x86.c   |  2 +-
 arch/x86/kvm/x86.h   |  1 +
 4 files changed, 14 insertions(+), 16 deletions(-)

Comments

Sean Christopherson Sept. 16, 2019, 6:55 p.m. UTC | #1
On Mon, Sep 16, 2019 at 04:39:59PM +0800, Wanpeng Li wrote:
> From: Wanpeng Li <wanpengli@tencent.com>
> 
> Filter out drastic fluctuation and random fluctuation, remove
> timer_advance_adjust_done altogether, the adjustment would be 
> continuous.
> 
> Signed-off-by: Wanpeng Li <wanpengli@tencent.com>
> ---
>  arch/x86/kvm/lapic.c | 26 ++++++++++++--------------
>  arch/x86/kvm/lapic.h |  1 -
>  arch/x86/kvm/x86.c   |  2 +-
>  arch/x86/kvm/x86.h   |  1 +
>  4 files changed, 14 insertions(+), 16 deletions(-)
> 
> diff --git a/arch/x86/kvm/lapic.c b/arch/x86/kvm/lapic.c
> index dbbe478..2585b86 100644
> --- a/arch/x86/kvm/lapic.c
> +++ b/arch/x86/kvm/lapic.c
> @@ -69,6 +69,7 @@
>  #define LAPIC_TIMER_ADVANCE_ADJUST_INIT 1000
>  /* step-by-step approximation to mitigate fluctuation */
>  #define LAPIC_TIMER_ADVANCE_ADJUST_STEP 8
> +#define LAPIC_TIMER_ADVANCE_FILTER 5000
>  
>  static inline int apic_test_vector(int vec, void *bitmap)
>  {
> @@ -1482,29 +1483,28 @@ static inline void adjust_lapic_timer_advance(struct kvm_vcpu *vcpu,
>  					      s64 advance_expire_delta)
>  {
>  	struct kvm_lapic *apic = vcpu->arch.apic;
> -	u32 timer_advance_ns = apic->lapic_timer.timer_advance_ns;
> -	u64 ns;
> +	u32 timer_advance_ns = apic->lapic_timer.timer_advance_ns, ns;

Is changing 'ns' to a u32 intentionaly?  It's still cast to a u32 in the
calculations, and set from @advance_expire_delta.

> +
> +	if (abs(advance_expire_delta) > LAPIC_TIMER_ADVANCE_FILTER ||

Shouldn't LAPIC_TIMER_ADVANCE_FILTER be used in the other "if ... > 5000"
check?
	
	if (unlikely(timer_advance_ns > 5000))

And maybe name it LAPIC_TIMER_ADVANCE_MAX or something?

> +		abs(advance_expire_delta) < LAPIC_TIMER_ADVANCE_ADJUST_DONE) {

This should be aligned with the other 'abs', e.g.:

	if (abs(...) ||
	    abs(...))
		return

> +		/* filter out random fluctuations */

If you put the comment above the if statement then you can drop the
parentheses.  And if you're going to bother with a comment, maybe be a bit
more explicit?  E.g.:

	/* Do not adjust for tiny fluctuations or large random spikes. */
	if (abs(...) ||
	    abs(...))
		return;

> +		return;
> +	}
>  
>  	/* too early */
>  	if (advance_expire_delta < 0) {
>  		ns = -advance_expire_delta * 1000000ULL;
>  		do_div(ns, vcpu->arch.virtual_tsc_khz);
> -		timer_advance_ns -= min((u32)ns,
> -			timer_advance_ns / LAPIC_TIMER_ADVANCE_ADJUST_STEP);
> +		timer_advance_ns -= ns/LAPIC_TIMER_ADVANCE_ADJUST_STEP;
>  	} else {
>  	/* too late */
>  		ns = advance_expire_delta * 1000000ULL;
>  		do_div(ns, vcpu->arch.virtual_tsc_khz);
> -		timer_advance_ns += min((u32)ns,
> -			timer_advance_ns / LAPIC_TIMER_ADVANCE_ADJUST_STEP);
> +		timer_advance_ns += ns/LAPIC_TIMER_ADVANCE_ADJUST_STEP;
>  	}
>  
> -	if (abs(advance_expire_delta) < LAPIC_TIMER_ADVANCE_ADJUST_DONE)
> -		apic->lapic_timer.timer_advance_adjust_done = true;
> -	if (unlikely(timer_advance_ns > 5000)) {
> +	if (unlikely(timer_advance_ns > 5000))
>  		timer_advance_ns = LAPIC_TIMER_ADVANCE_ADJUST_INIT;
> -		apic->lapic_timer.timer_advance_adjust_done = false;
> -	}
>  	apic->lapic_timer.timer_advance_ns = timer_advance_ns;
>  }
>  
> @@ -1524,7 +1524,7 @@ static void __kvm_wait_lapic_expire(struct kvm_vcpu *vcpu)
>  	if (guest_tsc < tsc_deadline)
>  		__wait_lapic_expire(vcpu, tsc_deadline - guest_tsc);
>  
> -	if (unlikely(!apic->lapic_timer.timer_advance_adjust_done))
> +	if (lapic_timer_advance_ns == -1)

Rather than expose 'lapic_timer_advance_ns' from x86.c, what if we add a
'static bool dynamically_adjust_timer_advance __read_mostly;' in lapic.c,
and have that be set in kvm_create_lapic() and checked here?  That'd make
this code a little more readable, would make this patch more obvious (it
wasn't immediately clear why lapic_timer_advance_ns was being exposed),
and would reduce the probability of unintentionally writing/corrupting the
module param.

>  		adjust_lapic_timer_advance(vcpu, apic->lapic_timer.advance_expire_delta);
>  }
>  
> @@ -2302,10 +2302,8 @@ int kvm_create_lapic(struct kvm_vcpu *vcpu, int timer_advance_ns)
>  	apic->lapic_timer.timer.function = apic_timer_fn;
>  	if (timer_advance_ns == -1) {
>  		apic->lapic_timer.timer_advance_ns = LAPIC_TIMER_ADVANCE_ADJUST_INIT;
> -		apic->lapic_timer.timer_advance_adjust_done = false;
>  	} else {
>  		apic->lapic_timer.timer_advance_ns = timer_advance_ns;
> -		apic->lapic_timer.timer_advance_adjust_done = true;
>  	}

Parentheses can be dropped (unless this is converted to a local global, as
above).

> diff --git a/arch/x86/kvm/lapic.h b/arch/x86/kvm/lapic.h
> index 50053d2..2aad7e2 100644
> --- a/arch/x86/kvm/lapic.h
> +++ b/arch/x86/kvm/lapic.h
> @@ -35,7 +35,6 @@ struct kvm_timer {
>  	s64 advance_expire_delta;
>  	atomic_t pending;			/* accumulated triggered timers */
>  	bool hv_timer_in_use;
> -	bool timer_advance_adjust_done;
>  };
>  
>  struct kvm_lapic {
> diff --git a/arch/x86/kvm/x86.c b/arch/x86/kvm/x86.c
> index 83288ba..bb4574c 100644
> --- a/arch/x86/kvm/x86.c
> +++ b/arch/x86/kvm/x86.c
> @@ -141,7 +141,7 @@ module_param(tsc_tolerance_ppm, uint, S_IRUGO | S_IWUSR);
>   * advancement entirely.  Any other value is used as-is and disables adaptive
>   * tuning, i.e. allows priveleged userspace to set an exact advancement time.
>   */
> -static int __read_mostly lapic_timer_advance_ns = -1;
> +int __read_mostly lapic_timer_advance_ns = -1;
>  module_param(lapic_timer_advance_ns, int, S_IRUGO | S_IWUSR);
>  
>  static bool __read_mostly vector_hashing = true;
> diff --git a/arch/x86/kvm/x86.h b/arch/x86/kvm/x86.h
> index b5274e2..1bf2d82 100644
> --- a/arch/x86/kvm/x86.h
> +++ b/arch/x86/kvm/x86.h
> @@ -301,6 +301,7 @@ extern unsigned int min_timer_period_us;
>  
>  extern bool enable_vmware_backdoor;
>  
> +extern int lapic_timer_advance_ns;
>  extern int pi_inject_timer;
>  
>  extern struct static_key kvm_no_apic_vcpu;
> -- 
> 2.7.4
>
Wanpeng Li Sept. 17, 2019, 8:20 a.m. UTC | #2
On Tue, 17 Sep 2019 at 02:55, Sean Christopherson
<sean.j.christopherson@intel.com> wrote:
>
> On Mon, Sep 16, 2019 at 04:39:59PM +0800, Wanpeng Li wrote:
> > From: Wanpeng Li <wanpengli@tencent.com>
> >
> > Filter out drastic fluctuation and random fluctuation, remove
> > timer_advance_adjust_done altogether, the adjustment would be
> > continuous.
> >
> > Signed-off-by: Wanpeng Li <wanpengli@tencent.com>
> > ---
> >  arch/x86/kvm/lapic.c | 26 ++++++++++++--------------
> >  arch/x86/kvm/lapic.h |  1 -
> >  arch/x86/kvm/x86.c   |  2 +-
> >  arch/x86/kvm/x86.h   |  1 +
> >  4 files changed, 14 insertions(+), 16 deletions(-)
> >
> > diff --git a/arch/x86/kvm/lapic.c b/arch/x86/kvm/lapic.c
> > index dbbe478..2585b86 100644
> > --- a/arch/x86/kvm/lapic.c
> > +++ b/arch/x86/kvm/lapic.c
> > @@ -69,6 +69,7 @@
> >  #define LAPIC_TIMER_ADVANCE_ADJUST_INIT 1000
> >  /* step-by-step approximation to mitigate fluctuation */
> >  #define LAPIC_TIMER_ADVANCE_ADJUST_STEP 8
> > +#define LAPIC_TIMER_ADVANCE_FILTER 5000
> >
> >  static inline int apic_test_vector(int vec, void *bitmap)
> >  {
> > @@ -1482,29 +1483,28 @@ static inline void adjust_lapic_timer_advance(struct kvm_vcpu *vcpu,
> >                                             s64 advance_expire_delta)
> >  {
> >       struct kvm_lapic *apic = vcpu->arch.apic;
> > -     u32 timer_advance_ns = apic->lapic_timer.timer_advance_ns;
> > -     u64 ns;
> > +     u32 timer_advance_ns = apic->lapic_timer.timer_advance_ns, ns;
>
> Is changing 'ns' to a u32 intentionaly?  It's still cast to a u32 in the
> calculations, and set from @advance_expire_delta.
>
> > +
> > +     if (abs(advance_expire_delta) > LAPIC_TIMER_ADVANCE_FILTER ||
>
> Shouldn't LAPIC_TIMER_ADVANCE_FILTER be used in the other "if ... > 5000"
> check?
>
>         if (unlikely(timer_advance_ns > 5000))
>
> And maybe name it LAPIC_TIMER_ADVANCE_MAX or something?
>
> > +             abs(advance_expire_delta) < LAPIC_TIMER_ADVANCE_ADJUST_DONE) {
>
> This should be aligned with the other 'abs', e.g.:
>
>         if (abs(...) ||
>             abs(...))
>                 return
>
> > +             /* filter out random fluctuations */
>
> If you put the comment above the if statement then you can drop the
> parentheses.  And if you're going to bother with a comment, maybe be a bit
> more explicit?  E.g.:
>
>         /* Do not adjust for tiny fluctuations or large random spikes. */
>         if (abs(...) ||
>             abs(...))
>                 return;
>
> > +             return;
> > +     }
> >
> >       /* too early */
> >       if (advance_expire_delta < 0) {
> >               ns = -advance_expire_delta * 1000000ULL;
> >               do_div(ns, vcpu->arch.virtual_tsc_khz);
> > -             timer_advance_ns -= min((u32)ns,
> > -                     timer_advance_ns / LAPIC_TIMER_ADVANCE_ADJUST_STEP);
> > +             timer_advance_ns -= ns/LAPIC_TIMER_ADVANCE_ADJUST_STEP;
> >       } else {
> >       /* too late */
> >               ns = advance_expire_delta * 1000000ULL;
> >               do_div(ns, vcpu->arch.virtual_tsc_khz);
> > -             timer_advance_ns += min((u32)ns,
> > -                     timer_advance_ns / LAPIC_TIMER_ADVANCE_ADJUST_STEP);
> > +             timer_advance_ns += ns/LAPIC_TIMER_ADVANCE_ADJUST_STEP;
> >       }
> >
> > -     if (abs(advance_expire_delta) < LAPIC_TIMER_ADVANCE_ADJUST_DONE)
> > -             apic->lapic_timer.timer_advance_adjust_done = true;
> > -     if (unlikely(timer_advance_ns > 5000)) {
> > +     if (unlikely(timer_advance_ns > 5000))
> >               timer_advance_ns = LAPIC_TIMER_ADVANCE_ADJUST_INIT;
> > -             apic->lapic_timer.timer_advance_adjust_done = false;
> > -     }
> >       apic->lapic_timer.timer_advance_ns = timer_advance_ns;
> >  }
> >
> > @@ -1524,7 +1524,7 @@ static void __kvm_wait_lapic_expire(struct kvm_vcpu *vcpu)
> >       if (guest_tsc < tsc_deadline)
> >               __wait_lapic_expire(vcpu, tsc_deadline - guest_tsc);
> >
> > -     if (unlikely(!apic->lapic_timer.timer_advance_adjust_done))
> > +     if (lapic_timer_advance_ns == -1)
>
> Rather than expose 'lapic_timer_advance_ns' from x86.c, what if we add a
> 'static bool dynamically_adjust_timer_advance __read_mostly;' in lapic.c,
> and have that be set in kvm_create_lapic() and checked here?  That'd make
> this code a little more readable, would make this patch more obvious (it
> wasn't immediately clear why lapic_timer_advance_ns was being exposed),
> and would reduce the probability of unintentionally writing/corrupting the
> module param.
>
> >               adjust_lapic_timer_advance(vcpu, apic->lapic_timer.advance_expire_delta);
> >  }
> >
> > @@ -2302,10 +2302,8 @@ int kvm_create_lapic(struct kvm_vcpu *vcpu, int timer_advance_ns)
> >       apic->lapic_timer.timer.function = apic_timer_fn;
> >       if (timer_advance_ns == -1) {
> >               apic->lapic_timer.timer_advance_ns = LAPIC_TIMER_ADVANCE_ADJUST_INIT;
> > -             apic->lapic_timer.timer_advance_adjust_done = false;
> >       } else {
> >               apic->lapic_timer.timer_advance_ns = timer_advance_ns;
> > -             apic->lapic_timer.timer_advance_adjust_done = true;
> >       }
>
> Parentheses can be dropped (unless this is converted to a local global, as
> above).

I just handle all the comments in the new version.

    Wanpeng

Patch
diff mbox series

diff --git a/arch/x86/kvm/lapic.c b/arch/x86/kvm/lapic.c
index dbbe478..2585b86 100644
--- a/arch/x86/kvm/lapic.c
+++ b/arch/x86/kvm/lapic.c
@@ -69,6 +69,7 @@ 
 #define LAPIC_TIMER_ADVANCE_ADJUST_INIT 1000
 /* step-by-step approximation to mitigate fluctuation */
 #define LAPIC_TIMER_ADVANCE_ADJUST_STEP 8
+#define LAPIC_TIMER_ADVANCE_FILTER 5000
 
 static inline int apic_test_vector(int vec, void *bitmap)
 {
@@ -1482,29 +1483,28 @@  static inline void adjust_lapic_timer_advance(struct kvm_vcpu *vcpu,
 					      s64 advance_expire_delta)
 {
 	struct kvm_lapic *apic = vcpu->arch.apic;
-	u32 timer_advance_ns = apic->lapic_timer.timer_advance_ns;
-	u64 ns;
+	u32 timer_advance_ns = apic->lapic_timer.timer_advance_ns, ns;
+
+	if (abs(advance_expire_delta) > LAPIC_TIMER_ADVANCE_FILTER ||
+		abs(advance_expire_delta) < LAPIC_TIMER_ADVANCE_ADJUST_DONE) {
+		/* filter out random fluctuations */
+		return;
+	}
 
 	/* too early */
 	if (advance_expire_delta < 0) {
 		ns = -advance_expire_delta * 1000000ULL;
 		do_div(ns, vcpu->arch.virtual_tsc_khz);
-		timer_advance_ns -= min((u32)ns,
-			timer_advance_ns / LAPIC_TIMER_ADVANCE_ADJUST_STEP);
+		timer_advance_ns -= ns/LAPIC_TIMER_ADVANCE_ADJUST_STEP;
 	} else {
 	/* too late */
 		ns = advance_expire_delta * 1000000ULL;
 		do_div(ns, vcpu->arch.virtual_tsc_khz);
-		timer_advance_ns += min((u32)ns,
-			timer_advance_ns / LAPIC_TIMER_ADVANCE_ADJUST_STEP);
+		timer_advance_ns += ns/LAPIC_TIMER_ADVANCE_ADJUST_STEP;
 	}
 
-	if (abs(advance_expire_delta) < LAPIC_TIMER_ADVANCE_ADJUST_DONE)
-		apic->lapic_timer.timer_advance_adjust_done = true;
-	if (unlikely(timer_advance_ns > 5000)) {
+	if (unlikely(timer_advance_ns > 5000))
 		timer_advance_ns = LAPIC_TIMER_ADVANCE_ADJUST_INIT;
-		apic->lapic_timer.timer_advance_adjust_done = false;
-	}
 	apic->lapic_timer.timer_advance_ns = timer_advance_ns;
 }
 
@@ -1524,7 +1524,7 @@  static void __kvm_wait_lapic_expire(struct kvm_vcpu *vcpu)
 	if (guest_tsc < tsc_deadline)
 		__wait_lapic_expire(vcpu, tsc_deadline - guest_tsc);
 
-	if (unlikely(!apic->lapic_timer.timer_advance_adjust_done))
+	if (lapic_timer_advance_ns == -1)
 		adjust_lapic_timer_advance(vcpu, apic->lapic_timer.advance_expire_delta);
 }
 
@@ -2302,10 +2302,8 @@  int kvm_create_lapic(struct kvm_vcpu *vcpu, int timer_advance_ns)
 	apic->lapic_timer.timer.function = apic_timer_fn;
 	if (timer_advance_ns == -1) {
 		apic->lapic_timer.timer_advance_ns = LAPIC_TIMER_ADVANCE_ADJUST_INIT;
-		apic->lapic_timer.timer_advance_adjust_done = false;
 	} else {
 		apic->lapic_timer.timer_advance_ns = timer_advance_ns;
-		apic->lapic_timer.timer_advance_adjust_done = true;
 	}
 
 
diff --git a/arch/x86/kvm/lapic.h b/arch/x86/kvm/lapic.h
index 50053d2..2aad7e2 100644
--- a/arch/x86/kvm/lapic.h
+++ b/arch/x86/kvm/lapic.h
@@ -35,7 +35,6 @@  struct kvm_timer {
 	s64 advance_expire_delta;
 	atomic_t pending;			/* accumulated triggered timers */
 	bool hv_timer_in_use;
-	bool timer_advance_adjust_done;
 };
 
 struct kvm_lapic {
diff --git a/arch/x86/kvm/x86.c b/arch/x86/kvm/x86.c
index 83288ba..bb4574c 100644
--- a/arch/x86/kvm/x86.c
+++ b/arch/x86/kvm/x86.c
@@ -141,7 +141,7 @@  module_param(tsc_tolerance_ppm, uint, S_IRUGO | S_IWUSR);
  * advancement entirely.  Any other value is used as-is and disables adaptive
  * tuning, i.e. allows priveleged userspace to set an exact advancement time.
  */
-static int __read_mostly lapic_timer_advance_ns = -1;
+int __read_mostly lapic_timer_advance_ns = -1;
 module_param(lapic_timer_advance_ns, int, S_IRUGO | S_IWUSR);
 
 static bool __read_mostly vector_hashing = true;
diff --git a/arch/x86/kvm/x86.h b/arch/x86/kvm/x86.h
index b5274e2..1bf2d82 100644
--- a/arch/x86/kvm/x86.h
+++ b/arch/x86/kvm/x86.h
@@ -301,6 +301,7 @@  extern unsigned int min_timer_period_us;
 
 extern bool enable_vmware_backdoor;
 
+extern int lapic_timer_advance_ns;
 extern int pi_inject_timer;
 
 extern struct static_key kvm_no_apic_vcpu;