| Message ID | 1565886293-115836-12-git-send-email-suravee.suthikulpanit@amd.com (mailing list archive) |
|---|---|
| State | New, archived |
| Headers | show |
| Series | kvm: x86: Support AMD SVM AVIC w/ in-kernel irqchip mode | expand |
On 15.08.19 18:25, Suthikulpanit, Suravee wrote: > AMD AVIC does not support ExtINT. Therefore, AVIC must be temporary > deactivated and fall back to using legacy interrupt injection via vINTR > and interrupt window. > > Signed-off-by: Suravee Suthikulpanit <suravee.suthikulpanit@amd.com> > --- > arch/x86/kvm/svm.c | 49 +++++++++++++++++++++++++++++++++++++++++++++---- > 1 file changed, 45 insertions(+), 4 deletions(-) > > diff --git a/arch/x86/kvm/svm.c b/arch/x86/kvm/svm.c > index cfa4b13..4690351 100644 > --- a/arch/x86/kvm/svm.c > +++ b/arch/x86/kvm/svm.c > @@ -384,6 +384,7 @@ struct amd_svm_iommu_ir { > static void svm_set_cr0(struct kvm_vcpu *vcpu, unsigned long cr0); > static void svm_flush_tlb(struct kvm_vcpu *vcpu, bool invalidate_gpa); > static void svm_complete_interrupts(struct vcpu_svm *svm); > +static void svm_request_activate_avic(struct kvm_vcpu *vcpu); > static bool svm_get_enable_apicv(struct kvm *kvm); > static inline void avic_post_state_restore(struct kvm_vcpu *vcpu); > > @@ -4494,6 +4495,15 @@ static int interrupt_window_interception(struct vcpu_svm *svm) > { > kvm_make_request(KVM_REQ_EVENT, &svm->vcpu); > svm_clear_vintr(svm); > + > + /* > + * For AVIC, the only reason to end up here is ExtINTs. > + * In this case AVIC was temporarily disabled for > + * requesting the IRQ window and we have to re-enable it. > + */ > + if (svm_get_enable_apicv(svm->vcpu.kvm)) > + svm_request_activate_avic(&svm->vcpu); Would it make sense to add a trace point here and to the other call sites, so that it becomes obvious in a trace when and why exactly avic was active/inactive? The trace point could add additional information on the why. > + > svm->vmcb->control.int_ctl &= ~V_IRQ_MASK; > mark_dirty(svm->vmcb, VMCB_INTR); > ++svm->vcpu.stat.irq_window_exits; > @@ -5181,7 +5191,33 @@ static void svm_hwapic_isr_update(struct kvm_vcpu *vcpu, int max_isr) > { > } > > -/* Note: Currently only used by Hyper-V. */ > +static bool is_avic_active(struct vcpu_svm *svm) > +{ > + return (svm_get_enable_apicv(svm->vcpu.kvm) && > + svm->vmcb->control.int_ctl & AVIC_ENABLE_MASK); > +} > + > +static void svm_request_activate_avic(struct kvm_vcpu *vcpu) > +{ > + struct vcpu_svm *svm = to_svm(vcpu); > + > + if (!lapic_in_kernel(vcpu) || is_avic_active(svm)) > + return; > + > + kvm_make_apicv_activate_request(vcpu); > +} > + > +static void svm_request_deactivate_avic(struct kvm_vcpu *vcpu) > +{ > + struct vcpu_svm *svm = to_svm(vcpu); > + > + if (!lapic_in_kernel(vcpu) || !is_avic_active(svm)) > + return; > + > + /* Request temporary deactivate apicv */ > + kvm_make_apicv_deactivate_request(vcpu, false); > +} > + > static void svm_refresh_apicv_exec_ctrl(struct kvm_vcpu *vcpu) > { > struct vcpu_svm *svm = to_svm(vcpu); > @@ -5522,9 +5558,6 @@ static void enable_irq_window(struct kvm_vcpu *vcpu) > { > struct vcpu_svm *svm = to_svm(vcpu); > > - if (kvm_vcpu_apicv_active(vcpu)) > - return; > - > /* > * In case GIF=0 we can't rely on the CPU to tell us when GIF becomes > * 1, because that's a separate STGI/VMRUN intercept. The next time we > @@ -5534,6 +5567,14 @@ static void enable_irq_window(struct kvm_vcpu *vcpu) > * window under the assumption that the hardware will set the GIF. > */ > if ((vgif_enabled(svm) || gif_set(svm)) && nested_svm_intr(svm)) { > + /* > + * IRQ window is not needed when AVIC is enabled, > + * unless we have pending ExtINT since it cannot be injected > + * via AVIC. In such case, we need to temporarily disable AVIC, > + * and fallback to injecting IRQ via V_IRQ. > + */ > + if (kvm_vcpu_apicv_active(vcpu)) > + svm_request_deactivate_avic(&svm->vcpu); Did you test AVIC with nesting? Did you actually run across this issue there? Alex > svm_set_vintr(svm); > svm_inject_irq(svm, 0x0); > } >
Alex, On 8/19/19 5:35 AM, Alexander Graf wrote: > > > On 15.08.19 18:25, Suthikulpanit, Suravee wrote: >> AMD AVIC does not support ExtINT. Therefore, AVIC must be temporary >> deactivated and fall back to using legacy interrupt injection via vINTR >> and interrupt window. >> >> Signed-off-by: Suravee Suthikulpanit <suravee.suthikulpanit@amd.com> >> --- >> arch/x86/kvm/svm.c | 49 >> +++++++++++++++++++++++++++++++++++++++++++++---- >> 1 file changed, 45 insertions(+), 4 deletions(-) >> >> diff --git a/arch/x86/kvm/svm.c b/arch/x86/kvm/svm.c >> index cfa4b13..4690351 100644 >> --- a/arch/x86/kvm/svm.c >> +++ b/arch/x86/kvm/svm.c >> @@ -384,6 +384,7 @@ struct amd_svm_iommu_ir { >> static void svm_set_cr0(struct kvm_vcpu *vcpu, unsigned long cr0); >> static void svm_flush_tlb(struct kvm_vcpu *vcpu, bool invalidate_gpa); >> static void svm_complete_interrupts(struct vcpu_svm *svm); >> +static void svm_request_activate_avic(struct kvm_vcpu *vcpu); >> static bool svm_get_enable_apicv(struct kvm *kvm); >> static inline void avic_post_state_restore(struct kvm_vcpu *vcpu); >> @@ -4494,6 +4495,15 @@ static int interrupt_window_interception(struct >> vcpu_svm *svm) >> { >> kvm_make_request(KVM_REQ_EVENT, &svm->vcpu); >> svm_clear_vintr(svm); >> + >> + /* >> + * For AVIC, the only reason to end up here is ExtINTs. >> + * In this case AVIC was temporarily disabled for >> + * requesting the IRQ window and we have to re-enable it. >> + */ >> + if (svm_get_enable_apicv(svm->vcpu.kvm)) >> + svm_request_activate_avic(&svm->vcpu); > > Would it make sense to add a trace point here and to the other call > sites, so that it becomes obvious in a trace when and why exactly avic > was active/inactive? > > The trace point could add additional information on the why. Sure, sounds good. >> .... >> @@ -5522,9 +5558,6 @@ static void enable_irq_window(struct kvm_vcpu >> *vcpu) >> { >> struct vcpu_svm *svm = to_svm(vcpu); >> - if (kvm_vcpu_apicv_active(vcpu)) >> - return; >> - >> /* >> * In case GIF=0 we can't rely on the CPU to tell us when GIF >> becomes >> * 1, because that's a separate STGI/VMRUN intercept. The next >> time we >> @@ -5534,6 +5567,14 @@ static void enable_irq_window(struct kvm_vcpu >> *vcpu) >> * window under the assumption that the hardware will set the GIF. >> */ >> if ((vgif_enabled(svm) || gif_set(svm)) && nested_svm_intr(svm)) { >> + /* >> + * IRQ window is not needed when AVIC is enabled, >> + * unless we have pending ExtINT since it cannot be injected >> + * via AVIC. In such case, we need to temporarily disable AVIC, >> + * and fallback to injecting IRQ via V_IRQ. >> + */ >> + if (kvm_vcpu_apicv_active(vcpu)) >> + svm_request_deactivate_avic(&svm->vcpu); > > Did you test AVIC with nesting? Did you actually run across this issue > there? Currently, we have not claimed that AVIC is supported w/ nested VM. That's why we have not enabled AVIC by default yet. We will be looking more into that next. Suravee
> Am 28.08.2019 um 17:19 schrieb Suthikulpanit, Suravee <Suravee.Suthikulpanit@amd.com>: > > Alex, > >> On 8/19/19 5:35 AM, Alexander Graf wrote: >> >> >>> On 15.08.19 18:25, Suthikulpanit, Suravee wrote: >>> AMD AVIC does not support ExtINT. Therefore, AVIC must be temporary >>> deactivated and fall back to using legacy interrupt injection via vINTR >>> and interrupt window. >>> >>> Signed-off-by: Suravee Suthikulpanit <suravee.suthikulpanit@amd.com> >>> --- >>> arch/x86/kvm/svm.c | 49 >>> +++++++++++++++++++++++++++++++++++++++++++++---- >>> 1 file changed, 45 insertions(+), 4 deletions(-) >>> >>> diff --git a/arch/x86/kvm/svm.c b/arch/x86/kvm/svm.c >>> index cfa4b13..4690351 100644 >>> --- a/arch/x86/kvm/svm.c >>> +++ b/arch/x86/kvm/svm.c >>> @@ -384,6 +384,7 @@ struct amd_svm_iommu_ir { >>> static void svm_set_cr0(struct kvm_vcpu *vcpu, unsigned long cr0); >>> static void svm_flush_tlb(struct kvm_vcpu *vcpu, bool invalidate_gpa); >>> static void svm_complete_interrupts(struct vcpu_svm *svm); >>> +static void svm_request_activate_avic(struct kvm_vcpu *vcpu); >>> static bool svm_get_enable_apicv(struct kvm *kvm); >>> static inline void avic_post_state_restore(struct kvm_vcpu *vcpu); >>> @@ -4494,6 +4495,15 @@ static int interrupt_window_interception(struct >>> vcpu_svm *svm) >>> { >>> kvm_make_request(KVM_REQ_EVENT, &svm->vcpu); >>> svm_clear_vintr(svm); >>> + >>> + /* >>> + * For AVIC, the only reason to end up here is ExtINTs. >>> + * In this case AVIC was temporarily disabled for >>> + * requesting the IRQ window and we have to re-enable it. >>> + */ >>> + if (svm_get_enable_apicv(svm->vcpu.kvm)) >>> + svm_request_activate_avic(&svm->vcpu); >> >> Would it make sense to add a trace point here and to the other call >> sites, so that it becomes obvious in a trace when and why exactly avic >> was active/inactive? >> >> The trace point could add additional information on the why. > > Sure, sounds good. > >>> .... >>> @@ -5522,9 +5558,6 @@ static void enable_irq_window(struct kvm_vcpu >>> *vcpu) >>> { >>> struct vcpu_svm *svm = to_svm(vcpu); >>> - if (kvm_vcpu_apicv_active(vcpu)) >>> - return; >>> - >>> /* >>> * In case GIF=0 we can't rely on the CPU to tell us when GIF >>> becomes >>> * 1, because that's a separate STGI/VMRUN intercept. The next >>> time we >>> @@ -5534,6 +5567,14 @@ static void enable_irq_window(struct kvm_vcpu >>> *vcpu) >>> * window under the assumption that the hardware will set the GIF. >>> */ >>> if ((vgif_enabled(svm) || gif_set(svm)) && nested_svm_intr(svm)) { >>> + /* >>> + * IRQ window is not needed when AVIC is enabled, >>> + * unless we have pending ExtINT since it cannot be injected >>> + * via AVIC. In such case, we need to temporarily disable AVIC, >>> + * and fallback to injecting IRQ via V_IRQ. >>> + */ >>> + if (kvm_vcpu_apicv_active(vcpu)) >>> + svm_request_deactivate_avic(&svm->vcpu); >> >> Did you test AVIC with nesting? Did you actually run across this issue >> there? > > Currently, we have not claimed that AVIC is supported w/ nested VM. > That's why we have not enabled AVIC by default yet. We will be looking > more into that next. If it's not supported, please suspend it when we enter a nested guest then? In that case, the above change is also unnecessary, as it only affects nested guests, no? Alex > > Suravee Amazon Development Center Germany GmbH Krausenstr. 38 10117 Berlin Geschaeftsfuehrung: Christian Schlaeger, Ralf Herbrich Eingetragen am Amtsgericht Charlottenburg unter HRB 149173 B Sitz: Berlin Ust-ID: DE 289 237 879
Alex, On 8/28/19 2:37 PM, Graf (AWS), Alexander wrote: >>>> @@ -5522,9 +5558,6 @@ static void enable_irq_window(struct kvm_vcpu >>>> *vcpu) >>>> { >>>> struct vcpu_svm *svm = to_svm(vcpu); >>>> - if (kvm_vcpu_apicv_active(vcpu)) >>>> - return; >>>> - >>>> /* >>>> * In case GIF=0 we can't rely on the CPU to tell us when GIF >>>> becomes >>>> * 1, because that's a separate STGI/VMRUN intercept. The next >>>> time we >>>> @@ -5534,6 +5567,14 @@ static void enable_irq_window(struct kvm_vcpu >>>> *vcpu) >>>> * window under the assumption that the hardware will set the GIF. >>>> */ >>>> if ((vgif_enabled(svm) || gif_set(svm)) && nested_svm_intr(svm)) { >>>> + /* >>>> + * IRQ window is not needed when AVIC is enabled, >>>> + * unless we have pending ExtINT since it cannot be injected >>>> + * via AVIC. In such case, we need to temporarily disable AVIC, >>>> + * and fallback to injecting IRQ via V_IRQ. >>>> + */ >>>> + if (kvm_vcpu_apicv_active(vcpu)) >>>> + svm_request_deactivate_avic(&svm->vcpu); >>> Did you test AVIC with nesting? Did you actually run across this issue >>> there? >> Currently, we have not claimed that AVIC is supported w/ nested VM. >> That's why we have not enabled AVIC by default yet. We will be looking >> more into that next. > If it's not supported, please suspend it when we enter a nested guest then? Ok, this makes sense. I'll update this in V3. > In that case, the above change is also unnecessary, as it only affects nested guests, no? Actually, the function name nested_svm_intr() is misleading. Here it returns true when _NOT_ in guest mode: /* This function returns true if it is save to enable the irq window */ static inline bool nested_svm_intr(struct vcpu_svm *svm) { if (!is_guest_mode(&svm->vcpu)) return true; .... So, the logic above does what we want when AVIC is enabled. Thanks, Suravee
diff --git a/arch/x86/kvm/svm.c b/arch/x86/kvm/svm.c index cfa4b13..4690351 100644 --- a/arch/x86/kvm/svm.c +++ b/arch/x86/kvm/svm.c @@ -384,6 +384,7 @@ struct amd_svm_iommu_ir { static void svm_set_cr0(struct kvm_vcpu *vcpu, unsigned long cr0); static void svm_flush_tlb(struct kvm_vcpu *vcpu, bool invalidate_gpa); static void svm_complete_interrupts(struct vcpu_svm *svm); +static void svm_request_activate_avic(struct kvm_vcpu *vcpu); static bool svm_get_enable_apicv(struct kvm *kvm); static inline void avic_post_state_restore(struct kvm_vcpu *vcpu); @@ -4494,6 +4495,15 @@ static int interrupt_window_interception(struct vcpu_svm *svm) { kvm_make_request(KVM_REQ_EVENT, &svm->vcpu); svm_clear_vintr(svm); + + /* + * For AVIC, the only reason to end up here is ExtINTs. + * In this case AVIC was temporarily disabled for + * requesting the IRQ window and we have to re-enable it. + */ + if (svm_get_enable_apicv(svm->vcpu.kvm)) + svm_request_activate_avic(&svm->vcpu); + svm->vmcb->control.int_ctl &= ~V_IRQ_MASK; mark_dirty(svm->vmcb, VMCB_INTR); ++svm->vcpu.stat.irq_window_exits; @@ -5181,7 +5191,33 @@ static void svm_hwapic_isr_update(struct kvm_vcpu *vcpu, int max_isr) { } -/* Note: Currently only used by Hyper-V. */ +static bool is_avic_active(struct vcpu_svm *svm) +{ + return (svm_get_enable_apicv(svm->vcpu.kvm) && + svm->vmcb->control.int_ctl & AVIC_ENABLE_MASK); +} + +static void svm_request_activate_avic(struct kvm_vcpu *vcpu) +{ + struct vcpu_svm *svm = to_svm(vcpu); + + if (!lapic_in_kernel(vcpu) || is_avic_active(svm)) + return; + + kvm_make_apicv_activate_request(vcpu); +} + +static void svm_request_deactivate_avic(struct kvm_vcpu *vcpu) +{ + struct vcpu_svm *svm = to_svm(vcpu); + + if (!lapic_in_kernel(vcpu) || !is_avic_active(svm)) + return; + + /* Request temporary deactivate apicv */ + kvm_make_apicv_deactivate_request(vcpu, false); +} + static void svm_refresh_apicv_exec_ctrl(struct kvm_vcpu *vcpu) { struct vcpu_svm *svm = to_svm(vcpu); @@ -5522,9 +5558,6 @@ static void enable_irq_window(struct kvm_vcpu *vcpu) { struct vcpu_svm *svm = to_svm(vcpu); - if (kvm_vcpu_apicv_active(vcpu)) - return; - /* * In case GIF=0 we can't rely on the CPU to tell us when GIF becomes * 1, because that's a separate STGI/VMRUN intercept. The next time we @@ -5534,6 +5567,14 @@ static void enable_irq_window(struct kvm_vcpu *vcpu) * window under the assumption that the hardware will set the GIF. */ if ((vgif_enabled(svm) || gif_set(svm)) && nested_svm_intr(svm)) { + /* + * IRQ window is not needed when AVIC is enabled, + * unless we have pending ExtINT since it cannot be injected + * via AVIC. In such case, we need to temporarily disable AVIC, + * and fallback to injecting IRQ via V_IRQ. + */ + if (kvm_vcpu_apicv_active(vcpu)) + svm_request_deactivate_avic(&svm->vcpu); svm_set_vintr(svm); svm_inject_irq(svm, 0x0); }
AMD AVIC does not support ExtINT. Therefore, AVIC must be temporary deactivated and fall back to using legacy interrupt injection via vINTR and interrupt window. Signed-off-by: Suravee Suthikulpanit <suravee.suthikulpanit@amd.com> --- arch/x86/kvm/svm.c | 49 +++++++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 45 insertions(+), 4 deletions(-)