| Message ID | 20161213163001.4567-3-rkrcmar@redhat.com (mailing list archive) |
|---|---|
| State | New, archived |
| Headers | show |
Am 13.12.2016 um 17:29 schrieb Radim Krčmář: > There were three calls sites: > - recalculate_apic_map and kvm_apic_match_physical_addr, where it would > only complicate implementation of x2APIC hotplug; > - in apic_debug, where it was still somewhat preserved, but keeping the > old function just for apic_debug was not worth it > > Signed-off-by: Radim Krčmář <rkrcmar@redhat.com> > --- > arch/x86/kvm/lapic.c | 41 ++++++++++++++++++++++++++++++----------- > arch/x86/kvm/lapic.h | 11 ----------- > 2 files changed, 30 insertions(+), 22 deletions(-) > > diff --git a/arch/x86/kvm/lapic.c b/arch/x86/kvm/lapic.c > index 09edd32b8e42..e645b4bc6437 100644 > --- a/arch/x86/kvm/lapic.c > +++ b/arch/x86/kvm/lapic.c > @@ -115,6 +115,16 @@ static inline int apic_enabled(struct kvm_lapic *apic) > (LVT_MASK | APIC_MODE_MASK | APIC_INPUT_POLARITY | \ > APIC_LVT_REMOTE_IRR | APIC_LVT_LEVEL_TRIGGER) > > +static inline u8 kvm_xapic_id(struct kvm_lapic *apic) > +{ > + return kvm_lapic_get_reg(apic, APIC_ID) >> 24; > +} > + > +static inline u32 kvm_x2apic_id(struct kvm_lapic *apic) > +{ > + return apic->vcpu->vcpu_id; > +} > + > static inline bool kvm_apic_map_get_logical_dest(struct kvm_apic_map *map, > u32 dest_id, struct kvm_lapic ***cluster, u16 *mask) { > switch (map->mode) { > @@ -159,13 +169,13 @@ static void recalculate_apic_map(struct kvm *kvm) > struct kvm_apic_map *new, *old = NULL; > struct kvm_vcpu *vcpu; > int i; > - u32 max_id = 255; > + u32 max_id = 256; /* enough space for any xAPIC ID */ Why exactly do we have to increase this? I thought the maximum xapic id is 255, and we correctly allocate 255+1 slots. Now, we would allocate 257 slots and set max_apic_id to 256. Can you elaborate?
2016-12-14 13:39+0100, David Hildenbrand: > Am 13.12.2016 um 17:29 schrieb Radim Krčmář: >> There were three calls sites: >> - recalculate_apic_map and kvm_apic_match_physical_addr, where it would >> only complicate implementation of x2APIC hotplug; >> - in apic_debug, where it was still somewhat preserved, but keeping the >> old function just for apic_debug was not worth it >> >> Signed-off-by: Radim Krčmář <rkrcmar@redhat.com> >> --- >> arch/x86/kvm/lapic.c | 41 ++++++++++++++++++++++++++++++----------- >> arch/x86/kvm/lapic.h | 11 ----------- >> 2 files changed, 30 insertions(+), 22 deletions(-) >> >> diff --git a/arch/x86/kvm/lapic.c b/arch/x86/kvm/lapic.c >> index 09edd32b8e42..e645b4bc6437 100644 >> --- a/arch/x86/kvm/lapic.c >> +++ b/arch/x86/kvm/lapic.c >> @@ -115,6 +115,16 @@ static inline int apic_enabled(struct kvm_lapic *apic) >> (LVT_MASK | APIC_MODE_MASK | APIC_INPUT_POLARITY | \ >> APIC_LVT_REMOTE_IRR | APIC_LVT_LEVEL_TRIGGER) >> >> +static inline u8 kvm_xapic_id(struct kvm_lapic *apic) >> +{ >> + return kvm_lapic_get_reg(apic, APIC_ID) >> 24; >> +} >> + >> +static inline u32 kvm_x2apic_id(struct kvm_lapic *apic) >> +{ >> + return apic->vcpu->vcpu_id; >> +} >> + >> static inline bool kvm_apic_map_get_logical_dest(struct kvm_apic_map *map, >> u32 dest_id, struct kvm_lapic ***cluster, u16 *mask) { >> switch (map->mode) { >> @@ -159,13 +169,13 @@ static void recalculate_apic_map(struct kvm *kvm) >> struct kvm_apic_map *new, *old = NULL; >> struct kvm_vcpu *vcpu; >> int i; >> - u32 max_id = 255; >> + u32 max_id = 256; /* enough space for any xAPIC ID */ > > Why exactly do we have to increase this? > > I thought the maximum xapic id is 255, and we correctly allocate > 255+1 slots. > > Now, we would allocate 257 slots and set max_apic_id to 256. Right, it is a typical off-by-one error. The comment is sufficient ... Queued for v3, Thanks. -- To unsubscribe from this list: send the line "unsubscribe kvm" in the body of a message to majordomo@vger.kernel.org More majordomo info at http://vger.kernel.org/majordomo-info.html
On 14/12/2016 14:41, Radim Krčmář wrote: > 2016-12-14 13:39+0100, David Hildenbrand: >> Am 13.12.2016 um 17:29 schrieb Radim Krčmář: >>> There were three calls sites: >>> - recalculate_apic_map and kvm_apic_match_physical_addr, where it would >>> only complicate implementation of x2APIC hotplug; >>> - in apic_debug, where it was still somewhat preserved, but keeping the >>> old function just for apic_debug was not worth it >>> >>> Signed-off-by: Radim Krčmář <rkrcmar@redhat.com> >>> --- >>> arch/x86/kvm/lapic.c | 41 ++++++++++++++++++++++++++++++----------- >>> arch/x86/kvm/lapic.h | 11 ----------- >>> 2 files changed, 30 insertions(+), 22 deletions(-) >>> >>> diff --git a/arch/x86/kvm/lapic.c b/arch/x86/kvm/lapic.c >>> index 09edd32b8e42..e645b4bc6437 100644 >>> --- a/arch/x86/kvm/lapic.c >>> +++ b/arch/x86/kvm/lapic.c >>> @@ -115,6 +115,16 @@ static inline int apic_enabled(struct kvm_lapic *apic) >>> (LVT_MASK | APIC_MODE_MASK | APIC_INPUT_POLARITY | \ >>> APIC_LVT_REMOTE_IRR | APIC_LVT_LEVEL_TRIGGER) >>> >>> +static inline u8 kvm_xapic_id(struct kvm_lapic *apic) >>> +{ >>> + return kvm_lapic_get_reg(apic, APIC_ID) >> 24; >>> +} >>> + >>> +static inline u32 kvm_x2apic_id(struct kvm_lapic *apic) >>> +{ >>> + return apic->vcpu->vcpu_id; >>> +} >>> + >>> static inline bool kvm_apic_map_get_logical_dest(struct kvm_apic_map *map, >>> u32 dest_id, struct kvm_lapic ***cluster, u16 *mask) { >>> switch (map->mode) { >>> @@ -159,13 +169,13 @@ static void recalculate_apic_map(struct kvm *kvm) >>> struct kvm_apic_map *new, *old = NULL; >>> struct kvm_vcpu *vcpu; >>> int i; >>> - u32 max_id = 255; >>> + u32 max_id = 256; /* enough space for any xAPIC ID */ >> >> Why exactly do we have to increase this? >> >> I thought the maximum xapic id is 255, and we correctly allocate >> 255+1 slots. >> >> Now, we would allocate 257 slots and set max_apic_id to 256. > > Right, it is a typical off-by-one error. The comment is sufficient ... I can change it myself too. Paolo -- To unsubscribe from this list: send the line "unsubscribe kvm" in the body of a message to majordomo@vger.kernel.org More majordomo info at http://vger.kernel.org/majordomo-info.html
> kvm_for_each_vcpu(i, vcpu, kvm) > if (kvm_apic_present(vcpu)) > - max_id = max(max_id, kvm_apic_id(vcpu->arch.apic)); > + max_id = max(max_id, kvm_x2apic_id(vcpu->arch.apic)); > > new = kvm_kvzalloc(sizeof(struct kvm_apic_map) + > sizeof(struct kvm_lapic *) * ((u64)max_id + 1)); > @@ -179,16 +189,23 @@ static void recalculate_apic_map(struct kvm *kvm) > struct kvm_lapic *apic = vcpu->arch.apic; > struct kvm_lapic **cluster; > u16 mask; > - u32 ldr, aid; > + u32 ldr; > + u8 xapic_id; > + u32 x2apic_id; > > if (!kvm_apic_present(vcpu)) > continue; > > - aid = kvm_apic_id(apic); think I'd even prefer here a simple aid = kvm_xapic_id(apic); if (apic_x2apic_mode(apic)) aid = kvm_x2apic_id(apic); that would keep changes minimal and I don't really see any benefit in the code when splitting handling up. Patch 4 then simply can fixup setting code if (aid <= new->max_apic_id && !new->phys_map[aid]) new->phys_map[aid] = apic; (if I am not missing some important corner case here) > - ldr = kvm_lapic_get_reg(apic, APIC_LDR); > + xapic_id = kvm_xapic_id(apic); > + x2apic_id = kvm_x2apic_id(apic); > > - if (aid <= new->max_apic_id) > - new->phys_map[aid] = apic; > + if (apic_x2apic_mode(apic) && > + x2apic_id <= new->max_apic_id) > + new->phys_map[x2apic_id] = apic; > + else if (!apic_x2apic_mode(apic)) This looks good to me.
On 14/12/2016 17:15, David Hildenbrand wrote: > >> kvm_for_each_vcpu(i, vcpu, kvm) >> if (kvm_apic_present(vcpu)) >> - max_id = max(max_id, kvm_apic_id(vcpu->arch.apic)); >> + max_id = max(max_id, kvm_x2apic_id(vcpu->arch.apic)); >> >> new = kvm_kvzalloc(sizeof(struct kvm_apic_map) + >> sizeof(struct kvm_lapic *) * ((u64)max_id + 1)); >> @@ -179,16 +189,23 @@ static void recalculate_apic_map(struct kvm *kvm) >> struct kvm_lapic *apic = vcpu->arch.apic; >> struct kvm_lapic **cluster; >> u16 mask; >> - u32 ldr, aid; >> + u32 ldr; >> + u8 xapic_id; >> + u32 x2apic_id; >> >> if (!kvm_apic_present(vcpu)) >> continue; >> >> - aid = kvm_apic_id(apic); > > think I'd even prefer here a simple > > aid = kvm_xapic_id(apic); > if (apic_x2apic_mode(apic)) > aid = kvm_x2apic_id(apic); > > that would keep changes minimal and I don't really see any benefit in > the code when splitting handling up. > > Patch 4 then simply can fixup setting code > > if (aid <= new->max_apic_id && !new->phys_map[aid]) > new->phys_map[aid] = apic; > > (if I am not missing some important corner case here) Radim, what do you think? I wanted to get these in before Christmas, but it's your call. Paolo > >> - ldr = kvm_lapic_get_reg(apic, APIC_LDR); >> + xapic_id = kvm_xapic_id(apic); >> + x2apic_id = kvm_x2apic_id(apic); >> >> - if (aid <= new->max_apic_id) >> - new->phys_map[aid] = apic; >> + if (apic_x2apic_mode(apic) && >> + x2apic_id <= new->max_apic_id) >> + new->phys_map[x2apic_id] = apic; >> + else if (!apic_x2apic_mode(apic)) > > > This looks good to me. > -- To unsubscribe from this list: send the line "unsubscribe kvm" in the body of a message to majordomo@vger.kernel.org More majordomo info at http://vger.kernel.org/majordomo-info.html
2016-12-14 17:15+0100, David Hildenbrand: >> kvm_for_each_vcpu(i, vcpu, kvm) >> if (kvm_apic_present(vcpu)) >> - max_id = max(max_id, kvm_apic_id(vcpu->arch.apic)); >> + max_id = max(max_id, kvm_x2apic_id(vcpu->arch.apic)); >> >> new = kvm_kvzalloc(sizeof(struct kvm_apic_map) + >> sizeof(struct kvm_lapic *) * ((u64)max_id + 1)); >> @@ -179,16 +189,23 @@ static void recalculate_apic_map(struct kvm *kvm) >> struct kvm_lapic *apic = vcpu->arch.apic; >> struct kvm_lapic **cluster; >> u16 mask; >> - u32 ldr, aid; >> + u32 ldr; >> + u8 xapic_id; >> + u32 x2apic_id; >> >> if (!kvm_apic_present(vcpu)) >> continue; >> >> - aid = kvm_apic_id(apic); > > think I'd even prefer here a simple > > aid = kvm_xapic_id(apic); > if (apic_x2apic_mode(apic)) > aid = kvm_x2apic_id(apic); > > that would keep changes minimal and I don't really see any benefit in the > code when splitting handling up. It is neccesassary to write an entry for both IDs and I wanted to split it before [4/4], because doing both changes at once seemed hard to grasp. Putting it here didn't work well either ... is a separate patch for the hunk below better, or would you prefer to have it in [4/4]? > Patch 4 then simply can fixup setting code > > if (aid <= new->max_apic_id && !new->phys_map[aid]) > new->phys_map[aid] = apic; > > (if I am not missing some important corner case here) The trick is that we want to do the following even in xAPIC mode: new->phys_map[kvm_x2apic_id(apic)] = apic; This is the main idea of the hotplug hack -- to allow unique addressing of processors that were reset in xAPIC mode. (And I add a disgusting "x2apic_id > 0xff" condition in [4/4], because we still allow guests to change xAPIC IDs, which wouldn't play nice with this.) Hardware does a superset of this, because it only looks at lower 8 bits of the desination ID when delivering to xAPIC. When kvm_x2apic_id(apic) != kvm_xapic_id(apic), then the APIC is in xAPIC mode so we definitely want to keep xAPIC working, hence if (!apic_x2apic_mode(apic)) new->phys_map[kvm_xapic_id(apic)] = apic; Two writes are necessary. And there can already be another_apic "kvm_x2apic_id(another_apic) == kvm_xapic_id(apic)" so we prevent hotplug from breaking existing x2APIC setups by doing "!new->phys_map[aid]" when setting xAPIC ID. I hope we get a better solution in the future, but it would have to be done at hardware (QEMU) level, because even firmware (seabios) doesn't have standard ways to deal with this situation ... >> - ldr = kvm_lapic_get_reg(apic, APIC_LDR); >> + xapic_id = kvm_xapic_id(apic); >> + x2apic_id = kvm_x2apic_id(apic); >> >> - if (aid <= new->max_apic_id) >> - new->phys_map[aid] = apic; >> + if (apic_x2apic_mode(apic) && >> + x2apic_id <= new->max_apic_id) >> + new->phys_map[x2apic_id] = apic; >> + else if (!apic_x2apic_mode(apic)) > > > This looks good to me. -- To unsubscribe from this list: send the line "unsubscribe kvm" in the body of a message to majordomo@vger.kernel.org More majordomo info at http://vger.kernel.org/majordomo-info.html
2016-12-14 17:59+0100, Paolo Bonzini: > On 14/12/2016 17:15, David Hildenbrand wrote: >> >>> kvm_for_each_vcpu(i, vcpu, kvm) >>> if (kvm_apic_present(vcpu)) >>> - max_id = max(max_id, kvm_apic_id(vcpu->arch.apic)); >>> + max_id = max(max_id, kvm_x2apic_id(vcpu->arch.apic)); >>> >>> new = kvm_kvzalloc(sizeof(struct kvm_apic_map) + >>> sizeof(struct kvm_lapic *) * ((u64)max_id + 1)); >>> @@ -179,16 +189,23 @@ static void recalculate_apic_map(struct kvm *kvm) >>> struct kvm_lapic *apic = vcpu->arch.apic; >>> struct kvm_lapic **cluster; >>> u16 mask; >>> - u32 ldr, aid; >>> + u32 ldr; >>> + u8 xapic_id; >>> + u32 x2apic_id; >>> >>> if (!kvm_apic_present(vcpu)) >>> continue; >>> >>> - aid = kvm_apic_id(apic); >> >> think I'd even prefer here a simple >> >> aid = kvm_xapic_id(apic); >> if (apic_x2apic_mode(apic)) >> aid = kvm_x2apic_id(apic); >> >> that would keep changes minimal and I don't really see any benefit in >> the code when splitting handling up. >> >> Patch 4 then simply can fixup setting code >> >> if (aid <= new->max_apic_id && !new->phys_map[aid]) >> new->phys_map[aid] = apic; >> >> (if I am not missing some important corner case here) > > Radim, what do you think? I wanted to get these in before Christmas, > but it's your call. There was a reason why it was so ugly ... it's not a hack for nothing. I can hope to make the patches/code more understandable, but the function shouldn't change, unless we want to take a different approach. -- To unsubscribe from this list: send the line "unsubscribe kvm" in the body of a message to majordomo@vger.kernel.org More majordomo info at http://vger.kernel.org/majordomo-info.html
>> think I'd even prefer here a simple >> >> aid = kvm_xapic_id(apic); >> if (apic_x2apic_mode(apic)) >> aid = kvm_x2apic_id(apic); >> >> that would keep changes minimal and I don't really see any benefit in the >> code when splitting handling up. > > It is neccesassary to write an entry for both IDs and I wanted to split > it before [4/4], because doing both changes at once seemed hard to > grasp. > > Putting it here didn't work well either ... is a separate patch for the > hunk below better, or would you prefer to have it in [4/4]? I actually would prefer to have it in 4/4, but not sure if it is worth yet another round. Anyhow, Reviewed-by: David Hildenbrand <david@redhat.com> for this patch (with the 256 fixed) > >> Patch 4 then simply can fixup setting code >> >> if (aid <= new->max_apic_id && !new->phys_map[aid]) >> new->phys_map[aid] = apic; >> >> (if I am not missing some important corner case here) > > The trick is that we want to do the following even in xAPIC mode: > > new->phys_map[kvm_x2apic_id(apic)] = apic; > > This is the main idea of the hotplug hack -- to allow unique addressing > of processors that were reset in xAPIC mode. (And I add a disgusting > "x2apic_id > 0xff" condition in [4/4], because we still allow guests to > change xAPIC IDs, which wouldn't play nice with this.) > > Hardware does a superset of this, because it only looks at lower 8 bits > of the desination ID when delivering to xAPIC. > > When kvm_x2apic_id(apic) != kvm_xapic_id(apic), then the APIC is in > xAPIC mode so we definitely want to keep xAPIC working, hence > > if (!apic_x2apic_mode(apic)) > new->phys_map[kvm_xapic_id(apic)] = apic; Okay, so this is is the case I missed in patch 4 :) Thanks for the explanation and sorry for the noise.
diff --git a/arch/x86/kvm/lapic.c b/arch/x86/kvm/lapic.c index 09edd32b8e42..e645b4bc6437 100644 --- a/arch/x86/kvm/lapic.c +++ b/arch/x86/kvm/lapic.c @@ -115,6 +115,16 @@ static inline int apic_enabled(struct kvm_lapic *apic) (LVT_MASK | APIC_MODE_MASK | APIC_INPUT_POLARITY | \ APIC_LVT_REMOTE_IRR | APIC_LVT_LEVEL_TRIGGER) +static inline u8 kvm_xapic_id(struct kvm_lapic *apic) +{ + return kvm_lapic_get_reg(apic, APIC_ID) >> 24; +} + +static inline u32 kvm_x2apic_id(struct kvm_lapic *apic) +{ + return apic->vcpu->vcpu_id; +} + static inline bool kvm_apic_map_get_logical_dest(struct kvm_apic_map *map, u32 dest_id, struct kvm_lapic ***cluster, u16 *mask) { switch (map->mode) { @@ -159,13 +169,13 @@ static void recalculate_apic_map(struct kvm *kvm) struct kvm_apic_map *new, *old = NULL; struct kvm_vcpu *vcpu; int i; - u32 max_id = 255; + u32 max_id = 256; /* enough space for any xAPIC ID */ mutex_lock(&kvm->arch.apic_map_lock); kvm_for_each_vcpu(i, vcpu, kvm) if (kvm_apic_present(vcpu)) - max_id = max(max_id, kvm_apic_id(vcpu->arch.apic)); + max_id = max(max_id, kvm_x2apic_id(vcpu->arch.apic)); new = kvm_kvzalloc(sizeof(struct kvm_apic_map) + sizeof(struct kvm_lapic *) * ((u64)max_id + 1)); @@ -179,16 +189,23 @@ static void recalculate_apic_map(struct kvm *kvm) struct kvm_lapic *apic = vcpu->arch.apic; struct kvm_lapic **cluster; u16 mask; - u32 ldr, aid; + u32 ldr; + u8 xapic_id; + u32 x2apic_id; if (!kvm_apic_present(vcpu)) continue; - aid = kvm_apic_id(apic); - ldr = kvm_lapic_get_reg(apic, APIC_LDR); + xapic_id = kvm_xapic_id(apic); + x2apic_id = kvm_x2apic_id(apic); - if (aid <= new->max_apic_id) - new->phys_map[aid] = apic; + if (apic_x2apic_mode(apic) && + x2apic_id <= new->max_apic_id) + new->phys_map[x2apic_id] = apic; + else if (!apic_x2apic_mode(apic)) + new->phys_map[xapic_id] = apic; + + ldr = kvm_lapic_get_reg(apic, APIC_LDR); if (apic_x2apic_mode(apic)) { new->mode |= KVM_APIC_MODE_X2APIC; @@ -250,6 +267,8 @@ static inline void kvm_apic_set_x2apic_id(struct kvm_lapic *apic, u32 id) { u32 ldr = ((id >> 4) << 16) | (1 << (id & 0xf)); + WARN_ON_ONCE(id != apic->vcpu->vcpu_id); + kvm_lapic_set_reg(apic, APIC_ID, id); kvm_lapic_set_reg(apic, APIC_LDR, ldr); recalculate_apic_map(apic->vcpu->kvm); @@ -591,9 +610,9 @@ static bool kvm_apic_match_physical_addr(struct kvm_lapic *apic, u32 mda) return true; if (apic_x2apic_mode(apic)) - return mda == kvm_apic_id(apic); + return mda == kvm_x2apic_id(apic); - return mda == SET_APIC_DEST_FIELD(kvm_apic_id(apic)); + return mda == SET_APIC_DEST_FIELD(kvm_xapic_id(apic)); } static bool kvm_apic_match_logical_addr(struct kvm_lapic *apic, u32 mda) @@ -1907,9 +1926,9 @@ void kvm_lapic_reset(struct kvm_vcpu *vcpu, bool init_event) vcpu->arch.apic_arb_prio = 0; vcpu->arch.apic_attention = 0; - apic_debug("%s: vcpu=%p, id=%d, base_msr=" + apic_debug("%s: vcpu=%p, id=0x%x, base_msr=" "0x%016" PRIx64 ", base_address=0x%0lx.\n", __func__, - vcpu, kvm_apic_id(apic), + vcpu, kvm_lapic_get_reg(apic, APIC_ID), vcpu->arch.apic_base, apic->base_address); } diff --git a/arch/x86/kvm/lapic.h b/arch/x86/kvm/lapic.h index e0c80233b3e1..cb16e6fd2330 100644 --- a/arch/x86/kvm/lapic.h +++ b/arch/x86/kvm/lapic.h @@ -202,17 +202,6 @@ static inline int kvm_lapic_latched_init(struct kvm_vcpu *vcpu) return lapic_in_kernel(vcpu) && test_bit(KVM_APIC_INIT, &vcpu->arch.apic->pending_events); } -static inline u32 kvm_apic_id(struct kvm_lapic *apic) -{ - /* To avoid a race between apic_base and following APIC_ID update when - * switching to x2apic_mode, the x2apic mode returns initial x2apic id. - */ - if (apic_x2apic_mode(apic)) - return apic->vcpu->vcpu_id; - - return kvm_lapic_get_reg(apic, APIC_ID) >> 24; -} - bool kvm_apic_pending_eoi(struct kvm_vcpu *vcpu, int vector); void wait_lapic_expire(struct kvm_vcpu *vcpu);
There were three calls sites: - recalculate_apic_map and kvm_apic_match_physical_addr, where it would only complicate implementation of x2APIC hotplug; - in apic_debug, where it was still somewhat preserved, but keeping the old function just for apic_debug was not worth it Signed-off-by: Radim Krčmář <rkrcmar@redhat.com> --- arch/x86/kvm/lapic.c | 41 ++++++++++++++++++++++++++++++----------- arch/x86/kvm/lapic.h | 11 ----------- 2 files changed, 30 insertions(+), 22 deletions(-)