| Message ID | 156879437195.18368.2222030761877686909.stgit@aravinda (mailing list archive) |
|---|---|
| State | New, archived |
| Headers | show |
| Series | target-ppc/spapr: Add FWNMI support in QEMU for PowerKVM guests | expand |
On Wed, Sep 18, 2019 at 01:42:51PM +0530, Aravinda Prasad wrote: > This patch includes migration support for machine check > handling. Especially this patch blocks VM migration > requests until the machine check error handling is > complete as these errors are specific to the source > hardware and is irrelevant on the target hardware. > > Signed-off-by: Aravinda Prasad <aravinda@linux.vnet.ibm.com> > --- > hw/ppc/spapr.c | 63 ++++++++++++++++++++++++++++++++++++++++++++++++ > hw/ppc/spapr_events.c | 16 +++++++++++- > hw/ppc/spapr_rtas.c | 2 ++ > include/hw/ppc/spapr.h | 2 ++ > 4 files changed, 82 insertions(+), 1 deletion(-) > > diff --git a/hw/ppc/spapr.c b/hw/ppc/spapr.c > index 6992b32..a72a4b1 100644 > --- a/hw/ppc/spapr.c > +++ b/hw/ppc/spapr.c > @@ -46,6 +46,7 @@ > #include "migration/qemu-file-types.h" > #include "migration/global_state.h" > #include "migration/register.h" > +#include "migration/blocker.h" > #include "mmu-hash64.h" > #include "mmu-book3s-v3.h" > #include "cpu-models.h" > @@ -1829,6 +1830,8 @@ static void spapr_machine_reset(MachineState *machine) > > /* Signal all vCPUs waiting on this condition */ > qemu_cond_broadcast(&spapr->mc_delivery_cond); > + > + migrate_del_blocker(spapr->fwnmi_migration_blocker); > } > > static void spapr_create_nvram(SpaprMachineState *spapr) > @@ -2119,6 +2122,60 @@ static const VMStateDescription vmstate_spapr_dtb = { > }, > }; > > +static bool spapr_fwnmi_needed(void *opaque) > +{ > + SpaprMachineState *spapr = (SpaprMachineState *)opaque; > + > + return spapr->guest_machine_check_addr != -1; > +} > + > +static int spapr_fwnmi_post_load(void *opaque, int version_id) > +{ > + SpaprMachineState *spapr = (SpaprMachineState *)opaque; > + > + if (spapr_get_cap(spapr, SPAPR_CAP_FWNMI_MCE) == SPAPR_CAP_ON) { > + > + if (kvmppc_has_cap_ppc_fwnmi()) { > + return 0; > + } > + > + return kvmppc_set_fwnmi(); > + } I don't see that you need this. The spapr caps need to be set the same on source and destination (the caps infrastructure handles that), so setup should already be handled by the caps .apply hooks. > + > + return 0; > +} > + > +static int spapr_fwnmi_pre_save(void *opaque) > +{ > + SpaprMachineState *spapr = (SpaprMachineState *)opaque; > + > + /* > + * With -only-migratable QEMU option, we cannot block migration. > + * Hence check if machine check handling is in progress and print > + * a warning message. > + */ > + if (spapr->mc_status != -1) { > + warn_report("A machine check is being handled during migration. The" > + "handler may run and log hardware error on the destination"); > + } > + > + return 0; > +} > + > +static const VMStateDescription vmstate_spapr_machine_check = { > + .name = "spapr_machine_check", > + .version_id = 1, > + .minimum_version_id = 1, > + .needed = spapr_fwnmi_needed, > + .post_load = spapr_fwnmi_post_load, > + .pre_save = spapr_fwnmi_pre_save, > + .fields = (VMStateField[]) { > + VMSTATE_UINT64(guest_machine_check_addr, SpaprMachineState), > + VMSTATE_INT32(mc_status, SpaprMachineState), > + VMSTATE_END_OF_LIST() > + }, > +}; So, I know I suggested earlier that you postpone the migration support to a later patch in the series. But at least for the actual vmstate stuff, I think that was due to considerations that have since gone. I don't see any reason you can't put this in as soon as you add the machine_check_addr and mc_status fields. The migration blocker stuff might have to come in a later patch, but that's ok. > + > static const VMStateDescription vmstate_spapr = { > .name = "spapr", > .version_id = 3, > @@ -2152,6 +2209,7 @@ static const VMStateDescription vmstate_spapr = { > &vmstate_spapr_dtb, > &vmstate_spapr_cap_large_decr, > &vmstate_spapr_cap_ccf_assist, > + &vmstate_spapr_machine_check, > NULL > } > }; > @@ -2948,6 +3006,11 @@ static void spapr_machine_init(MachineState *machine) > exit(1); > } > > + /* Create the error string for live migration blocker */ > + error_setg(&spapr->fwnmi_migration_blocker, > + "A machine check is being handled during migration. The handler" > + "may run and log hardware error on the destination"); > + > /* Register ibm,nmi-register and ibm,nmi-interlock RTAS calls */ > spapr_fwnmi_register(); > } > diff --git a/hw/ppc/spapr_events.c b/hw/ppc/spapr_events.c > index ecc3d68..71caa03 100644 > --- a/hw/ppc/spapr_events.c > +++ b/hw/ppc/spapr_events.c > @@ -43,6 +43,7 @@ > #include "qemu/main-loop.h" > #include "hw/ppc/spapr_ovec.h" > #include <libfdt.h> > +#include "migration/blocker.h" > > #define RTAS_LOG_VERSION_MASK 0xff000000 > #define RTAS_LOG_VERSION_6 0x06000000 > @@ -844,6 +845,8 @@ void spapr_mce_req_event(PowerPCCPU *cpu, bool recovered) > { > SpaprMachineState *spapr = SPAPR_MACHINE(qdev_get_machine()); > CPUState *cs = CPU(cpu); > + int ret; > + Error *local_err = NULL; > > if (spapr->guest_machine_check_addr == -1) { > /* > @@ -873,8 +876,19 @@ void spapr_mce_req_event(PowerPCCPU *cpu, bool recovered) > return; > } > } > - spapr->mc_status = cpu->vcpu_id; > > + ret = migrate_add_blocker(spapr->fwnmi_migration_blocker, &local_err); > + if (ret == -EBUSY) { > + /* > + * We don't want to abort so we let the migration to continue. > + * In a rare case, the machine check handler will run on the target. > + * Though this is not preferable, it is better than aborting > + * the migration or killing the VM. > + */ > + warn_report_err(local_err); > + } > + > + spapr->mc_status = cpu->vcpu_id; > spapr_mce_dispatch_elog(cpu, recovered); > } > > diff --git a/hw/ppc/spapr_rtas.c b/hw/ppc/spapr_rtas.c > index b569538..c652ec3 100644 > --- a/hw/ppc/spapr_rtas.c > +++ b/hw/ppc/spapr_rtas.c > @@ -50,6 +50,7 @@ > #include "hw/ppc/fdt.h" > #include "target/ppc/mmu-hash64.h" > #include "target/ppc/mmu-book3s-v3.h" > +#include "migration/blocker.h" > > static void rtas_display_character(PowerPCCPU *cpu, SpaprMachineState *spapr, > uint32_t token, uint32_t nargs, > @@ -446,6 +447,7 @@ static void rtas_ibm_nmi_interlock(PowerPCCPU *cpu, > */ > spapr->mc_status = -1; > qemu_cond_signal(&spapr->mc_delivery_cond); > + migrate_del_blocker(spapr->fwnmi_migration_blocker); > rtas_st(rets, 0, RTAS_OUT_SUCCESS); > } > > diff --git a/include/hw/ppc/spapr.h b/include/hw/ppc/spapr.h > index dada821..ea7625e 100644 > --- a/include/hw/ppc/spapr.h > +++ b/include/hw/ppc/spapr.h > @@ -217,6 +217,8 @@ struct SpaprMachineState { > > unsigned gpu_numa_id; > SpaprTpmProxy *tpm_proxy; > + > + Error *fwnmi_migration_blocker; > }; > > #define H_SUCCESS 0 >
On Wednesday 25 September 2019 07:09 AM, David Gibson wrote: > On Wed, Sep 18, 2019 at 01:42:51PM +0530, Aravinda Prasad wrote: >> This patch includes migration support for machine check >> handling. Especially this patch blocks VM migration >> requests until the machine check error handling is >> complete as these errors are specific to the source >> hardware and is irrelevant on the target hardware. >> >> Signed-off-by: Aravinda Prasad <aravinda@linux.vnet.ibm.com> >> --- >> hw/ppc/spapr.c | 63 ++++++++++++++++++++++++++++++++++++++++++++++++ >> hw/ppc/spapr_events.c | 16 +++++++++++- >> hw/ppc/spapr_rtas.c | 2 ++ >> include/hw/ppc/spapr.h | 2 ++ >> 4 files changed, 82 insertions(+), 1 deletion(-) >> >> diff --git a/hw/ppc/spapr.c b/hw/ppc/spapr.c >> index 6992b32..a72a4b1 100644 >> --- a/hw/ppc/spapr.c >> +++ b/hw/ppc/spapr.c >> @@ -46,6 +46,7 @@ >> #include "migration/qemu-file-types.h" >> #include "migration/global_state.h" >> #include "migration/register.h" >> +#include "migration/blocker.h" >> #include "mmu-hash64.h" >> #include "mmu-book3s-v3.h" >> #include "cpu-models.h" >> @@ -1829,6 +1830,8 @@ static void spapr_machine_reset(MachineState *machine) >> >> /* Signal all vCPUs waiting on this condition */ >> qemu_cond_broadcast(&spapr->mc_delivery_cond); >> + >> + migrate_del_blocker(spapr->fwnmi_migration_blocker); >> } >> >> static void spapr_create_nvram(SpaprMachineState *spapr) >> @@ -2119,6 +2122,60 @@ static const VMStateDescription vmstate_spapr_dtb = { >> }, >> }; >> >> +static bool spapr_fwnmi_needed(void *opaque) >> +{ >> + SpaprMachineState *spapr = (SpaprMachineState *)opaque; >> + >> + return spapr->guest_machine_check_addr != -1; >> +} >> + >> +static int spapr_fwnmi_post_load(void *opaque, int version_id) >> +{ >> + SpaprMachineState *spapr = (SpaprMachineState *)opaque; >> + >> + if (spapr_get_cap(spapr, SPAPR_CAP_FWNMI_MCE) == SPAPR_CAP_ON) { >> + >> + if (kvmppc_has_cap_ppc_fwnmi()) { >> + return 0; >> + } >> + >> + return kvmppc_set_fwnmi(); >> + } > > I don't see that you need this. The spapr caps need to be set the > same on source and destination (the caps infrastructure handles that), > so setup should already be handled by the caps .apply hooks. Will check. > >> + >> + return 0; >> +} >> + >> +static int spapr_fwnmi_pre_save(void *opaque) >> +{ >> + SpaprMachineState *spapr = (SpaprMachineState *)opaque; >> + >> + /* >> + * With -only-migratable QEMU option, we cannot block migration. >> + * Hence check if machine check handling is in progress and print >> + * a warning message. >> + */ >> + if (spapr->mc_status != -1) { >> + warn_report("A machine check is being handled during migration. The" >> + "handler may run and log hardware error on the destination"); >> + } >> + >> + return 0; >> +} >> + >> +static const VMStateDescription vmstate_spapr_machine_check = { >> + .name = "spapr_machine_check", >> + .version_id = 1, >> + .minimum_version_id = 1, >> + .needed = spapr_fwnmi_needed, >> + .post_load = spapr_fwnmi_post_load, >> + .pre_save = spapr_fwnmi_pre_save, >> + .fields = (VMStateField[]) { >> + VMSTATE_UINT64(guest_machine_check_addr, SpaprMachineState), >> + VMSTATE_INT32(mc_status, SpaprMachineState), >> + VMSTATE_END_OF_LIST() >> + }, >> +}; > > So, I know I suggested earlier that you postpone the migration support > to a later patch in the series. But at least for the actual vmstate > stuff, I think that was due to considerations that have since gone. I > don't see any reason you can't put this in as soon as you add the > machine_check_addr and mc_status fields. > > The migration blocker stuff might have to come in a later patch, but > that's ok. I feel I can retain this. Let me know if you want me to change. Regards, Aravinda > >> + >> static const VMStateDescription vmstate_spapr = { >> .name = "spapr", >> .version_id = 3, >> @@ -2152,6 +2209,7 @@ static const VMStateDescription vmstate_spapr = { >> &vmstate_spapr_dtb, >> &vmstate_spapr_cap_large_decr, >> &vmstate_spapr_cap_ccf_assist, >> + &vmstate_spapr_machine_check, >> NULL >> } >> }; >> @@ -2948,6 +3006,11 @@ static void spapr_machine_init(MachineState *machine) >> exit(1); >> } >> >> + /* Create the error string for live migration blocker */ >> + error_setg(&spapr->fwnmi_migration_blocker, >> + "A machine check is being handled during migration. The handler" >> + "may run and log hardware error on the destination"); >> + >> /* Register ibm,nmi-register and ibm,nmi-interlock RTAS calls */ >> spapr_fwnmi_register(); >> } >> diff --git a/hw/ppc/spapr_events.c b/hw/ppc/spapr_events.c >> index ecc3d68..71caa03 100644 >> --- a/hw/ppc/spapr_events.c >> +++ b/hw/ppc/spapr_events.c >> @@ -43,6 +43,7 @@ >> #include "qemu/main-loop.h" >> #include "hw/ppc/spapr_ovec.h" >> #include <libfdt.h> >> +#include "migration/blocker.h" >> >> #define RTAS_LOG_VERSION_MASK 0xff000000 >> #define RTAS_LOG_VERSION_6 0x06000000 >> @@ -844,6 +845,8 @@ void spapr_mce_req_event(PowerPCCPU *cpu, bool recovered) >> { >> SpaprMachineState *spapr = SPAPR_MACHINE(qdev_get_machine()); >> CPUState *cs = CPU(cpu); >> + int ret; >> + Error *local_err = NULL; >> >> if (spapr->guest_machine_check_addr == -1) { >> /* >> @@ -873,8 +876,19 @@ void spapr_mce_req_event(PowerPCCPU *cpu, bool recovered) >> return; >> } >> } >> - spapr->mc_status = cpu->vcpu_id; >> >> + ret = migrate_add_blocker(spapr->fwnmi_migration_blocker, &local_err); >> + if (ret == -EBUSY) { >> + /* >> + * We don't want to abort so we let the migration to continue. >> + * In a rare case, the machine check handler will run on the target. >> + * Though this is not preferable, it is better than aborting >> + * the migration or killing the VM. >> + */ >> + warn_report_err(local_err); >> + } >> + >> + spapr->mc_status = cpu->vcpu_id; >> spapr_mce_dispatch_elog(cpu, recovered); >> } >> >> diff --git a/hw/ppc/spapr_rtas.c b/hw/ppc/spapr_rtas.c >> index b569538..c652ec3 100644 >> --- a/hw/ppc/spapr_rtas.c >> +++ b/hw/ppc/spapr_rtas.c >> @@ -50,6 +50,7 @@ >> #include "hw/ppc/fdt.h" >> #include "target/ppc/mmu-hash64.h" >> #include "target/ppc/mmu-book3s-v3.h" >> +#include "migration/blocker.h" >> >> static void rtas_display_character(PowerPCCPU *cpu, SpaprMachineState *spapr, >> uint32_t token, uint32_t nargs, >> @@ -446,6 +447,7 @@ static void rtas_ibm_nmi_interlock(PowerPCCPU *cpu, >> */ >> spapr->mc_status = -1; >> qemu_cond_signal(&spapr->mc_delivery_cond); >> + migrate_del_blocker(spapr->fwnmi_migration_blocker); >> rtas_st(rets, 0, RTAS_OUT_SUCCESS); >> } >> >> diff --git a/include/hw/ppc/spapr.h b/include/hw/ppc/spapr.h >> index dada821..ea7625e 100644 >> --- a/include/hw/ppc/spapr.h >> +++ b/include/hw/ppc/spapr.h >> @@ -217,6 +217,8 @@ struct SpaprMachineState { >> >> unsigned gpu_numa_id; >> SpaprTpmProxy *tpm_proxy; >> + >> + Error *fwnmi_migration_blocker; >> }; >> >> #define H_SUCCESS 0 >> >
diff --git a/hw/ppc/spapr.c b/hw/ppc/spapr.c index 6992b32..a72a4b1 100644 --- a/hw/ppc/spapr.c +++ b/hw/ppc/spapr.c @@ -46,6 +46,7 @@ #include "migration/qemu-file-types.h" #include "migration/global_state.h" #include "migration/register.h" +#include "migration/blocker.h" #include "mmu-hash64.h" #include "mmu-book3s-v3.h" #include "cpu-models.h" @@ -1829,6 +1830,8 @@ static void spapr_machine_reset(MachineState *machine) /* Signal all vCPUs waiting on this condition */ qemu_cond_broadcast(&spapr->mc_delivery_cond); + + migrate_del_blocker(spapr->fwnmi_migration_blocker); } static void spapr_create_nvram(SpaprMachineState *spapr) @@ -2119,6 +2122,60 @@ static const VMStateDescription vmstate_spapr_dtb = { }, }; +static bool spapr_fwnmi_needed(void *opaque) +{ + SpaprMachineState *spapr = (SpaprMachineState *)opaque; + + return spapr->guest_machine_check_addr != -1; +} + +static int spapr_fwnmi_post_load(void *opaque, int version_id) +{ + SpaprMachineState *spapr = (SpaprMachineState *)opaque; + + if (spapr_get_cap(spapr, SPAPR_CAP_FWNMI_MCE) == SPAPR_CAP_ON) { + + if (kvmppc_has_cap_ppc_fwnmi()) { + return 0; + } + + return kvmppc_set_fwnmi(); + } + + return 0; +} + +static int spapr_fwnmi_pre_save(void *opaque) +{ + SpaprMachineState *spapr = (SpaprMachineState *)opaque; + + /* + * With -only-migratable QEMU option, we cannot block migration. + * Hence check if machine check handling is in progress and print + * a warning message. + */ + if (spapr->mc_status != -1) { + warn_report("A machine check is being handled during migration. The" + "handler may run and log hardware error on the destination"); + } + + return 0; +} + +static const VMStateDescription vmstate_spapr_machine_check = { + .name = "spapr_machine_check", + .version_id = 1, + .minimum_version_id = 1, + .needed = spapr_fwnmi_needed, + .post_load = spapr_fwnmi_post_load, + .pre_save = spapr_fwnmi_pre_save, + .fields = (VMStateField[]) { + VMSTATE_UINT64(guest_machine_check_addr, SpaprMachineState), + VMSTATE_INT32(mc_status, SpaprMachineState), + VMSTATE_END_OF_LIST() + }, +}; + static const VMStateDescription vmstate_spapr = { .name = "spapr", .version_id = 3, @@ -2152,6 +2209,7 @@ static const VMStateDescription vmstate_spapr = { &vmstate_spapr_dtb, &vmstate_spapr_cap_large_decr, &vmstate_spapr_cap_ccf_assist, + &vmstate_spapr_machine_check, NULL } }; @@ -2948,6 +3006,11 @@ static void spapr_machine_init(MachineState *machine) exit(1); } + /* Create the error string for live migration blocker */ + error_setg(&spapr->fwnmi_migration_blocker, + "A machine check is being handled during migration. The handler" + "may run and log hardware error on the destination"); + /* Register ibm,nmi-register and ibm,nmi-interlock RTAS calls */ spapr_fwnmi_register(); } diff --git a/hw/ppc/spapr_events.c b/hw/ppc/spapr_events.c index ecc3d68..71caa03 100644 --- a/hw/ppc/spapr_events.c +++ b/hw/ppc/spapr_events.c @@ -43,6 +43,7 @@ #include "qemu/main-loop.h" #include "hw/ppc/spapr_ovec.h" #include <libfdt.h> +#include "migration/blocker.h" #define RTAS_LOG_VERSION_MASK 0xff000000 #define RTAS_LOG_VERSION_6 0x06000000 @@ -844,6 +845,8 @@ void spapr_mce_req_event(PowerPCCPU *cpu, bool recovered) { SpaprMachineState *spapr = SPAPR_MACHINE(qdev_get_machine()); CPUState *cs = CPU(cpu); + int ret; + Error *local_err = NULL; if (spapr->guest_machine_check_addr == -1) { /* @@ -873,8 +876,19 @@ void spapr_mce_req_event(PowerPCCPU *cpu, bool recovered) return; } } - spapr->mc_status = cpu->vcpu_id; + ret = migrate_add_blocker(spapr->fwnmi_migration_blocker, &local_err); + if (ret == -EBUSY) { + /* + * We don't want to abort so we let the migration to continue. + * In a rare case, the machine check handler will run on the target. + * Though this is not preferable, it is better than aborting + * the migration or killing the VM. + */ + warn_report_err(local_err); + } + + spapr->mc_status = cpu->vcpu_id; spapr_mce_dispatch_elog(cpu, recovered); } diff --git a/hw/ppc/spapr_rtas.c b/hw/ppc/spapr_rtas.c index b569538..c652ec3 100644 --- a/hw/ppc/spapr_rtas.c +++ b/hw/ppc/spapr_rtas.c @@ -50,6 +50,7 @@ #include "hw/ppc/fdt.h" #include "target/ppc/mmu-hash64.h" #include "target/ppc/mmu-book3s-v3.h" +#include "migration/blocker.h" static void rtas_display_character(PowerPCCPU *cpu, SpaprMachineState *spapr, uint32_t token, uint32_t nargs, @@ -446,6 +447,7 @@ static void rtas_ibm_nmi_interlock(PowerPCCPU *cpu, */ spapr->mc_status = -1; qemu_cond_signal(&spapr->mc_delivery_cond); + migrate_del_blocker(spapr->fwnmi_migration_blocker); rtas_st(rets, 0, RTAS_OUT_SUCCESS); } diff --git a/include/hw/ppc/spapr.h b/include/hw/ppc/spapr.h index dada821..ea7625e 100644 --- a/include/hw/ppc/spapr.h +++ b/include/hw/ppc/spapr.h @@ -217,6 +217,8 @@ struct SpaprMachineState { unsigned gpu_numa_id; SpaprTpmProxy *tpm_proxy; + + Error *fwnmi_migration_blocker; }; #define H_SUCCESS 0
This patch includes migration support for machine check handling. Especially this patch blocks VM migration requests until the machine check error handling is complete as these errors are specific to the source hardware and is irrelevant on the target hardware. Signed-off-by: Aravinda Prasad <aravinda@linux.vnet.ibm.com> --- hw/ppc/spapr.c | 63 ++++++++++++++++++++++++++++++++++++++++++++++++ hw/ppc/spapr_events.c | 16 +++++++++++- hw/ppc/spapr_rtas.c | 2 ++ include/hw/ppc/spapr.h | 2 ++ 4 files changed, 82 insertions(+), 1 deletion(-)