| Message ID | 1456417362-20652-5-git-send-email-bharata@linux.vnet.ibm.com (mailing list archive) |
|---|---|
| State | New, archived |
| Headers | show |
On Thu, Feb 25, 2016 at 09:52:40PM +0530, Bharata B Rao wrote: > Set up device tree entries for the hotplugged CPU core and use the > exising EPOW event infrastructure to send CPU hotplug notification to > the guest. > > Signed-off-by: Bharata B Rao <bharata@linux.vnet.ibm.com> > --- > hw/ppc/spapr.c | 136 ++++++++++++++++++++++++++++++++++++++++++++++++- > hw/ppc/spapr_events.c | 3 ++ > hw/ppc/spapr_rtas.c | 24 +++++++++ > include/hw/ppc/spapr.h | 1 + > 4 files changed, 163 insertions(+), 1 deletion(-) > > diff --git a/hw/ppc/spapr.c b/hw/ppc/spapr.c > index 1f0d232..780cd00 100644 > --- a/hw/ppc/spapr.c > +++ b/hw/ppc/spapr.c > @@ -603,6 +603,18 @@ static void spapr_populate_cpu_dt(CPUState *cs, void *fdt, int offset, > size_t page_sizes_prop_size; > uint32_t vcpus_per_socket = smp_threads * smp_cores; > uint32_t pft_size_prop[] = {0, cpu_to_be32(spapr->htab_shift)}; > + sPAPRMachineClass *smc = SPAPR_MACHINE_GET_CLASS(qdev_get_machine()); > + sPAPRDRConnector *drc; > + sPAPRDRConnectorClass *drck; > + int drc_index; > + > + if (smc->dr_cpu_enabled) { > + drc = spapr_dr_connector_by_id(SPAPR_DR_CONNECTOR_TYPE_CPU, index); > + g_assert(drc); > + drck = SPAPR_DR_CONNECTOR_GET_CLASS(drc); > + drc_index = drck->get_index(drc); > + _FDT((fdt_setprop_cell(fdt, offset, "ibm,my-drc-index", drc_index))); > + } > > /* Note: we keep CI large pages off for now because a 64K capable guest > * provisioned with large pages might otherwise try to map a qemu > @@ -987,6 +999,16 @@ static void spapr_finalize_fdt(sPAPRMachineState *spapr, > _FDT(spapr_drc_populate_dt(fdt, 0, NULL, SPAPR_DR_CONNECTOR_TYPE_LMB)); > } > > + if (smc->dr_cpu_enabled) { > + int offset = fdt_path_offset(fdt, "/cpus"); > + ret = spapr_drc_populate_dt(fdt, offset, NULL, > + SPAPR_DR_CONNECTOR_TYPE_CPU); > + if (ret < 0) { > + fprintf(stderr, "Couldn't set up CPU DR device tree properties\n"); > + exit(1); > + } > + } > + > _FDT((fdt_pack(fdt))); > > if (fdt_totalsize(fdt) > FDT_MAX_SIZE) { > @@ -1759,6 +1781,7 @@ static void ppc_spapr_init(MachineState *machine) > char *filename; > int spapr_cores = smp_cpus / smp_threads; > int spapr_max_cores = max_cpus / smp_threads; > + int smt = kvmppc_smt_threads(); > > msi_supported = true; > > @@ -1813,6 +1836,15 @@ static void ppc_spapr_init(MachineState *machine) > spapr_validate_node_memory(machine, &error_fatal); > } > > + if (smc->dr_cpu_enabled) { > + for (i = 0; i < spapr_max_cores; i++) { > + sPAPRDRConnector *drc = > + spapr_dr_connector_new(OBJECT(spapr), > + SPAPR_DR_CONNECTOR_TYPE_CPU, i * smt); > + qemu_register_reset(spapr_drc_reset, drc); > + } > + } > + > /* init CPUs */ > if (machine->cpu_model == NULL) { > machine->cpu_model = kvm_enabled() ? "host" : "POWER7"; > @@ -2247,6 +2279,88 @@ out: > error_propagate(errp, local_err); > } > > +static void *spapr_populate_hotplug_cpu_dt(DeviceState *dev, CPUState *cs, > + int *fdt_offset, > + sPAPRMachineState *spapr) > +{ > + PowerPCCPU *cpu = POWERPC_CPU(cs); > + DeviceClass *dc = DEVICE_GET_CLASS(cs); > + int id = ppc_get_vcpu_dt_id(cpu); > + void *fdt; > + int offset, fdt_size; > + char *nodename; > + > + fdt = create_device_tree(&fdt_size); > + nodename = g_strdup_printf("%s@%x", dc->fw_name, id); > + offset = fdt_add_subnode(fdt, 0, nodename); > + > + spapr_populate_cpu_dt(cs, fdt, offset, spapr); > + g_free(nodename); > + > + *fdt_offset = offset; > + return fdt; > +} > + > +static void spapr_core_plug(HotplugHandler *hotplug_dev, DeviceState *dev, > + Error **errp) > +{ > + sPAPRMachineClass *smc = SPAPR_MACHINE_GET_CLASS(qdev_get_machine()); > + sPAPRMachineState *ms = SPAPR_MACHINE(qdev_get_machine()); > + sPAPRCPUCore *core = SPAPR_CPU_CORE(OBJECT(dev)); > + PowerPCCPU *cpu = &core->threads[0]; > + CPUState *cs = CPU(cpu); > + int id = ppc_get_vcpu_dt_id(cpu); > + sPAPRDRConnector *drc = > + spapr_dr_connector_by_id(SPAPR_DR_CONNECTOR_TYPE_CPU, id); > + sPAPRDRConnectorClass *drck; > + Error *local_err = NULL; > + void *fdt = NULL; > + int fdt_offset = 0; > + > + if (!smc->dr_cpu_enabled) { > + /* > + * This is a cold plugged CPU core but the machine doesn't support > + * DR. So skip the hotplug path ensuring that the core is brought > + * up online with out an associated DR connector. > + */ > + return; > + } > + > + g_assert(drc); > + > + /* > + * Setup CPU DT entries only for hotplugged CPUs. For boot time or > + * coldplugged CPUs DT entries are setup in spapr_finalize_fdt(). > + */ > + if (dev->hotplugged) { > + fdt = spapr_populate_hotplug_cpu_dt(dev, cs, &fdt_offset, ms); > + dev->hotplugged = true; > + } > + > + drck = SPAPR_DR_CONNECTOR_GET_CLASS(drc); > + drck->attach(drc, dev, fdt, fdt_offset, !dev->hotplugged, &local_err); > + if (local_err) { > + g_free(fdt); > + error_propagate(errp, local_err); > + return; > + } > + > + if (dev->hotplugged) { > + /* > + * Send hotplug notification interrupt to the guest only in case > + * of hotplugged CPUs. > + */ > + spapr_hotplug_req_add_by_index(drc); > + } else { > + /* > + * Set the right DRC states for cold plugged CPU. > + */ > + drck->set_allocation_state(drc, SPAPR_DR_ALLOCATION_STATE_USABLE); > + drck->set_isolation_state(drc, SPAPR_DR_ISOLATION_STATE_UNISOLATED); > + } > + return; > +} > + > static void spapr_machine_device_plug(HotplugHandler *hotplug_dev, > DeviceState *dev, Error **errp) > { > @@ -2291,8 +2405,25 @@ static void spapr_machine_device_plug(HotplugHandler *hotplug_dev, > } else if (object_dynamic_cast(OBJECT(dev), TYPE_CPU)) { > CPUState *cs = CPU(dev); > PowerPCCPU *cpu = POWERPC_CPU(cs); > + int i; > + > + if (!smc->dr_cpu_enabled && dev->hotplugged) { > + error_setg(errp, "CPU hotplug not supported for this machine"); > + return; > + } > + > + /* Set NUMA node for the added CPUs */ > + for (i = 0; i < nb_numa_nodes; i++) { > + if (test_bit(cs->cpu_index, numa_info[i].node_cpu)) { > + cs->numa_node = i; > + break; > + } > + } > > spapr_cpu_init(ms, cpu, errp); > + spapr_cpu_reset(cpu); It looks to me like all of this setup should be either in the last patch, or in this one, not split between them. > + } else if (object_dynamic_cast(OBJECT(dev), TYPE_SPAPR_CPU_CORE)) { > + spapr_core_plug(hotplug_dev, dev, errp); > } > } > > @@ -2308,7 +2439,8 @@ static HotplugHandler *spapr_get_hotpug_handler(MachineState *machine, > DeviceState *dev) > { > if (object_dynamic_cast(OBJECT(dev), TYPE_PC_DIMM) || > - object_dynamic_cast(OBJECT(dev), TYPE_CPU)) { > + object_dynamic_cast(OBJECT(dev), TYPE_CPU) || > + object_dynamic_cast(OBJECT(dev), TYPE_SPAPR_CPU_CORE)) { > return HOTPLUG_HANDLER(machine); > } > return NULL; > @@ -2352,6 +2484,7 @@ static void spapr_machine_class_init(ObjectClass *oc, void *data) > mc->cpu_index_to_socket_id = spapr_cpu_index_to_socket_id; > > smc->dr_lmb_enabled = true; > + smc->dr_cpu_enabled = true; > fwc->get_dev_path = spapr_get_fw_dev_path; > nc->nmi_monitor_handler = spapr_nmi; > } > @@ -2431,6 +2564,7 @@ static void spapr_machine_2_5_class_options(MachineClass *mc) > > spapr_machine_2_6_class_options(mc); > smc->use_ohci_by_default = true; > + smc->dr_cpu_enabled = false; > SET_MACHINE_COMPAT(mc, SPAPR_COMPAT_2_5); > } > > diff --git a/hw/ppc/spapr_events.c b/hw/ppc/spapr_events.c > index f5eac4b..e50bb16 100644 > --- a/hw/ppc/spapr_events.c > +++ b/hw/ppc/spapr_events.c > @@ -437,6 +437,9 @@ static void spapr_hotplug_req_event(uint8_t hp_id, uint8_t hp_action, > case SPAPR_DR_CONNECTOR_TYPE_LMB: > hp->hotplug_type = RTAS_LOG_V6_HP_TYPE_MEMORY; > break; > + case SPAPR_DR_CONNECTOR_TYPE_CPU: > + hp->hotplug_type = RTAS_LOG_V6_HP_TYPE_CPU; > + break; > default: > /* we shouldn't be signaling hotplug events for resources > * that don't support them > diff --git a/hw/ppc/spapr_rtas.c b/hw/ppc/spapr_rtas.c > index b7c5ebd..cc0369e 100644 > --- a/hw/ppc/spapr_rtas.c > +++ b/hw/ppc/spapr_rtas.c > @@ -34,6 +34,7 @@ > > #include "hw/ppc/spapr.h" > #include "hw/ppc/spapr_vio.h" > +#include "hw/ppc/ppc.h" > #include "qapi-event.h" > #include "hw/boards.h" > > @@ -161,6 +162,27 @@ static void rtas_query_cpu_stopped_state(PowerPCCPU *cpu_, > rtas_st(rets, 0, RTAS_OUT_PARAM_ERROR); > } > > +/* > + * Set the timebase offset of the CPU to that of first CPU. > + * This helps hotplugged CPU to have the correct timebase offset. > + */ > +static void spapr_cpu_update_tb_offset(PowerPCCPU *cpu) > +{ > + PowerPCCPU *fcpu = POWERPC_CPU(first_cpu); > + > + cpu->env.tb_env->tb_offset = fcpu->env.tb_env->tb_offset; > +} > + > +static void spapr_cpu_set_endianness(PowerPCCPU *cpu) > +{ > + PowerPCCPU *fcpu = POWERPC_CPU(first_cpu); > + PowerPCCPUClass *pcc = POWERPC_CPU_GET_CLASS(fcpu); > + > + if (!pcc->interrupts_big_endian(fcpu)) { > + cpu->env.spr[SPR_LPCR] |= LPCR_ILE; > + } > +} > + Any particular reason for doing these things at rtas_start_cpu() time, but other initialization at plug time? Could you consolidate it to one place or the other? > static void rtas_start_cpu(PowerPCCPU *cpu_, sPAPRMachineState *spapr, > uint32_t token, uint32_t nargs, > target_ulong args, > @@ -197,6 +219,8 @@ static void rtas_start_cpu(PowerPCCPU *cpu_, sPAPRMachineState *spapr, > env->nip = start; > env->gpr[3] = r3; > cs->halted = 0; > + spapr_cpu_set_endianness(cpu); > + spapr_cpu_update_tb_offset(cpu); > > qemu_cpu_kick(cs); > > diff --git a/include/hw/ppc/spapr.h b/include/hw/ppc/spapr.h > index 20b3417..eb4ccd1 100644 > --- a/include/hw/ppc/spapr.h > +++ b/include/hw/ppc/spapr.h > @@ -36,6 +36,7 @@ struct sPAPRMachineClass { > > /*< public >*/ > bool dr_lmb_enabled; /* enable dynamic-reconfig/hotplug of LMBs */ > + bool dr_cpu_enabled; /* enable dynamic-reconfig/hotplug of CPUs */ > bool use_ohci_by_default; /* use USB-OHCI instead of XHCI */ > }; >
On 25.02.2016 17:22, Bharata B Rao wrote: > Set up device tree entries for the hotplugged CPU core and use the > exising EPOW event infrastructure to send CPU hotplug notification to > the guest. > > Signed-off-by: Bharata B Rao <bharata@linux.vnet.ibm.com> > --- > hw/ppc/spapr.c | 136 ++++++++++++++++++++++++++++++++++++++++++++++++- > hw/ppc/spapr_events.c | 3 ++ > hw/ppc/spapr_rtas.c | 24 +++++++++ > include/hw/ppc/spapr.h | 1 + > 4 files changed, 163 insertions(+), 1 deletion(-) > > diff --git a/hw/ppc/spapr.c b/hw/ppc/spapr.c > index 1f0d232..780cd00 100644 > --- a/hw/ppc/spapr.c > +++ b/hw/ppc/spapr.c > @@ -603,6 +603,18 @@ static void spapr_populate_cpu_dt(CPUState *cs, void *fdt, int offset, > size_t page_sizes_prop_size; > uint32_t vcpus_per_socket = smp_threads * smp_cores; > uint32_t pft_size_prop[] = {0, cpu_to_be32(spapr->htab_shift)}; > + sPAPRMachineClass *smc = SPAPR_MACHINE_GET_CLASS(qdev_get_machine()); > + sPAPRDRConnector *drc; > + sPAPRDRConnectorClass *drck; > + int drc_index; > + > + if (smc->dr_cpu_enabled) { > + drc = spapr_dr_connector_by_id(SPAPR_DR_CONNECTOR_TYPE_CPU, index); > + g_assert(drc); > + drck = SPAPR_DR_CONNECTOR_GET_CLASS(drc); > + drc_index = drck->get_index(drc); > + _FDT((fdt_setprop_cell(fdt, offset, "ibm,my-drc-index", drc_index))); > + } > > /* Note: we keep CI large pages off for now because a 64K capable guest > * provisioned with large pages might otherwise try to map a qemu > @@ -987,6 +999,16 @@ static void spapr_finalize_fdt(sPAPRMachineState *spapr, > _FDT(spapr_drc_populate_dt(fdt, 0, NULL, SPAPR_DR_CONNECTOR_TYPE_LMB)); > } > > + if (smc->dr_cpu_enabled) { > + int offset = fdt_path_offset(fdt, "/cpus"); > + ret = spapr_drc_populate_dt(fdt, offset, NULL, > + SPAPR_DR_CONNECTOR_TYPE_CPU); > + if (ret < 0) { > + fprintf(stderr, "Couldn't set up CPU DR device tree properties\n"); I think it's better to use error_report() nowadays instead. > + exit(1); > + } > + } > + > _FDT((fdt_pack(fdt))); > > if (fdt_totalsize(fdt) > FDT_MAX_SIZE) { > @@ -1759,6 +1781,7 @@ static void ppc_spapr_init(MachineState *machine) > char *filename; > int spapr_cores = smp_cpus / smp_threads; > int spapr_max_cores = max_cpus / smp_threads; > + int smt = kvmppc_smt_threads(); > > msi_supported = true; > > @@ -1813,6 +1836,15 @@ static void ppc_spapr_init(MachineState *machine) > spapr_validate_node_memory(machine, &error_fatal); > } > > + if (smc->dr_cpu_enabled) { > + for (i = 0; i < spapr_max_cores; i++) { > + sPAPRDRConnector *drc = > + spapr_dr_connector_new(OBJECT(spapr), > + SPAPR_DR_CONNECTOR_TYPE_CPU, i * smt); > + qemu_register_reset(spapr_drc_reset, drc); > + } > + } > + > /* init CPUs */ > if (machine->cpu_model == NULL) { > machine->cpu_model = kvm_enabled() ? "host" : "POWER7"; > @@ -2247,6 +2279,88 @@ out: > error_propagate(errp, local_err); > } > > +static void *spapr_populate_hotplug_cpu_dt(DeviceState *dev, CPUState *cs, > + int *fdt_offset, > + sPAPRMachineState *spapr) > +{ > + PowerPCCPU *cpu = POWERPC_CPU(cs); > + DeviceClass *dc = DEVICE_GET_CLASS(cs); > + int id = ppc_get_vcpu_dt_id(cpu); > + void *fdt; > + int offset, fdt_size; > + char *nodename; > + > + fdt = create_device_tree(&fdt_size); > + nodename = g_strdup_printf("%s@%x", dc->fw_name, id); > + offset = fdt_add_subnode(fdt, 0, nodename); > + > + spapr_populate_cpu_dt(cs, fdt, offset, spapr); > + g_free(nodename); > + > + *fdt_offset = offset; > + return fdt; > +} > + > +static void spapr_core_plug(HotplugHandler *hotplug_dev, DeviceState *dev, > + Error **errp) > +{ > + sPAPRMachineClass *smc = SPAPR_MACHINE_GET_CLASS(qdev_get_machine()); > + sPAPRMachineState *ms = SPAPR_MACHINE(qdev_get_machine()); > + sPAPRCPUCore *core = SPAPR_CPU_CORE(OBJECT(dev)); > + PowerPCCPU *cpu = &core->threads[0]; > + CPUState *cs = CPU(cpu); > + int id = ppc_get_vcpu_dt_id(cpu); > + sPAPRDRConnector *drc = > + spapr_dr_connector_by_id(SPAPR_DR_CONNECTOR_TYPE_CPU, id); > + sPAPRDRConnectorClass *drck; > + Error *local_err = NULL; > + void *fdt = NULL; > + int fdt_offset = 0; > + > + if (!smc->dr_cpu_enabled) { > + /* > + * This is a cold plugged CPU core but the machine doesn't support > + * DR. So skip the hotplug path ensuring that the core is brought > + * up online with out an associated DR connector. > + */ > + return; > + } > + > + g_assert(drc); > + > + /* > + * Setup CPU DT entries only for hotplugged CPUs. For boot time or > + * coldplugged CPUs DT entries are setup in spapr_finalize_fdt(). > + */ > + if (dev->hotplugged) { > + fdt = spapr_populate_hotplug_cpu_dt(dev, cs, &fdt_offset, ms); > + dev->hotplugged = true; That looks strange ... why "dev->hotplugged = true" after you've already checked that with the above if-statement? > + } > + > + drck = SPAPR_DR_CONNECTOR_GET_CLASS(drc); > + drck->attach(drc, dev, fdt, fdt_offset, !dev->hotplugged, &local_err); > + if (local_err) { > + g_free(fdt); > + error_propagate(errp, local_err); > + return; > + } > + > + if (dev->hotplugged) { > + /* > + * Send hotplug notification interrupt to the guest only in case > + * of hotplugged CPUs. > + */ > + spapr_hotplug_req_add_by_index(drc); > + } else { > + /* > + * Set the right DRC states for cold plugged CPU. > + */ > + drck->set_allocation_state(drc, SPAPR_DR_ALLOCATION_STATE_USABLE); > + drck->set_isolation_state(drc, SPAPR_DR_ISOLATION_STATE_UNISOLATED); > + } > + return; Superfluous return statement. > +} > + > static void spapr_machine_device_plug(HotplugHandler *hotplug_dev, > DeviceState *dev, Error **errp) > { > @@ -2291,8 +2405,25 @@ static void spapr_machine_device_plug(HotplugHandler *hotplug_dev, > } else if (object_dynamic_cast(OBJECT(dev), TYPE_CPU)) { > CPUState *cs = CPU(dev); > PowerPCCPU *cpu = POWERPC_CPU(cs); > + int i; > + > + if (!smc->dr_cpu_enabled && dev->hotplugged) { > + error_setg(errp, "CPU hotplug not supported for this machine"); > + return; > + } > + > + /* Set NUMA node for the added CPUs */ > + for (i = 0; i < nb_numa_nodes; i++) { > + if (test_bit(cs->cpu_index, numa_info[i].node_cpu)) { > + cs->numa_node = i; > + break; > + } > + } > > spapr_cpu_init(ms, cpu, errp); > + spapr_cpu_reset(cpu); > + } else if (object_dynamic_cast(OBJECT(dev), TYPE_SPAPR_CPU_CORE)) { > + spapr_core_plug(hotplug_dev, dev, errp); > } > } > > @@ -2308,7 +2439,8 @@ static HotplugHandler *spapr_get_hotpug_handler(MachineState *machine, > DeviceState *dev) > { > if (object_dynamic_cast(OBJECT(dev), TYPE_PC_DIMM) || > - object_dynamic_cast(OBJECT(dev), TYPE_CPU)) { > + object_dynamic_cast(OBJECT(dev), TYPE_CPU) || > + object_dynamic_cast(OBJECT(dev), TYPE_SPAPR_CPU_CORE)) { > return HOTPLUG_HANDLER(machine); > } > return NULL; > @@ -2352,6 +2484,7 @@ static void spapr_machine_class_init(ObjectClass *oc, void *data) > mc->cpu_index_to_socket_id = spapr_cpu_index_to_socket_id; > > smc->dr_lmb_enabled = true; > + smc->dr_cpu_enabled = true; > fwc->get_dev_path = spapr_get_fw_dev_path; > nc->nmi_monitor_handler = spapr_nmi; > } > @@ -2431,6 +2564,7 @@ static void spapr_machine_2_5_class_options(MachineClass *mc) > > spapr_machine_2_6_class_options(mc); > smc->use_ohci_by_default = true; > + smc->dr_cpu_enabled = false; > SET_MACHINE_COMPAT(mc, SPAPR_COMPAT_2_5); > } > Thomas
On Fri, Feb 26, 2016 at 02:03:57PM +0100, Thomas Huth wrote: > On 25.02.2016 17:22, Bharata B Rao wrote: > > Set up device tree entries for the hotplugged CPU core and use the > > exising EPOW event infrastructure to send CPU hotplug notification to > > the guest. > > > > Signed-off-by: Bharata B Rao <bharata@linux.vnet.ibm.com> > > --- > > hw/ppc/spapr.c | 136 ++++++++++++++++++++++++++++++++++++++++++++++++- > > hw/ppc/spapr_events.c | 3 ++ > > hw/ppc/spapr_rtas.c | 24 +++++++++ > > include/hw/ppc/spapr.h | 1 + > > 4 files changed, 163 insertions(+), 1 deletion(-) > > > > diff --git a/hw/ppc/spapr.c b/hw/ppc/spapr.c > > index 1f0d232..780cd00 100644 > > --- a/hw/ppc/spapr.c > > +++ b/hw/ppc/spapr.c > > @@ -603,6 +603,18 @@ static void spapr_populate_cpu_dt(CPUState *cs, void *fdt, int offset, > > size_t page_sizes_prop_size; > > uint32_t vcpus_per_socket = smp_threads * smp_cores; > > uint32_t pft_size_prop[] = {0, cpu_to_be32(spapr->htab_shift)}; > > + sPAPRMachineClass *smc = SPAPR_MACHINE_GET_CLASS(qdev_get_machine()); > > + sPAPRDRConnector *drc; > > + sPAPRDRConnectorClass *drck; > > + int drc_index; > > + > > + if (smc->dr_cpu_enabled) { > > + drc = spapr_dr_connector_by_id(SPAPR_DR_CONNECTOR_TYPE_CPU, index); > > + g_assert(drc); > > + drck = SPAPR_DR_CONNECTOR_GET_CLASS(drc); > > + drc_index = drck->get_index(drc); > > + _FDT((fdt_setprop_cell(fdt, offset, "ibm,my-drc-index", drc_index))); > > + } > > > > /* Note: we keep CI large pages off for now because a 64K capable guest > > * provisioned with large pages might otherwise try to map a qemu > > @@ -987,6 +999,16 @@ static void spapr_finalize_fdt(sPAPRMachineState *spapr, > > _FDT(spapr_drc_populate_dt(fdt, 0, NULL, SPAPR_DR_CONNECTOR_TYPE_LMB)); > > } > > > > + if (smc->dr_cpu_enabled) { > > + int offset = fdt_path_offset(fdt, "/cpus"); > > + ret = spapr_drc_populate_dt(fdt, offset, NULL, > > + SPAPR_DR_CONNECTOR_TYPE_CPU); > > + if (ret < 0) { > > + fprintf(stderr, "Couldn't set up CPU DR device tree properties\n"); > > I think it's better to use error_report() nowadays instead. Yeah, have been carrying this old hunk, will fix. > > > + exit(1); > > + } > > + } > > + > > _FDT((fdt_pack(fdt))); > > > > if (fdt_totalsize(fdt) > FDT_MAX_SIZE) { > > @@ -1759,6 +1781,7 @@ static void ppc_spapr_init(MachineState *machine) > > char *filename; > > int spapr_cores = smp_cpus / smp_threads; > > int spapr_max_cores = max_cpus / smp_threads; > > + int smt = kvmppc_smt_threads(); > > > > msi_supported = true; > > > > @@ -1813,6 +1836,15 @@ static void ppc_spapr_init(MachineState *machine) > > spapr_validate_node_memory(machine, &error_fatal); > > } > > > > + if (smc->dr_cpu_enabled) { > > + for (i = 0; i < spapr_max_cores; i++) { > > + sPAPRDRConnector *drc = > > + spapr_dr_connector_new(OBJECT(spapr), > > + SPAPR_DR_CONNECTOR_TYPE_CPU, i * smt); > > + qemu_register_reset(spapr_drc_reset, drc); > > + } > > + } > > + > > /* init CPUs */ > > if (machine->cpu_model == NULL) { > > machine->cpu_model = kvm_enabled() ? "host" : "POWER7"; > > @@ -2247,6 +2279,88 @@ out: > > error_propagate(errp, local_err); > > } > > > > +static void *spapr_populate_hotplug_cpu_dt(DeviceState *dev, CPUState *cs, > > + int *fdt_offset, > > + sPAPRMachineState *spapr) > > +{ > > + PowerPCCPU *cpu = POWERPC_CPU(cs); > > + DeviceClass *dc = DEVICE_GET_CLASS(cs); > > + int id = ppc_get_vcpu_dt_id(cpu); > > + void *fdt; > > + int offset, fdt_size; > > + char *nodename; > > + > > + fdt = create_device_tree(&fdt_size); > > + nodename = g_strdup_printf("%s@%x", dc->fw_name, id); > > + offset = fdt_add_subnode(fdt, 0, nodename); > > + > > + spapr_populate_cpu_dt(cs, fdt, offset, spapr); > > + g_free(nodename); > > + > > + *fdt_offset = offset; > > + return fdt; > > +} > > + > > +static void spapr_core_plug(HotplugHandler *hotplug_dev, DeviceState *dev, > > + Error **errp) > > +{ > > + sPAPRMachineClass *smc = SPAPR_MACHINE_GET_CLASS(qdev_get_machine()); > > + sPAPRMachineState *ms = SPAPR_MACHINE(qdev_get_machine()); > > + sPAPRCPUCore *core = SPAPR_CPU_CORE(OBJECT(dev)); > > + PowerPCCPU *cpu = &core->threads[0]; > > + CPUState *cs = CPU(cpu); > > + int id = ppc_get_vcpu_dt_id(cpu); > > + sPAPRDRConnector *drc = > > + spapr_dr_connector_by_id(SPAPR_DR_CONNECTOR_TYPE_CPU, id); > > + sPAPRDRConnectorClass *drck; > > + Error *local_err = NULL; > > + void *fdt = NULL; > > + int fdt_offset = 0; > > + > > + if (!smc->dr_cpu_enabled) { > > + /* > > + * This is a cold plugged CPU core but the machine doesn't support > > + * DR. So skip the hotplug path ensuring that the core is brought > > + * up online with out an associated DR connector. > > + */ > > + return; > > + } > > + > > + g_assert(drc); > > + > > + /* > > + * Setup CPU DT entries only for hotplugged CPUs. For boot time or > > + * coldplugged CPUs DT entries are setup in spapr_finalize_fdt(). > > + */ > > + if (dev->hotplugged) { > > + fdt = spapr_populate_hotplug_cpu_dt(dev, cs, &fdt_offset, ms); > > + dev->hotplugged = true; > > That looks strange ... why "dev->hotplugged = true" after you've already > checked that with the above if-statement? Yeah this is a remnant of the implmentation that was based on David's proposal where all the devices were pre-created and I had to set dev->hotplugged manually here. Will remove as it is not needed here in this device_add based implementation. Regards, Bharata.
On Fri, Feb 26, 2016 at 02:51:41PM +1100, David Gibson wrote: > On Thu, Feb 25, 2016 at 09:52:40PM +0530, Bharata B Rao wrote: > > Set up device tree entries for the hotplugged CPU core and use the > > exising EPOW event infrastructure to send CPU hotplug notification to > > the guest. > > > > Signed-off-by: Bharata B Rao <bharata@linux.vnet.ibm.com> > > --- > > hw/ppc/spapr.c | 136 ++++++++++++++++++++++++++++++++++++++++++++++++- > > hw/ppc/spapr_events.c | 3 ++ > > hw/ppc/spapr_rtas.c | 24 +++++++++ > > include/hw/ppc/spapr.h | 1 + > > 4 files changed, 163 insertions(+), 1 deletion(-) > > > > diff --git a/hw/ppc/spapr.c b/hw/ppc/spapr.c > > index 1f0d232..780cd00 100644 > > --- a/hw/ppc/spapr.c > > +++ b/hw/ppc/spapr.c > > @@ -603,6 +603,18 @@ static void spapr_populate_cpu_dt(CPUState *cs, void *fdt, int offset, > > size_t page_sizes_prop_size; > > uint32_t vcpus_per_socket = smp_threads * smp_cores; > > uint32_t pft_size_prop[] = {0, cpu_to_be32(spapr->htab_shift)}; > > + sPAPRMachineClass *smc = SPAPR_MACHINE_GET_CLASS(qdev_get_machine()); > > + sPAPRDRConnector *drc; > > + sPAPRDRConnectorClass *drck; > > + int drc_index; > > + > > + if (smc->dr_cpu_enabled) { > > + drc = spapr_dr_connector_by_id(SPAPR_DR_CONNECTOR_TYPE_CPU, index); > > + g_assert(drc); > > + drck = SPAPR_DR_CONNECTOR_GET_CLASS(drc); > > + drc_index = drck->get_index(drc); > > + _FDT((fdt_setprop_cell(fdt, offset, "ibm,my-drc-index", drc_index))); > > + } > > > > /* Note: we keep CI large pages off for now because a 64K capable guest > > * provisioned with large pages might otherwise try to map a qemu > > @@ -987,6 +999,16 @@ static void spapr_finalize_fdt(sPAPRMachineState *spapr, > > _FDT(spapr_drc_populate_dt(fdt, 0, NULL, SPAPR_DR_CONNECTOR_TYPE_LMB)); > > } > > > > + if (smc->dr_cpu_enabled) { > > + int offset = fdt_path_offset(fdt, "/cpus"); > > + ret = spapr_drc_populate_dt(fdt, offset, NULL, > > + SPAPR_DR_CONNECTOR_TYPE_CPU); > > + if (ret < 0) { > > + fprintf(stderr, "Couldn't set up CPU DR device tree properties\n"); > > + exit(1); > > + } > > + } > > + > > _FDT((fdt_pack(fdt))); > > > > if (fdt_totalsize(fdt) > FDT_MAX_SIZE) { > > @@ -1759,6 +1781,7 @@ static void ppc_spapr_init(MachineState *machine) > > char *filename; > > int spapr_cores = smp_cpus / smp_threads; > > int spapr_max_cores = max_cpus / smp_threads; > > + int smt = kvmppc_smt_threads(); > > > > msi_supported = true; > > > > @@ -1813,6 +1836,15 @@ static void ppc_spapr_init(MachineState *machine) > > spapr_validate_node_memory(machine, &error_fatal); > > } > > > > + if (smc->dr_cpu_enabled) { > > + for (i = 0; i < spapr_max_cores; i++) { > > + sPAPRDRConnector *drc = > > + spapr_dr_connector_new(OBJECT(spapr), > > + SPAPR_DR_CONNECTOR_TYPE_CPU, i * smt); > > + qemu_register_reset(spapr_drc_reset, drc); > > + } > > + } > > + > > /* init CPUs */ > > if (machine->cpu_model == NULL) { > > machine->cpu_model = kvm_enabled() ? "host" : "POWER7"; > > @@ -2247,6 +2279,88 @@ out: > > error_propagate(errp, local_err); > > } > > > > +static void *spapr_populate_hotplug_cpu_dt(DeviceState *dev, CPUState *cs, > > + int *fdt_offset, > > + sPAPRMachineState *spapr) > > +{ > > + PowerPCCPU *cpu = POWERPC_CPU(cs); > > + DeviceClass *dc = DEVICE_GET_CLASS(cs); > > + int id = ppc_get_vcpu_dt_id(cpu); > > + void *fdt; > > + int offset, fdt_size; > > + char *nodename; > > + > > + fdt = create_device_tree(&fdt_size); > > + nodename = g_strdup_printf("%s@%x", dc->fw_name, id); > > + offset = fdt_add_subnode(fdt, 0, nodename); > > + > > + spapr_populate_cpu_dt(cs, fdt, offset, spapr); > > + g_free(nodename); > > + > > + *fdt_offset = offset; > > + return fdt; > > +} > > + > > +static void spapr_core_plug(HotplugHandler *hotplug_dev, DeviceState *dev, > > + Error **errp) > > +{ > > + sPAPRMachineClass *smc = SPAPR_MACHINE_GET_CLASS(qdev_get_machine()); > > + sPAPRMachineState *ms = SPAPR_MACHINE(qdev_get_machine()); > > + sPAPRCPUCore *core = SPAPR_CPU_CORE(OBJECT(dev)); > > + PowerPCCPU *cpu = &core->threads[0]; > > + CPUState *cs = CPU(cpu); > > + int id = ppc_get_vcpu_dt_id(cpu); > > + sPAPRDRConnector *drc = > > + spapr_dr_connector_by_id(SPAPR_DR_CONNECTOR_TYPE_CPU, id); > > + sPAPRDRConnectorClass *drck; > > + Error *local_err = NULL; > > + void *fdt = NULL; > > + int fdt_offset = 0; > > + > > + if (!smc->dr_cpu_enabled) { > > + /* > > + * This is a cold plugged CPU core but the machine doesn't support > > + * DR. So skip the hotplug path ensuring that the core is brought > > + * up online with out an associated DR connector. > > + */ > > + return; > > + } > > + > > + g_assert(drc); > > + > > + /* > > + * Setup CPU DT entries only for hotplugged CPUs. For boot time or > > + * coldplugged CPUs DT entries are setup in spapr_finalize_fdt(). > > + */ > > + if (dev->hotplugged) { > > + fdt = spapr_populate_hotplug_cpu_dt(dev, cs, &fdt_offset, ms); > > + dev->hotplugged = true; > > + } > > + > > + drck = SPAPR_DR_CONNECTOR_GET_CLASS(drc); > > + drck->attach(drc, dev, fdt, fdt_offset, !dev->hotplugged, &local_err); > > + if (local_err) { > > + g_free(fdt); > > + error_propagate(errp, local_err); > > + return; > > + } > > + > > + if (dev->hotplugged) { > > + /* > > + * Send hotplug notification interrupt to the guest only in case > > + * of hotplugged CPUs. > > + */ > > + spapr_hotplug_req_add_by_index(drc); > > + } else { > > + /* > > + * Set the right DRC states for cold plugged CPU. > > + */ > > + drck->set_allocation_state(drc, SPAPR_DR_ALLOCATION_STATE_USABLE); > > + drck->set_isolation_state(drc, SPAPR_DR_ISOLATION_STATE_UNISOLATED); > > + } > > + return; > > +} > > + > > static void spapr_machine_device_plug(HotplugHandler *hotplug_dev, > > DeviceState *dev, Error **errp) > > { > > @@ -2291,8 +2405,25 @@ static void spapr_machine_device_plug(HotplugHandler *hotplug_dev, > > } else if (object_dynamic_cast(OBJECT(dev), TYPE_CPU)) { > > CPUState *cs = CPU(dev); > > PowerPCCPU *cpu = POWERPC_CPU(cs); > > + int i; > > + > > + if (!smc->dr_cpu_enabled && dev->hotplugged) { > > + error_setg(errp, "CPU hotplug not supported for this machine"); > > + return; > > + } > > + > > + /* Set NUMA node for the added CPUs */ > > + for (i = 0; i < nb_numa_nodes; i++) { > > + if (test_bit(cs->cpu_index, numa_info[i].node_cpu)) { > > + cs->numa_node = i; > > + break; > > + } > > + } > > > > spapr_cpu_init(ms, cpu, errp); > > + spapr_cpu_reset(cpu); > > It looks to me like all of this setup should be either in the last > patch, or in this one, not split between them. The way I have done the functional split is like this: - Introduce the spapr-cpu-core device as the 2nd patch in the series - The previous patch converts the boot CPU initialization into spapr-cpu-core creation. So until this point, we can just boot a guest with this CPUs created as core devices. - Then this patch adds the bits needed to actually perform a device_add operation of core device. But I am open to re-arrange things if you feel it is necessary. > > > + } else if (object_dynamic_cast(OBJECT(dev), TYPE_SPAPR_CPU_CORE)) { > > + spapr_core_plug(hotplug_dev, dev, errp); > > } > > } > > > > @@ -2308,7 +2439,8 @@ static HotplugHandler *spapr_get_hotpug_handler(MachineState *machine, > > DeviceState *dev) > > { > > if (object_dynamic_cast(OBJECT(dev), TYPE_PC_DIMM) || > > - object_dynamic_cast(OBJECT(dev), TYPE_CPU)) { > > + object_dynamic_cast(OBJECT(dev), TYPE_CPU) || > > + object_dynamic_cast(OBJECT(dev), TYPE_SPAPR_CPU_CORE)) { > > return HOTPLUG_HANDLER(machine); > > } > > return NULL; > > @@ -2352,6 +2484,7 @@ static void spapr_machine_class_init(ObjectClass *oc, void *data) > > mc->cpu_index_to_socket_id = spapr_cpu_index_to_socket_id; > > > > smc->dr_lmb_enabled = true; > > + smc->dr_cpu_enabled = true; > > fwc->get_dev_path = spapr_get_fw_dev_path; > > nc->nmi_monitor_handler = spapr_nmi; > > } > > @@ -2431,6 +2564,7 @@ static void spapr_machine_2_5_class_options(MachineClass *mc) > > > > spapr_machine_2_6_class_options(mc); > > smc->use_ohci_by_default = true; > > + smc->dr_cpu_enabled = false; > > SET_MACHINE_COMPAT(mc, SPAPR_COMPAT_2_5); > > } > > > > diff --git a/hw/ppc/spapr_events.c b/hw/ppc/spapr_events.c > > index f5eac4b..e50bb16 100644 > > --- a/hw/ppc/spapr_events.c > > +++ b/hw/ppc/spapr_events.c > > @@ -437,6 +437,9 @@ static void spapr_hotplug_req_event(uint8_t hp_id, uint8_t hp_action, > > case SPAPR_DR_CONNECTOR_TYPE_LMB: > > hp->hotplug_type = RTAS_LOG_V6_HP_TYPE_MEMORY; > > break; > > + case SPAPR_DR_CONNECTOR_TYPE_CPU: > > + hp->hotplug_type = RTAS_LOG_V6_HP_TYPE_CPU; > > + break; > > default: > > /* we shouldn't be signaling hotplug events for resources > > * that don't support them > > diff --git a/hw/ppc/spapr_rtas.c b/hw/ppc/spapr_rtas.c > > index b7c5ebd..cc0369e 100644 > > --- a/hw/ppc/spapr_rtas.c > > +++ b/hw/ppc/spapr_rtas.c > > @@ -34,6 +34,7 @@ > > > > #include "hw/ppc/spapr.h" > > #include "hw/ppc/spapr_vio.h" > > +#include "hw/ppc/ppc.h" > > #include "qapi-event.h" > > #include "hw/boards.h" > > > > @@ -161,6 +162,27 @@ static void rtas_query_cpu_stopped_state(PowerPCCPU *cpu_, > > rtas_st(rets, 0, RTAS_OUT_PARAM_ERROR); > > } > > > > +/* > > + * Set the timebase offset of the CPU to that of first CPU. > > + * This helps hotplugged CPU to have the correct timebase offset. > > + */ > > +static void spapr_cpu_update_tb_offset(PowerPCCPU *cpu) > > +{ > > + PowerPCCPU *fcpu = POWERPC_CPU(first_cpu); > > + > > + cpu->env.tb_env->tb_offset = fcpu->env.tb_env->tb_offset; > > +} > > + > > +static void spapr_cpu_set_endianness(PowerPCCPU *cpu) > > +{ > > + PowerPCCPU *fcpu = POWERPC_CPU(first_cpu); > > + PowerPCCPUClass *pcc = POWERPC_CPU_GET_CLASS(fcpu); > > + > > + if (!pcc->interrupts_big_endian(fcpu)) { > > + cpu->env.spr[SPR_LPCR] |= LPCR_ILE; > > + } > > +} > > + > > Any particular reason for doing these things at rtas_start_cpu() time, > but other initialization at plug time? Could you consolidate it to > one place or the other? Those board specific things that are needed to be done have been consolidated into spapr_cpu_init() which will be called from the plug handler. We have discussed this earlier at: https://lists.nongnu.org/archive/html/qemu-devel/2015-02/msg04399.html It has been a while but there was a good reason why setting endianness here rather than in plug handler is necessary. W/o this LE hotplug on guests wouldn't work, I will dig up and come back on what exactly necessiated this change. Regards, Bharata.
On Mon, Feb 29, 2016 at 10:12:10AM +0530, Bharata B Rao wrote: > > > diff --git a/hw/ppc/spapr_rtas.c b/hw/ppc/spapr_rtas.c > > > index b7c5ebd..cc0369e 100644 > > > --- a/hw/ppc/spapr_rtas.c > > > +++ b/hw/ppc/spapr_rtas.c > > > @@ -34,6 +34,7 @@ > > > > > > #include "hw/ppc/spapr.h" > > > #include "hw/ppc/spapr_vio.h" > > > +#include "hw/ppc/ppc.h" > > > #include "qapi-event.h" > > > #include "hw/boards.h" > > > > > > @@ -161,6 +162,27 @@ static void rtas_query_cpu_stopped_state(PowerPCCPU *cpu_, > > > rtas_st(rets, 0, RTAS_OUT_PARAM_ERROR); > > > } > > > > > > +/* > > > + * Set the timebase offset of the CPU to that of first CPU. > > > + * This helps hotplugged CPU to have the correct timebase offset. > > > + */ > > > +static void spapr_cpu_update_tb_offset(PowerPCCPU *cpu) > > > +{ > > > + PowerPCCPU *fcpu = POWERPC_CPU(first_cpu); > > > + > > > + cpu->env.tb_env->tb_offset = fcpu->env.tb_env->tb_offset; > > > +} > > > + > > > +static void spapr_cpu_set_endianness(PowerPCCPU *cpu) > > > +{ > > > + PowerPCCPU *fcpu = POWERPC_CPU(first_cpu); > > > + PowerPCCPUClass *pcc = POWERPC_CPU_GET_CLASS(fcpu); > > > + > > > + if (!pcc->interrupts_big_endian(fcpu)) { > > > + cpu->env.spr[SPR_LPCR] |= LPCR_ILE; > > > + } > > > +} > > > + > > > > Any particular reason for doing these things at rtas_start_cpu() time, > > but other initialization at plug time? Could you consolidate it to > > one place or the other? > > Those board specific things that are needed to be done have been consolidated > into spapr_cpu_init() which will be called from the plug handler. We have > discussed this earlier at: > > https://lists.nongnu.org/archive/html/qemu-devel/2015-02/msg04399.html > > It has been a while but there was a good reason why setting endianness > here rather than in plug handler is necessary. W/o this LE hotplug on guests > wouldn't work, I will dig up and come back on what exactly necessiated > this change. If we set LPCR_ILE in cpu->env.spr[SPR_LPCR] at plug time (from spapr_cpu_init()), there are at least two places later where it gets over-written. One is spapr_cpu_reset() and the other one when kvm_cpu_synchronize_state() is called from rtas_start_cpu(). We could probably issue a kvm_arch_put_registers(), but I found rtas_start_cpu() as a place where this change is guaranteed to get reflected. Regards, Bharata.
On Tue, Mar 01, 2016 at 01:28:56PM +0530, Bharata B Rao wrote: > On Mon, Feb 29, 2016 at 10:12:10AM +0530, Bharata B Rao wrote: > > > > diff --git a/hw/ppc/spapr_rtas.c b/hw/ppc/spapr_rtas.c > > > > index b7c5ebd..cc0369e 100644 > > > > --- a/hw/ppc/spapr_rtas.c > > > > +++ b/hw/ppc/spapr_rtas.c > > > > @@ -34,6 +34,7 @@ > > > > > > > > #include "hw/ppc/spapr.h" > > > > #include "hw/ppc/spapr_vio.h" > > > > +#include "hw/ppc/ppc.h" > > > > #include "qapi-event.h" > > > > #include "hw/boards.h" > > > > > > > > @@ -161,6 +162,27 @@ static void rtas_query_cpu_stopped_state(PowerPCCPU *cpu_, > > > > rtas_st(rets, 0, RTAS_OUT_PARAM_ERROR); > > > > } > > > > > > > > +/* > > > > + * Set the timebase offset of the CPU to that of first CPU. > > > > + * This helps hotplugged CPU to have the correct timebase offset. > > > > + */ > > > > +static void spapr_cpu_update_tb_offset(PowerPCCPU *cpu) > > > > +{ > > > > + PowerPCCPU *fcpu = POWERPC_CPU(first_cpu); > > > > + > > > > + cpu->env.tb_env->tb_offset = fcpu->env.tb_env->tb_offset; > > > > +} > > > > + > > > > +static void spapr_cpu_set_endianness(PowerPCCPU *cpu) > > > > +{ > > > > + PowerPCCPU *fcpu = POWERPC_CPU(first_cpu); > > > > + PowerPCCPUClass *pcc = POWERPC_CPU_GET_CLASS(fcpu); > > > > + > > > > + if (!pcc->interrupts_big_endian(fcpu)) { > > > > + cpu->env.spr[SPR_LPCR] |= LPCR_ILE; > > > > + } > > > > +} > > > > + > > > > > > Any particular reason for doing these things at rtas_start_cpu() time, > > > but other initialization at plug time? Could you consolidate it to > > > one place or the other? > > > > Those board specific things that are needed to be done have been consolidated > > into spapr_cpu_init() which will be called from the plug handler. We have > > discussed this earlier at: > > > > https://lists.nongnu.org/archive/html/qemu-devel/2015-02/msg04399.html > > > > It has been a while but there was a good reason why setting endianness > > here rather than in plug handler is necessary. W/o this LE hotplug on guests > > wouldn't work, I will dig up and come back on what exactly necessiated > > this change. > > If we set LPCR_ILE in cpu->env.spr[SPR_LPCR] at plug time > (from spapr_cpu_init()), there are at least two places later where it gets > over-written. One is spapr_cpu_reset() and the other one when > kvm_cpu_synchronize_state() is called from rtas_start_cpu(). We could > probably issue a kvm_arch_put_registers(), but I found rtas_start_cpu() > as a place where this change is guaranteed to get reflected. Ok, makes sense. In that case can we move all, or nearly all, of the PAPR specific thread initialization to rtas_start_cpu()? Obviously we'd need a separate call to the same stuff for the boot cpu.
diff --git a/hw/ppc/spapr.c b/hw/ppc/spapr.c index 1f0d232..780cd00 100644 --- a/hw/ppc/spapr.c +++ b/hw/ppc/spapr.c @@ -603,6 +603,18 @@ static void spapr_populate_cpu_dt(CPUState *cs, void *fdt, int offset, size_t page_sizes_prop_size; uint32_t vcpus_per_socket = smp_threads * smp_cores; uint32_t pft_size_prop[] = {0, cpu_to_be32(spapr->htab_shift)}; + sPAPRMachineClass *smc = SPAPR_MACHINE_GET_CLASS(qdev_get_machine()); + sPAPRDRConnector *drc; + sPAPRDRConnectorClass *drck; + int drc_index; + + if (smc->dr_cpu_enabled) { + drc = spapr_dr_connector_by_id(SPAPR_DR_CONNECTOR_TYPE_CPU, index); + g_assert(drc); + drck = SPAPR_DR_CONNECTOR_GET_CLASS(drc); + drc_index = drck->get_index(drc); + _FDT((fdt_setprop_cell(fdt, offset, "ibm,my-drc-index", drc_index))); + } /* Note: we keep CI large pages off for now because a 64K capable guest * provisioned with large pages might otherwise try to map a qemu @@ -987,6 +999,16 @@ static void spapr_finalize_fdt(sPAPRMachineState *spapr, _FDT(spapr_drc_populate_dt(fdt, 0, NULL, SPAPR_DR_CONNECTOR_TYPE_LMB)); } + if (smc->dr_cpu_enabled) { + int offset = fdt_path_offset(fdt, "/cpus"); + ret = spapr_drc_populate_dt(fdt, offset, NULL, + SPAPR_DR_CONNECTOR_TYPE_CPU); + if (ret < 0) { + fprintf(stderr, "Couldn't set up CPU DR device tree properties\n"); + exit(1); + } + } + _FDT((fdt_pack(fdt))); if (fdt_totalsize(fdt) > FDT_MAX_SIZE) { @@ -1759,6 +1781,7 @@ static void ppc_spapr_init(MachineState *machine) char *filename; int spapr_cores = smp_cpus / smp_threads; int spapr_max_cores = max_cpus / smp_threads; + int smt = kvmppc_smt_threads(); msi_supported = true; @@ -1813,6 +1836,15 @@ static void ppc_spapr_init(MachineState *machine) spapr_validate_node_memory(machine, &error_fatal); } + if (smc->dr_cpu_enabled) { + for (i = 0; i < spapr_max_cores; i++) { + sPAPRDRConnector *drc = + spapr_dr_connector_new(OBJECT(spapr), + SPAPR_DR_CONNECTOR_TYPE_CPU, i * smt); + qemu_register_reset(spapr_drc_reset, drc); + } + } + /* init CPUs */ if (machine->cpu_model == NULL) { machine->cpu_model = kvm_enabled() ? "host" : "POWER7"; @@ -2247,6 +2279,88 @@ out: error_propagate(errp, local_err); } +static void *spapr_populate_hotplug_cpu_dt(DeviceState *dev, CPUState *cs, + int *fdt_offset, + sPAPRMachineState *spapr) +{ + PowerPCCPU *cpu = POWERPC_CPU(cs); + DeviceClass *dc = DEVICE_GET_CLASS(cs); + int id = ppc_get_vcpu_dt_id(cpu); + void *fdt; + int offset, fdt_size; + char *nodename; + + fdt = create_device_tree(&fdt_size); + nodename = g_strdup_printf("%s@%x", dc->fw_name, id); + offset = fdt_add_subnode(fdt, 0, nodename); + + spapr_populate_cpu_dt(cs, fdt, offset, spapr); + g_free(nodename); + + *fdt_offset = offset; + return fdt; +} + +static void spapr_core_plug(HotplugHandler *hotplug_dev, DeviceState *dev, + Error **errp) +{ + sPAPRMachineClass *smc = SPAPR_MACHINE_GET_CLASS(qdev_get_machine()); + sPAPRMachineState *ms = SPAPR_MACHINE(qdev_get_machine()); + sPAPRCPUCore *core = SPAPR_CPU_CORE(OBJECT(dev)); + PowerPCCPU *cpu = &core->threads[0]; + CPUState *cs = CPU(cpu); + int id = ppc_get_vcpu_dt_id(cpu); + sPAPRDRConnector *drc = + spapr_dr_connector_by_id(SPAPR_DR_CONNECTOR_TYPE_CPU, id); + sPAPRDRConnectorClass *drck; + Error *local_err = NULL; + void *fdt = NULL; + int fdt_offset = 0; + + if (!smc->dr_cpu_enabled) { + /* + * This is a cold plugged CPU core but the machine doesn't support + * DR. So skip the hotplug path ensuring that the core is brought + * up online with out an associated DR connector. + */ + return; + } + + g_assert(drc); + + /* + * Setup CPU DT entries only for hotplugged CPUs. For boot time or + * coldplugged CPUs DT entries are setup in spapr_finalize_fdt(). + */ + if (dev->hotplugged) { + fdt = spapr_populate_hotplug_cpu_dt(dev, cs, &fdt_offset, ms); + dev->hotplugged = true; + } + + drck = SPAPR_DR_CONNECTOR_GET_CLASS(drc); + drck->attach(drc, dev, fdt, fdt_offset, !dev->hotplugged, &local_err); + if (local_err) { + g_free(fdt); + error_propagate(errp, local_err); + return; + } + + if (dev->hotplugged) { + /* + * Send hotplug notification interrupt to the guest only in case + * of hotplugged CPUs. + */ + spapr_hotplug_req_add_by_index(drc); + } else { + /* + * Set the right DRC states for cold plugged CPU. + */ + drck->set_allocation_state(drc, SPAPR_DR_ALLOCATION_STATE_USABLE); + drck->set_isolation_state(drc, SPAPR_DR_ISOLATION_STATE_UNISOLATED); + } + return; +} + static void spapr_machine_device_plug(HotplugHandler *hotplug_dev, DeviceState *dev, Error **errp) { @@ -2291,8 +2405,25 @@ static void spapr_machine_device_plug(HotplugHandler *hotplug_dev, } else if (object_dynamic_cast(OBJECT(dev), TYPE_CPU)) { CPUState *cs = CPU(dev); PowerPCCPU *cpu = POWERPC_CPU(cs); + int i; + + if (!smc->dr_cpu_enabled && dev->hotplugged) { + error_setg(errp, "CPU hotplug not supported for this machine"); + return; + } + + /* Set NUMA node for the added CPUs */ + for (i = 0; i < nb_numa_nodes; i++) { + if (test_bit(cs->cpu_index, numa_info[i].node_cpu)) { + cs->numa_node = i; + break; + } + } spapr_cpu_init(ms, cpu, errp); + spapr_cpu_reset(cpu); + } else if (object_dynamic_cast(OBJECT(dev), TYPE_SPAPR_CPU_CORE)) { + spapr_core_plug(hotplug_dev, dev, errp); } } @@ -2308,7 +2439,8 @@ static HotplugHandler *spapr_get_hotpug_handler(MachineState *machine, DeviceState *dev) { if (object_dynamic_cast(OBJECT(dev), TYPE_PC_DIMM) || - object_dynamic_cast(OBJECT(dev), TYPE_CPU)) { + object_dynamic_cast(OBJECT(dev), TYPE_CPU) || + object_dynamic_cast(OBJECT(dev), TYPE_SPAPR_CPU_CORE)) { return HOTPLUG_HANDLER(machine); } return NULL; @@ -2352,6 +2484,7 @@ static void spapr_machine_class_init(ObjectClass *oc, void *data) mc->cpu_index_to_socket_id = spapr_cpu_index_to_socket_id; smc->dr_lmb_enabled = true; + smc->dr_cpu_enabled = true; fwc->get_dev_path = spapr_get_fw_dev_path; nc->nmi_monitor_handler = spapr_nmi; } @@ -2431,6 +2564,7 @@ static void spapr_machine_2_5_class_options(MachineClass *mc) spapr_machine_2_6_class_options(mc); smc->use_ohci_by_default = true; + smc->dr_cpu_enabled = false; SET_MACHINE_COMPAT(mc, SPAPR_COMPAT_2_5); } diff --git a/hw/ppc/spapr_events.c b/hw/ppc/spapr_events.c index f5eac4b..e50bb16 100644 --- a/hw/ppc/spapr_events.c +++ b/hw/ppc/spapr_events.c @@ -437,6 +437,9 @@ static void spapr_hotplug_req_event(uint8_t hp_id, uint8_t hp_action, case SPAPR_DR_CONNECTOR_TYPE_LMB: hp->hotplug_type = RTAS_LOG_V6_HP_TYPE_MEMORY; break; + case SPAPR_DR_CONNECTOR_TYPE_CPU: + hp->hotplug_type = RTAS_LOG_V6_HP_TYPE_CPU; + break; default: /* we shouldn't be signaling hotplug events for resources * that don't support them diff --git a/hw/ppc/spapr_rtas.c b/hw/ppc/spapr_rtas.c index b7c5ebd..cc0369e 100644 --- a/hw/ppc/spapr_rtas.c +++ b/hw/ppc/spapr_rtas.c @@ -34,6 +34,7 @@ #include "hw/ppc/spapr.h" #include "hw/ppc/spapr_vio.h" +#include "hw/ppc/ppc.h" #include "qapi-event.h" #include "hw/boards.h" @@ -161,6 +162,27 @@ static void rtas_query_cpu_stopped_state(PowerPCCPU *cpu_, rtas_st(rets, 0, RTAS_OUT_PARAM_ERROR); } +/* + * Set the timebase offset of the CPU to that of first CPU. + * This helps hotplugged CPU to have the correct timebase offset. + */ +static void spapr_cpu_update_tb_offset(PowerPCCPU *cpu) +{ + PowerPCCPU *fcpu = POWERPC_CPU(first_cpu); + + cpu->env.tb_env->tb_offset = fcpu->env.tb_env->tb_offset; +} + +static void spapr_cpu_set_endianness(PowerPCCPU *cpu) +{ + PowerPCCPU *fcpu = POWERPC_CPU(first_cpu); + PowerPCCPUClass *pcc = POWERPC_CPU_GET_CLASS(fcpu); + + if (!pcc->interrupts_big_endian(fcpu)) { + cpu->env.spr[SPR_LPCR] |= LPCR_ILE; + } +} + static void rtas_start_cpu(PowerPCCPU *cpu_, sPAPRMachineState *spapr, uint32_t token, uint32_t nargs, target_ulong args, @@ -197,6 +219,8 @@ static void rtas_start_cpu(PowerPCCPU *cpu_, sPAPRMachineState *spapr, env->nip = start; env->gpr[3] = r3; cs->halted = 0; + spapr_cpu_set_endianness(cpu); + spapr_cpu_update_tb_offset(cpu); qemu_cpu_kick(cs); diff --git a/include/hw/ppc/spapr.h b/include/hw/ppc/spapr.h index 20b3417..eb4ccd1 100644 --- a/include/hw/ppc/spapr.h +++ b/include/hw/ppc/spapr.h @@ -36,6 +36,7 @@ struct sPAPRMachineClass { /*< public >*/ bool dr_lmb_enabled; /* enable dynamic-reconfig/hotplug of LMBs */ + bool dr_cpu_enabled; /* enable dynamic-reconfig/hotplug of CPUs */ bool use_ohci_by_default; /* use USB-OHCI instead of XHCI */ };
Set up device tree entries for the hotplugged CPU core and use the exising EPOW event infrastructure to send CPU hotplug notification to the guest. Signed-off-by: Bharata B Rao <bharata@linux.vnet.ibm.com> --- hw/ppc/spapr.c | 136 ++++++++++++++++++++++++++++++++++++++++++++++++- hw/ppc/spapr_events.c | 3 ++ hw/ppc/spapr_rtas.c | 24 +++++++++ include/hw/ppc/spapr.h | 1 + 4 files changed, 163 insertions(+), 1 deletion(-)