diff mbox

[RFC,v2,9/9] spapr: CPU hot unplug support

Message ID 1457672078-17307-10-git-send-email-bharata@linux.vnet.ibm.com (mailing list archive)
State New, archived
Headers show

Commit Message

Bharata B Rao March 11, 2016, 4:54 a.m. UTC 
Remove the CPU core device by removing the underlying CPU thread devices.
Hot removal of CPU for sPAPR guests is achieved by sending the hot unplug
notification to the guest. Release the vCPU object after CPU hot unplug so
that vCPU fd can be parked and reused.

Signed-off-by: Bharata B Rao <bharata@linux.vnet.ibm.com>
---
 hw/ppc/spapr.c                  | 21 ++++++++++
 hw/ppc/spapr_cpu_core.c         | 86 +++++++++++++++++++++++++++++++++++++++++
 include/hw/ppc/spapr.h          |  1 +
 include/hw/ppc/spapr_cpu_core.h | 12 ++++++
 4 files changed, 120 insertions(+)

Comments

David Gibson March 16, 2016, 5:27 a.m. UTC | #1 
On Fri, Mar 11, 2016 at 10:24:38AM +0530, Bharata B Rao wrote:
> Remove the CPU core device by removing the underlying CPU thread devices.
> Hot removal of CPU for sPAPR guests is achieved by sending the hot unplug
> notification to the guest. Release the vCPU object after CPU hot unplug so
> that vCPU fd can be parked and reused.
> 
> Signed-off-by: Bharata B Rao <bharata@linux.vnet.ibm.com>
> ---
>  hw/ppc/spapr.c                  | 21 ++++++++++
>  hw/ppc/spapr_cpu_core.c         | 86 +++++++++++++++++++++++++++++++++++++++++
>  include/hw/ppc/spapr.h          |  1 +
>  include/hw/ppc/spapr_cpu_core.h | 12 ++++++
>  4 files changed, 120 insertions(+)
> 
> diff --git a/hw/ppc/spapr.c b/hw/ppc/spapr.c
> index 822c87d..b1e9ba2 100644
> --- a/hw/ppc/spapr.c
> +++ b/hw/ppc/spapr.c
> @@ -2345,7 +2345,12 @@ static void spapr_machine_device_plug(HotplugHandler *hotplug_dev,
>  
>          spapr_memory_plug(hotplug_dev, dev, node, errp);
>      } else if (object_dynamic_cast(OBJECT(dev), TYPE_SPAPR_CPU_CORE)) {
> +        /*
> +         * TODO: Move this check to pre_plug handler at which point
> +         * spapr_core_release() won't be necessary.
> +         */
>          if (!smc->dr_cpu_enabled && dev->hotplugged) {
> +            spapr_core_release(dev);
>              error_setg(errp, "CPU hotplug not supported for this machine");
>              return;
>          }

This hunk doesn't look like its related to unplug.  Did it belong in
another patch?

> @@ -2353,11 +2358,27 @@ static void spapr_machine_device_plug(HotplugHandler *hotplug_dev,
>      }
>  }
>  
> +void spapr_cpu_destroy(PowerPCCPU *cpu)
> +{
> +    sPAPRMachineState *spapr = SPAPR_MACHINE(qdev_get_machine());
> +
> +    xics_cpu_destroy(spapr->icp, cpu);
> +    qemu_unregister_reset(spapr_cpu_reset, cpu);
> +}
> +
>  static void spapr_machine_device_unplug(HotplugHandler *hotplug_dev,
>                                        DeviceState *dev, Error **errp)
>  {
> +    sPAPRMachineClass *smc = SPAPR_MACHINE_GET_CLASS(qdev_get_machine());
> +
>      if (object_dynamic_cast(OBJECT(dev), TYPE_PC_DIMM)) {
>          error_setg(errp, "Memory hot unplug not supported by sPAPR");
> +    } else if (object_dynamic_cast(OBJECT(dev), TYPE_SPAPR_CPU_CORE)) {
> +        if (!smc->dr_cpu_enabled) {
> +            error_setg(errp, "CPU hot unplug not supported on this machine");
> +            return;
> +        }
> +        spapr_core_unplug(hotplug_dev, dev, errp);
>      }
>  }
>  
> diff --git a/hw/ppc/spapr_cpu_core.c b/hw/ppc/spapr_cpu_core.c
> index db8de32..dd391bd 100644
> --- a/hw/ppc/spapr_cpu_core.c
> +++ b/hw/ppc/spapr_cpu_core.c
> @@ -84,6 +84,92 @@ void spapr_core_plug(HotplugHandler *hotplug_dev, DeviceState *dev,
>      }
>  }
>  
> +static void spapr_cpu_core_cleanup(struct sPAPRCPUUnplugList *unplug_list)
> +{
> +    sPAPRCPUUnplug *unplug, *next;
> +    Object *cpu;
> +
> +    QLIST_FOREACH_SAFE(unplug, unplug_list, node, next) {
> +        cpu = unplug->cpu;
> +        object_unparent(cpu);
> +        QLIST_REMOVE(unplug, node);
> +        g_free(unplug);
> +    }
> +}
> +
> +static void spapr_add_cpu_to_unplug_list(Object *cpu,
> +                                         struct sPAPRCPUUnplugList *unplug_list)
> +{
> +    sPAPRCPUUnplug *unplug = g_malloc(sizeof(*unplug));
> +
> +    unplug->cpu = cpu;
> +    QLIST_INSERT_HEAD(unplug_list, unplug, node);
> +}
> +
> +static int spapr_cpu_release(Object *obj, void *opaque)
> +{
> +    DeviceState *dev = DEVICE(obj);
> +    CPUState *cs = CPU(dev);
> +    PowerPCCPU *cpu = POWERPC_CPU(cs);
> +    struct sPAPRCPUUnplugList *unplug_list = opaque;
> +
> +    spapr_cpu_destroy(cpu);
> +    cpu_remove_sync(cs);
> +
> +    /*
> +     * We are still walking the core object's children list, and
> +     * hence can't cleanup this CPU thread object just yet. Put
> +     * it on a list for later removal.
> +     */
> +    spapr_add_cpu_to_unplug_list(obj, unplug_list);
> +    return 0;
> +}
> +
> +void spapr_core_release(DeviceState *dev)
> +{
> +    struct sPAPRCPUUnplugList unplug_list;
> +    sPAPRMachineState *spapr = SPAPR_MACHINE(qdev_get_machine());
> +    sPAPRCPUCore *core = SPAPR_CPU_CORE(OBJECT(dev));
> +    int core_dt_id = object_property_get_int(OBJECT(dev), "core", NULL);
> +    int smt = kvmppc_smt_threads();
> +
> +    QLIST_INIT(&unplug_list);
> +    object_child_foreach(OBJECT(dev), spapr_cpu_release, &unplug_list);
> +    spapr_cpu_core_cleanup(&unplug_list);
> +    spapr->cores[core_dt_id / smt] = NULL;
> +
> +    g_free(core->threads);
> +}
> +
> +static void spapr_core_release_unparent(DeviceState *dev, void *opaque)
> +{
> +    spapr_core_release(dev);
> +    object_unparent(OBJECT(dev));
> +}
> +
> +void spapr_core_unplug(HotplugHandler *hotplug_dev, DeviceState *dev,
> +                       Error **errp)
> +{
> +    sPAPRCPUCore *core = SPAPR_CPU_CORE(OBJECT(dev));
> +    PowerPCCPU *cpu = &core->threads[0];
> +    int id = ppc_get_vcpu_dt_id(cpu);
> +    sPAPRDRConnector *drc =
> +        spapr_dr_connector_by_id(SPAPR_DR_CONNECTOR_TYPE_CPU, id);
> +    sPAPRDRConnectorClass *drck;
> +    Error *local_err = NULL;
> +
> +    g_assert(drc);
> +
> +    drck = SPAPR_DR_CONNECTOR_GET_CLASS(drc);
> +    drck->detach(drc, dev, spapr_core_release_unparent, NULL, &local_err);
> +    if (local_err) {
> +        error_propagate(errp, local_err);
> +        return;
> +    }
> +
> +    spapr_hotplug_req_remove_by_index(drc);
> +}
> +
>  static void spapr_cpu_core_create_threads(sPAPRCPUCore *core, int threads,
>                                            Error **errp)
>  {
> diff --git a/include/hw/ppc/spapr.h b/include/hw/ppc/spapr.h
> index 8957072..41d0928 100644
> --- a/include/hw/ppc/spapr.h
> +++ b/include/hw/ppc/spapr.h
> @@ -591,6 +591,7 @@ void spapr_cpu_init(sPAPRMachineState *spapr, PowerPCCPU *cpu, Error **errp);
>  void spapr_cpu_reset(void *opaque);
>  void *spapr_populate_hotplug_cpu_dt(DeviceState *dev, CPUState *cs,
>                                      int *fdt_offset, sPAPRMachineState *spapr);
> +void spapr_cpu_destroy(PowerPCCPU *cpu);
>  
>  /* rtas-configure-connector state */
>  struct sPAPRConfigureConnectorState {
> diff --git a/include/hw/ppc/spapr_cpu_core.h b/include/hw/ppc/spapr_cpu_core.h
> index 980d8ae..75c6c58 100644
> --- a/include/hw/ppc/spapr_cpu_core.h
> +++ b/include/hw/ppc/spapr_cpu_core.h
> @@ -27,4 +27,16 @@ typedef struct sPAPRCPUCore {
>  
>  void spapr_core_plug(HotplugHandler *hotplug_dev, DeviceState *dev,
>                       Error **errp);
> +void spapr_core_unplug(HotplugHandler *hotplug_dev, DeviceState *dev,
> +                       Error **errp);
> +void spapr_core_release(DeviceState *dev);
> +
> +/* List to store unplugged CPU objects for cleanup during unplug */
> +typedef struct sPAPRCPUUnplug {
> +    Object *cpu;
> +    QLIST_ENTRY(sPAPRCPUUnplug) node;
> +} sPAPRCPUUnplug;
> +
> +QLIST_HEAD(sPAPRCPUUnplugList, sPAPRCPUUnplug);
> +
>  #endif
Bharata B Rao March 16, 2016, 5:37 a.m. UTC | #2 
On Wed, Mar 16, 2016 at 04:27:04PM +1100, David Gibson wrote:
> On Fri, Mar 11, 2016 at 10:24:38AM +0530, Bharata B Rao wrote:
> > Remove the CPU core device by removing the underlying CPU thread devices.
> > Hot removal of CPU for sPAPR guests is achieved by sending the hot unplug
> > notification to the guest. Release the vCPU object after CPU hot unplug so
> > that vCPU fd can be parked and reused.
> > 
> > Signed-off-by: Bharata B Rao <bharata@linux.vnet.ibm.com>
> > ---
> >  hw/ppc/spapr.c                  | 21 ++++++++++
> >  hw/ppc/spapr_cpu_core.c         | 86 +++++++++++++++++++++++++++++++++++++++++
> >  include/hw/ppc/spapr.h          |  1 +
> >  include/hw/ppc/spapr_cpu_core.h | 12 ++++++
> >  4 files changed, 120 insertions(+)
> > 
> > diff --git a/hw/ppc/spapr.c b/hw/ppc/spapr.c
> > index 822c87d..b1e9ba2 100644
> > --- a/hw/ppc/spapr.c
> > +++ b/hw/ppc/spapr.c
> > @@ -2345,7 +2345,12 @@ static void spapr_machine_device_plug(HotplugHandler *hotplug_dev,
> >  
> >          spapr_memory_plug(hotplug_dev, dev, node, errp);
> >      } else if (object_dynamic_cast(OBJECT(dev), TYPE_SPAPR_CPU_CORE)) {
> > +        /*
> > +         * TODO: Move this check to pre_plug handler at which point
> > +         * spapr_core_release() won't be necessary.
> > +         */
> >          if (!smc->dr_cpu_enabled && dev->hotplugged) {
> > +            spapr_core_release(dev);
> >              error_setg(errp, "CPU hotplug not supported for this machine");
> >              return;
> >          }
> 
> This hunk doesn't look like its related to unplug.  Did it belong in
> another patch?

Yes, it actually belongs to hot-plug but the whole infrastructure to release
the core and associated threads get introduced in this patch, hence put
this hunk here.

However, if pre_plug is the way to go forward, we woudn't need this
altogether as roll-back is much easier from there than from here.

Regards,
Bharata.
diff mbox

Patch

diff --git a/hw/ppc/spapr.c b/hw/ppc/spapr.c
index 822c87d..b1e9ba2 100644
--- a/hw/ppc/spapr.c
+++ b/hw/ppc/spapr.c
@@ -2345,7 +2345,12 @@  static void spapr_machine_device_plug(HotplugHandler *hotplug_dev,
 
         spapr_memory_plug(hotplug_dev, dev, node, errp);
     } else if (object_dynamic_cast(OBJECT(dev), TYPE_SPAPR_CPU_CORE)) {
+        /*
+         * TODO: Move this check to pre_plug handler at which point
+         * spapr_core_release() won't be necessary.
+         */
         if (!smc->dr_cpu_enabled && dev->hotplugged) {
+            spapr_core_release(dev);
             error_setg(errp, "CPU hotplug not supported for this machine");
             return;
         }
@@ -2353,11 +2358,27 @@  static void spapr_machine_device_plug(HotplugHandler *hotplug_dev,
     }
 }
 
+void spapr_cpu_destroy(PowerPCCPU *cpu)
+{
+    sPAPRMachineState *spapr = SPAPR_MACHINE(qdev_get_machine());
+
+    xics_cpu_destroy(spapr->icp, cpu);
+    qemu_unregister_reset(spapr_cpu_reset, cpu);
+}
+
 static void spapr_machine_device_unplug(HotplugHandler *hotplug_dev,
                                       DeviceState *dev, Error **errp)
 {
+    sPAPRMachineClass *smc = SPAPR_MACHINE_GET_CLASS(qdev_get_machine());
+
     if (object_dynamic_cast(OBJECT(dev), TYPE_PC_DIMM)) {
         error_setg(errp, "Memory hot unplug not supported by sPAPR");
+    } else if (object_dynamic_cast(OBJECT(dev), TYPE_SPAPR_CPU_CORE)) {
+        if (!smc->dr_cpu_enabled) {
+            error_setg(errp, "CPU hot unplug not supported on this machine");
+            return;
+        }
+        spapr_core_unplug(hotplug_dev, dev, errp);
     }
 }
 
diff --git a/hw/ppc/spapr_cpu_core.c b/hw/ppc/spapr_cpu_core.c
index db8de32..dd391bd 100644
--- a/hw/ppc/spapr_cpu_core.c
+++ b/hw/ppc/spapr_cpu_core.c
@@ -84,6 +84,92 @@  void spapr_core_plug(HotplugHandler *hotplug_dev, DeviceState *dev,
     }
 }
 
+static void spapr_cpu_core_cleanup(struct sPAPRCPUUnplugList *unplug_list)
+{
+    sPAPRCPUUnplug *unplug, *next;
+    Object *cpu;
+
+    QLIST_FOREACH_SAFE(unplug, unplug_list, node, next) {
+        cpu = unplug->cpu;
+        object_unparent(cpu);
+        QLIST_REMOVE(unplug, node);
+        g_free(unplug);
+    }
+}
+
+static void spapr_add_cpu_to_unplug_list(Object *cpu,
+                                         struct sPAPRCPUUnplugList *unplug_list)
+{
+    sPAPRCPUUnplug *unplug = g_malloc(sizeof(*unplug));
+
+    unplug->cpu = cpu;
+    QLIST_INSERT_HEAD(unplug_list, unplug, node);
+}
+
+static int spapr_cpu_release(Object *obj, void *opaque)
+{
+    DeviceState *dev = DEVICE(obj);
+    CPUState *cs = CPU(dev);
+    PowerPCCPU *cpu = POWERPC_CPU(cs);
+    struct sPAPRCPUUnplugList *unplug_list = opaque;
+
+    spapr_cpu_destroy(cpu);
+    cpu_remove_sync(cs);
+
+    /*
+     * We are still walking the core object's children list, and
+     * hence can't cleanup this CPU thread object just yet. Put
+     * it on a list for later removal.
+     */
+    spapr_add_cpu_to_unplug_list(obj, unplug_list);
+    return 0;
+}
+
+void spapr_core_release(DeviceState *dev)
+{
+    struct sPAPRCPUUnplugList unplug_list;
+    sPAPRMachineState *spapr = SPAPR_MACHINE(qdev_get_machine());
+    sPAPRCPUCore *core = SPAPR_CPU_CORE(OBJECT(dev));
+    int core_dt_id = object_property_get_int(OBJECT(dev), "core", NULL);
+    int smt = kvmppc_smt_threads();
+
+    QLIST_INIT(&unplug_list);
+    object_child_foreach(OBJECT(dev), spapr_cpu_release, &unplug_list);
+    spapr_cpu_core_cleanup(&unplug_list);
+    spapr->cores[core_dt_id / smt] = NULL;
+
+    g_free(core->threads);
+}
+
+static void spapr_core_release_unparent(DeviceState *dev, void *opaque)
+{
+    spapr_core_release(dev);
+    object_unparent(OBJECT(dev));
+}
+
+void spapr_core_unplug(HotplugHandler *hotplug_dev, DeviceState *dev,
+                       Error **errp)
+{
+    sPAPRCPUCore *core = SPAPR_CPU_CORE(OBJECT(dev));
+    PowerPCCPU *cpu = &core->threads[0];
+    int id = ppc_get_vcpu_dt_id(cpu);
+    sPAPRDRConnector *drc =
+        spapr_dr_connector_by_id(SPAPR_DR_CONNECTOR_TYPE_CPU, id);
+    sPAPRDRConnectorClass *drck;
+    Error *local_err = NULL;
+
+    g_assert(drc);
+
+    drck = SPAPR_DR_CONNECTOR_GET_CLASS(drc);
+    drck->detach(drc, dev, spapr_core_release_unparent, NULL, &local_err);
+    if (local_err) {
+        error_propagate(errp, local_err);
+        return;
+    }
+
+    spapr_hotplug_req_remove_by_index(drc);
+}
+
 static void spapr_cpu_core_create_threads(sPAPRCPUCore *core, int threads,
                                           Error **errp)
 {
diff --git a/include/hw/ppc/spapr.h b/include/hw/ppc/spapr.h
index 8957072..41d0928 100644
--- a/include/hw/ppc/spapr.h
+++ b/include/hw/ppc/spapr.h
@@ -591,6 +591,7 @@  void spapr_cpu_init(sPAPRMachineState *spapr, PowerPCCPU *cpu, Error **errp);
 void spapr_cpu_reset(void *opaque);
 void *spapr_populate_hotplug_cpu_dt(DeviceState *dev, CPUState *cs,
                                     int *fdt_offset, sPAPRMachineState *spapr);
+void spapr_cpu_destroy(PowerPCCPU *cpu);
 
 /* rtas-configure-connector state */
 struct sPAPRConfigureConnectorState {
diff --git a/include/hw/ppc/spapr_cpu_core.h b/include/hw/ppc/spapr_cpu_core.h
index 980d8ae..75c6c58 100644
--- a/include/hw/ppc/spapr_cpu_core.h
+++ b/include/hw/ppc/spapr_cpu_core.h
@@ -27,4 +27,16 @@  typedef struct sPAPRCPUCore {
 
 void spapr_core_plug(HotplugHandler *hotplug_dev, DeviceState *dev,
                      Error **errp);
+void spapr_core_unplug(HotplugHandler *hotplug_dev, DeviceState *dev,
+                       Error **errp);
+void spapr_core_release(DeviceState *dev);
+
+/* List to store unplugged CPU objects for cleanup during unplug */
+typedef struct sPAPRCPUUnplug {
+    Object *cpu;
+    QLIST_ENTRY(sPAPRCPUUnplug) node;
+} sPAPRCPUUnplug;
+
+QLIST_HEAD(sPAPRCPUUnplugList, sPAPRCPUUnplug);
+
 #endif