diff mbox series

[v5,6/6] hw/arm/virt: Add 'compact-highmem' property

Message ID 20221011231832.149839-7-gshan@redhat.com (mailing list archive)
State New, archived
Headers show
Series hw/arm/virt: Improve address assignment for high memory regions | expand

Commit Message

Gavin Shan Oct. 11, 2022, 11:18 p.m. UTC
After the improvement to high memory region address assignment is
applied, the memory layout can be changed, introducing possible
migration breakage. For example, VIRT_HIGH_PCIE_MMIO memory region
is disabled or enabled when the optimization is applied or not, with
the following configuration.

  pa_bits              = 40;
  vms->highmem_redists = false;
  vms->highmem_ecam    = false;
  vms->highmem_mmio    = true;

  # qemu-system-aarch64 -accel kvm -cpu host    \
    -machine virt-7.2,compact-highmem={on, off} \
    -m 4G,maxmem=511G -monitor stdio

  Region            compact-highmem=off         compact-highmem=on
  ----------------------------------------------------------------
  RAM               [1GB         512GB]        [1GB         512GB]
  HIGH_GIC_REDISTS  [512GB       512GB+64MB]   [disabled]
  HIGH_PCIE_ECAM    [512GB+256MB 512GB+512MB]  [disabled]
  HIGH_PCIE_MMIO    [disabled]                 [512GB       1TB]

In order to keep backwords compatibility, we need to disable the
optimization on machines, which is virt-7.1 or ealier than it. It
means the optimization is enabled by default from virt-7.2. Besides,
'compact-highmem' property is added so that the optimization can be
explicitly enabled or disabled on all machine types by users.

Signed-off-by: Gavin Shan <gshan@redhat.com>
Tested-by: Zhenyu Zhang <zhenyzha@redhat.com>
---
 docs/system/arm/virt.rst |  4 ++++
 hw/arm/virt.c            | 47 ++++++++++++++++++++++++++++++++++++++++
 include/hw/arm/virt.h    |  1 +
 3 files changed, 52 insertions(+)

Comments

Cornelia Huck Oct. 19, 2022, 2 p.m. UTC | #1
On Wed, Oct 12 2022, Gavin Shan <gshan@redhat.com> wrote:

> After the improvement to high memory region address assignment is
> applied, the memory layout can be changed, introducing possible
> migration breakage. For example, VIRT_HIGH_PCIE_MMIO memory region
> is disabled or enabled when the optimization is applied or not, with
> the following configuration.
>
>   pa_bits              = 40;
>   vms->highmem_redists = false;
>   vms->highmem_ecam    = false;
>   vms->highmem_mmio    = true;
>
>   # qemu-system-aarch64 -accel kvm -cpu host    \
>     -machine virt-7.2,compact-highmem={on, off} \
>     -m 4G,maxmem=511G -monitor stdio
>
>   Region            compact-highmem=off         compact-highmem=on
>   ----------------------------------------------------------------
>   RAM               [1GB         512GB]        [1GB         512GB]
>   HIGH_GIC_REDISTS  [512GB       512GB+64MB]   [disabled]
>   HIGH_PCIE_ECAM    [512GB+256MB 512GB+512MB]  [disabled]
>   HIGH_PCIE_MMIO    [disabled]                 [512GB       1TB]
>
> In order to keep backwords compatibility, we need to disable the
> optimization on machines, which is virt-7.1 or ealier than it. It
> means the optimization is enabled by default from virt-7.2. Besides,
> 'compact-highmem' property is added so that the optimization can be
> explicitly enabled or disabled on all machine types by users.
>
> Signed-off-by: Gavin Shan <gshan@redhat.com>
> Tested-by: Zhenyu Zhang <zhenyzha@redhat.com>
> ---
>  docs/system/arm/virt.rst |  4 ++++
>  hw/arm/virt.c            | 47 ++++++++++++++++++++++++++++++++++++++++
>  include/hw/arm/virt.h    |  1 +
>  3 files changed, 52 insertions(+)
>
> diff --git a/docs/system/arm/virt.rst b/docs/system/arm/virt.rst
> index 20442ea2c1..75bf5a4994 100644
> --- a/docs/system/arm/virt.rst
> +++ b/docs/system/arm/virt.rst
> @@ -94,6 +94,10 @@ highmem
>    address space above 32 bits. The default is ``on`` for machine types
>    later than ``virt-2.12``.
>  
> +compact-highmem
> +  Set ``on``/``off`` to enable/disable compact space for high memory regions.

Maybe s/compact space/the compact layout/ ?

> +  The default is ``on`` for machine types later than ``virt-7.2``
> +
>  gic-version
>    Specify the version of the Generic Interrupt Controller (GIC) to provide.
>    Valid values are:
> diff --git a/hw/arm/virt.c b/hw/arm/virt.c
> index c05cfb5314..8f1dba0ece 100644
> --- a/hw/arm/virt.c
> +++ b/hw/arm/virt.c
> @@ -174,6 +174,27 @@ static const MemMapEntry base_memmap[] = {
>   * Note the extended_memmap is sized so that it eventually also includes the
>   * base_memmap entries (VIRT_HIGH_GIC_REDIST2 index is greater than the last
>   * index of base_memmap).
> + *
> + * The addresses assigned to these regions are affected by 'compact-highmem'
> + * property, which is to enable or disable the compact space in the Highmem

s/space in/layout for/ ?

> + * IO regions. For example, VIRT_HIGH_PCIE_MMIO can be disabled or enabled
> + * depending on the property in the following scenario.
> + *
> + * pa_bits              = 40;
> + * vms->highmem_redists = false;
> + * vms->highmem_ecam    = false;
> + * vms->highmem_mmio    = true;
> + *
> + * # qemu-system-aarch64 -accel kvm -cpu host    \
> + *   -machine virt-7.2,compact-highmem={on, off} \
> + *   -m 4G,maxmem=511G -monitor stdio
> + *
> + * Region            compact-highmem=off        compact-highmem=on
> + * ----------------------------------------------------------------
> + * RAM               [1GB         512GB]        [1GB         512GB]
> + * HIGH_GIC_REDISTS  [512GB       512GB+64MB]   [disabled]
> + * HIGH_PCIE_ECAM    [512GB+256GB 512GB+512MB]  [disabled]
> + * HIGH_PCIE_MMIO    [disabled]                 [512GB       1TB]
>   */
>  static MemMapEntry extended_memmap[] = {
>      /* Additional 64 MB redist region (can contain up to 512 redistributors) */

(...)

> @@ -3124,8 +3167,12 @@ DEFINE_VIRT_MACHINE_AS_LATEST(7, 2)
>  
>  static void virt_machine_7_1_options(MachineClass *mc)
>  {
> +    VirtMachineClass *vmc = VIRT_MACHINE_CLASS(OBJECT_CLASS(mc));
> +
>      virt_machine_7_2_options(mc);
>      compat_props_add(mc->compat_props, hw_compat_7_1, hw_compat_7_1_len);
> +    /* Compact space for high memory regions was introduced with 7.2 */

s/space/layout/ ?

> +    vmc->no_highmem_compact = true;
>  }
>  DEFINE_VIRT_MACHINE(7, 1)
>  

Other than the wording nits, lgtm.

Reviewed-by: Cornelia Huck <cohuck@redhat.com>
Eric Auger Oct. 19, 2022, 8:18 p.m. UTC | #2
Hi Gavin,

On 10/12/22 01:18, Gavin Shan wrote:
> After the improvement to high memory region address assignment is
> applied, the memory layout can be changed, introducing possible
> migration breakage. For example, VIRT_HIGH_PCIE_MMIO memory region
> is disabled or enabled when the optimization is applied or not, with
> the following configuration.
>
>   pa_bits              = 40;
>   vms->highmem_redists = false;
>   vms->highmem_ecam    = false;
>   vms->highmem_mmio    = true;
>
>   # qemu-system-aarch64 -accel kvm -cpu host    \
>     -machine virt-7.2,compact-highmem={on, off} \
>     -m 4G,maxmem=511G -monitor stdio
>
>   Region            compact-highmem=off         compact-highmem=on
>   ----------------------------------------------------------------
>   RAM               [1GB         512GB]        [1GB         512GB]
>   HIGH_GIC_REDISTS  [512GB       512GB+64MB]   [disabled]
>   HIGH_PCIE_ECAM    [512GB+256MB 512GB+512MB]  [disabled]
>   HIGH_PCIE_MMIO    [disabled]                 [512GB       1TB]
>
> In order to keep backwords compatibility, we need to disable the
> optimization on machines, which is virt-7.1 or ealier than it. It
> means the optimization is enabled by default from virt-7.2. Besides,
> 'compact-highmem' property is added so that the optimization can be
> explicitly enabled or disabled on all machine types by users.
>
> Signed-off-by: Gavin Shan <gshan@redhat.com>
> Tested-by: Zhenyu Zhang <zhenyzha@redhat.com>
> ---
>  docs/system/arm/virt.rst |  4 ++++
>  hw/arm/virt.c            | 47 ++++++++++++++++++++++++++++++++++++++++
>  include/hw/arm/virt.h    |  1 +
>  3 files changed, 52 insertions(+)
>
> diff --git a/docs/system/arm/virt.rst b/docs/system/arm/virt.rst
> index 20442ea2c1..75bf5a4994 100644
> --- a/docs/system/arm/virt.rst
> +++ b/docs/system/arm/virt.rst
> @@ -94,6 +94,10 @@ highmem
>    address space above 32 bits. The default is ``on`` for machine types
>    later than ``virt-2.12``.
>  
> +compact-highmem
> +  Set ``on``/``off`` to enable/disable compact space for high memory regions.
> +  The default is ``on`` for machine types later than ``virt-7.2``
> +
>  gic-version
>    Specify the version of the Generic Interrupt Controller (GIC) to provide.
>    Valid values are:
> diff --git a/hw/arm/virt.c b/hw/arm/virt.c
> index c05cfb5314..8f1dba0ece 100644
> --- a/hw/arm/virt.c
> +++ b/hw/arm/virt.c
> @@ -174,6 +174,27 @@ static const MemMapEntry base_memmap[] = {
>   * Note the extended_memmap is sized so that it eventually also includes the
>   * base_memmap entries (VIRT_HIGH_GIC_REDIST2 index is greater than the last
>   * index of base_memmap).
> + *
> + * The addresses assigned to these regions are affected by 'compact-highmem'
> + * property, which is to enable or disable the compact space in the Highmem
> + * IO regions. For example, VIRT_HIGH_PCIE_MMIO can be disabled or enabled
> + * depending on the property in the following scenario.
To me you shall rather explain here what is the so-called "compact"
space vs the legacy highmem layout.

If I understand correctly the example rather legitimates the use of a
compat option showing how the layout can be affected by the option. I
would put that in the commit msg instead. Also in your example I see
VIRT_HIGH_GIC_REDISTS is disabled but the code does not disable the
region excpet if it does not fit within the PA. This does not match your
example. Also the region is named VIRT_HIGH_GIC_REDIST2.

In v4, Marc also suggested to have individual options for each highmem
region.
https://lore.kernel.org/qemu-devel/0f8e6a58-0dde-fb80-6966-7bb32c4df552@redhat.com/

Have you considered that option?

Thanks

Eric
> + *
> + * pa_bits              = 40;
> + * vms->highmem_redists = false;
> + * vms->highmem_ecam    = false;
> + * vms->highmem_mmio    = true;
> + *
> + * # qemu-system-aarch64 -accel kvm -cpu host    \
> + *   -machine virt-7.2,compact-highmem={on, off} \
> + *   -m 4G,maxmem=511G -monitor stdio
> + *
> + * Region            compact-highmem=off        compact-highmem=on
> + * ----------------------------------------------------------------
> + * RAM               [1GB         512GB]        [1GB         512GB]
> + * HIGH_GIC_REDISTS  [512GB       512GB+64MB]   [disabled]
> + * HIGH_PCIE_ECAM    [512GB+256GB 512GB+512MB]  [disabled]
> + * HIGH_PCIE_MMIO    [disabled]                 [512GB       1TB]
>   */
>  static MemMapEntry extended_memmap[] = {
>      /* Additional 64 MB redist region (can contain up to 512 redistributors) */
> @@ -2353,6 +2374,20 @@ static void virt_set_highmem(Object *obj, bool value, Error **errp)
>      vms->highmem = value;
>  }
>  
> +static bool virt_get_compact_highmem(Object *obj, Error **errp)
> +{
> +    VirtMachineState *vms = VIRT_MACHINE(obj);
> +
> +    return vms->highmem_compact;
> +}
> +
> +static void virt_set_compact_highmem(Object *obj, bool value, Error **errp)
> +{
> +    VirtMachineState *vms = VIRT_MACHINE(obj);
> +
> +    vms->highmem_compact = value;
> +}
> +
>  static bool virt_get_its(Object *obj, Error **errp)
>  {
>      VirtMachineState *vms = VIRT_MACHINE(obj);
> @@ -2971,6 +3006,13 @@ static void virt_machine_class_init(ObjectClass *oc, void *data)
>                                            "Set on/off to enable/disable using "
>                                            "physical address space above 32 bits");
>  
> +    object_class_property_add_bool(oc, "compact-highmem",
> +                                   virt_get_compact_highmem,
> +                                   virt_set_compact_highmem);
> +    object_class_property_set_description(oc, "compact-highmem",
> +                                          "Set on/off to enable/disable compact "
> +                                          "space for high memory regions");
> +
>      object_class_property_add_str(oc, "gic-version", virt_get_gic_version,
>                                    virt_set_gic_version);
>      object_class_property_set_description(oc, "gic-version",
> @@ -3055,6 +3097,7 @@ static void virt_instance_init(Object *obj)
>  
>      /* High memory is enabled by default */
>      vms->highmem = true;
> +    vms->highmem_compact = !vmc->no_highmem_compact;
>      vms->gic_version = VIRT_GIC_VERSION_NOSEL;
>  
>      vms->highmem_ecam = !vmc->no_highmem_ecam;
> @@ -3124,8 +3167,12 @@ DEFINE_VIRT_MACHINE_AS_LATEST(7, 2)
>  
>  static void virt_machine_7_1_options(MachineClass *mc)
>  {
> +    VirtMachineClass *vmc = VIRT_MACHINE_CLASS(OBJECT_CLASS(mc));
> +
>      virt_machine_7_2_options(mc);
>      compat_props_add(mc->compat_props, hw_compat_7_1, hw_compat_7_1_len);
> +    /* Compact space for high memory regions was introduced with 7.2 */
> +    vmc->no_highmem_compact = true;
>  }
>  DEFINE_VIRT_MACHINE(7, 1)
>  
> diff --git a/include/hw/arm/virt.h b/include/hw/arm/virt.h
> index 709f623741..c7dd59d7f1 100644
> --- a/include/hw/arm/virt.h
> +++ b/include/hw/arm/virt.h
> @@ -125,6 +125,7 @@ struct VirtMachineClass {
>      bool no_pmu;
>      bool claim_edge_triggered_timers;
>      bool smbios_old_sys_ver;
> +    bool no_highmem_compact;
>      bool no_highmem_ecam;
>      bool no_ged;   /* Machines < 4.2 have no support for ACPI GED device */
>      bool kvm_no_adjvtime;
Gavin Shan Oct. 19, 2022, 11:08 p.m. UTC | #3
Hi Connie,

On 10/19/22 10:00 PM, Cornelia Huck wrote:
> On Wed, Oct 12 2022, Gavin Shan <gshan@redhat.com> wrote:
> 
>> After the improvement to high memory region address assignment is
>> applied, the memory layout can be changed, introducing possible
>> migration breakage. For example, VIRT_HIGH_PCIE_MMIO memory region
>> is disabled or enabled when the optimization is applied or not, with
>> the following configuration.
>>
>>    pa_bits              = 40;
>>    vms->highmem_redists = false;
>>    vms->highmem_ecam    = false;
>>    vms->highmem_mmio    = true;
>>
>>    # qemu-system-aarch64 -accel kvm -cpu host    \
>>      -machine virt-7.2,compact-highmem={on, off} \
>>      -m 4G,maxmem=511G -monitor stdio
>>
>>    Region            compact-highmem=off         compact-highmem=on
>>    ----------------------------------------------------------------
>>    RAM               [1GB         512GB]        [1GB         512GB]
>>    HIGH_GIC_REDISTS  [512GB       512GB+64MB]   [disabled]
>>    HIGH_PCIE_ECAM    [512GB+256MB 512GB+512MB]  [disabled]
>>    HIGH_PCIE_MMIO    [disabled]                 [512GB       1TB]
>>
>> In order to keep backwords compatibility, we need to disable the
>> optimization on machines, which is virt-7.1 or ealier than it. It
>> means the optimization is enabled by default from virt-7.2. Besides,
>> 'compact-highmem' property is added so that the optimization can be
>> explicitly enabled or disabled on all machine types by users.
>>
>> Signed-off-by: Gavin Shan <gshan@redhat.com>
>> Tested-by: Zhenyu Zhang <zhenyzha@redhat.com>
>> ---
>>   docs/system/arm/virt.rst |  4 ++++
>>   hw/arm/virt.c            | 47 ++++++++++++++++++++++++++++++++++++++++
>>   include/hw/arm/virt.h    |  1 +
>>   3 files changed, 52 insertions(+)
>>
>> diff --git a/docs/system/arm/virt.rst b/docs/system/arm/virt.rst
>> index 20442ea2c1..75bf5a4994 100644
>> --- a/docs/system/arm/virt.rst
>> +++ b/docs/system/arm/virt.rst
>> @@ -94,6 +94,10 @@ highmem
>>     address space above 32 bits. The default is ``on`` for machine types
>>     later than ``virt-2.12``.
>>   
>> +compact-highmem
>> +  Set ``on``/``off`` to enable/disable compact space for high memory regions.
> 
> Maybe s/compact space/the compact layout/ ?
> 

Yeah, 'compact layout' is better. I will amend in next respin.

>> +  The default is ``on`` for machine types later than ``virt-7.2``
>> +
>>   gic-version
>>     Specify the version of the Generic Interrupt Controller (GIC) to provide.
>>     Valid values are:
>> diff --git a/hw/arm/virt.c b/hw/arm/virt.c
>> index c05cfb5314..8f1dba0ece 100644
>> --- a/hw/arm/virt.c
>> +++ b/hw/arm/virt.c
>> @@ -174,6 +174,27 @@ static const MemMapEntry base_memmap[] = {
>>    * Note the extended_memmap is sized so that it eventually also includes the
>>    * base_memmap entries (VIRT_HIGH_GIC_REDIST2 index is greater than the last
>>    * index of base_memmap).
>> + *
>> + * The addresses assigned to these regions are affected by 'compact-highmem'
>> + * property, which is to enable or disable the compact space in the Highmem
> 
> s/space in/layout for/ ?
> 

Agreed.

>> + * IO regions. For example, VIRT_HIGH_PCIE_MMIO can be disabled or enabled
>> + * depending on the property in the following scenario.
>> + *
>> + * pa_bits              = 40;
>> + * vms->highmem_redists = false;
>> + * vms->highmem_ecam    = false;
>> + * vms->highmem_mmio    = true;
>> + *
>> + * # qemu-system-aarch64 -accel kvm -cpu host    \
>> + *   -machine virt-7.2,compact-highmem={on, off} \
>> + *   -m 4G,maxmem=511G -monitor stdio
>> + *
>> + * Region            compact-highmem=off        compact-highmem=on
>> + * ----------------------------------------------------------------
>> + * RAM               [1GB         512GB]        [1GB         512GB]
>> + * HIGH_GIC_REDISTS  [512GB       512GB+64MB]   [disabled]
>> + * HIGH_PCIE_ECAM    [512GB+256GB 512GB+512MB]  [disabled]
>> + * HIGH_PCIE_MMIO    [disabled]                 [512GB       1TB]
>>    */
>>   static MemMapEntry extended_memmap[] = {
>>       /* Additional 64 MB redist region (can contain up to 512 redistributors) */
> 
> (...)
> 
>> @@ -3124,8 +3167,12 @@ DEFINE_VIRT_MACHINE_AS_LATEST(7, 2)
>>   
>>   static void virt_machine_7_1_options(MachineClass *mc)
>>   {
>> +    VirtMachineClass *vmc = VIRT_MACHINE_CLASS(OBJECT_CLASS(mc));
>> +
>>       virt_machine_7_2_options(mc);
>>       compat_props_add(mc->compat_props, hw_compat_7_1, hw_compat_7_1_len);
>> +    /* Compact space for high memory regions was introduced with 7.2 */
> 
> s/space/layout/ ?
> 

Ack.

>> +    vmc->no_highmem_compact = true;
>>   }
>>   DEFINE_VIRT_MACHINE(7, 1)
>>   
> 
> Other than the wording nits, lgtm.
> 
> Reviewed-by: Cornelia Huck <cohuck@redhat.com>
> 

Thanks,
Gavin
Gavin Shan Oct. 19, 2022, 11:57 p.m. UTC | #4
Hi Eric,

On 10/20/22 4:18 AM, Eric Auger wrote:
> On 10/12/22 01:18, Gavin Shan wrote:
>> After the improvement to high memory region address assignment is
>> applied, the memory layout can be changed, introducing possible
>> migration breakage. For example, VIRT_HIGH_PCIE_MMIO memory region
>> is disabled or enabled when the optimization is applied or not, with
>> the following configuration.
>>
>>    pa_bits              = 40;
>>    vms->highmem_redists = false;
>>    vms->highmem_ecam    = false;
>>    vms->highmem_mmio    = true;
>>
>>    # qemu-system-aarch64 -accel kvm -cpu host    \
>>      -machine virt-7.2,compact-highmem={on, off} \
>>      -m 4G,maxmem=511G -monitor stdio
>>
>>    Region            compact-highmem=off         compact-highmem=on
>>    ----------------------------------------------------------------
>>    RAM               [1GB         512GB]        [1GB         512GB]
>>    HIGH_GIC_REDISTS  [512GB       512GB+64MB]   [disabled]
>>    HIGH_PCIE_ECAM    [512GB+256MB 512GB+512MB]  [disabled]
>>    HIGH_PCIE_MMIO    [disabled]                 [512GB       1TB]
>>
>> In order to keep backwords compatibility, we need to disable the
>> optimization on machines, which is virt-7.1 or ealier than it. It
>> means the optimization is enabled by default from virt-7.2. Besides,
>> 'compact-highmem' property is added so that the optimization can be
>> explicitly enabled or disabled on all machine types by users.
>>
>> Signed-off-by: Gavin Shan <gshan@redhat.com>
>> Tested-by: Zhenyu Zhang <zhenyzha@redhat.com>
>> ---
>>   docs/system/arm/virt.rst |  4 ++++
>>   hw/arm/virt.c            | 47 ++++++++++++++++++++++++++++++++++++++++
>>   include/hw/arm/virt.h    |  1 +
>>   3 files changed, 52 insertions(+)
>>
>> diff --git a/docs/system/arm/virt.rst b/docs/system/arm/virt.rst
>> index 20442ea2c1..75bf5a4994 100644
>> --- a/docs/system/arm/virt.rst
>> +++ b/docs/system/arm/virt.rst
>> @@ -94,6 +94,10 @@ highmem
>>     address space above 32 bits. The default is ``on`` for machine types
>>     later than ``virt-2.12``.
>>   
>> +compact-highmem
>> +  Set ``on``/``off`` to enable/disable compact space for high memory regions.
>> +  The default is ``on`` for machine types later than ``virt-7.2``
>> +
>>   gic-version
>>     Specify the version of the Generic Interrupt Controller (GIC) to provide.
>>     Valid values are:
>> diff --git a/hw/arm/virt.c b/hw/arm/virt.c
>> index c05cfb5314..8f1dba0ece 100644
>> --- a/hw/arm/virt.c
>> +++ b/hw/arm/virt.c
>> @@ -174,6 +174,27 @@ static const MemMapEntry base_memmap[] = {
>>    * Note the extended_memmap is sized so that it eventually also includes the
>>    * base_memmap entries (VIRT_HIGH_GIC_REDIST2 index is greater than the last
>>    * index of base_memmap).
>> + *
>> + * The addresses assigned to these regions are affected by 'compact-highmem'
>> + * property, which is to enable or disable the compact space in the Highmem
>> + * IO regions. For example, VIRT_HIGH_PCIE_MMIO can be disabled or enabled
>> + * depending on the property in the following scenario.
> To me you shall rather explain here what is the so-called "compact"
> space vs the legacy highmem layout.
> 
> If I understand correctly the example rather legitimates the use of a
> compat option showing how the layout can be affected by the option. I
> would put that in the commit msg instead. Also in your example I see
> VIRT_HIGH_GIC_REDISTS is disabled but the code does not disable the
> region excpet if it does not fit within the PA. This does not match your
> example. Also the region is named VIRT_HIGH_GIC_REDIST2.
> 
> In v4, Marc also suggested to have individual options for each highmem
> region.
> https://lore.kernel.org/qemu-devel/0f8e6a58-0dde-fb80-6966-7bb32c4df552@redhat.com/
> 
> Have you considered that option?
> 

I think your comments make sense to me. So lets put the following comments
to the code and move the example to commit log.

   /*
    * The memory map for these Highmem IO Regions can be in legacy or compact
    * layout, depending on 'compact-highmem' property. In legacy layout, the
    * PA space for one specific region is always reserved, even the region has
    * been disabled or doesn't fit into the PA space. However, the PA space for
    * the region won't be reserved in these circumstances.
    */

You're correct about the example. VIRT_HIGH_GIC_REDIST2 should be used. Besides,
the configuration is only achievable by modifying source code at present, until
Marc's suggestion rolls in to allow users disable one particular high memory
regions by more properties. I will amend the commit log to have something like
below.

     For example, VIRT_HIGH_PCIE_MMIO memory region is disabled or enabled when
     the optimization is applied or not, with the following configuration. The
     configuration is only achievable by modifying source code, until more properties
     are added to allow user selectively disable those high memory regions.

For Marc's suggestion to add properties so that these high memory regions can
be disabled by users. I can add one patch after this one to introduce the following
3 properties. Could you please confirm the property names are good enough? It's
nice if Marc can help to confirm before I'm going to work on next revision.

     "highmem-ecam":    "on"/"off" on vms->highmem_ecam
     "highmem-mmio":    "on"/"off" on vms->highmem_mmio
     "highmem-redists": "on"/"off" on vms->highmem_redists

>> + *
>> + * pa_bits              = 40;
>> + * vms->highmem_redists = false;
>> + * vms->highmem_ecam    = false;
>> + * vms->highmem_mmio    = true;
>> + *
>> + * # qemu-system-aarch64 -accel kvm -cpu host    \
>> + *   -machine virt-7.2,compact-highmem={on, off} \
>> + *   -m 4G,maxmem=511G -monitor stdio
>> + *
>> + * Region            compact-highmem=off        compact-highmem=on
>> + * ----------------------------------------------------------------
>> + * RAM               [1GB         512GB]        [1GB         512GB]
>> + * HIGH_GIC_REDISTS  [512GB       512GB+64MB]   [disabled]
>> + * HIGH_PCIE_ECAM    [512GB+256GB 512GB+512MB]  [disabled]
>> + * HIGH_PCIE_MMIO    [disabled]                 [512GB       1TB]
>>    */
>>   static MemMapEntry extended_memmap[] = {
>>       /* Additional 64 MB redist region (can contain up to 512 redistributors) */
>> @@ -2353,6 +2374,20 @@ static void virt_set_highmem(Object *obj, bool value, Error **errp)
>>       vms->highmem = value;
>>   }
>>   
>> +static bool virt_get_compact_highmem(Object *obj, Error **errp)
>> +{
>> +    VirtMachineState *vms = VIRT_MACHINE(obj);
>> +
>> +    return vms->highmem_compact;
>> +}
>> +
>> +static void virt_set_compact_highmem(Object *obj, bool value, Error **errp)
>> +{
>> +    VirtMachineState *vms = VIRT_MACHINE(obj);
>> +
>> +    vms->highmem_compact = value;
>> +}
>> +
>>   static bool virt_get_its(Object *obj, Error **errp)
>>   {
>>       VirtMachineState *vms = VIRT_MACHINE(obj);
>> @@ -2971,6 +3006,13 @@ static void virt_machine_class_init(ObjectClass *oc, void *data)
>>                                             "Set on/off to enable/disable using "
>>                                             "physical address space above 32 bits");
>>   
>> +    object_class_property_add_bool(oc, "compact-highmem",
>> +                                   virt_get_compact_highmem,
>> +                                   virt_set_compact_highmem);
>> +    object_class_property_set_description(oc, "compact-highmem",
>> +                                          "Set on/off to enable/disable compact "
>> +                                          "space for high memory regions");
>> +
>>       object_class_property_add_str(oc, "gic-version", virt_get_gic_version,
>>                                     virt_set_gic_version);
>>       object_class_property_set_description(oc, "gic-version",
>> @@ -3055,6 +3097,7 @@ static void virt_instance_init(Object *obj)
>>   
>>       /* High memory is enabled by default */
>>       vms->highmem = true;
>> +    vms->highmem_compact = !vmc->no_highmem_compact;
>>       vms->gic_version = VIRT_GIC_VERSION_NOSEL;
>>   
>>       vms->highmem_ecam = !vmc->no_highmem_ecam;
>> @@ -3124,8 +3167,12 @@ DEFINE_VIRT_MACHINE_AS_LATEST(7, 2)
>>   
>>   static void virt_machine_7_1_options(MachineClass *mc)
>>   {
>> +    VirtMachineClass *vmc = VIRT_MACHINE_CLASS(OBJECT_CLASS(mc));
>> +
>>       virt_machine_7_2_options(mc);
>>       compat_props_add(mc->compat_props, hw_compat_7_1, hw_compat_7_1_len);
>> +    /* Compact space for high memory regions was introduced with 7.2 */
>> +    vmc->no_highmem_compact = true;
>>   }
>>   DEFINE_VIRT_MACHINE(7, 1)
>>   
>> diff --git a/include/hw/arm/virt.h b/include/hw/arm/virt.h
>> index 709f623741..c7dd59d7f1 100644
>> --- a/include/hw/arm/virt.h
>> +++ b/include/hw/arm/virt.h
>> @@ -125,6 +125,7 @@ struct VirtMachineClass {
>>       bool no_pmu;
>>       bool claim_edge_triggered_timers;
>>       bool smbios_old_sys_ver;
>> +    bool no_highmem_compact;
>>       bool no_highmem_ecam;
>>       bool no_ged;   /* Machines < 4.2 have no support for ACPI GED device */
>>       bool kvm_no_adjvtime;
> 

Thanks,
Gavin
Marc Zyngier Oct. 20, 2022, 9:44 a.m. UTC | #5
On Thu, 20 Oct 2022 00:57:32 +0100,
Gavin Shan <gshan@redhat.com> wrote:
> 
> For Marc's suggestion to add properties so that these high memory
> regions can be disabled by users. I can add one patch after this one
> to introduce the following 3 properties. Could you please confirm
> the property names are good enough? It's nice if Marc can help to
> confirm before I'm going to work on next revision.
> 
>     "highmem-ecam":    "on"/"off" on vms->highmem_ecam
>     "highmem-mmio":    "on"/"off" on vms->highmem_mmio
>     "highmem-redists": "on"/"off" on vms->highmem_redists

I think that'd be reasonable, and would give the user some actual
control over what gets exposed in the highmem region.

I guess that the annoying thing with these options is that they allow
the user to request conflicting settings (256 CPUs and
highmem-redists=off, for example). You'll need to make this fail more
or less gracefully.

Thanks,

	M.
Gavin Shan Oct. 20, 2022, 11:13 a.m. UTC | #6
Hi Marc,

On 10/20/22 5:44 PM, Marc Zyngier wrote:
> On Thu, 20 Oct 2022 00:57:32 +0100,
> Gavin Shan <gshan@redhat.com> wrote:
>>
>> For Marc's suggestion to add properties so that these high memory
>> regions can be disabled by users. I can add one patch after this one
>> to introduce the following 3 properties. Could you please confirm
>> the property names are good enough? It's nice if Marc can help to
>> confirm before I'm going to work on next revision.
>>
>>      "highmem-ecam":    "on"/"off" on vms->highmem_ecam
>>      "highmem-mmio":    "on"/"off" on vms->highmem_mmio
>>      "highmem-redists": "on"/"off" on vms->highmem_redists
> 
> I think that'd be reasonable, and would give the user some actual
> control over what gets exposed in the highmem region.
> 
> I guess that the annoying thing with these options is that they allow
> the user to request conflicting settings (256 CPUs and
> highmem-redists=off, for example). You'll need to make this fail more
> or less gracefully.
> 

Thanks for the quick confirm. The check is already existing in machvirt_init().
I think what we need is simply to calculate 'virt_max_cpus' with consideration
to 'highmem-redists' property there.

     if (max_cpus > virt_max_cpus) {
         error_report("Number of SMP CPUs requested (%d) exceeds max CPUs "
                      "supported by machine 'mach-virt' (%d)",
                      max_cpus, virt_max_cpus);
         exit(1);
     }

Thanks,
Gavin
diff mbox series

Patch

diff --git a/docs/system/arm/virt.rst b/docs/system/arm/virt.rst
index 20442ea2c1..75bf5a4994 100644
--- a/docs/system/arm/virt.rst
+++ b/docs/system/arm/virt.rst
@@ -94,6 +94,10 @@  highmem
   address space above 32 bits. The default is ``on`` for machine types
   later than ``virt-2.12``.
 
+compact-highmem
+  Set ``on``/``off`` to enable/disable compact space for high memory regions.
+  The default is ``on`` for machine types later than ``virt-7.2``
+
 gic-version
   Specify the version of the Generic Interrupt Controller (GIC) to provide.
   Valid values are:
diff --git a/hw/arm/virt.c b/hw/arm/virt.c
index c05cfb5314..8f1dba0ece 100644
--- a/hw/arm/virt.c
+++ b/hw/arm/virt.c
@@ -174,6 +174,27 @@  static const MemMapEntry base_memmap[] = {
  * Note the extended_memmap is sized so that it eventually also includes the
  * base_memmap entries (VIRT_HIGH_GIC_REDIST2 index is greater than the last
  * index of base_memmap).
+ *
+ * The addresses assigned to these regions are affected by 'compact-highmem'
+ * property, which is to enable or disable the compact space in the Highmem
+ * IO regions. For example, VIRT_HIGH_PCIE_MMIO can be disabled or enabled
+ * depending on the property in the following scenario.
+ *
+ * pa_bits              = 40;
+ * vms->highmem_redists = false;
+ * vms->highmem_ecam    = false;
+ * vms->highmem_mmio    = true;
+ *
+ * # qemu-system-aarch64 -accel kvm -cpu host    \
+ *   -machine virt-7.2,compact-highmem={on, off} \
+ *   -m 4G,maxmem=511G -monitor stdio
+ *
+ * Region            compact-highmem=off        compact-highmem=on
+ * ----------------------------------------------------------------
+ * RAM               [1GB         512GB]        [1GB         512GB]
+ * HIGH_GIC_REDISTS  [512GB       512GB+64MB]   [disabled]
+ * HIGH_PCIE_ECAM    [512GB+256GB 512GB+512MB]  [disabled]
+ * HIGH_PCIE_MMIO    [disabled]                 [512GB       1TB]
  */
 static MemMapEntry extended_memmap[] = {
     /* Additional 64 MB redist region (can contain up to 512 redistributors) */
@@ -2353,6 +2374,20 @@  static void virt_set_highmem(Object *obj, bool value, Error **errp)
     vms->highmem = value;
 }
 
+static bool virt_get_compact_highmem(Object *obj, Error **errp)
+{
+    VirtMachineState *vms = VIRT_MACHINE(obj);
+
+    return vms->highmem_compact;
+}
+
+static void virt_set_compact_highmem(Object *obj, bool value, Error **errp)
+{
+    VirtMachineState *vms = VIRT_MACHINE(obj);
+
+    vms->highmem_compact = value;
+}
+
 static bool virt_get_its(Object *obj, Error **errp)
 {
     VirtMachineState *vms = VIRT_MACHINE(obj);
@@ -2971,6 +3006,13 @@  static void virt_machine_class_init(ObjectClass *oc, void *data)
                                           "Set on/off to enable/disable using "
                                           "physical address space above 32 bits");
 
+    object_class_property_add_bool(oc, "compact-highmem",
+                                   virt_get_compact_highmem,
+                                   virt_set_compact_highmem);
+    object_class_property_set_description(oc, "compact-highmem",
+                                          "Set on/off to enable/disable compact "
+                                          "space for high memory regions");
+
     object_class_property_add_str(oc, "gic-version", virt_get_gic_version,
                                   virt_set_gic_version);
     object_class_property_set_description(oc, "gic-version",
@@ -3055,6 +3097,7 @@  static void virt_instance_init(Object *obj)
 
     /* High memory is enabled by default */
     vms->highmem = true;
+    vms->highmem_compact = !vmc->no_highmem_compact;
     vms->gic_version = VIRT_GIC_VERSION_NOSEL;
 
     vms->highmem_ecam = !vmc->no_highmem_ecam;
@@ -3124,8 +3167,12 @@  DEFINE_VIRT_MACHINE_AS_LATEST(7, 2)
 
 static void virt_machine_7_1_options(MachineClass *mc)
 {
+    VirtMachineClass *vmc = VIRT_MACHINE_CLASS(OBJECT_CLASS(mc));
+
     virt_machine_7_2_options(mc);
     compat_props_add(mc->compat_props, hw_compat_7_1, hw_compat_7_1_len);
+    /* Compact space for high memory regions was introduced with 7.2 */
+    vmc->no_highmem_compact = true;
 }
 DEFINE_VIRT_MACHINE(7, 1)
 
diff --git a/include/hw/arm/virt.h b/include/hw/arm/virt.h
index 709f623741..c7dd59d7f1 100644
--- a/include/hw/arm/virt.h
+++ b/include/hw/arm/virt.h
@@ -125,6 +125,7 @@  struct VirtMachineClass {
     bool no_pmu;
     bool claim_edge_triggered_timers;
     bool smbios_old_sys_ver;
+    bool no_highmem_compact;
     bool no_highmem_ecam;
     bool no_ged;   /* Machines < 4.2 have no support for ACPI GED device */
     bool kvm_no_adjvtime;