diff mbox series

[RFC,v3,08/16] KVM: arm64: Add a new VCPU device control group for SPE

Message ID 20201027172705.15181-9-alexandru.elisei@arm.com (mailing list archive)
State New, archived
Headers show
Series KVM: arm64: Add Statistical Profiling Extension (SPE) support | expand

Commit Message

Alexandru Elisei Oct. 27, 2020, 5:26 p.m. UTC 
From: Sudeep Holla <sudeep.holla@arm.com>

To configure the virtual SPE buffer management interrupt number, we use a
VCPU kvm_device ioctl, encapsulating the KVM_ARM_VCPU_SPE_IRQ attribute
within the KVM_ARM_VCPU_SPE_CTRL group.

After configuring the SPE, userspace is required to call the VCPU ioctl
with the attribute KVM_ARM_VCPU_SPE_INIT to initialize SPE on the VCPU.

[Alexandru E: Fixed compilation errors, don't allow userspace to set the
	VCPU feature, removed unused functions, fixed mismatched
	descriptions, comments and error codes, reworked logic, rebased on
	top of v5.10-rc1]

Signed-off-by: Sudeep Holla <sudeep.holla@arm.com>
Signed-off-by: Alexandru Elisei <alexandru.elisei@arm.com>
---
 Documentation/virt/kvm/devices/vcpu.rst |  40 ++++++++
 arch/arm64/include/uapi/asm/kvm.h       |   3 +
 arch/arm64/kvm/Makefile                 |   1 +
 arch/arm64/kvm/guest.c                  |   9 ++
 arch/arm64/kvm/reset.c                  |  23 +++++
 arch/arm64/kvm/spe.c                    | 129 ++++++++++++++++++++++++
 include/kvm/arm_spe.h                   |  27 +++++
 include/uapi/linux/kvm.h                |   1 +
 8 files changed, 233 insertions(+)
 create mode 100644 arch/arm64/kvm/spe.c

Comments

Haibo Xu Nov. 5, 2020, 9:58 a.m. UTC | #1 
On Wed, 28 Oct 2020 at 01:26, Alexandru Elisei <alexandru.elisei@arm.com> wrote:
>
> From: Sudeep Holla <sudeep.holla@arm.com>
>
> To configure the virtual SPE buffer management interrupt number, we use a
> VCPU kvm_device ioctl, encapsulating the KVM_ARM_VCPU_SPE_IRQ attribute
> within the KVM_ARM_VCPU_SPE_CTRL group.
>
> After configuring the SPE, userspace is required to call the VCPU ioctl
> with the attribute KVM_ARM_VCPU_SPE_INIT to initialize SPE on the VCPU.
>
> [Alexandru E: Fixed compilation errors, don't allow userspace to set the
>         VCPU feature, removed unused functions, fixed mismatched
>         descriptions, comments and error codes, reworked logic, rebased on
>         top of v5.10-rc1]
>
> Signed-off-by: Sudeep Holla <sudeep.holla@arm.com>
> Signed-off-by: Alexandru Elisei <alexandru.elisei@arm.com>
> ---
>  Documentation/virt/kvm/devices/vcpu.rst |  40 ++++++++
>  arch/arm64/include/uapi/asm/kvm.h       |   3 +
>  arch/arm64/kvm/Makefile                 |   1 +
>  arch/arm64/kvm/guest.c                  |   9 ++
>  arch/arm64/kvm/reset.c                  |  23 +++++
>  arch/arm64/kvm/spe.c                    | 129 ++++++++++++++++++++++++
>  include/kvm/arm_spe.h                   |  27 +++++
>  include/uapi/linux/kvm.h                |   1 +
>  8 files changed, 233 insertions(+)
>  create mode 100644 arch/arm64/kvm/spe.c
>
> diff --git a/Documentation/virt/kvm/devices/vcpu.rst b/Documentation/virt/kvm/devices/vcpu.rst
> index 2acec3b9ef65..6135b9827fbe 100644
> --- a/Documentation/virt/kvm/devices/vcpu.rst
> +++ b/Documentation/virt/kvm/devices/vcpu.rst
> @@ -161,3 +161,43 @@ Specifies the base address of the stolen time structure for this VCPU. The
>  base address must be 64 byte aligned and exist within a valid guest memory
>  region. See Documentation/virt/kvm/arm/pvtime.rst for more information
>  including the layout of the stolen time structure.
> +
> +4. GROUP: KVM_ARM_VCPU_SPE_CTRL
> +===============================
> +
> +:Architectures: ARM64
> +
> +4.1 ATTRIBUTE: KVM_ARM_VCPU_SPE_IRQ
> +-----------------------------------
> +
> +:Parameters: in kvm_device_attr.addr the address for the SPE buffer management
> +             interrupt is a pointer to an int
> +
> +Returns:
> +
> +        =======  ========================================================
> +        -EBUSY   The SPE buffer management interrupt is already set
> +        -EINVAL  Invalid SPE overflow interrupt number
> +        -EFAULT  Could not read the buffer management interrupt number
> +        -ENXIO   SPE not supported or not properly configured
> +        =======  ========================================================
> +
> +A value describing the SPE (Statistical Profiling Extension) overflow interrupt
> +number for this vcpu. This interrupt should be a PPI and the interrupt type and
> +number must be same for each vcpu.
> +
> +4.2 ATTRIBUTE: KVM_ARM_VCPU_SPE_INIT
> +------------------------------------
> +
> +:Parameters: no additional parameter in kvm_device_attr.addr
> +
> +Returns:
> +
> +        =======  ======================================================
> +        -EBUSY   SPE already initialized
> +        -ENODEV  GIC not initialized
> +        -ENXIO   SPE not supported or not properly configured
> +        =======  ======================================================
> +
> +Request the initialization of the SPE. Must be done after initializing the
> +in-kernel irqchip and after setting the interrupt number for the VCPU.
> diff --git a/arch/arm64/include/uapi/asm/kvm.h b/arch/arm64/include/uapi/asm/kvm.h
> index 489e12304dbb..ca57dfb7abf0 100644
> --- a/arch/arm64/include/uapi/asm/kvm.h
> +++ b/arch/arm64/include/uapi/asm/kvm.h
> @@ -360,6 +360,9 @@ struct kvm_vcpu_events {
>  #define   KVM_ARM_VCPU_TIMER_IRQ_PTIMER                1
>  #define KVM_ARM_VCPU_PVTIME_CTRL       2
>  #define   KVM_ARM_VCPU_PVTIME_IPA      0
> +#define KVM_ARM_VCPU_SPE_CTRL          3
> +#define   KVM_ARM_VCPU_SPE_IRQ         0
> +#define   KVM_ARM_VCPU_SPE_INIT                1
>
>  /* KVM_IRQ_LINE irq field index values */
>  #define KVM_ARM_IRQ_VCPU2_SHIFT                28
> diff --git a/arch/arm64/kvm/Makefile b/arch/arm64/kvm/Makefile
> index 1504c81fbf5d..f6e76f64ffbe 100644
> --- a/arch/arm64/kvm/Makefile
> +++ b/arch/arm64/kvm/Makefile
> @@ -25,3 +25,4 @@ kvm-y := $(KVM)/kvm_main.o $(KVM)/coalesced_mmio.o $(KVM)/eventfd.o \
>          vgic/vgic-its.o vgic/vgic-debug.o
>
>  kvm-$(CONFIG_KVM_ARM_PMU)  += pmu-emul.o
> +kvm-$(CONFIG_KVM_ARM_SPE)  += spe.o
> diff --git a/arch/arm64/kvm/guest.c b/arch/arm64/kvm/guest.c
> index dfb5218137ca..2ba790eeb782 100644
> --- a/arch/arm64/kvm/guest.c
> +++ b/arch/arm64/kvm/guest.c
> @@ -926,6 +926,9 @@ int kvm_arm_vcpu_arch_set_attr(struct kvm_vcpu *vcpu,
>         case KVM_ARM_VCPU_PVTIME_CTRL:
>                 ret = kvm_arm_pvtime_set_attr(vcpu, attr);
>                 break;
> +       case KVM_ARM_VCPU_SPE_CTRL:
> +               ret = kvm_arm_spe_set_attr(vcpu, attr);
> +               break;
>         default:
>                 ret = -ENXIO;
>                 break;
> @@ -949,6 +952,9 @@ int kvm_arm_vcpu_arch_get_attr(struct kvm_vcpu *vcpu,
>         case KVM_ARM_VCPU_PVTIME_CTRL:
>                 ret = kvm_arm_pvtime_get_attr(vcpu, attr);
>                 break;
> +       case KVM_ARM_VCPU_SPE_CTRL:
> +               ret = kvm_arm_spe_get_attr(vcpu, attr);
> +               break;
>         default:
>                 ret = -ENXIO;
>                 break;
> @@ -972,6 +978,9 @@ int kvm_arm_vcpu_arch_has_attr(struct kvm_vcpu *vcpu,
>         case KVM_ARM_VCPU_PVTIME_CTRL:
>                 ret = kvm_arm_pvtime_has_attr(vcpu, attr);
>                 break;
> +       case KVM_ARM_VCPU_SPE_CTRL:
> +               ret = kvm_arm_spe_has_attr(vcpu, attr);
> +               break;
>         default:
>                 ret = -ENXIO;
>                 break;
> diff --git a/arch/arm64/kvm/reset.c b/arch/arm64/kvm/reset.c
> index f32490229a4c..4dc205fa4be1 100644
> --- a/arch/arm64/kvm/reset.c
> +++ b/arch/arm64/kvm/reset.c
> @@ -87,6 +87,9 @@ int kvm_arch_vm_ioctl_check_extension(struct kvm *kvm, long ext)
>         case KVM_CAP_ARM_PTRAUTH_GENERIC:
>                 r = system_has_full_ptr_auth();
>                 break;
> +       case KVM_CAP_ARM_SPE:
> +               r = kvm_arm_supports_spe();
> +               break;
>         default:
>                 r = 0;
>         }
> @@ -223,6 +226,19 @@ static int kvm_vcpu_enable_ptrauth(struct kvm_vcpu *vcpu)
>         return 0;
>  }
>
> +static int kvm_vcpu_enable_spe(struct kvm_vcpu *vcpu)
> +{
> +       if (!kvm_arm_supports_spe())
> +               return -EINVAL;
> +
> +       /* SPE is disabled if the PE is in AArch32 state */
> +       if (test_bit(KVM_ARM_VCPU_EL1_32BIT, vcpu->arch.features))
> +               return -EINVAL;
> +
> +       vcpu->arch.flags |= KVM_ARM64_GUEST_HAS_SPE;
> +       return 0;
> +}
> +
>  /**
>   * kvm_reset_vcpu - sets core registers and sys_regs to reset value
>   * @vcpu: The VCPU pointer
> @@ -274,6 +290,13 @@ int kvm_reset_vcpu(struct kvm_vcpu *vcpu)
>                 }
>         }
>
> +       if (test_bit(KVM_ARM_VCPU_SPE, vcpu->arch.features)) {
> +               if (kvm_vcpu_enable_spe(vcpu)) {
> +                       ret = -EINVAL;
> +                       goto out;
> +               }
> +       }
> +
>         switch (vcpu->arch.target) {
>         default:
>                 if (test_bit(KVM_ARM_VCPU_EL1_32BIT, vcpu->arch.features)) {
> diff --git a/arch/arm64/kvm/spe.c b/arch/arm64/kvm/spe.c
> new file mode 100644
> index 000000000000..f91a52cd7cd3
> --- /dev/null
> +++ b/arch/arm64/kvm/spe.c
> @@ -0,0 +1,129 @@
> +// SPDX-License-Identifier: GPL-2.0
> +/*
> + * Copyright (C) 2019 ARM Ltd.
> + */
> +
> +#include <linux/kvm.h>
> +#include <linux/kvm_host.h>
> +#include <linux/uaccess.h>
> +
> +#include <kvm/arm_spe.h>
> +#include <kvm/arm_vgic.h>
> +
> +static bool kvm_arm_vcpu_supports_spe(struct kvm_vcpu *vcpu)
> +{
> +       if (!vcpu_has_spe(vcpu))
> +               return false;
> +
> +       if (!irqchip_in_kernel(vcpu->kvm))
> +               return false;
> +

nit: should we move the irqchip_in_kernel() check to the caller?

> +       return true;
> +}
> +
> +static int kvm_arm_spe_init(struct kvm_vcpu *vcpu)
> +{
> +       if (!kvm_arm_spe_irq_initialized(vcpu))
> +               return -ENXIO;
> +
> +       if (!vgic_initialized(vcpu->kvm))
> +               return -ENODEV;
> +
> +       if (kvm_arm_spe_vcpu_initialized(vcpu))
> +               return -EBUSY;
> +
> +       if (kvm_vgic_set_owner(vcpu, vcpu->arch.spe_cpu.irq_num, &vcpu->arch.spe_cpu))
> +               return -ENXIO;
> +
> +       vcpu->arch.spe_cpu.initialized = true;
> +
> +       return 0;
> +}
> +
> +static bool kvm_arm_spe_irq_is_valid(struct kvm *kvm, int irq)
> +{
> +       int i;
> +       struct kvm_vcpu *vcpu;
> +
> +       /* The SPE overflow interrupt can be a PPI only */
> +       if (!irq_is_ppi(irq))
> +               return false;
> +
> +       kvm_for_each_vcpu(i, vcpu, kvm) {
> +               if (!kvm_arm_spe_irq_initialized(vcpu))
> +                       continue;
> +
> +               if (vcpu->arch.spe_cpu.irq_num != irq)
> +                       return false;
> +       }
> +
> +       return true;
> +}
> +
> +int kvm_arm_spe_set_attr(struct kvm_vcpu *vcpu, struct kvm_device_attr *attr)
> +{
> +       switch (attr->attr) {
> +       case KVM_ARM_VCPU_SPE_IRQ: {
> +               int __user *uaddr = (int __user *)(long)attr->addr;
> +               int irq;
> +
> +               if (!kvm_arm_vcpu_supports_spe(vcpu))
> +                       return -ENXIO;
> +
> +               if (get_user(irq, uaddr))
> +                       return -EFAULT;
> +
> +               if (!kvm_arm_spe_irq_is_valid(vcpu->kvm, irq))
> +                       return -EINVAL;
> +
> +               if (kvm_arm_spe_irq_initialized(vcpu))
> +                       return -EBUSY;
> +
> +               kvm_debug("Set kvm ARM SPE irq: %d\n", irq);
> +               vcpu->arch.spe_cpu.irq_num = irq;
> +
> +               return 0;
> +       }
> +       case KVM_ARM_VCPU_SPE_INIT:
> +               return kvm_arm_spe_init(vcpu);
> +       }
> +
> +       return -ENXIO;
> +}
> +
> +int kvm_arm_spe_get_attr(struct kvm_vcpu *vcpu, struct kvm_device_attr *attr)
> +{
> +       switch (attr->attr) {
> +       case KVM_ARM_VCPU_SPE_IRQ: {
> +               int __user *uaddr = (int __user *)(long)attr->addr;
> +               int irq;
> +
> +               if (!kvm_arm_vcpu_supports_spe(vcpu))
> +                       return -ENXIO;
> +
> +               if (!kvm_arm_spe_irq_initialized(vcpu))
> +                       return -ENXIO;
> +
> +               irq = vcpu->arch.spe_cpu.irq_num;
> +               if (put_user(irq, uaddr))
> +                       return -EFAULT;
> +
> +               return 0;
> +       }
> +       }
> +
> +       return -ENXIO;
> +}
> +
> +int kvm_arm_spe_has_attr(struct kvm_vcpu *vcpu, struct kvm_device_attr *attr)
> +{
> +       switch (attr->attr) {
> +       case KVM_ARM_VCPU_SPE_IRQ:
> +               fallthrough;
> +       case KVM_ARM_VCPU_SPE_INIT:
> +               if (kvm_arm_vcpu_supports_spe(vcpu))
> +                       return 0;
> +       }
> +
> +       return -ENXIO;
> +}
> diff --git a/include/kvm/arm_spe.h b/include/kvm/arm_spe.h
> index 46ec447ed013..0275e8097529 100644
> --- a/include/kvm/arm_spe.h
> +++ b/include/kvm/arm_spe.h
> @@ -18,11 +18,38 @@ struct kvm_spe_cpu {
>         bool initialized;       /* Feature is initialized on VCPU */
>  };
>
> +#define kvm_arm_spe_irq_initialized(v)                 \
> +       ((v)->arch.spe_cpu.irq_num >= VGIC_NR_SGIS &&   \
> +        (v)->arch.spe_cpu.irq_num < VGIC_MAX_PRIVATE)
> +#define kvm_arm_spe_vcpu_initialized(v)        ((v)->arch.spe_cpu.initialized)
> +
> +int kvm_arm_spe_set_attr(struct kvm_vcpu *vcpu, struct kvm_device_attr *attr);
> +int kvm_arm_spe_get_attr(struct kvm_vcpu *vcpu, struct kvm_device_attr *attr);
> +int kvm_arm_spe_has_attr(struct kvm_vcpu *vcpu, struct kvm_device_attr *attr);
> +
>  #else
>  #define kvm_arm_supports_spe() false
>
>  struct kvm_spe_cpu {
>  };
>
> +#define kvm_arm_spe_irq_initialized(v) false
> +#define kvm_arm_spe_vcpu_initialized(v)        false
> +
> +static inline int kvm_arm_spe_set_attr(struct kvm_vcpu *vcpu,
> +                                      struct kvm_device_attr *attr)
> +{
> +       return -ENXIO;
> +}
> +static inline int kvm_arm_spe_get_attr(struct kvm_vcpu *vcpu,
> +                                      struct kvm_device_attr *attr)
> +{
> +       return -ENXIO;
> +}
> +static inline int kvm_arm_spe_has_attr(struct kvm_vcpu *vcpu,
> +                                      struct kvm_device_attr *attr)
> +{
> +       return -ENXIO;
> +}
>  #endif /* CONFIG_KVM_ARM_SPE */
>  #endif /* __ASM_ARM_KVM_SPE_H */
> diff --git a/include/uapi/linux/kvm.h b/include/uapi/linux/kvm.h
> index ca41220b40b8..96228b823711 100644
> --- a/include/uapi/linux/kvm.h
> +++ b/include/uapi/linux/kvm.h
> @@ -1053,6 +1053,7 @@ struct kvm_ppc_resize_hpt {
>  #define KVM_CAP_X86_USER_SPACE_MSR 188
>  #define KVM_CAP_X86_MSR_FILTER 189
>  #define KVM_CAP_ENFORCE_PV_FEATURE_CPUID 190
> +#define KVM_CAP_ARM_SPE 191
>
>  #ifdef KVM_CAP_IRQ_ROUTING
>
> --
> 2.29.1
>
> _______________________________________________
> kvmarm mailing list
> kvmarm@lists.cs.columbia.edu
> https://lists.cs.columbia.edu/mailman/listinfo/kvmarm
James Morse Nov. 19, 2020, 4:58 p.m. UTC | #2 
Hi Alex,

On 27/10/2020 17:26, Alexandru Elisei wrote:
> From: Sudeep Holla <sudeep.holla@arm.com>
> 
> To configure the virtual SPE buffer management interrupt number, we use a
> VCPU kvm_device ioctl, encapsulating the KVM_ARM_VCPU_SPE_IRQ attribute
> within the KVM_ARM_VCPU_SPE_CTRL group.
> 
> After configuring the SPE, userspace is required to call the VCPU ioctl
> with the attribute KVM_ARM_VCPU_SPE_INIT to initialize SPE on the VCPU.

> diff --git a/Documentation/virt/kvm/devices/vcpu.rst b/Documentation/virt/kvm/devices/vcpu.rst
> index 2acec3b9ef65..6135b9827fbe 100644
> --- a/Documentation/virt/kvm/devices/vcpu.rst
> +++ b/Documentation/virt/kvm/devices/vcpu.rst
> @@ -161,3 +161,43 @@ Specifies the base address of the stolen time structure for this VCPU. The
>  base address must be 64 byte aligned and exist within a valid guest memory
>  region. See Documentation/virt/kvm/arm/pvtime.rst for more information
>  including the layout of the stolen time structure.
> +
> +4. GROUP: KVM_ARM_VCPU_SPE_CTRL
> +===============================
> +
> +:Architectures: ARM64
> +
> +4.1 ATTRIBUTE: KVM_ARM_VCPU_SPE_IRQ
> +-----------------------------------
> +
> +:Parameters: in kvm_device_attr.addr the address for the SPE buffer management
> +             interrupt is a pointer to an int
> +
> +Returns:
> +
> +	 =======  ========================================================
> +	 -EBUSY   The SPE buffer management interrupt is already set
> +	 -EINVAL  Invalid SPE overflow interrupt number
> +	 -EFAULT  Could not read the buffer management interrupt number
> +	 -ENXIO   SPE not supported or not properly configured
> +	 =======  ========================================================
> +
> +A value describing the SPE (Statistical Profiling Extension) overflow interrupt
> +number for this vcpu. This interrupt should be a PPI and the interrupt type and
> +number must be same for each vcpu.
> +
> +4.2 ATTRIBUTE: KVM_ARM_VCPU_SPE_INIT
> +------------------------------------
> +
> +:Parameters: no additional parameter in kvm_device_attr.addr
> +
> +Returns:
> +
> +	 =======  ======================================================
> +	 -EBUSY   SPE already initialized
> +	 -ENODEV  GIC not initialized
> +	 -ENXIO   SPE not supported or not properly configured
> +	 =======  ======================================================

> +Request the initialization of the SPE. Must be done after initializing the
> +in-kernel irqchip and after setting the interrupt number for the VCPU.

Fantastic!


> diff --git a/arch/arm64/kvm/reset.c b/arch/arm64/kvm/reset.c
> index f32490229a4c..4dc205fa4be1 100644
> --- a/arch/arm64/kvm/reset.c
> +++ b/arch/arm64/kvm/reset.c
> @@ -87,6 +87,9 @@ int kvm_arch_vm_ioctl_check_extension(struct kvm *kvm, long ext)
>  	case KVM_CAP_ARM_PTRAUTH_GENERIC:
>  		r = system_has_full_ptr_auth();
>  		break;
> +	case KVM_CAP_ARM_SPE:
> +		r = kvm_arm_supports_spe();
> +		break;
>  	default:
>  		r = 0;
>  	}
> @@ -223,6 +226,19 @@ static int kvm_vcpu_enable_ptrauth(struct kvm_vcpu *vcpu)
>  	return 0;
>  }
>  
> +static int kvm_vcpu_enable_spe(struct kvm_vcpu *vcpu)
> +{
> +	if (!kvm_arm_supports_spe())
> +		return -EINVAL;
> +
> +	/* SPE is disabled if the PE is in AArch32 state */
> +	if (test_bit(KVM_ARM_VCPU_EL1_32BIT, vcpu->arch.features))
> +		return -EINVAL;
> +
> +	vcpu->arch.flags |= KVM_ARM64_GUEST_HAS_SPE;
> +	return 0;
> +}

VCPU-reset promotes the VMM feature into flags. How does this interact with
kvm_arm_spe_init()?

It doesn't look like this resets any state, couldn't it be done once by kvm_arm_spe_init()?


>  /**
>   * kvm_reset_vcpu - sets core registers and sys_regs to reset value
>   * @vcpu: The VCPU pointer
> @@ -274,6 +290,13 @@ int kvm_reset_vcpu(struct kvm_vcpu *vcpu)
>  		}
>  	}
>  
> +	if (test_bit(KVM_ARM_VCPU_SPE, vcpu->arch.features)) {
> +		if (kvm_vcpu_enable_spe(vcpu)) {
> +			ret = -EINVAL;
> +			goto out;
> +		}
> +	}
> +
>  	switch (vcpu->arch.target) {
>  	default:
>  		if (test_bit(KVM_ARM_VCPU_EL1_32BIT, vcpu->arch.features)) {

> diff --git a/arch/arm64/kvm/spe.c b/arch/arm64/kvm/spe.c
> new file mode 100644
> index 000000000000..f91a52cd7cd3
> --- /dev/null
> +++ b/arch/arm64/kvm/spe.c
> @@ -0,0 +1,129 @@

> +static bool kvm_arm_spe_irq_is_valid(struct kvm *kvm, int irq)
> +{
> +	int i;
> +	struct kvm_vcpu *vcpu;
> +
> +	/* The SPE overflow interrupt can be a PPI only */
> +	if (!irq_is_ppi(irq))
> +		return false;
> +
> +	kvm_for_each_vcpu(i, vcpu, kvm) {
> +		if (!kvm_arm_spe_irq_initialized(vcpu))
> +			continue;
> +
> +		if (vcpu->arch.spe_cpu.irq_num != irq)
> +			return false;
> +	}

Looks like you didn't really want a vcpu property! (huh, patch 10 adds a vm property too)
We're making this a vcpu property because of the PPI and system registers? (both good reasons)

If the PPI number lived in struct kvm_arch, you'd only only need to check it was
uninitialised, or the same to get the same behaviour, which would save some of this error
handling.


> +	return true;
> +}

> diff --git a/include/kvm/arm_spe.h b/include/kvm/arm_spe.h
> index 46ec447ed013..0275e8097529 100644
> --- a/include/kvm/arm_spe.h
> +++ b/include/kvm/arm_spe.h
> @@ -18,11 +18,38 @@ struct kvm_spe_cpu {
>  	bool initialized; 	/* Feature is initialized on VCPU */
>  };
>  
> +#define kvm_arm_spe_irq_initialized(v)			\
> +	((v)->arch.spe_cpu.irq_num >= VGIC_NR_SGIS &&	\
> +	 (v)->arch.spe_cpu.irq_num < VGIC_MAX_PRIVATE)

Didn't GICv(mumbles) add an additional PPI range? Could this be made irq_is_ppi(), that
way if the vgic gains support for that, we don't get weird behaviour here?


Thanks,

James
Alexandru Elisei Dec. 2, 2020, 3:20 p.m. UTC | #3 
Hi Haibu,

Thanks for having a look at the patches!

On 11/5/20 9:58 AM, Haibo Xu wrote:
> On Wed, 28 Oct 2020 at 01:26, Alexandru Elisei <alexandru.elisei@arm.com> wrote:
>> From: Sudeep Holla <sudeep.holla@arm.com>
>>
>> To configure the virtual SPE buffer management interrupt number, we use a
>> VCPU kvm_device ioctl, encapsulating the KVM_ARM_VCPU_SPE_IRQ attribute
>> within the KVM_ARM_VCPU_SPE_CTRL group.
>>
>> After configuring the SPE, userspace is required to call the VCPU ioctl
>> with the attribute KVM_ARM_VCPU_SPE_INIT to initialize SPE on the VCPU.
>>
>> [Alexandru E: Fixed compilation errors, don't allow userspace to set the
>>         VCPU feature, removed unused functions, fixed mismatched
>>         descriptions, comments and error codes, reworked logic, rebased on
>>         top of v5.10-rc1]
>>
>> Signed-off-by: Sudeep Holla <sudeep.holla@arm.com>
>> Signed-off-by: Alexandru Elisei <alexandru.elisei@arm.com>
>> ---
>>  Documentation/virt/kvm/devices/vcpu.rst |  40 ++++++++
>>  arch/arm64/include/uapi/asm/kvm.h       |   3 +
>>  arch/arm64/kvm/Makefile                 |   1 +
>>  arch/arm64/kvm/guest.c                  |   9 ++
>>  arch/arm64/kvm/reset.c                  |  23 +++++
>>  arch/arm64/kvm/spe.c                    | 129 ++++++++++++++++++++++++
>>  include/kvm/arm_spe.h                   |  27 +++++
>>  include/uapi/linux/kvm.h                |   1 +
>>  8 files changed, 233 insertions(+)
>>  create mode 100644 arch/arm64/kvm/spe.c
>>
>> diff --git a/Documentation/virt/kvm/devices/vcpu.rst b/Documentation/virt/kvm/devices/vcpu.rst
>> index 2acec3b9ef65..6135b9827fbe 100644
>> --- a/Documentation/virt/kvm/devices/vcpu.rst
>> +++ b/Documentation/virt/kvm/devices/vcpu.rst
>> @@ -161,3 +161,43 @@ Specifies the base address of the stolen time structure for this VCPU. The
>>  base address must be 64 byte aligned and exist within a valid guest memory
>>  region. See Documentation/virt/kvm/arm/pvtime.rst for more information
>>  including the layout of the stolen time structure.
>> +
>> +4. GROUP: KVM_ARM_VCPU_SPE_CTRL
>> +===============================
>> +
>> +:Architectures: ARM64
>> +
>> +4.1 ATTRIBUTE: KVM_ARM_VCPU_SPE_IRQ
>> +-----------------------------------
>> +
>> +:Parameters: in kvm_device_attr.addr the address for the SPE buffer management
>> +             interrupt is a pointer to an int
>> +
>> +Returns:
>> +
>> +        =======  ========================================================
>> +        -EBUSY   The SPE buffer management interrupt is already set
>> +        -EINVAL  Invalid SPE overflow interrupt number
>> +        -EFAULT  Could not read the buffer management interrupt number
>> +        -ENXIO   SPE not supported or not properly configured
>> +        =======  ========================================================
>> +
>> +A value describing the SPE (Statistical Profiling Extension) overflow interrupt
>> +number for this vcpu. This interrupt should be a PPI and the interrupt type and
>> +number must be same for each vcpu.
>> +
>> +4.2 ATTRIBUTE: KVM_ARM_VCPU_SPE_INIT
>> +------------------------------------
>> +
>> +:Parameters: no additional parameter in kvm_device_attr.addr
>> +
>> +Returns:
>> +
>> +        =======  ======================================================
>> +        -EBUSY   SPE already initialized
>> +        -ENODEV  GIC not initialized
>> +        -ENXIO   SPE not supported or not properly configured
>> +        =======  ======================================================
>> +
>> +Request the initialization of the SPE. Must be done after initializing the
>> +in-kernel irqchip and after setting the interrupt number for the VCPU.
>> diff --git a/arch/arm64/include/uapi/asm/kvm.h b/arch/arm64/include/uapi/asm/kvm.h
>> index 489e12304dbb..ca57dfb7abf0 100644
>> --- a/arch/arm64/include/uapi/asm/kvm.h
>> +++ b/arch/arm64/include/uapi/asm/kvm.h
>> @@ -360,6 +360,9 @@ struct kvm_vcpu_events {
>>  #define   KVM_ARM_VCPU_TIMER_IRQ_PTIMER                1
>>  #define KVM_ARM_VCPU_PVTIME_CTRL       2
>>  #define   KVM_ARM_VCPU_PVTIME_IPA      0
>> +#define KVM_ARM_VCPU_SPE_CTRL          3
>> +#define   KVM_ARM_VCPU_SPE_IRQ         0
>> +#define   KVM_ARM_VCPU_SPE_INIT                1
>>
>>  /* KVM_IRQ_LINE irq field index values */
>>  #define KVM_ARM_IRQ_VCPU2_SHIFT                28
>> diff --git a/arch/arm64/kvm/Makefile b/arch/arm64/kvm/Makefile
>> index 1504c81fbf5d..f6e76f64ffbe 100644
>> --- a/arch/arm64/kvm/Makefile
>> +++ b/arch/arm64/kvm/Makefile
>> @@ -25,3 +25,4 @@ kvm-y := $(KVM)/kvm_main.o $(KVM)/coalesced_mmio.o $(KVM)/eventfd.o \
>>          vgic/vgic-its.o vgic/vgic-debug.o
>>
>>  kvm-$(CONFIG_KVM_ARM_PMU)  += pmu-emul.o
>> +kvm-$(CONFIG_KVM_ARM_SPE)  += spe.o
>> diff --git a/arch/arm64/kvm/guest.c b/arch/arm64/kvm/guest.c
>> index dfb5218137ca..2ba790eeb782 100644
>> --- a/arch/arm64/kvm/guest.c
>> +++ b/arch/arm64/kvm/guest.c
>> @@ -926,6 +926,9 @@ int kvm_arm_vcpu_arch_set_attr(struct kvm_vcpu *vcpu,
>>         case KVM_ARM_VCPU_PVTIME_CTRL:
>>                 ret = kvm_arm_pvtime_set_attr(vcpu, attr);
>>                 break;
>> +       case KVM_ARM_VCPU_SPE_CTRL:
>> +               ret = kvm_arm_spe_set_attr(vcpu, attr);
>> +               break;
>>         default:
>>                 ret = -ENXIO;
>>                 break;
>> @@ -949,6 +952,9 @@ int kvm_arm_vcpu_arch_get_attr(struct kvm_vcpu *vcpu,
>>         case KVM_ARM_VCPU_PVTIME_CTRL:
>>                 ret = kvm_arm_pvtime_get_attr(vcpu, attr);
>>                 break;
>> +       case KVM_ARM_VCPU_SPE_CTRL:
>> +               ret = kvm_arm_spe_get_attr(vcpu, attr);
>> +               break;
>>         default:
>>                 ret = -ENXIO;
>>                 break;
>> @@ -972,6 +978,9 @@ int kvm_arm_vcpu_arch_has_attr(struct kvm_vcpu *vcpu,
>>         case KVM_ARM_VCPU_PVTIME_CTRL:
>>                 ret = kvm_arm_pvtime_has_attr(vcpu, attr);
>>                 break;
>> +       case KVM_ARM_VCPU_SPE_CTRL:
>> +               ret = kvm_arm_spe_has_attr(vcpu, attr);
>> +               break;
>>         default:
>>                 ret = -ENXIO;
>>                 break;
>> diff --git a/arch/arm64/kvm/reset.c b/arch/arm64/kvm/reset.c
>> index f32490229a4c..4dc205fa4be1 100644
>> --- a/arch/arm64/kvm/reset.c
>> +++ b/arch/arm64/kvm/reset.c
>> @@ -87,6 +87,9 @@ int kvm_arch_vm_ioctl_check_extension(struct kvm *kvm, long ext)
>>         case KVM_CAP_ARM_PTRAUTH_GENERIC:
>>                 r = system_has_full_ptr_auth();
>>                 break;
>> +       case KVM_CAP_ARM_SPE:
>> +               r = kvm_arm_supports_spe();
>> +               break;
>>         default:
>>                 r = 0;
>>         }
>> @@ -223,6 +226,19 @@ static int kvm_vcpu_enable_ptrauth(struct kvm_vcpu *vcpu)
>>         return 0;
>>  }
>>
>> +static int kvm_vcpu_enable_spe(struct kvm_vcpu *vcpu)
>> +{
>> +       if (!kvm_arm_supports_spe())
>> +               return -EINVAL;
>> +
>> +       /* SPE is disabled if the PE is in AArch32 state */
>> +       if (test_bit(KVM_ARM_VCPU_EL1_32BIT, vcpu->arch.features))
>> +               return -EINVAL;
>> +
>> +       vcpu->arch.flags |= KVM_ARM64_GUEST_HAS_SPE;
>> +       return 0;
>> +}
>> +
>>  /**
>>   * kvm_reset_vcpu - sets core registers and sys_regs to reset value
>>   * @vcpu: The VCPU pointer
>> @@ -274,6 +290,13 @@ int kvm_reset_vcpu(struct kvm_vcpu *vcpu)
>>                 }
>>         }
>>
>> +       if (test_bit(KVM_ARM_VCPU_SPE, vcpu->arch.features)) {
>> +               if (kvm_vcpu_enable_spe(vcpu)) {
>> +                       ret = -EINVAL;
>> +                       goto out;
>> +               }
>> +       }
>> +
>>         switch (vcpu->arch.target) {
>>         default:
>>                 if (test_bit(KVM_ARM_VCPU_EL1_32BIT, vcpu->arch.features)) {
>> diff --git a/arch/arm64/kvm/spe.c b/arch/arm64/kvm/spe.c
>> new file mode 100644
>> index 000000000000..f91a52cd7cd3
>> --- /dev/null
>> +++ b/arch/arm64/kvm/spe.c
>> @@ -0,0 +1,129 @@
>> +// SPDX-License-Identifier: GPL-2.0
>> +/*
>> + * Copyright (C) 2019 ARM Ltd.
>> + */
>> +
>> +#include <linux/kvm.h>
>> +#include <linux/kvm_host.h>
>> +#include <linux/uaccess.h>
>> +
>> +#include <kvm/arm_spe.h>
>> +#include <kvm/arm_vgic.h>
>> +
>> +static bool kvm_arm_vcpu_supports_spe(struct kvm_vcpu *vcpu)
>> +{
>> +       if (!vcpu_has_spe(vcpu))
>> +               return false;
>> +
>> +       if (!irqchip_in_kernel(vcpu->kvm))
>> +               return false;
>> +
> nit: should we move the irqchip_in_kernel() check to the caller?

Yes, definitely, I can move the irqchip_in_kernel() check to the callers because
it's a VM, not VCPU property, and use vcpu_has_spe() directly.

Thanks,
Alex
Alexandru Elisei Dec. 2, 2020, 4:28 p.m. UTC | #4 
Hi James,

On 11/19/20 4:58 PM, James Morse wrote:
> Hi Alex,
>
> On 27/10/2020 17:26, Alexandru Elisei wrote:
>> From: Sudeep Holla <sudeep.holla@arm.com>
>>
>> To configure the virtual SPE buffer management interrupt number, we use a
>> VCPU kvm_device ioctl, encapsulating the KVM_ARM_VCPU_SPE_IRQ attribute
>> within the KVM_ARM_VCPU_SPE_CTRL group.
>>
>> After configuring the SPE, userspace is required to call the VCPU ioctl
>> with the attribute KVM_ARM_VCPU_SPE_INIT to initialize SPE on the VCPU.
>> diff --git a/Documentation/virt/kvm/devices/vcpu.rst b/Documentation/virt/kvm/devices/vcpu.rst
>> index 2acec3b9ef65..6135b9827fbe 100644
>> --- a/Documentation/virt/kvm/devices/vcpu.rst
>> +++ b/Documentation/virt/kvm/devices/vcpu.rst
>> @@ -161,3 +161,43 @@ Specifies the base address of the stolen time structure for this VCPU. The
>>  base address must be 64 byte aligned and exist within a valid guest memory
>>  region. See Documentation/virt/kvm/arm/pvtime.rst for more information
>>  including the layout of the stolen time structure.
>> +
>> +4. GROUP: KVM_ARM_VCPU_SPE_CTRL
>> +===============================
>> +
>> +:Architectures: ARM64
>> +
>> +4.1 ATTRIBUTE: KVM_ARM_VCPU_SPE_IRQ
>> +-----------------------------------
>> +
>> +:Parameters: in kvm_device_attr.addr the address for the SPE buffer management
>> +             interrupt is a pointer to an int
>> +
>> +Returns:
>> +
>> +	 =======  ========================================================
>> +	 -EBUSY   The SPE buffer management interrupt is already set
>> +	 -EINVAL  Invalid SPE overflow interrupt number
>> +	 -EFAULT  Could not read the buffer management interrupt number
>> +	 -ENXIO   SPE not supported or not properly configured
>> +	 =======  ========================================================
>> +
>> +A value describing the SPE (Statistical Profiling Extension) overflow interrupt
>> +number for this vcpu. This interrupt should be a PPI and the interrupt type and
>> +number must be same for each vcpu.
>> +
>> +4.2 ATTRIBUTE: KVM_ARM_VCPU_SPE_INIT
>> +------------------------------------
>> +
>> +:Parameters: no additional parameter in kvm_device_attr.addr
>> +
>> +Returns:
>> +
>> +	 =======  ======================================================
>> +	 -EBUSY   SPE already initialized
>> +	 -ENODEV  GIC not initialized
>> +	 -ENXIO   SPE not supported or not properly configured
>> +	 =======  ======================================================
>> +Request the initialization of the SPE. Must be done after initializing the
>> +in-kernel irqchip and after setting the interrupt number for the VCPU.
> Fantastic!
>
>
>> diff --git a/arch/arm64/kvm/reset.c b/arch/arm64/kvm/reset.c
>> index f32490229a4c..4dc205fa4be1 100644
>> --- a/arch/arm64/kvm/reset.c
>> +++ b/arch/arm64/kvm/reset.c
>> @@ -87,6 +87,9 @@ int kvm_arch_vm_ioctl_check_extension(struct kvm *kvm, long ext)
>>  	case KVM_CAP_ARM_PTRAUTH_GENERIC:
>>  		r = system_has_full_ptr_auth();
>>  		break;
>> +	case KVM_CAP_ARM_SPE:
>> +		r = kvm_arm_supports_spe();
>> +		break;
>>  	default:
>>  		r = 0;
>>  	}
>> @@ -223,6 +226,19 @@ static int kvm_vcpu_enable_ptrauth(struct kvm_vcpu *vcpu)
>>  	return 0;
>>  }
>>  
>> +static int kvm_vcpu_enable_spe(struct kvm_vcpu *vcpu)
>> +{
>> +	if (!kvm_arm_supports_spe())
>> +		return -EINVAL;
>> +
>> +	/* SPE is disabled if the PE is in AArch32 state */
>> +	if (test_bit(KVM_ARM_VCPU_EL1_32BIT, vcpu->arch.features))
>> +		return -EINVAL;
>> +
>> +	vcpu->arch.flags |= KVM_ARM64_GUEST_HAS_SPE;
>> +	return 0;
>> +}
> VCPU-reset promotes the VMM feature into flags. How does this interact with
> kvm_arm_spe_init()?
>
> It doesn't look like this resets any state, couldn't it be done once by kvm_arm_spe_init()?

I need to check here for incompatible features (KVM_ARM_VCPU_EL1_32BIT) or
unsupported SPE to return an error code to KVM_ARM_VCPU_INIT (the ioctl is handled
in kvm_arch_vcpu_ioctl_vcpu_init -> kvm_vcpu_set_target)

We need to track the feature per-vcpu, to refuse finalization if the feature was
not set on all of them. I could use features, but I think using flags will end up
slightly faster.

When KVM SPE is optimized, I will probably add at least one vcpu->arch.flags flag
(something like KVM_ARM64_HOST_USES_SPE) to allow for lazy save/restore of the
host SPE context when the flag is missing. I was thinking that in that case
checking vcpu->arch.flags will be cheaper. This is also the approach that SVE
uses, from what I can tell.

For now, I would prefer to keep it as a vcpu flag and make the decision once I
start implementing performance optimizations. But using features is definitely
doable if there are objections against using flags.

>
>
>>  /**
>>   * kvm_reset_vcpu - sets core registers and sys_regs to reset value
>>   * @vcpu: The VCPU pointer
>> @@ -274,6 +290,13 @@ int kvm_reset_vcpu(struct kvm_vcpu *vcpu)
>>  		}
>>  	}
>>  
>> +	if (test_bit(KVM_ARM_VCPU_SPE, vcpu->arch.features)) {
>> +		if (kvm_vcpu_enable_spe(vcpu)) {
>> +			ret = -EINVAL;
>> +			goto out;
>> +		}
>> +	}
>> +
>>  	switch (vcpu->arch.target) {
>>  	default:
>>  		if (test_bit(KVM_ARM_VCPU_EL1_32BIT, vcpu->arch.features)) {
>> diff --git a/arch/arm64/kvm/spe.c b/arch/arm64/kvm/spe.c
>> new file mode 100644
>> index 000000000000..f91a52cd7cd3
>> --- /dev/null
>> +++ b/arch/arm64/kvm/spe.c
>> @@ -0,0 +1,129 @@
>> +static bool kvm_arm_spe_irq_is_valid(struct kvm *kvm, int irq)
>> +{
>> +	int i;
>> +	struct kvm_vcpu *vcpu;
>> +
>> +	/* The SPE overflow interrupt can be a PPI only */
>> +	if (!irq_is_ppi(irq))
>> +		return false;
>> +
>> +	kvm_for_each_vcpu(i, vcpu, kvm) {
>> +		if (!kvm_arm_spe_irq_initialized(vcpu))
>> +			continue;
>> +
>> +		if (vcpu->arch.spe_cpu.irq_num != irq)
>> +			return false;
>> +	}
> Looks like you didn't really want a vcpu property! (huh, patch 10 adds a vm property too)
> We're making this a vcpu property because of the PPI and system registers? (both good reasons)
>
> If the PPI number lived in struct kvm_arch, you'd only only need to check it was
> uninitialised, or the same to get the same behaviour, which would save some of this error
> handling.

The Arm ARM mandates that the SPE interrupt must be a PPI. I think it makes more
sense to have a Private Peripheral Interrupt ID saved as a per-vcpu variable than
a per-VM one, like a SPI.

>
>
>> +	return true;
>> +}
>> diff --git a/include/kvm/arm_spe.h b/include/kvm/arm_spe.h
>> index 46ec447ed013..0275e8097529 100644
>> --- a/include/kvm/arm_spe.h
>> +++ b/include/kvm/arm_spe.h
>> @@ -18,11 +18,38 @@ struct kvm_spe_cpu {
>>  	bool initialized; 	/* Feature is initialized on VCPU */
>>  };
>>  
>> +#define kvm_arm_spe_irq_initialized(v)			\
>> +	((v)->arch.spe_cpu.irq_num >= VGIC_NR_SGIS &&	\
>> +	 (v)->arch.spe_cpu.irq_num < VGIC_MAX_PRIVATE)
> Didn't GICv(mumbles) add an additional PPI range? Could this be made irq_is_ppi(), that
> way if the vgic gains support for that, we don't get weird behaviour here?

You're right, the macro reimplements irq_is_ppi(), I'll rewrite it to use
irq_is_ppi().

Thanks,
Alex
diff mbox series

Patch

diff --git a/Documentation/virt/kvm/devices/vcpu.rst b/Documentation/virt/kvm/devices/vcpu.rst
index 2acec3b9ef65..6135b9827fbe 100644
--- a/Documentation/virt/kvm/devices/vcpu.rst
+++ b/Documentation/virt/kvm/devices/vcpu.rst
@@ -161,3 +161,43 @@  Specifies the base address of the stolen time structure for this VCPU. The
 base address must be 64 byte aligned and exist within a valid guest memory
 region. See Documentation/virt/kvm/arm/pvtime.rst for more information
 including the layout of the stolen time structure.
+
+4. GROUP: KVM_ARM_VCPU_SPE_CTRL
+===============================
+
+:Architectures: ARM64
+
+4.1 ATTRIBUTE: KVM_ARM_VCPU_SPE_IRQ
+-----------------------------------
+
+:Parameters: in kvm_device_attr.addr the address for the SPE buffer management
+             interrupt is a pointer to an int
+
+Returns:
+
+	 =======  ========================================================
+	 -EBUSY   The SPE buffer management interrupt is already set
+	 -EINVAL  Invalid SPE overflow interrupt number
+	 -EFAULT  Could not read the buffer management interrupt number
+	 -ENXIO   SPE not supported or not properly configured
+	 =======  ========================================================
+
+A value describing the SPE (Statistical Profiling Extension) overflow interrupt
+number for this vcpu. This interrupt should be a PPI and the interrupt type and
+number must be same for each vcpu.
+
+4.2 ATTRIBUTE: KVM_ARM_VCPU_SPE_INIT
+------------------------------------
+
+:Parameters: no additional parameter in kvm_device_attr.addr
+
+Returns:
+
+	 =======  ======================================================
+	 -EBUSY   SPE already initialized
+	 -ENODEV  GIC not initialized
+	 -ENXIO   SPE not supported or not properly configured
+	 =======  ======================================================
+
+Request the initialization of the SPE. Must be done after initializing the
+in-kernel irqchip and after setting the interrupt number for the VCPU.
diff --git a/arch/arm64/include/uapi/asm/kvm.h b/arch/arm64/include/uapi/asm/kvm.h
index 489e12304dbb..ca57dfb7abf0 100644
--- a/arch/arm64/include/uapi/asm/kvm.h
+++ b/arch/arm64/include/uapi/asm/kvm.h
@@ -360,6 +360,9 @@  struct kvm_vcpu_events {
 #define   KVM_ARM_VCPU_TIMER_IRQ_PTIMER		1
 #define KVM_ARM_VCPU_PVTIME_CTRL	2
 #define   KVM_ARM_VCPU_PVTIME_IPA	0
+#define KVM_ARM_VCPU_SPE_CTRL		3
+#define   KVM_ARM_VCPU_SPE_IRQ		0
+#define   KVM_ARM_VCPU_SPE_INIT		1
 
 /* KVM_IRQ_LINE irq field index values */
 #define KVM_ARM_IRQ_VCPU2_SHIFT		28
diff --git a/arch/arm64/kvm/Makefile b/arch/arm64/kvm/Makefile
index 1504c81fbf5d..f6e76f64ffbe 100644
--- a/arch/arm64/kvm/Makefile
+++ b/arch/arm64/kvm/Makefile
@@ -25,3 +25,4 @@  kvm-y := $(KVM)/kvm_main.o $(KVM)/coalesced_mmio.o $(KVM)/eventfd.o \
 	 vgic/vgic-its.o vgic/vgic-debug.o
 
 kvm-$(CONFIG_KVM_ARM_PMU)  += pmu-emul.o
+kvm-$(CONFIG_KVM_ARM_SPE)  += spe.o
diff --git a/arch/arm64/kvm/guest.c b/arch/arm64/kvm/guest.c
index dfb5218137ca..2ba790eeb782 100644
--- a/arch/arm64/kvm/guest.c
+++ b/arch/arm64/kvm/guest.c
@@ -926,6 +926,9 @@  int kvm_arm_vcpu_arch_set_attr(struct kvm_vcpu *vcpu,
 	case KVM_ARM_VCPU_PVTIME_CTRL:
 		ret = kvm_arm_pvtime_set_attr(vcpu, attr);
 		break;
+	case KVM_ARM_VCPU_SPE_CTRL:
+		ret = kvm_arm_spe_set_attr(vcpu, attr);
+		break;
 	default:
 		ret = -ENXIO;
 		break;
@@ -949,6 +952,9 @@  int kvm_arm_vcpu_arch_get_attr(struct kvm_vcpu *vcpu,
 	case KVM_ARM_VCPU_PVTIME_CTRL:
 		ret = kvm_arm_pvtime_get_attr(vcpu, attr);
 		break;
+	case KVM_ARM_VCPU_SPE_CTRL:
+		ret = kvm_arm_spe_get_attr(vcpu, attr);
+		break;
 	default:
 		ret = -ENXIO;
 		break;
@@ -972,6 +978,9 @@  int kvm_arm_vcpu_arch_has_attr(struct kvm_vcpu *vcpu,
 	case KVM_ARM_VCPU_PVTIME_CTRL:
 		ret = kvm_arm_pvtime_has_attr(vcpu, attr);
 		break;
+	case KVM_ARM_VCPU_SPE_CTRL:
+		ret = kvm_arm_spe_has_attr(vcpu, attr);
+		break;
 	default:
 		ret = -ENXIO;
 		break;
diff --git a/arch/arm64/kvm/reset.c b/arch/arm64/kvm/reset.c
index f32490229a4c..4dc205fa4be1 100644
--- a/arch/arm64/kvm/reset.c
+++ b/arch/arm64/kvm/reset.c
@@ -87,6 +87,9 @@  int kvm_arch_vm_ioctl_check_extension(struct kvm *kvm, long ext)
 	case KVM_CAP_ARM_PTRAUTH_GENERIC:
 		r = system_has_full_ptr_auth();
 		break;
+	case KVM_CAP_ARM_SPE:
+		r = kvm_arm_supports_spe();
+		break;
 	default:
 		r = 0;
 	}
@@ -223,6 +226,19 @@  static int kvm_vcpu_enable_ptrauth(struct kvm_vcpu *vcpu)
 	return 0;
 }
 
+static int kvm_vcpu_enable_spe(struct kvm_vcpu *vcpu)
+{
+	if (!kvm_arm_supports_spe())
+		return -EINVAL;
+
+	/* SPE is disabled if the PE is in AArch32 state */
+	if (test_bit(KVM_ARM_VCPU_EL1_32BIT, vcpu->arch.features))
+		return -EINVAL;
+
+	vcpu->arch.flags |= KVM_ARM64_GUEST_HAS_SPE;
+	return 0;
+}
+
 /**
  * kvm_reset_vcpu - sets core registers and sys_regs to reset value
  * @vcpu: The VCPU pointer
@@ -274,6 +290,13 @@  int kvm_reset_vcpu(struct kvm_vcpu *vcpu)
 		}
 	}
 
+	if (test_bit(KVM_ARM_VCPU_SPE, vcpu->arch.features)) {
+		if (kvm_vcpu_enable_spe(vcpu)) {
+			ret = -EINVAL;
+			goto out;
+		}
+	}
+
 	switch (vcpu->arch.target) {
 	default:
 		if (test_bit(KVM_ARM_VCPU_EL1_32BIT, vcpu->arch.features)) {
diff --git a/arch/arm64/kvm/spe.c b/arch/arm64/kvm/spe.c
new file mode 100644
index 000000000000..f91a52cd7cd3
--- /dev/null
+++ b/arch/arm64/kvm/spe.c
@@ -0,0 +1,129 @@ 
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright (C) 2019 ARM Ltd.
+ */
+
+#include <linux/kvm.h>
+#include <linux/kvm_host.h>
+#include <linux/uaccess.h>
+
+#include <kvm/arm_spe.h>
+#include <kvm/arm_vgic.h>
+
+static bool kvm_arm_vcpu_supports_spe(struct kvm_vcpu *vcpu)
+{
+	if (!vcpu_has_spe(vcpu))
+		return false;
+
+	if (!irqchip_in_kernel(vcpu->kvm))
+		return false;
+
+	return true;
+}
+
+static int kvm_arm_spe_init(struct kvm_vcpu *vcpu)
+{
+	if (!kvm_arm_spe_irq_initialized(vcpu))
+		return -ENXIO;
+
+	if (!vgic_initialized(vcpu->kvm))
+		return -ENODEV;
+
+	if (kvm_arm_spe_vcpu_initialized(vcpu))
+		return -EBUSY;
+
+	if (kvm_vgic_set_owner(vcpu, vcpu->arch.spe_cpu.irq_num, &vcpu->arch.spe_cpu))
+		return -ENXIO;
+
+	vcpu->arch.spe_cpu.initialized = true;
+
+	return 0;
+}
+
+static bool kvm_arm_spe_irq_is_valid(struct kvm *kvm, int irq)
+{
+	int i;
+	struct kvm_vcpu *vcpu;
+
+	/* The SPE overflow interrupt can be a PPI only */
+	if (!irq_is_ppi(irq))
+		return false;
+
+	kvm_for_each_vcpu(i, vcpu, kvm) {
+		if (!kvm_arm_spe_irq_initialized(vcpu))
+			continue;
+
+		if (vcpu->arch.spe_cpu.irq_num != irq)
+			return false;
+	}
+
+	return true;
+}
+
+int kvm_arm_spe_set_attr(struct kvm_vcpu *vcpu, struct kvm_device_attr *attr)
+{
+	switch (attr->attr) {
+	case KVM_ARM_VCPU_SPE_IRQ: {
+		int __user *uaddr = (int __user *)(long)attr->addr;
+		int irq;
+
+		if (!kvm_arm_vcpu_supports_spe(vcpu))
+			return -ENXIO;
+
+		if (get_user(irq, uaddr))
+			return -EFAULT;
+
+		if (!kvm_arm_spe_irq_is_valid(vcpu->kvm, irq))
+			return -EINVAL;
+
+		if (kvm_arm_spe_irq_initialized(vcpu))
+			return -EBUSY;
+
+		kvm_debug("Set kvm ARM SPE irq: %d\n", irq);
+		vcpu->arch.spe_cpu.irq_num = irq;
+
+		return 0;
+	}
+	case KVM_ARM_VCPU_SPE_INIT:
+		return kvm_arm_spe_init(vcpu);
+	}
+
+	return -ENXIO;
+}
+
+int kvm_arm_spe_get_attr(struct kvm_vcpu *vcpu, struct kvm_device_attr *attr)
+{
+	switch (attr->attr) {
+	case KVM_ARM_VCPU_SPE_IRQ: {
+		int __user *uaddr = (int __user *)(long)attr->addr;
+		int irq;
+
+		if (!kvm_arm_vcpu_supports_spe(vcpu))
+			return -ENXIO;
+
+		if (!kvm_arm_spe_irq_initialized(vcpu))
+			return -ENXIO;
+
+		irq = vcpu->arch.spe_cpu.irq_num;
+		if (put_user(irq, uaddr))
+			return -EFAULT;
+
+		return 0;
+	}
+	}
+
+	return -ENXIO;
+}
+
+int kvm_arm_spe_has_attr(struct kvm_vcpu *vcpu, struct kvm_device_attr *attr)
+{
+	switch (attr->attr) {
+	case KVM_ARM_VCPU_SPE_IRQ:
+		fallthrough;
+	case KVM_ARM_VCPU_SPE_INIT:
+		if (kvm_arm_vcpu_supports_spe(vcpu))
+			return 0;
+	}
+
+	return -ENXIO;
+}
diff --git a/include/kvm/arm_spe.h b/include/kvm/arm_spe.h
index 46ec447ed013..0275e8097529 100644
--- a/include/kvm/arm_spe.h
+++ b/include/kvm/arm_spe.h
@@ -18,11 +18,38 @@  struct kvm_spe_cpu {
 	bool initialized; 	/* Feature is initialized on VCPU */
 };
 
+#define kvm_arm_spe_irq_initialized(v)			\
+	((v)->arch.spe_cpu.irq_num >= VGIC_NR_SGIS &&	\
+	 (v)->arch.spe_cpu.irq_num < VGIC_MAX_PRIVATE)
+#define kvm_arm_spe_vcpu_initialized(v)	((v)->arch.spe_cpu.initialized)
+
+int kvm_arm_spe_set_attr(struct kvm_vcpu *vcpu, struct kvm_device_attr *attr);
+int kvm_arm_spe_get_attr(struct kvm_vcpu *vcpu, struct kvm_device_attr *attr);
+int kvm_arm_spe_has_attr(struct kvm_vcpu *vcpu, struct kvm_device_attr *attr);
+
 #else
 #define kvm_arm_supports_spe()	false
 
 struct kvm_spe_cpu {
 };
 
+#define kvm_arm_spe_irq_initialized(v)	false
+#define kvm_arm_spe_vcpu_initialized(v)	false
+
+static inline int kvm_arm_spe_set_attr(struct kvm_vcpu *vcpu,
+				       struct kvm_device_attr *attr)
+{
+	return -ENXIO;
+}
+static inline int kvm_arm_spe_get_attr(struct kvm_vcpu *vcpu,
+				       struct kvm_device_attr *attr)
+{
+	return -ENXIO;
+}
+static inline int kvm_arm_spe_has_attr(struct kvm_vcpu *vcpu,
+				       struct kvm_device_attr *attr)
+{
+	return -ENXIO;
+}
 #endif /* CONFIG_KVM_ARM_SPE */
 #endif /* __ASM_ARM_KVM_SPE_H */
diff --git a/include/uapi/linux/kvm.h b/include/uapi/linux/kvm.h
index ca41220b40b8..96228b823711 100644
--- a/include/uapi/linux/kvm.h
+++ b/include/uapi/linux/kvm.h
@@ -1053,6 +1053,7 @@  struct kvm_ppc_resize_hpt {
 #define KVM_CAP_X86_USER_SPACE_MSR 188
 #define KVM_CAP_X86_MSR_FILTER 189
 #define KVM_CAP_ENFORCE_PV_FEATURE_CPUID 190
+#define KVM_CAP_ARM_SPE 191
 
 #ifdef KVM_CAP_IRQ_ROUTING