@@ -28,6 +28,7 @@
#include <linux/sched/clock.h>
#include <linux/mm.h>
#include <linux/slab.h>
+#include <linux/set_memory.h>
#include <asm/hypervisor.h>
#include <asm/mem_encrypt.h>
@@ -61,8 +62,8 @@ early_param("no-kvmclock-vsyscall", parse_no_kvmclock_vsyscall);
(PAGE_SIZE / sizeof(struct pvclock_vsyscall_time_info))
static struct pvclock_vsyscall_time_info
- hv_clock_boot[HVC_BOOT_ARRAY_SIZE] __aligned(PAGE_SIZE);
-static struct pvclock_wall_clock wall_clock;
+ hv_clock_boot[HVC_BOOT_ARRAY_SIZE] __decrypted __aligned(PAGE_SIZE);
+static struct pvclock_wall_clock wall_clock __decrypted;
static DEFINE_PER_CPU(struct pvclock_vsyscall_time_info *, hv_clock_per_cpu);
static inline struct pvclock_vcpu_time_info *this_cpu_pvti(void)
@@ -267,10 +268,29 @@ static int kvmclock_setup_percpu(unsigned int cpu)
return 0;
/* Use the static page for the first CPUs, allocate otherwise */
- if (cpu < HVC_BOOT_ARRAY_SIZE)
+ if (cpu < HVC_BOOT_ARRAY_SIZE) {
p = &hv_clock_boot[cpu];
- else
- p = kzalloc(sizeof(*p), GFP_KERNEL);
+ } else {
+ int rc;
+ unsigned int sz = sizeof(*p);
+
+ if (sev_active())
+ sz = PAGE_ALIGN(sz);
+
+ p = kzalloc(sz, GFP_KERNEL);
+
+ /*
+ * The physical address of per-cpu variable will be shared with
+ * the hypervisor. Let's clear the C-bit before we assign the
+ * memory to per_cpu variable.
+ */
+ if (p && sev_active()) {
+ rc = set_memory_decrypted((unsigned long)p, sz >> PAGE_SHIFT);
+ if (rc)
+ return rc;
+ memset(p, 0, sz);
+ }
+ }
per_cpu(hv_clock_per_cpu, cpu) = p;
return p ? 0 : -ENOMEM;
The following commit: 368a540e0232 (x86/kvmclock: Remove memblock dependency) caused SEV guest regression. When SEV is active, we map the shared variables (wall_clock and hv_clock_boot) with C=0 to ensure that both the guest and the hypervisor is able to access the data. To map the variables we use kernel_physical_mapping_init() to split the large pages, but this routine fails to allocate a new page. Before the above commit, kvmclock initialization was called after memory allocator was available but now its called very early in the boot process. Recently we added a special .data..decrypted section to hold the shared variables. This section is mapped with C=0 very early. Use __decrypted attribute to put the wall_clock and hv_clock_boot in .data..decrypted section so that they are mapped with C=0. Signed-off-by: Brijesh Singh <brijesh.singh@amd.com> Fixes: 368a540e0232 ("x86/kvmclock: Remove memblock dependency") Cc: stable@vger.kernel.org Cc: Tom Lendacky <thomas.lendacky@amd.com> Cc: kvm@vger.kernel.org Cc: Thomas Gleixner <tglx@linutronix.de> Cc: Borislav Petkov <bp@suse.de> Cc: "H. Peter Anvin" <hpa@zytor.com> Cc: linux-kernel@vger.kernel.org Cc: Paolo Bonzini <pbonzini@redhat.com> Cc: Sean Christopherson <sean.j.christopherson@intel.com> Cc: kvm@vger.kernel.org Cc: "Radim Krčmář" <rkrcmar@redhat.com> --- arch/x86/kernel/kvmclock.c | 30 +++++++++++++++++++++++++----- 1 file changed, 25 insertions(+), 5 deletions(-)