@@ -62,6 +62,8 @@ extern unsigned long initial_gs;
extern unsigned long initial_stack;
#ifdef CONFIG_AMD_MEM_ENCRYPT
extern unsigned long initial_vc_handler;
+extern unsigned char rm_ap_jump_table_blob[];
+extern unsigned char rm_ap_jump_table_blob_end[];
#endif
extern unsigned char real_mode_blob[];
new file mode 100644
@@ -0,0 +1,25 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * AMD Encrypted Register State Support
+ *
+ * Author: Joerg Roedel <jroedel@suse.de>
+ */
+#ifndef __ASM_SEV_AP_JUMPTABLE_H
+#define __ASM_SEV_AP_JUMPTABLE_H
+
+#define SEV_APJT_CS16 0x8
+#define SEV_APJT_DS16 0x10
+
+#define SEV_APJT_ENTRY 0x10
+
+#ifndef __ASSEMBLY__
+
+struct sev_ap_jump_table_header {
+ u16 reset_ip;
+ u16 reset_cs;
+ u16 gdt_offset;
+};
+
+#endif /* !__ASSEMBLY__ */
+
+#endif /* __ASM_SEV_AP_JUMPTABLE_H */
@@ -19,6 +19,7 @@
#include <linux/kernel.h>
#include <linux/mm.h>
+#include <asm/sev-ap-jumptable.h>
#include <asm/cpu_entry_area.h>
#include <asm/stacktrace.h>
#include <asm/sev.h>
@@ -45,6 +46,9 @@ static struct ghcb __initdata *boot_ghcb;
/* Cached AP Jump Table Address */
static phys_addr_t sev_es_jump_table_pa;
+/* Whether the AP Jump Table blob was successfully installed */
+static bool sev_ap_jumptable_blob_installed __ro_after_init;
+
/* #VC handler runtime per-CPU data */
struct sev_es_runtime_data {
struct ghcb ghcb_page;
@@ -749,6 +753,107 @@ static void __init sev_es_setup_play_dead(void)
static inline void sev_es_setup_play_dead(void) { }
#endif
+/*
+ * This function make the necessary runtime changes to the AP Jump Table blob.
+ * For now this only sets up the GDT used while the code executes. The GDT needs
+ * to contain 16-bit code and data segments with a base that points to AP Jump
+ * Table page.
+ */
+void __init sev_es_setup_ap_jump_table_data(void *base, u32 pa)
+{
+ struct sev_ap_jump_table_header *header;
+ struct desc_ptr *gdt_descr;
+ u64 *ap_jumptable_gdt;
+
+ header = base;
+
+ /*
+ * Setup 16-bit protected mode code and data segments for AP Jumptable.
+ * Set the segment limits to 0xffff to already be compatible with
+ * real-mode.
+ */
+ ap_jumptable_gdt = (u64 *)(base + header->gdt_offset);
+ ap_jumptable_gdt[SEV_APJT_CS16 / 8] = GDT_ENTRY(0x9b, pa, 0xffff);
+ ap_jumptable_gdt[SEV_APJT_DS16 / 8] = GDT_ENTRY(0x93, pa, 0xffff);
+
+ /* Write correct GDT base address into GDT descriptor */
+ gdt_descr = (struct desc_ptr *)(base + header->gdt_offset);
+ gdt_descr->address += pa;
+}
+
+/*
+ * This function sets up the AP Jump Table blob which contains code which runs
+ * in 16-bit protected mode to park an AP. After the AP is woken up again the
+ * code will disable protected mode and jump to the reset vector which is also
+ * stored in the AP Jump Table.
+ *
+ * The Jump Table is a safe place to park an AP, because it is owned by the
+ * BIOS and writable by the OS. Putting the code in kernel memory would break
+ * with kexec, because by the time th APs wake up the memory is owned by
+ * the new kernel, and possibly already overwritten.
+ *
+ * Kexec is also the reason this function is called as an init-call after SMP
+ * bringup. Only after all CPUs are up there is a guarantee that no AP is still
+ * parked in AP jump-table code.
+ */
+static int __init sev_es_setup_ap_jump_table_blob(void)
+{
+ size_t blob_size = rm_ap_jump_table_blob_end - rm_ap_jump_table_blob;
+ u16 startup_cs, startup_ip;
+ u16 __iomem *jump_table;
+ phys_addr_t pa;
+
+ if (!sev_es_active())
+ return 0;
+
+ if (sev_get_ghcb_proto_ver() < 2) {
+ pr_info("AP Jump Table parking requires at least GHCB protocol version 2\n");
+ return 0;
+ }
+
+ pa = get_jump_table_addr();
+
+ /* Overflow and size checks for untrusted Jump Table address */
+ if (pa + PAGE_SIZE < pa || pa + PAGE_SIZE > SZ_4G) {
+ pr_info("AP Jump Table is above 4GB - not enabling AP Jump Table parking\n");
+ return 0;
+ }
+
+ /* On UP guests there is no jump table so this is not a failure */
+ if (!pa)
+ return 0;
+
+ jump_table = ioremap_encrypted(pa, PAGE_SIZE);
+ if (WARN_ON(!jump_table))
+ return -EINVAL;
+
+ /*
+ * Safe reset vector to restore it later because the blob will
+ * overwrite it.
+ */
+ startup_ip = jump_table[0];
+ startup_cs = jump_table[1];
+
+ /* Install AP Jump Table Blob with real mode AP parking code */
+ memcpy_toio(jump_table, rm_ap_jump_table_blob, blob_size);
+
+ /* Setup AP Jumptable GDT */
+ sev_es_setup_ap_jump_table_data(jump_table, (u32)pa);
+
+ writew(startup_ip, &jump_table[0]);
+ writew(startup_cs, &jump_table[1]);
+
+ iounmap(jump_table);
+
+ pr_info("AP Jump Table Blob successfully set up\n");
+
+ /* Mark AP Jump Table blob as available */
+ sev_ap_jumptable_blob_installed = true;
+
+ return 0;
+}
+core_initcall(sev_es_setup_ap_jump_table_blob);
+
static void __init alloc_runtime_data(int cpu)
{
struct sev_es_runtime_data *data;
@@ -11,12 +11,19 @@
KASAN_SANITIZE := n
KCSAN_SANITIZE := n
+RMPIGGY-y = $(obj)/rm/realmode.bin
+RMPIGGY-$(CONFIG_AMD_MEM_ENCRYPT) += $(obj)/sev/ap_jump_table.bin
+
subdir- := rm
+subdir- := sev
obj-y += init.o
obj-y += rmpiggy.o
-$(obj)/rmpiggy.o: $(obj)/rm/realmode.bin
+$(obj)/rmpiggy.o: $(RMPIGGY-y)
$(obj)/rm/realmode.bin: FORCE
$(Q)$(MAKE) $(build)=$(obj)/rm $@
+
+$(obj)/sev/ap_jump_table.bin: FORCE
+ $(Q)$(MAKE) $(build)=$(obj)/sev $@
@@ -17,3 +17,9 @@ SYM_DATA_END_LABEL(real_mode_blob, SYM_L_GLOBAL, real_mode_blob_end)
SYM_DATA_START(real_mode_relocs)
.incbin "arch/x86/realmode/rm/realmode.relocs"
SYM_DATA_END(real_mode_relocs)
+
+#ifdef CONFIG_AMD_MEM_ENCRYPT
+SYM_DATA_START(rm_ap_jump_table_blob)
+ .incbin "arch/x86/realmode/sev/ap_jump_table.bin"
+SYM_DATA_END_LABEL(rm_ap_jump_table_blob, SYM_L_GLOBAL, rm_ap_jump_table_blob_end)
+#endif
new file mode 100644
@@ -0,0 +1,41 @@
+#
+# arch/x86/sev/Makefile
+#
+# This file is subject to the terms and conditions of the GNU General Public
+# License. See the file "COPYING" in the main directory of this archive
+# for more details.
+#
+
+# Sanitizer runtimes are unavailable and cannot be linked here.
+KASAN_SANITIZE := n
+KCSAN_SANITIZE := n
+OBJECT_FILES_NON_STANDARD := y
+
+# Prevents link failures: __sanitizer_cov_trace_pc() is not linked in.
+KCOV_INSTRUMENT := n
+
+always-y := ap_jump_table.bin
+
+ap_jump_table-y += ap_jump_table.o
+
+targets += $(ap_jump_table-y)
+
+APJUMPTABLE_OBJS = $(addprefix $(obj)/,$(ap_jump_table-y))
+
+LDFLAGS_ap_jump_table.elf := -m elf_i386 -T
+
+targets += ap_jump_table.elf
+$(obj)/ap_jump_table.elf: $(obj)/ap_jump_table.lds $(APJUMPTABLE_OBJS) FORCE
+ $(call if_changed,ld)
+
+OBJCOPYFLAGS_ap_jump_table.bin := -O binary
+
+targets += ap_jump_table.bin
+$(obj)/ap_jump_table.bin: $(obj)/ap_jump_table.elf FORCE
+ $(call if_changed,objcopy)
+
+# ---------------------------------------------------------------------------
+
+KBUILD_AFLAGS := $(REALMODE_CFLAGS) -D__ASSEMBLY__
+GCOV_PROFILE := n
+UBSAN_SANITIZE := n
new file mode 100644
@@ -0,0 +1,130 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+
+#include <linux/linkage.h>
+#include <asm/sev-ap-jumptable.h>
+
+/*
+ * This file contains the source code for the binary blob which gets copied to
+ * the SEV-ES AP Jumptable to park APs while offlining CPUs or booting a new
+ * kernel via KEXEC.
+ *
+ * The AP Jumptable is the only safe place to put this code, as any memory the
+ * kernel allocates will be owned (and possibly overwritten) by the new kernel
+ * once the APs are woken up.
+ *
+ * This code runs in 16-bit protected mode, the CS, DS, and SS segment bases are
+ * set to the beginning of the AP Jumptable page.
+ *
+ * Since the GDT will also be gone when the AP wakes up, this blob contains its
+ * own GDT, which is set up by the AP Jumptable setup code with the correct
+ * offsets.
+ *
+ * Author: Joerg Roedel <jroedel@suse.de>
+ */
+
+ .text
+ .org 0x0
+ .code16
+SYM_DATA_START(ap_jumptable_header)
+ .word 0 /* reset IP */
+ .word 0 /* reset CS */
+ .word ap_jumptable_gdt /* GDT Offset */
+SYM_DATA_END(ap_jumptable_header)
+
+ .org SEV_APJT_ENTRY
+SYM_CODE_START(ap_park_asm)
+
+ /* Switch to AP Jumptable GDT first */
+ lgdtl ap_jumptable_gdt
+
+ /* Reload CS */
+ ljmpw $SEV_APJT_CS16, $1f
+1:
+
+ /* Reload DS and SS */
+ movl $SEV_APJT_DS16, %ecx
+ movl %ecx, %ds
+ movl %ecx, %ss
+
+ /*
+ * Setup a stack pointing to the end of the AP Jumptable page.
+ * The stack is needed ot reset EFLAGS after wakeup.
+ */
+ movl $0x1000, %esp
+
+ /* Execute AP reset hold VMGEXIT */
+2: xorl %edx, %edx
+ movl $0x6, %eax
+ movl $0xc0010130, %ecx
+ wrmsr
+ rep; vmmcall
+ rdmsr
+ movl %eax, %ecx
+ andl $0xfff, %ecx
+ cmpl $0x7, %ecx
+ jne 2b
+ shrl $12, %eax
+ jnz 3f
+ testl %edx, %edx
+ jnz 3f
+ jmp 2b
+3:
+ /*
+ * Successfully woken up - Patch the correct target into the far jump at
+ * the end. An indirect far jump does not work here, because at the time
+ * the jump is executed DS is already loaded with real-mode values.
+ */
+
+ /* Jump target is at address 0x0 - copy it to the far jump instruction */
+ movl $0, %ecx
+ movl (%ecx), %eax
+ movl %eax, jump_target
+
+ /* Reset EFLAGS */
+ pushl $2
+ popfl
+
+ /* Setup DS and SS for real-mode */
+ movl $0x18, %ecx
+ movl %ecx, %ds
+ movl %ecx, %ss
+
+ /* Reset remaining registers */
+ movl $0, %esp
+ movl $0, %eax
+ movl $0, %ebx
+ movl $0, %edx
+
+ /* Reset CR0 to get out of protected mode */
+ movl $0x60000010, %ecx
+ movl %ecx, %cr0
+
+ /*
+ * The below sums up to a far-jump instruction which jumps to the reset
+ * vector configured in the AP Jumptable and to real-mode. An indirect
+ * jump would be cleaner, but requires a working DS base/limit. DS is
+ * already loaded with real-mode values, therefore a direct far jump is
+ * used which got the correct target patched in.
+ */
+ .byte 0xea
+SYM_DATA_LOCAL(jump_target, .long 0)
+
+SYM_CODE_END(ap_park_asm)
+ /* Here comes the GDT */
+ .balign 16
+SYM_DATA_START_LOCAL(ap_jumptable_gdt)
+ /* Offset zero used for GDT descriptor */
+ .word ap_jumptable_gdt_end - ap_jumptable_gdt - 1
+ .long ap_jumptable_gdt
+ .word 0
+
+ /* 16 bit code segment - setup at boot */
+ .quad 0
+
+ /* 16 bit data segment - setup at boot */
+ .quad 0
+
+ /* Offset 0x8 - real-mode data segment */
+ .long 0xffff0180
+ .long 0x00009300
+SYM_DATA_END_LABEL(ap_jumptable_gdt, SYM_L_LOCAL, ap_jumptable_gdt_end)
new file mode 100644
@@ -0,0 +1,24 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * ap_jump_table.lds
+ *
+ * Linker script for the SEV-ES AP Jump Table Code
+ */
+
+OUTPUT_FORMAT("elf32-i386")
+OUTPUT_ARCH(i386)
+ENTRY(ap_park_asm)
+
+SECTIONS
+{
+ . = 0;
+ .text : {
+ *(.text)
+ *(.text.*)
+ }
+
+ /DISCARD/ : {
+ *(.note*)
+ *(.debug*)
+ }
+}