@@ -73,6 +73,8 @@ vmlinux-objs-y := $(obj)/vmlinux.lds $(obj)/head_$(BITS).o $(obj)/misc.o \
$(obj)/string.o $(obj)/cmdline.o $(obj)/error.o \
$(obj)/piggy.o $(obj)/cpuflags.o
+vmlinux-objs-$(CONFIG_X86_64) += $(obj)/mem_encrypt.o
+
vmlinux-objs-$(CONFIG_EARLY_PRINTK) += $(obj)/early_serial_console.o
vmlinux-objs-$(CONFIG_RANDOMIZE_BASE) += $(obj)/kaslr.o
ifdef CONFIG_X86_64
@@ -131,6 +131,19 @@ ENTRY(startup_32)
/*
* Build early 4G boot pagetable
*/
+ /*
+ * If SEV is active set the encryption mask in the page tables. This
+ * will insure that when the kernel is copied and decompressed it
+ * will be done so encrypted.
+ */
+ call sev_active
+ xorl %edx, %edx
+ testl %eax, %eax
+ jz 1f
+ subl $32, %eax /* Encryption bit is always above bit 31 */
+ bts %eax, %edx /* Set encryption mask for page tables */
+1:
+
/* Initialize Page tables to 0 */
leal pgtable(%ebx), %edi
xorl %eax, %eax
@@ -141,12 +154,14 @@ ENTRY(startup_32)
leal pgtable + 0(%ebx), %edi
leal 0x1007 (%edi), %eax
movl %eax, 0(%edi)
+ addl %edx, 4(%edi)
/* Build Level 3 */
leal pgtable + 0x1000(%ebx), %edi
leal 0x1007(%edi), %eax
movl $4, %ecx
1: movl %eax, 0x00(%edi)
+ addl %edx, 0x04(%edi)
addl $0x00001000, %eax
addl $8, %edi
decl %ecx
@@ -157,6 +172,7 @@ ENTRY(startup_32)
movl $0x00000183, %eax
movl $2048, %ecx
1: movl %eax, 0(%edi)
+ addl %edx, 4(%edi)
addl $0x00200000, %eax
addl $8, %edi
decl %ecx
@@ -344,6 +360,9 @@ preferred_addr:
subl $_end, %ebx
addq %rbp, %rbx
+ /* Check for SEV and adjust page tables as necessary */
+ call sev_adjust
+
/* Set up the stack */
leaq boot_stack_end(%rbx), %rsp
new file mode 100644
@@ -0,0 +1,123 @@
+/*
+ * AMD Memory Encryption Support
+ *
+ * Copyright (C) 2016 Advanced Micro Devices, Inc.
+ *
+ * Author: Tom Lendacky <thomas.lendacky@amd.com>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ */
+
+#include <linux/linkage.h>
+
+#include <asm/processor-flags.h>
+#include <asm/msr.h>
+#include <asm/asm-offsets.h>
+#include <uapi/asm/kvm_para.h>
+
+ .text
+ .code32
+ENTRY(sev_active)
+ xor %eax, %eax
+
+#ifdef CONFIG_AMD_MEM_ENCRYPT
+ push %ebx
+ push %ecx
+ push %edx
+
+ /* Check if running under a hypervisor */
+ movl $0x40000000, %eax
+ cpuid
+ cmpl $0x40000001, %eax
+ jb .Lno_sev
+
+ movl $0x40000001, %eax
+ cpuid
+ bt $KVM_FEATURE_SEV, %eax
+ jnc .Lno_sev
+
+ /*
+ * Check for memory encryption feature:
+ * CPUID Fn8000_001F[EAX] - Bit 0
+ */
+ movl $0x8000001f, %eax
+ cpuid
+ bt $0, %eax
+ jnc .Lno_sev
+
+ /*
+ * Get memory encryption information:
+ * CPUID Fn8000_001F[EBX] - Bits 5:0
+ * Pagetable bit position used to indicate encryption
+ */
+ movl %ebx, %eax
+ andl $0x3f, %eax
+ jmp .Lsev_exit
+
+.Lno_sev:
+ xor %eax, %eax
+
+.Lsev_exit:
+ pop %edx
+ pop %ecx
+ pop %ebx
+
+#endif /* CONFIG_AMD_MEM_ENCRYPT */
+
+ ret
+ENDPROC(sev_active)
+
+ .code64
+ENTRY(sev_adjust)
+#ifdef CONFIG_AMD_MEM_ENCRYPT
+ push %rax
+ push %rbx
+ push %rcx
+ push %rdx
+
+ /* Check if running under a hypervisor */
+ movl $0x40000000, %eax
+ cpuid
+ cmpl $0x40000001, %eax
+ jb .Lno_adjust
+
+ movl $0x40000001, %eax
+ cpuid
+ bt $KVM_FEATURE_SEV, %eax
+ jnc .Lno_adjust
+
+ /*
+ * Check for memory encryption feature:
+ * CPUID Fn8000_001F[EAX] - Bit 0
+ */
+ movl $0x8000001f, %eax
+ cpuid
+ bt $0, %eax
+ jnc .Lno_adjust
+
+ /*
+ * Get memory encryption information:
+ * CPUID Fn8000_001F[EBX] - Bits 5:0
+ * Pagetable bit position used to indicate encryption
+ */
+ movl %ebx, %ecx
+ andl $0x3f, %ecx
+ jz .Lno_adjust
+
+ /*
+ * Adjust/verify the page table entries to include the encryption
+ * mask for the area where the compressed kernel is copied and
+ * the area the kernel is decompressed into
+ */
+
+.Lno_adjust:
+ pop %rdx
+ pop %rcx
+ pop %rbx
+ pop %rax
+#endif /* CONFIG_AMD_MEM_ENCRYPT */
+
+ ret
+ENDPROC(sev_adjust)
@@ -3,6 +3,8 @@
#include <linux/types.h>
+#ifndef __ASSEMBLY__
+
/*
* The below CPUID leaves are present if VersionAndFeatures.HypervisorPresent
* is set by CPUID(HvCpuIdFunctionVersionAndFeatures).
@@ -363,4 +365,6 @@ struct hv_timer_message_payload {
#define HV_STIMER_AUTOENABLE (1ULL << 3)
#define HV_STIMER_SINT(config) (__u8)(((config) >> 16) & 0x0F)
+#endif /* __ASSEMBLY__ */
+
#endif
@@ -26,6 +26,8 @@
#define KVM_FEATURE_PV_UNHALT 7
#define KVM_FEATURE_SEV 8
+#ifndef __ASSEMBLY__
+
/* The last 8 bits are used to indicate how to interpret the flags field
* in pvclock structure. If no bits are set, all flags are ignored.
*/
@@ -98,5 +100,6 @@ struct kvm_vcpu_pv_apf_data {
#define KVM_PV_EOI_ENABLED KVM_PV_EOI_MASK
#define KVM_PV_EOI_DISABLED 0x0
+#endif /* __ASSEMBLY__ */
#endif /* _UAPI_ASM_X86_KVM_PARA_H */
@@ -17,11 +17,47 @@
#include <asm/page.h>
#include <asm/msr.h>
#include <asm/asm-offsets.h>
+#include <uapi/asm/kvm_para.h>
.text
.code64
ENTRY(sme_enable)
#ifdef CONFIG_AMD_MEM_ENCRYPT
+ /* Check if running under a hypervisor */
+ movl $0x40000000, %eax
+ cpuid
+ cmpl $0x40000001, %eax
+ jb .Lno_hyp
+
+ movl $0x40000001, %eax
+ cpuid
+ bt $KVM_FEATURE_SEV, %eax
+ jnc .Lno_mem_encrypt
+
+ /*
+ * Check for memory encryption feature:
+ * CPUID Fn8000_001F[EAX] - Bit 0
+ */
+ movl $0x8000001f, %eax
+ cpuid
+ bt $0, %eax
+ jnc .Lno_mem_encrypt
+
+ /*
+ * Get memory encryption information:
+ * CPUID Fn8000_001F[EBX] - Bits 5:0
+ * Pagetable bit position used to indicate encryption
+ */
+ movl %ebx, %ecx
+ andl $0x3f, %ecx
+ jz .Lno_mem_encrypt
+ bts %ecx, sme_me_mask(%rip)
+
+ /* Indicate that SEV is active */
+ movl $1, sev_active(%rip)
+ jmp .Lmem_encrypt_exit
+
+.Lno_hyp:
/* Check for AMD processor */
xorl %eax, %eax
cpuid