@@ -24,6 +24,7 @@ kvm-$(CONFIG_KVM_XEN) += xen.o
kvm-intel-y += vmx/vmx.o vmx/vmenter.o vmx/pmu_intel.o vmx/vmcs12.o \
vmx/evmcs.o vmx/nested.o vmx/posted_intr.o vmx/main.o
kvm-intel-$(CONFIG_X86_SGX_KVM) += vmx/sgx.o
+kvm-intel-$(CONFIG_INTEL_TDX_HOST) += vmx/tdx.o
kvm-amd-y += svm/svm.o svm/vmenter.o svm/pmu.o svm/nested.o svm/avic.o svm/sev.o
@@ -7,6 +7,22 @@
#include "pmu.h"
#include "tdx.h"
+static bool __read_mostly enable_tdx = IS_ENABLED(CONFIG_INTEL_TDX_HOST);
+module_param_named(tdx, enable_tdx, bool, 0444);
+
+static __init int vt_hardware_setup(void)
+{
+ int ret;
+
+ ret = vmx_hardware_setup();
+ if (ret)
+ return ret;
+
+ enable_tdx = enable_tdx && !tdx_hardware_setup(&vt_x86_ops);
+
+ return 0;
+}
+
struct kvm_x86_ops vt_x86_ops __initdata = {
.name = "kvm_intel",
@@ -148,7 +164,7 @@ struct kvm_x86_ops vt_x86_ops __initdata = {
struct kvm_x86_init_ops vt_init_ops __initdata = {
.cpu_has_kvm_support = vmx_cpu_has_kvm_support,
.disabled_by_bios = vmx_disabled_by_bios,
- .hardware_setup = vmx_hardware_setup,
+ .hardware_setup = vt_hardware_setup,
.handle_intel_pt_intr = NULL,
.runtime_ops = &vt_x86_ops,
new file mode 100644
@@ -0,0 +1,99 @@
+// SPDX-License-Identifier: GPL-2.0
+#include <linux/cpu.h>
+
+#include <asm/tdx.h>
+
+#include "capabilities.h"
+#include "x86_ops.h"
+#include "tdx.h"
+#include "x86.h"
+
+#undef pr_fmt
+#define pr_fmt(fmt) "tdx: " fmt
+
+#define TDX_MAX_NR_CPUID_CONFIGS \
+ ((sizeof(struct tdsysinfo_struct) - \
+ offsetof(struct tdsysinfo_struct, cpuid_configs)) \
+ / sizeof(struct tdx_cpuid_config))
+
+struct tdx_capabilities {
+ u8 tdcs_nr_pages;
+ u8 tdvpx_nr_pages;
+
+ u64 attrs_fixed0;
+ u64 attrs_fixed1;
+ u64 xfam_fixed0;
+ u64 xfam_fixed1;
+
+ u32 nr_cpuid_configs;
+ struct tdx_cpuid_config cpuid_configs[TDX_MAX_NR_CPUID_CONFIGS];
+};
+
+/* Capabilities of KVM + the TDX module. */
+static struct tdx_capabilities tdx_caps;
+
+static int __init tdx_module_setup(void)
+{
+ const struct tdsysinfo_struct *tdsysinfo;
+ int ret = 0;
+
+ BUILD_BUG_ON(sizeof(*tdsysinfo) != 1024);
+ BUILD_BUG_ON(TDX_MAX_NR_CPUID_CONFIGS != 37);
+
+ ret = tdx_init();
+ if (ret) {
+ pr_info("Failed to initialize TDX module.\n");
+ return ret;
+ }
+
+ tdsysinfo = tdx_get_sysinfo();
+ if (tdsysinfo->num_cpuid_config > TDX_MAX_NR_CPUID_CONFIGS)
+ return -EIO;
+
+ tdx_caps = (struct tdx_capabilities) {
+ .tdcs_nr_pages = tdsysinfo->tdcs_base_size / PAGE_SIZE,
+ /*
+ * TDVPS = TDVPR(4K page) + TDVPX(multiple 4K pages).
+ * -1 for TDVPR.
+ */
+ .tdvpx_nr_pages = tdsysinfo->tdvps_base_size / PAGE_SIZE - 1,
+ .attrs_fixed0 = tdsysinfo->attributes_fixed0,
+ .attrs_fixed1 = tdsysinfo->attributes_fixed1,
+ .xfam_fixed0 = tdsysinfo->xfam_fixed0,
+ .xfam_fixed1 = tdsysinfo->xfam_fixed1,
+ .nr_cpuid_configs = tdsysinfo->num_cpuid_config,
+ };
+ if (!memcpy(tdx_caps.cpuid_configs, tdsysinfo->cpuid_configs,
+ tdsysinfo->num_cpuid_config *
+ sizeof(struct tdx_cpuid_config)))
+ return -EIO;
+
+ pr_info("kvm: TDX is supported. x86 phys bits %d\n",
+ boot_cpu_data.x86_phys_bits);
+
+ return 0;
+}
+
+int __init tdx_hardware_setup(struct kvm_x86_ops *x86_ops)
+{
+ int r;
+
+ if (!enable_ept) {
+ pr_warn("Cannot enable TDX with EPT disabled\n");
+ return -EINVAL;
+ }
+
+ /* MOVDIR64B instruction is needed. */
+ if (!static_cpu_has(X86_FEATURE_MOVDIR64B)) {
+ pr_warn("Cannot enable TDX with MOVDIR64B supported ");
+ return -ENODEV;
+ }
+
+ /* TDX requires VMX. */
+ r = vmxon_all();
+ if (!r)
+ r = tdx_module_setup();
+ vmxoff_all();
+
+ return r;
+}
@@ -2464,6 +2464,35 @@ int vmx_hardware_enable(void)
return 0;
}
+static void __init vmxon(void *arg)
+{
+ int cpu = raw_smp_processor_id();
+ u64 phys_addr = __pa(per_cpu(vmxarea, cpu));
+ atomic_t *failed = arg;
+ int r;
+
+ if (cr4_read_shadow() & X86_CR4_VMXE) {
+ r = -EBUSY;
+ goto out;
+ }
+
+ r = kvm_cpu_vmxon(phys_addr);
+out:
+ if (r)
+ atomic_inc(failed);
+}
+
+int __init vmxon_all(void)
+{
+ atomic_t failed = ATOMIC_INIT(0);
+
+ on_each_cpu(vmxon, &failed, 1);
+
+ if (atomic_read(&failed))
+ return -EBUSY;
+ return 0;
+}
+
static void vmclear_local_loaded_vmcss(void)
{
int cpu = raw_smp_processor_id();
@@ -2484,6 +2513,16 @@ void vmx_hardware_disable(void)
intel_pt_handle_vmx(0);
}
+static void __init vmxoff(void *junk)
+{
+ cpu_vmxoff();
+}
+
+void __init vmxoff_all(void)
+{
+ on_each_cpu(vmxoff, NULL, 1);
+}
+
/*
* There is no X86_FEATURE for SGX yet, but anyway we need to query CPUID
* directly instead of going through cpu_has(), to ensure KVM is trapping
@@ -17,6 +17,9 @@ __init int vmx_cpu_has_kvm_support(void);
__init int vmx_disabled_by_bios(void);
__init int vmx_hardware_setup(void);
+int __init vmxon_all(void);
+void __init vmxoff_all(void);
+
extern struct kvm_x86_ops vt_x86_ops __initdata;
extern struct kvm_x86_init_ops vt_init_ops __initdata;
@@ -127,4 +130,10 @@ void vmx_cancel_hv_timer(struct kvm_vcpu *vcpu);
#endif
void vmx_setup_mce(struct kvm_vcpu *vcpu);
+#ifdef CONFIG_INTEL_TDX_HOST
+int __init tdx_hardware_setup(struct kvm_x86_ops *x86_ops);
+#else
+static inline int tdx_hardware_setup(struct kvm_x86_ops *x86_ops) { return 0; }
+#endif
+
#endif /* __KVM_X86_VMX_X86_OPS_H */
@@ -11975,6 +11975,16 @@ static void hardware_enable(void *arg)
atomic_inc(failed);
}
+static int kvm_hardware_enable_all(void)
+{
+ atomic_t failed = ATOMIC_INIT(0);
+
+ on_each_cpu(hardware_enable, &failed, 1);
+ if (atomic_read(&failed))
+ return -EBUSY;
+ return 0;
+}
+
static void hardware_disable(void *junk)
{
WARN_ON_ONCE(preemptible());
@@ -11982,29 +11992,29 @@ static void hardware_disable(void *junk)
drop_user_return_notifiers();
}
+static void kvm_hardware_disable_all(void)
+{
+ on_each_cpu(hardware_disable, NULL, 1);
+}
+
/*
* Called after the VM is otherwise initialized, but just before adding it to
* the vm_list.
*/
int kvm_arch_add_vm(struct kvm *kvm, int usage_count)
{
- atomic_t failed = ATOMIC_INIT(0);
- int r = 0;
+ int r;
if (usage_count != 1)
return kvm_mmu_post_init_vm(kvm);
- on_each_cpu(hardware_enable, &failed, 1);
-
- if (atomic_read(&failed)) {
- r = -EBUSY;
- goto err;
- }
+ r = kvm_hardware_enable_all();
+ if (r)
+ return r;
r = kvm_mmu_post_init_vm(kvm);
-err:
if (r)
- on_each_cpu(hardware_disable, NULL, 1);
+ kvm_hardware_disable_all();
return r;
}
@@ -12013,7 +12023,7 @@ int kvm_arch_del_vm(int usage_count)
if (usage_count)
return 0;
- on_each_cpu(hardware_disable, NULL, 1);
+ kvm_hardware_disable_all();
return 0;
}