@@ -42,6 +42,7 @@ struct vmcs_host_state {
#ifdef CONFIG_X86_64
u16 ds_sel, es_sel;
#endif
+ u32 pkrs;
};
struct vmcs_controls_shadow {
@@ -28,6 +28,7 @@
#include <linux/tboot.h>
#include <linux/trace_events.h>
#include <linux/entry-kvm.h>
+#include <linux/pkeys.h>
#include <asm/apic.h>
#include <asm/asm.h>
@@ -172,6 +173,7 @@ static u32 vmx_possible_passthrough_msrs[MAX_POSSIBLE_PASSTHROUGH_MSRS] = {
MSR_CORE_C3_RESIDENCY,
MSR_CORE_C6_RESIDENCY,
MSR_CORE_C7_RESIDENCY,
+ MSR_IA32_PKRS,
};
/*
@@ -1121,6 +1123,7 @@ void vmx_prepare_switch_to_guest(struct kvm_vcpu *vcpu)
#endif
unsigned long fs_base, gs_base;
u16 fs_sel, gs_sel;
+ u32 host_pkrs;
int i;
vmx->req_immediate_exit = false;
@@ -1156,6 +1159,20 @@ void vmx_prepare_switch_to_guest(struct kvm_vcpu *vcpu)
*/
host_state->ldt_sel = kvm_read_ldt();
+ /*
+ * Update the host pkrs vmcs field before vcpu runs.
+ * The setting of VM_EXIT_LOAD_IA32_PKRS can ensure
+ * kvm_cpu_cap_has(X86_FEATURE_PKS) &&
+ * guest_cpuid_has(vcpu, X86_FEATURE_PKS)
+ */
+ if (vm_exit_controls_get(vmx) & VM_EXIT_LOAD_IA32_PKRS) {
+ host_pkrs = get_current_pkrs();
+ if (unlikely(host_pkrs != host_state->pkrs)) {
+ vmcs_write64(HOST_IA32_PKRS, host_pkrs);
+ host_state->pkrs = host_pkrs;
+ }
+ }
+
#ifdef CONFIG_X86_64
savesegment(ds, host_state->ds_sel);
savesegment(es, host_state->es_sel);
@@ -1912,6 +1929,13 @@ static int vmx_get_msr(struct kvm_vcpu *vcpu, struct msr_data *msr_info)
case MSR_IA32_DEBUGCTLMSR:
msr_info->data = vmcs_read64(GUEST_IA32_DEBUGCTL);
break;
+ case MSR_IA32_PKRS:
+ if (!kvm_cpu_cap_has(X86_FEATURE_PKS) ||
+ (!msr_info->host_initiated &&
+ !guest_cpuid_has(vcpu, X86_FEATURE_PKS)))
+ return 1;
+ msr_info->data = kvm_read_pkrs(vcpu);
+ break;
default:
find_uret_msr:
msr = vmx_find_uret_msr(vmx, msr_info->index);
@@ -2253,7 +2277,17 @@ static int vmx_set_msr(struct kvm_vcpu *vcpu, struct msr_data *msr_info)
}
ret = kvm_set_msr_common(vcpu, msr_info);
break;
-
+ case MSR_IA32_PKRS:
+ if (!kvm_pkrs_valid(data))
+ return 1;
+ if (!kvm_cpu_cap_has(X86_FEATURE_PKS) ||
+ (!msr_info->host_initiated &&
+ !guest_cpuid_has(vcpu, X86_FEATURE_PKS)))
+ return 1;
+ vcpu->arch.pkrs = data;
+ kvm_register_mark_available(vcpu, VCPU_EXREG_PKRS);
+ vmcs_write64(GUEST_IA32_PKRS, data);
+ break;
default:
find_uret_msr:
msr = vmx_find_uret_msr(vmx, msr_index);
@@ -2544,7 +2578,8 @@ static __init int setup_vmcs_config(struct vmcs_config *vmcs_conf,
VM_EXIT_LOAD_IA32_EFER |
VM_EXIT_CLEAR_BNDCFGS |
VM_EXIT_PT_CONCEAL_PIP |
- VM_EXIT_CLEAR_IA32_RTIT_CTL;
+ VM_EXIT_CLEAR_IA32_RTIT_CTL |
+ VM_EXIT_LOAD_IA32_PKRS;
if (adjust_vmx_controls(min, opt, MSR_IA32_VMX_EXIT_CTLS,
&_vmexit_control) < 0)
return -EIO;
@@ -2568,7 +2603,8 @@ static __init int setup_vmcs_config(struct vmcs_config *vmcs_conf,
VM_ENTRY_LOAD_IA32_EFER |
VM_ENTRY_LOAD_BNDCFGS |
VM_ENTRY_PT_CONCEAL_PIP |
- VM_ENTRY_LOAD_IA32_RTIT_CTL;
+ VM_ENTRY_LOAD_IA32_RTIT_CTL |
+ VM_ENTRY_LOAD_IA32_PKRS;
if (adjust_vmx_controls(min, opt, MSR_IA32_VMX_ENTRY_CTLS,
&_vmentry_control) < 0)
return -EIO;
@@ -4162,7 +4198,8 @@ static u32 vmx_vmentry_ctrl(void)
VM_ENTRY_LOAD_IA32_RTIT_CTL);
/* Loading of EFER and PERF_GLOBAL_CTRL are toggled dynamically */
return vmentry_ctrl &
- ~(VM_ENTRY_LOAD_IA32_PERF_GLOBAL_CTRL | VM_ENTRY_LOAD_IA32_EFER);
+ ~(VM_ENTRY_LOAD_IA32_PERF_GLOBAL_CTRL | VM_ENTRY_LOAD_IA32_EFER |
+ VM_ENTRY_LOAD_IA32_PKRS);
}
static u32 vmx_vmexit_ctrl(void)
@@ -4174,7 +4211,8 @@ static u32 vmx_vmexit_ctrl(void)
VM_EXIT_CLEAR_IA32_RTIT_CTL);
/* Loading of EFER and PERF_GLOBAL_CTRL are toggled dynamically */
return vmexit_ctrl &
- ~(VM_EXIT_LOAD_IA32_PERF_GLOBAL_CTRL | VM_EXIT_LOAD_IA32_EFER);
+ ~(VM_EXIT_LOAD_IA32_PERF_GLOBAL_CTRL | VM_EXIT_LOAD_IA32_EFER |
+ VM_EXIT_LOAD_IA32_PKRS);
}
static void vmx_refresh_apicv_exec_ctrl(struct kvm_vcpu *vcpu)
@@ -5887,6 +5925,8 @@ void dump_vmcs(struct kvm_vcpu *vcpu)
vmcs_read64(GUEST_IA32_PERF_GLOBAL_CTRL));
if (vmentry_ctl & VM_ENTRY_LOAD_BNDCFGS)
pr_err("BndCfgS = 0x%016llx\n", vmcs_read64(GUEST_BNDCFGS));
+ if (vmentry_ctl & VM_ENTRY_LOAD_IA32_PKRS)
+ pr_err("PKRS = 0x%016llx\n", vmcs_read64(GUEST_IA32_PKRS));
pr_err("Interruptibility = %08x ActivityState = %08x\n",
vmcs_read32(GUEST_INTERRUPTIBILITY_INFO),
vmcs_read32(GUEST_ACTIVITY_STATE));
@@ -5928,6 +5968,8 @@ void dump_vmcs(struct kvm_vcpu *vcpu)
vmcs_read64(HOST_IA32_PERF_GLOBAL_CTRL));
if (vmcs_read32(VM_EXIT_MSR_LOAD_COUNT) > 0)
vmx_dump_msrs("host autoload", &vmx->msr_autoload.host);
+ if (vmexit_ctl & VM_EXIT_LOAD_IA32_PKRS)
+ pr_err("PKRS = 0x%016llx\n", vmcs_read64(HOST_IA32_PKRS));
pr_err("*** Control State ***\n");
pr_err("PinBased=%08x CPUBased=%08x SecondaryExec=%08x\n",
@@ -7357,6 +7399,20 @@ static void vmx_vcpu_after_set_cpuid(struct kvm_vcpu *vcpu)
/* Refresh #PF interception to account for MAXPHYADDR changes. */
vmx_update_exception_bitmap(vcpu);
+
+ if (kvm_cpu_cap_has(X86_FEATURE_PKS)) {
+ if (guest_cpuid_has(vcpu, X86_FEATURE_PKS)) {
+ vmx_disable_intercept_for_msr(vcpu, MSR_IA32_PKRS, MSR_TYPE_RW);
+
+ vm_entry_controls_setbit(vmx, VM_ENTRY_LOAD_IA32_PKRS);
+ vm_exit_controls_setbit(vmx, VM_EXIT_LOAD_IA32_PKRS);
+ } else {
+ vmx_enable_intercept_for_msr(vcpu, MSR_IA32_PKRS, MSR_TYPE_RW);
+
+ vm_entry_controls_clearbit(vmx, VM_ENTRY_LOAD_IA32_PKRS);
+ vm_exit_controls_clearbit(vmx, VM_EXIT_LOAD_IA32_PKRS);
+ }
+ }
}
static __init void vmx_set_cpu_caps(void)
@@ -352,7 +352,7 @@ struct vcpu_vmx {
struct lbr_desc lbr_desc;
/* Save desired MSR intercept (read: pass-through) state */
-#define MAX_POSSIBLE_PASSTHROUGH_MSRS 15
+#define MAX_POSSIBLE_PASSTHROUGH_MSRS 16
struct {
DECLARE_BITMAP(read, MAX_POSSIBLE_PASSTHROUGH_MSRS);
DECLARE_BITMAP(write, MAX_POSSIBLE_PASSTHROUGH_MSRS);
@@ -1356,7 +1356,7 @@ static const u32 msrs_to_save_all[] = {
MSR_IA32_RTIT_ADDR1_A, MSR_IA32_RTIT_ADDR1_B,
MSR_IA32_RTIT_ADDR2_A, MSR_IA32_RTIT_ADDR2_B,
MSR_IA32_RTIT_ADDR3_A, MSR_IA32_RTIT_ADDR3_B,
- MSR_IA32_UMWAIT_CONTROL,
+ MSR_IA32_UMWAIT_CONTROL, MSR_IA32_PKRS,
MSR_ARCH_PERFMON_FIXED_CTR0, MSR_ARCH_PERFMON_FIXED_CTR1,
MSR_ARCH_PERFMON_FIXED_CTR0 + 2,
@@ -6506,6 +6506,10 @@ static void kvm_init_msr_list(void)
intel_pt_validate_hw_cap(PT_CAP_num_address_ranges) * 2)
continue;
break;
+ case MSR_IA32_PKRS:
+ if (!kvm_cpu_cap_has(X86_FEATURE_PKS))
+ continue;
+ break;
case MSR_ARCH_PERFMON_PERFCTR0 ... MSR_ARCH_PERFMON_PERFCTR0 + 17:
if (msrs_to_save_all[i] - MSR_ARCH_PERFMON_PERFCTR0 >=
min(INTEL_PMC_MAX_GENERIC, x86_pmu.num_counters_gp))
@@ -11233,6 +11237,10 @@ void kvm_vcpu_reset(struct kvm_vcpu *vcpu, bool init_event)
kvm_set_rflags(vcpu, X86_EFLAGS_FIXED);
kvm_rip_write(vcpu, 0xfff0);
+ /* PKRS is preserved on INIT */
+ if (!init_event && kvm_cpu_cap_has(X86_FEATURE_PKS))
+ __kvm_set_msr(vcpu, MSR_IA32_PKRS, 0, true);
+
vcpu->arch.cr3 = 0;
kvm_register_mark_dirty(vcpu, VCPU_EXREG_CR3);
@@ -452,6 +452,12 @@ static inline void kvm_machine_check(void)
#endif
}
+static inline bool kvm_pkrs_valid(u64 data)
+{
+ /* bit[63,32] must be zero */
+ return !(data >> 32);
+}
+
void kvm_load_guest_xsave_state(struct kvm_vcpu *vcpu);
void kvm_load_host_xsave_state(struct kvm_vcpu *vcpu);
int kvm_spec_ctrl_test_value(u64 value);
@@ -490,4 +490,10 @@ void pks_update_exception(struct pt_regs *regs, int pkey, u32 protection)
}
EXPORT_SYMBOL_GPL(pks_update_exception);
+u32 get_current_pkrs(void)
+{
+ return this_cpu_read(pkrs_cache);
+}
+EXPORT_SYMBOL_GPL(get_current_pkrs);
+
#endif /* CONFIG_ARCH_ENABLE_SUPERVISOR_PKEYS */
@@ -98,6 +98,8 @@ static inline void pks_mk_readwrite(int pkey)
typedef bool (*pks_key_callback)(struct pt_regs *regs, unsigned long address,
bool write);
+u32 get_current_pkrs(void);
+
#else /* !CONFIG_ARCH_ENABLE_SUPERVISOR_PKEYS */
static inline bool pks_available(void)
@@ -112,6 +114,10 @@ static inline void pks_update_exception(struct pt_regs *regs,
int pkey,
u32 protection)
{ }
+static inline u32 get_current_pkrs(void)
+{
+ return 0;
+}
#endif /* CONFIG_ARCH_ENABLE_SUPERVISOR_PKEYS */
Protection Key for Superviosr Pages (PKS) uses IA32_PKRS MSR (PKRS) at index 0x6E1 to allow software to manage superviosr key rights, i.e. it can enforce additional permissions checks besides normal paging protections via a MSR update without TLB flushes when permissions change. For performance consideration, PKRS intercept in KVM will be disabled when PKS is supported in guest so that PKRS can be accessed without VM exit. PKS introduces dedicated control fields in VMCS to switch PKRS, which only does the retore part. In addition, every VM exit saves PKRS into the guest-state area in VMCS, while VM enter won't save the host value due to the expectation that the host won't change the MSR often. Update the host's value in VMCS manually if the MSR has been changed by the kernel since the last time the VMCS was run. Introduce a function get_current_pkrs() in arch/x86/mm/pkeys.c to export the per-cpu variable pkrs_cache to avoid frequent rdmsr of PKRS. Signed-off-by: Chenyi Qiang <chenyi.qiang@intel.com> --- arch/x86/kvm/vmx/vmcs.h | 1 + arch/x86/kvm/vmx/vmx.c | 66 +++++++++++++++++++++++++++++++++++++---- arch/x86/kvm/vmx/vmx.h | 2 +- arch/x86/kvm/x86.c | 10 ++++++- arch/x86/kvm/x86.h | 6 ++++ arch/x86/mm/pkeys.c | 6 ++++ include/linux/pkeys.h | 6 ++++ 7 files changed, 90 insertions(+), 7 deletions(-)