@@ -3132,12 +3132,12 @@ static int host_pfn_mapping_level(struct kvm *kvm, gfn_t gfn,
return level;
}
-int kvm_mmu_max_mapping_level(struct kvm *kvm,
- const struct kvm_memory_slot *slot, gfn_t gfn,
- int max_level, bool is_private)
+static int __kvm_mmu_max_mapping_level(struct kvm *kvm,
+ const struct kvm_memory_slot *slot,
+ gfn_t gfn, int max_level, int host_level,
+ bool faultin_private)
{
struct kvm_lpage_info *linfo;
- int host_level;
max_level = min(max_level, max_huge_page_level);
for ( ; max_level > PG_LEVEL_4K; max_level--) {
@@ -3146,16 +3146,24 @@ int kvm_mmu_max_mapping_level(struct kvm *kvm,
break;
}
- if (is_private)
- return max_level;
-
if (max_level == PG_LEVEL_4K)
return PG_LEVEL_4K;
- host_level = host_pfn_mapping_level(kvm, gfn, slot);
+ if (!faultin_private) {
+ WARN_ON_ONCE(host_level != PG_LEVEL_NONE);
+ host_level = host_pfn_mapping_level(kvm, gfn, slot);
+ }
+ WARN_ON_ONCE(host_level == PG_LEVEL_NONE);
return min(host_level, max_level);
}
+int kvm_mmu_max_mapping_level(struct kvm *kvm,
+ const struct kvm_memory_slot *slot, gfn_t gfn,
+ int max_level, bool faultin_private)
+{
+ return __kvm_mmu_max_mapping_level(kvm, slot, gfn, max_level, PG_LEVEL_NONE, faultin_private);
+}
+
void kvm_mmu_hugepage_adjust(struct kvm_vcpu *vcpu, struct kvm_page_fault *fault)
{
struct kvm_memory_slot *slot = fault->slot;
@@ -3176,9 +3184,10 @@ void kvm_mmu_hugepage_adjust(struct kvm_vcpu *vcpu, struct kvm_page_fault *fault
* Enforce the iTLB multihit workaround after capturing the requested
* level, which will be used to do precise, accurate accounting.
*/
- fault->req_level = kvm_mmu_max_mapping_level(vcpu->kvm, slot,
- fault->gfn, fault->max_level,
- fault->is_private);
+ fault->req_level = __kvm_mmu_max_mapping_level(vcpu->kvm, slot,
+ fault->gfn, fault->max_level,
+ fault->host_level,
+ kvm_is_faultin_private(fault));
if (fault->req_level == PG_LEVEL_4K || fault->huge_page_disallowed)
return;
@@ -4311,7 +4320,8 @@ static int kvm_faultin_pfn_private(struct kvm_vcpu *vcpu,
if (kvm_restricted_mem_get_pfn(slot, fault->gfn, &fault->pfn, &order))
return RET_PF_RETRY;
- fault->max_level = min(order_to_level(order), fault->max_level);
+ fault->host_level = order_to_level(order);
+ fault->max_level = min((u8)fault->host_level, fault->max_level);
fault->map_writable = !(slot->flags & KVM_MEM_READONLY);
return RET_PF_CONTINUE;
}
@@ -4355,7 +4365,7 @@ static int __kvm_faultin_pfn(struct kvm_vcpu *vcpu, struct kvm_page_fault *fault
if (fault->is_private != kvm_mem_is_private(vcpu->kvm, fault->gfn))
return kvm_do_memory_fault_exit(vcpu, fault);
- if (fault->is_private && kvm_slot_can_be_private(slot))
+ if (kvm_is_faultin_private(fault))
return kvm_faultin_pfn_private(vcpu, fault);
async = false;
@@ -333,6 +333,7 @@ struct kvm_page_fault {
kvm_pfn_t pfn;
hva_t hva;
bool map_writable;
+ enum pg_level host_level; /* valid only for private memslot && private gfn */
};
int kvm_tdp_page_fault(struct kvm_vcpu *vcpu, struct kvm_page_fault *fault);
@@ -446,4 +447,11 @@ static inline int kvm_restricted_mem_get_pfn(struct kvm_memory_slot *slot,
}
#endif /* CONFIG_HAVE_KVM_RESTRICTED_MEM */
+static inline bool kvm_is_faultin_private(const struct kvm_page_fault *fault)
+{
+ if (IS_ENABLED(CONFIG_HAVE_KVM_RESTRICTED_MEM))
+ return fault->is_private && kvm_slot_can_be_private(fault->slot);
+ return false;
+}
+
#endif /* __KVM_X86_MMU_INTERNAL_H */