| Message ID | 20230215174046.2201432-2-ricarkol@google.com (mailing list archive) |
|---|---|
| State | New, archived |
| Headers | show |
| Series | Implement Eager Page Splitting for ARM | expand |
Hi Ricardo, On 2/16/23 01:40, Ricardo Koller wrote: > Add two flags to kvm_pgtable_visit_ctx, KVM_PGTABLE_WALK_SKIP_BBM and > KVM_PGTABLE_WALK_SKIP_CMO, to indicate that the walk should not > perform break-before-make (BBM) nor cache maintenance operations > (CMO). This will by a future commit to create unlinked tables not > accessible to the HW page-table walker. This is safe as these removed > tables are not visible to the HW page-table walker. > > Signed-off-by: Ricardo Koller <ricarkol@google.com> > --- > arch/arm64/include/asm/kvm_pgtable.h | 18 ++++++++++++++++++ > arch/arm64/kvm/hyp/pgtable.c | 27 ++++++++++++++++----------- > 2 files changed, 34 insertions(+), 11 deletions(-) > > diff --git a/arch/arm64/include/asm/kvm_pgtable.h b/arch/arm64/include/asm/kvm_pgtable.h > index 63f81b27a4e3..3339192a97a9 100644 > --- a/arch/arm64/include/asm/kvm_pgtable.h > +++ b/arch/arm64/include/asm/kvm_pgtable.h > @@ -188,12 +188,20 @@ typedef bool (*kvm_pgtable_force_pte_cb_t)(u64 addr, u64 end, > * children. > * @KVM_PGTABLE_WALK_SHARED: Indicates the page-tables may be shared > * with other software walkers. > + * @KVM_PGTABLE_WALK_SKIP_BBM: Visit and update table entries > + * without Break-before-make > + * requirements. > + * @KVM_PGTABLE_WALK_SKIP_CMO: Visit and update table entries > + * without Cache maintenance > + * operations required. > */ > enum kvm_pgtable_walk_flags { > KVM_PGTABLE_WALK_LEAF = BIT(0), > KVM_PGTABLE_WALK_TABLE_PRE = BIT(1), > KVM_PGTABLE_WALK_TABLE_POST = BIT(2), > KVM_PGTABLE_WALK_SHARED = BIT(3), > + KVM_PGTABLE_WALK_SKIP_BBM = BIT(4), > + KVM_PGTABLE_WALK_SKIP_CMO = BIT(4), The KVM_PGTABLE_WALK_SKIP_BBM and KVM_PGTABLE_WALK_SKIP_CMO use the same BIT(4), if I understand correctly, the two flags are used in different operation and will never be used at the same time. Maybe add some comments to illustrate why the two use the same bit can be better. > }; > > struct kvm_pgtable_visit_ctx { > @@ -215,6 +223,16 @@ static inline bool kvm_pgtable_walk_shared(const struct kvm_pgtable_visit_ctx *c > return ctx->flags & KVM_PGTABLE_WALK_SHARED; > } > > +static inline bool kvm_pgtable_walk_skip_bbm(const struct kvm_pgtable_visit_ctx *ctx) > +{ > + return ctx->flags & KVM_PGTABLE_WALK_SKIP_BBM; > +} > + > +static inline bool kvm_pgtable_walk_skip_cmo(const struct kvm_pgtable_visit_ctx *ctx) > +{ > + return ctx->flags & KVM_PGTABLE_WALK_SKIP_CMO; > +} > + > /** > * struct kvm_pgtable_walker - Hook into a page-table walk. > * @cb: Callback function to invoke during the walk. > diff --git a/arch/arm64/kvm/hyp/pgtable.c b/arch/arm64/kvm/hyp/pgtable.c > index b11cf2c618a6..e093e222daf3 100644 > --- a/arch/arm64/kvm/hyp/pgtable.c > +++ b/arch/arm64/kvm/hyp/pgtable.c > @@ -717,14 +717,17 @@ static bool stage2_try_break_pte(const struct kvm_pgtable_visit_ctx *ctx, > if (!stage2_try_set_pte(ctx, KVM_INVALID_PTE_LOCKED)) > return false; > > - /* > - * Perform the appropriate TLB invalidation based on the evicted pte > - * value (if any). > - */ > - if (kvm_pte_table(ctx->old, ctx->level)) > - kvm_call_hyp(__kvm_tlb_flush_vmid, mmu); > - else if (kvm_pte_valid(ctx->old)) > - kvm_call_hyp(__kvm_tlb_flush_vmid_ipa, mmu, ctx->addr, ctx->level); > + if (!kvm_pgtable_walk_skip_bbm(ctx)) { > + /* > + * Perform the appropriate TLB invalidation based on the > + * evicted pte value (if any). > + */ > + if (kvm_pte_table(ctx->old, ctx->level)) > + kvm_call_hyp(__kvm_tlb_flush_vmid, mmu); > + else if (kvm_pte_valid(ctx->old)) > + kvm_call_hyp(__kvm_tlb_flush_vmid_ipa, mmu, > + ctx->addr, ctx->level); > + } > > if (stage2_pte_is_counted(ctx->old)) > mm_ops->put_page(ctx->ptep); > @@ -808,11 +811,13 @@ static int stage2_map_walker_try_leaf(const struct kvm_pgtable_visit_ctx *ctx, > return -EAGAIN; > > /* Perform CMOs before installation of the guest stage-2 PTE */ > - if (mm_ops->dcache_clean_inval_poc && stage2_pte_cacheable(pgt, new)) > + if (!kvm_pgtable_walk_skip_cmo(ctx) && mm_ops->dcache_clean_inval_poc && > + stage2_pte_cacheable(pgt, new)) > mm_ops->dcache_clean_inval_poc(kvm_pte_follow(new, mm_ops), > - granule); > + granule); > > - if (mm_ops->icache_inval_pou && stage2_pte_executable(new)) > + if (!kvm_pgtable_walk_skip_cmo(ctx) && mm_ops->icache_inval_pou && > + stage2_pte_executable(new)) > mm_ops->icache_inval_pou(kvm_pte_follow(new, mm_ops), granule); > > stage2_make_pte(ctx, new);
On Wed, Feb 15, 2023 at 6:56 PM Shaoqin Huang <shahuang@redhat.com> wrote: > > Hi Ricardo, > > On 2/16/23 01:40, Ricardo Koller wrote: > > Add two flags to kvm_pgtable_visit_ctx, KVM_PGTABLE_WALK_SKIP_BBM and > > KVM_PGTABLE_WALK_SKIP_CMO, to indicate that the walk should not > > perform break-before-make (BBM) nor cache maintenance operations > > (CMO). This will by a future commit to create unlinked tables not > > accessible to the HW page-table walker. This is safe as these removed > > tables are not visible to the HW page-table walker. > > > > Signed-off-by: Ricardo Koller <ricarkol@google.com> > > --- > > arch/arm64/include/asm/kvm_pgtable.h | 18 ++++++++++++++++++ > > arch/arm64/kvm/hyp/pgtable.c | 27 ++++++++++++++++----------- > > 2 files changed, 34 insertions(+), 11 deletions(-) > > > > diff --git a/arch/arm64/include/asm/kvm_pgtable.h b/arch/arm64/include/asm/kvm_pgtable.h > > index 63f81b27a4e3..3339192a97a9 100644 > > --- a/arch/arm64/include/asm/kvm_pgtable.h > > +++ b/arch/arm64/include/asm/kvm_pgtable.h > > @@ -188,12 +188,20 @@ typedef bool (*kvm_pgtable_force_pte_cb_t)(u64 addr, u64 end, > > * children. > > * @KVM_PGTABLE_WALK_SHARED: Indicates the page-tables may be shared > > * with other software walkers. > > + * @KVM_PGTABLE_WALK_SKIP_BBM: Visit and update table entries > > + * without Break-before-make > > + * requirements. > > + * @KVM_PGTABLE_WALK_SKIP_CMO: Visit and update table entries > > + * without Cache maintenance > > + * operations required. > > */ > > enum kvm_pgtable_walk_flags { > > KVM_PGTABLE_WALK_LEAF = BIT(0), > > KVM_PGTABLE_WALK_TABLE_PRE = BIT(1), > > KVM_PGTABLE_WALK_TABLE_POST = BIT(2), > > KVM_PGTABLE_WALK_SHARED = BIT(3), > > + KVM_PGTABLE_WALK_SKIP_BBM = BIT(4), > > + KVM_PGTABLE_WALK_SKIP_CMO = BIT(4), > > The KVM_PGTABLE_WALK_SKIP_BBM and KVM_PGTABLE_WALK_SKIP_CMO use the same > BIT(4), if I understand correctly, the two flags are used in different > operation and will never be used at the same time. > That was supposed to be a BIT(5): KVM_PGTABLE_WALK_SKIP_BBM = BIT(4), KVM_PGTABLE_WALK_SKIP_CMO = BIT(5), > > Maybe add some comments to illustrate why the two use the same bit can > be better. > > > > }; > > > > struct kvm_pgtable_visit_ctx { > > @@ -215,6 +223,16 @@ static inline bool kvm_pgtable_walk_shared(const struct kvm_pgtable_visit_ctx *c > > return ctx->flags & KVM_PGTABLE_WALK_SHARED; > > } > > > > +static inline bool kvm_pgtable_walk_skip_bbm(const struct kvm_pgtable_visit_ctx *ctx) > > +{ > > + return ctx->flags & KVM_PGTABLE_WALK_SKIP_BBM; > > +} > > + > > +static inline bool kvm_pgtable_walk_skip_cmo(const struct kvm_pgtable_visit_ctx *ctx) > > +{ > > + return ctx->flags & KVM_PGTABLE_WALK_SKIP_CMO; > > +} > > + > > /** > > * struct kvm_pgtable_walker - Hook into a page-table walk. > > * @cb: Callback function to invoke during the walk. > > diff --git a/arch/arm64/kvm/hyp/pgtable.c b/arch/arm64/kvm/hyp/pgtable.c > > index b11cf2c618a6..e093e222daf3 100644 > > --- a/arch/arm64/kvm/hyp/pgtable.c > > +++ b/arch/arm64/kvm/hyp/pgtable.c > > @@ -717,14 +717,17 @@ static bool stage2_try_break_pte(const struct kvm_pgtable_visit_ctx *ctx, > > if (!stage2_try_set_pte(ctx, KVM_INVALID_PTE_LOCKED)) > > return false; > > > > - /* > > - * Perform the appropriate TLB invalidation based on the evicted pte > > - * value (if any). > > - */ > > - if (kvm_pte_table(ctx->old, ctx->level)) > > - kvm_call_hyp(__kvm_tlb_flush_vmid, mmu); > > - else if (kvm_pte_valid(ctx->old)) > > - kvm_call_hyp(__kvm_tlb_flush_vmid_ipa, mmu, ctx->addr, ctx->level); > > + if (!kvm_pgtable_walk_skip_bbm(ctx)) { > > + /* > > + * Perform the appropriate TLB invalidation based on the > > + * evicted pte value (if any). > > + */ > > + if (kvm_pte_table(ctx->old, ctx->level)) > > + kvm_call_hyp(__kvm_tlb_flush_vmid, mmu); > > + else if (kvm_pte_valid(ctx->old)) > > + kvm_call_hyp(__kvm_tlb_flush_vmid_ipa, mmu, > > + ctx->addr, ctx->level); > > + } > > > > if (stage2_pte_is_counted(ctx->old)) > > mm_ops->put_page(ctx->ptep); > > @@ -808,11 +811,13 @@ static int stage2_map_walker_try_leaf(const struct kvm_pgtable_visit_ctx *ctx, > > return -EAGAIN; > > > > /* Perform CMOs before installation of the guest stage-2 PTE */ > > - if (mm_ops->dcache_clean_inval_poc && stage2_pte_cacheable(pgt, new)) > > + if (!kvm_pgtable_walk_skip_cmo(ctx) && mm_ops->dcache_clean_inval_poc && > > + stage2_pte_cacheable(pgt, new)) > > mm_ops->dcache_clean_inval_poc(kvm_pte_follow(new, mm_ops), > > - granule); > > + granule); > > > > - if (mm_ops->icache_inval_pou && stage2_pte_executable(new)) > > + if (!kvm_pgtable_walk_skip_cmo(ctx) && mm_ops->icache_inval_pou && > > + stage2_pte_executable(new)) > > mm_ops->icache_inval_pou(kvm_pte_follow(new, mm_ops), granule); > > > > stage2_make_pte(ctx, new); > > -- > Regards, > Shaoqin >
diff --git a/arch/arm64/include/asm/kvm_pgtable.h b/arch/arm64/include/asm/kvm_pgtable.h index 63f81b27a4e3..3339192a97a9 100644 --- a/arch/arm64/include/asm/kvm_pgtable.h +++ b/arch/arm64/include/asm/kvm_pgtable.h @@ -188,12 +188,20 @@ typedef bool (*kvm_pgtable_force_pte_cb_t)(u64 addr, u64 end, * children. * @KVM_PGTABLE_WALK_SHARED: Indicates the page-tables may be shared * with other software walkers. + * @KVM_PGTABLE_WALK_SKIP_BBM: Visit and update table entries + * without Break-before-make + * requirements. + * @KVM_PGTABLE_WALK_SKIP_CMO: Visit and update table entries + * without Cache maintenance + * operations required. */ enum kvm_pgtable_walk_flags { KVM_PGTABLE_WALK_LEAF = BIT(0), KVM_PGTABLE_WALK_TABLE_PRE = BIT(1), KVM_PGTABLE_WALK_TABLE_POST = BIT(2), KVM_PGTABLE_WALK_SHARED = BIT(3), + KVM_PGTABLE_WALK_SKIP_BBM = BIT(4), + KVM_PGTABLE_WALK_SKIP_CMO = BIT(4), }; struct kvm_pgtable_visit_ctx { @@ -215,6 +223,16 @@ static inline bool kvm_pgtable_walk_shared(const struct kvm_pgtable_visit_ctx *c return ctx->flags & KVM_PGTABLE_WALK_SHARED; } +static inline bool kvm_pgtable_walk_skip_bbm(const struct kvm_pgtable_visit_ctx *ctx) +{ + return ctx->flags & KVM_PGTABLE_WALK_SKIP_BBM; +} + +static inline bool kvm_pgtable_walk_skip_cmo(const struct kvm_pgtable_visit_ctx *ctx) +{ + return ctx->flags & KVM_PGTABLE_WALK_SKIP_CMO; +} + /** * struct kvm_pgtable_walker - Hook into a page-table walk. * @cb: Callback function to invoke during the walk. diff --git a/arch/arm64/kvm/hyp/pgtable.c b/arch/arm64/kvm/hyp/pgtable.c index b11cf2c618a6..e093e222daf3 100644 --- a/arch/arm64/kvm/hyp/pgtable.c +++ b/arch/arm64/kvm/hyp/pgtable.c @@ -717,14 +717,17 @@ static bool stage2_try_break_pte(const struct kvm_pgtable_visit_ctx *ctx, if (!stage2_try_set_pte(ctx, KVM_INVALID_PTE_LOCKED)) return false; - /* - * Perform the appropriate TLB invalidation based on the evicted pte - * value (if any). - */ - if (kvm_pte_table(ctx->old, ctx->level)) - kvm_call_hyp(__kvm_tlb_flush_vmid, mmu); - else if (kvm_pte_valid(ctx->old)) - kvm_call_hyp(__kvm_tlb_flush_vmid_ipa, mmu, ctx->addr, ctx->level); + if (!kvm_pgtable_walk_skip_bbm(ctx)) { + /* + * Perform the appropriate TLB invalidation based on the + * evicted pte value (if any). + */ + if (kvm_pte_table(ctx->old, ctx->level)) + kvm_call_hyp(__kvm_tlb_flush_vmid, mmu); + else if (kvm_pte_valid(ctx->old)) + kvm_call_hyp(__kvm_tlb_flush_vmid_ipa, mmu, + ctx->addr, ctx->level); + } if (stage2_pte_is_counted(ctx->old)) mm_ops->put_page(ctx->ptep); @@ -808,11 +811,13 @@ static int stage2_map_walker_try_leaf(const struct kvm_pgtable_visit_ctx *ctx, return -EAGAIN; /* Perform CMOs before installation of the guest stage-2 PTE */ - if (mm_ops->dcache_clean_inval_poc && stage2_pte_cacheable(pgt, new)) + if (!kvm_pgtable_walk_skip_cmo(ctx) && mm_ops->dcache_clean_inval_poc && + stage2_pte_cacheable(pgt, new)) mm_ops->dcache_clean_inval_poc(kvm_pte_follow(new, mm_ops), - granule); + granule); - if (mm_ops->icache_inval_pou && stage2_pte_executable(new)) + if (!kvm_pgtable_walk_skip_cmo(ctx) && mm_ops->icache_inval_pou && + stage2_pte_executable(new)) mm_ops->icache_inval_pou(kvm_pte_follow(new, mm_ops), granule); stage2_make_pte(ctx, new);
Add two flags to kvm_pgtable_visit_ctx, KVM_PGTABLE_WALK_SKIP_BBM and KVM_PGTABLE_WALK_SKIP_CMO, to indicate that the walk should not perform break-before-make (BBM) nor cache maintenance operations (CMO). This will by a future commit to create unlinked tables not accessible to the HW page-table walker. This is safe as these removed tables are not visible to the HW page-table walker. Signed-off-by: Ricardo Koller <ricarkol@google.com> --- arch/arm64/include/asm/kvm_pgtable.h | 18 ++++++++++++++++++ arch/arm64/kvm/hyp/pgtable.c | 27 ++++++++++++++++----------- 2 files changed, 34 insertions(+), 11 deletions(-)