| Message ID | 20250223194943.3518952-6-riel@surriel.com (mailing list archive) |
|---|---|
| State | New |
| Headers | show |
| Series | AMD broadcast TLB invalidation | expand |
On Sun, Feb 23, 2025 at 02:48:55PM -0500, Rik van Riel wrote: > The flush_tlb_all() function is not used a whole lot, but we might > as well use broadcast TLB flushing there, too. > > Signed-off-by: Rik van Riel <riel@surriel.com> > Tested-by: Manali Shukla <Manali.Shukla@amd.com> > Tested-by: Brendan Jackman <jackmanb@google.com> > Tested-by: Michael Kelley <mhklinux@outlook.com> > --- > arch/x86/mm/tlb.c | 16 ++++++++++++++++ > 1 file changed, 16 insertions(+) > > diff --git a/arch/x86/mm/tlb.c b/arch/x86/mm/tlb.c > index 59396a3c6e9c..2d7ed0fda61f 100644 > --- a/arch/x86/mm/tlb.c > +++ b/arch/x86/mm/tlb.c > @@ -1065,6 +1065,16 @@ void flush_tlb_mm_range(struct mm_struct *mm, unsigned long start, > } > > > +static bool broadcast_flush_tlb_all(void) broadcast? All those INVLPGB functions have a "invlpgb_" prefix... In any case, get rid of that function: diff --git a/arch/x86/mm/tlb.c b/arch/x86/mm/tlb.c index 2d7ed0fda61f..feaca53b7685 100644 --- a/arch/x86/mm/tlb.c +++ b/arch/x86/mm/tlb.c @@ -1065,16 +1065,6 @@ void flush_tlb_mm_range(struct mm_struct *mm, unsigned long start, } -static bool broadcast_flush_tlb_all(void) -{ - if (!cpu_feature_enabled(X86_FEATURE_INVLPGB)) - return false; - - guard(preempt)(); - invlpgb_flush_all(); - return true; -} - static void do_flush_tlb_all(void *info) { count_vm_tlb_event(NR_TLB_REMOTE_FLUSH_RECEIVED); @@ -1085,9 +1075,11 @@ void flush_tlb_all(void) { count_vm_tlb_event(NR_TLB_REMOTE_FLUSH); - /* First try (faster) hardware-assisted TLB invalidation. */ - if (broadcast_flush_tlb_all()) + if (cpu_feature_enabled(X86_FEATURE_INVLPGB)) { + guard(preempt)(); + invlpgb_flush_all(); return; + } /* Fall back to the IPI-based invalidation. */ on_each_cpu(do_flush_tlb_all, NULL, 1);
diff --git a/arch/x86/mm/tlb.c b/arch/x86/mm/tlb.c index 59396a3c6e9c..2d7ed0fda61f 100644 --- a/arch/x86/mm/tlb.c +++ b/arch/x86/mm/tlb.c @@ -1065,6 +1065,16 @@ void flush_tlb_mm_range(struct mm_struct *mm, unsigned long start, } +static bool broadcast_flush_tlb_all(void) +{ + if (!cpu_feature_enabled(X86_FEATURE_INVLPGB)) + return false; + + guard(preempt)(); + invlpgb_flush_all(); + return true; +} + static void do_flush_tlb_all(void *info) { count_vm_tlb_event(NR_TLB_REMOTE_FLUSH_RECEIVED); @@ -1074,6 +1084,12 @@ static void do_flush_tlb_all(void *info) void flush_tlb_all(void) { count_vm_tlb_event(NR_TLB_REMOTE_FLUSH); + + /* First try (faster) hardware-assisted TLB invalidation. */ + if (broadcast_flush_tlb_all()) + return; + + /* Fall back to the IPI-based invalidation. */ on_each_cpu(do_flush_tlb_all, NULL, 1); }