| Message ID | 20240202124711.256146-1-alexghiti@rivosinc.com (mailing list archive) |
|---|---|
| State | Accepted |
| Commit | 01261e24cfab69c65043e1e61168348ae23a64c2 |
| Headers | show |
| Series | riscv: Only flush the mm icache when setting an exec pte | expand |
On Fri, Feb 02, 2024 at 01:47:11PM +0100, Alexandre Ghiti wrote: > We used to emit a flush_icache_all() whenever a dirty executable > mapping is set in the page table but we can instead call > flush_icache_mm() which will only send IPIs to cores that currently run > this mm and add a deferred icache flush to the others. > > The number of calls to sbi_remote_fence_i() (tested without IPI > support): > > With a simple buildroot rootfs: > * Before: ~5k > * After : 4 (!) > > Tested on HW, the boot to login is ~4.5% faster. > > With an ubuntu rootfs: > * Before: ~24k > * After : ~13k > > Signed-off-by: Alexandre Ghiti <alexghiti@rivosinc.com> > --- > arch/riscv/include/asm/pgtable.h | 14 +++++++------- > arch/riscv/mm/cacheflush.c | 4 ++-- > arch/riscv/mm/pgtable.c | 2 +- > 3 files changed, 10 insertions(+), 10 deletions(-) > > diff --git a/arch/riscv/include/asm/pgtable.h b/arch/riscv/include/asm/pgtable.h > index 0c94260b5d0c..058cfa89dd29 100644 > --- a/arch/riscv/include/asm/pgtable.h > +++ b/arch/riscv/include/asm/pgtable.h > @@ -513,12 +513,12 @@ static inline void set_pte(pte_t *ptep, pte_t pteval) > WRITE_ONCE(*ptep, pteval); > } > > -void flush_icache_pte(pte_t pte); > +void flush_icache_pte(struct mm_struct *mm, pte_t pte); > > -static inline void __set_pte_at(pte_t *ptep, pte_t pteval) > +static inline void __set_pte_at(struct mm_struct *mm, pte_t *ptep, pte_t pteval) > { > if (pte_present(pteval) && pte_exec(pteval)) > - flush_icache_pte(pteval); > + flush_icache_pte(mm, pteval); > > set_pte(ptep, pteval); > } > @@ -529,7 +529,7 @@ static inline void set_ptes(struct mm_struct *mm, unsigned long addr, > page_table_check_ptes_set(mm, ptep, pteval, nr); > > for (;;) { > - __set_pte_at(ptep, pteval); > + __set_pte_at(mm, ptep, pteval); > if (--nr == 0) > break; > ptep++; > @@ -541,7 +541,7 @@ static inline void set_ptes(struct mm_struct *mm, unsigned long addr, > static inline void pte_clear(struct mm_struct *mm, > unsigned long addr, pte_t *ptep) > { > - __set_pte_at(ptep, __pte(0)); > + __set_pte_at(mm, ptep, __pte(0)); > } > > #define __HAVE_ARCH_PTEP_SET_ACCESS_FLAGS /* defined in mm/pgtable.c */ > @@ -707,14 +707,14 @@ static inline void set_pmd_at(struct mm_struct *mm, unsigned long addr, > pmd_t *pmdp, pmd_t pmd) > { > page_table_check_pmd_set(mm, pmdp, pmd); > - return __set_pte_at((pte_t *)pmdp, pmd_pte(pmd)); > + return __set_pte_at(mm, (pte_t *)pmdp, pmd_pte(pmd)); > } > > static inline void set_pud_at(struct mm_struct *mm, unsigned long addr, > pud_t *pudp, pud_t pud) > { > page_table_check_pud_set(mm, pudp, pud); > - return __set_pte_at((pte_t *)pudp, pud_pte(pud)); > + return __set_pte_at(mm, (pte_t *)pudp, pud_pte(pud)); > } > > #ifdef CONFIG_PAGE_TABLE_CHECK > diff --git a/arch/riscv/mm/cacheflush.c b/arch/riscv/mm/cacheflush.c > index b0774993c0f3..3e234e95d952 100644 > --- a/arch/riscv/mm/cacheflush.c > +++ b/arch/riscv/mm/cacheflush.c > @@ -85,12 +85,12 @@ void flush_icache_mm(struct mm_struct *mm, bool local) > #endif /* CONFIG_SMP */ > > #ifdef CONFIG_MMU > -void flush_icache_pte(pte_t pte) > +void flush_icache_pte(struct mm_struct *mm, pte_t pte) > { > struct folio *folio = page_folio(pte_page(pte)); > > if (!test_bit(PG_dcache_clean, &folio->flags)) { > - flush_icache_all(); > + flush_icache_mm(mm, false); > set_bit(PG_dcache_clean, &folio->flags); > } > } > diff --git a/arch/riscv/mm/pgtable.c b/arch/riscv/mm/pgtable.c > index ef887efcb679..533ec9055fa0 100644 > --- a/arch/riscv/mm/pgtable.c > +++ b/arch/riscv/mm/pgtable.c > @@ -10,7 +10,7 @@ int ptep_set_access_flags(struct vm_area_struct *vma, > pte_t entry, int dirty) > { > if (!pte_same(ptep_get(ptep), entry)) > - __set_pte_at(ptep, entry); > + __set_pte_at(vma->vm_mm, ptep, entry); > /* > * update_mmu_cache will unconditionally execute, handling both > * the case that the PTE changed and the spurious fault case. > -- > 2.39.2 > > > _______________________________________________ > linux-riscv mailing list > linux-riscv@lists.infradead.org > http://lists.infradead.org/mailman/listinfo/linux-riscv Reviewed-by: Charlie Jenkins <charlie@rivosinc.com>
Hello: This patch was applied to riscv/linux.git (for-next) by Palmer Dabbelt <palmer@rivosinc.com>: On Fri, 2 Feb 2024 13:47:11 +0100 you wrote: > We used to emit a flush_icache_all() whenever a dirty executable > mapping is set in the page table but we can instead call > flush_icache_mm() which will only send IPIs to cores that currently run > this mm and add a deferred icache flush to the others. > > The number of calls to sbi_remote_fence_i() (tested without IPI > support): > > [...] Here is the summary with links: - riscv: Only flush the mm icache when setting an exec pte https://git.kernel.org/riscv/c/01261e24cfab You are awesome, thank you!
diff --git a/arch/riscv/include/asm/pgtable.h b/arch/riscv/include/asm/pgtable.h index 0c94260b5d0c..058cfa89dd29 100644 --- a/arch/riscv/include/asm/pgtable.h +++ b/arch/riscv/include/asm/pgtable.h @@ -513,12 +513,12 @@ static inline void set_pte(pte_t *ptep, pte_t pteval) WRITE_ONCE(*ptep, pteval); } -void flush_icache_pte(pte_t pte); +void flush_icache_pte(struct mm_struct *mm, pte_t pte); -static inline void __set_pte_at(pte_t *ptep, pte_t pteval) +static inline void __set_pte_at(struct mm_struct *mm, pte_t *ptep, pte_t pteval) { if (pte_present(pteval) && pte_exec(pteval)) - flush_icache_pte(pteval); + flush_icache_pte(mm, pteval); set_pte(ptep, pteval); } @@ -529,7 +529,7 @@ static inline void set_ptes(struct mm_struct *mm, unsigned long addr, page_table_check_ptes_set(mm, ptep, pteval, nr); for (;;) { - __set_pte_at(ptep, pteval); + __set_pte_at(mm, ptep, pteval); if (--nr == 0) break; ptep++; @@ -541,7 +541,7 @@ static inline void set_ptes(struct mm_struct *mm, unsigned long addr, static inline void pte_clear(struct mm_struct *mm, unsigned long addr, pte_t *ptep) { - __set_pte_at(ptep, __pte(0)); + __set_pte_at(mm, ptep, __pte(0)); } #define __HAVE_ARCH_PTEP_SET_ACCESS_FLAGS /* defined in mm/pgtable.c */ @@ -707,14 +707,14 @@ static inline void set_pmd_at(struct mm_struct *mm, unsigned long addr, pmd_t *pmdp, pmd_t pmd) { page_table_check_pmd_set(mm, pmdp, pmd); - return __set_pte_at((pte_t *)pmdp, pmd_pte(pmd)); + return __set_pte_at(mm, (pte_t *)pmdp, pmd_pte(pmd)); } static inline void set_pud_at(struct mm_struct *mm, unsigned long addr, pud_t *pudp, pud_t pud) { page_table_check_pud_set(mm, pudp, pud); - return __set_pte_at((pte_t *)pudp, pud_pte(pud)); + return __set_pte_at(mm, (pte_t *)pudp, pud_pte(pud)); } #ifdef CONFIG_PAGE_TABLE_CHECK diff --git a/arch/riscv/mm/cacheflush.c b/arch/riscv/mm/cacheflush.c index b0774993c0f3..3e234e95d952 100644 --- a/arch/riscv/mm/cacheflush.c +++ b/arch/riscv/mm/cacheflush.c @@ -85,12 +85,12 @@ void flush_icache_mm(struct mm_struct *mm, bool local) #endif /* CONFIG_SMP */ #ifdef CONFIG_MMU -void flush_icache_pte(pte_t pte) +void flush_icache_pte(struct mm_struct *mm, pte_t pte) { struct folio *folio = page_folio(pte_page(pte)); if (!test_bit(PG_dcache_clean, &folio->flags)) { - flush_icache_all(); + flush_icache_mm(mm, false); set_bit(PG_dcache_clean, &folio->flags); } } diff --git a/arch/riscv/mm/pgtable.c b/arch/riscv/mm/pgtable.c index ef887efcb679..533ec9055fa0 100644 --- a/arch/riscv/mm/pgtable.c +++ b/arch/riscv/mm/pgtable.c @@ -10,7 +10,7 @@ int ptep_set_access_flags(struct vm_area_struct *vma, pte_t entry, int dirty) { if (!pte_same(ptep_get(ptep), entry)) - __set_pte_at(ptep, entry); + __set_pte_at(vma->vm_mm, ptep, entry); /* * update_mmu_cache will unconditionally execute, handling both * the case that the PTE changed and the spurious fault case.
We used to emit a flush_icache_all() whenever a dirty executable mapping is set in the page table but we can instead call flush_icache_mm() which will only send IPIs to cores that currently run this mm and add a deferred icache flush to the others. The number of calls to sbi_remote_fence_i() (tested without IPI support): With a simple buildroot rootfs: * Before: ~5k * After : 4 (!) Tested on HW, the boot to login is ~4.5% faster. With an ubuntu rootfs: * Before: ~24k * After : ~13k Signed-off-by: Alexandre Ghiti <alexghiti@rivosinc.com> --- arch/riscv/include/asm/pgtable.h | 14 +++++++------- arch/riscv/mm/cacheflush.c | 4 ++-- arch/riscv/mm/pgtable.c | 2 +- 3 files changed, 10 insertions(+), 10 deletions(-)