@@ -3364,11 +3364,12 @@ static vm_fault_t filemap_map_folio_range(struct vm_fault *vmf,
struct file *file = vma->vm_file;
struct page *page = folio_page(folio, start);
unsigned int mmap_miss = READ_ONCE(file->f_ra.mmap_miss);
- unsigned int ref_count = 0, count = 0;
+ unsigned int mapped = 0;
+ pte_t *pte = vmf->pte;
do {
if (PageHWPoison(page))
- continue;
+ goto map;
if (mmap_miss > 0)
mmap_miss--;
@@ -3378,20 +3379,33 @@ static vm_fault_t filemap_map_folio_range(struct vm_fault *vmf,
* handled in the specific fault path, and it'll prohibit the
* fault-around logic.
*/
- if (!pte_none(*vmf->pte))
- continue;
+ if (!pte_none(pte[mapped]))
+ goto map;
if (vmf->address == addr)
ret = VM_FAULT_NOPAGE;
- ref_count++;
- do_set_pte(vmf, page, addr);
- } while (vmf->pte++, page++, addr += PAGE_SIZE, ++count < nr_pages);
+ mapped++;
+ continue;
- /* Restore the vmf->pte */
- vmf->pte -= nr_pages;
+map:
+ if (mapped) {
+ do_set_pte_range(vmf, folio, addr, pte, start, mapped);
+ folio_ref_add(folio, mapped);
+ }
+
+ /* advance 1 to jump the HWPoison or !pte_none entry */
+ start += mapped + 1;
+ pte += mapped + 1;
+ addr += (mapped + 1) * PAGE_SIZE;
+ mapped = 0;
+ } while (page++, --nr_pages > 0);
+
+ if (mapped) {
+ do_set_pte_range(vmf, folio, addr, pte, start, mapped);
+ folio_ref_add(folio, mapped);
+ }
- folio_ref_add(folio, ref_count);
WRITE_ONCE(file->f_ra.mmap_miss, mmap_miss);
return ret;
Use do_set_pte_range() in filemap_map_folio_range(). Which batched updates mm counter, rmap for large folio. With a will-it-scale.page_fault3 like app (change file write fault testing to read fault testing. Trying to upstream it to will-it-scale at [1]) got 15% performance gain. Perf data collected before/after the change: 18.73%--page_add_file_rmap | --11.60%--__mod_lruvec_page_state | |--7.40%--__mod_memcg_lruvec_state | | | --5.58%--cgroup_rstat_updated | --2.53%--__mod_lruvec_state | --1.48%--__mod_node_page_state 9.93%--page_add_file_rmap_range | --2.67%--__mod_lruvec_page_state | |--1.95%--__mod_memcg_lruvec_state | | | --1.57%--cgroup_rstat_updated | --0.61%--__mod_lruvec_state | --0.54%--__mod_node_page_state The running time of __mode_lruvec_page_state() is reduced about 9%. [1]: https://github.com/antonblanchard/will-it-scale/pull/37 Signed-off-by: Yin Fengwei <fengwei.yin@intel.com> --- mm/filemap.c | 34 ++++++++++++++++++++++++---------- 1 file changed, 24 insertions(+), 10 deletions(-)