[RFC,v12,15/26] fs, filemap: refactor to gather the scattered ->write_{begin,end}() calls

Message ID	20250220052027.58847-16-byungchul@sk.com (mailing list archive)
State	New
Headers	show Return-Path: <owner-linux-mm@kvack.org> From: Byungchul Park <byungchul@sk.com> To: linux-kernel@vger.kernel.org, linux-mm@kvack.org Cc: kernel_team@skhynix.com, akpm@linux-foundation.org, ying.huang@intel.com, vernhao@tencent.com, mgorman@techsingularity.net, hughd@google.com, willy@infradead.org, david@redhat.com, peterz@infradead.org, luto@kernel.org, tglx@linutronix.de, mingo@redhat.com, bp@alien8.de, dave.hansen@linux.intel.com, rjgolo@gmail.com Subject: [RFC PATCH v12 15/26] fs, filemap: refactor to gather the scattered ->write_{begin,end}() calls Date: Thu, 20 Feb 2025 14:20:16 +0900 Message-Id: <20250220052027.58847-16-byungchul@sk.com> In-Reply-To: <20250220052027.58847-1-byungchul@sk.com> References: <20250220052027.58847-1-byungchul@sk.com> Sender: owner-linux-mm@kvack.org Precedence: bulk
Series	LUF(Lazy Unmap Flush) reducing tlb numbers over 90% \| expand [RFC,v12,00/26] LUF(Lazy Unmap Flush) reducing tlb numbers over 90% [RFC,v12,01/26] x86/tlb: add APIs manipulating tlb batch's arch data [RFC,v12,02/26] arm64/tlbflush: add APIs manipulating tlb batch's arch data [RFC,v12,03/26] riscv/tlb: add APIs manipulating tlb batch's arch data [RFC,v12,04/26] x86/tlb, riscv/tlb, mm/rmap: separate arch_tlbbatch_clear() out of arch_tlbbatch_fl… [RFC,v12,05/26] mm/buddy: make room for a new variable, luf_key, in struct page [RFC,v12,06/26] mm: move should_skip_kasan_poison() to mm/internal.h [RFC,v12,07/26] mm: introduce luf_ugen to be used as a global timestamp [RFC,v12,08/26] mm: introduce luf_batch to be used as hash table to store luf meta data [RFC,v12,09/26] mm: introduce API to perform tlb shootdown on exit from page allocator [RFC,v12,10/26] mm: introduce APIs to check if the page allocation is tlb shootdownable [RFC,v12,11/26] mm: deliver luf_key to pcp or buddy on free after unmapping [RFC,v12,12/26] mm: delimit critical sections to take off pages from pcp or buddy alloctor [RFC,v12,13/26] mm: introduce pend_list in struct free_area to track luf'd pages [RFC,v12,14/26] mm/rmap: recognize read-only tlb entries during batched tlb flush [RFC,v12,15/26] fs, filemap: refactor to gather the scattered ->write_{begin,end}() calls [RFC,v12,16/26] mm: implement LUF(Lazy Unmap Flush) defering tlb flush when folios get unmapped [RFC,v12,17/26] x86/tlb, riscv/tlb, arm64/tlbflush, mm: remove cpus from tlb shootdown that already… [RFC,v12,18/26] mm/page_alloc: retry 3 times to take pcp pages on luf check failure [RFC,v12,19/26] mm: skip luf tlb flush for luf'd mm that already has been done [RFC,v12,20/26] mm, fs: skip tlb flushes for luf'd filemap that already has been done [RFC,v12,21/26] mm: perform luf tlb shootdown per zone in batched manner [RFC,v12,22/26] mm/page_alloc: not allow to tlb shootdown if !preemptable() && non_luf_pages_ok() [RFC,v12,23/26] mm: separate move/undo parts from migrate_pages_batch() [RFC,v12,24/26] mm/migrate: apply luf mechanism to unmapping during migration [RFC,v12,25/26] mm/vmscan: apply luf mechanism to unmapping during folio reclaim [RFC,v12,26/26] mm/luf: implement luf debug feature

diff --git a/drivers/gpu/drm/i915/gem/i915_gem_shmem.c b/drivers/gpu/drm/i915/gem/i915_gem_shmem.c index fe69f2c8527d7..1d475d681d3de 100644 --- a/drivers/gpu/drm/i915/gem/i915_gem_shmem.c +++ b/drivers/gpu/drm/i915/gem/i915_gem_shmem.c @@ -422,7 +422,6 @@ shmem_pwrite(struct drm_i915_gem_object *obj, const struct drm_i915_gem_pwrite *arg) { struct address_space *mapping = obj->base.filp->f_mapping; - const struct address_space_operations *aops = mapping->a_ops; char __user *user_data = u64_to_user_ptr(arg->data_ptr); u64 remain; loff_t pos; @@ -481,7 +480,7 @@ shmem_pwrite(struct drm_i915_gem_object *obj, if (err) return err; - err = aops->write_begin(obj->base.filp, mapping, pos, len, + err = mapping_write_begin(obj->base.filp, mapping, pos, len, &folio, &data); if (err < 0) return err; @@ -492,7 +491,7 @@ shmem_pwrite(struct drm_i915_gem_object *obj, pagefault_enable(); kunmap_local(vaddr); - err = aops->write_end(obj->base.filp, mapping, pos, len, + err = mapping_write_end(obj->base.filp, mapping, pos, len, len - unwritten, folio, data); if (err < 0) return err; @@ -658,7 +657,6 @@ i915_gem_object_create_shmem_from_data(struct drm_i915_private *i915, { struct drm_i915_gem_object *obj; struct file *file; - const struct address_space_operations *aops; loff_t pos; int err; @@ -670,21 +668,20 @@ i915_gem_object_create_shmem_from_data(struct drm_i915_private *i915, GEM_BUG_ON(obj->write_domain != I915_GEM_DOMAIN_CPU); file = obj->base.filp; - aops = file->f_mapping->a_ops; pos = 0; do { unsigned int len = min_t(typeof(size), size, PAGE_SIZE); struct folio *folio; void *fsdata; - err = aops->write_begin(file, file->f_mapping, pos, len, + err = mapping_write_begin(file, file->f_mapping, pos, len, &folio, &fsdata); if (err < 0) goto fail; memcpy_to_folio(folio, offset_in_folio(folio, pos), data, len); - err = aops->write_end(file, file->f_mapping, pos, len, len, + err = mapping_write_end(file, file->f_mapping, pos, len, len, folio, fsdata); if (err < 0) goto fail; diff --git a/fs/affs/file.c b/fs/affs/file.c index a5a861dd52230..10e7f53828e93 100644 --- a/fs/affs/file.c +++ b/fs/affs/file.c @@ -885,9 +885,9 @@ affs_truncate(struct inode *inode) loff_t isize = inode->i_size; int res; - res = mapping->a_ops->write_begin(NULL, mapping, isize, 0, &folio, &fsdata); + res = mapping_write_begin(NULL, mapping, isize, 0, &folio, &fsdata); if (!res) - res = mapping->a_ops->write_end(NULL, mapping, isize, 0, 0, folio, fsdata); + res = mapping_write_end(NULL, mapping, isize, 0, 0, folio, fsdata); else inode->i_size = AFFS_I(inode)->mmu_private; mark_inode_dirty(inode); diff --git a/fs/buffer.c b/fs/buffer.c index 88e765b0699fe..7cb0295500937 100644 --- a/fs/buffer.c +++ b/fs/buffer.c @@ -2456,7 +2456,6 @@ EXPORT_SYMBOL(block_read_full_folio); int generic_cont_expand_simple(struct inode *inode, loff_t size) { struct address_space *mapping = inode->i_mapping; - const struct address_space_operations *aops = mapping->a_ops; struct folio *folio; void *fsdata = NULL; int err; @@ -2465,11 +2464,11 @@ int generic_cont_expand_simple(struct inode *inode, loff_t size) if (err) goto out; - err = aops->write_begin(NULL, mapping, size, 0, &folio, &fsdata); + err = mapping_write_begin(NULL, mapping, size, 0, &folio, &fsdata); if (err) goto out; - err = aops->write_end(NULL, mapping, size, 0, 0, folio, fsdata); + err = mapping_write_end(NULL, mapping, size, 0, 0, folio, fsdata); BUG_ON(err > 0); out: @@ -2481,7 +2480,6 @@ static int cont_expand_zero(struct file *file, struct address_space *mapping, loff_t pos, loff_t *bytes) { struct inode *inode = mapping->host; - const struct address_space_operations *aops = mapping->a_ops; unsigned int blocksize = i_blocksize(inode); struct folio *folio; void *fsdata = NULL; @@ -2501,12 +2499,12 @@ static int cont_expand_zero(struct file *file, struct address_space *mapping, } len = PAGE_SIZE - zerofrom; - err = aops->write_begin(file, mapping, curpos, len, + err = mapping_write_begin(file, mapping, curpos, len, &folio, &fsdata); if (err) goto out; folio_zero_range(folio, offset_in_folio(folio, curpos), len); - err = aops->write_end(file, mapping, curpos, len, len, + err = mapping_write_end(file, mapping, curpos, len, len, folio, fsdata); if (err < 0) goto out; @@ -2534,12 +2532,12 @@ static int cont_expand_zero(struct file *file, struct address_space *mapping, } len = offset - zerofrom; - err = aops->write_begin(file, mapping, curpos, len, + err = mapping_write_begin(file, mapping, curpos, len, &folio, &fsdata); if (err) goto out; folio_zero_range(folio, offset_in_folio(folio, curpos), len); - err = aops->write_end(file, mapping, curpos, len, len, + err = mapping_write_end(file, mapping, curpos, len, len, folio, fsdata); if (err < 0) goto out; diff --git a/fs/exfat/file.c b/fs/exfat/file.c index a25d7eb789f4c..242563b9dec95 100644 --- a/fs/exfat/file.c +++ b/fs/exfat/file.c @@ -539,7 +539,6 @@ static int exfat_extend_valid_size(struct file *file, loff_t new_valid_size) struct inode *inode = file_inode(file); struct exfat_inode_info *ei = EXFAT_I(inode); struct address_space *mapping = inode->i_mapping; - const struct address_space_operations *ops = mapping->a_ops; pos = ei->valid_size; while (pos < new_valid_size) { @@ -550,11 +549,11 @@ static int exfat_extend_valid_size(struct file *file, loff_t new_valid_size) if (pos + len > new_valid_size) len = new_valid_size - pos; - err = ops->write_begin(file, mapping, pos, len, &folio, NULL); + err = mapping_write_begin(file, mapping, pos, len, &folio, NULL); if (err) goto out; - err = ops->write_end(file, mapping, pos, len, len, folio, NULL); + err = mapping_write_end(file, mapping, pos, len, len, folio, NULL); if (err < 0) goto out; pos += len; diff --git a/fs/ext4/verity.c b/fs/ext4/verity.c index d9203228ce979..64fa43f80c73e 100644 --- a/fs/ext4/verity.c +++ b/fs/ext4/verity.c @@ -68,7 +68,6 @@ static int pagecache_write(struct inode *inode, const void *buf, size_t count, loff_t pos) { struct address_space *mapping = inode->i_mapping; - const struct address_space_operations *aops = mapping->a_ops; if (pos + count > inode->i_sb->s_maxbytes) return -EFBIG; @@ -80,13 +79,13 @@ static int pagecache_write(struct inode *inode, const void *buf, size_t count, void *fsdata = NULL; int res; - res = aops->write_begin(NULL, mapping, pos, n, &folio, &fsdata); + res = mapping_write_begin(NULL, mapping, pos, n, &folio, &fsdata); if (res) return res; memcpy_to_folio(folio, offset_in_folio(folio, pos), buf, n); - res = aops->write_end(NULL, mapping, pos, n, n, folio, fsdata); + res = mapping_write_end(NULL, mapping, pos, n, n, folio, fsdata); if (res < 0) return res; if (res != n) diff --git a/fs/f2fs/super.c b/fs/f2fs/super.c index 87ab5696bd482..f8d5ee466807c 100644 --- a/fs/f2fs/super.c +++ b/fs/f2fs/super.c @@ -2678,7 +2678,6 @@ static ssize_t f2fs_quota_write(struct super_block *sb, int type, { struct inode *inode = sb_dqopt(sb)->files[type]; struct address_space *mapping = inode->i_mapping; - const struct address_space_operations *a_ops = mapping->a_ops; int offset = off & (sb->s_blocksize - 1); size_t towrite = len; struct folio *folio; @@ -2690,7 +2689,7 @@ static ssize_t f2fs_quota_write(struct super_block *sb, int type, tocopy = min_t(unsigned long, sb->s_blocksize - offset, towrite); retry: - err = a_ops->write_begin(NULL, mapping, off, tocopy, + err = mapping_write_begin(NULL, mapping, off, tocopy, &folio, &fsdata); if (unlikely(err)) { if (err == -ENOMEM) { @@ -2703,7 +2702,7 @@ static ssize_t f2fs_quota_write(struct super_block *sb, int type, memcpy_to_folio(folio, offset_in_folio(folio, off), data, tocopy); - a_ops->write_end(NULL, mapping, off, tocopy, tocopy, + mapping_write_end(NULL, mapping, off, tocopy, tocopy, folio, fsdata); offset = 0; towrite -= tocopy; diff --git a/fs/f2fs/verity.c b/fs/f2fs/verity.c index 2287f238ae09e..b232589546d39 100644 --- a/fs/f2fs/verity.c +++ b/fs/f2fs/verity.c @@ -72,7 +72,6 @@ static int pagecache_write(struct inode *inode, const void *buf, size_t count, loff_t pos) { struct address_space *mapping = inode->i_mapping; - const struct address_space_operations *aops = mapping->a_ops; if (pos + count > F2FS_BLK_TO_BYTES(max_file_blocks(inode))) return -EFBIG; @@ -84,13 +83,13 @@ static int pagecache_write(struct inode *inode, const void *buf, size_t count, void *fsdata = NULL; int res; - res = aops->write_begin(NULL, mapping, pos, n, &folio, &fsdata); + res = mapping_write_begin(NULL, mapping, pos, n, &folio, &fsdata); if (res) return res; memcpy_to_folio(folio, offset_in_folio(folio, pos), buf, n); - res = aops->write_end(NULL, mapping, pos, n, n, folio, fsdata); + res = mapping_write_end(NULL, mapping, pos, n, n, folio, fsdata); if (res < 0) return res; if (res != n) diff --git a/fs/namei.c b/fs/namei.c index 4a4a22a08ac20..14a701ecf1a7e 100644 --- a/fs/namei.c +++ b/fs/namei.c @@ -5349,7 +5349,6 @@ EXPORT_SYMBOL(page_readlink); int page_symlink(struct inode *inode, const char *symname, int len) { struct address_space *mapping = inode->i_mapping; - const struct address_space_operations *aops = mapping->a_ops; bool nofs = !mapping_gfp_constraint(mapping, __GFP_FS); struct folio *folio; void *fsdata = NULL; @@ -5359,7 +5358,7 @@ int page_symlink(struct inode *inode, const char *symname, int len) retry: if (nofs) flags = memalloc_nofs_save(); - err = aops->write_begin(NULL, mapping, 0, len-1, &folio, &fsdata); + err = mapping_write_begin(NULL, mapping, 0, len-1, &folio, &fsdata); if (nofs) memalloc_nofs_restore(flags); if (err) @@ -5367,7 +5366,7 @@ int page_symlink(struct inode *inode, const char *symname, int len) memcpy(folio_address(folio), symname, len - 1); - err = aops->write_end(NULL, mapping, 0, len - 1, len - 1, + err = mapping_write_end(NULL, mapping, 0, len - 1, len - 1, folio, fsdata); if (err < 0) goto fail; diff --git a/include/linux/fs.h b/include/linux/fs.h index 3559446279c15..bfd8aaeb78bb8 100644 --- a/include/linux/fs.h +++ b/include/linux/fs.h @@ -494,6 +494,24 @@ struct address_space { #define PAGECACHE_TAG_WRITEBACK XA_MARK_1 #define PAGECACHE_TAG_TOWRITE XA_MARK_2 +static inline int mapping_write_begin(struct file *file, + struct address_space *mapping, + loff_t pos, unsigned len, + struct folio **foliop, void **fsdata) +{ + return mapping->a_ops->write_begin(file, mapping, pos, len, foliop, + fsdata); +} + +static inline int mapping_write_end(struct file *file, + struct address_space *mapping, + loff_t pos, unsigned len, unsigned copied, + struct folio *folio, void *fsdata) +{ + return mapping->a_ops->write_end(file, mapping, pos, len, copied, + folio, fsdata); +} + /* * Returns true if any of the pages in the mapping are marked with the tag. */ diff --git a/mm/filemap.c b/mm/filemap.c index e582a1545d2ae..a4930449fc705 100644 --- a/mm/filemap.c +++ b/mm/filemap.c @@ -4016,7 +4016,6 @@ ssize_t generic_perform_write(struct kiocb *iocb, struct iov_iter *i) struct file *file = iocb->ki_filp; loff_t pos = iocb->ki_pos; struct address_space *mapping = file->f_mapping; - const struct address_space_operations *a_ops = mapping->a_ops; size_t chunk = mapping_max_folio_size(mapping); long status = 0; ssize_t written = 0; @@ -4050,7 +4049,7 @@ ssize_t generic_perform_write(struct kiocb *iocb, struct iov_iter *i) break; } - status = a_ops->write_begin(file, mapping, pos, bytes, + status = mapping_write_begin(file, mapping, pos, bytes, &folio, &fsdata); if (unlikely(status < 0)) break; @@ -4065,7 +4064,7 @@ ssize_t generic_perform_write(struct kiocb *iocb, struct iov_iter *i) copied = copy_folio_from_iter_atomic(folio, offset, bytes, i); flush_dcache_folio(folio); - status = a_ops->write_end(file, mapping, pos, bytes, copied, + status = mapping_write_end(file, mapping, pos, bytes, copied, folio, fsdata); if (unlikely(status != copied)) { iov_iter_revert(i, copied - max(status, 0L));

[RFC,v12,15/26] fs, filemap: refactor to gather the scattered ->write_{begin,end}() calls

Commit Message

Patch