[04/25] mm: Create FolioFlags

Message ID	20201216182335.27227-5-willy@infradead.org (mailing list archive)
State	New, archived
Headers	show Return-Path: <SRS0=WzRt=FU=kvack.org=owner-linux-mm@kernel.org> DMARC-Filter: OpenDMARC Filter v1.3.2 mail.kernel.org 5E19125E3C From: "Matthew Wilcox (Oracle)" <willy@infradead.org> To: linux-fsdevel@vger.kernel.org, linux-mm@kvack.org Cc: "Matthew Wilcox (Oracle)" <willy@infradead.org>, linux-kernel@vger.kernel.org Subject: [PATCH 04/25] mm: Create FolioFlags Date: Wed, 16 Dec 2020 18:23:14 +0000 Message-Id: <20201216182335.27227-5-willy@infradead.org> In-Reply-To: <20201216182335.27227-1-willy@infradead.org> References: <20201216182335.27227-1-willy@infradead.org> MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable Sender: owner-linux-mm@kvack.org Precedence: bulk
Series	Page folios \| expand [00/25] Page folios [01/25] mm: Introduce struct folio [02/25] mm: Add put_folio [03/25] mm: Add get_folio [04/25] mm: Create FolioFlags [05/25] mm: Add unlock_folio [06/25] mm: Add lock_folio [07/25] mm: Add lock_folio_killable [08/25] mm: Add __alloc_folio_node and alloc_folio [09/25] mm: Convert __page_cache_alloc to return a folio [10/25] mm/filemap: Convert end_page_writeback to use a folio [11/25] mm: Convert mapping_get_entry to return a folio [12/25] mm: Add mark_folio_accessed [13/25] mm: Add filemap_get_folio and find_get_folio [14/25] mm/filemap: Add folio_add_to_page_cache [15/25] mm/swap: Convert rotate_reclaimable_page to folio [16/25] mm: Add folio_mapping [17/25] mm: Rename THP_SUPPORT to MULTI_PAGE_FOLIOS [18/25] btrfs: Use readahead_batch_length [19/25] fs: Change page refcount rules for readahead [20/25] fs: Change readpage to take a folio [21/25] mm: Convert wait_on_page_bit to wait_on_folio_bit [22/25] mm: Add wait_on_folio_locked & wait_on_folio_locked_killable [23/25] mm: Add flush_dcache_folio [24/25] mm: Add read_cache_folio and read_mapping_folio [25/25] fs: Convert vfs_dedupe_file_range_compare to folios

diff --git a/include/linux/page-flags.h b/include/linux/page-flags.h index ec5d0290e0ee..446217ce13e9 100644 --- a/include/linux/page-flags.h +++ b/include/linux/page-flags.h @@ -212,6 +212,12 @@ static inline void page_init_poison(struct page *page, size_t size) } #endif +static unsigned long *folio_flags(struct folio *folio) +{ + VM_BUG_ON_PGFLAGS(PagePoisoned(&folio->page), &folio->page); + return &folio->page.flags; +} + /* * Page flags policies wrt compound pages * @@ -260,30 +266,44 @@ static inline void page_init_poison(struct page *page, size_t size) * Macros to create function definitions for page flags */ #define TESTPAGEFLAG(uname, lname, policy) \ +static __always_inline int Folio##uname(struct folio *folio) \ + { return test_bit(PG_##lname, folio_flags(folio)); } \ static __always_inline int Page##uname(struct page *page) \ { return test_bit(PG_##lname, &policy(page, 0)->flags); } #define SETPAGEFLAG(uname, lname, policy) \ +static __always_inline void SetFolio##uname(struct folio *folio) \ + { set_bit(PG_##lname, folio_flags(folio)); } \ static __always_inline void SetPage##uname(struct page *page) \ { set_bit(PG_##lname, &policy(page, 1)->flags); } #define CLEARPAGEFLAG(uname, lname, policy) \ +static __always_inline void ClearFolio##uname(struct folio *folio) \ + { clear_bit(PG_##lname, folio_flags(folio)); } \ static __always_inline void ClearPage##uname(struct page *page) \ { clear_bit(PG_##lname, &policy(page, 1)->flags); } #define __SETPAGEFLAG(uname, lname, policy) \ +static __always_inline void __SetFolio##uname(struct folio *folio) \ + { __set_bit(PG_##lname, folio_flags(folio)); } \ static __always_inline void __SetPage##uname(struct page *page) \ { __set_bit(PG_##lname, &policy(page, 1)->flags); } #define __CLEARPAGEFLAG(uname, lname, policy) \ +static __always_inline void __ClearFolio##uname(struct folio *folio) \ + { __clear_bit(PG_##lname, folio_flags(folio)); } \ static __always_inline void __ClearPage##uname(struct page *page) \ { __clear_bit(PG_##lname, &policy(page, 1)->flags); } #define TESTSETFLAG(uname, lname, policy) \ +static __always_inline int TestSetFolio##uname(struct folio *folio) \ + { return test_and_set_bit(PG_##lname, folio_flags(folio)); } \ static __always_inline int TestSetPage##uname(struct page *page) \ { return test_and_set_bit(PG_##lname, &policy(page, 1)->flags); } #define TESTCLEARFLAG(uname, lname, policy) \ +static __always_inline int TestClearFolio##uname(struct folio *folio) \ + { return test_and_clear_bit(PG_##lname, folio_flags(folio)); } \ static __always_inline int TestClearPage##uname(struct page *page) \ { return test_and_clear_bit(PG_##lname, &policy(page, 1)->flags); } @@ -302,21 +322,27 @@ static __always_inline int TestClearPage##uname(struct page *page) \ TESTCLEARFLAG(uname, lname, policy) #define TESTPAGEFLAG_FALSE(uname) \ +static inline int Folio##uname(const struct folio *folio) { return 0; } \ static inline int Page##uname(const struct page *page) { return 0; } #define SETPAGEFLAG_NOOP(uname) \ +static inline void SetFolio##uname(struct folio *folio) { } \ static inline void SetPage##uname(struct page *page) { } #define CLEARPAGEFLAG_NOOP(uname) \ +static inline void ClearFolio##uname(struct folio *folio) { } \ static inline void ClearPage##uname(struct page *page) { } #define __CLEARPAGEFLAG_NOOP(uname) \ +static inline void __ClearFolio##uname(struct folio *folio) { } \ static inline void __ClearPage##uname(struct page *page) { } #define TESTSETFLAG_FALSE(uname) \ +static inline int TestSetFolio##uname(struct folio *folio) { return 0; } \ static inline int TestSetPage##uname(struct page *page) { return 0; } #define TESTCLEARFLAG_FALSE(uname) \ +static inline int TestClearFolio##uname(struct folio *folio) { return 0; } \ static inline int TestClearPage##uname(struct page *page) { return 0; } #define PAGEFLAG_FALSE(uname) TESTPAGEFLAG_FALSE(uname) \ @@ -393,14 +419,18 @@ PAGEFLAG_FALSE(HighMem) #endif #ifdef CONFIG_SWAP -static __always_inline int PageSwapCache(struct page *page) +static __always_inline bool FolioSwapCache(struct folio *folio) { -#ifdef CONFIG_THP_SWAP - page = compound_head(page); -#endif - return PageSwapBacked(page) && test_bit(PG_swapcache, &page->flags); + return FolioSwapBacked(folio) && + test_bit(PG_swapcache, folio_flags(folio)); + +} +static __always_inline bool PageSwapCache(struct page *page) +{ + return FolioSwapCache(page_folio(page)); } + SETPAGEFLAG(SwapCache, swapcache, PF_NO_TAIL) CLEARPAGEFLAG(SwapCache, swapcache, PF_NO_TAIL) #else @@ -509,18 +539,16 @@ TESTPAGEFLAG_FALSE(Ksm) u64 stable_page_flags(struct page *page); -static inline int PageUptodate(struct page *page) +static inline int FolioUptodate(struct folio *folio) { - int ret; - page = compound_head(page); - ret = test_bit(PG_uptodate, &(page)->flags); + int ret = test_bit(PG_uptodate, folio_flags(folio)); /* * Must ensure that the data we read out of the page is loaded * _after_ we've loaded page->flags to check for PageUptodate. * We can skip the barrier if the page is not uptodate, because * we wouldn't be reading anything from it. * - * See SetPageUptodate() for the other side of the story. + * See SetFolioUptodate() for the other side of the story. */ if (ret) smp_rmb(); @@ -528,23 +556,38 @@ static inline int PageUptodate(struct page *page) return ret; } -static __always_inline void __SetPageUptodate(struct page *page) +static inline int PageUptodate(struct page *page) +{ + return FolioUptodate(page_folio(page)); +} + +static __always_inline void __SetFolioUptodate(struct folio *folio) { - VM_BUG_ON_PAGE(PageTail(page), page); smp_wmb(); - __set_bit(PG_uptodate, &page->flags); + __set_bit(PG_uptodate, folio_flags(folio)); } -static __always_inline void SetPageUptodate(struct page *page) +static __always_inline void SetFolioUptodate(struct folio *folio) { - VM_BUG_ON_PAGE(PageTail(page), page); /* * Memory barrier must be issued before setting the PG_uptodate bit, * so that all previous stores issued in order to bring the page * uptodate are actually visible before PageUptodate becomes true. */ smp_wmb(); - set_bit(PG_uptodate, &page->flags); + set_bit(PG_uptodate, folio_flags(folio)); +} + +static __always_inline void __SetPageUptodate(struct page *page) +{ + VM_BUG_ON_PAGE(PageTail(page), page); + __SetFolioUptodate((struct folio *)page); +} + +static __always_inline void SetPageUptodate(struct page *page) +{ + VM_BUG_ON_PAGE(PageTail(page), page); + SetFolioUptodate((struct folio *)page); } CLEARPAGEFLAG(Uptodate, uptodate, PF_NO_TAIL) @@ -593,6 +636,10 @@ static inline void ClearPageCompound(struct page *page) int PageHuge(struct page *page); int PageHeadHuge(struct page *page); bool page_huge_active(struct page *page); +static inline bool FolioHuge(struct folio *folio) +{ + return PageHeadHuge(&folio->page); +} #else TESTPAGEFLAG_FALSE(Huge) TESTPAGEFLAG_FALSE(HeadHuge) @@ -603,7 +650,6 @@ static inline bool page_huge_active(struct page *page) } #endif - #ifdef CONFIG_TRANSPARENT_HUGEPAGE /* * PageHuge() only returns true for hugetlbfs pages, but not for

[04/25] mm: Create FolioFlags

Commit Message

Patch