[v9,6/8] mm: buddy: make room for a new variable, mgen, in struct page

Commit Message

Byungchul Park April 17, 2024, 7:18 a.m. UTC

Functionally, no change.  This is a preparation for migrc mechanism that
tracks need of tlb flush for each page residing in buddy, using a
generation number in struct page.

Fortunately, since the private field in struct page is used only to
store page order in buddy, ranging from 0 to MAX_PAGE_ORDER, that can be
covered with unsigned short int.  So splitted it into two smaller ones,
order and mgen, so that the both can be used in buddy at the same time.

Signed-off-by: Byungchul Park <[e-mail]>
---
 include/linux/mm_types.h | 39 ++++++++++++++++++++++++++++++++-------
 mm/internal.h            |  4 ++--
 mm/page_alloc.c          | 13 ++++++++-----
 3 files changed, 42 insertions(+), 14 deletions(-)

Patch

diff --git a/include/linux/mm_types.h b/include/linux/mm_types.h
index 5240bd7bca33..4130d22977d6 100644
--- a/include/linux/mm_types.h
+++ b/include/linux/mm_types.h
@@ -108,13 +108,24 @@  struct page {
 				pgoff_t index;		/* Our offset within mapping. */
 				unsigned long share;	/* share count for fsdax */
 			};
-			/**
-			 * @private: Mapping-private opaque data.
-			 * Usually used for buffer_heads if PagePrivate.
-			 * Used for swp_entry_t if PageSwapCache.
-			 * Indicates order in the buddy system if PageBuddy.
-			 */
-			unsigned long private;
+			union {
+				/**
+				 * @private: Mapping-private opaque data.
+				 * Usually used for buffer_heads if PagePrivate.
+				 * Used for swp_entry_t if PageSwapCache.
+				 */
+				unsigned long private;
+				struct {
+					/*
+					 * Indicates order in the buddy system if PageBuddy.
+					 */
+					unsigned short int order;
+					/*
+					 * Tracks need of tlb flush used by migrc
+					 */
+					unsigned short int mgen;
+				};
+			};
 		};
 		struct {	/* page_pool used by netstack */
 			/**
@@ -520,6 +531,20 @@  static inline void set_page_private(struct page *page, unsigned long private)
 	page->private = private;
 }
 
+#define page_buddy_order(page)		((page)->order)
+
+static inline void set_page_buddy_order(struct page *page, unsigned int order)
+{
+	page->order = (unsigned short int)order;
+}
+
+#define page_buddy_mgen(page)		((page)->mgen)
+
+static inline void set_page_buddy_mgen(struct page *page, unsigned short int mgen)
+{
+	page->mgen = mgen;
+}
+
 static inline void *folio_get_private(struct folio *folio)
 {
 	return folio->private;
diff --git a/mm/internal.h b/mm/internal.h
index bbaf1fd97b1c..e3a8b77b58ca 100644
--- a/mm/internal.h
+++ b/mm/internal.h
@@ -364,7 +364,7 @@  struct alloc_context {
 static inline unsigned int buddy_order(struct page *page)
 {
 	/* PageBuddy() must be checked by the caller */
-	return page_private(page);
+	return page_buddy_order(page);
 }
 
 /*
@@ -378,7 +378,7 @@  static inline unsigned int buddy_order(struct page *page)
  * times, potentially observing different values in the tests and the actual
  * use of the result.
  */
-#define buddy_order_unsafe(page)	READ_ONCE(page_private(page))
+#define buddy_order_unsafe(page)	READ_ONCE(page_buddy_order(page))
 
 /*
  * This function checks whether a page is free && is the buddy
diff --git a/mm/page_alloc.c b/mm/page_alloc.c
index 14d39f34d336..555f6d2e33ea 100644
--- a/mm/page_alloc.c
+++ b/mm/page_alloc.c
@@ -604,9 +604,12 @@  void destroy_large_folio(struct folio *folio)
 	free_the_page(&folio->page, folio_order(folio));
 }
 
-static inline void set_buddy_order(struct page *page, unsigned int order)
+static inline void set_buddy_order_mgen(struct page *page,
+					unsigned int order,
+					unsigned short int mgen)
 {
-	set_page_private(page, order);
+	set_page_buddy_order(page, order);
+	set_page_buddy_mgen(page, order);
 	__SetPageBuddy(page);
 }
 
@@ -824,7 +827,7 @@  static inline void __free_one_page(struct page *page,
 	}
 
 done_merging:
-	set_buddy_order(page, order);
+	set_buddy_order_mgen(page, order, 0);
 
 	if (fpi_flags & FPI_TO_TAIL)
 		to_tail = true;
@@ -1404,7 +1407,7 @@  static inline void expand(struct zone *zone, struct page *page,
 			continue;
 
 		add_to_free_list(&page[size], zone, high, migratetype);
-		set_buddy_order(&page[size], high);
+		set_buddy_order_mgen(&page[size], high, 0);
 	}
 }
 
@@ -6710,7 +6713,7 @@  static void break_down_buddy_pages(struct zone *zone, struct page *page,
 			continue;
 
 		add_to_free_list(current_buddy, zone, high, migratetype);
-		set_buddy_order(current_buddy, high);
+		set_buddy_order_mgen(current_buddy, high, 0);
 	}
 }