@@ -1152,11 +1152,18 @@ static inline void set_page_node(struct page *page, unsigned long node)
page->flags |= (node & NODES_MASK) << NODES_PGSHIFT;
}
+static inline void set_page_reserved(struct page *page, bool reserved)
+{
+ page->flags &= ~(1ul << PG_reserved);
+ page->flags |= (unsigned long)(!!reserved) << PG_reserved;
+}
+
static inline void set_page_links(struct page *page, enum zone_type zone,
- unsigned long node, unsigned long pfn)
+ unsigned long node, unsigned long pfn, bool reserved)
{
set_page_zone(page, zone);
set_page_node(page, node);
+ set_page_reserved(page, reserved);
#ifdef SECTION_IN_PAGE_FLAGS
set_page_section(page, pfn_to_section_nr(pfn));
#endif
@@ -1179,7 +1179,7 @@ static void __meminit __init_single_page(struct page *page, unsigned long pfn,
unsigned long zone, int nid)
{
mm_zero_struct_page(page);
- set_page_links(page, zone, nid, pfn);
+ set_page_links(page, zone, nid, pfn, false);
init_page_count(page);
page_mapcount_reset(page);
page_cpupid_reset_last(page);
@@ -1232,20 +1232,16 @@ static void __meminit __init_pageblock(unsigned long start_pfn,
* call because of the fact that the pfn number is used to
* get the section_nr and this function should not be
* spanning more than a single section.
+ *
+ * We can use a non-atomic operation for setting the
+ * PG_reserved flag as we are still initializing the pages.
*/
- set_page_links(page, zone, nid, start_pfn);
+ set_page_links(page, zone, nid, start_pfn, is_reserved);
init_page_count(page);
page_mapcount_reset(page);
page_cpupid_reset_last(page);
/*
- * We can use the non-atomic __set_bit operation for setting
- * the flag as we are still initializing the pages.
- */
- if (is_reserved)
- __SetPageReserved(page);
-
- /*
* ZONE_DEVICE pages union ->lru with a ->pgmap back
* pointer and hmm_data. It is a bug if a ZONE_DEVICE
* page is ever freed or placed on a driver-private list.
This patch modifies the set_page_links function to include the setting of the reserved flag via a simple AND and OR operation. The motivation for this is the fact that the existing __set_bit call still seems to have effects on performance as replacing the call with the AND and OR can reduce initialization time by as much as 4 seconds, ~26s down to ~22s, per NUMA node on a 4 socket system with 3TB of persistent memory per node. Looking over the assembly code before and after the change the main difference between the two is that the reserved bit is stored in a value that is generated outside of the main initialization loop and is then written with the other flags field values in one write to the page->flags value. Previously the generated value was written and then then btsq instruction was issued after testing the is_reserved value. Signed-off-by: Alexander Duyck <alexander.h.duyck@linux.intel.com> --- include/linux/mm.h | 9 ++++++++- mm/page_alloc.c | 14 +++++--------- 2 files changed, 13 insertions(+), 10 deletions(-)