@@ -111,6 +111,7 @@ enum vm_event_item { PGPGIN, PGPGOUT, PSWPIN, PSWPOUT,
#ifdef CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD
THP_SPLIT_PUD,
#endif
+ THP_SPLIT_FREE,
THP_ZERO_PAGE_ALLOC,
THP_ZERO_PAGE_ALLOC_FAILED,
THP_SWPOUT,
@@ -2505,6 +2505,8 @@ static void __split_huge_page(struct page *page, struct list_head *list,
struct address_space *swap_cache = NULL;
unsigned long offset = 0;
unsigned int nr = thp_nr_pages(head);
+ LIST_HEAD(pages_to_free);
+ int nr_pages_to_free = 0;
int i;
/* complete memcg works before add pages to LRU */
@@ -2581,6 +2583,34 @@ static void __split_huge_page(struct page *page, struct list_head *list,
continue;
unlock_page(subpage);
+ /*
+ * If a tail page has only two references left, one inherited
+ * from the isolation of its head and the other from
+ * lru_add_page_tail() which we are about to drop, it means this
+ * tail page was concurrently zapped. Then we can safely free it
+ * and save page reclaim or migration the trouble of trying it.
+ */
+ if (list && page_ref_freeze(subpage, 2)) {
+ VM_BUG_ON_PAGE(PageLRU(subpage), subpage);
+ VM_BUG_ON_PAGE(PageCompound(subpage), subpage);
+ VM_BUG_ON_PAGE(page_mapped(subpage), subpage);
+
+ ClearPageActive(subpage);
+ ClearPageUnevictable(subpage);
+ list_move(&subpage->lru, &pages_to_free);
+ nr_pages_to_free++;
+ continue;
+ }
+
+ /*
+ * If a tail page has only one reference left, it will be freed
+ * by the call to free_page_and_swap_cache below. Since zero
+ * subpages are no longer remapped, there will only be one
+ * reference left in cases outside of reclaim or migration.
+ */
+ if (page_ref_count(subpage) == 1)
+ nr_pages_to_free++;
+
/*
* Subpages may be freed if there wasn't any mapping
* like if add_to_swap() is running on a lru page that
@@ -2590,6 +2620,13 @@ static void __split_huge_page(struct page *page, struct list_head *list,
*/
free_page_and_swap_cache(subpage);
}
+
+ if (!nr_pages_to_free)
+ return;
+
+ mem_cgroup_uncharge_list(&pages_to_free);
+ free_unref_page_list(&pages_to_free);
+ count_vm_events(THP_SPLIT_FREE, nr_pages_to_free);
}
/* Racy check whether the huge page can be split */
@@ -1359,6 +1359,7 @@ const char * const vmstat_text[] = {
#ifdef CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD
"thp_split_pud",
#endif
+ "thp_split_free",
"thp_zero_page_alloc",
"thp_zero_page_alloc_failed",
"thp_swpout",