@@ -193,6 +193,8 @@ static inline int split_huge_page(struct page *page)
}
void deferred_split_huge_page(struct page *page);
+void add_underutilized_thp(struct page *page);
+
void __split_huge_pmd(struct vm_area_struct *vma, pmd_t *pmd,
unsigned long address, bool freeze, struct folio *folio);
@@ -303,6 +305,11 @@ static inline struct list_head *page_deferred_list(struct page *page)
return &page[2].deferred_list;
}
+static inline struct list_head *page_underutilized_thp_list(struct page *page)
+{
+ return &page[3].underutilized_thp_list;
+}
+
#else /* CONFIG_TRANSPARENT_HUGEPAGE */
#define HPAGE_PMD_SHIFT ({ BUILD_BUG(); 0; })
#define HPAGE_PMD_MASK ({ BUILD_BUG(); 0; })
@@ -89,6 +89,18 @@ void memcg_reparent_list_lrus(struct mem_cgroup *memcg, struct mem_cgroup *paren
*/
bool list_lru_add(struct list_lru *lru, struct list_head *item);
+/**
+ * list_lru_add_page: add an element to the lru list's tail
+ * @list_lru: the lru pointer
+ * @page: the page containing the item
+ * @item: the item to be deleted.
+ *
+ * This function works the same as list_lru_add in terms of list
+ * manipulation. Used for non slab objects contained in the page.
+ *
+ * Return value: true if the list was updated, false otherwise
+ */
+bool list_lru_add_page(struct list_lru *lru, struct page *page, struct list_head *item);
/**
* list_lru_del: delete an element to the lru list
* @list_lru: the lru pointer
@@ -102,6 +114,18 @@ bool list_lru_add(struct list_lru *lru, struct list_head *item);
*/
bool list_lru_del(struct list_lru *lru, struct list_head *item);
+/**
+ * list_lru_del_page: delete an element to the lru list
+ * @list_lru: the lru pointer
+ * @page: the page containing the item
+ * @item: the item to be deleted.
+ *
+ * This function works the same as list_lru_del in terms of list
+ * manipulation. Used for non slab objects contained in the page.
+ *
+ * Return value: true if the list was updated, false otherwise
+ */
+bool list_lru_del_page(struct list_lru *lru, struct page *page, struct list_head *item);
/**
* list_lru_count_one: return the number of objects currently held by @lru
* @lru: the lru pointer.
@@ -151,6 +151,11 @@ struct page {
/* For both global and memcg */
struct list_head deferred_list;
};
+ struct { /* Third tail page of compound page */
+ unsigned long _compound_pad_3; /* compound_head */
+ unsigned long _compound_pad_4;
+ struct list_head underutilized_thp_list;
+ };
struct { /* Page table pages */
unsigned long _pt_pad_1; /* compound_head */
pgtable_t pmd_huge_pte; /* protected by page->ptl */
@@ -80,6 +80,8 @@ static atomic_t huge_zero_refcount;
struct page *huge_zero_page __read_mostly;
unsigned long huge_zero_pfn __read_mostly = ~0UL;
+struct list_lru huge_low_util_page_lru;
+
static void thp_utilization_workfn(struct work_struct *work);
static DECLARE_DELAYED_WORK(thp_utilization_work, thp_utilization_workfn);
@@ -264,6 +266,51 @@ static struct shrinker huge_zero_page_shrinker = {
.seeks = DEFAULT_SEEKS,
};
+static enum lru_status low_util_free_page(struct list_head *item,
+ struct list_lru_one *lru,
+ spinlock_t *lock,
+ void *cb_arg)
+{
+ int bucket, num_utilized_pages;
+ struct page *head = compound_head(list_entry(item,
+ struct page,
+ underutilized_thp_list));
+
+ if (get_page_unless_zero(head)) {
+ lock_page(head);
+ list_lru_isolate(lru, item);
+ num_utilized_pages = thp_number_utilized_pages(head);
+ bucket = thp_utilization_bucket(num_utilized_pages);
+ if (bucket < THP_UTIL_BUCKET_NR - 1)
+ split_huge_page(head);
+ unlock_page(head);
+ put_page(head);
+ }
+
+ return LRU_REMOVED_RETRY;
+}
+
+static unsigned long shrink_huge_low_util_page_count(struct shrinker *shrink,
+ struct shrink_control *sc)
+{
+ return HPAGE_PMD_NR * list_lru_shrink_count(&huge_low_util_page_lru, sc);
+}
+
+static unsigned long shrink_huge_low_util_page_scan(struct shrinker *shrink,
+ struct shrink_control *sc)
+{
+ return HPAGE_PMD_NR * list_lru_shrink_walk(&huge_low_util_page_lru,
+ sc, low_util_free_page, NULL);
+}
+
+static struct shrinker huge_low_util_page_shrinker = {
+ .count_objects = shrink_huge_low_util_page_count,
+ .scan_objects = shrink_huge_low_util_page_scan,
+ .seeks = DEFAULT_SEEKS,
+ .flags = SHRINKER_NUMA_AWARE | SHRINKER_MEMCG_AWARE |
+ SHRINKER_NONSLAB,
+};
+
#ifdef CONFIG_SYSFS
static ssize_t enabled_show(struct kobject *kobj,
struct kobj_attribute *attr, char *buf)
@@ -516,13 +563,18 @@ static int __init hugepage_init(void)
goto err_slab;
schedule_delayed_work(&thp_utilization_work, HZ);
+ err = register_shrinker(&huge_low_util_page_shrinker, "thp-low-util");
+ if (err)
+ goto err_low_util_shrinker;
err = register_shrinker(&huge_zero_page_shrinker, "thp-zero");
if (err)
goto err_hzp_shrinker;
err = register_shrinker(&deferred_split_shrinker, "thp-deferred_split");
if (err)
goto err_split_shrinker;
-
+ err = list_lru_init_memcg(&huge_low_util_page_lru, &huge_low_util_page_shrinker);
+ if (err)
+ goto err_low_util_list_lru;
/*
* By default disable transparent hugepages on smaller systems,
* where the extra memory used could hurt more than TLB overhead
@@ -538,11 +590,16 @@ static int __init hugepage_init(void)
goto err_khugepaged;
return 0;
+
err_khugepaged:
+ list_lru_destroy(&huge_low_util_page_lru);
+err_low_util_list_lru:
unregister_shrinker(&deferred_split_shrinker);
err_split_shrinker:
unregister_shrinker(&huge_zero_page_shrinker);
err_hzp_shrinker:
+ unregister_shrinker(&huge_low_util_page_shrinker);
+err_low_util_shrinker:
khugepaged_destroy();
err_slab:
hugepage_exit_sysfs(hugepage_kobj);
@@ -617,6 +674,7 @@ void prep_transhuge_page(struct page *page)
*/
INIT_LIST_HEAD(page_deferred_list(page));
+ INIT_LIST_HEAD(page_underutilized_thp_list(page));
set_compound_page_dtor(page, TRANSHUGE_PAGE_DTOR);
}
@@ -2519,8 +2577,7 @@ static void __split_huge_page_tail(struct page *head, int tail,
(1L << PG_dirty)));
/* ->mapping in first tail page is compound_mapcount */
- VM_BUG_ON_PAGE(tail > 2 && page_tail->mapping != TAIL_MAPPING,
- page_tail);
+ VM_BUG_ON_PAGE(tail > 3 && page_tail->mapping != TAIL_MAPPING, page_tail);
page_tail->mapping = head->mapping;
page_tail->index = head->index + tail;
page_tail->private = 0;
@@ -2727,6 +2784,7 @@ int split_huge_page_to_list(struct page *page, struct list_head *list)
struct folio *folio = page_folio(page);
struct page *head = &folio->page;
struct deferred_split *ds_queue = get_deferred_split_queue(head);
+ struct list_head *underutilized_thp_list = page_underutilized_thp_list(head);
XA_STATE(xas, &head->mapping->i_pages, head->index);
struct anon_vma *anon_vma = NULL;
struct address_space *mapping = NULL;
@@ -2825,6 +2883,8 @@ int split_huge_page_to_list(struct page *page, struct list_head *list)
list_del(page_deferred_list(head));
}
spin_unlock(&ds_queue->split_queue_lock);
+ if (!list_empty(underutilized_thp_list))
+ list_lru_del_page(&huge_low_util_page_lru, head, underutilized_thp_list);
if (mapping) {
int nr = thp_nr_pages(head);
@@ -2867,6 +2927,7 @@ int split_huge_page_to_list(struct page *page, struct list_head *list)
void free_transhuge_page(struct page *page)
{
struct deferred_split *ds_queue = get_deferred_split_queue(page);
+ struct list_head *underutilized_thp_list = page_underutilized_thp_list(page);
unsigned long flags;
spin_lock_irqsave(&ds_queue->split_queue_lock, flags);
@@ -2875,6 +2936,12 @@ void free_transhuge_page(struct page *page)
list_del(page_deferred_list(page));
}
spin_unlock_irqrestore(&ds_queue->split_queue_lock, flags);
+ if (!list_empty(underutilized_thp_list))
+ list_lru_del_page(&huge_low_util_page_lru, page, underutilized_thp_list);
+
+ if (PageLRU(page))
+ __clear_page_lru_flags(page);
+
free_compound_page(page);
}
@@ -2915,6 +2982,26 @@ void deferred_split_huge_page(struct page *page)
spin_unlock_irqrestore(&ds_queue->split_queue_lock, flags);
}
+void add_underutilized_thp(struct page *page)
+{
+ VM_BUG_ON_PAGE(!PageTransHuge(page), page);
+
+ if (PageSwapCache(page))
+ return;
+
+ /*
+ * Need to take a reference on the page to prevent the page from getting free'd from
+ * under us while we are adding the THP to the shrinker.
+ */
+ if (!get_page_unless_zero(page))
+ return;
+
+ if (!is_huge_zero_page(page) && is_anon_transparent_hugepage(page))
+ list_lru_add_page(&huge_low_util_page_lru, page, page_underutilized_thp_list(page));
+
+ put_page(page);
+}
+
static unsigned long deferred_split_count(struct shrinker *shrink,
struct shrink_control *sc)
{
@@ -3449,6 +3536,9 @@ static void thp_util_scan(unsigned long pfn_end)
if (bucket < 0)
continue;
+ if (bucket < THP_UTIL_BUCKET_NR - 1)
+ add_underutilized_thp(page);
+
thp_scan.buckets[bucket].nr_thps++;
thp_scan.buckets[bucket].nr_zero_pages += (HPAGE_PMD_NR - num_utilized_pages);
}
@@ -140,6 +140,32 @@ bool list_lru_add(struct list_lru *lru, struct list_head *item)
}
EXPORT_SYMBOL_GPL(list_lru_add);
+bool list_lru_add_page(struct list_lru *lru, struct page *page, struct list_head *item)
+{
+ int nid = page_to_nid(page);
+ struct list_lru_node *nlru = &lru->node[nid];
+ struct list_lru_one *l;
+ struct mem_cgroup *memcg;
+
+ spin_lock(&nlru->lock);
+ if (list_empty(item)) {
+ memcg = page_memcg(page);
+ memcg_list_lru_alloc(memcg, lru, GFP_KERNEL);
+ l = list_lru_from_memcg_idx(lru, nid, memcg_kmem_id(memcg));
+ list_add_tail(item, &l->list);
+ /* Set shrinker bit if the first element was added */
+ if (!l->nr_items++)
+ set_shrinker_bit(memcg, nid,
+ lru_shrinker_id(lru));
+ nlru->nr_items++;
+ spin_unlock(&nlru->lock);
+ return true;
+ }
+ spin_unlock(&nlru->lock);
+ return false;
+}
+EXPORT_SYMBOL_GPL(list_lru_add_page);
+
bool list_lru_del(struct list_lru *lru, struct list_head *item)
{
int nid = page_to_nid(virt_to_page(item));
@@ -160,6 +186,29 @@ bool list_lru_del(struct list_lru *lru, struct list_head *item)
}
EXPORT_SYMBOL_GPL(list_lru_del);
+bool list_lru_del_page(struct list_lru *lru, struct page *page, struct list_head *item)
+{
+ int nid = page_to_nid(page);
+ struct list_lru_node *nlru = &lru->node[nid];
+ struct list_lru_one *l;
+ struct mem_cgroup *memcg;
+
+ spin_lock(&nlru->lock);
+ if (!list_empty(item)) {
+ memcg = page_memcg(page);
+ memcg_list_lru_alloc(memcg, lru, GFP_KERNEL);
+ l = list_lru_from_memcg_idx(lru, nid, memcg_kmem_id(memcg));
+ list_del_init(item);
+ l->nr_items--;
+ nlru->nr_items--;
+ spin_unlock(&nlru->lock);
+ return true;
+ }
+ spin_unlock(&nlru->lock);
+ return false;
+}
+EXPORT_SYMBOL_GPL(list_lru_del_page);
+
void list_lru_isolate(struct list_lru_one *list, struct list_head *item)
{
list_del_init(item);
@@ -1327,6 +1327,12 @@ static int free_tail_pages_check(struct page *head_page, struct page *page)
* deferred_list.next -- ignore value.
*/
break;
+ case 3:
+ /*
+ * the third tail page: ->mapping is
+ * underutilized_thp_list.next -- ignore value.
+ */
+ break;
default:
if (page->mapping != TAIL_MAPPING) {
bad_page(page, "corrupted mapping in tail page");