@@ -24,7 +24,7 @@ struct pagevec {
};
void __pagevec_release(struct pagevec *pvec);
-void __pagevec_lru_add(struct pagevec *pvec);
+void __pagevec_lru_add(struct pagevec *pvec, bool new_anon);
unsigned pagevec_lookup_entries(struct pagevec *pvec,
struct address_space *mapping,
pgoff_t start, unsigned nr_entries,
@@ -334,6 +334,7 @@ extern void lru_cache_add_anon(struct page *page);
extern void lru_cache_add_file(struct page *page);
extern void lru_add_page_tail(struct page *page, struct page *page_tail,
struct lruvec *lruvec, struct list_head *head);
+extern void update_anon_page_reclaim_stat(struct page *page);
extern void activate_page(struct page *);
extern void mark_page_accessed(struct page *);
extern void lru_add_drain(void);
@@ -27,7 +27,7 @@ struct reclaim_stat {
unsigned nr_writeback;
unsigned nr_immediate;
unsigned nr_activate[2];
- unsigned nr_ref_keep;
+ unsigned nr_ref_keep[2];
unsigned nr_unmap_fail;
};
@@ -366,7 +366,8 @@ TRACE_EVENT(mm_vmscan_lru_shrink_inactive,
__entry->nr_immediate = stat->nr_immediate;
__entry->nr_activate0 = stat->nr_activate[0];
__entry->nr_activate1 = stat->nr_activate[1];
- __entry->nr_ref_keep = stat->nr_ref_keep;
+ __entry->nr_ref_keep = stat->nr_ref_keep[0] +
+ stat->nr_ref_keep[1];
__entry->nr_unmap_fail = stat->nr_unmap_fail;
__entry->priority = priority;
__entry->reclaim_flags = trace_reclaim_flags(file);
@@ -3099,6 +3099,12 @@ vm_fault_t do_swap_page(struct vm_fault *vmf)
} else {
do_page_add_anon_rmap(page, vma, vmf->address, exclusive);
mem_cgroup_commit_charge(page, memcg, true, false);
+
+ /*
+ * To reduce difference of the reclaim ratio caused by LRU
+ * algorithm change, update reclaim_stat manually.
+ */
+ update_anon_page_reclaim_stat(page);
}
if (shadow) {
@@ -273,7 +273,7 @@ static void __putback_lru_fast(struct pagevec *pvec, int pgrescued)
*__pagevec_lru_add() calls release_pages() so we don't call
* put_page() explicitly
*/
- __pagevec_lru_add(pvec);
+ __pagevec_lru_add(pvec, false);
count_vm_events(UNEVICTABLE_PGRESCUED, pgrescued);
}
@@ -45,6 +45,7 @@
int page_cluster;
static DEFINE_PER_CPU(struct pagevec, lru_add_pvec);
+static DEFINE_PER_CPU(struct pagevec, lru_add_new_anon_pvec);
static DEFINE_PER_CPU(struct pagevec, lru_rotate_pvecs);
static DEFINE_PER_CPU(struct pagevec, lru_deactivate_file_pvecs);
static DEFINE_PER_CPU(struct pagevec, lru_deactivate_pvecs);
@@ -272,6 +273,23 @@ static void update_page_reclaim_stat(struct lruvec *lruvec,
reclaim_stat->recent_rotated[file]++;
}
+void update_anon_page_reclaim_stat(struct page *page)
+{
+ pg_data_t *pgdat;
+ struct lruvec *lruvec;
+ struct zone_reclaim_stat *reclaim_stat;
+
+ page = compound_head(page);
+ pgdat = page_pgdat(page);
+ spin_lock_irq(&pgdat->lru_lock);
+ if (PageLRU(page) && !PageActive(page) && !PageUnevictable(page)) {
+ lruvec = mem_cgroup_page_lruvec(page, pgdat);
+ reclaim_stat = &lruvec->reclaim_stat;
+ update_page_reclaim_stat(lruvec, 0, 1);
+ }
+ spin_unlock_irq(&pgdat->lru_lock);
+}
+
static void __activate_page(struct page *page, struct lruvec *lruvec,
void *arg)
{
@@ -333,9 +351,8 @@ void activate_page(struct page *page)
}
#endif
-static void __lru_cache_activate_page(struct page *page)
+static bool lookup_activate_page(struct pagevec *pvec, struct page *page)
{
- struct pagevec *pvec = &get_cpu_var(lru_add_pvec);
int i;
/*
@@ -352,12 +369,35 @@ static void __lru_cache_activate_page(struct page *page)
struct page *pagevec_page = pvec->pages[i];
if (pagevec_page == page) {
- SetPageActive(page);
- break;
+ return true;
}
}
+ return false;
+}
+
+static void __lru_cache_activate_page(struct page *page)
+{
+ struct pagevec *pvec;
+ bool ret;
+ pvec = &get_cpu_var(lru_add_pvec);
+ ret = lookup_activate_page(pvec, page);
put_cpu_var(lru_add_pvec);
+
+ if (ret) {
+ SetPageActive(page);
+ return;
+ }
+
+ if (!PageSwapBacked(page))
+ return;
+
+ pvec = &get_cpu_var(lru_add_new_anon_pvec);
+ ret = lookup_activate_page(pvec, page);
+ put_cpu_var(lru_add_new_anon_pvec);
+
+ if (ret)
+ SetPageActive(page);
}
/*
@@ -407,7 +447,17 @@ static void __lru_cache_add(struct page *page)
get_page(page);
if (!pagevec_add(pvec, page) || PageCompound(page))
- __pagevec_lru_add(pvec);
+ __pagevec_lru_add(pvec, false);
+ put_cpu_var(lru_add_pvec);
+}
+
+static void __lru_cache_add_new_anon(struct page *page)
+{
+ struct pagevec *pvec = &get_cpu_var(lru_add_new_anon_pvec);
+
+ get_page(page);
+ if (!pagevec_add(pvec, page) || PageCompound(page))
+ __pagevec_lru_add(pvec, true);
put_cpu_var(lru_add_pvec);
}
@@ -474,6 +524,11 @@ void lru_cache_add_inactive_or_unevictable(struct page *page,
hpage_nr_pages(page));
count_vm_event(UNEVICTABLE_PGMLOCKED);
}
+
+ if (PageSwapBacked(page) && !unevictable) {
+ __lru_cache_add_new_anon(page);
+ return;
+ }
lru_cache_add(page);
}
@@ -596,7 +651,11 @@ void lru_add_drain_cpu(int cpu)
struct pagevec *pvec = &per_cpu(lru_add_pvec, cpu);
if (pagevec_count(pvec))
- __pagevec_lru_add(pvec);
+ __pagevec_lru_add(pvec, false);
+
+ pvec = &per_cpu(lru_add_new_anon_pvec, cpu);
+ if (pagevec_count(pvec))
+ __pagevec_lru_add(pvec, true);
pvec = &per_cpu(lru_rotate_pvecs, cpu);
if (pagevec_count(pvec)) {
@@ -744,6 +803,7 @@ void lru_add_drain_all(void)
struct work_struct *work = &per_cpu(lru_add_drain_work, cpu);
if (pagevec_count(&per_cpu(lru_add_pvec, cpu)) ||
+ pagevec_count(&per_cpu(lru_add_new_anon_pvec, cpu)) ||
pagevec_count(&per_cpu(lru_rotate_pvecs, cpu)) ||
pagevec_count(&per_cpu(lru_deactivate_file_pvecs, cpu)) ||
pagevec_count(&per_cpu(lru_deactivate_pvecs, cpu)) ||
@@ -928,6 +988,7 @@ static void __pagevec_lru_add_fn(struct page *page, struct lruvec *lruvec,
{
enum lru_list lru;
int was_unevictable = TestClearPageUnevictable(page);
+ bool new_anon = (bool)arg;
VM_BUG_ON_PAGE(PageLRU(page), page);
@@ -962,8 +1023,18 @@ static void __pagevec_lru_add_fn(struct page *page, struct lruvec *lruvec,
if (page_evictable(page)) {
lru = page_lru(page);
+ /*
+ * Count new_anon page for rotation due to performance reason.
+ * Previous LRU management algorithm for anonymous page attaches
+ * a new anonymous page to the active list and it is counted as
+ * rotation. However, current one attaches a new anonymous page
+ * to the inactive list so rotation isn't counted. This count
+ * difference results in difference of file/anon reclaim ratio
+ * and then performance regression. To workaround it, new_anon
+ * is counted for rotation here.
+ */
update_page_reclaim_stat(lruvec, page_is_file_cache(page),
- PageActive(page));
+ PageActive(page) | new_anon);
if (was_unevictable)
count_vm_event(UNEVICTABLE_PGRESCUED);
} else {
@@ -982,9 +1053,9 @@ static void __pagevec_lru_add_fn(struct page *page, struct lruvec *lruvec,
* Add the passed pages to the LRU, then drop the caller's refcount
* on them. Reinitialises the caller's pagevec.
*/
-void __pagevec_lru_add(struct pagevec *pvec)
+void __pagevec_lru_add(struct pagevec *pvec, bool new_anon)
{
- pagevec_lru_move_fn(pvec, __pagevec_lru_add_fn, NULL);
+ pagevec_lru_move_fn(pvec, __pagevec_lru_add_fn, (void *)new_anon);
}
EXPORT_SYMBOL(__pagevec_lru_add);
@@ -1088,11 +1088,13 @@ static unsigned long shrink_page_list(struct list_head *page_list,
enum page_references references = PAGEREF_RECLAIM;
bool dirty, writeback;
unsigned int nr_pages;
+ int file;
cond_resched();
page = lru_to_page(page_list);
list_del(&page->lru);
+ file = page_is_file_cache(page);
if (!trylock_page(page))
goto keep;
@@ -1223,7 +1225,7 @@ static unsigned long shrink_page_list(struct list_head *page_list,
case PAGEREF_ACTIVATE:
goto activate_locked;
case PAGEREF_KEEP:
- stat->nr_ref_keep += nr_pages;
+ stat->nr_ref_keep[file] += nr_pages;
goto keep_locked;
case PAGEREF_RECLAIM:
case PAGEREF_RECLAIM_CLEAN:
@@ -1316,7 +1318,7 @@ static unsigned long shrink_page_list(struct list_head *page_list,
* the rest of the LRU for clean pages and see
* the same dirty pages again (PageReclaim).
*/
- if (page_is_file_cache(page) &&
+ if (file &&
(!current_is_kswapd() || !PageReclaim(page) ||
!test_bit(PGDAT_DIRTY, &pgdat->flags))) {
/*
@@ -1460,9 +1462,8 @@ static unsigned long shrink_page_list(struct list_head *page_list,
try_to_free_swap(page);
VM_BUG_ON_PAGE(PageActive(page), page);
if (!PageMlocked(page)) {
- int type = page_is_file_cache(page);
SetPageActive(page);
- stat->nr_activate[type] += nr_pages;
+ stat->nr_activate[file] += nr_pages;
count_memcg_page_event(page, PGACTIVATE);
workingset_activation(page);
}
@@ -1955,6 +1956,9 @@ shrink_inactive_list(unsigned long nr_to_scan, struct lruvec *lruvec,
reclaim_stat->recent_rotated[0] += stat.nr_activate[0];
reclaim_stat->recent_rotated[1] += stat.nr_activate[1];
+ /* to mitigate impact on scan ratio due to LRU algorithm change */
+ reclaim_stat->recent_rotated[0] += stat.nr_ref_keep[0];
+
move_pages_to_lru(lruvec, &page_list);
__mod_node_page_state(pgdat, NR_ISOLATED_ANON + file, -nr_taken);