[v2,08/12] mm/khugepaged: add flag to ignore page young/referenced requirement

Commit Message

Zach O'Keefe April 14, 2022, 6:06 p.m. UTC

Add enforce_young flag to struct collapse_control that allows context to
ignore requirement that some pages in region being collapsed be young or
referenced.  Set this flag in khugepaged collapse context to preserve
existing khugepaged behavior and unset the flag in madvise collapse
context since the user presumably has reason to believe the collapse
will be beneficial.

Signed-off-by: Zach O'Keefe <[e-mail]>
---
 mm/khugepaged.c | 24 ++++++++++++++++--------
 1 file changed, 16 insertions(+), 8 deletions(-)

Patch

diff --git a/mm/khugepaged.c b/mm/khugepaged.c
index 2f95f60431aa..b9bf15faba26 100644
--- a/mm/khugepaged.c
+++ b/mm/khugepaged.c
@@ -90,6 +90,9 @@  struct collapse_control {
 	/* Respect khugepaged_max_ptes_[none|swap|shared] */
 	bool enforce_pte_scan_limits;
 
+	/* Require memory to be young */
+	bool enforce_young;
+
 	/* Num pages scanned per node */
 	int node_load[MAX_NUMNODES];
 
@@ -737,9 +740,10 @@  static int __collapse_huge_page_isolate(struct vm_area_struct *vma,
 			list_add_tail(&page->lru, compound_pagelist);
 next:
 		/* There should be enough young pte to collapse the page */
-		if (pte_young(pteval) ||
-		    page_is_young(page) || PageReferenced(page) ||
-		    mmu_notifier_test_young(vma->vm_mm, address))
+		if (cc->enforce_young &&
+		    (pte_young(pteval) || page_is_young(page) ||
+		     PageReferenced(page) || mmu_notifier_test_young(vma->vm_mm,
+								     address)))
 			referenced++;
 
 		if (pte_write(pteval))
@@ -748,7 +752,7 @@  static int __collapse_huge_page_isolate(struct vm_area_struct *vma,
 
 	if (unlikely(!writable)) {
 		result = SCAN_PAGE_RO;
-	} else if (unlikely(!referenced)) {
+	} else if (unlikely(cc->enforce_young && !referenced)) {
 		result = SCAN_LACK_REFERENCED_PAGE;
 	} else {
 		result = SCAN_SUCCEED;
@@ -1408,14 +1412,16 @@  static void scan_pmd(struct mm_struct *mm, struct vm_area_struct *vma,
 			cr->result = SCAN_PAGE_COUNT;
 			goto out_unmap;
 		}
-		if (pte_young(pteval) ||
-		    page_is_young(page) || PageReferenced(page) ||
-		    mmu_notifier_test_young(vma->vm_mm, address))
+		if (cc->enforce_young &&
+		    (pte_young(pteval) || page_is_young(page) ||
+		     PageReferenced(page) || mmu_notifier_test_young(vma->vm_mm,
+								     address)))
 			referenced++;
 	}
 	if (!writable) {
 		cr->result = SCAN_PAGE_RO;
-	} else if (!referenced || (unmapped && referenced < HPAGE_PMD_NR/2)) {
+	} else if (cc->enforce_young && (!referenced || (unmapped && referenced
+							 < HPAGE_PMD_NR / 2))) {
 		cr->result = SCAN_LACK_REFERENCED_PAGE;
 	} else {
 		cr->result = SCAN_SUCCEED;
@@ -2375,6 +2381,7 @@  static int khugepaged(void *none)
 	struct mm_slot *mm_slot;
 	struct collapse_control cc = {
 		.enforce_pte_scan_limits = true,
+		.enforce_young = true,
 		.last_target_node = NUMA_NO_NODE,
 		.alloc_hpage = &khugepaged_alloc_page,
 	};
@@ -2517,6 +2524,7 @@  int madvise_collapse(struct vm_area_struct *vma, struct vm_area_struct **prev,
 {
 	struct collapse_control cc = {
 		.enforce_pte_scan_limits = false,
+		.enforce_young = false,
 		.last_target_node = NUMA_NO_NODE,
 		.hpage = NULL,
 		.alloc_hpage = &alloc_hpage,