@@ -5899,66 +5899,6 @@ EXPORT_SYMBOL(__might_fault);
#endif
#if defined(CONFIG_TRANSPARENT_HUGEPAGE) || defined(CONFIG_HUGETLBFS)
-/*
- * Process all subpages of the specified huge page with the specified
- * operation. The target subpage will be processed last to keep its
- * cache lines hot.
- */
-static inline int process_huge_page(
- unsigned long addr_hint, unsigned int pages_per_huge_page,
- int (*process_subpage)(unsigned long addr, int idx, void *arg),
- void *arg)
-{
- int i, n, base, l, ret;
- unsigned long addr = addr_hint &
- ~(((unsigned long)pages_per_huge_page << PAGE_SHIFT) - 1);
-
- /* Process target subpage last to keep its cache lines hot */
- might_sleep();
- n = (addr_hint - addr) / PAGE_SIZE;
- if (2 * n <= pages_per_huge_page) {
- /* If target subpage in first half of huge page */
- base = 0;
- l = n;
- /* Process subpages at the end of huge page */
- for (i = pages_per_huge_page - 1; i >= 2 * n; i--) {
- cond_resched();
- ret = process_subpage(addr + i * PAGE_SIZE, i, arg);
- if (ret)
- return ret;
- }
- } else {
- /* If target subpage in second half of huge page */
- base = pages_per_huge_page - 2 * (pages_per_huge_page - n);
- l = pages_per_huge_page - n;
- /* Process subpages at the begin of huge page */
- for (i = 0; i < base; i++) {
- cond_resched();
- ret = process_subpage(addr + i * PAGE_SIZE, i, arg);
- if (ret)
- return ret;
- }
- }
- /*
- * Process remaining subpages in left-right-left-right pattern
- * towards the target subpage
- */
- for (i = 0; i < l; i++) {
- int left_idx = base + i;
- int right_idx = base + 2 * l - 1 - i;
-
- cond_resched();
- ret = process_subpage(addr + left_idx * PAGE_SIZE, left_idx, arg);
- if (ret)
- return ret;
- cond_resched();
- ret = process_subpage(addr + right_idx * PAGE_SIZE, right_idx, arg);
- if (ret)
- return ret;
- }
- return 0;
-}
-
static void clear_gigantic_page(struct page *page,
unsigned long addr,
unsigned int pages_per_huge_page)
@@ -5982,6 +5922,65 @@ static int clear_subpage(unsigned long addr, int idx, void *arg)
return 0;
}
+/*
+ * Clear subpages of the specified huge page. The target subpage will be
+ * processed last to keep its cache lines hot.
+ */
+static int __clear_huge_page(
+ unsigned long addr_hint, unsigned int pages_per_huge_page,
+ int (*process_subpage)(unsigned long addr, int idx, void *arg),
+ void *arg)
+{
+ int i, n, base, l, ret;
+ unsigned long addr = addr_hint &
+ ~(((unsigned long)pages_per_huge_page << PAGE_SHIFT) - 1);
+
+ /* Process target subpage last to keep its cache lines hot */
+ might_sleep();
+ n = (addr_hint - addr) / PAGE_SIZE;
+ if (2 * n <= pages_per_huge_page) {
+ /* If target subpage in first half of huge page */
+ base = 0;
+ l = n;
+ /* Process subpages at the end of huge page */
+ for (i = pages_per_huge_page - 1; i >= 2 * n; i--) {
+ cond_resched();
+ ret = process_subpage(addr + i * PAGE_SIZE, i, arg);
+ if (ret)
+ return ret;
+ }
+ } else {
+ /* If target subpage in second half of huge page */
+ base = pages_per_huge_page - 2 * (pages_per_huge_page - n);
+ l = pages_per_huge_page - n;
+ /* Process subpages at the begin of huge page */
+ for (i = 0; i < base; i++) {
+ cond_resched();
+ ret = process_subpage(addr + i * PAGE_SIZE, i, arg);
+ if (ret)
+ return ret;
+ }
+ }
+ /*
+ * Process remaining subpages in left-right-left-right pattern
+ * towards the target subpage
+ */
+ for (i = 0; i < l; i++) {
+ int left_idx = base + i;
+ int right_idx = base + 2 * l - 1 - i;
+
+ cond_resched();
+ ret = process_subpage(addr + left_idx * PAGE_SIZE, left_idx, arg);
+ if (ret)
+ return ret;
+ cond_resched();
+ ret = process_subpage(addr + right_idx * PAGE_SIZE, right_idx, arg);
+ if (ret)
+ return ret;
+ }
+ return 0;
+}
+
__weak void clear_huge_page(struct page *page,
unsigned long addr_hint,
unsigned int pages_per_huge_page)
@@ -5994,7 +5993,7 @@ __weak void clear_huge_page(struct page *page,
return;
}
- process_huge_page(addr_hint, pages_per_huge_page, clear_subpage, page);
+ __clear_huge_page(addr_hint, pages_per_huge_page, clear_subpage, page);
}
static int copy_user_gigantic_page(struct folio *dst, struct folio *src,
@@ -6038,6 +6037,65 @@ static int copy_subpage(unsigned long addr, int idx, void *arg)
return 0;
}
+/*
+ * Copy subpages of the specified huge page. The target subpage will be
+ * processed last to keep its cache lines hot.
+ */
+static int __copy_huge_page(
+ unsigned long addr_hint, unsigned int pages_per_huge_page,
+ int (*process_subpage)(unsigned long addr, int idx, void *arg),
+ void *arg)
+{
+ int i, n, base, l, ret;
+ unsigned long addr = addr_hint &
+ ~(((unsigned long)pages_per_huge_page << PAGE_SHIFT) - 1);
+
+ /* Process target subpage last to keep its cache lines hot */
+ might_sleep();
+ n = (addr_hint - addr) / PAGE_SIZE;
+ if (2 * n <= pages_per_huge_page) {
+ /* If target subpage in first half of huge page */
+ base = 0;
+ l = n;
+ /* Process subpages at the end of huge page */
+ for (i = pages_per_huge_page - 1; i >= 2 * n; i--) {
+ cond_resched();
+ ret = process_subpage(addr + i * PAGE_SIZE, i, arg);
+ if (ret)
+ return ret;
+ }
+ } else {
+ /* If target subpage in second half of huge page */
+ base = pages_per_huge_page - 2 * (pages_per_huge_page - n);
+ l = pages_per_huge_page - n;
+ /* Process subpages at the begin of huge page */
+ for (i = 0; i < base; i++) {
+ cond_resched();
+ ret = process_subpage(addr + i * PAGE_SIZE, i, arg);
+ if (ret)
+ return ret;
+ }
+ }
+ /*
+ * Process remaining subpages in left-right-left-right pattern
+ * towards the target subpage
+ */
+ for (i = 0; i < l; i++) {
+ int left_idx = base + i;
+ int right_idx = base + 2 * l - 1 - i;
+
+ cond_resched();
+ ret = process_subpage(addr + left_idx * PAGE_SIZE, left_idx, arg);
+ if (ret)
+ return ret;
+ cond_resched();
+ ret = process_subpage(addr + right_idx * PAGE_SIZE, right_idx, arg);
+ if (ret)
+ return ret;
+ }
+ return 0;
+}
+
int copy_user_large_folio(struct folio *dst, struct folio *src,
unsigned long addr_hint, struct vm_area_struct *vma)
{
@@ -6054,7 +6112,7 @@ int copy_user_large_folio(struct folio *dst, struct folio *src,
return copy_user_gigantic_page(dst, src, addr, vma,
pages_per_huge_page);
- return process_huge_page(addr_hint, pages_per_huge_page, copy_subpage, &arg);
+ return __copy_huge_page(addr_hint, pages_per_huge_page, copy_subpage, &arg);
}
long copy_folio_from_user(struct folio *dst_folio,
Clearing and copying of huge-pages is done via process_huge_page() which subsequently calls process_subpage(). This preserves the structural similarities while processing the pages but needs some ugly casting and even uglier indexing: for instance in the call for process_subpage(), we pass the indexed address, along with the index so the handler can also do the indexing. Additionally, both of these paths have diverged since this code was written: for instance clear_subpage() cannot fail but copy_subpage() can fail in copying. Note that there's no runtime cost in having this code be common, as most current compilers inline process_huge_page() (though some older, supported ones do not), but it's unnecessary complexity for something that only has two users. Accordingly, fold process_huge_page() back in its callers. Suggested-by: Linus Torvalds <torvalds@linux-foundation.org> Signed-off-by: Ankur Arora <ankur.a.arora@oracle.com> --- mm/memory.c | 182 ++++++++++++++++++++++++++++++++++------------------ 1 file changed, 120 insertions(+), 62 deletions(-)