| Message ID | 20250103172419.4148674-6-ziy@nvidia.com (mailing list archive) |
|---|---|
| State | New |
| Headers | show |
| Series | Accelerate page migration with batching and multi threads | expand |
On Fri, Jan 03, 2025 at 12:24:19PM -0500, Zi Yan wrote: ... snip ... > + { > + .procname = "use_mt_copy", > + .data = &use_mt_copy, > + .maxlen = sizeof(use_mt_copy), > + .mode = 0644, > + .proc_handler = proc_dointvec_minmax, > + .extra1 = SYSCTL_ZERO, > + .extra2 = SYSCTL_ONE, > + }, > + { > + .procname = "limit_mt_num", > + .data = &limit_mt_num, > + .maxlen = sizeof(limit_mt_num), > + .mode = 0644, > + .proc_handler = proc_dointvec_minmax, > + .extra1 = SYSCTL_ONE, > + .extra2 = SYSCTL_32, > + }, > + { > + .procname = "push_0_pull_1", > + .data = &push_0_pull_1, > + .maxlen = sizeof(push_0_pull_1), > + .mode = 0644, > + .proc_handler = proc_dointvec_minmax, > + .extra1 = SYSCTL_ZERO, > + .extra2 = SYSCTL_ONE, > + }, > { > .procname = "drop_caches", > .data = &sysctl_drop_caches, Build errors here ~Gregory --- diff --git a/kernel/sysctl.c b/kernel/sysctl.c index f9ba48cd6e09..bca82e6132b3 100644 --- a/kernel/sysctl.c +++ b/kernel/sysctl.c @@ -2093,8 +2093,8 @@ static struct ctl_table vm_table[] = { #endif { .procname = "use_mt_copy", - .data = &use_mt_copy, - .maxlen = sizeof(use_mt_copy), + .data = &sysctl_use_mt_copy, + .maxlen = sizeof(sysctl_use_mt_copy), .mode = 0644, .proc_handler = proc_dointvec_minmax, .extra1 = SYSCTL_ZERO, @@ -2102,8 +2102,8 @@ static struct ctl_table vm_table[] = { }, { .procname = "limit_mt_num", - .data = &limit_mt_num, - .maxlen = sizeof(limit_mt_num), + .data = &sysctl_limit_mt_num, + .maxlen = sizeof(sysctl_limit_mt_num), .mode = 0644, .proc_handler = proc_dointvec_minmax, .extra1 = SYSCTL_ONE, @@ -2111,8 +2111,8 @@ static struct ctl_table vm_table[] = { }, { .procname = "push_0_pull_1", - .data = &push_0_pull_1, - .maxlen = sizeof(push_0_pull_1), + .data = &sysctl_push_0_pull_1, + .maxlen = sizeof(sysctl_push_0_pull_1), .mode = 0644, .proc_handler = proc_dointvec_minmax, .extra1 = SYSCTL_ZERO,
On 3 Jan 2025, at 17:21, Gregory Price wrote: > On Fri, Jan 03, 2025 at 12:24:19PM -0500, Zi Yan wrote: > ... snip ... >> + { >> + .procname = "use_mt_copy", >> + .data = &use_mt_copy, >> + .maxlen = sizeof(use_mt_copy), >> + .mode = 0644, >> + .proc_handler = proc_dointvec_minmax, >> + .extra1 = SYSCTL_ZERO, >> + .extra2 = SYSCTL_ONE, >> + }, >> + { >> + .procname = "limit_mt_num", >> + .data = &limit_mt_num, >> + .maxlen = sizeof(limit_mt_num), >> + .mode = 0644, >> + .proc_handler = proc_dointvec_minmax, >> + .extra1 = SYSCTL_ONE, >> + .extra2 = SYSCTL_32, >> + }, >> + { >> + .procname = "push_0_pull_1", >> + .data = &push_0_pull_1, >> + .maxlen = sizeof(push_0_pull_1), >> + .mode = 0644, >> + .proc_handler = proc_dointvec_minmax, >> + .extra1 = SYSCTL_ZERO, >> + .extra2 = SYSCTL_ONE, >> + }, >> { >> .procname = "drop_caches", >> .data = &sysctl_drop_caches, > > Build errors here Thanks, these changes must be lost during my patch clean time. > > ~Gregory > > --- > > diff --git a/kernel/sysctl.c b/kernel/sysctl.c > index f9ba48cd6e09..bca82e6132b3 100644 > --- a/kernel/sysctl.c > +++ b/kernel/sysctl.c > @@ -2093,8 +2093,8 @@ static struct ctl_table vm_table[] = { > #endif > { > .procname = "use_mt_copy", > - .data = &use_mt_copy, > - .maxlen = sizeof(use_mt_copy), > + .data = &sysctl_use_mt_copy, > + .maxlen = sizeof(sysctl_use_mt_copy), > .mode = 0644, > .proc_handler = proc_dointvec_minmax, > .extra1 = SYSCTL_ZERO, > @@ -2102,8 +2102,8 @@ static struct ctl_table vm_table[] = { > }, > { > .procname = "limit_mt_num", > - .data = &limit_mt_num, > - .maxlen = sizeof(limit_mt_num), > + .data = &sysctl_limit_mt_num, > + .maxlen = sizeof(sysctl_limit_mt_num), > .mode = 0644, > .proc_handler = proc_dointvec_minmax, > .extra1 = SYSCTL_ONE, > @@ -2111,8 +2111,8 @@ static struct ctl_table vm_table[] = { > }, > { > .procname = "push_0_pull_1", > - .data = &push_0_pull_1, > - .maxlen = sizeof(push_0_pull_1), > + .data = &sysctl_push_0_pull_1, > + .maxlen = sizeof(sysctl_push_0_pull_1), > .mode = 0644, > .proc_handler = proc_dointvec_minmax, > .extra1 = SYSCTL_ZERO, Best Regards, Yan, Zi
diff --git a/include/linux/mm.h b/include/linux/mm.h index 1a11f9df5c2d..277b12b9ef0d 100644 --- a/include/linux/mm.h +++ b/include/linux/mm.h @@ -42,6 +42,10 @@ struct pt_regs; struct folio_batch; extern int sysctl_page_lock_unfairness; +extern int sysctl_use_mt_copy; +extern unsigned int sysctl_limit_mt_num; +extern unsigned int sysctl_push_0_pull_1; + void mm_core_init(void); void init_mm_internals(void); diff --git a/include/linux/sysctl.h b/include/linux/sysctl.h index 40a6ac6c9713..f33dafea2533 100644 --- a/include/linux/sysctl.h +++ b/include/linux/sysctl.h @@ -52,6 +52,7 @@ struct ctl_dir; /* this is needed for the proc_dointvec_minmax for [fs_]overflow UID and GID */ #define SYSCTL_MAXOLDUID ((void *)&sysctl_vals[10]) #define SYSCTL_NEG_ONE ((void *)&sysctl_vals[11]) +#define SYSCTL_32 ((void *)&sysctl_vals[12]) extern const int sysctl_vals[]; diff --git a/kernel/sysctl.c b/kernel/sysctl.c index 5c9202cb8f59..f9ba48cd6e09 100644 --- a/kernel/sysctl.c +++ b/kernel/sysctl.c @@ -82,7 +82,7 @@ #endif /* shared constants to be used in various sysctls */ -const int sysctl_vals[] = { 0, 1, 2, 3, 4, 100, 200, 1000, 3000, INT_MAX, 65535, -1 }; +const int sysctl_vals[] = { 0, 1, 2, 3, 4, 100, 200, 1000, 3000, INT_MAX, 65535, -1, 32 }; EXPORT_SYMBOL(sysctl_vals); const unsigned long sysctl_long_vals[] = { 0, 1, LONG_MAX }; @@ -2091,6 +2091,33 @@ static struct ctl_table vm_table[] = { .extra2 = SYSCTL_ONE, }, #endif + { + .procname = "use_mt_copy", + .data = &use_mt_copy, + .maxlen = sizeof(use_mt_copy), + .mode = 0644, + .proc_handler = proc_dointvec_minmax, + .extra1 = SYSCTL_ZERO, + .extra2 = SYSCTL_ONE, + }, + { + .procname = "limit_mt_num", + .data = &limit_mt_num, + .maxlen = sizeof(limit_mt_num), + .mode = 0644, + .proc_handler = proc_dointvec_minmax, + .extra1 = SYSCTL_ONE, + .extra2 = SYSCTL_32, + }, + { + .procname = "push_0_pull_1", + .data = &push_0_pull_1, + .maxlen = sizeof(push_0_pull_1), + .mode = 0644, + .proc_handler = proc_dointvec_minmax, + .extra1 = SYSCTL_ZERO, + .extra2 = SYSCTL_ONE, + }, { .procname = "drop_caches", .data = &sysctl_drop_caches, diff --git a/mm/copy_pages.c b/mm/copy_pages.c index 0e2231199f66..257034550c86 100644 --- a/mm/copy_pages.c +++ b/mm/copy_pages.c @@ -10,7 +10,9 @@ #include <linux/migrate.h> -unsigned int limit_mt_num = 4; +unsigned int sysctl_limit_mt_num = 4; +/* push by default */ +unsigned int sysctl_push_0_pull_1; struct copy_item { char *to; @@ -45,11 +47,13 @@ int copy_page_lists_mt(struct list_head *dst_folios, struct list_head *src_folios, int nr_items) { int err = 0; - unsigned int total_mt_num = limit_mt_num; + unsigned int total_mt_num = sysctl_limit_mt_num; int to_node = folio_nid(list_first_entry(dst_folios, struct folio, lru)); + int from_node = folio_nid(list_first_entry(src_folios, struct folio, lru)); int i; struct copy_page_info *work_items[32] = {0}; - const struct cpumask *per_node_cpumask = cpumask_of_node(to_node); + const struct cpumask *per_node_cpumask = + cpumask_of_node(sysctl_push_0_pull_1 ? to_node : from_node); int cpu_id_list[32] = {0}; int cpu; int max_items_per_thread; diff --git a/mm/migrate.c b/mm/migrate.c index 18440180d747..0f7a4b09acda 100644 --- a/mm/migrate.c +++ b/mm/migrate.c @@ -51,6 +51,7 @@ #include "internal.h" +int sysctl_use_mt_copy; bool isolate_movable_page(struct page *page, isolate_mode_t mode) { @@ -1621,7 +1622,7 @@ static inline int try_split_folio(struct folio *folio, struct list_head *split_f } #ifdef CONFIG_TRANSPARENT_HUGEPAGE -#define NR_MAX_BATCHED_MIGRATION HPAGE_PMD_NR +#define NR_MAX_BATCHED_MIGRATION HPAGE_PUD_NR #else #define NR_MAX_BATCHED_MIGRATION 512 #endif @@ -1868,7 +1869,8 @@ static void migrate_folios_batch_move(struct list_head *src_folios, goto out; /* Batch copy the folios */ - if (total_nr_pages > 32) { + /* if (total_nr_pages > 32) { */ + if (sysctl_use_mt_copy) { copy_page_lists_mt(dst_folios, src_folios, total_nr_folios); } else { dst = list_first_entry(dst_folios, struct folio, lru);
1. enable multi-threaded copy 2. specify how many CPU threads to use 3. push from local CPUs or pull from remote CPUs 4. change NR_MAX_BATCHED_MIGRATION to HPAGE_PUD_NR to allow batching THP copies. These are for testing purpose only. Signed-off-by: Zi Yan <ziy@nvidia.com> --- include/linux/mm.h | 4 ++++ include/linux/sysctl.h | 1 + kernel/sysctl.c | 29 ++++++++++++++++++++++++++++- mm/copy_pages.c | 10 +++++++--- mm/migrate.c | 6 ++++-- 5 files changed, 44 insertions(+), 6 deletions(-)