| Message ID | 20240722172917.503370-2-ziy@nvidia.com (mailing list archive) |
|---|---|
| State | New |
| Headers | show |
| Series | [v2,1/3] memory tiering: read last_cpupid correctly in do_huge_pmd_numa_page() | expand |
On Mon, Jul 22, 2024 at 01:29:16PM GMT, Zi Yan wrote: > Instead of open coded check for if memory tiering mode is on and a folio > is in the top tier memory, use a function to encapsulate the check. > > Signed-off-by: Zi Yan <ziy@nvidia.com> > Reviewed-by: "Huang, Ying" <ying.huang@intel.com> > --- > include/linux/mm.h | 6 ++++++ > kernel/sched/fair.c | 3 +-- > mm/huge_memory.c | 6 ++---- > mm/memory-tiers.c | 17 +++++++++++++++++ > mm/memory.c | 3 +-- > mm/mprotect.c | 3 +-- > 6 files changed, 28 insertions(+), 10 deletions(-) > > diff --git a/include/linux/mm.h b/include/linux/mm.h > index c227f22ba810..048b2a56d8a3 100644 > --- a/include/linux/mm.h > +++ b/include/linux/mm.h > @@ -1738,6 +1738,8 @@ static inline void vma_set_access_pid_bit(struct vm_area_struct *vma) > __set_bit(pid_bit, &vma->numab_state->pids_active[1]); > } > } > + > +bool folio_has_cpupid(struct folio *folio); > #else /* !CONFIG_NUMA_BALANCING */ > static inline int folio_xchg_last_cpupid(struct folio *folio, int cpupid) > { > @@ -1791,6 +1793,10 @@ static inline bool cpupid_match_pid(struct task_struct *task, int cpupid) > static inline void vma_set_access_pid_bit(struct vm_area_struct *vma) > { > } > +static inline bool folio_has_cpupid(struct folio *folio) > +{ > + return true; > +} > #endif /* CONFIG_NUMA_BALANCING */ > > #if defined(CONFIG_KASAN_SW_TAGS) || defined(CONFIG_KASAN_HW_TAGS) > diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c > index 8a5b1ae0aa55..03de808cb3cc 100644 > --- a/kernel/sched/fair.c > +++ b/kernel/sched/fair.c > @@ -1840,8 +1840,7 @@ bool should_numa_migrate_memory(struct task_struct *p, struct folio *folio, > * The pages in slow memory node should be migrated according > * to hot/cold instead of private/shared. > */ > - if (sysctl_numa_balancing_mode & NUMA_BALANCING_MEMORY_TIERING && > - !node_is_toptier(src_nid)) { > + if (!folio_has_cpupid(folio)) { > struct pglist_data *pgdat; > unsigned long rate_limit; > unsigned int latency, th, def_th; > diff --git a/mm/huge_memory.c b/mm/huge_memory.c > index 825317aee88e..d925a93bb9ed 100644 > --- a/mm/huge_memory.c > +++ b/mm/huge_memory.c > @@ -1712,8 +1712,7 @@ vm_fault_t do_huge_pmd_numa_page(struct vm_fault *vmf) > * For memory tiering mode, cpupid of slow memory page is used > * to record page access time. So use default value. > */ > - if (!(sysctl_numa_balancing_mode & NUMA_BALANCING_MEMORY_TIERING) || > - node_is_toptier(nid)) > + if (folio_has_cpupid(folio)) > last_cpupid = folio_last_cpupid(folio); > target_nid = numa_migrate_prep(folio, vmf, haddr, nid, &flags); > if (target_nid == NUMA_NO_NODE) > @@ -2066,8 +2065,7 @@ int change_huge_pmd(struct mmu_gather *tlb, struct vm_area_struct *vma, > toptier) > goto unlock; > > - if (sysctl_numa_balancing_mode & NUMA_BALANCING_MEMORY_TIERING && > - !toptier) > + if (!folio_has_cpupid(folio)) > folio_xchg_access_time(folio, > jiffies_to_msecs(jiffies)); > } > diff --git a/mm/memory-tiers.c b/mm/memory-tiers.c > index 4775b3a3dabe..7f0360d4e3a0 100644 > --- a/mm/memory-tiers.c > +++ b/mm/memory-tiers.c > @@ -6,6 +6,7 @@ > #include <linux/memory.h> > #include <linux/memory-tiers.h> > #include <linux/notifier.h> > +#include <linux/sched/sysctl.h> > > #include "internal.h" > > @@ -50,6 +51,22 @@ static const struct bus_type memory_tier_subsys = { > .dev_name = "memory_tier", > }; > > +/** > + * folio_has_cpupid - check if a folio has cpupid information > + * @folio: folio to check > + * > + * folio's _last_cpupid field is repurposed by memory tiering. In memory > + * tiering mode, cpupid of slow memory folio (not toptier memory) is used to > + * record page access time. > + * > + * Return: the folio _last_cpupid is used as cpupid > + */ > +bool folio_has_cpupid(struct folio *folio) > +{ > + return !(sysctl_numa_balancing_mode & NUMA_BALANCING_MEMORY_TIERING) || > + node_is_toptier(folio_nid(folio)); > +} > + The static version of folio_has_cpupid() is defined in include/linux/mm.h if !CONFIG_NUMA_BALANCING but you define the function in memory-tiers.c unconditionally, a file that is compiled predicated on CONFIG_NUMA. So a config with !CONFIG_NUMA_BALANCING but CONFIG_NUMA set results in a compilation error (I just hit it this morning in mm-unstable). A minimal fix for this is to wrap the declaration in: #ifdef CONFIG_NUMA_BALANCING ... #endif I've tried this locally and it resolves the issue. > #ifdef CONFIG_MIGRATION > static int top_tier_adistance; > /* > diff --git a/mm/memory.c b/mm/memory.c > index 802d0d8a40f9..105e1a0157dd 100644 > --- a/mm/memory.c > +++ b/mm/memory.c > @@ -5337,8 +5337,7 @@ static vm_fault_t do_numa_page(struct vm_fault *vmf) > * For memory tiering mode, cpupid of slow memory page is used > * to record page access time. So use default value. > */ > - if ((sysctl_numa_balancing_mode & NUMA_BALANCING_MEMORY_TIERING) && > - !node_is_toptier(nid)) > + if (!folio_has_cpupid(folio)) > last_cpupid = (-1 & LAST_CPUPID_MASK); > else > last_cpupid = folio_last_cpupid(folio); > diff --git a/mm/mprotect.c b/mm/mprotect.c > index 222ab434da54..787c3c2bf1b6 100644 > --- a/mm/mprotect.c > +++ b/mm/mprotect.c > @@ -161,8 +161,7 @@ static long change_pte_range(struct mmu_gather *tlb, > if (!(sysctl_numa_balancing_mode & NUMA_BALANCING_NORMAL) && > toptier) > continue; > - if (sysctl_numa_balancing_mode & NUMA_BALANCING_MEMORY_TIERING && > - !toptier) > + if (!folio_has_cpupid(folio)) > folio_xchg_access_time(folio, > jiffies_to_msecs(jiffies)); > } > -- > 2.43.0 >
> > The static version of folio_has_cpupid() is defined in include/linux/mm.h > if !CONFIG_NUMA_BALANCING but you define the function in memory-tiers.c > unconditionally, a file that is compiled predicated on CONFIG_NUMA. > > So a config with !CONFIG_NUMA_BALANCING but CONFIG_NUMA set results in a > compilation error (I just hit it this morning in mm-unstable). > > A minimal fix for this is to wrap the declaration in: > > #ifdef CONFIG_NUMA_BALANCING > ... > #endif > > I've tried this locally and it resolves the issue. Agreed, with that Acked-by: David Hildenbrand <david@redhat.com>
On Tue Jul 23, 2024 at 6:14 AM EDT, David Hildenbrand wrote: > > > > The static version of folio_has_cpupid() is defined in include/linux/mm.h > > if !CONFIG_NUMA_BALANCING but you define the function in memory-tiers.c > > unconditionally, a file that is compiled predicated on CONFIG_NUMA. > > > > So a config with !CONFIG_NUMA_BALANCING but CONFIG_NUMA set results in a > > compilation error (I just hit it this morning in mm-unstable). > > > > A minimal fix for this is to wrap the declaration in: > > > > #ifdef CONFIG_NUMA_BALANCING > > ... > > #endif > > > > I've tried this locally and it resolves the issue. Will fix it. Thanks. > > Agreed, with that > > Acked-by: David Hildenbrand <david@redhat.com> Thanks.
diff --git a/include/linux/mm.h b/include/linux/mm.h index c227f22ba810..048b2a56d8a3 100644 --- a/include/linux/mm.h +++ b/include/linux/mm.h @@ -1738,6 +1738,8 @@ static inline void vma_set_access_pid_bit(struct vm_area_struct *vma) __set_bit(pid_bit, &vma->numab_state->pids_active[1]); } } + +bool folio_has_cpupid(struct folio *folio); #else /* !CONFIG_NUMA_BALANCING */ static inline int folio_xchg_last_cpupid(struct folio *folio, int cpupid) { @@ -1791,6 +1793,10 @@ static inline bool cpupid_match_pid(struct task_struct *task, int cpupid) static inline void vma_set_access_pid_bit(struct vm_area_struct *vma) { } +static inline bool folio_has_cpupid(struct folio *folio) +{ + return true; +} #endif /* CONFIG_NUMA_BALANCING */ #if defined(CONFIG_KASAN_SW_TAGS) || defined(CONFIG_KASAN_HW_TAGS) diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c index 8a5b1ae0aa55..03de808cb3cc 100644 --- a/kernel/sched/fair.c +++ b/kernel/sched/fair.c @@ -1840,8 +1840,7 @@ bool should_numa_migrate_memory(struct task_struct *p, struct folio *folio, * The pages in slow memory node should be migrated according * to hot/cold instead of private/shared. */ - if (sysctl_numa_balancing_mode & NUMA_BALANCING_MEMORY_TIERING && - !node_is_toptier(src_nid)) { + if (!folio_has_cpupid(folio)) { struct pglist_data *pgdat; unsigned long rate_limit; unsigned int latency, th, def_th; diff --git a/mm/huge_memory.c b/mm/huge_memory.c index 825317aee88e..d925a93bb9ed 100644 --- a/mm/huge_memory.c +++ b/mm/huge_memory.c @@ -1712,8 +1712,7 @@ vm_fault_t do_huge_pmd_numa_page(struct vm_fault *vmf) * For memory tiering mode, cpupid of slow memory page is used * to record page access time. So use default value. */ - if (!(sysctl_numa_balancing_mode & NUMA_BALANCING_MEMORY_TIERING) || - node_is_toptier(nid)) + if (folio_has_cpupid(folio)) last_cpupid = folio_last_cpupid(folio); target_nid = numa_migrate_prep(folio, vmf, haddr, nid, &flags); if (target_nid == NUMA_NO_NODE) @@ -2066,8 +2065,7 @@ int change_huge_pmd(struct mmu_gather *tlb, struct vm_area_struct *vma, toptier) goto unlock; - if (sysctl_numa_balancing_mode & NUMA_BALANCING_MEMORY_TIERING && - !toptier) + if (!folio_has_cpupid(folio)) folio_xchg_access_time(folio, jiffies_to_msecs(jiffies)); } diff --git a/mm/memory-tiers.c b/mm/memory-tiers.c index 4775b3a3dabe..7f0360d4e3a0 100644 --- a/mm/memory-tiers.c +++ b/mm/memory-tiers.c @@ -6,6 +6,7 @@ #include <linux/memory.h> #include <linux/memory-tiers.h> #include <linux/notifier.h> +#include <linux/sched/sysctl.h> #include "internal.h" @@ -50,6 +51,22 @@ static const struct bus_type memory_tier_subsys = { .dev_name = "memory_tier", }; +/** + * folio_has_cpupid - check if a folio has cpupid information + * @folio: folio to check + * + * folio's _last_cpupid field is repurposed by memory tiering. In memory + * tiering mode, cpupid of slow memory folio (not toptier memory) is used to + * record page access time. + * + * Return: the folio _last_cpupid is used as cpupid + */ +bool folio_has_cpupid(struct folio *folio) +{ + return !(sysctl_numa_balancing_mode & NUMA_BALANCING_MEMORY_TIERING) || + node_is_toptier(folio_nid(folio)); +} + #ifdef CONFIG_MIGRATION static int top_tier_adistance; /* diff --git a/mm/memory.c b/mm/memory.c index 802d0d8a40f9..105e1a0157dd 100644 --- a/mm/memory.c +++ b/mm/memory.c @@ -5337,8 +5337,7 @@ static vm_fault_t do_numa_page(struct vm_fault *vmf) * For memory tiering mode, cpupid of slow memory page is used * to record page access time. So use default value. */ - if ((sysctl_numa_balancing_mode & NUMA_BALANCING_MEMORY_TIERING) && - !node_is_toptier(nid)) + if (!folio_has_cpupid(folio)) last_cpupid = (-1 & LAST_CPUPID_MASK); else last_cpupid = folio_last_cpupid(folio); diff --git a/mm/mprotect.c b/mm/mprotect.c index 222ab434da54..787c3c2bf1b6 100644 --- a/mm/mprotect.c +++ b/mm/mprotect.c @@ -161,8 +161,7 @@ static long change_pte_range(struct mmu_gather *tlb, if (!(sysctl_numa_balancing_mode & NUMA_BALANCING_NORMAL) && toptier) continue; - if (sysctl_numa_balancing_mode & NUMA_BALANCING_MEMORY_TIERING && - !toptier) + if (!folio_has_cpupid(folio)) folio_xchg_access_time(folio, jiffies_to_msecs(jiffies)); }