| Message ID | 20220808062601.836025-7-aneesh.kumar@linux.ibm.com (mailing list archive) |
|---|---|
| State | New |
| Headers | show |
| Series | mm/demotion: Memory tiers and demotion | expand |
"Aneesh Kumar K.V" <aneesh.kumar@linux.ibm.com> writes: > Also update different helpes to use NODE_DATA()->memtier. Since > node specific memtier can change based on the reassignment of > NUMA node to a different memory tiers, accessing NODE_DATA()->memtier > needs to happen under an rcu read lock or memory_tier_lock. > > Signed-off-by: Aneesh Kumar K.V <aneesh.kumar@linux.ibm.com> > --- > include/linux/mmzone.h | 3 +++ > mm/memory-tiers.c | 38 ++++++++++++++++++++++++++++++++------ > 2 files changed, 35 insertions(+), 6 deletions(-) > > diff --git a/include/linux/mmzone.h b/include/linux/mmzone.h > index aab70355d64f..353812495a70 100644 > --- a/include/linux/mmzone.h > +++ b/include/linux/mmzone.h > @@ -928,6 +928,9 @@ typedef struct pglist_data { > /* Per-node vmstats */ > struct per_cpu_nodestat __percpu *per_cpu_nodestats; > atomic_long_t vm_stat[NR_VM_NODE_STAT_ITEMS]; > +#ifdef CONFIG_NUMA > + struct memory_tier __rcu *memtier; > +#endif > } pg_data_t; > > #define node_present_pages(nid) (NODE_DATA(nid)->node_present_pages) > diff --git a/mm/memory-tiers.c b/mm/memory-tiers.c > index 02e514e87d5c..3778ac6a44a1 100644 > --- a/mm/memory-tiers.c > +++ b/mm/memory-tiers.c > @@ -5,6 +5,7 @@ > #include <linux/kobject.h> > #include <linux/memory.h> > #include <linux/random.h> > +#include <linux/mmzone.h> > #include <linux/memory-tiers.h> > > #include "internal.h" > @@ -137,12 +138,18 @@ static struct memory_tier *find_create_memory_tier(struct memory_dev_type *memty > > static struct memory_tier *__node_get_memory_tier(int node) > { > - struct memory_dev_type *memtype; > + pg_data_t *pgdat; > > - memtype = node_memory_types[node]; > - if (memtype && node_isset(node, memtype->nodes)) > - return memtype->memtier; > - return NULL; > + pgdat = NODE_DATA(node); > + if (!pgdat) > + return NULL; > + /* > + * Since we hold memory_tier_lock, we can avoid > + * RCU read locks when accessing the details. No > + * parallel updates are possible here. > + */ > + return rcu_dereference_check(pgdat->memtier, > + lockdep_is_held(&memory_tier_lock)); > } > > #ifdef CONFIG_MIGRATION > @@ -295,6 +302,8 @@ static struct memory_tier *set_node_memory_tier(int node) > { > struct memory_tier *memtier; > struct memory_dev_type *memtype; > + pg_data_t *pgdat = NODE_DATA(node); > + > > lockdep_assert_held_once(&memory_tier_lock); > > @@ -307,6 +316,8 @@ static struct memory_tier *set_node_memory_tier(int node) > memtype = node_memory_types[node]; > node_set(node, memtype->nodes); > memtier = find_create_memory_tier(memtype); > + if (!IS_ERR(memtier)) > + rcu_assign_pointer(pgdat->memtier, memtier); > return memtier; > } > > @@ -319,12 +330,25 @@ static void destroy_memory_tier(struct memory_tier *memtier) > static bool clear_node_memory_tier(int node) > { > bool cleared = false; > + pg_data_t *pgdat; > struct memory_tier *memtier; > > + pgdat = NODE_DATA(node); > + if (!pgdat) > + return false; > + > + /* > + * Make sure that anybody looking at NODE_DATA who finds > + * a valid memtier finds memory_dev_types with nodes still > + * linked to the memtier. We achieve this by waiting for > + * rcu read section to finish using synchronize_rcu. > + */ The synchronize_rcu() is also needed because we may free the memory tier. Best Regards, Huang, Ying > memtier = __node_get_memory_tier(node); > if (memtier) { > struct memory_dev_type *memtype; > > + rcu_assign_pointer(pgdat->memtier, NULL); > + synchronize_rcu(); > memtype = node_memory_types[node]; > node_clear(node, memtype->nodes); > if (nodes_empty(memtype->nodes)) { > @@ -422,8 +446,10 @@ static int __init memory_tier_init(void) > panic("%s() failed to register memory tier: %ld\n", > __func__, PTR_ERR(memtier)); > > - for_each_node_state(node, N_MEMORY) > + for_each_node_state(node, N_MEMORY) { > __init_node_memory_type(node, default_dram_type); > + rcu_assign_pointer(NODE_DATA(node)->memtier, memtier); > + } > > mutex_unlock(&memory_tier_lock); > #ifdef CONFIG_MIGRATION
"Aneesh Kumar K.V" <aneesh.kumar@linux.ibm.com> writes: > Also update different helpes to use NODE_DATA()->memtier. Since > node specific memtier can change based on the reassignment of > NUMA node to a different memory tiers, accessing NODE_DATA()->memtier > needs to happen under an rcu read lock or memory_tier_lock. > > Signed-off-by: Aneesh Kumar K.V <aneesh.kumar@linux.ibm.com> > --- > include/linux/mmzone.h | 3 +++ > mm/memory-tiers.c | 38 ++++++++++++++++++++++++++++++++------ > 2 files changed, 35 insertions(+), 6 deletions(-) > > diff --git a/include/linux/mmzone.h b/include/linux/mmzone.h > index aab70355d64f..353812495a70 100644 > --- a/include/linux/mmzone.h > +++ b/include/linux/mmzone.h > @@ -928,6 +928,9 @@ typedef struct pglist_data { > /* Per-node vmstats */ > struct per_cpu_nodestat __percpu *per_cpu_nodestats; > atomic_long_t vm_stat[NR_VM_NODE_STAT_ITEMS]; > +#ifdef CONFIG_NUMA > + struct memory_tier __rcu *memtier; > +#endif > } pg_data_t; > > #define node_present_pages(nid) (NODE_DATA(nid)->node_present_pages) > diff --git a/mm/memory-tiers.c b/mm/memory-tiers.c > index 02e514e87d5c..3778ac6a44a1 100644 > --- a/mm/memory-tiers.c > +++ b/mm/memory-tiers.c > @@ -5,6 +5,7 @@ > #include <linux/kobject.h> > #include <linux/memory.h> > #include <linux/random.h> > +#include <linux/mmzone.h> > #include <linux/memory-tiers.h> > > #include "internal.h" > @@ -137,12 +138,18 @@ static struct memory_tier *find_create_memory_tier(struct memory_dev_type *memty > > static struct memory_tier *__node_get_memory_tier(int node) > { > - struct memory_dev_type *memtype; > + pg_data_t *pgdat; > > - memtype = node_memory_types[node]; > - if (memtype && node_isset(node, memtype->nodes)) > - return memtype->memtier; > - return NULL; After adding pgdat->memtier, it appears there's unnecessary to keep memtype->memtier? Best Regards, Huang, Ying > + pgdat = NODE_DATA(node); > + if (!pgdat) > + return NULL; > + /* > + * Since we hold memory_tier_lock, we can avoid > + * RCU read locks when accessing the details. No > + * parallel updates are possible here. > + */ > + return rcu_dereference_check(pgdat->memtier, > + lockdep_is_held(&memory_tier_lock)); > } > > #ifdef CONFIG_MIGRATION > @@ -295,6 +302,8 @@ static struct memory_tier *set_node_memory_tier(int node) > { > struct memory_tier *memtier; > struct memory_dev_type *memtype; > + pg_data_t *pgdat = NODE_DATA(node); > + > > lockdep_assert_held_once(&memory_tier_lock); > > @@ -307,6 +316,8 @@ static struct memory_tier *set_node_memory_tier(int node) > memtype = node_memory_types[node]; > node_set(node, memtype->nodes); > memtier = find_create_memory_tier(memtype); > + if (!IS_ERR(memtier)) > + rcu_assign_pointer(pgdat->memtier, memtier); > return memtier; > } > > @@ -319,12 +330,25 @@ static void destroy_memory_tier(struct memory_tier *memtier) > static bool clear_node_memory_tier(int node) > { > bool cleared = false; > + pg_data_t *pgdat; > struct memory_tier *memtier; > > + pgdat = NODE_DATA(node); > + if (!pgdat) > + return false; > + > + /* > + * Make sure that anybody looking at NODE_DATA who finds > + * a valid memtier finds memory_dev_types with nodes still > + * linked to the memtier. We achieve this by waiting for > + * rcu read section to finish using synchronize_rcu. > + */ > memtier = __node_get_memory_tier(node); > if (memtier) { > struct memory_dev_type *memtype; > > + rcu_assign_pointer(pgdat->memtier, NULL); > + synchronize_rcu(); > memtype = node_memory_types[node]; > node_clear(node, memtype->nodes); > if (nodes_empty(memtype->nodes)) { > @@ -422,8 +446,10 @@ static int __init memory_tier_init(void) > panic("%s() failed to register memory tier: %ld\n", > __func__, PTR_ERR(memtier)); > > - for_each_node_state(node, N_MEMORY) > + for_each_node_state(node, N_MEMORY) { > __init_node_memory_type(node, default_dram_type); > + rcu_assign_pointer(NODE_DATA(node)->memtier, memtier); > + } > > mutex_unlock(&memory_tier_lock); > #ifdef CONFIG_MIGRATION
On 8/9/22 10:51 AM, Huang, Ying wrote: > "Aneesh Kumar K.V" <aneesh.kumar@linux.ibm.com> writes: > >> Also update different helpes to use NODE_DATA()->memtier. Since >> node specific memtier can change based on the reassignment of >> NUMA node to a different memory tiers, accessing NODE_DATA()->memtier >> needs to happen under an rcu read lock or memory_tier_lock. >> >> Signed-off-by: Aneesh Kumar K.V <aneesh.kumar@linux.ibm.com> >> --- >> include/linux/mmzone.h | 3 +++ >> mm/memory-tiers.c | 38 ++++++++++++++++++++++++++++++++------ >> 2 files changed, 35 insertions(+), 6 deletions(-) >> >> diff --git a/include/linux/mmzone.h b/include/linux/mmzone.h >> index aab70355d64f..353812495a70 100644 >> --- a/include/linux/mmzone.h >> +++ b/include/linux/mmzone.h >> @@ -928,6 +928,9 @@ typedef struct pglist_data { >> /* Per-node vmstats */ >> struct per_cpu_nodestat __percpu *per_cpu_nodestats; >> atomic_long_t vm_stat[NR_VM_NODE_STAT_ITEMS]; >> +#ifdef CONFIG_NUMA >> + struct memory_tier __rcu *memtier; >> +#endif >> } pg_data_t; >> >> #define node_present_pages(nid) (NODE_DATA(nid)->node_present_pages) >> diff --git a/mm/memory-tiers.c b/mm/memory-tiers.c >> index 02e514e87d5c..3778ac6a44a1 100644 >> --- a/mm/memory-tiers.c >> +++ b/mm/memory-tiers.c >> @@ -5,6 +5,7 @@ >> #include <linux/kobject.h> >> #include <linux/memory.h> >> #include <linux/random.h> >> +#include <linux/mmzone.h> >> #include <linux/memory-tiers.h> >> >> #include "internal.h" >> @@ -137,12 +138,18 @@ static struct memory_tier *find_create_memory_tier(struct memory_dev_type *memty >> >> static struct memory_tier *__node_get_memory_tier(int node) >> { >> - struct memory_dev_type *memtype; >> + pg_data_t *pgdat; >> >> - memtype = node_memory_types[node]; >> - if (memtype && node_isset(node, memtype->nodes)) >> - return memtype->memtier; >> - return NULL; > > After adding pgdat->memtier, it appears there's unnecessary to keep > memtype->memtier? > It do simplify find_create_memory_tier() where I use if (memtype->memtier) to check whether the memtype is already added to a memory tier. I could switch that to list_empty(memtype->tier_sibiling). But I felt the current one is much cleaner . -aneesh
Aneesh Kumar K V <aneesh.kumar@linux.ibm.com> writes: > On 8/9/22 10:51 AM, Huang, Ying wrote: >> "Aneesh Kumar K.V" <aneesh.kumar@linux.ibm.com> writes: >> >>> Also update different helpes to use NODE_DATA()->memtier. Since >>> node specific memtier can change based on the reassignment of >>> NUMA node to a different memory tiers, accessing NODE_DATA()->memtier >>> needs to happen under an rcu read lock or memory_tier_lock. >>> >>> Signed-off-by: Aneesh Kumar K.V <aneesh.kumar@linux.ibm.com> >>> --- >>> include/linux/mmzone.h | 3 +++ >>> mm/memory-tiers.c | 38 ++++++++++++++++++++++++++++++++------ >>> 2 files changed, 35 insertions(+), 6 deletions(-) >>> >>> diff --git a/include/linux/mmzone.h b/include/linux/mmzone.h >>> index aab70355d64f..353812495a70 100644 >>> --- a/include/linux/mmzone.h >>> +++ b/include/linux/mmzone.h >>> @@ -928,6 +928,9 @@ typedef struct pglist_data { >>> /* Per-node vmstats */ >>> struct per_cpu_nodestat __percpu *per_cpu_nodestats; >>> atomic_long_t vm_stat[NR_VM_NODE_STAT_ITEMS]; >>> +#ifdef CONFIG_NUMA >>> + struct memory_tier __rcu *memtier; >>> +#endif >>> } pg_data_t; >>> >>> #define node_present_pages(nid) (NODE_DATA(nid)->node_present_pages) >>> diff --git a/mm/memory-tiers.c b/mm/memory-tiers.c >>> index 02e514e87d5c..3778ac6a44a1 100644 >>> --- a/mm/memory-tiers.c >>> +++ b/mm/memory-tiers.c >>> @@ -5,6 +5,7 @@ >>> #include <linux/kobject.h> >>> #include <linux/memory.h> >>> #include <linux/random.h> >>> +#include <linux/mmzone.h> >>> #include <linux/memory-tiers.h> >>> >>> #include "internal.h" >>> @@ -137,12 +138,18 @@ static struct memory_tier *find_create_memory_tier(struct memory_dev_type *memty >>> >>> static struct memory_tier *__node_get_memory_tier(int node) >>> { >>> - struct memory_dev_type *memtype; >>> + pg_data_t *pgdat; >>> >>> - memtype = node_memory_types[node]; >>> - if (memtype && node_isset(node, memtype->nodes)) >>> - return memtype->memtier; >>> - return NULL; >> >> After adding pgdat->memtier, it appears there's unnecessary to keep >> memtype->memtier? >> > > It do simplify find_create_memory_tier() where I use if (memtype->memtier) > to check whether the memtype is already added to a memory tier. I could switch > that to list_empty(memtype->tier_sibiling). But I felt the current one is much > cleaner I prefer "list_empty(memtype->tier_sibiling)". But I will let you to decide. Best Regards, Huang, Ying
diff --git a/include/linux/mmzone.h b/include/linux/mmzone.h index aab70355d64f..353812495a70 100644 --- a/include/linux/mmzone.h +++ b/include/linux/mmzone.h @@ -928,6 +928,9 @@ typedef struct pglist_data { /* Per-node vmstats */ struct per_cpu_nodestat __percpu *per_cpu_nodestats; atomic_long_t vm_stat[NR_VM_NODE_STAT_ITEMS]; +#ifdef CONFIG_NUMA + struct memory_tier __rcu *memtier; +#endif } pg_data_t; #define node_present_pages(nid) (NODE_DATA(nid)->node_present_pages) diff --git a/mm/memory-tiers.c b/mm/memory-tiers.c index 02e514e87d5c..3778ac6a44a1 100644 --- a/mm/memory-tiers.c +++ b/mm/memory-tiers.c @@ -5,6 +5,7 @@ #include <linux/kobject.h> #include <linux/memory.h> #include <linux/random.h> +#include <linux/mmzone.h> #include <linux/memory-tiers.h> #include "internal.h" @@ -137,12 +138,18 @@ static struct memory_tier *find_create_memory_tier(struct memory_dev_type *memty static struct memory_tier *__node_get_memory_tier(int node) { - struct memory_dev_type *memtype; + pg_data_t *pgdat; - memtype = node_memory_types[node]; - if (memtype && node_isset(node, memtype->nodes)) - return memtype->memtier; - return NULL; + pgdat = NODE_DATA(node); + if (!pgdat) + return NULL; + /* + * Since we hold memory_tier_lock, we can avoid + * RCU read locks when accessing the details. No + * parallel updates are possible here. + */ + return rcu_dereference_check(pgdat->memtier, + lockdep_is_held(&memory_tier_lock)); } #ifdef CONFIG_MIGRATION @@ -295,6 +302,8 @@ static struct memory_tier *set_node_memory_tier(int node) { struct memory_tier *memtier; struct memory_dev_type *memtype; + pg_data_t *pgdat = NODE_DATA(node); + lockdep_assert_held_once(&memory_tier_lock); @@ -307,6 +316,8 @@ static struct memory_tier *set_node_memory_tier(int node) memtype = node_memory_types[node]; node_set(node, memtype->nodes); memtier = find_create_memory_tier(memtype); + if (!IS_ERR(memtier)) + rcu_assign_pointer(pgdat->memtier, memtier); return memtier; } @@ -319,12 +330,25 @@ static void destroy_memory_tier(struct memory_tier *memtier) static bool clear_node_memory_tier(int node) { bool cleared = false; + pg_data_t *pgdat; struct memory_tier *memtier; + pgdat = NODE_DATA(node); + if (!pgdat) + return false; + + /* + * Make sure that anybody looking at NODE_DATA who finds + * a valid memtier finds memory_dev_types with nodes still + * linked to the memtier. We achieve this by waiting for + * rcu read section to finish using synchronize_rcu. + */ memtier = __node_get_memory_tier(node); if (memtier) { struct memory_dev_type *memtype; + rcu_assign_pointer(pgdat->memtier, NULL); + synchronize_rcu(); memtype = node_memory_types[node]; node_clear(node, memtype->nodes); if (nodes_empty(memtype->nodes)) { @@ -422,8 +446,10 @@ static int __init memory_tier_init(void) panic("%s() failed to register memory tier: %ld\n", __func__, PTR_ERR(memtier)); - for_each_node_state(node, N_MEMORY) + for_each_node_state(node, N_MEMORY) { __init_node_memory_type(node, default_dram_type); + rcu_assign_pointer(NODE_DATA(node)->memtier, memtier); + } mutex_unlock(&memory_tier_lock); #ifdef CONFIG_MIGRATION
Also update different helpes to use NODE_DATA()->memtier. Since node specific memtier can change based on the reassignment of NUMA node to a different memory tiers, accessing NODE_DATA()->memtier needs to happen under an rcu read lock or memory_tier_lock. Signed-off-by: Aneesh Kumar K.V <aneesh.kumar@linux.ibm.com> --- include/linux/mmzone.h | 3 +++ mm/memory-tiers.c | 38 ++++++++++++++++++++++++++++++++------ 2 files changed, 35 insertions(+), 6 deletions(-)