| Message ID | 20240212213922.783301-33-surenb@google.com (mailing list archive) |
|---|---|
| State | New |
| Headers | show |
| Series | Memory allocation profiling | expand |
On 2/12/24 22:39, Suren Baghdasaryan wrote: > objext objects are created with __GFP_NO_OBJ_EXT flag and therefore have > no corresponding objext themselves (otherwise we would get an infinite > recursion). When freeing these objects their codetag will be empty and > when CONFIG_MEM_ALLOC_PROFILING_DEBUG is enabled this will lead to false > warnings. Introduce CODETAG_EMPTY special codetag value to mark > allocations which intentionally lack codetag to avoid these warnings. > Set objext codetags to CODETAG_EMPTY before freeing to indicate that > the codetag is expected to be empty. > > Signed-off-by: Suren Baghdasaryan <surenb@google.com> > --- > include/linux/alloc_tag.h | 26 ++++++++++++++++++++++++++ > mm/slab.h | 25 +++++++++++++++++++++++++ > mm/slab_common.c | 1 + > mm/slub.c | 8 ++++++++ > 4 files changed, 60 insertions(+) > > diff --git a/include/linux/alloc_tag.h b/include/linux/alloc_tag.h > index 0a5973c4ad77..1f3207097b03 100644 ... > index c4bd0d5348cb..cf332a839bf4 100644 > --- a/mm/slab.h > +++ b/mm/slab.h > @@ -567,6 +567,31 @@ static inline struct slabobj_ext *slab_obj_exts(struct slab *slab) > int alloc_slab_obj_exts(struct slab *slab, struct kmem_cache *s, > gfp_t gfp, bool new_slab); > > + > +#ifdef CONFIG_MEM_ALLOC_PROFILING_DEBUG > + > +static inline void mark_objexts_empty(struct slabobj_ext *obj_exts) > +{ > + struct slabobj_ext *slab_exts; > + struct slab *obj_exts_slab; > + > + obj_exts_slab = virt_to_slab(obj_exts); > + slab_exts = slab_obj_exts(obj_exts_slab); > + if (slab_exts) { > + unsigned int offs = obj_to_index(obj_exts_slab->slab_cache, > + obj_exts_slab, obj_exts); > + /* codetag should be NULL */ > + WARN_ON(slab_exts[offs].ref.ct); > + set_codetag_empty(&slab_exts[offs].ref); > + } > +} > + > +#else /* CONFIG_MEM_ALLOC_PROFILING_DEBUG */ > + > +static inline void mark_objexts_empty(struct slabobj_ext *obj_exts) {} > + > +#endif /* CONFIG_MEM_ALLOC_PROFILING_DEBUG */ > + I assume with alloc_slab_obj_exts() moved to slub.c, mark_objexts_empty() could move there too. > static inline bool need_slab_obj_ext(void) > { > #ifdef CONFIG_MEM_ALLOC_PROFILING > diff --git a/mm/slab_common.c b/mm/slab_common.c > index 21b0b9e9cd9e..d5f75d04ced2 100644 > --- a/mm/slab_common.c > +++ b/mm/slab_common.c > @@ -242,6 +242,7 @@ int alloc_slab_obj_exts(struct slab *slab, struct kmem_cache *s, > * assign slabobj_exts in parallel. In this case the existing > * objcg vector should be reused. > */ > + mark_objexts_empty(vec); > kfree(vec); > return 0; > } > diff --git a/mm/slub.c b/mm/slub.c > index 4d480784942e..1136ff18b4fe 100644 > --- a/mm/slub.c > +++ b/mm/slub.c > @@ -1890,6 +1890,14 @@ static inline void free_slab_obj_exts(struct slab *slab) > if (!obj_exts) > return; > > + /* > + * obj_exts was created with __GFP_NO_OBJ_EXT flag, therefore its > + * corresponding extension will be NULL. alloc_tag_sub() will throw a > + * warning if slab has extensions but the extension of an object is > + * NULL, therefore replace NULL with CODETAG_EMPTY to indicate that > + * the extension for obj_exts is expected to be NULL. > + */ > + mark_objexts_empty(obj_exts); > kfree(obj_exts); > slab->obj_exts = 0; > }
On Fri, Feb 16, 2024 at 6:39 PM Vlastimil Babka <vbabka@suse.cz> wrote: > > On 2/12/24 22:39, Suren Baghdasaryan wrote: > > objext objects are created with __GFP_NO_OBJ_EXT flag and therefore have > > no corresponding objext themselves (otherwise we would get an infinite > > recursion). When freeing these objects their codetag will be empty and > > when CONFIG_MEM_ALLOC_PROFILING_DEBUG is enabled this will lead to false > > warnings. Introduce CODETAG_EMPTY special codetag value to mark > > allocations which intentionally lack codetag to avoid these warnings. > > Set objext codetags to CODETAG_EMPTY before freeing to indicate that > > the codetag is expected to be empty. > > > > Signed-off-by: Suren Baghdasaryan <surenb@google.com> > > --- > > include/linux/alloc_tag.h | 26 ++++++++++++++++++++++++++ > > mm/slab.h | 25 +++++++++++++++++++++++++ > > mm/slab_common.c | 1 + > > mm/slub.c | 8 ++++++++ > > 4 files changed, 60 insertions(+) > > > > diff --git a/include/linux/alloc_tag.h b/include/linux/alloc_tag.h > > index 0a5973c4ad77..1f3207097b03 100644 > > ... > > > index c4bd0d5348cb..cf332a839bf4 100644 > > --- a/mm/slab.h > > +++ b/mm/slab.h > > @@ -567,6 +567,31 @@ static inline struct slabobj_ext *slab_obj_exts(struct slab *slab) > > int alloc_slab_obj_exts(struct slab *slab, struct kmem_cache *s, > > gfp_t gfp, bool new_slab); > > > > + > > +#ifdef CONFIG_MEM_ALLOC_PROFILING_DEBUG > > + > > +static inline void mark_objexts_empty(struct slabobj_ext *obj_exts) > > +{ > > + struct slabobj_ext *slab_exts; > > + struct slab *obj_exts_slab; > > + > > + obj_exts_slab = virt_to_slab(obj_exts); > > + slab_exts = slab_obj_exts(obj_exts_slab); > > + if (slab_exts) { > > + unsigned int offs = obj_to_index(obj_exts_slab->slab_cache, > > + obj_exts_slab, obj_exts); > > + /* codetag should be NULL */ > > + WARN_ON(slab_exts[offs].ref.ct); > > + set_codetag_empty(&slab_exts[offs].ref); > > + } > > +} > > + > > +#else /* CONFIG_MEM_ALLOC_PROFILING_DEBUG */ > > + > > +static inline void mark_objexts_empty(struct slabobj_ext *obj_exts) {} > > + > > +#endif /* CONFIG_MEM_ALLOC_PROFILING_DEBUG */ > > + > > I assume with alloc_slab_obj_exts() moved to slub.c, mark_objexts_empty() > could move there too. No, I think mark_objexts_empty() belongs here. This patch introduced the function and uses it. Makes sense to me to keep it all together. > > > static inline bool need_slab_obj_ext(void) > > { > > #ifdef CONFIG_MEM_ALLOC_PROFILING > > diff --git a/mm/slab_common.c b/mm/slab_common.c > > index 21b0b9e9cd9e..d5f75d04ced2 100644 > > --- a/mm/slab_common.c > > +++ b/mm/slab_common.c > > @@ -242,6 +242,7 @@ int alloc_slab_obj_exts(struct slab *slab, struct kmem_cache *s, > > * assign slabobj_exts in parallel. In this case the existing > > * objcg vector should be reused. > > */ > > + mark_objexts_empty(vec); > > kfree(vec); > > return 0; > > } > > diff --git a/mm/slub.c b/mm/slub.c > > index 4d480784942e..1136ff18b4fe 100644 > > --- a/mm/slub.c > > +++ b/mm/slub.c > > @@ -1890,6 +1890,14 @@ static inline void free_slab_obj_exts(struct slab *slab) > > if (!obj_exts) > > return; > > > > + /* > > + * obj_exts was created with __GFP_NO_OBJ_EXT flag, therefore its > > + * corresponding extension will be NULL. alloc_tag_sub() will throw a > > + * warning if slab has extensions but the extension of an object is > > + * NULL, therefore replace NULL with CODETAG_EMPTY to indicate that > > + * the extension for obj_exts is expected to be NULL. > > + */ > > + mark_objexts_empty(obj_exts); > > kfree(obj_exts); > > slab->obj_exts = 0; > > } >
On 2/19/24 02:04, Suren Baghdasaryan wrote: > On Fri, Feb 16, 2024 at 6:39 PM Vlastimil Babka <vbabka@suse.cz> wrote: >> >> On 2/12/24 22:39, Suren Baghdasaryan wrote: >> > objext objects are created with __GFP_NO_OBJ_EXT flag and therefore have >> > no corresponding objext themselves (otherwise we would get an infinite >> > recursion). When freeing these objects their codetag will be empty and >> > when CONFIG_MEM_ALLOC_PROFILING_DEBUG is enabled this will lead to false >> > warnings. Introduce CODETAG_EMPTY special codetag value to mark >> > allocations which intentionally lack codetag to avoid these warnings. >> > Set objext codetags to CODETAG_EMPTY before freeing to indicate that >> > the codetag is expected to be empty. >> > >> > Signed-off-by: Suren Baghdasaryan <surenb@google.com> >> > --- >> > include/linux/alloc_tag.h | 26 ++++++++++++++++++++++++++ >> > mm/slab.h | 25 +++++++++++++++++++++++++ >> > mm/slab_common.c | 1 + >> > mm/slub.c | 8 ++++++++ >> > 4 files changed, 60 insertions(+) >> > >> > diff --git a/include/linux/alloc_tag.h b/include/linux/alloc_tag.h >> > index 0a5973c4ad77..1f3207097b03 100644 >> >> ... >> >> > index c4bd0d5348cb..cf332a839bf4 100644 >> > --- a/mm/slab.h >> > +++ b/mm/slab.h >> > @@ -567,6 +567,31 @@ static inline struct slabobj_ext *slab_obj_exts(struct slab *slab) >> > int alloc_slab_obj_exts(struct slab *slab, struct kmem_cache *s, >> > gfp_t gfp, bool new_slab); >> > >> > + >> > +#ifdef CONFIG_MEM_ALLOC_PROFILING_DEBUG >> > + >> > +static inline void mark_objexts_empty(struct slabobj_ext *obj_exts) >> > +{ >> > + struct slabobj_ext *slab_exts; >> > + struct slab *obj_exts_slab; >> > + >> > + obj_exts_slab = virt_to_slab(obj_exts); >> > + slab_exts = slab_obj_exts(obj_exts_slab); >> > + if (slab_exts) { >> > + unsigned int offs = obj_to_index(obj_exts_slab->slab_cache, >> > + obj_exts_slab, obj_exts); >> > + /* codetag should be NULL */ >> > + WARN_ON(slab_exts[offs].ref.ct); >> > + set_codetag_empty(&slab_exts[offs].ref); >> > + } >> > +} >> > + >> > +#else /* CONFIG_MEM_ALLOC_PROFILING_DEBUG */ >> > + >> > +static inline void mark_objexts_empty(struct slabobj_ext *obj_exts) {} >> > + >> > +#endif /* CONFIG_MEM_ALLOC_PROFILING_DEBUG */ >> > + >> >> I assume with alloc_slab_obj_exts() moved to slub.c, mark_objexts_empty() >> could move there too. > > No, I think mark_objexts_empty() belongs here. This patch introduced > the function and uses it. Makes sense to me to keep it all together. Hi, here I didn't mean moving between patches, but files. alloc_slab_obj_exts() in slub.c means all callers of mark_objexts_empty() are in slub.c so it doesn't need to be in slab.h Also same thing with mark_failed_objexts_alloc() and handle_failed_objexts_alloc() in patch 34/35.
On Mon, Feb 19, 2024 at 1:17 AM Vlastimil Babka <vbabka@suse.cz> wrote: > > On 2/19/24 02:04, Suren Baghdasaryan wrote: > > On Fri, Feb 16, 2024 at 6:39 PM Vlastimil Babka <vbabka@suse.cz> wrote: > >> > >> On 2/12/24 22:39, Suren Baghdasaryan wrote: > >> > objext objects are created with __GFP_NO_OBJ_EXT flag and therefore have > >> > no corresponding objext themselves (otherwise we would get an infinite > >> > recursion). When freeing these objects their codetag will be empty and > >> > when CONFIG_MEM_ALLOC_PROFILING_DEBUG is enabled this will lead to false > >> > warnings. Introduce CODETAG_EMPTY special codetag value to mark > >> > allocations which intentionally lack codetag to avoid these warnings. > >> > Set objext codetags to CODETAG_EMPTY before freeing to indicate that > >> > the codetag is expected to be empty. > >> > > >> > Signed-off-by: Suren Baghdasaryan <surenb@google.com> > >> > --- > >> > include/linux/alloc_tag.h | 26 ++++++++++++++++++++++++++ > >> > mm/slab.h | 25 +++++++++++++++++++++++++ > >> > mm/slab_common.c | 1 + > >> > mm/slub.c | 8 ++++++++ > >> > 4 files changed, 60 insertions(+) > >> > > >> > diff --git a/include/linux/alloc_tag.h b/include/linux/alloc_tag.h > >> > index 0a5973c4ad77..1f3207097b03 100644 > >> > >> ... > >> > >> > index c4bd0d5348cb..cf332a839bf4 100644 > >> > --- a/mm/slab.h > >> > +++ b/mm/slab.h > >> > @@ -567,6 +567,31 @@ static inline struct slabobj_ext *slab_obj_exts(struct slab *slab) > >> > int alloc_slab_obj_exts(struct slab *slab, struct kmem_cache *s, > >> > gfp_t gfp, bool new_slab); > >> > > >> > + > >> > +#ifdef CONFIG_MEM_ALLOC_PROFILING_DEBUG > >> > + > >> > +static inline void mark_objexts_empty(struct slabobj_ext *obj_exts) > >> > +{ > >> > + struct slabobj_ext *slab_exts; > >> > + struct slab *obj_exts_slab; > >> > + > >> > + obj_exts_slab = virt_to_slab(obj_exts); > >> > + slab_exts = slab_obj_exts(obj_exts_slab); > >> > + if (slab_exts) { > >> > + unsigned int offs = obj_to_index(obj_exts_slab->slab_cache, > >> > + obj_exts_slab, obj_exts); > >> > + /* codetag should be NULL */ > >> > + WARN_ON(slab_exts[offs].ref.ct); > >> > + set_codetag_empty(&slab_exts[offs].ref); > >> > + } > >> > +} > >> > + > >> > +#else /* CONFIG_MEM_ALLOC_PROFILING_DEBUG */ > >> > + > >> > +static inline void mark_objexts_empty(struct slabobj_ext *obj_exts) {} > >> > + > >> > +#endif /* CONFIG_MEM_ALLOC_PROFILING_DEBUG */ > >> > + > >> > >> I assume with alloc_slab_obj_exts() moved to slub.c, mark_objexts_empty() > >> could move there too. > > > > No, I think mark_objexts_empty() belongs here. This patch introduced > > the function and uses it. Makes sense to me to keep it all together. > > Hi, > > here I didn't mean moving between patches, but files. alloc_slab_obj_exts() > in slub.c means all callers of mark_objexts_empty() are in slub.c so it > doesn't need to be in slab.h Ah, I see. I misunderstood your comment. Yes, after slab/slob cleanup this makes sense. > > Also same thing with mark_failed_objexts_alloc() and > handle_failed_objexts_alloc() in patch 34/35. Ack. Thanks! >
diff --git a/include/linux/alloc_tag.h b/include/linux/alloc_tag.h index 0a5973c4ad77..1f3207097b03 100644 --- a/include/linux/alloc_tag.h +++ b/include/linux/alloc_tag.h @@ -77,6 +77,27 @@ static inline struct alloc_tag_counters alloc_tag_read(struct alloc_tag *tag) return v; } +#ifdef CONFIG_MEM_ALLOC_PROFILING_DEBUG + +#define CODETAG_EMPTY (void *)1 + +static inline bool is_codetag_empty(union codetag_ref *ref) +{ + return ref->ct == CODETAG_EMPTY; +} + +static inline void set_codetag_empty(union codetag_ref *ref) +{ + if (ref) + ref->ct = CODETAG_EMPTY; +} + +#else /* CONFIG_MEM_ALLOC_PROFILING_DEBUG */ + +static inline bool is_codetag_empty(union codetag_ref *ref) { return false; } + +#endif /* CONFIG_MEM_ALLOC_PROFILING_DEBUG */ + static inline void __alloc_tag_sub(union codetag_ref *ref, size_t bytes) { struct alloc_tag *tag; @@ -87,6 +108,11 @@ static inline void __alloc_tag_sub(union codetag_ref *ref, size_t bytes) if (!ref || !ref->ct) return; + if (is_codetag_empty(ref)) { + ref->ct = NULL; + return; + } + tag = ct_to_alloc_tag(ref->ct); this_cpu_sub(tag->counters->bytes, bytes); diff --git a/mm/slab.h b/mm/slab.h index c4bd0d5348cb..cf332a839bf4 100644 --- a/mm/slab.h +++ b/mm/slab.h @@ -567,6 +567,31 @@ static inline struct slabobj_ext *slab_obj_exts(struct slab *slab) int alloc_slab_obj_exts(struct slab *slab, struct kmem_cache *s, gfp_t gfp, bool new_slab); + +#ifdef CONFIG_MEM_ALLOC_PROFILING_DEBUG + +static inline void mark_objexts_empty(struct slabobj_ext *obj_exts) +{ + struct slabobj_ext *slab_exts; + struct slab *obj_exts_slab; + + obj_exts_slab = virt_to_slab(obj_exts); + slab_exts = slab_obj_exts(obj_exts_slab); + if (slab_exts) { + unsigned int offs = obj_to_index(obj_exts_slab->slab_cache, + obj_exts_slab, obj_exts); + /* codetag should be NULL */ + WARN_ON(slab_exts[offs].ref.ct); + set_codetag_empty(&slab_exts[offs].ref); + } +} + +#else /* CONFIG_MEM_ALLOC_PROFILING_DEBUG */ + +static inline void mark_objexts_empty(struct slabobj_ext *obj_exts) {} + +#endif /* CONFIG_MEM_ALLOC_PROFILING_DEBUG */ + static inline bool need_slab_obj_ext(void) { #ifdef CONFIG_MEM_ALLOC_PROFILING diff --git a/mm/slab_common.c b/mm/slab_common.c index 21b0b9e9cd9e..d5f75d04ced2 100644 --- a/mm/slab_common.c +++ b/mm/slab_common.c @@ -242,6 +242,7 @@ int alloc_slab_obj_exts(struct slab *slab, struct kmem_cache *s, * assign slabobj_exts in parallel. In this case the existing * objcg vector should be reused. */ + mark_objexts_empty(vec); kfree(vec); return 0; } diff --git a/mm/slub.c b/mm/slub.c index 4d480784942e..1136ff18b4fe 100644 --- a/mm/slub.c +++ b/mm/slub.c @@ -1890,6 +1890,14 @@ static inline void free_slab_obj_exts(struct slab *slab) if (!obj_exts) return; + /* + * obj_exts was created with __GFP_NO_OBJ_EXT flag, therefore its + * corresponding extension will be NULL. alloc_tag_sub() will throw a + * warning if slab has extensions but the extension of an object is + * NULL, therefore replace NULL with CODETAG_EMPTY to indicate that + * the extension for obj_exts is expected to be NULL. + */ + mark_objexts_empty(obj_exts); kfree(obj_exts); slab->obj_exts = 0; }
objext objects are created with __GFP_NO_OBJ_EXT flag and therefore have no corresponding objext themselves (otherwise we would get an infinite recursion). When freeing these objects their codetag will be empty and when CONFIG_MEM_ALLOC_PROFILING_DEBUG is enabled this will lead to false warnings. Introduce CODETAG_EMPTY special codetag value to mark allocations which intentionally lack codetag to avoid these warnings. Set objext codetags to CODETAG_EMPTY before freeing to indicate that the codetag is expected to be empty. Signed-off-by: Suren Baghdasaryan <surenb@google.com> --- include/linux/alloc_tag.h | 26 ++++++++++++++++++++++++++ mm/slab.h | 25 +++++++++++++++++++++++++ mm/slab_common.c | 1 + mm/slub.c | 8 ++++++++ 4 files changed, 60 insertions(+)