| Message ID | 88f2964f4063aa6fd935ef8c8302d02d8d67005b.1640891329.git.andreyknvl@google.com (mailing list archive) |
|---|---|
| State | New, archived |
| Headers | show |
| Series | kasan, vmalloc, arm64: add vmalloc tagging support for SW/HW_TAGS | expand |
On Fri, 2021-12-31 at 03:14 +0800, andrey.konovalov@linux.dev wrote: > From: Andrey Konovalov <andreyknvl@google.com> > > Add a new GFP flag __GFP_SKIP_ZERO that allows to skip memory > initialization. The flag is only effective with HW_TAGS KASAN. > > This flag will be used by vmalloc code for page_alloc allocations > backing vmalloc() mappings in a following patch. The reason to skip > memory initialization for these pages in page_alloc is because > vmalloc > code will be initializing them instead. > > With the current implementation, when __GFP_SKIP_ZERO is provided, > __GFP_ZEROTAGS is ignored. This doesn't matter, as these two flags > are > never provided at the same time. However, if this is changed in the > future, this particular implementation detail can be changed as well. > > Signed-off-by: Andrey Konovalov <andreyknvl@google.com> > > --- > > Changes v4->v5: > - Cosmetic changes to __def_gfpflag_names_kasan and __GFP_BITS_SHIFT. > > Changes v3->v4: > - Only define __GFP_SKIP_ZERO when CONFIG_KASAN_HW_TAGS is enabled. > - Add __GFP_SKIP_ZERO to include/trace/events/mmflags.h. > - Use proper kasan_hw_tags_enabled() check instead of > IS_ENABLED(CONFIG_KASAN_HW_TAGS). Also add explicit checks for > software modes. > > Changes v2->v3: > - Update patch description. > > Changes v1->v2: > - Add this patch. > --- > include/linux/gfp.h | 18 +++++++++++------- > include/trace/events/mmflags.h | 1 + > mm/page_alloc.c | 18 +++++++++++++++++- > 3 files changed, 29 insertions(+), 8 deletions(-) > > diff --git a/include/linux/gfp.h b/include/linux/gfp.h > index 487126f089e1..6eef3e447540 100644 > --- a/include/linux/gfp.h > +++ b/include/linux/gfp.h > @@ -55,14 +55,16 @@ struct vm_area_struct; > #define ___GFP_ACCOUNT 0x400000u > #define ___GFP_ZEROTAGS 0x800000u > #ifdef CONFIG_KASAN_HW_TAGS > -#define ___GFP_SKIP_KASAN_UNPOISON 0x1000000u > -#define ___GFP_SKIP_KASAN_POISON 0x2000000u > +#define ___GFP_SKIP_ZERO 0x1000000u > +#define ___GFP_SKIP_KASAN_UNPOISON 0x2000000u > +#define ___GFP_SKIP_KASAN_POISON 0x4000000u > #else > +#define ___GFP_SKIP_ZERO 0 > #define ___GFP_SKIP_KASAN_UNPOISON 0 > #define ___GFP_SKIP_KASAN_POISON 0 > #endif > #ifdef CONFIG_LOCKDEP > -#define ___GFP_NOLOCKDEP 0x4000000u > +#define ___GFP_NOLOCKDEP 0x8000000u > #else > #define ___GFP_NOLOCKDEP 0 > #endif > @@ -235,9 +237,10 @@ struct vm_area_struct; > * %__GFP_ZERO returns a zeroed page on success. > * > * %__GFP_ZEROTAGS zeroes memory tags at allocation time if the > memory itself > - * is being zeroed (either via __GFP_ZERO or via init_on_alloc). > This flag is > - * intended for optimization: setting memory tags at the same time > as zeroing > - * memory has minimal additional performace impact. > + * is being zeroed (either via __GFP_ZERO or via init_on_alloc, > provided that > + * __GFP_SKIP_ZERO is not set). This flag is intended for > optimization: setting > + * memory tags at the same time as zeroing memory has minimal > additional > + * performace impact. > * > * %__GFP_SKIP_KASAN_UNPOISON makes KASAN skip unpoisoning on page > allocation. > * Only effective in HW_TAGS mode. > @@ -249,6 +252,7 @@ struct vm_area_struct; > #define __GFP_COMP ((__force gfp_t)___GFP_COMP) > #define __GFP_ZERO ((__force gfp_t)___GFP_ZERO) > #define __GFP_ZEROTAGS ((__force gfp_t)___GFP_ZEROTAGS) > +#define __GFP_SKIP_ZERO ((__force gfp_t)___GFP_SKIP_ZERO) > #define __GFP_SKIP_KASAN_UNPOISON ((__force > gfp_t)___GFP_SKIP_KASAN_UNPOISON) > #define __GFP_SKIP_KASAN_POISON ((__force > gfp_t)___GFP_SKIP_KASAN_POISON) > > @@ -257,7 +261,7 @@ struct vm_area_struct; > > /* Room for N __GFP_FOO bits */ > #define __GFP_BITS_SHIFT (24 + > \ > - 2 * IS_ENABLED(CONFIG_KASAN_HW_TAGS) + \ > + 3 * IS_ENABLED(CONFIG_KASAN_HW_TAGS) + \ > IS_ENABLED(CONFIG_LOCKDEP)) > #define __GFP_BITS_MASK ((__force gfp_t)((1 << __GFP_BITS_SHIFT) - > 1)) > > diff --git a/include/trace/events/mmflags.h > b/include/trace/events/mmflags.h > index 5ffc7bdce91f..0698c5d0f194 100644 > --- a/include/trace/events/mmflags.h > +++ b/include/trace/events/mmflags.h > @@ -52,6 +52,7 @@ > > #ifdef CONFIG_KASAN_HW_TAGS > #define __def_gfpflag_names_kasan , > \ > + {(unsigned long)__GFP_SKIP_ZERO, "__GFP_SKIP_ZERO"}, > \ > {(unsigned > long)__GFP_SKIP_KASAN_POISON, "__GFP_SKIP_KASAN_POISON"}, \ > {(unsigned long)__GFP_SKIP_KASAN_UNPOISON, > "__GFP_SKIP_KASAN_UNPOISON"} > #else > diff --git a/mm/page_alloc.c b/mm/page_alloc.c > index 102f0cd8815e..30da0e1f94f8 100644 > --- a/mm/page_alloc.c > +++ b/mm/page_alloc.c > @@ -2415,10 +2415,26 @@ static inline bool > should_skip_kasan_unpoison(gfp_t flags, bool init_tags) > return init_tags || (flags & __GFP_SKIP_KASAN_UNPOISON); > } > > +static inline bool should_skip_init(gfp_t flags) > +{ > + /* Don't skip if a software KASAN mode is enabled. */ > + if (IS_ENABLED(CONFIG_KASAN_GENERIC) || > + IS_ENABLED(CONFIG_KASAN_SW_TAGS)) > + return false; Forget to drop the above check? I saw v4 mentioned that this check can be dropped. [1] Do I miss something? [1] https://lkml.org/lkml/2021/12/30/450 > + > + /* Don't skip, if hardware tag-based KASAN is not enabled. */ > + if (!kasan_hw_tags_enabled()) > + return false; > + > + /* For hardware tag-based KASAN, skip if requested. */ > + return (flags & __GFP_SKIP_ZERO); > +} > + > inline void post_alloc_hook(struct page *page, unsigned int order, > gfp_t gfp_flags) > { > - bool init = !want_init_on_free() && > want_init_on_alloc(gfp_flags); > + bool init = !want_init_on_free() && > want_init_on_alloc(gfp_flags) && > + !should_skip_init(gfp_flags); > bool init_tags = init && (gfp_flags & __GFP_ZEROTAGS); > > set_page_private(page, 0); > -- > 2.25.1 > >
On Mon, Jan 3, 2022 at 3:32 AM Kuan-Ying Lee <Kuan-Ying.Lee@mediatek.com> wrote: > > On Fri, 2021-12-31 at 03:14 +0800, andrey.konovalov@linux.dev wrote: > > From: Andrey Konovalov <andreyknvl@google.com> > > > > Add a new GFP flag __GFP_SKIP_ZERO that allows to skip memory > > initialization. The flag is only effective with HW_TAGS KASAN. > > > > This flag will be used by vmalloc code for page_alloc allocations > > backing vmalloc() mappings in a following patch. The reason to skip > > memory initialization for these pages in page_alloc is because > > vmalloc > > code will be initializing them instead. > > > > With the current implementation, when __GFP_SKIP_ZERO is provided, > > __GFP_ZEROTAGS is ignored. This doesn't matter, as these two flags > > are > > never provided at the same time. However, if this is changed in the > > future, this particular implementation detail can be changed as well. > > > > Signed-off-by: Andrey Konovalov <andreyknvl@google.com> > > > > --- > > > > Changes v4->v5: > > - Cosmetic changes to __def_gfpflag_names_kasan and __GFP_BITS_SHIFT. > > > > Changes v3->v4: > > - Only define __GFP_SKIP_ZERO when CONFIG_KASAN_HW_TAGS is enabled. > > - Add __GFP_SKIP_ZERO to include/trace/events/mmflags.h. > > - Use proper kasan_hw_tags_enabled() check instead of > > IS_ENABLED(CONFIG_KASAN_HW_TAGS). Also add explicit checks for > > software modes. > > > > Changes v2->v3: > > - Update patch description. > > > > Changes v1->v2: > > - Add this patch. > > --- > > include/linux/gfp.h | 18 +++++++++++------- > > include/trace/events/mmflags.h | 1 + > > mm/page_alloc.c | 18 +++++++++++++++++- > > 3 files changed, 29 insertions(+), 8 deletions(-) > > > > diff --git a/include/linux/gfp.h b/include/linux/gfp.h > > index 487126f089e1..6eef3e447540 100644 > > --- a/include/linux/gfp.h > > +++ b/include/linux/gfp.h > > @@ -55,14 +55,16 @@ struct vm_area_struct; > > #define ___GFP_ACCOUNT 0x400000u > > #define ___GFP_ZEROTAGS 0x800000u > > #ifdef CONFIG_KASAN_HW_TAGS > > -#define ___GFP_SKIP_KASAN_UNPOISON 0x1000000u > > -#define ___GFP_SKIP_KASAN_POISON 0x2000000u > > +#define ___GFP_SKIP_ZERO 0x1000000u > > +#define ___GFP_SKIP_KASAN_UNPOISON 0x2000000u > > +#define ___GFP_SKIP_KASAN_POISON 0x4000000u > > #else > > +#define ___GFP_SKIP_ZERO 0 > > #define ___GFP_SKIP_KASAN_UNPOISON 0 > > #define ___GFP_SKIP_KASAN_POISON 0 > > #endif > > #ifdef CONFIG_LOCKDEP > > -#define ___GFP_NOLOCKDEP 0x4000000u > > +#define ___GFP_NOLOCKDEP 0x8000000u > > #else > > #define ___GFP_NOLOCKDEP 0 > > #endif > > @@ -235,9 +237,10 @@ struct vm_area_struct; > > * %__GFP_ZERO returns a zeroed page on success. > > * > > * %__GFP_ZEROTAGS zeroes memory tags at allocation time if the > > memory itself > > - * is being zeroed (either via __GFP_ZERO or via init_on_alloc). > > This flag is > > - * intended for optimization: setting memory tags at the same time > > as zeroing > > - * memory has minimal additional performace impact. > > + * is being zeroed (either via __GFP_ZERO or via init_on_alloc, > > provided that > > + * __GFP_SKIP_ZERO is not set). This flag is intended for > > optimization: setting > > + * memory tags at the same time as zeroing memory has minimal > > additional > > + * performace impact. > > * > > * %__GFP_SKIP_KASAN_UNPOISON makes KASAN skip unpoisoning on page > > allocation. > > * Only effective in HW_TAGS mode. > > @@ -249,6 +252,7 @@ struct vm_area_struct; > > #define __GFP_COMP ((__force gfp_t)___GFP_COMP) > > #define __GFP_ZERO ((__force gfp_t)___GFP_ZERO) > > #define __GFP_ZEROTAGS ((__force gfp_t)___GFP_ZEROTAGS) > > +#define __GFP_SKIP_ZERO ((__force gfp_t)___GFP_SKIP_ZERO) > > #define __GFP_SKIP_KASAN_UNPOISON ((__force > > gfp_t)___GFP_SKIP_KASAN_UNPOISON) > > #define __GFP_SKIP_KASAN_POISON ((__force > > gfp_t)___GFP_SKIP_KASAN_POISON) > > > > @@ -257,7 +261,7 @@ struct vm_area_struct; > > > > /* Room for N __GFP_FOO bits */ > > #define __GFP_BITS_SHIFT (24 + > > \ > > - 2 * IS_ENABLED(CONFIG_KASAN_HW_TAGS) + \ > > + 3 * IS_ENABLED(CONFIG_KASAN_HW_TAGS) + \ > > IS_ENABLED(CONFIG_LOCKDEP)) > > #define __GFP_BITS_MASK ((__force gfp_t)((1 << __GFP_BITS_SHIFT) - > > 1)) > > > > diff --git a/include/trace/events/mmflags.h > > b/include/trace/events/mmflags.h > > index 5ffc7bdce91f..0698c5d0f194 100644 > > --- a/include/trace/events/mmflags.h > > +++ b/include/trace/events/mmflags.h > > @@ -52,6 +52,7 @@ > > > > #ifdef CONFIG_KASAN_HW_TAGS > > #define __def_gfpflag_names_kasan , > > \ > > + {(unsigned long)__GFP_SKIP_ZERO, "__GFP_SKIP_ZERO"}, > > \ > > {(unsigned > > long)__GFP_SKIP_KASAN_POISON, "__GFP_SKIP_KASAN_POISON"}, \ > > {(unsigned long)__GFP_SKIP_KASAN_UNPOISON, > > "__GFP_SKIP_KASAN_UNPOISON"} > > #else > > diff --git a/mm/page_alloc.c b/mm/page_alloc.c > > index 102f0cd8815e..30da0e1f94f8 100644 > > --- a/mm/page_alloc.c > > +++ b/mm/page_alloc.c > > @@ -2415,10 +2415,26 @@ static inline bool > > should_skip_kasan_unpoison(gfp_t flags, bool init_tags) > > return init_tags || (flags & __GFP_SKIP_KASAN_UNPOISON); > > } > > > > +static inline bool should_skip_init(gfp_t flags) > > +{ > > + /* Don't skip if a software KASAN mode is enabled. */ > > + if (IS_ENABLED(CONFIG_KASAN_GENERIC) || > > + IS_ENABLED(CONFIG_KASAN_SW_TAGS)) > > + return false; > > Forget to drop the above check? > > I saw v4 mentioned that this check can be dropped. [1] > > Do I miss something? > > [1] https://lkml.org/lkml/2021/12/30/450 Right, forgot to include this change. Will include into v5 or post as a standalone fix closer to rc1. Thanks for noticing!
diff --git a/include/linux/gfp.h b/include/linux/gfp.h index 487126f089e1..6eef3e447540 100644 --- a/include/linux/gfp.h +++ b/include/linux/gfp.h @@ -55,14 +55,16 @@ struct vm_area_struct; #define ___GFP_ACCOUNT 0x400000u #define ___GFP_ZEROTAGS 0x800000u #ifdef CONFIG_KASAN_HW_TAGS -#define ___GFP_SKIP_KASAN_UNPOISON 0x1000000u -#define ___GFP_SKIP_KASAN_POISON 0x2000000u +#define ___GFP_SKIP_ZERO 0x1000000u +#define ___GFP_SKIP_KASAN_UNPOISON 0x2000000u +#define ___GFP_SKIP_KASAN_POISON 0x4000000u #else +#define ___GFP_SKIP_ZERO 0 #define ___GFP_SKIP_KASAN_UNPOISON 0 #define ___GFP_SKIP_KASAN_POISON 0 #endif #ifdef CONFIG_LOCKDEP -#define ___GFP_NOLOCKDEP 0x4000000u +#define ___GFP_NOLOCKDEP 0x8000000u #else #define ___GFP_NOLOCKDEP 0 #endif @@ -235,9 +237,10 @@ struct vm_area_struct; * %__GFP_ZERO returns a zeroed page on success. * * %__GFP_ZEROTAGS zeroes memory tags at allocation time if the memory itself - * is being zeroed (either via __GFP_ZERO or via init_on_alloc). This flag is - * intended for optimization: setting memory tags at the same time as zeroing - * memory has minimal additional performace impact. + * is being zeroed (either via __GFP_ZERO or via init_on_alloc, provided that + * __GFP_SKIP_ZERO is not set). This flag is intended for optimization: setting + * memory tags at the same time as zeroing memory has minimal additional + * performace impact. * * %__GFP_SKIP_KASAN_UNPOISON makes KASAN skip unpoisoning on page allocation. * Only effective in HW_TAGS mode. @@ -249,6 +252,7 @@ struct vm_area_struct; #define __GFP_COMP ((__force gfp_t)___GFP_COMP) #define __GFP_ZERO ((__force gfp_t)___GFP_ZERO) #define __GFP_ZEROTAGS ((__force gfp_t)___GFP_ZEROTAGS) +#define __GFP_SKIP_ZERO ((__force gfp_t)___GFP_SKIP_ZERO) #define __GFP_SKIP_KASAN_UNPOISON ((__force gfp_t)___GFP_SKIP_KASAN_UNPOISON) #define __GFP_SKIP_KASAN_POISON ((__force gfp_t)___GFP_SKIP_KASAN_POISON) @@ -257,7 +261,7 @@ struct vm_area_struct; /* Room for N __GFP_FOO bits */ #define __GFP_BITS_SHIFT (24 + \ - 2 * IS_ENABLED(CONFIG_KASAN_HW_TAGS) + \ + 3 * IS_ENABLED(CONFIG_KASAN_HW_TAGS) + \ IS_ENABLED(CONFIG_LOCKDEP)) #define __GFP_BITS_MASK ((__force gfp_t)((1 << __GFP_BITS_SHIFT) - 1)) diff --git a/include/trace/events/mmflags.h b/include/trace/events/mmflags.h index 5ffc7bdce91f..0698c5d0f194 100644 --- a/include/trace/events/mmflags.h +++ b/include/trace/events/mmflags.h @@ -52,6 +52,7 @@ #ifdef CONFIG_KASAN_HW_TAGS #define __def_gfpflag_names_kasan , \ + {(unsigned long)__GFP_SKIP_ZERO, "__GFP_SKIP_ZERO"}, \ {(unsigned long)__GFP_SKIP_KASAN_POISON, "__GFP_SKIP_KASAN_POISON"}, \ {(unsigned long)__GFP_SKIP_KASAN_UNPOISON, "__GFP_SKIP_KASAN_UNPOISON"} #else diff --git a/mm/page_alloc.c b/mm/page_alloc.c index 102f0cd8815e..30da0e1f94f8 100644 --- a/mm/page_alloc.c +++ b/mm/page_alloc.c @@ -2415,10 +2415,26 @@ static inline bool should_skip_kasan_unpoison(gfp_t flags, bool init_tags) return init_tags || (flags & __GFP_SKIP_KASAN_UNPOISON); } +static inline bool should_skip_init(gfp_t flags) +{ + /* Don't skip if a software KASAN mode is enabled. */ + if (IS_ENABLED(CONFIG_KASAN_GENERIC) || + IS_ENABLED(CONFIG_KASAN_SW_TAGS)) + return false; + + /* Don't skip, if hardware tag-based KASAN is not enabled. */ + if (!kasan_hw_tags_enabled()) + return false; + + /* For hardware tag-based KASAN, skip if requested. */ + return (flags & __GFP_SKIP_ZERO); +} + inline void post_alloc_hook(struct page *page, unsigned int order, gfp_t gfp_flags) { - bool init = !want_init_on_free() && want_init_on_alloc(gfp_flags); + bool init = !want_init_on_free() && want_init_on_alloc(gfp_flags) && + !should_skip_init(gfp_flags); bool init_tags = init && (gfp_flags & __GFP_ZEROTAGS); set_page_private(page, 0);