Message ID | ce573435398f21d3e604f104c29ba65eca70d9e7.1603372719.git.andreyknvl@google.com (mailing list archive) |
---|---|
State | New, archived |
Headers | show |
Series | kasan: hardware tag-based mode for production use on arm64 | expand |
On Thu, Oct 22, 2020 at 3:20 PM Andrey Konovalov <andreyknvl@google.com> wrote: > > Since kasan_kmalloc() always follows kasan_slab_alloc(), there's no need > to reunpoison the object data, only to poison the redzone. > > This requires changing kasan annotation for early SLUB cache to > kasan_slab_alloc(). Otherwise kasan_kmalloc() doesn't untag the object. > This doesn't do any functional changes, as kmem_cache_node->object_size > is equal to sizeof(struct kmem_cache_node). > > Similarly for kasan_krealloc(), as it's called after ksize(), which > already unpoisoned the object, there's no need to do it again. Have you considered doing this the other way around: make krealloc call __ksize and unpoison in kasan_krealloc? This has the advantage of more precise poisoning as ksize will unpoison the whole underlying object. But then maybe we will need to move first checks in ksize into __ksize as we may need them in krealloc as well. > Signed-off-by: Andrey Konovalov <andreyknvl@google.com> > Link: https://linux-review.googlesource.com/id/I4083d3b55605f70fef79bca9b90843c4390296f2 > --- > mm/kasan/common.c | 31 +++++++++++++++++++++---------- > mm/slub.c | 3 +-- > 2 files changed, 22 insertions(+), 12 deletions(-) > > diff --git a/mm/kasan/common.c b/mm/kasan/common.c > index c5ec60e1a4d2..a581937c2a44 100644 > --- a/mm/kasan/common.c > +++ b/mm/kasan/common.c > @@ -360,8 +360,14 @@ static void *____kasan_kmalloc(struct kmem_cache *cache, const void *object, > if (IS_ENABLED(CONFIG_KASAN_SW_TAGS) || IS_ENABLED(CONFIG_KASAN_HW_TAGS)) > tag = assign_tag(cache, object, false, keep_tag); > > - /* Tag is ignored in set_tag without CONFIG_KASAN_SW/HW_TAGS */ > - kasan_unpoison_memory(set_tag(object, tag), size); > + /* > + * Don't unpoison the object when keeping the tag. Tag is kept for: > + * 1. krealloc(), and then the memory has already been unpoisoned via ksize(); > + * 2. kmalloc(), and then the memory has already been unpoisoned by kasan_kmalloc(). > + * Tag is ignored in set_tag() without CONFIG_KASAN_SW/HW_TAGS. > + */ > + if (!keep_tag) > + kasan_unpoison_memory(set_tag(object, tag), size); > kasan_poison_memory((void *)redzone_start, redzone_end - redzone_start, > KASAN_KMALLOC_REDZONE); > > @@ -384,10 +390,9 @@ void * __must_check __kasan_kmalloc(struct kmem_cache *cache, const void *object > } > EXPORT_SYMBOL(__kasan_kmalloc); > > -void * __must_check __kasan_kmalloc_large(const void *ptr, size_t size, > - gfp_t flags) > +static void * __must_check ____kasan_kmalloc_large(struct page *page, const void *ptr, > + size_t size, gfp_t flags, bool realloc) > { > - struct page *page; > unsigned long redzone_start; > unsigned long redzone_end; > > @@ -397,18 +402,24 @@ void * __must_check __kasan_kmalloc_large(const void *ptr, size_t size, > if (unlikely(ptr == NULL)) > return NULL; > > - page = virt_to_page(ptr); > - redzone_start = round_up((unsigned long)(ptr + size), > - KASAN_GRANULE_SIZE); > + redzone_start = round_up((unsigned long)(ptr + size), KASAN_GRANULE_SIZE); > redzone_end = (unsigned long)ptr + page_size(page); > > - kasan_unpoison_memory(ptr, size); > + /* ksize() in __do_krealloc() already unpoisoned the memory. */ > + if (!realloc) > + kasan_unpoison_memory(ptr, size); > kasan_poison_memory((void *)redzone_start, redzone_end - redzone_start, > KASAN_PAGE_REDZONE); > > return (void *)ptr; > } > > +void * __must_check __kasan_kmalloc_large(const void *ptr, size_t size, > + gfp_t flags) > +{ > + return ____kasan_kmalloc_large(virt_to_page(ptr), ptr, size, flags, false); > +} > + > void * __must_check __kasan_krealloc(const void *object, size_t size, gfp_t flags) > { > struct page *page; > @@ -419,7 +430,7 @@ void * __must_check __kasan_krealloc(const void *object, size_t size, gfp_t flag > page = virt_to_head_page(object); > > if (unlikely(!PageSlab(page))) > - return __kasan_kmalloc_large(object, size, flags); > + return ____kasan_kmalloc_large(page, object, size, flags, true); > else > return ____kasan_kmalloc(page->slab_cache, object, size, > flags, true); > diff --git a/mm/slub.c b/mm/slub.c > index 1d3f2355df3b..afb035b0bf2d 100644 > --- a/mm/slub.c > +++ b/mm/slub.c > @@ -3535,8 +3535,7 @@ static void early_kmem_cache_node_alloc(int node) > init_object(kmem_cache_node, n, SLUB_RED_ACTIVE); > init_tracking(kmem_cache_node, n); > #endif > - n = kasan_kmalloc(kmem_cache_node, n, sizeof(struct kmem_cache_node), > - GFP_KERNEL); > + n = kasan_slab_alloc(kmem_cache_node, n, GFP_KERNEL); > page->freelist = get_freepointer(kmem_cache_node, n); > page->inuse = 1; > page->frozen = 0; > -- > 2.29.0.rc1.297.gfa9743e501-goog >
On Wed, Oct 28, 2020 at 5:55 PM Dmitry Vyukov <dvyukov@google.com> wrote: > > On Thu, Oct 22, 2020 at 3:20 PM Andrey Konovalov <andreyknvl@google.com> wrote: > > > > Since kasan_kmalloc() always follows kasan_slab_alloc(), there's no need > > to reunpoison the object data, only to poison the redzone. > > > > This requires changing kasan annotation for early SLUB cache to > > kasan_slab_alloc(). Otherwise kasan_kmalloc() doesn't untag the object. > > This doesn't do any functional changes, as kmem_cache_node->object_size > > is equal to sizeof(struct kmem_cache_node). > > > > Similarly for kasan_krealloc(), as it's called after ksize(), which > > already unpoisoned the object, there's no need to do it again. > > Have you considered doing this the other way around: make krealloc > call __ksize and unpoison in kasan_krealloc? > This has the advantage of more precise poisoning as ksize will > unpoison the whole underlying object. > > But then maybe we will need to move first checks in ksize into __ksize > as we may need them in krealloc as well. This might be a good idea. I won't implement this for the next version, but will look into this after that. Thanks! > > > > > > > Signed-off-by: Andrey Konovalov <andreyknvl@google.com> > > Link: https://linux-review.googlesource.com/id/I4083d3b55605f70fef79bca9b90843c4390296f2 > > --- > > mm/kasan/common.c | 31 +++++++++++++++++++++---------- > > mm/slub.c | 3 +-- > > 2 files changed, 22 insertions(+), 12 deletions(-) > > > > diff --git a/mm/kasan/common.c b/mm/kasan/common.c > > index c5ec60e1a4d2..a581937c2a44 100644 > > --- a/mm/kasan/common.c > > +++ b/mm/kasan/common.c > > @@ -360,8 +360,14 @@ static void *____kasan_kmalloc(struct kmem_cache *cache, const void *object, > > if (IS_ENABLED(CONFIG_KASAN_SW_TAGS) || IS_ENABLED(CONFIG_KASAN_HW_TAGS)) > > tag = assign_tag(cache, object, false, keep_tag); > > > > - /* Tag is ignored in set_tag without CONFIG_KASAN_SW/HW_TAGS */ > > - kasan_unpoison_memory(set_tag(object, tag), size); > > + /* > > + * Don't unpoison the object when keeping the tag. Tag is kept for: > > + * 1. krealloc(), and then the memory has already been unpoisoned via ksize(); > > + * 2. kmalloc(), and then the memory has already been unpoisoned by kasan_kmalloc(). > > + * Tag is ignored in set_tag() without CONFIG_KASAN_SW/HW_TAGS. > > + */ > > + if (!keep_tag) > > + kasan_unpoison_memory(set_tag(object, tag), size); > > kasan_poison_memory((void *)redzone_start, redzone_end - redzone_start, > > KASAN_KMALLOC_REDZONE); > > > > @@ -384,10 +390,9 @@ void * __must_check __kasan_kmalloc(struct kmem_cache *cache, const void *object > > } > > EXPORT_SYMBOL(__kasan_kmalloc); > > > > -void * __must_check __kasan_kmalloc_large(const void *ptr, size_t size, > > - gfp_t flags) > > +static void * __must_check ____kasan_kmalloc_large(struct page *page, const void *ptr, > > + size_t size, gfp_t flags, bool realloc) > > { > > - struct page *page; > > unsigned long redzone_start; > > unsigned long redzone_end; > > > > @@ -397,18 +402,24 @@ void * __must_check __kasan_kmalloc_large(const void *ptr, size_t size, > > if (unlikely(ptr == NULL)) > > return NULL; > > > > - page = virt_to_page(ptr); > > - redzone_start = round_up((unsigned long)(ptr + size), > > - KASAN_GRANULE_SIZE); > > + redzone_start = round_up((unsigned long)(ptr + size), KASAN_GRANULE_SIZE); > > redzone_end = (unsigned long)ptr + page_size(page); > > > > - kasan_unpoison_memory(ptr, size); > > + /* ksize() in __do_krealloc() already unpoisoned the memory. */ > > + if (!realloc) > > + kasan_unpoison_memory(ptr, size); > > kasan_poison_memory((void *)redzone_start, redzone_end - redzone_start, > > KASAN_PAGE_REDZONE); > > > > return (void *)ptr; > > } > > > > +void * __must_check __kasan_kmalloc_large(const void *ptr, size_t size, > > + gfp_t flags) > > +{ > > + return ____kasan_kmalloc_large(virt_to_page(ptr), ptr, size, flags, false); > > +} > > + > > void * __must_check __kasan_krealloc(const void *object, size_t size, gfp_t flags) > > { > > struct page *page; > > @@ -419,7 +430,7 @@ void * __must_check __kasan_krealloc(const void *object, size_t size, gfp_t flag > > page = virt_to_head_page(object); > > > > if (unlikely(!PageSlab(page))) > > - return __kasan_kmalloc_large(object, size, flags); > > + return ____kasan_kmalloc_large(page, object, size, flags, true); > > else > > return ____kasan_kmalloc(page->slab_cache, object, size, > > flags, true); > > diff --git a/mm/slub.c b/mm/slub.c > > index 1d3f2355df3b..afb035b0bf2d 100644 > > --- a/mm/slub.c > > +++ b/mm/slub.c > > @@ -3535,8 +3535,7 @@ static void early_kmem_cache_node_alloc(int node) > > init_object(kmem_cache_node, n, SLUB_RED_ACTIVE); > > init_tracking(kmem_cache_node, n); > > #endif > > - n = kasan_kmalloc(kmem_cache_node, n, sizeof(struct kmem_cache_node), > > - GFP_KERNEL); > > + n = kasan_slab_alloc(kmem_cache_node, n, GFP_KERNEL); > > page->freelist = get_freepointer(kmem_cache_node, n); > > page->inuse = 1; > > page->frozen = 0; > > -- > > 2.29.0.rc1.297.gfa9743e501-goog > >
diff --git a/mm/kasan/common.c b/mm/kasan/common.c index c5ec60e1a4d2..a581937c2a44 100644 --- a/mm/kasan/common.c +++ b/mm/kasan/common.c @@ -360,8 +360,14 @@ static void *____kasan_kmalloc(struct kmem_cache *cache, const void *object, if (IS_ENABLED(CONFIG_KASAN_SW_TAGS) || IS_ENABLED(CONFIG_KASAN_HW_TAGS)) tag = assign_tag(cache, object, false, keep_tag); - /* Tag is ignored in set_tag without CONFIG_KASAN_SW/HW_TAGS */ - kasan_unpoison_memory(set_tag(object, tag), size); + /* + * Don't unpoison the object when keeping the tag. Tag is kept for: + * 1. krealloc(), and then the memory has already been unpoisoned via ksize(); + * 2. kmalloc(), and then the memory has already been unpoisoned by kasan_kmalloc(). + * Tag is ignored in set_tag() without CONFIG_KASAN_SW/HW_TAGS. + */ + if (!keep_tag) + kasan_unpoison_memory(set_tag(object, tag), size); kasan_poison_memory((void *)redzone_start, redzone_end - redzone_start, KASAN_KMALLOC_REDZONE); @@ -384,10 +390,9 @@ void * __must_check __kasan_kmalloc(struct kmem_cache *cache, const void *object } EXPORT_SYMBOL(__kasan_kmalloc); -void * __must_check __kasan_kmalloc_large(const void *ptr, size_t size, - gfp_t flags) +static void * __must_check ____kasan_kmalloc_large(struct page *page, const void *ptr, + size_t size, gfp_t flags, bool realloc) { - struct page *page; unsigned long redzone_start; unsigned long redzone_end; @@ -397,18 +402,24 @@ void * __must_check __kasan_kmalloc_large(const void *ptr, size_t size, if (unlikely(ptr == NULL)) return NULL; - page = virt_to_page(ptr); - redzone_start = round_up((unsigned long)(ptr + size), - KASAN_GRANULE_SIZE); + redzone_start = round_up((unsigned long)(ptr + size), KASAN_GRANULE_SIZE); redzone_end = (unsigned long)ptr + page_size(page); - kasan_unpoison_memory(ptr, size); + /* ksize() in __do_krealloc() already unpoisoned the memory. */ + if (!realloc) + kasan_unpoison_memory(ptr, size); kasan_poison_memory((void *)redzone_start, redzone_end - redzone_start, KASAN_PAGE_REDZONE); return (void *)ptr; } +void * __must_check __kasan_kmalloc_large(const void *ptr, size_t size, + gfp_t flags) +{ + return ____kasan_kmalloc_large(virt_to_page(ptr), ptr, size, flags, false); +} + void * __must_check __kasan_krealloc(const void *object, size_t size, gfp_t flags) { struct page *page; @@ -419,7 +430,7 @@ void * __must_check __kasan_krealloc(const void *object, size_t size, gfp_t flag page = virt_to_head_page(object); if (unlikely(!PageSlab(page))) - return __kasan_kmalloc_large(object, size, flags); + return ____kasan_kmalloc_large(page, object, size, flags, true); else return ____kasan_kmalloc(page->slab_cache, object, size, flags, true); diff --git a/mm/slub.c b/mm/slub.c index 1d3f2355df3b..afb035b0bf2d 100644 --- a/mm/slub.c +++ b/mm/slub.c @@ -3535,8 +3535,7 @@ static void early_kmem_cache_node_alloc(int node) init_object(kmem_cache_node, n, SLUB_RED_ACTIVE); init_tracking(kmem_cache_node, n); #endif - n = kasan_kmalloc(kmem_cache_node, n, sizeof(struct kmem_cache_node), - GFP_KERNEL); + n = kasan_slab_alloc(kmem_cache_node, n, GFP_KERNEL); page->freelist = get_freepointer(kmem_cache_node, n); page->inuse = 1; page->frozen = 0;
Since kasan_kmalloc() always follows kasan_slab_alloc(), there's no need to reunpoison the object data, only to poison the redzone. This requires changing kasan annotation for early SLUB cache to kasan_slab_alloc(). Otherwise kasan_kmalloc() doesn't untag the object. This doesn't do any functional changes, as kmem_cache_node->object_size is equal to sizeof(struct kmem_cache_node). Similarly for kasan_krealloc(), as it's called after ksize(), which already unpoisoned the object, there's no need to do it again. Signed-off-by: Andrey Konovalov <andreyknvl@google.com> Link: https://linux-review.googlesource.com/id/I4083d3b55605f70fef79bca9b90843c4390296f2 --- mm/kasan/common.c | 31 +++++++++++++++++++++---------- mm/slub.c | 3 +-- 2 files changed, 22 insertions(+), 12 deletions(-)