diff mbox series

[RFC,04/10] mm, kfence: insert KFENCE hooks for SLAB

Message ID 20200907134055.2878499-5-elver@google.com
State New
Headers show
Series KFENCE: A low-overhead sampling-based memory safety error detector | expand

Commit Message

Marco Elver Sept. 7, 2020, 1:40 p.m. UTC
From: Alexander Potapenko <glider@google.com>

Inserts KFENCE hooks into the SLAB allocator.

We note the addition of the 'orig_size' argument to slab_alloc*()
functions, to be able to pass the originally requested size to KFENCE.
When KFENCE is disabled, there is no additional overhead, since these
functions are __always_inline.

Co-developed-by: Marco Elver <elver@google.com>
Signed-off-by: Marco Elver <elver@google.com>
Signed-off-by: Alexander Potapenko <glider@google.com>
---
 mm/slab.c        | 46 ++++++++++++++++++++++++++++++++++------------
 mm/slab_common.c |  6 +++++-
 2 files changed, 39 insertions(+), 13 deletions(-)

Comments

Dmitry Vyukov Sept. 11, 2020, 7:17 a.m. UTC | #1
On Mon, Sep 7, 2020 at 3:41 PM Marco Elver <elver@google.com> wrote:
>
> From: Alexander Potapenko <glider@google.com>
>
> Inserts KFENCE hooks into the SLAB allocator.
>
> We note the addition of the 'orig_size' argument to slab_alloc*()
> functions, to be able to pass the originally requested size to KFENCE.
> When KFENCE is disabled, there is no additional overhead, since these
> functions are __always_inline.
>
> Co-developed-by: Marco Elver <elver@google.com>
> Signed-off-by: Marco Elver <elver@google.com>
> Signed-off-by: Alexander Potapenko <glider@google.com>
> ---
>  mm/slab.c        | 46 ++++++++++++++++++++++++++++++++++------------
>  mm/slab_common.c |  6 +++++-
>  2 files changed, 39 insertions(+), 13 deletions(-)
>
> diff --git a/mm/slab.c b/mm/slab.c
> index 3160dff6fd76..30aba06ae02b 100644
> --- a/mm/slab.c
> +++ b/mm/slab.c
> @@ -100,6 +100,7 @@
>  #include       <linux/seq_file.h>
>  #include       <linux/notifier.h>
>  #include       <linux/kallsyms.h>
> +#include       <linux/kfence.h>
>  #include       <linux/cpu.h>
>  #include       <linux/sysctl.h>
>  #include       <linux/module.h>
> @@ -3206,7 +3207,7 @@ static void *____cache_alloc_node(struct kmem_cache *cachep, gfp_t flags,
>  }
>
>  static __always_inline void *
> -slab_alloc_node(struct kmem_cache *cachep, gfp_t flags, int nodeid,
> +slab_alloc_node(struct kmem_cache *cachep, gfp_t flags, int nodeid, size_t orig_size,
>                    unsigned long caller)
>  {
>         unsigned long save_flags;
> @@ -3219,6 +3220,10 @@ slab_alloc_node(struct kmem_cache *cachep, gfp_t flags, int nodeid,
>         if (unlikely(!cachep))
>                 return NULL;
>
> +       ptr = kfence_alloc(cachep, orig_size, flags);
> +       if (unlikely(ptr))
> +               goto out_hooks;
> +
>         cache_alloc_debugcheck_before(cachep, flags);
>         local_irq_save(save_flags);
>
> @@ -3251,6 +3256,7 @@ slab_alloc_node(struct kmem_cache *cachep, gfp_t flags, int nodeid,
>         if (unlikely(slab_want_init_on_alloc(flags, cachep)) && ptr)
>                 memset(ptr, 0, cachep->object_size);
>
> +out_hooks:
>         slab_post_alloc_hook(cachep, objcg, flags, 1, &ptr);
>         return ptr;
>  }
> @@ -3288,7 +3294,7 @@ __do_cache_alloc(struct kmem_cache *cachep, gfp_t flags)
>  #endif /* CONFIG_NUMA */
>
>  static __always_inline void *
> -slab_alloc(struct kmem_cache *cachep, gfp_t flags, unsigned long caller)
> +slab_alloc(struct kmem_cache *cachep, gfp_t flags, size_t orig_size, unsigned long caller)
>  {
>         unsigned long save_flags;
>         void *objp;
> @@ -3299,6 +3305,10 @@ slab_alloc(struct kmem_cache *cachep, gfp_t flags, unsigned long caller)
>         if (unlikely(!cachep))
>                 return NULL;
>
> +       objp = kfence_alloc(cachep, orig_size, flags);
> +       if (unlikely(objp))
> +               goto leave;
> +
>         cache_alloc_debugcheck_before(cachep, flags);
>         local_irq_save(save_flags);
>         objp = __do_cache_alloc(cachep, flags);
> @@ -3309,6 +3319,7 @@ slab_alloc(struct kmem_cache *cachep, gfp_t flags, unsigned long caller)
>         if (unlikely(slab_want_init_on_alloc(flags, cachep)) && objp)
>                 memset(objp, 0, cachep->object_size);
>
> +leave:
>         slab_post_alloc_hook(cachep, objcg, flags, 1, &objp);
>         return objp;
>  }
> @@ -3414,6 +3425,11 @@ static void cache_flusharray(struct kmem_cache *cachep, struct array_cache *ac)
>  static __always_inline void __cache_free(struct kmem_cache *cachep, void *objp,
>                                          unsigned long caller)
>  {
> +       if (kfence_free(objp)) {
> +               kmemleak_free_recursive(objp, cachep->flags);
> +               return;
> +       }
> +
>         /* Put the object into the quarantine, don't touch it for now. */
>         if (kasan_slab_free(cachep, objp, _RET_IP_))
>                 return;
> @@ -3479,7 +3495,7 @@ void ___cache_free(struct kmem_cache *cachep, void *objp,
>   */
>  void *kmem_cache_alloc(struct kmem_cache *cachep, gfp_t flags)
>  {
> -       void *ret = slab_alloc(cachep, flags, _RET_IP_);
> +       void *ret = slab_alloc(cachep, flags, cachep->object_size, _RET_IP_);


It's kinda minor, but since we are talking about malloc fast path:
will passing 0 instead of cachep->object_size (here and everywhere
else) and then using cachep->object_size on the slow path if 0 is
passed as size improve codegen?


>         trace_kmem_cache_alloc(_RET_IP_, ret,
>                                cachep->object_size, cachep->size, flags);
> @@ -3512,7 +3528,7 @@ int kmem_cache_alloc_bulk(struct kmem_cache *s, gfp_t flags, size_t size,
>
>         local_irq_disable();
>         for (i = 0; i < size; i++) {
> -               void *objp = __do_cache_alloc(s, flags);
> +               void *objp = kfence_alloc(s, s->object_size, flags) ?: __do_cache_alloc(s, flags);
>
>                 if (unlikely(!objp))
>                         goto error;
> @@ -3545,7 +3561,7 @@ kmem_cache_alloc_trace(struct kmem_cache *cachep, gfp_t flags, size_t size)
>  {
>         void *ret;
>
> -       ret = slab_alloc(cachep, flags, _RET_IP_);
> +       ret = slab_alloc(cachep, flags, size, _RET_IP_);
>
>         ret = kasan_kmalloc(cachep, ret, size, flags);
>         trace_kmalloc(_RET_IP_, ret,
> @@ -3571,7 +3587,7 @@ EXPORT_SYMBOL(kmem_cache_alloc_trace);
>   */
>  void *kmem_cache_alloc_node(struct kmem_cache *cachep, gfp_t flags, int nodeid)
>  {
> -       void *ret = slab_alloc_node(cachep, flags, nodeid, _RET_IP_);
> +       void *ret = slab_alloc_node(cachep, flags, nodeid, cachep->object_size, _RET_IP_);
>
>         trace_kmem_cache_alloc_node(_RET_IP_, ret,
>                                     cachep->object_size, cachep->size,
> @@ -3589,7 +3605,7 @@ void *kmem_cache_alloc_node_trace(struct kmem_cache *cachep,
>  {
>         void *ret;
>
> -       ret = slab_alloc_node(cachep, flags, nodeid, _RET_IP_);
> +       ret = slab_alloc_node(cachep, flags, nodeid, size, _RET_IP_);
>
>         ret = kasan_kmalloc(cachep, ret, size, flags);
>         trace_kmalloc_node(_RET_IP_, ret,
> @@ -3650,7 +3666,7 @@ static __always_inline void *__do_kmalloc(size_t size, gfp_t flags,
>         cachep = kmalloc_slab(size, flags);
>         if (unlikely(ZERO_OR_NULL_PTR(cachep)))
>                 return cachep;
> -       ret = slab_alloc(cachep, flags, caller);
> +       ret = slab_alloc(cachep, flags, size, caller);
>
>         ret = kasan_kmalloc(cachep, ret, size, flags);
>         trace_kmalloc(caller, ret,
> @@ -4138,18 +4154,24 @@ void __check_heap_object(const void *ptr, unsigned long n, struct page *page,
>                          bool to_user)
>  {
>         struct kmem_cache *cachep;
> -       unsigned int objnr;
> +       unsigned int objnr = 0;
>         unsigned long offset;
> +       bool is_kfence = is_kfence_address(ptr);
>
>         ptr = kasan_reset_tag(ptr);
>
>         /* Find and validate object. */
>         cachep = page->slab_cache;
> -       objnr = obj_to_index(cachep, page, (void *)ptr);
> -       BUG_ON(objnr >= cachep->num);
> +       if (!is_kfence) {
> +               objnr = obj_to_index(cachep, page, (void *)ptr);
> +               BUG_ON(objnr >= cachep->num);
> +       }
>
>         /* Find offset within object. */
> -       offset = ptr - index_to_obj(cachep, page, objnr) - obj_offset(cachep);
> +       if (is_kfence_address(ptr))
> +               offset = ptr - kfence_object_start(ptr);
> +       else
> +               offset = ptr - index_to_obj(cachep, page, objnr) - obj_offset(cachep);
>
>         /* Allow address range falling entirely within usercopy region. */
>         if (offset >= cachep->useroffset &&
> diff --git a/mm/slab_common.c b/mm/slab_common.c
> index f9ccd5dc13f3..6e35e273681a 100644
> --- a/mm/slab_common.c
> +++ b/mm/slab_common.c
> @@ -12,6 +12,7 @@
>  #include <linux/memory.h>
>  #include <linux/cache.h>
>  #include <linux/compiler.h>
> +#include <linux/kfence.h>
>  #include <linux/module.h>
>  #include <linux/cpu.h>
>  #include <linux/uaccess.h>
> @@ -448,6 +449,9 @@ static int shutdown_cache(struct kmem_cache *s)
>         /* free asan quarantined objects */
>         kasan_cache_shutdown(s);
>
> +       if (!kfence_shutdown_cache(s))
> +               return -EBUSY;
> +
>         if (__kmem_cache_shutdown(s) != 0)
>                 return -EBUSY;
>
> @@ -1171,7 +1175,7 @@ size_t ksize(const void *objp)
>         if (unlikely(ZERO_OR_NULL_PTR(objp)) || !__kasan_check_read(objp, 1))
>                 return 0;
>
> -       size = __ksize(objp);
> +       size = kfence_ksize(objp) ?: __ksize(objp);
>         /*
>          * We assume that ksize callers could use whole allocated area,
>          * so we need to unpoison this area.
> --
> 2.28.0.526.ge36021eeef-goog
>
Marco Elver Sept. 11, 2020, 12:24 p.m. UTC | #2
On Fri, 11 Sep 2020 at 09:17, Dmitry Vyukov <dvyukov@google.com> wrote:
>
> On Mon, Sep 7, 2020 at 3:41 PM Marco Elver <elver@google.com> wrote:
> >
> > From: Alexander Potapenko <glider@google.com>
> >
> > Inserts KFENCE hooks into the SLAB allocator.
> >
> > We note the addition of the 'orig_size' argument to slab_alloc*()
> > functions, to be able to pass the originally requested size to KFENCE.
> > When KFENCE is disabled, there is no additional overhead, since these
> > functions are __always_inline.
> >
> > Co-developed-by: Marco Elver <elver@google.com>
> > Signed-off-by: Marco Elver <elver@google.com>
> > Signed-off-by: Alexander Potapenko <glider@google.com>
> > ---
> >  mm/slab.c        | 46 ++++++++++++++++++++++++++++++++++------------
> >  mm/slab_common.c |  6 +++++-
> >  2 files changed, 39 insertions(+), 13 deletions(-)
> >
> > diff --git a/mm/slab.c b/mm/slab.c
> > index 3160dff6fd76..30aba06ae02b 100644
> > --- a/mm/slab.c
> > +++ b/mm/slab.c
> > @@ -100,6 +100,7 @@
> >  #include       <linux/seq_file.h>
> >  #include       <linux/notifier.h>
> >  #include       <linux/kallsyms.h>
> > +#include       <linux/kfence.h>
> >  #include       <linux/cpu.h>
> >  #include       <linux/sysctl.h>
> >  #include       <linux/module.h>
> > @@ -3206,7 +3207,7 @@ static void *____cache_alloc_node(struct kmem_cache *cachep, gfp_t flags,
> >  }
> >
> >  static __always_inline void *
> > -slab_alloc_node(struct kmem_cache *cachep, gfp_t flags, int nodeid,
> > +slab_alloc_node(struct kmem_cache *cachep, gfp_t flags, int nodeid, size_t orig_size,
> >                    unsigned long caller)
> >  {
> >         unsigned long save_flags;
> > @@ -3219,6 +3220,10 @@ slab_alloc_node(struct kmem_cache *cachep, gfp_t flags, int nodeid,
> >         if (unlikely(!cachep))
> >                 return NULL;
> >
> > +       ptr = kfence_alloc(cachep, orig_size, flags);
> > +       if (unlikely(ptr))
> > +               goto out_hooks;
> > +
> >         cache_alloc_debugcheck_before(cachep, flags);
> >         local_irq_save(save_flags);
> >
> > @@ -3251,6 +3256,7 @@ slab_alloc_node(struct kmem_cache *cachep, gfp_t flags, int nodeid,
> >         if (unlikely(slab_want_init_on_alloc(flags, cachep)) && ptr)
> >                 memset(ptr, 0, cachep->object_size);
> >
> > +out_hooks:
> >         slab_post_alloc_hook(cachep, objcg, flags, 1, &ptr);
> >         return ptr;
> >  }
> > @@ -3288,7 +3294,7 @@ __do_cache_alloc(struct kmem_cache *cachep, gfp_t flags)
> >  #endif /* CONFIG_NUMA */
> >
> >  static __always_inline void *
> > -slab_alloc(struct kmem_cache *cachep, gfp_t flags, unsigned long caller)
> > +slab_alloc(struct kmem_cache *cachep, gfp_t flags, size_t orig_size, unsigned long caller)
> >  {
> >         unsigned long save_flags;
> >         void *objp;
> > @@ -3299,6 +3305,10 @@ slab_alloc(struct kmem_cache *cachep, gfp_t flags, unsigned long caller)
> >         if (unlikely(!cachep))
> >                 return NULL;
> >
> > +       objp = kfence_alloc(cachep, orig_size, flags);
> > +       if (unlikely(objp))
> > +               goto leave;
> > +
> >         cache_alloc_debugcheck_before(cachep, flags);
> >         local_irq_save(save_flags);
> >         objp = __do_cache_alloc(cachep, flags);
> > @@ -3309,6 +3319,7 @@ slab_alloc(struct kmem_cache *cachep, gfp_t flags, unsigned long caller)
> >         if (unlikely(slab_want_init_on_alloc(flags, cachep)) && objp)
> >                 memset(objp, 0, cachep->object_size);
> >
> > +leave:
> >         slab_post_alloc_hook(cachep, objcg, flags, 1, &objp);
> >         return objp;
> >  }
> > @@ -3414,6 +3425,11 @@ static void cache_flusharray(struct kmem_cache *cachep, struct array_cache *ac)
> >  static __always_inline void __cache_free(struct kmem_cache *cachep, void *objp,
> >                                          unsigned long caller)
> >  {
> > +       if (kfence_free(objp)) {
> > +               kmemleak_free_recursive(objp, cachep->flags);
> > +               return;
> > +       }
> > +
> >         /* Put the object into the quarantine, don't touch it for now. */
> >         if (kasan_slab_free(cachep, objp, _RET_IP_))
> >                 return;
> > @@ -3479,7 +3495,7 @@ void ___cache_free(struct kmem_cache *cachep, void *objp,
> >   */
> >  void *kmem_cache_alloc(struct kmem_cache *cachep, gfp_t flags)
> >  {
> > -       void *ret = slab_alloc(cachep, flags, _RET_IP_);
> > +       void *ret = slab_alloc(cachep, flags, cachep->object_size, _RET_IP_);
>
>
> It's kinda minor, but since we are talking about malloc fast path:
> will passing 0 instead of cachep->object_size (here and everywhere
> else) and then using cachep->object_size on the slow path if 0 is
> passed as size improve codegen?

It doesn't save us much, maybe 1 instruction based on what I'm looking
at right now. The main worry I have is that the 'orig_size' argument
is now part of slab_alloc, and changing its semantics may cause
problems in future if it's no longer just passed to kfence_alloc().
Today, we can do the 'size = size ?: cache->object_size' trick inside
kfence_alloc(), but at the cost breaking the intuitive semantics of
slab_alloc's orig_size argument for future users. Is it worth it?

Thanks,
-- Marco
Dmitry Vyukov Sept. 11, 2020, 1:03 p.m. UTC | #3
On Fri, Sep 11, 2020 at 2:24 PM Marco Elver <elver@google.com> wrote:
> > > From: Alexander Potapenko <glider@google.com>
> > >
> > > Inserts KFENCE hooks into the SLAB allocator.
> > >
> > > We note the addition of the 'orig_size' argument to slab_alloc*()
> > > functions, to be able to pass the originally requested size to KFENCE.
> > > When KFENCE is disabled, there is no additional overhead, since these
> > > functions are __always_inline.
> > >
> > > Co-developed-by: Marco Elver <elver@google.com>
> > > Signed-off-by: Marco Elver <elver@google.com>
> > > Signed-off-by: Alexander Potapenko <glider@google.com>
> > > ---
> > >  mm/slab.c        | 46 ++++++++++++++++++++++++++++++++++------------
> > >  mm/slab_common.c |  6 +++++-
> > >  2 files changed, 39 insertions(+), 13 deletions(-)
> > >
> > > diff --git a/mm/slab.c b/mm/slab.c
> > > index 3160dff6fd76..30aba06ae02b 100644
> > > --- a/mm/slab.c
> > > +++ b/mm/slab.c
> > > @@ -100,6 +100,7 @@
> > >  #include       <linux/seq_file.h>
> > >  #include       <linux/notifier.h>
> > >  #include       <linux/kallsyms.h>
> > > +#include       <linux/kfence.h>
> > >  #include       <linux/cpu.h>
> > >  #include       <linux/sysctl.h>
> > >  #include       <linux/module.h>
> > > @@ -3206,7 +3207,7 @@ static void *____cache_alloc_node(struct kmem_cache *cachep, gfp_t flags,
> > >  }
> > >
> > >  static __always_inline void *
> > > -slab_alloc_node(struct kmem_cache *cachep, gfp_t flags, int nodeid,
> > > +slab_alloc_node(struct kmem_cache *cachep, gfp_t flags, int nodeid, size_t orig_size,
> > >                    unsigned long caller)
> > >  {
> > >         unsigned long save_flags;
> > > @@ -3219,6 +3220,10 @@ slab_alloc_node(struct kmem_cache *cachep, gfp_t flags, int nodeid,
> > >         if (unlikely(!cachep))
> > >                 return NULL;
> > >
> > > +       ptr = kfence_alloc(cachep, orig_size, flags);
> > > +       if (unlikely(ptr))
> > > +               goto out_hooks;
> > > +
> > >         cache_alloc_debugcheck_before(cachep, flags);
> > >         local_irq_save(save_flags);
> > >
> > > @@ -3251,6 +3256,7 @@ slab_alloc_node(struct kmem_cache *cachep, gfp_t flags, int nodeid,
> > >         if (unlikely(slab_want_init_on_alloc(flags, cachep)) && ptr)
> > >                 memset(ptr, 0, cachep->object_size);
> > >
> > > +out_hooks:
> > >         slab_post_alloc_hook(cachep, objcg, flags, 1, &ptr);
> > >         return ptr;
> > >  }
> > > @@ -3288,7 +3294,7 @@ __do_cache_alloc(struct kmem_cache *cachep, gfp_t flags)
> > >  #endif /* CONFIG_NUMA */
> > >
> > >  static __always_inline void *
> > > -slab_alloc(struct kmem_cache *cachep, gfp_t flags, unsigned long caller)
> > > +slab_alloc(struct kmem_cache *cachep, gfp_t flags, size_t orig_size, unsigned long caller)
> > >  {
> > >         unsigned long save_flags;
> > >         void *objp;
> > > @@ -3299,6 +3305,10 @@ slab_alloc(struct kmem_cache *cachep, gfp_t flags, unsigned long caller)
> > >         if (unlikely(!cachep))
> > >                 return NULL;
> > >
> > > +       objp = kfence_alloc(cachep, orig_size, flags);
> > > +       if (unlikely(objp))
> > > +               goto leave;
> > > +
> > >         cache_alloc_debugcheck_before(cachep, flags);
> > >         local_irq_save(save_flags);
> > >         objp = __do_cache_alloc(cachep, flags);
> > > @@ -3309,6 +3319,7 @@ slab_alloc(struct kmem_cache *cachep, gfp_t flags, unsigned long caller)
> > >         if (unlikely(slab_want_init_on_alloc(flags, cachep)) && objp)
> > >                 memset(objp, 0, cachep->object_size);
> > >
> > > +leave:
> > >         slab_post_alloc_hook(cachep, objcg, flags, 1, &objp);
> > >         return objp;
> > >  }
> > > @@ -3414,6 +3425,11 @@ static void cache_flusharray(struct kmem_cache *cachep, struct array_cache *ac)
> > >  static __always_inline void __cache_free(struct kmem_cache *cachep, void *objp,
> > >                                          unsigned long caller)
> > >  {
> > > +       if (kfence_free(objp)) {
> > > +               kmemleak_free_recursive(objp, cachep->flags);
> > > +               return;
> > > +       }
> > > +
> > >         /* Put the object into the quarantine, don't touch it for now. */
> > >         if (kasan_slab_free(cachep, objp, _RET_IP_))
> > >                 return;
> > > @@ -3479,7 +3495,7 @@ void ___cache_free(struct kmem_cache *cachep, void *objp,
> > >   */
> > >  void *kmem_cache_alloc(struct kmem_cache *cachep, gfp_t flags)
> > >  {
> > > -       void *ret = slab_alloc(cachep, flags, _RET_IP_);
> > > +       void *ret = slab_alloc(cachep, flags, cachep->object_size, _RET_IP_);
> >
> >
> > It's kinda minor, but since we are talking about malloc fast path:
> > will passing 0 instead of cachep->object_size (here and everywhere
> > else) and then using cachep->object_size on the slow path if 0 is
> > passed as size improve codegen?
>
> It doesn't save us much, maybe 1 instruction based on what I'm looking
> at right now. The main worry I have is that the 'orig_size' argument
> is now part of slab_alloc, and changing its semantics may cause
> problems in future if it's no longer just passed to kfence_alloc().
> Today, we can do the 'size = size ?: cache->object_size' trick inside
> kfence_alloc(), but at the cost breaking the intuitive semantics of
> slab_alloc's orig_size argument for future users. Is it worth it?

I don't have an answer to this question. I will leave this to others.
If nobody has strong support for changing semantics, let's leave it as
is. Maybe keep in mind as potential ballast.
FWIW most likely misuse of 0 size for other future purposes should
manifest itself in a quite straightforward way.
diff mbox series

Patch

diff --git a/mm/slab.c b/mm/slab.c
index 3160dff6fd76..30aba06ae02b 100644
--- a/mm/slab.c
+++ b/mm/slab.c
@@ -100,6 +100,7 @@ 
 #include	<linux/seq_file.h>
 #include	<linux/notifier.h>
 #include	<linux/kallsyms.h>
+#include	<linux/kfence.h>
 #include	<linux/cpu.h>
 #include	<linux/sysctl.h>
 #include	<linux/module.h>
@@ -3206,7 +3207,7 @@  static void *____cache_alloc_node(struct kmem_cache *cachep, gfp_t flags,
 }
 
 static __always_inline void *
-slab_alloc_node(struct kmem_cache *cachep, gfp_t flags, int nodeid,
+slab_alloc_node(struct kmem_cache *cachep, gfp_t flags, int nodeid, size_t orig_size,
 		   unsigned long caller)
 {
 	unsigned long save_flags;
@@ -3219,6 +3220,10 @@  slab_alloc_node(struct kmem_cache *cachep, gfp_t flags, int nodeid,
 	if (unlikely(!cachep))
 		return NULL;
 
+	ptr = kfence_alloc(cachep, orig_size, flags);
+	if (unlikely(ptr))
+		goto out_hooks;
+
 	cache_alloc_debugcheck_before(cachep, flags);
 	local_irq_save(save_flags);
 
@@ -3251,6 +3256,7 @@  slab_alloc_node(struct kmem_cache *cachep, gfp_t flags, int nodeid,
 	if (unlikely(slab_want_init_on_alloc(flags, cachep)) && ptr)
 		memset(ptr, 0, cachep->object_size);
 
+out_hooks:
 	slab_post_alloc_hook(cachep, objcg, flags, 1, &ptr);
 	return ptr;
 }
@@ -3288,7 +3294,7 @@  __do_cache_alloc(struct kmem_cache *cachep, gfp_t flags)
 #endif /* CONFIG_NUMA */
 
 static __always_inline void *
-slab_alloc(struct kmem_cache *cachep, gfp_t flags, unsigned long caller)
+slab_alloc(struct kmem_cache *cachep, gfp_t flags, size_t orig_size, unsigned long caller)
 {
 	unsigned long save_flags;
 	void *objp;
@@ -3299,6 +3305,10 @@  slab_alloc(struct kmem_cache *cachep, gfp_t flags, unsigned long caller)
 	if (unlikely(!cachep))
 		return NULL;
 
+	objp = kfence_alloc(cachep, orig_size, flags);
+	if (unlikely(objp))
+		goto leave;
+
 	cache_alloc_debugcheck_before(cachep, flags);
 	local_irq_save(save_flags);
 	objp = __do_cache_alloc(cachep, flags);
@@ -3309,6 +3319,7 @@  slab_alloc(struct kmem_cache *cachep, gfp_t flags, unsigned long caller)
 	if (unlikely(slab_want_init_on_alloc(flags, cachep)) && objp)
 		memset(objp, 0, cachep->object_size);
 
+leave:
 	slab_post_alloc_hook(cachep, objcg, flags, 1, &objp);
 	return objp;
 }
@@ -3414,6 +3425,11 @@  static void cache_flusharray(struct kmem_cache *cachep, struct array_cache *ac)
 static __always_inline void __cache_free(struct kmem_cache *cachep, void *objp,
 					 unsigned long caller)
 {
+	if (kfence_free(objp)) {
+		kmemleak_free_recursive(objp, cachep->flags);
+		return;
+	}
+
 	/* Put the object into the quarantine, don't touch it for now. */
 	if (kasan_slab_free(cachep, objp, _RET_IP_))
 		return;
@@ -3479,7 +3495,7 @@  void ___cache_free(struct kmem_cache *cachep, void *objp,
  */
 void *kmem_cache_alloc(struct kmem_cache *cachep, gfp_t flags)
 {
-	void *ret = slab_alloc(cachep, flags, _RET_IP_);
+	void *ret = slab_alloc(cachep, flags, cachep->object_size, _RET_IP_);
 
 	trace_kmem_cache_alloc(_RET_IP_, ret,
 			       cachep->object_size, cachep->size, flags);
@@ -3512,7 +3528,7 @@  int kmem_cache_alloc_bulk(struct kmem_cache *s, gfp_t flags, size_t size,
 
 	local_irq_disable();
 	for (i = 0; i < size; i++) {
-		void *objp = __do_cache_alloc(s, flags);
+		void *objp = kfence_alloc(s, s->object_size, flags) ?: __do_cache_alloc(s, flags);
 
 		if (unlikely(!objp))
 			goto error;
@@ -3545,7 +3561,7 @@  kmem_cache_alloc_trace(struct kmem_cache *cachep, gfp_t flags, size_t size)
 {
 	void *ret;
 
-	ret = slab_alloc(cachep, flags, _RET_IP_);
+	ret = slab_alloc(cachep, flags, size, _RET_IP_);
 
 	ret = kasan_kmalloc(cachep, ret, size, flags);
 	trace_kmalloc(_RET_IP_, ret,
@@ -3571,7 +3587,7 @@  EXPORT_SYMBOL(kmem_cache_alloc_trace);
  */
 void *kmem_cache_alloc_node(struct kmem_cache *cachep, gfp_t flags, int nodeid)
 {
-	void *ret = slab_alloc_node(cachep, flags, nodeid, _RET_IP_);
+	void *ret = slab_alloc_node(cachep, flags, nodeid, cachep->object_size, _RET_IP_);
 
 	trace_kmem_cache_alloc_node(_RET_IP_, ret,
 				    cachep->object_size, cachep->size,
@@ -3589,7 +3605,7 @@  void *kmem_cache_alloc_node_trace(struct kmem_cache *cachep,
 {
 	void *ret;
 
-	ret = slab_alloc_node(cachep, flags, nodeid, _RET_IP_);
+	ret = slab_alloc_node(cachep, flags, nodeid, size, _RET_IP_);
 
 	ret = kasan_kmalloc(cachep, ret, size, flags);
 	trace_kmalloc_node(_RET_IP_, ret,
@@ -3650,7 +3666,7 @@  static __always_inline void *__do_kmalloc(size_t size, gfp_t flags,
 	cachep = kmalloc_slab(size, flags);
 	if (unlikely(ZERO_OR_NULL_PTR(cachep)))
 		return cachep;
-	ret = slab_alloc(cachep, flags, caller);
+	ret = slab_alloc(cachep, flags, size, caller);
 
 	ret = kasan_kmalloc(cachep, ret, size, flags);
 	trace_kmalloc(caller, ret,
@@ -4138,18 +4154,24 @@  void __check_heap_object(const void *ptr, unsigned long n, struct page *page,
 			 bool to_user)
 {
 	struct kmem_cache *cachep;
-	unsigned int objnr;
+	unsigned int objnr = 0;
 	unsigned long offset;
+	bool is_kfence = is_kfence_address(ptr);
 
 	ptr = kasan_reset_tag(ptr);
 
 	/* Find and validate object. */
 	cachep = page->slab_cache;
-	objnr = obj_to_index(cachep, page, (void *)ptr);
-	BUG_ON(objnr >= cachep->num);
+	if (!is_kfence) {
+		objnr = obj_to_index(cachep, page, (void *)ptr);
+		BUG_ON(objnr >= cachep->num);
+	}
 
 	/* Find offset within object. */
-	offset = ptr - index_to_obj(cachep, page, objnr) - obj_offset(cachep);
+	if (is_kfence_address(ptr))
+		offset = ptr - kfence_object_start(ptr);
+	else
+		offset = ptr - index_to_obj(cachep, page, objnr) - obj_offset(cachep);
 
 	/* Allow address range falling entirely within usercopy region. */
 	if (offset >= cachep->useroffset &&
diff --git a/mm/slab_common.c b/mm/slab_common.c
index f9ccd5dc13f3..6e35e273681a 100644
--- a/mm/slab_common.c
+++ b/mm/slab_common.c
@@ -12,6 +12,7 @@ 
 #include <linux/memory.h>
 #include <linux/cache.h>
 #include <linux/compiler.h>
+#include <linux/kfence.h>
 #include <linux/module.h>
 #include <linux/cpu.h>
 #include <linux/uaccess.h>
@@ -448,6 +449,9 @@  static int shutdown_cache(struct kmem_cache *s)
 	/* free asan quarantined objects */
 	kasan_cache_shutdown(s);
 
+	if (!kfence_shutdown_cache(s))
+		return -EBUSY;
+
 	if (__kmem_cache_shutdown(s) != 0)
 		return -EBUSY;
 
@@ -1171,7 +1175,7 @@  size_t ksize(const void *objp)
 	if (unlikely(ZERO_OR_NULL_PTR(objp)) || !__kasan_check_read(objp, 1))
 		return 0;
 
-	size = __ksize(objp);
+	size = kfence_ksize(objp) ?: __ksize(objp);
 	/*
 	 * We assume that ksize callers could use whole allocated area,
 	 * so we need to unpoison this area.