diff mbox series

[RFC,net-next,v7,06/14] page_pool: convert to use netmem

Message ID 20240326225048.785801-7-almasrymina@google.com (mailing list archive)
State Superseded
Headers show
Series Device Memory TCP | expand

Commit Message

Mina Almasry March 26, 2024, 10:50 p.m. UTC
Abstrace the memory type from the page_pool so we can later add support
for new memory types. Convert the page_pool to use the new netmem type
abstraction, rather than use struct page directly.

As of this patch the netmem type is a no-op abstraction: it's always a
struct page underneath. All the page pool internals are converted to
use struct netmem instead of struct page, and the page pool now exports
2 APIs:

1. The existing struct page API.
2. The new struct netmem API.

Keeping the existing API is transitional; we do not want to refactor all
the current drivers using the page pool at once.

The netmem abstraction is currently a no-op. The page_pool uses
page_to_netmem() to convert allocated pages to netmem, and uses
netmem_to_page() to convert the netmem back to pages to pass to mm APIs,

Follow up patches to this series add non-paged netmem support to the
page_pool. This change is factored out on its own to limit the code
churn to this 1 patch, for ease of code review.

Signed-off-by: Mina Almasry <almasrymina@google.com>

---

v6:

- Rebased on top of the merged netmem_ref type.

To: linux-mm@kvack.org
Cc: Matthew Wilcox <willy@infradead.org>

---
 include/linux/skbuff.h           |   4 +-
 include/net/netmem.h             |  15 ++
 include/net/page_pool/helpers.h  | 122 +++++++++----
 include/net/page_pool/types.h    |  17 +-
 include/trace/events/page_pool.h |  29 +--
 net/bpf/test_run.c               |   5 +-
 net/core/page_pool.c             | 303 +++++++++++++++++--------------
 net/core/skbuff.c                |   7 +-
 8 files changed, 302 insertions(+), 200 deletions(-)

Comments

Mina Almasry March 27, 2024, 8:25 a.m. UTC | #1
On Tue, Mar 26, 2024 at 3:51 PM Mina Almasry <almasrymina@google.com> wrote:
>
> Abstrace the memory type from the page_pool so we can later add support
> for new memory types. Convert the page_pool to use the new netmem type
> abstraction, rather than use struct page directly.
>
> As of this patch the netmem type is a no-op abstraction: it's always a
> struct page underneath. All the page pool internals are converted to
> use struct netmem instead of struct page, and the page pool now exports
> 2 APIs:
>
> 1. The existing struct page API.
> 2. The new struct netmem API.
>
> Keeping the existing API is transitional; we do not want to refactor all
> the current drivers using the page pool at once.
>
> The netmem abstraction is currently a no-op. The page_pool uses
> page_to_netmem() to convert allocated pages to netmem, and uses
> netmem_to_page() to convert the netmem back to pages to pass to mm APIs,
>
> Follow up patches to this series add non-paged netmem support to the
> page_pool. This change is factored out on its own to limit the code
> churn to this 1 patch, for ease of code review.
>
> Signed-off-by: Mina Almasry <almasrymina@google.com>
>
> ---
>
> v6:
>
> - Rebased on top of the merged netmem_ref type.
>
> To: linux-mm@kvack.org

It looks like this tag to add linux-mm did not work as intended. CCing
linux-mm manually.

> Cc: Matthew Wilcox <willy@infradead.org>
>
> ---
>  include/linux/skbuff.h           |   4 +-
>  include/net/netmem.h             |  15 ++
>  include/net/page_pool/helpers.h  | 122 +++++++++----
>  include/net/page_pool/types.h    |  17 +-
>  include/trace/events/page_pool.h |  29 +--
>  net/bpf/test_run.c               |   5 +-
>  net/core/page_pool.c             | 303 +++++++++++++++++--------------
>  net/core/skbuff.c                |   7 +-
>  8 files changed, 302 insertions(+), 200 deletions(-)
>
> diff --git a/include/linux/skbuff.h b/include/linux/skbuff.h
> index b945af8a6208..78659c8efa4e 100644
> --- a/include/linux/skbuff.h
> +++ b/include/linux/skbuff.h
> @@ -3521,7 +3521,7 @@ int skb_pp_cow_data(struct page_pool *pool, struct sk_buff **pskb,
>                     unsigned int headroom);
>  int skb_cow_data_for_xdp(struct page_pool *pool, struct sk_buff **pskb,
>                          struct bpf_prog *prog);
> -bool napi_pp_put_page(struct page *page, bool napi_safe);
> +bool napi_pp_put_page(netmem_ref netmem, bool napi_safe);
>
>  static inline void
>  skb_page_unref(const struct sk_buff *skb, struct page *page, bool napi_safe)
> @@ -3539,7 +3539,7 @@ napi_frag_unref(skb_frag_t *frag, bool recycle, bool napi_safe)
>         struct page *page = skb_frag_page(frag);
>
>  #ifdef CONFIG_PAGE_POOL
> -       if (recycle && napi_pp_put_page(page, napi_safe))
> +       if (recycle && napi_pp_put_page(page_to_netmem(page), napi_safe))
>                 return;
>  #endif
>         put_page(page);
> diff --git a/include/net/netmem.h b/include/net/netmem.h
> index ca17ea1d33f8..21f53b29e5fe 100644
> --- a/include/net/netmem.h
> +++ b/include/net/netmem.h
> @@ -88,4 +88,19 @@ static inline netmem_ref page_to_netmem(struct page *page)
>         return (__force netmem_ref)page;
>  }
>
> +static inline int netmem_ref_count(netmem_ref netmem)
> +{
> +       return page_ref_count(netmem_to_page(netmem));
> +}
> +
> +static inline unsigned long netmem_to_pfn(netmem_ref netmem)
> +{
> +       return page_to_pfn(netmem_to_page(netmem));
> +}
> +
> +static inline netmem_ref netmem_compound_head(netmem_ref netmem)
> +{
> +       return page_to_netmem(compound_head(netmem_to_page(netmem)));
> +}
> +
>  #endif /* _NET_NETMEM_H */
> diff --git a/include/net/page_pool/helpers.h b/include/net/page_pool/helpers.h
> index 1d397c1a0043..61814f91a458 100644
> --- a/include/net/page_pool/helpers.h
> +++ b/include/net/page_pool/helpers.h
> @@ -53,6 +53,8 @@
>  #define _NET_PAGE_POOL_HELPERS_H
>
>  #include <net/page_pool/types.h>
> +#include <net/net_debug.h>
> +#include <net/netmem.h>
>
>  #ifdef CONFIG_PAGE_POOL_STATS
>  /* Deprecated driver-facing API, use netlink instead */
> @@ -101,7 +103,7 @@ static inline struct page *page_pool_dev_alloc_pages(struct page_pool *pool)
>   * Get a page fragment from the page allocator or page_pool caches.
>   *
>   * Return:
> - * Return allocated page fragment, otherwise return NULL.
> + * Return allocated page fragment, otherwise return 0.
>   */
>  static inline struct page *page_pool_dev_alloc_frag(struct page_pool *pool,
>                                                     unsigned int *offset,
> @@ -112,22 +114,22 @@ static inline struct page *page_pool_dev_alloc_frag(struct page_pool *pool,
>         return page_pool_alloc_frag(pool, offset, size, gfp);
>  }
>
> -static inline struct page *page_pool_alloc(struct page_pool *pool,
> -                                          unsigned int *offset,
> -                                          unsigned int *size, gfp_t gfp)
> +static inline netmem_ref page_pool_alloc(struct page_pool *pool,
> +                                        unsigned int *offset,
> +                                        unsigned int *size, gfp_t gfp)
>  {
>         unsigned int max_size = PAGE_SIZE << pool->p.order;
> -       struct page *page;
> +       netmem_ref netmem;
>
>         if ((*size << 1) > max_size) {
>                 *size = max_size;
>                 *offset = 0;
> -               return page_pool_alloc_pages(pool, gfp);
> +               return page_pool_alloc_netmem(pool, gfp);
>         }
>
> -       page = page_pool_alloc_frag(pool, offset, *size, gfp);
> -       if (unlikely(!page))
> -               return NULL;
> +       netmem = page_pool_alloc_frag_netmem(pool, offset, *size, gfp);
> +       if (unlikely(!netmem))
> +               return 0;
>
>         /* There is very likely not enough space for another fragment, so append
>          * the remaining size to the current fragment to avoid truesize
> @@ -138,7 +140,7 @@ static inline struct page *page_pool_alloc(struct page_pool *pool,
>                 pool->frag_offset = max_size;
>         }
>
> -       return page;
> +       return netmem;
>  }
>
>  /**
> @@ -152,7 +154,7 @@ static inline struct page *page_pool_alloc(struct page_pool *pool,
>   * utilization and performance penalty.
>   *
>   * Return:
> - * Return allocated page or page fragment, otherwise return NULL.
> + * Return allocated page or page fragment, otherwise return 0.
>   */
>  static inline struct page *page_pool_dev_alloc(struct page_pool *pool,
>                                                unsigned int *offset,
> @@ -160,7 +162,7 @@ static inline struct page *page_pool_dev_alloc(struct page_pool *pool,
>  {
>         gfp_t gfp = (GFP_ATOMIC | __GFP_NOWARN);
>
> -       return page_pool_alloc(pool, offset, size, gfp);
> +       return netmem_to_page(page_pool_alloc(pool, offset, size, gfp));
>  }
>
>  static inline void *page_pool_alloc_va(struct page_pool *pool,
> @@ -170,9 +172,10 @@ static inline void *page_pool_alloc_va(struct page_pool *pool,
>         struct page *page;
>
>         /* Mask off __GFP_HIGHMEM to ensure we can use page_address() */
> -       page = page_pool_alloc(pool, &offset, size, gfp & ~__GFP_HIGHMEM);
> +       page = netmem_to_page(
> +               page_pool_alloc(pool, &offset, size, gfp & ~__GFP_HIGHMEM));
>         if (unlikely(!page))
> -               return NULL;
> +               return 0;
>
>         return page_address(page) + offset;
>  }
> @@ -187,7 +190,7 @@ static inline void *page_pool_alloc_va(struct page_pool *pool,
>   * it returns va of the allocated page or page fragment.
>   *
>   * Return:
> - * Return the va for the allocated page or page fragment, otherwise return NULL.
> + * Return the va for the allocated page or page fragment, otherwise return 0.
>   */
>  static inline void *page_pool_dev_alloc_va(struct page_pool *pool,
>                                            unsigned int *size)
> @@ -210,6 +213,11 @@ inline enum dma_data_direction page_pool_get_dma_dir(struct page_pool *pool)
>         return pool->p.dma_dir;
>  }
>
> +static inline void page_pool_fragment_netmem(netmem_ref netmem, long nr)
> +{
> +       atomic_long_set(&netmem_to_page(netmem)->pp_ref_count, nr);
> +}
> +
>  /**
>   * page_pool_fragment_page() - split a fresh page into fragments
>   * @page:      page to split
> @@ -230,11 +238,12 @@ inline enum dma_data_direction page_pool_get_dma_dir(struct page_pool *pool)
>   */
>  static inline void page_pool_fragment_page(struct page *page, long nr)
>  {
> -       atomic_long_set(&page->pp_ref_count, nr);
> +       page_pool_fragment_netmem(page_to_netmem(page), nr);
>  }
>
> -static inline long page_pool_unref_page(struct page *page, long nr)
> +static inline long page_pool_unref_netmem(netmem_ref netmem, long nr)
>  {
> +       struct page *page = netmem_to_page(netmem);
>         long ret;
>
>         /* If nr == pp_ref_count then we have cleared all remaining
> @@ -277,15 +286,41 @@ static inline long page_pool_unref_page(struct page *page, long nr)
>         return ret;
>  }
>
> +static inline long page_pool_unref_page(struct page *page, long nr)
> +{
> +       return page_pool_unref_netmem(page_to_netmem(page), nr);
> +}
> +
> +static inline void page_pool_ref_netmem(netmem_ref netmem)
> +{
> +       atomic_long_inc(&netmem_to_page(netmem)->pp_ref_count);
> +}
> +
>  static inline void page_pool_ref_page(struct page *page)
>  {
> -       atomic_long_inc(&page->pp_ref_count);
> +       page_pool_ref_netmem(page_to_netmem(page));
>  }
>
> -static inline bool page_pool_is_last_ref(struct page *page)
> +static inline bool page_pool_is_last_ref(netmem_ref netmem)
>  {
>         /* If page_pool_unref_page() returns 0, we were the last user */
> -       return page_pool_unref_page(page, 1) == 0;
> +       return page_pool_unref_netmem(netmem, 1) == 0;
> +}
> +
> +static inline void page_pool_put_netmem(struct page_pool *pool,
> +                                       netmem_ref netmem,
> +                                       unsigned int dma_sync_size,
> +                                       bool allow_direct)
> +{
> +       /* When page_pool isn't compiled-in, net/core/xdp.c doesn't
> +        * allow registering MEM_TYPE_PAGE_POOL, but shield linker.
> +        */
> +#ifdef CONFIG_PAGE_POOL
> +       if (!page_pool_is_last_ref(netmem))
> +               return;
> +
> +       page_pool_put_unrefed_netmem(pool, netmem, dma_sync_size, allow_direct);
> +#endif
>  }
>
>  /**
> @@ -306,15 +341,15 @@ static inline void page_pool_put_page(struct page_pool *pool,
>                                       unsigned int dma_sync_size,
>                                       bool allow_direct)
>  {
> -       /* When page_pool isn't compiled-in, net/core/xdp.c doesn't
> -        * allow registering MEM_TYPE_PAGE_POOL, but shield linker.
> -        */
> -#ifdef CONFIG_PAGE_POOL
> -       if (!page_pool_is_last_ref(page))
> -               return;
> +       page_pool_put_netmem(pool, page_to_netmem(page), dma_sync_size,
> +                            allow_direct);
> +}
>
> -       page_pool_put_unrefed_page(pool, page, dma_sync_size, allow_direct);
> -#endif
> +static inline void page_pool_put_full_netmem(struct page_pool *pool,
> +                                            netmem_ref netmem,
> +                                            bool allow_direct)
> +{
> +       page_pool_put_netmem(pool, netmem, -1, allow_direct);
>  }
>
>  /**
> @@ -329,7 +364,7 @@ static inline void page_pool_put_page(struct page_pool *pool,
>  static inline void page_pool_put_full_page(struct page_pool *pool,
>                                            struct page *page, bool allow_direct)
>  {
> -       page_pool_put_page(pool, page, -1, allow_direct);
> +       page_pool_put_netmem(pool, page_to_netmem(page), -1, allow_direct);
>  }
>
>  /**
> @@ -363,6 +398,18 @@ static inline void page_pool_free_va(struct page_pool *pool, void *va,
>         page_pool_put_page(pool, virt_to_head_page(va), -1, allow_direct);
>  }
>
> +static inline dma_addr_t page_pool_get_dma_addr_netmem(netmem_ref netmem)
> +{
> +       struct page *page = netmem_to_page(netmem);
> +
> +       dma_addr_t ret = page->dma_addr;
> +
> +       if (PAGE_POOL_32BIT_ARCH_WITH_64BIT_DMA)
> +               ret <<= PAGE_SHIFT;
> +
> +       return ret;
> +}
> +
>  /**
>   * page_pool_get_dma_addr() - Retrieve the stored DMA address.
>   * @page:      page allocated from a page pool
> @@ -372,16 +419,14 @@ static inline void page_pool_free_va(struct page_pool *pool, void *va,
>   */
>  static inline dma_addr_t page_pool_get_dma_addr(struct page *page)
>  {
> -       dma_addr_t ret = page->dma_addr;
> -
> -       if (PAGE_POOL_32BIT_ARCH_WITH_64BIT_DMA)
> -               ret <<= PAGE_SHIFT;
> -
> -       return ret;
> +       return page_pool_get_dma_addr_netmem(page_to_netmem(page));
>  }
>
> -static inline bool page_pool_set_dma_addr(struct page *page, dma_addr_t addr)
> +static inline bool page_pool_set_dma_addr_netmem(netmem_ref netmem,
> +                                                dma_addr_t addr)
>  {
> +       struct page *page = netmem_to_page(netmem);
> +
>         if (PAGE_POOL_32BIT_ARCH_WITH_64BIT_DMA) {
>                 page->dma_addr = addr >> PAGE_SHIFT;
>
> @@ -395,6 +440,11 @@ static inline bool page_pool_set_dma_addr(struct page *page, dma_addr_t addr)
>         return false;
>  }
>
> +static inline bool page_pool_set_dma_addr(struct page *page, dma_addr_t addr)
> +{
> +       return page_pool_set_dma_addr_netmem(page_to_netmem(page), addr);
> +}
> +
>  static inline bool page_pool_put(struct page_pool *pool)
>  {
>         return refcount_dec_and_test(&pool->user_cnt);
> diff --git a/include/net/page_pool/types.h b/include/net/page_pool/types.h
> index 07e6afafedbe..0d164624f16d 100644
> --- a/include/net/page_pool/types.h
> +++ b/include/net/page_pool/types.h
> @@ -40,7 +40,7 @@
>  #define PP_ALLOC_CACHE_REFILL  64
>  struct pp_alloc_cache {
>         u32 count;
> -       struct page *cache[PP_ALLOC_CACHE_SIZE];
> +       netmem_ref cache[PP_ALLOC_CACHE_SIZE];
>  };
>
>  /**
> @@ -73,7 +73,7 @@ struct page_pool_params {
>         struct_group_tagged(page_pool_params_slow, slow,
>                 struct net_device *netdev;
>  /* private: used by test code only */
> -               void (*init_callback)(struct page *page, void *arg);
> +               void (*init_callback)(netmem_ref netmem, void *arg);
>                 void *init_arg;
>         );
>  };
> @@ -131,8 +131,8 @@ struct page_pool_stats {
>  struct memory_provider_ops {
>         int (*init)(struct page_pool *pool);
>         void (*destroy)(struct page_pool *pool);
> -       struct page *(*alloc_pages)(struct page_pool *pool, gfp_t gfp);
> -       bool (*release_page)(struct page_pool *pool, struct page *page);
> +       netmem_ref (*alloc_pages)(struct page_pool *pool, gfp_t gfp);
> +       bool (*release_page)(struct page_pool *pool, netmem_ref netmem);
>  };
>
>  struct pp_memory_provider_params {
> @@ -147,7 +147,7 @@ struct page_pool {
>         bool has_init_callback;
>
>         long frag_users;
> -       struct page *frag_page;
> +       netmem_ref frag_page;
>         unsigned int frag_offset;
>         u32 pages_state_hold_cnt;
>
> @@ -219,8 +219,12 @@ struct page_pool {
>  };
>
>  struct page *page_pool_alloc_pages(struct page_pool *pool, gfp_t gfp);
> +netmem_ref page_pool_alloc_netmem(struct page_pool *pool, gfp_t gfp);
>  struct page *page_pool_alloc_frag(struct page_pool *pool, unsigned int *offset,
>                                   unsigned int size, gfp_t gfp);
> +netmem_ref page_pool_alloc_frag_netmem(struct page_pool *pool,
> +                                      unsigned int *offset, unsigned int size,
> +                                      gfp_t gfp);
>  struct page_pool *page_pool_create(const struct page_pool_params *params);
>  struct page_pool *page_pool_create_percpu(const struct page_pool_params *params,
>                                           int cpuid);
> @@ -250,6 +254,9 @@ static inline void page_pool_put_page_bulk(struct page_pool *pool, void **data,
>  }
>  #endif
>
> +void page_pool_put_unrefed_netmem(struct page_pool *pool, netmem_ref netmem,
> +                                 unsigned int dma_sync_size,
> +                                 bool allow_direct);
>  void page_pool_put_unrefed_page(struct page_pool *pool, struct page *page,
>                                 unsigned int dma_sync_size,
>                                 bool allow_direct);
> diff --git a/include/trace/events/page_pool.h b/include/trace/events/page_pool.h
> index 6834356b2d2a..c5b6383ff276 100644
> --- a/include/trace/events/page_pool.h
> +++ b/include/trace/events/page_pool.h
> @@ -42,51 +42,52 @@ TRACE_EVENT(page_pool_release,
>  TRACE_EVENT(page_pool_state_release,
>
>         TP_PROTO(const struct page_pool *pool,
> -                const struct page *page, u32 release),
> +                netmem_ref netmem, u32 release),
>
> -       TP_ARGS(pool, page, release),
> +       TP_ARGS(pool, netmem, release),
>
>         TP_STRUCT__entry(
>                 __field(const struct page_pool *,       pool)
> -               __field(const struct page *,            page)
> +               __field(netmem_ref,                     netmem)
>                 __field(u32,                            release)
>                 __field(unsigned long,                  pfn)
>         ),
>
>         TP_fast_assign(
>                 __entry->pool           = pool;
> -               __entry->page           = page;
> +               __entry->netmem         = netmem;
>                 __entry->release        = release;
> -               __entry->pfn            = page_to_pfn(page);
> +               __entry->pfn            = netmem_to_pfn(netmem);
>         ),
>
> -       TP_printk("page_pool=%p page=%p pfn=0x%lx release=%u",
> -                 __entry->pool, __entry->page, __entry->pfn, __entry->release)
> +       TP_printk("page_pool=%p netmem=%lu pfn=0x%lx release=%u",
> +                 __entry->pool, (__force unsigned long)__entry->netmem,
> +                 __entry->pfn, __entry->release)
>  );
>
>  TRACE_EVENT(page_pool_state_hold,
>
>         TP_PROTO(const struct page_pool *pool,
> -                const struct page *page, u32 hold),
> +                netmem_ref netmem, u32 hold),
>
> -       TP_ARGS(pool, page, hold),
> +       TP_ARGS(pool, netmem, hold),
>
>         TP_STRUCT__entry(
>                 __field(const struct page_pool *,       pool)
> -               __field(const struct page *,            page)
> +               __field(netmem_ref,                     netmem)
>                 __field(u32,                            hold)
>                 __field(unsigned long,                  pfn)
>         ),
>
>         TP_fast_assign(
>                 __entry->pool   = pool;
> -               __entry->page   = page;
> +               __entry->netmem = netmem;
>                 __entry->hold   = hold;
> -               __entry->pfn    = page_to_pfn(page);
> +               __entry->pfn    = netmem_to_pfn(netmem);
>         ),
>
> -       TP_printk("page_pool=%p page=%p pfn=0x%lx hold=%u",
> -                 __entry->pool, __entry->page, __entry->pfn, __entry->hold)
> +       TP_printk("page_pool=%p netmem=%lu pfn=0x%lx hold=%u",
> +                 __entry->pool, __entry->netmem, __entry->pfn, __entry->hold)
>  );
>
>  TRACE_EVENT(page_pool_update_nid,
> diff --git a/net/bpf/test_run.c b/net/bpf/test_run.c
> index 61efeadaff8d..fc300e807e1d 100644
> --- a/net/bpf/test_run.c
> +++ b/net/bpf/test_run.c
> @@ -127,9 +127,10 @@ struct xdp_test_data {
>  #define TEST_XDP_FRAME_SIZE (PAGE_SIZE - sizeof(struct xdp_page_head))
>  #define TEST_XDP_MAX_BATCH 256
>
> -static void xdp_test_run_init_page(struct page *page, void *arg)
> +static void xdp_test_run_init_page(netmem_ref netmem, void *arg)
>  {
> -       struct xdp_page_head *head = phys_to_virt(page_to_phys(page));
> +       struct xdp_page_head *head =
> +               phys_to_virt(page_to_phys(netmem_to_page(netmem)));
>         struct xdp_buff *new_ctx, *orig_ctx;
>         u32 headroom = XDP_PACKET_HEADROOM;
>         struct xdp_test_data *xdp = arg;
> diff --git a/net/core/page_pool.c b/net/core/page_pool.c
> index 795b7ff1c01f..c8125be3a6e2 100644
> --- a/net/core/page_pool.c
> +++ b/net/core/page_pool.c
> @@ -329,19 +329,18 @@ struct page_pool *page_pool_create(const struct page_pool_params *params)
>  }
>  EXPORT_SYMBOL(page_pool_create);
>
> -static void page_pool_return_page(struct page_pool *pool, struct page *page);
> +static void page_pool_return_page(struct page_pool *pool, netmem_ref netmem);
>
> -noinline
> -static struct page *page_pool_refill_alloc_cache(struct page_pool *pool)
> +static noinline netmem_ref page_pool_refill_alloc_cache(struct page_pool *pool)
>  {
>         struct ptr_ring *r = &pool->ring;
> -       struct page *page;
> +       netmem_ref netmem;
>         int pref_nid; /* preferred NUMA node */
>
>         /* Quicker fallback, avoid locks when ring is empty */
>         if (__ptr_ring_empty(r)) {
>                 alloc_stat_inc(pool, empty);
> -               return NULL;
> +               return 0;
>         }
>
>         /* Softirq guarantee CPU and thus NUMA node is stable. This,
> @@ -356,56 +355,56 @@ static struct page *page_pool_refill_alloc_cache(struct page_pool *pool)
>
>         /* Refill alloc array, but only if NUMA match */
>         do {
> -               page = __ptr_ring_consume(r);
> -               if (unlikely(!page))
> +               netmem = (__force netmem_ref)__ptr_ring_consume(r);
> +               if (unlikely(!netmem))
>                         break;
>
> -               if (likely(page_to_nid(page) == pref_nid)) {
> -                       pool->alloc.cache[pool->alloc.count++] = page;
> +               if (likely(page_to_nid(netmem_to_page(netmem)) == pref_nid)) {
> +                       pool->alloc.cache[pool->alloc.count++] = netmem;
>                 } else {
>                         /* NUMA mismatch;
>                          * (1) release 1 page to page-allocator and
>                          * (2) break out to fallthrough to alloc_pages_node.
>                          * This limit stress on page buddy alloactor.
>                          */
> -                       page_pool_return_page(pool, page);
> +                       page_pool_return_page(pool, netmem);
>                         alloc_stat_inc(pool, waive);
> -                       page = NULL;
> +                       netmem = 0;
>                         break;
>                 }
>         } while (pool->alloc.count < PP_ALLOC_CACHE_REFILL);
>
>         /* Return last page */
>         if (likely(pool->alloc.count > 0)) {
> -               page = pool->alloc.cache[--pool->alloc.count];
> +               netmem = pool->alloc.cache[--pool->alloc.count];
>                 alloc_stat_inc(pool, refill);
>         }
>
> -       return page;
> +       return netmem;
>  }
>
>  /* fast path */
> -static struct page *__page_pool_get_cached(struct page_pool *pool)
> +static netmem_ref __page_pool_get_cached(struct page_pool *pool)
>  {
> -       struct page *page;
> +       netmem_ref netmem;
>
>         /* Caller MUST guarantee safe non-concurrent access, e.g. softirq */
>         if (likely(pool->alloc.count)) {
>                 /* Fast-path */
> -               page = pool->alloc.cache[--pool->alloc.count];
> +               netmem = pool->alloc.cache[--pool->alloc.count];
>                 alloc_stat_inc(pool, fast);
>         } else {
> -               page = page_pool_refill_alloc_cache(pool);
> +               netmem = page_pool_refill_alloc_cache(pool);
>         }
>
> -       return page;
> +       return netmem;
>  }
>
>  static void page_pool_dma_sync_for_device(struct page_pool *pool,
> -                                         struct page *page,
> +                                         netmem_ref netmem,
>                                           unsigned int dma_sync_size)
>  {
> -       dma_addr_t dma_addr = page_pool_get_dma_addr(page);
> +       dma_addr_t dma_addr = page_pool_get_dma_addr_netmem(netmem);
>
>         dma_sync_size = min(dma_sync_size, pool->p.max_len);
>         dma_sync_single_range_for_device(pool->p.dev, dma_addr,
> @@ -413,7 +412,7 @@ static void page_pool_dma_sync_for_device(struct page_pool *pool,
>                                          pool->p.dma_dir);
>  }
>
> -static bool page_pool_dma_map(struct page_pool *pool, struct page *page)
> +static bool page_pool_dma_map(struct page_pool *pool, netmem_ref netmem)
>  {
>         dma_addr_t dma;
>
> @@ -422,18 +421,18 @@ static bool page_pool_dma_map(struct page_pool *pool, struct page *page)
>          * into page private data (i.e 32bit cpu with 64bit DMA caps)
>          * This mapping is kept for lifetime of page, until leaving pool.
>          */
> -       dma = dma_map_page_attrs(pool->p.dev, page, 0,
> -                                (PAGE_SIZE << pool->p.order),
> -                                pool->p.dma_dir, DMA_ATTR_SKIP_CPU_SYNC |
> -                                                 DMA_ATTR_WEAK_ORDERING);
> +       dma = dma_map_page_attrs(pool->p.dev, netmem_to_page(netmem), 0,
> +                                (PAGE_SIZE << pool->p.order), pool->p.dma_dir,
> +                                DMA_ATTR_SKIP_CPU_SYNC |
> +                                        DMA_ATTR_WEAK_ORDERING);
>         if (dma_mapping_error(pool->p.dev, dma))
>                 return false;
>
> -       if (page_pool_set_dma_addr(page, dma))
> +       if (page_pool_set_dma_addr_netmem(netmem, dma))
>                 goto unmap_failed;
>
>         if (pool->p.flags & PP_FLAG_DMA_SYNC_DEV)
> -               page_pool_dma_sync_for_device(pool, page, pool->p.max_len);
> +               page_pool_dma_sync_for_device(pool, netmem, pool->p.max_len);
>
>         return true;
>
> @@ -445,9 +444,10 @@ static bool page_pool_dma_map(struct page_pool *pool, struct page *page)
>         return false;
>  }
>
> -static void page_pool_set_pp_info(struct page_pool *pool,
> -                                 struct page *page)
> +static void page_pool_set_pp_info(struct page_pool *pool, netmem_ref netmem)
>  {
> +       struct page *page = netmem_to_page(netmem);
> +
>         page->pp = pool;
>         page->pp_magic |= PP_SIGNATURE;
>
> @@ -457,13 +457,15 @@ static void page_pool_set_pp_info(struct page_pool *pool,
>          * is dirtying the same cache line as the page->pp_magic above, so
>          * the overhead is negligible.
>          */
> -       page_pool_fragment_page(page, 1);
> +       page_pool_fragment_netmem(netmem, 1);
>         if (pool->has_init_callback)
> -               pool->slow.init_callback(page, pool->slow.init_arg);
> +               pool->slow.init_callback(netmem, pool->slow.init_arg);
>  }
>
> -static void page_pool_clear_pp_info(struct page *page)
> +static void page_pool_clear_pp_info(netmem_ref netmem)
>  {
> +       struct page *page = netmem_to_page(netmem);
> +
>         page->pp_magic = 0;
>         page->pp = NULL;
>  }
> @@ -479,34 +481,34 @@ static struct page *__page_pool_alloc_page_order(struct page_pool *pool,
>                 return NULL;
>
>         if ((pool->p.flags & PP_FLAG_DMA_MAP) &&
> -           unlikely(!page_pool_dma_map(pool, page))) {
> +           unlikely(!page_pool_dma_map(pool, page_to_netmem(page)))) {
>                 put_page(page);
>                 return NULL;
>         }
>
>         alloc_stat_inc(pool, slow_high_order);
> -       page_pool_set_pp_info(pool, page);
> +       page_pool_set_pp_info(pool, page_to_netmem(page));
>
>         /* Track how many pages are held 'in-flight' */
>         pool->pages_state_hold_cnt++;
> -       trace_page_pool_state_hold(pool, page, pool->pages_state_hold_cnt);
> +       trace_page_pool_state_hold(pool, page_to_netmem(page),
> +                                  pool->pages_state_hold_cnt);
>         return page;
>  }
>
>  /* slow path */
> -noinline
> -static struct page *__page_pool_alloc_pages_slow(struct page_pool *pool,
> -                                                gfp_t gfp)
> +static noinline netmem_ref __page_pool_alloc_pages_slow(struct page_pool *pool,
> +                                                       gfp_t gfp)
>  {
>         const int bulk = PP_ALLOC_CACHE_REFILL;
>         unsigned int pp_flags = pool->p.flags;
>         unsigned int pp_order = pool->p.order;
> -       struct page *page;
> +       netmem_ref netmem;
>         int i, nr_pages;
>
>         /* Don't support bulk alloc for high-order pages */
>         if (unlikely(pp_order))
> -               return __page_pool_alloc_page_order(pool, gfp);
> +               return page_to_netmem(__page_pool_alloc_page_order(pool, gfp));
>
>         /* Unnecessary as alloc cache is empty, but guarantees zero count */
>         if (unlikely(pool->alloc.count > 0))
> @@ -515,60 +517,67 @@ static struct page *__page_pool_alloc_pages_slow(struct page_pool *pool,
>         /* Mark empty alloc.cache slots "empty" for alloc_pages_bulk_array */
>         memset(&pool->alloc.cache, 0, sizeof(void *) * bulk);
>
> -       nr_pages = alloc_pages_bulk_array_node(gfp, pool->p.nid, bulk,
> -                                              pool->alloc.cache);
> +       nr_pages = alloc_pages_bulk_array_node(gfp,
> +                                              pool->p.nid, bulk,
> +                                              (struct page **)pool->alloc.cache);
>         if (unlikely(!nr_pages))
> -               return NULL;
> +               return 0;
>
>         /* Pages have been filled into alloc.cache array, but count is zero and
>          * page element have not been (possibly) DMA mapped.
>          */
>         for (i = 0; i < nr_pages; i++) {
> -               page = pool->alloc.cache[i];
> +               netmem = pool->alloc.cache[i];
>                 if ((pp_flags & PP_FLAG_DMA_MAP) &&
> -                   unlikely(!page_pool_dma_map(pool, page))) {
> -                       put_page(page);
> +                   unlikely(!page_pool_dma_map(pool, netmem))) {
> +                       put_page(netmem_to_page(netmem));
>                         continue;
>                 }
>
> -               page_pool_set_pp_info(pool, page);
> -               pool->alloc.cache[pool->alloc.count++] = page;
> +               page_pool_set_pp_info(pool, netmem);
> +               pool->alloc.cache[pool->alloc.count++] = netmem;
>                 /* Track how many pages are held 'in-flight' */
>                 pool->pages_state_hold_cnt++;
> -               trace_page_pool_state_hold(pool, page,
> +               trace_page_pool_state_hold(pool, netmem,
>                                            pool->pages_state_hold_cnt);
>         }
>
>         /* Return last page */
>         if (likely(pool->alloc.count > 0)) {
> -               page = pool->alloc.cache[--pool->alloc.count];
> +               netmem = pool->alloc.cache[--pool->alloc.count];
>                 alloc_stat_inc(pool, slow);
>         } else {
> -               page = NULL;
> +               netmem = 0;
>         }
>
>         /* When page just alloc'ed is should/must have refcnt 1. */
> -       return page;
> +       return netmem;
>  }
>
>  /* For using page_pool replace: alloc_pages() API calls, but provide
>   * synchronization guarantee for allocation side.
>   */
> -struct page *page_pool_alloc_pages(struct page_pool *pool, gfp_t gfp)
> +netmem_ref page_pool_alloc_netmem(struct page_pool *pool, gfp_t gfp)
>  {
> -       struct page *page;
> +       netmem_ref netmem;
>
>         /* Fast-path: Get a page from cache */
> -       page = __page_pool_get_cached(pool);
> -       if (page)
> -               return page;
> +       netmem = __page_pool_get_cached(pool);
> +       if (netmem)
> +               return netmem;
>
>         /* Slow-path: cache empty, do real allocation */
>         if (static_branch_unlikely(&page_pool_mem_providers) && pool->mp_ops)
> -               page = pool->mp_ops->alloc_pages(pool, gfp);
> +               netmem = pool->mp_ops->alloc_pages(pool, gfp);
>         else
> -               page = __page_pool_alloc_pages_slow(pool, gfp);
> -       return page;
> +               netmem = __page_pool_alloc_pages_slow(pool, gfp);
> +       return netmem;
> +}
> +EXPORT_SYMBOL(page_pool_alloc_netmem);
> +
> +struct page *page_pool_alloc_pages(struct page_pool *pool, gfp_t gfp)
> +{
> +       return netmem_to_page(page_pool_alloc_netmem(pool, gfp));
>  }
>  EXPORT_SYMBOL(page_pool_alloc_pages);
>
> @@ -596,8 +605,8 @@ s32 page_pool_inflight(const struct page_pool *pool, bool strict)
>         return inflight;
>  }
>
> -static __always_inline
> -void __page_pool_release_page_dma(struct page_pool *pool, struct page *page)
> +static __always_inline void __page_pool_release_page_dma(struct page_pool *pool,
> +                                                        netmem_ref netmem)
>  {
>         dma_addr_t dma;
>
> @@ -607,13 +616,13 @@ void __page_pool_release_page_dma(struct page_pool *pool, struct page *page)
>                  */
>                 return;
>
> -       dma = page_pool_get_dma_addr(page);
> +       dma = page_pool_get_dma_addr_netmem(netmem);
>
>         /* When page is unmapped, it cannot be returned to our pool */
>         dma_unmap_page_attrs(pool->p.dev, dma,
>                              PAGE_SIZE << pool->p.order, pool->p.dma_dir,
>                              DMA_ATTR_SKIP_CPU_SYNC | DMA_ATTR_WEAK_ORDERING);
> -       page_pool_set_dma_addr(page, 0);
> +       page_pool_set_dma_addr_netmem(netmem, 0);
>  }
>
>  /* Disconnects a page (from a page_pool).  API users can have a need
> @@ -621,26 +630,26 @@ void __page_pool_release_page_dma(struct page_pool *pool, struct page *page)
>   * a regular page (that will eventually be returned to the normal
>   * page-allocator via put_page).
>   */
> -void page_pool_return_page(struct page_pool *pool, struct page *page)
> +void page_pool_return_page(struct page_pool *pool, netmem_ref netmem)
>  {
>         int count;
>         bool put;
>
>         put = true;
>         if (static_branch_unlikely(&page_pool_mem_providers) && pool->mp_ops)
> -               put = pool->mp_ops->release_page(pool, page);
> +               put = pool->mp_ops->release_page(pool, netmem);
>         else
> -               __page_pool_release_page_dma(pool, page);
> +               __page_pool_release_page_dma(pool, netmem);
>
>         /* This may be the last page returned, releasing the pool, so
>          * it is not safe to reference pool afterwards.
>          */
>         count = atomic_inc_return_relaxed(&pool->pages_state_release_cnt);
> -       trace_page_pool_state_release(pool, page, count);
> +       trace_page_pool_state_release(pool, netmem, count);
>
>         if (put) {
> -               page_pool_clear_pp_info(page);
> -               put_page(page);
> +               page_pool_clear_pp_info(netmem);
> +               put_page(netmem_to_page(netmem));
>         }
>         /* An optimization would be to call __free_pages(page, pool->p.order)
>          * knowing page is not part of page-cache (thus avoiding a
> @@ -648,14 +657,14 @@ void page_pool_return_page(struct page_pool *pool, struct page *page)
>          */
>  }
>
> -static bool page_pool_recycle_in_ring(struct page_pool *pool, struct page *page)
> +static bool page_pool_recycle_in_ring(struct page_pool *pool, netmem_ref netmem)
>  {
>         int ret;
>         /* BH protection not needed if current is softirq */
>         if (in_softirq())
> -               ret = ptr_ring_produce(&pool->ring, page);
> +               ret = ptr_ring_produce(&pool->ring, (__force void *)netmem);
>         else
> -               ret = ptr_ring_produce_bh(&pool->ring, page);
> +               ret = ptr_ring_produce_bh(&pool->ring, (__force void *)netmem);
>
>         if (!ret) {
>                 recycle_stat_inc(pool, ring);
> @@ -670,7 +679,7 @@ static bool page_pool_recycle_in_ring(struct page_pool *pool, struct page *page)
>   *
>   * Caller must provide appropriate safe context.
>   */
> -static bool page_pool_recycle_in_cache(struct page *page,
> +static bool page_pool_recycle_in_cache(netmem_ref netmem,
>                                        struct page_pool *pool)
>  {
>         if (unlikely(pool->alloc.count == PP_ALLOC_CACHE_SIZE)) {
> @@ -679,14 +688,15 @@ static bool page_pool_recycle_in_cache(struct page *page,
>         }
>
>         /* Caller MUST have verified/know (page_ref_count(page) == 1) */
> -       pool->alloc.cache[pool->alloc.count++] = page;
> +       pool->alloc.cache[pool->alloc.count++] = netmem;
>         recycle_stat_inc(pool, cached);
>         return true;
>  }
>
> -static bool __page_pool_page_can_be_recycled(const struct page *page)
> +static bool __page_pool_page_can_be_recycled(netmem_ref netmem)
>  {
> -       return page_ref_count(page) == 1 && !page_is_pfmemalloc(page);
> +       return page_ref_count(netmem_to_page(netmem)) == 1 &&
> +              !page_is_pfmemalloc(netmem_to_page(netmem));
>  }
>
>  /* If the page refcnt == 1, this will try to recycle the page.
> @@ -695,8 +705,8 @@ static bool __page_pool_page_can_be_recycled(const struct page *page)
>   * If the page refcnt != 1, then the page will be returned to memory
>   * subsystem.
>   */
> -static __always_inline struct page *
> -__page_pool_put_page(struct page_pool *pool, struct page *page,
> +static __always_inline netmem_ref
> +__page_pool_put_page(struct page_pool *pool, netmem_ref netmem,
>                      unsigned int dma_sync_size, bool allow_direct)
>  {
>         lockdep_assert_no_hardirq();
> @@ -710,19 +720,19 @@ __page_pool_put_page(struct page_pool *pool, struct page *page,
>          * page is NOT reusable when allocated when system is under
>          * some pressure. (page_is_pfmemalloc)
>          */
> -       if (likely(__page_pool_page_can_be_recycled(page))) {
> +       if (likely(__page_pool_page_can_be_recycled(netmem))) {
>                 /* Read barrier done in page_ref_count / READ_ONCE */
>
>                 if (pool->p.flags & PP_FLAG_DMA_SYNC_DEV)
> -                       page_pool_dma_sync_for_device(pool, page,
> +                       page_pool_dma_sync_for_device(pool, netmem,
>                                                       dma_sync_size);
>
>                 if (allow_direct && in_softirq() &&
> -                   page_pool_recycle_in_cache(page, pool))
> -                       return NULL;
> +                   page_pool_recycle_in_cache(netmem, pool))
> +                       return 0;
>
>                 /* Page found as candidate for recycling */
> -               return page;
> +               return netmem;
>         }
>         /* Fallback/non-XDP mode: API user have elevated refcnt.
>          *
> @@ -738,21 +748,30 @@ __page_pool_put_page(struct page_pool *pool, struct page *page,
>          * will be invoking put_page.
>          */
>         recycle_stat_inc(pool, released_refcnt);
> -       page_pool_return_page(pool, page);
> +       page_pool_return_page(pool, netmem);
>
> -       return NULL;
> +       return 0;
>  }
>
> -void page_pool_put_unrefed_page(struct page_pool *pool, struct page *page,
> -                               unsigned int dma_sync_size, bool allow_direct)
> +void page_pool_put_unrefed_netmem(struct page_pool *pool, netmem_ref netmem,
> +                                 unsigned int dma_sync_size, bool allow_direct)
>  {
> -       page = __page_pool_put_page(pool, page, dma_sync_size, allow_direct);
> -       if (page && !page_pool_recycle_in_ring(pool, page)) {
> +       netmem =
> +               __page_pool_put_page(pool, netmem, dma_sync_size, allow_direct);
> +       if (netmem && !page_pool_recycle_in_ring(pool, netmem)) {
>                 /* Cache full, fallback to free pages */
>                 recycle_stat_inc(pool, ring_full);
> -               page_pool_return_page(pool, page);
> +               page_pool_return_page(pool, netmem);
>         }
>  }
> +EXPORT_SYMBOL(page_pool_put_unrefed_netmem);
> +
> +void page_pool_put_unrefed_page(struct page_pool *pool, struct page *page,
> +                               unsigned int dma_sync_size, bool allow_direct)
> +{
> +       page_pool_put_unrefed_netmem(pool, page_to_netmem(page), dma_sync_size,
> +                                    allow_direct);
> +}
>  EXPORT_SYMBOL(page_pool_put_unrefed_page);
>
>  /**
> @@ -777,16 +796,16 @@ void page_pool_put_page_bulk(struct page_pool *pool, void **data,
>         bool in_softirq;
>
>         for (i = 0; i < count; i++) {
> -               struct page *page = virt_to_head_page(data[i]);
> +               netmem_ref netmem = page_to_netmem(virt_to_head_page(data[i]));
>
>                 /* It is not the last user for the page frag case */
> -               if (!page_pool_is_last_ref(page))
> +               if (!page_pool_is_last_ref(netmem))
>                         continue;
>
> -               page = __page_pool_put_page(pool, page, -1, false);
> +               netmem = __page_pool_put_page(pool, netmem, -1, false);
>                 /* Approved for bulk recycling in ptr_ring cache */
> -               if (page)
> -                       data[bulk_len++] = page;
> +               if (netmem)
> +                       data[bulk_len++] = (__force void *)netmem;
>         }
>
>         if (unlikely(!bulk_len))
> @@ -812,100 +831,108 @@ void page_pool_put_page_bulk(struct page_pool *pool, void **data,
>          * since put_page() with refcnt == 1 can be an expensive operation
>          */
>         for (; i < bulk_len; i++)
> -               page_pool_return_page(pool, data[i]);
> +               page_pool_return_page(pool, (__force netmem_ref)data[i]);
>  }
>  EXPORT_SYMBOL(page_pool_put_page_bulk);
>
> -static struct page *page_pool_drain_frag(struct page_pool *pool,
> -                                        struct page *page)
> +static netmem_ref page_pool_drain_frag(struct page_pool *pool,
> +                                      netmem_ref netmem)
>  {
>         long drain_count = BIAS_MAX - pool->frag_users;
>
>         /* Some user is still using the page frag */
> -       if (likely(page_pool_unref_page(page, drain_count)))
> -               return NULL;
> +       if (likely(page_pool_unref_netmem(netmem, drain_count)))
> +               return 0;
>
> -       if (__page_pool_page_can_be_recycled(page)) {
> +       if (__page_pool_page_can_be_recycled(netmem)) {
>                 if (pool->p.flags & PP_FLAG_DMA_SYNC_DEV)
> -                       page_pool_dma_sync_for_device(pool, page, -1);
> +                       page_pool_dma_sync_for_device(pool, netmem, -1);
>
> -               return page;
> +               return netmem;
>         }
>
> -       page_pool_return_page(pool, page);
> -       return NULL;
> +       page_pool_return_page(pool, netmem);
> +       return 0;
>  }
>
>  static void page_pool_free_frag(struct page_pool *pool)
>  {
>         long drain_count = BIAS_MAX - pool->frag_users;
> -       struct page *page = pool->frag_page;
> +       netmem_ref netmem = pool->frag_page;
>
> -       pool->frag_page = NULL;
> +       pool->frag_page = 0;
>
> -       if (!page || page_pool_unref_page(page, drain_count))
> +       if (!netmem || page_pool_unref_netmem(netmem, drain_count))
>                 return;
>
> -       page_pool_return_page(pool, page);
> +       page_pool_return_page(pool, netmem);
>  }
>
> -struct page *page_pool_alloc_frag(struct page_pool *pool,
> -                                 unsigned int *offset,
> -                                 unsigned int size, gfp_t gfp)
> +netmem_ref page_pool_alloc_frag_netmem(struct page_pool *pool,
> +                                      unsigned int *offset, unsigned int size,
> +                                      gfp_t gfp)
>  {
>         unsigned int max_size = PAGE_SIZE << pool->p.order;
> -       struct page *page = pool->frag_page;
> +       netmem_ref netmem = pool->frag_page;
>
>         if (WARN_ON(size > max_size))
> -               return NULL;
> +               return 0;
>
>         size = ALIGN(size, dma_get_cache_alignment());
>         *offset = pool->frag_offset;
>
> -       if (page && *offset + size > max_size) {
> -               page = page_pool_drain_frag(pool, page);
> -               if (page) {
> +       if (netmem && *offset + size > max_size) {
> +               netmem = page_pool_drain_frag(pool, netmem);
> +               if (netmem) {
>                         alloc_stat_inc(pool, fast);
>                         goto frag_reset;
>                 }
>         }
>
> -       if (!page) {
> -               page = page_pool_alloc_pages(pool, gfp);
> -               if (unlikely(!page)) {
> -                       pool->frag_page = NULL;
> -                       return NULL;
> +       if (!netmem) {
> +               netmem = page_pool_alloc_netmem(pool, gfp);
> +               if (unlikely(!netmem)) {
> +                       pool->frag_page = 0;
> +                       return 0;
>                 }
>
> -               pool->frag_page = page;
> +               pool->frag_page = netmem;
>
>  frag_reset:
>                 pool->frag_users = 1;
>                 *offset = 0;
>                 pool->frag_offset = size;
> -               page_pool_fragment_page(page, BIAS_MAX);
> -               return page;
> +               page_pool_fragment_netmem(netmem, BIAS_MAX);
> +               return netmem;
>         }
>
>         pool->frag_users++;
>         pool->frag_offset = *offset + size;
>         alloc_stat_inc(pool, fast);
> -       return page;
> +       return netmem;
> +}
> +EXPORT_SYMBOL(page_pool_alloc_frag_netmem);
> +
> +struct page *page_pool_alloc_frag(struct page_pool *pool, unsigned int *offset,
> +                                 unsigned int size, gfp_t gfp)
> +{
> +       return netmem_to_page(page_pool_alloc_frag_netmem(pool, offset, size,
> +                                                         gfp));
>  }
>  EXPORT_SYMBOL(page_pool_alloc_frag);
>
>  static void page_pool_empty_ring(struct page_pool *pool)
>  {
> -       struct page *page;
> +       netmem_ref netmem;
>
>         /* Empty recycle ring */
> -       while ((page = ptr_ring_consume_bh(&pool->ring))) {
> +       while ((netmem = (__force netmem_ref)ptr_ring_consume_bh(&pool->ring))) {
>                 /* Verify the refcnt invariant of cached pages */
> -               if (!(page_ref_count(page) == 1))
> +               if (!(page_ref_count(netmem_to_page(netmem)) == 1))
>                         pr_crit("%s() page_pool refcnt %d violation\n",
> -                               __func__, page_ref_count(page));
> +                               __func__, netmem_ref_count(netmem));
>
> -               page_pool_return_page(pool, page);
> +               page_pool_return_page(pool, netmem);
>         }
>  }
>
> @@ -927,7 +954,7 @@ static void __page_pool_destroy(struct page_pool *pool)
>
>  static void page_pool_empty_alloc_cache_once(struct page_pool *pool)
>  {
> -       struct page *page;
> +       netmem_ref netmem;
>
>         if (pool->destroy_cnt)
>                 return;
> @@ -937,8 +964,8 @@ static void page_pool_empty_alloc_cache_once(struct page_pool *pool)
>          * call concurrently.
>          */
>         while (pool->alloc.count) {
> -               page = pool->alloc.cache[--pool->alloc.count];
> -               page_pool_return_page(pool, page);
> +               netmem = pool->alloc.cache[--pool->alloc.count];
> +               page_pool_return_page(pool, netmem);
>         }
>  }
>
> @@ -1044,15 +1071,15 @@ EXPORT_SYMBOL(page_pool_destroy);
>  /* Caller must provide appropriate safe context, e.g. NAPI. */
>  void page_pool_update_nid(struct page_pool *pool, int new_nid)
>  {
> -       struct page *page;
> +       netmem_ref netmem;
>
>         trace_page_pool_update_nid(pool, new_nid);
>         pool->p.nid = new_nid;
>
>         /* Flush pool alloc cache, as refill will check NUMA node */
>         while (pool->alloc.count) {
> -               page = pool->alloc.cache[--pool->alloc.count];
> -               page_pool_return_page(pool, page);
> +               netmem = pool->alloc.cache[--pool->alloc.count];
> +               page_pool_return_page(pool, netmem);
>         }
>  }
>  EXPORT_SYMBOL(page_pool_update_nid);
> diff --git a/net/core/skbuff.c b/net/core/skbuff.c
> index 17617c29be2d..7193ee9737a0 100644
> --- a/net/core/skbuff.c
> +++ b/net/core/skbuff.c
> @@ -1005,8 +1005,9 @@ int skb_cow_data_for_xdp(struct page_pool *pool, struct sk_buff **pskb,
>  EXPORT_SYMBOL(skb_cow_data_for_xdp);
>
>  #if IS_ENABLED(CONFIG_PAGE_POOL)
> -bool napi_pp_put_page(struct page *page, bool napi_safe)
> +bool napi_pp_put_page(netmem_ref netmem, bool napi_safe)
>  {
> +       struct page *page = netmem_to_page(netmem);
>         bool allow_direct = false;
>         struct page_pool *pp;
>
> @@ -1043,7 +1044,7 @@ bool napi_pp_put_page(struct page *page, bool napi_safe)
>          * The page will be returned to the pool here regardless of the
>          * 'flipped' fragment being in use or not.
>          */
> -       page_pool_put_full_page(pp, page, allow_direct);
> +       page_pool_put_full_netmem(pp, page_to_netmem(page), allow_direct);
>
>         return true;
>  }
> @@ -1054,7 +1055,7 @@ static bool skb_pp_recycle(struct sk_buff *skb, void *data, bool napi_safe)
>  {
>         if (!IS_ENABLED(CONFIG_PAGE_POOL) || !skb->pp_recycle)
>                 return false;
> -       return napi_pp_put_page(virt_to_page(data), napi_safe);
> +       return napi_pp_put_page(page_to_netmem(virt_to_page(data)), napi_safe);
>  }
>
>  /**
> --
> 2.44.0.396.g6e790dbe36-goog
>
diff mbox series

Patch

diff --git a/include/linux/skbuff.h b/include/linux/skbuff.h
index b945af8a6208..78659c8efa4e 100644
--- a/include/linux/skbuff.h
+++ b/include/linux/skbuff.h
@@ -3521,7 +3521,7 @@  int skb_pp_cow_data(struct page_pool *pool, struct sk_buff **pskb,
 		    unsigned int headroom);
 int skb_cow_data_for_xdp(struct page_pool *pool, struct sk_buff **pskb,
 			 struct bpf_prog *prog);
-bool napi_pp_put_page(struct page *page, bool napi_safe);
+bool napi_pp_put_page(netmem_ref netmem, bool napi_safe);
 
 static inline void
 skb_page_unref(const struct sk_buff *skb, struct page *page, bool napi_safe)
@@ -3539,7 +3539,7 @@  napi_frag_unref(skb_frag_t *frag, bool recycle, bool napi_safe)
 	struct page *page = skb_frag_page(frag);
 
 #ifdef CONFIG_PAGE_POOL
-	if (recycle && napi_pp_put_page(page, napi_safe))
+	if (recycle && napi_pp_put_page(page_to_netmem(page), napi_safe))
 		return;
 #endif
 	put_page(page);
diff --git a/include/net/netmem.h b/include/net/netmem.h
index ca17ea1d33f8..21f53b29e5fe 100644
--- a/include/net/netmem.h
+++ b/include/net/netmem.h
@@ -88,4 +88,19 @@  static inline netmem_ref page_to_netmem(struct page *page)
 	return (__force netmem_ref)page;
 }
 
+static inline int netmem_ref_count(netmem_ref netmem)
+{
+	return page_ref_count(netmem_to_page(netmem));
+}
+
+static inline unsigned long netmem_to_pfn(netmem_ref netmem)
+{
+	return page_to_pfn(netmem_to_page(netmem));
+}
+
+static inline netmem_ref netmem_compound_head(netmem_ref netmem)
+{
+	return page_to_netmem(compound_head(netmem_to_page(netmem)));
+}
+
 #endif /* _NET_NETMEM_H */
diff --git a/include/net/page_pool/helpers.h b/include/net/page_pool/helpers.h
index 1d397c1a0043..61814f91a458 100644
--- a/include/net/page_pool/helpers.h
+++ b/include/net/page_pool/helpers.h
@@ -53,6 +53,8 @@ 
 #define _NET_PAGE_POOL_HELPERS_H
 
 #include <net/page_pool/types.h>
+#include <net/net_debug.h>
+#include <net/netmem.h>
 
 #ifdef CONFIG_PAGE_POOL_STATS
 /* Deprecated driver-facing API, use netlink instead */
@@ -101,7 +103,7 @@  static inline struct page *page_pool_dev_alloc_pages(struct page_pool *pool)
  * Get a page fragment from the page allocator or page_pool caches.
  *
  * Return:
- * Return allocated page fragment, otherwise return NULL.
+ * Return allocated page fragment, otherwise return 0.
  */
 static inline struct page *page_pool_dev_alloc_frag(struct page_pool *pool,
 						    unsigned int *offset,
@@ -112,22 +114,22 @@  static inline struct page *page_pool_dev_alloc_frag(struct page_pool *pool,
 	return page_pool_alloc_frag(pool, offset, size, gfp);
 }
 
-static inline struct page *page_pool_alloc(struct page_pool *pool,
-					   unsigned int *offset,
-					   unsigned int *size, gfp_t gfp)
+static inline netmem_ref page_pool_alloc(struct page_pool *pool,
+					 unsigned int *offset,
+					 unsigned int *size, gfp_t gfp)
 {
 	unsigned int max_size = PAGE_SIZE << pool->p.order;
-	struct page *page;
+	netmem_ref netmem;
 
 	if ((*size << 1) > max_size) {
 		*size = max_size;
 		*offset = 0;
-		return page_pool_alloc_pages(pool, gfp);
+		return page_pool_alloc_netmem(pool, gfp);
 	}
 
-	page = page_pool_alloc_frag(pool, offset, *size, gfp);
-	if (unlikely(!page))
-		return NULL;
+	netmem = page_pool_alloc_frag_netmem(pool, offset, *size, gfp);
+	if (unlikely(!netmem))
+		return 0;
 
 	/* There is very likely not enough space for another fragment, so append
 	 * the remaining size to the current fragment to avoid truesize
@@ -138,7 +140,7 @@  static inline struct page *page_pool_alloc(struct page_pool *pool,
 		pool->frag_offset = max_size;
 	}
 
-	return page;
+	return netmem;
 }
 
 /**
@@ -152,7 +154,7 @@  static inline struct page *page_pool_alloc(struct page_pool *pool,
  * utilization and performance penalty.
  *
  * Return:
- * Return allocated page or page fragment, otherwise return NULL.
+ * Return allocated page or page fragment, otherwise return 0.
  */
 static inline struct page *page_pool_dev_alloc(struct page_pool *pool,
 					       unsigned int *offset,
@@ -160,7 +162,7 @@  static inline struct page *page_pool_dev_alloc(struct page_pool *pool,
 {
 	gfp_t gfp = (GFP_ATOMIC | __GFP_NOWARN);
 
-	return page_pool_alloc(pool, offset, size, gfp);
+	return netmem_to_page(page_pool_alloc(pool, offset, size, gfp));
 }
 
 static inline void *page_pool_alloc_va(struct page_pool *pool,
@@ -170,9 +172,10 @@  static inline void *page_pool_alloc_va(struct page_pool *pool,
 	struct page *page;
 
 	/* Mask off __GFP_HIGHMEM to ensure we can use page_address() */
-	page = page_pool_alloc(pool, &offset, size, gfp & ~__GFP_HIGHMEM);
+	page = netmem_to_page(
+		page_pool_alloc(pool, &offset, size, gfp & ~__GFP_HIGHMEM));
 	if (unlikely(!page))
-		return NULL;
+		return 0;
 
 	return page_address(page) + offset;
 }
@@ -187,7 +190,7 @@  static inline void *page_pool_alloc_va(struct page_pool *pool,
  * it returns va of the allocated page or page fragment.
  *
  * Return:
- * Return the va for the allocated page or page fragment, otherwise return NULL.
+ * Return the va for the allocated page or page fragment, otherwise return 0.
  */
 static inline void *page_pool_dev_alloc_va(struct page_pool *pool,
 					   unsigned int *size)
@@ -210,6 +213,11 @@  inline enum dma_data_direction page_pool_get_dma_dir(struct page_pool *pool)
 	return pool->p.dma_dir;
 }
 
+static inline void page_pool_fragment_netmem(netmem_ref netmem, long nr)
+{
+	atomic_long_set(&netmem_to_page(netmem)->pp_ref_count, nr);
+}
+
 /**
  * page_pool_fragment_page() - split a fresh page into fragments
  * @page:	page to split
@@ -230,11 +238,12 @@  inline enum dma_data_direction page_pool_get_dma_dir(struct page_pool *pool)
  */
 static inline void page_pool_fragment_page(struct page *page, long nr)
 {
-	atomic_long_set(&page->pp_ref_count, nr);
+	page_pool_fragment_netmem(page_to_netmem(page), nr);
 }
 
-static inline long page_pool_unref_page(struct page *page, long nr)
+static inline long page_pool_unref_netmem(netmem_ref netmem, long nr)
 {
+	struct page *page = netmem_to_page(netmem);
 	long ret;
 
 	/* If nr == pp_ref_count then we have cleared all remaining
@@ -277,15 +286,41 @@  static inline long page_pool_unref_page(struct page *page, long nr)
 	return ret;
 }
 
+static inline long page_pool_unref_page(struct page *page, long nr)
+{
+	return page_pool_unref_netmem(page_to_netmem(page), nr);
+}
+
+static inline void page_pool_ref_netmem(netmem_ref netmem)
+{
+	atomic_long_inc(&netmem_to_page(netmem)->pp_ref_count);
+}
+
 static inline void page_pool_ref_page(struct page *page)
 {
-	atomic_long_inc(&page->pp_ref_count);
+	page_pool_ref_netmem(page_to_netmem(page));
 }
 
-static inline bool page_pool_is_last_ref(struct page *page)
+static inline bool page_pool_is_last_ref(netmem_ref netmem)
 {
 	/* If page_pool_unref_page() returns 0, we were the last user */
-	return page_pool_unref_page(page, 1) == 0;
+	return page_pool_unref_netmem(netmem, 1) == 0;
+}
+
+static inline void page_pool_put_netmem(struct page_pool *pool,
+					netmem_ref netmem,
+					unsigned int dma_sync_size,
+					bool allow_direct)
+{
+	/* When page_pool isn't compiled-in, net/core/xdp.c doesn't
+	 * allow registering MEM_TYPE_PAGE_POOL, but shield linker.
+	 */
+#ifdef CONFIG_PAGE_POOL
+	if (!page_pool_is_last_ref(netmem))
+		return;
+
+	page_pool_put_unrefed_netmem(pool, netmem, dma_sync_size, allow_direct);
+#endif
 }
 
 /**
@@ -306,15 +341,15 @@  static inline void page_pool_put_page(struct page_pool *pool,
 				      unsigned int dma_sync_size,
 				      bool allow_direct)
 {
-	/* When page_pool isn't compiled-in, net/core/xdp.c doesn't
-	 * allow registering MEM_TYPE_PAGE_POOL, but shield linker.
-	 */
-#ifdef CONFIG_PAGE_POOL
-	if (!page_pool_is_last_ref(page))
-		return;
+	page_pool_put_netmem(pool, page_to_netmem(page), dma_sync_size,
+			     allow_direct);
+}
 
-	page_pool_put_unrefed_page(pool, page, dma_sync_size, allow_direct);
-#endif
+static inline void page_pool_put_full_netmem(struct page_pool *pool,
+					     netmem_ref netmem,
+					     bool allow_direct)
+{
+	page_pool_put_netmem(pool, netmem, -1, allow_direct);
 }
 
 /**
@@ -329,7 +364,7 @@  static inline void page_pool_put_page(struct page_pool *pool,
 static inline void page_pool_put_full_page(struct page_pool *pool,
 					   struct page *page, bool allow_direct)
 {
-	page_pool_put_page(pool, page, -1, allow_direct);
+	page_pool_put_netmem(pool, page_to_netmem(page), -1, allow_direct);
 }
 
 /**
@@ -363,6 +398,18 @@  static inline void page_pool_free_va(struct page_pool *pool, void *va,
 	page_pool_put_page(pool, virt_to_head_page(va), -1, allow_direct);
 }
 
+static inline dma_addr_t page_pool_get_dma_addr_netmem(netmem_ref netmem)
+{
+	struct page *page = netmem_to_page(netmem);
+
+	dma_addr_t ret = page->dma_addr;
+
+	if (PAGE_POOL_32BIT_ARCH_WITH_64BIT_DMA)
+		ret <<= PAGE_SHIFT;
+
+	return ret;
+}
+
 /**
  * page_pool_get_dma_addr() - Retrieve the stored DMA address.
  * @page:	page allocated from a page pool
@@ -372,16 +419,14 @@  static inline void page_pool_free_va(struct page_pool *pool, void *va,
  */
 static inline dma_addr_t page_pool_get_dma_addr(struct page *page)
 {
-	dma_addr_t ret = page->dma_addr;
-
-	if (PAGE_POOL_32BIT_ARCH_WITH_64BIT_DMA)
-		ret <<= PAGE_SHIFT;
-
-	return ret;
+	return page_pool_get_dma_addr_netmem(page_to_netmem(page));
 }
 
-static inline bool page_pool_set_dma_addr(struct page *page, dma_addr_t addr)
+static inline bool page_pool_set_dma_addr_netmem(netmem_ref netmem,
+						 dma_addr_t addr)
 {
+	struct page *page = netmem_to_page(netmem);
+
 	if (PAGE_POOL_32BIT_ARCH_WITH_64BIT_DMA) {
 		page->dma_addr = addr >> PAGE_SHIFT;
 
@@ -395,6 +440,11 @@  static inline bool page_pool_set_dma_addr(struct page *page, dma_addr_t addr)
 	return false;
 }
 
+static inline bool page_pool_set_dma_addr(struct page *page, dma_addr_t addr)
+{
+	return page_pool_set_dma_addr_netmem(page_to_netmem(page), addr);
+}
+
 static inline bool page_pool_put(struct page_pool *pool)
 {
 	return refcount_dec_and_test(&pool->user_cnt);
diff --git a/include/net/page_pool/types.h b/include/net/page_pool/types.h
index 07e6afafedbe..0d164624f16d 100644
--- a/include/net/page_pool/types.h
+++ b/include/net/page_pool/types.h
@@ -40,7 +40,7 @@ 
 #define PP_ALLOC_CACHE_REFILL	64
 struct pp_alloc_cache {
 	u32 count;
-	struct page *cache[PP_ALLOC_CACHE_SIZE];
+	netmem_ref cache[PP_ALLOC_CACHE_SIZE];
 };
 
 /**
@@ -73,7 +73,7 @@  struct page_pool_params {
 	struct_group_tagged(page_pool_params_slow, slow,
 		struct net_device *netdev;
 /* private: used by test code only */
-		void (*init_callback)(struct page *page, void *arg);
+		void (*init_callback)(netmem_ref netmem, void *arg);
 		void *init_arg;
 	);
 };
@@ -131,8 +131,8 @@  struct page_pool_stats {
 struct memory_provider_ops {
 	int (*init)(struct page_pool *pool);
 	void (*destroy)(struct page_pool *pool);
-	struct page *(*alloc_pages)(struct page_pool *pool, gfp_t gfp);
-	bool (*release_page)(struct page_pool *pool, struct page *page);
+	netmem_ref (*alloc_pages)(struct page_pool *pool, gfp_t gfp);
+	bool (*release_page)(struct page_pool *pool, netmem_ref netmem);
 };
 
 struct pp_memory_provider_params {
@@ -147,7 +147,7 @@  struct page_pool {
 	bool has_init_callback;
 
 	long frag_users;
-	struct page *frag_page;
+	netmem_ref frag_page;
 	unsigned int frag_offset;
 	u32 pages_state_hold_cnt;
 
@@ -219,8 +219,12 @@  struct page_pool {
 };
 
 struct page *page_pool_alloc_pages(struct page_pool *pool, gfp_t gfp);
+netmem_ref page_pool_alloc_netmem(struct page_pool *pool, gfp_t gfp);
 struct page *page_pool_alloc_frag(struct page_pool *pool, unsigned int *offset,
 				  unsigned int size, gfp_t gfp);
+netmem_ref page_pool_alloc_frag_netmem(struct page_pool *pool,
+				       unsigned int *offset, unsigned int size,
+				       gfp_t gfp);
 struct page_pool *page_pool_create(const struct page_pool_params *params);
 struct page_pool *page_pool_create_percpu(const struct page_pool_params *params,
 					  int cpuid);
@@ -250,6 +254,9 @@  static inline void page_pool_put_page_bulk(struct page_pool *pool, void **data,
 }
 #endif
 
+void page_pool_put_unrefed_netmem(struct page_pool *pool, netmem_ref netmem,
+				  unsigned int dma_sync_size,
+				  bool allow_direct);
 void page_pool_put_unrefed_page(struct page_pool *pool, struct page *page,
 				unsigned int dma_sync_size,
 				bool allow_direct);
diff --git a/include/trace/events/page_pool.h b/include/trace/events/page_pool.h
index 6834356b2d2a..c5b6383ff276 100644
--- a/include/trace/events/page_pool.h
+++ b/include/trace/events/page_pool.h
@@ -42,51 +42,52 @@  TRACE_EVENT(page_pool_release,
 TRACE_EVENT(page_pool_state_release,
 
 	TP_PROTO(const struct page_pool *pool,
-		 const struct page *page, u32 release),
+		 netmem_ref netmem, u32 release),
 
-	TP_ARGS(pool, page, release),
+	TP_ARGS(pool, netmem, release),
 
 	TP_STRUCT__entry(
 		__field(const struct page_pool *,	pool)
-		__field(const struct page *,		page)
+		__field(netmem_ref,			netmem)
 		__field(u32,				release)
 		__field(unsigned long,			pfn)
 	),
 
 	TP_fast_assign(
 		__entry->pool		= pool;
-		__entry->page		= page;
+		__entry->netmem		= netmem;
 		__entry->release	= release;
-		__entry->pfn		= page_to_pfn(page);
+		__entry->pfn		= netmem_to_pfn(netmem);
 	),
 
-	TP_printk("page_pool=%p page=%p pfn=0x%lx release=%u",
-		  __entry->pool, __entry->page, __entry->pfn, __entry->release)
+	TP_printk("page_pool=%p netmem=%lu pfn=0x%lx release=%u",
+		  __entry->pool, (__force unsigned long)__entry->netmem,
+		  __entry->pfn, __entry->release)
 );
 
 TRACE_EVENT(page_pool_state_hold,
 
 	TP_PROTO(const struct page_pool *pool,
-		 const struct page *page, u32 hold),
+		 netmem_ref netmem, u32 hold),
 
-	TP_ARGS(pool, page, hold),
+	TP_ARGS(pool, netmem, hold),
 
 	TP_STRUCT__entry(
 		__field(const struct page_pool *,	pool)
-		__field(const struct page *,		page)
+		__field(netmem_ref,			netmem)
 		__field(u32,				hold)
 		__field(unsigned long,			pfn)
 	),
 
 	TP_fast_assign(
 		__entry->pool	= pool;
-		__entry->page	= page;
+		__entry->netmem	= netmem;
 		__entry->hold	= hold;
-		__entry->pfn	= page_to_pfn(page);
+		__entry->pfn	= netmem_to_pfn(netmem);
 	),
 
-	TP_printk("page_pool=%p page=%p pfn=0x%lx hold=%u",
-		  __entry->pool, __entry->page, __entry->pfn, __entry->hold)
+	TP_printk("page_pool=%p netmem=%lu pfn=0x%lx hold=%u",
+		  __entry->pool, __entry->netmem, __entry->pfn, __entry->hold)
 );
 
 TRACE_EVENT(page_pool_update_nid,
diff --git a/net/bpf/test_run.c b/net/bpf/test_run.c
index 61efeadaff8d..fc300e807e1d 100644
--- a/net/bpf/test_run.c
+++ b/net/bpf/test_run.c
@@ -127,9 +127,10 @@  struct xdp_test_data {
 #define TEST_XDP_FRAME_SIZE (PAGE_SIZE - sizeof(struct xdp_page_head))
 #define TEST_XDP_MAX_BATCH 256
 
-static void xdp_test_run_init_page(struct page *page, void *arg)
+static void xdp_test_run_init_page(netmem_ref netmem, void *arg)
 {
-	struct xdp_page_head *head = phys_to_virt(page_to_phys(page));
+	struct xdp_page_head *head =
+		phys_to_virt(page_to_phys(netmem_to_page(netmem)));
 	struct xdp_buff *new_ctx, *orig_ctx;
 	u32 headroom = XDP_PACKET_HEADROOM;
 	struct xdp_test_data *xdp = arg;
diff --git a/net/core/page_pool.c b/net/core/page_pool.c
index 795b7ff1c01f..c8125be3a6e2 100644
--- a/net/core/page_pool.c
+++ b/net/core/page_pool.c
@@ -329,19 +329,18 @@  struct page_pool *page_pool_create(const struct page_pool_params *params)
 }
 EXPORT_SYMBOL(page_pool_create);
 
-static void page_pool_return_page(struct page_pool *pool, struct page *page);
+static void page_pool_return_page(struct page_pool *pool, netmem_ref netmem);
 
-noinline
-static struct page *page_pool_refill_alloc_cache(struct page_pool *pool)
+static noinline netmem_ref page_pool_refill_alloc_cache(struct page_pool *pool)
 {
 	struct ptr_ring *r = &pool->ring;
-	struct page *page;
+	netmem_ref netmem;
 	int pref_nid; /* preferred NUMA node */
 
 	/* Quicker fallback, avoid locks when ring is empty */
 	if (__ptr_ring_empty(r)) {
 		alloc_stat_inc(pool, empty);
-		return NULL;
+		return 0;
 	}
 
 	/* Softirq guarantee CPU and thus NUMA node is stable. This,
@@ -356,56 +355,56 @@  static struct page *page_pool_refill_alloc_cache(struct page_pool *pool)
 
 	/* Refill alloc array, but only if NUMA match */
 	do {
-		page = __ptr_ring_consume(r);
-		if (unlikely(!page))
+		netmem = (__force netmem_ref)__ptr_ring_consume(r);
+		if (unlikely(!netmem))
 			break;
 
-		if (likely(page_to_nid(page) == pref_nid)) {
-			pool->alloc.cache[pool->alloc.count++] = page;
+		if (likely(page_to_nid(netmem_to_page(netmem)) == pref_nid)) {
+			pool->alloc.cache[pool->alloc.count++] = netmem;
 		} else {
 			/* NUMA mismatch;
 			 * (1) release 1 page to page-allocator and
 			 * (2) break out to fallthrough to alloc_pages_node.
 			 * This limit stress on page buddy alloactor.
 			 */
-			page_pool_return_page(pool, page);
+			page_pool_return_page(pool, netmem);
 			alloc_stat_inc(pool, waive);
-			page = NULL;
+			netmem = 0;
 			break;
 		}
 	} while (pool->alloc.count < PP_ALLOC_CACHE_REFILL);
 
 	/* Return last page */
 	if (likely(pool->alloc.count > 0)) {
-		page = pool->alloc.cache[--pool->alloc.count];
+		netmem = pool->alloc.cache[--pool->alloc.count];
 		alloc_stat_inc(pool, refill);
 	}
 
-	return page;
+	return netmem;
 }
 
 /* fast path */
-static struct page *__page_pool_get_cached(struct page_pool *pool)
+static netmem_ref __page_pool_get_cached(struct page_pool *pool)
 {
-	struct page *page;
+	netmem_ref netmem;
 
 	/* Caller MUST guarantee safe non-concurrent access, e.g. softirq */
 	if (likely(pool->alloc.count)) {
 		/* Fast-path */
-		page = pool->alloc.cache[--pool->alloc.count];
+		netmem = pool->alloc.cache[--pool->alloc.count];
 		alloc_stat_inc(pool, fast);
 	} else {
-		page = page_pool_refill_alloc_cache(pool);
+		netmem = page_pool_refill_alloc_cache(pool);
 	}
 
-	return page;
+	return netmem;
 }
 
 static void page_pool_dma_sync_for_device(struct page_pool *pool,
-					  struct page *page,
+					  netmem_ref netmem,
 					  unsigned int dma_sync_size)
 {
-	dma_addr_t dma_addr = page_pool_get_dma_addr(page);
+	dma_addr_t dma_addr = page_pool_get_dma_addr_netmem(netmem);
 
 	dma_sync_size = min(dma_sync_size, pool->p.max_len);
 	dma_sync_single_range_for_device(pool->p.dev, dma_addr,
@@ -413,7 +412,7 @@  static void page_pool_dma_sync_for_device(struct page_pool *pool,
 					 pool->p.dma_dir);
 }
 
-static bool page_pool_dma_map(struct page_pool *pool, struct page *page)
+static bool page_pool_dma_map(struct page_pool *pool, netmem_ref netmem)
 {
 	dma_addr_t dma;
 
@@ -422,18 +421,18 @@  static bool page_pool_dma_map(struct page_pool *pool, struct page *page)
 	 * into page private data (i.e 32bit cpu with 64bit DMA caps)
 	 * This mapping is kept for lifetime of page, until leaving pool.
 	 */
-	dma = dma_map_page_attrs(pool->p.dev, page, 0,
-				 (PAGE_SIZE << pool->p.order),
-				 pool->p.dma_dir, DMA_ATTR_SKIP_CPU_SYNC |
-						  DMA_ATTR_WEAK_ORDERING);
+	dma = dma_map_page_attrs(pool->p.dev, netmem_to_page(netmem), 0,
+				 (PAGE_SIZE << pool->p.order), pool->p.dma_dir,
+				 DMA_ATTR_SKIP_CPU_SYNC |
+					 DMA_ATTR_WEAK_ORDERING);
 	if (dma_mapping_error(pool->p.dev, dma))
 		return false;
 
-	if (page_pool_set_dma_addr(page, dma))
+	if (page_pool_set_dma_addr_netmem(netmem, dma))
 		goto unmap_failed;
 
 	if (pool->p.flags & PP_FLAG_DMA_SYNC_DEV)
-		page_pool_dma_sync_for_device(pool, page, pool->p.max_len);
+		page_pool_dma_sync_for_device(pool, netmem, pool->p.max_len);
 
 	return true;
 
@@ -445,9 +444,10 @@  static bool page_pool_dma_map(struct page_pool *pool, struct page *page)
 	return false;
 }
 
-static void page_pool_set_pp_info(struct page_pool *pool,
-				  struct page *page)
+static void page_pool_set_pp_info(struct page_pool *pool, netmem_ref netmem)
 {
+	struct page *page = netmem_to_page(netmem);
+
 	page->pp = pool;
 	page->pp_magic |= PP_SIGNATURE;
 
@@ -457,13 +457,15 @@  static void page_pool_set_pp_info(struct page_pool *pool,
 	 * is dirtying the same cache line as the page->pp_magic above, so
 	 * the overhead is negligible.
 	 */
-	page_pool_fragment_page(page, 1);
+	page_pool_fragment_netmem(netmem, 1);
 	if (pool->has_init_callback)
-		pool->slow.init_callback(page, pool->slow.init_arg);
+		pool->slow.init_callback(netmem, pool->slow.init_arg);
 }
 
-static void page_pool_clear_pp_info(struct page *page)
+static void page_pool_clear_pp_info(netmem_ref netmem)
 {
+	struct page *page = netmem_to_page(netmem);
+
 	page->pp_magic = 0;
 	page->pp = NULL;
 }
@@ -479,34 +481,34 @@  static struct page *__page_pool_alloc_page_order(struct page_pool *pool,
 		return NULL;
 
 	if ((pool->p.flags & PP_FLAG_DMA_MAP) &&
-	    unlikely(!page_pool_dma_map(pool, page))) {
+	    unlikely(!page_pool_dma_map(pool, page_to_netmem(page)))) {
 		put_page(page);
 		return NULL;
 	}
 
 	alloc_stat_inc(pool, slow_high_order);
-	page_pool_set_pp_info(pool, page);
+	page_pool_set_pp_info(pool, page_to_netmem(page));
 
 	/* Track how many pages are held 'in-flight' */
 	pool->pages_state_hold_cnt++;
-	trace_page_pool_state_hold(pool, page, pool->pages_state_hold_cnt);
+	trace_page_pool_state_hold(pool, page_to_netmem(page),
+				   pool->pages_state_hold_cnt);
 	return page;
 }
 
 /* slow path */
-noinline
-static struct page *__page_pool_alloc_pages_slow(struct page_pool *pool,
-						 gfp_t gfp)
+static noinline netmem_ref __page_pool_alloc_pages_slow(struct page_pool *pool,
+							gfp_t gfp)
 {
 	const int bulk = PP_ALLOC_CACHE_REFILL;
 	unsigned int pp_flags = pool->p.flags;
 	unsigned int pp_order = pool->p.order;
-	struct page *page;
+	netmem_ref netmem;
 	int i, nr_pages;
 
 	/* Don't support bulk alloc for high-order pages */
 	if (unlikely(pp_order))
-		return __page_pool_alloc_page_order(pool, gfp);
+		return page_to_netmem(__page_pool_alloc_page_order(pool, gfp));
 
 	/* Unnecessary as alloc cache is empty, but guarantees zero count */
 	if (unlikely(pool->alloc.count > 0))
@@ -515,60 +517,67 @@  static struct page *__page_pool_alloc_pages_slow(struct page_pool *pool,
 	/* Mark empty alloc.cache slots "empty" for alloc_pages_bulk_array */
 	memset(&pool->alloc.cache, 0, sizeof(void *) * bulk);
 
-	nr_pages = alloc_pages_bulk_array_node(gfp, pool->p.nid, bulk,
-					       pool->alloc.cache);
+	nr_pages = alloc_pages_bulk_array_node(gfp,
+					       pool->p.nid, bulk,
+					       (struct page **)pool->alloc.cache);
 	if (unlikely(!nr_pages))
-		return NULL;
+		return 0;
 
 	/* Pages have been filled into alloc.cache array, but count is zero and
 	 * page element have not been (possibly) DMA mapped.
 	 */
 	for (i = 0; i < nr_pages; i++) {
-		page = pool->alloc.cache[i];
+		netmem = pool->alloc.cache[i];
 		if ((pp_flags & PP_FLAG_DMA_MAP) &&
-		    unlikely(!page_pool_dma_map(pool, page))) {
-			put_page(page);
+		    unlikely(!page_pool_dma_map(pool, netmem))) {
+			put_page(netmem_to_page(netmem));
 			continue;
 		}
 
-		page_pool_set_pp_info(pool, page);
-		pool->alloc.cache[pool->alloc.count++] = page;
+		page_pool_set_pp_info(pool, netmem);
+		pool->alloc.cache[pool->alloc.count++] = netmem;
 		/* Track how many pages are held 'in-flight' */
 		pool->pages_state_hold_cnt++;
-		trace_page_pool_state_hold(pool, page,
+		trace_page_pool_state_hold(pool, netmem,
 					   pool->pages_state_hold_cnt);
 	}
 
 	/* Return last page */
 	if (likely(pool->alloc.count > 0)) {
-		page = pool->alloc.cache[--pool->alloc.count];
+		netmem = pool->alloc.cache[--pool->alloc.count];
 		alloc_stat_inc(pool, slow);
 	} else {
-		page = NULL;
+		netmem = 0;
 	}
 
 	/* When page just alloc'ed is should/must have refcnt 1. */
-	return page;
+	return netmem;
 }
 
 /* For using page_pool replace: alloc_pages() API calls, but provide
  * synchronization guarantee for allocation side.
  */
-struct page *page_pool_alloc_pages(struct page_pool *pool, gfp_t gfp)
+netmem_ref page_pool_alloc_netmem(struct page_pool *pool, gfp_t gfp)
 {
-	struct page *page;
+	netmem_ref netmem;
 
 	/* Fast-path: Get a page from cache */
-	page = __page_pool_get_cached(pool);
-	if (page)
-		return page;
+	netmem = __page_pool_get_cached(pool);
+	if (netmem)
+		return netmem;
 
 	/* Slow-path: cache empty, do real allocation */
 	if (static_branch_unlikely(&page_pool_mem_providers) && pool->mp_ops)
-		page = pool->mp_ops->alloc_pages(pool, gfp);
+		netmem = pool->mp_ops->alloc_pages(pool, gfp);
 	else
-		page = __page_pool_alloc_pages_slow(pool, gfp);
-	return page;
+		netmem = __page_pool_alloc_pages_slow(pool, gfp);
+	return netmem;
+}
+EXPORT_SYMBOL(page_pool_alloc_netmem);
+
+struct page *page_pool_alloc_pages(struct page_pool *pool, gfp_t gfp)
+{
+	return netmem_to_page(page_pool_alloc_netmem(pool, gfp));
 }
 EXPORT_SYMBOL(page_pool_alloc_pages);
 
@@ -596,8 +605,8 @@  s32 page_pool_inflight(const struct page_pool *pool, bool strict)
 	return inflight;
 }
 
-static __always_inline
-void __page_pool_release_page_dma(struct page_pool *pool, struct page *page)
+static __always_inline void __page_pool_release_page_dma(struct page_pool *pool,
+							 netmem_ref netmem)
 {
 	dma_addr_t dma;
 
@@ -607,13 +616,13 @@  void __page_pool_release_page_dma(struct page_pool *pool, struct page *page)
 		 */
 		return;
 
-	dma = page_pool_get_dma_addr(page);
+	dma = page_pool_get_dma_addr_netmem(netmem);
 
 	/* When page is unmapped, it cannot be returned to our pool */
 	dma_unmap_page_attrs(pool->p.dev, dma,
 			     PAGE_SIZE << pool->p.order, pool->p.dma_dir,
 			     DMA_ATTR_SKIP_CPU_SYNC | DMA_ATTR_WEAK_ORDERING);
-	page_pool_set_dma_addr(page, 0);
+	page_pool_set_dma_addr_netmem(netmem, 0);
 }
 
 /* Disconnects a page (from a page_pool).  API users can have a need
@@ -621,26 +630,26 @@  void __page_pool_release_page_dma(struct page_pool *pool, struct page *page)
  * a regular page (that will eventually be returned to the normal
  * page-allocator via put_page).
  */
-void page_pool_return_page(struct page_pool *pool, struct page *page)
+void page_pool_return_page(struct page_pool *pool, netmem_ref netmem)
 {
 	int count;
 	bool put;
 
 	put = true;
 	if (static_branch_unlikely(&page_pool_mem_providers) && pool->mp_ops)
-		put = pool->mp_ops->release_page(pool, page);
+		put = pool->mp_ops->release_page(pool, netmem);
 	else
-		__page_pool_release_page_dma(pool, page);
+		__page_pool_release_page_dma(pool, netmem);
 
 	/* This may be the last page returned, releasing the pool, so
 	 * it is not safe to reference pool afterwards.
 	 */
 	count = atomic_inc_return_relaxed(&pool->pages_state_release_cnt);
-	trace_page_pool_state_release(pool, page, count);
+	trace_page_pool_state_release(pool, netmem, count);
 
 	if (put) {
-		page_pool_clear_pp_info(page);
-		put_page(page);
+		page_pool_clear_pp_info(netmem);
+		put_page(netmem_to_page(netmem));
 	}
 	/* An optimization would be to call __free_pages(page, pool->p.order)
 	 * knowing page is not part of page-cache (thus avoiding a
@@ -648,14 +657,14 @@  void page_pool_return_page(struct page_pool *pool, struct page *page)
 	 */
 }
 
-static bool page_pool_recycle_in_ring(struct page_pool *pool, struct page *page)
+static bool page_pool_recycle_in_ring(struct page_pool *pool, netmem_ref netmem)
 {
 	int ret;
 	/* BH protection not needed if current is softirq */
 	if (in_softirq())
-		ret = ptr_ring_produce(&pool->ring, page);
+		ret = ptr_ring_produce(&pool->ring, (__force void *)netmem);
 	else
-		ret = ptr_ring_produce_bh(&pool->ring, page);
+		ret = ptr_ring_produce_bh(&pool->ring, (__force void *)netmem);
 
 	if (!ret) {
 		recycle_stat_inc(pool, ring);
@@ -670,7 +679,7 @@  static bool page_pool_recycle_in_ring(struct page_pool *pool, struct page *page)
  *
  * Caller must provide appropriate safe context.
  */
-static bool page_pool_recycle_in_cache(struct page *page,
+static bool page_pool_recycle_in_cache(netmem_ref netmem,
 				       struct page_pool *pool)
 {
 	if (unlikely(pool->alloc.count == PP_ALLOC_CACHE_SIZE)) {
@@ -679,14 +688,15 @@  static bool page_pool_recycle_in_cache(struct page *page,
 	}
 
 	/* Caller MUST have verified/know (page_ref_count(page) == 1) */
-	pool->alloc.cache[pool->alloc.count++] = page;
+	pool->alloc.cache[pool->alloc.count++] = netmem;
 	recycle_stat_inc(pool, cached);
 	return true;
 }
 
-static bool __page_pool_page_can_be_recycled(const struct page *page)
+static bool __page_pool_page_can_be_recycled(netmem_ref netmem)
 {
-	return page_ref_count(page) == 1 && !page_is_pfmemalloc(page);
+	return page_ref_count(netmem_to_page(netmem)) == 1 &&
+	       !page_is_pfmemalloc(netmem_to_page(netmem));
 }
 
 /* If the page refcnt == 1, this will try to recycle the page.
@@ -695,8 +705,8 @@  static bool __page_pool_page_can_be_recycled(const struct page *page)
  * If the page refcnt != 1, then the page will be returned to memory
  * subsystem.
  */
-static __always_inline struct page *
-__page_pool_put_page(struct page_pool *pool, struct page *page,
+static __always_inline netmem_ref
+__page_pool_put_page(struct page_pool *pool, netmem_ref netmem,
 		     unsigned int dma_sync_size, bool allow_direct)
 {
 	lockdep_assert_no_hardirq();
@@ -710,19 +720,19 @@  __page_pool_put_page(struct page_pool *pool, struct page *page,
 	 * page is NOT reusable when allocated when system is under
 	 * some pressure. (page_is_pfmemalloc)
 	 */
-	if (likely(__page_pool_page_can_be_recycled(page))) {
+	if (likely(__page_pool_page_can_be_recycled(netmem))) {
 		/* Read barrier done in page_ref_count / READ_ONCE */
 
 		if (pool->p.flags & PP_FLAG_DMA_SYNC_DEV)
-			page_pool_dma_sync_for_device(pool, page,
+			page_pool_dma_sync_for_device(pool, netmem,
 						      dma_sync_size);
 
 		if (allow_direct && in_softirq() &&
-		    page_pool_recycle_in_cache(page, pool))
-			return NULL;
+		    page_pool_recycle_in_cache(netmem, pool))
+			return 0;
 
 		/* Page found as candidate for recycling */
-		return page;
+		return netmem;
 	}
 	/* Fallback/non-XDP mode: API user have elevated refcnt.
 	 *
@@ -738,21 +748,30 @@  __page_pool_put_page(struct page_pool *pool, struct page *page,
 	 * will be invoking put_page.
 	 */
 	recycle_stat_inc(pool, released_refcnt);
-	page_pool_return_page(pool, page);
+	page_pool_return_page(pool, netmem);
 
-	return NULL;
+	return 0;
 }
 
-void page_pool_put_unrefed_page(struct page_pool *pool, struct page *page,
-				unsigned int dma_sync_size, bool allow_direct)
+void page_pool_put_unrefed_netmem(struct page_pool *pool, netmem_ref netmem,
+				  unsigned int dma_sync_size, bool allow_direct)
 {
-	page = __page_pool_put_page(pool, page, dma_sync_size, allow_direct);
-	if (page && !page_pool_recycle_in_ring(pool, page)) {
+	netmem =
+		__page_pool_put_page(pool, netmem, dma_sync_size, allow_direct);
+	if (netmem && !page_pool_recycle_in_ring(pool, netmem)) {
 		/* Cache full, fallback to free pages */
 		recycle_stat_inc(pool, ring_full);
-		page_pool_return_page(pool, page);
+		page_pool_return_page(pool, netmem);
 	}
 }
+EXPORT_SYMBOL(page_pool_put_unrefed_netmem);
+
+void page_pool_put_unrefed_page(struct page_pool *pool, struct page *page,
+				unsigned int dma_sync_size, bool allow_direct)
+{
+	page_pool_put_unrefed_netmem(pool, page_to_netmem(page), dma_sync_size,
+				     allow_direct);
+}
 EXPORT_SYMBOL(page_pool_put_unrefed_page);
 
 /**
@@ -777,16 +796,16 @@  void page_pool_put_page_bulk(struct page_pool *pool, void **data,
 	bool in_softirq;
 
 	for (i = 0; i < count; i++) {
-		struct page *page = virt_to_head_page(data[i]);
+		netmem_ref netmem = page_to_netmem(virt_to_head_page(data[i]));
 
 		/* It is not the last user for the page frag case */
-		if (!page_pool_is_last_ref(page))
+		if (!page_pool_is_last_ref(netmem))
 			continue;
 
-		page = __page_pool_put_page(pool, page, -1, false);
+		netmem = __page_pool_put_page(pool, netmem, -1, false);
 		/* Approved for bulk recycling in ptr_ring cache */
-		if (page)
-			data[bulk_len++] = page;
+		if (netmem)
+			data[bulk_len++] = (__force void *)netmem;
 	}
 
 	if (unlikely(!bulk_len))
@@ -812,100 +831,108 @@  void page_pool_put_page_bulk(struct page_pool *pool, void **data,
 	 * since put_page() with refcnt == 1 can be an expensive operation
 	 */
 	for (; i < bulk_len; i++)
-		page_pool_return_page(pool, data[i]);
+		page_pool_return_page(pool, (__force netmem_ref)data[i]);
 }
 EXPORT_SYMBOL(page_pool_put_page_bulk);
 
-static struct page *page_pool_drain_frag(struct page_pool *pool,
-					 struct page *page)
+static netmem_ref page_pool_drain_frag(struct page_pool *pool,
+				       netmem_ref netmem)
 {
 	long drain_count = BIAS_MAX - pool->frag_users;
 
 	/* Some user is still using the page frag */
-	if (likely(page_pool_unref_page(page, drain_count)))
-		return NULL;
+	if (likely(page_pool_unref_netmem(netmem, drain_count)))
+		return 0;
 
-	if (__page_pool_page_can_be_recycled(page)) {
+	if (__page_pool_page_can_be_recycled(netmem)) {
 		if (pool->p.flags & PP_FLAG_DMA_SYNC_DEV)
-			page_pool_dma_sync_for_device(pool, page, -1);
+			page_pool_dma_sync_for_device(pool, netmem, -1);
 
-		return page;
+		return netmem;
 	}
 
-	page_pool_return_page(pool, page);
-	return NULL;
+	page_pool_return_page(pool, netmem);
+	return 0;
 }
 
 static void page_pool_free_frag(struct page_pool *pool)
 {
 	long drain_count = BIAS_MAX - pool->frag_users;
-	struct page *page = pool->frag_page;
+	netmem_ref netmem = pool->frag_page;
 
-	pool->frag_page = NULL;
+	pool->frag_page = 0;
 
-	if (!page || page_pool_unref_page(page, drain_count))
+	if (!netmem || page_pool_unref_netmem(netmem, drain_count))
 		return;
 
-	page_pool_return_page(pool, page);
+	page_pool_return_page(pool, netmem);
 }
 
-struct page *page_pool_alloc_frag(struct page_pool *pool,
-				  unsigned int *offset,
-				  unsigned int size, gfp_t gfp)
+netmem_ref page_pool_alloc_frag_netmem(struct page_pool *pool,
+				       unsigned int *offset, unsigned int size,
+				       gfp_t gfp)
 {
 	unsigned int max_size = PAGE_SIZE << pool->p.order;
-	struct page *page = pool->frag_page;
+	netmem_ref netmem = pool->frag_page;
 
 	if (WARN_ON(size > max_size))
-		return NULL;
+		return 0;
 
 	size = ALIGN(size, dma_get_cache_alignment());
 	*offset = pool->frag_offset;
 
-	if (page && *offset + size > max_size) {
-		page = page_pool_drain_frag(pool, page);
-		if (page) {
+	if (netmem && *offset + size > max_size) {
+		netmem = page_pool_drain_frag(pool, netmem);
+		if (netmem) {
 			alloc_stat_inc(pool, fast);
 			goto frag_reset;
 		}
 	}
 
-	if (!page) {
-		page = page_pool_alloc_pages(pool, gfp);
-		if (unlikely(!page)) {
-			pool->frag_page = NULL;
-			return NULL;
+	if (!netmem) {
+		netmem = page_pool_alloc_netmem(pool, gfp);
+		if (unlikely(!netmem)) {
+			pool->frag_page = 0;
+			return 0;
 		}
 
-		pool->frag_page = page;
+		pool->frag_page = netmem;
 
 frag_reset:
 		pool->frag_users = 1;
 		*offset = 0;
 		pool->frag_offset = size;
-		page_pool_fragment_page(page, BIAS_MAX);
-		return page;
+		page_pool_fragment_netmem(netmem, BIAS_MAX);
+		return netmem;
 	}
 
 	pool->frag_users++;
 	pool->frag_offset = *offset + size;
 	alloc_stat_inc(pool, fast);
-	return page;
+	return netmem;
+}
+EXPORT_SYMBOL(page_pool_alloc_frag_netmem);
+
+struct page *page_pool_alloc_frag(struct page_pool *pool, unsigned int *offset,
+				  unsigned int size, gfp_t gfp)
+{
+	return netmem_to_page(page_pool_alloc_frag_netmem(pool, offset, size,
+							  gfp));
 }
 EXPORT_SYMBOL(page_pool_alloc_frag);
 
 static void page_pool_empty_ring(struct page_pool *pool)
 {
-	struct page *page;
+	netmem_ref netmem;
 
 	/* Empty recycle ring */
-	while ((page = ptr_ring_consume_bh(&pool->ring))) {
+	while ((netmem = (__force netmem_ref)ptr_ring_consume_bh(&pool->ring))) {
 		/* Verify the refcnt invariant of cached pages */
-		if (!(page_ref_count(page) == 1))
+		if (!(page_ref_count(netmem_to_page(netmem)) == 1))
 			pr_crit("%s() page_pool refcnt %d violation\n",
-				__func__, page_ref_count(page));
+				__func__, netmem_ref_count(netmem));
 
-		page_pool_return_page(pool, page);
+		page_pool_return_page(pool, netmem);
 	}
 }
 
@@ -927,7 +954,7 @@  static void __page_pool_destroy(struct page_pool *pool)
 
 static void page_pool_empty_alloc_cache_once(struct page_pool *pool)
 {
-	struct page *page;
+	netmem_ref netmem;
 
 	if (pool->destroy_cnt)
 		return;
@@ -937,8 +964,8 @@  static void page_pool_empty_alloc_cache_once(struct page_pool *pool)
 	 * call concurrently.
 	 */
 	while (pool->alloc.count) {
-		page = pool->alloc.cache[--pool->alloc.count];
-		page_pool_return_page(pool, page);
+		netmem = pool->alloc.cache[--pool->alloc.count];
+		page_pool_return_page(pool, netmem);
 	}
 }
 
@@ -1044,15 +1071,15 @@  EXPORT_SYMBOL(page_pool_destroy);
 /* Caller must provide appropriate safe context, e.g. NAPI. */
 void page_pool_update_nid(struct page_pool *pool, int new_nid)
 {
-	struct page *page;
+	netmem_ref netmem;
 
 	trace_page_pool_update_nid(pool, new_nid);
 	pool->p.nid = new_nid;
 
 	/* Flush pool alloc cache, as refill will check NUMA node */
 	while (pool->alloc.count) {
-		page = pool->alloc.cache[--pool->alloc.count];
-		page_pool_return_page(pool, page);
+		netmem = pool->alloc.cache[--pool->alloc.count];
+		page_pool_return_page(pool, netmem);
 	}
 }
 EXPORT_SYMBOL(page_pool_update_nid);
diff --git a/net/core/skbuff.c b/net/core/skbuff.c
index 17617c29be2d..7193ee9737a0 100644
--- a/net/core/skbuff.c
+++ b/net/core/skbuff.c
@@ -1005,8 +1005,9 @@  int skb_cow_data_for_xdp(struct page_pool *pool, struct sk_buff **pskb,
 EXPORT_SYMBOL(skb_cow_data_for_xdp);
 
 #if IS_ENABLED(CONFIG_PAGE_POOL)
-bool napi_pp_put_page(struct page *page, bool napi_safe)
+bool napi_pp_put_page(netmem_ref netmem, bool napi_safe)
 {
+	struct page *page = netmem_to_page(netmem);
 	bool allow_direct = false;
 	struct page_pool *pp;
 
@@ -1043,7 +1044,7 @@  bool napi_pp_put_page(struct page *page, bool napi_safe)
 	 * The page will be returned to the pool here regardless of the
 	 * 'flipped' fragment being in use or not.
 	 */
-	page_pool_put_full_page(pp, page, allow_direct);
+	page_pool_put_full_netmem(pp, page_to_netmem(page), allow_direct);
 
 	return true;
 }
@@ -1054,7 +1055,7 @@  static bool skb_pp_recycle(struct sk_buff *skb, void *data, bool napi_safe)
 {
 	if (!IS_ENABLED(CONFIG_PAGE_POOL) || !skb->pp_recycle)
 		return false;
-	return napi_pp_put_page(virt_to_page(data), napi_safe);
+	return napi_pp_put_page(page_to_netmem(virt_to_page(data)), napi_safe);
 }
 
 /**