| Message ID | 20240305020153.2787423-8-almasrymina@google.com (mailing list archive) |
|---|---|
| State | New, archived |
| Headers | show |
| Series | Device Memory TCP | expand |
Hi Mina, I recommend you cc linux-mm and Matthew Wilcox on these two patches also. David
On Mon, Mar 4, 2024 at 6:02 PM Mina Almasry <almasrymina@google.com> wrote: > > Abstrace the memory type from the page_pool so we can later add support > for new memory types. Convert the page_pool to use the new netmem type > abstraction, rather than use struct page directly. > > As of this patch the netmem type is a no-op abstraction: it's always a > struct page underneath. All the page pool internals are converted to > use struct netmem instead of struct page, and the page pool now exports > 2 APIs: > > 1. The existing struct page API. > 2. The new struct netmem API. > > Keeping the existing API is transitional; we do not want to refactor all > the current drivers using the page pool at once. > > The netmem abstraction is currently a no-op. The page_pool uses > page_to_netmem() to convert allocated pages to netmem, and uses > netmem_to_page() to convert the netmem back to pages to pass to mm APIs, > > Follow up patches to this series add non-paged netmem support to the > page_pool. This change is factored out on its own to limit the code > churn to this 1 patch, for ease of code review. > > Signed-off-by: Mina Almasry <almasrymina@google.com> > Per David Howell's request, I'm forwarding these 2 patches from the series to linux-mm & Mathew Wilcox. https://lore.kernel.org/netdev/950858.1709622997@warthog.procyon.org.uk/ +linux-mm +Mathew Wilcox +David Howells > --- > > v6: > > - Rebased on top of the merged netmem_ref type. > > --- > include/linux/skbuff.h | 4 +- > include/net/netmem.h | 15 ++ > include/net/page_pool/helpers.h | 122 +++++++++---- > include/net/page_pool/types.h | 17 +- > include/trace/events/page_pool.h | 29 +-- > net/bpf/test_run.c | 5 +- > net/core/page_pool.c | 303 +++++++++++++++++-------------- > net/core/skbuff.c | 7 +- > 8 files changed, 302 insertions(+), 200 deletions(-) > > diff --git a/include/linux/skbuff.h b/include/linux/skbuff.h > index d577e0bee18d..ca29d1fd4561 100644 > --- a/include/linux/skbuff.h > +++ b/include/linux/skbuff.h > @@ -3504,7 +3504,7 @@ int skb_pp_cow_data(struct page_pool *pool, struct sk_buff **pskb, > unsigned int headroom); > int skb_cow_data_for_xdp(struct page_pool *pool, struct sk_buff **pskb, > struct bpf_prog *prog); > -bool napi_pp_put_page(struct page *page, bool napi_safe); > +bool napi_pp_put_page(netmem_ref netmem, bool napi_safe); > > static inline void > napi_frag_unref(skb_frag_t *frag, bool recycle, bool napi_safe) > @@ -3512,7 +3512,7 @@ napi_frag_unref(skb_frag_t *frag, bool recycle, bool napi_safe) > struct page *page = skb_frag_page(frag); > > #ifdef CONFIG_PAGE_POOL > - if (recycle && napi_pp_put_page(page, napi_safe)) > + if (recycle && napi_pp_put_page(page_to_netmem(page), napi_safe)) > return; > #endif > put_page(page); > diff --git a/include/net/netmem.h b/include/net/netmem.h > index ca17ea1d33f8..21f53b29e5fe 100644 > --- a/include/net/netmem.h > +++ b/include/net/netmem.h > @@ -88,4 +88,19 @@ static inline netmem_ref page_to_netmem(struct page *page) > return (__force netmem_ref)page; > } > > +static inline int netmem_ref_count(netmem_ref netmem) > +{ > + return page_ref_count(netmem_to_page(netmem)); > +} > + > +static inline unsigned long netmem_to_pfn(netmem_ref netmem) > +{ > + return page_to_pfn(netmem_to_page(netmem)); > +} > + > +static inline netmem_ref netmem_compound_head(netmem_ref netmem) > +{ > + return page_to_netmem(compound_head(netmem_to_page(netmem))); > +} > + > #endif /* _NET_NETMEM_H */ > diff --git a/include/net/page_pool/helpers.h b/include/net/page_pool/helpers.h > index 1d397c1a0043..61814f91a458 100644 > --- a/include/net/page_pool/helpers.h > +++ b/include/net/page_pool/helpers.h > @@ -53,6 +53,8 @@ > #define _NET_PAGE_POOL_HELPERS_H > > #include <net/page_pool/types.h> > +#include <net/net_debug.h> > +#include <net/netmem.h> > > #ifdef CONFIG_PAGE_POOL_STATS > /* Deprecated driver-facing API, use netlink instead */ > @@ -101,7 +103,7 @@ static inline struct page *page_pool_dev_alloc_pages(struct page_pool *pool) > * Get a page fragment from the page allocator or page_pool caches. > * > * Return: > - * Return allocated page fragment, otherwise return NULL. > + * Return allocated page fragment, otherwise return 0. > */ > static inline struct page *page_pool_dev_alloc_frag(struct page_pool *pool, > unsigned int *offset, > @@ -112,22 +114,22 @@ static inline struct page *page_pool_dev_alloc_frag(struct page_pool *pool, > return page_pool_alloc_frag(pool, offset, size, gfp); > } > > -static inline struct page *page_pool_alloc(struct page_pool *pool, > - unsigned int *offset, > - unsigned int *size, gfp_t gfp) > +static inline netmem_ref page_pool_alloc(struct page_pool *pool, > + unsigned int *offset, > + unsigned int *size, gfp_t gfp) > { > unsigned int max_size = PAGE_SIZE << pool->p.order; > - struct page *page; > + netmem_ref netmem; > > if ((*size << 1) > max_size) { > *size = max_size; > *offset = 0; > - return page_pool_alloc_pages(pool, gfp); > + return page_pool_alloc_netmem(pool, gfp); > } > > - page = page_pool_alloc_frag(pool, offset, *size, gfp); > - if (unlikely(!page)) > - return NULL; > + netmem = page_pool_alloc_frag_netmem(pool, offset, *size, gfp); > + if (unlikely(!netmem)) > + return 0; > > /* There is very likely not enough space for another fragment, so append > * the remaining size to the current fragment to avoid truesize > @@ -138,7 +140,7 @@ static inline struct page *page_pool_alloc(struct page_pool *pool, > pool->frag_offset = max_size; > } > > - return page; > + return netmem; > } > > /** > @@ -152,7 +154,7 @@ static inline struct page *page_pool_alloc(struct page_pool *pool, > * utilization and performance penalty. > * > * Return: > - * Return allocated page or page fragment, otherwise return NULL. > + * Return allocated page or page fragment, otherwise return 0. > */ > static inline struct page *page_pool_dev_alloc(struct page_pool *pool, > unsigned int *offset, > @@ -160,7 +162,7 @@ static inline struct page *page_pool_dev_alloc(struct page_pool *pool, > { > gfp_t gfp = (GFP_ATOMIC | __GFP_NOWARN); > > - return page_pool_alloc(pool, offset, size, gfp); > + return netmem_to_page(page_pool_alloc(pool, offset, size, gfp)); > } > > static inline void *page_pool_alloc_va(struct page_pool *pool, > @@ -170,9 +172,10 @@ static inline void *page_pool_alloc_va(struct page_pool *pool, > struct page *page; > > /* Mask off __GFP_HIGHMEM to ensure we can use page_address() */ > - page = page_pool_alloc(pool, &offset, size, gfp & ~__GFP_HIGHMEM); > + page = netmem_to_page( > + page_pool_alloc(pool, &offset, size, gfp & ~__GFP_HIGHMEM)); > if (unlikely(!page)) > - return NULL; > + return 0; > > return page_address(page) + offset; > } > @@ -187,7 +190,7 @@ static inline void *page_pool_alloc_va(struct page_pool *pool, > * it returns va of the allocated page or page fragment. > * > * Return: > - * Return the va for the allocated page or page fragment, otherwise return NULL. > + * Return the va for the allocated page or page fragment, otherwise return 0. > */ > static inline void *page_pool_dev_alloc_va(struct page_pool *pool, > unsigned int *size) > @@ -210,6 +213,11 @@ inline enum dma_data_direction page_pool_get_dma_dir(struct page_pool *pool) > return pool->p.dma_dir; > } > > +static inline void page_pool_fragment_netmem(netmem_ref netmem, long nr) > +{ > + atomic_long_set(&netmem_to_page(netmem)->pp_ref_count, nr); > +} > + > /** > * page_pool_fragment_page() - split a fresh page into fragments > * @page: page to split > @@ -230,11 +238,12 @@ inline enum dma_data_direction page_pool_get_dma_dir(struct page_pool *pool) > */ > static inline void page_pool_fragment_page(struct page *page, long nr) > { > - atomic_long_set(&page->pp_ref_count, nr); > + page_pool_fragment_netmem(page_to_netmem(page), nr); > } > > -static inline long page_pool_unref_page(struct page *page, long nr) > +static inline long page_pool_unref_netmem(netmem_ref netmem, long nr) > { > + struct page *page = netmem_to_page(netmem); > long ret; > > /* If nr == pp_ref_count then we have cleared all remaining > @@ -277,15 +286,41 @@ static inline long page_pool_unref_page(struct page *page, long nr) > return ret; > } > > +static inline long page_pool_unref_page(struct page *page, long nr) > +{ > + return page_pool_unref_netmem(page_to_netmem(page), nr); > +} > + > +static inline void page_pool_ref_netmem(netmem_ref netmem) > +{ > + atomic_long_inc(&netmem_to_page(netmem)->pp_ref_count); > +} > + > static inline void page_pool_ref_page(struct page *page) > { > - atomic_long_inc(&page->pp_ref_count); > + page_pool_ref_netmem(page_to_netmem(page)); > } > > -static inline bool page_pool_is_last_ref(struct page *page) > +static inline bool page_pool_is_last_ref(netmem_ref netmem) > { > /* If page_pool_unref_page() returns 0, we were the last user */ > - return page_pool_unref_page(page, 1) == 0; > + return page_pool_unref_netmem(netmem, 1) == 0; > +} > + > +static inline void page_pool_put_netmem(struct page_pool *pool, > + netmem_ref netmem, > + unsigned int dma_sync_size, > + bool allow_direct) > +{ > + /* When page_pool isn't compiled-in, net/core/xdp.c doesn't > + * allow registering MEM_TYPE_PAGE_POOL, but shield linker. > + */ > +#ifdef CONFIG_PAGE_POOL > + if (!page_pool_is_last_ref(netmem)) > + return; > + > + page_pool_put_unrefed_netmem(pool, netmem, dma_sync_size, allow_direct); > +#endif > } > > /** > @@ -306,15 +341,15 @@ static inline void page_pool_put_page(struct page_pool *pool, > unsigned int dma_sync_size, > bool allow_direct) > { > - /* When page_pool isn't compiled-in, net/core/xdp.c doesn't > - * allow registering MEM_TYPE_PAGE_POOL, but shield linker. > - */ > -#ifdef CONFIG_PAGE_POOL > - if (!page_pool_is_last_ref(page)) > - return; > + page_pool_put_netmem(pool, page_to_netmem(page), dma_sync_size, > + allow_direct); > +} > > - page_pool_put_unrefed_page(pool, page, dma_sync_size, allow_direct); > -#endif > +static inline void page_pool_put_full_netmem(struct page_pool *pool, > + netmem_ref netmem, > + bool allow_direct) > +{ > + page_pool_put_netmem(pool, netmem, -1, allow_direct); > } > > /** > @@ -329,7 +364,7 @@ static inline void page_pool_put_page(struct page_pool *pool, > static inline void page_pool_put_full_page(struct page_pool *pool, > struct page *page, bool allow_direct) > { > - page_pool_put_page(pool, page, -1, allow_direct); > + page_pool_put_netmem(pool, page_to_netmem(page), -1, allow_direct); > } > > /** > @@ -363,6 +398,18 @@ static inline void page_pool_free_va(struct page_pool *pool, void *va, > page_pool_put_page(pool, virt_to_head_page(va), -1, allow_direct); > } > > +static inline dma_addr_t page_pool_get_dma_addr_netmem(netmem_ref netmem) > +{ > + struct page *page = netmem_to_page(netmem); > + > + dma_addr_t ret = page->dma_addr; > + > + if (PAGE_POOL_32BIT_ARCH_WITH_64BIT_DMA) > + ret <<= PAGE_SHIFT; > + > + return ret; > +} > + > /** > * page_pool_get_dma_addr() - Retrieve the stored DMA address. > * @page: page allocated from a page pool > @@ -372,16 +419,14 @@ static inline void page_pool_free_va(struct page_pool *pool, void *va, > */ > static inline dma_addr_t page_pool_get_dma_addr(struct page *page) > { > - dma_addr_t ret = page->dma_addr; > - > - if (PAGE_POOL_32BIT_ARCH_WITH_64BIT_DMA) > - ret <<= PAGE_SHIFT; > - > - return ret; > + return page_pool_get_dma_addr_netmem(page_to_netmem(page)); > } > > -static inline bool page_pool_set_dma_addr(struct page *page, dma_addr_t addr) > +static inline bool page_pool_set_dma_addr_netmem(netmem_ref netmem, > + dma_addr_t addr) > { > + struct page *page = netmem_to_page(netmem); > + > if (PAGE_POOL_32BIT_ARCH_WITH_64BIT_DMA) { > page->dma_addr = addr >> PAGE_SHIFT; > > @@ -395,6 +440,11 @@ static inline bool page_pool_set_dma_addr(struct page *page, dma_addr_t addr) > return false; > } > > +static inline bool page_pool_set_dma_addr(struct page *page, dma_addr_t addr) > +{ > + return page_pool_set_dma_addr_netmem(page_to_netmem(page), addr); > +} > + > static inline bool page_pool_put(struct page_pool *pool) > { > return refcount_dec_and_test(&pool->user_cnt); > diff --git a/include/net/page_pool/types.h b/include/net/page_pool/types.h > index ffe5f31fb0da..68a24c5ae827 100644 > --- a/include/net/page_pool/types.h > +++ b/include/net/page_pool/types.h > @@ -40,7 +40,7 @@ > #define PP_ALLOC_CACHE_REFILL 64 > struct pp_alloc_cache { > u32 count; > - struct page *cache[PP_ALLOC_CACHE_SIZE]; > + netmem_ref cache[PP_ALLOC_CACHE_SIZE]; > }; > > /** > @@ -73,7 +73,7 @@ struct page_pool_params { > struct_group_tagged(page_pool_params_slow, slow, > struct net_device *netdev; > /* private: used by test code only */ > - void (*init_callback)(struct page *page, void *arg); > + void (*init_callback)(netmem_ref netmem, void *arg); > void *init_arg; > ); > }; > @@ -131,8 +131,8 @@ struct page_pool_stats { > struct memory_provider_ops { > int (*init)(struct page_pool *pool); > void (*destroy)(struct page_pool *pool); > - struct page *(*alloc_pages)(struct page_pool *pool, gfp_t gfp); > - bool (*release_page)(struct page_pool *pool, struct page *page); > + netmem_ref (*alloc_pages)(struct page_pool *pool, gfp_t gfp); > + bool (*release_page)(struct page_pool *pool, netmem_ref netmem); > }; > > struct page_pool { > @@ -142,7 +142,7 @@ struct page_pool { > bool has_init_callback; > > long frag_users; > - struct page *frag_page; > + netmem_ref frag_page; > unsigned int frag_offset; > u32 pages_state_hold_cnt; > > @@ -214,8 +214,12 @@ struct page_pool { > }; > > struct page *page_pool_alloc_pages(struct page_pool *pool, gfp_t gfp); > +netmem_ref page_pool_alloc_netmem(struct page_pool *pool, gfp_t gfp); > struct page *page_pool_alloc_frag(struct page_pool *pool, unsigned int *offset, > unsigned int size, gfp_t gfp); > +netmem_ref page_pool_alloc_frag_netmem(struct page_pool *pool, > + unsigned int *offset, unsigned int size, > + gfp_t gfp); > struct page_pool *page_pool_create(const struct page_pool_params *params); > struct page_pool *page_pool_create_percpu(const struct page_pool_params *params, > int cpuid); > @@ -245,6 +249,9 @@ static inline void page_pool_put_page_bulk(struct page_pool *pool, void **data, > } > #endif > > +void page_pool_put_unrefed_netmem(struct page_pool *pool, netmem_ref netmem, > + unsigned int dma_sync_size, > + bool allow_direct); > void page_pool_put_unrefed_page(struct page_pool *pool, struct page *page, > unsigned int dma_sync_size, > bool allow_direct); > diff --git a/include/trace/events/page_pool.h b/include/trace/events/page_pool.h > index 6834356b2d2a..c5b6383ff276 100644 > --- a/include/trace/events/page_pool.h > +++ b/include/trace/events/page_pool.h > @@ -42,51 +42,52 @@ TRACE_EVENT(page_pool_release, > TRACE_EVENT(page_pool_state_release, > > TP_PROTO(const struct page_pool *pool, > - const struct page *page, u32 release), > + netmem_ref netmem, u32 release), > > - TP_ARGS(pool, page, release), > + TP_ARGS(pool, netmem, release), > > TP_STRUCT__entry( > __field(const struct page_pool *, pool) > - __field(const struct page *, page) > + __field(netmem_ref, netmem) > __field(u32, release) > __field(unsigned long, pfn) > ), > > TP_fast_assign( > __entry->pool = pool; > - __entry->page = page; > + __entry->netmem = netmem; > __entry->release = release; > - __entry->pfn = page_to_pfn(page); > + __entry->pfn = netmem_to_pfn(netmem); > ), > > - TP_printk("page_pool=%p page=%p pfn=0x%lx release=%u", > - __entry->pool, __entry->page, __entry->pfn, __entry->release) > + TP_printk("page_pool=%p netmem=%lu pfn=0x%lx release=%u", > + __entry->pool, (__force unsigned long)__entry->netmem, > + __entry->pfn, __entry->release) > ); > > TRACE_EVENT(page_pool_state_hold, > > TP_PROTO(const struct page_pool *pool, > - const struct page *page, u32 hold), > + netmem_ref netmem, u32 hold), > > - TP_ARGS(pool, page, hold), > + TP_ARGS(pool, netmem, hold), > > TP_STRUCT__entry( > __field(const struct page_pool *, pool) > - __field(const struct page *, page) > + __field(netmem_ref, netmem) > __field(u32, hold) > __field(unsigned long, pfn) > ), > > TP_fast_assign( > __entry->pool = pool; > - __entry->page = page; > + __entry->netmem = netmem; > __entry->hold = hold; > - __entry->pfn = page_to_pfn(page); > + __entry->pfn = netmem_to_pfn(netmem); > ), > > - TP_printk("page_pool=%p page=%p pfn=0x%lx hold=%u", > - __entry->pool, __entry->page, __entry->pfn, __entry->hold) > + TP_printk("page_pool=%p netmem=%lu pfn=0x%lx hold=%u", > + __entry->pool, __entry->netmem, __entry->pfn, __entry->hold) > ); > > TRACE_EVENT(page_pool_update_nid, > diff --git a/net/bpf/test_run.c b/net/bpf/test_run.c > index 5535f9adc658..bc8f7ab88f86 100644 > --- a/net/bpf/test_run.c > +++ b/net/bpf/test_run.c > @@ -126,9 +126,10 @@ struct xdp_test_data { > #define TEST_XDP_FRAME_SIZE (PAGE_SIZE - sizeof(struct xdp_page_head)) > #define TEST_XDP_MAX_BATCH 256 > > -static void xdp_test_run_init_page(struct page *page, void *arg) > +static void xdp_test_run_init_page(netmem_ref netmem, void *arg) > { > - struct xdp_page_head *head = phys_to_virt(page_to_phys(page)); > + struct xdp_page_head *head = > + phys_to_virt(page_to_phys(netmem_to_page(netmem))); > struct xdp_buff *new_ctx, *orig_ctx; > u32 headroom = XDP_PACKET_HEADROOM; > struct xdp_test_data *xdp = arg; > diff --git a/net/core/page_pool.c b/net/core/page_pool.c > index fe9de4ecce94..24d5236b2efc 100644 > --- a/net/core/page_pool.c > +++ b/net/core/page_pool.c > @@ -329,19 +329,18 @@ struct page_pool *page_pool_create(const struct page_pool_params *params) > } > EXPORT_SYMBOL(page_pool_create); > > -static void page_pool_return_page(struct page_pool *pool, struct page *page); > +static void page_pool_return_page(struct page_pool *pool, netmem_ref netmem); > > -noinline > -static struct page *page_pool_refill_alloc_cache(struct page_pool *pool) > +static noinline netmem_ref page_pool_refill_alloc_cache(struct page_pool *pool) > { > struct ptr_ring *r = &pool->ring; > - struct page *page; > + netmem_ref netmem; > int pref_nid; /* preferred NUMA node */ > > /* Quicker fallback, avoid locks when ring is empty */ > if (__ptr_ring_empty(r)) { > alloc_stat_inc(pool, empty); > - return NULL; > + return 0; > } > > /* Softirq guarantee CPU and thus NUMA node is stable. This, > @@ -356,56 +355,56 @@ static struct page *page_pool_refill_alloc_cache(struct page_pool *pool) > > /* Refill alloc array, but only if NUMA match */ > do { > - page = __ptr_ring_consume(r); > - if (unlikely(!page)) > + netmem = (__force netmem_ref)__ptr_ring_consume(r); > + if (unlikely(!netmem)) > break; > > - if (likely(page_to_nid(page) == pref_nid)) { > - pool->alloc.cache[pool->alloc.count++] = page; > + if (likely(page_to_nid(netmem_to_page(netmem)) == pref_nid)) { > + pool->alloc.cache[pool->alloc.count++] = netmem; > } else { > /* NUMA mismatch; > * (1) release 1 page to page-allocator and > * (2) break out to fallthrough to alloc_pages_node. > * This limit stress on page buddy alloactor. > */ > - page_pool_return_page(pool, page); > + page_pool_return_page(pool, netmem); > alloc_stat_inc(pool, waive); > - page = NULL; > + netmem = 0; > break; > } > } while (pool->alloc.count < PP_ALLOC_CACHE_REFILL); > > /* Return last page */ > if (likely(pool->alloc.count > 0)) { > - page = pool->alloc.cache[--pool->alloc.count]; > + netmem = pool->alloc.cache[--pool->alloc.count]; > alloc_stat_inc(pool, refill); > } > > - return page; > + return netmem; > } > > /* fast path */ > -static struct page *__page_pool_get_cached(struct page_pool *pool) > +static netmem_ref __page_pool_get_cached(struct page_pool *pool) > { > - struct page *page; > + netmem_ref netmem; > > /* Caller MUST guarantee safe non-concurrent access, e.g. softirq */ > if (likely(pool->alloc.count)) { > /* Fast-path */ > - page = pool->alloc.cache[--pool->alloc.count]; > + netmem = pool->alloc.cache[--pool->alloc.count]; > alloc_stat_inc(pool, fast); > } else { > - page = page_pool_refill_alloc_cache(pool); > + netmem = page_pool_refill_alloc_cache(pool); > } > > - return page; > + return netmem; > } > > static void page_pool_dma_sync_for_device(struct page_pool *pool, > - struct page *page, > + netmem_ref netmem, > unsigned int dma_sync_size) > { > - dma_addr_t dma_addr = page_pool_get_dma_addr(page); > + dma_addr_t dma_addr = page_pool_get_dma_addr_netmem(netmem); > > dma_sync_size = min(dma_sync_size, pool->p.max_len); > dma_sync_single_range_for_device(pool->p.dev, dma_addr, > @@ -413,7 +412,7 @@ static void page_pool_dma_sync_for_device(struct page_pool *pool, > pool->p.dma_dir); > } > > -static bool page_pool_dma_map(struct page_pool *pool, struct page *page) > +static bool page_pool_dma_map(struct page_pool *pool, netmem_ref netmem) > { > dma_addr_t dma; > > @@ -422,18 +421,18 @@ static bool page_pool_dma_map(struct page_pool *pool, struct page *page) > * into page private data (i.e 32bit cpu with 64bit DMA caps) > * This mapping is kept for lifetime of page, until leaving pool. > */ > - dma = dma_map_page_attrs(pool->p.dev, page, 0, > - (PAGE_SIZE << pool->p.order), > - pool->p.dma_dir, DMA_ATTR_SKIP_CPU_SYNC | > - DMA_ATTR_WEAK_ORDERING); > + dma = dma_map_page_attrs(pool->p.dev, netmem_to_page(netmem), 0, > + (PAGE_SIZE << pool->p.order), pool->p.dma_dir, > + DMA_ATTR_SKIP_CPU_SYNC | > + DMA_ATTR_WEAK_ORDERING); > if (dma_mapping_error(pool->p.dev, dma)) > return false; > > - if (page_pool_set_dma_addr(page, dma)) > + if (page_pool_set_dma_addr_netmem(netmem, dma)) > goto unmap_failed; > > if (pool->p.flags & PP_FLAG_DMA_SYNC_DEV) > - page_pool_dma_sync_for_device(pool, page, pool->p.max_len); > + page_pool_dma_sync_for_device(pool, netmem, pool->p.max_len); > > return true; > > @@ -445,9 +444,10 @@ static bool page_pool_dma_map(struct page_pool *pool, struct page *page) > return false; > } > > -static void page_pool_set_pp_info(struct page_pool *pool, > - struct page *page) > +static void page_pool_set_pp_info(struct page_pool *pool, netmem_ref netmem) > { > + struct page *page = netmem_to_page(netmem); > + > page->pp = pool; > page->pp_magic |= PP_SIGNATURE; > > @@ -457,13 +457,15 @@ static void page_pool_set_pp_info(struct page_pool *pool, > * is dirtying the same cache line as the page->pp_magic above, so > * the overhead is negligible. > */ > - page_pool_fragment_page(page, 1); > + page_pool_fragment_netmem(netmem, 1); > if (pool->has_init_callback) > - pool->slow.init_callback(page, pool->slow.init_arg); > + pool->slow.init_callback(netmem, pool->slow.init_arg); > } > > -static void page_pool_clear_pp_info(struct page *page) > +static void page_pool_clear_pp_info(netmem_ref netmem) > { > + struct page *page = netmem_to_page(netmem); > + > page->pp_magic = 0; > page->pp = NULL; > } > @@ -479,34 +481,34 @@ static struct page *__page_pool_alloc_page_order(struct page_pool *pool, > return NULL; > > if ((pool->p.flags & PP_FLAG_DMA_MAP) && > - unlikely(!page_pool_dma_map(pool, page))) { > + unlikely(!page_pool_dma_map(pool, page_to_netmem(page)))) { > put_page(page); > return NULL; > } > > alloc_stat_inc(pool, slow_high_order); > - page_pool_set_pp_info(pool, page); > + page_pool_set_pp_info(pool, page_to_netmem(page)); > > /* Track how many pages are held 'in-flight' */ > pool->pages_state_hold_cnt++; > - trace_page_pool_state_hold(pool, page, pool->pages_state_hold_cnt); > + trace_page_pool_state_hold(pool, page_to_netmem(page), > + pool->pages_state_hold_cnt); > return page; > } > > /* slow path */ > -noinline > -static struct page *__page_pool_alloc_pages_slow(struct page_pool *pool, > - gfp_t gfp) > +static noinline netmem_ref __page_pool_alloc_pages_slow(struct page_pool *pool, > + gfp_t gfp) > { > const int bulk = PP_ALLOC_CACHE_REFILL; > unsigned int pp_flags = pool->p.flags; > unsigned int pp_order = pool->p.order; > - struct page *page; > + netmem_ref netmem; > int i, nr_pages; > > /* Don't support bulk alloc for high-order pages */ > if (unlikely(pp_order)) > - return __page_pool_alloc_page_order(pool, gfp); > + return page_to_netmem(__page_pool_alloc_page_order(pool, gfp)); > > /* Unnecessary as alloc cache is empty, but guarantees zero count */ > if (unlikely(pool->alloc.count > 0)) > @@ -515,60 +517,67 @@ static struct page *__page_pool_alloc_pages_slow(struct page_pool *pool, > /* Mark empty alloc.cache slots "empty" for alloc_pages_bulk_array */ > memset(&pool->alloc.cache, 0, sizeof(void *) * bulk); > > - nr_pages = alloc_pages_bulk_array_node(gfp, pool->p.nid, bulk, > - pool->alloc.cache); > + nr_pages = alloc_pages_bulk_array_node(gfp, > + pool->p.nid, bulk, > + (struct page **)pool->alloc.cache); > if (unlikely(!nr_pages)) > - return NULL; > + return 0; > > /* Pages have been filled into alloc.cache array, but count is zero and > * page element have not been (possibly) DMA mapped. > */ > for (i = 0; i < nr_pages; i++) { > - page = pool->alloc.cache[i]; > + netmem = pool->alloc.cache[i]; > if ((pp_flags & PP_FLAG_DMA_MAP) && > - unlikely(!page_pool_dma_map(pool, page))) { > - put_page(page); > + unlikely(!page_pool_dma_map(pool, netmem))) { > + put_page(netmem_to_page(netmem)); > continue; > } > > - page_pool_set_pp_info(pool, page); > - pool->alloc.cache[pool->alloc.count++] = page; > + page_pool_set_pp_info(pool, netmem); > + pool->alloc.cache[pool->alloc.count++] = netmem; > /* Track how many pages are held 'in-flight' */ > pool->pages_state_hold_cnt++; > - trace_page_pool_state_hold(pool, page, > + trace_page_pool_state_hold(pool, netmem, > pool->pages_state_hold_cnt); > } > > /* Return last page */ > if (likely(pool->alloc.count > 0)) { > - page = pool->alloc.cache[--pool->alloc.count]; > + netmem = pool->alloc.cache[--pool->alloc.count]; > alloc_stat_inc(pool, slow); > } else { > - page = NULL; > + netmem = 0; > } > > /* When page just alloc'ed is should/must have refcnt 1. */ > - return page; > + return netmem; > } > > /* For using page_pool replace: alloc_pages() API calls, but provide > * synchronization guarantee for allocation side. > */ > -struct page *page_pool_alloc_pages(struct page_pool *pool, gfp_t gfp) > +netmem_ref page_pool_alloc_netmem(struct page_pool *pool, gfp_t gfp) > { > - struct page *page; > + netmem_ref netmem; > > /* Fast-path: Get a page from cache */ > - page = __page_pool_get_cached(pool); > - if (page) > - return page; > + netmem = __page_pool_get_cached(pool); > + if (netmem) > + return netmem; > > /* Slow-path: cache empty, do real allocation */ > if (static_branch_unlikely(&page_pool_mem_providers) && pool->mp_ops) > - page = pool->mp_ops->alloc_pages(pool, gfp); > + netmem = pool->mp_ops->alloc_pages(pool, gfp); > else > - page = __page_pool_alloc_pages_slow(pool, gfp); > - return page; > + netmem = __page_pool_alloc_pages_slow(pool, gfp); > + return netmem; > +} > +EXPORT_SYMBOL(page_pool_alloc_netmem); > + > +struct page *page_pool_alloc_pages(struct page_pool *pool, gfp_t gfp) > +{ > + return netmem_to_page(page_pool_alloc_netmem(pool, gfp)); > } > EXPORT_SYMBOL(page_pool_alloc_pages); > > @@ -596,8 +605,8 @@ s32 page_pool_inflight(const struct page_pool *pool, bool strict) > return inflight; > } > > -static __always_inline > -void __page_pool_release_page_dma(struct page_pool *pool, struct page *page) > +static __always_inline void __page_pool_release_page_dma(struct page_pool *pool, > + netmem_ref netmem) > { > dma_addr_t dma; > > @@ -607,13 +616,13 @@ void __page_pool_release_page_dma(struct page_pool *pool, struct page *page) > */ > return; > > - dma = page_pool_get_dma_addr(page); > + dma = page_pool_get_dma_addr_netmem(netmem); > > /* When page is unmapped, it cannot be returned to our pool */ > dma_unmap_page_attrs(pool->p.dev, dma, > PAGE_SIZE << pool->p.order, pool->p.dma_dir, > DMA_ATTR_SKIP_CPU_SYNC | DMA_ATTR_WEAK_ORDERING); > - page_pool_set_dma_addr(page, 0); > + page_pool_set_dma_addr_netmem(netmem, 0); > } > > /* Disconnects a page (from a page_pool). API users can have a need > @@ -621,26 +630,26 @@ void __page_pool_release_page_dma(struct page_pool *pool, struct page *page) > * a regular page (that will eventually be returned to the normal > * page-allocator via put_page). > */ > -void page_pool_return_page(struct page_pool *pool, struct page *page) > +void page_pool_return_page(struct page_pool *pool, netmem_ref netmem) > { > int count; > bool put; > > put = true; > if (static_branch_unlikely(&page_pool_mem_providers) && pool->mp_ops) > - put = pool->mp_ops->release_page(pool, page); > + put = pool->mp_ops->release_page(pool, netmem); > else > - __page_pool_release_page_dma(pool, page); > + __page_pool_release_page_dma(pool, netmem); > > /* This may be the last page returned, releasing the pool, so > * it is not safe to reference pool afterwards. > */ > count = atomic_inc_return_relaxed(&pool->pages_state_release_cnt); > - trace_page_pool_state_release(pool, page, count); > + trace_page_pool_state_release(pool, netmem, count); > > if (put) { > - page_pool_clear_pp_info(page); > - put_page(page); > + page_pool_clear_pp_info(netmem); > + put_page(netmem_to_page(netmem)); > } > /* An optimization would be to call __free_pages(page, pool->p.order) > * knowing page is not part of page-cache (thus avoiding a > @@ -648,14 +657,14 @@ void page_pool_return_page(struct page_pool *pool, struct page *page) > */ > } > > -static bool page_pool_recycle_in_ring(struct page_pool *pool, struct page *page) > +static bool page_pool_recycle_in_ring(struct page_pool *pool, netmem_ref netmem) > { > int ret; > /* BH protection not needed if current is softirq */ > if (in_softirq()) > - ret = ptr_ring_produce(&pool->ring, page); > + ret = ptr_ring_produce(&pool->ring, (__force void *)netmem); > else > - ret = ptr_ring_produce_bh(&pool->ring, page); > + ret = ptr_ring_produce_bh(&pool->ring, (__force void *)netmem); > > if (!ret) { > recycle_stat_inc(pool, ring); > @@ -670,7 +679,7 @@ static bool page_pool_recycle_in_ring(struct page_pool *pool, struct page *page) > * > * Caller must provide appropriate safe context. > */ > -static bool page_pool_recycle_in_cache(struct page *page, > +static bool page_pool_recycle_in_cache(netmem_ref netmem, > struct page_pool *pool) > { > if (unlikely(pool->alloc.count == PP_ALLOC_CACHE_SIZE)) { > @@ -679,14 +688,15 @@ static bool page_pool_recycle_in_cache(struct page *page, > } > > /* Caller MUST have verified/know (page_ref_count(page) == 1) */ > - pool->alloc.cache[pool->alloc.count++] = page; > + pool->alloc.cache[pool->alloc.count++] = netmem; > recycle_stat_inc(pool, cached); > return true; > } > > -static bool __page_pool_page_can_be_recycled(const struct page *page) > +static bool __page_pool_page_can_be_recycled(netmem_ref netmem) > { > - return page_ref_count(page) == 1 && !page_is_pfmemalloc(page); > + return page_ref_count(netmem_to_page(netmem)) == 1 && > + !page_is_pfmemalloc(netmem_to_page(netmem)); > } > > /* If the page refcnt == 1, this will try to recycle the page. > @@ -695,8 +705,8 @@ static bool __page_pool_page_can_be_recycled(const struct page *page) > * If the page refcnt != 1, then the page will be returned to memory > * subsystem. > */ > -static __always_inline struct page * > -__page_pool_put_page(struct page_pool *pool, struct page *page, > +static __always_inline netmem_ref > +__page_pool_put_page(struct page_pool *pool, netmem_ref netmem, > unsigned int dma_sync_size, bool allow_direct) > { > lockdep_assert_no_hardirq(); > @@ -710,19 +720,19 @@ __page_pool_put_page(struct page_pool *pool, struct page *page, > * page is NOT reusable when allocated when system is under > * some pressure. (page_is_pfmemalloc) > */ > - if (likely(__page_pool_page_can_be_recycled(page))) { > + if (likely(__page_pool_page_can_be_recycled(netmem))) { > /* Read barrier done in page_ref_count / READ_ONCE */ > > if (pool->p.flags & PP_FLAG_DMA_SYNC_DEV) > - page_pool_dma_sync_for_device(pool, page, > + page_pool_dma_sync_for_device(pool, netmem, > dma_sync_size); > > if (allow_direct && in_softirq() && > - page_pool_recycle_in_cache(page, pool)) > - return NULL; > + page_pool_recycle_in_cache(netmem, pool)) > + return 0; > > /* Page found as candidate for recycling */ > - return page; > + return netmem; > } > /* Fallback/non-XDP mode: API user have elevated refcnt. > * > @@ -738,21 +748,30 @@ __page_pool_put_page(struct page_pool *pool, struct page *page, > * will be invoking put_page. > */ > recycle_stat_inc(pool, released_refcnt); > - page_pool_return_page(pool, page); > + page_pool_return_page(pool, netmem); > > - return NULL; > + return 0; > } > > -void page_pool_put_unrefed_page(struct page_pool *pool, struct page *page, > - unsigned int dma_sync_size, bool allow_direct) > +void page_pool_put_unrefed_netmem(struct page_pool *pool, netmem_ref netmem, > + unsigned int dma_sync_size, bool allow_direct) > { > - page = __page_pool_put_page(pool, page, dma_sync_size, allow_direct); > - if (page && !page_pool_recycle_in_ring(pool, page)) { > + netmem = > + __page_pool_put_page(pool, netmem, dma_sync_size, allow_direct); > + if (netmem && !page_pool_recycle_in_ring(pool, netmem)) { > /* Cache full, fallback to free pages */ > recycle_stat_inc(pool, ring_full); > - page_pool_return_page(pool, page); > + page_pool_return_page(pool, netmem); > } > } > +EXPORT_SYMBOL(page_pool_put_unrefed_netmem); > + > +void page_pool_put_unrefed_page(struct page_pool *pool, struct page *page, > + unsigned int dma_sync_size, bool allow_direct) > +{ > + page_pool_put_unrefed_netmem(pool, page_to_netmem(page), dma_sync_size, > + allow_direct); > +} > EXPORT_SYMBOL(page_pool_put_unrefed_page); > > /** > @@ -777,16 +796,16 @@ void page_pool_put_page_bulk(struct page_pool *pool, void **data, > bool in_softirq; > > for (i = 0; i < count; i++) { > - struct page *page = virt_to_head_page(data[i]); > + netmem_ref netmem = page_to_netmem(virt_to_head_page(data[i])); > > /* It is not the last user for the page frag case */ > - if (!page_pool_is_last_ref(page)) > + if (!page_pool_is_last_ref(netmem)) > continue; > > - page = __page_pool_put_page(pool, page, -1, false); > + netmem = __page_pool_put_page(pool, netmem, -1, false); > /* Approved for bulk recycling in ptr_ring cache */ > - if (page) > - data[bulk_len++] = page; > + if (netmem) > + data[bulk_len++] = (__force void *)netmem; > } > > if (unlikely(!bulk_len)) > @@ -812,100 +831,108 @@ void page_pool_put_page_bulk(struct page_pool *pool, void **data, > * since put_page() with refcnt == 1 can be an expensive operation > */ > for (; i < bulk_len; i++) > - page_pool_return_page(pool, data[i]); > + page_pool_return_page(pool, (__force netmem_ref)data[i]); > } > EXPORT_SYMBOL(page_pool_put_page_bulk); > > -static struct page *page_pool_drain_frag(struct page_pool *pool, > - struct page *page) > +static netmem_ref page_pool_drain_frag(struct page_pool *pool, > + netmem_ref netmem) > { > long drain_count = BIAS_MAX - pool->frag_users; > > /* Some user is still using the page frag */ > - if (likely(page_pool_unref_page(page, drain_count))) > - return NULL; > + if (likely(page_pool_unref_netmem(netmem, drain_count))) > + return 0; > > - if (__page_pool_page_can_be_recycled(page)) { > + if (__page_pool_page_can_be_recycled(netmem)) { > if (pool->p.flags & PP_FLAG_DMA_SYNC_DEV) > - page_pool_dma_sync_for_device(pool, page, -1); > + page_pool_dma_sync_for_device(pool, netmem, -1); > > - return page; > + return netmem; > } > > - page_pool_return_page(pool, page); > - return NULL; > + page_pool_return_page(pool, netmem); > + return 0; > } > > static void page_pool_free_frag(struct page_pool *pool) > { > long drain_count = BIAS_MAX - pool->frag_users; > - struct page *page = pool->frag_page; > + netmem_ref netmem = pool->frag_page; > > - pool->frag_page = NULL; > + pool->frag_page = 0; > > - if (!page || page_pool_unref_page(page, drain_count)) > + if (!netmem || page_pool_unref_netmem(netmem, drain_count)) > return; > > - page_pool_return_page(pool, page); > + page_pool_return_page(pool, netmem); > } > > -struct page *page_pool_alloc_frag(struct page_pool *pool, > - unsigned int *offset, > - unsigned int size, gfp_t gfp) > +netmem_ref page_pool_alloc_frag_netmem(struct page_pool *pool, > + unsigned int *offset, unsigned int size, > + gfp_t gfp) > { > unsigned int max_size = PAGE_SIZE << pool->p.order; > - struct page *page = pool->frag_page; > + netmem_ref netmem = pool->frag_page; > > if (WARN_ON(size > max_size)) > - return NULL; > + return 0; > > size = ALIGN(size, dma_get_cache_alignment()); > *offset = pool->frag_offset; > > - if (page && *offset + size > max_size) { > - page = page_pool_drain_frag(pool, page); > - if (page) { > + if (netmem && *offset + size > max_size) { > + netmem = page_pool_drain_frag(pool, netmem); > + if (netmem) { > alloc_stat_inc(pool, fast); > goto frag_reset; > } > } > > - if (!page) { > - page = page_pool_alloc_pages(pool, gfp); > - if (unlikely(!page)) { > - pool->frag_page = NULL; > - return NULL; > + if (!netmem) { > + netmem = page_pool_alloc_netmem(pool, gfp); > + if (unlikely(!netmem)) { > + pool->frag_page = 0; > + return 0; > } > > - pool->frag_page = page; > + pool->frag_page = netmem; > > frag_reset: > pool->frag_users = 1; > *offset = 0; > pool->frag_offset = size; > - page_pool_fragment_page(page, BIAS_MAX); > - return page; > + page_pool_fragment_netmem(netmem, BIAS_MAX); > + return netmem; > } > > pool->frag_users++; > pool->frag_offset = *offset + size; > alloc_stat_inc(pool, fast); > - return page; > + return netmem; > +} > +EXPORT_SYMBOL(page_pool_alloc_frag_netmem); > + > +struct page *page_pool_alloc_frag(struct page_pool *pool, unsigned int *offset, > + unsigned int size, gfp_t gfp) > +{ > + return netmem_to_page(page_pool_alloc_frag_netmem(pool, offset, size, > + gfp)); > } > EXPORT_SYMBOL(page_pool_alloc_frag); > > static void page_pool_empty_ring(struct page_pool *pool) > { > - struct page *page; > + netmem_ref netmem; > > /* Empty recycle ring */ > - while ((page = ptr_ring_consume_bh(&pool->ring))) { > + while ((netmem = (__force netmem_ref)ptr_ring_consume_bh(&pool->ring))) { > /* Verify the refcnt invariant of cached pages */ > - if (!(page_ref_count(page) == 1)) > + if (!(page_ref_count(netmem_to_page(netmem)) == 1)) > pr_crit("%s() page_pool refcnt %d violation\n", > - __func__, page_ref_count(page)); > + __func__, netmem_ref_count(netmem)); > > - page_pool_return_page(pool, page); > + page_pool_return_page(pool, netmem); > } > } > > @@ -927,7 +954,7 @@ static void __page_pool_destroy(struct page_pool *pool) > > static void page_pool_empty_alloc_cache_once(struct page_pool *pool) > { > - struct page *page; > + netmem_ref netmem; > > if (pool->destroy_cnt) > return; > @@ -937,8 +964,8 @@ static void page_pool_empty_alloc_cache_once(struct page_pool *pool) > * call concurrently. > */ > while (pool->alloc.count) { > - page = pool->alloc.cache[--pool->alloc.count]; > - page_pool_return_page(pool, page); > + netmem = pool->alloc.cache[--pool->alloc.count]; > + page_pool_return_page(pool, netmem); > } > } > > @@ -1044,15 +1071,15 @@ EXPORT_SYMBOL(page_pool_destroy); > /* Caller must provide appropriate safe context, e.g. NAPI. */ > void page_pool_update_nid(struct page_pool *pool, int new_nid) > { > - struct page *page; > + netmem_ref netmem; > > trace_page_pool_update_nid(pool, new_nid); > pool->p.nid = new_nid; > > /* Flush pool alloc cache, as refill will check NUMA node */ > while (pool->alloc.count) { > - page = pool->alloc.cache[--pool->alloc.count]; > - page_pool_return_page(pool, page); > + netmem = pool->alloc.cache[--pool->alloc.count]; > + page_pool_return_page(pool, netmem); > } > } > EXPORT_SYMBOL(page_pool_update_nid); > diff --git a/net/core/skbuff.c b/net/core/skbuff.c > index 1f918e602bc4..e1118b637085 100644 > --- a/net/core/skbuff.c > +++ b/net/core/skbuff.c > @@ -1006,8 +1006,9 @@ int skb_cow_data_for_xdp(struct page_pool *pool, struct sk_buff **pskb, > EXPORT_SYMBOL(skb_cow_data_for_xdp); > > #if IS_ENABLED(CONFIG_PAGE_POOL) > -bool napi_pp_put_page(struct page *page, bool napi_safe) > +bool napi_pp_put_page(netmem_ref netmem, bool napi_safe) > { > + struct page *page = netmem_to_page(netmem); > bool allow_direct = false; > struct page_pool *pp; > > @@ -1044,7 +1045,7 @@ bool napi_pp_put_page(struct page *page, bool napi_safe) > * The page will be returned to the pool here regardless of the > * 'flipped' fragment being in use or not. > */ > - page_pool_put_full_page(pp, page, allow_direct); > + page_pool_put_full_netmem(pp, page_to_netmem(page), allow_direct); > > return true; > } > @@ -1055,7 +1056,7 @@ static bool skb_pp_recycle(struct sk_buff *skb, void *data, bool napi_safe) > { > if (!IS_ENABLED(CONFIG_PAGE_POOL) || !skb->pp_recycle) > return false; > - return napi_pp_put_page(virt_to_page(data), napi_safe); > + return napi_pp_put_page(page_to_netmem(virt_to_page(data)), napi_safe); > } > > /** > -- > 2.44.0.rc1.240.g4c46232300-goog >
diff --git a/include/linux/skbuff.h b/include/linux/skbuff.h index d577e0bee18d..ca29d1fd4561 100644 --- a/include/linux/skbuff.h +++ b/include/linux/skbuff.h @@ -3504,7 +3504,7 @@ int skb_pp_cow_data(struct page_pool *pool, struct sk_buff **pskb, unsigned int headroom); int skb_cow_data_for_xdp(struct page_pool *pool, struct sk_buff **pskb, struct bpf_prog *prog); -bool napi_pp_put_page(struct page *page, bool napi_safe); +bool napi_pp_put_page(netmem_ref netmem, bool napi_safe); static inline void napi_frag_unref(skb_frag_t *frag, bool recycle, bool napi_safe) @@ -3512,7 +3512,7 @@ napi_frag_unref(skb_frag_t *frag, bool recycle, bool napi_safe) struct page *page = skb_frag_page(frag); #ifdef CONFIG_PAGE_POOL - if (recycle && napi_pp_put_page(page, napi_safe)) + if (recycle && napi_pp_put_page(page_to_netmem(page), napi_safe)) return; #endif put_page(page); diff --git a/include/net/netmem.h b/include/net/netmem.h index ca17ea1d33f8..21f53b29e5fe 100644 --- a/include/net/netmem.h +++ b/include/net/netmem.h @@ -88,4 +88,19 @@ static inline netmem_ref page_to_netmem(struct page *page) return (__force netmem_ref)page; } +static inline int netmem_ref_count(netmem_ref netmem) +{ + return page_ref_count(netmem_to_page(netmem)); +} + +static inline unsigned long netmem_to_pfn(netmem_ref netmem) +{ + return page_to_pfn(netmem_to_page(netmem)); +} + +static inline netmem_ref netmem_compound_head(netmem_ref netmem) +{ + return page_to_netmem(compound_head(netmem_to_page(netmem))); +} + #endif /* _NET_NETMEM_H */ diff --git a/include/net/page_pool/helpers.h b/include/net/page_pool/helpers.h index 1d397c1a0043..61814f91a458 100644 --- a/include/net/page_pool/helpers.h +++ b/include/net/page_pool/helpers.h @@ -53,6 +53,8 @@ #define _NET_PAGE_POOL_HELPERS_H #include <net/page_pool/types.h> +#include <net/net_debug.h> +#include <net/netmem.h> #ifdef CONFIG_PAGE_POOL_STATS /* Deprecated driver-facing API, use netlink instead */ @@ -101,7 +103,7 @@ static inline struct page *page_pool_dev_alloc_pages(struct page_pool *pool) * Get a page fragment from the page allocator or page_pool caches. * * Return: - * Return allocated page fragment, otherwise return NULL. + * Return allocated page fragment, otherwise return 0. */ static inline struct page *page_pool_dev_alloc_frag(struct page_pool *pool, unsigned int *offset, @@ -112,22 +114,22 @@ static inline struct page *page_pool_dev_alloc_frag(struct page_pool *pool, return page_pool_alloc_frag(pool, offset, size, gfp); } -static inline struct page *page_pool_alloc(struct page_pool *pool, - unsigned int *offset, - unsigned int *size, gfp_t gfp) +static inline netmem_ref page_pool_alloc(struct page_pool *pool, + unsigned int *offset, + unsigned int *size, gfp_t gfp) { unsigned int max_size = PAGE_SIZE << pool->p.order; - struct page *page; + netmem_ref netmem; if ((*size << 1) > max_size) { *size = max_size; *offset = 0; - return page_pool_alloc_pages(pool, gfp); + return page_pool_alloc_netmem(pool, gfp); } - page = page_pool_alloc_frag(pool, offset, *size, gfp); - if (unlikely(!page)) - return NULL; + netmem = page_pool_alloc_frag_netmem(pool, offset, *size, gfp); + if (unlikely(!netmem)) + return 0; /* There is very likely not enough space for another fragment, so append * the remaining size to the current fragment to avoid truesize @@ -138,7 +140,7 @@ static inline struct page *page_pool_alloc(struct page_pool *pool, pool->frag_offset = max_size; } - return page; + return netmem; } /** @@ -152,7 +154,7 @@ static inline struct page *page_pool_alloc(struct page_pool *pool, * utilization and performance penalty. * * Return: - * Return allocated page or page fragment, otherwise return NULL. + * Return allocated page or page fragment, otherwise return 0. */ static inline struct page *page_pool_dev_alloc(struct page_pool *pool, unsigned int *offset, @@ -160,7 +162,7 @@ static inline struct page *page_pool_dev_alloc(struct page_pool *pool, { gfp_t gfp = (GFP_ATOMIC | __GFP_NOWARN); - return page_pool_alloc(pool, offset, size, gfp); + return netmem_to_page(page_pool_alloc(pool, offset, size, gfp)); } static inline void *page_pool_alloc_va(struct page_pool *pool, @@ -170,9 +172,10 @@ static inline void *page_pool_alloc_va(struct page_pool *pool, struct page *page; /* Mask off __GFP_HIGHMEM to ensure we can use page_address() */ - page = page_pool_alloc(pool, &offset, size, gfp & ~__GFP_HIGHMEM); + page = netmem_to_page( + page_pool_alloc(pool, &offset, size, gfp & ~__GFP_HIGHMEM)); if (unlikely(!page)) - return NULL; + return 0; return page_address(page) + offset; } @@ -187,7 +190,7 @@ static inline void *page_pool_alloc_va(struct page_pool *pool, * it returns va of the allocated page or page fragment. * * Return: - * Return the va for the allocated page or page fragment, otherwise return NULL. + * Return the va for the allocated page or page fragment, otherwise return 0. */ static inline void *page_pool_dev_alloc_va(struct page_pool *pool, unsigned int *size) @@ -210,6 +213,11 @@ inline enum dma_data_direction page_pool_get_dma_dir(struct page_pool *pool) return pool->p.dma_dir; } +static inline void page_pool_fragment_netmem(netmem_ref netmem, long nr) +{ + atomic_long_set(&netmem_to_page(netmem)->pp_ref_count, nr); +} + /** * page_pool_fragment_page() - split a fresh page into fragments * @page: page to split @@ -230,11 +238,12 @@ inline enum dma_data_direction page_pool_get_dma_dir(struct page_pool *pool) */ static inline void page_pool_fragment_page(struct page *page, long nr) { - atomic_long_set(&page->pp_ref_count, nr); + page_pool_fragment_netmem(page_to_netmem(page), nr); } -static inline long page_pool_unref_page(struct page *page, long nr) +static inline long page_pool_unref_netmem(netmem_ref netmem, long nr) { + struct page *page = netmem_to_page(netmem); long ret; /* If nr == pp_ref_count then we have cleared all remaining @@ -277,15 +286,41 @@ static inline long page_pool_unref_page(struct page *page, long nr) return ret; } +static inline long page_pool_unref_page(struct page *page, long nr) +{ + return page_pool_unref_netmem(page_to_netmem(page), nr); +} + +static inline void page_pool_ref_netmem(netmem_ref netmem) +{ + atomic_long_inc(&netmem_to_page(netmem)->pp_ref_count); +} + static inline void page_pool_ref_page(struct page *page) { - atomic_long_inc(&page->pp_ref_count); + page_pool_ref_netmem(page_to_netmem(page)); } -static inline bool page_pool_is_last_ref(struct page *page) +static inline bool page_pool_is_last_ref(netmem_ref netmem) { /* If page_pool_unref_page() returns 0, we were the last user */ - return page_pool_unref_page(page, 1) == 0; + return page_pool_unref_netmem(netmem, 1) == 0; +} + +static inline void page_pool_put_netmem(struct page_pool *pool, + netmem_ref netmem, + unsigned int dma_sync_size, + bool allow_direct) +{ + /* When page_pool isn't compiled-in, net/core/xdp.c doesn't + * allow registering MEM_TYPE_PAGE_POOL, but shield linker. + */ +#ifdef CONFIG_PAGE_POOL + if (!page_pool_is_last_ref(netmem)) + return; + + page_pool_put_unrefed_netmem(pool, netmem, dma_sync_size, allow_direct); +#endif } /** @@ -306,15 +341,15 @@ static inline void page_pool_put_page(struct page_pool *pool, unsigned int dma_sync_size, bool allow_direct) { - /* When page_pool isn't compiled-in, net/core/xdp.c doesn't - * allow registering MEM_TYPE_PAGE_POOL, but shield linker. - */ -#ifdef CONFIG_PAGE_POOL - if (!page_pool_is_last_ref(page)) - return; + page_pool_put_netmem(pool, page_to_netmem(page), dma_sync_size, + allow_direct); +} - page_pool_put_unrefed_page(pool, page, dma_sync_size, allow_direct); -#endif +static inline void page_pool_put_full_netmem(struct page_pool *pool, + netmem_ref netmem, + bool allow_direct) +{ + page_pool_put_netmem(pool, netmem, -1, allow_direct); } /** @@ -329,7 +364,7 @@ static inline void page_pool_put_page(struct page_pool *pool, static inline void page_pool_put_full_page(struct page_pool *pool, struct page *page, bool allow_direct) { - page_pool_put_page(pool, page, -1, allow_direct); + page_pool_put_netmem(pool, page_to_netmem(page), -1, allow_direct); } /** @@ -363,6 +398,18 @@ static inline void page_pool_free_va(struct page_pool *pool, void *va, page_pool_put_page(pool, virt_to_head_page(va), -1, allow_direct); } +static inline dma_addr_t page_pool_get_dma_addr_netmem(netmem_ref netmem) +{ + struct page *page = netmem_to_page(netmem); + + dma_addr_t ret = page->dma_addr; + + if (PAGE_POOL_32BIT_ARCH_WITH_64BIT_DMA) + ret <<= PAGE_SHIFT; + + return ret; +} + /** * page_pool_get_dma_addr() - Retrieve the stored DMA address. * @page: page allocated from a page pool @@ -372,16 +419,14 @@ static inline void page_pool_free_va(struct page_pool *pool, void *va, */ static inline dma_addr_t page_pool_get_dma_addr(struct page *page) { - dma_addr_t ret = page->dma_addr; - - if (PAGE_POOL_32BIT_ARCH_WITH_64BIT_DMA) - ret <<= PAGE_SHIFT; - - return ret; + return page_pool_get_dma_addr_netmem(page_to_netmem(page)); } -static inline bool page_pool_set_dma_addr(struct page *page, dma_addr_t addr) +static inline bool page_pool_set_dma_addr_netmem(netmem_ref netmem, + dma_addr_t addr) { + struct page *page = netmem_to_page(netmem); + if (PAGE_POOL_32BIT_ARCH_WITH_64BIT_DMA) { page->dma_addr = addr >> PAGE_SHIFT; @@ -395,6 +440,11 @@ static inline bool page_pool_set_dma_addr(struct page *page, dma_addr_t addr) return false; } +static inline bool page_pool_set_dma_addr(struct page *page, dma_addr_t addr) +{ + return page_pool_set_dma_addr_netmem(page_to_netmem(page), addr); +} + static inline bool page_pool_put(struct page_pool *pool) { return refcount_dec_and_test(&pool->user_cnt); diff --git a/include/net/page_pool/types.h b/include/net/page_pool/types.h index ffe5f31fb0da..68a24c5ae827 100644 --- a/include/net/page_pool/types.h +++ b/include/net/page_pool/types.h @@ -40,7 +40,7 @@ #define PP_ALLOC_CACHE_REFILL 64 struct pp_alloc_cache { u32 count; - struct page *cache[PP_ALLOC_CACHE_SIZE]; + netmem_ref cache[PP_ALLOC_CACHE_SIZE]; }; /** @@ -73,7 +73,7 @@ struct page_pool_params { struct_group_tagged(page_pool_params_slow, slow, struct net_device *netdev; /* private: used by test code only */ - void (*init_callback)(struct page *page, void *arg); + void (*init_callback)(netmem_ref netmem, void *arg); void *init_arg; ); }; @@ -131,8 +131,8 @@ struct page_pool_stats { struct memory_provider_ops { int (*init)(struct page_pool *pool); void (*destroy)(struct page_pool *pool); - struct page *(*alloc_pages)(struct page_pool *pool, gfp_t gfp); - bool (*release_page)(struct page_pool *pool, struct page *page); + netmem_ref (*alloc_pages)(struct page_pool *pool, gfp_t gfp); + bool (*release_page)(struct page_pool *pool, netmem_ref netmem); }; struct page_pool { @@ -142,7 +142,7 @@ struct page_pool { bool has_init_callback; long frag_users; - struct page *frag_page; + netmem_ref frag_page; unsigned int frag_offset; u32 pages_state_hold_cnt; @@ -214,8 +214,12 @@ struct page_pool { }; struct page *page_pool_alloc_pages(struct page_pool *pool, gfp_t gfp); +netmem_ref page_pool_alloc_netmem(struct page_pool *pool, gfp_t gfp); struct page *page_pool_alloc_frag(struct page_pool *pool, unsigned int *offset, unsigned int size, gfp_t gfp); +netmem_ref page_pool_alloc_frag_netmem(struct page_pool *pool, + unsigned int *offset, unsigned int size, + gfp_t gfp); struct page_pool *page_pool_create(const struct page_pool_params *params); struct page_pool *page_pool_create_percpu(const struct page_pool_params *params, int cpuid); @@ -245,6 +249,9 @@ static inline void page_pool_put_page_bulk(struct page_pool *pool, void **data, } #endif +void page_pool_put_unrefed_netmem(struct page_pool *pool, netmem_ref netmem, + unsigned int dma_sync_size, + bool allow_direct); void page_pool_put_unrefed_page(struct page_pool *pool, struct page *page, unsigned int dma_sync_size, bool allow_direct); diff --git a/include/trace/events/page_pool.h b/include/trace/events/page_pool.h index 6834356b2d2a..c5b6383ff276 100644 --- a/include/trace/events/page_pool.h +++ b/include/trace/events/page_pool.h @@ -42,51 +42,52 @@ TRACE_EVENT(page_pool_release, TRACE_EVENT(page_pool_state_release, TP_PROTO(const struct page_pool *pool, - const struct page *page, u32 release), + netmem_ref netmem, u32 release), - TP_ARGS(pool, page, release), + TP_ARGS(pool, netmem, release), TP_STRUCT__entry( __field(const struct page_pool *, pool) - __field(const struct page *, page) + __field(netmem_ref, netmem) __field(u32, release) __field(unsigned long, pfn) ), TP_fast_assign( __entry->pool = pool; - __entry->page = page; + __entry->netmem = netmem; __entry->release = release; - __entry->pfn = page_to_pfn(page); + __entry->pfn = netmem_to_pfn(netmem); ), - TP_printk("page_pool=%p page=%p pfn=0x%lx release=%u", - __entry->pool, __entry->page, __entry->pfn, __entry->release) + TP_printk("page_pool=%p netmem=%lu pfn=0x%lx release=%u", + __entry->pool, (__force unsigned long)__entry->netmem, + __entry->pfn, __entry->release) ); TRACE_EVENT(page_pool_state_hold, TP_PROTO(const struct page_pool *pool, - const struct page *page, u32 hold), + netmem_ref netmem, u32 hold), - TP_ARGS(pool, page, hold), + TP_ARGS(pool, netmem, hold), TP_STRUCT__entry( __field(const struct page_pool *, pool) - __field(const struct page *, page) + __field(netmem_ref, netmem) __field(u32, hold) __field(unsigned long, pfn) ), TP_fast_assign( __entry->pool = pool; - __entry->page = page; + __entry->netmem = netmem; __entry->hold = hold; - __entry->pfn = page_to_pfn(page); + __entry->pfn = netmem_to_pfn(netmem); ), - TP_printk("page_pool=%p page=%p pfn=0x%lx hold=%u", - __entry->pool, __entry->page, __entry->pfn, __entry->hold) + TP_printk("page_pool=%p netmem=%lu pfn=0x%lx hold=%u", + __entry->pool, __entry->netmem, __entry->pfn, __entry->hold) ); TRACE_EVENT(page_pool_update_nid, diff --git a/net/bpf/test_run.c b/net/bpf/test_run.c index 5535f9adc658..bc8f7ab88f86 100644 --- a/net/bpf/test_run.c +++ b/net/bpf/test_run.c @@ -126,9 +126,10 @@ struct xdp_test_data { #define TEST_XDP_FRAME_SIZE (PAGE_SIZE - sizeof(struct xdp_page_head)) #define TEST_XDP_MAX_BATCH 256 -static void xdp_test_run_init_page(struct page *page, void *arg) +static void xdp_test_run_init_page(netmem_ref netmem, void *arg) { - struct xdp_page_head *head = phys_to_virt(page_to_phys(page)); + struct xdp_page_head *head = + phys_to_virt(page_to_phys(netmem_to_page(netmem))); struct xdp_buff *new_ctx, *orig_ctx; u32 headroom = XDP_PACKET_HEADROOM; struct xdp_test_data *xdp = arg; diff --git a/net/core/page_pool.c b/net/core/page_pool.c index fe9de4ecce94..24d5236b2efc 100644 --- a/net/core/page_pool.c +++ b/net/core/page_pool.c @@ -329,19 +329,18 @@ struct page_pool *page_pool_create(const struct page_pool_params *params) } EXPORT_SYMBOL(page_pool_create); -static void page_pool_return_page(struct page_pool *pool, struct page *page); +static void page_pool_return_page(struct page_pool *pool, netmem_ref netmem); -noinline -static struct page *page_pool_refill_alloc_cache(struct page_pool *pool) +static noinline netmem_ref page_pool_refill_alloc_cache(struct page_pool *pool) { struct ptr_ring *r = &pool->ring; - struct page *page; + netmem_ref netmem; int pref_nid; /* preferred NUMA node */ /* Quicker fallback, avoid locks when ring is empty */ if (__ptr_ring_empty(r)) { alloc_stat_inc(pool, empty); - return NULL; + return 0; } /* Softirq guarantee CPU and thus NUMA node is stable. This, @@ -356,56 +355,56 @@ static struct page *page_pool_refill_alloc_cache(struct page_pool *pool) /* Refill alloc array, but only if NUMA match */ do { - page = __ptr_ring_consume(r); - if (unlikely(!page)) + netmem = (__force netmem_ref)__ptr_ring_consume(r); + if (unlikely(!netmem)) break; - if (likely(page_to_nid(page) == pref_nid)) { - pool->alloc.cache[pool->alloc.count++] = page; + if (likely(page_to_nid(netmem_to_page(netmem)) == pref_nid)) { + pool->alloc.cache[pool->alloc.count++] = netmem; } else { /* NUMA mismatch; * (1) release 1 page to page-allocator and * (2) break out to fallthrough to alloc_pages_node. * This limit stress on page buddy alloactor. */ - page_pool_return_page(pool, page); + page_pool_return_page(pool, netmem); alloc_stat_inc(pool, waive); - page = NULL; + netmem = 0; break; } } while (pool->alloc.count < PP_ALLOC_CACHE_REFILL); /* Return last page */ if (likely(pool->alloc.count > 0)) { - page = pool->alloc.cache[--pool->alloc.count]; + netmem = pool->alloc.cache[--pool->alloc.count]; alloc_stat_inc(pool, refill); } - return page; + return netmem; } /* fast path */ -static struct page *__page_pool_get_cached(struct page_pool *pool) +static netmem_ref __page_pool_get_cached(struct page_pool *pool) { - struct page *page; + netmem_ref netmem; /* Caller MUST guarantee safe non-concurrent access, e.g. softirq */ if (likely(pool->alloc.count)) { /* Fast-path */ - page = pool->alloc.cache[--pool->alloc.count]; + netmem = pool->alloc.cache[--pool->alloc.count]; alloc_stat_inc(pool, fast); } else { - page = page_pool_refill_alloc_cache(pool); + netmem = page_pool_refill_alloc_cache(pool); } - return page; + return netmem; } static void page_pool_dma_sync_for_device(struct page_pool *pool, - struct page *page, + netmem_ref netmem, unsigned int dma_sync_size) { - dma_addr_t dma_addr = page_pool_get_dma_addr(page); + dma_addr_t dma_addr = page_pool_get_dma_addr_netmem(netmem); dma_sync_size = min(dma_sync_size, pool->p.max_len); dma_sync_single_range_for_device(pool->p.dev, dma_addr, @@ -413,7 +412,7 @@ static void page_pool_dma_sync_for_device(struct page_pool *pool, pool->p.dma_dir); } -static bool page_pool_dma_map(struct page_pool *pool, struct page *page) +static bool page_pool_dma_map(struct page_pool *pool, netmem_ref netmem) { dma_addr_t dma; @@ -422,18 +421,18 @@ static bool page_pool_dma_map(struct page_pool *pool, struct page *page) * into page private data (i.e 32bit cpu with 64bit DMA caps) * This mapping is kept for lifetime of page, until leaving pool. */ - dma = dma_map_page_attrs(pool->p.dev, page, 0, - (PAGE_SIZE << pool->p.order), - pool->p.dma_dir, DMA_ATTR_SKIP_CPU_SYNC | - DMA_ATTR_WEAK_ORDERING); + dma = dma_map_page_attrs(pool->p.dev, netmem_to_page(netmem), 0, + (PAGE_SIZE << pool->p.order), pool->p.dma_dir, + DMA_ATTR_SKIP_CPU_SYNC | + DMA_ATTR_WEAK_ORDERING); if (dma_mapping_error(pool->p.dev, dma)) return false; - if (page_pool_set_dma_addr(page, dma)) + if (page_pool_set_dma_addr_netmem(netmem, dma)) goto unmap_failed; if (pool->p.flags & PP_FLAG_DMA_SYNC_DEV) - page_pool_dma_sync_for_device(pool, page, pool->p.max_len); + page_pool_dma_sync_for_device(pool, netmem, pool->p.max_len); return true; @@ -445,9 +444,10 @@ static bool page_pool_dma_map(struct page_pool *pool, struct page *page) return false; } -static void page_pool_set_pp_info(struct page_pool *pool, - struct page *page) +static void page_pool_set_pp_info(struct page_pool *pool, netmem_ref netmem) { + struct page *page = netmem_to_page(netmem); + page->pp = pool; page->pp_magic |= PP_SIGNATURE; @@ -457,13 +457,15 @@ static void page_pool_set_pp_info(struct page_pool *pool, * is dirtying the same cache line as the page->pp_magic above, so * the overhead is negligible. */ - page_pool_fragment_page(page, 1); + page_pool_fragment_netmem(netmem, 1); if (pool->has_init_callback) - pool->slow.init_callback(page, pool->slow.init_arg); + pool->slow.init_callback(netmem, pool->slow.init_arg); } -static void page_pool_clear_pp_info(struct page *page) +static void page_pool_clear_pp_info(netmem_ref netmem) { + struct page *page = netmem_to_page(netmem); + page->pp_magic = 0; page->pp = NULL; } @@ -479,34 +481,34 @@ static struct page *__page_pool_alloc_page_order(struct page_pool *pool, return NULL; if ((pool->p.flags & PP_FLAG_DMA_MAP) && - unlikely(!page_pool_dma_map(pool, page))) { + unlikely(!page_pool_dma_map(pool, page_to_netmem(page)))) { put_page(page); return NULL; } alloc_stat_inc(pool, slow_high_order); - page_pool_set_pp_info(pool, page); + page_pool_set_pp_info(pool, page_to_netmem(page)); /* Track how many pages are held 'in-flight' */ pool->pages_state_hold_cnt++; - trace_page_pool_state_hold(pool, page, pool->pages_state_hold_cnt); + trace_page_pool_state_hold(pool, page_to_netmem(page), + pool->pages_state_hold_cnt); return page; } /* slow path */ -noinline -static struct page *__page_pool_alloc_pages_slow(struct page_pool *pool, - gfp_t gfp) +static noinline netmem_ref __page_pool_alloc_pages_slow(struct page_pool *pool, + gfp_t gfp) { const int bulk = PP_ALLOC_CACHE_REFILL; unsigned int pp_flags = pool->p.flags; unsigned int pp_order = pool->p.order; - struct page *page; + netmem_ref netmem; int i, nr_pages; /* Don't support bulk alloc for high-order pages */ if (unlikely(pp_order)) - return __page_pool_alloc_page_order(pool, gfp); + return page_to_netmem(__page_pool_alloc_page_order(pool, gfp)); /* Unnecessary as alloc cache is empty, but guarantees zero count */ if (unlikely(pool->alloc.count > 0)) @@ -515,60 +517,67 @@ static struct page *__page_pool_alloc_pages_slow(struct page_pool *pool, /* Mark empty alloc.cache slots "empty" for alloc_pages_bulk_array */ memset(&pool->alloc.cache, 0, sizeof(void *) * bulk); - nr_pages = alloc_pages_bulk_array_node(gfp, pool->p.nid, bulk, - pool->alloc.cache); + nr_pages = alloc_pages_bulk_array_node(gfp, + pool->p.nid, bulk, + (struct page **)pool->alloc.cache); if (unlikely(!nr_pages)) - return NULL; + return 0; /* Pages have been filled into alloc.cache array, but count is zero and * page element have not been (possibly) DMA mapped. */ for (i = 0; i < nr_pages; i++) { - page = pool->alloc.cache[i]; + netmem = pool->alloc.cache[i]; if ((pp_flags & PP_FLAG_DMA_MAP) && - unlikely(!page_pool_dma_map(pool, page))) { - put_page(page); + unlikely(!page_pool_dma_map(pool, netmem))) { + put_page(netmem_to_page(netmem)); continue; } - page_pool_set_pp_info(pool, page); - pool->alloc.cache[pool->alloc.count++] = page; + page_pool_set_pp_info(pool, netmem); + pool->alloc.cache[pool->alloc.count++] = netmem; /* Track how many pages are held 'in-flight' */ pool->pages_state_hold_cnt++; - trace_page_pool_state_hold(pool, page, + trace_page_pool_state_hold(pool, netmem, pool->pages_state_hold_cnt); } /* Return last page */ if (likely(pool->alloc.count > 0)) { - page = pool->alloc.cache[--pool->alloc.count]; + netmem = pool->alloc.cache[--pool->alloc.count]; alloc_stat_inc(pool, slow); } else { - page = NULL; + netmem = 0; } /* When page just alloc'ed is should/must have refcnt 1. */ - return page; + return netmem; } /* For using page_pool replace: alloc_pages() API calls, but provide * synchronization guarantee for allocation side. */ -struct page *page_pool_alloc_pages(struct page_pool *pool, gfp_t gfp) +netmem_ref page_pool_alloc_netmem(struct page_pool *pool, gfp_t gfp) { - struct page *page; + netmem_ref netmem; /* Fast-path: Get a page from cache */ - page = __page_pool_get_cached(pool); - if (page) - return page; + netmem = __page_pool_get_cached(pool); + if (netmem) + return netmem; /* Slow-path: cache empty, do real allocation */ if (static_branch_unlikely(&page_pool_mem_providers) && pool->mp_ops) - page = pool->mp_ops->alloc_pages(pool, gfp); + netmem = pool->mp_ops->alloc_pages(pool, gfp); else - page = __page_pool_alloc_pages_slow(pool, gfp); - return page; + netmem = __page_pool_alloc_pages_slow(pool, gfp); + return netmem; +} +EXPORT_SYMBOL(page_pool_alloc_netmem); + +struct page *page_pool_alloc_pages(struct page_pool *pool, gfp_t gfp) +{ + return netmem_to_page(page_pool_alloc_netmem(pool, gfp)); } EXPORT_SYMBOL(page_pool_alloc_pages); @@ -596,8 +605,8 @@ s32 page_pool_inflight(const struct page_pool *pool, bool strict) return inflight; } -static __always_inline -void __page_pool_release_page_dma(struct page_pool *pool, struct page *page) +static __always_inline void __page_pool_release_page_dma(struct page_pool *pool, + netmem_ref netmem) { dma_addr_t dma; @@ -607,13 +616,13 @@ void __page_pool_release_page_dma(struct page_pool *pool, struct page *page) */ return; - dma = page_pool_get_dma_addr(page); + dma = page_pool_get_dma_addr_netmem(netmem); /* When page is unmapped, it cannot be returned to our pool */ dma_unmap_page_attrs(pool->p.dev, dma, PAGE_SIZE << pool->p.order, pool->p.dma_dir, DMA_ATTR_SKIP_CPU_SYNC | DMA_ATTR_WEAK_ORDERING); - page_pool_set_dma_addr(page, 0); + page_pool_set_dma_addr_netmem(netmem, 0); } /* Disconnects a page (from a page_pool). API users can have a need @@ -621,26 +630,26 @@ void __page_pool_release_page_dma(struct page_pool *pool, struct page *page) * a regular page (that will eventually be returned to the normal * page-allocator via put_page). */ -void page_pool_return_page(struct page_pool *pool, struct page *page) +void page_pool_return_page(struct page_pool *pool, netmem_ref netmem) { int count; bool put; put = true; if (static_branch_unlikely(&page_pool_mem_providers) && pool->mp_ops) - put = pool->mp_ops->release_page(pool, page); + put = pool->mp_ops->release_page(pool, netmem); else - __page_pool_release_page_dma(pool, page); + __page_pool_release_page_dma(pool, netmem); /* This may be the last page returned, releasing the pool, so * it is not safe to reference pool afterwards. */ count = atomic_inc_return_relaxed(&pool->pages_state_release_cnt); - trace_page_pool_state_release(pool, page, count); + trace_page_pool_state_release(pool, netmem, count); if (put) { - page_pool_clear_pp_info(page); - put_page(page); + page_pool_clear_pp_info(netmem); + put_page(netmem_to_page(netmem)); } /* An optimization would be to call __free_pages(page, pool->p.order) * knowing page is not part of page-cache (thus avoiding a @@ -648,14 +657,14 @@ void page_pool_return_page(struct page_pool *pool, struct page *page) */ } -static bool page_pool_recycle_in_ring(struct page_pool *pool, struct page *page) +static bool page_pool_recycle_in_ring(struct page_pool *pool, netmem_ref netmem) { int ret; /* BH protection not needed if current is softirq */ if (in_softirq()) - ret = ptr_ring_produce(&pool->ring, page); + ret = ptr_ring_produce(&pool->ring, (__force void *)netmem); else - ret = ptr_ring_produce_bh(&pool->ring, page); + ret = ptr_ring_produce_bh(&pool->ring, (__force void *)netmem); if (!ret) { recycle_stat_inc(pool, ring); @@ -670,7 +679,7 @@ static bool page_pool_recycle_in_ring(struct page_pool *pool, struct page *page) * * Caller must provide appropriate safe context. */ -static bool page_pool_recycle_in_cache(struct page *page, +static bool page_pool_recycle_in_cache(netmem_ref netmem, struct page_pool *pool) { if (unlikely(pool->alloc.count == PP_ALLOC_CACHE_SIZE)) { @@ -679,14 +688,15 @@ static bool page_pool_recycle_in_cache(struct page *page, } /* Caller MUST have verified/know (page_ref_count(page) == 1) */ - pool->alloc.cache[pool->alloc.count++] = page; + pool->alloc.cache[pool->alloc.count++] = netmem; recycle_stat_inc(pool, cached); return true; } -static bool __page_pool_page_can_be_recycled(const struct page *page) +static bool __page_pool_page_can_be_recycled(netmem_ref netmem) { - return page_ref_count(page) == 1 && !page_is_pfmemalloc(page); + return page_ref_count(netmem_to_page(netmem)) == 1 && + !page_is_pfmemalloc(netmem_to_page(netmem)); } /* If the page refcnt == 1, this will try to recycle the page. @@ -695,8 +705,8 @@ static bool __page_pool_page_can_be_recycled(const struct page *page) * If the page refcnt != 1, then the page will be returned to memory * subsystem. */ -static __always_inline struct page * -__page_pool_put_page(struct page_pool *pool, struct page *page, +static __always_inline netmem_ref +__page_pool_put_page(struct page_pool *pool, netmem_ref netmem, unsigned int dma_sync_size, bool allow_direct) { lockdep_assert_no_hardirq(); @@ -710,19 +720,19 @@ __page_pool_put_page(struct page_pool *pool, struct page *page, * page is NOT reusable when allocated when system is under * some pressure. (page_is_pfmemalloc) */ - if (likely(__page_pool_page_can_be_recycled(page))) { + if (likely(__page_pool_page_can_be_recycled(netmem))) { /* Read barrier done in page_ref_count / READ_ONCE */ if (pool->p.flags & PP_FLAG_DMA_SYNC_DEV) - page_pool_dma_sync_for_device(pool, page, + page_pool_dma_sync_for_device(pool, netmem, dma_sync_size); if (allow_direct && in_softirq() && - page_pool_recycle_in_cache(page, pool)) - return NULL; + page_pool_recycle_in_cache(netmem, pool)) + return 0; /* Page found as candidate for recycling */ - return page; + return netmem; } /* Fallback/non-XDP mode: API user have elevated refcnt. * @@ -738,21 +748,30 @@ __page_pool_put_page(struct page_pool *pool, struct page *page, * will be invoking put_page. */ recycle_stat_inc(pool, released_refcnt); - page_pool_return_page(pool, page); + page_pool_return_page(pool, netmem); - return NULL; + return 0; } -void page_pool_put_unrefed_page(struct page_pool *pool, struct page *page, - unsigned int dma_sync_size, bool allow_direct) +void page_pool_put_unrefed_netmem(struct page_pool *pool, netmem_ref netmem, + unsigned int dma_sync_size, bool allow_direct) { - page = __page_pool_put_page(pool, page, dma_sync_size, allow_direct); - if (page && !page_pool_recycle_in_ring(pool, page)) { + netmem = + __page_pool_put_page(pool, netmem, dma_sync_size, allow_direct); + if (netmem && !page_pool_recycle_in_ring(pool, netmem)) { /* Cache full, fallback to free pages */ recycle_stat_inc(pool, ring_full); - page_pool_return_page(pool, page); + page_pool_return_page(pool, netmem); } } +EXPORT_SYMBOL(page_pool_put_unrefed_netmem); + +void page_pool_put_unrefed_page(struct page_pool *pool, struct page *page, + unsigned int dma_sync_size, bool allow_direct) +{ + page_pool_put_unrefed_netmem(pool, page_to_netmem(page), dma_sync_size, + allow_direct); +} EXPORT_SYMBOL(page_pool_put_unrefed_page); /** @@ -777,16 +796,16 @@ void page_pool_put_page_bulk(struct page_pool *pool, void **data, bool in_softirq; for (i = 0; i < count; i++) { - struct page *page = virt_to_head_page(data[i]); + netmem_ref netmem = page_to_netmem(virt_to_head_page(data[i])); /* It is not the last user for the page frag case */ - if (!page_pool_is_last_ref(page)) + if (!page_pool_is_last_ref(netmem)) continue; - page = __page_pool_put_page(pool, page, -1, false); + netmem = __page_pool_put_page(pool, netmem, -1, false); /* Approved for bulk recycling in ptr_ring cache */ - if (page) - data[bulk_len++] = page; + if (netmem) + data[bulk_len++] = (__force void *)netmem; } if (unlikely(!bulk_len)) @@ -812,100 +831,108 @@ void page_pool_put_page_bulk(struct page_pool *pool, void **data, * since put_page() with refcnt == 1 can be an expensive operation */ for (; i < bulk_len; i++) - page_pool_return_page(pool, data[i]); + page_pool_return_page(pool, (__force netmem_ref)data[i]); } EXPORT_SYMBOL(page_pool_put_page_bulk); -static struct page *page_pool_drain_frag(struct page_pool *pool, - struct page *page) +static netmem_ref page_pool_drain_frag(struct page_pool *pool, + netmem_ref netmem) { long drain_count = BIAS_MAX - pool->frag_users; /* Some user is still using the page frag */ - if (likely(page_pool_unref_page(page, drain_count))) - return NULL; + if (likely(page_pool_unref_netmem(netmem, drain_count))) + return 0; - if (__page_pool_page_can_be_recycled(page)) { + if (__page_pool_page_can_be_recycled(netmem)) { if (pool->p.flags & PP_FLAG_DMA_SYNC_DEV) - page_pool_dma_sync_for_device(pool, page, -1); + page_pool_dma_sync_for_device(pool, netmem, -1); - return page; + return netmem; } - page_pool_return_page(pool, page); - return NULL; + page_pool_return_page(pool, netmem); + return 0; } static void page_pool_free_frag(struct page_pool *pool) { long drain_count = BIAS_MAX - pool->frag_users; - struct page *page = pool->frag_page; + netmem_ref netmem = pool->frag_page; - pool->frag_page = NULL; + pool->frag_page = 0; - if (!page || page_pool_unref_page(page, drain_count)) + if (!netmem || page_pool_unref_netmem(netmem, drain_count)) return; - page_pool_return_page(pool, page); + page_pool_return_page(pool, netmem); } -struct page *page_pool_alloc_frag(struct page_pool *pool, - unsigned int *offset, - unsigned int size, gfp_t gfp) +netmem_ref page_pool_alloc_frag_netmem(struct page_pool *pool, + unsigned int *offset, unsigned int size, + gfp_t gfp) { unsigned int max_size = PAGE_SIZE << pool->p.order; - struct page *page = pool->frag_page; + netmem_ref netmem = pool->frag_page; if (WARN_ON(size > max_size)) - return NULL; + return 0; size = ALIGN(size, dma_get_cache_alignment()); *offset = pool->frag_offset; - if (page && *offset + size > max_size) { - page = page_pool_drain_frag(pool, page); - if (page) { + if (netmem && *offset + size > max_size) { + netmem = page_pool_drain_frag(pool, netmem); + if (netmem) { alloc_stat_inc(pool, fast); goto frag_reset; } } - if (!page) { - page = page_pool_alloc_pages(pool, gfp); - if (unlikely(!page)) { - pool->frag_page = NULL; - return NULL; + if (!netmem) { + netmem = page_pool_alloc_netmem(pool, gfp); + if (unlikely(!netmem)) { + pool->frag_page = 0; + return 0; } - pool->frag_page = page; + pool->frag_page = netmem; frag_reset: pool->frag_users = 1; *offset = 0; pool->frag_offset = size; - page_pool_fragment_page(page, BIAS_MAX); - return page; + page_pool_fragment_netmem(netmem, BIAS_MAX); + return netmem; } pool->frag_users++; pool->frag_offset = *offset + size; alloc_stat_inc(pool, fast); - return page; + return netmem; +} +EXPORT_SYMBOL(page_pool_alloc_frag_netmem); + +struct page *page_pool_alloc_frag(struct page_pool *pool, unsigned int *offset, + unsigned int size, gfp_t gfp) +{ + return netmem_to_page(page_pool_alloc_frag_netmem(pool, offset, size, + gfp)); } EXPORT_SYMBOL(page_pool_alloc_frag); static void page_pool_empty_ring(struct page_pool *pool) { - struct page *page; + netmem_ref netmem; /* Empty recycle ring */ - while ((page = ptr_ring_consume_bh(&pool->ring))) { + while ((netmem = (__force netmem_ref)ptr_ring_consume_bh(&pool->ring))) { /* Verify the refcnt invariant of cached pages */ - if (!(page_ref_count(page) == 1)) + if (!(page_ref_count(netmem_to_page(netmem)) == 1)) pr_crit("%s() page_pool refcnt %d violation\n", - __func__, page_ref_count(page)); + __func__, netmem_ref_count(netmem)); - page_pool_return_page(pool, page); + page_pool_return_page(pool, netmem); } } @@ -927,7 +954,7 @@ static void __page_pool_destroy(struct page_pool *pool) static void page_pool_empty_alloc_cache_once(struct page_pool *pool) { - struct page *page; + netmem_ref netmem; if (pool->destroy_cnt) return; @@ -937,8 +964,8 @@ static void page_pool_empty_alloc_cache_once(struct page_pool *pool) * call concurrently. */ while (pool->alloc.count) { - page = pool->alloc.cache[--pool->alloc.count]; - page_pool_return_page(pool, page); + netmem = pool->alloc.cache[--pool->alloc.count]; + page_pool_return_page(pool, netmem); } } @@ -1044,15 +1071,15 @@ EXPORT_SYMBOL(page_pool_destroy); /* Caller must provide appropriate safe context, e.g. NAPI. */ void page_pool_update_nid(struct page_pool *pool, int new_nid) { - struct page *page; + netmem_ref netmem; trace_page_pool_update_nid(pool, new_nid); pool->p.nid = new_nid; /* Flush pool alloc cache, as refill will check NUMA node */ while (pool->alloc.count) { - page = pool->alloc.cache[--pool->alloc.count]; - page_pool_return_page(pool, page); + netmem = pool->alloc.cache[--pool->alloc.count]; + page_pool_return_page(pool, netmem); } } EXPORT_SYMBOL(page_pool_update_nid); diff --git a/net/core/skbuff.c b/net/core/skbuff.c index 1f918e602bc4..e1118b637085 100644 --- a/net/core/skbuff.c +++ b/net/core/skbuff.c @@ -1006,8 +1006,9 @@ int skb_cow_data_for_xdp(struct page_pool *pool, struct sk_buff **pskb, EXPORT_SYMBOL(skb_cow_data_for_xdp); #if IS_ENABLED(CONFIG_PAGE_POOL) -bool napi_pp_put_page(struct page *page, bool napi_safe) +bool napi_pp_put_page(netmem_ref netmem, bool napi_safe) { + struct page *page = netmem_to_page(netmem); bool allow_direct = false; struct page_pool *pp; @@ -1044,7 +1045,7 @@ bool napi_pp_put_page(struct page *page, bool napi_safe) * The page will be returned to the pool here regardless of the * 'flipped' fragment being in use or not. */ - page_pool_put_full_page(pp, page, allow_direct); + page_pool_put_full_netmem(pp, page_to_netmem(page), allow_direct); return true; } @@ -1055,7 +1056,7 @@ static bool skb_pp_recycle(struct sk_buff *skb, void *data, bool napi_safe) { if (!IS_ENABLED(CONFIG_PAGE_POOL) || !skb->pp_recycle) return false; - return napi_pp_put_page(virt_to_page(data), napi_safe); + return napi_pp_put_page(page_to_netmem(virt_to_page(data)), napi_safe); } /**
Abstrace the memory type from the page_pool so we can later add support for new memory types. Convert the page_pool to use the new netmem type abstraction, rather than use struct page directly. As of this patch the netmem type is a no-op abstraction: it's always a struct page underneath. All the page pool internals are converted to use struct netmem instead of struct page, and the page pool now exports 2 APIs: 1. The existing struct page API. 2. The new struct netmem API. Keeping the existing API is transitional; we do not want to refactor all the current drivers using the page pool at once. The netmem abstraction is currently a no-op. The page_pool uses page_to_netmem() to convert allocated pages to netmem, and uses netmem_to_page() to convert the netmem back to pages to pass to mm APIs, Follow up patches to this series add non-paged netmem support to the page_pool. This change is factored out on its own to limit the code churn to this 1 patch, for ease of code review. Signed-off-by: Mina Almasry <almasrymina@google.com> --- v6: - Rebased on top of the merged netmem_ref type. --- include/linux/skbuff.h | 4 +- include/net/netmem.h | 15 ++ include/net/page_pool/helpers.h | 122 +++++++++---- include/net/page_pool/types.h | 17 +- include/trace/events/page_pool.h | 29 +-- net/bpf/test_run.c | 5 +- net/core/page_pool.c | 303 +++++++++++++++++-------------- net/core/skbuff.c | 7 +- 8 files changed, 302 insertions(+), 200 deletions(-)