@@ -51,6 +51,11 @@
struct pp_alloc_cache {
u32 count;
netmem_ref cache[PP_ALLOC_CACHE_SIZE];
+
+ /* Keep batched refilled pages here to avoid doing the atomic operation
+ * for each page.
+ */
+ struct page_pool_item *refill;
};
/**
@@ -681,11 +681,13 @@ static void __page_pool_return_page(struct page_pool *pool, netmem_ref netmem,
static noinline netmem_ref page_pool_refill_alloc_cache(struct page_pool *pool)
{
+ struct page_pool_item *refill;
netmem_ref netmem;
int pref_nid; /* preferred NUMA node */
/* Quicker fallback, avoid locks when ring is empty */
- if (unlikely(!READ_ONCE(pool->ring.list))) {
+ refill = pool->alloc.refill;
+ if (unlikely(!refill && !READ_ONCE(pool->ring.list))) {
alloc_stat_inc(pool, empty);
return 0;
}
@@ -702,10 +704,14 @@ static noinline netmem_ref page_pool_refill_alloc_cache(struct page_pool *pool)
/* Refill alloc array, but only if NUMA match */
do {
- netmem = page_pool_consume_ring(pool);
- if (unlikely(!netmem))
- break;
+ if (unlikely(!refill)) {
+ refill = xchg(&pool->ring.list, NULL);
+ if (!refill)
+ break;
+ }
+ netmem = refill->pp_netmem;
+ refill = page_pool_item_get_next(refill);
if (likely(netmem_is_pref_nid(netmem, pref_nid))) {
pool->alloc.cache[pool->alloc.count++] = netmem;
} else {
@@ -715,14 +721,18 @@ static noinline netmem_ref page_pool_refill_alloc_cache(struct page_pool *pool)
* This limit stress on page buddy alloactor.
*/
__page_pool_return_page(pool, netmem, false);
+ atomic_dec(&pool->ring.count);
alloc_stat_inc(pool, waive);
netmem = 0;
break;
}
} while (pool->alloc.count < PP_ALLOC_CACHE_REFILL);
+ pool->alloc.refill = refill;
+
/* Return last page */
if (likely(pool->alloc.count > 0)) {
+ atomic_sub(pool->alloc.count, &pool->ring.count);
netmem = pool->alloc.cache[--pool->alloc.count];
alloc_stat_inc(pool, refill);
}
@@ -1410,6 +1420,7 @@ static void __page_pool_destroy(struct page_pool *pool)
static void page_pool_empty_alloc_cache_once(struct page_pool *pool)
{
+ struct page_pool_item *refill;
netmem_ref netmem;
if (pool->destroy_cnt)
@@ -1423,6 +1434,12 @@ static void page_pool_empty_alloc_cache_once(struct page_pool *pool)
netmem = pool->alloc.cache[--pool->alloc.count];
page_pool_return_page(pool, netmem);
}
+
+ while ((refill = pool->alloc.refill)) {
+ pool->alloc.refill = page_pool_item_get_next(refill);
+ page_pool_return_page(pool, refill->pp_netmem);
+ atomic_dec(&pool->ring.count);
+ }
}
static void page_pool_scrub(struct page_pool *pool)
Add refill variable in alloc cache to keep batched refilled pages to avoid doing the atomic operation for each page. Testing shows there is about 10ns improvement for the performance of 'time_bench_page_pool02_ptr_ring' test case. CC: Robin Murphy <robin.murphy@arm.com> CC: Alexander Duyck <alexander.duyck@gmail.com> CC: IOMMU <iommu@lists.linux.dev> Signed-off-by: Yunsheng Lin <linyunsheng@huawei.com> --- include/net/page_pool/types.h | 5 +++++ net/core/page_pool.c | 25 +++++++++++++++++++++---- 2 files changed, 26 insertions(+), 4 deletions(-)