@@ -278,9 +278,6 @@ static int page_pool_init(struct page_pool *pool,
/* Driver calling page_pool_create() also call page_pool_destroy() */
refcount_set(&pool->user_cnt, 1);
- if (pool->dma_map)
- get_device(pool->p.dev);
-
if (pool->slow.flags & PP_FLAG_ALLOW_UNREADABLE_NETMEM) {
/* We rely on rtnl_lock()ing to make sure netdev_rx_queue
* configuration doesn't change while we're initializing
@@ -318,9 +315,6 @@ static void page_pool_uninit(struct page_pool *pool)
{
ptr_ring_cleanup(&pool->ring, NULL);
- if (pool->dma_map)
- put_device(pool->p.dev);
-
#ifdef CONFIG_PAGE_POOL_STATS
if (!pool->system)
free_percpu(pool->recycle_stats);
@@ -747,6 +741,25 @@ page_pool_dma_sync_for_device(const struct page_pool *pool,
__page_pool_dma_sync_for_device(pool, netmem, dma_sync_size);
}
+static __always_inline void
+page_pool_dma_sync_for_device_rcu(const struct page_pool *pool,
+ netmem_ref netmem,
+ u32 dma_sync_size)
+{
+ if (!pool->dma_sync || !dma_dev_need_sync(pool->p.dev))
+ return;
+
+ rcu_read_lock();
+
+ /* Recheck the dma_sync under rcu lock to pair with rcu sync operation
+ * in page_pool_destroy().
+ */
+ if (pool->dma_sync)
+ __page_pool_dma_sync_for_device(pool, netmem, dma_sync_size);
+
+ rcu_read_unlock();
+}
+
static bool page_pool_dma_map(struct page_pool *pool, netmem_ref netmem)
{
struct page_pool_item *item;
@@ -985,7 +998,8 @@ void page_pool_return_page(struct page_pool *pool, netmem_ref netmem)
*/
}
-static bool page_pool_recycle_in_ring(struct page_pool *pool, netmem_ref netmem)
+static bool page_pool_recycle_in_ring(struct page_pool *pool, netmem_ref netmem,
+ unsigned int dma_sync_size)
{
int ret;
/* BH protection not needed if current is softirq */
@@ -994,12 +1008,12 @@ static bool page_pool_recycle_in_ring(struct page_pool *pool, netmem_ref netmem)
else
ret = ptr_ring_produce_bh(&pool->ring, (__force void *)netmem);
- if (!ret) {
+ if (likely(!ret)) {
+ page_pool_dma_sync_for_device_rcu(pool, netmem, dma_sync_size);
recycle_stat_inc(pool, ring);
- return true;
}
- return false;
+ return !ret;
}
/* Only allow direct recycling in special circumstances, into the
@@ -1052,10 +1066,10 @@ __page_pool_put_page(struct page_pool *pool, netmem_ref netmem,
if (likely(__page_pool_page_can_be_recycled(netmem))) {
/* Read barrier done in page_ref_count / READ_ONCE */
- page_pool_dma_sync_for_device(pool, netmem, dma_sync_size);
-
- if (allow_direct && page_pool_recycle_in_cache(netmem, pool))
+ if (allow_direct && page_pool_recycle_in_cache(netmem, pool)) {
+ page_pool_dma_sync_for_device(pool, netmem, dma_sync_size);
return 0;
+ }
/* Page found as candidate for recycling */
return netmem;
@@ -1116,7 +1130,7 @@ void page_pool_put_unrefed_netmem(struct page_pool *pool, netmem_ref netmem,
netmem =
__page_pool_put_page(pool, netmem, dma_sync_size, allow_direct);
- if (netmem && !page_pool_recycle_in_ring(pool, netmem)) {
+ if (netmem && !page_pool_recycle_in_ring(pool, netmem, dma_sync_size)) {
/* Cache full, fallback to free pages */
recycle_stat_inc(pool, ring_full);
page_pool_return_page(pool, netmem);
@@ -1174,13 +1188,17 @@ void page_pool_put_page_bulk(struct page_pool *pool, void **data,
/* Bulk producer into ptr_ring page_pool cache */
in_softirq = page_pool_producer_lock(pool);
+ rcu_read_lock();
for (i = 0; i < bulk_len; i++) {
if (__ptr_ring_produce(&pool->ring, data[i])) {
/* ring full */
recycle_stat_inc(pool, ring_full);
break;
}
+ page_pool_dma_sync_for_device(pool, (__force netmem_ref)data[i],
+ -1);
}
+ rcu_read_unlock();
recycle_stat_add(pool, ring, i);
page_pool_producer_unlock(pool, in_softirq);
@@ -1421,6 +1439,16 @@ void page_pool_destroy(struct page_pool *pool)
if (!page_pool_release(pool))
return;
+ /* After page_pool_destroy() is called, the page is not expected to be
+ * recycled back to pool->alloc cache and dma sync operation is not
+ * needed when the page is not recyclable or pool->ring is full, skip
+ * the dma sync operation for the infilght pages by clearing the
+ * pool->dma_sync, and the below synchronize_net() is paired with rcu
+ * lock when page is recycled back into ptr_ring to ensure that there is
+ * no dma sync operation called after rcu sync operation.
+ */
+ pool->dma_sync = false;
+
/* Paired with RCU-bh read-side critical section to enable clearing
* of pool->p.napi in page_pool_disable_direct_recycling() is seen
* before returning to driver to free the napi instance.
Skip dma sync operation for inflight pages before the page_pool_destroy() returns to the driver as DMA API expects to be called with a valid device bound to a driver as mentioned in [1]. After page_pool_destroy() is called, the page is not expected to be recycled back to pool->alloc cache and dma sync operation is not needed when the page is not recyclable or pool->ring is full, so only skip the dma sync operation for the infilght pages by clearing the pool->dma_sync, as rcu sync operation in page_pool_destroy() is paired with rcu lock in page_pool_recycle_in_ring() to ensure that there is no dma sync operation called after rcu sync operation. 1. https://lore.kernel.org/all/caf31b5e-0e8f-4844-b7ba-ef59ed13b74e@arm.com/ CC: Robin Murphy <robin.murphy@arm.com> CC: Alexander Duyck <alexander.duyck@gmail.com> CC: IOMMU <iommu@lists.linux.dev> Fixes: f71fec47c2df ("page_pool: make sure struct device is stable") Signed-off-by: Yunsheng Lin <linyunsheng@huawei.com> --- net/core/page_pool.c | 56 +++++++++++++++++++++++++++++++++----------- 1 file changed, 42 insertions(+), 14 deletions(-)