From patchwork Wed Feb 5 03:12:10 2025 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Jakub Kicinski X-Patchwork-Id: 13960488 X-Patchwork-Delegate: kuba@kernel.org Received: from smtp.kernel.org (aws-us-west-2-korg-mail-1.web.codeaurora.org [10.30.226.201]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by smtp.subspace.kernel.org (Postfix) with ESMTPS id 82F3B212FB9 for ; Wed, 5 Feb 2025 03:12:46 +0000 (UTC) Authentication-Results: smtp.subspace.kernel.org; arc=none smtp.client-ip=10.30.226.201 ARC-Seal: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1738725166; cv=none; b=QH7UgK2Ru6NNw8eLOqVwnGAue5+6oxpFbEnopfgnDKECBfZ9QC8zOt1U551aTLkVpd6mAIiybZW/F7KAPuxxzwpj/2FLOfAMSz3xyrIlIQyUmBvn0bGbF7SnMjDsU3B6mDKdwEhGBsHCrnnWvJ5wGwTfbmpVGXUcAqq2Kigpx8Y= ARC-Message-Signature: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1738725166; c=relaxed/simple; bh=21PdyqdVZaQwS6A74Dvn9/fchNwQCpV6x0EdY8s6XGM=; h=From:To:Cc:Subject:Date:Message-ID:In-Reply-To:References: MIME-Version; b=pJFsTbZZLTyITBe+TN79RMP/1fIdsMdQKxQu/c+1IYlVvSg2lxwiqK7J5M/zrqlGhdl0fJhd/LlJR7RKNdDO+JG2w5eeN0hKHQxoPoDSUPWH9LrYdfcSAE6bzJYx8hQKPPuBuIRsvPKL9lHABUKNfEZYQmEV2mJfNlo1J+keD3Q= ARC-Authentication-Results: i=1; smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b=udq+gqUO; arc=none smtp.client-ip=10.30.226.201 Authentication-Results: smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b="udq+gqUO" Received: by smtp.kernel.org (Postfix) with ESMTPSA id 828B4C4CEE8; Wed, 5 Feb 2025 03:12:45 +0000 (UTC) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/simple; d=kernel.org; s=k20201202; t=1738725166; bh=21PdyqdVZaQwS6A74Dvn9/fchNwQCpV6x0EdY8s6XGM=; h=From:To:Cc:Subject:Date:In-Reply-To:References:From; b=udq+gqUOkt+oJEF9TL4XjHCuWTmc1pV1Cx4Zy5Y8x+wmzZn6EgTy2ofEewsdFYjO4 pi5U4049EzS9dcbD+y79PBuimRPIchbSmJ5/0lIiv4ZPbkugh5+5sw2kHYQaGlaKZm uyi3MiTsvo9mAsrcsclmWyYk7Sfd/BoUAGmTwtQU3mI28Ivx1svf8GS0PIpOs0RS+e tfEvNE5vDvGhvjrU/lZ9kzMxgWkybAK0OOLtEh/iHrzrQ5L0/l4T5Kn4e2Kg5tcYG1 MyEvrP2Wa6G1I09HEQrgD/hS5V09c+cL9oYxh0003OwUlQS3TOGf9VkrZjNozcQwGc 5swjX6lTcfwdA== From: Jakub Kicinski To: davem@davemloft.net Cc: netdev@vger.kernel.org, edumazet@google.com, pabeni@redhat.com, andrew+netdev@lunn.ch, horms@kernel.org, tariqt@nvidia.com, hawk@kernel.org, Jakub Kicinski Subject: [PATCH net-next 1/4] eth: mlx4: create a page pool for Rx Date: Tue, 4 Feb 2025 19:12:10 -0800 Message-ID: <20250205031213.358973-2-kuba@kernel.org> X-Mailer: git-send-email 2.48.1 In-Reply-To: <20250205031213.358973-1-kuba@kernel.org> References: <20250205031213.358973-1-kuba@kernel.org> Precedence: bulk X-Mailing-List: netdev@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: MIME-Version: 1.0 X-Patchwork-Delegate: kuba@kernel.org Create a pool per rx queue. Subsequent patches will make use of it. Move fcs_del to a hole to make space for the pointer. Signed-off-by: Jakub Kicinski --- drivers/net/ethernet/mellanox/mlx4/mlx4_en.h | 3 ++- drivers/net/ethernet/mellanox/mlx4/en_rx.c | 24 +++++++++++++++++++- 2 files changed, 25 insertions(+), 2 deletions(-) diff --git a/drivers/net/ethernet/mellanox/mlx4/mlx4_en.h b/drivers/net/ethernet/mellanox/mlx4/mlx4_en.h index 28b70dcc652e..29f48e63081b 100644 --- a/drivers/net/ethernet/mellanox/mlx4/mlx4_en.h +++ b/drivers/net/ethernet/mellanox/mlx4/mlx4_en.h @@ -335,10 +335,11 @@ struct mlx4_en_rx_ring { u16 stride; u16 log_stride; u16 cqn; /* index of port CQ associated with this ring */ + u8 fcs_del; u32 prod; u32 cons; u32 buf_size; - u8 fcs_del; + struct page_pool *pp; void *buf; void *rx_info; struct bpf_prog __rcu *xdp_prog; diff --git a/drivers/net/ethernet/mellanox/mlx4/en_rx.c b/drivers/net/ethernet/mellanox/mlx4/en_rx.c index 15c57e9517e9..2c23d75baf14 100644 --- a/drivers/net/ethernet/mellanox/mlx4/en_rx.c +++ b/drivers/net/ethernet/mellanox/mlx4/en_rx.c @@ -48,6 +48,7 @@ #if IS_ENABLED(CONFIG_IPV6) #include #endif +#include #include "mlx4_en.h" @@ -268,6 +269,7 @@ int mlx4_en_create_rx_ring(struct mlx4_en_priv *priv, u32 size, u16 stride, int node, int queue_index) { struct mlx4_en_dev *mdev = priv->mdev; + struct page_pool_params pp = {}; struct mlx4_en_rx_ring *ring; int err = -ENOMEM; int tmp; @@ -286,9 +288,26 @@ int mlx4_en_create_rx_ring(struct mlx4_en_priv *priv, ring->log_stride = ffs(ring->stride) - 1; ring->buf_size = ring->size * ring->stride + TXBB_SIZE; - if (xdp_rxq_info_reg(&ring->xdp_rxq, priv->dev, queue_index, 0) < 0) + pp.flags = PP_FLAG_DMA_MAP; + pp.pool_size = MLX4_EN_MAX_RX_SIZE; + pp.nid = node; + pp.napi = &priv->rx_cq[queue_index]->napi; + pp.netdev = priv->dev; + pp.dev = &mdev->dev->persist->pdev->dev; + pp.dma_dir = DMA_BIDIRECTIONAL; + + ring->pp = page_pool_create(&pp); + if (!ring->pp) goto err_ring; + if (xdp_rxq_info_reg(&ring->xdp_rxq, priv->dev, queue_index, 0) < 0) + goto err_pp; + + err = xdp_rxq_info_reg_mem_model(&ring->xdp_rxq, MEM_TYPE_PAGE_POOL, + ring->pp); + if (err) + goto err_xdp_info; + tmp = size * roundup_pow_of_two(MLX4_EN_MAX_RX_FRAGS * sizeof(struct mlx4_en_rx_alloc)); ring->rx_info = kvzalloc_node(tmp, GFP_KERNEL, node); @@ -319,6 +338,8 @@ int mlx4_en_create_rx_ring(struct mlx4_en_priv *priv, ring->rx_info = NULL; err_xdp_info: xdp_rxq_info_unreg(&ring->xdp_rxq); +err_pp: + page_pool_destroy(ring->pp); err_ring: kfree(ring); *pring = NULL; @@ -445,6 +466,7 @@ void mlx4_en_destroy_rx_ring(struct mlx4_en_priv *priv, xdp_rxq_info_unreg(&ring->xdp_rxq); mlx4_free_hwq_res(mdev->dev, &ring->wqres, size * stride + TXBB_SIZE); kvfree(ring->rx_info); + page_pool_destroy(ring->pp); ring->rx_info = NULL; kfree(ring); *pring = NULL; From patchwork Wed Feb 5 03:12:11 2025 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Jakub Kicinski X-Patchwork-Id: 13960489 X-Patchwork-Delegate: kuba@kernel.org Received: from smtp.kernel.org (aws-us-west-2-korg-mail-1.web.codeaurora.org [10.30.226.201]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by smtp.subspace.kernel.org (Postfix) with ESMTPS id 52679213258 for ; Wed, 5 Feb 2025 03:12:46 +0000 (UTC) Authentication-Results: smtp.subspace.kernel.org; arc=none smtp.client-ip=10.30.226.201 ARC-Seal: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1738725167; cv=none; b=kb4/hmNcmMV3q9ViDXv2ridnqHzWoLK3AARXJCxVdx9jiAfRavltcrCpmN0M1++Oa9XddVFoVvwDs4h50DiUF/5U2c5pKpd5bs1oJgpltMPQLUfBpzrPg8Dt+XQBXD8NIbTK1Tc/A89sTyHY+lD3N3OnzN0QXcnxKT/1cPfHFJs= ARC-Message-Signature: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1738725167; c=relaxed/simple; bh=R/h0DxOX/MuT9LoObDZtK+tS6CyeQglfavPnl9opZNg=; h=From:To:Cc:Subject:Date:Message-ID:In-Reply-To:References: MIME-Version; b=JVwMXy0WOYU4kN2yNCLVN5rh+k/AWsFlKNo++VRp2+ktLRDp48H0PRS7XeOgEAc/7D8QvpEfIfE7O99BuM5LokZJM/r/QUIywvgJvSOEx1Xwz0nw1Gw3yim0qzHMb11n07cw4nIL6eFcMAvJEhkY7RpOnkA+RQM6tLGVAqzmmPk= ARC-Authentication-Results: i=1; smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b=lArvJU9q; arc=none smtp.client-ip=10.30.226.201 Authentication-Results: smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b="lArvJU9q" Received: by smtp.kernel.org (Postfix) with ESMTPSA id 35F1FC4CEE4; Wed, 5 Feb 2025 03:12:46 +0000 (UTC) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/simple; d=kernel.org; s=k20201202; t=1738725166; bh=R/h0DxOX/MuT9LoObDZtK+tS6CyeQglfavPnl9opZNg=; h=From:To:Cc:Subject:Date:In-Reply-To:References:From; b=lArvJU9qp8K01sAkEVD4O5BLQD5XJ61DTFzGSQDBCFwZrcsxusuffSufdwahSbAFw 7VMRLI+OWiIZiCplVttMILtA59znACXCGLCfe1HfmfIAyKYxcNEKypFSEKKJTIqJ8e oenUfDQ8KN1laFORRFlYuUnrLqIi2QKMKALz/XJFS3mHr+NJvYEZLbsWYCz8iayiAo 3hSk2jAq2Ayebd8VJQF14D29Hj1rdWtXJALsIhasrWZyidQV7p4lqdtl7RNY4JeQpX 74ydlsKwF6LWXjeOJaJtCT9N7kq9ctgk+m5g5xTLhbfqKvA3V14ZUA5ruqS0mYtu5c LVqWY1ZFocmhw== From: Jakub Kicinski To: davem@davemloft.net Cc: netdev@vger.kernel.org, edumazet@google.com, pabeni@redhat.com, andrew+netdev@lunn.ch, horms@kernel.org, tariqt@nvidia.com, hawk@kernel.org, Jakub Kicinski Subject: [PATCH net-next 2/4] eth: mlx4: don't try to complete XDP frames in netpoll Date: Tue, 4 Feb 2025 19:12:11 -0800 Message-ID: <20250205031213.358973-3-kuba@kernel.org> X-Mailer: git-send-email 2.48.1 In-Reply-To: <20250205031213.358973-1-kuba@kernel.org> References: <20250205031213.358973-1-kuba@kernel.org> Precedence: bulk X-Mailing-List: netdev@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: MIME-Version: 1.0 X-Patchwork-Delegate: kuba@kernel.org mlx4 doesn't support xdp xmit and wasn't using page pool until now, so it could run XDP completions in netpoll (NAPI budget == 0) just fine. Page pool has calling context requirements, make sure we don't try to call it from what is potentially HW IRQ context. Signed-off-by: Jakub Kicinski --- drivers/net/ethernet/mellanox/mlx4/en_tx.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/drivers/net/ethernet/mellanox/mlx4/en_tx.c b/drivers/net/ethernet/mellanox/mlx4/en_tx.c index 1ddb11cb25f9..6e077d202827 100644 --- a/drivers/net/ethernet/mellanox/mlx4/en_tx.c +++ b/drivers/net/ethernet/mellanox/mlx4/en_tx.c @@ -450,6 +450,8 @@ int mlx4_en_process_tx_cq(struct net_device *dev, if (unlikely(!priv->port_up)) return 0; + if (unlikely(!napi_budget) && cq->type == TX_XDP) + return 0; netdev_txq_bql_complete_prefetchw(ring->tx_queue); From patchwork Wed Feb 5 03:12:12 2025 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Jakub Kicinski X-Patchwork-Id: 13960490 X-Patchwork-Delegate: kuba@kernel.org Received: from smtp.kernel.org (aws-us-west-2-korg-mail-1.web.codeaurora.org [10.30.226.201]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by smtp.subspace.kernel.org (Postfix) with ESMTPS id A8E102135AF for ; Wed, 5 Feb 2025 03:12:47 +0000 (UTC) Authentication-Results: smtp.subspace.kernel.org; arc=none smtp.client-ip=10.30.226.201 ARC-Seal: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1738725167; cv=none; b=s08YcMwxGgq7SjefCoGA3NGm25XC5GU4qVm2MlihIx551d6u2ZaBLM4ETiu/whbHEO8S/0tU40ylcw3bQyk4WjhsDAKx57nDcMZiX3hQygiUz5FthSOIJf+EALYKUPL1f4YLPFq9tRMImIjxVuiZ3zgWQzkgPlOXO/Dm/MMPrcM= ARC-Message-Signature: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1738725167; c=relaxed/simple; bh=5QiXuyyyXDix7sFVat21/KxZlz14J27l6B75OFPh0Ls=; h=From:To:Cc:Subject:Date:Message-ID:In-Reply-To:References: MIME-Version; b=Bwf6bgHQbgUVZt+rfUlJlaX2cYJcI+xsMursayhDQQETWB12Rzw8FppkZO4cQncD10yTLlspBApe4WolJ/n8ndgDNd8guUzHku4CFTMidaGauL/J/wISlGdm0AiY0guf9OuxqaIKZH1r/PfBJkEh16xQyQ0YD3uhQVIuyCG9Rc8= ARC-Authentication-Results: i=1; smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b=Zjuw1ZpQ; arc=none smtp.client-ip=10.30.226.201 Authentication-Results: smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b="Zjuw1ZpQ" Received: by smtp.kernel.org (Postfix) with ESMTPSA id E1E34C4CEE7; Wed, 5 Feb 2025 03:12:46 +0000 (UTC) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/simple; d=kernel.org; s=k20201202; t=1738725167; bh=5QiXuyyyXDix7sFVat21/KxZlz14J27l6B75OFPh0Ls=; h=From:To:Cc:Subject:Date:In-Reply-To:References:From; b=Zjuw1ZpQVyQChWjVAQnRsO6fblvCRp6D8E8Knqd0XxYwysQgdcybtVWBvdYWq+q7D 2gq/0DZyHeyRBn0zI/Ox2Dojju/enZo1JuEy/QaNni8DFewaEJVBpGKZwRG7C7VsY+ sjilEyKErClfMw/8rSPJ+ylZCWir9vbPt76pHxtIBfX9hChNlzsQ9pVef/5cWaBos9 CY0+CKYYdYy9xbJ3B/JXJMnsM7X4YXJ3daKyBkQtT502PByLVuCM9ug9bMVFD2GwwH 9tdoSaUw9hXQ3svXa4zLD0p4ePt/SzD7XK9Z2TSXFtZ+kZF//qKRRXDyDzP65f1IU6 fAH0xXsGEFQkw== From: Jakub Kicinski To: davem@davemloft.net Cc: netdev@vger.kernel.org, edumazet@google.com, pabeni@redhat.com, andrew+netdev@lunn.ch, horms@kernel.org, tariqt@nvidia.com, hawk@kernel.org, Jakub Kicinski Subject: [PATCH net-next 3/4] eth: mlx4: remove the local XDP fast-recycling ring Date: Tue, 4 Feb 2025 19:12:12 -0800 Message-ID: <20250205031213.358973-4-kuba@kernel.org> X-Mailer: git-send-email 2.48.1 In-Reply-To: <20250205031213.358973-1-kuba@kernel.org> References: <20250205031213.358973-1-kuba@kernel.org> Precedence: bulk X-Mailing-List: netdev@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: MIME-Version: 1.0 X-Patchwork-Delegate: kuba@kernel.org It will be replaced with page pool's built-in recycling. Signed-off-by: Jakub Kicinski --- drivers/net/ethernet/mellanox/mlx4/mlx4_en.h | 11 ------ drivers/net/ethernet/mellanox/mlx4/en_rx.c | 40 -------------------- drivers/net/ethernet/mellanox/mlx4/en_tx.c | 11 +----- 3 files changed, 2 insertions(+), 60 deletions(-) diff --git a/drivers/net/ethernet/mellanox/mlx4/mlx4_en.h b/drivers/net/ethernet/mellanox/mlx4/mlx4_en.h index 29f48e63081b..97311c98569f 100644 --- a/drivers/net/ethernet/mellanox/mlx4/mlx4_en.h +++ b/drivers/net/ethernet/mellanox/mlx4/mlx4_en.h @@ -253,14 +253,6 @@ struct mlx4_en_rx_alloc { #define MLX4_EN_CACHE_SIZE (2 * NAPI_POLL_WEIGHT) -struct mlx4_en_page_cache { - u32 index; - struct { - struct page *page; - dma_addr_t dma; - } buf[MLX4_EN_CACHE_SIZE]; -}; - enum { MLX4_EN_TX_RING_STATE_RECOVERING, }; @@ -343,7 +335,6 @@ struct mlx4_en_rx_ring { void *buf; void *rx_info; struct bpf_prog __rcu *xdp_prog; - struct mlx4_en_page_cache page_cache; unsigned long bytes; unsigned long packets; unsigned long csum_ok; @@ -708,8 +699,6 @@ netdev_tx_t mlx4_en_xmit_frame(struct mlx4_en_rx_ring *rx_ring, struct mlx4_en_priv *priv, unsigned int length, int tx_ind, bool *doorbell_pending); void mlx4_en_xmit_doorbell(struct mlx4_en_tx_ring *ring); -bool mlx4_en_rx_recycle(struct mlx4_en_rx_ring *ring, - struct mlx4_en_rx_alloc *frame); int mlx4_en_create_tx_ring(struct mlx4_en_priv *priv, struct mlx4_en_tx_ring **pring, diff --git a/drivers/net/ethernet/mellanox/mlx4/en_rx.c b/drivers/net/ethernet/mellanox/mlx4/en_rx.c index 2c23d75baf14..9de5449667bb 100644 --- a/drivers/net/ethernet/mellanox/mlx4/en_rx.c +++ b/drivers/net/ethernet/mellanox/mlx4/en_rx.c @@ -142,18 +142,6 @@ static int mlx4_en_prepare_rx_desc(struct mlx4_en_priv *priv, (index << ring->log_stride); struct mlx4_en_rx_alloc *frags = ring->rx_info + (index << priv->log_rx_info); - if (likely(ring->page_cache.index > 0)) { - /* XDP uses a single page per frame */ - if (!frags->page) { - ring->page_cache.index--; - frags->page = ring->page_cache.buf[ring->page_cache.index].page; - frags->dma = ring->page_cache.buf[ring->page_cache.index].dma; - } - frags->page_offset = XDP_PACKET_HEADROOM; - rx_desc->data[0].addr = cpu_to_be64(frags->dma + - XDP_PACKET_HEADROOM); - return 0; - } return mlx4_en_alloc_frags(priv, ring, rx_desc, frags, gfp); } @@ -430,26 +418,6 @@ void mlx4_en_recover_from_oom(struct mlx4_en_priv *priv) } } -/* When the rx ring is running in page-per-packet mode, a released frame can go - * directly into a small cache, to avoid unmapping or touching the page - * allocator. In bpf prog performance scenarios, buffers are either forwarded - * or dropped, never converted to skbs, so every page can come directly from - * this cache when it is sized to be a multiple of the napi budget. - */ -bool mlx4_en_rx_recycle(struct mlx4_en_rx_ring *ring, - struct mlx4_en_rx_alloc *frame) -{ - struct mlx4_en_page_cache *cache = &ring->page_cache; - - if (cache->index >= MLX4_EN_CACHE_SIZE) - return false; - - cache->buf[cache->index].page = frame->page; - cache->buf[cache->index].dma = frame->dma; - cache->index++; - return true; -} - void mlx4_en_destroy_rx_ring(struct mlx4_en_priv *priv, struct mlx4_en_rx_ring **pring, u32 size, u16 stride) @@ -475,14 +443,6 @@ void mlx4_en_destroy_rx_ring(struct mlx4_en_priv *priv, void mlx4_en_deactivate_rx_ring(struct mlx4_en_priv *priv, struct mlx4_en_rx_ring *ring) { - int i; - - for (i = 0; i < ring->page_cache.index; i++) { - dma_unmap_page(priv->ddev, ring->page_cache.buf[i].dma, - PAGE_SIZE, priv->dma_dir); - put_page(ring->page_cache.buf[i].page); - } - ring->page_cache.index = 0; mlx4_en_free_rx_buf(priv, ring); if (ring->stride <= TXBB_SIZE) ring->buf -= TXBB_SIZE; diff --git a/drivers/net/ethernet/mellanox/mlx4/en_tx.c b/drivers/net/ethernet/mellanox/mlx4/en_tx.c index 6e077d202827..fe1378a689a1 100644 --- a/drivers/net/ethernet/mellanox/mlx4/en_tx.c +++ b/drivers/net/ethernet/mellanox/mlx4/en_tx.c @@ -350,16 +350,9 @@ u32 mlx4_en_recycle_tx_desc(struct mlx4_en_priv *priv, int napi_mode) { struct mlx4_en_tx_info *tx_info = &ring->tx_info[index]; - struct mlx4_en_rx_alloc frame = { - .page = tx_info->page, - .dma = tx_info->map0_dma, - }; - if (!napi_mode || !mlx4_en_rx_recycle(ring->recycle_ring, &frame)) { - dma_unmap_page(priv->ddev, tx_info->map0_dma, - PAGE_SIZE, priv->dma_dir); - put_page(tx_info->page); - } + dma_unmap_page(priv->ddev, tx_info->map0_dma, PAGE_SIZE, priv->dma_dir); + put_page(tx_info->page); return tx_info->nr_txbb; } From patchwork Wed Feb 5 03:12:13 2025 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Jakub Kicinski X-Patchwork-Id: 13960491 X-Patchwork-Delegate: kuba@kernel.org Received: from smtp.kernel.org (aws-us-west-2-korg-mail-1.web.codeaurora.org [10.30.226.201]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by smtp.subspace.kernel.org (Postfix) with ESMTPS id B53C9213E9F for ; Wed, 5 Feb 2025 03:12:48 +0000 (UTC) Authentication-Results: smtp.subspace.kernel.org; arc=none smtp.client-ip=10.30.226.201 ARC-Seal: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1738725168; cv=none; b=I9XDtFAUrRiMfad1T/XVJTvh4iwJAOjSb3DPFoKBuLCFM0rOkAt4YU6KURZ6t3SUOE8jwcA7v9WQIZrC5jfI3Lb7/UU5qicRFTIf5N/2XXQrUJlpw05SvpeGBtJRFwY3RcYSuzO2GggRIfgDWIL2TXkzc4+gakFShzP9bQDVjI4= ARC-Message-Signature: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1738725168; c=relaxed/simple; bh=iQflgnpPvfysSR4BJjjx5SjHiZ33lOkxq2QKi0oja+Y=; h=From:To:Cc:Subject:Date:Message-ID:In-Reply-To:References: MIME-Version; b=RYgQW2JFqDFkHHe9udl/kIezGz53Oi7QgFW4cu9NCNDD42y1qNIHZQmSFgJLLhivpozEJgZTHabOVXTBecmGbW3E+fPAcSAlBrrLTeyJ9e1wne7Aj9ibjprKELGdXiB1xOF4HHipEwhq5ctnb6Hch5QQ7NABejufoBjOTH7ibO0= ARC-Authentication-Results: i=1; smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b=GADOvLe+; arc=none smtp.client-ip=10.30.226.201 Authentication-Results: smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b="GADOvLe+" Received: by smtp.kernel.org (Postfix) with ESMTPSA id A36C1C4CEDF; Wed, 5 Feb 2025 03:12:47 +0000 (UTC) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/simple; d=kernel.org; s=k20201202; t=1738725168; bh=iQflgnpPvfysSR4BJjjx5SjHiZ33lOkxq2QKi0oja+Y=; h=From:To:Cc:Subject:Date:In-Reply-To:References:From; b=GADOvLe+Zhr8LmCENN4GbSciHnTg4pgb169HL19BgtpjkWHuRzqjMSkcUEZsUG7fb COERN34n57PqhFnjzeXn1UcQE1CC7fIcNWD+ZbkjcSRl1gpJHtGDrsmMILJ3bowNee /CPnuy17kLzVB02WOUWwMrE+qRrvclFaJdjQPct9kFpzJIBxGC9DCucGcPaQH94eVd eC7yJGgcXAzeCjq5kMHvX5jUsS+0I6j1FP52GfkR3BH8auZQL26g6M/caIIiln1I/t X/vCWzkKlhMzsGgIa4cJEZkEPu6C23Qs8zUtg/NAoGZqjKINfQUb8484NvWkssvpux 3pPJepgCZSGVw== From: Jakub Kicinski To: davem@davemloft.net Cc: netdev@vger.kernel.org, edumazet@google.com, pabeni@redhat.com, andrew+netdev@lunn.ch, horms@kernel.org, tariqt@nvidia.com, hawk@kernel.org, Jakub Kicinski Subject: [PATCH net-next 4/4] eth: mlx4: use the page pool for Rx buffers Date: Tue, 4 Feb 2025 19:12:13 -0800 Message-ID: <20250205031213.358973-5-kuba@kernel.org> X-Mailer: git-send-email 2.48.1 In-Reply-To: <20250205031213.358973-1-kuba@kernel.org> References: <20250205031213.358973-1-kuba@kernel.org> Precedence: bulk X-Mailing-List: netdev@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: MIME-Version: 1.0 X-Patchwork-Delegate: kuba@kernel.org Simple conversion to page pool. Preserve the current fragmentation logic / page splitting. Each page starts with a single frag reference, and then we bump that when attaching to skbs. This can likely be optimized further. Signed-off-by: Jakub Kicinski --- drivers/net/ethernet/mellanox/mlx4/mlx4_en.h | 1 - drivers/net/ethernet/mellanox/mlx4/en_rx.c | 56 ++++++++------------ drivers/net/ethernet/mellanox/mlx4/en_tx.c | 8 +-- 3 files changed, 26 insertions(+), 39 deletions(-) diff --git a/drivers/net/ethernet/mellanox/mlx4/mlx4_en.h b/drivers/net/ethernet/mellanox/mlx4/mlx4_en.h index 97311c98569f..ad0d91a75184 100644 --- a/drivers/net/ethernet/mellanox/mlx4/mlx4_en.h +++ b/drivers/net/ethernet/mellanox/mlx4/mlx4_en.h @@ -247,7 +247,6 @@ struct mlx4_en_tx_desc { struct mlx4_en_rx_alloc { struct page *page; - dma_addr_t dma; u32 page_offset; }; diff --git a/drivers/net/ethernet/mellanox/mlx4/en_rx.c b/drivers/net/ethernet/mellanox/mlx4/en_rx.c index 9de5449667bb..0e74d9c75c71 100644 --- a/drivers/net/ethernet/mellanox/mlx4/en_rx.c +++ b/drivers/net/ethernet/mellanox/mlx4/en_rx.c @@ -52,57 +52,39 @@ #include "mlx4_en.h" -static int mlx4_alloc_page(struct mlx4_en_priv *priv, - struct mlx4_en_rx_alloc *frag, - gfp_t gfp) -{ - struct page *page; - dma_addr_t dma; - - page = alloc_page(gfp); - if (unlikely(!page)) - return -ENOMEM; - dma = dma_map_page(priv->ddev, page, 0, PAGE_SIZE, priv->dma_dir); - if (unlikely(dma_mapping_error(priv->ddev, dma))) { - __free_page(page); - return -ENOMEM; - } - frag->page = page; - frag->dma = dma; - frag->page_offset = priv->rx_headroom; - return 0; -} - static int mlx4_en_alloc_frags(struct mlx4_en_priv *priv, struct mlx4_en_rx_ring *ring, struct mlx4_en_rx_desc *rx_desc, struct mlx4_en_rx_alloc *frags, gfp_t gfp) { + dma_addr_t dma; int i; for (i = 0; i < priv->num_frags; i++, frags++) { if (!frags->page) { - if (mlx4_alloc_page(priv, frags, gfp)) { + frags->page = page_pool_alloc_pages(ring->pp, gfp); + if (!frags->page) { ring->alloc_fail++; return -ENOMEM; } + page_pool_fragment_page(frags->page, 1); + frags->page_offset = priv->rx_headroom; + ring->rx_alloc_pages++; } - rx_desc->data[i].addr = cpu_to_be64(frags->dma + - frags->page_offset); + dma = page_pool_get_dma_addr(frags->page); + rx_desc->data[i].addr = cpu_to_be64(dma + frags->page_offset); } return 0; } static void mlx4_en_free_frag(const struct mlx4_en_priv *priv, + struct mlx4_en_rx_ring *ring, struct mlx4_en_rx_alloc *frag) { - if (frag->page) { - dma_unmap_page(priv->ddev, frag->dma, - PAGE_SIZE, priv->dma_dir); - __free_page(frag->page); - } + if (frag->page) + page_pool_put_full_page(ring->pp, frag->page, false); /* We need to clear all fields, otherwise a change of priv->log_rx_info * could lead to see garbage later in frag->page. */ @@ -167,7 +149,7 @@ static void mlx4_en_free_rx_desc(const struct mlx4_en_priv *priv, frags = ring->rx_info + (index << priv->log_rx_info); for (nr = 0; nr < priv->num_frags; nr++) { en_dbg(DRV, priv, "Freeing fragment:%d\n", nr); - mlx4_en_free_frag(priv, frags + nr); + mlx4_en_free_frag(priv, ring, frags + nr); } } @@ -283,6 +265,7 @@ int mlx4_en_create_rx_ring(struct mlx4_en_priv *priv, pp.netdev = priv->dev; pp.dev = &mdev->dev->persist->pdev->dev; pp.dma_dir = DMA_BIDIRECTIONAL; + pp.max_len = PAGE_SIZE; ring->pp = page_pool_create(&pp); if (!ring->pp) @@ -469,7 +452,7 @@ static int mlx4_en_complete_rx_desc(struct mlx4_en_priv *priv, if (unlikely(!page)) goto fail; - dma = frags->dma; + dma = page_pool_get_dma_addr(page); dma_sync_single_range_for_cpu(priv->ddev, dma, frags->page_offset, frag_size, priv->dma_dir); @@ -480,6 +463,7 @@ static int mlx4_en_complete_rx_desc(struct mlx4_en_priv *priv, if (frag_info->frag_stride == PAGE_SIZE / 2) { frags->page_offset ^= PAGE_SIZE / 2; release = page_count(page) != 1 || + atomic_long_read(&page->pp_ref_count) != 1 || page_is_pfmemalloc(page) || page_to_nid(page) != numa_mem_id(); } else if (!priv->rx_headroom) { @@ -493,10 +477,9 @@ static int mlx4_en_complete_rx_desc(struct mlx4_en_priv *priv, release = frags->page_offset + frag_info->frag_size > PAGE_SIZE; } if (release) { - dma_unmap_page(priv->ddev, dma, PAGE_SIZE, priv->dma_dir); frags->page = NULL; } else { - page_ref_inc(page); + page_pool_ref_page(page); } nr++; @@ -766,7 +749,8 @@ int mlx4_en_process_rx_cq(struct net_device *dev, struct mlx4_en_cq *cq, int bud /* Get pointer to first fragment since we haven't * skb yet and cast it to ethhdr struct */ - dma = frags[0].dma + frags[0].page_offset; + dma = page_pool_get_dma_addr(frags[0].page); + dma += frags[0].page_offset; dma_sync_single_for_cpu(priv->ddev, dma, sizeof(*ethh), DMA_FROM_DEVICE); @@ -805,7 +789,8 @@ int mlx4_en_process_rx_cq(struct net_device *dev, struct mlx4_en_cq *cq, int bud void *orig_data; u32 act; - dma = frags[0].dma + frags[0].page_offset; + dma = page_pool_get_dma_addr(frags[0].page); + dma += frags[0].page_offset; dma_sync_single_for_cpu(priv->ddev, dma, priv->frag_info[0].frag_size, DMA_FROM_DEVICE); @@ -868,6 +853,7 @@ int mlx4_en_process_rx_cq(struct net_device *dev, struct mlx4_en_cq *cq, int bud skb = napi_get_frags(&cq->napi); if (unlikely(!skb)) goto next; + skb_mark_for_recycle(skb); if (unlikely(ring->hwtstamp_rx_filter == HWTSTAMP_FILTER_ALL)) { u64 timestamp = mlx4_en_get_cqe_ts(cqe); diff --git a/drivers/net/ethernet/mellanox/mlx4/en_tx.c b/drivers/net/ethernet/mellanox/mlx4/en_tx.c index fe1378a689a1..87f35bcbeff8 100644 --- a/drivers/net/ethernet/mellanox/mlx4/en_tx.c +++ b/drivers/net/ethernet/mellanox/mlx4/en_tx.c @@ -44,6 +44,7 @@ #include #include #include +#include #include "mlx4_en.h" @@ -350,9 +351,10 @@ u32 mlx4_en_recycle_tx_desc(struct mlx4_en_priv *priv, int napi_mode) { struct mlx4_en_tx_info *tx_info = &ring->tx_info[index]; + struct page_pool *pool = ring->recycle_ring->pp; - dma_unmap_page(priv->ddev, tx_info->map0_dma, PAGE_SIZE, priv->dma_dir); - put_page(tx_info->page); + /* Note that napi_mode = 0 means ndo_close() path, not budget = 0 */ + page_pool_put_full_page(pool, tx_info->page, !!napi_mode); return tx_info->nr_txbb; } @@ -1189,7 +1191,7 @@ netdev_tx_t mlx4_en_xmit_frame(struct mlx4_en_rx_ring *rx_ring, tx_desc = ring->buf + (index << LOG_TXBB_SIZE); data = &tx_desc->data; - dma = frame->dma; + dma = page_pool_get_dma_addr(frame->page); tx_info->page = frame->page; frame->page = NULL;