diff mbox series

[net-next,v6,6/6] octeontx2-pf: AF_XDP zero copy transmit support

Message ID 20250213053141.2833254-7-sumang@marvell.com (mailing list archive)
State Accepted
Commit 53616af09b5a55174e470b456461fa45d8abeb35
Delegated to: Netdev Maintainers
Headers show
Series Add af_xdp support for cn10k | expand

Checks

Context Check Description
netdev/series_format success Posting correctly formatted
netdev/tree_selection success Clearly marked for net-next
netdev/ynl success Generated files up to date; no warnings/errors; no diff in generated;
netdev/fixes_present success Fixes tag not required for -next series
netdev/header_inline success No static functions without inline keyword in header files
netdev/build_32bit success Errors and warnings before: 0 this patch: 0
netdev/build_tools success No tools touched, skip
netdev/cc_maintainers success CCed 15 of 15 maintainers
netdev/build_clang success Errors and warnings before: 21 this patch: 21
netdev/verify_signedoff success Signed-off-by tag matches author and committer
netdev/deprecated_api success None detected
netdev/check_selftest success No net selftest shell script
netdev/verify_fixes success No Fixes tag
netdev/build_allmodconfig_warn success Errors and warnings before: 9 this patch: 9
netdev/checkpatch warning WARNING: line length of 83 exceeds 80 columns WARNING: line length of 85 exceeds 80 columns WARNING: line length of 86 exceeds 80 columns WARNING: line length of 99 exceeds 80 columns
netdev/build_clang_rust success No Rust files in patch. Skipping build
netdev/kdoc success Errors and warnings before: 0 this patch: 0
netdev/source_inline success Was 0 now: 0

Commit Message

Suman Ghosh Feb. 13, 2025, 5:31 a.m. UTC 
This patch implements below changes,

1. To avoid concurrency with normal traffic uses
   XDP queues.
2. Since there are chances that XDP and AF_XDP can
   fall under same queue uses separate flags to handle
   dma buffers.

Signed-off-by: Hariprasad Kelam <hkelam@marvell.com>
Signed-off-by: Suman Ghosh <sumang@marvell.com>
---
 .../marvell/octeontx2/nic/otx2_common.c       |  4 ++
 .../marvell/octeontx2/nic/otx2_common.h       |  6 +++
 .../ethernet/marvell/octeontx2/nic/otx2_pf.c  |  2 +-
 .../marvell/octeontx2/nic/otx2_txrx.c         | 49 +++++++++++++++----
 .../marvell/octeontx2/nic/otx2_txrx.h         |  2 +
 .../ethernet/marvell/octeontx2/nic/otx2_xsk.c | 43 +++++++++++++++-
 .../ethernet/marvell/octeontx2/nic/otx2_xsk.h |  3 ++
 7 files changed, 96 insertions(+), 13 deletions(-)

Comments

Paolo Abeni Feb. 18, 2025, 10:26 a.m. UTC | #1 
On 2/13/25 6:31 AM, Suman Ghosh wrote:
> +void otx2_zc_napi_handler(struct otx2_nic *pfvf, struct xsk_buff_pool *pool,
> +			  int queue, int budget)
> +{
> +	struct xdp_desc *xdp_desc = pool->tx_descs;
> +	int err, i, work_done = 0, batch;
> +
> +	budget = min(budget, otx2_read_free_sqe(pfvf, queue));
> +	batch = xsk_tx_peek_release_desc_batch(pool, budget);
> +	if (!batch)
> +		return;
> +
> +	for (i = 0; i < batch; i++) {
> +		dma_addr_t dma_addr;
> +
> +		dma_addr = xsk_buff_raw_get_dma(pool, xdp_desc[i].addr);
> +		err = otx2_xdp_sq_append_pkt(pfvf, NULL, dma_addr, xdp_desc[i].len,
> +					     queue, OTX2_AF_XDP_FRAME);
> +		if (!err) {
> +			netdev_err(pfvf->netdev, "AF_XDP: Unable to transfer packet err%d\n", err);

Here `err` is always 0, dumping it's value is quite confusing.

The root cause is that otx2_xdp_sq_append_pkt() returns a success
boolean value, the variable holding it should possibly be renamed
accordingly.

Since this is the only nit I could find, I think we are better without a
repost, but please follow-up on this chunk soon.

Thanks,

Paolo
Maciej Fijalkowski Feb. 18, 2025, 1:01 p.m. UTC | #2 
On Thu, Feb 13, 2025 at 11:01:41AM +0530, Suman Ghosh wrote:
> This patch implements below changes,
> 
> 1. To avoid concurrency with normal traffic uses
>    XDP queues.
> 2. Since there are chances that XDP and AF_XDP can
>    fall under same queue uses separate flags to handle
>    dma buffers.
> 
> Signed-off-by: Hariprasad Kelam <hkelam@marvell.com>
> Signed-off-by: Suman Ghosh <sumang@marvell.com>
> ---

[...]

> +void otx2_zc_napi_handler(struct otx2_nic *pfvf, struct xsk_buff_pool *pool,
> +			  int queue, int budget)
> +{
> +	struct xdp_desc *xdp_desc = pool->tx_descs;
> +	int err, i, work_done = 0, batch;
> +
> +	budget = min(budget, otx2_read_free_sqe(pfvf, queue));
> +	batch = xsk_tx_peek_release_desc_batch(pool, budget);
> +	if (!batch)
> +		return;
> +
> +	for (i = 0; i < batch; i++) {
> +		dma_addr_t dma_addr;
> +
> +		dma_addr = xsk_buff_raw_get_dma(pool, xdp_desc[i].addr);
> +		err = otx2_xdp_sq_append_pkt(pfvf, NULL, dma_addr, xdp_desc[i].len,
> +					     queue, OTX2_AF_XDP_FRAME);
> +		if (!err) {
> +			netdev_err(pfvf->netdev, "AF_XDP: Unable to transfer packet err%d\n", err);
> +			break;
> +		}
> +		work_done++;
> +	}
> +
> +	if (work_done)
> +		xsk_tx_release(pool);

this is broken actually. the batch api you're using above is doing tx
release internally for you.

Sorry for not catching this earlier but i was never CCed in this series.

> +}
> diff --git a/drivers/net/ethernet/marvell/octeontx2/nic/otx2_xsk.h b/drivers/net/ethernet/marvell/octeontx2/nic/otx2_xsk.h
> index 022b3433edbb..8047fafee8fe 100644
> --- a/drivers/net/ethernet/marvell/octeontx2/nic/otx2_xsk.h
> +++ b/drivers/net/ethernet/marvell/octeontx2/nic/otx2_xsk.h
> @@ -17,5 +17,8 @@ int otx2_xsk_pool_disable(struct otx2_nic *pf, u16 qid);
>  int otx2_xsk_pool_alloc_buf(struct otx2_nic *pfvf, struct otx2_pool *pool,
>  			    dma_addr_t *dma, int idx);
>  int otx2_xsk_wakeup(struct net_device *dev, u32 queue_id, u32 flags);
> +void otx2_zc_napi_handler(struct otx2_nic *pfvf, struct xsk_buff_pool *pool,
> +			  int queue, int budget);
> +void otx2_attach_xsk_buff(struct otx2_nic *pfvf, struct otx2_snd_queue *sq, int qidx);
>  
>  #endif /* OTX2_XSK_H */
> -- 
> 2.25.1
> 
>
Suman Ghosh Feb. 19, 2025, 9:46 a.m. UTC | #3 
>> +void otx2_zc_napi_handler(struct otx2_nic *pfvf, struct xsk_buff_pool
>*pool,
>> +			  int queue, int budget)
>> +{
>> +	struct xdp_desc *xdp_desc = pool->tx_descs;
>> +	int err, i, work_done = 0, batch;
>> +
>> +	budget = min(budget, otx2_read_free_sqe(pfvf, queue));
>> +	batch = xsk_tx_peek_release_desc_batch(pool, budget);
>> +	if (!batch)
>> +		return;
>> +
>> +	for (i = 0; i < batch; i++) {
>> +		dma_addr_t dma_addr;
>> +
>> +		dma_addr = xsk_buff_raw_get_dma(pool, xdp_desc[i].addr);
>> +		err = otx2_xdp_sq_append_pkt(pfvf, NULL, dma_addr,
>xdp_desc[i].len,
>> +					     queue, OTX2_AF_XDP_FRAME);
>> +		if (!err) {
>> +			netdev_err(pfvf->netdev, "AF_XDP: Unable to transfer
>packet
>> +err%d\n", err);
>
>Here `err` is always 0, dumping it's value is quite confusing.
>
>The root cause is that otx2_xdp_sq_append_pkt() returns a success
>boolean value, the variable holding it should possibly be renamed
>accordingly.
>
>Since this is the only nit I could find, I think we are better without a
>repost, but please follow-up on this chunk soon.
>
>Thanks,
>
>Paolo
[Suman] Sure, I will push a follow-up patch
Suman Ghosh Feb. 19, 2025, 9:47 a.m. UTC | #4 
>> +void otx2_zc_napi_handler(struct otx2_nic *pfvf, struct xsk_buff_pool
>*pool,
>> +			  int queue, int budget)
>> +{
>> +	struct xdp_desc *xdp_desc = pool->tx_descs;
>> +	int err, i, work_done = 0, batch;
>> +
>> +	budget = min(budget, otx2_read_free_sqe(pfvf, queue));
>> +	batch = xsk_tx_peek_release_desc_batch(pool, budget);
>> +	if (!batch)
>> +		return;
>> +
>> +	for (i = 0; i < batch; i++) {
>> +		dma_addr_t dma_addr;
>> +
>> +		dma_addr = xsk_buff_raw_get_dma(pool, xdp_desc[i].addr);
>> +		err = otx2_xdp_sq_append_pkt(pfvf, NULL, dma_addr,
>xdp_desc[i].len,
>> +					     queue, OTX2_AF_XDP_FRAME);
>> +		if (!err) {
>> +			netdev_err(pfvf->netdev, "AF_XDP: Unable to transfer
>packet err%d\n", err);
>> +			break;
>> +		}
>> +		work_done++;
>> +	}
>> +
>> +	if (work_done)
>> +		xsk_tx_release(pool);
>
>this is broken actually. the batch api you're using above is doing tx
>release internally for you.
>
>Sorry for not catching this earlier but i was never CCed in this series.
[Suman] Thanks for the comment, I will push a follow-up patch.
diff mbox series

Patch

diff --git a/drivers/net/ethernet/marvell/octeontx2/nic/otx2_common.c b/drivers/net/ethernet/marvell/octeontx2/nic/otx2_common.c
index 6f7c11d59942..84cd029a85aa 100644
--- a/drivers/net/ethernet/marvell/octeontx2/nic/otx2_common.c
+++ b/drivers/net/ethernet/marvell/octeontx2/nic/otx2_common.c
@@ -1037,6 +1037,10 @@  int otx2_sq_init(struct otx2_nic *pfvf, u16 qidx, u16 sqb_aura)
 
 	sq->stats.bytes = 0;
 	sq->stats.pkts = 0;
+	/* Attach XSK_BUFF_POOL to XDP queue */
+	if (qidx > pfvf->hw.xdp_queues)
+		otx2_attach_xsk_buff(pfvf, sq, (qidx - pfvf->hw.xdp_queues));
+
 
 	chan_offset = qidx % pfvf->hw.tx_chan_cnt;
 	err = pfvf->hw_ops->sq_aq_init(pfvf, qidx, chan_offset, sqb_aura);
diff --git a/drivers/net/ethernet/marvell/octeontx2/nic/otx2_common.h b/drivers/net/ethernet/marvell/octeontx2/nic/otx2_common.h
index 19e9e2e72233..1e88422825be 100644
--- a/drivers/net/ethernet/marvell/octeontx2/nic/otx2_common.h
+++ b/drivers/net/ethernet/marvell/octeontx2/nic/otx2_common.h
@@ -129,6 +129,12 @@  enum otx2_errcodes_re {
 	ERRCODE_IL4_CSUM = 0x22,
 };
 
+enum otx2_xdp_action {
+	OTX2_XDP_TX	  = BIT(0),
+	OTX2_XDP_REDIRECT = BIT(1),
+	OTX2_AF_XDP_FRAME = BIT(2),
+};
+
 struct otx2_dev_stats {
 	u64 rx_bytes;
 	u64 rx_frames;
diff --git a/drivers/net/ethernet/marvell/octeontx2/nic/otx2_pf.c b/drivers/net/ethernet/marvell/octeontx2/nic/otx2_pf.c
index 50a42cd5d50a..c7c562f0f5e5 100644
--- a/drivers/net/ethernet/marvell/octeontx2/nic/otx2_pf.c
+++ b/drivers/net/ethernet/marvell/octeontx2/nic/otx2_pf.c
@@ -2693,7 +2693,7 @@  static int otx2_xdp_xmit_tx(struct otx2_nic *pf, struct xdp_frame *xdpf,
 		return -ENOMEM;
 
 	err = otx2_xdp_sq_append_pkt(pf, xdpf, dma_addr, xdpf->len,
-				     qidx, XDP_REDIRECT);
+				     qidx, OTX2_XDP_REDIRECT);
 	if (!err) {
 		otx2_dma_unmap_page(pf, dma_addr, xdpf->len, DMA_TO_DEVICE);
 		xdp_return_frame(xdpf);
diff --git a/drivers/net/ethernet/marvell/octeontx2/nic/otx2_txrx.c b/drivers/net/ethernet/marvell/octeontx2/nic/otx2_txrx.c
index 9a6c1f1a3ee0..6bc5ce5a9f61 100644
--- a/drivers/net/ethernet/marvell/octeontx2/nic/otx2_txrx.c
+++ b/drivers/net/ethernet/marvell/octeontx2/nic/otx2_txrx.c
@@ -20,6 +20,7 @@ 
 #include "otx2_txrx.h"
 #include "otx2_ptp.h"
 #include "cn10k.h"
+#include "otx2_xsk.h"
 
 #define CQE_ADDR(CQ, idx) ((CQ)->cqe_base + ((CQ)->cqe_size * (idx)))
 #define PTP_PORT	        0x13F
@@ -103,13 +104,19 @@  static unsigned int frag_num(unsigned int i)
 
 static void otx2_xdp_snd_pkt_handler(struct otx2_nic *pfvf,
 				     struct otx2_snd_queue *sq,
-				     struct nix_cqe_tx_s *cqe)
+				     struct nix_cqe_tx_s *cqe,
+				     int *xsk_frames)
 {
 	struct nix_send_comp_s *snd_comp = &cqe->comp;
 	struct sg_list *sg;
 
 	sg = &sq->sg[snd_comp->sqe_id];
-	if (sg->flags & XDP_REDIRECT)
+	if (sg->flags & OTX2_AF_XDP_FRAME) {
+		(*xsk_frames)++;
+		return;
+	}
+
+	if (sg->flags & OTX2_XDP_REDIRECT)
 		otx2_dma_unmap_page(pfvf, sg->dma_addr[0], sg->size[0], DMA_TO_DEVICE);
 	xdp_return_frame((struct xdp_frame *)sg->skb);
 	sg->skb = (u64)NULL;
@@ -434,6 +441,18 @@  int otx2_refill_pool_ptrs(void *dev, struct otx2_cq_queue *cq)
 	return cnt - cq->pool_ptrs;
 }
 
+static void otx2_zc_submit_pkts(struct otx2_nic *pfvf, struct xsk_buff_pool *xsk_pool,
+				int *xsk_frames, int qidx, int budget)
+{
+	if (*xsk_frames)
+		xsk_tx_completed(xsk_pool, *xsk_frames);
+
+	if (xsk_uses_need_wakeup(xsk_pool))
+		xsk_set_tx_need_wakeup(xsk_pool);
+
+	otx2_zc_napi_handler(pfvf, xsk_pool, qidx, budget);
+}
+
 static int otx2_tx_napi_handler(struct otx2_nic *pfvf,
 				struct otx2_cq_queue *cq, int budget)
 {
@@ -442,16 +461,22 @@  static int otx2_tx_napi_handler(struct otx2_nic *pfvf,
 	struct nix_cqe_tx_s *cqe;
 	struct net_device *ndev;
 	int processed_cqe = 0;
+	int xsk_frames = 0;
+
+	qidx = cq->cq_idx - pfvf->hw.rx_queues;
+	sq = &pfvf->qset.sq[qidx];
 
 	if (cq->pend_cqe >= budget)
 		goto process_cqe;
 
-	if (otx2_nix_cq_op_status(pfvf, cq) || !cq->pend_cqe)
+	if (otx2_nix_cq_op_status(pfvf, cq) || !cq->pend_cqe) {
+		if (sq->xsk_pool)
+			otx2_zc_submit_pkts(pfvf, sq->xsk_pool, &xsk_frames,
+					    qidx, budget);
 		return 0;
+	}
 
 process_cqe:
-	qidx = cq->cq_idx - pfvf->hw.rx_queues;
-	sq = &pfvf->qset.sq[qidx];
 
 	while (likely(processed_cqe < budget) && cq->pend_cqe) {
 		cqe = (struct nix_cqe_tx_s *)otx2_get_next_cqe(cq);
@@ -461,10 +486,8 @@  static int otx2_tx_napi_handler(struct otx2_nic *pfvf,
 			break;
 		}
 
-		qidx = cq->cq_idx - pfvf->hw.rx_queues;
-
 		if (cq->cq_type == CQ_XDP)
-			otx2_xdp_snd_pkt_handler(pfvf, sq, cqe);
+			otx2_xdp_snd_pkt_handler(pfvf, sq, cqe, &xsk_frames);
 		else
 			otx2_snd_pkt_handler(pfvf, cq, &pfvf->qset.sq[qidx],
 					     cqe, budget, &tx_pkts, &tx_bytes);
@@ -505,6 +528,10 @@  static int otx2_tx_napi_handler(struct otx2_nic *pfvf,
 		    netif_carrier_ok(ndev))
 			netif_tx_wake_queue(txq);
 	}
+
+	if (sq->xsk_pool)
+		otx2_zc_submit_pkts(pfvf, sq->xsk_pool, &xsk_frames, qidx, budget);
+
 	return 0;
 }
 
@@ -1499,8 +1526,10 @@  static bool otx2_xdp_rcv_pkt_handler(struct otx2_nic *pfvf,
 		qidx += pfvf->hw.tx_queues;
 		cq->pool_ptrs++;
 		xdpf = xdp_convert_buff_to_frame(&xdp);
-		return otx2_xdp_sq_append_pkt(pfvf, xdpf, cqe->sg.seg_addr,
-					      cqe->sg.seg_size, qidx, XDP_TX);
+		return otx2_xdp_sq_append_pkt(pfvf, xdpf,
+					      cqe->sg.seg_addr,
+					      cqe->sg.seg_size,
+					      qidx, OTX2_XDP_TX);
 	case XDP_REDIRECT:
 		cq->pool_ptrs++;
 		if (xsk_buff) {
diff --git a/drivers/net/ethernet/marvell/octeontx2/nic/otx2_txrx.h b/drivers/net/ethernet/marvell/octeontx2/nic/otx2_txrx.h
index 8f346fbc8221..acf259d72008 100644
--- a/drivers/net/ethernet/marvell/octeontx2/nic/otx2_txrx.h
+++ b/drivers/net/ethernet/marvell/octeontx2/nic/otx2_txrx.h
@@ -106,6 +106,8 @@  struct otx2_snd_queue {
 	/* SQE ring and CPT response queue for Inline IPSEC */
 	struct qmem		*sqe_ring;
 	struct qmem		*cpt_resp;
+	/* Buffer pool for af_xdp zero-copy */
+	struct xsk_buff_pool    *xsk_pool;
 } ____cacheline_aligned_in_smp;
 
 enum cq_type {
diff --git a/drivers/net/ethernet/marvell/octeontx2/nic/otx2_xsk.c b/drivers/net/ethernet/marvell/octeontx2/nic/otx2_xsk.c
index 3bdee1603fac..ce10caea8511 100644
--- a/drivers/net/ethernet/marvell/octeontx2/nic/otx2_xsk.c
+++ b/drivers/net/ethernet/marvell/octeontx2/nic/otx2_xsk.c
@@ -140,11 +140,14 @@  int otx2_xsk_pool_disable(struct otx2_nic *pf, u16 qidx)
 {
 	struct net_device *netdev = pf->netdev;
 	struct xsk_buff_pool *pool;
+	struct otx2_snd_queue *sq;
 
 	pool = xsk_get_pool_from_qid(netdev, qidx);
 	if (!pool)
 		return -EINVAL;
 
+	sq = &pf->qset.sq[qidx + pf->hw.tx_queues];
+	sq->xsk_pool = NULL;
 	otx2_clean_up_rq(pf, qidx);
 	clear_bit(qidx, pf->af_xdp_zc_qidx);
 	xsk_pool_dma_unmap(pool, DMA_ATTR_SKIP_CPU_SYNC | DMA_ATTR_WEAK_ORDERING);
@@ -171,7 +174,7 @@  int otx2_xsk_wakeup(struct net_device *dev, u32 queue_id, u32 flags)
 	if (pf->flags & OTX2_FLAG_INTF_DOWN)
 		return -ENETDOWN;
 
-	if (queue_id >= pf->hw.rx_queues)
+	if (queue_id >= pf->hw.rx_queues || queue_id >= pf->hw.tx_queues)
 		return -EINVAL;
 
 	cq_poll = &qset->napi[queue_id];
@@ -179,8 +182,44 @@  int otx2_xsk_wakeup(struct net_device *dev, u32 queue_id, u32 flags)
 		return -EINVAL;
 
 	/* Trigger interrupt */
-	if (!napi_if_scheduled_mark_missed(&cq_poll->napi))
+	if (!napi_if_scheduled_mark_missed(&cq_poll->napi)) {
 		otx2_write64(pf, NIX_LF_CINTX_ENA_W1S(cq_poll->cint_idx), BIT_ULL(0));
+		otx2_write64(pf, NIX_LF_CINTX_INT_W1S(cq_poll->cint_idx), BIT_ULL(0));
+	}
 
 	return 0;
 }
+
+void otx2_attach_xsk_buff(struct otx2_nic *pfvf, struct otx2_snd_queue *sq, int qidx)
+{
+	if (test_bit(qidx, pfvf->af_xdp_zc_qidx))
+		sq->xsk_pool = xsk_get_pool_from_qid(pfvf->netdev, qidx);
+}
+
+void otx2_zc_napi_handler(struct otx2_nic *pfvf, struct xsk_buff_pool *pool,
+			  int queue, int budget)
+{
+	struct xdp_desc *xdp_desc = pool->tx_descs;
+	int err, i, work_done = 0, batch;
+
+	budget = min(budget, otx2_read_free_sqe(pfvf, queue));
+	batch = xsk_tx_peek_release_desc_batch(pool, budget);
+	if (!batch)
+		return;
+
+	for (i = 0; i < batch; i++) {
+		dma_addr_t dma_addr;
+
+		dma_addr = xsk_buff_raw_get_dma(pool, xdp_desc[i].addr);
+		err = otx2_xdp_sq_append_pkt(pfvf, NULL, dma_addr, xdp_desc[i].len,
+					     queue, OTX2_AF_XDP_FRAME);
+		if (!err) {
+			netdev_err(pfvf->netdev, "AF_XDP: Unable to transfer packet err%d\n", err);
+			break;
+		}
+		work_done++;
+	}
+
+	if (work_done)
+		xsk_tx_release(pool);
+}
diff --git a/drivers/net/ethernet/marvell/octeontx2/nic/otx2_xsk.h b/drivers/net/ethernet/marvell/octeontx2/nic/otx2_xsk.h
index 022b3433edbb..8047fafee8fe 100644
--- a/drivers/net/ethernet/marvell/octeontx2/nic/otx2_xsk.h
+++ b/drivers/net/ethernet/marvell/octeontx2/nic/otx2_xsk.h
@@ -17,5 +17,8 @@  int otx2_xsk_pool_disable(struct otx2_nic *pf, u16 qid);
 int otx2_xsk_pool_alloc_buf(struct otx2_nic *pfvf, struct otx2_pool *pool,
 			    dma_addr_t *dma, int idx);
 int otx2_xsk_wakeup(struct net_device *dev, u32 queue_id, u32 flags);
+void otx2_zc_napi_handler(struct otx2_nic *pfvf, struct xsk_buff_pool *pool,
+			  int queue, int budget);
+void otx2_attach_xsk_buff(struct otx2_nic *pfvf, struct otx2_snd_queue *sq, int qidx);
 
 #endif /* OTX2_XSK_H */