diff mbox series

[29/33] virtio_net: xsk: tx: support tx

Message ID 20230202110058.130695-30-xuanzhuo@linux.alibaba.com (mailing list archive)
State Changes Requested
Delegated to: Netdev Maintainers
Headers show
Series virtio-net: support AF_XDP zero copy | expand

Checks

Context Check Description
netdev/tree_selection success Guessed tree name to be net-next, async
netdev/fixes_present success Fixes tag not required for -next series
netdev/subject_prefix success Link
netdev/cover_letter success Series has a cover letter
netdev/patch_count fail Series longer than 15 patches (and no cover letter)
netdev/header_inline success No static functions without inline keyword in header files
netdev/build_32bit success Errors and warnings before: 5 this patch: 4
netdev/cc_maintainers success CCed 13 of 13 maintainers
netdev/build_clang success Errors and warnings before: 2 this patch: 0
netdev/module_param success Was 0 now: 0
netdev/verify_signedoff success Signed-off-by tag matches author and committer
netdev/check_selftest success No net selftest shell script
netdev/verify_fixes success No Fixes tag
netdev/build_allmodconfig_warn success Errors and warnings before: 5 this patch: 4
netdev/checkpatch success total: 0 errors, 0 warnings, 0 checks, 182 lines checked
netdev/kdoc success Errors and warnings before: 0 this patch: 0
netdev/source_inline success Was 0 now: 0

Commit Message

Xuan Zhuo Feb. 2, 2023, 11 a.m. UTC 
The driver's tx napi is very important for XSK. It is responsible for
obtaining data from the XSK queue and sending it out.

At the beginning, we need to trigger tx napi.

Signed-off-by: Xuan Zhuo <xuanzhuo@linux.alibaba.com>
---
 drivers/net/virtio/main.c |  12 +++-
 drivers/net/virtio/xsk.c  | 146 ++++++++++++++++++++++++++++++++++++++
 drivers/net/virtio/xsk.h  |   2 +
 3 files changed, 159 insertions(+), 1 deletion(-)

Comments

Maciej Fijalkowski Feb. 3, 2023, 8:39 a.m. UTC | #1 
On Thu, Feb 02, 2023 at 07:00:54PM +0800, Xuan Zhuo wrote:
> The driver's tx napi is very important for XSK. It is responsible for
> obtaining data from the XSK queue and sending it out.
> 
> At the beginning, we need to trigger tx napi.
> 
> Signed-off-by: Xuan Zhuo <xuanzhuo@linux.alibaba.com>
> ---
>  drivers/net/virtio/main.c |  12 +++-
>  drivers/net/virtio/xsk.c  | 146 ++++++++++++++++++++++++++++++++++++++
>  drivers/net/virtio/xsk.h  |   2 +
>  3 files changed, 159 insertions(+), 1 deletion(-)
> 

(...)

> +static int virtnet_xsk_xmit_batch(struct send_queue *sq,
> +				  struct xsk_buff_pool *pool,
> +				  unsigned int budget,
> +				  struct virtnet_sq_stats *stats)
> +{
> +	int ret = XSK_XMIT_NO_BUDGET;
> +	struct xdp_desc desc;
> +	int err, packet = 0;
> +
> +	while (budget-- > 0) {
> +		if (sq->vq->num_free < 2) {
> +			__free_old_xmit(sq, true, stats);
> +			if (sq->vq->num_free < 2) {
> +				ret = XSK_XMIT_DEV_BUSY;
> +				break;
> +			}
> +		}
> +
> +		if (!xsk_tx_peek_desc(pool, &desc)) {

anything that stopped from using xsk_tx_peek_release_desc_batch() ?

> +			ret = XSK_XMIT_DONE;
> +			break;
> +		}
> +
> +		err = virtnet_xsk_xmit_one(sq, pool, &desc);
> +		if (unlikely(err)) {
> +			ret = XSK_XMIT_DEV_BUSY;
> +			break;
> +		}
> +
> +		++packet;
> +
> +		if (virtqueue_kick_prepare(sq->vq) && virtqueue_notify(sq->vq))
> +			++stats->kicks;
> +	}
> +
> +	if (packet) {
> +		stats->xdp_tx += packet;
> +
> +		xsk_tx_release(pool);
> +	}
> +
> +	return ret;
> +}
> +
> +bool virtnet_xsk_xmit(struct send_queue *sq, struct xsk_buff_pool *pool,
> +		      int budget)
> +{
> +	struct virtnet_sq_stats stats = {};
> +	bool busy;
> +	int ret;
> +
> +	__free_old_xmit(sq, true, &stats);
> +
> +	if (xsk_uses_need_wakeup(pool))
> +		xsk_set_tx_need_wakeup(pool);
> +
> +	ret = virtnet_xsk_xmit_batch(sq, pool, budget, &stats);
> +	switch (ret) {
> +	case XSK_XMIT_DONE:
> +		/* xsk tx qeueu has been consumed done. should complete napi. */
> +		busy = false;
> +		break;
> +
> +	case XSK_XMIT_NO_BUDGET:
> +		/* reach the budget limit. should let napi run again. */
> +		busy = true;
> +		break;
> +
> +	case XSK_XMIT_DEV_BUSY:
> +		/* sq vring is full, should complete napi. wait for tx napi been
> +		 * triggered by interrupt.
> +		 */
> +		busy = false;
> +		break;
> +	}
> +
> +	virtnet_xsk_check_queue(sq);
> +
> +	u64_stats_update_begin(&sq->stats.syncp);
> +	sq->stats.packets += stats.packets;
> +	sq->stats.bytes += stats.bytes;
> +	sq->stats.kicks += stats.kicks;
> +	sq->stats.xdp_tx += stats.xdp_tx;
> +	u64_stats_update_end(&sq->stats.syncp);
> +
> +	return busy;
> +}
> +
>  static int virtnet_rq_bind_xsk_pool(struct virtnet_info *vi, struct receive_queue *rq,
>  				    struct xsk_buff_pool *pool, struct net_device *dev)
>  {
> diff --git a/drivers/net/virtio/xsk.h b/drivers/net/virtio/xsk.h
> index ad684c812091..15f1540a5803 100644
> --- a/drivers/net/virtio/xsk.h
> +++ b/drivers/net/virtio/xsk.h
> @@ -20,4 +20,6 @@ static inline u32 ptr_to_xsk(void *ptr)
>  }
>  
>  int virtnet_xsk_pool_setup(struct net_device *dev, struct netdev_bpf *xdp);
> +bool virtnet_xsk_xmit(struct send_queue *sq, struct xsk_buff_pool *pool,
> +		      int budget);
>  #endif
> -- 
> 2.32.0.3.g01195cf9f
>
Xuan Zhuo Feb. 3, 2023, 8:55 a.m. UTC | #2 
On Fri, 3 Feb 2023 09:39:25 +0100, Maciej Fijalkowski <maciej.fijalkowski@intel.com> wrote:
> On Thu, Feb 02, 2023 at 07:00:54PM +0800, Xuan Zhuo wrote:
> > The driver's tx napi is very important for XSK. It is responsible for
> > obtaining data from the XSK queue and sending it out.
> >
> > At the beginning, we need to trigger tx napi.
> >
> > Signed-off-by: Xuan Zhuo <xuanzhuo@linux.alibaba.com>
> > ---
> >  drivers/net/virtio/main.c |  12 +++-
> >  drivers/net/virtio/xsk.c  | 146 ++++++++++++++++++++++++++++++++++++++
> >  drivers/net/virtio/xsk.h  |   2 +
> >  3 files changed, 159 insertions(+), 1 deletion(-)
> >
>
> (...)
>
> > +static int virtnet_xsk_xmit_batch(struct send_queue *sq,
> > +				  struct xsk_buff_pool *pool,
> > +				  unsigned int budget,
> > +				  struct virtnet_sq_stats *stats)
> > +{
> > +	int ret = XSK_XMIT_NO_BUDGET;
> > +	struct xdp_desc desc;
> > +	int err, packet = 0;
> > +
> > +	while (budget-- > 0) {
> > +		if (sq->vq->num_free < 2) {
> > +			__free_old_xmit(sq, true, stats);
> > +			if (sq->vq->num_free < 2) {
> > +				ret = XSK_XMIT_DEV_BUSY;
> > +				break;
> > +			}
> > +		}
> > +
> > +		if (!xsk_tx_peek_desc(pool, &desc)) {
>
> anything that stopped from using xsk_tx_peek_release_desc_batch() ?


Great!

Will fix.

Thanks.


>
> > +			ret = XSK_XMIT_DONE;
> > +			break;
> > +		}
> > +
> > +		err = virtnet_xsk_xmit_one(sq, pool, &desc);
> > +		if (unlikely(err)) {
> > +			ret = XSK_XMIT_DEV_BUSY;
> > +			break;
> > +		}
> > +
> > +		++packet;
> > +
> > +		if (virtqueue_kick_prepare(sq->vq) && virtqueue_notify(sq->vq))
> > +			++stats->kicks;
> > +	}
> > +
> > +	if (packet) {
> > +		stats->xdp_tx += packet;
> > +
> > +		xsk_tx_release(pool);
> > +	}
> > +
> > +	return ret;
> > +}
> > +
> > +bool virtnet_xsk_xmit(struct send_queue *sq, struct xsk_buff_pool *pool,
> > +		      int budget)
> > +{
> > +	struct virtnet_sq_stats stats = {};
> > +	bool busy;
> > +	int ret;
> > +
> > +	__free_old_xmit(sq, true, &stats);
> > +
> > +	if (xsk_uses_need_wakeup(pool))
> > +		xsk_set_tx_need_wakeup(pool);
> > +
> > +	ret = virtnet_xsk_xmit_batch(sq, pool, budget, &stats);
> > +	switch (ret) {
> > +	case XSK_XMIT_DONE:
> > +		/* xsk tx qeueu has been consumed done. should complete napi. */
> > +		busy = false;
> > +		break;
> > +
> > +	case XSK_XMIT_NO_BUDGET:
> > +		/* reach the budget limit. should let napi run again. */
> > +		busy = true;
> > +		break;
> > +
> > +	case XSK_XMIT_DEV_BUSY:
> > +		/* sq vring is full, should complete napi. wait for tx napi been
> > +		 * triggered by interrupt.
> > +		 */
> > +		busy = false;
> > +		break;
> > +	}
> > +
> > +	virtnet_xsk_check_queue(sq);
> > +
> > +	u64_stats_update_begin(&sq->stats.syncp);
> > +	sq->stats.packets += stats.packets;
> > +	sq->stats.bytes += stats.bytes;
> > +	sq->stats.kicks += stats.kicks;
> > +	sq->stats.xdp_tx += stats.xdp_tx;
> > +	u64_stats_update_end(&sq->stats.syncp);
> > +
> > +	return busy;
> > +}
> > +
> >  static int virtnet_rq_bind_xsk_pool(struct virtnet_info *vi, struct receive_queue *rq,
> >  				    struct xsk_buff_pool *pool, struct net_device *dev)
> >  {
> > diff --git a/drivers/net/virtio/xsk.h b/drivers/net/virtio/xsk.h
> > index ad684c812091..15f1540a5803 100644
> > --- a/drivers/net/virtio/xsk.h
> > +++ b/drivers/net/virtio/xsk.h
> > @@ -20,4 +20,6 @@ static inline u32 ptr_to_xsk(void *ptr)
> >  }
> >
> >  int virtnet_xsk_pool_setup(struct net_device *dev, struct netdev_bpf *xdp);
> > +bool virtnet_xsk_xmit(struct send_queue *sq, struct xsk_buff_pool *pool,
> > +		      int budget);
> >  #endif
> > --
> > 2.32.0.3.g01195cf9f
> >
diff mbox series

Patch

diff --git a/drivers/net/virtio/main.c b/drivers/net/virtio/main.c
index 43249c78484a..02d2f7d21bdf 100644
--- a/drivers/net/virtio/main.c
+++ b/drivers/net/virtio/main.c
@@ -1607,6 +1607,7 @@  static int virtnet_poll_tx(struct napi_struct *napi, int budget)
 	struct send_queue *sq = container_of(napi, struct send_queue, napi);
 	struct virtnet_info *vi = sq->vq->vdev->priv;
 	unsigned int index = vq2txq(sq->vq);
+	struct xsk_buff_pool *pool;
 	struct netdev_queue *txq;
 	int busy = 0;
 	int opaque;
@@ -1621,7 +1622,16 @@  static int virtnet_poll_tx(struct napi_struct *napi, int budget)
 	txq = netdev_get_tx_queue(vi->dev, index);
 	__netif_tx_lock(txq, raw_smp_processor_id());
 	virtqueue_disable_cb(sq->vq);
-	free_old_xmit(sq, true);
+
+	rcu_read_lock();
+	pool = rcu_dereference(sq->xsk.pool);
+	if (pool) {
+		busy |= virtnet_xsk_xmit(sq, pool, budget);
+		rcu_read_unlock();
+	} else {
+		rcu_read_unlock();
+		free_old_xmit(sq, true);
+	}
 
 	if (sq->vq->num_free >= 2 + MAX_SKB_FRAGS)
 		netif_tx_wake_queue(txq);
diff --git a/drivers/net/virtio/xsk.c b/drivers/net/virtio/xsk.c
index b96af38a2608..04db80244dbd 100644
--- a/drivers/net/virtio/xsk.c
+++ b/drivers/net/virtio/xsk.c
@@ -7,6 +7,152 @@ 
 
 static struct virtio_net_hdr_mrg_rxbuf xsk_hdr;
 
+static void sg_fill_dma(struct scatterlist *sg, dma_addr_t addr, u32 len)
+{
+	sg->dma_address = addr;
+	sg->length = len;
+}
+
+static void virtnet_xsk_check_queue(struct send_queue *sq)
+{
+	struct virtnet_info *vi = sq->vq->vdev->priv;
+	struct net_device *dev = vi->dev;
+	int qnum = sq - vi->sq;
+
+	/* If it is a raw buffer queue, it does not check whether the status
+	 * of the queue is stopped when sending. So there is no need to check
+	 * the situation of the raw buffer queue.
+	 */
+	if (is_xdp_raw_buffer_queue(vi, qnum))
+		return;
+
+	/* If this sq is not the exclusive queue of the current cpu,
+	 * then it may be called by start_xmit, so check it running out
+	 * of space.
+	 *
+	 * Stop the queue to avoid getting packets that we are
+	 * then unable to transmit. Then wait the tx interrupt.
+	 */
+	if (sq->vq->num_free < 2 + MAX_SKB_FRAGS)
+		netif_stop_subqueue(dev, qnum);
+}
+
+static int virtnet_xsk_xmit_one(struct send_queue *sq,
+				struct xsk_buff_pool *pool,
+				struct xdp_desc *desc)
+{
+	struct virtnet_info *vi;
+	dma_addr_t addr;
+
+	vi = sq->vq->vdev->priv;
+
+	addr = xsk_buff_raw_get_dma(pool, desc->addr);
+	xsk_buff_raw_dma_sync_for_device(pool, addr, desc->len);
+
+	sg_init_table(sq->sg, 2);
+
+	sg_fill_dma(sq->sg, sq->xsk.hdr_dma_address, vi->hdr_len);
+	sg_fill_dma(sq->sg + 1, addr, desc->len);
+
+	return virtqueue_add_outbuf_premapped(sq->vq, sq->sg, 2,
+					      xsk_to_ptr(desc->len),
+					      GFP_ATOMIC);
+}
+
+enum {
+	XSK_XMIT_DONE,
+	XSK_XMIT_DEV_BUSY,
+	XSK_XMIT_NO_BUDGET
+};
+
+static int virtnet_xsk_xmit_batch(struct send_queue *sq,
+				  struct xsk_buff_pool *pool,
+				  unsigned int budget,
+				  struct virtnet_sq_stats *stats)
+{
+	int ret = XSK_XMIT_NO_BUDGET;
+	struct xdp_desc desc;
+	int err, packet = 0;
+
+	while (budget-- > 0) {
+		if (sq->vq->num_free < 2) {
+			__free_old_xmit(sq, true, stats);
+			if (sq->vq->num_free < 2) {
+				ret = XSK_XMIT_DEV_BUSY;
+				break;
+			}
+		}
+
+		if (!xsk_tx_peek_desc(pool, &desc)) {
+			ret = XSK_XMIT_DONE;
+			break;
+		}
+
+		err = virtnet_xsk_xmit_one(sq, pool, &desc);
+		if (unlikely(err)) {
+			ret = XSK_XMIT_DEV_BUSY;
+			break;
+		}
+
+		++packet;
+
+		if (virtqueue_kick_prepare(sq->vq) && virtqueue_notify(sq->vq))
+			++stats->kicks;
+	}
+
+	if (packet) {
+		stats->xdp_tx += packet;
+
+		xsk_tx_release(pool);
+	}
+
+	return ret;
+}
+
+bool virtnet_xsk_xmit(struct send_queue *sq, struct xsk_buff_pool *pool,
+		      int budget)
+{
+	struct virtnet_sq_stats stats = {};
+	bool busy;
+	int ret;
+
+	__free_old_xmit(sq, true, &stats);
+
+	if (xsk_uses_need_wakeup(pool))
+		xsk_set_tx_need_wakeup(pool);
+
+	ret = virtnet_xsk_xmit_batch(sq, pool, budget, &stats);
+	switch (ret) {
+	case XSK_XMIT_DONE:
+		/* xsk tx qeueu has been consumed done. should complete napi. */
+		busy = false;
+		break;
+
+	case XSK_XMIT_NO_BUDGET:
+		/* reach the budget limit. should let napi run again. */
+		busy = true;
+		break;
+
+	case XSK_XMIT_DEV_BUSY:
+		/* sq vring is full, should complete napi. wait for tx napi been
+		 * triggered by interrupt.
+		 */
+		busy = false;
+		break;
+	}
+
+	virtnet_xsk_check_queue(sq);
+
+	u64_stats_update_begin(&sq->stats.syncp);
+	sq->stats.packets += stats.packets;
+	sq->stats.bytes += stats.bytes;
+	sq->stats.kicks += stats.kicks;
+	sq->stats.xdp_tx += stats.xdp_tx;
+	u64_stats_update_end(&sq->stats.syncp);
+
+	return busy;
+}
+
 static int virtnet_rq_bind_xsk_pool(struct virtnet_info *vi, struct receive_queue *rq,
 				    struct xsk_buff_pool *pool, struct net_device *dev)
 {
diff --git a/drivers/net/virtio/xsk.h b/drivers/net/virtio/xsk.h
index ad684c812091..15f1540a5803 100644
--- a/drivers/net/virtio/xsk.h
+++ b/drivers/net/virtio/xsk.h
@@ -20,4 +20,6 @@  static inline u32 ptr_to_xsk(void *ptr)
 }
 
 int virtnet_xsk_pool_setup(struct net_device *dev, struct netdev_bpf *xdp);
+bool virtnet_xsk_xmit(struct send_queue *sq, struct xsk_buff_pool *pool,
+		      int budget);
 #endif