| Message ID | 20240130114224.86536-18-xuanzhuo@linux.alibaba.com (mailing list archive) |
|---|---|
| State | New, archived |
| Headers | show |
| Series | virtio: drivers maintain dma info for premapped vq | expand |
On Tue, Jan 30, 2024 at 7:43 PM Xuan Zhuo <xuanzhuo@linux.alibaba.com> wrote: > > If the xsk is enabling, the xsk tx will share the send queue. > But the xsk requires that the send queue use the premapped mode. > So the send queue must support premapped mode. > > Signed-off-by: Xuan Zhuo <xuanzhuo@linux.alibaba.com> > --- > drivers/net/virtio_net.c | 167 ++++++++++++++++++++++++++++++++++++++- > 1 file changed, 163 insertions(+), 4 deletions(-) > > diff --git a/drivers/net/virtio_net.c b/drivers/net/virtio_net.c > index 226ab830870e..cf0c67380b07 100644 > --- a/drivers/net/virtio_net.c > +++ b/drivers/net/virtio_net.c > @@ -46,6 +46,7 @@ module_param(napi_tx, bool, 0644); > #define VIRTIO_XDP_REDIR BIT(1) > > #define VIRTIO_XDP_FLAG BIT(0) > +#define VIRTIO_DMA_FLAG BIT(1) > > /* RX packet size EWMA. The average packet size is used to determine the packet > * buffer size when refilling RX rings. As the entire RX ring may be refilled > @@ -140,6 +141,21 @@ struct virtnet_rq_dma { > u16 need_sync; > }; > > +struct virtnet_sq_dma { > + union { > + struct virtnet_sq_dma *next; > + void *data; > + }; > + dma_addr_t addr; > + u32 len; > + bool is_tail; > +}; > + > +struct virtnet_sq_dma_head { > + struct virtnet_sq_dma *free; > + struct virtnet_sq_dma *head; Any reason the head must be a pointer instead of a simple index? > +}; > + > /* Internal representation of a send virtqueue */ > struct send_queue { > /* Virtqueue associated with this send _queue */ > @@ -159,6 +175,8 @@ struct send_queue { > > /* Record whether sq is in reset state. */ > bool reset; > + > + struct virtnet_sq_dma_head dmainfo; > }; > > /* Internal representation of a receive virtqueue */ > @@ -348,6 +366,131 @@ static struct xdp_frame *ptr_to_xdp(void *ptr) > return (struct xdp_frame *)((unsigned long)ptr & ~VIRTIO_XDP_FLAG); > } > > +static inline void *virtnet_sq_unmap(struct send_queue *sq, void *data) > +{ > + struct virtnet_sq_dma *head, *tail; > + > + if (!((unsigned long)data & VIRTIO_DMA_FLAG)) > + return data; > + > + head = (void *)((unsigned long)data & ~VIRTIO_DMA_FLAG); > + > + tail = head; > + > + while (true) { > + virtqueue_dma_unmap_page_attrs(sq->vq, tail->addr, tail->len, > + DMA_TO_DEVICE, 0); > + > + if (tail->is_tail) > + break; > + > + tail = tail->next; > + } > + > + data = tail->data; > + tail->is_tail = false; > + > + tail->next = sq->dmainfo.free; > + sq->dmainfo.free = head; > + > + return data; > +} > + > +static void *virtnet_sq_dma_splice(struct send_queue *sq, > + struct virtnet_sq_dma *head, > + struct virtnet_sq_dma *tail, > + void *data) > +{ > + sq->dmainfo.free = tail->next; > + > + tail->is_tail = true; > + tail->data = data; > + > + head = (void *)((unsigned long)head | VIRTIO_DMA_FLAG); > + > + return head; > +} > + > +static struct virtnet_sq_dma *virtnet_sq_map_sg(struct send_queue *sq, int nents, void *data) > +{ > + struct virtnet_sq_dma *head, *tail, *p; > + struct scatterlist *sg; > + dma_addr_t addr; > + int i; > + > + head = sq->dmainfo.free; > + p = head; > + > + tail = NULL; > + > + for_each_sg(sq->sg, sg, nents, i) { > + addr = virtqueue_dma_map_page_attrs(sq->vq, sg_page(sg), > + sg->offset, sg->length, > + DMA_TO_DEVICE, 0); > + if (virtqueue_dma_mapping_error(sq->vq, addr)) > + goto err; > + > + sg->dma_address = addr; > + > + tail = p; > + tail->addr = sg->dma_address; > + tail->len = sg->length; > + > + p = p->next; > + } > + > + return virtnet_sq_dma_splice(sq, head, tail, data); > + > +err: > + if (tail) > + virtnet_sq_unmap(sq, virtnet_sq_dma_splice(sq, head, tail, data)); > + > + return NULL; > +} > + > +static int virtnet_add_outbuf(struct send_queue *sq, u32 num, void *data) > +{ > + int ret; > + > + if (sq->vq->premapped) { > + data = virtnet_sq_map_sg(sq, num, data); > + if (!data) > + return -ENOMEM; > + } > + > + ret = virtqueue_add_outbuf(sq->vq, sq->sg, num, data, GFP_ATOMIC); > + if (ret && sq->vq->premapped) > + virtnet_sq_unmap(sq, data); > + > + return ret; > +} > + > +static int virtnet_sq_init_dma_mate(struct send_queue *sq) > +{ > + struct virtnet_sq_dma *d; > + int size, i; > + > + size = virtqueue_get_vring_size(sq->vq); > + > + size += MAX_SKB_FRAGS + 2; Is this enough for the case where an indirect descriptor is used? Thanks
On Wed, 31 Jan 2024 17:12:47 +0800, Jason Wang <jasowang@redhat.com> wrote: > On Tue, Jan 30, 2024 at 7:43 PM Xuan Zhuo <xuanzhuo@linux.alibaba.com> wrote: > > > > If the xsk is enabling, the xsk tx will share the send queue. > > But the xsk requires that the send queue use the premapped mode. > > So the send queue must support premapped mode. > > > > Signed-off-by: Xuan Zhuo <xuanzhuo@linux.alibaba.com> > > --- > > drivers/net/virtio_net.c | 167 ++++++++++++++++++++++++++++++++++++++- > > 1 file changed, 163 insertions(+), 4 deletions(-) > > > > diff --git a/drivers/net/virtio_net.c b/drivers/net/virtio_net.c > > index 226ab830870e..cf0c67380b07 100644 > > --- a/drivers/net/virtio_net.c > > +++ b/drivers/net/virtio_net.c > > @@ -46,6 +46,7 @@ module_param(napi_tx, bool, 0644); > > #define VIRTIO_XDP_REDIR BIT(1) > > > > #define VIRTIO_XDP_FLAG BIT(0) > > +#define VIRTIO_DMA_FLAG BIT(1) > > > > /* RX packet size EWMA. The average packet size is used to determine the packet > > * buffer size when refilling RX rings. As the entire RX ring may be refilled > > @@ -140,6 +141,21 @@ struct virtnet_rq_dma { > > u16 need_sync; > > }; > > > > +struct virtnet_sq_dma { > > + union { > > + struct virtnet_sq_dma *next; > > + void *data; > > + }; > > + dma_addr_t addr; > > + u32 len; > > + bool is_tail; > > +}; > > + > > +struct virtnet_sq_dma_head { > > + struct virtnet_sq_dma *free; > > + struct virtnet_sq_dma *head; > > Any reason the head must be a pointer instead of a simple index? The head is used for kfree. Maybe I need to rename it. About the index(next) of the virtnet_sq_dma. If we use the index, the struct will be: struct virtnet_sq_dma { dma_addr_t addr; u32 len; u32 next; void *data }; The size of virtnet_sq_dma is same. > > > +}; > > + > > /* Internal representation of a send virtqueue */ > > struct send_queue { > > /* Virtqueue associated with this send _queue */ > > @@ -159,6 +175,8 @@ struct send_queue { > > > > /* Record whether sq is in reset state. */ > > bool reset; > > + > > + struct virtnet_sq_dma_head dmainfo; > > }; > > .... > > + > > +static int virtnet_sq_init_dma_mate(struct send_queue *sq) > > +{ > > + struct virtnet_sq_dma *d; > > + int size, i; > > + > > + size = virtqueue_get_vring_size(sq->vq); > > + > > + size += MAX_SKB_FRAGS + 2; > > Is this enough for the case where an indirect descriptor is used? This is for the case, when the ring is full, the xmit_skb is called. I will add comment. Thanks. > > Thanks >
On Thu, Feb 1, 2024 at 11:28 AM Xuan Zhuo <xuanzhuo@linux.alibaba.com> wrote: > > On Wed, 31 Jan 2024 17:12:47 +0800, Jason Wang <jasowang@redhat.com> wrote: > > On Tue, Jan 30, 2024 at 7:43 PM Xuan Zhuo <xuanzhuo@linux.alibaba.com> wrote: > > > > > > If the xsk is enabling, the xsk tx will share the send queue. > > > But the xsk requires that the send queue use the premapped mode. > > > So the send queue must support premapped mode. > > > > > > Signed-off-by: Xuan Zhuo <xuanzhuo@linux.alibaba.com> > > > --- > > > drivers/net/virtio_net.c | 167 ++++++++++++++++++++++++++++++++++++++- > > > 1 file changed, 163 insertions(+), 4 deletions(-) > > > > > > diff --git a/drivers/net/virtio_net.c b/drivers/net/virtio_net.c > > > index 226ab830870e..cf0c67380b07 100644 > > > --- a/drivers/net/virtio_net.c > > > +++ b/drivers/net/virtio_net.c > > > @@ -46,6 +46,7 @@ module_param(napi_tx, bool, 0644); > > > #define VIRTIO_XDP_REDIR BIT(1) > > > > > > #define VIRTIO_XDP_FLAG BIT(0) > > > +#define VIRTIO_DMA_FLAG BIT(1) > > > > > > /* RX packet size EWMA. The average packet size is used to determine the packet > > > * buffer size when refilling RX rings. As the entire RX ring may be refilled > > > @@ -140,6 +141,21 @@ struct virtnet_rq_dma { > > > u16 need_sync; > > > }; > > > > > > +struct virtnet_sq_dma { > > > + union { > > > + struct virtnet_sq_dma *next; > > > + void *data; > > > + }; > > > + dma_addr_t addr; > > > + u32 len; > > > + bool is_tail; > > > +}; > > > + > > > +struct virtnet_sq_dma_head { > > > + struct virtnet_sq_dma *free; > > > + struct virtnet_sq_dma *head; > > > > Any reason the head must be a pointer instead of a simple index? > > > The head is used for kfree. > Maybe I need to rename it. > > About the index(next) of the virtnet_sq_dma. > If we use the index, the struct will be: > > struct virtnet_sq_dma { > dma_addr_t addr; > u32 len; > > u32 next; > void *data > }; > > The size of virtnet_sq_dma is same. Ok. > > > > > > > +}; > > > + > > > /* Internal representation of a send virtqueue */ > > > struct send_queue { > > > /* Virtqueue associated with this send _queue */ > > > @@ -159,6 +175,8 @@ struct send_queue { > > > > > > /* Record whether sq is in reset state. */ > > > bool reset; > > > + > > > + struct virtnet_sq_dma_head dmainfo; > > > }; > > > > > .... > > > > + > > > +static int virtnet_sq_init_dma_mate(struct send_queue *sq) > > > +{ > > > + struct virtnet_sq_dma *d; > > > + int size, i; > > > + > > > + size = virtqueue_get_vring_size(sq->vq); > > > + > > > + size += MAX_SKB_FRAGS + 2; > > > > Is this enough for the case where an indirect descriptor is used? > > > This is for the case, when the ring is full, the xmit_skb is called. > > I will add comment. Just to make sure we are at the same page. I meant, we could have more pending #sg than allocated here. For example, we can have up to (vring_size - 2 - MAX_SKB_FRAGS) * MAX_SKB_FRAGS number of pending sgs? Thanks > > Thanks. > > > > > > Thanks > > >
On Thu, 1 Feb 2024 13:36:46 +0800, Jason Wang <jasowang@redhat.com> wrote: > On Thu, Feb 1, 2024 at 11:28 AM Xuan Zhuo <xuanzhuo@linux.alibaba.com> wrote: > > > > On Wed, 31 Jan 2024 17:12:47 +0800, Jason Wang <jasowang@redhat.com> wrote: > > > On Tue, Jan 30, 2024 at 7:43 PM Xuan Zhuo <xuanzhuo@linux.alibaba.com> wrote: > > > > > > > > If the xsk is enabling, the xsk tx will share the send queue. > > > > But the xsk requires that the send queue use the premapped mode. > > > > So the send queue must support premapped mode. > > > > > > > > Signed-off-by: Xuan Zhuo <xuanzhuo@linux.alibaba.com> > > > > --- > > > > drivers/net/virtio_net.c | 167 ++++++++++++++++++++++++++++++++++++++- > > > > 1 file changed, 163 insertions(+), 4 deletions(-) > > > > > > > > diff --git a/drivers/net/virtio_net.c b/drivers/net/virtio_net.c > > > > index 226ab830870e..cf0c67380b07 100644 > > > > --- a/drivers/net/virtio_net.c > > > > +++ b/drivers/net/virtio_net.c > > > > @@ -46,6 +46,7 @@ module_param(napi_tx, bool, 0644); > > > > #define VIRTIO_XDP_REDIR BIT(1) > > > > > > > > #define VIRTIO_XDP_FLAG BIT(0) > > > > +#define VIRTIO_DMA_FLAG BIT(1) > > > > > > > > /* RX packet size EWMA. The average packet size is used to determine the packet > > > > * buffer size when refilling RX rings. As the entire RX ring may be refilled > > > > @@ -140,6 +141,21 @@ struct virtnet_rq_dma { > > > > u16 need_sync; > > > > }; > > > > > > > > +struct virtnet_sq_dma { > > > > + union { > > > > + struct virtnet_sq_dma *next; > > > > + void *data; > > > > + }; > > > > + dma_addr_t addr; > > > > + u32 len; > > > > + bool is_tail; > > > > +}; > > > > + > > > > +struct virtnet_sq_dma_head { > > > > + struct virtnet_sq_dma *free; > > > > + struct virtnet_sq_dma *head; > > > > > > Any reason the head must be a pointer instead of a simple index? > > > > > > The head is used for kfree. > > Maybe I need to rename it. > > > > About the index(next) of the virtnet_sq_dma. > > If we use the index, the struct will be: > > > > struct virtnet_sq_dma { > > dma_addr_t addr; > > u32 len; > > > > u32 next; > > void *data > > }; > > > > The size of virtnet_sq_dma is same. > > Ok. > > > > > > > > > > > > +}; > > > > + > > > > /* Internal representation of a send virtqueue */ > > > > struct send_queue { > > > > /* Virtqueue associated with this send _queue */ > > > > @@ -159,6 +175,8 @@ struct send_queue { > > > > > > > > /* Record whether sq is in reset state. */ > > > > bool reset; > > > > + > > > > + struct virtnet_sq_dma_head dmainfo; > > > > }; > > > > > > > > .... > > > > > > + > > > > +static int virtnet_sq_init_dma_mate(struct send_queue *sq) > > > > +{ > > > > + struct virtnet_sq_dma *d; > > > > + int size, i; > > > > + > > > > + size = virtqueue_get_vring_size(sq->vq); > > > > + > > > > + size += MAX_SKB_FRAGS + 2; > > > > > > Is this enough for the case where an indirect descriptor is used? > > > > > > This is for the case, when the ring is full, the xmit_skb is called. > > > > I will add comment. > > Just to make sure we are at the same page. > > I meant, we could have more pending #sg than allocated here. > > For example, we can have up to (vring_size - 2 - MAX_SKB_FRAGS) * > MAX_SKB_FRAGS number of pending sgs? > Oh, my was wrong. But the max value a But shouldn't the maximum value be vring_size * (2 + MAX_SKB_FRAGS)? And for the reason above, we should allocate (vring_size + 1) * (2 + MAX_SKB_FRAGS); Thanks. > Thanks > > > > > Thanks. > > > > > > > > > > Thanks > > > > > >
On Thu, Feb 1, 2024 at 2:01 PM Xuan Zhuo <xuanzhuo@linux.alibaba.com> wrote: > > On Thu, 1 Feb 2024 13:36:46 +0800, Jason Wang <jasowang@redhat.com> wrote: > > On Thu, Feb 1, 2024 at 11:28 AM Xuan Zhuo <xuanzhuo@linux.alibaba.com> wrote: > > > > > > On Wed, 31 Jan 2024 17:12:47 +0800, Jason Wang <jasowang@redhat.com> wrote: > > > > On Tue, Jan 30, 2024 at 7:43 PM Xuan Zhuo <xuanzhuo@linux.alibaba.com> wrote: > > > > > > > > > > If the xsk is enabling, the xsk tx will share the send queue. > > > > > But the xsk requires that the send queue use the premapped mode. > > > > > So the send queue must support premapped mode. > > > > > > > > > > Signed-off-by: Xuan Zhuo <xuanzhuo@linux.alibaba.com> > > > > > --- > > > > > drivers/net/virtio_net.c | 167 ++++++++++++++++++++++++++++++++++++++- > > > > > 1 file changed, 163 insertions(+), 4 deletions(-) > > > > > > > > > > diff --git a/drivers/net/virtio_net.c b/drivers/net/virtio_net.c > > > > > index 226ab830870e..cf0c67380b07 100644 > > > > > --- a/drivers/net/virtio_net.c > > > > > +++ b/drivers/net/virtio_net.c > > > > > @@ -46,6 +46,7 @@ module_param(napi_tx, bool, 0644); > > > > > #define VIRTIO_XDP_REDIR BIT(1) > > > > > > > > > > #define VIRTIO_XDP_FLAG BIT(0) > > > > > +#define VIRTIO_DMA_FLAG BIT(1) > > > > > > > > > > /* RX packet size EWMA. The average packet size is used to determine the packet > > > > > * buffer size when refilling RX rings. As the entire RX ring may be refilled > > > > > @@ -140,6 +141,21 @@ struct virtnet_rq_dma { > > > > > u16 need_sync; > > > > > }; > > > > > > > > > > +struct virtnet_sq_dma { > > > > > + union { > > > > > + struct virtnet_sq_dma *next; > > > > > + void *data; > > > > > + }; > > > > > + dma_addr_t addr; > > > > > + u32 len; > > > > > + bool is_tail; > > > > > +}; > > > > > + > > > > > +struct virtnet_sq_dma_head { > > > > > + struct virtnet_sq_dma *free; > > > > > + struct virtnet_sq_dma *head; > > > > > > > > Any reason the head must be a pointer instead of a simple index? > > > > > > > > > The head is used for kfree. > > > Maybe I need to rename it. > > > > > > About the index(next) of the virtnet_sq_dma. > > > If we use the index, the struct will be: > > > > > > struct virtnet_sq_dma { > > > dma_addr_t addr; > > > u32 len; > > > > > > u32 next; > > > void *data > > > }; > > > > > > The size of virtnet_sq_dma is same. > > > > Ok. > > > > > > > > > > > > > > > > > +}; > > > > > + > > > > > /* Internal representation of a send virtqueue */ > > > > > struct send_queue { > > > > > /* Virtqueue associated with this send _queue */ > > > > > @@ -159,6 +175,8 @@ struct send_queue { > > > > > > > > > > /* Record whether sq is in reset state. */ > > > > > bool reset; > > > > > + > > > > > + struct virtnet_sq_dma_head dmainfo; > > > > > }; > > > > > > > > > > > .... > > > > > > > > + > > > > > +static int virtnet_sq_init_dma_mate(struct send_queue *sq) > > > > > +{ > > > > > + struct virtnet_sq_dma *d; > > > > > + int size, i; > > > > > + > > > > > + size = virtqueue_get_vring_size(sq->vq); > > > > > + > > > > > + size += MAX_SKB_FRAGS + 2; > > > > > > > > Is this enough for the case where an indirect descriptor is used? > > > > > > > > > This is for the case, when the ring is full, the xmit_skb is called. > > > > > > I will add comment. > > > > Just to make sure we are at the same page. > > > > I meant, we could have more pending #sg than allocated here. > > > > For example, we can have up to (vring_size - 2 - MAX_SKB_FRAGS) * > > MAX_SKB_FRAGS number of pending sgs? > > > > Oh, my was wrong. > > But the max value a > But shouldn't the maximum value be vring_size * (2 + MAX_SKB_FRAGS)? This seems to be safer, yes. > > And for the reason above, we should allocate (vring_size + 1) * (2 + MAX_SKB_FRAGS); Then we need to benchmark to see if it has an impact on the performance. Thanks > > Thanks. > > > > Thanks > > > > > > > > Thanks. > > > > > > > > > > > > > > Thanks > > > > > > > > > >
diff --git a/drivers/net/virtio_net.c b/drivers/net/virtio_net.c index 226ab830870e..cf0c67380b07 100644 --- a/drivers/net/virtio_net.c +++ b/drivers/net/virtio_net.c @@ -46,6 +46,7 @@ module_param(napi_tx, bool, 0644); #define VIRTIO_XDP_REDIR BIT(1) #define VIRTIO_XDP_FLAG BIT(0) +#define VIRTIO_DMA_FLAG BIT(1) /* RX packet size EWMA. The average packet size is used to determine the packet * buffer size when refilling RX rings. As the entire RX ring may be refilled @@ -140,6 +141,21 @@ struct virtnet_rq_dma { u16 need_sync; }; +struct virtnet_sq_dma { + union { + struct virtnet_sq_dma *next; + void *data; + }; + dma_addr_t addr; + u32 len; + bool is_tail; +}; + +struct virtnet_sq_dma_head { + struct virtnet_sq_dma *free; + struct virtnet_sq_dma *head; +}; + /* Internal representation of a send virtqueue */ struct send_queue { /* Virtqueue associated with this send _queue */ @@ -159,6 +175,8 @@ struct send_queue { /* Record whether sq is in reset state. */ bool reset; + + struct virtnet_sq_dma_head dmainfo; }; /* Internal representation of a receive virtqueue */ @@ -348,6 +366,131 @@ static struct xdp_frame *ptr_to_xdp(void *ptr) return (struct xdp_frame *)((unsigned long)ptr & ~VIRTIO_XDP_FLAG); } +static inline void *virtnet_sq_unmap(struct send_queue *sq, void *data) +{ + struct virtnet_sq_dma *head, *tail; + + if (!((unsigned long)data & VIRTIO_DMA_FLAG)) + return data; + + head = (void *)((unsigned long)data & ~VIRTIO_DMA_FLAG); + + tail = head; + + while (true) { + virtqueue_dma_unmap_page_attrs(sq->vq, tail->addr, tail->len, + DMA_TO_DEVICE, 0); + + if (tail->is_tail) + break; + + tail = tail->next; + } + + data = tail->data; + tail->is_tail = false; + + tail->next = sq->dmainfo.free; + sq->dmainfo.free = head; + + return data; +} + +static void *virtnet_sq_dma_splice(struct send_queue *sq, + struct virtnet_sq_dma *head, + struct virtnet_sq_dma *tail, + void *data) +{ + sq->dmainfo.free = tail->next; + + tail->is_tail = true; + tail->data = data; + + head = (void *)((unsigned long)head | VIRTIO_DMA_FLAG); + + return head; +} + +static struct virtnet_sq_dma *virtnet_sq_map_sg(struct send_queue *sq, int nents, void *data) +{ + struct virtnet_sq_dma *head, *tail, *p; + struct scatterlist *sg; + dma_addr_t addr; + int i; + + head = sq->dmainfo.free; + p = head; + + tail = NULL; + + for_each_sg(sq->sg, sg, nents, i) { + addr = virtqueue_dma_map_page_attrs(sq->vq, sg_page(sg), + sg->offset, sg->length, + DMA_TO_DEVICE, 0); + if (virtqueue_dma_mapping_error(sq->vq, addr)) + goto err; + + sg->dma_address = addr; + + tail = p; + tail->addr = sg->dma_address; + tail->len = sg->length; + + p = p->next; + } + + return virtnet_sq_dma_splice(sq, head, tail, data); + +err: + if (tail) + virtnet_sq_unmap(sq, virtnet_sq_dma_splice(sq, head, tail, data)); + + return NULL; +} + +static int virtnet_add_outbuf(struct send_queue *sq, u32 num, void *data) +{ + int ret; + + if (sq->vq->premapped) { + data = virtnet_sq_map_sg(sq, num, data); + if (!data) + return -ENOMEM; + } + + ret = virtqueue_add_outbuf(sq->vq, sq->sg, num, data, GFP_ATOMIC); + if (ret && sq->vq->premapped) + virtnet_sq_unmap(sq, data); + + return ret; +} + +static int virtnet_sq_init_dma_mate(struct send_queue *sq) +{ + struct virtnet_sq_dma *d; + int size, i; + + size = virtqueue_get_vring_size(sq->vq); + + size += MAX_SKB_FRAGS + 2; + + sq->dmainfo.head = kcalloc(size, sizeof(*sq->dmainfo.head), GFP_KERNEL); + if (!sq->dmainfo.head) + return -ENOMEM; + + sq->dmainfo.free = sq->dmainfo.head; + + for (i = 0; i < size; ++i) { + d = &sq->dmainfo.head[i]; + d->is_tail = false; + d->next = d + 1; + } + + d->next = NULL; + + return 0; +} + static void __free_old_xmit(struct send_queue *sq, bool in_napi, u64 *bytes, u64 *packets) { @@ -355,6 +498,8 @@ static void __free_old_xmit(struct send_queue *sq, bool in_napi, void *ptr; while ((ptr = virtqueue_get_buf(sq->vq, &len)) != NULL) { + ptr = virtnet_sq_unmap(sq, ptr); + if (!is_xdp_frame(ptr)) { struct sk_buff *skb = ptr; @@ -865,8 +1010,7 @@ static int __virtnet_xdp_xmit_one(struct virtnet_info *vi, skb_frag_size(frag), skb_frag_off(frag)); } - err = virtqueue_add_outbuf(sq->vq, sq->sg, nr_frags + 1, - xdp_to_ptr(xdpf), GFP_ATOMIC); + err = virtnet_add_outbuf(sq, nr_frags + 1, xdp_to_ptr(xdpf)); if (unlikely(err)) return -ENOSPC; /* Caller handle free/refcnt */ @@ -2305,7 +2449,7 @@ static int xmit_skb(struct send_queue *sq, struct sk_buff *skb) return num_sg; num_sg++; } - return virtqueue_add_outbuf(sq->vq, sq->sg, num_sg, skb, GFP_ATOMIC); + return virtnet_add_outbuf(sq, num_sg, skb); } static netdev_tx_t start_xmit(struct sk_buff *skb, struct net_device *dev) @@ -3961,6 +4105,8 @@ static void virtnet_free_queues(struct virtnet_info *vi) for (i = 0; i < vi->max_queue_pairs; i++) { __netif_napi_del(&vi->rq[i].napi); __netif_napi_del(&vi->sq[i].napi); + + kfree(vi->sq[i].dmainfo.head); } /* We called __netif_napi_del(), @@ -4009,6 +4155,14 @@ static void free_receive_page_frags(struct virtnet_info *vi) static void virtnet_sq_free_unused_buf(struct virtqueue *vq, void *buf) { + struct virtnet_info *vi = vq->vdev->priv; + struct send_queue *sq; + int i = vq2rxq(vq); + + sq = &vi->sq[i]; + + buf = virtnet_sq_unmap(sq, buf); + if (!is_xdp_frame(buf)) dev_kfree_skb(buf); else @@ -4121,8 +4275,10 @@ static int virtnet_find_vqs(struct virtnet_info *vi) if (ctx) ctx[rxq2vq(i)] = true; - if (premapped) + if (premapped) { premapped[rxq2vq(i)] = true; + premapped[txq2vq(i)] = true; + } } cfg.nvqs = total_vqs; @@ -4146,6 +4302,9 @@ static int virtnet_find_vqs(struct virtnet_info *vi) vi->rq[i].vq = vqs[rxq2vq(i)]; vi->rq[i].min_buf_len = mergeable_min_buf_len(vi, vi->rq[i].vq); vi->sq[i].vq = vqs[txq2vq(i)]; + + if (vi->sq[i].vq->premapped) + virtnet_sq_init_dma_mate(&vi->sq[i]); } /* run here: ret == 0. */
If the xsk is enabling, the xsk tx will share the send queue. But the xsk requires that the send queue use the premapped mode. So the send queue must support premapped mode. Signed-off-by: Xuan Zhuo <xuanzhuo@linux.alibaba.com> --- drivers/net/virtio_net.c | 167 ++++++++++++++++++++++++++++++++++++++- 1 file changed, 163 insertions(+), 4 deletions(-)