| Message ID | 20190717113030.163499-3-sgarzare@redhat.com (mailing list archive) |
|---|---|
| State | New, archived |
| Headers | show |
| Series | vsock/virtio: optimizations to increase the throughput | expand |
On Wed, Jul 17, 2019 at 01:30:27PM +0200, Stefano Garzarella wrote: > In order to reduce the number of credit update messages, > we send them only when the space available seen by the > transmitter is less than VIRTIO_VSOCK_MAX_PKT_BUF_SIZE. > > Signed-off-by: Stefano Garzarella <sgarzare@redhat.com> > --- > include/linux/virtio_vsock.h | 1 + > net/vmw_vsock/virtio_transport_common.c | 16 +++++++++++++--- > 2 files changed, 14 insertions(+), 3 deletions(-) It's an arbitrary limit but the risk of regressions is low since the credit update traffic was so excessive: Reviewed-by: Stefan Hajnoczi <stefanha@redhat.com>
On Wed, Jul 17, 2019 at 01:30:27PM +0200, Stefano Garzarella wrote: > In order to reduce the number of credit update messages, > we send them only when the space available seen by the > transmitter is less than VIRTIO_VSOCK_MAX_PKT_BUF_SIZE. > > Signed-off-by: Stefano Garzarella <sgarzare@redhat.com> > --- > include/linux/virtio_vsock.h | 1 + > net/vmw_vsock/virtio_transport_common.c | 16 +++++++++++++--- > 2 files changed, 14 insertions(+), 3 deletions(-) > > diff --git a/include/linux/virtio_vsock.h b/include/linux/virtio_vsock.h > index 7d973903f52e..49fc9d20bc43 100644 > --- a/include/linux/virtio_vsock.h > +++ b/include/linux/virtio_vsock.h > @@ -41,6 +41,7 @@ struct virtio_vsock_sock { > > /* Protected by rx_lock */ > u32 fwd_cnt; > + u32 last_fwd_cnt; > u32 rx_bytes; > struct list_head rx_queue; > }; > diff --git a/net/vmw_vsock/virtio_transport_common.c b/net/vmw_vsock/virtio_transport_common.c > index 095221f94786..a85559d4d974 100644 > --- a/net/vmw_vsock/virtio_transport_common.c > +++ b/net/vmw_vsock/virtio_transport_common.c > @@ -211,6 +211,7 @@ static void virtio_transport_dec_rx_pkt(struct virtio_vsock_sock *vvs, > void virtio_transport_inc_tx_pkt(struct virtio_vsock_sock *vvs, struct virtio_vsock_pkt *pkt) > { > spin_lock_bh(&vvs->tx_lock); > + vvs->last_fwd_cnt = vvs->fwd_cnt; > pkt->hdr.fwd_cnt = cpu_to_le32(vvs->fwd_cnt); > pkt->hdr.buf_alloc = cpu_to_le32(vvs->buf_alloc); > spin_unlock_bh(&vvs->tx_lock); > @@ -261,6 +262,7 @@ virtio_transport_stream_do_dequeue(struct vsock_sock *vsk, > struct virtio_vsock_sock *vvs = vsk->trans; > struct virtio_vsock_pkt *pkt; > size_t bytes, total = 0; > + u32 free_space; > int err = -EFAULT; > > spin_lock_bh(&vvs->rx_lock); > @@ -291,11 +293,19 @@ virtio_transport_stream_do_dequeue(struct vsock_sock *vsk, > virtio_transport_free_pkt(pkt); > } > } > + > + free_space = vvs->buf_alloc - (vvs->fwd_cnt - vvs->last_fwd_cnt); > + > spin_unlock_bh(&vvs->rx_lock); > > - /* Send a credit pkt to peer */ > - virtio_transport_send_credit_update(vsk, VIRTIO_VSOCK_TYPE_STREAM, > - NULL); > + /* We send a credit update only when the space available seen > + * by the transmitter is less than VIRTIO_VSOCK_MAX_PKT_BUF_SIZE This is just repeating what code does though. Please include the *reason* for the condition. E.g. here's a better comment: /* To reduce number of credit update messages, * don't update credits as long as lots of space is available. * Note: the limit chosen here is arbitrary. Setting the limit * too high causes extra messages. Too low causes transmitter * stalls. As stalls are in theory more expensive than extra * messages, we set the limit to a high value. TODO: experiment * with different values. */ > + */ > + if (free_space < VIRTIO_VSOCK_MAX_PKT_BUF_SIZE) { > + virtio_transport_send_credit_update(vsk, > + VIRTIO_VSOCK_TYPE_STREAM, > + NULL); > + } > > return total; > > -- > 2.20.1
On Tue, Sep 03, 2019 at 12:38:02AM -0400, Michael S. Tsirkin wrote: > On Wed, Jul 17, 2019 at 01:30:27PM +0200, Stefano Garzarella wrote: > > In order to reduce the number of credit update messages, > > we send them only when the space available seen by the > > transmitter is less than VIRTIO_VSOCK_MAX_PKT_BUF_SIZE. > > > > Signed-off-by: Stefano Garzarella <sgarzare@redhat.com> > > --- > > include/linux/virtio_vsock.h | 1 + > > net/vmw_vsock/virtio_transport_common.c | 16 +++++++++++++--- > > 2 files changed, 14 insertions(+), 3 deletions(-) > > > > diff --git a/include/linux/virtio_vsock.h b/include/linux/virtio_vsock.h > > index 7d973903f52e..49fc9d20bc43 100644 > > --- a/include/linux/virtio_vsock.h > > +++ b/include/linux/virtio_vsock.h > > @@ -41,6 +41,7 @@ struct virtio_vsock_sock { > > > > /* Protected by rx_lock */ > > u32 fwd_cnt; > > + u32 last_fwd_cnt; > > u32 rx_bytes; > > struct list_head rx_queue; > > }; > > diff --git a/net/vmw_vsock/virtio_transport_common.c b/net/vmw_vsock/virtio_transport_common.c > > index 095221f94786..a85559d4d974 100644 > > --- a/net/vmw_vsock/virtio_transport_common.c > > +++ b/net/vmw_vsock/virtio_transport_common.c > > @@ -211,6 +211,7 @@ static void virtio_transport_dec_rx_pkt(struct virtio_vsock_sock *vvs, > > void virtio_transport_inc_tx_pkt(struct virtio_vsock_sock *vvs, struct virtio_vsock_pkt *pkt) > > { > > spin_lock_bh(&vvs->tx_lock); > > + vvs->last_fwd_cnt = vvs->fwd_cnt; > > pkt->hdr.fwd_cnt = cpu_to_le32(vvs->fwd_cnt); > > pkt->hdr.buf_alloc = cpu_to_le32(vvs->buf_alloc); > > spin_unlock_bh(&vvs->tx_lock); > > @@ -261,6 +262,7 @@ virtio_transport_stream_do_dequeue(struct vsock_sock *vsk, > > struct virtio_vsock_sock *vvs = vsk->trans; > > struct virtio_vsock_pkt *pkt; > > size_t bytes, total = 0; > > + u32 free_space; > > int err = -EFAULT; > > > > spin_lock_bh(&vvs->rx_lock); > > @@ -291,11 +293,19 @@ virtio_transport_stream_do_dequeue(struct vsock_sock *vsk, > > virtio_transport_free_pkt(pkt); > > } > > } > > + > > + free_space = vvs->buf_alloc - (vvs->fwd_cnt - vvs->last_fwd_cnt); > > + > > spin_unlock_bh(&vvs->rx_lock); > > > > - /* Send a credit pkt to peer */ > > - virtio_transport_send_credit_update(vsk, VIRTIO_VSOCK_TYPE_STREAM, > > - NULL); > > + /* We send a credit update only when the space available seen > > + * by the transmitter is less than VIRTIO_VSOCK_MAX_PKT_BUF_SIZE > > This is just repeating what code does though. > Please include the *reason* for the condition. > E.g. here's a better comment: > > /* To reduce number of credit update messages, > * don't update credits as long as lots of space is available. > * Note: the limit chosen here is arbitrary. Setting the limit > * too high causes extra messages. Too low causes transmitter > * stalls. As stalls are in theory more expensive than extra > * messages, we set the limit to a high value. TODO: experiment > * with different values. > */ > Yes, it is better, sorry for that. I'll try to avoid unnecessary comments, explaining the reason for certain changes. Since this patch is already queued in net-next, should I send another patch to fix the comment? Thanks, Stefano
On Tue, Sep 03, 2019 at 09:31:20AM +0200, Stefano Garzarella wrote: > On Tue, Sep 03, 2019 at 12:38:02AM -0400, Michael S. Tsirkin wrote: > > On Wed, Jul 17, 2019 at 01:30:27PM +0200, Stefano Garzarella wrote: > > > In order to reduce the number of credit update messages, > > > we send them only when the space available seen by the > > > transmitter is less than VIRTIO_VSOCK_MAX_PKT_BUF_SIZE. > > > > > > Signed-off-by: Stefano Garzarella <sgarzare@redhat.com> > > > --- > > > include/linux/virtio_vsock.h | 1 + > > > net/vmw_vsock/virtio_transport_common.c | 16 +++++++++++++--- > > > 2 files changed, 14 insertions(+), 3 deletions(-) > > > > > > diff --git a/include/linux/virtio_vsock.h b/include/linux/virtio_vsock.h > > > index 7d973903f52e..49fc9d20bc43 100644 > > > --- a/include/linux/virtio_vsock.h > > > +++ b/include/linux/virtio_vsock.h > > > @@ -41,6 +41,7 @@ struct virtio_vsock_sock { > > > > > > /* Protected by rx_lock */ > > > u32 fwd_cnt; > > > + u32 last_fwd_cnt; > > > u32 rx_bytes; > > > struct list_head rx_queue; > > > }; > > > diff --git a/net/vmw_vsock/virtio_transport_common.c b/net/vmw_vsock/virtio_transport_common.c > > > index 095221f94786..a85559d4d974 100644 > > > --- a/net/vmw_vsock/virtio_transport_common.c > > > +++ b/net/vmw_vsock/virtio_transport_common.c > > > @@ -211,6 +211,7 @@ static void virtio_transport_dec_rx_pkt(struct virtio_vsock_sock *vvs, > > > void virtio_transport_inc_tx_pkt(struct virtio_vsock_sock *vvs, struct virtio_vsock_pkt *pkt) > > > { > > > spin_lock_bh(&vvs->tx_lock); > > > + vvs->last_fwd_cnt = vvs->fwd_cnt; > > > pkt->hdr.fwd_cnt = cpu_to_le32(vvs->fwd_cnt); > > > pkt->hdr.buf_alloc = cpu_to_le32(vvs->buf_alloc); > > > spin_unlock_bh(&vvs->tx_lock); > > > @@ -261,6 +262,7 @@ virtio_transport_stream_do_dequeue(struct vsock_sock *vsk, > > > struct virtio_vsock_sock *vvs = vsk->trans; > > > struct virtio_vsock_pkt *pkt; > > > size_t bytes, total = 0; > > > + u32 free_space; > > > int err = -EFAULT; > > > > > > spin_lock_bh(&vvs->rx_lock); > > > @@ -291,11 +293,19 @@ virtio_transport_stream_do_dequeue(struct vsock_sock *vsk, > > > virtio_transport_free_pkt(pkt); > > > } > > > } > > > + > > > + free_space = vvs->buf_alloc - (vvs->fwd_cnt - vvs->last_fwd_cnt); > > > + > > > spin_unlock_bh(&vvs->rx_lock); > > > > > > - /* Send a credit pkt to peer */ > > > - virtio_transport_send_credit_update(vsk, VIRTIO_VSOCK_TYPE_STREAM, > > > - NULL); > > > + /* We send a credit update only when the space available seen > > > + * by the transmitter is less than VIRTIO_VSOCK_MAX_PKT_BUF_SIZE > > > > This is just repeating what code does though. > > Please include the *reason* for the condition. > > E.g. here's a better comment: > > > > /* To reduce number of credit update messages, > > * don't update credits as long as lots of space is available. > > * Note: the limit chosen here is arbitrary. Setting the limit > > * too high causes extra messages. Too low causes transmitter > > * stalls. As stalls are in theory more expensive than extra > > * messages, we set the limit to a high value. TODO: experiment > > * with different values. > > */ > > > > Yes, it is better, sorry for that. I'll try to avoid unnecessary comments, > explaining the reason for certain changes. > > Since this patch is already queued in net-next, should I send another > patch to fix the comment? > > Thanks, > Stefano I just sent a patch like that, pls ack it.
diff --git a/include/linux/virtio_vsock.h b/include/linux/virtio_vsock.h index 7d973903f52e..49fc9d20bc43 100644 --- a/include/linux/virtio_vsock.h +++ b/include/linux/virtio_vsock.h @@ -41,6 +41,7 @@ struct virtio_vsock_sock { /* Protected by rx_lock */ u32 fwd_cnt; + u32 last_fwd_cnt; u32 rx_bytes; struct list_head rx_queue; }; diff --git a/net/vmw_vsock/virtio_transport_common.c b/net/vmw_vsock/virtio_transport_common.c index 095221f94786..a85559d4d974 100644 --- a/net/vmw_vsock/virtio_transport_common.c +++ b/net/vmw_vsock/virtio_transport_common.c @@ -211,6 +211,7 @@ static void virtio_transport_dec_rx_pkt(struct virtio_vsock_sock *vvs, void virtio_transport_inc_tx_pkt(struct virtio_vsock_sock *vvs, struct virtio_vsock_pkt *pkt) { spin_lock_bh(&vvs->tx_lock); + vvs->last_fwd_cnt = vvs->fwd_cnt; pkt->hdr.fwd_cnt = cpu_to_le32(vvs->fwd_cnt); pkt->hdr.buf_alloc = cpu_to_le32(vvs->buf_alloc); spin_unlock_bh(&vvs->tx_lock); @@ -261,6 +262,7 @@ virtio_transport_stream_do_dequeue(struct vsock_sock *vsk, struct virtio_vsock_sock *vvs = vsk->trans; struct virtio_vsock_pkt *pkt; size_t bytes, total = 0; + u32 free_space; int err = -EFAULT; spin_lock_bh(&vvs->rx_lock); @@ -291,11 +293,19 @@ virtio_transport_stream_do_dequeue(struct vsock_sock *vsk, virtio_transport_free_pkt(pkt); } } + + free_space = vvs->buf_alloc - (vvs->fwd_cnt - vvs->last_fwd_cnt); + spin_unlock_bh(&vvs->rx_lock); - /* Send a credit pkt to peer */ - virtio_transport_send_credit_update(vsk, VIRTIO_VSOCK_TYPE_STREAM, - NULL); + /* We send a credit update only when the space available seen + * by the transmitter is less than VIRTIO_VSOCK_MAX_PKT_BUF_SIZE + */ + if (free_space < VIRTIO_VSOCK_MAX_PKT_BUF_SIZE) { + virtio_transport_send_credit_update(vsk, + VIRTIO_VSOCK_TYPE_STREAM, + NULL); + } return total;
In order to reduce the number of credit update messages, we send them only when the space available seen by the transmitter is less than VIRTIO_VSOCK_MAX_PKT_BUF_SIZE. Signed-off-by: Stefano Garzarella <sgarzare@redhat.com> --- include/linux/virtio_vsock.h | 1 + net/vmw_vsock/virtio_transport_common.c | 16 +++++++++++++--- 2 files changed, 14 insertions(+), 3 deletions(-)