From patchwork Thu May 26 19:42:40 2011 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Shirley Ma X-Patchwork-Id: 821982 Received: from vger.kernel.org (vger.kernel.org [209.132.180.67]) by demeter1.kernel.org (8.14.4/8.14.3) with ESMTP id p4QJhAQ6021177 for ; Thu, 26 May 2011 19:43:10 GMT Received: (majordomo@vger.kernel.org) by vger.kernel.org via listexpand id S1757985Ab1EZTmr (ORCPT ); Thu, 26 May 2011 15:42:47 -0400 Received: from e5.ny.us.ibm.com ([32.97.182.145]:43057 "EHLO e5.ny.us.ibm.com" rhost-flags-OK-OK-OK-OK) by vger.kernel.org with ESMTP id S1753168Ab1EZTmp (ORCPT ); Thu, 26 May 2011 15:42:45 -0400 Received: from d01relay01.pok.ibm.com (d01relay01.pok.ibm.com [9.56.227.233]) by e5.ny.us.ibm.com (8.14.4/8.13.1) with ESMTP id p4QJFUWr017766; Thu, 26 May 2011 15:15:30 -0400 Received: from d01av03.pok.ibm.com (d01av03.pok.ibm.com [9.56.224.217]) by d01relay01.pok.ibm.com (8.13.8/8.13.8/NCO v10.0) with ESMTP id p4QJgi7W130052; Thu, 26 May 2011 15:42:44 -0400 Received: from d01av03.pok.ibm.com (loopback [127.0.0.1]) by d01av03.pok.ibm.com (8.14.4/8.13.1/NCO v10.0 AVout) with ESMTP id p4QFgVcG018111; Thu, 26 May 2011 12:42:32 -0300 Received: from [9.48.55.170] (sig-9-48-55-170.mts.ibm.com [9.48.55.170]) by d01av03.pok.ibm.com (8.14.4/8.13.1/NCO v10.0 AVin) with ESMTP id p4QFgTG2017958; Thu, 26 May 2011 12:42:30 -0300 Subject: [PATCH V6 3/4]macvtap: macvtap TX zero-copy support From: Shirley Ma To: David Miller , mst@redhat.com, Eric Dumazet , Avi Kivity , Arnd Bergmann Cc: netdev@vger.kernel.org, kvm@vger.kernel.org, linux-kernel@vger.kernel.org Date: Thu, 26 May 2011 12:42:40 -0700 Message-ID: <1306438960.5180.39.camel@localhost.localdomain> Mime-Version: 1.0 X-Mailer: Evolution 2.28.3 (2.28.3-1.fc12) Sender: kvm-owner@vger.kernel.org Precedence: bulk List-ID: X-Mailing-List: kvm@vger.kernel.org X-Greylist: IP, sender and recipient auto-whitelisted, not delayed by milter-greylist-4.2.6 (demeter1.kernel.org [140.211.167.41]); Thu, 26 May 2011 19:43:11 +0000 (UTC) Only when buffer size is greater than GOODCOPY_LEN (256), macvtap enables zero-copy. Signed-off-by: Shirley Ma --- drivers/net/macvtap.c | 132 ++++++++++++++++++++++++++++++++++++++++++++---- 1 files changed, 121 insertions(+), 11 deletions(-) -- To unsubscribe from this list: send the line "unsubscribe kvm" in the body of a message to majordomo@vger.kernel.org More majordomo info at http://vger.kernel.org/majordomo-info.html diff --git a/drivers/net/macvtap.c b/drivers/net/macvtap.c index 6696e56..97ad224 100644 --- a/drivers/net/macvtap.c +++ b/drivers/net/macvtap.c @@ -60,6 +60,7 @@ static struct proto macvtap_proto = { */ static dev_t macvtap_major; #define MACVTAP_NUM_DEVS 65536 +#define GOODCOPY_LEN 256 static struct class *macvtap_class; static struct cdev macvtap_cdev; @@ -340,6 +341,7 @@ static int macvtap_open(struct inode *inode, struct file *file) { struct net *net = current->nsproxy->net_ns; struct net_device *dev = dev_get_by_index(net, iminor(inode)); + struct macvlan_dev *vlan = netdev_priv(dev); struct macvtap_queue *q; int err; @@ -369,6 +371,16 @@ static int macvtap_open(struct inode *inode, struct file *file) q->flags = IFF_VNET_HDR | IFF_NO_PI | IFF_TAP; q->vnet_hdr_sz = sizeof(struct virtio_net_hdr); + /* + * so far only KVM virtio_net uses macvtap, enable zero copy between + * guest kernel and host kernel when lower device supports highdma + * and sg + */ + if (vlan) { + if (vlan->lowerdev->features & (NETIF_F_HIGHDMA | NETIF_F_SG)) + sock_set_flag(&q->sk, SOCK_ZEROCOPY); + } + err = macvtap_set_queue(dev, file, q); if (err) sock_put(&q->sk); @@ -433,6 +445,80 @@ static inline struct sk_buff *macvtap_alloc_skb(struct sock *sk, size_t prepad, return skb; } +/* set skb frags from iovec, this can move to core network code for reuse */ +static int zerocopy_sg_from_iovec(struct sk_buff *skb, const struct iovec *from, + int offset, size_t count) +{ + int len = iov_length(from, count) - offset; + int copy = skb_headlen(skb); + int size, offset1 = 0; + int i = 0; + skb_frag_t *f; + + /* Skip over from offset */ + while (count && (offset >= from->iov_len)) { + offset -= from->iov_len; + ++from; + --count; + } + + /* copy up to skb headlen */ + while (count && (copy > 0)) { + size = min_t(unsigned int, copy, from->iov_len - offset); + if (copy_from_user(skb->data + offset1, from->iov_base + offset, + size)) + return -EFAULT; + if (copy > size) { + ++from; + --count; + } + copy -= size; + offset1 += size; + offset = 0; + } + + if (len == offset1) + return 0; + + while (count--) { + struct page *page[MAX_SKB_FRAGS]; + int num_pages; + unsigned long base; + + len = from->iov_len - offset1; + if (!len) { + offset1 = 0; + ++from; + continue; + } + base = (unsigned long)from->iov_base + offset1; + size = ((base & ~PAGE_MASK) + len + ~PAGE_MASK) >> PAGE_SHIFT; + num_pages = get_user_pages_fast(base, size, 0, &page[i]); + if ((num_pages != size) || + (num_pages > MAX_SKB_FRAGS - skb_shinfo(skb)->nr_frags)) + /* put_page is in skb free */ + return -EFAULT; + skb->data_len += len; + skb->len += len; + skb->truesize += len; + atomic_add(len, &skb->sk->sk_wmem_alloc); + while (len) { + f = &skb_shinfo(skb)->frags[i]; + f->page = page[i]; + f->page_offset = base & ~PAGE_MASK; + f->size = min_t(int, len, PAGE_SIZE - f->page_offset); + skb_shinfo(skb)->nr_frags++; + /* increase sk_wmem_alloc */ + base += f->size; + len -= f->size; + i++; + } + offset1 = 0; + ++from; + } + return 0; +} + /* * macvtap_skb_from_vnet_hdr and macvtap_skb_to_vnet_hdr should * be shared with the tun/tap driver. @@ -515,16 +601,18 @@ static int macvtap_skb_to_vnet_hdr(const struct sk_buff *skb, /* Get packet from user space buffer */ -static ssize_t macvtap_get_user(struct macvtap_queue *q, - const struct iovec *iv, size_t count, - int noblock) +static ssize_t macvtap_get_user(struct macvtap_queue *q, struct msghdr *m, + const struct iovec *iv, unsigned long total_len, + size_t count, int noblock) { struct sk_buff *skb; struct macvlan_dev *vlan; - size_t len = count; + unsigned long len = total_len; int err; struct virtio_net_hdr vnet_hdr = { 0 }; int vnet_hdr_len = 0; + int copylen; + bool zerocopy = false; if (q->flags & IFF_VNET_HDR) { vnet_hdr_len = q->vnet_hdr_sz; @@ -552,12 +640,30 @@ static ssize_t macvtap_get_user(struct macvtap_queue *q, if (unlikely(len < ETH_HLEN)) goto err; - skb = macvtap_alloc_skb(&q->sk, NET_IP_ALIGN, len, vnet_hdr.hdr_len, - noblock, &err); + if (m && m->msg_control) + zerocopy = true; + + if (zerocopy) { + /* There are 256 bytes to be copied in skb, so there is enough + * room for skb expand head in case it is used. + * The rest buffer is mapped from userspace. + */ + copylen = vnet_hdr.hdr_len; + if (!copylen) + copylen = GOODCOPY_LEN; + } else + copylen = len; + + skb = macvtap_alloc_skb(&q->sk, NET_IP_ALIGN, copylen, + vnet_hdr.hdr_len, noblock, &err); if (!skb) goto err; - err = skb_copy_datagram_from_iovec(skb, 0, iv, vnet_hdr_len, len); + if (zerocopy) + err = zerocopy_sg_from_iovec(skb, iv, vnet_hdr_len, count); + else + err = skb_copy_datagram_from_iovec(skb, 0, iv, vnet_hdr_len, + len); if (err) goto err_kfree; @@ -573,13 +679,17 @@ static ssize_t macvtap_get_user(struct macvtap_queue *q, rcu_read_lock_bh(); vlan = rcu_dereference_bh(q->vlan); + /* copy skb_ubuf_info for callback when skb has no error */ + if (zerocopy) + memcpy(&skb_shinfo(skb)->ubuf, m->msg_control, + sizeof(struct skb_ubuf_info)); if (vlan) macvlan_start_xmit(skb, vlan->dev); else kfree_skb(skb); rcu_read_unlock_bh(); - return count; + return total_len; err_kfree: kfree_skb(skb); @@ -601,8 +711,8 @@ static ssize_t macvtap_aio_write(struct kiocb *iocb, const struct iovec *iv, ssize_t result = -ENOLINK; struct macvtap_queue *q = file->private_data; - result = macvtap_get_user(q, iv, iov_length(iv, count), - file->f_flags & O_NONBLOCK); + result = macvtap_get_user(q, NULL, iv, iov_length(iv, count), count, + file->f_flags & O_NONBLOCK); return result; } @@ -815,7 +925,7 @@ static int macvtap_sendmsg(struct kiocb *iocb, struct socket *sock, struct msghdr *m, size_t total_len) { struct macvtap_queue *q = container_of(sock, struct macvtap_queue, sock); - return macvtap_get_user(q, m->msg_iov, total_len, + return macvtap_get_user(q, m, m->msg_iov, total_len, m->msg_iovlen, m->msg_flags & MSG_DONTWAIT); }