diff mbox series

[net-next,v4,2/3] sock: add MSG_ZEROCOPY notification mechanism based on msg_control

Message ID 20240513211755.2751955-3-zijianzhang@bytedance.com (mailing list archive)
State Deferred
Delegated to: Netdev Maintainers
Headers show
Series net: A lightweight zero-copy notification | expand

Checks

Context Check Description
netdev/series_format success Posting correctly formatted
netdev/tree_selection success Clearly marked for net-next, async
netdev/ynl success Generated files up to date; no warnings/errors; no diff in generated;
netdev/fixes_present success Fixes tag not required for -next series
netdev/header_inline success No static functions without inline keyword in header files
netdev/build_32bit fail Errors and warnings before: 6356 this patch: 6357
netdev/build_tools success Errors and warnings before: 0 this patch: 0
netdev/cc_maintainers warning 17 maintainers not CCed: James.Bottomley@HansenPartnership.com linux-alpha@vger.kernel.org deller@gmx.de mattst88@gmail.com richard.henderson@linaro.org kuba@kernel.org ink@jurassic.park.msu.ru arnd@arndb.de pabeni@redhat.com brauner@kernel.org linux-arch@vger.kernel.org sparclinux@vger.kernel.org andreas@gaisler.com alexander@mihalicyn.com tsbogend@alpha.franken.de linux-parisc@vger.kernel.org linux-mips@vger.kernel.org
netdev/build_clang fail Errors and warnings before: 2060 this patch: 2062
netdev/verify_signedoff success Signed-off-by tag matches author and committer
netdev/deprecated_api success None detected
netdev/check_selftest success No net selftest shell script
netdev/verify_fixes success No Fixes tag
netdev/build_allmodconfig_warn fail Errors and warnings before: 16204 this patch: 16205
netdev/checkpatch warning WARNING: line length of 81 exceeds 80 columns WARNING: line length of 94 exceeds 80 columns
netdev/build_clang_rust success No Rust files in patch. Skipping build
netdev/kdoc success Errors and warnings before: 9 this patch: 9
netdev/source_inline success Was 0 now: 0

Commit Message

Zijian Zhang May 13, 2024, 9:17 p.m. UTC
From: Zijian Zhang <zijianzhang@bytedance.com>

The MSG_ZEROCOPY flag enables copy avoidance for socket send calls.
However, zerocopy is not a free lunch. Apart from the management of user
pages, the combination of poll + recvmsg to receive notifications incurs
unignorable overhead in the applications. The overhead of such sometimes
might be more than the CPU savings from zerocopy. We try to solve this
problem with a new notification mechanism based on msgcontrol.
This new mechanism aims to reduce the overhead associated with receiving
notifications by embedding them directly into user arguments passed with
each sendmsg control message. By doing so, we can significantly reduce
the complexity and overhead for managing notifications. In an ideal
pattern, the user will keep calling sendmsg with SCM_ZC_NOTIFICATION
msg_control, and the notification will be delivered as soon as possible.

Signed-off-by: Zijian Zhang <zijianzhang@bytedance.com>
Signed-off-by: Xiaochun Lu <xiaochun.lu@bytedance.com>
---
 arch/alpha/include/uapi/asm/socket.h  |  2 +
 arch/mips/include/uapi/asm/socket.h   |  2 +
 arch/parisc/include/uapi/asm/socket.h |  2 +
 arch/sparc/include/uapi/asm/socket.h  |  2 +
 include/uapi/asm-generic/socket.h     |  2 +
 include/uapi/linux/socket.h           | 10 +++++
 net/core/sock.c                       | 65 +++++++++++++++++++++++++++
 7 files changed, 85 insertions(+)

Comments

kernel test robot May 14, 2024, 3:40 a.m. UTC | #1
Hi,

kernel test robot noticed the following build warnings:

[auto build test WARNING on net-next/main]

url:    https://github.com/intel-lab-lkp/linux/commits/zijianzhang-bytedance-com/selftests-fix-OOM-problem-in-msg_zerocopy-selftest/20240514-051839
base:   net-next/main
patch link:    https://lore.kernel.org/r/20240513211755.2751955-3-zijianzhang%40bytedance.com
patch subject: [PATCH net-next v4 2/3] sock: add MSG_ZEROCOPY notification mechanism based on msg_control
config: openrisc-defconfig (https://download.01.org/0day-ci/archive/20240514/202405141135.xZDS5YLg-lkp@intel.com/config)
compiler: or1k-linux-gcc (GCC) 13.2.0
reproduce (this is a W=1 build): (https://download.01.org/0day-ci/archive/20240514/202405141135.xZDS5YLg-lkp@intel.com/reproduce)

If you fix the issue in a separate patch/commit (i.e. not just a new version of
the same patch/commit), kindly add following tags
| Reported-by: kernel test robot <lkp@intel.com>
| Closes: https://lore.kernel.org/oe-kbuild-all/202405141135.xZDS5YLg-lkp@intel.com/

All warnings (new ones prefixed by >>):

   net/core/sock.c: In function '__sock_cmsg_send':
>> net/core/sock.c:2850:39: warning: unused variable 'tmp' [-Wunused-variable]
    2850 |                 struct sk_buff *skb, *tmp;
         |                                       ^~~


vim +/tmp +2850 net/core/sock.c

  2807	
  2808	int __sock_cmsg_send(struct sock *sk, struct cmsghdr *cmsg,
  2809			     struct sockcm_cookie *sockc)
  2810	{
  2811		u32 tsflags;
  2812	
  2813		switch (cmsg->cmsg_type) {
  2814		case SO_MARK:
  2815			if (!ns_capable(sock_net(sk)->user_ns, CAP_NET_RAW) &&
  2816			    !ns_capable(sock_net(sk)->user_ns, CAP_NET_ADMIN))
  2817				return -EPERM;
  2818			if (cmsg->cmsg_len != CMSG_LEN(sizeof(u32)))
  2819				return -EINVAL;
  2820			sockc->mark = *(u32 *)CMSG_DATA(cmsg);
  2821			break;
  2822		case SO_TIMESTAMPING_OLD:
  2823		case SO_TIMESTAMPING_NEW:
  2824			if (cmsg->cmsg_len != CMSG_LEN(sizeof(u32)))
  2825				return -EINVAL;
  2826	
  2827			tsflags = *(u32 *)CMSG_DATA(cmsg);
  2828			if (tsflags & ~SOF_TIMESTAMPING_TX_RECORD_MASK)
  2829				return -EINVAL;
  2830	
  2831			sockc->tsflags &= ~SOF_TIMESTAMPING_TX_RECORD_MASK;
  2832			sockc->tsflags |= tsflags;
  2833			break;
  2834		case SCM_TXTIME:
  2835			if (!sock_flag(sk, SOCK_TXTIME))
  2836				return -EINVAL;
  2837			if (cmsg->cmsg_len != CMSG_LEN(sizeof(u64)))
  2838				return -EINVAL;
  2839			sockc->transmit_time = get_unaligned((u64 *)CMSG_DATA(cmsg));
  2840			break;
  2841		/* SCM_RIGHTS and SCM_CREDENTIALS are semantically in SOL_UNIX. */
  2842		case SCM_RIGHTS:
  2843		case SCM_CREDENTIALS:
  2844			break;
  2845		case SCM_ZC_NOTIFICATION: {
  2846			struct zc_info_elem zc_info_kern[SOCK_ZC_INFO_MAX];
  2847			int cmsg_data_len, zc_info_elem_num;
  2848			struct sk_buff_head *q, local_q;
  2849			struct sock_exterr_skb *serr;
> 2850			struct sk_buff *skb, *tmp;
  2851			void __user	*usr_addr;
  2852			unsigned long flags;
  2853			int ret, i = 0;
  2854	
  2855			if (!sock_flag(sk, SOCK_ZEROCOPY) || sk->sk_family == PF_RDS)
  2856				return -EINVAL;
  2857	
  2858			cmsg_data_len = cmsg->cmsg_len - sizeof(struct cmsghdr);
  2859			if (cmsg_data_len % sizeof(struct zc_info_elem))
  2860				return -EINVAL;
  2861	
  2862			zc_info_elem_num = cmsg_data_len / sizeof(struct zc_info_elem);
  2863			if (!zc_info_elem_num || zc_info_elem_num > SOCK_ZC_INFO_MAX)
  2864				return -EINVAL;
  2865	
  2866			if (in_compat_syscall())
  2867				usr_addr = compat_ptr(*(compat_uptr_t *)CMSG_DATA(cmsg));
  2868			else
  2869				usr_addr = (void __user *)*(void **)CMSG_DATA(cmsg);
  2870			if (!access_ok(usr_addr, cmsg_data_len))
  2871				return -EFAULT;
  2872	
  2873			q = &sk->sk_error_queue;
  2874			skb_queue_head_init(&local_q);
  2875			spin_lock_irqsave(&q->lock, flags);
  2876			skb = skb_peek(q);
  2877			while (skb && i < zc_info_elem_num) {
  2878				struct sk_buff *skb_next = skb_peek_next(skb, q);
  2879	
  2880				serr = SKB_EXT_ERR(skb);
  2881				if (serr->ee.ee_errno == 0 &&
  2882				    serr->ee.ee_origin == SO_EE_ORIGIN_ZEROCOPY) {
  2883					zc_info_kern[i].hi = serr->ee.ee_data;
  2884					zc_info_kern[i].lo = serr->ee.ee_info;
  2885					zc_info_kern[i].zerocopy = !(serr->ee.ee_code
  2886									& SO_EE_CODE_ZEROCOPY_COPIED);
  2887					__skb_unlink(skb, q);
  2888					__skb_queue_tail(&local_q, skb);
  2889					i++;
  2890				}
  2891				skb = skb_next;
  2892			}
  2893			spin_unlock_irqrestore(&q->lock, flags);
  2894	
  2895			ret = copy_to_user(usr_addr,
  2896					   zc_info_kern,
  2897						i * sizeof(struct zc_info_elem));
  2898	
  2899			if (unlikely(ret)) {
  2900				spin_lock_irqsave(&q->lock, flags);
  2901				skb_queue_splice_init(&local_q, q);
  2902				spin_unlock_irqrestore(&q->lock, flags);
  2903				return -EFAULT;
  2904			}
  2905	
  2906			while ((skb = __skb_dequeue(&local_q)))
  2907				consume_skb(skb);
  2908			break;
  2909		}
  2910		default:
  2911			return -EINVAL;
  2912		}
  2913		return 0;
  2914	}
  2915	EXPORT_SYMBOL(__sock_cmsg_send);
  2916
diff mbox series

Patch

diff --git a/arch/alpha/include/uapi/asm/socket.h b/arch/alpha/include/uapi/asm/socket.h
index e94f621903fe..7761a4e0ea2c 100644
--- a/arch/alpha/include/uapi/asm/socket.h
+++ b/arch/alpha/include/uapi/asm/socket.h
@@ -140,6 +140,8 @@ 
 #define SO_PASSPIDFD		76
 #define SO_PEERPIDFD		77
 
+#define SCM_ZC_NOTIFICATION 78
+
 #if !defined(__KERNEL__)
 
 #if __BITS_PER_LONG == 64
diff --git a/arch/mips/include/uapi/asm/socket.h b/arch/mips/include/uapi/asm/socket.h
index 60ebaed28a4c..89edc51380f0 100644
--- a/arch/mips/include/uapi/asm/socket.h
+++ b/arch/mips/include/uapi/asm/socket.h
@@ -151,6 +151,8 @@ 
 #define SO_PASSPIDFD		76
 #define SO_PEERPIDFD		77
 
+#define SCM_ZC_NOTIFICATION 78
+
 #if !defined(__KERNEL__)
 
 #if __BITS_PER_LONG == 64
diff --git a/arch/parisc/include/uapi/asm/socket.h b/arch/parisc/include/uapi/asm/socket.h
index be264c2b1a11..2911b43e6a9d 100644
--- a/arch/parisc/include/uapi/asm/socket.h
+++ b/arch/parisc/include/uapi/asm/socket.h
@@ -132,6 +132,8 @@ 
 #define SO_PASSPIDFD		0x404A
 #define SO_PEERPIDFD		0x404B
 
+#define SCM_ZC_NOTIFICATION 0x404C
+
 #if !defined(__KERNEL__)
 
 #if __BITS_PER_LONG == 64
diff --git a/arch/sparc/include/uapi/asm/socket.h b/arch/sparc/include/uapi/asm/socket.h
index 682da3714686..dc045e87cc8e 100644
--- a/arch/sparc/include/uapi/asm/socket.h
+++ b/arch/sparc/include/uapi/asm/socket.h
@@ -133,6 +133,8 @@ 
 #define SO_PASSPIDFD             0x0055
 #define SO_PEERPIDFD             0x0056
 
+#define SCM_ZC_NOTIFICATION 0x0057
+
 #if !defined(__KERNEL__)
 
 
diff --git a/include/uapi/asm-generic/socket.h b/include/uapi/asm-generic/socket.h
index 8ce8a39a1e5f..7474c8a244bc 100644
--- a/include/uapi/asm-generic/socket.h
+++ b/include/uapi/asm-generic/socket.h
@@ -135,6 +135,8 @@ 
 #define SO_PASSPIDFD		76
 #define SO_PEERPIDFD		77
 
+#define SCM_ZC_NOTIFICATION 78
+
 #if !defined(__KERNEL__)
 
 #if __BITS_PER_LONG == 64 || (defined(__x86_64__) && defined(__ILP32__))
diff --git a/include/uapi/linux/socket.h b/include/uapi/linux/socket.h
index d3fcd3b5ec53..16911bca45f3 100644
--- a/include/uapi/linux/socket.h
+++ b/include/uapi/linux/socket.h
@@ -2,6 +2,8 @@ 
 #ifndef _UAPI_LINUX_SOCKET_H
 #define _UAPI_LINUX_SOCKET_H
 
+#include <linux/types.h>
+
 /*
  * Desired design of maximum size and alignment (see RFC2553)
  */
@@ -35,4 +37,12 @@  struct __kernel_sockaddr_storage {
 #define SOCK_TXREHASH_DISABLED	0
 #define SOCK_TXREHASH_ENABLED	1
 
+#define SOCK_ZC_INFO_MAX 64
+
+struct zc_info_elem {
+	__u32 lo;
+	__u32 hi;
+	__u8 zerocopy;
+};
+
 #endif /* _UAPI_LINUX_SOCKET_H */
diff --git a/net/core/sock.c b/net/core/sock.c
index 8d6e638b5426..eafa98c70d9a 100644
--- a/net/core/sock.c
+++ b/net/core/sock.c
@@ -2842,6 +2842,71 @@  int __sock_cmsg_send(struct sock *sk, struct cmsghdr *cmsg,
 	case SCM_RIGHTS:
 	case SCM_CREDENTIALS:
 		break;
+	case SCM_ZC_NOTIFICATION: {
+		struct zc_info_elem zc_info_kern[SOCK_ZC_INFO_MAX];
+		int cmsg_data_len, zc_info_elem_num;
+		struct sk_buff_head *q, local_q;
+		struct sock_exterr_skb *serr;
+		struct sk_buff *skb, *tmp;
+		void __user	*usr_addr;
+		unsigned long flags;
+		int ret, i = 0;
+
+		if (!sock_flag(sk, SOCK_ZEROCOPY) || sk->sk_family == PF_RDS)
+			return -EINVAL;
+
+		cmsg_data_len = cmsg->cmsg_len - sizeof(struct cmsghdr);
+		if (cmsg_data_len % sizeof(struct zc_info_elem))
+			return -EINVAL;
+
+		zc_info_elem_num = cmsg_data_len / sizeof(struct zc_info_elem);
+		if (!zc_info_elem_num || zc_info_elem_num > SOCK_ZC_INFO_MAX)
+			return -EINVAL;
+
+		if (in_compat_syscall())
+			usr_addr = compat_ptr(*(compat_uptr_t *)CMSG_DATA(cmsg));
+		else
+			usr_addr = (void __user *)*(void **)CMSG_DATA(cmsg);
+		if (!access_ok(usr_addr, cmsg_data_len))
+			return -EFAULT;
+
+		q = &sk->sk_error_queue;
+		skb_queue_head_init(&local_q);
+		spin_lock_irqsave(&q->lock, flags);
+		skb = skb_peek(q);
+		while (skb && i < zc_info_elem_num) {
+			struct sk_buff *skb_next = skb_peek_next(skb, q);
+
+			serr = SKB_EXT_ERR(skb);
+			if (serr->ee.ee_errno == 0 &&
+			    serr->ee.ee_origin == SO_EE_ORIGIN_ZEROCOPY) {
+				zc_info_kern[i].hi = serr->ee.ee_data;
+				zc_info_kern[i].lo = serr->ee.ee_info;
+				zc_info_kern[i].zerocopy = !(serr->ee.ee_code
+								& SO_EE_CODE_ZEROCOPY_COPIED);
+				__skb_unlink(skb, q);
+				__skb_queue_tail(&local_q, skb);
+				i++;
+			}
+			skb = skb_next;
+		}
+		spin_unlock_irqrestore(&q->lock, flags);
+
+		ret = copy_to_user(usr_addr,
+				   zc_info_kern,
+					i * sizeof(struct zc_info_elem));
+
+		if (unlikely(ret)) {
+			spin_lock_irqsave(&q->lock, flags);
+			skb_queue_splice_init(&local_q, q);
+			spin_unlock_irqrestore(&q->lock, flags);
+			return -EFAULT;
+		}
+
+		while ((skb = __skb_dequeue(&local_q)))
+			consume_skb(skb);
+		break;
+	}
 	default:
 		return -EINVAL;
 	}