@@ -522,6 +522,13 @@ enum {
#define SKBFL_ALL_ZEROCOPY (SKBFL_ZEROCOPY_FRAG | SKBFL_PURE_ZEROCOPY | \
SKBFL_DONT_ORPHAN | SKBFL_MANAGED_FRAG_REFS)
+enum {
+ /* The caller holds a reference during the submission so the ubuf won't
+ * be freed until we return.
+ */
+ UARGFL_CALLER_PINNED = BIT(0),
+};
+
/*
* The callback notifies userspace to release buffers when skb DMA is done in
* lower device, the skb last reference should be 0 when calling this.
@@ -1229,7 +1229,8 @@ int tcp_sendmsg_locked(struct sock *sk, struct msghdr *msg, size_t size)
if (msg->msg_ubuf) {
uarg = msg->msg_ubuf;
- net_zcopy_get(uarg);
+ if (!(uarg->flags & UARGFL_CALLER_PINNED))
+ net_zcopy_get(uarg);
zc = sk->sk_route_caps & NETIF_F_SG;
} else if (sock_flag(sk, SOCK_ZEROCOPY)) {
uarg = msg_zerocopy_realloc(sk, size, skb_zcopy(skb));
@@ -1455,7 +1456,8 @@ int tcp_sendmsg_locked(struct sock *sk, struct msghdr *msg, size_t size)
tcp_push(sk, flags, mss_now, tp->nonagle, size_goal);
}
out_nopush:
- net_zcopy_put(uarg);
+ if (uarg && !(uarg->flags & UARGFL_CALLER_PINNED))
+ net_zcopy_put(uarg);
return copied + copied_syn;
do_error:
@@ -1464,7 +1466,8 @@ int tcp_sendmsg_locked(struct sock *sk, struct msghdr *msg, size_t size)
if (copied + copied_syn)
goto out;
out_err:
- net_zcopy_put_abort(uarg, true);
+ if (uarg && !(uarg->flags & UARGFL_CALLER_PINNED))
+ net_zcopy_put_abort(uarg, true);
err = sk_stream_error(sk, flags, err);
/* make sure we wake any epoll edge trigger waiter */
if (unlikely(tcp_rtx_and_write_queues_empty(sk) && err == -EAGAIN)) {
Add UARGFL_CALLER_PINNED letting protocols know that the caller holds a reference to the ubuf_info and so it doesn't need additional refcounting for purposes of keeping it alive. With that TCP can save a refcount put/get pair per send when used with ->msg_ubuf. Signed-off-by: Pavel Begunkov <asml.silence@gmail.com> --- include/linux/skbuff.h | 7 +++++++ net/ipv4/tcp.c | 9 ++++++--- 2 files changed, 13 insertions(+), 3 deletions(-)