@@ -85,6 +85,7 @@ enum {
NETIF_F_HW_MACSEC_BIT, /* Offload MACsec operations */
NETIF_F_GRO_UDP_FWD_BIT, /* Allow UDP GRO for forwarding */
+ NETIF_F_HW_TCP_DDP_BIT, /* TCP direct data placement offload */
/*
* Add your fresh new feature above and remember to update
@@ -159,6 +160,7 @@ enum {
#define NETIF_F_GSO_FRAGLIST __NETIF_F(GSO_FRAGLIST)
#define NETIF_F_HW_MACSEC __NETIF_F(HW_MACSEC)
#define NETIF_F_GRO_UDP_FWD __NETIF_F(GRO_UDP_FWD)
+#define NETIF_F_HW_TCP_DDP __NETIF_F(HW_TCP_DDP)
/* Finds the next feature with the highest number of the range of start till 0.
*/
@@ -941,6 +941,7 @@ struct dev_ifalias {
struct devlink;
struct tlsdev_ops;
+struct tcp_ddp_dev_ops;
struct netdev_name_node {
struct hlist_node hlist;
@@ -1942,6 +1943,10 @@ struct net_device {
const struct tlsdev_ops *tlsdev_ops;
#endif
+#ifdef CONFIG_TCP_DDP
+ const struct tcp_ddp_dev_ops *tcp_ddp_ops;
+#endif
+
const struct header_ops *header_ops;
unsigned char operstate;
@@ -684,6 +684,7 @@ typedef unsigned char *sk_buff_data_t;
* CHECKSUM_UNNECESSARY (max 3)
* @dst_pending_confirm: need to confirm neighbour
* @decrypted: Decrypted SKB
+ * @ddp_crc: DDP or CRC offloaded
* @napi_id: id of the NAPI struct this skb came from
* @sender_cpu: (aka @napi_id) source CPU in XPS
* @secmark: security marking
@@ -860,6 +861,9 @@ struct sk_buff {
#ifdef CONFIG_TLS_DEVICE
__u8 decrypted:1;
#endif
+#ifdef CONFIG_TCP_DDP
+ __u8 ddp_crc:1;
+#endif
#ifdef CONFIG_NET_SCHED
__u16 tc_index; /* traffic control index */
@@ -66,6 +66,8 @@ struct inet_connection_sock_af_ops {
* @icsk_ulp_ops Pluggable ULP control hook
* @icsk_ulp_data ULP private data
* @icsk_clean_acked Clean acked data hook
+ * @icsk_ulp_ddp_ops Pluggable ULP direct data placement control hook
+ * @icsk_ulp_ddp_data ULP direct data placement private data
* @icsk_listen_portaddr_node hash to the portaddr listener hashtable
* @icsk_ca_state: Congestion control state
* @icsk_retransmits: Number of unrecovered [RTO] timeouts
@@ -96,6 +98,8 @@ struct inet_connection_sock {
const struct tcp_ulp_ops *icsk_ulp_ops;
void __rcu *icsk_ulp_data;
void (*icsk_clean_acked)(struct sock *sk, u32 acked_seq);
+ const struct tcp_ddp_ulp_ops *icsk_ulp_ddp_ops;
+ void __rcu *icsk_ulp_ddp_data;
struct hlist_node icsk_listen_portaddr_node;
unsigned int (*icsk_sync_mss)(struct sock *sk, u32 pmtu);
__u8 icsk_ca_state:5,
new file mode 100644
@@ -0,0 +1,136 @@
+/* SPDX-License-Identifier: GPL-2.0
+ *
+ * tcp_ddp.h
+ * Author: Boris Pismenny <borisp@mellanox.com>
+ * Copyright (C) 2021 Mellanox Technologies.
+ */
+#ifndef _TCP_DDP_H
+#define _TCP_DDP_H
+
+#include <linux/netdevice.h>
+#include <net/inet_connection_sock.h>
+#include <net/sock.h>
+
+/* limits returned by the offload driver, zero means don't care */
+struct tcp_ddp_limits {
+ int max_ddp_sgl_len;
+};
+
+enum tcp_ddp_type {
+ TCP_DDP_NVME = 1,
+};
+
+/**
+ * struct tcp_ddp_config - Generic tcp ddp configuration: tcp ddp IO queue
+ * config implementations must use this as the first member.
+ * Add new instances of tcp_ddp_config below (nvme-tcp, etc.).
+ */
+struct tcp_ddp_config {
+ enum tcp_ddp_type type;
+ unsigned char buf[];
+};
+
+/**
+ * struct nvme_tcp_ddp_config - nvme tcp ddp configuration for an IO queue
+ *
+ * @pfv: pdu version (e.g., NVME_TCP_PFV_1_0)
+ * @cpda: controller pdu data alignmend (dwords, 0's based)
+ * @dgst: digest types enabled.
+ * The netdev will offload crc if ddp_crc is supported.
+ * @queue_size: number of nvme-tcp IO queue elements
+ * @queue_id: queue identifier
+ * @cpu_io: cpu core running the IO thread for this queue
+ */
+struct nvme_tcp_ddp_config {
+ struct tcp_ddp_config cfg;
+
+ u16 pfv;
+ u8 cpda;
+ u8 dgst;
+ int queue_size;
+ int queue_id;
+ int io_cpu;
+};
+
+/**
+ * struct tcp_ddp_io - tcp ddp configuration for an IO request.
+ *
+ * @command_id: identifier on the wire associated with these buffers
+ * @nents: number of entries in the sg_table
+ * @sg_table: describing the buffers for this IO request
+ * @first_sgl: first SGL in sg_table
+ */
+struct tcp_ddp_io {
+ u32 command_id;
+ int nents;
+ struct sg_table sg_table;
+ struct scatterlist first_sgl[SG_CHUNK_SIZE];
+};
+
+/* struct tcp_ddp_dev_ops - operations used by an upper layer protocol to configure ddp offload
+ *
+ * @tcp_ddp_limits: limit the number of scatter gather entries per IO.
+ * the device driver can use this to limit the resources allocated per queue.
+ * @tcp_ddp_sk_add: add offload for the queue represennted by the socket+config pair.
+ * this function is used to configure either copy, crc or both offloads.
+ * @tcp_ddp_sk_del: remove offload from the socket, and release any device related resources.
+ * @tcp_ddp_setup: request copy offload for buffers associated with a command_id in tcp_ddp_io.
+ * @tcp_ddp_teardown: release offload resources association between buffers and command_id in
+ * tcp_ddp_io.
+ * @tcp_ddp_resync: respond to the driver's resync_request. Called only if resync is successful.
+ */
+struct tcp_ddp_dev_ops {
+ int (*tcp_ddp_limits)(struct net_device *netdev,
+ struct tcp_ddp_limits *limits);
+ int (*tcp_ddp_sk_add)(struct net_device *netdev,
+ struct sock *sk,
+ struct tcp_ddp_config *config);
+ void (*tcp_ddp_sk_del)(struct net_device *netdev,
+ struct sock *sk);
+ int (*tcp_ddp_setup)(struct net_device *netdev,
+ struct sock *sk,
+ struct tcp_ddp_io *io);
+ int (*tcp_ddp_teardown)(struct net_device *netdev,
+ struct sock *sk,
+ struct tcp_ddp_io *io,
+ void *ddp_ctx);
+ void (*tcp_ddp_resync)(struct net_device *netdev,
+ struct sock *sk, u32 seq);
+};
+
+#define TCP_DDP_RESYNC_REQ BIT(0)
+
+/**
+ * struct tcp_ddp_ulp_ops - Interface to register uppper layer Direct Data Placement (DDP) TCP offload
+ */
+struct tcp_ddp_ulp_ops {
+ /* NIC requests ulp to indicate if @seq is the start of a message */
+ bool (*resync_request)(struct sock *sk, u32 seq, u32 flags);
+ /* NIC driver informs the ulp that ddp teardown is done - used for async completions*/
+ void (*ddp_teardown_done)(void *ddp_ctx);
+};
+
+/**
+ * struct tcp_ddp_ctx - Generic tcp ddp context: device driver per queue contexts must
+ * use this as the first member.
+ */
+struct tcp_ddp_ctx {
+ enum tcp_ddp_type type;
+ unsigned char buf[];
+};
+
+static inline struct tcp_ddp_ctx *tcp_ddp_get_ctx(const struct sock *sk)
+{
+ struct inet_connection_sock *icsk = inet_csk(sk);
+
+ return (__force struct tcp_ddp_ctx *)icsk->icsk_ulp_ddp_data;
+}
+
+static inline void tcp_ddp_set_ctx(struct sock *sk, void *ctx)
+{
+ struct inet_connection_sock *icsk = inet_csk(sk);
+
+ rcu_assign_pointer(icsk->icsk_ulp_ddp_data, ctx);
+}
+
+#endif //_TCP_DDP_H
@@ -457,6 +457,15 @@ config ETHTOOL_NETLINK
netlink. It provides better extensibility and some new features,
e.g. notification messages.
+config TCP_DDP
+ bool "TCP direct data placement offload"
+ default n
+ help
+ Direct Data Placement (DDP) offload for TCP enables ULP, such as
+ NVMe-TCP/iSCSI, to request the NIC to place TCP payload data
+ of a command response directly into kernel pages.
+
+
endif # if NET
# Used by archs to tell that they support BPF JIT compiler plus which flavour.
@@ -69,6 +69,7 @@
#include <net/xfrm.h>
#include <net/mpls.h>
#include <net/mptcp.h>
+#include <net/tcp_ddp.h>
#include <linux/uaccess.h>
#include <trace/events/skb.h>
@@ -6191,9 +6192,14 @@ EXPORT_SYMBOL(pskb_extract);
*/
void skb_condense(struct sk_buff *skb)
{
+ bool is_ddp = false;
+
+#ifdef CONFIG_TCP_DDP
+ is_ddp = skb->ddp_crc;
+#endif
if (skb->data_len) {
if (skb->data_len > skb->end - skb->tail ||
- skb_cloned(skb))
+ skb_cloned(skb) || is_ddp)
return;
/* Nice, we can free page frag(s) right now */
@@ -69,6 +69,7 @@ const char netdev_features_strings[NETDEV_FEATURE_COUNT][ETH_GSTRING_LEN] = {
[NETIF_F_GRO_FRAGLIST_BIT] = "rx-gro-list",
[NETIF_F_HW_MACSEC_BIT] = "macsec-hw-offload",
[NETIF_F_GRO_UDP_FWD_BIT] = "rx-udp-gro-forwarding",
+ [NETIF_F_HW_TCP_DDP_BIT] = "tcp-ddp-offload",
};
const char
@@ -5149,6 +5149,9 @@ tcp_collapse(struct sock *sk, struct sk_buff_head *list, struct rb_root *root,
memcpy(nskb->cb, skb->cb, sizeof(skb->cb));
#ifdef CONFIG_TLS_DEVICE
nskb->decrypted = skb->decrypted;
+#endif
+#ifdef CONFIG_TCP_DDP
+ nskb->ddp_crc = skb->ddp_crc;
#endif
TCP_SKB_CB(nskb)->seq = TCP_SKB_CB(nskb)->end_seq = start;
if (list)
@@ -5182,6 +5185,11 @@ tcp_collapse(struct sock *sk, struct sk_buff_head *list, struct rb_root *root,
#ifdef CONFIG_TLS_DEVICE
if (skb->decrypted != nskb->decrypted)
goto end;
+#endif
+#ifdef CONFIG_TCP_DDP
+
+ if (skb->ddp_crc != nskb->ddp_crc)
+ goto end;
#endif
}
}
@@ -1813,6 +1813,9 @@ bool tcp_add_backlog(struct sock *sk, struct sk_buff *skb)
TCP_SKB_CB(skb)->tcp_flags) & (TCPHDR_ECE | TCPHDR_CWR)) ||
#ifdef CONFIG_TLS_DEVICE
tail->decrypted != skb->decrypted ||
+#endif
+#ifdef CONFIG_TCP_DDP
+ tail->ddp_crc != skb->ddp_crc ||
#endif
thtail->doff != th->doff ||
memcmp(thtail + 1, th + 1, hdrlen - sizeof(*th)))
@@ -262,6 +262,9 @@ struct sk_buff *tcp_gro_receive(struct list_head *head, struct sk_buff *skb)
#ifdef CONFIG_TLS_DEVICE
flush |= p->decrypted ^ skb->decrypted;
#endif
+#ifdef CONFIG_TCP_DDP
+ flush |= p->ddp_crc ^ skb->ddp_crc;
+#endif
if (flush || skb_gro_receive(p, skb)) {
mss = 1;