[RFC,bpf-next,41/52] net, xdp: replace net_device::xdp_prog pointer with &xdp_attachment_info

Commit Message

Alexander Lobakin June 28, 2022, 7:48 p.m. UTC

To have access and store not only BPF prog pointer, but also
auxiliary params on Generic (skb) XDP path, replace it with
an &xdp_attachment_info struct and use xdp_attachment_setup_rcu()
(since Generic XDP code RCU-protects the pointer already).
This slightly changes the struct &net_device cacheline layout, but
nothing performance-critical.

Signed-off-by: Alexander Lobakin <[e-mail]>
---
 include/linux/netdevice.h |  7 +++----
 net/bpf/dev.c             | 11 ++++-------
 net/core/dev.c            |  4 +++-
 net/core/rtnetlink.c      |  2 +-
 4 files changed, 11 insertions(+), 13 deletions(-)

Patch

diff --git a/include/linux/netdevice.h b/include/linux/netdevice.h
index 60df42b3f116..1c033c164257 100644
--- a/include/linux/netdevice.h
+++ b/include/linux/netdevice.h
@@ -2168,7 +2168,7 @@  struct net_device {
 	unsigned int		num_rx_queues;
 	unsigned int		real_num_rx_queues;
 
-	struct bpf_prog __rcu	*xdp_prog;
+	struct xdp_attachment_info xdp_info;
 	unsigned long		gro_flush_timeout;
 	int			napi_defer_hard_irqs;
 #define GRO_LEGACY_MAX_SIZE	65536u
@@ -2343,9 +2343,8 @@  struct net_device {
 
 static inline bool netif_elide_gro(const struct net_device *dev)
 {
-	if (!(dev->features & NETIF_F_GRO) || dev->xdp_prog)
-		return true;
-	return false;
+	return !(dev->features & NETIF_F_GRO) ||
+	       rcu_access_pointer(dev->xdp_info.prog_rcu);
 }
 
 #define	NETDEV_ALIGN		32
diff --git a/net/bpf/dev.c b/net/bpf/dev.c
index 82948d0536c8..cc43f73929f3 100644
--- a/net/bpf/dev.c
+++ b/net/bpf/dev.c
@@ -242,19 +242,16 @@  static void dev_disable_gro_hw(struct net_device *dev)
 
 static int generic_xdp_install(struct net_device *dev, struct netdev_bpf *xdp)
 {
-	struct bpf_prog *old = rtnl_dereference(dev->xdp_prog);
-	struct bpf_prog *new = xdp->prog;
+	bool old = !!rtnl_dereference(dev->xdp_info.prog_rcu);
 	int ret = 0;
 
 	switch (xdp->command) {
 	case XDP_SETUP_PROG:
-		rcu_assign_pointer(dev->xdp_prog, new);
-		if (old)
-			bpf_prog_put(old);
+		xdp_attachment_setup_rcu(&dev->xdp_info, xdp);
 
-		if (old && !new) {
+		if (old && !xdp->prog) {
 			static_branch_dec(&generic_xdp_needed_key);
-		} else if (new && !old) {
+		} else if (xdp->prog && !old) {
 			static_branch_inc(&generic_xdp_needed_key);
 			dev_disable_lro(dev);
 			dev_disable_gro_hw(dev);
diff --git a/net/core/dev.c b/net/core/dev.c
index 62bf6ee00741..e57ae87d619e 100644
--- a/net/core/dev.c
+++ b/net/core/dev.c
@@ -5055,10 +5055,12 @@  static int __netif_receive_skb_core(struct sk_buff **pskb, bool pfmemalloc,
 	__this_cpu_inc(softnet_data.processed);
 
 	if (static_branch_unlikely(&generic_xdp_needed_key)) {
+		struct bpf_prog *prog;
 		int ret2;
 
 		migrate_disable();
-		ret2 = do_xdp_generic(rcu_dereference(skb->dev->xdp_prog), skb);
+		prog = rcu_dereference(skb->dev->xdp_info.prog_rcu);
+		ret2 = do_xdp_generic(prog, skb);
 		migrate_enable();
 
 		if (ret2 != XDP_PASS) {
diff --git a/net/core/rtnetlink.c b/net/core/rtnetlink.c
index 500420d5017c..72f696b12df2 100644
--- a/net/core/rtnetlink.c
+++ b/net/core/rtnetlink.c
@@ -1451,7 +1451,7 @@  static u32 rtnl_xdp_prog_skb(struct net_device *dev)
 
 	ASSERT_RTNL();
 
-	generic_xdp_prog = rtnl_dereference(dev->xdp_prog);
+	generic_xdp_prog = rtnl_dereference(dev->xdp_info.prog_rcu);
 	if (!generic_xdp_prog)
 		return 0;
 	return generic_xdp_prog->aux->id;