@@ -379,7 +379,10 @@ int xdp_reg_mem_model(struct xdp_mem_info *mem,
void xdp_unreg_mem_model(struct xdp_mem_info *mem);
struct xdp_attachment_info {
- struct bpf_prog *prog;
+ union {
+ struct bpf_prog __rcu *prog_rcu;
+ struct bpf_prog *prog;
+ };
union {
__le64 btf_id_le;
u64 btf_id;
@@ -391,6 +394,8 @@ struct xdp_attachment_info {
struct netdev_bpf;
void xdp_attachment_setup(struct xdp_attachment_info *info,
struct netdev_bpf *bpf);
+void xdp_attachment_setup_rcu(struct xdp_attachment_info *info,
+ struct netdev_bpf *bpf);
#define DEV_MAP_BULK_SIZE XDP_BULK_QUEUE_SIZE
@@ -557,6 +557,34 @@ void xdp_attachment_setup(struct xdp_attachment_info *info,
}
EXPORT_SYMBOL_GPL(xdp_attachment_setup);
+/**
+ * xdp_attachment_setup_rcu - an RCU-powered version of xdp_attachment_setup()
+ * @info: pointer to the target container
+ * @bpf: pointer to the container passed to ::ndo_bpf()
+ *
+ * Protects sensitive values with RCU to allow program how-swaps without
+ * stopping an interface. Write side (this) must be called under the RTNL lock
+ * and reader sides must fetch any data only under the RCU read lock -- old BPF
+ * program will be freed only after a critical section is finished (see
+ * bpf_prog_put()).
+ */
+void xdp_attachment_setup_rcu(struct xdp_attachment_info *info,
+ struct netdev_bpf *bpf)
+{
+ struct bpf_prog *old_prog;
+
+ ASSERT_RTNL();
+
+ old_prog = rcu_replace_pointer(info->prog_rcu, bpf->prog,
+ lockdep_rtnl_is_held());
+ WRITE_ONCE(info->btf_id, bpf->btf_id);
+ WRITE_ONCE(info->meta_thresh, bpf->meta_thresh);
+
+ if (old_prog)
+ bpf_prog_put(old_prog);
+}
+EXPORT_SYMBOL_GPL(xdp_attachment_setup_rcu);
+
struct xdp_frame *xdp_convert_zc_to_xdp_frame(struct xdp_buff *xdp)
{
unsigned int metasize, totsize;
Currently, xdp_attachment_setup() uses plain assignments and puts the previous BPF program before updating the pointer, rendering itself dangerous for program hot-swaps due to pointer tearing and potential use-after-free's. At the same time, &xdp_attachment_info comes handy to use it in drivers as a main container including hotpath -- the BTF ID and meta threshold values are now being used there as well, not speaking of reducing some boilerplate code. Add an RCU-protected pointer to XDP program to that structure and an RCU version of xdp_attachment_setup(), which will make sure that all the values were not corrupted and that old BPF program was freed only after the pointer was updated. The only thing left is that RCU read critical sections might happen in between each assignment, but since the relations between XDP prog, BTF ID and meta threshold are not vital, it's totally fine to allow this. A caller must ensure it's being executed under the RTNL lock. Reader sides must ensure they're being executed under the RCU read lock. Once all the current users of xdp_attachment_setup() are switched to the RCU-aware version (with appropriate adjustments), the "regular" one will be removed. Partially inspired by commit fe45386a2082 ("net/mlx5e: Use RCU to protect rq->xdp_prog"). Signed-off-by: Alexander Lobakin <alexandr.lobakin@intel.com> --- include/net/xdp.h | 7 ++++++- net/bpf/core.c | 28 ++++++++++++++++++++++++++++ 2 files changed, 34 insertions(+), 1 deletion(-)