@@ -4957,6 +4957,17 @@ union bpf_attr {
* **-ENOENT** if *task->mm* is NULL, or no vma contains *addr*.
* **-EBUSY** if failed to try lock mmap_lock.
* **-EINVAL** for invalid **flags**.
+ *
+ * long bpf_redirect_xsk(void *ctx, struct bpf_map *map, u32 key, u64 flags)
+ * Description
+ * Redirect the packet to the XDP socket associated with the netdev queue if
+ * the socket has an rx ring configured and is the only socket attached to the
+ * queue. Fall back to bpf_redirect_map behavior if either condition is not met.
+ * Return
+ * **XDP_REDIRECT_XSK** if successful.
+ *
+ * **XDP_REDIRECT** if the fall back was successful, or the value of the
+ * two lower bits of the *flags* argument on error
*/
#define __BPF_FUNC_MAPPER(FN) \
FN(unspec), \
@@ -5140,6 +5151,7 @@ union bpf_attr {
FN(skc_to_unix_sock), \
FN(kallsyms_lookup_name), \
FN(find_vma), \
+ FN(redirect_xsk), \
/* */
/* integer value in 'imm' field of BPF_CALL instruction selects which helper
@@ -5520,6 +5532,7 @@ enum xdp_action {
XDP_PASS,
XDP_TX,
XDP_REDIRECT,
+ XDP_REDIRECT_XSK,
};
/* user accessible metadata for XDP packet hook
@@ -5526,7 +5526,8 @@ static int check_map_func_compatibility(struct bpf_verifier_env *env,
break;
case BPF_MAP_TYPE_XSKMAP:
if (func_id != BPF_FUNC_redirect_map &&
- func_id != BPF_FUNC_map_lookup_elem)
+ func_id != BPF_FUNC_map_lookup_elem &&
+ func_id != BPF_FUNC_redirect_xsk)
goto error;
break;
case BPF_MAP_TYPE_ARRAY_OF_MAPS:
@@ -5629,6 +5630,10 @@ static int check_map_func_compatibility(struct bpf_verifier_env *env,
map->map_type != BPF_MAP_TYPE_XSKMAP)
goto error;
break;
+ case BPF_FUNC_redirect_xsk:
+ if (map->map_type != BPF_MAP_TYPE_XSKMAP)
+ goto error;
+ break;
case BPF_FUNC_sk_redirect_map:
case BPF_FUNC_msg_redirect_map:
case BPF_FUNC_sock_map_update:
@@ -4140,6 +4140,26 @@ static const struct bpf_func_proto bpf_xdp_redirect_map_proto = {
.arg3_type = ARG_ANYTHING,
};
+BPF_CALL_4(bpf_xdp_redirect_xsk, struct xdp_buff *, xdp, struct bpf_map *, map,
+ u32, ifindex, u64, flags)
+{
+#ifdef CONFIG_XDP_SOCKETS
+ if (likely(refcount_read(&xdp->rxq->dev->_rx[xdp->rxq->queue_index].xsk_refcnt) == 1))
+ return XDP_REDIRECT_XSK;
+#endif
+ return map->ops->map_redirect(map, ifindex, flags);
+}
+
+static const struct bpf_func_proto bpf_xdp_redirect_xsk_proto = {
+ .func = bpf_xdp_redirect_xsk,
+ .gpl_only = false,
+ .ret_type = RET_INTEGER,
+ .arg1_type = ARG_PTR_TO_CTX,
+ .arg2_type = ARG_CONST_MAP_PTR,
+ .arg3_type = ARG_ANYTHING,
+ .arg4_type = ARG_ANYTHING,
+};
+
static unsigned long bpf_skb_copy(void *dst_buff, const void *skb,
unsigned long off, unsigned long len)
{
@@ -7469,6 +7489,8 @@ xdp_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog)
return &bpf_xdp_redirect_proto;
case BPF_FUNC_redirect_map:
return &bpf_xdp_redirect_map_proto;
+ case BPF_FUNC_redirect_xsk:
+ return &bpf_xdp_redirect_xsk_proto;
case BPF_FUNC_xdp_adjust_tail:
return &bpf_xdp_adjust_tail_proto;
case BPF_FUNC_fib_lookup:
Add a new XDP redirect helper called bpf_redirect_xsk which simply returns the new XDP_REDIRECT_XSK action if the xsk refcnt for the netdev_rx_queue is equal to one. Checking this value verifies that the AF_XDP socket Rx ring is configured and there is exactly one xsk attached to the queue. XDP_REDIRECT_XSK indicates to the driver that the XSKMAP lookup can be skipped and the pointer to the socket to redirect to can instead be retrieved from the netdev_rx_queue on which the packet was received. If the aforementioned conditions are not met, fallback to the behavior of xdp_redirect_map which returns XDP_REDIRECT for a successful XSKMAP lookup. Signed-off-by: Ciara Loftus <ciara.loftus@intel.com> --- include/uapi/linux/bpf.h | 13 +++++++++++++ kernel/bpf/verifier.c | 7 ++++++- net/core/filter.c | 22 ++++++++++++++++++++++ 3 files changed, 41 insertions(+), 1 deletion(-)