[bpf,v1,3/3] selftests/bpf: Add timer lockup selftest

Commit Message

Kumar Kartikeya Dwivedi July 9, 2024, 6:54 p.m. UTC

Add a selftest that tries to trigger a situation where two timer
callbacks are attempting to cancel each other's timer. By running them
continuously, we hit a condition where both run in parallel and cancel
each other. Without the fix in the previous patch, this would cause a
lockup as hrtimer_cancel on either side will wait for forward progress
from the callback.

Ensure that this situation leads to a EDEADLK error.

Signed-off-by: Kumar Kartikeya Dwivedi <[e-mail]>
---
 .../selftests/bpf/prog_tests/timer_lockup.c   | 65 ++++++++++++++
 .../selftests/bpf/progs/timer_lockup.c        | 85 +++++++++++++++++++
 2 files changed, 150 insertions(+)
 create mode 100644 tools/testing/selftests/bpf/prog_tests/timer_lockup.c
 create mode 100644 tools/testing/selftests/bpf/progs/timer_lockup.c

Comments

Kumar Kartikeya Dwivedi July 9, 2024, 9:06 p.m. UTC | #1

On Tue, 9 Jul 2024 at 20:54, Kumar Kartikeya Dwivedi <[e-mail]> wrote:
>
> Add a selftest that tries to trigger a situation where two timer
> callbacks are attempting to cancel each other's timer. By running them
> continuously, we hit a condition where both run in parallel and cancel
> each other. Without the fix in the previous patch, this would cause a
> lockup as hrtimer_cancel on either side will wait for forward progress
> from the callback.
>
> Ensure that this situation leads to a EDEADLK error.
>
> Signed-off-by: Kumar Kartikeya Dwivedi <[e-mail]>
> ---
>  .../selftests/bpf/prog_tests/timer_lockup.c   | 65 ++++++++++++++
>  .../selftests/bpf/progs/timer_lockup.c        | 85 +++++++++++++++++++
>  2 files changed, 150 insertions(+)
>  create mode 100644 tools/testing/selftests/bpf/prog_tests/timer_lockup.c
>  create mode 100644 tools/testing/selftests/bpf/progs/timer_lockup.c
>
> diff --git a/tools/testing/selftests/bpf/prog_tests/timer_lockup.c b/tools/testing/selftests/bpf/prog_tests/timer_lockup.c
> new file mode 100644
> index 000000000000..73e376fc5bbd
> --- /dev/null
> +++ b/tools/testing/selftests/bpf/prog_tests/timer_lockup.c
> @@ -0,0 +1,65 @@
> +// SPDX-License-Identifier: GPL-2.0
> +#define _GNU_SOURCE
> +#include <sched.h>
> +#include <test_progs.h>
> +#include <pthread.h>
> +#include <network_helpers.h>
> +#include "timer_lockup.skel.h"
> +
> +long cpu;
> +int *timer1_err;
> +int *timer2_err;
> +
> +static void *timer_lockup_thread(void *arg)
> +{
> +       LIBBPF_OPTS(bpf_test_run_opts, opts,
> +               .data_in = &pkt_v4,
> +               .data_size_in = sizeof(pkt_v4),
> +               .repeat = 10000,
> +       );
> +       int prog_fd = *(int *)arg;
> +       cpu_set_t cpuset;
> +
> +       CPU_ZERO(&cpuset);
> +       CPU_SET(__sync_fetch_and_add(&cpu, 1), &cpuset);
> +       ASSERT_OK(pthread_setaffinity_np(pthread_self(), sizeof(cpuset), &cpuset), "cpu affinity");
> +
> +       while (!*timer1_err && !*timer2_err)
> +               bpf_prog_test_run_opts(prog_fd, &opts);
> +
> +       return NULL;
> +}
> +
> +void test_timer_lockup(void)
> +{
> +       struct timer_lockup *skel;
> +       pthread_t thrds[2];
> +       void *ret;
> +
> +       skel = timer_lockup__open_and_load();
> +       if (!ASSERT_OK_PTR(skel, "timer_lockup__open_and_load"))
> +               return;
> +
> +       int timer1_prog = bpf_program__fd(skel->progs.timer1_prog);
> +       int timer2_prog = bpf_program__fd(skel->progs.timer2_prog);
> +
> +       timer1_err = &skel->bss->timer1_err;
> +       timer2_err = &skel->bss->timer2_err;
> +
> +       if (!ASSERT_OK(pthread_create(&thrds[0], NULL, timer_lockup_thread, &timer1_prog), "pthread_create thread1"))
> +               return;
> +       if (!ASSERT_OK(pthread_create(&thrds[1], NULL, timer_lockup_thread, &timer2_prog), "pthread_create thread2")) {
> +               pthread_exit(&thrds[0]);
> +               return;

A goto out: timer_lockup___destroy(skel) is missing here and above
this. Will wait for a day or so before respinning.

> [...]

Alexei Starovoitov July 10, 2024, 11:28 p.m. UTC | #2

On Tue, Jul 9, 2024 at 2:07 PM Kumar Kartikeya Dwivedi <[e-mail]> wrote:
> > +       while (!*timer1_err && !*timer2_err)
> > +               bpf_prog_test_run_opts(prog_fd, &opts);
> > +
> > +       return NULL;
> > +}
> > +
> > +void test_timer_lockup(void)
> > +{
> > +       struct timer_lockup *skel;
> > +       pthread_t thrds[2];
> > +       void *ret;
> > +
> > +       skel = timer_lockup__open_and_load();
> > +       if (!ASSERT_OK_PTR(skel, "timer_lockup__open_and_load"))
> > +               return;
> > +
> > +       int timer1_prog = bpf_program__fd(skel->progs.timer1_prog);
> > +       int timer2_prog = bpf_program__fd(skel->progs.timer2_prog);

Pls don't declare inline. Some compiler might warn.

> > +
> > +       timer1_err = &skel->bss->timer1_err;
> > +       timer2_err = &skel->bss->timer2_err;
> > +
> > +       if (!ASSERT_OK(pthread_create(&thrds[0], NULL, timer_lockup_thread, &timer1_prog), "pthread_create thread1"))

pls shorten the line.

> > +               return;
> > +       if (!ASSERT_OK(pthread_create(&thrds[1], NULL, timer_lockup_thread, &timer2_prog), "pthread_create thread2")) {
> > +               pthread_exit(&thrds[0]);
> > +               return;
>
> A goto out: timer_lockup___destroy(skel) is missing here and above
> this. Will wait for a day or so before respinning.

I was thinking to fix all these up while applying.
So I fixed it, but then noticed that the new test is quite flaky.
It seems it can get stuck in that while() loop forever.
Pls investigate.

So I applied the first two patches only.

Also pls fix:
+static int timer_cb2(void *map, int *k, struct bpf_timer *timer)
+{
+       int key = 0;
+
+       timer = bpf_map_lookup_elem(&timer1_map, &key);

1. no need to do a lookup.
2. the 3rd arg is not a bpf_timer. It's a pointer to map value.
So use
timer_cb2(void *map, int *k, struct elem *v)
then cast it to bpf_timer and use it w/o lookup.

Kumar Kartikeya Dwivedi July 11, 2024, 2:56 a.m. UTC | #3

On Thu, 11 Jul 2024 at 01:28, Alexei Starovoitov
<[e-mail]> wrote:
>
> On Tue, Jul 9, 2024 at 2:07 PM Kumar Kartikeya Dwivedi <[e-mail]> wrote:
> > > +       while (!*timer1_err && !*timer2_err)
> > > +               bpf_prog_test_run_opts(prog_fd, &opts);
> > > +
> > > +       return NULL;
> > > +}
> > > +
> > > +void test_timer_lockup(void)
> > > +{
> > > +       struct timer_lockup *skel;
> > > +       pthread_t thrds[2];
> > > +       void *ret;
> > > +
> > > +       skel = timer_lockup__open_and_load();
> > > +       if (!ASSERT_OK_PTR(skel, "timer_lockup__open_and_load"))
> > > +               return;
> > > +
> > > +       int timer1_prog = bpf_program__fd(skel->progs.timer1_prog);
> > > +       int timer2_prog = bpf_program__fd(skel->progs.timer2_prog);
>
> Pls don't declare inline. Some compiler might warn.
>

ack.

> > > +
> > > +       timer1_err = &skel->bss->timer1_err;
> > > +       timer2_err = &skel->bss->timer2_err;
> > > +
> > > +       if (!ASSERT_OK(pthread_create(&thrds[0], NULL, timer_lockup_thread, &timer1_prog), "pthread_create thread1"))
>
> pls shorten the line.
>

ack.

> > > +               return;
> > > +       if (!ASSERT_OK(pthread_create(&thrds[1], NULL, timer_lockup_thread, &timer2_prog), "pthread_create thread2")) {
> > > +               pthread_exit(&thrds[0]);
> > > +               return;
> >
> > A goto out: timer_lockup___destroy(skel) is missing here and above
> > this. Will wait for a day or so before respinning.
>
> I was thinking to fix all these up while applying.
> So I fixed it, but then noticed that the new test is quite flaky.
> It seems it can get stuck in that while() loop forever.
> Pls investigate.
>

Will do.

> So I applied the first two patches only.
>
> Also pls fix:
> +static int timer_cb2(void *map, int *k, struct bpf_timer *timer)
> +{
> +       int key = 0;
> +
> +       timer = bpf_map_lookup_elem(&timer1_map, &key);
>
> 1. no need to do a lookup.
> 2. the 3rd arg is not a bpf_timer. It's a pointer to map value.
> So use
> timer_cb2(void *map, int *k, struct elem *v)
> then cast it to bpf_timer and use it w/o lookup.

2 makes sense, will fix. Lookup is still needed. We need the timer
from timer1_map, while the callback gets timer of timer2_map.

Patch

diff --git a/tools/testing/selftests/bpf/prog_tests/timer_lockup.c b/tools/testing/selftests/bpf/prog_tests/timer_lockup.c
new file mode 100644
index 000000000000..73e376fc5bbd
--- /dev/null
+++ b/tools/testing/selftests/bpf/prog_tests/timer_lockup.c
@@ -0,0 +1,65 @@ 
+// SPDX-License-Identifier: GPL-2.0
+#define _GNU_SOURCE
+#include <sched.h>
+#include <test_progs.h>
+#include <pthread.h>
+#include <network_helpers.h>
+#include "timer_lockup.skel.h"
+
+long cpu;
+int *timer1_err;
+int *timer2_err;
+
+static void *timer_lockup_thread(void *arg)
+{
+	LIBBPF_OPTS(bpf_test_run_opts, opts,
+		.data_in = &pkt_v4,
+		.data_size_in = sizeof(pkt_v4),
+		.repeat = 10000,
+	);
+	int prog_fd = *(int *)arg;
+	cpu_set_t cpuset;
+
+	CPU_ZERO(&cpuset);
+	CPU_SET(__sync_fetch_and_add(&cpu, 1), &cpuset);
+	ASSERT_OK(pthread_setaffinity_np(pthread_self(), sizeof(cpuset), &cpuset), "cpu affinity");
+
+	while (!*timer1_err && !*timer2_err)
+		bpf_prog_test_run_opts(prog_fd, &opts);
+
+	return NULL;
+}
+
+void test_timer_lockup(void)
+{
+	struct timer_lockup *skel;
+	pthread_t thrds[2];
+	void *ret;
+
+	skel = timer_lockup__open_and_load();
+	if (!ASSERT_OK_PTR(skel, "timer_lockup__open_and_load"))
+		return;
+
+	int timer1_prog = bpf_program__fd(skel->progs.timer1_prog);
+	int timer2_prog = bpf_program__fd(skel->progs.timer2_prog);
+
+	timer1_err = &skel->bss->timer1_err;
+	timer2_err = &skel->bss->timer2_err;
+
+	if (!ASSERT_OK(pthread_create(&thrds[0], NULL, timer_lockup_thread, &timer1_prog), "pthread_create thread1"))
+		return;
+	if (!ASSERT_OK(pthread_create(&thrds[1], NULL, timer_lockup_thread, &timer2_prog), "pthread_create thread2")) {
+		pthread_exit(&thrds[0]);
+		return;
+	}
+
+	pthread_join(thrds[1], &ret);
+	pthread_join(thrds[0], &ret);
+
+	if (*timer1_err != -EDEADLK && *timer1_err != 0)
+		ASSERT_FAIL("timer1_err bad value");
+	if (*timer2_err != -EDEADLK && *timer2_err != 0)
+		ASSERT_FAIL("timer2_err bad value");
+
+	return;
+}
diff --git a/tools/testing/selftests/bpf/progs/timer_lockup.c b/tools/testing/selftests/bpf/progs/timer_lockup.c
new file mode 100644
index 000000000000..ca29da9ff25c
--- /dev/null
+++ b/tools/testing/selftests/bpf/progs/timer_lockup.c
@@ -0,0 +1,85 @@ 
+// SPDX-License-Identifier: GPL-2.0
+
+#include <linux/bpf.h>
+#include <time.h>
+#include <errno.h>
+#include <bpf/bpf_helpers.h>
+#include <bpf/bpf_tracing.h>
+#include "bpf_misc.h"
+
+char _license[] SEC("license") = "GPL";
+
+struct elem {
+	struct bpf_timer t;
+};
+
+struct {
+	__uint(type, BPF_MAP_TYPE_ARRAY);
+	__uint(max_entries, 1);
+	__type(key, int);
+	__type(value, struct elem);
+} timer1_map SEC(".maps");
+
+struct {
+	__uint(type, BPF_MAP_TYPE_ARRAY);
+	__uint(max_entries, 1);
+	__type(key, int);
+	__type(value, struct elem);
+} timer2_map SEC(".maps");
+
+int timer1_err;
+int timer2_err;
+
+static int timer_cb1(void *map, int *k, struct bpf_timer *timer)
+{
+	int key = 0;
+
+	timer = bpf_map_lookup_elem(&timer2_map, &key);
+	if (timer) {
+		timer2_err = bpf_timer_cancel(timer);
+	}
+	return 0;
+}
+
+static int timer_cb2(void *map, int *k, struct bpf_timer *timer)
+{
+	int key = 0;
+
+	timer = bpf_map_lookup_elem(&timer1_map, &key);
+	if (timer) {
+		timer1_err = bpf_timer_cancel(timer);
+	}
+	return 0;
+}
+
+SEC("tc")
+int timer1_prog(void *ctx)
+{
+	int key = 0;
+	struct bpf_timer *timer;
+
+	timer = bpf_map_lookup_elem(&timer1_map, &key);
+	if (timer) {
+		bpf_timer_init(timer, &timer1_map, CLOCK_BOOTTIME);
+		bpf_timer_set_callback(timer, timer_cb1);
+		bpf_timer_start(timer, 1, BPF_F_TIMER_CPU_PIN);
+	}
+
+	return 0;
+}
+
+SEC("tc")
+int timer2_prog(void *ctx)
+{
+	int key = 0;
+	struct bpf_timer *timer;
+
+	timer = bpf_map_lookup_elem(&timer2_map, &key);
+	if (timer) {
+		bpf_timer_init(timer, &timer2_map, CLOCK_BOOTTIME);
+		bpf_timer_set_callback(timer, timer_cb2);
+		bpf_timer_start(timer, 1, BPF_F_TIMER_CPU_PIN);
+	}
+
+	return 0;
+}