@@ -142,15 +142,14 @@ optional. The following modified excerpt is from
s32 prev_cpu, u64 wake_flags)
{
s32 cpu;
- /* Need to initialize or the BPF verifier will reject the program */
- bool direct = false;
- cpu = scx_bpf_select_cpu_dfl(p, prev_cpu, wake_flags, &direct);
-
- if (direct)
+ cpu = scx_bpf_select_cpu_and(p, prev_cpu, wake_flags, p->cpus_ptr, 0);
+ if (cpu >= 0)
scx_bpf_dsq_insert(p, SCX_DSQ_LOCAL, SCX_SLICE_DFL, 0);
+ return cpu;
+ }
- return cpu;
+ return prev_cpu;
}
/*
@@ -464,13 +464,12 @@ struct sched_ext_ops {
* state. By default, implementing this operation disables the built-in
* idle CPU tracking and the following helpers become unavailable:
*
- * - scx_bpf_select_cpu_dfl()
* - scx_bpf_select_cpu_and()
* - scx_bpf_test_and_clear_cpu_idle()
* - scx_bpf_pick_idle_cpu()
*
* The user also must implement ops.select_cpu() as the default
- * implementation relies on scx_bpf_select_cpu_dfl().
+ * implementation relies on scx_bpf_select_cpu_and().
*
* Specify the %SCX_OPS_KEEP_BUILTIN_IDLE flag to keep the built-in idle
* tracking.
@@ -872,26 +872,16 @@ __bpf_kfunc int scx_bpf_cpu_node(s32 cpu)
#endif
}
-/**
- * scx_bpf_select_cpu_dfl - The default implementation of ops.select_cpu()
- * @p: task_struct to select a CPU for
- * @prev_cpu: CPU @p was on previously
- * @wake_flags: %SCX_WAKE_* flags
- * @is_idle: out parameter indicating whether the returned CPU is idle
- *
- * Can only be called from ops.select_cpu() if the built-in CPU selection is
- * enabled - ops.update_idle() is missing or %SCX_OPS_KEEP_BUILTIN_IDLE is set.
- * @p, @prev_cpu and @wake_flags match ops.select_cpu().
- *
- * Returns the picked CPU with *@is_idle indicating whether the picked CPU is
- * currently idle and thus a good candidate for direct dispatching.
- */
+/* Provided for backward binary compatibility, will be removed in v6.17. */
__bpf_kfunc s32 scx_bpf_select_cpu_dfl(struct task_struct *p, s32 prev_cpu,
u64 wake_flags, bool *is_idle)
{
#ifdef CONFIG_SMP
s32 cpu;
#endif
+ printk_deferred_once(KERN_WARNING
+ "sched_ext: scx_bpf_select_cpu_dfl() deprecated in favor of scx_bpf_select_cpu_and()");
+
if (!ops_cpu_valid(prev_cpu, NULL))
goto prev_cpu;
@@ -47,7 +47,8 @@ static inline void ___vmlinux_h_sanity_check___(void)
}
s32 scx_bpf_create_dsq(u64 dsq_id, s32 node) __ksym;
-s32 scx_bpf_select_cpu_dfl(struct task_struct *p, s32 prev_cpu, u64 wake_flags, bool *is_idle) __ksym;
+s32 scx_bpf_select_cpu_dfl(struct task_struct *p,
+ s32 prev_cpu, u64 wake_flags, bool *is_idle) __ksym __weak;
s32 scx_bpf_select_cpu_and(struct task_struct *p, s32 prev_cpu, u64 wake_flags,
const struct cpumask *cpus_allowed, u64 flags) __ksym __weak;
void scx_bpf_dsq_insert(struct task_struct *p, u64 dsq_id, u64 slice, u64 enq_flags) __ksym __weak;
@@ -225,6 +225,43 @@ static inline bool __COMPAT_is_enq_cpu_selected(u64 enq_flags)
scx_bpf_pick_any_cpu_node(cpus_allowed, node, flags) : \
scx_bpf_pick_any_cpu(cpus_allowed, flags))
+/**
+ * scx_bpf_select_cpu_dfl - The default implementation of ops.select_cpu().
+ * We will preserve this compatible helper until v6.17.
+ *
+ * @p: task_struct to select a CPU for
+ * @prev_cpu: CPU @p was on previously
+ * @wake_flags: %SCX_WAKE_* flags
+ * @is_idle: out parameter indicating whether the returned CPU is idle
+ *
+ * Can only be called from ops.select_cpu() if the built-in CPU selection is
+ * enabled - ops.update_idle() is missing or %SCX_OPS_KEEP_BUILTIN_IDLE is set.
+ * @p, @prev_cpu and @wake_flags match ops.select_cpu().
+ *
+ * Returns the picked CPU with *@is_idle indicating whether the picked CPU is
+ * currently idle and thus a good candidate for direct dispatching.
+ */
+#define scx_bpf_select_cpu_dfl(p, prev_cpu, wake_flags, is_idle) \
+({ \
+ s32 __cpu; \
+ \
+ if (bpf_ksym_exists(scx_bpf_select_cpu_and)) { \
+ __cpu = scx_bpf_select_cpu_and((p), (prev_cpu), (wake_flags), \
+ (p)->cpus_ptr, 0); \
+ if (__cpu >= 0) { \
+ *(is_idle) = true; \
+ } else { \
+ *(is_idle) = false; \
+ __cpu = (prev_cpu); \
+ } \
+ } else { \
+ __cpu = scx_bpf_select_cpu_dfl((p), (prev_cpu), \
+ (wake_flags), (is_idle)); \
+ } \
+ \
+ __cpu; \
+})
+
/*
* Define sched_ext_ops. This may be expanded to define multiple variants for
* backward compatibility. See compat.h::SCX_OPS_LOAD/ATTACH().
@@ -317,15 +317,12 @@ static void set_bypassed_at(struct task_struct *p, struct fcg_task_ctx *taskc)
s32 BPF_STRUCT_OPS(fcg_select_cpu, struct task_struct *p, s32 prev_cpu, u64 wake_flags)
{
struct fcg_task_ctx *taskc;
- bool is_idle = false;
s32 cpu;
- cpu = scx_bpf_select_cpu_dfl(p, prev_cpu, wake_flags, &is_idle);
-
taskc = bpf_task_storage_get(&task_ctx, p, 0, 0);
if (!taskc) {
scx_bpf_error("task_ctx lookup failed");
- return cpu;
+ return prev_cpu;
}
/*
@@ -333,13 +330,16 @@ s32 BPF_STRUCT_OPS(fcg_select_cpu, struct task_struct *p, s32 prev_cpu, u64 wake
* idle. Follow it and charge the cgroup later in fcg_stopping() after
* the fact.
*/
- if (is_idle) {
+ cpu = scx_bpf_select_cpu_and(p, prev_cpu, wake_flags, p->cpus_ptr, 0);
+ if (cpu >= 0) {
set_bypassed_at(p, taskc);
stat_inc(FCG_STAT_LOCAL);
scx_bpf_dsq_insert(p, SCX_DSQ_LOCAL, SCX_SLICE_DFL, 0);
+
+ return cpu;
}
- return cpu;
+ return prev_cpu;
}
void BPF_STRUCT_OPS(fcg_enqueue, struct task_struct *p, u64 enq_flags)
@@ -54,16 +54,17 @@ static void stat_inc(u32 idx)
s32 BPF_STRUCT_OPS(simple_select_cpu, struct task_struct *p, s32 prev_cpu, u64 wake_flags)
{
- bool is_idle = false;
s32 cpu;
- cpu = scx_bpf_select_cpu_dfl(p, prev_cpu, wake_flags, &is_idle);
- if (is_idle) {
+ cpu = scx_bpf_select_cpu_and(p, prev_cpu, wake_flags, p->cpus_ptr, 0);
+ if (cpu >= 0) {
stat_inc(0); /* count local queueing */
scx_bpf_dsq_insert(p, SCX_DSQ_LOCAL, SCX_SLICE_DFL, 0);
+
+ return cpu;
}
- return cpu;
+ return prev_cpu;
}
void BPF_STRUCT_OPS(simple_enqueue, struct task_struct *p, u64 enq_flags)
@@ -9,10 +9,6 @@
char _license[] SEC("license") = "GPL";
-/* Manually specify the signature until the kfunc is added to the scx repo. */
-s32 scx_bpf_select_cpu_dfl(struct task_struct *p, s32 prev_cpu, u64 wake_flags,
- bool *found) __ksym;
-
s32 BPF_STRUCT_OPS(enq_select_cpu_fails_select_cpu, struct task_struct *p,
s32 prev_cpu, u64 wake_flags)
{
@@ -22,14 +18,8 @@ s32 BPF_STRUCT_OPS(enq_select_cpu_fails_select_cpu, struct task_struct *p,
void BPF_STRUCT_OPS(enq_select_cpu_fails_enqueue, struct task_struct *p,
u64 enq_flags)
{
- /*
- * Need to initialize the variable or the verifier will fail to load.
- * Improving these semantics is actively being worked on.
- */
- bool found = false;
-
/* Can only call from ops.select_cpu() */
- scx_bpf_select_cpu_dfl(p, 0, 0, &found);
+ scx_bpf_select_cpu_and(p, 0, 0, p->cpus_ptr, 0);
scx_bpf_dsq_insert(p, SCX_DSQ_GLOBAL, SCX_SLICE_DFL, enq_flags);
}
@@ -52,7 +52,7 @@ static void cleanup(void *ctx)
struct scx_test enq_select_cpu_fails = {
.name = "enq_select_cpu_fails",
- .description = "Verify we fail to call scx_bpf_select_cpu_dfl() "
+ .description = "Verify we fail to call scx_bpf_select_cpu_and() "
"from ops.enqueue()",
.setup = setup,
.run = run,
@@ -20,12 +20,14 @@ UEI_DEFINE(uei);
s32 BPF_STRUCT_OPS(exit_select_cpu, struct task_struct *p,
s32 prev_cpu, u64 wake_flags)
{
- bool found;
+ s32 cpu;
if (exit_point == EXIT_SELECT_CPU)
EXIT_CLEANLY();
- return scx_bpf_select_cpu_dfl(p, prev_cpu, wake_flags, &found);
+ cpu = scx_bpf_select_cpu_and(p, prev_cpu, wake_flags, p->cpus_ptr, 0);
+
+ return cpu >= 0 ? cpu : prev_cpu;
}
void BPF_STRUCT_OPS(exit_enqueue, struct task_struct *p, u64 enq_flags)
@@ -27,10 +27,6 @@ struct {
__type(value, struct task_ctx);
} task_ctx_stor SEC(".maps");
-/* Manually specify the signature until the kfunc is added to the scx repo. */
-s32 scx_bpf_select_cpu_dfl(struct task_struct *p, s32 prev_cpu, u64 wake_flags,
- bool *found) __ksym;
-
s32 BPF_STRUCT_OPS(select_cpu_dfl_nodispatch_select_cpu, struct task_struct *p,
s32 prev_cpu, u64 wake_flags)
{
@@ -43,10 +39,13 @@ s32 BPF_STRUCT_OPS(select_cpu_dfl_nodispatch_select_cpu, struct task_struct *p,
return -ESRCH;
}
- cpu = scx_bpf_select_cpu_dfl(p, prev_cpu, wake_flags,
- &tctx->force_local);
+ cpu = scx_bpf_select_cpu_and(p, prev_cpu, wake_flags, p->cpus_ptr, 0);
+ if (cpu >= 0) {
+ tctx->force_local = true;
+ return cpu;
+ }
- return cpu;
+ return prev_cpu;
}
void BPF_STRUCT_OPS(select_cpu_dfl_nodispatch_enqueue, struct task_struct *p,
@@ -66,7 +66,7 @@ static void cleanup(void *ctx)
struct scx_test select_cpu_dfl_nodispatch = {
.name = "select_cpu_dfl_nodispatch",
- .description = "Verify behavior of scx_bpf_select_cpu_dfl() in "
+ .description = "Verify behavior of scx_bpf_select_cpu_and() in "
"ops.select_cpu()",
.setup = setup,
.run = run,
With the introduction of scx_bpf_select_cpu_and(), we can deprecate scx_bpf_select_cpu_dfl(), as it offers only a subset of features and it's also more consistent with other idle-related APIs (returning a negative value when no idle CPU is found). Therefore, mark scx_bpf_select_cpu_dfl() as deprecated (printing a warning when it's used), update all the scheduler examples and kselftests to adopt the new API, and ensure backward (source and binary) compatibility by providing the necessary macros and hooks. Support for scx_bpf_select_cpu_dfl() can be maintained until v6.17. Signed-off-by: Andrea Righi <arighi@nvidia.com> --- Documentation/scheduler/sched-ext.rst | 11 +++--- kernel/sched/ext.c | 3 +- kernel/sched/ext_idle.c | 18 ++------- tools/sched_ext/include/scx/common.bpf.h | 3 +- tools/sched_ext/include/scx/compat.bpf.h | 37 +++++++++++++++++++ tools/sched_ext/scx_flatcg.bpf.c | 12 +++--- tools/sched_ext/scx_simple.bpf.c | 9 +++-- .../sched_ext/enq_select_cpu_fails.bpf.c | 12 +----- .../sched_ext/enq_select_cpu_fails.c | 2 +- tools/testing/selftests/sched_ext/exit.bpf.c | 6 ++- .../sched_ext/select_cpu_dfl_nodispatch.bpf.c | 13 +++---- .../sched_ext/select_cpu_dfl_nodispatch.c | 2 +- 12 files changed, 73 insertions(+), 55 deletions(-)