From 5e5a4b1bfab707715600dc2aa356b6690971e2c2 Mon Sep 17 00:00:00 2001 From: ferreo Date: Wed, 2 Oct 2024 18:22:22 +0200 Subject: [PATCH] Update patches/0002-sched-ext.patch --- patches/0002-sched-ext.patch | 717 +++++++++++++++++++++-------------- 1 file changed, 423 insertions(+), 294 deletions(-) diff --git a/patches/0002-sched-ext.patch b/patches/0002-sched-ext.patch index 0a4809b..8f30132 100644 --- a/patches/0002-sched-ext.patch +++ b/patches/0002-sched-ext.patch @@ -1,12 +1,12 @@ -From b9aae9642313560e22d9424b9b32c15dc23e36e1 Mon Sep 17 00:00:00 2001 -From: Piotr Gorski -Date: Thu, 12 Sep 2024 07:55:48 +0200 +From 02d7d0eafe23e91d14c2a57f1daa8e57d98e5d7d Mon Sep 17 00:00:00 2001 +From: Peter Jung +Date: Mon, 30 Sep 2024 16:59:41 +0200 Subject: [PATCH] sched-ext -Signed-off-by: Piotr Gorski +Signed-off-by: Peter Jung --- Documentation/scheduler/index.rst | 1 + - Documentation/scheduler/sched-ext.rst | 316 + + Documentation/scheduler/sched-ext.rst | 326 + MAINTAINERS | 13 + drivers/tty/sysrq.c | 1 + include/asm-generic/vmlinux.lds.h | 1 + @@ -24,11 +24,11 @@ Signed-off-by: Piotr Gorski kernel/sched/core.c | 288 +- kernel/sched/cpufreq_schedutil.c | 50 +- kernel/sched/debug.c | 3 + - kernel/sched/ext.c | 7213 +++++++++++++++++ + kernel/sched/ext.c | 7262 +++++++++++++++++ kernel/sched/ext.h | 91 + kernel/sched/fair.c | 22 +- kernel/sched/idle.c | 2 + - kernel/sched/sched.h | 193 +- + kernel/sched/sched.h | 203 +- kernel/sched/syscalls.c | 26 + lib/dump_stack.c | 1 + tools/Makefile | 10 +- @@ -36,19 +36,19 @@ Signed-off-by: Piotr Gorski tools/sched_ext/Makefile | 246 + tools/sched_ext/README.md | 270 + .../sched_ext/include/bpf-compat/gnu/stubs.h | 11 + - tools/sched_ext/include/scx/common.bpf.h | 412 + + tools/sched_ext/include/scx/common.bpf.h | 427 + tools/sched_ext/include/scx/common.h | 75 + - tools/sched_ext/include/scx/compat.bpf.h | 28 + + tools/sched_ext/include/scx/compat.bpf.h | 47 + tools/sched_ext/include/scx/compat.h | 186 + - tools/sched_ext/include/scx/user_exit_info.h | 111 + + tools/sched_ext/include/scx/user_exit_info.h | 115 + tools/sched_ext/scx_central.bpf.c | 361 + tools/sched_ext/scx_central.c | 135 + - tools/sched_ext/scx_flatcg.bpf.c | 949 +++ + tools/sched_ext/scx_flatcg.bpf.c | 957 +++ tools/sched_ext/scx_flatcg.c | 233 + tools/sched_ext/scx_flatcg.h | 51 + tools/sched_ext/scx_qmap.bpf.c | 813 ++ tools/sched_ext/scx_qmap.c | 153 + - tools/sched_ext/scx_show_state.py | 39 + + tools/sched_ext/scx_show_state.py | 40 + tools/sched_ext/scx_simple.bpf.c | 156 + tools/sched_ext/scx_simple.c | 107 + tools/testing/selftests/sched_ext/.gitignore | 6 + @@ -102,7 +102,7 @@ Signed-off-by: Piotr Gorski .../selftests/sched_ext/test_example.c | 49 + tools/testing/selftests/sched_ext/util.c | 71 + tools/testing/selftests/sched_ext/util.h | 13 + - 97 files changed, 16063 insertions(+), 126 deletions(-) + 97 files changed, 16175 insertions(+), 130 deletions(-) create mode 100644 Documentation/scheduler/sched-ext.rst create mode 100644 include/linux/sched/ext.h create mode 100644 include/trace/events/sched_ext.h @@ -180,7 +180,7 @@ Signed-off-by: Piotr Gorski create mode 100644 tools/testing/selftests/sched_ext/util.h diff --git a/Documentation/scheduler/index.rst b/Documentation/scheduler/index.rst -index 43bd8a145b7a..0611dc3dda8e 100644 +index 43bd8a145b7a9..0611dc3dda8ea 100644 --- a/Documentation/scheduler/index.rst +++ b/Documentation/scheduler/index.rst @@ -20,6 +20,7 @@ Scheduler @@ -193,10 +193,10 @@ index 43bd8a145b7a..0611dc3dda8e 100644 text_files diff --git a/Documentation/scheduler/sched-ext.rst b/Documentation/scheduler/sched-ext.rst new file mode 100644 -index 000000000000..a707d2181a77 +index 0000000000000..6c0d70e2e27df --- /dev/null +++ b/Documentation/scheduler/sched-ext.rst -@@ -0,0 +1,316 @@ +@@ -0,0 +1,326 @@ +========================== +Extensible Scheduler Class +========================== @@ -282,6 +282,15 @@ index 000000000000..a707d2181a77 + # cat /sys/kernel/sched_ext/root/ops + simple + ++You can check if any BPF scheduler has ever been loaded since boot by examining ++this monotonically incrementing counter (a value of zero indicates that no BPF ++scheduler has been loaded): ++ ++.. code-block:: none ++ ++ # cat /sys/kernel/sched_ext/enable_seq ++ 1 ++ +``tools/sched_ext/scx_show_state.py`` is a drgn script which shows more +detailed information: + @@ -295,6 +304,7 @@ index 000000000000..a707d2181a77 + enable_state : enabled (2) + bypass_depth : 0 + nr_rejected : 0 ++ enable_seq : 1 + +If ``CONFIG_SCHED_DEBUG`` is set, whether a given task is on sched_ext can +be determined as follows: @@ -514,10 +524,10 @@ index 000000000000..a707d2181a77 +possible, they are subject to change without warning between kernel +versions. diff --git a/MAINTAINERS b/MAINTAINERS -index 3b5c5c42eb03..827105d4441e 100644 +index 16df466c205dc..3345a15afded8 100644 --- a/MAINTAINERS +++ b/MAINTAINERS -@@ -20352,6 +20352,19 @@ F: include/linux/wait.h +@@ -20353,6 +20353,19 @@ F: include/linux/wait.h F: include/uapi/linux/sched.h F: kernel/sched/ @@ -538,7 +548,7 @@ index 3b5c5c42eb03..827105d4441e 100644 M: Gustavo Silva S: Maintained diff --git a/drivers/tty/sysrq.c b/drivers/tty/sysrq.c -index 14f8f00fdcf9..930b04e3d148 100644 +index 14f8f00fdcf9a..930b04e3d148f 100644 --- a/drivers/tty/sysrq.c +++ b/drivers/tty/sysrq.c @@ -531,6 +531,7 @@ static const struct sysrq_key_op *sysrq_key_table[62] = { @@ -550,7 +560,7 @@ index 14f8f00fdcf9..930b04e3d148 100644 NULL, /* T */ NULL, /* U */ diff --git a/include/asm-generic/vmlinux.lds.h b/include/asm-generic/vmlinux.lds.h -index 1ae44793132a..19ec49a9179b 100644 +index 1ae44793132a8..19ec49a9179b8 100644 --- a/include/asm-generic/vmlinux.lds.h +++ b/include/asm-generic/vmlinux.lds.h @@ -133,6 +133,7 @@ @@ -562,7 +572,7 @@ index 1ae44793132a..19ec49a9179b 100644 __sched_class_lowest = .; diff --git a/include/linux/cgroup.h b/include/linux/cgroup.h -index c60ba0ab1462..7139b33cb104 100644 +index c60ba0ab14627..7139b33cb104f 100644 --- a/include/linux/cgroup.h +++ b/include/linux/cgroup.h @@ -28,8 +28,6 @@ @@ -584,7 +594,7 @@ index c60ba0ab1462..7139b33cb104 100644 CSS_TASK_ITER_PROCS = (1U << 0), /* walk only threadgroup leaders */ CSS_TASK_ITER_THREADED = (1U << 1), /* walk all threaded css_sets in the domain */ diff --git a/include/linux/sched.h b/include/linux/sched.h -index f8d150343d42..5b4f78fe379d 100644 +index f8d150343d42d..5b4f78fe379d1 100644 --- a/include/linux/sched.h +++ b/include/linux/sched.h @@ -82,6 +82,8 @@ struct task_group; @@ -608,7 +618,7 @@ index f8d150343d42..5b4f78fe379d 100644 #ifdef CONFIG_SCHED_CORE diff --git a/include/linux/sched/ext.h b/include/linux/sched/ext.h new file mode 100644 -index 000000000000..76166d3b14fc +index 0000000000000..76166d3b14fcf --- /dev/null +++ b/include/linux/sched/ext.h @@ -0,0 +1,216 @@ @@ -829,7 +839,7 @@ index 000000000000..76166d3b14fc +#endif /* CONFIG_SCHED_CLASS_EXT */ +#endif /* _LINUX_SCHED_EXT_H */ diff --git a/include/linux/sched/task.h b/include/linux/sched/task.h -index d362aacf9f89..0f2aeb37bbb0 100644 +index d362aacf9f897..0f2aeb37bbb04 100644 --- a/include/linux/sched/task.h +++ b/include/linux/sched/task.h @@ -63,7 +63,8 @@ extern asmlinkage void schedule_tail(struct task_struct *prev); @@ -856,7 +866,7 @@ index d362aacf9f89..0f2aeb37bbb0 100644 diff --git a/include/trace/events/sched_ext.h b/include/trace/events/sched_ext.h new file mode 100644 -index 000000000000..fe19da7315a9 +index 0000000000000..fe19da7315a90 --- /dev/null +++ b/include/trace/events/sched_ext.h @@ -0,0 +1,32 @@ @@ -893,7 +903,7 @@ index 000000000000..fe19da7315a9 +/* This part must be outside protection */ +#include diff --git a/include/uapi/linux/sched.h b/include/uapi/linux/sched.h -index 3bac0a8ceab2..359a14cc76a4 100644 +index 3bac0a8ceab26..359a14cc76a40 100644 --- a/include/uapi/linux/sched.h +++ b/include/uapi/linux/sched.h @@ -118,6 +118,7 @@ struct clone_args { @@ -905,7 +915,7 @@ index 3bac0a8ceab2..359a14cc76a4 100644 /* Can be ORed in to make sure the process is reverted back to SCHED_NORMAL on fork */ #define SCHED_RESET_ON_FORK 0x40000000 diff --git a/init/Kconfig b/init/Kconfig -index 08a0d51afaae..e1a88d48d652 100644 +index 08a0d51afaae4..e1a88d48d652c 100644 --- a/init/Kconfig +++ b/init/Kconfig @@ -1028,9 +1028,13 @@ menuconfig CGROUP_SCHED @@ -936,7 +946,7 @@ index 08a0d51afaae..e1a88d48d652 100644 config SCHED_MM_CID diff --git a/init/init_task.c b/init/init_task.c -index eeb110c65fe2..e222722e790b 100644 +index eeb110c65fe22..e222722e790b1 100644 --- a/init/init_task.c +++ b/init/init_task.c @@ -6,6 +6,7 @@ @@ -966,7 +976,7 @@ index eeb110c65fe2..e222722e790b 100644 .ptraced = LIST_HEAD_INIT(init_task.ptraced), .ptrace_entry = LIST_HEAD_INIT(init_task.ptrace_entry), diff --git a/kernel/Kconfig.preempt b/kernel/Kconfig.preempt -index c2f1fd95a821..fe782cd77388 100644 +index c2f1fd95a8214..fe782cd773885 100644 --- a/kernel/Kconfig.preempt +++ b/kernel/Kconfig.preempt @@ -133,4 +133,29 @@ config SCHED_CORE @@ -1001,7 +1011,7 @@ index c2f1fd95a821..fe782cd77388 100644 + Documentation/scheduler/sched-ext.rst + https://github.com/sched-ext/scx diff --git a/kernel/fork.c b/kernel/fork.c -index 238695afc630..69a0a7210060 100644 +index 238695afc6304..69a0a7210060e 100644 --- a/kernel/fork.c +++ b/kernel/fork.c @@ -23,6 +23,7 @@ @@ -1079,7 +1089,7 @@ index 238695afc630..69a0a7210060 100644 lockdep_free_task(p); #ifdef CONFIG_NUMA diff --git a/kernel/sched/build_policy.c b/kernel/sched/build_policy.c -index 39c315182b35..fae1f5c921eb 100644 +index 39c315182b359..fae1f5c921eb3 100644 --- a/kernel/sched/build_policy.c +++ b/kernel/sched/build_policy.c @@ -16,18 +16,25 @@ @@ -1118,7 +1128,7 @@ index 39c315182b35..fae1f5c921eb 100644 + #include "syscalls.c" diff --git a/kernel/sched/core.c b/kernel/sched/core.c -index f3951e4a55e5..c792a6feb7a9 100644 +index f3951e4a55e5b..c792a6feb7a93 100644 --- a/kernel/sched/core.c +++ b/kernel/sched/core.c @@ -169,7 +169,10 @@ static inline int __task_prio(const struct task_struct *p) @@ -1777,7 +1787,7 @@ index f3951e4a55e5..c792a6feb7a9 100644 +} +#endif /* CONFIG_SCHED_CLASS_EXT */ diff --git a/kernel/sched/cpufreq_schedutil.c b/kernel/sched/cpufreq_schedutil.c -index eece6244f9d2..e683e5d08daa 100644 +index eece6244f9d2f..e683e5d08daae 100644 --- a/kernel/sched/cpufreq_schedutil.c +++ b/kernel/sched/cpufreq_schedutil.c @@ -197,8 +197,10 @@ unsigned long sugov_effective_cpu_perf(int cpu, unsigned long actual, @@ -1866,7 +1876,7 @@ index eece6244f9d2..e683e5d08daa 100644 cpufreq_driver_adjust_perf(sg_cpu->cpu, sg_cpu->bw_min, diff --git a/kernel/sched/debug.c b/kernel/sched/debug.c -index c1eb9a1afd13..c057ef46c5f8 100644 +index c1eb9a1afd13e..c057ef46c5f80 100644 --- a/kernel/sched/debug.c +++ b/kernel/sched/debug.c @@ -1090,6 +1090,9 @@ void proc_sched_show_task(struct task_struct *p, struct pid_namespace *ns, @@ -1881,10 +1891,10 @@ index c1eb9a1afd13..c057ef46c5f8 100644 diff --git a/kernel/sched/ext.c b/kernel/sched/ext.c new file mode 100644 -index 000000000000..4509fd38d92b +index 0000000000000..25fadfaace33c --- /dev/null +++ b/kernel/sched/ext.c -@@ -0,0 +1,7213 @@ +@@ -0,0 +1,7262 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * BPF extensible scheduler class: Documentation/scheduler/sched-ext.rst @@ -2668,7 +2678,6 @@ index 000000000000..4509fd38d92b +}; + +enum scx_ops_enable_state { -+ SCX_OPS_PREPPING, + SCX_OPS_ENABLING, + SCX_OPS_ENABLED, + SCX_OPS_DISABLING, @@ -2676,7 +2685,6 @@ index 000000000000..4509fd38d92b +}; + +static const char *scx_ops_enable_state_str[] = { -+ [SCX_OPS_PREPPING] = "prepping", + [SCX_OPS_ENABLING] = "enabling", + [SCX_OPS_ENABLED] = "enabled", + [SCX_OPS_DISABLING] = "disabling", @@ -2744,6 +2752,7 @@ index 000000000000..4509fd38d92b +DEFINE_STATIC_PERCPU_RWSEM(scx_fork_rwsem); +static atomic_t scx_ops_enable_state_var = ATOMIC_INIT(SCX_OPS_DISABLED); +static atomic_t scx_ops_bypass_depth = ATOMIC_INIT(0); ++static bool scx_ops_init_task_enabled; +static bool scx_switching_all; +DEFINE_STATIC_KEY_FALSE(__scx_switched_all); + @@ -2765,6 +2774,13 @@ index 000000000000..4509fd38d92b +static atomic_long_t scx_hotplug_seq = ATOMIC_LONG_INIT(0); + +/* ++ * A monotically increasing sequence number that is incremented every time a ++ * scheduler is enabled. This can be used by to check if any custom sched_ext ++ * scheduler has ever been used in the system. ++ */ ++static atomic_long_t scx_enable_seq = ATOMIC_LONG_INIT(0); ++ ++/* + * The maximum amount of time in jiffies that a task may be runnable without + * being scheduled on a CPU. If this timeout is exceeded, it will trigger + * scx_ops_error(). @@ -2808,8 +2824,15 @@ index 000000000000..4509fd38d92b + */ +static DEFINE_PER_CPU(struct task_struct *, direct_dispatch_task); + -+/* dispatch queues */ -+static struct scx_dispatch_q __cacheline_aligned_in_smp scx_dsq_global; ++/* ++ * Dispatch queues. ++ * ++ * The global DSQ (%SCX_DSQ_GLOBAL) is split per-node for scalability. This is ++ * to avoid live-locking in bypass mode where all tasks are dispatched to ++ * %SCX_DSQ_GLOBAL and all CPUs consume from it. If per-node split isn't ++ * sufficient, it can be further split. ++ */ ++static struct scx_dispatch_q **global_dsqs; + +static const struct rhashtable_params dsq_hash_params = { + .key_len = 8, @@ -2912,6 +2935,16 @@ index 000000000000..4509fd38d92b + return (s32)(a - b) < 0; +} + ++static struct scx_dispatch_q *find_global_dsq(struct task_struct *p) ++{ ++ return global_dsqs[cpu_to_node(task_cpu(p))]; ++} ++ ++static struct scx_dispatch_q *find_user_dsq(u64 dsq_id) ++{ ++ return rhashtable_lookup_fast(&dsq_hash, &dsq_id, dsq_hash_params); ++} ++ +/* + * scx_kf_mask enforcement. Some kfuncs can only be called from specific SCX + * ops. When invoking SCX ops, SCX_CALL_OP[_RET]() should be used to indicate @@ -3520,7 +3553,7 @@ index 000000000000..4509fd38d92b + scx_ops_error("attempting to dispatch to a destroyed dsq"); + /* fall back to the global dsq */ + raw_spin_unlock(&dsq->lock); -+ dsq = &scx_dsq_global; ++ dsq = find_global_dsq(p); + raw_spin_lock(&dsq->lock); + } + } @@ -3686,21 +3719,6 @@ index 000000000000..4509fd38d92b + raw_spin_unlock(&dsq->lock); +} + -+static struct scx_dispatch_q *find_user_dsq(u64 dsq_id) -+{ -+ return rhashtable_lookup_fast(&dsq_hash, &dsq_id, dsq_hash_params); -+} -+ -+static struct scx_dispatch_q *find_non_local_dsq(u64 dsq_id) -+{ -+ lockdep_assert(rcu_read_lock_any_held()); -+ -+ if (dsq_id == SCX_DSQ_GLOBAL) -+ return &scx_dsq_global; -+ else -+ return find_user_dsq(dsq_id); -+} -+ +static struct scx_dispatch_q *find_dsq_for_dispatch(struct rq *rq, u64 dsq_id, + struct task_struct *p) +{ @@ -3713,16 +3731,20 @@ index 000000000000..4509fd38d92b + s32 cpu = dsq_id & SCX_DSQ_LOCAL_CPU_MASK; + + if (!ops_cpu_valid(cpu, "in SCX_DSQ_LOCAL_ON dispatch verdict")) -+ return &scx_dsq_global; ++ return find_global_dsq(p); + + return &cpu_rq(cpu)->scx.local_dsq; + } + -+ dsq = find_non_local_dsq(dsq_id); ++ if (dsq_id == SCX_DSQ_GLOBAL) ++ dsq = find_global_dsq(p); ++ else ++ dsq = find_user_dsq(dsq_id); ++ + if (unlikely(!dsq)) { + scx_ops_error("non-existent DSQ 0x%llx for %s[%d]", + dsq_id, p->comm, p->pid); -+ return &scx_dsq_global; ++ return find_global_dsq(p); + } + + return dsq; @@ -3903,7 +3925,7 @@ index 000000000000..4509fd38d92b +global: + touch_core_sched(rq, p); /* see the comment in local: */ + p->scx.slice = SCX_SLICE_DFL; -+ dispatch_enqueue(&scx_dsq_global, p, enq_flags); ++ dispatch_enqueue(find_global_dsq(p), p, enq_flags); +} + +static bool task_runnable(const struct task_struct *p) @@ -4248,6 +4270,7 @@ index 000000000000..4509fd38d92b + } +} +#else /* CONFIG_SMP */ ++static inline void move_remote_task_to_local_dsq(struct task_struct *p, u64 enq_flags, struct rq *src_rq, struct rq *dst_rq) { WARN_ON_ONCE(1); } +static inline bool task_can_run_on_remote_rq(struct task_struct *p, struct rq *rq, bool trigger_error) { return false; } +static inline bool consume_remote_task(struct rq *this_rq, struct task_struct *p, struct scx_dispatch_q *dsq, struct rq *task_rq) { return false; } +#endif /* CONFIG_SMP */ @@ -4287,6 +4310,13 @@ index 000000000000..4509fd38d92b + return false; +} + ++static bool consume_global_dsq(struct rq *rq) ++{ ++ int node = cpu_to_node(cpu_of(rq)); ++ ++ return consume_dispatch_q(rq, global_dsqs[node]); ++} ++ +/** + * dispatch_to_local_dsq - Dispatch a task to a local dsq + * @rq: current rq which is locked @@ -4320,7 +4350,8 @@ index 000000000000..4509fd38d92b + +#ifdef CONFIG_SMP + if (unlikely(!task_can_run_on_remote_rq(p, dst_rq, true))) { -+ dispatch_enqueue(&scx_dsq_global, p, enq_flags | SCX_ENQ_CLEAR_OPSS); ++ dispatch_enqueue(find_global_dsq(p), p, ++ enq_flags | SCX_ENQ_CLEAR_OPSS); + return; + } + @@ -4527,7 +4558,7 @@ index 000000000000..4509fd38d92b + if (rq->scx.local_dsq.nr) + goto has_tasks; + -+ if (consume_dispatch_q(rq, &scx_dsq_global)) ++ if (consume_global_dsq(rq)) + goto has_tasks; + + if (!SCX_HAS_OP(dispatch) || scx_rq_bypassing(rq) || !scx_rq_online(rq)) @@ -4552,7 +4583,7 @@ index 000000000000..4509fd38d92b + + if (rq->scx.local_dsq.nr) + goto has_tasks; -+ if (consume_dispatch_q(rq, &scx_dsq_global)) ++ if (consume_global_dsq(rq)) + goto has_tasks; + + /* @@ -5465,7 +5496,7 @@ index 000000000000..4509fd38d92b +{ + percpu_rwsem_assert_held(&scx_fork_rwsem); + -+ if (scx_enabled()) ++ if (scx_ops_init_task_enabled) + return scx_ops_init_task(p, task_group(p), true); + else + return 0; @@ -5473,7 +5504,7 @@ index 000000000000..4509fd38d92b + +void scx_post_fork(struct task_struct *p) +{ -+ if (scx_enabled()) { ++ if (scx_ops_init_task_enabled) { + scx_set_task_state(p, SCX_TASK_READY); + + /* @@ -5605,6 +5636,7 @@ index 000000000000..4509fd38d92b +#ifdef CONFIG_EXT_GROUP_SCHED + +DEFINE_STATIC_PERCPU_RWSEM(scx_cgroup_rwsem); ++static bool scx_cgroup_enabled; +static bool cgroup_warned_missing_weight; +static bool cgroup_warned_missing_idle; + @@ -5624,8 +5656,7 @@ index 000000000000..4509fd38d92b + +static void scx_cgroup_warn_missing_idle(struct task_group *tg) +{ -+ if (scx_ops_enable_state() == SCX_OPS_DISABLED || -+ cgroup_warned_missing_idle) ++ if (!scx_cgroup_enabled || cgroup_warned_missing_idle) + return; + + if (!tg->idle) @@ -5646,15 +5677,18 @@ index 000000000000..4509fd38d92b + + scx_cgroup_warn_missing_weight(tg); + -+ if (SCX_HAS_OP(cgroup_init)) { -+ struct scx_cgroup_init_args args = { .weight = tg->scx_weight }; ++ if (scx_cgroup_enabled) { ++ if (SCX_HAS_OP(cgroup_init)) { ++ struct scx_cgroup_init_args args = ++ { .weight = tg->scx_weight }; + -+ ret = SCX_CALL_OP_RET(SCX_KF_UNLOCKED, cgroup_init, -+ tg->css.cgroup, &args); -+ if (!ret) ++ ret = SCX_CALL_OP_RET(SCX_KF_UNLOCKED, cgroup_init, ++ tg->css.cgroup, &args); ++ if (ret) ++ ret = ops_sanitize_err("cgroup_init", ret); ++ } ++ if (ret == 0) + tg->scx_flags |= SCX_TG_ONLINE | SCX_TG_INITED; -+ else -+ ret = ops_sanitize_err("cgroup_init", ret); + } else { + tg->scx_flags |= SCX_TG_ONLINE; + } @@ -5685,7 +5719,7 @@ index 000000000000..4509fd38d92b + /* released in scx_finish/cancel_attach() */ + percpu_down_read(&scx_cgroup_rwsem); + -+ if (!scx_enabled()) ++ if (!scx_cgroup_enabled) + return 0; + + cgroup_taskset_for_each(p, css, tset) { @@ -5728,7 +5762,7 @@ index 000000000000..4509fd38d92b + +void scx_move_task(struct task_struct *p) +{ -+ if (!scx_enabled()) ++ if (!scx_cgroup_enabled) + return; + + /* @@ -5764,7 +5798,7 @@ index 000000000000..4509fd38d92b + struct cgroup_subsys_state *css; + struct task_struct *p; + -+ if (!scx_enabled()) ++ if (!scx_cgroup_enabled) + goto out_unlock; + + cgroup_taskset_for_each(p, css, tset) { @@ -5781,7 +5815,7 @@ index 000000000000..4509fd38d92b +{ + percpu_down_read(&scx_cgroup_rwsem); + -+ if (tg->scx_weight != weight) { ++ if (scx_cgroup_enabled && tg->scx_weight != weight) { + if (SCX_HAS_OP(cgroup_set_weight)) + SCX_CALL_OP(SCX_KF_UNLOCKED, cgroup_set_weight, + tg_cgrp(tg), weight); @@ -5959,6 +5993,9 @@ index 000000000000..4509fd38d92b + + percpu_rwsem_assert_held(&scx_cgroup_rwsem); + ++ WARN_ON_ONCE(!scx_cgroup_enabled); ++ scx_cgroup_enabled = false; ++ + /* + * scx_tg_on/offline() are excluded through scx_cgroup_rwsem. If we walk + * cgroups and exit all the inited ones, all online cgroups are exited. @@ -6034,6 +6071,9 @@ index 000000000000..4509fd38d92b + } + rcu_read_unlock(); + ++ WARN_ON_ONCE(scx_cgroup_enabled); ++ scx_cgroup_enabled = true; ++ + return 0; +} + @@ -6082,11 +6122,19 @@ index 000000000000..4509fd38d92b +} +SCX_ATTR(hotplug_seq); + ++static ssize_t scx_attr_enable_seq_show(struct kobject *kobj, ++ struct kobj_attribute *ka, char *buf) ++{ ++ return sysfs_emit(buf, "%ld\n", atomic_long_read(&scx_enable_seq)); ++} ++SCX_ATTR(enable_seq); ++ +static struct attribute *scx_global_attrs[] = { + &scx_attr_state.attr, + &scx_attr_switch_all.attr, + &scx_attr_nr_rejected.attr, + &scx_attr_hotplug_seq.attr, ++ &scx_attr_enable_seq.attr, + NULL, +}; + @@ -6343,19 +6391,23 @@ index 000000000000..4509fd38d92b + WRITE_ONCE(scx_switching_all, false); + + /* -+ * Avoid racing against fork and cgroup changes. See scx_ops_enable() -+ * for explanation on the locking order. ++ * Shut down cgroup support before tasks so that the cgroup attach path ++ * doesn't race against scx_ops_exit_task(). + */ -+ percpu_down_write(&scx_fork_rwsem); -+ cpus_read_lock(); + scx_cgroup_lock(); ++ scx_cgroup_exit(); ++ scx_cgroup_unlock(); + -+ spin_lock_irq(&scx_tasks_lock); -+ scx_task_iter_init(&sti); + /* + * The BPF scheduler is going away. All tasks including %TASK_DEAD ones + * must be switched out and exited synchronously. + */ ++ percpu_down_write(&scx_fork_rwsem); ++ ++ scx_ops_init_task_enabled = false; ++ ++ spin_lock_irq(&scx_tasks_lock); ++ scx_task_iter_init(&sti); + while ((p = scx_task_iter_next_locked(&sti))) { + const struct sched_class *old_class = p->sched_class; + struct sched_enq_and_set_ctx ctx; @@ -6373,31 +6425,27 @@ index 000000000000..4509fd38d92b + } + scx_task_iter_exit(&sti); + spin_unlock_irq(&scx_tasks_lock); ++ percpu_up_write(&scx_fork_rwsem); + + /* no task is on scx, turn off all the switches and flush in-progress calls */ -+ static_branch_disable_cpuslocked(&__scx_ops_enabled); ++ static_branch_disable(&__scx_ops_enabled); + for (i = SCX_OPI_BEGIN; i < SCX_OPI_END; i++) -+ static_branch_disable_cpuslocked(&scx_has_op[i]); -+ static_branch_disable_cpuslocked(&scx_ops_enq_last); -+ static_branch_disable_cpuslocked(&scx_ops_enq_exiting); -+ static_branch_disable_cpuslocked(&scx_ops_cpu_preempt); -+ static_branch_disable_cpuslocked(&scx_builtin_idle_enabled); ++ static_branch_disable(&scx_has_op[i]); ++ static_branch_disable(&scx_ops_enq_last); ++ static_branch_disable(&scx_ops_enq_exiting); ++ static_branch_disable(&scx_ops_cpu_preempt); ++ static_branch_disable(&scx_builtin_idle_enabled); + synchronize_rcu(); + -+ scx_cgroup_exit(); -+ -+ scx_cgroup_unlock(); -+ cpus_read_unlock(); -+ percpu_up_write(&scx_fork_rwsem); -+ + if (ei->kind >= SCX_EXIT_ERROR) { + pr_err("sched_ext: BPF scheduler \"%s\" disabled (%s)\n", + scx_ops.name, ei->reason); + + if (ei->msg[0] != '\0') + pr_err("sched_ext: %s: %s\n", scx_ops.name, ei->msg); -+ ++#ifdef CONFIG_STACKTRACE + stack_trace_print(ei->bt, ei->bt_len, 2); ++#endif + } else { + pr_info("sched_ext: BPF scheduler \"%s\" disabled (%s)\n", + scx_ops.name, ei->reason); @@ -6774,10 +6822,10 @@ index 000000000000..4509fd38d92b + return; + + ei->exit_code = exit_code; -+ ++#ifdef CONFIG_STACKTRACE + if (kind >= SCX_EXIT_ERROR) + ei->bt_len = stack_trace_save(ei->bt, SCX_EXIT_BT_LEN, 1); -+ ++#endif + va_start(args, fmt); + vscnprintf(ei->msg, SCX_EXIT_MSG_LEN, fmt, args); + va_end(args); @@ -6840,7 +6888,7 @@ index 000000000000..4509fd38d92b + struct scx_task_iter sti; + struct task_struct *p; + unsigned long timeout; -+ int i, cpu, ret; ++ int i, cpu, node, ret; + + if (!cpumask_equal(housekeeping_cpumask(HK_TYPE_DOMAIN), + cpu_possible_mask)) { @@ -6859,6 +6907,34 @@ index 000000000000..4509fd38d92b + } + } + ++ if (!global_dsqs) { ++ struct scx_dispatch_q **dsqs; ++ ++ dsqs = kcalloc(nr_node_ids, sizeof(dsqs[0]), GFP_KERNEL); ++ if (!dsqs) { ++ ret = -ENOMEM; ++ goto err_unlock; ++ } ++ ++ for_each_node_state(node, N_POSSIBLE) { ++ struct scx_dispatch_q *dsq; ++ ++ dsq = kzalloc_node(sizeof(*dsq), GFP_KERNEL, node); ++ if (!dsq) { ++ for_each_node_state(node, N_POSSIBLE) ++ kfree(dsqs[node]); ++ kfree(dsqs); ++ ret = -ENOMEM; ++ goto err_unlock; ++ } ++ ++ init_dsq(dsq, SCX_DSQ_GLOBAL); ++ dsqs[node] = dsq; ++ } ++ ++ global_dsqs = dsqs; ++ } ++ + if (scx_ops_enable_state() != SCX_OPS_DISABLED) { + ret = -EBUSY; + goto err_unlock; @@ -6882,12 +6958,12 @@ index 000000000000..4509fd38d92b + } + + /* -+ * Set scx_ops, transition to PREPPING and clear exit info to arm the ++ * Set scx_ops, transition to ENABLING and clear exit info to arm the + * disable path. Failure triggers full disabling from here on. + */ + scx_ops = *ops; + -+ WARN_ON_ONCE(scx_ops_set_enable_state(SCX_OPS_PREPPING) != ++ WARN_ON_ONCE(scx_ops_set_enable_state(SCX_OPS_ENABLING) != + SCX_OPS_DISABLED); + + atomic_set(&scx_exit_kind, SCX_EXIT_NONE); @@ -6908,7 +6984,8 @@ index 000000000000..4509fd38d92b + ret = SCX_CALL_OP_RET(SCX_KF_UNLOCKED, init); + if (ret) { + ret = ops_sanitize_err("init", ret); -+ goto err_disable_unlock_cpus; ++ cpus_read_unlock(); ++ goto err_disable; + } + } + @@ -6916,6 +6993,7 @@ index 000000000000..4509fd38d92b + if (((void (**)(void))ops)[i]) + static_branch_enable_cpuslocked(&scx_has_op[i]); + ++ check_hotplug_seq(ops); + cpus_read_unlock(); + + ret = validate_ops(ops); @@ -6943,57 +7021,40 @@ index 000000000000..4509fd38d92b + scx_watchdog_timeout / 2); + + /* -+ * Lock out forks, cgroup on/offlining and moves before opening the -+ * floodgate so that they don't wander into the operations prematurely. -+ * -+ * We don't need to keep the CPUs stable but static_branch_*() requires -+ * cpus_read_lock() and scx_cgroup_rwsem must nest inside -+ * cpu_hotplug_lock because of the following dependency chain: -+ * -+ * cpu_hotplug_lock --> cgroup_threadgroup_rwsem --> scx_cgroup_rwsem -+ * -+ * So, we need to do cpus_read_lock() before scx_cgroup_lock() and use -+ * static_branch_*_cpuslocked(). -+ * -+ * Note that cpu_hotplug_lock must nest inside scx_fork_rwsem due to the -+ * following dependency chain: -+ * -+ * scx_fork_rwsem --> pernet_ops_rwsem --> cpu_hotplug_lock ++ * Once __scx_ops_enabled is set, %current can be switched to SCX ++ * anytime. This can lead to stalls as some BPF schedulers (e.g. ++ * userspace scheduling) may not function correctly before all tasks are ++ * switched. Init in bypass mode to guarantee forward progress. + */ -+ percpu_down_write(&scx_fork_rwsem); -+ cpus_read_lock(); -+ scx_cgroup_lock(); -+ -+ check_hotplug_seq(ops); ++ scx_ops_bypass(true); + + for (i = SCX_OPI_NORMAL_BEGIN; i < SCX_OPI_NORMAL_END; i++) + if (((void (**)(void))ops)[i]) -+ static_branch_enable_cpuslocked(&scx_has_op[i]); ++ static_branch_enable(&scx_has_op[i]); + + if (ops->flags & SCX_OPS_ENQ_LAST) -+ static_branch_enable_cpuslocked(&scx_ops_enq_last); ++ static_branch_enable(&scx_ops_enq_last); + + if (ops->flags & SCX_OPS_ENQ_EXITING) -+ static_branch_enable_cpuslocked(&scx_ops_enq_exiting); ++ static_branch_enable(&scx_ops_enq_exiting); + if (scx_ops.cpu_acquire || scx_ops.cpu_release) -+ static_branch_enable_cpuslocked(&scx_ops_cpu_preempt); ++ static_branch_enable(&scx_ops_cpu_preempt); + + if (!ops->update_idle || (ops->flags & SCX_OPS_KEEP_BUILTIN_IDLE)) { + reset_idle_masks(); -+ static_branch_enable_cpuslocked(&scx_builtin_idle_enabled); ++ static_branch_enable(&scx_builtin_idle_enabled); + } else { -+ static_branch_disable_cpuslocked(&scx_builtin_idle_enabled); ++ static_branch_disable(&scx_builtin_idle_enabled); + } + + /* -+ * All cgroups should be initialized before letting in tasks. cgroup -+ * on/offlining and task migrations are already locked out. ++ * Lock out forks, cgroup on/offlining and moves before opening the ++ * floodgate so that they don't wander into the operations prematurely. + */ -+ ret = scx_cgroup_init(); -+ if (ret) -+ goto err_disable_unlock_all; ++ percpu_down_write(&scx_fork_rwsem); + -+ static_branch_enable_cpuslocked(&__scx_ops_enabled); ++ WARN_ON_ONCE(scx_ops_init_task_enabled); ++ scx_ops_init_task_enabled = true; + + /* + * Enable ops for every task. Fork is excluded by scx_fork_rwsem @@ -7001,9 +7062,19 @@ index 000000000000..4509fd38d92b + * leaving as sched_ext_free() can handle both prepped and enabled + * tasks. Prep all tasks first and then enable them with preemption + * disabled. ++ * ++ * All cgroups should be initialized before scx_ops_init_task() so that ++ * the BPF scheduler can reliably track each task's cgroup membership ++ * from scx_ops_init_task(). Lock out cgroup on/offlining and task ++ * migrations while tasks are being initialized so that ++ * scx_cgroup_can_attach() never sees uninitialized tasks. + */ -+ spin_lock_irq(&scx_tasks_lock); ++ scx_cgroup_lock(); ++ ret = scx_cgroup_init(); ++ if (ret) ++ goto err_disable_unlock_all; + ++ spin_lock_irq(&scx_tasks_lock); + scx_task_iter_init(&sti); + while ((p = scx_task_iter_next_locked(&sti))) { + /* @@ -7028,43 +7099,30 @@ index 000000000000..4509fd38d92b + goto err_disable_unlock_all; + } + ++ scx_set_task_state(p, SCX_TASK_READY); ++ + put_task_struct(p); + spin_lock_irq(&scx_tasks_lock); + } + scx_task_iter_exit(&sti); ++ spin_unlock_irq(&scx_tasks_lock); ++ scx_cgroup_unlock(); ++ percpu_up_write(&scx_fork_rwsem); + + /* -+ * All tasks are prepped but are still ops-disabled. Ensure that -+ * %current can't be scheduled out and switch everyone. -+ * preempt_disable() is necessary because we can't guarantee that -+ * %current won't be starved if scheduled out while switching. ++ * All tasks are READY. It's safe to turn on scx_enabled() and switch ++ * all eligible tasks. + */ -+ preempt_disable(); ++ WRITE_ONCE(scx_switching_all, !(ops->flags & SCX_OPS_SWITCH_PARTIAL)); ++ static_branch_enable(&__scx_ops_enabled); + + /* -+ * From here on, the disable path must assume that tasks have ops -+ * enabled and need to be recovered. -+ * -+ * Transition to ENABLING fails iff the BPF scheduler has already -+ * triggered scx_bpf_error(). Returning an error code here would lose -+ * the recorded error information. Exit indicating success so that the -+ * error is notified through ops.exit() with all the details. -+ */ -+ if (!scx_ops_tryset_enable_state(SCX_OPS_ENABLING, SCX_OPS_PREPPING)) { -+ preempt_enable(); -+ spin_unlock_irq(&scx_tasks_lock); -+ WARN_ON_ONCE(atomic_read(&scx_exit_kind) == SCX_EXIT_NONE); -+ ret = 0; -+ goto err_disable_unlock_all; -+ } -+ -+ /* -+ * We're fully committed and can't fail. The PREPPED -> ENABLED ++ * We're fully committed and can't fail. The task READY -> ENABLED + * transitions here are synchronized against sched_ext_free() through + * scx_tasks_lock. + */ -+ WRITE_ONCE(scx_switching_all, !(ops->flags & SCX_OPS_SWITCH_PARTIAL)); -+ ++ percpu_down_write(&scx_fork_rwsem); ++ spin_lock_irq(&scx_tasks_lock); + scx_task_iter_init(&sti); + while ((p = scx_task_iter_next_locked(&sti))) { + const struct sched_class *old_class = p->sched_class; @@ -7072,7 +7130,6 @@ index 000000000000..4509fd38d92b + + sched_deq_and_put_task(p, DEQUEUE_SAVE | DEQUEUE_MOVE, &ctx); + -+ scx_set_task_state(p, SCX_TASK_READY); + __setscheduler_prio(p, p->prio); + check_class_changing(task_rq(p), p, old_class); + @@ -7081,14 +7138,16 @@ index 000000000000..4509fd38d92b + check_class_changed(task_rq(p), p, old_class, p->prio); + } + scx_task_iter_exit(&sti); -+ + spin_unlock_irq(&scx_tasks_lock); -+ preempt_enable(); -+ scx_cgroup_unlock(); -+ cpus_read_unlock(); + percpu_up_write(&scx_fork_rwsem); + -+ /* see above ENABLING transition for the explanation on exiting with 0 */ ++ scx_ops_bypass(false); ++ ++ /* ++ * Returning an error code here would lose the recorded error ++ * information. Exit indicating success so that the error is notified ++ * through ops.exit() with all the details. ++ */ + if (!scx_ops_tryset_enable_state(SCX_OPS_ENABLED, SCX_OPS_ENABLING)) { + WARN_ON_ONCE(atomic_read(&scx_exit_kind) == SCX_EXIT_NONE); + ret = 0; @@ -7103,6 +7162,8 @@ index 000000000000..4509fd38d92b + kobject_uevent(scx_root_kobj, KOBJ_ADD); + mutex_unlock(&scx_ops_enable_mutex); + ++ atomic_long_inc(&scx_enable_seq); ++ + return 0; + +err_del: @@ -7121,8 +7182,7 @@ index 000000000000..4509fd38d92b +err_disable_unlock_all: + scx_cgroup_unlock(); + percpu_up_write(&scx_fork_rwsem); -+err_disable_unlock_cpus: -+ cpus_read_unlock(); ++ scx_ops_bypass(false); +err_disable: + mutex_unlock(&scx_ops_enable_mutex); + /* must be fully disabled before returning */ @@ -7691,7 +7751,6 @@ index 000000000000..4509fd38d92b + SCX_TG_ONLINE); + + BUG_ON(rhashtable_init(&dsq_hash, &dsq_hash_params)); -+ init_dsq(&scx_dsq_global, SCX_DSQ_GLOBAL); +#ifdef CONFIG_SMP + BUG_ON(!alloc_cpumask_var(&idle_masks.cpu, GFP_KERNEL)); + BUG_ON(!alloc_cpumask_var(&idle_masks.smt, GFP_KERNEL)); @@ -7967,7 +8026,7 @@ index 000000000000..4509fd38d92b + if (dst_dsq->id == SCX_DSQ_LOCAL) { + dst_rq = container_of(dst_dsq, struct rq, scx.local_dsq); + if (!task_can_run_on_remote_rq(p, dst_rq, true)) { -+ dst_dsq = &scx_dsq_global; ++ dst_dsq = find_global_dsq(p); + dst_rq = src_rq; + } + } else { @@ -8084,7 +8143,7 @@ index 000000000000..4509fd38d92b + + flush_dispatch_buf(dspc->rq); + -+ dsq = find_non_local_dsq(dsq_id); ++ dsq = find_user_dsq(dsq_id); + if (unlikely(!dsq)) { + scx_ops_error("invalid DSQ ID 0x%016llx", dsq_id); + return false; @@ -8405,7 +8464,7 @@ index 000000000000..4509fd38d92b + goto out; + } + } else { -+ dsq = find_non_local_dsq(dsq_id); ++ dsq = find_user_dsq(dsq_id); + if (dsq) { + ret = READ_ONCE(dsq->nr); + goto out; @@ -8454,7 +8513,7 @@ index 000000000000..4509fd38d92b + if (flags & ~__SCX_DSQ_ITER_USER_FLAGS) + return -EINVAL; + -+ kit->dsq = find_non_local_dsq(dsq_id); ++ kit->dsq = find_user_dsq(dsq_id); + if (!kit->dsq) + return -ENOENT; + @@ -9100,7 +9159,7 @@ index 000000000000..4509fd38d92b +__initcall(scx_init); diff --git a/kernel/sched/ext.h b/kernel/sched/ext.h new file mode 100644 -index 000000000000..246019519231 +index 0000000000000..246019519231c --- /dev/null +++ b/kernel/sched/ext.h @@ -0,0 +1,91 @@ @@ -9196,7 +9255,7 @@ index 000000000000..246019519231 +#endif /* CONFIG_EXT_GROUP_SCHED */ +#endif /* CONFIG_CGROUP_SCHED */ diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c -index 5dc714fd8a22..2928026d76a3 100644 +index 5dc714fd8a226..2928026d76a39 100644 --- a/kernel/sched/fair.c +++ b/kernel/sched/fair.c @@ -3848,7 +3848,8 @@ static void reweight_entity(struct cfs_rq *cfs_rq, struct sched_entity *se, @@ -9259,7 +9318,7 @@ index 5dc714fd8a22..2928026d76a3 100644 .switched_from = switched_from_fair, .switched_to = switched_to_fair, diff --git a/kernel/sched/idle.c b/kernel/sched/idle.c -index 6e78d071beb5..c7a218123b7a 100644 +index 6e78d071beb58..c7a218123b7ac 100644 --- a/kernel/sched/idle.c +++ b/kernel/sched/idle.c @@ -452,11 +452,13 @@ static void wakeup_preempt_idle(struct rq *rq, struct task_struct *p, int flags) @@ -9277,7 +9336,7 @@ index 6e78d071beb5..c7a218123b7a 100644 } diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h -index 432b43aa091c..207a04f02b4c 100644 +index 432b43aa091cd..48d893de632bb 100644 --- a/kernel/sched/sched.h +++ b/kernel/sched/sched.h @@ -192,9 +192,18 @@ static inline int idle_policy(int policy) @@ -9325,7 +9384,29 @@ index 432b43aa091c..207a04f02b4c 100644 /* * !! For sched_setattr_nocheck() (kernel) only !! * -@@ -424,6 +451,11 @@ struct task_group { +@@ -397,16 +424,17 @@ struct cfs_bandwidth { + struct task_group { + struct cgroup_subsys_state css; + ++#ifdef CONFIG_GROUP_SCHED_WEIGHT ++ /* A positive value indicates that this is a SCHED_IDLE group. */ ++ int idle; ++#endif ++ + #ifdef CONFIG_FAIR_GROUP_SCHED + /* schedulable entities of this group on each CPU */ + struct sched_entity **se; + /* runqueue "owned" by this group on each CPU */ + struct cfs_rq **cfs_rq; + unsigned long shares; +- +- /* A positive value indicates that this is a SCHED_IDLE group. */ +- int idle; +- + #ifdef CONFIG_SMP + /* + * load_avg can be heavily contended at clock tick time, so put +@@ -424,6 +452,11 @@ struct task_group { struct rt_bandwidth rt_bandwidth; #endif @@ -9337,7 +9418,7 @@ index 432b43aa091c..207a04f02b4c 100644 struct rcu_head rcu; struct list_head list; -@@ -448,7 +480,7 @@ struct task_group { +@@ -448,7 +481,7 @@ struct task_group { }; @@ -9346,7 +9427,7 @@ index 432b43aa091c..207a04f02b4c 100644 #define ROOT_TASK_GROUP_LOAD NICE_0_LOAD /* -@@ -479,6 +511,11 @@ static inline int walk_tg_tree(tg_visitor down, tg_visitor up, void *data) +@@ -479,6 +512,11 @@ static inline int walk_tg_tree(tg_visitor down, tg_visitor up, void *data) return walk_tg_tree_from(&root_task_group, down, up, data); } @@ -9358,16 +9439,17 @@ index 432b43aa091c..207a04f02b4c 100644 extern int tg_nop(struct task_group *tg, void *data); #ifdef CONFIG_FAIR_GROUP_SCHED -@@ -535,6 +572,8 @@ extern void set_task_rq_fair(struct sched_entity *se, +@@ -535,6 +573,9 @@ extern void set_task_rq_fair(struct sched_entity *se, static inline void set_task_rq_fair(struct sched_entity *se, struct cfs_rq *prev, struct cfs_rq *next) { } #endif /* CONFIG_SMP */ +#else /* !CONFIG_FAIR_GROUP_SCHED */ +static inline int sched_group_set_shares(struct task_group *tg, unsigned long shares) { return 0; } ++static inline int sched_group_set_idle(struct task_group *tg, long idle) { return 0; } #endif /* CONFIG_FAIR_GROUP_SCHED */ #else /* CONFIG_CGROUP_SCHED */ -@@ -588,6 +627,11 @@ do { \ +@@ -588,6 +629,11 @@ do { \ # define u64_u32_load(var) u64_u32_load_copy(var, var##_copy) # define u64_u32_store(var, val) u64_u32_store_copy(var, var##_copy, val) @@ -9379,7 +9461,7 @@ index 432b43aa091c..207a04f02b4c 100644 /* CFS-related fields in a runqueue */ struct cfs_rq { struct load_weight load; -@@ -696,6 +740,43 @@ struct cfs_rq { +@@ -696,6 +742,43 @@ struct cfs_rq { #endif /* CONFIG_FAIR_GROUP_SCHED */ }; @@ -9423,7 +9505,7 @@ index 432b43aa091c..207a04f02b4c 100644 static inline int rt_bandwidth_enabled(void) { return sysctl_sched_rt_runtime >= 0; -@@ -996,11 +1077,6 @@ struct uclamp_rq { +@@ -996,11 +1079,6 @@ struct uclamp_rq { DECLARE_STATIC_KEY_FALSE(sched_uclamp_used); #endif /* CONFIG_UCLAMP_TASK */ @@ -9435,7 +9517,7 @@ index 432b43aa091c..207a04f02b4c 100644 /* * This is the main, per-CPU runqueue data structure. * -@@ -1043,6 +1119,9 @@ struct rq { +@@ -1043,6 +1121,9 @@ struct rq { struct cfs_rq cfs; struct rt_rq rt; struct dl_rq dl; @@ -9445,7 +9527,7 @@ index 432b43aa091c..207a04f02b4c 100644 #ifdef CONFIG_FAIR_GROUP_SCHED /* list of leaf cfs_rq on this CPU: */ -@@ -2291,13 +2370,15 @@ struct sched_class { +@@ -2291,13 +2372,15 @@ struct sched_class { void (*wakeup_preempt)(struct rq *rq, struct task_struct *p, int flags); @@ -9462,7 +9544,7 @@ index 432b43aa091c..207a04f02b4c 100644 int (*select_task_rq)(struct task_struct *p, int task_cpu, int flags); struct task_struct * (*pick_task)(struct rq *rq); -@@ -2323,8 +2404,11 @@ struct sched_class { +@@ -2323,8 +2406,11 @@ struct sched_class { * cannot assume the switched_from/switched_to pair is serialized by * rq->lock. They are however serialized by p->pi_lock. */ @@ -9474,7 +9556,7 @@ index 432b43aa091c..207a04f02b4c 100644 void (*prio_changed) (struct rq *this_rq, struct task_struct *task, int oldprio); -@@ -2373,19 +2457,54 @@ const struct sched_class name##_sched_class \ +@@ -2373,19 +2459,54 @@ const struct sched_class name##_sched_class \ extern struct sched_class __sched_class_highest[]; extern struct sched_class __sched_class_lowest[]; @@ -9535,7 +9617,7 @@ index 432b43aa091c..207a04f02b4c 100644 static inline bool sched_stop_runnable(struct rq *rq) { -@@ -2424,6 +2543,19 @@ extern void sched_balance_trigger(struct rq *rq); +@@ -2424,6 +2545,19 @@ extern void sched_balance_trigger(struct rq *rq); extern int __set_cpus_allowed_ptr(struct task_struct *p, struct affinity_context *ctx); extern void set_cpus_allowed_common(struct task_struct *p, struct affinity_context *ctx); @@ -9555,7 +9637,7 @@ index 432b43aa091c..207a04f02b4c 100644 static inline cpumask_t *alloc_user_cpus_ptr(int node) { /* -@@ -2457,6 +2589,11 @@ extern int push_cpu_stop(void *arg); +@@ -2457,6 +2591,11 @@ extern int push_cpu_stop(void *arg); #else /* !CONFIG_SMP: */ @@ -9567,7 +9649,7 @@ index 432b43aa091c..207a04f02b4c 100644 static inline int __set_cpus_allowed_ptr(struct task_struct *p, struct affinity_context *ctx) { -@@ -2510,8 +2647,6 @@ extern void init_sched_dl_class(void); +@@ -2510,8 +2649,6 @@ extern void init_sched_dl_class(void); extern void init_sched_rt_class(void); extern void init_sched_fair_class(void); @@ -9576,7 +9658,7 @@ index 432b43aa091c..207a04f02b4c 100644 extern void resched_curr(struct rq *rq); extern void resched_cpu(int cpu); -@@ -3056,6 +3191,8 @@ static inline void cpufreq_update_util(struct rq *rq, unsigned int flags) { } +@@ -3056,6 +3193,8 @@ static inline void cpufreq_update_util(struct rq *rq, unsigned int flags) { } #ifdef CONFIG_SMP @@ -9585,7 +9667,7 @@ index 432b43aa091c..207a04f02b4c 100644 unsigned long effective_cpu_util(int cpu, unsigned long util_cfs, unsigned long *min, unsigned long *max); -@@ -3099,6 +3236,8 @@ static inline unsigned long cpu_util_rt(struct rq *rq) +@@ -3099,6 +3238,8 @@ static inline unsigned long cpu_util_rt(struct rq *rq) return READ_ONCE(rq->avg_rt.util_avg); } @@ -9594,7 +9676,7 @@ index 432b43aa091c..207a04f02b4c 100644 #endif /* CONFIG_SMP */ #ifdef CONFIG_UCLAMP_TASK -@@ -3609,6 +3748,8 @@ extern void set_load_weight(struct task_struct *p, bool update_load); +@@ -3609,6 +3750,8 @@ extern void set_load_weight(struct task_struct *p, bool update_load); extern void enqueue_task(struct rq *rq, struct task_struct *p, int flags); extern void dequeue_task(struct rq *rq, struct task_struct *p, int flags); @@ -9603,7 +9685,7 @@ index 432b43aa091c..207a04f02b4c 100644 extern void check_class_changed(struct rq *rq, struct task_struct *p, const struct sched_class *prev_class, int oldprio); -@@ -3629,4 +3770,24 @@ static inline void balance_callbacks(struct rq *rq, struct balance_callback *hea +@@ -3629,4 +3772,24 @@ static inline void balance_callbacks(struct rq *rq, struct balance_callback *hea #endif @@ -9629,7 +9711,7 @@ index 432b43aa091c..207a04f02b4c 100644 + #endif /* _KERNEL_SCHED_SCHED_H */ diff --git a/kernel/sched/syscalls.c b/kernel/sched/syscalls.c -index ae1b42775ef9..4fa59c9f69ac 100644 +index ae1b42775ef95..4fa59c9f69ac7 100644 --- a/kernel/sched/syscalls.c +++ b/kernel/sched/syscalls.c @@ -259,6 +259,25 @@ int sched_core_idle_cpu(int cpu) @@ -9694,7 +9776,7 @@ index ae1b42775ef9..4fa59c9f69ac 100644 } return ret; diff --git a/lib/dump_stack.c b/lib/dump_stack.c -index 1a996fbbf50a..388da1aea14a 100644 +index 1a996fbbf50a0..388da1aea14a5 100644 --- a/lib/dump_stack.c +++ b/lib/dump_stack.c @@ -73,6 +73,7 @@ void dump_stack_print_info(const char *log_lvl) @@ -9706,7 +9788,7 @@ index 1a996fbbf50a..388da1aea14a 100644 /** diff --git a/tools/Makefile b/tools/Makefile -index 276f5d0d53a4..278d24723b74 100644 +index 276f5d0d53a44..278d24723b74c 100644 --- a/tools/Makefile +++ b/tools/Makefile @@ -28,6 +28,7 @@ help: @@ -9748,7 +9830,7 @@ index 276f5d0d53a4..278d24723b74 100644 .PHONY: FORCE diff --git a/tools/sched_ext/.gitignore b/tools/sched_ext/.gitignore new file mode 100644 -index 000000000000..d6264fe1c8cd +index 0000000000000..d6264fe1c8cdb --- /dev/null +++ b/tools/sched_ext/.gitignore @@ -0,0 +1,2 @@ @@ -9756,7 +9838,7 @@ index 000000000000..d6264fe1c8cd +build/ diff --git a/tools/sched_ext/Makefile b/tools/sched_ext/Makefile new file mode 100644 -index 000000000000..ca3815e572d8 +index 0000000000000..ca3815e572d8d --- /dev/null +++ b/tools/sched_ext/Makefile @@ -0,0 +1,246 @@ @@ -10008,7 +10090,7 @@ index 000000000000..ca3815e572d8 +.SECONDARY: diff --git a/tools/sched_ext/README.md b/tools/sched_ext/README.md new file mode 100644 -index 000000000000..16a42e4060f6 +index 0000000000000..16a42e4060f64 --- /dev/null +++ b/tools/sched_ext/README.md @@ -0,0 +1,270 @@ @@ -10284,7 +10366,7 @@ index 000000000000..16a42e4060f6 +Seeing `llvm: [ OFF ]` here is not an issue. You can safely ignore. diff --git a/tools/sched_ext/include/bpf-compat/gnu/stubs.h b/tools/sched_ext/include/bpf-compat/gnu/stubs.h new file mode 100644 -index 000000000000..ad7d139ce907 +index 0000000000000..ad7d139ce907b --- /dev/null +++ b/tools/sched_ext/include/bpf-compat/gnu/stubs.h @@ -0,0 +1,11 @@ @@ -10301,10 +10383,10 @@ index 000000000000..ad7d139ce907 + */ diff --git a/tools/sched_ext/include/scx/common.bpf.h b/tools/sched_ext/include/scx/common.bpf.h new file mode 100644 -index 000000000000..f538c75db183 +index 0000000000000..225f61f9bfca8 --- /dev/null +++ b/tools/sched_ext/include/scx/common.bpf.h -@@ -0,0 +1,412 @@ +@@ -0,0 +1,427 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (c) 2022 Meta Platforms, Inc. and affiliates. @@ -10314,7 +10396,13 @@ index 000000000000..f538c75db183 +#ifndef __SCX_COMMON_BPF_H +#define __SCX_COMMON_BPF_H + ++#ifdef LSP ++#define __bpf__ ++#include "../vmlinux/vmlinux.h" ++#else +#include "vmlinux.h" ++#endif ++ +#include +#include +#include @@ -10616,6 +10704,15 @@ index 000000000000..f538c75db183 +u32 bpf_cpumask_any_distribute(const struct cpumask *cpumask) __ksym; +u32 bpf_cpumask_any_and_distribute(const struct cpumask *src1, + const struct cpumask *src2) __ksym; ++u32 bpf_cpumask_weight(const struct cpumask *cpumask) __ksym; ++ ++/* ++ * Access a cpumask in read-only mode (typically to check bits). ++ */ ++const struct cpumask *cast_mask(struct bpf_cpumask *mask) ++{ ++ return (const struct cpumask *)mask; ++} + +/* rcu */ +void bpf_rcu_read_lock(void) __ksym; @@ -10719,7 +10816,7 @@ index 000000000000..f538c75db183 +#endif /* __SCX_COMMON_BPF_H */ diff --git a/tools/sched_ext/include/scx/common.h b/tools/sched_ext/include/scx/common.h new file mode 100644 -index 000000000000..5b0f90152152 +index 0000000000000..5b0f901521523 --- /dev/null +++ b/tools/sched_ext/include/scx/common.h @@ -0,0 +1,75 @@ @@ -10800,10 +10897,10 @@ index 000000000000..5b0f90152152 +#endif /* __SCHED_EXT_COMMON_H */ diff --git a/tools/sched_ext/include/scx/compat.bpf.h b/tools/sched_ext/include/scx/compat.bpf.h new file mode 100644 -index 000000000000..3d2fe1208900 +index 0000000000000..e5afe9efd3f31 --- /dev/null +++ b/tools/sched_ext/include/scx/compat.bpf.h -@@ -0,0 +1,28 @@ +@@ -0,0 +1,47 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (c) 2024 Meta Platforms, Inc. and affiliates. @@ -10821,6 +10918,25 @@ index 000000000000..3d2fe1208900 + __ret; \ +}) + ++/* v6.12: 819513666966 ("sched_ext: Add cgroup support") */ ++#define __COMPAT_scx_bpf_task_cgroup(p) \ ++ (bpf_ksym_exists(scx_bpf_task_cgroup) ? \ ++ scx_bpf_task_cgroup((p)) : NULL) ++ ++/* v6.12: 4c30f5ce4f7a ("sched_ext: Implement scx_bpf_dispatch[_vtime]_from_dsq()") */ ++#define __COMPAT_scx_bpf_dispatch_from_dsq_set_slice(it, slice) \ ++ (bpf_ksym_exists(scx_bpf_dispatch_from_dsq_set_slice) ? \ ++ scx_bpf_dispatch_from_dsq_set_slice((it), (slice)) : (void)0) ++#define __COMPAT_scx_bpf_dispatch_from_dsq_set_vtime(it, vtime) \ ++ (bpf_ksym_exists(scx_bpf_dispatch_from_dsq_set_vtime) ? \ ++ scx_bpf_dispatch_from_dsq_set_vtime((it), (vtime)) : (void)0) ++#define __COMPAT_scx_bpf_dispatch_from_dsq(it, p, dsq_id, enq_flags) \ ++ (bpf_ksym_exists(scx_bpf_dispatch_from_dsq) ? \ ++ scx_bpf_dispatch_from_dsq((it), (p), (dsq_id), (enq_flags)) : false) ++#define __COMPAT_scx_bpf_dispatch_vtime_from_dsq(it, p, dsq_id, enq_flags) \ ++ (bpf_ksym_exists(scx_bpf_dispatch_vtime_from_dsq) ? \ ++ scx_bpf_dispatch_vtime_from_dsq((it), (p), (dsq_id), (enq_flags)) : false) ++ +/* + * Define sched_ext_ops. This may be expanded to define multiple variants for + * backward compatibility. See compat.h::SCX_OPS_LOAD/ATTACH(). @@ -10834,7 +10950,7 @@ index 000000000000..3d2fe1208900 +#endif /* __SCX_COMPAT_BPF_H */ diff --git a/tools/sched_ext/include/scx/compat.h b/tools/sched_ext/include/scx/compat.h new file mode 100644 -index 000000000000..cc56ff9aa252 +index 0000000000000..cc56ff9aa2529 --- /dev/null +++ b/tools/sched_ext/include/scx/compat.h @@ -0,0 +1,186 @@ @@ -11026,10 +11142,10 @@ index 000000000000..cc56ff9aa252 +#endif /* __SCX_COMPAT_H */ diff --git a/tools/sched_ext/include/scx/user_exit_info.h b/tools/sched_ext/include/scx/user_exit_info.h new file mode 100644 -index 000000000000..891693ee604e +index 0000000000000..8ce2734402e1d --- /dev/null +++ b/tools/sched_ext/include/scx/user_exit_info.h -@@ -0,0 +1,111 @@ +@@ -0,0 +1,115 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Define struct user_exit_info which is shared between BPF and userspace parts @@ -11057,7 +11173,11 @@ index 000000000000..891693ee604e + +#ifdef __bpf__ + ++#ifdef LSP ++#include "../vmlinux/vmlinux.h" ++#else +#include "vmlinux.h" ++#endif +#include + +#define UEI_DEFINE(__name) \ @@ -11143,7 +11263,7 @@ index 000000000000..891693ee604e +#endif /* __USER_EXIT_INFO_H */ diff --git a/tools/sched_ext/scx_central.bpf.c b/tools/sched_ext/scx_central.bpf.c new file mode 100644 -index 000000000000..8dd8eb73b6b8 +index 0000000000000..8dd8eb73b6b8b --- /dev/null +++ b/tools/sched_ext/scx_central.bpf.c @@ -0,0 +1,361 @@ @@ -11510,7 +11630,7 @@ index 000000000000..8dd8eb73b6b8 + .name = "central"); diff --git a/tools/sched_ext/scx_central.c b/tools/sched_ext/scx_central.c new file mode 100644 -index 000000000000..21deea320bd7 +index 0000000000000..21deea320bd78 --- /dev/null +++ b/tools/sched_ext/scx_central.c @@ -0,0 +1,135 @@ @@ -11651,10 +11771,10 @@ index 000000000000..21deea320bd7 +} diff --git a/tools/sched_ext/scx_flatcg.bpf.c b/tools/sched_ext/scx_flatcg.bpf.c new file mode 100644 -index 000000000000..3ab2b60781a0 +index 0000000000000..b722baf6da4b9 --- /dev/null +++ b/tools/sched_ext/scx_flatcg.bpf.c -@@ -0,0 +1,949 @@ +@@ -0,0 +1,957 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * A demo sched_ext flattened cgroup hierarchy scheduler. It implements @@ -11706,7 +11826,10 @@ index 000000000000..3ab2b60781a0 +/* + * Maximum amount of retries to find a valid cgroup. + */ -+#define CGROUP_MAX_RETRIES 1024 ++enum { ++ FALLBACK_DSQ = 0, ++ CGROUP_MAX_RETRIES = 1024, ++}; + +char _license[] SEC("license") = "GPL"; + @@ -11882,7 +12005,7 @@ index 000000000000..3ab2b60781a0 + break; + + /* -+ * We can be oppotunistic here and not grab the ++ * We can be opportunistic here and not grab the + * cgv_tree_lock and deal with the occasional races. + * However, hweight updates are already cached and + * relatively low-frequency. Let's just do the @@ -11915,8 +12038,7 @@ index 000000000000..3ab2b60781a0 + * and thus can't be updated and repositioned. Instead, we collect the + * vtime deltas separately and apply it asynchronously here. + */ -+ delta = cgc->cvtime_delta; -+ __sync_fetch_and_sub(&cgc->cvtime_delta, delta); ++ delta = __sync_fetch_and_sub(&cgc->cvtime_delta, cgc->cvtime_delta); + cvtime = cgv_node->cvtime + delta; + + /* @@ -12035,12 +12157,12 @@ index 000000000000..3ab2b60781a0 + scx_bpf_dispatch(p, SCX_DSQ_LOCAL, SCX_SLICE_DFL, enq_flags); + } else { + stat_inc(FCG_STAT_GLOBAL); -+ scx_bpf_dispatch(p, SCX_DSQ_GLOBAL, SCX_SLICE_DFL, enq_flags); ++ scx_bpf_dispatch(p, FALLBACK_DSQ, SCX_SLICE_DFL, enq_flags); + } + return; + } + -+ cgrp = scx_bpf_task_cgroup(p); ++ cgrp = __COMPAT_scx_bpf_task_cgroup(p); + cgc = find_cgrp_ctx(cgrp); + if (!cgc) + goto out_release; @@ -12166,7 +12288,7 @@ index 000000000000..3ab2b60781a0 +{ + struct cgroup *cgrp; + -+ cgrp = scx_bpf_task_cgroup(p); ++ cgrp = __COMPAT_scx_bpf_task_cgroup(p); + update_active_weight_sums(cgrp, true); + bpf_cgroup_release(cgrp); +} @@ -12179,7 +12301,7 @@ index 000000000000..3ab2b60781a0 + if (fifo_sched) + return; + -+ cgrp = scx_bpf_task_cgroup(p); ++ cgrp = __COMPAT_scx_bpf_task_cgroup(p); + cgc = find_cgrp_ctx(cgrp); + if (cgc) { + /* @@ -12222,7 +12344,7 @@ index 000000000000..3ab2b60781a0 + if (!taskc->bypassed_at) + return; + -+ cgrp = scx_bpf_task_cgroup(p); ++ cgrp = __COMPAT_scx_bpf_task_cgroup(p); + cgc = find_cgrp_ctx(cgrp); + if (cgc) { + __sync_fetch_and_add(&cgc->cvtime_delta, @@ -12236,7 +12358,7 @@ index 000000000000..3ab2b60781a0 +{ + struct cgroup *cgrp; + -+ cgrp = scx_bpf_task_cgroup(p); ++ cgrp = __COMPAT_scx_bpf_task_cgroup(p); + update_active_weight_sums(cgrp, false); + bpf_cgroup_release(cgrp); +} @@ -12438,7 +12560,7 @@ index 000000000000..3ab2b60781a0 +pick_next_cgroup: + cpuc->cur_at = now; + -+ if (scx_bpf_consume(SCX_DSQ_GLOBAL)) { ++ if (scx_bpf_consume(FALLBACK_DSQ)) { + cpuc->cur_cgid = 0; + return; + } @@ -12495,7 +12617,7 @@ index 000000000000..3ab2b60781a0 + int ret; + + /* -+ * Technically incorrect as cgroup ID is full 64bit while dq ID is ++ * Technically incorrect as cgroup ID is full 64bit while dsq ID is + * 63bit. Should not be a problem in practice and easy to spot in the + * unlikely case that it breaks. + */ @@ -12583,6 +12705,11 @@ index 000000000000..3ab2b60781a0 + p->scx.dsq_vtime = to_cgc->tvtime_now + vtime_delta; +} + ++s32 BPF_STRUCT_OPS_SLEEPABLE(fcg_init) ++{ ++ return scx_bpf_create_dsq(FALLBACK_DSQ, -1); ++} ++ +void BPF_STRUCT_OPS(fcg_exit, struct scx_exit_info *ei) +{ + UEI_RECORD(uei, ei); @@ -12601,12 +12728,13 @@ index 000000000000..3ab2b60781a0 + .cgroup_init = (void *)fcg_cgroup_init, + .cgroup_exit = (void *)fcg_cgroup_exit, + .cgroup_move = (void *)fcg_cgroup_move, ++ .init = (void *)fcg_init, + .exit = (void *)fcg_exit, + .flags = SCX_OPS_HAS_CGROUP_WEIGHT | SCX_OPS_ENQ_EXITING, + .name = "flatcg"); diff --git a/tools/sched_ext/scx_flatcg.c b/tools/sched_ext/scx_flatcg.c new file mode 100644 -index 000000000000..5d24ca9c29d9 +index 0000000000000..5d24ca9c29d9e --- /dev/null +++ b/tools/sched_ext/scx_flatcg.c @@ -0,0 +1,233 @@ @@ -12845,7 +12973,7 @@ index 000000000000..5d24ca9c29d9 +} diff --git a/tools/sched_ext/scx_flatcg.h b/tools/sched_ext/scx_flatcg.h new file mode 100644 -index 000000000000..6f2ea50acb1c +index 0000000000000..6f2ea50acb1cb --- /dev/null +++ b/tools/sched_ext/scx_flatcg.h @@ -0,0 +1,51 @@ @@ -12902,7 +13030,7 @@ index 000000000000..6f2ea50acb1c +#endif /* __SCX_EXAMPLE_FLATCG_H */ diff --git a/tools/sched_ext/scx_qmap.bpf.c b/tools/sched_ext/scx_qmap.bpf.c new file mode 100644 -index 000000000000..391d80b4ac8e +index 0000000000000..5b39bee9eb232 --- /dev/null +++ b/tools/sched_ext/scx_qmap.bpf.c @@ -0,0 +1,813 @@ @@ -13227,11 +13355,11 @@ index 000000000000..391d80b4ac8e + + if (tctx->highpri) { + /* exercise the set_*() and vtime interface too */ -+ scx_bpf_dispatch_from_dsq_set_slice( ++ __COMPAT_scx_bpf_dispatch_from_dsq_set_slice( + BPF_FOR_EACH_ITER, slice_ns * 2); -+ scx_bpf_dispatch_from_dsq_set_vtime( ++ __COMPAT_scx_bpf_dispatch_from_dsq_set_vtime( + BPF_FOR_EACH_ITER, highpri_seq++); -+ scx_bpf_dispatch_vtime_from_dsq( ++ __COMPAT_scx_bpf_dispatch_vtime_from_dsq( + BPF_FOR_EACH_ITER, p, HIGHPRI_DSQ, 0); + } + } @@ -13249,9 +13377,9 @@ index 000000000000..391d80b4ac8e + else + cpu = scx_bpf_pick_any_cpu(p->cpus_ptr, 0); + -+ if (scx_bpf_dispatch_from_dsq(BPF_FOR_EACH_ITER, p, -+ SCX_DSQ_LOCAL_ON | cpu, -+ SCX_ENQ_PREEMPT)) { ++ if (__COMPAT_scx_bpf_dispatch_from_dsq(BPF_FOR_EACH_ITER, p, ++ SCX_DSQ_LOCAL_ON | cpu, ++ SCX_ENQ_PREEMPT)) { + if (cpu == this_cpu) { + dispatched = true; + __sync_fetch_and_add(&nr_expedited_local, 1); @@ -13721,7 +13849,7 @@ index 000000000000..391d80b4ac8e + .name = "qmap"); diff --git a/tools/sched_ext/scx_qmap.c b/tools/sched_ext/scx_qmap.c new file mode 100644 -index 000000000000..ac45a02b4055 +index 0000000000000..ac45a02b40559 --- /dev/null +++ b/tools/sched_ext/scx_qmap.c @@ -0,0 +1,153 @@ @@ -13880,10 +14008,10 @@ index 000000000000..ac45a02b4055 +} diff --git a/tools/sched_ext/scx_show_state.py b/tools/sched_ext/scx_show_state.py new file mode 100644 -index 000000000000..d457d2a74e1e +index 0000000000000..8bc626ede1c48 --- /dev/null +++ b/tools/sched_ext/scx_show_state.py -@@ -0,0 +1,39 @@ +@@ -0,0 +1,40 @@ +#!/usr/bin/env drgn +# +# Copyright (C) 2024 Tejun Heo @@ -13923,9 +14051,10 @@ index 000000000000..d457d2a74e1e +print(f'enable_state : {ops_state_str(enable_state)} ({enable_state})') +print(f'bypass_depth : {read_atomic("scx_ops_bypass_depth")}') +print(f'nr_rejected : {read_atomic("scx_nr_rejected")}') ++print(f'enable_seq : {read_atomic("scx_enable_seq")}') diff --git a/tools/sched_ext/scx_simple.bpf.c b/tools/sched_ext/scx_simple.bpf.c new file mode 100644 -index 000000000000..ed7e8d535fc5 +index 0000000000000..ed7e8d535fc5f --- /dev/null +++ b/tools/sched_ext/scx_simple.bpf.c @@ -0,0 +1,156 @@ @@ -14087,7 +14216,7 @@ index 000000000000..ed7e8d535fc5 + .name = "simple"); diff --git a/tools/sched_ext/scx_simple.c b/tools/sched_ext/scx_simple.c new file mode 100644 -index 000000000000..76d83199545c +index 0000000000000..76d83199545cb --- /dev/null +++ b/tools/sched_ext/scx_simple.c @@ -0,0 +1,107 @@ @@ -14200,7 +14329,7 @@ index 000000000000..76d83199545c +} diff --git a/tools/testing/selftests/sched_ext/.gitignore b/tools/testing/selftests/sched_ext/.gitignore new file mode 100644 -index 000000000000..ae5491a114c0 +index 0000000000000..ae5491a114c09 --- /dev/null +++ b/tools/testing/selftests/sched_ext/.gitignore @@ -0,0 +1,6 @@ @@ -14212,7 +14341,7 @@ index 000000000000..ae5491a114c0 +!config diff --git a/tools/testing/selftests/sched_ext/Makefile b/tools/testing/selftests/sched_ext/Makefile new file mode 100644 -index 000000000000..0754a2c110a1 +index 0000000000000..0754a2c110a1a --- /dev/null +++ b/tools/testing/selftests/sched_ext/Makefile @@ -0,0 +1,218 @@ @@ -14436,7 +14565,7 @@ index 000000000000..0754a2c110a1 +.SECONDARY: diff --git a/tools/testing/selftests/sched_ext/config b/tools/testing/selftests/sched_ext/config new file mode 100644 -index 000000000000..0de9b4ee249d +index 0000000000000..0de9b4ee249d3 --- /dev/null +++ b/tools/testing/selftests/sched_ext/config @@ -0,0 +1,9 @@ @@ -14451,7 +14580,7 @@ index 000000000000..0de9b4ee249d +CONFIG_DEBUG_INFO_BTF=y diff --git a/tools/testing/selftests/sched_ext/create_dsq.bpf.c b/tools/testing/selftests/sched_ext/create_dsq.bpf.c new file mode 100644 -index 000000000000..23f79ed343f0 +index 0000000000000..23f79ed343f02 --- /dev/null +++ b/tools/testing/selftests/sched_ext/create_dsq.bpf.c @@ -0,0 +1,58 @@ @@ -14515,7 +14644,7 @@ index 000000000000..23f79ed343f0 +}; diff --git a/tools/testing/selftests/sched_ext/create_dsq.c b/tools/testing/selftests/sched_ext/create_dsq.c new file mode 100644 -index 000000000000..fa946d9146d4 +index 0000000000000..fa946d9146d4d --- /dev/null +++ b/tools/testing/selftests/sched_ext/create_dsq.c @@ -0,0 +1,57 @@ @@ -14578,7 +14707,7 @@ index 000000000000..fa946d9146d4 +REGISTER_SCX_TEST(&create_dsq) diff --git a/tools/testing/selftests/sched_ext/ddsp_bogus_dsq_fail.bpf.c b/tools/testing/selftests/sched_ext/ddsp_bogus_dsq_fail.bpf.c new file mode 100644 -index 000000000000..e97ad41d354a +index 0000000000000..e97ad41d354ad --- /dev/null +++ b/tools/testing/selftests/sched_ext/ddsp_bogus_dsq_fail.bpf.c @@ -0,0 +1,42 @@ @@ -14626,7 +14755,7 @@ index 000000000000..e97ad41d354a +}; diff --git a/tools/testing/selftests/sched_ext/ddsp_bogus_dsq_fail.c b/tools/testing/selftests/sched_ext/ddsp_bogus_dsq_fail.c new file mode 100644 -index 000000000000..e65d22f23f3b +index 0000000000000..e65d22f23f3bc --- /dev/null +++ b/tools/testing/selftests/sched_ext/ddsp_bogus_dsq_fail.c @@ -0,0 +1,57 @@ @@ -14689,7 +14818,7 @@ index 000000000000..e65d22f23f3b +REGISTER_SCX_TEST(&ddsp_bogus_dsq_fail) diff --git a/tools/testing/selftests/sched_ext/ddsp_vtimelocal_fail.bpf.c b/tools/testing/selftests/sched_ext/ddsp_vtimelocal_fail.bpf.c new file mode 100644 -index 000000000000..dde7e7dafbfb +index 0000000000000..dde7e7dafbfbc --- /dev/null +++ b/tools/testing/selftests/sched_ext/ddsp_vtimelocal_fail.bpf.c @@ -0,0 +1,39 @@ @@ -14734,7 +14863,7 @@ index 000000000000..dde7e7dafbfb +}; diff --git a/tools/testing/selftests/sched_ext/ddsp_vtimelocal_fail.c b/tools/testing/selftests/sched_ext/ddsp_vtimelocal_fail.c new file mode 100644 -index 000000000000..abafee587cd6 +index 0000000000000..abafee587cd60 --- /dev/null +++ b/tools/testing/selftests/sched_ext/ddsp_vtimelocal_fail.c @@ -0,0 +1,56 @@ @@ -14796,7 +14925,7 @@ index 000000000000..abafee587cd6 +REGISTER_SCX_TEST(&ddsp_vtimelocal_fail) diff --git a/tools/testing/selftests/sched_ext/dsp_local_on.bpf.c b/tools/testing/selftests/sched_ext/dsp_local_on.bpf.c new file mode 100644 -index 000000000000..efb4672decb4 +index 0000000000000..efb4672decb41 --- /dev/null +++ b/tools/testing/selftests/sched_ext/dsp_local_on.bpf.c @@ -0,0 +1,65 @@ @@ -14867,7 +14996,7 @@ index 000000000000..efb4672decb4 +}; diff --git a/tools/testing/selftests/sched_ext/dsp_local_on.c b/tools/testing/selftests/sched_ext/dsp_local_on.c new file mode 100644 -index 000000000000..472851b56854 +index 0000000000000..472851b568548 --- /dev/null +++ b/tools/testing/selftests/sched_ext/dsp_local_on.c @@ -0,0 +1,58 @@ @@ -14931,7 +15060,7 @@ index 000000000000..472851b56854 +REGISTER_SCX_TEST(&dsp_local_on) diff --git a/tools/testing/selftests/sched_ext/enq_last_no_enq_fails.bpf.c b/tools/testing/selftests/sched_ext/enq_last_no_enq_fails.bpf.c new file mode 100644 -index 000000000000..b0b99531d5d5 +index 0000000000000..b0b99531d5d5d --- /dev/null +++ b/tools/testing/selftests/sched_ext/enq_last_no_enq_fails.bpf.c @@ -0,0 +1,21 @@ @@ -14958,7 +15087,7 @@ index 000000000000..b0b99531d5d5 +}; diff --git a/tools/testing/selftests/sched_ext/enq_last_no_enq_fails.c b/tools/testing/selftests/sched_ext/enq_last_no_enq_fails.c new file mode 100644 -index 000000000000..2a3eda5e2c0b +index 0000000000000..2a3eda5e2c0b4 --- /dev/null +++ b/tools/testing/selftests/sched_ext/enq_last_no_enq_fails.c @@ -0,0 +1,60 @@ @@ -15024,7 +15153,7 @@ index 000000000000..2a3eda5e2c0b +REGISTER_SCX_TEST(&enq_last_no_enq_fails) diff --git a/tools/testing/selftests/sched_ext/enq_select_cpu_fails.bpf.c b/tools/testing/selftests/sched_ext/enq_select_cpu_fails.bpf.c new file mode 100644 -index 000000000000..b3dfc1033cd6 +index 0000000000000..b3dfc1033cd6a --- /dev/null +++ b/tools/testing/selftests/sched_ext/enq_select_cpu_fails.bpf.c @@ -0,0 +1,43 @@ @@ -15073,7 +15202,7 @@ index 000000000000..b3dfc1033cd6 +}; diff --git a/tools/testing/selftests/sched_ext/enq_select_cpu_fails.c b/tools/testing/selftests/sched_ext/enq_select_cpu_fails.c new file mode 100644 -index 000000000000..dd1350e5f002 +index 0000000000000..dd1350e5f002d --- /dev/null +++ b/tools/testing/selftests/sched_ext/enq_select_cpu_fails.c @@ -0,0 +1,61 @@ @@ -15140,7 +15269,7 @@ index 000000000000..dd1350e5f002 +REGISTER_SCX_TEST(&enq_select_cpu_fails) diff --git a/tools/testing/selftests/sched_ext/exit.bpf.c b/tools/testing/selftests/sched_ext/exit.bpf.c new file mode 100644 -index 000000000000..ae12ddaac921 +index 0000000000000..ae12ddaac921b --- /dev/null +++ b/tools/testing/selftests/sched_ext/exit.bpf.c @@ -0,0 +1,84 @@ @@ -15230,7 +15359,7 @@ index 000000000000..ae12ddaac921 +}; diff --git a/tools/testing/selftests/sched_ext/exit.c b/tools/testing/selftests/sched_ext/exit.c new file mode 100644 -index 000000000000..31bcd06e21cd +index 0000000000000..31bcd06e21cd3 --- /dev/null +++ b/tools/testing/selftests/sched_ext/exit.c @@ -0,0 +1,55 @@ @@ -15291,7 +15420,7 @@ index 000000000000..31bcd06e21cd +REGISTER_SCX_TEST(&exit_test) diff --git a/tools/testing/selftests/sched_ext/exit_test.h b/tools/testing/selftests/sched_ext/exit_test.h new file mode 100644 -index 000000000000..94f0268b9cb8 +index 0000000000000..94f0268b9cb8e --- /dev/null +++ b/tools/testing/selftests/sched_ext/exit_test.h @@ -0,0 +1,20 @@ @@ -15317,7 +15446,7 @@ index 000000000000..94f0268b9cb8 +#endif // # __EXIT_TEST_H__ diff --git a/tools/testing/selftests/sched_ext/hotplug.bpf.c b/tools/testing/selftests/sched_ext/hotplug.bpf.c new file mode 100644 -index 000000000000..8f2601db39f3 +index 0000000000000..8f2601db39f37 --- /dev/null +++ b/tools/testing/selftests/sched_ext/hotplug.bpf.c @@ -0,0 +1,61 @@ @@ -15384,7 +15513,7 @@ index 000000000000..8f2601db39f3 +}; diff --git a/tools/testing/selftests/sched_ext/hotplug.c b/tools/testing/selftests/sched_ext/hotplug.c new file mode 100644 -index 000000000000..87bf220b1bce +index 0000000000000..87bf220b1bcee --- /dev/null +++ b/tools/testing/selftests/sched_ext/hotplug.c @@ -0,0 +1,168 @@ @@ -15558,7 +15687,7 @@ index 000000000000..87bf220b1bce +REGISTER_SCX_TEST(&hotplug_test) diff --git a/tools/testing/selftests/sched_ext/hotplug_test.h b/tools/testing/selftests/sched_ext/hotplug_test.h new file mode 100644 -index 000000000000..73d236f90787 +index 0000000000000..73d236f90787d --- /dev/null +++ b/tools/testing/selftests/sched_ext/hotplug_test.h @@ -0,0 +1,15 @@ @@ -15579,7 +15708,7 @@ index 000000000000..73d236f90787 +#endif // # __HOTPLUG_TEST_H__ diff --git a/tools/testing/selftests/sched_ext/init_enable_count.bpf.c b/tools/testing/selftests/sched_ext/init_enable_count.bpf.c new file mode 100644 -index 000000000000..47ea89a626c3 +index 0000000000000..47ea89a626c37 --- /dev/null +++ b/tools/testing/selftests/sched_ext/init_enable_count.bpf.c @@ -0,0 +1,53 @@ @@ -15638,7 +15767,7 @@ index 000000000000..47ea89a626c3 +}; diff --git a/tools/testing/selftests/sched_ext/init_enable_count.c b/tools/testing/selftests/sched_ext/init_enable_count.c new file mode 100644 -index 000000000000..97d45f1e5597 +index 0000000000000..97d45f1e5597e --- /dev/null +++ b/tools/testing/selftests/sched_ext/init_enable_count.c @@ -0,0 +1,166 @@ @@ -15810,7 +15939,7 @@ index 000000000000..97d45f1e5597 +REGISTER_SCX_TEST(&init_enable_count) diff --git a/tools/testing/selftests/sched_ext/maximal.bpf.c b/tools/testing/selftests/sched_ext/maximal.bpf.c new file mode 100644 -index 000000000000..00bfa9cb95d3 +index 0000000000000..00bfa9cb95d38 --- /dev/null +++ b/tools/testing/selftests/sched_ext/maximal.bpf.c @@ -0,0 +1,164 @@ @@ -15980,7 +16109,7 @@ index 000000000000..00bfa9cb95d3 +}; diff --git a/tools/testing/selftests/sched_ext/maximal.c b/tools/testing/selftests/sched_ext/maximal.c new file mode 100644 -index 000000000000..f38fc973c380 +index 0000000000000..f38fc973c3800 --- /dev/null +++ b/tools/testing/selftests/sched_ext/maximal.c @@ -0,0 +1,51 @@ @@ -16037,7 +16166,7 @@ index 000000000000..f38fc973c380 +REGISTER_SCX_TEST(&maximal) diff --git a/tools/testing/selftests/sched_ext/maybe_null.bpf.c b/tools/testing/selftests/sched_ext/maybe_null.bpf.c new file mode 100644 -index 000000000000..27d0f386acfb +index 0000000000000..27d0f386acfb1 --- /dev/null +++ b/tools/testing/selftests/sched_ext/maybe_null.bpf.c @@ -0,0 +1,36 @@ @@ -16079,7 +16208,7 @@ index 000000000000..27d0f386acfb +}; diff --git a/tools/testing/selftests/sched_ext/maybe_null.c b/tools/testing/selftests/sched_ext/maybe_null.c new file mode 100644 -index 000000000000..31cfafb0cf65 +index 0000000000000..31cfafb0cf65a --- /dev/null +++ b/tools/testing/selftests/sched_ext/maybe_null.c @@ -0,0 +1,49 @@ @@ -16134,7 +16263,7 @@ index 000000000000..31cfafb0cf65 +REGISTER_SCX_TEST(&maybe_null) diff --git a/tools/testing/selftests/sched_ext/maybe_null_fail_dsp.bpf.c b/tools/testing/selftests/sched_ext/maybe_null_fail_dsp.bpf.c new file mode 100644 -index 000000000000..c0641050271d +index 0000000000000..c0641050271d3 --- /dev/null +++ b/tools/testing/selftests/sched_ext/maybe_null_fail_dsp.bpf.c @@ -0,0 +1,25 @@ @@ -16165,7 +16294,7 @@ index 000000000000..c0641050271d +}; diff --git a/tools/testing/selftests/sched_ext/maybe_null_fail_yld.bpf.c b/tools/testing/selftests/sched_ext/maybe_null_fail_yld.bpf.c new file mode 100644 -index 000000000000..3c1740028e3b +index 0000000000000..3c1740028e3b9 --- /dev/null +++ b/tools/testing/selftests/sched_ext/maybe_null_fail_yld.bpf.c @@ -0,0 +1,28 @@ @@ -16199,7 +16328,7 @@ index 000000000000..3c1740028e3b +}; diff --git a/tools/testing/selftests/sched_ext/minimal.bpf.c b/tools/testing/selftests/sched_ext/minimal.bpf.c new file mode 100644 -index 000000000000..6a7eccef0104 +index 0000000000000..6a7eccef01045 --- /dev/null +++ b/tools/testing/selftests/sched_ext/minimal.bpf.c @@ -0,0 +1,21 @@ @@ -16226,7 +16355,7 @@ index 000000000000..6a7eccef0104 +}; diff --git a/tools/testing/selftests/sched_ext/minimal.c b/tools/testing/selftests/sched_ext/minimal.c new file mode 100644 -index 000000000000..6c5db8ebbf8a +index 0000000000000..6c5db8ebbf8ac --- /dev/null +++ b/tools/testing/selftests/sched_ext/minimal.c @@ -0,0 +1,58 @@ @@ -16290,7 +16419,7 @@ index 000000000000..6c5db8ebbf8a +REGISTER_SCX_TEST(&minimal) diff --git a/tools/testing/selftests/sched_ext/prog_run.bpf.c b/tools/testing/selftests/sched_ext/prog_run.bpf.c new file mode 100644 -index 000000000000..6a4d7c48e3f2 +index 0000000000000..6a4d7c48e3f22 --- /dev/null +++ b/tools/testing/selftests/sched_ext/prog_run.bpf.c @@ -0,0 +1,33 @@ @@ -16329,7 +16458,7 @@ index 000000000000..6a4d7c48e3f2 +}; diff --git a/tools/testing/selftests/sched_ext/prog_run.c b/tools/testing/selftests/sched_ext/prog_run.c new file mode 100644 -index 000000000000..3cd57ef8daaa +index 0000000000000..3cd57ef8daaa5 --- /dev/null +++ b/tools/testing/selftests/sched_ext/prog_run.c @@ -0,0 +1,78 @@ @@ -16413,7 +16542,7 @@ index 000000000000..3cd57ef8daaa +REGISTER_SCX_TEST(&prog_run) diff --git a/tools/testing/selftests/sched_ext/reload_loop.c b/tools/testing/selftests/sched_ext/reload_loop.c new file mode 100644 -index 000000000000..5cfba2d6e056 +index 0000000000000..5cfba2d6e0568 --- /dev/null +++ b/tools/testing/selftests/sched_ext/reload_loop.c @@ -0,0 +1,75 @@ @@ -16494,7 +16623,7 @@ index 000000000000..5cfba2d6e056 +REGISTER_SCX_TEST(&reload_loop) diff --git a/tools/testing/selftests/sched_ext/runner.c b/tools/testing/selftests/sched_ext/runner.c new file mode 100644 -index 000000000000..eab48c7ff309 +index 0000000000000..eab48c7ff3094 --- /dev/null +++ b/tools/testing/selftests/sched_ext/runner.c @@ -0,0 +1,201 @@ @@ -16701,7 +16830,7 @@ index 000000000000..eab48c7ff309 +} diff --git a/tools/testing/selftests/sched_ext/scx_test.h b/tools/testing/selftests/sched_ext/scx_test.h new file mode 100644 -index 000000000000..90b8d6915bb7 +index 0000000000000..90b8d6915bb7e --- /dev/null +++ b/tools/testing/selftests/sched_ext/scx_test.h @@ -0,0 +1,131 @@ @@ -16838,7 +16967,7 @@ index 000000000000..90b8d6915bb7 +#endif // # __SCX_TEST_H__ diff --git a/tools/testing/selftests/sched_ext/select_cpu_dfl.bpf.c b/tools/testing/selftests/sched_ext/select_cpu_dfl.bpf.c new file mode 100644 -index 000000000000..2ed2991afafe +index 0000000000000..2ed2991afafe3 --- /dev/null +++ b/tools/testing/selftests/sched_ext/select_cpu_dfl.bpf.c @@ -0,0 +1,40 @@ @@ -16884,7 +17013,7 @@ index 000000000000..2ed2991afafe +}; diff --git a/tools/testing/selftests/sched_ext/select_cpu_dfl.c b/tools/testing/selftests/sched_ext/select_cpu_dfl.c new file mode 100644 -index 000000000000..a53a40c2d2f0 +index 0000000000000..a53a40c2d2f0f --- /dev/null +++ b/tools/testing/selftests/sched_ext/select_cpu_dfl.c @@ -0,0 +1,72 @@ @@ -16962,7 +17091,7 @@ index 000000000000..a53a40c2d2f0 +REGISTER_SCX_TEST(&select_cpu_dfl) diff --git a/tools/testing/selftests/sched_ext/select_cpu_dfl_nodispatch.bpf.c b/tools/testing/selftests/sched_ext/select_cpu_dfl_nodispatch.bpf.c new file mode 100644 -index 000000000000..4bb5abb2d369 +index 0000000000000..4bb5abb2d3690 --- /dev/null +++ b/tools/testing/selftests/sched_ext/select_cpu_dfl_nodispatch.bpf.c @@ -0,0 +1,89 @@ @@ -17057,7 +17186,7 @@ index 000000000000..4bb5abb2d369 +}; diff --git a/tools/testing/selftests/sched_ext/select_cpu_dfl_nodispatch.c b/tools/testing/selftests/sched_ext/select_cpu_dfl_nodispatch.c new file mode 100644 -index 000000000000..1d85bf4bf3a3 +index 0000000000000..1d85bf4bf3a39 --- /dev/null +++ b/tools/testing/selftests/sched_ext/select_cpu_dfl_nodispatch.c @@ -0,0 +1,72 @@ @@ -17135,7 +17264,7 @@ index 000000000000..1d85bf4bf3a3 +REGISTER_SCX_TEST(&select_cpu_dfl_nodispatch) diff --git a/tools/testing/selftests/sched_ext/select_cpu_dispatch.bpf.c b/tools/testing/selftests/sched_ext/select_cpu_dispatch.bpf.c new file mode 100644 -index 000000000000..f0b96a4a04b2 +index 0000000000000..f0b96a4a04b2c --- /dev/null +++ b/tools/testing/selftests/sched_ext/select_cpu_dispatch.bpf.c @@ -0,0 +1,41 @@ @@ -17182,7 +17311,7 @@ index 000000000000..f0b96a4a04b2 +}; diff --git a/tools/testing/selftests/sched_ext/select_cpu_dispatch.c b/tools/testing/selftests/sched_ext/select_cpu_dispatch.c new file mode 100644 -index 000000000000..0309ca8785b3 +index 0000000000000..0309ca8785b36 --- /dev/null +++ b/tools/testing/selftests/sched_ext/select_cpu_dispatch.c @@ -0,0 +1,70 @@ @@ -17258,7 +17387,7 @@ index 000000000000..0309ca8785b3 +REGISTER_SCX_TEST(&select_cpu_dispatch) diff --git a/tools/testing/selftests/sched_ext/select_cpu_dispatch_bad_dsq.bpf.c b/tools/testing/selftests/sched_ext/select_cpu_dispatch_bad_dsq.bpf.c new file mode 100644 -index 000000000000..7b42ddce0f56 +index 0000000000000..7b42ddce0f56c --- /dev/null +++ b/tools/testing/selftests/sched_ext/select_cpu_dispatch_bad_dsq.bpf.c @@ -0,0 +1,37 @@ @@ -17301,7 +17430,7 @@ index 000000000000..7b42ddce0f56 +}; diff --git a/tools/testing/selftests/sched_ext/select_cpu_dispatch_bad_dsq.c b/tools/testing/selftests/sched_ext/select_cpu_dispatch_bad_dsq.c new file mode 100644 -index 000000000000..47eb6ed7627d +index 0000000000000..47eb6ed7627d9 --- /dev/null +++ b/tools/testing/selftests/sched_ext/select_cpu_dispatch_bad_dsq.c @@ -0,0 +1,56 @@ @@ -17363,7 +17492,7 @@ index 000000000000..47eb6ed7627d +REGISTER_SCX_TEST(&select_cpu_dispatch_bad_dsq) diff --git a/tools/testing/selftests/sched_ext/select_cpu_dispatch_dbl_dsp.bpf.c b/tools/testing/selftests/sched_ext/select_cpu_dispatch_dbl_dsp.bpf.c new file mode 100644 -index 000000000000..653e3dc0b4dc +index 0000000000000..653e3dc0b4dc8 --- /dev/null +++ b/tools/testing/selftests/sched_ext/select_cpu_dispatch_dbl_dsp.bpf.c @@ -0,0 +1,38 @@ @@ -17407,7 +17536,7 @@ index 000000000000..653e3dc0b4dc +}; diff --git a/tools/testing/selftests/sched_ext/select_cpu_dispatch_dbl_dsp.c b/tools/testing/selftests/sched_ext/select_cpu_dispatch_dbl_dsp.c new file mode 100644 -index 000000000000..48ff028a3c46 +index 0000000000000..48ff028a3c46d --- /dev/null +++ b/tools/testing/selftests/sched_ext/select_cpu_dispatch_dbl_dsp.c @@ -0,0 +1,56 @@ @@ -17469,7 +17598,7 @@ index 000000000000..48ff028a3c46 +REGISTER_SCX_TEST(&select_cpu_dispatch_dbl_dsp) diff --git a/tools/testing/selftests/sched_ext/select_cpu_vtime.bpf.c b/tools/testing/selftests/sched_ext/select_cpu_vtime.bpf.c new file mode 100644 -index 000000000000..7f3ebf4fc2ea +index 0000000000000..7f3ebf4fc2ead --- /dev/null +++ b/tools/testing/selftests/sched_ext/select_cpu_vtime.bpf.c @@ -0,0 +1,92 @@ @@ -17567,7 +17696,7 @@ index 000000000000..7f3ebf4fc2ea +}; diff --git a/tools/testing/selftests/sched_ext/select_cpu_vtime.c b/tools/testing/selftests/sched_ext/select_cpu_vtime.c new file mode 100644 -index 000000000000..b4629c2364f5 +index 0000000000000..b4629c2364f5d --- /dev/null +++ b/tools/testing/selftests/sched_ext/select_cpu_vtime.c @@ -0,0 +1,59 @@ @@ -17632,7 +17761,7 @@ index 000000000000..b4629c2364f5 +REGISTER_SCX_TEST(&select_cpu_vtime) diff --git a/tools/testing/selftests/sched_ext/test_example.c b/tools/testing/selftests/sched_ext/test_example.c new file mode 100644 -index 000000000000..ce36cdf03cdc +index 0000000000000..ce36cdf03cdc5 --- /dev/null +++ b/tools/testing/selftests/sched_ext/test_example.c @@ -0,0 +1,49 @@ @@ -17687,7 +17816,7 @@ index 000000000000..ce36cdf03cdc +REGISTER_SCX_TEST(&example) diff --git a/tools/testing/selftests/sched_ext/util.c b/tools/testing/selftests/sched_ext/util.c new file mode 100644 -index 000000000000..e47769c91918 +index 0000000000000..e47769c919187 --- /dev/null +++ b/tools/testing/selftests/sched_ext/util.c @@ -0,0 +1,71 @@ @@ -17764,7 +17893,7 @@ index 000000000000..e47769c91918 +} diff --git a/tools/testing/selftests/sched_ext/util.h b/tools/testing/selftests/sched_ext/util.h new file mode 100644 -index 000000000000..bc13dfec1267 +index 0000000000000..bc13dfec1267a --- /dev/null +++ b/tools/testing/selftests/sched_ext/util.h @@ -0,0 +1,13 @@ @@ -17782,4 +17911,4 @@ index 000000000000..bc13dfec1267 + +#endif // __SCX_TEST_H__ -- -2.45.2.606.g9005149a4a +2.47.0.rc0