From 02b40bc7cbcd8853260e543b85036e05ed24ebc7 Mon Sep 17 00:00:00 2001 From: ferreo Date: Wed, 2 Oct 2024 18:22:48 +0200 Subject: [PATCH] Update patches/0003-bore-cachy-ext.patch --- patches/0003-bore-cachy-ext.patch | 1003 +++++++++++++++-------------- 1 file changed, 524 insertions(+), 479 deletions(-) diff --git a/patches/0003-bore-cachy-ext.patch b/patches/0003-bore-cachy-ext.patch index c04adb7..e76e5a0 100644 --- a/patches/0003-bore-cachy-ext.patch +++ b/patches/0003-bore-cachy-ext.patch @@ -1,25 +1,39 @@ -From 7b726493e44bba968c8c0369c387b5e462d6840e Mon Sep 17 00:00:00 2001 +From 92130ba595a1a57cb0cf5c806be12bc3c0d315c3 Mon Sep 17 00:00:00 2001 From: Piotr Gorski -Date: Mon, 16 Sep 2024 13:24:13 +0200 +Date: Wed, 25 Sep 2024 08:07:10 +0200 Subject: [PATCH] bore-cachy-ext Signed-off-by: Piotr Gorski --- - include/linux/sched.h | 10 ++ - init/Kconfig | 17 ++ - kernel/Kconfig.hz | 16 ++ - kernel/sched/core.c | 141 +++++++++++++++ - kernel/sched/debug.c | 60 ++++++- - kernel/sched/fair.c | 383 +++++++++++++++++++++++++++++++++++++--- - kernel/sched/features.h | 16 +- - kernel/sched/sched.h | 7 + - 8 files changed, 623 insertions(+), 27 deletions(-) + include/linux/sched.h | 15 +- + include/linux/sched/bore.h | 38 ++++ + init/Kconfig | 17 ++ + kernel/Kconfig.hz | 16 ++ + kernel/fork.c | 5 + + kernel/sched/Makefile | 1 + + kernel/sched/bore.c | 413 +++++++++++++++++++++++++++++++++++++ + kernel/sched/core.c | 7 + + kernel/sched/debug.c | 60 +++++- + kernel/sched/fair.c | 99 +++++++-- + kernel/sched/features.h | 4 + + kernel/sched/sched.h | 7 + + 12 files changed, 664 insertions(+), 18 deletions(-) + create mode 100644 include/linux/sched/bore.h + create mode 100644 kernel/sched/bore.c diff --git a/include/linux/sched.h b/include/linux/sched.h -index 5b4f78fe3..b9e5ea2aa 100644 +index 5b4f78fe3..7912524df 100644 --- a/include/linux/sched.h +++ b/include/linux/sched.h -@@ -551,6 +551,16 @@ struct sched_entity { +@@ -545,12 +545,25 @@ struct sched_entity { + u64 min_vruntime; + + struct list_head group_node; +- unsigned int on_rq; ++ unsigned char on_rq; ++ unsigned char rel_deadline; + + u64 exec_start; u64 sum_exec_runtime; u64 prev_sum_exec_runtime; u64 vruntime; @@ -30,12 +44,58 @@ index 5b4f78fe3..b9e5ea2aa 100644 + u8 burst_penalty; + u8 burst_score; + u8 child_burst; ++ u8 tg_burst; + u32 child_burst_cnt; + u64 child_burst_last_cached; ++ u64 tg_burst_last_cached; +#endif // CONFIG_SCHED_BORE s64 vlag; u64 slice; +diff --git a/include/linux/sched/bore.h b/include/linux/sched/bore.h +new file mode 100644 +index 000000000..26abf8cfc +--- /dev/null ++++ b/include/linux/sched/bore.h +@@ -0,0 +1,38 @@ ++ ++#include ++#include ++ ++#ifndef _LINUX_SCHED_BORE_H ++#define _LINUX_SCHED_BORE_H ++ ++#ifdef CONFIG_SCHED_BORE ++extern u8 __read_mostly sched_bore; ++extern u8 __read_mostly sched_burst_exclude_kthreads; ++extern u8 __read_mostly sched_burst_smoothness_long; ++extern u8 __read_mostly sched_burst_smoothness_short; ++extern u8 __read_mostly sched_burst_atavistic_mask; ++extern u8 __read_mostly sched_burst_atavistic_depth; ++extern u8 __read_mostly sched_burst_parity_threshold; ++extern u8 __read_mostly sched_burst_penalty_offset; ++extern uint __read_mostly sched_burst_penalty_scale; ++extern uint __read_mostly sched_burst_cache_lifetime; ++extern uint __read_mostly sched_deadline_boost_mask; ++ ++extern void update_burst_score(struct sched_entity *se); ++extern void update_burst_penalty(struct sched_entity *se); ++ ++extern void restart_burst(struct sched_entity *se); ++extern void restart_burst_rescale_deadline(struct sched_entity *se); ++ ++extern int sched_bore_update_handler(const struct ctl_table *table, int write, ++ void __user *buffer, size_t *lenp, loff_t *ppos); ++ ++extern void sched_clone_bore( ++ struct task_struct *p, struct task_struct *parent, u64 clone_flags); ++ ++extern void init_task_bore(struct task_struct *p); ++ ++extern void reweight_entity( ++ struct cfs_rq *cfs_rq, struct sched_entity *se, unsigned long weight); ++#endif // CONFIG_SCHED_BORE ++#endif // _LINUX_SCHED_BORE_H diff --git a/init/Kconfig b/init/Kconfig index e1a88d48d..3aea8e43c 100644 --- a/init/Kconfig @@ -90,32 +150,191 @@ index 0f78364ef..b50189ee5 100644 + config SCHED_HRTICK def_bool HIGH_RES_TIMERS -diff --git a/kernel/sched/core.c b/kernel/sched/core.c -index c792a6feb..79a913e2f 100644 ---- a/kernel/sched/core.c -+++ b/kernel/sched/core.c -@@ -4336,6 +4336,136 @@ int wake_up_state(struct task_struct *p, unsigned int state) - return try_to_wake_up(p, state, 0); - } +diff --git a/kernel/fork.c b/kernel/fork.c +index 69a0a7210..68737ef70 100644 +--- a/kernel/fork.c ++++ b/kernel/fork.c +@@ -116,6 +116,8 @@ + #include + #include -+#ifdef CONFIG_SCHED_BORE -+extern u8 sched_burst_fork_atavistic; -+extern uint sched_burst_cache_lifetime; ++#include + -+static void __init sched_init_bore(void) { -+ init_task.se.burst_time = 0; -+ init_task.se.prev_burst_penalty = 0; -+ init_task.se.curr_burst_penalty = 0; -+ init_task.se.burst_penalty = 0; -+ init_task.se.burst_score = 0; -+ init_task.se.child_burst_last_cached = 0; + #include + + #define CREATE_TRACE_POINTS +@@ -2354,6 +2356,9 @@ __latent_entropy struct task_struct *copy_process( + retval = sched_fork(clone_flags, p); + if (retval) + goto bad_fork_cleanup_policy; ++#ifdef CONFIG_SCHED_BORE ++ sched_clone_bore(p, current, clone_flags); ++#endif // CONFIG_SCHED_BORE + + retval = perf_event_init_task(p, clone_flags); + if (retval) +diff --git a/kernel/sched/Makefile b/kernel/sched/Makefile +index 976092b7b..293aad675 100644 +--- a/kernel/sched/Makefile ++++ b/kernel/sched/Makefile +@@ -32,3 +32,4 @@ obj-y += core.o + obj-y += fair.o + obj-y += build_policy.o + obj-y += build_utility.o ++obj-y += bore.o +diff --git a/kernel/sched/bore.c b/kernel/sched/bore.c +new file mode 100644 +index 000000000..f2b1f1179 +--- /dev/null ++++ b/kernel/sched/bore.c +@@ -0,0 +1,413 @@ ++/* ++ * Burst-Oriented Response Enhancer (BORE) CPU Scheduler ++ * Copyright (C) 2021-2024 Masahito Suzuki ++ */ ++#include ++#include ++#include "sched.h" ++ ++#ifdef CONFIG_SCHED_BORE ++u8 __read_mostly sched_bore = 1; ++u8 __read_mostly sched_burst_exclude_kthreads = 1; ++u8 __read_mostly sched_burst_smoothness_long = 1; ++u8 __read_mostly sched_burst_smoothness_short = 0; ++u8 __read_mostly sched_burst_atavistic_mask = 0x2; ++u8 __read_mostly sched_burst_atavistic_depth = 2; ++u8 __read_mostly sched_burst_parity_threshold = 2; ++u8 __read_mostly sched_burst_penalty_offset = 22; ++uint __read_mostly sched_burst_penalty_scale = 1280; ++uint __read_mostly sched_burst_cache_lifetime = 60000000; ++uint __read_mostly sched_deadline_boost_mask = ENQUEUE_INITIAL ++ | ENQUEUE_WAKEUP; ++static int __maybe_unused sixty_four = 64; ++static int __maybe_unused maxval_u8 = 255; ++static int __maybe_unused maxval_12_bits = 4095; ++ ++#define MAX_BURST_PENALTY (39U <<2) ++ ++static inline u32 log2plus1_u64_u32f8(u64 v) { ++ u32 msb = fls64(v); ++ u8 fractional = (v << (64 - msb) >> 55); ++ return msb << 8 | fractional; +} + -+inline void sched_fork_bore(struct task_struct *p) { -+ p->se.burst_time = 0; -+ p->se.curr_burst_penalty = 0; -+ p->se.burst_score = 0; -+ p->se.child_burst_last_cached = 0; ++static inline u32 calc_burst_penalty(u64 burst_time) { ++ u32 greed, tolerance, penalty, scaled_penalty; ++ ++ greed = log2plus1_u64_u32f8(burst_time); ++ tolerance = sched_burst_penalty_offset << 8; ++ penalty = max(0, (s32)(greed - tolerance)); ++ scaled_penalty = penalty * sched_burst_penalty_scale >> 16; ++ ++ return min(MAX_BURST_PENALTY, scaled_penalty); ++} ++ ++static inline u64 __scale_slice(u64 delta, u8 score) { ++ return mul_u64_u32_shr(delta, sched_prio_to_wmult[score], 22); ++} ++ ++static inline u64 __unscale_slice(u64 delta, u8 score) { ++ return mul_u64_u32_shr(delta, sched_prio_to_weight[score], 10); ++} ++ ++static void reweight_task_by_prio(struct task_struct *p, int prio) { ++ struct sched_entity *se = &p->se; ++ unsigned long weight = scale_load(sched_prio_to_weight[prio]); ++ ++ reweight_entity(cfs_rq_of(se), se, weight); ++ se->load.inv_weight = sched_prio_to_wmult[prio]; ++} ++ ++static inline u8 effective_prio(struct task_struct *p) { ++ u8 prio = p->static_prio - MAX_RT_PRIO; ++ if (likely(sched_bore)) ++ prio += p->se.burst_score; ++ return min(39, prio); ++} ++ ++void update_burst_score(struct sched_entity *se) { ++ if (!entity_is_task(se)) return; ++ struct task_struct *p = task_of(se); ++ u8 prev_prio = effective_prio(p); ++ ++ u8 burst_score = 0; ++ if (!((p->flags & PF_KTHREAD) && likely(sched_burst_exclude_kthreads))) ++ burst_score = se->burst_penalty >> 2; ++ se->burst_score = burst_score; ++ ++ u8 new_prio = effective_prio(p); ++ if (new_prio != prev_prio) ++ reweight_task_by_prio(p, new_prio); ++} ++ ++void update_burst_penalty(struct sched_entity *se) { ++ se->curr_burst_penalty = calc_burst_penalty(se->burst_time); ++ se->burst_penalty = max(se->prev_burst_penalty, se->curr_burst_penalty); ++ update_burst_score(se); ++} ++ ++static inline u32 binary_smooth(u32 new, u32 old) { ++ int increment = new - old; ++ return (0 <= increment)? ++ old + ( increment >> (int)sched_burst_smoothness_long): ++ old - (-increment >> (int)sched_burst_smoothness_short); ++} ++ ++static void __revolve_burst_penalty(struct sched_entity *se) { ++ se->prev_burst_penalty = ++ binary_smooth(se->curr_burst_penalty, se->prev_burst_penalty); ++ se->burst_time = 0; ++ se->curr_burst_penalty = 0; ++} ++ ++inline void restart_burst(struct sched_entity *se) { ++ __revolve_burst_penalty(se); ++ se->burst_penalty = se->prev_burst_penalty; ++ update_burst_score(se); ++} ++ ++void restart_burst_rescale_deadline(struct sched_entity *se) { ++ s64 vscaled, wremain, vremain = se->deadline - se->vruntime; ++ struct task_struct *p = task_of(se); ++ u8 prev_prio = effective_prio(p); ++ restart_burst(se); ++ u8 new_prio = effective_prio(p); ++ if (prev_prio > new_prio) { ++ wremain = __unscale_slice(abs(vremain), prev_prio); ++ vscaled = __scale_slice(wremain, new_prio); ++ if (unlikely(vremain < 0)) ++ vscaled = -vscaled; ++ se->deadline = se->vruntime + vscaled; ++ } ++} ++ ++static void reset_task_weights_bore(void) { ++ struct task_struct *task; ++ struct rq *rq; ++ struct rq_flags rf; ++ ++ write_lock_irq(&tasklist_lock); ++ for_each_process(task) { ++ rq = task_rq(task); ++ rq_lock_irqsave(rq, &rf); ++ reweight_task_by_prio(task, effective_prio(task)); ++ rq_unlock_irqrestore(rq, &rf); ++ } ++ write_unlock_irq(&tasklist_lock); ++} ++ ++int sched_bore_update_handler(const struct ctl_table *table, int write, ++ void __user *buffer, size_t *lenp, loff_t *ppos) { ++ int ret = proc_dou8vec_minmax(table, write, buffer, lenp, ppos); ++ if (ret || !write) ++ return ret; ++ ++ reset_task_weights_bore(); ++ ++ return 0; +} + +static u32 count_child_tasks(struct task_struct *p) { @@ -137,16 +356,15 @@ index c792a6feb..79a913e2f 100644 + +static void __update_child_burst_cache( + struct task_struct *p, u32 cnt, u32 sum, u64 now) { -+ u8 avg = 0; -+ if (cnt) avg = sum / cnt; ++ u8 avg = cnt ? sum / cnt : 0; + p->se.child_burst = max(avg, p->se.burst_penalty); + p->se.child_burst_cnt = cnt; + p->se.child_burst_last_cached = now; +} + +static inline void update_child_burst_direct(struct task_struct *p, u64 now) { -+ struct task_struct *child; + u32 cnt = 0, sum = 0; ++ struct task_struct *child; + + list_for_each_entry(child, &p->children, sibling) { + if (!task_burst_inheritable(child)) continue; @@ -158,7 +376,7 @@ index c792a6feb..79a913e2f 100644 +} + +static inline u8 __inherit_burst_direct(struct task_struct *p, u64 now) { -+ struct task_struct *parent = p->real_parent; ++ struct task_struct *parent = p; + if (child_burst_cache_expired(parent, now)) + update_child_burst_direct(parent, now); + @@ -167,8 +385,8 @@ index c792a6feb..79a913e2f 100644 + +static void update_child_burst_topological( + struct task_struct *p, u64 now, u32 depth, u32 *acnt, u32 *asum) { -+ struct task_struct *child, *dec; + u32 cnt = 0, dcnt = 0, sum = 0; ++ struct task_struct *child, *dec; + + list_for_each_entry(child, &p->children, sibling) { + dec = child; @@ -195,7 +413,7 @@ index c792a6feb..79a913e2f 100644 +} + +static inline u8 __inherit_burst_topological(struct task_struct *p, u64 now) { -+ struct task_struct *anc = p->real_parent; ++ struct task_struct *anc = p; + u32 cnt = 0, sum = 0; + + while (anc->real_parent != anc && count_child_tasks(anc) == 1) @@ -203,61 +421,206 @@ index c792a6feb..79a913e2f 100644 + + if (child_burst_cache_expired(anc, now)) + update_child_burst_topological( -+ anc, now, sched_burst_fork_atavistic - 1, &cnt, &sum); ++ anc, now, sched_burst_atavistic_depth - 1, &cnt, &sum); + + return anc->se.child_burst; +} + -+static inline void inherit_burst(struct task_struct *p) { -+ u8 burst_cache; -+ u64 now = ktime_get_ns(); ++static inline bool tg_burst_cache_expired(struct task_struct *p, u64 now) { ++ u64 expiration_time = ++ p->se.tg_burst_last_cached + sched_burst_cache_lifetime; ++ return ((s64)(expiration_time - now) < 0); ++} + ++static void __update_tg_burst_cache( ++ struct task_struct *p, u32 cnt, u32 sum, u64 now) { ++ u8 avg = cnt ? sum / cnt : 0; ++ p->se.tg_burst = max(avg, p->se.burst_penalty); ++ p->se.tg_burst_last_cached = now; ++} ++ ++static inline void update_tg_burst(struct task_struct *p, u64 now) { ++ struct task_struct *task; ++ u32 cnt = 0, sum = 0; ++ ++ for_each_thread(p, task) { ++ if (!task_burst_inheritable(task)) continue; ++ cnt++; ++ sum += task->se.burst_penalty; ++ } ++ ++ __update_tg_burst_cache(p, cnt, sum, now); ++} ++ ++static inline u8 __inherit_burst_tg(struct task_struct *p, u64 now) { ++ struct task_struct *parent = p->group_leader; ++ if (tg_burst_cache_expired(parent, now)) ++ update_tg_burst(parent, now); ++ ++ return parent->se.tg_burst; ++} ++ ++void sched_clone_bore( ++ struct task_struct *p, struct task_struct *parent, u64 clone_flags) { ++ if (!task_burst_inheritable(p)) return; ++ ++ u8 penalty; ++ u8 clone_type = !(clone_flags & CLONE_THREAD) + 1; ++ u8 type_matched = sched_burst_atavistic_mask & clone_type; ++ ++ u64 now = ktime_get_ns(); + read_lock(&tasklist_lock); -+ burst_cache = likely(sched_burst_fork_atavistic)? -+ __inherit_burst_topological(p, now): -+ __inherit_burst_direct(p, now); ++ penalty = (type_matched && likely(sched_burst_atavistic_depth)) ? ++ __inherit_burst_topological(parent, now): ++ ((clone_type == 2) ? ++ __inherit_burst_direct(parent, now): ++ __inherit_burst_tg(parent, now)); + read_unlock(&tasklist_lock); + -+ p->se.prev_burst_penalty = max(p->se.prev_burst_penalty, burst_cache); ++ struct sched_entity *se = &p->se; ++ __revolve_burst_penalty(se); ++ se->burst_penalty = se->prev_burst_penalty = ++ max(se->prev_burst_penalty, penalty); ++ se->child_burst_last_cached = 0; ++ se->tg_burst_last_cached = 0; +} + -+static void sched_post_fork_bore(struct task_struct *p) { -+ if (task_burst_inheritable(p)) -+ inherit_burst(p); -+ p->se.burst_penalty = p->se.prev_burst_penalty; ++void init_task_bore(struct task_struct *p) { ++ p->se.burst_time = 0; ++ p->se.prev_burst_penalty = 0; ++ p->se.curr_burst_penalty = 0; ++ p->se.burst_penalty = 0; ++ p->se.burst_score = 0; ++ p->se.child_burst_last_cached = 0; ++ p->se.tg_burst_last_cached = 0; +} -+#endif // CONFIG_SCHED_BORE + - /* - * Perform scheduler related setup for a newly forked process p. - * p is forked by current. -@@ -4352,6 +4482,9 @@ static void __sched_fork(unsigned long clone_flags, struct task_struct *p) - p->se.prev_sum_exec_runtime = 0; - p->se.nr_migrations = 0; - p->se.vruntime = 0; -+#ifdef CONFIG_SCHED_BORE -+ sched_fork_bore(p); ++#ifdef CONFIG_SYSCTL ++static struct ctl_table sched_bore_sysctls[] = { ++ { ++ .procname = "sched_bore", ++ .data = &sched_bore, ++ .maxlen = sizeof(u8), ++ .mode = 0644, ++ .proc_handler = sched_bore_update_handler, ++ .extra1 = SYSCTL_ZERO, ++ .extra2 = SYSCTL_ONE, ++ }, ++ { ++ .procname = "sched_burst_exclude_kthreads", ++ .data = &sched_burst_exclude_kthreads, ++ .maxlen = sizeof(u8), ++ .mode = 0644, ++ .proc_handler = proc_dou8vec_minmax, ++ .extra1 = SYSCTL_ZERO, ++ .extra2 = SYSCTL_ONE, ++ }, ++ { ++ .procname = "sched_burst_smoothness_long", ++ .data = &sched_burst_smoothness_long, ++ .maxlen = sizeof(u8), ++ .mode = 0644, ++ .proc_handler = proc_dou8vec_minmax, ++ .extra1 = SYSCTL_ZERO, ++ .extra2 = SYSCTL_ONE, ++ }, ++ { ++ .procname = "sched_burst_smoothness_short", ++ .data = &sched_burst_smoothness_short, ++ .maxlen = sizeof(u8), ++ .mode = 0644, ++ .proc_handler = proc_dou8vec_minmax, ++ .extra1 = SYSCTL_ZERO, ++ .extra2 = SYSCTL_ONE, ++ }, ++ { ++ .procname = "sched_burst_atavistic_mask", ++ .data = &sched_burst_atavistic_mask, ++ .maxlen = sizeof(u8), ++ .mode = 0644, ++ .proc_handler = proc_dou8vec_minmax, ++ .extra1 = SYSCTL_ZERO, ++ .extra2 = SYSCTL_THREE, ++ }, ++ { ++ .procname = "sched_burst_atavistic_depth", ++ .data = &sched_burst_atavistic_depth, ++ .maxlen = sizeof(u8), ++ .mode = 0644, ++ .proc_handler = proc_dou8vec_minmax, ++ .extra1 = SYSCTL_ZERO, ++ .extra2 = SYSCTL_THREE, ++ }, ++ { ++ .procname = "sched_burst_parity_threshold", ++ .data = &sched_burst_parity_threshold, ++ .maxlen = sizeof(u8), ++ .mode = 0644, ++ .proc_handler = proc_dou8vec_minmax, ++ .extra1 = SYSCTL_ZERO, ++ .extra2 = &maxval_u8, ++ }, ++ { ++ .procname = "sched_burst_penalty_offset", ++ .data = &sched_burst_penalty_offset, ++ .maxlen = sizeof(u8), ++ .mode = 0644, ++ .proc_handler = proc_dou8vec_minmax, ++ .extra1 = SYSCTL_ZERO, ++ .extra2 = &sixty_four, ++ }, ++ { ++ .procname = "sched_burst_penalty_scale", ++ .data = &sched_burst_penalty_scale, ++ .maxlen = sizeof(uint), ++ .mode = 0644, ++ .proc_handler = proc_douintvec_minmax, ++ .extra1 = SYSCTL_ZERO, ++ .extra2 = &maxval_12_bits, ++ }, ++ { ++ .procname = "sched_burst_cache_lifetime", ++ .data = &sched_burst_cache_lifetime, ++ .maxlen = sizeof(uint), ++ .mode = 0644, ++ .proc_handler = proc_douintvec, ++ }, ++ { ++ .procname = "sched_deadline_boost_mask", ++ .data = &sched_deadline_boost_mask, ++ .maxlen = sizeof(uint), ++ .mode = 0644, ++ .proc_handler = proc_douintvec, ++ }, ++}; ++ ++static int __init sched_bore_sysctl_init(void) { ++ register_sysctl_init("kernel", sched_bore_sysctls); ++ return 0; ++} ++late_initcall(sched_bore_sysctl_init); ++#endif // CONFIG_SYSCTL +#endif // CONFIG_SCHED_BORE - p->se.vlag = 0; - p->se.slice = sysctl_sched_base_slice; - INIT_LIST_HEAD(&p->se.group_node); -@@ -4686,6 +4819,9 @@ void sched_cancel_fork(struct task_struct *p) +diff --git a/kernel/sched/core.c b/kernel/sched/core.c +index c792a6feb..2db0ce2be 100644 +--- a/kernel/sched/core.c ++++ b/kernel/sched/core.c +@@ -97,6 +97,8 @@ + #include "../../io_uring/io-wq.h" + #include "../smpboot.h" - void sched_post_fork(struct task_struct *p) - { -+#ifdef CONFIG_SCHED_BORE -+ sched_post_fork_bore(p); -+#endif // CONFIG_SCHED_BORE - uclamp_post_fork(p); - scx_post_fork(p); - } -@@ -8283,6 +8419,11 @@ void __init sched_init(void) ++#include ++ + EXPORT_TRACEPOINT_SYMBOL_GPL(ipi_send_cpu); + EXPORT_TRACEPOINT_SYMBOL_GPL(ipi_send_cpumask); + +@@ -8283,6 +8285,11 @@ void __init sched_init(void) BUG_ON(!sched_class_above(&ext_sched_class, &idle_sched_class)); #endif +#ifdef CONFIG_SCHED_BORE -+ sched_init_bore(); -+ printk(KERN_INFO "BORE (Burst-Oriented Response Enhancer) CPU Scheduler modification 5.2.12 by Masahito Suzuki"); ++ printk(KERN_INFO "BORE (Burst-Oriented Response Enhancer) CPU Scheduler modification 5.5.0 by Masahito Suzuki"); ++ init_task_bore(&init_task); +#endif // CONFIG_SCHED_BORE + wait_bit_init(); @@ -371,20 +734,19 @@ index c057ef46c..3cab39e34 100644 P(se.avg.runnable_sum); P(se.avg.util_sum); diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c -index 2928026d7..91a870173 100644 +index 2928026d7..b4260f1bb 100644 --- a/kernel/sched/fair.c +++ b/kernel/sched/fair.c -@@ -19,6 +19,9 @@ +@@ -55,6 +55,8 @@ + #include "stats.h" + #include "autogroup.h" + ++#include ++ + /* + * The initial- and re-scaling of tunables is configurable * - * Adaptive scheduling granularity, math enhancements by Peter Zijlstra - * Copyright (C) 2007 Red Hat, Inc., Peter Zijlstra -+ * -+ * Burst-Oriented Response Enhancer (BORE) CPU Scheduler -+ * Copyright (C) 2021-2024 Masahito Suzuki - */ - #include - #include -@@ -64,28 +67,176 @@ +@@ -64,28 +66,31 @@ * SCHED_TUNABLESCALING_LOG - scaled logarithmically, *1+ilog(ncpus) * SCHED_TUNABLESCALING_LINEAR - scaled linear, *ncpus * @@ -424,155 +786,10 @@ index 2928026d7..91a870173 100644 -#else const_debug unsigned int sysctl_sched_migration_cost = 500000UL; -#endif -+ -+#ifdef CONFIG_SCHED_BORE -+u8 __read_mostly sched_bore = 1; -+u8 __read_mostly sched_burst_exclude_kthreads = 1; -+u8 __read_mostly sched_burst_smoothness_long = 1; -+u8 __read_mostly sched_burst_smoothness_short = 0; -+u8 __read_mostly sched_burst_fork_atavistic = 2; -+u8 __read_mostly sched_burst_parity_threshold = 2; -+u8 __read_mostly sched_burst_penalty_offset = 22; -+uint __read_mostly sched_burst_penalty_scale = 1280; -+uint __read_mostly sched_burst_cache_lifetime = 60000000; -+uint __read_mostly sched_deadline_boost_mask = ENQUEUE_INITIAL -+ | ENQUEUE_WAKEUP; -+uint __read_mostly sched_deadline_preserve_mask = ENQUEUE_RESTORE -+ | ENQUEUE_MIGRATED; -+static int __maybe_unused sixty_four = 64; -+static int __maybe_unused maxval_u8 = 255; -+static int __maybe_unused maxval_12_bits = 4095; -+ -+#define MAX_BURST_PENALTY (39U <<2) -+ -+static inline u32 log2plus1_u64_u32f8(u64 v) { -+ u32 msb = fls64(v); -+ u8 fractional = (v << (64 - msb) >> 55); -+ return msb << 8 | fractional; -+} -+ -+static inline u32 calc_burst_penalty(u64 burst_time) { -+ u32 greed, tolerance, penalty, scaled_penalty; -+ -+ greed = log2plus1_u64_u32f8(burst_time); -+ tolerance = sched_burst_penalty_offset << 8; -+ penalty = max(0, (s32)greed - (s32)tolerance); -+ scaled_penalty = penalty * sched_burst_penalty_scale >> 16; -+ -+ return min(MAX_BURST_PENALTY, scaled_penalty); -+} -+ -+static inline u64 __scale_slice(u64 delta, u8 score) { -+ return mul_u64_u32_shr(delta, sched_prio_to_wmult[score], 22); -+} -+ -+static inline u64 __unscale_slice(u64 delta, u8 score) { -+ return mul_u64_u32_shr(delta, sched_prio_to_weight[score], 10); -+} -+ -+static void reweight_entity( -+ struct cfs_rq *cfs_rq, struct sched_entity *se, unsigned long weight); -+ -+static void reweight_task_by_prio(struct task_struct *p, int prio) -+{ -+ struct sched_entity *se = &p->se; -+ struct cfs_rq *cfs_rq = cfs_rq_of(se); -+ struct load_weight *load = &se->load; -+ unsigned long weight = scale_load(sched_prio_to_weight[prio]); -+ -+ reweight_entity(cfs_rq, se, weight); -+ load->inv_weight = sched_prio_to_wmult[prio]; -+} -+ -+static inline u8 effective_prio(struct task_struct *p) { -+ u8 prio = p->static_prio - MAX_RT_PRIO; -+ if (likely(sched_bore)) -+ prio += p->se.burst_score; -+ return min(39, prio); -+} -+ -+static void update_burst_score(struct sched_entity *se) { -+ if (!entity_is_task(se)) return; -+ struct task_struct *p = task_of(se); -+ u8 prev_prio = effective_prio(p); -+ -+ u8 burst_score = 0; -+ if (!((p->flags & PF_KTHREAD) && likely(sched_burst_exclude_kthreads))) -+ burst_score = se->burst_penalty >> 2; -+ se->burst_score = burst_score; -+ -+ u8 new_prio = effective_prio(p); -+ if (new_prio != prev_prio) -+ reweight_task_by_prio(p, new_prio); -+} -+ -+static void update_burst_penalty(struct sched_entity *se) { -+ se->curr_burst_penalty = calc_burst_penalty(se->burst_time); -+ se->burst_penalty = max(se->prev_burst_penalty, se->curr_burst_penalty); -+ update_burst_score(se); -+} -+ -+static inline u32 binary_smooth(u32 new, u32 old) { -+ int increment = new - old; -+ return (0 <= increment)? -+ old + ( increment >> (int)sched_burst_smoothness_long): -+ old - (-increment >> (int)sched_burst_smoothness_short); -+} -+ -+static void restart_burst(struct sched_entity *se) { -+ se->burst_penalty = se->prev_burst_penalty = -+ binary_smooth(se->curr_burst_penalty, se->prev_burst_penalty); -+ se->curr_burst_penalty = 0; -+ se->burst_time = 0; -+ update_burst_score(se); -+} -+ -+static void restart_burst_rescale_deadline(struct sched_entity *se) { -+ s64 vscaled, wremain, vremain = se->deadline - se->vruntime; -+ struct task_struct *p = task_of(se); -+ u8 prev_prio = effective_prio(p); -+ restart_burst(se); -+ u8 new_prio = effective_prio(p); -+ if (prev_prio > new_prio) { -+ wremain = __unscale_slice(abs(vremain), prev_prio); -+ vscaled = __scale_slice(wremain, new_prio); -+ if (unlikely(vremain < 0)) -+ vscaled = -vscaled; -+ se->deadline = se->vruntime + vscaled; -+ } -+} -+ -+static void reset_task_weights_bore(void) { -+ struct task_struct *task; -+ struct rq *rq; -+ struct rq_flags rf; -+ -+ write_lock_irq(&tasklist_lock); -+ for_each_process(task) { -+ rq = task_rq(task); -+ rq_lock_irqsave(rq, &rf); -+ reweight_task_by_prio(task, effective_prio(task)); -+ rq_unlock_irqrestore(rq, &rf); -+ } -+ write_unlock_irq(&tasklist_lock); -+} -+ -+int sched_bore_update_handler(const struct ctl_table *table, int write, -+ void __user *buffer, size_t *lenp, loff_t *ppos) -+{ -+ int ret = proc_dou8vec_minmax(table, write, buffer, lenp, ppos); -+ if (ret || !write) -+ return ret; -+ -+ reset_task_weights_bore(); -+ -+ return 0; -+} -+#endif // CONFIG_SCHED_BORE static int __init setup_sched_thermal_decay_shift(char *str) { -@@ -130,12 +281,8 @@ int __weak arch_asym_cpu_priority(int cpu) +@@ -130,12 +135,8 @@ int __weak arch_asym_cpu_priority(int cpu) * * (default: 5 msec, units: microseconds) */ @@ -585,109 +802,7 @@ index 2928026d7..91a870173 100644 #ifdef CONFIG_NUMA_BALANCING /* Restrict the NUMA promotion throughput (MB/s) for each target node. */ -@@ -144,6 +291,101 @@ static unsigned int sysctl_numa_balancing_promote_rate_limit = 65536; - - #ifdef CONFIG_SYSCTL - static struct ctl_table sched_fair_sysctls[] = { -+#ifdef CONFIG_SCHED_BORE -+ { -+ .procname = "sched_bore", -+ .data = &sched_bore, -+ .maxlen = sizeof(u8), -+ .mode = 0644, -+ .proc_handler = sched_bore_update_handler, -+ .extra1 = SYSCTL_ZERO, -+ .extra2 = SYSCTL_ONE, -+ }, -+ { -+ .procname = "sched_burst_exclude_kthreads", -+ .data = &sched_burst_exclude_kthreads, -+ .maxlen = sizeof(u8), -+ .mode = 0644, -+ .proc_handler = proc_dou8vec_minmax, -+ .extra1 = SYSCTL_ZERO, -+ .extra2 = SYSCTL_ONE, -+ }, -+ { -+ .procname = "sched_burst_smoothness_long", -+ .data = &sched_burst_smoothness_long, -+ .maxlen = sizeof(u8), -+ .mode = 0644, -+ .proc_handler = proc_dou8vec_minmax, -+ .extra1 = SYSCTL_ZERO, -+ .extra2 = SYSCTL_ONE, -+ }, -+ { -+ .procname = "sched_burst_smoothness_short", -+ .data = &sched_burst_smoothness_short, -+ .maxlen = sizeof(u8), -+ .mode = 0644, -+ .proc_handler = proc_dou8vec_minmax, -+ .extra1 = SYSCTL_ZERO, -+ .extra2 = SYSCTL_ONE, -+ }, -+ { -+ .procname = "sched_burst_fork_atavistic", -+ .data = &sched_burst_fork_atavistic, -+ .maxlen = sizeof(u8), -+ .mode = 0644, -+ .proc_handler = proc_dou8vec_minmax, -+ .extra1 = SYSCTL_ZERO, -+ .extra2 = SYSCTL_THREE, -+ }, -+ { -+ .procname = "sched_burst_parity_threshold", -+ .data = &sched_burst_parity_threshold, -+ .maxlen = sizeof(u8), -+ .mode = 0644, -+ .proc_handler = proc_dou8vec_minmax, -+ .extra1 = SYSCTL_ZERO, -+ .extra2 = &maxval_u8, -+ }, -+ { -+ .procname = "sched_burst_penalty_offset", -+ .data = &sched_burst_penalty_offset, -+ .maxlen = sizeof(u8), -+ .mode = 0644, -+ .proc_handler = proc_dou8vec_minmax, -+ .extra1 = SYSCTL_ZERO, -+ .extra2 = &sixty_four, -+ }, -+ { -+ .procname = "sched_burst_penalty_scale", -+ .data = &sched_burst_penalty_scale, -+ .maxlen = sizeof(uint), -+ .mode = 0644, -+ .proc_handler = proc_douintvec_minmax, -+ .extra1 = SYSCTL_ZERO, -+ .extra2 = &maxval_12_bits, -+ }, -+ { -+ .procname = "sched_burst_cache_lifetime", -+ .data = &sched_burst_cache_lifetime, -+ .maxlen = sizeof(uint), -+ .mode = 0644, -+ .proc_handler = proc_douintvec, -+ }, -+ { -+ .procname = "sched_deadline_boost_mask", -+ .data = &sched_deadline_boost_mask, -+ .maxlen = sizeof(uint), -+ .mode = 0644, -+ .proc_handler = proc_douintvec, -+ }, -+ { -+ .procname = "sched_deadline_preserve_mask", -+ .data = &sched_deadline_preserve_mask, -+ .maxlen = sizeof(uint), -+ .mode = 0644, -+ .proc_handler = proc_douintvec, -+ }, -+#endif // CONFIG_SCHED_BORE - #ifdef CONFIG_CFS_BANDWIDTH - { - .procname = "sched_cfs_bandwidth_slice_us", -@@ -201,6 +443,13 @@ static inline void update_load_set(struct load_weight *lw, unsigned long w) +@@ -201,6 +202,13 @@ static inline void update_load_set(struct load_weight *lw, unsigned long w) * * This idea comes from the SD scheduler of Con Kolivas: */ @@ -701,7 +816,7 @@ index 2928026d7..91a870173 100644 static unsigned int get_update_sysctl_factor(void) { unsigned int cpus = min_t(unsigned int, num_online_cpus(), 8); -@@ -231,6 +480,7 @@ static void update_sysctl(void) +@@ -231,6 +239,7 @@ static void update_sysctl(void) SET_SYSCTL(sched_base_slice); #undef SET_SYSCTL } @@ -709,7 +824,7 @@ index 2928026d7..91a870173 100644 void __init sched_init_granularity(void) { -@@ -708,6 +958,9 @@ static s64 entity_lag(u64 avruntime, struct sched_entity *se) +@@ -708,6 +717,9 @@ static s64 entity_lag(u64 avruntime, struct sched_entity *se) vlag = avruntime - se->vruntime; limit = calc_delta_fair(max_t(u64, 2*se->slice, TICK_NSEC), se); @@ -719,89 +834,18 @@ index 2928026d7..91a870173 100644 return clamp(vlag, -limit, limit); } -@@ -868,6 +1121,45 @@ struct sched_entity *__pick_first_entity(struct cfs_rq *cfs_rq) - return __node_2_se(left); - } - -+static inline bool pick_curr(struct cfs_rq *cfs_rq, -+ struct sched_entity *curr, struct sched_entity *wakee) -+{ -+ /* -+ * Nothing to preserve... -+ */ -+ if (!curr || !sched_feat(RESPECT_SLICE)) -+ return false; -+ +@@ -909,6 +921,10 @@ static struct sched_entity *pick_eevdf(struct cfs_rq *cfs_rq) + * until it gets a new slice. See the HACK in set_next_entity(). + */ + if (sched_feat(RUN_TO_PARITY) && curr && curr->vlag == curr->deadline) +#ifdef CONFIG_SCHED_BORE -+ if (likely(sched_bore) && likely(sched_burst_parity_threshold) && -+ sched_burst_parity_threshold < cfs_rq->nr_running) -+ return false; ++ if (!(likely(sched_bore) && likely(sched_burst_parity_threshold) && ++ sched_burst_parity_threshold < cfs_rq->nr_running)) +#endif // CONFIG_SCHED_BORE -+ -+ /* -+ * Allow preemption at the 0-lag point -- even if not all of the slice -+ * is consumed. Note: placement of positive lag can push V left and render -+ * @curr instantly ineligible irrespective the time on-cpu. -+ */ -+ if (sched_feat(RUN_TO_PARITY) && !entity_eligible(cfs_rq, curr)) -+ return false; -+ -+ /* -+ * Don't preserve @curr when the @wakee has a shorter slice and earlier -+ * deadline. IOW, explicitly allow preemption. -+ */ -+ if (sched_feat(PREEMPT_SHORT) && wakee && -+ wakee->slice < curr->slice && -+ (s64)(wakee->deadline - curr->deadline) < 0) -+ return false; -+ -+ /* -+ * Preserve @curr to allow it to finish its first slice. -+ * See the HACK in set_next_entity(). -+ */ -+ return curr->vlag == curr->deadline; -+} -+ - /* - * Earliest Eligible Virtual Deadline First - * -@@ -887,28 +1179,27 @@ struct sched_entity *__pick_first_entity(struct cfs_rq *cfs_rq) - * - * Which allows tree pruning through eligibility. - */ --static struct sched_entity *pick_eevdf(struct cfs_rq *cfs_rq) -+static struct sched_entity *pick_eevdf(struct cfs_rq *cfs_rq, struct sched_entity *wakee) - { - struct rb_node *node = cfs_rq->tasks_timeline.rb_root.rb_node; - struct sched_entity *se = __pick_first_entity(cfs_rq); - struct sched_entity *curr = cfs_rq->curr; - struct sched_entity *best = NULL; - -+ if (curr && !curr->on_rq) -+ curr = NULL; -+ - /* - * We can safely skip eligibility check if there is only one entity - * in this cfs_rq, saving some cycles. - */ - if (cfs_rq->nr_running == 1) -- return curr && curr->on_rq ? curr : se; -- -- if (curr && (!curr->on_rq || !entity_eligible(cfs_rq, curr))) -- curr = NULL; -+ return curr ?: se; - - /* -- * Once selected, run a task until it either becomes non-eligible or -- * until it gets a new slice. See the HACK in set_next_entity(). -+ * Preserve @curr to let it finish its slice. - */ -- if (sched_feat(RUN_TO_PARITY) && curr && curr->vlag == curr->deadline) -+ if (pick_curr(cfs_rq, curr, wakee)) return curr; /* Pick the leftmost entity if it's eligible */ -@@ -967,6 +1258,7 @@ struct sched_entity *__pick_last_entity(struct cfs_rq *cfs_rq) +@@ -967,6 +983,7 @@ struct sched_entity *__pick_last_entity(struct cfs_rq *cfs_rq) * Scheduling class statistics methods: */ #ifdef CONFIG_SMP @@ -809,7 +853,7 @@ index 2928026d7..91a870173 100644 int sched_update_scaling(void) { unsigned int factor = get_update_sysctl_factor(); -@@ -978,6 +1270,7 @@ int sched_update_scaling(void) +@@ -978,6 +995,7 @@ int sched_update_scaling(void) return 0; } @@ -817,7 +861,7 @@ index 2928026d7..91a870173 100644 #endif #endif -@@ -1178,6 +1471,10 @@ static void update_curr(struct cfs_rq *cfs_rq) +@@ -1178,6 +1196,10 @@ static void update_curr(struct cfs_rq *cfs_rq) if (unlikely(delta_exec <= 0)) return; @@ -828,20 +872,16 @@ index 2928026d7..91a870173 100644 curr->vruntime += calc_delta_fair(delta_exec, curr); update_deadline(cfs_rq, curr); update_min_vruntime(cfs_rq); -@@ -5193,6 +5490,12 @@ place_entity(struct cfs_rq *cfs_rq, struct sched_entity *se, int flags) - s64 lag = 0; +@@ -3795,7 +3817,7 @@ static void reweight_eevdf(struct sched_entity *se, u64 avruntime, + se->deadline = avruntime + vslice; + } - se->slice = sysctl_sched_base_slice; -+#ifdef CONFIG_SCHED_BORE -+ if (likely(sched_bore) && -+ (flags & ~sched_deadline_boost_mask & sched_deadline_preserve_mask)) -+ vslice = se->deadline - se->vruntime; -+ else -+#endif // CONFIG_SCHED_BORE - vslice = calc_delta_fair(se->slice, se); - - /* -@@ -5203,6 +5506,9 @@ place_entity(struct cfs_rq *cfs_rq, struct sched_entity *se, int flags) +-static void reweight_entity(struct cfs_rq *cfs_rq, struct sched_entity *se, ++void reweight_entity(struct cfs_rq *cfs_rq, struct sched_entity *se, + unsigned long weight) + { + bool curr = cfs_rq->curr == se; +@@ -5203,6 +5225,9 @@ place_entity(struct cfs_rq *cfs_rq, struct sched_entity *se, int flags) * * EEVDF: placement strategy #1 / #2 */ @@ -851,30 +891,46 @@ index 2928026d7..91a870173 100644 if (sched_feat(PLACE_LAG) && cfs_rq->nr_running) { struct sched_entity *curr = cfs_rq->curr; unsigned long load; -@@ -5278,6 +5584,13 @@ place_entity(struct cfs_rq *cfs_rq, struct sched_entity *se, int flags) - * on average, halfway through their slice, as such start tasks - * off with half a slice to ease into the competition. - */ +@@ -5273,6 +5298,18 @@ place_entity(struct cfs_rq *cfs_rq, struct sched_entity *se, int flags) + + se->vruntime = vruntime - lag; + ++ if (sched_feat(PLACE_REL_DEADLINE) && se->rel_deadline) { ++ se->deadline += se->vruntime; ++ se->rel_deadline = 0; ++ return; ++ } +#ifdef CONFIG_SCHED_BORE -+ if (likely(sched_bore)) { ++ else if (likely(sched_bore)) { + if (flags & sched_deadline_boost_mask) + vslice /= 2; + } + else +#endif // CONFIG_SCHED_BORE - if (sched_feat(PLACE_DEADLINE_INITIAL) && (flags & ENQUEUE_INITIAL)) - vslice /= 2; + /* + * When joining the competition; the existing tasks will be, + * on average, halfway through their slice, as such start tasks +@@ -5382,6 +5419,7 @@ static __always_inline void return_cfs_rq_runtime(struct cfs_rq *cfs_rq); + static void + dequeue_entity(struct cfs_rq *cfs_rq, struct sched_entity *se, int flags) + { ++ bool sleep = flags & DEQUEUE_SLEEP; + int action = UPDATE_TG; -@@ -5492,7 +5805,7 @@ pick_next_entity(struct cfs_rq *cfs_rq) - cfs_rq->next && entity_eligible(cfs_rq, cfs_rq->next)) - return cfs_rq->next; + if (entity_is_task(se) && task_on_rq_migrating(task_of(se))) +@@ -5409,6 +5447,11 @@ dequeue_entity(struct cfs_rq *cfs_rq, struct sched_entity *se, int flags) + clear_buddies(cfs_rq, se); -- return pick_eevdf(cfs_rq); -+ return pick_eevdf(cfs_rq, NULL); - } - - static bool check_cfs_rq_runtime(struct cfs_rq *cfs_rq); -@@ -6860,6 +7173,14 @@ static void dequeue_task_fair(struct rq *rq, struct task_struct *p, int flags) + update_entity_lag(cfs_rq, se); ++ if (sched_feat(PLACE_REL_DEADLINE) && !sleep) { ++ se->deadline -= se->vruntime; ++ se->rel_deadline = 1; ++ } ++ + if (se != cfs_rq->curr) + __dequeue_entity(cfs_rq, se); + se->on_rq = 0; +@@ -6860,6 +6903,14 @@ static void dequeue_task_fair(struct rq *rq, struct task_struct *p, int flags) bool was_sched_idle = sched_idle_rq(rq); util_est_dequeue(&rq->cfs, p); @@ -889,16 +945,7 @@ index 2928026d7..91a870173 100644 for_each_sched_entity(se) { cfs_rq = cfs_rq_of(se); -@@ -8428,7 +8749,7 @@ static void check_preempt_wakeup_fair(struct rq *rq, struct task_struct *p, int - /* - * XXX pick_eevdf(cfs_rq) != se ? - */ -- if (pick_eevdf(cfs_rq) == pse) -+ if (pick_eevdf(cfs_rq, pse) == pse) - goto preempt; - - return; -@@ -8646,16 +8967,25 @@ static void yield_task_fair(struct rq *rq) +@@ -8646,16 +8697,25 @@ static void yield_task_fair(struct rq *rq) /* * Are we the only task in the tree? */ @@ -924,7 +971,7 @@ index 2928026d7..91a870173 100644 /* * Tell update_rq_clock() that we've just updated, * so we don't do microscopic update in schedule() -@@ -12720,6 +13050,9 @@ static void task_fork_fair(struct task_struct *p) +@@ -12720,6 +12780,9 @@ static void task_fork_fair(struct task_struct *p) curr = cfs_rq->curr; if (curr) update_curr(cfs_rq); @@ -934,34 +981,32 @@ index 2928026d7..91a870173 100644 place_entity(cfs_rq, se, ENQUEUE_INITIAL); rq_unlock(rq, &rf); } +@@ -12832,6 +12895,10 @@ static void attach_task_cfs_rq(struct task_struct *p) + + static void switched_from_fair(struct rq *rq, struct task_struct *p) + { ++ p->se.rel_deadline = 0; ++#ifdef CONFIG_SCHED_BORE ++ init_task_bore(p); ++#endif // CONFIG_SCHED_BORE + detach_task_cfs_rq(p); + } + diff --git a/kernel/sched/features.h b/kernel/sched/features.h -index 143f55df8..09599a67c 100644 +index 143f55df8..e97b7b68b 100644 --- a/kernel/sched/features.h +++ b/kernel/sched/features.h -@@ -5,8 +5,22 @@ - * sleep+wake cycles. EEVDF placement strategy #1, #2 if disabled. +@@ -6,6 +6,10 @@ */ SCHED_FEAT(PLACE_LAG, true) -+/* -+ * Give new tasks half a slice to ease into the competition. -+ */ SCHED_FEAT(PLACE_DEADLINE_INITIAL, true) --SCHED_FEAT(RUN_TO_PARITY, true) +/* -+ * Inhibit (wakeup) preemption until the current task has exhausted its slice. ++ * Preserve relative virtual deadline on 'migration'. + */ -+SCHED_FEAT(RESPECT_SLICE, true) -+/* -+ * Relax RESPECT_SLICE to allow preemption once current has reached 0-lag. -+ */ -+SCHED_FEAT(RUN_TO_PARITY, false) -+/* -+ * Allow tasks with a shorter slice to disregard RESPECT_SLICE -+ */ -+SCHED_FEAT(PREEMPT_SHORT, true) ++SCHED_FEAT(PLACE_REL_DEADLINE, true) + SCHED_FEAT(RUN_TO_PARITY, true) /* - * Prefer to schedule the task we woke last (assuming it failed diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h index 207a04f02..c99430161 100644 --- a/kernel/sched/sched.h @@ -989,4 +1034,4 @@ index 207a04f02..c99430161 100644 #ifdef CONFIG_SCHED_DEBUG extern int sysctl_resched_latency_warn_ms; -- -2.45.2.606.g9005149a4a \ No newline at end of file +2.46.0.469.g4590f2e941