From f77d25758ec26016bfeed8e8ecacb9f786aa908d Mon Sep 17 00:00:00 2001 From: Piotr Gorski Date: Mon, 17 Aug 2026 17:46:40 +0200 Subject: [PATCH 1/5] Revert POC Signed-off-by: Piotr Gorski --- include/linux/sched/topology.h | 46 - init/Kconfig | 13 - kernel/sched/ext/ext.c | 7 - kernel/sched/fair.c | 159 +-- kernel/sched/idle.c | 10 - kernel/sched/poc_selector.c | 2038 -------------------------------- kernel/sched/sched.h | 118 -- kernel/sched/topology.c | 3 - 8 files changed, 36 insertions(+), 2358 deletions(-) delete mode 100644 kernel/sched/poc_selector.c diff --git a/include/linux/sched/topology.h b/include/linux/sched/topology.h index 2d939fa46..b5d9d7c2b 100644 --- a/include/linux/sched/topology.h +++ b/include/linux/sched/topology.h @@ -86,52 +86,6 @@ struct sched_domain_shared { unsigned long util_avg; unsigned long capacity; #endif -#ifdef CONFIG_SCHED_POC_SELECTOR - /* - * POC Selector: per-LLC idle CPU tracking - */ - u64 poc_llc_members; /* bitmask of valid CPUs (relative to base) */ - int poc_cpu_base; /* smallest CPU ID in this LLC */ - u8 poc_affinity_shift; /* bit shift for cpumask alignment */ - bool poc_fast_eligible; /* true when LLC CPU count <= 64 */ - bool poc_cluster_valid; /* true when cluster mask is usable */ -#ifdef CONFIG_SCHED_SMT - u8 poc_smt_shift; /* bit distance between SMT siblings */ - u64 poc_primary_mask; /* bitmask of core representative CPUs */ -#endif - - /* - * Hot write path: idle state flag arrays (lock-free mode). - * Each array = exactly 1 cache line (64B). - * Writers: WRITE_ONCE (plain MOV, no LOCK prefix). - * Readers: snapshot to stack, then multiply-and-shift aggregation. - * Active only when sched_poc_atomic_bitmap=0. - */ - u8 poc_idle_cpus[64] ____cacheline_aligned; -#ifdef CONFIG_SCHED_SMT - u8 poc_idle_cores[64] ____cacheline_aligned; -#endif /* CONFIG_SCHED_SMT */ - - /* - * Hot read/write path: idle state bitmaps (bitmap mode, default). - * Readers: single atomic64_read (MOV on x86). - * Writers: atomic64_or / atomic64_andnot (LOCK'd on x86). - * Active only when sched_poc_atomic_bitmap=1. - */ - atomic64_t poc_idle_cpus_mask ____cacheline_aligned; -#ifdef CONFIG_SCHED_SMT - atomic64_t poc_idle_cores_mask ____cacheline_aligned; -#endif /* CONFIG_SCHED_SMT */ - - /* - * Read-only lookup tables (written once at init). - * Cacheline-aligned for exact prefetch targeting. - */ - u64 poc_cluster_mask[64] ____cacheline_aligned; -#ifdef CONFIG_SCHED_SMT - u64 poc_smt_mask[64] ____cacheline_aligned; -#endif /* CONFIG_SCHED_SMT */ -#endif /* CONFIG_SCHED_POC_SELECTOR */ }; struct sched_domain { diff --git a/init/Kconfig b/init/Kconfig index 2cd2ab2d7..8937400a3 100644 --- a/init/Kconfig +++ b/init/Kconfig @@ -1496,19 +1496,6 @@ config SCHED_AUTOGROUP desktop applications. Task group autogeneration is currently based upon task session. -config SCHED_POC_SELECTOR - bool "Piece-Of-Cake Fast Idle CPU Selector" - depends on SMP - default y - help - Idle CPU selector using cached bitmasks inspired by the scx_cake BPF - scheduler. Reduces select_idle_cpu overhead by using bitmap scanning. - - This optimization does not affect scheduler fairness - it only - speeds up the process of finding an idle CPU for task wakeup. - - If unsure, say Y. - config RELAY bool "Kernel->user space relay support (formerly relayfs)" select IRQ_WORK diff --git a/kernel/sched/ext/ext.c b/kernel/sched/ext/ext.c index e0132294b..18183062f 100644 --- a/kernel/sched/ext/ext.c +++ b/kernel/sched/ext/ext.c @@ -6236,9 +6236,6 @@ static void scx_root_disable(struct scx_sched *sch) mutex_unlock(&scx_enable_mutex); WARN_ON_ONCE(scx_set_enable_state(SCX_DISABLED) != SCX_DISABLING); -#ifdef CONFIG_SCHED_POC_SELECTOR - poc_notify_scx(false); -#endif done: scx_bypass(sch, false); } @@ -7453,10 +7450,6 @@ static void scx_root_enable_workfn(struct kthread_work *work) } } -#ifdef CONFIG_SCHED_POC_SELECTOR - poc_notify_scx(true); -#endif - pr_info("sched_ext: BPF scheduler \"%s\" enabled%s\n", sch->ops.name, scx_switched_all() ? "" : " (partial)"); kobject_uevent(&sch->kobj, KOBJ_ADD); diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c index c4d098bdb..d309809ec 100644 --- a/kernel/sched/fair.c +++ b/kernel/sched/fair.c @@ -50,7 +50,6 @@ #include #include #include -#include #include @@ -1240,7 +1239,7 @@ static bool update_deadline(struct cfs_rq *cfs_rq, struct sched_entity *se) #include "pelt.h" -static int select_idle_sibling(struct task_struct *p, int prev_cpu, int cpu, int sync); +static int select_idle_sibling(struct task_struct *p, int prev_cpu, int cpu); static unsigned long task_h_load(struct task_struct *p); static unsigned long capacity_of(int cpu); @@ -8538,20 +8537,6 @@ void __update_idle_core(struct rq *rq) rcu_read_unlock(); } -/* - * Check if the entire core (all SMT siblings) containing @cpu is idle. - */ -static inline bool is_idle_core(int cpu) -{ - int sibling; - - for_each_cpu(sibling, cpu_smt_mask(cpu)) { - if (!available_idle_cpu(sibling)) - return false; - } - return true; -} - /* * Scan the entire LLC domain for idle cores; this dynamically switches off if * there are no idle cores left in the system; tracked through @@ -8879,38 +8864,16 @@ static inline bool asym_fits_cpu(unsigned long util, return true; } -#ifdef CONFIG_SCHED_POC_SELECTOR -#include "poc_selector.c" -#endif /* * Try and locate an idle core/thread in the LLC cache domain. */ -static int select_idle_sibling(struct task_struct *p, int prev, int target, int sync) +static int select_idle_sibling(struct task_struct *p, int prev, int target) { bool has_idle_core = false; struct sched_domain *sd; unsigned long task_util, util_min, util_max; int i, recent_used_cpu, prev_aff = -1; - /* Check a recently used CPU as a potential idle candidate: */ - recent_used_cpu = p->recent_used_cpu; - p->recent_used_cpu = prev; - if (recent_used_cpu != prev && - recent_used_cpu != target && - cpus_share_cache(recent_used_cpu, target) && - choose_idle_cpu(recent_used_cpu, p) && - cpumask_test_cpu(recent_used_cpu, p->cpus_ptr)) { -#ifdef CONFIG_SCHED_POC_SELECTOR - if (!static_branch_likely(&poc_selector_active) || - static_branch_unlikely(&sched_poc_early_select)) -#endif - if ((unsigned int)recent_used_cpu < nr_cpumask_bits && - is_idle_core(recent_used_cpu)) - return recent_used_cpu; - } else { - recent_used_cpu = -1; - } - /* * On asymmetric system, update task utilization because we will check * that the task fits with CPU's capacity. @@ -8927,13 +8890,23 @@ static int select_idle_sibling(struct task_struct *p, int prev, int target, int */ lockdep_assert_irqs_disabled(); -#ifdef CONFIG_SCHED_POC_SELECTOR - if (static_branch_likely(&poc_selector_active) && - static_branch_unlikely(&sched_poc_early_select) && - is_idle_core(target) && + if (choose_idle_cpu(target, p) && asym_fits_cpu(task_util, util_min, util_max, target)) return target; -#endif + + /* + * If the previous CPU is cache affine and idle, don't be stupid: + */ + if (prev != target && cpus_share_cache(prev, target) && + choose_idle_cpu(prev, p) && + asym_fits_cpu(task_util, util_min, util_max, prev)) { + + if (!static_branch_unlikely(&sched_cluster_active) || + cpus_share_resources(prev, target)) + return prev; + + prev_aff = prev; + } /* * Allow a per-cpu kthread to stack with the wakee if the @@ -8951,6 +8924,24 @@ static int select_idle_sibling(struct task_struct *p, int prev, int target, int return prev; } + /* Check a recently used CPU as a potential idle candidate: */ + recent_used_cpu = p->recent_used_cpu; + p->recent_used_cpu = prev; + if (recent_used_cpu != prev && + recent_used_cpu != target && + cpus_share_cache(recent_used_cpu, target) && + choose_idle_cpu(recent_used_cpu, p) && + cpumask_test_cpu(recent_used_cpu, p->cpus_ptr) && + asym_fits_cpu(task_util, util_min, util_max, recent_used_cpu)) { + + if (!static_branch_unlikely(&sched_cluster_active) || + cpus_share_resources(recent_used_cpu, target)) + return recent_used_cpu; + + } else { + recent_used_cpu = -1; + } + /* * For asymmetric CPU capacity systems, our domain of interest is * sd_asym_cpucapacity rather than sd_llc. @@ -8975,74 +8966,6 @@ static int select_idle_sibling(struct task_struct *p, int prev, int target, int if (!sd) return target; -#ifdef CONFIG_SCHED_POC_SELECTOR - { - struct sched_domain_shared *sd_share = - rcu_dereference(per_cpu(sd_llc_shared, target)); - if (static_branch_likely(&poc_selector_active) - && !sched_asym_cpucap_active() - && sd_share && likely(sd_share->poc_fast_eligible)) { - int poc_cpu = select_idle_cpu_poc(target, prev, - recent_used_cpu, sync, - sd_share, p->cpus_ptr); - if (poc_cpu >= 0) { - return poc_cpu; - } - /* - * POC returns -2 when the SIS_UTIL overload gate fires - * (smt_fallback=0 only). POC has already checked - * prev's SMT sibling (Level 4) and decided broader - * search is not worthwhile. CFS would reach the same - * conclusion, so skip select_idle_smt/select_idle_cpu. - * - * POC returns -1 for Level 0 saturation (no idle CPUs - * in bitmap), but CFS may still find sched_idle CPUs, - * so we must NOT skip CFS in that case. - */ - if (poc_cpu == -2) - goto give_up; - } else { - /* - * poc_selector_active is off — POC is either disabled - * by sysctl or suppressed while scx is running. - * If an scx scheduler called us, flip poc_selector_skip - * and schedule a workqueue item to re-enable POC with - * bitmap resync. - */ - poc_check_skip_fallback(); - } - } - poc_count(POC_FALLBACK); -#endif /* CONFIG_SCHED_POC_SELECTOR */ - - if ((unsigned int)recent_used_cpu < nr_cpumask_bits) { - if (choose_idle_cpu(recent_used_cpu, p)) { - if (is_idle_core(recent_used_cpu)) - return recent_used_cpu; - /* idle CPU but not idle core → preserve for give_up */ - } else { - recent_used_cpu = -1; /* not idle → discard */ - } - } - - if (sync && is_idle_core(target) && - asym_fits_cpu(task_util, util_min, util_max, target)) - return target; - - /* - * If the previous CPU is cache affine and idle, don't be stupid: - */ - if (prev != target && cpus_share_cache(prev, target) && - choose_idle_cpu(prev, p) && - asym_fits_cpu(task_util, util_min, util_max, prev)) { - - if (!static_branch_unlikely(&sched_cluster_active) || - cpus_share_resources(prev, target)) - return prev; - - prev_aff = prev; - } - if (sched_smt_active()) { has_idle_core = test_idle_cores(target); @@ -9057,9 +8980,6 @@ static int select_idle_sibling(struct task_struct *p, int prev, int target, int if ((unsigned)i < nr_cpumask_bits) return i; -#ifdef CONFIG_SCHED_POC_SELECTOR -give_up: -#endif /* * For cluster machines which have lower sharing cache like L2 or * LLC Tag, we tend to find an idle CPU in the target's cluster @@ -9071,13 +8991,6 @@ static int select_idle_sibling(struct task_struct *p, int prev, int target, int if ((unsigned int)recent_used_cpu < nr_cpumask_bits) return recent_used_cpu; -#ifdef CONFIG_SCHED_POC_SELECTOR - /* Last resort: avoid enqueuing behind RT/DL tasks on target */ - if (static_branch_likely(&poc_selector_active) && - rt_task(cpu_rq(target)->curr) && - prev != target && !rt_task(cpu_rq(prev)->curr)) - return prev; -#endif return target; } @@ -9760,7 +9673,7 @@ select_task_rq_fair(struct task_struct *p, int prev_cpu, int wake_flags) /* Fast path */ if (wake_flags & WF_TTWU) - return select_idle_sibling(p, prev_cpu, new_cpu, sync); + return select_idle_sibling(p, prev_cpu, new_cpu); return new_cpu; } diff --git a/kernel/sched/idle.c b/kernel/sched/idle.c index d4d77f281..052435f4d 100644 --- a/kernel/sched/idle.c +++ b/kernel/sched/idle.c @@ -305,11 +305,6 @@ static void do_idle(void) __current_set_polling(); tick_nohz_idle_enter(); -#ifdef CONFIG_SCHED_POC_SELECTOR - /* POC Selector: mark CPU as idle */ - set_cpu_idle_state_poc(cpu, 1); -#endif /* CONFIG_SCHED_POC_SELECTOR */ - while (!need_resched()) { /* @@ -363,11 +358,6 @@ static void do_idle(void) arch_cpu_idle_exit(); } -#ifdef CONFIG_SCHED_POC_SELECTOR - /* POC Selector: mark CPU as busy */ - set_cpu_idle_state_poc(cpu, 0); -#endif /* CONFIG_SCHED_POC_SELECTOR */ - /* * Since we fell out of the loop above, we know TIF_NEED_RESCHED must * be set, propagate it into PREEMPT_NEED_RESCHED. diff --git a/kernel/sched/poc_selector.c b/kernel/sched/poc_selector.c deleted file mode 100644 index d0f1a4532..000000000 --- a/kernel/sched/poc_selector.c +++ /dev/null @@ -1,2038 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 -/* - * Piece-Of-Cake (POC) CPU Selector - * - * Fast idle CPU selector inspired by RitzDaCat's scx_cake scheduler - * "Piece of Cake" - making idle CPU search a piece of cake! - * - * Tracks idle state in per-LLC atomic64_t bitmaps with lock-free - * atomic64_read/or/andnot for O(1) idle CPU lookup. - * Supports up to 64 CPUs per LLC (single 64-bit word). - * Includes affinity-aware filtering via cpumask intersection. - * - * When the fast path is not eligible (LLC exceeds 64 CPUs), - * returns -1 to let CFS standard select_idle_cpu handle it. - * - * Copyright (C) 2026 Masahito Suzuki - * - * Acknowledgements: - * This work is heavily inspired by RitzDaCat's scx_cake scheduler. - * - * The select_idle_sibling() restructuring on which the POC fast - * path depends (is_idle_core() helper, @sync parameter, hoisting - * the recent_used_cpu check to the function top, switching the - * target fast-return to is_idle_core()) is adapted from: - * - * Andrea Righi, Mario Roy, and Eric Naim - * ("sched/fair: Prefer the previous cpu for wakeup") - * - * - Andrea Righi: is_idle_core() helper and idle-core - * target selection in select_idle_sibling() - * - Mario Roy: prev-cpu preference and recent_used_cpu - * hoisting with idle-core check - * - * Special thanks to the algorithm inventors whose research enabled - * the O(1) techniques used in this implementation: - * - * - Prashant Pandey, Michael A. Bender, Rob Johnson - * ("A Fast x86 Implementation of Select") - * - * - Daniel Lemire - * ("Fast Random Integer Generation in an Interval") - */ - -#ifdef CONFIG_SCHED_POC_SELECTOR - -/************************************************************** - * Version Information: - */ - -#define SCHED_POC_SELECTOR_AUTHOR "Masahito Suzuki" -#define SCHED_POC_SELECTOR_PROGNAME "Piece-Of-Cake (POC) CPU Selector" - -#define SCHED_POC_SELECTOR_VERSION "2.6.3" - -/************************************************************** - * Static keys: - */ - -/* - * Runtime control: poc_selector_active (static key) - * Derived from: sched_poc_selector && !poc_selector_skip - * - * sched_poc_selector: user-visible sysctl (kernel.sched_poc_selector), - * plain bool, default true. - * poc_selector_skip: set true while sched_ext is active to avoid - * idle-bitmap overhead in do_idle. - * poc_selector_active: the actual static key gating all POC hot paths. - * Enabled only when sched_poc_selector && !poc_selector_skip. - * On enable transition, poc_resync_idle_state() is called. - */ -DEFINE_STATIC_KEY_TRUE(poc_selector_active); -static bool sched_poc_selector = true; -static bool poc_selector_skip; - -/* - * SMT fallback control: sched_poc_smt_fallback - * (sysctl kernel.sched_poc_smt_fallback) - * - * When enabled, POC bails out to CFS when no idle - * cores exist (has_idle_cores == false). CFS then handles - * SMT sibling selection via select_idle_smt(prev) and - * nr_idle_scan-limited select_idle_cpu(). - * - * When disabled (default), POC handles SMT sibling selection - * itself, trying prev's SMT sibling for cache locality, then - * LLC-wide RR search for remaining idle CPUs. Level 5/6 is - * gated by nr_idle_scan (SIS_UTIL): when LLC utilization - * exceeds ~85%, broader SMT search is skipped. - */ -DEFINE_STATIC_KEY_FALSE(sched_poc_smt_fallback); - -/* - * SMT consecutive layout: sched_poc_smt_consecutive - * - * When true (default), SMT siblings occupy consecutive LLC-relative - * positions (e.g., CPU 0,1 / 2,3 / ...). The idle core mask is - * derived from the idle CPU mask via bit-parallel operations: - * core_mask = cpu_mask & (cpu_mask >> 1) & 0x5555555555555555ULL - * - * Disabled at boot if non-consecutive 2-way SMT or >2-way SMT - * is detected on any LLC. - */ -DEFINE_STATIC_KEY_TRUE(sched_poc_smt_consecutive); - -/* - * SMT uniform 2-way layout: sched_poc_smt_uniform - * - * When true (default), all cores in every LLC have uniform 2-way SMT - * with a constant stride between siblings. The idle core mask is - * derived at read time via: - * core_mask = cpu_mask & (cpu_mask >> poc_smt_shift) & poc_primary_mask - * - * This covers both consecutive (stride=1) and stride-N (e.g., Intel - * Xeon) layouts without write-path overhead. - * - * When false (>2-way SMT or non-uniform topology), falls back to - * write-time maintenance of poc_idle_cores_mask atomic64_t. - * - * Disabled at boot if any LLC contains non-2-way or non-uniform SMT. - */ -DEFINE_STATIC_KEY_TRUE(sched_poc_smt_uniform); - -/* - * Target CPU sticky: sched_poc_target_sticky - * (sysctl kernel.sched_poc_target_sticky) - * - * When enabled, if the target CPU is idle in the bitmap, return it - * immediately — regardless of whether its core is fully idle. - * This provides L1 cache affinity: the waking task reuses the CPU - * it ran on last, keeping warm TLB/L1/L2 state. - * - * Checked after Level 0 (saturation) and before core_mask derivation. - * Default: disabled. - */ -DEFINE_STATIC_KEY_FALSE(sched_poc_target_sticky); - -/* - * Early select: sched_poc_early_select - * (sysctl kernel.sched_poc_early_select) - * - * When enabled, select_idle_sibling performs idle-core checks - * for recent_used_cpu and target BEFORE entering POC search: - * - recent_used_cpu with fully idle core → return immediately - * (matches upstream CFS Gate 4 behavior) - * - target with fully idle core → return immediately - * (avoids POC overhead: RCU deref, bitmap read, mask ops) - * - * These two checks must be toggled together to preserve POC's - * internal priority order (Level 1r before 1t). Enabling only - * one would let the pre-POC path return a lower-priority result - * before POC can evaluate the higher-priority candidate. - * - * Default: enabled. - */ -DEFINE_STATIC_KEY_TRUE(sched_poc_early_select); - -/* - * Greedy search: sched_poc_greedy_search - * (sysctl kernel.sched_poc_greedy_search) - * - * When enabled, POC always attempts Level 5/6 (LLC-wide SMT sibling - * search) regardless of utilization, ignoring the SIS_UTIL overload - * gate (nr_idle_scan == 0). This may benefit latency-sensitive - * workloads that want to find any idle CPU at all costs. - * - * When disabled, POC skips Level 5/6 under overload, - * returning -2 to also skip CFS fallback search. - * - * Default: enabled. - */ -DEFINE_STATIC_KEY_TRUE(sched_poc_greedy_search); - -/* - * sched_poc_aligned: true when all LLCs have poc_cpu_base aligned to 64 - * - * When true, cpumask-to-POC conversion is a simple word load (zero shift). - * When false (e.g., Threadripper CCDs at CPU 8, 16, ...), bit shifting - * is needed to align cpumask bits with POC's LLC-relative positions. - * Defaults to true; disabled at boot if any LLC has non-aligned base. - */ -DEFINE_STATIC_KEY_TRUE(sched_poc_aligned); - -/* - * Packed priority search: sched_poc_packed - * - * When true (default), per-LLC CPU count is ≤ 32, enabling packed - * priority search. Cluster candidates (Level 2) and LLC-wide - * candidates (Level 3) are packed into a single 64-bit word: - * - * bits [31:0]: cluster idle candidates (high priority) - * bits [63:32]: all LLC idle candidates (low priority) - * - * A single TZCNT resolves both levels simultaneously. - * ror32-based rotation distributes selections across idle CPUs. - * - * When false (LLC > 32 CPUs), falls back to separate cluster - * search + PTSELECT-based RR. - * - * Disabled at boot if any LLC has > 32 CPUs. - */ -DEFINE_STATIC_KEY_TRUE(sched_poc_packed); - -/* - * Improved RR strategy: sched_poc_rr_improved - * (sysctl kernel.sched_poc_rr_improved) - * - * When enabled (default), idle CPU selection in poc_select_rr, - * poc_cluster_search, and the packed priority search uses an - * improved RR strategy combining two techniques: - * 1. total size case-split (1/2/>=3): direct / interleave / full - * 2. golden-ratio scrambling (Lemire fastrange) - * - * When disabled, the current strategy is used unchanged: - * - poc_select_rr: poc_rr_step[] table (perfect RR) - * - poc_cluster_search: ctz lowest-bit selection (no RR) - * - packed search: ror32(counter & 31) - * - * The current path is preserved as the A/B-testing baseline; - * once the improved path is validated, the legacy code will - * be removed in a follow-up. - */ -DEFINE_STATIC_KEY_TRUE(sched_poc_rr_improved); - -/* - * Lockless bitmap mode: sched_poc_lockless_bitmap - * (sysctl kernel.sched_poc_lockless_bitmap) - * - * When enabled, idle state is tracked in u8[64] flag arrays. - * Writers use plain WRITE_ONCE (no LOCK prefix); readers snapshot - * the 64-byte cache line to the stack, then use multiply-and-shift - * aggregation to assemble a u64 bitmask. - * - * When disabled (default), idle state is tracked in atomic64_t bitmaps. - * Readers use a single atomic64_read (MOV on x86); writers use - * atomic64_or / atomic64_andnot (LOCK'd on x86). - * - * Only one representation is maintained at a time (single-write). - * Switching via sysctl resyncs the newly-active representation - * before readers can observe it. - * - * Default: disabled. - */ -DEFINE_STATIC_KEY_FALSE(sched_poc_lockless_bitmap); - -/************************************************************** - * Debug counters (sysctl kernel.sched_poc_count): - * - * Per-CPU counters for each selection level hit. - * Guarded by static key — zero overhead when disabled (default). - * Aggregated across all CPUs and exposed via sysfs. - */ -enum poc_level { - POC_LV1S = 0, /* target CPU sticky (L1/TLB affinity) */ - POC_LV1T, /* target core idle */ - POC_LV1P, /* prev core idle */ - POC_LV1R, /* recent core idle */ - POC_LV2, /* idle core in L2 cluster */ - POC_LV3, /* idle core across LLC (RR) */ - POC_LV4S, /* sync + target CPU idle (no idle cores) */ - POC_LV4P, /* prev's SMT sibling (cache locality) */ - POC_LV4R, /* recent's SMT sibling (warm cache) */ - POC_LV4T, /* target's SMT sibling */ - POC_LV5, /* idle CPU in L2 cluster */ - POC_LV6, /* idle CPU across LLC (RR) */ - POC_FALLBACK, /* POC returned -1, CFS fallback */ - POC_NR_LEVELS -}; - -#define POC_SMT_LEVEL_OFFSET (POC_LV5 - POC_LV2) - -DEFINE_STATIC_KEY_FALSE(sched_poc_count_enabled); - -static DEFINE_PER_CPU(unsigned long[POC_NR_LEVELS], poc_debug_cnt); - -static __always_inline void poc_count(enum poc_level lv) -{ - if (static_branch_unlikely(&sched_poc_count_enabled)) - __this_cpu_inc(poc_debug_cnt[lv]); -} - -/************************************************************** - * Per-CPU round-robin counter and division-free mapping: - */ - -/* - * POC_HASH_MULT / POC_SCRAMBLE — Golden-ratio scrambling - * - * Multiplying a 32-bit counter by ⌊2^32 / φ⌋ = 0x9E3779B9 scatters - * consecutive values across the 32-bit output space with good - * avalanche properties (Knuth's multiplicative hash, TAOCP Vol. 3). - * The scrambled value feeds POC_FASTRANGE for uniform [0, range) - * mapping in the improved RR path, or is used directly with a bit - * shift to derive an uncorrelated rotation amount in packed search. - */ -#define POC_HASH_MULT 0x9E3779B9U /* golden ratio * 2^32 */ -#define POC_SCRAMBLE(counter) ((u32)(counter) * POC_HASH_MULT) - -/* - * Per-CPU round-robin counter for idle CPU selection. - * Each CPU starts at a different offset to reduce cross-CPU - * collision probability. Combined with poc_rr_step[] and - * POC_FIXED_MOD16, consecutive calls on the same CPU produce - * perfect round-robin: each call picks a different idle CPU - * until all candidates have been visited. - */ -static DEFINE_PER_CPU(u32, poc_rr_counter); - -/* - * Division-free modulo via 16-bit fixed-point reciprocal multiplication - * - * The multiply-and-shift technique is inspired by: - * D. Lemire, "Fast Random Integer Generation in an Interval", - * ACM Trans. Model. Comput. Simul. 29, 1, Article 3, 2019. - * - * Combined with poc_rr_step[], this replaces modulo with two - * multiplications and a shift: - * phase = (u16)(counter * poc_rr_step[total - 1]) - * pick = POC_FIXED_MOD16(phase, total) - * - * Proof that pick == counter % total (for total ≤ 64): - * Let S = ceil(2^16 / N). For k in [0, N): - * k*S*N / 2^16 ∈ [k, k + kN/2^16) - * Since kN < N² ≤ 64² = 4096 ≪ 2^16, floor(kN/2^16) = 0, - * so floor(k*S*N / 2^16) = k. QED. - */ -#define POC_FIXED_MOD16(phase, range) ((u32)(((u32)(phase) * (u32)(range)) >> 16)) - -/* - * POC_FASTRANGE — Map a 32-bit scrambled value to [0, range) - * - * Implements Lemire's fastrange technique: - * D. Lemire, "Fast Random Integer Generation in an Interval", - * ACM Trans. Model. Comput. Simul. 29, 1, Article 3, 2019. - * - * Computes (seed * range) >> 32, giving a uniform mapping of - * a 32-bit seed into [0, range) using only one 64-bit multiply - * and a shift. Used with golden-ratio hashing for pseudo-random - * RR distribution in the improved RR path. - */ -#define POC_FASTRANGE(seed, range) ((u32)(((u64)(seed) * (u32)(range)) >> 32)) - -/* - * RR step table: poc_rr_step[n-1] = ceil(2^16 / n) for n = 1..64 - * - * Indexed by (total - 1) where total = popcount(idle mask). - * total == 0 is unreachable (caller guarantees mask != 0). - * 64 entries × 2 bytes = 128 bytes = exactly 2 cache lines. - */ -static const u16 poc_rr_step[64] = { - 0, 0x8000, 0x5556, 0x4000, 0x3334, 0x2AAB, 0x2493, 0x2000, /* 1.. 8 */ - 0x1C72, 0x199A, 0x1746, 0x1556, 0x13B2, 0x124A, 0x1112, 0x1000, /* 9..16 */ - 0x0F10, 0x0E39, 0x0D7A, 0x0CCD, 0x0C31, 0x0BA3, 0x0B22, 0x0AAB, /* 17..24 */ - 0x0A3E, 0x09D9, 0x097C, 0x0925, 0x08D4, 0x0889, 0x0843, 0x0800, /* 25..32 */ - 0x07C2, 0x0788, 0x0751, 0x071D, 0x06EC, 0x06BD, 0x0691, 0x0667, /* 33..40 */ - 0x063F, 0x0619, 0x05F5, 0x05D2, 0x05B1, 0x0591, 0x0573, 0x0556, /* 41..48 */ - 0x053A, 0x051F, 0x0506, 0x04ED, 0x04D5, 0x04BE, 0x04A8, 0x0493, /* 49..56 */ - 0x047E, 0x046A, 0x0457, 0x0445, 0x0433, 0x0422, 0x0411, 0x0400, /* 57..64 */ -}; - -/************************************************************** - * Bit manipulation primitives: - */ - -/* - * POC_CTZ64 — Portable Count Trailing Zeros (64-bit) - * - * Three-tier architecture detection: - * - * Tier 1: Native hardware CTZ with well-defined zero semantics - * x86-64 + BMI1 (__BMI__): TZCNT — returns 64 for input 0 - * ARM64: RBIT + CLZ - * RISC-V Zbb: CTZ instruction - * - * Tier 2: x86-64 without BMI1 (Bulldozer, pre-Haswell, etc.) - * BSF is fast (~3 cyc) but UNDEFINED for input 0. - * On AMD Bulldozer: BSF(0) leaves dest register unchanged (stale value). - * On Intel pre-Haswell: BSF(0) is architecturally undefined. - * Wrap with explicit zero check to guarantee returning 64. - * - * Tier 3: De Bruijn fallback (BPF, unknown architectures) - * Software multiply + 64-entry table lookup, branchless O(1). - */ - -/* - * POC_CTZ64 is defined in sched.h for use by load balancer functions. - * Here we only define POC_CTZ64_NAME for sysfs hardware info display. - */ -#if defined(__x86_64__) && defined(__BMI__) -#define POC_CTZ64_NAME "HW (TZCNT)" -#elif defined(__aarch64__) -#define POC_CTZ64_NAME "HW (RBIT+CLZ)" -#elif defined(__riscv) && defined(__riscv_zbb) -#define POC_CTZ64_NAME "HW (ctz)" -#elif defined(__x86_64__) -#define POC_CTZ64_NAME "HW (BSF)" -#else -#define POC_CTZ64_NAME "SW (De Bruijn)" -#endif - -/* - * POC_PTSELECT — Select position of the j-th set bit in a 64-bit word - * - * Based on the algorithm described in: - * P. Pandey, M. A. Bender, R. Johnson, - * "A Fast x86 Implementation of Select", arXiv:1706.00990, 2017. - * - * Returns the bit position (0-indexed) of the j-th set bit in v. - * Undefined behavior if j >= popcount(v). - * - * Tier 1 (x86-64 + BMI2, excluding AMD Zen 1/2 slow microcode PDEP): - * PDEP + TZCNT — 4 instructions total. - * PDEP deposits the j-th source bit at the j-th mask position. - * - * Tier 2 (fallback): Iterative bit-clear — O(j) iterations - * Clears the lowest set bit j times, then CTZ on remainder. - */ - -#if defined(__x86_64__) && defined(__BMI2__) && \ - !defined(__znver1) && !defined(__znver2) -static __always_inline int poc_ptselect(u64 v, int j) -{ - u64 deposited; - - asm("pdep %2, %1, %0" : "=r"(deposited) : "r"(1ULL << j), "rm"(v)); - return POC_CTZ64(deposited); -} -#define POC_PTSELECT(v, j) poc_ptselect(v, j) -#define POC_PTSELECT_NAME "HW (PDEP)" - -/* - * Tier 2 (fallback): Iterative bit-clear — O(j) iterations. - * Clears the lowest set bit j times, then returns its position via CTZ. - */ -#else -static __always_inline int poc_ptselect_sw(u64 v, int j) -{ - int k; - - for (k = 0; k < j; k++) - v &= v - 1; /* clear lowest set bit */ - return POC_CTZ64(v); -} -#define POC_PTSELECT(v, j) poc_ptselect_sw(v, j) -#define POC_PTSELECT_NAME "SW (loop)" - -#endif /* POC_PTSELECT */ - -/************************************************************** - * Flag array to bitmask conversion (lock-free mode): - */ - -/* - * POC_BYTE_EXTRACT / POC_BYTE_PACK - constants for multiply-and-shift trick. - * - * Isolates bit 0 of each byte in a u64 word, then packs the 8 bits - * into the most significant byte via multiply. - */ -#define POC_BYTE_EXTRACT 0x0101010101010101ULL -#define POC_BYTE_PACK 0x0102040810204080ULL - -/* - * POC_BMP8 - Convert one 8-byte slice of the flag array to 8 packed bits. - * - * Tier 1 (x86-64 + BMI2, excluding AMD Zen 1/2 slow microcode PEXT): - * PEXT extracts bit 0 of each byte directly into 8 contiguous bits. - * Single instruction replaces AND + MUL + SHR. - * - * Tier 2 (fallback): Multiply-and-shift trick. - * Isolates bit 0 of each byte (AND), packs via MUL, shifts to position. - */ -#if defined(__x86_64__) && defined(__BMI2__) && \ - !defined(__znver1) && !defined(__znver2) - -static __always_inline u64 poc_bmp8_pext(u64 word, int i) -{ - u64 extracted; - - asm("pext %2, %1, %0" : "=r"(extracted) : "r"(word), "r"(POC_BYTE_EXTRACT)); - return extracted << (i * 8); -} -#define POC_BMP8(w, i) poc_bmp8_pext((w)[i], i) - -#else - -#define POC_BMP8(w, i) \ - ((((w)[i] & POC_BYTE_EXTRACT) * POC_BYTE_PACK >> 56) << ((i) * 8)) - -#endif /* POC_BMP8 */ - -/* - * poc_flags_to_u64 - Convert u8[64] flag array to u64 bitmask - * @flags: pointer to 64-byte flag array (cacheline-aligned) - * - * Phase 1 (memcpy): snapshot the 64-byte cache line to the stack. - * This eliminates the window in which a concurrent MESI invalidation - * could cause a re-fetch mid-computation. All 64 bytes land in one - * or two cache line transfers; subsequent computation is purely local. - * - * Phase 2: pack the stack-local copy into a u64 bitmask via - * multiply-and-shift (or PEXT on BMI2 x86). Always processes all - * 8 chunks — the extra iterations for small LLCs are negligible - * on stack-local data and avoid the poc_chunks_bit* dispatch tree. - * - * Returns: u64 bitmask with bit N set iff flags[N] != 0 - */ -static __always_inline u64 poc_flags_to_u64(const u8 *flags) -{ - u64 w[8]; - - /* Phase 1: snapshot shared cache line to stack */ - memcpy(w, flags, 64); - - /* Phase 2: pack stack-local copy into bitmask */ - return POC_BMP8(w, 0) | POC_BMP8(w, 1) | POC_BMP8(w, 2) | POC_BMP8(w, 3) | - POC_BMP8(w, 4) | POC_BMP8(w, 5) | POC_BMP8(w, 6) | POC_BMP8(w, 7); -} - -/************************************************************** - * Idle mask accessors: - */ - -/* - * poc_idle_cpu_mask - Get idle CPU bitmask filtered by LLC and affinity - * @affinity: task's allowed CPU mask (poc-relative, from poc_cpumask_to_u64) - * @sd_share: per-LLC shared data - * - * Returns a snapshot of idle CPUs within this LLC, masked by - * llc_members (valid CPUs) and @affinity (task placement). - * - * bitmap mode (default): single atomic64_read (MOV on x86). - * flag array mode: stack-snapshot + multiply-and-shift aggregation. - */ -static __always_inline u64 poc_idle_cpu_mask(u64 affinity, - struct sched_domain_shared *sd_share) -{ - u64 cpus; - - if (static_branch_unlikely(&sched_poc_lockless_bitmap)) - cpus = poc_flags_to_u64(sd_share->poc_idle_cpus); - else - cpus = (u64)atomic64_read(&sd_share->poc_idle_cpus_mask); - - return cpus & sd_share->poc_llc_members & affinity; -} - -#ifdef CONFIG_SCHED_SMT -/* - * poc_idle_core_mask - Get idle core bitmask - * @cpu_mask: snapshot of idle CPUs (already masked by llc_members & affinity) - * @sd_share: per-LLC shared data - * - * Returns a bitmask with bits set at core representative positions - * (lowest-numbered sibling) for cores where ALL SMT siblings are idle. - * - * Three-tier derivation: - * - * Tier 1 (consecutive 2-way SMT): 3 register ops with compile-time - * constants — AND, SHR 1, AND 0x5555... No memory loads. - * - * Tier 2 (uniform stride-N 2-way SMT): 3 register ops with - * precomputed per-LLC shift and primary mask — AND, SHR N, AND. - * Two extra loads (poc_smt_shift, poc_primary_mask) from sd_share, - * but no write-path overhead. - * - * Tier 3 (exotic: >2-way SMT or non-uniform topology): reads the - * separately-maintained poc_idle_cores_mask atomic64_t. Write path - * maintains this bitmap on every idle transition. - */ -static __always_inline u64 poc_idle_core_mask(u64 cpu_mask, - struct sched_domain_shared *sd_share) -{ - /* Tier 1: consecutive — constants only, zero loads */ - if (static_branch_likely(&sched_poc_smt_consecutive)) - return cpu_mask & (cpu_mask >> 1) & 0x5555555555555555ULL; - - /* Tier 2: uniform stride-N — precomputed shift + mask */ - if (static_branch_likely(&sched_poc_smt_uniform)) - return cpu_mask & (cpu_mask >> sd_share->poc_smt_shift) - & sd_share->poc_primary_mask; - - /* Tier 3: exotic — bitmap or flag array based on mode */ - if (static_branch_unlikely(&sched_poc_lockless_bitmap)) - return poc_flags_to_u64(sd_share->poc_idle_cores) & cpu_mask; - - return (u64)atomic64_read(&sd_share->poc_idle_cores_mask) & cpu_mask; -} -#endif /* CONFIG_SCHED_SMT */ - -/* - * __set_cpu_idle_state_poc - Update idle state in atomic64_t bitmap - * @cpu: CPU number - * @state: 0=busy, 1=idle - * - * Updates the atomic64_t cpus bitmap via atomic64_or/andnot (LOCK'd on x86). - * - * On uniform 2-way SMT (Tier 1 & 2: consecutive or stride-N), only - * the cpus state is updated; core idle state is derived at read time - * via bit-parallel operations. - * - * On exotic SMT (Tier 3: >2-way or non-uniform), also maintains the - * separate cores state (bitmap or flag array) for O(1) read-time lookup. - * - * Only one representation is maintained at a time (single-write), - * selected by sched_poc_lockless_bitmap. - * - * Caller (inline wrapper in sched.h) ensures poc_selector_active is on - * and sched_asym_cpucap_active() is false before calling here. - */ -void __set_cpu_idle_state_poc(int cpu, int state) -{ - struct rq *rq = cpu_rq(cpu); - if (!static_branch_unlikely(&sched_poc_lockless_bitmap) && - !state && READ_ONCE(rq->poc_idle_committed)) - return; - - guard(rcu)(); - struct sched_domain_shared *sd_share = - rcu_dereference(per_cpu(sd_llc_shared, cpu)); - if (!sd_share || !sd_share->poc_fast_eligible) - return; - - int bit = cpu - sd_share->poc_cpu_base; - u64 bit_mask = 1ULL << bit; - - if (static_branch_unlikely(&sched_poc_lockless_bitmap)) { - WRITE_ONCE(sd_share->poc_idle_cpus[bit], state > 0 ? 1 : 0); - } else if (state > 0) { - /* Entering idle: clear any stale committed flag */ - WRITE_ONCE(rq->poc_idle_committed, 0); - atomic64_or(bit_mask, &sd_share->poc_idle_cpus_mask); - } else { - /* - * Exiting idle: if a waker already committed (cleared the - * bitmap bit), skip the redundant atomic on the shared - * cacheline. The flag lives in rq's first cacheline — - * same line the waker already dirtied via ttwu_pending. - */ - atomic64_andnot(bit_mask, &sd_share->poc_idle_cpus_mask); - WRITE_ONCE(rq->poc_idle_committed, 1); - } - -#ifdef CONFIG_SCHED_SMT - if (sched_smt_active()) { - /* Tier 1 & 2: read-time derivation, no write-path cost */ - if (static_branch_likely(&sched_poc_smt_uniform)) - return; - /* - * Tier 3 (exotic SMT): maintain separate cores state. - * Check whether all SMT siblings are idle. - */ - u64 smt = sd_share->poc_smt_mask[bit]; - u64 core_bitmask = smt & (-smt); /* core representative */ - int core_bit = __builtin_ctzll(core_bitmask); - bool core_idle; - - if (static_branch_unlikely(&sched_poc_lockless_bitmap)) { - /* - * Flag array mode: check siblings via WRITE_ONCE-stored - * flags. smp_wmb() ensures our store to poc_idle_cpus[] - * is visible before we read sibling flags. - * On x86 TSO: compiler barrier only (~0 cyc). - * On ARM64: dmb ishst. - */ - smp_wmb(); - u64 tmp = smt; - - core_idle = state > 0; - while (core_idle && tmp) { - int s = __builtin_ctzll(tmp); - - if (!READ_ONCE(sd_share->poc_idle_cpus[s])) - core_idle = false; - tmp &= tmp - 1; - } - WRITE_ONCE(sd_share->poc_idle_cores[core_bit], - core_idle ? 1 : 0); - } else { - /* - * smp_mb__after_atomic() ensures our atomic store is - * visible before we read sibling bits. On x86 TSO this - * is a compiler barrier (~0 cyc); on ARM64: dmb ish. - */ - smp_mb__after_atomic(); - u64 cpus = (u64)atomic64_read(&sd_share->poc_idle_cpus_mask); - core_idle = (cpus & smt) == smt; - u64 cores = (u64)atomic64_read(&sd_share->poc_idle_cores_mask); - - if (core_idle) { - if (!(cores & core_bitmask)) - atomic64_or(core_bitmask, - &sd_share->poc_idle_cores_mask); - } else { - if (cores & core_bitmask) - atomic64_andnot(core_bitmask, - &sd_share->poc_idle_cores_mask); - } - } - } -#endif /* CONFIG_SCHED_SMT */ -} - -/************************************************************** - * Idle CPU selection helpers: - */ - -/* Test whether a single CPU is idle in a POC bitmap snapshot. - * Assumes cpu_mask is in scope — works in any function with that variable. */ -#define POC_IDLE_CPU(bit) (cpu_mask & (1ULL << (bit))) -/* Scope-free validity checks — usable in any function. */ -#define POC_CPU_VALID(cpu) ((cpu) >= 0) -#define POC_CPU_IN_LLC(bit) ((unsigned int)(bit) < 64) - -/* - * poc_select_rr_improved - Improved round-robin idle CPU selection - * @base: poc_cpu_base (smallest CPU ID in this LLC) - * @mask: idle bitmask (snapshot, caller guarantees non-zero) - * @counter: per-CPU round-robin counter value - * - * Improved RR with two techniques: - * 1. Case-split by total: - * total=1: direct ctz - * total=2: interleave by counter LSB (guarantees non-repeat), - * single CTZ via cmov-selected source mask - * total>=3: golden-ratio scramble + Lemire fastrange - * 2. Golden-ratio scrambling (counter * 0x9E3779B9) mapped via - * Lemire fastrange for pseudo-random uniform distribution. - * - * eager_commit (unconditional) already prevents burst wake-ups from - * re-selecting the same CPU by clearing the bitmap bit at selection - * time, so no previous-pick exclusion state is needed here. - * - * Returns: selected CPU number. - */ -static __always_inline int poc_select_rr_improved( - int base, u64 mask, unsigned int counter) -{ - int total = hweight64(mask); - - if (total <= 2) { - /* - * Pick the lower or upper set bit via counter LSB if total == 2. - * Select the mask first (cmov), then one CTZ — halves the - * cost on archs where CTZ64 is a SW fallback (De Bruijn). - */ - if ((total == 2) && (counter & 1)) - mask &= mask - 1; - - return base + POC_CTZ64(mask); - } - - /* total >= 3: golden-ratio scramble + Lemire fastrange */ - { - u32 scrambled = POC_SCRAMBLE(counter); - int pick = POC_FASTRANGE(scrambled, total); - - return base + POC_PTSELECT(mask, pick); - } -} - -/* - * poc_select_rr - Round-robin idle CPU selection from a single-word mask - * @base: poc_cpu_base (smallest CPU ID in this LLC) - * @mask: idle bitmask (snapshot) - * @counter: per-CPU round-robin counter value - * - * Division-free perfect round-robin via FASTRANGE16 + PTSELECT. - * Consecutive calls on the same CPU never repeat an idle CPU - * until all candidates have been visited. - * Caller must ensure at least one bit is set in mask. - * Returns: selected CPU number. - */ -static __always_inline int poc_select_rr(int base, u64 mask, unsigned int counter) -{ - if (static_branch_likely(&sched_poc_rr_improved)) - return poc_select_rr_improved(base, mask, counter); - - /* Current strategy: poc_rr_step[] table (perfect RR), unchanged */ - { - int total = hweight64(mask); - u16 phase = (u16)(counter * (u32)poc_rr_step[total - 1]); - int pick = POC_FIXED_MOD16(phase, total); - - return POC_PTSELECT(mask, pick) + base; - } -} - -/* - * poc_cluster_search - Search for an idle CPU within the target's L2 cluster - * @base: poc_cpu_base (smallest CPU ID in this LLC) - * @tgt_bit: target CPU's POC-relative bit position - * @sd_share: per-LLC shared data containing cluster geometry - * @mask: snapshot of idle bitmask (cores or cpus, caller decides) - * - * Uses pre-computed cluster mask for O(1) lookup via CTZ. - * Returns: idle CPU number if found within cluster, -1 otherwise. - */ -static __always_inline int poc_cluster_search(int base, int tgt_bit, - struct sched_domain_shared *sd_share, u64 mask) -{ - u64 cls_idle = mask & sd_share->poc_cluster_mask[tgt_bit]; - - if (!cls_idle) - return -1; - - if (static_branch_likely(&sched_poc_rr_improved)) { - /* Improved path: inc counter here so LV3 fallback sees fresh value */ - unsigned int counter = __this_cpu_inc_return(poc_rr_counter); - return poc_select_rr_improved(base, cls_idle, counter); - } - - /* Current strategy: ctz lowest-bit (no RR), unchanged */ - return base + POC_CTZ64(cls_idle); -} - -#ifdef CONFIG_SCHED_SMT -/* - * poc_smt_sibling_mask - Get SMT sibling bitmask for a given CPU - * @bit: POC-relative bit position - * @sd_share: per-LLC shared data - * - * Three-tier computation matching poc_idle_core_mask(): - * - * Tier 1 (consecutive): 3ULL << (bit & ~1) — shift only, zero loads. - * - * Tier 2 (uniform stride-N): determine sibling via poc_smt_shift - * and poc_primary_mask. Avoids poc_smt_mask[] array lookup. - * - * Tier 3 (exotic): loads from pre-computed poc_smt_mask[] table. - */ -static __always_inline u64 poc_smt_sibling_mask(int bit, - struct sched_domain_shared *sd_share) -{ - if (static_branch_likely(&sched_poc_smt_consecutive)) - return 3ULL << (bit & ~1); - - if (static_branch_likely(&sched_poc_smt_uniform)) { - u8 shift = sd_share->poc_smt_shift; - int sib = (sd_share->poc_primary_mask & (1ULL << bit)) - ? bit + shift : bit - shift; - if (unlikely(sib < 0 || sib >= 64)) - return 1ULL << bit; - return (1ULL << bit) | (1ULL << sib); - } - - return sd_share->poc_smt_mask[bit]; -} - -/* - * poc_find_idle_smt_sibling - Find an idle CPU among target and its SMT siblings - * @base: poc_cpu_base (smallest CPU ID in this LLC) - * @tgt_bit: target CPU's POC-relative bit position - * @cpu_mask: snapshot of idle CPU bitmask - * @smt_mask: pre-computed SMT sibling mask for target (includes self) - * - * Searches target itself and its SMT siblings for an idle CPU. - * Target is checked first for cache locality. - * Returns: idle CPU number if found, -1 otherwise - */ -static __always_inline int poc_find_idle_smt_sibling( - int base, int tgt_bit, u64 cpu_mask, u64 smt_mask) -{ - /* Check target first for cache locality */ - if (POC_IDLE_CPU(tgt_bit)) - return base + tgt_bit; - - u64 idle_sibs = cpu_mask & smt_mask; - - if (idle_sibs) - return base + POC_CTZ64(idle_sibs); - - return -1; -} -/* - * poc_try_idle_smt - Find an idle CPU among a CPU and its SMT siblings - * @base: poc_cpu_base (smallest CPU ID in this LLC) - * @cpu: the CPU to check (and its SMT siblings) - * @cpu_mask: snapshot of idle CPU bitmask - * @sd_share: per-LLC shared data - * - * Checks if the given CPU or any of its SMT siblings is idle. - * Caller is responsible for poc_count() and poc_commit_selection(). - * Returns: idle CPU number if found, -1 otherwise - */ -static __always_inline int poc_try_idle_smt(int base, int cpu, - u64 cpu_mask, struct sched_domain_shared *sd_share) -{ - int bit = cpu - base; - - if (sd_share->poc_llc_members & (1ULL << bit)) { - int smt_cpu = poc_find_idle_smt_sibling(base, bit, - cpu_mask, poc_smt_sibling_mask(bit, sd_share)); - if (POC_CPU_VALID(smt_cpu)) - return smt_cpu; - } - return -1; -} - -#endif /* CONFIG_SCHED_SMT */ - -/* - * poc_commit_selection - Atomically clear selected CPU from idle bitmap - * @cpu: the CPU number selected by POC - * @sd_share: per-LLC shared data - * - * Clears the selected CPU's bit in poc_idle_cpus_mask at selection - * time to close the race window where multiple waker CPUs read the - * same stale bitmap and select the same idle CPU. The do_idle() - * exit path performs an idempotent clear as a safety net for - * non-POC wakeups; poc_idle_committed gates that path so the atomic - * fires at most once per selection. - */ -static __always_inline void poc_commit_selection(int cpu, - struct sched_domain_shared *sd_share) -{ - if (cpu_rq(cpu)->nr_running <= 2) { - int bit = cpu - sd_share->poc_cpu_base; - - if (static_branch_unlikely(&sched_poc_lockless_bitmap)) { - WRITE_ONCE(sd_share->poc_idle_cpus[bit], 0); - smp_wmb(); - } else { - atomic64_andnot(1ULL << bit, &sd_share->poc_idle_cpus_mask); - smp_mb__after_atomic(); - /* Mark committed so target skips redundant andnot on wakeup */ - WRITE_ONCE(cpu_rq(cpu)->poc_idle_committed, 1); - } - } -} - -/* - * POC_IDLE_CORE - Test whether a CPU's core is fully idle. - * POC_IDLE_SMT - Find an idle CPU among @cpu and its SMT siblings. - * - * POC_RETURN - Record hit counter, clear bitmap, return selected CPU. - * POC_RETURN_IF - Same, but only if @cpu >= 0 (used after POC_IDLE_SMT). - * - * These assume core_mask, base, sd_share are in scope - * (only used inside select_idle_cpu_poc). - */ -#define POC_IDLE_CORE(bit) (core_mask & poc_smt_sibling_mask((bit), sd_share)) -#define POC_IDLE_SMT(cpu) poc_try_idle_smt(base, (cpu), cpu_mask, sd_share) - -#define POC_RETURN(cpu, level) do { \ - poc_count(level); \ - poc_commit_selection(cpu, sd_share); \ - return cpu; \ -} while (0) - -#define POC_RETURN_IF(cpu, level) do { \ - if ((cpu) >= 0) \ - POC_RETURN(cpu, level); \ -} while (0) - -/************************************************************** - * Fast path dispatcher: - */ - -/* - * select_idle_cpu_poc - Fast idle CPU selector (atomic64 bitmap path) - * @target: CPU chosen by wake_affine (Level 1 preferred CPU; - * search origin for L2/L3/L5/L6) - * @prev: task's previous CPU (Level 4 cache locality preference) - * @recent: task's recent_used_cpu (-1 if none; pre-filtered by caller) - * @sync: 1 if synchronous wakeup (Level 4s: waker yields CPU) - * @sd_share: per-LLC shared data (caller provides; never NULL) - * @allowed: task's cpumask (p->cpus_ptr) for affinity filtering - * - * Two operating modes (sysctl kernel.sched_poc_smt_fallback): - * - * smt_fallback=0 (default): POC handles all idle CPU - * selection itself, including SMT siblings. Prioritizes - * prev's SMT sibling for cache locality. Uses CFS's - * nr_idle_scan (SIS_UTIL) to gate Level 5/6 under overload. - * - * smt_fallback=1: Bails out to CFS when has_idle_cores is - * false. CFS handles SMT sibling selection via - * select_idle_smt(prev) and nr_idle_scan-limited - * select_idle_cpu(). - * - * Selection levels: - * - * Level 0: Saturation check -- no idle CPUs → return -1 - * (smt_fallback: also when has_idle_cores == false) - * Level 1r: Recent's core is fully idle → return recent (!early_select) - * Level 1s: Target CPU idle in bitmap → return target (L1/TLB affinity) - * Level 1t: Target CPU's core is fully idle → return target - * Level 1p: Prev's core is fully idle → return prev (prev != target) - * --- core_mask != 0: search idle-core bitmap --- - * Level 2: Idle core in L2 cluster (CTZ) - * Level 3: Idle core across LLC (RR PTSELECT) - * --- core_mask == 0: search idle-CPU bitmap --- - * Level 4s: sync + target CPU idle (waker frees core) - * Level 4p: Prev's SMT sibling (cache locality) - * Level 4t: Target's SMT sibling - * Level 4r: Recent's SMT sibling (warm cache, always) - * [SIS_UTIL gate: nr_idle_scan == 0 → return -2] - * Level 5: Idle CPU in L2 cluster (CTZ) - * Level 6: Idle CPU across LLC (RR PTSELECT) - * - * Non-SMT: Level 1r → 1t → 1p → Level 2 → Level 3 (core = CPU). - * - * Returns: idle CPU number if found, -1 if not found (CFS may retry), - * -2 if SIS_UTIL overload (caller should skip CFS) - */ -static __always_inline int select_idle_cpu_poc(int target, int prev, - int recent, int sync, - struct sched_domain_shared *sd_share, - const struct cpumask *allowed) -{ - int base = sd_share->poc_cpu_base; - int rct_bit = recent - base; - int tgt_bit = target - base; - int prv_bit = prev - base; -#ifdef CONFIG_SCHED_SMT - u64 core_mask __maybe_unused; -#endif - u64 affinity; - u64 cpu_mask; - int level_offset = 0; - -#ifdef CONFIG_SCHED_SMT - /* SMT fallback: bail to CFS for SMT sibling selection */ - if (sched_smt_active() && - static_branch_unlikely(&sched_poc_smt_fallback) && - !READ_ONCE(sd_share->has_idle_cores)) - return -1; -#endif - - if (static_branch_unlikely(&sched_poc_lockless_bitmap)) - prefetch(sd_share->poc_idle_cpus); - else - prefetch(&sd_share->poc_idle_cpus_mask); -#ifdef CONFIG_SCHED_SMT - if (sched_smt_active()) { - if (!static_branch_likely(&sched_poc_smt_uniform)) { - if (static_branch_unlikely(&sched_poc_lockless_bitmap)) - prefetch(sd_share->poc_idle_cores); - else - prefetch(&sd_share->poc_idle_cores_mask); - if (POC_CPU_VALID(recent)) - prefetch(&sd_share->poc_smt_mask[rct_bit]); - prefetch(&sd_share->poc_smt_mask[tgt_bit]); - prefetch(&sd_share->poc_smt_mask[prv_bit]); - } - } -#endif - if (static_branch_likely(&sched_cluster_active)) - prefetch(&sd_share->poc_cluster_mask[tgt_bit]); - - affinity = poc_cpumask_to_u64(allowed, sd_share); - cpu_mask = poc_idle_cpu_mask(affinity, sd_share); - - /* Level 0: Saturation — no idle CPU */ - if (!cpu_mask) - return -1; - -#ifdef CONFIG_SCHED_SMT - if (sched_smt_active()) { - core_mask = poc_idle_core_mask(cpu_mask, sd_share); - - /* Level 1r: recent's core is idle (warm cache) */ - if (!static_branch_likely(&sched_poc_early_select) && - core_mask && POC_CPU_IN_LLC(rct_bit) && POC_IDLE_CORE(rct_bit)) - POC_RETURN(recent, POC_LV1R); - - /* Level 1s: target CPU sticky — L1/TLB affinity shortcut */ - if (static_branch_unlikely(&sched_poc_target_sticky) && POC_IDLE_CPU(tgt_bit)) - POC_RETURN(target, POC_LV1S); - - if (core_mask) { - /* - * Idle core path: T → P order. - * Target first — wake_affine chose it for data sharing - * and the full core is free. - */ - - /* Level 1t: target CPU's core is idle → return it */ - if (!static_branch_likely(&sched_poc_early_select) && - POC_IDLE_CORE(tgt_bit)) - POC_RETURN(target, POC_LV1T); - - /* Level 1p: prev's core is idle (task's L1/L2 warm) */ - if (prev != target && POC_CPU_IN_LLC(prv_bit) && POC_IDLE_CORE(prv_bit)) - POC_RETURN(prev, POC_LV1P); - - cpu_mask = core_mask; - } else { - int cpu; - - /* Level 4s: sync wakeup + target CPU idle → - * waker will sleep imminently, freeing the core */ - if (sync && POC_IDLE_CPU(tgt_bit)) - POC_RETURN(target, POC_LV4S); - - /* - * No-idle-core path: P → T → R order. - * Target itself was already tried at Level 1s/4s; - * prioritize task's own cache (prev, recent) over - * waker locality (target's sibling). - */ - - /* Level 4p: prev's SMT sibling (cache locality) */ - if (prev != target && POC_CPU_IN_LLC(prv_bit)) { - cpu = POC_IDLE_SMT(prev); - POC_RETURN_IF(cpu, POC_LV4P); - } - - /* Level 4t: target's SMT sibling */ - cpu = POC_IDLE_SMT(target); - POC_RETURN_IF(cpu, POC_LV4T); - - /* Level 4r: recent's SMT sibling (warm cache) */ - if (POC_CPU_IN_LLC(rct_bit)) { - cpu = POC_IDLE_SMT(recent); - POC_RETURN_IF(cpu, POC_LV4R); - } - - /* SIS_UTIL overload gate for Level 5/6 */ - if (!static_branch_likely(&sched_poc_greedy_search) && - sched_feat(SIS_UTIL) && !READ_ONCE(sd_share->nr_idle_scan)) - return -2; - - level_offset = POC_SMT_LEVEL_OFFSET; - } - } - else -#endif - { - /* Level 1r: recent CPU is idle (non-SMT) */ - if (!static_branch_likely(&sched_poc_early_select) && - POC_CPU_IN_LLC(rct_bit) && POC_IDLE_CPU(rct_bit)) - POC_RETURN(recent, POC_LV1R); - /* Level 1t: target CPU is idle → return (non-SMT) */ - if (POC_IDLE_CPU(tgt_bit)) - POC_RETURN(target, POC_LV1T); - /* Level 1p: prev CPU is idle (non-SMT) */ - if (prev != target && POC_CPU_IN_LLC(prv_bit) && POC_IDLE_CPU(prv_bit)) - POC_RETURN(prev, POC_LV1P); - } - - if (static_branch_likely(&sched_poc_packed)) { - /* - * Level 2+3 / 5+6: packed priority search (≤32 CPUs/LLC) - * - * Packs cluster candidates (high priority) into lower 32 bits - * and all LLC candidates (low priority) into upper 32 bits. - * A single TZCNT resolves the highest-priority idle CPU. - * Level discrimination: (raw >> 5) yields 0 (cluster) or 1 (LLC). - * - * rr_improved=ON: rotation amount via golden-ratio scramble. - * rr_improved=OFF: rotation amount is (counter & 31). - */ - unsigned int counter = __this_cpu_inc_return(poc_rr_counter); - int rot; - u32 cls = 0; - u32 all; - u64 packed; - int raw, bit; - - if (static_branch_likely(&sched_poc_rr_improved)) - rot = (int)(POC_SCRAMBLE(counter) >> 27); - else - rot = counter & 31; - - if (static_branch_likely(&sched_cluster_active) && - sd_share->poc_cluster_valid) - cls = ror32((u32)(cpu_mask & - sd_share->poc_cluster_mask[tgt_bit]), rot); - - all = ror32((u32)cpu_mask, rot); - packed = (u64)cls | ((u64)all << 32); - - raw = POC_CTZ64(packed); - bit = ((raw & 31) + rot) & 31; - - POC_RETURN(base + bit, POC_LV2 + (raw >> 5) + level_offset); - } else { - /* Level 2/5: idle core/cpu in target's L2 cluster */ - if (static_branch_likely(&sched_cluster_active) - && sd_share->poc_cluster_valid) { - int cpu = poc_cluster_search( - base, tgt_bit, sd_share, cpu_mask); - if (POC_CPU_VALID(cpu)) - POC_RETURN(cpu, POC_LV2 + level_offset); - } - - /* Level 3/6: idle core/cpu across LLC via RR */ - { - unsigned int counter = __this_cpu_inc_return(poc_rr_counter); - int rr_cpu = poc_select_rr(base, cpu_mask, counter); - POC_RETURN(rr_cpu, POC_LV3 + level_offset); - } - } -} - -/************************************************************** - * Topology setup: - * - * poc_sd_shared_init - Initialize POC fields in sched_domain_shared - * @sd: the LLC-sharing sched_domain whose ->shared was just attached - * @sd_id: first CPU of @sd's span (used as poc_cpu_base) - * - * Called from build_sched_domains() right after sd->shared is attached - * for an SD_SHARE_LLC domain. Computes per-LLC bit-base and pre-builds - * member/SMT/cluster masks for O(1) lookup at wakeup time. - */ -void poc_sd_shared_init(struct sched_domain *sd, int sd_id) -{ - struct cpumask *sd_span = sched_domain_span(sd); - int range = cpumask_last(sd_span) - sd_id + 1; - - sd->shared->poc_cpu_base = sd_id; - sd->shared->poc_affinity_shift = sd_id & 63; - - if (range <= 64) { - sd->shared->poc_fast_eligible = true; - /* - * Disable aligned optimization if this LLC's base CPU - * is not 64-aligned (e.g., Threadripper CCDs). - */ - if (sd_id & 63) - static_branch_disable_cpuslocked(&sched_poc_aligned); - /* - * Disable packed priority search if this LLC - * has more than 32 CPUs. - */ - if (range > 32) - static_branch_disable_cpuslocked(&sched_poc_packed); - } else { - sd->shared->poc_fast_eligible = false; - static_branch_disable_cpuslocked(&sched_poc_packed); - } - memset(sd->shared->poc_idle_cpus, 0, - sizeof(sd->shared->poc_idle_cpus)); - atomic64_set(&sd->shared->poc_idle_cpus_mask, 0); -#ifdef CONFIG_SCHED_SMT - memset(sd->shared->poc_idle_cores, 0, - sizeof(sd->shared->poc_idle_cores)); - atomic64_set(&sd->shared->poc_idle_cores_mask, 0); -#endif - - /* Build LLC member bitmask for reader-side aggregation */ - { - u64 members = 0; - int cpu_iter; - - for_each_cpu(cpu_iter, sd_span) { - int bit = cpu_iter - sd_id; - - if ((unsigned int)bit < 64) - members |= 1ULL << bit; - } - sd->shared->poc_llc_members = members; - } - -#ifdef CONFIG_SCHED_SMT - /* - * Pre-compute SMT sibling masks for Level 4. - * Each entry contains a bitmask of SMT siblings (including self) - * for O(1) lookup via CTZ during wakeup. - */ - memset(sd->shared->poc_smt_mask, 0, - sizeof(sd->shared->poc_smt_mask)); - if (sd->shared->poc_fast_eligible) { - int cpu_iter; - - for_each_cpu(cpu_iter, sd_span) { - int bit = cpu_iter - sd_id; - int sibling; - u64 mask = 0; - - for_each_cpu(sibling, cpu_smt_mask(cpu_iter)) { - int sib_bit; - - sib_bit = sibling - sd_id; - if (sib_bit >= 0 && sib_bit < 64) - mask |= 1ULL << sib_bit; - } - if (bit >= 0 && bit < 64) - sd->shared->poc_smt_mask[bit] = mask; - } - } - - /* - * Detect SMT topology and classify for poc_idle_core_mask(): - * - * Tier 1 (consecutive): uniform 2-way SMT, siblings at - * consecutive bit positions (e.g., 0,1 / 2,3). - * Uses compile-time constants: shift=1, mask=0x5555... - * - * Tier 2 (uniform stride-N): uniform 2-way SMT with - * constant stride between siblings (e.g., Intel Xeon - * stride-8: CPU 0,8 / 1,9 / ...). Uses precomputed - * poc_smt_shift and poc_primary_mask for read-time - * derivation without write-path overhead. - * - * Tier 3 (exotic): >2-way SMT, non-uniform topology, - * or mixed SMT ways. Falls back to write-time - * maintenance of poc_idle_cores_mask atomic64_t. - * - * On pure non-SMT systems, the key values are irrelevant - * because sched_smt_active() gates all SMT paths. - */ - sd->shared->poc_smt_shift = 1; - sd->shared->poc_primary_mask = 0; - - if (sd->shared->poc_fast_eligible) { - int cpu_iter; - bool all_2way = true; - bool all_consecutive = true; - int uniform_stride = -1; - u64 primary_mask = 0; - - for_each_cpu(cpu_iter, sd_span) { - int bit = cpu_iter - sd_id; - - if (bit < 0 || bit >= 64) - continue; - u64 mask = sd->shared->poc_smt_mask[bit]; - int ways = hweight64(mask); - - if (ways != 2) { - all_2way = false; - all_consecutive = false; - break; - } - - int lo = __ffs(mask); - int hi = __fls(mask); - int stride = hi - lo; - - /* Track primary (lowest-numbered sibling) */ - primary_mask |= 1ULL << lo; - - /* Check consecutive: 0b11 at even position */ - if ((lo & 1) || mask != (3ULL << lo)) - all_consecutive = false; - - /* Check uniform stride */ - if (uniform_stride < 0) - uniform_stride = stride; - else if (stride != uniform_stride) - all_2way = false; - } - - if (!all_consecutive) - static_branch_disable_cpuslocked( - &sched_poc_smt_consecutive); - - if (all_2way && uniform_stride > 0) { - sd->shared->poc_smt_shift = (u8)uniform_stride; - sd->shared->poc_primary_mask = primary_mask; - } else { - static_branch_disable_cpuslocked( - &sched_poc_smt_consecutive); - static_branch_disable_cpuslocked( - &sched_poc_smt_uniform); - } - } -#endif /* CONFIG_SCHED_SMT */ - - memset(sd->shared->poc_cluster_mask, 0, - sizeof(sd->shared->poc_cluster_mask)); - - sd->shared->poc_cluster_valid = false; - -#ifdef CONFIG_SCHED_CLUSTER - /* - * Detect cluster (L2-sharing) topology for Level 2/5 - * cluster-local search in POC selector. - * - * Uses cpu_clustergroup_mask() which returns the L2 - * cache sharing mask on x86. Validates that all - * clusters are uniform (same size, power-of-2, and - * naturally aligned in POC bit space). - */ - if (sd->shared->poc_fast_eligible) { - const struct cpumask *cls_mask = cpu_clustergroup_mask(sd_id); - int cls_size = cpumask_weight(cls_mask); - int smt_size = cpumask_weight(cpu_smt_mask(sd_id)); - - if (cls_size > smt_size && is_power_of_2(cls_size)) { - bool valid = true; - int cpu_iter; - - for_each_cpu(cpu_iter, sd_span) { - const struct cpumask *m = - cpu_clustergroup_mask(cpu_iter); - int first = cpumask_first(m); - int rel = first - sd_id; - - if (cpumask_weight(m) != cls_size || - (rel & (cls_size - 1)) != 0) { - valid = false; - break; - } - } - if (valid) { - sd->shared->poc_cluster_valid = true; - - /* - * Pre-compute cluster masks for O(1) lookup. - * Each entry contains a bitmask of cluster - * members (excluding self) for fast search. - */ - for_each_cpu(cpu_iter, sd_span) { - const struct cpumask *m = - cpu_clustergroup_mask(cpu_iter); - int bit = cpu_iter - sd_id; - int member; - u64 cmask = 0; - - for_each_cpu(member, m) { - int mbit; - - if (member == cpu_iter) - continue; - mbit = member - sd_id; - if (mbit >= 0 && mbit < 64) - cmask |= 1ULL << mbit; - } - if (bit >= 0 && bit < 64) - sd->shared->poc_cluster_mask[bit] = cmask; - } - } - } - } -#endif /* CONFIG_SCHED_CLUSTER */ -} - -/************************************************************** - * Sysctl interface and initialization: - */ - -#if defined(CONFIG_SYSCTL) || defined(CONFIG_SCHED_CLASS_EXT) -/* - * poc_resync_idle_state - Resync POC idle bitmaps after re-enable - * - * When POC is re-enabled after a period of being disabled, - * the idle bitmaps may be stale. Walk all online CPUs and push - * the current idle state into poc_idle_cpus_mask (and poc_idle_cores_mask - * on non-consecutive SMT). - * - * Must be called AFTER static_branch_enable() so that concurrent - * idle transitions are also updating the flags. - * Caller must hold cpus_read_lock(). - */ -static void poc_resync_idle_state(void) -{ - int cpu; - - for_each_online_cpu(cpu) { - WRITE_ONCE(cpu_rq(cpu)->poc_idle_committed, 0); - __set_cpu_idle_state_poc(cpu, idle_cpu(cpu)); - } -} - -/* - * poc_reevaluate_active - Recompute poc_selector_active from inputs - * - * poc_selector_active = sched_poc_selector && !poc_selector_skip - * - * On transition to active: enable static key, then resync idle bitmaps. - * On transition to inactive: disable static key. - * Caller must hold cpus_read_lock(). - */ -static void poc_reevaluate_active(void) -{ - bool want = sched_poc_selector && !poc_selector_skip; - bool now = static_branch_likely(&poc_selector_active); - - if (want == now) - return; - - if (want) { - static_branch_enable_cpuslocked(&poc_selector_active); - poc_resync_idle_state(); - } else { - static_branch_disable_cpuslocked(&poc_selector_active); - } -} -#endif /* CONFIG_SYSCTL || CONFIG_SCHED_CLASS_EXT */ - -#ifdef CONFIG_SCHED_CLASS_EXT -/* - * poc_notify_scx - Called by sched_ext on enable/disable transitions - * @scx_active: true when scx scheduler is being enabled - */ -void poc_notify_scx(bool scx_active) -{ - cpus_read_lock(); - poc_selector_skip = scx_active; - poc_reevaluate_active(); - cpus_read_unlock(); -} - -/* - * poc_skip_fallback_work - Workqueue item to re-enable POC after scx fallback. - * - * Scheduled by poc_check_skip_fallback() when an scx scheduler calls - * select_idle_sibling. Runs poc_reevaluate_active() outside the hot path - * to avoid updating the static key and resyncing bitmaps inline. - */ -static void poc_skip_fallback_fn(struct work_struct *work); -static DECLARE_WORK(poc_skip_fallback_work, poc_skip_fallback_fn); - -static void poc_skip_fallback_fn(struct work_struct *work) -{ - cpus_read_lock(); - poc_reevaluate_active(); - cpus_read_unlock(); -} - -/* - * poc_check_skip_fallback - Hot-path detection for scx calling select_idle_sibling - * - * While scx is active, poc_selector_skip=true suppresses idle bitmap updates - * in do_idle. Some scx schedulers still call select_idle_sibling; when that - * happens, flip poc_selector_skip back to false and schedule a workqueue item - * to re-enable poc_selector_active and resync stale bitmaps. - * - * WRITE_ONCE(false) is idempotent across concurrent callers; schedule_work() - * silently drops duplicate requests when the item is already queued. - */ -void poc_check_skip_fallback(void) -{ - if (!sched_poc_selector || !READ_ONCE(poc_selector_skip)) - return; - WRITE_ONCE(poc_selector_skip, false); - schedule_work(&poc_skip_fallback_work); -} -#endif - -#ifdef CONFIG_SYSCTL -static int sched_poc_sysctl_handler(const struct ctl_table *table, int write, - void *buffer, size_t *lenp, loff_t *ppos) -{ - unsigned int val = sched_poc_selector ? 1 : 0; - struct ctl_table tmp = { - .data = &val, - .maxlen = sizeof(val), - .extra1 = SYSCTL_ZERO, - .extra2 = SYSCTL_ONE, - }; - int ret = proc_douintvec_minmax(&tmp, write, buffer, lenp, ppos); - - if (!ret && write) { - cpus_read_lock(); - sched_poc_selector = !!val; - poc_reevaluate_active(); - cpus_read_unlock(); - } - return ret; -} - -static int sched_poc_smt_fallback_sysctl_handler(const struct ctl_table *table, - int write, void *buffer, - size_t *lenp, loff_t *ppos) -{ - unsigned int val = static_branch_unlikely(&sched_poc_smt_fallback) ? 1 : 0; - struct ctl_table tmp = { - .data = &val, - .maxlen = sizeof(val), - .extra1 = SYSCTL_ZERO, - .extra2 = SYSCTL_ONE, - }; - int ret = proc_douintvec_minmax(&tmp, write, buffer, lenp, ppos); - - if (!ret && write) { - if (val) - static_branch_enable(&sched_poc_smt_fallback); - else - static_branch_disable(&sched_poc_smt_fallback); - } - return ret; -} - -static int sched_poc_rr_improved_sysctl_handler(const struct ctl_table *table, - int write, void *buffer, - size_t *lenp, loff_t *ppos) -{ - unsigned int val = static_branch_likely(&sched_poc_rr_improved) ? 1 : 0; - struct ctl_table tmp = { - .data = &val, - .maxlen = sizeof(val), - .extra1 = SYSCTL_ZERO, - .extra2 = SYSCTL_ONE, - }; - int ret = proc_douintvec_minmax(&tmp, write, buffer, lenp, ppos); - - if (!ret && write) { - if (val) - static_branch_enable(&sched_poc_rr_improved); - else - static_branch_disable(&sched_poc_rr_improved); - } - return ret; -} - -static int sched_poc_target_sticky_sysctl_handler(const struct ctl_table *table, - int write, void *buffer, - size_t *lenp, loff_t *ppos) -{ - unsigned int val = static_branch_unlikely(&sched_poc_target_sticky) ? 1 : 0; - struct ctl_table tmp = { - .data = &val, - .maxlen = sizeof(val), - .extra1 = SYSCTL_ZERO, - .extra2 = SYSCTL_ONE, - }; - int ret = proc_douintvec_minmax(&tmp, write, buffer, lenp, ppos); - - if (!ret && write) { - if (val) - static_branch_enable(&sched_poc_target_sticky); - else - static_branch_disable(&sched_poc_target_sticky); - } - return ret; -} - -static int sched_poc_early_select_handler(const struct ctl_table *table, - int write, void *buffer, - size_t *lenp, loff_t *ppos) -{ - unsigned int val = static_branch_likely(&sched_poc_early_select) ? 1 : 0; - struct ctl_table tmp = { - .data = &val, - .maxlen = sizeof(val), - .extra1 = SYSCTL_ZERO, - .extra2 = SYSCTL_ONE, - }; - int ret = proc_douintvec_minmax(&tmp, write, buffer, lenp, ppos); - - if (!ret && write) { - if (val) - static_branch_enable(&sched_poc_early_select); - else - static_branch_disable(&sched_poc_early_select); - } - return ret; -} - -static int sched_poc_greedy_search_handler(const struct ctl_table *table, - int write, void *buffer, - size_t *lenp, loff_t *ppos) -{ - unsigned int val = static_branch_likely(&sched_poc_greedy_search) ? 1 : 0; - struct ctl_table tmp = { - .data = &val, - .maxlen = sizeof(val), - .extra1 = SYSCTL_ZERO, - .extra2 = SYSCTL_ONE, - }; - int ret = proc_douintvec_minmax(&tmp, write, buffer, lenp, ppos); - - if (!ret && write) { - if (val) - static_branch_enable(&sched_poc_greedy_search); - else - static_branch_disable(&sched_poc_greedy_search); - } - return ret; -} - -static int sched_poc_count_sysctl_handler(const struct ctl_table *table, - int write, void *buffer, - size_t *lenp, loff_t *ppos) -{ - unsigned int val = static_branch_unlikely(&sched_poc_count_enabled) ? 1 : 0; - struct ctl_table tmp = { - .data = &val, - .maxlen = sizeof(val), - .extra1 = SYSCTL_ZERO, - .extra2 = SYSCTL_ONE, - }; - int ret = proc_douintvec_minmax(&tmp, write, buffer, lenp, ppos); - - if (!ret && write) { - if (val) - static_branch_enable(&sched_poc_count_enabled); - else - static_branch_disable(&sched_poc_count_enabled); - } - return ret; -} - -static int sched_poc_lockless_bitmap_sysctl_handler(const struct ctl_table *table, - int write, void *buffer, - size_t *lenp, loff_t *ppos) -{ - unsigned int val = static_branch_unlikely(&sched_poc_lockless_bitmap) ? 1 : 0; - struct ctl_table tmp = { - .data = &val, - .maxlen = sizeof(val), - .extra1 = SYSCTL_ZERO, - .extra2 = SYSCTL_ONE, - }; - int ret = proc_douintvec_minmax(&tmp, write, buffer, lenp, ppos); - - if (!ret && write) { - cpus_read_lock(); - if (val) - static_branch_enable_cpuslocked(&sched_poc_lockless_bitmap); - else - static_branch_disable_cpuslocked(&sched_poc_lockless_bitmap); - /* - * Resync the newly-active representation so readers see - * consistent state immediately after the mode switch. - */ - poc_resync_idle_state(); - cpus_read_unlock(); - } - return ret; -} - -static struct ctl_table sched_poc_sysctls[] = { - { - .procname = "sched_poc_selector", - .data = NULL, - .maxlen = sizeof(unsigned int), - .mode = 0644, - .proc_handler = sched_poc_sysctl_handler, - }, - { - .procname = "sched_poc_smt_fallback", - .data = NULL, - .maxlen = sizeof(unsigned int), - .mode = 0644, - .proc_handler = sched_poc_smt_fallback_sysctl_handler, - }, - { - .procname = "sched_poc_rr_improved", - .data = NULL, - .maxlen = sizeof(unsigned int), - .mode = 0644, - .proc_handler = sched_poc_rr_improved_sysctl_handler, - }, - { - .procname = "sched_poc_target_sticky", - .data = NULL, - .maxlen = sizeof(unsigned int), - .mode = 0644, - .proc_handler = sched_poc_target_sticky_sysctl_handler, - }, - { - .procname = "sched_poc_early_select", - .data = NULL, - .maxlen = sizeof(unsigned int), - .mode = 0644, - .proc_handler = sched_poc_early_select_handler, - }, - { - .procname = "sched_poc_greedy_search", - .data = NULL, - .maxlen = sizeof(unsigned int), - .mode = 0644, - .proc_handler = sched_poc_greedy_search_handler, - }, - { - .procname = "sched_poc_count", - .data = NULL, - .maxlen = sizeof(unsigned int), - .mode = 0644, - .proc_handler = sched_poc_count_sysctl_handler, - }, - { - .procname = "sched_poc_lockless_bitmap", - .data = NULL, - .maxlen = sizeof(unsigned int), - .mode = 0644, - .proc_handler = sched_poc_lockless_bitmap_sysctl_handler, - }, -}; - -static int __init sched_poc_sysctl_init(void) -{ - printk(KERN_INFO "%s %s by %s [CTZ: %s, PTSelect: %s]\n", - SCHED_POC_SELECTOR_PROGNAME, SCHED_POC_SELECTOR_VERSION, - SCHED_POC_SELECTOR_AUTHOR, POC_CTZ64_NAME, POC_PTSELECT_NAME); - - register_sysctl_init("kernel", sched_poc_sysctls); - return 0; -} -late_initcall(sched_poc_sysctl_init); - -#endif /* CONFIG_SYSCTL */ - -/* - * Initialize per-CPU RR counters with CPU ID offset. - * Different starting values shift the FASTRANGE16 phase per CPU, - * reducing cross-CPU collision probability when multiple CPUs - * perform burst wakeups against the same idle bitmap snapshot. - */ -static int __init sched_poc_rr_init(void) -{ - int cpu; - - for_each_possible_cpu(cpu) - per_cpu(poc_rr_counter, cpu) = (u32)cpu; - return 0; -} -early_initcall(sched_poc_rr_init); - -/************************************************************** - * Status: sysfs interface (always available) - * - * Exported at /sys/kernel/poc_selector/status/ for runtime status queries. - * Reports whether POC is actually active (combining all conditions). - */ - -#ifdef CONFIG_SYSFS - -/* Root kobject shared with debug section */ -static struct kobject *kobj_poc_root; - -static bool poc_check_all_llc_eligible(void) -{ - int cpu; - - for_each_online_cpu(cpu) { - struct sched_domain_shared *sd_share; - - scoped_guard(rcu) { - sd_share = rcu_dereference(per_cpu(sd_llc_shared, cpu)); - if (sd_share && !sd_share->poc_fast_eligible) - return false; - } - } - return true; -} - -static ssize_t active_show(struct kobject *kobj, - struct kobj_attribute *attr, char *buf) -{ - bool active = static_branch_likely(&poc_selector_active) && - !sched_asym_cpucap_active() && - poc_check_all_llc_eligible(); - return sysfs_emit(buf, "%d\n", active ? 1 : 0); -} - -static ssize_t symmetric_cpucap_show(struct kobject *kobj, - struct kobj_attribute *attr, char *buf) -{ - return sysfs_emit(buf, "%d\n", sched_asym_cpucap_active() ? 0 : 1); -} - -static ssize_t all_llc_eligible_show(struct kobject *kobj, - struct kobj_attribute *attr, char *buf) -{ - return sysfs_emit(buf, "%d\n", poc_check_all_llc_eligible() ? 1 : 0); -} - -static ssize_t version_show(struct kobject *kobj, - struct kobj_attribute *attr, char *buf) -{ - return sysfs_emit(buf, "%s\n", SCHED_POC_SELECTOR_VERSION); -} - -static struct kobj_attribute poc_status_active_attr = __ATTR_RO(active); -static struct kobj_attribute poc_status_asym_attr = __ATTR_RO(symmetric_cpucap); -static struct kobj_attribute poc_status_eligible_attr = __ATTR_RO(all_llc_eligible); -static struct kobj_attribute poc_status_version_attr = __ATTR_RO(version); - -static struct attribute *poc_status_attrs[] = { - &poc_status_active_attr.attr, - &poc_status_asym_attr.attr, - &poc_status_eligible_attr.attr, - &poc_status_version_attr.attr, - NULL, -}; - -static const struct attribute_group poc_status_group = { - .name = "status", - .attrs = poc_status_attrs, -}; - -/* --- hw_accel: expose which hardware acceleration is in use --- */ - -#define DEFINE_POC_HW_ATTR(fname, namestr) \ -static ssize_t poc_hw_##fname##_show(struct kobject *kobj, \ - struct kobj_attribute *attr, char *buf) \ -{ \ - return sysfs_emit(buf, "%s\n", namestr); \ -} \ -static struct kobj_attribute poc_hw_attr_##fname = { \ - .attr = { .name = #fname, .mode = 0444 }, \ - .show = poc_hw_##fname##_show, \ -} - -DEFINE_POC_HW_ATTR(ctz, POC_CTZ64_NAME); -DEFINE_POC_HW_ATTR(ptselect, POC_PTSELECT_NAME); - -/* popcnt: x86 uses runtime alternatives, detect via boot_cpu_has */ -static ssize_t poc_hw_popcnt_show(struct kobject *kobj, - struct kobj_attribute *attr, char *buf) -{ -#if defined(__x86_64__) - return sysfs_emit(buf, "%s\n", - boot_cpu_has(X86_FEATURE_POPCNT) ? "HW (POPCNT)" : "SW"); -#elif defined(__aarch64__) - return sysfs_emit(buf, "HW (CNT)\n"); -#elif defined(__riscv) && defined(__riscv_zbb) - return sysfs_emit(buf, "HW (cpop)\n"); -#else - return sysfs_emit(buf, "SW\n"); -#endif -} - -static struct kobj_attribute poc_hw_attr_popcnt = { - .attr = { .name = "popcnt", .mode = 0444 }, - .show = poc_hw_popcnt_show, -}; - -static struct attribute *poc_hw_attrs[] = { - &poc_hw_attr_popcnt.attr, - &poc_hw_attr_ctz.attr, - &poc_hw_attr_ptselect.attr, - NULL, -}; - -static const struct attribute_group poc_hw_group = { - .name = "hw_accel", - .attrs = poc_hw_attrs, -}; - -/* --- count: per-level hit counters (sysctl kernel.sched_poc_count) --- */ - -static unsigned long poc_sum_level(enum poc_level lvl) -{ - unsigned long sum = 0; - int cpu; - - for_each_possible_cpu(cpu) - sum += per_cpu(poc_debug_cnt[lvl], cpu); - return sum; -} - -#define DEFINE_POC_COUNT_ATTR(fname, level) \ -static ssize_t poc_count_##fname##_show(struct kobject *kobj, \ - struct kobj_attribute *attr, char *buf) \ -{ \ - return sysfs_emit(buf, "%lu\n", poc_sum_level(level)); \ -} \ -static struct kobj_attribute poc_count_##fname##_attr = { \ - .attr = { .name = #fname, .mode = 0444 }, \ - .show = poc_count_##fname##_show, \ -} - -DEFINE_POC_COUNT_ATTR(l1s, POC_LV1S); -DEFINE_POC_COUNT_ATTR(l1t, POC_LV1T); -DEFINE_POC_COUNT_ATTR(l1p, POC_LV1P); -DEFINE_POC_COUNT_ATTR(l1r, POC_LV1R); -DEFINE_POC_COUNT_ATTR(l2, POC_LV2); -DEFINE_POC_COUNT_ATTR(l3, POC_LV3); -DEFINE_POC_COUNT_ATTR(l4s, POC_LV4S); -DEFINE_POC_COUNT_ATTR(l4p, POC_LV4P); -DEFINE_POC_COUNT_ATTR(l4r, POC_LV4R); -DEFINE_POC_COUNT_ATTR(l4t, POC_LV4T); -DEFINE_POC_COUNT_ATTR(l5, POC_LV5); -DEFINE_POC_COUNT_ATTR(l6, POC_LV6); -DEFINE_POC_COUNT_ATTR(fallback, POC_FALLBACK); - -static ssize_t poc_count_reset_store(struct kobject *kobj, - struct kobj_attribute *attr, - const char *buf, size_t count) -{ - int cpu; - - for_each_possible_cpu(cpu) - memset(per_cpu_ptr(poc_debug_cnt, cpu), 0, - sizeof(poc_debug_cnt)); - return count; -} - -static struct kobj_attribute poc_count_reset_attr = { - .attr = { .name = "reset", .mode = 0200 }, - .store = poc_count_reset_store, -}; - -static struct attribute *poc_count_attrs[] = { - &poc_count_l1s_attr.attr, - &poc_count_l1t_attr.attr, - &poc_count_l1p_attr.attr, - &poc_count_l1r_attr.attr, - &poc_count_l2_attr.attr, - &poc_count_l3_attr.attr, - &poc_count_l4s_attr.attr, - &poc_count_l4p_attr.attr, - &poc_count_l4r_attr.attr, - &poc_count_l4t_attr.attr, - &poc_count_l5_attr.attr, - &poc_count_l6_attr.attr, - &poc_count_fallback_attr.attr, - &poc_count_reset_attr.attr, - NULL, -}; - -static const struct attribute_group poc_count_group = { - .name = "count", - .attrs = poc_count_attrs, -}; - -static int __init sched_poc_status_init(void) -{ - int ret; - - kobj_poc_root = kobject_create_and_add("poc_selector", kernel_kobj); - if (!kobj_poc_root) - return -ENOMEM; - - ret = sysfs_create_group(kobj_poc_root, &poc_status_group); - if (ret) - goto err_status; - - ret = sysfs_create_group(kobj_poc_root, &poc_hw_group); - if (ret) - goto err_hw; - - ret = sysfs_create_group(kobj_poc_root, &poc_count_group); - if (ret) - goto err_selected; - - return 0; - -err_selected: - sysfs_remove_group(kobj_poc_root, &poc_hw_group); -err_hw: - sysfs_remove_group(kobj_poc_root, &poc_status_group); -err_status: - kobject_put(kobj_poc_root); - kobj_poc_root = NULL; - return ret; -} -late_initcall(sched_poc_status_init); - -#endif /* CONFIG_SYSFS */ -#endif /* CONFIG_SCHED_POC_SELECTOR */ diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h index e25c5a986..4664869d8 100644 --- a/kernel/sched/sched.h +++ b/kernel/sched/sched.h @@ -1180,10 +1180,6 @@ struct rq { call_single_data_t nohz_csd; #endif /* CONFIG_NO_HZ_COMMON */ -#ifdef CONFIG_SCHED_POC_SELECTOR - unsigned int poc_idle_committed; -#endif - #ifdef CONFIG_UCLAMP_TASK /* Utilization clamp values based on CPU's RUNNABLE tasks */ struct uclamp_rq uclamp[UCLAMP_CNT] ____cacheline_aligned; @@ -2378,119 +2374,6 @@ static inline struct task_group *task_group(struct task_struct *p) #endif /* !CONFIG_CGROUP_SCHED */ -#ifdef CONFIG_SCHED_POC_SELECTOR -extern struct static_key_true poc_selector_active; -#ifdef CONFIG_SCHED_CLASS_EXT -extern void poc_notify_scx(bool scx_active); -extern void poc_check_skip_fallback(void); -#else -static inline void poc_check_skip_fallback(void) {} -#endif -extern struct static_key_true sched_poc_aligned; -extern struct static_key_true sched_poc_smt_consecutive; -extern struct static_key_true sched_poc_smt_uniform; -extern struct static_key_false sched_poc_target_sticky; -extern struct static_key_true sched_poc_packed; -extern struct static_key_false sched_poc_lockless_bitmap; -extern void __set_cpu_idle_state_poc(int cpu, int state); -extern void poc_sd_shared_init(struct sched_domain *sd, int sd_id); -static __always_inline void set_cpu_idle_state_poc(int cpu, int state) -{ - if (static_branch_likely(&poc_selector_active) && - !sched_asym_cpucap_active()) - __set_cpu_idle_state_poc(cpu, state); -} - -/* - * POC_CTZ64 - Count trailing zeros (find first set bit) - * - * Architecture-optimized CTZ for POC idle CPU selection. - * Returns 64 for input 0 (important for BSF-based implementations). - */ -#if defined(__x86_64__) && defined(__BMI__) -/* Tier 1: x86-64 with BMI1 - TZCNT is zero-safe */ -#define POC_CTZ64(v) ((int)__builtin_ctzll(v)) - -#elif defined(__aarch64__) -/* Tier 1: ARM64 - RBIT+CLZ is zero-safe */ -#define POC_CTZ64(v) ((int)__builtin_ctzll(v)) - -#elif defined(__riscv) && defined(__riscv_zbb) -/* Tier 1: RISC-V with Zbb - CTZ is zero-safe */ -#define POC_CTZ64(v) ((int)__builtin_ctzll(v)) - -#elif defined(__x86_64__) -/* Tier 2: x86-64 without BMI1 - BSF needs zero check */ -static __always_inline int poc_ctz64_bsf(u64 v) -{ - if (unlikely(!v)) - return 64; - return (int)__builtin_ctzll(v); -} -#define POC_CTZ64(v) poc_ctz64_bsf(v) - -#else -/* Tier 3: De Bruijn fallback for other architectures */ -#define POC_DEBRUIJN_CTZ64_CONST 0x03F79D71B4CA8B09ULL -static const u8 poc_debruijn_ctz64_tab[64] = { - 0, 1, 56, 2, 57, 49, 28, 3, - 61, 58, 42, 50, 38, 29, 17, 4, - 62, 47, 59, 36, 45, 43, 51, 22, - 53, 39, 33, 30, 24, 18, 12, 5, - 63, 55, 48, 27, 60, 41, 37, 16, - 46, 35, 44, 21, 52, 32, 23, 11, - 54, 26, 40, 15, 34, 20, 31, 10, - 25, 14, 19, 9, 13, 8, 7, 6, -}; -static __always_inline int poc_debruijn_ctz64(u64 v) -{ - u64 lsb; - u32 idx; - - if (unlikely(!v)) - return 64; - lsb = v & (-(s64)v); - idx = (u32)((lsb * POC_DEBRUIJN_CTZ64_CONST) >> 58); - return (int)poc_debruijn_ctz64_tab[idx & 63]; -} -#define POC_CTZ64(v) poc_debruijn_ctz64(v) - -#endif /* POC_CTZ64 */ - -/* - * POC helper: convert cpumask region to POC-relative u64 - * - * Extracts the 64-bit region of @mask corresponding to this LLC's - * CPU range and shifts it to align with POC's bit positions. - * - * Used by load balancer functions that need to intersect cpumasks - * with POC idle bitmaps. - */ -static __always_inline u64 poc_cpumask_to_u64(const struct cpumask *mask, - struct sched_domain_shared *sd_share) -{ - int base = sd_share->poc_cpu_base; - int base_word = base >> 6; - int shift = sd_share->poc_affinity_shift; - - if (likely(shift == 0)) { - /* - * Fast path: this LLC's own base is 64-aligned, so no - * cross-word shift is needed here even if some other LLC - * in the system is misaligned (sched_poc_aligned is a - * single system-wide flag and must not gate this branch). - */ - return cpumask_bits(mask)[base_word]; - } else { - /* Slow path: shift required (e.g., Threadripper) */ - u64 lo = cpumask_bits(mask)[base_word]; - u64 hi = (base_word + 1 < BITS_TO_LONGS(nr_cpumask_bits)) ? - cpumask_bits(mask)[base_word + 1] : 0; - return (lo >> shift) | (hi << (64 - shift)); - } -} -#endif /* CONFIG_SCHED_POC_SELECTOR */ - static inline void __set_task_cpu(struct task_struct *p, unsigned int cpu) { set_task_rq(p, cpu); @@ -3570,7 +3453,6 @@ extern void nohz_run_idle_balance(int cpu); static inline void nohz_run_idle_balance(int cpu) { } #endif - #include "stats.h" #if defined(CONFIG_SCHED_CORE) && defined(CONFIG_SCHEDSTATS) diff --git a/kernel/sched/topology.c b/kernel/sched/topology.c index 339d505c8..622e2e019 100644 --- a/kernel/sched/topology.c +++ b/kernel/sched/topology.c @@ -3170,9 +3170,6 @@ build_sched_domains(const struct cpumask *cpu_map, struct sched_domain_attr *att if (sd->flags & SD_SHARE_LLC) { init_sched_domain_shared(&d, sd, SD_SHARE_LLC); -#ifdef CONFIG_SCHED_POC_SELECTOR - poc_sd_shared_init(sd, cpumask_first(sched_domain_span(sd))); -#endif /* * In presence of higher domains, adjust the -- 2.55.0 From 4e62662e72a310a7b8de06e3d50569d6affdebca Mon Sep 17 00:00:00 2001 From: Piotr Gorski Date: Mon, 17 Aug 2026 17:47:25 +0200 Subject: [PATCH 2/5] Revert gaming-sched Signed-off-by: Piotr Gorski --- include/linux/cpuset.h | 6 - include/linux/sched.h | 1 - kernel/cgroup/cpuset.c | 22 - kernel/sched/core.c | 5 +- kernel/sched/debug.c | 89 +--- kernel/sched/fair.c | 941 ++++++++++++++++++++--------------------- kernel/sched/pelt.c | 6 +- kernel/sched/sched.h | 30 +- 8 files changed, 487 insertions(+), 613 deletions(-) diff --git a/include/linux/cpuset.h b/include/linux/cpuset.h index 9db2d4fce..65d76a389 100644 --- a/include/linux/cpuset.h +++ b/include/linux/cpuset.h @@ -80,7 +80,6 @@ extern void lockdep_assert_cpuset_lock_held(void); extern void cpuset_cpus_allowed_locked(struct task_struct *p, struct cpumask *mask); extern void cpuset_cpus_allowed(struct task_struct *p, struct cpumask *mask); extern bool cpuset_cpus_allowed_fallback(struct task_struct *p); -extern int cpuset_num_cpus(struct cgroup *cgroup); extern nodemask_t cpuset_mems_allowed(struct task_struct *p); #define cpuset_current_mems_allowed (current->mems_allowed) void cpuset_init_current_mems_allowed(void); @@ -217,11 +216,6 @@ static inline bool cpuset_cpus_allowed_fallback(struct task_struct *p) return false; } -static inline int cpuset_num_cpus(struct cgroup *cgroup) -{ - return num_online_cpus(); -} - static inline nodemask_t cpuset_mems_allowed(struct task_struct *p) { return node_possible_map; diff --git a/include/linux/sched.h b/include/linux/sched.h index 63536356d..5738c54eb 100644 --- a/include/linux/sched.h +++ b/include/linux/sched.h @@ -575,7 +575,6 @@ struct sched_statistics { struct sched_entity { /* For load-balancing: */ struct load_weight load; - struct load_weight h_load; struct rb_node run_node; u64 deadline; u64 min_vruntime; diff --git a/kernel/cgroup/cpuset.c b/kernel/cgroup/cpuset.c index e42f81a61..45944b3e3 100644 --- a/kernel/cgroup/cpuset.c +++ b/kernel/cgroup/cpuset.c @@ -4129,28 +4129,6 @@ bool cpuset_cpus_allowed_fallback(struct task_struct *tsk) return changed; } -/* - * Returns the number of CPUs available for this cgroup. - * - * This only really works for cgroup-v2 where all the controllers are mounted - * in the same hierarchy. If not cgroup-v2 or no cpuset controller is - * configured it reverts to num_online_cpus(). - */ -int cpuset_num_cpus(struct cgroup *cgrp) -{ - int nr = num_online_cpus(); - struct cpuset *cs; - - if (is_in_v2_mode()) { - guard(rcu)(); - cs = css_cs(cgroup_e_css(cgrp, &cpuset_cgrp_subsys)); - if (cs) - nr = cpumask_weight(cs->effective_cpus); - } - - return nr; -} - void __init cpuset_init_current_mems_allowed(void) { nodes_setall(current->mems_allowed); diff --git a/kernel/sched/core.c b/kernel/sched/core.c index edd1a3fc7..e55a8f257 100644 --- a/kernel/sched/core.c +++ b/kernel/sched/core.c @@ -5660,8 +5660,11 @@ EXPORT_PER_CPU_SYMBOL(kernel_cpustat); */ static inline void prefetch_curr_exec_start(struct task_struct *p) { +#ifdef CONFIG_FAIR_GROUP_SCHED + struct sched_entity *curr = p->se.cfs_rq->curr; +#else struct sched_entity *curr = task_rq(p)->cfs.curr; - +#endif prefetch(curr); prefetch(&curr->exec_start); } diff --git a/kernel/sched/debug.c b/kernel/sched/debug.c index c15291cbc..40584b27e 100644 --- a/kernel/sched/debug.c +++ b/kernel/sched/debug.c @@ -633,82 +633,6 @@ static void debugfs_fair_server_init(void) } } -#ifdef CONFIG_FAIR_GROUP_SCHED -static int cgroup_mode = 2; - -/* See __sched_cgroup_mode_update(). */ -static const char *cgroup_mode_str[] = { - "up", - "smp", - "concur", - "max", - "tasks", -}; - -static int sched_cgroup_mode(const char *str) -{ - for (int i = 0; i < ARRAY_SIZE(cgroup_mode_str); i++) { - if (!strcmp(str, cgroup_mode_str[i])) - return i; - } - return -EINVAL; -} - -static ssize_t sched_cgroup_write(struct file *filp, const char __user *ubuf, - size_t cnt, loff_t *ppos) -{ - char buf[16]; - int mode; - - if (cnt > 15) - cnt = 15; - - if (copy_from_user(buf, ubuf, cnt)) - return -EFAULT; - - buf[cnt] = 0; - mode = sched_cgroup_mode(strstrip(buf)); - if (mode < 0) - return mode; - - __sched_cgroup_mode_update(mode); - WRITE_ONCE(cgroup_mode, mode); - - *ppos += cnt; - return cnt; -} - -static int sched_cgroup_show(struct seq_file *m, void *v) -{ - int mode = READ_ONCE(cgroup_mode); - - for (int i = 0; i < ARRAY_SIZE(cgroup_mode_str); i++) { - if (mode == i) - seq_puts(m, "("); - seq_puts(m, cgroup_mode_str[i]); - if (mode == i) - seq_puts(m, ")"); - - seq_puts(m, " "); - } - seq_puts(m, "\n"); - return 0; -} - -static int sched_cgroup_open(struct inode *inode, struct file *filp) -{ - return single_open(filp, sched_cgroup_show, NULL); -} - -static const struct file_operations sched_cgroup_fops = { - .open = sched_cgroup_open, - .write = sched_cgroup_write, - .read = seq_read, - .llseek = seq_lseek, - .release = single_release, -}; -#endif - static __init int sched_init_debug(void) { struct dentry __maybe_unused *numa, *llc; @@ -762,10 +686,6 @@ static __init int sched_init_debug(void) debugfs_create_file("debug", 0444, debugfs_sched, NULL, &sched_debug_fops); -#ifdef CONFIG_FAIR_GROUP_SCHED - debugfs_create_file("cgroup_mode", 0644, debugfs_sched, NULL, &sched_cgroup_fops); -#endif - debugfs_fair_server_init(); #ifdef CONFIG_SCHED_CLASS_EXT debugfs_ext_server_init(); @@ -975,11 +895,10 @@ print_task(struct seq_file *m, struct rq *rq, struct task_struct *p) else SEQ_printf(m, " %c", task_state_to_char(p)); - SEQ_printf(m, " %15s %5d %10ld %9Ld.%06ld %c %9Ld.%06ld %c %9Ld.%06ld %9Ld.%06ld %9Ld %5d ", + SEQ_printf(m, " %15s %5d %9Ld.%06ld %c %9Ld.%06ld %c %9Ld.%06ld %9Ld.%06ld %9Ld %5d ", p->comm, task_pid_nr(p), - p->se.h_load.weight, SPLIT_NS(p->se.vruntime), - entity_eligible(&rq->cfs, &p->se) ? 'E' : 'N', + entity_eligible(cfs_rq_of(&p->se), &p->se) ? 'E' : 'N', SPLIT_NS(p->se.deadline), p->se.custom_slice ? 'S' : ' ', SPLIT_NS(p->se.slice), @@ -1008,7 +927,7 @@ static void print_rq(struct seq_file *m, struct rq *rq, int rq_cpu) SEQ_printf(m, "\n"); SEQ_printf(m, "runnable tasks:\n"); - SEQ_printf(m, " S task PID weight vruntime eligible " + SEQ_printf(m, " S task PID vruntime eligible " "deadline slice sum-exec switches " "prio wait-time sum-sleep sum-block" #ifdef CONFIG_NUMA_BALANCING @@ -1116,8 +1035,6 @@ void print_cfs_rq(struct seq_file *m, int cpu, struct cfs_rq *cfs_rq) cfs_rq->tg_load_avg_contrib); SEQ_printf(m, " .%-30s: %ld\n", "tg_load_avg", atomic_long_read(&cfs_rq->tg->load_avg)); - SEQ_printf(m, " .%-30s: %lu\n", "h_load", - cfs_rq->h_load); #endif /* CONFIG_FAIR_GROUP_SCHED */ #ifdef CONFIG_CFS_BANDWIDTH SEQ_printf(m, " .%-30s: %d\n", "throttled", diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c index d309809ec..bf5c9a01c 100644 --- a/kernel/sched/fair.c +++ b/kernel/sched/fair.c @@ -38,7 +38,6 @@ #include #include #include -#include #include #include @@ -306,8 +305,8 @@ static u64 __calc_delta(u64 delta_exec, unsigned long weight, struct load_weight */ static inline u64 calc_delta_fair(u64 delta, struct sched_entity *se) { - if (se->h_load.weight != NICE_0_LOAD) - delta = __calc_delta(delta, NICE_0_LOAD, &se->h_load); + if (unlikely(se->load.weight != NICE_0_LOAD)) + delta = __calc_delta(delta, NICE_0_LOAD, &se->load); return delta; } @@ -437,6 +436,38 @@ static inline struct sched_entity *parent_entity(const struct sched_entity *se) return se->parent; } +static void +find_matching_se(struct sched_entity **se, struct sched_entity **pse) +{ + int se_depth, pse_depth; + + /* + * preemption test can be made between sibling entities who are in the + * same cfs_rq i.e who have a common parent. Walk up the hierarchy of + * both tasks until we find their ancestors who are siblings of common + * parent. + */ + + /* First walk up until both entities are at same depth */ + se_depth = (*se)->depth; + pse_depth = (*pse)->depth; + + while (se_depth > pse_depth) { + se_depth--; + *se = parent_entity(*se); + } + + while (pse_depth > se_depth) { + pse_depth--; + *pse = parent_entity(*pse); + } + + while (!is_same_group(*se, *pse)) { + *se = parent_entity(*se); + *pse = parent_entity(*pse); + } +} + static int tg_is_idle(struct task_group *tg) { return tg->idle > 0; @@ -480,6 +511,11 @@ static inline struct sched_entity *parent_entity(struct sched_entity *se) return NULL; } +static inline void +find_matching_se(struct sched_entity **se, struct sched_entity **pse) +{ +} + static inline int tg_is_idle(struct task_group *tg) { return 0; @@ -658,7 +694,7 @@ static inline unsigned long avg_vruntime_weight(struct cfs_rq *cfs_rq, unsigned static inline void __sum_w_vruntime_add(struct cfs_rq *cfs_rq, struct sched_entity *se) { - unsigned long weight = avg_vruntime_weight(cfs_rq, se->h_load.weight); + unsigned long weight = avg_vruntime_weight(cfs_rq, se->load.weight); s64 w_vruntime, key = entity_key(cfs_rq, se); w_vruntime = key * weight; @@ -675,7 +711,7 @@ sum_w_vruntime_add_paranoid(struct cfs_rq *cfs_rq, struct sched_entity *se) s64 key, tmp; again: - weight = avg_vruntime_weight(cfs_rq, se->h_load.weight); + weight = avg_vruntime_weight(cfs_rq, se->load.weight); key = entity_key(cfs_rq, se); if (check_mul_overflow(key, weight, &key)) @@ -721,7 +757,7 @@ sum_w_vruntime_add(struct cfs_rq *cfs_rq, struct sched_entity *se) static void sum_w_vruntime_sub(struct cfs_rq *cfs_rq, struct sched_entity *se) { - unsigned long weight = avg_vruntime_weight(cfs_rq, se->h_load.weight); + unsigned long weight = avg_vruntime_weight(cfs_rq, se->load.weight); s64 key = entity_key(cfs_rq, se); cfs_rq->sum_w_vruntime -= key * weight; @@ -763,7 +799,7 @@ u64 avg_vruntime(struct cfs_rq *cfs_rq) s64 runtime = cfs_rq->sum_w_vruntime; if (curr) { - unsigned long w = avg_vruntime_weight(cfs_rq, curr->h_load.weight); + unsigned long w = avg_vruntime_weight(cfs_rq, curr->load.weight); runtime += entity_key(cfs_rq, curr) * w; weight += w; @@ -834,6 +870,8 @@ bool update_entity_lag(struct cfs_rq *cfs_rq, struct sched_entity *se) u64 avruntime = avg_vruntime(cfs_rq); s64 vlag = entity_lag(cfs_rq, se, avruntime); + WARN_ON_ONCE(!se->on_rq); + if (se->sched_delayed) { /* previous vlag < 0 otherwise se would not be delayed */ vlag = max(vlag, se->vlag); @@ -869,7 +907,7 @@ static int vruntime_eligible(struct cfs_rq *cfs_rq, u64 vruntime) long load = cfs_rq->sum_weight; if (curr && curr->on_rq) { - unsigned long weight = avg_vruntime_weight(cfs_rq, curr->h_load.weight); + unsigned long weight = avg_vruntime_weight(cfs_rq, curr->load.weight); avg += entity_key(cfs_rq, curr) * weight; load += weight; @@ -1010,9 +1048,6 @@ RB_DECLARE_CALLBACKS(static, min_vruntime_cb, struct sched_entity, */ static void __enqueue_entity(struct cfs_rq *cfs_rq, struct sched_entity *se) { - WARN_ON_ONCE(&rq_of(cfs_rq)->cfs != cfs_rq); - WARN_ON_ONCE(!entity_is_task(se)); - sum_w_vruntime_add(cfs_rq, se); se->min_vruntime = se->vruntime; se->min_slice = se->slice; @@ -1022,9 +1057,6 @@ static void __enqueue_entity(struct cfs_rq *cfs_rq, struct sched_entity *se) static void __dequeue_entity(struct cfs_rq *cfs_rq, struct sched_entity *se) { - WARN_ON_ONCE(&rq_of(cfs_rq)->cfs != cfs_rq); - WARN_ON_ONCE(!entity_is_task(se)); - rb_erase_augmented_cached(&se->run_node, &cfs_rq->tasks_timeline, &min_vruntime_cb); sum_w_vruntime_sub(cfs_rq, se); @@ -1121,7 +1153,7 @@ static struct sched_entity *pick_eevdf(struct cfs_rq *cfs_rq, bool protect) * We can safely skip eligibility check if there is only one entity * in this cfs_rq, saving some cycles. */ - if (cfs_rq->h_nr_queued == 1) + if (cfs_rq->nr_queued == 1) return curr && curr->on_rq ? curr : se; /* @@ -1371,6 +1403,8 @@ static s64 update_se(struct rq *rq, struct sched_entity *se) return delta_exec; } +static void set_next_buddy(struct sched_entity *se); + #ifdef CONFIG_SCHED_CACHE /* @@ -1965,7 +1999,7 @@ static void update_curr(struct cfs_rq *cfs_rq) * not necessarily be the actual task running * (rq->curr.se). This is easy to confuse! */ - struct sched_entity *curr = cfs_rq->h_curr; + struct sched_entity *curr = cfs_rq->curr; struct rq *rq = rq_of(cfs_rq); s64 delta_exec; bool resched; @@ -1977,29 +2011,26 @@ static void update_curr(struct cfs_rq *cfs_rq) if (unlikely(delta_exec <= 0)) return; - account_cfs_rq_runtime(cfs_rq, delta_exec); - - if (!entity_is_task(curr)) - return; - - cfs_rq = &rq->cfs; - curr->vruntime += calc_delta_fair(delta_exec, curr); resched = update_deadline(cfs_rq, curr); - /* - * If the fair_server is active, we need to account for the - * fair_server time whether or not the task is running on - * behalf of fair_server or not: - * - If the task is running on behalf of fair_server, we need - * to limit its time based on the assigned runtime. - * - Fair task that runs outside of fair_server should account - * against fair_server such that it can account for this time - * and possibly avoid running this period. - */ - dl_server_update(&rq->fair_server, delta_exec); + if (entity_is_task(curr)) { + /* + * If the fair_server is active, we need to account for the + * fair_server time whether or not the task is running on + * behalf of fair_server or not: + * - If the task is running on behalf of fair_server, we need + * to limit its time based on the assigned runtime. + * - Fair task that runs outside of fair_server should account + * against fair_server such that it can account for this time + * and possibly avoid running this period. + */ + dl_server_update(&rq->fair_server, delta_exec); + } - if (cfs_rq->h_nr_queued == 1) + account_cfs_rq_runtime(cfs_rq, delta_exec); + + if (cfs_rq->nr_queued == 1) return; if (resched || !protect_slice(curr)) { @@ -2010,10 +2041,7 @@ static void update_curr(struct cfs_rq *cfs_rq) static void update_curr_fair(struct rq *rq) { - struct sched_entity *se = &rq->donor->se; - - for_each_sched_entity(se) - update_curr(cfs_rq_of(se)); + update_curr(cfs_rq_of(&rq->donor->se)); } static inline void @@ -2089,7 +2117,7 @@ update_stats_enqueue_fair(struct cfs_rq *cfs_rq, struct sched_entity *se, int fl * Are we enqueueing a waiting task? (for current tasks * a dequeue/enqueue event is a NOP) */ - if (se != cfs_rq->h_curr) + if (se != cfs_rq->curr) update_stats_wait_start_fair(cfs_rq, se); if (flags & ENQUEUE_WAKEUP) @@ -2107,7 +2135,7 @@ update_stats_dequeue_fair(struct cfs_rq *cfs_rq, struct sched_entity *se, int fl * Mark the end of the wait period if dequeueing a * waiting task: */ - if (se != cfs_rq->h_curr) + if (se != cfs_rq->curr) update_stats_wait_end_fair(cfs_rq, se); if ((flags & DEQUEUE_SLEEP) && entity_is_task(se)) { @@ -4448,7 +4476,6 @@ static inline void update_scan_period(struct task_struct *p, int new_cpu) static void account_entity_enqueue(struct cfs_rq *cfs_rq, struct sched_entity *se) { - WARN_ON_ONCE(cfs_rq != cfs_rq_of(se)); update_load_add(&cfs_rq->load, se->load.weight); if (entity_is_task(se)) { struct task_struct *p = task_of(se); @@ -4464,7 +4491,6 @@ account_entity_enqueue(struct cfs_rq *cfs_rq, struct sched_entity *se) static void account_entity_dequeue(struct cfs_rq *cfs_rq, struct sched_entity *se) { - WARN_ON_ONCE(cfs_rq != cfs_rq_of(se)); update_load_sub(&cfs_rq->load, se->load.weight); if (entity_is_task(se)) { struct task_struct *p = task_of(se); @@ -4546,7 +4572,7 @@ dequeue_load_avg(struct cfs_rq *cfs_rq, struct sched_entity *se) static void rescale_entity(struct sched_entity *se, unsigned long weight, bool rel_vprot) { - long old_weight = se->h_load.weight; + unsigned long old_weight = se->load.weight; /* * VRUNTIME @@ -4646,17 +4672,16 @@ rescale_entity(struct sched_entity *se, unsigned long weight, bool rel_vprot) se->vprot = div64_long(se->vprot * old_weight, weight); } -static void reweight_eevdf(struct cfs_rq *cfs_rq, struct sched_entity *se, - unsigned long weight, bool on_rq) +static void reweight_entity(struct cfs_rq *cfs_rq, struct sched_entity *se, + unsigned long weight) { bool curr = cfs_rq->curr == se; bool rel_vprot = false; u64 avruntime = 0; - if (se->h_load.weight == weight) - return; - - if (on_rq) { + if (se->on_rq) { + /* commit outstanding execution time */ + update_curr(cfs_rq); avruntime = avg_vruntime(cfs_rq); se->vlag = entity_lag(cfs_rq, se, avruntime); se->deadline -= avruntime; @@ -4666,40 +4691,15 @@ static void reweight_eevdf(struct cfs_rq *cfs_rq, struct sched_entity *se, rel_vprot = true; } - cfs_rq->h_nr_queued--; + cfs_rq->nr_queued--; if (!curr) __dequeue_entity(cfs_rq, se); - } - - rescale_entity(se, weight, rel_vprot); - - update_load_set(&se->h_load, weight); - - if (on_rq) { - if (rel_vprot) - se->vprot += avruntime; - se->deadline += avruntime; - se->rel_deadline = 0; - se->vruntime = avruntime - se->vlag; - - if (!curr) - __enqueue_entity(cfs_rq, se); - cfs_rq->h_nr_queued++; - } -} - -static void reweight_entity(struct cfs_rq *cfs_rq, struct sched_entity *se, - unsigned long weight) -{ - if (se->load.weight == weight) - return; - - if (se->on_rq) { - WARN_ON_ONCE(cfs_rq != cfs_rq_of(se)); update_load_sub(&cfs_rq->load, se->load.weight); } dequeue_load_avg(cfs_rq, se); + rescale_entity(se, weight, rel_vprot); + update_load_set(&se->load, weight); do { @@ -4708,48 +4708,29 @@ static void reweight_entity(struct cfs_rq *cfs_rq, struct sched_entity *se, } while (0); enqueue_load_avg(cfs_rq, se); + if (se->on_rq) { + if (rel_vprot) + se->vprot += avruntime; + se->deadline += avruntime; + se->rel_deadline = 0; + se->vruntime = avruntime - se->vlag; - if (se->on_rq) update_load_add(&cfs_rq->load, se->load.weight); -} - -/* - * weight = NICE_0_LOAD; - * for_each_entity_se(se) - * weight = __calc_prop_weight(cfs_rq_of(se), se, weight); - */ -static __always_inline -unsigned long __calc_prop_weight(struct cfs_rq *cfs_rq, struct sched_entity *se, - unsigned long weight) -{ - weight *= se->load.weight; - if (parent_entity(se)) - weight /= cfs_rq->load.weight; - else - weight /= NICE_0_LOAD; - - return max(weight, MIN_SHARES); + if (!curr) + __enqueue_entity(cfs_rq, se); + cfs_rq->nr_queued++; + } } static void reweight_task_fair(struct rq *rq, struct task_struct *p, const struct load_weight *lw) { struct sched_entity *se = &p->se; - unsigned long weight = NICE_0_LOAD; + struct cfs_rq *cfs_rq = cfs_rq_of(se); + struct load_weight *load = &se->load; - if (se->on_rq) - update_curr_fair(rq); - - reweight_entity(cfs_rq_of(se), se, lw->weight); - se->load.inv_weight = lw->inv_weight; - - if (!se->on_rq) - return; - - for_each_sched_entity(se) - weight = __calc_prop_weight(cfs_rq_of(se), se, weight); - - reweight_eevdf(&rq->cfs, &p->se, weight, p->se.on_rq); + reweight_entity(cfs_rq, se, lw->weight); + load->inv_weight = lw->inv_weight; } static inline int throttled_hierarchy(struct cfs_rq *cfs_rq); @@ -4828,10 +4809,12 @@ static inline int throttled_hierarchy(struct cfs_rq *cfs_rq); * * hence icky! */ -static long __calc_smp_shares(struct cfs_rq *cfs_rq, long tg_shares, long shares_max) +static long calc_group_shares(struct cfs_rq *cfs_rq) { + long tg_weight, tg_shares, load, shares; struct task_group *tg = cfs_rq->tg; - long tg_weight, load, shares; + + tg_shares = READ_ONCE(tg->shares); load = max(scale_load_down(cfs_rq->load.weight), cfs_rq->avg.load_avg); @@ -4857,115 +4840,7 @@ static long __calc_smp_shares(struct cfs_rq *cfs_rq, long tg_shares, long shares * case no task is runnable on a CPU MIN_SHARES=2 should be returned * instead of 0. */ - return clamp_t(long, shares, MIN_SHARES, shares_max); -} - -static int tg_cpus(struct task_group *tg) -{ - int nr = num_online_cpus(); - - if (cpusets_enabled()) { - struct cgroup *cgrp = tg->css.cgroup; - if (cgrp) - nr = cpuset_num_cpus(cgrp); - } - - return nr; -} - -static inline int tg_tasks(struct task_group *tg) -{ - return max(1, atomic_long_read(&tg->runnable_avg) >> SCHED_CAPACITY_SHIFT); -} - -/* - * Func: fraction(nr_tasks * tg->shares) - * - * Scale tg->shares by the number of tasks. - */ -static long calc_tasks_shares(struct cfs_rq *cfs_rq) -{ - struct task_group *tg = cfs_rq->tg; - int nr = tg_tasks(tg); - long tg_shares = READ_ONCE(tg->shares); - return __calc_smp_shares(cfs_rq, nr * tg_shares, nr * tg_shares); -} - -/* - * Func: min(fraction(nr_cpus * tg->shares), nice -20) - * - * Scale tg->shares by the maximal number of CPUs; but clip the max shares at - * nice -20, otherwise a single spinner on a 512 CPU machine would result in - * 512*NICE_0_LOAD, which is also crazy. - */ -static long calc_max_shares(struct cfs_rq *cfs_rq) -{ - struct task_group *tg = cfs_rq->tg; - int nr = tg_cpus(tg); - long tg_shares = READ_ONCE(tg->shares); - long max_shares = scale_load(sched_prio_to_weight[0]); - return __calc_smp_shares(cfs_rq, tg_shares * nr, max_shares); -} - -/* - * Func: fraction(nr * tg->shares); nr = min(nr_tasks, nr_cpus) - * - * Scales between "smp" and "max" in a natural way. No longer needs clipping - * since there are no unnatural inflations like with "max". - */ -static long calc_concur_shares(struct cfs_rq *cfs_rq) -{ - struct task_group *tg = cfs_rq->tg; - int nr = min(tg_tasks(tg), tg_cpus(tg)); - long tg_shares = READ_ONCE(tg->shares); - return __calc_smp_shares(cfs_rq, nr * tg_shares, nr * tg_shares); -} - -/* - * Func: fraction(tg->shares) - * - * This infamously results in tiny shares when you have many CPUs. - */ -static long calc_smp_shares(struct cfs_rq *cfs_rq) -{ - struct task_group *tg = cfs_rq->tg; - long tg_shares = READ_ONCE(tg->shares); - return __calc_smp_shares(cfs_rq, tg_shares, tg_shares); -} - -/* - * Ignore this pesky SMP stuff, use (4). - */ -static long calc_up_shares(struct cfs_rq *cfs_rq) -{ - struct task_group *tg = cfs_rq->tg; - return READ_ONCE(tg->shares); -} - -DEFINE_STATIC_CALL(calc_group_shares, calc_concur_shares); - -void __sched_cgroup_mode_update(int mode) -{ - long (*func)(struct cfs_rq *); - switch (mode) { - case 0: - func = &calc_up_shares; - break; - case 1: - func = &calc_smp_shares; - break; - case 2: - default: - func = &calc_concur_shares; - break; - case 3: - func = &calc_max_shares; - break; - case 4: - func = &calc_tasks_shares; - break; - } - static_call_update(calc_group_shares, func); + return clamp_t(long, shares, MIN_SHARES, tg_shares); } /* @@ -4984,8 +4859,9 @@ static void update_cfs_group(struct sched_entity *se) if (!gcfs_rq || !gcfs_rq->load.weight) return; - shares = static_call(calc_group_shares)(gcfs_rq); - reweight_entity(cfs_rq_of(se), se, shares); + shares = calc_group_shares(gcfs_rq); + if (unlikely(se->load.weight != shares)) + reweight_entity(cfs_rq_of(se), se, shares); } #else /* !CONFIG_FAIR_GROUP_SCHED: */ @@ -5103,11 +4979,11 @@ static inline bool cfs_rq_is_decayed(struct cfs_rq *cfs_rq) * differential update where we store the last value we propagated. This in * turn allows skipping updates if the differential is 'small'. * - * Updating tg's load_avg is necessary before update_cfs_group(). + * Updating tg's load_avg is necessary before update_cfs_share(). */ static inline void update_tg_load_avg(struct cfs_rq *cfs_rq) { - long dl, dr; + long delta; u64 now; /* @@ -5128,21 +5004,17 @@ static inline void update_tg_load_avg(struct cfs_rq *cfs_rq) if (now - cfs_rq->last_update_tg_load_avg < NSEC_PER_MSEC) return; - dl = cfs_rq->avg.load_avg - cfs_rq->tg_load_avg_contrib; - dr = cfs_rq->avg.runnable_avg - cfs_rq->tg_runnable_avg_contrib; - if (abs(dl) > cfs_rq->tg_load_avg_contrib / 64 || - abs(dr) > cfs_rq->tg_runnable_avg_contrib / 64) { - atomic_long_add(dl, &cfs_rq->tg->load_avg); - atomic_long_add(dr, &cfs_rq->tg->runnable_avg); + delta = cfs_rq->avg.load_avg - cfs_rq->tg_load_avg_contrib; + if (abs(delta) > cfs_rq->tg_load_avg_contrib / 64) { + atomic_long_add(delta, &cfs_rq->tg->load_avg); cfs_rq->tg_load_avg_contrib = cfs_rq->avg.load_avg; - cfs_rq->tg_runnable_avg_contrib = cfs_rq->avg.runnable_avg; cfs_rq->last_update_tg_load_avg = now; } } static inline void clear_tg_load_avg(struct cfs_rq *cfs_rq) { - long dl, dr; + long delta; u64 now; /* @@ -5152,12 +5024,9 @@ static inline void clear_tg_load_avg(struct cfs_rq *cfs_rq) return; now = rq_clock(rq_of(cfs_rq)); - dl = 0 - cfs_rq->tg_load_avg_contrib; - dr = 0 - cfs_rq->tg_runnable_avg_contrib; - atomic_long_add(dl, &cfs_rq->tg->load_avg); - atomic_long_add(dr, &cfs_rq->tg->runnable_avg); + delta = 0 - cfs_rq->tg_load_avg_contrib; + atomic_long_add(delta, &cfs_rq->tg->load_avg); cfs_rq->tg_load_avg_contrib = 0; - cfs_rq->tg_runnable_avg_contrib = 0; cfs_rq->last_update_tg_load_avg = now; } @@ -5570,7 +5439,7 @@ static void migrate_se_pelt_lag(struct sched_entity *se) {} * The cfs_rq avg is the direct sum of all its entities (blocked and runnable) * avg. The immediate corollary is that all (fair) tasks must be attached. * - * cfs_rq->avg is used for task_h_load() and update_cfs_group() for example. + * cfs_rq->avg is used for task_h_load() and update_cfs_share() for example. * * Return: true if the load decayed or we removed load. * @@ -6108,7 +5977,6 @@ static void place_entity(struct cfs_rq *cfs_rq, struct sched_entity *se, int flags) { u64 vslice, vruntime = avg_vruntime(cfs_rq); - unsigned int nr_queued = cfs_rq->h_nr_queued; bool update_zero = false; s64 lag = 0; @@ -6116,9 +5984,6 @@ place_entity(struct cfs_rq *cfs_rq, struct sched_entity *se, int flags) se->slice = sysctl_sched_base_slice; vslice = calc_delta_fair(se->slice, se); - if (flags & ENQUEUE_QUEUED) - nr_queued -= 1; - /* * Due to how V is constructed as the weighted average of entities, * adding tasks with positive lag, or removing tasks with negative lag @@ -6127,7 +5992,7 @@ place_entity(struct cfs_rq *cfs_rq, struct sched_entity *se, int flags) * * EEVDF: placement strategy #1 / #2 */ - if (sched_feat(PLACE_LAG) && nr_queued && se->vlag) { + if (sched_feat(PLACE_LAG) && cfs_rq->nr_queued && se->vlag) { struct sched_entity *curr = cfs_rq->curr; long load, weight; @@ -6187,9 +6052,9 @@ place_entity(struct cfs_rq *cfs_rq, struct sched_entity *se, int flags) */ load = cfs_rq->sum_weight; if (curr && curr->on_rq) - load += avg_vruntime_weight(cfs_rq, curr->h_load.weight); + load += avg_vruntime_weight(cfs_rq, curr->load.weight); - weight = avg_vruntime_weight(cfs_rq, se->h_load.weight); + weight = avg_vruntime_weight(cfs_rq, se->load.weight); lag *= load + weight; if (WARN_ON_ONCE(!load)) load = 1; @@ -6247,9 +6112,23 @@ place_entity(struct cfs_rq *cfs_rq, struct sched_entity *se, int flags) static void check_enqueue_throttle(struct cfs_rq *cfs_rq); static inline int cfs_rq_throttled(struct cfs_rq *cfs_rq); +static void +requeue_delayed_entity(struct sched_entity *se); + static void enqueue_entity(struct cfs_rq *cfs_rq, struct sched_entity *se, int flags) { + bool curr = cfs_rq->curr == se; + + /* + * If we're the current task, we must renormalise before calling + * update_curr(). + */ + if (curr) + place_entity(cfs_rq, se, flags); + + update_curr(cfs_rq); + /* * When enqueuing a sched_entity, we must: * - Update loads to have both entity and cfs_rq synced with now. @@ -6268,6 +6147,13 @@ enqueue_entity(struct cfs_rq *cfs_rq, struct sched_entity *se, int flags) */ update_cfs_group(se); + /* + * XXX now that the entity has been re-weighted, and it's lag adjusted, + * we can place the entity. + */ + if (!curr) + place_entity(cfs_rq, se, flags); + account_entity_enqueue(cfs_rq, se); /* Entity has migrated, no longer consider this task hot */ @@ -6276,6 +6162,8 @@ enqueue_entity(struct cfs_rq *cfs_rq, struct sched_entity *se, int flags) check_schedstat_required(); update_stats_enqueue_fair(cfs_rq, se, flags); + if (!curr) + __enqueue_entity(cfs_rq, se); se->on_rq = 1; if (cfs_rq->nr_queued == 1) { @@ -6293,19 +6181,21 @@ enqueue_entity(struct cfs_rq *cfs_rq, struct sched_entity *se, int flags) } } -static void set_next_buddy(struct cfs_rq *cfs_rq, struct sched_entity *se) +static void __clear_buddies_next(struct sched_entity *se) { - if (WARN_ON_ONCE(!se->on_rq || se->sched_delayed)) - return; - if (se_is_idle(se)) - return; - cfs_rq->next = se; + for_each_sched_entity(se) { + struct cfs_rq *cfs_rq = cfs_rq_of(se); + if (cfs_rq->next != se) + break; + + cfs_rq->next = NULL; + } } static void clear_buddies(struct cfs_rq *cfs_rq, struct sched_entity *se) { if (cfs_rq->next == se) - cfs_rq->next = NULL; + __clear_buddies_next(se); } static __always_inline void return_cfs_rq_runtime(struct cfs_rq *cfs_rq); @@ -6316,7 +6206,7 @@ static void set_delayed(struct sched_entity *se) /* * Delayed se of cfs_rq have no tasks queued on them. - * Do not adjust h_nr_runnable since __dequeue_task() + * Do not adjust h_nr_runnable since dequeue_entities() * will account it for blocked tasks. */ if (!entity_is_task(se)) @@ -6349,16 +6239,45 @@ static void clear_delayed(struct sched_entity *se) } } -static void +static bool dequeue_entity(struct cfs_rq *cfs_rq, struct sched_entity *se, int flags) { - int action = UPDATE_TG; + bool sleep = flags & DEQUEUE_SLEEP; + int action = 0; + update_curr(cfs_rq); + clear_buddies(cfs_rq, se); + + if (flags & DEQUEUE_DELAYED) { + WARN_ON_ONCE(!se->sched_delayed); + } else { + bool delay = sleep; + /* + * DELAY_DEQUEUE relies on spurious wakeups, special task + * states must not suffer spurious wakeups, excempt them. + */ + if (flags & (DEQUEUE_SPECIAL | DEQUEUE_THROTTLE)) + delay = false; + + WARN_ON_ONCE(delay && se->sched_delayed); + + if (sched_feat(DELAY_DEQUEUE) && delay && + !entity_eligible(cfs_rq, se)) { + if (entity_is_task(se)) + action |= UPDATE_UTIL_EST; + update_load_avg(cfs_rq, se, action); + update_entity_lag(cfs_rq, se); + set_delayed(se); + return false; + } + } + + action = UPDATE_TG; if (entity_is_task(se)) { if (task_on_rq_migrating(task_of(se))) action |= DO_DETACH; - if ((flags & DEQUEUE_SLEEP) && !(flags & DEQUEUE_DELAYED)) + if (sleep && !(flags & DEQUEUE_DELAYED)) action |= UPDATE_UTIL_EST; } @@ -6376,6 +6295,14 @@ dequeue_entity(struct cfs_rq *cfs_rq, struct sched_entity *se, int flags) update_stats_dequeue_fair(cfs_rq, se, flags); + update_entity_lag(cfs_rq, se); + if (sched_feat(PLACE_REL_DEADLINE) && !sleep) { + se->deadline -= se->vruntime; + se->rel_deadline = 1; + } + + if (se != cfs_rq->curr) + __dequeue_entity(cfs_rq, se); se->on_rq = 0; account_entity_dequeue(cfs_rq, se); @@ -6384,6 +6311,9 @@ dequeue_entity(struct cfs_rq *cfs_rq, struct sched_entity *se, int flags) update_cfs_group(se); + if (flags & DEQUEUE_DELAYED) + clear_delayed(se); + if (cfs_rq->nr_queued == 0) { update_idle_cfs_rq_clock_pelt(cfs_rq); #ifdef CONFIG_CFS_BANDWIDTH @@ -6396,11 +6326,15 @@ dequeue_entity(struct cfs_rq *cfs_rq, struct sched_entity *se, int flags) } #endif } + + return true; } static void -set_next_entity(struct cfs_rq *cfs_rq, struct sched_entity *se) +set_next_entity(struct cfs_rq *cfs_rq, struct sched_entity *se, bool first) { + clear_buddies(cfs_rq, se); + /* 'current' is not kept within the tree. */ if (se->on_rq) { /* @@ -6409,12 +6343,16 @@ set_next_entity(struct cfs_rq *cfs_rq, struct sched_entity *se) * runqueue. */ update_stats_wait_end_fair(cfs_rq, se); + __dequeue_entity(cfs_rq, se); update_load_avg(cfs_rq, se, UPDATE_TG); + + if (first) + set_protect_slice(cfs_rq, se); } update_stats_curr_start(cfs_rq, se); - WARN_ON_ONCE(cfs_rq->h_curr); - cfs_rq->h_curr = se; + WARN_ON_ONCE(cfs_rq->curr); + cfs_rq->curr = se; /* * Track our maximum slice length, if the CPU's load is at @@ -6434,17 +6372,23 @@ set_next_entity(struct cfs_rq *cfs_rq, struct sched_entity *se) se->prev_sum_exec_runtime = se->sum_exec_runtime; } -static bool __dequeue_task(struct rq *rq, struct task_struct *p, int flags); +static int dequeue_entities(struct rq *rq, struct sched_entity *se, int flags); +/* + * Pick the next process, keeping these things in mind, in this order: + * 1) keep things fair between processes/task groups + * 2) pick the "next" process, since someone really wants that to run + * 3) pick the "last" process, for cache locality + * 4) do not run the "skip" process, if something else is available + */ static struct sched_entity * -pick_next_entity(struct rq *rq, bool protect) +pick_next_entity(struct rq *rq, struct cfs_rq *cfs_rq, bool protect) { - struct cfs_rq *cfs_rq = &rq->cfs; struct sched_entity *se; se = pick_eevdf(cfs_rq, protect); if (se->sched_delayed) { - __dequeue_task(rq, task_of(se), DEQUEUE_SLEEP | DEQUEUE_DELAYED); + dequeue_entities(rq, se, DEQUEUE_SLEEP | DEQUEUE_DELAYED); /* * Must not reference @se again, see __block_task(). */ @@ -6464,11 +6408,13 @@ static void put_prev_entity(struct cfs_rq *cfs_rq, struct sched_entity *prev) if (prev->on_rq) { update_stats_wait_start_fair(cfs_rq, prev); + /* Put 'current' back into the tree. */ + __enqueue_entity(cfs_rq, prev); /* in !on_rq case, update occurred at dequeue */ update_load_avg(cfs_rq, prev, 0); } - WARN_ON_ONCE(cfs_rq->h_curr != prev); - cfs_rq->h_curr = NULL; + WARN_ON_ONCE(cfs_rq->curr != prev); + cfs_rq->curr = NULL; } static void @@ -7023,7 +6969,7 @@ void unthrottle_cfs_rq(struct cfs_rq *cfs_rq) assert_list_leaf_cfs_rq(rq); /* Determine whether we need to wake up potentially idle CPU: */ - if (rq->curr == rq->idle && rq->cfs.h_nr_queued) + if (rq->curr == rq->idle && rq->cfs.nr_queued) resched_curr(rq); } @@ -7358,7 +7304,7 @@ static void check_enqueue_throttle(struct cfs_rq *cfs_rq) return; /* an active group must be handled by the update_curr() path */ - if (!cfs_rq->runtime_enabled || cfs_rq->h_curr) + if (!cfs_rq->runtime_enabled || cfs_rq->curr) return; /* ensure the group is not already throttled */ @@ -7730,7 +7676,7 @@ static void hrtick_start_fair(struct rq *rq, struct task_struct *p) resched_curr(rq); return; } - delta = (se->h_load.weight * vdelta) / NICE_0_LOAD; + delta = (se->load.weight * vdelta) / NICE_0_LOAD; /* * Correct for instantaneous load of other classes. @@ -7830,8 +7776,10 @@ static int choose_idle_cpu(int cpu, struct task_struct *p) } static void -requeue_delayed_entity(struct cfs_rq *cfs_rq, struct sched_entity *se) +requeue_delayed_entity(struct sched_entity *se) { + struct cfs_rq *cfs_rq = cfs_rq_of(se); + /* * se->sched_delayed should imply: se->on_rq == 1. * Because a delayed entity is one that is still on @@ -7843,58 +7791,19 @@ requeue_delayed_entity(struct cfs_rq *cfs_rq, struct sched_entity *se) update_curr(cfs_rq); if (update_entity_lag(cfs_rq, se)) { - cfs_rq->h_nr_queued--; + cfs_rq->nr_queued--; if (se != cfs_rq->curr) __dequeue_entity(cfs_rq, se); place_entity(cfs_rq, se, 0); if (se != cfs_rq->curr) __enqueue_entity(cfs_rq, se); - cfs_rq->h_nr_queued++; + cfs_rq->nr_queued++; } update_load_avg(cfs_rq, se, 0); clear_delayed(se); } -static unsigned long enqueue_hierarchy(struct task_struct *p, int flags) -{ - unsigned long weight = NICE_0_LOAD; - int task_new = !(flags & ENQUEUE_WAKEUP); - struct sched_entity *se = &p->se; - int h_nr_idle = task_has_idle_policy(p); - int h_nr_runnable = 1; - - if (task_new && se->sched_delayed) - h_nr_runnable = 0; - - for_each_sched_entity(se) { - struct cfs_rq *cfs_rq = cfs_rq_of(se); - - update_curr(cfs_rq); - - if (!se->on_rq) { - enqueue_entity(cfs_rq, se, flags); - } else { - update_load_avg(cfs_rq, se, UPDATE_TG); - se_update_runnable(se); - update_cfs_group(se); - } - - cfs_rq->h_nr_runnable += h_nr_runnable; - cfs_rq->h_nr_queued++; - cfs_rq->h_nr_idle += h_nr_idle; - - if (cfs_rq_is_idle(cfs_rq)) - h_nr_idle = 1; - - weight = __calc_prop_weight(cfs_rq, se, weight); - - flags = ENQUEUE_WAKEUP; - } - - return weight; -} - /* * The enqueue_task method is called before nr_running is * increased. Here we update the fair scheduling stats and @@ -7903,12 +7812,13 @@ static unsigned long enqueue_hierarchy(struct task_struct *p, int flags) static void enqueue_task_fair(struct rq *rq, struct task_struct *p, int flags) { - int rq_h_nr_queued = rq->cfs.h_nr_queued; - int task_new = !(flags & ENQUEUE_WAKEUP); + struct cfs_rq *cfs_rq; struct sched_entity *se = &p->se; - struct cfs_rq *cfs_rq = &rq->cfs; - unsigned long weight; - bool curr; + int h_nr_idle = task_has_idle_policy(p); + int h_nr_runnable = 1; + int task_new = !(flags & ENQUEUE_WAKEUP); + int rq_h_nr_queued = rq->cfs.h_nr_queued; + u64 slice = 0; if (task_is_throttled(p) && enqueue_throttled_task(p)) return; @@ -7920,10 +7830,10 @@ enqueue_task_fair(struct rq *rq, struct task_struct *p, int flags) * estimated utilization, before we update schedutil. */ if (!p->se.sched_delayed || (flags & ENQUEUE_DELAYED)) - util_est_enqueue(cfs_rq, p); + util_est_enqueue(&rq->cfs, p); if (flags & ENQUEUE_DELAYED) { - requeue_delayed_entity(cfs_rq, se); + requeue_delayed_entity(se); return; } @@ -7935,22 +7845,57 @@ enqueue_task_fair(struct rq *rq, struct task_struct *p, int flags) if (p->in_iowait) cpufreq_update_util(rq, SCHED_CPUFREQ_IOWAIT); - /* - * XXX comment on the curr thing - */ - curr = (cfs_rq->curr == se); - if (curr) - place_entity(cfs_rq, se, flags); + if (task_new && se->sched_delayed) + h_nr_runnable = 0; - if (se->on_rq && se->sched_delayed) - requeue_delayed_entity(cfs_rq, se); + for_each_sched_entity(se) { + if (se->on_rq) { + if (se->sched_delayed) + requeue_delayed_entity(se); + break; + } + cfs_rq = cfs_rq_of(se); - weight = enqueue_hierarchy(p, flags); + /* + * Basically set the slice of group entries to the min_slice of + * their respective cfs_rq. This ensures the group can service + * its entities in the desired time-frame. + */ + if (slice) { + se->slice = slice; + se->custom_slice = 1; + } + enqueue_entity(cfs_rq, se, flags); + slice = cfs_rq_min_slice(cfs_rq); - if (!curr) { - reweight_eevdf(cfs_rq, se, weight, false); - place_entity(cfs_rq, se, flags | ENQUEUE_QUEUED); - __enqueue_entity(cfs_rq, se); + cfs_rq->h_nr_runnable += h_nr_runnable; + cfs_rq->h_nr_queued++; + cfs_rq->h_nr_idle += h_nr_idle; + + if (cfs_rq_is_idle(cfs_rq)) + h_nr_idle = 1; + + flags = ENQUEUE_WAKEUP; + } + + for_each_sched_entity(se) { + cfs_rq = cfs_rq_of(se); + + update_load_avg(cfs_rq, se, UPDATE_TG); + se_update_runnable(se); + update_cfs_group(se); + + se->slice = slice; + if (se != cfs_rq->curr) + min_vruntime_cb_propagate(&se->run_node, NULL); + slice = cfs_rq_min_slice(cfs_rq); + + cfs_rq->h_nr_runnable += h_nr_runnable; + cfs_rq->h_nr_queued++; + cfs_rq->h_nr_idle += h_nr_idle; + + if (cfs_rq_is_idle(cfs_rq)) + h_nr_idle = 1; } if (!rq_h_nr_queued && rq->cfs.h_nr_queued) @@ -7981,109 +7926,105 @@ enqueue_task_fair(struct rq *rq, struct task_struct *p, int flags) hrtick_update(rq); } -static void dequeue_hierarchy(struct task_struct *p, int flags) +/* + * Basically dequeue_task_fair(), except it can deal with dequeue_entity() + * failing half-way through and resume the dequeue later. + * + * Returns: + * -1 - dequeue delayed + * 0 - dequeue throttled + * 1 - dequeue complete + */ +static int dequeue_entities(struct rq *rq, struct sched_entity *se, int flags) { - struct sched_entity *se = &p->se; + bool was_sched_idle = sched_idle_rq(rq); bool task_sleep = flags & DEQUEUE_SLEEP; bool task_delayed = flags & DEQUEUE_DELAYED; bool task_throttled = flags & DEQUEUE_THROTTLE; + struct task_struct *p = NULL; + int h_nr_idle = 0; + int h_nr_queued = 0; int h_nr_runnable = 0; - int h_nr_idle = task_has_idle_policy(p); - bool dequeue = true; + struct cfs_rq *cfs_rq; + u64 slice = 0; - if (task_sleep || task_delayed || !se->sched_delayed) - h_nr_runnable = 1; + if (entity_is_task(se)) { + p = task_of(se); + h_nr_queued = 1; + h_nr_idle = task_has_idle_policy(p); + if (task_sleep || task_delayed || !se->sched_delayed) + h_nr_runnable = 1; + } for_each_sched_entity(se) { - struct cfs_rq *cfs_rq = cfs_rq_of(se); + cfs_rq = cfs_rq_of(se); - update_curr(cfs_rq); + if (!dequeue_entity(cfs_rq, se, flags)) { + if (p && &p->se == se) + return -1; - if (dequeue) { - dequeue_entity(cfs_rq, se, flags); - /* Don't dequeue parent if it has other entities besides us */ - if (cfs_rq->load.weight) - dequeue = false; - } else { - update_load_avg(cfs_rq, se, UPDATE_TG); - se_update_runnable(se); - update_cfs_group(se); + slice = cfs_rq_min_slice(cfs_rq); + break; } cfs_rq->h_nr_runnable -= h_nr_runnable; - cfs_rq->h_nr_queued--; + cfs_rq->h_nr_queued -= h_nr_queued; cfs_rq->h_nr_idle -= h_nr_idle; if (cfs_rq_is_idle(cfs_rq)) - h_nr_idle = 1; + h_nr_idle = h_nr_queued; if (throttled_hierarchy(cfs_rq) && task_throttled) record_throttle_clock(cfs_rq); + /* Don't dequeue parent if it has other entities besides us */ + if (cfs_rq->load.weight) { + slice = cfs_rq_min_slice(cfs_rq); + + /* Avoid re-evaluating load for this entity: */ + se = parent_entity(se); + /* + * Bias pick_next to pick a task from this cfs_rq, as + * p is sleeping when it is within its sched_slice. + */ + if (task_sleep && se) + set_next_buddy(se); + break; + } flags |= DEQUEUE_SLEEP; flags &= ~(DEQUEUE_DELAYED | DEQUEUE_SPECIAL); } -} -/* - * The part of dequeue_task_fair() that is needed to dequeue delayed tasks. - * - * Returns: - * true - dequeued - * false - delayed - */ -static bool __dequeue_task(struct rq *rq, struct task_struct *p, int flags) -{ - struct sched_entity *se = &p->se; - struct cfs_rq *cfs_rq = &rq->cfs; - bool was_sched_idle = sched_idle_rq(rq); - bool task_sleep = flags & DEQUEUE_SLEEP; - bool task_delayed = flags & DEQUEUE_DELAYED; + for_each_sched_entity(se) { + cfs_rq = cfs_rq_of(se); - clear_buddies(cfs_rq, se); + update_load_avg(cfs_rq, se, UPDATE_TG); + se_update_runnable(se); + update_cfs_group(se); - update_curr(cfs_rq_of(se)); - update_entity_lag(cfs_rq, se); + se->slice = slice; + if (se != cfs_rq->curr) + min_vruntime_cb_propagate(&se->run_node, NULL); + slice = cfs_rq_min_slice(cfs_rq); - if (flags & DEQUEUE_DELAYED) { - WARN_ON_ONCE(!se->sched_delayed); - } else { - bool delay = task_sleep; - /* - * DELAY_DEQUEUE relies on spurious wakeups, special task - * states must not suffer spurious wakeups, excempt them. - */ - if (flags & (DEQUEUE_SPECIAL | DEQUEUE_THROTTLE)) - delay = false; + cfs_rq->h_nr_runnable -= h_nr_runnable; + cfs_rq->h_nr_queued -= h_nr_queued; + cfs_rq->h_nr_idle -= h_nr_idle; - WARN_ON_ONCE(delay && se->sched_delayed); + if (cfs_rq_is_idle(cfs_rq)) + h_nr_idle = h_nr_queued; - if (sched_feat(DELAY_DEQUEUE) && delay && - !entity_eligible(cfs_rq, se)) { - update_load_avg(cfs_rq_of(se), se, UPDATE_UTIL_EST); - set_delayed(se); - return false; - } + if (throttled_hierarchy(cfs_rq) && task_throttled) + record_throttle_clock(cfs_rq); } - dequeue_hierarchy(p, flags); - - if (sched_feat(PLACE_REL_DEADLINE) && !task_sleep) { - se->deadline -= se->vruntime; - se->rel_deadline = 1; - } - if (se != cfs_rq->curr) - __dequeue_entity(cfs_rq, se); - - sub_nr_running(rq, 1); + sub_nr_running(rq, h_nr_queued); /* balance early to pull high priority tasks */ if (unlikely(!was_sched_idle && sched_idle_rq(rq))) rq->next_balance = jiffies; - if (task_delayed) { - clear_delayed(se); - + if (p && task_delayed) { WARN_ON_ONCE(!task_sleep); WARN_ON_ONCE(p->on_rq != 1); @@ -8095,7 +8036,7 @@ static bool __dequeue_task(struct rq *rq, struct task_struct *p, int flags) __block_task(rq, p); } - return true; + return 1; } /* @@ -8113,11 +8054,11 @@ static bool dequeue_task_fair(struct rq *rq, struct task_struct *p, int flags) if (!p->se.sched_delayed) util_est_dequeue(&rq->cfs, p); - if (!__dequeue_task(rq, p, flags)) + if (dequeue_entities(rq, &p->se, flags) < 0) return false; /* - * Must not reference @p after __dequeue_task(DEQUEUE_DELAYED). + * Must not reference @p after dequeue_entities(DEQUEUE_DELAYED). */ return true; } @@ -9712,6 +9653,19 @@ static void migrate_task_rq_fair(struct task_struct *p, int new_cpu) static void task_dead_fair(struct task_struct *p) { struct sched_entity *se = &p->se; + + if (se->sched_delayed) { + struct rq_flags rf; + struct rq *rq; + + rq = task_rq_lock(p, &rf); + if (se->sched_delayed) { + update_rq_clock(rq); + dequeue_entities(rq, se, DEQUEUE_SLEEP | DEQUEUE_DELAYED); + } + task_rq_unlock(rq, p, &rf); + } + remove_entity_load_avg(se); } @@ -9745,10 +9699,21 @@ static void set_cpus_allowed_fair(struct task_struct *p, struct affinity_context set_task_max_allowed_capacity(p); } +static void set_next_buddy(struct sched_entity *se) +{ + for_each_sched_entity(se) { + if (WARN_ON_ONCE(!se->on_rq)) + return; + if (se_is_idle(se)) + return; + cfs_rq_of(se)->next = se; + } +} + enum preempt_wakeup_action { PREEMPT_WAKEUP_NONE, /* No preemption. */ PREEMPT_WAKEUP_SHORT, /* Ignore slice protection. */ - PREEMPT_WAKEUP_PICK, /* Let pick_eevdf() decide. */ + PREEMPT_WAKEUP_PICK, /* Let __pick_eevdf() decide. */ PREEMPT_WAKEUP_RESCHED, /* Force reschedule. */ }; @@ -9765,7 +9730,7 @@ set_preempt_buddy(struct cfs_rq *cfs_rq, int wake_flags, if (cfs_rq->next && entity_before(cfs_rq->next, pse)) return false; - set_next_buddy(cfs_rq, pse); + set_next_buddy(pse); return true; } @@ -9818,7 +9783,7 @@ static void wakeup_preempt_fair(struct rq *rq, struct task_struct *p, int wake_f enum preempt_wakeup_action preempt_action = PREEMPT_WAKEUP_PICK; struct task_struct *donor = rq->donor; struct sched_entity *nse, *se = &donor->se, *pse = &p->se; - struct cfs_rq *cfs_rq = &rq->cfs; + struct cfs_rq *cfs_rq = task_cfs_rq(donor); int cse_is_idle, pse_is_idle; /* @@ -9855,6 +9820,7 @@ static void wakeup_preempt_fair(struct rq *rq, struct task_struct *p, int wake_f if (!sched_feat(WAKEUP_PREEMPTION)) return; + find_matching_se(&se, &pse); WARN_ON_ONCE(!pse); cse_is_idle = se_is_idle(se); @@ -9882,7 +9848,8 @@ static void wakeup_preempt_fair(struct rq *rq, struct task_struct *p, int wake_f if (unlikely(!normal_policy(p->policy))) return; - update_curr_fair(rq); + cfs_rq = cfs_rq_of(se); + update_curr(cfs_rq); /* * If @p has a shorter slice than current and @p is eligible, override * current's slice protection in order to allow preemption. @@ -9926,15 +9893,18 @@ static void wakeup_preempt_fair(struct rq *rq, struct task_struct *p, int wake_f } pick: - if (cfs_rq->h_nr_queued) { - nse = pick_next_entity(rq, preempt_action != PREEMPT_WAKEUP_SHORT); - if (unlikely(!nse)) - goto pick; + nse = pick_next_entity(rq, cfs_rq, preempt_action != PREEMPT_WAKEUP_SHORT); + /* If @p has become the most eligible task, force preemption */ + if (nse == pse) + goto preempt; - /* If @p has become the most eligible task, force preemption */ - if (nse == pse) - goto preempt; - } + /* + * Because p is enqueued, nse being null can only mean that we + * dequeued a delayed task. If there are still entities queued in + * cfs, check if the next one will be p. + */ + if (!nse && cfs_rq->nr_queued) + goto pick; if (sched_feat(RUN_TO_PARITY)) update_protect_slice(cfs_rq, se); @@ -9953,24 +9923,33 @@ static void wakeup_preempt_fair(struct rq *rq, struct task_struct *p, int wake_f struct task_struct *pick_task_fair(struct rq *rq, struct rq_flags *rf) __must_hold(__rq_lockp(rq)) { - struct cfs_rq *cfs_rq = &rq->cfs; struct sched_entity *se; + struct cfs_rq *cfs_rq; struct task_struct *p; + bool throttled; int new_tasks; again: - if (!cfs_rq->h_nr_queued) + cfs_rq = &rq->cfs; + if (!cfs_rq->nr_queued) goto idle; - /* Might not have done put_prev_entity() */ - if (cfs_rq->curr && cfs_rq->curr->on_rq) - update_curr(cfs_rq); + throttled = false; - se = pick_next_entity(rq, true); - if (!se) - goto again; + do { + /* Might not have done put_prev_entity() */ + if (cfs_rq->curr && cfs_rq->curr->on_rq) + update_curr(cfs_rq); + + se = pick_next_entity(rq, cfs_rq, true); + if (!se) + goto again; + cfs_rq = group_cfs_rq(se); + } while (cfs_rq); p = task_of(se); + if (unlikely(throttled)) + task_throttle_setup_work(p); return p; idle: @@ -10007,7 +9986,7 @@ void fair_server_init(struct rq *rq) static void put_prev_task_fair(struct rq *rq, struct task_struct *prev, struct task_struct *next) { struct sched_entity *se = &prev->se; - struct cfs_rq *cfs_rq = &rq->cfs; + struct cfs_rq *cfs_rq; struct sched_entity *nse = NULL; #ifdef CONFIG_FAIR_GROUP_SCHED @@ -10017,7 +9996,7 @@ static void put_prev_task_fair(struct rq *rq, struct task_struct *prev, struct t while (se) { cfs_rq = cfs_rq_of(se); - if (!nse || cfs_rq->h_curr) + if (!nse || cfs_rq->curr) put_prev_entity(cfs_rq, se); #ifdef CONFIG_FAIR_GROUP_SCHED if (nse) { @@ -10036,14 +10015,6 @@ static void put_prev_task_fair(struct rq *rq, struct task_struct *prev, struct t #endif se = parent_entity(se); } - - /* Put 'current' back into the tree. */ - cfs_rq = &rq->cfs; - se = &prev->se; - WARN_ON_ONCE(cfs_rq->curr != se); - cfs_rq->curr = NULL; - if (se->on_rq) - __enqueue_entity(cfs_rq, se); } /* @@ -10052,8 +10023,8 @@ static void put_prev_task_fair(struct rq *rq, struct task_struct *prev, struct t static void yield_task_fair(struct rq *rq) { struct task_struct *curr = rq->donor; + struct cfs_rq *cfs_rq = task_cfs_rq(curr); struct sched_entity *se = &curr->se; - struct cfs_rq *cfs_rq = &rq->cfs; /* * Are we the only task in the tree? @@ -10094,11 +10065,11 @@ static bool yield_to_task_fair(struct rq *rq, struct task_struct *p) struct sched_entity *se = &p->se; /* !se->on_rq also covers throttled task */ - if (!se->on_rq || se->sched_delayed) + if (!se->on_rq) return false; /* Tell the scheduler that we'd really like se to run next. */ - set_next_buddy(&task_rq(p)->cfs, se); + set_next_buddy(se); yield_task_fair(rq); @@ -10437,10 +10408,15 @@ static inline long migrate_degrades_locality(struct task_struct *p, */ static inline int task_is_ineligible_on_dst_cpu(struct task_struct *p, int dest_cpu) { - struct cfs_rq *dst_cfs_rq = &cpu_rq(dest_cpu)->cfs; + struct cfs_rq *dst_cfs_rq; - if (sched_feat(PLACE_LAG) && dst_cfs_rq->h_nr_queued && - !entity_eligible(&task_rq(p)->cfs, &p->se)) +#ifdef CONFIG_FAIR_GROUP_SCHED + dst_cfs_rq = tg_cfs_rq(task_group(p), dest_cpu); +#else + dst_cfs_rq = &cpu_rq(dest_cpu)->cfs; +#endif + if (sched_feat(PLACE_LAG) && dst_cfs_rq->nr_queued && + !entity_eligible(task_cfs_rq(p), &p->se)) return 1; return 0; @@ -11223,7 +11199,7 @@ static void update_cfs_rq_h_load(struct cfs_rq *cfs_rq) while ((se = READ_ONCE(cfs_rq->h_load_next)) != NULL) { load = cfs_rq->h_load; load = div64_ul(load * se->avg.load_avg, - cfs_rq_load_avg(cfs_rq) + 1); + cfs_rq_load_avg(cfs_rq) + 1); cfs_rq = group_cfs_rq(se); cfs_rq->h_load = load; cfs_rq->last_h_load_update = now; @@ -14618,7 +14594,7 @@ static inline void task_tick_core(struct rq *rq, struct task_struct *curr) * MIN_NR_TASKS_DURING_FORCEIDLE - 1 tasks and use that to check * if we need to give up the CPU. */ - if (rq->core->core_forceidle_count && rq->cfs.h_nr_queued == 1 && + if (rq->core->core_forceidle_count && rq->cfs.nr_queued == 1 && __entity_slice_used(&curr->se, MIN_NR_TASKS_DURING_FORCEIDLE)) resched_curr(rq); } @@ -14827,8 +14803,30 @@ bool cfs_prio_less(const struct task_struct *a, const struct task_struct *b, WARN_ON_ONCE(task_rq(b)->core != rq->core); +#ifdef CONFIG_FAIR_GROUP_SCHED + /* + * Find an se in the hierarchy for tasks a and b, such that the se's + * are immediate siblings. + */ + while (sea->cfs_rq->tg != seb->cfs_rq->tg) { + int sea_depth = sea->depth; + int seb_depth = seb->depth; + + if (sea_depth >= seb_depth) + sea = parent_entity(sea); + if (sea_depth <= seb_depth) + seb = parent_entity(seb); + } + + se_fi_update(sea, rq->core->core_forceidle_seq, in_fi); + se_fi_update(seb, rq->core->core_forceidle_seq, in_fi); + + cfs_rqa = sea->cfs_rq; + cfs_rqb = seb->cfs_rq; +#else /* !CONFIG_FAIR_GROUP_SCHED: */ cfs_rqa = &task_rq(a)->cfs; cfs_rqb = &task_rq(b)->cfs; +#endif /* !CONFIG_FAIR_GROUP_SCHED */ /* * Find delta after normalizing se's vruntime with its cfs_rq's @@ -14867,20 +14865,11 @@ static inline void task_tick_core(struct rq *rq, struct task_struct *curr) {} static void task_tick_fair(struct rq *rq, struct task_struct *curr, int queued) { struct sched_entity *se = &curr->se; + struct cfs_rq *cfs_rq; - if (se->on_rq) { - unsigned long weight = NICE_0_LOAD; - struct cfs_rq *cfs_rq; - - for_each_sched_entity(se) { - cfs_rq = cfs_rq_of(se); - entity_tick(cfs_rq, se, queued); - - weight = __calc_prop_weight(cfs_rq, se, weight); - } - - se = &curr->se; - reweight_eevdf(cfs_rq, se, weight, se->on_rq); + for_each_sched_entity(se) { + cfs_rq = cfs_rq_of(se); + entity_tick(cfs_rq, se, queued); } if (queued) @@ -14920,7 +14909,7 @@ prio_changed_fair(struct rq *rq, struct task_struct *p, u64 oldprio) if (p->prio == oldprio) return; - if (rq->cfs.h_nr_queued == 1) + if (rq->cfs.nr_queued == 1) return; /* @@ -15049,44 +15038,33 @@ static void switched_to_fair(struct rq *rq, struct task_struct *p) } } +/* + * Account for a task changing its policy or group. + * + * This routine is mostly called to set cfs_rq->curr field when a task + * migrates between groups/classes. + */ static void set_next_task_fair(struct rq *rq, struct task_struct *p, bool first) { struct sched_entity *se = &p->se; bool throttled = false; - struct cfs_rq *cfs_rq = &rq->cfs; - unsigned long weight = NICE_0_LOAD; - bool on_rq = se->on_rq; - - clear_buddies(cfs_rq, se); - - if (on_rq) - __dequeue_entity(cfs_rq, se); for_each_sched_entity(se) { - cfs_rq = cfs_rq_of(se); + struct cfs_rq *cfs_rq = cfs_rq_of(se); - if (!IS_ENABLED(CONFIG_FAIR_GROUP_SCHED) || - !first || !cfs_rq->h_curr) - set_next_entity(cfs_rq, se); + if (IS_ENABLED(CONFIG_FAIR_GROUP_SCHED) && + first && cfs_rq->curr) + break; + set_next_entity(cfs_rq, se, first); /* ensure bandwidth has been allocated on our new cfs_rq */ throttled |= account_cfs_rq_runtime(cfs_rq, 0); - - if (on_rq) - weight = __calc_prop_weight(cfs_rq, se, weight); } if (throttled) task_throttle_setup_work(p); se = &p->se; - cfs_rq->curr = se; - - if (on_rq) { - reweight_eevdf(cfs_rq, se, weight, se->on_rq); - if (first) - set_protect_slice(cfs_rq, se); - } if (task_on_rq_queued(p)) { /* @@ -15199,8 +15177,17 @@ void unregister_fair_sched_group(struct task_group *tg) struct sched_entity *se = tg_se(tg, cpu); struct rq *rq = cpu_rq(cpu); - if (se) + if (se) { + if (se->sched_delayed) { + guard(rq_lock_irqsave)(rq); + if (se->sched_delayed) { + update_rq_clock(rq); + dequeue_entities(rq, se, DEQUEUE_SLEEP | DEQUEUE_DELAYED); + } + list_del_leaf_cfs_rq(cfs_rq); + } remove_entity_load_avg(se); + } /* * Only empty task groups can be destroyed; so we can speculatively diff --git a/kernel/sched/pelt.c b/kernel/sched/pelt.c index 779eb58a4..897790889 100644 --- a/kernel/sched/pelt.c +++ b/kernel/sched/pelt.c @@ -206,7 +206,7 @@ ___update_load_sum(u64 now, struct sched_avg *sa, /* * running is a subset of runnable (weight) so running can't be set if * runnable is clear. But there are some corner cases where the current - * se has been already dequeued but cfs_rq->h_curr still points to it. + * se has been already dequeued but cfs_rq->curr still points to it. * This means that weight will be 0 but not running for a sched_entity * but also for a cfs_rq if the latter becomes idle. As an example, * this happens during sched_balance_newidle() which calls @@ -307,7 +307,7 @@ int __update_load_avg_blocked_se(u64 now, struct sched_entity *se) int __update_load_avg_se(u64 now, struct cfs_rq *cfs_rq, struct sched_entity *se) { if (___update_load_sum(now, &se->avg, !!se->on_rq, se_runnable(se), - cfs_rq->h_curr == se)) { + cfs_rq->curr == se)) { ___update_load_avg(&se->avg, se_weight(se)); cfs_se_util_change(&se->avg); @@ -323,7 +323,7 @@ int __update_load_avg_cfs_rq(u64 now, struct cfs_rq *cfs_rq) if (___update_load_sum(now, &cfs_rq->avg, scale_load_down(cfs_rq->load.weight), cfs_rq->h_nr_runnable, - cfs_rq->h_curr != NULL)) { + cfs_rq->curr != NULL)) { ___update_load_avg(&cfs_rq->avg, 1); trace_pelt_cfs_tp(cfs_rq); diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h index 4664869d8..a4a1eb57a 100644 --- a/kernel/sched/sched.h +++ b/kernel/sched/sched.h @@ -493,8 +493,6 @@ struct task_group { * will also be accessed at each tick. */ atomic_long_t load_avg ____cacheline_aligned; - atomic_long_t runnable_avg; - #endif /* CONFIG_FAIR_GROUP_SCHED */ #ifdef CONFIG_RT_GROUP_SCHED @@ -530,8 +528,21 @@ struct task_group { }; +#ifdef CONFIG_GROUP_SCHED_WEIGHT #define ROOT_TASK_GROUP_LOAD NICE_0_LOAD +/* + * A weight of 0 or 1 can cause arithmetics problems. + * A weight of a cfs_rq is the sum of weights of which entities + * are queued on this cfs_rq, so a weight of a entity should not be + * too large, so as the shares value of a task group. + * (The default weight is 1024 - so there's no practical + * limitation from this.) + */ +#define MIN_SHARES (1UL << 1) +#define MAX_SHARES (1UL << 18) +#endif + typedef int (*tg_visitor)(struct task_group *, void *); extern int walk_tg_tree_from(struct task_group *from, @@ -560,7 +571,6 @@ extern void free_fair_sched_group(struct task_group *tg); extern int alloc_fair_sched_group(struct task_group *tg, struct task_group *parent); extern void online_fair_sched_group(struct task_group *tg); extern void unregister_fair_sched_group(struct task_group *tg); -extern void __sched_cgroup_mode_update(int mode); #else /* !CONFIG_FAIR_GROUP_SCHED: */ static inline void free_fair_sched_group(struct task_group *tg) { } static inline int alloc_fair_sched_group(struct task_group *tg, struct task_group *parent) @@ -618,17 +628,6 @@ static inline bool cfs_task_bw_constrained(struct task_struct *p) { return false #endif /* !CONFIG_CGROUP_SCHED */ -/* - * A weight of 0 or 1 can cause arithmetics problems. - * A weight of a cfs_rq is the sum of weights of which entities - * are queued on this cfs_rq, so a weight of a entity should not be - * too large, so as the shares value of a task group. - * (The default weight is 1024 - so there's no practical - * limitation from this.) - */ -#define MIN_SHARES (1UL << 1) -#define MAX_SHARES (1UL << 18) - extern void unregister_rt_sched_group(struct task_group *tg); extern void free_rt_sched_group(struct task_group *tg); extern int alloc_rt_sched_group(struct task_group *tg, struct task_group *parent); @@ -707,7 +706,6 @@ struct cfs_rq { /* * CFS load tracking */ - struct sched_entity *h_curr; struct sched_avg avg; #ifndef CONFIG_64BIT u64 last_update_time_copy; @@ -723,7 +721,6 @@ struct cfs_rq { #ifdef CONFIG_FAIR_GROUP_SCHED u64 last_update_tg_load_avg; unsigned long tg_load_avg_contrib; - unsigned long tg_runnable_avg_contrib; long propagate; long prop_runnable_sum; @@ -2574,7 +2571,6 @@ extern const u32 sched_prio_to_wmult[40]; #define ENQUEUE_MIGRATED 0x00040000 #define ENQUEUE_INITIAL 0x00080000 #define ENQUEUE_RQ_SELECTED 0x00100000 -#define ENQUEUE_QUEUED 0x00200000 #define RETRY_TASK ((void *)-1UL) -- 2.55.0 From e5dd11e52ec3c3bc3139b03cfe01263de67d48d5 Mon Sep 17 00:00:00 2001 From: Piotr Gorski Date: Mon, 17 Aug 2026 22:57:38 +0200 Subject: [PATCH 3/5] muqss-cachy Signed-off-by: Piotr Gorski --- Makefile | 4 + arch/m68k/coldfire/pci.c | 2 +- arch/powerpc/platforms/powernv/vas-window.c | 4 +- block/Makefile | 1 + block/bio.c | 12 +- block/blk-iotime.c | 749 ++ block/blk-merge.c | 3 + block/blk-mq-debugfs.c | 2 + block/blk-mq.c | 58 +- block/blk-rq-qos.h | 1 + block/blk-sysfs.c | 2 + .../accessibility/speakup/speakup_acntpc.c | 4 +- .../accessibility/speakup/speakup_apollo.c | 8 +- .../accessibility/speakup/speakup_decext.c | 5 +- drivers/accessibility/speakup/speakup_decpc.c | 5 +- .../accessibility/speakup/speakup_dectlk.c | 5 +- drivers/accessibility/speakup/speakup_dtlk.c | 4 +- drivers/accessibility/speakup/speakup_keypc.c | 4 +- drivers/accessibility/speakup/synth.c | 8 +- drivers/auxdisplay/hd44780_common.c | 2 +- drivers/block/swim.c | 6 +- drivers/char/ipmi/ipmi_msghandler.c | 2 +- drivers/char/ipmi/ipmi_ssif.c | 2 +- drivers/comedi/drivers/ni_mio_common.c | 2 +- drivers/crypto/ccp/tee-dev.c | 2 +- drivers/gpib/nec7210/nec7210.c | 2 +- drivers/gpu/drm/vmwgfx/vmwgfx_cmd.c | 2 +- drivers/gpu/drm/vmwgfx/vmwgfx_irq.c | 2 +- drivers/hwmon/adt7470.c | 2 +- drivers/hwmon/fam15h_power.c | 2 +- drivers/hwmon/ibmaem.c | 5 +- drivers/iio/light/tsl2563.c | 6 +- drivers/md/bcache/btree.c | 3 +- drivers/media/i2c/bt866.c | 2 +- drivers/media/i2c/ir-kbd-i2c.c | 2 +- drivers/media/i2c/msp3400-driver.c | 4 +- drivers/media/i2c/saa7110.c | 4 +- drivers/media/pci/cx18/cx18-gpio.c | 4 +- drivers/media/pci/hws/hws_pci.c | 4 +- drivers/media/pci/ivtv/ivtv-gpio.c | 6 +- drivers/media/pci/ivtv/ivtv-ioctl.c | 2 +- drivers/media/pci/ivtv/ivtv-streams.c | 2 +- drivers/media/pci/ivtv/ivtvfb.c | 2 +- drivers/media/pci/saa7134/saa7134-tvaudio.c | 3 +- drivers/media/radio/radio-mr800.c | 2 +- drivers/media/radio/radio-tea5777.c | 2 +- drivers/media/radio/tea575x.c | 2 +- drivers/media/usb/gspca/m5602/m5602_s5k83a.c | 2 +- drivers/mfd/ucb1x00-core.c | 2 +- drivers/misc/mei/hw-me.c | 2 +- drivers/misc/sgi-xp/xpc_channel.c | 2 +- drivers/net/can/usb/peak_usb/pcan_usb.c | 2 +- .../ethernet/cavium/liquidio/lio_ethtool.c | 2 +- .../net/ethernet/cavium/liquidio/lio_main.c | 2 +- drivers/net/usb/lan78xx.c | 2 +- drivers/net/usb/usbnet.c | 2 +- drivers/net/wireless/intel/ipw2x00/ipw2100.c | 4 +- drivers/parport/ieee1284.c | 2 +- drivers/parport/ieee1284_ops.c | 2 +- drivers/parport/parport_pc.c | 3 +- drivers/platform/x86/intel_ips.c | 8 +- drivers/rtc/rtc-wm8350.c | 6 +- drivers/s390/char/sclp_sdias.c | 2 +- drivers/scsi/be2iscsi/be_cmds.c | 2 +- drivers/scsi/esas2r/esas2r_flash.c | 2 +- drivers/scsi/esas2r/esas2r_init.c | 12 +- drivers/scsi/esas2r/esas2r_main.c | 2 +- drivers/scsi/fnic/fnic_scsi.c | 10 +- drivers/scsi/lpfc/lpfc_scsi.c | 4 +- drivers/scsi/qla4xxx/ql4_nx.c | 2 +- drivers/scsi/snic/snic_scsi.c | 4 +- drivers/usb/class/usblp.c | 2 +- drivers/video/fbdev/omap/hwa742.c | 2 +- drivers/video/fbdev/pxafb.c | 2 +- fs/f2fs/checkpoint.c | 2 +- fs/jffs2/background.c | 2 +- fs/proc/base.c | 110 +- fs/xfs/xfs_trans_ail.c | 2 +- include/linux/blk-mq.h | 17 + include/linux/blk_types.h | 16 +- include/linux/freezer.h | 1 + include/linux/ioprio.h | 2 + include/linux/mm.h | 55 + include/linux/mmzone.h | 3 + include/linux/muqss.h | 51 + include/linux/muqss_iotime.h | 166 + include/linux/page-flags-layout.h | 32 +- include/linux/sched.h | 90 + include/linux/sched/deadline.h | 24 +- include/linux/sched/nohz.h | 2 +- include/linux/sched/prio.h | 15 + include/linux/skip_list.h | 41 + include/linux/workqueue.h | 2 + include/linux/workqueue_types.h | 42 + include/uapi/linux/sched.h | 2 +- init/Kconfig | 15 +- init/init_task.c | 10 + io_uring/io_uring.c | 13 + kernel/Kconfig.MuQSS | 219 + kernel/Kconfig.hz | 62 +- kernel/Kconfig.preempt | 50 +- kernel/cgroup/cpuset.c | 2 +- kernel/delayacct.c | 2 +- kernel/exit.c | 10 +- kernel/locking/rtmutex.c | 4 +- kernel/sched/Makefile | 7 + kernel/sched/MuQSS.c | 10778 ++++++++++++++++ kernel/sched/MuQSS.h | 1253 ++ kernel/sched/build_muqss.c | 131 + kernel/sched/cputime.c | 19 +- kernel/sched/idle.c | 2 + kernel/sched/sched.h | 6 + kernel/sched/skip_list.c | 187 + kernel/sched/stats.h | 21 + kernel/time/Kconfig | 29 +- kernel/time/clockevents.c | 4 +- kernel/time/posix-cpu-timers.c | 8 +- kernel/time/sleep_timeout.c | 222 +- kernel/time/tick-internal.h | 3 + kernel/trace/rv/monitors/deadline/Kconfig | 2 +- kernel/workqueue.c | 37 +- mm/khugepaged.c | 2 +- mm/page-writeback.c | 7 + mm/page_alloc.c | 1 + net/core/pktgen.c | 8 +- sound/hda/core/controller.c | 2 +- sound/isa/msnd/msnd_pinnacle.c | 2 +- sound/pci/maestro3.c | 6 +- sound/pci/ymfpci/ymfpci_main.c | 2 +- sound/soc/codecs/rt5631.c | 10 +- sound/soc/codecs/wm8350.c | 21 +- sound/soc/codecs/wm8900.c | 6 +- sound/soc/codecs/wm9713.c | 4 +- sound/soc/soc-dapm.c | 2 +- sound/usb/line6/pcm.c | 2 +- 135 files changed, 14681 insertions(+), 218 deletions(-) create mode 100644 block/blk-iotime.c create mode 100644 include/linux/muqss.h create mode 100644 include/linux/muqss_iotime.h create mode 100644 include/linux/skip_list.h create mode 100644 kernel/Kconfig.MuQSS create mode 100644 kernel/sched/MuQSS.c create mode 100644 kernel/sched/MuQSS.h create mode 100644 kernel/sched/build_muqss.c create mode 100644 kernel/sched/skip_list.c diff --git a/Makefile b/Makefile index ad0caa9b4..1c2689a1b 100644 --- a/Makefile +++ b/Makefile @@ -22,6 +22,10 @@ $(if $(filter __%, $(MAKECMDGOALS)), \ PHONY := __all __all: +CKVERSION = -ck1 +CKNAME = MuQSS Powered +EXTRAVERSION := $(EXTRAVERSION)$(CKVERSION) + # We are using a recursive build, so we need to do a little thinking # to get the ordering right. # diff --git a/arch/m68k/coldfire/pci.c b/arch/m68k/coldfire/pci.c index 5afb207cc..5606bc0a9 100644 --- a/arch/m68k/coldfire/pci.c +++ b/arch/m68k/coldfire/pci.c @@ -226,7 +226,7 @@ static int __init mcf_pci_init(void) /* Turn of PCI reset, and wait for devices to settle */ mcf_write32(0, PCIGSCR); set_current_state(TASK_UNINTERRUPTIBLE); - schedule_timeout(msecs_to_jiffies(200)); + schedule_msec_hrtimeout(200); pci_add_resource(&bridge->windows, &ioport_resource); diff --git a/arch/powerpc/platforms/powernv/vas-window.c b/arch/powerpc/platforms/powernv/vas-window.c index 9f093176b..8de2dd4af 100644 --- a/arch/powerpc/platforms/powernv/vas-window.c +++ b/arch/powerpc/platforms/powernv/vas-window.c @@ -1185,7 +1185,7 @@ static void poll_window_credits(struct pnv_vas_window *window) if (creds < window->vas_win.wcreds_max) { val = 0; set_current_state(TASK_UNINTERRUPTIBLE); - schedule_timeout(msecs_to_jiffies(10)); + schedule_msec_hrtimeout(10); count++; /* * Process can not close send window until all credits are @@ -1218,7 +1218,7 @@ static void poll_window_busy_state(struct pnv_vas_window *window) if (busy) { val = 0; set_current_state(TASK_UNINTERRUPTIBLE); - schedule_timeout(msecs_to_jiffies(10)); + schedule_msec_hrtimeout(10); count++; /* * Takes around few milliseconds to process all pending diff --git a/block/Makefile b/block/Makefile index 5523e5f62..326b576fb 100644 --- a/block/Makefile +++ b/block/Makefile @@ -14,6 +14,7 @@ obj-y := bdev.o fops.o bio.o elevator.o blk-core.o blk-sysfs.o \ disk-events.o blk-ia-ranges.o early-lookup.o obj-$(CONFIG_BLK_ERROR_INJECTION) += error-injection.o +obj-$(CONFIG_MUQSS_IOTIME) += blk-iotime.o obj-$(CONFIG_BLK_DEV_BSG_COMMON) += bsg.o obj-$(CONFIG_BLK_DEV_BSGLIB) += bsg-lib.o obj-$(CONFIG_BLK_CGROUP) += blk-cgroup.o diff --git a/block/bio.c b/block/bio.c index 6a2f6fc34..4bda05e09 100644 --- a/block/bio.c +++ b/block/bio.c @@ -19,6 +19,7 @@ #include #include #include +#include #include #include "blk.h" @@ -185,6 +186,8 @@ void bio_uninit(struct bio *bio) bio->bi_blkg = NULL; } #endif + muqss_iotime_put_owner(bio); + if (bio_integrity(bio)) bio_integrity_free(bio); @@ -234,13 +237,18 @@ void bio_init(struct bio *bio, struct block_device *bdev, struct bio_vec *table, bio->bi_private = NULL; #ifdef CONFIG_BLK_CGROUP bio->bi_blkg = NULL; - bio->issue_time_ns = 0; if (bdev) bio_associate_blkg(bio); #ifdef CONFIG_BLK_CGROUP_IOCOST bio->bi_iocost_cost = 0; #endif #endif +#if defined(CONFIG_BLK_CGROUP) || defined(CONFIG_MUQSS_IOTIME) + bio->issue_time_ns = 0; +#endif +#ifdef CONFIG_MUQSS_IOTIME + bio->bi_muqss_owner = NULL; +#endif #ifdef CONFIG_BLK_INLINE_ENCRYPTION bio->bi_crypt_context = NULL; #endif @@ -867,6 +875,7 @@ static int __bio_clone(struct bio *bio, struct bio *bio_src, gfp_t gfp) bio_set_flag(bio, BIO_REMAPPED); bio_clone_blkg_association(bio, bio_src); } + muqss_iotime_clone_owner(bio, bio_src); if (bio_crypt_clone(bio, bio_src, gfp) < 0) return -ENOMEM; @@ -1814,6 +1823,7 @@ void bio_endio(struct bio *bio) bio->bi_blkg = NULL; } #endif + muqss_iotime_put_owner(bio); if (bio->bi_end_io) bio->bi_end_io(bio); diff --git a/block/blk-iotime.c b/block/blk-iotime.c new file mode 100644 index 000000000..edf4d683f --- /dev/null +++ b/block/blk-iotime.c @@ -0,0 +1,749 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * I/O aware scheduling support for MuQSS. + * + * The CPU scheduler can see how much CPU a task uses but nothing of what it + * costs the storage device. This attaches an rq_qos policy that attributes + * block device time back to the task that caused the I/O, so that cost is + * visible to the scheduler. + * + * Two different quantities are recorded and deliberately kept apart: + * + * Latency is what a waiting task experiences, measured per bio from + * bio->issue_time_ns, which is stamped in blk_mq_submit_bio() before merging + * and request allocation. It therefore includes plug, merge and queue wait as + * well as service. + * + * Occupancy is what the task cost everybody else, measured per request from + * rq->io_start_time_ns at dispatch to rq->muqss_done_ns at completion. This is + * the quantity the scheduler charges for. + * + * Attribution has two halves. Reads and synchronous writes are submitted by + * the originating task, so the owner is simply current. Writeback is not: the + * dirtier is recorded on the folio when it is dirtied and recovered when the + * flusher thread submits the I/O. See the owner table below. + * + * The owner table earns its keep a second time over. A kworker running a work + * item declares itself to be acting for whoever queued that work, which both + * attributes the I/O the work item issues and lets the CPU time it costs be + * charged back as kern_debt_ns. See muqss_kerntime_begin(). + * + * The occupancy is charged one for one against the deadline, with nothing to + * tune it by: the accounting below and the scheduling it feeds are one + * feature, and only CONFIG_MUQSS_IOTIME=n removes either. See + * consume_iotime_penalty() in kernel/sched/MuQSS.c and + * MuQSS-iotime-design.md. + */ +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "blk.h" +#include "blk-mq.h" +#include "blk-mq-debugfs.h" +#include "blk-rq-qos.h" +#include "blk-stat.h" + +struct blk_iotime { + struct rq_qos rqos; + + /* Unattributed device busy time, see comment above. */ + atomic64_t occupancy_ns; + atomic64_t requests; + + /* + * Bios that completed with no owner recorded. Counted so the size of + * the attribution gap stays visible rather than silently skewing the + * per task numbers. + */ + atomic64_t unattributed; + + /* Requests disowned because a merge brought in another task's bio. */ + atomic64_t mixed; + + /* + * Requests that reached completion with no captured timestamp, and so + * could not be measured at all. See muqss_iotime_done(). + */ + atomic64_t unstamped; + + /* Bios that completed with an owner but no submit timestamp. */ + atomic64_t nostamp_bio; +}; + +static inline struct blk_iotime *BLK_IOTIME(struct rq_qos *rqos) +{ + return container_of(rqos, struct blk_iotime, rqos); +} + +/* + * Owner table. + * + * Reads are attributable at submit time because the submitting task is the + * originating one. Writeback is not: the folio is dirtied by one task and + * written out much later by a flusher thread. The dirtier is therefore + * recorded on the folio itself, in spare page->flags bits, as an index into + * this table. + * + * Only MUQSS_IOWNER_WIDTH bits are available, so the table is small and the + * index carries no generation. Slot 0 is reserved to mean "no owner". A slot + * is held from the task's first dirty until it exits, and the table holds a + * reference for that whole period, so a resolved pointer is always a live + * task_struct. + * + * Slots are released on exit and never stolen from a live task, so a resolved + * index is never charged to the wrong task. The cost of that choice is that + * beyond IOWNER_SLOTS concurrent dirtiers, further tasks get no slot and their + * writeback goes unattributed; iowner_exhausted counts it. + * + * Writeback still in flight for a task that has exited resolves to nothing, + * which is correct: there is no longer anybody to charge. + */ +#define IOWNER_SLOTS (1U << MUQSS_IOWNER_SHIFT) + +static struct task_struct *iowner_table[IOWNER_SLOTS]; +static DEFINE_SPINLOCK(iowner_lock); + +static atomic64_t iowner_exhausted; + +static atomic64_t iowner_proxied; + +/* Work items charged back to a task, and the CPU time they cost. */ +static atomic64_t kernwork_charged; +static atomic64_t kernwork_ns; + +/* + * Slot for @tsk, allocating one on first use. + * + * @tsk is not necessarily current: an io-wq worker allocates on behalf of the + * task that queued the request it is running, so that pages it dirties are + * tagged with the real owner. The caller holds a reference to @tsk. + */ +static unsigned int iotime_task_slot(struct task_struct *tsk) +{ + unsigned int slot, scan; + + if (!MUQSS_IOWNER_WIDTH) + return 0; + + slot = READ_ONCE(tsk->io_owner_slot); + if (slot) + return slot; + + if (tsk->flags & (PF_KTHREAD | PF_EXITING)) + return 0; + + spin_lock(&iowner_lock); + + /* + * Recheck PF_EXITING under the lock. It is set well before + * muqss_iotime_release_slot() runs, and that also takes this lock, so + * a task seen here as not exiting either has not reached exit or will + * release the slot we are about to hand it. Without the recheck a slot + * claimed for a remote task racing through exit would never be freed. + */ + if (tsk->flags & PF_EXITING) + goto out; + + /* Raced with another dirty on the same task. */ + if (tsk->io_owner_slot) { + slot = tsk->io_owner_slot; + goto out; + } + + /* Prefer a free slot. */ + for (scan = 1; scan < IOWNER_SLOTS; scan++) { + if (!iowner_table[scan]) { + slot = scan; + goto claim; + } + } + + /* + * Table full of live dirtiers. Leave this task unattributed rather + * than evicting somebody still using their slot: a stolen slot would + * silently charge one task's writeback to another, which is worse + * than not charging it at all. + */ + atomic64_inc(&iowner_exhausted); + slot = 0; + goto out; + +claim: + iowner_table[slot] = get_task_struct(tsk); + WRITE_ONCE(tsk->io_owner_slot, slot); +out: + spin_unlock(&iowner_lock); + return slot; +} + +unsigned int muqss_iotime_owner_slot(void) +{ + struct task_struct *tsk = current; + struct task_struct *owner; + + /* + * A thread running inside a proxy window dirties pages on behalf of + * the task that window belongs to, so tag them with that task. Doing + * otherwise would both misattribute the writeback and, for io-wq + * workers, let a churn of short lived workers consume the whole slot + * table. + * + * An io-wq worker outside a window has nobody to speak for and must + * not claim a slot of its own. A kworker outside a window needs no + * such test: iotime_task_slot() refuses PF_KTHREAD outright. + */ + owner = READ_ONCE(tsk->io_owner_override); + if (owner) + tsk = owner; + else if (tsk->flags & PF_IO_WORKER) + return 0; + + return iotime_task_slot(tsk); +} + +struct task_struct *muqss_iotime_proxy_begin(struct task_struct *owner) +{ + struct task_struct *prev = current->io_owner_override; + + WRITE_ONCE(current->io_owner_override, owner); + return prev; +} + +void muqss_iotime_proxy_end(struct task_struct *prev) +{ + WRITE_ONCE(current->io_owner_override, prev); +} + +void muqss_work_set_owner(struct work_struct *work) +{ + work->muqss_owner_slot = muqss_iotime_owner_slot(); +} + +void muqss_kerntime_begin(struct muqss_kern_window *w, unsigned int slot) +{ + w->prev = NULL; + w->start = 0; + + /* + * Resolving the slot is what costs here, so it gates everything else. + * Work queued by kernel threads on their own account carries slot 0 + * and leaves this function having done one branch, which matters: + * timers, readahead, RCU and any amount of driver housekeeping run + * through work items that belong to nobody. + */ + w->owner = muqss_iotime_owner_task(slot); + if (!w->owner) + return; + + w->prev = muqss_iotime_proxy_begin(w->owner); + w->start = muqss_task_runtime_live(); +} + +void muqss_kerntime_end(struct muqss_kern_window *w) +{ + s64 ns; + + if (!w->owner) + return; + + ns = muqss_task_runtime_live() - w->start; + muqss_iotime_proxy_end(w->prev); + + /* + * Runtime rather than elapsed time, so a worker preempted for a whole + * scheduling round does not hand the bill for it to the task it is + * working for. Negative or absurd deltas are possible if the two ends + * of the window landed either side of a clock warp, and are dropped + * rather than clamped: there is no sensible value to substitute. + */ + if (ns > 0) { + atomic64_add(ns, &w->owner->kern_time_ns); + atomic64_add(ns, &w->owner->kern_debt_ns); + atomic64_inc(&kernwork_charged); + atomic64_add(ns, &kernwork_ns); + } + + put_task_struct(w->owner); + w->owner = NULL; +} + +struct task_struct *muqss_iotime_owner_task(unsigned int slot) +{ + struct task_struct *tsk = NULL; + + if (!slot || slot >= IOWNER_SLOTS) + return NULL; + + spin_lock(&iowner_lock); + if (iowner_table[slot]) + tsk = get_task_struct(iowner_table[slot]); + spin_unlock(&iowner_lock); + + return tsk; +} + +void muqss_iotime_release_slot(struct task_struct *p) +{ + unsigned int slot = READ_ONCE(p->io_owner_slot); + + if (!slot) + return; + + spin_lock(&iowner_lock); + if (iowner_table[slot] == p) { + iowner_table[slot] = NULL; + WRITE_ONCE(p->io_owner_slot, 0); + spin_unlock(&iowner_lock); + put_task_struct(p); + return; + } + WRITE_ONCE(p->io_owner_slot, 0); + spin_unlock(&iowner_lock); +} + +void muqss_iotime_dirty_folio(struct folio *folio) +{ + unsigned int slot = muqss_iotime_owner_slot(); + + if (slot) + folio_set_io_owner(folio, slot); +} + +void muqss_iotime_task_init(struct task_struct *p) +{ + atomic64_set(&p->io_latency_ns, 0); + atomic64_set(&p->io_count, 0); + atomic64_set(&p->io_occupancy_ns, 0); + atomic64_set(&p->io_debt_ns, 0); + atomic64_set(&p->kern_time_ns, 0); + atomic64_set(&p->kern_debt_ns, 0); + p->io_owner_slot = 0; + p->io_owner_override = NULL; +} + +void muqss_iotime_set_owner(struct bio *bio) +{ + struct task_struct *tsk = current; + + if (bio->bi_muqss_owner) + return; + + /* + * Not in task context: current is whatever this interrupted, which + * has nothing to do with the I/O. Bios submitted from softirq or hard + * IRQ, such as a filesystem completing one write by starting another, + * would otherwise be charged to a bystander. + */ + if (!in_task()) + return; + + /* + * Passthrough requests are not issued on any task's behalf. Neither + * is the flush machinery's own empty bio, but REQ_PREFLUSH is also set + * on an ordinary write that wants a cache flush ahead of it, and that + * write is somebody's. Test for the payload rather than the flag. + */ + if (blk_op_is_passthrough(bio->bi_opf)) + return; + if ((bio->bi_opf & REQ_PREFLUSH) && !bio_has_data(bio)) + return; + + if (tsk->flags & (PF_KTHREAD | PF_IO_WORKER)) { + struct task_struct *owner; + struct folio *folio; + + /* + * io_uring punted this submission to an io-wq worker, which + * published the task that queued the request. Charge that + * task: the worker is an anonymous thread that exists only to + * run somebody else's I/O. + * + * Writeback that happens to be issued from inside the same + * window is charged here too, which is a small misattribution + * within one application rather than a loss. + */ + owner = READ_ONCE(tsk->io_owner_override); + if (owner) { + atomic64_inc(&iowner_proxied); + bio->bi_muqss_owner = get_task_struct(owner); + return; + } + + /* + * Writeback. current is a flusher thread rather than the task + * that dirtied the data, so recover the owner from the folio, + * where it was recorded at dirty time. + * + * The first page decides for the whole bio. A writeback bio + * normally covers one file's pages dirtied by one task, and + * reading every page to take a vote would put a loop on the + * submit path for a heuristic that does not need it. + * + * Cloned bios are skipped: their bi_io_vec belongs to the + * parent, and muqss_iotime_clone_owner() has already carried + * the owner across. + */ + if (op_is_write(bio->bi_opf) && bio->bi_vcnt && + !bio_flagged(bio, BIO_CLONED)) { + folio = page_folio(bio->bi_io_vec[0].bv_page); + bio->bi_muqss_owner = + muqss_iotime_owner_task(folio_io_owner(folio)); + return; + } + + /* + * An io_uring SQPOLL thread submits for whoever shares its + * ring, and the request does not record which task that was; + * with IORING_SETUP_ATTACH_WQ it need not be a single one. + * The thread is a long lived schedulable task in its own + * right though, so charge it directly rather than lose the + * time: the device cost is real, and the sq thread is the + * entity the scheduler can actually act on. + * + * Kernel threads proper get nothing. + */ + if (tsk->flags & PF_IO_WORKER) + bio->bi_muqss_owner = get_task_struct(tsk); + return; + } + + bio->bi_muqss_owner = get_task_struct(tsk); +} + +void muqss_iotime_put_owner(struct bio *bio) +{ + if (bio->bi_muqss_owner) { + put_task_struct(bio->bi_muqss_owner); + bio->bi_muqss_owner = NULL; + } +} + +void muqss_iotime_clone_owner(struct bio *bio, struct bio *bio_src) +{ + muqss_iotime_put_owner(bio); + + if (bio_src->bi_muqss_owner) + bio->bi_muqss_owner = get_task_struct(bio_src->bi_muqss_owner); +} + +/* + * Per bio completion. The owner reference is not dropped here: bio_endio() + * and bio_uninit() release it alongside the blkg reference, which keeps the + * lifetime rules identical to the ones the block layer already follows. + */ +static void muqss_iotime_done_bio(struct rq_qos *rqos, struct bio *bio) +{ + struct blk_iotime *bit = BLK_IOTIME(rqos); + struct task_struct *tsk = bio->bi_muqss_owner; + u64 now; + + if (!tsk) { + atomic64_inc(&bit->unattributed); + return; + } + if (!bio->issue_time_ns) { + atomic64_inc(&bit->nostamp_bio); + return; + } + + now = blk_time_get_ns(); + if (now <= bio->issue_time_ns) + return; + + atomic64_add(now - bio->issue_time_ns, &tsk->io_latency_ns); + atomic64_inc(&tsk->io_count); +} + +/* + * A request is being built from its first bio. Take our own reference for the + * request: the bio's reference is released by bio_endio() during + * blk_update_request(), which runs before rq_qos_done(), so borrowing it would + * leave a dangling pointer at completion. + */ +static void muqss_iotime_track(struct rq_qos *rqos, struct request *rq, + struct bio *bio) +{ + if (bio->bi_muqss_owner) + rq->muqss_owner = get_task_struct(bio->bi_muqss_owner); +} + +/* + * A further bio is merging into an existing request. Occupancy is measured + * per request and cannot be split between owners, so a request that ends up + * serving more than one task is disowned rather than charged to whichever + * task happened to be first. + */ +static void muqss_iotime_merge(struct rq_qos *rqos, struct request *rq, + struct bio *bio) +{ + if (!rq->muqss_owner || rq->muqss_owner == bio->bi_muqss_owner) + return; + + put_task_struct(rq->muqss_owner); + rq->muqss_owner = NULL; + atomic64_inc(&BLK_IOTIME(rqos)->mixed); +} + +/* + * Two whole requests are being merged: next's bios are about to be appended to + * rq and next freed. The rq_qos ->merge hook does not cover this, it only sees + * a bio being merged into a request, so without this the survivor keeps its own + * owner and is charged for both requests' device time. + * + * Called from attempt_merge() rather than being an rq_qos op because the + * framework has no request-to-request callback to hang it on. + */ +void muqss_iotime_merge_requests(struct request *rq, struct request *next) +{ + struct rq_qos *rqos; + + if (!rq->muqss_owner || rq->muqss_owner == next->muqss_owner) + return; + + put_task_struct(rq->muqss_owner); + rq->muqss_owner = NULL; + + rqos = rq_qos_id(rq->q, RQ_QOS_MUQSS_IOTIME); + if (rqos) + atomic64_inc(&BLK_IOTIME(rqos)->mixed); +} + +/* Per request completion. This is where device occupancy is attributed. */ +static void muqss_iotime_done(struct rq_qos *rqos, struct request *rq) +{ + struct blk_iotime *bit = BLK_IOTIME(rqos); + struct task_struct *tsk = rq->muqss_owner; + u64 now, occupancy; + + if (!(rq->rq_flags & RQF_STATS) || !rq->io_start_time_ns) + goto out; + + /* + * Use the timestamp captured when the request completed, not one + * taken here: rq_qos_done() runs after blk_update_request() has + * completed every bio, so measuring now would fold the completion + * path into what is supposed to be device service time. That bias is + * roughly constant per request, which would quietly turn part of the + * charge into a per-I/O-count charge and penalise small random I/O. + * + * Consuming the stamp also makes this idempotent. rq_qos_done() runs + * twice for a request with an end_io handler that frees it: once from + * __blk_mq_end_request() and again from blk_mq_free_request(). Every + * policy has to tolerate that. Clearing muqss_owner alone stopped the + * task being charged twice but left the device totals below counting + * the same request on both passes. + * + * A request that never went through blk_mq_end_request() has no stamp + * and is not measured. Timing it here instead would report the + * completion path as device time, which is the bias this exists to + * avoid; unstamped counts them so the loss stays visible. + */ + now = rq->muqss_done_ns; + if (!now) { + atomic64_inc(&bit->unstamped); + goto out; + } + rq->muqss_done_ns = 0; + + if (now <= rq->io_start_time_ns) + goto out; + + occupancy = now - rq->io_start_time_ns; + atomic64_add(occupancy, &bit->occupancy_ns); + atomic64_inc(&bit->requests); + + if (tsk) { + atomic64_add(occupancy, &tsk->io_occupancy_ns); + atomic64_add(occupancy, &tsk->io_debt_ns); + } +out: + if (tsk) { + put_task_struct(tsk); + rq->muqss_owner = NULL; + } +} + +static void muqss_iotime_exit(struct rq_qos *rqos) +{ + struct blk_iotime *bit = BLK_IOTIME(rqos); + + /* + * QUEUE_FLAG_BIO_ISSUE_TIME is deliberately left set. blk-iolatency + * uses the same flag, and the queue is being torn down here anyway. + */ + blk_stat_disable_accounting(rqos->disk->queue); + kfree(bit); +} + +#ifdef CONFIG_BLK_DEBUG_FS +static int muqss_iotime_occupancy_show(void *data, struct seq_file *m) +{ + struct rq_qos *rqos = data; + + seq_printf(m, "%llu\n", + (u64)atomic64_read(&BLK_IOTIME(rqos)->occupancy_ns)); + return 0; +} + +static int muqss_iotime_requests_show(void *data, struct seq_file *m) +{ + struct rq_qos *rqos = data; + + seq_printf(m, "%llu\n", + (u64)atomic64_read(&BLK_IOTIME(rqos)->requests)); + return 0; +} + +static int muqss_iotime_unattributed_show(void *data, struct seq_file *m) +{ + struct rq_qos *rqos = data; + + seq_printf(m, "%llu\n", + (u64)atomic64_read(&BLK_IOTIME(rqos)->unattributed)); + return 0; +} + +static int muqss_iotime_mixed_show(void *data, struct seq_file *m) +{ + struct rq_qos *rqos = data; + + seq_printf(m, "%llu\n", (u64)atomic64_read(&BLK_IOTIME(rqos)->mixed)); + return 0; +} + +static int muqss_iotime_unstamped_show(void *data, struct seq_file *m) +{ + struct rq_qos *rqos = data; + + seq_printf(m, "%llu\n", + (u64)atomic64_read(&BLK_IOTIME(rqos)->unstamped)); + return 0; +} + +static int muqss_iotime_nostamp_bio_show(void *data, struct seq_file *m) +{ + struct rq_qos *rqos = data; + + seq_printf(m, "%llu\n", + (u64)atomic64_read(&BLK_IOTIME(rqos)->nostamp_bio)); + return 0; +} + +static int muqss_iotime_exhausted_show(void *data, struct seq_file *m) +{ + seq_printf(m, "%llu\n", (u64)atomic64_read(&iowner_exhausted)); + return 0; +} + +static int muqss_iotime_proxied_show(void *data, struct seq_file *m) +{ + seq_printf(m, "%llu\n", (u64)atomic64_read(&iowner_proxied)); + return 0; +} + +static int muqss_iotime_owner_width_show(void *data, struct seq_file *m) +{ + seq_printf(m, "%u\n", MUQSS_IOWNER_WIDTH); + return 0; +} + +static int muqss_iotime_kernwork_show(void *data, struct seq_file *m) +{ + seq_printf(m, "%llu\n", (u64)atomic64_read(&kernwork_charged)); + return 0; +} + +static int muqss_iotime_kernwork_ns_show(void *data, struct seq_file *m) +{ + seq_printf(m, "%llu\n", (u64)atomic64_read(&kernwork_ns)); + return 0; +} + +static const struct blk_mq_debugfs_attr muqss_iotime_debugfs_attrs[] = { + {"occupancy_ns", 0400, muqss_iotime_occupancy_show}, + {"requests", 0400, muqss_iotime_requests_show}, + {"unattributed", 0400, muqss_iotime_unattributed_show}, + {"mixed", 0400, muqss_iotime_mixed_show}, + {"unstamped", 0400, muqss_iotime_unstamped_show}, + {"nostamp_bio", 0400, muqss_iotime_nostamp_bio_show}, + {"owner_slots_exhausted", 0400, muqss_iotime_exhausted_show}, + {"proxied", 0400, muqss_iotime_proxied_show}, + {"owner_tag_bits", 0400, muqss_iotime_owner_width_show}, + {"kernwork", 0400, muqss_iotime_kernwork_show}, + {"kernwork_ns", 0400, muqss_iotime_kernwork_ns_show}, + {}, +}; +#endif + +static const struct rq_qos_ops muqss_iotime_ops = { + .track = muqss_iotime_track, + .merge = muqss_iotime_merge, + .done = muqss_iotime_done, + .done_bio = muqss_iotime_done_bio, + .exit = muqss_iotime_exit, +#ifdef CONFIG_BLK_DEBUG_FS + .debugfs_attrs = muqss_iotime_debugfs_attrs, +#endif +}; + +void muqss_iotime_enable(struct gendisk *disk) +{ + struct request_queue *q = disk->queue; + unsigned int memflags; + struct blk_iotime *bit; + int ret; + + if (!queue_is_mq(q)) + return; + + /* + * Only attach to queues backed by real hardware. + * + * A stacking driver that uses blk-mq -- loop is the common one -- + * serves its requests by doing I/O to another block device, which runs + * this policy too. Accounting both charges the task twice for one + * trip to the disk, and with the deadline charge live that is a real + * distortion rather than a cosmetic one. + * + * Drivers for physical devices pass their parent through + * device_add_disk(): nvme passes ctrl->device, sd its scsi_device, + * virtio_blk its virtio device. Virtual ones call add_disk(), which + * is device_add_disk(NULL, ...) -- loop, brd, zram, nbd. So the + * absence of a parent is a serviceable test for "not a device whose + * busy time means anything". + * + * Bio based stacking drivers (dm, md) never reach here at all, having + * failed queue_is_mq() above. + */ + if (!disk_to_dev(disk)->parent) + return; + + bit = kzalloc_obj(*bit); + if (!bit) + return; + + mutex_lock(&q->rq_qos_mutex); + ret = rq_qos_add(&bit->rqos, disk, RQ_QOS_MUQSS_IOTIME, + &muqss_iotime_ops); + mutex_unlock(&q->rq_qos_mutex); + if (ret) { + kfree(bit); + return; + } + + /* Needed for rq->io_start_time_ns to be stamped at dispatch. */ + blk_stat_enable_accounting(q); + /* Needed for bio->issue_time_ns to be stamped at submit. */ + blk_queue_flag_set(QUEUE_FLAG_BIO_ISSUE_TIME, q); + + memflags = blk_debugfs_lock(q); + blk_mq_debugfs_register_rq_qos(q); + blk_debugfs_unlock(q, memflags); +} diff --git a/block/blk-merge.c b/block/blk-merge.c index ab1161ca6..16d2fde03 100644 --- a/block/blk-merge.c +++ b/block/blk-merge.c @@ -12,6 +12,8 @@ #include +#include + #include "blk.h" #include "blk-mq-sched.h" #include "blk-rq-qos.h" @@ -848,6 +850,7 @@ static struct request *attempt_merge(struct request_queue *q, * 'next' is going away, so update stats accordingly */ blk_account_io_merge_request(next); + muqss_iotime_merge_requests(req, next); trace_block_rq_merge(next); diff --git a/block/blk-mq-debugfs.c b/block/blk-mq-debugfs.c index 6754d8f94..a88819bfd 100644 --- a/block/blk-mq-debugfs.c +++ b/block/blk-mq-debugfs.c @@ -757,6 +757,8 @@ static const char *rq_qos_id_to_name(enum rq_qos_id id) return "latency"; case RQ_QOS_COST: return "cost"; + case RQ_QOS_MUQSS_IOTIME: + return "iotime"; } return "unknown"; } diff --git a/block/blk-mq.c b/block/blk-mq.c index 2c850330a..4b0ca1871 100644 --- a/block/blk-mq.c +++ b/block/blk-mq.c @@ -41,6 +41,7 @@ #include "blk-stat.h" #include "blk-mq-sched.h" #include "blk-rq-qos.h" +#include static DEFINE_PER_CPU(struct llist_head, blk_cpu_done); static DEFINE_PER_CPU(call_single_data_t, blk_cpu_csd); @@ -398,12 +399,48 @@ static inline void blk_mq_rq_time_init(struct request *rq, u64 alloc_time_ns) #endif } +/* + * Record the completion timestamp for MuQSS I/O accounting. + * + * rq_qos_done() is reached only after blk_update_request() has run bio_endio() + * on every bio of the request, so a timestamp taken inside the policy is later + * than the request actually finished, and lands after the per bio latency + * stamp. Occupancy then measures longer than the end to end latency it is + * supposed to be a subset of. + * + * Does not overwrite: the first caller to record a stamp is the earliest and + * therefore the most accurate one. + */ +static inline void blk_mq_set_iotime_done(struct request *rq, u64 now) +{ +#ifdef CONFIG_MUQSS_IOTIME + if (!rq->muqss_done_ns) + rq->muqss_done_ns = now; +#endif +} + +/* + * Take the completion stamp before bio completion. Costs an extra timestamp + * on the completion path, which cannot be avoided by reusing the one taken + * for blk-stat: that one is deliberately read after blk_update_request() and + * moving it would change accounting shared with everybody else. + */ +static inline void blk_mq_capture_iotime_done(struct request *rq) +{ +#ifdef CONFIG_MUQSS_IOTIME + if (blk_mq_need_time_stamp(rq)) + rq->muqss_done_ns = blk_time_get_ns(); +#endif +} + static inline void blk_mq_bio_issue_init(struct request_queue *q, struct bio *bio) { -#ifdef CONFIG_BLK_CGROUP - if (test_bit(QUEUE_FLAG_BIO_ISSUE_TIME, &q->queue_flags)) +#if defined(CONFIG_BLK_CGROUP) || defined(CONFIG_MUQSS_IOTIME) + if (test_bit(QUEUE_FLAG_BIO_ISSUE_TIME, &q->queue_flags)) { bio->issue_time_ns = blk_time_get_ns(); + muqss_iotime_set_owner(bio); + } #endif } @@ -435,6 +472,10 @@ static struct request *blk_mq_rq_ctx_init(struct blk_mq_alloc_data *data, rq->part = NULL; rq->io_start_time_ns = 0; +#ifdef CONFIG_MUQSS_IOTIME + rq->muqss_owner = NULL; + rq->muqss_done_ns = 0; +#endif rq->stats_sectors = 0; rq->nr_phys_segments = 0; rq->nr_integrity_segments = 0; @@ -1127,8 +1168,12 @@ static inline void __blk_mq_end_request_acct(struct request *rq, u64 now) inline void __blk_mq_end_request(struct request *rq, blk_status_t error) { - if (blk_mq_need_time_stamp(rq)) - __blk_mq_end_request_acct(rq, blk_time_get_ns()); + if (blk_mq_need_time_stamp(rq)) { + u64 now = blk_time_get_ns(); + + blk_mq_set_iotime_done(rq, now); + __blk_mq_end_request_acct(rq, now); + } blk_mq_finish_request(rq); @@ -1144,6 +1189,7 @@ EXPORT_SYMBOL(__blk_mq_end_request); void blk_mq_end_request(struct request *rq, blk_status_t error) { + blk_mq_capture_iotime_done(rq); if (blk_update_request(rq, error, blk_rq_bytes(rq))) BUG(); __blk_mq_end_request(rq, error); @@ -1178,8 +1224,10 @@ void blk_mq_end_request_batch(struct io_comp_batch *iob) prefetch(rq->rq_next); blk_complete_request(rq); - if (iob->need_ts) + if (iob->need_ts) { + blk_mq_set_iotime_done(rq, now); __blk_mq_end_request_acct(rq, now); + } blk_mq_finish_request(rq); diff --git a/block/blk-rq-qos.h b/block/blk-rq-qos.h index a747a504f..66913f3df 100644 --- a/block/blk-rq-qos.h +++ b/block/blk-rq-qos.h @@ -17,6 +17,7 @@ enum rq_qos_id { RQ_QOS_WBT, RQ_QOS_LATENCY, RQ_QOS_COST, + RQ_QOS_MUQSS_IOTIME, }; struct rq_wait { diff --git a/block/blk-sysfs.c b/block/blk-sysfs.c index 520972676..be36ea5bf 100644 --- a/block/blk-sysfs.c +++ b/block/blk-sysfs.c @@ -20,6 +20,7 @@ #include "blk-cgroup.h" #include "blk-throttle.h" #include "error-injection.h" +#include struct queue_sysfs_entry { struct attribute attr; @@ -993,6 +994,7 @@ int blk_register_queue(struct gendisk *disk) blk_queue_flag_set(QUEUE_FLAG_REGISTERED, q); wbt_init_enable_default(disk); + muqss_iotime_enable(disk); /* Now everything is ready and send out KOBJ_ADD uevent */ kobject_uevent(&disk->queue_kobj, KOBJ_ADD); diff --git a/drivers/accessibility/speakup/speakup_acntpc.c b/drivers/accessibility/speakup/speakup_acntpc.c index a27e6bbf0..af7c1c4f2 100644 --- a/drivers/accessibility/speakup/speakup_acntpc.c +++ b/drivers/accessibility/speakup/speakup_acntpc.c @@ -207,7 +207,7 @@ static void do_catch_up(struct spk_synth *synth) full_time_val = full_time->u.n.value; spin_unlock_irqrestore(&speakup_info.spinlock, flags); if (synth_full()) { - schedule_timeout(msecs_to_jiffies(full_time_val)); + schedule_msec_hrtimeout(full_time_val); continue; } set_current_state(TASK_RUNNING); @@ -235,7 +235,7 @@ static void do_catch_up(struct spk_synth *synth) jiffy_delta_val = jiffy_delta->u.n.value; delay_time_val = delay_time->u.n.value; spin_unlock_irqrestore(&speakup_info.spinlock, flags); - schedule_timeout(msecs_to_jiffies(delay_time_val)); + schedule_msec_hrtimeout(delay_time_val); jiff_max = jiffies + jiffy_delta_val; } } diff --git a/drivers/accessibility/speakup/speakup_apollo.c b/drivers/accessibility/speakup/speakup_apollo.c index d2fbb3f57..abd9f3409 100644 --- a/drivers/accessibility/speakup/speakup_apollo.c +++ b/drivers/accessibility/speakup/speakup_apollo.c @@ -178,7 +178,7 @@ static void do_catch_up(struct spk_synth *synth) if (!synth->io_ops->synth_out(synth, ch)) { synth->io_ops->tiocmset(synth, 0, UART_MCR_RTS); synth->io_ops->tiocmset(synth, UART_MCR_RTS, 0); - schedule_timeout(msecs_to_jiffies(full_time_val)); + schedule_msec_hrtimeout(full_time_val); continue; } if (time_after_eq(jiffies, jiff_max) && (ch == SPACE)) { @@ -188,11 +188,9 @@ static void do_catch_up(struct spk_synth *synth) delay_time_val = delay_time->u.n.value; spin_unlock_irqrestore(&speakup_info.spinlock, flags); if (synth->io_ops->synth_out(synth, synth->procspeech)) - schedule_timeout(msecs_to_jiffies - (delay_time_val)); + schedule_msec_hrtimeout(delay_time_val); else - schedule_timeout(msecs_to_jiffies - (full_time_val)); + schedule_msec_hrtimeout(full_time_val); jiff_max = jiffies + jiffy_delta_val; } set_current_state(TASK_RUNNING); diff --git a/drivers/accessibility/speakup/speakup_decext.c b/drivers/accessibility/speakup/speakup_decext.c index 271bcf279..ec430f790 100644 --- a/drivers/accessibility/speakup/speakup_decext.c +++ b/drivers/accessibility/speakup/speakup_decext.c @@ -189,7 +189,7 @@ static void do_catch_up(struct spk_synth *synth) if (ch == '\n') ch = 0x0D; if (synth_full() || !synth->io_ops->synth_out(synth, ch)) { - schedule_timeout(msecs_to_jiffies(delay_time_val)); + schedule_msec_hrtimeout(delay_time_val); continue; } set_current_state(TASK_RUNNING); @@ -213,8 +213,7 @@ static void do_catch_up(struct spk_synth *synth) delay_time_val = delay_time->u.n.value; spin_unlock_irqrestore(&speakup_info.spinlock, flags); - schedule_timeout(msecs_to_jiffies - (delay_time_val)); + schedule_msec_hrtimeout(delay_time_val); jiff_max = jiffies + jiffy_delta_val; } } diff --git a/drivers/accessibility/speakup/speakup_decpc.c b/drivers/accessibility/speakup/speakup_decpc.c index 083ca9265..ef9466c76 100644 --- a/drivers/accessibility/speakup/speakup_decpc.c +++ b/drivers/accessibility/speakup/speakup_decpc.c @@ -409,7 +409,7 @@ static void do_catch_up(struct spk_synth *synth) if (ch == '\n') ch = 0x0D; if (dt_sendchar(ch)) { - schedule_timeout(msecs_to_jiffies(delay_time_val)); + schedule_msec_hrtimeout(delay_time_val); continue; } set_current_state(TASK_RUNNING); @@ -432,8 +432,7 @@ static void do_catch_up(struct spk_synth *synth) delay_time_val = delay_time->u.n.value; spin_unlock_irqrestore(&speakup_info.spinlock, flags); - schedule_timeout(msecs_to_jiffies - (delay_time_val)); + schedule_msec_hrtimeout(delay_time_val); jiff_max = jiffies + jiffy_delta_val; } } diff --git a/drivers/accessibility/speakup/speakup_dectlk.c b/drivers/accessibility/speakup/speakup_dectlk.c index 56334405d..bbcdf011f 100644 --- a/drivers/accessibility/speakup/speakup_dectlk.c +++ b/drivers/accessibility/speakup/speakup_dectlk.c @@ -265,7 +265,7 @@ static void do_catch_up(struct spk_synth *synth) if (ch == '\n') ch = 0x0D; if (synth_full_val || !synth->io_ops->synth_out(synth, ch)) { - schedule_timeout(msecs_to_jiffies(delay_time_val)); + schedule_msec_hrtimeout(delay_time_val); continue; } set_current_state(TASK_RUNNING); @@ -289,8 +289,7 @@ static void do_catch_up(struct spk_synth *synth) delay_time_val = delay_time->u.n.value; spin_unlock_irqrestore(&speakup_info.spinlock, flags); - schedule_timeout(msecs_to_jiffies - (delay_time_val)); + schedule_msec_hrtimeout(delay_time_val); jiff_max = jiffies + jiffy_delta_val; } } diff --git a/drivers/accessibility/speakup/speakup_dtlk.c b/drivers/accessibility/speakup/speakup_dtlk.c index fa8265689..1745519f8 100644 --- a/drivers/accessibility/speakup/speakup_dtlk.c +++ b/drivers/accessibility/speakup/speakup_dtlk.c @@ -221,7 +221,7 @@ static void do_catch_up(struct spk_synth *synth) delay_time_val = delay_time->u.n.value; spin_unlock_irqrestore(&speakup_info.spinlock, flags); if (synth_full()) { - schedule_timeout(msecs_to_jiffies(delay_time_val)); + schedule_msec_hrtimeout(delay_time_val); continue; } set_current_state(TASK_RUNNING); @@ -237,7 +237,7 @@ static void do_catch_up(struct spk_synth *synth) delay_time_val = delay_time->u.n.value; jiffy_delta_val = jiffy_delta->u.n.value; spin_unlock_irqrestore(&speakup_info.spinlock, flags); - schedule_timeout(msecs_to_jiffies(delay_time_val)); + schedule_msec_hrtimeout(delay_time_val); jiff_max = jiffies + jiffy_delta_val; } } diff --git a/drivers/accessibility/speakup/speakup_keypc.c b/drivers/accessibility/speakup/speakup_keypc.c index 9356f6379..ab5cde814 100644 --- a/drivers/accessibility/speakup/speakup_keypc.c +++ b/drivers/accessibility/speakup/speakup_keypc.c @@ -208,7 +208,7 @@ static void do_catch_up(struct spk_synth *synth) full_time_val = full_time->u.n.value; spin_unlock_irqrestore(&speakup_info.spinlock, flags); if (synth_full()) { - schedule_timeout(msecs_to_jiffies(full_time_val)); + schedule_msec_hrtimeout(full_time_val); continue; } set_current_state(TASK_RUNNING); @@ -241,7 +241,7 @@ static void do_catch_up(struct spk_synth *synth) jiffy_delta_val = jiffy_delta->u.n.value; delay_time_val = delay_time->u.n.value; spin_unlock_irqrestore(&speakup_info.spinlock, flags); - schedule_timeout(msecs_to_jiffies(delay_time_val)); + schedule_msec_hrtimeout(delay_time_val); jiff_max = jiffies + jiffy_delta_val; } } diff --git a/drivers/accessibility/speakup/synth.c b/drivers/accessibility/speakup/synth.c index d8addbf3a..d28c0f4e0 100644 --- a/drivers/accessibility/speakup/synth.c +++ b/drivers/accessibility/speakup/synth.c @@ -98,7 +98,7 @@ static void _spk_do_catch_up(struct spk_synth *synth, int unicode) else ret = synth->io_ops->synth_out(synth, ch); if (!ret) { - schedule_timeout(msecs_to_jiffies(full_time_val)); + schedule_msec_hrtimeout(full_time_val); continue; } if (time_after_eq(jiffies, jiff_max) && (ch == SPACE)) { @@ -108,11 +108,9 @@ static void _spk_do_catch_up(struct spk_synth *synth, int unicode) full_time_val = full_time->u.n.value; spin_unlock_irqrestore(&speakup_info.spinlock, flags); if (synth->io_ops->synth_out(synth, synth->procspeech)) - schedule_timeout( - msecs_to_jiffies(delay_time_val)); + schedule_msec_hrtimeout(delay_time_val); else - schedule_timeout( - msecs_to_jiffies(full_time_val)); + schedule_msec_hrtimeout(full_time_val); jiff_max = jiffies + jiffy_delta_val; } set_current_state(TASK_RUNNING); diff --git a/drivers/auxdisplay/hd44780_common.c b/drivers/auxdisplay/hd44780_common.c index b71db39f9..f7056362b 100644 --- a/drivers/auxdisplay/hd44780_common.c +++ b/drivers/auxdisplay/hd44780_common.c @@ -34,7 +34,7 @@ /* sleeps that many milliseconds with a reschedule */ static void long_sleep(int ms) { - schedule_timeout_interruptible(msecs_to_jiffies(ms)); + schedule_msec_hrtimeout_interruptible(ms); } int hd44780_common_print(struct charlcd *lcd, int c) diff --git a/drivers/block/swim.c b/drivers/block/swim.c index 0ccc12a72..612ff677d 100644 --- a/drivers/block/swim.c +++ b/drivers/block/swim.c @@ -330,7 +330,7 @@ static inline void swim_motor(struct swim __iomem *base, if (swim_readbit(base, MOTOR_ON)) break; set_current_state(TASK_INTERRUPTIBLE); - schedule_timeout(1); + schedule_min_hrtimeout(); } } else if (action == OFF) { swim_action(base, MOTOR_OFF); @@ -349,7 +349,7 @@ static inline void swim_eject(struct swim __iomem *base) if (!swim_readbit(base, DISK_IN)) break; set_current_state(TASK_INTERRUPTIBLE); - schedule_timeout(1); + schedule_min_hrtimeout(); } swim_select(base, RELAX); } @@ -373,7 +373,7 @@ static inline int swim_step(struct swim __iomem *base) for (wait = 0; wait < HZ; wait++) { set_current_state(TASK_INTERRUPTIBLE); - schedule_timeout(1); + schedule_min_hrtimeout(); swim_select(base, RELAX); if (!swim_readbit(base, STEP)) diff --git a/drivers/char/ipmi/ipmi_msghandler.c b/drivers/char/ipmi/ipmi_msghandler.c index ab4c85f3d..6d3a54ee7 100644 --- a/drivers/char/ipmi/ipmi_msghandler.c +++ b/drivers/char/ipmi/ipmi_msghandler.c @@ -3800,7 +3800,7 @@ static void cleanup_smi_msgs(struct ipmi_smi *intf) /* Current message first, to preserve order */ while (intf->curr_msg && !list_empty(&intf->waiting_rcv_msgs)) { /* Wait for the message to clear out. */ - schedule_timeout(1); + schedule_min_hrtimeout(); } /* No need for locks, the interface is down. */ diff --git a/drivers/char/ipmi/ipmi_ssif.c b/drivers/char/ipmi/ipmi_ssif.c index 07f1d2327..52767cc1e 100644 --- a/drivers/char/ipmi/ipmi_ssif.c +++ b/drivers/char/ipmi/ipmi_ssif.c @@ -1282,7 +1282,7 @@ static void shutdown_ssif(void *send_info) /* make sure the driver is not looking for flags any more. */ while (ssif_info->ssif_state != SSIF_IDLE) - schedule_timeout(1); + schedule_min_hrtimeout(); ssif_info->stopping = true; timer_delete_sync(&ssif_info->watch_timer); diff --git a/drivers/comedi/drivers/ni_mio_common.c b/drivers/comedi/drivers/ni_mio_common.c index 3acb449d2..06fb51fe6 100644 --- a/drivers/comedi/drivers/ni_mio_common.c +++ b/drivers/comedi/drivers/ni_mio_common.c @@ -4762,7 +4762,7 @@ static int cs5529_wait_for_idle(struct comedi_device *dev) if ((status & NI67XX_CAL_STATUS_BUSY) == 0) break; set_current_state(TASK_INTERRUPTIBLE); - if (schedule_timeout(1)) + if (schedule_min_hrtimeout()) return -EIO; } if (i == timeout) { diff --git a/drivers/crypto/ccp/tee-dev.c b/drivers/crypto/ccp/tee-dev.c index 3e3645980..3464825c9 100644 --- a/drivers/crypto/ccp/tee-dev.c +++ b/drivers/crypto/ccp/tee-dev.c @@ -270,7 +270,7 @@ static int tee_submit_cmd(struct psp_tee_device *tee, enum tee_cmd_id cmd_id, /* Wait if ring buffer is full or TEE is processing data */ mutex_unlock(&tee->rb_mgr.mutex); - schedule_timeout_interruptible(msecs_to_jiffies(10)); + schedule_msec_hrtimeout_interruptible(10); mutex_lock(&tee->rb_mgr.mutex); } while (--nloop); diff --git a/drivers/gpib/nec7210/nec7210.c b/drivers/gpib/nec7210/nec7210.c index f15d38dfa..f259d43a7 100644 --- a/drivers/gpib/nec7210/nec7210.c +++ b/drivers/gpib/nec7210/nec7210.c @@ -317,7 +317,7 @@ int nec7210_go_to_standby(struct gpib_board *board, struct nec7210_priv *priv) if (i == timeout) { for (i = 0; i < HZ; i++) { set_current_state(TASK_INTERRUPTIBLE); - if (schedule_timeout(1)) + if (schedule_min_hrtimeout()) return -ERESTARTSYS; adsr_bits = read_byte(priv, ADSR); if (adsr_bits & HR_NATN) diff --git a/drivers/gpu/drm/vmwgfx/vmwgfx_cmd.c b/drivers/gpu/drm/vmwgfx/vmwgfx_cmd.c index d932e38d7..0c9218eef 100644 --- a/drivers/gpu/drm/vmwgfx/vmwgfx_cmd.c +++ b/drivers/gpu/drm/vmwgfx/vmwgfx_cmd.c @@ -215,7 +215,7 @@ static int vmw_fifo_wait_noirq(struct vmw_private *dev_priv, DRM_ERROR("SVGA device lockup.\n"); break; } - schedule_timeout(1); + schedule_min_hrtimeout(); if (interruptible && signal_pending(current)) { ret = -ERESTARTSYS; break; diff --git a/drivers/gpu/drm/vmwgfx/vmwgfx_irq.c b/drivers/gpu/drm/vmwgfx/vmwgfx_irq.c index 05773eb39..ac18b0378 100644 --- a/drivers/gpu/drm/vmwgfx/vmwgfx_irq.c +++ b/drivers/gpu/drm/vmwgfx/vmwgfx_irq.c @@ -200,7 +200,7 @@ int vmw_fallback_wait(struct vmw_private *dev_priv, break; } if (lazy) - schedule_timeout(1); + schedule_min_hrtimeout(); else if ((++count & 0x0F) == 0) { /** * FIXME: Use schedule_hr_timeout here for diff --git a/drivers/hwmon/adt7470.c b/drivers/hwmon/adt7470.c index c45b984c0..50c42c06f 100644 --- a/drivers/hwmon/adt7470.c +++ b/drivers/hwmon/adt7470.c @@ -332,7 +332,7 @@ static int adt7470_update_thread(void *p) if (kthread_should_stop()) break; - schedule_timeout_interruptible(msecs_to_jiffies(data->auto_update_interval)); + schedule_msec_hrtimeout_interruptible(data->auto_update_interval); } return 0; diff --git a/drivers/hwmon/fam15h_power.c b/drivers/hwmon/fam15h_power.c index ad4ed4162..b6cf6d66c 100644 --- a/drivers/hwmon/fam15h_power.c +++ b/drivers/hwmon/fam15h_power.c @@ -221,7 +221,7 @@ static ssize_t power1_average_show(struct device *dev, prev_ptsc[cu] = data->cpu_sw_pwr_ptsc[cu]; } - leftover = schedule_timeout_interruptible(msecs_to_jiffies(data->power_period)); + leftover = schedule_msec_hrtimeout_interruptible(data->power_period); if (leftover) return 0; diff --git a/drivers/hwmon/ibmaem.c b/drivers/hwmon/ibmaem.c index 0a9c3a29e..df56c7104 100644 --- a/drivers/hwmon/ibmaem.c +++ b/drivers/hwmon/ibmaem.c @@ -827,9 +827,8 @@ static ssize_t aem_show_power(struct device *dev, time = ktime_get_ns(); before = data->energy[attr->index]; - leftover = schedule_timeout_interruptible( - msecs_to_jiffies(data->power_period[attr->index]) - ); + leftover = schedule_msec_hrtimeout_interruptible( + data->power_period[attr->index]); if (leftover) { mutex_unlock(&data->lock); return 0; diff --git a/drivers/iio/light/tsl2563.c b/drivers/iio/light/tsl2563.c index 45f3513d9..3428708c5 100644 --- a/drivers/iio/light/tsl2563.c +++ b/drivers/iio/light/tsl2563.c @@ -274,11 +274,7 @@ static void tsl2563_wait_adc(struct tsl2563_chip *chip) default: delay = 402; } - /* - * TODO: Make sure that we wait at least required delay but why we - * have to extend it one tick more? - */ - schedule_timeout_interruptible(msecs_to_jiffies(delay) + 2); + schedule_msec_hrtimeout_interruptible(delay + 1); } static int tsl2563_adjust_gainlevel(struct tsl2563_chip *chip, u16 adc) diff --git a/drivers/md/bcache/btree.c b/drivers/md/bcache/btree.c index 27a129d47..207dcfd35 100644 --- a/drivers/md/bcache/btree.c +++ b/drivers/md/bcache/btree.c @@ -1846,8 +1846,7 @@ static void bch_btree_gc(struct cache_set *c) cond_resched(); if (ret == -EAGAIN) - schedule_timeout_interruptible( - msecs_to_jiffies(btree_gc_sleep_ms(c))); + schedule_msec_hrtimeout_interruptible(btree_gc_sleep_ms(c)); else if (ret) pr_warn("gc failed!\n"); } while (ret && !test_bit(CACHE_SET_IO_DISABLE, &c->flags)); diff --git a/drivers/media/i2c/bt866.c b/drivers/media/i2c/bt866.c index f5104c7d9..f41bce209 100644 --- a/drivers/media/i2c/bt866.c +++ b/drivers/media/i2c/bt866.c @@ -65,7 +65,7 @@ static int bt866_write(struct bt866 *encoder, u8 subaddr, u8 data) err++; v4l_warn(client, "error #%d writing to 0x%02x\n", err, subaddr); - schedule_timeout_interruptible(msecs_to_jiffies(100)); + schedule_msec_hrtimeout_interruptible(100); } if (err == 3) { v4l_warn(client, "giving up\n"); diff --git a/drivers/media/i2c/ir-kbd-i2c.c b/drivers/media/i2c/ir-kbd-i2c.c index f2bf2b354..8ef956189 100644 --- a/drivers/media/i2c/ir-kbd-i2c.c +++ b/drivers/media/i2c/ir-kbd-i2c.c @@ -698,7 +698,7 @@ static int zilog_tx(struct rc_dev *rcdev, unsigned int *txbuf, */ for (i = 0; i < 20; ++i) { set_current_state(TASK_UNINTERRUPTIBLE); - schedule_timeout(msecs_to_jiffies(50)); + schedule_msec_hrtimeout(50); ret = i2c_master_send(ir->tx_c, buf, 1); if (ret == 1) break; diff --git a/drivers/media/i2c/msp3400-driver.c b/drivers/media/i2c/msp3400-driver.c index 413cfbc2d..f0a76210a 100644 --- a/drivers/media/i2c/msp3400-driver.c +++ b/drivers/media/i2c/msp3400-driver.c @@ -170,7 +170,7 @@ static int msp_read(struct i2c_client *client, int dev, int addr) break; dev_warn(&client->dev, "I/O error #%d (read 0x%02x/0x%02x)\n", err, dev, addr); - schedule_timeout_interruptible(msecs_to_jiffies(10)); + schedule_msec_hrtimeout_interruptible(10); } if (err == 3) { dev_warn(&client->dev, "resetting chip, sound will go off.\n"); @@ -211,7 +211,7 @@ static int msp_write(struct i2c_client *client, int dev, int addr, int val) break; dev_warn(&client->dev, "I/O error #%d (write 0x%02x/0x%02x)\n", err, dev, addr); - schedule_timeout_interruptible(msecs_to_jiffies(10)); + schedule_msec_hrtimeout_interruptible(10); } if (err == 3) { dev_warn(&client->dev, "resetting chip, sound will go off.\n"); diff --git a/drivers/media/i2c/saa7110.c b/drivers/media/i2c/saa7110.c index 652058b8f..82639968d 100644 --- a/drivers/media/i2c/saa7110.c +++ b/drivers/media/i2c/saa7110.c @@ -184,7 +184,7 @@ static v4l2_std_id determine_norm(struct v4l2_subdev *sd) saa7110_write_block(sd, initseq, sizeof(initseq)); saa7110_selmux(sd, decoder->input); prepare_to_wait(&decoder->wq, &wait, TASK_UNINTERRUPTIBLE); - schedule_timeout(msecs_to_jiffies(250)); + schedule_msec_hrtimeout(250); finish_wait(&decoder->wq, &wait); status = saa7110_read(sd); if (status & 0x40) { @@ -219,7 +219,7 @@ static v4l2_std_id determine_norm(struct v4l2_subdev *sd) /*saa7110_write(sd,0x2E,0x9A);*/ prepare_to_wait(&decoder->wq, &wait, TASK_UNINTERRUPTIBLE); - schedule_timeout(msecs_to_jiffies(250)); + schedule_msec_hrtimeout(250); finish_wait(&decoder->wq, &wait); status = saa7110_read(sd); diff --git a/drivers/media/pci/cx18/cx18-gpio.c b/drivers/media/pci/cx18/cx18-gpio.c index 4aea92639..baacf7820 100644 --- a/drivers/media/pci/cx18/cx18-gpio.c +++ b/drivers/media/pci/cx18/cx18-gpio.c @@ -81,11 +81,11 @@ static void gpio_reset_seq(struct cx18 *cx, u32 active_lo, u32 active_hi, /* Assert */ gpio_update(cx, mask, ~active_lo); - schedule_timeout_uninterruptible(msecs_to_jiffies(assert_msecs)); + schedule_msec_hrtimeout_uninterruptible(assert_msecs); /* Deassert */ gpio_update(cx, mask, ~active_hi); - schedule_timeout_uninterruptible(msecs_to_jiffies(recovery_msecs)); + schedule_msec_hrtimeout_uninterruptible(recovery_msecs); } /* diff --git a/drivers/media/pci/hws/hws_pci.c b/drivers/media/pci/hws/hws_pci.c index 30bb7d344..75b6527ee 100644 --- a/drivers/media/pci/hws/hws_pci.c +++ b/drivers/media/pci/hws/hws_pci.c @@ -215,7 +215,7 @@ static int main_ks_thread_handle(void *data) /* If we're suspending, don't touch hardware; just sleep/freeze. */ if (READ_ONCE(pdx->suspended)) { try_to_freeze(); - schedule_timeout_interruptible(msecs_to_jiffies(1000)); + schedule_msec_hrtimeout_interruptible(1000); continue; } @@ -225,7 +225,7 @@ static int main_ks_thread_handle(void *data) try_to_freeze(); /* cooperate with freezer each loop */ /* Sleep 1s or until signaled to wake/stop */ - schedule_timeout_interruptible(msecs_to_jiffies(1000)); + schedule_msec_hrtimeout_interruptible(1000); } dev_dbg(&pdx->pdev->dev, "%s: exiting\n", __func__); diff --git a/drivers/media/pci/ivtv/ivtv-gpio.c b/drivers/media/pci/ivtv/ivtv-gpio.c index d3477e152..1ca2b4d05 100644 --- a/drivers/media/pci/ivtv/ivtv-gpio.c +++ b/drivers/media/pci/ivtv/ivtv-gpio.c @@ -105,7 +105,7 @@ void ivtv_reset_ir_gpio(struct ivtv *itv) curout = (curout & ~0xF) | 1; write_reg(curout, IVTV_REG_GPIO_OUT); /* We could use something else for smaller time */ - schedule_timeout_interruptible(msecs_to_jiffies(1)); + schedule_msec_hrtimeout_interruptible(1); curout |= 2; write_reg(curout, IVTV_REG_GPIO_OUT); curdir &= ~0x80; @@ -125,11 +125,11 @@ int ivtv_reset_tuner_gpio(void *dev, int component, int cmd, int value) curout = read_reg(IVTV_REG_GPIO_OUT); curout &= ~(1 << itv->card->xceive_pin); write_reg(curout, IVTV_REG_GPIO_OUT); - schedule_timeout_interruptible(msecs_to_jiffies(1)); + schedule_msec_hrtimeout_interruptible(1); curout |= 1 << itv->card->xceive_pin; write_reg(curout, IVTV_REG_GPIO_OUT); - schedule_timeout_interruptible(msecs_to_jiffies(1)); + schedule_msec_hrtimeout_interruptible(1); return 0; } diff --git a/drivers/media/pci/ivtv/ivtv-ioctl.c b/drivers/media/pci/ivtv/ivtv-ioctl.c index fc95f0bf4..4e78628e4 100644 --- a/drivers/media/pci/ivtv/ivtv-ioctl.c +++ b/drivers/media/pci/ivtv/ivtv-ioctl.c @@ -1141,7 +1141,7 @@ void ivtv_s_std_dec(struct ivtv *itv, v4l2_std_id std) TASK_UNINTERRUPTIBLE); if ((read_reg(IVTV_REG_DEC_LINE_FIELD) >> 16) < 100) break; - schedule_timeout(msecs_to_jiffies(25)); + schedule_msec_hrtimeout(25); } finish_wait(&itv->vsync_waitq, &wait); mutex_lock(&itv->serialize_lock); diff --git a/drivers/media/pci/ivtv/ivtv-streams.c b/drivers/media/pci/ivtv/ivtv-streams.c index d98fe0c9d..45456ef79 100644 --- a/drivers/media/pci/ivtv/ivtv-streams.c +++ b/drivers/media/pci/ivtv/ivtv-streams.c @@ -835,7 +835,7 @@ int ivtv_stop_v4l2_encode_stream(struct ivtv_stream *s, int gop_end) while (!test_bit(IVTV_F_I_EOS, &itv->i_flags) && time_before(jiffies, then + msecs_to_jiffies(2000))) { - schedule_timeout(msecs_to_jiffies(10)); + schedule_msec_hrtimeout(10); } /* To convert jiffies to ms, we must multiply by 1000 diff --git a/drivers/media/pci/ivtv/ivtvfb.c b/drivers/media/pci/ivtv/ivtvfb.c index 4f55b4473..e9f345a8c 100644 --- a/drivers/media/pci/ivtv/ivtvfb.c +++ b/drivers/media/pci/ivtv/ivtvfb.c @@ -474,7 +474,7 @@ static int ivtvfb_ioctl(struct fb_info *info, unsigned int cmd, unsigned long ar case FBIO_WAITFORVSYNC: prepare_to_wait(&itv->vsync_waitq, &wait, TASK_INTERRUPTIBLE); - if (!schedule_timeout(msecs_to_jiffies(50))) + if (!schedule_msec_hrtimeout(50)) rc = -ETIMEDOUT; finish_wait(&itv->vsync_waitq, &wait); return rc; diff --git a/drivers/media/pci/saa7134/saa7134-tvaudio.c b/drivers/media/pci/saa7134/saa7134-tvaudio.c index 9e0c442ab..3476d4c25 100644 --- a/drivers/media/pci/saa7134/saa7134-tvaudio.c +++ b/drivers/media/pci/saa7134/saa7134-tvaudio.c @@ -307,8 +307,7 @@ static int tvaudio_sleep(struct saa7134_dev *dev, int timeout) set_current_state(TASK_INTERRUPTIBLE); schedule(); } else { - schedule_timeout_interruptible - (msecs_to_jiffies(timeout)); + schedule_msec_hrtimeout_interruptible(timeout); } } return dev->thread.scan1 != dev->thread.scan2; diff --git a/drivers/media/radio/radio-mr800.c b/drivers/media/radio/radio-mr800.c index d5a6053e1..835b78ff5 100644 --- a/drivers/media/radio/radio-mr800.c +++ b/drivers/media/radio/radio-mr800.c @@ -366,7 +366,7 @@ static int vidioc_s_hw_freq_seek(struct file *file, void *priv, retval = -ENODATA; break; } - if (schedule_timeout_interruptible(msecs_to_jiffies(10))) { + if (schedule_msec_hrtimeout_interruptible(10)) { retval = -ERESTARTSYS; break; } diff --git a/drivers/media/radio/radio-tea5777.c b/drivers/media/radio/radio-tea5777.c index fb9de7bbc..8417b0ab5 100644 --- a/drivers/media/radio/radio-tea5777.c +++ b/drivers/media/radio/radio-tea5777.c @@ -235,7 +235,7 @@ static int radio_tea5777_update_read_reg(struct radio_tea5777 *tea, int wait) } if (wait) { - if (schedule_timeout_interruptible(msecs_to_jiffies(wait))) + if (schedule_msec_hrtimeout_interruptible(wait)) return -ERESTARTSYS; } diff --git a/drivers/media/radio/tea575x.c b/drivers/media/radio/tea575x.c index c37315226..a7b182070 100644 --- a/drivers/media/radio/tea575x.c +++ b/drivers/media/radio/tea575x.c @@ -401,7 +401,7 @@ int snd_tea575x_s_hw_freq_seek(struct file *file, struct snd_tea575x *tea, for (;;) { if (time_after(jiffies, timeout)) break; - if (schedule_timeout_interruptible(msecs_to_jiffies(10))) { + if (schedule_msec_hrtimeout_interruptible(10)) { /* some signal arrived, stop search */ tea->val &= ~TEA575X_BIT_SEARCH; snd_tea575x_set_freq(tea); diff --git a/drivers/media/usb/gspca/m5602/m5602_s5k83a.c b/drivers/media/usb/gspca/m5602/m5602_s5k83a.c index 8ef010a87..7da826099 100644 --- a/drivers/media/usb/gspca/m5602/m5602_s5k83a.c +++ b/drivers/media/usb/gspca/m5602/m5602_s5k83a.c @@ -297,7 +297,7 @@ static int rotation_thread_function(void *data) __s32 vflip, hflip; set_current_state(TASK_INTERRUPTIBLE); - while (!schedule_timeout(msecs_to_jiffies(100))) { + while (!schedule_msec_hrtimeout(100)) { if (mutex_lock_interruptible(&sd->gspca_dev.usb_lock)) break; diff --git a/drivers/mfd/ucb1x00-core.c b/drivers/mfd/ucb1x00-core.c index 16f64e2b2..6439b0b29 100644 --- a/drivers/mfd/ucb1x00-core.c +++ b/drivers/mfd/ucb1x00-core.c @@ -253,7 +253,7 @@ unsigned int ucb1x00_adc_read(struct ucb1x00 *ucb, int adc_channel, int sync) break; /* yield to other processes */ set_current_state(TASK_INTERRUPTIBLE); - schedule_timeout(1); + schedule_min_hrtimeout(); } return UCB_ADC_DAT(val); diff --git a/drivers/misc/mei/hw-me.c b/drivers/misc/mei/hw-me.c index e7fbc02fb..9e54e2be2 100644 --- a/drivers/misc/mei/hw-me.c +++ b/drivers/misc/mei/hw-me.c @@ -1464,7 +1464,7 @@ int mei_me_polling_thread(void *_dev) MEI_POLLING_TIMEOUT_IDLE); } - schedule_timeout_interruptible(msecs_to_jiffies(polling_timeout)); + schedule_msec_hrtimeout_interruptible(polling_timeout); } return 0; diff --git a/drivers/misc/sgi-xp/xpc_channel.c b/drivers/misc/sgi-xp/xpc_channel.c index 8e6607fc8..b9ab770bb 100644 --- a/drivers/misc/sgi-xp/xpc_channel.c +++ b/drivers/misc/sgi-xp/xpc_channel.c @@ -834,7 +834,7 @@ xpc_allocate_msg_wait(struct xpc_channel *ch) atomic_inc(&ch->n_on_msg_allocate_wq); prepare_to_wait(&ch->msg_allocate_wq, &wait, TASK_INTERRUPTIBLE); - ret = schedule_timeout(1); + ret = schedule_min_hrtimeout(); finish_wait(&ch->msg_allocate_wq, &wait); atomic_dec(&ch->n_on_msg_allocate_wq); diff --git a/drivers/net/can/usb/peak_usb/pcan_usb.c b/drivers/net/can/usb/peak_usb/pcan_usb.c index 8fd058c32..67a2022f5 100644 --- a/drivers/net/can/usb/peak_usb/pcan_usb.c +++ b/drivers/net/can/usb/peak_usb/pcan_usb.c @@ -308,7 +308,7 @@ static int pcan_usb_write_mode(struct peak_usb_device *dev, u8 onoff) } else { /* the PCAN-USB needs time to init */ set_current_state(TASK_INTERRUPTIBLE); - schedule_timeout(msecs_to_jiffies(PCAN_USB_STARTUP_TIMEOUT)); + schedule_msec_hrtimeout(PCAN_USB_STARTUP_TIMEOUT); } return err; diff --git a/drivers/net/ethernet/cavium/liquidio/lio_ethtool.c b/drivers/net/ethernet/cavium/liquidio/lio_ethtool.c index c849e2c87..29404d5fe 100644 --- a/drivers/net/ethernet/cavium/liquidio/lio_ethtool.c +++ b/drivers/net/ethernet/cavium/liquidio/lio_ethtool.c @@ -1090,7 +1090,7 @@ static int lio_reset_queues(struct net_device *netdev, uint32_t num_qs) struct napi_struct *napi, *n; int ret; - schedule_timeout_uninterruptible(msecs_to_jiffies(100)); + schedule_msec_hrtimeout_uninterruptible(100); if (wait_for_pending_requests(oct)) dev_err(&oct->pci_dev->dev, "There were pending requests\n"); diff --git a/drivers/net/ethernet/cavium/liquidio/lio_main.c b/drivers/net/ethernet/cavium/liquidio/lio_main.c index e303956b4..19c601516 100644 --- a/drivers/net/ethernet/cavium/liquidio/lio_main.c +++ b/drivers/net/ethernet/cavium/liquidio/lio_main.c @@ -758,7 +758,7 @@ static int liquidio_watchdog(void *param) while (!kthread_should_stop()) { /* sleep for a couple of seconds so that we don't hog the CPU */ set_current_state(TASK_INTERRUPTIBLE); - schedule_timeout(msecs_to_jiffies(2000)); + schedule_msec_hrtimeout(2000); mask_of_crashed_or_stuck_cores = (u16)octeon_read_csr64(oct, CN23XX_SLI_SCRATCH2); diff --git a/drivers/net/usb/lan78xx.c b/drivers/net/usb/lan78xx.c index cb782d81d..734f9bcfe 100644 --- a/drivers/net/usb/lan78xx.c +++ b/drivers/net/usb/lan78xx.c @@ -3514,7 +3514,7 @@ static void lan78xx_terminate_urbs(struct lan78xx_net *dev) /* maybe wait for deletions to finish. */ while (!skb_queue_empty(&dev->rxq) || !skb_queue_empty(&dev->txq)) { - schedule_timeout(msecs_to_jiffies(UNLINK_TIMEOUT_MS)); + schedule_msec_hrtimeout(UNLINK_TIMEOUT_MS); set_current_state(TASK_UNINTERRUPTIBLE); netif_dbg(dev, ifdown, dev->net, "waited for %d urb completions", temp); diff --git a/drivers/net/usb/usbnet.c b/drivers/net/usb/usbnet.c index a19ecf718..bbc47dd8c 100644 --- a/drivers/net/usb/usbnet.c +++ b/drivers/net/usb/usbnet.c @@ -791,7 +791,7 @@ static void wait_skb_queue_empty(struct sk_buff_head *q) spin_lock_irqsave(&q->lock, flags); while (!skb_queue_empty(q)) { spin_unlock_irqrestore(&q->lock, flags); - schedule_timeout(msecs_to_jiffies(UNLINK_TIMEOUT_MS)); + schedule_msec_hrtimeout(UNLINK_TIMEOUT_MS); set_current_state(TASK_UNINTERRUPTIBLE); spin_lock_irqsave(&q->lock, flags); } diff --git a/drivers/net/wireless/intel/ipw2x00/ipw2100.c b/drivers/net/wireless/intel/ipw2x00/ipw2100.c index 2b8a23865..e5bd4439d 100644 --- a/drivers/net/wireless/intel/ipw2x00/ipw2100.c +++ b/drivers/net/wireless/intel/ipw2x00/ipw2100.c @@ -799,7 +799,7 @@ static int ipw2100_hw_send_command(struct ipw2100_priv *priv, * doesn't seem to have as many firmware restart cycles... * * As a test, we're sticking in a 1/100s delay here */ - schedule_timeout_uninterruptible(msecs_to_jiffies(10)); + schedule_msec_hrtimeout_uninterruptible(10); return 0; @@ -1250,7 +1250,7 @@ static int ipw2100_start_adapter(struct ipw2100_priv *priv) IPW_DEBUG_FW("Waiting for f/w initialization to complete...\n"); i = 5000; do { - schedule_timeout_uninterruptible(msecs_to_jiffies(40)); + schedule_msec_hrtimeout_uninterruptible(40); /* Todo... wait for sync command ... */ read_register(priv->net_dev, IPW_REG_INTA, &inta); diff --git a/drivers/parport/ieee1284.c b/drivers/parport/ieee1284.c index 403501024..333faf78b 100644 --- a/drivers/parport/ieee1284.c +++ b/drivers/parport/ieee1284.c @@ -202,7 +202,7 @@ int parport_wait_peripheral(struct parport *port, /* parport_wait_event didn't time out, but the * peripheral wasn't actually ready either. * Wait for another 10ms. */ - schedule_timeout_interruptible(msecs_to_jiffies(10)); + schedule_msec_hrtimeout_interruptible(10); } } diff --git a/drivers/parport/ieee1284_ops.c b/drivers/parport/ieee1284_ops.c index 17061f1df..447bfe476 100644 --- a/drivers/parport/ieee1284_ops.c +++ b/drivers/parport/ieee1284_ops.c @@ -520,7 +520,7 @@ size_t parport_ieee1284_ecp_read_data (struct parport *port, /* Yield the port for a while. */ if (dev->port->irq != PARPORT_IRQ_NONE) { parport_release (dev); - schedule_timeout_interruptible(msecs_to_jiffies(40)); + schedule_msec_hrtimeout_interruptible(40); parport_claim_or_block (dev); } else diff --git a/drivers/parport/parport_pc.c b/drivers/parport/parport_pc.c index c75abdd8e..63fbb7d55 100644 --- a/drivers/parport/parport_pc.c +++ b/drivers/parport/parport_pc.c @@ -177,8 +177,7 @@ static int change_mode(struct parport *p, int m) if (time_after_eq(jiffies, expire)) /* The FIFO is stuck. */ return -EBUSY; - schedule_timeout_interruptible( - msecs_to_jiffies(10)); + schedule_msec_hrtimeout_interruptible(10); if (signal_pending(current)) break; } diff --git a/drivers/platform/x86/intel_ips.c b/drivers/platform/x86/intel_ips.c index b1b2d9cab..2a282f8cd 100644 --- a/drivers/platform/x86/intel_ips.c +++ b/drivers/platform/x86/intel_ips.c @@ -806,7 +806,7 @@ static int ips_adjust(void *data) ips_gpu_lower(ips); sleep: - schedule_timeout_interruptible(msecs_to_jiffies(IPS_ADJUST_PERIOD)); + schedule_msec_hrtimeout_interruptible(IPS_ADJUST_PERIOD); } while (!kthread_should_stop()); dev_dbg(ips->dev, "ips-adjust thread stopped\n"); @@ -984,7 +984,7 @@ static int ips_monitor(void *data) seqno_timestamp = get_jiffies_64(); old_cpu_power = thm_readl(THM_CEC); - schedule_timeout_interruptible(msecs_to_jiffies(IPS_SAMPLE_PERIOD)); + schedule_msec_hrtimeout_interruptible(IPS_SAMPLE_PERIOD); /* Collect an initial average */ for (i = 0; i < IPS_SAMPLE_COUNT; i++) { @@ -1011,7 +1011,7 @@ static int ips_monitor(void *data) mchp_samples[i] = mchp; } - schedule_timeout_interruptible(msecs_to_jiffies(IPS_SAMPLE_PERIOD)); + schedule_msec_hrtimeout_interruptible(IPS_SAMPLE_PERIOD); if (kthread_should_stop()) break; } @@ -1038,7 +1038,7 @@ static int ips_monitor(void *data) * us to reduce the sample frequency if the CPU and GPU are idle. */ old_cpu_power = thm_readl(THM_CEC); - schedule_timeout_interruptible(msecs_to_jiffies(IPS_SAMPLE_PERIOD)); + schedule_msec_hrtimeout_interruptible(IPS_SAMPLE_PERIOD); last_sample_period = IPS_SAMPLE_PERIOD; timer_setup(&ips->timer, monitor_timeout, TIMER_DEFERRABLE); diff --git a/drivers/rtc/rtc-wm8350.c b/drivers/rtc/rtc-wm8350.c index 3bd60d067..3c392987c 100644 --- a/drivers/rtc/rtc-wm8350.c +++ b/drivers/rtc/rtc-wm8350.c @@ -114,7 +114,7 @@ static int wm8350_rtc_settime(struct device *dev, struct rtc_time *tm) /* Wait until confirmation of stopping */ do { rtc_ctrl = wm8350_reg_read(wm8350, WM8350_RTC_TIME_CONTROL); - schedule_timeout_uninterruptible(msecs_to_jiffies(1)); + schedule_msec_hrtimeout_uninterruptible(1); } while (--retries && !(rtc_ctrl & WM8350_RTC_STS)); if (!retries) { @@ -197,7 +197,7 @@ static int wm8350_rtc_stop_alarm(struct wm8350 *wm8350) /* Wait until confirmation of stopping */ do { rtc_ctrl = wm8350_reg_read(wm8350, WM8350_RTC_TIME_CONTROL); - schedule_timeout_uninterruptible(msecs_to_jiffies(1)); + schedule_msec_hrtimeout_uninterruptible(1); } while (retries-- && !(rtc_ctrl & WM8350_RTC_ALMSTS)); if (!(rtc_ctrl & WM8350_RTC_ALMSTS)) @@ -220,7 +220,7 @@ static int wm8350_rtc_start_alarm(struct wm8350 *wm8350) /* Wait until confirmation */ do { rtc_ctrl = wm8350_reg_read(wm8350, WM8350_RTC_TIME_CONTROL); - schedule_timeout_uninterruptible(msecs_to_jiffies(1)); + schedule_msec_hrtimeout_uninterruptible(1); } while (retries-- && rtc_ctrl & WM8350_RTC_ALMSTS); if (rtc_ctrl & WM8350_RTC_ALMSTS) diff --git a/drivers/s390/char/sclp_sdias.c b/drivers/s390/char/sclp_sdias.c index ab8f1b758..a4a78b179 100644 --- a/drivers/s390/char/sclp_sdias.c +++ b/drivers/s390/char/sclp_sdias.c @@ -68,7 +68,7 @@ static int sdias_sclp_send(struct sclp_req *req) /* not initiated, wait some time and retry */ set_current_state(TASK_INTERRUPTIBLE); TRACE("add request failed: rc = %i\n",rc); - schedule_timeout(msecs_to_jiffies(500)); + schedule_msec_hrtimeout(500); continue; } /* initiated, wait for completion of service call */ diff --git a/drivers/scsi/be2iscsi/be_cmds.c b/drivers/scsi/be2iscsi/be_cmds.c index 0b59b63bc..9006c0b60 100644 --- a/drivers/scsi/be2iscsi/be_cmds.c +++ b/drivers/scsi/be2iscsi/be_cmds.c @@ -611,7 +611,7 @@ static int be_mbox_db_ready_poll(struct be_ctrl_info *ctrl) if (time_after(jiffies, timeout)) break; /* 1ms sleep is enough in most cases */ - schedule_timeout_uninterruptible(msecs_to_jiffies(1)); + schedule_msec_hrtimeout_uninterruptible(1); } while (!ready); beiscsi_log(phba, KERN_ERR, diff --git a/drivers/scsi/esas2r/esas2r_flash.c b/drivers/scsi/esas2r/esas2r_flash.c index f910e2553..548d16cd1 100644 --- a/drivers/scsi/esas2r/esas2r_flash.c +++ b/drivers/scsi/esas2r/esas2r_flash.c @@ -965,7 +965,7 @@ static bool esas2r_flash_access(struct esas2r_adapter *a, u32 function) break; } - schedule_timeout_interruptible(msecs_to_jiffies(100)); + schedule_msec_hrtimeout_interruptible(100); if ((jiffies_to_msecs(jiffies) - starttime) > timeout) { /* diff --git a/drivers/scsi/esas2r/esas2r_init.c b/drivers/scsi/esas2r/esas2r_init.c index 0a35f1953..edff2f622 100644 --- a/drivers/scsi/esas2r/esas2r_init.c +++ b/drivers/scsi/esas2r/esas2r_init.c @@ -994,7 +994,7 @@ bool esas2r_check_adapter(struct esas2r_adapter *a) break; } - schedule_timeout_interruptible(msecs_to_jiffies(100)); + schedule_msec_hrtimeout_interruptible(100); if ((jiffies_to_msecs(jiffies) - starttime) > 180000) { esas2r_hdebug("FW ready TMO"); @@ -1017,7 +1017,7 @@ bool esas2r_check_adapter(struct esas2r_adapter *a) break; } - schedule_timeout_interruptible(msecs_to_jiffies(50)); + schedule_msec_hrtimeout_interruptible(50); if ((jiffies_to_msecs(jiffies) - starttime) > 3000) { esas2r_hdebug("timeout waiting for interface down"); @@ -1106,7 +1106,7 @@ bool esas2r_check_adapter(struct esas2r_adapter *a) break; } - schedule_timeout_interruptible(msecs_to_jiffies(100)); + schedule_msec_hrtimeout_interruptible(100); if ((jiffies_to_msecs(jiffies) - starttime) > 3000) { esas2r_hdebug( @@ -1356,7 +1356,7 @@ bool esas2r_init_adapter_hw(struct esas2r_adapter *a, bool init_poll) atomic_dec(&a->disable_cnt); while (test_bit(AF_DISC_PENDING, &a->flags)) { - schedule_timeout_interruptible(msecs_to_jiffies(100)); + schedule_msec_hrtimeout_interruptible(100); /* * Determine the need for a timer tick based on the @@ -1512,7 +1512,7 @@ static void esas2r_power_down_notify_firmware(struct esas2r_adapter *a) break; } - schedule_timeout_interruptible(msecs_to_jiffies(100)); + schedule_msec_hrtimeout_interruptible(100); if ((jiffies_to_msecs(jiffies) - starttime) > 30000) { esas2r_hdebug("Timeout waiting for power down"); @@ -1557,7 +1557,7 @@ void esas2r_power_down(struct esas2r_adapter *a) break; } - schedule_timeout_interruptible(msecs_to_jiffies(100)); + schedule_msec_hrtimeout_interruptible(100); if ((jiffies_to_msecs(jiffies) - starttime) > 3000) { esas2r_hdebug( diff --git a/drivers/scsi/esas2r/esas2r_main.c b/drivers/scsi/esas2r/esas2r_main.c index ada278c24..1182781e3 100644 --- a/drivers/scsi/esas2r/esas2r_main.c +++ b/drivers/scsi/esas2r/esas2r_main.c @@ -1247,7 +1247,7 @@ void esas2r_wait_request(struct esas2r_adapter *a, struct esas2r_request *rq) if (rq->req_stat != RS_STARTED) break; - schedule_timeout_interruptible(msecs_to_jiffies(100)); + schedule_msec_hrtimeout_interruptible(100); if ((jiffies_to_msecs(jiffies) - starttime) > timeout) { esas2r_hdebug("request TMO"); diff --git a/drivers/scsi/fnic/fnic_scsi.c b/drivers/scsi/fnic/fnic_scsi.c index 6ee3c559e..d1a18b0c4 100644 --- a/drivers/scsi/fnic/fnic_scsi.c +++ b/drivers/scsi/fnic/fnic_scsi.c @@ -258,7 +258,7 @@ int fnic_fw_reset_handler(struct fnic *fnic) /* wait for io cmpl */ while (atomic_read(&fnic->in_flight)) - schedule_timeout(msecs_to_jiffies(1)); + schedule_msec_hrtimeout(1); spin_lock_irqsave(&fnic->wq_copy_lock[0], flags); @@ -1635,7 +1635,7 @@ static void fnic_cleanup_io(struct fnic *fnic, int exclude_id) io_count, atomic64_read(&fnic->fnic_stats.io_stats.active_ios)); - schedule_timeout(msecs_to_jiffies(100)); + schedule_msec_hrtimeout(100); } } @@ -1913,7 +1913,7 @@ void fnic_rport_exch_reset(struct fnic *fnic, u32 port_id) atomic_dec(&fnic->in_flight); while ((io_count = fnic_count_ioreqs(fnic, port_id))) - schedule_timeout(msecs_to_jiffies(1000)); + schedule_msec_hrtimeout(1000); FNIC_SCSI_DBG(KERN_DEBUG, fnic->host, fnic->fnic_num, "rport: 0x%x remaining portid-io-count: %d ", @@ -2511,7 +2511,7 @@ static int fnic_clean_pending_aborts(struct fnic *fnic, ret = iter_data.ret; goto clean_pending_aborts_end; } - schedule_timeout(msecs_to_jiffies(2 * fnic->config.ed_tov)); + schedule_msec_hrtimeout(2 * fnic->config.ed_tov); /* walk again to check, if IOs are still pending in fw */ if (fnic_is_abts_pending(fnic, lr_sc)) @@ -2813,7 +2813,7 @@ int fnic_device_reset(struct scsi_cmnd *sc) } FNIC_SCSI_DBG(KERN_ERR, fnic->host, fnic->fnic_num, "Cannot clean up all IOs for the LUN\n"); - schedule_timeout(msecs_to_jiffies(1000)); + schedule_msec_hrtimeout(1000); count++; } diff --git a/drivers/scsi/lpfc/lpfc_scsi.c b/drivers/scsi/lpfc/lpfc_scsi.c index f2cab134a..b766ed936 100644 --- a/drivers/scsi/lpfc/lpfc_scsi.c +++ b/drivers/scsi/lpfc/lpfc_scsi.c @@ -5920,7 +5920,7 @@ lpfc_chk_tgt_mapped(struct lpfc_vport *vport, struct fc_rport *rport) return FAILED; if (pnode->nlp_state == NLP_STE_MAPPED_NODE) return SUCCESS; - schedule_timeout_uninterruptible(msecs_to_jiffies(500)); + schedule_msec_hrtimeout_uninterruptible(500); rdata = rport->dd_data; if (!rdata) return FAILED; @@ -5962,7 +5962,7 @@ lpfc_reset_flush_io_context(struct lpfc_vport *vport, uint16_t tgt_id, tgt_id, lun_id, context); later = secs_to_jiffies(2 * vport->cfg_devloss_tmo) + jiffies; while (time_after(later, jiffies) && cnt) { - schedule_timeout_uninterruptible(msecs_to_jiffies(20)); + schedule_msec_hrtimeout_uninterruptible(20); cnt = lpfc_sli_sum_iocb(vport, tgt_id, lun_id, context); } if (cnt) { diff --git a/drivers/scsi/qla4xxx/ql4_nx.c b/drivers/scsi/qla4xxx/ql4_nx.c index f7340cfc9..d8c704589 100644 --- a/drivers/scsi/qla4xxx/ql4_nx.c +++ b/drivers/scsi/qla4xxx/ql4_nx.c @@ -1552,7 +1552,7 @@ static int qla4_82xx_cmdpeg_ready(struct scsi_qla_host *ha, int pegtune_val) (val == PHAN_INITIALIZE_ACK)) return 0; set_current_state(TASK_UNINTERRUPTIBLE); - schedule_timeout(msecs_to_jiffies(500)); + schedule_msec_hrtimeout(500); } while (--retries); diff --git a/drivers/scsi/snic/snic_scsi.c b/drivers/scsi/snic/snic_scsi.c index c6af3b8d2..204987f20 100644 --- a/drivers/scsi/snic/snic_scsi.c +++ b/drivers/scsi/snic/snic_scsi.c @@ -1863,7 +1863,7 @@ snic_dr_clean_pending_req(struct snic *snic, struct scsi_cmnd *lr_sc) } } - schedule_timeout(msecs_to_jiffies(100)); + schedule_msec_hrtimeout(100); /* Walk through all the cmds and check abts status. */ if (snic_is_abts_pending(snic, lr_sc)) @@ -2321,7 +2321,7 @@ snic_reset(struct Scsi_Host *shost, struct scsi_cmnd *sc) /* Wait for all the IOs that are entered in Qcmd */ while (atomic_read(&snic->ios_inflight)) - schedule_timeout(msecs_to_jiffies(1)); + schedule_msec_hrtimeout(1); ret = snic_issue_hba_reset(snic, sc); if (ret) { diff --git a/drivers/usb/class/usblp.c b/drivers/usb/class/usblp.c index 746414763..c568c0be1 100644 --- a/drivers/usb/class/usblp.c +++ b/drivers/usb/class/usblp.c @@ -923,7 +923,7 @@ static int usblp_wwait(struct usblp *usblp, int nonblock) if (rc <= 0) break; - if (schedule_timeout(msecs_to_jiffies(1500)) == 0) { + if (schedule_msec_hrtimeout(1500) == 0) { if (usblp->flags & LP_ABORT) { err = usblp_check_status(usblp, err); if (err == 1) { /* Paper out */ diff --git a/drivers/video/fbdev/omap/hwa742.c b/drivers/video/fbdev/omap/hwa742.c index 68a677f82..432469312 100644 --- a/drivers/video/fbdev/omap/hwa742.c +++ b/drivers/video/fbdev/omap/hwa742.c @@ -926,7 +926,7 @@ static void hwa742_resume(void) if (hwa742_read_reg(HWA742_PLL_DIV_REG) & (1 << 7)) break; set_current_state(TASK_UNINTERRUPTIBLE); - schedule_timeout(msecs_to_jiffies(5)); + schedule_msec_hrtimeout(5); } hwa742_set_update_mode(hwa742.update_mode_before_suspend); } diff --git a/drivers/video/fbdev/pxafb.c b/drivers/video/fbdev/pxafb.c index e418eee82..f49ae1dcf 100644 --- a/drivers/video/fbdev/pxafb.c +++ b/drivers/video/fbdev/pxafb.c @@ -1272,7 +1272,7 @@ static int pxafb_smart_thread(void *arg) mutex_unlock(&fbi->ctrlr_lock); set_current_state(TASK_INTERRUPTIBLE); - schedule_timeout(msecs_to_jiffies(30)); + schedule_msec_hrtimeout(30); } pr_debug("%s(): task ending\n", __func__); diff --git a/fs/f2fs/checkpoint.c b/fs/f2fs/checkpoint.c index 064f5b537..a81cbd7b8 100644 --- a/fs/f2fs/checkpoint.c +++ b/fs/f2fs/checkpoint.c @@ -28,7 +28,7 @@ static inline void get_lock_elapsed_time(struct f2fs_time_stat *ts) { ts->total_time = ktime_get(); #ifdef CONFIG_64BIT - ts->running_time = current->se.sum_exec_runtime; + ts->running_time = tsk_seruntime(current); #endif #if defined(CONFIG_SCHED_INFO) && defined(CONFIG_SCHEDSTATS) ts->runnable_time = current->sched_info.run_delay; diff --git a/fs/jffs2/background.c b/fs/jffs2/background.c index bb0ee1a59..1c0aaf07e 100644 --- a/fs/jffs2/background.c +++ b/fs/jffs2/background.c @@ -110,7 +110,7 @@ static int jffs2_garbage_collect_thread(void *_c) * This forces the GCD to slow the hell down. Pulling an * inode in with read_inode() is much preferable to having * the GC thread get there first. */ - schedule_timeout_interruptible(msecs_to_jiffies(50)); + schedule_msec_hrtimeout_interruptible(50); if (kthread_should_stop()) { jffs2_dbg(1, "%s(): kthread_stop() called\n", __func__); diff --git a/fs/proc/base.c b/fs/proc/base.c index 780f81259..ac935224d 100644 --- a/fs/proc/base.c +++ b/fs/proc/base.c @@ -513,7 +513,7 @@ static int proc_pid_schedstat(struct seq_file *m, struct pid_namespace *ns, seq_puts(m, "0 0 0\n"); else seq_printf(m, "%llu %llu %lu\n", - (unsigned long long)task->se.sum_exec_runtime, + (unsigned long long)tsk_seruntime(task), (unsigned long long)task->sched_info.run_delay, task->sched_info.pcount); @@ -521,6 +521,108 @@ static int proc_pid_schedstat(struct seq_file *m, struct pid_namespace *ns, } #endif +#ifdef CONFIG_MUQSS_IOTIME +/* + * Provides /proc/PID/iotime + * + * Block device time consumed on this task's behalf, in nanoseconds. + * + * io_latency_ns is end to end per bio and so includes queue wait: it is what + * this task waited for. io_occupancy_ns is device service time per request: + * it is what this task cost everybody else, and is what scheduling charges + * are based on. io_debt_ns is occupancy not yet charged to the deadline. + * See block/blk-iotime.c. + * + * kern_time_ns is CPU time rather than device time: kernel work done for this + * task in another thread's context, which today means kworkers running work + * items it queued. kern_debt_ns is the part of it not yet charged. Work the + * task asks for in its own context is not counted here, because that already + * comes out of its own timeslice. + * + * Access is restricted exactly as /proc/PID/io is, and for the same reason: + * these counters say when a task used the device and for how long, which is + * enough to infer a good deal about what it is doing. The exec_update_lock + * keeps the permission check and the read on the same side of an exec, so + * credentials cannot change between the two. + * + * Also as /proc/PID/io: the tgid file sums the thread group and the tid file + * reports one thread. Charging is per thread, since each has its own deadline, + * but a multithreaded writer's total is only meaningful summed. + */ +struct iotime_acct { + u64 latency_ns; + u64 count; + u64 occupancy_ns; + u64 debt_ns; + u64 kern_time_ns; + u64 kern_debt_ns; +}; + +static void iotime_add(struct iotime_acct *acct, struct task_struct *task) +{ + acct->latency_ns += atomic64_read(&task->io_latency_ns); + acct->count += atomic64_read(&task->io_count); + acct->occupancy_ns += atomic64_read(&task->io_occupancy_ns); + acct->debt_ns += atomic64_read(&task->io_debt_ns); + acct->kern_time_ns += atomic64_read(&task->kern_time_ns); + acct->kern_debt_ns += atomic64_read(&task->kern_debt_ns); +} + +static int do_iotime_accounting(struct task_struct *task, struct seq_file *m, + int whole) +{ + struct iotime_acct acct = { }; + int result; + + result = down_read_killable(&task->signal->exec_update_lock); + if (result) + return result; + + if (!ptrace_may_access(task, PTRACE_MODE_READ_FSCREDS)) { + result = -EACCES; + goto out_unlock; + } + + if (whole) { + struct task_struct *t; + + rcu_read_lock(); + for_each_thread(task, t) + iotime_add(&acct, t); + rcu_read_unlock(); + } else { + iotime_add(&acct, task); + } + + seq_printf(m, + "io_latency_ns %llu\n" + "io_count %llu\n" + "io_occupancy_ns %llu\n" + "io_debt_ns %llu\n" + "kern_time_ns %llu\n" + "kern_debt_ns %llu\n", + acct.latency_ns, acct.count, acct.occupancy_ns, + acct.debt_ns, acct.kern_time_ns, acct.kern_debt_ns); + +out_unlock: + up_read(&task->signal->exec_update_lock); + return result; +} + +static int proc_tgid_iotime(struct seq_file *m, struct pid_namespace *ns, + struct pid *pid, struct task_struct *task) +{ + return do_iotime_accounting(task, m, 1); +} + +static int proc_tid_iotime(struct seq_file *m, struct pid_namespace *ns, + struct pid *pid, struct task_struct *task) +{ + return do_iotime_accounting(task, m, 0); +} + +#endif + #ifdef CONFIG_LATENCYTOP static int lstats_show_proc(struct seq_file *m, void *v) { @@ -3336,6 +3438,9 @@ static const struct pid_entry tgid_base_stuff[] = { #ifdef CONFIG_SCHED_INFO ONE("schedstat", S_IRUGO, proc_pid_schedstat), #endif +#ifdef CONFIG_MUQSS_IOTIME + ONE("iotime", S_IRUSR, proc_tgid_iotime), +#endif #ifdef CONFIG_LATENCYTOP REG("latency", S_IRUGO, proc_lstats_operations), #endif @@ -3691,6 +3796,9 @@ static const struct pid_entry tid_base_stuff[] = { #ifdef CONFIG_SCHED_INFO ONE("schedstat", S_IRUGO, proc_pid_schedstat), #endif +#ifdef CONFIG_MUQSS_IOTIME + ONE("iotime", S_IRUSR, proc_tid_iotime), +#endif #ifdef CONFIG_LATENCYTOP REG("latency", S_IRUGO, proc_lstats_operations), #endif diff --git a/fs/xfs/xfs_trans_ail.c b/fs/xfs/xfs_trans_ail.c index 99a9bf376..73676f8c4 100644 --- a/fs/xfs/xfs_trans_ail.c +++ b/fs/xfs/xfs_trans_ail.c @@ -714,7 +714,7 @@ xfsaild( spin_unlock(&ailp->ail_lock); if (tout) - schedule_timeout(msecs_to_jiffies(tout)); + schedule_msec_hrtimeout(tout); __set_current_state(TASK_RUNNING); diff --git a/include/linux/blk-mq.h b/include/linux/blk-mq.h index af878597a..bb10e083f 100644 --- a/include/linux/blk-mq.h +++ b/include/linux/blk-mq.h @@ -137,6 +137,23 @@ struct request { /* Time that I/O was submitted to the device. */ u64 io_start_time_ns; +#ifdef CONFIG_MUQSS_IOTIME + /* + * Task this request is being served for, holding a reference. Set + * once from the first bio and cleared if a merge brings in a bio + * belonging to somebody else, since occupancy cannot be split + * between owners at completion time. + */ + struct task_struct *muqss_owner; + /* + * Completion time, captured where the block layer already takes it. + * rq_qos_done() runs after blk_update_request() has completed every + * bio, so a timestamp taken there would include the completion path + * and overstate device service time. + */ + u64 muqss_done_ns; +#endif + #ifdef CONFIG_BLK_WBT unsigned short wbt_flags; #endif diff --git a/include/linux/blk_types.h b/include/linux/blk_types.h index 8808ee76e..f00ed6c5e 100644 --- a/include/linux/blk_types.h +++ b/include/linux/blk_types.h @@ -252,13 +252,25 @@ struct bio { * on release of the bio. */ struct blkcg_gq *bi_blkg; - /* Time that this bio was issued. */ - u64 issue_time_ns; #ifdef CONFIG_BLK_CGROUP_IOCOST u64 bi_iocost_cost; #endif #endif + /* + * Time that this bio was issued. Wanted by blkcg accounting and by + * MuQSS I/O aware scheduling, which must work on kernels built + * without any block cgroup support. + */ +#if defined(CONFIG_BLK_CGROUP) || defined(CONFIG_MUQSS_IOTIME) + u64 issue_time_ns; +#endif + +#ifdef CONFIG_MUQSS_IOTIME + /* Task this I/O is being done for. Holds a reference. */ + struct task_struct *bi_muqss_owner; +#endif + #ifdef CONFIG_BLK_INLINE_ENCRYPTION struct bio_crypt_ctx *bi_crypt_context; #endif diff --git a/include/linux/freezer.h b/include/linux/freezer.h index 0a8c6c4d1..39ffe7835 100644 --- a/include/linux/freezer.h +++ b/include/linux/freezer.h @@ -91,6 +91,7 @@ static inline bool try_to_freeze(void) { return false; } static inline void set_freezable(void) {} +#define pm_freezing (false) #endif /* !CONFIG_FREEZER */ #endif /* FREEZER_H_INCLUDED */ diff --git a/include/linux/ioprio.h b/include/linux/ioprio.h index 5210e8371..e82669586 100644 --- a/include/linux/ioprio.h +++ b/include/linux/ioprio.h @@ -29,6 +29,8 @@ static inline bool ioprio_valid(unsigned short ioprio) */ static inline int task_nice_ioprio(struct task_struct *task) { + if (iso_task(task)) + return 0; return (task_nice(task) + 20) / 5; } diff --git a/include/linux/mm.h b/include/linux/mm.h index 485df9c2d..3a3ed9cb8 100644 --- a/include/linux/mm.h +++ b/include/linux/mm.h @@ -2460,6 +2460,61 @@ static inline bool folio_use_access_time(struct folio *folio) } #endif /* CONFIG_NUMA_BALANCING */ +#if MUQSS_IOWNER_WIDTH +/* + * MuQSS I/O owner tag: an index into the owner table in block/blk-iotime.c, + * recorded when a folio is dirtied so that the writeback issued for it later + * by a flusher thread can still be charged to the task responsible. Zero + * means no owner. + * + * The tag itself is lossy by design: which of two racing dirtiers wins the + * field does not matter, and a wrong answer costs one misattributed + * writeback. The *word* is not ours to be lossy with, though. Other fields in + * folio->flags are updated by atomic bit operations from contexts that do not + * hold the folio lock -- PG_waiters is set by lock waiters, and losing it + * skips the wakeup on unlock -- so a plain read-modify-write store here would + * discard concurrent updates to unrelated bits. Update the field the way + * folio_xchg_last_cpupid() does, with a cmpxchg loop that touches only our + * own bits. + */ +static inline unsigned int folio_io_owner(const struct folio *folio) +{ + return (READ_ONCE(folio->flags.f) >> MUQSS_IOWNER_PGSHIFT) & + MUQSS_IOWNER_MASK; +} + +static inline void folio_set_io_owner(struct folio *folio, unsigned int slot) +{ + unsigned long old_flags, flags; + + old_flags = READ_ONCE(folio->flags.f); + do { + flags = old_flags; + flags &= ~(MUQSS_IOWNER_MASK << MUQSS_IOWNER_PGSHIFT); + flags |= ((unsigned long)slot & MUQSS_IOWNER_MASK) << + MUQSS_IOWNER_PGSHIFT; + } while (unlikely(!try_cmpxchg(&folio->flags.f, &old_flags, flags))); +} + +static inline void page_io_owner_reset(struct page *page) +{ + page->flags.f &= ~(MUQSS_IOWNER_MASK << MUQSS_IOWNER_PGSHIFT); +} +#else +static inline unsigned int folio_io_owner(const struct folio *folio) +{ + return 0; +} + +static inline void folio_set_io_owner(struct folio *folio, unsigned int slot) +{ +} + +static inline void page_io_owner_reset(struct page *page) +{ +} +#endif /* MUQSS_IOWNER_WIDTH */ + #if defined(CONFIG_KASAN_SW_TAGS) || defined(CONFIG_KASAN_HW_TAGS) /* diff --git a/include/linux/mmzone.h b/include/linux/mmzone.h index ca2712187..c39859884 100644 --- a/include/linux/mmzone.h +++ b/include/linux/mmzone.h @@ -1240,6 +1240,7 @@ static inline bool zone_is_empty(const struct zone *zone) #define KASAN_TAG_PGOFF (LAST_CPUPID_PGOFF - KASAN_TAG_WIDTH) #define LRU_GEN_PGOFF (KASAN_TAG_PGOFF - LRU_GEN_WIDTH) #define LRU_REFS_PGOFF (LRU_GEN_PGOFF - LRU_REFS_WIDTH) +#define MUQSS_IOWNER_PGOFF (LRU_REFS_PGOFF - MUQSS_IOWNER_WIDTH) /* * Define the bit shifts to access each section. For non-existent @@ -1251,6 +1252,7 @@ static inline bool zone_is_empty(const struct zone *zone) #define ZONES_PGSHIFT (ZONES_PGOFF * (ZONES_WIDTH != 0)) #define LAST_CPUPID_PGSHIFT (LAST_CPUPID_PGOFF * (LAST_CPUPID_WIDTH != 0)) #define KASAN_TAG_PGSHIFT (KASAN_TAG_PGOFF * (KASAN_TAG_WIDTH != 0)) +#define MUQSS_IOWNER_PGSHIFT (MUQSS_IOWNER_PGOFF * (MUQSS_IOWNER_WIDTH != 0)) /* NODE:ZONE or SECTION:ZONE is used to ID a zone for the buddy allocator */ #ifdef NODE_NOT_IN_PAGE_FLAGS @@ -1269,6 +1271,7 @@ static inline bool zone_is_empty(const struct zone *zone) #define NODES_MASK ((1UL << NODES_WIDTH) - 1) #define SECTIONS_MASK ((1UL << SECTIONS_WIDTH) - 1) #define LAST_CPUPID_MASK ((1UL << LAST_CPUPID_SHIFT) - 1) +#define MUQSS_IOWNER_MASK ((1UL << MUQSS_IOWNER_WIDTH) - 1) #define KASAN_TAG_MASK ((1UL << KASAN_TAG_WIDTH) - 1) #define ZONEID_MASK ((1UL << ZONEID_SHIFT) - 1) diff --git a/include/linux/muqss.h b/include/linux/muqss.h new file mode 100644 index 000000000..3ada43230 --- /dev/null +++ b/include/linux/muqss.h @@ -0,0 +1,51 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Scheduler-neutral accessors for task fields that MuQSS keeps somewhere + * other than the mainline scheduling entities. + * + * Kept out of so that enabling MuQSS costs that header only + * the task_struct members themselves. Include this after task_struct is + * defined; already does so on your behalf. + */ +#ifndef _LINUX_MUQSS_H +#define _LINUX_MUQSS_H + +#ifdef CONFIG_SCHED_MUQSS + +/* MuQSS accounts runtime and RT timeouts directly on the task. */ +#define tsk_seruntime(t) ((t)->sched_time) +#define tsk_rttimeout(t) ((t)->rt_timeout) + +/* + * The rtmutex PI chain sorts equal-priority waiters by SCHED_DEADLINE + * deadline. MuQSS implements no deadline class, and that tie-break is only + * consulted when dl_prio(prio) holds - i.e. prio < MAX_DL_PRIO, which is 0 - + * so it is unreachable here. Report 0 rather than carry a dl entity. + */ +#define tsk_dl_deadline(t) (0) + +/* No deadline class: no budget to overrun and no bandwidth to migrate. */ +#define tsk_dl_overrun(t) (0) +#define tsk_dl_bw(t) (0) + +static inline bool iso_task(struct task_struct *p) +{ + return (p->policy == SCHED_ISO); +} + +#else /* CONFIG_SCHED_MUQSS */ + +#define tsk_seruntime(t) ((t)->se.sum_exec_runtime) +#define tsk_rttimeout(t) ((t)->rt.timeout) +#define tsk_dl_deadline(t) ((t)->dl.deadline) +#define tsk_dl_overrun(t) ((t)->dl.dl_overrun) +#define tsk_dl_bw(t) ((t)->dl.dl_bw) + +static inline bool iso_task(struct task_struct *p) +{ + return false; +} + +#endif /* CONFIG_SCHED_MUQSS */ + +#endif /* _LINUX_MUQSS_H */ diff --git a/include/linux/muqss_iotime.h b/include/linux/muqss_iotime.h new file mode 100644 index 000000000..e17c54e90 --- /dev/null +++ b/include/linux/muqss_iotime.h @@ -0,0 +1,166 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * I/O aware scheduling support for MuQSS. + * + * Attributes block device time back to the task that caused the I/O, so the + * CPU scheduler can see what a task costs the storage device rather than only + * what it costs the CPU. + * + * The attributed time is charged against the task's virtual deadline one for + * one, a nanosecond of deadline for each nanosecond of device time, so that + * I/O and CPU cost a task the same. + * + * The same attribution carries CPU time as well as device time. A kworker + * running a work item declares itself to be acting for whoever queued that + * work, so the time it spends is charged back to them rather than lost to a + * kernel thread. That half is charged one for one: a nanosecond of CPU time + * costs the same wherever the kernel chose to spend it, and work a task does + * in its own context already costs it exactly that. + * + * See MuQSS-iotime-design.md. + */ +#ifndef _LINUX_MUQSS_IOTIME_H +#define _LINUX_MUQSS_IOTIME_H + +struct bio; +struct folio; +struct request; +struct gendisk; +struct task_struct; +struct work_struct; + +#ifdef CONFIG_MUQSS_IOTIME + +/* + * A window during which current is running on behalf of another task, opened + * by a kworker around one work item. Lives on the stack of the thread doing + * the work; see muqss_kerntime_begin(). + */ +struct muqss_kern_window { + struct task_struct *owner; /* who to charge, NULL for nobody */ + struct task_struct *prev; /* override to restore on close */ + u64 start; /* current's runtime when opened */ +}; + +/* + * Tag @bio with the task that caused it. Called from the block layer submit + * path, where current is the originating task for reads, readahead and + * synchronous writes alike. Takes a reference that is dropped by + * muqss_iotime_put_owner(). + */ +void muqss_iotime_set_owner(struct bio *bio); + +/* + * Drop the owner reference taken above. Idempotent, so it can be called from + * every site that releases bio state without tracking which ran first. + */ +void muqss_iotime_put_owner(struct bio *bio); + +/* + * Propagate ownership from @bio_src to @bio. Stacked drivers (md, dm, loop) + * resubmit clones from their own threads, where current is a kernel thread + * rather than the originating task, so without this attribution is lost on + * every RAID and LUKS setup. + */ +void muqss_iotime_clone_owner(struct bio *bio, struct bio *bio_src); + +/* Attach the accounting policy to a newly registered queue. */ +void muqss_iotime_enable(struct gendisk *disk); + +/* Zero a new task's counters so nothing is inherited across fork. */ +void muqss_iotime_task_init(struct task_struct *p); + +/* + * Owner table, used to attribute writeback. A dirtying task is given a slot, + * whose index is small enough to store in spare page->flags bits; writeback + * issued later by a flusher thread reads the index back off the folio and + * resolves it to the task that dirtied it. + */ + +/* Slot for current, allocating one on first use. 0 if none is available. */ +unsigned int muqss_iotime_owner_slot(void); + +/* Resolve a slot to its task, taking a reference. NULL if the slot is stale. */ +struct task_struct *muqss_iotime_owner_task(unsigned int slot); + +/* Give up a task's slot. Called from exit. */ +void muqss_iotime_release_slot(struct task_struct *p); + +/* Record current as the dirtier of @folio, if it can be attributed. */ +void muqss_iotime_dirty_folio(struct folio *folio); + +/* + * Declare that current is about to do I/O on behalf of @owner, and undo it. + * Used by io_uring's io-wq workers, which issue requests that another task + * queued. Returns the previous value, to be handed back to _proxy_end(). + * + * The caller must keep a reference to @owner across the window. + */ +struct task_struct *muqss_iotime_proxy_begin(struct task_struct *owner); +void muqss_iotime_proxy_end(struct task_struct *prev); + +/* + * Two requests are being merged into one. Disowns @rq if @next belongs to a + * different task, so the survivor is not charged for both. + */ +void muqss_iotime_merge_requests(struct request *rq, struct request *next); + +/* + * Record who queued @work, so the kworker that eventually runs it can be + * charged to them. Called from the queueing task's own context, which rules + * out the deferred re-entries into __queue_work() made by the delayed work + * timer and the rcu_work callback: those run in interrupt and softirq + * context, where current is whoever was unlucky enough to be interrupted. + */ +void muqss_work_set_owner(struct work_struct *work); + +/* + * Open and close a window in which current is running work on behalf of the + * task in @slot. The CPU time current consumes across the window is charged + * to that task, and any I/O the work issues is attributed to it too. + * + * Cheap when there is nobody to charge, which is the common case: an unowned + * slot costs one comparison and reads no clocks. + */ +void muqss_kerntime_begin(struct muqss_kern_window *w, unsigned int slot); +void muqss_kerntime_end(struct muqss_kern_window *w); + +/* + * The CPU time current has consumed so far, including the part not yet banked + * into ->sched_time. Implemented by the scheduler; see MuQSS.c. + */ +u64 muqss_task_runtime_live(void); + +#else /* !CONFIG_MUQSS_IOTIME */ + +struct muqss_kern_window { }; + +static inline void muqss_iotime_set_owner(struct bio *bio) { } +static inline void muqss_iotime_put_owner(struct bio *bio) { } +static inline void muqss_iotime_clone_owner(struct bio *bio, + struct bio *bio_src) { } +static inline void muqss_iotime_enable(struct gendisk *disk) { } +static inline void muqss_iotime_task_init(struct task_struct *p) { } +static inline unsigned int muqss_iotime_owner_slot(void) { return 0; } +static inline struct task_struct *muqss_iotime_owner_task(unsigned int slot) +{ + return NULL; +} +static inline void muqss_iotime_release_slot(struct task_struct *p) { } +static inline void muqss_iotime_dirty_folio(struct folio *folio) { } +static inline struct task_struct * +muqss_iotime_proxy_begin(struct task_struct *owner) +{ + return NULL; +} +static inline void muqss_iotime_proxy_end(struct task_struct *prev) { } +static inline void muqss_iotime_merge_requests(struct request *rq, + struct request *next) { } +static inline void muqss_work_set_owner(struct work_struct *work) { } +static inline void muqss_kerntime_begin(struct muqss_kern_window *w, + unsigned int slot) { } +static inline void muqss_kerntime_end(struct muqss_kern_window *w) { } + +#endif /* CONFIG_MUQSS_IOTIME */ + +#endif /* _LINUX_MUQSS_IOTIME_H */ diff --git a/include/linux/page-flags-layout.h b/include/linux/page-flags-layout.h index 760006b1c..4a4beb2f3 100644 --- a/include/linux/page-flags-layout.h +++ b/include/linux/page-flags-layout.h @@ -113,9 +113,39 @@ ZONES_WIDTH - LRU_GEN_WIDTH - SECTIONS_WIDTH - \ NODES_WIDTH - KASAN_TAG_WIDTH - LAST_CPUPID_WIDTH) +/* + * MuQSS I/O aware scheduling stores an index into its owner table here, so + * that a page dirtied now can be charged to the task that dirtied it when it + * is written back later by a flusher thread. + * + * These bits are affordable because MuQSS excludes NUMA balancing, which is + * the only user of LAST_CPUPID: with CONFIG_NUMA_BALANCING off, + * LAST_CPUPID_SHIFT is 0 and its share of page->flags is unclaimed. + * + * Taken last so nothing else has to shrink. __LRU_REFS_WIDTH rather than + * LRU_REFS_WIDTH is used below because the latter is a min() expression and + * so cannot be evaluated by the preprocessor; the two are equal whenever + * there is room for MGLRU's full allocation, which is the only case in which + * anything is left over for us anyway. + */ +#ifdef CONFIG_MUQSS_IOTIME +#define MUQSS_IOWNER_SHIFT 8 +#else +#define MUQSS_IOWNER_SHIFT 0 +#endif + +#if ZONES_WIDTH + LRU_GEN_WIDTH + SECTIONS_WIDTH + NODES_WIDTH + \ + KASAN_TAG_WIDTH + LAST_CPUPID_WIDTH + __LRU_REFS_WIDTH + \ + MUQSS_IOWNER_SHIFT <= BITS_PER_LONG - NR_PAGEFLAGS +#define MUQSS_IOWNER_WIDTH MUQSS_IOWNER_SHIFT +#else +#define MUQSS_IOWNER_WIDTH 0 +#endif + #define NR_NON_PAGEFLAG_BITS (SECTIONS_WIDTH + NODES_WIDTH + ZONES_WIDTH + \ LAST_CPUPID_SHIFT + KASAN_TAG_WIDTH + \ - LRU_GEN_WIDTH + LRU_REFS_WIDTH) + LRU_GEN_WIDTH + LRU_REFS_WIDTH + \ + MUQSS_IOWNER_WIDTH) #define NR_UNUSED_PAGEFLAG_BITS (BITS_PER_LONG - \ (NR_NON_PAGEFLAG_BITS + NR_PAGEFLAGS)) diff --git a/include/linux/sched.h b/include/linux/sched.h index 5738c54eb..08ed3b38b 100644 --- a/include/linux/sched.h +++ b/include/linux/sched.h @@ -44,6 +44,9 @@ #include #include #include +#ifdef CONFIG_SCHED_MUQSS +#include +#endif #include #include #include @@ -334,6 +337,12 @@ extern long schedule_timeout_interruptible(long timeout); extern long schedule_timeout_killable(long timeout); extern long schedule_timeout_uninterruptible(long timeout); extern long schedule_timeout_idle(long timeout); + +extern long schedule_msec_hrtimeout(long timeout); +extern long schedule_min_hrtimeout(void); +extern long schedule_msec_hrtimeout_interruptible(long timeout); +extern long schedule_msec_hrtimeout_uninterruptible(long timeout); + asmlinkage void schedule(void); extern void schedule_preempt_disabled(void); asmlinkage void preempt_schedule_irq(void); @@ -888,6 +897,79 @@ struct task_struct { int normal_prio; unsigned int rt_priority; +#ifdef CONFIG_SCHED_MUQSS + int time_slice; + u64 deadline; + skiplist_node node; /* Skip list node */ + u64 last_ran; + u64 sched_time; /* sched_clock time spent running */ +#ifdef CONFIG_SMT_NICE + int smt_bias; /* Policy/nice level bias across smt siblings */ +#endif +#ifdef CONFIG_HOTPLUG_CPU + bool zerobound; /* Bound to CPU0 for hotplug */ +#endif + unsigned long rt_timeout; + /* Unbanked cpu time */ + unsigned long utime_ns, stime_ns; +#ifdef CONFIG_MUQSS_IOTIME + /* + * Block device time consumed on this task's behalf. Updated from + * I/O completion, which may be any CPU, so these are atomic. Read + * via /proc//iotime. Not inherited across fork. Bytes are not + * counted here, task_io_accounting already has them. + * + * io_latency_ns is end to end per bio, so it includes queue wait and + * is what a waiting task experiences. io_occupancy_ns is device + * service time per request, which is what this task cost everybody + * else, and is the only one of the two fit to charge for. + * + * io_debt_ns is occupancy not yet charged to the deadline. The + * scheduler consumes and zeroes it; see consume_iotime_penalty(). + */ + atomic64_t io_latency_ns; + atomic64_t io_count; + atomic64_t io_occupancy_ns; + atomic64_t io_debt_ns; + + /* + * CPU time burnt in some other thread's context on this task's + * behalf, currently kworkers running work items it queued. Unlike the + * I/O counters above this is CPU time, which the task would have been + * charged against its own time_slice had the kernel done the work + * synchronously instead of handing it to a worker. + * + * kern_time_ns is cumulative and only for reporting. kern_debt_ns is + * the part not yet charged to the deadline; the scheduler consumes + * and zeroes it, see consume_kerntime_penalty(). + */ + atomic64_t kern_time_ns; + atomic64_t kern_debt_ns; + + /* + * Index of this task's slot in the I/O owner table, stamped into + * page->flags when it dirties a folio so writeback can be charged + * back to it. 0 means no slot; allocated lazily on first dirty and + * released on exit. + */ + unsigned int io_owner_slot; + + /* + * The task this one is currently working on behalf of, or NULL. + * io_uring's io-wq workers publish the task that queued the request + * here for the duration of the issue, so a submission punted to a + * worker is still charged to the task that asked for it. kworkers do + * the same around each work item, which additionally attributes any + * I/O the work item submits, and any pages it dirties, to the task + * that queued the work rather than losing them to a kernel thread. + * + * Only ever written by the task itself and only read while it is + * running, so it needs no locking. The publisher holds a reference + * to the pointee for the whole window. + */ + struct task_struct *io_owner_override; +#endif +#else /* CONFIG_SCHED_MUQSS */ struct sched_entity se; struct sched_rt_entity rt; struct sched_dl_entity dl; @@ -896,6 +978,7 @@ struct task_struct { struct sched_ext_entity scx; #endif const struct sched_class *sched_class; +#endif /* CONFIG_SCHED_MUQSS */ #ifdef CONFIG_SCHED_CORE struct rb_node core_node; @@ -1681,6 +1764,8 @@ struct task_struct { randomized_struct_fields_end } __attribute__ ((aligned (64))); +#include + #ifdef CONFIG_SCHED_PROXY_EXEC DECLARE_STATIC_KEY_TRUE(__sched_proxy_exec); static inline bool sched_proxy_exec(void) @@ -2297,7 +2382,12 @@ static inline void set_task_cpu(struct task_struct *p, unsigned int cpu) static inline bool task_is_runnable(struct task_struct *p) { +#ifdef CONFIG_SCHED_MUQSS + /* MuQSS has no delayed dequeue: queued means runnable. */ + return p->on_rq; +#else return p->on_rq && !p->se.sched_delayed; +#endif } extern bool sched_task_on_rq(struct task_struct *p); diff --git a/include/linux/sched/deadline.h b/include/linux/sched/deadline.h index 273538200..d76aff3bc 100644 --- a/include/linux/sched/deadline.h +++ b/include/linux/sched/deadline.h @@ -30,6 +30,24 @@ static inline bool dl_time_before(u64 a, u64 b) } struct root_domain; +#ifdef CONFIG_SCHED_MUQSS +/* + * MuQSS implements no deadline scheduling class, so there is never any DL + * bandwidth to account for or migrate between root domains. + */ +static inline void dl_add_task_root_domain(struct task_struct *p) { } +static inline void dl_clear_root_domain(struct root_domain *rd) { } +static inline void dl_clear_root_domain_cpu(int cpu) { } +static inline bool dl_task_needs_bw_move(struct task_struct *p, + const struct cpumask *new_mask) +{ + return false; +} +static inline bool dl_bw_visited(int cpu, u64 cookie) +{ + return false; +} +#else /* CONFIG_SCHED_MUQSS */ extern void dl_add_task_root_domain(struct task_struct *p); extern void dl_clear_root_domain(struct root_domain *rd); extern void dl_clear_root_domain_cpu(int cpu); @@ -43,19 +61,23 @@ extern void dl_clear_root_domain_cpu(int cpu); extern bool dl_task_needs_bw_move(struct task_struct *p, const struct cpumask *new_mask); -extern u64 dl_cookie; extern bool dl_bw_visited(int cpu, u64 cookie); +#endif /* CONFIG_SCHED_MUQSS */ + +extern u64 dl_cookie; static inline bool dl_server(struct sched_dl_entity *dl_se) { return dl_se->dl_server; } +#ifndef CONFIG_SCHED_MUQSS static inline struct task_struct *dl_task_of(struct sched_dl_entity *dl_se) { BUG_ON(dl_server(dl_se)); return container_of(dl_se, struct task_struct, dl); } +#endif /* * Regarding the deadline, a task with implicit deadline has a relative diff --git a/include/linux/sched/nohz.h b/include/linux/sched/nohz.h index 0db7f6793..3fe4c8f7c 100644 --- a/include/linux/sched/nohz.h +++ b/include/linux/sched/nohz.h @@ -13,7 +13,7 @@ extern int get_nohz_timer_target(void); static inline void nohz_balance_enter_idle(int cpu) { } #endif -#ifdef CONFIG_NO_HZ_COMMON +#if defined(CONFIG_NO_HZ_COMMON) && !defined(CONFIG_SCHED_MUQSS) void calc_load_nohz_start(void); void calc_load_nohz_remote(struct rq *rq); void calc_load_nohz_stop(void); diff --git a/include/linux/sched/prio.h b/include/linux/sched/prio.h index 6ab43b4f7..122ea08e5 100644 --- a/include/linux/sched/prio.h +++ b/include/linux/sched/prio.h @@ -16,6 +16,21 @@ #define MAX_RT_PRIO 100 #define MAX_DL_PRIO 0 +#ifdef CONFIG_SCHED_MUQSS +/* + * MuQSS ranks its non-RT policies immediately below the RT range, and has no + * deadline class. SCHED_IDLEPRIO is an alias for the SCHED_IDLE policy value. + */ +#define ISO_PRIO (MAX_RT_PRIO) +#define NORMAL_PRIO (MAX_RT_PRIO + 1) +#define IDLE_PRIO (MAX_RT_PRIO + 2) +#define PRIO_LIMIT ((IDLE_PRIO) + 1) + +#define SCHED_IDLEPRIO SCHED_IDLE +#define SCHED_MAX (SCHED_IDLEPRIO) +#define SCHED_RANGE(policy) ((policy) <= SCHED_MAX) +#endif /* CONFIG_SCHED_MUQSS */ + #define MAX_PRIO (MAX_RT_PRIO + NICE_WIDTH) #define DEFAULT_PRIO (MAX_RT_PRIO + NICE_WIDTH / 2) diff --git a/include/linux/skip_list.h b/include/linux/skip_list.h new file mode 100644 index 000000000..4547346d8 --- /dev/null +++ b/include/linux/skip_list.h @@ -0,0 +1,41 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _LINUX_SKIP_LISTS_H +#define _LINUX_SKIP_LISTS_H + +/* + * Four levels is enough: randomLevel() returns 0..3, and insert only + * grows the list one level at a time, so next/prev[4..] were unused. + */ +#define SKIPLIST_MAXLEVEL 4 + +typedef u64 keyType; + +typedef struct nodeStructure skiplist_node; + +struct nodeStructure { + keyType key; + skiplist_node *next[SKIPLIST_MAXLEVEL]; + skiplist_node *prev[SKIPLIST_MAXLEVEL]; +}; + +typedef struct listStructure { + int entries; + int level; /* Maximum level of the list + (1 more than the number of levels in the list) */ + u64 best_key; /* Earliest queued key; lockless hint for EDT */ + skiplist_node *header; /* pointer to header */ +} skiplist; + +void skiplist_init(skiplist_node *slnode); +void skiplist_cache_init(void); +skiplist *new_skiplist(skiplist_node *slnode); +void skiplist_free(skiplist *l); +void free_skiplist(skiplist *l); +void skiplist_node_init(skiplist_node *node); +void skiplist_insert(skiplist *l, skiplist_node *node, keyType key, unsigned int randseed); +bool skiplist_delete(skiplist *l, skiplist_node *node); + +static inline bool skiplist_node_empty(skiplist_node *node) { + return (!node->next[0]); +} +#endif /* _LINUX_SKIP_LISTS_H */ diff --git a/include/linux/workqueue.h b/include/linux/workqueue.h index a283766a1..b9cb42ec2 100644 --- a/include/linux/workqueue.h +++ b/include/linux/workqueue.h @@ -288,6 +288,7 @@ static inline unsigned int work_static(struct work_struct *work) { return 0; } lockdep_init_map(&(_work)->lockdep_map, "(work_completion)"#_work, (_key), 0); \ INIT_LIST_HEAD(&(_work)->entry); \ (_work)->func = (_func); \ + muqss_work_init_owner(_work); \ } while (0) #else #define __INIT_WORK_KEY(_work, _func, _onstack, _key) \ @@ -296,6 +297,7 @@ static inline unsigned int work_static(struct work_struct *work) { return 0; } (_work)->data = (atomic_long_t) WORK_DATA_INIT(); \ INIT_LIST_HEAD(&(_work)->entry); \ (_work)->func = (_func); \ + muqss_work_init_owner(_work); \ } while (0) #endif diff --git a/include/linux/workqueue_types.h b/include/linux/workqueue_types.h index 4c38824f3..ce1927123 100644 --- a/include/linux/workqueue_types.h +++ b/include/linux/workqueue_types.h @@ -17,9 +17,51 @@ struct work_struct { atomic_long_t data; struct list_head entry; work_func_t func; +#ifdef CONFIG_MUQSS_IOTIME + /* + * Index in the I/O owner table of the task that queued this work, so + * the CPU time the kworker spends running it can be charged back to + * whoever asked for it. 0 means nobody to charge, which is the case + * for the great deal of work queued by kernel threads on their own + * account. + * + * A slot index rather than a task pointer deliberately: work items + * are cancelled and disabled through half a dozen paths that clear + * the pending bit without ever running the work, and none of them + * would drop a reference taken here. A stale slot simply resolves to + * NULL and the work is charged to nobody. + */ + unsigned int muqss_owner_slot; +#endif #ifdef CONFIG_LOCKDEP struct lockdep_map lockdep_map; #endif }; +#ifdef CONFIG_MUQSS_IOTIME +static inline unsigned int muqss_work_owner(struct work_struct *work) +{ + return work->muqss_owner_slot; +} + +/* + * Every queueing path stamps the owner, so this only matters for a work item + * initialised on memory that was never zeroed and then reaching execution by + * some path that does not. It is here so that such a path, if one exists, + * charges nobody rather than charging whichever unlucky task holds the slot + * the garbage happens to name. + */ +static inline void muqss_work_init_owner(struct work_struct *work) +{ + work->muqss_owner_slot = 0; +} +#else +static inline unsigned int muqss_work_owner(struct work_struct *work) +{ + return 0; +} + +static inline void muqss_work_init_owner(struct work_struct *work) { } +#endif + #endif /* _LINUX_WORKQUEUE_TYPES_H */ diff --git a/include/uapi/linux/sched.h b/include/uapi/linux/sched.h index 33a462428..facf41cd7 100644 --- a/include/uapi/linux/sched.h +++ b/include/uapi/linux/sched.h @@ -125,7 +125,7 @@ struct clone_args { #define SCHED_FIFO 1 #define SCHED_RR 2 #define SCHED_BATCH 3 -/* SCHED_ISO: reserved but not implemented yet */ +#define SCHED_ISO 4 /* Implemented on MuQSS only */ #define SCHED_IDLE 5 #define SCHED_DEADLINE 6 #define SCHED_EXT 7 diff --git a/init/Kconfig b/init/Kconfig index 8937400a3..623d8559c 100644 --- a/init/Kconfig +++ b/init/Kconfig @@ -222,6 +222,8 @@ config CACHY bool "Kernel tweaks by CachyOS" default y +source "kernel/Kconfig.MuQSS" + config BROKEN bool help @@ -891,6 +893,7 @@ menu "Scheduler features" config UCLAMP_TASK bool "Enable utilization clamping for RT/FAIR tasks" depends on CPU_FREQ_GOV_SCHEDUTIL + depends on !SCHED_MUQSS help This feature enables the scheduler to track the clamped utilization of each CPU based on RUNNABLE tasks scheduled on that CPU. @@ -939,6 +942,8 @@ config UCLAMP_BUCKETS_COUNT config SCHED_PROXY_EXEC bool "Proxy Execution" + # MuQSS has no sched_class donor/proxy plumbing + depends on !SCHED_MUQSS # Avoid some build failures w/ PREEMPT_RT until it can be fixed depends on !PREEMPT_RT # Need to investigate how to inform sched_ext of split contexts @@ -1016,6 +1021,7 @@ config ARCH_WANT_NUMA_VARIABLE_LOCALITY config NUMA_BALANCING bool "Memory placement aware NUMA scheduler" + depends on !SCHED_MUQSS depends on ARCH_SUPPORTS_NUMA_BALANCING depends on !ARCH_WANT_NUMA_VARIABLE_LOCALITY depends on SMP && NUMA_MIGRATION && !PREEMPT_RT @@ -1030,6 +1036,8 @@ config SCHED_CACHE bool "Cache aware load balance" default y depends on SMP + # Cache aware load balancing lives in CFS, which MuQSS replaces + depends on !SCHED_MUQSS help When enabled, the scheduler will attempt to aggregate tasks from the same process onto a single Last Level Cache (LLC) domain when @@ -1152,7 +1160,7 @@ menuconfig CGROUP_SCHED bandwidth allocation to such task groups. It uses cgroups to group tasks. -if CGROUP_SCHED +if CGROUP_SCHED && !SCHED_MUQSS config GROUP_SCHED_WEIGHT def_bool n @@ -1214,7 +1222,7 @@ config EXT_SUB_SCHED config SCHED_MM_CID def_bool y - depends on SMP && RSEQ + depends on SMP && RSEQ && !SCHED_MUQSS config UCLAMP_TASK_GROUP bool "Utilization clamping per group of tasks" @@ -1341,6 +1349,7 @@ config CGROUP_DEVICE config CGROUP_CPUACCT bool "Simple CPU accounting controller" + depends on !SCHED_MUQSS help Provides a simple controller for monitoring the total CPU consumed by the tasks in a cgroup. @@ -1486,6 +1495,7 @@ config CHECKPOINT_RESTORE config SCHED_AUTOGROUP bool "Automatic process group scheduling" + depends on !SCHED_MUQSS select CGROUPS select CGROUP_SCHED select FAIR_GROUP_SCHED @@ -2007,6 +2017,7 @@ config RSEQ config RSEQ_SLICE_EXTENSION bool "Enable rseq-based time slice extension mechanism" depends on RSEQ && HIGH_RES_TIMERS && GENERIC_ENTRY && HAVE_GENERIC_TIF_BITS + depends on !SCHED_MUQSS help Allows userspace to request a limited time slice extension when returning from an interrupt to user space via the RSEQ shared diff --git a/init/init_task.c b/init/init_task.c index b67ef6040..5d2e26a0e 100644 --- a/init/init_task.c +++ b/init/init_task.c @@ -111,9 +111,17 @@ struct task_struct init_task __aligned(L1_CACHE_BYTES) = { .stack = init_stack, .usage = REFCOUNT_INIT(2), .flags = PF_KTHREAD, +#ifdef CONFIG_SCHED_MUQSS + .prio = NORMAL_PRIO, + .static_prio = MAX_PRIO - 20, + .normal_prio = NORMAL_PRIO, + .deadline = 0, + .time_slice = 1000000, +#else .prio = MAX_PRIO - 20, .static_prio = MAX_PRIO - 20, .normal_prio = MAX_PRIO - 20, +#endif .policy = SCHED_NORMAL, .cpus_ptr = &init_task.cpus_mask, .user_cpus_ptr = NULL, @@ -126,6 +134,7 @@ struct task_struct init_task __aligned(L1_CACHE_BYTES) = { .restart_block = { .fn = do_no_restart_syscall, }, +#ifndef CONFIG_SCHED_MUQSS .se = { .group_node = LIST_HEAD_INIT(init_task.se.group_node), }, @@ -133,6 +142,7 @@ struct task_struct init_task __aligned(L1_CACHE_BYTES) = { .run_list = LIST_HEAD_INIT(init_task.rt.run_list), .time_slice = RR_TIMESLICE, }, +#endif .tasks = LIST_HEAD_INIT(init_task.tasks), #ifdef CONFIG_SMP .pushable_tasks = PLIST_NODE_INIT(init_task.pushable_tasks, MAX_PRIO), diff --git a/io_uring/io_uring.c b/io_uring/io_uring.c index 1ea2fca34..e3ef5db32 100644 --- a/io_uring/io_uring.c +++ b/io_uring/io_uring.c @@ -56,6 +56,7 @@ #include #include #include +#include #include #include #include @@ -1531,7 +1532,19 @@ void io_wq_submit_work(struct io_wq_work *work) } do { + struct task_struct *prev; + + /* + * Anything this issue submits to the block layer is being done + * on behalf of the task that queued the request, not this + * worker. Publish it so the I/O is charged to the right task; + * req->tctx pins it for the duration. + */ + prev = muqss_iotime_proxy_begin(req->tctx ? req->tctx->task : + NULL); ret = io_issue_sqe(req, issue_flags); + muqss_iotime_proxy_end(prev); + if (ret != -EAGAIN) break; diff --git a/kernel/Kconfig.MuQSS b/kernel/Kconfig.MuQSS new file mode 100644 index 000000000..3dad0e814 --- /dev/null +++ b/kernel/Kconfig.MuQSS @@ -0,0 +1,219 @@ +# SPDX-License-Identifier: GPL-2.0-only +# +# MuQSS - The Multiple Queue Skiplist Scheduler by Con Kolivas. +# +# Everything MuQSS needs from Kconfig lives in this file, including the +# negative dependencies that disable mainline scheduler features MuQSS +# replaces or does not implement. Kconfig merges properties from repeated +# "config" blocks, so those features are gated here rather than by editing +# each mainline Kconfig in place. The only change required outside this file +# is a single source line in init/Kconfig. + +config SCHED_MUQSS + bool "MuQSS cpu scheduler" + # Architectures the port is built and tested on. Widening this needs + # arch review: gains a NORMAL_PRIO definition, + # which collides with the private one in + # arch/powerpc/platforms/cell/spufs/sched.c. + depends on X86 || ARM64 + select HIGH_RES_TIMERS + default y + help + The Multiple Queue Skiplist Scheduler for excellent interactivity and + responsiveness on the desktop and highly scalable deterministic + low latency on any hardware. + + Say Y here. + +config SMT_NICE + bool "SMT (Hyperthreading) aware nice priority and policy support" + depends on SCHED_MUQSS && SCHED_SMT + default y + help + Enabling Hyperthreading on Intel CPUs decreases the effectiveness + of the use of 'nice' levels and different scheduling policies + (e.g. realtime) due to sharing of CPU power between hyperthreads. + SMT nice support makes each logical CPU aware of what is running on + its hyperthread siblings, maintaining appropriate distribution of + CPU according to nice levels and scheduling policies at the expense + of slightly increased overhead. + + If unsure say Y here. + +choice + depends on SMP + prompt "CPU scheduler runqueue sharing" + default RQ_MC if SCHED_MUQSS + default RQ_NONE + +config RQ_NONE + bool "No sharing" + help + This is the default behaviour where the CPU scheduler has one runqueue + per CPU, whether it is a physical or logical CPU (hyperthread). + + This can still be enabled runtime with the boot parameter + rqshare=none + + If unsure, say N. + +config RQ_SMT + bool "SMT (hyperthread) siblings" + depends on SCHED_SMT && SCHED_MUQSS + + help + With this option enabled, the CPU scheduler will have one runqueue + shared by SMT (hyperthread) siblings. As these logical cores share + one physical core, sharing the runqueue resource can lead to decreased + overhead, lower latency and higher throughput. + + This can still be enabled runtime with the boot parameter + rqshare=smt + + If unsure, say N. + +config RQ_MC + bool "Multicore siblings" + depends on SCHED_MC && SCHED_MUQSS + help + With this option enabled, the CPU scheduler will have one runqueue + shared by multicore siblings in addition to any SMT siblings. + As these physical cores share caches, sharing the runqueue resource + will lead to lower latency, but its effects on overhead and throughput + are less predictable. As a general rule, 6 or fewer cores will likely + benefit from this, while larger CPUs will only derive a latency + benefit. If your workloads are primarily single threaded, this will + possibly worsen throughput. If you are only concerned about latency + then enable this regardless of how many cores you have. + + This can still be enabled runtime with the boot parameter + rqshare=mc + + If unsure, say Y. + +config RQ_MC_LLC + bool "Multicore siblings (LLC)" + depends on SCHED_MC && SCHED_MUQSS + help + With this option enabled, the CPU scheduler will behave similarly as + with "Multicore siblings". + This option takes LLC cache into account when scheduling tasks. + Option may benefit CPUs with multiple LLC caches, such as Ryzen + and Xeon CPUs. + + This can still be enabled runtime with the boot parameter + rqshare=llc + + If unsure, say N. + +config RQ_SMP + bool "Symmetric Multi-Processing" + depends on SMP && SCHED_MUQSS + help + With this option enabled, the CPU scheduler will have one runqueue + shared by all physical CPUs unless they are on separate NUMA nodes. + As physical CPUs usually do not share resources, sharing the runqueue + will normally worsen throughput but improve latency. If you only + care about latency enable this. + + This can still be enabled runtime with the boot parameter + rqshare=smp + + If unsure, say N. + +config RQ_ALL + bool "NUMA" + depends on SMP && SCHED_MUQSS + help + With this option enabled, the CPU scheduler will have one runqueue + regardless of the architecture configuration, including across NUMA + nodes. This can substantially decrease throughput in NUMA + configurations, but light NUMA designs will not be dramatically + affected. This option should only be chosen if latency is the prime + concern. + + This can still be enabled runtime with the boot parameter + rqshare=all + + If unsure, say N. +endchoice + +config SHARERQ + int + default 0 if RQ_NONE + default 1 if RQ_SMT + default 2 if RQ_MC + default 3 if RQ_MC_LLC + default 4 if RQ_SMP + default 5 if RQ_ALL + +config MUQSS_IOTIME + bool "I/O aware CPU scheduling support" + depends on SCHED_MUQSS && BLOCK + default y + help + Ordinarily the CPU scheduler can see how much CPU a task uses, but it + is blind to how much block device time that task consumes. A task + that keeps a disk busy on everyone else's behalf looks identical to + one that is merely idle, so it keeps full CPU priority while other + tasks wait behind its I/O. + + This option attaches the scheduler to the block layer so that the + time a device spends servicing each request is attributed back to the + task that asked for it, and accumulates that as a per task I/O time + figure. It is exported per task through /proc//iotime and per + device through debugfs. + + That figure is charged against the task's virtual deadline, demoting a + task in proportion to the device time it consumed. This stops a task + that keeps a disk busy on everybody else's behalf from retaining full + CPU priority for free, while leaving tasks that merely wait on I/O as + responsive as they are now. + + The charge is one nanosecond of deadline per nanosecond of device + time, so I/O and CPU cost a task equally. That rate is what the + feature means rather than a setting within it, so there is nothing to + tune it by. + + The same machinery charges a task for kernel work done in another + thread's context on its behalf, which today means kworkers running + work items it queued. Kernel work a task asks for in its own context + needs no such help, since it already comes out of that task's own + timeslice; what escapes the scheduler is the work handed to somebody + else, and forcing IRQ threading moves more work into that blind spot. + That is charged one for one, since a nanosecond of CPU time costs the + same wherever the kernel chose to spend it. It is reported alongside + the I/O figures. + + Doing this costs a timestamp and an owner reference on every block + request, and an accounting update on every completion. On very fast + devices, or under heavy small random I/O, that overhead is measurable + and may reduce peak I/O throughput. Say N if you are tuning purely + for storage benchmarks. + + When disabled this is compiled out entirely and costs nothing. + + If unsure, say Y. + +# +# Mainline features MuQSS replaces or does not implement are gated with +# "depends on !SCHED_MUQSS" at their point of definition. Kconfig scopes a +# "depends on" to the properties declared in the same config block, so a +# dependency cannot be attached to a symbol from here; each gate has to live +# beside the definition it constrains. They are: +# +# init/Kconfig UCLAMP_TASK +# SCHED_PROXY_EXEC +# NUMA_BALANCING +# SCHED_MM_CID +# CGROUP_CPUACCT +# SCHED_AUTOGROUP +# RSEQ_SLICE_EXTENSION +# if CGROUP_SCHED (group scheduling +# block: FAIR_GROUP_SCHED, +# CFS_BANDWIDTH, RT_GROUP_SCHED, +# GROUP_SCHED_*, EXT_GROUP_SCHED) +# kernel/Kconfig.preempt PREEMPT_RT +# SCHED_CORE +# SCHED_CLASS_EXT +# kernel/trace/rv/monitors/deadline/Kconfig RV_MON_DEADLINE diff --git a/kernel/Kconfig.hz b/kernel/Kconfig.hz index e1359db55..ed3539ef3 100644 --- a/kernel/Kconfig.hz +++ b/kernel/Kconfig.hz @@ -3,9 +3,38 @@ # Timer Interrupt Frequency Configuration # +# +# Silent symbols keep CONFIG_HZ_* available to the rest of the tree and to +# inherited .configs, but they are not choice members. The promptable +# *_NODEF names are what olddefconfig can stick on; renaming every rate +# (including 100) means a saved "# CONFIG_HZ_100 is not set" cannot pin the +# choice away from the MuQSS default. +# +config HZ_100 + bool + +config HZ_250 + bool + +config HZ_300 + bool + +config HZ_500 + bool + +config HZ_600 + bool + +config HZ_750 + bool + +config HZ_1000 + bool + choice prompt "Timer frequency" - default HZ_250 + default HZ_100_NODEF if SCHED_MUQSS + default HZ_250_NODEF if !SCHED_MUQSS help Allows the configuration of the timer frequency. It is customary to have the timer interrupt run at 1000 Hz but 100 Hz may be more @@ -17,52 +46,69 @@ choice per second. - config HZ_100 + config HZ_100_NODEF bool "100 HZ" + select HZ_100 help + 100 Hz is a suitable choice in combination with MuQSS which does + not rely on ticks for rescheduling interrupts, and is not Hz limited + for timeouts and sleeps from both the kernel and userspace. + This allows us to benefit from the lower overhead and higher + throughput of fewer timer ticks. + + Non-MuQSS kernels: 100 Hz is a typical choice for servers, SMP and NUMA systems with lots of processors that may show reduced performance if too many timer interrupts are occurring. - config HZ_250 + config HZ_250_NODEF bool "250 HZ" + select HZ_250 help 250 Hz is a good compromise choice allowing server performance while also showing good interactive responsiveness even on SMP and NUMA systems. If you are going to be using NTSC video or multimedia, select 300Hz instead. - config HZ_300 + 250 Hz is the default choice for the mainline scheduler but not + advantageous in combination with MuQSS. + + config HZ_300_NODEF bool "300 HZ" + select HZ_300 help 300 Hz is a good compromise choice allowing server performance while also showing good interactive responsiveness even on SMP and NUMA systems and exactly dividing by both PAL and NTSC frame rates for video and multimedia work. - config HZ_500 + config HZ_500_NODEF bool "500 HZ" + select HZ_500 help 500 Hz is a balanced timer frequency. Provides fast interactivity on desktops with good smoothness without increasing CPU power consumption and sacrificing the battery life on laptops. - config HZ_600 + config HZ_600_NODEF bool "600 HZ" + select HZ_600 help 600 Hz is a balanced timer frequency. Provides fast interactivity on desktops with good smoothness without increasing CPU power consumption and sacrificing the battery life on laptops. - config HZ_750 + config HZ_750_NODEF bool "750 HZ" + select HZ_750 help 750 Hz is a balanced timer frequency. Provides fast interactivity on desktops with good smoothness without increasing CPU power consumption and sacrificing the battery life on laptops. - config HZ_1000 + config HZ_1000_NODEF bool "1000 HZ" + select HZ_1000 help 1000 Hz is the preferred choice for desktop systems and other systems requiring fast interactive responses to events. diff --git a/kernel/Kconfig.preempt b/kernel/Kconfig.preempt index 2471a502c..e839c3b36 100644 --- a/kernel/Kconfig.preempt +++ b/kernel/Kconfig.preempt @@ -14,15 +14,35 @@ config PREEMPT_BUILD config ARCH_HAS_PREEMPT_LAZY bool +# +# Silent symbols keep CONFIG_PREEMPT{,_NONE,_VOLUNTARY,_LAZY} available to +# the rest of the tree and to inherited .configs, but they are not choice +# members. The promptable *_NODEF names are what olddefconfig can stick on; +# renaming every model (including PREEMPT) means a saved +# "# CONFIG_PREEMPT is not set" or CONFIG_PREEMPT_{NONE,VOLUNTARY,LAZY}=y +# cannot pin the choice away from full preempt. +# +config PREEMPT_NONE + bool + +config PREEMPT_VOLUNTARY + bool + +config PREEMPT + bool + +config PREEMPT_LAZY + bool + choice prompt "Preemption Model" - default PREEMPT_LAZY if ARCH_HAS_PREEMPT_LAZY - default PREEMPT_NONE + default PREEMPT_NODEF -config PREEMPT_NONE +config PREEMPT_NONE_NODEF bool "No Forced Preemption (Server)" depends on !PREEMPT_RT depends on ARCH_NO_PREEMPT + select PREEMPT_NONE select PREEMPT_NONE_BUILD if !PREEMPT_DYNAMIC help This is the traditional Linux preemption model, geared towards @@ -35,11 +55,12 @@ config PREEMPT_NONE raw processing power of the kernel, irrespective of scheduling latencies. -config PREEMPT_VOLUNTARY - bool "Voluntary Kernel Preemption (Desktop)" +config PREEMPT_VOLUNTARY_NODEF + bool "Voluntary Kernel Preemption (Nothing)" depends on !ARCH_HAS_PREEMPT_LAZY depends on !ARCH_NO_PREEMPT depends on !PREEMPT_RT + select PREEMPT_VOLUNTARY select PREEMPT_VOLUNTARY_BUILD if !PREEMPT_DYNAMIC help This option reduces the latency of the kernel by adding more @@ -54,11 +75,13 @@ config PREEMPT_VOLUNTARY applications to run more 'smoothly' even when the system is under load. - Select this if you are building a kernel for a desktop system. + Select this for no system in particular (choose Preemptible + instead on a desktop if you know what's good for you). -config PREEMPT +config PREEMPT_NODEF bool "Preemptible Kernel (Low-Latency Desktop)" depends on !ARCH_NO_PREEMPT + select PREEMPT select PREEMPT_BUILD if !PREEMPT_DYNAMIC help This option reduces the latency of the kernel by making @@ -75,10 +98,11 @@ config PREEMPT embedded system with latency requirements in the milliseconds range. -config PREEMPT_LAZY +config PREEMPT_LAZY_NODEF bool "Scheduler controlled preemption model" depends on !ARCH_NO_PREEMPT depends on ARCH_HAS_PREEMPT_LAZY + select PREEMPT_LAZY select PREEMPT_BUILD if !PREEMPT_DYNAMIC help This option provides a scheduler driven preemption model that @@ -87,11 +111,17 @@ config PREEMPT_LAZY reduce lock holder preemption and recover some of the performance gains seen from using Voluntary preemption. + It trades the latency that makes a desktop feel responsive for + throughput, so choose Preemptible instead unless you are building + for a throughput oriented system. + endchoice config PREEMPT_RT bool "Fully Preemptible Kernel (Real-Time)" depends on ARCH_SUPPORTS_RT && !COMPILE_TEST + # MuQSS does not implement the PREEMPT_RT scheduler/locking contract + depends on !SCHED_MUQSS select PREEMPTION help This option turns the kernel into a real-time kernel by replacing @@ -150,7 +180,7 @@ config PREEMPT_DYNAMIC config SCHED_CORE bool "Core Scheduling for SMT" - depends on SCHED_SMT + depends on SCHED_SMT && !SCHED_MUQSS help This option permits Core Scheduling, a means of coordinated task selection across SMT siblings. When enabled -- see @@ -168,7 +198,7 @@ config SCHED_CORE config SCHED_CLASS_EXT bool "Extensible Scheduling Class" - depends on BPF_SYSCALL && BPF_JIT && DEBUG_INFO_BTF + depends on BPF_SYSCALL && BPF_JIT && DEBUG_INFO_BTF && !SCHED_MUQSS select STACKTRACE if STACKTRACE_SUPPORT help This option enables a new scheduler class sched_ext (SCX), which diff --git a/kernel/cgroup/cpuset.c b/kernel/cgroup/cpuset.c index 45944b3e3..c1c97d77a 100644 --- a/kernel/cgroup/cpuset.c +++ b/kernel/cgroup/cpuset.c @@ -3055,7 +3055,7 @@ static int cpuset_can_attach(struct cgroup_taskset *tset) */ cs->nr_migrate_dl_tasks++; if (dl_task_needs_bw_move(task, cs->effective_cpus)) - cs->sum_migrate_dl_bw += task->dl.dl_bw; + cs->sum_migrate_dl_bw += tsk_dl_bw(task); } } diff --git a/kernel/delayacct.c b/kernel/delayacct.c index 2e55c493c..5c2262a24 100644 --- a/kernel/delayacct.c +++ b/kernel/delayacct.c @@ -170,7 +170,7 @@ int delayacct_add_tsk(struct taskstats *d, struct task_struct *tsk) */ t1 = tsk->sched_info.pcount; t2 = tsk->sched_info.run_delay; - t3 = tsk->se.sum_exec_runtime; + t3 = tsk_seruntime(tsk); d->cpu_count += t1; diff --git a/kernel/exit.c b/kernel/exit.c index 2c0b1c029..e7bd45df3 100644 --- a/kernel/exit.c +++ b/kernel/exit.c @@ -34,6 +34,7 @@ #include #include #include +#include #include #include #include @@ -207,7 +208,7 @@ static void __exit_signal(struct release_task_post *post, struct task_struct *ts sig->inblock += task_io_get_inblock(tsk); sig->oublock += task_io_get_oublock(tsk); task_io_accounting_add(&sig->ioac, &tsk->ioac); - sig->sum_sched_runtime += tsk->se.sum_exec_runtime; + sig->sum_sched_runtime += tsk_seruntime(tsk); sig->nr_threads--; __unhash_process(post, tsk, group_dead); write_sequnlock(&sig->stats_lock); @@ -295,8 +296,8 @@ void release_task(struct task_struct *p) /* @thread_pid can't go away until free_pids() below */ proc_flush_pid(thread_pid); exit_cred_namespaces(p); - add_device_randomness(&p->se.sum_exec_runtime, - sizeof(p->se.sum_exec_runtime)); + add_device_randomness(&tsk_seruntime(p), + sizeof(tsk_seruntime(p))); free_pids(post.pids); release_thread(p); /* @@ -1033,6 +1034,9 @@ void __noreturn do_exit(long code) if (tsk->io_context) exit_io_context(tsk); + /* Give up the I/O owner table slot used to attribute writeback. */ + muqss_iotime_release_slot(tsk); + if (tsk->splice_pipe) free_pipe_info(tsk->splice_pipe); diff --git a/kernel/locking/rtmutex.c b/kernel/locking/rtmutex.c index 4728631ae..0200351ed 100644 --- a/kernel/locking/rtmutex.c +++ b/kernel/locking/rtmutex.c @@ -372,7 +372,7 @@ waiter_update_prio(struct rt_mutex_waiter *waiter, struct task_struct *task) lockdep_assert(RB_EMPTY_NODE(&waiter->tree.entry)); waiter->tree.prio = __waiter_prio(task); - waiter->tree.deadline = task->dl.deadline; + waiter->tree.deadline = tsk_dl_deadline(task); } /* @@ -393,7 +393,7 @@ waiter_clone_prio(struct rt_mutex_waiter *waiter, struct task_struct *task) * Only use with rt_waiter_node_{less,equal}() */ #define task_to_waiter_node(p) \ - &(struct rt_waiter_node){ .prio = __waiter_prio(p), .deadline = (p)->dl.deadline } + &(struct rt_waiter_node){ .prio = __waiter_prio(p), .deadline = tsk_dl_deadline(p) } #define task_to_waiter(p) \ &(struct rt_mutex_waiter){ .tree = *task_to_waiter_node(p) } diff --git a/kernel/sched/Makefile b/kernel/sched/Makefile index b1f1a3670..96b1c60cb 100644 --- a/kernel/sched/Makefile +++ b/kernel/sched/Makefile @@ -23,12 +23,14 @@ ifneq ($(CONFIG_SCHED_OMIT_FRAME_POINTER),y) # I turn this off for IA-64 only. Andreas Schwab says it's also needed on m68k # to get a correct value for the wait-channel (WCHAN in ps). --davidm CFLAGS_core.o := $(PROFILING) -fno-omit-frame-pointer +CFLAGS_MuQSS.o := $(PROFILING) -fno-omit-frame-pointer endif # Branch profiling isn't noinstr-safe ifdef CONFIG_TRACE_BRANCH_PROFILING CFLAGS_build_policy.o += -DDISABLE_BRANCH_PROFILING CFLAGS_build_utility.o += -DDISABLE_BRANCH_PROFILING +CFLAGS_build_muqss.o += -DDISABLE_BRANCH_PROFILING endif # # Build efficiency: @@ -36,7 +38,12 @@ endif # These compilation units have roughly the same size and complexity - so their # build parallelizes well and finishes roughly at once: # +ifdef CONFIG_SCHED_MUQSS +obj-y += MuQSS.o +obj-y += build_muqss.o +else obj-y += core.o obj-y += fair.o obj-y += build_policy.o obj-y += build_utility.o +endif diff --git a/kernel/sched/MuQSS.c b/kernel/sched/MuQSS.c new file mode 100644 index 000000000..a5712a76e --- /dev/null +++ b/kernel/sched/MuQSS.c @@ -0,0 +1,10778 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * kernel/sched/MuQSS.c, was kernel/sched.c + * + * Kernel scheduler and related syscalls + * + * Copyright (C) 1991-2002 Linus Torvalds + * + * 1996-12-23 Modified by Dave Grothe to fix bugs in semaphores and + * make semaphores SMP safe + * 1998-11-19 Implemented schedule_timeout() and related stuff + * by Andrea Arcangeli + * 2002-01-04 New ultra-scalable O(1) scheduler by Ingo Molnar: + * hybrid priority-list and round-robin design with + * an array-switch method of distributing timeslices + * and per-CPU runqueues. Cleanups and useful suggestions + * by Davide Libenzi, preemptible kernel bits by Robert Love. + * 2003-09-03 Interactivity tuning by Con Kolivas. + * 2004-04-02 Scheduler domains code by Nick Piggin + * 2007-04-15 Work begun on replacing all interactivity tuning with a + * fair scheduling design by Con Kolivas. + * 2007-05-05 Load balancing (smp-nice) and other improvements + * by Peter Williams + * 2007-05-06 Interactivity improvements to CFS by Mike Galbraith + * 2007-07-01 Group scheduling enhancements by Srivatsa Vaddagiri + * 2007-11-29 RT balancing improvements by Steven Rostedt, Gregory Haskins, + * Thomas Gleixner, Mike Kravetz + * 2009-08-13 Brainfuck deadline scheduling policy by Con Kolivas deletes + * a whole lot of those previous things. + * 2016-10-01 Multiple Queue Skiplist Scheduler scalable evolution of BFS + * scheduler by Con Kolivas. + * 2019-08-31 LLC bits by Eduards Bezverhijs + */ +/* Out-of-line migrate_{disable,enable} for modules; see linux/sched.h. */ +#define INSTANTIATE_EXPORTED_MIGRATE_DISABLE +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include + +#include "../workqueue_internal.h" +#include "../../io_uring/io-wq.h" +#include "../smpboot.h" + +#define CREATE_TRACE_POINTS +#include +#include +#include +#undef CREATE_TRACE_POINTS + +#include "MuQSS.h" +#include "smp.h" + +/* Modules probe these; core.c exports the same set under CFS. */ +EXPORT_TRACEPOINT_SYMBOL_GPL(ipi_send_cpu); +EXPORT_TRACEPOINT_SYMBOL_GPL(ipi_send_cpumask); +EXPORT_TRACEPOINT_SYMBOL(sched_set_state_tp); + +#define rt_prio(prio) unlikely((prio) < MAX_RT_PRIO) +#define rt_task(p) rt_prio((p)->prio) +#define batch_task(p) (unlikely((p)->policy == SCHED_BATCH)) +#define is_rt_policy(policy) ((policy) == SCHED_FIFO || \ + (policy) == SCHED_RR) +#define has_rt_policy(p) unlikely(is_rt_policy((p)->policy)) + +#define is_idle_policy(policy) ((policy) == SCHED_IDLEPRIO) +#define idleprio_task(p) unlikely(is_idle_policy((p)->policy)) +#define task_running_idle(p) unlikely((p)->prio == IDLE_PRIO) + +#define is_iso_policy(policy) ((policy) == SCHED_ISO) +#define iso_task(p) unlikely(is_iso_policy((p)->policy)) +#define task_running_iso(p) unlikely((p)->prio == ISO_PRIO) + +#define rq_idle(rq) ((rq)->rq_prio == PRIO_LIMIT) + +#define ISO_PERIOD (5 * HZ) + +/* + * 'User priority' is the nice value converted to something we + * can work with better when scaling various scheduler parameters, + * it's a [ 0 ... 39 ] range. + */ +#define USER_PRIO(p) ((p)-MAX_RT_PRIO) +#define TASK_USER_PRIO(p) USER_PRIO((p)->static_prio) +#define MAX_USER_PRIO (USER_PRIO(MAX_PRIO)) +#define STOP_PRIO (MAX_RT_PRIO - 1) + +/* + * Some helpers for converting to/from various scales. Use shifts to get + * approximate multiples of ten for less overhead. These are internal scales + * only - niffies themselves are real nanoseconds, so anything converting the + * tick into niffies must use TICK_NSEC rather than the approximations here. + */ +#define APPROX_NS_PS (1073741824) /* Approximate ns per second */ +#define JIFFY_NS (APPROX_NS_PS / HZ) +#define NS_TO_JIFFIES(TIME) ((TIME) / JIFFY_NS) +#define HALF_JIFFY_NS (APPROX_NS_PS / HZ / 2) +/* + * time_slice is banked in NS_TO_US() of real nanoseconds, so half a tick in + * those units has to come from TICK_NSEC, not from the approximate scale. + */ +#define HALF_JIFFY_US (NS_TO_US(TICK_NSEC) / 2) +#define MS_TO_NS(TIME) ((TIME) << 20) +#define MS_TO_US(TIME) ((TIME) << 10) +#define NS_TO_MS(TIME) ((TIME) >> 20) +#define NS_TO_US(TIME) ((TIME) >> 10) +#define US_TO_NS(TIME) ((TIME) << 10) +#define TICK_APPROX_NS ((APPROX_NS_PS+HZ/2)/HZ) + +#define RESCHED_US (100) /* Reschedule if less than this many μs left */ + +static void print_scheduler_version(void) +{ + printk(KERN_INFO "MuQSS CPU scheduler v0.31 by Con Kolivas.\n"); +} + +/* + * This is the time all tasks within the same priority round robin. + * Value is in ms and set to a minimum of 6ms. + * Tunable via /proc interface. + */ +int rr_interval __read_mostly = 6; + +/* + * Tunable to choose whether to prioritise latency or throughput, simple + * binary yes or no + */ +int sched_interactive __read_mostly = 1; + +/* + * sched_iso_cpu - sysctl which determines the cpu percentage SCHED_ISO tasks + * are allowed to run five seconds as real time tasks. This is the total over + * all online cpus. + */ +int sched_iso_cpu __read_mostly = 70; + +/* + * sched_yield_type - Choose what sort of yield sched_yield will perform. + * 0: No yield. + * 1: Yield only to better priority/deadline tasks. (default) + * 2: Expire timeslice and recalculate deadline. + */ +int sched_yield_type __read_mostly = 1; + +/* + * The relative length of deadline for each priority(nice) level. + */ +static int prio_ratios[NICE_WIDTH] __read_mostly; + + +/* + * The quota handed out to tasks of all priority levels when refilling their + * time_slice. + */ +static inline int timeslice(void) +{ + return MS_TO_US(rr_interval); +} + +DEFINE_PER_CPU_SHARED_ALIGNED(struct rq, runqueues); + +#ifdef CONFIG_SMP + +/* Define RQ share levels */ +#define RQSHARE_NONE 0 +#define RQSHARE_SMT 1 +#define RQSHARE_MC 2 +#define RQSHARE_MC_LLC 3 +#define RQSHARE_SMP 4 +#define RQSHARE_ALL 5 + +/* Define locality levels */ +#define LOCALITY_SAME 0 +#define LOCALITY_SMT 1 +#define LOCALITY_MC_LLC 2 +#define LOCALITY_MC 3 +#define LOCALITY_SMP 4 +#define LOCALITY_DISTANT 5 + +/* + * This determines what level of runqueue sharing will be done and is + * configurable at boot time with the bootparam rqshare = + */ +static int rqshare __read_mostly = CONFIG_SHARERQ; /* Default RQSHARE_MC */ + +static int __init set_rqshare(char *str) +{ + if (!strncmp(str, "none", 4)) { + rqshare = RQSHARE_NONE; + return 1; + } + if (!strncmp(str, "smt", 3)) { + rqshare = RQSHARE_SMT; + return 1; + } + if (!strncmp(str, "mc", 2)) { + rqshare = RQSHARE_MC; + return 1; + } + if (!strncmp(str, "llc", 3)) { + rqshare = RQSHARE_MC_LLC; + return 1; + } + if (!strncmp(str, "smp", 3)) { + rqshare = RQSHARE_SMP; + return 1; + } + if (!strncmp(str, "all", 3)) { + rqshare = RQSHARE_ALL; + return 1; + } + return 0; +} +__setup("rqshare=", set_rqshare); + +/* + * Total number of runqueues. Equals number of CPUs when there is no runqueue + * sharing but is usually less with SMT/MC sharing of runqueues. + */ +static int total_runqueues __read_mostly = 1; + +static cpumask_t cpu_idle_map ____cacheline_aligned_in_smp; + +/* + * For asym packing, by default the lower numbered cpu has higher priority. + */ +int __weak arch_asym_cpu_priority(int cpu) +{ + return -cpu; +} + +#else +struct rq *uprq; +#endif /* CONFIG_SMP */ + +/* + * sched_smt_active() is unconditionally available now that cpu_smt_mask() is + * cpumask_of(cpu) for !CONFIG_SCHED_SMT, so the key must always be defined. + */ +DEFINE_STATIC_KEY_FALSE(sched_smt_present); +EXPORT_SYMBOL_GPL(sched_smt_present); + +#include "stats.h" + +/* + * All common locking functions performed on rq->lock. rq->clock is local to + * the CPU accessing it so it can be modified just with interrupts disabled + * when we're not updating niffies. + * Looking up task_rq must be done under rq->lock to be safe. + */ + +/* + * RQ-clock updating methods: + */ + +#ifdef HAVE_SCHED_AVG_IRQ +static void update_irq_load_avg(struct rq *rq, long delta); +#else +static inline void update_irq_load_avg(struct rq *rq, long delta) {} +#endif + +/* Use CONFIG_PARAVIRT as this will avoid more #ifdef in arch code. */ +#ifdef CONFIG_PARAVIRT +struct static_key paravirt_steal_rq_enabled; +#endif + +static void update_rq_clock_task(struct rq *rq, s64 delta) +{ +/* + * In theory, the compile should just see 0 here, and optimize out the call + * to sched_rt_avg_update. But I don't trust it... + */ + s64 __maybe_unused steal = 0, irq_delta = 0; +#ifdef CONFIG_IRQ_TIME_ACCOUNTING + irq_delta = irq_time_read(cpu_of(rq)) - rq->prev_irq_time; + + /* + * Since irq_time is only updated on {soft,}irq_exit, we might run into + * this case when a previous update_rq_clock() happened inside a + * {soft,}irq region. + * + * When this happens, we stop ->clock_task and only update the + * prev_irq_time stamp to account for the part that fit, so that a next + * update will consume the rest. This ensures ->clock_task is + * monotonic. + * + * It does however cause some slight miss-attribution of {soft,}irq + * time, a more accurate solution would be to update the irq_time using + * the current rq->clock timestamp, except that would require using + * atomic ops. + */ + if (irq_delta > delta) + irq_delta = delta; + + rq->prev_irq_time += irq_delta; + delta -= irq_delta; +#endif +#ifdef CONFIG_PARAVIRT_TIME_ACCOUNTING + if (static_key_false((¶virt_steal_rq_enabled))) { + steal = paravirt_steal_clock(cpu_of(rq)); + steal -= rq->prev_steal_time_rq; + + if (unlikely(steal > delta)) + steal = delta; + + rq->prev_steal_time_rq += steal; + delta -= steal; + } +#endif + rq->clock_task += delta; + +#ifdef CONFIG_HAVE_SCHED_AVG_IRQ + if (irq_delta + steal) + update_irq_load_avg(rq, irq_delta + steal); +#endif +} + +static inline void update_rq_clock(struct rq *rq) +{ + s64 delta = sched_clock_cpu(cpu_of(rq)) - rq->clock; + + if (unlikely(delta < 0)) + return; + rq->clock += delta; + update_rq_clock_task(rq, delta); +} + +/* + * Niffies are a globally increasing nanosecond counter. They're only used by + * update_load_avg and time_slice_expired, however deadlines are based on them + * across CPUs. Update them whenever we will call one of those functions, and + * synchronise them across CPUs whenever we hold both runqueue locks. + * + * Niffies are the highest of two absolute values that each advance at real + * time: this runqueue's clock, and a tick line advanced by whole jiffies so + * that we keep counting if the rq clock stalls. Because both are absolute, + * time contributed by the tick line, or imported by synchronise_niffies(), is + * never counted again when the rq clock catches up - niffies simply stall + * until it does. Accumulating the maximum of the two *deltas* instead keeps + * every overshoot for good and drifts upwards without bound. + */ +static inline void update_clocks(struct rq *rq) +{ + long jdiff; + + update_rq_clock(rq); + jdiff = jiffies - rq->last_jiffy; + if (jdiff > 0) { + rq->last_jiffy += jdiff; + rq->jiffy_niffies += (u64)jdiff * TICK_NSEC; + } + if (rq->niffies < rq->jiffy_niffies) + rq->niffies = rq->jiffy_niffies; + if (rq->niffies < rq->clock) + rq->niffies = rq->clock; +} + +/* + * Any time we have two runqueues locked we use that as an opportunity to + * synchronise niffies to the highest value as idle ticks may have artificially + * kept niffies low on one CPU and the truth can only be later. + */ +static inline void synchronise_niffies(struct rq *rq1, struct rq *rq2) +{ + if (rq1->niffies > rq2->niffies) + rq2->niffies = rq1->niffies; + else + rq1->niffies = rq2->niffies; +} + +/* + * double_rq_lock - safely lock two runqueues + * + * Note this does not disable interrupts like task_rq_lock, + * you need to do so manually before calling. + */ + +/* For when we know rq1 != rq2 */ +static inline void __double_rq_lock(struct rq *rq1, struct rq *rq2) + __acquires(rq1->lock) + __acquires(rq2->lock) +{ + if (rq1 < rq2) { + raw_spin_lock(rq1->lock); + raw_spin_lock_nested(rq2->lock, SINGLE_DEPTH_NESTING); + } else { + raw_spin_lock(rq2->lock); + raw_spin_lock_nested(rq1->lock, SINGLE_DEPTH_NESTING); + } +} + +static inline void double_rq_lock(struct rq *rq1, struct rq *rq2) + __acquires(rq1->lock) + __acquires(rq2->lock) +{ + BUG_ON(!irqs_disabled()); + if (rq1->lock == rq2->lock) { + raw_spin_lock(rq1->lock); + __acquire(rq2->lock); /* Fake it out ;) */ + } else + __double_rq_lock(rq1, rq2); + synchronise_niffies(rq1, rq2); +} + +/* + * double_rq_unlock - safely unlock two runqueues + * + * Note this does not restore interrupts like task_rq_unlock, + * you need to do so manually after calling. + */ +static inline void double_rq_unlock(struct rq *rq1, struct rq *rq2) + __releases(rq1->lock) + __releases(rq2->lock) +{ + raw_spin_unlock(rq1->lock); + if (rq1->lock != rq2->lock) + raw_spin_unlock(rq2->lock); + else + __release(rq2->lock); +} + +static inline void lock_all_rqs(void) +{ + int cpu; + + preempt_disable(); + for_each_possible_cpu(cpu) { + struct rq *rq = cpu_rq(cpu); + + do_raw_spin_lock(rq->lock); + } +} + +static inline void unlock_all_rqs(void) +{ + int cpu; + + for_each_possible_cpu(cpu) { + struct rq *rq = cpu_rq(cpu); + + do_raw_spin_unlock(rq->lock); + } + preempt_enable(); +} + +/* Specially nest trylock an rq */ +static inline bool trylock_rq(struct rq *this_rq, struct rq *rq) +{ + if (unlikely(!do_raw_spin_trylock(rq->lock))) + return false; + spin_acquire(&rq->lock->dep_map, SINGLE_DEPTH_NESTING, 1, _RET_IP_); + synchronise_niffies(this_rq, rq); + return true; +} + +/* Unlock a specially nested trylocked rq */ +static inline void unlock_rq(struct rq *rq) +{ + spin_release(&rq->lock->dep_map, _RET_IP_); + do_raw_spin_unlock(rq->lock); +} + +/* + * cmpxchg based fetch_or, macro so it works for different integer types + */ +#define fetch_or(ptr, mask) \ + ({ \ + typeof(ptr) _ptr = (ptr); \ + typeof(mask) _mask = (mask); \ + typeof(*_ptr) _old, _val = *_ptr; \ + \ + for (;;) { \ + _old = cmpxchg(_ptr, _val, _val | _mask); \ + if (_old == _val) \ + break; \ + _val = _old; \ + } \ + _old; \ +}) + +#if defined(CONFIG_SMP) && defined(TIF_POLLING_NRFLAG) +/* + * Atomically set TIF_NEED_RESCHED and test for TIF_POLLING_NRFLAG, + * this avoids any races wrt polling state changes and thereby avoids + * spurious IPIs. + */ +static bool set_nr_and_not_polling(struct task_struct *p) +{ + struct thread_info *ti = task_thread_info(p); + return !(fetch_or(&ti->flags, _TIF_NEED_RESCHED) & _TIF_POLLING_NRFLAG); +} + +/* + * Atomically set TIF_NEED_RESCHED if TIF_POLLING_NRFLAG is set. + * + * If this returns true, then the idle task promises to call + * sched_ttwu_pending() and reschedule soon. + */ +static bool set_nr_if_polling(struct task_struct *p) +{ + struct thread_info *ti = task_thread_info(p); + typeof(ti->flags) old, val = READ_ONCE(ti->flags); + + for (;;) { + if (!(val & _TIF_POLLING_NRFLAG)) + return false; + if (val & _TIF_NEED_RESCHED) + return true; + old = cmpxchg(&ti->flags, val, val | _TIF_NEED_RESCHED); + if (old == val) + break; + val = old; + } + return true; +} + +#else +static bool set_nr_and_not_polling(struct task_struct *p) +{ + set_tsk_need_resched(p); + return true; +} + +#ifdef CONFIG_SMP +static bool set_nr_if_polling(struct task_struct *p) +{ + return false; +} +#endif +#endif + +static bool __wake_q_add(struct wake_q_head *head, struct task_struct *task) +{ + struct wake_q_node *node = &task->wake_q; + + /* + * Atomically grab the task, if ->wake_q is !nil already it means + * it's already queued (either by us or someone else) and will get the + * wakeup due to that. + * + * In order to ensure that a pending wakeup will observe our pending + * state, even in the failed case, an explicit smp_mb() must be used. + */ + smp_mb__before_atomic(); + if (unlikely(cmpxchg_relaxed(&node->next, NULL, WAKE_Q_TAIL))) + return false; + + /* + * The head is context local, there can be no concurrency. + */ + *head->lastp = node; + head->lastp = &node->next; + return true; +} + +/** + * wake_q_add() - queue a wakeup for 'later' waking. + * @head: the wake_q_head to add @task to + * @task: the task to queue for 'later' wakeup + * + * Queue a task for later wakeup, most likely by the wake_up_q() call in the + * same context, _HOWEVER_ this is not guaranteed, the wakeup can come + * instantly. + * + * This function must be used as-if it were wake_up_process(); IOW the task + * must be ready to be woken at this location. + */ +void wake_q_add(struct wake_q_head *head, struct task_struct *task) +{ + if (__wake_q_add(head, task)) + get_task_struct(task); +} + +/** + * wake_q_add_safe() - safely queue a wakeup for 'later' waking. + * @head: the wake_q_head to add @task to + * @task: the task to queue for 'later' wakeup + * + * Queue a task for later wakeup, most likely by the wake_up_q() call in the + * same context, _HOWEVER_ this is not guaranteed, the wakeup can come + * instantly. + * + * This function must be used as-if it were wake_up_process(); IOW the task + * must be ready to be woken at this location. + * + * This function is essentially a task-safe equivalent to wake_q_add(). Callers + * that already hold reference to @task can call the 'safe' version and trust + * wake_q to do the right thing depending whether or not the @task is already + * queued for wakeup. + */ +void wake_q_add_safe(struct wake_q_head *head, struct task_struct *task) +{ + if (!__wake_q_add(head, task)) + put_task_struct(task); +} + +void wake_up_q(struct wake_q_head *head) +{ + struct wake_q_node *node = head->first; + + while (node != WAKE_Q_TAIL) { + struct task_struct *task; + + task = container_of(node, struct task_struct, wake_q); + BUG_ON(!task); + /* Task can safely be re-inserted now */ + node = node->next; + task->wake_q.next = NULL; + + /* + * wake_up_process() executes a full barrier, which pairs with + * the queueing in wake_q_add() so as not to miss wakeups. + */ + wake_up_process(task); + put_task_struct(task); + } +} + +static inline void smp_sched_reschedule(int cpu) +{ + if (likely(cpu_online(cpu))) + smp_send_reschedule(cpu); +} + +/* + * resched_task - mark a task 'to be rescheduled now'. + * + * On UP this means the setting of the need_resched flag, on SMP it + * might also involve a cross-CPU call to trigger the scheduler on + * the target CPU. + */ +#ifdef CONFIG_PREEMPT_DYNAMIC +static DEFINE_STATIC_KEY_FALSE(sk_dynamic_preempt_lazy); +static __always_inline bool dynamic_preempt_lazy(void) +{ + return static_branch_unlikely(&sk_dynamic_preempt_lazy); +} +#else +static __always_inline bool dynamic_preempt_lazy(void) +{ + return IS_ENABLED(CONFIG_PREEMPT_LAZY); +} +#endif + +static void resched_task(struct task_struct *p) +{ + int cpu; +#ifdef CONFIG_LOCKDEP + /* Kernel threads call this when creating workqueues while still + * inactive from __kthread_bind_mask, holding only the pi_lock */ + if (!(p->flags & PF_KTHREAD)) { + struct rq *rq = task_rq(p); + + lockdep_assert_held(rq->lock); + } +#endif + if (test_tsk_need_resched(p)) + return; + + cpu = task_cpu(p); + if (cpu == smp_processor_id()) { + set_tsk_need_resched(p); + set_preempt_need_resched(); + return; + } + + if (set_nr_and_not_polling(p)) + smp_sched_reschedule(cpu); + else + trace_sched_wake_idle_without_ipi(cpu); +} + +/* + * A task that is not running or queued will not have a node set. + * A task that is queued but not running will have a node set. + * A task that is currently running will have ->on_cpu set but no node set. + */ +static inline bool task_queued(struct task_struct *p) +{ + return !skiplist_node_empty(&p->node); +} + +static void enqueue_task(struct rq *rq, struct task_struct *p, int flags); +static inline void resched_if_idle(struct rq *rq); + +static inline bool deadline_before(u64 deadline, u64 time) +{ + return (deadline < time); +} + +/* + * Deadline is "now" in niffies + (offset by priority). Setting the deadline + * is the key to everything. It distributes cpu fairly amongst tasks of the + * same nice value, it proportions cpu according to nice level, it means the + * task that last woke up the longest ago has the earliest deadline, thus + * ensuring that interactive tasks get low latency on wake up. The CPU + * proportion works out to the square of the virtual deadline difference, so + * this equation will give nice 19 3% CPU compared to nice 0. + */ +static inline u64 prio_deadline_diff(int user_prio) +{ + return (prio_ratios[user_prio] * rr_interval * (MS_TO_NS(1) / 128)); +} + +static inline u64 task_deadline_diff(struct task_struct *p) +{ + return prio_deadline_diff(TASK_USER_PRIO(p)); +} + +static inline u64 static_deadline_diff(int static_prio) +{ + return prio_deadline_diff(USER_PRIO(static_prio)); +} + +static inline int longest_deadline_diff(void) +{ + return prio_deadline_diff(39); +} + +static inline int ms_longest_deadline_diff(void) +{ + return NS_TO_MS(longest_deadline_diff()); +} + +#ifdef CONFIG_MUQSS_IOTIME +/* + * Scale an I/O time penalty by nice level, exactly as prio_deadline_diff() + * scales rr_interval. A deadline is virtual time, already stretched by + * prio_ratios[] before it means anything; adding raw nanoseconds of device + * time to it mixes two different currencies. Passing the penalty through the + * same ratio makes it commensurate with the deadline it is added to, so a + * given amount of I/O costs a task the same share of its own allotment + * whatever its nice level, and a niced down task is pushed back further in + * absolute terms than a niced up one for the same device time. + * + * The multiply is 64 bit, as it is in prio_deadline_diff(). Wrapping it would + * need a penalty over U64_MAX / prio_ratios[39], which is some forty days of + * device occupancy charged to one task between two scheduling events, so it + * is not a case worth writing code for. + */ +static inline u64 prio_penalty_diff(int user_prio, u64 penalty) +{ + return penalty * prio_ratios[user_prio] / 128; +} + +/* + * Idleprio tasks are charged at the highest nice level whatever nice value + * they happen to carry. SCHED_IDLEPRIO is a class beneath the whole nice + * range rather than a position within it, and a task there has said it should + * run only when nothing else wants the CPU. Scaling by its own nice would let + * an idleprio task sitting at nice -20 be charged the lightest rate of all + * for keeping the disk busy, which is backwards. Their deadlines already sort + * below everything else in enqueue_task(); this keeps what they are charged + * for I/O consistent with that. + */ +static inline u64 task_penalty_diff(struct task_struct *p, u64 penalty) +{ + if (idleprio_task(p)) + return prio_penalty_diff(39, penalty); + + return prio_penalty_diff(TASK_USER_PRIO(p), penalty); +} +#endif + +#ifdef CONFIG_MUQSS_IOTIME +/* + * Take the block device time accumulated on this task's behalf since it was + * last charged, and convert it to an amount of virtual deadline to push the + * task back by. The debt is consumed as it is read, so each nanosecond of + * device time demotes the task exactly once no matter which caller gets to it + * first. The charge is added to the deadline, so it is a demotion, not a + * boost: a task that keeps a device busy for everybody else stops getting + * full CPU priority for free. + * + * Charged one for one, a nanosecond of deadline for each nanosecond of device + * time consumed. That rate says device time and CPU time cost a task the + * same, which is the premise of the feature rather than a setting within it, + * so there is nothing to weight one against the other with. + * + * The charge is then scaled by nice through task_penalty_diff(), the same way + * rr_interval is, so that it is in the same virtual currency as the deadline + * it is added to. One for one is therefore one for one in deadline terms + * rather than in raw nanoseconds: at nice 0 the ratio is prio_ratios[20] / + * 128, about 6.7, exactly as an ordinary nice 0 timeslice is worth 6.7 + * rr_intervals of deadline. + * + * There is no ceiling. A task that keeps a device busy can be demoted + * arbitrarily far behind, past the deadline of the lowest nice level. See + * MuQSS-iotime-design.md. + * + * This has to be consumed from both time_slice_expired() and enqueue_task(). + * Expiry alone misses the streaming reader, which blocks before exhausting its + * slice and so never refreshes its deadline. Enqueue alone is not enough + * either: time_slice_expired() assigns the deadline absolutely, resetting the + * task to an uncharged baseline, and two of its callers (sched_yield() and + * yield_to()) have no enqueue behind them to reapply the charge. + */ +static inline u64 consume_iotime_penalty(struct task_struct *p) +{ + u64 debt = atomic64_xchg(&p->io_debt_ns, 0); + + if (!debt) + return 0; + + return task_penalty_diff(p, debt); +} + +/* + * The same for CPU time spent in another thread's context on this task's + * behalf, which today means kworkers running work items it queued. + * + * Charged one for one, as device time is. A nanosecond of CPU time is a + * nanosecond of CPU time wherever it was spent, and the whole premise + * here is that the thread it was spent in should not decide whether it counts. + * Work a task asks for in its own context is already charged at exactly that + * rate: update_cpu_clock_switch() and update_cpu_clock_tick() deduct it from + * ->time_slice with no regard for which mode it was spent in, so a task that + * burns its whole quantum inside a syscall is demoted as surely as one that + * spun in userspace. Charging deferred work at anything other than parity + * would say that where the kernel chose to do the work changes what it cost, + * which is the bug this exists to fix. Forcing IRQ threading, as -ck does, + * only moves more work into that blind spot. + * + * It is still scaled by nice through task_penalty_diff(), like every other + * addition to a deadline, so that it is in the same virtual currency as the + * deadline it lands on. + * + * As with iotime there is no ceiling. The runaway this invites is a different + * shape to the I/O case and worth naming, because here CPU consumption is + * being punished with CPU demotion: a demoted task need not stop generating + * the work, so it can be starved while kworkers keep charging it. It is + * bounded in practice because the debt a task can accrue is bounded by the + * work it queued, and a task demoted enough to stop running stops queueing + * more. + * + * Kept as a separate counter from io_debt_ns rather than folded into it, + * because the two are quantities of very different size: work item runtimes + * are microseconds where device occupancy is milliseconds, and sharing an + * accumulator would let one vanish into the rounding of the other. Separate + * also keeps them separable in /proc//iotime when the question is why a + * task was demoted. + */ +static inline u64 consume_kerntime_penalty(struct task_struct *p) +{ + u64 debt = atomic64_xchg(&p->kern_debt_ns, 0); + + if (!debt) + return 0; + + return task_penalty_diff(p, debt); +} +#else +static inline u64 consume_iotime_penalty(struct task_struct *p) +{ + return 0; +} + +static inline u64 consume_kerntime_penalty(struct task_struct *p) +{ + return 0; +} +#endif + +/* + * Everything charged against a task's deadline that did not come out of its + * own time_slice. + */ +static inline u64 consume_task_penalty(struct task_struct *p) +{ + return consume_iotime_penalty(p) + consume_kerntime_penalty(p); +} + +static inline bool rq_local(struct rq *rq); + +#ifndef SCHED_CAPACITY_SCALE +#define SCHED_CAPACITY_SCALE 1024 +#endif + +static inline int rq_load(struct rq *rq) +{ + return rq->nr_running; +} + +/* + * Update the load average for feeding into cpu frequency governors. Use a + * rough estimate of a rolling average with ~ time constant of 32ms. + * 80/128 ~ 0.63. * 80 / 32768 / 128 == * 5 / 262144 + * Make sure a call to update_clocks has been made before calling this to get + * an updated rq->niffies. + */ +static void update_load_avg(struct rq *rq, unsigned int flags) +{ + long us_interval, load; + + us_interval = NS_TO_US(rq->niffies - rq->load_update); + if (unlikely(us_interval <= 0)) + return; + + load = rq->load_avg - (rq->load_avg * us_interval * 5 / 262144); + if (unlikely(load < 0)) + load = 0; + load += rq_load(rq) * SCHED_CAPACITY_SCALE * us_interval * 5 / 262144; + rq->load_avg = load; + + rq->load_update = rq->niffies; + update_irq_load_avg(rq, 0); + if (likely(rq_local(rq))) + cpufreq_trigger(rq, flags); +} + +#ifdef HAVE_SCHED_AVG_IRQ +/* + * IRQ variant of update_load_avg below. delta is actually time in nanoseconds + * here so we scale curload to how long it's been since the last update. + */ +static void update_irq_load_avg(struct rq *rq, long delta) +{ + long us_interval, load; + + us_interval = NS_TO_US(rq->niffies - rq->irq_load_update); + if (unlikely(us_interval <= 0)) + return; + + load = rq->irq_load_avg - (rq->irq_load_avg * us_interval * 5 / 262144); + if (unlikely(load < 0)) + load = 0; + load += NS_TO_US(delta) * SCHED_CAPACITY_SCALE * 5 / 262144; + rq->irq_load_avg = load; + + rq->irq_load_update = rq->niffies; +} +#endif + +static inline void update_best_key(struct rq *rq) +{ + WRITE_ONCE(rq->sl->best_key, rq->node->next[0]->key); +} + +/* + * Removing from the runqueue. Enter with rq locked. Deleting a task + * from the skip list is done via the stored node reference in the task struct + * and does not require a full look up. Thus it occurs in O(k) time where k + * is the "level" of the list the task was stored at - usually 0, max 3. + */ +static void dequeue_task(struct rq *rq, struct task_struct *p, int flags) +{ + /* + * Everything below is accounting for @p having been on @rq, so none of + * it may happen if it was not: a task whose node is unlinked is on no + * runqueue at all and skiplist_delete() has left the list untouched. + * + * nr_running in particular is unsigned, so a single spurious decrement + * wraps it to ~0 and the runqueue never looks empty again. That wedges + * sched_cpu_wait_empty(), whose only exit is nr_running <= 1, and a CPU + * offline then hangs forever with the dying CPU idle. psi_dequeue() + * would likewise clear state that is counted somewhere else. Leaving + * best_key alone is right for the same reason - the list did not + * change. + * + * skiplist_delete() has already warned; this only keeps a lost race in + * the caller from becoming permanent corruption of the runqueue. + */ + if (unlikely(!skiplist_delete(rq->sl, &p->node))) + return; + + update_best_key(rq); + update_clocks(rq); + + if (!(flags & DEQUEUE_SAVE)) { + sched_info_dequeue(rq, p); + /* Pass full flags: DEQUEUE_SLEEP vs migration clear all state. */ + psi_dequeue(p, flags); + } + rq->nr_running--; + if (rt_task(p)) + rq->rt_nr_running--; + update_load_avg(rq, flags); +} + +#ifdef CONFIG_PREEMPT_RCU +static bool rcu_read_critical(struct task_struct *p) +{ + return p->rcu_read_unlock_special.b.blocked; +} +#else /* CONFIG_PREEMPT_RCU */ +#define rcu_read_critical(p) (false) +#endif /* CONFIG_PREEMPT_RCU */ + +/* + * To determine if it's safe for a task of SCHED_IDLEPRIO to actually run as + * an idle task, we ensure none of the following conditions are met. + */ +static bool idleprio_suitable(struct task_struct *p) +{ + return (!(p->sched_contributes_to_load) && !(p->flags & (PF_EXITING)) && + !signal_pending(p) && !rcu_read_critical(p) && !freezing(p)); +} + +/* + * To determine if a task of SCHED_ISO can run in pseudo-realtime, we check + * that the iso_refractory flag is not set. + */ +static inline bool isoprio_suitable(struct rq *rq) +{ + return !rq->iso_refractory; +} + +static inline void inc_nr_running(struct rq *rq) +{ + rq->nr_running++; + if (trace_sched_update_nr_running_tp_enabled()) { + call_trace_sched_update_nr_running(rq, 1); + } +} + +static inline void dec_nr_running(struct rq *rq) +{ + rq->nr_running--; + if (trace_sched_update_nr_running_tp_enabled()) { + call_trace_sched_update_nr_running(rq, -1); + } +} + +/* + * A running task is off the skiplist, so its share of rt_nr_running is not + * maintained by the enqueue/dequeue pair; __schedule() hands that slot over on + * every switch instead. When a *running* task's priority crosses the rt + * boundary neither happens, so fix the count up here. + */ +static inline void rt_running_reprio(struct rq *rq, int oldprio, int newprio) +{ + if (!rt_prio(newprio) == !rt_prio(oldprio)) + return; + + if (rt_prio(newprio)) + rq->rt_nr_running++; + else + rq->rt_nr_running--; +} + +/* + * Adding to the runqueue. Enter with rq locked. + */ +static void enqueue_task(struct rq *rq, struct task_struct *p, int flags) +{ + unsigned int randseed, cflags = 0; + u64 sl_id, penalty; + + if (!rt_task(p)) { + /* Check it hasn't gotten rt from PI */ + if ((idleprio_task(p) && idleprio_suitable(p)) || + (iso_task(p) && isoprio_suitable(rq))) + p->prio = p->normal_prio; + else + p->prio = NORMAL_PRIO; + } else + rq->rt_nr_running++; + /* + * The sl_id key passed to the skiplist generates a sorted list. + * Realtime and sched iso tasks run FIFO so they only need be sorted + * according to priority. The skiplist will put tasks of the same + * key inserted later in FIFO order. Tasks of sched normal, batch + * and idleprio are sorted according to their deadlines. Idleprio + * tasks are offset by an impossibly large deadline value ensuring + * they get sorted into last positions, but still according to their + * own deadlines. This creates a "landscape" of skiplists running + * from priority 0 realtime in first place to the lowest priority + * idleprio tasks last. Skiplist insertion is an O(log n) process. + */ + /* + * Charge for device time and kernel work consumed since this task was + * last queued. Consumed unconditionally so debt cannot accumulate + * while a task is realtime, but only applied below, since realtime and + * iso tasks sort by priority rather than by deadline and so are never + * demoted. + */ + penalty = consume_task_penalty(p); + + if (p->prio <= ISO_PRIO) { + sl_id = p->prio; + } else { + p->deadline += penalty; + sl_id = p->deadline; + if (idleprio_task(p)) { + if (p->prio == IDLE_PRIO) + sl_id |= 0xF000000000000000; + else + sl_id += longest_deadline_diff(); + } + } + /* + * Some architectures don't have better than microsecond resolution + * so mask out ~microseconds as the random seed for skiplist insertion. + */ + update_clocks(rq); + if (!(flags & ENQUEUE_RESTORE)) { + sched_info_enqueue(rq, p); + /* Full flags so ENQUEUE_MIGRATED is visible to psi_enqueue(). */ + psi_enqueue(p, flags); + } + + randseed = (rq->niffies >> 10) & 0xFFFFFFFF; + skiplist_insert(rq->sl, &p->node, sl_id, randseed); + update_best_key(rq); + if (p->in_iowait) + cflags |= SCHED_CPUFREQ_IOWAIT; + inc_nr_running(rq); + update_load_avg(rq, cflags); +} + +/* + * Returns the relative length of deadline all compared to the shortest + * deadline which is that of nice -20. + */ +static inline int task_prio_ratio(struct task_struct *p) +{ + return prio_ratios[TASK_USER_PRIO(p)]; +} + +/* + * task_timeslice - all tasks of all priorities get the exact same timeslice + * length. CPU distribution is handled by giving different deadlines to + * tasks of different priorities. Use 128 as the base value for fast shifts. + */ +static inline int task_timeslice(struct task_struct *p) +{ + return (rr_interval * task_prio_ratio(p) / 128); +} + +#ifdef CONFIG_SMP +/* Entered with rq locked */ +static inline void resched_if_idle(struct rq *rq) +{ + if (rq_idle(rq)) + resched_task(rq->curr); +} + +static inline bool rq_local(struct rq *rq) +{ + return (rq->cpu == smp_processor_id()); +} +#ifdef CONFIG_SMT_NICE +static const cpumask_t *thread_cpumask(int cpu); + +/* Find the best real time priority running on any SMT siblings of cpu and if + * none are running, the static priority of the best deadline task running. + * The lookups to the other runqueues is done lockless as the occasional wrong + * value would be harmless. */ +static int best_smt_bias(struct rq *this_rq) +{ + int other_cpu, best_bias = 0; + + for_each_cpu(other_cpu, &this_rq->thread_mask) { + struct rq *rq = cpu_rq(other_cpu); + + if (rq_idle(rq)) + continue; + if (unlikely(!rq->online)) + continue; + if (!rq->rq_mm) + continue; + if (likely(rq->rq_smt_bias > best_bias)) + best_bias = rq->rq_smt_bias; + } + return best_bias; +} + +static int task_prio_bias(struct task_struct *p) +{ + if (rt_task(p)) + return 1 << 30; + else if (task_running_iso(p)) + return 1 << 29; + else if (task_running_idle(p)) + return 0; + return MAX_PRIO - p->static_prio; +} + +static DEFINE_STATIC_KEY_FALSE(smt_nice_enabled); + +/* We've already decided p can run on CPU, now test if it shouldn't for SMT + * nice reasons. */ +static bool smt_should_schedule(struct task_struct *p, struct rq *this_rq) +{ + int best_bias, task_bias; + + if (!idleprio_suitable(p)) + return true; + best_bias = best_smt_bias(this_rq); + /* The smt siblings are all idle or running IDLEPRIO */ + if (best_bias < 1) + return true; + task_bias = task_prio_bias(p); + if (task_bias < 1) + return false; + if (task_bias >= best_bias) + return true; + /* Dither 25% cpu of normal tasks regardless of nice difference */ + if (best_bias % 4 == 1) + return true; + /* Sorry, you lose */ + return false; +} + +static inline bool smt_schedule(struct task_struct *p, struct rq *this_rq) +{ + if (!static_branch_unlikely(&smt_nice_enabled)) + return true; + /* Kernel threads and RT tasks always run */ + if (unlikely(!p->mm) || rt_task(p)) + return true; + return smt_should_schedule(p, this_rq); +} +#else /* CONFIG_SMT_NICE */ +#define smt_schedule(p, this_rq) (true) +#endif /* CONFIG_SMT_NICE */ + +static inline void atomic_set_cpu(int cpu, cpumask_t *cpumask) +{ + set_bit(cpu, (volatile unsigned long *)cpumask); +} + +/* + * The cpu_idle_map stores a bitmap of all the CPUs currently idle to + * allow easy lookup of whether any suitable idle CPUs are available. + * It's cheaper to maintain a binary yes/no if there are any idle CPUs on the + * idle_cpus variable than to do a full bitmask check when we are busy. The + * bits are set atomically but read locklessly as occasional false positive / + * negative is harmless. + */ +static inline void set_cpuidle_map(int cpu) +{ + /* + * Only an active CPU may advertise itself as a wakeup target. + * select_best_cpu() returns resched_best_idle()'s pick directly, + * and that pick comes straight out of this map without consulting + * is_cpu_allowed(), so a CPU on its way down (online but no longer + * active) would keep re-arming itself here every time it idled and + * collect fresh wakeups that sched_cpu_wait_empty() has to chase. + * set_rq_offline() clears the map; this keeps it clear. + */ + if (likely(cpu_active(cpu))) + atomic_set_cpu(cpu, &cpu_idle_map); +} + +static inline void atomic_clear_cpu(int cpu, cpumask_t *cpumask) +{ + clear_bit(cpu, (volatile unsigned long *)cpumask); +} + +static inline void clear_cpuidle_map(int cpu) +{ + atomic_clear_cpu(cpu, &cpu_idle_map); +} + +static bool suitable_idle_cpus(struct task_struct *p) +{ + return (cpumask_intersects(p->cpus_ptr, &cpu_idle_map)); +} + +/* + * Resched current on rq. We don't know if rq is local to this CPU nor if it + * is locked so we do not use an intermediate variable for the task to avoid + * having it dereferenced. + */ +static void resched_curr(struct rq *rq) +{ + int cpu; + + if (test_tsk_need_resched(rq->curr)) + return; + + rq->preempt = rq->curr; + cpu = rq->cpu; + + /* We're doing this without holding the rq lock if it's not task_rq */ + + if (cpu == smp_processor_id()) { + set_tsk_need_resched(rq->curr); + set_preempt_need_resched(); + return; + } + + if (set_nr_and_not_polling(rq->curr)) + smp_sched_reschedule(cpu); + else + trace_sched_wake_idle_without_ipi(cpu); +} + +#define CPUIDLE_NO_SIBLING (1) +#define CPUIDLE_DIFF_THREAD (2) +#define CPUIDLE_DIFF_CORE_LLC (4) +#define CPUIDLE_DIFF_CORE (8) +#define CPUIDLE_CACHE_BUSY (16) +#define CPUIDLE_DIFF_CPU (32) +#define CPUIDLE_THREAD_BUSY (64) +#define CPUIDLE_DIFF_NODE (128) + +/* + * The best idle CPU is chosen according to the CPUIDLE ranking above where the + * lowest value would give the most suitable CPU to schedule p onto next. The + * order works out to be the following: + * + * Same thread, idle or busy cache, idle or busy threads + * Other core, same cache, idle or busy cache, idle threads. + * Same node, other CPU, idle cache, idle threads. + * Same node, other CPU, busy cache, idle threads. + * Other core, same cache, busy threads. + * Same node, other CPU, busy threads. + * Other node, other CPU, idle cache, idle threads. + * Other node, other CPU, busy cache, idle threads. + * Other node, other CPU, busy threads. + * + * CPUIDLE_NO_SIBLING is the least significant rank so it only separates cores + * that are otherwise equal, preferring a core with an idle SMT sibling over one + * with no siblings at all. On hybrid CPUs that indirectly prefers P cores over + * E cores, since only the former have siblings. A core whose sibling is busy + * still ranks below a core with no siblings as it only offers half a core. + * This does not treat CPUs with a single offline sibling as idle for + * simplicity. + */ +static int best_mask_cpu(int best_cpu, struct rq *rq, cpumask_t *tmpmask) +{ + int best_ranking = CPUIDLE_DIFF_NODE | CPUIDLE_THREAD_BUSY | + CPUIDLE_DIFF_CPU | CPUIDLE_CACHE_BUSY | CPUIDLE_DIFF_CORE | + CPUIDLE_DIFF_CORE_LLC | CPUIDLE_DIFF_THREAD | CPUIDLE_NO_SIBLING; + unsigned long best_idle_jiffy = 0; + int cpu_tmp; + + if (cpumask_test_cpu(best_cpu, tmpmask)) + goto out; + + for_each_cpu(cpu_tmp, tmpmask) { + int ranking, locality; + struct rq *tmp_rq; + + ranking = 0; + tmp_rq = cpu_rq(cpu_tmp); + + locality = rq->cpu_locality[cpu_tmp]; +#ifdef CONFIG_NUMA + if (locality > LOCALITY_SMP) + ranking |= CPUIDLE_DIFF_NODE; + else +#endif + if (locality > LOCALITY_MC) + ranking |= CPUIDLE_DIFF_CPU; +#ifdef CONFIG_SCHED_MC + else if (locality == LOCALITY_MC_LLC) + ranking |= CPUIDLE_DIFF_CORE_LLC; + else if (locality == LOCALITY_MC) + ranking |= CPUIDLE_DIFF_CORE; + if (!(tmp_rq->cache_idle(tmp_rq))) + ranking |= CPUIDLE_CACHE_BUSY; +#endif +#ifdef CONFIG_SCHED_SMT + if (locality == LOCALITY_SMT) + ranking |= CPUIDLE_DIFF_THREAD; + if (!tmp_rq->has_smt_sibling) + ranking |= CPUIDLE_NO_SIBLING; + else if (!cpumask_subset(&tmp_rq->thread_mask, &cpu_idle_map)) + ranking |= CPUIDLE_THREAD_BUSY; +#endif + /* Also look for the most recently idled CPU as it will likely + * be still at a higher CPU frequency */ + if (ranking < best_ranking || + (ranking == best_ranking && tmp_rq->idle_jiffy > best_idle_jiffy)) { + best_cpu = cpu_tmp; + best_ranking = ranking; + best_idle_jiffy = tmp_rq->idle_jiffy; + } + } +out: + return best_cpu; +} + +bool cpus_share_cache(int this_cpu, int that_cpu) +{ + struct rq *this_rq = cpu_rq(this_cpu); + + return (this_rq->cpu_locality[that_cpu] < LOCALITY_SMP); +} + +/* As per resched_curr but only will resched idle task */ +static inline void resched_idle(struct rq *rq) +{ + if (test_tsk_need_resched(rq->idle)) + return; + + rq->preempt = rq->idle; + + if (rq_local(rq)) { + set_tsk_need_resched(rq->idle); + set_preempt_need_resched(); + return; + } + + /* + * Atomically set NEED_RESCHED and only IPI if the idle task is not + * polling — same protocol as resched_curr / wake_up_idle_cpu. + */ + if (set_nr_and_not_polling(rq->idle)) + smp_sched_reschedule(rq->cpu); + else + trace_sched_wake_idle_without_ipi(rq->cpu); +} + +DEFINE_PER_CPU(cpumask_t, idlemask); + +static struct rq *resched_best_idle(struct task_struct *p, int cpu) +{ + cpumask_t *tmpmask = &(per_cpu(idlemask, cpu)); + struct rq *rq; + int best_cpu; + + cpumask_and(tmpmask, p->cpus_ptr, &cpu_idle_map); + best_cpu = best_mask_cpu(cpu, task_rq(p), tmpmask); + rq = cpu_rq(best_cpu); + if (!smt_schedule(p, rq)) + return NULL; + rq->preempt = p; + resched_idle(rq); + return rq; +} + +static inline void resched_suitable_idle(struct task_struct *p) +{ + if (suitable_idle_cpus(p)) + resched_best_idle(p, task_cpu(p)); +} + +static inline struct rq *rq_order(struct rq *rq, int cpu) +{ + return rq->rq_order[cpu]; +} +#else /* CONFIG_SMP */ +static inline void set_cpuidle_map(int cpu) +{ +} + +static inline void clear_cpuidle_map(int cpu) +{ +} + +static inline bool suitable_idle_cpus(struct task_struct *p) +{ + return uprq->curr == uprq->idle; +} + +static inline void resched_suitable_idle(struct task_struct *p) +{ +} + +static inline void resched_curr(struct rq *rq) +{ + resched_task(rq->curr); +} + +static inline void resched_if_idle(struct rq *rq) +{ +} + +static inline bool rq_local(struct rq *rq) +{ + return true; +} + +static inline struct rq *rq_order(struct rq *rq, int cpu) +{ + return rq; +} + +static inline bool smt_schedule(struct task_struct *p, struct rq *rq) +{ + return true; +} + +/* One CPU shares its cache with itself. */ +bool cpus_share_cache(int this_cpu, int that_cpu) +{ + return true; +} +#endif /* CONFIG_SMP */ + +static inline int normal_prio(struct task_struct *p) +{ + if (has_rt_policy(p)) + return MAX_RT_PRIO - 1 - p->rt_priority; + if (idleprio_task(p)) + return IDLE_PRIO; + if (iso_task(p)) + return ISO_PRIO; + return NORMAL_PRIO; +} + +/* + * Calculate the current priority, i.e. the priority + * taken into account by the scheduler. This value might + * be boosted by RT tasks as it will be RT if the task got + * RT-boosted. If not then it returns p->normal_prio. + */ +static int effective_prio(struct task_struct *p) +{ + p->normal_prio = normal_prio(p); + /* + * If we are RT tasks or we were boosted to RT priority, + * keep the priority unchanged. Otherwise, update priority + * to the normal priority: + */ + if (!rt_prio(p->prio)) + return p->normal_prio; + return p->prio; +} + +/* + * activate_task - move a task to the runqueue. Enter with rq locked. + */ +static void activate_task(struct rq *rq, struct task_struct *p, int flags) +{ + resched_if_idle(rq); + + /* Sleep profiling (SLEEP_PROFILING) was removed upstream. */ + + p->prio = effective_prio(p); + enqueue_task(rq, p, flags); + p->on_rq = TASK_ON_RQ_QUEUED; +} + +/* + * deactivate_task - If it's running, it's not on the runqueue and we can just + * decrement the nr_running. Enter with rq locked. + */ +static inline void deactivate_task(struct task_struct *p, struct rq *rq) +{ + p->on_rq = 0; + sched_info_dequeue(rq, p); + /* deactivate_task is always DEQUEUE_SLEEP in muqss */ + psi_dequeue(p, DEQUEUE_SLEEP); +} + +/* + * PSI counts the state a task is in per CPU, indexed by task_cpu(), so + * whenever task_cpu() changes for a task that still carries state, that state + * has to be moved with it. That is TSK_RUNNING for a task being taken to + * another runqueue, and TSK_IOWAIT for one that is still asleep. Whoever + * clears them next does so against task_cpu(p), and if that is no longer the + * CPU they were counted on the per-CPU counter underflows. + * + * TSK_ONCPU is deliberately not moved. It belongs to psi_sched_switch(), and + * a task whose CPU is being changed here is by definition not on one. + */ +#ifdef CONFIG_PSI +static inline unsigned int psi_migrate_begin(struct task_struct *p) +{ + unsigned int migrate = p->psi_flags & ~TSK_ONCPU; + + if (migrate) + psi_task_change(p, migrate, 0); + return migrate; +} + +static inline void psi_migrate_end(struct task_struct *p, unsigned int migrate) +{ + if (migrate) + psi_task_change(p, 0, migrate); +} +#else /* !CONFIG_PSI */ +static inline unsigned int psi_migrate_begin(struct task_struct *p) +{ + return 0; +} + +static inline void psi_migrate_end(struct task_struct *p, unsigned int migrate) +{ +} +#endif /* CONFIG_PSI */ + +#ifdef CONFIG_SMP +void set_task_cpu(struct task_struct *p, unsigned int new_cpu) +{ + unsigned int migrate; + struct rq *rq; + + if (task_cpu(p) == new_cpu) + return; + + /* Do NOT call set_task_cpu on a currently queued task as we will not + * be reliably holding the rq lock after changing CPU. */ + BUG_ON(task_queued(p)); + rq = task_rq(p); + +#ifdef CONFIG_LOCKDEP + /* + * The caller should hold either p->pi_lock or rq->lock, when changing + * a task's CPU. ->pi_lock for waking tasks, rq->lock for runnable tasks. + * + * Furthermore, all task_rq users should acquire both locks, see + * task_rq_lock(). + */ + WARN_ON_ONCE(debug_locks && !(lockdep_is_held(&p->pi_lock) || + lockdep_is_held(rq->lock))); +#endif + + trace_sched_migrate_task(p, new_cpu); + rseq_sched_set_ids_changed(p); + perf_event_task_migrate(p); + + /* + * After ->cpu is set up to a new value, task_rq_lock(p, ...) can be + * successfully executed on another CPU. We must ensure that updates of + * per-task data have been completed by this moment. + */ + smp_wmb(); + + p->wake_cpu = new_cpu; + + if (task_running(rq, p)) { + /* + * We should only be calling this on a running task if we're + * holding rq lock. + */ + lockdep_assert_held(rq->lock); + + /* + * We can't change the task_thread_info CPU on a running task + * as p will still be protected by the rq lock of the CPU it + * is still running on so we only set the wake_cpu for it to be + * lazily updated once off the CPU. + */ + return; + } + + migrate = psi_migrate_begin(p); + WRITE_ONCE(task_thread_info(p)->cpu, new_cpu); + psi_migrate_end(p, migrate); + /* We're no longer protecting p after this point since we're holding + * the wrong runqueue lock. */ +} +#endif /* CONFIG_SMP */ + +/* + * Move a task off the runqueue and take it to a cpu for it will + * become the running task. + */ +static inline void take_task(struct rq *rq, int cpu, struct task_struct *p) +{ + struct rq *p_rq = task_rq(p); + + dequeue_task(p_rq, p, DEQUEUE_SAVE); + if (p_rq != rq) { + sched_info_dequeue(p_rq, p); + sched_info_enqueue(rq, p); + } + /* + * DEQUEUE_SAVE left TSK_RUNNING in place, and set_task_cpu() moves it + * to @cpu along with the task. + */ + set_task_cpu(p, cpu); +} + +/* + * Returns a descheduling task to the runqueue unless it is being + * deactivated. + */ +static inline void return_task(struct task_struct *p, struct rq *rq, + int cpu, bool deactivate) +{ + if (deactivate) + deactivate_task(p, rq); + else { +#ifdef CONFIG_SMP + /* + * set_task_cpu was called on the running task that doesn't + * want to deactivate so it has to be enqueued to a different + * CPU and we need its lock. Tag it to be moved with as the + * lock is dropped in finish_lock_switch. + */ + if (unlikely(p->wake_cpu != cpu)) + WRITE_ONCE(p->on_rq, TASK_ON_RQ_MIGRATING); + else +#endif + enqueue_task(rq, p, ENQUEUE_RESTORE); + } +} + +/* Enter with rq lock held. We know p is on the local cpu */ +static inline void __set_tsk_resched(struct task_struct *p) +{ + set_tsk_need_resched(p); + set_preempt_need_resched(); +} + +/** + * task_curr - is this task currently executing on a CPU? + * @p: the task in question. + * + * Return: 1 if the task is currently executing. 0 otherwise. + */ +inline int task_curr(const struct task_struct *p) +{ + return cpu_curr(task_cpu(p)) == p; +} + +static __always_inline +int __task_state_match(struct task_struct *p, unsigned int state) +{ + if (READ_ONCE(p->__state) & state) + return 1; + + if (READ_ONCE(p->saved_state) & state) + return -1; + + return 0; +} + +static __always_inline +int task_state_match(struct task_struct *p, unsigned int state) +{ + int match; + + /* + * Serialize against current_save_and_set_rtlock_wait_state(), + * current_restore_rtlock_saved_state(), and __refrigerator(). + */ + raw_spin_lock_irq(&p->pi_lock); + match = __task_state_match(p, state); + raw_spin_unlock_irq(&p->pi_lock); + + return match; +} + +/* + * wait_task_inactive - wait for a thread to unschedule. + * + * Wait for the thread to block in any of the states set in @match_state. + * If it changes, i.e. @p might have woken up, then return zero. When we + * succeed in waiting for @p to be off its CPU, we return a positive number + * (its total switch count). If a second call a short while later returns + * the same number, the caller can be sure that @p has remained unscheduled + * the whole time. + * + * The caller must ensure that the task *will* unschedule sometime soon, + * else this function might spin for a *long* time. This function can't + * be called with interrupts off, or it may introduce deadlock with + * smp_call_function() if an IPI is sent by the same process we are + * waiting to become inactive. + */ +unsigned long wait_task_inactive(struct task_struct *p, unsigned int match_state) +{ + int running, queued, match; + struct rq_flags rf; + unsigned long ncsw; + struct rq *rq; + + for (;;) { + rq = task_rq(p); + + /* + * If the task is actively running on another CPU + * still, just relax and busy-wait without holding + * any locks. + * + * NOTE! Since we don't hold any locks, it's not + * even sure that "rq" stays as the right runqueue! + * But we don't care, since this will return false + * if the runqueue has changed and p is actually now + * running somewhere else! + */ + while (task_running(rq, p)) { + if (!task_state_match(p, match_state)) + return 0; + cpu_relax(); + } + + /* + * Ok, time to look more closely! We need the rq + * lock now, to be *sure*. If we're wrong, we'll + * just go back and repeat. + */ + rq = task_rq_lock(p, &rf); + trace_sched_wait_task(p); + running = task_running(rq, p); + queued = task_on_rq_queued(p); + ncsw = 0; + if ((match = __task_state_match(p, match_state))) { + /* + * When matching on p->saved_state, consider this task + * still queued so it will wait. + */ + if (match < 0) + queued = 1; + ncsw = p->nvcsw | LONG_MIN; /* sets MSB */ + } + task_rq_unlock(rq, p, &rf); + + /* + * If it changed from the expected state, bail out now. + */ + if (unlikely(!ncsw)) + break; + + /* + * Was it really running after all now that we + * checked with the proper locks actually held? + * + * Oops. Go back and try again.. + */ + if (unlikely(running)) { + cpu_relax(); + continue; + } + + /* + * It's not enough that it's not actively running, + * it must be off the runqueue _entirely_, and not + * preempted! + * + * So if it was still runnable (but just not actively + * running right now), it's preempted, and we should + * yield - it could be a while. + */ + if (unlikely(queued)) { + ktime_t to = NSEC_PER_SEC / HZ; + + set_current_state(TASK_UNINTERRUPTIBLE); + schedule_hrtimeout(&to, HRTIMER_MODE_REL); + continue; + } + + /* + * Ahh, all good. It wasn't running, and it wasn't + * runnable, which means that it will never become + * running in the future either. We're all done! + */ + break; + } + + return ncsw; +} + +/*** + * kick_process - kick a running thread to enter/exit the kernel + * @p: the to-be-kicked thread + * + * Cause a process which is running on another CPU to enter + * kernel-mode, without any delay. (to get signals handled.) + * + * NOTE: this function doesn't have to take the runqueue lock, + * because all it wants to ensure is that the remote task enters + * the kernel. If the IPI races and the task has been migrated + * to another CPU then no harm is done and the purpose has been + * achieved as well. + */ +#ifdef CONFIG_SMP +void kick_process(struct task_struct *p) +{ + int cpu; + + preempt_disable(); + cpu = task_cpu(p); + if ((cpu != smp_processor_id()) && task_curr(p)) + smp_sched_reschedule(cpu); + preempt_enable(); +} +#else /* !CONFIG_SMP */ +/* @p can only be running on the CPU we are already running on. */ +void kick_process(struct task_struct *p) +{ +} +#endif /* CONFIG_SMP */ +EXPORT_SYMBOL_GPL(kick_process); + +/* + * RT tasks preempt purely on priority. SCHED_NORMAL tasks preempt on the + * basis of earlier deadlines. SCHED_IDLEPRIO don't preempt anything else or + * between themselves, they cooperatively multitask. An idle rq scores as + * prio PRIO_LIMIT so it is always preempted. + */ +static inline bool +can_preempt(struct task_struct *p, int prio, u64 deadline) +{ + /* Better static priority RT task or better policy preemption */ + if (p->prio < prio) + return true; + if (p->prio > prio) + return false; + if (p->policy == SCHED_BATCH) + return false; + /* SCHED_NORMAL and ISO will preempt based on deadline */ + if (!deadline_before(p->deadline, deadline)) + return false; + return true; +} + +#ifdef CONFIG_SMP + +/* + * Per-CPU kthreads are allowed to run on !active && online CPUs, see + * __set_cpus_allowed_ptr(). + */ +static inline bool is_cpu_allowed(struct task_struct *p, int cpu) +{ + if (!cpumask_test_cpu(cpu, p->cpus_ptr)) + return false; + + /* migrate_disable() must be allowed to finish on an online CPU. */ + if (is_migration_disabled(p)) + return cpu_online(cpu); + + if (!(p->flags & PF_KTHREAD)) + return cpu_active(cpu); + + /* KTHREAD_IS_PER_CPU is always allowed. */ + if (kthread_is_per_cpu(p)) + return cpu_online(cpu); + + /* Regular kernel threads don't get to stay during offline. */ + if (cpu_dying(cpu)) + return false; + + /* But are allowed during online. */ + return cpu_online(cpu); +} + +/* + * Check to see if p can run on cpu, and if not, whether there are any online + * CPUs it can run on instead. This only happens with the hotplug threads that + * bring up the CPUs. + */ +static inline bool sched_other_cpu(struct task_struct *p, int cpu) +{ + if (unlikely(is_migration_disabled(p) && task_cpu(p) != cpu)) + return true; + /* + * Defer to the single placement predicate: userspace is refused a + * deactivated CPU, a regular kthread is refused a dying one, while + * per-CPU kthreads and a migrate_disable() pin on this CPU still + * may run there. + */ + if (likely(cpumask_test_cpu(cpu, p->cpus_ptr))) + return !is_cpu_allowed(p, cpu); + if (p->nr_cpus_allowed == 1) { + cpumask_t valid_mask; + + /* + * Nowhere left to send it, so it may as well run here. The mask + * has to be the same one valid_task_cpu() uses to pick the + * destination, or the two disagree and the task is placed on a + * CPU that then refuses to run it. + * + * That is what a regular kthread affine to a single dying CPU + * hits: the CPU is still online, so intersecting with + * cpu_online_mask alone leaves it non-empty and this escape + * hatch never fires, while is_cpu_allowed() has already refused + * the dying CPU itself. It ends up runnable with no CPU willing + * to pick it. bind_zero() would free it by overriding the + * affinity, but that only runs from sched_cpu_wait_empty(), + * several hotplug states later - and anything the CPU going + * down waits for in between, such as blk_mq_hctx_notify_offline() + * waiting on a threaded completion interrupt, deadlocks against + * it. + */ + cpumask_and(&valid_mask, p->cpus_ptr, cpu_online_mask); + if (!kthread_is_per_cpu(p)) + cpumask_andnot(&valid_mask, &valid_mask, cpu_dying_mask); + if (unlikely(cpumask_empty(&valid_mask))) + return false; + } + return true; +} + +static inline bool needs_other_cpu(struct task_struct *p, int cpu) +{ + if (unlikely(is_migration_disabled(p))) + return task_cpu(p) != cpu; + return !is_cpu_allowed(p, cpu); +} + +/* + * Pin p->cpus_ptr to this CPU so EDT / select_best_cpu cannot steal a + * preempted migrate_disable() task. Called under rq->lock on prev. + */ +static void migrate_disable_switch(struct rq *rq, struct task_struct *p) +{ + if (likely(!p->migration_disabled)) + return; + if (p->cpus_ptr != &p->cpus_mask) + return; + p->cpus_ptr = cpumask_of(cpu_of(rq)); +} + +#define cpu_online_map (*(cpumask_t *)cpu_online_mask) + +static void try_preempt(struct task_struct *p, struct rq *this_rq) +{ + int i, this_entries = rq_load(this_rq); + cpumask_t tmp; + + /* + * An idle CPU first: waking one costs less than bouncing a task + * that is already running, and it is what keeps work spread out. + * Only if none is available do we consider preemption, and the + * loop below starts at cpu_order[0] - this_rq - so @p still gets + * to preempt its own dest before any remote runqueue. + */ + if (suitable_idle_cpus(p) && resched_best_idle(p, task_cpu(p))) + return; + + /* IDLEPRIO tasks never preempt anything but idle */ + if (p->policy == SCHED_IDLEPRIO) + return; + + cpumask_and(&tmp, &cpu_online_map, p->cpus_ptr); + + for (i = 0; i < num_online_cpus(); i++) { + struct rq *rq = this_rq->cpu_order[i]; + + if (!cpumask_test_cpu(rq->cpu, &tmp)) + continue; + + if (!sched_interactive && rq != this_rq && rq_load(rq) <= this_entries) + continue; + if (smt_schedule(p, rq) && can_preempt(p, rq->rq_prio, rq->rq_deadline)) { + /* We set rq->preempting lockless, it's a hint only */ + rq->preempting = p; + resched_curr(rq); + return; + } + } +} + +static int __set_cpus_allowed_ptr(struct task_struct *p, + const struct cpumask *new_mask, + u32 flags); +#else /* CONFIG_SMP */ +static inline bool needs_other_cpu(struct task_struct *p, int cpu) +{ + return false; +} + +static void try_preempt(struct task_struct *p, struct rq *this_rq) +{ + if (p->policy == SCHED_IDLEPRIO) + return; + if (can_preempt(p, uprq->rq_prio, uprq->rq_deadline)) + resched_curr(uprq); +} + +static inline int __set_cpus_allowed_ptr(struct task_struct *p, + const struct cpumask *new_mask, + u32 __always_unused flags) +{ + return set_cpus_allowed_ptr(p, new_mask); +} +#endif /* CONFIG_SMP */ + +static void +ttwu_stat(struct task_struct *p, int cpu, int wake_flags) +{ + struct rq *rq; + + if (!schedstat_enabled()) + return; + + rq = this_rq(); + +#ifdef CONFIG_SMP + if (cpu == rq->cpu) { + __schedstat_inc(rq->ttwu_local); + } else { + struct sched_domain *sd; + + rcu_read_lock(); + for_each_domain(rq->cpu, sd) { + if (cpumask_test_cpu(cpu, sched_domain_span(sd))) { + __schedstat_inc(sd->ttwu_wake_remote); + break; + } + } + rcu_read_unlock(); + } + +#endif /* CONFIG_SMP */ + + __schedstat_inc(rq->ttwu_count); +} + +/* + * Mark the task runnable and perform wakeup-preemption. + */ +static void ttwu_do_wakeup(struct rq *rq, struct task_struct *p, int wake_flags) +{ + /* + * WF_SYNC means the waker will leave the CPU shortly. Prefer an + * idle CPU and do not preempt the waker itself — it will pick the + * wakee up on the next schedule(). A remote dest still needs + * kicking, or the wakee sits until dest's next tick (or forever + * on a nohz_full hog). + */ + if ((wake_flags & WF_SYNC) && rq == this_rq()) + resched_suitable_idle(p); + else + try_preempt(p, rq); + WRITE_ONCE(p->__state, TASK_RUNNING); + trace_sched_wakeup(p); +} + +static void +ttwu_do_activate(struct rq *rq, struct task_struct *p, int wake_flags) +{ + int en_flags = ENQUEUE_WAKEUP; + + lockdep_assert_held(rq->lock); + + if (p->sched_contributes_to_load) + rq->nr_uninterruptible--; + +#ifdef CONFIG_SMP + if (wake_flags & WF_MIGRATED) + en_flags |= ENQUEUE_MIGRATED; + else +#endif + if (p->in_iowait) { + delayacct_blkio_end(p); + atomic_dec(&task_rq(p)->nr_iowait); + } + + activate_task(rq, p, en_flags); + ttwu_do_wakeup(rq, p, wake_flags); +} + +/* + * Consider @p being inside a wait loop: + * + * for (;;) { + * set_current_state(TASK_UNINTERRUPTIBLE); + * + * if (CONDITION) + * break; + * + * schedule(); + * } + * __set_current_state(TASK_RUNNING); + * + * between set_current_state() and schedule(). In this case @p is still + * runnable, so all that needs doing is change p->__state back to TASK_RUNNING in + * an atomic manner. + * + * By taking task_rq(p)->lock we serialize against schedule(), if @p->on_rq + * then schedule() must still happen and p->__state can be changed to + * TASK_RUNNING. Otherwise we lost the race, schedule() has happened, and we + * need to do a full wakeup with enqueue. + * + * Returns: %true when the wakeup is done, + * %false otherwise. + */ +static int ttwu_runnable(struct task_struct *p, int wake_flags) +{ + struct rq *rq; + int ret = 0; + + rq = __task_rq_lock(p, NULL); + if (likely(task_on_rq_queued(p))) { + ttwu_do_wakeup(rq, p, wake_flags); + ret = 1; + } + __task_rq_unlock(rq, p, NULL); + + return ret; +} + +#ifdef CONFIG_SMP +void sched_ttwu_pending(void *arg) +{ + struct llist_node *llist = arg; + struct rq *rq = this_rq(); + struct task_struct *p, *t; + struct rq_flags rf; + + if (!llist) + return; + + /* + * rq::ttwu_pending racy indication of out-standing wakeups. + * Races such that false-negatives are possible, since they + * are shorter lived that false-positives would be. + */ + WRITE_ONCE(rq->ttwu_pending, 0); + + rq_lock_irqsave(rq, &rf); + + llist_for_each_entry_safe(p, t, llist, wake_entry.llist) { + if (WARN_ON_ONCE(p->on_cpu)) + smp_cond_load_acquire(&p->on_cpu, !VAL); + + if (WARN_ON_ONCE(task_cpu(p) != cpu_of(rq))) + set_task_cpu(p, cpu_of(rq)); + + ttwu_do_activate(rq, p, p->sched_remote_wakeup ? WF_MIGRATED : 0); + } + + rq_unlock_irqrestore(rq, &rf); +} + +/* + * Called from smp.c: report whether the caller still needs to send the IPI, + * having woken a polling idle task in passing. + */ +bool call_function_single_prep_ipi(int cpu) +{ + if (set_nr_if_polling(cpu_rq(cpu)->idle)) { + trace_sched_wake_idle_without_ipi(cpu); + return false; + } + + return true; +} + +/* + * Queue a task on the target CPUs wake_list and wake the CPU via IPI if + * necessary. The wakee CPU on receipt of the IPI will queue the task + * via sched_ttwu_wakeup() for activation so the wakee incurs the cost + * of the wakeup instead of the waker. + */ +static void __ttwu_queue_wakelist(struct task_struct *p, int cpu, int wake_flags) +{ + struct rq *rq = cpu_rq(cpu); + + /* + * Carry WF_MIGRATED across to the CPU that will do the enqueue. Losing + * it there makes ttwu_do_activate() treat an already migrated wakeup + * as a local one, decrementing rq->nr_iowait a second time and telling + * PSI to clear an iowait that psi_ttwu_dequeue() has already dropped. + */ + p->sched_remote_wakeup = !!(wake_flags & WF_MIGRATED); + + WRITE_ONCE(rq->ttwu_pending, 1); + __smp_call_single_queue(cpu, &p->wake_entry.llist); +} + +void wake_up_if_idle(int cpu) +{ + struct rq *rq = cpu_rq(cpu); + struct rq_flags rf; + + rcu_read_lock(); + + if (!is_idle_task(rcu_dereference(rq->curr))) + goto out; + + /* + * Match mainline: always go through resched_curr so TIF_NEED_RESCHED + * is set before any IPI. Sending a reschedule IPI alone does nothing + * useful — scheduler_ipi() only folds an already-set need_resched. + */ + rq_lock_irqsave(rq, &rf); + if (is_idle_task(rq->curr)) + resched_curr(rq); + rq_unlock_irqrestore(rq, &rf); + +out: + rcu_read_unlock(); +} + +static inline bool ttwu_queue_cond(int cpu, int wake_flags) +{ + /* + * Do not complicate things with the async wake_list while the CPU is + * in hotplug state. + */ + if (!cpu_active(cpu)) + return false; + + /* + * If the CPU does not share cache, then queue the task on the + * remote rqs wakelist to avoid accessing remote data. + */ + if (!cpus_share_cache(smp_processor_id(), cpu)) + return true; + + /* + * If the task is descheduling and the only running task on the + * CPU then use the wakelist to offload the task activation to + * the soon-to-be-idle CPU as the current CPU is likely busy. + * nr_running is checked to avoid unnecessary task stacking. + */ + if ((wake_flags & WF_ON_CPU) && cpu_rq(cpu)->nr_running <= 1) + return true; + + return false; +} + +static bool ttwu_queue_wakelist(struct task_struct *p, int cpu, int wake_flags) +{ + /* CFS would require sched_feat(TTWU_QUEUE) here but that is + * fixed enabled */ + if (ttwu_queue_cond(cpu, wake_flags)) { + if (WARN_ON_ONCE(cpu == smp_processor_id())) + return false; + + sched_clock_cpu(cpu); /* Sync clocks across CPUs */ + __ttwu_queue_wakelist(p, cpu, wake_flags); + return true; + } + + return false; +} + +static int valid_task_cpu(struct task_struct *p) +{ + cpumask_t valid_mask; + + if (unlikely(is_migration_disabled(p))) + return task_cpu(p); + + /* + * Kthreads may be bound to a CPU that is not yet online (per-CPU + * hotplug threads created during smp_init). Placement for running + * must still pick an online CPU — sched_other_cpu() allows them to + * be selected despite the affinity mismatch until their CPU is up. + * Userspace tasks are restricted to the active mask as usual. + */ + if (p->flags & PF_KTHREAD) { + cpumask_and(&valid_mask, p->cpus_ptr, cpu_online_mask); + /* + * Only KTHREAD_IS_PER_CPU gets to stay on a CPU that is on + * its way down, so do not hand a regular kthread back the + * CPU bind_zero() just moved it off. Mirrors is_cpu_allowed(). + */ + if (!kthread_is_per_cpu(p)) + cpumask_andnot(&valid_mask, &valid_mask, cpu_dying_mask); + } else + cpumask_and(&valid_mask, p->cpus_ptr, cpu_active_mask); + + if (unlikely(!cpumask_weight(&valid_mask))) { + if ((p->flags & PF_KTHREAD) && num_online_cpus()) + return cpumask_any(cpu_online_mask); + /* We shouldn't be hitting this any more */ + printk(KERN_WARNING "SCHED: No cpumask for %s/%d weight %d\n", p->comm, + p->pid, cpumask_weight(p->cpus_ptr)); + return cpumask_any(p->cpus_ptr); + } + return cpumask_any(&valid_mask); +} + +/* + * For a task that's just being woken up we have a valuable balancing + * opportunity so choose the nearest cache most lightly loaded runqueue. + * Entered with rq locked and returns with the chosen runqueue locked. + */ +static inline int select_best_cpu(struct task_struct *p) +{ + unsigned int idlest = ~0U; + struct rq *rq = NULL; + int i; + + if (suitable_idle_cpus(p)) { + int cpu = task_cpu(p); + + if (unlikely(needs_other_cpu(p, cpu))) + cpu = valid_task_cpu(p); + rq = resched_best_idle(p, cpu); + if (likely(rq)) + return rq->cpu; + } + + for (i = 0; i < num_online_cpus(); i++) { + struct rq *other_rq = task_rq(p)->cpu_order[i]; + int entries; + + if (!other_rq->online) + continue; + if (needs_other_cpu(p, other_rq->cpu)) + continue; + entries = rq_load(other_rq); + if (entries >= idlest) + continue; + idlest = entries; + rq = other_rq; + } + if (unlikely(!rq)) { + /* + * The walk found nowhere to put @p. Its affinity may contain + * only offline CPUs (hotplug kthreads bound before their CPU + * is up), or the walk may simply not have reached an allowed + * one: cpu_order[] is built once at boot, so once any CPU is + * offline the num_online_cpus() bound above stops short of its + * tail and never examines what is there. + * + * Staying put is only an answer if this CPU is one @p is + * allowed to run on. Otherwise defer to valid_task_cpu(), + * which intersects the mask directly and so does not depend on + * that ordering at all. Never place a wakeup on a CPU the task + * is not allowed on, whether because it is offline or because + * it is not in the mask; a blocked task keeps a stale + * task_cpu() across an affinity change, so the latter is the + * common case here. + */ + if (unlikely(needs_other_cpu(p, task_cpu(p)))) + return valid_task_cpu(p); + return task_cpu(p); + } + return rq->cpu; +} +#else /* CONFIG_SMP */ + +static inline bool ttwu_queue_wakelist(struct task_struct *p, int cpu, int wake_flags) +{ + return false; +} + +static int valid_task_cpu(struct task_struct *p) +{ + return 0; +} + +static inline int select_best_cpu(struct task_struct *p) +{ + return 0; +} + +static struct rq *resched_best_idle(struct task_struct *p, int cpu) +{ + return NULL; +} +#endif /* CONFIG_SMP */ + +static void ttwu_queue(struct task_struct *p, int cpu, int wake_flags) +{ + struct rq *rq = cpu_rq(cpu); + + if (ttwu_queue_wakelist(p, cpu, wake_flags)) + return; + + rq_lock(rq); + update_rq_clock(rq); + ttwu_do_activate(rq, p, wake_flags); + rq_unlock(rq); +} + +/* + * Consider @state matched against @p, taking p->saved_state into account. + * + * The caller holds p::pi_lock if p != current or has preemption disabled + * when p == current. + * + * The rules of saved_state: + * + * The related locking code always holds p::pi_lock when updating + * p::saved_state, which means the code is fully serialized in both cases. + * + * For PREEMPT_RT, the lock wait and lock wakeups happen via TASK_RTLOCK_WAIT. + * No other bits set. This allows to distinguish all wakeup scenarios. + * + * For FREEZER, the wakeup happens via TASK_FROZEN. No other bits set. This + * allows us to prevent early wakeup of tasks before they can be run on + * asymmetric ISA architectures (eg ARMv9). + */ +static __always_inline +bool ttwu_state_match(struct task_struct *p, unsigned int state, int *success) +{ + int match; + + if (IS_ENABLED(CONFIG_DEBUG_PREEMPT)) { + WARN_ON_ONCE((state & TASK_RTLOCK_WAIT) && + state != TASK_RTLOCK_WAIT); + } + + *success = !!(match = __task_state_match(p, state)); + + /* + * Saved state preserves the task state across blocking on + * an RT lock or TASK_FREEZABLE tasks. If the state matches, + * set p::saved_state to TASK_RUNNING, but do not wake the task + * because it waits for a lock wakeup or __thaw_task(). Also + * indicate success because from the regular waker's point of + * view this has succeeded. + * + * After acquiring the lock the task will restore p::__state + * from p::saved_state which ensures that the regular + * wakeup is not lost. The restore will also set + * p::saved_state to TASK_RUNNING so any further tests will + * not result in false positives vs. @success + */ + if (match < 0) + p->saved_state = TASK_RUNNING; + + return match > 0; +} + +/*** + * try_to_wake_up - wake up a thread + * @p: the thread to be awakened + * @state: the mask of task states that can be woken + * @wake_flags: wake modifier flags (WF_*) + * + * Put it on the run-queue if it's not already there. The "current" + * thread is always on the run-queue (except when the actual + * re-schedule is in progress), and as such you're allowed to do + * the simpler "current->__state = TASK_RUNNING" to mark yourself + * runnable without the overhead of this. + * + * Return: %true if @p was woken up, %false if it was already running. + * or @state didn't match @p's state. + */ +int try_to_wake_up(struct task_struct *p, unsigned int state, int wake_flags) +{ + unsigned long flags; + int cpu, success = 0; + + preempt_disable(); + if (p == current) { + /* + * We're waking current, this means 'p->on_rq' and 'task_cpu(p) + * == smp_processor_id()'. Together this means we can special + * case the whole 'p->on_rq && ttwu_runnable()' case below + * without taking any locks. + * + * In particular: + * - we rely on Program-Order guarantees for all the ordering, + * - we're serialized against set_special_state() by virtue of + * it disabling IRQs (this allows not taking ->pi_lock). + */ + if (!ttwu_state_match(p, state, &success)) + goto out; + + trace_sched_waking(p); + p->__state = TASK_RUNNING; + trace_sched_wakeup(p); + goto out; + } + + /* + * If we are going to wake up a thread waiting for CONDITION we + * need to ensure that CONDITION=1 done by the caller can not be + * reordered with p->__state check below. This pairs with smp_store_mb() + * in set_current_state() that the waiting thread does. + */ + raw_spin_lock_irqsave(&p->pi_lock, flags); + smp_mb__after_spinlock(); + if (!ttwu_state_match(p, state, &success)) + goto unlock; + + trace_sched_waking(p); + + /* + * Ensure we load p->on_rq _after_ p->__state, otherwise it would + * be possible to, falsely, observe p->on_rq == 0 and get stuck + * in smp_cond_load_acquire() below. + * + * sched_ttwu_pending() try_to_wake_up() + * STORE p->on_rq = 1 LOAD p->__state + * UNLOCK rq->lock + * + * __schedule() (switch to task 'p') + * LOCK rq->lock smp_rmb(); + * smp_mb__after_spinlock(); + * UNLOCK rq->lock + * + * [task p] + * STORE p->__state = UNINTERRUPTIBLE LOAD p->on_rq + * + * Pairs with the LOCK+smp_mb__after_spinlock() on rq->lock in + * __schedule(). See the comment for smp_mb__after_spinlock(). + */ + smp_rmb(); + if (READ_ONCE(p->on_rq) && ttwu_runnable(p, wake_flags)) + goto unlock; + +#ifdef CONFIG_SMP + /* + * Ensure we load p->on_cpu _after_ p->on_rq, otherwise it would be + * possible to, falsely, observe p->on_cpu == 0. + * + * One must be running (->on_cpu == 1) in order to remove oneself + * from the runqueue. + * + * __schedule() (switch to task 'p') try_to_wake_up() + * STORE p->on_cpu = 1 LOAD p->on_rq + * UNLOCK rq->lock + * + * __schedule() (put 'p' to sleep) + * LOCK rq->lock smp_rmb(); + * smp_mb__after_spinlock(); + * STORE p->on_rq = 0 LOAD p->on_cpu + * + * Pairs with the LOCK+smp_mb__after_spinlock() on rq->lock in + * __schedule(). See the comment for smp_mb__after_spinlock(). + * + * Form a control-dep-acquire with p->on_rq == 0 above, to ensure + * schedule()'s deactivate_task() has 'happened' and p will no longer + * care about it's own p->__state. See the comment in __schedule(). + */ + smp_acquire__after_ctrl_dep(); + + /* + * We're doing the wakeup (@success == 1), they did a dequeue (p->on_rq + * == 0), which means we need to do an enqueue, change p->__state to + * TASK_WAKING such that we can unlock p->pi_lock before doing the + * enqueue, such as ttwu_queue_wakelist(). + */ + p->__state = TASK_WAKING; + + /* + * If the owning (remote) CPU is still in the middle of schedule() with + * this task as prev, considering queueing p on the remote CPUs wake_list + * which potentially sends an IPI instead of spinning on p->on_cpu to + * let the waker make forward progress. This is safe because IRQs are + * disabled and the IPI will deliver after on_cpu is cleared. + * + * Ensure we load task_cpu(p) after p->on_cpu: + * + * set_task_cpu(p, cpu); + * STORE task_cpu(p) = @cpu + * __schedule() (switch to task 'p') + * LOCK rq->lock + * smp_mb__after_spin_lock() smp_cond_load_acquire(&p->on_cpu) + * STORE p->on_cpu = 1 LOAD task_cpu(p) + * + * to ensure we observe the correct CPU on which the task is currently + * scheduling. + */ + if (smp_load_acquire(&p->on_cpu) && + ttwu_queue_wakelist(p, task_cpu(p), wake_flags | WF_ON_CPU)) + goto unlock; + + /* + * If the owning (remote) CPU is still in the middle of schedule() with + * this task as prev, wait until it's done referencing the task. + * + * Pairs with the smp_store_release() in finish_task(). + * + * This ensures that tasks getting woken will be fully ordered against + * their previous state and preserve Program Order. + */ + smp_cond_load_acquire(&p->on_cpu, !VAL); + + cpu = select_best_cpu(p); + if (task_cpu(p) != cpu) { + if (p->in_iowait) { + delayacct_blkio_end(p); + atomic_dec(&task_rq(p)->nr_iowait); + } + + wake_flags |= WF_MIGRATED; + psi_ttwu_dequeue(p); + set_task_cpu(p, cpu); + } + +#else + cpu = task_cpu(p); +#endif /* CONFIG_SMP */ + + ttwu_queue(p, cpu, wake_flags); +unlock: + raw_spin_unlock_irqrestore(&p->pi_lock, flags); +out: + if (success) + ttwu_stat(p, task_cpu(p), wake_flags); + preempt_enable(); + + return success; +} + +static bool __task_needs_rq_lock(struct task_struct *p) +{ + unsigned int state = READ_ONCE(p->__state); + + /* + * Since pi->lock blocks try_to_wake_up(), we don't need rq->lock when + * the task is blocked. Make sure to check @state since ttwu() can drop + * locks at the end, see ttwu_queue_wakelist(). + */ + if (state == TASK_RUNNING || state == TASK_WAKING) + return true; + + /* + * Ensure we load p->on_rq after p->__state, otherwise it would be + * possible to, falsely, observe p->on_rq == 0. + * + * See try_to_wake_up() for a longer comment. + */ + smp_rmb(); + if (p->on_rq) + return true; + + /* + * Ensure the task has finished __schedule() and will not be referenced + * anymore. Again, see try_to_wake_up() for a longer comment. + */ + smp_rmb(); + smp_cond_load_acquire(&p->on_cpu, !VAL); + + return false; +} + +/** + * task_call_func - Invoke a function on task in fixed state + * @p: Process for which the function is to be invoked, can be @current. + * @func: Function to invoke. + * @arg: Argument to function. + * + * Fix the task in it's current state by avoiding wakeups and or rq operations + * and call @func(@arg) on it. This function can use task_is_runnable() and + * task_curr() to work out what the state is, if required. Given that @func + * can be invoked with a runqueue lock held, it had better be quite + * lightweight. + * + * Returns: + * Whatever @func returns + */ +int task_call_func(struct task_struct *p, task_call_f func, void *arg) +{ + struct rq_flags rf; + int ret; + + raw_spin_lock_irqsave(&p->pi_lock, rf.flags); + + if (__task_needs_rq_lock(p)) { + struct rq *rq = __task_rq_lock(p, &rf); + + /* + * At this point the task is pinned; either: + * - blocked and we're holding off wakeups (pi->lock) + * - woken, and we're holding off enqueue (rq->lock) + * - queued, and we're holding off schedule (rq->lock) + * - running, and we're holding off de-schedule (rq->lock) + * + * The called function (@func) can use: task_curr(), p->on_rq and + * p->__state to differentiate between these states. + */ + ret = func(p, arg); + + __task_rq_unlock(rq, p, &rf); + } else { + ret = func(p, arg); + } + + raw_spin_unlock_irqrestore(&p->pi_lock, rf.flags); + return ret; +} + +/** + * wake_up_process - Wake up a specific process + * @p: The process to be woken up. + * + * Attempt to wake up the nominated process and move it to the set of runnable + * processes. + * + * Return: 1 if the process was woken up, 0 if it was already running. + * + * This function executes a full memory barrier before accessing the task state. + */ +int wake_up_process(struct task_struct *p) +{ + return try_to_wake_up(p, TASK_NORMAL, 0); +} +EXPORT_SYMBOL(wake_up_process); + +int wake_up_state(struct task_struct *p, unsigned int state) +{ + return try_to_wake_up(p, state, 0); +} + +static void time_slice_expired(struct task_struct *p, struct rq *rq); + +/* + * Perform scheduler related setup for a newly forked process p. + * p is forked by current. + */ +int sched_fork(u64 __maybe_unused clone_flags, struct task_struct *p) +{ + unsigned long flags; + +#ifdef CONFIG_PREEMPT_NOTIFIERS + INIT_HLIST_HEAD(&p->preempt_notifiers); +#endif + +#ifdef CONFIG_COMPACTION + p->capture_control = NULL; +#endif + +#ifdef CONFIG_SMP + p->wake_entry.u_flags = CSD_TYPE_TTWU; +#endif + /* A new task starts with no I/O history of its own. */ + muqss_iotime_task_init(p); + + /* + * We mark the process as NEW here. This guarantees that + * nobody will actually run it, and a signal or other external + * event cannot wake it up and insert it on the runqueue either. + */ + p->__state = TASK_NEW; + + /* + * The process state is set to the same value of the process executing + * do_fork() code. That is running. This guarantees that nobody will + * actually run it, and a signal or other external event cannot wake + * it up and insert it on the runqueue either. + */ + + /* Should be reset in fork.c but done here for ease of MuQSS patching */ + p->on_cpu = + p->on_rq = + p->utime = + p->stime = + p->sched_time = + p->stime_ns = + p->utime_ns = 0; + skiplist_node_init(&p->node); + + /* + * Revert to default priority/policy on fork if requested. + */ + if (unlikely(p->sched_reset_on_fork)) { + if (p->policy == SCHED_FIFO || p->policy == SCHED_RR || p-> policy == SCHED_ISO) { + /* + * __setscheduler() zeroes timer_slack_ns for rt tasks, + * so restore it when demoting back to SCHED_NORMAL, + * otherwise the child inherits zero slack forever. + */ + if (has_rt_policy(p)) + p->timer_slack_ns = p->default_timer_slack_ns; + p->policy = SCHED_NORMAL; + p->normal_prio = normal_prio(p); + } + + if (PRIO_TO_NICE(p->static_prio) < 0) { + p->static_prio = NICE_TO_PRIO(0); + p->normal_prio = p->static_prio; + } + + /* + * We don't need the reset flag anymore after the fork. It has + * fulfilled its duty: + */ + p->sched_reset_on_fork = 0; + } + + /* + * Silence PROVE_RCU. + */ + raw_spin_lock_irqsave(&p->pi_lock, flags); + rseq_sched_set_ids_changed(p); + set_task_cpu(p, smp_processor_id()); + raw_spin_unlock_irqrestore(&p->pi_lock, flags); + +#ifdef CONFIG_SCHED_INFO + if (unlikely(sched_info_on())) + memset(&p->sched_info, 0, sizeof(p->sched_info)); +#endif + init_task_preempt_count(p); + + return 0; +} + +void sched_post_fork(struct task_struct *p) +{ +} + +#ifdef CONFIG_SCHEDSTATS + +DEFINE_STATIC_KEY_FALSE(sched_schedstats); +static bool __initdata __sched_schedstats = false; + +static void set_schedstats(bool enabled) +{ + if (enabled) + static_branch_enable(&sched_schedstats); + else + static_branch_disable(&sched_schedstats); +} + +void force_schedstat_enabled(void) +{ + if (!schedstat_enabled()) { + pr_info("kernel profiling enabled schedstats, disable via kernel.sched_schedstats.\n"); + static_branch_enable(&sched_schedstats); + } +} + +static int __init setup_schedstats(char *str) +{ + int ret = 0; + if (!str) + goto out; + + /* + * This code is called before jump labels have been set up, so we can't + * change the static branch directly just yet. Instead set a temporary + * variable so init_schedstats() can do it later. + */ + if (!strcmp(str, "enable")) { + __sched_schedstats = true; + ret = 1; + } else if (!strcmp(str, "disable")) { + __sched_schedstats = false; + ret = 1; + } +out: + if (!ret) + pr_warn("Unable to parse schedstats=\n"); + + return ret; +} +__setup("schedstats=", setup_schedstats); + +static void __init init_schedstats(void) +{ + set_schedstats(__sched_schedstats); +} + +#ifdef CONFIG_SYSCTL +static int sysctl_schedstats(const struct ctl_table *table, int write, void *buffer, + size_t *lenp, loff_t *ppos) +{ + struct ctl_table t; + int err; + int state = static_branch_likely(&sched_schedstats); + + if (write && !capable(CAP_SYS_ADMIN)) + return -EPERM; + + t = *table; + t.data = &state; + err = proc_dointvec_minmax(&t, write, buffer, lenp, ppos); + if (err < 0) + return err; + if (write) + set_schedstats(state); + return err; +} +#endif /* CONFIG_SYSCTL */ +#else /* !CONFIG_SCHEDSTATS */ +static inline void init_schedstats(void) {} +#endif /* CONFIG_SCHEDSTATS */ + +#ifdef CONFIG_SYSCTL +/* + * MuQSS tunables used to live in kernel/sysctl.c. Mainline moved scheduler + * sysctls next to their implementation via register_sysctl_init(), so register + * ours the same way. CFS/rt/fair knobs are not present under MuQSS. + */ +static const struct ctl_table muqss_sysctls[] = { + { + .procname = "rr_interval", + .data = &rr_interval, + .maxlen = sizeof(int), + .mode = 0644, + .proc_handler = proc_dointvec_minmax, + .extra1 = SYSCTL_ONE, + .extra2 = SYSCTL_ONE_THOUSAND, + }, + { + .procname = "interactive", + .data = &sched_interactive, + .maxlen = sizeof(int), + .mode = 0644, + .proc_handler = proc_dointvec_minmax, + .extra1 = SYSCTL_ZERO, + .extra2 = SYSCTL_ONE, + }, + { + .procname = "iso_cpu", + .data = &sched_iso_cpu, + .maxlen = sizeof(int), + .mode = 0644, + .proc_handler = proc_dointvec_minmax, + .extra1 = SYSCTL_ZERO, + .extra2 = SYSCTL_ONE_HUNDRED, + }, + { + .procname = "yield_type", + .data = &sched_yield_type, + .maxlen = sizeof(int), + .mode = 0644, + .proc_handler = proc_dointvec_minmax, + .extra1 = SYSCTL_ZERO, + .extra2 = SYSCTL_TWO, + }, +#ifdef CONFIG_SCHEDSTATS + { + .procname = "sched_schedstats", + .data = NULL, + .maxlen = sizeof(unsigned int), + .mode = 0644, + .proc_handler = sysctl_schedstats, + .extra1 = SYSCTL_ZERO, + .extra2 = SYSCTL_ONE, + }, +#endif +}; + +static int __init muqss_sysctl_init(void) +{ + register_sysctl_init("kernel", muqss_sysctls); + return 0; +} +late_initcall(muqss_sysctl_init); +#endif /* CONFIG_SYSCTL */ + +static void update_cpu_clock_switch(struct rq *rq, struct task_struct *p); + +static void account_task_cpu(struct rq *rq, struct task_struct *p) +{ + update_clocks(rq); + /* This isn't really a context switch but accounting is the same */ + update_cpu_clock_switch(rq, p); + p->last_ran = rq->niffies; +} + +bool sched_smp_initialized __read_mostly; + +/* + * High-resolution timeslice expiry (MuQSS counterpart of mainline hrtick). + * + * Reprogramming a oneshot clockevent from set_rq_task() on every context + * switch under the rq lock races with the tick/timer softirq path on SMP + * and starves TIMER_SOFTIRQ (RCU "timer wakeup didn't happen"). Mirror + * mainline: HARD + LAZY_REARM setup, needs_rearm 5us threshold, and defer + * the actual start/cancel until hrexpiry_schedule_exit() after the pick. + */ +#ifdef CONFIG_HIGH_RES_TIMERS + +enum { + HREXPIRE_SCHED_NONE = 0, + HREXPIRE_SCHED_DEFER = BIT(1), + HREXPIRE_SCHED_START = BIT(2), + HREXPIRE_SCHED_REARM_HRTIMER = BIT(3), +}; + +static inline int hrexpiry_enabled(struct rq *rq) +{ + /* + * 5.12 used hrtimer_is_hres_active(); 7.1 exposes the same idea as + * hrtimer_resolution != LOW_RES_NSEC once highres has switched on. + */ + return hrtimer_resolution != LOW_RES_NSEC; +} + +static inline bool hrexpiry_needs_rearm(struct hrtimer *timer, ktime_t expires) +{ + /* + * Queued is false when not started or the callback is running. If + * already queued, only reprogram when the expiry moves substantially. + */ + return !hrtimer_is_queued(timer) || + abs(expires - hrtimer_get_expires(timer)) > 5000; +} + +static void hrexpiry_cond_restart(struct rq *rq) +{ + struct hrtimer *timer = &rq->hrexpiry_timer; + ktime_t time = rq->hrexpiry_time; + + if (hrexpiry_needs_rearm(timer, time)) + hrtimer_start(timer, time, HRTIMER_MODE_ABS_PINNED_HARD); +} + +/* + * Remote start IPI — wake_up_new_task may shorten a parent on another CPU. + * Runs hardirq/IPI context; take the rq lock like mainline __hrtick_start. + */ +static void __hrexpiry_start(void *arg) +{ + struct rq *rq = arg; + unsigned long flags; + + raw_spin_lock_irqsave(rq->lock, flags); + hrexpiry_cond_restart(rq); + raw_spin_unlock_irqrestore(rq->lock, flags); +} + +static inline void hrexpiry_clear(struct rq *rq) +{ + if (!hrexpiry_enabled(rq)) + return; + + /* + * Inside __schedule() only drop a pending deferred start; the actual + * cancel happens once in hrexpiry_schedule_exit(). + */ + if (rq->hrexpiry_sched) { + rq->hrexpiry_sched &= ~HREXPIRE_SCHED_START; + return; + } + + hrtimer_try_to_cancel(&rq->hrexpiry_timer); +} + +/* + * High-resolution time_slice expiry. + * Runs from hardirq context with interrupts disabled. + */ +static enum hrtimer_restart hrexpiry(struct hrtimer *timer) +{ + struct rq *rq = container_of(timer, struct rq, hrexpiry_timer); + struct task_struct *p; + + /* This can happen during CPU hotplug / resume */ + if (unlikely(cpu_of(rq) != smp_processor_id())) + goto out; + + /* + * Local CPU only; no rq lock. Force a reschedule when the slice + * expires — __schedule() will pick the next deadline task. + */ + p = rq->curr; + p->time_slice = 0; + __set_tsk_resched(p); +out: + return HRTIMER_NORESTART; +} + +/* + * Called with irqs disabled under the rq lock (set_rq_task / fork path). + * May target a remote rq — then arm via CSD like mainline hrtick_start. + */ +static void hrexpiry_start(struct rq *rq, u64 delay) +{ + s64 delta; + + if (!hrexpiry_enabled(rq)) + return; + + /* Slices < 10us are not useful and can DoS the timer subsystem. */ + delta = max_t(s64, delay, 10000LL); + + /* + * Mid-schedule: note the delay and let hrexpiry_schedule_exit() + * program the clockevent once. + */ + if (rq->hrexpiry_sched) { + rq->hrexpiry_sched |= HREXPIRE_SCHED_START; + rq->hrexpiry_delay = delta; + return; + } + + rq->hrexpiry_time = ktime_add_ns(ktime_get(), delta); + if (!hrexpiry_needs_rearm(&rq->hrexpiry_timer, rq->hrexpiry_time)) + return; + + if (rq == this_rq()) + hrtimer_start(&rq->hrexpiry_timer, rq->hrexpiry_time, + HRTIMER_MODE_ABS_PINNED_HARD); + else + smp_call_function_single_async(cpu_of(rq), &rq->hrexpiry_csd); +} + +static inline void hrexpiry_schedule_enter(struct rq *rq) +{ + rq->hrexpiry_sched = HREXPIRE_SCHED_DEFER; + if (hrtimer_test_and_clear_rearm_deferred()) + rq->hrexpiry_sched |= HREXPIRE_SCHED_REARM_HRTIMER; +} + +static inline void hrexpiry_schedule_exit(struct rq *rq) +{ + if (rq->hrexpiry_sched & HREXPIRE_SCHED_START) { + rq->hrexpiry_time = ktime_add_ns(ktime_get(), rq->hrexpiry_delay); + hrexpiry_cond_restart(rq); + } else if (rq->curr == rq->idle || rq->curr->policy == SCHED_FIFO) { + /* + * No slice timer needed. Local CPU, IRQs off: the HARD + * callback cannot be running, so cancel is safe. + */ + if (hrtimer_is_queued(&rq->hrexpiry_timer)) + hrtimer_cancel(&rq->hrexpiry_timer); + } + + if (rq->hrexpiry_sched & HREXPIRE_SCHED_REARM_HRTIMER) + __hrtimer_rearm_deferred(); + + rq->hrexpiry_sched = HREXPIRE_SCHED_NONE; +} + +static void init_rq_hrexpiry(struct rq *rq) +{ + INIT_CSD(&rq->hrexpiry_csd, __hrexpiry_start, rq); + rq->hrexpiry_sched = HREXPIRE_SCHED_NONE; + hrtimer_setup(&rq->hrexpiry_timer, hrexpiry, CLOCK_MONOTONIC, + HRTIMER_MODE_REL_HARD | HRTIMER_MODE_LAZY_REARM); +} + +#else /* !CONFIG_HIGH_RES_TIMERS */ + +static inline int hrexpiry_enabled(struct rq *rq) +{ + return 0; +} +static inline void hrexpiry_clear(struct rq *rq) { } +static inline void hrexpiry_start(struct rq *rq, u64 delay) { } +static inline void hrexpiry_schedule_enter(struct rq *rq) { } +static inline void hrexpiry_schedule_exit(struct rq *rq) { } +static inline void init_rq_hrexpiry(struct rq *rq) { } + +#endif /* CONFIG_HIGH_RES_TIMERS */ + +static inline int rq_dither(struct rq *rq) +{ + if (!hrexpiry_enabled(rq)) + return HALF_JIFFY_US; + return 0; +} + +/* + * wake_up_new_task - wake up a newly created task for the first time. + * + * This function will do some initial scheduler statistics housekeeping + * that must be done for every newly created context, then puts the task + * on the runqueue and wakes it. + */ +void wake_up_new_task(struct task_struct *p) +{ + struct task_struct *parent, *rq_curr; + struct rq *rq, *new_rq; + unsigned long flags; + + parent = p->parent; + + raw_spin_lock_irqsave(&p->pi_lock, flags); + p->__state = TASK_RUNNING; + /* Task_rq can't change yet on a new task */ + new_rq = rq = task_rq(p); + if (unlikely(needs_other_cpu(p, task_cpu(p)))) { + set_task_cpu(p, valid_task_cpu(p)); + new_rq = task_rq(p); + } + + double_rq_lock(rq, new_rq); + rq_curr = rq->curr; + + /* + * Make sure we do not leak PI boosting priority to the child. + */ + p->prio = rq_curr->normal_prio; + + trace_sched_wakeup_new(p); + + /* + * Share the timeslice between parent and child, thus the + * total amount of pending timeslices in the system doesn't change, + * resulting in more scheduling fairness. If it's negative, it won't + * matter since that's the same as being 0. rq->rq_deadline is only + * modified within schedule() so it is always equal to + * current->deadline. + */ + account_task_cpu(rq, rq_curr); + p->last_ran = rq_curr->last_ran; + if (likely(rq_curr->policy != SCHED_FIFO)) { + rq_curr->time_slice /= 2; + if (rq_curr->time_slice < RESCHED_US) { + /* + * Forking task has run out of timeslice. Reschedule it and + * start its child with a new time slice and deadline. The + * child will end up running first because its deadline will + * be slightly earlier. + */ + __set_tsk_resched(rq_curr); + time_slice_expired(p, new_rq); + if (suitable_idle_cpus(p)) + resched_best_idle(p, task_cpu(p)); + else if (unlikely(rq != new_rq)) + try_preempt(p, new_rq); + } else { + p->time_slice = rq_curr->time_slice; + if (rq_curr == parent && rq == new_rq && !suitable_idle_cpus(p)) { + /* + * The VM isn't cloned, so we're in a good position to + * do child-runs-first in anticipation of an exec. This + * usually avoids a lot of COW overhead. + */ + __set_tsk_resched(rq_curr); + } else { + /* + * Adjust the hrexpiry since rq_curr will keep + * running and its timeslice has been shortened. + */ + hrexpiry_start(rq, US_TO_NS(rq_curr->time_slice)); + try_preempt(p, new_rq); + } + } + } else { + time_slice_expired(p, new_rq); + try_preempt(p, new_rq); + } + activate_task(new_rq, p, 0); + double_rq_unlock(rq, new_rq); + raw_spin_unlock_irqrestore(&p->pi_lock, flags); +} + +#ifdef CONFIG_PREEMPT_NOTIFIERS + +static DEFINE_STATIC_KEY_FALSE(preempt_notifier_key); + +void preempt_notifier_inc(void) +{ + static_branch_inc(&preempt_notifier_key); +} +EXPORT_SYMBOL_GPL(preempt_notifier_inc); + +void preempt_notifier_dec(void) +{ + static_branch_dec(&preempt_notifier_key); +} +EXPORT_SYMBOL_GPL(preempt_notifier_dec); + +/** + * preempt_notifier_register - tell me when current is being preempted & rescheduled + * @notifier: notifier struct to register + */ +void preempt_notifier_register(struct preempt_notifier *notifier) +{ + if (!static_branch_unlikely(&preempt_notifier_key)) + WARN(1, "registering preempt_notifier while notifiers disabled\n"); + + hlist_add_head(¬ifier->link, ¤t->preempt_notifiers); +} +EXPORT_SYMBOL_GPL(preempt_notifier_register); + +/** + * preempt_notifier_unregister - no longer interested in preemption notifications + * @notifier: notifier struct to unregister + * + * This is *not* safe to call from within a preemption notifier. + */ +void preempt_notifier_unregister(struct preempt_notifier *notifier) +{ + hlist_del(¬ifier->link); +} +EXPORT_SYMBOL_GPL(preempt_notifier_unregister); + +static void __fire_sched_in_preempt_notifiers(struct task_struct *curr) +{ + struct preempt_notifier *notifier; + + hlist_for_each_entry(notifier, &curr->preempt_notifiers, link) + notifier->ops->sched_in(notifier, raw_smp_processor_id()); +} + +static __always_inline void fire_sched_in_preempt_notifiers(struct task_struct *curr) +{ + if (static_branch_unlikely(&preempt_notifier_key)) + __fire_sched_in_preempt_notifiers(curr); +} + +static void +__fire_sched_out_preempt_notifiers(struct task_struct *curr, + struct task_struct *next) +{ + struct preempt_notifier *notifier; + + hlist_for_each_entry(notifier, &curr->preempt_notifiers, link) + notifier->ops->sched_out(notifier, next); +} + +static __always_inline void +fire_sched_out_preempt_notifiers(struct task_struct *curr, + struct task_struct *next) +{ + if (static_branch_unlikely(&preempt_notifier_key)) + __fire_sched_out_preempt_notifiers(curr, next); +} + +#else /* !CONFIG_PREEMPT_NOTIFIERS */ + +static inline void fire_sched_in_preempt_notifiers(struct task_struct *curr) +{ +} + +static inline void +fire_sched_out_preempt_notifiers(struct task_struct *curr, + struct task_struct *next) +{ +} + +#endif /* CONFIG_PREEMPT_NOTIFIERS */ + +static inline void prepare_task(struct task_struct *next) +{ + /* + * Claim the task as running, we do this before switching to it + * such that any running task will have this set. + * + * See the ttwu() WF_ON_CPU case and its ordering comment. + */ + WRITE_ONCE(next->on_cpu, 1); +} + +static inline void finish_task(struct task_struct *prev) +{ + /* + * This must be the very last reference to @prev from this CPU. After + * p->on_cpu is cleared, the task can be moved to a different CPU. We + * must ensure this doesn't happen until the switch is completely + * finished. + * + * In particular, the load of prev->__state in finish_task_switch() must + * happen before this. + * + * Pairs with the smp_cond_load_acquire() in try_to_wake_up() and + * __task_needs_rq_lock(). + */ + smp_store_release(&prev->on_cpu, 0); +} + +static inline void +prepare_lock_switch(struct rq *rq, struct task_struct *next) +{ + /* + * Since the runqueue lock will be released by the next + * task (which is an invalid locking op but in the case + * of the scheduler it's an obvious special-case), so we + * do an early lockdep release here: + */ + spin_release(&rq->lock->dep_map, _THIS_IP_); +#ifdef CONFIG_DEBUG_SPINLOCK + /* this is a valid case when another task releases the spinlock */ + rq->lock->owner = next; +#endif +} + +static inline void finish_lock_switch(struct rq *rq, struct task_struct *prev) +{ + /* + * If we are tracking spinlock dependencies then we have to + * fix up the runqueue lock - which gets 'carried over' from + * prev into current: + */ + spin_acquire(&rq->lock->dep_map, 0, 0, _THIS_IP_); + +#ifdef CONFIG_SMP + /* + * If prev was marked as migrating to another CPU in return_task, drop + * the local runqueue lock but leave interrupts disabled and grab the + * remote lock we're migrating it to before enabling them. + */ + if (unlikely(task_on_rq_migrating(prev))) { + unsigned int migrate; + + /* + * Program/cancel hrexpiry on this CPU before dropping its + * rq lock; after the unlock `rq` may become the remote one. + */ + hrexpiry_schedule_exit(rq); + sched_info_dequeue(rq, prev); + /* + * We move the ownership of prev to the new cpu now. Note that + * this does not lock ttwu out: pointing task_cpu() at wake_cpu + * below sends a concurrent ttwu_runnable() to the *new* + * runqueue's lock rather than the one dropped here, and + * __task_rq_lock() does not spin on task_on_rq_migrating() the + * way mainline's does. See the re-check before enqueueing. + * + * This bypasses set_task_cpu(), so any PSI state prev is still + * counted for has to be moved by hand, exactly as that does. + * psi_sched_switch() saw prev off the CPU as a sleep, since it + * is no longer queued, so a task in iowait is carrying + * TSK_IOWAIT here and enqueue_task() below would clear it on + * the new CPU that never counted it. + */ + migrate = psi_migrate_begin(prev); + task_thread_info(prev)->cpu = prev->wake_cpu; + psi_migrate_end(prev, migrate); + raw_spin_unlock(rq->lock); + + raw_spin_lock(&prev->pi_lock); + rq = __task_rq_lock(prev, NULL); + /* + * Complete the handover only while it is still ours to + * complete. ttwu() reaches prev here despite the comment + * above: ttwu_runnable() only accepts TASK_ON_RQ_QUEUED, so a + * wakeup that finds prev still TASK_ON_RQ_MIGRATING falls + * through it and enqueues prev itself. Once it has, prev can be + * picked and run on another CPU, and take_task() empties its + * skiplist node again. + * + * task_queued() only asks whether that node is linked, so at + * that point it reads "prev still needs enqueueing" when it + * means "prev is running elsewhere", and puts a task that is on + * a CPU back on a runqueue for a second CPU to pick up. One + * task then runs on two CPUs off one stack, which shows up + * downstream as skiplist corruption, PSI counting the task + * twice, and a scribbled kernel stack. + * + * on_rq is what actually tracks the handover - return_task() + * set TASK_ON_RQ_MIGRATING and whoever takes prev over clears + * it - and pi_lock, held here and taken by ttwu() before it + * does anything, serialises the two. + */ + if (likely(task_on_rq_migrating(prev))) { + enqueue_task(rq, prev, 0); + prev->on_rq = TASK_ON_RQ_QUEUED; + /* Wake up the CPU if it's not already running */ + resched_if_idle(rq); + } + raw_spin_unlock(&prev->pi_lock); + raw_spin_unlock_irq(rq->lock); + return; + } +#endif + hrexpiry_schedule_exit(rq); + raw_spin_unlock_irq(rq->lock); +} + +#ifndef prepare_arch_switch +# define prepare_arch_switch(next) do { } while (0) +#endif +#ifndef finish_arch_switch +# define finish_arch_switch(prev) do { } while (0) +#endif +#ifndef finish_arch_post_lock_switch +# define finish_arch_post_lock_switch() do { } while (0) +#endif + +static inline void kmap_local_sched_out(void) +{ +#ifdef CONFIG_KMAP_LOCAL + if (unlikely(current->kmap_ctrl.idx)) + __kmap_local_sched_out(); +#endif +} + +static inline void kmap_local_sched_in(void) +{ +#ifdef CONFIG_KMAP_LOCAL + if (unlikely(current->kmap_ctrl.idx)) + __kmap_local_sched_in(); +#endif +} + +/** + * prepare_task_switch - prepare to switch tasks + * @rq: the runqueue preparing to switch + * @next: the task we are going to switch to. + * + * This is called with the rq lock held and interrupts off. It must + * be paired with a subsequent finish_task_switch after the context + * switch. + * + * prepare_task_switch sets up locking and calls architecture specific + * hooks. + */ +static inline void +prepare_task_switch(struct rq *rq, struct task_struct *prev, + struct task_struct *next) +{ + kcov_prepare_switch(prev); + sched_info_switch(rq, prev, next); + perf_event_task_sched_out(prev, next); + /* + * rseq_preempt(prev) became rseq_sched_switch_event(next): rseq is now + * told about the task being scheduled *in*. + */ + rseq_sched_switch_event(next); + fire_sched_out_preempt_notifiers(prev, next); + kmap_local_sched_out(); + prepare_task(next); + prepare_arch_switch(next); +} + +/** + * finish_task_switch - clean up after a task-switch + * @rq: runqueue associated with task-switch + * @prev: the thread we just switched away from. + * + * finish_task_switch must be called after the context switch, paired + * with a prepare_task_switch call before the context switch. + * finish_task_switch will reconcile locking set up by prepare_task_switch, + * and do any other architecture-specific cleanup actions. + * + * Note that we may have delayed dropping an mm in context_switch(). If + * so, we finish that here outside of the runqueue lock. (Doing it + * with the lock held can cause deadlocks; see schedule() for + * details.) + * + * The context switch have flipped the stack from under us and restored the + * local variables which were saved when this task called schedule() in the + * past. prev == current is still correct but we need to recalculate this_rq + * because prev may have moved to another CPU. + */ +static void finish_task_switch(struct task_struct *prev) + __releases(rq->lock) +{ + struct rq *rq = this_rq(); + struct mm_struct *mm = rq->prev_mm; + long prev_state; + + /* + * The previous task will have left us with a preempt_count of 2 + * because it left us after: + * + * schedule() + * preempt_disable(); // 1 + * __schedule() + * raw_spin_lock_irq(rq->lock) // 2 + * + * Also, see FORK_PREEMPT_COUNT. + */ + if (WARN_ONCE(preempt_count() != 2*PREEMPT_DISABLE_OFFSET, + "corrupted preempt_count: %s/%d/0x%x\n", + current->comm, current->pid, preempt_count())) + preempt_count_set(FORK_PREEMPT_COUNT); + + rq->prev_mm = NULL; + + /* + * A task struct has one reference for the use as "current". + * If a task dies, then it sets TASK_DEAD in tsk->__state and calls + * schedule one last time. The schedule call will never return, and + * the scheduled task must drop that reference. + * + * We must observe prev->__state before clearing prev->on_cpu (in + * finish_task), otherwise a concurrent wakeup can get prev + * running on another CPU and we could rave with its RUNNING -> DEAD + * transition, resulting in a double drop. + */ + prev_state = prev->__state; + vtime_task_switch(prev); + perf_event_task_sched_in(prev, current); + finish_task(prev); + finish_lock_switch(rq, prev); + finish_arch_post_lock_switch(); + kcov_finish_switch(current); + /* + * kmap_local_sched_out() is invoked with rq::lock held and + * interrupts disabled. There is no requirement for that, but the + * sched out code does not have an interrupt enabled section. + * Restoring the maps on sched in does not require interrupts being + * disabled either. + */ + kmap_local_sched_in(); + + /* + * Any cached block-layer timestamp (plug->cur_ktime) is stale now, + * invalidate it. + */ + blk_plug_invalidate_ts(); + + fire_sched_in_preempt_notifiers(current); + /* + * When switching through a kernel thread, the loop in + * membarrier_{private,global}_expedited() may have observed that + * kernel thread and not issued an IPI. It is therefore possible to + * schedule between user->kernel->user threads without passing though + * switch_mm(). Membarrier requires a barrier after storing to + * rq->curr, before returning to userspace, so provide them here: + * + * - a full memory barrier for {PRIVATE,GLOBAL}_EXPEDITED, implicitly + * provided by mmdrop_lazy_tlb(), + * - a sync_core for SYNC_CORE. + */ + if (mm) { + membarrier_mm_sync_core_before_usermode(mm); + mmdrop_lazy_tlb_sched(mm); + } + if (unlikely(prev_state == TASK_DEAD)) { + /* + * Remove function-return probe instances associated with this + * task and put them back on the free list. + */ + kprobe_flush_task(prev); + + /* + * Unlink from css_set after the final switch so rmdir can + * complete. Must precede put_task_struct_rcu_user(). + */ + cgroup_task_dead(prev); + + /* Task is done with its stack. */ + put_task_stack(prev); + + put_task_struct_rcu_user(prev); + } +} + +/** + * schedule_tail - first thing a freshly forked thread must call. + * @prev: the thread we just switched away from. + */ +asmlinkage __visible void schedule_tail(struct task_struct *prev) +{ + /* + * New tasks start with FORK_PREEMPT_COUNT, see there and + * finish_task_switch() for details. + * + * finish_task_switch() will drop rq->lock() and lower preempt_count + * and the preempt_enable() will end up enabling preemption (on + * PREEMPT_COUNT kernels). + */ + + finish_task_switch(prev); + preempt_enable(); + + if (current->set_child_tid) + put_user(task_pid_vnr(current), current->set_child_tid); + + calculate_sigpending(); +} + +/* + * context_switch - switch to the new MM and the new thread's register state. + */ +static __always_inline void +context_switch(struct rq *rq, struct task_struct *prev, + struct task_struct *next) +{ + prepare_task_switch(rq, prev, next); + + /* + * For paravirt, this is coupled with an exit in switch_to to + * combine the page table reload and the switch backend into + * one hypercall. + */ + arch_start_context_switch(prev); + + /* + * kernel -> kernel lazy + transfer active + * user -> kernel lazy + mmgrab_lazy_tlb() active + * + * kernel -> user switch + mmdrop_lazy_tlb() active + * user -> user switch + */ + if (!next->mm) { // to kernel + enter_lazy_tlb(prev->active_mm, next); + + next->active_mm = prev->active_mm; + if (prev->mm) // from user + mmgrab_lazy_tlb(prev->active_mm); + else + prev->active_mm = NULL; + } else { // to user + membarrier_switch_mm(rq, prev->active_mm, next->mm); + /* + * sys_membarrier() requires an smp_mb() between setting + * rq->curr / membarrier_switch_mm() and returning to userspace. + * + * The below provides this either through switch_mm(), or in + * case 'prev->active_mm == next->mm' through + * finish_task_switch()'s mmdrop_lazy_tlb(). + */ + switch_mm_irqs_off(prev->active_mm, next->mm, next); + lru_gen_use_mm(next->mm); + + if (!prev->mm) { // from kernel + /* will mmdrop_lazy_tlb() in finish_task_switch(). */ + rq->prev_mm = prev->active_mm; + prev->active_mm = NULL; + } + } + prepare_lock_switch(rq, next); + + /* Here we just switch the register state and the stack. */ + switch_to(prev, next, prev); + barrier(); + + finish_task_switch(prev); +} + +/* + * nr_running, nr_uninterruptible and nr_context_switches: + * + * externally visible scheduler statistics: current number of runnable + * threads, total number of context switches performed since bootup. + */ +unsigned int nr_running(void) +{ + unsigned long i, sum = 0; + + for_each_online_cpu(i) + sum += cpu_rq(i)->nr_running; + + return sum; +} + +static unsigned long nr_uninterruptible(void) +{ + unsigned long i, sum = 0; + + for_each_online_cpu(i) + sum += cpu_rq(i)->nr_uninterruptible; + + return sum; +} + +/* + * Check if only the current task is running on the CPU. + * + * Caution: this function does not check that the caller has disabled + * preemption, thus the result might have a time-of-check-to-time-of-use + * race. The caller is responsible to use it correctly, for example: + * + * - from a non-preemptible section (of course) + * + * - from a thread that is bound to a single CPU + * + * - in a loop with very short iterations (e.g. a polling loop) + */ +bool single_task_running(void) +{ + if (rq_load(raw_rq()) == 1) + return true; + else + return false; +} +EXPORT_SYMBOL(single_task_running); + +unsigned long long nr_context_switches(void) +{ + int cpu; + unsigned long long sum = 0; + + for_each_possible_cpu(cpu) + sum += cpu_rq(cpu)->nr_switches; + + return sum; +} + +/* + * Consumers of these two interfaces, like for example the cpufreq menu + * governor are using nonsensical data. Boosting frequency for a CPU that has + * IO-wait which might not even end up running the task when it does become + * runnable. + */ + +unsigned int nr_iowait_cpu(int cpu) +{ + return atomic_read(&cpu_rq(cpu)->nr_iowait); +} + +/* + * IO-wait accounting, and how it's mostly bollocks (on SMP). + * + * The idea behind IO-wait account is to account the idle time that we could + * have spend running if it were not for IO. That is, if we were to improve the + * storage performance, we'd have a proportional reduction in IO-wait time. + * + * This all works nicely on UP, where, when a task blocks on IO, we account + * idle time as IO-wait, because if the storage were faster, it could've been + * running and we'd not be idle. + * + * This has been extended to SMP, by doing the same for each CPU. This however + * is broken. + * + * Imagine for instance the case where two tasks block on one CPU, only the one + * CPU will have IO-wait accounted, while the other has regular idle. Even + * though, if the storage were faster, both could've ran at the same time, + * utilising both CPUs. + * + * This means, that when looking globally, the current IO-wait accounting on + * SMP is a lower bound, by reason of under accounting. + * + * Worse, since the numbers are provided per CPU, they are sometimes + * interpreted per CPU, and that is nonsensical. A blocked task isn't strictly + * associated with any one particular CPU, it can wake to another CPU than it + * blocked on. This means the per CPU IO-wait number is meaningless. + * + * Task CPU affinities can make all that even more 'interesting'. + */ + +unsigned int nr_iowait(void) +{ + unsigned long cpu, sum = 0; + + for_each_possible_cpu(cpu) + sum += nr_iowait_cpu(cpu); + + return sum; +} + +static unsigned long nr_active(void) +{ + return nr_running() + nr_uninterruptible(); +} + +/* Variables and functions for calc_load */ +static unsigned long calc_load_update; +unsigned long avenrun[3]; +EXPORT_SYMBOL(avenrun); + +/** + * get_avenrun - get the load average array + * @loads: pointer to dest load array + * @offset: offset to add + * @shift: shift count to shift the result left + * + * These values are estimates at best, so no need for locking. + */ +void get_avenrun(unsigned long *loads, unsigned long offset, int shift) +{ + loads[0] = (avenrun[0] + offset) << shift; + loads[1] = (avenrun[1] + offset) << shift; + loads[2] = (avenrun[2] + offset) << shift; +} + +/* + * calc_load - update the avenrun load estimates every LOAD_FREQ seconds. + */ +void calc_global_load(void) +{ + long active; + + if (time_before(jiffies, READ_ONCE(calc_load_update))) + return; + active = nr_active() * FIXED_1; + + avenrun[0] = calc_load(avenrun[0], EXP_1, active); + avenrun[1] = calc_load(avenrun[1], EXP_5, active); + avenrun[2] = calc_load(avenrun[2], EXP_15, active); + + calc_load_update = jiffies + LOAD_FREQ; +} + +/** + * fixed_power_int - compute: x^n, in O(log n) time + * + * @x: base of the power + * @frac_bits: fractional bits of @x + * @n: power to raise @x to. + * + * By exploiting the relation between the definition of the natural power + * function: x^n := x*x*...*x (x multiplied by itself for n times), and + * the binary encoding of numbers used by computers: n := \Sum n_i * 2^i, + * (where: n_i \elem {0, 1}, the binary vector representing n), + * we find: x^n := x^(\Sum n_i * 2^i) := \Prod x^(n_i * 2^i), which is + * of course trivially computable in O(log_2 n), the length of our binary + * vector. + */ +static unsigned long +fixed_power_int(unsigned long x, unsigned int frac_bits, unsigned int n) +{ + unsigned long result = 1UL << frac_bits; + + if (n) { + for (;;) { + if (n & 1) { + result *= x; + result += 1UL << (frac_bits - 1); + result >>= frac_bits; + } + n >>= 1; + if (!n) + break; + x *= x; + x += 1UL << (frac_bits - 1); + x >>= frac_bits; + } + } + + return result; +} + +/* + * a1 = a0 * e + a * (1 - e) + * + * a2 = a1 * e + a * (1 - e) + * = (a0 * e + a * (1 - e)) * e + a * (1 - e) + * = a0 * e^2 + a * (1 - e) * (1 + e) + * + * a3 = a2 * e + a * (1 - e) + * = (a0 * e^2 + a * (1 - e) * (1 + e)) * e + a * (1 - e) + * = a0 * e^3 + a * (1 - e) * (1 + e + e^2) + * + * ... + * + * an = a0 * e^n + a * (1 - e) * (1 + e + ... + e^n-1) [1] + * = a0 * e^n + a * (1 - e) * (1 - e^n)/(1 - e) + * = a0 * e^n + a * (1 - e^n) + * + * [1] application of the geometric series: + * + * n 1 - x^(n+1) + * S_n := \Sum x^i = ------------- + * i=0 1 - x + */ +unsigned long +calc_load_n(unsigned long load, unsigned long exp, + unsigned long active, unsigned int n) +{ + return calc_load(load, fixed_power_int(exp, FSHIFT, n), active); +} + +DEFINE_PER_CPU(struct kernel_stat, kstat); +DEFINE_PER_CPU(struct kernel_cpustat, kernel_cpustat) = { +#ifdef CONFIG_NO_HZ_COMMON + .idle_sleeptime_seq = SEQCNT_ZERO(kernel_cpustat.idle_sleeptime_seq) +#endif +}; + +EXPORT_PER_CPU_SYMBOL(kstat); +EXPORT_PER_CPU_SYMBOL(kernel_cpustat); + +#ifdef CONFIG_PARAVIRT +static inline u64 steal_ticks(u64 steal) +{ + if (unlikely(steal > NSEC_PER_SEC)) + return div_u64(steal, TICK_NSEC); + + return __iter_div_u64_rem(steal, TICK_NSEC, &steal); +} +#endif + +#ifndef nsecs_to_cputime +# define nsecs_to_cputime(__nsecs) nsecs_to_jiffies(__nsecs) +#endif + +/* + * On each tick, add the number of nanoseconds to the unbanked variables and + * once one tick's worth has accumulated, account it allowing for accurate + * sub-tick accounting and totals. Use the TICK_APPROX_NS to match the way we + * deduct nanoseconds. + */ +static void pc_idle_time(struct rq *rq, struct task_struct *idle, unsigned long ns) +{ + u64 *cpustat = kcpustat_this_cpu->cpustat; + unsigned long ticks; + + if (atomic_read(&rq->nr_iowait) > 0) { + rq->iowait_ns += ns; + if (rq->iowait_ns >= JIFFY_NS) { + ticks = NS_TO_JIFFIES(rq->iowait_ns); + cpustat[CPUTIME_IOWAIT] += (__force u64)TICK_APPROX_NS * ticks; + rq->iowait_ns %= JIFFY_NS; + } + } else { + rq->idle_ns += ns; + if (rq->idle_ns >= JIFFY_NS) { + ticks = NS_TO_JIFFIES(rq->idle_ns); + cpustat[CPUTIME_IDLE] += (__force u64)TICK_APPROX_NS * ticks; + rq->idle_ns %= JIFFY_NS; + } + } + acct_update_integrals(idle); +} + +static void pc_system_time(struct rq *rq, struct task_struct *p, + int hardirq_offset, unsigned long ns) +{ + u64 *cpustat = kcpustat_this_cpu->cpustat; + unsigned long ticks; + + p->stime_ns += ns; + if (p->stime_ns >= JIFFY_NS) { + ticks = NS_TO_JIFFIES(p->stime_ns); + p->stime_ns %= JIFFY_NS; + p->stime += (__force u64)TICK_APPROX_NS * ticks; + account_group_system_time(p, TICK_APPROX_NS * ticks); + } + p->sched_time += ns; + account_group_exec_runtime(p, ns); + + if (hardirq_count() - hardirq_offset) { + rq->irq_ns += ns; + if (rq->irq_ns >= JIFFY_NS) { + ticks = NS_TO_JIFFIES(rq->irq_ns); + cpustat[CPUTIME_IRQ] += (__force u64)TICK_APPROX_NS * ticks; + rq->irq_ns %= JIFFY_NS; + } + } else if (in_serving_softirq() || this_cpu_ksoftirqd() == p) { + /* + * ksoftirqd time does not get accounted in cpu_softirq_time, so + * it has to be handled separately here. Naming it explicitly + * also catches it between batches, where it has reenabled bh + * and in_serving_softirq() no longer holds - which is where the + * context switch that lands us here happens. + */ + rq->softirq_ns += ns; + if (rq->softirq_ns >= JIFFY_NS) { + ticks = NS_TO_JIFFIES(rq->softirq_ns); + cpustat[CPUTIME_SOFTIRQ] += (__force u64)TICK_APPROX_NS * ticks; + rq->softirq_ns %= JIFFY_NS; + } + } else { + rq->system_ns += ns; + if (rq->system_ns >= JIFFY_NS) { + ticks = NS_TO_JIFFIES(rq->system_ns); + cpustat[CPUTIME_SYSTEM] += (__force u64)TICK_APPROX_NS * ticks; + rq->system_ns %= JIFFY_NS; + } + } + acct_update_integrals(p); +} + +static void pc_user_time(struct rq *rq, struct task_struct *p, unsigned long ns) +{ + u64 *cpustat = kcpustat_this_cpu->cpustat; + unsigned long ticks; + + p->utime_ns += ns; + if (p->utime_ns >= JIFFY_NS) { + ticks = NS_TO_JIFFIES(p->utime_ns); + p->utime_ns %= JIFFY_NS; + p->utime += (__force u64)TICK_APPROX_NS * ticks; + account_group_user_time(p, TICK_APPROX_NS * ticks); + } + p->sched_time += ns; + account_group_exec_runtime(p, ns); + + if (task_nice(p) > 0 || idleprio_task(p)) { + rq->nice_ns += ns; + if (rq->nice_ns >= JIFFY_NS) { + ticks = NS_TO_JIFFIES(rq->nice_ns); + cpustat[CPUTIME_NICE] += (__force u64)TICK_APPROX_NS * ticks; + rq->nice_ns %= JIFFY_NS; + } + } else { + rq->user_ns += ns; + if (rq->user_ns >= JIFFY_NS) { + ticks = NS_TO_JIFFIES(rq->user_ns); + cpustat[CPUTIME_USER] += (__force u64)TICK_APPROX_NS * ticks; + rq->user_ns %= JIFFY_NS; + } + } + acct_update_integrals(p); +} + +/* + * This is called on clock ticks. + * Bank in p->sched_time the ns elapsed since the last tick or switch. + * CPU scheduler quota accounting is also performed here in microseconds. + */ +static void update_cpu_clock_tick(struct rq *rq, struct task_struct *p) +{ + s64 account_ns = rq->niffies - p->last_ran; + struct task_struct *idle = rq->idle; + + /* Accurate tick timekeeping */ + if (user_mode(get_irq_regs())) + pc_user_time(rq, p, account_ns); + else if (p != idle || (irq_count() != HARDIRQ_OFFSET)) { + pc_system_time(rq, p, HARDIRQ_OFFSET, account_ns); + } else + pc_idle_time(rq, idle, account_ns); + + /* time_slice accounting is done in usecs to avoid overflow on 32bit */ + if (p->policy != SCHED_FIFO && p != idle) + p->time_slice -= NS_TO_US(account_ns); + + p->last_ran = rq->niffies; +} + +/* + * This is called on context switches. + * Bank in p->sched_time the ns elapsed since the last tick or switch. + * CPU scheduler quota accounting is also performed here in microseconds. + */ +static void update_cpu_clock_switch(struct rq *rq, struct task_struct *p) +{ + s64 account_ns = rq->niffies - p->last_ran; + struct task_struct *idle = rq->idle; + + /* + * Accurate subtick timekeeping. There is no interrupt frame to sample + * here the way update_cpu_clock_tick() samples one: the entry code has + * already restored the previous value by the time irqentry_exit() + * reaches preempt_schedule_irq(), and a task that called schedule() + * itself never had a frame at all. What can be said for certain is that + * a task which never executes user code cannot have spent this interval + * in userspace, so charge all of those as system time. Without this the + * whole of every kernel thread's runtime that lands between two ticks + * is booked as user time, which is how irq threads, kworkers and + * ksoftirqd end up carrying utime they cannot possibly have accrued. + * + * A user task that blocks in a syscall genuinely does split its + * interval between the two modes, and knowing where it crossed over + * needs timestamped user/kernel transitions - vtime - that MuQSS + * deliberately does not take. Those keep the historical assumption. + */ + if (p == idle) + pc_idle_time(rq, idle, account_ns); + else if (is_user_task(p)) + pc_user_time(rq, p, account_ns); + else + pc_system_time(rq, p, 0, account_ns); + + /* time_slice accounting is done in usecs to avoid overflow on 32bit */ + if (p->policy != SCHED_FIFO && p != idle) + p->time_slice -= NS_TO_US(account_ns); +} + +#ifdef CONFIG_MUQSS_IOTIME +/* + * The CPU time current has consumed so far, for measuring how long a stretch + * of work took the thread doing it. + * + * ->sched_time alone is not enough. It is banked at ticks and at context + * switches, and a kworker's switches happen at the ends of a whole batch of + * work items, so the delta across any one of them is usually a flat zero. The + * unbanked remainder has to be added in, and that means reading a clock: + * sched_clock_cpu() rather than rq->niffies, which is only refreshed under + * the rq lock and would be stale by up to a tick here. + * + * niffies is monotonised forward of the raw clock, so last_ran can be ahead + * of what we read and the remainder can come out negative. Drop it in that + * case; it is bounded by the skew between the two and the banked figure is + * still right. + * + * Interrupts are off across the read so the tick cannot land between taking + * ->sched_time and taking ->last_ran and have the interval counted in both. + * That also pins us to this CPU, which is what makes the subtraction a + * same-clock one. + */ +u64 muqss_task_runtime_live(void) +{ + struct task_struct *p = current; + unsigned long flags; + s64 remainder; + u64 ns, ran; + + local_irq_save(flags); + ns = p->sched_time; + ran = p->last_ran; + remainder = sched_clock_cpu(smp_processor_id()) - ran; + local_irq_restore(flags); + + if (likely(remainder > 0)) + ns += remainder; + + return ns; +} +#endif + +/* + * Return any ns on the sched_clock that have not yet been accounted in + * @p in case that task is currently running. + * + * Called with task_rq_lock(p) held. + */ +static inline u64 do_task_delta_exec(struct task_struct *p, struct rq *rq) +{ + u64 ns = 0; + + /* + * Must be ->curr _and_ ->on_rq. If dequeued, we would + * project cycles that may never be accounted to this + * thread, breaking clock_gettime(). + */ + if (p == rq->curr && task_on_rq_queued(p)) { + update_clocks(rq); + ns = rq->niffies - p->last_ran; + } + + return ns; +} + +/* + * Return accounted runtime for the task. + * Return separately the current's pending runtime that have not been + * accounted yet. + */ +unsigned long long task_sched_runtime(struct task_struct *p) +{ + struct rq_flags rf; + struct rq *rq; + u64 ns; + +#if defined(CONFIG_64BIT) && defined(CONFIG_SMP) + /* + * 64-bit doesn't need locks to atomically read a 64-bit value. + * So we have a optimisation chance when the task's delta_exec is 0. + * Reading ->on_cpu is racy, but this is ok. + * + * If we race with it leaving CPU, we'll take a lock. So we're correct. + * If we race with it entering CPU, unaccounted time is 0. This is + * indistinguishable from the read occurring a few cycles earlier. + * If we see ->on_cpu without ->on_rq, the task is leaving, and has + * been accounted, so we're correct here as well. + */ + if (!p->on_cpu || !task_on_rq_queued(p)) + return tsk_seruntime(p); +#endif + + rq = task_rq_lock(p, &rf); + ns = p->sched_time + do_task_delta_exec(p, rq); + task_rq_unlock(rq, p, &rf); + + return ns; +} + +/* + * Functions to test for when SCHED_ISO tasks have used their allocated + * quota as real time scheduling and convert them back to SCHED_NORMAL. All + * data is modified only by the local runqueue during sched_tick with + * interrupts disabled. + */ + +/* + * Test if SCHED_ISO tasks have run longer than their alloted period as RT + * tasks and set the refractory flag if necessary. There is 10% hysteresis + * for unsetting the flag. 115/128 is ~90/100 as a fast shift instead of a + * slow division. + */ +static inline void iso_tick(struct rq *rq) +{ + rq->iso_ticks = rq->iso_ticks * (ISO_PERIOD - 1) / ISO_PERIOD; + rq->iso_ticks += 100; + if (rq->iso_ticks > ISO_PERIOD * sched_iso_cpu) { + rq->iso_refractory = true; + if (unlikely(rq->iso_ticks > ISO_PERIOD * 100)) + rq->iso_ticks = ISO_PERIOD * 100; + } +} + +/* No SCHED_ISO task was running so decrease rq->iso_ticks */ +static inline void no_iso_tick(struct rq *rq, int ticks) +{ + if (rq->iso_ticks > 0 || rq->iso_refractory) { + rq->iso_ticks = rq->iso_ticks * (ISO_PERIOD - ticks) / ISO_PERIOD; + if (rq->iso_ticks < ISO_PERIOD * (sched_iso_cpu * 115 / 128)) { + rq->iso_refractory = false; + if (unlikely(rq->iso_ticks < 0)) + rq->iso_ticks = 0; + } + } +} + +/* This manages tasks that have run out of timeslice during a sched_tick */ +static void task_running_tick(struct rq *rq) +{ + struct task_struct *p = rq->curr; + + /* + * If a SCHED_ISO task is running we increment the iso_ticks. In + * order to prevent SCHED_ISO tasks from causing starvation in the + * presence of true RT tasks we account those as iso_ticks as well. + */ + if (rt_task(p) || task_running_iso(p)) + iso_tick(rq); + else + no_iso_tick(rq, 1); + + /* SCHED_FIFO tasks never run out of timeslice. */ + if (p->policy == SCHED_FIFO) + return; + + if (iso_task(p)) { + if (task_running_iso(p)) { + if (rq->iso_refractory) { + /* + * SCHED_ISO task is running as RT and limit + * has been hit. Force it to reschedule as + * SCHED_NORMAL by zeroing its time_slice + */ + p->time_slice = 0; + } + } else if (!rq->iso_refractory) { + /* Can now run again ISO. Reschedule to pick up prio */ + goto out_resched; + } + } + + /* + * Tasks that were scheduled in the first half of a tick are not + * allowed to run into the 2nd half of the next tick if they will + * run out of time slice in the interim. Otherwise, if they have + * less than RESCHED_US μs of time slice left they will be rescheduled. + * Dither is used as a backup for when hrexpiry is disabled or high res + * timers not configured in. + */ + if (p->time_slice - rq->dither >= RESCHED_US) + return; +out_resched: + rq_lock(rq); + __set_tsk_resched(p); + rq_unlock(rq); +} + +static inline void task_tick(struct rq *rq) +{ + if (!rq_idle(rq)) + task_running_tick(rq); + else if (rq->last_jiffy > rq->last_scheduler_tick) + no_iso_tick(rq, rq->last_jiffy - rq->last_scheduler_tick); +} + +#ifdef CONFIG_NO_HZ_FULL +/* + * We can stop the timer tick any time highres timers are active since + * we rely entirely on highres timeouts for task expiry rescheduling. + */ +static void sched_stop_tick(struct rq *rq, int cpu) +{ + if (!hrexpiry_enabled(rq)) + return; + if (!tick_nohz_full_enabled()) + return; + if (!tick_nohz_full_cpu(cpu)) + return; + tick_nohz_dep_clear_cpu(cpu, TICK_DEP_BIT_SCHED); +} + +static inline void sched_start_tick(struct rq *rq, int cpu) +{ + tick_nohz_dep_set_cpu(cpu, TICK_DEP_BIT_SCHED); +} + +struct tick_work { + int cpu; + atomic_t state; + struct delayed_work work; +}; +/* Values for ->state, see diagram below. */ +#define TICK_SCHED_REMOTE_OFFLINE 0 +#define TICK_SCHED_REMOTE_OFFLINING 1 +#define TICK_SCHED_REMOTE_RUNNING 2 + +/* + * State diagram for ->state: + * + * + * TICK_SCHED_REMOTE_OFFLINE + * | ^ + * | | + * | | sched_tick_remote() + * | | + * | | + * +--TICK_SCHED_REMOTE_OFFLINING + * | ^ + * | | + * sched_tick_start() | | sched_tick_stop() + * | | + * V | + * TICK_SCHED_REMOTE_RUNNING + * + * + * Other transitions get WARN_ON_ONCE(), except that sched_tick_remote() + * and sched_tick_start() are happy to leave the state in RUNNING. + */ + +static struct tick_work __percpu *tick_work_cpu; + +static void sched_tick_remote(struct work_struct *work) +{ + struct delayed_work *dwork = to_delayed_work(work); + struct tick_work *twork = container_of(dwork, struct tick_work, work); + int cpu = twork->cpu; + struct rq *rq = cpu_rq(cpu); + struct task_struct *curr; + u64 delta; + int os; + + /* + * Handle the tick only if it appears the remote CPU is running in full + * dynticks mode. The check is racy by nature, but missing a tick or + * having one too much is no big deal because the scheduler tick updates + * statistics and checks timeslices in a time-independent way, regardless + * of when exactly it is running. + */ + if (!tick_nohz_tick_stopped_cpu(cpu)) + goto out_requeue; + + rq_lock_irq(rq); + if (cpu_is_offline(cpu)) + goto out_unlock; + + curr = rq->curr; + update_rq_clock(rq); + + if (!is_idle_task(curr)) { + /* + * Make sure the next tick runs within a reasonable + * amount of time. + */ + delta = rq_clock_task(rq) - curr->last_ran; + WARN_ON_ONCE(delta > (u64)NSEC_PER_SEC * 3); + } + /* + * task_tick() takes the rq lock itself when it needs to force a + * reschedule so we must drop it here, keeping interrupts disabled + * to match the context it is called in from sched_tick(). + */ + rq_unlock(rq); + task_tick(rq); + local_irq_enable(); + goto out_requeue; + +out_unlock: + rq_unlock_irq(rq, NULL); + +out_requeue: + + /* + * Run the remote tick once per second (1Hz). This arbitrary + * frequency is large enough to avoid overload but short enough + * to keep scheduler internal stats reasonably up to date. But + * first update state to reflect hotplug activity if required. + */ + os = atomic_fetch_add_unless(&twork->state, -1, TICK_SCHED_REMOTE_RUNNING); + WARN_ON_ONCE(os == TICK_SCHED_REMOTE_OFFLINE); + if (os == TICK_SCHED_REMOTE_RUNNING) + queue_delayed_work(system_unbound_wq, dwork, HZ); +} + +static void sched_tick_start(int cpu) +{ + struct tick_work *twork; + int os; + + if (housekeeping_cpu(cpu, HK_TYPE_TICK)) + return; + + WARN_ON_ONCE(!tick_work_cpu); + + twork = per_cpu_ptr(tick_work_cpu, cpu); + os = atomic_xchg(&twork->state, TICK_SCHED_REMOTE_RUNNING); + WARN_ON_ONCE(os == TICK_SCHED_REMOTE_RUNNING); + if (os == TICK_SCHED_REMOTE_OFFLINE) { + twork->cpu = cpu; + INIT_DELAYED_WORK(&twork->work, sched_tick_remote); + queue_delayed_work(system_unbound_wq, &twork->work, HZ); + } +} + +#ifdef CONFIG_HOTPLUG_CPU +static void sched_tick_stop(int cpu) +{ + struct tick_work *twork; + int os; + + if (housekeeping_cpu(cpu, HK_TYPE_TICK)) + return; + + WARN_ON_ONCE(!tick_work_cpu); + + twork = per_cpu_ptr(tick_work_cpu, cpu); + /* There cannot be competing actions, but don't rely on stop-machine. */ + os = atomic_xchg(&twork->state, TICK_SCHED_REMOTE_OFFLINING); + WARN_ON_ONCE(os != TICK_SCHED_REMOTE_RUNNING); + /* Don't cancel, as this would mess up the state machine. */ +} +#endif /* CONFIG_HOTPLUG_CPU */ + +int __init sched_tick_offload_init(void) +{ + tick_work_cpu = alloc_percpu(struct tick_work); + BUG_ON(!tick_work_cpu); + return 0; +} + +#else /* !CONFIG_NO_HZ_FULL */ +static inline void sched_stop_tick(struct rq *rq, int cpu) {} +static inline void sched_start_tick(struct rq *rq, int cpu) {} +static inline void sched_tick_start(int cpu) { } +static inline void sched_tick_stop(int cpu) { } +#endif + +/* + * This function gets called by the timer code, with HZ frequency. + * We call it with interrupts disabled. + */ +void sched_tick(void) +{ + int cpu __maybe_unused = smp_processor_id(); + struct rq *rq = cpu_rq(cpu); + + arch_scale_freq_tick(); + sched_clock_tick(); + update_clocks(rq); + update_load_avg(rq, 0); + update_cpu_clock_tick(rq, rq->curr); + task_tick(rq); + rq->last_scheduler_tick = rq->last_jiffy; + rq->last_tick = rq->clock; + perf_event_task_tick(); + sched_stop_tick(rq, cpu); +} + +#if defined(CONFIG_PREEMPTION) && (defined(CONFIG_DEBUG_PREEMPT) || \ + defined(CONFIG_TRACE_PREEMPT_TOGGLE)) +/* + * If the value passed in is equal to the current preempt count + * then we just disabled preemption. Start timing the latency. + */ +static inline void preempt_latency_start(int val) +{ + if (preempt_count() == val) { + unsigned long ip = get_lock_parent_ip(); +#ifdef CONFIG_DEBUG_PREEMPT + current->preempt_disable_ip = ip; +#endif + trace_preempt_off(CALLER_ADDR0, ip); + } +} + +void preempt_count_add(int val) +{ +#ifdef CONFIG_DEBUG_PREEMPT + /* + * Underflow? + */ + if (DEBUG_LOCKS_WARN_ON((preempt_count() < 0))) + return; +#endif + __preempt_count_add(val); +#ifdef CONFIG_DEBUG_PREEMPT + /* + * Spinlock count overflowing soon? + */ + DEBUG_LOCKS_WARN_ON((preempt_count() & PREEMPT_MASK) >= + PREEMPT_MASK - 10); +#endif + preempt_latency_start(val); +} +EXPORT_SYMBOL(preempt_count_add); +NOKPROBE_SYMBOL(preempt_count_add); + +/* + * If the value passed in equals to the current preempt count + * then we just enabled preemption. Stop timing the latency. + */ +static inline void preempt_latency_stop(int val) +{ + if (preempt_count() == val) + trace_preempt_on(CALLER_ADDR0, get_lock_parent_ip()); +} + +void preempt_count_sub(int val) +{ +#ifdef CONFIG_DEBUG_PREEMPT + /* + * Underflow? + */ + if (DEBUG_LOCKS_WARN_ON(val > preempt_count())) + return; + /* + * Is the spinlock portion underflowing? + */ + if (DEBUG_LOCKS_WARN_ON((val < PREEMPT_MASK) && + !(preempt_count() & PREEMPT_MASK))) + return; +#endif + + preempt_latency_stop(val); + __preempt_count_sub(val); +} +EXPORT_SYMBOL(preempt_count_sub); +NOKPROBE_SYMBOL(preempt_count_sub); + +#else +static inline void preempt_latency_start(int val) { } +static inline void preempt_latency_stop(int val) { } +#endif + +static inline unsigned long get_preempt_disable_ip(struct task_struct *p) +{ +#ifdef CONFIG_DEBUG_PREEMPT + return p->preempt_disable_ip; +#else + return 0; +#endif +} + +/* + * The time_slice is only refilled when it is empty and that is when we set a + * new deadline. Make sure update_clocks has been called recently to update + * rq->niffies. + */ +static void time_slice_expired(struct task_struct *p, struct rq *rq) +{ + p->time_slice = timeslice(); + /* + * This assignment is absolute, so the charge has to be folded in here + * rather than left to enqueue_task(), or expiry would reset the task + * to an uncharged baseline. + */ + p->deadline = rq->niffies + task_deadline_diff(p) + + consume_task_penalty(p); +#ifdef CONFIG_SMT_NICE + if (!p->mm) + p->smt_bias = 0; + else if (rt_task(p)) + p->smt_bias = 1 << 30; + else if (task_running_iso(p)) + p->smt_bias = 1 << 29; + else if (idleprio_task(p)) { + if (task_running_idle(p)) + p->smt_bias = 0; + else + p->smt_bias = 1; + } else if (--p->smt_bias < 1) + p->smt_bias = MAX_PRIO - p->static_prio; +#endif +} + +/* + * Timeslices below RESCHED_US are considered as good as expired as there's no + * point rescheduling when there's so little time left. SCHED_BATCH tasks + * have been flagged be not latency sensitive and likely to be fully CPU + * bound so every time they're rescheduled they have their time_slice + * refilled, but get a new later deadline to have little effect on + * SCHED_NORMAL tasks. + + */ +static inline void check_deadline(struct task_struct *p, struct rq *rq) +{ + if (p->time_slice < RESCHED_US || batch_task(p)) + time_slice_expired(p, rq); +} + +/* + * Task selection with skiplists is a simple matter of picking off the first + * task in the sorted list, an O(1) operation. The lookup is amortised O(1) + * being bound to the number of processors. + * + * Runqueues are selectively locked based on their unlocked data and then + * unlocked if not needed. At most 3 locks will be held at any time and are + * released as soon as they're no longer needed. All balancing between CPUs + * is thus done here in an extremely simple first come best fit manner. + * + * This iterates over runqueues in cache locality order. In interactive mode + * it iterates over all CPUs and finds the task with the best key/deadline. + * In non-interactive mode it will only take a task if it's from the current + * runqueue or a runqueue with more tasks than the current one with a better + * key/deadline. + */ +#ifdef CONFIG_SMP +static inline struct task_struct +*earliest_deadline_task(struct rq *rq, int cpu, struct task_struct *idle) +{ + struct rq *locked = NULL, *chosen = NULL; + struct task_struct *edt = idle; + int i, best_entries = 0; + u64 best_key = ~0ULL; + + for (i = 0; i < total_runqueues; i++) { + skiplist *sl = rq->sl_order[i]; + struct rq *other_rq = NULL; + skiplist_node *next; + int entries; + + entries = READ_ONCE(sl->entries); + /* + * Check for queued entres lockless first. The local runqueue + * is locked so entries will always be accurate. + */ + if (!sched_interactive) { + /* + * Don't reschedule balance across nodes unless the CPU + * is idle. Non-interactive needs the rq: cpu is read + * before the entries filter. + */ + other_rq = rq_order(rq, i); + if (edt != idle && rq->cpu_locality[other_rq->cpu] > LOCALITY_SMP) + break; + if (entries <= best_entries) + continue; + } else if (!entries) + continue; + + /* if (i) implies other_rq != rq */ + if (i) { + /* Check for best id queued lockless first */ + if (READ_ONCE(sl->best_key) >= best_key) + continue; + + other_rq = rq_order(rq, i); + if (unlikely(!trylock_rq(rq, other_rq))) + continue; + + /* Need to reevaluate entries after locking */ + entries = sl->entries; + if (unlikely(!entries)) { + unlock_rq(other_rq); + continue; + } + } + + next = sl->header; + /* + * In interactive mode we check beyond the best entry on other + * runqueues if we can't get the best for smt or affinity + * reasons. + */ + while ((next = next->next[0]) != sl->header) { + struct task_struct *p; + u64 key = next->key; + + /* Reevaluate key after locking */ + if (key >= best_key) + break; + + p = container_of(next, struct task_struct, node); + if (!smt_schedule(p, rq)) { + if (i && !sched_interactive) + break; + continue; + } + + if (sched_other_cpu(p, cpu)) { + if (sched_interactive || !i) + continue; + break; + } + /* Make sure affinity is ok */ + if (i) { + /* From this point on p is the best so far */ + if (locked) + unlock_rq(locked); + chosen = locked = other_rq; + } + best_entries = entries; + best_key = key; + edt = p; + break; + } + /* rq->preempting is a hint only as the state may have changed + * since it was set with the resched call but if we have met + * the condition we can break out here. */ + if (edt == rq->preempting) + break; + if (i && other_rq != chosen) + unlock_rq(other_rq); + } + + if (likely(edt != idle)) + take_task(rq, cpu, edt); + + if (locked) + unlock_rq(locked); + + rq->preempting = NULL; + + return edt; +} +#else /* CONFIG_SMP */ +static inline struct task_struct +*earliest_deadline_task(struct rq *rq, int cpu, struct task_struct *idle) +{ + struct task_struct *edt; + + if (unlikely(!rq->sl->entries)) + return idle; + edt = container_of(rq->node->next[0], struct task_struct, node); + take_task(rq, cpu, edt); + return edt; +} +#endif /* CONFIG_SMP */ + +/* + * Print scheduling while atomic bug: + */ +static noinline void __schedule_bug(struct task_struct *prev) +{ + /* Save this before calling printk(), since that will clobber it */ + unsigned long preempt_disable_ip = get_preempt_disable_ip(current); + + if (oops_in_progress) + return; + + printk(KERN_ERR "BUG: scheduling while atomic: %s/%d/0x%08x\n", + prev->comm, prev->pid, preempt_count()); + + debug_show_held_locks(prev); + print_modules(); + if (irqs_disabled()) + print_irqtrace_events(prev); + if (IS_ENABLED(CONFIG_DEBUG_PREEMPT) + && in_atomic_preempt_off()) { + pr_err("Preemption disabled at:"); + print_ip_sym(KERN_ERR, preempt_disable_ip); + } + dump_stack(); + add_taint(TAINT_WARN, LOCKDEP_STILL_OK); +} + +/* + * Various schedule()-time debugging checks and statistics: + */ +static inline void schedule_debug(struct task_struct *prev, bool preempt) +{ +#ifdef CONFIG_SCHED_STACK_END_CHECK + if (task_stack_end_corrupted(prev)) + panic("corrupted stack end detected inside scheduler\n"); + + if (task_scs_end_corrupted(prev)) + panic("corrupted shadow stack detected inside scheduler\n"); +#endif + +#ifdef CONFIG_DEBUG_ATOMIC_SLEEP + if (!preempt && prev->__state && prev->non_block_count) { + printk(KERN_ERR "BUG: scheduling in a non-blocking section: %s/%d/%i\n", + prev->comm, prev->pid, prev->non_block_count); + dump_stack(); + add_taint(TAINT_WARN, LOCKDEP_STILL_OK); + } +#endif + + if (unlikely(in_atomic_preempt_off())) { + __schedule_bug(prev); + preempt_count_set(PREEMPT_DISABLED); + } + rcu_sleep_check(); + SCHED_WARN_ON(ct_state() == CT_STATE_USER); + + profile_hit(SCHED_PROFILING, __builtin_return_address(0)); + + schedstat_inc(this_rq()->sched_count); +} + +/* + * The currently running task's information is all stored in rq local data + * which is only modified by the local CPU. + */ +static inline void set_rq_task(struct rq *rq, struct task_struct *p) +{ + if (p == rq->idle || p->policy == SCHED_FIFO) + hrexpiry_clear(rq); + else + hrexpiry_start(rq, US_TO_NS(p->time_slice)); + if (rq->clock - rq->last_tick > HALF_JIFFY_NS) + rq->dither = 0; + else + rq->dither = rq_dither(rq); + + rq->rq_deadline = p->deadline; + rq->rq_prio = p->prio; +#ifdef CONFIG_SMT_NICE + rq->rq_mm = p->mm; + rq->rq_smt_bias = p->smt_bias; +#endif +} + +#ifdef CONFIG_SMT_NICE +static void check_no_siblings(struct rq __maybe_unused *this_rq) {} +static void wake_no_siblings(struct rq __maybe_unused *this_rq) {} +static void (*check_siblings)(struct rq *this_rq) = &check_no_siblings; +static void (*wake_siblings)(struct rq *this_rq) = &wake_no_siblings; + +/* Iterate over smt siblings when we've scheduled a process on cpu and decide + * whether they should continue running or be descheduled. */ +static void check_smt_siblings(struct rq *this_rq) +{ + int other_cpu; + + for_each_cpu(other_cpu, &this_rq->thread_mask) { + struct task_struct *p; + struct rq *rq; + + rq = cpu_rq(other_cpu); + if (rq_idle(rq)) + continue; + p = rq->curr; + if (!smt_schedule(p, this_rq)) + resched_curr(rq); + } +} + +static void wake_smt_siblings(struct rq *this_rq) +{ + int other_cpu; + + for_each_cpu(other_cpu, &this_rq->thread_mask) { + struct rq *rq; + + rq = cpu_rq(other_cpu); + if (rq_idle(rq)) + resched_idle(rq); + } +} +#else +static void check_siblings(struct rq __maybe_unused *this_rq) {} +static void wake_siblings(struct rq __maybe_unused *this_rq) {} +#endif + +/* + * schedule() is the main scheduler function. + * + * The main means of driving the scheduler and thus entering this function are: + * + * 1. Explicit blocking: mutex, semaphore, waitqueue, etc. + * + * 2. TIF_NEED_RESCHED flag is checked on interrupt and userspace return + * paths. For example, see arch/x86/entry_64.S. + * + * To drive preemption between tasks, the scheduler sets the flag in timer + * interrupt handler sched_tick(). + * + * 3. Wakeups don't really cause entry into schedule(). They add a + * task to the run-queue and that's it. + * + * Now, if the new task added to the run-queue preempts the current + * task, then the wakeup sets TIF_NEED_RESCHED and schedule() gets + * called on the nearest possible occasion: + * + * - If the kernel is preemptible (CONFIG_PREEMPTION=y): + * + * - in syscall or exception context, at the next outmost + * preempt_enable(). (this might be as soon as the wake_up()'s + * spin_unlock()!) + * + * - in IRQ context, return from interrupt-handler to + * preemptible context + * + * - If the kernel is not preemptible (CONFIG_PREEMPTION is not set) + * then at the next: + * + * - cond_resched() call + * - explicit schedule() call + * - return from syscall or exception to user-space + * - return from interrupt-handler to user-space + * + * WARNING: must be called with preemption disabled! + */ +#define SM_IDLE (-1) +#define SM_NONE 0 +#define SM_PREEMPT 1 +#define SM_RTLOCK_WAIT 2 + +static void __sched notrace __schedule(int sched_mode) +{ + struct task_struct *prev, *next, *idle; + unsigned long *switch_count; + unsigned long prev_state; + bool deactivate = false; + struct rq *rq; + u64 niffies; + int cpu; + /* + * On PREEMPT_RT, SM_RTLOCK_WAIT is noted as a preemption by + * schedule_debug() and RCU. Task-state changes still treat + * only SM_PREEMPT as preemption so a sleeping lock wait can + * deactivate. + */ + bool preempt = sched_mode > SM_NONE; + + cpu = smp_processor_id(); + rq = cpu_rq(cpu); + prev = rq->curr; + idle = rq->idle; + + schedule_debug(prev, preempt); + + klp_sched_try_switch(prev); + + local_irq_disable(); + rcu_note_context_switch(preempt); + + /* + * Make sure that signal_pending_state()->signal_pending() below + * can't be reordered with __set_current_state(TASK_INTERRUPTIBLE) + * done by the caller to avoid the race with signal_wake_up(): + * + * __set_current_state(@state) signal_wake_up() + * schedule() set_tsk_thread_flag(p, TIF_SIGPENDING) + * wake_up_state(p, state) + * LOCK rq->lock LOCK p->pi_state + * smp_mb__after_spinlock() smp_mb__after_spinlock() + * if (signal_pending_state()) if (p->__state & @state) + * + * Also, the membarrier system call requires a full memory barrier + * after coming from user-space, before storing to rq->curr. + */ + rq_lock(rq); + smp_mb__after_spinlock(); +#ifdef CONFIG_SMP + if (rq->preempt) { + /* + * Make sure resched_curr hasn't triggered a preemption + * locklessly on a task that has since scheduled away. Spurious + * wakeup of idle is okay though. + */ + if (unlikely(sched_mode == SM_PREEMPT && prev != idle && + !test_tsk_need_resched(prev))) { + rq->preempt = NULL; + clear_preempt_need_resched(); + rq_unlock_irq(rq, NULL); + return; + } + rq->preempt = NULL; + } + migrate_disable_switch(rq, prev); +#endif + + /* + * Defer hrexpiry start/cancel until we leave __schedule so we do not + * thrash the oneshot clockevent under the rq lock (mainline hrtick). + */ + hrexpiry_schedule_enter(rq); + + switch_count = &prev->nivcsw; + + /* Task state changes only consider SM_PREEMPT as preemption */ + preempt = sched_mode == SM_PREEMPT; + + /* + * We must load prev->__state once (task_struct::state is volatile), such + * that: + * + * - we form a control dependency vs deactivate_task() below. + * - ptrace_{,un}freeze_traced() can change ->state underneath us. + */ + prev_state = prev->__state; + if (!preempt && prev_state) { + if (signal_pending_state(prev_state, prev)) { + prev->__state = TASK_RUNNING; + } else { + prev->sched_contributes_to_load = + (prev_state & TASK_UNINTERRUPTIBLE) && + !(prev_state & TASK_NOLOAD) && + !(prev_state & TASK_FROZEN); + + if (prev->sched_contributes_to_load) + rq->nr_uninterruptible++; + + /* + * __schedule() ttwu() + * prev_state = prev->__state; if (p->on_rq && ...) + * if (prev_state) goto out; + * p->on_rq = 0; smp_acquire__after_ctrl_dep(); + * p->__state = TASK_WAKING + * + * Where __schedule() and ttwu() have matching control dependencies. + * + * After this, schedule() must not care about p->__state any more. + */ + deactivate = true; + + if (prev->in_iowait) { + atomic_inc(&rq->nr_iowait); + delayacct_blkio_start(); + } + } + switch_count = &prev->nvcsw; + } + + /* + * Store the niffy value here for use by the next task's last_ran + * below to avoid losing niffies due to update_clocks being called + * again after this point. + */ + update_clocks(rq); + niffies = rq->niffies; + update_cpu_clock_switch(rq, prev); + + clear_tsk_need_resched(prev); + clear_preempt_need_resched(); + + if (idle != prev) { + check_deadline(prev, rq); + return_task(prev, rq, cpu, deactivate); + } + + next = earliest_deadline_task(rq, cpu, idle); + if (likely(next->prio != PRIO_LIMIT)) + clear_cpuidle_map(cpu); + else { +#ifdef CONFIG_SMP + if (prev != idle) + rq->idle_jiffy = jiffies; +#endif + set_cpuidle_map(cpu); + update_load_avg(rq, 0); + } + + set_rq_task(rq, next); + next->last_ran = niffies; + + if (likely(prev != next)) { + /* + * Don't reschedule an idle task or deactivated tasks + */ + if (prev == idle) + inc_nr_running(rq); + else if (!deactivate) + resched_suitable_idle(prev); + /* + * The task on the CPU is not on the skiplist, so it holds a + * count of its own on top of the queued ones. nr_running only + * changes when that slot is created or destroyed - i.e. on the + * idle transitions above and below - but rt_nr_running also + * depends on *which* task holds it, so hand it over on every + * switch. Leaving that to the idle transitions leaked a count + * on every rt -> non-rt switch and underflowed on the reverse. + */ + if (prev != idle && rt_task(prev)) + rq->rt_nr_running--; + if (next != idle && rt_task(next)) + rq->rt_nr_running++; + if (unlikely(next == idle)) { + dec_nr_running(rq); + wake_siblings(rq); + } else + check_siblings(rq); + rq->nr_switches++; + /* + * RCU users of rcu_dereference(rq->curr) may not see + * changes to task_struct made by pick_next_task(). + */ + RCU_INIT_POINTER(rq->curr, next); + /* + * The membarrier system call requires each architecture + * to have a full memory barrier after updating + * rq->curr, before returning to user-space. + * + * Here are the schemes providing that barrier on the + * various architectures: + * - mm ? switch_mm() : mmdrop() for x86, s390, sparc, PowerPC. + * switch_mm() rely on membarrier_arch_switch_mm() on PowerPC. + * - finish_lock_switch() for weakly-ordered + * architectures where spin_unlock is a full barrier, + * - switch_to() for arm64 (weakly-ordered, spin_unlock + * is a RELEASE barrier), + */ + ++*switch_count; + + psi_sched_switch(prev, next, !task_on_rq_queued(prev)); + + trace_sched_switch(preempt, prev, next, prev->__state); + context_switch(rq, prev, next); /* unlocks the rq via finish_lock_switch */ + } else { + check_siblings(rq); + hrexpiry_schedule_exit(rq); + rq_unlock(rq); + local_irq_enable(); + } +} + +void __noreturn do_task_dead(void) +{ + /* Causes final put_task_struct in finish_task_switch(). */ + set_special_state(TASK_DEAD); + + /* Tell freezer to ignore us: */ + current->flags |= PF_NOFREEZE; + __schedule(SM_NONE); + BUG(); + + /* Avoid "noreturn function does return" - but don't continue if BUG() is a NOP: */ + for (;;) + cpu_relax(); +} + +static inline void sched_submit_work(struct task_struct *tsk) +{ + unsigned int task_flags; + + if (!tsk->__state) + return; + + task_flags = tsk->flags; + /* + * If a worker went to sleep, notify and ask workqueue whether + * it wants to wake up a task to maintain concurrency. + * As this function is called inside the schedule() context, + * we disable preemption to avoid it calling schedule() again + * in the possible wakeup of a kworker and because wq_worker_sleeping() + * requires it. + */ + if (task_flags & (PF_WQ_WORKER | PF_IO_WORKER)) { + preempt_disable(); + if (task_flags & PF_WQ_WORKER) + wq_worker_sleeping(tsk); + else + io_wq_worker_sleeping(tsk); + preempt_enable_no_resched(); + } + + if (tsk->pi_blocked_on) + return; + + /* + * If we are going to sleep and we have plugged IO queued, + * make sure to submit it to avoid deadlocks. + */ + blk_flush_plug(tsk->plug, true); +} + +static inline void sched_update_worker(struct task_struct *tsk) +{ + if (tsk->flags & (PF_WQ_WORKER | PF_IO_WORKER)) { + if (tsk->flags & PF_WQ_WORKER) + wq_worker_running(tsk); + else + io_wq_worker_running(tsk); + } +} + +static __always_inline void __schedule_loop(int sched_mode) +{ + do { + preempt_disable(); + __schedule(sched_mode); + sched_preempt_enable_no_resched(); + } while (need_resched()); +} + +asmlinkage __visible void __sched schedule(void) +{ + struct task_struct *tsk = current; + +#ifdef CONFIG_RT_MUTEXES + lockdep_assert(!tsk->sched_rt_mutex); +#endif + sched_submit_work(tsk); + __schedule_loop(SM_NONE); + sched_update_worker(tsk); +} + +EXPORT_SYMBOL(schedule); + +/* + * synchronize_rcu_tasks() makes sure that no task is stuck in preempted + * state (have scheduled out non-voluntarily) by making sure that all + * tasks have either left the run queue or have gone into user space. + * As idle tasks do not do either, they must not ever be preempted + * (schedule out non-voluntarily). + * + * schedule_idle() is similar to schedule_preempt_disable() except that it + * never enables preemption because it does not call sched_submit_work(). + */ +void __sched schedule_idle(void) +{ + /* + * As this skips calling sched_submit_work(), which the idle task does + * regardless because that function is a nop when the task is in a + * TASK_RUNNING state, make sure this isn't used someplace that the + * current task can be in any other state. Note, idle is always in the + * TASK_RUNNING state. + */ + WARN_ON_ONCE(current->__state); + do { + __schedule(SM_IDLE); + } while (need_resched()); +} + +#if defined(CONFIG_CONTEXT_TRACKING) && !defined(CONFIG_HAVE_CONTEXT_TRACKING_OFFSTACK) +asmlinkage __visible void __sched schedule_user(void) +{ + /* + * If we come here after a random call to set_need_resched(), + * or we have been woken up remotely but the IPI has not yet arrived, + * we haven't yet exited the RCU idle mode. Do it here manually until + * we find a better solution. + * + * NB: There are buggy callers of this function. Ideally we + * should warn if prev_state != IN_USER, but that will trigger + * too frequently to make sense yet. + */ + enum ctx_state prev_state = exception_enter(); + schedule(); + exception_exit(prev_state); +} +#endif + +/** + * schedule_preempt_disabled - called with preemption disabled + * + * Returns with preemption disabled. Note: preempt_count must be 1 + */ +void __sched schedule_preempt_disabled(void) +{ + sched_preempt_enable_no_resched(); + schedule(); + preempt_disable(); +} + +#ifdef CONFIG_PREEMPT_RT +void __sched notrace schedule_rtlock(void) +{ + __schedule_loop(SM_RTLOCK_WAIT); +} +NOKPROBE_SYMBOL(schedule_rtlock); +#endif + +static void __sched notrace preempt_schedule_common(void) +{ + do { + /* + * Because the function tracer can trace preempt_count_sub() + * and it also uses preempt_enable/disable_notrace(), if + * NEED_RESCHED is set, the preempt_enable_notrace() called + * by the function tracer will call this function again and + * cause infinite recursion. + * + * Preemption must be disabled here before the function + * tracer can trace. Break up preempt_disable() into two + * calls. One to disable preemption without fear of being + * traced. The other to still record the preemption latency, + * which can also be traced by the function tracer. + */ + preempt_disable_notrace(); + preempt_latency_start(1); + __schedule(SM_PREEMPT); + preempt_latency_stop(1); + preempt_enable_no_resched_notrace(); + + /* + * Check again in case we missed a preemption opportunity + * between schedule and now. + */ + } while (need_resched()); +} + +#ifdef CONFIG_PREEMPTION +/* + * This is the entry point to schedule() from in-kernel preemption + * off of preempt_enable. + */ +asmlinkage __visible void __sched notrace preempt_schedule(void) +{ + /* + * If there is a non-zero preempt_count or interrupts are disabled, + * we do not want to preempt the current task. Just return.. + */ + if (likely(!preemptible())) + return; + + preempt_schedule_common(); +} +NOKPROBE_SYMBOL(preempt_schedule); +EXPORT_SYMBOL(preempt_schedule); + +#ifdef CONFIG_PREEMPT_DYNAMIC +# ifdef CONFIG_HAVE_PREEMPT_DYNAMIC_CALL +# ifndef preempt_schedule_dynamic_enabled +# define preempt_schedule_dynamic_enabled preempt_schedule +# define preempt_schedule_dynamic_disabled NULL +# endif +DEFINE_STATIC_CALL(preempt_schedule, preempt_schedule_dynamic_enabled); +EXPORT_STATIC_CALL_TRAMP(preempt_schedule); +# elif defined(CONFIG_HAVE_PREEMPT_DYNAMIC_KEY) +static DEFINE_STATIC_KEY_TRUE(sk_dynamic_preempt_schedule); +void __sched notrace dynamic_preempt_schedule(void) +{ + if (!static_branch_unlikely(&sk_dynamic_preempt_schedule)) + return; + preempt_schedule(); +} +NOKPROBE_SYMBOL(dynamic_preempt_schedule); +EXPORT_SYMBOL(dynamic_preempt_schedule); +# endif +#endif + + +/** + * preempt_schedule_notrace - preempt_schedule called by tracing + * + * The tracing infrastructure uses preempt_enable_notrace to prevent + * recursion and tracing preempt enabling caused by the tracing + * infrastructure itself. But as tracing can happen in areas coming + * from userspace or just about to enter userspace, a preempt enable + * can occur before user_exit() is called. This will cause the scheduler + * to be called when the system is still in usermode. + * + * To prevent this, the preempt_enable_notrace will use this function + * instead of preempt_schedule() to exit user context if needed before + * calling the scheduler. + */ +asmlinkage __visible void __sched notrace preempt_schedule_notrace(void) +{ + enum ctx_state prev_ctx; + + if (likely(!preemptible())) + return; + + do { + /* + * Because the function tracer can trace preempt_count_sub() + * and it also uses preempt_enable/disable_notrace(), if + * NEED_RESCHED is set, the preempt_enable_notrace() called + * by the function tracer will call this function again and + * cause infinite recursion. + * + * Preemption must be disabled here before the function + * tracer can trace. Break up preempt_disable() into two + * calls. One to disable preemption without fear of being + * traced. The other to still record the preemption latency, + * which can also be traced by the function tracer. + */ + preempt_disable_notrace(); + preempt_latency_start(1); + /* + * Needs preempt disabled in case user_exit() is traced + * and the tracer calls preempt_enable_notrace() causing + * an infinite recursion. + */ + prev_ctx = exception_enter(); + __schedule(SM_PREEMPT); + exception_exit(prev_ctx); + + preempt_latency_stop(1); + preempt_enable_no_resched_notrace(); + } while (need_resched()); +} +EXPORT_SYMBOL_GPL(preempt_schedule_notrace); + +#ifdef CONFIG_PREEMPT_DYNAMIC +# if defined(CONFIG_HAVE_PREEMPT_DYNAMIC_CALL) +# ifndef preempt_schedule_notrace_dynamic_enabled +# define preempt_schedule_notrace_dynamic_enabled preempt_schedule_notrace +# define preempt_schedule_notrace_dynamic_disabled NULL +# endif +DEFINE_STATIC_CALL(preempt_schedule_notrace, preempt_schedule_notrace_dynamic_enabled); +EXPORT_STATIC_CALL_TRAMP(preempt_schedule_notrace); +# elif defined(CONFIG_HAVE_PREEMPT_DYNAMIC_KEY) +static DEFINE_STATIC_KEY_TRUE(sk_dynamic_preempt_schedule_notrace); +void __sched notrace dynamic_preempt_schedule_notrace(void) +{ + if (!static_branch_unlikely(&sk_dynamic_preempt_schedule_notrace)) + return; + preempt_schedule_notrace(); +} +NOKPROBE_SYMBOL(dynamic_preempt_schedule_notrace); +EXPORT_SYMBOL(dynamic_preempt_schedule_notrace); +# endif +#endif + +#endif /* CONFIG_PREEMPTION */ + +#ifdef CONFIG_PREEMPT_DYNAMIC +/* + * These aliases sit beside __cond_resched() in core.c, which MuQSS.c defines + * much further down; declare them here so sched_dynamic_update() can see them. + */ +# ifdef CONFIG_HAVE_PREEMPT_DYNAMIC_CALL +# define cond_resched_dynamic_enabled __cond_resched +# define cond_resched_dynamic_disabled ((void *)&__static_call_return0) +# define might_resched_dynamic_enabled __cond_resched +# define might_resched_dynamic_disabled ((void *)&__static_call_return0) +# endif +#endif + +#ifdef CONFIG_PREEMPT_DYNAMIC + +# ifdef CONFIG_GENERIC_IRQ_ENTRY +# include +# endif + +/* + * SC:cond_resched + * SC:might_resched + * SC:preempt_schedule + * SC:preempt_schedule_notrace + * SC:irqentry_exit_cond_resched + * + * + * NONE: + * cond_resched <- __cond_resched + * might_resched <- RET0 + * preempt_schedule <- NOP + * preempt_schedule_notrace <- NOP + * irqentry_exit_cond_resched <- NOP + * dynamic_preempt_lazy <- false + * + * VOLUNTARY: + * cond_resched <- __cond_resched + * might_resched <- __cond_resched + * preempt_schedule <- NOP + * preempt_schedule_notrace <- NOP + * irqentry_exit_cond_resched <- NOP + * dynamic_preempt_lazy <- false + * + * FULL: + * cond_resched <- RET0 + * might_resched <- RET0 + * preempt_schedule <- preempt_schedule + * preempt_schedule_notrace <- preempt_schedule_notrace + * irqentry_exit_cond_resched <- irqentry_exit_cond_resched + * dynamic_preempt_lazy <- false + * + * LAZY: + * cond_resched <- RET0 + * might_resched <- RET0 + * preempt_schedule <- preempt_schedule + * preempt_schedule_notrace <- preempt_schedule_notrace + * irqentry_exit_cond_resched <- irqentry_exit_cond_resched + * dynamic_preempt_lazy <- true + */ + +enum { + preempt_dynamic_undefined = -1, + preempt_dynamic_none, + preempt_dynamic_voluntary, + preempt_dynamic_full, + preempt_dynamic_lazy, +}; + +int preempt_dynamic_mode = preempt_dynamic_undefined; + +int sched_dynamic_mode(const char *str) +{ +# if !(defined(CONFIG_PREEMPT_RT) || defined(CONFIG_ARCH_HAS_PREEMPT_LAZY)) + if (!strcmp(str, "none")) + return preempt_dynamic_none; + + if (!strcmp(str, "voluntary")) + return preempt_dynamic_voluntary; +# endif + + if (!strcmp(str, "full")) + return preempt_dynamic_full; + +# ifdef CONFIG_ARCH_HAS_PREEMPT_LAZY + if (!strcmp(str, "lazy")) + return preempt_dynamic_lazy; +# endif + + return -EINVAL; +} + +# define preempt_dynamic_key_enable(f) static_key_enable(&sk_dynamic_##f.key) +# define preempt_dynamic_key_disable(f) static_key_disable(&sk_dynamic_##f.key) + +# if defined(CONFIG_HAVE_PREEMPT_DYNAMIC_CALL) +# define preempt_dynamic_enable(f) static_call_update(f, f##_dynamic_enabled) +# define preempt_dynamic_disable(f) static_call_update(f, f##_dynamic_disabled) +# elif defined(CONFIG_HAVE_PREEMPT_DYNAMIC_KEY) +# define preempt_dynamic_enable(f) preempt_dynamic_key_enable(f) +# define preempt_dynamic_disable(f) preempt_dynamic_key_disable(f) +# else +# error "Unsupported PREEMPT_DYNAMIC mechanism" +# endif + +static DEFINE_MUTEX(sched_dynamic_mutex); + +static void __sched_dynamic_update(int mode) +{ + /* + * Avoid {NONE,VOLUNTARY} -> FULL transitions from ever ending up in + * the ZERO state, which is invalid. + */ + preempt_dynamic_enable(cond_resched); + preempt_dynamic_enable(might_resched); + preempt_dynamic_enable(preempt_schedule); + preempt_dynamic_enable(preempt_schedule_notrace); + preempt_dynamic_enable(irqentry_exit_cond_resched); + preempt_dynamic_key_disable(preempt_lazy); + + switch (mode) { + case preempt_dynamic_none: + preempt_dynamic_enable(cond_resched); + preempt_dynamic_disable(might_resched); + preempt_dynamic_disable(preempt_schedule); + preempt_dynamic_disable(preempt_schedule_notrace); + preempt_dynamic_disable(irqentry_exit_cond_resched); + preempt_dynamic_key_disable(preempt_lazy); + if (mode != preempt_dynamic_mode) + pr_info("Dynamic Preempt: none\n"); + break; + + case preempt_dynamic_voluntary: + preempt_dynamic_enable(cond_resched); + preempt_dynamic_enable(might_resched); + preempt_dynamic_disable(preempt_schedule); + preempt_dynamic_disable(preempt_schedule_notrace); + preempt_dynamic_disable(irqentry_exit_cond_resched); + preempt_dynamic_key_disable(preempt_lazy); + if (mode != preempt_dynamic_mode) + pr_info("Dynamic Preempt: voluntary\n"); + break; + + case preempt_dynamic_full: + preempt_dynamic_disable(cond_resched); + preempt_dynamic_disable(might_resched); + preempt_dynamic_enable(preempt_schedule); + preempt_dynamic_enable(preempt_schedule_notrace); + preempt_dynamic_enable(irqentry_exit_cond_resched); + preempt_dynamic_key_disable(preempt_lazy); + if (mode != preempt_dynamic_mode) + pr_info("Dynamic Preempt: full\n"); + break; + + case preempt_dynamic_lazy: + preempt_dynamic_disable(cond_resched); + preempt_dynamic_disable(might_resched); + preempt_dynamic_enable(preempt_schedule); + preempt_dynamic_enable(preempt_schedule_notrace); + preempt_dynamic_enable(irqentry_exit_cond_resched); + preempt_dynamic_key_enable(preempt_lazy); + if (mode != preempt_dynamic_mode) + pr_info("Dynamic Preempt: lazy\n"); + break; + } + + WRITE_ONCE(preempt_dynamic_mode, mode); +} + +void sched_dynamic_update(int mode) +{ + mutex_lock(&sched_dynamic_mutex); + __sched_dynamic_update(mode); + mutex_unlock(&sched_dynamic_mutex); +} + +static int __init setup_preempt_mode(char *str) +{ + int mode = sched_dynamic_mode(str); + if (mode < 0) { + pr_warn("Dynamic Preempt: unsupported mode: %s\n", str); + return 0; + } + + sched_dynamic_update(mode); + return 1; +} +__setup("preempt=", setup_preempt_mode); + +static void __init preempt_dynamic_init(void) +{ + if (preempt_dynamic_mode == preempt_dynamic_undefined) { + if (IS_ENABLED(CONFIG_PREEMPT_NONE)) { + sched_dynamic_update(preempt_dynamic_none); + } else if (IS_ENABLED(CONFIG_PREEMPT_VOLUNTARY)) { + sched_dynamic_update(preempt_dynamic_voluntary); + } else if (IS_ENABLED(CONFIG_PREEMPT_LAZY)) { + sched_dynamic_update(preempt_dynamic_lazy); + } else { + /* Default static call setting, nothing to do */ + WARN_ON_ONCE(!IS_ENABLED(CONFIG_PREEMPT)); + preempt_dynamic_mode = preempt_dynamic_full; + pr_info("Dynamic Preempt: full\n"); + } + } +} + +# define PREEMPT_MODEL_ACCESSOR(mode) \ + bool preempt_model_##mode(void) \ + { \ + int mode = READ_ONCE(preempt_dynamic_mode); \ + WARN_ON_ONCE(mode == preempt_dynamic_undefined); \ + return mode == preempt_dynamic_##mode; \ + } \ + EXPORT_SYMBOL_GPL(preempt_model_##mode) + +PREEMPT_MODEL_ACCESSOR(none); +PREEMPT_MODEL_ACCESSOR(voluntary); +PREEMPT_MODEL_ACCESSOR(full); +PREEMPT_MODEL_ACCESSOR(lazy); + +#else /* !CONFIG_PREEMPT_DYNAMIC: */ + +#define preempt_dynamic_mode -1 + +static inline void preempt_dynamic_init(void) { } + +#endif /* CONFIG_PREEMPT_DYNAMIC */ + +const char *preempt_modes[] = { + "none", "voluntary", "full", "lazy", NULL, +}; +/* + * This is the entry point to schedule() from kernel preemption + * off of irq context. + * Note, that this is called and return with irqs disabled. This will + * protect us against recursive calling from irq. + */ +asmlinkage __visible void __sched preempt_schedule_irq(void) +{ + enum ctx_state prev_state; + + /* Catch callers which need to be fixed */ + BUG_ON(preempt_count() || !irqs_disabled()); + + prev_state = exception_enter(); + + do { + preempt_disable(); + local_irq_enable(); + __schedule(SM_PREEMPT); + local_irq_disable(); + sched_preempt_enable_no_resched(); + } while (need_resched()); + + exception_exit(prev_state); +} + +int default_wake_function(wait_queue_entry_t *curr, unsigned mode, int wake_flags, + void *key) +{ + WARN_ON_ONCE(IS_ENABLED(CONFIG_SCHED_DEBUG) && wake_flags & ~WF_SYNC); + return try_to_wake_up(curr->private, mode, wake_flags); +} +EXPORT_SYMBOL(default_wake_function); + +#ifdef CONFIG_RT_MUTEXES + +static inline int __rt_effective_prio(struct task_struct *pi_task, int prio) +{ + if (pi_task) + prio = min(prio, pi_task->prio); + + return prio; +} + +static inline int rt_effective_prio(struct task_struct *p, int prio) +{ + struct task_struct *pi_task = rt_mutex_get_top_task(p); + + return __rt_effective_prio(pi_task, prio); +} + +/* + * rt_mutex_setprio - set the current priority of a task + * @p: task to boost + * @pi_task: donor task + * + * This function changes the 'effective' priority of a task. It does + * not touch ->normal_prio like __setscheduler(). + * + * Used by the rt_mutex code to implement priority inheritance + * logic. Call site only calls if the priority of the task changed. + */ +void rt_mutex_setprio(struct task_struct *p, struct task_struct *pi_task) +{ + int prio, oldprio; + struct rq *rq; + + /* XXX used to be waiter->prio, not waiter->task->prio */ + prio = __rt_effective_prio(pi_task, p->normal_prio); + + /* + * If nothing changed; bail early. + */ + if (p->pi_top_task == pi_task && prio == p->prio) + return; + + rq = __task_rq_lock(p, NULL); + update_rq_clock(rq); + /* + * Set under pi_lock && rq->lock, such that the value can be used under + * either lock. + * + * Note that there is loads of tricky to make this pointer cache work + * right. rt_mutex_slowunlock()+rt_mutex_postunlock() work together to + * ensure a task is de-boosted (pi_task is set to NULL) before the + * task is allowed to run again (and can exit). This ensures the pointer + * points to a blocked task -- which guarantees the task is present. + */ + p->pi_top_task = pi_task; + + /* + * For FIFO/RR we only need to set prio, if that matches we're done. + */ + if (prio == p->prio) + goto out_unlock; + + /* + * Idle task boosting is a nono in general. There is one + * exception, when PREEMPT_RT and NOHZ is active: + * + * The idle task calls get_next_timer_interrupt() and holds + * the timer wheel base->lock on the CPU and another CPU wants + * to access the timer (probably to cancel it). We can safely + * ignore the boosting request, as the idle CPU runs this code + * with interrupts disabled and will complete the lock + * protected section without being interrupted. So there is no + * real need to boost. + */ + if (unlikely(p == rq->idle)) { + WARN_ON(p != rq->curr); + WARN_ON(p->pi_blocked_on); + goto out_unlock; + } + + trace_sched_pi_setprio(p, pi_task); + oldprio = p->prio; + p->prio = prio; + if (task_running(rq, p)){ + rt_running_reprio(rq, oldprio, prio); + if (prio > oldprio) + resched_task(p); + } else if (task_queued(p)) { + dequeue_task(rq, p, DEQUEUE_SAVE); + enqueue_task(rq, p, ENQUEUE_RESTORE); + if (prio < oldprio) + try_preempt(p, rq); + } +out_unlock: + /* Avoid rq from going away on us: */ + preempt_disable(); + __task_rq_unlock(rq, p, NULL); + + preempt_enable(); +} +#else +static inline int rt_effective_prio(struct task_struct *p, int prio) +{ + return prio; +} +#endif + +/* + * Adjust the deadline for when the priority is to change, before it's + * changed. + */ +static inline void adjust_deadline(struct task_struct *p, int new_prio) +{ + p->deadline += static_deadline_diff(new_prio) - task_deadline_diff(p); +} + +void set_user_nice(struct task_struct *p, long nice) +{ + int new_static, old_static; + struct rq_flags rf; + struct rq *rq; + + if (task_nice(p) == nice || nice < MIN_NICE || nice > MAX_NICE) + return; + new_static = NICE_TO_PRIO(nice); + /* + * We have to be careful, if called from sys_setpriority(), + * the task might be in the middle of scheduling on another CPU. + */ + rq = task_rq_lock(p, &rf); + update_rq_clock(rq); + + /* + * The RT priorities are set via sched_setscheduler(), but we still + * allow the 'normal' nice value to be set - but as expected + * it won't have any effect on scheduling until the task is + * not SCHED_NORMAL/SCHED_BATCH: + */ + if (has_rt_policy(p)) { + p->static_prio = new_static; + goto out_unlock; + } + + adjust_deadline(p, new_static); + old_static = p->static_prio; + p->static_prio = new_static; + p->prio = effective_prio(p); + + if (task_queued(p)) { + dequeue_task(rq, p, DEQUEUE_SAVE); + enqueue_task(rq, p, ENQUEUE_RESTORE); + if (new_static < old_static) + try_preempt(p, rq); + } else if (task_running(rq, p)) { + set_rq_task(rq, p); + if (old_static < new_static) + resched_task(p); + } +out_unlock: + task_rq_unlock(rq, p, &rf); +} +EXPORT_SYMBOL(set_user_nice); + +/* + * can_nice - check if a task can reduce its nice value + * @p: task + * @nice: nice value + */ +int can_nice(const struct task_struct *p, const int nice) +{ + /* Convert nice value [19,-20] to rlimit style value [1,40] */ + int nice_rlim = nice_to_rlimit(nice); + + return (nice_rlim <= task_rlimit(p, RLIMIT_NICE) || + capable(CAP_SYS_NICE)); +} + +#ifdef __ARCH_WANT_SYS_NICE + +/* + * sys_nice - change the priority of the current process. + * @increment: priority increment + * + * sys_setpriority is a more generic, but much slower function that + * does similar things. + */ +SYSCALL_DEFINE1(nice, int, increment) +{ + long nice, retval; + + /* + * Setpriority might change our priority at the same moment. + * We don't have to worry. Conceptually one call occurs first + * and we have a single winner. + */ + + increment = clamp(increment, -NICE_WIDTH, NICE_WIDTH); + nice = task_nice(current) + increment; + + nice = clamp_val(nice, MIN_NICE, MAX_NICE); + if (increment < 0 && !can_nice(current, nice)) + return -EPERM; + + retval = security_task_setnice(current, nice); + if (retval) + return retval; + + set_user_nice(current, nice); + return 0; +} + +#endif + +/** + * task_prio - return the priority value of a given task. + * @p: the task in question. + * + * Return: The priority value as seen by users in /proc. + * + * sched policy return value kernel prio user prio/nice + * + * normal, batch, [1 ... 41] 101 0/[-20 ... 19] + * idle [42 ... 81] 102 0/[-20 ... 19] + * iso [0 ... 41] 100 0/[-20 ... 19] + * fifo, rr [-2 ... -100] [98 ... 0] [1 ... 99] + */ +int task_prio(const struct task_struct *p) +{ + int delta, prio = p->prio - MAX_RT_PRIO; + + /* rt tasks and iso tasks */ + if (prio <= 0) + goto out; + + /* Convert to ms to avoid overflows */ + delta = NS_TO_MS(p->deadline - task_rq(p)->niffies); + if (unlikely(delta < 0)) + delta = 0; + delta = delta * 40 / ms_longest_deadline_diff(); + if (delta <= 80) + prio += delta; + if (idleprio_task(p)) + prio += 40; +out: + return prio; +} + +#ifdef CONFIG_SMP +static inline bool rt_rq_is_runnable(struct rq *rt_rq) +{ + return rt_rq->rt_nr_running; +} + +/* + * This function computes an effective utilization for the given CPU, to be + * used for frequency selection given the linear relation: f = u * f_max. + * + * The scheduler tracks the following metrics: + * + * cpu_util_{cfs,rt,dl,irq}() + * cpu_bw_dl() + * + * Where the cfs,rt and dl util numbers are tracked with the same metric and + * synchronized windows and are thus directly comparable. + * + * The cfs,rt,dl utilization are the running times measured with rq->clock_task + * which excludes things like IRQ and steal-time. These latter are then accrued + * in the irq utilization. + * + * The DL bandwidth number otoh is not a measured metric but a value computed + * based on the task model parameters and gives the minimal utilization + * required to meet deadlines. + */ +unsigned long effective_cpu_util(int cpu, unsigned long util_cfs, + unsigned long *min, unsigned long *max) +{ + unsigned long util, irq, scale; + struct rq *rq = cpu_rq(cpu); + + scale = arch_scale_cpu_capacity(cpu); + + /* + * Early check to see if IRQ/steal time saturates the CPU, can be + * because of inaccuracies in how we track these. + */ + irq = cpu_util_irq(rq); + if (unlikely(irq >= scale)) { + if (min) + *min = scale; + if (max) + *max = scale; + return scale; + } + + /* + * MuQSS has no utilisation clamping and no deadline bandwidth, so the + * usable range is simply the whole capacity. A runnable realtime task + * asks for the maximum. + */ + if (min) + *min = rt_rq_is_runnable(rq) ? scale : 0; + if (max) + *max = scale; + + util = util_cfs + cpu_util_rt(rq); + util = scale_irq_capacity(util, irq, scale); + + return min_t(unsigned long, scale, util); +} + +/* + * Available idle CPU capacity, used by the energy model. MuQSS does not + * implement energy aware scheduling, but the interface is still called. + */ +unsigned long sched_cpu_util(int cpu) +{ + unsigned long min, max; + + return effective_cpu_util(cpu, cpu_util_cfs(cpu_rq(cpu)), &min, &max); +} +#endif /* CONFIG_SMP */ + +/** + * idle_cpu - is a given CPU idle currently? + * @cpu: the processor in question. + * + * Return: 1 if the CPU is currently idle. 0 otherwise. + */ +int idle_cpu(int cpu) +{ + struct rq *rq = cpu_rq(cpu); + + if (rq->curr != rq->idle) + return 0; + + if (rq->nr_running) + return 0; + +#ifdef CONFIG_SMP + if (rq->ttwu_pending) + return 0; +#endif + + return 1; +} + +/** + * available_idle_cpu - is a given CPU idle for enqueuing work. + * @cpu: the CPU in question. + * + * Return: 1 if the CPU is currently idle. 0 otherwise. + */ +bool available_idle_cpu(int cpu) +{ + if (!idle_cpu(cpu)) + return 0; + + if (vcpu_is_preempted(cpu)) + return 0; + + return 1; +} + +/** + * idle_task - return the idle task for a given CPU. + * @cpu: the processor in question. + * + * Return: The idle task for the CPU @cpu. + */ +struct task_struct *idle_task(int cpu) +{ + return cpu_rq(cpu)->idle; +} + +/** + * find_process_by_pid - find a process with a matching PID value. + * @pid: the pid in question. + * + * The task of @pid, if found. %NULL otherwise. + */ +static inline struct task_struct *find_process_by_pid(pid_t pid) +{ + return pid ? find_task_by_vpid(pid) : current; +} + +/* Actually do priority change: must hold rq lock. */ +static void __setscheduler(struct task_struct *p, struct rq *rq, int policy, + int prio, const struct sched_attr *attr, + bool keep_boost) +{ + int oldrtprio, oldprio; + + /* + * If params can't change scheduling class changes aren't allowed + * either. + */ + if (attr->sched_flags & SCHED_FLAG_KEEP_PARAMS) + return; + + p->policy = policy; + oldrtprio = p->rt_priority; + p->rt_priority = prio; + + /* rt-policy tasks do not have a timerslack */ + if (has_rt_policy(p)) { + p->timer_slack_ns = 0; + } else if (p->timer_slack_ns == 0) { + /* when switching back to non-rt policy, restore timerslack */ + p->timer_slack_ns = p->default_timer_slack_ns; + } + + p->normal_prio = normal_prio(p); + oldprio = p->prio; + /* + * Keep a potential priority boosting if called from + * sched_setscheduler(). + */ + p->prio = normal_prio(p); + if (keep_boost) + p->prio = rt_effective_prio(p, p->prio); + + if (task_running(rq, p)) { + rt_running_reprio(rq, oldprio, p->prio); + set_rq_task(rq, p); + resched_task(p); + } else if (task_queued(p)) { + dequeue_task(rq, p, DEQUEUE_SAVE); + enqueue_task(rq, p, ENQUEUE_RESTORE); + if (p->prio < oldprio || p->rt_priority > oldrtprio) + try_preempt(p, rq); + } +} + +/* + * Check the target process has a UID that matches the current process's + */ +static bool check_same_owner(struct task_struct *p) +{ + const struct cred *cred = current_cred(), *pcred; + bool match; + + rcu_read_lock(); + pcred = __task_cred(p); + match = (uid_eq(cred->euid, pcred->euid) || + uid_eq(cred->euid, pcred->uid)); + rcu_read_unlock(); + return match; +} + +static int __sched_setscheduler(struct task_struct *p, + const struct sched_attr *attr, + bool user, bool pi) +{ + int retval, policy = attr->sched_policy, oldpolicy = -1, priority = attr->sched_priority; + unsigned long rlim_rtprio = 0; + struct rq_flags rf; + int reset_on_fork; + struct rq *rq; + + /* The pi code expects interrupts enabled */ + BUG_ON(pi && in_interrupt()); + + if (is_rt_policy(policy) && !capable(CAP_SYS_NICE)) { + unsigned long lflags; + + if (!lock_task_sighand(p, &lflags)) + return -ESRCH; + rlim_rtprio = task_rlimit(p, RLIMIT_RTPRIO); + unlock_task_sighand(p, &lflags); + if (rlim_rtprio) + goto recheck; + /* + * If the caller requested an RT policy without having the + * necessary rights, we downgrade the policy to SCHED_ISO. + * We also set the parameter to zero to pass the checks. + */ + policy = SCHED_ISO; + priority = 0; + } +recheck: + /* Double check policy once rq lock held */ + if (policy < 0) { + reset_on_fork = p->sched_reset_on_fork; + policy = oldpolicy = p->policy; + } else { + reset_on_fork = !!(policy & SCHED_RESET_ON_FORK); + policy &= ~SCHED_RESET_ON_FORK; + + if (!SCHED_RANGE(policy)) + return -EINVAL; + } + + if (attr->sched_flags & ~(SCHED_FLAG_ALL | SCHED_FLAG_SUGOV)) + return -EINVAL; + + /* + * Valid priorities for SCHED_FIFO and SCHED_RR are + * 1..MAX_RT_PRIO-1, valid priority for SCHED_NORMAL and + * SCHED_BATCH is 0. + */ + if (priority > MAX_RT_PRIO-1) + return -EINVAL; + if (is_rt_policy(policy) != (priority != 0)) + return -EINVAL; + + /* + * Allow unprivileged RT tasks to decrease priority: + */ + if (user && !capable(CAP_SYS_NICE)) { + if (is_rt_policy(policy)) { + unsigned long rlim_rtprio = + task_rlimit(p, RLIMIT_RTPRIO); + + /* Can't set/change the rt policy */ + if (policy != p->policy && !rlim_rtprio) + return -EPERM; + + /* Can't increase priority */ + if (priority > p->rt_priority && + priority > rlim_rtprio) + return -EPERM; + } else { + switch (p->policy) { + /* + * Can only downgrade policies but not back to + * SCHED_NORMAL + */ + case SCHED_ISO: + if (policy == SCHED_ISO) + goto out; + if (policy != SCHED_NORMAL) + return -EPERM; + break; + case SCHED_BATCH: + if (policy == SCHED_BATCH) + goto out; + if (policy != SCHED_IDLEPRIO) + return -EPERM; + break; + case SCHED_IDLEPRIO: + if (policy == SCHED_IDLEPRIO) + goto out; + return -EPERM; + default: + break; + } + } + + /* Can't change other user's priorities */ + if (!check_same_owner(p)) + return -EPERM; + + /* Normal users shall not reset the sched_reset_on_fork flag: */ + if (p->sched_reset_on_fork && !reset_on_fork) + return -EPERM; + } + + if (user) { + retval = security_task_setscheduler(p); + if (retval) + return retval; + } + + if (pi) + cpuset_lock(); + + /* + * Make sure no PI-waiters arrive (or leave) while we are + * changing the priority of the task: + * + * To be able to change p->policy safely, the runqueue lock must be + * held. + */ + rq = task_rq_lock(p, &rf); + update_rq_clock(rq); + + /* + * Changing the policy of the stop threads its a very bad idea: + */ + if (p == rq->stop) { + retval = -EINVAL; + goto unlock; + } + + /* + * If not changing anything there's no need to proceed further, + * but store a possible modification of reset_on_fork. + */ + if (unlikely(policy == p->policy && (!is_rt_policy(policy) || + priority == p->rt_priority))) { + p->sched_reset_on_fork = reset_on_fork; + retval = 0; + goto unlock; + } + + /* Re-check policy now with rq lock held */ + if (unlikely(oldpolicy != -1 && oldpolicy != p->policy)) { + policy = oldpolicy = -1; + task_rq_unlock(rq, p, &rf); + if (pi) + cpuset_unlock(); + goto recheck; + } + p->sched_reset_on_fork = reset_on_fork; + + __setscheduler(p, rq, policy, priority, attr, pi); + + /* Avoid rq from going away on us: */ + preempt_disable(); + task_rq_unlock(rq, p, &rf); + + if (pi) { + cpuset_unlock(); + rt_mutex_adjust_pi(p); + } + preempt_enable(); +out: + return 0; + +unlock: + task_rq_unlock(rq, p, &rf); + if (pi) + cpuset_unlock(); + return retval; +} + +static int _sched_setscheduler(struct task_struct *p, int policy, + const struct sched_param *param, bool check) +{ + struct sched_attr attr = { + .sched_policy = policy, + .sched_priority = param->sched_priority, + .sched_nice = PRIO_TO_NICE(p->static_prio), + }; + + return __sched_setscheduler(p, &attr, check, true); +} +/** + * sched_setscheduler - change the scheduling policy and/or RT priority of a thread. + * @p: the task in question. + * @policy: new policy. + * @param: structure containing the new RT priority. + * + * Use sched_set_fifo(), read its comment. + * + * Return: 0 on success. An error code otherwise. + * + * NOTE that the task may be already dead. + */ +int sched_setscheduler(struct task_struct *p, int policy, + const struct sched_param *param) +{ + return _sched_setscheduler(p, policy, param, true); +} + + +int sched_setattr(struct task_struct *p, const struct sched_attr *attr) +{ + return __sched_setscheduler(p, attr, true, true); +} + +int sched_setattr_nocheck(struct task_struct *p, const struct sched_attr *attr) +{ + return __sched_setscheduler(p, attr, false, true); +} + +/** + * sched_setscheduler_nocheck - change the scheduling policy and/or RT priority of a thread from kernelspace. + * @p: the task in question. + * @policy: new policy. + * @param: structure containing the new RT priority. + * + * Just like sched_setscheduler, only don't bother checking if the + * current context has permission. For example, this is needed in + * stop_machine(): we create temporary high priority worker threads, + * but our caller might not have that capability. + * + * Return: 0 on success. An error code otherwise. + */ +int sched_setscheduler_nocheck(struct task_struct *p, int policy, + const struct sched_param *param) +{ + return _sched_setscheduler(p, policy, param, false); +} + +/* + * SCHED_FIFO is a broken scheduler model; that is, it is fundamentally + * incapable of resource management, which is the one thing an OS really should + * be doing. + * + * This is of course the reason it is limited to privileged users only. + * + * Worse still; it is fundamentally impossible to compose static priority + * workloads. You cannot take two correctly working static prio workloads + * and smash them together and still expect them to work. + * + * For this reason 'all' FIFO tasks the kernel creates are basically at: + * + * MAX_RT_PRIO / 2 + * + * The administrator _MUST_ configure the system, the kernel simply doesn't + * know enough information to make a sensible choice. + */ +void sched_set_fifo(struct task_struct *p) +{ + struct sched_param sp = { .sched_priority = MAX_RT_PRIO / 2 }; + WARN_ON_ONCE(sched_setscheduler_nocheck(p, SCHED_FIFO, &sp) != 0); +} +EXPORT_SYMBOL_GPL(sched_set_fifo); + +/* + * For when you don't much care about FIFO, but want to be above SCHED_NORMAL. + */ +void sched_set_fifo_low(struct task_struct *p) +{ + struct sched_param sp = { .sched_priority = 1 }; + WARN_ON_ONCE(sched_setscheduler_nocheck(p, SCHED_FIFO, &sp) != 0); +} +EXPORT_SYMBOL_GPL(sched_set_fifo_low); + +void sched_set_normal(struct task_struct *p, int nice) +{ + struct sched_attr attr = { + .sched_policy = SCHED_NORMAL, + .sched_nice = nice, + }; + WARN_ON_ONCE(sched_setattr_nocheck(p, &attr) != 0); +} +EXPORT_SYMBOL_GPL(sched_set_normal); + +static int +do_sched_setscheduler(pid_t pid, int policy, struct sched_param __user *param) +{ + struct sched_param lparam; + struct task_struct *p; + int retval; + + if (!param || pid < 0) + return -EINVAL; + if (copy_from_user(&lparam, param, sizeof(struct sched_param))) + return -EFAULT; + + rcu_read_lock(); + retval = -ESRCH; + p = find_process_by_pid(pid); + if (likely(p)) + get_task_struct(p); + rcu_read_unlock(); + + if (likely(p)) { + retval = sched_setscheduler(p, policy, &lparam); + put_task_struct(p); + } + + return retval; +} + +/* + * Mimics kernel/events/core.c perf_copy_attr(). + */ +static int sched_copy_attr(struct sched_attr __user *uattr, + struct sched_attr *attr) +{ + u32 size; + int ret; + + /* Zero the full structure, so that a short copy will be nice: */ + memset(attr, 0, sizeof(*attr)); + + ret = get_user(size, &uattr->size); + if (ret) + return ret; + + /* ABI compatibility quirk: */ + if (!size) + size = SCHED_ATTR_SIZE_VER0; + + if (size < SCHED_ATTR_SIZE_VER0 || size > PAGE_SIZE) + goto err_size; + + ret = copy_struct_from_user(attr, sizeof(*attr), uattr, size); + if (ret) { + if (ret == -E2BIG) + goto err_size; + return ret; + } + + /* + * XXX: Do we want to be lenient like existing syscalls; or do we want + * to be strict and return an error on out-of-bounds values? + */ + attr->sched_nice = clamp(attr->sched_nice, -20, 19); + + /* sched/core.c uses zero here but we already know ret is zero */ + return 0; + +err_size: + put_user(sizeof(*attr), &uattr->size); + return -E2BIG; +} + +/* + * sched_setparam() passes in -1 for its policy, to let the functions + * it calls know not to change it. + */ +#define SETPARAM_POLICY -1 + +/** + * sys_sched_setscheduler - set/change the scheduler policy and RT priority + * @pid: the pid in question. + * @policy: new policy. + * @param: structure containing the new RT priority. + * + * Return: 0 on success. An error code otherwise. + */ +SYSCALL_DEFINE3(sched_setscheduler, pid_t, pid, int, policy, struct sched_param __user *, param) +{ + if (policy < 0) + return -EINVAL; + + return do_sched_setscheduler(pid, policy, param); +} + +/** + * sys_sched_setparam - set/change the RT priority of a thread + * @pid: the pid in question. + * @param: structure containing the new RT priority. + * + * Return: 0 on success. An error code otherwise. + */ +SYSCALL_DEFINE2(sched_setparam, pid_t, pid, struct sched_param __user *, param) +{ + return do_sched_setscheduler(pid, SETPARAM_POLICY, param); +} + +/** + * sys_sched_setattr - same as above, but with extended sched_attr + * @pid: the pid in question. + * @uattr: structure containing the extended parameters. + */ +SYSCALL_DEFINE3(sched_setattr, pid_t, pid, struct sched_attr __user *, uattr, + unsigned int, flags) +{ + struct sched_attr attr; + struct task_struct *p; + int retval; + + if (!uattr || pid < 0 || flags) + return -EINVAL; + + retval = sched_copy_attr(uattr, &attr); + if (retval) + return retval; + + if ((int)attr.sched_policy < 0) + return -EINVAL; + if (attr.sched_flags & SCHED_FLAG_KEEP_POLICY) + attr.sched_policy = SETPARAM_POLICY; + + rcu_read_lock(); + retval = -ESRCH; + p = find_process_by_pid(pid); + if (likely(p)) + get_task_struct(p); + rcu_read_unlock(); + + if (likely(p)) { + retval = sched_setattr(p, &attr); + put_task_struct(p); + } + + return retval; +} + +/** + * sys_sched_getscheduler - get the policy (scheduling class) of a thread + * @pid: the pid in question. + * + * Return: On success, the policy of the thread. Otherwise, a negative error + * code. + */ +SYSCALL_DEFINE1(sched_getscheduler, pid_t, pid) +{ + struct task_struct *p; + int retval = -EINVAL; + + if (pid < 0) + goto out_nounlock; + + retval = -ESRCH; + rcu_read_lock(); + p = find_process_by_pid(pid); + if (p) { + retval = security_task_getscheduler(p); + if (!retval) + retval = p->policy; + } + rcu_read_unlock(); + +out_nounlock: + return retval; +} + +/** + * sys_sched_getscheduler - get the RT priority of a thread + * @pid: the pid in question. + * @param: structure containing the RT priority. + * + * Return: On success, 0 and the RT priority is in @param. Otherwise, an error + * code. + */ +SYSCALL_DEFINE2(sched_getparam, pid_t, pid, struct sched_param __user *, param) +{ + struct sched_param lp = { .sched_priority = 0 }; + struct task_struct *p; + int retval = -EINVAL; + + if (!param || pid < 0) + goto out_nounlock; + + rcu_read_lock(); + p = find_process_by_pid(pid); + retval = -ESRCH; + if (!p) + goto out_unlock; + + retval = security_task_getscheduler(p); + if (retval) + goto out_unlock; + + if (has_rt_policy(p)) + lp.sched_priority = p->rt_priority; + rcu_read_unlock(); + + /* + * This one might sleep, we cannot do it with a spinlock held ... + */ + retval = copy_to_user(param, &lp, sizeof(*param)) ? -EFAULT : 0; + +out_nounlock: + return retval; + +out_unlock: + rcu_read_unlock(); + return retval; +} + +/* + * Copy the kernel size attribute structure (which might be larger + * than what user-space knows about) to user-space. + * + * Note that all cases are valid: user-space buffer can be larger or + * smaller than the kernel-space buffer. The usual case is that both + * have the same size. + */ +static int +sched_attr_copy_to_user(struct sched_attr __user *uattr, + struct sched_attr *kattr, + unsigned int usize) +{ + unsigned int ksize = sizeof(*kattr); + + if (!access_ok(uattr, usize)) + return -EFAULT; + + /* + * sched_getattr() ABI forwards and backwards compatibility: + * + * If usize == ksize then we just copy everything to user-space and all is good. + * + * If usize < ksize then we only copy as much as user-space has space for, + * this keeps ABI compatibility as well. We skip the rest. + * + * If usize > ksize then user-space is using a newer version of the ABI, + * which part the kernel doesn't know about. Just ignore it - tooling can + * detect the kernel's knowledge of attributes from the attr->size value + * which is set to ksize in this case. + */ + kattr->size = min(usize, ksize); + + if (copy_to_user(uattr, kattr, kattr->size)) + return -EFAULT; + + return 0; +} + +/** + * sys_sched_getattr - similar to sched_getparam, but with sched_attr + * @pid: the pid in question. + * @uattr: structure containing the extended parameters. + * @usize: sizeof(attr) for fwd/bwd comp. + * @flags: for future extension. + */ +SYSCALL_DEFINE4(sched_getattr, pid_t, pid, struct sched_attr __user *, uattr, + unsigned int, usize, unsigned int, flags) +{ + struct sched_attr kattr = { }; + struct task_struct *p; + int retval; + + if (!uattr || pid < 0 || usize > PAGE_SIZE || + usize < SCHED_ATTR_SIZE_VER0 || flags) + return -EINVAL; + + rcu_read_lock(); + p = find_process_by_pid(pid); + retval = -ESRCH; + if (!p) + goto out_unlock; + + retval = security_task_getscheduler(p); + if (retval) + goto out_unlock; + + kattr.sched_policy = p->policy; + if (rt_task(p)) + kattr.sched_priority = p->rt_priority; + else + kattr.sched_nice = task_nice(p); + + rcu_read_unlock(); + + return sched_attr_copy_to_user(uattr, &kattr, usize); + +out_unlock: + rcu_read_unlock(); + return retval; +} + +long sched_setaffinity(pid_t pid, const struct cpumask *in_mask) +{ + cpumask_var_t cpus_allowed, new_mask; + struct task_struct *p; + int retval; + + rcu_read_lock(); + + p = find_process_by_pid(pid); + if (!p) { + rcu_read_unlock(); + return -ESRCH; + } + + /* Prevent p going away */ + get_task_struct(p); + rcu_read_unlock(); + + if (p->flags & PF_NO_SETAFFINITY) { + retval = -EINVAL; + goto out_put_task; + } + if (!alloc_cpumask_var(&cpus_allowed, GFP_KERNEL)) { + retval = -ENOMEM; + goto out_put_task; + } + if (!alloc_cpumask_var(&new_mask, GFP_KERNEL)) { + retval = -ENOMEM; + goto out_free_cpus_allowed; + } + retval = -EPERM; + if (!check_same_owner(p)) { + rcu_read_lock(); + if (!ns_capable(__task_cred(p)->user_ns, CAP_SYS_NICE)) { + rcu_read_unlock(); + goto out_unlock; + } + rcu_read_unlock(); + } + + retval = security_task_setscheduler(p); + if (retval) + goto out_unlock; + + cpuset_cpus_allowed(p, cpus_allowed); + cpumask_and(new_mask, in_mask, cpus_allowed); +again: + retval = __set_cpus_allowed_ptr(p, new_mask, SCA_CHECK | SCA_USER); + + if (!retval) { + cpuset_cpus_allowed(p, cpus_allowed); + if (!cpumask_subset(new_mask, cpus_allowed)) { + /* + * We must have raced with a concurrent cpuset + * update. Just reset the cpus_allowed to the + * cpuset's cpus_allowed + */ + cpumask_copy(new_mask, cpus_allowed); + goto again; + } + } +out_unlock: + free_cpumask_var(new_mask); +out_free_cpus_allowed: + free_cpumask_var(cpus_allowed); +out_put_task: + put_task_struct(p); + return retval; +} + +static int get_user_cpu_mask(unsigned long __user *user_mask_ptr, unsigned len, + cpumask_t *new_mask) +{ + if (len < cpumask_size()) + cpumask_clear(new_mask); + else if (len > cpumask_size()) + len = cpumask_size(); + + return copy_from_user(new_mask, user_mask_ptr, len) ? -EFAULT : 0; +} + + +/** + * sys_sched_setaffinity - set the CPU affinity of a process + * @pid: pid of the process + * @len: length in bytes of the bitmask pointed to by user_mask_ptr + * @user_mask_ptr: user-space pointer to the new CPU mask + * + * Return: 0 on success. An error code otherwise. + */ +SYSCALL_DEFINE3(sched_setaffinity, pid_t, pid, unsigned int, len, + unsigned long __user *, user_mask_ptr) +{ + cpumask_var_t new_mask; + int retval; + + if (!alloc_cpumask_var(&new_mask, GFP_KERNEL)) + return -ENOMEM; + + retval = get_user_cpu_mask(user_mask_ptr, len, new_mask); + if (retval == 0) + retval = sched_setaffinity(pid, new_mask); + free_cpumask_var(new_mask); + return retval; +} + +long sched_getaffinity(pid_t pid, cpumask_t *mask) +{ + struct task_struct *p; + unsigned long flags; + int retval; + + cpus_read_lock(); + rcu_read_lock(); + + retval = -ESRCH; + p = find_process_by_pid(pid); + if (!p) + goto out_unlock; + + retval = security_task_getscheduler(p); + if (retval) + goto out_unlock; + + raw_spin_lock_irqsave(&p->pi_lock, flags); + cpumask_and(mask, &p->cpus_mask, cpu_active_mask); + raw_spin_unlock_irqrestore(&p->pi_lock, flags); + +out_unlock: + rcu_read_unlock(); + cpus_read_unlock(); + + return retval; +} + +/** + * sys_sched_getaffinity - get the CPU affinity of a process + * @pid: pid of the process + * @len: length in bytes of the bitmask pointed to by user_mask_ptr + * @user_mask_ptr: user-space pointer to hold the current CPU mask + * + * Return: 0 on success. An error code otherwise. + */ +SYSCALL_DEFINE3(sched_getaffinity, pid_t, pid, unsigned int, len, + unsigned long __user *, user_mask_ptr) +{ + int ret; + cpumask_var_t mask; + + if ((len * BITS_PER_BYTE) < nr_cpu_ids) + return -EINVAL; + if (len & (sizeof(unsigned long)-1)) + return -EINVAL; + + if (!alloc_cpumask_var(&mask, GFP_KERNEL)) + return -ENOMEM; + + ret = sched_getaffinity(pid, mask); + if (ret == 0) { + unsigned int retlen = min(len, cpumask_size()); + + if (copy_to_user(user_mask_ptr, mask, retlen)) + ret = -EFAULT; + else + ret = retlen; + } + free_cpumask_var(mask); + + return ret; +} + +static void do_sched_yield(void) +{ + struct rq_flags rf; + struct rq *rq; + + if (!sched_yield_type) + return; + + rq = this_rq_lock_irq(&rf); + + if (sched_yield_type > 1) + time_slice_expired(current, rq); + schedstat_inc(rq->yld_count); + + preempt_disable(); + rq_unlock_irq(rq, &rf); + sched_preempt_enable_no_resched(); + + schedule(); +} + +/** + * sys_sched_yield - yield the current processor to other threads. + * + * This function yields the current CPU to other tasks. If there are no + * other threads running on this CPU then this function will return. + * + * Return: 0. + */ +SYSCALL_DEFINE0(sched_yield) +{ + do_sched_yield(); + return 0; +} + +#if !defined(CONFIG_PREEMPTION) || defined(CONFIG_PREEMPT_DYNAMIC) +int __sched __cond_resched(void) +{ + if (should_resched(0)) { + preempt_schedule_common(); + return 1; + } +#ifndef CONFIG_PREEMPT_RCU + rcu_all_qs(); +#endif + return 0; +} +EXPORT_SYMBOL(__cond_resched); +#endif + +#ifdef CONFIG_PREEMPT_DYNAMIC +# ifdef CONFIG_HAVE_PREEMPT_DYNAMIC_CALL +DEFINE_STATIC_CALL_RET0(cond_resched, __cond_resched); +EXPORT_STATIC_CALL_TRAMP(cond_resched); + +DEFINE_STATIC_CALL_RET0(might_resched, __cond_resched); +EXPORT_STATIC_CALL_TRAMP(might_resched); +# elif defined(CONFIG_HAVE_PREEMPT_DYNAMIC_KEY) +static DEFINE_STATIC_KEY_FALSE(sk_dynamic_cond_resched); +int __sched dynamic_cond_resched(void) +{ + if (!static_branch_unlikely(&sk_dynamic_cond_resched)) + return 0; + return __cond_resched(); +} +EXPORT_SYMBOL(dynamic_cond_resched); + +static DEFINE_STATIC_KEY_FALSE(sk_dynamic_might_resched); +int __sched dynamic_might_resched(void) +{ + if (!static_branch_unlikely(&sk_dynamic_might_resched)) + return 0; + return __cond_resched(); +} +EXPORT_SYMBOL(dynamic_might_resched); +# endif +#endif /* CONFIG_PREEMPT_DYNAMIC */ + +/* + * __cond_resched_lock() - if a reschedule is pending, drop the given lock, + * call schedule, and on return reacquire the lock. + * + * This works OK both with and without CONFIG_PREEMPTION. We do strange low-level + * operations here to prevent schedule() from being called twice (once via + * spin_unlock(), once by hand). + */ +int __cond_resched_lock(spinlock_t *lock) +{ + int resched = should_resched(PREEMPT_LOCK_OFFSET); + int ret = 0; + + lockdep_assert_held(lock); + + if (spin_needbreak(lock) || resched) { + spin_unlock(lock); + if (resched) + preempt_schedule_common(); + else + cpu_relax(); + ret = 1; + spin_lock(lock); + } + return ret; +} +EXPORT_SYMBOL(__cond_resched_lock); + +int __cond_resched_rwlock_read(rwlock_t *lock) +{ + int resched = should_resched(PREEMPT_LOCK_OFFSET); + int ret = 0; + + lockdep_assert_held_read(lock); + + if (rwlock_needbreak(lock) || resched) { + read_unlock(lock); + if (resched) + preempt_schedule_common(); + else + cpu_relax(); + ret = 1; + read_lock(lock); + } + return ret; +} +EXPORT_SYMBOL(__cond_resched_rwlock_read); + +int __cond_resched_rwlock_write(rwlock_t *lock) +{ + int resched = should_resched(PREEMPT_LOCK_OFFSET); + int ret = 0; + + lockdep_assert_held_write(lock); + + if (rwlock_needbreak(lock) || resched) { + write_unlock(lock); + if (resched) + preempt_schedule_common(); + else + cpu_relax(); + ret = 1; + write_lock(lock); + } + return ret; +} +EXPORT_SYMBOL(__cond_resched_rwlock_write); + +/** + * yield - yield the current processor to other threads. + * + * Do not ever use this function, there's a 99% chance you're doing it wrong. + * + * The scheduler is at all times free to pick the calling task as the most + * eligible task to run, if removing the yield() call from your code breaks + * it, it's already broken. + * + * Typical broken usage is: + * + * while (!event) + * yield(); + * + * where one assumes that yield() will let 'the other' process run that will + * make event true. If the current task is a SCHED_FIFO task that will never + * happen. Never use yield() as a progress guarantee!! + * + * If you want to use yield() to wait for something, use wait_event(). + * If you want to use yield() to be 'nice' for others, use cond_resched(). + * If you still want to use yield(), do not! + */ +void __sched yield(void) +{ + set_current_state(TASK_RUNNING); + do_sched_yield(); +} +EXPORT_SYMBOL(yield); + +/** + * yield_to - yield the current processor to another thread in + * your thread group, or accelerate that thread toward the + * processor it's on. + * @p: target task + * @preempt: whether task preemption is allowed or not + * + * It's the caller's job to ensure that the target task struct + * can't go away on us before we can do any checks. + * + * Return: + * true (>0) if we indeed boosted the target task. + * false (0) if we failed to boost the target. + * -ESRCH if there's no task to yield to. + */ +int __sched yield_to(struct task_struct *p, bool preempt) +{ + struct task_struct *rq_p; + struct rq *rq, *p_rq; + unsigned long flags; + int yielded = 0; + + local_irq_save(flags); + rq = this_rq(); + +again: + p_rq = task_rq(p); + /* + * If we're the only runnable task on the rq and target rq also + * has only one task, there's absolutely no point in yielding. + */ + if (task_running(p_rq, p) || p->__state) { + yielded = -ESRCH; + goto out_irq; + } + + double_rq_lock(rq, p_rq); + if (unlikely(task_rq(p) != p_rq)) { + double_rq_unlock(rq, p_rq); + goto again; + } + + yielded = 1; + schedstat_inc(rq->yld_count); + rq_p = rq->curr; + if (p->deadline > rq_p->deadline) + p->deadline = rq_p->deadline; + p->time_slice += rq_p->time_slice; + if (p->time_slice > timeslice()) + p->time_slice = timeslice(); + time_slice_expired(rq_p, rq); + if (preempt && rq != p_rq) + resched_task(p_rq->curr); + double_rq_unlock(rq, p_rq); +out_irq: + local_irq_restore(flags); + + if (yielded > 0) + schedule(); + return yielded; +} +EXPORT_SYMBOL_GPL(yield_to); + +int io_schedule_prepare(void) +{ + int old_iowait = current->in_iowait; + + current->in_iowait = 1; + blk_flush_plug(current->plug, true); + + return old_iowait; +} + +void io_schedule_finish(int token) +{ + current->in_iowait = token; +} + +/* + * This task is about to go to sleep on IO. Increment rq->nr_iowait so + * that process accounting knows that this is a task in IO wait state. + * + * But don't do that if it is a deliberate, throttling IO wait (this task + * has set its backing_dev_info: the queue against which it should throttle) + */ + +long __sched io_schedule_timeout(long timeout) +{ + int token; + long ret; + + token = io_schedule_prepare(); + ret = schedule_timeout(timeout); + io_schedule_finish(token); + + return ret; +} +EXPORT_SYMBOL(io_schedule_timeout); + +void __sched io_schedule(void) +{ + int token; + + token = io_schedule_prepare(); + schedule(); + io_schedule_finish(token); +} +EXPORT_SYMBOL(io_schedule); + +/** + * sys_sched_get_priority_max - return maximum RT priority. + * @policy: scheduling class. + * + * Return: On success, this syscall returns the maximum + * rt_priority that can be used by a given scheduling class. + * On failure, a negative error code is returned. + */ +SYSCALL_DEFINE1(sched_get_priority_max, int, policy) +{ + int ret = -EINVAL; + + switch (policy) { + case SCHED_FIFO: + case SCHED_RR: + ret = MAX_RT_PRIO-1; + break; + case SCHED_NORMAL: + case SCHED_BATCH: + case SCHED_ISO: + case SCHED_IDLEPRIO: + ret = 0; + break; + } + return ret; +} + +/** + * sys_sched_get_priority_min - return minimum RT priority. + * @policy: scheduling class. + * + * Return: On success, this syscall returns the minimum + * rt_priority that can be used by a given scheduling class. + * On failure, a negative error code is returned. + */ +SYSCALL_DEFINE1(sched_get_priority_min, int, policy) +{ + int ret = -EINVAL; + + switch (policy) { + case SCHED_FIFO: + case SCHED_RR: + ret = 1; + break; + case SCHED_NORMAL: + case SCHED_BATCH: + case SCHED_ISO: + case SCHED_IDLEPRIO: + ret = 0; + break; + } + return ret; +} + +static int sched_rr_get_interval(pid_t pid, struct timespec64 *t) +{ + struct task_struct *p; + unsigned int time_slice; + struct rq_flags rf; + struct rq *rq; + int retval; + + if (pid < 0) + return -EINVAL; + + retval = -ESRCH; + rcu_read_lock(); + p = find_process_by_pid(pid); + if (!p) + goto out_unlock; + + retval = security_task_getscheduler(p); + if (retval) + goto out_unlock; + + rq = task_rq_lock(p, &rf); + time_slice = p->policy == SCHED_FIFO ? 0 : MS_TO_NS(task_timeslice(p)); + task_rq_unlock(rq, p, &rf); + + rcu_read_unlock(); + *t = ns_to_timespec64(time_slice); + return 0; + +out_unlock: + rcu_read_unlock(); + return retval; +} + +/** + * sys_sched_rr_get_interval - return the default timeslice of a process. + * @pid: pid of the process. + * @interval: userspace pointer to the timeslice value. + * + * this syscall writes the default timeslice value of a given process + * into the user-space timespec buffer. A value of '0' means infinity. + * + * Return: On success, 0 and the timeslice is in @interval. Otherwise, + * an error code. + */ +SYSCALL_DEFINE2(sched_rr_get_interval, pid_t, pid, + struct __kernel_timespec __user *, interval) +{ + struct timespec64 t; + int retval = sched_rr_get_interval(pid, &t); + + if (retval == 0) + retval = put_timespec64(&t, interval); + + return retval; +} + +#ifdef CONFIG_COMPAT_32BIT_TIME +SYSCALL_DEFINE2(sched_rr_get_interval_time32, pid_t, pid, + struct old_timespec32 __user *, interval) +{ + struct timespec64 t; + int retval = sched_rr_get_interval(pid, &t); + + if (retval == 0) + retval = put_old_timespec32(&t, interval); + return retval; +} +#endif + +void sched_show_task(struct task_struct *p) +{ + unsigned long free = 0; + int ppid; + + if (!try_get_task_stack(p)) + return; + + printk(KERN_INFO "%-15.15s %c", p->comm, task_state_to_char(p)); + + if (p->__state == TASK_RUNNING) + printk(KERN_CONT " running task "); +#ifdef CONFIG_DEBUG_STACK_USAGE + free = stack_not_used(p); +#endif + ppid = 0; + rcu_read_lock(); + if (pid_alive(p)) + ppid = task_pid_nr(rcu_dereference(p->real_parent)); + rcu_read_unlock(); + pr_cont(" stack:%5lu pid:%5d ppid:%6d flags:0x%08lx\n", + free, task_pid_nr(p), ppid, + (unsigned long)task_thread_info(p)->flags); + + print_worker_info(KERN_INFO, p); + print_stop_info(KERN_INFO, p); + show_stack(p, NULL, KERN_INFO); + put_task_stack(p); +} +EXPORT_SYMBOL_GPL(sched_show_task); + +static inline bool +state_filter_match(unsigned long state_filter, struct task_struct *p) +{ + /* no filter, everything matches */ + if (!state_filter) + return true; + + /* filter, but doesn't match */ + if (!(p->__state & state_filter)) + return false; + + /* + * When looking for TASK_UNINTERRUPTIBLE skip TASK_IDLE (allows + * TASK_KILLABLE). + */ + if (state_filter == TASK_UNINTERRUPTIBLE && p->__state == TASK_IDLE) + return false; + + return true; +} + +void show_state_filter(unsigned int state_filter) +{ + struct task_struct *g, *p; + + rcu_read_lock(); + for_each_process_thread(g, p) { + /* + * reset the NMI-timeout, listing all files on a slow + * console might take a lot of time: + * Also, reset softlockup watchdogs on all CPUs, because + * another CPU might be blocked waiting for us to process + * an IPI. + */ + touch_nmi_watchdog(); + touch_all_softlockup_watchdogs(); + if (state_filter_match(state_filter, p)) + sched_show_task(p); + } + + rcu_read_unlock(); + /* + * Only show locks if all tasks are dumped: + */ + if (!state_filter) + debug_show_all_locks(); +} + +void dump_cpu_task(int cpu) +{ + pr_info("Task dump for CPU %d:\n", cpu); + sched_show_task(cpu_curr(cpu)); +} + +#ifdef CONFIG_SMP +void set_cpus_allowed_common(struct task_struct *p, struct affinity_context *ctx) +{ + if (ctx->flags & (SCA_MIGRATE_ENABLE | SCA_MIGRATE_DISABLE)) { + p->cpus_ptr = ctx->new_mask; + return; + } + + cpumask_copy(&p->cpus_mask, ctx->new_mask); + p->nr_cpus_allowed = cpumask_weight(ctx->new_mask); +} + +void +__do_set_cpus_allowed(struct task_struct *p, struct affinity_context *ctx) +{ + struct rq *rq = task_rq(p); + + lockdep_assert_held(&p->pi_lock); + + /* + * Keep cpus_mask and nr_cpus_allowed in lockstep. sched_other_cpu() + * relies on nr_cpus_allowed == 1 to allow hotplug kthreads bound to + * offline CPUs to still be picked on an online runqueue. + */ + set_cpus_allowed_common(p, ctx); + + if (task_queued(p)) { + /* + * Because __kthread_bind() calls this on blocked tasks without + * holding rq->lock. + */ + lockdep_assert_held(rq->lock); + } +} + +static struct rq *move_queued_task(struct rq *rq, struct rq_flags *rf, + struct task_struct *p, int new_cpu); + +/* + * Calling do_set_cpus_allowed from outside the scheduler code should not be + * called on a running or queued task. We should be holding pi_lock. + */ +void do_set_cpus_allowed(struct task_struct *p, struct affinity_context *ctx) +{ + __do_set_cpus_allowed(p, ctx); + + /* + * A mask forced by cpuset or kthread_bind is a change of what the + * task wants, so an override bind_zero() had to make must not be + * restored over the top of it later. What was saved is left in place + * rather than freed, which here would be under pi_lock. + */ + if (ctx->flags & SCA_USER) + p->zerobound = false; + + if (needs_other_cpu(p, task_cpu(p))) { + struct rq *rq; + + rq = __task_rq_lock(p, NULL); + /* + * A blocked task is on no runqueue, so task_cpu() only says + * which rq lock protects it and which CPU PSI counts its state + * on. It is left exactly as it is. + * + * Moving it would leave the TSK_IOWAIT it is carrying counted + * on the CPU it came from, and the wakeup would then clear it + * on a CPU that never counted it and underflow that counter. + * Nor may the destination be recorded in wake_cpu: that is not + * a hint, it is the bit that arms the lazy migration handshake. + * return_task() reads wake_cpu != task_cpu() as "set_task_cpu() + * moved this running task, hand it over in + * finish_lock_switch()", so stamping it here arms a migration + * no one arranged. Widening the mask again before the task + * wakes is enough to leave it armed, since ttwu only calls + * set_task_cpu() - the one place that resyncs wake_cpu - when + * it picks a different CPU, and a wakeup back onto task_cpu() + * therefore keeps the stale value until the next deschedule + * migrates the task out from under the switch. + * + * Nothing needs either. select_best_cpu() honours the new mask + * when the task wakes and set_task_cpu() takes the PSI state + * with it, which is why affine_move_task() leaves a blocked + * task alone here as well. + */ + if (task_running(rq, p)) { + /* + * A running task is not on any skiplist, so + * set_task_cpu() only tags wake_cpu for it; resched_task() + * gets it off the CPU so finish_lock_switch() can + * complete the move. + */ + set_task_cpu(p, valid_task_cpu(p)); + resched_task(p); + } else if (task_queued(p)) { + /* + * A queued task's skiplist node is linked into this + * rq's list, so its CPU must not be rewritten + * underneath it: task_rq(p) would then name a runqueue + * the node is not in, and the next dequeue_task() would + * try to unlink it from there and trip + * skiplist_delete()'s "m < 0" while leaving the node + * linked in the list it really is on. Unlink it first, + * exactly as every other migration of a queued task + * does. + */ + rq = move_queued_task(rq, NULL, p, valid_task_cpu(p)); + } + __task_rq_unlock(rq, p, NULL); + } +} + +/* migrate_disable()/migrate_enable() are inline in now. */ +#endif + +/** + * init_idle - set up an idle thread for a given CPU + * @idle: task in question + * @cpu: cpu the idle task belongs to + * + * NOTE: this function does not set the idle thread's NEED_RESCHED + * flag, to make booting more robust. + */ +void init_idle(struct task_struct *idle, int cpu) +{ + struct rq *rq = cpu_rq(cpu); + unsigned long flags; + + raw_spin_lock_irqsave(&idle->pi_lock, flags); + raw_spin_lock(rq->lock); + idle->last_ran = rq->niffies; + time_slice_expired(idle, rq); + idle->__state = TASK_RUNNING; + /* Setting prio to illegal value shouldn't matter when never queued */ + idle->prio = PRIO_LIMIT; + idle->flags |= PF_IDLE; + + scs_task_reset(idle); + kasan_unpoison_task_stack(idle); + +#ifdef CONFIG_SMP + /* + * It's possible that init_idle() gets called multiple times on a task, + * in that case do_set_cpus_allowed() will not do the right thing. + * + * And since this is boot we can forgo the serialisation. + */ + { + struct affinity_context ac = { .new_mask = cpumask_of(cpu) }; + + set_cpus_allowed_common(idle, &ac); + } +#ifdef CONFIG_SMT_NICE + idle->smt_bias = 0; +#endif +#endif + set_rq_task(rq, idle); + + /* Silence PROVE_RCU */ + rcu_read_lock(); + set_task_cpu(idle, cpu); + rcu_read_unlock(); + + rq->idle = idle; + rcu_assign_pointer(rq->curr, idle); + idle->on_rq = TASK_ON_RQ_QUEUED; + raw_spin_unlock(rq->lock); + raw_spin_unlock_irqrestore(&idle->pi_lock, flags); + + /* Set the preempt count _outside_ the spinlocks! */ + init_idle_preempt_count(idle, cpu); + + ftrace_graph_init_idle_task(idle, cpu); + vtime_init_idle(idle, cpu); +#ifdef CONFIG_SMP + sprintf(idle->comm, "%s/%d", INIT_TASK_COMM, cpu); +#endif +} + +int cpuset_cpumask_can_shrink(const struct cpumask __maybe_unused *cur, + const struct cpumask __maybe_unused *trial) +{ + return 1; +} + +int task_can_attach(struct task_struct *p) +{ + int ret = 0; + + /* + * Kthreads which disallow setaffinity shouldn't be moved + * to a new cpuset; we don't want to change their CPU + * affinity and isolating such threads by their set of + * allowed nodes is unnecessary. Thus, cpusets are not + * applicable for such threads. This prevents checking for + * success of set_cpus_allowed_ptr() on all attached tasks + * before cpus_mask may be changed. + */ + if (p->flags & PF_NO_SETAFFINITY) + ret = -EINVAL; + + return ret; +} + +void resched_cpu(int cpu) +{ + struct rq *rq = cpu_rq(cpu); + struct rq_flags rf; + + rq_lock_irqsave(rq, &rf); + if (cpu_online(cpu) || cpu == smp_processor_id()) + resched_curr(rq); + rq_unlock_irqrestore(rq, &rf); +} + +#ifdef CONFIG_SMP +#ifdef CONFIG_NO_HZ_COMMON +void nohz_balance_enter_idle(int cpu) {} + +/* + * In the semi idle case, use the nearest busy CPU for migrating timers + * from an idle CPU. This is good for power-savings. + * + * We don't do similar optimization for completely idle system, as + * selecting an idle CPU will add more delays to the timers than intended + * (as that CPU's timer base may not be uptodate wrt jiffies etc). + */ +int get_nohz_timer_target(void) +{ + int i, cpu = smp_processor_id(), default_cpu = -1; + struct sched_domain *sd; + + if (housekeeping_cpu(cpu, HK_TYPE_TIMER)) { + if (!idle_cpu(cpu)) + return cpu; + default_cpu = cpu; + } + + rcu_read_lock(); + for_each_domain(cpu, sd) { + for_each_cpu_and(i, sched_domain_span(sd), + housekeeping_cpumask(HK_TYPE_TIMER)) { + if (cpu == i) + continue; + + if (!idle_cpu(i)) { + cpu = i; + goto unlock; + } + } + } + + if (default_cpu == -1) + default_cpu = housekeeping_any_cpu(HK_TYPE_TIMER); + cpu = default_cpu; +unlock: + rcu_read_unlock(); + return cpu; +} + +/* + * When add_timer_on() enqueues a timer into the timer wheel of an + * idle CPU then this timer might expire before the next timer event + * which is scheduled to wake up that CPU. In case of a completely + * idle system the next event might even be infinite time into the + * future. wake_up_idle_cpu() ensures that the CPU is woken up and + * leaves the inner idle loop so the newly added timer is taken into + * account when the CPU goes back to idle and evaluates the timer + * wheel for the next timer event. + */ +static void wake_up_idle_cpu(int cpu) +{ + if (cpu == smp_processor_id()) + return; + + if (set_nr_and_not_polling(cpu_rq(cpu)->idle)) + smp_sched_reschedule(cpu); + else + trace_sched_wake_idle_without_ipi(cpu); +} + +static bool wake_up_full_nohz_cpu(int cpu) +{ + /* + * We just need the target to call irq_exit() and re-evaluate + * the next tick. The nohz full kick at least implies that. + * If needed we can still optimize that later with an + * empty IRQ. + */ + if (cpu_is_offline(cpu)) + return true; /* Don't try to wake offline CPUs. */ + if (tick_nohz_full_cpu(cpu)) { + if (cpu != smp_processor_id() || + tick_nohz_tick_stopped()) + tick_nohz_full_kick_cpu(cpu); + return true; + } + + return false; +} + +/* + * Wake up the specified CPU. If the CPU is going offline, it is the + * caller's responsibility to deal with the lost wakeup, for example, + * by hooking into the CPU_DEAD notifier like timers and hrtimers do. + */ +void wake_up_nohz_cpu(int cpu) +{ + if (!wake_up_full_nohz_cpu(cpu)) + wake_up_idle_cpu(cpu); +} +#endif /* CONFIG_NO_HZ_COMMON */ + +/* + * This is how migration works: + * + * 1) we invoke migration_cpu_stop() on the target CPU using + * stop_one_cpu(). + * 2) stopper starts to run (implicitly forcing the migrated thread + * off the CPU) + * 3) it checks whether the migrated task is still in the wrong runqueue. + * 4) if it's in the wrong runqueue then the migration thread removes + * it and puts it into the right queue. + * 5) stopper completes and stop_one_cpu() returns and the migration + * is done. + */ + +/* + * move_queued_task - move a queued task to new rq. + * + * Returns (locked) new rq. Old rq's lock is released. Running tasks + * are not on the skiplist, so callers must use task_queued(). + */ +static struct rq *move_queued_task(struct rq *rq, + struct rq_flags __always_unused *rf, + struct task_struct *p, int new_cpu) +{ + lockdep_assert_held(rq->lock); + + dequeue_task(rq, p, 0); + set_task_cpu(p, new_cpu); + rq_unlock(rq); + + rq = cpu_rq(new_cpu); + + rq_lock(rq); + WARN_ON_ONCE(task_cpu(p) != new_cpu); + enqueue_task(rq, p, ENQUEUE_MIGRATED); + try_preempt(p, rq); + + return rq; +} + +struct set_affinity_pending; + +struct migration_arg { + struct task_struct *task; + int dest_cpu; + struct set_affinity_pending *pending; +}; + +/* + * @refs: number of wait_for_completion() + * @stop_pending: is @stop_work in use + */ +struct set_affinity_pending { + refcount_t refs; + unsigned int stop_pending; + struct completion done; + struct cpu_stop_work stop_work; + struct migration_arg arg; +}; + +static struct rq *__migrate_task(struct rq *rq, struct rq_flags *rf, + struct task_struct *p, int dest_cpu) +{ + /* Affinity changed (again). */ + if (!is_cpu_allowed(p, dest_cpu)) + return rq; + + return move_queued_task(rq, rf, p, dest_cpu); +} + +static int migration_cpu_stop(void *data) +{ + struct migration_arg *arg = data; + struct set_affinity_pending *pending = arg->pending; + struct task_struct *p = arg->task; + struct rq *rq = this_rq(); + bool complete = false; + struct rq_flags rf; + + /* + * The original target CPU might have gone down and we might + * be on another CPU but it doesn't matter. + */ + local_irq_save(rf.flags); + /* + * Flush pending wakeups so we do not miss enforcing cpus_ptr, + * see set_cpus_allowed_ptr()'s TASK_WAKING test. + */ + flush_smp_call_function_queue(); + + raw_spin_lock(&p->pi_lock); + rq_lock(rq); + + /* + * If we were passed a pending, then ->stop_pending was set, thus + * p->migration_pending must have remained stable. + */ + WARN_ON_ONCE(pending && pending != p->migration_pending); + + /* + * If task_rq(p) != rq, it cannot be migrated here, because we're + * holding rq->lock. If p is not queued it cannot get enqueued + * because we're holding p->pi_lock. + */ + if (task_rq(p) == rq) { + if (is_migration_disabled(p)) + goto out; + + if (pending) { + p->migration_pending = NULL; + complete = true; + + if (cpumask_test_cpu(task_cpu(p), &p->cpus_mask)) + goto out; + } + + /* + * Mainline updates the rq clock before __migrate_task() here. + * MuQSS has no need to: dequeue_task() and enqueue_task() each + * call update_clocks() on the runqueue they touch, so niffies, + * which deadlines are based on, is current on both sides of + * the migration. Adding one here would only be redundant. + */ + if (task_queued(p)) + rq = __migrate_task(rq, &rf, p, arg->dest_cpu); + else + p->wake_cpu = arg->dest_cpu; + } else if (pending) { + /* + * The task moved before the stopper got to run. We're + * holding ->pi_lock, so the allowed mask is stable - if + * it got somewhere allowed, we're done. + */ + if (cpumask_test_cpu(task_cpu(p), p->cpus_ptr)) { + p->migration_pending = NULL; + complete = true; + goto out; + } + + /* + * When migrate_enable() hits a rq mis-match we can't + * reliably determine is_migration_disabled() and so + * have to chase after it. + */ + WARN_ON_ONCE(!pending->stop_pending); + preempt_disable(); + rq_unlock(rq); + raw_spin_unlock_irqrestore(&p->pi_lock, rf.flags); + stop_one_cpu_nowait(task_cpu(p), migration_cpu_stop, + &pending->arg, &pending->stop_work); + preempt_enable(); + return 0; + } +out: + if (pending) + pending->stop_pending = false; + rq_unlock(rq); + raw_spin_unlock_irqrestore(&p->pi_lock, rf.flags); + + if (complete) + complete_all(&pending->done); + + return 0; +} + +/* + * When given a valid mask, __set_cpus_allowed_ptr() must block until + * the designated task is enqueued on an allowed CPU. A running task + * is kicked off with the CPU stopper. A migrate_disable() region on + * the target delays that move until the outermost migrate_enable(); + * concurrent waiters share one set_affinity_pending. + */ +static int affine_move_task(struct rq *rq, struct task_struct *p, + struct rq_flags *rf, int dest_cpu, unsigned int flags) +{ + struct set_affinity_pending my_pending = { }, *pending = NULL; + bool stop_pending, complete = false; + + /* Can the task run on the task's current CPU? If so, we're done */ + if (cpumask_test_cpu(task_cpu(p), &p->cpus_mask)) { + /* + * If there are pending waiters, but no pending stop_work, + * then complete now. + */ + pending = p->migration_pending; + if (pending && !pending->stop_pending) { + p->migration_pending = NULL; + complete = true; + } + + task_rq_unlock(rq, p, rf); + + if (complete) + complete_all(&pending->done); + + return 0; + } + + if (!(flags & SCA_MIGRATE_ENABLE)) { + /* serialized by p->pi_lock */ + if (!p->migration_pending) { + refcount_set(&my_pending.refs, 1); + init_completion(&my_pending.done); + my_pending.arg = (struct migration_arg) { + .task = p, + .dest_cpu = dest_cpu, + .pending = &my_pending, + }; + + p->migration_pending = &my_pending; + } else { + pending = p->migration_pending; + refcount_inc(&pending->refs); + /* + * Affinity has changed, but we've already installed a + * pending. migration_cpu_stop() *must* see this, else + * we risk completing despite a task on a disallowed + * CPU. Serialized by p->pi_lock. + */ + pending->arg.dest_cpu = dest_cpu; + } + } + pending = p->migration_pending; + /* + * !MIGRATE_ENABLE installs a pending if there wasn't one. + * MIGRATE_ENABLE only gets here because the current CPU no + * longer matches, so a concurrent set_cpus_allowed_ptr() + * should still be pending completion. + */ + if (WARN_ON_ONCE(!pending)) { + task_rq_unlock(rq, p, rf); + return -EINVAL; + } + + if (task_running(rq, p) || READ_ONCE(p->__state) == TASK_WAKING) { + /* + * MIGRATE_ENABLE gets here because 'p == current'. For + * anything else we cannot do is_migration_disabled(); + * punt and have the stopper handle it race-free. + */ + stop_pending = pending->stop_pending; + if (!stop_pending) + pending->stop_pending = true; + + preempt_disable(); + task_rq_unlock(rq, p, rf); + if (!stop_pending) { + stop_one_cpu_nowait(cpu_of(rq), migration_cpu_stop, + &pending->arg, &pending->stop_work); + } + preempt_enable(); + + if (flags & SCA_MIGRATE_ENABLE) + return 0; + } else { + if (!is_migration_disabled(p)) { + if (task_queued(p)) + rq = move_queued_task(rq, rf, p, dest_cpu); + + if (!pending->stop_pending) { + p->migration_pending = NULL; + complete = true; + } + } + task_rq_unlock(rq, p, rf); + + if (complete) + complete_all(&pending->done); + } + + wait_for_completion(&pending->done); + + if (refcount_dec_and_test(&pending->refs)) + wake_up_var(&pending->refs); /* No UaF, just an address */ + + /* + * Block the original owner of &pending until all subsequent + * callers have seen the completion and decremented the refcount + */ + wait_var_event(&my_pending.refs, !refcount_read(&my_pending.refs)); + + WARN_ON_ONCE(my_pending.stop_pending); + + return 0; +} + +/* + * Record @new_mask as the affinity userspace asked for, handing back the + * allocation it displaced for the caller to free once the locks are + * dropped. An explicit request also supersedes any override bind_zero() + * had to force, since this mask is the one that is wanted now. + */ +static cpumask_t *set_user_cpus_ptr(struct task_struct *p, + const struct cpumask *new_mask, + cpumask_t *user_mask) +{ + lockdep_assert_held(&p->pi_lock); + + if (user_mask) { + cpumask_copy(user_mask, new_mask); + swap(p->user_cpus_ptr, user_mask); + } + p->zerobound = false; + + return user_mask; +} + +/* + * Change a given task's CPU affinity. Migrate the thread to a + * proper CPU and schedule it away if the CPU it's executing on + * is removed from the allowed bitmask. + * + * NOTE: the caller must have a valid reference to the task, the + * task must not exit() & deallocate itself prematurely. The + * call is not atomic; no spinlocks may be held. + */ +static int __set_cpus_allowed_ptr(struct task_struct *p, + const struct cpumask *new_mask, + u32 flags) +{ + const struct cpumask *cpu_valid_mask = cpu_active_mask; + cpumask_t *user_mask = NULL; + unsigned int dest_cpu; + struct rq_flags rf; + struct rq *rq; + int ret = 0; + + /* + * user_cpus_ptr keeps what userspace asked for so that + * relax_compatible_cpus_allowed_ptr() and unbind_zero() have + * something to put back. Only sched_setaffinity() passes SCA_USER + * and it is always sleepable, so make room before taking any lock; + * failing here only costs the task that restore, not the syscall. + */ + if (flags & SCA_USER) + user_mask = kmalloc(cpumask_size(), GFP_KERNEL); + + rq = task_rq_lock(p, &rf); + update_rq_clock(rq); + + if ((p->flags & PF_KTHREAD) || is_migration_disabled(p)) { + /* + * Kernel threads are allowed on online && !active CPUs. + * migrate_disabled() tasks must not fail the dest pick + * on SCA_MIGRATE_ENABLE or we skip set_cpus_allowed_common() + * and never reset p->cpus_ptr. + */ + cpu_valid_mask = cpu_online_mask; + } + + /* + * Must re-check here, to close a race against __kthread_bind(), + * sched_setaffinity() is not guaranteed to observe the flag. + */ + if ((flags & SCA_CHECK) && (p->flags & PF_NO_SETAFFINITY)) { + ret = -EINVAL; + goto out; + } + + if (!(flags & SCA_MIGRATE_ENABLE)) { + if (cpumask_equal(&p->cpus_mask, new_mask)) { + /* + * Nothing to change, but asking for exactly the mask + * bind_zero() forced is still userspace choosing it, + * and takes over from the restore. An internal caller + * changing nothing must not cost the task that. + */ + if (flags & SCA_USER) + user_mask = set_user_cpus_ptr(p, new_mask, + user_mask); + goto out; + } + + if (WARN_ON_ONCE(p == current && + is_migration_disabled(p) && + !cpumask_test_cpu(task_cpu(p), new_mask))) { + ret = -EBUSY; + goto out; + } + } + + /* + * Picking a ~random cpu helps in cases where we are changing affinity + * for groups of tasks (ie. cpuset), so that load balancing is not + * immediately required to distribute the tasks within their new mask. + */ + dest_cpu = cpumask_any_and_distribute(cpu_valid_mask, new_mask); + if (dest_cpu >= nr_cpu_ids) { + ret = -EINVAL; + goto out; + } + + { + struct affinity_context ac = { + .new_mask = new_mask, + .flags = flags, + }; + + __do_set_cpus_allowed(p, &ac); + } + + /* + * migrate_enable() only reinstates cpus_mask, so it is not a change + * of what the task wants and must not disturb what is saved. + */ + if (!(flags & SCA_MIGRATE_ENABLE)) + user_mask = set_user_cpus_ptr(p, new_mask, user_mask); + + if (p->flags & PF_KTHREAD) { + /* + * For kernel threads that do indeed end up on online && + * !active we want to ensure they are strict per-CPU threads. + */ + WARN_ON(cpumask_intersects(new_mask, cpu_online_mask) && + !cpumask_intersects(new_mask, cpu_active_mask) && + p->nr_cpus_allowed != 1); + } + + ret = affine_move_task(rq, p, &rf, dest_cpu, flags); + kfree(user_mask); + + return ret; + +out: + task_rq_unlock(rq, p, &rf); + kfree(user_mask); + + return ret; +} + +int set_cpus_allowed_ptr(struct task_struct *p, const struct cpumask *new_mask) +{ + return __set_cpus_allowed_ptr(p, new_mask, 0); +} +EXPORT_SYMBOL_GPL(set_cpus_allowed_ptr); + +#ifdef CONFIG_HOTPLUG_CPU +/* + * Skip affinity rewrite for tasks whose mask is managed elsewhere: the + * hotplug/stopper threads, idle, and kthreads that forbid setaffinity + * (including KTHREAD_IS_PER_CPU, rebound on unpark). + */ +static bool bind_zero_skip_affinity(struct task_struct *p, struct rq *rq) +{ + return p == current || is_idle_task(p) || p == rq->stop || + (p->flags & PF_NO_SETAFFINITY) || kthread_is_per_cpu(p); +} + +/* + * Move @p off @src_cpu under pi+rq lock, and only override its affinity + * if losing @src_cpu would leave it with no active CPU at all. + * + * Like mainline, the common case does not touch cpus_mask: an offline or + * deactivated CPU is already refused by is_cpu_allowed(), sched_other_cpu(), + * needs_other_cpu() and valid_task_cpu(), so the mask needs no editing and + * the task keeps the affinity userspace asked for across a hotplug cycle. + * Narrowing here would be permanent, as unbind_zero() only restores tasks + * that were forced. + * + * The desperate case still has to put the task somewhere: substitute an + * active fallback (usually CPU0) for @src_cpu and record that with + * zerobound, stashing the mask being replaced in user_cpus_ptr so + * unbind_zero() can put back exactly what userspace asked for once + * @src_cpu returns. + * + * migrate_disable() tasks keep their pin (cpus_ptr) and are left for + * wait_empty to wait out; only cpus_mask is updated so migrate_enable() + * will affine_move them. + */ +static int bind_zero_one(struct task_struct *p, int src_cpu) +{ + cpumask_t *user_mask, new_mask; + struct rq_flags rf; + struct rq *rq; + int dest, bound = 0; + + /* + * The override below cannot allocate under the runqueue lock, so + * speculatively provide it room for the old mask here. Almost every + * task takes the fast path and hands this straight back. + */ + user_mask = kmalloc(cpumask_size(), GFP_ATOMIC); + + rq = task_rq_lock(p, &rf); + + if (!bind_zero_skip_affinity(p, rq) && + cpumask_test_cpu(src_cpu, &p->cpus_mask)) { + cpumask_copy(&new_mask, &p->cpus_mask); + cpumask_clear_cpu(src_cpu, &new_mask); + if (!cpumask_intersects(&new_mask, cpu_active_mask)) { + dest = cpumask_any(cpu_active_mask); + if (dest >= nr_cpu_ids) + dest = cpumask_any(cpu_online_mask); + if (dest < nr_cpu_ids) { + struct affinity_context ac = { + .new_mask = &new_mask + }; + + /* + * Only the first override saves; a second one + * would only be replacing an already forced + * mask with another. What is being displaced + * is what has to come back, so it takes over + * any mask sched_setaffinity() left here - + * reusing its allocation - rather than let a + * request cpusets have since narrowed be + * restored wider than the cpuset allows. + */ + if (!p->zerobound) { + if (p->user_cpus_ptr) { + cpumask_copy(p->user_cpus_ptr, + &p->cpus_mask); + } else if (user_mask) { + cpumask_copy(user_mask, + &p->cpus_mask); + p->user_cpus_ptr = user_mask; + user_mask = NULL; + } + } + cpumask_set_cpu(dest, &new_mask); + p->zerobound = true; + __do_set_cpus_allowed(p, &ac); + bound = 1; + } + } + } + + if (task_cpu(p) == src_cpu && p != current && + !is_idle_task(p) && p != rq->stop && + !is_migration_disabled(p)) { + dest = valid_task_cpu(p); + if (dest != src_cpu) { + /* + * Only a queued task has to be moved. A blocked one is + * on no runqueue, so it does not hold @src_cpu up, and + * its destination must not be stamped into wake_cpu: + * that arms return_task()'s migration handshake, which + * would then fire on a task nobody is migrating. Its + * wakeup picks an allowed CPU by itself. + */ + if (task_queued(p)) + rq = move_queued_task(rq, &rf, p, dest); + } + } + + task_rq_unlock(rq, p, &rf); + kfree(user_mask); + return bound; +} + +/* + * Called from sched_cpu_wait_empty() (sleepable) on the outgoing CPU. + * Walk every task and move anyone still queued here. Affinity is only + * overridden for the few tasks @src_cpu leaving would otherwise strand, + * so on a normal offline this reports nothing. + */ +static void bind_zero(int src_cpu) +{ + struct task_struct *g, *p; + int bound = 0; + + if (src_cpu == 0) + return; + + rcu_read_lock(); + for_each_process_thread(g, p) { + if (task_cpu(p) != src_cpu && + !cpumask_test_cpu(src_cpu, p->cpus_ptr) && + !cpumask_test_cpu(src_cpu, &p->cpus_mask)) + continue; + bound += bind_zero_one(p, src_cpu); + } + rcu_read_unlock(); + + if (bound) { + printk(KERN_INFO "MuQSS overrode affinity for %d processes left with no active cpu by offlining cpu %d\n", + bound, src_cpu); + } +} + +/* + * Undo the affinity override bind_zero() had to force onto tasks that + * @src_cpu leaving would have stranded. Hold pi+rq so cpus_mask / + * nr_cpus_allowed stay in lockstep. Unbound kthreads are eligible — only + * per-CPU / PF_NO_SETAFFINITY threads were skipped on the way out. + * + * Restoring the saved mask can take the CPU the task is on right now away + * from it, so kick it off exactly like do_set_cpus_allowed() does. + */ +static void unbind_zero(int src_cpu) +{ + int restored = 0, unbound = 0; + struct task_struct *g, *p; + + if (src_cpu == 0) + return; + + rcu_read_lock(); + for_each_process_thread(g, p) { + struct rq_flags rf; + struct rq *rq; + + if (!p->zerobound) + continue; + + rq = task_rq_lock(p, &rf); + if (!p->zerobound) { + task_rq_unlock(rq, p, &rf); + continue; + } + + if (likely(p->user_cpus_ptr)) { + /* + * Wait for one of the CPUs the task actually wanted, + * or restoring would strand it all over again. The + * saved mask stays put once copied back: it is still + * what the task wants, and what a later override or + * relax_compatible_cpus_allowed_ptr() restores. + */ + if (cpumask_test_cpu(src_cpu, p->user_cpus_ptr)) { + struct affinity_context ac = { + .new_mask = p->user_cpus_ptr + }; + + __do_set_cpus_allowed(p, &ac); + p->zerobound = false; + restored++; + } + } else if (!cpumask_test_cpu(src_cpu, &p->cpus_mask)) { + /* + * bind_zero() had no room to save the old mask, so + * hand @src_cpu back and stop there rather than + * accumulate every CPU that ever comes online. + */ + cpumask_set_cpu(src_cpu, &p->cpus_mask); + p->nr_cpus_allowed = cpumask_weight(&p->cpus_mask); + p->zerobound = false; + unbound++; + } + + if (!p->zerobound && needs_other_cpu(p, task_cpu(p))) { + int dest = valid_task_cpu(p); + + /* + * As in bind_zero_one(), a blocked task is left alone + * rather than having @dest stamped into its wake_cpu, + * which would arm return_task()'s migration handshake + * for a migration that is not happening. + */ + if (task_queued(p)) + rq = move_queued_task(rq, &rf, p, dest); + else if (task_running(rq, p)) { + set_task_cpu(p, dest); + resched_task(p); + } + } + + task_rq_unlock(rq, p, &rf); + } + rcu_read_unlock(); + + if (restored) { + printk(KERN_INFO "MuQSS restored the original affinity of %d processes by onlining cpu %d\n", + restored, src_cpu); + } + if (unbound) { + printk(KERN_INFO "MuQSS added affinity for %d processes to cpu %d\n", + unbound, src_cpu); + } +} + +/* + * idle_task_exit() is an empty inline in now; the + * outgoing CPU switches back to init_mm from sched_cpu_wait_empty() instead. + */ +#else /* CONFIG_HOTPLUG_CPU */ +static void unbind_zero(int src_cpu) {} +#endif /* CONFIG_HOTPLUG_CPU */ + +void sched_set_stop_task(int cpu, struct task_struct *stop) +{ + struct sched_param stop_param = { .sched_priority = STOP_PRIO }; + struct sched_param start_param = { .sched_priority = 0 }; + struct task_struct *old_stop = cpu_rq(cpu)->stop; + + if (stop) { + /* + * Make it appear like a SCHED_FIFO task, its something + * userspace knows about and won't get confused about. + * + * Also, it will make PI more or less work without too + * much confusion -- but then, stop work should not + * rely on PI working anyway. + */ + sched_setscheduler_nocheck(stop, SCHED_FIFO, &stop_param); + } + + cpu_rq(cpu)->stop = stop; + + if (old_stop) { + /* + * Reset it back to a normal scheduling policy so that + * it can die in pieces. + */ + sched_setscheduler_nocheck(old_stop, SCHED_NORMAL, &start_param); + } +} + +#if defined(CONFIG_SCHED_DEBUG) && defined(CONFIG_SYSCTL) + +static struct ctl_table sd_ctl_dir[] = { + { + .procname = "sched_domain", + .mode = 0555, + }, + {} +}; + +static struct ctl_table sd_ctl_root[] = { + { + .procname = "kernel", + .mode = 0555, + .child = sd_ctl_dir, + }, + {} +}; + +static struct ctl_table *sd_alloc_ctl_entry(int n) +{ + struct ctl_table *entry = + kcalloc(n, sizeof(struct ctl_table), GFP_KERNEL); + + return entry; +} + +static void sd_free_ctl_entry(struct ctl_table **tablep) +{ + struct ctl_table *entry; + + /* + * In the intermediate directories, both the child directory and + * procname are dynamically allocated and could fail but the mode + * will always be set. In the lowest directory the names are + * static strings and all have proc handlers. + */ + for (entry = *tablep; entry->mode; entry++) { + if (entry->child) + sd_free_ctl_entry(&entry->child); + if (entry->proc_handler == NULL) + kfree(entry->procname); + } + + kfree(*tablep); + *tablep = NULL; +} + +static void +set_table_entry(struct ctl_table *entry, + const char *procname, void *data, int maxlen, + umode_t mode, proc_handler *proc_handler) +{ + entry->procname = procname; + entry->data = data; + entry->maxlen = maxlen; + entry->mode = mode; + entry->proc_handler = proc_handler; +} + +static struct ctl_table * +sd_alloc_ctl_domain_table(struct sched_domain *sd) +{ + struct ctl_table *table = sd_alloc_ctl_entry(9); + + if (table == NULL) + return NULL; + + set_table_entry(&table[0], "min_interval", &sd->min_interval, sizeof(long), 0644, proc_doulongvec_minmax); + set_table_entry(&table[1], "max_interval", &sd->max_interval, sizeof(long), 0644, proc_doulongvec_minmax); + set_table_entry(&table[2], "busy_factor", &sd->busy_factor, sizeof(int), 0644, proc_dointvec_minmax); + set_table_entry(&table[3], "imbalance_pct", &sd->imbalance_pct, sizeof(int), 0644, proc_dointvec_minmax); + set_table_entry(&table[4], "cache_nice_tries", &sd->cache_nice_tries, sizeof(int), 0644, proc_dointvec_minmax); + set_table_entry(&table[5], "flags", &sd->flags, sizeof(int), 0644, proc_dointvec_minmax); + set_table_entry(&table[6], "max_newidle_lb_cost", &sd->max_newidle_lb_cost, sizeof(long), 0644, proc_doulongvec_minmax); + set_table_entry(&table[7], "name", sd->name, CORENAME_MAX_SIZE, 0444, proc_dostring); + /* &table[8] is terminator */ + + return table; +} + +static struct ctl_table *sd_alloc_ctl_cpu_table(int cpu) +{ + struct ctl_table *entry, *table; + struct sched_domain *sd; + int domain_num = 0, i; + char buf[32]; + + for_each_domain(cpu, sd) + domain_num++; + entry = table = sd_alloc_ctl_entry(domain_num + 1); + if (table == NULL) + return NULL; + + i = 0; + for_each_domain(cpu, sd) { + snprintf(buf, 32, "domain%d", i); + entry->procname = kstrdup(buf, GFP_KERNEL); + entry->mode = 0555; + entry->child = sd_alloc_ctl_domain_table(sd); + entry++; + i++; + } + return table; +} + +static cpumask_var_t sd_sysctl_cpus; +static struct ctl_table_header *sd_sysctl_header; + +void register_sched_domain_sysctl(void) +{ + static struct ctl_table *cpu_entries; + static struct ctl_table **cpu_idx; + char buf[32]; + int i; + + if (!cpu_entries) { + cpu_entries = sd_alloc_ctl_entry(num_possible_cpus() + 1); + if (!cpu_entries) + return; + + WARN_ON(sd_ctl_dir[0].child); + sd_ctl_dir[0].child = cpu_entries; + } + + if (!cpu_idx) { + struct ctl_table *e = cpu_entries; + + cpu_idx = kcalloc(nr_cpu_ids, sizeof(struct ctl_table*), GFP_KERNEL); + if (!cpu_idx) + return; + + /* deal with sparse possible map */ + for_each_possible_cpu(i) { + cpu_idx[i] = e; + e++; + } + } + + if (!cpumask_available(sd_sysctl_cpus)) { + if (!alloc_cpumask_var(&sd_sysctl_cpus, GFP_KERNEL)) + return; + + /* init to possible to not have holes in @cpu_entries */ + cpumask_copy(sd_sysctl_cpus, cpu_possible_mask); + } + + for_each_cpu(i, sd_sysctl_cpus) { + struct ctl_table *e = cpu_idx[i]; + + if (e->child) + sd_free_ctl_entry(&e->child); + + if (!e->procname) { + snprintf(buf, 32, "cpu%d", i); + e->procname = kstrdup(buf, GFP_KERNEL); + } + e->mode = 0555; + e->child = sd_alloc_ctl_cpu_table(i); + + __cpumask_clear_cpu(i, sd_sysctl_cpus); + } + + WARN_ON(sd_sysctl_header); + sd_sysctl_header = register_sysctl_table(sd_ctl_root); +} + +void dirty_sched_domain_sysctl(int cpu) +{ + if (cpumask_available(sd_sysctl_cpus)) + __cpumask_set_cpu(cpu, sd_sysctl_cpus); +} + +/* may be called multiple times per register */ +void unregister_sched_domain_sysctl(void) +{ + unregister_sysctl_table(sd_sysctl_header); + sd_sysctl_header = NULL; +} +#endif /* CONFIG_SYSCTL */ + +void set_rq_online(struct rq *rq) +{ + if (!rq->online) { + cpumask_set_cpu(cpu_of(rq), rq->rd->online); + rq->online = true; + } +} + +void set_rq_offline(struct rq *rq) +{ + if (rq->online) { + int cpu = cpu_of(rq); + + cpumask_clear_cpu(cpu, rq->rd->online); + rq->online = false; + clear_cpuidle_map(cpu); + } +} + +/* + * used to mark begin/end of suspend/resume: + */ +static int num_cpus_frozen; + +/* + * Update cpusets according to cpu_active mask. If cpusets are + * disabled, cpuset_update_active_cpus() becomes a simple wrapper + * around partition_sched_domains(). + * + * If we come here as part of a suspend/resume, don't touch cpusets because we + * want to restore it back to its original state upon resume anyway. + */ +static void cpuset_cpu_active(void) +{ + if (cpuhp_tasks_frozen) { + /* + * num_cpus_frozen tracks how many CPUs are involved in suspend + * resume sequence. As long as this is not the last online + * operation in the resume sequence, just build a single sched + * domain, ignoring cpusets. + */ + partition_sched_domains(1, NULL, NULL); + if (--num_cpus_frozen) + return; + /* + * This is the last CPU online operation. So fall through and + * restore the original sched domains by considering the + * cpuset configurations. + */ + cpuset_force_rebuild(); + } + + cpuset_update_active_cpus(); +} + +static int cpuset_cpu_inactive(unsigned int cpu) +{ + if (!cpuhp_tasks_frozen) { + cpuset_update_active_cpus(); + } else { + num_cpus_frozen++; + partition_sched_domains(1, NULL, NULL); + } + return 0; +} + +int sched_cpu_activate(unsigned int cpu) +{ + struct rq *rq = cpu_rq(cpu); + struct rq_flags rf; + + /* + * When going up, increment the number of cores with SMT present. + * cpu_smt_mask() is cpumask_of(cpu) when !CONFIG_SCHED_SMT, so this + * never fires there. + */ + if (cpumask_weight(cpu_smt_mask(cpu)) == 2) + static_branch_inc_cpuslocked(&sched_smt_present); + + set_cpu_active(cpu, true); + + if (sched_smp_initialized) { + sched_domains_numa_masks_set(cpu); + cpuset_cpu_active(); + } + + /* + * Put the rq online, if not already. This happens: + * + * 1) In the early boot process, because we build the real domains + * after all CPUs have been brought up. + * + * 2) At runtime, if cpuset_cpu_active() fails to rebuild the + * domains. + */ + rq_lock_irqsave(rq, &rf); + if (rq->rd) { + BUG_ON(!cpumask_test_cpu(cpu, rq->rd->span)); + set_rq_online(rq); + } + rq_unlock_irqrestore(rq, &rf); + /* + * unbind_zero() takes each task's pi+rq lock; do not nest that + * under this CPU's rq lock. + */ + unbind_zero(cpu); + + return 0; +} + +int sched_cpu_deactivate(unsigned int cpu) +{ + struct rq *rq = cpu_rq(cpu); + struct rq_flags rf; + int ret; + + set_cpu_active(cpu, false); + /* + * We've cleared cpu_active_mask, wait for all preempt-disabled and RCU + * users of this state to go away such that all new such users will + * observe it. + * + * Do sync before park smpboot threads to take care the rcu boost case. + */ + synchronize_rcu(); + + sched_domains_free_llc_id(cpu); + + rq_lock_irqsave(rq, &rf); + if (rq->rd) { + update_rq_clock(rq); + BUG_ON(!cpumask_test_cpu(cpu, rq->rd->span)); + set_rq_offline(rq); + } + rq_unlock_irqrestore(rq, &rf); + + /* + * When going down, decrement the number of cores with SMT present. + */ + if (cpumask_weight(cpu_smt_mask(cpu)) == 2) + static_branch_dec_cpuslocked(&sched_smt_present); + + if (!sched_smp_initialized) + return 0; + + ret = cpuset_cpu_inactive(cpu); + if (ret) { + set_cpu_active(cpu, true); + return ret; + } + sched_domains_numa_masks_clear(cpu); + return 0; +} + +int sched_cpu_starting(unsigned int cpu) +{ + sched_tick_start(cpu); + return 0; +} + +#ifdef CONFIG_HOTPLUG_CPU +/* + * Invoked on the outgoing CPU in context of the CPU hotplug thread after + * ensuring that there are no user space tasks left on the CPU. + * + * If there is a lazy mm in use on the hotplug thread, drop it and switch to + * init_mm. finish_cpu() on the control CPU drops the init_mm reference and + * WARNs if we left anything else behind. 5.12 did this from idle_task_exit(); + * mainline now does it here from sched_cpu_wait_empty(). + */ +static void __sched_force_init_mm(void) +{ + struct mm_struct *mm = current->active_mm; + + if (mm == &init_mm) + return; + + mmgrab_lazy_tlb(&init_mm); + current->active_mm = &init_mm; + switch_mm_irqs_off(mm, &init_mm, current); + finish_arch_post_lock_switch(); + mmdrop_lazy_tlb(mm); +} + +static void sched_force_init_mm(void) +{ + unsigned long flags; + + local_irq_save(flags); + __sched_force_init_mm(); + local_irq_restore(flags); +} + +static void dump_rq_tasks(struct rq *rq, const char *loglvl) +{ + struct task_struct *g, *p; + int cpu = cpu_of(rq); + + lockdep_assert_rq_held(rq); + + printk("%sCPU%d tasks (nr_running=%u nr_pinned=%u):\n", + loglvl, cpu, rq->nr_running, rq->nr_pinned); + rcu_read_lock(); + for_each_process_thread(g, p) { + if (task_cpu(p) != cpu) + continue; + printk("%s\tpid: %d, name: %s, queued: %d, on_rq: %d\n", + loglvl, p->pid, p->comm, task_queued(p), p->on_rq); + } + rcu_read_unlock(); +} + +/* + * Invoked on the outgoing CPU after per-CPU kthreads have been parked. + * bind_zero() takes affinity off this CPU and moves anyone still queued + * here. Then wait until only this hotplug thread remains and every + * migrate_disable() pin has dropped, so finish_cpu() cannot see a user mm. + */ +int sched_cpu_wait_empty(unsigned int cpu) +{ + struct rq *rq = cpu_rq(cpu); + + WARN_ON_ONCE(cpu != smp_processor_id()); + + for (;;) { + bind_zero(cpu); + if (READ_ONCE(rq->nr_running) <= 1 && !READ_ONCE(rq->nr_pinned)) + break; + schedule_timeout_uninterruptible(1); + } + + sched_force_init_mm(); + return 0; +} + +int sched_cpu_dying(unsigned int cpu) +{ + struct rq *rq = cpu_rq(cpu); + struct rq_flags rf; + + /* Handle pending wakeups; tasks were already moved in wait_empty. */ + sched_tick_stop(cpu); + + rq_lock_irqsave(rq, &rf); + if (rq->rd) { + BUG_ON(!cpumask_test_cpu(cpu, rq->rd->span)); + set_rq_offline(rq); + } + if (rq->nr_running > 1 || rq->nr_pinned) { + WARN(true, "Dying CPU not properly vacated!"); + dump_rq_tasks(rq, KERN_WARNING); + } + rq_unlock_irqrestore(rq, &rf); + + sched_start_tick(rq, cpu); + hrexpiry_clear(rq); + /* + * Belt-and-suspenders: wait_empty already switched to init_mm, + * but a preempting migrate_disable() task could have left a + * lazy user mm on current. Drop it again now that this is the + * last thing to run before idle takes over. + */ + local_irq_disable(); + __sched_force_init_mm(); + local_irq_enable(); + + return 0; +} +#endif + +#if defined(CONFIG_SCHED_SMT) || defined(CONFIG_SCHED_MC) +/* + * Cheaper version of the below functions in case support for SMT and MC is + * compiled in but CPUs have no siblings. + */ +static bool sole_cpu_idle(struct rq *rq) +{ + return rq_idle(rq); +} +#endif +#ifdef CONFIG_SCHED_SMT +static const cpumask_t *thread_cpumask(int cpu) +{ + return topology_sibling_cpumask(cpu); +} +#endif +#ifdef CONFIG_SCHED_MC +static const cpumask_t *core_cpumask(int cpu) +{ + return topology_core_cpumask(cpu); +} +/* All this CPU's shared cache siblings are idle */ +static bool cache_cpu_idle(struct rq *rq) +{ + return cpumask_subset(&rq->core_mask, &cpu_idle_map); +} +/* MC siblings CPU mask which share the same LLC */ +static const cpumask_t *llc_core_cpumask(int cpu) +{ +#ifdef CONFIG_X86 + return per_cpu(cpu_llc_shared_map, cpu); +#else + return topology_core_cpumask(cpu); +#endif +} +#endif + +enum sched_domain_level { + SD_LV_NONE = 0, + SD_LV_SIBLING, + SD_LV_MC, + SD_LV_BOOK, + SD_LV_CPU, + SD_LV_NODE, + SD_LV_ALLNODES, + SD_LV_MAX +}; + +#ifdef CONFIG_SMT_NICE +/* + * Recorded by select_leaders() and consumed by sched_init_smp() once the + * runqueue locks have been dropped. See the comment where it is set. + */ +static bool __initdata smt_nice_needed; +#endif + +/* + * Set up the relative cache distance of each online cpu from each + * other in a simple array for quick lookup. Locality is determined + * by the closest sched_domain that CPUs are separated by. CPUs with + * shared cache in SMT and MC are treated as local. Separate CPUs + * (within the same package or physically) within the same node are + * treated as not local. CPUs not even in the same domain (different + * nodes) are treated as very distant. + * + * Called with interrupts disabled and every runqueue lock held, so nothing + * here may sleep or wait on another CPU. + */ +static void __init select_leaders(void) +{ + struct rq *rq, *other_rq, *leader; + struct sched_domain *sd; + int cpu, other_cpu; +#ifdef CONFIG_SCHED_SMT + bool smt_threads = false; +#endif + + for (cpu = 0; cpu < num_online_cpus(); cpu++) { + rq = cpu_rq(cpu); + leader = NULL; + /* First check if this cpu is in the same node */ + for_each_domain(cpu, sd) { + if (sd->level > SD_LV_MC) + continue; + if (rqshare != RQSHARE_ALL) + leader = NULL; + /* Set locality to local node if not already found lower */ + for_each_cpu(other_cpu, sched_domain_span(sd)) { + if (rqshare >= RQSHARE_SMP) { + other_rq = cpu_rq(other_cpu); + + /* Set the smp_leader to the first CPU */ + if (!leader) + leader = rq; + if (!other_rq->smp_leader) + other_rq->smp_leader = leader; + } + if (rq->cpu_locality[other_cpu] > LOCALITY_SMP) + rq->cpu_locality[other_cpu] = LOCALITY_SMP; + } + } + + /* + * Each runqueue has its own function in case it doesn't have + * siblings of its own allowing mixed topologies. + */ +#ifdef CONFIG_SCHED_MC + leader = NULL; + if (cpumask_weight(core_cpumask(cpu)) > 1) { + cpumask_copy(&rq->core_mask, llc_core_cpumask(cpu)); + cpumask_clear_cpu(cpu, &rq->core_mask); + for_each_cpu(other_cpu, core_cpumask(cpu)) { + if (rqshare == RQSHARE_MC || + (rqshare == RQSHARE_MC_LLC && cpumask_test_cpu(other_cpu, llc_core_cpumask(cpu)))) { + other_rq = cpu_rq(other_cpu); + + /* Set the mc_leader to the first CPU */ + if (!leader) + leader = rq; + if (!other_rq->mc_leader) + other_rq->mc_leader = leader; + } + if (rq->cpu_locality[other_cpu] > LOCALITY_MC) { + /* this is to get LLC into play even in case LLC sharing is not used */ + if (cpumask_test_cpu(other_cpu, llc_core_cpumask(cpu))) + rq->cpu_locality[other_cpu] = LOCALITY_MC_LLC; + else + rq->cpu_locality[other_cpu] = LOCALITY_MC; + } + } + rq->cache_idle = cache_cpu_idle; + } +#endif +#ifdef CONFIG_SCHED_SMT + leader = NULL; + if (cpumask_weight(thread_cpumask(cpu)) > 1) { + cpumask_copy(&rq->thread_mask, thread_cpumask(cpu)); + cpumask_clear_cpu(cpu, &rq->thread_mask); + for_each_cpu(other_cpu, thread_cpumask(cpu)) { + if (rqshare == RQSHARE_SMT) { + other_rq = cpu_rq(other_cpu); + + /* Set the smt_leader to the first CPU */ + if (!leader) + leader = rq; + if (!other_rq->smt_leader) + other_rq->smt_leader = leader; + } + if (rq->cpu_locality[other_cpu] > LOCALITY_SMT) + rq->cpu_locality[other_cpu] = LOCALITY_SMT; + } + rq->has_smt_sibling = true; + smt_threads = true; + } +#endif + } + +#ifdef CONFIG_SMT_NICE + /* + * Only record it here; sched_init_smp() turns SMT nice on once it has + * dropped the runqueue locks and re-enabled interrupts. + * + * static_branch_enable() must not be called from this context. It + * takes cpus_read_lock() and jump_label_mutex, either of which can + * sleep - and scheduling from here would try to take a runqueue lock + * this CPU already holds - and it then patches the jump site with + * text_poke_bp(), which waits for every other CPU to answer a sync + * IPI. Any CPU spinning on one of the runqueue locks held here has + * interrupts disabled and can never answer, so the wait never ends and + * the machine is gone with nothing on the console. + */ + smt_nice_needed = smt_threads; +#endif + + for_each_online_cpu(cpu) { + rq = cpu_rq(cpu); + for_each_online_cpu(other_cpu) { + printk(KERN_DEBUG "MuQSS locality CPU %d to %d: %d\n", cpu, other_cpu, rq->cpu_locality[other_cpu]); + } + } +} + +/* + * Fold @rq into @leader while all runqueue locks are already held by + * lock_all_rqs(). Secondary CPUs may already have tasks queued by the time + * sched_init_smp() runs (hotplug kthreads, RCU, early kworkers), so the + * skiplist must be drained onto the leader before its storage is freed. + * + * Lock handoff: rq_lock() is a plain raw_spin_lock(rq->lock) with no re-check, + * so a CPU that had already read the follower's lock pointer would acquire the + * old lock after we drop it and then walk the leader's skiplist without the + * leader's lock. share_rqs() therefore runs from stop_machine(), where no + * other CPU can be inside or waiting on any rq lock, which is what makes + * repointing rq->lock and freeing the old one safe. + */ +static void __init share_and_free_rq(struct rq *leader, struct rq *rq) +{ + raw_spinlock_t *old_lock = rq->lock; + skiplist_node *old_node = rq->node; + skiplist *old_sl = rq->sl; + + /* + * Move every queued task onto the leader skiplist. nr_running is + * accounted against the rq the task belongs to (task_rq(p), which is + * unchanged here) rather than the rq owning the skiplist, so hand the + * counts straight back or the follower underflows on the next dequeue + * and the leader keeps a phantom entry forever. + */ + while (rq->sl->entries > 0) { + struct task_struct *p = container_of(rq->node->next[0], + struct task_struct, node); + + dequeue_task(rq, p, DEQUEUE_SAVE); + enqueue_task(leader, p, ENQUEUE_RESTORE); + leader->nr_running--; + rq->nr_running++; + if (rt_task(p)) { + leader->rt_nr_running--; + rq->rt_nr_running++; + } + } + + /* + * A non-idle curr is not on the skiplist but still accounts for one + * nr_running. Leave that count on @rq; only the shared skiplist and + * lock are merged. + */ + WARN_ON_ONCE(rq->sl->entries != 0); + + /* Point the follower at the leader's skiplist and lock. */ + rq->node = leader->node; + rq->sl = leader->sl; + rq->lock = leader->lock; + rq->is_leader = false; + + /* + * Drop the follower's private lock, taken by lock_all_rqs(). Only the + * leader lock remains held for @rq, and nothing can be waiting on the + * old one, so free it. + */ + do_raw_spin_unlock(old_lock); + + kfree(old_node); + skiplist_free(old_sl); + kfree(old_lock); +} + +/* + * Called with every runqueue lock held via lock_all_rqs() and IRQs off. + * Must not take locks again. After this, only leader rqs own a unique lock; + * unlock_all_rqs() is replaced by unlock_leader_rqs(). + */ +static void __init share_rqs(void) +{ + struct rq *rq, *leader; + int cpu; + + for_each_online_cpu(cpu) { + rq = cpu_rq(cpu); + leader = rq->smp_leader; + + if (leader && rq != leader) { + printk(KERN_INFO "MuQSS sharing SMP runqueue from CPU %d to CPU %d\n", + leader->cpu, rq->cpu); + share_and_free_rq(leader, rq); + } + } + +#ifdef CONFIG_SCHED_MC + for_each_online_cpu(cpu) { + rq = cpu_rq(cpu); + leader = rq->mc_leader; + + if (leader && rq != leader) { + printk(KERN_INFO "MuQSS sharing MC runqueue from CPU %d to CPU %d\n", + leader->cpu, rq->cpu); + share_and_free_rq(leader, rq); + } + } +#endif /* CONFIG_SCHED_MC */ + +#ifdef CONFIG_SCHED_SMT + for_each_online_cpu(cpu) { + rq = cpu_rq(cpu); + leader = rq->smt_leader; + + if (leader && rq != leader) { + printk(KERN_INFO "MuQSS sharing SMT runqueue from CPU %d to CPU %d\n", + leader->cpu, rq->cpu); + share_and_free_rq(leader, rq); + } + } +#endif /* CONFIG_SCHED_SMT */ +} + +/* Unlock each unique runqueue lock once after share_rqs(). */ +static inline void unlock_leader_rqs(void) +{ + int cpu; + + for_each_possible_cpu(cpu) { + struct rq *rq = cpu_rq(cpu); + + if (rq->is_leader) + do_raw_spin_unlock(rq->lock); + } + preempt_enable(); +} + +/* + * Fold the runqueues from stop_machine() context. Every other CPU is parked + * in the stopper with interrupts disabled, so none of them is inside an rq + * lock or spinning on one while share_and_free_rq() repoints a follower's + * rq->lock at its leader and frees the old lock. Any single CPU can run this. + */ +static int __init share_rqs_stopper(void *unused) +{ + lock_all_rqs(); + share_rqs(); + unlock_leader_rqs(); + + return 0; +} + +static void __init set_rq_order(struct rq *rq, int idx, struct rq *other) +{ + rq->rq_order[idx] = other; + rq->sl_order[idx] = other->sl; +} + +static void __init setup_rq_orders(void) +{ + int *selected_cpus, *ordered_cpus; + struct rq *rq, *other_rq; + int cpu, other_cpu, i; + + selected_cpus = kmalloc(sizeof(int) * NR_CPUS, GFP_ATOMIC); + ordered_cpus = kmalloc(sizeof(int) * NR_CPUS, GFP_ATOMIC); + + total_runqueues = 0; + for_each_online_cpu(cpu) { + int locality, total_rqs = 0, total_cpus = 0; + + rq = cpu_rq(cpu); + if (rq->is_leader) + total_runqueues++; + + for (locality = LOCALITY_SAME; locality <= LOCALITY_DISTANT; locality++) { + int selected_cpu_cnt, selected_cpu_idx, test_cpu_idx, cpu_idx, best_locality, test_cpu; + int ordered_cpus_idx; + + ordered_cpus_idx = -1; + selected_cpu_cnt = 0; + + for_each_online_cpu(test_cpu) { + if (cpu < num_online_cpus() / 2) + other_cpu = cpu + test_cpu; + else + other_cpu = cpu - test_cpu; + if (other_cpu < 0) + other_cpu += num_online_cpus(); + else + other_cpu %= num_online_cpus(); + /* gather CPUs of the same locality */ + if (rq->cpu_locality[other_cpu] == locality) { + selected_cpus[selected_cpu_cnt] = other_cpu; + selected_cpu_cnt++; + } + } + + /* reserve first CPU as starting point */ + if (selected_cpu_cnt > 0) { + ordered_cpus_idx++; + ordered_cpus[ordered_cpus_idx] = selected_cpus[ordered_cpus_idx]; + selected_cpus[ordered_cpus_idx] = -1; + } + + /* take each CPU and sort it within the same locality based on each inter-CPU localities */ + for (test_cpu_idx = 1; test_cpu_idx < selected_cpu_cnt; test_cpu_idx++) { + /* starting point with worst locality and current CPU */ + best_locality = LOCALITY_DISTANT; + selected_cpu_idx = test_cpu_idx; + + /* try to find the best locality within group */ + for (cpu_idx = 1; cpu_idx < selected_cpu_cnt; cpu_idx++) { + /* if CPU has not been used and locality is better */ + if (selected_cpus[cpu_idx] > -1) { + other_rq = cpu_rq(ordered_cpus[ordered_cpus_idx]); + if (best_locality > other_rq->cpu_locality[selected_cpus[cpu_idx]]) { + /* assign best locality and best CPU idx in array */ + best_locality = other_rq->cpu_locality[selected_cpus[cpu_idx]]; + selected_cpu_idx = cpu_idx; + } + } + } + + /* add our next best CPU to ordered list */ + ordered_cpus_idx++; + ordered_cpus[ordered_cpus_idx] = selected_cpus[selected_cpu_idx]; + /* mark this CPU as used */ + selected_cpus[selected_cpu_idx] = -1; + } + + /* set up RQ and CPU orders */ + for (test_cpu = 0; test_cpu <= ordered_cpus_idx; test_cpu++) { + other_rq = cpu_rq(ordered_cpus[test_cpu]); + /* set up cpu orders */ + rq->cpu_order[total_cpus++] = other_rq; + if (other_rq->is_leader) { + /* set up RQ orders */ + set_rq_order(rq, total_rqs++, other_rq); + } + } + } + } + + kfree(selected_cpus); + kfree(ordered_cpus); + +#ifdef CONFIG_X86 + for_each_online_cpu(cpu) { + rq = cpu_rq(cpu); + for (i = 0; i < total_runqueues; i++) { + printk(KERN_DEBUG "MuQSS CPU %d llc %d RQ order %d RQ %d llc %d\n", cpu, per_cpu_llc_id(cpu), i, + rq->rq_order[i]->cpu, per_cpu_llc_id(rq->rq_order[i]->cpu)); + } + } + + for_each_online_cpu(cpu) { + rq = cpu_rq(cpu); + for (i = 0; i < num_online_cpus(); i++) { + printk(KERN_DEBUG "MuQSS CPU %d llc %d CPU order %d RQ %d llc %d\n", cpu, per_cpu_llc_id(cpu), i, + rq->cpu_order[i]->cpu, per_cpu_llc_id(rq->cpu_order[i]->cpu)); + } + } +#endif +} + +void __init sched_init_smp(void) +{ + sched_init_numa(NUMA_NO_NODE); + + /* + * There's no userspace yet to cause hotplug operations; hence all the + * cpu masks are stable and all blatant races in the below code cannot + * happen. + */ + mutex_lock(&sched_domains_mutex); + sched_init_domains(cpu_active_mask); + mutex_unlock(&sched_domains_mutex); + + /* Move init over to a non-isolated CPU */ + if (set_cpus_allowed_ptr(current, housekeeping_cpumask(HK_TYPE_DOMAIN)) < 0) + BUG(); + + /* + * Take the sleeping lock before disabling interrupts - the 5.12 order + * trips "sleeping function called from invalid context" under + * CONFIG_DEBUG_ATOMIC_SLEEP. select_leaders() walks the domain tree, + * which is what the mutex is for. + */ + mutex_lock(&sched_domains_mutex); + local_irq_disable(); + lock_all_rqs(); + + printk(KERN_INFO "MuQSS possible/present/online CPUs: %d/%d/%d\n", + num_possible_cpus(), num_present_cpus(), num_online_cpus()); + + select_leaders(); + + unlock_all_rqs(); + local_irq_enable(); + mutex_unlock(&sched_domains_mutex); + +#ifdef CONFIG_SMT_NICE + /* + * Safe to patch the jump site only now that the runqueue locks are + * dropped and interrupts are back on; see select_leaders(). Until + * this point smt_schedule() just returns true and check_siblings()/ + * wake_siblings() are the no-op variants, which is correct behaviour, + * merely without SMT nice. + */ + if (smt_nice_needed) { + check_siblings = &check_smt_siblings; + wake_siblings = &wake_smt_siblings; + static_branch_enable(&smt_nice_enabled); + } +#endif + + /* + * Only now fold the runqueues together, and do it from stop_machine(): + * 7.1 has the secondary CPUs running by this point, so a follower's rq + * lock can be held - or waited on - by another CPU exactly while it is + * handed over to the leader and freed. Quiescing everybody closes that + * window without putting a re-check in the rq_lock() fast path. Note + * stop_machine() takes cpus_read_lock(), so it must not nest inside + * sched_domains_mutex. + */ + stop_machine(share_rqs_stopper, NULL, cpumask_of(raw_smp_processor_id())); + + setup_rq_orders(); + + switch (rqshare) { + case RQSHARE_ALL: + /* This should only ever read 1 */ + printk(KERN_INFO "MuQSS runqueue share type ALL total runqueues: %d\n", + total_runqueues); + break; + case RQSHARE_SMP: + printk(KERN_INFO "MuQSS runqueue share type SMP total runqueues: %d\n", + total_runqueues); + break; + case RQSHARE_MC: + printk(KERN_INFO "MuQSS runqueue share type MC total runqueues: %d\n", + total_runqueues); + break; + case RQSHARE_MC_LLC: + printk(KERN_INFO "MuQSS runqueue share type LLC total runqueues: %d\n", + total_runqueues); + break; + case RQSHARE_SMT: + printk(KERN_INFO "MuQSS runqueue share type SMT total runqueues: %d\n", + total_runqueues); + break; + case RQSHARE_NONE: + printk(KERN_INFO "MuQSS runqueue share type NONE total runqueues: %d\n", + total_runqueues); + break; + } + + sched_smp_initialized = true; +} +#else /* !CONFIG_SMP */ +void __init sched_init_smp(void) +{ + sched_smp_initialized = true; +} + +/* + * 7.1 dropped the !CONFIG_SMP inline stubs these used to have in + * , so the scheduler has to provide them on UP too. There is + * only ever cpu 0 to run on, so affinity is either trivially satisfied or + * impossible. + */ +int set_cpus_allowed_ptr(struct task_struct *p, const struct cpumask *new_mask) +{ + if (!cpumask_test_cpu(0, new_mask)) + return -EINVAL; + return 0; +} +EXPORT_SYMBOL_GPL(set_cpus_allowed_ptr); + +void set_cpus_allowed_force(struct task_struct *p, const struct cpumask *new_mask) +{ + cpumask_copy(&p->cpus_mask, new_mask); + p->nr_cpus_allowed = cpumask_weight(new_mask); +} + +#ifdef CONFIG_NO_HZ_COMMON +void nohz_balance_enter_idle(int cpu) {} + +/* + * The only CPU is the one already running this, so it is by definition not + * idle and needs no kick. + */ +void wake_up_nohz_cpu(int cpu) +{ +} +#endif /* CONFIG_NO_HZ_COMMON */ + +/* + * topology.c is not built on UP, but declares this + * unconditionally and amd-pstate calls it. There are no sched domains to + * update. + */ +void sched_update_asym_prefer_cpu(int cpu, int old_prio, int new_prio) +{ +} +#endif /* CONFIG_SMP */ + +int in_sched_functions(unsigned long addr) +{ + return in_lock_functions(addr) || + (addr >= (unsigned long)__sched_text_start + && addr < (unsigned long)__sched_text_end); +} + +#ifdef CONFIG_CGROUP_SCHED +/* + * Default CFS bandwidth period (100ms). Used only for accept-and-ignore + * readback of cpu.max / cfs_period_us — MuQSS does not enforce quotas. + */ +#define MUQSS_CGROUP_PERIOD_DFL_US 100000ULL + +/* task group related information */ +struct task_group { + struct cgroup_subsys_state css; + + struct rcu_head rcu; + struct list_head list; + + struct task_group *parent; + struct list_head siblings; + struct list_head children; + + /* + * cpu controller knobs stored for ABI/readback only. Writes are + * validated then ignored for scheduling — MuQSS has no group fairness + * or bandwidth enforcement. Keeps systemd/docker/podman happy. + */ + unsigned long weight; /* cgroup weight [CGROUP_WEIGHT_MIN, MAX] */ + s64 idle; + u64 period_us; + s64 quota_us; /* -1 == unlimited ("max") */ + u64 burst_us; +}; + +/* + * Default task group. + * Every task in system belongs to this group at bootup. + */ +struct task_group root_task_group; +LIST_HEAD(task_groups); + +/* Cacheline aligned slab cache for task_group */ +static struct kmem_cache *task_group_cache __read_mostly; + +static void init_tg_cgroup_defaults(struct task_group *tg) +{ + tg->weight = CGROUP_WEIGHT_DFL; + tg->idle = 0; + tg->period_us = MUQSS_CGROUP_PERIOD_DFL_US; + tg->quota_us = -1; + tg->burst_us = 0; +} +#endif /* CONFIG_CGROUP_SCHED */ + +void __init sched_init(void) +{ +#ifdef CONFIG_SMP + int cpu_ids; +#endif + int i; + struct rq *rq; + + wait_bit_init(); + + prio_ratios[0] = 128; + for (i = 1 ; i < NICE_WIDTH ; i++) + prio_ratios[i] = prio_ratios[i - 1] * 11 / 10; + + skiplist_node_init(&init_task.node); + +#ifdef CONFIG_SMP + init_defrootdomain(); + cpumask_clear(&cpu_idle_map); +#else + uprq = &per_cpu(runqueues, 0); +#endif + +#ifdef CONFIG_CGROUP_SCHED + task_group_cache = KMEM_CACHE(task_group, 0); + + list_add(&root_task_group.list, &task_groups); + INIT_LIST_HEAD(&root_task_group.children); + INIT_LIST_HEAD(&root_task_group.siblings); + init_tg_cgroup_defaults(&root_task_group); +#endif /* CONFIG_CGROUP_SCHED */ + skiplist_cache_init(); + for_each_possible_cpu(i) { + rq = cpu_rq(i); + rq->node = kmalloc(sizeof(skiplist_node), GFP_ATOMIC); + skiplist_init(rq->node); + rq->sl = new_skiplist(rq->node); + rq->lock = kmalloc(sizeof(raw_spinlock_t), GFP_ATOMIC); + raw_spin_lock_init(rq->lock); + rq->nr_running = 0; + rq->nr_uninterruptible = 0; + rq->nr_switches = 0; + rq->clock = rq->niffies = rq->jiffy_niffies = 0; + rq->last_jiffy = jiffies; + rq->user_ns = rq->nice_ns = rq->softirq_ns = rq->system_ns = + rq->iowait_ns = rq->idle_ns = 0; + rq->dither = 0; + set_rq_task(rq, &init_task); + rq->iso_ticks = 0; + rq->iso_refractory = false; +#ifdef CONFIG_SMP + rq->is_leader = true; + rq->smp_leader = NULL; +#ifdef CONFIG_SCHED_MC + rq->mc_leader = NULL; +#endif +#ifdef CONFIG_SCHED_SMT + rq->smt_leader = NULL; +#endif + rq->sd = NULL; + rq->rd = NULL; + rq->online = false; + rq->cpu = i; + rq_attach_root(rq, &def_root_domain); +#endif /* CONFIG_SMP */ + init_rq_hrexpiry(rq); + atomic_set(&rq->nr_iowait, 0); + } + +#ifdef CONFIG_SMP + cpu_ids = i; + /* + * Set the base locality for cpu cache distance calculation to + * "distant" (3). Make sure the distance from a CPU to itself is 0. + */ + for_each_possible_cpu(i) { + int j; + + rq = cpu_rq(i); +#ifdef CONFIG_SCHED_MC + rq->cache_idle = sole_cpu_idle; +#endif + rq->cpu_locality = kmalloc(cpu_ids * sizeof(int *), GFP_ATOMIC); + for_each_possible_cpu(j) { + if (i == j) + rq->cpu_locality[j] = LOCALITY_SAME; + else + rq->cpu_locality[j] = LOCALITY_DISTANT; + } + /* sl_order is O(possible_cpus²) pointers, same as rq_order. */ + rq->rq_order = kmalloc(cpu_ids * sizeof(struct rq *), GFP_ATOMIC); + rq->sl_order = kmalloc(cpu_ids * sizeof(skiplist *), GFP_ATOMIC); + rq->cpu_order = kmalloc(cpu_ids * sizeof(struct rq *), GFP_ATOMIC); + set_rq_order(rq, 0, rq); + rq->cpu_order[0] = rq; + for (j = 1; j < cpu_ids; j++) { + set_rq_order(rq, j, cpu_rq(j)); + rq->cpu_order[j] = cpu_rq(j); + } + } +#endif + + /* + * The boot idle thread does lazy MMU switching as well: + */ + mmgrab_lazy_tlb(&init_mm); + enter_lazy_tlb(&init_mm, current); + + /* + * Make us the idle thread. Technically, schedule() should not be + * called from this thread, however somewhere below it might be, + * but because we are the idle thread, we just pick up running again + * when this runqueue becomes "idle". + */ + init_idle(current, smp_processor_id()); + +#ifdef CONFIG_SMP + idle_thread_set_boot_cpu(); +#endif /* SMP */ + + init_schedstats(); + + psi_init(); + + preempt_dynamic_init(); + + print_scheduler_version(); +} + +#ifdef CONFIG_DEBUG_ATOMIC_SLEEP +static inline int preempt_count_equals(int preempt_offset) +{ + int nested = preempt_count() + rcu_preempt_depth(); + + return (nested == preempt_offset); +} + +void __might_sleep(const char *file, int line) +{ + unsigned int state = get_current_state(); + /* + * Blocking primitives will set (and therefore destroy) current->__state, + * since we will exit with TASK_RUNNING make sure we enter with it, + * otherwise we will destroy state. + */ + WARN_ONCE(state != TASK_RUNNING && current->task_state_change, + "do not call blocking ops when !TASK_RUNNING; " + "state=%x set at [<%p>] %pS\n", state, + (void *)current->task_state_change, + (void *)current->task_state_change); + + __might_resched(file, line, 0); +} +EXPORT_SYMBOL(__might_sleep); + +void __cant_migrate(const char *file, int line) +{ + static unsigned long prev_jiffy; + + if (irqs_disabled()) + return; + + if (is_migration_disabled(current)) + return; + + if (!IS_ENABLED(CONFIG_PREEMPT_COUNT)) + return; + + if (preempt_count() > 0) + return; + + if (time_before(jiffies, prev_jiffy + HZ) && prev_jiffy) + return; + prev_jiffy = jiffies; + + pr_err("BUG: assuming non migratable context at %s:%d\n", file, line); + pr_err("in_atomic(): %d, irqs_disabled(): %d, migration_disabled() %u pid: %d, name: %s\n", + in_atomic(), irqs_disabled(), is_migration_disabled(current), + current->pid, current->comm); + + debug_show_held_locks(current); + dump_stack(); + add_taint(TAINT_WARN, LOCKDEP_STILL_OK); +} +EXPORT_SYMBOL_GPL(__cant_migrate); + +void __cant_sleep(const char *file, int line, int preempt_offset) +{ + static unsigned long prev_jiffy; + + if (irqs_disabled()) + return; + + if (!IS_ENABLED(CONFIG_PREEMPT_COUNT)) + return; + + if (preempt_count() > preempt_offset) + return; + + if (time_before(jiffies, prev_jiffy + HZ) && prev_jiffy) + return; + prev_jiffy = jiffies; + + printk(KERN_ERR "BUG: assuming atomic context at %s:%d\n", file, line); + printk(KERN_ERR "in_atomic(): %d, irqs_disabled(): %d, pid: %d, name: %s\n", + in_atomic(), irqs_disabled(), + current->pid, current->comm); + + debug_show_held_locks(current); + dump_stack(); + add_taint(TAINT_WARN, LOCKDEP_STILL_OK); +} +EXPORT_SYMBOL_GPL(__cant_sleep); + +void __might_resched(const char *file, int line, unsigned int offsets) +{ + /* Ratelimiting timestamp: */ + static unsigned long prev_jiffy; + + unsigned long preempt_disable_ip; + + /* WARN_ON_ONCE() by default, no rate limit required: */ + rcu_sleep_check(); + + if ((preempt_count_equals(offsets) && !irqs_disabled() && + !is_idle_task(current) && !current->non_block_count) || + system_state == SYSTEM_BOOTING || system_state > SYSTEM_RUNNING || + oops_in_progress) + return; + + if (time_before(jiffies, prev_jiffy + HZ) && prev_jiffy) + return; + prev_jiffy = jiffies; + + /* Save this before calling printk(), since that will clobber it: */ + preempt_disable_ip = get_preempt_disable_ip(current); + + printk(KERN_ERR + "BUG: sleeping function called from invalid context at %s:%d\n", + file, line); + printk(KERN_ERR + "in_atomic(): %d, irqs_disabled(): %d, non_block: %d, pid: %d, name: %s\n", + in_atomic(), irqs_disabled(), current->non_block_count, + current->pid, current->comm); + + if (task_stack_end_corrupted(current)) + printk(KERN_EMERG "Thread overran stack, or stack corrupted\n"); + + debug_show_held_locks(current); + if (irqs_disabled()) + print_irqtrace_events(current); + if (IS_ENABLED(CONFIG_DEBUG_PREEMPT) + && !preempt_count_equals(offsets)) { + pr_err("Preemption disabled at:"); + print_ip_sym(KERN_ERR, preempt_disable_ip); + } + dump_stack(); + add_taint(TAINT_WARN, LOCKDEP_STILL_OK); +} +EXPORT_SYMBOL(__might_resched); +#endif + +#ifdef CONFIG_MAGIC_SYSRQ +static inline void normalise_rt_tasks(void) +{ + struct sched_attr attr = {}; + struct task_struct *g, *p; + struct rq_flags rf; + struct rq *rq; + + read_lock(&tasklist_lock); + for_each_process_thread(g, p) { + /* + * Only normalize user tasks: + */ + if (p->flags & PF_KTHREAD) + continue; + + if (!rt_task(p) && !iso_task(p)) + continue; + + rq = task_rq_lock(p, &rf); + __setscheduler(p, rq, SCHED_NORMAL, 0, &attr, false); + task_rq_unlock(rq, p, &rf); + } + read_unlock(&tasklist_lock); +} + +void normalize_rt_tasks(void) +{ + normalise_rt_tasks(); +} +#endif /* CONFIG_MAGIC_SYSRQ */ + +#if defined(CONFIG_IA64) || defined(CONFIG_KGDB_KDB) +/* + * These functions are only useful for the IA64 MCA handling, or kdb. + * + * They can only be called when the whole system has been + * stopped - every CPU needs to be quiescent, and no scheduling + * activity can take place. Using them for anything else would + * be a serious bug, and as a result, they aren't even visible + * under any other configuration. + */ + +/** + * curr_task - return the current task for a given CPU. + * @cpu: the processor in question. + * + * ONLY VALID WHEN THE WHOLE SYSTEM IS STOPPED! + * + * Return: The current task for @cpu. + */ +struct task_struct *curr_task(int cpu) +{ + return cpu_curr(cpu); +} + +#endif /* defined(CONFIG_IA64) || defined(CONFIG_KGDB_KDB) */ + +#ifdef CONFIG_IA64 +/** + * ia64_set_curr_task - set the current task for a given CPU. + * @cpu: the processor in question. + * @p: the task pointer to set. + * + * Description: This function must only be used when non-maskable interrupts + * are serviced on a separate stack. It allows the architecture to switch the + * notion of the current task on a CPU in a non-blocking manner. This function + * must be called with all CPU's synchronised, and interrupts disabled, the + * and caller must save the original value of the current task (see + * curr_task() above) and restore that value before reenabling interrupts and + * re-starting the system. + * + * ONLY VALID WHEN THE WHOLE SYSTEM IS STOPPED! + */ +void ia64_set_curr_task(int cpu, struct task_struct *p) +{ + cpu_curr(cpu) = p; +} + +#endif + +#ifdef CONFIG_SCHED_DEBUG +__read_mostly bool sched_debug_enabled; + +void proc_sched_show_task(struct task_struct *p, struct pid_namespace *ns, + struct seq_file *m) +{ + seq_printf(m, "%s (%d, #threads: %d)\n", p->comm, task_pid_nr_ns(p, ns), + get_nr_threads(p)); +} + +void proc_sched_set_task(struct task_struct *p) +{} +#endif + +#ifdef CONFIG_CGROUP_SCHED +static void sched_free_group(struct task_group *tg) +{ + kmem_cache_free(task_group_cache, tg); +} + +/* allocate runqueue etc for a new task group */ +struct task_group *sched_create_group(struct task_group *parent) +{ + struct task_group *tg; + + tg = kmem_cache_alloc(task_group_cache, GFP_KERNEL | __GFP_ZERO); + if (!tg) + return ERR_PTR(-ENOMEM); + + init_tg_cgroup_defaults(tg); + return tg; +} + +void sched_online_group(struct task_group *tg, struct task_group *parent) +{ +} + +/* rcu callback to free various structures associated with a task group */ +static void sched_free_group_rcu(struct rcu_head *rhp) +{ + /* Now it should be safe to free those cfs_rqs */ + sched_free_group(container_of(rhp, struct task_group, rcu)); +} + +void sched_destroy_group(struct task_group *tg) +{ + /* Wait for possible concurrent references to cfs_rqs complete */ + call_rcu(&tg->rcu, sched_free_group_rcu); +} + +void sched_release_group(struct task_group *tg) +{ +} + +static inline struct task_group *css_tg(struct cgroup_subsys_state *css) +{ + return css ? container_of(css, struct task_group, css) : NULL; +} + +static struct cgroup_subsys_state * +cpu_cgroup_css_alloc(struct cgroup_subsys_state *parent_css) +{ + struct task_group *parent = css_tg(parent_css); + struct task_group *tg; + + if (!parent) { + /* This is early initialization for the top cgroup */ + return &root_task_group.css; + } + + tg = sched_create_group(parent); + if (IS_ERR(tg)) + return ERR_PTR(-ENOMEM); + return &tg->css; +} + +/* Expose task group only after completing cgroup initialization */ +static int cpu_cgroup_css_online(struct cgroup_subsys_state *css) +{ + struct task_group *tg = css_tg(css); + struct task_group *parent = css_tg(css->parent); + + if (parent) + sched_online_group(tg, parent); + return 0; +} + +static void cpu_cgroup_css_released(struct cgroup_subsys_state *css) +{ + struct task_group *tg = css_tg(css); + + sched_release_group(tg); +} + +static void cpu_cgroup_css_free(struct cgroup_subsys_state *css) +{ + struct task_group *tg = css_tg(css); + + /* + * Relies on the RCU grace period between css_released() and this. + */ + sched_free_group(tg); +} + +static void cpu_cgroup_fork(struct task_struct *task) +{ +} + +static int cpu_cgroup_can_attach(struct cgroup_taskset *tset) +{ + return 0; +} + +static void cpu_cgroup_attach(struct cgroup_taskset *tset) +{ +} + +/* + * Accept-and-ignore cpu controller files. + * + * Mainline wires these to CFS shares/bandwidth. MuQSS has neither, but + * container runtimes and systemd write CPUWeight=/CPUQuota=/--cpus and + * fail if the files are missing. Validate ranges, store for readback, + * and leave scheduling unaffected. + * + * nice↔weight table matches CFS (sched_prio_to_weight) so weight.nice + * round-trips to the same values userspace expects. + */ +static const int muqss_prio_to_weight[40] = { + /* -20 */ 88761, 71755, 56483, 46273, 36291, + /* -15 */ 29154, 23254, 18705, 14949, 11916, + /* -10 */ 9548, 7620, 6100, 4904, 3906, + /* -5 */ 3121, 2501, 1991, 1586, 1277, + /* 0 */ 1024, 820, 655, 526, 423, + /* 5 */ 335, 272, 215, 172, 137, + /* 10 */ 110, 87, 70, 56, 45, + /* 15 */ 36, 29, 23, 18, 15, +}; + +static unsigned long muqss_weight_from_cgroup(unsigned long cgrp_weight) +{ + return DIV_ROUND_CLOSEST_ULL(cgrp_weight * 1024, CGROUP_WEIGHT_DFL); +} + +static unsigned long muqss_weight_to_cgroup(unsigned long weight) +{ + return clamp_t(unsigned long, + DIV_ROUND_CLOSEST_ULL(weight * CGROUP_WEIGHT_DFL, 1024), + CGROUP_WEIGHT_MIN, CGROUP_WEIGHT_MAX); +} + +static u64 cpu_weight_read_u64(struct cgroup_subsys_state *css, + struct cftype *cft) +{ + return css_tg(css)->weight; +} + +static int cpu_weight_write_u64(struct cgroup_subsys_state *css, + struct cftype *cft, u64 cgrp_weight) +{ + if (cgrp_weight < CGROUP_WEIGHT_MIN || cgrp_weight > CGROUP_WEIGHT_MAX) + return -ERANGE; + + css_tg(css)->weight = cgrp_weight; + return 0; +} + +static s64 cpu_weight_nice_read_s64(struct cgroup_subsys_state *css, + struct cftype *cft) +{ + unsigned long weight = muqss_weight_from_cgroup(css_tg(css)->weight); + int last_delta = INT_MAX; + int prio, delta; + + for (prio = 0; prio < ARRAY_SIZE(muqss_prio_to_weight); prio++) { + delta = abs(muqss_prio_to_weight[prio] - (int)weight); + if (delta >= last_delta) + break; + last_delta = delta; + } + + return PRIO_TO_NICE(prio - 1 + MAX_RT_PRIO); +} + +static int cpu_weight_nice_write_s64(struct cgroup_subsys_state *css, + struct cftype *cft, s64 nice) +{ + int idx; + + if (nice < MIN_NICE || nice > MAX_NICE) + return -ERANGE; + + idx = NICE_TO_PRIO(nice) - MAX_RT_PRIO; + idx = array_index_nospec(idx, ARRAY_SIZE(muqss_prio_to_weight)); + css_tg(css)->weight = + muqss_weight_to_cgroup(muqss_prio_to_weight[idx]); + return 0; +} + +static s64 cpu_idle_read_s64(struct cgroup_subsys_state *css, + struct cftype *cft) +{ + return css_tg(css)->idle; +} + +static int cpu_idle_write_s64(struct cgroup_subsys_state *css, + struct cftype *cft, s64 idle) +{ + if (idle != 0 && idle != 1) + return -EINVAL; + + css_tg(css)->idle = idle; + return 0; +} + +static void cpu_period_quota_print(struct seq_file *sf, long period, long quota) +{ + if (quota < 0) + seq_puts(sf, "max"); + else + seq_printf(sf, "%ld", quota); + + seq_printf(sf, " %ld\n", period); +} + +static int cpu_period_quota_parse(char *buf, u64 *period_us_p, u64 *quota_us_p) +{ + char tok[21]; /* U64_MAX */ + + if (sscanf(buf, "%20s %llu", tok, period_us_p) < 1) + return -EINVAL; + + if (sscanf(tok, "%llu", quota_us_p) < 1) { + if (!strcmp(tok, "max")) + *quota_us_p = U64_MAX; + else + return -EINVAL; + } + + return 0; +} + +static int cpu_max_show(struct seq_file *sf, void *v) +{ + struct task_group *tg = css_tg(seq_css(sf)); + s64 quota = tg->quota_us; + + cpu_period_quota_print(sf, tg->period_us, quota); + return 0; +} + +static ssize_t cpu_max_write(struct kernfs_open_file *of, + char *buf, size_t nbytes, loff_t off) +{ + struct task_group *tg = css_tg(of_css(of)); + u64 period_us = tg->period_us, quota_us; + int ret; + + ret = cpu_period_quota_parse(buf, &period_us, "a_us); + if (ret) + return ret; + + if (!period_us || period_us > USEC_PER_SEC) + return -EINVAL; + + tg->period_us = period_us; + tg->quota_us = (quota_us == U64_MAX) ? -1 : (s64)quota_us; + return nbytes; +} + +static u64 cpu_burst_read_u64(struct cgroup_subsys_state *css, + struct cftype *cft) +{ + return css_tg(css)->burst_us; +} + +static int cpu_burst_write_u64(struct cgroup_subsys_state *css, + struct cftype *cft, u64 burst_us) +{ + struct task_group *tg = css_tg(css); + + /* Burst must not exceed a finite quota when one is set. */ + if (tg->quota_us >= 0 && burst_us > (u64)tg->quota_us) + return -EINVAL; + + tg->burst_us = burst_us; + return 0; +} + +/* Legacy v1 interfaces */ +static u64 cpu_shares_read_u64(struct cgroup_subsys_state *css, + struct cftype *cft) +{ + return muqss_weight_from_cgroup(css_tg(css)->weight); +} + +static int cpu_shares_write_u64(struct cgroup_subsys_state *css, + struct cftype *cft, u64 share) +{ + if (share < 2 || share > 262144) + return -ERANGE; + + css_tg(css)->weight = muqss_weight_to_cgroup(share); + return 0; +} + +static u64 cpu_period_read_u64(struct cgroup_subsys_state *css, + struct cftype *cft) +{ + return css_tg(css)->period_us; +} + +static int cpu_period_write_u64(struct cgroup_subsys_state *css, + struct cftype *cft, u64 period_us) +{ + if (!period_us || period_us > USEC_PER_SEC) + return -EINVAL; + + css_tg(css)->period_us = period_us; + return 0; +} + +static s64 cpu_quota_read_s64(struct cgroup_subsys_state *css, + struct cftype *cft) +{ + return css_tg(css)->quota_us; +} + +static int cpu_quota_write_s64(struct cgroup_subsys_state *css, + struct cftype *cft, s64 quota_us) +{ + if (quota_us < -1 || quota_us > (s64)USEC_PER_SEC * 1024) + return -EINVAL; + + css_tg(css)->quota_us = quota_us; + return 0; +} + +static struct cftype cpu_legacy_files[] = { + { + .name = "shares", + .read_u64 = cpu_shares_read_u64, + .write_u64 = cpu_shares_write_u64, + }, + { + .name = "idle", + .read_s64 = cpu_idle_read_s64, + .write_s64 = cpu_idle_write_s64, + }, + { + .name = "cfs_period_us", + .read_u64 = cpu_period_read_u64, + .write_u64 = cpu_period_write_u64, + }, + { + .name = "cfs_quota_us", + .read_s64 = cpu_quota_read_s64, + .write_s64 = cpu_quota_write_s64, + }, + { + .name = "cfs_burst_us", + .read_u64 = cpu_burst_read_u64, + .write_u64 = cpu_burst_write_u64, + }, + { } /* Terminate */ +}; + +static struct cftype cpu_files[] = { + { + .name = "weight", + .flags = CFTYPE_NOT_ON_ROOT, + .read_u64 = cpu_weight_read_u64, + .write_u64 = cpu_weight_write_u64, + }, + { + .name = "weight.nice", + .flags = CFTYPE_NOT_ON_ROOT, + .read_s64 = cpu_weight_nice_read_s64, + .write_s64 = cpu_weight_nice_write_s64, + }, + { + .name = "idle", + .flags = CFTYPE_NOT_ON_ROOT, + .read_s64 = cpu_idle_read_s64, + .write_s64 = cpu_idle_write_s64, + }, + { + .name = "max", + .flags = CFTYPE_NOT_ON_ROOT, + .seq_show = cpu_max_show, + .write = cpu_max_write, + }, + { + .name = "max.burst", + .flags = CFTYPE_NOT_ON_ROOT, + .read_u64 = cpu_burst_read_u64, + .write_u64 = cpu_burst_write_u64, + }, + { } /* terminate */ +}; + +static int cpu_extra_stat_show(struct seq_file *sf, + struct cgroup_subsys_state *css) +{ + return 0; +} + +struct cgroup_subsys cpu_cgrp_subsys = { + .css_alloc = cpu_cgroup_css_alloc, + .css_online = cpu_cgroup_css_online, + .css_released = cpu_cgroup_css_released, + .css_free = cpu_cgroup_css_free, + .css_extra_stat_show = cpu_extra_stat_show, + .fork = cpu_cgroup_fork, + .can_attach = cpu_cgroup_can_attach, + .attach = cpu_cgroup_attach, + .legacy_cftypes = cpu_legacy_files, + .dfl_cftypes = cpu_files, + .early_init = true, + .threaded = true, +}; +#endif /* CONFIG_CGROUP_SCHED */ + +void call_trace_sched_update_nr_running(struct rq *rq, int count) +{ + trace_sched_update_nr_running_tp(rq, count); +} + +/* CFS Compat */ +#ifdef CONFIG_RCU_TORTURE_TEST +int sysctl_sched_rt_runtime; +#endif + +/* + * Compatibility layer for core.c interfaces added after 5.12. + * + * These are all consumed by code outside the scheduler. Where the feature + * behind them does not exist under MuQSS (deadline bandwidth, sched_ext, + * mm_cid, the CFS runqueue debugfs) the implementation is deliberately inert + * rather than absent, so mainline callers need no #ifdef. + */ + +/* + * Tracepoint helpers behind set_current_state()/set_need_resched(). Callers + * MUST guard these with a tracepoint_enabled() check, which is why they use + * the unguarded trace_call__() form. + */ +void __trace_set_current_state(int state_value) +{ + trace_call__sched_set_state_tp(current, state_value); +} +EXPORT_SYMBOL(__trace_set_current_state); + +void __trace_set_need_resched(struct task_struct *curr, int tif) +{ + trace_call__sched_set_need_resched_tp(curr, smp_processor_id(), tif); +} +EXPORT_SYMBOL_GPL(__trace_set_need_resched); + +unsigned long long nr_context_switches_cpu(int cpu) +{ + return cpu_rq(cpu)->nr_switches; +} + +/* + * External (non-sched/) callers cannot see task_on_rq_queued(); wrap it. + * Used by tick-sched nohz full path among others. + */ +bool sched_task_on_rq(struct task_struct *p) +{ + return task_on_rq_queued(p); +} + +unsigned long get_wchan(struct task_struct *p) +{ + unsigned long ip = 0; + unsigned int state; + + if (!p || p == current) + return 0; + + /* Only get wchan if task is blocked and we can keep it that way. */ + raw_spin_lock_irq(&p->pi_lock); + state = READ_ONCE(p->__state); + smp_rmb(); /* see try_to_wake_up() */ + if (state != TASK_RUNNING && state != TASK_WAKING && !p->on_rq) + ip = __get_wchan(p); + raw_spin_unlock_irq(&p->pi_lock); + + return ip; +} + +/* + * Fork path. MuQSS does no cgroup bandwidth accounting and has no sched_ext + * to cancel, so these only need to exist. + */ +int sched_cgroup_fork(struct task_struct *p, struct kernel_clone_args *kargs) +{ + return 0; +} + +void sched_cancel_fork(struct task_struct *p) +{ +} + +/* + * rt_mutex helpers. MuQSS has no proxy execution; pre/post still run the + * worker submit/update pair so blocking on an rt_mutex flushes plugged IO. + * rt_mutex_schedule() must not re-enter schedule() or submit_work runs twice. + */ +#ifdef CONFIG_RT_MUTEXES +#define fetch_and_set(x, v) ({ int _x = (x); (x) = (v); _x; }) +#endif + +void rt_mutex_pre_schedule(void) +{ +#ifdef CONFIG_RT_MUTEXES + lockdep_assert(!fetch_and_set(current->sched_rt_mutex, 1)); +#endif + sched_submit_work(current); +} + +void rt_mutex_schedule(void) +{ +#ifdef CONFIG_RT_MUTEXES + lockdep_assert(current->sched_rt_mutex); +#endif + __schedule_loop(SM_NONE); +} + +void rt_mutex_post_schedule(void) +{ + sched_update_worker(current); +#ifdef CONFIG_RT_MUTEXES + lockdep_assert(fetch_and_set(current->sched_rt_mutex, 0)); +#endif +} + +int dl_task_check_affinity(struct task_struct *p __always_unused, + const struct cpumask *mask __always_unused) +{ + /* MuQSS has no deadline bandwidth admission. */ + return 0; +} + +#ifdef CONFIG_SMP +/* Callers must hold p->pi_lock across the read and its use. */ +static const struct cpumask *task_user_cpus(struct task_struct *p) +{ + lockdep_assert_held(&p->pi_lock); + + if (!p->user_cpus_ptr) + return cpu_possible_mask; + return p->user_cpus_ptr; +} + +/* + * Copy out the intersection of the task's user-requested mask and @mask. + * pi_lock keeps user_cpus_ptr alive for the copy: everything that frees it + * clears the pointer under task_rq_lock(), which nests pi_lock, and only + * frees once that has been dropped. + */ +static bool user_cpus_and(struct task_struct *p, struct cpumask *dst, + const struct cpumask *mask) +{ + unsigned long flags; + bool ret; + + raw_spin_lock_irqsave(&p->pi_lock, flags); + ret = cpumask_and(dst, task_user_cpus(p), mask); + raw_spin_unlock_irqrestore(&p->pi_lock, flags); + + return ret; +} + +/* + * Intersect the task's user-requested mask with @subset_mask and apply it. + * Empty intersection leaves affinity unchanged. + */ +static int restrict_cpus_allowed_ptr(struct task_struct *p, + struct cpumask *new_mask, + const struct cpumask *subset_mask) +{ + if (!user_cpus_and(p, new_mask, subset_mask)) + return -EINVAL; + + return __set_cpus_allowed_ptr(p, new_mask, 0); +} +#endif + +/* + * Restrict @p to CPUs it can actually run on (task_cpu_possible_mask()). + * ARM64 32-bit execve uses this when 64-bit-only CPUs exist. + */ +void force_compatible_cpus_allowed_ptr(struct task_struct *p) +{ + const struct cpumask *override_mask = task_cpu_possible_mask(p); +#ifdef CONFIG_SMP + cpumask_var_t new_mask; + + alloc_cpumask_var(&new_mask, GFP_KERNEL); + + /* + * __migrate_task() can fail silently if the dest CPU is offlined + * concurrently, so hold the hotplug lock across the move. + */ + cpus_read_lock(); + if (!cpumask_available(new_mask)) + goto out_set_mask; + + if (!restrict_cpus_allowed_ptr(p, new_mask, override_mask)) + goto out_free_mask; + + cpuset_cpus_allowed(p, new_mask); + override_mask = new_mask; + +out_set_mask: + if (printk_ratelimit()) { + printk_deferred("Overriding affinity for process %d (%s) to CPUs %*pbl\n", + task_pid_nr(p), p->comm, + cpumask_pr_args(override_mask)); + } + + WARN_ON(set_cpus_allowed_ptr(p, override_mask)); +out_free_mask: + cpus_read_unlock(); + free_cpumask_var(new_mask); +#else + WARN_ON(set_cpus_allowed_ptr(p, override_mask)); +#endif +} + +/* + * Restore the affinity previously restricted by + * force_compatible_cpus_allowed_ptr(). Caller serialises the pair. + */ +void relax_compatible_cpus_allowed_ptr(struct task_struct *p) +{ +#ifdef CONFIG_SMP + cpumask_var_t cpus_allowed, new_mask; + int ret = -ENOMEM; + + if (!alloc_cpumask_var(&cpus_allowed, GFP_KERNEL)) + goto warn; + if (!alloc_cpumask_var(&new_mask, GFP_KERNEL)) + goto out_free_cpus_allowed; + + cpuset_cpus_allowed(p, cpus_allowed); + user_cpus_and(p, new_mask, cpus_allowed); + ret = __set_cpus_allowed_ptr(p, new_mask, SCA_CHECK); + + free_cpumask_var(new_mask); +out_free_cpus_allowed: + free_cpumask_var(cpus_allowed); +warn: + WARN_ON_ONCE(ret); +#else + WARN_ON(set_cpus_allowed_ptr(p, cpu_possible_mask)); +#endif +} + +const char *preempt_model_str(void) +{ + if (IS_ENABLED(CONFIG_PREEMPT_RT)) + return "PREEMPT_RT"; + if (preempt_model_full()) + return "PREEMPT"; + if (preempt_model_voluntary()) + return "VOLUNTARY"; + return "NONE"; +} + +#ifdef CONFIG_SMP +bool cpus_equal_capacity(int this_cpu, int that_cpu) +{ + if (!sched_asym_cpucap_active()) + return true; + + if (this_cpu == that_cpu) + return true; + + return arch_scale_cpu_capacity(this_cpu) == arch_scale_cpu_capacity(that_cpu); +} + +void set_cpus_allowed_force(struct task_struct *p, const struct cpumask *new_mask) +{ + struct affinity_context ac = { + .new_mask = new_mask, + .user_mask = NULL, + .flags = SCA_USER, + }; + + do_set_cpus_allowed(p, &ac); +} + +void ___migrate_enable(void) +{ + __set_cpus_allowed_ptr(current, ¤t->cpus_mask, + SCA_MIGRATE_ENABLE); +} +#else /* !CONFIG_SMP */ +/* + * Nothing ever repoints cpus_ptr away from cpus_mask on UP, so + * __migrate_enable() never reaches here - it just has to link. + */ +void ___migrate_enable(void) +{ +} +#endif /* CONFIG_SMP */ +EXPORT_SYMBOL_GPL(___migrate_enable); + +/* + * user_cpus_ptr records the affinity a task asked for so that it can be + * restored after a temporary restriction: sched_setaffinity() stores it, and + * bind_zero() stores the mask it displaces from a task that would otherwise + * have been stranded by a CPU going down. Either way the child inherits both + * the restriction (zerobound, copied by dup_task_struct()) and the mask that + * has to be put back for it. + */ +int dup_user_cpus_ptr(struct task_struct *dst, struct task_struct *src, + int node) +{ + cpumask_t *user_mask; + unsigned long flags; + + dst->user_cpus_ptr = NULL; + + /* + * Racing here is harmless: losing it just means the child forgoes a + * restore it would have got for free, and taking the pi_lock on every + * fork to close it is not worth that. + */ + if (data_race(!src->user_cpus_ptr)) + return 0; + + user_mask = kmalloc_node(cpumask_size(), GFP_KERNEL, node); + if (!user_mask) + return -ENOMEM; + + raw_spin_lock_irqsave(&src->pi_lock, flags); + if (src->user_cpus_ptr) { + swap(dst->user_cpus_ptr, user_mask); + cpumask_copy(dst->user_cpus_ptr, src->user_cpus_ptr); + } + raw_spin_unlock_irqrestore(&src->pi_lock, flags); + + kfree(user_mask); + + return 0; +} + +void release_user_cpus_ptr(struct task_struct *p) +{ + kfree(p->user_cpus_ptr); + p->user_cpus_ptr = NULL; +} + +/* + * Modules cannot see the runqueues layout, so export out-of-line wrappers + * (INSTANTIATE_EXPORTED_MIGRATE_DISABLE makes the inlines become externs). + */ +void migrate_disable(void) +{ + __migrate_disable(); +} +EXPORT_SYMBOL_GPL(migrate_disable); + +void migrate_enable(void) +{ + __migrate_enable(); +} +EXPORT_SYMBOL_GPL(migrate_enable); + +/* + * MuQSS picks the CPU for a task at schedule() time rather than at exec, so + * there is nothing useful to do here. + */ +void sched_exec(void) +{ +} + +/* + * Deadline bandwidth accounting. There is no deadline class, so no bandwidth + * is ever reserved and cpuset has nothing to move between root domains. + */ +u64 dl_cookie; + +int dl_bw_alloc(int cpu, u64 dl_bw) +{ + return 0; +} + +void dl_bw_free(int cpu, u64 dl_bw) +{ +} + +void sched_set_fifo_secondary(struct task_struct *p) +{ + struct sched_param sp = { .sched_priority = MAX_RT_PRIO / 2 - 1 }; + + WARN_ON_ONCE(sched_setscheduler_nocheck(p, SCHED_FIFO, &sp) != 0); +} +EXPORT_SYMBOL_GPL(sched_set_fifo_secondary); + +/* + * Scheduler debugfs. debug.c is CFS/EEVDF runqueue introspection and is not + * built under MuQSS; /proc//sched reports nothing. + */ +bool sched_debug_verbose; + +#ifdef CONFIG_SMP +/* Only topology.c calls these, and it is not built on UP. */ +void update_sched_domain_debugfs(void) +{ +} + +void dirty_sched_domain_sysctl(int cpu) +{ +} +#endif /* CONFIG_SMP */ + +void proc_sched_show_task(struct task_struct *p, struct pid_namespace *ns, + struct seq_file *m) +{ +} + +void proc_sched_set_task(struct task_struct *p) +{ +} diff --git a/kernel/sched/MuQSS.h b/kernel/sched/MuQSS.h new file mode 100644 index 000000000..0c461e650 --- /dev/null +++ b/kernel/sched/MuQSS.h @@ -0,0 +1,1253 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef MUQSS_SCHED_H +#define MUQSS_SCHED_H + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#ifdef CONFIG_PARAVIRT +#include +#endif + +/* + * cpupri.h is deliberately not included: MuQSS has no RT class priority + * bitmap, root_domain keeps an opaque pointer instead, and pulling the header + * in would clash with the no-op cpupri_init()/cpupri_cleanup() below. + */ + +#include + +#ifdef CONFIG_SCHED_DEBUG +# define SCHED_WARN_ON(x) WARN_ONCE(x, #x) +#else +# define SCHED_WARN_ON(x) ((void)(x)) +#endif + +/* Wake flags. The first three directly map to some SD flag value */ +#define WF_EXEC 0x02 /* Wakeup after exec; maps to SD_BALANCE_EXEC */ +#define WF_FORK 0x04 /* Wakeup after fork; maps to SD_BALANCE_FORK */ +#define WF_TTWU 0x08 /* Wakeup; maps to SD_BALANCE_WAKE */ + +#define WF_SYNC 0x10 /* Waker goes to sleep after wakeup */ +#define WF_MIGRATED 0x20 /* Internal use, task got migrated */ +#define WF_CURRENT_CPU 0x40 /* Prefer to move the wakee to the current CPU. */ +#define WF_ON_CPU 0x80 /* Wakee is on_cpu */ + +#ifdef CONFIG_SMP +static_assert(WF_EXEC == SD_BALANCE_EXEC); +static_assert(WF_FORK == SD_BALANCE_FORK); +static_assert(WF_TTWU == SD_BALANCE_WAKE); +#endif + +/* task_struct::on_rq states: */ +#define TASK_ON_RQ_QUEUED 1 +#define TASK_ON_RQ_MIGRATING 2 + +extern void call_trace_sched_update_nr_running(struct rq *rq, int count); + +struct rq; + +#ifdef CONFIG_SMP + +static inline bool sched_asym_prefer(int a, int b) +{ + return arch_asym_cpu_priority(a) > arch_asym_cpu_priority(b); +} + +struct perf_domain { + struct em_perf_domain *em_pd; + struct perf_domain *next; + struct rcu_head rcu; +}; + +/* Scheduling group status flags */ +#define SG_OVERLOAD 0x1 /* More than one runnable task on a CPU. */ +#define SG_OVERUTILIZED 0x2 /* One or more CPUs are over-utilized. */ + +/* + * We add the notion of a root-domain which will be used to define per-domain + * variables. Each exclusive cpuset essentially defines an island domain by + * fully partitioning the member cpus from any other cpuset. Whenever a new + * exclusive cpuset is created, we also create and attach a new root-domain + * object. + * + */ +struct root_domain { + atomic_t refcount; + atomic_t rto_count; + struct rcu_head rcu; + cpumask_var_t span; + cpumask_var_t online; + + /* + * Indicate pullable load on at least one CPU, e.g: + * - More than one runnable task + * - Running task is misfit + */ + int overload; + + /* Indicate one or more cpus over-utilized (tipping point) */ + int overutilized; + + /* + * The bit corresponding to a CPU gets set here if such CPU has more + * than one runnable -deadline task (as it is below for RT tasks). + */ + cpumask_var_t dlo_mask; + atomic_t dlo_count; + + /* Replace unused CFS structures with void */ + //struct dl_bw dl_bw; + //struct cpudl cpudl; + void *dl_bw; + void *cpudl; + u64 visit_cookie; + + /* + * The "RT overload" flag: it gets set if a CPU has more than + * one runnable RT task. + */ + cpumask_var_t rto_mask; + //struct cpupri cpupri; + void *cpupri; + + unsigned long max_cpu_capacity; + + /* + * NULL-terminated list of performance domains intersecting with the + * CPUs of the rd. Protected by RCU. + */ + struct perf_domain *pd; +}; + +extern void init_defrootdomain(void); +extern int sched_init_domains(const struct cpumask *cpu_map); +extern void rq_attach_root(struct rq *rq, struct root_domain *rd); + +static inline void cpupri_cleanup(void __maybe_unused *cpupri) +{ +} + +static inline void cpudl_cleanup(void __maybe_unused *cpudl) +{ +} + +static inline void init_dl_bw(void __maybe_unused *dl_bw) +{ +} + +static inline int cpudl_init(void __maybe_unused *dl_bw) +{ + return 0; +} + +static inline int cpupri_init(void __maybe_unused *cpupri) +{ + return 0; +} +#endif /* CONFIG_SMP */ + +/* + * This is the main, per-CPU runqueue data structure. + * This data should only be modified by the local cpu. + */ +struct rq { + raw_spinlock_t *lock; + raw_spinlock_t *orig_lock; + + struct task_struct __rcu *curr; + struct task_struct *idle; + struct task_struct *stop; + struct mm_struct *prev_mm; + + unsigned int nr_running; + /* + * This is part of a global counter where only the total sum + * over all CPUs matters. A task can increase this counter on + * one CPU and if it got migrated afterwards it may decrease + * it on another CPU. Always updated under the runqueue lock: + */ + unsigned long nr_uninterruptible; +#ifdef CONFIG_SMP + unsigned int ttwu_pending; +#endif + /* + * migrate_disable() nesting count. Mainline's migrate_disable() and + * __migrate_enable() are inline in and reach this via + * the RQ_nr_pinned offset generated from rq-offsets.c, so the member + * is required whether or not MuQSS itself consults it. + */ + unsigned int nr_pinned; + u64 nr_switches; + + /* + * Isolated curr snapshot for lockless peeks. Own cacheline so + * last_jiffy / niffies (and remote synchronise_niffies()) do not + * invalidate it. Under SMT_NICE this is four fields; without it, + * deadline + prio only. Do not pull nr_running, online, or sl in. + */ + u64 rq_deadline ____cacheline_aligned; + int rq_prio; +#ifdef CONFIG_SMT_NICE + struct mm_struct *rq_mm; + int rq_smt_bias; /* Policy/nice level bias across smt siblings */ +#endif + u8 __rq_snap_pad[0] ____cacheline_aligned; + + unsigned long last_scheduler_tick; /* Last jiffy this RQ ticked */ + unsigned long last_jiffy; /* Last jiffy this RQ updated rq clock */ + u64 niffies; /* Last time this RQ updated rq clock */ + u64 jiffy_niffies; /* Niffies as counted by the tick alone */ + + u64 load_update; /* When we last updated load */ + unsigned long load_avg; /* Rolling load average */ +#ifdef CONFIG_HAVE_SCHED_AVG_IRQ + u64 irq_load_update; /* When we last updated IRQ load */ + unsigned long irq_load_avg; /* Rolling IRQ load average */ +#endif + /* Accurate timekeeping data */ + unsigned long user_ns, nice_ns, irq_ns, softirq_ns, system_ns, + iowait_ns, idle_ns; + atomic_t nr_iowait; + +#ifdef CONFIG_MEMBARRIER + int membarrier_state; +#endif + + /* + * MuQSS runs no deadline server. topology.c unconditionally tests + * rq->fair_server.dl_server when attaching a root domain, so carry a + * one-byte flag that is never set rather than #ifdef that code out. + * __dl_server_attach_root() below is a no-op taking void *, so the + * member's type does not have to match sched_dl_entity. + */ + struct { + bool dl_server; + } fair_server; + + skiplist_node *node; + skiplist *sl; +#ifdef CONFIG_SMP + struct task_struct *preempt; /* Preempt triggered on this task */ + struct task_struct *preempting; /* Hint only, what task is preempting */ + + int cpu; /* cpu of this runqueue */ + bool online; + + struct root_domain *rd; + struct sched_domain *sd; + + unsigned long cpu_capacity_orig; + + int *cpu_locality; /* CPU relative cache distance */ + struct rq **rq_order; /* Shared RQs ordered by relative cache distance */ + skiplist **sl_order; /* Leaders' skiplists, parallel to rq_order */ + struct rq **cpu_order; /* RQs of discrete CPUs ordered by distance */ + + bool is_leader; + struct rq *smp_leader; /* First physical CPU per node */ +#ifdef CONFIG_SCHED_THERMAL_PRESSURE + struct sched_avg avg_thermal; +#endif /* CONFIG_SCHED_THERMAL_PRESSURE */ +#ifdef CONFIG_SCHED_SMT + struct rq *smt_leader; /* First logical CPU in SMT siblings */ + cpumask_t thread_mask; + bool has_smt_sibling; /* This CPU has SMT siblings at all */ +#endif /* CONFIG_SCHED_SMT */ +#ifdef CONFIG_SCHED_MC + struct rq *mc_leader; /* First logical CPU in MC siblings */ + cpumask_t core_mask; + bool (*cache_idle)(struct rq *rq); + /* See if all cache siblings are idle */ +#endif /* CONFIG_SCHED_MC */ + /* last busy→idle, wakeup idle pick */ + unsigned long idle_jiffy; +#endif /* CONFIG_SMP */ + +#ifdef CONFIG_IRQ_TIME_ACCOUNTING + u64 prev_irq_time; +#endif /* CONFIG_IRQ_TIME_ACCOUNTING */ +#ifdef CONFIG_PARAVIRT + u64 prev_steal_time; +#endif /* CONFIG_PARAVIRT */ +#ifdef CONFIG_PARAVIRT_TIME_ACCOUNTING + u64 prev_steal_time_rq; +#endif /* CONFIG_PARAVIRT_TIME_ACCOUNTING */ + + u64 clock, last_tick; + /* Ensure that all clocks are in the same cache line */ + u64 clock_task ____cacheline_aligned; + int dither; + + int iso_ticks; + bool iso_refractory; + +#ifdef CONFIG_HIGH_RES_TIMERS + struct hrtimer hrexpiry_timer; + /* + * Deferred start/cancel around __schedule() — same scheme as mainline + * hrtick. Reprogramming the oneshot clockevent from set_rq_task() on + * every context switch under the rq lock starves TIMER_SOFTIRQ on SMP. + */ + call_single_data_t hrexpiry_csd; + unsigned int hrexpiry_sched; + s64 hrexpiry_delay; + ktime_t hrexpiry_time; +#endif + + int rt_nr_running; /* Number real time tasks running */ +#ifdef CONFIG_SCHEDSTATS + + /* latency stats */ + struct sched_info rq_sched_info; + unsigned long long rq_cpu_time; + /* could above be rq->cfs_rq.exec_clock + rq->rt_rq.rt_runtime ? */ + + /* sys_sched_yield() stats */ + unsigned int yld_count; + + /* schedule() stats */ + unsigned int sched_switch; + unsigned int sched_count; + unsigned int sched_goidle; + + /* try_to_wake_up() stats */ + unsigned int ttwu_count; + unsigned int ttwu_local; +#endif /* CONFIG_SCHEDSTATS */ + +#ifdef CONFIG_CPU_IDLE + /* Must be inspected within a rcu lock section */ + struct cpuidle_state *idle_state; +#endif +}; + +static inline u64 __rq_clock_broken(struct rq *rq) +{ + return READ_ONCE(rq->clock); +} + +static inline u64 rq_clock(struct rq *rq) +{ + lockdep_assert_held(rq->lock); + + return rq->clock; +} + +static inline u64 rq_clock_task(struct rq *rq) +{ + lockdep_assert_held(rq->lock); + + return rq->clock_task; +} + +/** + * By default the decay is the default pelt decay period. + * The decay shift can change the decay period in + * multiples of 32. + * Decay shift Decay period(ms) + * 0 32 + * 1 64 + * 2 128 + * 3 256 + * 4 512 + */ +extern int sched_thermal_decay_shift; + +static inline u64 rq_clock_thermal(struct rq *rq) +{ + return rq_clock_task(rq) >> sched_thermal_decay_shift; +} + +struct rq_flags { + unsigned long flags; +}; + +#ifdef CONFIG_SMP +struct rq *cpu_rq(int cpu); +#endif + +#ifndef CONFIG_SMP +extern struct rq *uprq; +#define cpu_rq(cpu) (uprq) +#define this_rq() (uprq) +#define raw_rq() (uprq) +#define task_rq(p) (uprq) +#define cpu_curr(cpu) ((uprq)->curr) +#else /* CONFIG_SMP */ +DECLARE_PER_CPU_SHARED_ALIGNED(struct rq, runqueues); +#define cpu_rq(cpu) (&per_cpu(runqueues, (cpu))) +#define this_rq() this_cpu_ptr(&runqueues) +#define task_rq(p) cpu_rq(task_cpu(p)) +#define cpu_curr(cpu) (cpu_rq(cpu)->curr) +#define raw_rq() raw_cpu_ptr(&runqueues) +#endif /* CONFIG_SMP */ + +static inline int task_current(struct rq *rq, struct task_struct *p) +{ + return rq->curr == p; +} + +static inline int task_running(struct rq *rq, struct task_struct *p) +{ +#ifdef CONFIG_SMP + return p->on_cpu; +#else + return task_current(rq, p); +#endif +} + +/* + * Mainline name used by stats.h / psi.c / RT helpers. Equivalent to + * task_running() under SMP; always uses p->on_cpu so the @rq argument is + * unused (kept for API compatibility with mainline call sites). + */ +static inline int task_on_cpu(struct rq *rq, struct task_struct *p) +{ +#ifdef CONFIG_SMP + return p->on_cpu; +#else + return task_current(rq, p); +#endif +} + +static inline int task_on_rq_queued(struct task_struct *p) +{ + return p->on_rq == TASK_ON_RQ_QUEUED; +} + +static inline int task_on_rq_migrating(struct task_struct *p) +{ + return READ_ONCE(p->on_rq) == TASK_ON_RQ_MIGRATING; +} + +static inline void lockdep_assert_rq_held(struct rq *rq) +{ + lockdep_assert_held(rq->lock); +} + +static inline void rq_lock(struct rq *rq) + __acquires(rq->lock) +{ + raw_spin_lock(rq->lock); +} + +static inline void rq_unlock(struct rq *rq) + __releases(rq->lock) +{ + raw_spin_unlock(rq->lock); +} + +static inline void rq_lock_irq(struct rq *rq) + __acquires(rq->lock) +{ + raw_spin_lock_irq(rq->lock); +} + +static inline void rq_unlock_irq(struct rq *rq, struct rq_flags __always_unused *rf) + __releases(rq->lock) +{ + raw_spin_unlock_irq(rq->lock); +} + +static inline void rq_lock_irqsave(struct rq *rq, struct rq_flags *rf) + __acquires(rq->lock) +{ + raw_spin_lock_irqsave(rq->lock, rf->flags); +} + +static inline void rq_unlock_irqrestore(struct rq *rq, struct rq_flags *rf) + __releases(rq->lock) +{ + raw_spin_unlock_irqrestore(rq->lock, rf->flags); +} + +static inline struct rq *task_rq_lock(struct task_struct *p, struct rq_flags *rf) + __acquires(p->pi_lock) + __acquires(rq->lock) +{ + struct rq *rq; + + while (42) { + raw_spin_lock_irqsave(&p->pi_lock, rf->flags); + rq = task_rq(p); + raw_spin_lock(rq->lock); + if (likely(rq == task_rq(p))) + break; + raw_spin_unlock(rq->lock); + raw_spin_unlock_irqrestore(&p->pi_lock, rf->flags); + } + return rq; +} + +static inline void task_rq_unlock(struct rq *rq, struct task_struct *p, struct rq_flags *rf) + __releases(rq->lock) + __releases(p->pi_lock) +{ + rq_unlock(rq); + raw_spin_unlock_irqrestore(&p->pi_lock, rf->flags); +} + +static inline struct rq *__task_rq_lock(struct task_struct *p, struct rq_flags __always_unused *rf) + __acquires(rq->lock) +{ + struct rq *rq; + + lockdep_assert_held(&p->pi_lock); + + while (42) { + rq = task_rq(p); + raw_spin_lock(rq->lock); + if (likely(rq == task_rq(p))) + break; + raw_spin_unlock(rq->lock); + } + return rq; +} + +/* + * Match mainline's 3-arg signature (stats.h / psi paths pass @p). MuQSS does + * not pin the lock cookie, so @p and @rf are unused beyond API compatibility. + */ +static inline void __task_rq_unlock(struct rq *rq, + struct task_struct __always_unused *p, + struct rq_flags __always_unused *rf) +{ + rq_unlock(rq); +} + +/* + * scoped_guard / guard(rq_lock_irq) used by psi_cgroup_restart() and similar. + * MuQSS rq_lock_irq takes only the rq (no pin cookie). + */ +DEFINE_LOCK_GUARD_1(rq_lock_irq, struct rq, + rq_lock_irq(_T->lock), + rq_unlock_irq(_T->lock, &_T->rf), + struct rq_flags rf) + +DEFINE_LOCK_GUARD_1(rq_lock_irqsave, struct rq, + rq_lock_irqsave(_T->lock, &_T->rf), + rq_unlock_irqrestore(_T->lock, &_T->rf), + struct rq_flags rf) + +static inline struct rq * +this_rq_lock_irq(struct rq_flags *rf) + __acquires(rq->lock) +{ + struct rq *rq; + + local_irq_disable(); + rq = this_rq(); + rq_lock(rq); + return rq; +} + +/* + * {de,en}queue flags: Most not used on MuQSS. + * + * DEQUEUE_SLEEP - task is no longer runnable + * ENQUEUE_WAKEUP - task just became runnable + * + * SAVE/RESTORE - an otherwise spurious dequeue/enqueue, done to ensure tasks + * are in a known state which allows modification. Such pairs + * should preserve as much state as possible. + * + * MOVE - paired with SAVE/RESTORE, explicitly does not preserve the location + * in the runqueue. + * + * ENQUEUE_HEAD - place at front of runqueue (tail if not specified) + * ENQUEUE_REPLENISH - CBS (replenish runtime and postpone deadline) + * ENQUEUE_MIGRATED - the task was migrated during wakeup + * + */ + +#define DEQUEUE_SLEEP 0x01 +#define DEQUEUE_SAVE 0x02 /* matches ENQUEUE_RESTORE */ + +#define ENQUEUE_WAKEUP 0x01 +#define ENQUEUE_RESTORE 0x02 + +#ifdef CONFIG_SMP +#define ENQUEUE_MIGRATED 0x40 +#else +#define ENQUEUE_MIGRATED 0x00 +#endif + +#ifdef CONFIG_NUMA +enum numa_topology_type { + NUMA_DIRECT, + NUMA_GLUELESS_MESH, + NUMA_BACKPLANE, +}; +extern enum numa_topology_type sched_numa_topology_type; +extern int sched_max_numa_distance; +extern bool find_numa_distance(int distance); +extern void sched_init_numa(int offline_node); +extern void sched_domains_numa_masks_set(unsigned int cpu); +extern void sched_domains_numa_masks_clear(unsigned int cpu); +extern int sched_numa_find_closest(const struct cpumask *cpus, int cpu); +#else +static inline void sched_init_numa(int offline_node) { } +static inline void sched_domains_numa_masks_set(unsigned int cpu) { } +static inline void sched_domains_numa_masks_clear(unsigned int cpu) { } +static inline int sched_numa_find_closest(const struct cpumask *cpus, int cpu) +{ + return nr_cpu_ids; +} +#endif + +extern struct mutex sched_domains_mutex; +extern struct static_key_false sched_schedstats; + +#define rcu_dereference_sched_domain(p) \ + rcu_dereference_all_check((p), lockdep_is_held(&sched_domains_mutex)) + +struct affinity_context { + const struct cpumask *new_mask; + struct cpumask *user_mask; + unsigned int flags; +}; + +#define MDF_PUSH 0x01 + +static inline bool is_migration_disabled(struct task_struct *p) +{ + return p->migration_disabled; +} + +#define SCA_CHECK 0x01 +#define SCA_MIGRATE_DISABLE 0x02 +#define SCA_MIGRATE_ENABLE 0x04 +#define SCA_USER 0x08 + +#ifdef CONFIG_SMP + +/* + * The domain tree (rq->sd) is protected by RCU's quiescent state transition. + * See destroy_sched_domains: call_rcu for details. + * + * The domain tree of any CPU may only be accessed from within + * preempt-disabled sections. + */ +#define for_each_domain(cpu, __sd) \ + for (__sd = rcu_dereference_sched_domain(cpu_rq(cpu)->sd); \ + __sd; __sd = __sd->parent) + +/** + * highest_flag_domain - Return highest sched_domain containing flag. + * @cpu: The cpu whose highest level of sched domain is to + * be returned. + * @flag: The flag to check for the highest sched_domain + * for the given cpu. + * + * Returns the highest sched_domain of a cpu which contains the given flag. + */ +static inline struct sched_domain *highest_flag_domain(int cpu, int flag) +{ + struct sched_domain *sd, *hsd = NULL; + + for_each_domain(cpu, sd) { + if (!(sd->flags & flag)) + break; + hsd = sd; + } + + return hsd; +} + +static inline struct sched_domain *lowest_flag_domain(int cpu, int flag) +{ + struct sched_domain *sd; + + for_each_domain(cpu, sd) { + if (sd->flags & flag) + break; + } + + return sd; +} + +DECLARE_PER_CPU(struct sched_domain __rcu *, sd_llc); +DECLARE_PER_CPU(int, sd_llc_size); +DECLARE_PER_CPU(int, sd_llc_id); +DECLARE_PER_CPU(int, sd_share_id); +DECLARE_PER_CPU(struct sched_domain_shared __rcu *, sd_llc_shared); +DECLARE_PER_CPU(struct sched_domain_shared __rcu *, sd_balance_shared); +DECLARE_PER_CPU(struct sched_domain __rcu *, sd_numa); +DECLARE_PER_CPU(struct sched_domain __rcu *, sd_asym_packing); +DECLARE_PER_CPU(struct sched_domain __rcu *, sd_asym_cpucapacity); + +void sched_domains_free_llc_id(int cpu); + +extern struct static_key_false sched_asym_cpucapacity; +extern struct static_key_false sched_cluster_active; + +static __always_inline bool sched_asym_cpucap_active(void) +{ + return static_branch_unlikely(&sched_asym_cpucapacity); +} + +struct sched_group_capacity { + atomic_t ref; + /* + * CPU capacity of this group, SCHED_CAPACITY_SCALE being max capacity + * for a single CPU. + */ + unsigned long capacity; + unsigned long min_capacity; /* Min per-CPU capacity in group */ + unsigned long max_capacity; /* Max per-CPU capacity in group */ + unsigned long next_update; + int imbalance; /* XXX unrelated to capacity but shared group state */ + + int id; + + unsigned long cpumask[]; /* Balance mask */ +}; + +struct sched_group { + struct sched_group *next; /* Must be a circular list */ + atomic_t ref; + + unsigned int group_weight; + unsigned int cores; + struct sched_group_capacity *sgc; + int asym_prefer_cpu; /* CPU of highest priority in group */ + int flags; + + /* + * The CPUs this group covers. + * + * NOTE: this field is variable length. (Allocated dynamically + * by attaching extra space to the end of the structure, + * depending on how many CPUs the kernel has booted up with) + */ + unsigned long cpumask[]; +}; + +static inline struct cpumask *sched_group_span(struct sched_group *sg) +{ + return to_cpumask(sg->cpumask); +} + +/* + * See build_balance_mask(). + */ +static inline struct cpumask *group_balance_mask(struct sched_group *sg) +{ + return to_cpumask(sg->sgc->cpumask); +} + +extern int group_balance_cpu(struct sched_group *sg); +extern void update_sched_domain_debugfs(void); +extern void dirty_sched_domain_sysctl(int cpu); +extern void sched_update_numa(int cpu, bool online); +extern void sched_get_rd(struct root_domain *rd); +extern void sched_put_rd(struct root_domain *rd); +extern bool sched_debug_verbose; + +/** + * group_first_cpu - Returns the first cpu in the cpumask of a sched_group. + * @group: The group whose first cpu is to be returned. + */ +static inline unsigned int group_first_cpu(struct sched_group *group) +{ + return cpumask_first(sched_group_span(group)); +} + + +/* + * register_sched_domain_sysctl()/unregister_sched_domain_sysctl() were + * replaced upstream by update_sched_domain_debugfs(), declared above with + * dirty_sched_domain_sysctl(). flush_smp_call_function_from_idle() likewise + * became flush_smp_call_function_queue(), declared in smp.h. + */ + +extern void set_cpus_allowed_common(struct task_struct *p, struct affinity_context *ctx); + +extern void set_rq_online (struct rq *rq); +extern void set_rq_offline(struct rq *rq); +extern bool sched_smp_initialized; + +static inline void update_group_capacity(struct sched_domain *sd, int cpu) +{ +} + +static inline void trigger_load_balance(struct rq *rq) +{ +} + +#else /* CONFIG_SMP */ + +static inline void flush_smp_call_function_from_idle(void) { } + +/* + * UP has no sched domains and MuQSS's rq has no ->sd, but stats.c walks the + * domain list unconditionally. Hand it an empty list. + */ +#define for_each_domain(cpu, __sd) \ + for (__sd = NULL; __sd; __sd = __sd->parent) + +#endif /* CONFIG_SMP */ + +#ifdef CONFIG_CPU_IDLE +static inline void idle_set_state(struct rq *rq, + struct cpuidle_state *idle_state) +{ + rq->idle_state = idle_state; +} + +static inline struct cpuidle_state *idle_get_state(struct rq *rq) +{ + SCHED_WARN_ON(!rcu_read_lock_held()); + return rq->idle_state; +} +#else +static inline void idle_set_state(struct rq *rq, + struct cpuidle_state *idle_state) +{ +} + +static inline struct cpuidle_state *idle_get_state(struct rq *rq) +{ + return NULL; +} +#endif + +#ifdef CONFIG_SCHED_DEBUG +extern bool sched_debug_enabled; +#endif + +extern void schedule_idle(void); + +#ifdef CONFIG_IRQ_TIME_ACCOUNTING +struct irqtime { + u64 total; + u64 tick_delta; + u64 irq_start_time; + struct u64_stats_sync sync; +}; + +DECLARE_PER_CPU(struct irqtime, cpu_irqtime); + +/* + * Returns the irqtime minus the softirq time computed by ksoftirqd. + * Otherwise ksoftirqd's sum_exec_runtime is substracted its own runtime + * and never move forward. + */ +static inline u64 irq_time_read(int cpu) +{ + struct irqtime *irqtime = &per_cpu(cpu_irqtime, cpu); + unsigned int seq; + u64 total; + + do { + seq = __u64_stats_fetch_begin(&irqtime->sync); + total = irqtime->total; + } while (__u64_stats_fetch_retry(&irqtime->sync, seq)); + + return total; +} +#endif /* CONFIG_IRQ_TIME_ACCOUNTING */ + +static inline bool sched_stop_runnable(struct rq *rq) +{ + return rq->stop && task_on_rq_queued(rq->stop); +} + +#ifdef CONFIG_SMP +static inline int cpu_of(struct rq *rq) +{ + return rq->cpu; +} +#else /* CONFIG_SMP */ +static inline int cpu_of(struct rq *rq) +{ + return 0; +} +#endif + +#ifdef CONFIG_CPU_FREQ +DECLARE_PER_CPU(struct update_util_data *, cpufreq_update_util_data); + +static inline void cpufreq_trigger(struct rq *rq, unsigned int flags) +{ + struct update_util_data *data; + + data = rcu_dereference_sched(*per_cpu_ptr(&cpufreq_update_util_data, + cpu_of(rq))); + + if (data) + data->func(data, rq->niffies, flags); +} +#else +static inline void cpufreq_trigger(struct rq *rq, unsigned int flag) +{ +} +#endif /* CONFIG_CPU_FREQ */ + +static __always_inline +unsigned int uclamp_rq_util_with(struct rq __maybe_unused *rq, unsigned int util, + struct task_struct __maybe_unused *p) +{ + return util; +} + +static inline bool uclamp_is_used(void) +{ + return false; +} + +#ifndef arch_scale_freq_tick +static __always_inline +void arch_scale_freq_tick(void) +{ +} +#endif + +#ifdef arch_scale_freq_capacity +#ifndef arch_scale_freq_invariant +#define arch_scale_freq_invariant() (true) +#endif +#else /* arch_scale_freq_capacity */ +#define arch_scale_freq_invariant() (false) +#endif + +/* + * read_sum_exec_runtime() lives in cputime.c and now goes through + * tsk_seruntime(), so MuQSS needs no copy of it here. + */ + +#ifndef arch_scale_freq_capacity +/** + * arch_scale_freq_capacity - get the frequency scale factor of a given CPU. + * @cpu: the CPU in question. + * + * Return: the frequency scale factor normalized against SCHED_CAPACITY_SCALE, i.e. + * + * f_curr + * ------ * SCHED_CAPACITY_SCALE + * f_max + */ +static __always_inline +unsigned long arch_scale_freq_capacity(int cpu) +{ + return SCHED_CAPACITY_SCALE; +} +#endif + +#ifdef CONFIG_NO_HZ_FULL +extern bool sched_can_stop_tick(struct rq *rq); +extern int __init sched_tick_offload_init(void); + +/* + * Tick may be needed by tasks in the runqueue depending on their policy and + * requirements. If tick is needed, lets send the target an IPI to kick it out of + * nohz mode if necessary. + */ +static inline void sched_update_tick_dependency(struct rq *rq) +{ + int cpu = cpu_of(rq); + + if (!tick_nohz_full_cpu(cpu)) + return; + + if (sched_can_stop_tick(rq)) + tick_nohz_dep_clear_cpu(cpu, TICK_DEP_BIT_SCHED); + else + tick_nohz_dep_set_cpu(cpu, TICK_DEP_BIT_SCHED); +} +#else +static inline int sched_tick_offload_init(void) { return 0; } +static inline void sched_update_tick_dependency(struct rq *rq) { } +#endif + +#define SCHED_FLAG_SUGOV 0x10000000 + +#ifdef CONFIG_SMP +/* + * enum cpu_util_type and its FREQUENCY_UTIL/ENERGY_UTIL members were removed + * upstream; effective_cpu_util() now reports a utilisation figure plus the + * usable capacity range. MuQSS supplies it from MuQSS.c. + */ +unsigned long effective_cpu_util(int cpu, unsigned long util_cfs, + unsigned long *min, unsigned long *max); + +static inline unsigned long cpu_bw_dl(struct rq *rq) +{ + return 0; +} + +static inline unsigned long cpu_util_dl(struct rq *rq) +{ + return 0; +} + +static inline unsigned long cpu_util_cfs(struct rq *rq) +{ + unsigned long ret = READ_ONCE(rq->load_avg); + + if (ret > SCHED_CAPACITY_SCALE) + ret = SCHED_CAPACITY_SCALE; + return ret; +} + +static inline unsigned long cpu_util_rt(struct rq *rq) +{ + unsigned long ret = READ_ONCE(rq->rt_nr_running); + + if (ret > SCHED_CAPACITY_SCALE) + ret = SCHED_CAPACITY_SCALE; + return ret; +} + +#ifdef CONFIG_HAVE_SCHED_AVG_IRQ +static inline unsigned long cpu_util_irq(struct rq *rq) +{ + unsigned long ret = READ_ONCE(rq->irq_load_avg); + + if (ret > SCHED_CAPACITY_SCALE) + ret = SCHED_CAPACITY_SCALE; + return ret; +} + +static inline +unsigned long scale_irq_capacity(unsigned long util, unsigned long irq, unsigned long max) +{ + util *= (max - irq); + util /= max; + + return util; + +} +#else /* CONFIG_HAVE_SCHED_AVG_IRQ */ +static inline unsigned long cpu_util_irq(struct rq *rq) +{ + return 0; +} + +static inline +unsigned long scale_irq_capacity(unsigned long util, unsigned long irq, unsigned long max) +{ + return util; +} +#endif /* CONFIG_HAVE_SCHED_AVG_IRQ */ +#endif /* CONFIG_SMP */ + +#if defined(CONFIG_ENERGY_MODEL) && defined(CONFIG_CPU_FREQ_GOV_SCHEDUTIL) +#define perf_domain_span(pd) (to_cpumask(((pd)->em_pd->cpus))) + +DECLARE_STATIC_KEY_FALSE(sched_energy_present); + +static inline bool sched_energy_enabled(void) +{ + return static_branch_unlikely(&sched_energy_present); +} + +#else /* ! (CONFIG_ENERGY_MODEL && CONFIG_CPU_FREQ_GOV_SCHEDUTIL) */ + +#define perf_domain_span(pd) NULL +static inline bool sched_energy_enabled(void) { return false; } + +#endif /* CONFIG_ENERGY_MODEL && CONFIG_CPU_FREQ_GOV_SCHEDUTIL */ + +#ifdef CONFIG_MEMBARRIER +/* + * The scheduler provides memory barriers required by membarrier between: + * - prior user-space memory accesses and store to rq->membarrier_state, + * - store to rq->membarrier_state and following user-space memory accesses. + * In the same way it provides those guarantees around store to rq->curr. + */ +static inline void membarrier_switch_mm(struct rq *rq, + struct mm_struct *prev_mm, + struct mm_struct *next_mm) +{ + int membarrier_state; + + if (prev_mm == next_mm) + return; + + membarrier_state = atomic_read(&next_mm->membarrier_state); + if (READ_ONCE(rq->membarrier_state) == membarrier_state) + return; + + WRITE_ONCE(rq->membarrier_state, membarrier_state); +} +#else +static inline void membarrier_switch_mm(struct rq *rq, + struct mm_struct *prev_mm, + struct mm_struct *next_mm) +{ +} +#endif + +#ifdef CONFIG_SMP +static inline bool is_per_cpu_kthread(struct task_struct *p) +{ + if (!(p->flags & PF_KTHREAD)) + return false; + + if (p->nr_cpus_allowed != 1) + return false; + + return true; +} +#endif + +void swake_up_all_locked(struct swait_queue_head *q); +void __prepare_to_swait(struct swait_queue_head *q, struct swait_queue *wait); + +/* pelt.h compat CONFIG_SCHED_THERMAL_PRESSURE impossible with MUQSS */ +static inline int +update_thermal_load_avg(u64 now, struct rq *rq, u64 capacity) +{ + return 0; +} + +static inline u64 thermal_load_avg(struct rq *rq) +{ + return 0; +} + +#ifdef CONFIG_RCU_TORTURE_TEST +extern int sysctl_sched_rt_runtime; +#endif + +/* + * Asymmetric CPU capacity bits, used by topology.c. MuQSS does not do + * capacity-aware placement, but the domain build code still maintains the + * list. + */ +struct asym_cap_data { + struct list_head link; + struct rcu_head rcu; + unsigned long capacity; + unsigned long cpus[]; +}; + +extern struct list_head asym_cap_list; + +#define cpu_capacity_span(asym_data) to_cpumask((asym_data)->cpus) + +/* + * Stubs for mainline scheduler machinery MuQSS does not implement. These are + * referenced by the support files retained in build_muqss.c. + */ + +/* No deadline server: topology.c attaches none. See rq->fair_server. */ +static inline void __dl_server_attach_root(void *dl_se, struct rq *rq) +{ +} + +/* sched_ext is mutually exclusive with MuQSS; schedutil still asks. */ +static inline u32 scx_cpuperf_target(s32 cpu) +{ + return 0; +} + +static inline bool scx_switched_all(void) +{ + return false; +} + +/* No nohz idle balancing: MuQSS balances implicitly at task selection. */ +static inline void nohz_run_idle_balance(int cpu) +{ +} + +/* No utilisation clamping, and no CFS utilisation to boost or clamp. */ +static inline bool uclamp_rq_is_capped(struct rq *rq) +{ + return false; +} + +static inline unsigned long cpu_util_cfs_boost(int cpu) +{ + return 0; +} + +#ifdef CONFIG_IRQ_TIME_ACCOUNTING +DECLARE_STATIC_KEY_FALSE(sched_clock_irqtime); + +static inline int irqtime_enabled(void) +{ + return static_branch_likely(&sched_clock_irqtime); +} +#else +static inline int irqtime_enabled(void) +{ + return 0; +} +#endif + +extern unsigned long sugov_effective_cpu_perf(int cpu, unsigned long actual, + unsigned long min, + unsigned long max); + +extern int try_to_wake_up(struct task_struct *p, unsigned int state, + int wake_flags); + +asmlinkage void schedule_user(void); +extern void resched_cpu(int cpu); +extern bool available_idle_cpu(int cpu); +extern void do_set_cpus_allowed(struct task_struct *p, + struct affinity_context *ctx); +extern void __do_set_cpus_allowed(struct task_struct *p, + struct affinity_context *ctx); + +#ifdef CONFIG_PREEMPT_DYNAMIC +extern int sched_dynamic_mode(const char *str); +extern void sched_dynamic_update(int mode); +#endif + +#ifdef CONFIG_CGROUP_SCHED +extern struct task_group *sched_create_group(struct task_group *parent); +extern void sched_online_group(struct task_group *tg, + struct task_group *parent); +extern void sched_destroy_group(struct task_group *tg); +extern void sched_release_group(struct task_group *tg); +#endif + +#endif /* MUQSS_SCHED_H */ diff --git a/kernel/sched/build_muqss.c b/kernel/sched/build_muqss.c new file mode 100644 index 000000000..d1fc94c44 --- /dev/null +++ b/kernel/sched/build_muqss.c @@ -0,0 +1,131 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * The scheduler support files MuQSS retains from mainline, built in a single + * compilation unit for the same build-efficiency reasons as build_policy.c + * and build_utility.c. + * + * MuQSS replaces core.c, fair.c, rt.c, deadline.c, syscalls.c and stop_task.c + * outright, so neither of the mainline aggregates can be used as-is. The + * files gathered here are the subset of build_utility.c and build_policy.c + * that survives, and they are #included rather than built individually + * because several of them (clock.c, completion.c, psi.c) carry no includes of + * their own and only compile as part of an aggregate. + * + * Deliberately absent, and why: + * + * fair.c rt.c deadline.c ext.c stop_task.c policy replaced by MuQSS + * cpudeadline.c cpupri.c pelt.c supporting data for those classes + * syscalls.c MuQSS implements the syscalls + * core_sched.c SCHED_CORE gated off + * debug.c CFS/EEVDF runqueue introspection + * loadavg.c MuQSS keeps its own load average + * autogroup.c cpuacct.c gated off in Kconfig.MuQSS + */ + +/* Headers: */ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include + +#include + +#include + +#include "sched.h" +#include "stats.h" + +/* psi.c needs wq_worker_last_func(); mainline gets this via sched.h. */ +#include "../workqueue_internal.h" + +/* Source code modules: */ + +#include "clock.c" + +#ifdef CONFIG_CPU_FREQ +# include "cpufreq.c" +#endif + +#ifdef CONFIG_CPU_FREQ_GOV_SCHEDUTIL +# include "cpufreq_schedutil.c" +#endif + +#ifdef CONFIG_SCHEDSTATS +# include "stats.c" +#endif + +#include "completion.c" +#include "swait.c" +#include "wait_bit.c" +#include "wait.c" + +#include "cputime.c" +#include "idle.c" + +#ifdef CONFIG_SMP +# include "topology.c" +#endif + +#ifdef CONFIG_PSI +# include "psi.c" +#endif + +#ifdef CONFIG_MEMBARRIER +# include "membarrier.c" +#endif + +#ifdef CONFIG_CPU_ISOLATION +# include "isolation.c" +#endif + +#include "skip_list.c" diff --git a/kernel/sched/cputime.c b/kernel/sched/cputime.c index 06bddaa73..41eca983a 100644 --- a/kernel/sched/cputime.c +++ b/kernel/sched/cputime.c @@ -305,7 +305,7 @@ static inline u64 account_other_time(u64 max) #ifdef CONFIG_64BIT static inline u64 read_sum_exec_runtime(struct task_struct *t) { - return t->se.sum_exec_runtime; + return tsk_seruntime(t); } #else /* !CONFIG_64BIT: */ static u64 read_sum_exec_runtime(struct task_struct *t) @@ -315,7 +315,7 @@ static u64 read_sum_exec_runtime(struct task_struct *t) struct rq *rq; rq = task_rq_lock(t, &rf); - ns = t->se.sum_exec_runtime; + ns = tsk_seruntime(t); task_rq_unlock(rq, t, &rf); return ns; @@ -689,6 +689,19 @@ void account_process_tick(struct task_struct *p, int user_tick) { u64 cputime, steal; + /* + * MuQSS accounts this tick itself, from sched_tick() -> + * update_cpu_clock_tick(), which update_process_times() calls a few + * lines after calling us. Its accounting is in real nanoseconds rather + * than whole ticks, and it is the same p->utime/p->stime and the same + * kcpustat[] fields we would be adding to here, so running both counts + * every busy tick twice. That is invisible in per-task figures, since + * cputime_adjust() rescales utime/stime to sched_time, but it doubles + * the busy buckets of /proc/stat. Leave it all to MuQSS. + */ + if (IS_ENABLED(CONFIG_SCHED_MUQSS)) + return; + if (vtime_accounting_enabled_this_cpu()) return; @@ -811,7 +824,7 @@ void cputime_adjust(struct task_cputime *curr, struct prev_cputime *prev, void task_cputime_adjusted(struct task_struct *p, u64 *ut, u64 *st) { struct task_cputime cputime = { - .sum_exec_runtime = p->se.sum_exec_runtime, + .sum_exec_runtime = tsk_seruntime(p), }; if (task_cputime(p, &cputime.utime, &cputime.stime)) diff --git a/kernel/sched/idle.c b/kernel/sched/idle.c index 052435f4d..120bce36e 100644 --- a/kernel/sched/idle.c +++ b/kernel/sched/idle.c @@ -454,6 +454,7 @@ void cpu_startup_entry(enum cpuhp_state state) do_idle(); } +#ifndef CONFIG_SCHED_MUQSS /* * idle-task scheduling class. */ @@ -591,3 +592,4 @@ DEFINE_SCHED_CLASS(idle) = { .switching_to = switching_to_idle, .update_curr = update_curr_idle, }; +#endif /* CONFIG_SCHED_MUQSS */ diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h index a4a1eb57a..8efa0de07 100644 --- a/kernel/sched/sched.h +++ b/kernel/sched/sched.h @@ -5,6 +5,10 @@ #ifndef _KERNEL_SCHED_SCHED_H #define _KERNEL_SCHED_SCHED_H +#ifdef CONFIG_SCHED_MUQSS +#include "MuQSS.h" +#else /* CONFIG_SCHED_MUQSS */ + #include #include #include @@ -4213,4 +4217,6 @@ DEFINE_CLASS_IS_UNCONDITIONAL(sched_change) #include "ext/ext.h" +#endif /* CONFIG_SCHED_MUQSS */ + #endif /* _KERNEL_SCHED_SCHED_H */ diff --git a/kernel/sched/skip_list.c b/kernel/sched/skip_list.c new file mode 100644 index 000000000..ddc494d56 --- /dev/null +++ b/kernel/sched/skip_list.c @@ -0,0 +1,187 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + Copyright (C) 2011,2016 Con Kolivas. + + Code based on example originally by William Pugh. + +Skip Lists are a probabilistic alternative to balanced trees, as +described in the June 1990 issue of CACM and were invented by +William Pugh in 1987. + +A couple of comments about this implementation: +The routine randomLevel has been hard-coded to generate random +levels using p=0.25. It can be easily changed. + +The insertion routine has been implemented so as to use the +dirty hack described in the CACM paper: if a random level is +generated that is more than the current maximum level, the +current maximum level plus one is used instead. + +Levels start at zero and go up to MaxLevel (which is equal to +MaxNumberOfLevels-1). + +The routines defined in this file are: + +init: defines slnode + +new_skiplist: returns a new, empty list + +randomLevel: Returns a random level based on a u64 random seed passed to it. +In MuQSS, the "niffy" time is used for this purpose. + +insert(l, node, key): inserts node into l ordered by key. This operation +occurs in O(log n) time. + +delnode(slnode, l, node): deletes the node from l. This operation occurs +in O(k) time where k is the number of levels of the node in question +(max 4). The original delete function occurred in O(log n) time and +involved a search. + +MuQSS Notes: In this implementation of skiplists, there are bidirectional +next/prev pointers. The node is embedded in task_struct; the task is +recovered with container_of(). The key here is chosen by the scheduler +so as to sort tasks according to the priority list requirements. The +scheduler lookup occurs in O(1) time because it is always the first +item in the level 0 linked list. Since the task struct embeds the node, +it can also remove it much faster than the original implementation with +the aid of prev<->next pointer manipulation and no searching. + +*/ + +#include +#include +#include +#include + +static struct kmem_cache *skiplist_cache __read_mostly; + +#define MaxNumberOfLevels SKIPLIST_MAXLEVEL +#define MaxLevel (MaxNumberOfLevels - 1) + +void skiplist_init(skiplist_node *slnode) +{ + int i; + + slnode->key = 0xFFFFFFFFFFFFFFFF; + for (i = 0; i < MaxNumberOfLevels; i++) + slnode->next[i] = slnode->prev[i] = slnode; +} + +void __init skiplist_cache_init(void) +{ + skiplist_cache = kmem_cache_create("skiplist", sizeof(skiplist), + 0, SLAB_HWCACHE_ALIGN, NULL); + BUG_ON(!skiplist_cache); +} + +skiplist *new_skiplist(skiplist_node *slnode) +{ + skiplist *l = kmem_cache_zalloc(skiplist_cache, GFP_ATOMIC); + + BUG_ON(!l); + l->header = slnode; + l->best_key = ~0ULL; + return l; +} + +void skiplist_free(skiplist *l) +{ + kmem_cache_free(skiplist_cache, l); +} + +void free_skiplist(skiplist *l) +{ + skiplist_node *p, *q; + + p = l->header; + do { + q = p->next[0]; + p->next[0]->prev[0] = q->prev[0]; + skiplist_node_init(p); + p = q; + } while (p != l->header); + skiplist_free(l); +} + +void skiplist_node_init(skiplist_node *node) +{ + memset(node, 0, sizeof(skiplist_node)); +} + +static inline unsigned int randomLevel(const long unsigned int randseed) +{ + /* + * Two bits per extra level (p = 0.25). Search 2*MaxLevel+1 bits + * so the result stays in 0..MaxLevel; this matches the old + * 8-slot arrays whose random height never exceeded 3. + */ + return find_first_bit(&randseed, MaxLevel * 2 + 1) / 2; +} + +void skiplist_insert(skiplist *l, skiplist_node *node, keyType key, unsigned int randseed) +{ + skiplist_node *update[MaxNumberOfLevels]; + skiplist_node *p, *q; + int k = l->level; + + p = l->header; + do { + while (q = p->next[k], q->key <= key) + p = q; + update[k] = p; + } while (--k >= 0); + + WRITE_ONCE(l->entries, l->entries + 1); + k = randomLevel(randseed); + if (k > l->level) { + k = ++l->level; + update[k] = l->header; + } + + node->key = key; + do { + p = update[k]; + node->next[k] = p->next[k]; + p->next[k] = node; + node->prev[k] = p; + node->next[k]->prev[k] = node; + } while (--k >= 0); +} + +/* + * Unlink @node from @l. Returns true if it was linked and has been removed, + * false if it was not on any list and nothing was changed - the caller's own + * accounting has to be skipped along with the removal in that case. + */ +bool skiplist_delete(skiplist *l, skiplist_node *node) +{ + int k, m; + + /* + * Insert fills levels contiguously from 0 up and removal zeroes the + * whole node, so a NULL next[] marks the top of this node and stands + * in for the level count the node used to carry. + */ + for (k = 0; k < MaxNumberOfLevels && node->next[k]; k++) { + node->prev[k]->next[k] = node->next[k]; + node->next[k]->prev[k] = node->prev[k]; + } + m = k - 1; + /* + * A node that was never queued has no levels linked at all. Leave + * before decrementing entries, which would otherwise go negative and + * make the emptiness tests in earliest_deadline_task() pass on an + * empty list, handing the caller the header as if it were a task. + */ + if (WARN_ON_ONCE(m < 0)) + return false; + skiplist_node_init(node); + if (m == l->level) { + while (l->header->next[m] == l->header && l->header->prev[m] == l->header && m > 0) + m--; + l->level = m; + } + WRITE_ONCE(l->entries, l->entries - 1); + + return true; +} diff --git a/kernel/sched/stats.h b/kernel/sched/stats.h index ebe0a7765..c662b16fb 100644 --- a/kernel/sched/stats.h +++ b/kernel/sched/stats.h @@ -89,6 +89,8 @@ static inline void rq_sched_info_depart (struct rq *rq, unsigned long long delt #endif /* CONFIG_SCHEDSTATS */ +#ifndef CONFIG_SCHED_MUQSS +/* Only reachable from fair.c, debug.c and core.c, none of which MuQSS builds. */ static inline struct sched_statistics * __schedstats_from_se(struct sched_entity *se) { @@ -98,6 +100,7 @@ __schedstats_from_se(struct sched_entity *se) #endif return &task_of(se)->stats; } +#endif /* CONFIG_SCHED_MUQSS */ #ifdef CONFIG_PSI void psi_task_change(struct task_struct *task, int clear, int set); @@ -135,6 +138,23 @@ static inline void psi_enqueue(struct task_struct *p, int flags) if (task_on_cpu(task_rq(p), p)) return; +#ifdef CONFIG_SCHED_MUQSS + /* + * MuQSS has no delayed-dequeue (no task_struct::se), so a migration + * always moves a runnable task and a non-migration is a wakeup. + */ + if (flags & ENQUEUE_MIGRATED) { + set = TSK_RUNNING; + if (p->in_memstall) + set |= TSK_MEMSTALL | TSK_MEMSTALL_RUNNING; + } else { + if (p->in_iowait) + clear |= TSK_IOWAIT; + set = TSK_RUNNING; + if (p->in_memstall) + set |= TSK_MEMSTALL_RUNNING; + } +#else if (p->se.sched_delayed) { /* CPU migration of "sleeping" task */ WARN_ON_ONCE(!(flags & ENQUEUE_MIGRATED)); @@ -155,6 +175,7 @@ static inline void psi_enqueue(struct task_struct *p, int flags) if (p->in_memstall) set |= TSK_MEMSTALL_RUNNING; } +#endif /* CONFIG_SCHED_MUQSS */ psi_task_change(p, clear, set); } diff --git a/kernel/time/Kconfig b/kernel/time/Kconfig index d098ac39b..8fffe5e0e 100644 --- a/kernel/time/Kconfig +++ b/kernel/time/Kconfig @@ -103,18 +103,36 @@ config NO_HZ_COMMON bool select TICK_ONESHOT +# +# Silent symbols keep CONFIG_NO_HZ_* / CONFIG_HZ_PERIODIC available to the +# rest of the tree. The promptable *_NODEF names are what olddefconfig can +# stick on; renaming every choice member (including idle) means a saved +# "# CONFIG_NO_HZ_IDLE is not set" cannot pin the choice onto full dynticks. +# +config HZ_PERIODIC + bool + +config NO_HZ_IDLE + bool + +config NO_HZ_FULL + bool + choice prompt "Timer tick handling" - default NO_HZ_IDLE if NO_HZ + default NO_HZ_IDLE_NODEF if NO_HZ + default NO_HZ_IDLE_NODEF -config HZ_PERIODIC +config HZ_PERIODIC_NODEF bool "Periodic timer ticks (constant rate, no dynticks)" + select HZ_PERIODIC help This option keeps the tick running periodically at a constant rate, even when the CPU doesn't need it. -config NO_HZ_IDLE +config NO_HZ_IDLE_NODEF bool "Idle dynticks system (tickless idle)" + select NO_HZ_IDLE select NO_HZ_COMMON help This option enables a tickless idle system: timer interrupts @@ -123,8 +141,9 @@ config NO_HZ_IDLE Most of the time you want to say Y here. -config NO_HZ_FULL +config NO_HZ_FULL_NODEF bool "Full dynticks system (tickless)" + select NO_HZ_FULL # NO_HZ_COMMON dependency # We need at least one periodic CPU for timekeeping depends on SMP @@ -150,6 +169,8 @@ config NO_HZ_FULL By default, without passing the nohz_full parameter, this behaves just like NO_HZ_IDLE. + Not recommended for desktops, laptops, or mobile devices. + If you're a distro say Y. endchoice diff --git a/kernel/time/clockevents.c b/kernel/time/clockevents.c index 0014d163f..048aedc35 100644 --- a/kernel/time/clockevents.c +++ b/kernel/time/clockevents.c @@ -194,8 +194,8 @@ int clockevents_tick_resume(struct clock_event_device *dev) #ifdef CONFIG_GENERIC_CLOCKEVENTS_MIN_ADJUST -/* Limit min_delta to a jiffy */ -#define MIN_DELTA_LIMIT (NSEC_PER_SEC / HZ) +/* Limit min_delta to the configurable hrtimer granularity */ +#define MIN_DELTA_LIMIT (hrtimer_granularity_us * NSEC_PER_USEC) /** * clockevents_increase_min_delta - raise minimum delta of a clock event device diff --git a/kernel/time/posix-cpu-timers.c b/kernel/time/posix-cpu-timers.c index a7d3e8229..a8d12763c 100644 --- a/kernel/time/posix-cpu-timers.c +++ b/kernel/time/posix-cpu-timers.c @@ -223,7 +223,7 @@ static void task_sample_cputime(struct task_struct *p, u64 *samples) u64 stime, utime; task_cputime(p, &utime, &stime); - store_samples(samples, stime, utime, p->se.sum_exec_runtime); + store_samples(samples, stime, utime, tsk_seruntime(p)); } static void proc_sample_cputime_atomic(struct task_cputime_atomic *at, @@ -901,10 +901,12 @@ static void collect_posix_cputimers(struct posix_cputimers *pct, u64 *samples, static inline void check_dl_overrun(struct task_struct *tsk) { +#ifndef CONFIG_SCHED_MUQSS if (tsk->dl.dl_overrun) { tsk->dl.dl_overrun = 0; send_signal_locked(SIGXCPU, SEND_SIG_PRIV, tsk, PIDTYPE_TGID); } +#endif } static bool check_rlimit(u64 time, u64 limit, int signo, bool rt, bool hard) @@ -948,7 +950,7 @@ static void check_thread_timers(struct task_struct *tsk, soft = task_rlimit(tsk, RLIMIT_RTTIME); if (soft != RLIM_INFINITY) { /* Task RT timeout is accounted in jiffies. RTTIME is usec */ - unsigned long rttime = tsk->rt.timeout * (USEC_PER_SEC / HZ); + unsigned long rttime = tsk_rttimeout(tsk) * (USEC_PER_SEC / HZ); unsigned long hard = task_rlimit_max(tsk, RLIMIT_RTTIME); /* At the hard limit, send SIGKILL. No further action. */ @@ -1179,7 +1181,7 @@ static inline bool fastpath_timer_check(struct task_struct *tsk) return true; } - if (dl_task(tsk) && tsk->dl.dl_overrun) + if (dl_task(tsk) && tsk_dl_overrun(tsk)) return true; return false; diff --git a/kernel/time/sleep_timeout.c b/kernel/time/sleep_timeout.c index 3c90574bd..fa7af0ead 100644 --- a/kernel/time/sleep_timeout.c +++ b/kernel/time/sleep_timeout.c @@ -4,13 +4,57 @@ */ #include +#include #include #include #include #include +#include #include "tick-internal.h" +/* + * Granularity, in microseconds, below which a high resolution timeout is not + * worth arming. Also caps how far clockevents_increase_min_delta() is allowed + * to raise a clock event device's minimum delta. + */ +int __read_mostly hrtimer_granularity_us = 100; + +/* The timeout, in microseconds, used by schedule_min_hrtimeout(). */ +int __read_mostly hrtimeout_min_us = 500; + +#ifdef CONFIG_SYSCTL +static const int hrtimeout_us_max = 10000; + +static const struct ctl_table hrtimeout_sysctl[] = { + { + .procname = "hrtimer_granularity_us", + .data = &hrtimer_granularity_us, + .maxlen = sizeof(int), + .mode = 0644, + .proc_handler = proc_dointvec_minmax, + .extra1 = SYSCTL_ONE, + .extra2 = (void *)&hrtimeout_us_max, + }, + { + .procname = "hrtimeout_min_us", + .data = &hrtimeout_min_us, + .maxlen = sizeof(int), + .mode = 0644, + .proc_handler = proc_dointvec_minmax, + .extra1 = SYSCTL_ONE, + .extra2 = (void *)&hrtimeout_us_max, + }, +}; + +static int __init hrtimeout_sysctl_init(void) +{ + register_sysctl("kernel", hrtimeout_sysctl); + return 0; +} +device_initcall(hrtimeout_sysctl_init); +#endif /* CONFIG_SYSCTL */ + /* * Since schedule_timeout()'s timer is defined on the stack, it must store * the target task on the stack as well. @@ -279,10 +323,164 @@ int __sched schedule_hrtimeout(ktime_t *expires, const enum hrtimer_mode mode) } EXPORT_SYMBOL_GPL(schedule_hrtimeout); +#ifdef CONFIG_HIGH_RES_TIMERS +/* + * As per schedule_hrtimeout but takes a millisecond value and returns how + * many milliseconds are left. + */ +long __sched schedule_msec_hrtimeout(long timeout) +{ + struct hrtimer_sleeper t; + int delta, jiffs; + ktime_t expires; + + if (!timeout) { + __set_current_state(TASK_RUNNING); + return 0; + } + + jiffs = msecs_to_jiffies(timeout); + /* + * If regular timer resolution is adequate or hrtimer resolution is not + * (yet) better than Hz, as would occur during startup, use regular + * timers. Freezing gets the same treatment as some drivers still do not + * correctly use freezable timeouts. + */ + if (jiffs > 4 || hrtimer_resolution >= NSEC_PER_SEC / HZ || pm_freezing) + return schedule_timeout(jiffs); + + /* The jiffs > 4 test above bounds timeout well under one second. */ + delta = (timeout % 1000) * NSEC_PER_MSEC; + expires = ktime_set(0, delta); + + hrtimer_setup_sleeper_on_stack(&t, CLOCK_MONOTONIC, HRTIMER_MODE_REL); + /* + * No slack is given here. The whole point of taking the high resolution + * path is that the timeout is too short for the tick to resolve, and a + * slack of delta would push the expiry out to twice the request. + */ + hrtimer_set_expires(&t.timer, expires); + + hrtimer_sleeper_start_expires(&t, HRTIMER_MODE_REL); + + if (likely(t.task)) + schedule(); + + hrtimer_cancel(&t.timer); + destroy_hrtimer_on_stack(&t.timer); + + __set_current_state(TASK_RUNNING); + + expires = hrtimer_expires_remaining(&t.timer); + timeout = ktime_to_ms(expires); + return timeout < 0 ? 0 : timeout; +} + +EXPORT_SYMBOL(schedule_msec_hrtimeout); + +/* + * As per schedule_msec_hrtimeout but takes a microsecond value and returns + * how many microseconds are left. + */ +static long __sched schedule_usec_hrtimeout(long timeout) +{ + struct hrtimer_sleeper t; + ktime_t expires; + int delta; + + if (!timeout) { + __set_current_state(TASK_RUNNING); + return 0; + } + + if (hrtimer_resolution >= NSEC_PER_SEC / HZ) + return schedule_timeout(usecs_to_jiffies(timeout)); + + if (timeout < hrtimer_granularity_us) + timeout = hrtimer_granularity_us; + delta = (timeout % USEC_PER_SEC) * NSEC_PER_USEC; + expires = ktime_set(0, delta); + + hrtimer_setup_sleeper_on_stack(&t, CLOCK_MONOTONIC, HRTIMER_MODE_REL); + /* No slack is given here, as per schedule_msec_hrtimeout(). */ + hrtimer_set_expires(&t.timer, expires); + + hrtimer_sleeper_start_expires(&t, HRTIMER_MODE_REL); + + if (likely(t.task)) + schedule(); + + hrtimer_cancel(&t.timer); + destroy_hrtimer_on_stack(&t.timer); + + __set_current_state(TASK_RUNNING); + + expires = hrtimer_expires_remaining(&t.timer); + timeout = ktime_to_us(expires); + return timeout < 0 ? 0 : timeout; +} + +long __sched schedule_min_hrtimeout(void) +{ + return usecs_to_jiffies(schedule_usec_hrtimeout(hrtimeout_min_us)); +} + +EXPORT_SYMBOL(schedule_min_hrtimeout); + +long __sched schedule_msec_hrtimeout_interruptible(long timeout) +{ + __set_current_state(TASK_INTERRUPTIBLE); + return schedule_msec_hrtimeout(timeout); +} +EXPORT_SYMBOL(schedule_msec_hrtimeout_interruptible); + +long __sched schedule_msec_hrtimeout_uninterruptible(long timeout) +{ + __set_current_state(TASK_UNINTERRUPTIBLE); + return schedule_msec_hrtimeout(timeout); +} +EXPORT_SYMBOL(schedule_msec_hrtimeout_uninterruptible); + +#else /* !CONFIG_HIGH_RES_TIMERS */ + +/* + * Without high resolution timers there is nothing better than the regular + * jiffy based timeouts to fall back on. + */ +long __sched schedule_msec_hrtimeout(long timeout) +{ + return schedule_timeout(msecs_to_jiffies(timeout)); +} +EXPORT_SYMBOL(schedule_msec_hrtimeout); + +long __sched schedule_min_hrtimeout(void) +{ + return schedule_timeout(1); +} +EXPORT_SYMBOL(schedule_min_hrtimeout); + +long __sched schedule_msec_hrtimeout_interruptible(long timeout) +{ + return schedule_timeout_interruptible(msecs_to_jiffies(timeout)); +} +EXPORT_SYMBOL(schedule_msec_hrtimeout_interruptible); + +long __sched schedule_msec_hrtimeout_uninterruptible(long timeout) +{ + return schedule_timeout_uninterruptible(msecs_to_jiffies(timeout)); +} +EXPORT_SYMBOL(schedule_msec_hrtimeout_uninterruptible); + +#endif /* CONFIG_HIGH_RES_TIMERS */ + /** * msleep - sleep safely even with waitqueue interruptions * @msecs: Requested sleep duration in milliseconds * + * Sleeps shorter than five ticks are handed to a high resolution timer instead, + * where that has better resolution than the tick, so the slack described below + * only applies to the jiffy based path. + * * msleep() uses jiffy based timeouts for the sleep duration. Because of the * design of the timer wheel, the maximum additional percentage delay (slack) is * 12.5%. This is only valid for timers which will end up in level 1 or a higher @@ -312,7 +510,19 @@ EXPORT_SYMBOL_GPL(schedule_hrtimeout); */ void msleep(unsigned int msecs) { - unsigned long timeout = msecs_to_jiffies(msecs); + int jiffs = msecs_to_jiffies(msecs); + unsigned long timeout; + + /* + * Use high resolution timers where the resolution of tick based + * timers is inadequate. + */ + if (jiffs < 5 && hrtimer_resolution < NSEC_PER_SEC / HZ && !pm_freezing) { + while (msecs) + msecs = schedule_msec_hrtimeout_uninterruptible(msecs); + return; + } + timeout = jiffs; while (timeout) timeout = schedule_timeout_uninterruptible(timeout); @@ -333,7 +543,15 @@ EXPORT_SYMBOL(msleep); */ unsigned long msleep_interruptible(unsigned int msecs) { - unsigned long timeout = msecs_to_jiffies(msecs); + int jiffs = msecs_to_jiffies(msecs); + unsigned long timeout; + + if (jiffs < 5 && hrtimer_resolution < NSEC_PER_SEC / HZ && !pm_freezing) { + while (msecs && !signal_pending(current)) + msecs = schedule_msec_hrtimeout_interruptible(msecs); + return msecs; + } + timeout = jiffs; while (timeout && !signal_pending(current)) timeout = schedule_timeout_interruptible(timeout); diff --git a/kernel/time/tick-internal.h b/kernel/time/tick-internal.h index 597d816d2..e32c0f230 100644 --- a/kernel/time/tick-internal.h +++ b/kernel/time/tick-internal.h @@ -13,6 +13,9 @@ struct timer_events { u64 global; }; +/* Defined in sleep_timeout.c, tunable via sysctl */ +extern int hrtimer_granularity_us; + #ifdef CONFIG_GENERIC_CLOCKEVENTS # define TICK_DO_TIMER_NONE -1 diff --git a/kernel/trace/rv/monitors/deadline/Kconfig b/kernel/trace/rv/monitors/deadline/Kconfig index 38804a6ad..ed2844a2b 100644 --- a/kernel/trace/rv/monitors/deadline/Kconfig +++ b/kernel/trace/rv/monitors/deadline/Kconfig @@ -1,5 +1,5 @@ config RV_MON_DEADLINE - depends on RV + depends on RV && !SCHED_MUQSS bool "deadline monitor" help Collection of monitors to check the deadline scheduler and server diff --git a/kernel/workqueue.c b/kernel/workqueue.c index 78068ae8f..e95563d80 100644 --- a/kernel/workqueue.c +++ b/kernel/workqueue.c @@ -56,6 +56,7 @@ #include #include #include +#include #include "workqueue_internal.h" @@ -1438,7 +1439,7 @@ void wq_worker_running(struct task_struct *task) * CPU intensive auto-detection cares about how long a work item hogged * CPU without sleeping. Reset the starting timestamp on wakeup. */ - worker->current_at = worker->task->se.sum_exec_runtime; + worker->current_at = tsk_seruntime(worker->task); WRITE_ONCE(worker->sleeping, 0); } @@ -1523,7 +1524,7 @@ void wq_worker_tick(struct task_struct *task) * We probably want to make this prettier in the future. */ if ((worker->flags & WORKER_NOT_RUNNING) || READ_ONCE(worker->sleeping) || - worker->task->se.sum_exec_runtime - worker->current_at < + tsk_seruntime(worker->task) - worker->current_at < wq_cpu_intensive_thresh_us * NSEC_PER_USEC) return; @@ -2449,6 +2450,7 @@ bool queue_work_on(int cpu, struct workqueue_struct *wq, if (!test_and_set_bit(WORK_STRUCT_PENDING_BIT, work_data_bits(work)) && !clear_pending_if_disabled(work)) { + muqss_work_set_owner(work); __queue_work(cpu, wq, work); ret = true; } @@ -2530,6 +2532,7 @@ bool queue_work_node(int node, struct workqueue_struct *wq, !clear_pending_if_disabled(work)) { int cpu = select_numa_node_cpu(node); + muqss_work_set_owner(work); __queue_work(cpu, wq, work); ret = true; } @@ -2617,6 +2620,7 @@ bool queue_delayed_work_on(int cpu, struct workqueue_struct *wq, if (!test_and_set_bit(WORK_STRUCT_PENDING_BIT, work_data_bits(work)) && !clear_pending_if_disabled(work)) { + muqss_work_set_owner(work); __queue_delayed_work(cpu, wq, dwork, delay); ret = true; } @@ -2652,8 +2656,10 @@ bool mod_delayed_work_on(int cpu, struct workqueue_struct *wq, ret = work_grab_pending(&dwork->work, WORK_CANCEL_DELAYED, &irq_flags); - if (!clear_pending_if_disabled(&dwork->work)) + if (!clear_pending_if_disabled(&dwork->work)) { + muqss_work_set_owner(&dwork->work); __queue_delayed_work(cpu, wq, dwork, delay); + } local_irq_restore(irq_flags); return ret; @@ -2690,6 +2696,7 @@ bool queue_rcu_work(struct workqueue_struct *wq, struct rcu_work *rwork) */ if (!test_and_set_bit(WORK_STRUCT_PENDING_BIT, work_data_bits(work)) && !WARN_ON_ONCE(clear_pending_if_disabled(work))) { + muqss_work_set_owner(work); rwork->wq = wq; call_rcu_hurry(&rwork->rcu, rcu_work_rcufn); return true; @@ -3223,7 +3230,9 @@ __acquires(&pool->lock) { struct pool_workqueue *pwq = get_work_pwq(work); struct worker_pool *pool = worker->pool; + struct muqss_kern_window kern_window; unsigned long work_data; + unsigned int muqss_owner; int lockdep_start_depth, rcu_start_depth; bool bh_draining = pool->flags & POOL_BH_DRAINING; #ifdef CONFIG_LOCKDEP @@ -3249,11 +3258,23 @@ __acquires(&pool->lock) worker->current_func = work->func; worker->current_pwq = pwq; if (worker->task) - worker->current_at = worker->task->se.sum_exec_runtime; + worker->current_at = tsk_seruntime(worker->task); worker->current_start = jiffies; work_data = *work_data_bits(work); worker->current_color = get_work_color(work_data); + /* + * Read while PENDING still holds this work item for us. Once it is + * cleared below another CPU may requeue the same item and restamp it + * with a different owner, and the one we are about to run belongs to + * whoever queued it, not to them. + * + * A BH pool runs work in softirq context on top of whichever task was + * interrupted, so worker->task is NULL and there is no worker runtime + * to measure. Charge nobody there. + */ + muqss_owner = worker->task ? muqss_work_owner(work) : 0; + /* * Record wq name for cmdline and debug reporting, may get * overridden through set_worker_desc(). @@ -3318,6 +3339,13 @@ __acquires(&pool->lock) * workqueues), so hiding them isn't a problem. */ lockdep_invariant_state(true); + /* + * Everything from here to muqss_kerntime_end() is done on the queueing + * task's account: the CPU time it costs this worker is charged back to + * it, and so is any I/O the work item submits or any page it dirties, + * which would otherwise be lost to a kernel thread. + */ + muqss_kerntime_begin(&kern_window, muqss_owner); trace_workqueue_execute_start(work); worker->current_func(work); /* @@ -3325,6 +3353,7 @@ __acquires(&pool->lock) * point will only record its address. */ trace_workqueue_execute_end(work, worker->current_func); + muqss_kerntime_end(&kern_window); lock_map_release(&lockdep_map); if (!bh_draining) diff --git a/mm/khugepaged.c b/mm/khugepaged.c index 617bca76d..ccff7f1be 100644 --- a/mm/khugepaged.c +++ b/mm/khugepaged.c @@ -975,7 +975,7 @@ static void khugepaged_alloc_sleep(void) add_wait_queue(&khugepaged_wait, &wait); __set_current_state(TASK_INTERRUPTIBLE|TASK_FREEZABLE); - schedule_timeout(msecs_to_jiffies(khugepaged_alloc_sleep_millisecs)); + schedule_msec_hrtimeout(khugepaged_alloc_sleep_millisecs); remove_wait_queue(&khugepaged_wait, &wait); } diff --git a/mm/page-writeback.c b/mm/page-writeback.c index e98748112..3eac217a6 100644 --- a/mm/page-writeback.c +++ b/mm/page-writeback.c @@ -38,6 +38,7 @@ #include #include #include +#include #include #include @@ -2636,6 +2637,12 @@ static void folio_account_dirtied(struct folio *folio, current->nr_dirtied += nr; __this_cpu_add(bdp_ratelimits, nr); + /* + * Record who dirtied this so the writeback issued for it later + * by a flusher thread can still be charged to them. + */ + muqss_iotime_dirty_folio(folio); + mem_cgroup_track_foreign_dirty(folio, wb); } } diff --git a/mm/page_alloc.c b/mm/page_alloc.c index f5290d209..bed8be6bb 100644 --- a/mm/page_alloc.c +++ b/mm/page_alloc.c @@ -1405,6 +1405,7 @@ static __always_inline bool __free_pages_prepare(struct page *page, } page_cpupid_reset_last(page); + page_io_owner_reset(page); page->flags.f &= ~PAGE_FLAGS_CHECK_AT_PREP; page->private = 0; reset_page_owner(page, order); diff --git a/net/core/pktgen.c b/net/core/pktgen.c index ee64f3012..3379657a1 100644 --- a/net/core/pktgen.c +++ b/net/core/pktgen.c @@ -1992,7 +1992,7 @@ static ssize_t pktgen_thread_write(struct file *file, mutex_lock(&pktgen_thread_lock); t->control |= T_REMDEVALL; mutex_unlock(&pktgen_thread_lock); - schedule_timeout_interruptible(msecs_to_jiffies(125)); /* Propagate thread->control */ + schedule_msec_hrtimeout_interruptible(125); /* Propagate thread->control */ ret = count; sprintf(pg_result, "OK: rem_device_all"); goto out; @@ -2064,7 +2064,7 @@ static void pktgen_mark_device(const struct pktgen_net *pn, const char *ifname) mutex_unlock(&pktgen_thread_lock); pr_debug("%s: waiting for %s to disappear....\n", __func__, ifname); - schedule_timeout_interruptible(msecs_to_jiffies(msec_per_try)); + schedule_msec_hrtimeout_interruptible(msec_per_try); mutex_lock(&pktgen_thread_lock); if (++i >= max_tries) { @@ -3321,7 +3321,7 @@ static void pktgen_run_all_threads(struct pktgen_net *pn) pktgen_handle_all_threads(pn, T_RUN); /* Propagate thread->control */ - schedule_timeout_interruptible(msecs_to_jiffies(125)); + schedule_msec_hrtimeout_interruptible(125); pktgen_wait_all_threads_run(pn); } @@ -3333,7 +3333,7 @@ static void pktgen_reset_all_threads(struct pktgen_net *pn) pktgen_handle_all_threads(pn, T_REMDEVALL); /* Propagate thread->control */ - schedule_timeout_interruptible(msecs_to_jiffies(125)); + schedule_msec_hrtimeout_interruptible(125); pktgen_wait_all_threads_run(pn); } diff --git a/sound/hda/core/controller.c b/sound/hda/core/controller.c index 69e11d62b..4496fea04 100644 --- a/sound/hda/core/controller.c +++ b/sound/hda/core/controller.c @@ -340,7 +340,7 @@ static int snd_hdac_bus_get_response_rirb(struct hdac_bus *bus, break; #define LOOP_COUNT_MAX 3000 if (!bus->polling_mode) { - schedule_timeout(msecs_to_jiffies(2)); + schedule_msec_hrtimeout(2); } else if (bus->needs_damn_long_delay || loopcounter > LOOP_COUNT_MAX) { if (loopcounter > LOOP_COUNT_MAX && !warned) { diff --git a/sound/isa/msnd/msnd_pinnacle.c b/sound/isa/msnd/msnd_pinnacle.c index 0d5f4461a..215fff762 100644 --- a/sound/isa/msnd/msnd_pinnacle.c +++ b/sound/isa/msnd/msnd_pinnacle.c @@ -490,7 +490,7 @@ static int snd_msnd_calibrate_adc(struct snd_msnd *chip, u16 srate) & ~0x0001, chip->SMA + SMA_wCurrHostStatusFlags); if (snd_msnd_send_word(chip, 0, 0, HDEXAR_CAL_A_TO_D) == 0 && snd_msnd_send_dsp_cmd_chk(chip, HDEX_AUX_REQ) == 0) { - schedule_timeout_interruptible(msecs_to_jiffies(333)); + schedule_msec_hrtimeout_interruptible(333); return 0; } dev_warn(chip->card->dev, LOGNAME ": ADC calibration failed\n"); diff --git a/sound/pci/maestro3.c b/sound/pci/maestro3.c index dd45ffa17..56b075dec 100644 --- a/sound/pci/maestro3.c +++ b/sound/pci/maestro3.c @@ -1999,7 +1999,7 @@ static void snd_m3_ac97_reset(struct snd_m3 *chip) outw(0, io + GPIO_DATA); outw(dir | GPO_PRIMARY_AC97, io + GPIO_DIRECTION); - schedule_timeout_uninterruptible(msecs_to_jiffies(delay1)); + schedule_msec_hrtimeout_uninterruptible(delay1); outw(GPO_PRIMARY_AC97, io + GPIO_DATA); udelay(5); @@ -2007,7 +2007,7 @@ static void snd_m3_ac97_reset(struct snd_m3 *chip) outw(IO_SRAM_ENABLE | SERIAL_AC_LINK_ENABLE, io + RING_BUS_CTRL_A); outw(~0, io + GPIO_MASK); - schedule_timeout_uninterruptible(msecs_to_jiffies(delay2)); + schedule_msec_hrtimeout_uninterruptible(delay2); if (! snd_m3_try_read_vendor(chip)) break; @@ -2054,7 +2054,7 @@ static int snd_m3_mixer(struct snd_m3 *chip) /* seems ac97 PCM needs initialization.. hack hack.. */ snd_ac97_write(chip->ac97, AC97_PCM, 0x8000 | (15 << 8) | 15); - schedule_timeout_uninterruptible(msecs_to_jiffies(100)); + schedule_msec_hrtimeout_uninterruptible(100); snd_ac97_write(chip->ac97, AC97_PCM, 0); #ifndef CONFIG_SND_MAESTRO3_INPUT diff --git a/sound/pci/ymfpci/ymfpci_main.c b/sound/pci/ymfpci/ymfpci_main.c index 2ccb976e6..ffcfe3923 100644 --- a/sound/pci/ymfpci/ymfpci_main.c +++ b/sound/pci/ymfpci/ymfpci_main.c @@ -742,7 +742,7 @@ static void snd_ymfpci_irq_wait(struct snd_ymfpci *chip) init_waitqueue_entry(&wait, current); add_wait_queue(&chip->interrupt_sleep, &wait); atomic_inc(&chip->interrupt_sleep_count); - schedule_timeout_uninterruptible(msecs_to_jiffies(50)); + schedule_msec_hrtimeout_uninterruptible(50); remove_wait_queue(&chip->interrupt_sleep, &wait); } } diff --git a/sound/soc/codecs/rt5631.c b/sound/soc/codecs/rt5631.c index ed1233973..5e019fdd3 100644 --- a/sound/soc/codecs/rt5631.c +++ b/sound/soc/codecs/rt5631.c @@ -417,7 +417,7 @@ static void onebit_depop_mute_stage(struct snd_soc_component *component, int ena hp_zc = snd_soc_component_read(component, RT5631_INT_ST_IRQ_CTRL_2); snd_soc_component_write(component, RT5631_INT_ST_IRQ_CTRL_2, hp_zc & 0xf7ff); if (enable) { - schedule_timeout_uninterruptible(msecs_to_jiffies(10)); + schedule_msec_hrtimeout_uninterruptible(10); /* config one-bit depop parameter */ rt5631_write_index(component, RT5631_SPK_INTL_CTRL, 0x307f); snd_soc_component_update_bits(component, RT5631_HP_OUT_VOL, @@ -529,7 +529,7 @@ static void depop_seq_mute_stage(struct snd_soc_component *component, int enable hp_zc = snd_soc_component_read(component, RT5631_INT_ST_IRQ_CTRL_2); snd_soc_component_write(component, RT5631_INT_ST_IRQ_CTRL_2, hp_zc & 0xf7ff); if (enable) { - schedule_timeout_uninterruptible(msecs_to_jiffies(10)); + schedule_msec_hrtimeout_uninterruptible(10); /* config depop sequence parameter */ rt5631_write_index(component, RT5631_SPK_INTL_CTRL, 0x302f); @@ -1496,8 +1496,7 @@ static int rt5631_codec_set_dai_pll(struct snd_soc_dai *codec_dai, int pll_id, "change PLL in master mode\n"); snd_soc_component_write(component, RT5631_PLL_CTRL, codec_master_pll_div[i].reg_val); - schedule_timeout_uninterruptible( - msecs_to_jiffies(20)); + schedule_msec_hrtimeout_uninterruptible(20); snd_soc_component_update_bits(component, RT5631_GLOBAL_CLK_CTRL, RT5631_SYSCLK_SOUR_SEL_MASK | @@ -1515,8 +1514,7 @@ static int rt5631_codec_set_dai_pll(struct snd_soc_dai *codec_dai, int pll_id, "change PLL in slave mode\n"); snd_soc_component_write(component, RT5631_PLL_CTRL, codec_slave_pll_div[i].reg_val); - schedule_timeout_uninterruptible( - msecs_to_jiffies(20)); + schedule_msec_hrtimeout_uninterruptible(20); snd_soc_component_update_bits(component, RT5631_GLOBAL_CLK_CTRL, RT5631_SYSCLK_SOUR_SEL_MASK | diff --git a/sound/soc/codecs/wm8350.c b/sound/soc/codecs/wm8350.c index ad68ff179..aee9093db 100644 --- a/sound/soc/codecs/wm8350.c +++ b/sound/soc/codecs/wm8350.c @@ -234,11 +234,9 @@ static void wm8350_pga_work(struct work_struct *work) out2->ramp == WM8350_RAMP_UP) { /* delay is longer over 0dB as increases are larger */ if (i >= WM8350_OUTn_0dB) - schedule_timeout_interruptible(msecs_to_jiffies - (2)); + schedule_msec_hrtimeout_interruptible(2); else - schedule_timeout_interruptible(msecs_to_jiffies - (1)); + schedule_msec_hrtimeout_interruptible(1); } else udelay(50); /* doesn't matter if we delay longer */ } @@ -1122,9 +1120,7 @@ static int wm8350_set_bias_level(struct snd_soc_component *component, (platform->dis_out4 << 6)); /* wait for discharge */ - schedule_timeout_interruptible(msecs_to_jiffies - (platform-> - cap_discharge_msecs)); + schedule_msec_hrtimeout_interruptible(platform->cap_discharge_msecs); /* enable antipop */ wm8350_reg_write(wm8350, WM8350_ANTI_POP_CONTROL, @@ -1138,9 +1134,7 @@ static int wm8350_set_bias_level(struct snd_soc_component *component, WM8350_VBUFEN); /* wait for vmid */ - schedule_timeout_interruptible(msecs_to_jiffies - (platform-> - vmid_charge_msecs)); + schedule_msec_hrtimeout_interruptible(platform->vmid_charge_msecs); /* turn on vmid 300k */ pm1 = wm8350_reg_read(wm8350, WM8350_POWER_MGMT_1) & @@ -1189,9 +1183,7 @@ static int wm8350_set_bias_level(struct snd_soc_component *component, wm8350_reg_write(wm8350, WM8350_POWER_MGMT_1, pm1); /* wait */ - schedule_timeout_interruptible(msecs_to_jiffies - (platform-> - vmid_discharge_msecs)); + schedule_msec_hrtimeout_interruptible(platform->vmid_discharge_msecs); wm8350_reg_write(wm8350, WM8350_ANTI_POP_CONTROL, (platform->vmid_s_curve << 8) | @@ -1207,8 +1199,7 @@ static int wm8350_set_bias_level(struct snd_soc_component *component, pm1 | WM8350_OUTPUT_DRAIN_EN); /* wait */ - schedule_timeout_interruptible(msecs_to_jiffies - (platform->drain_msecs)); + schedule_msec_hrtimeout_interruptible(platform->drain_msecs); pm1 &= ~WM8350_BIASEN; wm8350_reg_write(wm8350, WM8350_POWER_MGMT_1, pm1); diff --git a/sound/soc/codecs/wm8900.c b/sound/soc/codecs/wm8900.c index a9128cfa6..484377bd6 100644 --- a/sound/soc/codecs/wm8900.c +++ b/sound/soc/codecs/wm8900.c @@ -1057,7 +1057,7 @@ static int wm8900_set_bias_level(struct snd_soc_component *component, WM8900_REG_POWER1_STARTUP_BIAS_ENA | 0x1); /* Allow capacitors to charge */ - schedule_timeout_interruptible(msecs_to_jiffies(400)); + schedule_msec_hrtimeout_interruptible(400); /* Enable bias */ snd_soc_component_write(component, WM8900_REG_POWER1, @@ -1091,7 +1091,7 @@ static int wm8900_set_bias_level(struct snd_soc_component *component, /* Discharge caps */ snd_soc_component_write(component, WM8900_REG_POWER1, WM8900_REG_POWER1_STARTUP_BIAS_ENA); - schedule_timeout_interruptible(msecs_to_jiffies(500)); + schedule_msec_hrtimeout_interruptible(500); /* Remove clamp */ snd_soc_component_write(component, WM8900_REG_HPCTL1, 0); @@ -1105,7 +1105,7 @@ static int wm8900_set_bias_level(struct snd_soc_component *component, /* Need to let things settle before stopping the clock * to ensure that restart works, see "Stopping the * master clock" in the datasheet. */ - schedule_timeout_interruptible(msecs_to_jiffies(1)); + schedule_msec_hrtimeout_interruptible(1); snd_soc_component_write(component, WM8900_REG_POWER2, WM8900_REG_POWER2_SYSCLK_ENA); break; diff --git a/sound/soc/codecs/wm9713.c b/sound/soc/codecs/wm9713.c index b3bbecf07..7bda26410 100644 --- a/sound/soc/codecs/wm9713.c +++ b/sound/soc/codecs/wm9713.c @@ -199,7 +199,7 @@ static int wm9713_voice_shutdown(struct snd_soc_dapm_widget *w, /* Gracefully shut down the voice interface. */ snd_soc_component_update_bits(component, AC97_HANDSET_RATE, 0x0f00, 0x0200); - schedule_timeout_interruptible(msecs_to_jiffies(1)); + schedule_msec_hrtimeout_interruptible(1); snd_soc_component_update_bits(component, AC97_HANDSET_RATE, 0x0f00, 0x0f00); snd_soc_component_update_bits(component, AC97_EXTENDED_MID, 0x1000, 0x1000); @@ -864,7 +864,7 @@ static int wm9713_set_pll(struct snd_soc_component *component, wm9713->pll_in = freq_in; /* wait 10ms AC97 link frames for the link to stabilise */ - schedule_timeout_interruptible(msecs_to_jiffies(10)); + schedule_msec_hrtimeout_interruptible(10); return 0; } diff --git a/sound/soc/soc-dapm.c b/sound/soc/soc-dapm.c index 4ad126bd4..58c08cf6a 100644 --- a/sound/soc/soc-dapm.c +++ b/sound/soc/soc-dapm.c @@ -166,7 +166,7 @@ static void dapm_assert_locked(struct snd_soc_dapm_context *dapm) static void dapm_pop_wait(void) { if (pop_time) - schedule_timeout_uninterruptible(msecs_to_jiffies(pop_time)); + schedule_msec_hrtimeout_uninterruptible(pop_time); } __printf(2, 3) diff --git a/sound/usb/line6/pcm.c b/sound/usb/line6/pcm.c index 2932eaf15..6cb136235 100644 --- a/sound/usb/line6/pcm.c +++ b/sound/usb/line6/pcm.c @@ -127,7 +127,7 @@ static void line6_wait_clear_audio_urbs(struct snd_line6_pcm *line6pcm, if (!alive) break; set_current_state(TASK_UNINTERRUPTIBLE); - schedule_timeout(1); + schedule_min_hrtimeout(); } while (--timeout > 0); if (alive) dev_err(line6pcm->line6->ifcdev, -- 2.55.0 From 9c4b3d047c3ac89989f3ceec94124751f62d808f Mon Sep 17 00:00:00 2001 From: dvize Date: Tue, 18 Aug 2026 10:56:10 +0200 Subject: [PATCH 4/5] Fix PSI irq time accounting build for default arch config. --- kernel/sched/MuQSS.h | 1 + 1 file changed, 1 insertion(+) diff --git a/kernel/sched/MuQSS.h b/kernel/sched/MuQSS.h index 0c461e650..7c1a90e7b 100644 --- a/kernel/sched/MuQSS.h +++ b/kernel/sched/MuQSS.h @@ -309,6 +309,7 @@ struct rq { #ifdef CONFIG_IRQ_TIME_ACCOUNTING u64 prev_irq_time; + u64 psi_irq_time; #endif /* CONFIG_IRQ_TIME_ACCOUNTING */ #ifdef CONFIG_PARAVIRT u64 prev_steal_time; -- 2.55.0 From 3f5909cf57d7a78840a9a37de274b1b3c3076844 Mon Sep 17 00:00:00 2001 From: Piotr Gorski Date: Thu, 27 Aug 2026 13:45:31 +0200 Subject: [PATCH 5/5] sched/muqss: sync with CK repo Signed-off-by: Piotr Gorski --- kernel/Kconfig.MuQSS | 1 + tools/objtool/Makefile | 11 +++++++++++ 2 files changed, 12 insertions(+) diff --git a/kernel/Kconfig.MuQSS b/kernel/Kconfig.MuQSS index 3dad0e814..6b0ec452d 100644 --- a/kernel/Kconfig.MuQSS +++ b/kernel/Kconfig.MuQSS @@ -205,6 +205,7 @@ config MUQSS_IOTIME # init/Kconfig UCLAMP_TASK # SCHED_PROXY_EXEC # NUMA_BALANCING +# SCHED_CACHE # SCHED_MM_CID # CGROUP_CPUACCT # SCHED_AUTOGROUP diff --git a/tools/objtool/Makefile b/tools/objtool/Makefile index a4484fd22..3271b1de5 100644 --- a/tools/objtool/Makefile +++ b/tools/objtool/Makefile @@ -99,9 +99,19 @@ LIBOPCODES_LIBS := $(shell \ HAVE_DISASM_STYLED := $(shell echo '$(pound)include ' | \ $(HOSTCC) -E -xc - 2>/dev/null | grep -q disassembler_style && echo y) +# +# Linking -lopcodes can succeed when the shared library is present without +# development headers. That is common on Gentoo without sys-libs/binutils-libs +# (bfd.h lives under /usr/lib64/binutils///include, not +# /usr/include). Do not enable DISAS unless the headers actually compile. +# +HAVE_DISAS_HEADERS := $(shell printf "$(pound)include \n$(pound)include \nint main(void) { return 0; }\n" | \ + $(HOSTCC) $(HOSTCFLAGS) -DPACKAGE='"objtool"' -xc - -c -o /dev/null 2>/dev/null && echo y) + BUILD_DISAS := n ifneq ($(LIBOPCODES_LIBS),) +ifeq ($(HAVE_DISAS_HEADERS),y) BUILD_DISAS := y OBJTOOL_CFLAGS += -DDISAS -DPACKAGE='"objtool"' OBJTOOL_LDFLAGS += $(LIBOPCODES_LIBS) @@ -109,6 +119,7 @@ ifeq ($(HAVE_DISASM_STYLED),y) OBJTOOL_CFLAGS += -DDISASM_INIT_STYLED endif endif +endif export BUILD_DISAS -- 2.55.0