diff --git a/CHANGELOG.md b/CHANGELOG.md index c5ab072..6c5a841 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,15 @@ and the versions follow [semver](https://semver.org). ## [Unreleased] +### Changed + +- macOS: the kernel's memory warning level no longer stops every new job. + At warning, jobs start while memory fits without any leeway, and not while + a first run still grows (at most two minutes). A dev stack kept the level + at warning with half of memory free, and the browser proof that needed the + stack waited behind it for up to an hour. Critical still stops every new + job. `warn_admit = false` brings back the old rule. (#22) + ## [0.8.0] - 2026-10-09 ### Added diff --git a/README.md b/README.md index a373f4a..ea0fa56 100644 --- a/README.md +++ b/README.md @@ -94,8 +94,16 @@ below). it still promises to take keeps up with what it takes. The Queue shows in red how far jobs are above the needs they started with. - **Memory pressure stops everything new.** When the kernel reports memory - pressure (macOS: warning or critical; Linux: PSI above 20%), nothing new - starts until it eases, whatever the estimates say. + pressure (macOS: critical; Linux: PSI above 20%), nothing new starts until + it eases, whatever the estimates say. macOS keeps its warning level for a + long time, also with half of memory free, for example while a dev stack + runs. At warning, jobs still start, but only while memory fits without any + leeway, and not while a first run still grows: a first run books a guess, + so the next job waits until it has shown what it takes (at most two + minutes). A job with a history has its whole need booked from its first + second, so the next job may start at once. So the job that needs that + stack does not wait behind it. `warn_admit = false` makes warning stop + every new job again. - **Memory is read at its recent peak.** Admission uses the highest reading of the last 10 seconds, so a job never starts in a short dip. - **Short jobs skip the CPU reading, not the CPU budget.** A job that usually diff --git a/src/commands.rs b/src/commands.rs index 7d2183e..b8d1e93 100644 --- a/src/commands.rs +++ b/src/commands.rs @@ -21,6 +21,8 @@ pub fn limits(cfg: &Config) -> Limits { max_backfill: cfg.max_backfill as f64, cpu_min_duration: cfg.cpu_min_duration, pressure_max: cfg.pressure_max, + // Only macOS has a warning level; Linux reports a share of time. + warn_admit: cfg!(target_os = "macos") && cfg.warn_admit, noise_mem_pct: cfg.noise_mem, noise_cpu: cfg.noise_cpu, outside_admit: cfg.outside_admit, diff --git a/src/config.rs b/src/config.rs index 971e151..8932157 100644 --- a/src/config.rs +++ b/src/config.rs @@ -32,6 +32,7 @@ pub struct Layer { pub learn_stagger: Option, pub cpu_min_duration: Option, pub pressure_max: Option, + pub warn_admit: Option, pub priority: Option, pub auto_pause: Option, pub pause_at: Option, @@ -125,6 +126,9 @@ pub struct Config { pub learn_stagger: f64, pub cpu_min_duration: f64, pub pressure_max: f64, + /// macOS: at the kernel's warning level, jobs still start while memory + /// fits and no first run still grows. Off: warning stops every new job. + pub warn_admit: bool, /// The priority of a job that no rule or flag gives one. pub priority: i32, /// Pause running jobs when memory fills up, and resume them when it frees @@ -187,6 +191,7 @@ impl Default for Config { learn_stagger: 5.0, cpu_min_duration: 5.0, pressure_max: 20.0, + warn_admit: true, priority: 0, auto_pause: false, pause_at: 92.0, @@ -607,6 +612,7 @@ impl Config { set!(learn_stagger); set!(cpu_min_duration); set!(pressure_max); + set!(warn_admit); set!(priority); set!(auto_pause); set!(pause_at); diff --git a/src/queue.rs b/src/queue.rs index 531a6c5..7ee0cce 100644 --- a/src/queue.rs +++ b/src/queue.rs @@ -532,6 +532,10 @@ pub struct Limits { pub cpu_min_duration: f64, /// Memory pressure (0-100) at which nothing new starts. pub pressure_max: f64, + /// At the kernel's warning level (under 100), jobs still start while + /// memory fits, and while no first run still grows. Off: warning stops + /// every new job, as critical always does. + pub warn_admit: bool, /// A job short of memory by at most this share of RAM (0-100) starts. pub noise_mem_pct: f64, /// A job short of CPU by at most this many cores starts. @@ -583,10 +587,19 @@ pub enum Blocker { key: String, wait_s: f64, }, - /// The kernel reports memory pressure: nothing new starts. + /// The kernel reports memory pressure: nothing new starts. At the + /// warning level with `warn_admit` on (`paced`), jobs still start, but + /// not while the first run `after` still grows: for at most `next_in` + /// seconds. Pressure { level: f64, limit: f64, + #[serde(skip_serializing_if = "std::ops::Not::not")] + paced: bool, + #[serde(skip_serializing_if = "Option::is_none")] + after: Option, + #[serde(skip_serializing_if = "Option::is_none")] + next_in: Option, }, /// A running job is paused for memory: it resumes before anything new starts. Paused { @@ -701,8 +714,32 @@ pub fn decide( // taskguard runs nothing: then the pressure comes from other programs, and // one more job only makes it worse. This cannot deadlock the queue: jobs // start again as soon as the pressure eases, and --now still skips it. + // + // macOS keeps its warning level for a long time, also with half of memory + // free: a running dev stack holds it there, and the job that needs that + // stack waited behind it. So at warning (`warn_admit`), jobs still start, + // only while memory fits without any leeway. A job with a history has its + // whole need booked from its first second, so the next job may start at + // once. A first run books a guess, so nothing new starts until it has + // shown what it takes (`settling_for`). Critical stops every new job. if let Some(p) = m.mem_pressure.filter(|p| *p >= lim.pressure_max) { - return Decision::Wait { blockers: vec![Blocker::Pressure { level: p, limit: lim.pressure_max }] }; + let paced = p < 100.0 && lim.warn_admit; + let growing = running.iter().filter_map(|e| e.settling_for(now).map(|w| (e, w))).max_by(|a, b| a.1.total_cmp(&b.1)); + let pressure = Blocker::Pressure { + level: p, + limit: lim.pressure_max, + paced, + after: growing.filter(|_| paced).map(|(e, _)| e.key.clone()), + next_in: growing.filter(|_| paced).map(|(_, w)| w), + }; + if !paced || growing.is_some() { + return Decision::Wait { blockers: vec![pressure] }; + } + let room = Room::new(m, lim, running); + if let Some(mem) = room.blockers(m, lim, running, me, now, unknown_starts).into_iter().find(|b| matches!(b, Blocker::Memory { .. })) + { + return Decision::Wait { blockers: vec![mem, pressure] }; + } } // A job paused for memory gets its room back before anything new starts. @@ -1284,6 +1321,7 @@ mod tests { max_backfill: 0.0, cpu_min_duration: 5.0, pressure_max: 20.0, + warn_admit: false, noise_mem_pct: 0.0, noise_cpu: 0.0, outside_admit: false, @@ -2228,6 +2266,51 @@ mod tests { assert!(blockers(decide(&m, &LIM, &[], std::slice::from_ref(&me), &me, 1000.0, &[])).is_empty(), "it eases: the job starts"); } + #[test] + fn at_warning_jobs_start_while_memory_fits_and_no_first_run_grows() { + // A dev stack past its start-up holds macOS at warning with half of + // memory free. The browser proof that needs it starts at once. + let lim = Limits { warn_admit: true, ..LIM }; + let stack = Entry { started_at: Some(100.0), steady_since: Some(220.0), live_mem_kb: 8 * GB, ..job(1, "stack", 1.0, 8) }; + let proof = job(2, "proof", 2.2, 3); + let mut m = machine(2.0, 16); + m.mem_pressure = Some(50.0); + let d = decide(&m, &lim, std::slice::from_ref(&stack), std::slice::from_ref(&proof), &proof, 1000.0, &[]); + assert!(blockers(d).is_empty(), "the proof starts next to its stack"); + + // A job with a history has its whole need booked: the next starts at once. + let started = Entry { started_at: Some(999.0), ..proof.clone() }; + let lint = job(3, "lint", 1.0, 1); + let d = decide(&m, &lim, &[stack.clone(), started], std::slice::from_ref(&lint), &lint, 1000.0, &[]); + assert!(blockers(d).is_empty(), "known jobs need no gap"); + + // A first run books a guess: nothing new starts while it still grows. + let first = Entry { known: false, started_at: Some(990.0), mem_grew_at: Some(995.0), ..job(4, "first", 1.0, 1) }; + let both = [stack.clone(), first]; + let d = decide(&m, &lim, &both, std::slice::from_ref(&lint), &lint, 1000.0, &[]); + assert!( + matches!(&d, Decision::Wait { blockers } if matches!(&blockers[0], Blocker::Pressure { after: Some(k), next_in: Some(t), .. } if k == "first" && *t == 15.0)) + ); + assert!(blockers(decide(&m, &lim, &both, std::slice::from_ref(&lint), &lint, 1015.0, &[])).is_empty(), "it has settled"); + + // At warning, memory has no leeway, also when only long-lived jobs run. + let big = job(5, "big", 1.0, 12); + let d = decide(&m, &lim, std::slice::from_ref(&stack), std::slice::from_ref(&big), &big, 1000.0, &[]); + assert_eq!(blockers(d), vec!["memory", "pressure"]); + + // Critical stops every new job, and warn_admit off keeps the old rule. + m.mem_pressure = Some(100.0); + assert_eq!( + blockers(decide(&m, &lim, std::slice::from_ref(&stack), std::slice::from_ref(&proof), &proof, 1000.0, &[])), + vec!["pressure"] + ); + m.mem_pressure = Some(50.0); + assert_eq!( + blockers(decide(&m, &LIM, std::slice::from_ref(&stack), std::slice::from_ref(&proof), &proof, 1000.0, &[])), + vec!["pressure"] + ); + } + #[test] fn admission_uses_the_peak_of_the_last_seconds() { let running = job(1, "r", 1.0, 1); diff --git a/src/report.rs b/src/report.rs index 7aab550..33acaec 100644 --- a/src/report.rs +++ b/src/report.rs @@ -46,6 +46,17 @@ pub fn blocker_text(b: &Blocker) -> String { wait_s.ceil() ) } + Blocker::Pressure { level, after: Some(key), next_in, .. } => format!( + "memory pressure: the kernel reports {} ({level:.0}); jobs still start while memory fits, the next once the first run {key} has shown what it takes (at most {:.0}s)", + pressure_word(*level), + next_in.unwrap_or(0.0).ceil() + ), + Blocker::Pressure { level, paced: true, .. } => { + format!( + "memory pressure: the kernel reports {} ({level:.0}); jobs still start, but only while memory fits", + pressure_word(*level) + ) + } Blocker::Pressure { level, .. } => { format!("memory pressure: the kernel reports {} ({level:.0}); nothing new starts until it eases", pressure_word(*level)) } @@ -146,6 +157,10 @@ fn unblock(b: &Blocker, running: &[Entry], now: f64) -> (String, Option) { }, Blocker::LearnStagger { wait_s } => (format!("starts in about {:.0}s", wait_s.ceil()), Some(*wait_s)), Blocker::Settling { key, wait_s } => (format!("starts when {key} stops growing (at most {:.0}s)", wait_s.ceil()), Some(*wait_s)), + Blocker::Pressure { after: Some(key), next_in, .. } => { + (format!("starts when {key} has shown what it takes, if memory fits"), *next_in) + } + Blocker::Pressure { paced: true, .. } => ("starts when memory fits".into(), None), Blocker::Pressure { .. } => ("starts when the memory pressure eases".into(), None), Blocker::Paused { key } => (format!("starts after {key} has resumed"), None), Blocker::Memory { short_kb, .. } => { @@ -665,6 +680,7 @@ mod tests { max_backfill: 600.0, cpu_min_duration: 5.0, pressure_max: 20.0, + warn_admit: false, noise_mem_pct: 0.0, noise_cpu: 0.0, outside_admit: false, @@ -806,6 +822,7 @@ mod tests { max_backfill: 0.0, cpu_min_duration: 5.0, pressure_max: 20.0, + warn_admit: false, noise_mem_pct: 0.0, noise_cpu: 0.0, outside_admit: false, @@ -833,6 +850,7 @@ mod tests { max_backfill: 0.0, cpu_min_duration: 5.0, pressure_max: 20.0, + warn_admit: false, noise_mem_pct: 0.0, noise_cpu: 0.0, outside_admit: false, diff --git a/src/sim.rs b/src/sim.rs index e1e9edd..2fe34a4 100644 --- a/src/sim.rs +++ b/src/sim.rs @@ -306,6 +306,7 @@ mod tests { max_backfill: 0.0, cpu_min_duration: 5.0, pressure_max: 20.0, + warn_admit: false, noise_mem_pct: 0.0, noise_cpu: 0.0, outside_admit: false, diff --git a/taskguard.example.toml b/taskguard.example.toml index 0da13cc..708c597 100644 --- a/taskguard.example.toml +++ b/taskguard.example.toml @@ -30,6 +30,7 @@ cpu_max = 100 # percent of all cores mem_max = 85 # percent of RAM pressure_max = 20 # nothing new starts at this memory pressure (macOS: 50 warning, 100 critical) +warn_admit = true # macOS warning: jobs still start while memory fits and no first run grows; false = warning stops all # ------------------------------------------------------------ auto pause ---- # Off by default. When memory in use reaches pause_at, taskguard pauses a