fix: at macOS memory warning, start jobs while memory fits

CHANGELOG.md

@@ -10,6 +10,15 @@ and the versions follow [semver](https://semver.org).
 
 ## [Unreleased]
 
+### Changed
+
+- macOS: the kernel's memory warning level no longer stops every new job.
+  At warning, jobs start while memory fits without any leeway, and not while
+  a first run still grows (at most two minutes). A dev stack kept the level
+  at warning with half of memory free, and the browser proof that needed the
+  stack waited behind it for up to an hour. Critical still stops every new
+  job. `warn_admit = false` brings back the old rule. (#22)
+
 ## [0.8.0] - 2026-10-09
 
 ### Added

README.md

@@ -94,8 +94,16 @@ below).
   it still promises to take keeps up with what it takes. The Queue shows in red
   how far jobs are above the needs they started with.
 - **Memory pressure stops everything new.** When the kernel reports memory
-  pressure (macOS: warning or critical; Linux: PSI above 20%), nothing new
-  starts until it eases, whatever the estimates say.
+  pressure (macOS: critical; Linux: PSI above 20%), nothing new starts until
+  it eases, whatever the estimates say. macOS keeps its warning level for a
+  long time, also with half of memory free, for example while a dev stack
+  runs. At warning, jobs still start, but only while memory fits without any
+  leeway, and not while a first run still grows: a first run books a guess,
+  so the next job waits until it has shown what it takes (at most two
+  minutes). A job with a history has its whole need booked from its first
+  second, so the next job may start at once. So the job that needs that
+  stack does not wait behind it. `warn_admit = false` makes warning stop
+  every new job again.
 - **Memory is read at its recent peak.** Admission uses the highest reading
   of the last 10 seconds, so a job never starts in a short dip.
 - **Short jobs skip the CPU reading, not the CPU budget.** A job that usually

src/commands.rs

@@ -21,6 +21,8 @@ pub fn limits(cfg: &Config) -> Limits {
         max_backfill: cfg.max_backfill as f64,
         cpu_min_duration: cfg.cpu_min_duration,
         pressure_max: cfg.pressure_max,
+        // Only macOS has a warning level; Linux reports a share of time.
+        warn_admit: cfg!(target_os = "macos") && cfg.warn_admit,
         noise_mem_pct: cfg.noise_mem,
         noise_cpu: cfg.noise_cpu,
         outside_admit: cfg.outside_admit,

src/config.rs

@@ -32,6 +32,7 @@ pub struct Layer {
     pub learn_stagger: Option<f64>,
     pub cpu_min_duration: Option<f64>,
     pub pressure_max: Option<f64>,
+    pub warn_admit: Option<bool>,
     pub priority: Option<i32>,
     pub auto_pause: Option<bool>,
     pub pause_at: Option<f64>,
@@ -125,6 +126,9 @@ pub struct Config {
     pub learn_stagger: f64,
     pub cpu_min_duration: f64,
     pub pressure_max: f64,
+    /// macOS: at the kernel's warning level, jobs still start while memory
+    /// fits and no first run still grows. Off: warning stops every new job.
+    pub warn_admit: bool,
     /// The priority of a job that no rule or flag gives one.
     pub priority: i32,
     /// Pause running jobs when memory fills up, and resume them when it frees
@@ -187,6 +191,7 @@ impl Default for Config {
             learn_stagger: 5.0,
             cpu_min_duration: 5.0,
             pressure_max: 20.0,
+            warn_admit: true,
             priority: 0,
             auto_pause: false,
             pause_at: 92.0,
@@ -607,6 +612,7 @@ impl Config {
         set!(learn_stagger);
         set!(cpu_min_duration);
         set!(pressure_max);
+        set!(warn_admit);
         set!(priority);
         set!(auto_pause);
         set!(pause_at);

src/queue.rs

@@ -532,6 +532,10 @@ pub struct Limits {
     pub cpu_min_duration: f64,
     /// Memory pressure (0-100) at which nothing new starts.
     pub pressure_max: f64,
+    /// At the kernel's warning level (under 100), jobs still start while
+    /// memory fits, and while no first run still grows. Off: warning stops
+    /// every new job, as critical always does.
+    pub warn_admit: bool,
     /// A job short of memory by at most this share of RAM (0-100) starts.
     pub noise_mem_pct: f64,
     /// A job short of CPU by at most this many cores starts.
@@ -583,10 +587,19 @@ pub enum Blocker {
         key: String,
         wait_s: f64,
     },
-    /// The kernel reports memory pressure: nothing new starts.
+    /// The kernel reports memory pressure: nothing new starts. At the
+    /// warning level with `warn_admit` on (`paced`), jobs still start, but
+    /// not while the first run `after` still grows: for at most `next_in`
+    /// seconds.
     Pressure {
         level: f64,
         limit: f64,
+        #[serde(skip_serializing_if = "std::ops::Not::not")]
+        paced: bool,
+        #[serde(skip_serializing_if = "Option::is_none")]
+        after: Option<String>,
+        #[serde(skip_serializing_if = "Option::is_none")]
+        next_in: Option<f64>,
     },
     /// A running job is paused for memory: it resumes before anything new starts.
     Paused {
@@ -701,8 +714,32 @@ pub fn decide(
     // taskguard runs nothing: then the pressure comes from other programs, and
     // one more job only makes it worse. This cannot deadlock the queue: jobs
     // start again as soon as the pressure eases, and --now still skips it.
+    //
+    // macOS keeps its warning level for a long time, also with half of memory
+    // free: a running dev stack holds it there, and the job that needs that
+    // stack waited behind it. So at warning (`warn_admit`), jobs still start,
+    // only while memory fits without any leeway. A job with a history has its
+    // whole need booked from its first second, so the next job may start at
+    // once. A first run books a guess, so nothing new starts until it has
+    // shown what it takes (`settling_for`). Critical stops every new job.
     if let Some(p) = m.mem_pressure.filter(|p| *p >= lim.pressure_max) {
-        return Decision::Wait { blockers: vec![Blocker::Pressure { level: p, limit: lim.pressure_max }] };
+        let paced = p < 100.0 && lim.warn_admit;
+        let growing = running.iter().filter_map(|e| e.settling_for(now).map(|w| (e, w))).max_by(|a, b| a.1.total_cmp(&b.1));
+        let pressure = Blocker::Pressure {
+            level: p,
+            limit: lim.pressure_max,
+            paced,
+            after: growing.filter(|_| paced).map(|(e, _)| e.key.clone()),
+            next_in: growing.filter(|_| paced).map(|(_, w)| w),
+        };
+        if !paced || growing.is_some() {
+            return Decision::Wait { blockers: vec![pressure] };
+        }
+        let room = Room::new(m, lim, running);
+        if let Some(mem) = room.blockers(m, lim, running, me, now, unknown_starts).into_iter().find(|b| matches!(b, Blocker::Memory { .. }))
+        {
+            return Decision::Wait { blockers: vec![mem, pressure] };
+        }
     }
 
     // A job paused for memory gets its room back before anything new starts.
@@ -1284,6 +1321,7 @@ mod tests {
         max_backfill: 0.0,
         cpu_min_duration: 5.0,
         pressure_max: 20.0,
+        warn_admit: false,
         noise_mem_pct: 0.0,
         noise_cpu: 0.0,
         outside_admit: false,
@@ -2228,6 +2266,51 @@ mod tests {
         assert!(blockers(decide(&m, &LIM, &[], std::slice::from_ref(&me), &me, 1000.0, &[])).is_empty(), "it eases: the job starts");
     }
 
+    #[test]
+    fn at_warning_jobs_start_while_memory_fits_and_no_first_run_grows() {
+        // A dev stack past its start-up holds macOS at warning with half of
+        // memory free. The browser proof that needs it starts at once.
+        let lim = Limits { warn_admit: true, ..LIM };
+        let stack = Entry { started_at: Some(100.0), steady_since: Some(220.0), live_mem_kb: 8 * GB, ..job(1, "stack", 1.0, 8) };
+        let proof = job(2, "proof", 2.2, 3);
+        let mut m = machine(2.0, 16);
+        m.mem_pressure = Some(50.0);
+        let d = decide(&m, &lim, std::slice::from_ref(&stack), std::slice::from_ref(&proof), &proof, 1000.0, &[]);
+        assert!(blockers(d).is_empty(), "the proof starts next to its stack");
+
+        // A job with a history has its whole need booked: the next starts at once.
+        let started = Entry { started_at: Some(999.0), ..proof.clone() };
+        let lint = job(3, "lint", 1.0, 1);
+        let d = decide(&m, &lim, &[stack.clone(), started], std::slice::from_ref(&lint), &lint, 1000.0, &[]);
+        assert!(blockers(d).is_empty(), "known jobs need no gap");
+
+        // A first run books a guess: nothing new starts while it still grows.
+        let first = Entry { known: false, started_at: Some(990.0), mem_grew_at: Some(995.0), ..job(4, "first", 1.0, 1) };
+        let both = [stack.clone(), first];
+        let d = decide(&m, &lim, &both, std::slice::from_ref(&lint), &lint, 1000.0, &[]);
+        assert!(
+            matches!(&d, Decision::Wait { blockers } if matches!(&blockers[0], Blocker::Pressure { after: Some(k), next_in: Some(t), .. } if k == "first" && *t == 15.0))
+        );
+        assert!(blockers(decide(&m, &lim, &both, std::slice::from_ref(&lint), &lint, 1015.0, &[])).is_empty(), "it has settled");
+
+        // At warning, memory has no leeway, also when only long-lived jobs run.
+        let big = job(5, "big", 1.0, 12);
+        let d = decide(&m, &lim, std::slice::from_ref(&stack), std::slice::from_ref(&big), &big, 1000.0, &[]);
+        assert_eq!(blockers(d), vec!["memory", "pressure"]);
+
+        // Critical stops every new job, and warn_admit off keeps the old rule.
+        m.mem_pressure = Some(100.0);
+        assert_eq!(
+            blockers(decide(&m, &lim, std::slice::from_ref(&stack), std::slice::from_ref(&proof), &proof, 1000.0, &[])),
+            vec!["pressure"]
+        );
+        m.mem_pressure = Some(50.0);
+        assert_eq!(
+            blockers(decide(&m, &LIM, std::slice::from_ref(&stack), std::slice::from_ref(&proof), &proof, 1000.0, &[])),
+            vec!["pressure"]
+        );
+    }
+
     #[test]
     fn admission_uses_the_peak_of_the_last_seconds() {
         let running = job(1, "r", 1.0, 1);

src/report.rs

@@ -46,6 +46,17 @@ pub fn blocker_text(b: &Blocker) -> String {
                 wait_s.ceil()
             )
         }
+        Blocker::Pressure { level, after: Some(key), next_in, .. } => format!(
+            "memory pressure: the kernel reports {} ({level:.0}); jobs still start while memory fits, the next once the first run {key} has shown what it takes (at most {:.0}s)",
+            pressure_word(*level),
+            next_in.unwrap_or(0.0).ceil()
+        ),
+        Blocker::Pressure { level, paced: true, .. } => {
+            format!(
+                "memory pressure: the kernel reports {} ({level:.0}); jobs still start, but only while memory fits",
+                pressure_word(*level)
+            )
+        }
         Blocker::Pressure { level, .. } => {
             format!("memory pressure: the kernel reports {} ({level:.0}); nothing new starts until it eases", pressure_word(*level))
         }
@@ -146,6 +157,10 @@ fn unblock(b: &Blocker, running: &[Entry], now: f64) -> (String, Option<f64>) {
         },
         Blocker::LearnStagger { wait_s } => (format!("starts in about {:.0}s", wait_s.ceil()), Some(*wait_s)),
         Blocker::Settling { key, wait_s } => (format!("starts when {key} stops growing (at most {:.0}s)", wait_s.ceil()), Some(*wait_s)),
+        Blocker::Pressure { after: Some(key), next_in, .. } => {
+            (format!("starts when {key} has shown what it takes, if memory fits"), *next_in)
+        }
+        Blocker::Pressure { paced: true, .. } => ("starts when memory fits".into(), None),
         Blocker::Pressure { .. } => ("starts when the memory pressure eases".into(), None),
         Blocker::Paused { key } => (format!("starts after {key} has resumed"), None),
         Blocker::Memory { short_kb, .. } => {
@@ -665,6 +680,7 @@ mod tests {
             max_backfill: 600.0,
             cpu_min_duration: 5.0,
             pressure_max: 20.0,
+            warn_admit: false,
             noise_mem_pct: 0.0,
             noise_cpu: 0.0,
             outside_admit: false,
@@ -806,6 +822,7 @@ mod tests {
             max_backfill: 0.0,
             cpu_min_duration: 5.0,
             pressure_max: 20.0,
+            warn_admit: false,
             noise_mem_pct: 0.0,
             noise_cpu: 0.0,
             outside_admit: false,
@@ -833,6 +850,7 @@ mod tests {
             max_backfill: 0.0,
             cpu_min_duration: 5.0,
             pressure_max: 20.0,
+            warn_admit: false,
             noise_mem_pct: 0.0,
             noise_cpu: 0.0,
             outside_admit: false,

src/sim.rs

@@ -306,6 +306,7 @@ mod tests {
         max_backfill: 0.0,
         cpu_min_duration: 5.0,
         pressure_max: 20.0,
+        warn_admit: false,
         noise_mem_pct: 0.0,
         noise_cpu: 0.0,
         outside_admit: false,

taskguard.example.toml

@@ -30,6 +30,7 @@
 cpu_max = 100         # percent of all cores
 mem_max = 85          # percent of RAM
 pressure_max = 20     # nothing new starts at this memory pressure (macOS: 50 warning, 100 critical)
+warn_admit = true     # macOS warning: jobs still start while memory fits and no first run grows; false = warning stops all
 
 # ------------------------------------------------------------ auto pause ----
 # Off by default. When memory in use reaches pause_at, taskguard pauses a