From bb887bd0f85dd96d88c374bfb0e5f83a103b50af Mon Sep 17 00:00:00 2001 From: Vincent De Snerck <10668+SpoBo@users.noreply.github.com> Date: Fri, 16 Oct 2026 11:37:16 +0200 Subject: [PATCH 01/22] docs(queue): a steady job books what it holds, not what past runs grew to The doc comment on Entry::follow and LongLived::steady_mem_kb, and a test comment, still said a steady job keeps what its past runs grew to after their start-up. Since 0.7.1 it books what it holds now plus a quarter, which the test already checks. --- README.md | 6 +++--- src/queue.rs | 17 +++++++++-------- 2 files changed, 12 insertions(+), 11 deletions(-) diff --git a/README.md b/README.md index 9cf82d6..734d786 100644 --- a/README.md +++ b/README.md @@ -202,9 +202,9 @@ below). a job starts when there is room for its start-up peak, and holds that peak for its start-up (`startup`, 300 seconds by default). Then its needs start again from what it uses: its CPU need is what it wanted over the last 10 - seconds, up or down, and its memory need what it holds plus 25%, or what - its past runs grew to after their start-up when that is more, and it follows - its peak from there. A DALP dev stack wants almost every core while it + seconds, up or down, and its memory need what it holds plus 25%, up or + down with each reading, so an idle stack releases its booking. A DALP dev + stack wants almost every core while it starts and seeds, then idles at a few percent; without this it kept every core for hours. Its history keeps both: the start-up peak, which the next run is admitted against, and what it took after. Past its start-up it diff --git a/src/queue.rs b/src/queue.rs index 1e35be3..e6cebdc 100644 --- a/src/queue.rs +++ b/src/queue.rs @@ -199,8 +199,8 @@ impl Entry { pub struct LongLived { /// Seconds after its start that it holds its start-up peak. pub startup: f64, - /// The highest memory its past runs took after their start-up: what it - /// still grows to once it settles. + /// The highest memory its past runs took after their start-up. `follow` + /// does not book it: once steady, a job books what it holds now. pub steady_mem_kb: Option, } @@ -219,11 +219,11 @@ impl Entry { /// A long-lived job follows its peak through its start-up only. After that its needs start again /// from what it uses: a dev stack that took every core while it started /// and seeded, and idles at a few percent, must not keep every core for - /// hours. Its memory need is what it holds plus a quarter, or what its - /// past runs grew to after their start-up when that is more, and follows - /// its peak from there: running out of memory kills processes. Its CPU - /// need is measured use plus a margin, up or down, because runnable wait - /// can reflect priority and contention rather than useful work. + /// hours. Its memory need is what it holds now plus a quarter, up or + /// down with each reading, so an idle stack releases its booking; what + /// its past runs grew to after their start-up is not booked. Its CPU need + /// is measured use plus a margin, up or down, because runnable wait can + /// reflect priority and contention rather than useful work. pub fn follow(&mut self, mem_kb: u64, used_recent: f64, ncpu: f64, now: f64, long: Option) { if let Some(l) = long && self.steady_since.is_none() @@ -2030,7 +2030,8 @@ mod tests { s.follow(2 * GB, 0.3, 12.0, 1460.0, Some(STACK)); assert_eq!(s.need_mem_kb, 2 * GB + (2 * GB) / 4); - // What its past runs grew to after their start-up stays reserved. + // What its past runs grew to after their start-up is not booked: + // only what it holds now, plus a quarter. let mut again = Entry { started_at: Some(1000.0), ..stack() }; again.follow(3 * GB, 0.2, 12.0, 1300.0, Some(LongLived { steady_mem_kb: Some(5 * GB), ..STACK })); assert_eq!(again.need_mem_kb, 3 * GB + (3 * GB) / 4); From 1dbcd9b114438bd96afff8791f51e934b0d901bb Mon Sep 17 00:00:00 2001 From: Vincent De Snerck <10668+SpoBo@users.noreply.github.com> Date: Fri, 16 Oct 2026 11:38:51 +0200 Subject: [PATCH 02/22] fix: outside_admit admits again when mem_max is at or above pause_at outside_admit lets a job start that only programs outside taskguard keep out, while memory stays under pause_at (92%). A job it would let start is over mem_max already, so with mem_max = 95 (the MacBook) it never let anything start. The ceiling is now halfway from mem_max to all of RAM when mem_max is at or above pause_at. --- CHANGELOG.md | 7 +++++++ README.md | 9 ++++++--- src/commands.rs | 45 +++++++++++++++++++++++++++++++++++++++++- src/config.rs | 2 +- taskguard.example.toml | 3 ++- 5 files changed, 60 insertions(+), 6 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c2971f4..9f8751e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -22,6 +22,13 @@ and the versions follow [semver](https://semver.org). - Each run now stores how its need was made (`runs.need_why`), so `dump` can explain an old run too, not only the next one. +### Fixed + +- `outside_admit` works again when `mem_max` is at or above `pause_at` + (92% by default), as with `mem_max = 95`. It capped memory at `pause_at`, + under the limit, so it could never let a job start; the cap is now + halfway from `mem_max` to all of RAM in that case. + ## [0.10.0] - 2026-10-16 ### Fixed diff --git a/README.md b/README.md index 734d786..c1406bd 100644 --- a/README.md +++ b/README.md @@ -127,8 +127,10 @@ below). starts anyway: readings move by that much from one second to the next. So does a job that only programs outside taskguard keep out, while taskguard's own jobs fit under the limits (`outside_admit`). Memory stays a - hard rule at `pause_at` (92%). Each such start prints a warning with what - the job lacks and the rule that let it start. + hard rule at `pause_at` (92%), or, when `mem_max` is at or above `pause_at`, + halfway from `mem_max` to all of RAM (97.5% for `mem_max = 95`). Each such + start prints a warning with what the job lacks and the rule that let it + start. - **Nothing running means the job starts,** whatever the CPU and memory readings say, so the queue cannot deadlock on a wrong reading. Memory pressure is the one exception: then the load comes from other programs, and @@ -594,7 +596,8 @@ repo. Put a repo's own scripts in its `.taskguard.toml`: `outlier_weights = [0.5]` to learn sooner, or `outlier_ratio = 0` to learn every peak at once. - A job that starts early on `noise_mem` or `outside_admit` can push memory - past `mem_max`, up to `pause_at`. Set `outside_admit = false` and + past `mem_max`, up to `pause_at` (or halfway from `mem_max` to all of RAM + when `mem_max` is at or above `pause_at`). Set `outside_admit = false` and `noise_mem = 0` to keep `mem_max` a hard rule. ## Moving from tsc-queue diff --git a/src/commands.rs b/src/commands.rs index 58186b5..4a24f55 100644 --- a/src/commands.rs +++ b/src/commands.rs @@ -26,11 +26,20 @@ pub fn limits(cfg: &Config) -> Limits { noise_mem_pct: cfg.noise_mem, noise_cpu: cfg.noise_cpu, outside_admit: cfg.outside_admit, - outside_mem_max_pct: cfg.pause_at, + outside_mem_max_pct: outside_ceiling(cfg.mem_max, cfg.pause_at), partial_fit: cfg.partial_fit.clamp(0.0, 1.0), } } +/// The share of RAM up to which `outside_admit` lets a job start: `pause_at`, +/// where auto_pause would start pausing jobs. A job it lets start is over +/// `mem_max` already, so with `mem_max` at or above `pause_at` the rule could +/// never admit anything; then the ceiling is halfway from `mem_max` to all of +/// RAM. +pub fn outside_ceiling(mem_max: f64, pause_at: f64) -> f64 { + if pause_at > mem_max { pause_at } else { (mem_max + 100.0) / 2.0 } +} + /// Everything the status screen and the dashboard's Queue view show. pub fn snapshot(dir: &Path, cfg: &Config, db: Option<&Db>) -> Result { let mut s = queue_snapshot(dir, cfg, db)?; @@ -734,4 +743,38 @@ mod tests { assert!(parse_prune(&args("--bogus")).is_err()); assert!(parse_prune(&args("--ratio")).is_err()); } + + #[test] + fn outside_admit_still_works_with_mem_max_above_pause_at() { + use crate::machine::MachineSample; + use crate::queue::{Decision, decide}; + const GB: u64 = 1024 * 1024; + // The MacBook: mem_max 95, pause_at 92 by default. + let cfg = Config { mem_max: 95.0, noise_mem: 0.0, ..Config::default() }; + let lim = limits(&cfg); + assert_eq!(lim.outside_mem_max_pct, 97.5); + assert_eq!(limits(&Config::default()).outside_mem_max_pct, 92.0, "under pause_at the ceiling stays pause_at"); + // 29.5 GB held, 1 GB of it taskguard's; a 1 GB job would reach 95.3% + // of 32 GB, over mem_max only because of other programs. It starts, + // where a ceiling of 92% under a limit of 95% let nothing through. + let mut running = + Entry { key: "api".into(), need_mem_kb: GB, live_mem_kb: GB, known: true, started_at: Some(1.0), ..Default::default() }; + running.need_cpu = 0.5; + let me = + Entry { ticket: 2, key: "worker".into(), need_cpu: 0.5, need_mem_kb: GB, known: true, queued_at: 1.0, ..Default::default() }; + let m = MachineSample::fixed(1.0, 12, 29 * GB + GB / 2, 32 * GB); + let d = decide(&m, &lim, std::slice::from_ref(&running), std::slice::from_ref(&me), &me, 2.0, &[]); + assert!(matches!(&d, Decision::Admit { early: Some(why), .. } if why.contains("outside_admit")), "{d:?}"); + let old = crate::queue::Limits { outside_mem_max_pct: 92.0, ..lim }; + assert!(matches!( + decide(&m, &old, std::slice::from_ref(&running), std::slice::from_ref(&me), &me, 2.0, &[]), + Decision::Wait { .. } + )); + // Above the new ceiling memory stays a hard rule. + let m = MachineSample::fixed(1.0, 12, 30 * GB + GB / 2, 32 * GB); + assert!(matches!( + decide(&m, &lim, std::slice::from_ref(&running), std::slice::from_ref(&me), &me, 2.0, &[]), + Decision::Wait { .. } + )); + } } diff --git a/src/config.rs b/src/config.rs index bbbb4d8..42fb266 100644 --- a/src/config.rs +++ b/src/config.rs @@ -154,7 +154,7 @@ pub struct Config { /// A job short of CPU by at most this many cores starts anyway. pub noise_cpu: f64, /// A job that only programs outside taskguard keep out starts anyway, - /// while memory stays under `pause_at`. + /// while memory stays under `pause_at` (see `commands::outside_ceiling`). pub outside_admit: bool, /// A job that cannot fit now starts on this share of its CPU need, when /// the machine is near its recent low. 0 turns it off. diff --git a/taskguard.example.toml b/taskguard.example.toml index 669aa7f..aad5cdd 100644 --- a/taskguard.example.toml +++ b/taskguard.example.toml @@ -85,7 +85,8 @@ outlier_weights = [0, 0.5] noise_mem = 2 # percent of RAM noise_cpu = 0.5 # cores # A job that only programs outside taskguard keep out starts anyway, with a -# warning, while memory stays under pause_at. +# warning, while memory stays under pause_at (with mem_max at or above +# pause_at: halfway from mem_max to all of RAM). outside_admit = true # A job that cannot fit now (its CPU need is above the whole limit, or it has # waited past max_bypass) starts on this share of its CPU need, once the From 862f5e1f4eda37b180a1f8c3b859285cd179b8e6 Mon Sep 17 00:00:00 2001 From: Vincent De Snerck <10668+SpoBo@users.noreply.github.com> Date: Fri, 16 Oct 2026 11:43:18 +0200 Subject: [PATCH 03/22] test: a model of dev stacks, their runs, and a turbo run's lints src/stall.rs replays the MacBook morning second by second through the real decide() and Entry::follow(): two agents bring up a dev stack each and queue their browser run once it is up, give up on the run after 10 minutes and take the stack down, while an older turbo run keeps queueing lints. It also replays Omarchy's head of line: a second stack that does not fit, queued in front of 25 jobs. With today's rules the runs are given up (5 to 8 of about 12 in two hours), and the second Omarchy stack starts at once to 114% of RAM. --- src/main.rs | 2 + src/stall.rs | 594 +++++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 596 insertions(+) create mode 100644 src/stall.rs diff --git a/src/main.rs b/src/main.rs index f26ddc9..5c70a9f 100644 --- a/src/main.rs +++ b/src/main.rs @@ -21,6 +21,8 @@ mod report; mod runner; #[cfg(test)] mod sim; +#[cfg(test)] +mod stall; mod sys; mod top; diff --git a/src/stall.rs b/src/stall.rs new file mode 100644 index 0000000..baf0ba6 --- /dev/null +++ b/src/stall.rs @@ -0,0 +1,594 @@ +//! A model of dev stacks and the runs they exist for, run as tests. +//! +//! On a MacBook (10 cores, 32 GB, mem_max 95) two agents each brought up a +//! dev stack and, once it was up, queued the browser run that needs it. +//! Meanwhile an older turbo run kept queueing lints. The second stack was +//! admitted while the first one's run waited, the runs never fit next to +//! both stacks, and each agent gave up on its run after its 10-minute command +//! timeout and took its stack down. This model replays that morning second +//! by second through the real `decide()` and `Entry::follow()`, so a change +//! to the rules shows what it does to it. + +use crate::machine::{MachineSample, PEAK_WINDOW}; +use crate::queue::{Decision, Entry, Limits, LongLived, decide}; + +const GB: u64 = 1024 * 1024; +const TICK: f64 = 1.0; + +/// The machine and the load outside taskguard. +#[derive(Clone, Copy)] +pub struct Host { + pub total_kb: u64, + pub ncpu: usize, + /// Memory programs outside taskguard hold, on average; it swings by + /// `outside_swing_kb` around that. + pub outside_kb: u64, + pub outside_swing_kb: u64, + /// macOS: the kernel's warning level above this share of RAM in use, and + /// critical above `critical_pct`. Linux reports no levels: None. + pub warning_pct: Option, + pub critical_pct: f64, +} + +impl Host { + fn outside(&self, t: f64) -> u64 { + let swing = (t / 37.0).sin() * 0.7 + (t / 11.0).sin() * 0.3; + (self.outside_kb as f64 + swing * self.outside_swing_kb as f64) as u64 + } + + fn pressure(&self, pct: f64) -> f64 { + match self.warning_pct { + Some(_) if pct > self.critical_pct => 100.0, + Some(w) if pct > w => 50.0, + _ => 0.0, + } + } +} + +/// The MacBook of the morning: 10 cores, 32 GB, the kernel at warning above +/// 78% held (where that morning's readings flipped), critical above 97%. +pub fn macbook(outside_gb: f64) -> Host { + Host { + total_kb: 32 * GB, + ncpu: 10, + outside_kb: (outside_gb * GB as f64) as u64, + outside_swing_kb: 6 * GB / 10, + warning_pct: Some(78.0), + critical_pct: 97.0, + } +} + +/// The MacBook's limits: mem_max 95 (its user config), DALP's +/// max_backfill 1800, and the built-in rest. +pub fn macbook_limits() -> Limits { + Limits { + cpu_max_pct: 100.0, + mem_max_pct: 95.0, + learn_stagger: 5.0, + max_bypass: 120.0, + max_backfill: 1800.0, + cpu_min_duration: 5.0, + pressure_max: 20.0, + warn_admit: true, + noise_mem_pct: 2.0, + noise_cpu: 0.5, + outside_admit: true, + outside_mem_max_pct: crate::commands::outside_ceiling(95.0, 92.0), + partial_fit: 0.5, + } +} + +/// What a job is, for the model. +#[derive(Clone, Copy, PartialEq, Debug)] +enum Kind { + /// The dev stack of agent `n`. + Stack(usize), + /// The browser run of agent `n`, against its stack. + Run(usize), + Lint, + /// `Scenario::jobs[n]`. + Job(usize), +} + +/// A job that queues at `at` (or, before 0, already runs since then): it +/// holds `peak_kb` after a few seconds and ends after `secs`. A long-lived +/// one runs to the end. +#[derive(Clone)] +pub struct Job { + pub at: f64, + pub key: String, + pub need_kb: u64, + pub peak_kb: u64, + pub cpu: f64, + pub secs: f64, + pub long_lived: bool, + pub priority: i32, +} + +/// One agent: it brings up a stack, queues its browser run once the stack is +/// up, gives up on the run after `give_up_s` in the queue and takes the stack +/// down, or takes it down `down_after_s` after the run ended. Then it starts +/// over after `pause_s`. +#[derive(Clone, Copy)] +pub struct Agent { + pub first_at: f64, + /// Memory the stack holds once it has started, and how long that takes. + pub stack_idle_kb: u64, + pub stack_ramp_s: f64, + /// How much the stack grows while its run drives it. + pub stack_grows: f64, + pub stack_need_kb: u64, + pub run_need_kb: u64, + pub run_peak_kb: u64, + pub run_cpu: f64, + pub run_s: f64, + pub give_up_s: f64, + pub down_after_s: f64, + pub pause_s: f64, +} + +pub fn agent(first_at: f64, idle_gb: f64) -> Agent { + Agent { + first_at, + stack_idle_kb: (idle_gb * GB as f64) as u64, + stack_ramp_s: 360.0, + stack_grows: 1.13, + stack_need_kb: 94 * GB / 10, + run_need_kb: 43 * GB / 10, + run_peak_kb: 3 * GB, + run_cpu: 3.0, + run_s: 240.0, + give_up_s: 600.0, + down_after_s: 20.0, + pause_s: 30.0, + } +} + +/// Lints of a turbo run that began an hour earlier: it keeps `concurrency` +/// lints queued or running, each one `need_kb` (in turn), peaking at +/// `peak_kb` for `secs`. +#[derive(Clone)] +pub struct Lints { + pub concurrency: usize, + pub need_kb: Vec, + pub peak_kb: u64, + pub cpu: f64, + pub secs: f64, +} + +pub fn macbook_lints() -> Lints { + Lints { + concurrency: 3, + need_kb: [46, 58, 66, 53, 73].iter().map(|g| g * GB / 10).collect(), + peak_kb: 46 * GB / 10, + cpu: 3.5, + secs: 10.0, + } +} + +pub struct Scenario { + pub host: Host, + pub lim: Limits, + pub agents: Vec, + pub lints: Option, + pub jobs: Vec, + pub secs: f64, +} + +/// What came of it. +#[derive(Debug, Default, Clone)] +pub struct Outcome { + pub runs_done: usize, + pub runs_given_up: usize, + /// The longest a run waited in the queue, started or not. + pub worst_run_wait: f64, + /// Seconds stacks stood up while their run was not running. + pub stack_idle_s: f64, + pub max_stacks: usize, + pub worst_mem_pct: f64, + /// Seconds memory in use was above the limit. + pub over_limit_s: f64, + pub lints_done: usize, + pub worst_stack_wait: f64, + /// When each of `Scenario::jobs` started, if it did. + pub job_started: Vec>, +} + +struct Live { + e: Entry, + kind: Kind, + started: f64, +} + +#[derive(Clone, Copy, PartialEq, Debug)] +enum Phase { + Idle { until: f64 }, + StackQueued, + StackUp, + RunQueued { since: f64 }, + RunRunning { until: f64 }, + Down { at: f64 }, +} + +pub fn run(s: &Scenario) -> Outcome { + let mut out = Outcome { job_started: vec![None; s.jobs.len()], ..Default::default() }; + let mut waiting: Vec<(Entry, Kind)> = Vec::new(); + let mut running: Vec = Vec::new(); + let mut phase: Vec = s.agents.iter().map(|a| Phase::Idle { until: a.first_at }).collect(); + let mut ticket = 0u64; + let mut recent: Vec<(f64, u64, u64)> = Vec::new(); + let mut lint_n = 0usize; + let long = Some(LongLived { startup: 120.0, steady_mem_kb: None }); + let checkout = |n: usize| format!("/w/dalp/{n}"); + let job_entry = |n: usize, j: &Job, ticket: u64| Entry { + ticket, + pid: 8000 + n as i32, + key: j.key.clone(), + checkout: "/w/dalp/main".into(), + queued_at: j.at, + need_cpu: j.cpu, + need_mem_kb: j.need_kb, + known: true, + est_dur_s: (!j.long_lived).then_some(j.secs), + priority: j.priority, + ..Default::default() + }; + for (n, j) in s.jobs.iter().enumerate().filter(|(_, j)| j.at < 0.0) { + ticket += 1; + let e = Entry { started_at: Some(j.at), ..job_entry(n, j, ticket) }; + running.push(Live { e, kind: Kind::Job(n), started: j.at }); + out.job_started[n] = Some(j.at); + } + let mut t = 0.0; + while t < s.secs { + for (n, j) in s.jobs.iter().enumerate().filter(|(_, j)| j.at >= t && j.at < t + TICK) { + ticket += 1; + waiting.push((job_entry(n, j, ticket), Kind::Job(n))); + } + // Jobs that end by themselves. + running.retain(|r| { + let done = match r.kind { + Kind::Lint => t - r.started >= s.lints.as_ref().map_or(0.0, |l| l.secs), + Kind::Job(n) => !s.jobs[n].long_lived && t - r.started >= s.jobs[n].secs, + _ => false, + }; + if done { + out.lints_done += 1; + } + !done + }); + // The agents. + for (n, a) in s.agents.iter().enumerate() { + let stack_pid = 1000 + n as i32; + match phase[n] { + Phase::Idle { until } if t >= until => { + ticket += 1; + let e = Entry { + ticket, + pid: stack_pid, + key: "dev-env:stack".into(), + pool: Some("stack".into()), + checkout: checkout(n), + queued_at: t, + need_cpu: 3.0, + need_mem_kb: a.stack_need_kb, + known: true, + priority: 10, + ..Default::default() + }; + waiting.push((e, Kind::Stack(n))); + phase[n] = Phase::StackQueued; + } + Phase::StackQueued => { + if let Some(r) = running.iter().find(|r| r.kind == Kind::Stack(n)) + && t - r.started >= a.stack_ramp_s + { + phase[n] = Phase::StackUp; + } + } + Phase::StackUp => { + ticket += 1; + let e = Entry { + ticket, + pid: 2000 + n as i32, + key: format!("e2e/agentic:bun_src/pr.ts_--pr_{}", 17397 + n), + pool: Some("browser".into()), + checkout: checkout(n), + queued_at: t, + need_cpu: a.run_cpu, + need_mem_kb: a.run_need_kb, + known: true, + raised_by_min: true, + est_dur_s: Some(a.run_s), + ..Default::default() + }; + waiting.push((e, Kind::Run(n))); + phase[n] = Phase::RunQueued { since: t }; + } + Phase::RunQueued { since } => { + if let Some(r) = running.iter().find(|r| r.kind == Kind::Run(n)) { + out.worst_run_wait = out.worst_run_wait.max(r.started - since); + phase[n] = Phase::RunRunning { until: r.started + a.run_s }; + } else if t - since >= a.give_up_s { + // The agent's command timeout: the run leaves the queue + // and the agent takes its stack down. + out.worst_run_wait = out.worst_run_wait.max(t - since); + out.runs_given_up += 1; + waiting.retain(|(_, k)| *k != Kind::Run(n)); + phase[n] = Phase::Down { at: t }; + } + } + Phase::RunRunning { until } if t >= until => { + running.retain(|r| r.kind != Kind::Run(n)); + out.runs_done += 1; + phase[n] = Phase::Down { at: t + a.down_after_s }; + } + Phase::Down { at } if t >= at => { + running.retain(|r| r.kind != Kind::Stack(n)); + phase[n] = Phase::Idle { until: t + a.pause_s }; + } + _ => {} + } + } + // The turbo run keeps its lints queued. + if let Some(l) = &s.lints { + let open = waiting.iter().filter(|(_, k)| *k == Kind::Lint).count() + running.iter().filter(|r| r.kind == Kind::Lint).count(); + for _ in open..l.concurrency { + ticket += 1; + lint_n += 1; + let e = Entry { + ticket, + pid: 5000 + lint_n as i32, + key: format!("pkg{}:oxlint", lint_n % 40), + checkout: "/w/dalp/main".into(), + queued_at: t, + need_cpu: l.cpu, + need_mem_kb: l.need_kb[lint_n % l.need_kb.len()], + known: true, + est_dur_s: Some(l.secs), + pipeline: Some("turbo:1".into()), + pipeline_since: Some(-3600.0), + ..Default::default() + }; + waiting.push((e, Kind::Lint)); + } + } + // What each running job holds and uses now. + for r in running.iter_mut() { + let age = t - r.started; + let (mem, cpu) = match r.kind { + Kind::Stack(n) => { + let a = &s.agents[n]; + let busy = running_kind_any(&phase, n); + let grow = if busy { a.stack_grows } else { 1.0 }; + let ramp = (age / a.stack_ramp_s).min(1.0); + ((a.stack_idle_kb as f64 * ramp * grow) as u64, if age < a.stack_ramp_s { 3.0 } else { 0.1 }) + } + Kind::Run(n) => { + let a = &s.agents[n]; + ((a.run_peak_kb as f64 * (age / 20.0).min(1.0)) as u64, a.run_cpu) + } + Kind::Lint => { + let l = s.lints.as_ref().expect("lints"); + ((l.peak_kb as f64 * (age / 3.0).min(1.0)) as u64, l.cpu) + } + Kind::Job(n) => { + let j = &s.jobs[n]; + let ramp = if j.long_lived { 300.0 } else { 3.0 }; + ((j.peak_kb as f64 * (age / ramp).min(1.0)) as u64, if j.long_lived && age > ramp { 0.1 } else { j.cpu }) + } + }; + r.e.live_mem_kb = mem; + r.e.live_cpu = cpu; + let lived = match r.kind { + Kind::Stack(_) => long, + Kind::Job(n) if s.jobs[n].long_lived => long, + _ => None, + }; + r.e.follow(mem, cpu, s.host.ncpu as f64, t, lived); + } + let ours: u64 = running.iter().map(|r| r.e.live_mem_kb).sum(); + let used = s.host.outside(t) + ours; + let pct = used as f64 * 100.0 / s.host.total_kb as f64; + out.worst_mem_pct = out.worst_mem_pct.max(pct); + if pct > s.lim.mem_max_pct { + out.over_limit_s += TICK; + } + let stacks = + running.iter().filter(|r| matches!(r.kind, Kind::Stack(_)) || matches!(r.kind, Kind::Job(n) if s.jobs[n].long_lived)).count(); + out.max_stacks = out.max_stacks.max(stacks); + for (n, p) in phase.iter().enumerate() { + if matches!(p, Phase::RunQueued { .. }) && running.iter().any(|r| r.kind == Kind::Stack(n)) { + out.stack_idle_s += TICK; + } + } + recent.retain(|(ts, _, _)| t - ts < PEAK_WINDOW); + recent.push((t, used, ours)); + let busy = (1.0 + running.iter().map(|r| r.e.live_cpu).sum::()).min(s.host.ncpu as f64); + let m = MachineSample { + ts: t, + cpu_busy: busy, + ncpu: s.host.ncpu, + mem_used_kb: used, + mem_total_kb: s.host.total_kb, + mem_pressure: Some(s.host.pressure(pct)), + recent_mem: recent.clone(), + ..Default::default() + }; + // Every waiting job decides once per tick, as the runners do. + let mut i = 0; + while i < waiting.len() { + let run_now: Vec = running.iter().map(|r| r.e.clone()).collect(); + let all: Vec = waiting.iter().map(|(e, _)| e.clone()).collect(); + let (me, kind) = waiting[i].clone(); + if matches!(decide(&m, &s.lim, &run_now, &all, &me, t, &[]), Decision::Admit { .. }) { + for (w, _) in waiting.iter_mut().filter(|(w, _)| w.ahead_of(&me) && w.bypassed_since.is_none()) { + w.bypassed_since = Some(t); + } + if let Kind::Stack(_) = kind { + out.worst_stack_wait = out.worst_stack_wait.max(t - me.queued_at); + } + if let Kind::Job(n) = kind { + out.job_started[n] = Some(t); + } + let e = Entry { started_at: Some(t), start_need_cpu: Some(me.need_cpu), start_need_mem_kb: Some(me.need_mem_kb), ..me }; + running.push(Live { e, kind, started: t }); + waiting.remove(i); + } else { + i += 1; + } + } + t += TICK; + } + out +} + +/// Does agent `n`'s run run now? +fn running_kind_any(phase: &[Phase], n: usize) -> bool { + matches!(phase[n], Phase::RunRunning { .. }) +} + +/// The MacBook morning with programs outside taskguard at `outside_gb`. +pub fn morning(outside_gb: f64) -> Scenario { + Scenario { + host: macbook(outside_gb), + lim: macbook_limits(), + agents: vec![agent(0.0, 7.0), agent(380.0, 4.6)], + lints: Some(macbook_lints()), + jobs: Vec::new(), + secs: 7200.0, + } +} + +/// Omarchy (24 cores, 31.3 GB, Linux, mem_max 95, DALP's max_backfill 1800): +/// a stack has run for an hour, a second stack of 9.1 GB queues that does +/// not fit next to it, and 25 jobs queue behind it over five minutes: lints +/// of about 5 GB, which fit only one at a time, and small tests, which fit. +pub fn head_of_line() -> Scenario { + let gb = |g: f64| (g * GB as f64) as u64; + let mut jobs = vec![ + Job { + at: -3600.0, + key: "dev-env:stack".into(), + need_kb: gb(10.2), + peak_kb: gb(8.2), + cpu: 3.0, + secs: 0.0, + long_lived: true, + priority: 10, + }, + Job { + at: 0.0, + key: "dev-env:stack".into(), + need_kb: gb(9.1), + peak_kb: gb(7.0), + cpu: 3.0, + secs: 0.0, + long_lived: true, + priority: 10, + }, + ]; + for i in 0..25 { + let at = 2.0 + 12.0 * i as f64; + jobs.push(if i % 5 < 3 { + let need = [4.7, 5.0, 5.5, 6.1, 4.9][i % 5]; + Job { + at, + key: format!("pkg{i}:oxlint"), + need_kb: gb(need), + peak_kb: gb(4.5), + cpu: 3.0, + secs: 60.0, + long_lived: false, + priority: 0, + } + } else { + let need = [0.2, 0.9, 0.5][i % 3]; + Job { + at, + key: format!("pkg{i}:bun_test"), + need_kb: gb(need), + peak_kb: gb(need), + cpu: 1.5, + secs: 30.0, + long_lived: false, + priority: 0, + } + }); + } + let host = + Host { total_kb: gb(31.3), ncpu: 24, outside_kb: gb(15.8), outside_swing_kb: gb(0.3), warning_pct: None, critical_pct: 100.0 }; + let lim = Limits { warn_admit: false, ..macbook_limits() }; + Scenario { host, lim, agents: Vec::new(), lints: None, jobs, secs: 1800.0 } +} + +/// The longest a small job (under 1 GB) of `s` waited, or the scenario's +/// length for one that never started. +pub fn worst_small_wait(s: &Scenario, o: &Outcome) -> f64 { + s.jobs + .iter() + .zip(&o.job_started) + .filter(|(j, _)| j.at >= 0.0 && j.need_kb < GB) + .map(|(j, t)| t.unwrap_or(s.secs) - j.at) + .fold(0.0, f64::max) +} + +/// How many of the jobs of `s` that end by themselves started. +pub fn jobs_started(s: &Scenario, o: &Outcome) -> usize { + s.jobs.iter().zip(&o.job_started).filter(|(j, t)| !j.long_lived && t.is_some()).count() +} + +pub fn header() -> String { + format!( + "{:<34} {:>5} {:>6} {:>9} {:>8} {:>6} {:>6} {:>6} {:>6} {:>9}", + "variant", "runs", "gave", "worst run", "stack", "stacks", "worst", "over", "lints", "worst" + ) + &format!( + "\n{:<34} {:>5} {:>6} {:>9} {:>8} {:>6} {:>6} {:>6} {:>6} {:>9}", + "", "done", "up", "wait", "idle", "at once", "mem", "limit", "done", "stack wait" + ) +} + +pub fn row(name: &str, o: &Outcome) -> String { + format!( + "{name:<34} {:>5} {:>6} {:>8.0}s {:>7.0}s {:>6} {:>5.1}% {:>5.0}s {:>6} {:>8.0}s", + o.runs_done, + o.runs_given_up, + o.worst_run_wait, + o.stack_idle_s, + o.max_stacks, + o.worst_mem_pct, + o.over_limit_s, + o.lints_done, + o.worst_stack_wait + ) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn the_macbook_morning() { + eprintln!("{}", header()); + for outside in [11.0, 12.8, 14.5] { + let o = run(&morning(outside)); + eprintln!("{}", row(&format!("outside {outside} GB"), &o)); + // 0.10.0 also capped outside_admit at pause_at, under mem_max. + let old = Scenario { lim: Limits { outside_mem_max_pct: 92.0, ..macbook_limits() }, ..morning(outside) }; + eprintln!("{}", row(&format!("outside {outside} GB, outside cap 92%"), &run(&old))); + } + } + + #[test] + fn a_large_job_at_the_front_on_omarchy() { + let s = head_of_line(); + let o = run(&s); + eprintln!( + "head of line: second stack at {:?}, {} of 25 jobs started, small jobs waited up to {:.0}s, worst memory {:.1}%", + o.job_started[1], + jobs_started(&s, &o), + worst_small_wait(&s, &o), + o.worst_mem_pct + ); + } +} From e2abb6301b5a492277e48601f68fc3ae3e9525c0 Mon Sep 17 00:00:00 2001 From: Vincent De Snerck <10668+SpoBo@users.noreply.github.com> Date: Fri, 16 Oct 2026 11:51:03 +0200 Subject: [PATCH 04/22] fix(queue): the progress rule starts only a job that fits, and holds no one back for a job that cannot With nothing running, or nothing but long-lived jobs past their start-up, the head of the line started whatever the readings said. On the MacBook that head was a second dev stack (priority 10) that did not fit: it started over memory while the first stack's run waited, and the run waited until its caller gave up. At macOS warning the rule was skipped altogether. The rule now starts the first job in line that fits: a long-lived job under mem_max (it stays), any other job up to all of RAM. A job that does not fit holds nobody back. At warning the rule still decides when only long-lived jobs run; critical still stops everything. Head of line: a waiting job that would not fit by that rule even once every job that ends by itself has ended can only start when a long-lived job is stopped. It no longer holds others back by priority or reservation. On Omarchy a 5.5 GB lint that had waited 55 minutes kept 23 jobs out, among them a 189 MB test. Entry gains long_lived (serde default), set by the owner from the config. The NEXT line names a long-lived holder as one that must be stopped. --- CHANGELOG.md | 19 ++++ README.md | 40 +++++-- src/queue.rs | 302 ++++++++++++++++++++++++++++++++++++++++++-------- src/report.rs | 46 ++++++-- src/runner.rs | 1 + src/stall.rs | 2 + 6 files changed, 342 insertions(+), 68 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9f8751e..d9d03e7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -22,8 +22,27 @@ and the versions follow [semver](https://semver.org). - Each run now stores how its need was made (`runs.need_why`), so `dump` can explain an old run too, not only the next one. +### Changed + +- With nothing running, or nothing but long-lived jobs past their start-up, + the first job in line that fits starts, up to all of RAM, rather than the + first in line whatever the memory reading says. A long-lived job, such as a + second dev stack, starts that way only under `mem_max`. A job that does not + fit holds nobody back, so the jobs behind it that fit start. At macOS + warning this rule still decides when only long-lived jobs run. +- A waiting job that would not fit even once every job that ends by itself + has ended holds nobody back, whatever its priority or reservation: it can + only start once a long-lived job is stopped or other programs free memory, + and the small jobs behind it start meanwhile. + ### Fixed +- A second dev stack is no longer started over memory while the first one's + run waits: the rule for "nothing runs but long-lived jobs" started the head + of the line whatever the readings said, and skipped itself at macOS + warning, so the run waited until its caller gave up. +- The NEXT line names a long-lived job that holds the room as one that must + be stopped, not as one that will finish. - `outside_admit` works again when `mem_max` is at or above `pause_at` (92% by default), as with `mem_max = 95`. It capped memory at `pause_at`, under the limit, so it could never let a job start; the cap is now diff --git a/README.md b/README.md index c1406bd..203f6c9 100644 --- a/README.md +++ b/README.md @@ -107,8 +107,10 @@ below). so the next job waits until it has shown what it takes (at most two minutes). A job with a history has its whole need booked from its first second, so the next job may start at once. So the job that needs that - stack does not wait behind it. `warn_admit = false` makes warning stop - every new job again. + stack does not wait behind it. When nothing runs but long-lived jobs past + their start-up, the rule below for nothing running decides at warning too, + up to all of RAM. `warn_admit = false` makes warning stop every new job + again. - **Memory is read at its recent peak.** Admission uses the highest reading of the last 10 seconds, so a job never starts in a short dip. - **Short jobs skip the CPU reading, not the CPU budget.** A job that usually @@ -131,10 +133,16 @@ below). halfway from `mem_max` to all of RAM (97.5% for `mem_max = 95`). Each such start prints a warning with what the job lacks and the rule that let it start. -- **Nothing running means the job starts,** whatever the CPU and memory - readings say, so the queue cannot deadlock on a wrong reading. Memory - pressure is the one exception: then the load comes from other programs, and - jobs wait until it eases. +- **Nothing running means the first job in line that fits starts,** whatever + the CPU reading says and up to all of RAM, so the queue cannot deadlock on + a wrong reading or on programs outside taskguard. The same holds when + nothing runs but long-lived jobs past their start-up. A job that would not + fit in RAM even so waits for memory and holds nobody back: the jobs behind + it that fit start. A long-lived job must fit under `mem_max` (with + `noise_mem`): it does not end by itself, so starting it over the limit is + no progress but load that stays. Memory pressure is the one exception: + then the load comes from other programs, and jobs wait until it eases + (at macOS warning, see above). - **Any order, oldest first.** A small job may pass a big one that does not fit yet. This is safe: a job only reaches taskguard when its launcher has decided that it may run. Turbo starts a task only after the tasks it depends @@ -183,7 +191,7 @@ below). its learned duration says it ends before the reserved job could start anyway: before the running jobs have freed the room it lacks, or, when programs outside taskguard keep it out, before none of them runs (then the - first in line starts whatever the readings say). Such a job takes no room + first in line that fits in RAM starts). Such a job takes no room the reserved one could use, so a steady stream of small jobs cannot keep it out, and a lint is not held up for half an hour behind a job that waits for two long test runs. A job with no learned duration waits. For a reserved job @@ -192,6 +200,14 @@ below). when no other job runs, also when its need is above the whole limit. On by default for 10 minutes (`max_backfill = 600`); `0` turns it off, `1800` lets every job that fits through for half an hour. +- **A job that only a stopped stack can make room for holds nobody back.** + A job ahead in line that would not fit by the rule for nothing running even + once every job that ends by itself has ended can only start when a + long-lived job is stopped, or when programs outside taskguard free memory. + Holding the jobs behind it gains it nothing, so it holds none of them back, + whatever its priority or reservation: on Omarchy one 5.5 GB lint that only + fit once a dev stack stopped kept two dozen others out, among them a test of + 189 MB. - **Pools** add a slot ceiling where jobs share something: one database, one set of services, one lock file. A slot ceiling only stops such jobs from running at the same time and breaking each other; CPU and memory are always @@ -211,8 +227,10 @@ below). core for hours. Its history keeps both: the start-up peak, which the next run is admitted against, and what it took after. Past its start-up it counts as load from outside taskguard: no job waits for it to end, and when - nothing else runs, the first in line starts whatever the readings say. - `auto_pause` leaves it alone. `taskguard status` marks it `STEADY`. + nothing else runs, the first in line that fits starts. A waiting long-lived + job itself starts that way only under `mem_max`. `auto_pause` leaves it + alone. `taskguard status` marks it `STEADY`, and a job it keeps out says it + starts when the stack is stopped, not when it finishes. ### What is measured @@ -657,6 +675,10 @@ A version only follows the rules it knows. A version that does not know `long_lived` runs such a job as any other: it holds its start-up peak until it ends. Its waiting jobs do read the lower needs of a long-lived job that a newer version runs, but they still wait for it to end when nothing else runs. +Versions up to 0.10.0 start the first in line whatever the readings say when +nothing runs but long-lived jobs past their start-up, also a second dev stack +over `mem_max`, and skip that rule at macOS warning; their jobs decide so +whatever newer versions do. A version before 0.4.0 does not know runs: its jobs never step aside for an older run. A version before 0.2.0 does not diff --git a/src/queue.rs b/src/queue.rs index e6cebdc..583644d 100644 --- a/src/queue.rs +++ b/src/queue.rs @@ -106,6 +106,10 @@ pub struct Entry { /// A long-lived job (a dev stack) left its start-up phase then: from /// that moment its needs follow what it uses. Older versions leave it out. pub steady_since: Option, + /// A long-lived job, from its start: it runs until someone stops it. + /// Older versions leave it out; their long-lived jobs show as such only + /// once they are steady. + pub long_lived: bool, /// Another job saw this one able to start for `STALL_S` while it still /// waited (`stall/.`). It holds no job back any more. Read /// from the queue, never written into the entry. @@ -126,6 +130,11 @@ impl Entry { self.steady_since.is_some() } + /// A job that does not end by itself: long-lived, or steady. + pub fn lasting(&self) -> bool { + self.long_lived || self.steady() + } + /// Seconds a running job has left by its learned duration, at least one. pub fn remaining(&self, now: f64) -> Option { self.est_dur_s.map(|d| (d - (now - self.started_at.unwrap_or(now))).max(1.0)) @@ -782,10 +791,17 @@ pub fn decide( if !paced || growing.is_some() { return Decision::Wait { blockers: vec![pressure] }; } - let room = Room::new(m, lim, running); - if let Some(mem) = room.blockers(m, lim, running, me, now, unknown_starts).into_iter().find(|b| matches!(b, Blocker::Memory { .. })) - { - return Decision::Wait { blockers: vec![mem, pressure] }; + // With only long-lived jobs past their start-up running, the progress + // rule below decides, up to all of RAM: they do not end by themselves, + // and the job that needs one of them would wait as long as the + // warning lasts, which can be all morning. + if !running.iter().all(Entry::steady) { + let room = Room::new(m, lim, running); + if let Some(mem) = + room.blockers(m, lim, running, me, now, unknown_starts).into_iter().find(|b| matches!(b, Blocker::Memory { .. })) + { + return Decision::Wait { blockers: vec![mem, pressure] }; + } } } @@ -794,25 +810,49 @@ pub fn decide( return Decision::Wait { blockers: vec![Blocker::Paused { key: p.key.clone() }] }; } - // Always make progress: with nothing running, the oldest job starts, - // whatever the readings say. This is what keeps the queue from deadlocking - // on a machine that is busy with work outside taskguard. A long-lived job - // past its start-up counts as such work: it may run for hours, and a job - // it keeps out would otherwise hold up the whole queue until it is stopped. + // Always make progress: with nothing running, the first job in line that + // fits starts, whatever the CPU reading says and up to all of RAM. This is + // what keeps the queue from deadlocking on a machine that is busy with + // work outside taskguard. A long-lived job past its start-up counts as + // such work: it may run for hours, and a job it keeps out would otherwise + // hold up the whole queue until it is stopped. + // + // A job that does not fit even so holds nobody back: it waits for memory, + // and the jobs behind it that fit start meanwhile. A long-lived job must + // fit under `mem_max`: it does not end by itself, so starting it over the + // limit is no progress but load that stays, and a second dev stack + // started that way kept the first one's run out until it was given up. + let room = Room::new(m, lim, running); if running.iter().all(Entry::steady) { - return match older.first() { - None if running.is_empty() => Decision::Admit { reason: "nothing else is running".into(), early: None, booked_cpu: None }, - None => Decision::Admit { + let fits = |w: &Entry| room.fits_on_progress(m, lim, running, w); + let head = older.iter().copied().chain(std::iter::once(me)).find(|w| fits(w)); + return match head { + Some(h) if !h.ahead_of(me) && running.is_empty() => { + Decision::Admit { reason: "nothing else is running".into(), early: None, booked_cpu: None } + } + Some(h) if !h.ahead_of(me) => Decision::Admit { reason: "nothing else is running but long-lived jobs past their start-up".into(), early: None, booked_cpu: None, }, Some(o) => Decision::Wait { blockers: vec![Blocker::Older { key: o.key.clone() }] }, + None => { + let mut blockers: Vec = room + .blockers(m, lim, running, me, now, unknown_starts) + .into_iter() + .filter(|b| matches!(b, Blocker::Memory { .. } | Blocker::Slots { .. })) + .collect(); + if blockers.is_empty() { + blockers.push(room.memory_blocker(m, lim, me)); + } + if let Some(p) = m.mem_pressure.filter(|p| *p >= lim.pressure_max) { + blockers.push(Blocker::Pressure { level: p, limit: lim.pressure_max, paced: true, after: None, next_in: None }); + } + Decision::Wait { blockers } + } }; } - let room = Room::new(m, lim, running); - let mut blockers = Vec::new(); // With backfill on, a job ahead that goes first holds its turn only while // it could start itself: a job that waits for memory cannot use the room @@ -836,14 +876,19 @@ pub fn decide( } now - w.queued_at > lim.max_backfill && !me.est_dur_s.zip(soonest_start(&theirs, running, now)).is_some_and(|(d, t)| d <= t) }; + // A job ahead that would not fit even once every job that ends by itself + // has ended can only start when a long-lived job is stopped, or programs + // outside taskguard free memory. It holds nobody back: holding the jobs + // behind it that fit gains it nothing, and on Omarchy one such job kept + // two dozen others out for half an hour. + let may_hold = |w: &Entry| room.fits_once_drained(m, lim, running, w); // An older job that newer ones have passed for `max_bypass` goes first. // Only a job ahead in line holds `me` back, so a job of an older run is // never held back for one of a newer run, or for a job that queued after // that run began. - if let Some(r) = older - .iter() - .find(|w| w.bypassed_since.is_some() && now - w.queued_at > lim.max_bypass && (lim.max_backfill <= lim.max_bypass || holds(w))) - { + if let Some(r) = older.iter().find(|w| { + w.bypassed_since.is_some() && now - w.queued_at > lim.max_bypass && may_hold(w) && (lim.max_backfill <= lim.max_bypass || holds(w)) + }) { blockers.push(Blocker::Reserved { key: r.key.clone(), waited_s: now - r.queued_at }); } // A waiting job with a higher priority goes first, unless its own pool is @@ -853,7 +898,9 @@ pub fn decide( (Some(pk), Some(max)) => running.iter().filter(|e| e.pool_key.as_ref() == Some(pk)).count() as u32 >= max, _ => false, }; - if let Some(h) = older.iter().find(|w| w.priority > me.priority && !pool_full(w) && (lim.max_backfill <= 0.0 || holds(w))) { + if let Some(h) = + older.iter().find(|w| w.priority > me.priority && !pool_full(w) && may_hold(w) && (lim.max_backfill <= 0.0 || holds(w))) + { blockers.push(Blocker::Priority { key: h.key.clone(), priority: h.priority }); } // A job of an older task-runner run that could start now goes first, so @@ -888,8 +935,9 @@ pub fn decide( /// In how many seconds a job that `blockers` keep out could start at the /// soonest, by the learned durations of the running jobs: once they have /// freed what it lacks, or else once none of them runs but long-lived jobs -/// past their start-up, as then the first in line starts whatever the readings -/// say. None when that hangs on a job whose duration is unknown. +/// past their start-up, as then the first in line that fits starts whatever +/// the CPU reading says. None when that hangs on a job whose duration is +/// unknown. fn soonest_start(blockers: &[Blocker], running: &[Entry], now: f64) -> Option { let drained = running.iter().filter(|e| !e.steady()).try_fold(0.0, |t: f64, e| Some(t.max(e.remaining(now)?))); blockers.iter().try_fold(0.0, |t: f64, b| Some(t.max(crate::report::start_eta(b, running, now).or(drained)?))) @@ -1077,6 +1125,7 @@ pub fn signal_tree(root: i32, sig: i32) { } /// The room the running jobs leave, read once per decision. +#[derive(Clone, Copy)] struct Room { res_cpu: f64, res_mem: u64, @@ -1087,12 +1136,17 @@ struct Room { promised_cpu: f64, /// What the running jobs hold now. ours_mem: u64, + /// What the running jobs that end by themselves hold, and still + /// reserve: the memory that frees up once they have ended. + passing_live: u64, + passing_res: u64, } impl Room { fn new(m: &MachineSample, lim: &Limits, running: &[Entry]) -> Room { let (res_cpu, res_mem) = reserve(running); let ours_mem = running.iter().map(|e| e.live_mem_kb).sum(); + let passing: Vec = running.iter().filter(|e| !e.lasting()).cloned().collect(); Room { res_cpu, res_mem, @@ -1101,6 +1155,8 @@ impl Room { mem_limit: m.mem_total_kb as f64 * lim.mem_max_pct / 100.0, mem_used: m.mem_for_admission(ours_mem), ours_mem, + passing_live: passing.iter().map(|e| e.live_mem_kb).sum(), + passing_res: reserve(&passing).1, } } @@ -1108,6 +1164,51 @@ impl Room { (self.mem_used + self.res_mem + job.need_mem_kb) as f64 } + /// The most memory a job may take the machine to by the progress rule: + /// all of RAM, as nothing else of taskguard's would come after it; a + /// long-lived job, which stays, only `mem_max` (with `noise_mem`). + fn progress_ceiling(&self, m: &MachineSample, lim: &Limits, job: &Entry) -> f64 { + let total = m.mem_total_kb as f64; + if job.long_lived { self.mem_limit + total * lim.noise_mem_pct / 100.0 } else { total } + } + + /// May `job` start by the progress rule now: its pool has a free slot + /// and it fits under `progress_ceiling`. + fn fits_on_progress(&self, m: &MachineSample, lim: &Limits, running: &[Entry], job: &Entry) -> bool { + let pool_free = match (&job.pool_key, job.pool_slots) { + (Some(pk), Some(max)) => (running.iter().filter(|e| e.pool_key.as_ref() == Some(pk)).count() as u32) < max, + _ => true, + }; + pool_free && self.mem_would(job) <= self.progress_ceiling(m, lim, job) + } + + /// Would `job` fit by the progress rule once every running job that ends + /// by itself has ended? If not, it starts only when a long-lived job is + /// stopped or programs outside taskguard free memory. + fn fits_once_drained(&self, m: &MachineSample, lim: &Limits, running: &[Entry], job: &Entry) -> bool { + let lasting: Vec = running.iter().filter(|e| e.lasting()).cloned().collect(); + let drained = Room { + mem_used: self.mem_used.saturating_sub(self.passing_live), + res_mem: self.res_mem.saturating_sub(self.passing_res), + ..*self + }; + drained.fits_on_progress(m, lim, &lasting, job) + } + + /// The memory blocker for `job`, whether or not it fits under the limit. + fn memory_blocker(&self, m: &MachineSample, lim: &Limits, job: &Entry) -> Blocker { + let mem_would = self.mem_would(job); + Blocker::Memory { + would_pct: pct(mem_would, m.mem_total_kb as f64), + limit_pct: lim.mem_max_pct, + short_kb: (mem_would - self.mem_limit).max(0.0) as u64, + used_kb: self.mem_used, + reserve_kb: self.res_mem, + need_kb: job.need_mem_kb, + total_kb: m.mem_total_kb, + } + } + /// What keeps `job` from starting, after the leeway: a job that falls /// short only by a little, or only because of programs outside taskguard, /// or that cannot fit now and gets its `partial_fit`, starts anyway. Then @@ -1250,17 +1351,8 @@ impl Room { blockers.push(Blocker::LearnStagger { wait_s: (lim.learn_stagger - (now - newest)).max(0.0) }); } } - let mem_would = self.mem_would(job); - if mem_would > self.mem_limit { - blockers.push(Blocker::Memory { - would_pct: pct(mem_would, m.mem_total_kb as f64), - limit_pct: lim.mem_max_pct, - short_kb: (mem_would - self.mem_limit) as u64, - used_kb: self.mem_used, - reserve_kb: self.res_mem, - need_kb: job.need_mem_kb, - total_kb: m.mem_total_kb, - }); + if self.mem_would(job) > self.mem_limit { + blockers.push(self.memory_blocker(m, lim, job)); } // A short job is over before the CPU reading shows it, so the reading // does not hold it back. What taskguard has promised its own jobs @@ -1479,11 +1571,18 @@ mod tests { } #[test] - fn nothing_running_always_admits_the_oldest() { - let big = job(1, "big", 64.0, 500); - assert!(blockers(decide(&machine(12.0, 31), &LIM, &[], std::slice::from_ref(&big), &big, 1000.0, &[])).is_empty()); + fn nothing_running_admits_the_first_in_line_that_fits_in_ram() { + // Whatever the CPU reading says, and past mem_max: 20 GB held and 10 + // GB more is 94% of 32 GB, over the 85% limit, under all of RAM. + let old = job(1, "old", 64.0, 10); + assert!(blockers(decide(&machine(12.0, 20), &LIM, &[], std::slice::from_ref(&old), &old, 1000.0, &[])).is_empty()); let young = job(2, "young", 1.0, 1); - assert_eq!(blockers(decide(&machine(0.0, 1), &LIM, &[], &[big.clone(), young.clone()], &young, 1000.0, &[])), vec!["order"]); + assert_eq!(blockers(decide(&machine(0.0, 1), &LIM, &[], &[old.clone(), young.clone()], &young, 1000.0, &[])), vec!["order"]); + // A job that would not fit in all of RAM waits for memory, and holds + // nobody back: the young job that fits starts. + let big = job(1, "big", 64.0, 500); + assert_eq!(blockers(decide(&machine(12.0, 31), &LIM, &[], std::slice::from_ref(&big), &big, 1000.0, &[])), vec!["memory"]); + assert!(blockers(decide(&machine(0.0, 1), &LIM, &[], &[big.clone(), young.clone()], &young, 1000.0, &[])).is_empty()); } #[test] @@ -1703,7 +1802,7 @@ mod tests { let r = std::slice::from_ref(&running); // The urgent job is newer and waits for memory; the small old one fits. let old = job(2, "old", 1.0, 1); - let mut urgent = job(3, "urgent", 1.0, 30); + let mut urgent = job(3, "urgent", 1.0, 20); urgent.priority = 1; let waiting = [old.clone(), urgent.clone()]; assert_eq!(blockers(decide(&machine(1.0, 8), &LIM, r, &waiting, &old, 1000.0, &[])), vec!["priority"]); @@ -1819,14 +1918,23 @@ mod tests { #[test] fn a_job_passed_for_too_long_is_reserved() { let running = job(1, "r", 1.0, 1); - let mut big = job(2, "big", 2.0, 30); + let mut big = job(2, "big", 2.0, 20); big.bypassed_since = Some(1010.0); let small = job(3, "small", 1.0, 1); let waiting = [big.clone(), small.clone()]; // Not yet past max_bypass. assert!(blockers(decide(&machine(1.0, 8), &LIM, std::slice::from_ref(&running), &waiting, &small, 1100.0, &[])).is_empty()); // Past it: nobody newer may start. - assert_eq!(blockers(decide(&machine(1.0, 8), &LIM, &[running], &waiting, &small, 1200.0, &[])), vec!["reserved"]); + assert_eq!( + blockers(decide(&machine(1.0, 8), &LIM, std::slice::from_ref(&running), &waiting, &small, 1200.0, &[])), + vec!["reserved"] + ); + // A job that would not fit in RAM even once r has ended (8 GB held by + // other programs and 30 GB more) waits for them, not for r: it holds + // nobody back. + let huge = Entry { need_mem_kb: 30 * GB, ..big }; + let waiting = [huge, small.clone()]; + assert!(blockers(decide(&machine(1.0, 8), &LIM, &[running], &waiting, &small, 1200.0, &[])).is_empty()); } /// A 20 GB compile has run since 1000 and holds its memory. At 1000 a @@ -2063,24 +2171,25 @@ mod tests { /// A reserved job that a stack keeps out: the stack never ends by itself, /// so nothing counts on it to make room. It counts as load from outside - /// taskguard: once only stacks run, the first in line starts whatever the - /// readings say, and past `max_backfill` a newer job passes only when it - /// ends before that. + /// taskguard: once only stacks run, the first in line that fits in RAM + /// starts, and past `max_backfill` a newer job passes only when it ends + /// before that. #[test] fn backfill_never_waits_for_a_long_lived_job_to_end() { let backfill = Limits { max_backfill: 1800.0, ..LIM }; - // Two stacks past their start-up hold 10 GB each, other programs 6. + // Two stacks past their start-up hold 7 GB each, other programs 5. let steady = |pid: i32| Entry { pid, started_at: Some(0.0), steady_since: Some(300.0), + long_lived: true, live_cpu: 0.3, - live_mem_kb: 10 * GB, - ..job(1, "dalp:stack", 0.3, 10) + live_mem_kb: 7 * GB, + ..job(1, "dalp:stack", 0.3, 7) }; let stacks = [steady(11), steady(12)]; - let m = machine(2.0, 26); - // A 12 GB head would reach 38 GB of 27.2. + let m = machine(2.0, 19); + // A 12 GB head would reach 31 GB, over the 27.2 GB limit, in RAM. let (_, head) = blocked_head(); let quick = Entry { est_dur_s: Some(60.0), ..job(3, "install", 0.2, 1) }; let waiting = [head.clone(), quick.clone()]; @@ -2107,6 +2216,92 @@ mod tests { assert_eq!(blockers(decide(&m, &backfill, &running, &waiting, &quick, 2900.0, &[])), vec!["reserved"]); } + /// A steady dev stack of `live_gb`, as the MacBook's at 10:09. + fn steady_stack(pid: i32, live_gb: u64) -> Entry { + Entry { + pid, + started_at: Some(0.0), + steady_since: Some(120.0), + long_lived: true, + priority: 10, + live_cpu: 0.1, + live_mem_kb: live_gb * GB, + ..job(pid as u64, "dev-env:stack", 0.1, live_gb + live_gb / 4) + } + } + + #[test] + fn the_progress_rule_never_starts_a_stack_that_does_not_fit() { + // The MacBook at 10:09:56: one steady stack, 24 GB held of 32, mem_max + // 95. A second stack of 9.4 GB queued at priority 10, ahead of the run + // of the first one (4.3 GB). + let lim = Limits { mem_max_pct: 95.0, ..LIM }; + let a = steady_stack(11, 7); + let stack_b = Entry { long_lived: true, priority: 10, need_mem_kb: 94 * GB / 10, ..job(2, "dev-env:stack", 3.0, 0) }; + let run_a = Entry { need_mem_kb: 43 * GB / 10, ..job(3, "pr.ts --pr 17398", 3.0, 0) }; + let waiting = [stack_b.clone(), run_a.clone()]; + let m = machine(1.0, 24); + let r = std::slice::from_ref(&a); + // 24 + 1.75 still reserved by the stack + 9.4 = 35 GB, over 30.4: the + // stack waits, and holds nobody back. + assert_eq!(blockers(decide(&m, &lim, r, &waiting, &stack_b, 1000.0, &[])), vec!["memory"]); + // The run fits under all of RAM (30 GB of 32), and starts. + let d = decide(&m, &lim, r, &waiting, &run_a, 1000.0, &[]); + assert!(matches!(&d, Decision::Admit { reason, .. } if reason.contains("long-lived")), "{d:?}"); + // With nothing running at all, a stack still has to fit under mem_max. + assert_eq!(blockers(decide(&machine(1.0, 22), &lim, &[], &waiting, &stack_b, 1000.0, &[])), vec!["memory"]); + assert!(blockers(decide(&machine(1.0, 20), &lim, &[], &waiting, &stack_b, 1000.0, &[])).is_empty()); + // At macOS warning the same holds: the run starts next to its stack. + let mut warn = m.clone(); + warn.mem_pressure = Some(50.0); + let warned = Limits { warn_admit: true, ..lim }; + assert!(blockers(decide(&warn, &warned, r, &waiting, &run_a, 1000.0, &[])).is_empty()); + assert_eq!(blockers(decide(&warn, &warned, r, &waiting, &stack_b, 1000.0, &[])), vec!["memory", "pressure"]); + // Critical still stops every new job. + warn.mem_pressure = Some(100.0); + assert_eq!(blockers(decide(&warn, &warned, r, &waiting, &run_a, 1000.0, &[])), vec!["pressure"]); + } + + #[test] + fn a_job_that_only_fits_once_a_stack_stops_holds_nobody_back() { + // Omarchy at 11:33: two stacks run, one past its start-up, and 18.9 GB + // is held of 31.3. An oxlint of 5.5 GB has waited 55 minutes, past + // max_backfill, and would reach 110%: it fits only once a stack is + // stopped. A test of 189 MB queued behind it. + let lim = Limits { mem_max_pct: 95.0, max_backfill: 1800.0, ..LIM }; + let m = MachineSample::fixed(5.7, 24, 189 * GB / 10, 313 * GB / 10); + let steady = Entry { live_mem_kb: 82 * GB / 10, need_mem_kb: 102 * GB / 10, ..steady_stack(11, 8) }; + let starting = Entry { + steady_since: None, + started_at: Some(3000.0), + live_mem_kb: 11 * GB / 10, + need_mem_kb: 91 * GB / 10, + ..steady_stack(12, 1) + }; + let running = [steady.clone(), starting]; + let mut lint = job(1, "domains/token-features/app:oxlint", 3.6, 0); + (lint.need_mem_kb, lint.queued_at, lint.bypassed_since, lint.est_dur_s) = (55 * GB / 10, 0.0, Some(60.0), Some(40.0)); + let test = Entry { need_mem_kb: 189 * 1024, est_dur_s: Some(30.0), ..job(2, "custody-ops/flows:bun_test", 1.3, 0) }; + let waiting = [lint.clone(), test.clone()]; + assert_eq!(blockers(decide(&m, &lim, &running, &waiting, &lint, 3300.0, &[])), vec!["memory"]); + assert!(blockers(decide(&m, &lim, &running, &waiting, &test, 3300.0, &[])).is_empty(), "the test fits and starts"); + // So in the progress rule: with only the steady stack running, the + // lint does not fit in RAM; the test behind it does. + let m = MachineSample::fixed(1.0, 24, 28 * GB, 313 * GB / 10); + let r = std::slice::from_ref(&steady); + assert_eq!(blockers(decide(&m, &lim, r, &waiting, &lint, 3300.0, &[])), vec!["memory"]); + assert!(blockers(decide(&m, &lim, r, &waiting, &test, 3300.0, &[])).is_empty()); + // A lint that fits once a job that ends by itself has ended still + // holds back a job that would not end before then, as before. + let compile = Entry { started_at: Some(3200.0), live_mem_kb: 6 * GB, est_dur_s: Some(600.0), ..job(3, "compile", 1.0, 6) }; + let m = MachineSample::fixed(5.0, 24, 24 * GB, 313 * GB / 10); + let running = [steady, compile]; + let slow = Entry { est_dur_s: Some(1000.0), ..test }; + let waiting = [lint.clone(), slow.clone()]; + assert_eq!(blockers(decide(&m, &lim, &running, &waiting, &lint, 3300.0, &[])), vec!["memory"]); + assert_eq!(blockers(decide(&m, &lim, &running, &waiting, &slow, 3300.0, &[])), vec!["reserved"]); + } + #[test] fn backfill_keeps_pools_and_pressure() { let (mut running, mut head) = blocked_head(); @@ -2357,10 +2552,19 @@ mod tests { ); assert!(blockers(decide(&m, &lim, &both, std::slice::from_ref(&lint), &lint, 1015.0, &[])).is_empty(), "it has settled"); - // At warning, memory has no leeway, also when only long-lived jobs run. + // At warning, memory has no leeway while a job that ends by itself + // runs. let big = job(5, "big", 1.0, 12); - let d = decide(&m, &lim, std::slice::from_ref(&stack), std::slice::from_ref(&big), &big, 1000.0, &[]); + let busy = [stack.clone(), Entry { started_at: Some(999.0), ..lint.clone() }]; + let d = decide(&m, &lim, &busy, std::slice::from_ref(&big), &big, 1000.0, &[]); assert_eq!(blockers(d), vec!["memory", "pressure"]); + // With only long-lived jobs running, the progress rule decides, up to + // all of RAM: 16 + 12 GB is 88% of 32, over the 85% limit. + let d = decide(&m, &lim, std::slice::from_ref(&stack), std::slice::from_ref(&big), &big, 1000.0, &[]); + assert!(blockers(d).is_empty(), "the progress rule also works at warning"); + let bigger = job(5, "bigger", 1.0, 17); + let d = decide(&m, &lim, std::slice::from_ref(&stack), std::slice::from_ref(&bigger), &bigger, 1000.0, &[]); + assert_eq!(blockers(d), vec!["memory", "pressure"], "over all of RAM it waits"); // Critical stops every new job, and warn_admit off keeps the old rule. m.mem_pressure = Some(100.0); diff --git a/src/report.rs b/src/report.rs index 33acaec..649a0b4 100644 --- a/src/report.rs +++ b/src/report.rs @@ -131,28 +131,32 @@ pub fn eta_text(eta: f64) -> String { fn unblock(b: &Blocker, running: &[Entry], now: f64) -> (String, Option) { let remaining = |e: &Entry| e.remaining(now); - let first_to_free = |need: &dyn Fn(&Entry) -> f64, short: f64| -> Option<(String, Option)> { + let first_to_free = |need: &dyn Fn(&Entry) -> f64, short: f64| -> Option<&Entry> { let mut rs: Vec<&Entry> = running.iter().collect(); - rs.sort_by(|a, b| remaining(a).unwrap_or(f64::MAX).total_cmp(&remaining(b).unwrap_or(f64::MAX))); + // A long-lived job comes last: it ends only when someone stops it. + rs.sort_by(|a, b| { + a.lasting().cmp(&b.lasting()).then(remaining(a).unwrap_or(f64::MAX).total_cmp(&remaining(b).unwrap_or(f64::MAX))) + }); let mut freed = 0.0; for e in rs { freed += need(e); if freed >= short { - return Some((e.key.clone(), remaining(e))); + return Some(e); } } None }; - let when = |key: String, rem: Option| match rem { - Some(r) => (format!("starts when {key} finishes (about {} left)", dur(r)), Some(r)), - None => (format!("starts when {key} finishes"), None), + let when = |e: &Entry| match remaining(e) { + _ if e.lasting() => (format!("starts when {} is stopped (long-lived: it does not end by itself)", e.key), None), + Some(r) => (format!("starts when {} finishes (about {} left)", e.key, dur(r)), Some(r)), + None => (format!("starts when {} finishes", e.key), None), }; match b { Blocker::Older { key } | Blocker::Reserved { key, .. } | Blocker::Priority { key, .. } | Blocker::Pipeline { key, .. } => { (format!("starts after {key} has started"), None) } - Blocker::Slots { holders, .. } => match holders.first() { - Some(h) => when(h.clone(), running.iter().find(|e| &e.key == h).and_then(remaining)), + Blocker::Slots { holders, .. } => match holders.first().and_then(|h| running.iter().find(|e| &e.key == h)) { + Some(e) => when(e), None => ("starts when a slot frees up".into(), None), }, Blocker::LearnStagger { wait_s } => (format!("starts in about {:.0}s", wait_s.ceil()), Some(*wait_s)), @@ -166,14 +170,14 @@ fn unblock(b: &Blocker, running: &[Entry], now: f64) -> (String, Option) { Blocker::Memory { short_kb, .. } => { let need = |e: &Entry| e.need_mem_kb.max(e.live_mem_kb) as f64; match first_to_free(&need, *short_kb as f64) { - Some((k, r)) => when(k, r), + Some(e) => when(e), None => (format!("starts when {} of memory frees up", gb(*short_kb)), None), } } Blocker::Cpu { short, .. } => { let need = |e: &Entry| e.need_cpu.max(e.live_cpu); match first_to_free(&need, *short) { - Some((k, r)) => when(k, r), + Some(e) => when(e), None => (format!("starts when {short:.1} cores free up"), None), } } @@ -888,4 +892,26 @@ mod tests { assert_eq!(s.waiting[0].eta_s.map(|t| t.round()), Some(43.0)); assert!(text.contains("packages/api:tsc")); } + + #[test] + fn a_long_lived_holder_is_named_as_stopped_not_finished() { + let stack = Entry { + key: "dev-env:stack".into(), + need_mem_kb: 9 * GB, + live_mem_kb: 7 * GB, + long_lived: true, + started_at: Some(0.0), + ..Default::default() + }; + let b = Blocker::Memory { + would_pct: 100.0, + limit_pct: 95.0, + short_kb: 2 * GB, + used_kb: 0, + reserve_kb: 0, + need_kb: 0, + total_kb: 32 * GB, + }; + assert_eq!(unblock_text(&b, &[stack], 1000.0), "starts when dev-env:stack is stopped (long-lived: it does not end by itself)"); + } } diff --git a/src/runner.rs b/src/runner.rs index 3f0d0c3..af7cba8 100644 --- a/src/runner.rs +++ b/src/runner.rs @@ -388,6 +388,7 @@ pub fn run(mut o: Opts) -> Result { // until someone stops it, so no one counts on it ending. est_dur_s: learned.dur_s.or(est_dur).filter(|_| startup.is_none()), priority: o.priority.or(cls.priority).unwrap_or(cfg.priority), + long_lived: startup.is_some(), pausable: true, version: Some(env!("CARGO_PKG_VERSION").to_string()), ..Default::default() diff --git a/src/stall.rs b/src/stall.rs index baf0ba6..3825a08 100644 --- a/src/stall.rs +++ b/src/stall.rs @@ -231,6 +231,7 @@ pub fn run(s: &Scenario) -> Outcome { known: true, est_dur_s: (!j.long_lived).then_some(j.secs), priority: j.priority, + long_lived: j.long_lived, ..Default::default() }; for (n, j) in s.jobs.iter().enumerate().filter(|(_, j)| j.at < 0.0) { @@ -274,6 +275,7 @@ pub fn run(s: &Scenario) -> Outcome { need_mem_kb: a.stack_need_kb, known: true, priority: 10, + long_lived: true, ..Default::default() }; waiting.push((e, Kind::Stack(n))); From bf4bf0aabc6a671f8bb662311d6360093d04d700 Mon Sep 17 00:00:00 2001 From: Vincent De Snerck <10668+SpoBo@users.noreply.github.com> Date: Fri, 16 Oct 2026 12:02:39 +0200 Subject: [PATCH 05/22] feat: a long-lived job books room for the work it expects, and that work goes first A dev stack and the run it exists for were two jobs nothing related: the caller queued the run only once the stack was up, minutes later, so taskguard admitted the stack without room for its run and let other jobs, or a second stack, take that room. The caller can now declare the work: --expect WORK on the stack's job (repeatable), or TASKGUARD_EXPECT=WORK,WORK in its environment, which DALP's devenv up passes through as it is. WORK is a pool name, a job key, or a command pattern. The stack books each item's expected need, learned as the work itself would learn it, and starts only with room for itself and its work. A job in the same checkout that matches links to the stack: it goes one priority above it (an ordinary priority, which older versions honour), takes over the booked room when it starts (the stack drops it on a nudge), and waits, holding nobody back, while its stack still waits. Status shows "(for stack KEY)" on the work and "(expects WORK, books SIZE)" on the stack. Data formats, all additive with serde defaults: queue::Entry gains expects and work_for, queue::Nudge gains work_started. No change to the machine sample, the database schema or job keys. In the stall model, declaring the browser run clears the MacBook morning at 11, 12.8 and 14.5 GB outside: no run given up, worst run wait 9 s at 12.8 and 14.5 GB (CPU-bound 120 s at 11 GB), at the cost of one stack at a time. --- CHANGELOG.md | 9 ++ README.md | 29 +++++- src/db.rs | 6 +- src/help.rs | 4 + src/main.rs | 1 + src/need.rs | 33 +++++++ src/queue.rs | 258 ++++++++++++++++++++++++++++++++++++++++++++++++-- src/report.rs | 33 +++++-- src/runner.rs | 97 +++++++++++++++++++ src/stall.rs | 54 ++++++++++- tests/cli.rs | 37 ++++++++ 11 files changed, 543 insertions(+), 18 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d9d03e7..20d9316 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -21,6 +21,15 @@ and the versions follow [semver](https://semver.org). Use it to find out why a job reserved what it did. - Each run now stores how its need was made (`runs.need_why`), so `dump` can explain an old run too, not only the next one. +- A long-lived job can declare the work that will run on it: + `--expect WORK` (repeatable) or `TASKGUARD_EXPECT=WORK,WORK`, where WORK is + a pool name, a job key or a command pattern. The stack books room for that + work (learned as the work learns its own need) and starts only with room + for both. A job in the same checkout that matches is its work: it goes one + priority above the stack, takes over the booked room when it starts, and + waits while its stack still waits. `taskguard status` shows `(for stack + KEY)` on the work and `(expects WORK, books SIZE)` on the stack. Without a + declaration nothing changes. ### Changed diff --git a/README.md b/README.md index 203f6c9..56503ba 100644 --- a/README.md +++ b/README.md @@ -231,6 +231,24 @@ below). job itself starts that way only under `mem_max`. `auto_pause` leaves it alone. `taskguard status` marks it `STEADY`, and a job it keeps out says it starts when the stack is stopped, not when it finishes. +- **Work for a long-lived job goes first.** The caller that starts a dev + stack usually knows what will run on it: the browser run it brings the + stack up for. It can say so, with `--expect WORK` on the stack's job + (repeatable) or `TASKGUARD_EXPECT=WORK,WORK` in its environment, so it + passes through scripts such as `devenv up`. WORK is a pool name, a job key, + or a command pattern as in `[[job]] match`. The stack then books room for + each piece of work, learned as that work learns its own need: what jobs of + the pool typically took, or the history or first guess of the command. It + starts only when the machine has room for itself and that work, so the + whole load is known before it starts. A job that queues in the same + checkout and matches WORK is that work: it goes one priority above the + stack, so it starts before the jobs that queued for room meanwhile and + before a next stack, and when it starts it takes over the room booked for + it, which the stack then drops. `taskguard status` shows both ends: `(for + stack dev-env:stack)` on the work, `(expects browser, books 3.6 GB)` on the + stack. Work that queues while its stack still waits waits for the stack and + holds nobody back meanwhile. A stack started without `--expect` behaves as + before; the variable means nothing to any other job. ### What is measured @@ -322,6 +340,7 @@ taskguard --wait [--id NAME] | `--min-cpu N` | Never start with fewer than N free cores. | | `--min-mem SIZE` | Never start with less than SIZE free (`6G`, `512M`). | | `--priority N` | Higher starts first. The default is 0, or `priority` from the config. | +| `--expect WORK` | For a long-lived job: work that will run on it (a pool name, job key or command pattern). It books room for it, and that work goes first. Repeatable; also `TASKGUARD_EXPECT=WORK,WORK`. | | `--now` | Skip the queue, but still measure and learn. | | `--bg` | Wait for room, then return and let the job run on. | | `--pipe` | With `--bg`: pass stdin to the command. | @@ -617,6 +636,10 @@ repo. Put a repo's own scripts in its `.taskguard.toml`: past `mem_max`, up to `pause_at` (or halfway from `mem_max` to all of RAM when `mem_max` is at or above `pause_at`). Set `outside_admit = false` and `noise_mem = 0` to keep `mem_max` a hard rule. +- A stack that expects work (`--expect`) books room for it until it starts, + however long that takes: work that never comes keeps that room booked + while the stack runs. A stack plus its work that do not fit next to another + stack wait for room, where before the stack started and its work waited. ## Moving from tsc-queue @@ -678,7 +701,11 @@ version runs, but they still wait for it to end when nothing else runs. Versions up to 0.10.0 start the first in line whatever the readings say when nothing runs but long-lived jobs past their start-up, also a second dev stack over `mem_max`, and skip that rule at macOS warning; their jobs decide so -whatever newer versions do. +whatever newer versions do. They do not know `--expect` either: they ignore +`TASKGUARD_EXPECT`, do not see what a newer stack books for its work, and may +start their own jobs into that room. The priority a newer job takes as work +for a stack is an ordinary priority, so their waiting jobs do let it go +first. A version before 0.4.0 does not know runs: its jobs never step aside for an older run. A version before 0.2.0 does not diff --git a/src/db.rs b/src/db.rs index 0b08557..aa93fb4 100644 --- a/src/db.rs +++ b/src/db.rs @@ -1006,7 +1006,11 @@ impl Db { if let Some(p) = pool { tiers.push((None, None, Some(p), None, format!("jobs in {p}"))); } - tiers.push((None, None, None, Some(tool), format!("{tool} jobs"))); + // A guess without a program (work named only by its pool) does not + // fall back on every job there is. + if !tool.is_empty() { + tiers.push((None, None, None, Some(tool), format!("{tool} jobs"))); + } for (same, l, p, t, what) in tiers { let mut stmt = self.conn.prepare_cached( "SELECT max(peak_mem_kb), avg(coalesce(cores_used, cores_wanted)), avg(ended_at - started_at - coalesce(paused_s, 0)) FROM runs diff --git a/src/help.rs b/src/help.rs index d8dbb23..474dbd6 100644 --- a/src/help.rs +++ b/src/help.rs @@ -12,6 +12,10 @@ options (sem style; the default is to run in the foreground): --min-cpu N never start with fewer than N free cores --min-mem SIZE never start with less than SIZE free (6G, 512M) --priority N higher starts first (default 0; negative is allowed) + --expect WORK a long-lived job (a dev stack): work that will run on + it, a pool name, job key or command pattern; it books + room for that work and the work goes first once it + queues. Repeatable; also TASKGUARD_EXPECT=WORK,WORK --now skip the queue, but still measure and learn --wraps COMMAND runs other taskguard jobs (turbo, a package script): run it straight through and let them queue diff --git a/src/main.rs b/src/main.rs index 5c70a9f..b9a3653 100644 --- a/src/main.rs +++ b/src/main.rs @@ -124,6 +124,7 @@ pub fn parse_opts(args: &[String]) -> Result { o.min_cpu = Some(v.parse().map_err(|_| anyhow::anyhow!("--min-cpu needs a number of cores"))?); } "--min-mem" => o.min_mem_kb = Some(config::parse_size_kb(&take_value(args, &mut i, name)?)?), + "--expect" => o.expect.push(take_value(args, &mut i, name)?), "--priority" => { let v = take_value(args, &mut i, name)?; o.priority = Some(v.parse().map_err(|_| anyhow::anyhow!("--priority needs a whole number"))?); diff --git a/src/need.rs b/src/need.rs index 353d413..ff4ea13 100644 --- a/src/need.rs +++ b/src/need.rs @@ -185,6 +185,39 @@ pub fn guess(db: Option<&Db>, cfg: &Config, package: Option<&str>, argv: &[Strin } } +/// What a long-lived job books for work it expects (`--expect WORK`): the +/// need that work will start with, learned as the work itself learns it. +/// WORK is a pool name, a job key, or a command (pattern) run from `cwd`. +/// A pool books what jobs of that pool typically took; a key or command +/// its history, else a first guess as for a new job, with the minimums of +/// the job rules it matches. +pub fn expected(db: Option<&Db>, cfg: &Config, cwd: &Path, work: &str) -> crate::queue::Expect { + let expect = |mem_kb: u64, from: String| crate::queue::Expect { work: work.to_string(), mem_kb, from, ..Default::default() }; + if cfg.pools.iter().any(|p| p.name == work) { + let est = db.and_then(|d| d.estimate(None, None, "", Some(work)).ok().flatten()); + return match est.and_then(|e| e.mem_kb.map(|m| (m, e.from))) { + Some((m, from)) => expect(m, from), + None => expect(cfg.new_job_mem_kb, format!("no jobs in pool {work} yet: the new_job_mem default")), + }; + } + let argv = crate::matcher::split_shell(work); + let has_history = |k: &str| db.and_then(|d| d.learned(k, &cfg.learn()).ok()).is_some_and(|l| l.runs > 0); + let key = if has_history(work) { work.to_string() } else { crate::key::run_key(cwd, &argv) }; + let cls = cfg.classify(&argv, &key).unwrap_or_default(); + let (learned, history_key) = match db { + Some(d) => history(d, cfg, cwd, &argv, &key), + None => (Default::default(), key.clone()), + }; + let pool = cls.pool.as_ref().map(|p| p.0.as_str()); + let guess = (learned.runs == 0).then(|| self::guess(db, cfg, None, &argv, cls.label.as_deref(), pool)); + let need = derive(&inputs(cfg, &learned, Some(&history_key), guess.as_ref(), cls.min_cpu, cls.min_mem_kb)); + let from = match need.steps.last() { + Some(s) => format!("{}: {}", s.step, s.why), + None => "nothing known".into(), + }; + expect(need.mem_kb, from) +} + /// The program a command runs: `tsc` for `bunx tsc -p .`. pub fn tool(argv: &[String]) -> String { crate::matcher::effective(argv).first().map(|c| c.rsplit('/').next().unwrap_or(c).to_string()).unwrap_or_default() diff --git a/src/queue.rs b/src/queue.rs index 583644d..cb1593c 100644 --- a/src/queue.rs +++ b/src/queue.rs @@ -110,6 +110,13 @@ pub struct Entry { /// Older versions leave it out; their long-lived jobs show as such only /// once they are steady. pub long_lived: bool, + /// For a long-lived job: the work its caller said will run on it + /// (`--expect`), with the memory booked for each until that work starts. + /// Older versions leave it out, and do not see the booking. + pub expects: Vec, + /// The long-lived job this job is work for: one in the same checkout + /// that expects it. Older versions leave it out. + pub work_for: Option, /// Another job saw this one able to start for `STALL_S` while it still /// waited (`stall/.`). It holds no job back any more. Read /// from the queue, never written into the entry. @@ -117,6 +124,31 @@ pub struct Entry { pub stalled: bool, } +/// Work a long-lived job expects: a pool name, a job key, or a command +/// pattern as in `[[job]] match`. Until a job that matches it starts, the +/// long-lived job books `mem_kb` for it, so the room the work needs is +/// there when it comes. +#[derive(Debug, Clone, Default, Serialize, Deserialize, PartialEq)] +#[serde(default)] +pub struct Expect { + pub work: String, + pub mem_kb: u64, + /// Where `mem_kb` comes from, in words. + pub from: String, + /// A job for it has started: the booking is gone. + pub started: bool, +} + +/// The long-lived job a job is work for, and which of its `expects` it +/// takes over (none when every one of them has started already). +#[derive(Debug, Clone, Default, Serialize, Deserialize, PartialEq)] +#[serde(default)] +pub struct WorkFor { + pub pid: i32, + pub key: String, + pub item: Option, +} + impl Entry { /// Paused by auto_pause, not by hand: it resumes before anything new starts. pub fn auto_paused(&self) -> bool { @@ -264,6 +296,9 @@ pub struct Nudge { pub pause: Option, pub need_cpu: Option, pub need_mem_kb: Option, + /// To a long-lived job: work it expects has started (`Expect::started`), + /// by index. + pub work_started: Vec, } pub struct Queue { @@ -661,6 +696,10 @@ pub enum Blocker { Paused { key: String, }, + /// Work for a long-lived job that does not run yet: it waits for it. + Stack { + key: String, + }, Memory { would_pct: f64, limit_pct: f64, @@ -692,6 +731,7 @@ impl Blocker { Blocker::Memory { .. } => "memory", Blocker::Pressure { .. } => "pressure", Blocker::Paused { .. } => "paused", + Blocker::Stack { .. } => "stack", Blocker::Cpu { .. } => "cpu", } } @@ -723,13 +763,84 @@ pub enum Decision { /// A job paused by hand reserves no CPU: it uses none until someone resumes /// it, and the room it leaves is what the pause was for. Its memory stays /// reserved, since it keeps what it holds and grows again when it resumes. +/// +/// A long-lived job also reserves what it books for the work it expects. pub fn reserve(running: &[Entry]) -> (f64, u64) { running.iter().fold((0.0, 0), |(c, m), e| { let cpu = if e.paused_by_hand { 0.0 } else { (e.need_cpu - e.live_cpu).max(0.0) }; - (c + cpu, m + e.need_mem_kb.saturating_sub(e.live_mem_kb)) + (c + cpu, m + e.need_mem_kb.saturating_sub(e.live_mem_kb) + booked(e, running)) }) } +/// What `e` still books for the work it expects: each item whose work has +/// not started, and that no running job has taken over yet. A waiting job +/// books all of them, so it starts only with room for its work too. +pub fn booked(e: &Entry, running: &[Entry]) -> u64 { + let taken = |i: usize| { + e.started_at.is_some() && running.iter().any(|r| r.work_for.as_ref().is_some_and(|w| w.pid == e.pid && w.item == Some(i as u32))) + }; + e.expects.iter().enumerate().filter(|(i, x)| !x.started && !taken(*i)).map(|(_, x)| x.mem_kb).sum() +} + +/// The memory `job` adds when it starts: its need, plus what it books for +/// the work it expects, less what its long-lived job booked for it, which it +/// takes over. +pub fn need_with_booking(job: &Entry, running: &[Entry]) -> u64 { + let own = if job.started_at.is_none() { booked(job, &[]) } else { 0 }; + (job.need_mem_kb + own).saturating_sub(booking_taken_over(job, running)) +} + +/// What the long-lived job of `job` books for it now, which `job` takes over +/// when it starts. +fn booking_taken_over(job: &Entry, running: &[Entry]) -> u64 { + let Some(WorkFor { pid, item: Some(i), .. }) = &job.work_for else { return 0 }; + if job.started_at.is_some() { + return 0; + } + let Some(stack) = running.iter().find(|e| e.pid == *pid) else { return 0 }; + let taken = running.iter().any(|r| r.pid != job.pid && r.work_for.as_ref().is_some_and(|w| w.pid == *pid && w.item == Some(*i))); + stack.expects.get(*i as usize).filter(|x| !x.started && !taken).map_or(0, |x| x.mem_kb) +} + +/// Is `job` work for a long-lived job that does not run yet? It waits for +/// it: the work needs it up. +pub fn waits_for_stack(job: &Entry, running: &[Entry]) -> bool { + job.started_at.is_none() && job.work_for.as_ref().is_some_and(|w| !running.iter().any(|e| e.pid == w.pid)) +} + +/// The long-lived job in `me`'s checkout that expects it, among `running` +/// and `waiting`, by `matches` (does `me` match this declared work?). A +/// running one comes first, then the item whose work no other job has taken. +/// Returns the link and the long-lived job's priority. +pub fn link_work(me: &Entry, running: &[Entry], waiting: &[Entry], matches: impl Fn(&str) -> bool) -> Option<(WorkFor, i32)> { + if me.long_lived { + return None; + } + let claimed = |pid: i32, i: usize| { + running + .iter() + .chain(waiting) + .any(|r| r.pid != me.pid && r.work_for.as_ref().is_some_and(|w| w.pid == pid && w.item == Some(i as u32))) + }; + let stacks = running.iter().chain(waiting).filter(|e| e.long_lived && e.pid != me.pid && e.checkout == me.checkout); + let mut best: Option<(u8, WorkFor, i32)> = None; + for e in stacks { + for (i, x) in e.expects.iter().enumerate().filter(|(_, x)| matches(&x.work)) { + let free = !x.started && !claimed(e.pid, i); + // A stack that waits takes only work that can take over a booking. + if e.started_at.is_none() && !free { + continue; + } + let rank = u8::from(e.started_at.is_none()) * 2 + u8::from(!free); + if best.as_ref().is_none_or(|(r, _, _)| rank < *r) { + let item = free.then_some(i as u32); + best = Some((rank, WorkFor { pid: e.pid, key: e.key.clone(), item }, e.priority)); + } + } + } + best.map(|(_, w, p)| (w, p)) +} + /// Does `a` belong to an older task-runner run than `b`? Only jobs that are /// both in a run, in different runs, are ordered this way. pub fn runs_before(a: &Entry, b: &Entry) -> bool { @@ -840,7 +951,7 @@ pub fn decide( let mut blockers: Vec = room .blockers(m, lim, running, me, now, unknown_starts) .into_iter() - .filter(|b| matches!(b, Blocker::Memory { .. } | Blocker::Slots { .. })) + .filter(|b| matches!(b, Blocker::Memory { .. } | Blocker::Slots { .. } | Blocker::Stack { .. })) .collect(); if blockers.is_empty() { blockers.push(room.memory_blocker(m, lim, me)); @@ -1125,7 +1236,7 @@ pub fn signal_tree(root: i32, sig: i32) { } /// The room the running jobs leave, read once per decision. -#[derive(Clone, Copy)] +#[derive(Clone)] struct Room { res_cpu: f64, res_mem: u64, @@ -1140,6 +1251,9 @@ struct Room { /// reserve: the memory that frees up once they have ended. passing_live: u64, passing_res: u64, + /// The running jobs: what a job takes over from its long-lived job + /// depends on them. + running: Vec, } impl Room { @@ -1157,11 +1271,12 @@ impl Room { ours_mem, passing_live: passing.iter().map(|e| e.live_mem_kb).sum(), passing_res: reserve(&passing).1, + running: running.to_vec(), } } fn mem_would(&self, job: &Entry) -> f64 { - (self.mem_used + self.res_mem + job.need_mem_kb) as f64 + (self.mem_used + self.res_mem + need_with_booking(job, &self.running)) as f64 } /// The most memory a job may take the machine to by the progress rule: @@ -1179,7 +1294,7 @@ impl Room { (Some(pk), Some(max)) => (running.iter().filter(|e| e.pool_key.as_ref() == Some(pk)).count() as u32) < max, _ => true, }; - pool_free && self.mem_would(job) <= self.progress_ceiling(m, lim, job) + pool_free && !waits_for_stack(job, running) && self.mem_would(job) <= self.progress_ceiling(m, lim, job) } /// Would `job` fit by the progress rule once every running job that ends @@ -1190,7 +1305,7 @@ impl Room { let drained = Room { mem_used: self.mem_used.saturating_sub(self.passing_live), res_mem: self.res_mem.saturating_sub(self.passing_res), - ..*self + ..self.clone() }; drained.fits_on_progress(m, lim, &lasting, job) } @@ -1204,7 +1319,7 @@ impl Room { short_kb: (mem_would - self.mem_limit).max(0.0) as u64, used_kb: self.mem_used, reserve_kb: self.res_mem, - need_kb: job.need_mem_kb, + need_kb: need_with_booking(job, &self.running), total_kb: m.mem_total_kb, } } @@ -1262,7 +1377,7 @@ impl Room { return None; } let outside = self.mem_used.saturating_sub(self.ours_mem); - let ours = (self.ours_mem + self.res_mem + job.need_mem_kb) as f64; + let ours = (self.ours_mem + self.res_mem + need_with_booking(job, &self.running)) as f64; if short_kb as f64 <= total * lim.noise_mem_pct / 100.0 { why.push(format!( "memory short by {:.1} GB, within noise_mem {:.0}% of RAM", @@ -1314,6 +1429,10 @@ impl Room { /// of it in line. fn blockers(&self, m: &MachineSample, lim: &Limits, running: &[Entry], job: &Entry, now: f64, unknown_starts: &[f64]) -> Vec { let mut blockers = Vec::new(); + // Work for a long-lived job that does not run yet needs it up first. + if waits_for_stack(job, running) { + blockers.push(Blocker::Stack { key: job.work_for.as_ref().map(|w| w.key.clone()).unwrap_or_default() }); + } if let (Some(pk), Some(max)) = (&job.pool_key, job.pool_slots) { let holders: Vec = running.iter().filter(|e| e.pool_key.as_ref() == Some(pk)).map(|e| e.key.clone()).collect(); if holders.len() as u32 >= max { @@ -1506,6 +1625,18 @@ mod tests { assert_eq!(serde_json::from_str::(&newer).unwrap().ticket, 7); let older: Entry = serde_json::from_str(r#"{"ticket":3,"pid":10,"run_id":1,"key":"k"}"#).unwrap(); assert_eq!((older.ticket, older.need_cpu), (3, 0.0)); + assert!(!older.long_lived && older.expects.is_empty() && older.work_for.is_none()); + // What this version adds reads back, and leaves the rest alone. + let stack = Entry { + long_lived: true, + expects: vec![Expect { work: "browser".into(), mem_kb: 4, from: "x".into(), started: true }], + work_for: Some(WorkFor { pid: 9, key: "s".into(), item: Some(0) }), + ..older + }; + let back: Entry = serde_json::from_str(&serde_json::to_string(&stack).unwrap()).unwrap(); + assert_eq!(back, stack); + let nudge: Nudge = serde_json::from_str(r#"{"start":false,"work_started":[0]}"#).unwrap(); + assert_eq!(nudge.work_started, vec![0]); } #[test] @@ -2302,6 +2433,117 @@ mod tests { assert_eq!(blockers(decide(&m, &lim, &running, &waiting, &slow, 3300.0, &[])), vec!["reserved"]); } + /// A stack that expects a browser run of 4.3 GB, in checkout /w/a. + fn expecting(mut stack: Entry) -> Entry { + stack.checkout = "/w/a".into(); + stack.expects = vec![Expect { work: "browser".into(), mem_kb: 43 * GB / 10, ..Default::default() }]; + stack + } + + /// The browser run of checkout /w/a, linked to the stack of pid 11. + fn run_for(item: Option) -> Entry { + Entry { + pool: Some("browser".into()), + checkout: "/w/a".into(), + need_mem_kb: 43 * GB / 10, + priority: 11, + work_for: Some(WorkFor { pid: 11, key: "dev-env:stack".into(), item }), + ..job(3, "pr.ts --pr 17398", 3.0, 0) + } + } + + #[test] + fn a_stack_starts_only_with_room_for_the_work_it_expects() { + let lim = Limits { mem_max_pct: 95.0, ..LIM }; + // 18 GB held of 32 (limit 30.4): the stack alone (9.4 GB) fits, with + // its 4.3 GB run it does not. + let alone = Entry { long_lived: true, priority: 10, need_mem_kb: 94 * GB / 10, ..job(2, "dev-env:stack", 3.0, 0) }; + let stack = Entry { pid: 11, ..expecting(alone.clone()) }; + let m = machine(1.0, 18); + let busy = [Entry { started_at: Some(990.0), ..job(5, "lint", 1.0, 1) }]; + assert!(blockers(decide(&m, &lim, &busy, std::slice::from_ref(&alone), &alone, 1000.0, &[])).is_empty()); + assert_eq!(blockers(decide(&m, &lim, &busy, std::slice::from_ref(&stack), &stack, 1000.0, &[])), vec!["memory"]); + assert_eq!(need_with_booking(&stack, &busy), 94 * GB / 10 + 43 * GB / 10); + assert!(blockers(decide(&machine(1.0, 15), &lim, &busy, std::slice::from_ref(&stack), &stack, 1000.0, &[])).is_empty()); + } + + #[test] + fn a_running_stack_keeps_the_room_for_its_run_until_the_run_starts() { + let lim = Limits { mem_max_pct: 95.0, ..LIM }; + let stack = expecting(steady_stack(11, 7)); + let r = std::slice::from_ref(&stack); + // What the stack may still grow to (1 GB), and 4.3 GB booked for its run. + let grows = stack.need_mem_kb - stack.live_mem_kb; + assert_eq!(reserve(r).1, grows + 43 * GB / 10); + let lint = Entry { need_mem_kb: 46 * GB / 10, ..job(4, "lint", 3.5, 0) }; + let run = run_for(Some(0)); + let waiting = [lint.clone(), run.clone()]; + let m = machine(1.0, 24); + let other = [stack.clone(), Entry { started_at: Some(990.0), ..job(5, "build", 0.5, 0) }]; + // 24 GB held + 5.3 reserved: a lint that needs that room waits; the + // run takes over the booking, and goes first. + assert_eq!(blockers(decide(&m, &lim, &other, &waiting, &lint, 1000.0, &[])), vec!["priority", "memory"]); + assert!(blockers(decide(&m, &lim, &other, &waiting, &run, 1000.0, &[])).is_empty()); + let alone = [lint.clone()]; + assert_eq!(blockers(decide(&m, &lim, &other, &alone, &lint, 1000.0, &[])), vec!["memory"]); + // Once the run runs, the stack books nothing for it: the run's own + // need counts instead. + let running = [stack.clone(), Entry { started_at: Some(1000.0), ..run.clone() }]; + assert_eq!(reserve(&running).1, grows + 43 * GB / 10); + assert_eq!(super::booked(&stack, &running), 0); + // And once its owner has marked it started, it stays dropped. + let mut done = stack.clone(); + done.expects[0].started = true; + assert_eq!(reserve(std::slice::from_ref(&done)).1, grows); + } + + #[test] + fn work_links_to_the_stack_of_its_checkout_that_expects_it() { + let stack = Entry { pid: 11, ..expecting(steady_stack(11, 7)) }; + let me = Entry { pool: Some("browser".into()), checkout: "/w/a".into(), ..job(3, "pr.ts", 3.0, 4) }; + let is = |w: &str| w == "browser"; + let (link, priority) = link_work(&me, std::slice::from_ref(&stack), &[], is).unwrap(); + assert_eq!((link.pid, link.item, priority), (11, Some(0), 10)); + // Not from another checkout, and not work it does not expect. + let elsewhere = Entry { checkout: "/w/b".into(), ..me.clone() }; + assert!(link_work(&elsewhere, std::slice::from_ref(&stack), &[], is).is_none()); + assert!(link_work(&me, std::slice::from_ref(&stack), &[], |w| w == "lint").is_none()); + // A second run while the first one has the booking still goes first, + // without a booking to take over. + let first = Entry { pid: 30, started_at: Some(990.0), ..run_for(Some(0)) }; + let (link, _) = link_work(&me, &[stack.clone(), first], &[], is).unwrap(); + assert_eq!(link.item, None); + // A stack that still waits takes only work that can take over its booking. + let waiting = Entry { started_at: None, ..stack.clone() }; + let claimed = Entry { pid: 30, ..run_for(Some(0)) }; + assert!(link_work(&me, &[], &[waiting.clone(), claimed], is).is_none()); + assert_eq!(link_work(&me, &[], std::slice::from_ref(&waiting), is).unwrap().0.item, Some(0)); + // A long-lived job is nobody's work. + assert!(link_work(&Entry { long_lived: true, ..me }, std::slice::from_ref(&stack), &[], is).is_none()); + } + + #[test] + fn work_waits_for_its_stack_and_holds_nobody_back_meanwhile() { + let stack = Entry { pid: 11, long_lived: true, priority: 10, ..expecting(job(2, "dev-env:stack", 3.0, 9)) }; + let run = run_for(Some(0)); + let lint = job(4, "lint", 1.0, 1); + let waiting = [stack.clone(), run.clone(), lint.clone()]; + let busy = [Entry { started_at: Some(990.0), ..job(5, "build", 0.5, 2) }]; + let m = machine(1.0, 4); + assert_eq!(blockers(decide(&m, &LIM, &busy, &waiting, &run, 1000.0, &[])), vec!["stack"]); + // Its priority is above the stack's, but it cannot start: the lint + // that fits is held back only by the stack itself. + assert_eq!(blockers(decide(&m, &LIM, &busy, &waiting, &lint, 1000.0, &[])), vec!["priority"]); + let d = decide(&m, &LIM, &busy, &waiting, &lint, 1000.0, &[]); + assert!( + matches!(&d, Decision::Wait { blockers } if matches!(&blockers[0], Blocker::Priority { key, priority: 10 } if key == "dev-env:stack")) + ); + // With nothing running, the progress rule skips it too: the stack + // behind it in line starts. + assert_eq!(blockers(decide(&m, &LIM, &[], &waiting, &run, 1000.0, &[])), vec!["stack"]); + assert!(blockers(decide(&m, &LIM, &[], &waiting, &stack, 1000.0, &[])).is_empty()); + } + #[test] fn backfill_keeps_pools_and_pressure() { let (mut running, mut head) = blocked_head(); diff --git a/src/report.rs b/src/report.rs index 649a0b4..f4a4164 100644 --- a/src/report.rs +++ b/src/report.rs @@ -3,7 +3,7 @@ //! the scheduler really thinks. use crate::machine::MachineSample; -use crate::queue::{Blocker, Decision, Entry, Limits, reserve}; +use crate::queue::{Blocker, Decision, Entry, Limits, booked, need_with_booking, reserve}; use serde::Serialize; use std::fmt::Write as _; @@ -61,6 +61,7 @@ pub fn blocker_text(b: &Blocker) -> String { format!("memory pressure: the kernel reports {} ({level:.0}); nothing new starts until it eases", pressure_word(*level)) } Blocker::Paused { key } => format!("{key} is paused for memory; it resumes before anything new starts"), + Blocker::Stack { key } => format!("work for {key}, which has not started yet; it starts after it"), Blocker::Memory { would_pct, limit_pct, short_kb, .. } => { format!("memory: would reach {would_pct:.0}% (limit {limit_pct:.0}%), {} short", gb(*short_kb)) } @@ -167,6 +168,7 @@ fn unblock(b: &Blocker, running: &[Entry], now: f64) -> (String, Option) { Blocker::Pressure { paced: true, .. } => ("starts when memory fits".into(), None), Blocker::Pressure { .. } => ("starts when the memory pressure eases".into(), None), Blocker::Paused { key } => (format!("starts after {key} has resumed"), None), + Blocker::Stack { key } => (format!("starts after {key} has started"), None), Blocker::Memory { short_kb, .. } => { let need = |e: &Entry| e.need_mem_kb.max(e.live_mem_kb) as f64; match first_to_free(&need, *short_kb as f64) { @@ -486,14 +488,16 @@ pub fn rule_checks(s: &Snapshot, w: &Waiting) -> Vec<(bool, String)> { ), )); } - let mem_would = m.mem_used_kb + s.reserve_mem_kb + e.need_mem_kb; + let need = need_with_booking(e, &s.running); + let mem_would = m.mem_used_kb + s.reserve_mem_kb + need; out.push(( checked("memory", mem_would <= s.mem_limit_kb), format!( - "memory: {} held + {} promised + {} for this job = {:.0}% (limit {:.0}%)", + "memory: {} held + {} promised + {} for this job{} = {:.0}% (limit {:.0}%)", gb(m.mem_used_kb), gb(s.reserve_mem_kb), - gb(e.need_mem_kb), + gb(need), + link_text(e, &s.running), mem_would as f64 * 100.0 / m.mem_total_kb.max(1) as f64, s.limits.mem_max_pct ), @@ -532,6 +536,21 @@ pub fn bar(used: f64, over: f64, reserved: f64, total: f64, limit: f64, width: u .collect() } +/// What a job's link to a long-lived job adds to its line: the work a +/// long-lived job expects and books room for, or the long-lived job a job +/// is work for. Empty for any other job. +pub fn link_text(e: &Entry, running: &[Entry]) -> String { + let mut t = String::new(); + if let Some(w) = &e.work_for { + t += &format!(" (for stack {})", w.key); + } + let open: Vec<&str> = e.expects.iter().filter(|x| !x.started).map(|x| x.work.as_str()).collect(); + if !open.is_empty() { + t += &format!(" (expects {}, books {})", open.join(", "), gb(booked(e, running))); + } + t +} + /// The job's key cut to `n` characters, with its priority when it has one: "api:tsc p2". pub fn job_name(e: &Entry, n: usize) -> String { if e.priority == 0 { @@ -597,14 +616,15 @@ pub fn render_status(s: &Snapshot) -> String { let min = if e.raised_by_min { " min" } else { "" }; let _ = writeln!( o, - " {:<36} {:<12} {:>6} {:>4.1} / {:<9} {:>8} / {}{min}{tag}", + " {:<36} {:<12} {:>6} {:>4.1} / {:<9} {:>8} / {}{min}{tag}{}", job_name(e, 36), trunc(e.pool.as_deref().unwrap_or("-"), 12), dur(s.now - e.started_at.unwrap_or(s.now)), e.live_cpu, needs_cpu, gb(e.live_mem_kb), - needs_mem + needs_mem, + link_text(e, &s.running) ); } let _ = writeln!(o); @@ -632,6 +652,7 @@ pub fn render_status(s: &Snapshot) -> String { if e.stalled { blocked += " [stalled: holds no job back]"; } + blocked += &link_text(e, &s.running); let _ = writeln!( o, " {:<36} {:<12} {:>6} {:<21} {}", diff --git a/src/runner.rs b/src/runner.rs index af7cba8..b2ef3f9 100644 --- a/src/runner.rs +++ b/src/runner.rs @@ -45,9 +45,36 @@ pub struct Opts { pub wait: bool, /// Higher goes first; the default comes from the config, else 0. pub priority: Option, + /// For a long-lived job: the work that will run on it (`--expect`). + pub expect: Vec, pub cmd: Vec, } +/// Work a long-lived job's caller says will run on it: `--expect`, then +/// `TASKGUARD_EXPECT` (comma-separated). Any other job inherits the variable +/// from whoever started it and leaves it alone. +fn declared_work(o: &Opts) -> Vec { + let env = std::env::var("TASKGUARD_EXPECT").unwrap_or_default(); + o.expect.iter().cloned().chain(env.split(',').map(str::trim).filter(|w| !w.is_empty()).map(str::to_string)).collect() +} + +/// Is this job the work `work` names: its pool, its key, or its command? +fn is_work(work: &str, me: &Entry, cmd: &[String]) -> bool { + me.pool.as_deref() == Some(work) || me.key == work || matcher::matches(work, cmd) +} + +/// Mark the expected work a nudge says has started. +fn apply_work_started(me: &mut Entry, n: &queue::Nudge) -> bool { + let mut changed = false; + for i in &n.work_started { + if let Some(x) = me.expects.get_mut(*i as usize).filter(|x| !x.started) { + x.started = true; + changed = true; + } + } + changed +} + /// sem's `-j`: N, +N (cores + N), -N (cores - N) or N% of cores, at least 1. pub fn parse_jobs(s: &str, ncpu: usize) -> Result { let n = ncpu as f64; @@ -83,6 +110,30 @@ fn apply_nudge(me: &mut Entry, n: &queue::Nudge, database: Option<&Db>) { } } +/// Link a waiting job to the long-lived job in its checkout that expects it, +/// or drop the link when that job is gone. Work for a long-lived job goes +/// one priority above it, so it starts before the jobs that queued for room +/// meanwhile, and before a next stack. +fn link_to_stack(me: &mut Entry, own_priority: i32, running: &[Entry], waiting: &[Entry], cmd: &[String], quiet: bool) { + let gone = me.work_for.as_ref().is_some_and(|w| !running.iter().chain(waiting).any(|e| e.pid == w.pid)); + if gone { + me.work_for = None; + me.priority = own_priority; + } + if me.work_for.is_some() { + return; + } + let probe = me.clone(); + if let Some((link, priority)) = queue::link_work(&probe, running, waiting, |w| is_work(w, &probe, cmd)) { + if !quiet { + let booked = if link.item.is_some() { "; it takes over the room booked for it" } else { "" }; + say(&format!("for stack {} - {} is work it expects, and goes first{booked}", link.key, me.key)); + } + me.priority = own_priority.max(priority + 1); + me.work_for = Some(link); + } +} + fn cleanup_files(q: &Queue, e: &Entry) { let _ = std::fs::remove_file(q.wait_path(e)); let _ = std::fs::remove_file(q.run_path(e.pid)); @@ -318,6 +369,16 @@ pub fn run(mut o: Opts) -> Result { } let (need_cpu, need_mem, raised) = (need.cpu, need.mem_kb, need.raised_by_min); let need_why = serde_json::to_string(&need).ok(); + // A long-lived job books room for the work its caller says will run on + // it, learned as that work would learn it. + let work = declared_work(&o); + if startup.is_none() && !o.expect.is_empty() && !o.quiet { + say(&format!("warning {jkey} - --expect is for a long-lived job (long_lived = true); ignored")); + } + let expects: Vec = match startup { + Some(_) => work.iter().map(|w| need::expected(database.as_ref(), &cfg, &cwd, w)).collect(), + None => Vec::new(), + }; let estimate_from = guess.as_ref().map(|g| g.from.clone()); let est_dur = guess.as_ref().and_then(|g| g.dur_s); @@ -389,6 +450,7 @@ pub fn run(mut o: Opts) -> Result { est_dur_s: learned.dur_s.or(est_dur).filter(|_| startup.is_none()), priority: o.priority.or(cls.priority).unwrap_or(cfg.priority), long_lived: startup.is_some(), + expects, pausable: true, version: Some(env!("CARGO_PKG_VERSION").to_string()), ..Default::default() @@ -403,6 +465,14 @@ pub fn run(mut o: Opts) -> Result { let conf_mtime = |p: &Path| std::fs::metadata(p).and_then(|m| m.modified()).ok(); let mut seen_mtime = conf_mtime(&conf_path); + if !o.quiet { + for x in &me.expects { + say(&format!("expects {} - {} books {} for it until it starts ({})", x.work, me.key, report::gb(x.mem_kb), x.from)); + } + } + // The priority without the lift that work for a long-lived job gets. + let own_priority = me.priority; + // ---- wait for room let t0 = db::now(); let mut main_blocker: Option = None; @@ -491,12 +561,29 @@ pub fn run(mut o: Opts) -> Result { } if let Some(n) = q.take_nudge(me_pid) { apply_nudge(&mut me, &n, database.as_ref()); + apply_work_started(&mut me, &n); started_by_hand |= n.start; } + link_to_stack(&mut me, own_priority, &running, &waiting, &o.cmd, o.quiet); + // Work that queued for this long-lived job may need more than + // was booked for it: book that. + for (i, x) in me.expects.iter_mut().enumerate() { + let theirs = waiting + .iter() + .filter(|w| w.work_for.as_ref().is_some_and(|l| l.pid == me_pid && l.item == Some(i as u32))) + .max_by_key(|w| w.need_mem_kb); + if let Some(w) = theirs.filter(|w| w.need_mem_kb > x.mem_kb) { + x.mem_kb = w.need_mem_kb; + x.from = format!("the need of {}, which queued for it", w.key); + } + } let mut waiting = waiting; if let Some(w) = waiting.iter_mut().find(|w| w.pid == me_pid) { w.need_cpu = me.need_cpu; w.need_mem_kb = me.need_mem_kb; + w.priority = me.priority; + w.work_for = me.work_for.clone(); + w.expects = me.expects.clone(); } let unknown_starts = q.unknown_starts(); decision = queue::decide(&m, &limits, &running, &waiting, &me, now, &unknown_starts); @@ -538,6 +625,10 @@ pub fn run(mut o: Opts) -> Result { me.blocker = None; q.write(&q.run_path(me_pid), &me)?; let _ = std::fs::remove_file(q.wait_path(&me)); + // The long-lived job this is work for drops what it booked for it. + if let Some(queue::WorkFor { pid, item: Some(i), .. }) = &me.work_for { + let _ = q.nudge(*pid, |n| n.work_started.push(*i)); + } q.touch_activity(); } else { let text = report::main_blocker(&decision).map(report::blocker_text); @@ -726,6 +817,9 @@ pub fn run(mut o: Opts) -> Result { } if let Some(n) = q.take_nudge(me_pid) { apply_nudge(&mut me, &n, database.as_ref()); + if apply_work_started(&mut me, &n) { + let _ = q.write(&q.run_path(me_pid), &me); + } match (n.pause, me.paused_since.is_some()) { (Some(true), false) => pause_cmd(&q, &mut me, now, true), // An auto pause becomes a pause by hand: it stays until a resume by hand. @@ -1079,6 +1173,9 @@ fn run_bg(o: &Opts) -> Result { if let Some(v) = o.priority { args.extend(["--priority".into(), v.to_string()]); } + for w in &o.expect { + args.extend(["--expect".into(), w.clone()]); + } if o.now { args.push("--now".into()); } diff --git a/src/stall.rs b/src/stall.rs index 3825a08..1732c3d 100644 --- a/src/stall.rs +++ b/src/stall.rs @@ -10,7 +10,7 @@ //! to the rules shows what it does to it. use crate::machine::{MachineSample, PEAK_WINDOW}; -use crate::queue::{Decision, Entry, Limits, LongLived, decide}; +use crate::queue::{Decision, Entry, Expect, Limits, LongLived, decide, link_work}; const GB: u64 = 1024 * 1024; const TICK: f64 = 1.0; @@ -125,6 +125,11 @@ pub struct Agent { pub give_up_s: f64, pub down_after_s: f64, pub pause_s: f64, + /// The agent declares its browser run when it brings the stack up + /// (`TASKGUARD_EXPECT=browser`), and the stack books `booking_kb` for it: + /// what browser runs typically took, less than the run's own need. + pub declares: bool, + pub booking_kb: u64, } pub fn agent(first_at: f64, idle_gb: f64) -> Agent { @@ -141,6 +146,8 @@ pub fn agent(first_at: f64, idle_gb: f64) -> Agent { give_up_s: 600.0, down_after_s: 20.0, pause_s: 30.0, + declares: false, + booking_kb: 36 * GB / 10, } } @@ -173,6 +180,9 @@ pub struct Scenario { pub lints: Option, pub jobs: Vec, pub secs: f64, + /// The scout's fallback, for comparison: a steady stack books what it + /// holds, without room to grow. + pub steady_books_live: bool, } /// What came of it. @@ -276,6 +286,11 @@ pub fn run(s: &Scenario) -> Outcome { known: true, priority: 10, long_lived: true, + expects: if a.declares { + vec![Expect { work: "browser".into(), mem_kb: a.booking_kb, ..Default::default() }] + } else { + Vec::new() + }, ..Default::default() }; waiting.push((e, Kind::Stack(n))); @@ -388,6 +403,9 @@ pub fn run(s: &Scenario) -> Outcome { _ => None, }; r.e.follow(mem, cpu, s.host.ncpu as f64, t, lived); + if s.steady_books_live && r.e.steady() { + r.e.need_mem_kb = mem; + } } let ours: u64 = running.iter().map(|r| r.e.live_mem_kb).sum(); let used = s.host.outside(t) + ours; @@ -422,8 +440,25 @@ pub fn run(s: &Scenario) -> Outcome { while i < waiting.len() { let run_now: Vec = running.iter().map(|r| r.e.clone()).collect(); let all: Vec = waiting.iter().map(|(e, _)| e.clone()).collect(); + // As the runner does: work links to the stack that expects it, + // and goes one priority above it. + if waiting[i].0.work_for.is_none() { + let me = waiting[i].0.clone(); + if let Some((link, p)) = link_work(&me, &run_now, &all, |w| me.pool.as_deref() == Some(w)) { + waiting[i].0.priority = me.priority.max(p + 1); + waiting[i].0.work_for = Some(link); + } + } + let all: Vec = waiting.iter().map(|(e, _)| e.clone()).collect(); let (me, kind) = waiting[i].clone(); if matches!(decide(&m, &s.lim, &run_now, &all, &me, t, &[]), Decision::Admit { .. }) { + // The nudge to the stack: its work has started. + if let Some(w) = &me.work_for + && let Some(i) = w.item + && let Some(st) = running.iter_mut().find(|r| r.e.pid == w.pid) + { + st.e.expects[i as usize].started = true; + } for (w, _) in waiting.iter_mut().filter(|(w, _)| w.ahead_of(&me) && w.bypassed_since.is_none()) { w.bypassed_since = Some(t); } @@ -459,7 +494,17 @@ pub fn morning(outside_gb: f64) -> Scenario { lints: Some(macbook_lints()), jobs: Vec::new(), secs: 7200.0, + steady_books_live: false, + } +} + +/// The same morning, with each agent declaring its browser run on its stack. +pub fn declared_morning(outside_gb: f64) -> Scenario { + let mut s = morning(outside_gb); + for a in &mut s.agents { + a.declares = true; } + s } /// Omarchy (24 cores, 31.3 GB, Linux, mem_max 95, DALP's max_backfill 1800): @@ -521,7 +566,7 @@ pub fn head_of_line() -> Scenario { let host = Host { total_kb: gb(31.3), ncpu: 24, outside_kb: gb(15.8), outside_swing_kb: gb(0.3), warning_pct: None, critical_pct: 100.0 }; let lim = Limits { warn_admit: false, ..macbook_limits() }; - Scenario { host, lim, agents: Vec::new(), lints: None, jobs, secs: 1800.0 } + Scenario { host, lim, agents: Vec::new(), lints: None, jobs, secs: 1800.0, steady_books_live: false } } /// The longest a small job (under 1 GB) of `s` waited, or the scenario's @@ -578,6 +623,11 @@ mod tests { // 0.10.0 also capped outside_admit at pause_at, under mem_max. let old = Scenario { lim: Limits { outside_mem_max_pct: 92.0, ..macbook_limits() }, ..morning(outside) }; eprintln!("{}", row(&format!("outside {outside} GB, outside cap 92%"), &run(&old))); + eprintln!("{}", row(&format!("outside {outside} GB, declared"), &run(&declared_morning(outside)))); + let live = Scenario { steady_books_live: true, ..morning(outside) }; + eprintln!("{}", row(&format!("outside {outside} GB, steady books live"), &run(&live))); + let both = Scenario { steady_books_live: true, ..declared_morning(outside) }; + eprintln!("{}", row(&format!("outside {outside} GB, declared + live"), &run(&both))); } } diff --git a/tests/cli.rs b/tests/cli.rs index 8a7b065..b239db1 100644 --- a/tests/cli.rs +++ b/tests/cli.rs @@ -566,6 +566,43 @@ fn a_long_lived_job_holds_its_peak_only_through_its_start_up() { assert!(!explain.contains("long-lived"), "only the stack pool is: {explain}"); } +#[test] +fn a_stack_books_the_work_it_expects_and_that_work_goes_first() { + let e = Env::new( + "sample_every = 0.2\nnew_job_mem = \"700M\"\n[pool.stack]\nlong_lived = true\nstartup = 1\n[pool.browser]\nunlimited = true\n", + ); + // The caller of the stack says a browser run will follow (DALP passes the + // variable through `devenv up`). + let mut stack = e.cmd(&["--id", "stack", "--priority", "10", "--", "sh", "-c", "touch up; sleep 30"]); + stack.env("TASKGUARD_EXPECT", "browser"); + let mut stack = stack.stdout(Stdio::piped()).stderr(Stdio::piped()).spawn().unwrap(); + wait_for(&e.file("up")); + let r = running_job(&e).unwrap(); + assert_eq!(r["expects"][0]["work"], "browser", "{r}"); + assert_eq!(r["expects"][0]["mem_kb"], 700 * 1024, "no browser run is known yet: new_job_mem: {r}"); + let status = String::from_utf8_lossy(&e.run(&["status"]).stdout).into_owned(); + assert!(status.contains("(expects browser, books 700 MB)"), "{status}"); + + // The run queues in the same checkout: it is the work the stack expects, + // and goes one priority above it. + let run = e.spawn(&["--id", "browser", "--", "sh", "-c", "touch ran; sleep 1"]); + wait_for(&e.file("ran")); + let out = e.run(&["status", "--json"]); + let v: serde_json::Value = serde_json::from_slice(&out.stdout).unwrap(); + let r = v["running"].as_array().unwrap().iter().find(|r| r["pool"] == "browser").cloned().unwrap(); + assert_eq!((r["priority"].as_i64(), r["work_for"]["item"].as_i64()), (Some(11), Some(0)), "{r}"); + let run = run.wait_with_output().unwrap(); + assert!(run.status.success() && stderr(&run).contains("for stack "), "{}", stderr(&run)); + // Once it has started, the stack books nothing for it any more. + wait_until("the stack drops its booking", || running_job(&e).is_some_and(|r| r["expects"][0]["started"] == true)); + + // Any other job ignores the variable; --expect on it says so. + let other = e.run(&["--expect", "browser", "--", "true"]); + assert!(stderr(&other).contains("--expect is for a long-lived job"), "{}", stderr(&other)); + unsafe { libc::kill(stack.id() as i32, libc::SIGTERM) }; + let _ = stack.wait(); +} + #[test] fn enabled_false_runs_every_command_straight_through() { // A one-slot pool would queue the second job; switched off, nothing queues From 0cb5a39ea2d612c5d459a6836ad98c2b1a10bb5c Mon Sep 17 00:00:00 2001 From: Vincent De Snerck <10668+SpoBo@users.noreply.github.com> Date: Fri, 16 Oct 2026 12:16:19 +0200 Subject: [PATCH 06/22] feat: a stack waits a few seconds for the work it expects, and work queued for it goes first The captain could not tell whether a stack and the run that needs it arrive together. When they do, the stack should start with room for both. A long-lived job that has reason to expect work now waits up to expect_wait (5 s, 0 = off, at most 60) before it starts, for that work to queue, and starts as soon as it has. The reason is a declaration (--expect), or the job history for that checkout: at least two of the stack's last five runs there, and on at least half of them, a job of the same pool (or key, outside any pool) queued there while it ran. Work the history implies books nothing and only links while the stack waits; once it queues, the stack books its need. A stack with neither reason starts at once. A stack with work queued for it, and work whose stack runs, now hold their turn as a strict reservation instead of by backfill, as long as they could fit once the jobs that end by themselves have ended. Without that, a stream of lints kept taking the room a stack and its run needed together until the caller gave up (17 of 23 runs at 14.5 GB outside in the model). Entry gains expect_until, Expect gains implied (serde defaults). New setting expect_wait. The database gains no column: the history comes from runs. --- CHANGELOG.md | 9 +++ README.md | 28 ++++++++- src/commands.rs | 4 +- src/config.rs | 6 ++ src/db.rs | 71 +++++++++++++++++++++ src/queue.rs | 109 +++++++++++++++++++++++++++++++- src/report.rs | 8 ++- src/runner.rs | 30 ++++++++- src/stall.rs | 137 ++++++++++++++++++++++++++++++++--------- taskguard.example.toml | 4 ++ tests/cli.rs | 22 ++++++- 11 files changed, 386 insertions(+), 42 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 20d9316..998553b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -30,6 +30,12 @@ and the versions follow [semver](https://semver.org). waits while its stack still waits. `taskguard status` shows `(for stack KEY)` on the work and `(expects WORK, books SIZE)` on the stack. Without a declaration nothing changes. +- `expect_wait` (5 seconds by default, 0 turns it off): a long-lived job that + declares work, or whose history in its checkout says work follows it (on at + least half of its last runs there, two at least), waits that long before it + starts for the work to queue, so a stack and a run started together start + with room for both. Work the history implies books nothing until it + queues. A stack with neither starts at once. ### Changed @@ -43,6 +49,9 @@ and the versions follow [semver](https://semver.org). has ended holds nobody back, whatever its priority or reservation: it can only start once a long-lived job is stopped or other programs free memory, and the small jobs behind it start meanwhile. +- A stack with work queued for it, and work whose stack runs, hold their turn + as a strict reservation, also while they do not fit yet, rather than letting + jobs that fit start meanwhile (backfill). ### Fixed diff --git a/README.md b/README.md index 56503ba..0ddf142 100644 --- a/README.md +++ b/README.md @@ -247,8 +247,23 @@ below). it, which the stack then drops. `taskguard status` shows both ends: `(for stack dev-env:stack)` on the work, `(expects browser, books 3.6 GB)` on the stack. Work that queues while its stack still waits waits for the stack and - holds nobody back meanwhile. A stack started without `--expect` behaves as - before; the variable means nothing to any other job. + holds nobody back meanwhile. A stack with work queued for it, and work whose + stack runs, hold their turn as a strict reservation, also while they do not + fit yet: lints that fit do not take the room meanwhile, unless the stack + could only start once another stack is stopped. A stack started without + `--expect` behaves as before; the variable means nothing to any other job. +- **A stack waits a few seconds for its work.** A caller may start the stack + and its work at the same moment, not one after the other. So a long-lived + job that has reason to expect work waits up to `expect_wait` (5) seconds, + before it starts, for that work to queue, and starts as soon as it has: + with room for both, and the work right after it. The reason is either a + declaration (`--expect`), or its history in that checkout: at least two of + its last five runs there, and on at least half of them, a job of the same + pool (or with the same key, outside any pool) queued in that checkout while + it ran. Work the history implies books nothing, and only counts when it + queues within those seconds; then the stack books what it needs. A stack + with neither reason starts at once, as before: sometimes you only want the + stack. `expect_wait = 0` turns the wait off. ### What is measured @@ -580,6 +595,7 @@ outlier_ratio = 1.5 # a peak this many times the next one is an outlier; 0 = outside_admit = true # start jobs that only other programs keep out partial_fit = 0.5 # a job that cannot fit starts on half its CPU need; 0 = off max_backfill = 600 # small jobs pass a stuck job this long, then the machine drains +expect_wait = 5 # a dev stack waits this long for the work it expects to queue [pool.e2e] # a slot ceiling for one kind of job, per worktree max_slots = 1 @@ -640,6 +656,9 @@ repo. Put a repo's own scripts in its `.taskguard.toml`: however long that takes: work that never comes keeps that room booked while the stack runs. A stack plus its work that do not fit next to another stack wait for room, where before the stack started and its work waited. + Work only links to a stack in its own checkout. +- The history only suggests that work follows a stack; it never books room + by itself. Only work that queues within `expect_wait` of the stack counts. ## Moving from tsc-queue @@ -703,7 +722,10 @@ nothing runs but long-lived jobs past their start-up, also a second dev stack over `mem_max`, and skip that rule at macOS warning; their jobs decide so whatever newer versions do. They do not know `--expect` either: they ignore `TASKGUARD_EXPECT`, do not see what a newer stack books for its work, and may -start their own jobs into that room. The priority a newer job takes as work +start their own jobs into that room. Nor do they know `expect_wait`: they see +a stack that waits a few seconds for its work as one that could start, and +mark it as stalled only after 10 seconds, so keep `expect_wait` under 10 +while such versions share the machine. The priority a newer job takes as work for a stack is an ordinary priority, so their waiting jobs do let it go first. A version before 0.4.0 does not diff --git a/src/commands.rs b/src/commands.rs index 4a24f55..f8bd0d6 100644 --- a/src/commands.rs +++ b/src/commands.rs @@ -483,8 +483,8 @@ pub fn doctor(args: &[String]) -> Result { println!("namespace here: {}", key::namespace(&cwd)); println!(); println!( - "limits: cpu_max {:.0}% mem_max {:.0}% learn_stagger {}s max_bypass {}s max_backfill {}s partial_fit {} hints {}", - cfg.cpu_max, cfg.mem_max, cfg.learn_stagger, cfg.max_bypass, cfg.max_backfill, cfg.partial_fit, cfg.hints + "limits: cpu_max {:.0}% mem_max {:.0}% learn_stagger {}s max_bypass {}s max_backfill {}s partial_fit {} expect_wait {}s hints {}", + cfg.cpu_max, cfg.mem_max, cfg.learn_stagger, cfg.max_bypass, cfg.max_backfill, cfg.partial_fit, cfg.expect_wait, cfg.hints ); for (k, v) in &cfg.origin { if v != "built-in" { diff --git a/src/config.rs b/src/config.rs index 42fb266..350eabe 100644 --- a/src/config.rs +++ b/src/config.rs @@ -40,6 +40,7 @@ pub struct Layer { pub new_job_mem: Option, pub max_bypass: Option, pub max_backfill: Option, + pub expect_wait: Option, pub boost_runs: Option, pub outlier_ratio: Option, pub outlier_min: Option, @@ -139,6 +140,9 @@ pub struct Config { pub new_job_mem_kb: u64, pub max_bypass: u64, pub max_backfill: u64, + /// Seconds a long-lived job that expects work waits, before it starts, + /// for that work to queue: see `queue::Entry::expect_until`. 0 = never. + pub expect_wait: f64, pub boost_runs: usize, /// A memory peak more than this many times the next one below it is an /// outlier, and counts by `outlier_weights`. At or under 1: no outliers. @@ -199,6 +203,7 @@ impl Default for Config { new_job_mem_kb: 1536 * 1024, max_bypass: 120, max_backfill: 600, + expect_wait: 5.0, boost_runs: 5, outlier_ratio: 1.5, outlier_min_kb: 1024 * 1024, @@ -633,6 +638,7 @@ impl Config { } set!(max_bypass); set!(max_backfill); + set!(expect_wait); set!(boost_runs); set!(outlier_ratio); if let Some(v) = &l.outlier_min diff --git a/src/db.rs b/src/db.rs index aa93fb4..d18fb4e 100644 --- a/src/db.rs +++ b/src/db.rs @@ -1042,6 +1042,43 @@ impl Db { Ok(None) } + /// The work that followed a long-lived job before: for the last few + /// runs of `key` in `checkout` (at most `n`, of the last two weeks), the + /// pool, or the key of a job without one, of every other job that queued + /// in that checkout while the run ran. Returns how many runs were looked + /// at, and what followed with on how many of them, most first. + pub fn followers(&self, key: &str, pool: Option<&str>, checkout: &str, n: usize) -> Result<(usize, Vec<(String, usize)>)> { + let now = now(); + let mut stmt = self.conn.prepare_cached( + "SELECT id, started_at, coalesce(ended_at, ?3) FROM runs + WHERE key = ?1 AND started_at IS NOT NULL AND started_at > ?3 - 14 * 86400 + AND (cwd = ?2 OR substr(cwd, 1, length(?2) + 1) = ?2 || '/') + ORDER BY started_at DESC LIMIT ?4", + )?; + let runs: Vec<(i64, f64, f64)> = stmt + .query_map(params![key, checkout, now, n as i64], |r| Ok((r.get(0)?, r.get(1)?, r.get(2)?)))? + .collect::>()?; + let mut seen: Vec<(String, usize)> = Vec::new(); + let mut stmt = self.conn.prepare_cached( + "SELECT DISTINCT coalesce(pool, key) FROM runs + WHERE id != ?1 AND key != ?2 AND (pool IS NULL OR ?5 IS NULL OR pool != ?5) + AND queued_at >= ?3 AND queued_at <= ?4 + AND (cwd = ?6 OR substr(cwd, 1, length(?6) + 1) = ?6 || '/')", + )?; + for (id, from, to) in &runs { + let works: Vec = + stmt.query_map(params![id, key, from, to, pool, checkout], |r| r.get(0))?.collect::>()?; + for w in works { + match seen.iter_mut().find(|(s, _)| *s == w) { + Some((_, c)) => *c += 1, + None => seen.push((w, 1)), + } + } + } + seen.sort_by(|a, b| b.1.cmp(&a.1).then(a.0.cmp(&b.0))); + Ok((runs.len(), seen)) + } + /// How many of the key's most recent runs in a row were starved. pub fn starved_streak(&self, key: &str) -> Result { let mut stmt = @@ -1859,6 +1896,40 @@ mod tests { assert_eq!((l.steady_mem_kb, l.steady_cpu), (None, None)); } + #[test] + fn the_work_that_followed_a_stack_in_its_checkout() { + let db = Db::open(Path::new(":memory:")).unwrap(); + let t = now() - 86400.0; + let at = |key: &str, pool: Option<&str>, cwd: &str, queued: f64, started: f64, ended: Option| { + let id = db.insert_run(&NewRun { ns: "n", key, pool, cwd, ..Default::default() }).unwrap(); + db.conn + .execute( + "UPDATE runs SET queued_at = ?2, started_at = ?3, ended_at = ?4 WHERE id = ?1", + params![id, queued, started, ended], + ) + .unwrap(); + }; + // Three stacks in /w/dalp; a browser run followed two of them, and a + // lint each. The third still runs. + for (i, run) in [(0, true), (1, true), (2, false)] { + let s = t + 3600.0 * i as f64; + at("dev-env:stack", Some("stack"), "/w/dalp", s - 10.0, s, (i < 2).then_some(s + 1800.0)); + if run { + at("e2e/agentic:pr.ts", Some("browser"), "/w/dalp/e2e/agentic", s + 400.0, s + 400.0, Some(s + 700.0)); + } + at(&format!("pkg{i}:oxlint"), None, "/w/dalp/pkg", s + 60.0, s + 60.0, Some(s + 70.0)); + } + // Nothing of another checkout counts, nor whatever ran before a stack. + at("e2e/agentic:pr.ts", Some("browser"), "/w/dalp-2/e2e/agentic", t + 7300.0, t + 7300.0, Some(t + 7400.0)); + at("root:tsc", None, "/w/dalp", t - 600.0, t - 600.0, Some(t - 500.0)); + let (n, works) = db.followers("dev-env:stack", Some("stack"), "/w/dalp", 5).unwrap(); + assert_eq!(n, 3); + assert_eq!(works[0], ("browser".to_string(), 2)); + assert!(works[1..].iter().all(|(w, c)| w.ends_with(":oxlint") && *c == 1), "{works:?}"); + let (n, works) = db.followers("dev-env:stack", Some("stack"), "/w/elsewhere", 5).unwrap(); + assert_eq!((n, works.len()), (0, 0)); + } + #[test] fn opens_a_database_from_before_long_lived_jobs() { let path = std::env::temp_dir().join(format!("tg-old-steady-{}.db", std::process::id())); diff --git a/src/queue.rs b/src/queue.rs index cb1593c..ee639a9 100644 --- a/src/queue.rs +++ b/src/queue.rs @@ -117,6 +117,10 @@ pub struct Entry { /// The long-lived job this job is work for: one in the same checkout /// that expects it. Older versions leave it out. pub work_for: Option, + /// For a long-lived job that expects work: until then it waits, before + /// it starts, for that work to queue (`expect_wait`), so it starts with + /// room for both. Older versions leave it out. + pub expect_until: Option, /// Another job saw this one able to start for `STALL_S` while it still /// waited (`stall/.`). It holds no job back any more. Read /// from the queue, never written into the entry. @@ -137,6 +141,9 @@ pub struct Expect { pub from: String, /// A job for it has started: the booking is gone. pub started: bool, + /// Not declared: the history says such work followed this job before. + /// It books nothing until a job for it queues while this job waits. + pub implied: bool, } /// The long-lived job a job is work for, and which of its `expects` it @@ -700,6 +707,12 @@ pub enum Blocker { Stack { key: String, }, + /// A long-lived job waits, for at most `wait_s` more, for the work it + /// expects to queue. + Expect { + work: Vec, + wait_s: f64, + }, Memory { would_pct: f64, limit_pct: f64, @@ -732,6 +745,7 @@ impl Blocker { Blocker::Pressure { .. } => "pressure", Blocker::Paused { .. } => "paused", Blocker::Stack { .. } => "stack", + Blocker::Expect { .. } => "expect", Blocker::Cpu { .. } => "cpu", } } @@ -827,8 +841,9 @@ pub fn link_work(me: &Entry, running: &[Entry], waiting: &[Entry], matches: impl for e in stacks { for (i, x) in e.expects.iter().enumerate().filter(|(_, x)| matches(&x.work)) { let free = !x.started && !claimed(e.pid, i); - // A stack that waits takes only work that can take over a booking. - if e.started_at.is_none() && !free { + // A stack that waits takes only work that can take over a booking; + // work the history implies only while the stack still waits. + if (e.started_at.is_none() && !free) || (x.implied && e.started_at.is_some()) { continue; } let rank = u8::from(e.started_at.is_none()) * 2 + u8::from(!free); @@ -921,6 +936,17 @@ pub fn decide( return Decision::Wait { blockers: vec![Blocker::Paused { key: p.key.clone() }] }; } + // A long-lived job that expects work waits a few seconds for it to + // queue, unless it already has: its caller may start the work right + // after it, and then it starts with room for both, and the work goes + // first, instead of the stack starting alone. + if let Some(until) = me.expect_until.filter(|u| *u > now && me.started_at.is_none()) + && !waiting.iter().any(|w| w.work_for.as_ref().is_some_and(|l| l.pid == me.pid)) + { + let work = me.expects.iter().map(|x| x.work.clone()).collect(); + return Decision::Wait { blockers: vec![Blocker::Expect { work, wait_s: until - now }] }; + } + // Always make progress: with nothing running, the first job in line that // fits starts, whatever the CPU reading says and up to all of RAM. This is // what keeps the queue from deadlocking on a machine that is busy with @@ -973,7 +999,17 @@ pub fn decide( // keep it out: it starts once the running jobs have made room for it, or // when none runs, which they do not put off. A job that may start on part // of its need (`partial_fit`) waits only for that part. + // + // Work queued for a long-lived job is what that job exists for: the + // long-lived job it waits for, and that work once its long-lived job + // runs, hold their turn as a strict reservation does, also while they do + // not fit yet. Otherwise a stream of lints keeps taking the room a stack + // and its run need together, until the caller gives up on the run. + let queued_for = |w: &Entry| w.work_for.is_some() || waiting.iter().any(|x| x.work_for.as_ref().is_some_and(|l| l.pid == w.pid)); let holds = |w: &Entry| { + if queued_for(w) { + return true; + } let (mut theirs, _) = room.left(m, lim, running, w, now, unknown_starts); if theirs.is_empty() { return true; @@ -1629,7 +1665,8 @@ mod tests { // What this version adds reads back, and leaves the rest alone. let stack = Entry { long_lived: true, - expects: vec![Expect { work: "browser".into(), mem_kb: 4, from: "x".into(), started: true }], + expects: vec![Expect { work: "browser".into(), mem_kb: 4, from: "x".into(), started: true, implied: true }], + expect_until: Some(5.0), work_for: Some(WorkFor { pid: 9, key: "s".into(), item: Some(0) }), ..older }; @@ -2544,6 +2581,72 @@ mod tests { assert!(blockers(decide(&m, &LIM, &[], &waiting, &stack, 1000.0, &[])).is_empty()); } + #[test] + fn a_stack_that_expects_work_waits_a_few_seconds_for_it_to_queue() { + let stack = Entry { + pid: 11, + long_lived: true, + priority: 10, + queued_at: 1000.0, + expect_until: Some(1005.0), + ..expecting(job(2, "dev-env:stack", 3.0, 9)) + }; + let m = machine(1.0, 4); + let alone = std::slice::from_ref(&stack); + let d = decide(&m, &LIM, &[], alone, &stack, 1002.0, &[]); + assert!( + matches!(&d, Decision::Wait { blockers } if matches!(&blockers[0], Blocker::Expect { wait_s, .. } if *wait_s == 3.0)), + "{d:?}" + ); + // The run queued: the stack starts now, with room for both, and the + // run waits for it. + let run = run_for(Some(0)); + let both = [stack.clone(), run.clone()]; + assert!(blockers(decide(&m, &LIM, &[], &both, &stack, 1002.0, &[])).is_empty()); + assert_eq!(blockers(decide(&m, &LIM, &[], &both, &run, 1002.0, &[])), vec!["stack"]); + // Nothing queued: after the wait it starts alone. + assert!(blockers(decide(&m, &LIM, &[], alone, &stack, 1005.0, &[])).is_empty()); + // A stack that expects nothing never waits. + let plain = Entry { expects: Vec::new(), expect_until: None, ..stack.clone() }; + assert!(blockers(decide(&m, &LIM, &[], std::slice::from_ref(&plain), &plain, 1000.0, &[])).is_empty()); + } + + #[test] + fn a_stack_with_work_queued_for_it_holds_its_turn_also_before_it_fits() { + // DALP's max_backfill: a job ahead holds its turn only while it could + // start, for half an hour. + let lim = Limits { max_backfill: 1800.0, ..LIM }; + let stack = Entry { pid: 11, long_lived: true, priority: 10, ..expecting(job(2, "dev-env:stack", 3.0, 9)) }; + let lint = job(4, "lint", 1.0, 1); + let busy = [Entry { started_at: Some(990.0), est_dur_s: Some(20.0), ..job(5, "build", 0.5, 12) }]; + // 4 GB held + 12 reserved + 13.3 for the stack and its run: it waits. + let m = machine(1.0, 4); + // Alone, it lets the lint that fits start meanwhile, as before. + let waiting = [stack.clone(), lint.clone()]; + assert!(blockers(decide(&m, &lim, &busy, &waiting, &lint, 1000.0, &[])).is_empty()); + // With its run queued for it, the lint waits: the room is theirs. + let waiting = [stack.clone(), run_for(Some(0)), lint.clone()]; + assert_eq!(blockers(decide(&m, &lim, &busy, &waiting, &lint, 1000.0, &[])), vec!["priority"]); + // Not when it could only start once a stack is stopped: 14 GB held + // by other programs and 13.3 GB more is over the 27.2 GB limit. + let m = machine(1.0, 14); + assert!(blockers(decide(&m, &lim, &busy, &waiting, &lint, 1000.0, &[])).is_empty()); + } + + #[test] + fn work_the_history_implies_links_only_while_the_stack_waits() { + let mut stack = Entry { pid: 11, ..expecting(steady_stack(11, 7)) }; + stack.expects[0] = Expect { work: "browser".into(), implied: true, ..Default::default() }; + let me = Entry { pool: Some("browser".into()), checkout: "/w/a".into(), ..job(3, "pr.ts", 3.0, 4) }; + let is = |w: &str| w == "browser"; + // It books nothing. + assert_eq!(reserve(std::slice::from_ref(&stack)).1, stack.need_mem_kb - stack.live_mem_kb); + // Up already: a run that queues now is no evidence of anything. + assert!(link_work(&me, std::slice::from_ref(&stack), &[], is).is_none()); + let waiting = Entry { started_at: None, ..stack }; + assert_eq!(link_work(&me, &[], std::slice::from_ref(&waiting), is).unwrap().0.item, Some(0)); + } + #[test] fn backfill_keeps_pools_and_pressure() { let (mut running, mut head) = blocked_head(); diff --git a/src/report.rs b/src/report.rs index f4a4164..51ea577 100644 --- a/src/report.rs +++ b/src/report.rs @@ -62,6 +62,11 @@ pub fn blocker_text(b: &Blocker) -> String { } Blocker::Paused { key } => format!("{key} is paused for memory; it resumes before anything new starts"), Blocker::Stack { key } => format!("work for {key}, which has not started yet; it starts after it"), + Blocker::Expect { work, wait_s } => format!( + "waits up to {:.0}s more for the work it expects ({}) to queue, so it starts with room for both", + wait_s.ceil(), + work.join(", ") + ), Blocker::Memory { would_pct, limit_pct, short_kb, .. } => { format!("memory: would reach {would_pct:.0}% (limit {limit_pct:.0}%), {} short", gb(*short_kb)) } @@ -169,6 +174,7 @@ fn unblock(b: &Blocker, running: &[Entry], now: f64) -> (String, Option) { Blocker::Pressure { .. } => ("starts when the memory pressure eases".into(), None), Blocker::Paused { key } => (format!("starts after {key} has resumed"), None), Blocker::Stack { key } => (format!("starts after {key} has started"), None), + Blocker::Expect { wait_s, .. } => (format!("decides in at most {:.0}s", wait_s.ceil()), Some(*wait_s)), Blocker::Memory { short_kb, .. } => { let need = |e: &Entry| e.need_mem_kb.max(e.live_mem_kb) as f64; match first_to_free(&need, *short_kb as f64) { @@ -544,7 +550,7 @@ pub fn link_text(e: &Entry, running: &[Entry]) -> String { if let Some(w) = &e.work_for { t += &format!(" (for stack {})", w.key); } - let open: Vec<&str> = e.expects.iter().filter(|x| !x.started).map(|x| x.work.as_str()).collect(); + let open: Vec<&str> = e.expects.iter().filter(|x| !x.started && (!x.implied || x.mem_kb > 0)).map(|x| x.work.as_str()).collect(); if !open.is_empty() { t += &format!(" (expects {}, books {})", open.join(", "), gb(booked(e, running))); } diff --git a/src/runner.rs b/src/runner.rs index b2ef3f9..3762e3b 100644 --- a/src/runner.rs +++ b/src/runner.rs @@ -375,10 +375,26 @@ pub fn run(mut o: Opts) -> Result { if startup.is_none() && !o.expect.is_empty() && !o.quiet { say(&format!("warning {jkey} - --expect is for a long-lived job (long_lived = true); ignored")); } - let expects: Vec = match startup { + let mut expects: Vec = match startup { Some(_) => work.iter().map(|w| need::expected(database.as_ref(), &cfg, &cwd, w)).collect(), None => Vec::new(), }; + // Without a declaration, the history may still say that work follows + // this job in this checkout: what queued there while at least half of + // its last runs (two at least) ran. That books nothing; it only makes + // the job wait `expect_wait` for such work to queue with it. + let expect_wait = cfg.expect_wait.clamp(0.0, 60.0); + if startup.is_some() + && expects.is_empty() + && expect_wait > 0.0 + && let Some((n, works)) = database.as_ref().and_then(|d| d.followers(&jkey, pool.as_deref(), &checkout.to_string_lossy(), 5).ok()) + && n >= 2 + { + for (w, c) in works.into_iter().filter(|(_, c)| c * 2 >= n) { + let from = format!("not declared; it queued while {c} of the last {n} runs of {jkey} in this checkout ran"); + expects.push(queue::Expect { work: w, from, implied: true, ..Default::default() }); + } + } let estimate_from = guess.as_ref().map(|g| g.from.clone()); let est_dur = guess.as_ref().and_then(|g| g.dur_s); @@ -465,9 +481,19 @@ pub fn run(mut o: Opts) -> Result { let conf_mtime = |p: &Path| std::fs::metadata(p).and_then(|m| m.modified()).ok(); let mut seen_mtime = conf_mtime(&conf_path); + // It waits a few seconds for that work to queue, so it starts with room + // for both; a job with nothing to expect starts as before. + if !me.expects.is_empty() && expect_wait > 0.0 { + me.expect_until = Some(me.queued_at + expect_wait); + } if !o.quiet { for x in &me.expects { - say(&format!("expects {} - {} books {} for it until it starts ({})", x.work, me.key, report::gb(x.mem_kb), x.from)); + let wait = me.expect_until.map(|_| format!("; it waits up to {expect_wait:.0}s for it to queue")).unwrap_or_default(); + if x.implied { + say(&format!("expects {} - {}: it books room only once that work queues{wait}", x.work, x.from)); + } else { + say(&format!("expects {} - {} books {} for it until it starts ({}){wait}", x.work, me.key, report::gb(x.mem_kb), x.from)); + } } } // The priority without the lift that work for a long-lived job gets. diff --git a/src/stall.rs b/src/stall.rs index 1732c3d..239f7ca 100644 --- a/src/stall.rs +++ b/src/stall.rs @@ -130,6 +130,10 @@ pub struct Agent { /// what browser runs typically took, less than the run's own need. pub declares: bool, pub booking_kb: u64, + /// The agent queues its run 2 s after its stack, not once the stack is up. + pub with_stack: bool, + /// Undeclared, but the history says a browser run follows its stacks. + pub implied: bool, } pub fn agent(first_at: f64, idle_gb: f64) -> Agent { @@ -148,6 +152,8 @@ pub fn agent(first_at: f64, idle_gb: f64) -> Agent { pause_s: 30.0, declares: false, booking_kb: 36 * GB / 10, + with_stack: false, + implied: false, } } @@ -183,6 +189,8 @@ pub struct Scenario { /// The scout's fallback, for comparison: a steady stack books what it /// holds, without room to grow. pub steady_books_live: bool, + /// `expect_wait`. + pub expect_wait: f64, } /// What came of it. @@ -200,6 +208,8 @@ pub struct Outcome { pub over_limit_s: f64, pub lints_done: usize, pub worst_stack_wait: f64, + /// Runs that started while their stack did not run: they fail. + pub runs_without_stack: usize, /// When each of `Scenario::jobs` started, if it did. pub job_started: Vec>, } @@ -212,12 +222,24 @@ struct Live { #[derive(Clone, Copy, PartialEq, Debug)] enum Phase { - Idle { until: f64 }, - StackQueued, + Idle { + until: f64, + }, + /// The stack waits or starts; the run queues at `run_at` when the agent + /// queues it together with the stack. + StackQueued { + run_at: Option, + }, StackUp, - RunQueued { since: f64 }, - RunRunning { until: f64 }, - Down { at: f64 }, + RunQueued { + since: f64, + }, + RunRunning { + until: f64, + }, + Down { + at: f64, + }, } pub fn run(s: &Scenario) -> Outcome { @@ -274,6 +296,13 @@ pub fn run(s: &Scenario) -> Outcome { match phase[n] { Phase::Idle { until } if t >= until => { ticket += 1; + let expects = if a.declares { + vec![Expect { work: "browser".into(), mem_kb: a.booking_kb, ..Default::default() }] + } else if a.implied { + vec![Expect { work: "browser".into(), implied: true, ..Default::default() }] + } else { + Vec::new() + }; let e = Entry { ticket, pid: stack_pid, @@ -286,18 +315,19 @@ pub fn run(s: &Scenario) -> Outcome { known: true, priority: 10, long_lived: true, - expects: if a.declares { - vec![Expect { work: "browser".into(), mem_kb: a.booking_kb, ..Default::default() }] - } else { - Vec::new() - }, + expect_until: (!expects.is_empty() && s.expect_wait > 0.0).then_some(t + s.expect_wait), + expects, ..Default::default() }; waiting.push((e, Kind::Stack(n))); - phase[n] = Phase::StackQueued; + phase[n] = Phase::StackQueued { run_at: a.with_stack.then_some(t + 2.0) }; } - Phase::StackQueued => { - if let Some(r) = running.iter().find(|r| r.kind == Kind::Stack(n)) + Phase::StackQueued { run_at } => { + if run_at.is_some_and(|at| t >= at) { + ticket += 1; + waiting.push((run_entry(n, a, ticket, t), Kind::Run(n))); + phase[n] = Phase::RunQueued { since: t }; + } else if let Some(r) = running.iter().find(|r| r.kind == Kind::Stack(n)) && t - r.started >= a.stack_ramp_s { phase[n] = Phase::StackUp; @@ -305,21 +335,7 @@ pub fn run(s: &Scenario) -> Outcome { } Phase::StackUp => { ticket += 1; - let e = Entry { - ticket, - pid: 2000 + n as i32, - key: format!("e2e/agentic:bun_src/pr.ts_--pr_{}", 17397 + n), - pool: Some("browser".into()), - checkout: checkout(n), - queued_at: t, - need_cpu: a.run_cpu, - need_mem_kb: a.run_need_kb, - known: true, - raised_by_min: true, - est_dur_s: Some(a.run_s), - ..Default::default() - }; - waiting.push((e, Kind::Run(n))); + waiting.push((run_entry(n, a, ticket, t), Kind::Run(n))); phase[n] = Phase::RunQueued { since: t }; } Phase::RunQueued { since } => { @@ -342,6 +358,7 @@ pub fn run(s: &Scenario) -> Outcome { } Phase::Down { at } if t >= at => { running.retain(|r| r.kind != Kind::Stack(n)); + waiting.retain(|(_, k)| *k != Kind::Stack(n)); phase[n] = Phase::Idle { until: t + a.pause_s }; } _ => {} @@ -449,9 +466,25 @@ pub fn run(s: &Scenario) -> Outcome { waiting[i].0.work_for = Some(link); } } + // As the runner does: a waiting stack books at least what the + // work that queued for it needs. + if waiting[i].0.long_lived { + let pid = waiting[i].0.pid; + for x in 0..waiting[i].0.expects.len() { + let theirs = all + .iter() + .filter(|w| w.work_for.as_ref().is_some_and(|l| l.pid == pid && l.item == Some(x as u32))) + .map(|w| w.need_mem_kb) + .max() + .unwrap_or(0); + let item = &mut waiting[i].0.expects[x]; + item.mem_kb = item.mem_kb.max(theirs); + } + } let all: Vec = waiting.iter().map(|(e, _)| e.clone()).collect(); let (me, kind) = waiting[i].clone(); - if matches!(decide(&m, &s.lim, &run_now, &all, &me, t, &[]), Decision::Admit { .. }) { + let d = decide(&m, &s.lim, &run_now, &all, &me, t, &[]); + if matches!(d, Decision::Admit { .. }) { // The nudge to the stack: its work has started. if let Some(w) = &me.work_for && let Some(i) = w.item @@ -468,6 +501,11 @@ pub fn run(s: &Scenario) -> Outcome { if let Kind::Job(n) = kind { out.job_started[n] = Some(t); } + if let Kind::Run(n) = kind + && !running.iter().any(|r| r.kind == Kind::Stack(n)) + { + out.runs_without_stack += 1; + } let e = Entry { started_at: Some(t), start_need_cpu: Some(me.need_cpu), start_need_mem_kb: Some(me.need_mem_kb), ..me }; running.push(Live { e, kind, started: t }); waiting.remove(i); @@ -480,6 +518,24 @@ pub fn run(s: &Scenario) -> Outcome { out } +/// The browser run of agent `n`. +fn run_entry(n: usize, a: &Agent, ticket: u64, t: f64) -> Entry { + Entry { + ticket, + pid: 2000 + n as i32, + key: format!("e2e/agentic:bun_src/pr.ts_--pr_{}", 17397 + n), + pool: Some("browser".into()), + checkout: format!("/w/dalp/{n}"), + queued_at: t, + need_cpu: a.run_cpu, + need_mem_kb: a.run_need_kb, + known: true, + raised_by_min: true, + est_dur_s: Some(a.run_s), + ..Default::default() + } +} + /// Does agent `n`'s run run now? fn running_kind_any(phase: &[Phase], n: usize) -> bool { matches!(phase[n], Phase::RunRunning { .. }) @@ -495,6 +551,7 @@ pub fn morning(outside_gb: f64) -> Scenario { jobs: Vec::new(), secs: 7200.0, steady_books_live: false, + expect_wait: 5.0, } } @@ -566,7 +623,7 @@ pub fn head_of_line() -> Scenario { let host = Host { total_kb: gb(31.3), ncpu: 24, outside_kb: gb(15.8), outside_swing_kb: gb(0.3), warning_pct: None, critical_pct: 100.0 }; let lim = Limits { warn_admit: false, ..macbook_limits() }; - Scenario { host, lim, agents: Vec::new(), lints: None, jobs, secs: 1800.0, steady_books_live: false } + Scenario { host, lim, agents: Vec::new(), lints: None, jobs, secs: 1800.0, steady_books_live: false, expect_wait: 5.0 } } /// The longest a small job (under 1 GB) of `s` waited, or the scenario's @@ -631,6 +688,26 @@ mod tests { } } + #[test] + fn a_stack_and_its_run_queued_together() { + eprintln!("{}", header()); + for outside in [11.0, 12.8, 14.5] { + for (name, declares, implied, wait) in [ + ("nothing", false, false, 5.0), + ("history implies", false, true, 5.0), + ("declared", true, false, 5.0), + ("declared, no wait", true, false, 0.0), + ] { + let mut s = Scenario { expect_wait: wait, ..morning(outside) }; + for a in &mut s.agents { + (a.with_stack, a.declares, a.implied) = (true, declares, implied); + } + let o = run(&s); + eprintln!("{} {:>3} without stack", row(&format!("together {outside} GB, {name}"), &o), o.runs_without_stack); + } + } + } + #[test] fn a_large_job_at_the_front_on_omarchy() { let s = head_of_line(); diff --git a/taskguard.example.toml b/taskguard.example.toml index aad5cdd..c520d2e 100644 --- a/taskguard.example.toml +++ b/taskguard.example.toml @@ -71,6 +71,10 @@ max_backfill = 600 # until a reserved job has waited this many seconds, newer # before it could start anyway, so the machine drains for it. # 0 (or anything up to max_bypass) = never. # taskguard 0.1.6 and older reject a file that sets it. +expect_wait = 5 # seconds a long-lived job (a dev stack) waits before it starts for the + # work it expects to queue, so it starts with room for both. Only when + # it declares work (--expect, TASKGUARD_EXPECT) or its history says work + # follows it in that checkout. 0 = never; at most 60. boost_runs = 5 # runs that get +25% memory after a memory-starved run # A peak more than outlier_ratio times the next peak below it, and outlier_min # above it, is an outlier. Outliers count by outlier_weights: the first entry diff --git a/tests/cli.rs b/tests/cli.rs index b239db1..6645f35 100644 --- a/tests/cli.rs +++ b/tests/cli.rs @@ -569,7 +569,7 @@ fn a_long_lived_job_holds_its_peak_only_through_its_start_up() { #[test] fn a_stack_books_the_work_it_expects_and_that_work_goes_first() { let e = Env::new( - "sample_every = 0.2\nnew_job_mem = \"700M\"\n[pool.stack]\nlong_lived = true\nstartup = 1\n[pool.browser]\nunlimited = true\n", + "sample_every = 0.2\nnew_job_mem = \"700M\"\nexpect_wait = 0.5\n[pool.stack]\nlong_lived = true\nstartup = 1\n[pool.browser]\nunlimited = true\n", ); // The caller of the stack says a browser run will follow (DALP passes the // variable through `devenv up`). @@ -603,6 +603,26 @@ fn a_stack_books_the_work_it_expects_and_that_work_goes_first() { let _ = stack.wait(); } +#[test] +fn a_stack_waits_for_the_work_it_expects_when_both_queue_together() { + let e = + Env::new("sample_every = 0.2\nexpect_wait = 20\n[pool.stack]\nlong_lived = true\nstartup = 1\n[pool.browser]\nunlimited = true\n"); + let mut stack = e.cmd(&["--id", "stack", "--expect", "browser", "--", "sh", "-c", "touch up; sleep 30"]); + let mut stack = stack.stdout(Stdio::piped()).stderr(Stdio::piped()).spawn().unwrap(); + let t = Instant::now(); + // The run queues a moment later: the stack, which waits for it, starts + // at once, and the run after it. + std::thread::sleep(Duration::from_millis(300)); + let run = e.run(&["--id", "browser", "--", "true"]); + assert!(run.status.success(), "{}", stderr(&run)); + assert!(t.elapsed() < Duration::from_secs(15), "the stack did not wait out expect_wait: {:?}", t.elapsed()); + assert!(stderr(&run).contains("for stack "), "{}", stderr(&run)); + let started = |pool: &str| -> f64 { e.db().query_row("SELECT started_at FROM runs WHERE pool = ?1", [pool], |r| r.get(0)).unwrap() }; + assert!(started("browser") >= started("stack"), "the run starts only once its stack runs"); + unsafe { libc::kill(stack.id() as i32, libc::SIGTERM) }; + let _ = stack.wait(); +} + #[test] fn enabled_false_runs_every_command_straight_through() { // A one-slot pool would queue the second job; switched off, nothing queues From 7defbe5c666d9b35bdf2a11cf03fdb21e2a34ba8 Mon Sep 17 00:00:00 2001 From: Vincent De Snerck <10668+SpoBo@users.noreply.github.com> Date: Fri, 16 Oct 2026 12:17:51 +0200 Subject: [PATCH 07/22] test: the stall model asserts the scenarios that clear The MacBook morning: a caller that declares its run never gives it up, at 11, 12.8 and 14.5 GB held outside taskguard, and memory stays under the limit; without a declaration the progress rule alone clears 11 and 12.8 GB. A stack and its run queued together: with a declaration or the history, no run is given up and none starts before its stack. Omarchy: the second stack stays out and the small jobs behind it start within minutes. The scout's fallback (a steady stack books what it holds, without room to grow) stays a printed comparison only: it gives up 5 runs at 12.8 GB. --- src/stall.rs | 49 ++++++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 46 insertions(+), 3 deletions(-) diff --git a/src/stall.rs b/src/stall.rs index 239f7ca..cf28641 100644 --- a/src/stall.rs +++ b/src/stall.rs @@ -6,8 +6,19 @@ //! admitted while the first one's run waited, the runs never fit next to //! both stacks, and each agent gave up on its run after its 10-minute command //! timeout and took its stack down. This model replays that morning second -//! by second through the real `decide()` and `Entry::follow()`, so a change -//! to the rules shows what it does to it. +//! by second through the real `decide()`, `Entry::follow()` and +//! `link_work()`, so a change to the rules shows what it does to it: +//! +//! - `the_macbook_morning`: the agents queue the run once the stack is up, +//! with and without declaring it on the stack, at 11, 12.8 and 14.5 GB +//! held by programs outside taskguard. +//! - `a_stack_and_its_run_queued_together`: the agents queue the run two +//! seconds after the stack, with no declaration, a declaration, or only +//! the history to go on. +//! - `a_large_job_at_the_front_on_omarchy`: a second stack that does not fit +//! at the front of 25 jobs that do. +//! +//! `cargo test stall -- --nocapture --test-threads 1` prints the tables. use crate::machine::{MachineSample, PEAK_WINDOW}; use crate::queue::{Decision, Entry, Expect, Limits, LongLived, decide, link_work}; @@ -680,11 +691,26 @@ mod tests { // 0.10.0 also capped outside_admit at pause_at, under mem_max. let old = Scenario { lim: Limits { outside_mem_max_pct: 92.0, ..macbook_limits() }, ..morning(outside) }; eprintln!("{}", row(&format!("outside {outside} GB, outside cap 92%"), &run(&old))); - eprintln!("{}", row(&format!("outside {outside} GB, declared"), &run(&declared_morning(outside)))); + let declared = run(&declared_morning(outside)); + eprintln!("{}", row(&format!("outside {outside} GB, declared"), &declared)); + // The scout's fallback, not taken: a steady stack books what it + // holds. It gives runs up at 12.8 GB. let live = Scenario { steady_books_live: true, ..morning(outside) }; eprintln!("{}", row(&format!("outside {outside} GB, steady books live"), &run(&live))); let both = Scenario { steady_books_live: true, ..declared_morning(outside) }; eprintln!("{}", row(&format!("outside {outside} GB, declared + live"), &run(&both))); + + // A caller that declares its run never gives it up, and memory + // stays under the limit. + assert_eq!(declared.runs_given_up, 0, "{outside} GB declared: {declared:?}"); + assert!(declared.runs_done >= 10, "{outside} GB declared: {declared:?}"); + assert!(declared.worst_mem_pct <= 95.0 && declared.over_limit_s == 0.0, "{outside} GB declared: {declared:?}"); + // A caller that does not declare: the progress rule alone clears + // the morning's load, never a second stack over the limit. + if outside < 14.0 { + assert_eq!(o.runs_given_up, 0, "{outside} GB: {o:?}"); + } + assert!(o.worst_mem_pct <= 95.0, "{outside} GB: {o:?}"); } } @@ -704,6 +730,16 @@ mod tests { } let o = run(&s); eprintln!("{} {:>3} without stack", row(&format!("together {outside} GB, {name}"), &o), o.runs_without_stack); + if declares || implied { + // The run waits for its stack, and the two fit together. + assert_eq!((o.runs_given_up, o.runs_without_stack), (0, 0), "{outside} GB, {name}: {o:?}"); + assert!(o.worst_mem_pct <= 95.0, "{outside} GB, {name}: {o:?}"); + } + if implied { + // A run the history implied waits at most a few seconds for + // its stack after the stack has started. + assert!(o.stack_idle_s <= 60.0 || outside < 12.0, "{outside} GB, {name}: {o:?}"); + } } } } @@ -719,5 +755,12 @@ mod tests { worst_small_wait(&s, &o), o.worst_mem_pct ); + // 0.10.0 started the second stack at once, to 114% of RAM, and small + // jobs waited 1000 s. It does not fit next to the first: it waits, + // and holds nobody back. + assert_eq!(o.job_started[1], None); + assert!(o.worst_mem_pct < 100.0, "{o:?}"); + assert!(jobs_started(&s, &o) >= 24, "{o:?}"); + assert!(worst_small_wait(&s, &o) < 300.0, "{o:?}"); } } From 3a77071328da330fc686ec8038315c1a4e2b4613 Mon Sep 17 00:00:00 2001 From: Vincent De Snerck <10668+SpoBo@users.noreply.github.com> Date: Fri, 16 Oct 2026 12:38:54 +0200 Subject: [PATCH 08/22] feat: without a declaration, a stack books the work its history says follows it, as a bounded guess The captain: "would be cool if we could grab something from the context to determine that there will indeed be work incoming for the dev stack. And ideally we know what work so we know the full scope of the load on the machine." A long-lived job with no declared work now looks at its last infer_work_of (5) runs in its checkout. Work of one kind (a pool, or the key of a job outside any pool) that queued there within infer_work_within (900) seconds of their start on at least infer_work_min (3) of them is expected again: the stack books its usual need (the median need it queued with), status shows it as "from history", and the work links and goes first as declared work. This replaces the looser history rule that only drove expect_wait. Bounds, so a wrong guess cannot hurt: - the booking ends twice the usual delay after the stack started, ten minutes at most, and once work for the stack starts; - the rule for nothing running ignores a guess, so the queue never stops for one, and a job waits for it at most up to all of RAM; - a stack whose caller declared no work ignores guesses, its own included: it starts as it would without the history; - a declaration wins, --expect none expects nothing, infer_work = false turns it off; - once the guessed work queues, its room is booked as declared work. In the model the undeclared MacBook morning now clears at 14.5 GB too (no run given up), and a bad guess (no run ever follows) starves nothing: no lint or stack waits more than a minute longer than without the guess, though about a quarter fewer lints run at 14.5 GB. Data formats, additive: Expect gains expire_after_s and expired. New settings infer_work, infer_work_min, infer_work_of, infer_work_within. No database change: the history query reads runs as they are. --- CHANGELOG.md | 16 +++-- README.md | 57 +++++++++++----- src/commands.rs | 11 ++++ src/config.rs | 20 ++++++ src/db.rs | 105 +++++++++++++++++++---------- src/help.rs | 3 +- src/queue.rs | 106 ++++++++++++++++++++++++------ src/report.rs | 7 +- src/runner.rs | 146 ++++++++++++++++++++++++++++++++++++----- src/stall.rs | 93 +++++++++++++++++++++++++- taskguard.example.toml | 10 +++ tests/cli.rs | 5 +- 12 files changed, 478 insertions(+), 101 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 998553b..aba031c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -30,12 +30,20 @@ and the versions follow [semver](https://semver.org). waits while its stack still waits. `taskguard status` shows `(for stack KEY)` on the work and `(expects WORK, books SIZE)` on the stack. Without a declaration nothing changes. +- A long-lived job with no declared work books the work its history says + follows it: work of one kind (a pool, or a key outside any pool) that + queued in its checkout within `infer_work_within` (900) seconds of its + start on at least `infer_work_min` (3) of its last `infer_work_of` (5) + runs there. It books that work's usual need, shown as `(expects WORK from + history, ...)`, for twice its usual delay and ten minutes at most, and + drops it once work for the stack starts. The guess never stops the rule + for nothing running and never keeps out a stack started with no declared + work. A declaration wins; `--expect none` expects nothing; + `infer_work = false` turns it off. - `expect_wait` (5 seconds by default, 0 turns it off): a long-lived job that - declares work, or whose history in its checkout says work follows it (on at - least half of its last runs there, two at least), waits that long before it + expects work, declared or from its history, waits that long before it starts for the work to queue, so a stack and a run started together start - with room for both. Work the history implies books nothing until it - queues. A stack with neither starts at once. + with room for both. A stack that expects nothing starts at once. ### Changed diff --git a/README.md b/README.md index 0ddf142..3ae97f1 100644 --- a/README.md +++ b/README.md @@ -252,18 +252,34 @@ below). fit yet: lints that fit do not take the room meanwhile, unless the stack could only start once another stack is stopped. A stack started without `--expect` behaves as before; the variable means nothing to any other job. +- **Without a declaration, the history says what work follows a stack.** + When at least `infer_work_min` (3) of a stack's last `infer_work_of` (5) + runs in its checkout had work of one kind queue there within + `infer_work_within` (900) seconds of their start, that work is expected + again. A kind of work is a pool, or the key of a job outside any pool. The + stack books that work's usual need, which `taskguard status` shows as + `(expects browser from history, books 4.3 GB)`, and the work links and + goes first as declared work does. It is a guess, so it is bounded: + - it books room for twice the work's usual delay after the stack started, + ten minutes at most, and drops it once work for the stack starts or the + stack stops; + - no job waits for it by the rule for nothing running, so it can never + stop the queue, and a job waits for it at most up to all of RAM; + - it never keeps out a stack whose caller declared no work, its own guess + included: such a stack starts as it would without the history; + - a declaration always wins: a stack with `--expect` books only what was + declared, and `--expect none` expects nothing at all; + - `infer_work = false` turns it off. + + Once the work has queued, its room is no guess any more and is booked as + declared work is. - **A stack waits a few seconds for its work.** A caller may start the stack and its work at the same moment, not one after the other. So a long-lived - job that has reason to expect work waits up to `expect_wait` (5) seconds, - before it starts, for that work to queue, and starts as soon as it has: - with room for both, and the work right after it. The reason is either a - declaration (`--expect`), or its history in that checkout: at least two of - its last five runs there, and on at least half of them, a job of the same - pool (or with the same key, outside any pool) queued in that checkout while - it ran. Work the history implies books nothing, and only counts when it - queues within those seconds; then the stack books what it needs. A stack - with neither reason starts at once, as before: sometimes you only want the - stack. `expect_wait = 0` turns the wait off. + job that expects work, declared or from its history, waits up to + `expect_wait` (5) seconds, before it starts, for that work to queue, and + starts as soon as it has: with room for both, and the work right after it. + A stack that expects nothing starts at once, as before: sometimes you only + want the stack. `expect_wait = 0` turns the wait off. ### What is measured @@ -355,7 +371,7 @@ taskguard --wait [--id NAME] | `--min-cpu N` | Never start with fewer than N free cores. | | `--min-mem SIZE` | Never start with less than SIZE free (`6G`, `512M`). | | `--priority N` | Higher starts first. The default is 0, or `priority` from the config. | -| `--expect WORK` | For a long-lived job: work that will run on it (a pool name, job key or command pattern). It books room for it, and that work goes first. Repeatable; also `TASKGUARD_EXPECT=WORK,WORK`. | +| `--expect WORK` | For a long-lived job: work that will run on it (a pool name, job key or command pattern). It books room for it, and that work goes first. Repeatable; also `TASKGUARD_EXPECT=WORK,WORK`. `none`: no work, not even what its history suggests. | | `--now` | Skip the queue, but still measure and learn. | | `--bg` | Wait for room, then return and let the job run on. | | `--pipe` | With `--bg`: pass stdin to the command. | @@ -596,6 +612,7 @@ outside_admit = true # start jobs that only other programs keep out partial_fit = 0.5 # a job that cannot fit starts on half its CPU need; 0 = off max_backfill = 600 # small jobs pass a stuck job this long, then the machine drains expect_wait = 5 # a dev stack waits this long for the work it expects to queue +infer_work = true # a dev stack books the work its history says follows it [pool.e2e] # a slot ceiling for one kind of job, per worktree max_slots = 1 @@ -657,8 +674,11 @@ repo. Put a repo's own scripts in its `.taskguard.toml`: while the stack runs. A stack plus its work that do not fit next to another stack wait for room, where before the stack started and its work waited. Work only links to a stack in its own checkout. -- The history only suggests that work follows a stack; it never books room - by itself. Only work that queues within `expect_wait` of the stack counts. +- A guess of the history that does not come true books room for up to ten + minutes per stack: while it does, jobs that do not fit next to it wait, + unless nothing but long-lived jobs runs. In the model, when no run follows + any stack, about a quarter fewer lints ran at 14.5 GB held by other + programs, and none waited more than a minute longer. ## Moving from tsc-queue @@ -721,11 +741,12 @@ Versions up to 0.10.0 start the first in line whatever the readings say when nothing runs but long-lived jobs past their start-up, also a second dev stack over `mem_max`, and skip that rule at macOS warning; their jobs decide so whatever newer versions do. They do not know `--expect` either: they ignore -`TASKGUARD_EXPECT`, do not see what a newer stack books for its work, and may -start their own jobs into that room. Nor do they know `expect_wait`: they see -a stack that waits a few seconds for its work as one that could start, and -mark it as stalled only after 10 seconds, so keep `expect_wait` under 10 -while such versions share the machine. The priority a newer job takes as work +`TASKGUARD_EXPECT`, do not see what a newer stack books for its work, +declared or from its history, and may start their own jobs into that room. +Nor do they know `expect_wait`: they see a stack that waits a few seconds for +its work as one that could start, and mark it as stalled only after 10 +seconds, so keep `expect_wait` under 10 while such versions share the +machine. The priority a newer job takes as work for a stack is an ordinary priority, so their waiting jobs do let it go first. A version before 0.4.0 does not diff --git a/src/commands.rs b/src/commands.rs index f8bd0d6..9fae078 100644 --- a/src/commands.rs +++ b/src/commands.rs @@ -486,6 +486,17 @@ pub fn doctor(args: &[String]) -> Result { "limits: cpu_max {:.0}% mem_max {:.0}% learn_stagger {}s max_bypass {}s max_backfill {}s partial_fit {} expect_wait {}s hints {}", cfg.cpu_max, cfg.mem_max, cfg.learn_stagger, cfg.max_bypass, cfg.max_backfill, cfg.partial_fit, cfg.expect_wait, cfg.hints ); + println!( + "work from history: {}", + if cfg.infer_work { + format!( + "on, when it followed {} of a stack's last {} runs within {}s of its start (infer_work)", + cfg.infer_work_min, cfg.infer_work_of, cfg.infer_work_within + ) + } else { + "off (infer_work = false)".into() + } + ); for (k, v) in &cfg.origin { if v != "built-in" { println!(" {k} set by {v}"); diff --git a/src/config.rs b/src/config.rs index 350eabe..1feeaa1 100644 --- a/src/config.rs +++ b/src/config.rs @@ -41,6 +41,10 @@ pub struct Layer { pub max_bypass: Option, pub max_backfill: Option, pub expect_wait: Option, + pub infer_work: Option, + pub infer_work_min: Option, + pub infer_work_of: Option, + pub infer_work_within: Option, pub boost_runs: Option, pub outlier_ratio: Option, pub outlier_min: Option, @@ -143,6 +147,14 @@ pub struct Config { /// Seconds a long-lived job that expects work waits, before it starts, /// for that work to queue: see `queue::Entry::expect_until`. 0 = never. pub expect_wait: f64, + /// A long-lived job with no declared work books what its history says + /// follows it: work that queued within `infer_work_within` seconds of + /// its start on at least `infer_work_min` of its last `infer_work_of` + /// runs in that checkout. + pub infer_work: bool, + pub infer_work_min: usize, + pub infer_work_of: usize, + pub infer_work_within: u64, pub boost_runs: usize, /// A memory peak more than this many times the next one below it is an /// outlier, and counts by `outlier_weights`. At or under 1: no outliers. @@ -204,6 +216,10 @@ impl Default for Config { max_bypass: 120, max_backfill: 600, expect_wait: 5.0, + infer_work: true, + infer_work_min: 3, + infer_work_of: 5, + infer_work_within: 900, boost_runs: 5, outlier_ratio: 1.5, outlier_min_kb: 1024 * 1024, @@ -639,6 +655,10 @@ impl Config { set!(max_bypass); set!(max_backfill); set!(expect_wait); + set!(infer_work); + set!(infer_work_min); + set!(infer_work_of); + set!(infer_work_within); set!(boost_runs); set!(outlier_ratio); if let Some(v) = &l.outlier_min diff --git a/src/db.rs b/src/db.rs index d18fb4e..4284e6c 100644 --- a/src/db.rs +++ b/src/db.rs @@ -555,6 +555,18 @@ fn median(v: &mut [f64]) -> Option { Some(if n % 2 == 1 { v[n / 2] } else { (v[n / 2 - 1] + v[n / 2]) / 2.0 }) } +/// Work that followed runs of a long-lived job: see `Db::followers`. +#[derive(Debug, Clone, PartialEq)] +pub struct Follower { + pub work: String, + /// On how many of the runs looked at it followed. + pub runs: usize, + /// How long after the start it first queued, the median. + pub lag_s: f64, + /// The memory it queued with, the median. + pub need_kb: Option, +} + pub fn median_of(mut v: Vec) -> Option { median(&mut v) } @@ -1042,12 +1054,15 @@ impl Db { Ok(None) } - /// The work that followed a long-lived job before: for the last few - /// runs of `key` in `checkout` (at most `n`, of the last two weeks), the - /// pool, or the key of a job without one, of every other job that queued - /// in that checkout while the run ran. Returns how many runs were looked - /// at, and what followed with on how many of them, most first. - pub fn followers(&self, key: &str, pool: Option<&str>, checkout: &str, n: usize) -> Result<(usize, Vec<(String, usize)>)> { + /// The work that followed a long-lived job before: for the last `of` + /// runs of `key` in `checkout` (of the last two weeks), every job of + /// another kind that queued in that checkout within `within` seconds of + /// the run's start, while it ran. Work is a job's pool, or its key when + /// it has none. Returns how many runs were looked at, and for each kind + /// of work on how many of them it followed, how long after the start it + /// first queued (the median), and the need it queued with (the median), + /// most often first. + pub fn followers(&self, key: &str, pool: Option<&str>, checkout: &str, of: usize, within: f64) -> Result<(usize, Vec)> { let now = now(); let mut stmt = self.conn.prepare_cached( "SELECT id, started_at, coalesce(ended_at, ?3) FROM runs @@ -1056,27 +1071,43 @@ impl Db { ORDER BY started_at DESC LIMIT ?4", )?; let runs: Vec<(i64, f64, f64)> = stmt - .query_map(params![key, checkout, now, n as i64], |r| Ok((r.get(0)?, r.get(1)?, r.get(2)?)))? + .query_map(params![key, checkout, now, of as i64], |r| Ok((r.get(0)?, r.get(1)?, r.get(2)?)))? .collect::>()?; - let mut seen: Vec<(String, usize)> = Vec::new(); let mut stmt = self.conn.prepare_cached( - "SELECT DISTINCT coalesce(pool, key) FROM runs + "SELECT coalesce(pool, key), min(queued_at) - ?3, need_mem_kb FROM runs WHERE id != ?1 AND key != ?2 AND (pool IS NULL OR ?5 IS NULL OR pool != ?5) AND queued_at >= ?3 AND queued_at <= ?4 - AND (cwd = ?6 OR substr(cwd, 1, length(?6) + 1) = ?6 || '/')", + AND (cwd = ?6 OR substr(cwd, 1, length(?6) + 1) = ?6 || '/') + GROUP BY coalesce(pool, key)", )?; + let mut seen: Vec<(String, Vec, Vec)> = Vec::new(); for (id, from, to) in &runs { - let works: Vec = - stmt.query_map(params![id, key, from, to, pool, checkout], |r| r.get(0))?.collect::>()?; - for w in works { - match seen.iter_mut().find(|(s, _)| *s == w) { - Some((_, c)) => *c += 1, - None => seen.push((w, 1)), - } + let rows: Vec<(String, f64, Option)> = stmt + .query_map(params![id, key, from, to.min(from + within), pool, checkout], |r| Ok((r.get(0)?, r.get(1)?, r.get(2)?)))? + .collect::>()?; + for (w, lag, need) in rows { + let i = match seen.iter().position(|(s, _, _)| *s == w) { + Some(i) => i, + None => { + seen.push((w, Vec::new(), Vec::new())); + seen.len() - 1 + } + }; + seen[i].1.push(lag); + seen[i].2.extend(need.filter(|n| *n > 0).map(|n| n as f64)); } } - seen.sort_by(|a, b| b.1.cmp(&a.1).then(a.0.cmp(&b.0))); - Ok((runs.len(), seen)) + let mut out: Vec = seen + .into_iter() + .map(|(work, lags, needs)| Follower { + work, + runs: lags.len(), + lag_s: median_of(lags).unwrap_or(0.0), + need_kb: median_of(needs).map(|n| n as u64), + }) + .collect(); + out.sort_by(|a, b| b.runs.cmp(&a.runs).then(a.work.cmp(&b.work))); + Ok((runs.len(), out)) } /// How many of the key's most recent runs in a row were starved. @@ -1900,8 +1931,8 @@ mod tests { fn the_work_that_followed_a_stack_in_its_checkout() { let db = Db::open(Path::new(":memory:")).unwrap(); let t = now() - 86400.0; - let at = |key: &str, pool: Option<&str>, cwd: &str, queued: f64, started: f64, ended: Option| { - let id = db.insert_run(&NewRun { ns: "n", key, pool, cwd, ..Default::default() }).unwrap(); + let at = |key: &str, pool: Option<&str>, cwd: &str, need: u64, queued: f64, started: f64, ended: Option| { + let id = db.insert_run(&NewRun { ns: "n", key, pool, cwd, need_mem_kb: need, ..Default::default() }).unwrap(); db.conn .execute( "UPDATE runs SET queued_at = ?2, started_at = ?3, ended_at = ?4 WHERE id = ?1", @@ -1909,24 +1940,26 @@ mod tests { ) .unwrap(); }; - // Three stacks in /w/dalp; a browser run followed two of them, and a - // lint each. The third still runs. - for (i, run) in [(0, true), (1, true), (2, false)] { + // Four stacks in /w/dalp. A browser run followed three of them, 6 to + // 8 minutes after their start, and the fourth only after 20 minutes. + // A lint followed each. The last stack still runs. + for (i, lag) in [(0, 360.0), (1, 420.0), (2, 480.0), (3, 1200.0)] { let s = t + 3600.0 * i as f64; - at("dev-env:stack", Some("stack"), "/w/dalp", s - 10.0, s, (i < 2).then_some(s + 1800.0)); - if run { - at("e2e/agentic:pr.ts", Some("browser"), "/w/dalp/e2e/agentic", s + 400.0, s + 400.0, Some(s + 700.0)); - } - at(&format!("pkg{i}:oxlint"), None, "/w/dalp/pkg", s + 60.0, s + 60.0, Some(s + 70.0)); + at("dev-env:stack", Some("stack"), "/w/dalp", 9000, s - 10.0, s, (i < 3).then_some(s + 1800.0)); + at("e2e/agentic:pr.ts", Some("browser"), "/w/dalp/e2e/agentic", 4000 + i as u64, s + lag, s + lag, Some(s + lag + 300.0)); + at(&format!("pkg{i}:oxlint"), None, "/w/dalp/pkg", 5000, s + 60.0, s + 60.0, Some(s + 70.0)); } // Nothing of another checkout counts, nor whatever ran before a stack. - at("e2e/agentic:pr.ts", Some("browser"), "/w/dalp-2/e2e/agentic", t + 7300.0, t + 7300.0, Some(t + 7400.0)); - at("root:tsc", None, "/w/dalp", t - 600.0, t - 600.0, Some(t - 500.0)); - let (n, works) = db.followers("dev-env:stack", Some("stack"), "/w/dalp", 5).unwrap(); - assert_eq!(n, 3); - assert_eq!(works[0], ("browser".to_string(), 2)); - assert!(works[1..].iter().all(|(w, c)| w.ends_with(":oxlint") && *c == 1), "{works:?}"); - let (n, works) = db.followers("dev-env:stack", Some("stack"), "/w/elsewhere", 5).unwrap(); + at("e2e/agentic:pr.ts", Some("browser"), "/w/dalp-2/e2e/agentic", 4000, t + 7300.0, t + 7300.0, Some(t + 7400.0)); + at("root:tsc", None, "/w/dalp", 1000, t - 600.0, t - 600.0, Some(t - 500.0)); + let (n, works) = db.followers("dev-env:stack", Some("stack"), "/w/dalp", 5, 900.0).unwrap(); + assert_eq!(n, 4); + assert_eq!(works[0], Follower { work: "browser".into(), runs: 3, lag_s: 420.0, need_kb: Some(4001) }); + assert!(works[1..].iter().all(|f| f.work.ends_with(":oxlint") && f.runs == 1), "{works:?}"); + // Only the last three stacks: two browser runs within the window. + let (n, works) = db.followers("dev-env:stack", Some("stack"), "/w/dalp", 3, 900.0).unwrap(); + assert_eq!((n, works[0].runs), (3, 2)); + let (n, works) = db.followers("dev-env:stack", Some("stack"), "/w/elsewhere", 5, 900.0).unwrap(); assert_eq!((n, works.len()), (0, 0)); } diff --git a/src/help.rs b/src/help.rs index 474dbd6..02db3b0 100644 --- a/src/help.rs +++ b/src/help.rs @@ -15,7 +15,8 @@ options (sem style; the default is to run in the foreground): --expect WORK a long-lived job (a dev stack): work that will run on it, a pool name, job key or command pattern; it books room for that work and the work goes first once it - queues. Repeatable; also TASKGUARD_EXPECT=WORK,WORK + queues. Repeatable; also TASKGUARD_EXPECT=WORK,WORK. + none: no work, not even what its history suggests --now skip the queue, but still measure and learn --wraps COMMAND runs other taskguard jobs (turbo, a package script): run it straight through and let them queue diff --git a/src/queue.rs b/src/queue.rs index ee639a9..ce2cc08 100644 --- a/src/queue.rs +++ b/src/queue.rs @@ -141,9 +141,21 @@ pub struct Expect { pub from: String, /// A job for it has started: the booking is gone. pub started: bool, - /// Not declared: the history says such work followed this job before. - /// It books nothing until a job for it queues while this job waits. + /// Not declared: the history says such work followed this job before + /// (`infer_work`). Its booking is a guess: it ends `expire_after_s` + /// after the job started, and no job waits for it under the progress + /// rule. pub implied: bool, + pub expire_after_s: Option, + /// The guess did not come true in time: the booking is gone. + pub expired: bool, +} + +impl Expect { + /// Still booked: its work has not started, and the guess has not expired. + pub fn open(&self) -> bool { + !self.started && !self.expired + } } /// The long-lived job a job is work for, and which of its `expects` it @@ -793,7 +805,14 @@ pub fn booked(e: &Entry, running: &[Entry]) -> u64 { let taken = |i: usize| { e.started_at.is_some() && running.iter().any(|r| r.work_for.as_ref().is_some_and(|w| w.pid == e.pid && w.item == Some(i as u32))) }; - e.expects.iter().enumerate().filter(|(i, x)| !x.started && !taken(*i)).map(|(_, x)| x.mem_kb).sum() + e.expects.iter().enumerate().filter(|(i, x)| x.open() && !taken(*i)).map(|(_, x)| x.mem_kb).sum() +} + +/// The part of `booked` that only the history guessed (`infer_work`). +fn guessed(e: &Entry, running: &[Entry]) -> u64 { + let guesses = + Entry { expects: e.expects.iter().map(|x| Expect { started: x.started || !x.implied, ..x.clone() }).collect(), ..e.clone() }; + booked(&guesses, running) } /// The memory `job` adds when it starts: its need, plus what it books for @@ -813,7 +832,7 @@ fn booking_taken_over(job: &Entry, running: &[Entry]) -> u64 { } let Some(stack) = running.iter().find(|e| e.pid == *pid) else { return 0 }; let taken = running.iter().any(|r| r.pid != job.pid && r.work_for.as_ref().is_some_and(|w| w.pid == *pid && w.item == Some(*i))); - stack.expects.get(*i as usize).filter(|x| !x.started && !taken).map_or(0, |x| x.mem_kb) + stack.expects.get(*i as usize).filter(|x| x.open() && !taken).map_or(0, |x| x.mem_kb) } /// Is `job` work for a long-lived job that does not run yet? It waits for @@ -840,10 +859,9 @@ pub fn link_work(me: &Entry, running: &[Entry], waiting: &[Entry], matches: impl let mut best: Option<(u8, WorkFor, i32)> = None; for e in stacks { for (i, x) in e.expects.iter().enumerate().filter(|(_, x)| matches(&x.work)) { - let free = !x.started && !claimed(e.pid, i); - // A stack that waits takes only work that can take over a booking; - // work the history implies only while the stack still waits. - if (e.started_at.is_none() && !free) || (x.implied && e.started_at.is_some()) { + let free = x.open() && !claimed(e.pid, i); + // A stack that waits takes only work that can take over a booking. + if e.started_at.is_none() && !free { continue; } let rank = u8::from(e.started_at.is_none()) * 2 + u8::from(!free); @@ -1290,6 +1308,9 @@ struct Room { /// The running jobs: what a job takes over from its long-lived job /// depends on them. running: Vec, + /// The part of `res_mem` that only the history guessed: what long-lived + /// jobs book for work nobody declared (`infer_work`). + res_guessed: u64, } impl Room { @@ -1308,11 +1329,24 @@ impl Room { passing_live: passing.iter().map(|e| e.live_mem_kb).sum(), passing_res: reserve(&passing).1, running: running.to_vec(), + res_guessed: running.iter().map(|e| guessed(e, running)).sum(), } } fn mem_would(&self, job: &Entry) -> f64 { - (self.mem_used + self.res_mem + need_with_booking(job, &self.running)) as f64 + self.mem_would_on(job, false) + } + + /// As `mem_would`, by the progress rule (`progress`) or not. A guess of + /// the history never keeps a job out by the progress rule, so it cannot + /// stop the queue. Nor does it keep out a long-lived job started with no + /// declared work, its own guess included: such a stack starts as it + /// would without the history, and books its guess from then on. + fn mem_would_on(&self, job: &Entry, progress: bool) -> f64 { + let undeclared = job.long_lived && !job.expects.iter().any(|x| !x.implied); + let guess = if (progress && !job.long_lived) || undeclared { self.res_guessed } else { 0 }; + let own = if undeclared && job.started_at.is_none() { guessed(job, &[]) } else { 0 }; + (self.mem_used + self.res_mem - guess.min(self.res_mem) + need_with_booking(job, &self.running) - own) as f64 } /// The most memory a job may take the machine to by the progress rule: @@ -1330,7 +1364,7 @@ impl Room { (Some(pk), Some(max)) => (running.iter().filter(|e| e.pool_key.as_ref() == Some(pk)).count() as u32) < max, _ => true, }; - pool_free && !waits_for_stack(job, running) && self.mem_would(job) <= self.progress_ceiling(m, lim, job) + pool_free && !waits_for_stack(job, running) && self.mem_would_on(job, true) <= self.progress_ceiling(m, lim, job) } /// Would `job` fit by the progress rule once every running job that ends @@ -1665,7 +1699,15 @@ mod tests { // What this version adds reads back, and leaves the rest alone. let stack = Entry { long_lived: true, - expects: vec![Expect { work: "browser".into(), mem_kb: 4, from: "x".into(), started: true, implied: true }], + expects: vec![Expect { + work: "browser".into(), + mem_kb: 4, + from: "x".into(), + started: true, + implied: true, + expire_after_s: Some(600.0), + expired: true, + }], expect_until: Some(5.0), work_for: Some(WorkFor { pid: 9, key: "s".into(), item: Some(0) }), ..older @@ -2634,17 +2676,41 @@ mod tests { } #[test] - fn work_the_history_implies_links_only_while_the_stack_waits() { + fn a_guess_of_the_history_books_room_but_never_stops_the_queue() { + let lim = Limits { mem_max_pct: 95.0, ..LIM }; let mut stack = Entry { pid: 11, ..expecting(steady_stack(11, 7)) }; - stack.expects[0] = Expect { work: "browser".into(), implied: true, ..Default::default() }; + stack.expects[0].implied = true; + stack.expects[0].expire_after_s = Some(600.0); + let grows = stack.need_mem_kb - stack.live_mem_kb; + let r = std::slice::from_ref(&stack); + // It books the run's usual 4.3 GB, and the run takes that over. + assert_eq!(reserve(r).1, grows + 43 * GB / 10); let me = Entry { pool: Some("browser".into()), checkout: "/w/a".into(), ..job(3, "pr.ts", 3.0, 4) }; - let is = |w: &str| w == "browser"; - // It books nothing. - assert_eq!(reserve(std::slice::from_ref(&stack)).1, stack.need_mem_kb - stack.live_mem_kb); - // Up already: a run that queues now is no evidence of anything. - assert!(link_work(&me, std::slice::from_ref(&stack), &[], is).is_none()); - let waiting = Entry { started_at: None, ..stack }; - assert_eq!(link_work(&me, &[], std::slice::from_ref(&waiting), is).unwrap().0.item, Some(0)); + assert_eq!(link_work(&me, r, &[], |w| w == "browser").unwrap().0.item, Some(0)); + // 24 GB held of 32 and only the stack runs: a 5 GB lint does not fit + // with the guess booked (35 GB), but the progress rule ignores a + // guess, up to all of RAM (30 GB): it starts. + let m = machine(1.0, 24); + let lint = job(4, "lint", 3.5, 5); + assert!(blockers(decide(&m, &lim, r, std::slice::from_ref(&lint), &lint, 1000.0, &[])).is_empty()); + // While a job that ends by itself runs, the guess holds like a booking. + let busy = [stack.clone(), Entry { started_at: Some(990.0), ..job(5, "build", 0.5, 0) }]; + assert_eq!(blockers(decide(&m, &lim, &busy, std::slice::from_ref(&lint), &lint, 1000.0, &[])), vec!["memory"]); + // A second stack whose caller declared work respects the guess. One + // started with no declared work does not, nor its own guess: it + // starts as it would without the history. + let plain = Entry { long_lived: true, priority: 10, ..job(6, "dev-env:stack", 1.0, 4) }; + assert!(blockers(decide(&m, &lim, r, std::slice::from_ref(&plain), &plain, 1000.0, &[])).is_empty()); + let mut guessing = Entry { checkout: "/w/b".into(), ..plain.clone() }; + guessing.expects = vec![Expect { work: "browser".into(), mem_kb: GB, implied: true, ..Default::default() }]; + assert!(blockers(decide(&m, &lim, r, std::slice::from_ref(&guessing), &guessing, 1000.0, &[])).is_empty()); + let mut declared = guessing.clone(); + declared.expects[0].implied = false; + assert_eq!(blockers(decide(&m, &lim, r, std::slice::from_ref(&declared), &declared, 1000.0, &[])), vec!["memory"]); + // Expired, it books nothing, and a run for the stack still goes first. + stack.expects[0].expired = true; + assert_eq!(reserve(std::slice::from_ref(&stack)).1, grows); + assert_eq!(link_work(&me, std::slice::from_ref(&stack), &[], |w| w == "browser").unwrap().0.item, None); } #[test] diff --git a/src/report.rs b/src/report.rs index 51ea577..5743595 100644 --- a/src/report.rs +++ b/src/report.rs @@ -550,7 +550,12 @@ pub fn link_text(e: &Entry, running: &[Entry]) -> String { if let Some(w) = &e.work_for { t += &format!(" (for stack {})", w.key); } - let open: Vec<&str> = e.expects.iter().filter(|x| !x.started && (!x.implied || x.mem_kb > 0)).map(|x| x.work.as_str()).collect(); + let open: Vec = e + .expects + .iter() + .filter(|x| x.open()) + .map(|x| if x.implied { format!("{} from history", x.work) } else { x.work.clone() }) + .collect(); if !open.is_empty() { t += &format!(" (expects {}, books {})", open.join(", "), gb(booked(e, running))); } diff --git a/src/runner.rs b/src/runner.rs index 3762e3b..6728e8a 100644 --- a/src/runner.rs +++ b/src/runner.rs @@ -52,7 +52,8 @@ pub struct Opts { /// Work a long-lived job's caller says will run on it: `--expect`, then /// `TASKGUARD_EXPECT` (comma-separated). Any other job inherits the variable -/// from whoever started it and leaves it alone. +/// from whoever started it and leaves it alone. `none` says it expects no +/// work, not even what its history suggests. fn declared_work(o: &Opts) -> Vec { let env = std::env::var("TASKGUARD_EXPECT").unwrap_or_default(); o.expect.iter().cloned().chain(env.split(',').map(str::trim).filter(|w| !w.is_empty()).map(str::to_string)).collect() @@ -64,6 +65,8 @@ fn is_work(work: &str, me: &Entry, cmd: &[String]) -> bool { } /// Mark the expected work a nudge says has started. +/// Once work for it has started, a long-lived job also drops what it booked +/// on the history's guess: the work came. fn apply_work_started(me: &mut Entry, n: &queue::Nudge) -> bool { let mut changed = false; for i in &n.work_started { @@ -72,9 +75,61 @@ fn apply_work_started(me: &mut Entry, n: &queue::Nudge) -> bool { changed = true; } } + if !n.work_started.is_empty() { + for x in me.expects.iter_mut().filter(|x| x.implied && x.open()) { + x.started = true; + changed = true; + } + } changed } +/// A guess of the history that did not come true in time: its booking ends. +fn expire_guesses(me: &mut Entry, now: f64) -> bool { + let started = me.started_at.unwrap_or(now); + let mut changed = false; + for x in me.expects.iter_mut().filter(|x| x.open() && x.expire_after_s.is_some_and(|a| now - started > a)) { + x.expired = true; + changed = true; + } + changed +} + +/// The longest a booking on the history's guess lasts after its job started. +const GUESS_MAX_S: f64 = 600.0; + +/// The work the history says follows a long-lived job `key` in `checkout`: +/// each kind of work that queued within `infer_work_within` seconds of its +/// start on at least `infer_work_min` of its last `infer_work_of` runs there. +/// It books that work's usual need, for twice its usual delay at most. +fn inferred_work(d: &Db, cfg: &Config, key: &str, pool: Option<&str>, checkout: &str) -> Vec { + let within = cfg.infer_work_within as f64; + let Ok((n, works)) = d.followers(key, pool, checkout, cfg.infer_work_of.max(1), within) else { return Vec::new() }; + works + .into_iter() + .filter(|f| f.runs >= cfg.infer_work_min.max(1)) + .map(|f| { + let mem_kb = f.need_kb.unwrap_or(cfg.new_job_mem_kb); + let expire = (2.0 * f.lag_s).clamp(60.0, GUESS_MAX_S); + queue::Expect { + from: format!( + "from history: it queued within {}s of the start of {} of the last {n} runs of {key} here, after {} as a rule, needing {}; booked for {} at most", + within, + f.runs, + report::dur(f.lag_s), + report::gb(mem_kb), + report::dur(expire) + ), + work: f.work, + mem_kb, + implied: true, + expire_after_s: Some(expire), + ..Default::default() + } + }) + .collect() +} + /// sem's `-j`: N, +N (cores + N), -N (cores - N) or N% of cores, at least 1. pub fn parse_jobs(s: &str, ncpu: usize) -> Result { let n = ncpu as f64; @@ -379,22 +434,20 @@ pub fn run(mut o: Opts) -> Result { Some(_) => work.iter().map(|w| need::expected(database.as_ref(), &cfg, &cwd, w)).collect(), None => Vec::new(), }; - // Without a declaration, the history may still say that work follows - // this job in this checkout: what queued there while at least half of - // its last runs (two at least) ran. That books nothing; it only makes - // the job wait `expect_wait` for such work to queue with it. - let expect_wait = cfg.expect_wait.clamp(0.0, 60.0); + // `--expect none`: no work, not even what the history suggests. + let none = work.iter().any(|w| w == "none"); + expects.retain(|x| x.work != "none"); + // Without a declaration, the history may say what work follows this job + // in this checkout: it books that, as a guess with a time limit. if startup.is_some() && expects.is_empty() - && expect_wait > 0.0 - && let Some((n, works)) = database.as_ref().and_then(|d| d.followers(&jkey, pool.as_deref(), &checkout.to_string_lossy(), 5).ok()) - && n >= 2 + && !none + && cfg.infer_work + && let Some(d) = &database { - for (w, c) in works.into_iter().filter(|(_, c)| c * 2 >= n) { - let from = format!("not declared; it queued while {c} of the last {n} runs of {jkey} in this checkout ran"); - expects.push(queue::Expect { work: w, from, implied: true, ..Default::default() }); - } + expects = inferred_work(d, &cfg, &jkey, pool.as_deref(), &checkout.to_string_lossy()); } + let expect_wait = cfg.expect_wait.clamp(0.0, 60.0); let estimate_from = guess.as_ref().map(|g| g.from.clone()); let est_dur = guess.as_ref().and_then(|g| g.dur_s); @@ -490,7 +543,7 @@ pub fn run(mut o: Opts) -> Result { for x in &me.expects { let wait = me.expect_until.map(|_| format!("; it waits up to {expect_wait:.0}s for it to queue")).unwrap_or_default(); if x.implied { - say(&format!("expects {} - {}: it books room only once that work queues{wait}", x.work, x.from)); + say(&format!("expects {} - {} books {} for it ({}){wait}", x.work, me.key, report::gb(x.mem_kb), x.from)); } else { say(&format!("expects {} - {} books {} for it until it starts ({}){wait}", x.work, me.key, report::gb(x.mem_kb), x.from)); } @@ -592,15 +645,17 @@ pub fn run(mut o: Opts) -> Result { } link_to_stack(&mut me, own_priority, &running, &waiting, &o.cmd, o.quiet); // Work that queued for this long-lived job may need more than - // was booked for it: book that. + // was booked for it: book that. Work the history guessed that + // has queued is a guess no more. for (i, x) in me.expects.iter_mut().enumerate() { let theirs = waiting .iter() .filter(|w| w.work_for.as_ref().is_some_and(|l| l.pid == me_pid && l.item == Some(i as u32))) .max_by_key(|w| w.need_mem_kb); - if let Some(w) = theirs.filter(|w| w.need_mem_kb > x.mem_kb) { - x.mem_kb = w.need_mem_kb; + if let Some(w) = theirs.filter(|w| w.need_mem_kb > x.mem_kb || x.implied) { + x.mem_kb = x.mem_kb.max(w.need_mem_kb); x.from = format!("the need of {}, which queued for it", w.key); + (x.implied, x.expire_after_s) = (false, None); } } let mut waiting = waiting; @@ -863,6 +918,10 @@ pub fn run(mut o: Opts) -> Result { say(&format!("paused {} - by hand; it stays paused until taskguard resume, or p in taskguard top", me.key)); } } + if expire_guesses(&mut me, now) { + let _ = q.write(&q.run_path(me_pid), &me); + say(&format!("expected {} - the work its history suggested did not come in time; it no longer books room for it", me.key)); + } if let Some(r) = tracker.sample(child_pid, now, &procs) { me.live_cpu = r.used; me.live_mem_kb = r.mem_kb; @@ -1256,6 +1315,59 @@ pub fn wait_pool(id: Option<&str>) -> Result { mod tests { use super::*; + #[test] + fn a_guess_of_the_history_ends_when_the_work_comes_or_its_time_is_up() { + let guess = queue::Expect { work: "browser".into(), mem_kb: 4, implied: true, expire_after_s: Some(600.0), ..Default::default() }; + let declared = queue::Expect { work: "e2e".into(), mem_kb: 4, ..Default::default() }; + let stack = Entry { started_at: Some(1000.0), expects: vec![declared, guess], ..Default::default() }; + let mut s = stack.clone(); + assert!(!expire_guesses(&mut s, 1600.0)); + assert!(expire_guesses(&mut s, 1601.0)); + assert!(!s.expects[0].expired && s.expects[1].expired, "only the guess expires"); + // Work for the stack started: the guess goes with it. + let mut s = stack; + assert!(apply_work_started(&mut s, &queue::Nudge { work_started: vec![0], ..Default::default() })); + assert!(s.expects.iter().all(|x| x.started)); + } + + #[test] + fn the_history_infers_work_only_when_it_followed_often_enough() { + let d = Db::open(Path::new(":memory:")).unwrap(); + let t = db::now() - 86400.0; + let at = |key: &str, pool: &str, cwd: &str, queued: f64, ended: Option| { + let id = d.insert_run(&NewRun { ns: "n", key, pool: Some(pool), cwd, need_mem_kb: 4_000_000, ..Default::default() }).unwrap(); + d.conn.execute("UPDATE runs SET queued_at = ?2 WHERE id = ?1", rusqlite::params![id, queued]).unwrap(); + d.mark_started(id, queued, 0.0, None).unwrap(); + if let Some(e) = ended { + d.finish_run(id, &RunResult { ended_at: e, ..Default::default() }).unwrap(); + } + id + }; + let infer = |cfg: &Config| inferred_work(&d, cfg, "dev-env:stack", Some("stack"), "/w/dalp"); + let cfg = Config::default(); + // Two stacks, each followed by a browser run 6 minutes in: not yet 3 of 5. + for i in 0..3 { + let s = t + 3600.0 * i as f64; + at("dev-env:stack", "stack", "/w/dalp", s, Some(s + 1800.0)); + if i < 2 { + at("pr.ts", "browser", "/w/dalp/e2e", s + 360.0, Some(s + 600.0)); + } + } + assert!(infer(&cfg).is_empty()); + let s = t + 3600.0 * 3.0; + at("dev-env:stack", "stack", "/w/dalp", s, Some(s + 1800.0)); + at("pr.ts", "browser", "/w/dalp/e2e", s + 360.0, Some(s + 600.0)); + // Now 3 of the last 4: it books the run's usual need, for twice its + // usual delay, ten minutes at most. + let x = infer(&cfg); + assert_eq!(x.len(), 1, "{x:?}"); + assert_eq!((x[0].work.as_str(), x[0].mem_kb, x[0].implied, x[0].expire_after_s), ("browser", 4_000_000, true, Some(600.0))); + assert!(x[0].from.starts_with("from history"), "{}", x[0].from); + // Stricter settings, or later than the window, infer nothing. + assert!(infer(&Config { infer_work_min: 4, ..Config::default() }).is_empty()); + assert!(infer(&Config { infer_work_within: 300, ..Config::default() }).is_empty()); + } + #[test] fn jobs_like_sem() { assert_eq!(parse_jobs("4", 12).unwrap(), 4); diff --git a/src/stall.rs b/src/stall.rs index cf28641..876680f 100644 --- a/src/stall.rs +++ b/src/stall.rs @@ -143,8 +143,14 @@ pub struct Agent { pub booking_kb: u64, /// The agent queues its run 2 s after its stack, not once the stack is up. pub with_stack: bool, - /// Undeclared, but the history says a browser run follows its stacks. + /// Undeclared, but the history says a browser run follows its stacks + /// (`infer_work`): the stack books the run's usual need, for at most + /// ten minutes. pub implied: bool, + /// A bad guess: the agent only wants the stack, for `hold_s`, and never + /// queues a run. + pub no_run: bool, + pub hold_s: f64, } pub fn agent(first_at: f64, idle_gb: f64) -> Agent { @@ -165,6 +171,8 @@ pub fn agent(first_at: f64, idle_gb: f64) -> Agent { booking_kb: 36 * GB / 10, with_stack: false, implied: false, + no_run: false, + hold_s: 1200.0, } } @@ -221,6 +229,8 @@ pub struct Outcome { pub worst_stack_wait: f64, /// Runs that started while their stack did not run: they fail. pub runs_without_stack: usize, + /// The longest a lint waited. + pub worst_lint_wait: f64, /// When each of `Scenario::jobs` started, if it did. pub job_started: Vec>, } @@ -310,7 +320,13 @@ pub fn run(s: &Scenario) -> Outcome { let expects = if a.declares { vec![Expect { work: "browser".into(), mem_kb: a.booking_kb, ..Default::default() }] } else if a.implied { - vec![Expect { work: "browser".into(), implied: true, ..Default::default() }] + vec![Expect { + work: "browser".into(), + mem_kb: a.run_need_kb, + implied: true, + expire_after_s: Some(600.0), + ..Default::default() + }] } else { Vec::new() }; @@ -331,7 +347,7 @@ pub fn run(s: &Scenario) -> Outcome { ..Default::default() }; waiting.push((e, Kind::Stack(n))); - phase[n] = Phase::StackQueued { run_at: a.with_stack.then_some(t + 2.0) }; + phase[n] = Phase::StackQueued { run_at: (a.with_stack && !a.no_run).then_some(t + 2.0) }; } Phase::StackQueued { run_at } => { if run_at.is_some_and(|at| t >= at) { @@ -344,6 +360,10 @@ pub fn run(s: &Scenario) -> Outcome { phase[n] = Phase::StackUp; } } + Phase::StackUp if a.no_run => { + let started = running.iter().find(|r| r.kind == Kind::Stack(n)).map_or(t, |r| r.started); + phase[n] = Phase::Down { at: started + a.hold_s }; + } Phase::StackUp => { ticket += 1; waiting.push((run_entry(n, a, ticket, t), Kind::Run(n))); @@ -431,6 +451,12 @@ pub fn run(s: &Scenario) -> Outcome { _ => None, }; r.e.follow(mem, cpu, s.host.ncpu as f64, t, lived); + // As the owner does: a guess of the history that did not come + // true in time stops booking. + let age_now = t - r.started; + for x in r.e.expects.iter_mut().filter(|x| x.open() && x.expire_after_s.is_some_and(|a| age_now > a)) { + x.expired = true; + } if s.steady_books_live && r.e.steady() { r.e.need_mem_kb = mem; } @@ -490,6 +516,9 @@ pub fn run(s: &Scenario) -> Outcome { .unwrap_or(0); let item = &mut waiting[i].0.expects[x]; item.mem_kb = item.mem_kb.max(theirs); + if theirs > 0 { + (item.implied, item.expire_after_s) = (false, None); + } } } let all: Vec = waiting.iter().map(|(e, _)| e.clone()).collect(); @@ -502,6 +531,9 @@ pub fn run(s: &Scenario) -> Outcome { && let Some(st) = running.iter_mut().find(|r| r.e.pid == w.pid) { st.e.expects[i as usize].started = true; + for x in st.e.expects.iter_mut().filter(|x| x.implied) { + x.started = true; + } } for (w, _) in waiting.iter_mut().filter(|(w, _)| w.ahead_of(&me) && w.bypassed_since.is_none()) { w.bypassed_since = Some(t); @@ -512,6 +544,9 @@ pub fn run(s: &Scenario) -> Outcome { if let Kind::Job(n) = kind { out.job_started[n] = Some(t); } + if kind == Kind::Lint { + out.worst_lint_wait = out.worst_lint_wait.max(t - me.queued_at); + } if let Kind::Run(n) = kind && !running.iter().any(|r| r.kind == Kind::Stack(n)) { @@ -575,6 +610,25 @@ pub fn declared_morning(outside_gb: f64) -> Scenario { s } +/// The same morning, undeclared, with the history inferring each run. +pub fn inferred_morning(outside_gb: f64) -> Scenario { + let mut s = morning(outside_gb); + for a in &mut s.agents { + a.implied = true; + } + s +} + +/// A bad guess: the history says a run follows each stack, but the agents +/// only want their stacks, for 20 minutes each, while the lints keep coming. +pub fn bad_guess(outside_gb: f64, infer: bool) -> Scenario { + let mut s = morning(outside_gb); + for a in &mut s.agents { + (a.implied, a.no_run) = (infer, true); + } + s +} + /// Omarchy (24 cores, 31.3 GB, Linux, mem_max 95, DALP's max_backfill 1800): /// a stack has run for an hour, a second stack of 9.1 GB queues that does /// not fit next to it, and 25 jobs queue behind it over five minutes: lints @@ -693,6 +747,8 @@ mod tests { eprintln!("{}", row(&format!("outside {outside} GB, outside cap 92%"), &run(&old))); let declared = run(&declared_morning(outside)); eprintln!("{}", row(&format!("outside {outside} GB, declared"), &declared)); + let inferred = run(&inferred_morning(outside)); + eprintln!("{}", row(&format!("outside {outside} GB, inferred"), &inferred)); // The scout's fallback, not taken: a steady stack books what it // holds. It gives runs up at 12.8 GB. let live = Scenario { steady_books_live: true, ..morning(outside) }; @@ -705,6 +761,9 @@ mod tests { assert_eq!(declared.runs_given_up, 0, "{outside} GB declared: {declared:?}"); assert!(declared.runs_done >= 10, "{outside} GB declared: {declared:?}"); assert!(declared.worst_mem_pct <= 95.0 && declared.over_limit_s == 0.0, "{outside} GB declared: {declared:?}"); + // Nor does one whose run the history infers. + assert_eq!(inferred.runs_given_up, 0, "{outside} GB inferred: {inferred:?}"); + assert!(inferred.worst_mem_pct <= 95.0, "{outside} GB inferred: {inferred:?}"); // A caller that does not declare: the progress rule alone clears // the morning's load, never a second stack over the limit. if outside < 14.0 { @@ -744,6 +803,34 @@ mod tests { } } + #[test] + fn a_bad_guess_of_the_history_starves_nothing() { + eprintln!("{:<34} {:>6} {:>10} {:>6} {:>9} {:>11}", "variant", "lints", "worst lint", "worst", "stacks", "worst stack"); + eprintln!("{:<34} {:>6} {:>10} {:>6} {:>9} {:>11}", "", "done", "wait", "mem", "at once", "wait"); + for outside in [11.0, 12.8, 14.5] { + let plain = run(&bad_guess(outside, false)); + let guessed = run(&bad_guess(outside, true)); + for (name, o) in [("no run follows, no inference", &plain), ("no run follows, inferred", &guessed)] { + eprintln!( + "{:<34} {:>6} {:>9.0}s {:>5.1}% {:>9} {:>10.0}s", + format!("{outside} GB, {name}"), + o.lints_done, + o.worst_lint_wait, + o.worst_mem_pct, + o.max_stacks, + o.worst_stack_wait + ); + } + // The guess books room for ten minutes at most and never stops + // the progress rule: no lint waits longer than without it, give + // or take a minute, though fewer run while a guess books room. + assert!(guessed.worst_lint_wait <= plain.worst_lint_wait + 60.0, "{outside} GB: {guessed:?} vs {plain:?}"); + assert!(guessed.lints_done * 10 >= plain.lints_done * 7, "{outside} GB: {guessed:?} vs {plain:?}"); + // It never keeps a stack started with no work out. + assert!(guessed.worst_stack_wait <= plain.worst_stack_wait + 60.0, "{outside} GB: {guessed:?} vs {plain:?}"); + } + } + #[test] fn a_large_job_at_the_front_on_omarchy() { let s = head_of_line(); diff --git a/taskguard.example.toml b/taskguard.example.toml index c520d2e..3afd596 100644 --- a/taskguard.example.toml +++ b/taskguard.example.toml @@ -75,6 +75,16 @@ expect_wait = 5 # seconds a long-lived job (a dev stack) waits before it s # work it expects to queue, so it starts with room for both. Only when # it declares work (--expect, TASKGUARD_EXPECT) or its history says work # follows it in that checkout. 0 = never; at most 60. +# A long-lived job that declares no work books the work its history says +# follows it: work of one kind (a pool, or a key outside any pool) that queued +# in its checkout within infer_work_within seconds of its start on at least +# infer_work_min of its last infer_work_of runs there. It books that work's +# typical need until the work starts, at most twice its typical delay (10 +# minutes at most). --expect none on a job turns it off for that job. +infer_work = true +infer_work_min = 3 +infer_work_of = 5 +infer_work_within = 900 boost_runs = 5 # runs that get +25% memory after a memory-starved run # A peak more than outlier_ratio times the next peak below it, and outlier_min # above it, is an outlier. Outliers count by outlier_weights: the first entry diff --git a/tests/cli.rs b/tests/cli.rs index 6645f35..dd7e60c 100644 --- a/tests/cli.rs +++ b/tests/cli.rs @@ -612,7 +612,10 @@ fn a_stack_waits_for_the_work_it_expects_when_both_queue_together() { let t = Instant::now(); // The run queues a moment later: the stack, which waits for it, starts // at once, and the run after it. - std::thread::sleep(Duration::from_millis(300)); + wait_until("the stack queues", || { + let out = e.run(&["status", "--json"]); + serde_json::from_slice::(&out.stdout).is_ok_and(|v| v["waiting"].as_array().is_some_and(|w| !w.is_empty())) + }); let run = e.run(&["--id", "browser", "--", "true"]); assert!(run.status.success(), "{}", stderr(&run)); assert!(t.elapsed() < Duration::from_secs(15), "the stack did not wait out expect_wait: {:?}", t.elapsed()); From 744ec327b081bb3aaf559d82c8315529716af745 Mon Sep 17 00:00:00 2001 From: Vincent De Snerck <10668+SpoBo@users.noreply.github.com> Date: Fri, 16 Oct 2026 12:51:40 +0200 Subject: [PATCH 09/22] no-mistakes(review): Bound history guesses; respect open guesses; outside_admit lone-stack ceiling --- CHANGELOG.md | 21 ++++++--- README.md | 25 +++++++---- src/queue.rs | 118 +++++++++++++++++++++++++++++++++++++++++++------- src/runner.rs | 8 ++-- src/stall.rs | 20 ++++++--- 5 files changed, 152 insertions(+), 40 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index aba031c..7faaee5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -29,17 +29,22 @@ and the versions follow [semver](https://semver.org). priority above the stack, takes over the booked room when it starts, and waits while its stack still waits. `taskguard status` shows `(for stack KEY)` on the work and `(expects WORK, books SIZE)` on the stack. Without a - declaration nothing changes. + declaration, only what the history infers applies (`infer_work`, below). - A long-lived job with no declared work books the work its history says follows it: work of one kind (a pool, or a key outside any pool) that queued in its checkout within `infer_work_within` (900) seconds of its start on at least `infer_work_min` (3) of its last `infer_work_of` (5) runs there. It books that work's usual need, shown as `(expects WORK from history, ...)`, for twice its usual delay and ten minutes at most, and - drops it once work for the stack starts. The guess never stops the rule - for nothing running and never keeps out a stack started with no declared - work. A declaration wins; `--expect none` expects nothing; - `infer_work = false` turns it off. + drops it once work for the stack starts. While the guess is open the + stack waits `expect_wait` for the work, and matching work links to it and + goes first; an expired or started guess links nothing, and work linked to + it drops back to its own priority. The guess never stops the rule for + nothing running. Another stack respects it while it is open, also one + started with no declared work, so such a stack waits at most until the + guess expires; a stack that still fits next to it starts at once. A + declaration wins; `--expect none` expects nothing; `infer_work = false` + turns it off. - `expect_wait` (5 seconds by default, 0 turns it off): a long-lived job that expects work, declared or from its history, waits that long before it starts for the work to queue, so a stack and a run started together start @@ -50,8 +55,10 @@ and the versions follow [semver](https://semver.org). - With nothing running, or nothing but long-lived jobs past their start-up, the first job in line that fits starts, up to all of RAM, rather than the first in line whatever the memory reading says. A long-lived job, such as a - second dev stack, starts that way only under `mem_max`. A job that does not - fit holds nobody back, so the jobs behind it that fit start. At macOS + second dev stack, starts that way only under `mem_max`, or, when only + programs outside taskguard keep it out, up to the ceiling `outside_admit` + allows. A job that does not fit holds nobody back, so the jobs behind it + that fit start. At macOS warning this rule still decides when only long-lived jobs run. - A waiting job that would not fit even once every job that ends by itself has ended holds nobody back, whatever its priority or reservation: it can diff --git a/README.md b/README.md index 3ae97f1..1f687e7 100644 --- a/README.md +++ b/README.md @@ -228,7 +228,9 @@ below). run is admitted against, and what it took after. Past its start-up it counts as load from outside taskguard: no job waits for it to end, and when nothing else runs, the first in line that fits starts. A waiting long-lived - job itself starts that way only under `mem_max`. `auto_pause` leaves it + job itself starts that way only under `mem_max`, or, when only programs + outside taskguard keep it out, up to the ceiling `outside_admit` allows: + nothing taskguard runs would free that memory. `auto_pause` leaves it alone. `taskguard status` marks it `STEADY`, and a job it keeps out says it starts when the stack is stopped, not when it finishes. - **Work for a long-lived job goes first.** The caller that starts a dev @@ -251,22 +253,29 @@ below). stack runs, hold their turn as a strict reservation, also while they do not fit yet: lints that fit do not take the room meanwhile, unless the stack could only start once another stack is stopped. A stack started without - `--expect` behaves as before; the variable means nothing to any other job. + `--expect` books only what its history infers (below); the variable means + nothing to any other job. - **Without a declaration, the history says what work follows a stack.** When at least `infer_work_min` (3) of a stack's last `infer_work_of` (5) runs in its checkout had work of one kind queue there within `infer_work_within` (900) seconds of their start, that work is expected again. A kind of work is a pool, or the key of a job outside any pool. The stack books that work's usual need, which `taskguard status` shows as - `(expects browser from history, books 4.3 GB)`, and the work links and - goes first as declared work does. It is a guess, so it is bounded: + `(expects browser from history, books 4.3 GB)`, and while the guess is + open the work links and goes first as declared work does. It is a guess, + so it is bounded: - it books room for twice the work's usual delay after the stack started, ten minutes at most, and drops it once work for the stack starts or the stack stops; - - no job waits for it by the rule for nothing running, so it can never - stop the queue, and a job waits for it at most up to all of RAM; - - it never keeps out a stack whose caller declared no work, its own guess - included: such a stack starts as it would without the history; + - once it has expired or come true, no work links to it, and work linked + to it drops back to its own priority and holds its turn no more; + - no job that ends by itself waits for it by the rule for nothing running, + so it can never stop the queue, and such a job waits for it at most up + to all of RAM; + - another stack respects it while it is open, also one whose caller + declared no work, so the room is there when the work comes. Such a stack + leaves only its own guess out, starts at once when it still fits next to + the open guess, and otherwise waits at most until the guess expires; - a declaration always wins: a stack with `--expect` books only what was declared, and `--expect none` expects nothing at all; - `infer_work = false` turns it off. diff --git a/src/queue.rs b/src/queue.rs index ce2cc08..a712140 100644 --- a/src/queue.rs +++ b/src/queue.rs @@ -835,6 +835,12 @@ fn booking_taken_over(job: &Entry, running: &[Entry]) -> u64 { stack.expects.get(*i as usize).filter(|x| x.open() && !taken).map_or(0, |x| x.mem_kb) } +/// The part of `booking_taken_over` that only the history guessed. +fn guess_taken_over(job: &Entry, running: &[Entry]) -> u64 { + let item = job.work_for.as_ref().and_then(|w| running.iter().find(|e| e.pid == w.pid)?.expects.get(w.item? as usize)); + if item.is_some_and(|x| x.implied) { booking_taken_over(job, running) } else { 0 } +} + /// Is `job` work for a long-lived job that does not run yet? It waits for /// it: the work needs it up. pub fn waits_for_stack(job: &Entry, running: &[Entry]) -> bool { @@ -858,7 +864,7 @@ pub fn link_work(me: &Entry, running: &[Entry], waiting: &[Entry], matches: impl let stacks = running.iter().chain(waiting).filter(|e| e.long_lived && e.pid != me.pid && e.checkout == me.checkout); let mut best: Option<(u8, WorkFor, i32)> = None; for e in stacks { - for (i, x) in e.expects.iter().enumerate().filter(|(_, x)| matches(&x.work)) { + for (i, x) in e.expects.iter().enumerate().filter(|(_, x)| matches(&x.work) && still_expected(x)) { let free = x.open() && !claimed(e.pid, i); // A stack that waits takes only work that can take over a booking. if e.started_at.is_none() && !free { @@ -874,6 +880,23 @@ pub fn link_work(me: &Entry, running: &[Entry], waiting: &[Entry], matches: impl best.map(|(_, w, p)| (w, p)) } +/// Is this work still expected: declared, or a guess that has neither +/// expired nor come true? Only then does work link to it and go first. +fn still_expected(x: &Expect) -> bool { + !x.implied || x.open() +} + +/// Has `me`'s link to its long-lived job lapsed: that job is gone, or what +/// it expected of `me` was a guess that has expired or come true? +pub fn link_lapsed(me: &Entry, running: &[Entry], waiting: &[Entry], matches: impl Fn(&str) -> bool) -> bool { + let Some(w) = &me.work_for else { return false }; + running + .iter() + .chain(waiting) + .find(|e| e.pid == w.pid) + .is_none_or(|e| !e.expects.iter().any(|x| matches(&x.work) && still_expected(x))) +} + /// Does `a` belong to an older task-runner run than `b`? Only jobs that are /// both in a run, in different runs, are ordered this way. pub fn runs_before(a: &Entry, b: &Entry) -> bool { @@ -977,6 +1000,8 @@ pub fn decide( // fit under `mem_max`: it does not end by itself, so starting it over the // limit is no progress but load that stays, and a second dev stack // started that way kept the first one's run out until it was given up. + // Only when programs outside taskguard alone keep it out may it go up to + // the ceiling of `outside_admit`: nothing taskguard runs would free that. let room = Room::new(m, lim, running); if running.iter().all(Entry::steady) { let fits = |w: &Entry| room.fits_on_progress(m, lim, running, w); @@ -1338,23 +1363,32 @@ impl Room { } /// As `mem_would`, by the progress rule (`progress`) or not. A guess of - /// the history never keeps a job out by the progress rule, so it cannot - /// stop the queue. Nor does it keep out a long-lived job started with no - /// declared work, its own guess included: such a stack starts as it - /// would without the history, and books its guess from then on. + /// the history never keeps a job that ends by itself out by the progress + /// rule, so it cannot stop the queue; the part of it that job takes over + /// counts once. A long-lived job respects the guesses of others while + /// they are open, but one started with no declared work leaves its own + /// guess out: such a stack starts as it would without the history, and + /// books its guess from then on. fn mem_would_on(&self, job: &Entry, progress: bool) -> f64 { let undeclared = job.long_lived && !job.expects.iter().any(|x| !x.implied); - let guess = if (progress && !job.long_lived) || undeclared { self.res_guessed } else { 0 }; + let guess = if progress && !job.long_lived { self.res_guessed.saturating_sub(guess_taken_over(job, &self.running)) } else { 0 }; let own = if undeclared && job.started_at.is_none() { guessed(job, &[]) } else { 0 }; (self.mem_used + self.res_mem - guess.min(self.res_mem) + need_with_booking(job, &self.running) - own) as f64 } /// The most memory a job may take the machine to by the progress rule: /// all of RAM, as nothing else of taskguard's would come after it; a - /// long-lived job, which stays, only `mem_max` (with `noise_mem`). + /// long-lived job, which stays, only `mem_max` (with `noise_mem`), or + /// when only programs outside taskguard keep it out, the ceiling of + /// `outside_admit`: nothing taskguard runs would free that. fn progress_ceiling(&self, m: &MachineSample, lim: &Limits, job: &Entry) -> f64 { let total = m.mem_total_kb as f64; - if job.long_lived { self.mem_limit + total * lim.noise_mem_pct / 100.0 } else { total } + if !job.long_lived { + return total; + } + let ceiling = self.mem_limit + total * lim.noise_mem_pct / 100.0; + let ours = self.mem_would_on(job, true) - self.mem_used.saturating_sub(self.ours_mem) as f64; + if lim.outside_admit && ours <= self.mem_limit { ceiling.max(total * lim.outside_mem_max_pct / 100.0).min(total) } else { ceiling } } /// May `job` start by the progress rule now: its pool has a free slot @@ -1374,6 +1408,7 @@ impl Room { let lasting: Vec = running.iter().filter(|e| e.lasting()).cloned().collect(); let drained = Room { mem_used: self.mem_used.saturating_sub(self.passing_live), + ours_mem: self.ours_mem.saturating_sub(self.passing_live), res_mem: self.res_mem.saturating_sub(self.passing_res), ..self.clone() }; @@ -2696,21 +2731,72 @@ mod tests { // While a job that ends by itself runs, the guess holds like a booking. let busy = [stack.clone(), Entry { started_at: Some(990.0), ..job(5, "build", 0.5, 0) }]; assert_eq!(blockers(decide(&m, &lim, &busy, std::slice::from_ref(&lint), &lint, 1000.0, &[])), vec!["memory"]); - // A second stack whose caller declared work respects the guess. One - // started with no declared work does not, nor its own guess: it - // starts as it would without the history. + // A second stack respects the open guess, declared work or not (24 + + // 5.3 + 4 GB is over 30.4); one with no declared work leaves only its + // own guess out. let plain = Entry { long_lived: true, priority: 10, ..job(6, "dev-env:stack", 1.0, 4) }; - assert!(blockers(decide(&m, &lim, r, std::slice::from_ref(&plain), &plain, 1000.0, &[])).is_empty()); + assert_eq!(blockers(decide(&m, &lim, r, std::slice::from_ref(&plain), &plain, 1000.0, &[])), vec!["memory"]); let mut guessing = Entry { checkout: "/w/b".into(), ..plain.clone() }; guessing.expects = vec![Expect { work: "browser".into(), mem_kb: GB, implied: true, ..Default::default() }]; - assert!(blockers(decide(&m, &lim, r, std::slice::from_ref(&guessing), &guessing, 1000.0, &[])).is_empty()); + assert_eq!(blockers(decide(&m, &lim, r, std::slice::from_ref(&guessing), &guessing, 1000.0, &[])), vec!["memory"]); let mut declared = guessing.clone(); declared.expects[0].implied = false; assert_eq!(blockers(decide(&m, &lim, r, std::slice::from_ref(&declared), &declared, 1000.0, &[])), vec!["memory"]); - // Expired, it books nothing, and a run for the stack still goes first. + // A run that linked to the open guess keeps its link. + let linked = Entry { work_for: Some(WorkFor { pid: 11, key: "dev-env:stack".into(), item: Some(0) }), ..me.clone() }; + assert!(!link_lapsed(&linked, r, &[], |w| w == "browser")); + // Expired, it books nothing: the stack with no declared work starts + // (24 + 1 + 4 GB), and no run links to the guess or keeps its link. stack.expects[0].expired = true; - assert_eq!(reserve(std::slice::from_ref(&stack)).1, grows); - assert_eq!(link_work(&me, std::slice::from_ref(&stack), &[], |w| w == "browser").unwrap().0.item, None); + let r = std::slice::from_ref(&stack); + assert_eq!(reserve(r).1, grows); + assert!(blockers(decide(&m, &lim, r, std::slice::from_ref(&plain), &plain, 1000.0, &[])).is_empty()); + assert!(blockers(decide(&m, &lim, r, std::slice::from_ref(&guessing), &guessing, 1000.0, &[])).is_empty()); + assert!(link_work(&me, r, &[], |w| w == "browser").is_none()); + assert!(link_lapsed(&linked, r, &[], |w| w == "browser")); + // Nor once it came true. + stack.expects[0] = Expect { expired: false, started: true, ..stack.expects[0].clone() }; + let r = std::slice::from_ref(&stack); + assert!(link_work(&me, r, &[], |w| w == "browser").is_none()); + assert!(link_lapsed(&linked, r, &[], |w| w == "browser")); + // Declared work that has started still links: the stack exists for it. + stack.expects[0].implied = false; + let r = std::slice::from_ref(&stack); + assert_eq!(link_work(&me, r, &[], |w| w == "browser").unwrap().0.item, None); + assert!(!link_lapsed(&linked, r, &[], |w| w == "browser")); + } + + #[test] + fn a_run_takes_over_a_guessed_booking_once_by_the_progress_rule() { + let lim = Limits { mem_max_pct: 95.0, ..LIM }; + let mut stack = Entry { pid: 11, ..expecting(steady_stack(11, 7)) }; + stack.expects[0].implied = true; + let r = std::slice::from_ref(&stack); + let run = run_for(Some(0)); + // 29 GB held, 1 still to grow and the 4.3 GB the run takes over: + // 34.3 GB of 32. The guess it takes over counts once, not as nothing. + assert_eq!(blockers(decide(&machine(1.0, 29), &lim, r, std::slice::from_ref(&run), &run, 1000.0, &[])), vec!["memory"]); + // 26 GB held: 31.3 GB, under all of RAM, and it starts. + assert!(blockers(decide(&machine(1.0, 26), &lim, r, std::slice::from_ref(&run), &run, 1000.0, &[])).is_empty()); + } + + #[test] + fn a_lone_stack_is_kept_out_by_other_programs_only_up_to_the_outside_admit_ceiling() { + // 32 GB, mem_max 85 (27.2 GB), outside_admit up to 95% (30.4 GB). Nothing + // of taskguard runs; other programs hold 20 GB and a 9 GB stack would + // take the machine to 29 GB: only they keep it out, and nothing + // taskguard runs would free that. It starts as outside_admit allows. + let lim = Limits { outside_admit: true, ..LIM }; + let stack = Entry { long_lived: true, priority: 10, ..job(1, "dev-env:stack", 3.0, 9) }; + let me = std::slice::from_ref(&stack); + assert!(blockers(decide(&machine(1.0, 20), &lim, &[], me, &stack, 1000.0, &[])).is_empty()); + // Past that ceiling memory stays a hard rule; without the leeway it waits. + assert_eq!(blockers(decide(&machine(1.0, 22), &lim, &[], me, &stack, 1000.0, &[])), vec!["memory"]); + assert_eq!(blockers(decide(&machine(1.0, 20), &LIM, &[], me, &stack, 1000.0, &[])), vec!["memory"]); + // When a steady stack of taskguard's fills the room, outside programs + // are no excuse: 16 + 4 + 9 GB of its own is over 27.2. + let a = steady_stack(12, 16); + assert_eq!(blockers(decide(&machine(1.0, 17), &lim, std::slice::from_ref(&a), me, &stack, 1000.0, &[])), vec!["memory"]); } #[test] diff --git a/src/runner.rs b/src/runner.rs index 6728e8a..e6882e5 100644 --- a/src/runner.rs +++ b/src/runner.rs @@ -166,12 +166,12 @@ fn apply_nudge(me: &mut Entry, n: &queue::Nudge, database: Option<&Db>) { } /// Link a waiting job to the long-lived job in its checkout that expects it, -/// or drop the link when that job is gone. Work for a long-lived job goes -/// one priority above it, so it starts before the jobs that queued for room +/// or drop the link when that job is gone, or only guessed the work and the +/// guess has expired or come true. Work for a long-lived job goes one +/// priority above it, so it starts before the jobs that queued for room /// meanwhile, and before a next stack. fn link_to_stack(me: &mut Entry, own_priority: i32, running: &[Entry], waiting: &[Entry], cmd: &[String], quiet: bool) { - let gone = me.work_for.as_ref().is_some_and(|w| !running.iter().chain(waiting).any(|e| e.pid == w.pid)); - if gone { + if queue::link_lapsed(me, running, waiting, |w| is_work(w, me, cmd)) { me.work_for = None; me.priority = own_priority; } diff --git a/src/stall.rs b/src/stall.rs index 876680f..f87e007 100644 --- a/src/stall.rs +++ b/src/stall.rs @@ -21,7 +21,7 @@ //! `cargo test stall -- --nocapture --test-threads 1` prints the tables. use crate::machine::{MachineSample, PEAK_WINDOW}; -use crate::queue::{Decision, Entry, Expect, Limits, LongLived, decide, link_work}; +use crate::queue::{Decision, Entry, Expect, Limits, LongLived, decide, link_lapsed, link_work}; const GB: u64 = 1024 * 1024; const TICK: f64 = 1.0; @@ -495,7 +495,16 @@ pub fn run(s: &Scenario) -> Outcome { let run_now: Vec = running.iter().map(|r| r.e.clone()).collect(); let all: Vec = waiting.iter().map(|(e, _)| e.clone()).collect(); // As the runner does: work links to the stack that expects it, - // and goes one priority above it. + // and goes one priority above it, until the stack is gone or only + // guessed it and the guess has expired or come true. + let me = waiting[i].0.clone(); + if link_lapsed(&me, &run_now, &all, |w| me.pool.as_deref() == Some(w)) { + waiting[i].0.work_for = None; + waiting[i].0.priority = match waiting[i].1 { + Kind::Job(n) => s.jobs[n].priority, + _ => 0, + }; + } if waiting[i].0.work_for.is_none() { let me = waiting[i].0.clone(); if let Some((link, p)) = link_work(&me, &run_now, &all, |w| me.pool.as_deref() == Some(w)) { @@ -761,7 +770,7 @@ mod tests { assert_eq!(declared.runs_given_up, 0, "{outside} GB declared: {declared:?}"); assert!(declared.runs_done >= 10, "{outside} GB declared: {declared:?}"); assert!(declared.worst_mem_pct <= 95.0 && declared.over_limit_s == 0.0, "{outside} GB declared: {declared:?}"); - // Nor does one whose run the history infers. + // Nor does one that does not declare, whose run the history infers. assert_eq!(inferred.runs_given_up, 0, "{outside} GB inferred: {inferred:?}"); assert!(inferred.worst_mem_pct <= 95.0, "{outside} GB inferred: {inferred:?}"); // A caller that does not declare: the progress rule alone clears @@ -826,8 +835,9 @@ mod tests { // or take a minute, though fewer run while a guess books room. assert!(guessed.worst_lint_wait <= plain.worst_lint_wait + 60.0, "{outside} GB: {guessed:?} vs {plain:?}"); assert!(guessed.lints_done * 10 >= plain.lints_done * 7, "{outside} GB: {guessed:?} vs {plain:?}"); - // It never keeps a stack started with no work out. - assert!(guessed.worst_stack_wait <= plain.worst_stack_wait + 60.0, "{outside} GB: {guessed:?} vs {plain:?}"); + // A stack started with no work respects the other stack's open + // guess: it waits at most until that guess expires. + assert!(guessed.worst_stack_wait <= plain.worst_stack_wait + 600.0 + 60.0, "{outside} GB: {guessed:?} vs {plain:?}"); } } From 460f7638adfe86705c1b7e725fcc24ff2db5095d Mon Sep 17 00:00:00 2001 From: Vincent De Snerck <10668+SpoBo@users.noreply.github.com> Date: Fri, 16 Oct 2026 12:58:47 +0200 Subject: [PATCH 10/22] no-mistakes(review): Keep came-true guesses booked; lone job may use all RAM --- CHANGELOG.md | 26 +++++------ README.md | 13 +++--- src/queue.rs | 116 +++++++++++++++++++++++++++++++++++++++----------- src/runner.rs | 17 ++------ src/stall.rs | 8 ++-- 5 files changed, 120 insertions(+), 60 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7faaee5..ae20760 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -39,12 +39,13 @@ and the versions follow [semver](https://semver.org). drops it once work for the stack starts. While the guess is open the stack waits `expect_wait` for the work, and matching work links to it and goes first; an expired or started guess links nothing, and work linked to - it drops back to its own priority. The guess never stops the rule for - nothing running. Another stack respects it while it is open, also one - started with no declared work, so such a stack waits at most until the - guess expires; a stack that still fits next to it starts at once. A - declaration wins; `--expect none` expects nothing; `infer_work = false` - turns it off. + it drops back to its own priority. A guess whose work has queued and + linked to it came true: it does not expire while that work waits. The + guess never stops the rule for nothing running. Another stack respects it + while it is open, also one started with no declared work, so such a stack + waits at most until the guess expires or its work starts; a stack that + still fits next to it starts at once. A declaration wins; `--expect none` + expects nothing; `infer_work = false` turns it off. - `expect_wait` (5 seconds by default, 0 turns it off): a long-lived job that expects work, declared or from its history, waits that long before it starts for the work to queue, so a stack and a run started together start @@ -54,12 +55,13 @@ and the versions follow [semver](https://semver.org). - With nothing running, or nothing but long-lived jobs past their start-up, the first job in line that fits starts, up to all of RAM, rather than the - first in line whatever the memory reading says. A long-lived job, such as a - second dev stack, starts that way only under `mem_max`, or, when only - programs outside taskguard keep it out, up to the ceiling `outside_admit` - allows. A job that does not fit holds nobody back, so the jobs behind it - that fit start. At macOS - warning this rule still decides when only long-lived jobs run. + first in line whatever the memory reading says. Next to long-lived jobs, a + long-lived job, such as a second dev stack, starts that way only under + `mem_max`, or, when only programs outside taskguard keep it out, up to the + ceiling `outside_admit` allows. With nothing of taskguard's running, it + starts up to all of RAM, as before. A job that does not fit holds nobody + back, so the jobs behind it that fit start. At macOS warning this rule + still decides when only long-lived jobs run. - A waiting job that would not fit even once every job that ends by itself has ended holds nobody back, whatever its priority or reservation: it can only start once a long-lived job is stopped or other programs free memory, diff --git a/README.md b/README.md index 1f687e7..6ad62e1 100644 --- a/README.md +++ b/README.md @@ -228,9 +228,10 @@ below). run is admitted against, and what it took after. Past its start-up it counts as load from outside taskguard: no job waits for it to end, and when nothing else runs, the first in line that fits starts. A waiting long-lived - job itself starts that way only under `mem_max`, or, when only programs - outside taskguard keep it out, up to the ceiling `outside_admit` allows: - nothing taskguard runs would free that memory. `auto_pause` leaves it + job itself starts that way next to it only under `mem_max`, or, when only + programs outside taskguard keep it out, up to the ceiling `outside_admit` + allows. With nothing of taskguard's running at all, it starts up to all of + RAM: nothing taskguard runs would free that memory. `auto_pause` leaves it alone. `taskguard status` marks it `STEADY`, and a job it keeps out says it starts when the stack is stopped, not when it finishes. - **Work for a long-lived job goes first.** The caller that starts a dev @@ -275,13 +276,15 @@ below). - another stack respects it while it is open, also one whose caller declared no work, so the room is there when the work comes. Such a stack leaves only its own guess out, starts at once when it still fits next to - the open guess, and otherwise waits at most until the guess expires; + the open guess, and otherwise waits at most until the guess expires or + its work starts; - a declaration always wins: a stack with `--expect` books only what was declared, and `--expect none` expects nothing at all; - `infer_work = false` turns it off. Once the work has queued, its room is no guess any more and is booked as - declared work is. + declared work is: it does not expire while that work waits, and it ends + when the work starts, or, past its time, when the work leaves the queue. - **A stack waits a few seconds for its work.** A caller may start the stack and its work at the same moment, not one after the other. So a long-lived job that expects work, declared or from its history, waits up to diff --git a/src/queue.rs b/src/queue.rs index a712140..f1d3675 100644 --- a/src/queue.rs +++ b/src/queue.rs @@ -156,6 +156,27 @@ impl Expect { pub fn open(&self) -> bool { !self.started && !self.expired } + + /// A guess whose time is up, for a job that started at `started`. + pub fn due(&self, started: f64, now: f64) -> bool { + self.open() && self.expire_after_s.is_some_and(|a| now - started > a) + } +} + +/// A guess of the history that did not come true in time: its booking ends. +/// One whose work has queued and linked to it came true: it stays booked +/// while that work waits, until the work starts or leaves the queue. +pub fn expire_guesses(e: &mut Entry, waiting: &[Entry], now: f64) -> bool { + let (pid, started) = (e.pid, e.started_at.unwrap_or(now)); + let linked = |i: usize| waiting.iter().any(|w| w.work_for.as_ref().is_some_and(|l| l.pid == pid && l.item == Some(i as u32))); + let mut changed = false; + for (i, x) in e.expects.iter_mut().enumerate() { + if x.due(started, now) && !linked(i) { + x.expired = true; + changed = true; + } + } + changed } /// The long-lived job a job is work for, and which of its `expects` it @@ -996,12 +1017,14 @@ pub fn decide( // hold up the whole queue until it is stopped. // // A job that does not fit even so holds nobody back: it waits for memory, - // and the jobs behind it that fit start meanwhile. A long-lived job must - // fit under `mem_max`: it does not end by itself, so starting it over the - // limit is no progress but load that stays, and a second dev stack - // started that way kept the first one's run out until it was given up. - // Only when programs outside taskguard alone keep it out may it go up to - // the ceiling of `outside_admit`: nothing taskguard runs would free that. + // and the jobs behind it that fit start meanwhile. Next to long-lived + // jobs, a long-lived job must fit under `mem_max`: it does not end by + // itself, so starting it over the limit is no progress but load that + // stays, and a second dev stack started that way kept the first one's + // run out until it was given up. Only when programs outside taskguard + // alone keep it out may it go up to the ceiling of `outside_admit`, and + // with nothing of taskguard's running, up to all of RAM: nothing taskguard + // runs would free that. let room = Room::new(m, lim, running); if running.iter().all(Entry::steady) { let fits = |w: &Entry| room.fits_on_progress(m, lim, running, w); @@ -1377,13 +1400,15 @@ impl Room { } /// The most memory a job may take the machine to by the progress rule: - /// all of RAM, as nothing else of taskguard's would come after it; a - /// long-lived job, which stays, only `mem_max` (with `noise_mem`), or - /// when only programs outside taskguard keep it out, the ceiling of - /// `outside_admit`: nothing taskguard runs would free that. + /// all of RAM, as nothing else of taskguard's would come after it. A + /// long-lived job, which stays, next to long-lived jobs of taskguard's, + /// only `mem_max` (with `noise_mem`), or when only programs outside + /// taskguard keep it out, the ceiling of `outside_admit`. With nothing of + /// taskguard's running, it too goes up to all of RAM: nothing taskguard + /// runs would free the rest. fn progress_ceiling(&self, m: &MachineSample, lim: &Limits, job: &Entry) -> f64 { let total = m.mem_total_kb as f64; - if !job.long_lived { + if !job.long_lived || self.running.is_empty() { return total; } let ceiling = self.mem_limit + total * lim.noise_mem_pct / 100.0; @@ -1410,6 +1435,7 @@ impl Room { mem_used: self.mem_used.saturating_sub(self.passing_live), ours_mem: self.ours_mem.saturating_sub(self.passing_live), res_mem: self.res_mem.saturating_sub(self.passing_res), + running: lasting.clone(), ..self.clone() }; drained.fits_on_progress(m, lim, &lasting, job) @@ -2493,9 +2519,10 @@ mod tests { // The run fits under all of RAM (30 GB of 32), and starts. let d = decide(&m, &lim, r, &waiting, &run_a, 1000.0, &[]); assert!(matches!(&d, Decision::Admit { reason, .. } if reason.contains("long-lived")), "{d:?}"); - // With nothing running at all, a stack still has to fit under mem_max. - assert_eq!(blockers(decide(&machine(1.0, 22), &lim, &[], &waiting, &stack_b, 1000.0, &[])), vec!["memory"]); - assert!(blockers(decide(&machine(1.0, 20), &lim, &[], &waiting, &stack_b, 1000.0, &[])).is_empty()); + // With nothing running at all, a stack starts up to all of RAM: 22 + + // 9.4 GB fits in 32, 23 + 9.4 does not. + assert!(blockers(decide(&machine(1.0, 22), &lim, &[], &waiting, &stack_b, 1000.0, &[])).is_empty()); + assert_eq!(blockers(decide(&machine(1.0, 23), &lim, &[], &waiting, &stack_b, 1000.0, &[])), vec!["memory"]); // At macOS warning the same holds: the run starts next to its stack. let mut warn = m.clone(); warn.mem_pressure = Some(50.0); @@ -2781,24 +2808,65 @@ mod tests { } #[test] - fn a_lone_stack_is_kept_out_by_other_programs_only_up_to_the_outside_admit_ceiling() { - // 32 GB, mem_max 85 (27.2 GB), outside_admit up to 95% (30.4 GB). Nothing - // of taskguard runs; other programs hold 20 GB and a 9 GB stack would - // take the machine to 29 GB: only they keep it out, and nothing - // taskguard runs would free that. It starts as outside_admit allows. - let lim = Limits { outside_admit: true, ..LIM }; + fn a_lone_stack_is_never_kept_out_by_other_programs_alone() { + // 32 GB, mem_max 85 (27.2 GB). Nothing of taskguard runs; other + // programs hold 22 GB and a 9 GB stack would take the machine to 31 + // GB: only they keep it out, and nothing taskguard runs would free + // that. It starts up to all of RAM, whatever outside_admit and its + // ceiling say. let stack = Entry { long_lived: true, priority: 10, ..job(1, "dev-env:stack", 3.0, 9) }; let me = std::slice::from_ref(&stack); - assert!(blockers(decide(&machine(1.0, 20), &lim, &[], me, &stack, 1000.0, &[])).is_empty()); - // Past that ceiling memory stays a hard rule; without the leeway it waits. - assert_eq!(blockers(decide(&machine(1.0, 22), &lim, &[], me, &stack, 1000.0, &[])), vec!["memory"]); - assert_eq!(blockers(decide(&machine(1.0, 20), &LIM, &[], me, &stack, 1000.0, &[])), vec!["memory"]); + for lim in [ + LIM, + Limits { outside_admit: true, ..LIM }, + Limits { outside_admit: true, outside_mem_max_pct: 92.0, ..LIM }, + Limits { outside_admit: false, outside_mem_max_pct: 50.0, ..LIM }, + ] { + assert!(blockers(decide(&machine(1.0, 22), &lim, &[], me, &stack, 1000.0, &[])).is_empty(), "{lim:?}"); + // Never past all of RAM: 24 + 9 GB is over 32. + assert_eq!(blockers(decide(&machine(1.0, 24), &lim, &[], me, &stack, 1000.0, &[])), vec!["memory"], "{lim:?}"); + } + // Next to a steady stack, only programs outside taskguard keep it out + // (4 + 1 + 9 GB of taskguard's is under 27.2): it starts up to the + // outside_admit ceiling (30.4 GB), and without that leeway waits. + let small = steady_stack(12, 4); + let r = std::slice::from_ref(&small); + let lim = Limits { outside_admit: true, ..LIM }; + assert!(blockers(decide(&machine(1.0, 20), &lim, r, me, &stack, 1000.0, &[])).is_empty()); + assert_eq!(blockers(decide(&machine(1.0, 21), &lim, r, me, &stack, 1000.0, &[])), vec!["memory"]); + assert_eq!(blockers(decide(&machine(1.0, 20), &LIM, r, me, &stack, 1000.0, &[])), vec!["memory"]); // When a steady stack of taskguard's fills the room, outside programs // are no excuse: 16 + 4 + 9 GB of its own is over 27.2. let a = steady_stack(12, 16); assert_eq!(blockers(decide(&machine(1.0, 17), &lim, std::slice::from_ref(&a), me, &stack, 1000.0, &[])), vec!["memory"]); } + #[test] + fn a_guess_whose_work_queued_stays_while_that_work_waits() { + // The history says the run follows about 6 minutes after the stack + // starts: the guess expires 10 minutes in. The run queues at minute 6 + // and links; memory pressure keeps it out past minute 10. + let mut stack = Entry { pid: 11, ..expecting(steady_stack(11, 7)) }; + stack.expects[0].implied = true; + stack.expects[0].expire_after_s = Some(600.0); + let me = Entry { pool: Some("browser".into()), checkout: "/w/a".into(), queued_at: 360.0, ..job(3, "pr.ts", 3.0, 4) }; + let (link, priority) = link_work(&me, std::slice::from_ref(&stack), &[], |w| w == "browser").unwrap(); + assert_eq!((link.item, priority + 1), (Some(0), 11)); + let run = Entry { work_for: Some(link), priority: priority + 1, ..me }; + let waiting = std::slice::from_ref(&run); + // Minute 11: the guess came true, so it does not expire. + assert!(!expire_guesses(&mut stack, waiting, 660.0)); + let r = std::slice::from_ref(&stack); + assert!(!link_lapsed(&run, r, waiting, |w| w == "browser")); + assert_eq!(reserve(r).1, GB + 43 * GB / 10, "it keeps the room booked for the run"); + let mut m = machine(1.0, 20); + m.mem_pressure = Some(50.0); + assert_eq!(blockers(decide(&m, &LIM, r, waiting, &run, 660.0, &[])), vec!["pressure"]); + // Once the run has left the queue, the guess that is past its time ends. + assert!(expire_guesses(&mut stack, &[], 660.0)); + assert_eq!(reserve(std::slice::from_ref(&stack)).1, GB); + } + #[test] fn backfill_keeps_pools_and_pressure() { let (mut running, mut head) = blocked_head(); diff --git a/src/runner.rs b/src/runner.rs index e6882e5..433a0f0 100644 --- a/src/runner.rs +++ b/src/runner.rs @@ -84,17 +84,6 @@ fn apply_work_started(me: &mut Entry, n: &queue::Nudge) -> bool { changed } -/// A guess of the history that did not come true in time: its booking ends. -fn expire_guesses(me: &mut Entry, now: f64) -> bool { - let started = me.started_at.unwrap_or(now); - let mut changed = false; - for x in me.expects.iter_mut().filter(|x| x.open() && x.expire_after_s.is_some_and(|a| now - started > a)) { - x.expired = true; - changed = true; - } - changed -} - /// The longest a booking on the history's guess lasts after its job started. const GUESS_MAX_S: f64 = 600.0; @@ -918,7 +907,7 @@ pub fn run(mut o: Opts) -> Result { say(&format!("paused {} - by hand; it stays paused until taskguard resume, or p in taskguard top", me.key)); } } - if expire_guesses(&mut me, now) { + if me.expects.iter().any(|x| x.due(me.started_at.unwrap_or(now), now)) && queue::expire_guesses(&mut me, &q.waiting(), now) { let _ = q.write(&q.run_path(me_pid), &me); say(&format!("expected {} - the work its history suggested did not come in time; it no longer books room for it", me.key)); } @@ -1321,8 +1310,8 @@ mod tests { let declared = queue::Expect { work: "e2e".into(), mem_kb: 4, ..Default::default() }; let stack = Entry { started_at: Some(1000.0), expects: vec![declared, guess], ..Default::default() }; let mut s = stack.clone(); - assert!(!expire_guesses(&mut s, 1600.0)); - assert!(expire_guesses(&mut s, 1601.0)); + assert!(!queue::expire_guesses(&mut s, &[], 1600.0)); + assert!(queue::expire_guesses(&mut s, &[], 1601.0)); assert!(!s.expects[0].expired && s.expects[1].expired, "only the guess expires"); // Work for the stack started: the guess goes with it. let mut s = stack; diff --git a/src/stall.rs b/src/stall.rs index f87e007..d5aaf7f 100644 --- a/src/stall.rs +++ b/src/stall.rs @@ -21,7 +21,7 @@ //! `cargo test stall -- --nocapture --test-threads 1` prints the tables. use crate::machine::{MachineSample, PEAK_WINDOW}; -use crate::queue::{Decision, Entry, Expect, Limits, LongLived, decide, link_lapsed, link_work}; +use crate::queue::{Decision, Entry, Expect, Limits, LongLived, decide, expire_guesses, link_lapsed, link_work}; const GB: u64 = 1024 * 1024; const TICK: f64 = 1.0; @@ -419,6 +419,7 @@ pub fn run(s: &Scenario) -> Outcome { } } // What each running job holds and uses now. + let queued: Vec = waiting.iter().map(|(e, _)| e.clone()).collect(); for r in running.iter_mut() { let age = t - r.started; let (mem, cpu) = match r.kind { @@ -453,10 +454,7 @@ pub fn run(s: &Scenario) -> Outcome { r.e.follow(mem, cpu, s.host.ncpu as f64, t, lived); // As the owner does: a guess of the history that did not come // true in time stops booking. - let age_now = t - r.started; - for x in r.e.expects.iter_mut().filter(|x| x.open() && x.expire_after_s.is_some_and(|a| age_now > a)) { - x.expired = true; - } + expire_guesses(&mut r.e, &queued, t); if s.steady_books_live && r.e.steady() { r.e.need_mem_kb = mem; } From ea521721beaae779ec05825d4b0a88053de9130b Mon Sep 17 00:00:00 2001 From: Vincent De Snerck <10668+SpoBo@users.noreply.github.com> Date: Fri, 16 Oct 2026 13:18:49 +0200 Subject: [PATCH 11/22] no-mistakes(test): Book a pool's typical start need for declared stack work --- CHANGELOG.md | 4 ++-- README.md | 7 ++++--- src/db.rs | 19 +++++++++++++++++++ src/need.rs | 43 +++++++++++++++++++++++++++++++++++++------ src/stall.rs | 8 +++----- 5 files changed, 65 insertions(+), 16 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ae20760..0b0d76f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -24,8 +24,8 @@ and the versions follow [semver](https://semver.org). - A long-lived job can declare the work that will run on it: `--expect WORK` (repeatable) or `TASKGUARD_EXPECT=WORK,WORK`, where WORK is a pool name, a job key or a command pattern. The stack books room for that - work (learned as the work learns its own need) and starts only with room - for both. A job in the same checkout that matches is its work: it goes one + work (learned as the work learns its own need; for a pool, the need its + jobs typically started with) and starts only with room for both. A job in the same checkout that matches is its work: it goes one priority above the stack, takes over the booked room when it starts, and waits while its stack still waits. `taskguard status` shows `(for stack KEY)` on the work and `(expects WORK, books SIZE)` on the stack. Without a diff --git a/README.md b/README.md index 6ad62e1..10b6354 100644 --- a/README.md +++ b/README.md @@ -240,15 +240,16 @@ below). (repeatable) or `TASKGUARD_EXPECT=WORK,WORK` in its environment, so it passes through scripts such as `devenv up`. WORK is a pool name, a job key, or a command pattern as in `[[job]] match`. The stack then books room for - each piece of work, learned as that work learns its own need: what jobs of - the pool typically took, or the history or first guess of the command. It + each piece of work, learned as that work learns its own need: the need + jobs of the pool typically started with (their peak with headroom, or a + minimum), or the history or first guess of the command. It starts only when the machine has room for itself and that work, so the whole load is known before it starts. A job that queues in the same checkout and matches WORK is that work: it goes one priority above the stack, so it starts before the jobs that queued for room meanwhile and before a next stack, and when it starts it takes over the room booked for it, which the stack then drops. `taskguard status` shows both ends: `(for - stack dev-env:stack)` on the work, `(expects browser, books 3.6 GB)` on the + stack dev-env:stack)` on the work, `(expects browser, books 4.3 GB)` on the stack. Work that queues while its stack still waits waits for the stack and holds nobody back meanwhile. A stack with work queued for it, and work whose stack runs, hold their turn as a strict reservation, also while they do not diff --git a/src/db.rs b/src/db.rs index 4284e6c..d232451 100644 --- a/src/db.rs +++ b/src/db.rs @@ -1054,6 +1054,25 @@ impl Db { Ok(None) } + /// The memory jobs of `pool` typically started with, over the last 30 + /// days: the 90th percentile of each key's highest need, and how many + /// keys that is. A need is what a job is admitted with (its peak with + /// headroom, or a minimum), so it is what room booked for such a job + /// must cover; their peaks would leave the difference unbooked. + pub fn pool_need(&self, pool: &str) -> Result> { + let mut stmt = self.conn.prepare_cached( + "SELECT max(need_mem_kb) FROM runs + WHERE pool = ?1 AND need_mem_kb > 0 AND started_at IS NOT NULL AND queued_at > ?2 + GROUP BY key", + )?; + let mut v: Vec = stmt.query_map(params![pool, now() - 30.0 * 86400.0], |r| r.get(0))?.collect::>()?; + if v.is_empty() { + return Ok(None); + } + v.sort(); + Ok(Some((v[((v.len() as f64 * 0.9) as usize).min(v.len() - 1)] as u64, v.len()))) + } + /// The work that followed a long-lived job before: for the last `of` /// runs of `key` in `checkout` (of the last two weeks), every job of /// another kind that queued in that checkout within `within` seconds of diff --git a/src/need.rs b/src/need.rs index ff4ea13..64a6015 100644 --- a/src/need.rs +++ b/src/need.rs @@ -188,15 +188,14 @@ pub fn guess(db: Option<&Db>, cfg: &Config, package: Option<&str>, argv: &[Strin /// What a long-lived job books for work it expects (`--expect WORK`): the /// need that work will start with, learned as the work itself learns it. /// WORK is a pool name, a job key, or a command (pattern) run from `cwd`. -/// A pool books what jobs of that pool typically took; a key or command -/// its history, else a first guess as for a new job, with the minimums of -/// the job rules it matches. +/// A pool books what jobs of that pool typically started with; a key or +/// command its history, else a first guess as for a new job, with the +/// minimums of the job rules it matches. pub fn expected(db: Option<&Db>, cfg: &Config, cwd: &Path, work: &str) -> crate::queue::Expect { let expect = |mem_kb: u64, from: String| crate::queue::Expect { work: work.to_string(), mem_kb, from, ..Default::default() }; if cfg.pools.iter().any(|p| p.name == work) { - let est = db.and_then(|d| d.estimate(None, None, "", Some(work)).ok().flatten()); - return match est.and_then(|e| e.mem_kb.map(|m| (m, e.from))) { - Some((m, from)) => expect(m, from), + return match db.and_then(|d| d.pool_need(work).ok().flatten()) { + Some((m, n)) => expect(m, format!("the need of {n} jobs in {work}")), None => expect(cfg.new_job_mem_kb, format!("no jobs in pool {work} yet: the new_job_mem default")), }; } @@ -320,6 +319,38 @@ mod tests { assert_eq!(n.steps[0].why, "no runs yet: typical of 3 tsc jobs"); } + #[test] + fn a_pool_of_expected_work_books_what_its_jobs_started_with() { + let tmp = tempfile::tempdir().unwrap(); + let db = Db::open_dir(tmp.path()).unwrap(); + // Browser runs peak at 3 GB, but start with 4.3 GB (a min_mem). + for i in 0..4 { + let key = format!("e2e:pr_{i}"); + let run = crate::db::NewRun { ns: "n", key: &key, pool: Some("browser"), need_mem_kb: 43 * GB / 10, ..Default::default() }; + let id = db.insert_run(&run).unwrap(); + db.mark_started(id, crate::db::now() - 60.0, 0.0, None).unwrap(); + let done = crate::db::RunResult { ended_at: crate::db::now(), peak_mem_kb: 3 * GB, measured: true, ..Default::default() }; + db.finish_run(id, &done).unwrap(); + } + let pool = crate::config::Pool { + name: "browser".into(), + max_slots: None, + per_checkout: true, + patterns: Vec::new(), + long_lived: false, + startup: None, + }; + let empty = crate::config::Pool { name: "storybook".into(), ..pool.clone() }; + let cfg = Config { pools: vec![pool, empty], ..Config::default() }; + let x = expected(Some(&db), &cfg, Path::new("/w"), "browser"); + // Booked at their peak, a run would find 1.3 GB of its room taken. + assert_eq!(x.mem_kb, 43 * GB / 10, "{}", x.from); + assert_eq!(x.from, "the need of 4 jobs in browser"); + // No jobs in the pool yet: the default for a new job. + let x = expected(Some(&db), &cfg, Path::new("/w"), "storybook"); + assert_eq!(x.mem_kb, cfg.new_job_mem_kb, "{}", x.from); + } + #[test] fn ignored_failed_runs_are_named() { let failed = crate::db::Outlier { run_id: 7, peak_kb: 14 * GB, ended_at: 0.0, exit: Some(1) }; diff --git a/src/stall.rs b/src/stall.rs index d5aaf7f..e737411 100644 --- a/src/stall.rs +++ b/src/stall.rs @@ -137,10 +137,9 @@ pub struct Agent { pub down_after_s: f64, pub pause_s: f64, /// The agent declares its browser run when it brings the stack up - /// (`TASKGUARD_EXPECT=browser`), and the stack books `booking_kb` for it: - /// what browser runs typically took, less than the run's own need. + /// (`TASKGUARD_EXPECT=browser`), and the stack books for it what browser + /// runs typically started with: the run's need. pub declares: bool, - pub booking_kb: u64, /// The agent queues its run 2 s after its stack, not once the stack is up. pub with_stack: bool, /// Undeclared, but the history says a browser run follows its stacks @@ -168,7 +167,6 @@ pub fn agent(first_at: f64, idle_gb: f64) -> Agent { down_after_s: 20.0, pause_s: 30.0, declares: false, - booking_kb: 36 * GB / 10, with_stack: false, implied: false, no_run: false, @@ -318,7 +316,7 @@ pub fn run(s: &Scenario) -> Outcome { Phase::Idle { until } if t >= until => { ticket += 1; let expects = if a.declares { - vec![Expect { work: "browser".into(), mem_kb: a.booking_kb, ..Default::default() }] + vec![Expect { work: "browser".into(), mem_kb: a.run_need_kb, ..Default::default() }] } else if a.implied { vec![Expect { work: "browser".into(), From 8d5e037a6d7278f3f06b513fce3f7574e02058b4 Mon Sep 17 00:00:00 2001 From: Vincent De Snerck <10668+SpoBo@users.noreply.github.com> Date: Fri, 16 Oct 2026 13:27:56 +0200 Subject: [PATCH 12/22] no-mistakes(test): Align stack-holds-turn test with ruled lone-job RAM ceiling --- src/queue.rs | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/src/queue.rs b/src/queue.rs index f1d3675..7b3d698 100644 --- a/src/queue.rs +++ b/src/queue.rs @@ -2731,9 +2731,16 @@ mod tests { // With its run queued for it, the lint waits: the room is theirs. let waiting = [stack.clone(), run_for(Some(0)), lint.clone()]; assert_eq!(blockers(decide(&m, &lim, &busy, &waiting, &lint, 1000.0, &[])), vec!["priority"]); - // Not when it could only start once a stack is stopped: 14 GB held - // by other programs and 13.3 GB more is over the 27.2 GB limit. + // Still when 14 GB is held by other programs: once the build has + // ended nothing of taskguard's runs, and a lone job may then take + // the machine up to all of RAM (14 + 13.3 of 32 GB), so the stack + // and its run will start, and the room is theirs. let m = machine(1.0, 14); + assert_eq!(blockers(decide(&m, &lim, &busy, &waiting, &lint, 1000.0, &[])), vec!["priority"]); + // Not when it could not start even then: 14 GB held and 22.3 GB more + // for a bigger stack and its run is over all of RAM. + let big = Entry { pid: 11, long_lived: true, priority: 10, ..expecting(job(2, "dev-env:stack", 3.0, 18)) }; + let waiting = [big, run_for(Some(0)), lint.clone()]; assert!(blockers(decide(&m, &lim, &busy, &waiting, &lint, 1000.0, &[])).is_empty()); } From c87d101ee6dd81ef1564683837ead355179f877f Mon Sep 17 00:00:00 2001 From: Vincent De Snerck <10668+SpoBo@users.noreply.github.com> Date: Fri, 16 Oct 2026 13:42:34 +0200 Subject: [PATCH 13/22] no-mistakes(document): Clarify lone-stack ceiling and guess-link wording; apply rustfmt --- README.md | 12 +++++++----- src/db.rs | 3 ++- src/queue.rs | 11 ++++------- 3 files changed, 13 insertions(+), 13 deletions(-) diff --git a/README.md b/README.md index 10b6354..11e143e 100644 --- a/README.md +++ b/README.md @@ -138,9 +138,10 @@ below). a wrong reading or on programs outside taskguard. The same holds when nothing runs but long-lived jobs past their start-up. A job that would not fit in RAM even so waits for memory and holds nobody back: the jobs behind - it that fit start. A long-lived job must fit under `mem_max` (with - `noise_mem`): it does not end by itself, so starting it over the limit is - no progress but load that stays. Memory pressure is the one exception: + it that fit start. Next to long-lived jobs, a long-lived job must fit under + `mem_max` (with `noise_mem`), or the ceiling `outside_admit` allows: it does + not end by itself, so starting it over the limit is no progress but load + that stays. Memory pressure is the one exception: then the load comes from other programs, and jobs wait until it eases (at macOS warning, see above). - **Any order, oldest first.** A small job may pass a big one that does not @@ -269,8 +270,9 @@ below). - it books room for twice the work's usual delay after the stack started, ten minutes at most, and drops it once work for the stack starts or the stack stops; - - once it has expired or come true, no work links to it, and work linked - to it drops back to its own priority and holds its turn no more; + - once it has expired or its work has started, no work links to it, and + work linked to it drops back to its own priority and holds its turn no + more; - no job that ends by itself waits for it by the rule for nothing running, so it can never stop the queue, and such a job waits for it at most up to all of RAM; diff --git a/src/db.rs b/src/db.rs index d232451..f46279c 100644 --- a/src/db.rs +++ b/src/db.rs @@ -1065,7 +1065,8 @@ impl Db { WHERE pool = ?1 AND need_mem_kb > 0 AND started_at IS NOT NULL AND queued_at > ?2 GROUP BY key", )?; - let mut v: Vec = stmt.query_map(params![pool, now() - 30.0 * 86400.0], |r| r.get(0))?.collect::>()?; + let mut v: Vec = + stmt.query_map(params![pool, now() - 30.0 * 86400.0], |r| r.get(0))?.collect::>()?; if v.is_empty() { return Ok(None); } diff --git a/src/queue.rs b/src/queue.rs index 7b3d698..09e890f 100644 --- a/src/queue.rs +++ b/src/queue.rs @@ -902,20 +902,17 @@ pub fn link_work(me: &Entry, running: &[Entry], waiting: &[Entry], matches: impl } /// Is this work still expected: declared, or a guess that has neither -/// expired nor come true? Only then does work link to it and go first. +/// expired nor seen its work start? Only then does work link to it and go +/// first. fn still_expected(x: &Expect) -> bool { !x.implied || x.open() } /// Has `me`'s link to its long-lived job lapsed: that job is gone, or what -/// it expected of `me` was a guess that has expired or come true? +/// it expected of `me` was a guess that has expired or whose work started? pub fn link_lapsed(me: &Entry, running: &[Entry], waiting: &[Entry], matches: impl Fn(&str) -> bool) -> bool { let Some(w) = &me.work_for else { return false }; - running - .iter() - .chain(waiting) - .find(|e| e.pid == w.pid) - .is_none_or(|e| !e.expects.iter().any(|x| matches(&x.work) && still_expected(x))) + running.iter().chain(waiting).find(|e| e.pid == w.pid).is_none_or(|e| !e.expects.iter().any(|x| matches(&x.work) && still_expected(x))) } /// Does `a` belong to an older task-runner run than `b`? Only jobs that are From 50795e83fdfec37830d3b0a438a1d0c935f71404 Mon Sep 17 00:00:00 2001 From: Vincent De Snerck <10668+SpoBo@users.noreply.github.com> Date: Fri, 16 Oct 2026 13:51:34 +0200 Subject: [PATCH 14/22] test: the stack-and-run CLI test books little memory, so it passes on a full macOS runner On macos-latest the test waited 195 s against its 15 s bound: the stack and its run are first runs that book new_job_mem (1.5 GB) each, and the runner reports most of its 7 GB in use. A lone stack that does not fit in all of RAM waits by the ruled hard ceiling, so the test measured the host's memory instead of the expect_wait logic. It now books 64 MB each. --- tests/cli.rs | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/tests/cli.rs b/tests/cli.rs index dd7e60c..f2594a5 100644 --- a/tests/cli.rs +++ b/tests/cli.rs @@ -605,8 +605,13 @@ fn a_stack_books_the_work_it_expects_and_that_work_goes_first() { #[test] fn a_stack_waits_for_the_work_it_expects_when_both_queue_together() { - let e = - Env::new("sample_every = 0.2\nexpect_wait = 20\n[pool.stack]\nlong_lived = true\nstartup = 1\n[pool.browser]\nunlimited = true\n"); + // Both are first runs and book new_job_mem each. Small, so the stack and + // its run fit together also on a CI runner whose memory is mostly in use: + // a lone stack that does not fit in RAM waits, and this test is about the + // stack waiting for its work, not about room. + let e = Env::new( + "sample_every = 0.2\nexpect_wait = 20\nnew_job_mem = \"64M\"\n[pool.stack]\nlong_lived = true\nstartup = 1\n[pool.browser]\nunlimited = true\n", + ); let mut stack = e.cmd(&["--id", "stack", "--expect", "browser", "--", "sh", "-c", "touch up; sleep 30"]); let mut stack = stack.stdout(Stdio::piped()).stderr(Stdio::piped()).spawn().unwrap(); let t = Instant::now(); From 965cdaa90afece9463c6597779095d9de29b2e53 Mon Sep 17 00:00:00 2001 From: Vincent De Snerck <10668+SpoBo@users.noreply.github.com> Date: Fri, 16 Oct 2026 14:00:10 +0200 Subject: [PATCH 15/22] no-mistakes(review): Define "still a guess" once; came-true guesses count everywhere --- CHANGELOG.md | 3 +- README.md | 5 +- src/queue.rs | 134 ++++++++++++++++++++++++++++++++++---------------- src/runner.rs | 30 +++++------ src/stall.rs | 12 ++--- 5 files changed, 116 insertions(+), 68 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0b0d76f..5d4c9c0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -40,7 +40,8 @@ and the versions follow [semver](https://semver.org). stack waits `expect_wait` for the work, and matching work links to it and goes first; an expired or started guess links nothing, and work linked to it drops back to its own priority. A guess whose work has queued and - linked to it came true: it does not expire while that work waits. The + linked to it came true: it is declared work from then on, also for the + rule for nothing running, and does not expire while that work waits. The guess never stops the rule for nothing running. Another stack respects it while it is open, also one started with no declared work, so such a stack waits at most until the guess expires or its work starts; a stack that diff --git a/README.md b/README.md index 11e143e..fe3569b 100644 --- a/README.md +++ b/README.md @@ -285,8 +285,9 @@ below). declared, and `--expect none` expects nothing at all; - `infer_work = false` turns it off. - Once the work has queued, its room is no guess any more and is booked as - declared work is: it does not expire while that work waits, and it ends + Once the work has queued and linked to it, its room is no guess any more + and is booked as declared work is, by every rule, also the rule for + nothing running: it does not expire while that work waits, and it ends when the work starts, or, past its time, when the work leaves the queue. - **A stack waits a few seconds for its work.** A caller may start the stack and its work at the same moment, not one after the other. So a long-lived diff --git a/src/queue.rs b/src/queue.rs index 09e890f..f8131e7 100644 --- a/src/queue.rs +++ b/src/queue.rs @@ -142,9 +142,9 @@ pub struct Expect { /// A job for it has started: the booking is gone. pub started: bool, /// Not declared: the history says such work followed this job before - /// (`infer_work`). Its booking is a guess: it ends `expire_after_s` - /// after the job started, and no job waits for it under the progress - /// rule. + /// (`infer_work`). While no job links to it, it is a guess (`is_guess`): + /// its booking ends `expire_after_s` after the job started, and no job + /// waits for it under the progress rule. pub implied: bool, pub expire_after_s: Option, /// The guess did not come true in time: the booking is gone. @@ -163,20 +163,23 @@ impl Expect { } } +/// Is item `i` of long-lived job `e` still a guess of the history: implied, +/// and no job among `running` and `waiting` links to it? Work that queued +/// for a guess and linked to it made it come true: from then on it is +/// declared work in every rule. +pub fn is_guess(e: &Entry, i: usize, running: &[Entry], waiting: &[Entry]) -> bool { + e.expects.get(i).is_some_and(|x| x.implied) + && !running.iter().chain(waiting).any(|j| j.work_for.as_ref().is_some_and(|w| w.pid == e.pid && w.item == Some(i as u32))) +} + /// A guess of the history that did not come true in time: its booking ends. -/// One whose work has queued and linked to it came true: it stays booked -/// while that work waits, until the work starts or leaves the queue. -pub fn expire_guesses(e: &mut Entry, waiting: &[Entry], now: f64) -> bool { - let (pid, started) = (e.pid, e.started_at.unwrap_or(now)); - let linked = |i: usize| waiting.iter().any(|w| w.work_for.as_ref().is_some_and(|l| l.pid == pid && l.item == Some(i as u32))); - let mut changed = false; - for (i, x) in e.expects.iter_mut().enumerate() { - if x.due(started, now) && !linked(i) { - x.expired = true; - changed = true; - } +pub fn expire_guesses(e: &mut Entry, running: &[Entry], waiting: &[Entry], now: f64) -> bool { + let started = e.started_at.unwrap_or(now); + let due: Vec = (0..e.expects.len()).filter(|&i| e.expects[i].due(started, now) && is_guess(e, i, running, waiting)).collect(); + for &i in &due { + e.expects[i].expired = true; } - changed + !due.is_empty() } /// The long-lived job a job is work for, and which of its `expects` it @@ -829,11 +832,11 @@ pub fn booked(e: &Entry, running: &[Entry]) -> u64 { e.expects.iter().enumerate().filter(|(i, x)| x.open() && !taken(*i)).map(|(_, x)| x.mem_kb).sum() } -/// The part of `booked` that only the history guessed (`infer_work`). -fn guessed(e: &Entry, running: &[Entry]) -> u64 { - let guesses = - Entry { expects: e.expects.iter().map(|x| Expect { started: x.started || !x.implied, ..x.clone() }).collect(), ..e.clone() }; - booked(&guesses, running) +/// The part of `booked` that only the history guessed (`is_guess`). +fn guessed(e: &Entry, running: &[Entry], waiting: &[Entry]) -> u64 { + let expects = + e.expects.iter().enumerate().map(|(i, x)| Expect { started: x.started || !is_guess(e, i, running, waiting), ..x.clone() }); + booked(&Entry { expects: expects.collect(), ..e.clone() }, running) } /// The memory `job` adds when it starts: its need, plus what it books for @@ -856,10 +859,12 @@ fn booking_taken_over(job: &Entry, running: &[Entry]) -> u64 { stack.expects.get(*i as usize).filter(|x| x.open() && !taken).map_or(0, |x| x.mem_kb) } -/// The part of `booking_taken_over` that only the history guessed. -fn guess_taken_over(job: &Entry, running: &[Entry]) -> u64 { - let item = job.work_for.as_ref().and_then(|w| running.iter().find(|e| e.pid == w.pid)?.expects.get(w.item? as usize)); - if item.is_some_and(|x| x.implied) { booking_taken_over(job, running) } else { 0 } +/// The part of `booking_taken_over` that only the history guessed: none +/// when `job` is among `waiting`, as its link made the guess come true. +fn guess_taken_over(job: &Entry, running: &[Entry], waiting: &[Entry]) -> u64 { + let Some(WorkFor { pid, item: Some(i), .. }) = &job.work_for else { return 0 }; + let guess = running.iter().find(|e| e.pid == *pid).is_some_and(|e| is_guess(e, *i as usize, running, waiting)); + if guess { booking_taken_over(job, running) } else { 0 } } /// Is `job` work for a long-lived job that does not run yet? It waits for @@ -885,7 +890,7 @@ pub fn link_work(me: &Entry, running: &[Entry], waiting: &[Entry], matches: impl let stacks = running.iter().chain(waiting).filter(|e| e.long_lived && e.pid != me.pid && e.checkout == me.checkout); let mut best: Option<(u8, WorkFor, i32)> = None; for e in stacks { - for (i, x) in e.expects.iter().enumerate().filter(|(_, x)| matches(&x.work) && still_expected(x)) { + for (i, x) in e.expects.iter().enumerate().filter(|(i, x)| matches(&x.work) && still_expected(e, *i, running, waiting)) { let free = x.open() && !claimed(e.pid, i); // A stack that waits takes only work that can take over a booking. if e.started_at.is_none() && !free { @@ -901,18 +906,21 @@ pub fn link_work(me: &Entry, running: &[Entry], waiting: &[Entry], matches: impl best.map(|(_, w, p)| (w, p)) } -/// Is this work still expected: declared, or a guess that has neither -/// expired nor seen its work start? Only then does work link to it and go -/// first. -fn still_expected(x: &Expect) -> bool { - !x.implied || x.open() +/// Is item `i` of `e` still expected: declared or come true, or a guess that +/// has neither expired nor seen its work start? Only then does work link +/// to it and go first. +fn still_expected(e: &Entry, i: usize, running: &[Entry], waiting: &[Entry]) -> bool { + e.expects[i].open() || !is_guess(e, i, running, waiting) } /// Has `me`'s link to its long-lived job lapsed: that job is gone, or what /// it expected of `me` was a guess that has expired or whose work started? +/// The item `me` links to is no guess: `me` made it come true. pub fn link_lapsed(me: &Entry, running: &[Entry], waiting: &[Entry], matches: impl Fn(&str) -> bool) -> bool { let Some(w) = &me.work_for else { return false }; - running.iter().chain(waiting).find(|e| e.pid == w.pid).is_none_or(|e| !e.expects.iter().any(|x| matches(&x.work) && still_expected(x))) + let expected = + |e: &Entry, i: usize| matches(&e.expects[i].work) && (w.item == Some(i as u32) || still_expected(e, i, running, waiting)); + running.iter().chain(waiting).find(|e| e.pid == w.pid).is_none_or(|e| !(0..e.expects.len()).any(|i| expected(e, i))) } /// Does `a` belong to an older task-runner run than `b`? Only jobs that are @@ -981,7 +989,7 @@ pub fn decide( // and the job that needs one of them would wait as long as the // warning lasts, which can be all morning. if !running.iter().all(Entry::steady) { - let room = Room::new(m, lim, running); + let room = Room::new(m, lim, running, waiting); if let Some(mem) = room.blockers(m, lim, running, me, now, unknown_starts).into_iter().find(|b| matches!(b, Blocker::Memory { .. })) { @@ -1022,7 +1030,7 @@ pub fn decide( // alone keep it out may it go up to the ceiling of `outside_admit`, and // with nothing of taskguard's running, up to all of RAM: nothing taskguard // runs would free that. - let room = Room::new(m, lim, running); + let room = Room::new(m, lim, running, waiting); if running.iter().all(Entry::steady) { let fits = |w: &Entry| room.fits_on_progress(m, lim, running, w); let head = older.iter().copied().chain(std::iter::once(me)).find(|w| fits(w)); @@ -1353,14 +1361,19 @@ struct Room { /// The running jobs: what a job takes over from its long-lived job /// depends on them. running: Vec, + /// The waiting jobs linked to a long-lived job: what is still a guess + /// depends on them. + linked: Vec, /// The part of `res_mem` that only the history guessed: what long-lived - /// jobs book for work nobody declared (`infer_work`). + /// jobs book for work nobody declared and no job has queued for yet + /// (`is_guess`). res_guessed: u64, } impl Room { - fn new(m: &MachineSample, lim: &Limits, running: &[Entry]) -> Room { + fn new(m: &MachineSample, lim: &Limits, running: &[Entry], waiting: &[Entry]) -> Room { let (res_cpu, res_mem) = reserve(running); + let linked: Vec = waiting.iter().filter(|w| w.work_for.is_some()).cloned().collect(); let ours_mem = running.iter().map(|e| e.live_mem_kb).sum(); let passing: Vec = running.iter().filter(|e| !e.lasting()).cloned().collect(); Room { @@ -1374,7 +1387,8 @@ impl Room { passing_live: passing.iter().map(|e| e.live_mem_kb).sum(), passing_res: reserve(&passing).1, running: running.to_vec(), - res_guessed: running.iter().map(|e| guessed(e, running)).sum(), + res_guessed: running.iter().map(|e| guessed(e, running, &linked)).sum(), + linked, } } @@ -1390,9 +1404,13 @@ impl Room { /// guess out: such a stack starts as it would without the history, and /// books its guess from then on. fn mem_would_on(&self, job: &Entry, progress: bool) -> f64 { - let undeclared = job.long_lived && !job.expects.iter().any(|x| !x.implied); - let guess = if progress && !job.long_lived { self.res_guessed.saturating_sub(guess_taken_over(job, &self.running)) } else { 0 }; - let own = if undeclared && job.started_at.is_none() { guessed(job, &[]) } else { 0 }; + let undeclared = job.long_lived && (0..job.expects.len()).all(|i| is_guess(job, i, &self.running, &self.linked)); + let guess = if progress && !job.long_lived { + self.res_guessed.saturating_sub(guess_taken_over(job, &self.running, &self.linked)) + } else { + 0 + }; + let own = if undeclared && job.started_at.is_none() { guessed(job, &self.running, &self.linked) } else { 0 }; (self.mem_used + self.res_mem - guess.min(self.res_mem) + need_with_booking(job, &self.running) - own) as f64 } @@ -2773,11 +2791,15 @@ mod tests { let mut declared = guessing.clone(); declared.expects[0].implied = false; assert_eq!(blockers(decide(&m, &lim, r, std::slice::from_ref(&declared), &declared, 1000.0, &[])), vec!["memory"]); - // A run that linked to the open guess keeps its link. + // A run that linked to the open guess keeps its link, and so does one + // that linked while another run held the booking. let linked = Entry { work_for: Some(WorkFor { pid: 11, key: "dev-env:stack".into(), item: Some(0) }), ..me.clone() }; assert!(!link_lapsed(&linked, r, &[], |w| w == "browser")); + let linked = Entry { work_for: Some(WorkFor { pid: 11, key: "dev-env:stack".into(), item: None }), ..me.clone() }; + assert!(!link_lapsed(&linked, r, &[], |w| w == "browser")); // Expired, it books nothing: the stack with no declared work starts - // (24 + 1 + 4 GB), and no run links to the guess or keeps its link. + // (24 + 1 + 4 GB), and no run links to the guess or keeps a link + // that did not take its booking. stack.expects[0].expired = true; let r = std::slice::from_ref(&stack); assert_eq!(reserve(r).1, grows); @@ -2859,7 +2881,7 @@ mod tests { let run = Entry { work_for: Some(link), priority: priority + 1, ..me }; let waiting = std::slice::from_ref(&run); // Minute 11: the guess came true, so it does not expire. - assert!(!expire_guesses(&mut stack, waiting, 660.0)); + assert!(!expire_guesses(&mut stack, &[], waiting, 660.0)); let r = std::slice::from_ref(&stack); assert!(!link_lapsed(&run, r, waiting, |w| w == "browser")); assert_eq!(reserve(r).1, GB + 43 * GB / 10, "it keeps the room booked for the run"); @@ -2867,10 +2889,36 @@ mod tests { m.mem_pressure = Some(50.0); assert_eq!(blockers(decide(&m, &LIM, r, waiting, &run, 660.0, &[])), vec!["pressure"]); // Once the run has left the queue, the guess that is past its time ends. - assert!(expire_guesses(&mut stack, &[], 660.0)); + assert!(expire_guesses(&mut stack, &[], &[], 660.0)); assert_eq!(reserve(std::slice::from_ref(&stack)).1, GB); } + #[test] + fn a_guess_whose_work_queued_is_declared_work_also_by_the_progress_rule() { + // 32 GB, 28 GB held; only a steady stack runs, still to grow 1 GB, + // and the history guessed a 4.3 GB run for it. + let lim = Limits { mem_max_pct: 95.0, ..LIM }; + let mut stack = Entry { pid: 11, ..expecting(steady_stack(11, 7)) }; + stack.expects[0].implied = true; + stack.expects[0].expire_after_s = Some(600.0); + let r = std::slice::from_ref(&stack); + let m = machine(1.0, 28); + let lint = job(4, "lint", 0.5, 2); + // While nothing queued for it, the guess is no reason to wait: the + // lint starts (28 + 1 + 2 GB). + assert!(blockers(decide(&m, &lim, r, std::slice::from_ref(&lint), &lint, 1000.0, &[])).is_empty()); + // The run queued and linked: the guess came true, and its room is + // the run's. The lint does not start into it (28 + 1 + 4.3 + 2 GB). + let run = run_for(Some(0)); + let waiting = [run.clone(), lint.clone()]; + assert_eq!(blockers(decide(&m, &lim, r, &waiting, &lint, 1000.0, &[])), vec!["memory"]); + assert!(!expire_guesses(&mut stack, &[], &waiting, 2000.0)); + // A guess no job queued for still expires, and then books nothing. + let waiting = std::slice::from_ref(&lint); + assert!(expire_guesses(&mut stack, &[], waiting, 2000.0)); + assert!(blockers(decide(&m, &lim, std::slice::from_ref(&stack), waiting, &lint, 2000.0, &[])).is_empty()); + } + #[test] fn backfill_keeps_pools_and_pressure() { let (mut running, mut head) = blocked_head(); diff --git a/src/runner.rs b/src/runner.rs index 433a0f0..186ec2d 100644 --- a/src/runner.rs +++ b/src/runner.rs @@ -66,8 +66,8 @@ fn is_work(work: &str, me: &Entry, cmd: &[String]) -> bool { /// Mark the expected work a nudge says has started. /// Once work for it has started, a long-lived job also drops what it booked -/// on the history's guess: the work came. -fn apply_work_started(me: &mut Entry, n: &queue::Nudge) -> bool { +/// on the history's guess (`queue::is_guess`): the work came. +fn apply_work_started(me: &mut Entry, n: &queue::Nudge, running: &[Entry], waiting: &[Entry]) -> bool { let mut changed = false; for i in &n.work_started { if let Some(x) = me.expects.get_mut(*i as usize).filter(|x| !x.started) { @@ -76,8 +76,8 @@ fn apply_work_started(me: &mut Entry, n: &queue::Nudge) -> bool { } } if !n.work_started.is_empty() { - for x in me.expects.iter_mut().filter(|x| x.implied && x.open()) { - x.started = true; + for i in (0..me.expects.len()).filter(|&i| me.expects[i].open() && queue::is_guess(me, i, running, waiting)).collect::>() { + me.expects[i].started = true; changed = true; } } @@ -629,22 +629,20 @@ pub fn run(mut o: Opts) -> Result { } if let Some(n) = q.take_nudge(me_pid) { apply_nudge(&mut me, &n, database.as_ref()); - apply_work_started(&mut me, &n); + apply_work_started(&mut me, &n, &running, &waiting); started_by_hand |= n.start; } link_to_stack(&mut me, own_priority, &running, &waiting, &o.cmd, o.quiet); // Work that queued for this long-lived job may need more than - // was booked for it: book that. Work the history guessed that - // has queued is a guess no more. + // was booked for it: book that. for (i, x) in me.expects.iter_mut().enumerate() { let theirs = waiting .iter() .filter(|w| w.work_for.as_ref().is_some_and(|l| l.pid == me_pid && l.item == Some(i as u32))) .max_by_key(|w| w.need_mem_kb); - if let Some(w) = theirs.filter(|w| w.need_mem_kb > x.mem_kb || x.implied) { - x.mem_kb = x.mem_kb.max(w.need_mem_kb); + if let Some(w) = theirs.filter(|w| w.need_mem_kb > x.mem_kb) { + x.mem_kb = w.need_mem_kb; x.from = format!("the need of {}, which queued for it", w.key); - (x.implied, x.expire_after_s) = (false, None); } } let mut waiting = waiting; @@ -887,7 +885,7 @@ pub fn run(mut o: Opts) -> Result { } if let Some(n) = q.take_nudge(me_pid) { apply_nudge(&mut me, &n, database.as_ref()); - if apply_work_started(&mut me, &n) { + if !n.work_started.is_empty() && apply_work_started(&mut me, &n, &q.running(), &q.waiting()) { let _ = q.write(&q.run_path(me_pid), &me); } match (n.pause, me.paused_since.is_some()) { @@ -907,7 +905,9 @@ pub fn run(mut o: Opts) -> Result { say(&format!("paused {} - by hand; it stays paused until taskguard resume, or p in taskguard top", me.key)); } } - if me.expects.iter().any(|x| x.due(me.started_at.unwrap_or(now), now)) && queue::expire_guesses(&mut me, &q.waiting(), now) { + if me.expects.iter().any(|x| x.due(me.started_at.unwrap_or(now), now)) + && queue::expire_guesses(&mut me, &q.running(), &q.waiting(), now) + { let _ = q.write(&q.run_path(me_pid), &me); say(&format!("expected {} - the work its history suggested did not come in time; it no longer books room for it", me.key)); } @@ -1310,12 +1310,12 @@ mod tests { let declared = queue::Expect { work: "e2e".into(), mem_kb: 4, ..Default::default() }; let stack = Entry { started_at: Some(1000.0), expects: vec![declared, guess], ..Default::default() }; let mut s = stack.clone(); - assert!(!queue::expire_guesses(&mut s, &[], 1600.0)); - assert!(queue::expire_guesses(&mut s, &[], 1601.0)); + assert!(!queue::expire_guesses(&mut s, &[], &[], 1600.0)); + assert!(queue::expire_guesses(&mut s, &[], &[], 1601.0)); assert!(!s.expects[0].expired && s.expects[1].expired, "only the guess expires"); // Work for the stack started: the guess goes with it. let mut s = stack; - assert!(apply_work_started(&mut s, &queue::Nudge { work_started: vec![0], ..Default::default() })); + assert!(apply_work_started(&mut s, &queue::Nudge { work_started: vec![0], ..Default::default() }, &[], &[])); assert!(s.expects.iter().all(|x| x.started)); } diff --git a/src/stall.rs b/src/stall.rs index e737411..67ff170 100644 --- a/src/stall.rs +++ b/src/stall.rs @@ -21,7 +21,7 @@ //! `cargo test stall -- --nocapture --test-threads 1` prints the tables. use crate::machine::{MachineSample, PEAK_WINDOW}; -use crate::queue::{Decision, Entry, Expect, Limits, LongLived, decide, expire_guesses, link_lapsed, link_work}; +use crate::queue::{Decision, Entry, Expect, Limits, LongLived, decide, expire_guesses, is_guess, link_lapsed, link_work}; const GB: u64 = 1024 * 1024; const TICK: f64 = 1.0; @@ -418,6 +418,7 @@ pub fn run(s: &Scenario) -> Outcome { } // What each running job holds and uses now. let queued: Vec = waiting.iter().map(|(e, _)| e.clone()).collect(); + let ran: Vec = running.iter().map(|r| r.e.clone()).collect(); for r in running.iter_mut() { let age = t - r.started; let (mem, cpu) = match r.kind { @@ -452,7 +453,7 @@ pub fn run(s: &Scenario) -> Outcome { r.e.follow(mem, cpu, s.host.ncpu as f64, t, lived); // As the owner does: a guess of the history that did not come // true in time stops booking. - expire_guesses(&mut r.e, &queued, t); + expire_guesses(&mut r.e, &ran, &queued, t); if s.steady_books_live && r.e.steady() { r.e.need_mem_kb = mem; } @@ -521,9 +522,6 @@ pub fn run(s: &Scenario) -> Outcome { .unwrap_or(0); let item = &mut waiting[i].0.expects[x]; item.mem_kb = item.mem_kb.max(theirs); - if theirs > 0 { - (item.implied, item.expire_after_s) = (false, None); - } } } let all: Vec = waiting.iter().map(|(e, _)| e.clone()).collect(); @@ -536,8 +534,8 @@ pub fn run(s: &Scenario) -> Outcome { && let Some(st) = running.iter_mut().find(|r| r.e.pid == w.pid) { st.e.expects[i as usize].started = true; - for x in st.e.expects.iter_mut().filter(|x| x.implied) { - x.started = true; + for g in (0..st.e.expects.len()).filter(|&g| is_guess(&st.e, g, &run_now, &all)).collect::>() { + st.e.expects[g].started = true; } } for (w, _) in waiting.iter_mut().filter(|(w, _)| w.ahead_of(&me) && w.bypassed_since.is_none()) { From 0e96cc0e0e65439efe39c18abd17ad7c3c796db1 Mon Sep 17 00:00:00 2001 From: Vincent De Snerck <10668+SpoBo@users.noreply.github.com> Date: Fri, 16 Oct 2026 16:13:59 +0200 Subject: [PATCH 16/22] fix: a second run does not link to a guess the first run made come true Restore still_expected to '!x.implied || x.open()': once the run that took over a history guess has started, a second matching run in the checkout keeps its own priority instead of linking to the stack and holding its turn. taskguard status no longer labels a guess that came true as "from history". --- CHANGELOG.md | 4 ++-- README.md | 5 +++-- src/queue.rs | 42 ++++++++++++++++++++++++++++++++++-------- src/report.rs | 36 ++++++++++++++++++++++++++++-------- 4 files changed, 67 insertions(+), 20 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5d4c9c0..1171e5f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -35,8 +35,8 @@ and the versions follow [semver](https://semver.org). queued in its checkout within `infer_work_within` (900) seconds of its start on at least `infer_work_min` (3) of its last `infer_work_of` (5) runs there. It books that work's usual need, shown as `(expects WORK from - history, ...)`, for twice its usual delay and ten minutes at most, and - drops it once work for the stack starts. While the guess is open the + history, ...)` until work queues for it, for twice its usual delay and ten + minutes at most, and drops it once work for the stack starts. While the guess is open the stack waits `expect_wait` for the work, and matching work links to it and goes first; an expired or started guess links nothing, and work linked to it drops back to its own priority. A guess whose work has queued and diff --git a/README.md b/README.md index fe3569b..4148e38 100644 --- a/README.md +++ b/README.md @@ -264,8 +264,9 @@ below). `infer_work_within` (900) seconds of their start, that work is expected again. A kind of work is a pool, or the key of a job outside any pool. The stack books that work's usual need, which `taskguard status` shows as - `(expects browser from history, books 4.3 GB)`, and while the guess is - open the work links and goes first as declared work does. It is a guess, + `(expects browser from history, books 4.3 GB)` until work queues for + it, and while the guess is open the work links and goes first as declared + work does. It is a guess, so it is bounded: - it books room for twice the work's usual delay after the stack started, ten minutes at most, and drops it once work for the stack starts or the diff --git a/src/queue.rs b/src/queue.rs index f8131e7..f6bd216 100644 --- a/src/queue.rs +++ b/src/queue.rs @@ -890,7 +890,7 @@ pub fn link_work(me: &Entry, running: &[Entry], waiting: &[Entry], matches: impl let stacks = running.iter().chain(waiting).filter(|e| e.long_lived && e.pid != me.pid && e.checkout == me.checkout); let mut best: Option<(u8, WorkFor, i32)> = None; for e in stacks { - for (i, x) in e.expects.iter().enumerate().filter(|(i, x)| matches(&x.work) && still_expected(e, *i, running, waiting)) { + for (i, x) in e.expects.iter().enumerate().filter(|(_, x)| matches(&x.work) && still_expected(x)) { let free = x.open() && !claimed(e.pid, i); // A stack that waits takes only work that can take over a booking. if e.started_at.is_none() && !free { @@ -906,11 +906,13 @@ pub fn link_work(me: &Entry, running: &[Entry], waiting: &[Entry], matches: impl best.map(|(_, w, p)| (w, p)) } -/// Is item `i` of `e` still expected: declared or come true, or a guess that -/// has neither expired nor seen its work start? Only then does work link -/// to it and go first. -fn still_expected(e: &Entry, i: usize, running: &[Entry], waiting: &[Entry]) -> bool { - e.expects[i].open() || !is_guess(e, i, running, waiting) +/// Is this work still expected: declared, or implied by the history and +/// neither expired nor started? Only then does work link to it and go +/// first. Once the run that made a guess come true has started, a second +/// matching run is not the work the history foresaw: it keeps its own +/// priority. +fn still_expected(x: &Expect) -> bool { + !x.implied || x.open() } /// Has `me`'s link to its long-lived job lapsed: that job is gone, or what @@ -918,8 +920,7 @@ fn still_expected(e: &Entry, i: usize, running: &[Entry], waiting: &[Entry]) -> /// The item `me` links to is no guess: `me` made it come true. pub fn link_lapsed(me: &Entry, running: &[Entry], waiting: &[Entry], matches: impl Fn(&str) -> bool) -> bool { let Some(w) = &me.work_for else { return false }; - let expected = - |e: &Entry, i: usize| matches(&e.expects[i].work) && (w.item == Some(i as u32) || still_expected(e, i, running, waiting)); + let expected = |e: &Entry, i: usize| matches(&e.expects[i].work) && (w.item == Some(i as u32) || still_expected(&e.expects[i])); running.iter().chain(waiting).find(|e| e.pid == w.pid).is_none_or(|e| !(0..e.expects.len()).any(|i| expected(e, i))) } @@ -2893,6 +2894,31 @@ mod tests { assert_eq!(reserve(std::slice::from_ref(&stack)).1, GB); } + #[test] + fn a_second_run_does_not_ride_on_a_guess_the_first_run_made_come_true() { + let mut stack = Entry { pid: 11, ..expecting(steady_stack(11, 7)) }; + stack.expects[0].implied = true; + stack.expects[0].expire_after_s = Some(600.0); + let is = |w: &str| w == "browser"; + // The first run linked to the guess, took over its booking and runs. + stack.expects[0].started = true; + let first = Entry { pid: 30, started_at: Some(990.0), ..run_for(Some(0)) }; + let running = [stack.clone(), first.clone()]; + assert!(!link_lapsed(&first, &running, &[], is), "the first run keeps its link"); + // A second matching run in the checkout is not the work the history + // foresaw: it does not link, so it gets no lift and does not wait + // for the stack. + let second = Entry { pool: Some("browser".into()), checkout: "/w/a".into(), ..job(31, "pr.ts --pr 17399", 3.0, 4) }; + assert!(link_work(&second, &running, &[], is).is_none()); + // One that linked before the first run started lets go of its link. + let linked = Entry { work_for: Some(WorkFor { pid: 11, key: "dev-env:stack".into(), item: None }), ..second.clone() }; + assert!(link_lapsed(&linked, &running, &[], is)); + // Declared work still links a second run: the caller said it comes. + stack.expects[0].implied = false; + let running = [stack, first]; + assert_eq!(link_work(&second, &running, &[], is).unwrap().0.item, None); + } + #[test] fn a_guess_whose_work_queued_is_declared_work_also_by_the_progress_rule() { // 32 GB, 28 GB held; only a steady stack runs, still to grow 1 GB, diff --git a/src/report.rs b/src/report.rs index 5743595..9de1b55 100644 --- a/src/report.rs +++ b/src/report.rs @@ -3,7 +3,7 @@ //! the scheduler really thinks. use crate::machine::MachineSample; -use crate::queue::{Blocker, Decision, Entry, Limits, booked, need_with_booking, reserve}; +use crate::queue::{Blocker, Decision, Entry, Limits, booked, is_guess, need_with_booking, reserve}; use serde::Serialize; use std::fmt::Write as _; @@ -503,7 +503,7 @@ pub fn rule_checks(s: &Snapshot, w: &Waiting) -> Vec<(bool, String)> { gb(m.mem_used_kb), gb(s.reserve_mem_kb), gb(need), - link_text(e, &s.running), + link_text(e, &s.running, &queued(s)), mem_would as f64 * 100.0 / m.mem_total_kb.max(1) as f64, s.limits.mem_max_pct ), @@ -542,10 +542,16 @@ pub fn bar(used: f64, over: f64, reserved: f64, total: f64, limit: f64, width: u .collect() } +/// The entries of the jobs that wait. +fn queued(s: &Snapshot) -> Vec { + s.waiting.iter().map(|w| w.entry.clone()).collect() +} + /// What a job's link to a long-lived job adds to its line: the work a /// long-lived job expects and books room for, or the long-lived job a job -/// is work for. Empty for any other job. -pub fn link_text(e: &Entry, running: &[Entry]) -> String { +/// is work for. Empty for any other job. Work the history guessed reads +/// "from history" until a job queues for it and makes it come true. +pub fn link_text(e: &Entry, running: &[Entry], waiting: &[Entry]) -> String { let mut t = String::new(); if let Some(w) = &e.work_for { t += &format!(" (for stack {})", w.key); @@ -553,8 +559,9 @@ pub fn link_text(e: &Entry, running: &[Entry]) -> String { let open: Vec = e .expects .iter() - .filter(|x| x.open()) - .map(|x| if x.implied { format!("{} from history", x.work) } else { x.work.clone() }) + .enumerate() + .filter(|(_, x)| x.open()) + .map(|(i, x)| if is_guess(e, i, running, waiting) { format!("{} from history", x.work) } else { x.work.clone() }) .collect(); if !open.is_empty() { t += &format!(" (expects {}, books {})", open.join(", "), gb(booked(e, running))); @@ -611,6 +618,7 @@ pub fn render_status(s: &Snapshot) -> String { let _ = writeln!(o); let _ = writeln!(o, "RUNNING ({}) pool time CPU now / needs MEMORY now / needs", s.running.len()); + let queue = queued(s); if s.running.is_empty() { let _ = writeln!(o, " (nothing running)"); } @@ -635,7 +643,7 @@ pub fn render_status(s: &Snapshot) -> String { needs_cpu, gb(e.live_mem_kb), needs_mem, - link_text(e, &s.running) + link_text(e, &s.running, &queue) ); } let _ = writeln!(o); @@ -663,7 +671,7 @@ pub fn render_status(s: &Snapshot) -> String { if e.stalled { blocked += " [stalled: holds no job back]"; } - blocked += &link_text(e, &s.running); + blocked += &link_text(e, &s.running, &queue); let _ = writeln!( o, " {:<36} {:<12} {:>6} {:<21} {}", @@ -725,6 +733,18 @@ mod tests { } } + #[test] + fn a_guess_reads_from_history_until_work_queues_for_it() { + let guess = crate::queue::Expect { work: "browser".into(), mem_kb: GB, implied: true, ..Default::default() }; + let stack = Entry { pid: 11, long_lived: true, expects: vec![guess], ..Default::default() }; + let running = std::slice::from_ref(&stack); + assert_eq!(link_text(&stack, running, &[]), " (expects browser from history, books 1.0 GB)"); + let link = crate::queue::WorkFor { pid: 11, key: "dev-env:stack".into(), item: Some(0) }; + let run = Entry { pid: 30, work_for: Some(link), ..Default::default() }; + assert_eq!(link_text(&stack, running, std::slice::from_ref(&run)), " (expects browser, books 1.0 GB)"); + assert_eq!(link_text(&run, running, &[]), " (for stack dev-env:stack)"); + } + /// Voyager: 18 cores at cpu_max 150 is 27 cores. A typecheck that needs /// 30 can never fully fit; its wait output says why and what to do. #[test] From 79beb9edcd8de1c0c477816a7034e391ba8a11ed Mon Sep 17 00:00:00 2001 From: Vincent De Snerck <10668+SpoBo@users.noreply.github.com> Date: Fri, 16 Oct 2026 16:14:22 +0200 Subject: [PATCH 17/22] ci(release): a tag with a suffix publishes a pre-release, never latest --- .claude/skills/release/SKILL.md | 15 +++++++++++++++ .github/workflows/release.yml | 10 +++++++++- 2 files changed, 24 insertions(+), 1 deletion(-) diff --git a/.claude/skills/release/SKILL.md b/.claude/skills/release/SKILL.md index 4a0a39e..65d0331 100644 --- a/.claude/skills/release/SKILL.md +++ b/.claude/skills/release/SKILL.md @@ -91,6 +91,21 @@ build or the tests need it. A local `rust-objcopy ... libLLVM.dylib` warning on macOS comes from the local toolchain. It does not affect the release: CI builds the binaries. +### Pre-release + +A version with a suffix, such as `0.11.0-rc.1`, makes a pre-release. The +steps are the same, with the suffix in `Cargo.toml`, the `CHANGELOG.md` +section and the tag (`v0.11.0-rc.1`). The workflow marks a tag with a `-` +as a pre-release and never as latest. Check it after the workflow: + +```sh +gh release view vNEW -R SpoBo/taskguard --json isPrerelease --jq .isPrerelease # true +gh release view -R SpoBo/taskguard --json tagName --jq .tagName # the last full release +``` + +A pre-release can be tagged on a branch other than `master` when the user +asks for it; the workflow builds the tagged commit. + ### Dry run To test changes to `release.yml` without a release, start the workflow by diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index a33a95f..0859a94 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -78,11 +78,19 @@ jobs: exit 1 fi printf '\nPrebuilt binaries for macOS (Apple silicon, Intel) and Linux (x86_64, ARM, static musl). Each archive holds the `taskguard` binary; a `.sha256` file sits next to it.\n' >> notes.md + # A tag with a suffix, such as v0.11.0-rc.1, is a pre-release: it is + # never marked latest, so whoever installs the latest release keeps + # the last full one. - name: Publish the release env: GH_TOKEN: ${{ github.token }} run: | + pre=() + case "${{ github.ref_name }}" in + *-*) pre=(--prerelease --latest=false) ;; + esac gh release create "${{ github.ref_name }}" dist/* \ --repo "${{ github.repository }}" \ --title "taskguard ${{ github.ref_name }}" \ - --notes-file notes.md + --notes-file notes.md \ + "${pre[@]}" From ff7e3b51aa9ff279f5ef4e2c9de6db31bb52a08d Mon Sep 17 00:00:00 2001 From: Vincent De Snerck <10668+SpoBo@users.noreply.github.com> Date: Fri, 16 Oct 2026 16:15:33 +0200 Subject: [PATCH 18/22] chore: release 0.11.0-rc.1 --- CHANGELOG.md | 5 ++++- Cargo.lock | 2 +- Cargo.toml | 2 +- 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1171e5f..bb4e05b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,8 @@ and the versions follow [semver](https://semver.org). ## [Unreleased] +## [0.11.0-rc.1] - 2026-10-16 + ### Added - `taskguard dump PATTERN | --cmd "COMMAND" | --run ID` prints everything @@ -398,7 +400,8 @@ and the versions follow [semver](https://semver.org). needs, and starts it when the machine has room for it. - Prebuilt binaries for macOS and Linux on each version tag. -[Unreleased]: https://github.com/SpoBo/taskguard/compare/v0.10.0...HEAD +[Unreleased]: https://github.com/SpoBo/taskguard/compare/v0.11.0-rc.1...HEAD +[0.11.0-rc.1]: https://github.com/SpoBo/taskguard/compare/v0.10.0...v0.11.0-rc.1 [0.10.0]: https://github.com/SpoBo/taskguard/compare/v0.9.0...v0.10.0 [0.9.0]: https://github.com/SpoBo/taskguard/compare/v0.8.0...v0.9.0 [0.8.0]: https://github.com/SpoBo/taskguard/compare/v0.7.1...v0.8.0 diff --git a/Cargo.lock b/Cargo.lock index db42faf..af5c78d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1434,7 +1434,7 @@ dependencies = [ [[package]] name = "taskguard" -version = "0.10.0" +version = "0.11.0-rc.1" dependencies = [ "anyhow", "crossterm", diff --git a/Cargo.toml b/Cargo.toml index cc87f38..a0f1072 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "taskguard" -version = "0.10.0" +version = "0.11.0-rc.1" edition = "2024" rust-version = "1.89" description = "A sem that learns what each job needs: start a command only when the machine has room for its CPU and memory" From 3a73eed3def074895ca3c439ff91193a594d6cb2 Mon Sep 17 00:00:00 2001 From: Vincent De Snerck <10668+SpoBo@users.noreply.github.com> Date: Fri, 16 Oct 2026 16:20:58 +0200 Subject: [PATCH 19/22] no-mistakes(review): Explain's memory line leaves out a stack's own guessed booking --- src/queue.rs | 11 +++++++++-- src/report.rs | 27 +++++++++++++++++++++++++-- 2 files changed, 34 insertions(+), 4 deletions(-) diff --git a/src/queue.rs b/src/queue.rs index f6bd216..c6a50c7 100644 --- a/src/queue.rs +++ b/src/queue.rs @@ -839,6 +839,14 @@ fn guessed(e: &Entry, running: &[Entry], waiting: &[Entry]) -> u64 { booked(&Entry { expects: expects.collect(), ..e.clone() }, running) } +/// The part of `job`'s own booking that it leaves out when it starts: all +/// it guessed, when `job` is a waiting long-lived job with no declared work. +/// Such a stack starts as it would without the history. +pub fn own_guess(job: &Entry, running: &[Entry], waiting: &[Entry]) -> u64 { + let undeclared = job.long_lived && (0..job.expects.len()).all(|i| is_guess(job, i, running, waiting)); + if undeclared && job.started_at.is_none() { guessed(job, running, waiting) } else { 0 } +} + /// The memory `job` adds when it starts: its need, plus what it books for /// the work it expects, less what its long-lived job booked for it, which it /// takes over. @@ -1405,13 +1413,12 @@ impl Room { /// guess out: such a stack starts as it would without the history, and /// books its guess from then on. fn mem_would_on(&self, job: &Entry, progress: bool) -> f64 { - let undeclared = job.long_lived && (0..job.expects.len()).all(|i| is_guess(job, i, &self.running, &self.linked)); let guess = if progress && !job.long_lived { self.res_guessed.saturating_sub(guess_taken_over(job, &self.running, &self.linked)) } else { 0 }; - let own = if undeclared && job.started_at.is_none() { guessed(job, &self.running, &self.linked) } else { 0 }; + let own = own_guess(job, &self.running, &self.linked); (self.mem_used + self.res_mem - guess.min(self.res_mem) + need_with_booking(job, &self.running) - own) as f64 } diff --git a/src/report.rs b/src/report.rs index 9de1b55..06223ec 100644 --- a/src/report.rs +++ b/src/report.rs @@ -3,7 +3,7 @@ //! the scheduler really thinks. use crate::machine::MachineSample; -use crate::queue::{Blocker, Decision, Entry, Limits, booked, is_guess, need_with_booking, reserve}; +use crate::queue::{Blocker, Decision, Entry, Limits, booked, is_guess, need_with_booking, own_guess, reserve}; use serde::Serialize; use std::fmt::Write as _; @@ -494,7 +494,7 @@ pub fn rule_checks(s: &Snapshot, w: &Waiting) -> Vec<(bool, String)> { ), )); } - let need = need_with_booking(e, &s.running); + let need = need_with_booking(e, &s.running).saturating_sub(own_guess(e, &s.running, &queued(s))); let mem_would = m.mem_used_kb + s.reserve_mem_kb + need; out.push(( checked("memory", mem_would <= s.mem_limit_kb), @@ -745,6 +745,29 @@ mod tests { assert_eq!(link_text(&run, running, &[]), " (for stack dev-env:stack)"); } + #[test] + fn the_memory_rule_leaves_out_what_a_waiting_stack_only_guessed() { + let lim = Limits { partial_fit: 0.0, max_backfill: 0.0, ..advice_limits() }; + let m = MachineSample::fixed(1.0, 12, 20 * GB, 32 * GB); + let guess = crate::queue::Expect { work: "browser".into(), mem_kb: 43 * GB / 10, implied: true, ..Default::default() }; + let stack = Entry { + ticket: 1, + pid: 11, + key: "dev-env:stack".into(), + long_lived: true, + need_mem_kb: 6 * GB, + known: true, + expects: vec![guess], + ..Default::default() + }; + let s = Snapshot::build(m, lim, vec![], vec![stack], 1000.0, &[]); + assert!(matches!(s.waiting[0].decision, Decision::Admit { .. }), "26 GB of 27.2 fits without the guess"); + let checks = rule_checks(&s, &s.waiting[0]); + let (ok, text) = checks.iter().find(|(_, t)| t.starts_with("memory:")).cloned().unwrap(); + assert!(ok, "{text}"); + assert!(text.contains("6.0 GB for this job"), "{text}"); + } + /// Voyager: 18 cores at cpu_max 150 is 27 cores. A typecheck that needs /// 30 can never fully fit; its wait output says why and what to do. #[test] From db822784ad4a1b83764ed4e9f75963a49e07b1c8 Mon Sep 17 00:00:00 2001 From: Vincent De Snerck <10668+SpoBo@users.noreply.github.com> Date: Fri, 16 Oct 2026 16:30:39 +0200 Subject: [PATCH 20/22] no-mistakes(document): Reflow history-guess paragraphs in README and CHANGELOG --- CHANGELOG.md | 8 ++++---- README.md | 3 +-- 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index bb4e05b..b39bd11 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -38,10 +38,10 @@ and the versions follow [semver](https://semver.org). start on at least `infer_work_min` (3) of its last `infer_work_of` (5) runs there. It books that work's usual need, shown as `(expects WORK from history, ...)` until work queues for it, for twice its usual delay and ten - minutes at most, and drops it once work for the stack starts. While the guess is open the - stack waits `expect_wait` for the work, and matching work links to it and - goes first; an expired or started guess links nothing, and work linked to - it drops back to its own priority. A guess whose work has queued and + minutes at most, and drops it once work for the stack starts. While the + guess is open the stack waits `expect_wait` for the work, and matching work + links to it and goes first; an expired or started guess links nothing, and + work linked to it drops back to its own priority. A guess whose work has queued and linked to it came true: it is declared work from then on, also for the rule for nothing running, and does not expire while that work waits. The guess never stops the rule for nothing running. Another stack respects it diff --git a/README.md b/README.md index 4148e38..62db95f 100644 --- a/README.md +++ b/README.md @@ -266,8 +266,7 @@ below). stack books that work's usual need, which `taskguard status` shows as `(expects browser from history, books 4.3 GB)` until work queues for it, and while the guess is open the work links and goes first as declared - work does. It is a guess, - so it is bounded: + work does. It is a guess, so it is bounded: - it books room for twice the work's usual delay after the stack started, ten minutes at most, and drops it once work for the stack starts or the stack stops; From 2c5c3c6a091fa4adea86eff6e730e837ae2a52d3 Mon Sep 17 00:00:00 2001 From: Vincent De Snerck <10668+SpoBo@users.noreply.github.com> Date: Sat, 17 Oct 2026 13:02:04 +0200 Subject: [PATCH 21/22] feat(top): show the version at the right of the key bar The keys keep their room: the version shows as 'taskguard VERSION', or 'vVERSION' when narrow, and not at all when the keys fill the line. --- CHANGELOG.md | 5 +++++ README.md | 5 +++-- src/top/views.rs | 32 ++++++++++++++++++++++++++++++++ 3 files changed, 40 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b39bd11..4890113 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,11 @@ and the versions follow [semver](https://semver.org). ## [Unreleased] +### Added + +- `taskguard top` shows its version at the right of the key bar, in the + room the keys leave: `taskguard VERSION`, or `vVERSION` when narrow. + ## [0.11.0-rc.1] - 2026-10-16 ### Added diff --git a/README.md b/README.md index 62db95f..1e0e58e 100644 --- a/README.md +++ b/README.md @@ -528,8 +528,9 @@ signs of starvation. `Esc` or `Backspace` goes back to the same row, and `j` opens the last job again. `?` lists every key of the view you are in, and the keys of every view; the -bottom line starts with it and shows the keys of the view that fit. `?` or -`Esc` closes the list, and any other key closes it and does its work. +bottom line starts with it and shows the keys of the view that fit, with +the version of taskguard at the right when room is left. `?` or `Esc` +closes the list, and any other key closes it and does its work. Keys: `1`-`9`, `Tab`, or Shift, Option or Cmd with `←`/`→` change views. `t` time range (5m to 7d), `n` namespace, `/` filter, `←`/`→` time cursor, diff --git a/src/top/views.rs b/src/top/views.rs index b13ab0c..5d32b98 100644 --- a/src/top/views.rs +++ b/src/top/views.rs @@ -142,6 +142,20 @@ fn draw_footer(f: &mut Frame, app: &mut App, area: Rect) { spans.push(Span::styled(format!(" {what} "), DIM)); } f.render_widget(Paragraph::new(Line::from(spans)), area); + // The version of this binary at the right, in what room the keys leave. + let free = (area.x + area.width).saturating_sub(x); + if let Some(v) = version_label(free) { + let w = v.chars().count() as u16; + let at = Rect { x: area.x + area.width - w, width: w, ..area }; + f.render_widget(Paragraph::new(v).style(DIM), at); + } +} + +/// The version of this binary, as long as fits in `room` columns: in full, +/// then the number alone, else nothing. The keys keep their room. +fn version_label(room: u16) -> Option { + let v = env!("CARGO_PKG_VERSION"); + [format!("taskguard {v}"), format!("v{v}")].into_iter().find(|t| t.chars().count() as u16 <= room) } /// A key, what it does, and the key a click presses. @@ -2034,6 +2048,24 @@ mod tests { assert!(app.paused && !app.keys_open, "a click on 'space' in the list pauses and closes it"); } + #[test] + fn the_footer_shows_the_version_at_the_right_when_the_keys_leave_room() { + let (_t, mut app) = fixture(); + let v = env!("CARGO_PKG_VERSION"); + let s = screen(&mut app, 200, 20); + let foot = s.lines().last().unwrap().trim_end(); + assert!(foot.starts_with("? keys") && foot.ends_with(&format!("taskguard {v}")), "{foot:?}"); + // Narrow, the keys keep their room: the number alone, or nothing. + for w in [40, 60, 80, 100, 120] { + let s = screen(&mut app, w, 20); + let foot = s.lines().last().unwrap(); + assert!(foot.starts_with("? keys") && foot.chars().count() <= w as usize, "{w}: {foot:?}"); + let keys = foot.trim_end().trim_end_matches(&format!("taskguard {v}")).trim_end_matches(&format!("v{v}")).trim_end(); + let wide = screen(&mut app, 200, 20); + assert!(wide.lines().last().unwrap().starts_with(keys), "{w}: the version hid a key: {foot:?}"); + } + } + #[test] fn question_mark_lists_every_key_and_the_footer_starts_with_it() { let (_t, mut app) = fixture(); From 4ae9d40a6aa32a38c49bb0f2566d6d576219f3b0 Mon Sep 17 00:00:00 2001 From: Vincent De Snerck <10668+SpoBo@users.noreply.github.com> Date: Sat, 17 Oct 2026 13:02:16 +0200 Subject: [PATCH 22/22] chore: release 0.11.0-rc.2 --- CHANGELOG.md | 5 ++++- Cargo.lock | 2 +- Cargo.toml | 2 +- 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4890113..9a2f107 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,8 @@ and the versions follow [semver](https://semver.org). ## [Unreleased] +## [0.11.0-rc.2] - 2026-10-17 + ### Added - `taskguard top` shows its version at the right of the key bar, in the @@ -405,7 +407,8 @@ and the versions follow [semver](https://semver.org). needs, and starts it when the machine has room for it. - Prebuilt binaries for macOS and Linux on each version tag. -[Unreleased]: https://github.com/SpoBo/taskguard/compare/v0.11.0-rc.1...HEAD +[Unreleased]: https://github.com/SpoBo/taskguard/compare/v0.11.0-rc.2...HEAD +[0.11.0-rc.2]: https://github.com/SpoBo/taskguard/compare/v0.11.0-rc.1...v0.11.0-rc.2 [0.11.0-rc.1]: https://github.com/SpoBo/taskguard/compare/v0.10.0...v0.11.0-rc.1 [0.10.0]: https://github.com/SpoBo/taskguard/compare/v0.9.0...v0.10.0 [0.9.0]: https://github.com/SpoBo/taskguard/compare/v0.8.0...v0.9.0 diff --git a/Cargo.lock b/Cargo.lock index af5c78d..c989ec2 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1434,7 +1434,7 @@ dependencies = [ [[package]] name = "taskguard" -version = "0.11.0-rc.1" +version = "0.11.0-rc.2" dependencies = [ "anyhow", "crossterm", diff --git a/Cargo.toml b/Cargo.toml index a0f1072..b66fa28 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "taskguard" -version = "0.11.0-rc.1" +version = "0.11.0-rc.2" edition = "2024" rust-version = "1.89" description = "A sem that learns what each job needs: start a command only when the machine has room for its CPU and memory"