From b82a63421becf5920cf1fef0598c828485f9ccef Mon Sep 17 00:00:00 2001 From: Ryan Duguid Date: Sat, 31 Oct 2026 08:22:28 +1100 Subject: [PATCH 1/2] Keep the typed answer deadline when an answer wait starts after recovery's budget An answer wait reached just before the recovery episode ended reported the typed answer deadline. Reached just after it, the shared budget guard returned an untyped error without polling the wait. Provider eligibility depends on the type, so the same expiry withheld the Grok fallback on one side of that instant. The answer wait now always runs. Its limit is zero once the episode has ended, so it mints its own typed deadline. The acknowledgement keeps the guard: an acknowledgement that does not land stays ineligible. --- CHANGELOG.md | 1 + src/browser_runner.rs | 2 +- src/browser_runner/read_recovery.rs | 17 ++++++++++ src/browser_runner_tests.rs | 49 +++++++++++++++++++++++++++-- 4 files changed, 65 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5588961..8a1618f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,4 +14,5 @@ First public release. Industrious is a Rust port of Calvino, a maintained fork o - File selection fails closed: `.gitignore` rules apply from the root of the repository holding each selected path, a `.gitignore` that cannot be read or parsed is an error, files read into an API prompt or a browser text bundle must be UTF-8 text rather than being garbled (browser uploads and ZIP bundles still send the original bytes), and credential and private-key files are refused by name and skipped in expansion. - File sections in rendered prompts and bundles keep every line that ended, dropping only the final line ending, so a file ending in a whitespace-only line (as a diff can) arrives whole. - A tab retirement that has reached its outcome always records it: a slow disk write at the end of the 5 s budget no longer reports a false cleanup failure and leaves the record undecided, with the tab already closed. +- An answer wait reached just after the bounded original-target observation ended reports the answer deadline, as it does just before: a stall there no longer replaces that failure with an untyped error that withholds the Grok fallback. - Licensed under FSL-1.1-ALv2; each version also becomes available under Apache-2.0 two years after its release. diff --git a/src/browser_runner.rs b/src/browser_runner.rs index 4a56393..31cb1d8 100644 --- a/src/browser_runner.rs +++ b/src/browser_runner.rs @@ -4658,7 +4658,7 @@ async fn wait_original_answer( )?; let bounded = recovery.limits(limits); let initial = recovery - .within(wait_answer( + .answer_within(wait_answer( store, id, reservation, diff --git a/src/browser_runner/read_recovery.rs b/src/browser_runner/read_recovery.rs index 22defa7..342ab24 100644 --- a/src/browser_runner/read_recovery.rs +++ b/src/browser_runner/read_recovery.rs @@ -37,6 +37,23 @@ impl Budget { Instant::now() < deadline, "Original-target observation budget expired; recovery retained" ); + Self::settle(deadline, future).await + } + /// An answer wait is always polled, even after the episode ended: its limit from `limits` is + /// then zero, so it reports its own typed deadline, which decides provider eligibility. + pub(super) async fn answer_within( + &self, + future: impl std::future::Future>, + ) -> Result { + match self.deadline { + Some(deadline) => Self::settle(deadline, future).await, + None => future.await, + } + } + async fn settle( + deadline: Instant, + future: impl std::future::Future>, + ) -> Result { tokio::time::timeout_at(deadline, future).await.context( "Original-target observation did not settle before its deadline; recovery retained", )? diff --git a/src/browser_runner_tests.rs b/src/browser_runner_tests.rs index 1b119c4..64565d2 100644 --- a/src/browser_runner_tests.rs +++ b/src/browser_runner_tests.rs @@ -37,6 +37,7 @@ enum Fault { RetryOffered, ProviderLateCommit, ProviderLateCommitPendingAnswer, + ProviderLateCommitStalledBeforeAnswer, ProviderLateCommitLostAck, ProviderLateCommitChangedOwner, ProviderLateAnswer, @@ -558,6 +559,7 @@ impl Exchange { self.fault, Fault::ProviderLateCommit | Fault::ProviderLateCommitPendingAnswer + | Fault::ProviderLateCommitStalledBeforeAnswer | Fault::ProviderLateCommitLostAck | Fault::ProviderLateCommitChangedOwner ) && !self.provider_recovery_started() @@ -678,6 +680,9 @@ impl Exchange { } if cfg["action"] == "acknowledge" { self.commit = cfg["receipt"].clone(); + if matches!(self.fault, Fault::ProviderLateCommitStalledBeforeAnswer) { + hold_session_lock(&self.store, &self.id, RECOVERY_BUDGET); + } } return json!(true); } @@ -1023,7 +1028,9 @@ impl Exchange { let recovering = self.provider_recovery_started(); if matches!( self.fault, - Fault::ProviderLateCommitPendingAnswer | Fault::ProviderLateCommitChangedOwner + Fault::ProviderLateCommitPendingAnswer + | Fault::ProviderLateCommitStalledBeforeAnswer + | Fault::ProviderLateCommitChangedOwner ) || matches!(self.fault, Fault::ProviderLateAnswerDeadline) && !recovering { result["completionVisible"] = json!(false); @@ -2068,13 +2075,29 @@ fn limits() -> RunnerLimits { const WAITED_OUT_ATTACHMENT_FLOOR: Duration = Duration::from_secs(5); /// For closes the fixture refuses, which production watches for its whole budget. const REFUSED_CLOSE_WAIT: Duration = Duration::from_secs(1); +/// Keeps the session's writes waiting for `hold` from another thread, as a contended store would. +fn hold_session_lock(store: &SessionStore, id: &str, hold: Duration) { + let lock = DirectoryLock::acquire( + &store + .home() + .join("sessions-write-locks") + .join(format!("{id}.lock")), + Duration::from_secs(1), + ) + .unwrap(); + std::thread::spawn(move || { + std::thread::sleep(hold); + drop(lock); + }); +} +const RECOVERY_BUDGET: Duration = Duration::from_secs(3); /// Limits for cases whose commit or answer deadline expires before a bounded provider read. /// That read's whole budget is min(commit, answer), so the fixture's 500 ms leaves too little /// for its identity checks and receipts once a parallel suite stalls native publication. fn recovery_limits() -> RunnerLimits { RunnerLimits { - commit: Duration::from_secs(3), - answer: Duration::from_secs(3), + commit: RECOVERY_BUDGET, + answer: RECOVERY_BUDGET, ..limits() } } @@ -3994,6 +4017,26 @@ async fn bounded_provider_read_does_not_restart_full_answer_timeout_after_commit stop(f).await; } #[tokio::test] +async fn an_answer_wait_reached_after_the_recovery_budget_keeps_the_typed_answer_deadline() { + let f = fixture(Fault::ProviderLateCommitStalledBeforeAnswer, false).await; + let mut bounded = recovery_limits(); + bounded.answer = Duration::from_secs(60); + // The session lock is held from the acknowledgement for the whole budget, so the answer wait + // starts after the episode that recovered the commitment has ended. + let error = run_within(&f, &bounded).await.unwrap_err(); + assert!( + matches!( + error.downcast_ref::(), + Some(crate::browser_provider::Failure::AnswerDeadline) + ), + "{error:#}" + ); + assert_eq!(f.sends.load(Ordering::SeqCst), 1); + assert_eq!(f.closes.load(Ordering::SeqCst), 0); + assert!(captured(&f.store, &f.id).is_err()); + stop(f).await; +} +#[tokio::test] async fn bounded_provider_read_caps_recovered_ack_and_revalidates_used_episode() { for fault in [ Fault::ProviderLateCommitLostAck, From aa342f024cf33be3dc28c1d34fc2f7508ce80b7b Mon Sep 17 00:00:00 2001 From: Ryan Duguid Date: Sat, 31 Oct 2026 09:00:35 +1100 Subject: [PATCH 2/2] Do not start an answer wait once recovery's budget has expired Tokio polls a timeout's future before its deadline, so the wait could still run after expiry. A wait whose limit predated the expiry then went Pending and returned the untyped timeout instead of the answer deadline. Return the typed deadline without polling the wait. A wait still unsettled when the deadline passes stays ineligible, as a hung read says nothing of the provider. --- src/browser_runner/read_recovery.rs | 55 ++++++++++++++++++++++++++--- 1 file changed, 50 insertions(+), 5 deletions(-) diff --git a/src/browser_runner/read_recovery.rs b/src/browser_runner/read_recovery.rs index 342ab24..d55d83b 100644 --- a/src/browser_runner/read_recovery.rs +++ b/src/browser_runner/read_recovery.rs @@ -39,16 +39,19 @@ impl Budget { ); Self::settle(deadline, future).await } - /// An answer wait is always polled, even after the episode ended: its limit from `limits` is - /// then zero, so it reports its own typed deadline, which decides provider eligibility. + /// An answer wait due to start after the episode ended has met its deadline, so it never + /// starts. The typed failure decides provider eligibility; `observe` revalidates authority. pub(super) async fn answer_within( &self, future: impl std::future::Future>, ) -> Result { - match self.deadline { - Some(deadline) => Self::settle(deadline, future).await, - None => future.await, + let Some(deadline) = self.deadline else { + return future.await; + }; + if Instant::now() >= deadline { + return Err(crate::browser_provider::Failure::AnswerDeadline.into()); } + Self::settle(deadline, future).await } async fn settle( deadline: Instant, @@ -286,3 +289,45 @@ impl<'a> Context<'a> { self.require(cancel) } } + +#[cfg(test)] +mod tests { + use super::*; + use std::{cell::Cell, task::Poll}; + + fn budget(deadline: Instant) -> Budget { + Budget { + enabled: true, + deadline: Some(deadline), + } + } + #[tokio::test] + async fn an_answer_wait_after_the_budget_is_not_polled_and_reports_the_answer_deadline() { + let polls = Cell::new(0); + let wait = std::future::poll_fn(|_| { + polls.set(polls.get() + 1); + Poll::>::Pending + }); + let error = budget(Instant::now()) + .answer_within(wait) + .await + .unwrap_err(); + assert!( + matches!( + error.downcast_ref::(), + Some(crate::browser_provider::Failure::AnswerDeadline) + ), + "{error:#}" + ); + assert_eq!(polls.get(), 0); + } + #[tokio::test] + async fn an_answer_wait_still_unsettled_at_the_deadline_stays_ineligible() { + let deadline = Instant::now() + Duration::from_millis(50); + let error = budget(deadline) + .answer_within(std::future::pending::>()) + .await + .unwrap_err(); + assert!(!eligible(&error), "{error:#}"); + } +}