From 53ba20530a03f083ce5a10ae7e32c7783803a048 Mon Sep 17 00:00:00 2001 From: Ryan Duguid Date: Sat, 31 Oct 2026 00:53:04 +1100 Subject: [PATCH 1/4] Record a decided tab retirement even when the write outlasts its budget Retirement ran its operation to 100 ms before the deadline, then published the outcome under the same deadline. A durable write that missed the tail returned an error, leaving the record at its earlier state; one that landed but finished late returned an error with the outcome already on disk. Neither depends on the size of the budget, because the tail is fixed. Publish the decided outcome without a deadline, under the record's lock. The deadline still bounds lock acquisition and the operation, and a budget that has already expired still publishes nothing. --- CHANGELOG.md | 1 + src/browser_disposition.rs | 34 ++++++++++++++++++++++++++++++++-- 2 files changed, 33 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f5dcebd..5588961 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,4 +13,5 @@ First public release. Industrious is a Rust port of Calvino, a maintained fork o - `serve` prints its access token only when it generates one, adds `--token-file`, compares tokens without early exit, refuses an empty token and answers HTTP 413 `payload_too_large` for run requests over 1,432,704,342 bytes (Oracle's limit). - File selection fails closed: `.gitignore` rules apply from the root of the repository holding each selected path, a `.gitignore` that cannot be read or parsed is an error, files read into an API prompt or a browser text bundle must be UTF-8 text rather than being garbled (browser uploads and ZIP bundles still send the original bytes), and credential and private-key files are refused by name and skipped in expansion. - File sections in rendered prompts and bundles keep every line that ended, dropping only the final line ending, so a file ending in a whitespace-only line (as a diff can) arrives whole. +- A tab retirement that has reached its outcome always records it: a slow disk write at the end of the 5 s budget no longer reports a false cleanup failure and leaves the record undecided, with the tab already closed. - Licensed under FSL-1.1-ALv2; each version also becomes available under Apache-2.0 two years after its release. diff --git a/src/browser_disposition.rs b/src/browser_disposition.rs index 341f13c..74eedb2 100644 --- a/src/browser_disposition.rs +++ b/src/browser_disposition.rs @@ -386,7 +386,7 @@ impl TargetJournal { Self::saved_retirement_id(&record)?; } remaining(operation_deadline)?; - // Leave a tail for the disposition write if a CDP operation stalls. + // Stop a stalled CDP operation early so its disposition normally lands in budget. let result = tokio::time::timeout(remaining(operation_deadline)?, retire(record.clone())) .await; @@ -426,7 +426,10 @@ impl TargetJournal { &reason, &absence, ); - let saved = self.save_until(&record, deadline)?; + // The outcome is decided, so its record outlasts the deadline: a stalled write that gave + // up here would report a false failure and leave the retirement undecided. The deadline + // bounds the operation and the lock above. This write stays under the record's lock. + let saved = self.save(&record)?; lock.release()?; Ok(saved["disposition"].clone()) } @@ -1526,6 +1529,33 @@ mod tests { assert!(journal.retirement_id(&creator, &changed).is_err()); assert!(journal.retirement_id(&creator, &commit).is_err()); } + #[tokio::test] + async fn a_decision_reached_as_the_budget_ends_is_published_despite_the_stall() { + for (budget, absence, decision) in [ + (Duration::from_secs(2), "confirmed-absent", "retired"), + (Duration::from_millis(500), "unknown", "retained"), + ] { + let home = tempfile::tempdir().unwrap(); + let (journal, creator, commit) = native(home.path()); + journal.commit(&creator, &commit).unwrap(); + journal.finalise(&commit).unwrap(); + let deadline = Instant::now() + budget; + // Blocking stands in for a stall between the evidence and its durable write; the + // operation still completes on its first poll, as a real close can. + let result = journal + .retire_record_until(&creator, deadline, |_| async move { + std::thread::sleep(budget + Duration::from_millis(50)); + Ok((absence.to_owned(), "PUBLIC stalled evidence".to_owned())) + }) + .await + .unwrap(); + assert_eq!(result["decision"], decision); + assert_eq!(result["absence"], absence); + assert!(Instant::now() >= deadline); + let saved = journal.read(&creator["allocationId"]).unwrap(); + assert_eq!(saved["disposition"], result); + } + } #[test] fn creator_replacement_and_directory_replacement_refuse_writes() { let home = tempfile::tempdir().unwrap(); From 9ec0dbdf27c2beb3d840b3c999b686f7360c38c5 Mon Sep 17 00:00:00 2001 From: Ryan Duguid Date: Sat, 31 Oct 2026 01:55:28 +1100 Subject: [PATCH 2/4] Release a late retirement's lock on its own deadline The lock kept the deadline it was acquired under, so after a late publication its release had no time left. If another process held the recovery mutex at that moment, the release failed and was not retried, stranding the lock and returning an error for a disposition already on disk. --- src/browser_disposition.rs | 35 ++++++++++++++++++++++++++++++++++- src/storage.rs | 14 ++++++++++++++ 2 files changed, 48 insertions(+), 1 deletion(-) diff --git a/src/browser_disposition.rs b/src/browser_disposition.rs index 74eedb2..8b3f267 100644 --- a/src/browser_disposition.rs +++ b/src/browser_disposition.rs @@ -430,7 +430,9 @@ impl TargetJournal { // up here would report a false failure and leave the retirement undecided. The deadline // bounds the operation and the lock above. This write stays under the record's lock. let saved = self.save(&record)?; - lock.release()?; + // The lock would otherwise inherit the spent deadline, and a release that fails once is + // not retried, so a contended release would strand it. + lock.release_until(Instant::now() + Duration::from_secs(5))?; Ok(saved["disposition"].clone()) } pub(crate) async fn retire_settled_before_dispatch( @@ -1556,6 +1558,37 @@ mod tests { assert_eq!(saved["disposition"], result); } } + #[tokio::test] + async fn a_late_publication_releases_its_lock_despite_contention() { + let home = tempfile::tempdir().unwrap(); + let (journal, creator, commit) = native(home.path()); + journal.commit(&creator, &commit).unwrap(); + journal.finalise(&commit).unwrap(); + let budget = Duration::from_millis(500); + let lock = journal + .file(&creator["allocationId"]) + .unwrap() + .with_extension("lock"); + let contender = lock.clone(); + let holder = std::cell::Cell::new(None); + let result = journal + .retire_record_until(&creator, Instant::now() + budget, |_| { + let holder = &holder; + async move { + std::thread::sleep(budget + Duration::from_millis(50)); + holder.set(Some(crate::storage::hold_recovery_mutex( + &contender, + Duration::from_millis(500), + ))); + Ok(("unknown".to_owned(), "PUBLIC stalled evidence".to_owned())) + } + }) + .await + .unwrap(); + assert_eq!(result["decision"], "retained"); + assert!(!lock.exists()); + holder.take().unwrap().join().unwrap(); + } #[test] fn creator_replacement_and_directory_replacement_refuse_writes() { let home = tempfile::tempdir().unwrap(); diff --git a/src/storage.rs b/src/storage.rs index b810c41..fb2cc4a 100644 --- a/src/storage.rs +++ b/src/storage.rs @@ -552,6 +552,20 @@ fn normalise_windows_path_identity(raw: &str) -> String { raw.to_lowercase() } +/// Holds a lock's recovery mutex on another thread, returning once it is held. +#[cfg(test)] +pub(crate) fn hold_recovery_mutex(path: &Path, hold: Duration) -> std::thread::JoinHandle<()> { + let path = path.to_owned(); + let (held, wait) = std::sync::mpsc::channel(); + let holder = std::thread::spawn(move || { + let mutex = RecoveryMutex::acquire(&path, Duration::from_secs(5)).unwrap(); + held.send(()).unwrap(); + std::thread::sleep(hold); + drop(mutex); + }); + wait.recv().unwrap(); + holder +} struct RecoveryMutex { #[cfg(unix)] _file: File, From e130e7d2bc265953160a9c82d73eb326d1bda867 Mon Sep 17 00:00:00 2001 From: Ryan Duguid Date: Sat, 31 Oct 2026 02:21:26 +1100 Subject: [PATCH 3/4] Stall the retirement tests before the operation's future The blocking sleep sat inside async code, which SonarCloud reports as a reliability bug. It now runs in the closure, so the operation still completes on its first poll. --- src/browser_disposition.rs | 30 +++++++++++++++++------------- 1 file changed, 17 insertions(+), 13 deletions(-) diff --git a/src/browser_disposition.rs b/src/browser_disposition.rs index 8b3f267..faf7a3f 100644 --- a/src/browser_disposition.rs +++ b/src/browser_disposition.rs @@ -1542,12 +1542,16 @@ mod tests { journal.commit(&creator, &commit).unwrap(); journal.finalise(&commit).unwrap(); let deadline = Instant::now() + budget; - // Blocking stands in for a stall between the evidence and its durable write; the - // operation still completes on its first poll, as a real close can. + // Blocking before the operation starts stands in for a stall between the evidence and + // its durable write; the operation still completes on its first poll, as a real close + // can. let result = journal - .retire_record_until(&creator, deadline, |_| async move { + .retire_record_until(&creator, deadline, |_| { std::thread::sleep(budget + Duration::from_millis(50)); - Ok((absence.to_owned(), "PUBLIC stalled evidence".to_owned())) + std::future::ready(Ok(( + absence.to_owned(), + "PUBLIC stalled evidence".to_owned(), + ))) }) .await .unwrap(); @@ -1573,15 +1577,15 @@ mod tests { let holder = std::cell::Cell::new(None); let result = journal .retire_record_until(&creator, Instant::now() + budget, |_| { - let holder = &holder; - async move { - std::thread::sleep(budget + Duration::from_millis(50)); - holder.set(Some(crate::storage::hold_recovery_mutex( - &contender, - Duration::from_millis(500), - ))); - Ok(("unknown".to_owned(), "PUBLIC stalled evidence".to_owned())) - } + std::thread::sleep(budget + Duration::from_millis(50)); + holder.set(Some(crate::storage::hold_recovery_mutex( + &contender, + Duration::from_millis(500), + ))); + std::future::ready(Ok(( + "unknown".to_owned(), + "PUBLIC stalled evidence".to_owned(), + ))) }) .await .unwrap(); From 183081726f994c6914b1f3b8a795486471886b51 Mon Sep 17 00:00:00 2001 From: Ryan Duguid Date: Sat, 31 Oct 2026 02:26:00 +1100 Subject: [PATCH 4/4] Keep the retirement tests' stall out of async code SonarCloud also reads a closure inside an async test as async, so the blocking stall moves to a plain helper. --- src/browser_disposition.rs | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/browser_disposition.rs b/src/browser_disposition.rs index faf7a3f..e5abec7 100644 --- a/src/browser_disposition.rs +++ b/src/browser_disposition.rs @@ -1531,6 +1531,10 @@ mod tests { assert!(journal.retirement_id(&creator, &changed).is_err()); assert!(journal.retirement_id(&creator, &commit).is_err()); } + /// Blocks the thread past the budget, as a stalled disk would; a timer would yield instead. + fn stall_past(budget: Duration) { + std::thread::sleep(budget + Duration::from_millis(50)); + } #[tokio::test] async fn a_decision_reached_as_the_budget_ends_is_published_despite_the_stall() { for (budget, absence, decision) in [ @@ -1547,7 +1551,7 @@ mod tests { // can. let result = journal .retire_record_until(&creator, deadline, |_| { - std::thread::sleep(budget + Duration::from_millis(50)); + stall_past(budget); std::future::ready(Ok(( absence.to_owned(), "PUBLIC stalled evidence".to_owned(), @@ -1577,7 +1581,7 @@ mod tests { let holder = std::cell::Cell::new(None); let result = journal .retire_record_until(&creator, Instant::now() + budget, |_| { - std::thread::sleep(budget + Duration::from_millis(50)); + stall_past(budget); holder.set(Some(crate::storage::hold_recovery_mutex( &contender, Duration::from_millis(500),