From 267b27d385e7aa5f235da448b40da01df9f0f6bd Mon Sep 17 00:00:00 2001 From: joshyorko Date: Fri, 2 Oct 2026 15:04:50 -0400 Subject: [PATCH 01/18] test(dreamer): capture native startup and input-budget regressions --- tests/cli_smoke.rs | 124 ++++++++++++++++++++++++++++ tests/dream_command_scheduler.rs | 136 +++++++++++++++++++++++++++++++ 2 files changed, 260 insertions(+) diff --git a/tests/cli_smoke.rs b/tests/cli_smoke.rs index 8ba198a..c5a8f5e 100644 --- a/tests/cli_smoke.rs +++ b/tests/cli_smoke.rs @@ -5872,6 +5872,130 @@ fn cli_native_runtime_env_status_keeps_daemon_status_contract() { assert!(body.contains("\"dream_worker\"")); } +#[cfg(target_os = "linux")] +#[test] +fn cli_native_up_keeps_dreamer_degraded_daemon_live_and_recallable() { + use codex_memoryd::protocol::ConclusionsRequest; + use codex_memoryd::store::DreamRunRecord; + use codex_memoryd::{config::Config, service::Service, store::Store}; + + let dir = TempDir::new().unwrap(); + let home = dir.path().join("managed-home"); + let db = dir.path().join("synthetic.db"); + let bind = unused_loopback_addr(); + let url = format!("http://{bind}"); + + let store = Store::open(&db).unwrap(); + let mut config = Config::default(); + config.default_profile = "personal".into(); + config.default_workspace = "synthetic-scope".into(); + let service = Service::new(store.clone(), config); + service + .conclusions(ConclusionsRequest { + profile: Some("personal".into()), + workspace: Some("synthetic-scope".into()), + repo: None, + target: Some("user".into()), + conclusions: Some(vec![ + "Decision: synthetic lifecycle sentinel remains recallable".into(), + ]), + metadata: None, + record_type: Some("decision".into()), + }) + .unwrap(); + store + .record_dream_run(&DreamRunRecord { + run_id: "dream_failed_preview_fixture".into(), + profile_id: "personal".into(), + workspace_id: "synthetic-scope".into(), + mode: "preview".into(), + kind: "dream_preview".into(), + status: "error".into(), + started_at: "2030-01-01T00:00:00Z".into(), + completed_at: Some("2030-01-01T00:00:01Z".into()), + error: Some("dream provider input byte budget exhausted".into()), + ..Default::default() + }) + .unwrap(); + drop(service); + drop(store); + + let mut up = bin(); + clear_runtime_env(&mut up); + let up_output = up + .env("CODEX_MEMORYD_HOME", &home) + .env("CODEX_MEMORYD_DB", &db) + .env("CODEX_MEMORYD_HOST", "127.0.0.1") + .env("CODEX_MEMORYD_PORT", bind.rsplit_once(':').unwrap().1) + .env("CODEX_MEMORYD_PROFILE", "personal") + .env("CODEX_MEMORYD_WORKSPACE", "synthetic-scope") + .args(["--runtime", "native", "up"]) + .output() + .unwrap(); + + let mut status_ok = false; + let mut recall_ok = false; + if up_output.status.success() { + let http = reqwest::blocking::Client::new(); + let response = http.get(format!("{url}/v1/status")).send().unwrap(); + let body: serde_json::Value = response.json().unwrap(); + status_ok = body.pointer("/data/status").and_then(Value::as_str) == Some("degraded") + && body.pointer("/data/storage/writable").and_then(Value::as_bool) == Some(true) + && body + .pointer("/data/degraded_reasons") + .and_then(Value::as_array) + .is_some_and(|reasons| { + reasons.iter().any(|reason| { + reason.as_str().is_some_and(|text| { + text.contains("dream provider input byte budget exhausted") + }) + }) + }); + let mut recall = bin(); + clear_runtime_env(&mut recall); + let recall_output = recall + .arg("--url") + .arg(&url) + .args([ + "recall", + "--profile", + "personal", + "--workspace", + "synthetic-scope", + "--query", + "lifecycle sentinel", + ]) + .output() + .unwrap(); + recall_ok = recall_output.status.success() + && String::from_utf8_lossy(&recall_output.stdout).contains("synthetic lifecycle sentinel remains recallable"); + } + + let mut down = bin(); + clear_runtime_env(&mut down); + let down_output = down + .env("CODEX_MEMORYD_HOME", &home) + .env("CODEX_MEMORYD_DB", &db) + .env("CODEX_MEMORYD_HOST", "127.0.0.1") + .env("CODEX_MEMORYD_PORT", bind.rsplit_once(':').unwrap().1) + .args(["--runtime", "native", "down"]) + .output() + .unwrap(); + + assert!( + up_output.status.success(), + "native up failed: {}", + String::from_utf8_lossy(&up_output.stderr) + ); + assert!(status_ok, "degraded Dreamer receipt was not preserved in live status"); + assert!(recall_ok, "configured-scope recall failed after native startup"); + assert!( + down_output.status.success(), + "native down failed: {}", + String::from_utf8_lossy(&down_output.stderr) + ); +} + #[test] fn cli_rejects_invalid_runtime_environment_for_status_and_lifecycle() { let dir = TempDir::new().unwrap(); diff --git a/tests/dream_command_scheduler.rs b/tests/dream_command_scheduler.rs index 9531b4e..d0e4f2d 100644 --- a/tests/dream_command_scheduler.rs +++ b/tests/dream_command_scheduler.rs @@ -58,6 +58,142 @@ fn failed_scheduled_command_does_not_advance_success_watermark() { .is_none()); } +#[cfg(target_os = "linux")] +#[test] +fn oversized_command_preview_records_a_limit_and_advances_without_skipping_tail() { + let store = Store::open(":memory:").unwrap(); + store.ensure_workspace("personal", "ws").unwrap(); + store + .ensure_session( + "oversized-session", + "personal", + "ws", + None, + None, + "fixture", + ) + .unwrap(); + store + .insert_visible_turn(&VisibleTurn { + id: "oversized-turn".into(), + session_id: "oversized-session".into(), + actor: "user".into(), + content: format!("synthetic {}", "x".repeat(40_000)), + created_at: "2026-09-11T00:00:00Z".into(), + metadata: json!({}), + }) + .unwrap(); + + let temp = tempfile::tempdir().unwrap(); + let marker = temp.path().join("provider-called"); + let mut config = Config::default(); + config.default_profile = "personal".into(); + config.default_workspace = "ws".into(); + config.dream_scheduler.enabled = true; + config.dream_scheduler.scheduled_provider_enabled = true; + config.dream_scheduler.max_batch_size = 1; + config.dream_scheduler.idle_window_seconds = 0; + config.dream_scheduler.min_session_age_seconds = 0; + config.dream_scheduler.min_turn_count = 0; + config.dream_provider.enabled = true; + config.dream_provider.adapter = "command".into(); + config.dream_provider.model = "synthetic-model".into(); + config.dream_provider.command = vec![ + "/bin/sh".into(), + "-c".into(), + format!( + "touch '{}'; cat >/dev/null; printf '{{\"schema_version\":\"dream-preview-v1\",\"profile\":\"personal\",\"workspace\":\"ws\",\"candidates\":[]}}'", + marker.display() + ), + ]; + let service = Service::new(store.clone(), config); + + let first = service + .scheduled_dream(Some("2030-01-01T00:00:00Z".into())) + .unwrap(); + assert_eq!(first.status, "ok_with_limits"); + assert!(first + .limits_hit + .iter() + .any(|limit| limit == "max_input_bytes")); + assert!(first.watermark_after.is_some()); + assert!(!marker.exists(), "oversized evidence must not reach the provider"); + + let second = service + .scheduled_dream(Some("2030-01-02T00:00:00Z".into())) + .unwrap(); + assert_ne!(second.watermark_before, first.watermark_before); + assert!(!marker.exists()); +} + +#[cfg(target_os = "linux")] +#[test] +fn command_budget_reduces_a_window_then_processes_its_source_tail() { + let store = Store::open(":memory:").unwrap(); + store.ensure_workspace("personal", "ws").unwrap(); + store + .ensure_session( + "budget-window-session", + "personal", + "ws", + None, + None, + "fixture", + ) + .unwrap(); + for (id, minute) in [("turn-a", "00"), ("turn-b", "01")] { + store + .insert_visible_turn(&VisibleTurn { + id: id.into(), + session_id: "budget-window-session".into(), + actor: "user".into(), + content: format!("synthetic {id} {}", "x".repeat(18_000)), + created_at: format!("2026-09-11T00:{minute}:00Z"), + metadata: json!({}), + }) + .unwrap(); + } + let temp = tempfile::tempdir().unwrap(); + let calls = temp.path().join("provider-calls"); + let mut config = Config::default(); + config.default_profile = "personal".into(); + config.default_workspace = "ws".into(); + config.dream_scheduler.enabled = true; + config.dream_scheduler.scheduled_provider_enabled = true; + config.dream_scheduler.max_batch_size = 2; + config.dream_scheduler.idle_window_seconds = 0; + config.dream_scheduler.min_session_age_seconds = 0; + config.dream_scheduler.min_turn_count = 0; + config.dream_provider.enabled = true; + config.dream_provider.adapter = "command".into(); + config.dream_provider.model = "synthetic-model".into(); + config.dream_provider.command = vec![ + "/bin/sh".into(), + "-c".into(), + format!( + "cat >/dev/null; echo call >> '{}'; printf '{{\"schema_version\":\"dream-preview-v1\",\"profile\":\"personal\",\"workspace\":\"ws\",\"candidates\":[]}}'", + calls.display() + ), + ]; + let service = Service::new(store, config); + + let first = service + .scheduled_dream(Some("2030-01-01T00:00:00Z".into())) + .unwrap(); + assert!(first + .limits_hit + .iter() + .any(|limit| limit == "max_input_tokens" || limit == "max_input_bytes")); + assert_eq!(first.run.as_ref().unwrap().evidence_window.visible_turns.count, 1); + assert_eq!(std::fs::read_to_string(&calls).unwrap().lines().count(), 1); + + let second = service + .scheduled_dream(Some("2030-01-02T00:00:00Z".into())) + .unwrap(); + assert_eq!(second.run.as_ref().unwrap().evidence_window.visible_turns.count, 1); + assert_eq!(std::fs::read_to_string(&calls).unwrap().lines().count(), 2); +} + #[cfg(target_os = "linux")] #[test] fn scheduled_command_excludes_archived_sources_from_preview() { From ed16cccc174430a3ecd8b99a07076de8a0ce2755 Mon Sep 17 00:00:00 2001 From: joshyorko Date: Fri, 2 Oct 2026 15:16:17 -0400 Subject: [PATCH 02/18] style(tests): format Dreamer regression checkpoint --- tests/cli_smoke.rs | 18 ++++++++++++---- tests/dream_command_scheduler.rs | 36 ++++++++++++++++++++++---------- 2 files changed, 39 insertions(+), 15 deletions(-) diff --git a/tests/cli_smoke.rs b/tests/cli_smoke.rs index c5a8f5e..e6c0b04 100644 --- a/tests/cli_smoke.rs +++ b/tests/cli_smoke.rs @@ -5940,7 +5940,10 @@ fn cli_native_up_keeps_dreamer_degraded_daemon_live_and_recallable() { let response = http.get(format!("{url}/v1/status")).send().unwrap(); let body: serde_json::Value = response.json().unwrap(); status_ok = body.pointer("/data/status").and_then(Value::as_str) == Some("degraded") - && body.pointer("/data/storage/writable").and_then(Value::as_bool) == Some(true) + && body + .pointer("/data/storage/writable") + .and_then(Value::as_bool) + == Some(true) && body .pointer("/data/degraded_reasons") .and_then(Value::as_array) @@ -5968,7 +5971,8 @@ fn cli_native_up_keeps_dreamer_degraded_daemon_live_and_recallable() { .output() .unwrap(); recall_ok = recall_output.status.success() - && String::from_utf8_lossy(&recall_output.stdout).contains("synthetic lifecycle sentinel remains recallable"); + && String::from_utf8_lossy(&recall_output.stdout) + .contains("synthetic lifecycle sentinel remains recallable"); } let mut down = bin(); @@ -5987,8 +5991,14 @@ fn cli_native_up_keeps_dreamer_degraded_daemon_live_and_recallable() { "native up failed: {}", String::from_utf8_lossy(&up_output.stderr) ); - assert!(status_ok, "degraded Dreamer receipt was not preserved in live status"); - assert!(recall_ok, "configured-scope recall failed after native startup"); + assert!( + status_ok, + "degraded Dreamer receipt was not preserved in live status" + ); + assert!( + recall_ok, + "configured-scope recall failed after native startup" + ); assert!( down_output.status.success(), "native down failed: {}", diff --git a/tests/dream_command_scheduler.rs b/tests/dream_command_scheduler.rs index d0e4f2d..f88e057 100644 --- a/tests/dream_command_scheduler.rs +++ b/tests/dream_command_scheduler.rs @@ -64,14 +64,7 @@ fn oversized_command_preview_records_a_limit_and_advances_without_skipping_tail( let store = Store::open(":memory:").unwrap(); store.ensure_workspace("personal", "ws").unwrap(); store - .ensure_session( - "oversized-session", - "personal", - "ws", - None, - None, - "fixture", - ) + .ensure_session("oversized-session", "personal", "ws", None, None, "fixture") .unwrap(); store .insert_visible_turn(&VisibleTurn { @@ -117,7 +110,10 @@ fn oversized_command_preview_records_a_limit_and_advances_without_skipping_tail( .iter() .any(|limit| limit == "max_input_bytes")); assert!(first.watermark_after.is_some()); - assert!(!marker.exists(), "oversized evidence must not reach the provider"); + assert!( + !marker.exists(), + "oversized evidence must not reach the provider" + ); let second = service .scheduled_dream(Some("2030-01-02T00:00:00Z".into())) @@ -184,13 +180,31 @@ fn command_budget_reduces_a_window_then_processes_its_source_tail() { .limits_hit .iter() .any(|limit| limit == "max_input_tokens" || limit == "max_input_bytes")); - assert_eq!(first.run.as_ref().unwrap().evidence_window.visible_turns.count, 1); + assert_eq!( + first + .run + .as_ref() + .unwrap() + .evidence_window + .visible_turns + .count, + 1 + ); assert_eq!(std::fs::read_to_string(&calls).unwrap().lines().count(), 1); let second = service .scheduled_dream(Some("2030-01-02T00:00:00Z".into())) .unwrap(); - assert_eq!(second.run.as_ref().unwrap().evidence_window.visible_turns.count, 1); + assert_eq!( + second + .run + .as_ref() + .unwrap() + .evidence_window + .visible_turns + .count, + 1 + ); assert_eq!(std::fs::read_to_string(&calls).unwrap().lines().count(), 2); } From 45e3c1fdedc052779b89b6af00d34f99049cd15f Mon Sep 17 00:00:00 2001 From: joshyorko Date: Fri, 2 Oct 2026 15:58:46 -0400 Subject: [PATCH 03/18] test(dreamer): reproduce full-wire exhaustion and native rejection --- tests/cli_smoke.rs | 113 +++++++++++++++++++++++--- tests/dream_command_scheduler.rs | 131 ++++++++++++++++++++++++++----- 2 files changed, 213 insertions(+), 31 deletions(-) diff --git a/tests/cli_smoke.rs b/tests/cli_smoke.rs index e6c0b04..c633a54 100644 --- a/tests/cli_smoke.rs +++ b/tests/cli_smoke.rs @@ -5876,7 +5876,7 @@ fn cli_native_runtime_env_status_keeps_daemon_status_contract() { #[test] fn cli_native_up_keeps_dreamer_degraded_daemon_live_and_recallable() { use codex_memoryd::protocol::ConclusionsRequest; - use codex_memoryd::store::DreamRunRecord; + use codex_memoryd::protocol::{DreamJobBudget, DreamJobRunRequest}; use codex_memoryd::{config::Config, service::Service, store::Store}; let dir = TempDir::new().unwrap(); @@ -5884,11 +5884,24 @@ fn cli_native_up_keeps_dreamer_degraded_daemon_live_and_recallable() { let db = dir.path().join("synthetic.db"); let bind = unused_loopback_addr(); let url = format!("http://{bind}"); + let provider_called = dir.path().join("provider-called"); + fs::create_dir_all(&home).unwrap(); let store = Store::open(&db).unwrap(); let mut config = Config::default(); config.default_profile = "personal".into(); config.default_workspace = "synthetic-scope".into(); + config.dream_provider.enabled = true; + config.dream_provider.adapter = "command".into(); + config.dream_provider.model = "synthetic-model".into(); + config.dream_provider.command = vec![ + "/bin/sh".into(), + "-c".into(), + format!( + "touch '{}'; cat >/dev/null; printf '{{\"schema_version\":\"dream-preview-v1\",\"profile\":\"personal\",\"workspace\":\"synthetic-scope\",\"candidates\":[]}}'", + provider_called.display() + ), + ]; let service = Service::new(store.clone(), config); service .conclusions(ConclusionsRequest { @@ -5904,25 +5917,98 @@ fn cli_native_up_keeps_dreamer_degraded_daemon_live_and_recallable() { }) .unwrap(); store - .record_dream_run(&DreamRunRecord { - run_id: "dream_failed_preview_fixture".into(), - profile_id: "personal".into(), - workspace_id: "synthetic-scope".into(), - mode: "preview".into(), - kind: "dream_preview".into(), - status: "error".into(), - started_at: "2030-01-01T00:00:00Z".into(), - completed_at: Some("2030-01-01T00:00:01Z".into()), - error: Some("dream provider input byte budget exhausted".into()), - ..Default::default() + .ensure_session( + "startup-budget-session", + "personal", + "synthetic-scope", + None, + None, + "fixture", + ) + .unwrap(); + store + .insert_visible_turn(&codex_memoryd::domain::VisibleTurn { + id: "startup-oversized-turn".into(), + session_id: "startup-budget-session".into(), + actor: "user".into(), + content: format!("synthetic {}", "🦉".repeat(9_000)), + created_at: "2026-10-01T00:00:00Z".into(), + metadata: serde_json::json!({}), }) .unwrap(); + + let request = |job_id: &str, mode: &str| DreamJobRunRequest { + job_id: Some(job_id.into()), + profile: Some("personal".into()), + workspace: Some("synthetic-scope".into()), + repo: None, + now: Some("2030-01-01T00:00:00Z".into()), + since: None, + since_explicit: false, + kind: "dream_preview".into(), + mode: Some(mode.into()), + budget: DreamJobBudget { + max_runtime_seconds: 10, + // The recall sentinel occupies one slot before visible evidence. + max_input_records: 2, + max_candidates: 5, + max_input_tokens: 8000, + max_input_bytes: 32000, + max_provider_calls: 1, + ..Default::default() + }, + provider: None, + }; + let selected = service + .run_dream_job(request("startup-budget-diagnostic", "deterministic")) + .unwrap(); + assert_eq!(selected.preview.evidence_window.visible_turns.count, 1); + assert_eq!( + selected.preview.evidence_window.visible_turns.sources[0].id, + "startup-oversized-turn" + ); + let preflight = service + .run_dream_job(request("startup-budget-failure", "command")) + .unwrap_err(); + assert_eq!( + preflight.message, + "dream provider input byte budget exhausted" + ); + assert!( + !provider_called.exists(), + "budget check must precede dispatch" + ); + let status_before_up = service.status().unwrap(); + assert_eq!(status_before_up.status, "degraded"); + assert!(status_before_up.storage.writable); + assert_eq!( + status_before_up.storage_schema_version, + codex_memoryd::store::STORAGE_SCHEMA_VERSION + ); + assert_eq!( + status_before_up + .features + .get("exposure") + .and_then(Value::as_str), + Some("local_only") + ); + assert!(status_before_up + .degraded_reasons + .iter() + .any(|reason| { reason.contains("dream provider input byte budget exhausted") })); + let persisted_failure = store.last_dream_run().unwrap().unwrap(); + assert_eq!(persisted_failure.status, "error"); + assert!(persisted_failure + .error_summary + .as_deref() + .is_some_and(|error| error.contains("dream provider input byte budget exhausted"))); drop(service); drop(store); let mut up = bin(); clear_runtime_env(&mut up); let up_output = up + .env("HOME", &home) .env("CODEX_MEMORYD_HOME", &home) .env("CODEX_MEMORYD_DB", &db) .env("CODEX_MEMORYD_HOST", "127.0.0.1") @@ -5957,6 +6043,8 @@ fn cli_native_up_keeps_dreamer_degraded_daemon_live_and_recallable() { let mut recall = bin(); clear_runtime_env(&mut recall); let recall_output = recall + .env("HOME", &home) + .env("CODEX_MEMORYD_HOME", &home) .arg("--url") .arg(&url) .args([ @@ -5978,6 +6066,7 @@ fn cli_native_up_keeps_dreamer_degraded_daemon_live_and_recallable() { let mut down = bin(); clear_runtime_env(&mut down); let down_output = down + .env("HOME", &home) .env("CODEX_MEMORYD_HOME", &home) .env("CODEX_MEMORYD_DB", &db) .env("CODEX_MEMORYD_HOST", "127.0.0.1") diff --git a/tests/dream_command_scheduler.rs b/tests/dream_command_scheduler.rs index f88e057..7c154cd 100644 --- a/tests/dream_command_scheduler.rs +++ b/tests/dream_command_scheduler.rs @@ -1,10 +1,34 @@ use codex_memoryd::domain::{Portability, RecordType, Scope, Sensitivity, VisibleTurn}; use codex_memoryd::ids; -use codex_memoryd::protocol::ConclusionsRequest; +use codex_memoryd::protocol::{ConclusionsRequest, DreamJobBudget, DreamJobRunRequest}; use codex_memoryd::store::NewRecord; use codex_memoryd::{config::Config, service::Service, store::Store}; use serde_json::json; +fn provider_job(job_id: &str, mode: &str, max_input_records: usize) -> DreamJobRunRequest { + DreamJobRunRequest { + job_id: Some(job_id.into()), + profile: Some("personal".into()), + workspace: Some("ws".into()), + repo: None, + now: Some("2030-01-01T00:00:00Z".into()), + since: None, + since_explicit: false, + kind: "dream_preview".into(), + mode: Some(mode.into()), + budget: DreamJobBudget { + max_runtime_seconds: 10, + max_input_records, + max_candidates: 5, + max_input_tokens: 8000, + max_input_bytes: 32000, + max_provider_calls: 1, + ..Default::default() + }, + provider: None, + } +} + fn service(script: &str) -> (Service, Store) { let store = Store::open(":memory:").unwrap(); let mut config = Config::default(); @@ -71,8 +95,8 @@ fn oversized_command_preview_records_a_limit_and_advances_without_skipping_tail( id: "oversized-turn".into(), session_id: "oversized-session".into(), actor: "user".into(), - content: format!("synthetic {}", "x".repeat(40_000)), - created_at: "2026-09-11T00:00:00Z".into(), + content: format!("synthetic {}", "🦉".repeat(9_000)), + created_at: "2026-10-01T00:00:00Z".into(), metadata: json!({}), }) .unwrap(); @@ -85,6 +109,7 @@ fn oversized_command_preview_records_a_limit_and_advances_without_skipping_tail( config.dream_scheduler.enabled = true; config.dream_scheduler.scheduled_provider_enabled = true; config.dream_scheduler.max_batch_size = 1; + config.dream_scheduler.max_candidates = 5; config.dream_scheduler.idle_window_seconds = 0; config.dream_scheduler.min_session_age_seconds = 0; config.dream_scheduler.min_turn_count = 0; @@ -101,9 +126,27 @@ fn oversized_command_preview_records_a_limit_and_advances_without_skipping_tail( ]; let service = Service::new(store.clone(), config); - let first = service - .scheduled_dream(Some("2030-01-01T00:00:00Z".into())) + let selected = service + .run_dream_job(provider_job("oversized-diagnostic", "deterministic", 1)) .unwrap(); + assert_eq!(selected.preview.evidence_window.visible_turns.count, 1); + assert_eq!( + selected.preview.evidence_window.visible_turns.sources[0].id, + "oversized-turn" + ); + let preflight = service + .run_dream_job(provider_job("oversized-preflight", "command", 1)) + .unwrap_err(); + assert_eq!( + preflight.message, + "dream provider input byte budget exhausted" + ); + assert!( + !marker.exists(), + "budget rejection must happen before dispatch" + ); + + let first = service.scheduled_dream(None).unwrap(); assert_eq!(first.status, "ok_with_limits"); assert!(first .limits_hit @@ -115,9 +158,7 @@ fn oversized_command_preview_records_a_limit_and_advances_without_skipping_tail( "oversized evidence must not reach the provider" ); - let second = service - .scheduled_dream(Some("2030-01-02T00:00:00Z".into())) - .unwrap(); + let second = service.scheduled_dream(None).unwrap(); assert_ne!(second.watermark_before, first.watermark_before); assert!(!marker.exists()); } @@ -143,8 +184,8 @@ fn command_budget_reduces_a_window_then_processes_its_source_tail() { id: id.into(), session_id: "budget-window-session".into(), actor: "user".into(), - content: format!("synthetic {id} {}", "x".repeat(18_000)), - created_at: format!("2026-09-11T00:{minute}:00Z"), + content: format!("synthetic {id} {}", "🦉".repeat(2_000)), + created_at: format!("2026-10-01T00:{minute}:00Z"), metadata: json!({}), }) .unwrap(); @@ -157,6 +198,7 @@ fn command_budget_reduces_a_window_then_processes_its_source_tail() { config.dream_scheduler.enabled = true; config.dream_scheduler.scheduled_provider_enabled = true; config.dream_scheduler.max_batch_size = 2; + config.dream_scheduler.max_candidates = 5; config.dream_scheduler.idle_window_seconds = 0; config.dream_scheduler.min_session_age_seconds = 0; config.dream_scheduler.min_turn_count = 0; @@ -173,13 +215,40 @@ fn command_budget_reduces_a_window_then_processes_its_source_tail() { ]; let service = Service::new(store, config); - let first = service - .scheduled_dream(Some("2030-01-01T00:00:00Z".into())) + let selected = service + .run_dream_job(provider_job("window-diagnostic", "deterministic", 2)) .unwrap(); - assert!(first - .limits_hit - .iter() - .any(|limit| limit == "max_input_tokens" || limit == "max_input_bytes")); + assert_eq!(selected.preview.evidence_window.visible_turns.count, 2); + assert_eq!( + selected + .preview + .evidence_window + .visible_turns + .sources + .iter() + .map(|source| source.id.as_str()) + .collect::>(), + ["turn-a", "turn-b"].into_iter().collect() + ); + let preflight = service + .run_dream_job(provider_job("window-preflight", "command", 2)) + .unwrap_err(); + assert_eq!( + preflight.message, + "dream provider input byte budget exhausted" + ); + assert!( + !calls.exists(), + "budget rejection must happen before dispatch" + ); + + let first = service.scheduled_dream(None).unwrap(); + assert!(first.limits_hit.iter().any(|limit| { + matches!( + limit.as_str(), + "max_input_records" | "max_input_tokens" | "max_input_bytes" + ) + })); assert_eq!( first .run @@ -192,9 +261,23 @@ fn command_budget_reduces_a_window_then_processes_its_source_tail() { ); assert_eq!(std::fs::read_to_string(&calls).unwrap().lines().count(), 1); - let second = service - .scheduled_dream(Some("2030-01-02T00:00:00Z".into())) - .unwrap(); + let second = service.scheduled_dream(None).unwrap(); + let first_source = &first + .run + .as_ref() + .unwrap() + .evidence_window + .visible_turns + .sources[0] + .id; + let second_source = &second + .run + .as_ref() + .unwrap() + .evidence_window + .visible_turns + .sources[0] + .id; assert_eq!( second .run @@ -205,6 +288,16 @@ fn command_budget_reduces_a_window_then_processes_its_source_tail() { .count, 1 ); + assert_ne!( + first_source, second_source, + "the reduced window must leave its source tail for the next run" + ); + assert_eq!( + [first_source.as_str(), second_source.as_str()] + .into_iter() + .collect::>(), + ["turn-a", "turn-b"].into_iter().collect() + ); assert_eq!(std::fs::read_to_string(&calls).unwrap().lines().count(), 2); } From 565c40774770a7376a2b2dc893da8bb54a8ee098 Mon Sep 17 00:00:00 2001 From: joshyorko Date: Fri, 2 Oct 2026 16:37:55 -0400 Subject: [PATCH 04/18] fix(native): preserve readiness and bound scheduled Dreamer input --- docs/dreamer-native-provider.md | 19 ++- src/native_runtime.rs | 263 +++++++++++++++++++++++++++++++- src/service.rs | 163 +++++++++++++++----- 3 files changed, 400 insertions(+), 45 deletions(-) diff --git a/docs/dreamer-native-provider.md b/docs/dreamer-native-provider.md index f432657..110db51 100644 --- a/docs/dreamer-native-provider.md +++ b/docs/dreamer-native-provider.md @@ -54,8 +54,23 @@ Scheduled command mode uses the existing typed preview job: one provider call, zero provider retries, bounded input/output, and a deadline covering pipe I/O. The Linux runner uses nonblocking pipes, not detached reader/writer threads. An oversized response is stopped at the smaller job/configured byte limit. -Failure does not advance the successful scheduled watermark. Process-group -children are terminated; orphan reaping remains the operating system's job. +Input byte/token checks include the complete serialized provider envelope and +response schema. If pre-dispatch sizing exceeds an input budget, scheduled +command mode reselects fewer whole evidence records before its single provider +call, under the same byte, token, candidate, call, and runtime limits; it does +not truncate evidence text. If one selected record still cannot fit, the +command is not dispatched. A deterministic preview processes that one record, +records the input limit, and advances only that source-kind cursor so later +evidence remains eligible. Other provider failures do not advance the successful +scheduled watermark. Process-group children are terminated; orphan reaping +remains the operating system's job. + +Native managed readiness accepts `local_only`, or `degraded` only when the +reported store is the configured writable SQLite path at the current schema, +the configured scope is available, exposure/auth fields are loopback-only, and +the sole degradation reason matches a persisted failed Dreamer receipt. +Storage, schema, scope, authorization, exposure, malformed-status, and other +degradation failures still block startup. The scheduled command path is **preview-only**. Candidates preserve scoped source references and remain subject to normal MemoryD validation and review. diff --git a/src/native_runtime.rs b/src/native_runtime.rs index 75ca17f..a73bb65 100644 --- a/src/native_runtime.rs +++ b/src/native_runtime.rs @@ -16,7 +16,7 @@ use serde::Serialize; use crate::config; use crate::error::{Error, Result}; -use crate::store::Store; +use crate::store::{Store, STORAGE_SCHEMA_VERSION}; pub const DEFAULT_URL: &str = "http://127.0.0.1:8787"; @@ -714,16 +714,17 @@ fn wait_for_ready(opts: &RuntimeOptions) -> Result<()> { for _ in 0..30 { if let Ok(body) = http_get(&status_url) { let parsed: serde_json::Value = serde_json::from_str(&body)?; - if parsed - .pointer("/data/status") - .and_then(|v| v.as_str()) - .is_some_and(|status| status == "local_only") - { + if managed_status_ready( + &parsed, + &opts.profile, + &opts.workspace, + &opts.db.to_string_lossy(), + ) { return Ok(()); } if let Some(status) = parsed.pointer("/data/status").and_then(|v| v.as_str()) { return Err(Error::storage(format!( - "daemon status is '{status}', expected local_only" + "daemon status is '{status}', core readiness requirements are not met" ))); } } @@ -734,6 +735,100 @@ fn wait_for_ready(opts: &RuntimeOptions) -> Result<()> { ))) } +fn managed_status_ready( + body: &serde_json::Value, + profile: &str, + workspace: &str, + db_path: &str, +) -> bool { + let Some(data) = body.get("data") else { + return false; + }; + let status = data.get("status").and_then(|value| value.as_str()); + let Some(storage) = data.get("storage") else { + return false; + }; + let Some(features) = data.get("features") else { + return false; + }; + if storage.get("kind").and_then(|value| value.as_str()) != Some("sqlite") + || storage.get("path").and_then(|value| value.as_str()) != Some(db_path) + || storage.get("writable").and_then(|value| value.as_bool()) != Some(true) + || data + .get("storage_schema_version") + .and_then(|value| value.as_i64()) + != Some(STORAGE_SCHEMA_VERSION) + || features.get("exposure").and_then(|value| value.as_str()) != Some("local_only") + || features.get("auth").and_then(|value| value.as_str()) != Some("none") + || !managed_scope_available(data, status, profile, workspace) + { + return false; + } + + let Some(reasons) = data + .get("degraded_reasons") + .and_then(|value| value.as_array()) + else { + return false; + }; + match status { + Some("local_only") => reasons.is_empty(), + Some("degraded") => degraded_only_by_dreamer(data, reasons), + _ => false, + } +} + +fn managed_scope_available( + data: &serde_json::Value, + status: Option<&str>, + profile: &str, + workspace: &str, +) -> bool { + let Some(profiles) = data + .get("active_profiles") + .and_then(|value| value.as_array()) + else { + return false; + }; + let Some(workspaces) = data + .get("active_workspaces") + .and_then(|value| value.as_array()) + else { + return false; + }; + if profiles.is_empty() && workspaces.is_empty() { + return status == Some("local_only"); + } + if profiles.is_empty() || workspaces.is_empty() { + return false; + } + profiles.iter().any(|value| value.as_str() == Some(profile)) + && workspaces + .iter() + .any(|value| value.as_str() == Some(workspace)) +} + +fn degraded_only_by_dreamer(data: &serde_json::Value, reasons: &[serde_json::Value]) -> bool { + let Some(last_dream) = data.get("last_dream") else { + return false; + }; + if last_dream.get("status").and_then(|value| value.as_str()) != Some("error") + || last_dream.get("mode").and_then(|value| value.as_str()) != Some("preview") + || reasons.len() != 1 + { + return false; + } + let error = last_dream + .get("error_summary") + .and_then(|value| value.as_str()) + .filter(|value| !value.is_empty()); + let expected = error.map_or_else( + || "last Dreamer run failed".to_string(), + |error| format!("last Dreamer run failed: {error}"), + ); + reasons[0].as_str() == Some(expected.as_str()) +} + fn status_container(opts: &RuntimeOptions) -> RuntimeStatusReport { let process = container_runtime(opts) .ok() @@ -1197,3 +1292,157 @@ fn parse_loopback_url(url: &str) -> Result { path: format!("/{}", path), }) } + +#[cfg(test)] +mod managed_readiness_tests { + use super::managed_status_ready; + use crate::store::STORAGE_SCHEMA_VERSION; + use serde_json::json; + + fn local_status() -> serde_json::Value { + json!({"data": { + "status": "local_only", + "storage_schema_version": STORAGE_SCHEMA_VERSION, + "storage": {"kind": "sqlite", "path": "/tmp/synthetic.db", "writable": true}, + "active_profiles": ["personal"], + "active_workspaces": ["synthetic-scope"], + "last_dream": null, + "degraded_reasons": [], + "features": {"exposure": "local_only", "auth": "none"} + }}) + } + + fn dreamer_degraded_status() -> serde_json::Value { + json!({"data": { + "status": "degraded", + "storage_schema_version": STORAGE_SCHEMA_VERSION, + "storage": {"kind": "sqlite", "path": "/tmp/synthetic.db", "writable": true}, + "active_profiles": ["personal"], + "active_workspaces": ["synthetic-scope"], + "last_dream": { + "mode": "preview", + "status": "error", + "error_summary": "dream provider input byte budget exhausted" + }, + "degraded_reasons": ["last Dreamer run failed: dream provider input byte budget exhausted"], + "features": {"exposure": "local_only", "auth": "none"} + }}) + } + + fn ready(body: &serde_json::Value, profile: &str, workspace: &str) -> bool { + managed_status_ready(body, profile, workspace, "/tmp/synthetic.db") + } + + #[test] + fn accepts_core_ready_and_dreamer_only_degraded_statuses() { + assert!(ready(&local_status(), "personal", "synthetic-scope")); + assert!(ready( + &dreamer_degraded_status(), + "personal", + "synthetic-scope" + )); + + let empty_store = json!({"data": { + "status": "local_only", + "storage_schema_version": STORAGE_SCHEMA_VERSION, + "storage": {"kind": "sqlite", "path": "/tmp/empty.db", "writable": true}, + "active_profiles": [], + "active_workspaces": [], + "last_dream": null, + "degraded_reasons": [], + "features": {"exposure": "local_only", "auth": "none"} + }}); + assert!(managed_status_ready( + &empty_store, + "personal", + "default", + "/tmp/empty.db" + )); + + let mut missing_degraded_scope = dreamer_degraded_status(); + missing_degraded_scope["data"]["active_profiles"] = json!([]); + missing_degraded_scope["data"]["active_workspaces"] = json!([]); + assert!(!ready( + &missing_degraded_scope, + "personal", + "synthetic-scope" + )); + } + + #[test] + fn rejects_non_dreamer_and_unverifiable_degradation() { + let mut mixed_reason = dreamer_degraded_status(); + mixed_reason["data"]["degraded_reasons"] + .as_array_mut() + .unwrap() + .push(json!("storage is not writable")); + assert!(!ready(&mixed_reason, "personal", "synthetic-scope")); + + let mut no_receipt = dreamer_degraded_status(); + no_receipt["data"]["last_dream"] = json!(null); + assert!(!ready(&no_receipt, "personal", "synthetic-scope")); + + let mut failed_apply = dreamer_degraded_status(); + failed_apply["data"]["last_dream"]["mode"] = json!("apply"); + assert!(!ready(&failed_apply, "personal", "synthetic-scope")); + + let mut no_reason = dreamer_degraded_status(); + no_reason["data"]["degraded_reasons"] = json!([]); + assert!(!ready(&no_reason, "personal", "synthetic-scope")); + + let caller_boolean = json!({"ready": true, "data": {"status": "degraded"}}); + assert!(!ready(&caller_boolean, "personal", "synthetic-scope")); + } + + #[test] + fn rejects_storage_schema_authorization_exposure_and_scope_failures() { + let mut unreadable = local_status(); + unreadable["data"]["storage"]["writable"] = json!(false); + assert!(!ready(&unreadable, "personal", "synthetic-scope")); + + let mut malformed_storage = local_status(); + malformed_storage["data"]["storage"] = json!(null); + assert!(!ready(&malformed_storage, "personal", "synthetic-scope")); + + let mut wrong_schema = local_status(); + wrong_schema["data"]["storage_schema_version"] = json!(STORAGE_SCHEMA_VERSION - 1); + assert!(!ready(&wrong_schema, "personal", "synthetic-scope")); + + let mut missing_schema = local_status(); + missing_schema["data"] + .as_object_mut() + .unwrap() + .remove("storage_schema_version"); + assert!(!ready(&missing_schema, "personal", "synthetic-scope")); + + let mut remote_exposure = local_status(); + remote_exposure["data"]["features"]["exposure"] = json!("auth_missing"); + assert!(!ready(&remote_exposure, "personal", "synthetic-scope")); + + let mut missing_exposure = local_status(); + missing_exposure["data"]["features"] + .as_object_mut() + .unwrap() + .remove("exposure"); + assert!(!ready(&missing_exposure, "personal", "synthetic-scope")); + + let mut other_database = local_status(); + other_database["data"]["storage"]["path"] = json!("/tmp/other.db"); + assert!(!ready(&other_database, "personal", "synthetic-scope")); + + let mut authorization_failure = local_status(); + authorization_failure["data"]["features"]["auth"] = json!("required"); + assert!(!ready( + &authorization_failure, + "personal", + "synthetic-scope" + )); + + let mut inconsistent_status = local_status(); + inconsistent_status["data"]["degraded_reasons"] = json!(["storage is not writable"]); + assert!(!ready(&inconsistent_status, "personal", "synthetic-scope")); + + assert!(!ready(&local_status(), "other-profile", "synthetic-scope")); + assert!(!ready(&local_status(), "personal", "other-workspace")); + } +} diff --git a/src/service.rs b/src/service.rs index 01afe9b..3f4ff0b 100644 --- a/src/service.rs +++ b/src/service.rs @@ -1672,6 +1672,21 @@ impl Service { } pub fn run_dream_job(&self, req: DreamJobRunRequest) -> Result { + self.run_dream_job_inner(req, false) + } + + fn run_scheduled_command_dream_job( + &self, + req: DreamJobRunRequest, + ) -> Result { + self.run_dream_job_inner(req, true) + } + + fn run_dream_job_inner( + &self, + req: DreamJobRunRequest, + skip_empty_provider_input: bool, + ) -> Result { let profile = self.resolve_profile(&req.profile)?; let workspace = self.resolve_workspace(&req.workspace); let repo_id = self.register_repo(&req.repo)?; @@ -1782,6 +1797,7 @@ impl Service { resolved_provider.as_ref(), &req.budget, deadline, + skip_empty_provider_input, ) }); if started.elapsed().as_secs() >= req.budget.max_runtime_seconds && result.is_ok() { @@ -2102,6 +2118,7 @@ impl Service { provider: Option<&ResolvedDreamProvider>, budget: &DreamJobBudget, deadline: Instant, + skip_empty_provider_input: bool, ) -> Result<( DreamResponse, bool, @@ -2124,6 +2141,17 @@ impl Service { } let provider = provider .ok_or_else(|| Error::internal("model-backed Dream provider was not resolved"))?; + if skip_empty_provider_input && !scheduled_provider_has_evidence(&response) { + return Ok(( + response, + max_candidates_hit, + None, + Some(DreamBudgetUsage { + input_records, + ..DreamBudgetUsage::default() + }), + )); + } if adapter == DreamProviderAdapter::Provider && budget.max_cost_micros == 0 { return Err(Error::invalid_request( "provider jobs require max_cost_micros > 0", @@ -2565,47 +2593,101 @@ impl Service { && self.config.dream_provider.enabled && DreamProviderAdapter::parse(&self.config.dream_provider.adapter) == Some(DreamProviderAdapter::Command); + let mut input_window_limited = false; + let mut input_budget_limit = None; let result = if command_mode { if cfg.automatic_apply { Err(Error::invalid_request( "scheduled command providers are preview-only", )) } else { - self.run_dream_job(DreamJobRunRequest { - job_id: None, - profile: Some(profile.as_str().to_string()), - workspace: Some(workspace.clone()), - repo: None, - now: Some(now.clone()), - since: watermark_before.clone(), - since_explicit: false, - kind: "dream_preview".to_string(), - mode: Some("command".to_string()), - provider: None, - budget: DreamJobBudget { - max_runtime_seconds: cfg.max_runtime_seconds, - max_input_records: cfg.max_batch_size, - max_candidates: cfg.max_candidates, - max_input_tokens: 8000, - max_output_tokens: 2048, - max_input_bytes: 32000, - max_output_bytes: 262144, - max_provider_calls: 1, - max_retries: 0, - max_cost_micros: 0, - daily_cost_ceiling_micros: self - .config - .dream_provider - .daily_cost_ceiling_micros, - }, - }) - .and_then(|job| { - if job.status == "error" { - Err(Error::internal("scheduled native provider preview failed")) - } else { - Ok((job.preview, !job.limits_hit.is_empty())) + let mut max_input_records = cfg.max_batch_size; + loop { + let remaining_runtime = cfg + .max_runtime_seconds + .saturating_sub(started.elapsed().as_secs()); + if remaining_runtime == 0 { + break Err(Error::internal("dream runtime budget exhausted")); } - }) + match self.run_scheduled_command_dream_job(DreamJobRunRequest { + job_id: None, + profile: Some(profile.as_str().to_string()), + workspace: Some(workspace.clone()), + repo: None, + now: Some(now.clone()), + since: watermark_before.clone(), + since_explicit: false, + kind: "dream_preview".to_string(), + mode: Some("command".to_string()), + provider: None, + budget: DreamJobBudget { + max_runtime_seconds: remaining_runtime, + max_input_records, + max_candidates: cfg.max_candidates, + max_input_tokens: 8000, + max_output_tokens: 2048, + max_input_bytes: 32000, + max_output_bytes: 262144, + max_provider_calls: 1, + max_retries: 0, + max_cost_micros: 0, + daily_cost_ceiling_micros: self + .config + .dream_provider + .daily_cost_ceiling_micros, + }, + }) { + Ok(job) if job.status != "error" => { + input_window_limited |= max_input_records < cfg.max_batch_size; + break Ok((job.preview, !job.limits_hit.is_empty())); + } + Ok(_) => { + break Err(Error::internal("scheduled native provider preview failed")); + } + Err(error) + if error.message == "dream provider input byte budget exhausted" + || error.message + == "dream provider input token budget exhausted" => + { + input_window_limited = true; + input_budget_limit = Some(if error.message.contains("byte budget") { + "max_input_bytes" + } else { + "max_input_tokens" + }); + if max_input_records > 1 { + // Keep re-selection bounded even if the configured batch cap is large. + max_input_records = max_input_records / 2 + max_input_records % 2; + continue; + } + let Some(deadline) = started + .checked_add(StdDuration::from_secs(cfg.max_runtime_seconds)) + else { + break Err(Error::invalid_request( + "dream scheduler max_runtime_seconds is too large", + )); + }; + break dream::run( + &self.store, + &dream::DreamParams { + profile: profile.clone(), + workspace: &workspace, + repo_id: None, + mode: "preview", + now: &now, + source_window_end: Some(&now), + recency_cutoff: watermark_before.as_deref(), + include_archived_sources: false, + max_records: 1, + max_candidates: Some(cfg.max_candidates), + patch_run_id: None, + deadline: Some(deadline), + }, + ); + } + Err(error) => break Err(error), + } + } } } else { dream::run( @@ -2633,6 +2715,9 @@ impl Service { } match result { Ok((mut run, mut max_candidates_hit)) => { + if let Some(limit) = input_budget_limit { + limits_hit.push(limit.to_string()); + } if cfg.automatic_apply && !command_mode { max_candidates_hit |= self.filter_applied_scheduled_candidates( &mut run, @@ -2692,7 +2777,8 @@ impl Service { // A full bounded input window is not proof the entire frontier // was covered, even when all selected candidates were consumed. let input_window_full = cfg.max_batch_size > 0 - && evidence_window_count(&run.evidence_window) >= cfg.max_batch_size; + && (evidence_window_count(&run.evidence_window) >= cfg.max_batch_size + || input_window_limited); if input_window_full { limits_hit.push("max_input_records".to_string()); } @@ -2710,7 +2796,12 @@ impl Service { let watermark_after = if limits_hit.is_empty() { Some(now.clone()) } else if input_window_full - && limits_hit.iter().all(|limit| limit == "max_input_records") + && limits_hit.iter().all(|limit| { + matches!( + limit.as_str(), + "max_input_records" | "max_input_bytes" | "max_input_tokens" + ) + }) { dream::scheduler_watermark_after( watermark_before.as_deref(), From eefc7eae10463db6574fb417e3f5002f0c17f530 Mon Sep 17 00:00:00 2001 From: joshyorko Date: Fri, 2 Oct 2026 17:52:19 -0400 Subject: [PATCH 05/18] test(native): checkpoint managed-container readiness regression --- src/native_runtime.rs | 135 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 135 insertions(+) diff --git a/src/native_runtime.rs b/src/native_runtime.rs index a73bb65..92c8363 100644 --- a/src/native_runtime.rs +++ b/src/native_runtime.rs @@ -1446,3 +1446,138 @@ mod managed_readiness_tests { assert!(!ready(&local_status(), "personal", "other-workspace")); } } + +#[cfg(all(test, target_os = "linux"))] +mod container_readiness_regressions { + use super::{up, RuntimeKind, RuntimeOptions}; + use crate::store::STORAGE_SCHEMA_VERSION; + use serde_json::{json, Value}; + use std::fs; + use std::io::{Read, Write}; + use std::net::TcpListener; + use std::os::unix::fs::PermissionsExt; + use std::path::PathBuf; + use std::sync::atomic::{AtomicBool, Ordering}; + use std::sync::Arc; + use std::thread::JoinHandle; + use std::time::{Duration, Instant}; + + fn container_status(path: &str) -> Value { + json!({"data": { + "status": "local_only", + "storage_schema_version": STORAGE_SCHEMA_VERSION, + "storage": {"kind": "sqlite", "path": path, "writable": true}, + "active_profiles": ["personal"], + "active_workspaces": ["synthetic-scope"], + "last_dream": null, + "degraded_reasons": [], + "features": {"exposure": "local_only", "auth": "none"} + }}) + } + + fn status_server( + status: Value, + ) -> (String, Arc, JoinHandle<()>) { + let listener = TcpListener::bind("127.0.0.1:0").unwrap(); + let address = listener.local_addr().unwrap(); + listener.set_nonblocking(true).unwrap(); + let stop = Arc::new(AtomicBool::new(false)); + let stop_server = Arc::clone(&stop); + let status_body = status.to_string(); + let server = std::thread::spawn(move || { + let deadline = Instant::now() + Duration::from_secs(5); + while !stop_server.load(Ordering::Relaxed) && Instant::now() < deadline { + match listener.accept() { + Ok((mut stream, _)) => { + let mut request = [0_u8; 2048]; + let read = stream.read(&mut request).unwrap_or(0); + let request = String::from_utf8_lossy(&request[..read]); + let path = request + .lines() + .next() + .and_then(|line| line.split_whitespace().nth(1)) + .unwrap_or("/"); + let body = if path == "/healthz" { + "{}" + } else { + status_body.as_str() + }; + let response = format!( + "HTTP/1.1 200 OK\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{}", + body.len(), + body + ); + let _ = stream.write_all(response.as_bytes()); + } + Err(error) if error.kind() == std::io::ErrorKind::WouldBlock => { + std::thread::sleep(Duration::from_millis(5)); + } + Err(_) => break, + } + } + }); + (format!("http://{address}"), stop, server) + } + + fn fake_container_runtime(path: &std::path::Path) { + fs::write( + path, + "#!/bin/sh\nif [ \"$1\" = \"--version\" ]; then exit 0; fi\nif [ \"$1\" = \"container\" ] && [ \"$2\" = \"inspect\" ]; then if [ \"$3\" = \"--format\" ]; then printf true; fi; exit 0; fi\nif [ \"$1\" = \"start\" ]; then exit 0; fi\nexit 1\n", + ) + .unwrap(); + let mut permissions = fs::metadata(path).unwrap().permissions(); + permissions.set_mode(0o755); + fs::set_permissions(path, permissions).unwrap(); + } + + fn run_managed_container_up(status_path: &str) -> crate::error::Result { + let temp = tempfile::tempdir().unwrap(); + let runtime = temp.path().join("fake-container-runtime"); + fake_container_runtime(&runtime); + let (url, stop, server) = status_server(container_status(status_path)); + let address = url.strip_prefix("http://").unwrap(); + let port = address.rsplit_once(':').unwrap().1.parse().unwrap(); + let options = RuntimeOptions { + runtime: RuntimeKind::Container, + home: temp.path().join("runtime-home"), + url, + host: "127.0.0.1".into(), + port, + bind: format!("127.0.0.1:{port}"), + db: PathBuf::from("/tmp/host-memory.db"), + pid_file: temp.path().join("unused.pid"), + log_file: temp.path().join("unused.log"), + profile: "personal".into(), + workspace: "synthetic-scope".into(), + log_level: "info".into(), + binary: PathBuf::from("unused-binary"), + allow_non_loopback: false, + image: "synthetic-image".into(), + container_name: "synthetic-memoryd".into(), + container_runtime: Some(runtime.to_string_lossy().into_owned()), + codex_memories_dir: temp.path().join("synthetic-memories"), + uid: None, + gid: None, + endpoint_configured: true, + }; + + let result = up(&options); + stop.store(true, Ordering::Relaxed); + server.join().unwrap(); + result + } + + #[test] + fn up_accepts_healthy_container_status_at_container_database_path() { + let result = run_managed_container_up("/data/memory.db"); + assert!(result.is_ok(), "managed container startup failed: {result:?}"); + assert_eq!(result.unwrap().health, "ok"); + } + + #[test] + fn up_rejects_container_status_for_wrong_database_path() { + let result = run_managed_container_up("/data/wrong.db"); + let error = result.expect_err("wrong managed-container storage path must fail readiness"); + assert!(error.message.contains("core readiness requirements are not met")); + } +} From 4972855747250a8814020a6dd0e2dd8753eee758 Mon Sep 17 00:00:00 2001 From: Josh Yorko <54248591+joshyorko@users.noreply.github.com> Date: Fri, 2 Oct 2026 15:48:16 -0700 Subject: [PATCH 06/18] fix(native): preserve usable startup and bounded Dreamer previews --- AGENTS.md | 31 +++++ docs/dreamer-native-provider.md | 32 ++++-- scripts/run-with-child-reaper.py | 111 ++++++++++++++++++ scripts/v0.1-release-gate.sh | 2 + src/native_runtime.rs | 109 ++++++++++++++---- src/service.rs | 190 ++++++++++++++++++++++++------- src/status.rs | 5 + src/store.rs | 11 ++ tests/cli_smoke.rs | 166 ++++++++++++++------------- tests/dream_command_scheduler.rs | 24 +++- tests/test_child_reaper.py | 153 +++++++++++++++++++++++++ 11 files changed, 674 insertions(+), 160 deletions(-) create mode 100755 scripts/run-with-child-reaper.py create mode 100644 tests/test_child_reaper.py diff --git a/AGENTS.md b/AGENTS.md index 371ac2b..857303f 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -52,3 +52,34 @@ gate. - Before merge or handoff, report the reviewed head, a link to the completed connector review, unresolved findings, and required check status. Review completion does not itself authorize merging or replace other merge gates. + +## Isolated Linux lifecycle validation + +Native lifecycle checks start detached fixture daemons. Run them with a reaping +PID 1 or the repository's child-subreaper launcher: + +```sh +python3 scripts/run-with-child-reaper.py cargo test --test cli_smoke +python3 scripts/run-with-child-reaper.py scripts/v0.1-release-gate.sh +``` + +Use the launcher only for isolated synthetic checks. When the command exits or +is interrupted, it terminates and reaps the command's remaining descendants; +do not wrap a real operator service or `--include-dogfood` with it. + +A non-reaping PID 1 can leave an exited daemon as a zombie. Linux `kill(pid, 0)` +still reports that PID as present, so `src/native_runtime.rs::process_alive` +cannot establish clean shutdown in that environment. Inspect `/proc//status` +and PID 1 before classifying such a fixture failure as a startup regression. +Do not waive the shutdown assertion or change production lifecycle behavior to +make the test environment pass. + +The agent that starts a release gate must keep ownership until it has the +command's exit receipt. Shared log files do not transfer ownership of an +agent-local execution session; do not hand off a still-running gate as verified. + +Validate the launcher with +`python3 -m unittest discover -s tests -p test_child_reaper.py -v`. +Its `run` function owns the main command's exit status separately from adopted +child reaping and bounds signal cleanup. The canonical release gate includes +these launcher checks. diff --git a/docs/dreamer-native-provider.md b/docs/dreamer-native-provider.md index 110db51..5d80e76 100644 --- a/docs/dreamer-native-provider.md +++ b/docs/dreamer-native-provider.md @@ -51,24 +51,32 @@ children in their process group. The native companion has no tool executor. ## Budgets and review Scheduled command mode uses the existing typed preview job: one provider call, -zero provider retries, bounded input/output, and a deadline covering pipe I/O. +zero provider retries, bounded input/output, and one absolute deadline covering +evidence selection, every sizing retry, and pipe I/O. Retrying input selection +does not replenish fractional seconds already consumed. The Linux runner uses nonblocking pipes, not detached reader/writer threads. An oversized response is stopped at the smaller job/configured byte limit. Input byte/token checks include the complete serialized provider envelope and response schema. If pre-dispatch sizing exceeds an input budget, scheduled command mode reselects fewer whole evidence records before its single provider call, under the same byte, token, candidate, call, and runtime limits; it does -not truncate evidence text. If one selected record still cannot fit, the +not truncate evidence text. Safety screening still covers the entire source, +including its tail. If one selected record still cannot fit, the command is not dispatched. A deterministic preview processes that one record, records the input limit, and advances only that source-kind cursor so later evidence remains eligible. Other provider failures do not advance the successful scheduled watermark. Process-group children are terminated; orphan reaping remains the operating system's job. -Native managed readiness accepts `local_only`, or `degraded` only when the -reported store is the configured writable SQLite path at the current schema, -the configured scope is available, exposure/auth fields are loopback-only, and -the sole degradation reason matches a persisted failed Dreamer receipt. +Managed readiness accepts `local_only`, or `degraded` only when the reported +store is writable SQLite at the current schema, the exact configured +profile/workspace pair exists, exposure/auth fields are loopback-only, and +the sole degradation reason matches a persisted failed preview Dreamer receipt. +Native readiness checks the configured host database path; managed containers +check `/data/memory.db`, the path passed to the daemon inside the container. +`/v1/status` reports `features.configured_scope` using an exact workspace query, +rather than combining global profile and workspace lists. A healthy empty store +can start before its first workspace is created. Storage, schema, scope, authorization, exposure, malformed-status, and other degradation failures still block startup. @@ -82,17 +90,17 @@ accepted or useful. ## Verification ```sh -python -m unittest discover -s tests -p test_dreamer_native_provider.py -v +python3 -m unittest discover -s tests -p test_dreamer_native_provider.py -v cargo test --lib cargo test --test dream_command_scheduler --test command_no_http_fallback --test command_job_policy --test config_env ``` -The command tests are Linux-scoped. Use a disposable database for actual +The command tests are Linux-scoped. Native lifecycle acceptance also needs a +reaping PID 1 or `scripts/run-with-child-reaper.py`; see `AGENTS.md` for the +isolated-check commands and the zombie-PID counterexample. The launcher cleans +up only its owned test descendants and preserves the main command's exit code. +Use a disposable database for actual subscription tests; `dream --scheduled` exercises the scheduler's configured provider, not a separate shell smoke test. Verify `command` provenance, the selected model, validated `dream_provider_` observations, and zero accepted memory writes. Keep raw receipts and profile paths private. - -Three `dream_jobs` fixture failures also reproduce on the unchanged base: -HTTP fixtures used in HTTPS-only Provider mode, and a configured-credential -endpoint mismatch. They are not waived security checks or a green full suite. diff --git a/scripts/run-with-child-reaper.py b/scripts/run-with-child-reaper.py new file mode 100755 index 0000000..1db555e --- /dev/null +++ b/scripts/run-with-child-reaper.py @@ -0,0 +1,111 @@ +#!/usr/bin/env python3 +"""Run isolated Linux checks with orphan reaping and bounded owned cleanup.""" +import argparse +import ctypes +import os +from pathlib import Path +import signal +import subprocess +import sys +import time + +CLEANUP_GRACE_SECONDS = 2.0 +KILL_WAIT_SECONDS = 1.0 + + +def adopted_children(main_pid): + """Only direct children of this launcher, excluding the Popen owner.""" + children = Path(f"/proc/self/task/{os.getpid()}/children").read_text().split() + return [int(pid) for pid in children if int(pid) != main_pid] + + +def reap_adopted(main_pid): + for pid in adopted_children(main_pid): + try: + os.waitpid(pid, os.WNOHANG) + except ChildProcessError: + pass + return adopted_children(main_pid) + + +def send_signal(pid, signum, process_group=False): + try: + if process_group: + os.killpg(pid, signum) + else: + os.kill(pid, signum) + except ProcessLookupError: + pass + + +def run(command): + if sys.platform != "linux": + raise RuntimeError("child-subreaper checks require Linux") + libc = ctypes.CDLL(None, use_errno=True) + if libc.prctl(36, 1, 0, 0, 0) != 0: # PR_SET_CHILD_SUBREAPER + raise OSError(ctypes.get_errno(), "PR_SET_CHILD_SUBREAPER failed") + + pending_signals = [] + for signum in (signal.SIGINT, signal.SIGTERM, signal.SIGHUP): + signal.signal(signum, lambda received, _frame: pending_signals.append(received)) + child = subprocess.Popen(command, start_new_session=True) + cleanup_started = None + signaled_children = set() + while True: + # Popen alone owns the main command's wait status. Never waitpid(-1), + # which could steal its exit code while reaping adopted descendants. + returncode = child.poll() + children = reap_adopted(child.pid) + now = time.monotonic() + if pending_signals: + cleanup_started = cleanup_started or now + for signum in pending_signals: + if returncode is None: + send_signal(child.pid, signum, process_group=True) + for pid in children: + send_signal(pid, signum) + pending_signals.clear() + if returncode is not None: + if not children: + return returncode if returncode >= 0 else 128 - returncode + cleanup_started = cleanup_started or now + + if cleanup_started is not None: + elapsed = now - cleanup_started + signum = signal.SIGKILL if elapsed >= CLEANUP_GRACE_SECONDS else signal.SIGTERM + if signum == signal.SIGKILL and returncode is None: + send_signal(child.pid, signum, process_group=True) + for pid in children: + if signum == signal.SIGKILL or pid not in signaled_children: + send_signal(pid, signum) + signaled_children.add(pid) + if elapsed >= CLEANUP_GRACE_SECONDS + KILL_WAIT_SECONDS: + returncode = child.poll() + remaining = reap_adopted(child.pid) + if returncode is not None and not remaining: + return returncode if returncode >= 0 else 128 - returncode + print("child reaper: owned fixture cleanup exceeded its deadline", file=sys.stderr) + if returncode is not None and returncode != 0: + return returncode if returncode > 0 else 128 - returncode + return 1 + time.sleep(0.01) + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("command", nargs=argparse.REMAINDER) + args = parser.parse_args() + command = args.command + if command[:1] == ["--"]: + command = command[1:] + if not command: + parser.error("an isolated check command is required") + try: + return run(command) + except (OSError, RuntimeError) as error: + print(f"child reaper: {error}", file=sys.stderr) + return 125 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/v0.1-release-gate.sh b/scripts/v0.1-release-gate.sh index a6a59ad..baa6fc7 100755 --- a/scripts/v0.1-release-gate.sh +++ b/scripts/v0.1-release-gate.sh @@ -31,6 +31,7 @@ dry_run() { v0.1 local release gate: - cargo fmt --all --check - git diff --check +- isolated Linux child-reaper checks - cargo test - codex-memoryd doctor --format json against a temp DB - codex-memoryd eval substrate --compare --format json @@ -223,6 +224,7 @@ PY : >"$ARTIFACT_DIR/steps.jsonl" +run "child-reaper-tests" python3 -m unittest discover -s tests -p test_child_reaper.py -v run "cargo-fmt" cargo fmt --all --check run "git-diff-check" git diff --check diff --git a/src/native_runtime.rs b/src/native_runtime.rs index 92c8363..bda0e5e 100644 --- a/src/native_runtime.rs +++ b/src/native_runtime.rs @@ -586,7 +586,7 @@ fn up_native(opts: &RuntimeOptions) -> Result { fs::write(&opts.pid_file, child.id().to_string()).map_err(Error::from)?; } - wait_for_ready(opts)?; + wait_for_ready(opts, &opts.db.to_string_lossy())?; Ok(status(opts)) } @@ -666,7 +666,7 @@ fn up_container(opts: &RuntimeOptions) -> Result { )?; } - wait_for_ready(opts)?; + wait_for_ready(opts, "/data/memory.db")?; Ok(status_container(opts)) } @@ -708,18 +708,13 @@ fn wait_for_health(opts: &RuntimeOptions) -> Result<()> { Err(Error::storage(format!("timed out waiting for {url}"))) } -fn wait_for_ready(opts: &RuntimeOptions) -> Result<()> { +fn wait_for_ready(opts: &RuntimeOptions, db_path: &str) -> Result<()> { wait_for_health(opts)?; let status_url = format!("{}/v1/status", opts.url.trim_end_matches('/')); for _ in 0..30 { if let Ok(body) = http_get(&status_url) { let parsed: serde_json::Value = serde_json::from_str(&body)?; - if managed_status_ready( - &parsed, - &opts.profile, - &opts.workspace, - &opts.db.to_string_lossy(), - ) { + if managed_status_ready(&parsed, &opts.profile, &opts.workspace, db_path) { return Ok(()); } if let Some(status) = parsed.pointer("/data/status").and_then(|v| v.as_str()) { @@ -796,16 +791,23 @@ fn managed_scope_available( else { return false; }; - if profiles.is_empty() && workspaces.is_empty() { - return status == Some("local_only"); - } - if profiles.is_empty() || workspaces.is_empty() { + let Some(scope) = data.pointer("/features/configured_scope") else { + return false; + }; + if scope.get("profile").and_then(|value| value.as_str()) != Some(profile) + || scope.get("workspace").and_then(|value| value.as_str()) != Some(workspace) + { return false; } + if profiles.is_empty() && workspaces.is_empty() { + return status == Some("local_only") + && scope.get("available").and_then(|value| value.as_bool()) == Some(false); + } profiles.iter().any(|value| value.as_str() == Some(profile)) && workspaces .iter() .any(|value| value.as_str() == Some(workspace)) + && scope.get("available").and_then(|value| value.as_bool()) == Some(true) } fn degraded_only_by_dreamer(data: &serde_json::Value, reasons: &[serde_json::Value]) -> bool { @@ -1308,7 +1310,8 @@ mod managed_readiness_tests { "active_workspaces": ["synthetic-scope"], "last_dream": null, "degraded_reasons": [], - "features": {"exposure": "local_only", "auth": "none"} + "features": {"exposure": "local_only", "auth": "none", + "configured_scope": {"profile": "personal", "workspace": "synthetic-scope", "available": true}} }}) } @@ -1325,7 +1328,8 @@ mod managed_readiness_tests { "error_summary": "dream provider input byte budget exhausted" }, "degraded_reasons": ["last Dreamer run failed: dream provider input byte budget exhausted"], - "features": {"exposure": "local_only", "auth": "none"} + "features": {"exposure": "local_only", "auth": "none", + "configured_scope": {"profile": "personal", "workspace": "synthetic-scope", "available": true}} }}) } @@ -1350,7 +1354,8 @@ mod managed_readiness_tests { "active_workspaces": [], "last_dream": null, "degraded_reasons": [], - "features": {"exposure": "local_only", "auth": "none"} + "features": {"exposure": "local_only", "auth": "none", + "configured_scope": {"profile": "personal", "workspace": "default", "available": false}} }}); assert!(managed_status_ready( &empty_store, @@ -1369,6 +1374,58 @@ mod managed_readiness_tests { )); } + #[test] + fn rejects_profile_workspace_cross_product_from_real_status() { + let store = crate::store::Store::open(":memory:").unwrap(); + store + .ensure_workspace("personal", "other-workspace") + .unwrap(); + store.ensure_workspace("work", "synthetic-scope").unwrap(); + let mut config = crate::config::Config::default(); + config.default_profile = "personal".into(); + config.default_workspace = "synthetic-scope".into(); + let service = crate::service::Service::new(store.clone(), config); + let status = serde_json::to_value(service.status().unwrap()).unwrap(); + let path = status["storage"]["path"].as_str().unwrap().to_string(); + assert!( + !managed_status_ready( + &json!({"data": status}), + "personal", + "synthetic-scope", + &path + ), + "unrelated profile and workspace must not establish a ready scope" + ); + + store + .ensure_workspace("personal", "synthetic-scope") + .unwrap(); + let status = serde_json::to_value(service.status().unwrap()).unwrap(); + assert!(managed_status_ready( + &json!({"data": status}), + "personal", + "synthetic-scope", + &path + )); + } + + #[test] + fn rejects_inconsistent_empty_scope_claims() { + let mut status = local_status(); + status["data"]["active_profiles"] = json!([]); + status["data"]["active_workspaces"] = json!([]); + assert!(!ready(&status, "personal", "synthetic-scope")); + status["data"]["features"]["configured_scope"]["available"] = json!(false); + assert!(ready(&status, "personal", "synthetic-scope")); + assert!(!ready(&status, "work", "synthetic-scope")); + assert!(!ready(&status, "personal", "other-workspace")); + status["data"]["features"] + .as_object_mut() + .unwrap() + .remove("configured_scope"); + assert!(!ready(&status, "personal", "synthetic-scope")); + } + #[test] fn rejects_non_dreamer_and_unverifiable_degradation() { let mut mixed_reason = dreamer_degraded_status(); @@ -1471,13 +1528,12 @@ mod container_readiness_regressions { "active_workspaces": ["synthetic-scope"], "last_dream": null, "degraded_reasons": [], - "features": {"exposure": "local_only", "auth": "none"} + "features": {"exposure": "local_only", "auth": "none", + "configured_scope": {"profile": "personal", "workspace": "synthetic-scope", "available": true}} }}) } - fn status_server( - status: Value, - ) -> (String, Arc, JoinHandle<()>) { + fn status_server(status: Value) -> (String, Arc, JoinHandle<()>) { let listener = TcpListener::bind("127.0.0.1:0").unwrap(); let address = listener.local_addr().unwrap(); listener.set_nonblocking(true).unwrap(); @@ -1530,7 +1586,9 @@ mod container_readiness_regressions { fs::set_permissions(path, permissions).unwrap(); } - fn run_managed_container_up(status_path: &str) -> crate::error::Result { + fn run_managed_container_up( + status_path: &str, + ) -> crate::error::Result { let temp = tempfile::tempdir().unwrap(); let runtime = temp.path().join("fake-container-runtime"); fake_container_runtime(&runtime); @@ -1570,7 +1628,10 @@ mod container_readiness_regressions { #[test] fn up_accepts_healthy_container_status_at_container_database_path() { let result = run_managed_container_up("/data/memory.db"); - assert!(result.is_ok(), "managed container startup failed: {result:?}"); + assert!( + result.is_ok(), + "managed container startup failed: {result:?}" + ); assert_eq!(result.unwrap().health, "ok"); } @@ -1578,6 +1639,8 @@ mod container_readiness_regressions { fn up_rejects_container_status_for_wrong_database_path() { let result = run_managed_container_up("/data/wrong.db"); let error = result.expect_err("wrong managed-container storage path must fail readiness"); - assert!(error.message.contains("core readiness requirements are not met")); + assert!(error + .message + .contains("core readiness requirements are not met")); } } diff --git a/src/service.rs b/src/service.rs index 3f4ff0b..9376a34 100644 --- a/src/service.rs +++ b/src/service.rs @@ -1672,20 +1672,22 @@ impl Service { } pub fn run_dream_job(&self, req: DreamJobRunRequest) -> Result { - self.run_dream_job_inner(req, false) + self.run_dream_job_inner(req, false, None) } fn run_scheduled_command_dream_job( &self, req: DreamJobRunRequest, + deadline: Instant, ) -> Result { - self.run_dream_job_inner(req, true) + self.run_dream_job_inner(req, true, Some(deadline)) } fn run_dream_job_inner( &self, req: DreamJobRunRequest, skip_empty_provider_input: bool, + scheduler_deadline: Option, ) -> Result { let profile = self.resolve_profile(&req.profile)?; let workspace = self.resolve_workspace(&req.workspace); @@ -1738,6 +1740,9 @@ impl Service { "dream job max_runtime_seconds is too large", )); }; + // Sizing retries share the scheduler's original deadline, including + // fractional seconds already spent selecting and serializing sources. + let deadline = scheduler_deadline.map_or(deadline, |outer| outer.min(deadline)); let job_id = req.job_id.unwrap_or_else(|| ids::new_id("dream_job")); let provider = req.provider.unwrap_or_default(); let resolved_provider = self.resolve_dream_provider(adapter, &provider, &req.budget)?; @@ -1800,7 +1805,7 @@ impl Service { skip_empty_provider_input, ) }); - if started.elapsed().as_secs() >= req.budget.max_runtime_seconds && result.is_ok() { + if Instant::now() >= deadline && result.is_ok() { let completed_at = ids::now_rfc3339(); let summary = sanitize_error_summary("dream job exceeded max_runtime_seconds"); let audit = dream_error_audit( @@ -1840,7 +1845,7 @@ impl Service { if max_candidates_hit { limits_hit.push("max_candidates".to_string()); } - if started.elapsed().as_secs() >= req.budget.max_runtime_seconds { + if Instant::now() >= deadline { limits_hit.push("max_runtime_seconds".to_string()); } let status = if limits_hit.is_empty() { @@ -2589,6 +2594,11 @@ impl Service { } let started = Instant::now(); + let deadline = started + .checked_add(StdDuration::from_secs(cfg.max_runtime_seconds)) + .ok_or_else(|| { + Error::invalid_request("dream scheduler max_runtime_seconds is too large") + })?; let command_mode = cfg.scheduled_provider_enabled && self.config.dream_provider.enabled && DreamProviderAdapter::parse(&self.config.dream_provider.adapter) @@ -2603,40 +2613,40 @@ impl Service { } else { let mut max_input_records = cfg.max_batch_size; loop { - let remaining_runtime = cfg - .max_runtime_seconds - .saturating_sub(started.elapsed().as_secs()); - if remaining_runtime == 0 { + if Instant::now() >= deadline { break Err(Error::internal("dream runtime budget exhausted")); } - match self.run_scheduled_command_dream_job(DreamJobRunRequest { - job_id: None, - profile: Some(profile.as_str().to_string()), - workspace: Some(workspace.clone()), - repo: None, - now: Some(now.clone()), - since: watermark_before.clone(), - since_explicit: false, - kind: "dream_preview".to_string(), - mode: Some("command".to_string()), - provider: None, - budget: DreamJobBudget { - max_runtime_seconds: remaining_runtime, - max_input_records, - max_candidates: cfg.max_candidates, - max_input_tokens: 8000, - max_output_tokens: 2048, - max_input_bytes: 32000, - max_output_bytes: 262144, - max_provider_calls: 1, - max_retries: 0, - max_cost_micros: 0, - daily_cost_ceiling_micros: self - .config - .dream_provider - .daily_cost_ceiling_micros, + match self.run_scheduled_command_dream_job( + DreamJobRunRequest { + job_id: None, + profile: Some(profile.as_str().to_string()), + workspace: Some(workspace.clone()), + repo: None, + now: Some(now.clone()), + since: watermark_before.clone(), + since_explicit: false, + kind: "dream_preview".to_string(), + mode: Some("command".to_string()), + provider: None, + budget: DreamJobBudget { + max_runtime_seconds: cfg.max_runtime_seconds, + max_input_records, + max_candidates: cfg.max_candidates, + max_input_tokens: 8000, + max_output_tokens: 2048, + max_input_bytes: 32000, + max_output_bytes: 262144, + max_provider_calls: 1, + max_retries: 0, + max_cost_micros: 0, + daily_cost_ceiling_micros: self + .config + .dream_provider + .daily_cost_ceiling_micros, + }, }, - }) { + deadline, + ) { Ok(job) if job.status != "error" => { input_window_limited |= max_input_records < cfg.max_batch_size; break Ok((job.preview, !job.limits_hit.is_empty())); @@ -2660,13 +2670,6 @@ impl Service { max_input_records = max_input_records / 2 + max_input_records % 2; continue; } - let Some(deadline) = started - .checked_add(StdDuration::from_secs(cfg.max_runtime_seconds)) - else { - break Err(Error::invalid_request( - "dream scheduler max_runtime_seconds is too large", - )); - }; break dream::run( &self.store, &dream::DreamParams { @@ -3906,6 +3909,8 @@ fn scheduled_provider_has_evidence(response: &DreamResponse) -> bool { } fn dream_provider_context(response: &DreamResponse) -> Result { + // Durable record limits do not authorize shortening provider evidence. + // Screen whole source text; complete-envelope budgets decide whether it fits. let evidence_content = response .evidence_window .visible_turns @@ -3924,7 +3929,7 @@ fn dream_provider_context(response: &DreamResponse) -> Result { .filter_map(|source| { source.content.as_deref().map(|content| { json!({"id": source.id, "kind": source.kind, "content": - match policy::screen_content(content, policy::MAX_RECORD_CHARS) { + match policy::screen_content(content, content.len()) { PolicyDecision::Accept(value) => value, PolicyDecision::Reject { .. } => "[screened]".to_string(), }}) @@ -3956,7 +3961,7 @@ fn provider_evidence_stream(stream: &DreamEvidenceStream) -> Value { .iter() .map(|source| { let content = source.content.as_deref().map(|content| { - match policy::screen_content(content, policy::MAX_RECORD_CHARS) { + match policy::screen_content(content, content.len()) { PolicyDecision::Accept(value) => value, PolicyDecision::Reject { .. } => "[screened]".to_string(), } @@ -5618,6 +5623,103 @@ mod scheduled_dream_mode_tests { #[cfg(test)] mod provider_projection_review_tests { use super::*; + #[cfg(target_os = "linux")] + #[test] + fn scheduled_job_honors_partially_consumed_absolute_deadline() { + let store = Store::open(":memory:").unwrap(); + store.ensure_workspace("personal", "ws").unwrap(); + store + .ensure_session("deadline-session", "personal", "ws", None, None, "fixture") + .unwrap(); + store + .insert_visible_turn(&VisibleTurn { + id: "deadline-turn".into(), + session_id: "deadline-session".into(), + actor: "user".into(), + content: "ordinary synthetic test evidence".into(), + created_at: "2026-10-01T00:00:00Z".into(), + metadata: json!({}), + }) + .unwrap(); + let directory = tempfile::tempdir().unwrap(); + let marker = directory.path().join("completed-provider"); + let mut config = Config::default(); + config.dream_provider.enabled = true; + config.dream_provider.adapter = "command".into(); + config.dream_provider.model = "synthetic-model".into(); + config.dream_provider.command = vec!["/bin/sh".into(), "-c".into(), format!( + "cat >/dev/null; sleep 0.7; touch '{}'; printf '{{\"schema_version\":\"dream-preview-v1\",\"profile\":\"personal\",\"workspace\":\"ws\",\"candidates\":[]}}'", marker.display() + )]; + let svc = Service::new(store, config); + let started = Instant::now(); + let result = svc.run_scheduled_command_dream_job( + DreamJobRunRequest { + job_id: Some("deadline-job".into()), + profile: Some("personal".into()), + workspace: Some("ws".into()), + repo: None, + now: None, + since: None, + since_explicit: true, + kind: "dream_preview".into(), + mode: Some("command".into()), + budget: DreamJobBudget { + max_runtime_seconds: 1, + max_input_records: 1, + max_candidates: 5, + max_provider_calls: 1, + ..Default::default() + }, + provider: None, + }, + started + StdDuration::from_millis(150), + ); + let error = result.expect_err("a sizing retry must not grant a fresh whole second"); + assert!( + matches!( + error.message.as_str(), + "provider command timed out" + | "dream provider runtime budget exhausted" + | "dream job exceeded max_runtime_seconds" + ), + "{error:?}" + ); + assert!(started.elapsed() < StdDuration::from_millis(600)); + // Wait beyond the original command's completion time to catch children + // that escaped timeout cleanup and would write the marker later. + std::thread::sleep(StdDuration::from_millis(750)); + assert!( + !marker.exists(), + "timed-out owned command must not complete later" + ); + } + + #[test] + fn provider_projection_retains_whole_screened_ascii_evidence() { + let svc = Service::new(Store::open(":memory:").unwrap(), Config::default()); + svc.conclusions(serde_json::from_value(json!({"profile":"personal","workspace":"ws","conclusions":["Preference: concise updates"]})).unwrap()).unwrap(); + let mut response = svc + .dream( + serde_json::from_value( + json!({"profile":"personal","workspace":"ws","mode":"preview"}), + ) + .unwrap(), + ) + .unwrap(); + let content = format!("synthetic {} END-OF-WHOLE-RECORD", "a".repeat(9_000)); + response.evidence_window.conclusions.sources[0].content = Some(content.clone()); + let context: Value = + serde_json::from_str(&dream_provider_context(&response).unwrap()).unwrap(); + assert!( + context["evidence_content"][0]["content"] == content, + "provider evidence must retain its ASCII tail" + ); + assert!( + context["evidence_window"]["conclusions"]["sources"][0]["content"] == content, + "window evidence must retain its ASCII tail" + ); + } + #[test] fn provider_projection_omits_unneeded_raw_evidence_metadata() { let svc = Service::new(Store::open(":memory:").unwrap(), Config::default()); diff --git a/src/status.rs b/src/status.rs index 251b3e3..ea9a01b 100644 --- a/src/status.rs +++ b/src/status.rs @@ -101,6 +101,11 @@ pub fn build_status(store: &Store, config: &Config, metrics: &Metrics) -> Result "max_recall_tokens": config.max_recall_tokens, "exposure": if loopback_only { "local_only" } else { "auth_missing" }, "auth": "none", + "configured_scope": { + "profile": config.default_profile, + "workspace": config.default_workspace, + "available": store.workspace_exists(&config.default_profile, &config.default_workspace)?, + }, }); Ok(StatusResponse { diff --git a/src/store.rs b/src/store.rs index b01521b..e13b45a 100644 --- a/src/store.rs +++ b/src/store.rs @@ -1812,6 +1812,17 @@ impl Store { Ok(()) } + /// Check the exact scope without creating it or combining global lists. + pub fn workspace_exists(&self, profile_id: &str, workspace_id: &str) -> Result { + let conn = self.conn()?; + conn.query_row( + "SELECT EXISTS(SELECT 1 FROM workspaces WHERE profile_id = ?1 AND id = ?2)", + params![profile_id, workspace_id], + |row| row.get(0), + ) + .map_err(Error::from) + } + pub fn active_profiles(&self) -> Result> { let conn = self.conn()?; let mut stmt = conn.prepare("SELECT id FROM profiles ORDER BY id")?; diff --git a/tests/cli_smoke.rs b/tests/cli_smoke.rs index c633a54..bc182ce 100644 --- a/tests/cli_smoke.rs +++ b/tests/cli_smoke.rs @@ -6005,93 +6005,99 @@ fn cli_native_up_keeps_dreamer_degraded_daemon_live_and_recallable() { drop(service); drop(store); - let mut up = bin(); - clear_runtime_env(&mut up); - let up_output = up - .env("HOME", &home) - .env("CODEX_MEMORYD_HOME", &home) - .env("CODEX_MEMORYD_DB", &db) - .env("CODEX_MEMORYD_HOST", "127.0.0.1") - .env("CODEX_MEMORYD_PORT", bind.rsplit_once(':').unwrap().1) - .env("CODEX_MEMORYD_PROFILE", "personal") - .env("CODEX_MEMORYD_WORKSPACE", "synthetic-scope") - .args(["--runtime", "native", "up"]) - .output() - .unwrap(); - - let mut status_ok = false; - let mut recall_ok = false; - if up_output.status.success() { - let http = reqwest::blocking::Client::new(); - let response = http.get(format!("{url}/v1/status")).send().unwrap(); - let body: serde_json::Value = response.json().unwrap(); - status_ok = body.pointer("/data/status").and_then(Value::as_str) == Some("degraded") - && body - .pointer("/data/storage/writable") - .and_then(Value::as_bool) - == Some(true) - && body - .pointer("/data/degraded_reasons") - .and_then(Value::as_array) - .is_some_and(|reasons| { - reasons.iter().any(|reason| { - reason.as_str().is_some_and(|text| { - text.contains("dream provider input byte budget exhausted") - }) - }) - }); - let mut recall = bin(); - clear_runtime_env(&mut recall); - let recall_output = recall + for _ in 0..3 { + let mut up = bin(); + clear_runtime_env(&mut up); + let up_output = up .env("HOME", &home) .env("CODEX_MEMORYD_HOME", &home) - .arg("--url") - .arg(&url) - .args([ - "recall", - "--profile", - "personal", - "--workspace", - "synthetic-scope", - "--query", - "lifecycle sentinel", - ]) + .env("CODEX_MEMORYD_DB", &db) + .env("CODEX_MEMORYD_HOST", "127.0.0.1") + .env("CODEX_MEMORYD_PORT", bind.rsplit_once(':').unwrap().1) + .env("CODEX_MEMORYD_PROFILE", "personal") + .env("CODEX_MEMORYD_WORKSPACE", "synthetic-scope") + .args(["--runtime", "native", "up"]) .output() .unwrap(); - recall_ok = recall_output.status.success() - && String::from_utf8_lossy(&recall_output.stdout) - .contains("synthetic lifecycle sentinel remains recallable"); - } - let mut down = bin(); - clear_runtime_env(&mut down); - let down_output = down - .env("HOME", &home) - .env("CODEX_MEMORYD_HOME", &home) - .env("CODEX_MEMORYD_DB", &db) - .env("CODEX_MEMORYD_HOST", "127.0.0.1") - .env("CODEX_MEMORYD_PORT", bind.rsplit_once(':').unwrap().1) - .args(["--runtime", "native", "down"]) - .output() - .unwrap(); + let mut status_ok = false; + let mut recall_ok = false; + if up_output.status.success() { + let http = reqwest::blocking::Client::new(); + let response = http.get(format!("{url}/v1/status")).send().unwrap(); + let body: serde_json::Value = response.json().unwrap(); + status_ok = body.pointer("/data/status").and_then(Value::as_str) == Some("degraded") + && body + .pointer("/data/storage/writable") + .and_then(Value::as_bool) + == Some(true) + && body + .pointer("/data/degraded_reasons") + .and_then(Value::as_array) + .is_some_and(|reasons| { + reasons.iter().any(|reason| { + reason.as_str().is_some_and(|text| { + text.contains("dream provider input byte budget exhausted") + }) + }) + }); + let mut recall = bin(); + clear_runtime_env(&mut recall); + let recall_output = recall + .env("HOME", &home) + .env("CODEX_MEMORYD_HOME", &home) + .arg("--url") + .arg(&url) + .args([ + "recall", + "--profile", + "personal", + "--workspace", + "synthetic-scope", + "--query", + "lifecycle sentinel", + ]) + .output() + .unwrap(); + recall_ok = recall_output.status.success() + && String::from_utf8_lossy(&recall_output.stdout) + .contains("synthetic lifecycle sentinel remains recallable"); + } + let mut down = bin(); + clear_runtime_env(&mut down); + let down_output = down + .env("HOME", &home) + .env("CODEX_MEMORYD_HOME", &home) + .env("CODEX_MEMORYD_DB", &db) + .env("CODEX_MEMORYD_HOST", "127.0.0.1") + .env("CODEX_MEMORYD_PORT", bind.rsplit_once(':').unwrap().1) + .args(["--runtime", "native", "down"]) + .output() + .unwrap(); + + assert!( + up_output.status.success(), + "native up failed: {}", + String::from_utf8_lossy(&up_output.stderr) + ); + assert!( + status_ok, + "degraded Dreamer receipt was not preserved in live status" + ); + assert!( + recall_ok, + "configured-scope recall failed after native startup" + ); + assert!( + down_output.status.success(), + "native down failed: {}", + String::from_utf8_lossy(&down_output.stderr) + ); + } assert!( - up_output.status.success(), - "native up failed: {}", - String::from_utf8_lossy(&up_output.stderr) - ); - assert!( - status_ok, - "degraded Dreamer receipt was not preserved in live status" - ); - assert!( - recall_ok, - "configured-scope recall failed after native startup" - ); - assert!( - down_output.status.success(), - "native down failed: {}", - String::from_utf8_lossy(&down_output.stderr) + !provider_called.exists(), + "restart cycles must not dispatch the failed oversized job" ); } diff --git a/tests/dream_command_scheduler.rs b/tests/dream_command_scheduler.rs index 7c154cd..c7b957b 100644 --- a/tests/dream_command_scheduler.rs +++ b/tests/dream_command_scheduler.rs @@ -85,6 +85,20 @@ fn failed_scheduled_command_does_not_advance_success_watermark() { #[cfg(target_os = "linux")] #[test] fn oversized_command_preview_records_a_limit_and_advances_without_skipping_tail() { + check_oversized_command_preview(format!("synthetic {}", "🦉".repeat(9_000))); +} + +#[cfg(target_os = "linux")] +#[test] +fn oversized_ascii_record_is_not_dispatched_with_a_missing_tail() { + check_oversized_command_preview(format!( + "# synthetic {} END-OF-WHOLE-RECORD", + "a".repeat(17_000) + )); +} + +#[cfg(target_os = "linux")] +fn check_oversized_command_preview(content: String) { let store = Store::open(":memory:").unwrap(); store.ensure_workspace("personal", "ws").unwrap(); store @@ -95,7 +109,7 @@ fn oversized_command_preview_records_a_limit_and_advances_without_skipping_tail( id: "oversized-turn".into(), session_id: "oversized-session".into(), actor: "user".into(), - content: format!("synthetic {}", "🦉".repeat(9_000)), + content, created_at: "2026-10-01T00:00:00Z".into(), metadata: json!({}), }) @@ -161,6 +175,14 @@ fn oversized_command_preview_records_a_limit_and_advances_without_skipping_tail( let second = service.scheduled_dream(None).unwrap(); assert_ne!(second.watermark_before, first.watermark_before); assert!(!marker.exists()); + for _ in 0..4 { + let idle = service.scheduled_dream(None).unwrap(); + assert_eq!(idle.status, "ok"); + assert!( + !marker.exists(), + "unchanged ticks must not redispatch oversized evidence" + ); + } } #[cfg(target_os = "linux")] diff --git a/tests/test_child_reaper.py b/tests/test_child_reaper.py new file mode 100644 index 0000000..044fa4f --- /dev/null +++ b/tests/test_child_reaper.py @@ -0,0 +1,153 @@ +"""Exercise the Linux isolated-test launcher against real owned processes.""" +import os +from pathlib import Path +import signal +import subprocess +import sys +import tempfile +import time +import unittest + +LAUNCHER = Path(__file__).resolve().parents[1] / "scripts" / "run-with-child-reaper.py" + + +@unittest.skipUnless(sys.platform == "linux", "Linux child-subreaper contract") +class ChildReaperTests(unittest.TestCase): + def run_fixture(self, code, directory, **kwargs): + return subprocess.Popen( + [sys.executable, str(LAUNCHER), sys.executable, "-c", code, + "child-reaper-fixture", str(directory)], + stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True, **kwargs, + ) + + def cleanup_fixture(self, process, directory): + if process.poll() is None: + process.kill() + for path in directory.glob("*.pid"): + pid = int(path.read_text()) + try: + command = Path(f"/proc/{pid}/cmdline").read_bytes() + if b"child-reaper-fixture" in command: + os.kill(pid, signal.SIGKILL) + except (FileNotFoundError, ProcessLookupError): + pass + process.communicate(timeout=5) + + def test_success_preserves_output_and_exit(self): + with tempfile.TemporaryDirectory() as name: + directory = Path(name) + process = self.run_fixture("print('completed isolated command')", directory) + try: + stdout, stderr = process.communicate(timeout=5) + self.assertEqual(process.returncode, 0, stderr) + self.assertEqual(stdout, "completed isolated command\n") + finally: + self.cleanup_fixture(process, directory) + + def test_main_exit_seven_is_preserved_while_orphans_are_cleaned(self): + code = """ +import os, pathlib, subprocess, sys +root = pathlib.Path(sys.argv[2]) +child = subprocess.Popen([sys.executable, '-c', 'import time; time.sleep(60)', 'child-reaper-fixture']) +(root / 'orphan.pid').write_text(str(child.pid)) +sys.exit(7) +""" + with tempfile.TemporaryDirectory() as name: + directory = Path(name) + process = self.run_fixture(code, directory) + try: + self.assertEqual(process.wait(timeout=5), 7) + pid = int((directory / "orphan.pid").read_text()) + self.assertFalse(Path(f"/proc/{pid}").exists(), "owned orphan must be reaped before handoff") + finally: + self.cleanup_fixture(process, directory) + + def test_real_double_fork_orphan_is_reaped_while_command_runs(self): + code = """ +import os, pathlib, sys, time +root = pathlib.Path(sys.argv[2]) +intermediate = os.fork() +if intermediate == 0: + orphan = os.fork() + if orphan == 0: + os._exit(0) + (root / 'orphan.pid').write_text(str(orphan)) + os._exit(0) +os.waitpid(intermediate, 0) +pid = int((root / 'orphan.pid').read_text()) +deadline = time.monotonic() + 2 +while pathlib.Path(f'/proc/{pid}').exists() and time.monotonic() < deadline: + time.sleep(0.01) +if pathlib.Path(f'/proc/{pid}').exists(): + sys.exit(9) +print('orphan reaped while main command remains live') +""" + with tempfile.TemporaryDirectory() as name: + directory = Path(name) + process = self.run_fixture(code, directory) + try: + stdout, stderr = process.communicate(timeout=5) + self.assertEqual(process.returncode, 0, stderr) + self.assertIn("orphan reaped", stdout) + finally: + self.cleanup_fixture(process, directory) + + def test_interruption_reaches_main_and_cleans_owned_descendants(self): + code = """ +import os, pathlib, signal, subprocess, sys, time +root = pathlib.Path(sys.argv[2]) +(root / 'main.pid').write_text(str(os.getpid())) +signal.signal(signal.SIGTERM, lambda *_: sys.exit(17)) +child = subprocess.Popen([sys.executable, '-c', 'import time; time.sleep(60)', 'child-reaper-fixture']) +(root / 'orphan.pid').write_text(str(child.pid)) +(root / 'ready').write_text('ready') +time.sleep(60) +""" + with tempfile.TemporaryDirectory() as name: + directory = Path(name) + process = self.run_fixture(code, directory) + try: + deadline = time.monotonic() + 3 + while not (directory / "ready").exists() and time.monotonic() < deadline: + time.sleep(0.01) + self.assertTrue((directory / "ready").exists(), "fixture must be live before interruption") + started = time.monotonic() + process.send_signal(signal.SIGTERM) + self.assertEqual(process.wait(timeout=4), 17) + self.assertLess(time.monotonic() - started, 4) + for path in directory.glob("*.pid"): + self.assertFalse(Path(f"/proc/{int(path.read_text())}").exists(), "owned fixture must not survive interruption") + finally: + self.cleanup_fixture(process, directory) + + def test_ignored_interrupt_is_escalated_within_cleanup_deadline(self): + code = """ +import os, pathlib, signal, subprocess, sys, time +root = pathlib.Path(sys.argv[2]) +(root / 'main.pid').write_text(str(os.getpid())) +signal.signal(signal.SIGTERM, signal.SIG_IGN) +child = subprocess.Popen([sys.executable, '-c', 'import time; time.sleep(60)', 'child-reaper-fixture']) +(root / 'orphan.pid').write_text(str(child.pid)) +(root / 'ready').write_text('ready') +time.sleep(60) +""" + with tempfile.TemporaryDirectory() as name: + directory = Path(name) + process = self.run_fixture(code, directory) + try: + deadline = time.monotonic() + 3 + while not (directory / "ready").exists() and time.monotonic() < deadline: + time.sleep(0.01) + self.assertTrue((directory / "ready").exists()) + started = time.monotonic() + process.send_signal(signal.SIGTERM) + self.assertEqual(process.wait(timeout=4), 137) + self.assertLess(time.monotonic() - started, 4) + for path in directory.glob("*.pid"): + self.assertFalse(Path(f"/proc/{int(path.read_text())}").exists()) + finally: + self.cleanup_fixture(process, directory) + + +if __name__ == "__main__": + unittest.main() From 3bfe494589e5cbe0cc14043913cae66f14117b90 Mon Sep 17 00:00:00 2001 From: Josh Yorko Date: Wed, 14 Oct 2026 12:09:33 +0000 Subject: [PATCH 07/18] fix(test): qualify lifecycle checks under restricted procfs --- docs/dakota-native-codex-instance.md | 2 ++ scripts/run-with-child-reaper.py | 38 ++++++++++++++++++++++++++-- tests/native_instance_isolation.rs | 1 + tests/test_child_reaper.py | 15 +++++++++++ 4 files changed, 54 insertions(+), 2 deletions(-) diff --git a/docs/dakota-native-codex-instance.md b/docs/dakota-native-codex-instance.md index 15804a6..18d9fea 100644 --- a/docs/dakota-native-codex-instance.md +++ b/docs/dakota-native-codex-instance.md @@ -29,6 +29,8 @@ md() { PATH="$PATH" \ TMPDIR="${TMPDIR:-/tmp}" \ CODEX_MEMORYD_HOME="$MEMORYD_CODEX_HOME" \ + CODEX_MEMORYD_PROFILE=personal \ + CODEX_MEMORYD_WORKSPACE=dakota \ codex-memoryd "$@" } ``` diff --git a/scripts/run-with-child-reaper.py b/scripts/run-with-child-reaper.py index 1db555e..e67f35f 100755 --- a/scripts/run-with-child-reaper.py +++ b/scripts/run-with-child-reaper.py @@ -11,11 +11,45 @@ CLEANUP_GRACE_SECONDS = 2.0 KILL_WAIT_SECONDS = 1.0 +_CHILDREN_PATH_USABLE = True def adopted_children(main_pid): - """Only direct children of this launcher, excluding the Popen owner.""" - children = Path(f"/proc/self/task/{os.getpid()}/children").read_text().split() + """Only direct children of this launcher, excluding the Popen owner. + + Some restricted Linux procfs mounts omit the task ``children`` file. In + that case, recover the same direct-child list from process status records. + """ + global _CHILDREN_PATH_USABLE + if _CHILDREN_PATH_USABLE: + children_path = Path(f"/proc/self/task/{os.getpid()}/children") + try: + children = children_path.read_text().split() + except (FileNotFoundError, PermissionError): + _CHILDREN_PATH_USABLE = False + else: + return [int(pid) for pid in children if int(pid) != main_pid] + + children = [] + launcher_pid = os.getpid() + for process in Path("/proc").iterdir(): + if not process.name.isdecimal(): + continue + try: + status = (process / "status").read_text() + except (FileNotFoundError, ProcessLookupError): + # The process exited while procfs was being enumerated. + continue + parent_pid = next( + ( + int(line.split()[1]) + for line in status.splitlines() + if line.startswith("PPid:") + ), + None, + ) + if parent_pid == launcher_pid and int(process.name) != main_pid: + children.append(int(process.name)) return [int(pid) for pid in children if int(pid) != main_pid] diff --git a/tests/native_instance_isolation.rs b/tests/native_instance_isolation.rs index 6111fa4..19b084e 100644 --- a/tests/native_instance_isolation.rs +++ b/tests/native_instance_isolation.rs @@ -1057,6 +1057,7 @@ md down > "$MEMORYD_CODEX_HOME/down.json" ) .expect("resolved runbook config JSON"); assert_eq!(config["runtime"]["home"], memory_home.display().to_string()); + assert_eq!(config["runtime"]["workspace"], "dakota"); assert_eq!( config["daemon"]["storage_path"], memory_home.join("memory.db").display().to_string() diff --git a/tests/test_child_reaper.py b/tests/test_child_reaper.py index 044fa4f..a0e69ba 100644 --- a/tests/test_child_reaper.py +++ b/tests/test_child_reaper.py @@ -1,4 +1,5 @@ """Exercise the Linux isolated-test launcher against real owned processes.""" +import importlib.util import os from pathlib import Path import signal @@ -9,6 +10,9 @@ import unittest LAUNCHER = Path(__file__).resolve().parents[1] / "scripts" / "run-with-child-reaper.py" +SPEC = importlib.util.spec_from_file_location("memoryd_child_reaper", LAUNCHER) +CHILD_REAPER = importlib.util.module_from_spec(SPEC) +SPEC.loader.exec_module(CHILD_REAPER) @unittest.skipUnless(sys.platform == "linux", "Linux child-subreaper contract") @@ -33,6 +37,17 @@ def cleanup_fixture(self, process, directory): pass process.communicate(timeout=5) + def test_proc_status_fallback_finds_direct_child(self): + process = subprocess.Popen([sys.executable, "-c", "import time; time.sleep(5)"]) + previous = CHILD_REAPER._CHILDREN_PATH_USABLE + CHILD_REAPER._CHILDREN_PATH_USABLE = False + try: + self.assertIn(process.pid, CHILD_REAPER.adopted_children(-1)) + finally: + CHILD_REAPER._CHILDREN_PATH_USABLE = previous + process.terminate() + process.wait(timeout=5) + def test_success_preserves_output_and_exit(self): with tempfile.TemporaryDirectory() as name: directory = Path(name) From 8c94d1a6f2871719af17f07aa04214c6c3b063be Mon Sep 17 00:00:00 2001 From: Josh Yorko Date: Wed, 14 Oct 2026 12:54:27 +0000 Subject: [PATCH 08/18] fix(container): honor configured database path --- docs/getting-started.md | 5 + src/native_runtime.rs | 199 +++++++++++++++++++++++++++++++++++++--- 2 files changed, 190 insertions(+), 14 deletions(-) diff --git a/docs/getting-started.md b/docs/getting-started.md index f3f7021..61c537a 100644 --- a/docs/getting-started.md +++ b/docs/getting-started.md @@ -108,6 +108,11 @@ directory of the persistent SQLite database at `/data`, preserving SQLite sidecar files such as WAL and shared-memory files. Docker or Podman must be installed and available to the CLI. +When `--db` or `CODEX_MEMORYD_DB` selects a custom database, managed container +startup mounts that database's parent directory and configures the matching +`/data/` path inside the container. The host root cannot be used as +the database parent; choose a dedicated data directory instead. + To connect Codex to the container-backed read-only MCP server: ```zsh diff --git a/src/native_runtime.rs b/src/native_runtime.rs index 0d1ebe0..ffe6112 100644 --- a/src/native_runtime.rs +++ b/src/native_runtime.rs @@ -604,6 +604,13 @@ fn up_container(opts: &RuntimeOptions) -> Result { if !opts.codex_memories_dir.exists() { fs::create_dir_all(&opts.codex_memories_dir).map_err(Error::from)?; } + let database_parent = opts + .db + .parent() + .filter(|parent| !parent.as_os_str().is_empty()) + .unwrap_or_else(|| Path::new(".")); + fs::create_dir_all(database_parent).map_err(Error::from)?; + let (database_parent, container_db_path) = container_database_mount(opts)?; if container_exists(&runtime, &opts.container_name)? { if !container_running(&runtime, &opts.container_name)? { @@ -628,7 +635,7 @@ fn up_container(opts: &RuntimeOptions) -> Result { "--publish".to_string(), format!("{}:{}:8787", opts.host, opts.port), "--volume".to_string(), - format!("{}:/data", opts.home.display()), + format!("{}:/data", database_parent.display()), "--volume".to_string(), format!( "{}:/host-codex-memories:ro", @@ -639,7 +646,7 @@ fn up_container(opts: &RuntimeOptions) -> Result { "--env".to_string(), "CODEX_MEMORYD_DECLARE_LOOPBACK_PUBLISH=1".to_string(), "--env".to_string(), - "CODEX_MEMORYD_DB=/data/memory.db".to_string(), + format!("CODEX_MEMORYD_DB={container_db_path}"), "--env".to_string(), format!("CODEX_MEMORYD_PROFILE={}", opts.profile), "--env".to_string(), @@ -674,10 +681,139 @@ fn up_container(opts: &RuntimeOptions) -> Result { )?; } - wait_for_ready(opts, "/data/memory.db")?; + wait_for_ready(opts, &container_db_path)?; Ok(status_container(opts)) } +fn container_database_mount(opts: &RuntimeOptions) -> Result<(PathBuf, String)> { + let database = if opts.db.is_absolute() { + opts.db.clone() + } else { + std::env::current_dir() + .map_err(|err| { + Error::invalid_request(format!( + "cannot resolve relative container database path {}: {err}", + opts.db.display() + )) + })? + .join(&opts.db) + }; + let filename = database + .file_name() + .and_then(|name| name.to_str()) + .filter(|name| !name.is_empty() && *name != "." && *name != "..") + .ok_or_else(|| { + Error::invalid_request(format!( + "container database path must name a UTF-8 file: {}", + database.display() + )) + })?; + if filename + .chars() + .any(|character| character.is_control() || character == ':') + { + return Err(Error::invalid_request( + "container database filename contains characters that cannot be represented safely in a Docker/Podman mount", + )); + } + let configured_parent = database.parent().ok_or_else(|| { + Error::invalid_request(format!( + "container database path has no usable parent directory: {}", + database.display() + )) + })?; + let configured_metadata = fs::metadata(configured_parent).map_err(|err| { + Error::invalid_request(format!( + "container database parent directory is not usable: {} ({err})", + configured_parent.display() + )) + })?; + if !configured_metadata.is_dir() { + return Err(Error::invalid_request(format!( + "container database parent is not a directory: {}", + configured_parent.display() + ))); + } + let configured_parent = fs::canonicalize(configured_parent).map_err(|err| { + Error::invalid_request(format!( + "container database parent cannot be resolved safely: {} ({err})", + database + .parent() + .unwrap_or_else(|| Path::new(".")) + .display() + )) + })?; + let resolved_database = match fs::symlink_metadata(&database) { + Ok(_) => { + let resolved = fs::canonicalize(&database).map_err(|err| { + Error::invalid_request(format!( + "container database cannot be resolved safely: {} ({err})", + database.display() + )) + })?; + if !fs::metadata(&resolved).map_err(Error::from)?.is_file() { + return Err(Error::invalid_request(format!( + "container database path is not a file: {}", + database.display() + ))); + } + resolved + } + Err(err) if err.kind() == std::io::ErrorKind::NotFound => configured_parent.join(filename), + Err(err) => { + return Err(Error::invalid_request(format!( + "container database path is not usable: {} ({err})", + database.display() + ))) + } + }; + let database_parent = resolved_database.parent().ok_or_else(|| { + Error::invalid_request(format!( + "container database path has no usable parent directory: {}", + resolved_database.display() + )) + })?; + let database_parent = fs::canonicalize(database_parent).map_err(|err| { + Error::invalid_request(format!( + "container database parent cannot be resolved safely: {} ({err})", + database_parent.display() + )) + })?; + if database_parent == Path::new("/") { + return Err(Error::invalid_request( + "container runtime refuses to mount the host root as the database parent; set CODEX_MEMORYD_DB inside a dedicated data directory", + )); + } + let database_parent_text = database_parent.to_str().ok_or_else(|| { + Error::invalid_request( + "container database parent is not valid UTF-8 and cannot be represented safely", + ) + })?; + if database_parent_text.contains(':') || database_parent_text.chars().any(char::is_control) { + return Err(Error::invalid_request(format!( + "container database parent cannot be represented safely in a Docker/Podman volume mount: {database_parent_text}" + ))); + } + let filename = resolved_database + .file_name() + .and_then(|name| name.to_str()) + .ok_or_else(|| { + Error::invalid_request(format!( + "container database filename is not valid UTF-8: {}", + resolved_database.display() + )) + })?; + if filename + .chars() + .any(|character| character.is_control() || character == ':') + { + return Err(Error::invalid_request( + "resolved container database filename contains characters that cannot be represented safely in a Docker/Podman mount", + )); + } + Ok((database_parent, format!("/data/{filename}"))) +} + fn down_native(opts: &RuntimeOptions) -> Result { let Some(pid) = running_pid(&opts.pid_file) else { let _ = fs::remove_file(&opts.pid_file); @@ -860,7 +996,9 @@ fn status_container(opts: &RuntimeOptions) -> RuntimeStatusReport { pid: None, url: opts.url.clone(), bind: format!("{}:{} -> container:8787", opts.host, opts.port), - db: format!("{}:/data/memory.db", opts.home.display()), + db: container_database_mount(opts) + .map(|(parent, path)| format!("{}:{path}", parent.display())) + .unwrap_or_else(|_| opts.db.display().to_string()), log_file: format!( "{} logs {}", container_runtime_name(opts), @@ -1583,10 +1721,19 @@ mod container_readiness_regressions { (format!("http://{address}"), stop, server) } - fn fake_container_runtime(path: &std::path::Path) { + fn fake_container_runtime( + path: &std::path::Path, + invocation_log: &std::path::Path, + state_file: &std::path::Path, + ) { fs::write( path, - "#!/bin/sh\nif [ \"$1\" = \"--version\" ]; then exit 0; fi\nif [ \"$1\" = \"container\" ] && [ \"$2\" = \"inspect\" ]; then if [ \"$3\" = \"--format\" ]; then printf true; fi; exit 0; fi\nif [ \"$1\" = \"start\" ]; then exit 0; fi\nexit 1\n", + format!( + "#!/bin/sh\nif [ \"$1\" = \"--version\" ]; then exit 0; fi\nif [ \"$1\" = \"container\" ] && [ \"$2\" = \"inspect\" ]; then if [ -f '{}' ]; then if [ \"$3\" = \"--format\" ]; then printf true; fi; exit 0; fi; exit 1; fi\nif [ \"$1\" = \"run\" ]; then printf '%s\\n' \"$*\" >> '{}'; touch '{}'; exit 0; fi\nif [ \"$1\" = \"start\" ]; then exit 0; fi\nexit 1\n", + state_file.display(), + invocation_log.display(), + state_file.display() + ), ) .unwrap(); let mut permissions = fs::metadata(path).unwrap().permissions(); @@ -1596,13 +1743,21 @@ mod container_readiness_regressions { fn run_managed_container_up( status_path: &str, - ) -> crate::error::Result { + db_relative_path: &str, + ) -> ( + crate::error::Result, + String, + String, + ) { let temp = tempfile::tempdir().unwrap(); let runtime = temp.path().join("fake-container-runtime"); - fake_container_runtime(&runtime); + let invocation_log = temp.path().join("container-runtime.log"); + let state_file = temp.path().join("container-created"); + fake_container_runtime(&runtime, &invocation_log, &state_file); let (url, stop, server) = status_server(container_status(status_path)); - let address = url.strip_prefix("http://").unwrap(); - let port = address.rsplit_once(':').unwrap().1.parse().unwrap(); + let port = 0; + let db = temp.path().join(db_relative_path); + let database_parent = db.parent().unwrap().to_string_lossy().into_owned(); let options = RuntimeOptions { runtime: RuntimeKind::Container, home: temp.path().join("runtime-home"), @@ -1610,7 +1765,7 @@ mod container_readiness_regressions { host: "127.0.0.1".into(), port, bind: format!("127.0.0.1:{port}"), - db: PathBuf::from("/tmp/host-memory.db"), + db, pid_file: temp.path().join("unused.pid"), log_file: temp.path().join("unused.log"), profile: "personal".into(), @@ -1630,25 +1785,41 @@ mod container_readiness_regressions { let result = up(&options); stop.store(true, Ordering::Relaxed); server.join().unwrap(); - result + let invocations = fs::read_to_string(invocation_log).unwrap_or_default(); + (result, invocations, database_parent) } #[test] fn up_accepts_healthy_container_status_at_container_database_path() { - let result = run_managed_container_up("/data/memory.db"); + let (result, invocations, database_parent) = + run_managed_container_up("/data/memory.db", "runtime-home/memory.db"); assert!( result.is_ok(), "managed container startup failed: {result:?}" ); assert_eq!(result.unwrap().health, "ok"); + assert!(invocations.contains(&format!("--volume {database_parent}:/data"))); + assert!(invocations.contains("CODEX_MEMORYD_DB=/data/memory.db")); } #[test] fn up_rejects_container_status_for_wrong_database_path() { - let result = run_managed_container_up("/data/wrong.db"); + let (result, _, _) = run_managed_container_up("/data/wrong.db", "runtime-home/memory.db"); let error = result.expect_err("wrong managed-container storage path must fail readiness"); assert!(error .message .contains("core readiness requirements are not met")); } + + #[test] + fn up_mounts_and_verifies_the_configured_custom_database() { + let (result, invocations, database_parent) = + run_managed_container_up("/data/custom.sqlite", "custom-store/custom.sqlite"); + assert!( + result.is_ok(), + "managed container startup failed for configured database: {result:?}" + ); + assert!(invocations.contains(&format!("--volume {database_parent}:/data"))); + assert!(invocations.contains("CODEX_MEMORYD_DB=/data/custom.sqlite")); + } } From e2d9f66a4eaa88f84956d6d7b0cc779021365ed6 Mon Sep 17 00:00:00 2001 From: Josh Yorko Date: Wed, 14 Oct 2026 13:33:05 +0000 Subject: [PATCH 09/18] fix: close bounded scheduler and container review gaps --- AGENTS.md | 2 +- scripts/v0.1-release-gate.sh | 2 +- src/dream.rs | 94 ++++++++++++++++++++--- src/native_runtime.rs | 110 +++++++++++++++++++++++++-- src/service.rs | 19 +++-- tests/scheduler_review_boundaries.rs | 73 ++++++++++++++++++ 6 files changed, 269 insertions(+), 31 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 857303f..b90e71d 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -79,7 +79,7 @@ command's exit receipt. Shared log files do not transfer ownership of an agent-local execution session; do not hand off a still-running gate as verified. Validate the launcher with -`python3 -m unittest discover -s tests -p test_child_reaper.py -v`. +`python3 -B -m unittest discover -s tests -p test_child_reaper.py -v`. Its `run` function owns the main command's exit status separately from adopted child reaping and bounds signal cleanup. The canonical release gate includes these launcher checks. diff --git a/scripts/v0.1-release-gate.sh b/scripts/v0.1-release-gate.sh index baa6fc7..386c57a 100755 --- a/scripts/v0.1-release-gate.sh +++ b/scripts/v0.1-release-gate.sh @@ -224,7 +224,7 @@ PY : >"$ARTIFACT_DIR/steps.jsonl" -run "child-reaper-tests" python3 -m unittest discover -s tests -p test_child_reaper.py -v +run "child-reaper-tests" python3 -B -m unittest discover -s tests -p test_child_reaper.py -v run "cargo-fmt" cargo fmt --all --check run "git-diff-check" git diff --check diff --git a/src/dream.rs b/src/dream.rs index f403db7..55b81ed 100644 --- a/src/dream.rs +++ b/src/dream.rs @@ -60,6 +60,8 @@ struct DreamSchedulerCursor { #[serde(default)] lower_bound: Option, #[serde(default)] + upper_bound: Option, + #[serde(default)] visible_turns: Option, #[serde(default)] conclusions: Option, @@ -71,6 +73,24 @@ struct DreamSchedulerCursor { active_memory_records: Option, } +impl DreamSchedulerCursor { + fn has_source_position(&self) -> bool { + self.visible_turns.is_some() + || self.conclusions.is_some() + || self.checkpoints.is_some() + || self.imported_memories.is_some() + || self.active_memory_records.is_some() + } + + fn clear_source_positions(&mut self) { + self.visible_turns = None; + self.conclusions = None; + self.checkpoints = None; + self.imported_memories = None; + self.active_memory_records = None; + } +} + static RELATIVE_TIME: Lazy = Lazy::new(|| { Regex::new( r"(?i)\b(today|tomorrow|tonight|this week|next week|this weekend|currently|right now|soon|as of now|going to|planning to|will\s+(add|build|complete|deploy|fix|implement|merge|migrate|patch|release|remove|replace|resolve|run|ship|switch|test|update|write))\b", @@ -282,7 +302,19 @@ pub struct DreamParams<'a> { pub fn run(store: &Store, params: &DreamParams) -> Result<(DreamResponse, bool)> { check_deadline(params)?; - let scheduler_cursor = params.recency_cutoff.and_then(parse_scheduler_cursor); + let mut scheduler_cursor = params.recency_cutoff.and_then(parse_scheduler_cursor); + if let Some(cursor) = scheduler_cursor.as_mut() { + // Cursors written before the fixed upper frontier was introduced can + // only be resumed safely by rescanning their current bounded window. + if cursor.has_source_position() && cursor.upper_bound.is_none() { + cursor.clear_source_positions(); + } + } + let source_window_end = scheduler_cursor + .as_ref() + .and_then(|cursor| cursor.upper_bound.as_deref()) + .or(params.source_window_end) + .unwrap_or(params.now); let source_window_start = match scheduler_cursor.as_ref() { Some(cursor) => cursor.lower_bound.as_deref(), None => params.recency_cutoff, @@ -305,7 +337,7 @@ pub fn run(store: &Store, params: &DreamParams) -> Result<(DreamResponse, bool)> params.workspace, params.repo_id, source_window_start, - params.source_window_end, + Some(source_window_end), cursor.active_memory_records.as_ref(), params.max_records, params.include_archived_sources, @@ -315,7 +347,6 @@ pub fn run(store: &Store, params: &DreamParams) -> Result<(DreamResponse, bool)> None => store.query_records(&query)?, }, }; - let source_window_end = params.source_window_end.unwrap_or(params.now); check_deadline(params)?; let imported_limit = params.max_records.saturating_sub(records.len()); if imported_limit > 0 { @@ -324,6 +355,7 @@ pub fn run(store: &Store, params: &DreamParams) -> Result<(DreamResponse, bool)> params, imported_limit, source_window_start, + source_window_end, scheduler_cursor.as_ref(), )?); } @@ -672,6 +704,7 @@ fn imported_chatgpt_candidate_records( params: &DreamParams, limit: usize, source_window_start: Option<&str>, + source_window_end: &str, scheduler_cursor: Option<&DreamSchedulerCursor>, ) -> Result> { let turns = match scheduler_cursor { @@ -681,7 +714,7 @@ fn imported_chatgpt_candidate_records( params.workspace, params.repo_id, source_window_start, - params.source_window_end, + Some(source_window_end), cursor.visible_turns.as_ref(), params.max_records, )?, @@ -853,8 +886,47 @@ fn scheduler_cursor_from_watermark(previous: Option<&str>) -> DreamSchedulerCurs pub fn scheduler_watermark_after( previous: Option<&str>, window: &DreamEvidenceWindow, + now: &str, + window_full: bool, ) -> Result> { let mut cursor = scheduler_cursor_from_watermark(previous); + if window_full { + // Keep the original scan frontier stable while each source stream is + // drained. Rows arriving after it are scanned in a new bounded band. + if cursor.upper_bound.is_none() { + cursor.clear_source_positions(); + cursor.upper_bound = Some(window.end.clone()); + } + update_scheduler_cursor(&mut cursor, window); + let encoded = serde_json::to_string(&cursor)?; + return Ok(Some(format!("{SCHEDULER_CURSOR_PREFIX}{encoded}"))); + } + + if let Some(mut previous_cursor) = previous.and_then(parse_scheduler_cursor) { + if previous_cursor.has_source_position() { + if let Some(frontier) = previous_cursor.upper_bound.take() { + let now_is_after_frontier = OffsetDateTime::parse(now, &Rfc3339) + .ok() + .zip(OffsetDateTime::parse(&frontier, &Rfc3339).ok()) + .is_some_and(|(now, frontier)| now > frontier); + if now_is_after_frontier { + let next = DreamSchedulerCursor { + lower_bound: Some(frontier), + upper_bound: Some(now.to_string()), + ..DreamSchedulerCursor::default() + }; + let encoded = serde_json::to_string(&next)?; + return Ok(Some(format!("{SCHEDULER_CURSOR_PREFIX}{encoded}"))); + } + return Ok(Some(frontier)); + } + } + } + + Ok(Some(now.to_string())) +} + +fn update_scheduler_cursor(cursor: &mut DreamSchedulerCursor, window: &DreamEvidenceWindow) { update_stream_cursor(&mut cursor.visible_turns, &window.visible_turns, false); update_stream_cursor(&mut cursor.conclusions, &window.conclusions, false); update_stream_cursor(&mut cursor.checkpoints, &window.checkpoints, false); @@ -883,9 +955,6 @@ pub fn scheduler_watermark_after( ); } } - - let encoded = serde_json::to_string(&cursor)?; - Ok(Some(format!("{SCHEDULER_CURSOR_PREFIX}{encoded}"))) } fn update_stream_cursor( @@ -1225,6 +1294,9 @@ fn build_evidence_window( active_records: &[MemoryRecord], ) -> Result { let mut remaining = params.max_records.saturating_sub(active_records.len()); + let cursor_end = scheduler_cursor + .map(|_| Some(end)) + .unwrap_or(params.source_window_end); let visible_turns = if remaining == 0 { Vec::new() } else if let Some(cursor) = scheduler_cursor { @@ -1234,7 +1306,7 @@ fn build_evidence_window( params.workspace, params.repo_id, start, - params.source_window_end, + cursor_end, cursor.visible_turns.as_ref(), remaining, )?; @@ -1246,7 +1318,7 @@ fn build_evidence_window( params.workspace, params.repo_id, start, - params.source_window_end, + cursor_end, remaining, )?; remaining = remaining.saturating_sub(records.len()); @@ -1261,7 +1333,7 @@ fn build_evidence_window( params.workspace, params.repo_id, start, - params.source_window_end, + cursor_end, cursor.conclusions.as_ref(), remaining, )?; @@ -1314,7 +1386,7 @@ fn build_evidence_window( params.profile.as_str(), params.workspace, start, - params.source_window_end, + cursor_end, cursor.imported_memories.as_ref(), remaining, )? diff --git a/src/native_runtime.rs b/src/native_runtime.rs index ffe6112..1e8a560 100644 --- a/src/native_runtime.rs +++ b/src/native_runtime.rs @@ -613,6 +613,7 @@ fn up_container(opts: &RuntimeOptions) -> Result { let (database_parent, container_db_path) = container_database_mount(opts)?; if container_exists(&runtime, &opts.container_name)? { + ensure_container_database_mount(&runtime, &opts.container_name, &database_parent)?; if !container_running(&runtime, &opts.container_name)? { ensure_bind_available(opts)?; } @@ -814,6 +815,44 @@ fn container_database_mount(opts: &RuntimeOptions) -> Result<(PathBuf, String)> Ok((database_parent, format!("/data/{filename}"))) } +fn ensure_container_database_mount( + runtime: &str, + name: &str, + expected_parent: &Path, +) -> Result<()> { + let template = r#"{{range .Mounts}}{{if eq .Destination "/data"}}{{.Source}}{{end}}{{end}}"#; + let output = Command::new(runtime) + .args(["container", "inspect", "--format", template, name]) + .output() + .map_err(Error::from)?; + if !output.status.success() { + return Err(Error::storage(format!( + "could not verify the /data database mount for managed container {name}: {}", + String::from_utf8_lossy(&output.stderr).trim() + ))); + } + let actual_source = String::from_utf8_lossy(&output.stdout).trim().to_string(); + if actual_source.is_empty() { + return Err(Error::storage(format!( + "managed container {name} has no verifiable /data bind mount; refusing to start it" + ))); + } + let actual_parent = fs::canonicalize(&actual_source).map_err(|err| { + Error::storage(format!( + "managed container {name} /data mount source cannot be resolved safely: {} ({err})", + actual_source + )) + })?; + if actual_parent != expected_parent { + return Err(Error::storage(format!( + "managed container {name} is mounted from {} at /data, but the configured database requires {}; refusing to start it because it would use the wrong database", + actual_parent.display(), + expected_parent.display() + ))); + } + Ok(()) +} + fn down_native(opts: &RuntimeOptions) -> Result { let Some(pid) = running_pid(&opts.pid_file) else { let _ = fs::remove_file(&opts.pid_file); @@ -1725,17 +1764,31 @@ mod container_readiness_regressions { path: &std::path::Path, invocation_log: &std::path::Path, state_file: &std::path::Path, + mounted_data_source: &std::path::Path, + initial_existing_container: bool, ) { + let mount_source_file = path.with_extension("mount-source"); fs::write( path, format!( - "#!/bin/sh\nif [ \"$1\" = \"--version\" ]; then exit 0; fi\nif [ \"$1\" = \"container\" ] && [ \"$2\" = \"inspect\" ]; then if [ -f '{}' ]; then if [ \"$3\" = \"--format\" ]; then printf true; fi; exit 0; fi; exit 1; fi\nif [ \"$1\" = \"run\" ]; then printf '%s\\n' \"$*\" >> '{}'; touch '{}'; exit 0; fi\nif [ \"$1\" = \"start\" ]; then exit 0; fi\nexit 1\n", - state_file.display(), + "#!/bin/sh\nif [ \"$1\" = \"--version\" ]; then exit 0; fi\nprintf '%s\\n' \"$*\" >> '{}';\nif [ \"$1\" = \"container\" ] && [ \"$2\" = \"inspect\" ]; then if [ -f '{}' ]; then if [ \"$3\" = \"--format\" ]; then case \"$4\" in *State.Running*) printf true ;; *Mounts*) cat '{}' ;; esac; fi; exit 0; fi; exit 1; fi\nif [ \"$1\" = \"run\" ]; then printf '%s' '{}' > '{}'; touch '{}'; exit 0; fi\nif [ \"$1\" = \"start\" ]; then exit 0; fi\nexit 1\n", invocation_log.display(), + state_file.display(), + mount_source_file.display(), + mounted_data_source.display(), + mount_source_file.display(), state_file.display() ), ) .unwrap(); + if initial_existing_container { + fs::write( + &mount_source_file, + mounted_data_source.as_os_str().to_string_lossy().as_bytes(), + ) + .unwrap(); + fs::write(state_file, "existing").unwrap(); + } let mut permissions = fs::metadata(path).unwrap().permissions(); permissions.set_mode(0o755); fs::set_permissions(path, permissions).unwrap(); @@ -1744,6 +1797,7 @@ mod container_readiness_regressions { fn run_managed_container_up( status_path: &str, db_relative_path: &str, + existing_mount_relative_path: Option<&str>, ) -> ( crate::error::Result, String, @@ -1753,11 +1807,23 @@ mod container_readiness_regressions { let runtime = temp.path().join("fake-container-runtime"); let invocation_log = temp.path().join("container-runtime.log"); let state_file = temp.path().join("container-created"); - fake_container_runtime(&runtime, &invocation_log, &state_file); - let (url, stop, server) = status_server(container_status(status_path)); - let port = 0; let db = temp.path().join(db_relative_path); let database_parent = db.parent().unwrap().to_string_lossy().into_owned(); + let mounted_source = existing_mount_relative_path + .map(|path| temp.path().join(path)) + .unwrap_or_else(|| PathBuf::from(&database_parent)); + if existing_mount_relative_path.is_some() { + fs::create_dir_all(&mounted_source).unwrap(); + } + fake_container_runtime( + &runtime, + &invocation_log, + &state_file, + &mounted_source, + existing_mount_relative_path.is_some(), + ); + let (url, stop, server) = status_server(container_status(status_path)); + let port = 0; let options = RuntimeOptions { runtime: RuntimeKind::Container, home: temp.path().join("runtime-home"), @@ -1792,7 +1858,7 @@ mod container_readiness_regressions { #[test] fn up_accepts_healthy_container_status_at_container_database_path() { let (result, invocations, database_parent) = - run_managed_container_up("/data/memory.db", "runtime-home/memory.db"); + run_managed_container_up("/data/memory.db", "runtime-home/memory.db", None); assert!( result.is_ok(), "managed container startup failed: {result:?}" @@ -1804,7 +1870,8 @@ mod container_readiness_regressions { #[test] fn up_rejects_container_status_for_wrong_database_path() { - let (result, _, _) = run_managed_container_up("/data/wrong.db", "runtime-home/memory.db"); + let (result, _, _) = + run_managed_container_up("/data/wrong.db", "runtime-home/memory.db", None); let error = result.expect_err("wrong managed-container storage path must fail readiness"); assert!(error .message @@ -1814,7 +1881,7 @@ mod container_readiness_regressions { #[test] fn up_mounts_and_verifies_the_configured_custom_database() { let (result, invocations, database_parent) = - run_managed_container_up("/data/custom.sqlite", "custom-store/custom.sqlite"); + run_managed_container_up("/data/custom.sqlite", "custom-store/custom.sqlite", None); assert!( result.is_ok(), "managed container startup failed for configured database: {result:?}" @@ -1822,4 +1889,31 @@ mod container_readiness_regressions { assert!(invocations.contains(&format!("--volume {database_parent}:/data"))); assert!(invocations.contains("CODEX_MEMORYD_DB=/data/custom.sqlite")); } + + #[test] + fn up_rejects_an_existing_container_with_a_stale_database_mount() { + let (result, invocations, database_parent) = + run_managed_container_up("/data/memory.db", "new-store/memory.db", Some("old-store")); + let error = result.expect_err("old container must not start with a different DB mount"); + assert!(error.message.contains("mounted from"), "{error:?}"); + assert!(error.message.contains(&database_parent), "{error:?}"); + assert!(!invocations.contains("start synthetic-memoryd")); + assert!(!invocations.contains("run -d")); + } + + #[test] + fn up_reuses_an_existing_container_with_the_configured_database_mount() { + let (result, invocations, _database_parent) = run_managed_container_up( + "/data/memory.db", + "runtime-home/memory.db", + Some("runtime-home"), + ); + assert!( + result.is_ok(), + "matching managed container should be reusable: {result:?}" + ); + assert!(invocations.contains("container inspect --format")); + assert!(invocations.contains("start synthetic-memoryd")); + assert!(!invocations.contains("run -d")); + } } diff --git a/src/service.rs b/src/service.rs index 9376a34..70719c1 100644 --- a/src/service.rs +++ b/src/service.rs @@ -2796,19 +2796,18 @@ impl Service { // A limited run has not durably covered the source frontier. // Preserve a bounded source cursor for input-only limits; keep // the previous cursor for runtime, provider, or candidate limits. - let watermark_after = if limits_hit.is_empty() { - Some(now.clone()) - } else if input_window_full - && limits_hit.iter().all(|limit| { - matches!( - limit.as_str(), - "max_input_records" | "max_input_bytes" | "max_input_tokens" - ) - }) - { + let input_only_limits = limits_hit.iter().all(|limit| { + matches!( + limit.as_str(), + "max_input_records" | "max_input_bytes" | "max_input_tokens" + ) + }); + let watermark_after = if input_only_limits { dream::scheduler_watermark_after( watermark_before.as_deref(), &run.evidence_window, + &now, + input_window_full, )? } else { None diff --git a/tests/scheduler_review_boundaries.rs b/tests/scheduler_review_boundaries.rs index 46f7304..5c5d212 100644 --- a/tests/scheduler_review_boundaries.rs +++ b/tests/scheduler_review_boundaries.rs @@ -279,6 +279,79 @@ fn scheduler_cursor_keeps_lower_bound_across_all_streams_and_ties() { } } +#[test] +fn scheduler_cursor_keeps_new_evidence_eligible_while_draining() { + let store = Store::open(":memory:").expect("store"); + let service = Service::new(store.clone(), scheduler_config("workspace", 2, false)); + service + .scheduled_dream(Some(BEFORE_FRONTIER.into())) + .expect("seed watermark"); + store + .ensure_workspace(PROFILE, "workspace") + .expect("workspace"); + + for (id, created_at) in [ + ("visible-newest-before-drain", "2026-09-10T00:00:00Z"), + ("visible-middle-before-drain", "2026-09-09T00:00:00Z"), + ("visible-oldest-before-drain", "2026-09-08T00:00:00Z"), + ] { + insert_chatgpt_turn( + &store, + "cursor-session", + id, + &format!("generic evidence {id}"), + created_at, + None, + ); + } + + let first = service + .scheduled_dream(Some(RUN_NOW.into())) + .expect("first bounded scheduler page"); + assert!(first.limits_hit.contains(&"max_input_records".to_string())); + assert_eq!( + evidence_ids(first.run.as_ref().expect("first run")).len(), + 2, + "first page should be bounded to two sources" + ); + + insert_chatgpt_turn( + &store, + "cursor-session", + "visible-arrived-during-drain", + "generic evidence arrived during the bounded scan", + "2030-01-01T00:00:01Z", + None, + ); + + let mut drained = false; + for tick in 2..=10 { + let now = format!("2030-01-01T00:00:{tick:02}Z"); + let page = service + .scheduled_dream(Some(now.into())) + .expect("drain bounded scheduler page"); + let input_limited = page.limits_hit.contains(&"max_input_records".to_string()); + if !input_limited { + drained = true; + break; + } + } + assert!( + drained, + "the original bounded window should eventually drain" + ); + + let arrived = service + .scheduled_dream(Some("2030-01-01T00:00:11Z".into())) + .expect("scan evidence that arrived during the drain"); + let seen = evidence_ids(arrived.run.as_ref().expect("arrival scan")); + assert!( + seen.contains("visible-arrived-during-drain"), + "newer evidence must remain eligible until the cursor reaches its frontier; evidence={seen:?}; limits={:?}", + arrived.limits_hit + ); +} + #[test] fn synthetic_repository_evidence_resolves_trusted_session_boundary() { let store = Store::open(":memory:").expect("store"); From 83149509d3488a85ae6e0b5c048dde81ff8608ba Mon Sep 17 00:00:00 2001 From: Josh Yorko Date: Wed, 14 Oct 2026 15:59:11 +0000 Subject: [PATCH 10/18] fix: address latest PR 261 review findings --- AGENTS.md | 5 + docs/dreamer-consolidation.md | 5 +- docs/dreamer-native-provider.md | 4 +- scripts/run-with-child-reaper.py | 4 +- src/dream.rs | 29 +++- src/native_runtime.rs | 189 +++++++++++++++++++++++++-- tests/scheduler_review_boundaries.rs | 104 +++++++++++++++ tests/test_child_reaper.py | 29 ++++ 8 files changed, 350 insertions(+), 19 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index b90e71d..39ba11b 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -67,6 +67,11 @@ Use the launcher only for isolated synthetic checks. When the command exits or is interrupted, it terminates and reaps the command's remaining descendants; do not wrap a real operator service or `--include-dogfood` with it. +On restricted procfs mounts, the launcher falls back to scanning +`/proc//status` for direct children. Process records that disappear or +cannot be read due to permissions are skipped; the focused test +`test_proc_status_fallback_skips_unreadable_status_entries` covers that case. + A non-reaping PID 1 can leave an exited daemon as a zombie. Linux `kill(pid, 0)` still reports that PID as present, so `src/native_runtime.rs::process_alive` cannot establish clean shutdown in that environment. Inspect `/proc//status` diff --git a/docs/dreamer-consolidation.md b/docs/dreamer-consolidation.md index 622daf2..2aff5bb 100644 --- a/docs/dreamer-consolidation.md +++ b/docs/dreamer-consolidation.md @@ -49,7 +49,10 @@ rejects the transaction as stale. A scheduled run that fills its bounded input window persists a per-stream continuation cursor, including timestamp ties, rather than skipping to the run's -end. Later ticks consume the remaining pages before advancing the time watermark. +end. Every source stream uses that cursor's fixed upper frontier. Later ticks +consume the remaining pages before advancing the time watermark; a new time band +starts strictly after the prior frontier so boundary evidence is not selected +twice. Runtime or candidate limits retain the previous cursor. A scheduled command's watermark does not authorize replay of archived or withdrawn evidence. diff --git a/docs/dreamer-native-provider.md b/docs/dreamer-native-provider.md index 5d80e76..d5d2d33 100644 --- a/docs/dreamer-native-provider.md +++ b/docs/dreamer-native-provider.md @@ -73,7 +73,9 @@ store is writable SQLite at the current schema, the exact configured profile/workspace pair exists, exposure/auth fields are loopback-only, and the sole degradation reason matches a persisted failed preview Dreamer receipt. Native readiness checks the configured host database path; managed containers -check `/data/memory.db`, the path passed to the daemon inside the container. +check both the host directory mounted at `/data` and the exact configured +`/data/` path passed to the daemon inside the container. Reusing a +container with a stale database filename is refused before it is started. `/v1/status` reports `features.configured_scope` using an exact workspace query, rather than combining global profile and workspace lists. A healthy empty store can start before its first workspace is created. diff --git a/scripts/run-with-child-reaper.py b/scripts/run-with-child-reaper.py index e67f35f..fb46d99 100755 --- a/scripts/run-with-child-reaper.py +++ b/scripts/run-with-child-reaper.py @@ -37,8 +37,8 @@ def adopted_children(main_pid): continue try: status = (process / "status").read_text() - except (FileNotFoundError, ProcessLookupError): - # The process exited while procfs was being enumerated. + except (FileNotFoundError, ProcessLookupError, PermissionError): + # The process exited or its status record is unreadable. continue parent_pid = next( ( diff --git a/src/dream.rs b/src/dream.rs index 55b81ed..719c663 100644 --- a/src/dream.rs +++ b/src/dream.rs @@ -60,6 +60,8 @@ struct DreamSchedulerCursor { #[serde(default)] lower_bound: Option, #[serde(default)] + lower_bound_exclusive: bool, + #[serde(default)] upper_bound: Option, #[serde(default)] visible_turns: Option, @@ -337,6 +339,7 @@ pub fn run(store: &Store, params: &DreamParams) -> Result<(DreamResponse, bool)> params.workspace, params.repo_id, source_window_start, + cursor.lower_bound_exclusive, Some(source_window_end), cursor.active_memory_records.as_ref(), params.max_records, @@ -714,6 +717,7 @@ fn imported_chatgpt_candidate_records( params.workspace, params.repo_id, source_window_start, + cursor.lower_bound_exclusive, Some(source_window_end), cursor.visible_turns.as_ref(), params.max_records, @@ -912,6 +916,7 @@ pub fn scheduler_watermark_after( if now_is_after_frontier { let next = DreamSchedulerCursor { lower_bound: Some(frontier), + lower_bound_exclusive: true, upper_bound: Some(now.to_string()), ..DreamSchedulerCursor::default() }; @@ -995,6 +1000,7 @@ fn query_dream_rows( timestamp_column: &str, id_column: &str, start: Option<&str>, + start_exclusive: bool, end: Option<&str>, position: Option<&DreamSourceCursor>, limit: usize, @@ -1007,7 +1013,8 @@ where return Ok(Vec::new()); } if let Some(start) = start { - sql.push_str(&format!(" AND {timestamp_column} >= ?")); + let operator = if start_exclusive { ">" } else { ">=" }; + sql.push_str(&format!(" AND {timestamp_column} {operator} ?")); args.push(Box::new(start.to_string())); } if let Some(position) = position { @@ -1041,6 +1048,7 @@ fn query_visible_turns_with_frontier( workspace_id: &str, repo_id: Option<&str>, start: Option<&str>, + start_exclusive: bool, end: Option<&str>, position: Option<&DreamSourceCursor>, limit: usize, @@ -1065,6 +1073,7 @@ fn query_visible_turns_with_frontier( "t.created_at", "t.id", start, + start_exclusive, end, position, limit, @@ -1087,6 +1096,7 @@ fn query_conclusions_with_frontier( workspace_id: &str, repo_id: Option<&str>, start: Option<&str>, + start_exclusive: bool, end: Option<&str>, position: Option<&DreamSourceCursor>, limit: usize, @@ -1111,6 +1121,7 @@ fn query_conclusions_with_frontier( "created_at", "id", start, + start_exclusive, end, position, limit, @@ -1136,6 +1147,7 @@ fn query_checkpoints_with_frontier( workspace_id: &str, repo_id: Option<&str>, start: Option<&str>, + start_exclusive: bool, end: Option<&str>, position: Option<&DreamSourceCursor>, limit: usize, @@ -1161,6 +1173,7 @@ fn query_checkpoints_with_frontier( "created_at", "id", start, + start_exclusive, end, position, limit, @@ -1191,6 +1204,7 @@ fn query_memory_sources_with_frontier( profile_id: &str, workspace_id: &str, start: Option<&str>, + start_exclusive: bool, end: Option<&str>, position: Option<&DreamSourceCursor>, limit: usize, @@ -1211,6 +1225,7 @@ fn query_memory_sources_with_frontier( "ingested_at", "id", start, + start_exclusive, end, position, limit, @@ -1236,6 +1251,7 @@ fn query_memory_records_with_frontier( workspace_id: &str, repo_id: Option<&str>, start: Option<&str>, + start_exclusive: bool, end: Option<&str>, position: Option<&DreamSourceCursor>, limit: usize, @@ -1263,6 +1279,7 @@ fn query_memory_records_with_frontier( "updated_at", "id", start, + start_exclusive, end, position, limit, @@ -1297,6 +1314,8 @@ fn build_evidence_window( let cursor_end = scheduler_cursor .map(|_| Some(end)) .unwrap_or(params.source_window_end); + let cursor_start_exclusive = + scheduler_cursor.is_some_and(|cursor| cursor.lower_bound_exclusive); let visible_turns = if remaining == 0 { Vec::new() } else if let Some(cursor) = scheduler_cursor { @@ -1306,6 +1325,7 @@ fn build_evidence_window( params.workspace, params.repo_id, start, + cursor_start_exclusive, cursor_end, cursor.visible_turns.as_ref(), remaining, @@ -1333,6 +1353,7 @@ fn build_evidence_window( params.workspace, params.repo_id, start, + cursor_start_exclusive, cursor_end, cursor.conclusions.as_ref(), remaining, @@ -1345,7 +1366,7 @@ fn build_evidence_window( params.workspace, params.repo_id, start, - params.source_window_end, + cursor_end, remaining, )?; remaining = remaining.saturating_sub(records.len()); @@ -1360,7 +1381,8 @@ fn build_evidence_window( params.workspace, params.repo_id, start, - params.source_window_end, + cursor_start_exclusive, + cursor_end, cursor.checkpoints.as_ref(), remaining, )?; @@ -1386,6 +1408,7 @@ fn build_evidence_window( params.profile.as_str(), params.workspace, start, + cursor_start_exclusive, cursor_end, cursor.imported_memories.as_ref(), remaining, diff --git a/src/native_runtime.rs b/src/native_runtime.rs index 1e8a560..8728a75 100644 --- a/src/native_runtime.rs +++ b/src/native_runtime.rs @@ -344,6 +344,17 @@ pub fn init(opts: &RuntimeOptions, mode: InitMode) -> Result ensure_dir(&dir, &mut created, &mut reused)?; } + // Persist one stable database identity so a later `up` from another + // working directory cannot silently select a different relative path. + fs::create_dir_all( + opts.db + .parent() + .filter(|parent| !parent.as_os_str().is_empty()) + .unwrap_or_else(|| Path::new(".")), + ) + .map_err(Error::from)?; + opts.db = resolve_database_path(&opts.db)?; + let config_path = opts.home.join("config.toml"); write_or_update( &config_path, @@ -563,12 +574,14 @@ fn http_request(method: &str, url: &str, body: Option<&str>) -> Result { fn up_native(opts: &RuntimeOptions) -> Result { require_safe_bind(opts)?; + let mut opts = opts.clone(); fs::create_dir_all(opts.log_file.parent().unwrap_or_else(|| Path::new("."))) .map_err(Error::from)?; fs::create_dir_all(opts.db.parent().unwrap_or_else(|| Path::new("."))).map_err(Error::from)?; + opts.db = resolve_database_path(&opts.db)?; if running_pid(&opts.pid_file).is_none() { - ensure_bind_available(opts)?; + ensure_bind_available(&opts)?; let log = fs::OpenOptions::new() .create(true) .append(true) @@ -594,8 +607,8 @@ fn up_native(opts: &RuntimeOptions) -> Result { fs::write(&opts.pid_file, child.id().to_string()).map_err(Error::from)?; } - wait_for_ready(opts, &opts.db.to_string_lossy())?; - Ok(status(opts)) + wait_for_ready(&opts, &opts.db.to_string_lossy())?; + Ok(status(&opts)) } fn up_container(opts: &RuntimeOptions) -> Result { @@ -613,7 +626,12 @@ fn up_container(opts: &RuntimeOptions) -> Result { let (database_parent, container_db_path) = container_database_mount(opts)?; if container_exists(&runtime, &opts.container_name)? { - ensure_container_database_mount(&runtime, &opts.container_name, &database_parent)?; + ensure_container_database_mount( + &runtime, + &opts.container_name, + &database_parent, + &container_db_path, + )?; if !container_running(&runtime, &opts.container_name)? { ensure_bind_available(opts)?; } @@ -819,6 +837,7 @@ fn ensure_container_database_mount( runtime: &str, name: &str, expected_parent: &Path, + expected_database_path: &str, ) -> Result<()> { let template = r#"{{range .Mounts}}{{if eq .Destination "/data"}}{{.Source}}{{end}}{{end}}"#; let output = Command::new(runtime) @@ -850,9 +869,78 @@ fn ensure_container_database_mount( expected_parent.display() ))); } + + let template = r#"{{range .Config.Env}}{{println .}}{{end}}"#; + let output = Command::new(runtime) + .args(["container", "inspect", "--format", template, name]) + .output() + .map_err(Error::from)?; + if !output.status.success() { + return Err(Error::storage(format!( + "could not verify CODEX_MEMORYD_DB for managed container {name}: {}", + String::from_utf8_lossy(&output.stderr).trim() + ))); + } + let inspected_environment = String::from_utf8_lossy(&output.stdout); + let configured_databases = inspected_environment + .lines() + .filter_map(|line| line.strip_prefix("CODEX_MEMORYD_DB=")) + .collect::>(); + if configured_databases.as_slice() != [expected_database_path] { + let actual = match configured_databases.as_slice() { + [] => "missing".to_string(), + [only] => (*only).to_string(), + _ => "multiple values".to_string(), + }; + return Err(Error::storage(format!( + "managed container {name} has CODEX_MEMORYD_DB={actual}, but the configured database requires {expected_database_path}; refusing to start it because it would use the wrong database" + ))); + } Ok(()) } +fn resolve_database_path(database: &Path) -> Result { + let cwd = std::env::current_dir().map_err(Error::from)?; + resolve_database_path_from(database, &cwd) +} + +fn resolve_database_path_from(database: &Path, cwd: &Path) -> Result { + if database == Path::new(":memory:") { + return Ok(database.to_path_buf()); + } + let database = if database.is_absolute() { + database.to_path_buf() + } else { + cwd.join(database) + }; + let filename = database.file_name().ok_or_else(|| { + Error::invalid_request(format!( + "native database path has no filename: {}", + database.display() + )) + })?; + let parent = database + .parent() + .filter(|path| !path.as_os_str().is_empty()) + .unwrap_or(Path::new(".")); + let canonical_parent = fs::canonicalize(parent).map_err(|err| { + Error::invalid_request(format!( + "native database parent cannot be resolved safely: {} ({err})", + parent.display() + )) + })?; + match fs::canonicalize(&database) { + Ok(path) => Ok(path), + Err(err) if err.kind() == std::io::ErrorKind::NotFound => { + Ok(canonical_parent.join(filename)) + } + Err(err) => Err(Error::invalid_request(format!( + "native database path is not usable: {} ({err})", + database.display() + ))), + } +} + fn down_native(opts: &RuntimeOptions) -> Result { let Some(pid) = running_pid(&opts.pid_file) else { let _ = fs::remove_file(&opts.pid_file); @@ -1482,9 +1570,11 @@ fn parse_loopback_url(url: &str) -> Result { #[cfg(test)] mod managed_readiness_tests { - use super::managed_status_ready; + use super::{managed_status_ready, resolve_database_path_from}; use crate::store::STORAGE_SCHEMA_VERSION; use serde_json::json; + use std::fs; + use std::path::Path; fn local_status() -> serde_json::Value { json!({"data": { @@ -1687,6 +1777,37 @@ mod managed_readiness_tests { assert!(!ready(&local_status(), "other-profile", "synthetic-scope")); assert!(!ready(&local_status(), "personal", "other-workspace")); } + + #[test] + fn relative_database_identity_is_bound_to_the_starting_working_directory() { + let temp = tempfile::tempdir().expect("temp directory"); + let first_cwd = temp.path().join("first"); + let second_cwd = temp.path().join("second"); + fs::create_dir_all(first_cwd.join("data")).expect("first database directory"); + fs::create_dir_all(second_cwd.join("data")).expect("second database directory"); + + let configured = Path::new("data/memory.db"); + let first_path = resolve_database_path_from(configured, &first_cwd) + .expect("resolve first database identity"); + let second_path = resolve_database_path_from(configured, &second_cwd) + .expect("resolve second database identity"); + assert_ne!(first_path, second_path); + + let mut status = local_status(); + status["data"]["storage"]["path"] = json!(first_path.display().to_string()); + assert!(managed_status_ready( + &status, + "personal", + "synthetic-scope", + &first_path.display().to_string() + )); + assert!(!managed_status_ready( + &status, + "personal", + "synthetic-scope", + &second_path.display().to_string() + )); + } } #[cfg(all(test, target_os = "linux"))] @@ -1765,18 +1886,24 @@ mod container_readiness_regressions { invocation_log: &std::path::Path, state_file: &std::path::Path, mounted_data_source: &std::path::Path, + container_database_path: &str, + existing_database_path: Option<&str>, initial_existing_container: bool, ) { let mount_source_file = path.with_extension("mount-source"); + let database_env_file = path.with_extension("database-env"); fs::write( path, format!( - "#!/bin/sh\nif [ \"$1\" = \"--version\" ]; then exit 0; fi\nprintf '%s\\n' \"$*\" >> '{}';\nif [ \"$1\" = \"container\" ] && [ \"$2\" = \"inspect\" ]; then if [ -f '{}' ]; then if [ \"$3\" = \"--format\" ]; then case \"$4\" in *State.Running*) printf true ;; *Mounts*) cat '{}' ;; esac; fi; exit 0; fi; exit 1; fi\nif [ \"$1\" = \"run\" ]; then printf '%s' '{}' > '{}'; touch '{}'; exit 0; fi\nif [ \"$1\" = \"start\" ]; then exit 0; fi\nexit 1\n", + "#!/bin/sh\nif [ \"$1\" = \"--version\" ]; then exit 0; fi\nprintf '%s\\n' \"$*\" >> '{}';\nif [ \"$1\" = \"container\" ] && [ \"$2\" = \"inspect\" ]; then if [ -f '{}' ]; then if [ \"$3\" = \"--format\" ]; then case \"$4\" in *State.Running*) printf true ;; *Mounts*) cat '{}' ;; *Config.Env*) cat '{}' ;; esac; fi; exit 0; fi; exit 1; fi\nif [ \"$1\" = \"run\" ]; then printf '%s' '{}' > '{}'; printf 'CODEX_MEMORYD_DB=%s\\n' '{}' > '{}'; touch '{}'; exit 0; fi\nif [ \"$1\" = \"start\" ]; then exit 0; fi\nexit 1\n", invocation_log.display(), state_file.display(), mount_source_file.display(), + database_env_file.display(), mounted_data_source.display(), mount_source_file.display(), + container_database_path, + database_env_file.display(), state_file.display() ), ) @@ -1787,6 +1914,14 @@ mod container_readiness_regressions { mounted_data_source.as_os_str().to_string_lossy().as_bytes(), ) .unwrap(); + fs::write( + &database_env_file, + format!( + "CODEX_MEMORYD_DB={}\n", + existing_database_path.unwrap_or(container_database_path) + ), + ) + .unwrap(); fs::write(state_file, "existing").unwrap(); } let mut permissions = fs::metadata(path).unwrap().permissions(); @@ -1798,6 +1933,7 @@ mod container_readiness_regressions { status_path: &str, db_relative_path: &str, existing_mount_relative_path: Option<&str>, + existing_database_path: Option<&str>, ) -> ( crate::error::Result, String, @@ -1809,6 +1945,8 @@ mod container_readiness_regressions { let state_file = temp.path().join("container-created"); let db = temp.path().join(db_relative_path); let database_parent = db.parent().unwrap().to_string_lossy().into_owned(); + let container_database_path = + format!("/data/{}", db.file_name().unwrap().to_string_lossy()); let mounted_source = existing_mount_relative_path .map(|path| temp.path().join(path)) .unwrap_or_else(|| PathBuf::from(&database_parent)); @@ -1820,6 +1958,8 @@ mod container_readiness_regressions { &invocation_log, &state_file, &mounted_source, + &container_database_path, + existing_database_path, existing_mount_relative_path.is_some(), ); let (url, stop, server) = status_server(container_status(status_path)); @@ -1858,7 +1998,7 @@ mod container_readiness_regressions { #[test] fn up_accepts_healthy_container_status_at_container_database_path() { let (result, invocations, database_parent) = - run_managed_container_up("/data/memory.db", "runtime-home/memory.db", None); + run_managed_container_up("/data/memory.db", "runtime-home/memory.db", None, None); assert!( result.is_ok(), "managed container startup failed: {result:?}" @@ -1871,7 +2011,7 @@ mod container_readiness_regressions { #[test] fn up_rejects_container_status_for_wrong_database_path() { let (result, _, _) = - run_managed_container_up("/data/wrong.db", "runtime-home/memory.db", None); + run_managed_container_up("/data/wrong.db", "runtime-home/memory.db", None, None); let error = result.expect_err("wrong managed-container storage path must fail readiness"); assert!(error .message @@ -1880,8 +2020,12 @@ mod container_readiness_regressions { #[test] fn up_mounts_and_verifies_the_configured_custom_database() { - let (result, invocations, database_parent) = - run_managed_container_up("/data/custom.sqlite", "custom-store/custom.sqlite", None); + let (result, invocations, database_parent) = run_managed_container_up( + "/data/custom.sqlite", + "custom-store/custom.sqlite", + None, + None, + ); assert!( result.is_ok(), "managed container startup failed for configured database: {result:?}" @@ -1892,8 +2036,12 @@ mod container_readiness_regressions { #[test] fn up_rejects_an_existing_container_with_a_stale_database_mount() { - let (result, invocations, database_parent) = - run_managed_container_up("/data/memory.db", "new-store/memory.db", Some("old-store")); + let (result, invocations, database_parent) = run_managed_container_up( + "/data/memory.db", + "new-store/memory.db", + Some("old-store"), + None, + ); let error = result.expect_err("old container must not start with a different DB mount"); assert!(error.message.contains("mounted from"), "{error:?}"); assert!(error.message.contains(&database_parent), "{error:?}"); @@ -1907,6 +2055,7 @@ mod container_readiness_regressions { "/data/memory.db", "runtime-home/memory.db", Some("runtime-home"), + None, ); assert!( result.is_ok(), @@ -1916,4 +2065,20 @@ mod container_readiness_regressions { assert!(invocations.contains("start synthetic-memoryd")); assert!(!invocations.contains("run -d")); } + + #[test] + fn up_rejects_an_existing_container_with_a_stale_database_filename() { + let (result, invocations, _) = run_managed_container_up( + "/data/old.db", + "runtime-home/new.db", + Some("runtime-home"), + Some("/data/old.db"), + ); + let error = result.expect_err("existing container must not start with a stale DB filename"); + assert!(error.message.contains("CODEX_MEMORYD_DB"), "{error:?}"); + assert!(error.message.contains("/data/new.db"), "{error:?}"); + assert!(error.message.contains("/data/old.db"), "{error:?}"); + assert!(!invocations.contains("start synthetic-memoryd")); + assert!(!invocations.contains("run -d")); + } } diff --git a/tests/scheduler_review_boundaries.rs b/tests/scheduler_review_boundaries.rs index 5c5d212..d6790d7 100644 --- a/tests/scheduler_review_boundaries.rs +++ b/tests/scheduler_review_boundaries.rs @@ -352,6 +352,110 @@ fn scheduler_cursor_keeps_new_evidence_eligible_while_draining() { ); } +#[test] +fn scheduler_cursor_applies_fixed_frontier_to_checkpoint_queries() { + let store = Store::open(":memory:").expect("store"); + let service = Service::new(store.clone(), scheduler_config("workspace", 1, false)); + service + .scheduled_dream(Some(BEFORE_FRONTIER.into())) + .expect("seed watermark"); + store + .ensure_workspace(PROFILE, "workspace") + .expect("workspace"); + insert_chatgpt_turn( + &store, + "checkpoint-frontier-session", + "visible-before-frontier", + "generic evidence before the checkpoint frontier", + "2026-09-10T00:00:00Z", + None, + ); + + let first = service + .scheduled_dream(Some(RUN_NOW.into())) + .expect("first bounded page"); + assert!( + first.limits_hit.contains(&"max_input_records".to_string()), + "first page should retain a bounded cursor" + ); + assert!( + evidence_ids(first.run.as_ref().expect("first run")).contains("visible-before-frontier") + ); + + store + .insert_checkpoint(&Checkpoint { + id: "checkpoint-arrived-after-frontier".into(), + session_id: None, + profile_id: PROFILE.into(), + workspace_id: "workspace".into(), + repo_id: None, + summary: "Synthetic checkpoint beyond the saved scan frontier".into(), + changed_files: vec![], + decisions: vec![], + blockers: vec![], + next_steps: vec![], + tests_run: vec![], + tests_not_run: vec![], + branch: None, + commit: None, + created_at: "2030-01-01T00:00:01Z".into(), + }) + .expect("late checkpoint"); + + let drain = service + .scheduled_dream(Some("2030-01-01T00:00:02Z".into())) + .expect("drain the original bounded window"); + let drain_evidence = drain.run.as_ref().map(evidence_ids).unwrap_or_default(); + assert!( + !drain_evidence.contains("checkpoint-arrived-after-frontier"), + "a checkpoint after the saved upper frontier must not enter the old scan" + ); + + let next_band = service + .scheduled_dream(Some("2030-01-01T00:00:03Z".into())) + .expect("scan the later bounded interval"); + let next_evidence = evidence_ids(next_band.run.as_ref().expect("next band run")); + assert!(next_evidence.contains("checkpoint-arrived-after-frontier")); +} + +#[test] +fn scheduler_cursor_does_not_reselect_the_inclusive_frontier_boundary() { + let store = Store::open(":memory:").expect("store"); + let service = Service::new(store.clone(), scheduler_config("workspace", 1, false)); + service + .scheduled_dream(Some(BEFORE_FRONTIER.into())) + .expect("seed watermark"); + store + .ensure_workspace(PROFILE, "workspace") + .expect("workspace"); + insert_chatgpt_turn( + &store, + "boundary-session", + "visible-at-original-frontier", + "generic evidence exactly at the original upper frontier", + RUN_NOW, + None, + ); + + let first = service + .scheduled_dream(Some(RUN_NOW.into())) + .expect("scan inclusive upper frontier"); + assert!(evidence_ids(first.run.as_ref().expect("first run")) + .contains("visible-at-original-frontier")); + + service + .scheduled_dream(Some("2030-01-01T00:00:01Z".into())) + .expect("finish the original band"); + let next_band = service + .scheduled_dream(Some("2030-01-01T00:00:02Z".into())) + .expect("scan the next band"); + let next_evidence = next_band.run.as_ref().map(evidence_ids).unwrap_or_default(); + assert!( + !next_evidence.contains("visible-at-original-frontier"), + "the inclusive end of the prior band must be exclusive at the next band's start" + ); +} + #[test] fn synthetic_repository_evidence_resolves_trusted_session_boundary() { let store = Store::open(":memory:").expect("store"); diff --git a/tests/test_child_reaper.py b/tests/test_child_reaper.py index a0e69ba..c6141f8 100644 --- a/tests/test_child_reaper.py +++ b/tests/test_child_reaper.py @@ -8,6 +8,7 @@ import tempfile import time import unittest +from unittest import mock LAUNCHER = Path(__file__).resolve().parents[1] / "scripts" / "run-with-child-reaper.py" SPEC = importlib.util.spec_from_file_location("memoryd_child_reaper", LAUNCHER) @@ -48,6 +49,34 @@ def test_proc_status_fallback_finds_direct_child(self): process.terminate() process.wait(timeout=5) + def test_proc_status_fallback_skips_unreadable_status_entries(self): + previous = CHILD_REAPER._CHILDREN_PATH_USABLE + CHILD_REAPER._CHILDREN_PATH_USABLE = False + denied_status = Path("/proc/41001/status") + child_status = Path("/proc/41002/status") + original_iterdir = Path.iterdir + original_read_text = Path.read_text + + def fake_iterdir(path): + if path == Path("/proc"): + return iter((Path("/proc/41001"), Path("/proc/41002"))) + return original_iterdir(path) + + def fake_read_text(path, *args, **kwargs): + if path == denied_status: + raise PermissionError("synthetic restricted procfs entry") + if path == child_status: + return f"Name:\towned-child\nPPid:\t{os.getpid()}\n" + return original_read_text(path, *args, **kwargs) + + try: + with mock.patch.object(Path, "iterdir", fake_iterdir), mock.patch.object( + Path, "read_text", fake_read_text + ): + self.assertEqual(CHILD_REAPER.adopted_children(-1), [41002]) + finally: + CHILD_REAPER._CHILDREN_PATH_USABLE = previous + def test_success_preserves_output_and_exit(self): with tempfile.TemporaryDirectory() as name: directory = Path(name) From 26713beef487ff46f730fdffd3607c5b9af7aaef Mon Sep 17 00:00:00 2001 From: Josh Yorko Date: Wed, 14 Oct 2026 17:28:31 +0000 Subject: [PATCH 11/18] fix: preserve late scheduler frontier evidence --- src/cli.rs | 75 ++++- src/dream.rs | 427 ++++++++++++++++++++++++--- src/service.rs | 17 +- tests/cli_smoke.rs | 67 +++++ tests/scheduler_review_boundaries.rs | 138 ++++++++- 5 files changed, 669 insertions(+), 55 deletions(-) diff --git a/src/cli.rs b/src/cli.rs index 49f245c..2e30143 100644 --- a/src/cli.rs +++ b/src/cli.rs @@ -2750,37 +2750,76 @@ fn container_mcp_database(runtime: &RuntimeOptions) -> Result<(String, String)> )); } - let parent = database.parent().ok_or_else(|| { + let configured_parent = database.parent().ok_or_else(|| { error::Error::invalid_request(format!( "container MCP database path has no usable parent directory: {}; set CODEX_MEMORYD_DB=/path/to/memory.db", database.display() )) })?; - let metadata = std::fs::metadata(parent).map_err(|err| { + let metadata = std::fs::metadata(configured_parent).map_err(|err| { if err.kind() == std::io::ErrorKind::NotFound { error::Error::invalid_request(format!( "container MCP database parent directory does not exist: {}; create it with mkdir -p {} or set CODEX_MEMORYD_DB to an existing directory", - parent.display(), - parent.display() + configured_parent.display(), + configured_parent.display() )) } else { error::Error::invalid_request(format!( "container MCP database parent directory is not usable: {} ({err}); check permissions or set CODEX_MEMORYD_DB to an accessible directory", - parent.display() + configured_parent.display() )) } })?; if !metadata.is_dir() { return Err(error::Error::invalid_request(format!( "container MCP database parent is not a directory: {}; set CODEX_MEMORYD_DB to a file inside an existing directory", - parent.display() + configured_parent.display() ))); } - let parent = std::fs::canonicalize(parent).map_err(|err| { + let configured_parent = std::fs::canonicalize(configured_parent).map_err(|err| { error::Error::invalid_request(format!( "container MCP database parent cannot be resolved safely: {} ({err}); set CODEX_MEMORYD_DB to an accessible absolute path", - parent.display() + configured_parent.display() + )) + })?; + let resolved_database = match std::fs::symlink_metadata(&database) { + Ok(_) => { + let resolved = std::fs::canonicalize(&database).map_err(|err| { + error::Error::invalid_request(format!( + "container MCP database cannot be resolved safely: {} ({err})", + database.display() + )) + })?; + if !std::fs::metadata(&resolved) + .map_err(error::Error::from)? + .is_file() + { + return Err(error::Error::invalid_request(format!( + "container MCP database path is not a file: {}", + database.display() + ))); + } + resolved + } + Err(err) if err.kind() == std::io::ErrorKind::NotFound => configured_parent.join(filename), + Err(err) => { + return Err(error::Error::invalid_request(format!( + "container MCP database path is not usable: {} ({err})", + database.display() + ))) + } + }; + let resolved_parent = resolved_database.parent().ok_or_else(|| { + error::Error::invalid_request(format!( + "container MCP database path has no usable parent directory: {}", + resolved_database.display() + )) + })?; + let parent = std::fs::canonicalize(resolved_parent).map_err(|err| { + error::Error::invalid_request(format!( + "container MCP database parent cannot be resolved safely: {} ({err}); set CODEX_MEMORYD_DB to an accessible absolute path", + resolved_parent.display() )) })?; if parent == Path::new("/") { @@ -2798,7 +2837,25 @@ fn container_mcp_database(runtime: &RuntimeOptions) -> Result<(String, String)> "container MCP database parent cannot be represented safely in a Docker/Podman --volume mount: {parent}; use a path without ':' or control characters, or set CODEX_MEMORYD_DB to a safe directory" ))); } - Ok((parent.to_string(), filename.to_string())) + let resolved_filename = resolved_database + .file_name() + .and_then(|name| name.to_str()) + .filter(|name| !name.is_empty() && *name != "." && *name != "..") + .ok_or_else(|| { + error::Error::invalid_request(format!( + "container MCP database path must name a UTF-8 file: {}; set CODEX_MEMORYD_DB=/path/to/memory.db", + resolved_database.display() + )) + })?; + if resolved_filename + .chars() + .any(|character| character.is_control() || character == ':') + { + return Err(error::Error::invalid_request( + "resolved container MCP database filename contains characters that cannot be represented safely; set CODEX_MEMORYD_DB to a normal filename", + )); + } + Ok((parent.to_string(), resolved_filename.to_string())) } fn toml_basic_string(value: &str, label: &str) -> Result { diff --git a/src/dream.rs b/src/dream.rs index 719c663..6b14685 100644 --- a/src/dream.rs +++ b/src/dream.rs @@ -53,6 +53,17 @@ const SCHEDULER_CURSOR_PREFIX: &str = "dream-cursor-v1:"; struct DreamSourceCursor { timestamp: String, id: String, + #[serde(default)] + rowid: Option, +} + +#[derive(Debug, Clone, Default, Serialize, Deserialize)] +struct DreamRowidFrontier { + visible_turns: i64, + conclusions: i64, + checkpoints: i64, + imported_memories: i64, + active_memory_records: i64, } #[derive(Debug, Clone, Default, Serialize, Deserialize)] @@ -64,6 +75,10 @@ struct DreamSchedulerCursor { #[serde(default)] upper_bound: Option, #[serde(default)] + lower_rowids: Option, + #[serde(default)] + upper_rowids: Option, + #[serde(default)] visible_turns: Option, #[serde(default)] conclusions: Option, @@ -91,6 +106,19 @@ impl DreamSchedulerCursor { self.imported_memories = None; self.active_memory_records = None; } + + fn has_legacy_position(&self) -> bool { + [ + self.visible_turns.as_ref(), + self.conclusions.as_ref(), + self.checkpoints.as_ref(), + self.imported_memories.as_ref(), + self.active_memory_records.as_ref(), + ] + .into_iter() + .flatten() + .any(|position| position.rowid.is_none()) + } } static RELATIVE_TIME: Lazy = Lazy::new(|| { @@ -311,6 +339,16 @@ pub fn run(store: &Store, params: &DreamParams) -> Result<(DreamResponse, bool)> if cursor.has_source_position() && cursor.upper_bound.is_none() { cursor.clear_source_positions(); } + // The old ID tie-break cannot be translated to SQLite insertion + // order. Rewind legacy pages before switching the keyset to rowid. + if cursor.has_legacy_position() { + cursor.clear_source_positions(); + cursor.lower_bound_exclusive = false; + cursor.lower_rowids = cursor + .lower_bound + .as_ref() + .map(|_| DreamRowidFrontier::default()); + } } let source_window_end = scheduler_cursor .as_ref() @@ -341,6 +379,14 @@ pub fn run(store: &Store, params: &DreamParams) -> Result<(DreamResponse, bool)> source_window_start, cursor.lower_bound_exclusive, Some(source_window_end), + cursor + .lower_rowids + .as_ref() + .map(|rowids| rowids.active_memory_records), + cursor + .upper_rowids + .as_ref() + .map(|rowids| rowids.active_memory_records), cursor.active_memory_records.as_ref(), params.max_records, params.include_archived_sources, @@ -463,7 +509,11 @@ pub fn run(store: &Store, params: &DreamParams) -> Result<(DreamResponse, bool)> } } - if params.recency_cutoff.is_none() || params.include_archived_sources { + let has_recency_lower_bound = scheduler_cursor + .as_ref() + .map(|cursor| cursor.lower_bound.is_some()) + .unwrap_or(params.recency_cutoff.is_some()); + if !has_recency_lower_bound || params.include_archived_sources { push_threshold_candidates(params, &records, &mut candidates, &mut rejected)?; } @@ -719,6 +769,14 @@ fn imported_chatgpt_candidate_records( source_window_start, cursor.lower_bound_exclusive, Some(source_window_end), + cursor + .lower_rowids + .as_ref() + .map(|rowids| rowids.visible_turns), + cursor + .upper_rowids + .as_ref() + .map(|rowids| rowids.visible_turns), cursor.visible_turns.as_ref(), params.max_records, )?, @@ -884,10 +942,30 @@ fn scheduler_cursor_from_watermark(previous: Option<&str>) -> DreamSchedulerCurs }) } +/// Pin a scheduled run's source bounds before selecting evidence. This keeps +/// the first page and every resumed page on the same rowid ordering/frontier. +pub fn scheduler_cursor_for_run( + store: &Store, + previous: Option<&str>, + now: &str, +) -> Result { + if let Some(previous) = previous.filter(|value| is_scheduler_cursor(value)) { + return Ok(previous.to_string()); + } + let cursor = DreamSchedulerCursor { + lower_bound: previous.map(str::to_string), + upper_bound: Some(now.to_string()), + upper_rowids: Some(scheduler_rowid_frontier(store)?), + ..DreamSchedulerCursor::default() + }; + encode_scheduler_cursor(&cursor) +} + /// Return a bounded scheduler cursor for a window that did not cover the /// entire input frontier. The cursor keeps one keyset position per source /// stream because the streams are budgeted in order and can advance unevenly. pub fn scheduler_watermark_after( + store: &Store, previous: Option<&str>, window: &DreamEvidenceWindow, now: &str, @@ -900,45 +978,178 @@ pub fn scheduler_watermark_after( if cursor.upper_bound.is_none() { cursor.clear_source_positions(); cursor.upper_bound = Some(window.end.clone()); + cursor.upper_rowids = Some(scheduler_rowid_frontier(store)?); + // The first bounded page used the ordinary ID ordering. Start the + // persisted cursor at the beginning so the remaining pages use + // the stable rowid ordering consistently. + return Ok(Some(encode_scheduler_cursor(&cursor)?)); + } + if cursor.upper_rowids.is_none() || cursor.has_legacy_position() { + cursor.clear_source_positions(); + cursor.upper_rowids = Some(scheduler_rowid_frontier(store)?); + if cursor.lower_bound.is_some() { + cursor.lower_bound_exclusive = false; + cursor.lower_rowids = Some(DreamRowidFrontier::default()); + } + return Ok(Some(encode_scheduler_cursor(&cursor)?)); } - update_scheduler_cursor(&mut cursor, window); - let encoded = serde_json::to_string(&cursor)?; - return Ok(Some(format!("{SCHEDULER_CURSOR_PREFIX}{encoded}"))); + update_scheduler_cursor(store, &mut cursor, window)?; + return Ok(Some(encode_scheduler_cursor(&cursor)?)); } if let Some(mut previous_cursor) = previous.and_then(parse_scheduler_cursor) { - if previous_cursor.has_source_position() { - if let Some(frontier) = previous_cursor.upper_bound.take() { - let now_is_after_frontier = OffsetDateTime::parse(now, &Rfc3339) - .ok() - .zip(OffsetDateTime::parse(&frontier, &Rfc3339).ok()) - .is_some_and(|(now, frontier)| now > frontier); - if now_is_after_frontier { - let next = DreamSchedulerCursor { - lower_bound: Some(frontier), - lower_bound_exclusive: true, - upper_bound: Some(now.to_string()), - ..DreamSchedulerCursor::default() - }; - let encoded = serde_json::to_string(&next)?; - return Ok(Some(format!("{SCHEDULER_CURSOR_PREFIX}{encoded}"))); - } - return Ok(Some(frontier)); + if previous_cursor.upper_bound.is_some() + && (previous_cursor.upper_rowids.is_none() || previous_cursor.has_legacy_position()) + { + previous_cursor.clear_source_positions(); + previous_cursor.upper_rowids = Some(scheduler_rowid_frontier(store)?); + if previous_cursor.lower_bound.is_some() { + previous_cursor.lower_bound_exclusive = false; + previous_cursor.lower_rowids = Some(DreamRowidFrontier::default()); } + return Ok(Some(encode_scheduler_cursor(&previous_cursor)?)); + } + if let Some(frontier) = previous_cursor.upper_bound.take() { + let now_is_after_frontier = OffsetDateTime::parse(now, &Rfc3339) + .ok() + .zip(OffsetDateTime::parse(&frontier, &Rfc3339).ok()) + .is_some_and(|(now, frontier)| now > frontier); + if now_is_after_frontier { + let next = DreamSchedulerCursor { + lower_bound: Some(frontier), + lower_bound_exclusive: false, + lower_rowids: previous_cursor.upper_rowids, + upper_bound: Some(now.to_string()), + upper_rowids: Some(scheduler_rowid_frontier(store)?), + ..DreamSchedulerCursor::default() + }; + return Ok(Some(encode_scheduler_cursor(&next)?)); + } + return Ok(Some(frontier)); } } Ok(Some(now.to_string())) } -fn update_scheduler_cursor(cursor: &mut DreamSchedulerCursor, window: &DreamEvidenceWindow) { - update_stream_cursor(&mut cursor.visible_turns, &window.visible_turns, false); - update_stream_cursor(&mut cursor.conclusions, &window.conclusions, false); - update_stream_cursor(&mut cursor.checkpoints, &window.checkpoints, false); +fn encode_scheduler_cursor(cursor: &DreamSchedulerCursor) -> Result { + Ok(format!( + "{SCHEDULER_CURSOR_PREFIX}{}", + serde_json::to_string(cursor)? + )) +} + +fn scheduler_rowid_frontier(store: &Store) -> Result { + store.transaction(|tx| { + Ok(DreamRowidFrontier { + visible_turns: tx.query_row( + "SELECT COALESCE(MAX(rowid), 0) FROM visible_turns", + [], + |row| row.get(0), + )?, + conclusions: tx.query_row( + "SELECT COALESCE(MAX(rowid), 0) FROM conclusions", + [], + |row| row.get(0), + )?, + checkpoints: tx.query_row( + "SELECT COALESCE(MAX(rowid), 0) FROM checkpoints", + [], + |row| row.get(0), + )?, + imported_memories: tx.query_row( + "SELECT COALESCE(MAX(rowid), 0) FROM memory_sources", + [], + |row| row.get(0), + )?, + active_memory_records: tx.query_row( + "SELECT COALESCE(MAX(rowid), 0) FROM memory_records", + [], + |row| row.get(0), + )?, + }) + }) +} + +fn update_scheduler_cursor( + store: &Store, + cursor: &mut DreamSchedulerCursor, + window: &DreamEvidenceWindow, +) -> Result<()> { + let mut visible_turn_ids = window + .visible_turns + .sources + .iter() + .map(|source| source.id.clone()) + .collect::>(); + for source in &window.active_memory_records.sources { + if source.kind == "imported_chat_turn" { + visible_turn_ids.extend(source.root_ids.iter().cloned()); + } + } + let visible_rowids = rowids_for_ids(store, "visible_turns", &visible_turn_ids)?; + let conclusion_rowids = rowids_for_ids( + store, + "conclusions", + &window + .conclusions + .sources + .iter() + .map(|source| source.id.clone()) + .collect(), + )?; + let checkpoint_rowids = rowids_for_ids( + store, + "checkpoints", + &window + .checkpoints + .sources + .iter() + .map(|source| source.id.clone()) + .collect(), + )?; + let source_rowids = rowids_for_ids( + store, + "memory_sources", + &window + .imported_memories + .sources + .iter() + .map(|source| source.id.clone()) + .collect(), + )?; + let active_record_ids = window + .active_memory_records + .sources + .iter() + .filter(|source| source.kind != "imported_chat_turn") + .map(|source| source.id.clone()) + .collect(); + let record_rowids = rowids_for_ids(store, "memory_records", &active_record_ids)?; + + update_stream_cursor( + &mut cursor.visible_turns, + &window.visible_turns, + false, + &visible_rowids, + ); + update_stream_cursor( + &mut cursor.conclusions, + &window.conclusions, + false, + &conclusion_rowids, + ); + update_stream_cursor( + &mut cursor.checkpoints, + &window.checkpoints, + false, + &checkpoint_rowids, + ); update_stream_cursor( &mut cursor.imported_memories, &window.imported_memories, true, + &source_rowids, ); for source in &window.active_memory_records.sources { if source.kind == "imported_chat_turn" && !source.root_ids.is_empty() { @@ -947,6 +1158,7 @@ fn update_scheduler_cursor(cursor: &mut DreamSchedulerCursor, window: &DreamEvid &mut cursor.visible_turns, source.created_at.clone(), root_id.clone(), + visible_rowids.get(root_id).copied(), ); } } else { @@ -957,15 +1169,18 @@ fn update_scheduler_cursor(cursor: &mut DreamSchedulerCursor, window: &DreamEvid .clone() .unwrap_or_else(|| source.created_at.clone()), source.id.clone(), + record_rowids.get(&source.id).copied(), ); } } + Ok(()) } fn update_stream_cursor( slot: &mut Option, stream: &DreamEvidenceStream, use_updated_at: bool, + rowids: &BTreeMap, ) { for source in &stream.sources { update_cursor_position( @@ -979,29 +1194,77 @@ fn update_stream_cursor( source.created_at.clone() }, source.id.clone(), + rowids.get(&source.id).copied(), ); } } -fn update_cursor_position(slot: &mut Option, timestamp: String, id: String) { - let candidate = DreamSourceCursor { timestamp, id }; +fn update_cursor_position( + slot: &mut Option, + timestamp: String, + id: String, + rowid: Option, +) { + let candidate = DreamSourceCursor { + timestamp, + id, + rowid, + }; if slot.as_ref().is_none_or(|current| { candidate.timestamp < current.timestamp - || (candidate.timestamp == current.timestamp && candidate.id > current.id) + || (candidate.timestamp == current.timestamp + && match (candidate.rowid, current.rowid) { + (Some(candidate), Some(current)) => candidate > current, + _ => candidate.id > current.id, + }) }) { *slot = Some(candidate); } } +fn rowids_for_ids( + store: &Store, + table: &str, + ids: &BTreeSet, +) -> Result> { + if ids.is_empty() { + return Ok(BTreeMap::new()); + } + store.transaction(|tx| { + let mut rowids = BTreeMap::new(); + for chunk in ids.iter().collect::>().chunks(900) { + let placeholders = vec!["?"; chunk.len()].join(", "); + let sql = format!("SELECT id, rowid FROM {table} WHERE id IN ({placeholders})"); + let args: Vec> = chunk + .iter() + .map(|id| Box::new((*id).clone()) as Box) + .collect(); + let args_ref = args.iter().map(|arg| arg.as_ref()).collect::>(); + let mut statement = tx.prepare(&sql)?; + let rows = statement.query_map(args_ref.as_slice(), |row| { + Ok((row.get::<_, String>(0)?, row.get::<_, i64>(1)?)) + })?; + for row in rows { + let (id, rowid) = row?; + rowids.insert(id, rowid); + } + } + Ok(rowids) + }) +} + fn query_dream_rows( store: &Store, mut sql: String, mut args: Vec>, timestamp_column: &str, id_column: &str, + rowid_column: &str, start: Option<&str>, start_exclusive: bool, + lower_rowid: Option, end: Option<&str>, + upper_rowid: Option, position: Option<&DreamSourceCursor>, limit: usize, mapper: F, @@ -1013,25 +1276,52 @@ where return Ok(Vec::new()); } if let Some(start) = start { - let operator = if start_exclusive { ">" } else { ">=" }; - sql.push_str(&format!(" AND {timestamp_column} {operator} ?")); - args.push(Box::new(start.to_string())); + if let Some(lower_rowid) = lower_rowid { + sql.push_str(&format!( + " AND ({timestamp_column} > ? OR ({timestamp_column} = ? AND {rowid_column} > ?))" + )); + args.push(Box::new(start.to_string())); + args.push(Box::new(start.to_string())); + args.push(Box::new(lower_rowid)); + } else { + let operator = if start_exclusive { ">" } else { ">=" }; + sql.push_str(&format!(" AND {timestamp_column} {operator} ?")); + args.push(Box::new(start.to_string())); + } } if let Some(position) = position { - sql.push_str(&format!( - " AND ({timestamp_column} < ? OR ({timestamp_column} = ? AND {id_column} > ?))" - )); - args.push(Box::new(position.timestamp.clone())); - args.push(Box::new(position.timestamp.clone())); - args.push(Box::new(position.id.clone())); + if let Some(rowid) = position.rowid { + sql.push_str(&format!( + " AND ({timestamp_column} < ? OR ({timestamp_column} = ? AND {rowid_column} > ?))" + )); + args.push(Box::new(position.timestamp.clone())); + args.push(Box::new(position.timestamp.clone())); + args.push(Box::new(rowid)); + } else { + sql.push_str(&format!( + " AND ({timestamp_column} < ? OR ({timestamp_column} = ? AND {id_column} > ?))" + )); + args.push(Box::new(position.timestamp.clone())); + args.push(Box::new(position.timestamp.clone())); + args.push(Box::new(position.id.clone())); + } } if let Some(end) = end { - sql.push_str(&format!(" AND {timestamp_column} <= ?")); - args.push(Box::new(end.to_string())); + if let Some(upper_rowid) = upper_rowid { + sql.push_str(&format!( + " AND ({timestamp_column} < ? OR ({timestamp_column} = ? AND {rowid_column} <= ?))" + )); + args.push(Box::new(end.to_string())); + args.push(Box::new(end.to_string())); + args.push(Box::new(upper_rowid)); + } else { + sql.push_str(&format!(" AND {timestamp_column} <= ?")); + args.push(Box::new(end.to_string())); + } } let params_ref: Vec<&dyn ToSql> = args.iter().map(|arg| arg.as_ref()).collect(); sql.push_str(&format!( - " ORDER BY {timestamp_column} DESC, {id_column} ASC LIMIT {limit}" + " ORDER BY {timestamp_column} DESC, {rowid_column} ASC LIMIT {limit}" )); store.transaction(|tx| { let mut statement = tx.prepare(&sql)?; @@ -1050,6 +1340,8 @@ fn query_visible_turns_with_frontier( start: Option<&str>, start_exclusive: bool, end: Option<&str>, + lower_rowid: Option, + upper_rowid: Option, position: Option<&DreamSourceCursor>, limit: usize, ) -> Result> { @@ -1072,9 +1364,12 @@ fn query_visible_turns_with_frontier( args, "t.created_at", "t.id", + "t.rowid", start, start_exclusive, + lower_rowid, end, + upper_rowid, position, limit, |row| { @@ -1098,6 +1393,8 @@ fn query_conclusions_with_frontier( start: Option<&str>, start_exclusive: bool, end: Option<&str>, + lower_rowid: Option, + upper_rowid: Option, position: Option<&DreamSourceCursor>, limit: usize, ) -> Result> { @@ -1120,9 +1417,12 @@ fn query_conclusions_with_frontier( args, "created_at", "id", + "rowid", start, start_exclusive, + lower_rowid, end, + upper_rowid, position, limit, |row| { @@ -1149,6 +1449,8 @@ fn query_checkpoints_with_frontier( start: Option<&str>, start_exclusive: bool, end: Option<&str>, + lower_rowid: Option, + upper_rowid: Option, position: Option<&DreamSourceCursor>, limit: usize, ) -> Result> { @@ -1172,9 +1474,12 @@ fn query_checkpoints_with_frontier( args, "created_at", "id", + "rowid", start, start_exclusive, + lower_rowid, end, + upper_rowid, position, limit, |row| { @@ -1206,6 +1511,8 @@ fn query_memory_sources_with_frontier( start: Option<&str>, start_exclusive: bool, end: Option<&str>, + lower_rowid: Option, + upper_rowid: Option, position: Option<&DreamSourceCursor>, limit: usize, ) -> Result> { @@ -1224,9 +1531,12 @@ fn query_memory_sources_with_frontier( args, "ingested_at", "id", + "rowid", start, start_exclusive, + lower_rowid, end, + upper_rowid, position, limit, |row| { @@ -1253,6 +1563,8 @@ fn query_memory_records_with_frontier( start: Option<&str>, start_exclusive: bool, end: Option<&str>, + lower_rowid: Option, + upper_rowid: Option, position: Option<&DreamSourceCursor>, limit: usize, include_archived: bool, @@ -1278,9 +1590,12 @@ fn query_memory_records_with_frontier( args, "updated_at", "id", + "rowid", start, start_exclusive, + lower_rowid, end, + upper_rowid, position, limit, |row| row.get::<_, String>(0), @@ -1327,6 +1642,14 @@ fn build_evidence_window( start, cursor_start_exclusive, cursor_end, + cursor + .lower_rowids + .as_ref() + .map(|rowids| rowids.visible_turns), + cursor + .upper_rowids + .as_ref() + .map(|rowids| rowids.visible_turns), cursor.visible_turns.as_ref(), remaining, )?; @@ -1355,6 +1678,14 @@ fn build_evidence_window( start, cursor_start_exclusive, cursor_end, + cursor + .lower_rowids + .as_ref() + .map(|rowids| rowids.conclusions), + cursor + .upper_rowids + .as_ref() + .map(|rowids| rowids.conclusions), cursor.conclusions.as_ref(), remaining, )?; @@ -1383,6 +1714,14 @@ fn build_evidence_window( start, cursor_start_exclusive, cursor_end, + cursor + .lower_rowids + .as_ref() + .map(|rowids| rowids.checkpoints), + cursor + .upper_rowids + .as_ref() + .map(|rowids| rowids.checkpoints), cursor.checkpoints.as_ref(), remaining, )?; @@ -1410,6 +1749,14 @@ fn build_evidence_window( start, cursor_start_exclusive, cursor_end, + cursor + .lower_rowids + .as_ref() + .map(|rowids| rowids.imported_memories), + cursor + .upper_rowids + .as_ref() + .map(|rowids| rowids.imported_memories), cursor.imported_memories.as_ref(), remaining, )? diff --git a/src/service.rs b/src/service.rs index 70719c1..7eec377 100644 --- a/src/service.rs +++ b/src/service.rs @@ -2603,6 +2603,14 @@ impl Service { && self.config.dream_provider.enabled && DreamProviderAdapter::parse(&self.config.dream_provider.adapter) == Some(DreamProviderAdapter::Command); + let scan_start = if command_mode && watermark_before.is_none() { + self.store + .dream_watermark(profile.as_str(), &workspace, None)? + } else { + watermark_before.clone() + }; + let scheduler_cursor = + dream::scheduler_cursor_for_run(&self.store, scan_start.as_deref(), &now)?; let mut input_window_limited = false; let mut input_budget_limit = None; let result = if command_mode { @@ -2623,7 +2631,7 @@ impl Service { workspace: Some(workspace.clone()), repo: None, now: Some(now.clone()), - since: watermark_before.clone(), + since: Some(scheduler_cursor.clone()), since_explicit: false, kind: "dream_preview".to_string(), mode: Some("command".to_string()), @@ -2679,7 +2687,7 @@ impl Service { mode: "preview", now: &now, source_window_end: Some(&now), - recency_cutoff: watermark_before.as_deref(), + recency_cutoff: Some(&scheduler_cursor), include_archived_sources: false, max_records: 1, max_candidates: Some(cfg.max_candidates), @@ -2702,7 +2710,7 @@ impl Service { mode: "preview", now: &now, source_window_end: (!simulated_now).then_some(now.as_str()), - recency_cutoff: watermark_before.as_deref(), + recency_cutoff: Some(&scheduler_cursor), include_archived_sources: false, max_records: cfg.max_batch_size, max_candidates: (!cfg.automatic_apply).then_some(cfg.max_candidates), @@ -2804,7 +2812,8 @@ impl Service { }); let watermark_after = if input_only_limits { dream::scheduler_watermark_after( - watermark_before.as_deref(), + &self.store, + Some(&scheduler_cursor), &run.evidence_window, &now, input_window_full, diff --git a/tests/cli_smoke.rs b/tests/cli_smoke.rs index bc182ce..dfcdc62 100644 --- a/tests/cli_smoke.rs +++ b/tests/cli_smoke.rs @@ -6912,6 +6912,73 @@ fn cli_mcp_codex_container_global_runtime_placements_and_engines_are_determinist let _: toml::Value = toml::from_str(podman_output["snippet"].as_str().unwrap()).unwrap(); } +#[cfg(unix)] +#[test] +fn cli_mcp_codex_container_mounts_the_resolved_symlink_database_path() { + use std::os::unix::fs::symlink; + + let dir = TempDir::new().unwrap(); + let memoryd_home = dir.path().join("memoryd-home"); + let codex_home = codex_home_path(&dir); + let alias_dir = dir.path().join("database-alias"); + let target_dir = dir.path().join("database-target"); + fs::create_dir_all(&alias_dir).unwrap(); + fs::create_dir_all(&target_dir).unwrap(); + let target_db = target_dir.join("memory.db"); + fs::write(&target_db, b"synthetic disposable database fixture").unwrap(); + let alias_db = alias_dir.join("configured.db"); + symlink(&target_db, &alias_db).unwrap(); + let runtime = fake_container_runtime_named(dir.path(), "docker"); + + let mut command = bin(); + clear_runtime_env(&mut command); + let output = command + .env("CODEX_MEMORYD_HOME", &memoryd_home) + .env("CODEX_HOME", &codex_home) + .env("CODEX_MEMORYD_CONTAINER_RUNTIME", &runtime) + .env("CODEX_MEMORYD_IMAGE", "codex-memoryd:local") + .env("CODEX_MEMORYD_UID", "1001") + .env("CODEX_MEMORYD_GID", "1002") + .arg("--db") + .arg(&alias_db) + .args(["mcp", "codex", "preview", "--runtime", "container"]) + .assert() + .success() + .get_output() + .stdout + .clone(); + let preview: Value = serde_json::from_slice(&output).unwrap(); + assert_eq!( + preview["resolved"]["args"], + serde_json::json!([ + "run", + "--rm", + "-i", + "--pull=missing", + "--user", + "1001:1002", + "--volume", + format!("{}:/data", target_dir.canonicalize().unwrap().display()), + "--env", + "CODEX_MEMORYD_PROFILE=personal", + "--env", + "CODEX_MEMORYD_WORKSPACE=default", + "--env", + "CODEX_MEMORYD_LOG=info", + "codex-memoryd:local", + "--db", + "/data/memory.db", + "mcp", + "stdio", + "--read-only" + ]) + ); + assert!( + !codex_home.exists(), + "previewing a symlinked database path must not write Codex configuration" + ); +} + #[cfg(unix)] #[test] fn cli_mcp_codex_each_action_accepts_both_global_runtime_placements() { diff --git a/tests/scheduler_review_boundaries.rs b/tests/scheduler_review_boundaries.rs index d6790d7..73a57b2 100644 --- a/tests/scheduler_review_boundaries.rs +++ b/tests/scheduler_review_boundaries.rs @@ -7,7 +7,7 @@ use codex_memoryd::domain::{ use codex_memoryd::ids; use codex_memoryd::policy; use codex_memoryd::service::Service; -use codex_memoryd::store::{NewRecord, Store, UpsertOutcome}; +use codex_memoryd::store::{DreamRunRecord, NewRecord, Store, UpsertOutcome}; use serde_json::json; const PROFILE: &str = "personal"; @@ -398,7 +398,7 @@ fn scheduler_cursor_applies_fixed_frontier_to_checkpoint_queries() { tests_not_run: vec![], branch: None, commit: None, - created_at: "2030-01-01T00:00:01Z".into(), + created_at: RUN_NOW.into(), }) .expect("late checkpoint"); @@ -456,6 +456,140 @@ fn scheduler_cursor_does_not_reselect_the_inclusive_frontier_boundary() { ); } +#[test] +fn scheduler_cursor_picks_up_late_row_before_frontier_keyset_position() { + let store = Store::open(":memory:").expect("store"); + let service = Service::new(store.clone(), scheduler_config("workspace", 1, false)); + service + .scheduled_dream(Some(BEFORE_FRONTIER.into())) + .expect("seed watermark"); + store + .ensure_workspace(PROFILE, "workspace") + .expect("workspace"); + insert_chatgpt_turn( + &store, + "frontier-session", + "visible-z-frontier", + "generic evidence already at the saved upper frontier", + RUN_NOW, + None, + ); + insert_chatgpt_turn( + &store, + "older-session", + "visible-old-frontier-band", + "generic evidence below the saved upper frontier", + "2029-12-31T00:00:00Z", + None, + ); + + let first = service + .scheduled_dream(Some(RUN_NOW.into())) + .expect("scan first row at inclusive frontier"); + assert!(evidence_ids(first.run.as_ref().expect("first run")).contains("visible-z-frontier")); + + insert_chatgpt_turn( + &store, + "late-session", + "visible-a-late-frontier", + "generic evidence inserted late at the saved upper frontier", + RUN_NOW, + None, + ); + + let drain = service + .scheduled_dream(Some("2030-01-01T00:00:01Z".into())) + .expect("continue draining the saved band"); + let old_band = evidence_ids(drain.run.as_ref().expect("drain run")); + assert!( + old_band.contains("visible-old-frontier-band"), + "the old bounded band must continue after its rowid position" + ); + assert!( + !old_band.contains("visible-a-late-frontier"), + "a row inserted beyond the saved rowid frontier must wait for the next band" + ); + + service + .scheduled_dream(Some("2030-01-01T00:00:02Z".into())) + .expect("finish the original bounded band"); + let next_band = service + .scheduled_dream(Some("2030-01-01T00:00:03Z".into())) + .expect("scan the band after the saved frontier"); + assert!(evidence_ids(next_band.run.as_ref().expect("next band run")) + .contains("visible-a-late-frontier")); +} + +#[test] +fn scheduler_cursor_rewinds_persisted_id_only_cursor_before_rowid_paging() { + let store = Store::open(":memory:").expect("store"); + let service = Service::new(store.clone(), scheduler_config("workspace", 1, false)); + store + .ensure_workspace(PROFILE, "workspace") + .expect("workspace"); + store + .ensure_session( + "legacy-session", + PROFILE, + "workspace", + None, + None, + "fixture", + ) + .expect("session"); + for id in ["visible-z-legacy-position", "visible-a-legacy-unvisited"] { + store + .insert_visible_turn(&VisibleTurn { + id: id.into(), + session_id: "legacy-session".into(), + actor: "user".into(), + content: format!("generic evidence {id}"), + created_at: RUN_NOW.into(), + metadata: json!({}), + }) + .expect("visible turn"); + } + + let legacy_cursor = format!( + "dream-cursor-v1:{}", + json!({ + "lower_bound": BEFORE_FRONTIER, + "lower_bound_exclusive": false, + "upper_bound": RUN_NOW, + "visible_turns": { + "timestamp": RUN_NOW, + "id": "visible-z-legacy-position" + } + }) + ); + store + .record_dream_run(&DreamRunRecord { + run_id: "legacy-cursor-seed".into(), + profile_id: PROFILE.into(), + workspace_id: "workspace".into(), + mode: "preview".into(), + kind: "scheduled".into(), + status: "ok_with_limits".into(), + started_at: "2030-01-01T00:00:00Z".into(), + completed_at: Some("2030-01-01T00:00:00Z".into()), + watermark_after: Some(legacy_cursor), + ..Default::default() + }) + .expect("persist legacy cursor"); + + service + .scheduled_dream(Some("2030-01-01T00:00:01Z".into())) + .expect("migrate and rewind the legacy cursor"); + service + .scheduled_dream(Some("2030-01-01T00:00:02Z".into())) + .expect("restart the bounded scan with rowid ordering"); + let resumed = service + .scheduled_dream(Some("2030-01-01T00:00:03Z".into())) + .expect("continue beyond the legacy ID keyset position"); + assert!(evidence_ids(resumed.run.as_ref().expect("resumed run")) + .contains("visible-a-legacy-unvisited")); +} + #[test] fn synthetic_repository_evidence_resolves_trusted_session_boundary() { let store = Store::open(":memory:").expect("store"); From 4d9424f247f3b0c2683337da29644d970a8c17ed Mon Sep 17 00:00:00 2001 From: Josh Yorko Date: Wed, 14 Oct 2026 17:36:49 +0000 Subject: [PATCH 12/18] fix: preserve unbounded simulated scheduler frontier --- src/dream.rs | 57 ++++++++++++++++++++++++++++++++++++++++++-------- src/service.rs | 19 +++++++++++------ 2 files changed, 61 insertions(+), 15 deletions(-) diff --git a/src/dream.rs b/src/dream.rs index 6b14685..c9d0257 100644 --- a/src/dream.rs +++ b/src/dream.rs @@ -75,6 +75,8 @@ struct DreamSchedulerCursor { #[serde(default)] upper_bound: Option, #[serde(default)] + upper_unbounded: bool, + #[serde(default)] lower_rowids: Option, #[serde(default)] upper_rowids: Option, @@ -336,7 +338,7 @@ pub fn run(store: &Store, params: &DreamParams) -> Result<(DreamResponse, bool)> if let Some(cursor) = scheduler_cursor.as_mut() { // Cursors written before the fixed upper frontier was introduced can // only be resumed safely by rescanning their current bounded window. - if cursor.has_source_position() && cursor.upper_bound.is_none() { + if cursor.has_source_position() && cursor.upper_bound.is_none() && !cursor.upper_unbounded { cursor.clear_source_positions(); } // The old ID tie-break cannot be translated to SQLite insertion @@ -355,6 +357,11 @@ pub fn run(store: &Store, params: &DreamParams) -> Result<(DreamResponse, bool)> .and_then(|cursor| cursor.upper_bound.as_deref()) .or(params.source_window_end) .unwrap_or(params.now); + let cursor_end = match scheduler_cursor.as_ref() { + Some(cursor) if cursor.upper_unbounded => None, + Some(_) => Some(source_window_end), + None => params.source_window_end, + }; let source_window_start = match scheduler_cursor.as_ref() { Some(cursor) => cursor.lower_bound.as_deref(), None => params.recency_cutoff, @@ -378,7 +385,7 @@ pub fn run(store: &Store, params: &DreamParams) -> Result<(DreamResponse, bool)> params.repo_id, source_window_start, cursor.lower_bound_exclusive, - Some(source_window_end), + cursor_end, cursor .lower_rowids .as_ref() @@ -404,7 +411,7 @@ pub fn run(store: &Store, params: &DreamParams) -> Result<(DreamResponse, bool)> params, imported_limit, source_window_start, - source_window_end, + cursor_end, scheduler_cursor.as_ref(), )?); } @@ -757,7 +764,7 @@ fn imported_chatgpt_candidate_records( params: &DreamParams, limit: usize, source_window_start: Option<&str>, - source_window_end: &str, + source_window_end: Option<&str>, scheduler_cursor: Option<&DreamSchedulerCursor>, ) -> Result> { let turns = match scheduler_cursor { @@ -768,7 +775,7 @@ fn imported_chatgpt_candidate_records( params.repo_id, source_window_start, cursor.lower_bound_exclusive, - Some(source_window_end), + source_window_end, cursor .lower_rowids .as_ref() @@ -948,13 +955,15 @@ pub fn scheduler_cursor_for_run( store: &Store, previous: Option<&str>, now: &str, + upper_unbounded: bool, ) -> Result { if let Some(previous) = previous.filter(|value| is_scheduler_cursor(value)) { return Ok(previous.to_string()); } let cursor = DreamSchedulerCursor { lower_bound: previous.map(str::to_string), - upper_bound: Some(now.to_string()), + upper_bound: (!upper_unbounded).then(|| now.to_string()), + upper_unbounded, upper_rowids: Some(scheduler_rowid_frontier(store)?), ..DreamSchedulerCursor::default() }; @@ -975,6 +984,19 @@ pub fn scheduler_watermark_after( if window_full { // Keep the original scan frontier stable while each source stream is // drained. Rows arriving after it are scanned in a new bounded band. + if cursor.upper_unbounded { + if cursor.upper_rowids.is_none() || cursor.has_legacy_position() { + cursor.clear_source_positions(); + cursor.upper_rowids = Some(scheduler_rowid_frontier(store)?); + if cursor.lower_bound.is_some() { + cursor.lower_bound_exclusive = false; + cursor.lower_rowids = Some(DreamRowidFrontier::default()); + } + } else { + update_scheduler_cursor(store, &mut cursor, window)?; + } + return Ok(Some(encode_scheduler_cursor(&cursor)?)); + } if cursor.upper_bound.is_none() { cursor.clear_source_positions(); cursor.upper_bound = Some(window.end.clone()); @@ -998,6 +1020,18 @@ pub fn scheduler_watermark_after( } if let Some(mut previous_cursor) = previous.and_then(parse_scheduler_cursor) { + if previous_cursor.upper_unbounded { + if previous_cursor.upper_rowids.is_none() || previous_cursor.has_legacy_position() { + previous_cursor.clear_source_positions(); + previous_cursor.upper_rowids = Some(scheduler_rowid_frontier(store)?); + if previous_cursor.lower_bound.is_some() { + previous_cursor.lower_bound_exclusive = false; + previous_cursor.lower_rowids = Some(DreamRowidFrontier::default()); + } + return Ok(Some(encode_scheduler_cursor(&previous_cursor)?)); + } + return Ok(Some(now.to_string())); + } if previous_cursor.upper_bound.is_some() && (previous_cursor.upper_rowids.is_none() || previous_cursor.has_legacy_position()) { @@ -1318,6 +1352,9 @@ where sql.push_str(&format!(" AND {timestamp_column} <= ?")); args.push(Box::new(end.to_string())); } + } else if let Some(upper_rowid) = upper_rowid { + sql.push_str(&format!(" AND {rowid_column} <= ?")); + args.push(Box::new(upper_rowid)); } let params_ref: Vec<&dyn ToSql> = args.iter().map(|arg| arg.as_ref()).collect(); sql.push_str(&format!( @@ -1626,9 +1663,11 @@ fn build_evidence_window( active_records: &[MemoryRecord], ) -> Result { let mut remaining = params.max_records.saturating_sub(active_records.len()); - let cursor_end = scheduler_cursor - .map(|_| Some(end)) - .unwrap_or(params.source_window_end); + let cursor_end = match scheduler_cursor { + Some(cursor) if cursor.upper_unbounded => None, + Some(_) => Some(end), + None => params.source_window_end, + }; let cursor_start_exclusive = scheduler_cursor.is_some_and(|cursor| cursor.lower_bound_exclusive); let visible_turns = if remaining == 0 { diff --git a/src/service.rs b/src/service.rs index 7eec377..e07d1f6 100644 --- a/src/service.rs +++ b/src/service.rs @@ -2609,8 +2609,13 @@ impl Service { } else { watermark_before.clone() }; - let scheduler_cursor = - dream::scheduler_cursor_for_run(&self.store, scan_start.as_deref(), &now)?; + let scheduler_cursor = Some(dream::scheduler_cursor_for_run( + &self.store, + scan_start.as_deref(), + &now, + simulated_now && !command_mode && watermark_before.is_none(), + )?); + let scheduler_previous = scheduler_cursor.as_deref(); let mut input_window_limited = false; let mut input_budget_limit = None; let result = if command_mode { @@ -2631,7 +2636,7 @@ impl Service { workspace: Some(workspace.clone()), repo: None, now: Some(now.clone()), - since: Some(scheduler_cursor.clone()), + since: scheduler_cursor.clone(), since_explicit: false, kind: "dream_preview".to_string(), mode: Some("command".to_string()), @@ -2687,7 +2692,9 @@ impl Service { mode: "preview", now: &now, source_window_end: Some(&now), - recency_cutoff: Some(&scheduler_cursor), + recency_cutoff: scheduler_cursor + .as_deref() + .or(watermark_before.as_deref()), include_archived_sources: false, max_records: 1, max_candidates: Some(cfg.max_candidates), @@ -2710,7 +2717,7 @@ impl Service { mode: "preview", now: &now, source_window_end: (!simulated_now).then_some(now.as_str()), - recency_cutoff: Some(&scheduler_cursor), + recency_cutoff: scheduler_cursor.as_deref().or(watermark_before.as_deref()), include_archived_sources: false, max_records: cfg.max_batch_size, max_candidates: (!cfg.automatic_apply).then_some(cfg.max_candidates), @@ -2813,7 +2820,7 @@ impl Service { let watermark_after = if input_only_limits { dream::scheduler_watermark_after( &self.store, - Some(&scheduler_cursor), + scheduler_previous, &run.evidence_window, &now, input_window_full, From 01b24b2282327ae7a6cd495c13474c496180cb5f Mon Sep 17 00:00:00 2001 From: Josh Yorko Date: Wed, 14 Oct 2026 18:29:05 +0000 Subject: [PATCH 13/18] fix: preserve completed scheduler frontiers and status --- docs/dreamer-native-provider.md | 9 +- src/dream.rs | 198 +++++++++++++++++++++++++++--- src/service.rs | 47 ++++++- src/store.rs | 20 ++- tests/consolidation_acceptance.rs | 134 ++++++++++++++++++++ tests/dream_command_scheduler.rs | 12 ++ 6 files changed, 396 insertions(+), 24 deletions(-) diff --git a/docs/dreamer-native-provider.md b/docs/dreamer-native-provider.md index d5d2d33..e92b261 100644 --- a/docs/dreamer-native-provider.md +++ b/docs/dreamer-native-provider.md @@ -65,8 +65,13 @@ including its tail. If one selected record still cannot fit, the command is not dispatched. A deterministic preview processes that one record, records the input limit, and advances only that source-kind cursor so later evidence remains eligible. Other provider failures do not advance the successful -scheduled watermark. Process-group children are terminated; orphan reaping -remains the operating system's job. +scheduled watermark. A recovered deterministic fallback also records a later +status-visible `ok_with_limits` receipt while retaining the provider budget +rejection in audit history. Completed unbounded simulated-clock scans retain +their rowid frontier, so a later clock does not redispatch prior evidence and +new rows remain eligible when they meet the timestamp watermark, even when +they predate future-dated rows from the earlier scan. Process-group +children are terminated; orphan reaping remains the operating system's job. Managed readiness accepts `local_only`, or `degraded` only when the reported store is writable SQLite at the current schema, the exact configured diff --git a/src/dream.rs b/src/dream.rs index c9d0257..03e11e0 100644 --- a/src/dream.rs +++ b/src/dream.rs @@ -73,6 +73,8 @@ struct DreamSchedulerCursor { #[serde(default)] lower_bound_exclusive: bool, #[serde(default)] + lower_rowid_strict: bool, + #[serde(default)] upper_bound: Option, #[serde(default)] upper_unbounded: bool, @@ -385,6 +387,7 @@ pub fn run(store: &Store, params: &DreamParams) -> Result<(DreamResponse, bool)> params.repo_id, source_window_start, cursor.lower_bound_exclusive, + cursor.lower_rowid_strict, cursor_end, cursor .lower_rowids @@ -416,11 +419,61 @@ pub fn run(store: &Store, params: &DreamParams) -> Result<(DreamResponse, bool)> )?); } check_deadline(params)?; + // Previously processed active records remain the comparison state for a + // newly admitted record (for example, a correction must still supersede + // the current fact). Keep them out of the provider evidence window, which + // is limited to rows after the scheduler's captured source frontier. + let mut candidate_records = if scheduler_cursor + .as_ref() + .is_some_and(|cursor| cursor.lower_rowid_strict) + { + let context_limit = params.max_records.saturating_sub(records.len()); + let mut current_records = if context_limit > 0 { + query_memory_records_with_frontier( + store, + params.profile.as_str(), + params.workspace, + params.repo_id, + None, + false, + false, + cursor_end, + None, + scheduler_cursor.as_ref().and_then(|cursor| { + cursor + .upper_rowids + .as_ref() + .map(|rowids| rowids.active_memory_records) + }), + None, + context_limit, + params.include_archived_sources, + )? + } else { + Vec::new() + }; + for record in &records { + if !current_records + .iter() + .any(|current| current.id == record.id) + { + current_records.push(record.clone()); + } + } + current_records + } else { + records.clone() + }; records.sort_by(|a, b| { b.updated_at .cmp(&a.updated_at) .then_with(|| a.id.cmp(&b.id)) }); + candidate_records.sort_by(|a, b| { + b.updated_at + .cmp(&a.updated_at) + .then_with(|| a.id.cmp(&b.id)) + }); let evidence_window = build_evidence_window( store, params, @@ -435,7 +488,7 @@ pub fn run(store: &Store, params: &DreamParams) -> Result<(DreamResponse, bool)> check_deadline(params)?; - for record in &records { + for record in &candidate_records { check_deadline(params)?; let state = state_for_record(record); let drift_prone = state != "historical" && is_drift_prone(&record.content); @@ -483,10 +536,10 @@ pub fn run(store: &Store, params: &DreamParams) -> Result<(DreamResponse, bool)> check_deadline(params)?; - for newer in &records { + for newer in &candidate_records { check_deadline(params)?; let newer_state = state_for_record(newer); - for older in &records { + for older in &candidate_records { check_deadline(params)?; if newer.id == older.id || !same_boundary(newer, older) @@ -521,11 +574,11 @@ pub fn run(store: &Store, params: &DreamParams) -> Result<(DreamResponse, bool)> .map(|cursor| cursor.lower_bound.is_some()) .unwrap_or(params.recency_cutoff.is_some()); if !has_recency_lower_bound || params.include_archived_sources { - push_threshold_candidates(params, &records, &mut candidates, &mut rejected)?; + push_threshold_candidates(params, &candidate_records, &mut candidates, &mut rejected)?; } check_deadline(params)?; - attach_counter_evidence_retires(params, &records, &mut candidates)?; + attach_counter_evidence_retires(params, &candidate_records, &mut candidates)?; dedupe_candidates(&mut candidates); let mut max_candidates_hit = false; if let Some(max) = params.max_candidates { @@ -536,7 +589,7 @@ pub fn run(store: &Store, params: &DreamParams) -> Result<(DreamResponse, bool)> } check_deadline(params)?; - let run_id = stable_run_id(params, &evidence_window, &records); + let run_id = stable_run_id(params, &evidence_window, &candidate_records); let mut archived = Vec::new(); let mut created = Vec::new(); if params.mode == "apply" { @@ -775,6 +828,7 @@ fn imported_chatgpt_candidate_records( params.repo_id, source_window_start, cursor.lower_bound_exclusive, + cursor.lower_rowid_strict, source_window_end, cursor .lower_rowids @@ -940,6 +994,22 @@ pub fn is_scheduler_cursor(value: &str) -> bool { parse_scheduler_cursor(value).is_some() } +/// Return the public timestamp watermark when a completed scheduler cursor +/// retains a private rowid frontier. In-progress cursors stay opaque so +/// callers can continue a bounded scan. +pub fn scheduler_cursor_completed_watermark(value: &str) -> Option { + let cursor = parse_scheduler_cursor(value)?; + if cursor.lower_rowid_strict + && cursor.upper_bound.is_none() + && !cursor.upper_unbounded + && !cursor.has_source_position() + { + cursor.lower_bound + } else { + None + } +} + fn scheduler_cursor_from_watermark(previous: Option<&str>) -> DreamSchedulerCursor { previous .and_then(parse_scheduler_cursor) @@ -958,6 +1028,30 @@ pub fn scheduler_cursor_for_run( upper_unbounded: bool, ) -> Result { if let Some(previous) = previous.filter(|value| is_scheduler_cursor(value)) { + let mut cursor = scheduler_cursor_from_watermark(Some(previous)); + if cursor.lower_rowid_strict + && cursor.upper_bound.is_none() + && cursor.lower_rowids.is_some() + && !cursor.has_source_position() + { + // Start a new fixed band from the completed scan's timestamp and + // rowid frontiers. Both are needed to exclude already-seen rows + // without admitting newly inserted evidence below the watermark. + let next_start = cursor + .lower_bound + .as_deref() + .filter(|start| { + OffsetDateTime::parse(now, &Rfc3339) + .ok() + .zip(OffsetDateTime::parse(start, &Rfc3339).ok()) + .is_some_and(|(now, start)| now <= start) + }) + .unwrap_or(now); + cursor.upper_unbounded = false; + cursor.upper_bound = Some(next_start.to_string()); + cursor.upper_rowids = Some(scheduler_rowid_frontier(store)?); + return encode_scheduler_cursor(&cursor); + } return Ok(previous.to_string()); } let cursor = DreamSchedulerCursor { @@ -990,7 +1084,9 @@ pub fn scheduler_watermark_after( cursor.upper_rowids = Some(scheduler_rowid_frontier(store)?); if cursor.lower_bound.is_some() { cursor.lower_bound_exclusive = false; - cursor.lower_rowids = Some(DreamRowidFrontier::default()); + if !cursor.lower_rowid_strict { + cursor.lower_rowids = Some(DreamRowidFrontier::default()); + } } } else { update_scheduler_cursor(store, &mut cursor, window)?; @@ -1011,7 +1107,9 @@ pub fn scheduler_watermark_after( cursor.upper_rowids = Some(scheduler_rowid_frontier(store)?); if cursor.lower_bound.is_some() { cursor.lower_bound_exclusive = false; - cursor.lower_rowids = Some(DreamRowidFrontier::default()); + if !cursor.lower_rowid_strict { + cursor.lower_rowids = Some(DreamRowidFrontier::default()); + } } return Ok(Some(encode_scheduler_cursor(&cursor)?)); } @@ -1026,11 +1124,21 @@ pub fn scheduler_watermark_after( previous_cursor.upper_rowids = Some(scheduler_rowid_frontier(store)?); if previous_cursor.lower_bound.is_some() { previous_cursor.lower_bound_exclusive = false; - previous_cursor.lower_rowids = Some(DreamRowidFrontier::default()); + if !previous_cursor.lower_rowid_strict { + previous_cursor.lower_rowids = Some(DreamRowidFrontier::default()); + } } return Ok(Some(encode_scheduler_cursor(&previous_cursor)?)); } - return Ok(Some(now.to_string())); + previous_cursor.lower_bound = Some(now.to_string()); + previous_cursor.lower_bound_exclusive = false; + previous_cursor.lower_rowids = previous_cursor.upper_rowids.clone(); + previous_cursor.lower_rowid_strict = true; + previous_cursor.upper_bound = None; + previous_cursor.upper_unbounded = false; + previous_cursor.upper_rowids = None; + previous_cursor.clear_source_positions(); + return Ok(Some(encode_scheduler_cursor(&previous_cursor)?)); } if previous_cursor.upper_bound.is_some() && (previous_cursor.upper_rowids.is_none() || previous_cursor.has_legacy_position()) @@ -1039,10 +1147,38 @@ pub fn scheduler_watermark_after( previous_cursor.upper_rowids = Some(scheduler_rowid_frontier(store)?); if previous_cursor.lower_bound.is_some() { previous_cursor.lower_bound_exclusive = false; - previous_cursor.lower_rowids = Some(DreamRowidFrontier::default()); + if !previous_cursor.lower_rowid_strict { + previous_cursor.lower_rowids = Some(DreamRowidFrontier::default()); + } } return Ok(Some(encode_scheduler_cursor(&previous_cursor)?)); } + if previous_cursor.lower_rowid_strict { + if let Some(frontier) = previous_cursor.upper_bound.take() { + let now_is_after_frontier = OffsetDateTime::parse(now, &Rfc3339) + .ok() + .zip(OffsetDateTime::parse(&frontier, &Rfc3339).ok()) + .is_some_and(|(now, frontier)| now > frontier); + if now_is_after_frontier { + let next = DreamSchedulerCursor { + lower_bound: Some(frontier), + lower_rowids: previous_cursor.upper_rowids, + lower_rowid_strict: true, + upper_bound: Some(now.to_string()), + upper_rowids: Some(scheduler_rowid_frontier(store)?), + ..DreamSchedulerCursor::default() + }; + return Ok(Some(encode_scheduler_cursor(&next)?)); + } + previous_cursor.lower_bound = Some(frontier); + previous_cursor.lower_bound_exclusive = false; + previous_cursor.lower_rowids = previous_cursor.upper_rowids.take(); + previous_cursor.upper_bound = None; + previous_cursor.upper_unbounded = false; + previous_cursor.clear_source_positions(); + return Ok(Some(encode_scheduler_cursor(&previous_cursor)?)); + } + } if let Some(frontier) = previous_cursor.upper_bound.take() { let now_is_after_frontier = OffsetDateTime::parse(now, &Rfc3339) .ok() @@ -1296,6 +1432,7 @@ fn query_dream_rows( rowid_column: &str, start: Option<&str>, start_exclusive: bool, + lower_rowid_strict: bool, lower_rowid: Option, end: Option<&str>, upper_rowid: Option, @@ -1311,17 +1448,30 @@ where } if let Some(start) = start { if let Some(lower_rowid) = lower_rowid { - sql.push_str(&format!( - " AND ({timestamp_column} > ? OR ({timestamp_column} = ? AND {rowid_column} > ?))" - )); - args.push(Box::new(start.to_string())); - args.push(Box::new(start.to_string())); + if lower_rowid_strict { + let operator = if start_exclusive { ">" } else { ">=" }; + sql.push_str(&format!( + " AND {timestamp_column} {operator} ? AND {rowid_column} > ?" + )); + args.push(Box::new(start.to_string())); + } else { + sql.push_str(&format!( + " AND ({timestamp_column} > ? OR ({timestamp_column} = ? AND {rowid_column} > ?))" + )); + args.push(Box::new(start.to_string())); + args.push(Box::new(start.to_string())); + } args.push(Box::new(lower_rowid)); } else { let operator = if start_exclusive { ">" } else { ">=" }; sql.push_str(&format!(" AND {timestamp_column} {operator} ?")); args.push(Box::new(start.to_string())); } + } else if let Some(lower_rowid) = lower_rowid { + // A cursor without a timestamp start can still resume strictly after a + // per-source insertion frontier. + sql.push_str(&format!(" AND {rowid_column} > ?")); + args.push(Box::new(lower_rowid)); } if let Some(position) = position { if let Some(rowid) = position.rowid { @@ -1376,6 +1526,7 @@ fn query_visible_turns_with_frontier( repo_id: Option<&str>, start: Option<&str>, start_exclusive: bool, + lower_rowid_strict: bool, end: Option<&str>, lower_rowid: Option, upper_rowid: Option, @@ -1404,6 +1555,7 @@ fn query_visible_turns_with_frontier( "t.rowid", start, start_exclusive, + lower_rowid_strict, lower_rowid, end, upper_rowid, @@ -1429,6 +1581,7 @@ fn query_conclusions_with_frontier( repo_id: Option<&str>, start: Option<&str>, start_exclusive: bool, + lower_rowid_strict: bool, end: Option<&str>, lower_rowid: Option, upper_rowid: Option, @@ -1457,6 +1610,7 @@ fn query_conclusions_with_frontier( "rowid", start, start_exclusive, + lower_rowid_strict, lower_rowid, end, upper_rowid, @@ -1485,6 +1639,7 @@ fn query_checkpoints_with_frontier( repo_id: Option<&str>, start: Option<&str>, start_exclusive: bool, + lower_rowid_strict: bool, end: Option<&str>, lower_rowid: Option, upper_rowid: Option, @@ -1514,6 +1669,7 @@ fn query_checkpoints_with_frontier( "rowid", start, start_exclusive, + lower_rowid_strict, lower_rowid, end, upper_rowid, @@ -1547,6 +1703,7 @@ fn query_memory_sources_with_frontier( workspace_id: &str, start: Option<&str>, start_exclusive: bool, + lower_rowid_strict: bool, end: Option<&str>, lower_rowid: Option, upper_rowid: Option, @@ -1571,6 +1728,7 @@ fn query_memory_sources_with_frontier( "rowid", start, start_exclusive, + lower_rowid_strict, lower_rowid, end, upper_rowid, @@ -1599,6 +1757,7 @@ fn query_memory_records_with_frontier( repo_id: Option<&str>, start: Option<&str>, start_exclusive: bool, + lower_rowid_strict: bool, end: Option<&str>, lower_rowid: Option, upper_rowid: Option, @@ -1630,6 +1789,7 @@ fn query_memory_records_with_frontier( "rowid", start, start_exclusive, + lower_rowid_strict, lower_rowid, end, upper_rowid, @@ -1670,6 +1830,8 @@ fn build_evidence_window( }; let cursor_start_exclusive = scheduler_cursor.is_some_and(|cursor| cursor.lower_bound_exclusive); + let cursor_lower_rowid_strict = + scheduler_cursor.is_some_and(|cursor| cursor.lower_rowid_strict); let visible_turns = if remaining == 0 { Vec::new() } else if let Some(cursor) = scheduler_cursor { @@ -1680,6 +1842,7 @@ fn build_evidence_window( params.repo_id, start, cursor_start_exclusive, + cursor_lower_rowid_strict, cursor_end, cursor .lower_rowids @@ -1716,6 +1879,7 @@ fn build_evidence_window( params.repo_id, start, cursor_start_exclusive, + cursor_lower_rowid_strict, cursor_end, cursor .lower_rowids @@ -1752,6 +1916,7 @@ fn build_evidence_window( params.repo_id, start, cursor_start_exclusive, + cursor_lower_rowid_strict, cursor_end, cursor .lower_rowids @@ -1787,6 +1952,7 @@ fn build_evidence_window( params.workspace, start, cursor_start_exclusive, + cursor_lower_rowid_strict, cursor_end, cursor .lower_rowids diff --git a/src/service.rs b/src/service.rs index e07d1f6..15f2a66 100644 --- a/src/service.rs +++ b/src/service.rs @@ -2528,9 +2528,12 @@ impl Service { }); } - let watermark_before = + let scheduler_before = self.store - .scheduled_dream_watermark(profile.as_str(), &workspace, None)?; + .scheduled_dream_cursor(profile.as_str(), &workspace, None)?; + let watermark_before = scheduler_before.as_ref().map(|state| { + dream::scheduler_cursor_completed_watermark(state).unwrap_or_else(|| state.clone()) + }); if cfg.max_runtime_seconds == 0 { let limits_hit = vec!["max_runtime_seconds".to_string()]; self.store.record_dream_run(&DreamRunRecord { @@ -2603,11 +2606,11 @@ impl Service { && self.config.dream_provider.enabled && DreamProviderAdapter::parse(&self.config.dream_provider.adapter) == Some(DreamProviderAdapter::Command); - let scan_start = if command_mode && watermark_before.is_none() { + let scan_start = if command_mode && scheduler_before.is_none() { self.store .dream_watermark(profile.as_str(), &workspace, None)? } else { - watermark_before.clone() + scheduler_before.clone() }; let scheduler_cursor = Some(dream::scheduler_cursor_for_run( &self.store, @@ -2808,6 +2811,36 @@ impl Service { } else { "ok_with_limits" }; + if input_budget_limit.is_some() { + // The failed command sizing receipt is useful audit data, + // but this deterministic fallback completed the run. Add + // a later status-visible success receipt without replacing + // the provider's original budget audit. + let completed_at = ids::now_rfc3339(); + self.store.insert_dream_run(&DreamRunAudit { + id: ids::new_id("dream"), + profile_id: run.profile.clone(), + workspace_id: run.workspace.clone(), + repo_id: run.repo_id.clone(), + mode: run.mode.clone(), + status: status.to_string(), + started_at: completed_at.clone(), + completed_at: Some(completed_at), + implementation_version: dream::DREAM_IMPLEMENTATION_VERSION.to_string(), + config_hash: dream::config_hash(), + ruleset_version: dream::DREAM_RULESET_VERSION.to_string(), + fixture_schema_version: dream::DREAM_FIXTURE_SCHEMA_VERSION + .map(str::to_string), + source_window_start: run.evidence_window.start.clone(), + source_window_end: Some(run.evidence_window.end.clone()), + source_counts: dream_audit_source_counts(&run, None, None), + candidate_counts: dream_audit_candidate_counts(&run, None, None), + created_count: run.created.len() as i64, + archived_count: run.archived.len() as i64, + rejected_count: run.rejected.len() as i64, + error_summary: None, + })?; + } // A limited run has not durably covered the source frontier. // Preserve a bounded source cursor for input-only limits; keep // the previous cursor for runtime, provider, or candidate limits. @@ -2828,6 +2861,10 @@ impl Service { } else { None }; + let public_watermark_after = watermark_after.as_ref().map(|state| { + dream::scheduler_cursor_completed_watermark(state) + .unwrap_or_else(|| state.clone()) + }); self.store.record_dream_run(&DreamRunRecord { run_id: run.run_id.clone(), profile_id: run.profile.clone(), @@ -2851,7 +2888,7 @@ impl Service { reason: (!limits_hit.is_empty()).then(|| limits_hit.join(",")), run: Some(run), watermark_before, - watermark_after, + watermark_after: public_watermark_after, limits_hit, }) } diff --git a/src/store.rs b/src/store.rs index e13b45a..8e2fa9c 100644 --- a/src/store.rs +++ b/src/store.rs @@ -4273,11 +4273,14 @@ impl Store { LIMIT 1", [], |row| { + let watermark_after = row.get::<_, Option>(3)?.map(|state| { + crate::dream::scheduler_cursor_completed_watermark(&state).unwrap_or(state) + }); Ok(ScheduledDreamRunSummary { run_id: row.get(0)?, status: row.get(1)?, completed_at: row.get(2)?, - watermark_after: row.get(3)?, + watermark_after, error: row.get(4)?, }) }, @@ -4291,6 +4294,21 @@ impl Store { profile_id: &str, workspace_id: &str, repo_id: Option<&str>, + ) -> Result> { + Ok(self + .scheduled_dream_cursor(profile_id, workspace_id, repo_id)? + .map(|cursor| { + crate::dream::scheduler_cursor_completed_watermark(&cursor).unwrap_or(cursor) + })) + } + + /// Return the persisted scheduler state, including any private keyset + /// cursor used to resume or deduplicate a completed scan. + pub fn scheduled_dream_cursor( + &self, + profile_id: &str, + workspace_id: &str, + repo_id: Option<&str>, ) -> Result> { let conn = self.conn()?; let result = if let Some(repo_id) = repo_id { diff --git a/tests/consolidation_acceptance.rs b/tests/consolidation_acceptance.rs index 34be329..1e9cdb8 100644 --- a/tests/consolidation_acceptance.rs +++ b/tests/consolidation_acceptance.rs @@ -303,6 +303,140 @@ fn scheduled_provider_receives_conclusion_only_evidence() { server.join().expect("provider server"); } +#[test] +fn simulated_unbounded_scan_does_not_dispatch_frontier_evidence_again() { + let listener = TcpListener::bind("127.0.0.1:0").expect("provider listener"); + listener + .set_nonblocking(true) + .expect("nonblocking listener"); + let address = listener.local_addr().expect("provider address"); + let (body_tx, body_rx) = mpsc::channel(); + let server = thread::spawn(move || { + let deadline = Instant::now() + Duration::from_millis(800); + while Instant::now() < deadline { + match listener.accept() { + Ok((mut stream, _)) => { + let mut request = Vec::new(); + let mut chunk = [0_u8; 8192]; + loop { + let read = stream.read(&mut chunk).expect("read provider request"); + request.extend_from_slice(&chunk[..read]); + if let Some(header_end) = + request.windows(4).position(|window| window == b"\r\n\r\n") + { + let headers = String::from_utf8_lossy(&request[..header_end]); + let content_length = headers + .lines() + .find_map(|line| { + line.to_ascii_lowercase() + .strip_prefix("content-length:") + .map(str::to_string) + }) + .and_then(|value| value.trim().parse::().ok()) + .unwrap_or(0); + if request.len() >= header_end + 4 + content_length { + let body = String::from_utf8_lossy( + &request[header_end + 4..header_end + 4 + content_length], + ) + .into_owned(); + body_tx.send(body).expect("send provider body"); + let response = "HTTP/1.1 200 OK\r\nContent-Length: 2\r\nConnection: close\r\n\r\n[]"; + stream + .write_all(response.as_bytes()) + .expect("write provider response"); + break; + } + } + } + } + Err(error) + if error.kind() == std::io::ErrorKind::WouldBlock + && Instant::now() < deadline => + { + thread::sleep(Duration::from_millis(5)); + } + Err(error) => panic!("accept provider request: {error}"), + } + } + }); + + let store = Store::open(":memory:").expect("store"); + store + .insert_conclusion(&Conclusion { + id: "future-frontier-conclusion".into(), + profile_id: "personal".into(), + workspace_id: "simulated-frontier".into(), + repo_id: None, + target: "user".into(), + content: "Decision: simulated frontier evidence is processed once.".into(), + source_id: None, + created_at: "2030-01-01T00:00:00Z".into(), + metadata: json!({}), + }) + .expect("future-dated conclusion"); + let mut config = Config::default(); + config.default_profile = "personal".into(); + config.default_workspace = "simulated-frontier".into(); + config.dream_scheduler.enabled = true; + config.dream_scheduler.idle_window_seconds = 0; + config.dream_scheduler.min_session_age_seconds = 0; + config.dream_scheduler.min_turn_count = 0; + config.dream_scheduler.scheduled_provider_enabled = true; + config.dream_provider = DreamProviderConfig { + enabled: true, + endpoint: format!("http://{address}/v1"), + model: "fixture-model".into(), + ..DreamProviderConfig::default() + }; + let service = Service::new(store, config); + + let first = service + .scheduled_dream(Some("2000-01-01T00:00:00Z".into())) + .expect("initial unbounded simulated scan"); + assert_eq!( + first.watermark_after.as_deref(), + Some("2000-01-01T00:00:00Z"), + "the response keeps the public timestamp watermark" + ); + assert!(service + .store + .scheduled_dream_cursor("personal", "simulated-frontier", None) + .expect("persisted scheduler cursor") + .as_deref() + .is_some_and(codex_memoryd::dream::is_scheduler_cursor)); + service + .store + .insert_conclusion(&Conclusion { + id: "late-old-timestamp-conclusion".into(), + profile_id: "personal".into(), + workspace_id: "simulated-frontier".into(), + repo_id: None, + target: "user".into(), + content: "Decision: late evidence with an older timestamp stays eligible.".into(), + source_id: None, + created_at: "2001-01-01T00:00:00Z".into(), + metadata: json!({}), + }) + .expect("late conclusion"); + service + .scheduled_dream(Some("2030-01-01T00:00:00Z".into())) + .expect("later simulated scan"); + service + .scheduled_dream(Some("2030-01-01T00:00:00Z".into())) + .expect("repeat at the consumed source timestamp"); + + server.join().expect("provider server"); + let calls = body_rx.try_iter().collect::>(); + assert_eq!( + calls.len(), + 2, + "the later scan should dispatch only evidence added after the prior frontier" + ); + assert!(calls[0].contains("simulated frontier evidence is processed once")); + assert!(calls[1].contains("late evidence with an older timestamp stays eligible")); + assert!(!calls[1].contains("simulated frontier evidence is processed once")); +} + #[test] fn capped_scheduled_run_preserves_and_continues_unprocessed_tail() { let store = Store::open(":memory:").expect("store"); diff --git a/tests/dream_command_scheduler.rs b/tests/dream_command_scheduler.rs index c7b957b..9278784 100644 --- a/tests/dream_command_scheduler.rs +++ b/tests/dream_command_scheduler.rs @@ -171,6 +171,18 @@ fn check_oversized_command_preview(content: String) { !marker.exists(), "oversized evidence must not reach the provider" ); + let status = service.status().expect("status after recovered fallback"); + assert!( + !status + .degraded_reasons + .iter() + .any(|reason| reason.contains("last Dreamer run failed")), + "a successful scheduled fallback must replace the failed preflight in status" + ); + assert!(status + .last_dream + .as_ref() + .is_some_and(|run| run.status == "ok_with_limits")); let second = service.scheduled_dream(None).unwrap(); assert_ne!(second.watermark_before, first.watermark_before); From e5bcd6b6a1e92ed3c8c9361ecb41fdd66f983a82 Mon Sep 17 00:00:00 2001 From: Josh Yorko Date: Wed, 14 Oct 2026 19:16:01 +0000 Subject: [PATCH 14/18] fix: preserve scheduler mutation and rowid frontiers --- docs/dreamer-native-provider.md | 9 +- src/dream.rs | 195 +++++++++++--- src/store.rs | 6 +- tests/scheduler_review_boundaries.rs | 369 +++++++++++++++++++++++++++ 4 files changed, 534 insertions(+), 45 deletions(-) diff --git a/docs/dreamer-native-provider.md b/docs/dreamer-native-provider.md index e92b261..6016e19 100644 --- a/docs/dreamer-native-provider.md +++ b/docs/dreamer-native-provider.md @@ -68,9 +68,12 @@ evidence remains eligible. Other provider failures do not advance the successful scheduled watermark. A recovered deterministic fallback also records a later status-visible `ok_with_limits` receipt while retaining the provider budget rejection in audit history. Completed unbounded simulated-clock scans retain -their rowid frontier, so a later clock does not redispatch prior evidence and -new rows remain eligible when they meet the timestamp watermark, even when -they predate future-dated rows from the earlier scan. Process-group +their rowid frontier even when the simulated clock does not advance. The +scheduler also pins the active-memory `updated_at` version frontier across +bounded pages, so restore and promotion operations on an existing row remain +eligible after the insertion frontier advances. Previously selected active +records may provide comparison context, but only a record selected in the +current evidence window can emit stale or supersession candidates. Process-group children are terminated; orphan reaping remains the operating system's job. Managed readiness accepts `local_only`, or `degraded` only when the reported diff --git a/src/dream.rs b/src/dream.rs index 03e11e0..00424f5 100644 --- a/src/dream.rs +++ b/src/dream.rs @@ -83,6 +83,10 @@ struct DreamSchedulerCursor { #[serde(default)] upper_rowids: Option, #[serde(default)] + active_memory_version_lower: Option, + #[serde(default)] + active_memory_version_upper: Option, + #[serde(default)] visible_turns: Option, #[serde(default)] conclusions: Option, @@ -379,6 +383,15 @@ pub fn run(store: &Store, params: &DreamParams) -> Result<(DreamResponse, bool)> limit: params.max_records, offset: 0, }; + let active_memory_mutation_scan = scheduler_cursor.as_ref().and_then(|cursor| { + if !cursor.lower_rowid_strict { + return None; + } + let lower = cursor.active_memory_version_lower.as_deref()?; + let upper = cursor.active_memory_version_upper.as_deref()?; + let upper_rowid = cursor.upper_rowids.as_ref()?.active_memory_records; + Some((lower, upper, upper_rowid)) + }); let mut records = match scheduler_cursor.as_ref() { Some(cursor) => query_memory_records_with_frontier( store, @@ -400,6 +413,7 @@ pub fn run(store: &Store, params: &DreamParams) -> Result<(DreamResponse, bool)> cursor.active_memory_records.as_ref(), params.max_records, params.include_archived_sources, + active_memory_mutation_scan, )?, None => match params.source_window_end { Some(end) => store.query_records_until(&query, end)?, @@ -448,6 +462,7 @@ pub fn run(store: &Store, params: &DreamParams) -> Result<(DreamResponse, bool)> None, context_limit, params.include_archived_sources, + None, )? } else { Vec::new() @@ -485,11 +500,18 @@ pub fn run(store: &Store, params: &DreamParams) -> Result<(DreamResponse, bool)> let mut candidates = Vec::new(); let mut stale = Vec::new(); let mut rejected = Vec::new(); + let current_record_ids = records + .iter() + .map(|record| record.id.as_str()) + .collect::>(); check_deadline(params)?; for record in &candidate_records { check_deadline(params)?; + if !current_record_ids.contains(record.id.as_str()) { + continue; + } let state = state_for_record(record); let drift_prone = state != "historical" && is_drift_prone(&record.content); let valid_until = valid_until_for(record); @@ -542,6 +564,8 @@ pub fn run(store: &Store, params: &DreamParams) -> Result<(DreamResponse, bool)> for older in &candidate_records { check_deadline(params)?; if newer.id == older.id + || (!current_record_ids.contains(newer.id.as_str()) + && !current_record_ids.contains(older.id.as_str())) || !same_boundary(newer, older) || newer.created_at <= older.created_at { @@ -1049,21 +1073,58 @@ pub fn scheduler_cursor_for_run( .unwrap_or(now); cursor.upper_unbounded = false; cursor.upper_bound = Some(next_start.to_string()); + begin_active_memory_version_scan(&mut cursor); cursor.upper_rowids = Some(scheduler_rowid_frontier(store)?); return encode_scheduler_cursor(&cursor); } + if cursor.active_memory_version_upper.is_none() { + // Older persisted cursors have no mutation frontier. Establish a + // baseline when they are next resumed so later restores and + // promotions can be detected without replaying every old row. + if cursor.active_memory_version_lower.is_none() { + cursor.active_memory_version_lower = Some(ids::now_rfc3339()); + } + cursor.active_memory_version_upper = Some(ids::now_rfc3339()); + return encode_scheduler_cursor(&cursor); + } return Ok(previous.to_string()); } - let cursor = DreamSchedulerCursor { + let mut cursor = DreamSchedulerCursor { lower_bound: previous.map(str::to_string), upper_bound: (!upper_unbounded).then(|| now.to_string()), upper_unbounded, - upper_rowids: Some(scheduler_rowid_frontier(store)?), ..DreamSchedulerCursor::default() }; + cursor.active_memory_version_upper = Some(ids::now_rfc3339()); + cursor.upper_rowids = Some(scheduler_rowid_frontier(store)?); encode_scheduler_cursor(&cursor) } +fn begin_active_memory_version_scan(cursor: &mut DreamSchedulerCursor) { + if cursor.active_memory_version_lower.is_none() { + cursor.active_memory_version_lower = Some(ids::now_rfc3339()); + } + cursor.active_memory_version_upper = Some(ids::now_rfc3339()); +} + +fn complete_active_memory_version_scan(cursor: &mut DreamSchedulerCursor) { + if let Some(upper) = cursor.active_memory_version_upper.take() { + cursor.active_memory_version_lower = Some(upper); + } +} + +fn next_active_memory_version_scan( + cursor: &mut DreamSchedulerCursor, + previous: &DreamSchedulerCursor, +) { + cursor.active_memory_version_lower = previous + .active_memory_version_upper + .clone() + .or_else(|| previous.active_memory_version_lower.clone()) + .or_else(|| Some(ids::now_rfc3339())); + cursor.active_memory_version_upper = Some(ids::now_rfc3339()); +} + /// Return a bounded scheduler cursor for a window that did not cover the /// entire input frontier. The cursor keeps one keyset position per source /// stream because the streams are budgeted in order and can advance unevenly. @@ -1097,6 +1158,9 @@ pub fn scheduler_watermark_after( cursor.clear_source_positions(); cursor.upper_bound = Some(window.end.clone()); cursor.upper_rowids = Some(scheduler_rowid_frontier(store)?); + if cursor.active_memory_version_upper.is_none() { + begin_active_memory_version_scan(&mut cursor); + } // The first bounded page used the ordinary ID ordering. Start the // persisted cursor at the beginning so the remaining pages use // the stable rowid ordering consistently. @@ -1111,6 +1175,9 @@ pub fn scheduler_watermark_after( cursor.lower_rowids = Some(DreamRowidFrontier::default()); } } + if cursor.active_memory_version_upper.is_none() { + begin_active_memory_version_scan(&mut cursor); + } return Ok(Some(encode_scheduler_cursor(&cursor)?)); } update_scheduler_cursor(store, &mut cursor, window)?; @@ -1138,6 +1205,7 @@ pub fn scheduler_watermark_after( previous_cursor.upper_unbounded = false; previous_cursor.upper_rowids = None; previous_cursor.clear_source_positions(); + complete_active_memory_version_scan(&mut previous_cursor); return Ok(Some(encode_scheduler_cursor(&previous_cursor)?)); } if previous_cursor.upper_bound.is_some() @@ -1160,14 +1228,15 @@ pub fn scheduler_watermark_after( .zip(OffsetDateTime::parse(&frontier, &Rfc3339).ok()) .is_some_and(|(now, frontier)| now > frontier); if now_is_after_frontier { - let next = DreamSchedulerCursor { + let mut next = DreamSchedulerCursor { lower_bound: Some(frontier), - lower_rowids: previous_cursor.upper_rowids, + lower_rowids: previous_cursor.upper_rowids.clone(), lower_rowid_strict: true, upper_bound: Some(now.to_string()), - upper_rowids: Some(scheduler_rowid_frontier(store)?), ..DreamSchedulerCursor::default() }; + next_active_memory_version_scan(&mut next, &previous_cursor); + next.upper_rowids = Some(scheduler_rowid_frontier(store)?); return Ok(Some(encode_scheduler_cursor(&next)?)); } previous_cursor.lower_bound = Some(frontier); @@ -1176,6 +1245,7 @@ pub fn scheduler_watermark_after( previous_cursor.upper_bound = None; previous_cursor.upper_unbounded = false; previous_cursor.clear_source_positions(); + complete_active_memory_version_scan(&mut previous_cursor); return Ok(Some(encode_scheduler_cursor(&previous_cursor)?)); } } @@ -1185,17 +1255,27 @@ pub fn scheduler_watermark_after( .zip(OffsetDateTime::parse(&frontier, &Rfc3339).ok()) .is_some_and(|(now, frontier)| now > frontier); if now_is_after_frontier { - let next = DreamSchedulerCursor { + let mut next = DreamSchedulerCursor { lower_bound: Some(frontier), lower_bound_exclusive: false, - lower_rowids: previous_cursor.upper_rowids, + lower_rowids: previous_cursor.upper_rowids.clone(), + lower_rowid_strict: true, upper_bound: Some(now.to_string()), - upper_rowids: Some(scheduler_rowid_frontier(store)?), ..DreamSchedulerCursor::default() }; + next_active_memory_version_scan(&mut next, &previous_cursor); + next.upper_rowids = Some(scheduler_rowid_frontier(store)?); return Ok(Some(encode_scheduler_cursor(&next)?)); } - return Ok(Some(frontier)); + previous_cursor.lower_bound = Some(frontier); + previous_cursor.lower_bound_exclusive = false; + previous_cursor.lower_rowids = previous_cursor.upper_rowids.take(); + previous_cursor.lower_rowid_strict = true; + previous_cursor.upper_bound = None; + previous_cursor.upper_unbounded = false; + previous_cursor.clear_source_positions(); + complete_active_memory_version_scan(&mut previous_cursor); + return Ok(Some(encode_scheduler_cursor(&previous_cursor)?)); } } @@ -1437,6 +1517,7 @@ fn query_dream_rows( end: Option<&str>, upper_rowid: Option, position: Option<&DreamSourceCursor>, + mutation_scan: Option<(&str, &str, i64)>, limit: usize, mapper: F, ) -> Result> @@ -1446,33 +1527,79 @@ where if limit == 0 { return Ok(Vec::new()); } + let mut start_filter = String::new(); + let mut start_args: Vec> = Vec::new(); if let Some(start) = start { if let Some(lower_rowid) = lower_rowid { if lower_rowid_strict { let operator = if start_exclusive { ">" } else { ">=" }; - sql.push_str(&format!( - " AND {timestamp_column} {operator} ? AND {rowid_column} > ?" - )); - args.push(Box::new(start.to_string())); + start_filter = format!("{timestamp_column} {operator} ? AND {rowid_column} > ?"); + start_args.push(Box::new(start.to_string())); } else { - sql.push_str(&format!( - " AND ({timestamp_column} > ? OR ({timestamp_column} = ? AND {rowid_column} > ?))" - )); - args.push(Box::new(start.to_string())); - args.push(Box::new(start.to_string())); + start_filter = format!( + "({timestamp_column} > ? OR ({timestamp_column} = ? AND {rowid_column} > ?))" + ); + start_args.push(Box::new(start.to_string())); + start_args.push(Box::new(start.to_string())); } - args.push(Box::new(lower_rowid)); + start_args.push(Box::new(lower_rowid)); } else { let operator = if start_exclusive { ">" } else { ">=" }; - sql.push_str(&format!(" AND {timestamp_column} {operator} ?")); - args.push(Box::new(start.to_string())); + start_filter = format!("{timestamp_column} {operator} ?"); + start_args.push(Box::new(start.to_string())); } } else if let Some(lower_rowid) = lower_rowid { // A cursor without a timestamp start can still resume strictly after a // per-source insertion frontier. - sql.push_str(&format!(" AND {rowid_column} > ?")); - args.push(Box::new(lower_rowid)); + start_filter = format!("{rowid_column} > ?"); + start_args.push(Box::new(lower_rowid)); + } + + let mut end_filter = String::new(); + let mut end_args: Vec> = Vec::new(); + if let Some(end) = end { + if let Some(upper_rowid) = upper_rowid { + end_filter = format!( + " AND ({timestamp_column} < ? OR ({timestamp_column} = ? AND {rowid_column} <= ?))" + ); + end_args.push(Box::new(end.to_string())); + end_args.push(Box::new(end.to_string())); + end_args.push(Box::new(upper_rowid)); + } else { + end_filter = format!(" AND {timestamp_column} <= ?"); + end_args.push(Box::new(end.to_string())); + } + } else if let Some(upper_rowid) = upper_rowid { + end_filter = format!(" AND {rowid_column} <= ?"); + end_args.push(Box::new(upper_rowid)); + } + + if let Some((version_lower, version_upper, version_upper_rowid)) = mutation_scan { + sql.push_str(" AND (("); + if start_filter.is_empty() { + sql.push_str("1 = 1"); + } else { + sql.push_str(&start_filter); + } + sql.push_str(&end_filter); + sql.push_str(&format!( + ") OR ({timestamp_column} > ? AND {timestamp_column} <= ? AND {rowid_column} <= ?))" + )); + args.extend(start_args); + args.extend(end_args); + args.push(Box::new(version_lower.to_string())); + args.push(Box::new(version_upper.to_string())); + args.push(Box::new(version_upper_rowid)); + } else { + if !start_filter.is_empty() { + sql.push_str(" AND "); + sql.push_str(&start_filter); + args.extend(start_args); + } + sql.push_str(&end_filter); + args.extend(end_args); } + if let Some(position) = position { if let Some(rowid) = position.rowid { sql.push_str(&format!( @@ -1490,22 +1617,6 @@ where args.push(Box::new(position.id.clone())); } } - if let Some(end) = end { - if let Some(upper_rowid) = upper_rowid { - sql.push_str(&format!( - " AND ({timestamp_column} < ? OR ({timestamp_column} = ? AND {rowid_column} <= ?))" - )); - args.push(Box::new(end.to_string())); - args.push(Box::new(end.to_string())); - args.push(Box::new(upper_rowid)); - } else { - sql.push_str(&format!(" AND {timestamp_column} <= ?")); - args.push(Box::new(end.to_string())); - } - } else if let Some(upper_rowid) = upper_rowid { - sql.push_str(&format!(" AND {rowid_column} <= ?")); - args.push(Box::new(upper_rowid)); - } let params_ref: Vec<&dyn ToSql> = args.iter().map(|arg| arg.as_ref()).collect(); sql.push_str(&format!( " ORDER BY {timestamp_column} DESC, {rowid_column} ASC LIMIT {limit}" @@ -1560,6 +1671,7 @@ fn query_visible_turns_with_frontier( end, upper_rowid, position, + None, limit, |row| { Ok(VisibleTurn { @@ -1615,6 +1727,7 @@ fn query_conclusions_with_frontier( end, upper_rowid, position, + None, limit, |row| { Ok(Conclusion { @@ -1674,6 +1787,7 @@ fn query_checkpoints_with_frontier( end, upper_rowid, position, + None, limit, |row| { Ok(Checkpoint { @@ -1733,6 +1847,7 @@ fn query_memory_sources_with_frontier( end, upper_rowid, position, + None, limit, |row| { Ok(MemorySource { @@ -1764,6 +1879,7 @@ fn query_memory_records_with_frontier( position: Option<&DreamSourceCursor>, limit: usize, include_archived: bool, + mutation_scan: Option<(&str, &str, i64)>, ) -> Result> { let mut sql = "SELECT id FROM memory_records WHERE profile_id = ? AND workspace_id = ?" @@ -1794,6 +1910,7 @@ fn query_memory_records_with_frontier( end, upper_rowid, position, + mutation_scan, limit, |row| row.get::<_, String>(0), )?; diff --git a/src/store.rs b/src/store.rs index 8e2fa9c..b1f493c 100644 --- a/src/store.rs +++ b/src/store.rs @@ -4269,7 +4269,7 @@ impl Store { "SELECT run_id, status, completed_at, watermark_after, error FROM dream_runs WHERE kind = 'scheduled' - ORDER BY started_at DESC, run_id DESC + ORDER BY started_at DESC, rowid DESC LIMIT 1", [], |row| { @@ -4318,7 +4318,7 @@ impl Store { WHERE kind = 'scheduled' AND status IN ('ok', 'ok_with_limits') AND watermark_after IS NOT NULL AND profile_id = ?1 AND workspace_id = ?2 AND repo_id = ?3 - ORDER BY completed_at DESC, run_id DESC + ORDER BY completed_at DESC, rowid DESC LIMIT 1", params![profile_id, workspace_id, repo_id], |row| row.get(0), @@ -4331,7 +4331,7 @@ impl Store { WHERE kind = 'scheduled' AND status IN ('ok', 'ok_with_limits') AND watermark_after IS NOT NULL AND profile_id = ?1 AND workspace_id = ?2 AND repo_id IS NULL - ORDER BY completed_at DESC, run_id DESC + ORDER BY completed_at DESC, rowid DESC LIMIT 1", params![profile_id, workspace_id], |row| row.get(0), diff --git a/tests/scheduler_review_boundaries.rs b/tests/scheduler_review_boundaries.rs index 73a57b2..5a38051 100644 --- a/tests/scheduler_review_boundaries.rs +++ b/tests/scheduler_review_boundaries.rs @@ -1,4 +1,8 @@ use std::collections::BTreeSet; +use std::io::{Read, Write}; +use std::net::TcpListener; +use std::thread; +use std::time::{Duration, Instant}; use codex_memoryd::config::Config; use codex_memoryd::domain::{ @@ -30,6 +34,106 @@ fn scheduler_config(workspace: &str, max_batch_size: usize, automatic_apply: boo config } +fn insert_memory_record(store: &Store, content: &str, metadata: serde_json::Value) -> String { + let class = policy::classify(content, Profile::Personal, false); + match store + .upsert_record(&NewRecord { + profile_id: PROFILE.into(), + workspace_id: "workspace".into(), + repo_id: None, + subject_id: None, + episode_id: None, + scope: class.scope, + record_type: class.record_type, + content: content.into(), + related_files: vec![], + tags: vec![], + sensitivity: Sensitivity::Personal, + portability: Portability::ProfileOnly, + confidence: class.confidence, + source_ids: vec![], + content_hash: ids::exact_content_hash( + PROFILE, + "workspace", + None, + class.record_type.as_str(), + class.scope.as_str(), + content, + ), + supersedes: vec![], + metadata, + }) + .expect("synthetic memory record") + { + UpsertOutcome::Created(id) | UpsertOutcome::Skipped(id) => id, + } +} + +fn capture_provider_requests(expected: usize) -> (String, thread::JoinHandle>) { + let listener = TcpListener::bind("127.0.0.1:0").expect("provider listener"); + listener + .set_nonblocking(true) + .expect("nonblocking provider listener"); + let address = listener.local_addr().expect("provider address"); + let server = thread::spawn(move || { + let deadline = Instant::now() + Duration::from_secs(3); + let mut bodies = Vec::new(); + while bodies.len() < expected && Instant::now() < deadline { + match listener.accept() { + Ok((mut stream, _)) => { + stream + .set_read_timeout(Some(Duration::from_secs(1))) + .expect("provider read timeout"); + let mut request = Vec::new(); + let mut chunk = [0_u8; 8192]; + loop { + let read = stream.read(&mut chunk).expect("read provider request"); + if read == 0 { + break; + } + request.extend_from_slice(&chunk[..read]); + if let Some(header_end) = + request.windows(4).position(|window| window == b"\r\n\r\n") + { + let headers = String::from_utf8_lossy(&request[..header_end]); + let content_length = headers + .lines() + .find_map(|line| { + line.to_ascii_lowercase() + .strip_prefix("content-length:") + .map(str::to_string) + }) + .and_then(|value| value.trim().parse::().ok()) + .unwrap_or(0); + if request.len() >= header_end + 4 + content_length { + bodies.push( + String::from_utf8_lossy( + &request[header_end + 4..header_end + 4 + content_length], + ) + .into_owned(), + ); + stream + .write_all(b"HTTP/1.1 200 OK\r\nContent-Length: 2\r\nConnection: close\r\n\r\n[]") + .expect("write provider response"); + break; + } + } + } + } + Err(error) + if error.kind() == std::io::ErrorKind::WouldBlock + && Instant::now() < deadline => + { + thread::sleep(Duration::from_millis(5)); + } + Err(error) => panic!("accept provider request: {error}"), + } + } + bodies + }); + (format!("http://{address}/v1"), server) +} + fn evidence_ids(run: &codex_memoryd::protocol::DreamResponse) -> BTreeSet { run.evidence_window .visible_turns @@ -710,3 +814,268 @@ fn archived_exact_hash_does_not_suppress_new_synthetic_evidence() { "re-adoption must retain fresh provenance" ); } + +#[test] +fn scheduler_scan_emits_restored_and_promoted_memory_versions() { + let (endpoint, server) = capture_provider_requests(3); + let store = Store::open(":memory:").expect("store"); + let mut config = scheduler_config("workspace", 1, false); + config.dream_scheduler.scheduled_provider_enabled = true; + config.dream_provider = codex_memoryd::config::DreamProviderConfig { + enabled: true, + endpoint, + model: "fixture-model".into(), + ..Default::default() + }; + let service = Service::new(store.clone(), config); + + let restored_id = insert_memory_record( + &store, + "Synthetic restored record reaches the scheduled provider.", + json!({"origin": "fixture"}), + ); + store + .archive_records_with_metadata( + PROFILE, + Some("workspace"), + std::slice::from_ref(&restored_id), + "historical", + "synthetic restore fixture", + Some("restore-fixture-run"), + ) + .expect("archive restore fixture"); + let promoted_id = insert_memory_record( + &store, + "Synthetic promoted record reaches the scheduled provider.", + json!({"origin": "fixture"}), + ); + store + .quarantine_records( + PROFILE, + Some("workspace"), + std::slice::from_ref(&promoted_id), + "synthetic promotion fixture", + ) + .expect("quarantine promotion fixture"); + store + .insert_conclusion(&Conclusion { + id: "scheduler-version-seed".into(), + profile_id: PROFILE.into(), + workspace_id: "workspace".into(), + repo_id: None, + target: "user".into(), + content: "Conclusion: seed the simulated scheduler version scan.".into(), + source_id: None, + created_at: FRONTIER.into(), + metadata: json!({}), + }) + .expect("seed conclusion"); + + let first = service + .scheduled_dream(Some("2000-01-01T00:00:00Z".into())) + .expect("initial simulated scan"); + assert!(evidence_ids(first.run.as_ref().expect("seed run")).contains("scheduler-version-seed")); + service + .scheduled_dream(Some("2000-01-01T00:00:01Z".into())) + .expect("complete initial unbounded scan"); + assert_eq!( + store + .scheduled_dream_cursor(PROFILE, "workspace", None) + .expect("completed cursor") + .as_deref() + .and_then(codex_memoryd::dream::scheduler_cursor_completed_watermark), + Some("2000-01-01T00:00:01Z".into()), + "the no-input drain should persist a completed rowid cursor" + ); + + assert_eq!( + store + .restore_records_with_metadata( + PROFILE, + Some("workspace"), + std::slice::from_ref(&restored_id), + "restore-fixture-run", + "synthetic restore fixture", + ) + .expect("restore synthetic record") + .0, + vec![restored_id.clone()] + ); + assert_eq!( + store + .promote_quarantined_records( + PROFILE, + Some("workspace"), + std::slice::from_ref(&promoted_id), + ) + .expect("promote synthetic record") + .0, + vec![promoted_id.clone()] + ); + + let first_changed_page = service + .scheduled_dream(Some(RUN_NOW.into())) + .expect("scan restored or promoted record"); + let first_changed_ids = evidence_ids(first_changed_page.run.as_ref().expect("changed run")); + let second_changed_page = service + .scheduled_dream(Some(RUN_NOW.into())) + .expect("continue scanning changed records"); + let second_changed_ids = evidence_ids(second_changed_page.run.as_ref().expect("changed run")); + assert_eq!( + first_changed_ids.union(&second_changed_ids).cloned().collect::>(), + BTreeSet::from([restored_id.clone(), promoted_id.clone()]), + "existing rowids made eligible by restore or promotion must enter the current evidence window" + ); + service + .scheduled_dream(Some(RUN_NOW.into())) + .expect("finish changed-record scan"); + let next_tick = service + .scheduled_dream(Some(RUN_NOW.into())) + .expect("verify changed records are not redispatched"); + assert!( + next_tick + .run + .as_ref() + .map(evidence_ids) + .unwrap_or_default() + .is_disjoint(&BTreeSet::from([restored_id.clone(), promoted_id.clone()])), + "a completed version scan must not redispatch its restored or promoted records" + ); + + let provider_bodies = server.join().expect("provider capture"); + assert_eq!( + provider_bodies.len(), + 3, + "seed and both changed pages dispatch" + ); + let changed_provider_input = provider_bodies.iter().skip(1).cloned().collect::(); + assert!(changed_provider_input + .contains("Synthetic restored record reaches the scheduled provider.")); + assert!(changed_provider_input + .contains("Synthetic promoted record reaches the scheduled provider.")); +} + +#[test] +fn scheduler_idle_preview_does_not_emit_candidates_from_comparison_context() { + let store = Store::open(":memory:").expect("store"); + let service = Service::new(store.clone(), scheduler_config("workspace", 2, false)); + let old_id = insert_memory_record( + &store, + "I plan to deploy the widget tomorrow.", + json!({"state": "planned", "subject_key": "widget-deployment"}), + ); + let new_id = insert_memory_record( + &store, + "Widget deployment is fixed.", + json!({"state": "completed", "subject_key": "widget-deployment"}), + ); + store + .transaction(|tx| { + tx.execute( + "UPDATE memory_records SET created_at = ?1 WHERE id = ?2", + rusqlite::params!["2020-01-01T00:00:00Z", old_id], + )?; + tx.execute( + "UPDATE memory_records SET created_at = ?1 WHERE id = ?2", + rusqlite::params!["2021-01-01T00:00:00Z", new_id], + )?; + Ok(()) + }) + .expect("age synthetic comparison fixtures"); + + let first = service + .scheduled_dream(Some("2100-01-01T00:00:00Z".into())) + .expect("scan current records"); + let first_run = first.run.expect("initial run"); + assert!(first_run + .candidates + .iter() + .any(|candidate| candidate.action == "rewrite_historical" + && candidate.supersedes.contains(&old_id))); + assert!( + first_run + .candidates + .iter() + .any(|candidate| candidate.action == "supersede" + && candidate.supersedes.contains(&old_id)) + ); + + let idle = service + .scheduled_dream(Some("2100-01-01T00:00:01Z".into())) + .expect("complete current-record scan"); + assert!(idle.limits_hit.is_empty()); + let idle = service + .scheduled_dream(Some("2100-01-01T00:00:02Z".into())) + .expect("idle comparison-only preview"); + let idle_run = idle.run.expect("idle run"); + assert!(idle_run + .evidence_window + .active_memory_records + .sources + .is_empty()); + assert!( + idle_run.candidates.is_empty(), + "prior active rows should provide comparison context only when current evidence is absent; candidates={:?}", + idle_run.candidates + ); + assert!( + idle_run.stale.is_empty(), + "comparison-only expired rows must not be re-reported as current stale evidence" + ); +} + +#[test] +fn scheduler_same_clock_completion_retains_its_rowid_frontier() { + let store = Store::open(":memory:").expect("store"); + let service = Service::new(store.clone(), scheduler_config("workspace", 1, false)); + store + .record_dream_run(&DreamRunRecord { + // This key sorts after generated `dream_...` run ids. Equal + // simulated timestamps must still read the newest inserted run. + run_id: "zzzz-scheduler-seed-watermark".into(), + profile_id: PROFILE.into(), + workspace_id: "workspace".into(), + mode: "preview".into(), + kind: "scheduled".into(), + status: "ok".into(), + started_at: RUN_NOW.into(), + completed_at: Some(RUN_NOW.into()), + watermark_after: Some(BEFORE_FRONTIER.into()), + ..Default::default() + }) + .expect("seed bounded cursor watermark"); + for id in ["same-clock-frontier-a", "same-clock-frontier-b"] { + insert_chatgpt_turn( + &store, + "same-clock-session", + id, + &format!("generic same-clock scheduler evidence {id}"), + RUN_NOW, + None, + ); + } + + let first = service + .scheduled_dream(Some(RUN_NOW.into())) + .expect("first bounded page"); + let first_ids = evidence_ids(first.run.as_ref().expect("first run")); + assert_eq!(first_ids.len(), 1); + let second = service + .scheduled_dream(Some(RUN_NOW.into())) + .expect("second bounded page at unchanged clock"); + let second_ids = evidence_ids(second.run.as_ref().expect("second run")); + assert_eq!(second_ids.len(), 1); + assert!(first_ids.is_disjoint(&second_ids)); + + service + .scheduled_dream(Some(RUN_NOW.into())) + .expect("complete bounded cursor without advancing the clock"); + let next_tick = service + .scheduled_dream(Some(RUN_NOW.into())) + .expect("start next same-clock scan"); + let next_ids = next_tick.run.as_ref().map(evidence_ids).unwrap_or_default(); + assert!( + next_ids.is_empty(), + "a completed bounded scan must not redispatch rows at its saved upper timestamp; got {next_ids:?}" + ); +} From 720f0f1be826af7dc3cdf25e4dfc2ee91362b4de Mon Sep 17 00:00:00 2001 From: Josh Yorko Date: Wed, 14 Oct 2026 20:28:48 +0000 Subject: [PATCH 15/18] fix: make dream scheduler frontiers race-safe --- docs/dreamer-native-provider.md | 16 +- .../0014_scheduler_mutation_frontier.sql | 8 + src/dream.rs | 233 +++++++---- src/store.rs | 96 ++++- tests/dream_command_scheduler.rs | 8 +- tests/fixtures/status.response.json | 2 +- tests/scheduler_review_boundaries.rs | 374 +++++++++++++++++- tests/schema_upgrade.rs | 21 + 8 files changed, 672 insertions(+), 86 deletions(-) create mode 100644 migrations/0014_scheduler_mutation_frontier.sql diff --git a/docs/dreamer-native-provider.md b/docs/dreamer-native-provider.md index 6016e19..156accf 100644 --- a/docs/dreamer-native-provider.md +++ b/docs/dreamer-native-provider.md @@ -68,12 +68,16 @@ evidence remains eligible. Other provider failures do not advance the successful scheduled watermark. A recovered deterministic fallback also records a later status-visible `ok_with_limits` receipt while retaining the provider budget rejection in audit history. Completed unbounded simulated-clock scans retain -their rowid frontier even when the simulated clock does not advance. The -scheduler also pins the active-memory `updated_at` version frontier across -bounded pages, so restore and promotion operations on an existing row remain -eligible after the insertion frontier advances. Previously selected active -records may provide comparison context, but only a record selected in the -current evidence window can emit stale or supersession candidates. Process-group +their rowid frontier even when the simulated clock does not advance. Each +bounded band caps every source stream at its captured rowid frontier. The next +band also checks rows inserted after that cap, including rows whose source +timestamp was backdated below the prior band's end. Restore and promotion +operations allocate a per-record mutation sequence in the same SQLite +transaction as the eligibility change; the scheduler captures that sequence +with the rowid frontier so an uncommitted mutation cannot be skipped because +its wall-clock timestamp predates a scan. Previously selected active records +may provide comparison context, but only a record selected in the current +evidence window can emit stale or supersession candidates. Process-group children are terminated; orphan reaping remains the operating system's job. Managed readiness accepts `local_only`, or `degraded` only when the reported diff --git a/migrations/0014_scheduler_mutation_frontier.sql b/migrations/0014_scheduler_mutation_frontier.sql new file mode 100644 index 0000000..afe1512 --- /dev/null +++ b/migrations/0014_scheduler_mutation_frontier.sql @@ -0,0 +1,8 @@ +-- Commit-ordered mutation markers for scheduler-visible memory reactivation. +CREATE TABLE IF NOT EXISTS active_memory_mutation_counter ( + singleton INTEGER PRIMARY KEY CHECK (singleton = 1), + sequence INTEGER NOT NULL +); + +INSERT OR IGNORE INTO active_memory_mutation_counter(singleton, sequence) +VALUES (1, 0); diff --git a/src/dream.rs b/src/dream.rs index 00424f5..5ab508c 100644 --- a/src/dream.rs +++ b/src/dream.rs @@ -64,6 +64,8 @@ struct DreamRowidFrontier { checkpoints: i64, imported_memories: i64, active_memory_records: i64, + #[serde(default)] + active_memory_mutation_sequence: i64, } #[derive(Debug, Clone, Default, Serialize, Deserialize)] @@ -75,6 +77,8 @@ struct DreamSchedulerCursor { #[serde(default)] lower_rowid_strict: bool, #[serde(default)] + include_post_frontier_rowids: bool, + #[serde(default)] upper_bound: Option, #[serde(default)] upper_unbounded: bool, @@ -83,9 +87,9 @@ struct DreamSchedulerCursor { #[serde(default)] upper_rowids: Option, #[serde(default)] - active_memory_version_lower: Option, + active_memory_mutation_lower: Option, #[serde(default)] - active_memory_version_upper: Option, + active_memory_mutation_upper: Option, #[serde(default)] visible_turns: Option, #[serde(default)] @@ -384,13 +388,9 @@ pub fn run(store: &Store, params: &DreamParams) -> Result<(DreamResponse, bool)> offset: 0, }; let active_memory_mutation_scan = scheduler_cursor.as_ref().and_then(|cursor| { - if !cursor.lower_rowid_strict { - return None; - } - let lower = cursor.active_memory_version_lower.as_deref()?; - let upper = cursor.active_memory_version_upper.as_deref()?; - let upper_rowid = cursor.upper_rowids.as_ref()?.active_memory_records; - Some((lower, upper, upper_rowid)) + let lower = cursor.active_memory_mutation_lower.unwrap_or(0); + let upper = cursor.active_memory_mutation_upper?; + Some(("active_memory_mutation_sequence", lower, upper)) }); let mut records = match scheduler_cursor.as_ref() { Some(cursor) => query_memory_records_with_frontier( @@ -401,6 +401,7 @@ pub fn run(store: &Store, params: &DreamParams) -> Result<(DreamResponse, bool)> source_window_start, cursor.lower_bound_exclusive, cursor.lower_rowid_strict, + cursor.include_post_frontier_rowids, cursor_end, cursor .lower_rowids @@ -451,6 +452,7 @@ pub fn run(store: &Store, params: &DreamParams) -> Result<(DreamResponse, bool)> None, false, false, + false, cursor_end, None, scheduler_cursor.as_ref().and_then(|cursor| { @@ -853,6 +855,7 @@ fn imported_chatgpt_candidate_records( source_window_start, cursor.lower_bound_exclusive, cursor.lower_rowid_strict, + cursor.include_post_frontier_rowids, source_window_end, cursor .lower_rowids @@ -1073,18 +1076,28 @@ pub fn scheduler_cursor_for_run( .unwrap_or(now); cursor.upper_unbounded = false; cursor.upper_bound = Some(next_start.to_string()); - begin_active_memory_version_scan(&mut cursor); - cursor.upper_rowids = Some(scheduler_rowid_frontier(store)?); + let frontiers = scheduler_rowid_frontier(store)?; + begin_active_memory_mutation_scan( + &mut cursor, + frontiers.active_memory_mutation_sequence, + ); + cursor.upper_rowids = Some(frontiers); return encode_scheduler_cursor(&cursor); } - if cursor.active_memory_version_upper.is_none() { - // Older persisted cursors have no mutation frontier. Establish a - // baseline when they are next resumed so later restores and - // promotions can be detected without replaying every old row. - if cursor.active_memory_version_lower.is_none() { - cursor.active_memory_version_lower = Some(ids::now_rfc3339()); - } - cursor.active_memory_version_upper = Some(ids::now_rfc3339()); + if cursor.active_memory_mutation_upper.is_none() { + // Rewind pre-sequence cursors to a single captured source snapshot. + // Their historical rows have sequence zero; changes committed + // after this baseline receive a greater sequence and stay eligible. + let frontiers = scheduler_rowid_frontier(store)?; + cursor.active_memory_mutation_lower = Some( + cursor + .upper_rowids + .as_ref() + .map(|saved| saved.active_memory_mutation_sequence) + .unwrap_or(0), + ); + cursor.active_memory_mutation_upper = Some(frontiers.active_memory_mutation_sequence); + cursor.upper_rowids = Some(frontiers); return encode_scheduler_cursor(&cursor); } return Ok(previous.to_string()); @@ -1095,34 +1108,44 @@ pub fn scheduler_cursor_for_run( upper_unbounded, ..DreamSchedulerCursor::default() }; - cursor.active_memory_version_upper = Some(ids::now_rfc3339()); - cursor.upper_rowids = Some(scheduler_rowid_frontier(store)?); + let frontiers = scheduler_rowid_frontier(store)?; + let mutation_baseline = if previous.is_some() { + frontiers.active_memory_mutation_sequence + } else { + 0 + }; + cursor.active_memory_mutation_lower = Some(mutation_baseline); + cursor.active_memory_mutation_upper = Some(frontiers.active_memory_mutation_sequence); + cursor.upper_rowids = Some(frontiers); encode_scheduler_cursor(&cursor) } -fn begin_active_memory_version_scan(cursor: &mut DreamSchedulerCursor) { - if cursor.active_memory_version_lower.is_none() { - cursor.active_memory_version_lower = Some(ids::now_rfc3339()); +fn begin_active_memory_mutation_scan(cursor: &mut DreamSchedulerCursor, upper: i64) { + if cursor.active_memory_mutation_lower.is_none() { + cursor.active_memory_mutation_lower = + Some(cursor.active_memory_mutation_upper.unwrap_or(0)); } - cursor.active_memory_version_upper = Some(ids::now_rfc3339()); + cursor.active_memory_mutation_upper = Some(upper); } -fn complete_active_memory_version_scan(cursor: &mut DreamSchedulerCursor) { - if let Some(upper) = cursor.active_memory_version_upper.take() { - cursor.active_memory_version_lower = Some(upper); +fn complete_active_memory_mutation_scan(cursor: &mut DreamSchedulerCursor) { + if let Some(upper) = cursor.active_memory_mutation_upper.take() { + cursor.active_memory_mutation_lower = Some(upper); } } -fn next_active_memory_version_scan( +fn next_active_memory_mutation_scan( cursor: &mut DreamSchedulerCursor, previous: &DreamSchedulerCursor, + upper: i64, ) { - cursor.active_memory_version_lower = previous - .active_memory_version_upper - .clone() - .or_else(|| previous.active_memory_version_lower.clone()) - .or_else(|| Some(ids::now_rfc3339())); - cursor.active_memory_version_upper = Some(ids::now_rfc3339()); + cursor.active_memory_mutation_lower = Some( + previous + .active_memory_mutation_upper + .or(previous.active_memory_mutation_lower) + .unwrap_or(0), + ); + cursor.active_memory_mutation_upper = Some(upper); } /// Return a bounded scheduler cursor for a window that did not cover the @@ -1142,7 +1165,12 @@ pub fn scheduler_watermark_after( if cursor.upper_unbounded { if cursor.upper_rowids.is_none() || cursor.has_legacy_position() { cursor.clear_source_positions(); - cursor.upper_rowids = Some(scheduler_rowid_frontier(store)?); + let frontiers = scheduler_rowid_frontier(store)?; + begin_active_memory_mutation_scan( + &mut cursor, + frontiers.active_memory_mutation_sequence, + ); + cursor.upper_rowids = Some(frontiers); if cursor.lower_bound.is_some() { cursor.lower_bound_exclusive = false; if !cursor.lower_rowid_strict { @@ -1157,10 +1185,12 @@ pub fn scheduler_watermark_after( if cursor.upper_bound.is_none() { cursor.clear_source_positions(); cursor.upper_bound = Some(window.end.clone()); - cursor.upper_rowids = Some(scheduler_rowid_frontier(store)?); - if cursor.active_memory_version_upper.is_none() { - begin_active_memory_version_scan(&mut cursor); - } + let frontiers = scheduler_rowid_frontier(store)?; + begin_active_memory_mutation_scan( + &mut cursor, + frontiers.active_memory_mutation_sequence, + ); + cursor.upper_rowids = Some(frontiers); // The first bounded page used the ordinary ID ordering. Start the // persisted cursor at the beginning so the remaining pages use // the stable rowid ordering consistently. @@ -1168,16 +1198,18 @@ pub fn scheduler_watermark_after( } if cursor.upper_rowids.is_none() || cursor.has_legacy_position() { cursor.clear_source_positions(); - cursor.upper_rowids = Some(scheduler_rowid_frontier(store)?); + let frontiers = scheduler_rowid_frontier(store)?; + begin_active_memory_mutation_scan( + &mut cursor, + frontiers.active_memory_mutation_sequence, + ); + cursor.upper_rowids = Some(frontiers); if cursor.lower_bound.is_some() { cursor.lower_bound_exclusive = false; if !cursor.lower_rowid_strict { cursor.lower_rowids = Some(DreamRowidFrontier::default()); } } - if cursor.active_memory_version_upper.is_none() { - begin_active_memory_version_scan(&mut cursor); - } return Ok(Some(encode_scheduler_cursor(&cursor)?)); } update_scheduler_cursor(store, &mut cursor, window)?; @@ -1188,7 +1220,12 @@ pub fn scheduler_watermark_after( if previous_cursor.upper_unbounded { if previous_cursor.upper_rowids.is_none() || previous_cursor.has_legacy_position() { previous_cursor.clear_source_positions(); - previous_cursor.upper_rowids = Some(scheduler_rowid_frontier(store)?); + let frontiers = scheduler_rowid_frontier(store)?; + begin_active_memory_mutation_scan( + &mut previous_cursor, + frontiers.active_memory_mutation_sequence, + ); + previous_cursor.upper_rowids = Some(frontiers); if previous_cursor.lower_bound.is_some() { previous_cursor.lower_bound_exclusive = false; if !previous_cursor.lower_rowid_strict { @@ -1205,14 +1242,20 @@ pub fn scheduler_watermark_after( previous_cursor.upper_unbounded = false; previous_cursor.upper_rowids = None; previous_cursor.clear_source_positions(); - complete_active_memory_version_scan(&mut previous_cursor); + complete_active_memory_mutation_scan(&mut previous_cursor); + previous_cursor.include_post_frontier_rowids = false; return Ok(Some(encode_scheduler_cursor(&previous_cursor)?)); } if previous_cursor.upper_bound.is_some() && (previous_cursor.upper_rowids.is_none() || previous_cursor.has_legacy_position()) { previous_cursor.clear_source_positions(); - previous_cursor.upper_rowids = Some(scheduler_rowid_frontier(store)?); + let frontiers = scheduler_rowid_frontier(store)?; + begin_active_memory_mutation_scan( + &mut previous_cursor, + frontiers.active_memory_mutation_sequence, + ); + previous_cursor.upper_rowids = Some(frontiers); if previous_cursor.lower_bound.is_some() { previous_cursor.lower_bound_exclusive = false; if !previous_cursor.lower_rowid_strict { @@ -1232,11 +1275,17 @@ pub fn scheduler_watermark_after( lower_bound: Some(frontier), lower_rowids: previous_cursor.upper_rowids.clone(), lower_rowid_strict: true, + include_post_frontier_rowids: true, upper_bound: Some(now.to_string()), ..DreamSchedulerCursor::default() }; - next_active_memory_version_scan(&mut next, &previous_cursor); - next.upper_rowids = Some(scheduler_rowid_frontier(store)?); + let frontiers = scheduler_rowid_frontier(store)?; + next_active_memory_mutation_scan( + &mut next, + &previous_cursor, + frontiers.active_memory_mutation_sequence, + ); + next.upper_rowids = Some(frontiers); return Ok(Some(encode_scheduler_cursor(&next)?)); } previous_cursor.lower_bound = Some(frontier); @@ -1245,7 +1294,8 @@ pub fn scheduler_watermark_after( previous_cursor.upper_bound = None; previous_cursor.upper_unbounded = false; previous_cursor.clear_source_positions(); - complete_active_memory_version_scan(&mut previous_cursor); + complete_active_memory_mutation_scan(&mut previous_cursor); + previous_cursor.include_post_frontier_rowids = true; return Ok(Some(encode_scheduler_cursor(&previous_cursor)?)); } } @@ -1260,11 +1310,17 @@ pub fn scheduler_watermark_after( lower_bound_exclusive: false, lower_rowids: previous_cursor.upper_rowids.clone(), lower_rowid_strict: true, + include_post_frontier_rowids: true, upper_bound: Some(now.to_string()), ..DreamSchedulerCursor::default() }; - next_active_memory_version_scan(&mut next, &previous_cursor); - next.upper_rowids = Some(scheduler_rowid_frontier(store)?); + let frontiers = scheduler_rowid_frontier(store)?; + next_active_memory_mutation_scan( + &mut next, + &previous_cursor, + frontiers.active_memory_mutation_sequence, + ); + next.upper_rowids = Some(frontiers); return Ok(Some(encode_scheduler_cursor(&next)?)); } previous_cursor.lower_bound = Some(frontier); @@ -1274,7 +1330,8 @@ pub fn scheduler_watermark_after( previous_cursor.upper_bound = None; previous_cursor.upper_unbounded = false; previous_cursor.clear_source_positions(); - complete_active_memory_version_scan(&mut previous_cursor); + complete_active_memory_mutation_scan(&mut previous_cursor); + previous_cursor.include_post_frontier_rowids = true; return Ok(Some(encode_scheduler_cursor(&previous_cursor)?)); } } @@ -1317,6 +1374,11 @@ fn scheduler_rowid_frontier(store: &Store) -> Result { [], |row| row.get(0), )?, + active_memory_mutation_sequence: tx.query_row( + "SELECT sequence FROM active_memory_mutation_counter WHERE singleton = 1", + [], + |row| row.get(0), + )?, }) }) } @@ -1513,11 +1575,12 @@ fn query_dream_rows( start: Option<&str>, start_exclusive: bool, lower_rowid_strict: bool, + include_post_frontier_rowids: bool, lower_rowid: Option, end: Option<&str>, upper_rowid: Option, position: Option<&DreamSourceCursor>, - mutation_scan: Option<(&str, &str, i64)>, + mutation_scan: Option<(&str, i64, i64)>, limit: usize, mapper: F, ) -> Result> @@ -1559,10 +1622,7 @@ where let mut end_args: Vec> = Vec::new(); if let Some(end) = end { if let Some(upper_rowid) = upper_rowid { - end_filter = format!( - " AND ({timestamp_column} < ? OR ({timestamp_column} = ? AND {rowid_column} <= ?))" - ); - end_args.push(Box::new(end.to_string())); + end_filter = format!(" AND {timestamp_column} <= ? AND {rowid_column} <= ?"); end_args.push(Box::new(end.to_string())); end_args.push(Box::new(upper_rowid)); } else { @@ -1574,7 +1634,12 @@ where end_args.push(Box::new(upper_rowid)); } - if let Some((version_lower, version_upper, version_upper_rowid)) = mutation_scan { + let post_frontier_rowids = if include_post_frontier_rowids { + lower_rowid.zip(upper_rowid) + } else { + None + }; + if mutation_scan.is_some() || post_frontier_rowids.is_some() { sql.push_str(" AND (("); if start_filter.is_empty() { sql.push_str("1 = 1"); @@ -1582,14 +1647,27 @@ where sql.push_str(&start_filter); } sql.push_str(&end_filter); - sql.push_str(&format!( - ") OR ({timestamp_column} > ? AND {timestamp_column} <= ? AND {rowid_column} <= ?))" - )); + sql.push(')'); args.extend(start_args); args.extend(end_args); - args.push(Box::new(version_lower.to_string())); - args.push(Box::new(version_upper.to_string())); - args.push(Box::new(version_upper_rowid)); + if let Some((lower_rowid, upper_rowid)) = post_frontier_rowids { + sql.push_str(&format!(" OR ({rowid_column} > ? AND {rowid_column} <= ?")); + args.push(Box::new(lower_rowid)); + args.push(Box::new(upper_rowid)); + if let Some(end) = end { + sql.push_str(&format!(" AND {timestamp_column} <= ?")); + args.push(Box::new(end.to_string())); + } + sql.push(')'); + } + if let Some((mutation_column, lower, upper)) = mutation_scan { + sql.push_str(&format!( + " OR ({mutation_column} > ? AND {mutation_column} <= ?)" + )); + args.push(Box::new(lower)); + args.push(Box::new(upper)); + } + sql.push(')'); } else { if !start_filter.is_empty() { sql.push_str(" AND "); @@ -1638,6 +1716,7 @@ fn query_visible_turns_with_frontier( start: Option<&str>, start_exclusive: bool, lower_rowid_strict: bool, + include_post_frontier_rowids: bool, end: Option<&str>, lower_rowid: Option, upper_rowid: Option, @@ -1667,6 +1746,7 @@ fn query_visible_turns_with_frontier( start, start_exclusive, lower_rowid_strict, + include_post_frontier_rowids, lower_rowid, end, upper_rowid, @@ -1694,6 +1774,7 @@ fn query_conclusions_with_frontier( start: Option<&str>, start_exclusive: bool, lower_rowid_strict: bool, + include_post_frontier_rowids: bool, end: Option<&str>, lower_rowid: Option, upper_rowid: Option, @@ -1723,6 +1804,7 @@ fn query_conclusions_with_frontier( start, start_exclusive, lower_rowid_strict, + include_post_frontier_rowids, lower_rowid, end, upper_rowid, @@ -1753,6 +1835,7 @@ fn query_checkpoints_with_frontier( start: Option<&str>, start_exclusive: bool, lower_rowid_strict: bool, + include_post_frontier_rowids: bool, end: Option<&str>, lower_rowid: Option, upper_rowid: Option, @@ -1783,6 +1866,7 @@ fn query_checkpoints_with_frontier( start, start_exclusive, lower_rowid_strict, + include_post_frontier_rowids, lower_rowid, end, upper_rowid, @@ -1818,6 +1902,7 @@ fn query_memory_sources_with_frontier( start: Option<&str>, start_exclusive: bool, lower_rowid_strict: bool, + include_post_frontier_rowids: bool, end: Option<&str>, lower_rowid: Option, upper_rowid: Option, @@ -1843,6 +1928,7 @@ fn query_memory_sources_with_frontier( start, start_exclusive, lower_rowid_strict, + include_post_frontier_rowids, lower_rowid, end, upper_rowid, @@ -1873,13 +1959,14 @@ fn query_memory_records_with_frontier( start: Option<&str>, start_exclusive: bool, lower_rowid_strict: bool, + include_post_frontier_rowids: bool, end: Option<&str>, lower_rowid: Option, upper_rowid: Option, position: Option<&DreamSourceCursor>, limit: usize, include_archived: bool, - mutation_scan: Option<(&str, &str, i64)>, + mutation_scan: Option<(&str, i64, i64)>, ) -> Result> { let mut sql = "SELECT id FROM memory_records WHERE profile_id = ? AND workspace_id = ?" @@ -1892,6 +1979,13 @@ fn query_memory_records_with_frontier( sql.push_str(" AND repo_id = ?"); args.push(Box::new(repo_id.to_string())); } + if let Some((mutation_column, _, upper)) = mutation_scan { + // Query the active-memory state as it existed at the captured + // mutation frontier; a restore or promotion committed later must wait + // for the next scan even if its updated_at falls inside this band. + sql.push_str(&format!(" AND {mutation_column} <= ?")); + args.push(Box::new(upper)); + } if !include_archived { sql.push_str(" AND archived = 0"); } @@ -1906,6 +2000,7 @@ fn query_memory_records_with_frontier( start, start_exclusive, lower_rowid_strict, + include_post_frontier_rowids, lower_rowid, end, upper_rowid, @@ -1949,6 +2044,8 @@ fn build_evidence_window( scheduler_cursor.is_some_and(|cursor| cursor.lower_bound_exclusive); let cursor_lower_rowid_strict = scheduler_cursor.is_some_and(|cursor| cursor.lower_rowid_strict); + let cursor_post_frontier_rowids = + scheduler_cursor.is_some_and(|cursor| cursor.include_post_frontier_rowids); let visible_turns = if remaining == 0 { Vec::new() } else if let Some(cursor) = scheduler_cursor { @@ -1960,6 +2057,7 @@ fn build_evidence_window( start, cursor_start_exclusive, cursor_lower_rowid_strict, + cursor_post_frontier_rowids, cursor_end, cursor .lower_rowids @@ -1997,6 +2095,7 @@ fn build_evidence_window( start, cursor_start_exclusive, cursor_lower_rowid_strict, + cursor_post_frontier_rowids, cursor_end, cursor .lower_rowids @@ -2034,6 +2133,7 @@ fn build_evidence_window( start, cursor_start_exclusive, cursor_lower_rowid_strict, + cursor_post_frontier_rowids, cursor_end, cursor .lower_rowids @@ -2070,6 +2170,7 @@ fn build_evidence_window( start, cursor_start_exclusive, cursor_lower_rowid_strict, + cursor_post_frontier_rowids, cursor_end, cursor .lower_rowids diff --git a/src/store.rs b/src/store.rs index b1f493c..3b1b5bd 100644 --- a/src/store.rs +++ b/src/store.rs @@ -49,7 +49,7 @@ use crate::ids; use crate::protocol::DreamJobBudget; use crate::protocol::DreamJobProvider; -pub const STORAGE_SCHEMA_VERSION: i64 = 13; +pub const STORAGE_SCHEMA_VERSION: i64 = 14; const MIGRATION_INIT: &str = include_str!("../migrations/0001_init.sql"); const MIGRATION_FTS: &str = include_str!("../migrations/0002_fts.sql"); @@ -67,6 +67,8 @@ const MIGRATION_DREAM_JOBS: &str = include_str!("../migrations/0011_dream_jobs.s const MIGRATION_PORTABLE_BUNDLES: &str = include_str!("../migrations/0012_portable_bundles.sql"); const MIGRATION_CONSOLIDATION_PROPOSALS: &str = include_str!("../migrations/0013_consolidation_proposals.sql"); +const MIGRATION_SCHEDULER_MUTATION_FRONTIER: &str = + include_str!("../migrations/0014_scheduler_mutation_frontier.sql"); type SqlitePool = Pool; @@ -1638,6 +1640,17 @@ impl Store { conn.execute_batch(MIGRATION_DREAM_JOBS)?; conn.execute_batch(MIGRATION_PORTABLE_BUNDLES)?; conn.execute_batch(MIGRATION_CONSOLIDATION_PROPOSALS)?; + conn.execute_batch(MIGRATION_SCHEDULER_MUTATION_FRONTIER)?; + ensure_column( + &conn, + "memory_records", + "active_memory_mutation_sequence", + "INTEGER NOT NULL DEFAULT 0", + )?; + conn.execute_batch( + "CREATE INDEX IF NOT EXISTS idx_memory_records_active_mutation_sequence + ON memory_records(active_memory_mutation_sequence);", + )?; ensure_column( &conn, "consolidation_proposals", @@ -3145,24 +3158,47 @@ impl Store { workspace_id: Option<&str>, ids_to_promote: &[String], ) -> Result<(Vec, Vec)> { - let now = ids::now_rfc3339(); let mut promoted = Vec::new(); let mut not_found = Vec::new(); let mut conn = self.conn()?; let tx = conn.transaction()?; + // Serialize mutations before reading state so this transaction cannot + // keep a stale WAL snapshot while waiting for the mutation counter. + tx.execute( + "UPDATE active_memory_mutation_counter + SET sequence = sequence + WHERE singleton = 1", + [], + )?; + let mut mutation_sequence = None; + let mut now = None; { let mut stmt = tx.prepare( "UPDATE memory_records SET trust_state = 'trusted', trust_score = 1.0, - quarantine_reason = NULL, promoted_at = ?1, updated_at = ?1 - WHERE id = ?2 AND trust_state = 'quarantined'", + quarantine_reason = NULL, promoted_at = ?1, updated_at = ?1, + active_memory_mutation_sequence = ?2 + WHERE id = ?3 AND trust_state = 'quarantined'", )?; for id in ids_to_promote { if !self.scoped_record_exists(&tx, id, profile_id, workspace_id)? { not_found.push(id.clone()); continue; } - if stmt.execute(params![now, id])? > 0 { + if mutation_sequence.is_none() { + // The sequence is invisible to scheduler snapshots until + // this transaction commits, so it orders the eligibility + // change after snapshots that cannot yet observe it. + let sequence = next_active_memory_mutation_sequence(&tx)?; + mutation_sequence = Some(sequence); + now = Some(ids::now_rfc3339()); + } + let changed = stmt.execute(params![ + now.as_deref().expect("timestamp initialized with sequence"), + mutation_sequence.expect("sequence initialized before update"), + id + ])?; + if changed > 0 { promoted.push(id.clone()); } else { not_found.push(id.clone()); @@ -3276,16 +3312,27 @@ impl Store { patch_run_id: &str, reason: &str, ) -> Result<(Vec, Vec)> { - let now = ids::now_rfc3339(); let mut restored = Vec::new(); let mut not_found = Vec::new(); let mut conn = self.conn()?; let tx = conn.transaction()?; + // Serialize mutations before reading state so this transaction cannot + // keep a stale WAL snapshot while waiting for the mutation counter. + tx.execute( + "UPDATE active_memory_mutation_counter + SET sequence = sequence + WHERE singleton = 1", + [], + )?; + let mut mutation_sequence = None; + let mut now = None; { let mut select = tx.prepare("SELECT metadata, archived FROM memory_records WHERE id = ?1")?; let mut update = tx.prepare( - "UPDATE memory_records SET archived = 0, updated_at = ?1, metadata = ?2 WHERE id = ?3 AND archived = 1", + "UPDATE memory_records SET archived = 0, updated_at = ?1, metadata = ?2, + active_memory_mutation_sequence = ?3 + WHERE id = ?4 AND archived = 1", )?; for id in ids_to_restore { if !self.scoped_record_exists(&tx, id, profile_id, workspace_id)? { @@ -3312,13 +3359,24 @@ impl Store { not_found.push(id.clone()); continue; } + if mutation_sequence.is_none() { + // Acquire the writer lock and allocate a commit-ordered + // sequence before taking the mutation timestamp. + let sequence = next_active_memory_mutation_sequence(&tx)?; + mutation_sequence = Some(sequence); + now = Some(ids::now_rfc3339()); + } + let updated_at = now.as_deref().expect("timestamp initialized with sequence"); if let Some(obj) = metadata.as_object_mut() { obj.insert("state".to_string(), Value::String("restored".to_string())); obj.insert( "policy_outcome".to_string(), Value::String("restored".to_string()), ); - obj.insert("restored_at".to_string(), Value::String(now.clone())); + obj.insert( + "restored_at".to_string(), + Value::String(updated_at.to_string()), + ); obj.insert( "restored_by_patch_run_id".to_string(), Value::String(patch_run_id.to_string()), @@ -3328,7 +3386,13 @@ impl Store { Value::String(reason.to_string()), ); } - if update.execute(params![now, metadata.to_string(), id])? > 0 { + if update.execute(params![ + updated_at, + metadata.to_string(), + mutation_sequence.expect("sequence initialized before update"), + id + ])? > 0 + { restored.push(id.clone()); } } @@ -5156,6 +5220,20 @@ fn normalized_trust_value(value: &str) -> String { value.trim().to_ascii_lowercase().replace('-', "_") } +fn next_active_memory_mutation_sequence(tx: &rusqlite::Transaction<'_>) -> Result { + tx.execute( + "UPDATE active_memory_mutation_counter + SET sequence = sequence + 1 + WHERE singleton = 1", + [], + )?; + Ok(tx.query_row( + "SELECT sequence FROM active_memory_mutation_counter WHERE singleton = 1", + [], + |row| row.get(0), + )?) +} + fn ensure_column( conn: &rusqlite::Connection, table: &str, diff --git a/tests/dream_command_scheduler.rs b/tests/dream_command_scheduler.rs index 9278784..26dd30c 100644 --- a/tests/dream_command_scheduler.rs +++ b/tests/dream_command_scheduler.rs @@ -443,9 +443,15 @@ fn scheduled_command_excludes_archived_sources_from_preview() { }) .unwrap(); - service + let _second = service .scheduled_dream(Some("2040-01-01T00:00:00Z".into())) .unwrap(); + // This invocation drains the first fixed-frontier band. The late row is + // eligible in the next band, which is opened by the following invocation. + let third = service + .scheduled_dream(Some("2050-01-01T00:00:00Z".into())) + .unwrap(); + assert_eq!(third.status, "ok_with_limits"); let body = std::fs::read_to_string(capture_path).unwrap(); assert!(body.contains("Post-watermark source"), "body={body}"); assert!(!body.contains("Pre-watermark source"), "body={body}"); diff --git a/tests/fixtures/status.response.json b/tests/fixtures/status.response.json index 6dc8af1..db9fd36 100644 --- a/tests/fixtures/status.response.json +++ b/tests/fixtures/status.response.json @@ -4,7 +4,7 @@ "provider_name": "codex-memoryd", "provider_version": "0.1.0", "api_version": "v1", - "storage_schema_version": 8, + "storage_schema_version": 14, "status": "local_only", "storage": { "kind": "sqlite", diff --git a/tests/scheduler_review_boundaries.rs b/tests/scheduler_review_boundaries.rs index 5a38051..35fd8d6 100644 --- a/tests/scheduler_review_boundaries.rs +++ b/tests/scheduler_review_boundaries.rs @@ -1,6 +1,7 @@ use std::collections::BTreeSet; use std::io::{Read, Write}; use std::net::TcpListener; +use std::sync::mpsc; use std::thread; use std::time::{Duration, Instant}; @@ -147,6 +148,32 @@ fn evidence_ids(run: &codex_memoryd::protocol::DreamResponse) -> BTreeSet codex_memoryd::protocol::DreamResponse { + codex_memoryd::dream::run( + store, + &codex_memoryd::dream::DreamParams { + profile: Profile::Personal, + workspace: "workspace", + repo_id: None, + mode: "preview", + now, + source_window_end: Some(now), + recency_cutoff: Some(cursor), + include_archived_sources: false, + max_records: 10, + max_candidates: Some(10), + patch_run_id: None, + deadline: None, + }, + ) + .expect("run scheduler cursor") + .0 +} + fn insert_chatgpt_turn( store: &Store, session_id: &str, @@ -360,7 +387,7 @@ fn scheduler_cursor_keeps_lower_bound_across_all_streams_and_ties() { ] { assert!( !seen.contains(old_id), - "selected below-bound source {old_id}" + "selected below-bound source {old_id}; seen={seen:?}" ); } for current_id in [ @@ -624,6 +651,68 @@ fn scheduler_cursor_picks_up_late_row_before_frontier_keyset_position() { .contains("visible-a-late-frontier")); } +#[test] +fn scheduler_bounded_band_catches_backdated_rows_after_its_rowid_frontier() { + let store = Store::open(":memory:").expect("store"); + let service = Service::new(store.clone(), scheduler_config("workspace", 1, false)); + service + .scheduled_dream(Some(BEFORE_FRONTIER.into())) + .expect("seed watermark"); + store + .ensure_workspace(PROFILE, "workspace") + .expect("workspace"); + insert_chatgpt_turn( + &store, + "backdated-frontier-session", + "visible-position-before-backdate", + "generic evidence at the current bounded position", + "2029-12-31T23:59:00Z", + None, + ); + insert_chatgpt_turn( + &store, + "backdated-tail-session", + "visible-tail-before-backdate", + "generic older evidence that drains the bounded band", + "2029-12-31T23:58:00Z", + None, + ); + + let first = service + .scheduled_dream(Some(RUN_NOW.into())) + .expect("scan first bounded page"); + assert!(evidence_ids(first.run.as_ref().expect("first run")) + .contains("visible-position-before-backdate")); + + insert_chatgpt_turn( + &store, + "late-backdated-session", + "visible-backdated-after-frontier", + "generic late row with a backdated timestamp", + "2029-12-31T23:59:30Z", + None, + ); + + let drain = service + .scheduled_dream(Some("2030-01-01T00:00:01Z".into())) + .expect("drain original fixed band"); + let drained = evidence_ids(drain.run.as_ref().expect("drain run")); + assert!(drained.contains("visible-tail-before-backdate")); + assert!( + !drained.contains("visible-backdated-after-frontier"), + "rows inserted after the fixed upper rowid frontier must stay out of the old band" + ); + + service + .scheduled_dream(Some("2030-01-01T00:00:02Z".into())) + .expect("complete original bounded band"); + let next_band = service + .scheduled_dream(Some("2030-01-01T00:00:03Z".into())) + .expect("scan post-frontier rows in the next band"); + assert!(evidence_ids(next_band.run.as_ref().expect("next-band run")) + .contains("visible-backdated-after-frontier")); +} + #[test] fn scheduler_cursor_rewinds_persisted_id_only_cursor_before_rowid_paging() { let store = Store::open(":memory:").expect("store"); @@ -690,8 +779,12 @@ fn scheduler_cursor_rewinds_persisted_id_only_cursor_before_rowid_paging() { let resumed = service .scheduled_dream(Some("2030-01-01T00:00:03Z".into())) .expect("continue beyond the legacy ID keyset position"); - assert!(evidence_ids(resumed.run.as_ref().expect("resumed run")) - .contains("visible-a-legacy-unvisited")); + assert!( + evidence_ids(resumed.run.as_ref().expect("resumed run")) + .contains("visible-a-legacy-unvisited"), + "resumed run={:?}", + resumed.run + ); } #[test] @@ -912,6 +1005,25 @@ fn scheduler_scan_emits_restored_and_promoted_memory_versions() { .0, vec![promoted_id.clone()] ); + let mutation_sequences = store + .transaction(|tx| { + let restored_sequence: i64 = tx.query_row( + "SELECT active_memory_mutation_sequence FROM memory_records WHERE id = ?1", + rusqlite::params![restored_id], + |row| row.get(0), + )?; + let promoted_sequence: i64 = tx.query_row( + "SELECT active_memory_mutation_sequence FROM memory_records WHERE id = ?1", + rusqlite::params![promoted_id], + |row| row.get(0), + )?; + Ok((restored_sequence, promoted_sequence)) + }) + .expect("read active-memory mutation sequences"); + assert!( + mutation_sequences.0 > 0 && mutation_sequences.1 > mutation_sequences.0, + "restore and promotion must receive increasing commit-ordered sequences: {mutation_sequences:?}" + ); let first_changed_page = service .scheduled_dream(Some(RUN_NOW.into())) @@ -955,6 +1067,262 @@ fn scheduler_scan_emits_restored_and_promoted_memory_versions() { .contains("Synthetic promoted record reaches the scheduled provider.")); } +#[test] +fn scheduler_scan_catches_a_restore_committed_after_its_snapshot() { + let temp = tempfile::tempdir().expect("temporary database directory"); + let db_path = temp.path().join("scheduler-race.db"); + let store = Store::open(&db_path).expect("store"); + let writer_store = Store::open(&db_path).expect("independent writer pool"); + store + .ensure_workspace(PROFILE, "workspace") + .expect("workspace"); + let archived_id = insert_memory_record( + &store, + "Synthetic record is restored after the scheduler snapshot.", + json!({"origin": "fixture"}), + ); + store + .archive_records_with_metadata( + PROFILE, + Some("workspace"), + std::slice::from_ref(&archived_id), + "historical", + "synthetic race fixture", + Some("race-fixture-run"), + ) + .expect("archive synthetic record"); + let quarantined_id = insert_memory_record( + &store, + "Synthetic record is promoted after the scheduler snapshot.", + json!({"origin": "fixture"}), + ); + store + .quarantine_records( + PROFILE, + Some("workspace"), + std::slice::from_ref(&quarantined_id), + "synthetic race fixture", + ) + .expect("quarantine synthetic record"); + + let initial_cursor = + codex_memoryd::dream::scheduler_cursor_for_run(&store, None, "2000-01-01T00:00:00Z", true) + .expect("capture initial frontier"); + let initial_run = run_scheduler_cursor(&store, &initial_cursor, "2000-01-01T00:00:00Z"); + let completed_initial = codex_memoryd::dream::scheduler_watermark_after( + &store, + Some(&initial_cursor), + &initial_run.evidence_window, + "2000-01-01T00:00:00Z", + false, + ) + .expect("complete initial scan") + .expect("completed cursor"); + + let (staged_tx, staged_rx) = mpsc::channel(); + let (commit_tx, commit_rx) = mpsc::channel(); + let writer_record_id = archived_id.clone(); + let writer_quarantined_id = quarantined_id.clone(); + let writer = thread::spawn(move || { + writer_store + .transaction(|tx| { + // This deliberately old timestamp models restore code that + // computes updated_at before waiting to commit its transaction. + tx.execute( + "UPDATE active_memory_mutation_counter + SET sequence = sequence + 1 + WHERE singleton = 1", + [], + )?; + let mutation_sequence: i64 = tx.query_row( + "SELECT sequence FROM active_memory_mutation_counter WHERE singleton = 1", + [], + |row| row.get(0), + )?; + tx.execute( + "UPDATE memory_records + SET archived = 0, updated_at = ?1, + active_memory_mutation_sequence = ?2 + WHERE id = ?3", + rusqlite::params!["1990-01-01T00:00:00Z", mutation_sequence, writer_record_id], + )?; + tx.execute( + "UPDATE memory_records + SET trust_state = 'trusted', updated_at = ?1, + active_memory_mutation_sequence = ?2 + WHERE id = ?3", + rusqlite::params![ + "1990-01-01T00:00:00Z", + mutation_sequence, + writer_quarantined_id + ], + )?; + staged_tx.send(()).expect("signal staged restore"); + commit_rx + .recv_timeout(Duration::from_secs(10)) + .expect("wait for scheduler snapshot"); + Ok(()) + }) + .expect("commit staged restore"); + }); + staged_rx + .recv_timeout(Duration::from_secs(5)) + .expect("restore remains uncommitted"); + + let in_flight_cursor = codex_memoryd::dream::scheduler_cursor_for_run( + &store, + Some(&completed_initial), + RUN_NOW, + false, + ) + .expect("capture frontier while restore is uncommitted"); + let in_flight = run_scheduler_cursor(&store, &in_flight_cursor, RUN_NOW); + assert!(!evidence_ids(&in_flight).contains(&archived_id)); + let completed_in_flight = codex_memoryd::dream::scheduler_watermark_after( + &store, + Some(&in_flight_cursor), + &in_flight.evidence_window, + RUN_NOW, + false, + ) + .expect("complete scan before restore commit") + .expect("completed in-flight cursor"); + + commit_tx.send(()).expect("allow restore commit"); + writer.join().expect("restore writer thread"); + + let after_restore_cursor = codex_memoryd::dream::scheduler_cursor_for_run( + &store, + Some(&completed_in_flight), + "2030-01-01T00:00:01Z", + false, + ) + .expect("capture next frontier after restore commit"); + let after_restore = run_scheduler_cursor(&store, &after_restore_cursor, "2030-01-01T00:00:01Z"); + assert_eq!( + evidence_ids(&after_restore), + BTreeSet::from([archived_id, quarantined_id]), + "restores and promotions committed after a read snapshot must be selected even when updated_at predates that snapshot" + ); +} + +#[test] +fn scheduler_frontier_excludes_mutations_committed_after_capture_until_next_scan() { + let temp = tempfile::tempdir().expect("temporary database directory"); + let db_path = temp.path().join("scheduler-frontier-race.db"); + let store = Store::open(&db_path).expect("store"); + let writer_store = Store::open(&db_path).expect("independent writer pool"); + store + .ensure_workspace(PROFILE, "workspace") + .expect("workspace"); + + let initial_cursor = + codex_memoryd::dream::scheduler_cursor_for_run(&store, None, "2000-01-01T00:00:00Z", true) + .expect("capture empty initial frontier"); + let initial_run = run_scheduler_cursor(&store, &initial_cursor, "2000-01-01T00:00:00Z"); + let completed_initial = codex_memoryd::dream::scheduler_watermark_after( + &store, + Some(&initial_cursor), + &initial_run.evidence_window, + "2000-01-01T00:00:00Z", + false, + ) + .expect("complete initial scan") + .expect("completed cursor"); + + let archived_id = insert_memory_record( + &store, + "Synthetic archived row becomes active after frontier capture.", + json!({"origin": "fixture"}), + ); + store + .archive_records_with_metadata( + PROFILE, + Some("workspace"), + std::slice::from_ref(&archived_id), + "historical", + "synthetic post-frontier fixture", + Some("post-frontier-fixture-run"), + ) + .expect("archive synthetic record"); + let quarantined_id = insert_memory_record( + &store, + "Synthetic quarantined row becomes active after frontier capture.", + json!({"origin": "fixture"}), + ); + store + .quarantine_records( + PROFILE, + Some("workspace"), + std::slice::from_ref(&quarantined_id), + "synthetic post-frontier fixture", + ) + .expect("quarantine synthetic record"); + + let in_flight_cursor = codex_memoryd::dream::scheduler_cursor_for_run( + &store, + Some(&completed_initial), + RUN_NOW, + false, + ) + .expect("capture bounded frontier before eligibility changes"); + let writer_archived_id = archived_id.clone(); + let writer_quarantined_id = quarantined_id.clone(); + writer_store + .transaction(|tx| { + tx.execute( + "UPDATE active_memory_mutation_counter + SET sequence = sequence + 1 WHERE singleton = 1", + [], + )?; + let sequence: i64 = tx.query_row( + "SELECT sequence FROM active_memory_mutation_counter WHERE singleton = 1", + [], + |row| row.get(0), + )?; + tx.execute( + "UPDATE memory_records SET archived = 0, updated_at = ?1, + active_memory_mutation_sequence = ?2 WHERE id = ?3", + rusqlite::params![RUN_NOW, sequence, writer_archived_id], + )?; + tx.execute( + "UPDATE memory_records SET trust_state = 'trusted', updated_at = ?1, + active_memory_mutation_sequence = ?2 WHERE id = ?3", + rusqlite::params![RUN_NOW, sequence, writer_quarantined_id], + )?; + Ok(()) + }) + .expect("commit post-frontier restore and promotion"); + + let current_scan = run_scheduler_cursor(&store, &in_flight_cursor, RUN_NOW); + assert!( + evidence_ids(¤t_scan).is_empty(), + "a commit after the captured sequence frontier must not leak into the current band" + ); + let completed_band = codex_memoryd::dream::scheduler_watermark_after( + &store, + Some(&in_flight_cursor), + ¤t_scan.evidence_window, + RUN_NOW, + false, + ) + .expect("complete bounded scan") + .expect("completed bounded cursor"); + let next_cursor = codex_memoryd::dream::scheduler_cursor_for_run( + &store, + Some(&completed_band), + "2030-01-01T00:00:01Z", + false, + ) + .expect("capture next mutation frontier"); + let next_scan = run_scheduler_cursor(&store, &next_cursor, "2030-01-01T00:00:01Z"); + assert_eq!( + evidence_ids(&next_scan), + BTreeSet::from([archived_id, quarantined_id]), + "the next scan must select both committed eligibility changes" + ); +} + #[test] fn scheduler_idle_preview_does_not_emit_candidates_from_comparison_context() { let store = Store::open(":memory:").expect("store"); diff --git a/tests/schema_upgrade.rs b/tests/schema_upgrade.rs index 73a5902..2c88e1d 100644 --- a/tests/schema_upgrade.rs +++ b/tests/schema_upgrade.rs @@ -225,6 +225,27 @@ fn upgrades_v1_database_and_backfills_columns() { // Records preserved. assert_eq!(store.count_records().unwrap(), 2, "v1 records preserved"); + let mutation_state = store + .transaction(|tx| { + let record_sequence: i64 = tx.query_row( + "SELECT active_memory_mutation_sequence FROM memory_records WHERE id = 'mem_1'", + [], + |row| row.get(0), + )?; + let sequence: i64 = tx.query_row( + "SELECT sequence FROM active_memory_mutation_counter WHERE singleton = 1", + [], + |row| row.get(0), + )?; + Ok((record_sequence, sequence)) + }) + .expect("scheduler mutation frontier migration"); + assert_eq!( + mutation_state, + (0, 0), + "legacy memory rows are baseline data" + ); + // The read path needs the back-filled columns (subject_id, episode_id, // trust_state, ...). If `ensure_*` did not run, this query errors. let rec = store From a97105e614926e57d55d82a508271531823a23df Mon Sep 17 00:00:00 2001 From: Josh Yorko Date: Wed, 14 Oct 2026 21:00:59 +0000 Subject: [PATCH 16/18] fix: preserve scheduler mutations across frontiers --- docs/dreamer-native-provider.md | 22 +- src/dream.rs | 1 + src/store.rs | 99 +++++- tests/scheduler_review_boundaries.rs | 492 +++++++++++++++++++++++++-- 4 files changed, 559 insertions(+), 55 deletions(-) diff --git a/docs/dreamer-native-provider.md b/docs/dreamer-native-provider.md index 156accf..e03fd79 100644 --- a/docs/dreamer-native-provider.md +++ b/docs/dreamer-native-provider.md @@ -71,14 +71,20 @@ rejection in audit history. Completed unbounded simulated-clock scans retain their rowid frontier even when the simulated clock does not advance. Each bounded band caps every source stream at its captured rowid frontier. The next band also checks rows inserted after that cap, including rows whose source -timestamp was backdated below the prior band's end. Restore and promotion -operations allocate a per-record mutation sequence in the same SQLite -transaction as the eligibility change; the scheduler captures that sequence -with the rowid frontier so an uncommitted mutation cannot be skipped because -its wall-clock timestamp predates a scan. Previously selected active records -may provide comparison context, but only a record selected in the current -evidence window can emit stale or supersession candidates. Process-group -children are terminated; orphan reaping remains the operating system's job. +timestamp was backdated below the prior band's end. The first bounded band +after an unbounded scan uses the same post-frontier catch-up, so arrivals +between that snapshot and the next band are not lost below the timestamp +watermark. Existing-row updates that affect scheduler-visible evidence and +eligibility changes allocate a per-record mutation sequence in the same SQLite +transaction: provenance upserts, supersession and invalidation, consolidation +reuse and undo, restore, and promotion. The scheduler captures that sequence +with the rowid frontier so an uncommitted update cannot be skipped because its +wall-clock timestamp predates a scan. Archiving and quarantine remove rows from +eligible selection; a later reactivation advances the sequence. Previously +selected active records may provide comparison context, but only a record +selected in the current evidence window can emit stale or supersession +candidates. Process-group children are terminated; orphan reaping remains the +operating system's job. Managed readiness accepts `local_only`, or `degraded` only when the reported store is writable SQLite at the current schema, the exact configured diff --git a/src/dream.rs b/src/dream.rs index 5ab508c..d0ab9b1 100644 --- a/src/dream.rs +++ b/src/dream.rs @@ -1076,6 +1076,7 @@ pub fn scheduler_cursor_for_run( .unwrap_or(now); cursor.upper_unbounded = false; cursor.upper_bound = Some(next_start.to_string()); + cursor.include_post_frontier_rowids = true; let frontiers = scheduler_rowid_frontier(store)?; begin_active_memory_mutation_scan( &mut cursor, diff --git a/src/store.rs b/src/store.rs index 3b1b5bd..763274c 100644 --- a/src/store.rs +++ b/src/store.rs @@ -963,6 +963,7 @@ impl Store { .ok_or_else(|| Error::invalid_request("consolidation batch profile is invalid"))?; enforce_consolidation_budget(&batch, policy)?; let mut applied = Vec::new(); + let mut active_memory_mutation_sequence = None; for decision in decisions { let Some(candidate) = batch.candidates.iter().find(|item| item.candidate_id == decision.candidate_id) else { return Err(Error::invalid_request("consolidation decision references unknown candidate")); @@ -1107,6 +1108,10 @@ impl Store { (id, false) }; if reused_existing { + if active_memory_mutation_sequence.is_none() { + active_memory_mutation_sequence = + Some(next_active_memory_mutation_sequence(tx)?); + } let (raw_source_ids, raw_metadata): (String, String) = tx.query_row( "SELECT source_ids, metadata FROM memory_records WHERE id = ?1", rusqlite::params![&record_id], @@ -1130,8 +1135,9 @@ impl Store { temporal_state = COALESCE(?2, temporal_state), valid_until = COALESCE(?3, valid_until), historical_reason = COALESCE(?4, historical_reason), - metadata = ?5, updated_at = ?6, archived = 0 - WHERE id = ?7 AND (archived = 0 OR ?8 = 1) + metadata = ?5, updated_at = ?6, archived = 0, + active_memory_mutation_sequence = ?7 + WHERE id = ?8 AND (archived = 0 OR ?9 = 1) AND COALESCE(temporal_state, 'current') = 'current'", rusqlite::params![ serde_json::to_string(&source_ids)?, @@ -1140,6 +1146,8 @@ impl Store { candidate.historical_reason.as_deref(), metadata.to_string(), ids::now_rfc3339(), + active_memory_mutation_sequence + .expect("sequence initialized before reused-record update"), &record_id, decision.supersedes.is_empty(), ], @@ -1204,6 +1212,7 @@ impl Store { .collect::, _>>()?; drop(stmt); let mut reverted = Vec::new(); + let mut reactivation_sequence = None; for replacement_id in &ids { let old_records = tx .prepare( @@ -1254,11 +1263,21 @@ impl Store { let previous_historical_reason = previous .get("historical_reason") .and_then(Value::as_str); + let mutation_sequence = if previous_archived == 0 { + if reactivation_sequence.is_none() { + reactivation_sequence = + Some(next_active_memory_mutation_sequence(tx)?); + } + reactivation_sequence + } else { + None + }; tx.execute( "UPDATE memory_records SET archived = ?1, temporal_state = ?2, superseded_by = ?3, valid_until = ?4, historical_reason = ?5, - updated_at = ?6, metadata = ?7 + updated_at = ?6, metadata = ?7, + active_memory_mutation_sequence = COALESCE(?9, active_memory_mutation_sequence) WHERE id = ?8", rusqlite::params![ previous_archived, @@ -1269,6 +1288,7 @@ impl Store { ids::now_rfc3339(), previous_metadata.to_string(), old_id, + mutation_sequence, ], )?; } @@ -3045,12 +3065,45 @@ impl Store { merged.push(sid.clone()); } } - let now = ids::now_rfc3339(); - let conn = self.conn()?; - conn.execute( - "UPDATE memory_records SET source_ids = ?1, metadata = ?2, updated_at = ?3 WHERE id = ?4", - params![serde_json::to_string(&merged)?, merged_metadata.to_string(), if new.source_ids.is_empty() { existing.updated_at.clone() } else { now }, existing.id], - )?; + let existing_id = existing.id.clone(); + let merged_source_ids = serde_json::to_string(&merged)?; + let merged_metadata = merged_metadata.to_string(); + self.transaction_immediate(|tx| { + let is_scheduler_eligible: bool = tx.query_row( + "SELECT EXISTS( + SELECT 1 FROM memory_records + WHERE id = ?1 AND archived = 0 + AND sensitivity != 'secret_blocked' + AND trust_state != 'quarantined' + )", + params![existing_id], + |row| row.get(0), + )?; + let mutation_sequence = if is_scheduler_eligible { + Some(next_active_memory_mutation_sequence(tx)?) + } else { + None + }; + let updated_at = if new.source_ids.is_empty() { + existing.updated_at.clone() + } else { + ids::now_rfc3339() + }; + tx.execute( + "UPDATE memory_records + SET source_ids = ?1, metadata = ?2, updated_at = ?3, + active_memory_mutation_sequence = COALESCE(?4, active_memory_mutation_sequence) + WHERE id = ?5", + params![ + merged_source_ids, + merged_metadata, + updated_at, + mutation_sequence, + existing_id + ], + )?; + Ok(()) + })?; } return Ok(UpsertOutcome::Skipped(existing.id)); } @@ -3468,13 +3521,14 @@ impl Store { now: &str, ) -> Result> { let mut conn = self.conn()?; - let tx = conn.transaction()?; + let tx = conn.transaction_with_behavior(rusqlite::TransactionBehavior::Immediate)?; if !self.scoped_record_exists(&tx, old_id, profile_id, Some(workspace_id))? || !self.scoped_record_exists(&tx, new_id, profile_id, Some(workspace_id))? { tx.commit()?; return Ok(None); } + let mutation_sequence = next_active_memory_mutation_sequence(&tx)?; tx.execute( "UPDATE memory_records @@ -3482,9 +3536,18 @@ impl Store { superseded_by = ?4, valid_until = COALESCE(valid_until, ?5), historical_reason = ?6, - updated_at = ?5 + updated_at = ?5, + active_memory_mutation_sequence = ?7 WHERE profile_id = ?1 AND workspace_id = ?2 AND id = ?3", - params![profile_id, workspace_id, old_id, new_id, now, reason], + params![ + profile_id, + workspace_id, + old_id, + new_id, + now, + reason, + mutation_sequence + ], )?; let raw: String = tx.query_row( @@ -3498,7 +3561,8 @@ impl Store { supersedes.push(old_id.to_string()); tx.execute( "UPDATE memory_records - SET supersedes = ?4, updated_at = ?5 + SET supersedes = ?4, updated_at = ?5, + active_memory_mutation_sequence = ?6 WHERE profile_id = ?1 AND workspace_id = ?2 AND id = ?3", params![ profile_id, @@ -3506,6 +3570,7 @@ impl Store { new_id, serde_json::to_string(&supersedes)?, now, + mutation_sequence, ], )?; } @@ -3524,20 +3589,22 @@ impl Store { now: &str, ) -> Result> { let mut conn = self.conn()?; - let tx = conn.transaction()?; + let tx = conn.transaction_with_behavior(rusqlite::TransactionBehavior::Immediate)?; if !self.scoped_record_exists(&tx, id, profile_id, Some(workspace_id))? { tx.commit()?; return Ok(None); } + let mutation_sequence = next_active_memory_mutation_sequence(&tx)?; tx.execute( "UPDATE memory_records SET temporal_state = 'invalidated', invalidated_at = ?4, valid_until = COALESCE(valid_until, ?4), historical_reason = ?5, - updated_at = ?4 + updated_at = ?4, + active_memory_mutation_sequence = ?6 WHERE profile_id = ?1 AND workspace_id = ?2 AND id = ?3", - params![profile_id, workspace_id, id, now, reason], + params![profile_id, workspace_id, id, now, reason, mutation_sequence], )?; tx.commit()?; drop(conn); diff --git a/tests/scheduler_review_boundaries.rs b/tests/scheduler_review_boundaries.rs index 35fd8d6..19fa1fc 100644 --- a/tests/scheduler_review_boundaries.rs +++ b/tests/scheduler_review_boundaries.rs @@ -6,6 +6,11 @@ use std::thread; use std::time::{Duration, Instant}; use codex_memoryd::config::Config; +use codex_memoryd::consolidation::{ + ConsolidationBatch, ConsolidationBudget, ConsolidationCandidate, ConsolidationDecision, + ConsolidationMode, ConsolidationOperation, ConsolidationPolicy, ConsolidationSourceCursor, + CONSOLIDATION_CONTRACT_VERSION, +}; use codex_memoryd::domain::{ Checkpoint, Conclusion, Portability, Profile, Sensitivity, VisibleTurn, }; @@ -36,40 +41,48 @@ fn scheduler_config(workspace: &str, max_batch_size: usize, automatic_apply: boo } fn insert_memory_record(store: &Store, content: &str, metadata: serde_json::Value) -> String { - let class = policy::classify(content, Profile::Personal, false); match store - .upsert_record(&NewRecord { - profile_id: PROFILE.into(), - workspace_id: "workspace".into(), - repo_id: None, - subject_id: None, - episode_id: None, - scope: class.scope, - record_type: class.record_type, - content: content.into(), - related_files: vec![], - tags: vec![], - sensitivity: Sensitivity::Personal, - portability: Portability::ProfileOnly, - confidence: class.confidence, - source_ids: vec![], - content_hash: ids::exact_content_hash( - PROFILE, - "workspace", - None, - class.record_type.as_str(), - class.scope.as_str(), - content, - ), - supersedes: vec![], - metadata, - }) + .upsert_record(&new_memory_record(content, metadata, vec![])) .expect("synthetic memory record") { UpsertOutcome::Created(id) | UpsertOutcome::Skipped(id) => id, } } +fn new_memory_record( + content: &str, + metadata: serde_json::Value, + source_ids: Vec, +) -> NewRecord { + let class = policy::classify(content, Profile::Personal, false); + NewRecord { + profile_id: PROFILE.into(), + workspace_id: "workspace".into(), + repo_id: None, + subject_id: None, + episode_id: None, + scope: class.scope, + record_type: class.record_type, + content: content.into(), + related_files: vec![], + tags: vec![], + sensitivity: Sensitivity::Personal, + portability: Portability::ProfileOnly, + confidence: class.confidence, + source_ids, + content_hash: ids::exact_content_hash( + PROFILE, + "workspace", + None, + class.record_type.as_str(), + class.scope.as_str(), + content, + ), + supersedes: vec![], + metadata, + } +} + fn capture_provider_requests(expected: usize) -> (String, thread::JoinHandle>) { let listener = TcpListener::bind("127.0.0.1:0").expect("provider listener"); listener @@ -174,6 +187,77 @@ fn run_scheduler_cursor( .0 } +fn apply_synthetic_consolidation( + store: &Store, + batch_id: &str, + content: &str, + supersedes: &[String], +) -> Vec { + let policy = ConsolidationPolicy { + contract_version: CONSOLIDATION_CONTRACT_VERSION.into(), + mode: ConsolidationMode::Automatic, + scopes: vec!["personal".into()], + claim_classes: vec!["preference".into()], + source_classes: vec!["user_statement".into()], + operations: vec![ConsolidationOperation::AdoptStatement], + budget: ConsolidationBudget { + max_candidates: 10, + max_source_records: 10, + max_provider_calls: 1, + max_input_bytes: 4096, + max_output_bytes: 4096, + }, + retention_days: 30, + semantic_validation: false, + legacy_metadata: None, + }; + let candidate = ConsolidationCandidate { + candidate_id: format!("{batch_id}-candidate"), + output_digest: format!("{batch_id}-output"), + claim: content.into(), + claim_class: "preference".into(), + subject: "user".into(), + inferred: false, + source_ids: vec![format!("{batch_id}-synthetic-source")], + supporting_spans: vec![content.into()], + supersedes: supersedes.to_vec(), + temporal_state: None, + valid_until: None, + historical_reason: None, + }; + let batch = ConsolidationBatch { + contract_version: CONSOLIDATION_CONTRACT_VERSION.into(), + batch_id: batch_id.into(), + policy_digest: policy.digest(), + profile: PROFILE.into(), + workspace: "workspace".into(), + repo_id: None, + scope: "personal".into(), + source_cursor: ConsolidationSourceCursor { + since: None, + until: Some(RUN_NOW.into()), + explicit_since: false, + }, + snapshot_digest: format!("{batch_id}-snapshot"), + candidates: vec![candidate.clone()], + }; + let decisions = vec![ConsolidationDecision { + candidate_id: candidate.candidate_id.clone(), + output_digest: candidate.output_digest.clone(), + operation: ConsolidationOperation::AdoptStatement, + reason: "synthetic consolidation scheduler regression".into(), + distinct_evidence_roots: candidate.source_ids.clone(), + supersedes: supersedes.to_vec(), + validator: None, + }]; + store + .persist_consolidation_batch(&batch, Some(&decisions), "validated") + .expect("persist synthetic consolidation batch"); + store + .apply_consolidation_proposal(&batch.batch_id, &policy, &decisions) + .expect("apply synthetic consolidation batch") +} + fn insert_chatgpt_turn( store: &Store, session_id: &str, @@ -214,10 +298,6 @@ fn insert_chatgpt_turn( fn scheduler_cursor_keeps_lower_bound_across_all_streams_and_ties() { let store = Store::open(":memory:").expect("store"); let service = Service::new(store.clone(), scheduler_config("workspace", 3, false)); - - service - .scheduled_dream(Some(FRONTIER.into())) - .expect("seed watermark"); store .ensure_workspace(PROFILE, "workspace") .expect("workspace"); @@ -359,6 +439,21 @@ fn scheduler_cursor_keeps_lower_bound_across_all_streams_and_ties() { }) .expect("source timestamps"); + store + .record_dream_run(&DreamRunRecord { + run_id: "zzzz-scheduler-seed-watermark".into(), + profile_id: PROFILE.into(), + workspace_id: "workspace".into(), + mode: "preview".into(), + kind: "scheduled".into(), + status: "ok".into(), + started_at: FRONTIER.into(), + completed_at: Some(FRONTIER.into()), + watermark_after: Some(FRONTIER.into()), + ..Default::default() + }) + .expect("seed timestamp watermark without a rowid frontier"); + let mut seen = BTreeSet::new(); for now in [ "2030-01-01T00:00:00Z", @@ -713,6 +808,341 @@ fn scheduler_bounded_band_catches_backdated_rows_after_its_rowid_frontier() { .contains("visible-backdated-after-frontier")); } +#[test] +fn scheduler_unbounded_completion_catches_backdated_rows_in_next_fixed_band() { + let store = Store::open(":memory:").expect("store"); + store + .ensure_workspace(PROFILE, "workspace") + .expect("workspace"); + let unbounded = codex_memoryd::dream::scheduler_cursor_for_run(&store, None, RUN_NOW, true) + .expect("capture unbounded frontier"); + let backdated_at = "2029-12-31T23:59:00Z"; + insert_chatgpt_turn( + &store, + "late-unbounded-session", + "visible-after-unbounded-frontier", + "generic late source with a timestamp before simulated now", + backdated_at, + None, + ); + store + .insert_conclusion(&Conclusion { + id: "conclusion-after-unbounded-frontier".into(), + profile_id: PROFILE.into(), + workspace_id: "workspace".into(), + repo_id: None, + target: "user".into(), + content: "Conclusion: late unbounded frontier source".into(), + source_id: None, + created_at: backdated_at.into(), + metadata: json!({}), + }) + .expect("backdated conclusion"); + store + .insert_checkpoint(&Checkpoint { + id: "checkpoint-after-unbounded-frontier".into(), + session_id: Some("late-unbounded-session".into()), + profile_id: PROFILE.into(), + workspace_id: "workspace".into(), + repo_id: None, + summary: "generic late unbounded frontier checkpoint".into(), + changed_files: vec![], + decisions: vec![], + blockers: vec![], + next_steps: vec![], + tests_run: vec![], + tests_not_run: vec![], + branch: None, + commit: None, + created_at: backdated_at.into(), + }) + .expect("backdated checkpoint"); + let (source, _) = store + .upsert_source( + PROFILE, + "workspace", + "fixture", + Some("unbounded-late-source"), + "unbounded-late-hash", + &json!({}), + ) + .expect("backdated memory source"); + let memory_record_id = insert_memory_record( + &store, + "Preference: late active record after unbounded frontier", + json!({"origin": "fixture"}), + ); + store + .transaction(|tx| { + tx.execute( + "UPDATE memory_sources SET ingested_at = ?1 WHERE id = ?2", + rusqlite::params![backdated_at, source.id], + )?; + tx.execute( + "UPDATE memory_records SET updated_at = ?1 WHERE id = ?2", + rusqlite::params![backdated_at, memory_record_id], + )?; + Ok(()) + }) + .expect("set synthetic backdated timestamps"); + + let first_scan = run_scheduler_cursor(&store, &unbounded, RUN_NOW); + assert!( + evidence_ids(&first_scan).is_empty(), + "the late row must remain outside the captured unbounded frontier" + ); + let completed_unbounded = codex_memoryd::dream::scheduler_watermark_after( + &store, + Some(&unbounded), + &first_scan.evidence_window, + RUN_NOW, + false, + ) + .expect("complete unbounded scan") + .expect("completed unbounded cursor"); + + let next_band = codex_memoryd::dream::scheduler_cursor_for_run( + &store, + Some(&completed_unbounded), + "2030-01-01T00:00:01Z", + false, + ) + .expect("open fixed band after unbounded scan"); + let next_scan = run_scheduler_cursor(&store, &next_band, "2030-01-01T00:00:01Z"); + let next_ids = evidence_ids(&next_scan); + for id in [ + "visible-after-unbounded-frontier", + "conclusion-after-unbounded-frontier", + "checkpoint-after-unbounded-frontier", + source.id.as_str(), + memory_record_id.as_str(), + ] { + assert!( + next_ids.contains(id), + "the first fixed band after unbounded completion must include every post-frontier stream, even with backdated timestamps; missing={id}, selected={next_ids:?}" + ); + } +} + +#[test] +fn scheduler_observes_exact_hash_consolidation_reactivation_after_rowid_frontier() { + let store = Store::open(":memory:").expect("store"); + store + .ensure_workspace(PROFILE, "workspace") + .expect("workspace"); + let content = "Preference: preserve exact-hash scheduler eligibility"; + let archived_id = insert_memory_record(&store, content, json!({"origin": "fixture"})); + store + .archive_records( + PROFILE, + Some("workspace"), + std::slice::from_ref(&archived_id), + ) + .expect("archive synthetic exact-hash record"); + + let initial_cursor = + codex_memoryd::dream::scheduler_cursor_for_run(&store, None, "2000-01-01T00:00:00Z", true) + .expect("capture initial active-memory frontier"); + let initial_run = run_scheduler_cursor(&store, &initial_cursor, "2000-01-01T00:00:00Z"); + assert!(!evidence_ids(&initial_run).contains(&archived_id)); + let completed_initial = codex_memoryd::dream::scheduler_watermark_after( + &store, + Some(&initial_cursor), + &initial_run.evidence_window, + "2000-01-01T00:00:00Z", + false, + ) + .expect("complete initial scan") + .expect("completed initial cursor"); + + assert_eq!( + apply_synthetic_consolidation(&store, "reactivate-exact-hash-batch", content, &[],), + vec![archived_id.clone()] + ); + let next_cursor = codex_memoryd::dream::scheduler_cursor_for_run( + &store, + Some(&completed_initial), + RUN_NOW, + false, + ) + .expect("capture post-reactivation mutation frontier"); + let next_run = run_scheduler_cursor(&store, &next_cursor, RUN_NOW); + assert!( + evidence_ids(&next_run).contains(&archived_id), + "an exact-hash reactivation must be selected after its old rowid was scanned" + ); + let reactivated_sequence = store + .transaction(|tx| { + tx.query_row( + "SELECT active_memory_mutation_sequence FROM memory_records WHERE id = ?1", + rusqlite::params![archived_id], + |row| row.get::<_, i64>(0), + ) + .map_err(Into::into) + }) + .expect("read reactivated mutation sequence"); + assert!( + reactivated_sequence > 0, + "reactivation must allocate a mutation sequence" + ); +} + +#[test] +fn scheduler_rechecks_eligible_existing_record_updates_after_rowid_frontier() { + let store = Store::open(":memory:").expect("store"); + store + .ensure_workspace(PROFILE, "workspace") + .expect("workspace"); + let upsert_content = "Preference: merge a later synthetic source"; + let upsert_id = insert_memory_record(&store, upsert_content, json!({"origin": "fixture"})); + let superseded_id = insert_memory_record( + &store, + "Preference: retain supersession context after the frontier", + json!({"origin": "fixture"}), + ); + let replacement_id = insert_memory_record( + &store, + "Preference: replacement context after the frontier", + json!({"origin": "fixture"}), + ); + let invalidated_id = insert_memory_record( + &store, + "Preference: retain invalidation context after the frontier", + json!({"origin": "fixture"}), + ); + + let initial_cursor = + codex_memoryd::dream::scheduler_cursor_for_run(&store, None, "2000-01-01T00:00:00Z", true) + .expect("capture initial existing-record frontier"); + let initial_run = run_scheduler_cursor(&store, &initial_cursor, "2000-01-01T00:00:00Z"); + let initial_ids = evidence_ids(&initial_run); + assert!(initial_ids.contains(&upsert_id)); + assert!(initial_ids.contains(&superseded_id)); + assert!(initial_ids.contains(&replacement_id)); + assert!(initial_ids.contains(&invalidated_id)); + let completed_initial = codex_memoryd::dream::scheduler_watermark_after( + &store, + Some(&initial_cursor), + &initial_run.evidence_window, + "2000-01-01T00:00:00Z", + false, + ) + .expect("complete initial existing-record scan") + .expect("completed existing-record cursor"); + + store + .upsert_record(&new_memory_record( + upsert_content, + json!({"origin": "fixture"}), + vec!["synthetic-late-source".into()], + )) + .expect("merge source into existing active record"); + store + .supersede_record( + PROFILE, + "workspace", + &superseded_id, + &replacement_id, + "synthetic scheduler regression", + RUN_NOW, + ) + .expect("update existing supersession records"); + store + .invalidate_record( + PROFILE, + "workspace", + &invalidated_id, + "synthetic scheduler regression", + RUN_NOW, + ) + .expect("update existing invalidation record"); + + let next_cursor = codex_memoryd::dream::scheduler_cursor_for_run( + &store, + Some(&completed_initial), + RUN_NOW, + false, + ) + .expect("capture updates to existing rows"); + let next_run = run_scheduler_cursor(&store, &next_cursor, RUN_NOW); + let updated_ids = evidence_ids(&next_run); + for id in [upsert_id, superseded_id, replacement_id, invalidated_id] { + assert!( + updated_ids.contains(&id), + "existing eligible record update was skipped after the rowid frontier: {id}; selected={updated_ids:?}" + ); + } +} + +#[test] +fn scheduler_observes_consolidation_undo_reactivation_after_rowid_frontier() { + let store = Store::open(":memory:").expect("store"); + store + .ensure_workspace(PROFILE, "workspace") + .expect("workspace"); + let superseded_id = insert_memory_record( + &store, + "Preference: original claim restored by consolidation undo", + json!({"origin": "fixture"}), + ); + let batch_id = "undo-reactivation-batch"; + let replacement_ids = apply_synthetic_consolidation( + &store, + batch_id, + "Preference: governed replacement for the original claim", + std::slice::from_ref(&superseded_id), + ); + assert_eq!(replacement_ids.len(), 1); + assert!( + store + .get_record(&superseded_id) + .expect("read target") + .unwrap() + .archived + ); + + let initial_cursor = + codex_memoryd::dream::scheduler_cursor_for_run(&store, None, "2000-01-01T00:00:00Z", true) + .expect("capture frontier before consolidation undo"); + let initial_run = run_scheduler_cursor(&store, &initial_cursor, "2000-01-01T00:00:00Z"); + assert!(!evidence_ids(&initial_run).contains(&superseded_id)); + let completed_initial = codex_memoryd::dream::scheduler_watermark_after( + &store, + Some(&initial_cursor), + &initial_run.evidence_window, + "2000-01-01T00:00:00Z", + false, + ) + .expect("complete pre-undo scan") + .expect("completed pre-undo cursor"); + + assert_eq!( + store + .undo_consolidation_proposal(batch_id) + .expect("undo synthetic consolidation"), + replacement_ids + ); + assert!( + !store + .get_record(&superseded_id) + .expect("read restored target") + .unwrap() + .archived + ); + let next_cursor = codex_memoryd::dream::scheduler_cursor_for_run( + &store, + Some(&completed_initial), + RUN_NOW, + false, + ) + .expect("capture consolidation undo mutation"); + let next_run = run_scheduler_cursor(&store, &next_cursor, RUN_NOW); + assert!( + evidence_ids(&next_run).contains(&superseded_id), + "undo must re-emit a restored existing row after its old rowid was scanned" + ); +} + #[test] fn scheduler_cursor_rewinds_persisted_id_only_cursor_before_rowid_paging() { let store = Store::open(":memory:").expect("store"); From 49ea61e1b0b3e248fb9e466c0bb8b3d0b7b2af4b Mon Sep 17 00:00:00 2001 From: Josh Yorko Date: Wed, 14 Oct 2026 21:42:58 +0000 Subject: [PATCH 17/18] fix: preserve scheduler frontiers across rowid reuse --- docs/dreamer-native-provider.md | 17 +- ...15_scheduler_nonreusable_source_rowids.sql | 97 ++++++ src/dream.rs | 190 ++++++++++-- src/store.rs | 67 ++++- tests/fixtures/status.response.json | 2 +- tests/scheduler_review_boundaries.rs | 276 ++++++++++++++++++ tests/schema_upgrade.rs | 88 +++++- 7 files changed, 701 insertions(+), 36 deletions(-) create mode 100644 migrations/0015_scheduler_nonreusable_source_rowids.sql diff --git a/docs/dreamer-native-provider.md b/docs/dreamer-native-provider.md index e03fd79..691b24d 100644 --- a/docs/dreamer-native-provider.md +++ b/docs/dreamer-native-provider.md @@ -68,11 +68,18 @@ evidence remains eligible. Other provider failures do not advance the successful scheduled watermark. A recovered deterministic fallback also records a later status-visible `ok_with_limits` receipt while retaining the provider budget rejection in audit history. Completed unbounded simulated-clock scans retain -their rowid frontier even when the simulated clock does not advance. Each -bounded band caps every source stream at its captured rowid frontier. The next -band also checks rows inserted after that cap, including rows whose source -timestamp was backdated below the prior band's end. The first bounded band -after an unbounded scan uses the same post-frontier catch-up, so arrivals +their rowid frontier even when the simulated clock does not advance. Scheduler +frontiers use persisted logical rowids for visible turns, conclusions, +checkpoints, imported sources, and memory records. Existing rows retain their +SQLite rowid during migration, while later inserts receive a logical rowid +above every saved cursor frontier; hard-deleting a physical maximum can no +longer make a replacement insert invisible. Each bounded band caps every +source stream at its captured logical rowid frontier. Memory record contents +are materialized by the same filtered query that checks the frontier, so a +later pooled read cannot substitute a post-frontier update into the evidence +window. The next band also checks rows inserted after that cap, including rows +whose source timestamp was backdated below the prior band's end. The first +bounded band after an unbounded scan uses the same post-frontier catch-up, so arrivals between that snapshot and the next band are not lost below the timestamp watermark. Existing-row updates that affect scheduler-visible evidence and eligibility changes allocate a per-record mutation sequence in the same SQLite diff --git a/migrations/0015_scheduler_nonreusable_source_rowids.sql b/migrations/0015_scheduler_nonreusable_source_rowids.sql new file mode 100644 index 0000000..65e3f03 --- /dev/null +++ b/migrations/0015_scheduler_nonreusable_source_rowids.sql @@ -0,0 +1,97 @@ +-- Stable scheduler row IDs decouple saved cursors from SQLite rowid reuse. +CREATE TABLE IF NOT EXISTS scheduler_source_rowid_counter ( + singleton INTEGER PRIMARY KEY CHECK (singleton = 1), + sequence INTEGER NOT NULL CHECK (sequence >= 0) +); + +INSERT OR IGNORE INTO scheduler_source_rowid_counter(singleton, sequence) +VALUES (1, 0); + +CREATE TABLE IF NOT EXISTS scheduler_source_rowids ( + stream TEXT NOT NULL, + source_key TEXT NOT NULL, + scheduler_rowid INTEGER NOT NULL CHECK (scheduler_rowid > 0), + PRIMARY KEY (stream, source_key), + UNIQUE (stream, scheduler_rowid) +); + +INSERT OR IGNORE INTO scheduler_source_rowids(stream, source_key, scheduler_rowid) +SELECT 'visible_turns', id, rowid FROM visible_turns; +INSERT OR IGNORE INTO scheduler_source_rowids(stream, source_key, scheduler_rowid) +SELECT 'conclusions', id, rowid FROM conclusions; +INSERT OR IGNORE INTO scheduler_source_rowids(stream, source_key, scheduler_rowid) +SELECT 'checkpoints', id, rowid FROM checkpoints; +INSERT OR IGNORE INTO scheduler_source_rowids(stream, source_key, scheduler_rowid) +SELECT 'memory_sources', id, rowid FROM memory_sources; +INSERT OR IGNORE INTO scheduler_source_rowids(stream, source_key, scheduler_rowid) +SELECT 'memory_records', id, rowid FROM memory_records; + +UPDATE scheduler_source_rowid_counter +SET sequence = MAX( + sequence, + COALESCE((SELECT MAX(scheduler_rowid) FROM scheduler_source_rowids), 0) +) +WHERE singleton = 1; + +CREATE INDEX IF NOT EXISTS idx_scheduler_source_rowids_rowid + ON scheduler_source_rowids(stream, scheduler_rowid); + +CREATE TRIGGER IF NOT EXISTS visible_turns_scheduler_rowid_insert +AFTER INSERT ON visible_turns BEGIN + UPDATE scheduler_source_rowid_counter SET sequence = sequence + 1 WHERE singleton = 1; + INSERT OR REPLACE INTO scheduler_source_rowids(stream, source_key, scheduler_rowid) + SELECT 'visible_turns', NEW.id, sequence + FROM scheduler_source_rowid_counter WHERE singleton = 1; +END; +CREATE TRIGGER IF NOT EXISTS visible_turns_scheduler_rowid_delete +AFTER DELETE ON visible_turns BEGIN + DELETE FROM scheduler_source_rowids WHERE stream = 'visible_turns' AND source_key = OLD.id; +END; + +CREATE TRIGGER IF NOT EXISTS conclusions_scheduler_rowid_insert +AFTER INSERT ON conclusions BEGIN + UPDATE scheduler_source_rowid_counter SET sequence = sequence + 1 WHERE singleton = 1; + INSERT OR REPLACE INTO scheduler_source_rowids(stream, source_key, scheduler_rowid) + SELECT 'conclusions', NEW.id, sequence + FROM scheduler_source_rowid_counter WHERE singleton = 1; +END; +CREATE TRIGGER IF NOT EXISTS conclusions_scheduler_rowid_delete +AFTER DELETE ON conclusions BEGIN + DELETE FROM scheduler_source_rowids WHERE stream = 'conclusions' AND source_key = OLD.id; +END; + +CREATE TRIGGER IF NOT EXISTS checkpoints_scheduler_rowid_insert +AFTER INSERT ON checkpoints BEGIN + UPDATE scheduler_source_rowid_counter SET sequence = sequence + 1 WHERE singleton = 1; + INSERT OR REPLACE INTO scheduler_source_rowids(stream, source_key, scheduler_rowid) + SELECT 'checkpoints', NEW.id, sequence + FROM scheduler_source_rowid_counter WHERE singleton = 1; +END; +CREATE TRIGGER IF NOT EXISTS checkpoints_scheduler_rowid_delete +AFTER DELETE ON checkpoints BEGIN + DELETE FROM scheduler_source_rowids WHERE stream = 'checkpoints' AND source_key = OLD.id; +END; + +CREATE TRIGGER IF NOT EXISTS memory_sources_scheduler_rowid_insert +AFTER INSERT ON memory_sources BEGIN + UPDATE scheduler_source_rowid_counter SET sequence = sequence + 1 WHERE singleton = 1; + INSERT OR REPLACE INTO scheduler_source_rowids(stream, source_key, scheduler_rowid) + SELECT 'memory_sources', NEW.id, sequence + FROM scheduler_source_rowid_counter WHERE singleton = 1; +END; +CREATE TRIGGER IF NOT EXISTS memory_sources_scheduler_rowid_delete +AFTER DELETE ON memory_sources BEGIN + DELETE FROM scheduler_source_rowids WHERE stream = 'memory_sources' AND source_key = OLD.id; +END; + +CREATE TRIGGER IF NOT EXISTS memory_records_scheduler_rowid_insert +AFTER INSERT ON memory_records BEGIN + UPDATE scheduler_source_rowid_counter SET sequence = sequence + 1 WHERE singleton = 1; + INSERT OR REPLACE INTO scheduler_source_rowids(stream, source_key, scheduler_rowid) + SELECT 'memory_records', NEW.id, sequence + FROM scheduler_source_rowid_counter WHERE singleton = 1; +END; +CREATE TRIGGER IF NOT EXISTS memory_records_scheduler_rowid_delete +AFTER DELETE ON memory_records BEGIN + DELETE FROM scheduler_source_rowids WHERE stream = 'memory_records' AND source_key = OLD.id; +END; diff --git a/src/dream.rs b/src/dream.rs index d0ab9b1..d170ac7 100644 --- a/src/dream.rs +++ b/src/dream.rs @@ -37,11 +37,13 @@ use crate::protocol::DreamRejection; use crate::protocol::DreamResponse; use crate::protocol::DreamStaleRecord; use crate::store::ledger_safe_summary; +use crate::store::row_to_record; use crate::store::EvidenceLedgerEntry; use crate::store::NewRecord; use crate::store::RecordQuery; use crate::store::Store; use crate::store::UpsertOutcome; +use crate::store::RECORD_COLS; pub const DREAM_IMPLEMENTATION_VERSION: &str = "heuristic-v1"; pub const DREAM_RULESET_VERSION: &str = "dreamer-heuristics-v1"; @@ -1351,27 +1353,32 @@ fn scheduler_rowid_frontier(store: &Store) -> Result { store.transaction(|tx| { Ok(DreamRowidFrontier { visible_turns: tx.query_row( - "SELECT COALESCE(MAX(rowid), 0) FROM visible_turns", + "SELECT COALESCE(MAX(scheduler_rowid), 0) + FROM scheduler_source_rowids WHERE stream = 'visible_turns'", [], |row| row.get(0), )?, conclusions: tx.query_row( - "SELECT COALESCE(MAX(rowid), 0) FROM conclusions", + "SELECT COALESCE(MAX(scheduler_rowid), 0) + FROM scheduler_source_rowids WHERE stream = 'conclusions'", [], |row| row.get(0), )?, checkpoints: tx.query_row( - "SELECT COALESCE(MAX(rowid), 0) FROM checkpoints", + "SELECT COALESCE(MAX(scheduler_rowid), 0) + FROM scheduler_source_rowids WHERE stream = 'checkpoints'", [], |row| row.get(0), )?, imported_memories: tx.query_row( - "SELECT COALESCE(MAX(rowid), 0) FROM memory_sources", + "SELECT COALESCE(MAX(scheduler_rowid), 0) + FROM scheduler_source_rowids WHERE stream = 'memory_sources'", [], |row| row.get(0), )?, active_memory_records: tx.query_row( - "SELECT COALESCE(MAX(rowid), 0) FROM memory_records", + "SELECT COALESCE(MAX(scheduler_rowid), 0) + FROM scheduler_source_rowids WHERE stream = 'memory_records'", [], |row| row.get(0), )?, @@ -1547,11 +1554,17 @@ fn rowids_for_ids( let mut rowids = BTreeMap::new(); for chunk in ids.iter().collect::>().chunks(900) { let placeholders = vec!["?"; chunk.len()].join(", "); - let sql = format!("SELECT id, rowid FROM {table} WHERE id IN ({placeholders})"); - let args: Vec> = chunk - .iter() - .map(|id| Box::new((*id).clone()) as Box) - .collect(); + let sql = format!( + "SELECT source_key, scheduler_rowid + FROM scheduler_source_rowids + WHERE stream = ? AND source_key IN ({placeholders})" + ); + let mut args = vec![Box::new(table.to_string()) as Box]; + args.extend( + chunk + .iter() + .map(|id| Box::new((*id).clone()) as Box), + ); let args_ref = args.iter().map(|arg| arg.as_ref()).collect::>(); let mut statement = tx.prepare(&sql)?; let rows = statement.query_map(args_ref.as_slice(), |row| { @@ -1727,6 +1740,8 @@ fn query_visible_turns_with_frontier( let mut sql = "SELECT t.id, t.session_id, t.actor, t.content, t.created_at, t.metadata FROM visible_turns t JOIN sessions s ON s.id = t.session_id + JOIN scheduler_source_rowids sr + ON sr.stream = 'visible_turns' AND sr.source_key = t.id WHERE s.profile_id = ? AND s.workspace_id = ?" .to_string(); let mut args: Vec> = vec![ @@ -1743,7 +1758,7 @@ fn query_visible_turns_with_frontier( args, "t.created_at", "t.id", - "t.rowid", + "sr.scheduler_rowid", start, start_exclusive, lower_rowid_strict, @@ -1785,6 +1800,8 @@ fn query_conclusions_with_frontier( let mut sql = "SELECT id, profile_id, workspace_id, repo_id, target, content, source_id, created_at, metadata FROM conclusions + JOIN scheduler_source_rowids sr + ON sr.stream = 'conclusions' AND sr.source_key = conclusions.id WHERE profile_id = ? AND workspace_id = ?" .to_string(); let mut args: Vec> = vec![ @@ -1801,7 +1818,7 @@ fn query_conclusions_with_frontier( args, "created_at", "id", - "rowid", + "sr.scheduler_rowid", start, start_exclusive, lower_rowid_strict, @@ -1847,6 +1864,8 @@ fn query_checkpoints_with_frontier( changed_files, decisions, blockers, next_steps, tests_run, tests_not_run, branch, commit_sha, created_at FROM checkpoints + JOIN scheduler_source_rowids sr + ON sr.stream = 'checkpoints' AND sr.source_key = checkpoints.id WHERE profile_id = ? AND workspace_id = ?" .to_string(); let mut args: Vec> = vec![ @@ -1863,7 +1882,7 @@ fn query_checkpoints_with_frontier( args, "created_at", "id", - "rowid", + "sr.scheduler_rowid", start, start_exclusive, lower_rowid_strict, @@ -1913,6 +1932,8 @@ fn query_memory_sources_with_frontier( let sql = "SELECT id, profile_id, workspace_id, kind, source_path, source_hash, created_at, ingested_at, metadata FROM memory_sources + JOIN scheduler_source_rowids sr + ON sr.stream = 'memory_sources' AND sr.source_key = memory_sources.id WHERE profile_id = ? AND workspace_id = ?" .to_string(); let args: Vec> = vec![ @@ -1925,7 +1946,7 @@ fn query_memory_sources_with_frontier( args, "ingested_at", "id", - "rowid", + "sr.scheduler_rowid", start, start_exclusive, lower_rowid_strict, @@ -1969,9 +1990,53 @@ fn query_memory_records_with_frontier( include_archived: bool, mutation_scan: Option<(&str, i64, i64)>, ) -> Result> { - let mut sql = "SELECT id FROM memory_records + query_memory_records_with_frontier_after_materialization( + store, + profile_id, + workspace_id, + repo_id, + start, + start_exclusive, + lower_rowid_strict, + include_post_frontier_rowids, + end, + lower_rowid, + upper_rowid, + position, + limit, + include_archived, + mutation_scan, + || {}, + ) +} + +fn query_memory_records_with_frontier_after_materialization( + store: &Store, + profile_id: &str, + workspace_id: &str, + repo_id: Option<&str>, + start: Option<&str>, + start_exclusive: bool, + lower_rowid_strict: bool, + include_post_frontier_rowids: bool, + end: Option<&str>, + lower_rowid: Option, + upper_rowid: Option, + position: Option<&DreamSourceCursor>, + limit: usize, + include_archived: bool, + mutation_scan: Option<(&str, i64, i64)>, + after_materialization: F, +) -> Result> +where + F: FnOnce(), +{ + let mut sql = format!( + "SELECT {RECORD_COLS} FROM memory_records + JOIN scheduler_source_rowids sr + ON sr.stream = 'memory_records' AND sr.source_key = memory_records.id WHERE profile_id = ? AND workspace_id = ?" - .to_string(); + ); let mut args: Vec> = vec![ Box::new(profile_id.to_string()), Box::new(workspace_id.to_string()), @@ -1991,13 +2056,13 @@ fn query_memory_records_with_frontier( sql.push_str(" AND archived = 0"); } sql.push_str(" AND sensitivity != 'secret_blocked' AND trust_state != 'quarantined'"); - let ids = query_dream_rows( + let records = query_dream_rows( store, sql, args, "updated_at", "id", - "rowid", + "sr.scheduler_rowid", start, start_exclusive, lower_rowid_strict, @@ -2008,14 +2073,9 @@ fn query_memory_records_with_frontier( position, mutation_scan, limit, - |row| row.get::<_, String>(0), + row_to_record, )?; - let mut records = Vec::with_capacity(ids.len()); - for id in ids { - if let Some(record) = store.get_record(&id)? { - records.push(record); - } - } + after_materialization(); Ok(records) } @@ -3799,4 +3859,84 @@ mod tests { assert_eq!(error.message, "dream job exceeded max_runtime_seconds"); } + + #[test] + fn scheduler_memory_record_frontier_hydration_uses_filtered_snapshot() { + let temp = tempfile::tempdir().expect("temporary database directory"); + let db_path = temp.path().join("memory-record-frontier-snapshot.db"); + let store = Store::open(&db_path).expect("store"); + let writer = Store::open(&db_path).expect("independent writer pool"); + let content = "Preference: content from the captured scheduler snapshot"; + let classification = policy::classify(content, Profile::Personal, false); + let content_hash = ids::exact_content_hash( + Profile::Personal.as_str(), + "workspace", + None, + classification.record_type.as_str(), + classification.scope.as_str(), + content, + ); + let id = match store + .upsert_record(&NewRecord { + profile_id: Profile::Personal.as_str().into(), + workspace_id: "workspace".into(), + repo_id: None, + subject_id: None, + episode_id: None, + scope: classification.scope, + record_type: classification.record_type, + content: content.into(), + related_files: classification.related_files, + tags: classification.tags, + sensitivity: classification.sensitivity, + portability: classification.portability, + confidence: classification.confidence, + source_ids: vec![], + content_hash, + supersedes: vec![], + metadata: json!({"origin": "fixture"}), + }) + .expect("insert synthetic record") + { + UpsertOutcome::Created(id) | UpsertOutcome::Skipped(id) => id, + }; + + let records = query_memory_records_with_frontier_after_materialization( + &store, + Profile::Personal.as_str(), + "workspace", + None, + None, + false, + false, + false, + None, + None, + None, + None, + 10, + false, + None, + || { + writer + .transaction_immediate(|tx| { + tx.execute( + "UPDATE memory_records SET content = ?1, updated_at = ?2 WHERE id = ?3", + rusqlite::params![ + "Preference: content updated after frontier selection", + "2030-01-01T00:00:00Z", + id, + ], + )?; + Ok(()) + }) + .expect("concurrent synthetic record update"); + }, + ) + .expect("read memory records at the scheduler frontier"); + + assert_eq!(records.len(), 1); + assert_eq!(records[0].id, id); + assert_eq!(records[0].content, content); + } } diff --git a/src/store.rs b/src/store.rs index 763274c..39e3960 100644 --- a/src/store.rs +++ b/src/store.rs @@ -49,7 +49,7 @@ use crate::ids; use crate::protocol::DreamJobBudget; use crate::protocol::DreamJobProvider; -pub const STORAGE_SCHEMA_VERSION: i64 = 14; +pub const STORAGE_SCHEMA_VERSION: i64 = 15; const MIGRATION_INIT: &str = include_str!("../migrations/0001_init.sql"); const MIGRATION_FTS: &str = include_str!("../migrations/0002_fts.sql"); @@ -69,6 +69,8 @@ const MIGRATION_CONSOLIDATION_PROPOSALS: &str = include_str!("../migrations/0013_consolidation_proposals.sql"); const MIGRATION_SCHEDULER_MUTATION_FRONTIER: &str = include_str!("../migrations/0014_scheduler_mutation_frontier.sql"); +const MIGRATION_SCHEDULER_NONREUSABLE_SOURCE_ROWIDS: &str = + include_str!("../migrations/0015_scheduler_nonreusable_source_rowids.sql"); type SqlitePool = Pool; @@ -1661,6 +1663,14 @@ impl Store { conn.execute_batch(MIGRATION_PORTABLE_BUNDLES)?; conn.execute_batch(MIGRATION_CONSOLIDATION_PROPOSALS)?; conn.execute_batch(MIGRATION_SCHEDULER_MUTATION_FRONTIER)?; + conn.execute_batch(MIGRATION_SCHEDULER_NONREUSABLE_SOURCE_ROWIDS)?; + let saved_scheduler_frontier = saved_scheduler_rowid_highwater(&conn)?; + conn.execute( + "UPDATE scheduler_source_rowid_counter + SET sequence = MAX(sequence, ?1) + WHERE singleton = 1", + params![saved_scheduler_frontier], + )?; ensure_column( &conn, "memory_records", @@ -5046,6 +5056,57 @@ fn is_safe_identifier(name: &str) -> bool { !name.is_empty() && name.chars().all(|c| c.is_ascii_alphanumeric() || c == '_') } +fn saved_scheduler_rowid_highwater(conn: &rusqlite::Connection) -> Result { + fn visit(value: &Value, key: Option<&str>, highwater: &mut i64) { + match value { + Value::Number(number) + if matches!( + key, + Some( + "rowid" + | "visible_turns" + | "conclusions" + | "checkpoints" + | "imported_memories" + | "active_memory_records" + ) + ) => + { + if let Some(value) = number.as_i64() { + *highwater = (*highwater).max(value); + } + } + Value::Number(_) => {} + Value::Array(values) => { + for value in values { + visit(value, None, highwater); + } + } + Value::Object(values) => { + for (key, value) in values { + visit(value, Some(key), highwater); + } + } + Value::Null | Value::Bool(_) | Value::String(_) => {} + } + } + + let mut highwater = 0; + let mut statement = + conn.prepare("SELECT watermark_after FROM dream_runs WHERE watermark_after IS NOT NULL")?; + let rows = statement.query_map([], |row| row.get::<_, String>(0))?; + for row in rows { + let watermark = row?; + let Some(json_start) = watermark.find('{') else { + continue; + }; + if let Ok(value) = serde_json::from_str::(&watermark[json_start..]) { + visit(&value, None, &mut highwater); + } + } + Ok(highwater) +} + fn evidence_ledger_event_key(entry: &EvidenceLedgerEntry) -> String { ids::sha256_hex( format!( @@ -5324,7 +5385,7 @@ fn ensure_column( // Row mappers and SQL helpers // ---------------------------------------------------------------------- -const RECORD_COLS: &str = "id, profile_id, workspace_id, repo_id, subject_id, episode_id, scope, type, content, related_files, tags, sensitivity, portability, confidence, source_ids, content_hash, supersedes, created_at, updated_at, last_used_at, archived, trust_state, trust_score, quarantine_reason, quarantined_at, promoted_at, valid_from, valid_until, observed_at, invalidated_at, superseded_by, historical_reason, temporal_state, metadata"; +pub(crate) const RECORD_COLS: &str = "id, profile_id, workspace_id, repo_id, subject_id, episode_id, scope, type, content, related_files, tags, sensitivity, portability, confidence, source_ids, content_hash, supersedes, created_at, updated_at, last_used_at, archived, trust_state, trust_score, quarantine_reason, quarantined_at, promoted_at, valid_from, valid_until, observed_at, invalidated_at, superseded_by, historical_reason, temporal_state, metadata"; const SUBJECT_COLS: &str = "id, profile_id, workspace_id, subject_key, kind, display_name, created_at, updated_at, metadata"; const SUBJECT_COLS_S: &str = @@ -5586,7 +5647,7 @@ fn validate_relation(relation: &Relation) -> Result<()> { Ok(()) } -fn row_to_record(row: &Row) -> rusqlite::Result { +pub(crate) fn row_to_record(row: &Row) -> rusqlite::Result { Ok(MemoryRecord { id: row.get(0)?, profile_id: row.get(1)?, diff --git a/tests/fixtures/status.response.json b/tests/fixtures/status.response.json index db9fd36..94d68d7 100644 --- a/tests/fixtures/status.response.json +++ b/tests/fixtures/status.response.json @@ -4,7 +4,7 @@ "provider_name": "codex-memoryd", "provider_version": "0.1.0", "api_version": "v1", - "storage_schema_version": 14, + "storage_schema_version": 15, "status": "local_only", "storage": { "kind": "sqlite", diff --git a/tests/scheduler_review_boundaries.rs b/tests/scheduler_review_boundaries.rs index 19fa1fc..c82c5e1 100644 --- a/tests/scheduler_review_boundaries.rs +++ b/tests/scheduler_review_boundaries.rs @@ -924,6 +924,282 @@ fn scheduler_unbounded_completion_catches_backdated_rows_in_next_fixed_band() { } } +#[test] +fn scheduler_catches_reinserted_sources_after_max_rowid_is_deleted() { + let store = Store::open(":memory:").expect("store"); + store + .ensure_workspace(PROFILE, "workspace") + .expect("workspace"); + + insert_chatgpt_turn( + &store, + "rowid-reuse-session", + "visible-before-reuse", + "generic old visible row before rowid reuse", + "2029-12-31T23:59:00Z", + None, + ); + store + .insert_conclusion(&Conclusion { + id: "conclusion-before-reuse".into(), + profile_id: PROFILE.into(), + workspace_id: "workspace".into(), + repo_id: None, + target: "user".into(), + content: "Conclusion: old conclusion before rowid reuse".into(), + source_id: None, + created_at: "2029-12-31T23:59:00Z".into(), + metadata: json!({}), + }) + .expect("old conclusion"); + store + .insert_checkpoint(&Checkpoint { + id: "checkpoint-before-reuse".into(), + session_id: Some("rowid-reuse-session".into()), + profile_id: PROFILE.into(), + workspace_id: "workspace".into(), + repo_id: None, + summary: "generic old checkpoint before rowid reuse".into(), + changed_files: vec![], + decisions: vec![], + blockers: vec![], + next_steps: vec![], + tests_run: vec![], + tests_not_run: vec![], + branch: None, + commit: None, + created_at: "2029-12-31T23:59:00Z".into(), + }) + .expect("old checkpoint"); + let (old_source, _) = store + .upsert_source( + PROFILE, + "workspace", + "fixture", + Some("rowid-reuse-old-source"), + "rowid-reuse-old-hash", + &json!({}), + ) + .expect("old memory source"); + let old_record = insert_memory_record( + &store, + "Preference: old active record before rowid reuse", + json!({"origin": "fixture"}), + ); + store + .transaction(|tx| { + tx.execute( + "UPDATE memory_sources SET ingested_at = ?1 WHERE id = ?2", + rusqlite::params!["2029-12-31T23:59:00Z", old_source.id], + )?; + tx.execute( + "UPDATE memory_records SET updated_at = ?1 WHERE id = ?2", + rusqlite::params!["2029-12-31T23:59:00Z", old_record], + )?; + Ok(()) + }) + .expect("backdate old synthetic rows"); + + let initial_cursor = + codex_memoryd::dream::scheduler_cursor_for_run(&store, None, RUN_NOW, true) + .expect("capture initial rowid frontier"); + let initial_run = run_scheduler_cursor(&store, &initial_cursor, RUN_NOW); + let completed_initial = codex_memoryd::dream::scheduler_watermark_after( + &store, + Some(&initial_cursor), + &initial_run.evidence_window, + RUN_NOW, + false, + ) + .expect("complete initial scan") + .expect("completed initial cursor"); + + let original_rowids = store + .transaction(|tx| { + Ok(( + tx.query_row( + "SELECT rowid FROM visible_turns WHERE id = 'visible-before-reuse'", + [], + |row| row.get::<_, i64>(0), + )?, + tx.query_row( + "SELECT rowid FROM conclusions WHERE id = 'conclusion-before-reuse'", + [], + |row| row.get::<_, i64>(0), + )?, + tx.query_row( + "SELECT rowid FROM checkpoints WHERE id = 'checkpoint-before-reuse'", + [], + |row| row.get::<_, i64>(0), + )?, + tx.query_row( + "SELECT rowid FROM memory_sources WHERE id = ?1", + rusqlite::params![old_source.id], + |row| row.get::<_, i64>(0), + )?, + tx.query_row( + "SELECT rowid FROM memory_records WHERE id = ?1", + rusqlite::params![old_record], + |row| row.get::<_, i64>(0), + )?, + )) + }) + .expect("read original rowids"); + + store + .transaction(|tx| { + tx.execute( + "DELETE FROM visible_turns WHERE id = 'visible-before-reuse'", + [], + )?; + tx.execute( + "DELETE FROM conclusions WHERE id = 'conclusion-before-reuse'", + [], + )?; + tx.execute( + "DELETE FROM checkpoints WHERE id = 'checkpoint-before-reuse'", + [], + )?; + tx.execute( + "DELETE FROM memory_sources WHERE id = ?1", + rusqlite::params![old_source.id], + )?; + tx.execute( + "DELETE FROM memory_records WHERE id = ?1", + rusqlite::params![old_record], + )?; + Ok(()) + }) + .expect("hard-delete each synthetic maximum row"); + + insert_chatgpt_turn( + &store, + "rowid-reuse-session", + "visible-after-reuse", + "generic backdated visible row after rowid reuse", + "2029-12-31T23:59:30Z", + None, + ); + store + .insert_conclusion(&Conclusion { + id: "conclusion-after-reuse".into(), + profile_id: PROFILE.into(), + workspace_id: "workspace".into(), + repo_id: None, + target: "user".into(), + content: "Conclusion: backdated conclusion after rowid reuse".into(), + source_id: None, + created_at: "2029-12-31T23:59:30Z".into(), + metadata: json!({}), + }) + .expect("new conclusion"); + store + .insert_checkpoint(&Checkpoint { + id: "checkpoint-after-reuse".into(), + session_id: Some("rowid-reuse-session".into()), + profile_id: PROFILE.into(), + workspace_id: "workspace".into(), + repo_id: None, + summary: "generic backdated checkpoint after rowid reuse".into(), + changed_files: vec![], + decisions: vec![], + blockers: vec![], + next_steps: vec![], + tests_run: vec![], + tests_not_run: vec![], + branch: None, + commit: None, + created_at: "2029-12-31T23:59:30Z".into(), + }) + .expect("new checkpoint"); + let (new_source, _) = store + .upsert_source( + PROFILE, + "workspace", + "fixture", + Some("rowid-reuse-new-source"), + "rowid-reuse-new-hash", + &json!({}), + ) + .expect("new memory source"); + let new_record = insert_memory_record( + &store, + "Preference: backdated active record after rowid reuse", + json!({"origin": "fixture"}), + ); + store + .transaction(|tx| { + tx.execute( + "UPDATE memory_sources SET ingested_at = ?1 WHERE id = ?2", + rusqlite::params!["2029-12-31T23:59:30Z", new_source.id], + )?; + tx.execute( + "UPDATE memory_records SET updated_at = ?1 WHERE id = ?2", + rusqlite::params!["2029-12-31T23:59:30Z", new_record], + )?; + Ok(()) + }) + .expect("backdate new synthetic rows"); + + let reused_rowids = store + .transaction(|tx| { + Ok(( + tx.query_row( + "SELECT rowid FROM visible_turns WHERE id = 'visible-after-reuse'", + [], + |row| row.get::<_, i64>(0), + )?, + tx.query_row( + "SELECT rowid FROM conclusions WHERE id = 'conclusion-after-reuse'", + [], + |row| row.get::<_, i64>(0), + )?, + tx.query_row( + "SELECT rowid FROM checkpoints WHERE id = 'checkpoint-after-reuse'", + [], + |row| row.get::<_, i64>(0), + )?, + tx.query_row( + "SELECT rowid FROM memory_sources WHERE id = ?1", + rusqlite::params![new_source.id], + |row| row.get::<_, i64>(0), + )?, + tx.query_row( + "SELECT rowid FROM memory_records WHERE id = ?1", + rusqlite::params![new_record], + |row| row.get::<_, i64>(0), + )?, + )) + }) + .expect("read reused rowids"); + assert_eq!( + reused_rowids, original_rowids, + "fixture must reuse max rowids" + ); + + let next_cursor = codex_memoryd::dream::scheduler_cursor_for_run( + &store, + Some(&completed_initial), + "2030-01-01T00:00:01Z", + false, + ) + .expect("open fixed band after deleted maximum rowids"); + let next_run = run_scheduler_cursor(&store, &next_cursor, "2030-01-01T00:00:01Z"); + let selected = evidence_ids(&next_run); + for id in [ + "visible-after-reuse", + "conclusion-after-reuse", + "checkpoint-after-reuse", + new_source.id.as_str(), + new_record.as_str(), + ] { + assert!( + selected.contains(id), + "new rows whose physical rowids were reused must remain eligible after the frontier; missing={id}, selected={selected:?}" + ); + } +} + #[test] fn scheduler_observes_exact_hash_consolidation_reactivation_after_rowid_frontier() { let store = Store::open(":memory:").expect("store"); diff --git a/tests/schema_upgrade.rs b/tests/schema_upgrade.rs index 2c88e1d..9f6b3c2 100644 --- a/tests/schema_upgrade.rs +++ b/tests/schema_upgrade.rs @@ -16,10 +16,10 @@ use std::path::Path; -use codex_memoryd::store::Store; -use codex_memoryd::store::STORAGE_SCHEMA_VERSION; +use codex_memoryd::store::{DreamRunRecord, Store, STORAGE_SCHEMA_VERSION}; use rusqlite::params; use rusqlite::Connection; +use serde_json::json; use tempfile::TempDir; /// Historical `memory_records` shape as it existed at schema v1 — before the @@ -453,6 +453,90 @@ fn upgrades_v7_database_preserves_legacy_records_and_creates_semantic_tables() { assert_eq!(relation_rows, 0, "relations should exist at v8"); } +#[test] +fn scheduler_rowid_upgrade_preserves_saved_cursor_highwater() { + let temp = TempDir::new().expect("temporary directory"); + let db_path = temp.path().join("scheduler-rowid-upgrade.db"); + let store = Store::open(&db_path).expect("initial store"); + store + .record_dream_run(&DreamRunRecord { + run_id: "legacy-scheduler-cursor".into(), + profile_id: "personal".into(), + workspace_id: "workspace".into(), + mode: "preview".into(), + kind: "scheduled".into(), + status: "ok".into(), + started_at: "2030-01-01T00:00:00Z".into(), + watermark_after: Some(format!( + "dream-cursor-v1:{}", + json!({ + "upper_rowids": { + "visible_turns": 80, + "conclusions": 81, + "checkpoints": 82, + "imported_memories": 83, + "active_memory_records": 84, + "active_memory_mutation_sequence": 9000 + }, + "active_memory_records": { + "timestamp": "2029-12-31T23:59:59Z", + "id": "old-position", + "rowid": 97 + } + }) + )), + ..DreamRunRecord::default() + }) + .expect("store legacy rowid cursor"); + store + .transaction(|tx| { + tx.execute_batch( + "DROP TRIGGER visible_turns_scheduler_rowid_insert; + DROP TRIGGER visible_turns_scheduler_rowid_delete; + DROP TRIGGER conclusions_scheduler_rowid_insert; + DROP TRIGGER conclusions_scheduler_rowid_delete; + DROP TRIGGER checkpoints_scheduler_rowid_insert; + DROP TRIGGER checkpoints_scheduler_rowid_delete; + DROP TRIGGER memory_sources_scheduler_rowid_insert; + DROP TRIGGER memory_sources_scheduler_rowid_delete; + DROP TRIGGER memory_records_scheduler_rowid_insert; + DROP TRIGGER memory_records_scheduler_rowid_delete; + DROP TABLE scheduler_source_rowids; + DROP TABLE scheduler_source_rowid_counter;", + )?; + Ok(()) + }) + .expect("simulate a pre-rowid-frontier schema"); + drop(store); + + let upgraded = Store::open(&db_path).expect("reopen through scheduler rowid migration"); + upgraded + .transaction(|tx| { + tx.execute( + "INSERT INTO visible_turns(id, session_id, actor, content, created_at) + VALUES ('new-after-upgrade', 'synthetic-session', 'user', 'synthetic', ?1)", + params!["2029-12-31T23:59:30Z"], + )?; + Ok(()) + }) + .expect("insert synthetic row after migration"); + let scheduler_rowid = upgraded + .transaction(|tx| { + tx.query_row( + "SELECT scheduler_rowid FROM scheduler_source_rowids + WHERE stream = 'visible_turns' AND source_key = 'new-after-upgrade'", + [], + |row| row.get::<_, i64>(0), + ) + .map_err(Into::into) + }) + .expect("read upgraded scheduler rowid"); + assert!( + scheduler_rowid > 97, + "new rows must sort after every rowid saved in a pre-migration cursor" + ); +} + #[test] fn reopening_current_database_is_idempotent() { // Opening an already-current DB twice must not change counts or version. From 7717d40759cf5134beb24e8d3baa6107b4c1292f Mon Sep 17 00:00:00 2001 From: Josh Yorko Date: Wed, 14 Oct 2026 22:11:15 +0000 Subject: [PATCH 18/18] Make scheduler rowid upgrade atomic --- src/store.rs | 145 +++++++++++++++++++++++++++++++++++++--- tests/schema_upgrade.rs | 46 ++++++++++++- 2 files changed, 181 insertions(+), 10 deletions(-) diff --git a/src/store.rs b/src/store.rs index 39e3960..915420f 100644 --- a/src/store.rs +++ b/src/store.rs @@ -71,6 +71,7 @@ const MIGRATION_SCHEDULER_MUTATION_FRONTIER: &str = include_str!("../migrations/0014_scheduler_mutation_frontier.sql"); const MIGRATION_SCHEDULER_NONREUSABLE_SOURCE_ROWIDS: &str = include_str!("../migrations/0015_scheduler_nonreusable_source_rowids.sql"); +const SCHEDULER_SOURCE_ROWIDS_MIGRATION_KEY: &str = "scheduler_source_rowids_migrated"; type SqlitePool = Pool; @@ -1635,7 +1636,7 @@ impl Store { /// Run migrations and probe FTS5. Idempotent. fn migrate(&mut self) -> Result<()> { - let conn = self.conn()?; + let mut conn = self.conn()?; conn.execute_batch(MIGRATION_INIT)?; ensure_memory_record_ref_columns(&conn)?; @@ -1663,14 +1664,7 @@ impl Store { conn.execute_batch(MIGRATION_PORTABLE_BUNDLES)?; conn.execute_batch(MIGRATION_CONSOLIDATION_PROPOSALS)?; conn.execute_batch(MIGRATION_SCHEDULER_MUTATION_FRONTIER)?; - conn.execute_batch(MIGRATION_SCHEDULER_NONREUSABLE_SOURCE_ROWIDS)?; - let saved_scheduler_frontier = saved_scheduler_rowid_highwater(&conn)?; - conn.execute( - "UPDATE scheduler_source_rowid_counter - SET sequence = MAX(sequence, ?1) - WHERE singleton = 1", - params![saved_scheduler_frontier], - )?; + migrate_scheduler_source_rowids(&mut conn)?; ensure_column( &conn, "memory_records", @@ -5107,6 +5101,48 @@ fn saved_scheduler_rowid_highwater(conn: &rusqlite::Connection) -> Result { Ok(highwater) } +/// Installs stable scheduler row IDs once, while holding SQLite's write lock. +/// The saved-cursor scan and high-water seed share the transaction with the +/// backfill and triggers so concurrent inserts cannot receive IDs below an +/// already-persisted cursor. The marker avoids rescanning dream history on +/// ordinary store opens. +fn migrate_scheduler_source_rowids(conn: &mut rusqlite::Connection) -> Result<()> { + migrate_scheduler_source_rowids_with_hook(conn, || {}) +} + +fn migrate_scheduler_source_rowids_with_hook( + conn: &mut rusqlite::Connection, + after_install: impl FnOnce(), +) -> Result<()> { + let tx = conn.transaction_with_behavior(rusqlite::TransactionBehavior::Immediate)?; + let already_migrated: bool = tx.query_row( + "SELECT EXISTS( + SELECT 1 FROM schema_meta WHERE key = ?1 + )", + params![SCHEDULER_SOURCE_ROWIDS_MIGRATION_KEY], + |row| row.get(0), + )?; + + if !already_migrated { + tx.execute_batch(MIGRATION_SCHEDULER_NONREUSABLE_SOURCE_ROWIDS)?; + after_install(); + let saved_scheduler_frontier = saved_scheduler_rowid_highwater(&tx)?; + tx.execute( + "UPDATE scheduler_source_rowid_counter + SET sequence = MAX(sequence, ?1) + WHERE singleton = 1", + params![saved_scheduler_frontier], + )?; + tx.execute( + "INSERT INTO schema_meta(key, value) VALUES (?1, '1')", + params![SCHEDULER_SOURCE_ROWIDS_MIGRATION_KEY], + )?; + } + + tx.commit()?; + Ok(()) +} + fn evidence_ledger_event_key(entry: &EvidenceLedgerEntry) -> String { ids::sha256_hex( format!( @@ -5873,6 +5909,97 @@ mod tests { Store::open(":memory:").expect("open in-memory store") } + #[test] + fn scheduler_rowid_migration_blocks_writes_until_cursor_highwater_is_seeded() { + let temp = tempfile::TempDir::new().expect("temporary directory"); + let db_path = temp.path().join("scheduler-rowid-atomic.db"); + let store = Store::open(&db_path).expect("initial store"); + store + .transaction(|tx| { + tx.execute( + "INSERT INTO dream_runs( + run_id, profile_id, workspace_id, mode, kind, status, + started_at, watermark_after + ) VALUES ('synthetic-cursor', 'personal', 'workspace', 'preview', 'scheduled', + 'ok', '2030-01-01T00:00:00Z', + 'dream-cursor-v1:{\"upper_rowids\":{\"visible_turns\":500}}')", + [], + )?; + tx.execute_batch( + "DROP TRIGGER visible_turns_scheduler_rowid_insert; + DROP TRIGGER visible_turns_scheduler_rowid_delete; + DROP TRIGGER conclusions_scheduler_rowid_insert; + DROP TRIGGER conclusions_scheduler_rowid_delete; + DROP TRIGGER checkpoints_scheduler_rowid_insert; + DROP TRIGGER checkpoints_scheduler_rowid_delete; + DROP TRIGGER memory_sources_scheduler_rowid_insert; + DROP TRIGGER memory_sources_scheduler_rowid_delete; + DROP TRIGGER memory_records_scheduler_rowid_insert; + DROP TRIGGER memory_records_scheduler_rowid_delete; + DROP TABLE scheduler_source_rowids; + DROP TABLE scheduler_source_rowid_counter; + DELETE FROM schema_meta + WHERE key = 'scheduler_source_rowids_migrated';", + )?; + Ok(()) + }) + .expect("prepare a synthetic pre-migration database"); + drop(store); + + let mut migration_conn = rusqlite::Connection::open(&db_path).expect("migration db"); + let concurrent_writer = rusqlite::Connection::open(&db_path).expect("writer db"); + concurrent_writer + .execute_batch("PRAGMA busy_timeout = 0;") + .expect("make the competing write fail immediately"); + let mut concurrent_insert = None; + + migrate_scheduler_source_rowids_with_hook(&mut migration_conn, || { + concurrent_insert = Some(concurrent_writer.execute( + "INSERT INTO visible_turns(id, session_id, actor, content, created_at) + VALUES ('concurrent-synthetic', 'synthetic-session', 'user', 'synthetic', + '2029-12-31T23:59:30Z')", + [], + )); + }) + .expect("atomically migrate scheduler row IDs"); + + let blocked = concurrent_insert + .expect("migration hook attempted the concurrent insert") + .expect_err("the migration's immediate transaction must hold the write lock"); + assert!( + matches!( + blocked, + rusqlite::Error::SqliteFailure(ref error, _) + if matches!( + error.code, + rusqlite::ErrorCode::DatabaseBusy | rusqlite::ErrorCode::DatabaseLocked + ) + ), + "expected a SQLite busy/locked error, got {blocked}" + ); + + concurrent_writer + .execute( + "INSERT INTO visible_turns(id, session_id, actor, content, created_at) + VALUES ('concurrent-synthetic', 'synthetic-session', 'user', 'synthetic', + '2029-12-31T23:59:30Z')", + [], + ) + .expect("write succeeds after migration commits"); + let scheduler_rowid = concurrent_writer + .query_row( + "SELECT scheduler_rowid FROM scheduler_source_rowids + WHERE stream = 'visible_turns' AND source_key = 'concurrent-synthetic'", + [], + |row| row.get::<_, i64>(0), + ) + .expect("read concurrent row's stable scheduler ID"); + assert!( + scheduler_rowid > 500, + "post-migration writes must sort after the saved cursor high-water" + ); + } + #[test] fn backend_info_describes_current_sqlite_rusqlite_backend() { let store = mem_store(); diff --git a/tests/schema_upgrade.rs b/tests/schema_upgrade.rs index 9f6b3c2..b583d6c 100644 --- a/tests/schema_upgrade.rs +++ b/tests/schema_upgrade.rs @@ -502,7 +502,8 @@ fn scheduler_rowid_upgrade_preserves_saved_cursor_highwater() { DROP TRIGGER memory_records_scheduler_rowid_insert; DROP TRIGGER memory_records_scheduler_rowid_delete; DROP TABLE scheduler_source_rowids; - DROP TABLE scheduler_source_rowid_counter;", + DROP TABLE scheduler_source_rowid_counter; + DELETE FROM schema_meta WHERE key = 'scheduler_source_rowids_migrated';", )?; Ok(()) }) @@ -535,6 +536,49 @@ fn scheduler_rowid_upgrade_preserves_saved_cursor_highwater() { scheduler_rowid > 97, "new rows must sort after every rowid saved in a pre-migration cursor" ); + + let counter_before_reopen = upgraded + .transaction(|tx| { + tx.query_row( + "SELECT sequence FROM scheduler_source_rowid_counter WHERE singleton = 1", + [], + |row| row.get::<_, i64>(0), + ) + .map_err(Into::into) + }) + .expect("read scheduler rowid counter before reopen"); + upgraded + .transaction(|tx| { + tx.execute( + "UPDATE dream_runs SET watermark_after = ?1 WHERE run_id = ?2", + params![ + format!( + "dream-cursor-v1:{}", + json!({"upper_rowids": {"visible_turns": 100_000}}) + ), + "legacy-scheduler-cursor" + ], + )?; + Ok(()) + }) + .expect("seed a later synthetic cursor"); + drop(upgraded); + + let reopened = Store::open(&db_path).expect("reopen migrated database"); + let counter_after_reopen = reopened + .transaction(|tx| { + tx.query_row( + "SELECT sequence FROM scheduler_source_rowid_counter WHERE singleton = 1", + [], + |row| row.get::<_, i64>(0), + ) + .map_err(Into::into) + }) + .expect("read scheduler rowid counter after reopen"); + assert_eq!( + counter_after_reopen, counter_before_reopen, + "an ordinary reopen must not rescan and reseed historical scheduler cursors" + ); } #[test]