From b6760b09ff7fe6e49f6635e30d10e3d4d98aa4fa Mon Sep 17 00:00:00 2001 From: daniel Date: Tue, 29 Sep 2026 04:05:18 +0100 Subject: [PATCH 1/4] feat(gateway): recover remote sessions with atomic transcript replay --- docs/user/gateway.md | 29 +- src/gateway.rs | 355 ++++++++-- src/gateway/http_client.rs | 83 +-- src/gateway/http_client/recovery.rs | 808 ++++++++++++++++++++++ src/gateway/http_client/recovery/tests.rs | 465 +++++++++++++ src/gateway/http_server.rs | 123 +++- src/gateway/tests.rs | 454 +++++++++++- src/main.rs | 60 +- src/tui/app.rs | 25 +- src/tui/app/recovery.rs | 465 +++++++++++++ src/tui/mod.rs | 148 +++- src/tui/recovery.rs | 404 +++++++++++ src/tui/ui.rs | 7 + tests/gateway.rs | 627 ++++++++++++++++- 14 files changed, 3898 insertions(+), 155 deletions(-) create mode 100644 src/gateway/http_client/recovery.rs create mode 100644 src/gateway/http_client/recovery/tests.rs create mode 100644 src/tui/app/recovery.rs create mode 100644 src/tui/recovery.rs diff --git a/docs/user/gateway.md b/docs/user/gateway.md index 08660ac..d600d33 100644 --- a/docs/user/gateway.md +++ b/docs/user/gateway.md @@ -56,6 +56,27 @@ If the child is resident, this attaches to that process. After a gateway restart The remote TUI supports normal prompting, model/config selection advertised by the host, steering, and explicit interruption. `Esc` or `Ctrl+C` during a running turn interrupts it. Quitting with `Ctrl+D` on an empty prompt, or losing the connection, detaches without interruption. One attachment controls a session at a time. Clean disconnection releases the attachment. An explicit resume replaces the previous controller, so a lost HTTP stream does not prevent reconnection. The old controller cannot submit new work once replaced. +### Automatic recovery and uncertain requests + +After a transient transport failure, the bridge keeps the terminal open and makes a bounded series of fresh ACP connections. The TUI shows **Reconnecting** while it initializes and resumes the same confirmed session ID. It does not assume reopening SSE recovers missed events: full session replay builds a private replacement view, followed by the authoritative current configuration and turn state. Only a successful coherent attachment replaces the visible transcript; a failed attempt leaves the previous view intact. Replay replaces rather than appends history, and stale attachment generations cannot update the recovered view. + +Recovery does **not** retry prompts, session creation, configuration changes, steering, or cancellation. A submitted operation can have an **unknown outcome** after a lost response. Inspect the recovered transcript and state before deciding what to send next; retained composer text is not evidence that its earlier submission was rejected. New submissions during recovery are rejected rather than saved for automatic execution. If creation may have succeeded but no session ID was received, use `kit gateway list` and select `--remote-session` explicitly; the bridge never creates another session to guess the outcome. + +Authentication, protocol, project, and replaced-controller errors stop automatic recovery. Automatic recovery is conditional on the previous attachment generation and cannot repeatedly steal control from a newer explicit attachment. Repeated transient failures exhaust a finite attempt/time budget and leave a visible disconnected state with recovery guidance. + +### Recovering without historical replay + +A full history that exceeds replay or transport limits is not silently truncated, and automatic recovery never silently switches to no-replay. To explicitly regain an existing session without its historical transcript, restart with: + +```sh +kit tui --remote http://127.0.0.1:7766 \ + --remote-credential-file "$HOME/.kit-gateway-token" \ + --root /absolute/server/project --remote-session SESSION_ID \ + --remote-no-replay +``` + +This option requires an existing session ID. The TUI marks **History unavailable**; an empty historical pane is not a claim that the session has no history. The attachment still requires the authoritative current configuration and active/idle state before enabling control. The durable transcript stays on the host, and in-flight work is not restarted. Direct ACP clients can request the same no-replay behavior by omitting `replayFrom` from `session/resume`; full replay uses `{"type":"start"}`. + To start another session, exit and omit `--remote-session` on the next invocation. To switch sessions, exit, list them, and supply the desired ID. In-place `/new`, session listing/resume/rename commands, local provider login/usage, and voice are disabled in a remote attachment. This milestone does not provide remote session deletion or a close command. Unsupported ACP close requests fail explicitly; they are not reported as successful no-ops. ## Standard ACP HTTP clients @@ -106,7 +127,7 @@ Unsupported requests fail explicitly. Client-to-agent request cancellation (`$/c ### Pinned wire fixture -`tests/gateway.rs::pinned_sdk_http_initialize_capabilities_and_session_stream_contract` is an executable HTTP fixture for the SDK pinned at `2f039993d1d6ed8da35b38c31f54a7cbb7338c70`. It validates these payloads and headers against a real gateway without any control-plane service. +`tests/gateway.rs::pinned_sdk_http_initialize_capabilities_and_session_stream_contract` is an executable HTTP fixture for the SDK pinned at `423ba77cd555a09f68472b93c142c8f0baaabf43`. It validates these payloads and headers against a real gateway without any control-plane service. Initialize POST body (send `Authorization: Bearer TOKEN` and `Content-Type: application/json`): @@ -123,7 +144,7 @@ The response is HTTP 200 JSON with `Acp-Connection-Id`. Its `result` contains `p Its `result._meta` explicitly identifies the experimental limits: ```json -{"kit/gateway":{"experimental":true,"transport":"bounded-http","maxFrameBytes":1048576,"coreBufferedBytesPerDirection":16777216,"httpEgressBytesPerConnection":4194304,"liveReplayLimitBytes":8388608,"liveReplayLimitEvents":4094}} +{"kit/gateway":{"experimental":true,"transport":"bounded-http","maxFrameBytes":1048576,"coreBufferedBytesPerDirection":16777216,"httpEgressBytesPerConnection":4194304,"liveReplayLimitBytes":8388608,"liveReplayLimitEvents":4093}} ``` Both connection and session SSE GETs return HTTP 200 with `Content-Type: text/event-stream`. The session stream echoes both ACP ID headers. Subsequent POST and transport DELETE return HTTP 202. SSE `data:` contains JSON-RPC replies or notifications, for example: @@ -142,10 +163,10 @@ The error above illustrates a session-routed revoke of an unknown pending messag - Client redirects are disabled so the bearer token cannot follow a redirect. Use a trusted URL and transport; HTTPS can be provided by a separately secured proxy, not by this command itself. The bundled client uses bounded HTTP/SSE parsing and charged core mailboxes. Transport limits do not make an untrusted gateway safe: it still supplies protocol data and can terminate the connection. - A failed send can have an **unknown outcome**. There is no automatic prompt retry. Reconnect and inspect the transcript before resubmitting, or you may duplicate work. A lost create response can leave a resident session; list sessions before creating another. - Live reattachment replays the resident child's notifications and current configuration, not old request responses. Replies are scoped to the attachment that submitted the request, so a stale response cannot resolve a new terminal's request ID. ACP v2 state-update notifications carry active/idle state across reconnects. -- Replay is an in-memory convenience, not a second durable session format. It is limited to 8 MiB and approximately 4,096 events per session/attachment. Overflow expires an attachment or rejects a full live reattach rather than silently claiming complete replay. For large histories, reconnect with `session/resume` and omit `replayFrom` (or set it to null) to regain control without replay. The bundled TUI requests full replay and can therefore fail to reconnect beyond these limits; a no-replay ACP client is required in that case. A no-replay resume does not currently provide an authoritative active/idle snapshot; durable assistant content alone does not prove the turn is idle. Do not automatically submit another prompt based on either. Restarting the gateway does not guarantee oversized full replay will fit, and burst replay can exhaust transport budgets even below the history limit. +- Replay is an in-memory convenience, not a second durable session format. It is limited to 8 MiB and approximately 4,096 events per session/attachment, including room for a current snapshot. Overflow rejects a full live reattach rather than falsely claiming complete replay. Use explicit `--remote-no-replay` only when historical omission is acceptable. Current configuration and actual ACP turn state follow historical events before resume completes; durable assistant text alone is never used as evidence that a turn is idle. Restarting the gateway does not guarantee oversized full replay will fit, and burst replay can exhaust transport budgets even below the history limit. - Healthy HTTP connections have no cumulative output cap. The SDK instead limits each core direction to 16 MiB / 256 frames and each connection's HTTP egress to 4 MiB / 64 frames. These are independent encoded-data reservations, not an aggregate heap limit. Charged envelopes remain owned through local decoding and serialization. Admission saturation fails explicitly and can terminate the connection; it does not silently drop output while claiming the stream is complete. Accepted resident work is not cancelled by transport failure. There is no delivery acknowledgement, and HTTP body handoff does not prove peer consumption. The bridge's stdin mailbox holds at most 16 capped frames; each frame is limited to 1 MiB before parsing. - The server admits at most 64 connections, 32 concurrent SDK POSTs, and 8 MiB of aggregate body reservations. Each connection permits at most 128 batch entries, 256 pending routes, 64 session registrations, and 65 streams. HTTP/core frames are limited to 1 MiB; SSE encoding and the client's 1 MiB parser limits can reject a near-limit frame. The bundled client independently caps HTTP reservations at 64 MiB / 128 slots, pending RPCs at 128, each POST lane at 32, and SSE streams at 8. Transport exhaustion is terminal rather than an unbounded wait. The gateway's outer admission layer admits at most 32 concurrent POST/DELETE operations before buffering or waiting on per-connection teardown. -- These bounds exclude allocator overhead, transient JSON conversion, arbitrary application state, and HTTP/TLS/socket buffers. They are not a hard process-memory ceiling or a denial-of-service-hardening claim. The bounded transport supports HTTP/SSE, not WebSocket upgrades. SSE has no replay cursor: reopening a stream alone cannot recover lost events. Automatic reconnect and authoritative TUI transcript replacement are not implemented. There is no idle connection expiration or body-read deadline: disconnected peers that do not DELETE or otherwise terminate their transport can retain connection slots. Slot exhaustion may require restarting the gateway; do not expose it to untrusted clients. +- These bounds exclude allocator overhead, transient JSON conversion, arbitrary application state, and HTTP/TLS/socket buffers. They are not a hard process-memory ceiling or a denial-of-service-hardening claim. The bounded transport supports HTTP/SSE, not WebSocket upgrades. SSE has no replay cursor: reopening a stream alone cannot recover lost events. Recovery uses fresh initialization and session-level resume, not SSE event IDs. There is no idle connection expiration or body-read deadline: disconnected peers that do not DELETE or otherwise terminate their transport can retain connection slots. Slot exhaustion may require restarting the gateway; do not expose it to untrusted clients. - At most 64 active or stopping actors occupy session slots. Listing or creating sessions reclaims completed actor slots without restarting the gateway or deleting durable transcripts. Detached actors, including idle actors, are not automatically terminated; accepted work continues. HTTP POST bodies are limited to 1 MiB inside SDK admission, including streamed bodies; supervisor command queues and pending child requests are also bounded. Child stdout frames are limited to 8 MiB before JSON parsing; oversized or malformed frames stop that child. - Graceful `Ctrl+C` shutdown stops serving, releases resident actor ownership, closes each ACP child’s input, drains its output, and waits for cleanup and exit. A 10-second timeout falls back to killing and reaping the child. That fallback can leave a stale transcript lock requiring operator intervention. Hard process termination has operating-system-dependent cleanup behavior; arbitrary tool descendants are not a managed process group. - Client-side ACP services, arbitrary extra directories, remote MCP injection, browser callbacks, and arbitrary ACP methods are not supported. Files, tool execution, and provider authentication belong to the gateway host. diff --git a/src/gateway.rs b/src/gateway.rs index 60ef1a6..3f0e105 100644 --- a/src/gateway.rs +++ b/src/gateway.rs @@ -63,6 +63,7 @@ enum InternalCommand { credential_file: PathBuf, root: PathBuf, session: Option, + no_replay: bool, }, } impl Args { @@ -116,7 +117,12 @@ impl Args { .required(true) .value_parser(value_parser!(PathBuf)), ) - .arg(Arg::new("session").long("session")), + .arg(Arg::new("session").long("session")) + .arg( + Arg::new("no_replay") + .long("no-replay") + .action(ArgAction::SetTrue), + ), ) } pub fn from_matches(matches: &clap::ArgMatches) -> Result { @@ -130,6 +136,7 @@ impl Args { credential_file: required_arg(args, "credential_file")?, root: required_arg(args, "root")?, session: optional_arg(args, "session")?, + no_replay: required_arg(args, "no_replay")?, }), Some((name, _)) => { return Err(clap::Error::raw( @@ -177,6 +184,7 @@ pub struct Remote { pub url: String, pub credential_file: PathBuf, pub session: Option, + pub no_replay: bool, } impl Remote { pub fn command(&self, root: &Path) -> io::Result { @@ -189,6 +197,9 @@ impl Remote { if let Some(id) = &self.session { command.arg("--session").arg(id); } + if self.no_replay { + command.arg("--no-replay"); + } Ok(command) } } @@ -214,6 +225,8 @@ enum Request { replay: bool, #[cfg_attr(test, serde(default))] replace: bool, + #[cfg_attr(test, serde(default))] + startup: Option, }, Poll { session: String, @@ -235,21 +248,41 @@ enum Request { } type ResultValue = Result; #[derive(Debug)] -struct Failure(StatusCode, String); +struct Failure(StatusCode, String, &'static str); impl Failure { + fn replay(message: impl Into) -> Self { + Self(StatusCode::CONFLICT, message.into(), "replay_unavailable") + } + fn data(&self) -> Value { + object([ + ("reason", self.2.into()), + ("terminal", (self.2 != "unavailable").into()), + ]) + } + fn root(message: impl Into) -> Self { + Self(StatusCode::BAD_REQUEST, message.into(), "root_denied") + } fn bad(message: impl Into) -> Self { - Self(StatusCode::BAD_REQUEST, message.into()) + Self(StatusCode::BAD_REQUEST, message.into(), "protocol") } fn conflict(message: impl Into) -> Self { - Self(StatusCode::CONFLICT, message.into()) + Self(StatusCode::CONFLICT, message.into(), "conflict") } fn unavailable(message: impl Into) -> Self { - Self(StatusCode::SERVICE_UNAVAILABLE, message.into()) + Self( + StatusCode::SERVICE_UNAVAILABLE, + message.into(), + "unavailable", + ) } } impl IntoResponse for Failure { fn into_response(self) -> Response { - (self.0, Json(object([("error", self.1.into())]))).into_response() + ( + self.0, + Json(object([("data", self.data()), ("error", self.1.into())])), + ) + .into_response() } } struct Gateway { @@ -343,12 +376,14 @@ pub async fn run(args: Args) -> Result<(), Box> { credential_file, root, session, + no_replay, } => { http_client::bridge( Remote { url, credential_file, session, + no_replay, }, root, ) @@ -362,6 +397,7 @@ pub async fn run(args: Args) -> Result<(), Box> { url, credential_file, session: None, + no_replay: false, }) .await } @@ -401,7 +437,7 @@ pub async fn run(args: Args) -> Result<(), Box> { // HTTP connections can own long-lived SSE streams. Stop the listener and // release resident ownership without waiting indefinitely for clients. eprintln!( - "Experimental gateway: bounded HTTP transport (1 MiB frames, 64 connections); live replay is limited to 8 MiB / 4094 events. Resume without replay when history exceeds these limits; transcripts remain on the host." + "Experimental gateway: bounded HTTP transport (1 MiB frames, 64 connections); live replay is limited to 8 MiB / 4093 events. Resume without replay when history exceeds these limits; transcripts remain on the host." ); let served = { use std::future::IntoFuture as _; @@ -542,9 +578,9 @@ async fn handle( } => { let root = root .canonicalize() - .map_err(|_| Failure::bad("project is not registered"))?; + .map_err(|_| Failure::root("project is not registered"))?; if !gateway.roots.contains(&root) { - return Err(Failure::bad("project is not registered")); + return Err(Failure::root("project is not registered")); } gateway.reclaim_exited().await; let restore = session.is_some(); @@ -553,7 +589,7 @@ async fn handle( let entries = gateway.sessions.lock().await; if let Some(entry) = entries.get(&id).filter(|e| !e.sender.is_closed()) { if entry.root != root { - return Err(Failure::bad("session belongs to another project")); + return Err(Failure::root("session belongs to another project")); } return Ok(Json(object([ ("session", id.into()), @@ -563,7 +599,7 @@ async fn handle( } drop(entries); if !crate::session::belongs_to_workspace(&root, &id).map_err(Failure::bad)? { - return Err(Failure::bad("session does not belong to this project")); + return Err(Failure::root("session does not belong to this project")); } } let mut entries = gateway.sessions.lock().await; @@ -572,7 +608,7 @@ async fn handle( } if let Some(entry) = entries.get(&id).filter(|e| !e.sender.is_closed()) { if entry.root != root { - return Err(Failure::bad("session belongs to another project")); + return Err(Failure::root("session belongs to another project")); } } else { if entries.len() >= MAX_SESSIONS && !entries.contains_key(&id) { @@ -610,6 +646,7 @@ async fn handle( Failure( StatusCode::NOT_FOUND, "session is not resident; create with session id to restore".into(), + "session_unavailable", ) })?; let (reply, result) = oneshot::channel(); @@ -627,6 +664,8 @@ async fn handle( } struct Attachment { id: String, + recovery_id: Option, + recovery_attempt: Option, touched: Instant, next: u64, events: VecDeque<(u64, Value)>, @@ -661,6 +700,9 @@ struct Actor { pending: HashMap, initialized: Option, session_result: Option, + // Native child sessions start idle; only lifecycle notifications change this. + // Independent of bounded transcript retention, never inferred from text. + state: Value, journal: VecDeque, journal_bytes: usize, replay_complete: bool, @@ -692,6 +734,10 @@ fn spawn(root: &Path, id: &str, restore: bool, force: bool) -> io::Result pending: HashMap::new(), initialized: None, session_result: None, + state: object([ + ("sessionUpdate", "state_update".into()), + ("state", "idle".into()), + ]), journal: VecDeque::new(), journal_bytes: 0, replay_complete: true, @@ -815,7 +861,7 @@ impl Actor { fn remember(&mut self, message: Value) { self.journal_bytes += message.to_string().len(); self.journal.push_back(message); - while self.journal_bytes > MAX_REPLAY || self.journal.len() > MAX_QUEUE - 2 { + while self.journal_bytes > MAX_REPLAY || self.journal.len() > MAX_QUEUE - 3 { if let Some(old) = self.journal.pop_front() { self.journal_bytes -= old.to_string().len(); } @@ -837,6 +883,12 @@ impl Actor { ), ])); } + if message["method"] == "session/update" + && message["params"]["sessionId"] == self.id + && message["params"]["update"]["sessionUpdate"] == "state_update" + { + self.state = message["params"]["update"].clone(); + } if let Some(options) = message["params"]["update"].get("configOptions") && let Some(result) = &mut self.session_result { @@ -880,32 +932,63 @@ impl Actor { if matches!(pending.method.as_str(), "session/new" | "session/resume") && message.get("result").is_some() { - if pending.replay && !self.replay_complete { - self.emit(object([ - ("jsonrpc", "2.0".into()), - ("id", message["id"].clone()), - ( - "error", - object([ - ("code", (-32000).into()), - ( - "message", - "session opened but full replay exceeds the gateway limit" - .into(), - ), - ]), - ), - ])); - return None; - } - message["result"]["sessionId"] = Value::String(self.id.clone()); - if pending.replay { - for update in self.journal.clone() { - self.emit(update); + let messages = self.startup_messages( + message["result"].clone(), + message["id"].clone(), + pending.replay, + &pending.attachment, + ); + match messages { + Ok(messages) => { + // Include any unpolled initialization response in admission. + let fits = self.attachment.as_ref().is_some_and(|a| { + a.events.len() + messages.len() <= MAX_QUEUE + && a.bytes + + messages + .iter() + .map(|m| m.to_string().len()) + .sum::() + <= MAX_REPLAY + }); + if fits { + for update in messages { + self.emit(update); + } + if let Some(attachment) = &mut self.attachment { + attachment.live = true; + } + return None; + } + message = object([ + ("jsonrpc", "2.0".into()), + ("id", message["id"].clone()), + ( + "error", + object([ + ("code", (-32000).into()), + ( + "message", + "session snapshot exceeds the gateway limit".into(), + ), + ("data", Failure::replay("snapshot overflow").data()), + ]), + ), + ]); + } + Err(error) => { + message = object([ + ("jsonrpc", "2.0".into()), + ("id", message["id"].clone()), + ( + "error", + object([ + ("code", (-32000).into()), + ("data", error.data()), + ("message", error.1.into()), + ]), + ), + ]); } - } - if let Some(attachment) = &mut self.attachment { - attachment.live = true; } } self.emit(message); @@ -913,11 +996,136 @@ impl Actor { } None } + fn startup_messages( + &self, + mut result: Value, + id: Value, + replay: bool, + attachment: &str, + ) -> Result, Failure> { + if replay && !self.replay_complete { + return Err(Failure::replay("full live replay is unavailable")); + } + let mut messages: Vec = if replay { + self.journal.iter().cloned().collect() + } else { + Vec::new() + }; + let options = result + .get("configOptions") + .cloned() + .unwrap_or_else(|| Value::Array(Vec::new())); + for update in [ + object([ + ("sessionUpdate", "config_option_update".into()), + ("configOptions", options), + ]), + self.state.clone(), + ] { + messages.push(object([ + ("jsonrpc", "2.0".into()), + ("method", "session/update".into()), + ( + "params", + object([("sessionId", self.id.clone().into()), ("update", update)]), + ), + ])); + } + result["sessionId"] = self.id.clone().into(); + result["_meta"]["kit/gateway"] = object([ + ("attachment", attachment.into()), + ("historyAvailable", replay.into()), + ("stateSnapshot", true.into()), + ("configSnapshot", true.into()), + ]); + messages.push(object([ + ("jsonrpc", "2.0".into()), + ("id", id), + ("result", result), + ])); + if messages + .iter() + .any(|message| message.to_string().len() > MAX_HTTP_FRAME) + || messages.len() > MAX_QUEUE + || messages.iter().map(|m| m.to_string().len()).sum::() > MAX_REPLAY + { + return Err(Failure::replay( + "session snapshot exceeds the gateway limit", + )); + } + Ok(messages) + } fn command(&mut self, request: Request, writes: &mpsc::Sender) -> ResultValue { if let Request::Attach { - replace, replay, .. + replace, + replay, + startup, + .. } = request { + let metadata = startup + .as_ref() + .map(|message| &message["params"]["_meta"]["kit/gateway"]); + let token = |name: &str| -> Result, Failure> { + metadata + .and_then(|metadata| metadata.get(name)) + .map(|value| { + value + .as_str() + .filter(|value| !value.is_empty() && value.len() <= 128) + .map(str::to_owned) + .ok_or_else(|| { + Failure::bad(format!( + "{name} must be a nonempty string of at most 128 bytes" + )) + }) + }) + .transpose() + }; + let previous = token("previousAttachment")?; + let recovery_id = token("recoveryId")?; + let recovery_attempt = metadata + .and_then(|metadata| metadata.get("recoveryAttempt")) + .map(|value| { + value + .as_u64() + .filter(|attempt| *attempt > 0) + .ok_or_else(|| Failure::bad("recoveryAttempt must be a positive u64")) + }) + .transpose()?; + if recovery_id.is_some() != recovery_attempt.is_some() { + return Err(Failure::bad( + "recoveryId and recoveryAttempt must be supplied together", + )); + } + if recovery_id.is_some() && previous.is_none() { + return Err(Failure::bad("recoveryId requires previousAttachment")); + } + // A matching predecessor must not bypass the sequence high-water mark. + // Validate before preparing anything, and commit the mark only with ownership. + if let Some(owner) = &self.attachment + && recovery_id.is_some() + && owner.recovery_id == recovery_id + && recovery_attempt <= owner.recovery_attempt + { + return Err(Failure( + StatusCode::CONFLICT, + "recovery attempt is stale; automatic resume refused".into(), + "stale_recovery", + )); + } + if let Some(previous) = previous + && self.attachment.as_ref().is_some_and(|owner| { + owner.id != previous + && !(recovery_id.is_some() && owner.recovery_id == recovery_id) + }) + { + return Err(Failure( + StatusCode::CONFLICT, + "controller was replaced; automatic resume refused".into(), + "controller_replaced", + )); + } if !replace && self .attachment @@ -929,20 +1137,44 @@ impl Actor { )); } if replay && !self.replay_complete { - return Err(Failure::conflict( + return Err(Failure::replay( "live replay limit reached; transcript is durable but a full live reattach is unavailable", )); } let id = crate::session::new_id(); - self.attachment = Some(Attachment { + let mut attachment = Attachment { id: id.clone(), + recovery_id, + recovery_attempt, touched: Instant::now(), next: 0, events: VecDeque::new(), bytes: 0, live: false, - }); + }; + let started = + if let (Some(startup), Some(result)) = (startup, self.session_result.clone()) { + let request_id = startup + .get("id") + .cloned() + .ok_or_else(|| Failure::bad("session startup requires a request id"))?; + for message in self.startup_messages(result, request_id, replay, &id)? { + if !attachment.push(message) { + return Err(Failure::replay( + "session snapshot exceeds the gateway limit", + )); + } + } + attachment.live = true; + true + } else { + false + }; + // No fallible work after the ownership commit. The previous controller + // survives every replay/snapshot preparation failure. + self.attachment = Some(attachment); return Ok(object([ + ("started", started.into()), ("attachment", id.into()), ("lease_seconds", LEASE.as_secs().into()), ])); @@ -957,7 +1189,13 @@ impl Actor { .attachment .as_mut() .filter(|a| a.id == *token && a.touched.elapsed() < LEASE) - .ok_or_else(|| Failure::conflict("attachment expired or was replaced; reconnect"))?; + .ok_or_else(|| { + Failure( + StatusCode::CONFLICT, + "attachment expired or was replaced; reconnect".into(), + "controller_replaced", + ) + })?; attachment.touched = Instant::now(); match request { Request::Detach { .. } => { @@ -1057,7 +1295,7 @@ impl Actor { let replay = method == "session/new" || message["params"]["replayFrom"] == object([("type", "start".into())]); if startup && replay && !self.replay_complete { - return Err(Failure::conflict("full live replay is unavailable")); + return Err(Failure::replay("full live replay is unavailable")); } let cached = match method.as_str() { "initialize" => self.initialized.clone(), @@ -1065,19 +1303,30 @@ impl Actor { _ => None, }; - if let Some(mut result) = cached { - if matches!(method.as_str(), "session/new" | "session/resume") { - result["sessionId"] = Value::String(self.id.clone()); - if replay { - for update in self.journal.clone() { - self.emit(update); - } + if let Some(result) = cached { + if startup { + let Some(id) = original else { + return Err(Failure::bad("session startup requires a request id")); + }; + let messages = self.startup_messages(result, id, replay, &attachment)?; + let fits = self.attachment.as_ref().is_some_and(|a| { + a.events.len() + messages.len() <= MAX_QUEUE + && a.bytes + + messages.iter().map(|m| m.to_string().len()).sum::() + <= MAX_REPLAY + }); + if !fits { + return Err(Failure::replay( + "session snapshot exceeds the gateway limit", + )); + } + for update in messages { + self.emit(update); } if let Some(attachment) = &mut self.attachment { attachment.live = true; } - } - if let Some(id) = original { + } else if let Some(id) = original { self.emit(object([ ("jsonrpc", "2.0".into()), ("id", id), diff --git a/src/gateway/http_client.rs b/src/gateway/http_client.rs index f1b181f..11b7aa6 100644 --- a/src/gateway/http_client.rs +++ b/src/gateway/http_client.rs @@ -1,6 +1,8 @@ //! Stdio relay for the SDK's ACP v2 HTTP transport. use super::object; -use agent_client_protocol::schema::v1::{RequestId, Response}; +use agent_client_protocol::schema::v1::RequestId; +#[cfg(test)] +use agent_client_protocol::schema::v1::Response; use agent_client_protocol::{BoundedChannel, ChargedFrame, RawJsonRpcMessage, TransportFrame}; use agent_client_protocol_http::{ BoundedHttpClient, HttpClient as AcpHttpClient, HttpClientLimits, @@ -83,7 +85,7 @@ fn rewrite( ); if request.method.as_ref() == "session/new" && *initial { *initial = false; - if let Some(id) = session.take() { + if let Some(id) = session.clone() { *resumed = Some((request.id.clone(), id.clone())); request.method = "session/resume".into(); params["sessionId"] = Value::String(id); @@ -109,6 +111,7 @@ fn rewrite( // The local TUI issued session/new, whose response requires sessionId. ACP // session/resume omits it, so restore that field only on the rewritten response. +#[cfg(test)] fn restore_new_response(frame: &mut TransportFrame, resumed: &mut Option<(RequestId, String)>) { fn message(value: &mut RawJsonRpcMessage, resumed: &mut Option<(RequestId, String)>) { let Some((expected, session)) = resumed.as_ref() else { @@ -140,14 +143,13 @@ fn restore_new_response(frame: &mut TransportFrame, resumed: &mut Option<(Reques } } +mod recovery; + pub(super) async fn bridge( remote: super::Remote, root: PathBuf, ) -> Result<(), Box> { - let (mut channel, mut transport) = client(&remote)?.into_bounded_channel_and_future(); - // A dedicated reader avoids Tokio's uncancellable blocking stdin read keeping - // the runtime alive after a remote disconnect. - let (send, mut lines) = tokio::sync::mpsc::channel(16); + let (send, lines) = tokio::sync::mpsc::channel(16); std::thread::spawn(move || { let mut stdin = io::stdin().lock(); loop { @@ -155,66 +157,19 @@ pub(super) async fn bridge( Ok(Some(line)) => Ok(line), Ok(None) => break, Err(error) => { - let _ = send.blocking_send(Err(error)); + let _ = send.blocking_send((std::time::Instant::now(), Err(error))); break; } }; - if send.blocking_send(line).is_err() { + if send + .blocking_send((std::time::Instant::now(), line)) + .is_err() + { break; } } }); - let mut session = remote.session; - let mut initial = true; - let mut resumed = None; - let mut input_open = true; - enum Event { - Frame(Option), - Transport(Result<(), agent_client_protocol::Error>), - Line(Option>), - } - loop { - let event = { - let frame = std::pin::pin!(channel.rx.next()); - let line = std::pin::pin!(async { - if input_open { - lines.recv().await - } else { - std::future::pending().await - } - }); - match select(frame, select(&mut transport, line)).await { - Either::Left((frame, _)) => Event::Frame(frame), - Either::Right((Either::Left((result, _)), _)) => Event::Transport(result), - Either::Right((Either::Right((line, _)), _)) => Event::Line(line), - } - }; - match event { - Event::Frame(Some(charged)) => { - // Keep the reservation until decoded data and stdout serialization - // have both been consumed; never queue an uncharged decoded frame. - let mut frame = charged.decode(); - restore_new_response(&mut frame, &mut resumed); - print_frame(&frame)?; - drop(frame); - drop(charged); - } - Event::Frame(None) => return transport.await.map_err(transport_error), - Event::Transport(result) => return result.map_err(transport_error), - Event::Line(Some(line)) => { - let mut frame = TransportFrame::parse_json(&line?); - rewrite(&mut frame, &root, &mut session, &mut initial, &mut resumed)?; - channel.tx.try_send(frame).map_err(transport_error)?; - } - Event::Line(None) => { - input_open = false; - // Keep polling the driver after EOF: it drains accepted POSTs - // and awaits HTTP DELETE before returning. Dropping it here - // would race process shutdown against best-effort cleanup. - channel.tx.close_channel(); - } - } - } + recovery::run(remote, root, lines, io::stdout()).await } // Bound before parsing or entering the 16-slot local mailbox. BufRead::lines() @@ -248,14 +203,6 @@ fn transport_error(error: agent_client_protocol::Error) -> Box Result<(), Box> { - use std::io::Write; - let mut stdout = io::stdout().lock(); - writeln!(stdout, "{}", frame.to_json()?)?; - stdout.flush()?; - Ok(()) -} - async fn request( channel: &mut BoundedChannel, id: i64, @@ -403,6 +350,7 @@ mod tests { url: format!("http://{address}"), credential_file: credential.path().to_owned(), session: None, + no_replay: false, }), ) .await; @@ -476,6 +424,7 @@ mod tests { assert_eq!(value["params"]["sessionId"], "remote-session"); assert_eq!(value["params"]["replayFrom"], json!({"type": "start"})); assert_eq!(value["id"], 7); + assert_eq!(session.as_deref(), Some("remote-session")); let mut response = TransportFrame::parse_json(r#"{"jsonrpc":"2.0","id":7,"result":{"configOptions":[]}}"#); restore_new_response(&mut response, &mut resumed); diff --git a/src/gateway/http_client/recovery.rs b/src/gateway/http_client/recovery.rs new file mode 100644 index 0000000..76de23c --- /dev/null +++ b/src/gateway/http_client/recovery.rs @@ -0,0 +1,808 @@ +//! Single-owner reconnect state. Only initialize and same-session resume retry. +use super::*; +use agent_client_protocol::schema::v2::UpdateSessionNotification; +use std::time::{Duration, Instant}; + +const PREFIX: &str = "kit.gateway.internal/"; +const MAX_PENDING: usize = 128; +const META: &str = "kit/gatewayRecovery"; +type Error = Box; +type Lines = tokio::sync::mpsc::Receiver<(Instant, io::Result)>; + +fn emit(output: &mut impl io::Write, value: &Value) -> Result<(), Error> { + writeln!(output, "{value}")?; + output.flush()?; + Ok(()) +} +fn entries(value: &Value) -> &[Value] { + match value { + Value::Array(values) => values, + _ => std::slice::from_ref(value), + } +} +fn reserved(id: &Value) -> bool { + id.as_str().is_some_and(|id| id.starts_with(PREFIX)) +} +fn reject( + output: &mut impl io::Write, + value: &Value, + reason: &str, + session: &Option, +) -> Result<(), Error> { + for value in entries(value) { + if value.get("method").is_some() && value.get("id").is_some() { + emit( + output, + &object([ + ("jsonrpc", "2.0".into()), + ("id", value["id"].clone()), + ( + "error", + object([ + ("code", (-32000).into()), + ( + "message", + "Gateway unavailable; inspect session before resubmitting".into(), + ), + ( + "data", + object([ + ("reason", reason.into()), + ("method", value["method"].clone()), + ( + "sessionId", + session.clone().map_or(Value::Null, Value::String), + ), + ]), + ), + ]), + ), + ]), + )?; + } + } + Ok(()) +} +fn notice( + output: &mut impl io::Write, + session: &str, + epoch: u64, + kind: &str, + attempt: u32, + message: &str, + bits: Value, +) -> Result<(), Error> { + let mut meta = object([ + ("epoch", epoch.into()), + ("kind", kind.into()), + ("attempt", attempt.into()), + ("maxAttempts", 5.into()), + ("message", message.into()), + ]); + if let Some(bits) = bits.as_object() { + for (key, value) in bits { + meta[key] = value.clone(); + } + } + emit( + output, + &object([ + ("jsonrpc", "2.0".into()), + ("method", "session/update".into()), + ( + "params", + object([ + ("sessionId", session.into()), + ( + "update", + object([ + ("sessionUpdate", "notice".into()), + ( + "severity", + (if kind == "failed" { "error" } else { "info" }).into(), + ), + ("title", "Gateway recovery".into()), + ("description", message.into()), + ]), + ), + ("_meta", object([(META, meta)])), + ]), + ), + ]), + ) +} +fn annotate(value: &mut Value, session: &str, epoch: u64, kind: &str) { + if value["method"] == "session/update" + && value["params"]["sessionId"] == session + && let Some(params) = value["params"].as_object_mut() + { + let meta = params.entry("_meta").or_insert_with(|| object([])); + if !meta.is_object() { + *meta = object([]); + } + meta[META] = object([("epoch", epoch.into()), ("kind", kind.into())]); + } +} + +fn snapshot(result: &Value, no_replay: bool) -> Option { + let bits = &result["_meta"]["kit/gateway"]; + if bits["stateSnapshot"] != true + || bits["configSnapshot"] != true + || bits["historyAvailable"].as_bool()? == no_replay + { + return None; + } + Some(object([ + ("stateSnapshot", true.into()), + ("configSnapshot", true.into()), + ("historyAvailable", (!no_replay).into()), + ])) +} +fn internal(epoch: u64, method: &str, params: Value) -> Value { + object([ + ("jsonrpc", "2.0".into()), + ("id", format!("{PREFIX}{epoch}/{method}").into()), + ("method", method.into()), + ("params", params), + ]) +} +fn send(channel: &mut BoundedChannel, value: &Value) -> Result<(), Error> { + channel + .tx + .try_send(TransportFrame::parse_json(&value.to_string())) + .map_err(|_| "Gateway transport unavailable; submission outcome may be unknown".into()) +} + +fn terminal_reason(value: &Value) -> Option<&str> { + let data = if value.get("error").is_some() { + &value["error"]["data"] + } else if value["method"] == "session/update" + && value["params"]["update"]["sessionUpdate"] == "_gateway_controller" + { + &value["params"]["update"]["_meta"]["kit/gateway"] + } else { + return None; + }; + if data["terminal"] != true { + return None; + } + Some(match data["reason"].as_str() { + Some( + reason @ ("controller_replaced" + | "replay_unavailable" + | "root_denied" + | "protocol" + | "conflict" + | "session_unavailable"), + ) => reason, + _ => "protocol", + }) +} + +// SDK 423ba's bounded transport exposes status only in Error.data diagnostics. +// Match its exact wrappers and canonical status phrase. The bounded path never +// includes response bodies here; do not accept legacy wrappers or body suffixes. +fn diagnostic_status(error: &agent_client_protocol::Error) -> Option { + let data = error.data.as_ref()?.as_str()?; + let http = [ + "bounded HTTP: initialize HTTP ", + "bounded HTTP: POST HTTP ", + "bounded HTTP: SSE HTTP ", + ] + .iter() + .find_map(|prefix| data.strip_prefix(prefix))?; + let (status, reason) = http.split_once(' ')?; + if status.len() != 3 || !status.bytes().all(|byte| byte.is_ascii_digit()) { + return None; + } + let status: u16 = status.parse().ok()?; + let canonical = reqwest::StatusCode::from_u16(status) + .ok()? + .canonical_reason()?; + (reason == canonical).then_some(status) +} + +pub(super) async fn run( + remote: super::super::Remote, + root: PathBuf, + mut lines: Lines, + mut output: impl io::Write, +) -> Result<(), Error> { + root.to_str().ok_or("remote root must be valid UTF-8")?; + let mut session = remote.session.clone(); + let mut attachment: Option = None; + let mut recovery_id = String::new(); + let mut initial = true; + let mut resumed = None; + let mut initialize: Option = None; + let mut initialized = false; + let mut pending: Vec = Vec::new(); + let mut epoch = 1; + let mut attempt = 0; + let mut recovery_start = None; + let mut gate = None; + let mut terminal: Option = None; + loop { + if let Some(message) = terminal.take() { + for request in pending.drain(..) { + reject(&mut output, &request, "outcome_unknown", &session)?; + } + if let Some(session) = &session { + notice( + &mut output, + session, + epoch, + "failed", + attempt, + &message, + Value::Null, + )?; + } + return Err(message.into()); + } + let recovering = recovery_start.is_some(); + if recovering { + if attempt >= 5 + || recovery_start + .is_some_and(|start: Instant| start.elapsed() >= Duration::from_secs(30)) + { + terminal = Some("Recovery exhausted; submission outcome unknown. Inspect with kit gateway list; restart with explicit --remote-session (or --remote-no-replay if history unavailable).".into()); + continue; + } + attempt += 1; + if session.is_some() { + epoch += 1; + } + if let Some(session) = &session { + notice( + &mut output, + session, + epoch, + "begin", + attempt, + "Reconnecting; previous submissions and notifications may have unknown outcomes", + object([("delayMs", (250u64 << (attempt - 1)).into())]), + )?; + } + let remaining = recovery_start.map_or(Duration::ZERO, |start: Instant| { + Duration::from_secs(30).saturating_sub(start.elapsed()) + }); + let delay = + tokio::time::sleep(Duration::from_millis(250u64 << (attempt - 1)).min(remaining)); + tokio::pin!(delay); + loop { + let line = std::pin::pin!(lines.recv()); + match select(&mut delay, line).await { + Either::Left(_) => break, + Either::Right((Some((_, line)), _)) => reject( + &mut output, + &serde_json::from_str::(&line?)?, + "not_sent", + &session, + )?, + Either::Right((None, _)) => return Ok(()), + } + } + } + let (mut channel, mut transport) = client(&remote)?.into_bounded_channel_and_future(); + let remaining = recovery_start.map_or(Duration::from_secs(30), |start: Instant| { + Duration::from_secs(30).saturating_sub(start.elapsed()) + }); + let deadline = tokio::time::sleep(Duration::from_secs(5).min(remaining)); + tokio::pin!(deadline); + let mut phase = if recovering { "initialize" } else { "live" }; + let mut expected = Value::Null; + let mut saw_state = false; + let mut saw_config = false; + let mut replay_bytes = 0usize; + let mut replay_events = 0usize; + if recovering { + let request = internal( + epoch, + "initialize", + initialize.clone().ok_or("Missing initialize template")?, + ); + expected = request["id"].clone(); + send(&mut channel, &request)?; + } + let mut lost = false; + let mut input_open = true; + let mut frames_open = true; + while !lost { + enum Event { + Frame(Option), + Driver(Result<(), agent_client_protocol::Error>), + Line(Option<(Instant, io::Result)>), + Timeout, + } + let event = { + // select polls its left branch first. Keep timeout > driver > + // inbound frame > stdin priority; disabled branches never poll + // their underlying receiver or deadline. + let timeout = std::pin::pin!(async { + if phase != "live" { + deadline.as_mut().await; + } else { + std::future::pending::<()>().await; + } + }); + let frame = std::pin::pin!(async { + if frames_open { + channel.rx.next().await + } else { + std::future::pending().await + } + }); + let line = std::pin::pin!(async { + if input_open { + lines.recv().await + } else { + std::future::pending().await + } + }); + match select(timeout, select(&mut transport, select(frame, line))).await { + Either::Left(_) => Event::Timeout, + Either::Right((Either::Left((result, _)), _)) => Event::Driver(result), + Either::Right((Either::Right((Either::Left((frame, _)), _)), _)) => { + Event::Frame(frame) + } + Either::Right((Either::Right((Either::Right((line, _)), _)), _)) => { + Event::Line(line) + } + } + }; + match event { + Event::Line(None) => { + if phase != "live" { + return Ok(()); + } + input_open = false; + channel.tx.close_channel(); + } + Event::Line(Some((admitted, line))) => { + let value: Value = serde_json::from_str(&line?)?; + if phase != "live" + || resumed.is_some() + || gate.is_some_and(|gate| admitted <= gate) + { + reject(&mut output, &value, "not_sent", &session)?; + continue; + } + if entries(&value).iter().any(|v| v["method"] == "initialize") + && (initialized || entries(&value).len() != 1) + { + reject(&mut output, &value, "not_sent", &session)?; + continue; + } + if entries(&value).iter().any(|v| reserved(&v["id"])) { + reject(&mut output, &value, "reserved_id", &session)?; + continue; + } + let requests = entries(&value) + .iter() + .filter(|v| v.get("id").is_some() && v.get("method").is_some()) + .count(); + let duplicate_batch = entries(&value).iter().enumerate().any(|(index, v)| { + v.get("id").is_some() + && v.get("method").is_some() + && entries(&value)[..index] + .iter() + .any(|p| p["id"] == v["id"] && p.get("method").is_some()) + }); + if duplicate_batch + || pending.len() + requests > MAX_PENDING + || entries(&value).iter().any(|v| { + v.get("method").is_some() + && v.get("id").is_some() + && pending.iter().any(|p| p["id"] == v["id"]) + }) + { + reject(&mut output, &value, "not_sent", &session)?; + continue; + } + for entry in entries(&value) { + if entry["method"] == "initialize" { + initialize = Some(entry["params"].clone()); + } + if entry.get("id").is_some() && entry.get("method").is_some() { + pending.push(object([ + ("id", entry["id"].clone()), + ("method", entry["method"].clone()), + ])); + } + } + let mut frame = TransportFrame::parse_json(&value.to_string()); + rewrite(&mut frame, &root, &mut session, &mut initial, &mut resumed)?; + if let Some((_, id)) = &resumed { + deadline + .as_mut() + .reset(tokio::time::Instant::now() + Duration::from_secs(5)); + phase = "initial_replay"; + notice( + &mut output, + id, + epoch, + "begin", + 0, + "Attaching durable session", + Value::Null, + )?; + if remote.no_replay { + let mut value: Value = serde_json::from_str(&frame.to_json()?)?; + let values = match &mut value { + Value::Array(values) => values.as_mut_slice(), + value => std::slice::from_mut(value), + }; + for value in values { + if value["method"] == "session/resume" + && let Some(params) = value["params"].as_object_mut() + { + params.remove("replayFrom"); + } + } + frame = TransportFrame::parse_json(&value.to_string()); + } + } + if entries(&value).iter().any(|v| v["method"] == "initialize") { + phase = "initial_initialize"; + deadline + .as_mut() + .reset(tokio::time::Instant::now() + Duration::from_secs(5)); + } + if channel.tx.try_send(frame).is_err() { + lost = true; + } + } + Event::Frame(Some(charged)) => { + let value: Value = serde_json::from_slice(charged.as_bytes())?; + for mut value in entries(&value).iter().cloned() { + if value.get("method").is_none() + && reserved(&value["id"]) + && value["id"] != expected + { + continue; + } + if value["method"] == "session/update" + && value["params"]["sessionId"].as_str() == session.as_deref() + { + if matches!( + value["params"]["update"]["sessionUpdate"].as_str(), + Some("state_update" | "config_option_update") + ) && serde_json::from_value::( + value["params"].clone(), + ) + .is_err() + { + terminal = Some( + "Gateway protocol: invalid session snapshot notification" + .into(), + ); + lost = true; + break; + } + saw_state |= + value["params"]["update"]["sessionUpdate"] == "state_update"; + saw_config |= value["params"]["update"]["sessionUpdate"] + == "config_option_update"; + } + if let Some(reason) = terminal_reason(&value) { + if value.get("method").is_none() + && let Some(index) = + pending.iter().position(|p| p["id"] == value["id"]) + { + emit( + &mut output, + &object([ + ("jsonrpc", "2.0".into()), + ("id", value["id"].clone()), + ( + "error", + object([ + ("code", (-32000).into()), + ("message", "Gateway rejected request".into()), + ( + "data", + object([ + ("reason", reason.into()), + ("terminal", true.into()), + ]), + ), + ]), + ), + ]), + )?; + pending.remove(index); + } + terminal = Some(format!( + "Gateway {reason}; submission outcome may be unknown. Inspect kit gateway list and restart explicitly; replay_unavailable requires explicit --remote-no-replay opt-in." + )); + lost = true; + break; + } + if value.get("id").is_some() + && value.get("method").is_none() + && reserved(&value["id"]) + { + if value["id"] != expected { + continue; + } + if value.get("error").is_some() { + if value["error"]["data"]["terminal"] != false { + terminal = Some("Gateway rejected recovery; inspect session and restart explicitly. Use --remote-no-replay only to opt out of history.".into()); + } + lost = true; + break; + } + if phase == "initialize" { + if value["result"]["protocolVersion"] != 2 { + terminal = Some("Gateway protocol mismatch".into()); + lost = true; + break; + } + if !initialized { + // Initial initialize is safe to retry; preserve its original local ID. + if let Some(index) = + pending.iter().position(|v| v["method"] == "initialize") + { + value["id"] = pending.remove(index)["id"].clone(); + emit(&mut output, &value)?; + } + initialized = true; + phase = "live"; + gate = Some(Instant::now()); + recovery_start = None; + attempt = 0; + } else { + let mut params = object([ + ( + "sessionId", + session.clone().map_or(Value::Null, Value::String), + ), + ( + "cwd", + root.to_str() + .ok_or("remote root must be valid UTF-8")? + .into(), + ), + ("mcpServers", Value::Array(Vec::new())), + ( + "_meta", + object([( + "kit/gateway", + object([ + ( + "previousAttachment", + attachment + .clone() + .map_or(Value::Null, Value::String), + ), + ("recoveryId", recovery_id.clone().into()), + ("recoveryAttempt", attempt.into()), + ]), + )]), + ), + ]); + if !remote.no_replay { + params["replayFrom"] = object([("type", "start".into())]); + } + let request = internal(epoch, "session/resume", params); + expected = request["id"].clone(); + send(&mut channel, &request)?; + phase = "replay"; + } + } else if phase == "replay" { + if value["result"] + .get("sessionId") + .is_some_and(|id| id.as_str() != session.as_deref()) + { + terminal = Some( + "Gateway protocol: resumed a different durable session" + .into(), + ); + lost = true; + break; + } + let token = value["result"]["_meta"]["kit/gateway"]["attachment"] + .as_str() + .filter(|s| !s.is_empty() && s.len() <= 128); + if let (Some(bits), Some(token)) = ( + snapshot(&value["result"], remote.no_replay) + .filter(|_| saw_state && saw_config), + token, + ) { + attachment = Some(token.to_owned()); + notice( + &mut output, + session.as_deref().ok_or("Missing durable target")?, + epoch, + "commit", + attempt, + "Reconnected", + bits, + )?; + phase = "live"; + gate = Some(Instant::now()); + recovery_start = None; + attempt = 0; + } else { + terminal = Some("Gateway recovery snapshot unavailable; session left unchanged".into()); + lost = true; + } + } + continue; + } + if (phase == "replay" || phase == "initial_replay") + && value.get("method").is_some() + { + if value["method"] != "session/update" + || value["params"]["sessionId"].as_str() != session.as_deref() + { + continue; + } + replay_events += 1; + replay_bytes = replay_bytes.saturating_add(value.to_string().len()); + if replay_events > 4096 || replay_bytes > 8 * 1024 * 1024 { + terminal = Some("Replay unavailable within local bounds; restart with explicit --remote-no-replay".into()); + lost = true; + break; + } + } + let mut completed = None; + let mut initial_commit = None; + if value.get("method").is_none() + && let Some(index) = pending.iter().position(|p| p["id"] == value["id"]) + { + let request = pending[index].clone(); + completed = Some(index); + if value.get("error").is_some() && resumed.is_some() { + emit(&mut output, &value)?; + pending.remove(index); + terminal = Some("Explicit session resume rejected".into()); + lost = true; + break; + } + if value.get("result").is_some() { + if request["method"] == "initialize" { + initialized = true; + phase = "live"; + } + if matches!( + request["method"].as_str(), + Some("session/new" | "session/resume" | "session/load") + ) { + if let Some(id) = value["result"]["sessionId"] + .as_str() + .filter(|id| !id.is_empty()) + { + if resumed.as_ref().is_some_and(|(_, target)| target != id) + { + terminal = Some("Gateway protocol: resumed a different durable session".into()); + lost = true; + break; + } + session = Some(id.to_owned()); + } else if request["method"] == "session/new" + && resumed.is_none() + { + terminal = Some("Gateway protocol: missing confirmed session ID; use kit gateway list and explicit --remote-session".into()); + lost = true; + break; + } + attachment = + value["result"]["_meta"]["kit/gateway"]["attachment"] + .as_str() + .filter(|s| !s.is_empty() && s.len() <= 128) + .map(str::to_owned); + if let Some((id, target)) = &resumed + && serde_json::to_value(id)? == value["id"] + { + value["result"]["sessionId"] = + Value::String(target.clone()); + initial_commit = + snapshot(&value["result"], remote.no_replay) + .filter(|_| saw_state && saw_config); + if initial_commit.is_none() { + terminal = Some( + "Gateway snapshot missing for initial resume" + .into(), + ); + lost = true; + break; + } + resumed = None; + phase = "live"; + gate = Some(Instant::now()); + } + } + } + } + if let Some(session) = &session { + annotate( + &mut value, + session, + epoch, + if phase == "replay" || resumed.is_some() { + "replay" + } else { + "live" + }, + ); + if let Some(bits) = initial_commit { + notice(&mut output, session, epoch, "commit", 0, "Attached", bits)?; + } + } + if session.is_none() + && let Some(id) = + value["params"]["sessionId"].as_str().map(str::to_owned) + { + annotate(&mut value, &id, epoch, "live"); + } + emit(&mut output, &value)?; + if let Some(index) = completed { + pending.remove(index); + } + } + // Keep SDK byte/frame admission charged through every stdout flush. + drop(charged); + } + Event::Frame(None) => { + frames_open = false; + if phase == "live" { + phase = "transport_closing"; + deadline + .as_mut() + .reset(tokio::time::Instant::now() + Duration::from_secs(5)); + } + } + Event::Timeout => lost = true, + Event::Driver(result) => { + if !input_open { + return result + .map_err(|_| "Gateway transport disconnected during shutdown".into()); + } + if let Err(error) = result + && let Some(status) = diagnostic_status(&error) + && matches!(status, 400 | 401 | 403 | 404 | 405 | 409 | 410 | 422 | 426) + { + terminal = Some(format!( + "Gateway HTTP {status}; authentication/protocol/session recovery rejected. Inspect session and restart explicitly." + )); + } + lost = true; + } + } + } + drop(transport); + drop(channel); + let keep_initialize = !initialized + && pending.iter().all(|p| p["method"] == "initialize") + && initialize.is_some(); + if !keep_initialize { + for request in pending.drain(..) { + reject(&mut output, &request, "outcome_unknown", &session)?; + } + } + if terminal.is_none() && session.is_none() && !keep_initialize { + return Err("Session creation outcome unknown; use kit gateway list and explicit --remote-session. No new session was retried.".into()); + } + if recovery_start.is_none() { + if terminal.is_none() && session.is_some() && attachment.is_none() && initialized { + terminal = Some("Gateway attachment metadata unavailable; automatic recovery disabled. Use explicit --remote-session selection.".into()); + } + let mut nonce = [0u8; 32]; + getrandom::fill(&mut nonce).map_err(|_| "Recovery entropy unavailable")?; + recovery_id = nonce.iter().map(|byte| format!("{byte:02x}")).collect(); + recovery_start = Some(Instant::now()); + } + } +} + +#[cfg(test)] +#[allow( + clippy::unwrap_used, + clippy::expect_used, + clippy::panic, + clippy::disallowed_methods, + clippy::disallowed_macros +)] +mod tests; diff --git a/src/gateway/http_client/recovery/tests.rs b/src/gateway/http_client/recovery/tests.rs new file mode 100644 index 0000000..c1da9ad --- /dev/null +++ b/src/gateway/http_client/recovery/tests.rs @@ -0,0 +1,465 @@ +use super::*; +use axum::{ + Json, Router, + http::{HeaderMap, StatusCode}, + response::{IntoResponse, Sse}, + routing::post, +}; +use serde_json::json; +use std::{ + convert::Infallible, + io::{BufRead, Write}, +}; +use tokio::sync::{broadcast, mpsc}; + +#[test] +fn opaque_http_status_parser_is_anchored_and_never_matches_body_text() { + for prefix in [ + "bounded HTTP: initialize HTTP ", + "bounded HTTP: POST HTTP ", + "bounded HTTP: SSE HTTP ", + ] { + let error = agent_client_protocol::Error::internal_error() + .data(format!("{prefix}401 Unauthorized")); + assert_eq!(diagnostic_status(&error), Some(401)); + } + for text in [ + "HTTP 401 Unauthorized", + "initialize: HTTP 401 Unauthorized: secret", + "bounded HTTP: initialize HTTP 503 Service Unavailable: HTTP 401 Unauthorized", + "bounded HTTP: POST HTTP 401 Unauthorizedevil", + "bounded HTTP: POST HTTP 401 Unauthorized: secret", + "bounded HTTP: POST HTTP 0401 Unauthorized", + "body: bounded HTTP: initialize HTTP 401 Unauthorized", + ] { + let error = agent_client_protocol::Error::internal_error().data(text); + assert_eq!(diagnostic_status(&error), None); + } +} + +#[test] +fn metadata_validation_requires_both_snapshot_bits_and_explicit_history_policy() { + let full = json!({"_meta":{"kit/gateway":{"stateSnapshot":true,"configSnapshot":true,"historyAvailable":true}}}); + assert!(snapshot(&full, false).is_some()); + assert!(snapshot(&full, true).is_none()); + let mut no_replay = full.clone(); + no_replay["_meta"]["kit/gateway"]["historyAvailable"] = false.into(); + assert!(snapshot(&no_replay, true).is_some()); + for key in ["stateSnapshot", "configSnapshot", "historyAvailable"] { + let mut missing = full.clone(); + missing["_meta"]["kit/gateway"] + .as_object_mut() + .unwrap() + .remove(key); + assert!(snapshot(&missing, false).is_none()); + } +} + +#[test] +fn terminal_classification_uses_structured_metadata_only() { + assert_eq!( + terminal_reason( + &json!({"error":{"message":"controller_replaced","data":{"terminal":false,"reason":"unavailable"}}}) + ), + None + ); + assert_eq!( + terminal_reason(&json!({"error":{"data":{"terminal":true,"reason":"root_denied"}}})), + Some("root_denied") + ); + assert_eq!( + terminal_reason( + &json!({"method":"session/update","params":{"update":{"sessionUpdate":"_gateway_controller","_meta":{"kit/gateway":{"terminal":true,"reason":"controller_replaced"}}}}}) + ), + Some("controller_replaced") + ); +} + +#[test] +fn batch_rejection_is_once_per_request_and_reserves_internal_ids() { + let mut output = Vec::new(); + reject(&mut output,&json!([{"id":1,"method":"session/prompt"},{"id":"two","method":"session/set_config_option"},{"method":"session/cancel"}]),"outcome_unknown",&Some("durable".into())).unwrap(); + let output: Vec = std::str::from_utf8(&output) + .unwrap() + .lines() + .map(|l| serde_json::from_str(l).unwrap()) + .collect(); + assert_eq!(output.len(), 2); + assert_eq!(output[0]["error"]["data"]["reason"], "outcome_unknown"); + assert_eq!(output[1]["error"]["data"]["sessionId"], "durable"); + assert!(reserved(&json!("kit.gateway.internal/2/initialize"))); + assert!(!reserved(&json!(9223372036854775807i64))); +} + +struct Server { + remote: super::super::super::Remote, + _credential: tempfile::NamedTempFile, + task: tokio::task::JoinHandle<()>, + requests: mpsc::UnboundedReceiver, + events: broadcast::Sender<(String, Option)>, +} +impl Drop for Server { + fn drop(&mut self) { + self.task.abort(); + } +} +impl Server { + async fn new(initial_status: Option) -> Self { + let (incoming, requests) = mpsc::unbounded_channel(); + let (events, _) = broadcast::channel::<(String, Option)>(128); + let get_events = events.clone(); + let app = Router::new().route("/acp/v2",post(move |Json(message):Json| { + let incoming = incoming.clone(); + async move { + incoming.send(message.clone()).unwrap(); + if message["method"] == "initialize" { + if let Some(status) = initial_status.filter(|_| message["id"].is_number()) { return (status,"secret-response-body").into_response(); } + return ([("acp-connection-id","connection")],Json(json!({"jsonrpc":"2.0","id":message["id"],"result":{"protocolVersion":2,"info":{"name":"fault-server","version":"1"},"capabilities":{}}}))).into_response(); + } + StatusCode::ACCEPTED.into_response() + } + }).get(move |headers:HeaderMap| { + let receiver = get_events.subscribe(); + let scope = headers.get("acp-session-id").and_then(|v| v.to_str().ok()).unwrap_or("").to_owned(); + async move { + Sse::new(futures_util::stream::unfold((receiver,scope), |(mut receiver,scope)| async move { + loop { + let (target,value) = receiver.recv().await.ok()?; + if target != scope && target != "*" { continue; } + let value = value?; + return Some((Ok::<_,Infallible>(axum::response::sse::Event::default().data(value.to_string())),(receiver,scope))); + } + })) + } + }).delete(|| async { StatusCode::ACCEPTED })); + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let address = listener.local_addr().unwrap(); + let task = tokio::spawn(async move { + axum::serve(listener, app).await.unwrap(); + }); + let mut credential = tempfile::NamedTempFile::new().unwrap(); + writeln!(credential, "test-token").unwrap(); + Self { + remote: super::super::super::Remote { + url: format!("http://{address}"), + credential_file: credential.path().to_owned(), + session: None, + no_replay: false, + }, + _credential: credential, + task, + requests, + events, + } + } + fn update(&self, update: Value) { + self.events.send(("durable".into(),Some(json!({"jsonrpc":"2.0","method":"session/update","params":{"sessionId":"durable","update":update}})))).unwrap(); + } + fn result(&self, request: &Value, result: Value, scope: &str) { + self.events + .send(( + scope.into(), + Some(json!({"jsonrpc":"2.0","id":request["id"],"result":result})), + )) + .unwrap(); + } +} + +// Real OS output stream, not a callback into the state machine. Assertions read +// the same serialized FIFO consumed by the local ACP client in production. +fn output_pipe() -> ( + std::os::unix::net::UnixStream, + mpsc::UnboundedReceiver, +) { + let (writer, reader) = std::os::unix::net::UnixStream::pair().unwrap(); + let (send, receive) = mpsc::unbounded_channel(); + std::thread::spawn(move || { + for line in io::BufReader::new(reader).lines() { + let Ok(line) = line else { + break; + }; + if send.send(serde_json::from_str(&line).unwrap()).is_err() { + break; + } + } + }); + (writer, receive) +} +fn frame(id: u64, method: &str, params: Value) -> (Instant, io::Result) { + ( + Instant::now(), + Ok(json!({"jsonrpc":"2.0","id":id,"method":method,"params":params}).to_string()), + ) +} +async fn until_id(output: &mut mpsc::UnboundedReceiver, id: Value) -> Value { + loop { + let value = output.recv().await.unwrap(); + if value.get("id") == Some(&id) { + return value; + } + } +} +async fn until_kind(output: &mut mpsc::UnboundedReceiver, kind: &str) -> Value { + loop { + let value = output.recv().await.unwrap(); + if value["params"]["_meta"][META]["kind"] == kind { + return value; + } + } +} + +#[tokio::test] +async fn real_http_401_is_terminal_and_does_not_leak_body() { + let mut server = Server::new(Some(StatusCode::UNAUTHORIZED)).await; + let (send, lines) = mpsc::channel(16); + send.send(frame(1, "initialize", initialize_params().unwrap())) + .await + .unwrap(); + let mut output = Vec::new(); + let result = tokio::time::timeout( + Duration::from_secs(3), + run(server.remote.clone(), "/remote".into(), lines, &mut output), + ) + .await + .unwrap(); + assert!(result.unwrap_err().to_string().contains("HTTP 401")); + assert!( + !String::from_utf8(output) + .unwrap() + .contains("secret-response-body") + ); + assert_eq!( + server.requests.recv().await.unwrap()["method"], + "initialize" + ); + assert!(server.requests.try_recv().is_err()); +} + +#[tokio::test] +async fn lost_create_response_fails_closed_without_creating_twice() { + let mut server = Server::new(None).await; + let remote = server.remote.clone(); + let (send, lines) = mpsc::channel(16); + let (writer, mut output) = output_pipe(); + let scenario = async { + send.send(frame(1, "initialize", initialize_params().unwrap())) + .await + .unwrap(); + until_id(&mut output, json!(1)).await; + assert_eq!( + server.requests.recv().await.unwrap()["method"], + "initialize" + ); + send.send(frame( + 2, + "session/new", + json!({"cwd":"/local","mcpServers":[]}), + )) + .await + .unwrap(); + let new = server.requests.recv().await.unwrap(); + assert_eq!(new["method"], "session/new"); + server.events.send(("*".into(), None)).unwrap(); + let error = until_id(&mut output, json!(2)).await; + assert_eq!(error["error"]["data"]["reason"], "outcome_unknown"); + }; + let (result, ()) = tokio::time::timeout(Duration::from_secs(5), async { + tokio::join!(run(remote, "/remote".into(), lines, writer), scenario) + }) + .await + .unwrap(); + assert!(result.unwrap_err().to_string().contains("kit gateway list")); + assert!(server.requests.try_recv().is_err()); +} + +#[tokio::test] +async fn lost_prompt_reinitializes_and_replays_same_session_before_commit() { + let mut server = Server::new(None).await; + let remote = server.remote.clone(); + let (send, lines) = mpsc::channel(16); + let (writer, mut output) = output_pipe(); + let scenario = async { + send.send(frame(1, "initialize", initialize_params().unwrap())) + .await + .unwrap(); + until_id(&mut output, json!(1)).await; + server.requests.recv().await.unwrap(); + send.send(frame( + 2, + "session/new", + json!({"cwd":"/local","mcpServers":[]}), + )) + .await + .unwrap(); + let new = server.requests.recv().await.unwrap(); + server.result(&new,json!({"sessionId":"durable","configOptions":[],"_meta":{"kit/gateway":{"attachment":"owner-1"}}}),""); + until_id(&mut output, json!(2)).await; + send.send(frame( + 3, + "session/prompt", + json!({"sessionId":"durable","prompt":[]}), + )) + .await + .unwrap(); + assert_eq!( + server.requests.recv().await.unwrap()["method"], + "session/prompt" + ); + server.events.send(("*".into(), None)).unwrap(); + assert_eq!( + until_id(&mut output, json!(3)).await["error"]["data"]["reason"], + "outcome_unknown" + ); + let begin = until_kind(&mut output, "begin").await; + send.send(frame( + 4, + "session/prompt", + json!({"sessionId":"durable","prompt":[]}), + )) + .await + .unwrap(); + assert_eq!( + until_id(&mut output, json!(4)).await["error"]["data"]["reason"], + "not_sent" + ); + let init = server.requests.recv().await.unwrap(); + assert_eq!(init["method"], "initialize"); + let resume = server.requests.recv().await.unwrap(); + assert_eq!(resume["method"], "session/resume"); + assert_eq!(resume["params"]["sessionId"], "durable"); + assert_eq!(resume["params"]["cwd"], "/remote"); + assert_eq!(resume["params"]["replayFrom"], json!({"type":"start"})); + let fence = &resume["params"]["_meta"]["kit/gateway"]; + assert_eq!(fence["previousAttachment"], "owner-1"); + assert_eq!(fence["recoveryAttempt"], 1); + assert_eq!(fence["recoveryId"].as_str().unwrap().len(), 64); + let recovery_id = fence["recoveryId"].clone(); + // The first resume committed remotely, but its response is lost. The + // next attempt must retain sequence authority and fence old work. + server.update(json!({"sessionUpdate":"agent_message_chunk","content":{"type":"text","text":"candidate one"}})); + until_kind(&mut output, "replay").await; + server.events.send(("*".into(), None)).unwrap(); + let second_begin = until_kind(&mut output, "begin").await; + assert!( + second_begin["params"]["_meta"][META]["epoch"] + .as_u64() + .unwrap() + > begin["params"]["_meta"][META]["epoch"].as_u64().unwrap() + ); + let begin = second_begin; + assert_eq!( + server.requests.recv().await.unwrap()["method"], + "initialize" + ); + let resume = server.requests.recv().await.unwrap(); + assert_eq!(resume["method"], "session/resume"); + assert_eq!( + resume["params"]["_meta"]["kit/gateway"]["previousAttachment"], + "owner-1" + ); + assert_eq!( + resume["params"]["_meta"]["kit/gateway"]["recoveryId"], + recovery_id + ); + assert_eq!( + resume["params"]["_meta"]["kit/gateway"]["recoveryAttempt"], + 2 + ); + server.update(json!({"sessionUpdate":"config_option_update","configOptions":[]})); + server.update(json!({"sessionUpdate":"state_update","state":"idle"})); + server.result(&resume,json!({"_meta":{"kit/gateway":{"attachment":"owner-2","historyAvailable":true,"stateSnapshot":true,"configSnapshot":true}}}),"durable"); + let config = until_kind(&mut output, "replay").await; + assert_eq!( + config["params"]["update"]["sessionUpdate"], + "config_option_update" + ); + let state = until_kind(&mut output, "replay").await; + assert_eq!(state["params"]["update"]["sessionUpdate"], "state_update"); + let commit = until_kind(&mut output, "commit").await; + assert_eq!( + commit["params"]["_meta"][META]["epoch"], + begin["params"]["_meta"][META]["epoch"] + ); + assert_eq!(commit["params"]["_meta"][META]["historyAvailable"], true); + drop(send); + }; + let (result, ()) = tokio::time::timeout(Duration::from_secs(8), async { + tokio::join!(run(remote, "/remote".into(), lines, writer), scenario) + }) + .await + .unwrap(); + result.unwrap(); + assert!(server.requests.try_recv().is_err()); +} + +#[tokio::test] +async fn initial_initialize_loss_retries_safely_and_preserves_local_id() { + let mut server = Server::new(Some(StatusCode::SERVICE_UNAVAILABLE)).await; + let mut remote = server.remote.clone(); + remote.session = Some("durable".into()); + let (send, lines) = mpsc::channel(16); + let (writer, mut output) = output_pipe(); + let scenario = async { + send.send(frame(7, "initialize", initialize_params().unwrap())) + .await + .unwrap(); + let result = until_id(&mut output, json!(7)).await; + assert_eq!(result["result"]["protocolVersion"], 2); + assert_eq!(server.requests.recv().await.unwrap()["id"], 7); + let retry = server.requests.recv().await.unwrap(); + assert_eq!(retry["method"], "initialize"); + assert!(reserved(&retry["id"])); + assert_eq!(retry["params"], initialize_params().unwrap()); + drop(send); + }; + let (result, ()) = tokio::time::timeout(Duration::from_secs(5), async { + tokio::join!(run(remote, "/remote".into(), lines, writer), scenario) + }) + .await + .unwrap(); + result.unwrap(); + assert!(server.requests.try_recv().is_err()); +} + +#[tokio::test] +async fn eof_during_backoff_cancels_recovery_without_another_http_attempt() { + let mut server = Server::new(None).await; + let remote = server.remote.clone(); + let (send, lines) = mpsc::channel(16); + let (writer, mut output) = output_pipe(); + let scenario = async { + send.send(frame(1, "initialize", initialize_params().unwrap())) + .await + .unwrap(); + until_id(&mut output, json!(1)).await; + server.requests.recv().await.unwrap(); + send.send(frame( + 2, + "session/new", + json!({"cwd":"/local","mcpServers":[]}), + )) + .await + .unwrap(); + let new = server.requests.recv().await.unwrap(); + server.result(&new,json!({"sessionId":"durable","configOptions":[],"_meta":{"kit/gateway":{"attachment":"owner-1"}}}),""); + until_id(&mut output, json!(2)).await; + send.send(frame( + 3, + "session/prompt", + json!({"sessionId":"durable","prompt":[]}), + )) + .await + .unwrap(); + server.requests.recv().await.unwrap(); + server.events.send(("*".into(), None)).unwrap(); + until_kind(&mut output, "begin").await; + drop(send); + }; + let (result, ()) = tokio::time::timeout(Duration::from_secs(5), async { + tokio::join!(run(remote, "/remote".into(), lines, writer), scenario) + }) + .await + .unwrap(); + result.unwrap(); + assert!(server.requests.try_recv().is_err()); +} diff --git a/src/gateway/http_server.rs b/src/gateway/http_server.rs index bdb66b9..b9f3501 100644 --- a/src/gateway/http_server.rs +++ b/src/gateway/http_server.rs @@ -162,7 +162,7 @@ impl Connection { ("coreBufferedBytesPerDirection", 16_777_216.into()), ("httpEgressBytesPerConnection", 4_194_304.into()), ("liveReplayLimitBytes", MAX_REPLAY.into()), - ("liveReplayLimitEvents", (MAX_QUEUE - 2).into()), + ("liveReplayLimitEvents", (MAX_QUEUE - 3).into()), ]), )]), ), @@ -271,17 +271,6 @@ impl Connection { .as_str() .ok_or_else(|| Failure::unavailable("missing session id"))? .to_owned(); - let attached = self - .operation(Request::Attach { - session: session.clone(), - replace: resume, - replay: !resume || message["params"].get("replayFrom").is_some(), - }) - .await?; - let attachment = attached["attachment"] - .as_str() - .ok_or_else(|| Failure::unavailable("missing controller id"))? - .to_owned(); let entry = self .gateway .sessions @@ -290,6 +279,28 @@ impl Connection { .get(&session) .cloned() .ok_or_else(|| Failure::unavailable("resident child exited"))?; + // Pin the same resident actor for claim and subsequent control. + let (reply, accepted) = oneshot::channel(); + entry + .sender + .send(Envelope { + request: Request::Attach { + session: session.clone(), + replace: resume, + replay: !resume || message["params"].get("replayFrom").is_some(), + startup: Some(message.clone()), + }, + reply, + }) + .await + .map_err(|_| Failure::unavailable("resident child exited"))?; + let attached = accepted + .await + .map_err(|_| Failure::unavailable("resident child exited"))??; + let attachment = attached["attachment"] + .as_str() + .ok_or_else(|| Failure::unavailable("missing controller id"))? + .to_owned(); let mut control = Control { session: session.clone(), attachment, @@ -297,6 +308,13 @@ impl Connection { cursor: 0, pending: HashMap::new(), }; + if attached["started"] == true { + if let Some(id) = message.get("id") { + control.pending.insert(id.to_string(), id.clone()); + } + self.controls.insert(session, control); + return Ok(None); + } let (mut initialize, initialize_charge) = self .initialize .clone() @@ -356,6 +374,26 @@ impl Connection { // Settle that controller's pending calls explicitly before forgetting it. for (id, error) in failed { if let Some(mut control) = self.controls.remove(&id) { + send( + channel, + object([ + ("jsonrpc", "2.0".into()), + ("method", "session/update".into()), + ( + "params", + object([ + ("sessionId", id.clone().into()), + ( + "update", + object([ + ("sessionUpdate", "_gateway_controller".into()), + ("_meta", object([("kit/gateway", error.data())])), + ]), + ), + ]), + ), + ]), + )?; for (_, request_id) in control.pending.drain() { send( channel, @@ -367,6 +405,7 @@ impl Connection { object([ ("code", (-32000).into()), ("message", error.1.clone().into()), + ("data", error.data()), ]), ), ]), @@ -432,6 +471,7 @@ impl Connection { "error", object([ ("code", (-32000).into()), + ("data", error.data()), ("message", error.1.into()), ]), ), @@ -572,6 +612,65 @@ mod tests { send(&channel, message).unwrap(); } + #[tokio::test] + async fn revoked_idle_controller_emits_terminal_notice_without_pending_calls() { + let directory = tempfile::tempdir().unwrap(); + let token = directory.path().join("token"); + std::fs::write(&token, "test-token").unwrap(); + let gateway = Arc::new(Gateway { + stopping: AtomicBool::new(false), + token: BearerToken::load(&token).unwrap(), + roots: Vec::new(), + sessions: Mutex::new(HashMap::new()), + }); + let (sender, mut receiver) = mpsc::channel::(2); + let (_stopped, stopped) = watch::channel(()); + let mut connection = Connection::new(gateway); + connection.controls.insert( + "session".into(), + Control { + session: "session".into(), + attachment: "old".into(), + entry: Entry { + root: PathBuf::new(), + sender, + stopped, + }, + cursor: 0, + pending: HashMap::new(), + }, + ); + let actor = async { + let envelope = receiver.recv().await.unwrap(); + assert!(matches!(envelope.request, Request::Poll { .. })); + envelope + .reply + .send(Err(Failure( + StatusCode::CONFLICT, + "replaced".into(), + "controller_replaced", + ))) + .unwrap(); + }; + let (channel, mut peer) = BoundedChannel::duplex(ChannelLimits::default()).unwrap(); + let (result, ()) = tokio::join!(connection.flush(&channel), actor); + result.unwrap(); + assert!(connection.controls.is_empty()); + let frame = peer.rx.next().await.unwrap(); + let TransportFrame::Single(message) = frame.decode() else { + panic!("expected notification") + }; + let message = serde_json::to_value(message).unwrap(); + assert_eq!( + message["params"]["update"]["_meta"]["kit/gateway"], + json!({"reason":"controller_replaced","terminal":true}) + ); + serde_json::from_value::( + message["params"].clone(), + ) + .unwrap(); + } + #[tokio::test] async fn pending_request_capacity_preserves_response_lane() { let (sender, mut receiver) = mpsc::channel::(1); diff --git a/src/gateway/tests.rs b/src/gateway/tests.rs index 4fc6997..a61f6ba 100644 --- a/src/gateway/tests.rs +++ b/src/gateway/tests.rs @@ -57,7 +57,7 @@ impl Harness { .await { Ok(Json(value)) => (StatusCode::OK, value), - Err(Failure(status, message)) => (status, json!({"error":message})), + Err(Failure(status, message, _)) => (status, json!({"error":message})), } } @@ -111,6 +111,7 @@ for line in sys.stdin: pending: HashMap::new(), initialized: None, session_result: None, + state: json!({"sessionUpdate":"state_update","state":"idle"}), journal: VecDeque::new(), journal_bytes: 0, replay_complete: true, @@ -401,6 +402,7 @@ fn isolated_actor() -> Actor { pending: HashMap::new(), initialized: None, session_result: None, + state: json!({"sessionUpdate":"state_update","state":"idle"}), journal: VecDeque::new(), journal_bytes: 0, replay_complete: true, @@ -429,6 +431,7 @@ async fn accepted_child_write_retains_charge_after_dequeue_until_consumed() { let attached = actor .command( Request::Attach { + startup: None, session: actor.id.clone(), replace: false, replay: true, @@ -468,6 +471,7 @@ fn failed_child_enqueue_does_not_publish_pending_request() { let attached = actor .command( Request::Attach { + startup: None, session: actor.id.clone(), replace: false, replay: true, @@ -504,6 +508,7 @@ fn failed_replay_claim_preserves_owner_and_stale_release_preserves_replacement() let mut actor = isolated_actor(); let (writes, _receiver) = mpsc::channel(1); let claim = |replace| Request::Attach { + startup: None, session: "resident".into(), replace, replay: true, @@ -584,6 +589,7 @@ fn transcript_replay_overflow_returns_error_not_truncated_success() { let attachment = actor .command( Request::Attach { + startup: None, session: actor.id.clone(), replace: false, replay: true, @@ -631,6 +637,7 @@ fn no_replay_restore_and_claim_survive_incomplete_internal_cache() { let attachment = actor .command( Request::Attach { + startup: None, session: actor.id.clone(), replace: true, replay: false, @@ -653,14 +660,19 @@ fn no_replay_restore_and_claim_survive_incomplete_internal_cache() { actor.output(json!({"jsonrpc":"2.0","id":1,"result":{"sessionId":"resident"}})); let owner = actor.attachment.as_ref().unwrap(); assert!(owner.live); - assert_eq!(owner.events.len(), 1); - assert_eq!(owner.events[0].1["id"], 42); - assert!(owner.events[0].1.get("error").is_none()); + assert_eq!(owner.events.len(), 3); + assert_eq!(owner.events[2].1["id"], 42); + assert!(owner.events[2].1.get("error").is_none()); + assert_eq!( + owner.events[2].1["result"]["_meta"]["kit/gateway"]["historyAvailable"], + false + ); // A later full-replay claimant still cannot evict this healthy controller. assert!( actor .command( Request::Attach { + startup: None, session: actor.id.clone(), replace: true, replay: true, @@ -1007,3 +1019,437 @@ for line in sys.stdin: } } } + +fn resident_claim(actor: &Actor, replay: bool, id: u64) -> Request { + Request::Attach { + session: actor.id.clone(), + replace: true, + replay, + startup: Some( + json!({"jsonrpc":"2.0","id":id,"method":"session/resume","params":{"sessionId":actor.id}}), + ), + } +} + +fn state_update(state: &str) -> Value { + json!({"jsonrpc":"2.0","method":"session/update","params":{"sessionId":"resident","update":{"sessionUpdate":"state_update","state":state}}}) +} + +#[test] +fn resident_snapshot_follows_replay_and_precedes_success() { + let mut actor = isolated_actor(); + let (writes, _receiver) = mpsc::channel(1); + actor.session_result = Some(json!({"configOptions":[]})); + actor.output(state_update("running")); + actor.output(json!({"jsonrpc":"2.0","method":"session/update","params":{"sessionId":"resident","update":{"sessionUpdate":"agent_message","id":"done","content":[]}}})); + // Assistant output is not evidence that foreground work stopped. + actor + .command(resident_claim(&actor, true, 50), &writes) + .unwrap(); + let events: Vec<_> = actor + .attachment + .as_ref() + .unwrap() + .events + .iter() + .map(|(_, m)| m) + .collect(); + assert_eq!(events.len(), 5); + assert_eq!(events[1]["params"]["update"]["id"], "done"); + assert_eq!( + events[2]["params"]["update"]["sessionUpdate"], + "config_option_update" + ); + assert_eq!(events[3]["params"]["update"]["state"], "running"); + assert_eq!(events[4]["id"], 50); + assert_eq!( + events[4]["result"]["_meta"]["kit/gateway"], + json!({"historyAvailable":true,"stateSnapshot":true,"configSnapshot":true,"attachment":actor.attachment.as_ref().unwrap().id}) + ); + // Validate snapshots using the actual ACP v2 SDK, not an invented shape. + for event in &events[2..4] { + serde_json::from_value::( + event["params"].clone(), + ) + .unwrap(); + } +} + +#[test] +fn no_replay_snapshot_retains_running_state_and_current_config_after_eviction() { + let mut actor = isolated_actor(); + let (writes, _receiver) = mpsc::channel(1); + actor.session_result = Some(json!({"configOptions":[]})); + actor.output(state_update("running")); + let options = json!([{"id":"model","name":"Model","type":"select","currentValue":"new","options":[{"value":"new","name":"New"}]}]); + actor.output(json!({"jsonrpc":"2.0","method":"session/update","params":{"sessionId":"resident","update":{"sessionUpdate":"config_option_update","configOptions":options}}})); + actor.remember(json!({"payload":"x".repeat(MAX_REPLAY+1)})); + actor + .command(resident_claim(&actor, false, 51), &writes) + .unwrap(); + let owner = actor.attachment.as_ref().unwrap(); + assert_eq!(owner.events.len(), 3); + assert_eq!( + owner.events[0].1["params"]["update"]["configOptions"], + options + ); + assert_eq!(owner.events[1].1["params"]["update"]["state"], "running"); + assert_eq!( + owner.events[2].1["result"]["_meta"]["kit/gateway"]["historyAvailable"], + false + ); + actor.output(state_update("idle")); + assert_eq!( + actor.attachment.as_ref().unwrap().events.back().unwrap().1["params"]["update"]["state"], + "idle" + ); +} + +#[test] +fn failed_resident_snapshot_preserves_owner_and_success_fences_old_generation() { + let mut actor = isolated_actor(); + let (writes, _receiver) = mpsc::channel(1); + actor.session_result = Some(json!({"configOptions":[]})); + actor + .command(resident_claim(&actor, false, 60), &writes) + .unwrap(); + let old = actor.attachment.as_ref().unwrap().id.clone(); + // Journal itself is below aggregate limits, but cannot cross HTTP as one frame. + actor.remember(json!({"payload":"x".repeat(MAX_HTTP_FRAME)})); + assert!(actor.replay_complete); + let error = actor + .command(resident_claim(&actor, true, 61), &writes) + .unwrap_err(); + assert_eq!(error.2, "replay_unavailable"); + assert_eq!(actor.attachment.as_ref().unwrap().id, old); + assert_eq!( + actor.attachment.as_ref().unwrap().events.back().unwrap().1["id"], + 60 + ); + actor.pending.insert( + 9, + Pending { + attachment: old.clone(), + original: json!(99), + method: "session/prompt".into(), + replay: false, + }, + ); + actor + .command(resident_claim(&actor, false, 62), &writes) + .unwrap(); + actor.output(json!({"jsonrpc":"2.0","id":9,"result":{"stopReason":"end_turn"}})); + let current = actor.attachment.as_ref().unwrap().id.clone(); + assert_ne!(old, current); + assert_eq!(actor.attachment.as_ref().unwrap().events.len(), 3); + let error = actor + .command( + Request::Detach { + session: actor.id.clone(), + attachment: old, + }, + &writes, + ) + .unwrap_err(); + assert_eq!(error.2, "controller_replaced"); + assert_eq!(actor.attachment.as_ref().unwrap().id, current); +} + +#[test] +fn automatic_resume_is_conditioned_on_previous_controller_generation() { + let mut actor = isolated_actor(); + let (writes, _receiver) = mpsc::channel(1); + actor.session_result = Some(json!({"configOptions":[]})); + actor + .command(resident_claim(&actor, false, 70), &writes) + .unwrap(); + let old = actor.attachment.as_ref().unwrap().id.clone(); + let mut recovery = resident_claim(&actor, false, 72); + if let Request::Attach { + startup: Some(message), + .. + } = &mut recovery + { + message["params"]["_meta"]["kit/gateway"]["previousAttachment"] = old.clone().into(); + } + // A separate explicit manual claim has replaced the transport being recovered. + actor + .command(resident_claim(&actor, false, 71), &writes) + .unwrap(); + let current = actor.attachment.as_ref().unwrap().id.clone(); + let error = actor.command(recovery, &writes).unwrap_err(); + assert_eq!(error.2, "controller_replaced"); + assert_eq!(actor.attachment.as_ref().unwrap().id, current); + assert_eq!( + actor.attachment.as_ref().unwrap().events.back().unwrap().1["id"], + 71 + ); + let mut recovery = resident_claim(&actor, false, 73); + if let Request::Attach { + startup: Some(message), + .. + } = &mut recovery + { + message["params"]["_meta"]["kit/gateway"]["previousAttachment"] = current.clone().into(); + } + actor.command(recovery, &writes).unwrap(); + assert_ne!(actor.attachment.as_ref().unwrap().id, current); + assert_eq!( + actor.attachment.as_ref().unwrap().events.back().unwrap().1["result"]["_meta"]["kit/gateway"] + ["attachment"], + actor.attachment.as_ref().unwrap().id + ); + let newest = actor.attachment.as_ref().unwrap().id.clone(); + actor + .command( + Request::Detach { + session: actor.id.clone(), + attachment: newest.clone(), + }, + &writes, + ) + .unwrap(); + let mut recovery = resident_claim(&actor, false, 74); + if let Request::Attach { + startup: Some(message), + .. + } = &mut recovery + { + message["params"]["_meta"]["kit/gateway"]["previousAttachment"] = newest.into(); + } + actor.command(recovery, &writes).unwrap(); +} + +#[test] +fn recovery_sequence_retries_own_unconfirmed_claim_but_not_manual_replacement() { + let mut actor = isolated_actor(); + let (writes, _receiver) = mpsc::channel(1); + actor.session_result = Some(json!({"configOptions":[]})); + actor + .command(resident_claim(&actor, false, 80), &writes) + .unwrap(); + let old = actor.attachment.as_ref().unwrap().id.clone(); + let recovery = |actor: &Actor, id| { + let mut request = resident_claim(actor, false, id); + if let Request::Attach { + startup: Some(message), + .. + } = &mut request + { + message["params"]["_meta"]["kit/gateway"] = json!({"previousAttachment":old,"recoveryId":"random-per-outage","recoveryAttempt":id}); + } + request + }; + actor.command(recovery(&actor, 81), &writes).unwrap(); + let unconfirmed = actor.attachment.as_ref().unwrap().id.clone(); + // The previous result was lost; reuse the same outage ID and old confirmed token. + actor.command(recovery(&actor, 82), &writes).unwrap(); + assert_ne!(actor.attachment.as_ref().unwrap().id, unconfirmed); + actor + .command(resident_claim(&actor, false, 83), &writes) + .unwrap(); + let manual = actor.attachment.as_ref().unwrap().id.clone(); + assert_eq!( + actor.command(recovery(&actor, 84), &writes).unwrap_err().2, + "controller_replaced" + ); + assert_eq!(actor.attachment.as_ref().unwrap().id, manual); + // Invalid recovery capabilities cannot mutate ownership either. + for metadata in [ + json!({"recoveryId":"orphan"}), + json!({"previousAttachment":manual,"recoveryId":""}), + json!({"previousAttachment":manual,"recoveryId":"x".repeat(129)}), + ] { + let mut request = resident_claim(&actor, false, 85); + if let Request::Attach { + startup: Some(message), + .. + } = &mut request + { + message["params"]["_meta"]["kit/gateway"] = metadata; + } + assert_eq!(actor.command(request, &writes).unwrap_err().2, "protocol"); + assert_eq!(actor.attachment.as_ref().unwrap().id, manual); + } +} + +#[test] +fn stale_config_response_refreshes_snapshot_without_cross_generation_response() { + let mut actor = isolated_actor(); + let (writes, _receiver) = mpsc::channel(1); + actor.session_result = Some(json!({"configOptions":[]})); + actor + .command(resident_claim(&actor, false, 90), &writes) + .unwrap(); + let old = actor.attachment.as_ref().unwrap().id.clone(); + actor.pending.insert( + 1, + Pending { + attachment: old, + original: json!(900), + method: "session/set_config_option".into(), + replay: false, + }, + ); + actor + .command(resident_claim(&actor, false, 91), &writes) + .unwrap(); + let options = json!([{"id":"model","name":"Model","type":"select","currentValue":"updated","options":[{"value":"updated","name":"Updated"}]}]); + actor.output(json!({"jsonrpc":"2.0","id":1,"result":{"configOptions":options}})); + assert_eq!(actor.attachment.as_ref().unwrap().events.len(), 3); + actor + .command(resident_claim(&actor, false, 92), &writes) + .unwrap(); + assert_eq!( + actor.attachment.as_ref().unwrap().events[0].1["params"]["update"]["configOptions"], + options + ); +} + +#[test] +fn aggregate_snapshot_overflow_preserves_prior_controller() { + let mut actor = isolated_actor(); + let (writes, _receiver) = mpsc::channel(1); + actor.session_result = Some(json!({"configOptions":[]})); + actor + .command(resident_claim(&actor, false, 93), &writes) + .unwrap(); + let old = actor.attachment.as_ref().unwrap().id.clone(); + // Every frame fits, and the journal alone fits, but trailing snapshots do not. + let frame = json!({"payload":"x".repeat(MAX_HTTP_FRAME / 2 - 100)}); + for _ in 0..16 { + actor.remember(frame.clone()); + } + let remaining = MAX_REPLAY - actor.journal_bytes; + actor.remember(json!({"payload":"x".repeat(remaining - 14)})); + assert!(actor.replay_complete); + assert!(actor.journal_bytes <= MAX_REPLAY); + assert_eq!( + actor + .command(resident_claim(&actor, true, 94), &writes) + .unwrap_err() + .2, + "replay_unavailable" + ); + assert_eq!(actor.attachment.as_ref().unwrap().id, old); +} + +#[test] +fn recovery_attempt_high_water_mark_fences_delayed_and_duplicate_claims() { + let mut actor = isolated_actor(); + let (writes, _receiver) = mpsc::channel(1); + actor.session_result = Some(json!({"configOptions":[]})); + actor + .command(resident_claim(&actor, false, 100), &writes) + .unwrap(); + let original = actor.attachment.as_ref().unwrap().id.clone(); + let recovery = |actor: &Actor, previous: &str, attempt: u64| { + let mut request = resident_claim(actor, false, 100 + attempt); + if let Request::Attach { + startup: Some(message), + .. + } = &mut request + { + message["params"]["_meta"]["kit/gateway"] = json!({ + "previousAttachment":previous,"recoveryId":"ordered-outage","recoveryAttempt":attempt + }); + } + request + }; + // Attempt 1 was delayed before mailbox admission; attempt 2 commits first. + let late = recovery(&actor, &original, 1); + actor + .command(recovery(&actor, &original, 2), &writes) + .unwrap(); + let second = actor.attachment.as_ref().unwrap().id.clone(); + let queued = actor.attachment.as_ref().unwrap().events.clone(); + assert_eq!( + actor.command(late, &writes).unwrap_err().2, + "stale_recovery" + ); + assert_eq!(actor.attachment.as_ref().unwrap().id, second); + assert_eq!(actor.attachment.as_ref().unwrap().events, queued); + // Even the current attachment ID must not bypass duplicate/older fencing. + for attempt in [1, 2] { + let error = actor + .command(recovery(&actor, &second, attempt), &writes) + .unwrap_err(); + assert_eq!( + error.data(), + json!({"reason":"stale_recovery","terminal":true}) + ); + assert_eq!(actor.attachment.as_ref().unwrap().events, queued); + } + actor + .command(recovery(&actor, &original, 3), &writes) + .unwrap(); + assert_ne!(actor.attachment.as_ref().unwrap().id, second); + assert_eq!(actor.attachment.as_ref().unwrap().recovery_attempt, Some(3)); + actor + .command(resident_claim(&actor, false, 104), &writes) + .unwrap(); + let manual = actor.attachment.as_ref().unwrap().id.clone(); + assert!(actor.attachment.as_ref().unwrap().recovery_id.is_none()); + assert!( + actor + .attachment + .as_ref() + .unwrap() + .recovery_attempt + .is_none() + ); + assert_eq!( + actor + .command(recovery(&actor, &original, 4), &writes) + .unwrap_err() + .2, + "controller_replaced" + ); + assert_eq!(actor.attachment.as_ref().unwrap().id, manual); +} + +#[test] +fn recovery_attempt_metadata_is_paired_positive_and_bounded() { + let mut actor = isolated_actor(); + let (writes, _receiver) = mpsc::channel(1); + actor.session_result = Some(json!({"configOptions":[]})); + actor + .command(resident_claim(&actor, false, 110), &writes) + .unwrap(); + let owner = actor.attachment.as_ref().unwrap().id.clone(); + let mut invalid = vec![ + json!({"previousAttachment":owner,"recoveryId":"missing-attempt"}), + json!({"previousAttachment":owner,"recoveryAttempt":1}), + json!({"recoveryId":"missing-predecessor","recoveryAttempt":1}), + ]; + for attempt in [ + json!(0), + json!(-1), + json!(1.5), + json!("1"), + Value::Null, + json!(18446744073709551616_f64), + ] { + invalid.push(json!({"previousAttachment":owner,"recoveryId":"invalid-attempt","recoveryAttempt":attempt})); + } + for metadata in invalid { + let mut request = resident_claim(&actor, false, 111); + if let Request::Attach { + startup: Some(message), + .. + } = &mut request + { + message["params"]["_meta"]["kit/gateway"] = metadata; + } + assert_eq!(actor.command(request, &writes).unwrap_err().2, "protocol"); + assert_eq!(actor.attachment.as_ref().unwrap().id, owner); + assert!( + actor + .attachment + .as_ref() + .unwrap() + .recovery_attempt + .is_none() + ); + } +} diff --git a/src/main.rs b/src/main.rs index d631968..652a9c1 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1073,7 +1073,11 @@ impl Command { .arg(clap::Arg::new("remote_credential_file").long("remote-credential-file") .value_name("FILE").value_parser(clap::value_parser!(PathBuf)).requires("remote")) .arg(clap::Arg::new("remote_session").long("remote-session") - .value_name("ID").requires("remote")); + .value_name("ID").requires("remote")) + .arg(clap::Arg::new("remote_no_replay").long("remote-no-replay") + .action(clap::ArgAction::SetTrue) + .requires_all(["remote", "remote_session"]) + .help("Attach without transcript history using the authoritative current snapshot")); let command = command.arg( clap::Arg::new("root") .long("root") @@ -1211,6 +1215,7 @@ impl Command { remote: optional_arg(matches, "remote")?, remote_credential_file: optional_arg(matches, "remote_credential_file")?, remote_session: optional_arg(matches, "remote_session")?, + remote_no_replay: required_arg(matches, "remote_no_replay")?, root: optional_arg(matches, "root")?, model: optional_arg(matches, "model")?, provider: optional_arg(matches, "provider")?, @@ -1752,6 +1757,7 @@ enum Command { remote: Option, remote_credential_file: Option, remote_session: Option, + remote_no_replay: bool, /// Working directory and project context (defaults to config or `.`). root: Option, /// Model name (defaults to config or `gpt-5.4`). @@ -2484,6 +2490,7 @@ async fn run_cli(cli: Cli) -> Result<(), Box> { remote, remote_credential_file, remote_session, + remote_no_replay, root, model, provider, @@ -2500,6 +2507,7 @@ async fn run_cli(cli: Cli) -> Result<(), Box> { credential_file: remote_credential_file .ok_or("--remote requires --remote-credential-file")?, session: remote_session, + no_replay: remote_no_replay, }; return kit::tui::run_remote(&root, &remote, &mut kit::tui::Stop::new()?).await; } @@ -2645,6 +2653,7 @@ mod tests { remote, remote_credential_file, remote_session, + remote_no_replay, root, resume, .. @@ -2658,6 +2667,7 @@ mod tests { Some(std::path::Path::new("/client/credential")) ); assert_eq!(remote_session.as_deref(), Some("s-remote")); + assert!(!remote_no_replay); assert_eq!( root.as_deref(), Some(std::path::Path::new("/server/project")) @@ -2681,6 +2691,54 @@ mod tests { assert!(Cli::try_parse_from(["kit", "tui", "--remote-session", "s-remote"]).is_err()); } + #[cfg(feature = "tui")] + #[test] + fn remote_no_replay_requires_an_explicit_remote_session() { + let base = [ + "kit", + "tui", + "--remote", + "https://gateway.example", + "--remote-credential-file", + "/client/credential", + "--root", + "/server/project", + ]; + for args in [vec!["kit", "tui"], base.to_vec()] { + let cli = Cli::try_parse_from(args).unwrap(); + let Command::Tui { + remote_no_replay, .. + } = cli.command + else { + panic!("expected tui command"); + }; + assert!(!remote_no_replay); + } + for extra in [ + vec!["--remote-no-replay"], + vec!["--remote-session", "s-remote", "--remote-no-replay"], + ] { + let mut args = vec!["kit", "tui"]; + args.extend(extra); + assert!(Cli::try_parse_from(args).is_err()); + } + let mut args = base.to_vec(); + args.push("--remote-no-replay"); + assert!(Cli::try_parse_from(args.clone()).is_err()); + args.extend(["--remote-session", "s-remote"]); + let cli = Cli::try_parse_from(args).unwrap(); + let Command::Tui { + remote_no_replay, + remote_session, + .. + } = cli.command + else { + panic!("expected tui command"); + }; + assert!(remote_no_replay); + assert_eq!(remote_session.as_deref(), Some("s-remote")); + } + #[cfg(feature = "tui")] #[test] fn tui_resume_forms_preserve_direct_and_normal_startup() { diff --git a/src/tui/app.rs b/src/tui/app.rs index 8f77dfe..e4b3601 100644 --- a/src/tui/app.rs +++ b/src/tui/app.rs @@ -43,11 +43,13 @@ use super::{ }; mod focus; +mod recovery; pub use focus::ChildView; /// Everything the client learns from the agent or its own runtime channel. #[derive(Debug)] pub enum Update { + GatewayRecovery(crate::tui::recovery::Event), ChildSteerFinished { id: String, generation: u64, @@ -105,7 +107,10 @@ pub enum Update { append: bool, }, /// Atomic replacement of an assistant message with ordered content parts. - AgentParts { id: String, parts: Vec }, + AgentParts { + id: String, + parts: Vec, + }, /// Agent reasoning, either appended as a chunk or replaced by an upsert. AgentThought { id: String, @@ -134,7 +139,10 @@ pub enum Update { backgrounded: bool, }, /// ACP parent identity; absence of this update preserves the relationship. - ToolParent { id: String, parent: Option }, + ToolParent { + id: String, + parent: Option, + }, /// Agent-advertised slash commands for one session. AvailableCommands { session_id: String, @@ -822,6 +830,10 @@ pub struct AgentCounts { } pub struct App { + pub(super) gateway_epoch: u64, + pub(super) gateway_blocked: bool, + pub(super) gateway_status: Option, + gateway_candidate: Option, pub child_focus: Option, child_ui: HashMap, pub child_views: HashMap, @@ -1103,6 +1115,10 @@ fn agent_status_rank(status: SubagentStatus) -> u8 { impl App { pub fn new(root: PathBuf, provider: String, model: String, a2a: String) -> Self { Self { + gateway_epoch: 0, + gateway_blocked: false, + gateway_status: None, + gateway_candidate: None, child_focus: None, child_ui: HashMap::new(), child_views: HashMap::new(), @@ -2141,6 +2157,10 @@ impl App { } pub(super) fn apply_materialized(&mut self, update: Update, images: Vec) { + if let Update::GatewayRecovery(event) = update { + self.apply_recovery(event, images); + return; + } if matches!(update, Update::OpenUserImage(_)) { use super::attachment::RetainOpenedError; let result = images @@ -2214,6 +2234,7 @@ impl App { fn apply_observed(&mut self, update: Update) { let at = self.observed_at(); match update { + Update::GatewayRecovery(event) => self.apply_recovery(event, Vec::new()), Update::ChildSteerFinished { id, generation, diff --git a/src/tui/app/recovery.rs b/src/tui/app/recovery.rs new file mode 100644 index 0000000..1da543e --- /dev/null +++ b/src/tui/app/recovery.rs @@ -0,0 +1,465 @@ +//! Single-owner transactional replay: visible App never observes a partial cut. +use super::*; +use crate::tui::recovery::{Event, Kind, MAX_BYTES, MAX_EVENTS}; + +pub(super) struct Candidate { + app: Box, + epoch: u64, + session: String, + bytes: usize, + events: usize, + config: bool, + state_after_config: bool, +} + +impl App { + fn recovery_failed(&mut self, message: &str) { + self.gateway_candidate = None; + self.gateway_blocked = true; + self.gateway_status = Some(format!( + "Recovery failed: {message}. Outcome unknown; do not retry blindly. Use kit gateway list and --remote-session; --remote-no-replay explicitly omits history." + )); + } + + pub(super) fn apply_recovery(&mut self, event: Event, images: Vec) { + let Event { + marker, + session, + bytes, + updates, + } = event; + if marker.epoch < self.gateway_epoch { + return; + } + match marker.kind { + Kind::Begin => { + if marker.epoch == self.gateway_epoch && self.gateway_candidate.is_some() { + return; + } + if self.session_id.as_ref().is_some_and(|id| id != &session) { + return; + } + let mut candidate = App::new( + self.root.clone(), + self.provider.clone(), + self.model.clone(), + self.a2a.clone(), + ); + candidate.session_id = Some(session.clone()); + candidate.can_steer = self.can_steer; + candidate.can_replace_steer = self.can_replace_steer; + self.gateway_candidate = Some(Candidate { + app: Box::new(candidate), + epoch: marker.epoch, + session, + bytes: 0, + events: 0, + config: false, + state_after_config: false, + }); + self.gateway_epoch = marker.epoch; + self.gateway_blocked = true; + self.model_switch = None; + self.model_dialog = None; + self.effort_dialog = None; + self.cancel_steer_edit(); + self.selected_steer = None; + self.queue_focused = false; + self.queue_handoff = false; + self.steer_mutations.clear(); + self.cancel_clipboard_placeholders(); + self.gateway_status = Some(format!( + "Reconnecting {}/{}{} · previous outcome unknown; no automatic resubmission", + marker.attempt.unwrap_or(1), + marker.max_attempts.unwrap_or(1), + marker + .delay_ms + .map(|ms| format!(" (retry in {ms}ms)")) + .unwrap_or_default(), + )); + } + Kind::Replay => { + let Some(candidate) = self.gateway_candidate.as_mut().filter(|candidate| { + candidate.epoch == marker.epoch && candidate.session == session + }) else { + return; + }; + candidate.bytes = candidate.bytes.saturating_add(bytes); + candidate.events = candidate.events.saturating_add(1); + if candidate.bytes > MAX_BYTES || candidate.events > MAX_EVENTS { + self.recovery_failed("replay limit exceeded"); + return; + } + for image in images { + candidate.app.attachment_cache.admit(image); + } + let follows_config = candidate.config; + candidate.config = matches!(updates.as_slice(), [Update::ConfigOptions(_)]); + candidate.state_after_config = + follows_config && matches!(updates.as_slice(), [Update::State(_)]); + for update in updates { + if let Update::ConfigOptions(options) = &update { + crate::tui::refresh_config_state(&mut candidate.app, Some(options)); + } + // Historical state has no live observation timestamp and + // never reaches voice or deferred-submit observers. + candidate.app.apply_at(update, None); + } + } + Kind::Commit => { + if !self.gateway_blocked && self.gateway_candidate.is_none() { + return; + } + if marker.epoch != self.gateway_epoch { + return; + } + let valid = self.gateway_candidate.as_ref().is_some_and(|candidate| { + candidate.epoch == marker.epoch + && candidate.session == session + && candidate.state_after_config + }) && marker.config_snapshot == Some(true) + && marker.state_snapshot == Some(true) + && marker.history_available.is_some(); + if !valid { + self.recovery_failed("incomplete or mismatched snapshot"); + return; + } + let Some(candidate) = self.gateway_candidate.take() else { + self.recovery_failed("missing validated snapshot"); + return; + }; + let mut candidate = candidate.app; + candidate.gateway_epoch = marker.epoch; + candidate.gateway_status = (marker.history_available == Some(false)).then(|| + "History unavailable · transcript omitted by explicit --remote-no-replay; current state is synchronized".into()); + // Preserve client-owned state AND attachment lifetime owners. + // All transcript indexes/caches/runtime state stay with the + // candidate. No callbacks, locks or awaits occur in this commit. + macro_rules! retain { ($($field:ident),* $(,)?) => { $( + std::mem::swap(&mut candidate.$field, &mut self.$field); + )* }; } + retain!( + editor, + attachments, + retained_attachment_files, + next_attachment, + submitted_attachment, + clipboard_route_epoch, + auth_methods, + voice_enabled, + can_steer, + can_replace_steer, + show_thoughts, + agents_visible, + show_logs, + logs, + storage_pending, + storage_exhausted, + next_model_switch, + next_steer_token, + next_file_search_revision + ); + candidate.next_attachment = candidate.next_attachment.max(self.next_attachment); + candidate.follow = self.follow; + candidate.scroll = if self.follow { usize::MAX } else { 0 }; + *self = *candidate; + } + Kind::Failed => { + if marker.epoch != self.gateway_epoch && self.gateway_epoch != 0 { + return; + } + self.gateway_epoch = marker.epoch; + self.recovery_failed(marker.message.as_deref().unwrap_or("gateway unavailable")); + } + Kind::Live => { + if self.gateway_blocked + || marker.epoch != self.gateway_epoch + || self.session_id.as_deref() != Some(session.as_str()) + { + return; + } + for image in images { + self.attachment_cache.admit(image); + } + for update in updates { + if let Update::ConfigOptions(options) = &update { + crate::tui::refresh_config_state(self, Some(options)); + } + self.apply(update); + } + } + } + } +} + +#[cfg(test)] +#[allow( + clippy::unwrap_used, + clippy::expect_used, + clippy::panic, + clippy::unreachable, + clippy::disallowed_methods, + clippy::disallowed_macros +)] +mod tests { + use super::*; + use crate::tui::recovery::Marker; + use agent_client_protocol::schema::v2::{IdleStateUpdate, RunningStateUpdate}; + + fn app() -> App { + let mut app = App::new( + "/tmp/kit".into(), + "provider".into(), + "old".into(), + String::new(), + ); + app.start_session("session".into()); + app.gateway_epoch = 1; + app.apply(Update::AgentMessage { + id: "old".into(), + text: "old transcript".into(), + append: false, + }); + app.paste("unsent draft"); + app + } + fn event(kind: Kind, updates: Vec) -> Event { + Event { + marker: Marker { + epoch: 2, + kind, + attempt: Some(2), + max_attempts: Some(5), + delay_ms: None, + message: None, + history_available: Some(true), + state_snapshot: Some(true), + config_snapshot: Some(true), + }, + session: "session".into(), + bytes: 100, + updates, + } + } + fn replay(app: &mut App, update: Update) { + app.apply(Update::GatewayRecovery(event(Kind::Replay, vec![update]))); + } + fn snapshot(app: &mut App, running: bool) { + use agent_client_protocol::schema::v2::{ + SessionConfigOption, SessionConfigSelectGroup, SessionConfigSelectOption, + }; + let model = if running { + "running-model" + } else { + "idle-model" + }; + let value = format!("openrouter:{model}"); + replay( + app, + Update::ConfigOptions(vec![SessionConfigOption::select( + "model", + "Model", + value.clone(), + vec![SessionConfigSelectGroup::new( + "openrouter", + "OpenRouter", + vec![SessionConfigSelectOption::new(value, model)], + )], + )]), + ); + replay( + app, + Update::State(if running { + StateUpdate::Running(RunningStateUpdate::new()) + } else { + StateUpdate::Idle(IdleStateUpdate::new()) + }), + ); + } + fn text(app: &App) -> String { + app.blocks + .iter() + .filter_map(|block| match block { + Block::Agent(text) => Some(text.as_str()), + _ => None, + }) + .collect::>() + .join("\n") + } + + #[test] + fn failed_and_invalid_replays_preserve_visible_transcript_and_draft() { + for failure in [Kind::Failed, Kind::Commit] { + let mut app = app(); + let old = text(&app); + app.apply(Update::GatewayRecovery(event(Kind::Begin, Vec::new()))); + replay( + &mut app, + Update::AgentMessage { + id: "new".into(), + text: "partial".into(), + append: false, + }, + ); + assert_eq!(text(&app), old); + app.apply(Update::GatewayRecovery(event(failure, Vec::new()))); + assert_eq!(text(&app), old); + assert_eq!(app.editor.text(), "unsent draft"); + assert!(app.gateway_blocked); + } + } + #[test] + fn commit_replaces_once_retains_draft_and_uses_authoritative_state() { + for running in [true, false] { + let mut app = app(); + app.apply(Update::GatewayRecovery(event(Kind::Begin, Vec::new()))); + replay( + &mut app, + Update::AgentMessage { + id: "new".into(), + text: "complete".into(), + append: false, + }, + ); + snapshot(&mut app, !running); + snapshot(&mut app, running); + app.apply(Update::GatewayRecovery(event(Kind::Commit, Vec::new()))); + assert_eq!( + app.blocks + .iter() + .filter(|block| matches!(block, Block::Agent(_))) + .count(), + 1 + ); + assert!(text(&app).contains("complete")); + assert!(!text(&app).contains("old transcript")); + assert_eq!(app.editor.text(), "unsent draft"); + assert!(app.phase == if running { Phase::Working } else { Phase::Idle }); + assert_eq!( + app.model, + if running { + "running-model" + } else { + "idle-model" + } + ); + assert!(!app.gateway_blocked); + let before = text(&app); + app.apply(Update::GatewayRecovery(event(Kind::Commit, Vec::new()))); + assert_eq!(text(&app), before); + assert!(!app.gateway_blocked); + } + } + #[test] + fn commit_preserves_attachment_ownership_and_composer() { + let mut app = app(); + app.attach( + "/tmp/draft.png".into(), + "image/png", + AttachmentKind::Image, + 3, + ); + let draft = app.editor.text().to_string(); + let placeholder = app.attachments[0].placeholder.clone(); + app.apply(Update::GatewayRecovery(event(Kind::Begin, Vec::new()))); + snapshot(&mut app, true); + app.apply(Update::GatewayRecovery(event(Kind::Commit, Vec::new()))); + assert_eq!(app.editor.text(), draft); + assert_eq!(app.attachments.len(), 1); + assert_eq!(app.attachments[0].placeholder, placeholder); + app.attach( + "/tmp/next.png".into(), + "image/png", + AttachmentKind::Image, + 3, + ); + assert_ne!( + app.attachments[0].placeholder, + app.attachments[1].placeholder + ); + } + + #[test] + fn nonadjacent_or_interrupted_snapshot_tail_is_rejected() { + for ignored in [true, false] { + let mut app = app(); + let old = text(&app); + app.apply(Update::GatewayRecovery(event(Kind::Begin, Vec::new()))); + snapshot(&mut app, true); + if ignored { + app.apply(Update::GatewayRecovery(event(Kind::Replay, Vec::new()))); + } else { + replay( + &mut app, + Update::AgentMessage { + id: "late".into(), + text: "not a snapshot tail".into(), + append: false, + }, + ); + replay( + &mut app, + Update::State(StateUpdate::Running(RunningStateUpdate::new())), + ); + } + app.apply(Update::GatewayRecovery(event(Kind::Commit, Vec::new()))); + assert_eq!(text(&app), old); + assert!(app.gateway_blocked); + } + } + + #[test] + fn wrong_session_live_update_cannot_mutate_confirmed_view() { + let mut app = app(); + let old = text(&app); + let mut wrong = event( + Kind::Live, + vec![Update::AgentMessage { + id: "wrong".into(), + text: "wrong session".into(), + append: false, + }], + ); + wrong.marker.epoch = 1; + wrong.session = "other".into(); + app.apply(Update::GatewayRecovery(wrong)); + assert_eq!(text(&app), old); + } + + #[test] + fn omitted_history_is_persistent_and_never_guesses_idle() { + let mut app = app(); + app.apply(Update::GatewayRecovery(event(Kind::Begin, Vec::new()))); + snapshot(&mut app, true); + let mut commit = event(Kind::Commit, Vec::new()); + commit.marker.history_available = Some(false); + app.apply(Update::GatewayRecovery(commit)); + assert!(app.blocks.is_empty()); + assert!(app.phase == Phase::Working); + assert!( + app.gateway_status + .as_deref() + .unwrap() + .contains("History unavailable") + ); + } + #[test] + fn over_limit_and_mismatched_commits_cannot_install_candidate() { + let mut app = app(); + let old = text(&app); + app.apply(Update::GatewayRecovery(event(Kind::Begin, Vec::new()))); + let mut oversized = event(Kind::Replay, Vec::new()); + oversized.bytes = MAX_BYTES + 1; + app.apply(Update::GatewayRecovery(oversized)); + snapshot(&mut app, true); + app.apply(Update::GatewayRecovery(event(Kind::Commit, Vec::new()))); + assert_eq!(text(&app), old); + assert!(app.gateway_blocked); + app.apply(Update::GatewayRecovery(event(Kind::Begin, Vec::new()))); + snapshot(&mut app, true); + let mut wrong = event(Kind::Commit, Vec::new()); + wrong.session = "other".into(); + app.apply(Update::GatewayRecovery(wrong)); + assert_eq!(text(&app), old); + } +} diff --git a/src/tui/mod.rs b/src/tui/mod.rs index a919c83..93ef110 100644 --- a/src/tui/mod.rs +++ b/src/tui/mod.rs @@ -17,6 +17,7 @@ mod input; #[cfg(all(test, unix))] mod keyboard_tests; mod markdown; +mod recovery; mod scheduler; mod source; mod startup; @@ -639,7 +640,13 @@ fn spawn_background_workers( break; } let mut remaining = 64 * 1024 * 1024; - let images = match &queued.update { + let materialize = match &queued.update { + Update::GatewayRecovery(event) => { + event.updates.first().unwrap_or(&queued.update) + } + update => update, + }; + let images = match materialize { Update::UserMessage { images, .. } => images .iter() // File-backed placeholders need materialization before they can @@ -1479,6 +1486,12 @@ async fn run_client( // Local agents fix themselves to a canonical root. Remote paths belong to // the gateway host and must never be resolved against the client filesystem. let is_remote = remote.is_some(); + let initial_remote_resume = remote.is_some_and(|remote| remote.session.is_some()); + let recovery_ingress = Arc::new(Mutex::new(recovery::Ingress::for_session( + remote + .and_then(|remote| remote.session.clone()) + .or_else(|| resume.map(str::to_owned)), + ))); let voice_enabled = voice_enabled && !is_remote; let root = &client_root(root, is_remote)?; let credential_storage = @@ -1636,6 +1649,7 @@ async fn run_client( let cleanup_root = root.clone(); let transition_session = Arc::clone(&active_persisted_id); let notification_session = Arc::clone(&active_persisted_id); + let notification_recovery = Arc::clone(&recovery_ingress); let result = { let root = root.clone(); let model = model.clone(); @@ -1646,6 +1660,12 @@ async fn run_client( .v2() .on_receive_notification( async move |notification: UpdateSessionNotification, _cx| { + if is_remote { + if let Some(queued) = recovery::notification(notification, ¬ification_recovery, ¬ification_session) { + let _ = notifications.send(queued); + } + return Ok(()); + } let current = notification_session.lock().ok().map(|route| route.clone()); for update in current.as_ref().map_or_else(Vec::new, |route| { translate_for_session(notification, &route.id) @@ -1748,7 +1768,7 @@ async fn run_client( }; let initial = match initial { Err(error) - if auth_methods.is_empty() + if is_remote || auth_methods.is_empty() || !authentication_required(&error, &auth_methods) => { return Err(error); @@ -1948,7 +1968,9 @@ async fn run_client( } } Ok(update) => match update { - Some(update) => app.apply(update.update), + Some(update) => { + if let Some(update) = accept_queued_update(&transition_session, update) { app.apply(update); } + }, None => { drop(events); leave(&mut terminal); @@ -1986,6 +2008,11 @@ async fn run_client( active.id = active_session_id.clone(); } app.start_session(active_session_id.clone()); + if is_remote { + recovery_ingress.lock().map_err(|_| agent_client_protocol::util::internal_error("recovery admission poisoned"))?.confirm_session(&active_session_id); + app.gateway_epoch = if initial_remote_resume { 0 } else { 1 }; + app.gateway_blocked = initial_remote_resume; + } let storage_shutdown = crate::resilient_fs::shutdown_token(); let mut voice = NativeVoice::default(); // This scope owns events (authentication moves/drops it) but only @@ -2019,6 +2046,17 @@ async fn run_client( let mut priority = scheduler::Priority::default(); let mut frames = scheduler::Frames::new(tokio::time::Instant::now()); loop { + let recovery_unavailable = is_remote && !recovery_available(&recovery_ingress, &transition_session, &app); + if recovery_unavailable { + if !app.gateway_blocked { + app.gateway_status = Some("Reconnecting · previous outcome unknown; mutations disabled".into()); + } + app.gateway_blocked = true; + if let Some(pending) = clipboard_pastes.pending.as_mut() { pending.submit = false; } + submit_after_paste = false; + child_read = None; + app.child_focus = None; + } let target = app.child_read_target().map(|target| (session_id.to_string(), target)); if child_read.as_ref().is_some_and(|read| Some(&read.target) != target.as_ref()) { // Dropping the future cancels the local request and frees its page. @@ -2136,7 +2174,8 @@ async fn run_client( } SessionEvent::ChildTranscript(result) => { if let Some(read) = child_read.take() - && read.target.0 == session_id.to_string() { + && read.target.0 == session_id.to_string() + && (!is_remote || recovery_available(&recovery_ingress, &transition_session, &app)) { finish_child_transcript(&mut app, &mut frames, &read.target.1, result); } } @@ -2195,6 +2234,9 @@ async fn run_client( }, SessionEvent::StorageShutdown => return Ok(()), SessionEvent::Terminal(terminal_event) => { + if is_remote && !recovery_available(&recovery_ingress, &transition_session, &app) { + app.gateway_blocked = true; + } // A paste is a burst: one bracketed-paste event, or // thousands of key events where the terminal cannot // bracket it. Applying everything the terminal has @@ -2219,7 +2261,15 @@ async fn run_client( break; } } + let action_generation = transition_session.lock().ok().map(|route| route.generation); match action { + action if is_remote && !recovery_available(&recovery_ingress, &transition_session, &app) && recovery_mutation(&action) => { + if let Action::Submit { prompt, .. } = action { + app.paste(&prompt.text); + app.restore_attachments(prompt.attachments); + } + app.toast("Gateway reconnecting; mutations disabled. Previous acceptance may be unknown."); + } Action::Usage(_) | Action::Voice(_) | Action::Login(_) | Action::New(_) | Action::ListSessions | Action::RenameSession { .. } | Action::Resume(_) if is_remote => { @@ -2280,6 +2330,12 @@ async fn run_client( .await .map(|_| None) }; + if is_remote && !route_generation_matches(&transition_session, action_generation) { + app.paste(&prompt.text); + app.restore_attachments(prompt.attachments); + app.toast("Message acceptance unknown; not retried. Review the recovered transcript."); + continue; + } match outcome { Ok(Some(message_id)) => { app.accept_attachments(&prompt.attachments); @@ -2293,12 +2349,12 @@ async fn run_client( Err(error) => { app.paste(&prompt.text); app.restore_attachments(prompt.attachments); - app.note(format!("message was not accepted: {}", error.message)); + app.note(recovery::submission_error(&error)); } } } Action::SteerChild { id, generation, text } => { - // transition_route is the only route writer. Snapshot its + // Route transitions and recovery admission fence writers. Snapshot its // generation synchronously with the root ACP identity, and // drop the guard before starting any asynchronous work. let route_generation = match transition_session.lock() { @@ -2405,7 +2461,7 @@ async fn run_client( .block_task() .await; if let Err(error) = outcome { - app.note(format!("message was not accepted: {}", error.message)); + app.note(recovery::submission_error(&error)); } } } @@ -2589,6 +2645,10 @@ async fn run_client( )) .block_task() .await; + if is_remote && !route_generation_matches(&transition_session, action_generation) { + app.toast("Configuration outcome unknown; recovered snapshot will be authoritative"); + continue; + } match response { Ok(response) => { refresh_config_state( @@ -2737,14 +2797,18 @@ async fn run_client( ); } Action::DetachCompose(call_id) => { - match connection + let response = connection .send_request(DetachComposeRequest { session_id: session_id.clone(), call_id, }) .block_task() - .await - { + .await; + if is_remote && !route_generation_matches(&transition_session, action_generation) { + app.toast("Backgrounding outcome unknown; not retried"); + continue; + } + match response { Ok(response) if !response.detached => { app.note("compose call is no longer running in the foreground"); } @@ -2762,6 +2826,10 @@ async fn run_client( }) .block_task() .await; + if is_remote && !route_generation_matches(&transition_session, action_generation) { + app.toast("Cancellation outcome unknown; not retried"); + continue; + } if let Err(error) = response { app.note(format!( "could not cancel background call: {}", error.message @@ -2773,6 +2841,7 @@ async fn run_client( revision, activation, } => { + let Some(generation) = action_generation else { continue; }; let connection = connection.clone(); let updates = updates_tx.clone(); tokio::spawn(async move { @@ -2782,7 +2851,7 @@ async fn run_client( .await .map(|response| response.matches) .map_err(|error| error.message.to_string()); - let _ = updates.send(QueuedUpdate::global( + let _ = updates.send(QueuedUpdate::for_session(generation, Update::FileMatches { revision, result }, )); }); @@ -3174,6 +3243,15 @@ fn pending_message_unavailable(error: &agent_client_protocol::Error) -> bool { /// Keep Enter behind earlier native pastes without blocking terminal events. fn handle_with_clipboard(app: &mut App, pastes: &mut ClipboardPastes, event: Event) -> Action { + if app.gateway_blocked { + if let Some(pending) = pastes.pending.as_mut() { + pending.submit = false; + } + if matches!(&event, Event::Key(key) if key.code == KeyCode::Enter) { + app.toast("Gateway recovery in progress; input retained, not submitted"); + return Action::None; + } + } if let Some(pending) = &mut pastes.pending { if pending.route != app.clipboard_route() { pastes.pending = None; @@ -3313,7 +3391,55 @@ fn handle(app: &mut App, event: Event) -> Action { action } +fn recovery_available( + ingress: &Arc>, + route: &Arc>, + app: &App, +) -> bool { + // Same ingress -> route ordering as callback admission. Poison isolates the + // remote attachment; never infer an epoch or permit mutations by default. + ingress + .lock() + .is_ok_and(|ingress| ingress.available(app) && route.lock().is_ok()) +} + +fn route_generation_matches(route: &Arc>, expected: Option) -> bool { + route + .lock() + .is_ok_and(|route| Some(route.generation) == expected) +} + +fn recovery_mutation(action: &Action) -> bool { + matches!( + action, + Action::Submit { .. } + | Action::ReplaceSteer { .. } + | Action::RevokeSteer { .. } + | Action::SteerChild { .. } + | Action::SelectModel { .. } + | Action::ConfirmModelSwitch(_) + | Action::SelectEffort { .. } + | Action::Cancel + | Action::DetachCompose(_) + | Action::CancelBackground(_) + | Action::New(_) + | Action::Resume(_) + | Action::RenameSession { .. } + | Action::Login(_) + | Action::Voice(_) + ) +} + fn handle_inner(app: &mut App, event: Event) -> Action { + if app.gateway_blocked + && matches!(&event, Event::Key(key) if key.code == KeyCode::Char('c') && key.modifiers.contains(KeyModifiers::CONTROL)) + { + return Action::Quit; + } + if app.gateway_blocked && matches!(&event, Event::Key(key) if key.code == KeyCode::Enter) { + app.toast("Gateway recovery in progress; input retained, not submitted"); + return Action::None; + } match event { Event::Key(key) if clipboard_paste_key(key) => { if paste_blocked(app) { diff --git a/src/tui/recovery.rs b/src/tui/recovery.rs new file mode 100644 index 0000000..ba64c9b --- /dev/null +++ b/src/tui/recovery.rs @@ -0,0 +1,404 @@ +//! Ephemeral bridge recovery protocol. Admission fences precede FIFO application. +use super::{ActiveSessionRoute, App, QueuedUpdate, Update, UpdateSessionNotification, translate}; +use serde::Deserialize; +use std::io::Write; +use std::sync::{Arc, Mutex}; +pub(super) const META: &str = "kit/gatewayRecovery"; +pub(super) const MAX_BYTES: usize = 8 * 1024 * 1024; +pub(super) const MAX_EVENTS: usize = 4096; +#[derive(Clone, Copy, Debug, Deserialize, PartialEq, Eq)] +#[serde(rename_all = "lowercase")] +pub(super) enum Kind { + Begin, + Replay, + Commit, + Failed, + Live, +} +#[derive(Debug, Deserialize)] +#[serde(rename_all = "camelCase")] +pub(super) struct Marker { + pub epoch: u64, + pub kind: Kind, + pub attempt: Option, + pub max_attempts: Option, + pub delay_ms: Option, + pub message: Option, + pub history_available: Option, + pub state_snapshot: Option, + pub config_snapshot: Option, +} +#[derive(Debug)] +pub(super) struct Event { + pub marker: Marker, + pub session: String, + pub bytes: usize, + pub updates: Vec, +} +/// Callback-only writes, event-loop read-only admission. Lock order: ingress +/// then route. No other owner takes ingress with route held. Guards never cross +/// send/await/application or App destruction. Poison fails closed. +pub(super) struct Ingress { + epoch: u64, + begun_epoch: Option, + recovering: bool, + session: Option, + bytes: usize, + events: usize, +} +impl Default for Ingress { + fn default() -> Self { + Self { + epoch: 1, + begun_epoch: None, + recovering: false, + session: None, + bytes: 0, + events: 0, + } + } +} +impl Ingress { + pub fn for_session(session: Option) -> Self { + Self { + session, + ..Self::default() + } + } + + pub fn confirm_session(&mut self, session: &str) { + self.session = Some(session.to_owned()); + } + + pub fn available(&self, app: &App) -> bool { + !self.recovering && !app.gateway_blocked && self.epoch == app.gateway_epoch + } +} +struct Size(usize); +impl Write for Size { + fn write(&mut self, bytes: &[u8]) -> std::io::Result { + self.0 = self.0.saturating_add(bytes.len()); + if self.0 > MAX_BYTES { + return Err(std::io::Error::other("replay limit")); + } + Ok(bytes.len()) + } + fn flush(&mut self) -> std::io::Result<()> { + Ok(()) + } +} +pub(super) fn notification( + notification: UpdateSessionNotification, + ingress: &Arc>, + route: &Arc>, +) -> Option { + let session = notification.session_id.to_string(); + let mut size = Size(0); + let sized = serde_json::to_writer(&mut size, ¬ification).is_ok(); + let marker = notification.meta.as_ref().and_then(|meta| meta.get(META)); + let parsed = marker.and_then(|marker| serde_json::from_value::(marker.clone()).ok()); + let mut ingress = ingress.lock().ok()?; + let mut route = route.lock().ok()?; + if ingress.session.as_ref().is_some_and(|id| id != &session) { + return None; + } + let mut marker = match parsed.filter(|marker| marker.epoch > 0) { + Some(marker) => marker, + None => Marker { + epoch: ingress.epoch, + kind: Kind::Failed, + attempt: None, + max_attempts: None, + delay_ms: None, + message: Some("Invalid gateway recovery metadata".into()), + history_available: None, + state_snapshot: None, + config_snapshot: None, + }, + }; + if marker.epoch < ingress.epoch { + return None; + } + match marker.kind { + Kind::Begin => { + if ingress + .begun_epoch + .is_some_and(|epoch| marker.epoch <= epoch) + { + return None; + } + ingress.epoch = marker.epoch; + ingress.begun_epoch = Some(marker.epoch); + ingress.recovering = true; + ingress.bytes = 0; + ingress.events = 0; + ingress.session = Some(session.clone()); + route.generation = route.generation.checked_add(1)?; + } + Kind::Live if marker.epoch == ingress.epoch && !ingress.recovering => {} + Kind::Replay if marker.epoch == ingress.epoch && ingress.recovering => { + ingress.bytes = ingress.bytes.saturating_add(size.0); + ingress.events = ingress.events.saturating_add(1); + if !sized || ingress.bytes > MAX_BYTES || ingress.events > MAX_EVENTS { + marker.kind = Kind::Failed; + marker.message = Some("Gateway replay exceeds the safe replay limit".into()); + } + } + Kind::Commit if marker.epoch == ingress.epoch && ingress.recovering => { + // App independently validates the snapshot and keeps its gate shut + // on failure. Epoch alone never authorizes user input. + ingress.recovering = false; + } + Kind::Failed if marker.epoch == ingress.epoch => { + if !ingress.recovering { + route.generation = route.generation.checked_add(1)?; + } + ingress.recovering = true; + } + _ => return None, + } + let generation = route.generation; + drop(route); + drop(ingress); + let updates = if matches!(marker.kind, Kind::Replay | Kind::Live) { + translate(notification).1 + } else { + Vec::new() + }; + Some(QueuedUpdate::for_session( + generation, + Update::GatewayRecovery(Event { + marker, + session, + bytes: size.0, + updates, + }), + )) +} +pub(super) fn submission_error(error: &agent_client_protocol::Error) -> String { + if error + .data + .as_ref() + .and_then(|data| data.get("reason")) + .and_then(|value| value.as_str()) + == Some("outcome_unknown") + { + "Message acceptance is unknown; it was not retried. Review the recovered transcript before submitting again.".into() + } else { + format!("message was not accepted: {}", error.message) + } +} + +#[cfg(test)] +#[allow( + clippy::unwrap_used, + clippy::expect_used, + clippy::panic, + clippy::unreachable, + clippy::disallowed_methods, + clippy::disallowed_macros +)] +mod tests { + use super::*; + use crate::tui::{BackgroundCompletion, accept_queued_update, spawn_background_workers, wire}; + use base64::Engine; + + fn marked(kind: &str, epoch: u64, update: wire::SessionUpdate) -> UpdateSessionNotification { + let mut notification = UpdateSessionNotification::new("session", update); + notification.meta = Some( + serde_json::from_value(serde_json::json!({ META: { + "kind": kind, "epoch": epoch, "attempt": 1, "maxAttempts": 5, + "historyAvailable": true, "configSnapshot": true, "stateSnapshot": true + }})) + .unwrap(), + ); + notification + } + fn state() -> wire::SessionUpdate { + wire::SessionUpdate::StateUpdate( + wire::StateUpdate::Running(wire::RunningStateUpdate::new()), + ) + } + fn route() -> Arc> { + Arc::new(Mutex::new(ActiveSessionRoute { + id: "session".into(), + generation: 0, + })) + } + #[test] + fn begin_fences_queued_updates_and_old_epoch_notifications() { + let route = route(); + let ingress = Arc::new(Mutex::new(Ingress::default())); + let old = QueuedUpdate::for_session(0, Update::ConfigOptions(Vec::new())); + let begin = notification(marked("begin", 2, state()), &ingress, &route).unwrap(); + assert!(accept_queued_update(&route, old).is_none()); + assert!(accept_queued_update(&route, begin).is_some()); + assert!(notification(marked("live", 1, state()), &ingress, &route).is_none()); + assert!(notification(marked("begin", 2, state()), &ingress, &route).is_none()); + } + #[test] + fn malformed_marker_is_not_translated_as_live_state() { + let route = route(); + let ingress = Arc::new(Mutex::new(Ingress::default())); + let mut bad = marked("live", 1, state()); + bad.meta + .as_mut() + .unwrap() + .insert(META.into(), serde_json::json!({"epoch":"1","kind":"live"})); + let queued = notification(bad, &ingress, &route).unwrap(); + assert!( + matches!(queued.update, Update::GatewayRecovery(Event { marker: Marker {kind: Kind::Failed, ..}, updates, .. }) if updates.is_empty()) + ); + } + #[test] + fn outcome_unknown_does_not_claim_rejection_or_authorize_retry() { + let mut error = agent_client_protocol::util::internal_error("lost request"); + error.data = + Some(serde_json::json!({"reason":"outcome_unknown", "method":"session/prompt"})); + let message = submission_error(&error); + assert!(message.contains("acceptance is unknown")); + assert!(message.contains("not retried")); + assert!(!message.contains("not accepted")); + } + #[test] + fn poisoned_route_isolates_recovery_and_remote_mutations() { + let route = route(); + let ingress = Arc::new(Mutex::new(Ingress::default())); + let _ = std::panic::catch_unwind(|| { + let _guard = route.lock().unwrap(); + panic!("poison route at ownership boundary"); + }); + let mut app = App::new( + "/tmp/kit".into(), + "provider".into(), + "model".into(), + String::new(), + ); + app.gateway_epoch = 1; + assert!(!crate::tui::recovery_available(&ingress, &route, &app)); + assert!(notification(marked("begin", 2, state()), &ingress, &route).is_none()); + } + + #[test] + fn recovery_enter_preserves_draft_without_deferring_a_submission() { + let mut app = App::new( + "/tmp/kit".into(), + "provider".into(), + "model".into(), + String::new(), + ); + app.paste("unsent"); + app.gateway_blocked = true; + let mut pastes = crate::tui::ClipboardPastes::default(); + let action = crate::tui::handle_with_clipboard( + &mut app, + &mut pastes, + crossterm::event::Event::Key(crossterm::event::KeyEvent::new( + crossterm::event::KeyCode::Enter, + crossterm::event::KeyModifiers::NONE, + )), + ); + assert!(matches!(action, crate::tui::Action::None)); + assert_eq!(app.editor.text(), "unsent"); + assert!(pastes.pending.is_none()); + } + + #[tokio::test] + async fn image_worker_keeps_commit_after_replay_and_installs_once() { + let route = route(); + let ingress = Arc::new(Mutex::new(Ingress::default())); + let (completed, mut completions) = tokio::sync::mpsc::channel(1); + let workers = spawn_background_workers(completed).unwrap(); + let mut png = std::io::Cursor::new(Vec::new()); + image::DynamicImage::new_rgb8(1, 1) + .write_to(&mut png, image::ImageFormat::Png) + .unwrap(); + let image = crate::tui::app::UserImage::new( + base64::engine::general_purpose::STANDARD.encode(png.into_inner()), + "image/png".into(), + 0, + ) + .unwrap(); + let begin = notification(marked("begin", 2, state()), &ingress, &route).unwrap(); + assert!(workers.try_update(begin).is_ok()); + let mut replay = notification(marked("replay", 2, state()), &ingress, &route).unwrap(); + let Update::GatewayRecovery(event) = &mut replay.update else { + panic!("recovery update"); + }; + event.updates = vec![Update::UserMessage { + id: "user".into(), + text: "replayed".into(), + images: vec![image], + append: false, + }]; + assert!(workers.try_update(replay).is_ok()); + for update in [ + marked( + "replay", + 2, + wire::SessionUpdate::ConfigOptionUpdate(wire::ConfigOptionUpdate::new(Vec::new())), + ), + marked("replay", 2, state()), + marked("commit", 2, state()), + ] { + assert!( + workers + .try_update(notification(update, &ingress, &route).unwrap()) + .is_ok() + ); + } + let mut app = App::new( + "/tmp/kit".into(), + "provider".into(), + "model".into(), + String::new(), + ); + app.start_session("session".into()); + app.gateway_epoch = 1; + app.note("old transcript"); + app.paste("unsent"); + let mut kinds = Vec::new(); + for _ in 0..5 { + let completion = + tokio::time::timeout(std::time::Duration::from_secs(5), completions.recv()) + .await + .unwrap() + .unwrap(); + let BackgroundCompletion::Update { queued, images } = completion else { + panic!("update completion"); + }; + let Update::GatewayRecovery(event) = &queued.update else { + panic!("recovery update"); + }; + let kind = event.marker.kind; + kinds.push(kind); + if matches!(event.updates.first(), Some(Update::UserMessage { .. })) { + assert_eq!(images.len(), 1); + } + app.apply_materialized(accept_queued_update(&route, queued).unwrap(), images); + if kind != Kind::Commit { + assert!( + matches!(app.blocks.as_slice(), [crate::tui::app::Block::Notice(text)] if text == "old transcript") + ); + } + } + assert_eq!( + kinds, + [ + Kind::Begin, + Kind::Replay, + Kind::Replay, + Kind::Replay, + Kind::Commit + ] + ); + assert!(matches!( + app.blocks.as_slice(), + [crate::tui::app::Block::User(_)] + )); + assert_eq!(app.editor.text(), "unsent"); + assert!(!app.gateway_blocked); + drop(completions); + drop(workers); + } +} diff --git a/src/tui/ui.rs b/src/tui/ui.rs index 78cb736..87c6dd9 100644 --- a/src/tui/ui.rs +++ b/src/tui/ui.rs @@ -3193,6 +3193,12 @@ fn draw_status(frame: &mut Frame<'_>, app: &App, area: Rect) { Span::styled(format!(" {}", timing_label(app.elapsed())), theme::dim()), ], }; + if let Some(status) = &app.gateway_status { + left = vec![Span::styled( + format!(" {status}"), + Style::default().fg(theme::warn_color()), + )]; + } let counts = app.agent_counts(); let active_agents = counts.starting + counts.working; if active_agents > 0 { @@ -6358,6 +6364,7 @@ mod tests { url: "https://gateway.example/kit?a=b&c=d".into(), credential_file: PathBuf::from("/client/team's credential"), session: Some("s-original".into()), + no_replay: false, }; let local = app.resume_command().unwrap(); let command = crate::tui::attachment_clipboard_text(&app, Some(&remote), local.clone()); diff --git a/tests/gateway.rs b/tests/gateway.rs index f069da9..ad2b82b 100644 --- a/tests/gateway.rs +++ b/tests/gateway.rs @@ -113,6 +113,10 @@ impl Fixture { } fn bridge(&self, url: &str, session: Option<&str>) -> Bridge { + self.bridge_options(url, session, false) + } + + fn bridge_options(&self, url: &str, session: Option<&str>, no_replay: bool) -> Bridge { let mut command = self.command(); command .args(["gateway", "bridge", "--url", url, "--credential-file"]) @@ -125,6 +129,9 @@ impl Fixture { if let Some(session) = session { command.args(["--session", session]); } + if no_replay { + command.arg("--no-replay"); + } let mut child = command.spawn().unwrap(); Bridge { updates: Vec::new(), @@ -1434,7 +1441,7 @@ async fn pinned_sdk_http_initialize_capabilities_and_session_stream_contract() { json!({"jsonrpc":"2.0","id":1,"result":{ "protocolVersion":2,"info":{"name":"kit-gateway","version":env!("CARGO_PKG_VERSION")}, "capabilities":{"session":{"prompt":{"image":{},"audio":{},"embeddedContext":{}},"inject":{"modes":["steer"],"steerInStream":["finish"],"pending":{"replace":true}},"list":{}}}, - "_meta":{"kit/gateway":{"experimental":true,"transport":"bounded-http","maxFrameBytes":1048576,"coreBufferedBytesPerDirection":16777216,"httpEgressBytesPerConnection":4194304,"liveReplayLimitBytes":8388608,"liveReplayLimitEvents":4094}} + "_meta":{"kit/gateway":{"experimental":true,"transport":"bounded-http","maxFrameBytes":1048576,"coreBufferedBytesPerDirection":16777216,"httpEgressBytesPerConnection":4194304,"liveReplayLimitBytes":8388608,"liveReplayLimitEvents":4093}} }}) ); let mut stream = fixture @@ -1781,3 +1788,621 @@ async fn unread_session_mailbox_terminates_http_but_preserves_resident_prompt() stop_gateway(&mut gateway).await; provider.abort(); } + +// Real reverse proxy: successful frames, statuses, and headers come from the +// gateway process. A single-owner arbiter selects faults from parsed wire data. +#[derive(Clone)] +struct RecoveryProxyState { + upstream: String, + client: reqwest::Client, + inspect: tokio::sync::mpsc::Sender, +} +struct ProxyInspection { + phase: &'static str, + frame: Value, + decision: tokio::sync::oneshot::Sender, +} +struct ProxyFault { + phase: &'static str, + pointer: &'static str, + value: Value, +} +struct RecoveryProxy { + url: String, + server: tokio::task::JoinHandle<()>, + arbiter: tokio::task::JoinHandle<()>, + observed: tokio::sync::mpsc::UnboundedReceiver, +} +impl RecoveryProxyState { + async fn drop_frame(&self, phase: &'static str, frame: Value) -> bool { + let (decision, result) = tokio::sync::oneshot::channel(); + self.inspect + .send(ProxyInspection { + phase, + frame, + decision, + }) + .await + .unwrap(); + result.await.unwrap() + } +} +fn broken_proxy_body() -> axum::body::Body { + axum::body::Body::from_stream(futures_util::stream::once(async { + Err::(std::io::Error::other("intentional test transport loss")) + })) +} +async fn recovery_proxy_forward( + axum::extract::State(state): axum::extract::State, + request: axum::extract::Request, +) -> axum::response::Response { + let (parts, body) = request.into_parts(); + let body = axum::body::to_bytes(body, 16 * 1024 * 1024).await.unwrap(); + if let Ok(frame) = serde_json::from_slice::(&body) + && state.drop_frame("request", frame).await + { + return axum::response::Response::new(broken_proxy_body()); + } + let mut headers = parts.headers; + headers.remove("host"); + headers.remove("content-length"); + let response = state + .client + .request(parts.method, format!("{}{}", state.upstream, parts.uri)) + .headers(headers) + .body(body) + .send() + .await + .unwrap(); + let status = response.status(); + let mut headers = response.headers().clone(); + headers.remove("content-length"); + headers.remove("transfer-encoding"); + let sse = headers + .get("content-type") + .is_some_and(|value| value == "text/event-stream"); + let body = if sse { + let (send, receive) = tokio::sync::mpsc::channel::, std::io::Error>>(8); + tokio::spawn(async move { + let mut stream = response.bytes_stream(); + let mut buffer = Vec::new(); + while let Some(chunk) = stream.next().await { + let Ok(chunk) = chunk else { + let _ = send + .send(Err(std::io::Error::other("upstream closed"))) + .await; + return; + }; + buffer.extend_from_slice(&chunk); + while let Some(end) = buffer.windows(2).position(|bytes| bytes == b"\n\n") { + let event: Vec<_> = buffer.drain(..end + 2).collect(); + let text = std::str::from_utf8(&event).unwrap(); + for data in text.lines().filter_map(|line| line.strip_prefix("data:")) { + let frame: Value = serde_json::from_str(data.trim()).unwrap(); + if state.drop_frame("sse", frame).await { + let _ = send + .send(Err(std::io::Error::other("intentional SSE loss"))) + .await; + return; + } + } + if send.send(Ok(event)).await.is_err() { + return; + } + } + } + }); + axum::body::Body::from_stream(futures_util::stream::unfold(receive, |mut receive| async { + receive.recv().await.map(|chunk| (chunk, receive)) + })) + } else { + axum::body::Body::from_stream(response.bytes_stream()) + }; + let mut response = axum::response::Response::new(body); + *response.status_mut() = status; + *response.headers_mut() = headers; + response +} +impl RecoveryProxy { + async fn start(upstream: &str, faults: Vec) -> Self { + let (inspect, mut inspections) = tokio::sync::mpsc::channel::(32); + let (observed, received) = tokio::sync::mpsc::unbounded_channel(); + let arbiter = tokio::spawn(async move { + let mut faults = std::collections::VecDeque::from(faults); + while let Some(inspection) = inspections.recv().await { + let drop_frame = faults.front().is_some_and(|fault| { + fault.phase == inspection.phase + && inspection.frame.pointer(fault.pointer) == Some(&fault.value) + }); + if drop_frame { + faults.pop_front(); + } + let _ = observed.send( + json!({"phase":inspection.phase,"frame":inspection.frame,"dropped":drop_frame}), + ); + let _ = inspection.decision.send(drop_frame); + } + }); + let state = RecoveryProxyState { + upstream: upstream.into(), + client: reqwest::Client::builder().no_proxy().build().unwrap(), + inspect, + }; + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let url = format!("http://{}", listener.local_addr().unwrap()); + let app = axum::Router::new() + .fallback(recovery_proxy_forward) + .with_state(state); + let server = tokio::spawn(async move { + axum::serve(listener, app).await.unwrap(); + }); + Self { + url, + server, + arbiter, + observed: received, + } + } + fn observations(&mut self) -> Vec { + let mut result = Vec::new(); + while let Ok(value) = self.observed.try_recv() { + result.push(value); + } + result + } +} +impl Drop for RecoveryProxy { + fn drop(&mut self) { + self.server.abort(); + self.arbiter.abort(); + } +} +impl Bridge { + async fn send_wire(&mut self, message: Value) { + self.stdin + .write_all(format!("{message}\n").as_bytes()) + .await + .unwrap(); + } + async fn read_wire(&mut self) -> Value { + let line = timeout(WAIT, self.stdout.next_line()) + .await + .expect("bridge wire output timed out") + .unwrap() + .expect("bridge stdout closed"); + serde_json::from_str(&line).unwrap() + } + async fn recovery_until(&mut self, kind: &str) -> Vec { + timeout(WAIT, async { + let mut frames = Vec::new(); + loop { + let frame = self.read_wire().await; + let done = frame["params"]["_meta"]["kit/gatewayRecovery"]["kind"] == kind; + frames.push(frame); + if done { + return frames; + } + } + }) + .await + .expect("recovery did not reach expected terminal boundary") + } +} +fn wire_method_count(observed: &[Value], method: &str) -> usize { + observed + .iter() + .filter(|entry| entry["phase"] == "request" && entry["frame"]["method"] == method) + .count() +} +#[tokio::test] +async fn bridge_recovery_transient_preinitialize_drop() { + let fixture = Fixture::new(); + let (mut gateway, url) = fixture.gateway().await; + let mut proxy = RecoveryProxy::start( + &url, + vec![ProxyFault { + phase: "request", + pointer: "/method", + value: json!("initialize"), + }], + ) + .await; + let mut bridge = fixture.bridge(&proxy.url, None); + let id = bridge.handshake().await; + assert!(!id.is_empty()); + let observed = proxy.observations(); + assert_eq!(wire_method_count(&observed, "initialize"), 2); + assert_eq!(wire_method_count(&observed, "session/new"), 1); + bridge.detach().await; + stop_gateway(&mut gateway).await; +} +#[tokio::test] +async fn bridge_recovery_lost_create_never_creates_again() { + let fixture = Fixture::new(); + let (mut gateway, url) = fixture.gateway().await; + let mut proxy = RecoveryProxy::start( + &url, + vec![ProxyFault { + phase: "sse", + pointer: "/id", + value: json!(2), + }], + ) + .await; + let mut bridge = fixture.bridge(&proxy.url, None); + bridge.request(1, "initialize", json!({"protocolVersion":2,"info":{"name":"fault-test","version":"0"},"capabilities":{}})).await; + bridge.send_wire(json!({"jsonrpc":"2.0","id":2,"method":"session/new","params":{"cwd":fixture.root,"mcpServers":[]}})).await; + let response = loop { + let frame = bridge.read_wire().await; + if frame["id"] == 2 { + break frame; + } + }; + assert_eq!( + response["error"]["data"]["reason"], "outcome_unknown", + "{response}" + ); + timeout(WAIT, bridge.child.wait()) + .await + .expect("unknown create outcome must terminate without retry") + .unwrap(); + let listed = fixture.catalog(&url).await; + let sessions = listed["sessions"].as_array().unwrap(); + assert_eq!(sessions.len(), 1, "{listed}"); + assert_eq!(sessions[0]["_meta"]["kit.gateway.state"], "resident"); + let observed = proxy.observations(); + assert_eq!(wire_method_count(&observed, "session/new"), 1); + let id = sessions[0]["sessionId"].as_str().unwrap(); + let mut explicit = fixture.bridge(&url, Some(id)); + assert_eq!(explicit.handshake().await, id); + explicit.detach().await; + stop_gateway(&mut gateway).await; +} + +async fn bridge_active_recovery_faults(interrupt_replay: bool) { + let (fixture, mut requests, provider) = controlled_provider().await; + let (mut gateway, url) = fixture.gateway().await; + let mut faults = vec![ProxyFault { + phase: "sse", + pointer: "/id", + value: json!(3), + }]; + if interrupt_replay { + faults.push(ProxyFault { + phase: "sse", + pointer: "/params/update/state", + value: json!("running"), + }); + } + let mut proxy = RecoveryProxy::start(&url, faults).await; + let mut bridge = fixture.bridge(&proxy.url, None); + let id = bridge.handshake().await; + bridge.send_wire(json!({"jsonrpc":"2.0","id":3,"method":"session/prompt","params":{"sessionId":id,"prompt":[{"type":"text","text":"Accepted exactly once"}]}})).await; + // The real provider's response remains held while HTTP recovery executes. + let (_, abandoned) = timeout(WAIT, requests.recv()).await.unwrap().unwrap(); + let frames = bridge.recovery_until("commit").await; + let metadata: Vec<_> = frames + .iter() + .filter_map(|frame| frame["params"]["_meta"].get("kit/gatewayRecovery")) + .collect(); + let committed = *metadata.last().unwrap(); + let epoch = committed["epoch"].as_u64().unwrap(); + assert!(epoch > 0); + assert_eq!(committed["historyAvailable"], true); + assert_eq!(committed["stateSnapshot"], true); + assert_eq!(committed["configSnapshot"], true); + assert_eq!( + metadata + .iter() + .filter(|meta| meta["kind"] == "commit") + .count(), + 1 + ); + assert!( + metadata + .iter() + .any(|meta| meta["kind"] == "begin" && meta["epoch"] == epoch) + ); + let candidate: Vec<_> = frames + .iter() + .filter(|frame| { + let meta = &frame["params"]["_meta"]["kit/gatewayRecovery"]; + meta["kind"] == "replay" && meta["epoch"] == epoch + }) + .collect(); + assert!(!candidate.is_empty()); + assert!( + candidate + .iter() + .all(|frame| frame["params"]["sessionId"] == id) + ); + assert!( + candidate + .iter() + .any(|frame| frame["params"]["update"]["sessionUpdate"] == "config_option_update") + ); + assert!( + candidate + .iter() + .any(|frame| frame["params"]["update"]["state"] == "running") + ); + assert!(candidate.iter().any(|frame| { + frame["params"]["update"] + .to_string() + .contains("Accepted exactly once") + })); + assert_eq!(frames.iter().filter(|frame| frame["id"] == 3).count(), 1); + let uncertain = frames.iter().find(|frame| frame["id"] == 3).unwrap(); + assert_eq!(uncertain["error"]["data"]["reason"], "outcome_unknown"); + assert_eq!(uncertain["error"]["data"]["method"], "session/prompt"); + assert_eq!(uncertain["error"]["data"]["sessionId"], id); + if interrupt_replay { + assert!( + metadata + .iter() + .any(|meta| meta["kind"] == "begin" && meta["epoch"].as_u64().unwrap() < epoch) + ); + } + // Cancellation acts on the original accepted run, without releasing its + // provider response. A subsequent prompt must be the second provider call. + bridge + .send_wire(json!({"jsonrpc":"2.0","method":"session/cancel","params":{"sessionId":id}})) + .await; + bridge.wait_wire_idle().await; + bridge + .request( + 4, + "session/prompt", + json!({"sessionId":id,"prompt":[{"type":"text","text":"Healthy after recovery"}]}), + ) + .await; + let (body, release) = timeout(WAIT, requests.recv()).await.unwrap().unwrap(); + assert!( + body["messages"] + .to_string() + .contains("Healthy after recovery"), + "{body}" + ); + release.send("Recovered exactly once").unwrap(); + let completed = bridge.wait_wire_idle().await; + let updates: Vec<_> = completed + .iter() + .map(|frame| frame["params"]["update"].clone()) + .collect(); + assert_replayed_answer(&updates, "Recovered exactly once"); + assert!(requests.try_recv().is_err(), "unexpected extra inference"); + let observed = proxy.observations(); + let attempts = if interrupt_replay { 3 } else { 2 }; + assert_eq!(wire_method_count(&observed, "initialize"), attempts); + assert_eq!(wire_method_count(&observed, "session/resume"), attempts - 1); + assert_eq!(wire_method_count(&observed, "session/new"), 1); + assert_eq!(wire_method_count(&observed, "session/prompt"), 2); + assert_eq!(wire_method_count(&observed, "session/cancel"), 1); + assert!( + observed + .iter() + .filter(|entry| entry["frame"]["method"] == "session/resume") + .all(|entry| entry["frame"]["params"]["sessionId"] == id) + ); + drop(abandoned); + bridge.detach().await; + stop_gateway(&mut gateway).await; + provider.abort(); +} + +impl Bridge { + async fn wait_wire_idle(&mut self) -> Vec { + timeout(WAIT, async { + let mut frames = Vec::new(); + loop { + let frame = self.read_wire().await; + let idle = frame["params"]["update"]["sessionUpdate"] == "state_update" + && frame["params"]["update"]["state"] == "idle"; + frames.push(frame); + if idle { + return frames; + } + } + }) + .await + .expect("recovered bridge did not become idle") + } +} + +#[tokio::test] +async fn bridge_recovery_lost_prompt_response_cancels_original_run() { + bridge_active_recovery_faults(false).await; +} + +#[tokio::test] +async fn bridge_recovery_interrupted_replay_commits_only_fresh_candidate() { + bridge_active_recovery_faults(true).await; +} + +#[tokio::test] +async fn bridge_recovery_wrong_auth_is_terminal_without_retry() { + let fixture = Fixture::new(); + let (mut gateway, url) = fixture.gateway().await; + let mut proxy = RecoveryProxy::start(&url, vec![]).await; + // The gateway has already read its credential; only this bridge reads the + // changed isolated fixture file. The proxy forwards the actual HTTP 401. + fs::write(&fixture.credential, "incorrect-test-token").unwrap(); + let mut bridge = fixture.bridge(&proxy.url, None); + bridge.send_wire(json!({"jsonrpc":"2.0","id":1,"method":"initialize","params":{"protocolVersion":2,"info":{"name":"wrong-auth","version":"0"},"capabilities":{}}})).await; + let response = loop { + let frame = bridge.read_wire().await; + if frame["id"] == 1 { + break frame; + } + }; + assert!(response.get("error").is_some(), "{response}"); + let Bridge { + stdin, + mut stdout, + mut child, + .. + } = bridge; + drop(stdin); + timeout(WAIT, async { + while stdout.next_line().await.unwrap().is_some() {} + child.wait().await.unwrap(); + }) + .await + .expect("unauthorized bridge did not stop"); + let observed = proxy.observations(); + assert_eq!(wire_method_count(&observed, "initialize"), 1); + assert_eq!(wire_method_count(&observed, "session/new"), 0); + assert_eq!(wire_method_count(&observed, "session/resume"), 0); + stop_gateway(&mut gateway).await; +} + +#[tokio::test] +async fn bridge_recovery_replaced_controller_never_steals_session_back() { + let (fixture, mut requests, provider) = controlled_provider().await; + let (mut gateway, url) = fixture.gateway().await; + let mut proxy = RecoveryProxy::start(&url, vec![]).await; + let mut old = fixture.bridge(&proxy.url, None); + let id = old.handshake().await; + let mut replacement = SdkClient::connect(&url).await; + let resumed = replacement + .request( + 2, + "session/resume", + json!({"sessionId":id,"cwd":fixture.root,"replayFrom":{"type":"start"}}), + ) + .await; + assert!(resumed.get("error").is_none(), "{resumed}"); + let frames = old.recovery_until("failed").await; + assert!( + !frames + .iter() + .any(|frame| frame["params"]["_meta"]["kit/gatewayRecovery"]["kind"] == "commit") + ); + replacement + .start_prompt(3, &id, "Only replacement controls this session") + .await; + let (_, release) = timeout(WAIT, requests.recv()).await.unwrap().unwrap(); + release.send("Replacement retained authority").unwrap(); + replacement.wait_idle().await; + let observed = proxy.observations(); + assert_eq!(wire_method_count(&observed, "initialize"), 1); + assert_eq!(wire_method_count(&observed, "session/resume"), 0); + assert_eq!(wire_method_count(&observed, "session/new"), 1); + let _ = old.child.kill().await; + replacement.detach().await; + stop_gateway(&mut gateway).await; + provider.abort(); +} + +#[tokio::test] +async fn bridge_recovery_explicit_no_replay_oversized_history_snapshots_idle_and_running() { + static ANSWER: std::sync::LazyLock = std::sync::LazyLock::new(|| "h".repeat(64 * 1024)); + let (fixture, mut requests, provider) = controlled_provider().await; + let (mut gateway, url) = fixture.gateway().await; + let mut seed = SdkClient::connect(&url).await; + let created = seed + .request( + 2, + "session/new", + json!({"cwd":fixture.root,"mcpServers":[]}), + ) + .await; + let id = created["result"]["sessionId"].as_str().unwrap(); + // Pace every turn on actual idle, not socket buffering or a sleep. Each + // individual turn fits egress while cumulative journal history exceeds 8MiB. + for turn in 0..129 { + seed.start_prompt(3 + turn, id, "Grow history").await; + let (_, release) = timeout(WAIT, requests.recv()).await.unwrap().unwrap(); + release.send(ANSWER.as_str()).unwrap(); + seed.wait_idle().await; + } + seed.detach().await; + let mut full = SdkClient::connect(&url).await; + let denied = full + .request( + 2, + "session/resume", + json!({"sessionId":id,"cwd":fixture.root,"replayFrom":{"type":"start"}}), + ) + .await; + assert!( + denied.get("error").is_some(), + "full replay unexpectedly succeeded" + ); + full.detach().await; + let mut idle = fixture.bridge_options(&url, Some(id), true); + assert_no_replay_snapshot(&mut idle, id, "idle").await; + idle.request( + 3, + "session/prompt", + json!({"sessionId":id,"prompt":[{"type":"text","text":"Remain running through snapshot"}]}), + ) + .await; + let (_, abandoned) = timeout(WAIT, requests.recv()).await.unwrap().unwrap(); + let mut active = fixture.bridge_options(&url, Some(id), true); + assert_no_replay_snapshot(&mut active, id, "running").await; + active + .send_wire(json!({"jsonrpc":"2.0","method":"session/cancel","params":{"sessionId":id}})) + .await; + active.wait_wire_idle().await; + drop(abandoned); + let _ = idle.child.kill().await; + active.detach().await; + stop_gateway(&mut gateway).await; + provider.abort(); +} + +async fn assert_no_replay_snapshot(bridge: &mut Bridge, id: &str, state: &str) { + bridge.request(1, "initialize", json!({"protocolVersion":2,"info":{"name":"snapshot-test","version":"0"},"capabilities":{}})).await; + bridge.send_wire(json!({"jsonrpc":"2.0","id":2,"method":"session/new","params":{"cwd":"/ignored","mcpServers":[]}})).await; + let frames = timeout(WAIT, async { + let mut frames = Vec::new(); + let mut response = false; + let mut commit = false; + while !response || !commit { + let frame = bridge.read_wire().await; + if frame["id"] == 2 { + assert!(frame.get("error").is_none(), "{frame}"); + assert_eq!(frame["result"]["sessionId"], id); + response = true; + } + commit |= frame["params"]["_meta"]["kit/gatewayRecovery"]["kind"] == "commit"; + frames.push(frame); + } + frames + }) + .await + .expect("explicit no-replay resume did not commit"); + let commit = frames + .iter() + .find(|frame| frame["params"]["_meta"]["kit/gatewayRecovery"]["kind"] == "commit") + .unwrap(); + let meta = &commit["params"]["_meta"]["kit/gatewayRecovery"]; + assert_eq!(meta["historyAvailable"], false); + assert_eq!(meta["stateSnapshot"], true); + assert_eq!(meta["configSnapshot"], true); + let candidate: Vec<_> = frames + .iter() + .filter(|frame| frame["params"]["_meta"]["kit/gatewayRecovery"]["kind"] == "replay") + .collect(); + assert_eq!( + candidate.len(), + 2, + "no-replay must contain only authoritative config and state" + ); + assert!( + candidate + .iter() + .all(|frame| frame["params"]["sessionId"] == id) + ); + assert_eq!( + candidate[0]["params"]["update"]["sessionUpdate"], + "config_option_update" + ); + assert!(candidate[0]["params"]["update"]["configOptions"].is_array()); + assert_eq!( + candidate[1]["params"]["update"]["sessionUpdate"], + "state_update" + ); + assert_eq!(candidate[1]["params"]["update"]["state"], state); +} From 8ff1342bcc21a5eea71c121742eb30d2d18b369c Mon Sep 17 00:00:00 2001 From: daniel Date: Tue, 29 Sep 2026 04:20:00 +0100 Subject: [PATCH 2/4] fix(gateway): admit requests before publishing recovery readiness --- src/gateway/http_client/recovery.rs | 20 ++-- src/gateway/http_client/recovery/tests.rs | 113 +++++++++++++++++++++- 2 files changed, 124 insertions(+), 9 deletions(-) diff --git a/src/gateway/http_client/recovery.rs b/src/gateway/http_client/recovery.rs index 76de23c..810ea57 100644 --- a/src/gateway/http_client/recovery.rs +++ b/src/gateway/http_client/recovery.rs @@ -540,6 +540,13 @@ pub(super) async fn run( break; } if !initialized { + // Open admission before publishing readiness: a local reader + // can enqueue its next request before emit/flush returns. + initialized = true; + phase = "live"; + gate = Some(Instant::now()); + recovery_start = None; + attempt = 0; // Initial initialize is safe to retry; preserve its original local ID. if let Some(index) = pending.iter().position(|v| v["method"] == "initialize") @@ -547,11 +554,6 @@ pub(super) async fn run( value["id"] = pending.remove(index)["id"].clone(); emit(&mut output, &value)?; } - initialized = true; - phase = "live"; - gate = Some(Instant::now()); - recovery_start = None; - attempt = 0; } else { let mut params = object([ ( @@ -611,6 +613,11 @@ pub(super) async fn run( token, ) { attachment = Some(token.to_owned()); + // Commit is the local readiness publication. Fence old + // queued input before it becomes visible to the consumer. + phase = "live"; + gate = Some(Instant::now()); + recovery_start = None; notice( &mut output, session.as_deref().ok_or("Missing durable target")?, @@ -620,9 +627,6 @@ pub(super) async fn run( "Reconnected", bits, )?; - phase = "live"; - gate = Some(Instant::now()); - recovery_start = None; attempt = 0; } else { terminal = Some("Gateway recovery snapshot unavailable; session left unchanged".into()); diff --git a/src/gateway/http_client/recovery/tests.rs b/src/gateway/http_client/recovery/tests.rs index c1da9ad..caf5d43 100644 --- a/src/gateway/http_client/recovery/tests.rs +++ b/src/gateway/http_client/recovery/tests.rs @@ -185,6 +185,60 @@ fn output_pipe() -> ( }); (writer, receive) } +// Output is visible to an independent ACP peer before flush returns. This +// acknowledged OS pipe deterministically lets that peer enqueue its follow-up +// in that window, without callbacks or instrumentation in production code. +struct AcknowledgedOutput { + writer: std::os::unix::net::UnixStream, + consumed: std::sync::mpsc::Receiver<()>, +} +impl Write for AcknowledgedOutput { + fn write(&mut self, bytes: &[u8]) -> io::Result { + self.writer.write(bytes) + } + fn flush(&mut self) -> io::Result<()> { + self.writer.flush()?; + self.consumed + .recv_timeout(Duration::from_secs(5)) + .map_err(io::Error::other) + } +} +fn follow_up_during_publication( + input: mpsc::Sender<(Instant, io::Result)>, + on_commit: bool, + next: Value, +) -> (AcknowledgedOutput, mpsc::UnboundedReceiver) { + let (writer, reader) = std::os::unix::net::UnixStream::pair().unwrap(); + let (consumed, acknowledgement) = std::sync::mpsc::channel(); + let (observed, receive) = mpsc::unbounded_channel(); + std::thread::spawn(move || { + let mut follow_up = Some((input, next)); + for line in io::BufReader::new(reader).lines() { + let value: Value = serde_json::from_str(&line.unwrap()).unwrap(); + let ready = if on_commit { + value["params"]["_meta"][META]["kind"] == "commit" + } else { + value["id"] == 1 && value.get("result").is_some() + }; + if ready && let Some((input, next)) = follow_up.take() { + input + .blocking_send((Instant::now(), Ok(next.to_string()))) + .unwrap(); + } + if observed.send(value).is_err() || consumed.send(()).is_err() { + break; + } + } + }); + ( + AcknowledgedOutput { + writer, + consumed: acknowledgement, + }, + receive, + ) +} + fn frame(id: u64, method: &str, params: Value) -> (Instant, io::Result) { ( Instant::now(), @@ -277,7 +331,11 @@ async fn lost_prompt_reinitializes_and_replays_same_session_before_commit() { let mut server = Server::new(None).await; let remote = server.remote.clone(); let (send, lines) = mpsc::channel(16); - let (writer, mut output) = output_pipe(); + let (writer, mut output) = follow_up_during_publication( + send.clone(), + true, + json!({"jsonrpc":"2.0","id":5,"method":"session/prompt","params":{"sessionId":"durable","prompt":[]}}), + ); let scenario = async { send.send(frame(1, "initialize", initialize_params().unwrap())) .await @@ -381,6 +439,18 @@ async fn lost_prompt_reinitializes_and_replays_same_session_before_commit() { begin["params"]["_meta"][META]["epoch"] ); assert_eq!(commit["params"]["_meta"][META]["historyAvailable"], true); + let follow_up = tokio::select! { + request = server.requests.recv() => request.unwrap(), + response = until_id(&mut output,json!(5)) => panic!("post-commit request rejected: {response}"), + }; + assert_eq!(follow_up["method"], "session/prompt"); + server.result(&follow_up, json!({"stopReason":"end_turn"}), "durable"); + assert!( + until_id(&mut output, json!(5)) + .await + .get("result") + .is_some() + ); drop(send); }; let (result, ()) = tokio::time::timeout(Duration::from_secs(8), async { @@ -463,3 +533,44 @@ async fn eof_during_backoff_cancels_recovery_without_another_http_attempt() { result.unwrap(); assert!(server.requests.try_recv().is_err()); } + +#[tokio::test] +async fn recovered_initialize_accepts_follow_up_before_output_flush_returns() { + let mut server = Server::new(Some(StatusCode::SERVICE_UNAVAILABLE)).await; + let remote = server.remote.clone(); + let (send, lines) = mpsc::channel(16); + let (writer, mut output) = follow_up_during_publication( + send.clone(), + false, + json!({"jsonrpc":"2.0","id":2,"method":"session/new","params":{"cwd":"/local","mcpServers":[]}}), + ); + let scenario = async { + send.send(frame(1, "initialize", initialize_params().unwrap())) + .await + .unwrap(); + assert_eq!(server.requests.recv().await.unwrap()["id"], 1); + assert!(reserved(&server.requests.recv().await.unwrap()["id"])); + assert_eq!( + until_id(&mut output, json!(1)).await["result"]["protocolVersion"], + 2 + ); + let new = tokio::select! { + request = server.requests.recv() => request.unwrap(), + response = until_id(&mut output,json!(2)) => panic!("post-initialize request rejected: {response}"), + }; + assert_eq!(new["method"], "session/new"); + server.result(&new,json!({"sessionId":"durable","configOptions":[],"_meta":{"kit/gateway":{"attachment":"owner-1"}}}),""); + assert_eq!( + until_id(&mut output, json!(2)).await["result"]["sessionId"], + "durable" + ); + drop(send); + }; + let (result, ()) = tokio::time::timeout(Duration::from_secs(8), async { + tokio::join!(run(remote, "/remote".into(), lines, writer), scenario) + }) + .await + .unwrap(); + result.unwrap(); + assert!(server.requests.try_recv().is_err()); +} From 091d65627cd76e7f3317a6cd937a71122ff8e3d8 Mon Sep 17 00:00:00 2001 From: daniel Date: Tue, 29 Sep 2026 04:29:02 +0100 Subject: [PATCH 3/4] test(gateway): accept closing connection status after stream EOF --- tests/gateway.rs | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/tests/gateway.rs b/tests/gateway.rs index ad2b82b..c08d80c 100644 --- a/tests/gateway.rs +++ b/tests/gateway.rs @@ -1747,7 +1747,12 @@ async fn unread_session_mailbox_terminates_http_but_preserves_resident_prompt() .send() .await .unwrap(); - assert_eq!(rejected.status(), reqwest::StatusCode::NOT_FOUND); + // Stream EOF can precede registry removal: the SDK rejects a registered + // closing connection with 410, and a removed connection with 404. + assert!(matches!( + rejected.status(), + reqwest::StatusCode::GONE | reqwest::StatusCode::NOT_FOUND + )); // The accepted resident turn must finish durably without a new controller // refreshing its lease, even though its original HTTP connection is gone. From 3b6b6f768cc57eacc02f1765ca77fa638699bbbf Mon Sep 17 00:00:00 2001 From: daniel Date: Tue, 29 Sep 2026 15:13:48 +0100 Subject: [PATCH 4/4] fix: preserve recovery ownership fences and steering drafts --- src/gateway.rs | 19 +++-- src/gateway/tests.rs | 158 +++++++++++++++++++++++++++++++++++++++- src/tui/app/recovery.rs | 94 +++++++++++++++++++++++- src/tui/editor.rs | 11 ++- 4 files changed, 269 insertions(+), 13 deletions(-) diff --git a/src/gateway.rs b/src/gateway.rs index 3f0e105..3c8ba54 100644 --- a/src/gateway.rs +++ b/src/gateway.rs @@ -662,10 +662,13 @@ async fn handle( } } } -struct Attachment { +struct CommittedOwner { id: String, recovery_id: Option, recovery_attempt: Option, +} +struct Attachment { + id: String, touched: Instant, next: u64, events: VecDeque<(u64, Value)>, @@ -696,6 +699,8 @@ struct Actor { root: PathBuf, restore: bool, attachment: Option, + // Survives detach, expiry and replay overflow; only a committed claim replaces it. + committed_owner: Option, serial: u64, pending: HashMap, initialized: Option, @@ -730,6 +735,7 @@ fn spawn(root: &Path, id: &str, restore: bool, force: bool) -> io::Result root: root.to_owned(), restore, attachment: None, + committed_owner: None, serial: 0, pending: HashMap::new(), initialized: None, @@ -1103,7 +1109,7 @@ impl Actor { } // A matching predecessor must not bypass the sequence high-water mark. // Validate before preparing anything, and commit the mark only with ownership. - if let Some(owner) = &self.attachment + if let Some(owner) = &self.committed_owner && recovery_id.is_some() && owner.recovery_id == recovery_id && recovery_attempt <= owner.recovery_attempt @@ -1115,7 +1121,7 @@ impl Actor { )); } if let Some(previous) = previous - && self.attachment.as_ref().is_some_and(|owner| { + && self.committed_owner.as_ref().is_some_and(|owner| { owner.id != previous && !(recovery_id.is_some() && owner.recovery_id == recovery_id) }) @@ -1144,8 +1150,6 @@ impl Actor { let id = crate::session::new_id(); let mut attachment = Attachment { id: id.clone(), - recovery_id, - recovery_attempt, touched: Instant::now(), next: 0, events: VecDeque::new(), @@ -1172,6 +1176,11 @@ impl Actor { }; // No fallible work after the ownership commit. The previous controller // survives every replay/snapshot preparation failure. + self.committed_owner = Some(CommittedOwner { + id: id.clone(), + recovery_id, + recovery_attempt, + }); self.attachment = Some(attachment); return Ok(object([ ("started", started.into()), diff --git a/src/gateway/tests.rs b/src/gateway/tests.rs index a61f6ba..974524b 100644 --- a/src/gateway/tests.rs +++ b/src/gateway/tests.rs @@ -107,6 +107,7 @@ for line in sys.stdin: root: self.gateway.roots[0].clone(), restore: false, attachment: None, + committed_owner: None, serial: 0, pending: HashMap::new(), initialized: None, @@ -398,6 +399,7 @@ fn isolated_actor() -> Actor { root: PathBuf::from("/approved"), restore: false, attachment: None, + committed_owner: None, serial: 0, pending: HashMap::new(), initialized: None, @@ -1384,15 +1386,25 @@ fn recovery_attempt_high_water_mark_fences_delayed_and_duplicate_claims() { .command(recovery(&actor, &original, 3), &writes) .unwrap(); assert_ne!(actor.attachment.as_ref().unwrap().id, second); - assert_eq!(actor.attachment.as_ref().unwrap().recovery_attempt, Some(3)); + assert_eq!( + actor.committed_owner.as_ref().unwrap().recovery_attempt, + Some(3) + ); actor .command(resident_claim(&actor, false, 104), &writes) .unwrap(); let manual = actor.attachment.as_ref().unwrap().id.clone(); - assert!(actor.attachment.as_ref().unwrap().recovery_id.is_none()); assert!( actor - .attachment + .committed_owner + .as_ref() + .unwrap() + .recovery_id + .is_none() + ); + assert!( + actor + .committed_owner .as_ref() .unwrap() .recovery_attempt @@ -1445,7 +1457,7 @@ fn recovery_attempt_metadata_is_paired_positive_and_bounded() { assert_eq!(actor.attachment.as_ref().unwrap().id, owner); assert!( actor - .attachment + .committed_owner .as_ref() .unwrap() .recovery_attempt @@ -1453,3 +1465,141 @@ fn recovery_attempt_metadata_is_paired_positive_and_bounded() { ); } } + +fn recovery_claim(actor: &Actor, previous: &str, outage: &str, attempt: u64) -> Request { + let mut request = resident_claim(actor, false, 200 + attempt); + if let Request::Attach { + startup: Some(message), + .. + } = &mut request + { + message["params"]["_meta"]["kit/gateway"] = json!({ + "previousAttachment": previous, "recoveryId": outage, "recoveryAttempt": attempt + }); + } + request +} + +#[test] +fn detached_owner_fences_replaced_controller_and_preserves_recovery_high_water_mark() { + for overflow in [false, true] { + let mut actor = isolated_actor(); + let (writes, _receiver) = mpsc::channel(1); + actor.session_result = Some(json!({"configOptions":[]})); + actor + .command(resident_claim(&actor, false, 200), &writes) + .unwrap(); + let a = actor.attachment.as_ref().unwrap().id.clone(); + actor + .command(resident_claim(&actor, false, 201), &writes) + .unwrap(); + let b = actor.attachment.as_ref().unwrap().id.clone(); + if overflow { + actor.emit(json!({"payload":"x".repeat(MAX_REPLAY)})); + } else { + actor + .command( + Request::Detach { + session: actor.id.clone(), + attachment: b.clone(), + }, + &writes, + ) + .unwrap(); + } + assert!(actor.attachment.is_none()); + assert_eq!( + actor + .command(recovery_claim(&actor, &a, "a-outage", 1), &writes) + .unwrap_err() + .2, + "controller_replaced" + ); + assert!(actor.attachment.is_none()); + + // Failed preparation of a manual replacement must not erase B's fence. + let mut invalid = resident_claim(&actor, false, 202); + if let Request::Attach { + startup: Some(message), + .. + } = &mut invalid + { + message.as_object_mut().unwrap().remove("id"); + } + assert!(actor.command(invalid, &writes).is_err()); + assert_eq!(actor.committed_owner.as_ref().unwrap().id, b); + assert_eq!( + actor + .command(recovery_claim(&actor, &a, "a-outage", 2), &writes) + .unwrap_err() + .2, + "controller_replaced" + ); + actor + .command(recovery_claim(&actor, &b, "b-outage", 2), &writes) + .unwrap(); + let recovered = actor.attachment.as_ref().unwrap().id.clone(); + actor + .command( + Request::Detach { + session: actor.id.clone(), + attachment: recovered, + }, + &writes, + ) + .unwrap(); + for attempt in [1, 2] { + assert_eq!( + actor + .command(recovery_claim(&actor, &b, "b-outage", attempt), &writes) + .unwrap_err() + .2, + "stale_recovery" + ); + assert!(actor.attachment.is_none()); + } + // Lost recovery result: original B token is still valid for the same outage. + actor + .command(recovery_claim(&actor, &b, "b-outage", 3), &writes) + .unwrap(); + assert_eq!( + actor.committed_owner.as_ref().unwrap().recovery_attempt, + Some(3) + ); + } +} + +#[tokio::test] +async fn cancelled_detach_reply_keeps_committed_generation_fence() { + let harness = Harness::new().await; + harness.child().await; + let a = harness.attach().await; + harness.handshake(&a).await; + let b = harness + .call(json!({"op":"attach","session":"resident","replace":true,"replay":false})) + .await["attachment"] + .as_str() + .unwrap() + .to_owned(); + let entry = harness.gateway.sessions.lock().await["resident"].clone(); + let (reply, accepted) = oneshot::channel(); + entry + .sender + .send(Envelope { + request: Request::Detach { + session: "resident".into(), + attachment: b.clone(), + }, + reply, + }) + .await + .unwrap(); + drop(accepted); + // FIFO mailbox order makes cancellation deterministic, without sleeps. + for (previous, expected) in [(&a, StatusCode::CONFLICT), (&b, StatusCode::OK)] { + let (status, _) = harness.raw(json!({"op":"attach","session":"resident","replace":true,"replay":false, + "startup":{"jsonrpc":"2.0","id":205,"method":"session/resume","params":{"sessionId":"resident","_meta":{"kit/gateway":{"previousAttachment":previous,"recoveryId":"cancelled-detach","recoveryAttempt":1}}}} + })).await; + assert_eq!(status, expected); + } +} diff --git a/src/tui/app/recovery.rs b/src/tui/app/recovery.rs index 1da543e..bed89d8 100644 --- a/src/tui/app/recovery.rs +++ b/src/tui/app/recovery.rs @@ -62,12 +62,22 @@ impl App { self.model_switch = None; self.model_dialog = None; self.effort_dialog = None; + self.cancel_clipboard_placeholders(); + // The replacement target belongs to the old connection, but + // both unsent drafts belong to the client. Restore the ordinary + // composer and keep the edited text available for manual recall. + let steering_draft = self + .steer_edit + .as_ref() + .map(|_| self.editor.text().to_owned()); self.cancel_steer_edit(); + if let Some(text) = steering_draft { + self.editor.remember(text); + } self.selected_steer = None; self.queue_focused = false; self.queue_handoff = false; self.steer_mutations.clear(); - self.cancel_clipboard_placeholders(); self.gateway_status = Some(format!( "Reconnecting {}/{}{} · previous outcome unknown; no automatic resubmission", marker.attempt.unwrap_or(1), @@ -287,6 +297,88 @@ mod tests { .join("\n") } + fn active_modified_steer(app: &mut App) { + app.can_steer = true; + app.can_replace_steer = true; + app.apply(Update::State(StateUpdate::Running( + RunningStateUpdate::new(), + ))); + app.apply(Update::SteerAccepted { + editable: true, + id: "old-target".into(), + text: "queued steering".into(), + }); + app.handle_key(KeyEvent::new(KeyCode::F(2), KeyModifiers::NONE)); + app.handle_key(KeyEvent::new(KeyCode::Enter, KeyModifiers::NONE)); + assert!(app.editing_steer()); + app.paste(" modified but unsent"); + assert_eq!(app.editor.text(), "queued steering modified but unsent"); + } + + fn assert_both_drafts_recallable(app: &mut App) { + assert!(!app.editing_steer()); + assert_eq!(app.editor.text(), "unsent draft"); + assert!(matches!( + app.handle_key(KeyEvent::new(KeyCode::Up, KeyModifiers::NONE)), + Action::None + )); + assert_eq!(app.editor.text(), "queued steering modified but unsent"); + assert!(matches!( + app.handle_key(KeyEvent::new(KeyCode::Down, KeyModifiers::NONE)), + Action::None + )); + assert_eq!(app.editor.text(), "unsent draft"); + } + + #[test] + fn failed_recovery_preserves_active_modified_steer_and_ordinary_draft() { + let mut app = app(); + active_modified_steer(&mut app); + let before = text(&app); + app.apply(Update::GatewayRecovery(event(Kind::Begin, Vec::new()))); + assert_both_drafts_recallable(&mut app); + app.apply(Update::GatewayRecovery(event(Kind::Failed, Vec::new()))); + assert!(app.gateway_blocked); + assert_eq!(text(&app), before); + assert_eq!(app.pending_steers[0].text, "queued steering"); + assert_both_drafts_recallable(&mut app); + } + + #[test] + fn committed_recovery_preserves_active_modified_steer_without_old_target() { + for running in [false, true] { + let mut app = app(); + active_modified_steer(&mut app); + app.apply(Update::GatewayRecovery(event(Kind::Begin, Vec::new()))); + // Even if the same target survives replay, the unsent edit must not + // retain replacement authority or be automatically submitted. + replay( + &mut app, + Update::SteerAccepted { + editable: true, + id: "old-target".into(), + text: "queued steering".into(), + }, + ); + snapshot(&mut app, running); + app.apply(Update::GatewayRecovery(event(Kind::Commit, Vec::new()))); + assert!(!app.gateway_blocked); + if running { + assert_eq!(app.pending_steers[0].text, "queued steering"); + } else { + assert!(app.pending_steers.is_empty()); + } + assert_both_drafts_recallable(&mut app); + app.handle_key(KeyEvent::new(KeyCode::Up, KeyModifiers::NONE)); + app.last_key = None; + assert!(matches!( + app.handle_key(KeyEvent::new(KeyCode::Enter, KeyModifiers::NONE)), + Action::Submit { prompt, .. } + if prompt.text == "queued steering modified but unsent" + )); + } + } + #[test] fn failed_and_invalid_replays_preserve_visible_transcript_and_draft() { for failure in [Kind::Failed, Kind::Commit] { diff --git a/src/tui/editor.rs b/src/tui/editor.rs index 012e90f..50ca404 100644 --- a/src/tui/editor.rs +++ b/src/tui/editor.rs @@ -163,12 +163,17 @@ impl Editor { self.cursor = 0; self.browsing = None; self.stashed.clear(); - if self.history.last().map(String::as_str) != Some(text.as_str()) { - self.history.push(text.clone()); - } + self.remember(text.clone()); text } + /// Keeps text available for manual recall without submitting or changing the draft. + pub fn remember(&mut self, text: String) { + if self.history.last() != Some(&text) { + self.history.push(text); + } + } + /// Drops the prompt without recording it in the history. pub fn clear(&mut self) { self.text.clear();