From 2cb7aa597536f2e4b1e0b2ddf8457ec76333ccea Mon Sep 17 00:00:00 2001 From: Harry Yep Date: Mon, 26 Oct 2026 20:28:05 +0800 Subject: [PATCH 01/13] feat(fusion): register host capacity and generic agent messages --- plugins/fusion/scripts/lib/user-messages.mjs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/plugins/fusion/scripts/lib/user-messages.mjs b/plugins/fusion/scripts/lib/user-messages.mjs index fbc5006..a1528df 100644 --- a/plugins/fusion/scripts/lib/user-messages.mjs +++ b/plugins/fusion/scripts/lib/user-messages.mjs @@ -12,6 +12,7 @@ const MESSAGE_REGISTRY = [ { slug: "inline-guard.no-op-heartbeat-deny", description: "Denial of a no-op Bash command while Fusion tasks are in flight." }, { slug: "inline-guard.reaped-worker-redirect", description: "Denial of a TaskOutput or TaskStop probe of a reaped worker with the result path." }, { slug: "inline-guard.narrow-wave-advisory", description: "Advisory after consecutive width one dispatch waves." }, + { slug: "inline-guard.generic-agent-advisory", description: "Advisory that a generic or forked Agent ran a work package outside Fusion tracking." }, { slug: "inline-guard.state-unavailable-deny", description: "Strict posture fail closed denial when guard state is unreadable." }, { slug: "worker-lifecycle.foreground-wrapper-deny", description: "Denial of background delivery for a peer wrapper Agent." }, { slug: "worker-lifecycle.lane-default-injection", description: "Notice naming routing table options filled for a peer wrapper request." }, @@ -20,6 +21,7 @@ const MESSAGE_REGISTRY = [ { slug: "worker-lifecycle.background-authorization-deny", description: "Denial of background execution without explicit user authorization." }, { slug: "worker-lifecycle.dispatch-advisory", description: "Dispatch time advisory on verification suites, sizing, or transcript size." }, { slug: "worker-lifecycle.launch-cap-advisory", description: "Notice after the harness refuses a subagent launch at its concurrency cap." }, + { slug: "worker-lifecycle.host-capacity-deny", description: "Denial of a peer launch while this host already runs its peer engine capacity." }, { slug: "worker-lifecycle.final-deliverable-allow", description: "Permission for a stopping worker's final deliverable write." }, { slug: "worker-lifecycle.worker-stop-deny", description: "Instruction that a stopping worker must return a concise partial result." }, { slug: "worker-lifecycle.budget-wind-down", description: "Wind down instruction as a worker approaches its token or turn budget." }, From b8c5aebb6ce72e333c18757bab3486a89d95b771 Mon Sep 17 00:00:00 2001 From: Harry Yep Date: Mon, 26 Oct 2026 20:31:30 +0800 Subject: [PATCH 02/13] feat(fusion): drop the jev mixed concern check, rules for host limits and cheaper waves --- bench/jev/briefs.json | 8 -------- plugins/fusion/rules-manifest.json | 1 + plugins/fusion/rules/orchestration.md | 10 +++++----- plugins/fusion/rules/troubleshooting.md | 8 ++++---- plugins/fusion/scripts/brief-sensor.mjs | 6 ------ tests/brief-sensor.test.mjs | 11 +++++------ tests/rules-sync.test.mjs | 3 ++- 7 files changed, 17 insertions(+), 30 deletions(-) diff --git a/bench/jev/briefs.json b/bench/jev/briefs.json index 52180a0..b5aacac 100644 --- a/bench/jev/briefs.json +++ b/bench/jev/briefs.json @@ -1,12 +1,4 @@ [ - { - "label": "mixedConcern", - "brief": "lane: codex | verification: npm test\nThe login test has been flaky since last week and nobody knows why. Figure out what is causing it and fix it. Files are somewhere under src/auth. Done when npm test passes ten times in a row." - }, - { - "label": "mixedConcern", - "brief": "lane: codex | verification: node --test tests/stats.test.mjs\nfusion-stats sometimes double counts grok jobs. Investigate the root cause, then implement the fix and add a regression test in tests/stats.test.mjs." - }, { "label": "tasteAsProse", "brief": "lane: codex | verification: npm run build\nRewrite the error messages in src/cli/errors.ts so they feel friendly, elegant, and on brand. Make the naming cleaner and more idiomatic. Done when npm run build passes." diff --git a/plugins/fusion/rules-manifest.json b/plugins/fusion/rules-manifest.json index 7dc1c85..169acae 100644 --- a/plugins/fusion/rules-manifest.json +++ b/plugins/fusion/rules-manifest.json @@ -41,6 +41,7 @@ "b3649bf8c8fb1f3ede0ccfc7fb727a8adf1e13628a18f4a84d08efbee75bd934", "b41dfd4862a69bb58ee79761b8f7055c26433b337156d001665176b3c77fb948", "b5283c0356c9004385157f3360ca81a3601253e370216b390049d069a29a2250", + "bc88b749a510ca4683f9f7df05ff4b28d75966d5daa00c7e9f46307fbdd1faa3", "c29808811e61113fd7861f5e973354c18d59341645e3fb5217fa89a81c5cd3da", "c435cc1486eeff00f5b3a1c76017d46d8f05d30ebd248435d575a1461d7da5d4", "c5cb1e9d17cd2771e0f320e760a249188d69658ef3a63b31556000a64fed88e1", diff --git a/plugins/fusion/rules/orchestration.md b/plugins/fusion/rules/orchestration.md index 0de7eb6..c084333 100644 --- a/plugins/fusion/rules/orchestration.md +++ b/plugins/fusion/rules/orchestration.md @@ -71,7 +71,7 @@ Scores are user-assigned via /fusion:config; re-score when the model lineup chan Scores feed the routing priorities above: intelligence proxies correctness and safety, taste is user facing quality, cost applies only as the final tie breaker. Scores are user-assigned via /fusion:config; re-score when the model lineup changes. Treat /fusion:config and live model listings as authoritative over any example model names in this document; a live model without a scored table row holds no routing seat, and `gpt-reserve` is never a candidate. -Dispatch gate, checked once before any dispatch: decide whether the message continues the active goal or starts a new one, apply the current posture, and only inside coordinate or a fresh goal's triage does the question versus change classification apply (a question or problem description is answered or diagnosed from the main loop with read only tools; a requested change proceeds). A requested change, a second inline edit for the same goal, a repeated runtime error, or a failed verification enters implement as soon as a verification command can be written. If one cannot be written yet, neither dispatch nor edit; continue triage or reconnaissance instead. Fan out independent packages in a single message rather than queueing them one at a time; width two is already a wave, and at three or more independent packages, convene the fan out through /fusion:ultra rather than hand rolling it. One message never launches more Agent calls than 20 minus the tasks still in flight (`CLAUDE_CODE_MAX_CONCURRENT_SUBAGENTS`); a refused launch is settled by the lifecycle as an infrastructure death and relaunched after a completion notification. +Dispatch gate, checked once before any dispatch: decide whether the message continues the active goal or starts a new one, apply the current posture, and only inside coordinate or a fresh goal's triage does the question versus change classification apply (a question or problem description is answered or diagnosed from the main loop with read only tools; a requested change proceeds). A requested change, a second inline edit for the same goal, a repeated runtime error, or a failed verification enters implement as soon as a verification command can be written. If one cannot be written yet, neither dispatch nor edit; continue triage or reconnaissance instead. Fan out independent packages in a single message rather than queueing them one at a time; width two is already a wave, and at three or more independent packages, convene the fan out through /fusion:ultra rather than hand rolling it. One message never launches more Agent calls than 20 minus the tasks still in flight (`CLAUDE_CODE_MAX_CONCURRENT_SUBAGENTS`); a refused launch is settled by the lifecycle as an infrastructure death and relaunched after a completion notification. Peer engines build and test on this host, so Fusion also denies a peer launch while every session together already runs the host's peer capacity (one peer per 3 GiB of memory, at least two); relaunch after a completion notification. A constraint the user states about the host, such as no local builds, rides in every peer brief as a MUST NOT line. Codex admission follows one ladder: use the current workspace when its Codex slot is free; otherwise use an isolated Codex worktree for an independent, worktree eligible package; otherwise route an independent package that fits Grok's or Cursor's safety and turn boundaries under that engine's `burst` role, choosing between them by breaker health and recorded acceptance; otherwise route generic overflow to the Codex volume tiers and reserve `fusion:claude-worker` for Claude-surface work or structurally stranded packages with a stated `claude-fallback: ` header. Worktree eligible means disjoint files and verification that needs no heavy dependency setup. Packages that overlap files, share generated state, require ordering, or cannot be merged independently never overflow in parallel merely by changing engines; consolidate or sequence them. Before dispatching Codex into a workspace, confirm its single-flight slot is free; a dispatch bounced with an already-running error is a routing error, not a retry candidate. @@ -126,8 +126,8 @@ Codex owns ordinary delegated implementation by default. Grok is a complementary Peers are executors with model aware lanes: live model listings and /fusion:config are authoritative for the model and effort inside a lane. Capability scores never change lane ownership by themselves. Changing primary ownership requires an explicit routing policy edit. A brief names the effort or model when correctness or latency warrants it. - codex rewards spec grade briefs: completion criteria, an output contract, boundaries, a verification command, and an explicit requirement to return the real result in the same turn. Codex briefs stay lean and gate shaped: hard constraints ride as MUST NOT lines and checkable contracts (exemplar files, signatures, named tests) rather than accumulated prose, and style or architecture intent expressed only as prose is undeliverable to this lane. Pass the prompt directly to `codex:codex-rescue`; do not create a transport payload with Write. Every Codex rescue and companion call stays foreground; only an incoming user request that already includes `--background` may detach Codex, and a receipt from a job created by Fusion orchestration requires one same turn bounded collection attempt through fusion:job-collector. The Codex lane is single flight per canonical workspace; three or more quick packages touching the same subsystem with long horizon shape consolidate into one bounded spec grade brief or split only where independent, without lowering the fleet decision's count. For peer implementation packages, verification confirms that tests were not deleted, skipped, or weakened to make the package pass. Codex live web search requires explicit `--web`; workspace network access additionally requires `--write --network`. If Codex is unavailable, a bounded, isolated implementation package may move to Grok under `burst`. -- grok is the complementary specialist and burst lane: `/grok:task --write` for bounded implementation under `burst` or `independence`, read only consult with `--web` under `live-web`, and read only file analysis under `large-context`. A Grok burst dispatch passes `--effort low` as a leading companion option, and the brief header line repeats it. Automatically routed Grok briefs never enable cross-session memory and do not receive configured automatic continuity affinity; an explicit `--resume` or `--resume-last` is only for a sequential follow up to the same package. Consult, write, and review all use `--sandbox strict`, with no silent downgrade to workspace. Design decisions stay out of Grok write briefs under the judgment dense authorship rule. Sandbox, tool allowlist, credential exposure, version interval, runtime socket, quota cooldown, and authentication refresh details are in the troubleshooting rules. -- cursor is the multi family breadth lane. Every automatically routed Cursor brief's header line includes `cursor-role: `, where `` is exactly `diversity` or `burst`; Fusion fills the role's lane default model and effort, and a direct `/cursor:*` command falls back to the model in the user's own Cursor configuration. `diversity` is read only; pick its model by family, a family that is neither the orchestrator's nor any family already seated on the same question, so a second opinion never comes from the weights it is checking. `burst` writes only inside an orchestrator owned worktree. Never route Auto, because no event reports the model it chose. When a native lane's breaker is open on `quota` or `rate_limited`, a package whose load bearing capability is that family's weights, not its harness or tools, may run on Cursor's variant of the same family with every other requirement unchanged; live-web, X reads, and Claude surface work never substitute. Cursor quota and rate limit failures are scoped to the billing pool that failed (`first-party` for Composer, `api` for every other model). +- grok is the complementary specialist and burst lane: `/grok:task --write` for bounded implementation under `burst` or `independence`, read only consult with `--web` under `live-web`, and read only file analysis under `large-context`. A Grok burst dispatch passes `--effort low` as a leading companion option, and the brief header line repeats it. Automatically routed Grok briefs never enable cross-session memory and do not receive configured automatic continuity affinity; an explicit `--resume` or `--resume-last` is only for a sequential follow up to the same package. Consult, write, and review all run in the companion's strict sandbox, which is not a companion option to pass, with no silent downgrade to workspace. Design decisions stay out of Grok write briefs under the judgment dense authorship rule. Sandbox, tool allowlist, credential exposure, version interval, runtime socket, quota cooldown, and authentication refresh details are in the troubleshooting rules. +- cursor is the multi family breadth lane. Every automatically routed Cursor brief's header line includes `cursor-role: `, where `` is exactly `diversity` or `burst`; Fusion fills the role's lane default model and effort, and a direct `/cursor:*` command falls back to the model in the user's own Cursor configuration. `diversity` is read only; pick its model by family, a family that is neither the orchestrator's nor any family already seated on the same question, so a second opinion never comes from the weights it is checking. `burst` writes only inside an orchestrator owned worktree. Never route Auto, because no event reports the model it chose. When a native lane's breaker is open on `quota` or `rate_limited`, a package whose load bearing capability is that family's weights, not its harness or tools, may run on Cursor's variant of the same family with every other requirement unchanged; live-web, X reads, and Claude surface work never substitute. Cursor quota and rate limit failures are scoped to the billing pool that failed (`first-party` for Composer, `api` for every other model). Anything a Cursor package reads, such as a diff under review, lives inside its workspace, because Cursor's sandbox cannot read the Claude scratchpad. - Cross engine review by default: the other peer or fusion:deep-reasoner reviews a substantial package before merge; /codex:adversarial-review challenges a design, /grok:review is the fast pass. - A plan with three or more independent packages uses every lane whose ownership or protected role fits, splitting proportionally rather than queueing generic work away from the Codex volume tiers; `fusion:claude-worker` only receives Claude-surface or structurally stranded packages. This proportional split is exactly what /fusion:ultra performs. Idle capacity alone never manufactures a Grok role. Multi source research fans out one track to a peer by default. - Balance check: on a delegating project, peer lanes carry a meaningful share of implementation output. A session where Claude worker lanes absorb spec grade or quick scoped packages while Codex sits idle is a routing defect. After two eligible packages have gone to Claude workers while Codex sits idle, route the next ordinary implementation package to Codex. Route to idle Grok only when a protected role fits and its circuit breaker is closed. @@ -161,9 +161,9 @@ Every brief is self contained: goal, constraints, relevant paths, and what done - Before a fleet wave over an uncommitted baseline, the orchestrator records a checkpoint so isolated worktrees fork from the accumulated state; when no commit is authorized, the baseline rides as an applied and staged patch in each worktree and the package delta is exactly the unstaged diff. A worktree isolated peer package is collected by merging its diff back into the main tree and re-running its verification command there before the package counts as done. - Independent packages and independent sections of one task are decomposed and dispatched in a single message; sequential dispatch of independent work is a defect. Repeated width one dispatch turns on a goal with multiple independent packages are the signature of that defect, and the narrow wave watch in `/fusion:stats` surfaces it. - Every ordinary Agent is foreground delivery. The harness launches Agent calls asynchronously by default, but a launch receipt is not delivery: completion arrives as a task notification, which arms mandatory collection. Launch independent Agents together in one tool message for concurrency, but do not set `run_in_background` and do not accept an async launch receipt as delivery. A width one Claude worker dispatch may instead pass `run_in_background: false` so the deliverable returns synchronously inside the tool result. Agent `run_in_background` is allowed only for `grok:grok-review-runner` and for a Fusion worker when the latest user message explicitly contains `--background`. Codex, Grok, and Cursor wrapper Agents always remain foreground; only an incoming user request that explicitly contains companion `--background` lets the companion child detach and return a manual receipt. Fusion records every Claude worker as an owned task: the Stop hook advises while owned work is in flight and blocks completion until collection and settlement are recorded, releasing after repeated identical blocks, and `TaskStop` is reserved for an expired lifecycle budget. The main loop never runs a work package or launches a detached shell. -- Heartbeat rule: a wakeup with delegated work in flight emits one short user visible status line naming what is still in flight and when the next check happens, and otherwise takes no tool action. A silent wakeup and a hand rolled Bash or ScheduleWakeup polling loop used in place of fusion:job-collector are defects. +- Heartbeat rule: a wakeup with delegated work in flight emits one short user visible status line naming what is still in flight and when the next check happens, and otherwise takes no tool action. A silent wakeup and a hand rolled Bash or ScheduleWakeup polling loop used in place of fusion:job-collector are defects. Waiting on external state such as CI or a deploy runs as one background command or Monitor that notifies on change, never as repeated sleep and status turns, because every main loop turn rereads the whole context. - Same turn collection attempt: this obligation applies to a manual receipt that crosses into the main session from a detached job launched by Fusion orchestration. It is collected in the same turn through fusion:job-collector with a closed direct prompt containing exactly one standalone `engine: codex|grok|cursor` line and one standalone `job: <32 lowercase hexadecimal characters>` line. Its single collection window is at most 540000ms. A timeout, dead job, or status error leaves the package uncollected and must reach the user with the Fusion task id, peer job id, the literal state `uncollected`, and the exact `/codex:result `, `/grok:result `, or `/cursor:result ` command. A foreground Codex timeout with a resumable thread is salvaged inside the rescue wrapper itself by one `wind-down` companion call; a second timeout terminalizes the package as a sizing error. The clocks differ: the Bash tool caps a foreground call at 600 seconds, the Codex engine flight is 570 seconds, and the collector window is 540 seconds. -- Transport completion is not semantic acceptance. The main session verifies every peer and Claude worker result before relying on it. Settlement is batched per wave: collect and verify every result in a wave, then record all verdicts in one `/fusion:stats --record` call with multiple `=` pairs, where each id is the fusion task id or the engine job id. Per package settlement turns are reserved for waves of one. One settlement validates every pair before writing and is idempotent to rerun after a partial transport failure; it writes the worker ledger and linked engine record for every recorded verdict. Strict record forms may pass as direct arguments, and anything carrying free text still uses the raw-args transport. Record accepted only when the verification command or explicit acceptance criteria pass. A rejected verdict carries `--reason` and, when the cause is judgment shaped, a `--failure-kind` of `intent_override`, `scope_rewrite`, `wrong_approach`, or `style_mismatch`; a wind down meta report or an empty delta is `--failure-kind oversized`, a sizing error to fix at the brief. Leave a result `unverified` only when no reliable judgment was possible, and never past the goal's completion. Infrastructure deaths are settled by the lifecycle and stay out of judged acceptance. Never infer acceptance from `state: done`, `delivery: complete`, or any transport status. A review job whose findings feed a follow-up dispatch is settled like any other package. +- Transport completion is not semantic acceptance. The main session verifies every peer and Claude worker result before relying on it. Settlement is batched per wave: collect and verify every result in a wave, then record all verdicts in one `/fusion:stats --record` call with multiple `=` pairs, where each id is the fusion task id or the engine job id. Chain that settlement behind the wave's verification in the same Bash call (` && `), so a passing wave costs one turn. Per package settlement turns are reserved for waves of one. One settlement validates every pair before writing and is idempotent to rerun after a partial transport failure; it writes the worker ledger and linked engine record for every recorded verdict. Strict record forms may pass as direct arguments, and anything carrying free text still uses the raw-args transport. Record accepted only when the verification command or explicit acceptance criteria pass. A rejected verdict carries `--reason` and, when the cause is judgment shaped, a `--failure-kind` of `intent_override`, `scope_rewrite`, `wrong_approach`, or `style_mismatch`; a wind down meta report or an empty delta is `--failure-kind oversized`, a sizing error to fix at the brief. Leave a result `unverified` only when no reliable judgment was possible, and never past the goal's completion. Infrastructure deaths are settled by the lifecycle and stay out of judged acceptance. Never infer acceptance from `state: done`, `delivery: complete`, or any transport status. A review job whose findings feed a follow-up dispatch is settled like any other package. - A non deliverable final message starts an obligation, not an outcome: resume a truncated, non forwarder agent with SendMessage to finish and report with verification; for a background receipt created by Fusion, dispatch fusion:job-collector instead. - Related follow up packages reuse the existing worker thread via SendMessage instead of cold starting a new agent. A follow up sent to a peer wrapper restates the leading companion options (`--model`, `--effort`, `--cwd`) exactly as a fresh dispatch would, because the continuation composes a new engine call; a continued task reopens and is collected and settled like a fresh dispatch. - When any package in a wave terminates by timeout with partial output, rerun that package's verification plus one broad typecheck or test pass over the merged tree before accepting sibling packages that share the subsystem. diff --git a/plugins/fusion/rules/troubleshooting.md b/plugins/fusion/rules/troubleshooting.md index 8d56877..faa78cc 100644 --- a/plugins/fusion/rules/troubleshooting.md +++ b/plugins/fusion/rules/troubleshooting.md @@ -11,7 +11,7 @@ Failure driven implementation escalation among Claude workers runs only from `fu ## Failure kinds and the circuit breaker -The shared companion failure kinds are quota, auth, missing_cli, setup, rate_limited, timeout, stall, transport, sandbox, error, permission, cancelled, input, died, and resource. Codex additionally emits protocol when the required JSONL lifecycle is incompatible or malformed, and process for an unexpected process signal. Grok additionally emits network for a terminal API connectivity failure (DNS resolution, connection refused or reset) so a transient outage stops masquerading as a generic error. Codex, Grok, and Cursor emit policy for engine-specific enforced-boundary failures; Grok uses it when positive tool-policy evidence is missing or a fallback, unmappable, or unmatched warning appears. Grok also emits `turn_limit` when its CLI reaches the forwarded max turns value after the companion salvages the final envelope. A Claude session limit remains a Claude side event. The missing agent type and unavailable line are also observed routing conditions. For engines that do not emit the shared kinds, map an observed failure onto the closest companion failure kind before acting. +The shared companion failure kinds are quota, auth, missing_cli, setup, rate_limited, timeout, stall, transport, sandbox, error, permission, cancelled, input, died, and resource. Codex additionally emits protocol when the required JSONL lifecycle is incompatible or malformed, and process for an unexpected process signal. Codex and Grok additionally emit network for a terminal API connectivity failure (DNS resolution, connection refused or reset, and for Codex a workspace routing discovery timeout or a dropped stream) so a transient outage stops masquerading as a generic error. Codex, Grok, and Cursor emit policy for engine-specific enforced-boundary failures; Grok uses it when positive tool-policy evidence is missing or a fallback, unmappable, or unmatched warning appears. Grok also emits `turn_limit` when its CLI reaches the forwarded max turns value after the companion salvages the final envelope. A Claude session limit remains a Claude side event. The missing agent type and unavailable line are also observed routing conditions. For engines that do not emit the shared kinds, map an observed failure onto the closest companion failure kind before acting. | Failure kind or observed condition | Immediate action | Circuit breaker effect | |---|---|---| @@ -143,11 +143,11 @@ A model that appears in the live Codex listing without a scored table row holds ### Grok peer engagement detail -- grok is the complementary specialist and burst lane. Use `/grok:task --write` for bounded implementation under `burst` or `independence`; use read only consult with `--web` under `live-web`; use read only file analysis under `large-context`. Consult, write, and review all use `--sandbox strict`, with no silent downgrade to workspace. Consult exposes only file read, list, and search tools, plus web search and fetch when requested. Write uses a fixed list containing `read_file`, `grep`, `list_dir`, `search_replace`, and `run_terminal_cmd`; upstream 1.0.0 ships an OpenCode-compatible `write` tool id, but the fixed allowlist keeps it and every other new default tool out of managed runs, and `search_replace` creates new files. Command-pattern denies for common direct grok, claude, and codex calls do not cover absolute paths, aliases or functions, or indirect scripts, so they are not a hard nested-execution boundary while `run_terminal_cmd` remains enabled. Hard prevention requires removing that tool or an OS-level executable or network policy. Every ordinary call uses bare `--disallowed-tools Agent`; every mode also denies `search_tool`, `use_tool`, `ask_user_question`, and MCP tools. Upstream parses `--no-subagents`, but the single-turn and agent resolvers do not forward it, while the interactive TUI does apply it; the hard Agent tool deny and `GROK_SUBAGENTS=0` are the effective headless controls. The child sets all eighteen `GROK_CLAUDE_*_ENABLED`, `GROK_CURSOR_*_ENABLED`, and `GROK_CODEX_*_ENABLED` bridge variables to false; upstream currently consumes the six Claude and six Cursor cells plus the Codex sessions cell and reserves the other five Codex cells, and the child also pins `GROK_MANAGED_MCPS_ENABLED=false` because that variable has highest precedence over `[managed_mcps]` in `~/.grok/config.toml` and remote settings. It removes inherited Claude plugin and companion variables plus `_GROK_CLAUDE_MARKER_OVERRIDE`, broadly scrubs secrets while preserving xAI authentication, and disables automatic updates with `--no-auto-update`. Cross-session memory is forced off unless a direct user-selected ordinary task explicitly passes `--memory`; automatically routed briefs, review, and stop gate cannot enable it. Grok enforces hard version requirements at startup regardless of `--no-auto-update` (introduced in 0.2.112, verified through 1.0.41); that death is classified `setup` and remains a residual boundary. The grok companion also carries a tested interval like the codex companion: a build below 1.0.14 fails managed runs closed as `setup` because the event log contract does not hold there, and a build at or past 1.0.42 makes `/grok:setup` report it as newer than the tested interval with a verification pass advised, which is the signal to run /fusion:smoke before trusting the lane after an auto update. Since the runtime-socket deny policy (introduced between 1.0.4 and 1.0.13, still present at 1.0.41), strict refuses to start while `/var/run/docker.sock` is a symlink, which is the resting state of macOS Docker engines such as OrbStack; the companion preflights this and fails the dispatch as `sandbox` in seconds, and the repair is host side (for OrbStack `orb config set setup.use_admin false` and removing the link, for Docker Desktop disabling the default socket), never a sandbox downgrade. OrbStack's non-admin socket under `~/.orbstack/run` is outside upstream's deny list and the macOS strict profile allows network-outbound, so a strict write shell can drive that engine (verified at 1.0.41); the companion refuses write runs as `sandbox` while the socket exists, consult and review are unaffected, and the repair is quitting the engine or routing the package to Codex, whose sandbox blocks the connect. Strict cannot write the Grok token store since 1.0.14, so the companion refreshes a stored OIDC token older than five hours with one `grok models` call outside the sandbox under the managed child environment before a managed run; a run that still fails as `auth` gets one refresh and one rerun, and an `auth` failure after that means the refresh token itself expired and needs `grok login`, never a sandbox downgrade. Every managed run preflights `--prompt-file`, `--output-format`, `--sandbox`, `--tools`, `--disallowed-tools`, `--deny`, `--max-turns`, and `--no-auto-update`; consult adds `--permission-mode` and `--allow`, no-web adds `--disable-web-search`, write adds `--always-approve`, review adds `--json-schema`, and an ordinary run requires `--no-wait-for-background` or the bounded wait flag. Forced builder tracing initializes after stdin is consumed, rejects fallback or unmatched warnings as soon as observed, and rejects a successful close without positive tool allowlist applied evidence; it is not a pre-prompt or pre-side-effect attestation. Codex also exposes opt in web search, but Grok remains the preferred live research lane. Design decisions stay out of Grok write briefs under the judgment dense authorship rule; reopening that requires a routing policy edit backed by a style fidelity bench result, not a capability score alone. +- grok is the complementary specialist and burst lane. Use `/grok:task --write` for bounded implementation under `burst` or `independence`; use read only consult with `--web` under `live-web`; use read only file analysis under `large-context`. Consult, write, and review all run in the companion's strict sandbox, which is not a companion option to pass, with no silent downgrade to workspace. Consult exposes only file read, list, and search tools, plus web search and fetch when requested. Write uses a fixed list containing `read_file`, `grep`, `list_dir`, `search_replace`, and `run_terminal_cmd`; upstream 1.0.0 ships an OpenCode-compatible `write` tool id, but the fixed allowlist keeps it and every other new default tool out of managed runs, and `search_replace` creates new files. Command-pattern denies for common direct grok, claude, and codex calls do not cover absolute paths, aliases or functions, or indirect scripts, so they are not a hard nested-execution boundary while `run_terminal_cmd` remains enabled. Hard prevention requires removing that tool or an OS-level executable or network policy. Every ordinary call uses bare `--disallowed-tools Agent`; every mode also denies `search_tool`, `use_tool`, `ask_user_question`, and MCP tools. Upstream parses `--no-subagents`, but the single-turn and agent resolvers do not forward it, while the interactive TUI does apply it; the hard Agent tool deny and `GROK_SUBAGENTS=0` are the effective headless controls. The child sets all eighteen `GROK_CLAUDE_*_ENABLED`, `GROK_CURSOR_*_ENABLED`, and `GROK_CODEX_*_ENABLED` bridge variables to false; upstream currently consumes the six Claude and six Cursor cells plus the Codex sessions cell and reserves the other five Codex cells, and the child also pins `GROK_MANAGED_MCPS_ENABLED=false` because that variable has highest precedence over `[managed_mcps]` in `~/.grok/config.toml` and remote settings. It removes inherited Claude plugin and companion variables plus `_GROK_CLAUDE_MARKER_OVERRIDE`, broadly scrubs secrets while preserving xAI authentication, and disables automatic updates with `--no-auto-update`. Cross-session memory is forced off unless a direct user-selected ordinary task explicitly passes `--memory`; automatically routed briefs, review, and stop gate cannot enable it. Grok enforces hard version requirements at startup regardless of `--no-auto-update` (introduced in 0.2.112, verified through 1.0.41); that death is classified `setup` and remains a residual boundary. The grok companion also carries a tested interval like the codex companion: a build below 1.0.14 fails managed runs closed as `setup` because the event log contract does not hold there, and a build at or past 1.0.42 makes `/grok:setup` report it as newer than the tested interval with a verification pass advised, which is the signal to run /fusion:smoke before trusting the lane after an auto update. Since the runtime-socket deny policy (introduced between 1.0.4 and 1.0.13, still present at 1.0.41), strict refuses to start while `/var/run/docker.sock` is a symlink, which is the resting state of macOS Docker engines such as OrbStack; the companion preflights this and fails the dispatch as `sandbox` in seconds, and the repair is host side (for OrbStack `orb config set setup.use_admin false` and removing the link, for Docker Desktop disabling the default socket), never a sandbox downgrade. OrbStack's non-admin socket under `~/.orbstack/run` is outside upstream's deny list and the macOS strict profile allows network-outbound, so a strict write shell can drive that engine (verified at 1.0.41); the companion refuses write runs as `sandbox` while the socket exists, consult and review are unaffected, and the repair is quitting the engine or routing the package to Codex, whose sandbox blocks the connect. Strict cannot write the Grok token store since 1.0.14, so the companion refreshes a stored OIDC token older than five hours with one `grok models` call outside the sandbox under the managed child environment before a managed run; a run that still fails as `auth` gets one refresh and one rerun, and an `auth` failure after that means the refresh token itself expired and needs `grok login`, never a sandbox downgrade. Every managed run preflights `--prompt-file`, `--output-format`, `--sandbox`, `--tools`, `--disallowed-tools`, `--deny`, `--max-turns`, and `--no-auto-update`; consult adds `--permission-mode` and `--allow`, no-web adds `--disable-web-search`, write adds `--always-approve`, review adds `--json-schema`, and an ordinary run requires `--no-wait-for-background` or the bounded wait flag. Forced builder tracing initializes after stdin is consumed, rejects fallback or unmatched warnings as soon as observed, and rejects a successful close without positive tool allowlist applied evidence; it is not a pre-prompt or pre-side-effect attestation. Codex also exposes opt in web search, but Grok remains the preferred live research lane. Design decisions stay out of Grok write briefs under the judgment dense authorship rule; reopening that requires a routing policy edit backed by a style fidelity bench result, not a capability score alone. ### Cursor peer engagement detail -- cursor is the multi family breadth lane. Every automatically routed Cursor brief's header line includes `cursor-role: `, where `` is exactly `diversity` or `burst`; Fusion fills the role's lane default model and effort, and a direct `/cursor:*` command is a user selected override that falls back to the model in the user's own Cursor configuration. `diversity` is read only: Cursor's ask mode, a read only sandbox, and read, search, and lint tools only. Pick its model by family, not by score alone: a family that is neither the orchestrator's nor any family already seated on the same question, so a second opinion never comes from the weights it is checking. `burst` writes only inside an orchestrator owned worktree, with shell and edit tools confined to that workspace by Cursor's sandbox and no network. Every run executes under a private home that removes Claude Code, Codex, Grok, and Cursor user configuration from the session; project instructions, project hooks, and the account's Cursor user rules still reach it, as the Cursor companion contract in the plugin repository documents. Never route Auto, because no event reports the model it chose. When a native lane's breaker is open on `quota` or `rate_limited`, a package whose load bearing capability is that family's weights, not its harness or tools, may run on Cursor's variant of the same family with every other requirement unchanged; live-web, X reads, and Claude surface work never substitute. Cursor quota and rate limit failures are scoped to the billing pool that failed (`first-party` for Composer, `api` for every other model), so an exhausted API pool leaves the Composer burst lane open. +- cursor is the multi family breadth lane. Every automatically routed Cursor brief's header line includes `cursor-role: `, where `` is exactly `diversity` or `burst`; Fusion fills the role's lane default model and effort, and a direct `/cursor:*` command is a user selected override that falls back to the model in the user's own Cursor configuration. `diversity` is read only: Cursor's ask mode, a read only sandbox, and read, search, and lint tools only. Pick its model by family, not by score alone: a family that is neither the orchestrator's nor any family already seated on the same question, so a second opinion never comes from the weights it is checking. `burst` writes only inside an orchestrator owned worktree, with shell and edit tools confined to that workspace by Cursor's sandbox and no network. Every run executes under a private home that removes Claude Code, Codex, Grok, and Cursor user configuration from the session; project instructions, project hooks, and the account's Cursor user rules still reach it, as the Cursor companion contract in the plugin repository documents. Never route Auto, because no event reports the model it chose. When a native lane's breaker is open on `quota` or `rate_limited`, a package whose load bearing capability is that family's weights, not its harness or tools, may run on Cursor's variant of the same family with every other requirement unchanged; live-web, X reads, and Claude surface work never substitute. Cursor quota and rate limit failures are scoped to the billing pool that failed (`first-party` for Composer, `api` for every other model), so an exhausted API pool leaves the Composer burst lane open. Anything a Cursor package reads, such as a diff under review, lives inside its workspace, because Cursor's sandbox cannot read the Claude scratchpad. ### Agent scheduling layers @@ -159,7 +159,7 @@ A model that appears in the live Codex listing without a scored table row holds ### Settlement, retention, and acceptance epoch -- Transport completion is not semantic acceptance. The main session verifies every peer and Claude worker result before relying on it. Settlement is batched per wave: collect and verify every result in a wave, then record all verdicts in one `/fusion:stats --record` call with multiple `=` pairs, where each id is the fusion task id or the engine job id. Per package settlement turns are reserved for waves of one. One settlement validates every pair before writing and is idempotent to rerun after a partial transport failure; it writes the worker ledger and linked engine record for every recorded verdict. Strict record forms may pass as direct arguments, and anything carrying free text still uses the raw-args transport. Record accepted only when the verification command or explicit acceptance criteria pass. Record `rejected` when the collected result or verification fails. A rejected verdict carries `--reason` and, when the cause is judgment shaped rather than mechanical, a `--failure-kind` of `intent_override`, `scope_rewrite`, `wrong_approach`, or `style_mismatch`, so intent and taste failures become measurable instead of living only in free text. A delivery that is only a wind down meta report or an empty delta traces to package sizing, not worker judgment: record it with `--failure-kind oversized` and resize or split the brief, never `scope_rewrite`. Leave it `unverified` only when no reliable judgment was possible. Infrastructure deaths, a launch the harness refused and a wrapper that died on an API error before any engine job ran, are settled by the lifecycle, need no verdict, and stay out of judged acceptance. Every collected result is settled in the wave in which it is collected. `unverified` is temporary and must not survive the goal's completion. Stats default every terminal job without a recorded judgment to `unverified`; never infer acceptance from `state: done`, `delivery: complete`, or any transport status. The worker ledger is not kept forever: terminal records and their sidecars expire after `FUSION_WORKER_RETENTION_DAYS`, ninety days by default and zero to keep everything, swept when the next task is created, while a record whose transport is still live is never removed. +- Transport completion is not semantic acceptance. The main session verifies every peer and Claude worker result before relying on it. Settlement is batched per wave: collect and verify every result in a wave, then record all verdicts in one `/fusion:stats --record` call with multiple `=` pairs, where each id is the fusion task id or the engine job id. Chain that settlement behind the wave's verification in the same Bash call (` && `), so a passing wave costs one turn. Per package settlement turns are reserved for waves of one. One settlement validates every pair before writing and is idempotent to rerun after a partial transport failure; it writes the worker ledger and linked engine record for every recorded verdict. Strict record forms may pass as direct arguments, and anything carrying free text still uses the raw-args transport. Record accepted only when the verification command or explicit acceptance criteria pass. Record `rejected` when the collected result or verification fails. A rejected verdict carries `--reason` and, when the cause is judgment shaped rather than mechanical, a `--failure-kind` of `intent_override`, `scope_rewrite`, `wrong_approach`, or `style_mismatch`, so intent and taste failures become measurable instead of living only in free text. A delivery that is only a wind down meta report or an empty delta traces to package sizing, not worker judgment: record it with `--failure-kind oversized` and resize or split the brief, never `scope_rewrite`. Leave it `unverified` only when no reliable judgment was possible. Infrastructure deaths, a launch the harness refused and a wrapper that died on an API error before any engine job ran, are settled by the lifecycle, need no verdict, and stay out of judged acceptance. Every collected result is settled in the wave in which it is collected. `unverified` is temporary and must not survive the goal's completion. Stats default every terminal job without a recorded judgment to `unverified`; never infer acceptance from `state: done`, `delivery: complete`, or any transport status. The worker ledger is not kept forever: terminal records and their sidecars expire after `FUSION_WORKER_RETENTION_DAYS`, ninety days by default and zero to keep everything, swept when the next task is created, while a record whose transport is still live is never removed. ### Worktree collection and engine created worktrees diff --git a/plugins/fusion/scripts/brief-sensor.mjs b/plugins/fusion/scripts/brief-sensor.mjs index 31dc46e..6dd7eec 100644 --- a/plugins/fusion/scripts/brief-sensor.mjs +++ b/plugins/fusion/scripts/brief-sensor.mjs @@ -9,12 +9,6 @@ import { tagMessage } from "./lib/user-messages.mjs"; const BRIEF_MIN_CHARS = 200; const FLAG_THRESHOLD = 0.7; const CHECKS = [ - { - id: "mixedConcern", - instructions: "Does this brief ask the worker to both find the unknown cause of a problem and change code to fix it within the same package?", - finding: "It asks one package to find a cause and to fix it", - remedy: "Split it into a diagnosis dispatch and an implementation dispatch written from that result." - }, { id: "tasteAsProse", instructions: "Does this brief demand a style, tone, naming, or design quality without pointing to an exemplar file, a signature, a test, or another checkable constraint?", diff --git a/tests/brief-sensor.test.mjs b/tests/brief-sensor.test.mjs index a815906..b1c9624 100644 --- a/tests/brief-sensor.test.mjs +++ b/tests/brief-sensor.test.mjs @@ -10,7 +10,7 @@ import { messageTag } from "../plugins/fusion/scripts/lib/user-messages.mjs"; const repoRoot = path.join(import.meta.dirname, ".."); const script = path.join(repoRoot, "plugins", "fusion", "scripts", "brief-sensor.mjs"); -const checkIds = ["mixedConcern", "tasteAsProse", "noDoneCriteria", "unbounded"]; +const checkIds = ["tasteAsProse", "noDoneCriteria", "unbounded"]; function closeServer(server) { return new Promise((resolve, reject) => server.close((error) => error ? reject(error) : resolve())); @@ -96,11 +96,11 @@ test("briefAdvisory ignores absent, low, and non-finite answers", () => { assert.strictEqual(briefAdvisory(null), null); assert.strictEqual(briefAdvisory({}), null); assert.strictEqual(briefAdvisory(answerSet(0.69)), null); - assert.strictEqual(briefAdvisory({ mixedConcern: { noul: Number.NaN }, unbounded: { noul: Infinity } }), null); + assert.strictEqual(briefAdvisory({ tasteAsProse: { noul: Number.NaN }, unbounded: { noul: Infinity } }), null); }); test("briefAdvisory flags its threshold and retains its message tag", () => { - const advisory = briefAdvisory({ mixedConcern: { noul: 0.7 } }); + const advisory = briefAdvisory({ tasteAsProse: { noul: 0.7 } }); assert.ok(advisory.startsWith("Jev flagged this peer brief.")); assert.ok(advisory.includes("(p=0.70)")); @@ -108,10 +108,9 @@ test("briefAdvisory flags its threshold and retains its message tag", () => { }); test("briefAdvisory preserves check order and excludes clean findings", () => { - const advisory = briefAdvisory({ mixedConcern: { noul: 0.93 }, tasteAsProse: { noul: 0.05 }, noDoneCriteria: { noul: 0.05 }, unbounded: { noul: 0.91 } }); + const advisory = briefAdvisory({ tasteAsProse: { noul: 0.93 }, noDoneCriteria: { noul: 0.05 }, unbounded: { noul: 0.91 } }); - assert.ok(advisory.indexOf("It asks one package to find a cause and to fix it") < advisory.indexOf("It asks for a sweep without named paths")); - assert.ok(!advisory.includes("It carries style or design intent only as prose")); + assert.ok(advisory.indexOf("It carries style or design intent only as prose") < advisory.indexOf("It asks for a sweep without named paths")); assert.ok(!advisory.includes("It names no checkable completion condition")); }); diff --git a/tests/rules-sync.test.mjs b/tests/rules-sync.test.mjs index cc2757c..b18cab3 100644 --- a/tests/rules-sync.test.mjs +++ b/tests/rules-sync.test.mjs @@ -303,7 +303,8 @@ test("Grok rules document source verified headless boundaries", () => { assert.ok(text.includes(subagentSentence), "Grok headless subagent wording must stay synchronized"); } - assert.match(rules, /--sandbox strict/); + assert.match(rules, /strict sandbox, which is not a companion option to pass/); + assert.doesNotMatch(rules, /use `--sandbox strict`/); assert.match(rules, /no silent downgrade to workspace/); assert.match(rules, /troubleshooting rules/); From 4e48d4fa2f27d7b22cc9af6bd4ea2eb016d108b5 Mon Sep 17 00:00:00 2001 From: Harry Yep Date: Mon, 26 Oct 2026 20:33:42 +0800 Subject: [PATCH 03/13] fix(fusion): settle engine verdicts without a workspace cwd, one salvage rule --- .../fusion/scripts/lib/engine-acceptance.mjs | 84 ++++++++++++++--- .../scripts/lib/lifecycle/collection.mjs | 5 +- .../fusion/scripts/lib/stats/settlement.mjs | 12 +-- tests/engine-acceptance.test.mjs | 93 +++++++++++++++++++ 4 files changed, 174 insertions(+), 20 deletions(-) create mode 100644 tests/engine-acceptance.test.mjs diff --git a/plugins/fusion/scripts/lib/engine-acceptance.mjs b/plugins/fusion/scripts/lib/engine-acceptance.mjs index f881d40..72da57e 100644 --- a/plugins/fusion/scripts/lib/engine-acceptance.mjs +++ b/plugins/fusion/scripts/lib/engine-acceptance.mjs @@ -5,6 +5,8 @@ import path from "node:path"; import { fileURLToPath } from "node:url"; import { ENGINES, isEngineId } from "./engines.mjs"; +import { readEngineJobRecord } from "./engine-job-state.mjs"; +import { RESUMABLE_CODEX_FAILURE_KINDS } from "./stats/evidence.mjs"; export class EnginePluginUpgradeRequiredError extends Error { constructor(message) { @@ -34,6 +36,65 @@ function requireEngine(engine) { return ENGINES[engine]; } +const moduleDirectory = path.dirname(fileURLToPath(import.meta.url)); + +function fusionVersion() { + const pluginRoot = path.resolve(moduleDirectory, "..", ".."); + if (path.basename(pluginRoot) !== "fusion") { + return path.basename(pluginRoot); + } + try { + return JSON.parse(fs.readFileSync(path.join(pluginRoot, ".claude-plugin", "plugin.json"), "utf8")).version; + } catch { + return null; + } +} + +function semanticVersion(version) { + const match = version.match(/^(\d+)\.(\d+)\.(\d+)(?:-([0-9A-Za-z.-]+))?(?:\+[0-9A-Za-z.-]+)?$/); + return match ? { core: match.slice(1, 4).map(Number), prerelease: match[4]?.split(".") ?? [] } : null; +} + +function compareVersions(left, right) { + const a = semanticVersion(left); + const b = semanticVersion(right); + if (!a || !b) { + return a ? 1 : b ? -1 : 0; + } + for (let index = 0; index < 3; index += 1) { + if (a.core[index] !== b.core[index]) { + return a.core[index] - b.core[index]; + } + } + if (a.prerelease.length === 0 || b.prerelease.length === 0) { + return Number(b.prerelease.length > 0) - Number(a.prerelease.length > 0); + } + for (let index = 0; index < Math.min(a.prerelease.length, b.prerelease.length); index += 1) { + const first = a.prerelease[index]; + const second = b.prerelease[index]; + const firstNumeric = /^\d+$/.test(first); + const secondNumeric = /^\d+$/.test(second); + if (firstNumeric && secondNumeric) { + const difference = Number(first) - Number(second); + if (difference) { + return difference; + } + } else if (firstNumeric !== secondNumeric) { + return firstNumeric ? -1 : 1; + } else if (first !== second) { + return first < second ? -1 : 1; + } + } + return a.prerelease.length - b.prerelease.length; +} + +export function engineAcceptanceTransport({ engine, jobId, acceptance, acceptFailedTransport = false, env = process.env }) { + const record = readEngineJobRecord(engine, jobId, env); + const failedTransport = record?.status === "error" || record?.transportStatus === "error"; + const salvage = failedTransport && RESUMABLE_CODEX_FAILURE_KINDS.has(record.failureKind); + return { failedTransport, salvage, acceptFailedTransport: acceptFailedTransport || (acceptance === "accepted" && salvage) }; +} + export function newestEngineCompanion(engine, env = process.env) { const { companionEnv, companionFile, id } = requireEngine(engine); const override = typeof env[companionEnv] === "string" ? env[companionEnv].trim() : ""; @@ -42,19 +103,20 @@ export function newestEngineCompanion(engine, env = process.env) { } const base = path.join(configuredHome(env), ".claude", "plugins", "cache", "claude-code-fusion", id); try { + const runningVersion = fusionVersion(); const candidates = fs .readdirSync(base) - .map((version) => path.join(base, version, "scripts", companionFile)) - .filter((candidate) => fs.existsSync(candidate)) - .map((candidate) => ({ candidate, mtime: fs.statSync(candidate).mtimeMs })) - .sort((left, right) => right.mtime - left.mtime || left.candidate.localeCompare(right.candidate)); + .map((version) => ({ version, candidate: path.join(base, version, "scripts", companionFile) })) + .filter(({ candidate }) => regularFile(candidate)) + .map((entry) => ({ ...entry, mtime: fs.statSync(entry.candidate).mtimeMs })) + .sort((left, right) => Number(right.version === runningVersion) - Number(left.version === runningVersion) || compareVersions(right.version, left.version) || right.mtime - left.mtime || left.candidate.localeCompare(right.candidate)); if (candidates.length > 0) { return candidates[0].candidate; } } catch { void 0; } - const sibling = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "..", "..", "..", id, "scripts", companionFile); + const sibling = path.resolve(moduleDirectory, "..", "..", "..", id, "scripts", companionFile); return fs.existsSync(sibling) ? sibling : null; } @@ -63,21 +125,21 @@ function companionFailureMessage(result) { } function acceptanceSubcommandUnavailable(result) { - if (result.error) { - return true; - } const message = companionFailureMessage(result); return /\b(?:unknown|unsupported|unrecognized)\s+(?:subcommand|command)\b/i.test(message) || /\brecord-acceptance\b.*\b(?:unknown|unsupported|unavailable|not found|not supported)\b/i.test(message) || /\b(?:cannot find module|ERR_MODULE_NOT_FOUND)\b/i.test(message); } -export function recordEngineAcceptance({ engine, jobId, acceptance, source, reason, failureKind, acceptFailedTransport = false, workspaceRoot, asJson = false, env = process.env, requireUpdate = true }) { +export function recordEngineAcceptance({ engine, jobId, acceptance, source, reason, failureKind, acceptFailedTransport = false, asJson = false, env = process.env, requireUpdate = true }) { const { displayName } = requireEngine(engine); const bin = newestEngineCompanion(engine, env); let updated = false; if (bin) { const argv = [bin, "record-acceptance", "--job-id", jobId, "--acceptance", acceptance, ...(source ? ["--source", source] : []), ...(reason ? ["--reason", reason] : []), ...(failureKind ? ["--failure-kind", failureKind] : []), ...(acceptFailedTransport ? ["--accept-failed-transport"] : []), ...(asJson ? ["--json"] : [])]; - const result = spawnSync(process.execPath, argv, { cwd: workspaceRoot, encoding: "utf8", env }); - if (!result.error && result.status === 0) { + const result = spawnSync(process.execPath, argv, { cwd: path.dirname(bin), encoding: "utf8", env }); + if (result.error) { + throw result.error; + } + if (result.status === 0) { updated = true; } else if (!acceptanceSubcommandUnavailable(result)) { throw new Error(companionFailureMessage(result) || `${displayName} acceptance record update failed.`); diff --git a/plugins/fusion/scripts/lib/lifecycle/collection.mjs b/plugins/fusion/scripts/lib/lifecycle/collection.mjs index 32fc764..6221105 100644 --- a/plugins/fusion/scripts/lib/lifecycle/collection.mjs +++ b/plugins/fusion/scripts/lib/lifecycle/collection.mjs @@ -1,7 +1,7 @@ import { PEER_RESCUE_AGENT_NAMES } from "../engines.mjs"; import fs from "node:fs"; import path from "node:path"; -import { recordEngineAcceptance } from "../engine-acceptance.mjs"; +import { engineAcceptanceTransport, recordEngineAcceptance } from "../engine-acceptance.mjs"; import { PEER_JOB_FOOTER_AGENTS, PEER_MANAGED_AGENTS, engineForAgentType, isEngineId } from "../engines.mjs"; import { readEngineJobFailureKind } from "../engine-job-state.mjs"; import { applyQueuedVerdict, backfillWorkerTaskOutputTelemetry, canonicalWorkerAgentType, isSettledWorker, WORKER_TERMINAL_STATUSES, markWorkerCollected, WORKER_COLLECTION_METHODS, workerRecordFile, writePrivateText } from "../worker-state.mjs"; @@ -321,6 +321,7 @@ function settleQueuedVerdict(record, now, env) { const errors = []; for (const jobId of peerJobIds) { try { + const transport = engineAcceptanceTransport({ engine: peerEngine, jobId, acceptance: settled.acceptance, acceptFailedTransport: pendingVerdict.acceptFailedTransport === true, env }); recordEngineAcceptance({ engine: peerEngine, jobId, @@ -328,7 +329,7 @@ function settleQueuedVerdict(record, now, env) { source: pendingVerdict.source, reason: settled.acceptanceReason, failureKind: settled.acceptanceFailureKind ?? null, - acceptFailedTransport: pendingVerdict.acceptFailedTransport === true, + acceptFailedTransport: transport.acceptFailedTransport, workspaceRoot: typeof settled.workspaceRoot === "string" ? settled.workspaceRoot : process.cwd(), env }); diff --git a/plugins/fusion/scripts/lib/stats/settlement.mjs b/plugins/fusion/scripts/lib/stats/settlement.mjs index fc10299..e427b43 100644 --- a/plugins/fusion/scripts/lib/stats/settlement.mjs +++ b/plugins/fusion/scripts/lib/stats/settlement.mjs @@ -1,9 +1,9 @@ -import { recordEngineAcceptance } from "../../lib/engine-acceptance.mjs"; +import { engineAcceptanceTransport, recordEngineAcceptance } from "../../lib/engine-acceptance.mjs"; import { readEngineJobRecord } from "../../lib/engine-job-state.mjs"; import { ENGINE_IDS, ENGINE_TERMINAL_STATUSES, engineDisplayName, engineDisplayNameList, engineStateRoots, isEngineId } from "../../lib/engines.mjs"; import { WORKER_TERMINAL_STATUSES, readWorkerRecord, readWorkerRecords, recordWorkerAcceptance, validateWorkerAcceptance } from "../../lib/worker-state.mjs"; import { FILE_ENGINE_DESCRIPTORS } from "./engine-evidence.mjs"; -import { ENGINE_JOB_ID_PATTERN, FUSION_TASK_ID_PATTERN, RESUMABLE_CODEX_FAILURE_KINDS, nonEmptyString } from "./evidence.mjs"; +import { ENGINE_JOB_ID_PATTERN, FUSION_TASK_ID_PATTERN, nonEmptyString } from "./evidence.mjs"; function resolveEngineJob(jobId, env) { const found = ENGINE_IDS.filter((engine) => FILE_ENGINE_DESCRIPTORS[engine].enumerateJobs(engineStateRoots(engine, env)[0], { env }).some((job) => nonEmptyString(job?.id) === jobId)); @@ -69,15 +69,13 @@ function settleWorkerRecord({ taskId, verdict, source, reason, failureKind, acce if (!finalPeer && !retainedEngines.includes(engine)) { continue; } - const engineRecord = finalPeer ? null : readEngineJobRecord(engine, jobId, env); - const failedTransport = engineRecord?.status === "error" || engineRecord?.transportStatus === "error"; - const salvage = failedTransport && RESUMABLE_CODEX_FAILURE_KINDS.has(engineRecord.failureKind); - if (!finalPeer && verdict === "accepted" && failedTransport && !salvage) { + const transport = engineAcceptanceTransport({ engine, jobId, acceptance: verdict, acceptFailedTransport, env }); + if (!finalPeer && verdict === "accepted" && transport.failedTransport && !transport.salvage) { stderr.write(`Skipped engine settlement for ${engineDisplayName(engine)} job ${jobId}: error transport is not a salvage failure.\n`); continue; } try { - recordEngineAcceptance({ engine, jobId, acceptance: verdict, source, reason, failureKind, acceptFailedTransport: acceptFailedTransport || (!finalPeer && verdict === "accepted" && salvage), workspaceRoot, asJson, env }); + recordEngineAcceptance({ engine, jobId, acceptance: verdict, source, reason, failureKind, acceptFailedTransport: transport.acceptFailedTransport, workspaceRoot, asJson, env }); } catch (error) { const message = error instanceof Error ? error.message : String(error); if (!finalPeer) { diff --git a/tests/engine-acceptance.test.mjs b/tests/engine-acceptance.test.mjs new file mode 100644 index 0000000..91cc2c9 --- /dev/null +++ b/tests/engine-acceptance.test.mjs @@ -0,0 +1,93 @@ +import assert from "node:assert/strict"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { test } from "node:test"; + +import { newestEngineCompanion, recordEngineAcceptance } from "../plugins/fusion/scripts/lib/engine-acceptance.mjs"; +import { settleQueuedVerdict } from "../plugins/fusion/scripts/lib/lifecycle/collection.mjs"; +import { createWorkerRecord, recordWorkerAcceptance } from "../plugins/fusion/scripts/lib/worker-state.mjs"; + +function sandbox(t) { + const root = fs.mkdtempSync(path.join(os.tmpdir(), "engine-acceptance-test-")); + t.after(() => fs.rmSync(root, { recursive: true, force: true })); + return root; +} + +function companion(root, source) { + const file = path.join(root, "codex-companion.mjs"); + fs.mkdirSync(root, { recursive: true }); + fs.writeFileSync(file, source); + return file; +} + +function cachedCompanion(home, version, modifiedAt) { + const file = companion(path.join(home, ".claude", "plugins", "cache", "claude-code-fusion", "codex", version, "scripts"), "export {};\n"); + fs.utimesSync(file, modifiedAt, modifiedAt); + return file; +} + +test("record acceptance survives a removed workspace", (t) => { + const root = sandbox(t); + const workspaceRoot = path.join(root, "removed-worktree"); + const ledger = path.join(root, "ledger.json"); + const bin = companion(path.join(root, "companion"), 'import fs from "node:fs";\nfs.writeFileSync(process.env.TEST_LEDGER, JSON.stringify({ cwd: process.cwd(), args: process.argv.slice(2) }));\n'); + fs.mkdirSync(workspaceRoot); + fs.rmdirSync(workspaceRoot); + + const result = recordEngineAcceptance({ engine: "codex", jobId: "a".repeat(32), acceptance: "accepted", workspaceRoot, env: { ...process.env, FUSION_CODEX_COMPANION: bin, TEST_LEDGER: ledger } }); + + assert.deepEqual(result, { updated: true }); + const written = JSON.parse(fs.readFileSync(ledger, "utf8")); + assert.equal(written.cwd, fs.realpathSync(path.dirname(bin))); + assert.deepEqual(written.args.slice(0, 5), ["record-acceptance", "--job-id", "a".repeat(32), "--acceptance", "accepted"]); +}); + +test("a spawn error reports its own cause", (t) => { + const root = sandbox(t); + const bin = companion(path.join(root, "companion"), "export {};\n"); + + assert.throws( + () => recordEngineAcceptance({ engine: "codex", jobId: "a".repeat(32), acceptance: "accepted", env: { ...process.env, FUSION_CODEX_COMPANION: bin, TEST_OVERSIZED_ENV: "x".repeat(4_000_000) } }), + (error) => error.code === "E2BIG" && !/Upgrade the Codex plugin/.test(error.message) + ); +}); + +test("matching fusion version wins, then highest semantic version wins", (t) => { + const root = sandbox(t); + const home = path.join(root, "home"); + const current = JSON.parse(fs.readFileSync(new URL("../plugins/fusion/.claude-plugin/plugin.json", import.meta.url), "utf8")).version; + const older = cachedCompanion(home, "0.0.9", new Date("2026-10-03")); + const newer = cachedCompanion(home, "0.0.10", new Date("2026-10-01")); + cachedCompanion(home, "0.0.10-beta.1", new Date("2026-10-04")); + + assert.equal(newestEngineCompanion("codex", { HOME: home }), newer); + const matching = cachedCompanion(home, current, new Date("2026-09-01")); + assert.equal(newestEngineCompanion("codex", { HOME: home }), matching); + assert.equal(newestEngineCompanion("codex", { HOME: home, FUSION_CODEX_COMPANION: older }), older); +}); + +test("queued accepted verdict enables failed transport for a salvaged timeout", (t) => { + const root = sandbox(t); + const workspaceRoot = path.join(root, "worktree"); + const stateRoot = path.join(root, "codex-state"); + const jobId = "b".repeat(32); + const taskId = `fusion-${"c".repeat(24)}`; + const calls = path.join(root, "calls.json"); + const bin = companion(path.join(root, "companion"), 'import fs from "node:fs";\nconst args = process.argv.slice(2);\nfs.writeFileSync(process.env.TEST_CALLS, JSON.stringify(args));\nif (!args.includes("--accept-failed-transport")) { process.stderr.write("has transport status error. Pass --accept-failed-transport"); process.exitCode = 1; }\n'); + const jobFile = path.join(stateRoot, "retained-workspace", "jobs", `${jobId}.json`); + fs.mkdirSync(path.dirname(jobFile), { recursive: true }); + fs.writeFileSync(jobFile, JSON.stringify({ id: jobId, workspaceRoot, status: "error", failureKind: "timeout" })); + fs.mkdirSync(workspaceRoot); + const env = { ...process.env, FUSION_WORKER_STATE_DIR: path.join(root, "workers"), FUSION_CODEX_STATE: stateRoot, FUSION_CODEX_COMPANION: bin, TEST_CALLS: calls }; + createWorkerRecord({ taskId, sessionId: "session-1", agentType: "codex:codex-rescue", workspaceRoot }, env); + const queued = recordWorkerAcceptance({ taskId, acceptance: "accepted", source: "main-loop", env }); + assert.equal(queued.queued, true); + fs.rmdirSync(workspaceRoot); + + const settled = settleQueuedVerdict({ ...queued.record, transportStatus: "done", peerEngine: "codex", peerJobId: jobId }, new Date().toISOString(), env); + + assert.equal(settled.acceptance, "accepted"); + assert.equal(settled.engineSettlementError, undefined); + assert.ok(JSON.parse(fs.readFileSync(calls, "utf8")).includes("--accept-failed-transport")); +}); From 95306aa90020db224a99f18035fba714faecb435 Mon Sep 17 00:00:00 2001 From: Harry Yep Date: Mon, 26 Oct 2026 20:34:35 +0800 Subject: [PATCH 04/13] refactor(fusion): drop the unused settlement workspace root --- plugins/fusion/scripts/fusion-stats.mjs | 2 +- .../fusion/scripts/lib/lifecycle/collection.mjs | 1 - plugins/fusion/scripts/lib/stats/settlement.mjs | 14 +++++++------- 3 files changed, 8 insertions(+), 9 deletions(-) diff --git a/plugins/fusion/scripts/fusion-stats.mjs b/plugins/fusion/scripts/fusion-stats.mjs index e8baa25..a53ba33 100644 --- a/plugins/fusion/scripts/fusion-stats.mjs +++ b/plugins/fusion/scripts/fusion-stats.mjs @@ -60,7 +60,7 @@ export function main(argv = process.argv.slice(2), { env = process.env, cwd = pr if (argv.includes("--record")) { const request = parseRecordArguments(argv, { allowPerPairReasons: rawArgs }); - const result = settleRecords({ ...request, workspaceRoot: cwd, env, stdout, stderr }); + const result = settleRecords({ ...request, env, stdout, stderr }); if (result.errors.length > 0) { throw new Error(`${result.errors.length} record settlement ${result.errors.length === 1 ? "failed" : "failures"}.`); } diff --git a/plugins/fusion/scripts/lib/lifecycle/collection.mjs b/plugins/fusion/scripts/lib/lifecycle/collection.mjs index 6221105..0629ab3 100644 --- a/plugins/fusion/scripts/lib/lifecycle/collection.mjs +++ b/plugins/fusion/scripts/lib/lifecycle/collection.mjs @@ -330,7 +330,6 @@ function settleQueuedVerdict(record, now, env) { reason: settled.acceptanceReason, failureKind: settled.acceptanceFailureKind ?? null, acceptFailedTransport: transport.acceptFailedTransport, - workspaceRoot: typeof settled.workspaceRoot === "string" ? settled.workspaceRoot : process.cwd(), env }); } catch (error) { diff --git a/plugins/fusion/scripts/lib/stats/settlement.mjs b/plugins/fusion/scripts/lib/stats/settlement.mjs index e427b43..585af39 100644 --- a/plugins/fusion/scripts/lib/stats/settlement.mjs +++ b/plugins/fusion/scripts/lib/stats/settlement.mjs @@ -33,7 +33,7 @@ function writeRecordConfirmation({ kind, engine = null, jobId = null, worker = n stdout.write(`Recorded ${worker.acceptance} for Fusion worker task ${worker.taskId}.\n`); } -function settleWorkerRecord({ taskId, verdict, source, reason, failureKind, acceptFailedTransport = false, asJson, workspaceRoot, env, stdout, stderr }) { +function settleWorkerRecord({ taskId, verdict, source, reason, failureKind, acceptFailedTransport = false, asJson, env, stdout, stderr }) { const current = readWorkerRecord(taskId, env); const terminal = WORKER_TERMINAL_STATUSES.has(current?.transportStatus); const peerJobId = terminal && typeof current.peerJobId === "string" && ENGINE_JOB_ID_PATTERN.test(current.peerJobId) ? current.peerJobId : null; @@ -75,7 +75,7 @@ function settleWorkerRecord({ taskId, verdict, source, reason, failureKind, acce continue; } try { - recordEngineAcceptance({ engine, jobId, acceptance: verdict, source, reason, failureKind, acceptFailedTransport: transport.acceptFailedTransport, workspaceRoot, asJson, env }); + recordEngineAcceptance({ engine, jobId, acceptance: verdict, source, reason, failureKind, acceptFailedTransport: transport.acceptFailedTransport, asJson, env }); } catch (error) { const message = error instanceof Error ? error.message : String(error); if (!finalPeer) { @@ -106,10 +106,10 @@ function settleWorkerRecord({ taskId, verdict, source, reason, failureKind, acce return [worker, ...settledPeers]; } -function settleEngineRecord({ jobId, verdict, source, reason, failureKind, acceptFailedTransport = false, asJson, workspaceRoot, env, stdout }) { +function settleEngineRecord({ jobId, verdict, source, reason, failureKind, acceptFailedTransport = false, asJson, env, stdout }) { const engine = resolveEngineJob(jobId, env); const confirmation = { engine, jobId, acceptance: verdict }; - recordEngineAcceptance({ engine, jobId, acceptance: verdict, source, reason, failureKind, acceptFailedTransport, workspaceRoot, asJson, env }); + recordEngineAcceptance({ engine, jobId, acceptance: verdict, source, reason, failureKind, acceptFailedTransport, asJson, env }); writeRecordConfirmation({ kind: "engine", engine, jobId, worker: confirmation, asJson, stdout }); const writes = [confirmation]; for (const record of readWorkerRecords(env).filter((candidate) => candidate.peerJobId === jobId)) { @@ -159,7 +159,7 @@ function validateRecordSettlements({ records, source, acceptFailedTransport, env } } -export function settleRecords({ records, source, acceptFailedTransport = false, asJson, workspaceRoot, env, stdout, stderr }) { +export function settleRecords({ records, source, acceptFailedTransport = false, asJson, env, stdout, stderr }) { validateRecordSettlements({ records, source, acceptFailedTransport, env }); const writes = []; const errors = []; @@ -167,9 +167,9 @@ export function settleRecords({ records, source, acceptFailedTransport = false, const pairOverride = acceptFailedTransport || pairAcceptFailedTransport; try { if (FUSION_TASK_ID_PATTERN.test(id)) { - writes.push(...settleWorkerRecord({ taskId: id, verdict, source, reason, failureKind, acceptFailedTransport: pairOverride, asJson, workspaceRoot, env, stdout, stderr })); + writes.push(...settleWorkerRecord({ taskId: id, verdict, source, reason, failureKind, acceptFailedTransport: pairOverride, asJson, env, stdout, stderr })); } else { - writes.push(...settleEngineRecord({ jobId: id, verdict, source, reason, failureKind, acceptFailedTransport: pairOverride, asJson, workspaceRoot, env, stdout })); + writes.push(...settleEngineRecord({ jobId: id, verdict, source, reason, failureKind, acceptFailedTransport: pairOverride, asJson, env, stdout })); } } catch (error) { const failure = { id, error }; From ecd2c1603de729a046617517239d8e59319bb8e7 Mon Sep 17 00:00:00 2001 From: Harry Yep Date: Mon, 26 Oct 2026 20:39:27 +0800 Subject: [PATCH 05/13] fix(fusion): fresh budgets for continued wrappers, role efforts, any header separator --- plugins/fusion/scripts/lib/engines.mjs | 4 +- .../fusion/scripts/lib/lifecycle/dispatch.mjs | 4 +- plugins/fusion/scripts/lib/worker-state.mjs | 16 ++--- tests/fusion-worker-state.test.mjs | 4 +- tests/lifecycle-defaults.test.mjs | 62 +++++++++++++++++++ 5 files changed, 76 insertions(+), 14 deletions(-) create mode 100644 tests/lifecycle-defaults.test.mjs diff --git a/plugins/fusion/scripts/lib/engines.mjs b/plugins/fusion/scripts/lib/engines.mjs index 6e3ee76..0d8ab46 100644 --- a/plugins/fusion/scripts/lib/engines.mjs +++ b/plugins/fusion/scripts/lib/engines.mjs @@ -163,12 +163,12 @@ function engineRoleFromBrief(id, brief) { const { defaultRole, roles } = ENGINES[id]; const lines = brief.split(/\r?\n/).filter((line) => line.trim()).slice(0, ROLE_HEADER_LINES); const header = lines[0]?.trim() ?? ""; - const lane = knownRole(header.match(new RegExp(`^lane:\\s*${id}\\s+([a-z][a-z-]*)(?=$|[\\s;,|])`, "i"))?.[1], roles); + const lane = knownRole(header.match(new RegExp(`^lane:\\s*${id}\\s+([a-z][a-z-]*)(?=$|[\\s;,|·])`, "i"))?.[1], roles); if (lane) { return lane; } for (const line of lines) { - const field = line.match(new RegExp(`(?:^|[\\s;,|])${id}-role:\\s*([^\\s;,|]+)`, "i")); + const field = line.match(new RegExp(`(?:^|[\\s;,|·])${id}-role:\\s*([^\\s;,|·]+)`, "i")); if (field) { return knownRole(field[1], roles); } diff --git a/plugins/fusion/scripts/lib/lifecycle/dispatch.mjs b/plugins/fusion/scripts/lib/lifecycle/dispatch.mjs index 91b2ff8..78d5d2f 100644 --- a/plugins/fusion/scripts/lib/lifecycle/dispatch.mjs +++ b/plugins/fusion/scripts/lib/lifecycle/dispatch.mjs @@ -55,7 +55,7 @@ function injectedPeerDefaults(prompt, agentType, env) { } const { options, brief, separated } = leadingCompanionOptions(prompt); const header = brief.split(/\r?\n/).find((line) => line.trim())?.trim() ?? ""; - const roleField = new RegExp(`(?:^|\\|\\s*)(?:${ENGINE_ID_ALTERNATION})-role:`, "i"); + const roleField = new RegExp(`(?:^|[|·,;]\\s*)(?:${ENGINE_ID_ALTERNATION})-role:`, "i"); if (!/^lane:/i.test(header) && !roleField.test(header)) { return null; } @@ -77,7 +77,7 @@ function injectedPeerDefaults(prompt, agentType, env) { filled.push("--model", choice.model); } if (effort === null) { - const matched = Object.values(defaults[engine]).find((entry) => entry.model === (model ?? choice.model)); + const matched = model !== null && model !== choice.model ? Object.values(defaults[engine]).find((entry) => entry.model === model) : null; filled.push("--effort", matched?.effort ?? choice.effort); } if (filled.length === 0) { diff --git a/plugins/fusion/scripts/lib/worker-state.mjs b/plugins/fusion/scripts/lib/worker-state.mjs index d2df2dd..8b03780 100644 --- a/plugins/fusion/scripts/lib/worker-state.mjs +++ b/plugins/fusion/scripts/lib/worker-state.mjs @@ -408,15 +408,15 @@ export function reopenWorkerContinuation(record, at = new Date().toISOString(), uncachedWindDownSentAt: null, lastActivityAt: at, inFlightSince: at, + budgetBaseline: { + at, + turns: record.turns ?? 0, + outputTokens: record.usage?.outputTokens ?? 0, + uncachedTokens: record.usage?.uncachedTokens ?? 0, + toolCalls: record.toolCalls ?? 0, + usage: Object.fromEntries(USAGE_KEYS.map((key) => [key, integer(record.usage?.[key])])) + }, ...(fusionWorker ? { - budgetBaseline: { - at, - turns: record.turns ?? 0, - outputTokens: record.usage?.outputTokens ?? 0, - uncachedTokens: record.usage?.uncachedTokens ?? 0, - toolCalls: record.toolCalls ?? 0, - usage: Object.fromEntries(USAGE_KEYS.map((key) => [key, integer(record.usage?.[key])])) - }, transcriptRoundPrefix: { turns: 0, toolCalls: 0, usage: emptyWorkerUsage() }, usageMessages: {}, turnIds: [], diff --git a/tests/fusion-worker-state.test.mjs b/tests/fusion-worker-state.test.mjs index 3a9a4bc..9cf3727 100644 --- a/tests/fusion-worker-state.test.mjs +++ b/tests/fusion-worker-state.test.mjs @@ -301,7 +301,7 @@ test("reopened tool-response worker counts new transcript usage above its prior assert.deepStrictEqual(refreshWorkerTranscript(refreshed, transcript), refreshed); }); -test("peer continuation resets round counters without adding a Fusion budget baseline", () => { +test("peer continuation resets round counters and records its budget baseline", () => { const at = "2026-09-27T00:01:00.000Z"; const prior = unverifiedRecord({ agentType: "codex:codex-rescue", transcriptPath: "/tmp/peer.jsonl", transcriptOffset: 512, transcriptCarry: "partial", transcriptSkippingLine: true, stopBlockCount: 6, cancelAttemptCount: 2, retryCount: 1, terminalWriteGraceUsedAt: at, cancelReason: "old round", cancelRequestedAt: at, windDownContextSentAt: at, tokenWindDownSentAt: at, uncachedWindDownSentAt: at }); const reopened = reopenWorkerContinuation(prior, at, 2048); @@ -309,7 +309,7 @@ test("peer continuation resets round counters without adding a Fusion budget bas assert.deepStrictEqual([reopened.transcriptPath, reopened.transcriptOffset, reopened.transcriptCarry, reopened.transcriptSkippingLine], [null, 0, "", false]); assert.deepStrictEqual([reopened.stopBlockCount, reopened.cancelAttemptCount, reopened.retryCount], [0, 0, 0]); assert.deepStrictEqual([reopened.terminalWriteGraceUsedAt, reopened.cancelReason, reopened.cancelRequestedAt, reopened.windDownContextSentAt, reopened.tokenWindDownSentAt, reopened.uncachedWindDownSentAt], [null, null, null, null, null, null]); - assert.strictEqual(reopened.budgetBaseline, undefined); + assert.deepStrictEqual(reopened.budgetBaseline, { at, turns: 0, outputTokens: 0, uncachedTokens: 0, toolCalls: 0, usage: { inputTokens: 0, cacheCreationInputTokens: 0, cacheReadInputTokens: 0, outputTokens: 0, totalTokens: 0, uncachedTokens: 0 } }); }); test("unsettled continuation keeps its job id available for the next verdict", () => { diff --git a/tests/lifecycle-defaults.test.mjs b/tests/lifecycle-defaults.test.mjs new file mode 100644 index 0000000..e587736 --- /dev/null +++ b/tests/lifecycle-defaults.test.mjs @@ -0,0 +1,62 @@ +import assert from "node:assert/strict"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { test } from "node:test"; + +import { workerBudgetFailure } from "../plugins/fusion/scripts/lib/lifecycle/budget.mjs"; +import { injectedPeerDefaults } from "../plugins/fusion/scripts/lib/lifecycle/dispatch.mjs"; +import { reopenWorkerContinuation } from "../plugins/fusion/scripts/lib/worker-state.mjs"; + +test("continued peer wrappers get a fresh wall clock and usage budget", () => { + const at = "2026-09-27T01:00:00.000Z"; + const wallClockMs = 2_400_000; + const prior = { + agentType: "codex:codex-rescue", + transportStatus: "done", + startedAt: "2026-09-27T00:00:00.000Z", + limits: { wallClockMs, stallMs: 600_000, maxTurns: 100, maxOutputTokens: 192_000, maxUncachedTokens: 960_000 }, + turns: 103, + toolCalls: 9, + usage: { inputTokens: 7, outputTokens: 195_000, uncachedTokens: 965_000 } + }; + const reopened = reopenWorkerContinuation(prior, at); + + assert.deepStrictEqual(reopened.budgetBaseline, { + at, + turns: 103, + outputTokens: 195_000, + uncachedTokens: 965_000, + toolCalls: 9, + usage: { inputTokens: 7, cacheCreationInputTokens: 0, cacheReadInputTokens: 0, outputTokens: 195_000, totalTokens: 0, uncachedTokens: 965_000 } + }); + assert.strictEqual(workerBudgetFailure(reopened, Date.parse(at)), null); + assert.strictEqual(workerBudgetFailure(reopened, Date.parse(at) + wallClockMs - 1), null); + assert.deepStrictEqual(workerBudgetFailure(reopened, Date.parse(at) + wallClockMs), { failureKind: "timeout", limitKind: "wall", reason: `wall clock budget reached (${wallClockMs}ms)` }); +}); + +test("Grok injects the selected role's effort when roles share a model", (t) => { + const directory = fs.mkdtempSync(path.join(os.tmpdir(), "fusion-lifecycle-defaults-")); + t.after(() => fs.rmSync(directory, { recursive: true, force: true })); + const env = { FUSION_MODEL_ROUTING: path.join(directory, "missing.json") }; + + for (const [role, effort] of [["independence", "high"], ["live-web", "high"], ["burst", "low"]]) { + const prompt = `grok-role: ${role}\ngoal: review`; + assert.strictEqual(injectedPeerDefaults(prompt, "grok:grok-rescue", env)?.prompt, `--model grok-4.7-build-fast --effort ${effort} -- ${prompt}`); + assert.strictEqual(injectedPeerDefaults(`--model grok-4.7-build-fast -- ${prompt}`, "grok:grok-rescue", env)?.prompt, `--effort ${effort} --model grok-4.7-build-fast -- ${prompt}`); + } + + const otherModel = "--model grok-4.6 -- grok-role: independence"; + assert.strictEqual(injectedPeerDefaults(otherModel, "grok:grok-rescue", env)?.prompt, `--effort medium ${otherModel}`); +}); + +test("Cursor role fields with middle dot, comma, and semicolon separators get defaults", (t) => { + const directory = fs.mkdtempSync(path.join(os.tmpdir(), "fusion-lifecycle-defaults-")); + t.after(() => fs.rmSync(directory, { recursive: true, force: true })); + const env = { FUSION_MODEL_ROUTING: path.join(directory, "missing.json") }; + + for (const header of ["cursor · cursor-role: diversity · acceptance: cited lines", "cursor·cursor-role:diversity·acceptance: cited lines", "cursor, cursor-role: diversity, acceptance: cited lines", "cursor; cursor-role: diversity; acceptance: cited lines"]) { + const prompt = `${header}\ngoal: consult`; + assert.strictEqual(injectedPeerDefaults(prompt, "cursor:cursor-rescue", env)?.prompt, `--model muse-spark-1.3 --effort xhigh -- ${prompt}`); + } +}); From b7fd7addf61a56904bfa9d5b3aae771615cc78b0 Mon Sep 17 00:00:00 2001 From: Harry Yep Date: Mon, 26 Oct 2026 20:39:33 +0800 Subject: [PATCH 06/13] fix(fusion): credit real verification output, flag untracked generic agents --- .../scripts/lib/guard/dispatch-events.mjs | 10 ++- .../scripts/lib/guard/verification-events.mjs | 16 +--- .../fusion/scripts/lib/guard/verification.mjs | 11 +-- .../scripts/lib/verification-command.mjs | 45 ++++++++-- tests/inline-delegation-guard.test.mjs | 86 ++++++++++++++++++- tests/verification-command.test.mjs | 35 +++++++- 6 files changed, 164 insertions(+), 39 deletions(-) diff --git a/plugins/fusion/scripts/lib/guard/dispatch-events.mjs b/plugins/fusion/scripts/lib/guard/dispatch-events.mjs index e9e9e0c..dd105a6 100644 --- a/plugins/fusion/scripts/lib/guard/dispatch-events.mjs +++ b/plugins/fusion/scripts/lib/guard/dispatch-events.mjs @@ -20,6 +20,7 @@ function handleDispatch(input, env, { sessionId, toolName, file, now, posture, w return; } const subagentType = toolName === "SendMessage" ? record.agentType : extractSubagentType(input.tool_input); + const genericAgent = toolName === "Agent" && !input.agent_id && (subagentType == null || ["fork", "general-purpose", "claude"].includes(subagentType)); const safeSubagentType = sanitizeIdentifier(subagentType); const lane = laneForSubagentType(subagentType); const description = extractDispatchDescription(input.tool_input); @@ -122,6 +123,7 @@ function handleDispatch(input, env, { sessionId, toolName, file, now, posture, w { at: now, session: sessionId, event: "dispatch", lane: confirmation.lane, tool: toolName, ...(confirmation.messageId ? { messageId: confirmation.messageId } : {}), ...(confirmation.description ? { description: confirmation.description } : {}) }, env ); + let advisoryContext = genericAgent ? tagMessage("inline-guard.generic-agent-advisory", "This Agent runs a work package outside Fusion collection and settlement. Use codex:codex-rescue or fusion:claude-worker for tracked work, and Explore for read only search.") : null; if (confirmation.shouldAdvise) { const declineStated = hasFleetDeclineAfterLastUserMessage(input.transcript_path); const advisory = claimAdvisory( @@ -150,12 +152,14 @@ function handleDispatch(input, env, { sessionId, toolName, file, now, posture, w env ); if (!advisory.suppressed && !declineStated) { - writeHookText( - `${JSON.stringify(postToolAdvisoryOutput(tagMessage(NARROW_WAVE_ADVISORY_MESSAGE_SLUG, `${confirmation.streak} ${NARROW_WAVE_ADVISORY_SUFFIX}`)))}\n` - ); + const narrowContext = tagMessage(NARROW_WAVE_ADVISORY_MESSAGE_SLUG, `${confirmation.streak} ${NARROW_WAVE_ADVISORY_SUFFIX}`); + advisoryContext = advisoryContext ? `${advisoryContext}\n${narrowContext}` : narrowContext; } } } + if (advisoryContext) { + writeHookText(`${JSON.stringify(postToolAdvisoryOutput(advisoryContext))}\n`); + } return; } } diff --git a/plugins/fusion/scripts/lib/guard/verification-events.mjs b/plugins/fusion/scripts/lib/guard/verification-events.mjs index 852e45a..5c71f59 100644 --- a/plugins/fusion/scripts/lib/guard/verification-events.mjs +++ b/plugins/fusion/scripts/lib/guard/verification-events.mjs @@ -1,11 +1,12 @@ import { JUDGMENT_POSTURE } from "../posture.mjs"; import { messageCode, tagMessage } from "../user-messages.mjs"; +import { verificationEvidence } from "../verification-command.mjs"; import { recordAuditEvent } from "./audit.mjs"; import { resolveBudget, resolveFleetWaveGapMs, resolveMode } from "./config.mjs"; import { BASH_TOOL, MAIN_LANE, NO_OP_BASH_COMMANDS, NO_OP_HEARTBEAT_DENY_MESSAGE_SLUG, NO_OP_HEARTBEAT_REASON } from "./constants.mjs"; import { inFlightWorkerTaskSignature, totalDispatches } from "./dispatch.mjs"; import { normalizeState, readState, resetWriteWindow, withStateLock, writeState } from "./state.mjs"; -import { extractBashCommand, verificationCommand, verificationOutputPasses, verificationResponseFailed } from "./verification.mjs"; +import { extractBashCommand } from "./verification.mjs"; import { denyOutput } from "./write.mjs"; function handleBash(input, env, { sessionId, toolName, file, now, posture, writeHookText }) { @@ -13,19 +14,10 @@ function handleBash(input, env, { sessionId, toolName, file, now, posture, write const command = extractBashCommand(input.tool_input); if (input.hook_event_name === "PostToolUse") { const toolResponse = input.tool_response; - const recognizedVerification = command === null ? null : verificationCommand(command, env); - if ( - posture !== JUDGMENT_POSTURE || - command === null || - !recognizedVerification.recognized || - !toolResponse || - typeof toolResponse !== "object" || - Array.isArray(toolResponse) || - verificationResponseFailed(toolResponse) - ) { + if (posture !== JUDGMENT_POSTURE || command === null) { return; } - const evidence = recognizedVerification.observable ? "exit-status" : verificationOutputPasses(toolResponse) ? "output-summary" : null; + const evidence = verificationEvidence(command, toolResponse, env); if (!evidence) { return; } diff --git a/plugins/fusion/scripts/lib/guard/verification.mjs b/plugins/fusion/scripts/lib/guard/verification.mjs index cc3e41d..0b9374a 100644 --- a/plugins/fusion/scripts/lib/guard/verification.mjs +++ b/plugins/fusion/scripts/lib/guard/verification.mjs @@ -1,7 +1,5 @@ import { TRANSCRIPT_TAIL_MAX_BYTES } from "./constants.mjs"; -import { appendToolResponseText } from "../verification-command.mjs"; import fs from "node:fs"; -export { verificationCommand, verificationResponseFailed } from "../verification-command.mjs"; function transcriptEntryContent(entry) { if (Array.isArray(entry?.message?.content)) { @@ -113,11 +111,4 @@ function extractBashCommand(toolInput) { return typeof toolInput.command === "string" ? toolInput.command.trim() : null; } -function verificationOutputPasses(toolResponse) { - const parts = []; - appendToolResponseText(toolResponse, parts); - const output = parts.join("\n"); - return /^\s*(?:ℹ\s+)?fail\s+0\b/im.test(output) && !/^\s*(?:✖\s|not ok\b)/im.test(output); -} - -export { transcriptEntryContent, isExternalUserMessage, readTranscriptTail, hasFleetDeclineAfterLastUserMessage, normalizeMessageId, messageIdForToolUse, extractBashCommand, verificationOutputPasses }; +export { transcriptEntryContent, isExternalUserMessage, readTranscriptTail, hasFleetDeclineAfterLastUserMessage, normalizeMessageId, messageIdForToolUse, extractBashCommand }; diff --git a/plugins/fusion/scripts/lib/verification-command.mjs b/plugins/fusion/scripts/lib/verification-command.mjs index dc863d8..fe97724 100644 --- a/plugins/fusion/scripts/lib/verification-command.mjs +++ b/plugins/fusion/scripts/lib/verification-command.mjs @@ -120,6 +120,7 @@ const PASS_SUMMARY_PATTERNS = [ const FAILURE_SUMMARY_PATTERNS = [ /^\s*(?:✖\s|not ok\b)/im, + /^\s*(?:ℹ\s+)?fail\s+[1-9]\d*\b/im, /[1-9]\d*\s+fail(?:ed|ures?)\b/i, /^\s*FAIL\b/m, /[1-9]\d*\s+error(s)?\b/i @@ -244,6 +245,7 @@ function isVerificationSegment(segment, pattern) { function commandSegments(command) { const segments = []; let start = 0; + let separator = null; let quote = null; let escaped = false; for (let index = 0; index < command.length; index += 1) { @@ -273,14 +275,15 @@ function commandSegments(command) { } const segment = command.slice(start, index).trim(); if (segment) { - segments.push(segment); + segments.push({ command: segment, separator }); } + separator = twoCharacterSeparator === "&&" || twoCharacterSeparator === "||" ? twoCharacterSeparator : character; index += separatorLength - 1; start = index + 1; } const segment = command.slice(start).trim(); if (segment) { - segments.push(segment); + segments.push({ command: segment, separator }); } return segments; } @@ -295,10 +298,22 @@ function verificationCommand(command, env = process.env) { } const segments = commandSegments(trimmed); const pattern = customPattern(env); - const matchingSegments = segments.map((segment) => isVerificationSegment(segment, pattern)); + const matchingSegments = segments.map((segment) => isVerificationSegment(segment.command, pattern)); + const lastVerificationIndex = matchingSegments.lastIndexOf(true); + let pipefail = false; + const pipefailAtSegment = segments.map((segment, index) => { + if (/^set\s+-[a-z]*o\s+pipefail$/i.test(segment.command) && segments[index + 1]?.separator !== "|") { + pipefail = true; + } else if (/^set\s+\+(?:[a-z]*o)\s+pipefail$/i.test(segment.command)) { + pipefail = false; + } + return pipefail; + }); return { - recognized: matchingSegments.some(Boolean), - observable: matchingSegments.at(-1) === true + recognized: lastVerificationIndex !== -1, + observable: lastVerificationIndex !== -1 && segments.slice(lastVerificationIndex + 1).every( + (segment, offset) => segment.separator === "&&" || (segment.separator === "|" && pipefailAtSegment[lastVerificationIndex + offset]) + ) }; } @@ -325,19 +340,27 @@ function appendToolResponseText(value, parts, depth = 0) { } } -function verificationOutputPasses(toolResponse) { +function verificationOutputSummary(toolResponse) { if ( !toolResponse || typeof toolResponse !== "object" || Array.isArray(toolResponse) || verificationResponseFailed(toolResponse) ) { - return false; + return { passes: false, fails: false }; } const parts = []; appendToolResponseText(toolResponse, parts); const output = parts.join("\n"); - return PASS_SUMMARY_PATTERNS.some((pattern) => pattern.test(output)) && !FAILURE_SUMMARY_PATTERNS.some((pattern) => pattern.test(output)); + return { + passes: PASS_SUMMARY_PATTERNS.some((pattern) => pattern.test(output)), + fails: FAILURE_SUMMARY_PATTERNS.some((pattern) => pattern.test(output)) + }; +} + +function verificationOutputPasses(toolResponse) { + const summary = verificationOutputSummary(toolResponse); + return summary.passes && !summary.fails; } function verificationResponseFailed(toolResponse) { @@ -355,7 +378,11 @@ function verificationEvidence(command, toolResponse, env = process.env) { if (verificationResponseFailed(toolResponse)) { return null; } - return verification.observable ? "exit-status" : verificationOutputPasses(toolResponse) ? "output-summary" : null; + const summary = verificationOutputSummary(toolResponse); + if (summary.fails) { + return null; + } + return verification.observable ? "exit-status" : summary.passes ? "output-summary" : null; } export { diff --git a/tests/inline-delegation-guard.test.mjs b/tests/inline-delegation-guard.test.mjs index 66d1952..cdc0be3 100644 --- a/tests/inline-delegation-guard.test.mjs +++ b/tests/inline-delegation-guard.test.mjs @@ -117,7 +117,7 @@ function writePayload(sandbox, { sessionId = "session-1", toolName = "Edit", fil return payload; } -function dispatchPayload(sandbox, { sessionId = "session-1", toolName = "Agent", subagentType, description = "do the thing", agentId } = {}) { +function dispatchPayload(sandbox, { sessionId = "session-1", toolName = "Agent", subagentType = "Explore", description = "do the thing", agentId } = {}) { const payload = { hook_event_name: "PostToolUse", session_id: sessionId, @@ -455,7 +455,7 @@ test("a passing verification resets the write window and stops the advisory from }); test("a verification whose exit status is masked by a later segment does not reset the window", (t) => { - for (const command of ["npm test 2>&1 | tail -20", "npm test || true", "npm test && git status --short"]) { + for (const command of ["npm test 2>&1 | tail -20", "npm test || true", "npm test; git status --short"]) { const sandbox = makeSandbox(t); for (let index = 0; index < 5; index += 1) { run(sandbox, writePayload(sandbox)); @@ -493,6 +493,21 @@ for (const { name, command, output } of [ name: "npm test after a manifest generator", command: 'node plugins/fusion/scripts/generate-rules-manifest.mjs && npm test 2>&1 | rg "^✖|^ℹ (tests|pass|fail|skipped)"', output: testSummary() + }, + { + name: "npm test piped to tail with vitest output", + command: "npm test 2>&1 | tail -20", + output: "Test Files 3 passed (3)\nTests 42 passed (42)" + }, + { + name: "jest piped to tail", + command: "npx jest --ci 2>&1 | tail -20", + output: "Tests: 12 passed, 12 total" + }, + { + name: "pytest piped to tail", + command: "pytest -q 2>&1 | tail -5", + output: "3 passed in 0.12s" } ]) { test(`an output summary resets an unobservable verification for ${name}`, (t) => { @@ -533,6 +548,11 @@ for (const { name, command, toolResponse } of [ name: "a failing npm summary after a manifest generator", command: 'node plugins/fusion/scripts/generate-rules-manifest.mjs && npm test 2>&1 | rg "^✖|^ℹ (tests|pass|fail|skipped)"', toolResponse: { is_error: false, output: testSummary({ fail: 1 }) } + }, + { + name: "a failing vitest summary after a passing count", + command: "npm test 2>&1 | tail -20", + toolResponse: { is_error: false, output: "Tests 42 passed\nTest Files 1 failed" } } ]) { test(`an unobservable verification does not reset for ${name}`, (t) => { @@ -549,7 +569,7 @@ for (const { name, command, toolResponse } of [ } test("a verification that ends the command resets the window through every separator", (t) => { - for (const command of ["pnpm lint && npm test", "echo starting; npm run test:unit", "npm test > /tmp/out 2>&1"]) { + for (const command of ["pnpm lint && npm test", "echo starting; npm run test:unit", "npm test > /tmp/out 2>&1", "npm test && git status --short", "set -o pipefail; npm test | tail", "set -euo pipefail; npm test 2>&1 | tail -20", "tsc --noEmit"]) { const sandbox = makeSandbox(t); for (let index = 0; index < 5; index += 1) { run(sandbox, writePayload(sandbox)); @@ -580,6 +600,20 @@ test("failed and interrupted verifications do not reset the write window", (t) = } }); +test("a failing summary prevents observable verification credit", (t) => { + const sandbox = makeSandbox(t); + for (let index = 0; index < 5; index += 1) { + run(sandbox, writePayload(sandbox)); + } + const result = run(sandbox, verificationPayload(sandbox, { + command: "npm test && echo done", + toolResponse: { is_error: false, output: "ℹ pass 11\nℹ fail 1" } + })); + assert.strictEqual(result.stdout, ""); + assert.strictEqual(readState(sandbox, "session-1").writesSinceDispatch, 5); + assert.strictEqual(readAuditRecords(sandbox).filter((record) => record.event === "verification").length, 0); +}); + test("zero exit codes reset the write window", (t) => { for (const key of ["exitCode", "exit_code", "code", "returnCode"]) { const sandbox = makeSandbox(t); @@ -1001,7 +1035,7 @@ test("agent dispatches are bucketed by lane from the subagent_type prefix", (t) run(sandbox, dispatchPayload(sandbox, { subagentType: "fusion:claude-worker" })); run(sandbox, dispatchPayload(sandbox, { subagentType: "fusion:trivial-worker" })); run(sandbox, dispatchPayload(sandbox, { subagentType: "Explore" })); - run(sandbox, dispatchPayload(sandbox, {})); + run(sandbox, dispatchPayload(sandbox, { subagentType: null })); const state = readState(sandbox, "session-1"); assert.deepStrictEqual(state.dispatches, { @@ -1013,6 +1047,50 @@ test("agent dispatches are bucketed by lane from the subagent_type prefix", (t) }); }); +test("generic Agent launches advise on PostToolUse without denying the dispatch", (t) => { + for (const subagentType of ["fork", "general-purpose", "claude", null]) { + const sandbox = makeSandbox(t); + const payload = dispatchPayload(sandbox, { subagentType }); + const launch = runRaw(sandbox, { ...payload, hook_event_name: "PreToolUse" }); + assert.strictEqual(launch.stdout, ""); + const result = runRaw(sandbox, payload); + const output = JSON.parse(result.stdout).hookSpecificOutput; + assert.strictEqual(output.hookEventName, "PostToolUse"); + assert.strictEqual(output.permissionDecision, undefined); + assertMessageTagged(output.additionalContext, "inline-guard.generic-agent-advisory"); + assert.match(output.additionalContext, /outside Fusion collection and settlement/); + assert.match(output.additionalContext, /codex:codex-rescue/); + assert.match(output.additionalContext, /fusion:claude-worker/); + assert.match(output.additionalContext, /Explore for read only search/); + assert.strictEqual(readState(sandbox, "session-1").dispatches.builtin, 1); + } +}); + +test("tracked agents, Explore, Task, and subagent launches do not receive the generic Agent advisory", (t) => { + for (const options of [ + { subagentType: "codex:codex-rescue" }, + { subagentType: "fusion:claude-worker" }, + { subagentType: "Explore" }, + { toolName: "Task", subagentType: "fork" }, + { subagentType: "fork", agentId: "nested-agent" } + ]) { + const sandbox = makeSandbox(t); + assert.strictEqual(run(sandbox, dispatchPayload(sandbox, options)).stdout, ""); + } +}); + +test("generic and narrow-wave advisories share one PostToolUse response", (t) => { + const sandbox = makeSandbox(t); + const result = triggerNarrowWaveAdvisory(sandbox, { FUSION_FLEET_WAVE_GAP_MS: "10000" }, dispatchPayload(sandbox, { subagentType: "fork" })); + const output = JSON.parse(result.stdout).hookSpecificOutput; + assert.strictEqual(output.hookEventName, "PostToolUse"); + assert.strictEqual(output.permissionDecision, undefined); + assert.match(output.additionalContext, /outside Fusion collection and settlement/); + assert.match(output.additionalContext, /2 consecutive width one dispatch waves/); + assert.ok(output.additionalContext.includes(messageTag("inline-guard.generic-agent-advisory"))); + assert.ok(output.additionalContext.includes(messageTag("inline-guard.narrow-wave-advisory"))); +}); + test("a Task dispatch is bucketed the same way as an Agent dispatch", (t) => { const sandbox = makeSandbox(t); const result = run(sandbox, dispatchPayload(sandbox, { toolName: "Task", subagentType: "codex:codex-rescue" })); diff --git a/tests/verification-command.test.mjs b/tests/verification-command.test.mjs index 51b74e7..78434b2 100644 --- a/tests/verification-command.test.mjs +++ b/tests/verification-command.test.mjs @@ -13,12 +13,19 @@ test("credits a piped vitest summary", () => { ); }); +test("credits a piped npm vitest summary", () => { + assert.equal(summaryEvidence("npm test 2>&1 | tail -20", "Test Files 3 passed (3)\nTests 42 passed (42)"), "output-summary"); + assert.equal(summaryEvidence("npm test | tail", "Tests 42 passed (42)"), "output-summary"); +}); + test("rejects a piped vitest summary with failures", () => { assert.equal(summaryEvidence("npx vitest run 2>&1 | tail -20", "Test Files 1 failed | 2 passed"), null); }); test("keeps node:test fail-zero summaries", () => { assert.equal(summaryEvidence("npm test | grep -E 'fail|pass'", "ℹ fail 0"), "output-summary"); + assert.equal(summaryEvidence("node --test | tail", "ℹ pass 12\nℹ fail 0"), "output-summary"); + assert.equal(summaryEvidence("node --test | tail", "ℹ pass 11\nℹ fail 1"), null); }); test("credits and rejects piped pytest summaries", () => { @@ -52,6 +59,32 @@ test("credits and rejects piped jest summaries", () => { assert.equal(summaryEvidence("npx jest --ci 2>&1 | tail", "Tests: 1 failed, 11 passed, 12 total"), null); }); +test("credits silent tsc only when its exit status is observable", () => { + assert.equal(summaryEvidence("tsc --noEmit", ""), "exit-status"); + assert.equal(summaryEvidence("tsc --noEmit | tail", ""), null); +}); + +test("preserves runner failures through && and through pipes with pipefail", () => { + for (const command of [ + "npm test && echo done", + "npm test && git status --short && echo done", + "set -o pipefail; npm test | tail", + "set -euo pipefail; npm test 2>&1 | tail -20", + "set -o pipefail; npm test | tail && echo done" + ]) { + assert.equal(summaryEvidence(command, ""), "exit-status", command); + } + for (const command of ["npm test | tail", "npm test; echo done", "npm test || true", "npm test && echo done; echo again", "set -o pipefail; npm test | tail; echo done", "set -o pipefail; npm test && set +o pipefail && tsc --noEmit | tail"]) { + assert.equal(summaryEvidence(command, ""), null, command); + } +}); + +test("rejects a failing summary anywhere in observable output", () => { + for (const output of ["ℹ pass 11\nℹ fail 1", "Tests 42 passed\nTest Files 1 failed", "Tests: 1 failed, 11 passed, 12 total", "1 failed, 2 passed in 0.2s"]) { + assert.equal(summaryEvidence("npm test && echo done", output), null, output); + } +}); + test("does not credit summaries for unrecognized commands", () => { assert.equal(summaryEvidence("./run-my-checks.sh", "3 passed"), null); }); @@ -63,7 +96,7 @@ test("does not credit failed or interrupted tool responses", () => { }); test("keeps final-segment exit-status evidence and rejects mutating commands", () => { - assert.equal(verificationEvidence("npm test", { is_error: false, output: "1 failed" }), "exit-status"); + assert.equal(verificationEvidence("npm test", { is_error: false, output: "1 failed" }), null); assert.equal(summaryEvidence("npm install | tail -20", "3 passed"), null); }); From 054e6e991720c56e9ed3aad71594664147203951 Mon Sep 17 00:00:00 2001 From: Harry Yep Date: Mon, 26 Oct 2026 20:43:41 +0800 Subject: [PATCH 07/13] fix: classify codex network outages, report grok's json error message --- plugins/codex/scripts/lib/codex-outcome.mjs | 3 ++ plugins/fusion/scripts/breaker-check.mjs | 4 +- plugins/grok/scripts/lib/grok-exec.mjs | 42 ++++++++++++++++++++- tests/breaker-check.test.mjs | 15 ++++++++ tests/codex-exec.test.mjs | 29 ++++++++++++++ tests/failure-kinds.test.mjs | 4 ++ tests/grok-quota.test.mjs | 29 +++++++++++++- 7 files changed, 123 insertions(+), 3 deletions(-) diff --git a/plugins/codex/scripts/lib/codex-outcome.mjs b/plugins/codex/scripts/lib/codex-outcome.mjs index e13f3fd..57f7817 100644 --- a/plugins/codex/scripts/lib/codex-outcome.mjs +++ b/plugins/codex/scripts/lib/codex-outcome.mjs @@ -48,6 +48,9 @@ function classifiedFailureKind(message, fallback = "error") { if (/unauthori[sz]ed|unauthenticated|authentication|login required|invalid api key|api key/i.test(text)) { return "auth"; } + if (/workspace routing discovery timed out|stream disconnected before completion|error sending request|connection (?:refused|reset)|\b(?:ECONNRESET|ECONNREFUSED|ENOTFOUND|EAI_AGAIN)\b|getaddrinfo|network is unreachable|Reconnecting\.\.\./i.test(text)) { + return "network"; + } if (/permission denied|operation not permitted|approval (?:required|denied)|sandbox (?:denied|violation)/i.test(text)) { return "permission"; } diff --git a/plugins/fusion/scripts/breaker-check.mjs b/plugins/fusion/scripts/breaker-check.mjs index 1a865c8..d81b8f0 100644 --- a/plugins/fusion/scripts/breaker-check.mjs +++ b/plugins/fusion/scripts/breaker-check.mjs @@ -23,9 +23,11 @@ const CODEX_FAILURE_PATTERNS = [ "quota", /quota|insufficient (?:account )?(?:balance|credit|credits|funds)|balance (?:is )?exhausted|exhausted (?:account )?balance|payment required|usage limit|billing|credit limit|http(?:\/\d(?:\.\d)?)?\s+402|status(?: code)?\s*[:=]?\s*402/i ], + ["rate_limited", /rate[ _-]?limit|too many requests|\b429\b/i], + ["refused", /misalignment policy|cyber policy|content policy|reason: content_filter|flagged for possible (?:cybersecurity|biological) risk/i], ["auth", /unauthori[sz]ed|unauthenticated|authentication|auth(?:entication)? failed|invalid api key|api key/i], + ["network", /workspace routing discovery timed out|stream disconnected before completion|error sending request|connection (?:refused|reset)|\b(?:ECONNRESET|ECONNREFUSED|ENOTFOUND|EAI_AGAIN)\b|getaddrinfo|network is unreachable|Reconnecting\.\.\./i], ["missing_cli", /missing[ _-]?cli|command not found|cli not found|no such file/i], - ["rate_limited", /rate[ _-]?limit|too many requests|\b429\b/i], ["timeout", /timed?\s*out|timeout/i], ["stall", /\bstall(?:ed|ing)?\b|no[- ]progress/i], ["protocol", /collaboration policy violation|companion protocol violation|transport protocol violation/i], diff --git a/plugins/grok/scripts/lib/grok-exec.mjs b/plugins/grok/scripts/lib/grok-exec.mjs index db29bc5..00023dd 100644 --- a/plugins/grok/scripts/lib/grok-exec.mjs +++ b/plugins/grok/scripts/lib/grok-exec.mjs @@ -546,6 +546,39 @@ function maxTurnsReachedError(stderr) { return null; } +function internalErrorFromStderr(stderr) { + const text = String(stderr ?? ""); + let error = null; + for (const match of text.matchAll(/Internal error:\s*\{/gi)) { + const start = match.index + match[0].length - 1; + let depth = 0; + let quoted = false; + let escaped = false; + for (let end = start; end < text.length; end += 1) { + const char = text[end]; + if (quoted) { + if (escaped) escaped = false; + else if (char === "\\") escaped = true; + else if (char === '"') quoted = false; + } else if (char === '"') { + quoted = true; + } else if (char === "{") { + depth += 1; + } else if (char === "}" && --depth === 0) { + const body = tryParseObject(text.slice(start, end + 1)); + if (typeof body?.message === "string" && body.message.trim()) { + error = { + message: body.message.trim().replace(/\s+/g, " ").slice(0, 500), + httpStatus: Number.isInteger(body.http_status) && body.http_status >= 100 && body.http_status <= 599 ? body.http_status : null + }; + } + break; + } + } + } + return error; +} + const DENIED_TOOL_STDERR_PATTERNS = [ /\b(?:tool|rule)\s+[`'"]?([A-Za-z_][\w./:-]*(?:\([^)\r\n]*\))?)[`'"]?\s+(?:(?:is|was)\s+)?(?:not allowed|denied|disallowed|blocked|rejected)/i, /\b(?:denied|disallowed|blocked|rejected)\s+(?:tool|rule)\s*[:=]?\s*[`'"]?([A-Za-z_][\w./:-]*(?:\([^)\r\n]*\))?)[`'"]?/i, @@ -2280,6 +2313,13 @@ async function closeGrokChild(run, code, signal) { parsed?.type === "error" && typeof parsed.message === "string" && parsed.message.trim() ? parsed.message.trim() : null; + const stderrError = exitCode !== 0 ? internalErrorFromStderr(run.stderr) : null; + const terminalErrorMessage = structuredErrorMessage && !/^Internal error:\s*\{$/i.test(structuredErrorMessage) + ? structuredErrorMessage + : stderrError?.message ?? structuredErrorMessage; + if (stderrError?.httpStatus != null) { + appendJobLog(run.options.logFile, `HTTP ${stderrError.httpStatus}`); + } const recognizedEnvelope = envelopeShapeIsValid || structuredErrorMessage != null; const metrics = recognizedEnvelope ? headlessMetricsObservation(parsed) : { error: null }; const resolved = resolvedOutputMetadata(parsed, run.options.cwd, run.childEnv); @@ -2291,7 +2331,7 @@ async function closeGrokChild(run, code, signal) { ? AUTH_RETRY_FAILURE_MESSAGE : run.promptTransportError ? `Grok prompt transport failed before stdin was accepted: ${run.promptTransportError.message}` - : structuredErrorMessage ?? maxTurnsReachedError(run.stderr); + : terminalErrorMessage ?? maxTurnsReachedError(run.stderr); const parseError = metrics.error ?? ( exitCode === 0 && !run.timedOut && !errorMessage && expectsJsonOutput(run.args) && !validEnvelope ? invalidSessionId diff --git a/tests/breaker-check.test.mjs b/tests/breaker-check.test.mjs index 1b8eaf7..6c9b03f 100644 --- a/tests/breaker-check.test.mjs +++ b/tests/breaker-check.test.mjs @@ -63,6 +63,21 @@ function run(sandbox, extraEnv = {}) { }); } +test("text classified Codex connectivity failures map to network after stronger evidence", () => { + for (const message of [ + "Reconnecting... (workspace routing discovery timed out)", + "stream disconnected before completion: upstream closed the stream", + "error sending request: connection refused", + "connection reset", "ECONNRESET", "ECONNREFUSED", "ENOTFOUND", "EAI_AGAIN", "getaddrinfo failed", "network is unreachable" + ]) { + assert.strictEqual(codexFailureKind(message), "network", message); + } + assert.strictEqual(codexFailureKind("quota exhausted; Reconnecting..."), "quota"); + assert.strictEqual(codexFailureKind("rate limit exceeded; connection reset"), "rate_limited"); + assert.strictEqual(codexFailureKind("stream disconnected before completion: Incomplete response returned, reason: content_filter"), "refused"); + assert.strictEqual(codexFailureKind("authentication failed; error sending request"), "auth"); +}); + test("worker breaker parses recent ledger records once and skips 2,000 old files", (t) => { const sandbox = makeSandbox(t); const env = envFor(sandbox); diff --git a/tests/codex-exec.test.mjs b/tests/codex-exec.test.mjs index 1d706eb..64b8d50 100644 --- a/tests/codex-exec.test.mjs +++ b/tests/codex-exec.test.mjs @@ -29,6 +29,7 @@ import { terminateProcessTree, terminateProcessTreeSync } from "../plugins/codex/scripts/lib/codex-exec.mjs"; +import { failureOutcome } from "../plugins/codex/scripts/lib/codex-outcome.mjs"; import { envFor as companionEnvFor, jobRecords as companionJobRecords, @@ -807,6 +808,34 @@ test("turn.failed remains the structured failure when the process exits nonzero" assert.equal(outcome.exitCode, 1); }); +test("terminal Codex connectivity diagnostics classify as network", () => { + for (const message of [ + "Reconnecting... (workspace routing discovery timed out)", + "stream disconnected before completion: upstream closed the stream", + "error sending request: connection refused", + "ECONNRESET", "ECONNREFUSED", "ENOTFOUND", "EAI_AGAIN", "getaddrinfo failed", "network is unreachable" + ]) { + const outcome = failureOutcome({ terminalEvent: { type: "turn.failed" }, turnFailureMessage: message }, { exitCode: 1 }, ""); + assert.equal(outcome.status, "error", message); + assert.equal(outcome.failureKind, "network", message); + assert.equal(outcome.errorMessage, message); + } + const stderrOnly = failureOutcome({ diagnostics: [] }, { exitCode: 1 }, "Reconnecting... (workspace routing discovery timed out)"); + assert.equal(stderrOnly.failureKind, "network"); +}); + +test("Codex quota, rate limit, refusal, and auth evidence precedes network evidence", () => { + for (const [message, expected] of [ + ["quota exhausted; Reconnecting...", "quota"], + ["rate limit exceeded; connection reset", "rate_limited"], + ["stream disconnected before completion: Incomplete response returned, reason: content_filter", "refused"], + ["authentication failed; error sending request", "auth"] + ]) { + const outcome = failureOutcome({ terminalEvent: { type: "turn.failed" }, turnFailureMessage: message }, { exitCode: 1 }, ""); + assert.equal(outcome.failureKind, expected, message); + } +}); + test("top-level and item errors are diagnostics and do not override a completed turn", async (t) => { const { outcome } = await runFixture(t, "retry-error-completed"); assert.equal(outcome.status, "done"); diff --git a/tests/failure-kinds.test.mjs b/tests/failure-kinds.test.mjs index e30a921..b1edae6 100644 --- a/tests/failure-kinds.test.mjs +++ b/tests/failure-kinds.test.mjs @@ -100,6 +100,10 @@ test("a Codex refusal turn failure yields failure kind refused", (t) => { for (const [message, failureKind] of [ ["Flex capacity unavailable.", "rate_limited"], ["stream disconnected before completion: Incomplete response returned, reason: content_filter", "refused"], + ["Reconnecting... (workspace routing discovery timed out)", "network"], + ["stream disconnected before completion: upstream connection closed", "network"], + ["error sending request: connection reset", "network"], + ["network is unreachable", "network"], ["This request has been flagged for possible cybersecurity risk.", "refused"], ["This content was flagged for possible biological risk.", "refused"], ["The local disk has capacity for this request.", "error"] diff --git a/tests/grok-quota.test.mjs b/tests/grok-quota.test.mjs index 4652019..eacb95c 100644 --- a/tests/grok-quota.test.mjs +++ b/tests/grok-quota.test.mjs @@ -5,7 +5,7 @@ import test from "node:test"; import { writeQuotaCooldown } from "../plugins/grok/scripts/lib/state.mjs"; import { initFixtureRepo } from "./lib/git-fixture.mjs"; -import { envFor, jobRecords, makeSandbox, readInvocations, runCompanion } from "./lib/companion-harness.mjs"; +import { envFor, fakeGrok, jobRecords, makeSandbox, readInvocations, runCompanion } from "./lib/companion-harness.mjs"; const cooldownFile = (sandbox) => path.join(sandbox.dataDir, "quota-cooldown.json"); const runTask = (sandbox, extra = {}, args = ["task", "check"]) => @@ -74,6 +74,33 @@ test("quota failure records cooldown and later task fails before Grok launch", ( assert.equal(JSON.parse(fs.readFileSync(cooldownFile(sandbox), "utf8")).jobId, failed.id); }); +test("multiline internal error reports the JSON message and uses HTTP status for quota cooldown", (t) => { + const marker = ' if (mode === "quota-json-error") {'; + const source = fs.readFileSync(fakeGrok, "utf8"); + assert.ok(source.includes(marker)); + for (const message of [ + "API error (status 402 Payment Required): Grok Build usage balance exhausted", + "Grok Build plan disabled" + ]) { + const sandbox = makeSandbox(t); + const bin = path.join(sandbox.root, "multiline-grok"); + fs.writeFileSync(bin, source.replace(marker, ` if (mode === "multiline-quota-error") { + emit({ type: "error", message: "Internal error: {" }); + process.stderr.write("Error: Internal error: " + JSON.stringify({ message: process.env.FAKE_GROK_QUOTA_MESSAGE, http_status: 402 }, null, 2) + "\\n"); + process.exit(1); + } + +${marker}`), { mode: 0o700 }); + const result = runTask(sandbox, { GROK_BIN: bin, FAKE_GROK_MODE: "multiline-quota-error", FAKE_GROK_QUOTA_MESSAGE: message }); + assert.notEqual(result.status, 0); + const [failed] = jobRecords(sandbox.dataDir); + assert.equal(failed.failureKind, "quota"); + assert.equal(failed.errorMessage, message); + assert.match(failed.errorTail, /HTTP 402/); + assert.equal(JSON.parse(fs.readFileSync(cooldownFile(sandbox), "utf8")).message, message); + } +}); + test("bare quota diagnostic classifies as quota without opening cooldown", (t) => { const sandbox = makeSandbox(t); const result = runTask(sandbox, { FAKE_GROK_MODE: "quota-error" }); From 67eb2d220b7424b1e71e87d17b57462e101bb404 Mon Sep 17 00:00:00 2001 From: Harry Yep Date: Mon, 26 Oct 2026 20:46:58 +0800 Subject: [PATCH 08/13] fix: migrate legacy codex and grok job records instead of rejecting them --- plugins/codex/scripts/lib/state.mjs | 78 +++++++++++----- plugins/grok/scripts/lib/state.mjs | 97 ++++++++++++++++---- tests/codex-state.test.mjs | 11 ++- tests/record-migration.test.mjs | 136 ++++++++++++++++++++++++++++ 4 files changed, 277 insertions(+), 45 deletions(-) create mode 100644 tests/record-migration.test.mjs diff --git a/plugins/codex/scripts/lib/state.mjs b/plugins/codex/scripts/lib/state.mjs index 68951be..005acc9 100644 --- a/plugins/codex/scripts/lib/state.mjs +++ b/plugins/codex/scripts/lib/state.mjs @@ -28,6 +28,10 @@ const DELIVERY_MODES = new Set(["foreground", "manual", "managed"]); const SEMANTIC_STATUSES = new Set(["accepted", "rejected", "unverified"]); const SEMANTIC_FAILURE_KINDS = new Set(["intent_override", "scope_rewrite", "wrong_approach", "style_mismatch", "oversized"]); const ACCEPTANCE_SOURCES = new Set(["collector", "main-loop", "stats"]); +const JOB_REQUIRED_FIELDS = ["claudeSessionId", "delivery", "deliveryCollectedAt", "artifactsPruned", + "acceptanceSource", "acceptanceRecordedAt", "fileChangeCount", "semanticStatus", + "semanticFailureKind", "semanticFailureMessage", "serviceTier", "resolvedModel", "resolvedEffort", + "timeoutMs", "companionVersion"]; const JOB_ID_PATTERN = /^[A-Za-z0-9][A-Za-z0-9._-]{0,127}$/; const WORKSPACE_SLUG_PATTERN = /^.+-[a-f0-9]{16}$/; const GIT_CACHE_TTL_MS = 3000; @@ -503,17 +507,41 @@ function assertJobRecord(record) { if (record.companionVersion != null && (typeof record.companionVersion !== "string" || !record.companionVersion.trim())) { throw new TypeError("Job companion version is invalid."); } - const requiredFields = ["claudeSessionId", "delivery", "deliveryCollectedAt", "artifactsPruned", - "acceptanceSource", "acceptanceRecordedAt", "fileChangeCount", "semanticStatus", - "semanticFailureKind", "semanticFailureMessage", "serviceTier", "resolvedModel", "resolvedEffort", - "timeoutMs", "companionVersion"]; if (record.schemaVersion !== JOB_SCHEMA_VERSION || record.engine !== "codex" || Object.hasOwn(record, "appliedServiceTier") || Object.hasOwn(record, "sessionId") - || requiredFields.some((field) => !Object.hasOwn(record, field))) { + || JOB_REQUIRED_FIELDS.some((field) => !Object.hasOwn(record, field))) { throw new TypeError("Job record schema is invalid."); } } +export function migrateJobRecord(raw) { + if (!raw || typeof raw !== "object" || Array.isArray(raw) || raw.engine !== "codex" + || (raw.schemaVersion != null && raw.schemaVersion !== JOB_SCHEMA_VERSION)) return raw; + if (raw.schemaVersion === JOB_SCHEMA_VERSION && !Object.hasOwn(raw, "sessionId") + && !Object.hasOwn(raw, "appliedServiceTier") + && JOB_REQUIRED_FIELDS.every((field) => Object.hasOwn(raw, field))) return raw; + const { appliedServiceTier, sessionId, ...record } = raw; + return { + ...record, + claudeSessionId: raw.claudeSessionId ?? sessionId ?? null, + delivery: raw.delivery ?? (raw.background ? "manual" : "foreground"), + deliveryCollectedAt: raw.deliveryCollectedAt ?? null, + acceptanceSource: raw.acceptanceSource ?? null, + acceptanceRecordedAt: raw.acceptanceRecordedAt ?? null, + fileChangeCount: raw.fileChangeCount ?? null, + semanticStatus: raw.semanticStatus ?? "unverified", + semanticFailureKind: raw.semanticFailureKind ?? null, + semanticFailureMessage: raw.semanticFailureMessage ?? null, + serviceTier: raw.serviceTier ?? raw.request?.serviceTier ?? null, + resolvedModel: raw.resolvedModel ?? null, + resolvedEffort: raw.resolvedEffort ?? null, + timeoutMs: raw.timeoutMs ?? null, + companionVersion: raw.companionVersion ?? null, + schemaVersion: raw.schemaVersion ?? JOB_SCHEMA_VERSION, + artifactsPruned: raw.artifactsPruned ?? false + }; +} + function ownerSessionId(record) { const value = record?.claudeSessionId ?? null; return typeof value === "string" && value.trim() ? value.trim() : null; @@ -774,7 +802,7 @@ export function createJobRecord(fields) { export function writeJobRecordFile(file, record) { assertJobRecord(record); withRecordLock(file, () => { - const existing = readJobRecordFile(file); + const existing = readJobRecordFile(file, true); if (existing && TERMINAL_STATUSES.has(existing.status)) { return; } @@ -783,7 +811,7 @@ export function writeJobRecordFile(file, record) { return file; } -export function readJobRecordFile(file) { +export function readJobRecordFile(file, locked = false) { let text; try { text = fs.readFileSync(file, "utf8"); @@ -800,12 +828,19 @@ export function readJobRecordFile(file) { quarantineCorruptRecord(file); return null; } + let migrated; try { - assertJobRecord(parsed); - return parsed; + migrated = migrateJobRecord(parsed); + assertJobRecord(migrated); } catch { return null; } + if (migrated === parsed) return parsed; + if (locked) { + atomicWriteFile(file, `${JSON.stringify(migrated, null, 2)}\n`); + return migrated; + } + return withRecordLock(file, () => readJobRecordFile(file, true)); } export function updateJobRecordFile(file, patch) { @@ -813,7 +848,7 @@ export function updateJobRecordFile(file, patch) { throw new TypeError("Job record patch must be an object."); } return withRecordLock(file, () => { - const existing = readJobRecordFile(file); + const existing = readJobRecordFile(file, true); if (!existing) { throw new Error(`No job record found at ${file}.`); } @@ -832,7 +867,7 @@ export function updateJobRecordFileWithCurrent(file, updater, options = {}) { } const allowTerminal = options.allowTerminal === true; return withRecordLock(file, () => { - const existing = readJobRecordFile(file); + const existing = readJobRecordFile(file, true); if (!existing) { throw new Error(`No job record found at ${file}.`); } @@ -857,7 +892,7 @@ export function finishJobRecordFile(file, patch) { throw new TypeError("A terminal job status is required."); } return withRecordLock(file, () => { - const existing = readJobRecordFile(file); + const existing = readJobRecordFile(file, true); if (!existing) { throw new Error(`No job record found at ${file}.`); } @@ -875,7 +910,7 @@ export function markJobCollectedFile(file, collectedAt = nowIso()) { throw new TypeError("A valid collection timestamp is required."); } return withRecordLock(file, () => { - const existing = readJobRecordFile(file); + const existing = readJobRecordFile(file, true); if (!existing) { throw new Error(`No job record found at ${file}.`); } @@ -1012,18 +1047,15 @@ function expectedManagedArtifacts(workspaceDir, id) { ]; } -function readManagedJobEntry(workspace, workspaceDir, file, id) { - const recordDescriptor = regularFileDescriptor(file); +function readManagedJobEntry(workspace, workspaceDir, file, id, locked = false) { + let recordDescriptor = regularFileDescriptor(file); if (recordDescriptor.kind !== "regular" || recordDescriptor.size > JOB_RECORD_SCAN_MAX_BYTES) { return null; } - let record; - try { - record = JSON.parse(fs.readFileSync(file, "utf8")); - assertJobRecord(record); - } catch { - return null; - } + const record = readJobRecordFile(file, locked); + if (!record) return null; + recordDescriptor = regularFileDescriptor(file); + if (recordDescriptor.kind !== "regular" || recordDescriptor.size > JOB_RECORD_SCAN_MAX_BYTES) return null; if (record.schemaVersion !== JOB_SCHEMA_VERSION || record.engine !== "codex" || record.id !== id) { return null; } @@ -1241,7 +1273,7 @@ function resumeProtectionChanged(entry, current, protection) { function deleteManagedJobEntry(entry, tier, protection) { return withRecordLock(entry.file, () => { const workspaceDir = path.dirname(path.dirname(entry.file)); - const refreshed = readManagedJobEntry(entry.workspace, workspaceDir, entry.file, entry.id); + const refreshed = readManagedJobEntry(entry.workspace, workspaceDir, entry.file, entry.id, true); if (!refreshed || !TERMINAL_STATUSES.has(refreshed.record.status) || (refreshed.record.background && !refreshed.record.collectedAt)) { return null; diff --git a/plugins/grok/scripts/lib/state.mjs b/plugins/grok/scripts/lib/state.mjs index 291977a..fa241a0 100644 --- a/plugins/grok/scripts/lib/state.mjs +++ b/plugins/grok/scripts/lib/state.mjs @@ -301,6 +301,68 @@ function withStructuredStatuses(record) { }; } +const JOB_RECORD_FIELDS = ["jobClass", "delivery", "semanticStatus", "request", "pidIdentity", + "grokPidIdentity", "grokVersion", "deliveryStatus", "transportStatus", "resultPayload", "cleanupRequired"]; + +function assertJobRecord(record) { + if (!record || typeof record !== "object" || Array.isArray(record) + || record.schemaVersion !== 1 || record.engine !== "grok" + || typeof record.id !== "string" || !record.id + || !TERMINAL_STATUSES.has(record.status) && record.status !== "running" + || typeof record.mode !== "string" || !record.mode + || typeof record.cwd !== "string" || !record.cwd + || typeof record.createdAt !== "string" || !Number.isFinite(Date.parse(record.createdAt)) + || typeof record.background !== "boolean" + || record.request != null && (typeof record.request !== "object" || Array.isArray(record.request)) + || JOB_RECORD_FIELDS.some((field) => !Object.hasOwn(record, field))) { + throw new TypeError("Job record schema is invalid."); + } +} + +export function migrateJobRecord(raw) { + if (!raw || typeof raw !== "object" || Array.isArray(raw) + || raw.engine != null && raw.engine !== "grok" + || raw.schemaVersion != null && raw.schemaVersion !== 1) return raw; + if (raw.schemaVersion === 1 && raw.engine === "grok" + && JOB_RECORD_FIELDS.every((field) => Object.hasOwn(raw, field))) return raw; + if (typeof raw.id !== "string" || !raw.id || typeof raw.mode !== "string" + || typeof raw.cwd !== "string" || typeof raw.createdAt !== "string" + || typeof raw.background !== "boolean" || !Object.hasOwn(raw, "request")) return raw; + return withStructuredStatuses({ + ...raw, + schemaVersion: 1, + engine: "grok", + companionVersion: raw.companionVersion ?? null, + grokVersion: typeof raw.grokVersion === "string" && raw.grokVersion.trim() ? raw.grokVersion.trim() : null, + jobClass: raw.jobClass ?? "unknown", + delivery: raw.delivery ?? (raw.background ? "manual" : "foreground"), + deliveryCollectedAt: raw.deliveryCollectedAt ?? null, + claudeSessionId: raw.claudeSessionId ?? null, + finishedAt: raw.finishedAt ?? null, + exitCode: raw.exitCode ?? null, + sessionId: raw.sessionId ?? null, + resultText: raw.resultText ?? null, + resolvedModel: raw.resolvedModel ?? null, + resolvedEffort: raw.resolvedEffort ?? null, + role: raw.role ?? null, + usage: raw.usage ?? null, + modelUsage: raw.modelUsage ?? null, + usageIsIncomplete: raw.usageIsIncomplete ?? null, + modelUsageIsIncomplete: raw.modelUsageIsIncomplete ?? null, + errorMessage: raw.errorMessage ?? null, + errorTail: raw.errorTail ?? null, + failureKind: raw.failureKind ?? null, + semanticStatus: raw.semanticStatus ?? "unverified", + semanticFailureKind: raw.semanticFailureKind ?? null, + semanticFailureMessage: raw.semanticFailureMessage ?? null, + pidIdentity: raw.pidIdentity ?? null, + grokPidIdentity: raw.grokPidIdentity ?? null, + cleanupRequired: raw.cleanupRequired ?? false, + resultPayload: raw.resultPayload ?? null, + cancelRequestedAt: raw.cancelRequestedAt ?? null + }); +} + function preserveTerminalRecord(existing, next) { if (existing && TERMINAL_STATUSES.has(existing.status)) { return withStructuredStatuses(existing); @@ -336,7 +398,7 @@ export function createJobRecordFile(file, record) { export function writeJobRecordFile(file, record) { withRecordLock(file, () => { - const existing = readJobRecordFile(file); + const existing = readJobRecordFile(file, true); const next = preserveTerminalRecord(existing, record); atomicWriteFile(file, `${JSON.stringify(next, null, 2)}\n`); }); @@ -345,7 +407,7 @@ export function writeJobRecordFile(file, record) { export function markManagedDeliveryCollectedFile(file, collectedAt = nowIso()) { return withRecordLock(file, () => { - const existing = readJobRecordFile(file); + const existing = readJobRecordFile(file, true); if (!existing || existing.delivery !== "managed" || !TERMINAL_STATUSES.has(existing.status) || existing.deliveryCollectedAt) { return existing; } @@ -357,7 +419,7 @@ export function markManagedDeliveryCollectedFile(file, collectedAt = nowIso()) { export function markDeliveryCollectedFile(file, collectedAt = nowIso()) { return withRecordLock(file, () => { - const existing = readJobRecordFile(file); + const existing = readJobRecordFile(file, true); if (!existing || existing.delivery === "foreground" || !TERMINAL_STATUSES.has(existing.status) || existing.deliveryCollectedAt) { return existing; } @@ -367,22 +429,21 @@ export function markDeliveryCollectedFile(file, collectedAt = nowIso()) { }); } -export function readJobRecordFile(file) { +export function readJobRecordFile(file, locked = false) { if (!fs.existsSync(file)) { return null; } try { const record = JSON.parse(fs.readFileSync(file, "utf8")); - if (record && typeof record === "object" && !Array.isArray(record) && Object.hasOwn(record, "id") && ( - record.schemaVersion !== 1 || record.engine !== "grok" || typeof record.id !== "string" || - !Object.hasOwn(record, "jobClass") || !Object.hasOwn(record, "delivery") || - !Object.hasOwn(record, "semanticStatus") || !Object.hasOwn(record, "request") || - !Object.hasOwn(record, "pidIdentity") || !Object.hasOwn(record, "grokPidIdentity") || - !Object.hasOwn(record, "grokVersion") || !Object.hasOwn(record, "deliveryStatus") || - !Object.hasOwn(record, "transportStatus") || !Object.hasOwn(record, "resultPayload") || - !Object.hasOwn(record, "cleanupRequired") - )) return null; - return record; + if (!record || typeof record !== "object" || Array.isArray(record) || !Object.hasOwn(record, "id")) return record; + const migrated = migrateJobRecord(record); + assertJobRecord(migrated); + if (migrated === record) return record; + if (locked) { + atomicWriteFile(file, `${JSON.stringify(migrated, null, 2)}\n`); + return migrated; + } + return withRecordLock(file, () => readJobRecordFile(file, true)); } catch { return null; } @@ -536,7 +597,7 @@ function sweepGrokState(dataDir, now = Date.now()) { export function updateJobRecordFile(file, patch) { return withRecordLock(file, () => { - const existing = readJobRecordFile(file); + const existing = readJobRecordFile(file, true); if (!existing) { throw new Error(`No job record found at ${file}.`); } @@ -553,7 +614,7 @@ export function launchRunningJobProcess(file, launch, options = {}) { return withRecordLock(file, () => { const processRole = options.processRole === "worker" ? "worker" : "grok"; const processLabel = processRole === "worker" ? "Background worker" : "Grok process"; - const existing = readJobRecordFile(file); + const existing = readJobRecordFile(file, true); if (!existing) { throw new Error(`No job record found at ${file}.`); } @@ -597,7 +658,7 @@ export function launchRunningJobProcess(file, launch, options = {}) { export function updateJobRecordFileWithCurrent(file, updater, options = {}) { return withRecordLock(file, () => { - const existing = readJobRecordFile(file); + const existing = readJobRecordFile(file, true); if (!existing) { throw new Error(`No job record found at ${file}.`); } @@ -626,7 +687,7 @@ export function updateJobRecordFileWithCurrent(file, updater, options = {}) { export function finishSuccessfulJobRecordFile(file, successPatch) { return withRecordLock(file, () => { - const existing = readJobRecordFile(file); + const existing = readJobRecordFile(file, true); if (!existing) { throw new Error(`No job record found at ${file}.`); } diff --git a/tests/codex-state.test.mjs b/tests/codex-state.test.mjs index 2375df7..34233db 100644 --- a/tests/codex-state.test.mjs +++ b/tests/codex-state.test.mjs @@ -964,23 +964,26 @@ test("semantic failure kinds accept current policy and oversized records and rej assert.equal(readJobRecordFile(invalidFile), null); }); -test("records missing current fields or carrying retired fields are unreadable", (t) => { +test("records missing current fields or carrying retired fields migrate on read", (t) => { const sandbox = makeSandbox(t); const file = jobFilePath(sandbox.dataDir, sandbox.workDir, "legacy-file-change-count"); const record = createJobRecord({ cwd: sandbox.workDir, id: "legacy-file-change-count" }); fs.mkdirSync(path.dirname(file), { recursive: true }); delete record.fileChangeCount; fs.writeFileSync(file, `${JSON.stringify(record)}\n`, { mode: 0o600 }); - assert.equal(readJobRecordFile(file), null); + assert.equal(readJobRecordFile(file).fileChangeCount, null); + assert.equal(Object.hasOwn(JSON.parse(fs.readFileSync(file, "utf8")), "fileChangeCount"), true); const oldSession = { ...createJobRecord({ cwd: sandbox.workDir, id: "old-session" }), sessionId: "legacy" }; delete oldSession.claudeSessionId; const oldSessionFile = jobFilePath(sandbox.dataDir, sandbox.workDir, oldSession.id); fs.writeFileSync(oldSessionFile, `${JSON.stringify(oldSession)}\n`, { mode: 0o600 }); - assert.equal(readJobRecordFile(oldSessionFile), null); + assert.equal(readJobRecordFile(oldSessionFile).claudeSessionId, "legacy"); + assert.equal(Object.hasOwn(JSON.parse(fs.readFileSync(oldSessionFile, "utf8")), "sessionId"), false); const oldTier = { ...createJobRecord({ cwd: sandbox.workDir, id: "old-tier" }), appliedServiceTier: "priority" }; const oldTierFile = jobFilePath(sandbox.dataDir, sandbox.workDir, oldTier.id); fs.writeFileSync(oldTierFile, `${JSON.stringify(oldTier)}\n`, { mode: 0o600 }); - assert.equal(readJobRecordFile(oldTierFile), null); + assert.equal(Object.hasOwn(readJobRecordFile(oldTierFile), "appliedServiceTier"), false); + assert.equal(Object.hasOwn(JSON.parse(fs.readFileSync(oldTierFile, "utf8")), "appliedServiceTier"), false); }); test("global lookup rejects an id that is ambiguous across workspaces", (t) => { diff --git a/tests/record-migration.test.mjs b/tests/record-migration.test.mjs new file mode 100644 index 0000000..9ed5485 --- /dev/null +++ b/tests/record-migration.test.mjs @@ -0,0 +1,136 @@ +import assert from "node:assert/strict"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import test from "node:test"; + +import * as codex from "../plugins/codex/scripts/lib/state.mjs"; +import * as grok from "../plugins/grok/scripts/lib/state.mjs"; + +const root = fs.mkdtempSync(path.join(os.tmpdir(), "record-migration-")); +const bin = path.join(root, "bin"); +fs.mkdirSync(bin); +fs.writeFileSync(path.join(bin, "ps"), '#!/bin/sh\npid=""\nwhile [ "$#" -gt 0 ]; do\n if [ "$1" = "-p" ]; then pid="$2"; shift 2; else shift; fi\ndone\nprintf "process-%s\\n" "$pid"\n'); +fs.writeFileSync(path.join(bin, "sysctl"), '#!/bin/sh\nprintf "{ sec = 1, usec = 0 }\\n"\n'); +fs.chmodSync(path.join(bin, "ps"), 0o755); +fs.chmodSync(path.join(bin, "sysctl"), 0o755); +const originalPath = process.env.PATH; +process.env.PATH = `${bin}${path.delimiter}${originalPath}`; +test.after(() => { + process.env.PATH = originalPath; + fs.rmSync(root, { recursive: true, force: true }); +}); + +function checkMigration(name, state, legacy, expected) { + const file = path.join(root, name, `${legacy.id}.json`); + fs.mkdirSync(path.dirname(file), { recursive: true }); + fs.writeFileSync(file, `${JSON.stringify(legacy)}\n`, { mode: 0o600 }); + const before = fs.statSync(file).ino; + const copy = structuredClone(legacy); + const migrated = state.migrateJobRecord(legacy); + assert.deepEqual(legacy, copy); + assert.deepEqual(state.migrateJobRecord(migrated), migrated); + const read = state.readJobRecordFile(file); + assert.deepEqual(read, migrated); + expected(read); + assert.notEqual(fs.statSync(file).ino, before); + assert.deepEqual(JSON.parse(fs.readFileSync(file, "utf8")), read); + const after = fs.statSync(file).ino; + assert.deepEqual(state.readJobRecordFile(file), read); + assert.equal(fs.statSync(file).ino, after); +} + +test("Codex legacy session and tier fields migrate onto the current schema", () => { + const current = codex.createJobRecord({ id: "codex-legacy", cwd: root, background: true, request: { serviceTier: "priority" } }); + const legacy = { ...current, sessionId: "old-session", appliedServiceTier: "priority", resumable: true, resumeCommand: "resume" }; + delete legacy.claudeSessionId; + delete legacy.artifactsPruned; + delete legacy.fileChangeCount; + delete legacy.serviceTier; + checkMigration("codex", codex, legacy, (record) => { + assert.equal(record.claudeSessionId, "old-session"); + assert.equal(record.serviceTier, "priority"); + assert.equal(record.artifactsPruned, false); + assert.equal(record.fileChangeCount, null); + assert.equal(record.resumable, true); + assert.equal(Object.hasOwn(record, "sessionId"), false); + assert.equal(Object.hasOwn(record, "appliedServiceTier"), false); + }); + const versioned = { ...codex.createJobRecord({ id: "codex-plain-legacy", cwd: root }), sessionId: "owner" }; + delete versioned.artifactsPruned; + checkMigration("codex-plain", codex, versioned, (record) => { + assert.equal(record.claudeSessionId, "owner"); + assert.equal(record.artifactsPruned, false); + assert.equal(Object.hasOwn(record, "sessionId"), false); + }); +}); + +test("Grok 0.0.53 and 0.0.55 records gain the missing version field", () => { + for (const version of ["0.0.53", "0.0.55"]) { + const legacy = grok.createJobRecord({ id: `grok-${version}`, mode: "consult", cwd: root, briefFile: null, background: true }); + legacy.companionVersion = version; + delete legacy.grokVersion; + checkMigration(`grok-${version}`, grok, legacy, (record) => { + assert.equal(record.grokVersion, null); + assert.equal(record.delivery, "manual"); + assert.equal(record.transportStatus, "running"); + }); + } +}); + +test("Grok pre-versioned records gain structured status and process fields", () => { + const legacy = { + background: false, briefFile: null, cancelRequestedAt: null, claudeSessionId: "owner", + createdAt: "2026-01-01T00:00:00.000Z", cwd: root, errorMessage: null, errorTail: null, + exitCode: 0, failureKind: null, finishedAt: "2026-01-01T00:01:00.000Z", grokPid: null, + id: "grok-pre-version", mode: "consult", pid: null, request: {}, resultText: "done", + sessionId: "grok-session", status: "done" + }; + checkMigration("grok-pre-version", grok, legacy, (record) => { + assert.equal(record.schemaVersion, 1); + assert.equal(record.engine, "grok"); + assert.equal(record.jobClass, "unknown"); + assert.equal(record.deliveryStatus, "delivered"); + assert.equal(record.semanticStatus, "unverified"); + assert.equal(record.cleanupRequired, false); + assert.equal(record.sessionId, "grok-session"); + }); + const shorter = { ...legacy, id: "grok-pre-version-short" }; + delete shorter.cancelRequestedAt; + delete shorter.errorMessage; + checkMigration("grok-pre-version-short", grok, shorter, (record) => { + assert.equal(record.schemaVersion, 1); + assert.equal(record.semanticStatus, "unverified"); + }); +}); + +test("Codex retention scan migrates legacy records", () => { + const dataDir = path.join(root, "retention-data"); + const cwd = path.join(root, "retention-work"); + fs.mkdirSync(cwd); + const file = codex.jobFilePath(dataDir, cwd, "retained-legacy"); + const legacy = { ...codex.createJobRecord({ id: "retained-legacy", cwd, status: "done" }), sessionId: "owner" }; + delete legacy.artifactsPruned; + fs.mkdirSync(path.dirname(file), { recursive: true }); + fs.writeFileSync(file, `${JSON.stringify(legacy)}\n`, { mode: 0o600 }); + const result = codex.pruneJobState(dataDir, { recordMaxRecords: 10, recordRetentionDays: 90, maxRecords: 10, maxBytes: 1024 * 1024 }); + assert.equal(result.ok, true); + assert.equal(result.unrecognizedRecords, 0); + assert.equal(result.afterRecords, 1); + assert.equal(codex.readJobRecordFile(file).artifactsPruned, false); + assert.equal(Object.hasOwn(JSON.parse(fs.readFileSync(file, "utf8")), "sessionId"), false); +}); + +test("irreparable records remain unreadable and unchanged", () => { + for (const [name, state, record] of [ + ["codex-invalid", codex, { schemaVersion: 1, engine: "codex", id: "broken-codex", status: "mystery" }], + ["grok-invalid", grok, { schemaVersion: 1, engine: "grok", id: "broken-grok", status: "done" }] + ]) { + const file = path.join(root, name, `${record.id}.json`); + fs.mkdirSync(path.dirname(file), { recursive: true }); + fs.writeFileSync(file, `${JSON.stringify(record)}\n`); + const before = fs.readFileSync(file, "utf8"); + assert.equal(state.readJobRecordFile(file), null); + assert.equal(fs.readFileSync(file, "utf8"), before); + } +}); From b2b31912e3c28c450ff8c813b8370ddf624b615d Mon Sep 17 00:00:00 2001 From: Harry Yep Date: Mon, 26 Oct 2026 20:47:35 +0800 Subject: [PATCH 09/13] feat(fusion): host wide peer capacity gate and parent context in stats --- .../scripts/lib/lifecycle/constants.mjs | 2 +- .../fusion/scripts/lib/lifecycle/dispatch.mjs | 28 +++++++- .../fusion/scripts/lib/lifecycle/tools.mjs | 7 +- .../fusion/scripts/lib/stats/aggregation.mjs | 19 ++++- .../fusion/scripts/lib/stats/rendering.mjs | 4 ++ tests/fusion-worker-compaction.test.mjs | 6 +- tests/fusion-worker-lifecycle.test.mjs | 23 +++++-- tests/host-capacity.test.mjs | 69 +++++++++++++++++++ 8 files changed, 146 insertions(+), 12 deletions(-) create mode 100644 tests/host-capacity.test.mjs diff --git a/plugins/fusion/scripts/lib/lifecycle/constants.mjs b/plugins/fusion/scripts/lib/lifecycle/constants.mjs index c69144c..fcbb332 100644 --- a/plugins/fusion/scripts/lib/lifecycle/constants.mjs +++ b/plugins/fusion/scripts/lib/lifecycle/constants.mjs @@ -6,7 +6,7 @@ const PARENT_CONTEXT_ADVISORY_TOKENS_ENV = "FUSION_PARENT_CONTEXT_ADVISORY_TOKEN const VERIFICATION_MANIFEST_ENV = "FUSION_VERIFICATION_MANIFEST"; const DEFAULT_BRIEF_MAX_BYTES = 16 * 1024; const SIZING_ADVISORY_BYTES = 8 * 1024; -const DEFAULT_PARENT_CONTEXT_ADVISORY_TOKENS = 600000; +const DEFAULT_PARENT_CONTEXT_ADVISORY_TOKENS = 250000; const PARENT_CONTEXT_TAIL_MAX_BYTES = 256 * 1024; const SETTLE_DEMAND_STALE_MS_ENV = "FUSION_SETTLE_DEMAND_STALE_MS"; const DEFAULT_SETTLE_DEMAND_STALE_MS = 1_800_000; diff --git a/plugins/fusion/scripts/lib/lifecycle/dispatch.mjs b/plugins/fusion/scripts/lib/lifecycle/dispatch.mjs index 91b2ff8..fea75e3 100644 --- a/plugins/fusion/scripts/lib/lifecycle/dispatch.mjs +++ b/plugins/fusion/scripts/lib/lifecycle/dispatch.mjs @@ -1,11 +1,12 @@ import { PEER_RESCUE_AGENT_NAMES } from "../engines.mjs"; import fs from "node:fs"; +import os from "node:os"; import path from "node:path"; import { fileURLToPath } from "node:url"; import { ENGINE_ID_ALTERNATION, PEER_JOB_FOOTER_AGENTS, engineForAgentType, engineRoleFromBrief } from "../engines.mjs"; import { readRoutingDefaults } from "../model-table.mjs"; import { tagMessage } from "../user-messages.mjs"; -import { canonicalWorkerAgentType, createWorkerRecord, createWorkerTaskId, updateWorkerSessionState, workerRecordFile, writePrivateText } from "../worker-state.mjs"; +import { canonicalWorkerAgentType, createWorkerRecord, createWorkerTaskId, readWorkerRecords, updateWorkerSessionState, workerRecordFile, writePrivateText, WORKER_TERMINAL_STATUSES } from "../worker-state.mjs"; import { workerLimits } from "./budget.mjs"; import { updateLifecycleWorkerRecord } from "./records.mjs"; import { PARENT_CONTEXT_ADVISORY_TOKENS_ENV, VERIFICATION_MANIFEST_ENV, DEFAULT_BRIEF_MAX_BYTES, SIZING_ADVISORY_BYTES, DEFAULT_PARENT_CONTEXT_ADVISORY_TOKENS, PARENT_CONTEXT_TAIL_MAX_BYTES, BRIEF_AGENTS } from "./constants.mjs"; @@ -98,6 +99,29 @@ function parentContextAdvisoryTokens(env) { return Number.isSafeInteger(parsed) && parsed >= 0 ? parsed : DEFAULT_PARENT_CONTEXT_ADVISORY_TOKENS; } +function hostPeerCapacity(totalMemoryBytes = os.totalmem()) { + return Math.max(2, Math.floor(totalMemoryBytes / (3 * 1024 ** 3))); +} + +function hostPeerCapacityDenial(env, now = Date.now(), totalMemoryBytes = os.totalmem()) { + const capacity = hostPeerCapacity(totalMemoryBytes); + const inFlight = readWorkerRecords(env, { strict: true }).filter((record) => { + if (!PEER_RESCUE_AGENT_NAMES.has(record.agentType) || WORKER_TERMINAL_STATUSES.has(record.transportStatus)) { + return false; + } + const livenessAt = Date.parse(record.lastLivenessAt ?? record.startedAt ?? record.createdAt ?? ""); + const stallMs = Number.isSafeInteger(record.limits?.stallMs) && record.limits.stallMs > 0 + ? record.limits.stallMs + : workerLimits(record.agentType, env).stallMs; + return Number.isFinite(livenessAt) && now - livenessAt <= stallMs; + }).length; + if (inFlight < capacity) { + return null; + } + const totalGiB = Math.round(totalMemoryBytes / 1024 ** 3 * 10) / 10; + return tagMessage("worker-lifecycle.host-capacity-deny", `Host peer capacity reached: ${inFlight} peer wrapper jobs in flight (capacity ${capacity}; total memory ${totalGiB} GiB). Relaunch after a completion notification.`); +} + function parentContextTokens(transcriptPath) { if (typeof transcriptPath !== "string" || !path.isAbsolute(transcriptPath)) { return null; @@ -473,4 +497,4 @@ function briefArtifactPath(record, env) { } -export { allowAgentWithTaskId, promptText, leadingCompanionOptions, optionValue, injectedPeerDefaults, parentContextAdvisoryTokens, parentContextTokens, verificationManifestPath, globToRegExp, readVerificationManifest, pathsNamedInBrief, missingVerificationSuites, briefAdvisories, validateWorkerBrief, externalUserText, latestUserRequestedBackground, safeDescription, completionContract, collectorRequestIdentity, createDispatch, claimParentContextAdvisory, briefArtifactPath }; +export { allowAgentWithTaskId, promptText, leadingCompanionOptions, optionValue, injectedPeerDefaults, parentContextAdvisoryTokens, parentContextTokens, hostPeerCapacity, hostPeerCapacityDenial, verificationManifestPath, globToRegExp, readVerificationManifest, pathsNamedInBrief, missingVerificationSuites, briefAdvisories, validateWorkerBrief, externalUserText, latestUserRequestedBackground, safeDescription, completionContract, collectorRequestIdentity, createDispatch, claimParentContextAdvisory, briefArtifactPath }; diff --git a/plugins/fusion/scripts/lib/lifecycle/tools.mjs b/plugins/fusion/scripts/lib/lifecycle/tools.mjs index 2c1732e..b72b11d 100644 --- a/plugins/fusion/scripts/lib/lifecycle/tools.mjs +++ b/plugins/fusion/scripts/lib/lifecycle/tools.mjs @@ -5,7 +5,7 @@ import { tagMessage } from "../user-messages.mjs"; import { canonicalWorkerAgentType, isFusionWorkerAgent, WORKER_TERMINAL_STATUSES, markWorkerCollected, readWorkerRecord, readWorkerOutputFileTaskIds, readSessionWorkerRecords, refreshWorkerTranscript, reopenWorkerContinuation, WORKER_COLLECTION_METHODS } from "../worker-state.mjs"; import { markBudgetFailure, workerBudgetFailure, workerRoundUsage } from "./budget.mjs"; import { backfillCollectedTaskOutput, capturedPeerIdentity, collectedHarnessAsyncDelivery, noTaskFoundError, outputFileFromLaunchResponse, peerIdentityBackfill, settleQueuedVerdict, settleReapedIfActive, taskOutputResultText, terminalCollectedRecord, terminalCollectionPending } from "./collection.mjs"; -import { allowAgentWithTaskId, claimParentContextAdvisory, collectorRequestIdentity, createDispatch, injectedPeerDefaults, latestUserRequestedBackground, parentContextTokens, promptText, validateWorkerBrief } from "./dispatch.mjs"; +import { allowAgentWithTaskId, claimParentContextAdvisory, collectorRequestIdentity, createDispatch, hostPeerCapacityDenial, injectedPeerDefaults, latestUserRequestedBackground, parentContextTokens, promptText, validateWorkerBrief } from "./dispatch.mjs"; import { reconcileTaskNotifications, taskNotificationTransition } from "./notifications.mjs"; import { denyTool, hookOutput, writeOutput } from "./output.mjs"; import { activeSessionRecords, clearContinuationRequest, findSessionWorkerRecord, isAttributedTranscriptPath, parentTranscriptByteSize, recordForAgent, refreshRecord, sendMessageResumed, updateLifecycleWorkerRecord } from "./records.mjs"; @@ -36,6 +36,11 @@ function handlePreToolUse(input, env) { writeOutput(denyTool(tagMessage("worker-lifecycle.foreground-wrapper-deny", `${engineDisplayNameList()} wrapper Agents use foreground delivery. Keep the Agent foreground; an explicitly requested companion \`--background\` receipt stays inside that foreground wrapper call.`))); return; } + const capacityDenial = hostPeerCapacityDenial(env); + if (capacityDenial) { + writeOutput(denyTool(capacityDenial)); + return; + } const injection = injectedPeerDefaults(input.tool_input?.prompt, agentType, env); const advisory = claimParentContextAdvisory(input.session_id, parentContextTokens(input.transcript_path), env); if (injection || advisory) { diff --git a/plugins/fusion/scripts/lib/stats/aggregation.mjs b/plugins/fusion/scripts/lib/stats/aggregation.mjs index fecfa9b..a02fbc9 100644 --- a/plugins/fusion/scripts/lib/stats/aggregation.mjs +++ b/plugins/fusion/scripts/lib/stats/aggregation.mjs @@ -568,8 +568,24 @@ function retiredWorkerStats(env) { return stats; } +function parentContextAtDispatch(records, all, workspaceRoot) { + const repositoryCache = new Map(); + const values = records + .filter((record) => all || workspaceRootsShareRepository(record.workspaceRoot, workspaceRoot, repositoryCache)) + .map((record) => record.parentContextTokensAtDispatch) + .filter((value) => Number.isSafeInteger(value) && value >= 0) + .sort((left, right) => left - right); + const middle = Math.floor(values.length / 2); + return { + dispatchCount: values.length, + medianTokens: values.length ? (values[middle] + values[Math.floor((values.length - 1) / 2)]) / 2 : null, + p90Tokens: values.length ? values[Math.ceil(values.length * 0.9) - 1] : null + }; +} + export function buildFusionStats({ all = false, env = process.env, cwd = process.cwd() } = {}) { const workers = workerSnapshot(env); + const workspaceRoot = resolveGitWorkspaceRoot(cwd); const options = { all, env, cwd, workers }; const engines = {}; for (const provider of STATS_PROVIDER_REGISTRY) { @@ -577,7 +593,8 @@ export function buildFusionStats({ all = false, env = process.env, cwd = process } const fleet = buildFleetUsageStats({ env }); return { - scope: all ? "all" : resolveGitWorkspaceRoot(cwd), + scope: all ? "all" : workspaceRoot, + parentContextAtDispatch: parentContextAtDispatch(workers.records, all, workspaceRoot), ...engines, ...(all ? { retiredWorkers: retiredWorkerStats(env) } : {}), fleet, diff --git a/plugins/fusion/scripts/lib/stats/rendering.mjs b/plugins/fusion/scripts/lib/stats/rendering.mjs index 5dd6bd5..0c25bb1 100644 --- a/plugins/fusion/scripts/lib/stats/rendering.mjs +++ b/plugins/fusion/scripts/lib/stats/rendering.mjs @@ -321,6 +321,10 @@ function renderCoercionLedger(lines, ledger) { export function renderFusionStats(report, { identities = false, byDay = false } = {}) { const lines = ["# Fusion stats", "", `Scope: ${report.scope === "all" ? "all workspaces" : `workspace ${report.scope}`}`]; + if (report.parentContextAtDispatch) { + const { dispatchCount, medianTokens, p90Tokens } = report.parentContextAtDispatch; + lines.push(`Parent context at dispatch: ${dispatchCount} measured; median ${medianTokens ?? "n/a"} tokens, p90 ${p90Tokens ?? "n/a"} tokens`); + } for (const provider of STATS_PROVIDER_REGISTRY) { if (report[provider.id]) { renderEngine(lines, provider.displayName, report[provider.id], { identities }); diff --git a/tests/fusion-worker-compaction.test.mjs b/tests/fusion-worker-compaction.test.mjs index 29a5a7b..a24e349 100644 --- a/tests/fusion-worker-compaction.test.mjs +++ b/tests/fusion-worker-compaction.test.mjs @@ -141,12 +141,12 @@ test("next write compacts legacy buffers and skips an oversized carry", (t) => { assert.equal(compacted.transcriptSkippingLine, true); }); -test("parent context advice fires at 600k and 800k once each", async (t) => { +test("parent context advice fires at 250k and 800k once each", async (t) => { const box = sandbox(t); const dispatch = (id) => hook(box, { hook_event_name: "PreToolUse", session_id: "session-1", tool_use_id: `tool-${id}`, transcript_path: box.transcript, cwd: box.root, tool_name: "Agent", tool_input: { subagent_type: "codex:codex-rescue", description: "peer", prompt: "bounded peer brief" } }); const usage = (tokens) => fs.appendFileSync(box.transcript, `${JSON.stringify({ type: "assistant", message: { usage: { input_tokens: tokens } } })}\n`); - usage(600000); - assert.match((await dispatch(1)).hookSpecificOutput.additionalContext, /main loop context is 600k/); + usage(250000); + assert.match((await dispatch(1)).hookSpecificOutput.additionalContext, /main loop context is 250k/); usage(700000); assert.equal(await dispatch(2), null); usage(800000); diff --git a/tests/fusion-worker-lifecycle.test.mjs b/tests/fusion-worker-lifecycle.test.mjs index 0d9c37c..ce0411c 100644 --- a/tests/fusion-worker-lifecycle.test.mjs +++ b/tests/fusion-worker-lifecycle.test.mjs @@ -590,13 +590,13 @@ test("oversized briefs without sizing advise a large package and combine with ve assert.strictEqual(combinedLaunch.additionalContext.match(/Verification advisory/g).length, 1); }); -test("parent context advisory fires once at 600k and once at 800k across peer dispatches", (t) => { +test("parent context advisory fires once at 250k and once at 800k across peer dispatches", (t) => { const box = sandbox(t); const usage = (tokens) => fs.appendFileSync(box.transcript, `${JSON.stringify({ type: "assistant", message: { usage: { input_tokens: 1000, cache_read_input_tokens: tokens - 1500, cache_creation_input_tokens: 500 } } })}\n`); - usage(600000); + usage(250000); const atThreshold = run(box, dispatch(box)); - assert.match(JSON.parse(atThreshold.stdout).hookSpecificOutput.additionalContext, /main loop context is 600k tokens/); - assert.strictEqual(readWorkerRecords(envFor(box))[0].parentContextTokensAtDispatch, 600000); + assert.match(JSON.parse(atThreshold.stdout).hookSpecificOutput.additionalContext, /main loop context is 250k tokens/); + assert.strictEqual(readWorkerRecords(envFor(box))[0].parentContextTokensAtDispatch, 250000); usage(601000); fs.appendFileSync(box.transcript, `${JSON.stringify({ type: "assistant", isSidechain: true, message: { usage: { input_tokens: 1, cache_read_input_tokens: 0, cache_creation_input_tokens: 0 } } })}\n`); @@ -5678,3 +5678,18 @@ test("cancellation demand and re-verification agree for an uncollected failed re assert.strictEqual(needsCancellation(candidate), true); assert.deepStrictEqual(reverifyCancellationRecords([candidate], envFor(box)).map((record) => record.taskId), [candidate.taskId]); }); + +test("peer wrapper launches are denied while the host runs its peer capacity", (t) => { + const box = sandbox(t); + const capacity = Math.max(2, Math.floor(os.totalmem() / (3 * 1024 ** 3))); + for (let index = 0; index < capacity; index += 1) { + createWorkerRecord({ taskId: `fusion-host-peer-${index}`, sessionId: `other-session-${index % 2}`, agentType: "codex:codex-rescue", workspaceRoot: box.workDir ?? box.root, limits: { stallMs: 600_000 } }, envFor(box)); + } + const denied = JSON.parse(run(box, dispatch(box, { subagent_type: "grok:grok-rescue", prompt: "bounded peer brief" })).stdout).hookSpecificOutput; + assert.strictEqual(denied.permissionDecision, "deny"); + assert.match(denied.permissionDecisionReason, /Host peer capacity reached/); + assert.ok(denied.permissionDecisionReason.endsWith(messageTag("worker-lifecycle.host-capacity-deny"))); + updateWorkerRecord("fusion-host-peer-0", envFor(box), (record) => ({ ...record, transportStatus: "done" })); + const allowed = run(box, dispatch(box, { subagent_type: "grok:grok-rescue", prompt: "bounded peer brief" })).stdout; + assert.ok(!allowed || JSON.parse(allowed).hookSpecificOutput.permissionDecision !== "deny"); +}); diff --git a/tests/host-capacity.test.mjs b/tests/host-capacity.test.mjs new file mode 100644 index 0000000..9e1c2f1 --- /dev/null +++ b/tests/host-capacity.test.mjs @@ -0,0 +1,69 @@ +import assert from "node:assert/strict"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { test } from "node:test"; + +import { hostPeerCapacity, hostPeerCapacityDenial, createDispatch } from "../plugins/fusion/scripts/lib/lifecycle/dispatch.mjs"; +import { buildFusionStats } from "../plugins/fusion/scripts/lib/stats/aggregation.mjs"; +import { renderFusionStats } from "../plugins/fusion/scripts/lib/stats/rendering.mjs"; +import { messageTag } from "../plugins/fusion/scripts/lib/user-messages.mjs"; +import { createWorkerRecord, readWorkerRecord, updateWorkerRecord } from "../plugins/fusion/scripts/lib/worker-state.mjs"; + +const GiB = 1024 ** 3; + +function sandbox(t) { + const root = fs.mkdtempSync(path.join(os.tmpdir(), "fusion-capacity-test-")); + t.after(() => fs.rmSync(root, { recursive: true, force: true })); + const cwd = path.join(root, "repo"); + fs.mkdirSync(cwd); + return { root, cwd, env: { ...process.env, FUSION_DATA_DIR: path.join(root, "data"), FUSION_WORKER_STATE_DIR: path.join(root, "data", "workers") } }; +} + +function worker(box, taskId, sessionId, agentType = "codex:codex-rescue", extra = {}) { + return createWorkerRecord({ taskId, sessionId, agentType, workspaceRoot: box.cwd, limits: { stallMs: 600_000 }, ...extra }, box.env); +} + +test("host peer capacity follows total physical memory with a floor of two", () => { + assert.equal(hostPeerCapacity(0), 2); + assert.equal(hostPeerCapacity(16 * GiB), 5); + assert.equal(hostPeerCapacity(64 * GiB), 21); +}); + +test("host capacity counts live peer wrappers across sessions and ignores stale and Fusion records", (t) => { + const box = sandbox(t); + for (let index = 0; index < 5; index += 1) { + worker(box, `fusion-peer-${index}`, index < 3 ? "session-a" : "session-b", index % 2 ? "grok:grok-rescue" : "codex:codex-rescue"); + } + worker(box, "fusion-claude", "session-c", "fusion:claude-worker"); + const now = Date.now(); + const denial = hostPeerCapacityDenial(box.env, now, 16 * GiB); + assert.match(denial, /5 peer wrapper jobs in flight \(capacity 5; total memory 16 GiB\)/); + assert.ok(denial.endsWith(messageTag("worker-lifecycle.host-capacity-deny"))); + assert.match(denial, /Relaunch after a completion notification/); + updateWorkerRecord("fusion-peer-0", box.env, (record) => ({ ...record, lastLivenessAt: new Date(now - 600_001).toISOString() })); + assert.equal(hostPeerCapacityDenial(box.env, now, 16 * GiB), null); + updateWorkerRecord("fusion-peer-1", box.env, (record) => ({ ...record, transportStatus: "done" })); + assert.equal(hostPeerCapacityDenial(box.env, now, 16 * GiB), null); +}); + +test("dispatch stores parent context tokens and stats render median and p90 for measured dispatches", (t) => { + const box = sandbox(t); + const transcript = path.join(box.root, "parent.jsonl"); + fs.writeFileSync(transcript, `${JSON.stringify({ type: "assistant", message: { usage: { input_tokens: 500, cache_read_input_tokens: 249_000, cache_creation_input_tokens: 500 } } })}\n`); + const dispatched = createDispatch({ session_id: "session-a", tool_use_id: "tool-context", cwd: box.cwd, transcript_path: transcript, tool_input: { prompt: "peer brief" } }, "codex:codex-rescue", false, box.env); + assert.equal(dispatched.parentContextTokensAtDispatch, 250_000); + assert.equal(readWorkerRecord(dispatched.taskId, box.env).parentContextTokensAtDispatch, 250_000); + for (const [index, tokens] of [100_000, 200_000, 400_000, 900_000].entries()) { + worker(box, `fusion-measured-${index}`, "session-b", "fusion:claude-worker", { parentContextTokensAtDispatch: tokens }); + } + worker(box, "fusion-unmeasured", "session-b", "fusion:claude-worker"); + const report = buildFusionStats({ all: true, cwd: box.cwd, env: box.env }); + assert.deepEqual(report.parentContextAtDispatch, { dispatchCount: 5, medianTokens: 250_000, p90Tokens: 900_000 }); + assert.match(renderFusionStats(report), /Parent context at dispatch: 5 measured; median 250000 tokens, p90 900000 tokens/); + const otherWorkspace = path.join(box.root, "other-repo"); + fs.mkdirSync(otherWorkspace); + createWorkerRecord({ taskId: "fusion-other-workspace", sessionId: "session-c", agentType: "fusion:claude-worker", workspaceRoot: otherWorkspace, parentContextTokensAtDispatch: 3_000_000 }, box.env); + assert.deepEqual(buildFusionStats({ cwd: box.cwd, env: box.env }).parentContextAtDispatch, report.parentContextAtDispatch); + assert.equal(buildFusionStats({ all: true, cwd: box.cwd, env: box.env }).parentContextAtDispatch.p90Tokens, 3_000_000); +}); From fb97104ee723da0444a3fbff4e06eeae12c010c1 Mon Sep 17 00:00:00 2001 From: Harry Yep Date: Mon, 26 Oct 2026 20:51:02 +0800 Subject: [PATCH 10/13] fix: keep grok's record predicate, report parent context only when measured, current lifecycle defaults --- plugins/fusion/rules/troubleshooting.md | 2 +- plugins/fusion/scripts/lib/stats/aggregation.mjs | 3 ++- plugins/grok/scripts/lib/state.mjs | 8 +------- 3 files changed, 4 insertions(+), 9 deletions(-) diff --git a/plugins/fusion/rules/troubleshooting.md b/plugins/fusion/rules/troubleshooting.md index faa78cc..c5af3de 100644 --- a/plugins/fusion/rules/troubleshooting.md +++ b/plugins/fusion/rules/troubleshooting.md @@ -62,7 +62,7 @@ Codex exec refuses to start outside a directory with an ancestor `.git` entry. T `/fusion:stats --record` writes the verdict back to the Codex job record through the companion's `record-acceptance` subcommand. Rejected verdicts require `--reason` via the raw args transport, and judgment shaped rejections may add `--failure-kind` (`intent_override`, `scope_rewrite`, `wrong_approach`, `style_mismatch`). A wind down meta report or empty delta is a sizing failure, recorded as `--failure-kind oversized` so brief sizing errors stay separable from worker judgment errors. Recording `accepted` for a job whose transport failed requires the explicit `--accept-failed-transport` override. The stats report includes an acceptance anomalies block when transport and acceptance evidence conflict or when other acceptance integrity checks find a problem. -Worker lifecycle defaults for `fusion:claude-worker` are 1200000ms of wall clock, 600000ms of stall time, 96000 output tokens, 720000 uncached tokens, and 60 turns; `fusion:deep-reasoner` runs 1200000ms wall clock, 900000ms stall, and 64000 output tokens because its whole deliverable is text. The stall clock measures time since the last successful tool call, so it is checked against sweeps only: a worker that surfaces with a tool call refreshes its own liveness at the gate instead of being denied retroactively for a long composition turn. The Stop hook adds a non-blocking advisory when collected workers remain acceptance unverified. +Worker lifecycle defaults for `fusion:claude-worker` and the peer wrappers are 2400000ms of wall clock, 600000ms of stall time, 192000 output tokens, 960000 uncached tokens, and 100 turns; `fusion:deep-reasoner` runs 2400000ms wall clock, 900000ms stall, and 128000 output tokens because its whole deliverable is text. A SendMessage continuation measures every budget, peer wrappers included, from the reopen. The stall clock measures time since the last successful tool call, so it is checked against sweeps only: a worker that surfaces with a tool call refreshes its own liveness at the gate instead of being denied retroactively for a long composition turn. The Stop hook adds a non-blocking advisory when collected workers remain acceptance unverified. ## Cursor headless guardrails diff --git a/plugins/fusion/scripts/lib/stats/aggregation.mjs b/plugins/fusion/scripts/lib/stats/aggregation.mjs index a02fbc9..648712b 100644 --- a/plugins/fusion/scripts/lib/stats/aggregation.mjs +++ b/plugins/fusion/scripts/lib/stats/aggregation.mjs @@ -592,9 +592,10 @@ export function buildFusionStats({ all = false, env = process.env, cwd = process engines[provider.id] = provider.collect(options); } const fleet = buildFleetUsageStats({ env }); + const parentContext = parentContextAtDispatch(workers.records, all, workspaceRoot); return { scope: all ? "all" : workspaceRoot, - parentContextAtDispatch: parentContextAtDispatch(workers.records, all, workspaceRoot), + ...(parentContext.dispatchCount > 0 ? { parentContextAtDispatch: parentContext } : {}), ...engines, ...(all ? { retiredWorkers: retiredWorkerStats(env) } : {}), fleet, diff --git a/plugins/grok/scripts/lib/state.mjs b/plugins/grok/scripts/lib/state.mjs index fa241a0..fd08037 100644 --- a/plugins/grok/scripts/lib/state.mjs +++ b/plugins/grok/scripts/lib/state.mjs @@ -307,13 +307,7 @@ const JOB_RECORD_FIELDS = ["jobClass", "delivery", "semanticStatus", "request", function assertJobRecord(record) { if (!record || typeof record !== "object" || Array.isArray(record) || record.schemaVersion !== 1 || record.engine !== "grok" - || typeof record.id !== "string" || !record.id - || !TERMINAL_STATUSES.has(record.status) && record.status !== "running" - || typeof record.mode !== "string" || !record.mode - || typeof record.cwd !== "string" || !record.cwd - || typeof record.createdAt !== "string" || !Number.isFinite(Date.parse(record.createdAt)) - || typeof record.background !== "boolean" - || record.request != null && (typeof record.request !== "object" || Array.isArray(record.request)) + || typeof record.id !== "string" || JOB_RECORD_FIELDS.some((field) => !Object.hasOwn(record, field))) { throw new TypeError("Job record schema is invalid."); } From 9f7b3e2c2b5365a59c266d46a529882b94fddb2a Mon Sep 17 00:00:00 2001 From: Harry Yep Date: Mon, 26 Oct 2026 20:53:22 +0800 Subject: [PATCH 11/13] chore: release 0.0.67 --- .claude-plugin/marketplace.json | 10 +++++----- CHANGELOG.md | 14 ++++++++++++++ plugins/codex/.claude-plugin/plugin.json | 2 +- plugins/cursor/.claude-plugin/plugin.json | 2 +- plugins/fusion/.claude-plugin/plugin.json | 2 +- plugins/grok/.claude-plugin/plugin.json | 2 +- 6 files changed, 23 insertions(+), 9 deletions(-) diff --git a/.claude-plugin/marketplace.json b/.claude-plugin/marketplace.json index 36328d2..2384ebb 100644 --- a/.claude-plugin/marketplace.json +++ b/.claude-plugin/marketplace.json @@ -6,14 +6,14 @@ "email": "hi@okis.dev" }, "description": "Multi-model orchestration marketplace for Claude Code.", - "version": "0.0.66", + "version": "0.0.67", "plugins": [ { "name": "grok", "source": "./plugins/grok", "displayName": "Grok Companion", "description": "Local Grok CLI delegation: task, review, resumable history, best-of-n tournaments, background jobs, stats, and setup health checks.", - "version": "0.0.66", + "version": "0.0.67", "author": { "name": "Harry Yep" }, @@ -34,7 +34,7 @@ "source": "./plugins/codex", "displayName": "Codex Companion", "description": "First party local Codex CLI delegation for tasks, reviews, resumable threads, and durable background jobs.", - "version": "0.0.66", + "version": "0.0.67", "author": { "name": "Harry Yep" }, @@ -55,7 +55,7 @@ "source": "./plugins/cursor", "displayName": "Cursor Companion", "description": "Local Cursor Agent CLI delegation: multi family consults, Composer implementation in owned worktrees, review, resumable history, background jobs, and setup health checks.", - "version": "0.0.66", + "version": "0.0.67", "author": { "name": "Harry Yep" }, @@ -76,7 +76,7 @@ "source": "./plugins/fusion", "displayName": "Fusion Orchestrator", "description": "Multi-model orchestration: tier agents, routing rules, blind panel, ultra fleet, model config, and drift doctor.", - "version": "0.0.66", + "version": "0.0.67", "author": { "name": "Harry Yep" }, diff --git a/CHANGELOG.md b/CHANGELOG.md index 1999dc5..0e2e37a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,19 @@ # changelog +## 0.0.67 + +- this release acts on the call records of a second host on 0.0.65: 24 sessions, 326 peer jobs, and 342 worker records since its upgrade. +- codex and grok job records written by older companions are migrated to the current schema on first read and rewritten under the record lock, instead of being rejected. 260 of 438 codex and 12 of 64 grok records on that host had become unreadable to `record-acceptance`, `/codex:result`, history, and retention. +- engine settlement runs the companion from its own directory, so a verdict queued for a worktree package still lands after the worktree is removed (all 43 "upgrade the plugin" settlement errors were this), and a spawn failure is reported as itself. the queued verdict path and `/fusion:stats --record` share one salvage rule, so an accepted salvaged timeout no longer fails with "pass --accept-failed-transport" (29 cases). settlement prefers the engine companion of fusion's own version, then the highest version, instead of the newest file. +- a SendMessage continuation of a peer wrapper measures its budget from the reopen; 33 of 40 continued codex wrappers had been stopped as over their wall clock right after continuing. +- grok `independence` and `live-web` dispatches get their own `high` effort default; sharing burst's model gave them `low`. cursor and grok role fields are recognized after a `·`, comma, or semicolon separator, so a cursor diversity brief no longer starts without a model. +- fusion denies a peer launch while all sessions on the host already run its peer capacity (one peer per 3 gib of memory, at least two). the host had panicked from memory exhaustion with up to 19 peer jobs in flight. +- the inline guard credits verification through the shared evidence rules (vitest, jest, pytest, and node test summaries) and treats `&&` chains and pipefail pipelines as exit observable; it had credited 30 of 1,136 passing verification commands. an Agent launched as `fork`, `general-purpose`, or `claude` gets an advisory that its work runs outside fusion collection. +- codex classifies upstream connectivity failures (workspace routing discovery timeouts, dropped streams, connection errors) as `network`, an infrastructure death, instead of `error`. grok records the message inside a multi line `Internal error: {` instead of its first line. +- `/fusion:stats` reports the median and p90 parent context at dispatch, and the parent context advisory defaults to 250,000 tokens (hosts compacting at 400,000 never reached 600,000). +- the jev brief sensor drops its mixed concern check: all 96 of its advisories were that check, and flagged dispatches were accepted as often as unflagged ones (89.2% against 90.5%). +- the rules say grok's strict sandbox is not a flag to pass (5 of 23 grok dispatches had failed on `--sandbox`), chain a wave's settlement behind its verification in one call, wait on ci with one background command instead of sleep turns, carry host constraints such as no local builds into peer briefs, and keep files a cursor package reads inside its workspace. + ## 0.0.66 - the orchestration rules give the rendered browser pass of a UI package to the orchestrator after merge, and a peer brief says so: of 24 dispatched codex runs that loaded a browser verification skill, none produced a screenshot, and 14 stopped at `listen EPERM` when their sandbox refused the dev server port. diff --git a/plugins/codex/.claude-plugin/plugin.json b/plugins/codex/.claude-plugin/plugin.json index 46955ae..388d88c 100644 --- a/plugins/codex/.claude-plugin/plugin.json +++ b/plugins/codex/.claude-plugin/plugin.json @@ -2,7 +2,7 @@ "$schema": "https://json.schemastore.org/claude-code-plugin-manifest.json", "name": "codex", "displayName": "Codex Companion", - "version": "0.0.66", + "version": "0.0.67", "description": "First party local Codex CLI delegation for tasks, reviews, resumable threads, and durable background jobs.", "author": { "name": "Harry Yep" diff --git a/plugins/cursor/.claude-plugin/plugin.json b/plugins/cursor/.claude-plugin/plugin.json index 9106d74..5ce1dab 100644 --- a/plugins/cursor/.claude-plugin/plugin.json +++ b/plugins/cursor/.claude-plugin/plugin.json @@ -2,7 +2,7 @@ "$schema": "https://json.schemastore.org/claude-code-plugin-manifest.json", "name": "cursor", "displayName": "Cursor Companion", - "version": "0.0.66", + "version": "0.0.67", "description": "Local Cursor Agent CLI delegation: multi family consults, Composer implementation in owned worktrees, review, resumable history, background jobs, and setup health checks.", "author": { "name": "Harry Yep" diff --git a/plugins/fusion/.claude-plugin/plugin.json b/plugins/fusion/.claude-plugin/plugin.json index d96dd58..9209a94 100644 --- a/plugins/fusion/.claude-plugin/plugin.json +++ b/plugins/fusion/.claude-plugin/plugin.json @@ -2,7 +2,7 @@ "$schema": "https://json.schemastore.org/claude-code-plugin-manifest.json", "name": "fusion", "displayName": "Fusion Orchestrator", - "version": "0.0.66", + "version": "0.0.67", "description": "Multi-model orchestration: tier agents, routing rules, blind panel, ultra fleet, model config, and drift doctor.", "author": { "name": "Harry Yep" diff --git a/plugins/grok/.claude-plugin/plugin.json b/plugins/grok/.claude-plugin/plugin.json index e96b5c4..3b7e737 100644 --- a/plugins/grok/.claude-plugin/plugin.json +++ b/plugins/grok/.claude-plugin/plugin.json @@ -2,7 +2,7 @@ "$schema": "https://json.schemastore.org/claude-code-plugin-manifest.json", "name": "grok", "displayName": "Grok Companion", - "version": "0.0.66", + "version": "0.0.67", "description": "Local Grok CLI delegation: task, review, resumable history, background jobs, stats, and setup health checks.", "author": { "name": "Harry Yep" From bac2075acbe48146facb90e9fc6495dc7f570730 Mon Sep 17 00:00:00 2001 From: Harry Yep Date: Mon, 26 Oct 2026 21:07:42 +0800 Subject: [PATCH 12/13] fix(fusion): reserve host peer capacity per launch and continuation, tighten review edge cases --- plugins/codex/scripts/lib/codex-outcome.mjs | 2 +- plugins/fusion/scripts/breaker-check.mjs | 2 +- .../fusion/scripts/lib/lifecycle/dispatch.mjs | 49 ++++++++++- .../fusion/scripts/lib/lifecycle/tools.mjs | 55 +++++++----- .../scripts/lib/verification-command.mjs | 4 +- plugins/fusion/scripts/lib/worker-state.mjs | 35 ++++++++ plugins/grok/scripts/lib/state.mjs | 5 +- tests/breaker-check.test.mjs | 5 ++ tests/codex-exec.test.mjs | 7 ++ tests/failure-kinds.test.mjs | 2 + tests/host-capacity.test.mjs | 83 ++++++++++++++++++- tests/record-migration.test.mjs | 13 +++ tests/verification-command.test.mjs | 4 + 13 files changed, 234 insertions(+), 32 deletions(-) diff --git a/plugins/codex/scripts/lib/codex-outcome.mjs b/plugins/codex/scripts/lib/codex-outcome.mjs index 57f7817..8239577 100644 --- a/plugins/codex/scripts/lib/codex-outcome.mjs +++ b/plugins/codex/scripts/lib/codex-outcome.mjs @@ -48,7 +48,7 @@ function classifiedFailureKind(message, fallback = "error") { if (/unauthori[sz]ed|unauthenticated|authentication|login required|invalid api key|api key/i.test(text)) { return "auth"; } - if (/workspace routing discovery timed out|stream disconnected before completion|error sending request|connection (?:refused|reset)|\b(?:ECONNRESET|ECONNREFUSED|ENOTFOUND|EAI_AGAIN)\b|getaddrinfo|network is unreachable|Reconnecting\.\.\./i.test(text)) { + if (/workspace routing discovery timed out|stream disconnected before completion|error sending request for url|connection (?:refused|reset)|\b(?:ECONNRESET|ECONNREFUSED|ENOTFOUND|EAI_AGAIN)\b|getaddrinfo|network is unreachable|Reconnecting\.\.\./i.test(text)) { return "network"; } if (/permission denied|operation not permitted|approval (?:required|denied)|sandbox (?:denied|violation)/i.test(text)) { diff --git a/plugins/fusion/scripts/breaker-check.mjs b/plugins/fusion/scripts/breaker-check.mjs index d81b8f0..7b902ec 100644 --- a/plugins/fusion/scripts/breaker-check.mjs +++ b/plugins/fusion/scripts/breaker-check.mjs @@ -26,7 +26,7 @@ const CODEX_FAILURE_PATTERNS = [ ["rate_limited", /rate[ _-]?limit|too many requests|\b429\b/i], ["refused", /misalignment policy|cyber policy|content policy|reason: content_filter|flagged for possible (?:cybersecurity|biological) risk/i], ["auth", /unauthori[sz]ed|unauthenticated|authentication|auth(?:entication)? failed|invalid api key|api key/i], - ["network", /workspace routing discovery timed out|stream disconnected before completion|error sending request|connection (?:refused|reset)|\b(?:ECONNRESET|ECONNREFUSED|ENOTFOUND|EAI_AGAIN)\b|getaddrinfo|network is unreachable|Reconnecting\.\.\./i], + ["network", /workspace routing discovery timed out|stream disconnected before completion|error sending request for url|connection (?:refused|reset)|\b(?:ECONNRESET|ECONNREFUSED|ENOTFOUND|EAI_AGAIN)\b|getaddrinfo|network is unreachable|Reconnecting\.\.\./i], ["missing_cli", /missing[ _-]?cli|command not found|cli not found|no such file/i], ["timeout", /timed?\s*out|timeout/i], ["stall", /\bstall(?:ed|ing)?\b|no[- ]progress/i], diff --git a/plugins/fusion/scripts/lib/lifecycle/dispatch.mjs b/plugins/fusion/scripts/lib/lifecycle/dispatch.mjs index b7647f9..58c8a0b 100644 --- a/plugins/fusion/scripts/lib/lifecycle/dispatch.mjs +++ b/plugins/fusion/scripts/lib/lifecycle/dispatch.mjs @@ -6,7 +6,7 @@ import { fileURLToPath } from "node:url"; import { ENGINE_ID_ALTERNATION, PEER_JOB_FOOTER_AGENTS, engineForAgentType, engineRoleFromBrief } from "../engines.mjs"; import { readRoutingDefaults } from "../model-table.mjs"; import { tagMessage } from "../user-messages.mjs"; -import { canonicalWorkerAgentType, createWorkerRecord, createWorkerTaskId, readWorkerRecords, updateWorkerSessionState, workerRecordFile, writePrivateText, WORKER_TERMINAL_STATUSES } from "../worker-state.mjs"; +import { canonicalWorkerAgentType, createWorkerRecord, createWorkerTaskId, readPeerCapacityReservations, readWorkerRecords, updateWorkerSessionState, withPeerCapacityReservations, workerRecordFile, writePrivateText, WORKER_TERMINAL_STATUSES } from "../worker-state.mjs"; import { workerLimits } from "./budget.mjs"; import { updateLifecycleWorkerRecord } from "./records.mjs"; import { PARENT_CONTEXT_ADVISORY_TOKENS_ENV, VERIFICATION_MANIFEST_ENV, DEFAULT_BRIEF_MAX_BYTES, SIZING_ADVISORY_BYTES, DEFAULT_PARENT_CONTEXT_ADVISORY_TOKENS, PARENT_CONTEXT_TAIL_MAX_BYTES, BRIEF_AGENTS } from "./constants.mjs"; @@ -103,12 +103,23 @@ function hostPeerCapacity(totalMemoryBytes = os.totalmem()) { return Math.max(2, Math.floor(totalMemoryBytes / (3 * 1024 ** 3))); } -function hostPeerCapacityDenial(env, now = Date.now(), totalMemoryBytes = os.totalmem()) { +function livePeerReservations(reservations, env, now) { + return reservations.filter((reservation) => PEER_RESCUE_AGENT_NAMES.has(reservation.agentType) + && Number.isFinite(reservation.at) && reservation.at <= now + && now - reservation.at <= workerLimits(reservation.agentType, env).wallClockMs); +} + +function hostPeerCapacityDenial(env, now = Date.now(), totalMemoryBytes = os.totalmem(), reservations = readPeerCapacityReservations(env)) { const capacity = hostPeerCapacity(totalMemoryBytes); - const inFlight = readWorkerRecords(env, { strict: true }).filter((record) => { + const liveReservations = livePeerReservations(reservations, env, now); + const inFlight = liveReservations.length + readWorkerRecords(env, { strict: true }).filter((record) => { if (!PEER_RESCUE_AGENT_NAMES.has(record.agentType) || WORKER_TERMINAL_STATUSES.has(record.transportStatus)) { return false; } + if (liveReservations.some((reservation) => reservation.taskId === record.taskId + || typeof reservation.toolUseId === "string" && reservation.sessionId === record.sessionId && reservation.toolUseId === record.dispatchToolUseId)) { + return false; + } const livenessAt = Date.parse(record.lastLivenessAt ?? record.startedAt ?? record.createdAt ?? ""); const stallMs = Number.isSafeInteger(record.limits?.stallMs) && record.limits.stallMs > 0 ? record.limits.stallMs @@ -122,6 +133,36 @@ function hostPeerCapacityDenial(env, now = Date.now(), totalMemoryBytes = os.tot return tagMessage("worker-lifecycle.host-capacity-deny", `Host peer capacity reached: ${inFlight} peer wrapper jobs in flight (capacity ${capacity}; total memory ${totalGiB} GiB). Relaunch after a completion notification.`); } +function peerReservationKey(input) { + return JSON.stringify([input.session_id, input.tool_use_id ?? `${input.tool_name}:${input.tool_input?.to ?? input.tool_input?.subagent_type}`]); +} + +function reserveHostPeerCapacity(input, agentType, env, taskId = null, now = Date.now(), totalMemoryBytes = os.totalmem()) { + return withPeerCapacityReservations(env, (reservations) => { + const live = livePeerReservations(reservations, env, now); + reservations.splice(0, reservations.length, ...live); + const key = peerReservationKey(input); + if (reservations.some((reservation) => reservation.key === key)) { + return null; + } + const denial = hostPeerCapacityDenial(env, now, totalMemoryBytes, reservations); + if (!denial) { + reservations.push({ key, sessionId: input.session_id, toolUseId: input.tool_use_id ?? null, taskId, agentType, at: now }); + } + return denial; + }); +} + +function releaseHostPeerCapacity(input, env) { + return withPeerCapacityReservations(env, (reservations) => { + const key = peerReservationKey(input); + const index = reservations.findIndex((reservation) => reservation.key === key); + if (index !== -1) { + reservations.splice(index, 1); + } + }); +} + function parentContextTokens(transcriptPath) { if (typeof transcriptPath !== "string" || !path.isAbsolute(transcriptPath)) { return null; @@ -497,4 +538,4 @@ function briefArtifactPath(record, env) { } -export { allowAgentWithTaskId, promptText, leadingCompanionOptions, optionValue, injectedPeerDefaults, parentContextAdvisoryTokens, parentContextTokens, hostPeerCapacity, hostPeerCapacityDenial, verificationManifestPath, globToRegExp, readVerificationManifest, pathsNamedInBrief, missingVerificationSuites, briefAdvisories, validateWorkerBrief, externalUserText, latestUserRequestedBackground, safeDescription, completionContract, collectorRequestIdentity, createDispatch, claimParentContextAdvisory, briefArtifactPath }; +export { allowAgentWithTaskId, promptText, leadingCompanionOptions, optionValue, injectedPeerDefaults, parentContextAdvisoryTokens, parentContextTokens, hostPeerCapacity, hostPeerCapacityDenial, reserveHostPeerCapacity, releaseHostPeerCapacity, verificationManifestPath, globToRegExp, readVerificationManifest, pathsNamedInBrief, missingVerificationSuites, briefAdvisories, validateWorkerBrief, externalUserText, latestUserRequestedBackground, safeDescription, completionContract, collectorRequestIdentity, createDispatch, claimParentContextAdvisory, briefArtifactPath }; diff --git a/plugins/fusion/scripts/lib/lifecycle/tools.mjs b/plugins/fusion/scripts/lib/lifecycle/tools.mjs index b72b11d..04a142b 100644 --- a/plugins/fusion/scripts/lib/lifecycle/tools.mjs +++ b/plugins/fusion/scripts/lib/lifecycle/tools.mjs @@ -5,7 +5,7 @@ import { tagMessage } from "../user-messages.mjs"; import { canonicalWorkerAgentType, isFusionWorkerAgent, WORKER_TERMINAL_STATUSES, markWorkerCollected, readWorkerRecord, readWorkerOutputFileTaskIds, readSessionWorkerRecords, refreshWorkerTranscript, reopenWorkerContinuation, WORKER_COLLECTION_METHODS } from "../worker-state.mjs"; import { markBudgetFailure, workerBudgetFailure, workerRoundUsage } from "./budget.mjs"; import { backfillCollectedTaskOutput, capturedPeerIdentity, collectedHarnessAsyncDelivery, noTaskFoundError, outputFileFromLaunchResponse, peerIdentityBackfill, settleQueuedVerdict, settleReapedIfActive, taskOutputResultText, terminalCollectedRecord, terminalCollectionPending } from "./collection.mjs"; -import { allowAgentWithTaskId, claimParentContextAdvisory, collectorRequestIdentity, createDispatch, hostPeerCapacityDenial, injectedPeerDefaults, latestUserRequestedBackground, parentContextTokens, promptText, validateWorkerBrief } from "./dispatch.mjs"; +import { allowAgentWithTaskId, claimParentContextAdvisory, collectorRequestIdentity, createDispatch, injectedPeerDefaults, latestUserRequestedBackground, parentContextTokens, promptText, releaseHostPeerCapacity, reserveHostPeerCapacity, validateWorkerBrief } from "./dispatch.mjs"; import { reconcileTaskNotifications, taskNotificationTransition } from "./notifications.mjs"; import { denyTool, hookOutput, writeOutput } from "./output.mjs"; import { activeSessionRecords, clearContinuationRequest, findSessionWorkerRecord, isAttributedTranscriptPath, parentTranscriptByteSize, recordForAgent, refreshRecord, sendMessageResumed, updateLifecycleWorkerRecord } from "./records.mjs"; @@ -36,7 +36,7 @@ function handlePreToolUse(input, env) { writeOutput(denyTool(tagMessage("worker-lifecycle.foreground-wrapper-deny", `${engineDisplayNameList()} wrapper Agents use foreground delivery. Keep the Agent foreground; an explicitly requested companion \`--background\` receipt stays inside that foreground wrapper call.`))); return; } - const capacityDenial = hostPeerCapacityDenial(env); + const capacityDenial = reserveHostPeerCapacity(input, agentType, env); if (capacityDenial) { writeOutput(denyTool(capacityDenial)); return; @@ -83,6 +83,13 @@ function handlePreToolUse(input, env) { const recipient = input.tool_input.to; const record = findSessionWorkerRecord((candidate) => (PEER_RESCUE_AGENT_NAMES.has(candidate.agentType) || isFusionWorkerAgent(candidate.agentType)) && [candidate.agentId, candidate.backgroundTaskId].includes(recipient), input.session_id, env); if (record) { + if (PEER_RESCUE_AGENT_NAMES.has(record.agentType) && WORKER_TERMINAL_STATUSES.has(record.transportStatus)) { + const capacityDenial = reserveHostPeerCapacity(input, record.agentType, env, record.taskId); + if (capacityDenial) { + writeOutput(denyTool(capacityDenial)); + return; + } + } updateLifecycleWorkerRecord(record.taskId, env, (current) => { if (!current || current.sessionId !== input.session_id || ![current.agentId, current.backgroundTaskId].includes(recipient)) { return null; @@ -175,11 +182,15 @@ function handlePostToolUse(input, env, failed = false) { return current.continuationRequestedAt ? clearContinuationRequest(next) : next === current ? null : next; }); } + releaseHostPeerCapacity(input, env); if (!failed) { reconcileTaskNotifications(input, env); } return; } + if (failed && ["Agent", "Task"].includes(input.tool_name) && PEER_RESCUE_AGENT_NAMES.has(input.tool_input?.subagent_type)) { + releaseHostPeerCapacity(input, env); + } if (failed && ["Agent", "Task"].includes(input.tool_name)) { const failureText = [input.error, input.tool_response].flatMap((value) => { if (typeof value === "string") { @@ -291,25 +302,29 @@ function handlePostToolUse(input, env, failed = false) { if (failed) { return; } - const response = input.tool_response && typeof input.tool_response === "object" ? input.tool_response : {}; - const nested = response.toolUseResult && typeof response.toolUseResult === "object" ? response.toolUseResult : {}; - const isAsync = response.isAsync === true || nested.isAsync === true || response.status === "async_launched" || nested.status === "async_launched"; - if (!isAsync) { - return; + try { + const response = input.tool_response && typeof input.tool_response === "object" ? input.tool_response : {}; + const nested = response.toolUseResult && typeof response.toolUseResult === "object" ? response.toolUseResult : {}; + const isAsync = response.isAsync === true || nested.isAsync === true || response.status === "async_launched" || nested.status === "async_launched"; + if (!isAsync) { + return; + } + const agentId = typeof response.agentId === "string" ? response.agentId : typeof nested.agentId === "string" ? nested.agentId : null; + const outputFile = outputFileFromLaunchResponse(response, nested); + const now = new Date().toISOString(); + const record = createDispatch(input, agentType, false, env); + updateLifecycleWorkerRecord(record.taskId, env, (current) => ({ + ...current, + ...(agentId ? { agentId, backgroundTaskId: agentId } : {}), + ...(outputFile ? { transcriptPath: outputFile } : {}), + transportStatus: "pending_async", + runtimeAsync: true, + failureKind: "unexpected_async", + startedAt: current.startedAt ?? now + })); + } finally { + releaseHostPeerCapacity(input, env); } - const agentId = typeof response.agentId === "string" ? response.agentId : typeof nested.agentId === "string" ? nested.agentId : null; - const outputFile = outputFileFromLaunchResponse(response, nested); - const now = new Date().toISOString(); - const record = createDispatch(input, agentType, false, env); - updateLifecycleWorkerRecord(record.taskId, env, (current) => ({ - ...current, - ...(agentId ? { agentId, backgroundTaskId: agentId } : {}), - ...(outputFile ? { transcriptPath: outputFile } : {}), - transportStatus: "pending_async", - runtimeAsync: true, - failureKind: "unexpected_async", - startedAt: current.startedAt ?? now - })); return; } if (!isFusionWorkerAgent(agentType)) { diff --git a/plugins/fusion/scripts/lib/verification-command.mjs b/plugins/fusion/scripts/lib/verification-command.mjs index fe97724..6b36ca6 100644 --- a/plugins/fusion/scripts/lib/verification-command.mjs +++ b/plugins/fusion/scripts/lib/verification-command.mjs @@ -120,7 +120,7 @@ const PASS_SUMMARY_PATTERNS = [ const FAILURE_SUMMARY_PATTERNS = [ /^\s*(?:✖\s|not ok\b)/im, - /^\s*(?:ℹ\s+)?fail\s+[1-9]\d*\b/im, + /^\s*(?:ℹ\s+|#\s+)fail\s+[1-9]\d*\b/im, /[1-9]\d*\s+fail(?:ed|ures?)\b/i, /^\s*FAIL\b/m, /[1-9]\d*\s+error(s)?\b/i @@ -302,7 +302,7 @@ function verificationCommand(command, env = process.env) { const lastVerificationIndex = matchingSegments.lastIndexOf(true); let pipefail = false; const pipefailAtSegment = segments.map((segment, index) => { - if (/^set\s+-[a-z]*o\s+pipefail$/i.test(segment.command) && segments[index + 1]?.separator !== "|") { + if (/^set\s+-[a-z]*o\s+pipefail$/i.test(segment.command) && segments[index + 1]?.separator !== "|" && segments.slice(1, index + 1).every((previous) => [";", "\n", "\r", "&&"].includes(previous.separator))) { pipefail = true; } else if (/^set\s+\+(?:[a-z]*o)\s+pipefail$/i.test(segment.command)) { pipefail = false; diff --git a/plugins/fusion/scripts/lib/worker-state.mjs b/plugins/fusion/scripts/lib/worker-state.mjs index 8b03780..30215db 100644 --- a/plugins/fusion/scripts/lib/worker-state.mjs +++ b/plugins/fusion/scripts/lib/worker-state.mjs @@ -146,6 +146,41 @@ export function resolveFusionDataDir(env = process.env) { return path.join(os.homedir(), ".claude", "plugins", "data", "fusion-claude-code-fusion"); } +function peerCapacityReservationsFile(env) { + return path.join(resolveFusionDataDir(env), "peer-capacity-reservations.json"); +} + +export function readPeerCapacityReservations(env = process.env) { + let text; + try { + text = fs.readFileSync(peerCapacityReservationsFile(env), "utf8"); + } catch (error) { + if (error?.code === "ENOENT") { + return []; + } + throw error; + } + try { + const reservations = JSON.parse(text); + return Array.isArray(reservations) ? reservations.filter((entry) => entry && typeof entry === "object") : []; + } catch { + return []; + } +} + +export function withPeerCapacityReservations(env, callback) { + const file = peerCapacityReservationsFile(env); + return withLock(file, () => { + const reservations = readPeerCapacityReservations(env); + const before = JSON.stringify(reservations); + const result = callback(reservations); + if (JSON.stringify(reservations) !== before) { + writePrivateFile(file, `${JSON.stringify(reservations)}\n`); + } + return result; + }); +} + export function resolveWorkerStateDir(env = process.env) { const override = env[WORKER_STATE_ENV]; if (override && String(override).trim()) { diff --git a/plugins/grok/scripts/lib/state.mjs b/plugins/grok/scripts/lib/state.mjs index fd08037..fa2e3a2 100644 --- a/plugins/grok/scripts/lib/state.mjs +++ b/plugins/grok/scripts/lib/state.mjs @@ -24,6 +24,7 @@ const STATE_SWEEP_INTERVAL_MS = 60 * 60 * 1000; const PRIVATE_DIR_MODE = 0o700; const PRIVATE_FILE_MODE = 0o600; const TERMINAL_STATUSES = new Set(["done", "error", "cancelled"]); +const JOB_STATUSES = new Set([...TERMINAL_STATUSES, "running", "queued"]); export const SESSION_ID_ENV = "CLAUDE_CODE_SESSION_ID"; export const DEFAULT_LOCK_TIMEOUT_MS = 5000; @@ -307,7 +308,7 @@ const JOB_RECORD_FIELDS = ["jobClass", "delivery", "semanticStatus", "request", function assertJobRecord(record) { if (!record || typeof record !== "object" || Array.isArray(record) || record.schemaVersion !== 1 || record.engine !== "grok" - || typeof record.id !== "string" + || typeof record.id !== "string" || !JOB_STATUSES.has(record.status) || JOB_RECORD_FIELDS.some((field) => !Object.hasOwn(record, field))) { throw new TypeError("Job record schema is invalid."); } @@ -316,7 +317,7 @@ function assertJobRecord(record) { export function migrateJobRecord(raw) { if (!raw || typeof raw !== "object" || Array.isArray(raw) || raw.engine != null && raw.engine !== "grok" - || raw.schemaVersion != null && raw.schemaVersion !== 1) return raw; + || raw.schemaVersion != null && raw.schemaVersion !== 1 || !JOB_STATUSES.has(raw.status)) return raw; if (raw.schemaVersion === 1 && raw.engine === "grok" && JOB_RECORD_FIELDS.every((field) => Object.hasOwn(raw, field))) return raw; if (typeof raw.id !== "string" || !raw.id || typeof raw.mode !== "string" diff --git a/tests/breaker-check.test.mjs b/tests/breaker-check.test.mjs index 6c9b03f..f8e4bd7 100644 --- a/tests/breaker-check.test.mjs +++ b/tests/breaker-check.test.mjs @@ -213,6 +213,11 @@ test("breaker quota recovery matches the companion diagnostic vocabulary", () => } }); +test("unsupported Codex model errors are not network failures", () => { + assert.strictEqual(codexFailureKind("Error sending request: model gpt-5 is not supported"), null); + assert.strictEqual(codexFailureKind("error sending request for url (https://example.test): connection failed"), "network"); +}); + test("typed grok failures are not reclassified from quota summaries or logs", (t) => { const sandbox = makeSandbox(t); const stateRoot = path.join(sandbox.grokData, "state"); diff --git a/tests/codex-exec.test.mjs b/tests/codex-exec.test.mjs index 64b8d50..139da94 100644 --- a/tests/codex-exec.test.mjs +++ b/tests/codex-exec.test.mjs @@ -813,6 +813,7 @@ test("terminal Codex connectivity diagnostics classify as network", () => { "Reconnecting... (workspace routing discovery timed out)", "stream disconnected before completion: upstream closed the stream", "error sending request: connection refused", + "error sending request for url (https://example.test): request failed", "ECONNRESET", "ECONNREFUSED", "ENOTFOUND", "EAI_AGAIN", "getaddrinfo failed", "network is unreachable" ]) { const outcome = failureOutcome({ terminalEvent: { type: "turn.failed" }, turnFailureMessage: message }, { exitCode: 1 }, ""); @@ -824,6 +825,12 @@ test("terminal Codex connectivity diagnostics classify as network", () => { assert.equal(stderrOnly.failureKind, "network"); }); +test("an unsupported Codex model is not classified as a network failure", () => { + const message = "Error sending request: model gpt-5 is not supported"; + const outcome = failureOutcome({ terminalEvent: { type: "turn.failed" }, turnFailureMessage: message }, { exitCode: 1 }, ""); + assert.equal(outcome.failureKind, "error"); +}); + test("Codex quota, rate limit, refusal, and auth evidence precedes network evidence", () => { for (const [message, expected] of [ ["quota exhausted; Reconnecting...", "quota"], diff --git a/tests/failure-kinds.test.mjs b/tests/failure-kinds.test.mjs index b1edae6..b5d0b89 100644 --- a/tests/failure-kinds.test.mjs +++ b/tests/failure-kinds.test.mjs @@ -103,6 +103,8 @@ for (const [message, failureKind] of [ ["Reconnecting... (workspace routing discovery timed out)", "network"], ["stream disconnected before completion: upstream connection closed", "network"], ["error sending request: connection reset", "network"], + ["error sending request for url (https://example.test): request failed", "network"], + ["Error sending request: model gpt-5 is not supported", "error"], ["network is unreachable", "network"], ["This request has been flagged for possible cybersecurity risk.", "refused"], ["This content was flagged for possible biological risk.", "refused"], diff --git a/tests/host-capacity.test.mjs b/tests/host-capacity.test.mjs index 9e1c2f1..6fbd79f 100644 --- a/tests/host-capacity.test.mjs +++ b/tests/host-capacity.test.mjs @@ -4,11 +4,12 @@ import os from "node:os"; import path from "node:path"; import { test } from "node:test"; -import { hostPeerCapacity, hostPeerCapacityDenial, createDispatch } from "../plugins/fusion/scripts/lib/lifecycle/dispatch.mjs"; +import { hostPeerCapacity, hostPeerCapacityDenial, createDispatch, reserveHostPeerCapacity } from "../plugins/fusion/scripts/lib/lifecycle/dispatch.mjs"; +import { handlePreToolUse, handlePostToolUse } from "../plugins/fusion/scripts/lib/lifecycle/tools.mjs"; import { buildFusionStats } from "../plugins/fusion/scripts/lib/stats/aggregation.mjs"; import { renderFusionStats } from "../plugins/fusion/scripts/lib/stats/rendering.mjs"; import { messageTag } from "../plugins/fusion/scripts/lib/user-messages.mjs"; -import { createWorkerRecord, readWorkerRecord, updateWorkerRecord } from "../plugins/fusion/scripts/lib/worker-state.mjs"; +import { createWorkerRecord, readPeerCapacityReservations, readWorkerRecord, updateWorkerRecord, withPeerCapacityReservations } from "../plugins/fusion/scripts/lib/worker-state.mjs"; const GiB = 1024 ** 3; @@ -47,6 +48,84 @@ test("host capacity counts live peer wrappers across sessions and ignores stale assert.equal(hostPeerCapacityDenial(box.env, now, 16 * GiB), null); }); +function hookOutput(callback) { + const original = process.stdout.write; + let output = ""; + process.stdout.write = (chunk) => { output += chunk; return true; }; + try { + callback(); + return output; + } finally { + process.stdout.write = original; + } +} + +test("synchronous peer PreToolUse bursts reserve host slots and PostToolUse releases one", (t) => { + const box = sandbox(t); + const capacity = hostPeerCapacity(os.totalmem()); + const input = (index) => ({ session_id: "session-a", tool_use_id: `peer-${index}`, tool_name: "Agent", tool_input: { subagent_type: "codex:codex-rescue", prompt: "brief" } }); + for (let index = 0; index < capacity; index += 1) { + assert.equal(hookOutput(() => handlePreToolUse(input(index), box.env)), ""); + } + assert.equal(readPeerCapacityReservations(box.env).length, capacity); + assert.match(hookOutput(() => handlePreToolUse(input(capacity), box.env)), new RegExp(messageTag("worker-lifecycle.host-capacity-deny"))); + hookOutput(() => handlePostToolUse(input(0), box.env)); + assert.equal(readPeerCapacityReservations(box.env).length, capacity - 1); + assert.equal(hookOutput(() => handlePreToolUse(input(capacity), box.env)), ""); + hookOutput(() => handlePostToolUse(input(1), box.env, true)); + assert.equal(hookOutput(() => handlePreToolUse(input(capacity + 1), box.env)), ""); +}); + +test("stale peer reservations are pruned before capacity is checked", (t) => { + const box = sandbox(t); + const input = (index) => ({ session_id: "session-a", tool_use_id: `peer-${index}`, tool_name: "Agent", tool_input: { subagent_type: "codex:codex-rescue" } }); + for (let index = 0; index < 5; index += 1) { + assert.equal(reserveHostPeerCapacity(input(index), "codex:codex-rescue", box.env, null, Date.now(), 16 * GiB), null); + } + assert.match(reserveHostPeerCapacity(input(5), "codex:codex-rescue", box.env, null, Date.now(), 16 * GiB), new RegExp(messageTag("worker-lifecycle.host-capacity-deny"))); + withPeerCapacityReservations(box.env, (reservations) => { reservations[0].at -= 2_400_001; }); + assert.equal(reserveHostPeerCapacity(input(5), "codex:codex-rescue", box.env, null, Date.now(), 16 * GiB), null); + assert.equal(readPeerCapacityReservations(box.env).length, 5); +}); + +test("a peer record and its launch reservation occupy one slot", (t) => { + const box = sandbox(t); + const input = (index) => ({ session_id: "session-a", tool_use_id: `peer-${index}`, tool_name: "Agent", tool_input: { subagent_type: "codex:codex-rescue" } }); + assert.equal(reserveHostPeerCapacity(input(0), "codex:codex-rescue", box.env, null, Date.now(), 16 * GiB), null); + worker(box, "fusion-async-peer", "session-a", "codex:codex-rescue", { dispatchToolUseId: "peer-0" }); + for (let index = 1; index < 5; index += 1) { + assert.equal(reserveHostPeerCapacity(input(index), "codex:codex-rescue", box.env, null, Date.now(), 16 * GiB), null); + } + assert.match(hostPeerCapacityDenial(box.env, Date.now(), 16 * GiB), /5 peer wrapper jobs in flight/); +}); + +test("reservations without tool ids do not mask unrelated peer records", (t) => { + const box = sandbox(t); + worker(box, "fusion-active-peer", "session-a"); + const input = (agentType) => ({ session_id: "session-a", tool_name: "Agent", tool_input: { subagent_type: agentType } }); + assert.equal(reserveHostPeerCapacity(input("codex:codex-rescue"), "codex:codex-rescue", box.env, null, Date.now(), 6 * GiB), null); + assert.match(reserveHostPeerCapacity(input("grok:grok-rescue"), "grok:grok-rescue", box.env, null, Date.now(), 6 * GiB), /Host peer capacity reached: 2/); +}); + +test("SendMessage cannot reopen a terminal peer when host capacity is full", (t) => { + const box = sandbox(t); + const capacity = hostPeerCapacity(os.totalmem()); + const taskId = "fusion-terminal-peer"; + worker(box, taskId, "session-a"); + updateWorkerRecord(taskId, box.env, (record) => ({ ...record, agentId: "terminal-peer", transportStatus: "done" })); + const launch = (index) => ({ session_id: "session-b", tool_use_id: `peer-${index}`, tool_name: "Agent", tool_input: { subagent_type: "codex:codex-rescue", prompt: "brief" } }); + for (let index = 0; index < capacity; index += 1) { + hookOutput(() => handlePreToolUse(launch(index), box.env)); + } + const message = { session_id: "session-a", tool_use_id: "message-1", tool_name: "SendMessage", tool_input: { to: "terminal-peer", message: "continue" } }; + assert.match(hookOutput(() => handlePreToolUse(message, box.env)), new RegExp(messageTag("worker-lifecycle.host-capacity-deny"))); + assert.equal(readWorkerRecord(taskId, box.env).continuationRequestedAt, undefined); + hookOutput(() => handlePostToolUse(launch(0), box.env)); + assert.equal(hookOutput(() => handlePreToolUse(message, box.env)), ""); + hookOutput(() => handlePostToolUse(message, box.env, true)); + assert.equal(readPeerCapacityReservations(box.env).length, capacity - 1); +}); + test("dispatch stores parent context tokens and stats render median and p90 for measured dispatches", (t) => { const box = sandbox(t); const transcript = path.join(box.root, "parent.jsonl"); diff --git a/tests/record-migration.test.mjs b/tests/record-migration.test.mjs index 9ed5485..45784b7 100644 --- a/tests/record-migration.test.mjs +++ b/tests/record-migration.test.mjs @@ -134,3 +134,16 @@ test("irreparable records remain unreadable and unchanged", () => { assert.equal(fs.readFileSync(file, "utf8"), before); } }); + +test("Grok legacy records with an unknown status stay unreadable and unchanged", () => { + const legacy = { ...grok.createJobRecord({ id: "grok-mystery", mode: "consult", cwd: root, briefFile: null, background: false }), status: "mystery" }; + delete legacy.schemaVersion; + delete legacy.engine; + const file = path.join(root, "grok-mystery", `${legacy.id}.json`); + fs.mkdirSync(path.dirname(file), { recursive: true }); + const before = `${JSON.stringify(legacy)}\n`; + fs.writeFileSync(file, before); + assert.deepEqual(grok.migrateJobRecord(legacy), legacy); + assert.equal(grok.readJobRecordFile(file), null); + assert.equal(fs.readFileSync(file, "utf8"), before); +}); diff --git a/tests/verification-command.test.mjs b/tests/verification-command.test.mjs index 78434b2..adef62f 100644 --- a/tests/verification-command.test.mjs +++ b/tests/verification-command.test.mjs @@ -26,6 +26,8 @@ test("keeps node:test fail-zero summaries", () => { assert.equal(summaryEvidence("npm test | grep -E 'fail|pass'", "ℹ fail 0"), "output-summary"); assert.equal(summaryEvidence("node --test | tail", "ℹ pass 12\nℹ fail 0"), "output-summary"); assert.equal(summaryEvidence("node --test | tail", "ℹ pass 11\nℹ fail 1"), null); + assert.equal(summaryEvidence("node --test | tail", "diagnostic: fail 1\nfail 1\nℹ fail 0"), "output-summary"); + assert.equal(summaryEvidence("node --test | tail", "# fail 1\nℹ fail 0"), null); }); test("credits and rejects piped pytest summaries", () => { @@ -77,6 +79,8 @@ test("preserves runner failures through && and through pipes with pipefail", () for (const command of ["npm test | tail", "npm test; echo done", "npm test || true", "npm test && echo done; echo again", "set -o pipefail; npm test | tail; echo done", "set -o pipefail; npm test && set +o pipefail && tsc --noEmit | tail"]) { assert.equal(summaryEvidence(command, ""), null, command); } + assert.equal(summaryEvidence("true || set -o pipefail; npm test | tail -1", ""), null); + assert.equal(summaryEvidence("set -o pipefail | cat; npm test | tail -1", ""), null); }); test("rejects a failing summary anywhere in observable output", () => { From 71ac594aa10d8ed8a5a917651d4816f6c3c2f6fd Mon Sep 17 00:00:00 2001 From: Harry Yep Date: Mon, 26 Oct 2026 21:18:38 +0800 Subject: [PATCH 13/13] test(fusion): complete peer calls so capacity reservations release --- tests/fusion-worker-compaction.test.mjs | 14 ++++++++++++-- tests/fusion-worker-lifecycle.test.mjs | 7 +++++++ 2 files changed, 19 insertions(+), 2 deletions(-) diff --git a/tests/fusion-worker-compaction.test.mjs b/tests/fusion-worker-compaction.test.mjs index a24e349..d212d04 100644 --- a/tests/fusion-worker-compaction.test.mjs +++ b/tests/fusion-worker-compaction.test.mjs @@ -143,7 +143,12 @@ test("next write compacts legacy buffers and skips an oversized carry", (t) => { test("parent context advice fires at 250k and 800k once each", async (t) => { const box = sandbox(t); - const dispatch = (id) => hook(box, { hook_event_name: "PreToolUse", session_id: "session-1", tool_use_id: `tool-${id}`, transcript_path: box.transcript, cwd: box.root, tool_name: "Agent", tool_input: { subagent_type: "codex:codex-rescue", description: "peer", prompt: "bounded peer brief" } }); + const dispatch = async (id) => { + const event = { session_id: "session-1", tool_use_id: `tool-${id}`, transcript_path: box.transcript, cwd: box.root, tool_name: "Agent", tool_input: { subagent_type: "codex:codex-rescue", description: "peer", prompt: "bounded peer brief" } }; + const output = await hook(box, { ...event, hook_event_name: "PreToolUse" }); + await hook(box, { ...event, hook_event_name: "PostToolUse", tool_response: { status: "completed", content: [{ type: "text", text: "done" }] } }); + return output; + }; const usage = (tokens) => fs.appendFileSync(box.transcript, `${JSON.stringify({ type: "assistant", message: { usage: { input_tokens: tokens } } })}\n`); usage(250000); assert.match((await dispatch(1)).hookSpecificOutput.additionalContext, /main loop context is 250k/); @@ -158,7 +163,12 @@ test("parent context advice fires at 250k and 800k once each", async (t) => { const configured = sandbox(t); configured.env.FUSION_PARENT_CONTEXT_ADVISORY_TOKENS = "500000"; - const configuredDispatch = (id) => hook(configured, { hook_event_name: "PreToolUse", session_id: "session-1", tool_use_id: `configured-${id}`, transcript_path: configured.transcript, cwd: configured.root, tool_name: "Agent", tool_input: { subagent_type: "codex:codex-rescue", description: "peer", prompt: "bounded peer brief" } }); + const configuredDispatch = async (id) => { + const event = { session_id: "session-1", tool_use_id: `configured-${id}`, transcript_path: configured.transcript, cwd: configured.root, tool_name: "Agent", tool_input: { subagent_type: "codex:codex-rescue", description: "peer", prompt: "bounded peer brief" } }; + const output = await hook(configured, { ...event, hook_event_name: "PreToolUse" }); + await hook(configured, { ...event, hook_event_name: "PostToolUse", tool_response: { status: "completed", content: [{ type: "text", text: "done" }] } }); + return output; + }; fs.appendFileSync(configured.transcript, `${JSON.stringify({ type: "assistant", message: { usage: { input_tokens: 500000 } } })}\n`); assert.match((await configuredDispatch(1)).hookSpecificOutput.additionalContext, /main loop context is 500k/); fs.appendFileSync(configured.transcript, `${JSON.stringify({ type: "assistant", message: { usage: { input_tokens: 800000 } } })}\n`); diff --git a/tests/fusion-worker-lifecycle.test.mjs b/tests/fusion-worker-lifecycle.test.mjs index ce0411c..3e7d4e2 100644 --- a/tests/fusion-worker-lifecycle.test.mjs +++ b/tests/fusion-worker-lifecycle.test.mjs @@ -590,6 +590,10 @@ test("oversized briefs without sizing advise a large package and combine with ve assert.strictEqual(combinedLaunch.additionalContext.match(/Verification advisory/g).length, 1); }); +function completePeer(box, payload) { + run(box, { ...payload, hook_event_name: "PostToolUse", tool_response: { status: "completed", content: [{ type: "text", text: "done" }] } }); +} + test("parent context advisory fires once at 250k and once at 800k across peer dispatches", (t) => { const box = sandbox(t); const usage = (tokens) => fs.appendFileSync(box.transcript, `${JSON.stringify({ type: "assistant", message: { usage: { input_tokens: 1000, cache_read_input_tokens: tokens - 1500, cache_creation_input_tokens: 500 } } })}\n`); @@ -601,10 +605,12 @@ test("parent context advisory fires once at 250k and once at 800k across peer di usage(601000); fs.appendFileSync(box.transcript, `${JSON.stringify({ type: "assistant", isSidechain: true, message: { usage: { input_tokens: 1, cache_read_input_tokens: 0, cache_creation_input_tokens: 0 } } })}\n`); const peer = run(box, { ...dispatch(box), tool_use_id: "peer-1", tool_input: { subagent_type: "codex:codex-rescue", prompt: "plain peer brief" } }); + completePeer(box, { ...dispatch(box), tool_use_id: "peer-1", tool_input: { subagent_type: "codex:codex-rescue", prompt: "plain peer brief" } }); assert.strictEqual(peer.stdout, ""); assert.strictEqual(readWorkerSessionState("session-1", envFor(box)).parentContextAdvisorySent, true); const repeated = run(box, { ...dispatch(box), tool_use_id: "peer-2", tool_input: { subagent_type: "codex:codex-rescue", prompt: "lane: codex quick\ngoal: repair" } }, routingFixture(box)); + completePeer(box, { ...dispatch(box), tool_use_id: "peer-2", tool_input: { subagent_type: "codex:codex-rescue", prompt: "lane: codex quick\ngoal: repair" } }); assert.match(JSON.parse(repeated.stdout).hookSpecificOutput.additionalContext, /lane default/); assert.doesNotMatch(JSON.parse(repeated.stdout).hookSpecificOutput.additionalContext, /main loop context/); @@ -615,6 +621,7 @@ test("parent context advisory fires once at 250k and once at 800k across peer di usage(601000); const injected = run(box, { ...dispatch(box), tool_use_id: "peer-3", tool_input: { subagent_type: "codex:codex-rescue", prompt: "lane: codex quick\ngoal: repair" } }, routingFixture(box)); + completePeer(box, { ...dispatch(box), tool_use_id: "peer-3", tool_input: { subagent_type: "codex:codex-rescue", prompt: "lane: codex quick\ngoal: repair" } }); const injectedContext = JSON.parse(injected.stdout).hookSpecificOutput.additionalContext; assert.match(injectedContext, /lane default/); assert.doesNotMatch(injectedContext, /main loop context/);