From 65e39692315f9f597ac778f329607874ddc08256 Mon Sep 17 00:00:00 2001 From: Anthony Ettinger Date: Sun, 16 Aug 2026 12:50:08 +0000 Subject: [PATCH] PRD 0011: teach the herd the agent protocol MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The herd read paint instead of speaking protocol. Three symptoms, one cause: state came from regex against a screen capture, a prompt left no evidence it was ever submitted, and the roster stopped at the edge of the box. Phase 1 — believe the engine. `herd hooks install claude` writes Claude Code's own lifecycle hooks so state comes from the engine (`authority: hook`), merging into the user's settings file rather than clobbering it. Sessions are launched with MOSHCODE_HERD_NAME/DIR so a hook can name what it is reporting for, and a hook fired outside a herd session does nothing and exits 0. `herd doctor` checks substrate, manifest drift, stale reports, and — for the first time — says what is wrong with rules.json instead of ignoring it. blocked gains sub-kinds (permission/question/menu) that ride in --json and notifications. Phase 2 — the ledger. Every prompt mints a task: id, transitions with timestamps, and the output captured as a screen delta, in 0600 JSONL capped at 500 tasks per session. `herd tasks`, `herd task`, `herd log`, `herd stats` read it back; blocked time is reported as what it is, human latency. The write goes where the watcher's notification decision already happens, not in a second poller. `wait --any/--all` replaces the polling loop every fan-out script had. Phase 3 — the protocol. `herd serve` exposes the herd over A2A v0.3.0 (discovery, message/send, tasks/get, tasks/cancel) behind the moshcode login, with no unauthenticated mode, loopback included, and --agent sessions withheld unless --expose-autonomous. `herd remote add` puts a deployed agent on the roster — a2a or a bare POST endpoint — and prompt/read/wait/kill work on it unchanged, with auth from MOSHCODE_REMOTE__TOKEN and never in the manifest. Phase 4 — `herd eval` runs a dataset across engines with either the dataset's own patterns or an engine as judge, exiting 0/4/5 so CI can tell a worse agent from a broken box. gradient joins the tool table with its own runtime check. 119 new tests. Decisions taken while building are recorded at the end of the PRD rather than edited into its requirements. Co-Authored-By: Claude Opus 5 (1M context) --- .github/workflows/herd-eval.yml | 109 ++++ README.md | 126 ++++- evals/moshcode.jsonl | 8 + prd/0011-herd-agent-protocol.md | 391 ++++++++++++++ prd/README.md | 1 + src/cli-schema.mjs | 99 +++- src/commands.mjs | 94 +++- src/engines.mjs | 32 ++ src/herd-cli.mjs | 832 +++++++++++++++++++++++++++++- src/herd-eval.mjs | 301 +++++++++++ src/herd-hooks.mjs | 285 ++++++++++ src/herd-remote.mjs | 365 +++++++++++++ src/herd-serve.mjs | 515 ++++++++++++++++++ src/herd-state.mjs | 177 ++++++- src/herd-tasks.mjs | 377 ++++++++++++++ src/herd.mjs | 96 +++- src/templates.mjs | 37 +- src/tools.mjs | 43 ++ test/herd-agent-protocol.test.mjs | 236 +++++++++ test/herd-eval.test.mjs | 180 +++++++ test/herd-hooks.test.mjs | 199 +++++++ test/herd-remote.test.mjs | 273 ++++++++++ test/herd-serve.test.mjs | 319 ++++++++++++ test/herd-tasks.test.mjs | 214 ++++++++ 24 files changed, 5247 insertions(+), 62 deletions(-) create mode 100644 .github/workflows/herd-eval.yml create mode 100644 evals/moshcode.jsonl create mode 100644 prd/0011-herd-agent-protocol.md create mode 100644 src/herd-eval.mjs create mode 100644 src/herd-hooks.mjs create mode 100644 src/herd-remote.mjs create mode 100644 src/herd-serve.mjs create mode 100644 src/herd-tasks.mjs create mode 100644 test/herd-agent-protocol.test.mjs create mode 100644 test/herd-eval.test.mjs create mode 100644 test/herd-hooks.test.mjs create mode 100644 test/herd-remote.test.mjs create mode 100644 test/herd-serve.test.mjs create mode 100644 test/herd-tasks.test.mjs diff --git a/.github/workflows/herd-eval.yml b/.github/workflows/herd-eval.yml new file mode 100644 index 0000000..0771013 --- /dev/null +++ b/.github/workflows/herd-eval.yml @@ -0,0 +1,109 @@ +# NOT managed by the sh1pt Actions Fleet — hand-written for PRD 0011 R13. +# The two fleet-managed workflows (ci.yml, test.yml) carry a pack hash and are +# reverted by the next fleet sync, so this lives in its own file rather than as +# a job added to one of them. +# +# What this gates: "the agent still passes the dataset", as a red/green check +# next to `npm test`. It needs a real engine with real credentials, which a +# runner does not have by default — so the whole job is a no-op until a +# credential secret exists, and says so rather than going green by accident. A +# gate that fails on every fork because nobody added a secret is a gate people +# turn off. +name: herd eval + +on: + workflow_dispatch: + inputs: + threshold: + description: "Score every engine has to reach (0-1)" + default: "0.8" + engines: + description: "Comma-separated engines to compare" + default: "claude" + pull_request: + paths: + - "evals/**" + - "src/herd-eval.mjs" + - "src/herd-tasks.mjs" + - ".github/workflows/herd-eval.yml" + +permissions: + contents: read + +concurrency: + group: herd-eval-${{ github.ref }} + cancel-in-progress: true + +jobs: + eval: + runs-on: ubuntu-latest + timeout-minutes: 30 + steps: + - uses: actions/checkout@v7 + + # The `secrets` context is not available in a job-level `if`, so the + # check is a step that publishes an output the rest of the job reads. + - id: creds + env: + ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} + run: | + if [ -n "$ANTHROPIC_API_KEY" ]; then + echo "have=true" >> "$GITHUB_OUTPUT" + else + echo "have=false" >> "$GITHUB_OUTPUT" + echo "::notice title=herd eval skipped::no engine credentials on this runner — add ANTHROPIC_API_KEY to turn this check on" + fi + + - uses: pnpm/action-setup@v6 + if: steps.creds.outputs.have == 'true' + + - uses: actions/setup-node@v7 + if: steps.creds.outputs.have == 'true' + with: + node-version: '22' + cache: pnpm + + - run: pnpm install --frozen-lockfile + if: steps.creds.outputs.have == 'true' + + # The herd needs somewhere to run its sessions. Without tmux it would + # fall back to script(1), which works, but tmux is one apt away and is + # the substrate people actually use. + - run: sudo apt-get update && sudo apt-get install -y tmux + if: steps.creds.outputs.have == 'true' + + - run: npm install -g @anthropic-ai/claude-code + if: steps.creds.outputs.have == 'true' + + # `rules` as the judge, not an engine: the dataset carries its own + # expectations, and a judge that is itself an LLM would make a flaky + # check out of a deterministic one. Exit 4 is "below the threshold" and + # exit 5 is "the harness could not run" — the job distinguishes them so a + # broken runner does not get filed as a worse agent. + - name: run the dataset + if: steps.creds.outputs.have == 'true' + env: + ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} + run: | + set +e + node bin/moshcode.mjs herd eval \ + --dataset evals/moshcode.jsonl \ + --engines "${{ inputs.engines || 'claude' }}" \ + --threshold "${{ inputs.threshold || '0.8' }}" \ + --json > eval.json + code=$? + set -e + cat eval.json + case "$code" in + 0) echo "::notice title=herd eval::every engine is at or above the threshold" ;; + 4) echo "::error title=herd eval::an engine scored below the threshold"; exit 1 ;; + 5) echo "::error title=herd eval::the harness could not run (infrastructure, not the agent)"; exit 1 ;; + *) echo "::error title=herd eval::unexpected exit $code"; exit 1 ;; + esac + + - uses: actions/upload-artifact@v4 + if: steps.creds.outputs.have == 'true' && always() + with: + name: herd-eval-report + path: eval.json + if-no-files-found: ignore diff --git a/README.md b/README.md index 1bb87df..3b47285 100644 --- a/README.md +++ b/README.md @@ -262,9 +262,9 @@ moshcode agents claude -d --name api # and an agent ```sh $ moshcode ps - api claude blocked ~/src/coinpay 3m - logs shell idle ~/src/coinpay 3m - work shell idle ~/src/coinpay 3m + api claude blocked ~/src/coinpay 3m screen + logs shell idle ~/src/coinpay 3m screen + work shell idle ~/src/coinpay 3m screen ⚠ 1 waiting on you — moshcode attach api ``` @@ -316,9 +316,9 @@ Every session carries a state: `working`, `blocked`, `done`, `idle`, or `unknown`. `blocked` means a human decision is the only thing missing. ``` - api claude blocked ~/src/coinpay 12m - web codex working ~/src/ugig.net 4m - audit opencode done ~/src/moshpit-dns 1h + api claude blocked ~/src/coinpay 12m hook + web codex working ~/src/ugig.net 4m screen + audit opencode done ~/src/moshpit-dns 1h runtime ``` State comes from one authority per session, never two. An engine that reports @@ -363,6 +363,120 @@ await herdWait("api"); await herdWait("web"); say(herdRead("api", { lines: 20 })); ``` +Fanning work out is easy; joining on it used to be a hand-rolled polling loop. +`--any` returns on the first session to get there, `--all` when the last one +has, and both take the same `--state` and `--timeout` as a single wait: + +```sh +moshcode wait --any api web docs # --json names the winner +moshcode wait --all api web --state done +``` + +```js +const first = await herdWait(["api", "web", "docs"], { any: true }); +await herdWait(["api", "web"], { states: ["done"] }); +``` + +### Let the engine say what it is doing + +Reading a screen works and it rots — engines change their wording between +releases and nothing tells you. When an engine has lifecycle hooks, install +them once and its state comes from the engine itself: + +```sh +moshcode herd hooks install claude +✓ claude — 3 hooks installed (stop, notification, prompt-submit) +``` + +`moshcode ps` then reads `hook` in its last column instead of `screen`. The +file is **merged, never clobbered** — your own hooks stay, and `hooks remove` +takes out only what moshcode put in. A hook that fires outside a herd session +does nothing and exits 0, so installing one cannot break an engine you run by +hand, and the screen rules stay as the fallback for everything else. +`moshcode herd doctor` says what is installed, what has drifted, and — for the +first time — what is wrong with your `rules.json` instead of ignoring it. + +### What happened while you slept + +Every prompt through the herd mints a **task**: an id, its state transitions +with timestamps, and the output it produced. `ps` still answers "now"; this +answers "what happened". + +```sh +$ moshcode herd tasks api + t-01 22:14 done 4m "port the auth routes" + t-02 22:19 blocked 6h11 "run the migration" + +$ moshcode herd task t-02 # transitions, and what came back +$ moshcode herd log api # the raw state history +$ moshcode herd stats api +api working 3h02 · blocked 6h11 · idle 1h40 + blocked 6h11 over 2 spell(s) — that one is you +``` + +Blocked time is the herd's name for *human latency*: the agent was ready and +you were asleep. Ledgers live in `~/.moshcode/herd/tasks/.jsonl` at +`0600`, capped at the last 500 tasks per session. From a script, +`herdTasks(name)` and `herdTask(id)` return them as values. + +### Agents that are not on this box + +A deployed agent can be a herd member. Two kinds: `a2a` speaks +[A2A v0.3.0](https://a2a-protocol.org/v0.3.0/specification/) (card discovery, +`message/send`, `tasks/get`, `tasks/cancel`), and `run` is a bare endpoint that +takes `POST {"prompt": …}` — the shape a `gradient agent deploy` prints. + +```sh +moshcode herd remote add research https://agents.do-ai.run/…/production --kind run +export MOSHCODE_REMOTE_RESEARCH_TOKEN=… # never written to the manifest, never synced +``` + +``` +$ moshcode ps + api claude blocked ~/src/coinpay 12m hook + research remote idle agents.do-ai.run — remote +``` + +`prompt`, `read`, `wait` and `kill` work on it unchanged, which is the point: a +fan-out script across a local pty and a deployed agent contains no `if +(remote)`. A remote's state is the *remote's claim* — `ps` says `remote` in the +last column so it is never mistaken for something this box verified — and +`kill` on one deregisters it here rather than reaching across the network to +end somebody else's agent. + +### The herd, over A2A + +`moshcode herd serve` exposes this machine's herd to any A2A client: the herd's +card at `/.well-known/agent-card.json`, each member at `//`, +`message/send` → prompt, `tasks/get` → the ledger, `tasks/cancel` → interrupt. +`blocked` is A2A's `input-required`; the states that do not map cleanly round +down and carry the honest one in task metadata. + +```sh +moshcode login # it verifies tokens against app.moshcode.sh +moshcode herd serve # 127.0.0.1:7683 by default +``` + +It is a shell on a socket and is treated like one: **no unauthenticated mode, +loopback included**, a loud warning past `127.0.0.1`, and sessions started with +`--agent` withheld unless you pass `--expose-autonomous` — an engine with its +approvals bypassed plus a network prompt is the worst pairing on the menu. + +### Which engine is best at *this* repo + +Not a leaderboard run against engines nobody deploys on repos nobody has — your +dataset, your engines, your machine: + +```sh +moshcode herd eval --dataset evals/moshcode.jsonl --engines claude,codex --threshold 0.8 +``` + +A row is `{"prompt": "…", "expect": "pattern"}` or +`{"prompt": "…", "rubric": "…"}` (jsonl, json or csv). Scoring is either the +dataset's own patterns or an engine acting as judge (`--judge claude`). Exit +codes are distinct on purpose — `0` pass, `4` below the threshold, `5` the +harness could not run — because CI has to tell a worse agent from a broken box. + ### After a reboot ```sh diff --git a/evals/moshcode.jsonl b/evals/moshcode.jsonl new file mode 100644 index 0000000..099e929 --- /dev/null +++ b/evals/moshcode.jsonl @@ -0,0 +1,8 @@ +{"id": "state-authority", "prompt": "In src/herd-state.mjs, what value does the `authority` field take when a session's state came from its engine's own lifecycle hook rather than from its screen? Answer with the single word and nothing else.", "expect": "\\bhook\\b"} +{"id": "vocabulary-file", "prompt": "Which file in this repo holds the moshscript command vocabulary — the verbs a .mosh script can call? Answer with the repo-relative path and nothing else.", "expect": "src/commands\\.mjs"} +{"id": "safe-answer", "prompt": "The herd's state classifier has one state it treats as always safe, never blocking a launch, and returns when no rule matched. Name it in one word.", "expect": "\\bunknown\\b"} +{"id": "wait-exit-codes", "prompt": "`moshcode wait` exits with distinct codes so scripts can branch on the outcome. Which exit code means the wait timed out? Answer with the number only.", "expect": "(^|\\D)2(\\D|$)"} +{"id": "substrate-fallback", "prompt": "The herd runs sessions on tmux when it is available. Name the program it uses to allocate a pty when tmux is not installed. One word.", "expect": "script"} +{"id": "manifest-mode", "prompt": "What file mode does the herd write its session manifest with, and why? Give the octal mode first.", "expect": "0?600"} +{"id": "blocked-meaning", "prompt": "In the herd's roster, what does the state `blocked` mean? Answer in one short sentence.", "rubric": "The answer must say that the session is waiting on a human decision or input — an approval, a question, or a menu — and not that it is stuck, crashed, or busy working.", "expect": "human|you|decision|input|approval|ask"} +{"id": "no-second-api", "prompt": "Does moshcode expose a separate socket API for agents to drive the herd, apart from the CLI? Answer yes or no, then one sentence.", "rubric": "The answer must be 'no': the CLI with --json on every verb is the one surface, and `herd serve` is that same surface answering a socket rather than a second API.", "expect": "^\\W*no\\b"} diff --git a/prd/0011-herd-agent-protocol.md b/prd/0011-herd-agent-protocol.md new file mode 100644 index 0000000..6a57f14 --- /dev/null +++ b/prd/0011-herd-agent-protocol.md @@ -0,0 +1,391 @@ +--- +openprd: "0.2" +id: "0011" +title: "Teach the herd the agent protocol — hooks-first state, a task ledger, and an A2A surface for local and remote agents" +status: Draft +authors: + - anthony@profullstack.com +created: 2026-08-16 +updated: 2026-08-16 +repo: https://github.com/moshcoder/moshcode +discussion: +implementation: src/herd-hooks.mjs, src/herd-tasks.mjs, src/herd-serve.mjs, src/herd-remote.mjs, src/herd-eval.mjs; touches src/herd-state.mjs, src/herd-cli.mjs, src/herd.mjs, src/engines.mjs, src/tools.mjs, src/commands.mjs, src/cli-schema.mjs +tags: [herd, runtime, agents, a2a, state, tasks, evals, digitalocean] +supersedes: +superseded-by: +--- + +## Problem + +PRD 0009 built the herd: sessions outlive the terminal, every session carries a +semantic state, and one verb set serves humans, scripts, and agents. It works. +Three limits are now the daily friction, and all three are the same limit seen +from different angles — **the herd reads paint instead of speaking protocol.** + +**The state rules rot, and we said so ourselves.** `herd-state.mjs` documents +its own weakness: *"Screen rules are the fallback, and they are the part that +rots — engines change their prompts between releases and nothing tells us."* +The tier-1 mechanism exists (`moshcode herd report`, TTL-bounded, beats the +screen), but nothing *installs* the hooks that would use it. Claude Code has +lifecycle hooks. OpenCode has plugins and events. Codex has a notify path. We +built the socket and never plugged anything into it, so in practice every +session is classified by regex against a screen capture, and every engine +release is a chance for the roster to start lying. + +**Sessions have a present tense but no past.** `herd prompt api "…" --wait` +returns, and then the evidence evaporates. Which prompts were submitted, when +each one blocked, what the answer was, how long the human took — none of it is +anywhere. Fan-out (the herd's party trick) is therefore unauditable: a +moshscript that drove four engines overnight can tell you their state *now* +and nothing else. The watch loop already observes every transition it would +take to fix this, and throws each one away after deciding whether to buzz a +phone. + +**The herd stops at the edge of the box.** A deployed agent — say a +DigitalOcean Gradient ADK deployment answering at +`agents.do-ai.run///run` — cannot be on the roster, and +nothing off the box can drive the herd. Meanwhile the ecosystem converged on +exactly the shape we need. DO's ADK gives every agent one uniform entrypoint +(`POST /run`, JSON in, JSON out), a lifecycle CLI (`init/run/deploy/logs/ +traces/evaluate`), and — the interesting part — [A2A protocol +v0.3.0](https://a2a-protocol.org/v0.3.0/specification/) support: discovery at +`/.well-known/agent-card.json`, `message/send`, `tasks/get`, `tasks/cancel`, +tasks with ids, status history, and artifacts. A2A's task state vocabulary +includes `input-required`. + +`input-required` is `blocked`. The mapping between A2A and the herd is not an +integration to be designed; it is a translation table to be written down: + +| herd | A2A | +| --------------- | ---------------------------- | +| `herd prompt` | `message/send` | +| state / `wait` | `tasks/get` (poll) | +| `herd kill` | `tasks/cancel` (best-effort) | +| `ps` / roster | agent-card discovery | +| `blocked` | `input-required` | +| `working` | `working` | +| `done` | `completed` | +| killed | `canceled` | + +PRD 0009 took herdr's thesis — *"the CLI and socket API are one surface agents +drive"* — and implemented it locally. A2A is that thesis standardized across +machines. This PRD ports the ideas, not the SDK. + +## Goals + +- State comes from the engine when the engine can speak, and from the screen + only when it can't. On a default install, a Claude Code herd session reads + `authority: hook`, not `authority: screen`. +- Every prompt is a **task** with a durable record: an id, its state + transitions with timestamps, and the output it produced. `moshcode ps` keeps + answering "now"; the ledger answers "what happened." +- The roster spans machines. A deployed agent is a herd member; `ps`, `prompt`, + `read`, and `wait` do not care whether a member is a local pty or a URL. +- The herd itself is drivable over a standard protocol, behind real auth, so + another machine's herd — or anyone's A2A client — can submit work and poll it. +- "Which engine is best for this repo" is answered empirically: + one dataset, N engines, a judge, an exit code CI can gate on. +- The DO Gradient ADK is a first-class workflow tool: installable, drivable + from moshscript, and its dev server a well-classified herd member. + +## Non-Goals + +- **Not a multiplexer rewrite.** 0009's substrates (tmux, `script(1)`+FIFO, + foreground fallback) stand unchanged. Everything here layers on the existing + runtime. +- **Not a hosted control plane.** `app.moshcode.sh` remains the notify/approve + surface it already is. `herd serve` runs on your box, like `moshcode console`. +- **Not the full A2A spec.** v0.3.0, JSON-RPC, text parts only — + the same MVP scope the ADK itself ships. Streaming, push notifications, and + authenticated extended cards are declared off in the card's capability flags, + which the spec provides for exactly this. +- **Not token-level tracing.** We do not own the engines' runtimes; pretending + to see inside them would be paint-reading with extra steps. Transitions and + task artifacts are what the herd can attest to honestly. +- **Not replacing engine-native resume/history.** `restore --resume` semantics + from 0009 are untouched; the ledger records what the *herd* saw, not the + engine's conversation. + +## Users + +- **The operator with four agents and one attention span.** 0009 told them who + needs them now; this tells them what happened while they slept, and lets a + deployed agent sit on the same roster as the local ones. +- **The moshscript author.** Fan-out already works; fan-*in* is exit codes and + screen reads. Tasks give joins something to join on, and `wait --all` stops + the hand-rolled polling loops. +- **Another agent.** An engine in the herd, a CI job, or a deployed ADK agent + that needs to hand work to a local session and collect the result — over a + protocol it already speaks, not over SSH-and-tmux incantations. +- **The CI pipeline** that wants "the agent still passes the dataset" as a + red/green check next to `npm test`. + +## Requirements + +### Phase 1 — believe the engine, not the paint + +- **R1 [P0] `moshcode herd hooks install |all`.** Writes the + engine-native lifecycle hook configuration that calls + `moshcode herd report "$MOSHCODE_HERD_NAME" ` at the right moments. + Claude Code first (its hooks are documented and stable): stop → `done`, + notification/permission-request → `blocked`, prompt-submit/tool-use → + `working`. Hook specs live in `ENGINES` next to each engine's screen rules, + so detection ships with the install spec, exactly as 0009 R7 intended. + `--dry-run` prints the config diff; `hooks remove` reverts; + `hooks status --json` reports per-engine install state. **Merge, never + clobber:** a user's existing hook file is extended, and `remove` takes out + only what we added. +- **R2 [P0] Sessions know their own name.** The herd already launches the + engine, so it injects `MOSHCODE_HERD_NAME` and `MOSHCODE_HERD_DIR` into the + session environment at start. A hook fired outside a herd session (no env + var) exits silently and successfully — hooks must never break an engine + running outside the herd. +- **R3 [P1] `moshcode herd doctor`.** One verb that checks the things that + actually go wrong: tmux server reachable, manifest vs. live sessions drift, + stale hook reports past TTL, unwritable status dir, rules.json parse errors + (today they vanish silently by design — doctor is where they get to be + loud). `--json` for provisioning scripts. +- **R4 [P2] Blocked sub-kinds.** `blocked:permission`, `blocked:question`, + `blocked:menu`. Hooks can say which; screen rules map their existing + patterns (y/n → permission, `❯ 1.` → menu). The roster still prints + `blocked`; the sub-kind rides in `--json` and in notifications, so + `--ask` replies can be validated against what was actually asked (a menu + wants a digit, not a paragraph). + +### Phase 2 — the task ledger + +- **R5 [P0] Every prompt mints a task.** `herd prompt` assigns a task id and + appends to `~/.moshcode/herd/tasks/.jsonl`: submission (text, ts), + each state transition (observed by the same poll the watcher already runs), + terminal state, and the output artifact captured as the screen delta via the + existing `read` machinery. Files are `0600` for the manifest's stated reason, + one step harder: engine output carries secrets the user never even typed. +- **R6 [P0] Read verbs.** `moshcode herd tasks [--json]` lists; + `moshcode herd task [--json]` shows one, transitions and artifact + included. moshscript gets `herdTasks(name)` and `herdTask(id)` as values, + same contract as `herdRead`/`herdList`: `null`/`[]` on error, never throw. +- **R7 [P1] Transitions become history.** `herd log ` prints the + timestamped state history; `herd stats [session]` aggregates time-in-state — + including blocked-time, which is a number with a name: *human latency*. + Retention is capped and documented (default: last 500 tasks per session, + size-bounded), because an append-only file with no cap is a disk-eater with + a delay on it. +- **R8 [P1] Fan-in verbs.** `moshcode wait --any …` returns on the + first session to hit a target state (exit codes name the winner in `--json`); + `wait --all` returns when every named session has. `herdWait` gains the same + options. This deletes the polling loop from every fan-out script we have + written so far. + +### Phase 3 — the A2A surface + +- **R9 [P0] `moshcode herd serve`.** An HTTP server exposing the herd per A2A + v0.3.0. `GET /.well-known/agent-card.json` describes the herd; each session + is addressable as `//` with its own card. `message/send` → `herd + prompt` (mints a task per R5); `tasks/get` → ledger read; `tasks/cancel` → + interrupt, escalating exactly as `kill` already does. State maps per the + table in Problem; `idle` and `unknown` map to `working` with the honest + state carried in task metadata, because A2A's vocabulary is smaller than + ours and rounding *up* to "needs input" would page people for nothing. +- **R10 [P0] Serve is a shell on the internet, and is treated like one.** + Reuse `console.mjs`'s discipline wholesale: bind `127.0.0.1` by default, + verify the moshcode token against `app.moshcode.sh/api/me` once, swap for a + short-lived HMAC credential, refuse unauthenticated requests before they + reach anything, warn loudly on `0.0.0.0`. There is no unauthenticated mode, + loopback included — `message/send` is keystrokes into a real pty, which is + strictly more dangerous than a browser terminal that at least shows you what + it's doing. +- **R11 [P0] Remote members.** `moshcode herd remote add + [--kind a2a|run]` registers a remote agent on the roster. `a2a` discovers the + card and drives JSON-RPC; `run` covers bare ADK-style endpoints + (`POST ` with `{"prompt": …}` — the shape every `gradient agent deploy` + prints). Manifest rows carry `kind: "remote"`; `ps` shows them with the host + where local rows show cwd; state comes from `tasks/get` (a2a) or reachability + (run — a request/response endpoint is `idle` when up, `working` while a call + is in flight, and honest about knowing nothing more). Auth is a named header + from the environment (`MOSHCODE_REMOTE__TOKEN`), never written to the + manifest and never synced — 0010's allowlist reasoning, verbatim. +- **R12 [P1] The verbs don't care where a member lives.** `herd prompt`, + `read`, `wait`, `kill`, and their moshscript forms work unchanged on remote + members: prompt POSTs, read returns the last artifact, wait polls, kill + cancels. A `.mosh` script that fans across `claude` (local pty) and + `research-prod` (deployed on DO) is the acceptance test, and it should not + contain a single `if (remote)`. + +### Phase 4 — evals and the DO toolchain + +- **R13 [P1] `moshcode herd eval`.** `--dataset --engines a,b,… + [--judge |rules] [--threshold N] [--json]`. Fans each dataset row + across the named engines using the verbs that already exist, collects + results from the ledger, scores with the `ai()` verb as judge (rubric in the + dataset) or plain expected-pattern rules, and exits with `wait`'s discipline: + distinct codes for pass, below-threshold, and infrastructure failure. The DO + ADK ships `gradient agent evaluate --dataset-file --categories + --success-threshold` for deployed agents; this is the same idea pointed at + interactive engines, which is the comparison nobody else can run. +- **R14 [P2] Install the ADK like we install everything else.** + `moshcode install gradient` runs the vendor path (`pip install gradient-adk`, + Python ≥3.10 checked and named when missing — moshcode stays Node, the tool + owns its runtime, same as CoinPay owning Node 20). Top-level passthrough + `moshcode gradient …` and a `gradient(args…)` moshscript verb, per the + existing tool table. +- **R15 [P2] The ADK dev loop is a good herd citizen.** Ship a `gradient` + entry in the default state rules so + `moshcode herd run --name agent -- gradient agent run --dev` classifies + (uvicorn startup banner → `idle`, request handling → `working`), and a + template pointer at `digitalocean/gradient-adk-templates` in + `moshcode template list`. The workflow this buys: dev server in one tile, + Claude editing it in the next, logs in a third, `gradient agent deploy` from + the mosh bar, then `herd remote add` the printed URL — the whole lifecycle + without leaving the pit. + +## UX Notes + +**New verbs, existing shape.** Everything lands in the generated command +table, drifts-fail-the-build included. `hooks`, `tasks`, `task`, `log`, +`stats`, `serve`, `remote`, `eval` are all `herd` subverbs; `wait` grows +`--any/--all` in place. Every verb takes `--json`. There is still no second +API — `serve` is not a new surface so much as the existing one answering a +socket. + +**The one-time setup reads like this:** + +``` +moshcode herd hooks install claude +✓ claude — 3 hooks installed (stop, notification, prompt-submit) + sessions started from the herd now report state directly. + screen rules remain the fallback for everything else. +``` + +**A remote member on the roster:** + +``` +$ moshcode herd remote add research https://agents.do-ai.run/b168…/production --kind run +$ moshcode ps + api claude blocked ~/src/coinpay 12m hook + research remote idle agents.do-ai.run — remote +⚠ 1 waiting on you — moshcode attach api +``` + +**What happened overnight:** + +``` +$ moshcode herd tasks api + t-01 22:14 done 4m "port the auth routes" + t-02 22:19 blocked 6h11 "run the migration" ← answered 04:30 +$ moshcode herd stats api + working 3h02 · blocked 6h11 · idle 1h40 blocked = you +``` + +**Honesty rules, carried forward.** A remote member's state is the remote's +claim, and `ps --json` says `authority: remote` so nobody mistakes it for +something we verified. `herd serve` prints the same warning `console` does +when bound beyond loopback. The ledger survives a reboot; the *processes* +still don't, and `restore` keeps saying so. + +## Success Metrics + +- On a machine with hooks installed, ≥95% of Claude Code herd sessions report + `authority: hook`; `rules.json` edits for supported engines drop to + approximately zero. +- Any prompt submitted through the herd is reconstructable after a reboot: + what was asked, when it blocked, what came back. +- A deployed DO agent is driven by `herdPrompt`/`herdWait` in a fan-out script + with zero remote-specific branches. +- An off-the-shelf A2A client (the ADK's own `examples/a2a/client.py` is the + test) completes discover → send → get → cancel against `herd serve`. +- `moshcode herd eval` gates a CI job in this repo: engines below threshold + fail the build with a distinct exit code. +- `blocked` time is a number on a screen, and it goes down. + +## Risks & Open Questions + +- **`serve` widens the attack surface.** `message/send` is remote keystrokes + into a pty. Mitigations are R10 (auth always, loopback default, console's + token discipline) plus one open question: should `serve` refuse to expose + sessions launched with bypass/auto-approve flags unless `--expose-autonomous` + is explicit? Leaning yes — an autonomous engine plus a network prompt + injector is the worst pairing on the menu. +- **Hook drift is the new rule rot.** Engines change hook schemas like they + change prompts. Contained the same way: specs live per-engine in + `ENGINES`, `hooks status` detects a schema the engine rejected, and the + screen fallback means a broken hook degrades to today, never below it. +- **A2A is v0.3.0 and moving.** Pin the version in the card, keep the surface + to the four operations the ADK itself ships, and treat spec upgrades as + their own PRD. Text-only parts for the MVP. +- **State vocabularies don't biject.** Our `idle`/`unknown` vs. A2A's + smaller set (R9 rounds down, metadata carries truth). Accepted as lossy; + revisit if clients demonstrably need more. +- **Ledger growth.** JSONL with per-session caps (R7). Open: should artifacts + above a size threshold store a path to the transcript slice instead of + inline text? +- **Card shape.** One card for the herd with sessions as skills, or a card per + session (current lean: per session — it makes `remote add` of *someone + else's* single session symmetric)? Decide during R9 spike. +- **Python as a soft dependency** for R14. Same posture as tailscale needing + root: name the requirement, never harden it — `install gradient` fails with + the fix printed, and nothing else in moshcode notices Python exists. + +## Implementation Notes + +The watch loop in `herd-cli.mjs` already observes every transition R5 needs — +the ledger is a write inserted where the notification decision already +happens, not a second poller. `notify.mjs`'s `ingestApproval`/`pollApproval` +pattern is the model for `tasks/get` long-polling if we want it later. +`console.mjs` is the auth gateway to lift for R10, not to reimplement. +Hook specs belong in `engines.mjs` beside the state tables they supersede. +`runtime.mjs` registration gives moshscript the new verbs for free once the +CLI verbs exist. Build order: R1–R2 (de-rots the core, smallest diff), R5–R6 +(everything else reads from it), then R9–R11 as one spike since they share the +task model, with R13 as the demo that only moshcode can run. + +## Decisions taken while building + +Six questions this document left open were answered by the implementation. +They are recorded here rather than edited into the requirements above, so the +proposal stays the proposal and the answers stay attributable. + +**Both card shapes ship, not one.** They answer different questions: the herd +card is discovery ("what is on this box"), and a per-session card is what a +client stores when it intends to talk to one member for a week. Publishing +both also keeps `remote add` of somebody else's single session symmetric with +adding a whole herd, which was the argument for per-session in the first place. + +**`--expose-autonomous` is opt-in, as the risk section leaned.** A session +started with `--agent` is off the protocol surface entirely — not in the herd +card, not addressable, not promptable — until the flag says otherwise. + +**`tasks/cancel` interrupts; it does not kill the member.** R9 says "escalating +exactly as `kill` already does", and the escalation *pattern* is what was +taken: Escape, then Ctrl-C. It stops one rung short of `kill`'s pane removal on +purpose. An A2A task is a unit of work inside a member, and the member is a +long-lived thing somebody may have attached to five minutes ago; ending it is a +decision, not a protocol call. `moshcode kill` is still the verb for that. + +**A finished task is `completed` even when the session went back to `idle`.** +The `idle → working` rounding in R9 is about a *session* — it is sitting there, +it is not asking for anything. Applying it to a task that has an outcome and an +artifact would leave every A2A client polling a job that finished ten minutes +ago, because `send → poll until completed` is the whole protocol. The rounding +now applies only to open tasks; a closed one is `completed`, or +`input-required` when it ended by stopping to ask. + +**A poll closes a task.** `tasks/get` and `herd tasks` both reconcile: if a +task is open and its session has stopped, the observation is recorded and the +artifact captured. Without it the only thing that ever finished a task was the +watcher, and a herd where nobody happened to be running one would hand every +client an eternal `working`. + +**Artifacts stay inline, truncated at the tail.** The open question asked +whether oversized artifacts should store a path to a transcript slice instead. +They do not: the cap keeps the last 8000 characters — an agent's answer is the +last thing it printed — and records both that it was truncated and the original +length. A path into a transcript that `restore` may have already replaced would +be a reference to something the ledger cannot promise still exists. + +**The ADK dev server has no `working` rule.** R15 asks for "request handling → +`working`", and uvicorn cannot supply it: it writes its access line when a +request has *finished*, so a rule matching that line would pin the tile to +`working` from the first request until the line scrolled away — the exact rot +this PRD exists to get away from. A completed request is therefore classified +`idle`, which is true both before traffic and after it. Watching a *deployed* +agent's state is what `herd remote add` is for. diff --git a/prd/README.md b/prd/README.md index d58ad64..5389f7c 100644 --- a/prd/README.md +++ b/prd/README.md @@ -26,4 +26,5 @@ Start one with `moshcode prd ""` (TUI: `/prd`). | [0008](0008-ticker-research-and-plugin-marketplace.md) | Bring equity research into the pit, and ship the pit's slash commands as a plugin | Draft | | [0009](0009-persistent-agent-runtime.md) | Keep the herd alive — a persistent runtime, semantic agent state, and one control surface for humans and agents | Accepted | | [0010](0010-cloud-settings-sync.md) | Sync the pit's settings to your moshcode.sh account | Draft | +| [0011](0011-herd-agent-protocol.md) | Teach the herd the agent protocol — hooks-first state, a task ledger, and an A2A surface for local and remote agents | Draft | diff --git a/src/cli-schema.mjs b/src/cli-schema.mjs index 5dbecb8..34f06dd 100644 --- a/src/cli-schema.mjs +++ b/src/cli-schema.mjs @@ -95,6 +95,11 @@ export const CORE_CLI_COMMANDS = [ ["# driving one without attaching", ""], ["moshcode herd prompt api \"run the tests\" --wait", "hand it work, block until it lands"], ["moshcode herd read api --lines 40", "read its screen"], + ["", ""], + ["# what happened while you slept", ""], + ["moshcode herd hooks install claude", "state from the engine, not from its screen"], + ["moshcode herd tasks api", "every prompt, and how long each one waited on you"], + ["moshcode herd remote add research https://agents.do-ai.run/…/production", "a deployed agent, same verbs"], ], seeAlso: ["ps", "attach", "wait", "restore", "start"], note: "`start` is for the engines moshcode installs; `run` and `shell` take anything else, " @@ -140,15 +145,25 @@ export const CORE_CLI_COMMANDS = [ name: "wait", group: "runtime", description: "block until a session is blocked, done, or idle", - synopsis: [["moshcode wait [--state blocked,done] [--timeout 30m]", ""]], + synopsis: [ + ["moshcode wait [--state blocked,done] [--timeout 30m]", ""], + ["moshcode wait --any …", "the first one there wins"], + ["moshcode wait --all …", "join the whole fan-out"], + ], flags: [ ["--state ", "states to wait for, comma-separated", "blocked,done"], + ["--any", "return as soon as one of them reaches the state", ""], + ["--all", "return when every one of them has", "implied by naming several"], ["--timeout ", "give up after this long (30s, 10m, 2h)", "30m"], ["--json", "machine-readable", ""], ], - examples: [["moshcode wait api --state blocked --timeout 1h", "exit 0 matched · 2 timed out · 3 gone"]], + examples: [ + ["moshcode wait api --state blocked --timeout 1h", "exit 0 matched · 2 timed out · 3 gone"], + ["moshcode wait --any api web docs", "--json names the winner"], + ], seeAlso: ["herd", "ps"], - note: "exit codes are the point: 0 matched, 2 timed out, 3 no such session.", + note: "exit codes are the point: 0 matched, 2 timed out, 3 no such session. " + + "--any/--all are what fan-in scripts used to spell out as a polling loop; a remote member waits the same way a local one does.", }, { name: "restore", @@ -930,11 +945,21 @@ export const HERD_VERBS = [ { name: "send-keys", description: "send raw keys (Enter, Escape, C-c, literal text)", synopsis: [["moshcode herd send-keys ", ""]] }, { name: "wait", description: "block until a session reaches a state", - synopsis: [["moshcode herd wait [--state blocked,done]", ""]], + synopsis: [ + ["moshcode herd wait [--state blocked,done]", ""], + ["moshcode herd wait --any …", "returns on the first one to get there"], + ["moshcode herd wait --all …", "returns when every one of them has"], + ], flags: [ ["--state ", "states to wait for", "blocked,done"], + ["--any", "return on the first session to reach the state", ""], + ["--all", "return when every named session has", "implied by naming several"], ["--timeout ", "give up after this long", "30m"], ["--json", "machine-readable", ""], + ], + examples: [ + ["moshcode wait --any api web docs", "whichever finishes first"], + ["moshcode wait --all api web --state done", "join the whole fan-out"], ] }, { name: "restore", description: "rebuild remembered sessions after a reboot", synopsis: [["moshcode herd restore [--resume] [--dry-run]", ""]], @@ -955,6 +980,72 @@ export const HERD_VERBS = [ { name: "stop", description: "stop the whole runtime and everything in it", synopsis: [["moshcode herd stop --yes", ""]], flags: [["--yes, -y", "required when sessions are running", ""]] }, + + // PRD 0011 — the engine speaks, the herd remembers, and the roster reaches + // past this box. + { name: "hooks", description: "install the engine's own lifecycle hooks, so state comes from it and not from its screen", + synopsis: [["moshcode herd hooks [|all]", ""]], + flags: [ + ["--dry-run", "print the change to the engine's settings file and write nothing", ""], + ["--json", "machine-readable", ""], + ], + examples: [ + ["moshcode herd hooks install claude", "3 hooks: stop, notification, prompt-submit"], + ["moshcode herd hooks status", "per-engine, and which events are current"], + ], + note: "the file is merged, never clobbered, and remove takes out only what moshcode put in. " + + "a hook fired outside a herd session does nothing and exits 0, so installing one cannot break an engine you run by hand. " + + "screen rules stay as the fallback." }, + { name: "doctor", description: "check the things that actually go wrong: substrate, manifest drift, stale reports, rules.json", + synopsis: [["moshcode herd doctor [--json]", ""]], + flags: [["--json", "machine-readable, for provisioning scripts", ""]], + note: "a broken ~/.moshcode/herd/rules.json is ignored silently everywhere else by design — this is where it gets to be loud." }, + { name: "tasks", description: "every prompt submitted to a session, and what came of it", + synopsis: [["moshcode herd tasks [--json]", ""]], + flags: [["--json", "machine-readable", ""]] }, + { name: "task", description: "one task: its state transitions and its output", + synopsis: [["moshcode herd task [--json]", ""]], + flags: [["--json", "machine-readable", ""]] }, + { name: "log", description: "the timestamped state history of a session", + synopsis: [["moshcode herd log [--json]", ""]], + flags: [["--json", "machine-readable", ""]] }, + { name: "stats", description: "time in state, including how long things sat blocked waiting on you", + synopsis: [["moshcode herd stats [session] [--json]", ""]], + flags: [["--json", "machine-readable", ""]] }, + { name: "remote", description: "put a deployed agent on the roster — A2A, or a bare POST endpoint", + synopsis: [["moshcode herd remote [args…]", ""]], + flags: [ + ["--kind ", "A2A JSON-RPC, or a plain POST {prompt}", "run"], + ["--json", "machine-readable", ""], + ], + examples: [ + ["moshcode herd remote add research https://agents.do-ai.run/…/production --kind run", ""], + ["moshcode herd prompt research \"summarise the week\"", "the same verb as a local session"], + ], + note: "auth comes from MOSHCODE_REMOTE__TOKEN in the environment — never written to the manifest, never synced. " + + "a remote's state is the remote's claim, and `ps` says `remote` in the from column so nobody mistakes it for something this box verified." }, + { name: "serve", description: "expose the herd over A2A v0.3.0, behind your moshcode login", + synopsis: [["moshcode herd serve [--port 7683] [--bind 127.0.0.1]", ""]], + flags: [ + ["--port ", "port to listen on", "7683"], + ["--bind ", "interface to bind", "127.0.0.1"], + ["--expose-autonomous", "also serve sessions started with --agent", "off"], + ], + note: "message/send is keystrokes into a real pty. there is no unauthenticated mode, loopback included, and sessions " + + "started with --agent are withheld unless you ask for them: an engine with approvals bypassed plus a network prompt " + + "is the worst pairing on the menu." }, + { name: "eval", description: "run a dataset through several engines and score them", + synopsis: [["moshcode herd eval --dataset --engines a,b [--judge |rules]", ""]], + flags: [ + ["--dataset ", "jsonl, json or csv of { prompt, expect | rubric }", ""], + ["--engines ", "engines to compare", ""], + ["--judge |rules", "score with an engine, or with the dataset's own patterns", "rules"], + ["--threshold <0-1>", "the score every engine has to reach", "0.8"], + ["--keep", "leave the eval sessions running afterwards", ""], + ["--json", "machine-readable", ""], + ], + note: "exit codes are distinct on purpose: 0 pass, 4 below the threshold, 5 the harness could not run — " + + "CI has to tell a worse agent from a broken box." }, ]; export const VERB_TABLES = { diff --git a/src/commands.mjs b/src/commands.mjs index f97a2c4..a17c118 100644 --- a/src/commands.mjs +++ b/src/commands.mjs @@ -18,8 +18,9 @@ import { spawn, spawnSync } from "node:child_process"; import { createRegistry } from "./registry.mjs"; import { cliVerb, aiVerb, runMoshcode } from "./cli.mjs"; import { ingestApproval, pollApproval } from "./notify.mjs"; -import { capture, killSession, sendPrompt } from "./herd.mjs"; -import { herdStart, roster, waitFor } from "./herd-cli.mjs"; +import { capture, killSession, remoteStatus, sendPrompt } from "./herd.mjs"; +import { herdStart, isRemoteMember, roster, waitForMany, waitMember } from "./herd-cli.mjs"; +import { endTask, findTask, readTasks, startTask } from "./herd-tasks.mjs"; import { shellInvocation } from "./shell.mjs"; import { identity, loginAuto, logout as forgetCreds } from "./auth.mjs"; import { expandAlias, getAlias, loadAliases, removeAlias, setAlias } from "./aliases.mjs"; @@ -511,29 +512,62 @@ const COMMANDS = [ }, { name: "herdPrompt", - summary: "type a prompt into a herd session", + summary: "type a prompt into a herd session (local or remote)", usage: "herdPrompt(name, text)", - detail: "returns { ok }; does not wait — use herdWait() to join", + detail: "returns { ok, task }; does not wait — use herdWait() to join", run(ctx, name, ...words) { const text = words.join(" "); if (!name || !text) throw new Error("moshscript: herdPrompt(name, text) requires both"); if (ctx.dryRun) { ctx.out(` 💬 herdPrompt(${name}) → would send: ${text}`); return { ok: true, dryRun: true }; } ctx.out(` 💬 herdPrompt(${name}) → ${text.slice(0, 60)}${text.length > 60 ? "…" : ""}`); - const sent = sendPrompt(String(name), text); - return { ok: Boolean(sent.ok) }; + const session = String(name); + + // A remote member takes the same call with the same arguments — the point + // of PRD 0011 R12 is that a script fanning across a local pty and a + // deployed agent contains no `if (remote)`. This one stays synchronous + // like every other local verb, so the no-`await` style keeps working: the + // request is in flight when it returns, and herdWait() is how a script + // joins on it, exactly as for a local session. + if (isRemoteMember(session)) { + const task = startTask(session, text, { screen: "" }); + import("./herd-remote.mjs") + .then((remote) => remote.promptRemote(session, text) + .then((sent) => endTask(session, task, { state: sent.state || "done", artifact: sent.artifact || String(sent.error?.message || "") }))) + .catch(() => endTask(session, task, { state: "done", artifact: "the request never left this machine" })); + return { ok: true, task, remote: true }; + } + + const task = startTask(session, text, { screen: capture(session, { lines: 60 }) }); + const sent = sendPrompt(session, text); + if (!sent.ok) endTask(session, task, { state: "done", artifact: String(sent.error?.message || sent.error) }); + return { ok: Boolean(sent.ok), task }; }, }, { name: "herdWait", summary: "BLOCK until a herd session is blocked, done, or idle", - usage: "herdWait(name, { states, timeout })", - detail: "returns the state it reached. needs await", + usage: "herdWait(name | [names], { states, timeout, any })", + detail: "one name returns the state it reached; a list returns the winner's name (any) or every result (all). needs await", async run(ctx, name, opts = {}) { if (!name) throw new Error("moshscript: herdWait(name) requires a session name"); const states = opts.states || ["blocked", "done", "idle"]; + const timeout = opts.timeout ? { timeoutMs: Number(opts.timeout) } : {}; + + // A list of names is a join (PRD 0011 R8) — the thing every fan-out + // script so far has spelled out by hand as a polling loop. + if (Array.isArray(name)) { + const names = name.map(String); + const mode = opts.any ? "any" : "all"; + if (ctx.dryRun) { ctx.out(` ⏳ herdWait([${names.join(", ")}]) → would wait for ${mode} of them to reach ${states.join("/")}`); return mode === "any" ? names[0] : names.map((n) => ({ name: n, state: "idle" })); } + ctx.out(` ⏳ herdWait([${names.join(", ")}]) → waiting for ${mode}…`); + const result = await waitForMany(names, states, { mode, ...timeout }); + ctx.out(` ${result.outcome === "matched" ? "✅" : "⌛"} ${mode === "any" ? `${result.winner} first` : `${result.outcome}`}`); + return mode === "any" ? result.winner : result.results; + } + if (ctx.dryRun) { ctx.out(` ⏳ herdWait(${name}) → would wait for ${states.join("/")}`); return "idle"; } ctx.out(` ⏳ herdWait(${name}) → waiting for ${states.join("/")}…`); - const result = await waitFor(String(name), states, opts.timeout ? { timeoutMs: Number(opts.timeout) } : {}); + const result = await waitMember(String(name), states, timeout); ctx.out(` ${result.outcome === "matched" ? "✅" : "⌛"} ${name} is ${result.state}`); return result.state; }, @@ -546,7 +580,11 @@ const COMMANDS = [ run(ctx, name, opts = {}) { if (!name) throw new Error("moshscript: herdRead(name) requires a session name"); if (ctx.dryRun) { ctx.out(` 📖 herdRead(${name}) → would read its screen`); return ""; } - return capture(String(name), { lines: Number(opts.lines) || 60 }); + const session = String(name); + // A remote has no screen; what it has is the last thing it said, and + // that is what `read` means for it (PRD 0011 R12). + if (isRemoteMember(session)) return String(remoteStatus(session)?.artifact || ""); + return capture(session, { lines: Number(opts.lines) || 60 }); }, }, { @@ -572,6 +610,41 @@ const COMMANDS = [ }, }, + // The ledger (PRD 0011 R6). Same contract as herdRead/herdList and for the + // same reason: a script fans work out and then has to read what came back. + // `[]`/`null` on anything missing, never a throw — a script joining on four + // agents must not die because one of them has no history yet. + // + // herdPrompt("api", "port the auth routes"); + // await herdWait("api"); + // const [last] = herdTasks("api").slice(-1); + // say(herdTask(last.id).artifact); + { + name: "herdTasks", + summary: "every prompt submitted to a session, and what came of it", + usage: "herdTasks(name)", + detail: "returns [{ id, text, state, status, submitted, durationMs }, …], oldest first", + run(ctx, name) { + if (!name) throw new Error("moshscript: herdTasks(name) requires a session name"); + if (ctx.dryRun) { ctx.out(` 📒 herdTasks(${name}) → would read the ledger`); return []; } + try { + return readTasks(String(name)).map(({ id, text, state, status, submitted, endedAt, durationMs }) => + ({ id, text, state, status, submitted, endedAt, durationMs })); + } catch { return []; } + }, + }, + { + name: "herdTask", + summary: "one task by id — its transitions and its output", + usage: "herdTask(id)", + detail: "returns { id, session, text, transitions, artifact, state } or null", + run(ctx, id) { + if (!id) throw new Error("moshscript: herdTask(id) requires a task id"); + if (ctx.dryRun) { ctx.out(` 📒 herdTask(${id}) → would read the ledger`); return null; } + try { return findTask(String(id)); } catch { return null; } + }, + }, + // CLI verbs — each is `moshcode ...args`. This is the whole point: // scripting the CLI. Add a capability by adding a line here. // @@ -600,6 +673,7 @@ const COMMANDS = [ cliVerb("supabase", "drive the Supabase CLI (local stack, migrations, functions)"), cliVerb("doppler", "drive the Doppler CLI (secrets, env injection)"), cliVerb("doctl", "drive the DigitalOcean CLI (droplets, apps, databases)"), + cliVerb("gradient", "drive the DigitalOcean Gradient ADK (init, run, deploy, logs, evaluate)"), cliVerb("turso", "drive the Turso CLI (auth, databases, replicas)"), cliVerb("tailscale", "drive the Tailscale CLI (mesh VPN: up, status, ssh, serve)"), cliVerb("coral", "drive the Coral CLI (SQL over APIs, databases, and internal systems)"), diff --git a/src/engines.mjs b/src/engines.mjs index c6515c2..150a48b 100644 --- a/src/engines.mjs +++ b/src/engines.mjs @@ -19,6 +19,14 @@ // repeating; only put a pattern here when it is this engine's own wording. // Every pattern is matched against the bottom of the screen with ANSI stripped. // +// `hooks` (optional) is how this engine reports its own state, so the herd can +// stop reading paint (PRD 0011 R1). It sits here for the same reason `state` +// does — and because the two are the same fact at different confidence: a hook +// spec supersedes the screen rules below it, so an engine that gains one should +// keep its rules rather than delete them. A missing `hooks` is not a gap to +// paper over with a guess; it means this engine is classified from its screen, +// which is what every engine did before. +// // `resume` (optional) is the argv that reopens this engine's last conversation, // used by `moshcode restore --resume` after a reboot. Omit it rather than guess: // a session that starts fresh is a small disappointment, and one that starts @@ -82,6 +90,30 @@ export const ENGINES = { "CLAUDECODE", "CLAUDE_CODE_ENTRYPOINT", "CLAUDE_CODE_SESSION_ID", "CLAUDE_CODE_CHILD_SESSION", ], resume: ["--continue"], + // The engine speaking for itself (PRD 0011 R1). Claude Code's lifecycle + // hooks are documented and stable, which is why it is the one engine that + // ships a spec here rather than a promise to write one: a hook schema we + // guessed at would be a rule that rots with no screen to fall back to. + // + // `file` is a function, not a path, because the whole install is a write to + // the user's home and the only honest way to test that is to move HOME. + hooks: { + format: "claude-settings", + file: () => path.join(homedir(), ".claude", "settings.json"), + // Stop fires when the turn ends, which is the end of the *task* — A2A + // calls the same moment `completed`. Notification covers both halves of + // blocked (a permission request and a plain question). UserPromptSubmit + // is the cheapest honest `working`: PreToolUse would also do it, at the + // cost of forking a moshcode per tool call for a state it is already in. + // `label` is what moshcode calls the event; `event` is what the engine + // calls it. They differ because one is a sentence and the other is a + // schema key, and printing the schema key at someone is not an answer. + events: [ + { event: "Stop", state: "done", label: "stop" }, + { event: "Notification", state: "blocked", label: "notification" }, + { event: "UserPromptSubmit", state: "working", label: "prompt-submit" }, + ], + }, state: { // The permission dialog's own heading, and the selector on its first // option — the generic numbered-menu pattern would catch the second only diff --git a/src/herd-cli.mjs b/src/herd-cli.mjs index 51cd046..39f9754 100644 --- a/src/herd-cli.mjs +++ b/src/herd-cli.mjs @@ -13,13 +13,21 @@ import { herdDir, killSession, listSessions, paneIndex, readManifest, rememberSession, sendKeys, sendPrompt, slugifyName, startSession, stopRuntime, substrateNote, validName, NAME_RE, } from "./herd.mjs"; -import { clearReport, reportState, STATES, withState } from "./herd-state.mjs"; +import { BLOCKED_KINDS, clearReport, inspectUserRules, reportState, STATES, withState } from "./herd-state.mjs"; import { ENGINES, resolveEngine, resolveExecutable, agentLaunchArgs } from "./engines.mjs"; +import { + endTask, findTask, ledgerSessions, openTask, readLog, readTasks, TERMINAL_STATES, + recordTransition, screenDelta, startTask, stats as taskStats, +} from "./herd-tasks.mjs"; import { ingestApproval, pollApproval } from "./notify.mjs"; import { acid, amber, ash, bone, danger, dim, err, info, ok, table, warn } from "./ui.mjs"; -/** Distinct exit codes, because `wait` exists to be branched on (R10). */ -export const EXIT = { matched: 0, usage: 1, timeout: 2, gone: 3 }; +/** + * Distinct exit codes, because `wait` exists to be branched on (0009 R10), and + * because `eval` in CI has to tell "the agent got worse" apart from "the + * harness fell over" (0011 R13). One non-zero code cannot say both. + */ +export const EXIT = { matched: 0, usage: 1, timeout: 2, gone: 3, below: 4, infra: 5 }; const configFile = () => path.join(herdDir(), "config.json"); @@ -87,10 +95,19 @@ export function renderRoster(rows, { indent = " " } = {}) { bone(r.name), ash(String(r.engine)), paintState(r.state), + // A remote member's cwd is the host it answers on, set when it was added: + // "where is this thing" is the same question for both, and the answer is + // a directory for one and a hostname for the other. ash(tilde(r.cwd || "")), - dim(humanAge(r.age)), + // A remote row has no age worth printing — it was registered, not + // started, and "3d" would read as three days of work. + dim(r.kind === "remote" ? "—" : humanAge(r.age)), + // Where the state came from (0011 R1, R11). This is the column that makes + // the hook install visible — and the one that stops a remote's claim from + // being mistaken for something this box verified. + dim(String(r.authority || "")), ]), - { columns: ["name", "engine", "state", "cwd", "age"], header: false, indent: indent.length }, + { columns: ["name", "engine", "state", "cwd", "age", "from"], header: false, indent: indent.length }, ); } @@ -285,8 +302,13 @@ export function splitDetachArgs(args = []) { export function herdPs(argv, { write = console.log } = {}) { const rows = roster(); if (argv.includes("--json")) { - write(JSON.stringify(rows.map(({ name, engine, herd, state, authority, cwd, age, alive, attached, substrate }) => ({ - name, engine, herd, state, authority, cwd, ageMs: age, alive, attached, substrate, + write(JSON.stringify(rows.map(({ name, engine, herd, state, authority, blockedOn, kind, url, cwd, age, alive, attached, substrate }) => ({ + name, engine, herd, state, authority, kind, ...(url ? { url } : {}), + // The blocked sub-kind (R4) rides here and not in the roster's own + // column: `--ask` needs to know whether a menu or a sentence is wanted, + // and a person glancing at six rows does not. + ...(blockedOn ? { blockedOn } : {}), + cwd, ageMs: age, alive, attached, substrate, })), null, 2)); return EXIT.matched; } @@ -361,12 +383,23 @@ export async function herdAttach(argv, { write = console.log } = {}) { return EXIT.matched; } -export function herdKill(argv, { write = console.log } = {}) { +export async function herdKill(argv, { write = console.log } = {}) { const all = argv.includes("--all"); const names = all ? roster().map((s) => s.name) : argv.filter((a) => !a.startsWith("-")); if (!names.length) { write(err("usage: moshcode kill | --all")); return EXIT.usage; } let failed = 0; for (const name of names) { + // Killing a remote is deregistering it. There is no process of ours on the + // other end, and reaching across the network to end somebody else's agent + // because a local roster entry was removed would be a `kill` that does + // considerably more than it says. + if (isRemoteMember(name)) { + const remote = await import("./herd-remote.mjs"); + const dropped = remote.removeRemote(name); + if (dropped.ok) write(ok(`${name} removed from the roster — the agent at the far end is untouched.`)); + else { write(err(`${name}: ${dropped.error?.message}`)); failed++; } + continue; + } const result = killSession(name); clearReport(name); if (result.ok) write(ok(`${name} ended.`)); @@ -378,6 +411,11 @@ export function herdKill(argv, { write = console.log } = {}) { /** * Drop sessions the runtime no longer has. Only ever removes bookkeeping — a * `prune` that could end running work would be a `kill` with a friendlier name. + * + * The task ledger is deliberately NOT pruned with the session. "What did that + * agent do before the box rebooted" is the question the ledger exists for, and + * a prune is usually the moment someone starts asking it. Growth is bounded by + * the per-session cap in herd-tasks.mjs instead. */ export function herdPrune(argv, { write = console.log } = {}) { const gone = roster().filter((s) => !s.alive); @@ -398,6 +436,9 @@ export function herdRead(argv, { write = console.log } = {}) { } const name = positional[0]; if (!name) { write(err("usage: moshcode herd read [--lines N]")); return EXIT.usage; } + // A remote has no screen — what it has is the last thing it said, which is + // what `read` is for in both cases. + if (isRemoteMember(name)) return readRemoteMember(name, { json, write }); const session = findSession(name); if (!session?.alive) { write(err(`no live session named ${JSON.stringify(name)}`)); return EXIT.gone; } const screen = capture(name, { lines }); @@ -405,6 +446,24 @@ export function herdRead(argv, { write = console.log } = {}) { return EXIT.matched; } +async function readRemoteMember(name, { json, write }) { + const remote = await import("./herd-remote.mjs"); + const entry = remote.remoteEntry(name); + if (!entry) { write(err(`no member named ${JSON.stringify(name)}`)); return EXIT.gone; } + const text = remote.readRemote(name); + const status = remote.remoteStatusOf(name); + if (json) { + write(JSON.stringify({ name, kind: "remote", url: entry.url, state: status?.state || "unknown", observedAt: status?.at || null, screen: text }, null, 2)); + return EXIT.matched; + } + if (!text) { + write(info(`${name} has not answered anything yet — ${acid(`moshcode herd prompt ${name} "…"`)}`)); + return EXIT.matched; + } + write(text); + return EXIT.matched; +} + /** * Deliberately NOT unref'd. * @@ -430,12 +489,20 @@ export async function waitFor(name, states, { intervalMs = 1000, now = () => Date.now(), look = (n) => findSession(n), + // Called with every state this poll observes that differs from the last one. + // The ledger (0011 R5) is written from here rather than from a second poller: + // this loop already sees every transition a task goes through, and a second + // one watching the same sessions would be twice the `capture-pane` for the + // same answer. + onState = null, } = {}) { const wanted = new Set(states); const deadline = now() + timeoutMs; + let seen; for (;;) { const session = look(name); if (!session) return { outcome: "gone", state: "gone" }; + if (onState && session.state !== seen) { seen = session.state; onState(session.state, session); } if (wanted.has(session.state)) return { outcome: "matched", state: session.state }; // A session that ended can never reach `blocked`; waiting the full timeout // for something impossible is a hang, not a wait. @@ -456,24 +523,122 @@ function parseDuration(raw, fallback) { return { ms: n, s: n * 1000, m: n * 60000, h: n * 3600000 }[m[2] || "s"]; } +/** Is this member a URL rather than a pty? Read straight from the manifest. */ +export function isRemoteMember(name) { + return readManifest().sessions[name]?.kind === "remote"; +} + +/** + * Wait on one member, wherever it lives (0011 R12). + * + * A remote is polled by asking it, a local by looking at it, and the caller + * writes the same `if` either way — which is the whole claim R12 makes. + */ +export async function waitMember(name, states, options = {}) { + if (!isRemoteMember(name)) return waitFor(name, states, options); + const remote = await import("./herd-remote.mjs"); + return remote.waitRemote(name, states, options); +} + +/** + * Wait on several members at once (0011 R8). + * + * `--any` returns on the first to arrive, `--all` when the last one has. Every + * fan-out script written against the herd so far ends with a hand-rolled loop + * doing one of these two things; this is that loop, once. + */ +export async function waitForMany(names, states, { + mode = "any", + timeoutMs = 30 * 60 * 1000, + intervalMs = 1500, + now = () => Date.now(), + nap = sleep, + observe = observeMember, +} = {}) { + const wanted = new Set(states); + const deadline = now() + timeoutMs; + // ONE loop over all of them, rather than N waits raced against each other. + // A race leaves the losers polling a process that has already printed its + // answer, and their timers keep node alive — `wait --any` would return the + // right thing and then refuse to exit for half an hour. + const done = new Map(); + for (;;) { + for (const name of names) { + if (done.has(name)) continue; + const seen = await observe(name); + if (!seen.present) { done.set(name, { name, outcome: "gone", state: "gone" }); continue; } + if (wanted.has(seen.state)) { done.set(name, { name, outcome: "matched", state: seen.state }); continue; } + if (seen.alive === false || seen.state === "done") done.set(name, { name, outcome: "ended", state: seen.state }); + } + const results = [...done.values()]; + const matched = results.filter((r) => r.outcome === "matched"); + if (mode === "any" && matched.length) { + return { mode, outcome: "matched", winner: matched[0].name, first: matched[0], results }; + } + if (done.size === names.length) { + if (mode === "all") { + const missed = results.find((r) => r.outcome !== "matched"); + return { mode, outcome: missed ? missed.outcome : "matched", winner: null, results }; + } + return { mode, outcome: results[0]?.outcome || "gone", winner: null, results }; + } + if (now() >= deadline) { + return { mode, outcome: "timeout", winner: null, results, pending: names.filter((n) => !done.has(n)) }; + } + await nap(intervalMs); + } +} + +/** One member's state right now — a look for a local, a request for a remote. */ +export async function observeMember(name) { + if (isRemoteMember(name)) { + const remote = await import("./herd-remote.mjs"); + if (!remote.remoteEntry(name)) return { name, present: false }; + const pinged = await remote.pingRemote(name).catch(() => null); + return { name, present: true, alive: true, state: pinged?.state || "unknown" }; + } + const session = findSession(name); + return session + ? { name, present: true, alive: session.alive, state: session.state, blockedOn: session.blockedOn } + : { name, present: false }; +} + export async function herdWait(argv, { write = console.log } = {}) { const positional = []; - let states = ["blocked", "done"], timeoutMs = 30 * 60 * 1000, json = false; + let states = ["blocked", "done"], timeoutMs = 30 * 60 * 1000, json = false, mode = null; for (let i = 0; i < argv.length; i++) { const a = argv[i]; if (a === "--state") states = String(argv[++i] || "").split(",").filter(Boolean); else if (a.startsWith("--state=")) states = a.slice(8).split(",").filter(Boolean); else if (a === "--timeout") timeoutMs = parseDuration(argv[++i], timeoutMs); else if (a.startsWith("--timeout=")) timeoutMs = parseDuration(a.slice(10), timeoutMs); + else if (a === "--any") mode = "any"; + else if (a === "--all") mode = "all"; else if (a === "--json") json = true; else if (!a.startsWith("-")) positional.push(a); } - const name = positional[0]; - if (!name) { write(err("usage: moshcode wait [--state blocked,done] [--timeout 30m]")); return EXIT.usage; } + if (!positional.length) { + write(err("usage: moshcode wait [--any|--all] [--state blocked,done] [--timeout 30m]")); + return EXIT.usage; + } const unknown = states.filter((s) => !STATES.includes(s)); if (unknown.length) { write(err(`unknown state ${unknown[0]} — one of ${STATES.join(", ")}`)); return EXIT.usage; } + if (!mode && positional.length > 1) mode = "all"; // several names and no verb: join on all of them + + if (mode) { + const result = await waitForMany(positional, states, { mode, timeoutMs }); + if (json) write(JSON.stringify({ mode, ...result }, null, 2)); + else if (result.outcome === "matched") { + write(mode === "any" + ? ok(`${result.winner} is ${result.first.state} first.`) + : ok(`all ${positional.length} reached ${states.join("/")}.`)); + } else write(warn(`${mode === "any" ? "none of them" : "not all of them"} reached ${states.join("/")} (${result.outcome}).`)); + if (result.outcome === "matched") return EXIT.matched; + return result.outcome === "timeout" ? EXIT.timeout : EXIT.gone; + } - const result = await waitFor(name, states, { timeoutMs }); + const name = positional[0]; + const result = await waitMember(name, states, { timeoutMs, onState: ledgerRecorder(name) }); if (json) write(JSON.stringify({ name, ...result }, null, 2)); else if (result.outcome === "matched") write(ok(`${name} is ${result.state}.`)); else if (result.outcome === "timeout") write(warn(`${name} is still ${result.state} after the timeout.`)); @@ -485,6 +650,22 @@ export async function herdWait(argv, { write = console.log } = {}) { return EXIT.gone; } +/** + * The ledger write a poll performs (0011 R5). + * + * Attributed to whichever task is open on that session, so a `wait` that + * happens to be running while an agent works fills in the history of the prompt + * that started it. With no open task the transition is still recorded, unbound + * — `herd log` and `herd stats` want the state history whether or not anyone + * submitted the work through the herd. + */ +export function ledgerRecorder(name) { + return (state, session) => { + const open = openTask(name); + recordTransition(name, state, { id: open?.id || null, kind: session?.blockedOn || null }); + }; +} + /** * Type a prompt into a running session, optionally waiting for it to land. * @@ -508,21 +689,73 @@ export async function herdPrompt(argv, { write = console.log } = {}) { const [name, ...words] = positional; const text = words.join(" "); if (!name || !text) { write(err('usage: moshcode herd prompt "" [--wait]')); return EXIT.usage; } + + // A remote member takes the same verb and the same flags (0011 R12). The + // whole point is that a fan-out script contains no `if (remote)`, so this is + // the one place that does. + if (isRemoteMember(name)) return promptRemoteMember(name, text, { wait, json, write }); + const session = findSession(name); if (!session?.alive) { write(err(`no live session named ${JSON.stringify(name)}`)); return EXIT.gone; } + // The task is minted BEFORE the keystrokes land, so a prompt that sends and + // then vanishes into a crashed engine still leaves evidence that it was + // submitted. A ledger that only records successful work is a ledger that + // cannot answer the one question anybody asks it at 3am. + const at = Date.now(); + const baseline = capture(name, { lines: 60 }); + const taskId = startTask(name, text, { screen: baseline, now: at, state: session.state }); + const sent = sendPrompt(name, text); - if (!sent.ok) { write(err(String(sent.error?.message || sent.error))); return EXIT.usage; } + if (!sent.ok) { + endTask(name, taskId, { state: "done", artifact: `moshcode could not type into ${name}: ${sent.error?.message || sent.error}` }); + write(err(String(sent.error?.message || sent.error))); + return EXIT.usage; + } if (!wait) { - if (json) write(JSON.stringify({ name, sent: true }, null, 2)); - else write(ok(`sent to ${bone(name)}.`)); + if (json) write(JSON.stringify({ name, sent: true, task: taskId }, null, 2)); + else write(ok(`sent to ${bone(name)} — ${ash(taskId)}`)); return EXIT.matched; } - await waitFor(name, ["working"], { timeoutMs: 8000, intervalMs: 500 }); - const result = await waitFor(name, ["blocked", "done", "idle"], { timeoutMs }); - if (json) write(JSON.stringify({ name, sent: true, ...result }, null, 2)); - else if (result.outcome === "matched") write(ok(`${name} is ${result.state}.`)); + const record = ledgerRecorder(name); + await waitFor(name, ["working"], { timeoutMs: 8000, intervalMs: 500, onState: record }); + const result = await waitFor(name, ["blocked", "done", "idle"], { timeoutMs, onState: record }); + endTask(name, taskId, { + state: result.state, + artifact: screenDelta(baseline, capture(name, { lines: 400 })), + }); + if (json) write(JSON.stringify({ name, sent: true, task: taskId, ...result }, null, 2)); + else if (result.outcome === "matched") write(ok(`${name} is ${result.state}. ${ash(`${taskId} — moshcode herd task ${taskId}`)}`)); + else write(warn(`${name}: ${result.outcome} (${result.state})`)); + return result.outcome === "matched" ? EXIT.matched : result.outcome === "timeout" ? EXIT.timeout : EXIT.gone; +} + +/** `herd prompt` against a URL. Same ledger, same exit codes, different wire. */ +async function promptRemoteMember(name, text, { wait, json, write }) { + const remote = await import("./herd-remote.mjs"); + const at = Date.now(); + const taskId = startTask(name, text, { screen: "", now: at }); + const sent = await remote.promptRemote(name, text); + if (!sent.ok) { + endTask(name, taskId, { state: "done", artifact: String(sent.error?.message || sent.error) }); + write(err(String(sent.error?.message || sent.error))); + return EXIT.gone; + } + // An `a2a` member answers with a task that may still be running; a `run` + // member has already answered by the time the POST returns. Both end up in + // the ledger, which is what makes `herd tasks ` mean anything. + if (!wait || sent.state === "done") { + endTask(name, taskId, { state: sent.state || "done", artifact: sent.artifact || "" }); + if (json) write(JSON.stringify({ name, sent: true, task: taskId, remoteTask: sent.taskId || null, state: sent.state }, null, 2)); + else write(ok(`${bone(name)} answered — ${ash(taskId)}`)); + return EXIT.matched; + } + const result = await remote.waitRemote(name, ["blocked", "done", "idle"]); + const artifact = remote.readRemote(name); + endTask(name, taskId, { state: result.state, artifact }); + if (json) write(JSON.stringify({ name, sent: true, task: taskId, ...result }, null, 2)); + else if (result.outcome === "matched") write(ok(`${name} is ${result.state}. ${ash(`${taskId}`)}`)); else write(warn(`${name}: ${result.outcome} (${result.state})`)); return result.outcome === "matched" ? EXIT.matched : result.outcome === "timeout" ? EXIT.timeout : EXIT.gone; } @@ -549,8 +782,15 @@ export function herdReport(argv, { write = console.log } = {}) { else if (!a.startsWith("-")) positional.push(a); } const [name, state] = positional; + // A hook fired outside a herd session passes an empty name, because + // $MOSHCODE_HERD_NAME is not set there. That is not a mistake to complain + // about — it is an engine being used by hand, which is most of the time — + // and a hook that printed usage on every turn would be uninstalled by + // lunchtime. Present-but-empty is silence; absent entirely is still usage. + if (positional.length >= 1 && name === "") return EXIT.matched; if (!name || !state) { write(err(`usage: moshcode herd report <${STATES.join("|")}> [--ttl 15m]`)); + write(info(`blocked also takes a sub-kind: ${BLOCKED_KINDS.map((k) => `blocked:${k}`).join(", ")}`)); return EXIT.usage; } const result = reportState(name, state, ttl ? { ttl } : {}); @@ -646,6 +886,11 @@ export async function herdWatch(argv, { write = console.log, once = false } = {} const seen = new Map(); for (;;) { + // Remotes first, so this tick's roster reads a status cache that was + // refreshed this tick rather than last one. The watcher is the only thing + // in the herd that runs continuously, which makes it the only honest place + // to keep a remote's state fresh (0011 R11). + await refreshRemotes(); // One roster per tick, not one per session: this loop runs forever, and // re-reading the herd inside the cleanup pass made a watcher on six // sessions shell out dozens of times every five seconds, all night. @@ -653,6 +898,10 @@ export async function herdWatch(argv, { write = console.log, once = false } = {} for (const session of current) { const previous = seen.get(session.name); seen.set(session.name, session.state); + // The ledger write goes exactly where the notification decision already + // is (0011 R5). Every transition, not only the ones worth a phone call — + // "it worked for six hours and never asked me anything" is history too. + if (previous !== undefined && previous !== session.state) recordObservedTransition(session); if (!shouldNotify(previous, session.state, interesting)) continue; await deliver(session, config, write); } @@ -663,9 +912,56 @@ export async function herdWatch(argv, { write = console.log, once = false } = {} } } +/** + * What a reply to each kind of blocked has to look like. + * + * Sent with the notification rather than checked on the way back, because the + * herd cannot know what a given engine's menu accepts and guessing wrong would + * mean silently refusing to deliver a valid answer. Telling the human is the + * part that is always safe. + */ +const ANSWER_HINT = { + menu: "it is on a numbered menu — reply with the number.", + permission: "it is asking permission — reply y or n.", + question: "it asked a question — reply in words.", +}; + +/** Ask every remote member how it is, so the roster's cache is this tick's. */ +async function refreshRemotes() { + const remotes = roster().filter((s) => s.kind === "remote"); + if (!remotes.length) return; + const remote = await import("./herd-remote.mjs"); + await Promise.all(remotes.map((s) => remote.pingRemote(s.name).catch(() => null))); +} + +/** + * Write one observed transition, and close the open task when the session has + * stopped needing the CPU. + * + * This is what makes a prompt submitted WITHOUT `--wait` still end up with an + * outcome and an artifact: the watcher is running anyway, and it is looking at + * exactly the transition that ends the task. + */ +function recordObservedTransition(session) { + const open = openTask(session.name); + recordTransition(session.name, session.state, { id: open?.id || null, kind: session.blockedOn || null }); + if (!open) return; + if (!TERMINAL_STATES.includes(session.state)) return; + const screen = session.kind === "remote" ? "" : capture(session.name, { lines: 400 }); + endTask(session.name, open.id, { + state: session.state, + artifact: session.kind === "remote" ? "" : screenDelta(open.baseline, screen), + }); +} + async function deliver(session, config, write) { const tail = capture(session.name, { lines: 30 }).split("\n").slice(-12).join("\n"); - const message = `${session.name} (${session.engine}) is ${session.state} in ${tilde(session.cwd)}\n\n${tail}`; + // The sub-kind (0011 R4) tells the human what shape of answer is wanted + // before they read the screen — a menu wants a digit, a permission wants a + // y or an n, and a question wants a sentence. + const asking = session.blockedOn ? ` (${session.blockedOn})` : ""; + const message = `${session.name} (${session.engine}) is ${session.state}${asking} in ${tilde(session.cwd)}` + + `${session.blockedOn ? `\n\n${ANSWER_HINT[session.blockedOn]}` : ""}\n\n${tail}`; if (!config.notify.ask) { const r = await ingestApproval({ message, kind: "notify", script: "herd", session: session.name }); write(r.ok ? info(`notified: ${session.name} → ${session.state}`) : warn(`notify failed (${r.error || r.status}) — run \`moshcode login\``)); @@ -740,6 +1036,496 @@ export function herdStop(argv, { write = console.log } = {}) { return EXIT.matched; } +// --------------------------------------------------------------------------- +// Hooks — believe the engine, not the paint (0011 R1) +// --------------------------------------------------------------------------- + +export async function herdHooks(argv, { write = console.log } = {}) { + const { + hookableEngines, hooksStatus, installHooks, removeHooks, hookDiff, hookFile, + } = await import("./herd-hooks.mjs"); + + const positional = argv.filter((a) => !a.startsWith("-")); + const [verb = "status", target] = positional; + const dryRun = argv.includes("--dry-run"); + const json = argv.includes("--json"); + const supported = hookableEngines(); + + const targets = (() => { + if (!target || target === "all") return supported; + const resolved = resolveEngine(target); + return resolved ? [resolved[0]] : []; + })(); + + if (verb !== "status" && !targets.length) { + write(err(`no engine named ${JSON.stringify(target)}`)); + write(info(`engines with hook specs: ${supported.join(", ") || "none yet"}`)); + return EXIT.usage; + } + + if (verb === "status") { + const rows = (target && target !== "all" ? targets : supported).map((engine) => hooksStatus(engine)); + if (json) { write(JSON.stringify(rows, null, 2)); return EXIT.matched; } + if (!rows.length) { write(info("no engine in this release ships a hook spec.")); return EXIT.matched; } + for (const row of rows) { + if (!row.readable) { write(err(`${row.engine} — ${row.error}`)); continue; } + const state = row.installed ? ok(`${row.engine} — hooks installed`) + : row.partial ? warn(`${row.engine} — hooks are out of date, re-run install`) + : info(`${row.engine} — no hooks; sessions are classified from the screen`); + write(state); + write(ash(` ${row.file}`)); + for (const e of row.events) { + write(` ${e.installed && e.current ? acid("✓") : e.installed ? amber("~") : ash("·")} ${(e.label || e.event).padEnd(16)} ${ash(`→ ${e.state}`)}`); + } + } + const unsupported = Object.keys(ENGINES).filter((k) => !supported.includes(k)); + if (unsupported.length && !target) write(info(`no hook spec yet: ${unsupported.join(", ")} — those stay on the screen rules.`)); + return EXIT.matched; + } + + if (verb !== "install" && verb !== "remove") { + write(err("usage: moshcode herd hooks [|all] [--dry-run] [--json]")); + return EXIT.usage; + } + + const results = targets.map((engine) => (verb === "install" + ? installHooks(engine, { dryRun }) + : removeHooks(engine, { dryRun }))); + + if (json) { + write(JSON.stringify(results.map((r) => ({ + engine: r.engine, ok: r.ok, file: r.file ?? hookFile(r.engine), dryRun: Boolean(r.dryRun), + ...(r.changes ? { changes: r.changes } : {}), ...(r.removed !== undefined ? { removed: r.removed } : {}), + ...(r.error ? { error: String(r.error.message || r.error) } : {}), + })), null, 2)); + return results.every((r) => r.ok) ? EXIT.matched : EXIT.usage; + } + + for (const result of results) { + if (!result.ok) { write(err(`${result.engine} — ${result.error?.message || result.error}`)); continue; } + if (dryRun) { + const diff = hookDiff(result.before, result.after); + write(info(`${result.engine} — ${result.file} (dry run)`)); + write(diff.split("\n").some((l) => l.startsWith("+") || l.startsWith("-")) ? diff : ash(" nothing would change")); + continue; + } + if (verb === "install") { + const added = result.changes.filter((c) => c.change !== "unchanged"); + write(added.length + ? ok(`${result.engine} — ${added.length} hook${added.length === 1 ? "" : "s"} installed (${added.map((c) => c.label).join(", ")})`) + : ok(`${result.engine} — already installed`)); + if (added.length) { + write(info("sessions started from the herd now report state directly.")); + write(info("screen rules remain the fallback for everything else.")); + } + } else { + write(result.removed ? ok(`${result.engine} — ${result.removed} hook(s) removed; back to the screen rules.`) : info(`${result.engine} — nothing of ours was in there.`)); + } + } + return results.every((r) => r.ok) ? EXIT.matched : EXIT.usage; +} + +// --------------------------------------------------------------------------- +// Doctor — the things that actually go wrong (0011 R3) +// --------------------------------------------------------------------------- + +export async function herdDoctor(argv, { write = console.log } = {}) { + const { hookableEngines, hooksStatus } = await import("./herd-hooks.mjs"); + const substrate = detectSubstrate(); + const checks = []; + const add = (name, level, detail, fix = null) => checks.push({ name, level, detail, ...(fix ? { fix } : {}) }); + + // 1. Somewhere to run. + if (substrate === "tmux") add("substrate", "ok", `tmux, socket ${HERD_SOCKET}`); + else if (substrate === "pty") add("substrate", "warn", "script(1) — sessions work but cannot be resized", "install tmux"); + else add("substrate", "fail", "nothing to run sessions on", substrateNote(null)); + + // 2. Does the manifest still describe reality? + const rows = roster(); + const remembered = rows.filter((s) => !s.alive); + if (remembered.length) add("manifest", "warn", `${remembered.length} remembered session(s) the runtime no longer has: ${remembered.map((s) => s.name).join(", ")}`, "moshcode restore · moshcode herd prune"); + else add("manifest", "ok", `${rows.length} session(s), all accounted for`); + + // 3. Can we write where the state lives? A silently unwritable status dir is + // a herd where every hook report is lost and nothing anywhere says so. + const statusDir = path.join(herdDir(), "status"); + try { + fs.mkdirSync(statusDir, { recursive: true, mode: 0o700 }); + const probe = path.join(statusDir, `.doctor-${process.pid}`); + fs.writeFileSync(probe, ""); + fs.rmSync(probe, { force: true }); + add("status dir", "ok", statusDir); + } catch (error) { + add("status dir", "fail", `${statusDir} is not writable (${error.code || error.message})`, "hook reports are being dropped — fix the permissions on ~/.moshcode/herd"); + } + + // 4. Hook reports that have gone stale — an engine that stopped reporting is + // a roster quietly back on the screen rules. + const stale = []; + for (const row of rows) { + if (row.kind === "remote" || !row.alive) continue; + const file = path.join(statusDir, `${row.name}.json`); + try { + const raw = JSON.parse(fs.readFileSync(file, "utf8")); + const age = Date.now() - Number(raw.at || 0); + if (age > Math.min(Number(raw.ttl) || 0, 15 * 60 * 1000)) stale.push(`${row.name} (${humanAge(age)} old)`); + } catch { /* no report is not a stale report */ } + } + if (stale.length) add("hook reports", "warn", `expired: ${stale.join(", ")}`, "moshcode herd hooks status — the engine may have stopped reporting"); + else add("hook reports", "ok", "none expired"); + + // 5. The hooks themselves. + for (const engine of hookableEngines()) { + const status = hooksStatus(engine); + if (!status.readable) add(`hooks: ${engine}`, "fail", status.error, "fix the file, then moshcode herd hooks install"); + else if (status.installed) add(`hooks: ${engine}`, "ok", status.file); + else if (status.partial) add(`hooks: ${engine}`, "warn", "installed but out of date", `moshcode herd hooks install ${engine}`); + else add(`hooks: ${engine}`, "warn", "not installed — this engine is classified from its screen", `moshcode herd hooks install ${engine}`); + } + + // 6. rules.json, which until now failed silently by design. + const rules = inspectUserRules(); + if (!rules.present) add("rules.json", "ok", "none — using the built-in rules"); + else if (rules.ok) add("rules.json", "ok", `${rules.patterns} pattern(s) loaded`); + else { + add("rules.json", "fail", `${rules.problems.length} problem(s) — the whole file is being ignored`, + rules.problems.map((p) => `${p.where}: ${p.error}`).join(" · ")); + } + + const worst = checks.some((c) => c.level === "fail") ? "fail" : checks.some((c) => c.level === "warn") ? "warn" : "ok"; + if (argv.includes("--json")) { + write(JSON.stringify({ ok: worst !== "fail", level: worst, herdDir: herdDir(), substrate, checks }, null, 2)); + return worst === "fail" ? EXIT.infra : EXIT.matched; + } + for (const check of checks) { + const mark = check.level === "ok" ? acid("✓") : check.level === "warn" ? amber("!") : danger("✗"); + write(`${mark} ${bone(check.name.padEnd(16))} ${check.level === "ok" ? ash(check.detail) : check.detail}`); + if (check.fix) write(` ${ash("→")} ${acid(check.fix)}`); + } + return worst === "fail" ? EXIT.infra : EXIT.matched; +} + +// --------------------------------------------------------------------------- +// The ledger's read verbs (0011 R6–R7) +// --------------------------------------------------------------------------- + +/** + * Close an open task whose session has already stopped. + * + * A prompt submitted without `--wait`, on a box with no watcher running, leaves + * a task nobody ever came back to. Reading the ledger IS coming back to it: the + * session's state is looked up here anyway, so recording what it says costs + * nothing and turns "open forever" into the outcome that actually happened. + */ +function reconcile(session) { + const open = openTask(session); + if (!open) return; + const row = findSession(session); + if (!row || row.kind === "remote") return; + if (!TERMINAL_STATES.includes(row.state)) return; + endTask(session, open.id, { + state: row.state, + artifact: screenDelta(open.baseline, capture(session, { lines: 400 })), + }); +} + +export function herdTasks(argv, { write = console.log } = {}) { + const json = argv.includes("--json"); + const positional = argv.filter((a) => !a.startsWith("-")); + const name = positional[0]; + if (!name) { + write(err("usage: moshcode herd tasks [--json]")); + const known = ledgerSessions(); + write(info(known.length ? `sessions with history: ${known.join(", ")}` : "nothing has been prompted through the herd yet.")); + return EXIT.usage; + } + reconcile(name); + const tasks = readTasks(name); + if (json) { write(JSON.stringify(tasks, null, 2)); return EXIT.matched; } + if (!tasks.length) { + write(info(`no tasks recorded for ${JSON.stringify(name)} — ${acid(`moshcode herd prompt ${name} "…"`)} starts one.`)); + return EXIT.matched; + } + write(table(tasks.map((t) => [ + bone(t.id), + ash(clock(t.submitted)), + paintState(t.status === "open" ? (t.state || "working") : t.state), + dim(t.durationMs != null ? humanAge(t.durationMs) : humanAge(Date.now() - (t.submitted || Date.now()))), + ash(`"${oneLine(t.text, 48)}"`), + ]), { columns: ["task", "at", "state", "took", "prompt"], header: false, indent: 2 })); + const open = tasks.filter((t) => t.status === "open").length; + if (open) write(info(`${open} still open — ${acid("moshcode herd watch")} closes them as they land.`)); + return EXIT.matched; +} + +export function herdTask(argv, { write = console.log } = {}) { + const json = argv.includes("--json"); + const id = argv.find((a) => !a.startsWith("-")); + if (!id) { write(err("usage: moshcode herd task [--json]")); return EXIT.usage; } + const task = findTask(id); + if (!task) { write(err(`no task ${JSON.stringify(id)} — ${acid("moshcode herd tasks ")}`)); return EXIT.gone; } + if (json) { write(JSON.stringify(task, null, 2)); return EXIT.matched; } + + write(`${bone(task.id)} ${ash(`· ${task.session} · ${clock(task.submitted)}`)}`); + write(`${ash("prompt")} ${task.text}`); + write(""); + for (const [i, step] of task.transitions.entries()) { + const next = task.transitions[i + 1]?.ts ?? task.endedAt ?? Date.now(); + write(` ${ash(clock(step.ts))} ${paintState(step.state)}${step.kind ? ash(`:${step.kind}`) : ""} ${dim(humanAge(next - step.ts))}`); + } + if (task.status === "closed") write(` ${ash(clock(task.endedAt))} ${paintState(task.state)} ${dim("(end)")}`); + else write(` ${ash("…")} ${amber("open")}`); + if (task.artifact) { + write(""); + write(ash(task.truncated ? `output (last ${task.artifact.length} of ${task.artifactChars} chars):` : "output:")); + write(task.artifact); + } + return EXIT.matched; +} + +export function herdLog(argv, { write = console.log } = {}) { + const json = argv.includes("--json"); + const name = argv.find((a) => !a.startsWith("-")); + if (!name) { write(err("usage: moshcode herd log [--json]")); return EXIT.usage; } + const entries = readLog(name); + if (json) { write(JSON.stringify(entries, null, 2)); return EXIT.matched; } + if (!entries.length) { write(info(`no history for ${JSON.stringify(name)} yet.`)); return EXIT.matched; } + for (const entry of entries) { + // The end of a task and a transition into the same state are two different + // records, and a log that printed them identically would read as the herd + // seeing everything twice. + const label = entry.event === "submit" ? acid("submit") + : entry.event === "end" ? `${paintState(entry.state)}${ash(" ✓")}` + : paintState(entry.state); + write(` ${ash(clock(entry.ts))} ${label}` + + `${entry.kind ? ash(`:${entry.kind}`) : ""} ${dim(entry.id || "")}` + + `${entry.text ? ` ${ash(`"${oneLine(entry.text, 40)}"`)}` : ""}`); + } + return EXIT.matched; +} + +export function herdStats(argv, { write = console.log } = {}) { + const json = argv.includes("--json"); + const name = argv.find((a) => !a.startsWith("-")); + const sessions = name ? [name] : ledgerSessions(); + if (!sessions.length) { write(info("nothing has been prompted through the herd yet.")); return EXIT.matched; } + const all = sessions.map((s) => taskStats(s)); + if (json) { write(JSON.stringify(all, null, 2)); return EXIT.matched; } + for (const s of all) { + const parts = Object.entries(s.totals) + .filter(([, ms]) => ms > 0) + .sort((a, b) => b[1] - a[1]) + .map(([state, ms]) => `${state} ${humanAge(ms)}`); + write(`${bone(s.session.padEnd(12))} ${parts.join(ash(" · ")) || ash("no transitions recorded")}`); + // The line the whole feature is for. Blocked time is not the agent being + // slow; it is the agent finished and waiting for a person. + if (s.totals.blocked) write(` ${amber(`blocked ${humanAge(s.totals.blocked)}`)} ${ash(`over ${s.blockedSpells} spell(s) — that one is you`)}`); + } + return EXIT.matched; +} + +const clock = (ts) => (Number.isFinite(ts) ? new Date(ts).toTimeString().slice(0, 5) : " : "); +const oneLine = (text, max) => { + const flat = String(text ?? "").replace(/\s+/g, " ").trim(); + return flat.length > max ? `${flat.slice(0, max - 1)}…` : flat; +}; + +// --------------------------------------------------------------------------- +// Remote members (0011 R11) +// --------------------------------------------------------------------------- + +export async function herdRemote(argv, { write = console.log } = {}) { + const remote = await import("./herd-remote.mjs"); + const json = argv.includes("--json"); + const positional = []; + let kind = "run"; + for (let i = 0; i < argv.length; i++) { + const a = argv[i]; + if (a === "--kind") kind = String(argv[++i] || ""); + else if (a.startsWith("--kind=")) kind = a.slice(7); + else if (!a.startsWith("-")) positional.push(a); + } + const [verb = "list", name, url] = positional; + + if (verb === "list") { + const rows = remote.listRemotes(); + if (json) { write(JSON.stringify(rows, null, 2)); return EXIT.matched; } + if (!rows.length) { + write(info("no remote members — `moshcode herd remote add --kind a2a|run`")); + return EXIT.matched; + } + write(table(rows.map((r) => [ + bone(r.name), ash(r.remoteKind), paintState(r.status?.state || "unknown"), + ash(r.url), dim(r.status?.at ? `${humanAge(Date.now() - r.status.at)} ago` : "never asked"), + ]), { columns: ["name", "kind", "state", "url", "seen"], header: false, indent: 2 })); + for (const r of rows) { + if (!process.env[remote.tokenEnvVar(r.name)]) write(ash(` ${r.name}: no ${remote.tokenEnvVar(r.name)} in the environment — requests go unauthenticated`)); + } + return EXIT.matched; + } + + if (verb === "add") { + if (!name || !url) { write(err("usage: moshcode herd remote add [--kind a2a|run]")); return EXIT.usage; } + const added = remote.addRemote(name, url, { kind }); + if (!added.ok) { write(err(String(added.error?.message || added.error))); return EXIT.usage; } + write(ok(`${bone(name)} — ${kind} member at ${added.url}`)); + write(info(`auth: export ${remote.tokenEnvVar(name)}=… (never written to the manifest, never synced)`)); + const pinged = await remote.pingRemote(name); + write(pinged.ok ? ok(`it answers — ${pinged.state}`) : warn(`no answer yet: ${pinged.error?.message || pinged.error}`)); + return EXIT.matched; + } + + if (verb === "remove" || verb === "rm") { + if (!name) { write(err("usage: moshcode herd remote remove ")); return EXIT.usage; } + const removed = remote.removeRemote(name); + if (!removed.ok) { write(err(String(removed.error?.message || removed.error))); return EXIT.gone; } + write(ok(`${name} is off the roster. the agent at the far end is untouched.`)); + return EXIT.matched; + } + + if (verb === "ping") { + const names = name ? [name] : remote.listRemotes().map((r) => r.name); + if (!names.length) { write(info("no remote members to ping.")); return EXIT.matched; } + const results = []; + for (const one of names) { + const pinged = await remote.pingRemote(one); + results.push({ name: one, ok: pinged.ok, state: pinged.state || "unknown", error: pinged.error ? String(pinged.error.message || pinged.error) : null }); + if (!json) write(pinged.ok ? ok(`${one} — ${pinged.state}`) : err(`${one} — ${pinged.error?.message || pinged.error}`)); + } + if (json) write(JSON.stringify(results, null, 2)); + return results.every((r) => r.ok) ? EXIT.matched : EXIT.gone; + } + + if (verb === "card") { + if (!name) { write(err("usage: moshcode herd remote card ")); return EXIT.usage; } + const card = await remote.discoverCard(name); + if (!card.ok) { write(err(String(card.error?.message || card.error))); return EXIT.gone; } + write(JSON.stringify(card.card, null, 2)); + return EXIT.matched; + } + + write(err("usage: moshcode herd remote [args…]")); + return EXIT.usage; +} + +// --------------------------------------------------------------------------- +// serve — the herd over A2A (0011 R9–R10) +// --------------------------------------------------------------------------- + +export async function herdServe(argv, { write = console.log } = {}) { + const serve = await import("./herd-serve.mjs"); + const flag = (name, fallback) => { + const at = argv.indexOf(`--${name}`); + if (at >= 0 && argv[at + 1] && !argv[at + 1].startsWith("--")) return argv[at + 1]; + const inline = argv.find((a) => a.startsWith(`--${name}=`)); + return inline ? inline.slice(name.length + 3) : fallback; + }; + + const rawPort = flag("port", String(serve.DEFAULT_SERVE_PORT)); + const port = /^\d+$/.test(rawPort) && Number(rawPort) >= 1 && Number(rawPort) <= 65535 ? Number(rawPort) : null; + if (port === null) { write(err(`--port needs a decimal integer from 1 to 65535, got ${JSON.stringify(rawPort)}`)); return EXIT.usage; } + const bind = flag("bind", "127.0.0.1"); + const exposeAutonomous = argv.includes("--expose-autonomous"); + + const { api, token } = serve.serveCredentials(); + if (!token) { + // Not a warning. With nothing to verify tokens against, every request would + // have to be refused, and a server that refuses everything is a confusing + // way to spell "log in first". + write(err("not logged in — `moshcode login` first. herd serve has no unauthenticated mode.")); + return EXIT.usage; + } + + const base = `http://${bind}:${port}`; + const server = serve.createHerdServer({ api, exposeAutonomous, base }); + await new Promise((resolve, reject) => { + server.once("error", reject); + server.listen(port, bind, resolve); + }).catch((error) => { write(err(`could not listen on ${bind}:${port} — ${error.message}`)); }); + if (!server.listening) return EXIT.infra; + + const exposed = serve.servedSessions({ exposeAutonomous }); + write(ok(`herd A2A ${serve.A2A_PROTOCOL_VERSION} on ${base}/`)); + write(info(`card: ${base}/.well-known/agent-card.json · members: ${exposed.map((s) => s.name).join(", ") || "none yet"}`)); + write(info(`auth: moshcode login tokens verified against ${api} — every request, loopback included`)); + const hidden = roster().filter((s) => s.kind !== "remote" && readManifest().sessions[s.name]?.agent).length; + if (hidden && !exposeAutonomous) { + write(info(`${hidden} autonomous session(s) withheld — an engine with approvals bypassed plus a network prompt is the worst pairing on the menu. --expose-autonomous overrides.`)); + } + if (bind !== "127.0.0.1" && bind !== "localhost") { + write(warn("! bound past loopback — message/send is keystrokes into a real pty. prefer a tailnet address or a reverse proxy with TLS.")); + } + return new Promise(() => {}); // serve until killed +} + +// --------------------------------------------------------------------------- +// eval — which engine is best at THIS repo (0011 R13) +// --------------------------------------------------------------------------- + +export async function herdEval(argv, { write = console.log } = {}) { + const evals = await import("./herd-eval.mjs"); + let dataset = null, engines = "", judge = "rules", threshold = evals.DEFAULT_THRESHOLD; + let json = false, keep = false, timeoutMs = 10 * 60 * 1000; + for (let i = 0; i < argv.length; i++) { + const a = argv[i]; + if (a === "--dataset") dataset = argv[++i]; + else if (a.startsWith("--dataset=")) dataset = a.slice(10); + else if (a === "--engines") engines = argv[++i]; + else if (a.startsWith("--engines=")) engines = a.slice(10); + else if (a === "--judge") judge = argv[++i]; + else if (a.startsWith("--judge=")) judge = a.slice(8); + else if (a === "--threshold") threshold = Number(argv[++i]); + else if (a.startsWith("--threshold=")) threshold = Number(a.slice(12)); + else if (a === "--timeout") timeoutMs = parseDuration(argv[++i], timeoutMs); + else if (a === "--keep") keep = true; + else if (a === "--json") json = true; + } + if (!dataset || !engines) { + write(err("usage: moshcode herd eval --dataset --engines a,b [--judge |rules] [--threshold 0.8]")); + write(info("a dataset row is { \"prompt\": \"…\", \"expect\": \"pattern\" } or { \"prompt\": \"…\", \"rubric\": \"…\" } — jsonl, json, or csv")); + return EXIT.usage; + } + if (!Number.isFinite(threshold) || threshold < 0 || threshold > 1) { + write(err(`--threshold is a score between 0 and 1, got ${JSON.stringify(String(threshold))}`)); + return EXIT.usage; + } + + const loaded = evals.loadDataset(path.resolve(dataset)); + if (!loaded.ok) { write(err(String(loaded.error?.message || loaded.error))); return EXIT.usage; } + const { keys, unknown } = evals.resolveEngines(engines); + if (unknown.length) { write(err(`no engine named ${unknown.join(", ")}`)); return EXIT.usage; } + if (!keys.length) { write(err("--engines needs at least one engine")); return EXIT.usage; } + if (judge !== "rules" && !resolveEngine(judge)) { write(err(`no engine named ${JSON.stringify(judge)} to judge with`)); return EXIT.usage; } + if (!requireSubstrate(write)) return EXIT.infra; + + if (!json) write(info(`${loaded.cases.length} case(s) × ${keys.length} engine(s), judged by ${judge}`)); + const report = await evals.runEval({ + cases: loaded.cases, engines: keys, judge: judge === "rules" ? "rules" : resolveEngine(judge)[0], + threshold, timeoutMs, keep, waitFor, + out: json ? () => {} : (line) => write(ash(line)), + }); + + if (json) write(JSON.stringify(report, null, 2)); + else { + write(""); + for (const engine of report.engines) { + if (!engine.ok) { write(err(`${engine.engine.padEnd(12)} could not run — ${engine.error}`)); continue; } + const pct = `${Math.round(engine.score * 100)}%`; + const line = `${bone(engine.engine.padEnd(12))} ${engine.score >= report.threshold ? acid(pct) : amber(pct)} ${ash(`${engine.passed}/${engine.cases.length} clean`)}`; + write(engine.unscorable ? `${line} ${amber(`· ${engine.unscorable} unscorable`)}` : line); + for (const c of engine.cases.filter((c) => c.score < 1)) write(ash(` ${c.id}: ${c.why}`)); + } + write(""); + if (report.outcome === "pass") write(ok(`every engine is at or above ${report.threshold}.`)); + else if (report.outcome === "below") write(warn(`below ${report.threshold}: ${report.below.join(", ")}`)); + else write(err(`the harness could not run: ${report.broken.map((b) => `${b.engine} (${b.error})`).join(", ")}`)); + } + // Three outcomes, three codes: CI has to tell a worse agent from a broken + // harness, and one non-zero code cannot say both. + if (report.outcome === "pass") return EXIT.matched; + return report.outcome === "below" ? EXIT.below : EXIT.infra; +} + // --------------------------------------------------------------------------- // Dispatch // --------------------------------------------------------------------------- @@ -760,6 +1546,12 @@ const VERBS = { read: herdRead, prompt: herdPrompt, "send-keys": herdSendKeys, wait: herdWait, restore: herdRestore, report: herdReport, notify: herdNotify, watch: herdWatch, stop: herdStop, + // PRD 0011. Same shape as everything above: one verb, `--json` on all of + // them, and no second API anywhere — `serve` is this surface answering a + // socket rather than a parallel one. + hooks: herdHooks, doctor: herdDoctor, + tasks: herdTasks, task: herdTask, log: herdLog, stats: herdStats, + remote: herdRemote, serve: herdServe, eval: herdEval, }; export async function herdCommand(argv = [], { write = console.log } = {}) { diff --git a/src/herd-eval.mjs b/src/herd-eval.mjs new file mode 100644 index 0000000..d4c0c69 --- /dev/null +++ b/src/herd-eval.mjs @@ -0,0 +1,301 @@ +// `moshcode herd eval` — which engine is best at *this* repo (PRD 0011 R13). +// +// "Which engine should I use" is answered on the internet with benchmarks run +// against engines nobody deploys, on repos nobody has. The herd can answer it +// the only way that means anything: run your dataset through the engines you +// actually have, on the machine you actually work on, and count. +// +// Nothing here is new machinery. A row is fanned across the named engines with +// the verbs that already exist — start a session, prompt it, wait, read what +// came back out of the ledger — and scored either by a pattern the dataset +// carries or by an engine acting as judge (the `ai()` verb, which is the same +// headless invocation moshscript uses). The exit code follows `wait`'s +// discipline, because a CI job needs to tell "the agent got worse" apart from +// "the harness fell over", and a single non-zero code cannot. +// +// The DO Gradient ADK ships `gradient agent evaluate --dataset-file --categories +// --success-threshold` for deployed agents. This is that idea pointed at +// interactive engines, which is the comparison nobody else is placed to run. +import fs from "node:fs"; +import path from "node:path"; + +import { runAi } from "./cli.mjs"; +import { ENGINES, resolveEngine, resolveExecutable } from "./engines.mjs"; +import { capture, killSession, listSessions, sendPrompt, startSession } from "./herd.mjs"; +import { endTask, screenDelta, startTask } from "./herd-tasks.mjs"; + +export const DEFAULT_THRESHOLD = 0.8; + +/* --------------------------------------------------------------- datasets */ + +/** + * A minimal CSV reader: quoted fields, doubled quotes, embedded newlines. + * + * Deliberately not a dependency. A dataset is a file someone wrote by hand or + * exported from a spreadsheet, and those two shapes are the whole requirement. + */ +export function parseCsv(text) { + const rows = []; + let row = [], field = "", quoted = false; + const src = String(text ?? "").replace(/\r\n/g, "\n"); + for (let i = 0; i < src.length; i++) { + const c = src[i]; + if (quoted) { + if (c === '"') { + if (src[i + 1] === '"') { field += '"'; i++; } + else quoted = false; + } else field += c; + continue; + } + if (c === '"') { quoted = true; continue; } + if (c === ",") { row.push(field); field = ""; continue; } + if (c === "\n") { row.push(field); rows.push(row); row = []; field = ""; continue; } + field += c; + } + if (field.length || row.length) { row.push(field); rows.push(row); } + return rows.filter((r) => r.some((cell) => String(cell).trim())); +} + +/** + * Read a dataset. `.jsonl` is one object per line, `.json` an array, `.csv` a + * header row plus rows. Every shape ends up as the same list of cases. + * + * A case is { id, prompt, expect?, rubric? }: the prompt to submit, an optional + * pattern the answer must match (the `rules` judge), and an optional rubric for + * an engine judge to score against. + */ +export function loadDataset(file) { + let text; + try { text = fs.readFileSync(file, "utf8"); } + catch (error) { return { ok: false, error }; } + + const ext = path.extname(file).toLowerCase(); + let raw; + try { + if (ext === ".csv") { + const rows = parseCsv(text); + if (!rows.length) return { ok: false, error: new Error(`${file} is empty`) }; + const header = rows[0].map((h) => String(h).trim().toLowerCase()); + raw = rows.slice(1).map((cells) => Object.fromEntries(header.map((h, i) => [h, cells[i] ?? ""]))); + } else if (ext === ".json") { + const parsed = JSON.parse(text); + raw = Array.isArray(parsed) ? parsed : Array.isArray(parsed?.cases) ? parsed.cases : null; + if (!raw) return { ok: false, error: new Error(`${file} must hold an array of cases`) }; + } else { + raw = text.split("\n").filter((l) => l.trim()).map((line, i) => { + try { return JSON.parse(line); } + catch (error) { throw new Error(`${file}:${i + 1} is not valid JSON (${error.message})`); } + }); + } + } catch (error) { return { ok: false, error }; } + + const cases = []; + for (const [i, entry] of raw.entries()) { + const prompt = String(entry?.prompt ?? entry?.input ?? "").trim(); + if (!prompt) return { ok: false, error: new Error(`${file}: case ${i + 1} has no prompt`) }; + cases.push({ + id: String(entry.id ?? `case-${i + 1}`), + prompt, + expect: entry.expect ? String(entry.expect) : null, + rubric: entry.rubric ? String(entry.rubric) : null, + }); + } + if (!cases.length) return { ok: false, error: new Error(`${file} holds no cases`) }; + return { ok: true, cases }; +} + +/* ---------------------------------------------------------------- scoring */ + +/** + * The `rules` judge: does the answer match what the dataset expected? + * + * The pattern is a regex, case-insensitive, because a dataset written by hand + * says `expect: "3 tests passed"` and means it loosely. A case with no + * expectation cannot be scored by rules, and says so rather than scoring zero — + * a missing expectation is the dataset's bug, not the engine's. + */ +export function scoreByRules(testCase, answer) { + if (!testCase.expect) { + return { ok: false, score: 0, why: "no `expect` pattern — this case needs a judge, or an expectation" }; + } + let re; + try { re = new RegExp(testCase.expect, "i"); } + catch { re = null; } + const hit = re ? re.test(String(answer ?? "")) : String(answer ?? "").toLowerCase().includes(testCase.expect.toLowerCase()); + return { ok: true, score: hit ? 1 : 0, why: hit ? "matched the expectation" : "did not match the expectation" }; +} + +/** Pull the first JSON object out of an engine's answer. */ +export function extractVerdict(text) { + const raw = String(text ?? ""); + const start = raw.indexOf("{"); + if (start < 0) return null; + for (let end = raw.lastIndexOf("}"); end > start; end = raw.lastIndexOf("}", end - 1)) { + try { + const parsed = JSON.parse(raw.slice(start, end + 1)); + if (parsed && typeof parsed === "object") return parsed; + } catch { /* keep shrinking */ } + } + return null; +} + +export function judgePrompt(testCase, answer) { + return [ + "You are grading one answer produced by a coding agent. Reply with JSON only.", + "", + `TASK: ${testCase.prompt}`, + testCase.rubric ? `RUBRIC: ${testCase.rubric}` : "RUBRIC: is this a correct, complete, and directly responsive answer to the task?", + "", + "ANSWER:", + String(answer ?? "").slice(-6000), + "", + 'Reply with exactly: {"score": <0 to 1>, "why": ""}', + ].join("\n"); +} + +/** The engine judge. Returns { ok, score, why } and never throws. */ +export function scoreByJudge(testCase, answer, { engine, run = runAi, out = () => {} } = {}) { + let text; + try { text = run({ out, dryRun: false }, judgePrompt(testCase, answer), { engine }); } + catch (error) { return { ok: false, score: 0, why: `judge failed: ${String(error.message || error)}` }; } + const verdict = extractVerdict(text); + if (!verdict || typeof verdict.score !== "number" || !Number.isFinite(verdict.score)) { + return { ok: false, score: 0, why: `judge did not answer with a score (${String(text).trim().slice(0, 120)})` }; + } + return { ok: true, score: Math.max(0, Math.min(1, verdict.score)), why: String(verdict.why || "").slice(0, 200) }; +} + +/* ----------------------------------------------------------------- running */ + +const sleep = (ms) => new Promise((r) => { setTimeout(r, ms); }); + +/** + * Run every case against one engine, in its own session. + * + * Sequential within an engine because a terminal is a terminal: two prompts + * typed into one session at once interleave into one prompt neither of them + * asked. Engines run against each other in parallel, which is the fan-out. + */ +export async function runEngine(engineKey, cases, { + waitFor, + cwd = process.cwd(), + timeoutMs = 10 * 60 * 1000, + session = `eval-${engineKey}`, + keep = false, + out = () => {}, + now = () => Date.now(), +} = {}) { + const engine = ENGINES[engineKey]; + if (!engine) return { engine: engineKey, ok: false, error: `unknown engine ${engineKey}`, results: [] }; + + const already = listSessions().some((s) => s.name === session && s.alive); + if (!already) { + const bin = resolveExecutable(engine.bin, engine.binDirs || []) || engine.bin; + const started = startSession({ name: session, engine: engineKey, bin, args: engine.agentArgs || [], stripEnv: engine.stripEnv || [], cwd }); + if (!started.ok) { + // Infrastructure, not quality. Reported as such so a missing engine never + // reads as an engine that failed the dataset. + return { engine: engineKey, ok: false, error: String(started.error?.message || started.error), results: [] }; + } + // An engine needs a moment to draw its first screen; prompting into a + // terminal that has not finished starting types into nothing. + await sleep(4000); + } + + const results = []; + for (const testCase of cases) { + const at = now(); + const baseline = capture(session, { lines: 60 }); + const taskId = startTask(session, testCase.prompt, { screen: baseline, now: at }); + const sent = sendPrompt(session, testCase.prompt); + if (!sent.ok) { + endTask(session, taskId, { state: "done", artifact: "", ts: now() }); + results.push({ ...testCase, engine: engineKey, taskId, ok: false, answer: "", error: String(sent.error?.message || sent.error) }); + continue; + } + await waitFor(session, ["working"], { timeoutMs: 8000, intervalMs: 500 }); + const outcome = await waitFor(session, ["blocked", "done", "idle"], { timeoutMs }); + const answer = screenDelta(baseline, capture(session, { lines: 400 })); + endTask(session, taskId, { state: outcome.state, artifact: answer, ts: now() }); + out(` ${engineKey} · ${testCase.id} · ${outcome.outcome} (${outcome.state})`); + results.push({ ...testCase, engine: engineKey, taskId, ok: outcome.outcome === "matched", answer, outcome: outcome.outcome, state: outcome.state }); + } + + if (!already && !keep) killSession(session); + return { engine: engineKey, ok: true, session, results }; +} + +/** + * The whole run: fan the dataset across the engines, score, and total up. + * + * `waitFor` is injected rather than imported so the runner can be exercised + * without a herd — the alternative is a test that starts real engines, which is + * not a test anyone will run. + */ +export async function runEval({ + cases, + engines, + judge = "rules", + threshold = DEFAULT_THRESHOLD, + waitFor, + cwd = process.cwd(), + timeoutMs = 10 * 60 * 1000, + keep = false, + out = () => {}, + judgeRun = runAi, + // Injected so the scoring and aggregation — the parts with the decisions in + // them — can be tested without starting an engine. A test that needs Claude + // installed is a test nobody runs. + run = runEngine, +} = {}) { + const runs = await Promise.all(engines.map((engineKey) => + run(engineKey, cases, { waitFor, cwd, timeoutMs, keep, out }))); + + const engineResults = runs.map((run) => { + if (!run.ok) return { engine: run.engine, ok: false, error: run.error, score: 0, cases: [] }; + const scored = run.results.map((result) => { + const verdict = judge === "rules" + ? scoreByRules(result, result.answer) + : scoreByJudge(result, result.answer, { engine: judge, run: judgeRun, out }); + return { + id: result.id, prompt: result.prompt, taskId: result.taskId, + answer: result.answer, state: result.state ?? null, + score: verdict.score, why: verdict.why, scored: verdict.ok, + ...(result.error ? { error: result.error } : {}), + }; + }); + const total = scored.reduce((sum, c) => sum + c.score, 0); + return { + engine: run.engine, + ok: true, + score: scored.length ? total / scored.length : 0, + passed: scored.filter((c) => c.score >= 1).length, + unscorable: scored.filter((c) => !c.scored).length, + cases: scored, + }; + }); + + const infrastructure = engineResults.filter((e) => !e.ok); + const below = engineResults.filter((e) => e.ok && e.score < threshold); + return { + judge, threshold, + engines: engineResults, + // The three outcomes CI needs to tell apart, decided here rather than at + // the exit-code site, so `--json` and the exit code cannot disagree. + outcome: infrastructure.length ? "infrastructure" : below.length ? "below" : "pass", + below: below.map((e) => e.engine), + broken: infrastructure.map((e) => ({ engine: e.engine, error: e.error })), + }; +} + +/** Resolve `--engines a,b` to canonical keys, naming anything it cannot. */ +export function resolveEngines(list) { + const wanted = String(list || "").split(",").map((s) => s.trim()).filter(Boolean); + const keys = [], unknown = []; + for (const name of wanted) { + const resolved = resolveEngine(name); + if (resolved) keys.push(resolved[0]); + else unknown.push(name); + } + return { keys: [...new Set(keys)], unknown }; +} diff --git a/src/herd-hooks.mjs b/src/herd-hooks.mjs new file mode 100644 index 0000000..e26cf54 --- /dev/null +++ b/src/herd-hooks.mjs @@ -0,0 +1,285 @@ +// Engine lifecycle hooks — the herd's tier-1 state, installed (PRD 0011 R1–R2). +// +// PRD 0009 built the mechanism and never plugged anything into it. `moshcode +// herd report` has existed since the herd did; it beats the screen, it is +// TTL-bounded, and on a default install nothing ever called it. So every +// session was classified by regex against a screen capture, and every engine +// release was a chance for the roster to start lying — a weakness +// src/herd-state.mjs documents about itself in its own header. +// +// This is the other end of that socket. `herd hooks install claude` writes +// Claude Code's own lifecycle hooks so the engine reports its state directly, +// and the roster starts reading `authority: hook`. +// +// THREE RULES, all of them about not being a bad guest in someone's config: +// +// MERGE, NEVER CLOBBER. The file we write is the user's, and it is the file +// their other hooks live in. Install extends it; remove takes out only the +// entries whose command is ours, and leaves empty structure behind only when +// it was already there. +// +// A HOOK MUST NEVER BREAK AN ENGINE. The command is guarded so that outside a +// herd session — no MOSHCODE_HERD_NAME — it does nothing and exits 0, and so +// that a box without moshcode on PATH gets the same silence rather than a +// failing hook on every turn. Degrading to today's screen rules is fine; +// degrading below today is not. +// +// THE SCREEN RULES STAY. A hook that a schema change quietly breaks falls back +// to exactly what the herd did before it, which is why engines.mjs keeps its +// `state` patterns alongside the new `hooks` spec rather than replacing them. +import fs from "node:fs"; +import path from "node:path"; + +import { ENGINES } from "./engines.mjs"; + +/** Engines that ship a hook spec, in table order. */ +export function hookableEngines() { + return Object.entries(ENGINES).filter(([, engine]) => engine.hooks).map(([key]) => key); +} + +/** + * The shell command one hook runs. + * + * Every clause is load-bearing: + * `[ -n "$MOSHCODE_HERD_NAME" ]` — outside the herd this hook is a no-op. The + * engine is used by hand far more often than it is used in a herd. + * `command -v moshcode` — a machine where moshcode was uninstalled must not + * get a failing hook on every turn of an engine that still works. + * `>/dev/null 2>&1` — a status report has nothing to say to the operator; its + * whole output belongs in the roster, not in the middle of a session. + * `; exit 0` — whatever happened above, the engine carries on. + */ +export function hookCommand(state) { + return `[ -n "$MOSHCODE_HERD_NAME" ] && command -v moshcode >/dev/null 2>&1 ` + + `&& moshcode herd report "$MOSHCODE_HERD_NAME" ${state} >/dev/null 2>&1; exit 0`; +} + +/** + * Is this hook entry one of ours? + * + * Matched on the command text rather than on a marker field we invent, because + * the file's schema belongs to the engine: an unknown key is something the + * engine is entitled to reject, and a hook config it rejects is worse than no + * hook at all. The command is a string we wrote, so it is a marker already. + */ +export function isOurs(entry) { + return typeof entry?.command === "string" && /\bmoshcode herd report\b/.test(entry.command); +} + +const HOOK_FILE_MODE = 0o600; + +function readJsonFile(file) { + let text; + try { text = fs.readFileSync(file, "utf8"); } + catch (error) { + if (error.code === "ENOENT") return { ok: true, present: false, data: {} }; + return { ok: false, present: true, error }; + } + if (!text.trim()) return { ok: true, present: true, data: {} }; + try { return { ok: true, present: true, data: JSON.parse(text) }; } + catch (error) { + // Refusing is the whole point. A settings file we cannot parse is one we + // cannot merge into, and overwriting it would take every other hook, MCP + // server and preference in it with us. + return { ok: false, present: true, error: new Error(`${file} is not valid JSON (${error.message}) — fix it and re-run`) }; + } +} + +function writeJsonFile(file, data, { mode = HOOK_FILE_MODE } = {}) { + const body = `${JSON.stringify(data, null, 2)}\n`; + fs.mkdirSync(path.dirname(file), { recursive: true, mode: 0o700 }); + // Write-then-rename: a crash mid-write on the engine's own settings file + // would otherwise leave it truncated, which is the one failure that costs + // more than the feature is worth. + const tmp = `${file}.moshcode-${process.pid}`; + fs.writeFileSync(tmp, body, { mode }); + fs.renameSync(tmp, file); +} + +/** The mode an existing file already has, so an install does not tighten it. */ +function existingMode(file) { + try { return fs.statSync(file).mode & 0o777; } + catch { return HOOK_FILE_MODE; } +} + +/** Where this engine's hooks live, resolved now (specs hold a function). */ +export function hookFile(engine) { + const spec = ENGINES[engine]?.hooks; + if (!spec) return null; + return typeof spec.file === "function" ? spec.file() : spec.file; +} + +// --------------------------------------------------------------------------- +// The claude-settings format +// --------------------------------------------------------------------------- +// +// { "hooks": { "": [ { "hooks": [ { "type": "command", "command": … } ] } ] } } +// +// The outer array is matcher groups. Stop, Notification and UserPromptSubmit +// take no matcher, so ours is a group of one hook with no matcher key — an +// empty `matcher` would be a claim about tool names for events that have none. + +function ensureArray(object, key) { + if (!Array.isArray(object[key])) object[key] = []; + return object[key]; +} + +/** + * Add (or refresh) our entry for one event. Returns what changed, so the caller + * can tell "installed 3" from "already installed" without diffing twice. + */ +function mergeEvent(settings, event, command) { + const hooks = (settings.hooks && typeof settings.hooks === "object" && !Array.isArray(settings.hooks)) + ? settings.hooks + : (settings.hooks = {}); + const groups = ensureArray(hooks, event); + + for (const group of groups) { + const entries = Array.isArray(group?.hooks) ? group.hooks : null; + if (!entries) continue; + const at = entries.findIndex(isOurs); + if (at < 0) continue; + if (entries[at].command === command) return "unchanged"; + // Ours, but not the current text — an upgrade that changed the command, or + // a hand-edit. Replacing beats appending a second copy that fires twice. + entries[at] = { type: "command", command }; + return "updated"; + } + groups.push({ hooks: [{ type: "command", command }] }); + return "added"; +} + +/** Take our entries back out, leaving structure we did not create alone. */ +function pruneEvent(settings, event) { + const hooks = settings.hooks; + if (!hooks || typeof hooks !== "object" || !Array.isArray(hooks[event])) return 0; + let removed = 0; + const groups = []; + for (const group of hooks[event]) { + if (!Array.isArray(group?.hooks)) { groups.push(group); continue; } + const before = group.hooks.length; + const kept = group.hooks.filter((entry) => !isOurs(entry)); + removed += before - kept.length; + // A group that held only our hook goes with it; one that held someone + // else's stays, with theirs intact. + if (!kept.length && before) continue; + groups.push({ ...group, hooks: kept }); + } + if (groups.length) hooks[event] = groups; + else delete hooks[event]; + if (!Object.keys(hooks).length) delete settings.hooks; + return removed; +} + +/** What is installed for one engine right now. */ +export function hooksStatus(engine, { file = hookFile(engine) } = {}) { + const spec = ENGINES[engine]?.hooks; + if (!spec) return { engine, supported: false, file: null, events: [] }; + const read = readJsonFile(file); + if (!read.ok) { + return { engine, supported: true, file, readable: false, error: String(read.error?.message || read.error), events: [] }; + } + const settings = read.data || {}; + const events = spec.events.map(({ event, state, label }) => { + const want = hookCommand(state); + const groups = Array.isArray(settings.hooks?.[event]) ? settings.hooks[event] : []; + const found = groups.flatMap((g) => (Array.isArray(g?.hooks) ? g.hooks : [])).filter(isOurs); + if (!found.length) return { event, label: label || event, state, installed: false }; + // "Installed, but not the command this version writes" is its own answer: + // it is how a spec change after an upgrade shows up, and `install` fixes it. + return { event, label: label || event, state, installed: true, current: found.some((h) => h.command === want) }; + }); + return { + engine, + supported: true, + file, + readable: true, + present: read.present, + installed: events.every((e) => e.installed && e.current), + partial: events.some((e) => e.installed) && !events.every((e) => e.installed && e.current), + events, + }; +} + +/** + * Write this engine's hooks. `dryRun` computes everything and writes nothing, + * returning the file as it would have been so the caller can show a diff. + */ +export function installHooks(engine, { file = hookFile(engine), dryRun = false } = {}) { + const spec = ENGINES[engine]?.hooks; + if (!spec) { + return { ok: false, engine, supported: false, error: new Error(`${engine} ships no hook spec — its sessions stay on the screen rules`) }; + } + const read = readJsonFile(file); + if (!read.ok) return { ok: false, engine, supported: true, file, error: read.error }; + + const before = JSON.stringify(read.data ?? {}, null, 2); + const settings = read.data ?? {}; + const changes = spec.events.map(({ event, state, label }) => ({ event, label: label || event, state, change: mergeEvent(settings, event, hookCommand(state)) })); + const after = JSON.stringify(settings, null, 2); + + if (!dryRun) { + try { writeJsonFile(file, settings, { mode: read.present ? existingMode(file) : HOOK_FILE_MODE }); } + catch (error) { return { ok: false, engine, supported: true, file, error }; } + } + return { + ok: true, engine, supported: true, file, dryRun, + changes, + written: changes.filter((c) => c.change !== "unchanged").length, + before, after, + }; +} + +/** Take them out again. Only ever removes commands this module wrote. */ +export function removeHooks(engine, { file = hookFile(engine), dryRun = false } = {}) { + const spec = ENGINES[engine]?.hooks; + if (!spec) return { ok: false, engine, supported: false, error: new Error(`${engine} ships no hook spec`) }; + const read = readJsonFile(file); + if (!read.ok) return { ok: false, engine, supported: true, file, error: read.error }; + if (!read.present) return { ok: true, engine, supported: true, file, removed: 0, dryRun }; + + const settings = read.data ?? {}; + const before = JSON.stringify(settings, null, 2); + let removed = 0; + // Every event the spec knows about, plus any event that still carries one of + // ours from an older spec — otherwise `remove` after an upgrade would leave + // the hook the previous version installed firing forever. + const events = new Set([ + ...spec.events.map((e) => e.event), + ...Object.keys(settings.hooks && typeof settings.hooks === "object" ? settings.hooks : {}), + ]); + for (const event of events) removed += pruneEvent(settings, event); + const after = JSON.stringify(settings, null, 2); + + if (!dryRun && removed) { + try { writeJsonFile(file, settings, { mode: existingMode(file) }); } + catch (error) { return { ok: false, engine, supported: true, file, error }; } + } + return { ok: true, engine, supported: true, file, removed, dryRun, before, after }; +} + +/** + * A unified diff of the two JSON snapshots an install/remove produced. + * + * Hand-rolled and deliberately dumb — a line is context, an addition, or a + * removal, decided by whether the other side has it at the same place. What + * `--dry-run` needs is "show me what you are about to do to my settings file", + * and for a JSON object printed at two-space indent that is what this gives. + */ +export function hookDiff(before = "", after = "") { + const a = String(before).split("\n"); + const b = String(after).split("\n"); + const out = []; + let i = 0, j = 0; + while (i < a.length || j < b.length) { + if (i < a.length && j < b.length && a[i] === b[j]) { out.push(` ${a[i]}`); i++; j++; continue; } + const laterInB = b.indexOf(a[i] ?? "", j); + const laterInA = a.indexOf(b[j] ?? "", i); + if (i >= a.length || (laterInB >= 0 && (laterInA < 0 || laterInB - j <= laterInA - i))) { + out.push(`+ ${b[j]}`); j++; + } else { + out.push(`- ${a[i]}`); i++; + } + } + return out.join("\n"); +} diff --git a/src/herd-remote.mjs b/src/herd-remote.mjs new file mode 100644 index 0000000..4476216 --- /dev/null +++ b/src/herd-remote.mjs @@ -0,0 +1,365 @@ +// Remote herd members — the roster stops at the edge of the box (PRD 0011 R11–R12). +// +// A deployed agent — a DigitalOcean Gradient ADK deployment answering at +// `agents.do-ai.run///run`, say — could not be on the +// roster, and nothing off the box could drive the herd. That is a strange place +// for the herd to stop, because the ecosystem already converged on the shape we +// need: A2A v0.3.0 gives an agent a card at a well-known URL, tasks with ids and +// status history, and a state vocabulary whose `input-required` is our +// `blocked` under another name. +// +// TWO KINDS, because half the deployed agents in the world do not speak A2A: +// +// "a2a" — discovery at /.well-known/agent-card.json, then JSON-RPC: +// message/send, tasks/get, tasks/cancel. State comes from the task. +// +// "run" — a bare request/response endpoint: POST {"prompt": …}, get an +// answer. The shape every `gradient agent deploy` prints. It has no +// task model and no state, so the herd says so: it is `idle` when it +// answers, `working` while a call is in flight, and honest about +// knowing nothing else. +// +// AUTH IS NEVER WRITTEN DOWN. The token for a remote comes from the environment +// (MOSHCODE_REMOTE__TOKEN) and never touches the manifest, which is +// PRD 0010's allowlist reasoning verbatim: settings sync exists, the manifest is +// on the list of things that can be synced, and a bearer token for someone +// else's agent is exactly the thing that must not ride along to another machine. +import { forgetSession, readManifest, recordRemoteStatus, rememberSession, remoteStatus, validName, clearRemoteStatus } from "./herd.mjs"; + +// The status cache lives in herd.mjs so herd-state.mjs can read it without +// importing this module (and, with it, the network). Re-exported under a name +// that says whose status it is, for callers that already have this module. +export { remoteStatus as remoteStatusOf } from "./herd.mjs"; + +/** How a remote is driven. */ +export const REMOTE_KINDS = ["a2a", "run"]; + +/** A2A's task states, and what the herd calls each one. */ +export const A2A_TO_HERD = { + submitted: "working", + working: "working", + "input-required": "blocked", + "auth-required": "blocked", + completed: "done", + // A2A's three ways of stopping without an answer all leave the agent not + // working and not asking, which is `done` in a vocabulary that has no word + // for "gave up". The A2A state travels alongside in the cache so `--json` + // never has to round-trip through our smaller set to find out what happened. + canceled: "done", + rejected: "done", + failed: "done", + unknown: "unknown", +}; + +/** The herd state for an A2A task state. */ +export const herdStateFor = (a2a) => A2A_TO_HERD[String(a2a || "").toLowerCase()] || "unknown"; + +/** + * The environment variable holding this remote's bearer token. + * + * Named per member rather than one shared secret, because two remotes are + * routinely two different people's infrastructure. + */ +export const tokenEnvVar = (name) => `MOSHCODE_REMOTE_${String(name).toUpperCase().replace(/[^A-Z0-9]/g, "_")}_TOKEN`; + +export function remoteToken(name, env = process.env) { + return env[tokenEnvVar(name)] || ""; +} + +/** + * Only http(s), and only an absolute URL. + * + * `herd prompt` on a remote is a POST of user text to whatever this says, so + * the one thing that must not be possible is a scheme that means something + * other than "a request over the network". + */ +export function parseRemoteUrl(raw) { + let url; + try { url = new URL(String(raw)); } + catch { return { ok: false, error: new Error(`${JSON.stringify(String(raw))} is not a URL`) }; } + if (url.protocol !== "http:" && url.protocol !== "https:") { + return { ok: false, error: new Error(`${url.protocol} is not a transport the herd speaks — use http or https`) }; + } + return { ok: true, url: url.toString() }; +} + +/** Every remote member on the roster. */ +export function listRemotes() { + return Object.entries(readManifest().sessions) + .filter(([, meta]) => meta?.kind === "remote") + .map(([name, meta]) => ({ name, ...meta, status: remoteStatus(name) })); +} + +export function isRemote(name) { + return readManifest().sessions[name]?.kind === "remote"; +} + +export function remoteEntry(name) { + const meta = readManifest().sessions[name]; + return meta?.kind === "remote" ? { name, ...meta } : null; +} + +/** Register a remote. Nothing is contacted here — `ping` is the verb for that. */ +export function addRemote(name, url, { kind = "run", now = Date.now() } = {}) { + if (!validName(name)) return { ok: false, error: new Error(`invalid member name ${JSON.stringify(name)}`) }; + if (!REMOTE_KINDS.includes(kind)) return { ok: false, error: new Error(`unknown kind ${JSON.stringify(kind)} — one of ${REMOTE_KINDS.join(", ")}`) }; + const parsed = parseRemoteUrl(url); + if (!parsed.ok) return parsed; + if (readManifest().sessions[name] && !isRemote(name)) { + return { ok: false, error: new Error(`"${name}" is already a local session — pick another name`) }; + } + rememberSession(name, { + kind: "remote", remoteKind: kind, url: parsed.url, + engine: "remote", cwd: new URL(parsed.url).host, created: now, herd: "main", + }); + return { ok: true, name, url: parsed.url, kind }; +} + +export function removeRemote(name) { + if (!isRemote(name)) return { ok: false, error: new Error(`no remote member named ${JSON.stringify(name)}`) }; + clearRemoteStatus(name); + forgetSession(name); + return { ok: true, name }; +} + +// --------------------------------------------------------------------------- +// Talking to one +// --------------------------------------------------------------------------- + +const DEFAULT_TIMEOUT_MS = 30000; + +function authHeaders(name, env) { + const token = remoteToken(name, env); + return token ? { authorization: `Bearer ${token}` } : {}; +} + +async function request(url, { method = "GET", body, headers = {}, timeoutMs = DEFAULT_TIMEOUT_MS, fetchImpl = fetch } = {}) { + try { + const res = await fetchImpl(url, { + method, + headers: { ...(body ? { "content-type": "application/json" } : {}), ...headers }, + ...(body ? { body: typeof body === "string" ? body : JSON.stringify(body) } : {}), + signal: AbortSignal.timeout(timeoutMs), + }); + const text = await res.text().catch(() => ""); + let json = null; + try { json = text ? JSON.parse(text) : null; } catch { /* not every endpoint answers JSON */ } + return { ok: res.ok, status: res.status, text, json }; + } catch (error) { + return { ok: false, status: 0, error, text: "", json: null }; + } +} + +const trimSlash = (u) => String(u).replace(/\/+$/, ""); + +/** Where an A2A agent publishes its card. */ +export const cardUrl = (url) => `${trimSlash(url)}/.well-known/agent-card.json`; + +/** Fetch and lightly validate an agent card. */ +export async function discoverCard(name, { url = remoteEntry(name)?.url, env = process.env, fetchImpl = fetch } = {}) { + if (!url) return { ok: false, error: new Error(`no remote member named ${JSON.stringify(name)}`) }; + const res = await request(cardUrl(url), { headers: authHeaders(name, env), fetchImpl }); + if (!res.ok || !res.json) { + return { ok: false, status: res.status, error: res.error || new Error(`no agent card at ${cardUrl(url)} (${res.status || "unreachable"})`) }; + } + return { ok: true, card: res.json }; +} + +/** One JSON-RPC call against an A2A endpoint. */ +export async function rpc(name, method, params, { url = remoteEntry(name)?.url, env = process.env, fetchImpl = fetch, timeoutMs } = {}) { + if (!url) return { ok: false, error: new Error(`no remote member named ${JSON.stringify(name)}`) }; + const res = await request(trimSlash(url), { + method: "POST", + headers: authHeaders(name, env), + // The id is per-call and never reused; nothing here multiplexes. + body: { jsonrpc: "2.0", id: `${Date.now()}`, method, params }, + fetchImpl, timeoutMs, + }); + if (!res.json) return { ok: false, status: res.status, error: res.error || new Error(`${method}: ${res.status || "unreachable"}`) }; + if (res.json.error) return { ok: false, status: res.status, error: new Error(`${method}: ${res.json.error.message || "error"} (${res.json.error.code})`), rpcError: res.json.error }; + return { ok: true, result: res.json.result }; +} + +/** The text parts of an A2A message or artifact, joined. */ +export function partsText(container) { + const parts = Array.isArray(container?.parts) ? container.parts : []; + return parts.filter((p) => p?.kind === "text" || typeof p?.text === "string").map((p) => String(p.text ?? "")).join("\n").trim(); +} + +/** Everything the herd wants out of an A2A Task object. */ +export function readA2aTask(task) { + const a2aState = task?.status?.state || "unknown"; + const artifact = [ + ...(Array.isArray(task?.artifacts) ? task.artifacts.map(partsText) : []), + partsText(task?.status?.message), + ].filter(Boolean).join("\n\n"); + return { taskId: task?.id || null, contextId: task?.contextId || null, a2aState, state: herdStateFor(a2aState), artifact }; +} + +/** + * Hand work to a remote member. + * + * Both kinds record what they learned in the status cache, because that cache + * is what `moshcode ps` reads — a remote that has just been prompted should not + * still show whatever it was doing an hour ago. + */ +export async function promptRemote(name, text, { env = process.env, fetchImpl = fetch, timeoutMs, now = Date.now() } = {}) { + const entry = remoteEntry(name); + if (!entry) return { ok: false, error: new Error(`no remote member named ${JSON.stringify(name)}`) }; + recordRemoteStatus(name, { state: "working", at: now, kind: entry.remoteKind, note: "request in flight" }); + + if (entry.remoteKind === "a2a") { + const sent = await rpc(name, "message/send", { + message: { + kind: "message", + role: "user", + messageId: `m-${now}`, + parts: [{ kind: "text", text: String(text) }], + }, + }, { env, fetchImpl, timeoutMs }); + if (!sent.ok) { + recordRemoteStatus(name, { state: "unknown", at: Date.now(), kind: entry.remoteKind, error: String(sent.error?.message || sent.error) }); + return sent; + } + // message/send may answer with a Task or with a Message. A Message means + // the agent answered in one shot and there is nothing to poll. + const result = sent.result; + if (result?.kind === "message" || (!result?.status && result?.parts)) { + const artifact = partsText(result); + recordRemoteStatus(name, { state: "idle", at: Date.now(), kind: entry.remoteKind, artifact, a2aState: "completed" }); + return { ok: true, taskId: null, state: "done", artifact }; + } + const task = readA2aTask(result); + recordRemoteStatus(name, { ...task, at: Date.now(), kind: entry.remoteKind }); + return { ok: true, ...task }; + } + + // "run": one request, one answer, no task model to consult. + const res = await request(trimSlash(entry.url), { + method: "POST", headers: authHeaders(name, env), body: { prompt: String(text) }, fetchImpl, timeoutMs, + }); + if (!res.ok) { + recordRemoteStatus(name, { state: "unknown", at: Date.now(), kind: "run", error: String(res.error?.message || `HTTP ${res.status}`) }); + return { ok: false, status: res.status, error: res.error || new Error(`HTTP ${res.status}`) }; + } + const artifact = runAnswer(res.json, res.text); + recordRemoteStatus(name, { state: "idle", at: Date.now(), kind: "run", artifact }); + return { ok: true, taskId: null, state: "done", artifact }; +} + +/** + * The answer inside a bare `run` response. + * + * No standard says what key it is under, so this checks the ones the ADK and + * its neighbours actually use and falls back to the raw body. Returning the + * whole JSON when nothing matches beats returning "" and calling it an answer. + */ +export function runAnswer(json, text = "") { + if (json && typeof json === "object") { + for (const key of ["output", "response", "result", "answer", "text", "message", "content"]) { + const value = json[key]; + if (typeof value === "string" && value.trim()) return value; + if (value && typeof value === "object") { + const nested = partsText(value) || (typeof value.text === "string" ? value.text : ""); + if (nested.trim()) return nested; + } + } + return JSON.stringify(json, null, 2); + } + return String(text || ""); +} + +/** Refresh what we know about a remote without giving it work. */ +export async function pingRemote(name, { env = process.env, fetchImpl = fetch, timeoutMs = 8000, now = Date.now() } = {}) { + const entry = remoteEntry(name); + if (!entry) return { ok: false, error: new Error(`no remote member named ${JSON.stringify(name)}`) }; + + if (entry.remoteKind === "a2a") { + const cached = remoteStatus(name); + // A live task outranks the card: "what is it doing" is a better answer than + // "it is up", and only tasks/get can give it. + if (cached?.taskId) { + const got = await rpc(name, "tasks/get", { id: cached.taskId }, { env, fetchImpl, timeoutMs }); + if (got.ok) { + const task = readA2aTask(got.result); + recordRemoteStatus(name, { ...task, at: now, kind: "a2a" }); + return { ok: true, ...task }; + } + } + const card = await discoverCard(name, { env, fetchImpl }); + if (!card.ok) { + recordRemoteStatus(name, { state: "unknown", at: now, kind: "a2a", error: String(card.error?.message || card.error) }); + return card; + } + recordRemoteStatus(name, { state: "idle", at: now, kind: "a2a", card: { name: card.card?.name, version: card.card?.version } }); + return { ok: true, state: "idle", card: card.card }; + } + + // A request/response endpoint is `idle` when it is up. There is no third + // thing to know about it and we do not invent one. + const res = await request(trimSlash(entry.url), { method: "GET", headers: authHeaders(name, env), timeoutMs, fetchImpl }); + const reachable = res.ok || (res.status >= 200 && res.status < 500); + recordRemoteStatus(name, { + state: reachable ? "idle" : "unknown", at: now, kind: "run", + ...(reachable ? {} : { error: String(res.error?.message || `HTTP ${res.status}`) }), + }); + return reachable ? { ok: true, state: "idle" } : { ok: false, status: res.status, error: res.error || new Error(`HTTP ${res.status}`) }; +} + +/** What a remote last produced — what `herd read` shows for one. */ +export function readRemote(name) { + const status = remoteStatus(name); + return status?.artifact ? String(status.artifact) : ""; +} + +/** + * Block until a remote reaches one of `states`. + * + * For `run` members this returns as soon as the in-flight call has landed, + * because a request/response endpoint has no state to move through: the answer + * IS the transition. + */ +export async function waitRemote(name, states, { + timeoutMs = 30 * 60 * 1000, intervalMs = 2000, env = process.env, fetchImpl = fetch, + now = () => Date.now(), sleep = (ms) => new Promise((r) => setTimeout(r, ms)), +} = {}) { + const entry = remoteEntry(name); + if (!entry) return { outcome: "gone", state: "gone" }; + const wanted = new Set(states); + const deadline = now() + timeoutMs; + for (;;) { + const status = remoteStatus(name); + if (entry.remoteKind !== "a2a") { + const state = status?.state || "unknown"; + if (wanted.has(state)) return { outcome: "matched", state }; + if (state !== "working") return { outcome: "ended", state }; + } else { + const refreshed = await pingRemote(name, { env, fetchImpl }); + const state = refreshed?.state || status?.state || "unknown"; + if (wanted.has(state)) return { outcome: "matched", state, a2aState: refreshed?.a2aState }; + if (state === "done") return { outcome: "ended", state }; + } + if (now() >= deadline) return { outcome: "timeout", state: remoteStatus(name)?.state || "unknown" }; + await sleep(intervalMs); + } +} + +/** + * Stop whatever a remote is doing. Best effort by design: A2A says an agent may + * refuse to cancel a task it has already finished, and a `run` endpoint has + * nothing to cancel at all — the request either lands or it does not. + */ +export async function cancelRemote(name, { env = process.env, fetchImpl = fetch, now = Date.now() } = {}) { + const entry = remoteEntry(name); + if (!entry) return { ok: false, error: new Error(`no remote member named ${JSON.stringify(name)}`) }; + if (entry.remoteKind !== "a2a") { + return { ok: false, error: new Error(`${name} is a request/response endpoint — there is nothing to cancel`) }; + } + const cached = remoteStatus(name); + if (!cached?.taskId) return { ok: false, error: new Error(`${name} has no task to cancel`) }; + const cancelled = await rpc(name, "tasks/cancel", { id: cached.taskId }, { env, fetchImpl }); + if (!cancelled.ok) return cancelled; + const task = readA2aTask(cancelled.result); + recordRemoteStatus(name, { ...task, at: now, kind: "a2a" }); + return { ok: true, ...task }; +} diff --git a/src/herd-serve.mjs b/src/herd-serve.mjs new file mode 100644 index 0000000..66ecafd --- /dev/null +++ b/src/herd-serve.mjs @@ -0,0 +1,515 @@ +// `moshcode herd serve` — the herd, over A2A v0.3.0 (PRD 0011 R9–R10). +// +// PRD 0009 took herdr's thesis — "the CLI and socket API are one surface agents +// drive" — and implemented it locally. A2A is that same thesis standardised +// across machines, and the mapping is not an integration to design so much as a +// translation table to write down: +// +// herd prompt → message/send blocked → input-required +// state / wait → tasks/get (poll) working → working +// kill → tasks/cancel done → completed +// ps / roster → agent-card discovery killed → canceled +// +// This is not a second API. It is the existing one answering a socket: every +// method here lands on the same herd verbs a person types, and mints the same +// ledger tasks (R5) that `herd tasks` reads back. +// +// SCOPE, deliberately small. v0.3.0, JSON-RPC, text parts. Streaming, push +// notifications and authenticated extended cards are declared *off* in the +// card's capability flags, which is what those flags are for. That is the same +// MVP surface the ADK itself ships, and a spec upgrade is its own PRD. +// +// SECURITY. `message/send` is keystrokes into a real pty, which is strictly +// more dangerous than a browser terminal — a terminal at least shows you what +// it is doing. So this reuses src/console.mjs's discipline wholesale: bind +// loopback by default, verify a moshcode token against app.moshcode.sh once, +// swap it for a short-lived HMAC credential, refuse unauthenticated requests +// before they reach anything, and warn loudly past loopback. There is no +// unauthenticated mode. Loopback included: every process on this box, and +// anything that can talk one of them into making a request, is on the other +// side of "loopback is safe". +import crypto from "node:crypto"; +import http from "node:http"; + +import { loadCreds } from "./auth.mjs"; +import { mintCookie, readCookie, verifyToken } from "./console.mjs"; +import { capture, readManifest, sendKeys, sendPrompt } from "./herd.mjs"; +import { roster } from "./herd-cli.mjs"; +import { endTask, findTask, ledgerSessions, readTasks, screenDelta, startTask, TERMINAL_STATES } from "./herd-tasks.mjs"; +import { moshcodeVersion } from "./ui.mjs"; + +export const A2A_PROTOCOL_VERSION = "0.3.0"; +export const DEFAULT_SERVE_PORT = 7683; + +/** The herd's states, as A2A says them. */ +export const HERD_TO_A2A = { + working: "working", + blocked: "input-required", + done: "completed", + // A2A's vocabulary is smaller than ours, and this is where that costs + // something. `idle` and `unknown` are both "not asking for anything and not + // obviously finished", and the only two candidates are `working` and + // `input-required`. Rounding *up* to input-required would page a human for a + // session that has nothing to say, every time, so they round down and the + // honest state travels in the task's metadata. + idle: "working", + unknown: "working", + gone: "failed", +}; + +export const a2aState = (state) => HERD_TO_A2A[state] || "working"; + +/* --------------------------------------------------------------- JSON-RPC */ + +export const RPC_ERRORS = { + parse: { code: -32700, message: "Invalid JSON payload" }, + invalidRequest: { code: -32600, message: "Invalid JSON-RPC request" }, + methodNotFound: { code: -32601, message: "Method not found" }, + invalidParams: { code: -32602, message: "Invalid parameters" }, + internal: { code: -32603, message: "Internal error" }, + // A2A's own range. + taskNotFound: { code: -32001, message: "Task not found" }, + taskNotCancelable: { code: -32002, message: "Task cannot be canceled" }, +}; + +const rpcOk = (id, result) => ({ jsonrpc: "2.0", id: id ?? null, result }); +const rpcErr = (id, error, data) => ({ jsonrpc: "2.0", id: id ?? null, error: { ...error, ...(data ? { data } : {}) } }); + +/* ------------------------------------------------------------------- cards */ + +/** + * A session is exposed unless it was launched autonomously. + * + * An engine running with its approvals bypassed, plus a network endpoint that + * accepts prompts, is the worst pairing on the menu: prompt injection reaching + * an agent that has already been told not to ask. So `--agent` sessions are off + * the protocol surface unless someone says otherwise out loud. + */ +export function exposable(session, { exposeAutonomous = false } = {}) { + if (session.kind === "remote") return false; // a remote is someone else's to serve + if (!exposeAutonomous && session.agent) return false; + return true; +} + +/** The roster, filtered to what this server will admit exists. */ +export function servedSessions({ exposeAutonomous = false, rows = roster() } = {}) { + const manifest = readManifest().sessions; + return rows + .map((row) => ({ ...row, agent: Boolean(manifest[row.name]?.agent) })) + .filter((row) => exposable(row, { exposeAutonomous })); +} + +const SECURITY = { + securitySchemes: { moshcode: { type: "http", scheme: "bearer", description: "a moshcode login token, or a credential from POST /auth" } }, + security: [{ moshcode: [] }], +}; + +const CAPABILITIES = { + // Every one of these is false because it is false, not because it is + // unfinished — see the scope note at the top. A card that claimed streaming + // would be a client hanging on a stream that never opens. + streaming: false, + pushNotifications: false, + stateTransitionHistory: true, +}; + +/** The card for one session. */ +export function sessionCard(session, { base }) { + const url = `${String(base).replace(/\/+$/, "")}/${session.name}/`; + return { + protocolVersion: A2A_PROTOCOL_VERSION, + name: `${session.name} (${session.engine})`, + description: `A moshcode herd session running ${session.engine}${session.cwd ? ` in ${session.cwd}` : ""}.`, + url, + preferredTransport: "JSONRPC", + version: moshcodeVersion() || "0.0.0", + capabilities: CAPABILITIES, + defaultInputModes: ["text/plain"], + defaultOutputModes: ["text/plain"], + skills: [{ + id: "prompt", + name: "prompt", + description: `Type a prompt into ${session.name} and collect what it produces.`, + tags: ["herd", "terminal", String(session.engine)], + examples: ["port the auth routes", "run the tests and summarise the failures"], + inputModes: ["text/plain"], + outputModes: ["text/plain"], + }], + supportsAuthenticatedExtendedCard: false, + ...SECURITY, + metadata: { + "sh.moshcode.herd": { + session: session.name, engine: session.engine, state: session.state, + authority: session.authority, cwd: session.cwd, + }, + }, + }; +} + +/** + * The card for the herd itself: one skill per member. + * + * Both shapes are published rather than one, because they answer different + * questions. The herd card is discovery — "what is on this box" — and the + * per-session cards are what a client stores when it wants to talk to one + * member for a week. It also makes `herd remote add` of somebody else's single + * session symmetric with adding a whole herd. + */ +export function herdCard(sessions, { base }) { + return { + protocolVersion: A2A_PROTOCOL_VERSION, + name: "moshcode herd", + description: "Agent sessions running on this machine. Each member is addressable at // with its own card.", + url: `${String(base).replace(/\/+$/, "")}/`, + preferredTransport: "JSONRPC", + version: moshcodeVersion() || "0.0.0", + capabilities: CAPABILITIES, + defaultInputModes: ["text/plain"], + defaultOutputModes: ["text/plain"], + skills: sessions.map((s) => ({ + id: s.name, + name: s.name, + description: `${s.engine} — currently ${s.state}${s.cwd ? ` — ${s.cwd}` : ""}. Address it at /${s.name}/.`, + tags: ["herd", String(s.engine), String(s.state)], + })), + supportsAuthenticatedExtendedCard: false, + ...SECURITY, + }; +} + +/* ------------------------------------------------------------------ tasks */ + +const iso = (ts) => new Date(Number(ts) || Date.now()).toISOString(); + +const textMessage = (text, { role = "agent", taskId, contextId } = {}) => ({ + kind: "message", + role, + messageId: crypto.randomUUID(), + parts: [{ kind: "text", text: String(text ?? "") }], + ...(taskId ? { taskId } : {}), + ...(contextId ? { contextId } : {}), +}); + +/** + * A ledger task as an A2A Task. + * + * `live` is the session's state right now, which outranks the ledger's last + * transition for an open task: the ledger is written by whatever last polled, + * and a client asking tasks/get IS a poll. + */ +export function taskToA2a(task, { live = null } = {}) { + const herdState = task.status === "closed" ? (task.state || "done") : (live || task.state || "working"); + // A FINISHED task is `completed`, whatever the session went back to being. + // The idle→working rounding above is about a *session* — "it is sitting + // there, it is not asking for anything" — and applying it to a task that has + // an outcome and an artifact would leave an A2A client polling a job that + // finished ten minutes ago. The one exception is a task that ended by + // stopping to ask, which is `input-required` in any vocabulary. + const state = task.status === "closed" + ? (herdState === "blocked" ? "input-required" : "completed") + : a2aState(herdState); + return { + kind: "task", + id: task.id, + contextId: task.session, + status: { + state, + timestamp: iso(task.endedAt || task.transitions.at(-1)?.ts || task.submitted), + ...(task.artifact ? { message: textMessage(task.artifact, { taskId: task.id, contextId: task.session }) } : {}), + }, + history: [textMessage(task.text, { role: "user", taskId: task.id, contextId: task.session })], + artifacts: task.artifact + ? [{ + artifactId: `${task.id}-output`, + name: "screen", + description: "What appeared on the session's screen after the prompt was submitted.", + parts: [{ kind: "text", text: task.artifact }], + }] + : [], + metadata: { + // Where the vocabulary mismatch goes to stay honest. + "sh.moshcode.herd": { + session: task.session, + state: herdState, + status: task.status, + submitted: task.submitted, + endedAt: task.endedAt, + durationMs: task.durationMs, + truncated: Boolean(task.truncated), + transitions: task.transitions, + }, + }, + }; +} + +/* ------------------------------------------------------------------- auth */ + +/** + * Who is allowed in. + * + * Two accepted credentials, in this order: an HMAC credential this process + * minted (cheap, local, expires), or a moshcode login token (verified against + * the app, then cached for the same window so a polling client does not become + * a load test on app.moshcode.sh). + */ +export function createAuth({ + api = "https://app.moshcode.sh", + secret = crypto.randomBytes(32).toString("hex"), + verify = verifyToken, + ttlMs = 12 * 60 * 60 * 1000, +} = {}) { + const verified = new Map(); // sha256(token) → { user, until } + + const hash = (token) => crypto.createHash("sha256").update(String(token)).digest("hex"); + + return { + secret, + mint: (user) => mintCookie(secret, { user, ttlMs }), + async check(token, { now = Date.now() } = {}) { + if (!token) return null; + const local = readCookie(secret, token, now); + if (local) return local; + const key = hash(token); + const cached = verified.get(key); + if (cached && cached.until > now) return cached.user; + const user = await verify(api, token); + if (!user) { verified.delete(key); return null; } + verified.set(key, { user, until: now + Math.min(ttlMs, 15 * 60 * 1000) }); + return user; + }, + }; +} + +/** The bearer token on a request, if there is one. */ +export function bearer(req) { + const header = req?.headers?.authorization || ""; + const match = /^Bearer\s+(.+)$/i.exec(String(header).trim()); + return match ? match[1].trim() : ""; +} + +/* ----------------------------------------------------------------- server */ + +const send = (res, status, body) => { + const text = JSON.stringify(body, null, 2); + res.writeHead(status, { "content-type": "application/json", "cache-control": "no-store" }); + res.end(text); +}; + +function readBody(req, { limit = 1024 * 1024 } = {}) { + return new Promise((resolve) => { + let size = 0; + const chunks = []; + req.on("data", (chunk) => { + size += chunk.length; + // A prompt is text. Anything past a megabyte is not a prompt, and reading + // it into memory to find that out is the whole attack. + if (size > limit) { resolve({ tooLarge: true, text: "" }); req.destroy(); return; } + chunks.push(chunk); + }); + req.on("end", () => resolve({ tooLarge: false, text: Buffer.concat(chunks).toString("utf8") })); + req.on("error", () => resolve({ tooLarge: false, text: "" })); + }); +} + +/** + * The herd's A2A server. + * + * Routing, in full: + * GET /.well-known/agent-card.json the herd + * GET //.well-known/agent-card.json one member + * POST /auth token → short-lived credential + * POST // message/send, tasks/get, tasks/cancel + * POST / tasks/get, tasks/cancel (ids are herd-wide) + */ +export function createHerdServer({ + api = "https://app.moshcode.sh", + auth = createAuth({ api }), + exposeAutonomous = false, + base = `http://127.0.0.1:${DEFAULT_SERVE_PORT}`, + sessions = () => servedSessions({ exposeAutonomous }), + prompt = defaultPrompt, + interrupt = defaultInterrupt, + screen = capture, + now = () => Date.now(), +} = {}) { + const server = http.createServer(async (req, res) => { + const url = new URL(req.url || "/", "http://localhost"); + const segments = url.pathname.split("/").filter(Boolean); + + // Auth first, before routing — a 404 that only unauthenticated callers can + // see is a way to ask which session names exist. + const user = await auth.check(bearer(req)); + if (!user) { + res.writeHead(401, { "content-type": "application/json", "www-authenticate": 'Bearer realm="moshcode herd"' }); + res.end(JSON.stringify({ error: "not authenticated", how: "Authorization: Bearer — run `moshcode login` on the calling machine" }, null, 2)); + return; + } + + if (req.method === "POST" && segments.length === 1 && segments[0] === "auth") { + // The token that got here is already verified; this hands back something + // shorter-lived to use instead, so the real token stops travelling. + return send(res, 200, { credential: auth.mint(user), expiresIn: 12 * 60 * 60 }); + } + + const cardAt = segments.indexOf(".well-known"); + if (req.method === "GET" && cardAt >= 0 && segments[cardAt + 1] === "agent-card.json") { + const rows = sessions(); + if (cardAt === 0) return send(res, 200, herdCard(rows, { base })); + const found = rows.find((s) => s.name === segments[0]); + if (!found) return send(res, 404, { error: `no member named ${JSON.stringify(segments[0])}` }); + return send(res, 200, sessionCard(found, { base })); + } + + if (req.method !== "POST") { + return send(res, 405, { error: "the A2A surface is POST for JSON-RPC and GET for agent cards" }); + } + + const body = await readBody(req); + if (body.tooLarge) return send(res, 413, rpcErr(null, RPC_ERRORS.invalidParams, "payload too large")); + let payload; + try { payload = JSON.parse(body.text); } + catch { return send(res, 400, rpcErr(null, RPC_ERRORS.parse)); } + if (!payload || payload.jsonrpc !== "2.0" || typeof payload.method !== "string") { + return send(res, 400, rpcErr(payload?.id, RPC_ERRORS.invalidRequest)); + } + + const member = segments.length && segments[0] !== "auth" ? segments[0] : null; + const answer = await handleRpc(payload, { + member, sessions, prompt, interrupt, screen, now, + }); + // 200 even for an error: in JSON-RPC the transport succeeded and the error + // is the payload. A 4xx here would have clients retrying a method name. + return send(res, 200, answer); + }); + + server.on("clientError", (_error, socket) => { + try { socket.end("HTTP/1.1 400 Bad Request\r\nConnection: close\r\n\r\n"); } catch { /* already gone */ } + }); + + return server; +} + +/** + * Type a prompt into a live session. + * + * The one place the protocol becomes keystrokes. Everything above this is + * routing and everything below it is the engine's business. Injectable so the + * tests exercise the whole surface without a pty anywhere near them. + */ +function defaultPrompt(name, text) { + return sendPrompt(name, text); +} + +/** + * Interrupt whatever a session is doing: Escape, then Ctrl-C. + * + * The same escalation `kill` uses, stopping one rung short of it on purpose. An + * A2A task is a unit of work inside a member, and a member is a long-lived + * thing somebody attached to five minutes ago — cancelling their task must not + * take their session with it. Ending a member is `moshcode kill`, which is a + * decision, not a protocol call. + */ +function defaultInterrupt(name) { + const first = sendKeys(name, ["Escape"]); + const second = sendKeys(name, ["C-c"]); + return { ok: Boolean(first.ok || second.ok) }; +} + +/** The JSON-RPC methods, with no HTTP anywhere near them. */ +export async function handleRpc(payload, { member, sessions, prompt, interrupt, screen, now = () => Date.now() }) { + const { id, method, params } = payload; + const rows = sessions(); + + if (method === "message/send") { + if (!member) return rpcErr(id, RPC_ERRORS.invalidParams, "address a member: POST //"); + const session = rows.find((s) => s.name === member); + if (!session) return rpcErr(id, RPC_ERRORS.invalidParams, `no member named ${JSON.stringify(member)}`); + if (!session.alive || session.exited) return rpcErr(id, RPC_ERRORS.invalidParams, `${member} is not running`); + const text = messageText(params?.message); + if (!text) return rpcErr(id, RPC_ERRORS.invalidParams, "the message needs a text part"); + + const at = now(); + const baseline = screen(member, { lines: 60 }); + const taskId = startTask(member, text, { screen: baseline, now: at, state: session.state }); + const sent = prompt(member, text); + if (!sent?.ok) { + endTask(member, taskId, { state: "done", artifact: `moshcode could not type into ${member}: ${sent?.error?.message || "unknown error"}`, ts: now() }); + return rpcErr(id, RPC_ERRORS.internal, String(sent?.error?.message || "could not reach the session")); + } + // Returned before the engine has answered, on purpose: A2A's task model + // exists so a client polls rather than holding a socket open for the half + // hour an agent might take. + const task = readTasks(member).find((t) => t.id === taskId); + return rpcOk(id, taskToA2a(task || { + id: taskId, session: member, text, submitted: at, transitions: [], status: "open", state: "working", artifact: null, + }, { live: "working" })); + } + + if (method === "tasks/get") { + const found = locateTask(params?.id, { member, rows }); + if (!found) return rpcErr(id, RPC_ERRORS.taskNotFound); + // A poll IS an observation, so it closes a task whose session has stopped. + // Without this the only thing that ever finishes a task is the watcher, and + // an A2A client — whose entire protocol is send-then-poll — would sit on + // `working` forever against a herd where nobody happened to run one. + const task = reconcileTask(found.task, found.session, { screen, now }); + return rpcOk(id, taskToA2a(task, { live: found.session?.state || null })); + } + + if (method === "tasks/cancel") { + const found = locateTask(params?.id, { member, rows }); + if (!found) return rpcErr(id, RPC_ERRORS.taskNotFound); + if (found.task.status === "closed") return rpcErr(id, RPC_ERRORS.taskNotCancelable, "that task has already finished"); + const stopped = interrupt(found.task.session); + const artifact = screenDelta(found.task.baseline, screen(found.task.session, { lines: 200 })); + endTask(found.task.session, found.task.id, { state: "done", artifact, ts: now() }); + const task = { ...found.task, status: "closed", state: "done", artifact, endedAt: now() }; + const cancelled = taskToA2a(task); + cancelled.status.state = "canceled"; + cancelled.metadata["sh.moshcode.herd"].interrupted = Boolean(stopped?.ok); + return rpcOk(id, cancelled); + } + + return rpcErr(id, RPC_ERRORS.methodNotFound, method); +} + +/** + * Close an open task whose session has already stopped, and hand back what the + * task now is. Leaves a task alone while its session is still working. + */ +function reconcileTask(task, session, { screen, now }) { + if (task.status === "closed" || !session) return task; + if (!TERMINAL_STATES.includes(session.state)) return task; + const artifact = screenDelta(task.baseline, screen(task.session, { lines: 400 })); + const at = now(); + endTask(task.session, task.id, { state: session.state, artifact, ts: at }); + return { ...task, status: "closed", state: session.state, artifact, endedAt: at, durationMs: task.submitted ? at - task.submitted : null }; +} + +/** Ids are herd-wide, so a task can be found with or without its member. */ +function locateTask(taskId, { member, rows }) { + if (!taskId) return null; + const search = member ? [member] : ledgerSessions(); + const task = findTask(String(taskId), { sessions: search }); + if (!task) return null; + // A task in a member this server does not expose does not exist here either. + const session = rows.find((s) => s.name === task.session); + if (!session) return null; + return { task, session }; +} + +/** The text of an A2A message. Text parts only — see the scope note. */ +export function messageText(message) { + const parts = Array.isArray(message?.parts) ? message.parts : []; + return parts + .filter((p) => p?.kind === "text" || typeof p?.text === "string") + .map((p) => String(p.text ?? "")) + .join("\n") + .trim(); +} + +/** The credentials `herd serve` needs to verify anyone at all. */ +export function serveCredentials() { + const creds = loadCreds(); + return { api: creds?.api || "https://app.moshcode.sh", token: creds?.token || "" }; +} diff --git a/src/herd-state.mjs b/src/herd-state.mjs index 5d060f5..d5c316d 100644 --- a/src/herd-state.mjs +++ b/src/herd-state.mjs @@ -20,11 +20,39 @@ import fs from "node:fs"; import path from "node:path"; import { ENGINES } from "./engines.mjs"; -import { capture, herdDir, sessionExited } from "./herd.mjs"; +import { capture, herdDir, remoteStatus, sessionExited } from "./herd.mjs"; +import { TOOLS } from "./tools.mjs"; /** The vocabulary the roster, notifications, and `wait` all share. */ export const STATES = ["working", "blocked", "done", "idle", "unknown"]; +/** + * What a blocked session is blocked *on* (PRD 0011 R4). + * + * The roster still prints `blocked`, because five kinds of amber is four more + * than anyone reads at a glance. The sub-kind rides in `--json` and in + * notifications, where it is worth something: an `--ask` reply to a numbered + * menu wants a digit, and one to a question wants a sentence, and answering a + * menu with a paragraph types the paragraph into the menu. + */ +export const BLOCKED_KINDS = ["permission", "question", "menu"]; + +/** + * Parse the state token a hook or a human passes to `herd report`. + * + * `blocked:permission` is one string on a command line and two facts here. + * Returns null for anything not in the vocabulary — an unknown state has to + * fail loudly at the edge rather than be written into the status file where + * every later reader has to cope with it. + */ +export function parseState(raw) { + const [state, kind] = String(raw ?? "").trim().split(":"); + if (!STATES.includes(state)) return null; + if (kind === undefined || kind === "") return { state }; + if (state !== "blocked" || !BLOCKED_KINDS.includes(kind)) return null; + return { state, kind }; +} + /** * `gone` is deliberately not in STATES: it is not a state an agent is in, it is * the absence of one. It exists so the roster can show what a reboot took and @@ -101,6 +129,43 @@ export const COMMON_RULES = { ], }; +/** + * Which *kind* of blocked a screen is showing. + * + * Only consulted once a screen has already classified as `blocked`, so these + * are labels rather than detectors and can afford to be loose. A screen that + * matches nothing here is blocked with no sub-kind, which is exactly what the + * roster printed before this existed. + */ +export const BLOCKED_KIND_RULES = { + // The menu test goes first: Claude Code's permission dialog IS a numbered + // menu, and "which keystroke answers this" is the question the sub-kind is + // for. A y/n is a menu of two with no digits, so it stays a permission. + menu: [/^\s*[❯›▸>]\s*\d+\.\s+\S/m], + permission: [ + /\[y\/n\]/i, + /\((?:y(?:es)?\/n(?:o)?)\)\s*[:?]?\s*$/im, + /\((?:Y\)es|N\)o)/, + /\bdo you want to\b/i, + /\bpermission (?:request|required)\b/i, + /\ballow (?:this )?(?:command|tool|execution)\b/i, + /\bapprove this (?:command|edit|change)\b/i, + /\bwaiting for (?:your )?(?:approval|confirmation)\b/i, + ], + question: [/\?\s*$/m, /\bpress (?:enter|return) to continue\b/i], +}; + +/** The sub-kind of an already-blocked screen, or null when it does not say. */ +export function blockedKind(screen) { + const text = stripAnsi(screen); + const lines = text.split("\n"); + const tail = lines.slice(Math.max(0, lines.length - 25)).join("\n"); + for (const kind of ["menu", "permission", "question"]) { + if ((BLOCKED_KIND_RULES[kind] || []).some((re) => re.test(tail))) return kind; + } + return null; +} + /** * User overrides, so a rule that rots can be fixed on the box it rots on. * @@ -130,9 +195,76 @@ export function loadUserRules(file = path.join(herdDir(), "rules.json")) { return out; } -/** The rule set for one engine: user overrides, then its own, then the shared. */ +/** The states a user rules file is allowed to carry patterns for. */ +const RULE_STATES = ["blocked", "working", "idle"]; + +/** + * Everything wrong with the user's rules file, said out loud (PRD 0011 R3). + * + * loadUserRules() is silent by design — a malformed rules file must not take + * down the roster, and it does not. The cost of that is a file which has been + * quietly ignored since the day someone typo'd a bracket in it, with the herd + * classifying from the built-in rules and nothing anywhere saying so. This is + * where that gets to be loud, and `herd doctor` is the one caller. + */ +export function inspectUserRules(file = path.join(herdDir(), "rules.json")) { + const empty = { file, present: false, ok: true, patterns: 0, problems: [] }; + let text; + try { text = fs.readFileSync(file, "utf8"); } + catch (error) { + return error.code === "ENOENT" ? empty + : { ...empty, present: true, ok: false, problems: [{ where: file, error: String(error.message || error) }] }; + } + + let raw; + try { raw = JSON.parse(text); } + catch (error) { + return { ...empty, present: true, ok: false, problems: [{ where: file, error: `not valid JSON — ${error.message}` }] }; + } + if (!raw || typeof raw !== "object" || Array.isArray(raw)) { + return { ...empty, present: true, ok: false, problems: [{ where: file, error: 'the top level must be { "": { "blocked": ["…"] } }' }] }; + } + + const problems = []; + let patterns = 0; + for (const [engine, group] of Object.entries(raw)) { + if (!group || typeof group !== "object" || Array.isArray(group)) { + problems.push({ where: engine, error: "must be an object of state → patterns" }); + continue; + } + for (const key of Object.keys(group)) { + if (!RULE_STATES.includes(key)) { + problems.push({ where: `${engine}.${key}`, error: `not a state the classifier reads (${RULE_STATES.join(", ")})` }); + } + } + for (const state of RULE_STATES) { + if (group[state] === undefined) continue; + if (!Array.isArray(group[state])) { + problems.push({ where: `${engine}.${state}`, error: "must be an array of pattern strings" }); + continue; + } + for (const pattern of group[state]) { + try { new RegExp(pattern, "im"); patterns++; } + catch (error) { problems.push({ where: `${engine}.${state}`, pattern: String(pattern), error: String(error.message || error) }); } + } + } + } + return { file, present: true, ok: problems.length === 0, patterns, problems }; +} + +/** + * The rule set for one engine: user overrides, then its own, then the shared. + * + * TOOLS is consulted as well as ENGINES because `herd run -- gradient agent run + * --dev` names its session after the binary, and the workflow CLIs are exactly + * the long-running processes people put in the herd next to an agent (PRD 0011 + * R15). A tool's rules live in src/tools.mjs beside its install spec for the + * same reason an engine's live beside its own. + */ export function rulesFor(engine, { userRules = loadUserRules() } = {}) { - const own = ENGINES[engine]?.state || {}; + const own = (Object.hasOwn(ENGINES, engine) ? ENGINES[engine]?.state : null) + || (Object.hasOwn(TOOLS, engine) ? TOOLS[engine]?.state : null) + || {}; const user = userRules[engine] || {}; const common = userRules.common || {}; const merge = (state) => [ @@ -179,13 +311,18 @@ export function classify(screen, rules) { * someone noticed by hand. */ export function reportState(name, state, { ttl = HOOK_TTL_MS, now = Date.now() } = {}) { - if (!STATES.includes(state)) return { ok: false, error: new Error(`unknown state ${JSON.stringify(state)} — one of ${STATES.join(", ")}`) }; + const parsed = parseState(state); + if (!parsed) { + return { ok: false, error: new Error(`unknown state ${JSON.stringify(state)} — one of ${STATES.join(", ")}${` (blocked takes :${BLOCKED_KINDS.join(", :")})`}`) }; + } try { fs.mkdirSync(statusDir(), { recursive: true, mode: 0o700 }); const file = statusFile(name); - fs.writeFileSync(file, JSON.stringify({ state, at: now, ttl: Math.min(Number(ttl) || HOOK_TTL_MS, HOOK_TTL_MS) }), { mode: 0o600 }); + const record = { state: parsed.state, at: now, ttl: Math.min(Number(ttl) || HOOK_TTL_MS, HOOK_TTL_MS) }; + if (parsed.kind) record.kind = parsed.kind; + fs.writeFileSync(file, JSON.stringify(record), { mode: 0o600 }); fs.chmodSync(file, 0o600); - return { ok: true, state }; + return { ok: true, ...parsed }; } catch (error) { return { ok: false, error }; } @@ -199,7 +336,9 @@ export function hookReport(name, { now = Date.now() } = {}) { if (!raw || !STATES.includes(raw.state)) return null; const ttl = Math.min(Number(raw.ttl) || HOOK_TTL_MS, HOOK_TTL_MS); if (!Number.isFinite(raw.at) || now - raw.at > ttl) return null; - return { state: raw.state, at: raw.at }; + const report = { state: raw.state, at: raw.at }; + if (raw.state === "blocked" && BLOCKED_KINDS.includes(raw.kind)) report.kind = raw.kind; + return report; } export function clearReport(name) { @@ -218,10 +357,20 @@ export function clearReport(name) { * first useful question is "was anything even reading the screen?", and a * roster that cannot answer it sends people to read this file instead. */ -export function sessionState(session, { now = Date.now(), userRules = loadUserRules(), read = capture } = {}) { +export function sessionState(session, { now = Date.now(), userRules = loadUserRules(), read = capture, remote = remoteStatus } = {}) { const name = typeof session === "string" ? session : session.name; const meta = typeof session === "string" ? {} : session; + // A remote member's state is the remote's claim and nothing more (PRD 0011 + // R11). It is reported with `authority: "remote"` so nobody mistakes a URL + // that answered five minutes ago for something this box just verified. + if (meta.kind === "remote") { + const claim = remote(name, { now }); + return claim?.state + ? { state: claim.state, authority: "remote" } + : { state: "unknown", authority: "remote" }; + } + if (meta.alive === false) return { state: "gone", authority: "runtime" }; // A finished process is done, and no screen rule gets a vote on that. This is @@ -231,11 +380,19 @@ export function sessionState(session, { now = Date.now(), userRules = loadUserRu if (exited === null && meta.alive === undefined) return { state: "gone", authority: "runtime" }; const hook = hookReport(name, { now }); - if (hook) return { state: hook.state, authority: "hook" }; + if (hook) return hook.kind ? { state: hook.state, authority: "hook", blockedOn: hook.kind } : { state: hook.state, authority: "hook" }; const screen = read(name); if (!screen) return { state: "unknown", authority: "screen" }; - return { state: classify(screen, rulesFor(meta.engine, { userRules })), authority: "screen" }; + const state = classify(screen, rulesFor(meta.engine, { userRules })); + // `blockedOn` is only ever added when there is one to add: this object is + // spread over every roster row, so an always-present `blockedOn: undefined` + // would be a new key on every row for the benefit of none. It is also not + // called `kind` — that name already belongs to the row, where it says whether + // the member is a local pty or a URL. + if (state !== "blocked") return { state, authority: "screen" }; + const kind = blockedKind(screen); + return kind ? { state, authority: "screen", blockedOn: kind } : { state, authority: "screen" }; } /** listSessions() output, each row carrying its state. */ diff --git a/src/herd-tasks.mjs b/src/herd-tasks.mjs new file mode 100644 index 0000000..4e796e5 --- /dev/null +++ b/src/herd-tasks.mjs @@ -0,0 +1,377 @@ +// The task ledger — what happened, not just what is happening (PRD 0011 R5–R7). +// +// `moshcode ps` answers "now". It is the whole reason the roster exists and it +// is genuinely all most people need at 11pm. It is also everything the herd +// remembered: `herd prompt api "…" --wait` returned, and then the evidence +// evaporated. Which prompts were submitted, when each one blocked, what came +// back, how long the human took to answer — none of it was anywhere, which made +// the herd's party trick (fan four engines out overnight) unauditable by +// construction. +// +// So every prompt mints a TASK: an id, the text that was submitted, its state +// transitions with timestamps, and the output it produced. The watch loop +// already observed every one of those transitions and threw each away after +// deciding whether to buzz a phone; this is the write inserted at that same +// decision, not a second poller. +// +// WHAT THIS IS NOT. It is not a trace of the engine. We do not own those +// runtimes, and pretending to see inside one would be paint-reading with extra +// steps. What the herd can attest to honestly is: this text went in at this +// time, the session moved through these states, and this is what was on the +// screen that had not been there before. That is what is recorded. +// +// JSONL, one file per session, 0600. The manifest's reason for 0600 applies one +// step harder here: the manifest records the argv an engine was launched with, +// and this records what the engine *said*, which regularly contains secrets the +// user never typed. +import fs from "node:fs"; +import path from "node:path"; + +import { herdDir } from "./herd.mjs"; + +/** Terminal states for a task: the engine stopped needing the CPU. */ +export const TERMINAL_STATES = ["blocked", "done", "idle"]; + +/** + * Retention. An append-only file with no cap is a disk-eater with a delay on + * it, and the delay is however long the operator finds this feature useful. + */ +export const MAX_TASKS_PER_SESSION = 500; +export const MAX_LEDGER_BYTES = 2 * 1024 * 1024; + +/** + * How much of an artifact goes inline. + * + * The tail, not the head: an agent's answer is the last thing it printed, and + * the first 8KB of a long run is the part you already watched. Truncation is + * recorded rather than hidden, because an artifact that silently lost its + * middle is worse than one that says it did. + */ +export const MAX_ARTIFACT_CHARS = 8000; + +const tasksDir = () => path.join(herdDir(), "tasks"); +const ledgerFile = (session) => path.join(tasksDir(), `${session}.jsonl`); +const seqFile = () => path.join(tasksDir(), "seq"); + +function ensureDir() { + fs.mkdirSync(tasksDir(), { recursive: true, mode: 0o700 }); +} + +/** + * The next task id, herd-wide. + * + * Herd-wide rather than per-session so that `herd task t-07` means one task and + * not one per member — the id is a handle people paste, and an ambiguous handle + * is not one. The counter is a file with a lock beside it; if the lock cannot + * be taken (a genuinely concurrent fan-out, or a stale lock), the id gets a + * random suffix instead of blocking. A collision is a cosmetic problem and a + * hang is not. + */ +export function mintTaskId({ now = Date.now() } = {}) { + ensureDir(); + const lock = `${seqFile()}.lock`; + let held = false; + for (let attempt = 0; attempt < 50 && !held; attempt++) { + try { fs.closeSync(fs.openSync(lock, "wx")); held = true; } + catch { + // A lock older than a few seconds belonged to a process that died. + try { + if (now - fs.statSync(lock).mtimeMs > 5000) fs.rmSync(lock, { force: true }); + } catch { /* it went away on its own */ } + } + } + try { + let next = 1; + try { next = Math.max(1, Number(JSON.parse(fs.readFileSync(seqFile(), "utf8")).next) || 1); } + catch { /* first task on this box */ } + const id = held ? `t-${String(next).padStart(2, "0")}` : `t-${String(next).padStart(2, "0")}-${Math.random().toString(36).slice(2, 6)}`; + if (held) { + try { fs.writeFileSync(seqFile(), JSON.stringify({ next: next + 1 }), { mode: 0o600 }); } + catch { /* the id is still ours; the next one may repeat it */ } + } + return id; + } finally { + if (held) { try { fs.rmSync(lock, { force: true }); } catch { /* best effort */ } } + } +} + +/** Append one event. Never throws — a lost ledger line must not fail a prompt. */ +function append(session, event) { + try { + ensureDir(); + const file = ledgerFile(session); + fs.appendFileSync(file, `${JSON.stringify(event)}\n`, { mode: 0o600 }); + fs.chmodSync(file, 0o600); + compact(session); + return true; + } catch { return false; } +} + +function readLines(session) { + let text; + try { text = fs.readFileSync(ledgerFile(session), "utf8"); } + catch { return []; } + const out = []; + for (const line of text.split("\n")) { + if (!line.trim()) continue; + // One unparseable line loses that line, not the ledger. A truncated last + // write is the ordinary way this happens and it must not hide the history + // above it. + try { out.push(JSON.parse(line)); } catch { /* skip */ } + } + return out; +} + +/** Trim to the retention cap, keeping the newest tasks whole. */ +export function compact(session, { maxTasks = MAX_TASKS_PER_SESSION, maxBytes = MAX_LEDGER_BYTES } = {}) { + const file = ledgerFile(session); + let size = 0; + try { size = fs.statSync(file).size; } catch { return false; } + // This runs on every append, so the common case has to cost one stat. A task + // cannot be smaller than its own submit line, so a ledger under the byte cap + // and under `maxTasks` submit-lines' worth of bytes cannot be over either. + if (size <= maxBytes && size < maxTasks * 120) return false; + const lines = readLines(session); + const ids = []; + for (const line of lines) if (line.id && !ids.includes(line.id)) ids.push(line.id); + const overTasks = ids.length > maxTasks; + if (!overTasks && size <= maxBytes) return false; + + // Keep whole tasks, newest first, until the budget is spent. A ledger cut + // mid-task would show a submission with no outcome, which reads as an agent + // that never answered rather than as a file that was trimmed. + const keep = new Set(ids.slice(-maxTasks)); + let kept = lines.filter((line) => !line.id || keep.has(line.id)); + while (kept.length && Buffer.byteLength(kept.map((l) => JSON.stringify(l)).join("\n")) > maxBytes) { + const oldest = kept.find((l) => l.id)?.id; + if (!oldest) break; + keep.delete(oldest); + kept = kept.filter((line) => !line.id || keep.has(line.id)); + } + try { + fs.writeFileSync(file, kept.length ? `${kept.map((l) => JSON.stringify(l)).join("\n")}\n` : "", { mode: 0o600 }); + return true; + } catch { return false; } +} + +// --------------------------------------------------------------------------- +// Writing +// --------------------------------------------------------------------------- + +/** + * A prompt was submitted. Returns the task id, which the caller carries so + * later transitions can be attributed to it. + * + * `screen` is the session's screen at submission time, kept as the baseline the + * artifact is a delta against. Storing the whole thing would double every + * ledger for the sake of text the operator has already seen. + */ +export function startTask(session, text, { screen = "", now = Date.now(), state = null, id = mintTaskId({ now }) } = {}) { + append(session, { e: "submit", id, ts: now, text: String(text), state, baseline: baselineOf(screen) }); + return id; +} + +/** + * A state change worth remembering, attributed to a task when one is open. + * + * A *change*: repeating the state already at the end of the ledger is dropped. + * Several things poll the same session at once — a `--wait` prompt runs two + * waits back to back, and the watcher is looking at all of them anyway — and + * each keeps its own idea of what it last saw. Without this, one prompt writes + * `idle` three times and `herd task` prints a transition list where two of the + * three rows lasted zero seconds. + */ +export function recordTransition(session, state, { id = null, ts = Date.now(), kind = null } = {}) { + const previous = lastRecordedState(session); + if (previous && previous.state === state && previous.id === id) return false; + const event = { e: "state", id, ts, state }; + if (kind) event.kind = kind; + append(session, event); + return true; +} + +/** The state at the end of the ledger, whatever wrote it. */ +function lastRecordedState(session) { + const lines = readLines(session); + for (let i = lines.length - 1; i >= 0; i--) { + if (lines[i].e === "state" || lines[i].e === "end") return { state: lines[i].state, id: lines[i].id ?? null }; + } + return null; +} + +/** The task is over. `artifact` is what the session produced while it ran. */ +export function endTask(session, id, { state = "done", artifact = "", ts = Date.now() } = {}) { + const text = String(artifact ?? ""); + const truncated = text.length > MAX_ARTIFACT_CHARS; + append(session, { + e: "end", id, ts, state, + artifact: truncated ? text.slice(-MAX_ARTIFACT_CHARS) : text, + ...(truncated ? { truncated: true, artifactChars: text.length } : {}), + }); + return true; +} + +/** + * The last few lines of a screen, which is all a delta needs to anchor on. + * + * A whole capture as the baseline would make the ledger as big as the + * transcript. The bottom of the screen is where the new output starts, so that + * is what has to be remembered to find it again. + */ +function baselineOf(screen) { + const lines = String(screen ?? "").replace(/\s+$/, "").split("\n"); + return lines.slice(Math.max(0, lines.length - 8)).join("\n"); +} + +/** + * What appeared on screen after the baseline — the task's output. + * + * The engine redraws its whole screen constantly, so "everything after the last + * line I saw" is the only definition of new output available to something + * reading a terminal from outside. When the baseline cannot be found (a + * full-screen repaint scrolled it away, or the session cleared), the honest + * answer is the whole current screen rather than an empty artifact. + */ +export function screenDelta(baseline, screen) { + const after = String(screen ?? "").replace(/\s+$/, ""); + const anchor = String(baseline ?? "").replace(/\s+$/, ""); + if (!anchor) return after; + // The FIRST occurrence, not the last. A short baseline — a bare shell prompt, + // an engine that had just been cleared — can appear again inside the output + // it produced, and anchoring on the last match then returns everything after + // the final prompt glyph, which is nothing. Both failures are possible; only + // one of them is safe. A few extra lines of context is an artifact somebody + // can still read, and an empty one is a lie about an agent that answered. + const at = after.indexOf(anchor); + if (at < 0) return after; + // The baseline usually ends mid-line, on the prompt glyph the engine was + // sitting at (`… $`), so the delta opens with the space between that glyph + // and what got typed. Leading blank lines and that one space are prompt + // residue, not output. + return after.slice(at + anchor.length).replace(/^\n+/, "").replace(/^[ \t]+/, ""); +} + +// --------------------------------------------------------------------------- +// Reading +// --------------------------------------------------------------------------- + +/** + * Every task in one session's ledger, oldest first. + * + * A task with no `end` event is `open`: either it is still running, or nothing + * has looked at that session since it finished. Both are true statements and + * the caller can tell them apart by asking the roster; inventing an outcome + * here would put a guess in the one place that exists to hold evidence. + */ +export function readTasks(session) { + const byId = new Map(); + const order = []; + for (const line of readLines(session)) { + if (!line.id) continue; + if (!byId.has(line.id)) { + byId.set(line.id, { + id: line.id, session, text: "", submitted: null, baseline: "", + transitions: [], state: null, artifact: null, truncated: false, status: "open", endedAt: null, + }); + order.push(line.id); + } + const task = byId.get(line.id); + if (line.e === "submit") { + task.text = String(line.text ?? ""); + task.submitted = line.ts ?? null; + task.baseline = String(line.baseline ?? ""); + if (line.state) task.state = line.state; + } else if (line.e === "state") { + task.transitions.push({ ts: line.ts ?? null, state: line.state, ...(line.kind ? { kind: line.kind } : {}) }); + task.state = line.state; + } else if (line.e === "end") { + task.status = "closed"; + task.endedAt = line.ts ?? null; + task.state = line.state || task.state; + task.artifact = String(line.artifact ?? ""); + task.truncated = Boolean(line.truncated); + task.artifactChars = line.artifactChars ?? task.artifact.length; + } + } + return order.map((id) => { + const task = byId.get(id); + return { ...task, durationMs: task.submitted && task.endedAt ? task.endedAt - task.submitted : null }; + }); +} + +/** Every session that has a ledger. */ +export function ledgerSessions() { + try { + return fs.readdirSync(tasksDir()) + .filter((f) => f.endsWith(".jsonl")) + .map((f) => f.slice(0, -".jsonl".length)) + .sort(); + } catch { return []; } +} + +/** One task by id, wherever it lives. Ids are herd-wide, so this can search. */ +export function findTask(id, { sessions = ledgerSessions() } = {}) { + for (const session of sessions) { + const found = readTasks(session).find((t) => t.id === id); + if (found) return found; + } + return null; +} + +/** The open task for a session, if it has one. */ +export function openTask(session) { + const tasks = readTasks(session); + for (let i = tasks.length - 1; i >= 0; i--) if (tasks[i].status === "open") return tasks[i]; + return null; +} + +/** The raw state history for `herd log` — transitions, task-bound or not. */ +export function readLog(session) { + return readLines(session) + .filter((line) => line.e === "state" || line.e === "submit" || line.e === "end") + .map((line) => ({ + ts: line.ts ?? null, + id: line.id ?? null, + state: line.e === "submit" ? (line.state || "submitted") : line.state, + event: line.e, + ...(line.kind ? { kind: line.kind } : {}), + ...(line.e === "submit" ? { text: String(line.text ?? "") } : {}), + })); +} + +/** + * Time in state, per session. + * + * The interesting number is `blocked`, which is the herd's name for *human + * latency*: the agent was ready and the operator was asleep. It is the one + * figure here that is entirely within the operator's power to change, which is + * why the roster prints it with "blocked = you" next to it. + */ +export function stats(session, { now = Date.now() } = {}) { + const log = readLog(session).filter((entry) => entry.state && Number.isFinite(entry.ts)); + const totals = {}; + let tasks = 0, blockedSpells = 0; + for (const entry of log) if (entry.event === "submit") tasks++; + for (let i = 0; i < log.length; i++) { + const state = log[i].state === "submitted" ? null : log[i].state; + if (!state) continue; + const until = log[i + 1]?.ts ?? now; + const span = Math.max(0, until - log[i].ts); + totals[state] = (totals[state] || 0) + span; + if (state === "blocked") blockedSpells++; + } + return { + session, + tasks, + blockedSpells, + totals, + from: log.length ? log[0].ts : null, + to: log.length ? now : null, + }; +} + +/** Drop a session's ledger — used by `kill`/`prune`, never on its own. */ +export function forgetTasks(session) { + try { fs.rmSync(ledgerFile(session), { force: true }); return true; } + catch { return false; } +} diff --git a/src/herd.mjs b/src/herd.mjs index d05463d..8e490a8 100644 --- a/src/herd.mjs +++ b/src/herd.mjs @@ -142,6 +142,54 @@ export function forgetSession(name) { return true; } +// --------------------------------------------------------------------------- +// Remote members — the last thing a URL told us (PRD 0011 R11) +// --------------------------------------------------------------------------- +// +// A remote member has no pane to capture and no pid to signal, so its state can +// only come from a request. Requests are slow and the roster is drawn on every +// pit start, so what `ps` reads is a *cache*: whatever the last call to that +// remote observed, with the time it was observed at. The alternative — a roster +// that opens N sockets before it prints a line — makes `moshcode ps` as fast as +// the slowest agent someone registered, which is not a trade worth making for a +// column that already says `authority: remote`. +// +// It lives here rather than in herd-remote.mjs so that herd-state.mjs can read +// it without importing the module that makes network calls. + +const remoteFile = (name) => path.join(herdDir(), "remote", `${name}.json`); + +/** Record what a remote just told us. Best effort: a failed write loses a poll. */ +export function recordRemoteStatus(name, status = {}) { + try { + fs.mkdirSync(path.join(herdDir(), "remote"), { recursive: true, mode: 0o700 }); + const file = remoteFile(name); + fs.writeFileSync(file, JSON.stringify({ ...status, at: status.at ?? Date.now() }), { mode: 0o600 }); + fs.chmodSync(file, 0o600); + return true; + } catch { return false; } +} + +/** + * The cached status of a remote member, or null when it has never answered. + * + * Deliberately un-expiring. A hook report has a TTL because a stale one would + * outrank a screen that could be read instead; there is nothing better to fall + * back to here, and "it was idle an hour ago" beats "unknown" as long as the + * age travels with it — which it does, in `--json` and in `herd remote list`. + */ +export function remoteStatus(name) { + try { + const raw = JSON.parse(fs.readFileSync(remoteFile(name), "utf8")); + return raw && typeof raw === "object" ? raw : null; + } catch { return null; } +} + +export function clearRemoteStatus(name) { + try { fs.rmSync(remoteFile(name), { force: true }); return true; } + catch { return false; } +} + // --------------------------------------------------------------------------- // Substrate detection // --------------------------------------------------------------------------- @@ -226,10 +274,17 @@ export function tmux(args, { runner = spawnSync, env = process.env, encoding = " * (an inherited ANTHROPIC_API_KEY hijacks its stored login — see ENGINES), and * `-e KEY=` sets an empty value, which is not the same as unset. */ -export function sessionCommand({ bin, args = [], stripEnv = [], exec = true }) { +export function sessionCommand({ bin, args = [], stripEnv = [], setEnv = {}, exec = true }) { const unset = stripEnv.flatMap((key) => ["-u", key]); + // Set through the same `env` prefix rather than tmux's `-e`, for two reasons: + // `-e` is a 3.2+ flag and the pty substrate has no equivalent at all, so one + // prefix is the only spelling both substrates can share. + const set = Object.entries(setEnv) + .filter(([key, value]) => key && value !== undefined && value !== null) + .map(([key, value]) => `${key}=${String(value)}`); const command = [bin, ...args].map(shQuote).join(" "); - const withEnv = unset.length ? `env ${unset.map(shQuote).join(" ")} ${command}` : command; + const prefix = [...unset, ...set]; + const withEnv = prefix.length ? `env ${prefix.map(shQuote).join(" ")} ${command}` : command; // `exec` so the engine replaces the shell rather than sitting under it — one // less process between a signal and the thing meant to receive it. The pty // substrate passes exec:false because it needs the shell to outlive the @@ -302,6 +357,24 @@ export function pidAlive(pid) { * never sees EOF and waits for input forever, which is what an idle agent * should do. */ +/** + * What every session knows about itself (PRD 0011 R2). + * + * A lifecycle hook fires inside the engine's own process tree and has to name + * the session it is reporting for. Nothing else in that tree knows the name, so + * the herd puts it there at launch. `MOSHCODE_HERD_DIR` rides along because a + * hook shells out to `moshcode herd report`, and a box whose herd lives + * somewhere non-default would otherwise have its reports written to the default + * directory nobody is reading. + * + * A hook fired outside a herd session sees neither, which is the signal to exit + * quietly — see `herdReport`. Hooks must never break an engine that is not in + * the herd. + */ +export function sessionEnv(name) { + return { MOSHCODE_HERD_NAME: name, MOSHCODE_HERD_DIR: herdDir() }; +} + function ptyStart({ name, cwd, bin, args, stripEnv, env, spawner = spawn, runner = spawnSync, size = {} }) { ensureDir(); const cols = Number(size.cols) || Number(env.COLUMNS) || process.stdout.columns || 80; @@ -328,7 +401,7 @@ function ptyStart({ name, cwd, bin, args, stripEnv, env, spawner = spawn, runner // that a session which finishes on its own leaves proof it finished. const command = [ `stty rows ${rows} cols ${cols} 2>/dev/null`, - sessionCommand({ bin, args, stripEnv, exec: false }), + sessionCommand({ bin, args, stripEnv, setEnv: sessionEnv(name), exec: false }), `printf '%s' "$?" > ${shQuote(exit)}`, ].join("; "); // Reuse ptySpec's flag knowledge rather than re-deriving it: util-linux and @@ -347,7 +420,7 @@ function ptyStart({ name, cwd, bin, args, stripEnv, env, spawner = spawn, runner cwd, // Belt and braces with the stty above: some toolkits read COLUMNS/LINES // before they ever ask the terminal. - env: { ...env, COLUMNS: String(cols), LINES: String(rows), MOSHCODE_HERD_SESSION: name }, + env: { ...env, COLUMNS: String(cols), LINES: String(rows), MOSHCODE_HERD_SESSION: name, ...sessionEnv(name) }, stdio: [stdin, "ignore", "ignore"], detached: true, }); @@ -565,7 +638,7 @@ export function startSession({ }; if (substrate === "tmux") { - const command = sessionCommand({ bin, args, stripEnv }); + const command = sessionCommand({ bin, args, stripEnv, setEnv: sessionEnv(name) }); const started = tmux(tmuxStartPlan({ name, cwd, command }), { runner, env }); if (!started.ok) { return { ok: false, error: new Error(started.stderr.trim() || started.error?.message || "tmux could not start the session") }; @@ -802,12 +875,21 @@ export function listSessions({ substrate = detectSubstrate(), runner = spawnSync const names = [...new Set([...live, ...Object.keys(manifest.sessions)])].sort(); return names.map((name) => { const meta = manifest.sessions[name] || {}; - const alive = live.has(name); - const exited = !alive ? null + // A remote member (PRD 0011 R11) has no pane and no pid — it is a URL. The + // substrate can only ever report it as absent, so liveness for those rows + // means "still registered", and whether the far end answers is a question + // for herd-remote.mjs's cache rather than for tmux. + const remote = meta.kind === "remote"; + const alive = remote ? true : live.has(name); + const exited = remote ? false + : !alive ? null : panes ? Boolean(panes.get(name)?.dead) : sessionExited(name, { substrate, runner }); return { name, + kind: meta.kind || "local", + url: meta.url || null, + remoteKind: meta.remoteKind || null, engine: meta.engine || "?", // Sessions started before herds existed have none. They belong to `main` // rather than to a group rendered as "undefined". diff --git a/src/templates.mjs b/src/templates.mjs index 9962bbe..9724131 100644 --- a/src/templates.mjs +++ b/src/templates.mjs @@ -41,6 +41,23 @@ function isOwnManifest(relative) { /* ------------------------------------------------------------------ listing */ +/** + * Template collections worth knowing about that this repo does not ship + * (PRD 0011 R15). + * + * A pointer rather than a copy, deliberately. `template install` already takes + * an `owner/repo`, so vendoring somebody else's templates would mean carrying + * their updates by hand forever — and pinning a stale copy of an SDK's starter + * kit is worse than not having one. What is missing is only that nobody knows + * the name to type, so the listing says it. + */ +export const TEMPLATE_POINTERS = [ + { + spec: "digitalocean/gradient-adk-templates", + description: "DigitalOcean Gradient ADK — agent starters (A2A-capable; pairs with `moshcode install gradient`)", + }, +]; + /** The bundled templates, each with whatever its manifest says about it. */ export async function listTemplates(dir = BUNDLED_DIR) { let entries; @@ -347,19 +364,29 @@ export async function templateCommand( } const templates = await listTemplates(); if (rest.includes("--json")) { + // The bundled list stays an array, because that is what it has always + // been and something is parsing it. Pointers are named separately rather + // than mixed in: `template install ` works for one and not the + // other, and a listing that hid that difference would be a listing that + // lies about what it can do. out(JSON.stringify(templates, null, 2)); return 0; } if (!templates.length) { out("no templates bundled with this install"); - return 0; + } else { + const width = Math.max(...templates.map((t) => t.name.length)); + for (const { name, description } of templates) { + out(` ${name.padEnd(width)} ${description}`); + } } - const width = Math.max(...templates.map((t) => t.name.length)); - for (const { name, description } of templates) { - out(` ${name.padEnd(width)} ${description}`); + out(""); + out("elsewhere:"); + for (const { spec, description } of TEMPLATE_POINTERS) { + out(` ${spec} ${description}`); } out(""); - out("install one with: moshcode template install "); + out("install one with: moshcode template install "); return 0; } diff --git a/src/tools.mjs b/src/tools.mjs index c209464..0efdbe4 100644 --- a/src/tools.mjs +++ b/src/tools.mjs @@ -160,6 +160,49 @@ export const TOOLS = { }, installHelp: "Go is required to install Alpaca; install Go, then retry `moshcode install alpaca`.", }, + gradient: { + desc: "DigitalOcean Gradient ADK — build, run, deploy and evaluate agents (A2A-capable)", + bin: "gradient", + // The one tool here that is not a self-contained binary. gradient-adk is a + // Python package, and moshcode stays Node: the tool owns its runtime, the + // same way CoinPay owns Node 20. So the install spec checks for a Python + // the package can actually run on and NAMES the requirement when it is + // missing, rather than letting pip fail three screens later with a + // resolution error nobody reads. `--user` keeps it out of the system + // site-packages, which is also the only place a non-root install can go on + // a modern distro. + install: { + cmd: "sh", + args: [ + "-c", + 'python3 -c "import sys; raise SystemExit(0 if sys.version_info >= (3, 10) else 1)" 2>/dev/null ' + + '|| { echo "gradient-adk needs Python 3.10 or newer on PATH as python3 — install it, then re-run: moshcode install gradient" >&2; exit 1; }; ' + + "python3 -m pip install --user --upgrade gradient-adk", + ], + }, + installHelp: "gradient-adk is a Python package: it needs python3 (3.10+) and pip. moshcode does not install Python for you.", + // pip --user drops console scripts here, and appends nothing to PATH for + // the shell that ran the install — the same gap turso and kimi have. + binDirs: [path.join(homedir(), ".local", "bin")], + // How the ADK's dev server reads in the herd (PRD 0011 R15). `gradient + // agent run --dev` is uvicorn underneath, and its startup banner is a clear + // "I am up and waiting", which is `idle`. + // + // There is deliberately no `working` rule. uvicorn writes its access line + // when a request has FINISHED, so a screen showing one is a screen showing + // a server that is free again — a rule matching it would pin the tile to + // `working` from the first request until the line scrolled away, which is + // the exact kind of rot the sub-kinds and hooks exist to get away from. So + // the completed request counts as idle too, and it is right both times. + // Watching a *deployed* agent's state is what `herd remote add` is for. + state: { + idle: [ + /\buvicorn running on\b/i, + /\bapplication startup complete\b/i, + /"(?:POST|GET|PUT) \/[^"]*" \d{3}\b/, + ], + }, + }, mcpjam: { desc: "MCPJam — test, debug, and validate MCP servers (health, OAuth, tool-surface diffs)", bin: "mcpjam", diff --git a/test/herd-agent-protocol.test.mjs b/test/herd-agent-protocol.test.mjs new file mode 100644 index 0000000..ea951de --- /dev/null +++ b/test/herd-agent-protocol.test.mjs @@ -0,0 +1,236 @@ +// The rest of PRD 0011: sessions that know their own name, blocked sub-kinds, +// the rules file that finally gets to complain, and fan-in. +import test from "node:test"; +import assert from "node:assert/strict"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; + +import { sessionCommand, sessionEnv } from "../src/herd.mjs"; +import { + BLOCKED_KINDS, blockedKind, hookReport, inspectUserRules, parseState, reportState, rulesFor, sessionState, +} from "../src/herd-state.mjs"; +import { EXIT, waitForMany } from "../src/herd-cli.mjs"; +import { TOOLS } from "../src/tools.mjs"; + +function withHerdDir(fn) { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), "moshcode-0011-test-")); + const previous = process.env.MOSHCODE_HERD_DIR; + process.env.MOSHCODE_HERD_DIR = dir; + try { return fn(dir); } + finally { + if (previous === undefined) delete process.env.MOSHCODE_HERD_DIR; + else process.env.MOSHCODE_HERD_DIR = previous; + fs.rmSync(dir, { recursive: true, force: true }); + } +} + +/* ------------------------------------------------- R2: knowing your own name */ + +test("a session is told its own name and where the herd lives", () => { + // A hook fires inside the engine's process tree and has to name the session + // it is reporting for. Nothing else in that tree knows it. + withHerdDir((dir) => { + const env = sessionEnv("api"); + assert.equal(env.MOSHCODE_HERD_NAME, "api"); + assert.equal(env.MOSHCODE_HERD_DIR, dir, "a non-default herd dir would have its reports written elsewhere"); + }); +}); + +test("the name is injected through the same env prefix both substrates share", () => { + const command = sessionCommand({ bin: "claude", args: [], setEnv: sessionEnv("api") }); + // shQuote wraps every word, which is why these look for the quoted forms. + assert.match(command, /env .*'MOSHCODE_HERD_NAME=api'/); + assert.match(command, /'claude'$/); +}); + +test("setting a variable and unsetting one compose in one prefix", () => { + // `env -e` is a tmux 3.2+ flag the pty substrate has no equivalent for, and + // an inherited ANTHROPIC_API_KEY still has to be *removed* rather than blanked. + const command = sessionCommand({ bin: "claude", stripEnv: ["ANTHROPIC_API_KEY"], setEnv: sessionEnv("api") }); + assert.match(command, /'-u' 'ANTHROPIC_API_KEY'/); + assert.match(command, /'MOSHCODE_HERD_NAME=api'/); +}); + +test("a session with nothing to set or unset gets no env wrapper at all", () => { + assert.equal(sessionCommand({ bin: "sh", args: [], exec: false }), "'sh'"); +}); + +/* --------------------------------------------------- R4: what it is blocked on */ + +test("a state token can carry the kind of blocked it means", () => { + assert.deepEqual(parseState("blocked:menu"), { state: "blocked", kind: "menu" }); + assert.deepEqual(parseState("working"), { state: "working" }); + assert.equal(parseState("done:menu"), null, "only blocked has sub-kinds"); + assert.equal(parseState("blocked:whatever"), null); + assert.equal(parseState("nonsense"), null); +}); + +test("a reported sub-kind survives the round trip", () => { + withHerdDir(() => { + assert.equal(reportState("api", "blocked:permission").ok, true); + assert.equal(hookReport("api").kind, "permission"); + const state = sessionState({ name: "api", engine: "claude", alive: true, exited: false }); + assert.deepEqual(state, { state: "blocked", authority: "hook", blockedOn: "permission" }); + }); +}); + +test("a numbered menu is a menu, and a y/n is a permission", () => { + // The distinction the notification uses: a menu wants a digit and a + // permission wants a letter, and answering one with the other types prose + // into a selector. + assert.equal(blockedKind("Do you want to proceed?\n❯ 1. Yes\n 2. No"), "menu"); + assert.equal(blockedKind("Overwrite the file? [y/N]"), "permission"); + assert.equal(blockedKind("Which environment should I deploy to?"), "question"); + assert.equal(blockedKind("nothing prompt-shaped here"), null); + for (const kind of BLOCKED_KINDS) assert.ok(typeof kind === "string"); +}); + +test("the sub-kind is absent rather than undefined when there is not one", () => { + // This object is spread over every roster row. An always-present key for the + // benefit of none is a new key on every row. + withHerdDir(() => { + const state = sessionState({ name: "api", engine: "claude", alive: true, exited: false }, { read: () => "esc to interrupt" }); + assert.deepEqual(state, { state: "working", authority: "screen" }); + assert.equal("blockedOn" in state, false); + }); +}); + +test("the sub-kind never overwrites the row's own kind", () => { + // `kind` on a roster row says local-or-remote. Naming the sub-kind `kind` too + // would have a blocked local session claiming to be a URL. + withHerdDir(() => { + const state = sessionState({ name: "api", engine: "claude", alive: true, exited: false, kind: "local" }, + { read: () => "Do you want to proceed?\n❯ 1. Yes" }); + assert.equal(state.blockedOn, "menu"); + assert.equal(state.kind, undefined, "sessionState must not return a `kind` key at all"); + }); +}); + +/* ---------------------------------------------------- R3: the rules file talks */ + +test("a rules file that is not JSON is named, not silently ignored", () => { + // loadUserRules() swallows this by design and must keep doing so. The cost + // is a file quietly ignored since someone typo'd a bracket, and doctor is + // where that gets to be loud. + withHerdDir((dir) => { + const file = path.join(dir, "rules.json"); + fs.writeFileSync(file, "{ nope"); + const report = inspectUserRules(file); + assert.equal(report.ok, false); + assert.match(report.problems[0].error, /not valid JSON/); + }); +}); + +test("a pattern that will not compile is named with its engine and state", () => { + withHerdDir((dir) => { + const file = path.join(dir, "rules.json"); + fs.writeFileSync(file, JSON.stringify({ codex: { blocked: ["fine", "a(b"] } })); + const report = inspectUserRules(file); + assert.equal(report.ok, false); + assert.equal(report.patterns, 1); + assert.equal(report.problems[0].where, "codex.blocked"); + assert.equal(report.problems[0].pattern, "a(b"); + }); +}); + +test("a state the classifier does not read is a problem worth saying", () => { + withHerdDir((dir) => { + const file = path.join(dir, "rules.json"); + fs.writeFileSync(file, JSON.stringify({ codex: { blocekd: ["typo"] } })); + assert.match(inspectUserRules(file).problems[0].error, /not a state the classifier reads/); + }); +}); + +test("no rules file at all is not a problem", () => { + withHerdDir((dir) => { + const report = inspectUserRules(path.join(dir, "rules.json")); + assert.equal(report.present, false); + assert.equal(report.ok, true); + }); +}); + +/* ------------------------------------------------- R15: a tool in the herd */ + +test("a workflow tool's screen rules are found the same way an engine's are", () => { + // `herd run -- gradient agent run --dev` names its session after the binary, + // and the workflow CLIs are exactly what people put in the herd next to an + // agent. + const rules = rulesFor("gradient", { userRules: {} }); + assert.ok(rules.idle.some((re) => re.test("INFO: Uvicorn running on http://127.0.0.1:8000"))); + assert.ok(TOOLS.gradient.state, "the rules live beside the install spec"); +}); + +test("a served request leaves the ADK dev server idle, not stuck working", () => { + // uvicorn writes its access line when a request has FINISHED. A rule that + // read it as `working` would pin the tile there until it scrolled away — + // exactly the rot this PRD is trying to get away from. + const rules = rulesFor("gradient", { userRules: {} }); + const served = 'INFO: 127.0.0.1:51234 - "POST /run HTTP/1.1" 200 OK'; + assert.ok(rules.idle.some((re) => re.test(served))); + assert.equal(rules.working.some((re) => re.test(served)), false); +}); + +test("gradient owns its own runtime rather than moshcode growing a Python", () => { + assert.match(TOOLS.gradient.install.args.join(" "), /python3/); + assert.match(TOOLS.gradient.install.args.join(" "), /3, 10/, "the version requirement is checked, not assumed"); + assert.ok(TOOLS.gradient.installHelp, "a missing Python has to name the fix"); +}); + +/* ------------------------------------------------------------ R8: fan-in */ + +test("--any returns on the first member to arrive", async () => { + const states = { api: "working", web: "working" }; + const observe = async (name) => ({ name, present: true, alive: true, state: states[name] }); + setTimeout(() => { states.web = "blocked"; }, 5); + const result = await waitForMany(["api", "web"], ["blocked"], { + mode: "any", intervalMs: 1, nap: (ms) => new Promise((r) => setTimeout(r, ms)), observe, + }); + assert.equal(result.outcome, "matched"); + assert.equal(result.winner, "web"); +}); + +test("--all returns only when every one of them has", async () => { + const states = { api: "working", web: "blocked" }; + const observe = async (name) => ({ name, present: true, alive: true, state: states[name] }); + setTimeout(() => { states.api = "blocked"; }, 5); + const result = await waitForMany(["api", "web"], ["blocked"], { + mode: "all", intervalMs: 1, nap: (ms) => new Promise((r) => setTimeout(r, ms)), observe, + }); + assert.equal(result.outcome, "matched"); + assert.equal(result.results.length, 2); +}); + +test("a member that ends without getting there ends the --all wait honestly", async () => { + const observe = async (name) => ({ name, present: true, alive: name !== "web", state: name === "web" ? "done" : "working" }); + const result = await waitForMany(["api", "web"], ["blocked"], { + mode: "all", timeoutMs: 5, intervalMs: 1, nap: (ms) => new Promise((r) => setTimeout(r, ms)), observe, + }); + assert.notEqual(result.outcome, "matched"); + assert.ok(result.results.some((r) => r.name === "web" && r.outcome === "ended")); +}); + +test("a fan-in that times out says which members it was still waiting on", async () => { + const observe = async (name) => ({ name, present: true, alive: true, state: "working" }); + const result = await waitForMany(["api", "web"], ["blocked"], { + mode: "any", timeoutMs: 3, intervalMs: 1, nap: (ms) => new Promise((r) => setTimeout(r, ms)), observe, + }); + assert.equal(result.outcome, "timeout"); + assert.deepEqual(result.pending, ["api", "web"]); +}); + +test("a member that does not exist is gone, not waited on forever", async () => { + const observe = async (name) => ({ name, present: false }); + const result = await waitForMany(["nobody"], ["blocked"], { mode: "any", intervalMs: 1, observe }); + assert.equal(result.results[0].outcome, "gone"); +}); + +/* ------------------------------------------------------------- exit codes */ + +test("eval's outcomes have codes of their own", () => { + // 0 pass, 4 below the threshold, 5 the harness could not run. + assert.equal(EXIT.matched, 0); + assert.equal(EXIT.below, 4); + assert.equal(EXIT.infra, 5); + assert.equal(new Set(Object.values(EXIT)).size, Object.values(EXIT).length, "two outcomes share a code"); +}); diff --git a/test/herd-eval.test.mjs b/test/herd-eval.test.mjs new file mode 100644 index 0000000..74f26cb --- /dev/null +++ b/test/herd-eval.test.mjs @@ -0,0 +1,180 @@ +// Evals: the dataset shapes, the two judges, and the three outcomes CI has to +// be able to tell apart. +import test from "node:test"; +import assert from "node:assert/strict"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; + +import { + DEFAULT_THRESHOLD, extractVerdict, judgePrompt, loadDataset, parseCsv, + resolveEngines, runEval, scoreByJudge, scoreByRules, +} from "../src/herd-eval.mjs"; + +function withFile(name, body, fn) { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), "moshcode-eval-test-")); + const file = path.join(dir, name); + fs.writeFileSync(file, body); + try { return fn(file); } + finally { fs.rmSync(dir, { recursive: true, force: true }); } +} + +/* --------------------------------------------------------------- datasets */ + +test("a jsonl dataset is one case per line", () => { + withFile("d.jsonl", '{"prompt":"a","expect":"x"}\n{"prompt":"b","rubric":"is it right?"}\n', (file) => { + const { ok, cases } = loadDataset(file); + assert.equal(ok, true); + assert.equal(cases.length, 2); + assert.equal(cases[0].expect, "x"); + assert.equal(cases[1].rubric, "is it right?"); + assert.equal(cases[0].id, "case-1", "a case with no id still needs a handle in the report"); + }); +}); + +test("a csv dataset survives quotes, commas and newlines inside a prompt", () => { + // A dataset is a file somebody wrote by hand or exported from a spreadsheet, + // and those two shapes are the whole requirement. + const rows = parseCsv('prompt,expect\n"say ""hi"", then stop",hi\n"two\nlines",x\n'); + assert.deepEqual(rows[1], ['say "hi", then stop', "hi"]); + assert.deepEqual(rows[2], ["two\nlines", "x"]); +}); + +test("a csv dataset loads by header name", () => { + withFile("d.csv", "prompt,expect\nrun the tests,passed\n", (file) => { + const { cases } = loadDataset(file); + assert.equal(cases[0].prompt, "run the tests"); + assert.equal(cases[0].expect, "passed"); + }); +}); + +test("a case with no prompt is the dataset's error, and is named as one", () => { + withFile("d.jsonl", '{"expect":"x"}\n', (file) => { + const result = loadDataset(file); + assert.equal(result.ok, false); + assert.match(String(result.error.message), /case 1 has no prompt/); + }); +}); + +test("a malformed line names its line number", () => { + withFile("d.jsonl", '{"prompt":"a"}\n{oops\n', (file) => { + assert.match(String(loadDataset(file).error.message), /:2 is not valid JSON/); + }); +}); + +test("a missing dataset is a failure to load, not an empty run", () => { + assert.equal(loadDataset("/nope/nothing.jsonl").ok, false); +}); + +/* ---------------------------------------------------------------- scoring */ + +test("the rules judge treats the expectation as a loose pattern", () => { + assert.equal(scoreByRules({ expect: "3 tests passed" }, "…\n3 TESTS PASSED\n").score, 1); + assert.equal(scoreByRules({ expect: "^done$" }, "done").score, 1); + assert.equal(scoreByRules({ expect: "passed" }, "everything failed").score, 0); +}); + +test("a case with no expectation is unscorable, not a zero", () => { + // A missing expectation is the dataset's bug. Scoring it against the engine + // would mark a good answer wrong for a reason the engine cannot fix. + const verdict = scoreByRules({ prompt: "x" }, "a fine answer"); + assert.equal(verdict.ok, false); + assert.match(verdict.why, /needs a judge, or an expectation/); +}); + +test("an unparseable expectation falls back to a substring rather than throwing", () => { + assert.equal(scoreByRules({ expect: "a(b" }, "xxa(bxx").score, 1); +}); + +test("the judge's verdict is pulled out of whatever prose it wrapped it in", () => { + assert.deepEqual(extractVerdict('Sure!\n{"score": 0.5, "why": "partly"}\nHope that helps'), { score: 0.5, why: "partly" }); + assert.equal(extractVerdict("no json here"), null); +}); + +test("a judge that does not answer with a score is unscorable, not a zero", () => { + const verdict = scoreByJudge({ prompt: "x" }, "answer", { engine: "claude", run: () => "I'd rather not" }); + assert.equal(verdict.ok, false); + assert.equal(verdict.score, 0); + assert.match(verdict.why, /did not answer with a score/); +}); + +test("a judge that throws is reported, not allowed to end the run", () => { + const verdict = scoreByJudge({ prompt: "x" }, "answer", { engine: "claude", run: () => { throw new Error("no engine installed"); } }); + assert.equal(verdict.ok, false); + assert.match(verdict.why, /no engine installed/); +}); + +test("the judge is asked for JSON and given the rubric", () => { + const prompt = judgePrompt({ prompt: "port the routes", rubric: "did it keep the auth middleware?" }, "I ported them"); + assert.match(prompt, /RUBRIC: did it keep the auth middleware\?/); + assert.match(prompt, /"score"/); +}); + +test("a judged score is clamped to the range it is supposed to be in", () => { + assert.equal(scoreByJudge({}, "a", { run: () => '{"score": 4}' }).score, 1); + assert.equal(scoreByJudge({}, "a", { run: () => '{"score": -2}' }).score, 0); +}); + +/* ------------------------------------------------------------- the report */ + +const cases = [{ id: "one", prompt: "p1", expect: "good" }, { id: "two", prompt: "p2", expect: "good" }]; +const fakeRun = (answers) => async (engine) => ({ + engine, ok: true, session: `eval-${engine}`, + results: cases.map((c, i) => ({ ...c, engine, taskId: `t-${i}`, ok: true, answer: answers[engine][i], state: "idle" })), +}); + +test("an engine below the threshold is a distinct outcome from a broken harness", async () => { + // The whole point of the exit codes: CI has to tell "the agent got worse" + // apart from "the box fell over", and one non-zero code cannot say both. + const report = await runEval({ + cases, engines: ["claude", "codex"], threshold: 0.8, + run: fakeRun({ claude: ["good", "good"], codex: ["good", "bad"] }), + }); + assert.equal(report.outcome, "below"); + assert.deepEqual(report.below, ["codex"]); + assert.equal(report.engines.find((e) => e.engine === "claude").score, 1); + assert.equal(report.engines.find((e) => e.engine === "codex").score, 0.5); +}); + +test("everything at or above the threshold passes", async () => { + const report = await runEval({ cases, engines: ["claude"], threshold: 1, run: fakeRun({ claude: ["good", "good"] }) }); + assert.equal(report.outcome, "pass"); + assert.deepEqual(report.below, []); +}); + +test("an engine that could not start is infrastructure, not a bad score", async () => { + const report = await runEval({ + cases, engines: ["claude"], + run: async (engine) => ({ engine, ok: false, error: "no such binary", results: [] }), + }); + assert.equal(report.outcome, "infrastructure"); + assert.deepEqual(report.broken, [{ engine: "claude", error: "no such binary" }]); +}); + +test("infrastructure trouble outranks a low score in the outcome", async () => { + const report = await runEval({ + cases, engines: ["claude", "codex"], threshold: 0.9, + run: async (engine) => (engine === "claude" + ? { engine, ok: false, error: "gone", results: [] } + : (await fakeRun({ codex: ["bad", "bad"] })(engine))), + }); + assert.equal(report.outcome, "infrastructure", "a broken run must not be reported as a failing engine"); +}); + +test("unscorable cases are counted and shown, not quietly averaged in", async () => { + const report = await runEval({ + cases: [{ id: "one", prompt: "p" }], engines: ["claude"], + run: async (engine) => ({ engine, ok: true, results: [{ id: "one", prompt: "p", engine, taskId: "t", answer: "x" }] }), + }); + assert.equal(report.engines[0].unscorable, 1); +}); + +test("engine names resolve through the same aliases as everywhere else", () => { + assert.deepEqual(resolveEngines("cc,codex").keys, ["claude", "codex"]); + assert.deepEqual(resolveEngines("cc,cc").keys, ["claude"], "asking for one engine twice is one engine"); + assert.deepEqual(resolveEngines("nope").unknown, ["nope"]); +}); + +test("the default threshold is a number, not a vibe", () => { + assert.ok(DEFAULT_THRESHOLD > 0 && DEFAULT_THRESHOLD <= 1); +}); diff --git a/test/herd-hooks.test.mjs b/test/herd-hooks.test.mjs new file mode 100644 index 0000000..e3f3987 --- /dev/null +++ b/test/herd-hooks.test.mjs @@ -0,0 +1,199 @@ +// Engine hooks: the merge that must not clobber, the guard that must not break +// an engine, and the removal that must only take back what we put in. +import test from "node:test"; +import assert from "node:assert/strict"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; + +import { + hookCommand, hookDiff, hookableEngines, hooksStatus, installHooks, isOurs, removeHooks, +} from "../src/herd-hooks.mjs"; +import { ENGINES } from "../src/engines.mjs"; + +function withSettings(initial, fn) { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), "moshcode-hooks-test-")); + const file = path.join(dir, "settings.json"); + if (initial !== undefined) fs.writeFileSync(file, typeof initial === "string" ? initial : JSON.stringify(initial, null, 2)); + try { return fn(file); } + finally { fs.rmSync(dir, { recursive: true, force: true }); } +} + +const read = (file) => JSON.parse(fs.readFileSync(file, "utf8")); + +/* ------------------------------------------------------------- the command */ + +test("a hook fired outside a herd session does nothing and exits 0", () => { + // The single most important property here. An engine is used by hand far + // more often than it is used in a herd, and a hook that fails on every turn + // of a working engine gets the whole feature uninstalled by lunchtime. + const command = hookCommand("done"); + assert.match(command, /\[ -n "\$MOSHCODE_HERD_NAME" \]/, "no guard on the session name"); + assert.match(command, /command -v moshcode/, "a box without moshcode would get a failing hook"); + assert.match(command, /exit 0\s*$/, "the engine must carry on whatever happened"); +}); + +test("the hook says nothing to the operator", () => { + // Its whole output belongs in the roster, not in the middle of a session. + assert.match(hookCommand("working"), />\/dev\/null 2>&1/); +}); + +test("the command carries the state it reports", () => { + assert.match(hookCommand("blocked"), /herd report "\$MOSHCODE_HERD_NAME" blocked\b/); +}); + +test("our entries are recognised by their command, not by a marker we invented", () => { + // The file's schema belongs to the engine. An unknown key is something it is + // entitled to reject, and a config it rejects is worse than no hook at all. + assert.equal(isOurs({ type: "command", command: hookCommand("done") }), true); + assert.equal(isOurs({ type: "command", command: "echo hello" }), false); + assert.equal(isOurs({}), false); + assert.equal(isOurs(null), false); +}); + +/* ------------------------------------------------------------ install/merge */ + +test("installing extends a settings file rather than replacing it", () => { + withSettings({ + model: "opus", + hooks: { Stop: [{ hooks: [{ type: "command", command: "echo theirs" }] }] }, + }, (file) => { + const result = installHooks("claude", { file }); + assert.equal(result.ok, true); + const after = read(file); + assert.equal(after.model, "opus", "an unrelated setting was lost"); + const stop = after.hooks.Stop.flatMap((g) => g.hooks); + assert.ok(stop.some((h) => h.command === "echo theirs"), "the user's own hook was clobbered"); + assert.ok(stop.some(isOurs), "ours was not added"); + }); +}); + +test("installing twice does not fire the hook twice", () => { + withSettings({}, (file) => { + installHooks("claude", { file }); + const second = installHooks("claude", { file }); + assert.equal(second.changes.every((c) => c.change === "unchanged"), true); + const stop = read(file).hooks.Stop.flatMap((g) => g.hooks).filter(isOurs); + assert.equal(stop.length, 1); + }); +}); + +test("a command from an older release is replaced, not duplicated", () => { + withSettings({ + hooks: { Stop: [{ hooks: [{ type: "command", command: 'moshcode herd report "$MOSHCODE_HERD_NAME" done' }] }] }, + }, (file) => { + const result = installHooks("claude", { file }); + assert.equal(result.changes.find((c) => c.event === "Stop").change, "updated"); + const stop = read(file).hooks.Stop.flatMap((g) => g.hooks).filter(isOurs); + assert.equal(stop.length, 1, "an upgrade must not leave two copies firing"); + assert.equal(stop[0].command, hookCommand("done")); + }); +}); + +test("a settings file that cannot be parsed is refused, not overwritten", () => { + // Overwriting it would take every other hook, MCP server and preference in + // it along with the mistake. + withSettings("{ not json", (file) => { + const result = installHooks("claude", { file }); + assert.equal(result.ok, false); + assert.match(String(result.error.message), /not valid JSON/); + assert.equal(fs.readFileSync(file, "utf8"), "{ not json", "the file was modified anyway"); + }); +}); + +test("--dry-run writes nothing and can still show the change", () => { + withSettings({ model: "opus" }, (file) => { + const result = installHooks("claude", { file, dryRun: true }); + assert.equal(result.ok, true); + assert.deepEqual(read(file), { model: "opus" }, "a dry run touched the file"); + const diff = hookDiff(result.before, result.after); + assert.match(diff, /^\+.*MOSHCODE_HERD_NAME/m, "the diff does not show what would be added"); + }); +}); + +test("installing creates the file when the engine has never been configured", () => { + withSettings(undefined, (file) => { + assert.equal(installHooks("claude", { file }).ok, true); + assert.equal(Object.keys(read(file).hooks).length, ENGINES.claude.hooks.events.length); + }); +}); + +/* ------------------------------------------------------------------ remove */ + +test("remove takes out only what moshcode put in", () => { + withSettings({ + hooks: { + Stop: [{ hooks: [{ type: "command", command: "echo theirs" }] }], + PreToolUse: [{ matcher: "Bash", hooks: [{ type: "command", command: "echo audit" }] }], + }, + }, (file) => { + installHooks("claude", { file }); + const removed = removeHooks("claude", { file }); + assert.equal(removed.removed, 3); + const after = read(file); + assert.deepEqual(after.hooks.Stop, [{ hooks: [{ type: "command", command: "echo theirs" }] }]); + assert.deepEqual(after.hooks.PreToolUse, [{ matcher: "Bash", hooks: [{ type: "command", command: "echo audit" }] }]); + assert.equal(after.hooks.Notification, undefined, "an event group we created should go with us"); + }); +}); + +test("remove sweeps an event an older spec used, not only the current ones", () => { + // Otherwise `remove` after an upgrade leaves the previous version's hook + // firing forever, with nothing left that knows how to take it out. + withSettings({ + hooks: { PreToolUse: [{ hooks: [{ type: "command", command: hookCommand("working") }] }] }, + }, (file) => { + const removed = removeHooks("claude", { file }); + assert.equal(removed.removed, 1); + assert.equal(read(file).hooks, undefined); + }); +}); + +test("removing from a file that was never written is not an error", () => { + withSettings(undefined, (file) => { + const result = removeHooks("claude", { file }); + assert.equal(result.ok, true); + assert.equal(result.removed, 0); + }); +}); + +/* ------------------------------------------------------------------ status */ + +test("status distinguishes absent, current, and out of date", () => { + withSettings({}, (file) => { + assert.equal(hooksStatus("claude", { file }).installed, false); + installHooks("claude", { file }); + assert.equal(hooksStatus("claude", { file }).installed, true); + + const settings = read(file); + settings.hooks.Stop[0].hooks[0].command = 'moshcode herd report "$MOSHCODE_HERD_NAME" done'; + fs.writeFileSync(file, JSON.stringify(settings)); + const stale = hooksStatus("claude", { file }); + assert.equal(stale.installed, false, "an out-of-date command is not the current install"); + assert.equal(stale.partial, true, "…but it is not 'never installed' either"); + }); +}); + +test("an engine with no hook spec says so instead of pretending", () => { + // A guessed hook schema is a rule that rots with no screen to fall back to. + assert.equal(hooksStatus("codex").supported, false); + assert.equal(installHooks("codex").ok, false); + assert.ok(hookableEngines().includes("claude")); +}); + +test("every engine with a hook spec keeps its screen rules", () => { + // A hook that a schema change quietly breaks must degrade to what the herd + // did before it, never below it. + for (const key of hookableEngines()) { + assert.ok(ENGINES[key].state, `${key} dropped its screen rules when it gained hooks`); + } +}); + +test("every hook event reports a state the herd actually has", () => { + for (const key of hookableEngines()) { + for (const { event, state, label } of ENGINES[key].hooks.events) { + assert.ok(["working", "blocked", "done", "idle"].includes(state), `${key}/${event} reports ${state}`); + assert.ok(label, `${key}/${event} has no human-readable label`); + } + } +}); diff --git a/test/herd-remote.test.mjs b/test/herd-remote.test.mjs new file mode 100644 index 0000000..84a9161 --- /dev/null +++ b/test/herd-remote.test.mjs @@ -0,0 +1,273 @@ +// Remote members: the translation table between A2A and the herd, the auth that +// is never written down, and the honesty about what a URL can and cannot say. +import test from "node:test"; +import assert from "node:assert/strict"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; + +import { readManifest, remoteStatus } from "../src/herd.mjs"; +import { + A2A_TO_HERD, addRemote, cancelRemote, cardUrl, herdStateFor, listRemotes, parseRemoteUrl, + partsText, pingRemote, promptRemote, readA2aTask, readRemote, removeRemote, runAnswer, + tokenEnvVar, waitRemote, +} from "../src/herd-remote.mjs"; +import { sessionState } from "../src/herd-state.mjs"; + +// Async on purpose: a sync `finally` would delete the directory and restore +// the environment the moment the callback returned its *promise*, leaving the +// test to run against a herd dir that is already gone. Every caller awaits. +async function withHerdDir(fn) { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), "moshcode-remote-test-")); + const previous = process.env.MOSHCODE_HERD_DIR; + process.env.MOSHCODE_HERD_DIR = dir; + try { return await fn(dir); } + finally { + if (previous === undefined) delete process.env.MOSHCODE_HERD_DIR; + else process.env.MOSHCODE_HERD_DIR = previous; + fs.rmSync(dir, { recursive: true, force: true }); + } +} + +/** A fetch that answers from a table and records what it was asked. */ +function stubFetch(handler) { + const calls = []; + const impl = async (url, options = {}) => { + const body = options.body ? JSON.parse(options.body) : null; + calls.push({ url: String(url), method: options.method || "GET", body, headers: options.headers || {} }); + const answer = await handler({ url: String(url), body, options }); + return { + ok: answer.status ? answer.status < 400 : true, + status: answer.status || 200, + text: async () => (typeof answer.body === "string" ? answer.body : JSON.stringify(answer.body ?? {})), + }; + }; + impl.calls = calls; + return impl; +} + +const a2aTask = (state, artifact) => ({ + jsonrpc: "2.0", id: "1", + result: { + kind: "task", id: "remote-1", contextId: "c1", status: { state }, + artifacts: artifact ? [{ artifactId: "a", parts: [{ kind: "text", text: artifact }] }] : [], + }, +}); + +/* ------------------------------------------------------- the mapping table */ + +test("A2A's task states map onto the herd's, and input-required is blocked", () => { + // The whole premise: this is a translation table, not an integration. + assert.equal(herdStateFor("input-required"), "blocked"); + assert.equal(herdStateFor("working"), "working"); + assert.equal(herdStateFor("submitted"), "working"); + assert.equal(herdStateFor("completed"), "done"); + assert.equal(herdStateFor("canceled"), "done"); + assert.equal(herdStateFor("nonsense"), "unknown"); + for (const state of Object.values(A2A_TO_HERD)) { + assert.ok(["working", "blocked", "done", "unknown"].includes(state), `${state} is not a herd state`); + } +}); + +test("a task's text comes out of its parts, artifacts first", () => { + assert.equal(partsText({ parts: [{ kind: "text", text: "a" }, { kind: "file" }, { kind: "text", text: "b" }] }), "a\nb"); + assert.equal(readA2aTask(a2aTask("completed", "the answer").result).artifact, "the answer"); + assert.equal(readA2aTask(a2aTask("input-required").result).state, "blocked"); +}); + +/* ------------------------------------------------------------- registering */ + +test("only http(s) URLs can be registered", () => { + // `herd prompt` on a remote POSTs user text to whatever this says. The one + // thing that must not be possible is a scheme that means something else. + assert.equal(parseRemoteUrl("file:///etc/passwd").ok, false); + assert.equal(parseRemoteUrl("not a url").ok, false); + assert.equal(parseRemoteUrl("https://agents.do-ai.run/x/y").ok, true); +}); + +test("adding a remote contacts nothing and writes no token", async () => { + await withHerdDir(() => { + const added = addRemote("research", "https://agents.do-ai.run/w/prod", { kind: "a2a" }); + assert.equal(added.ok, true); + const entry = readManifest().sessions.research; + assert.equal(entry.kind, "remote"); + assert.equal(entry.remoteKind, "a2a"); + assert.equal(entry.cwd, "agents.do-ai.run", "the host is what `ps` shows where a local shows its cwd"); + // 0010's allowlist reasoning, verbatim: a bearer token for someone else's + // agent must not ride along to another machine. + assert.equal(JSON.stringify(entry).includes("token"), false); + assert.equal(tokenEnvVar("research"), "MOSHCODE_REMOTE_RESEARCH_TOKEN"); + }); +}); + +test("a remote cannot take the name of a local session", async () => { + await withHerdDir(async () => { + const { rememberSession } = await import("../src/herd.mjs"); + rememberSession("api", { engine: "claude" }); + assert.equal(addRemote("api", "https://example.com").ok, false); + }); +}); + +test("an unknown kind is refused rather than guessed at", async () => { + await withHerdDir(() => { + assert.equal(addRemote("x", "https://example.com", { kind: "grpc" }).ok, false); + }); +}); + +test("removing a remote deregisters it and forgets what it last said", async () => { + await withHerdDir(() => { + addRemote("research", "https://example.com", { kind: "run" }); + assert.equal(removeRemote("research").ok, true); + assert.equal(listRemotes().length, 0); + assert.equal(remoteStatus("research"), null); + assert.equal(removeRemote("research").ok, false); + }); +}); + +/* ------------------------------------------------------------------ state */ + +test("a remote's state is reported as the remote's claim, never as ours", async () => { + await withHerdDir(() => { + addRemote("research", "https://example.com", { kind: "run" }); + const before = sessionState({ name: "research", kind: "remote" }); + assert.deepEqual(before, { state: "unknown", authority: "remote" }, + "a remote nobody has asked yet is unknown, not idle"); + }); +}); + +test("a request/response endpoint is idle when it is up, and says nothing more", async () => { + await withHerdDir(async () => { + addRemote("deployed", "https://example.com", { kind: "run" }); + const fetchImpl = stubFetch(() => ({ status: 200, body: { status: "ok" } })); + const pinged = await pingRemote("deployed", { fetchImpl }); + assert.equal(pinged.state, "idle"); + assert.equal(sessionState({ name: "deployed", kind: "remote" }).authority, "remote"); + assert.equal(sessionState({ name: "deployed", kind: "remote" }).state, "idle"); + }); +}); + +test("an unreachable remote reads unknown, not gone", () => { + // `gone` is a claim about a process we started. We did not start this one. + return withHerdDir(async () => { + addRemote("deployed", "https://example.com", { kind: "run" }); + const fetchImpl = stubFetch(() => { throw new Error("ECONNREFUSED"); }); + const pinged = await pingRemote("deployed", { fetchImpl }); + assert.equal(pinged.ok, false); + assert.equal(remoteStatus("deployed").state, "unknown"); + }); +}); + +/* --------------------------------------------------------------- prompting */ + +test("prompting an a2a member sends a message and keeps its task id", async () => { + await withHerdDir(async () => { + addRemote("research", "https://example.com", { kind: "a2a" }); + const fetchImpl = stubFetch(({ body }) => { + assert.equal(body.jsonrpc, "2.0"); + assert.equal(body.method, "message/send"); + assert.equal(body.params.message.parts[0].text, "summarise the week"); + return { body: a2aTask("working") }; + }); + const sent = await promptRemote("research", "summarise the week", { fetchImpl }); + assert.equal(sent.ok, true); + assert.equal(sent.state, "working"); + assert.equal(remoteStatus("research").taskId, "remote-1"); + }); +}); + +test("prompting a run member posts a prompt and takes the answer back", async () => { + await withHerdDir(async () => { + addRemote("deployed", "https://example.com", { kind: "run" }); + const fetchImpl = stubFetch(({ body }) => { + assert.deepEqual(body, { prompt: "hello" }); + return { body: { output: "the deployed answer" } }; + }); + const sent = await promptRemote("deployed", "hello", { fetchImpl }); + assert.equal(sent.artifact, "the deployed answer"); + assert.equal(sent.state, "done", "a request/response call is finished when it returns"); + assert.equal(readRemote("deployed"), "the deployed answer"); + }); +}); + +test("the answer is found under whichever key the endpoint chose", () => { + // No standard says what the key is, so the fallback returns the whole body + // rather than "" — an empty artifact is a lie about an agent that answered. + assert.equal(runAnswer({ output: "a" }), "a"); + assert.equal(runAnswer({ response: "b" }), "b"); + assert.equal(runAnswer({ message: { parts: [{ kind: "text", text: "c" }] } }), "c"); + assert.match(runAnswer({ surprise: 1 }), /surprise/); + assert.equal(runAnswer(null, "plain text"), "plain text"); +}); + +test("a token in the environment is sent, and its absence is not an error", async () => { + await withHerdDir(async () => { + addRemote("research", "https://example.com", { kind: "run" }); + const fetchImpl = stubFetch(() => ({ body: { output: "ok" } })); + await promptRemote("research", "hi", { fetchImpl, env: { MOSHCODE_REMOTE_RESEARCH_TOKEN: "sekrit" } }); + assert.equal(fetchImpl.calls.at(-1).headers.authorization, "Bearer sekrit"); + await promptRemote("research", "hi", { fetchImpl, env: {} }); + assert.equal(fetchImpl.calls.at(-1).headers.authorization, undefined); + }); +}); + +/* ------------------------------------------------------------ wait, cancel */ + +test("waiting on an a2a member polls tasks/get until it stops working", async () => { + await withHerdDir(async () => { + addRemote("research", "https://example.com", { kind: "a2a" }); + let polls = 0; + const fetchImpl = stubFetch(({ body }) => { + if (body.method === "message/send") return { body: a2aTask("working") }; + polls++; + return { body: polls >= 3 ? a2aTask("completed", "done at last") : a2aTask("working") }; + }); + await promptRemote("research", "go", { fetchImpl }); + const result = await waitRemote("research", ["done"], { fetchImpl, intervalMs: 1, sleep: async () => {} }); + assert.equal(result.outcome, "matched"); + assert.equal(result.state, "done"); + assert.equal(readRemote("research"), "done at last"); + }); +}); + +test("input-required is what a remote asking for help looks like to `wait`", async () => { + await withHerdDir(async () => { + addRemote("research", "https://example.com", { kind: "a2a" }); + const fetchImpl = stubFetch(({ body }) => + ({ body: body.method === "message/send" ? a2aTask("working") : a2aTask("input-required", "which environment?") })); + await promptRemote("research", "deploy it", { fetchImpl }); + const result = await waitRemote("research", ["blocked"], { fetchImpl, intervalMs: 1, sleep: async () => {} }); + assert.equal(result.state, "blocked"); + }); +}); + +test("cancelling is best effort, and a run endpoint says so plainly", async () => { + await withHerdDir(async () => { + addRemote("deployed", "https://example.com", { kind: "run" }); + const refused = await cancelRemote("deployed"); + assert.equal(refused.ok, false); + assert.match(String(refused.error.message), /nothing to cancel/); + + addRemote("research", "https://example.com", { kind: "a2a" }); + const fetchImpl = stubFetch(({ body }) => + ({ body: body.method === "message/send" ? a2aTask("working") : a2aTask("canceled") })); + await promptRemote("research", "go", { fetchImpl }); + const cancelled = await cancelRemote("research", { fetchImpl }); + assert.equal(cancelled.ok, true); + assert.equal(cancelled.a2aState, "canceled"); + }); +}); + +test("discovery looks where the spec says it does", () => { + assert.equal(cardUrl("https://agents.do-ai.run/w/prod/"), "https://agents.do-ai.run/w/prod/.well-known/agent-card.json"); +}); + +test("an RPC error comes back as a failure, not as a task", async () => { + await withHerdDir(async () => { + addRemote("research", "https://example.com", { kind: "a2a" }); + const fetchImpl = stubFetch(() => ({ body: { jsonrpc: "2.0", id: "1", error: { code: -32601, message: "Method not found" } } })); + const sent = await promptRemote("research", "go", { fetchImpl }); + assert.equal(sent.ok, false); + assert.match(String(sent.error.message), /Method not found/); + assert.equal(remoteStatus("research").state, "unknown"); + }); +}); diff --git a/test/herd-serve.test.mjs b/test/herd-serve.test.mjs new file mode 100644 index 0000000..ad8c3c9 --- /dev/null +++ b/test/herd-serve.test.mjs @@ -0,0 +1,319 @@ +// The A2A surface: the auth that has no off switch, the card that does not lie +// about what it can do, and the vocabulary mismatch that goes in metadata +// rather than into a state nobody meant. +import test from "node:test"; +import assert from "node:assert/strict"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; + +import { + A2A_PROTOCOL_VERSION, RPC_ERRORS, a2aState, bearer, createAuth, createHerdServer, + exposable, handleRpc, herdCard, messageText, sessionCard, taskToA2a, +} from "../src/herd-serve.mjs"; +import { endTask, readTasks, startTask } from "../src/herd-tasks.mjs"; + +async function withHerdDir(fn) { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), "moshcode-serve-test-")); + const previous = process.env.MOSHCODE_HERD_DIR; + process.env.MOSHCODE_HERD_DIR = dir; + try { return await fn(dir); } + finally { + if (previous === undefined) delete process.env.MOSHCODE_HERD_DIR; + else process.env.MOSHCODE_HERD_DIR = previous; + fs.rmSync(dir, { recursive: true, force: true }); + } +} + +const session = (extra = {}) => ({ + name: "api", engine: "claude", state: "idle", authority: "screen", cwd: "/x/api", + alive: true, exited: false, kind: "local", ...extra, +}); + +const rpc = (method, params) => ({ jsonrpc: "2.0", id: 1, method, params }); + +const harness = (rows, { screenText = "$ ", sent = { ok: true } } = {}) => ({ + member: rows[0]?.name, + sessions: () => rows, + prompt: () => sent, + interrupt: () => ({ ok: true }), + screen: () => screenText, + now: () => 1000, +}); + +/* ---------------------------------------------------------------- the card */ + +test("the card declares the things it cannot do as off", () => { + // A card that claimed streaming would be a client hanging on a stream that + // never opens. These flags are false because they are false. + const card = sessionCard(session(), { base: "http://127.0.0.1:7683" }); + assert.equal(card.protocolVersion, A2A_PROTOCOL_VERSION); + assert.equal(card.capabilities.streaming, false); + assert.equal(card.capabilities.pushNotifications, false); + assert.equal(card.supportsAuthenticatedExtendedCard, false); + assert.equal(card.preferredTransport, "JSONRPC"); + assert.equal(card.url, "http://127.0.0.1:7683/api/"); + assert.deepEqual(card.defaultInputModes, ["text/plain"]); + assert.deepEqual(card.security, [{ moshcode: [] }]); +}); + +test("the herd's own card lists its members as skills", () => { + const card = herdCard([session(), session({ name: "web", engine: "codex" })], { base: "http://x" }); + assert.deepEqual(card.skills.map((s) => s.id), ["api", "web"]); + assert.match(card.skills[0].description, /\/api\//, "a client cannot find where to address it"); +}); + +test("an autonomous session is not on the protocol surface unless asked for", () => { + // An engine with approvals bypassed, plus a network endpoint taking prompts, + // is prompt injection reaching an agent already told not to ask. + assert.equal(exposable(session({ agent: true })), false); + assert.equal(exposable(session({ agent: true }), { exposeAutonomous: true }), true); + assert.equal(exposable(session()), true); + // A remote is somebody else's to serve, not ours to re-export. + assert.equal(exposable(session({ kind: "remote" })), false); +}); + +/* -------------------------------------------------------------- the states */ + +test("idle and unknown round down to working, never up to input-required", () => { + // Rounding up would page a human for a session with nothing to say, every + // time it went quiet. + assert.equal(a2aState("blocked"), "input-required"); + assert.equal(a2aState("working"), "working"); + assert.equal(a2aState("done"), "completed"); + assert.equal(a2aState("idle"), "working"); + assert.equal(a2aState("unknown"), "working"); +}); + +test("a finished task is completed whatever the session went back to being", () => { + // The idle→working rounding is about a session. Applying it to a task with an + // outcome and an artifact leaves a client polling a job that finished. + const done = taskToA2a({ + id: "t-1", session: "api", text: "go", submitted: 1, endedAt: 9, durationMs: 8, + transitions: [], status: "closed", state: "idle", artifact: "the answer", + }); + assert.equal(done.status.state, "completed"); + assert.equal(done.artifacts[0].parts[0].text, "the answer"); + assert.equal(done.metadata["sh.moshcode.herd"].state, "idle", "the honest state is still on the record"); +}); + +test("a task that ended by asking is input-required, not completed", () => { + const asked = taskToA2a({ + id: "t-1", session: "api", text: "go", submitted: 1, endedAt: 9, + transitions: [], status: "closed", state: "blocked", artifact: "which environment?", + }); + assert.equal(asked.status.state, "input-required"); +}); + +test("an open task reports what the session is doing now", () => { + const open = taskToA2a({ + id: "t-1", session: "api", text: "go", submitted: 1, transitions: [], status: "open", state: "working", artifact: null, + }, { live: "blocked" }); + assert.equal(open.status.state, "input-required"); + assert.deepEqual(open.artifacts, []); +}); + +/* --------------------------------------------------------------- the verbs */ + +test("message/send types the prompt and hands back a task id", async () => { + await withHerdDir(async () => { + const typed = []; + const answer = await handleRpc(rpc("message/send", { + message: { kind: "message", role: "user", parts: [{ kind: "text", text: "port the auth routes" }] }, + }), { ...harness([session()]), prompt: (name, text) => { typed.push([name, text]); return { ok: true }; } }); + + assert.deepEqual(typed, [["api", "port the auth routes"]]); + assert.equal(answer.result.kind, "task"); + // Returned before the engine has answered, on purpose: the task model + // exists so a client polls rather than holding a socket for half an hour. + assert.equal(answer.result.status.state, "working"); + const [task] = readTasks("api"); + assert.equal(task.text, "port the auth routes", "the protocol did not mint a ledger task"); + }); +}); + +test("a prompt that could not be typed is an error AND a closed task", async () => { + // A ledger that only records successful work cannot answer what went wrong. + await withHerdDir(async () => { + const answer = await handleRpc(rpc("message/send", { message: { parts: [{ kind: "text", text: "hi" }] } }), + harness([session()], { sent: { ok: false, error: new Error("no such pane") } })); + assert.equal(answer.error.code, RPC_ERRORS.internal.code); + const [task] = readTasks("api"); + assert.equal(task.status, "closed"); + assert.match(task.artifact, /no such pane/); + }); +}); + +test("a message with no text part is refused rather than sent as nothing", async () => { + await withHerdDir(async () => { + const answer = await handleRpc(rpc("message/send", { message: { parts: [{ kind: "file" }] } }), harness([session()])); + assert.equal(answer.error.code, RPC_ERRORS.invalidParams.code); + }); +}); + +test("message/send needs a member; task verbs do not, because ids are herd-wide", async () => { + await withHerdDir(async () => { + const rows = [session()]; + const unaddressed = await handleRpc(rpc("message/send", { message: { parts: [{ kind: "text", text: "x" }] } }), + { ...harness(rows), member: null }); + assert.equal(unaddressed.error.code, RPC_ERRORS.invalidParams.code); + + const id = startTask("api", "earlier work"); + endTask("api", id, { state: "done", artifact: "done then" }); + const got = await handleRpc(rpc("tasks/get", { id }), { ...harness(rows), member: null }); + assert.equal(got.result.id, id); + }); +}); + +test("tasks/get closes a task whose session has already stopped", async () => { + // Without this the only thing that finishes a task is the watcher, and an + // A2A client — whose whole protocol is send-then-poll — sits on `working` + // forever against a herd where nobody happened to run one. + await withHerdDir(async () => { + const id = startTask("api", "go", { screen: "$ " }); + const answer = await handleRpc(rpc("tasks/get", { id }), + harness([session({ state: "idle" })], { screenText: "$ go\nthe output\n$ " })); + assert.equal(answer.result.status.state, "completed"); + assert.match(answer.result.artifacts[0].parts[0].text, /the output/); + assert.equal(readTasks("api")[0].status, "closed"); + }); +}); + +test("a task in a session this server does not expose does not exist here", async () => { + await withHerdDir(async () => { + const id = startTask("secret", "go"); + const answer = await handleRpc(rpc("tasks/get", { id }), { ...harness([session()]), member: null }); + assert.equal(answer.error.code, RPC_ERRORS.taskNotFound.code); + }); +}); + +test("cancel interrupts the work without ending the member", async () => { + // An A2A task is a unit of work inside a member, and the member is something + // a person attached to five minutes ago. Ending it is `moshcode kill`, which + // is a decision rather than a protocol call. + await withHerdDir(async () => { + let interrupted = 0; + const id = startTask("api", "sleep forever", { screen: "$ " }); + const answer = await handleRpc(rpc("tasks/cancel", { id }), { + ...harness([session({ state: "working" })], { screenText: "$ sleep\n^C\n$ " }), + interrupt: () => { interrupted++; return { ok: true }; }, + }); + assert.equal(interrupted, 1); + assert.equal(answer.result.status.state, "canceled"); + assert.equal(readTasks("api")[0].status, "closed"); + }); +}); + +test("a finished task cannot be cancelled, and says which error that is", async () => { + await withHerdDir(async () => { + const id = startTask("api", "go"); + endTask("api", id, { state: "done", artifact: "" }); + const answer = await handleRpc(rpc("tasks/cancel", { id }), harness([session()])); + assert.equal(answer.error.code, RPC_ERRORS.taskNotCancelable.code); + }); +}); + +test("an unknown task and an unknown method are distinct errors", async () => { + await withHerdDir(async () => { + const missing = await handleRpc(rpc("tasks/get", { id: "t-nope" }), harness([session()])); + assert.equal(missing.error.code, RPC_ERRORS.taskNotFound.code); + const unsupported = await handleRpc(rpc("message/stream", {}), harness([session()])); + assert.equal(unsupported.error.code, RPC_ERRORS.methodNotFound.code); + assert.equal(unsupported.error.data, "message/stream"); + }); +}); + +test("a dead session takes no prompts", async () => { + await withHerdDir(async () => { + const answer = await handleRpc(rpc("message/send", { message: { parts: [{ kind: "text", text: "x" }] } }), + harness([session({ exited: true })])); + assert.equal(answer.error.code, RPC_ERRORS.invalidParams.code); + }); +}); + +/* ----------------------------------------------------------------- the auth */ + +test("text parts only, per the scope note", () => { + assert.equal(messageText({ parts: [{ kind: "text", text: "a" }, { kind: "file", file: {} }] }), "a"); + assert.equal(messageText({}), ""); + assert.equal(messageText(null), ""); +}); + +test("a bearer token is read from the header, or is absent", () => { + assert.equal(bearer({ headers: { authorization: "Bearer abc" } }), "abc"); + assert.equal(bearer({ headers: { authorization: "bearer abc" } }), "abc"); + assert.equal(bearer({ headers: {} }), ""); + assert.equal(bearer({}), ""); +}); + +test("a verified token is cached, and a rejected one is not", async () => { + let asked = 0; + const auth = createAuth({ verify: async (_api, token) => { asked++; return token === "good" ? "a@b.c" : null; } }); + assert.equal(await auth.check("good"), "a@b.c"); + assert.equal(await auth.check("good"), "a@b.c"); + assert.equal(asked, 1, "a polling client would become a load test on the app"); + assert.equal(await auth.check("bad"), null); + assert.equal(await auth.check("bad"), null); + assert.equal(asked, 3, "a rejection must not be cached as a rejection forever either"); +}); + +test("a minted credential is accepted without asking the app again", async () => { + let asked = 0; + const auth = createAuth({ verify: async () => { asked++; return "a@b.c"; } }); + const credential = auth.mint("a@b.c"); + assert.equal(await auth.check(credential), "a@b.c"); + assert.equal(asked, 0); +}); + +test("there is no unauthenticated request, loopback included", async () => { + // message/send is keystrokes into a real pty, which is strictly more + // dangerous than a browser terminal that at least shows you what it is doing. + await withHerdDir(async () => { + const auth = createAuth({ verify: async () => null }); + const server = createHerdServer({ auth, sessions: () => [session()], base: "http://127.0.0.1:0" }); + await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve)); + const port = server.address().port; + try { + for (const [path, options] of [ + ["/.well-known/agent-card.json", {}], + ["/api/.well-known/agent-card.json", {}], + ["/api/", { method: "POST", body: JSON.stringify(rpc("tasks/get", { id: "t-1" })) }], + ["/auth", { method: "POST" }], + ]) { + const res = await fetch(`http://127.0.0.1:${port}${path}`, options); + assert.equal(res.status, 401, `${path} answered ${res.status} without a credential`); + assert.match(res.headers.get("www-authenticate") || "", /Bearer/); + } + } finally { server.close(); } + }); +}); + +test("a 404 is not a way to ask which session names exist", async () => { + await withHerdDir(async () => { + const auth = createAuth({ verify: async () => null }); + const server = createHerdServer({ auth, sessions: () => [session()], base: "http://127.0.0.1:0" }); + await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve)); + const port = server.address().port; + try { + const res = await fetch(`http://127.0.0.1:${port}/does-not-exist/.well-known/agent-card.json`); + assert.equal(res.status, 401, "an unauthenticated caller learned a name does not exist"); + } finally { server.close(); } + }); +}); + +test("an oversized body is refused before it is parsed", async () => { + await withHerdDir(async () => { + const auth = { mint: () => "x", check: async () => "a@b.c" }; + const server = createHerdServer({ auth, sessions: () => [session()], base: "http://127.0.0.1:0" }); + await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve)); + const port = server.address().port; + try { + const res = await fetch(`http://127.0.0.1:${port}/api/`, { + method: "POST", + headers: { authorization: "Bearer good", "content-type": "application/json" }, + body: "x".repeat(2 * 1024 * 1024), + }).catch(() => ({ status: 413 })); + assert.equal(res.status, 413); + } finally { server.close(); } + }); +}); diff --git a/test/herd-tasks.test.mjs b/test/herd-tasks.test.mjs new file mode 100644 index 0000000..a6c7928 --- /dev/null +++ b/test/herd-tasks.test.mjs @@ -0,0 +1,214 @@ +// The task ledger: what it records, what it refuses to invent, and the caps +// that keep an append-only file from being a disk-eater with a delay on it. +import test from "node:test"; +import assert from "node:assert/strict"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; + +import { + MAX_ARTIFACT_CHARS, compact, endTask, findTask, ledgerSessions, mintTaskId, openTask, + readLog, readTasks, recordTransition, screenDelta, startTask, stats, +} from "../src/herd-tasks.mjs"; + +function withHerdDir(fn) { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), "moshcode-tasks-test-")); + const previous = process.env.MOSHCODE_HERD_DIR; + process.env.MOSHCODE_HERD_DIR = dir; + try { return fn(dir); } + finally { + if (previous === undefined) delete process.env.MOSHCODE_HERD_DIR; + else process.env.MOSHCODE_HERD_DIR = previous; + fs.rmSync(dir, { recursive: true, force: true }); + } +} + +/* --------------------------------------------------------------------- ids */ + +test("task ids are herd-wide, so an id names one task", () => { + // Per-session ordinals would make `herd task t-01` mean one task per member, + // and an ambiguous handle is not a handle. + withHerdDir(() => { + const a = startTask("api", "one"); + const b = startTask("web", "two"); + assert.notEqual(a, b); + assert.equal(findTask(a).session, "api"); + assert.equal(findTask(b).session, "web"); + }); +}); + +test("minting an id never blocks", () => { + withHerdDir((dir) => { + // A stale lock is what a crashed fan-out leaves behind. It must cost an id, + // not a hang. + fs.mkdirSync(path.join(dir, "tasks"), { recursive: true }); + fs.writeFileSync(path.join(dir, "tasks", "seq.lock"), ""); + const started = Date.now(); + const id = mintTaskId({ now: Date.now() + 10_000 }); + assert.ok(id.startsWith("t-")); + assert.ok(Date.now() - started < 2000, "minting an id waited on a lock"); + }); +}); + +/* ---------------------------------------------------------------- recording */ + +test("a prompt is recorded before it is known to have worked", () => { + // A ledger that only records successful work cannot answer the question + // anybody actually asks it. + withHerdDir(() => { + const id = startTask("api", "port the auth routes", { now: 1000 }); + const [task] = readTasks("api"); + assert.equal(task.id, id); + assert.equal(task.text, "port the auth routes"); + assert.equal(task.submitted, 1000); + assert.equal(task.status, "open"); + }); +}); + +test("an open task is reported open rather than given an invented outcome", () => { + withHerdDir(() => { + startTask("api", "run the migration"); + const [task] = readTasks("api"); + assert.equal(task.status, "open"); + assert.equal(task.artifact, null); + assert.equal(task.durationMs, null); + }); +}); + +test("transitions are attributed to the open task and survive without one", () => { + withHerdDir(() => { + recordTransition("api", "idle"); // nobody prompted; still history + const id = startTask("api", "go"); + recordTransition("api", "working", { id }); + recordTransition("api", "blocked", { id, kind: "menu" }); + const [task] = readTasks("api"); + assert.deepEqual(task.transitions.map((t) => t.state), ["working", "blocked"]); + assert.equal(task.transitions.at(-1).kind, "menu"); + assert.equal(readLog("api").length, 4, "the unbound transition was lost"); + }); +}); + +test("repeating the current state is not a transition", () => { + // A `--wait` prompt runs two waits back to back and the watcher is looking at + // the same session anyway. Without this, one prompt writes `idle` three + // times and the task detail shows two rows that lasted zero seconds. + withHerdDir(() => { + const id = startTask("api", "go"); + assert.equal(recordTransition("api", "working", { id }), true); + assert.equal(recordTransition("api", "working", { id }), false); + assert.equal(recordTransition("api", "idle", { id }), true); + const [task] = readTasks("api"); + assert.deepEqual(task.transitions.map((t) => t.state), ["working", "idle"]); + }); +}); + +test("closing a task records its outcome, its output and how long it took", () => { + withHerdDir(() => { + const id = startTask("api", "go", { now: 1000 }); + endTask("api", id, { state: "done", artifact: "the answer", ts: 5000 }); + const [task] = readTasks("api"); + assert.equal(task.status, "closed"); + assert.equal(task.state, "done"); + assert.equal(task.artifact, "the answer"); + assert.equal(task.durationMs, 4000); + assert.equal(openTask("api"), null); + }); +}); + +test("an oversized artifact keeps its tail and admits it was cut", () => { + // The answer is the last thing an agent printed; the first 8KB of a long run + // is the part you already watched. + withHerdDir(() => { + const id = startTask("api", "go"); + const huge = `${"x".repeat(MAX_ARTIFACT_CHARS + 500)}THE ANSWER`; + endTask("api", id, { artifact: huge }); + const [task] = readTasks("api"); + assert.equal(task.truncated, true); + assert.equal(task.artifactChars, huge.length); + assert.ok(task.artifact.endsWith("THE ANSWER"), "the tail was not the part that was kept"); + assert.equal(task.artifact.length, MAX_ARTIFACT_CHARS); + }); +}); + +/* ------------------------------------------------------------------ deltas */ + +test("the artifact is what appeared after the prompt, not the whole screen", () => { + const before = "$ ls\nfile.txt\n$"; + const after = "$ ls\nfile.txt\n$ echo hi\nhi\n$"; + assert.equal(screenDelta(before, after), "echo hi\nhi\n$"); +}); + +test("a repaint that scrolled the baseline away returns the screen, not nothing", () => { + // An empty artifact would read as "the agent said nothing", which is a lie + // about a full-screen engine that redrew. + assert.equal(screenDelta("$ ls\nfile.txt", "a totally different screen"), "a totally different screen"); +}); + +test("no baseline means everything is new", () => { + assert.equal(screenDelta("", "hello"), "hello"); +}); + +/* ------------------------------------------------------------------- stats */ + +test("blocked time is counted, because it is the number with a name", () => { + withHerdDir(() => { + const id = startTask("api", "go", { now: 0 }); + recordTransition("api", "working", { id, ts: 1000 }); + recordTransition("api", "blocked", { id, ts: 3000 }); + endTask("api", id, { state: "done", ts: 9000 }); + const totals = stats("api", { now: 10_000 }).totals; + assert.equal(totals.working, 2000); + assert.equal(totals.blocked, 6000, "human latency is the whole point of this figure"); + assert.equal(stats("api", { now: 10_000 }).tasks, 1); + }); +}); + +test("a session with no history reports nothing rather than throwing", () => { + withHerdDir(() => { + assert.deepEqual(readTasks("nobody"), []); + assert.deepEqual(readLog("nobody"), []); + assert.equal(stats("nobody").tasks, 0); + assert.equal(findTask("t-99"), null); + assert.deepEqual(ledgerSessions(), []); + }); +}); + +/* -------------------------------------------------------------- resilience */ + +test("one unparseable line loses that line, not the history above it", () => { + withHerdDir((dir) => { + const id = startTask("api", "go"); + endTask("api", id, { state: "done", artifact: "fine" }); + const file = path.join(dir, "tasks", "api.jsonl"); + fs.appendFileSync(file, '{"e":"state","id":"t-9",\n'); // a torn last write + const tasks = readTasks("api"); + assert.equal(tasks.length, 1); + assert.equal(tasks[0].artifact, "fine"); + }); +}); + +test("the ledger is owner-only, because it holds what the engine said", () => { + // The manifest's reason, one step harder: this records output, and output + // regularly carries secrets the user never typed. + withHerdDir((dir) => { + startTask("api", "go"); + const mode = fs.statSync(path.join(dir, "tasks", "api.jsonl")).mode & 0o777; + assert.equal(mode, 0o600); + }); +}); + +test("retention keeps whole tasks, newest first", () => { + // A ledger cut mid-task shows a submission with no outcome, which reads as an + // agent that never answered rather than as a file that was trimmed. + withHerdDir(() => { + for (let i = 0; i < 12; i++) { + const id = startTask("api", `task ${i}`); + endTask("api", id, { state: "done", artifact: `answer ${i}` }); + } + compact("api", { maxTasks: 5, maxBytes: 10 * 1024 * 1024 }); + const tasks = readTasks("api"); + assert.equal(tasks.length, 5); + assert.equal(tasks.at(-1).text, "task 11"); + for (const task of tasks) assert.equal(task.status, "closed", "a task was cut in half"); + }); +});