diff --git a/.upstreamer/coverage-floor.txt b/.upstreamer/coverage-floor.txt index 92a5611..32b76f0 100644 --- a/.upstreamer/coverage-floor.txt +++ b/.upstreamer/coverage-floor.txt @@ -1 +1 @@ -72.0 +74.0 diff --git a/.upstreamer/eval-report.md b/.upstreamer/eval-report.md index 6e39cb4..36eab9d 100644 --- a/.upstreamer/eval-report.md +++ b/.upstreamer/eval-report.md @@ -1,296 +1,547 @@ -PASS WITH WARNINGS - -# Eval Report — go-agent, @openrouter/agent 0.7.2 → 0.8.0 (Round 4) - -Fresh review context. No prior reasoning trusted — every claim below was -independently verified by reading upstream source, reading the port's code, -and writing/running throwaway probes against `StateAccessor` (not the raw -`State` pointer). Probes were deleted before finishing; `git status` before -and after this review is identical except for this report file. - -## Headline - -The Round-3 finding (an approval/HITL resume that itself re-pauses never -called `StateAccessor.Save`, so a `StateAccessor`-only caller could reload -stale state and re-execute an already-approved tool) **is genuinely fixed**, -and — going beyond just re-checking that one line — I traced every pause -path in `run()` and independently confirmed via fresh probes that the two -sibling pause boundaries the task specifically flagged as at-risk -(`awaiting_hitl` and `awaiting_client_tools`), plus the mixed-approval pause, -**all correctly persist via `StateAccessor.Save`** through a different, -independent code path (a `break` out of the main loop that falls through to -an unconditional save at the end of `run()`, rather than a `return`). - -While hunting for "the next place a similar bug could hide" (as the task -explicitly asked), I found one real, reproducible, but narrower-scope gap: -**go-agent does not persist state incrementally per model turn the way -upstream does** (upstream's `saveResponseToState`/`saveToolResultsToState` -save after every turn; go-agent only saves at the very end of `run()` or at -an explicit pause). A `StateAccessor`-only caller who hits a genuine -mid-run turn-level error (network failure, rate limit, etc.) on turn N+1 -loses visibility into turn N's already-executed tool work in the accessor, -even though that tool's side effect already happened. This is real and -worth fixing, but — unlike the Round-3 bug — it is (a) triggered by an -exceptional/error condition outside the documented pause/resume contract, -not the normal, expected, celebrated approval-pause path; (b) pre-existing -since the very first port (confirmed present in upstream's 0.7.2-era source -too, so it predates this whole 0.8.0 delta, not just this round); and (c) -not a violation of eval.md's actual enumerated "State" requirement, which is -scoped to "State survives a pause/resume cycle mid-approval" — a bullet this -round's fix (plus my own probes of the sibling boundaries) now fully -satisfies. I'm flagging it as a warning/follow-up rather than a blocker. - -Verdict: **PASS WITH WARNINGS.** +VERDICT: FAIL + +# Port Parity Eval — go-agent @ upstream fac0029 (`@openrouter/agent` 0.10.0) + +## 1. Scope and method + +**Inputs read** + +- `.upstreamer/upstreamer.md` (contract), `.upstreamer/eval.md` (this spec). +- Upstream delta, read directly rather than via the converter's summary: + - `git -C tmp/upstreamer/upstream log --oneline 680bceb..fac0029 -- packages/agent` + (16 commits) and `diff --stat` for `packages/agent/src` (49 files, + +15018/−1065) and for `*.test.ts` (56 files). + - Per-commit diffs read in full or in relevant part for: `78c562e` (doom + loop), `75271c3` (fan-out), `231fb65` (hook-context tool call), `a629cf1` + (`getUsage`), `0efdbb0` (forced-toolChoice relaxation), `3028554` (`strict` + passthrough), `5a7ed03` (`validateFinalResponse` diagnostics), `e8d7d6d` + (async tools / unified `run()`), `8d2ed61` (tool-set), `787cbf8` (MCP + relocation), `66d7232` (`nextTurnParams.toolChoice`). + - Upstream sources compared line-by-line against their Go counterparts: + `lib/doom-loop.ts`, `lib/next-turn-params.ts`, `lib/tool-set.ts`, + `lib/tool-set-types.ts`, `lib/hooks-schemas.ts`, `lib/hooks-types.ts`, + `lib/async-params.ts`, `lib/conversation-state.ts`, `lib/reusable-stream.ts`, + `lib/stream-transformers.ts`, `lib/tool-context.ts`, `lib/tool-orchestrator.ts`, + `lib/claude-type-guards.ts`, `inner-loop/call-model.ts`, the toolChoice / + doom-loop / usage regions of `lib/model-result.ts`, and + `tests/vectors/doom-loop-fingerprints.json`. +- This repo: `doom_loop.go`, `doom_loop_engine.go`, `tool_choice.go`, + `tool_set.go`, `next_turn_params.go`, `model_result.go`, `tool.go`, + `tool_types.go`, `tool_context.go`, `tool_executor.go`, `hooks_*.go`, + `claude.go`, `conversation_state.go`, plus the new and changed `*_test.go`. +- `upstreamer-changelog.md`, `README.md`, `.upstreamer/coverage-floor.txt`. + +**Commands run** + +- `go build ./...`, `gofmt -l .`, `go vet ./...` — clean. +- `go test -count=1 ./...` — pass. `go test -count=1 -cover ./...` — 74.8%. +- `.upstreamer/scripts/verify.sh` — `=== PASS: 0 failures ===`, including + `-race`, the coverage ratchet (floor raised 72.0 → 74.0), the 24 + required-symbol presence + per-symbol coverage gates, the `go-sdk v0.5.4` + pin, the `service_tier` workaround check, and the repo-owned-files check. + One advisory note: the `steering-*` upstream test cluster has no Go mention. +- `git status --porcelain` / `git diff --stat` to isolate this run's changes: + 12 files modified, 10 added (5697 lines), `.upstreamer/state.yaml` untouched, + `.github/`, `LICENSE`, `scripts/upstream`, `.upstreamer/scripts/` untouched. + +**Method note.** I graded per-behavior, not per-test-count. Where the port and +upstream are structurally different by design (synchronous single-goroutine +loop vs. TS promise chain; `context.Context` vs `AbortSignal`; generics vs +conditional types) I looked for the *observable* consequence rather than the +shape. + +## 2. Findings + +### Blockers + +**B1 — Upstream 0.9.0's headline feature (#90, async tools / unified `run()`) +is entirely unported, and this run advances state past it.** + +- Upstream modules with **no Go counterpart at all**: + `packages/agent/src/inner-loop/resume-tool-results.ts` (372 lines), + `lib/agent-tool.ts` (365), `lib/async-tool-registry.ts` (424), + `lib/async-tools.ts` (31), `lib/tool-task.ts` (330), `lib/tool-check.ts` + (297), `lib/tool-concurrency.ts` (89), plus the async halves of + `lib/tool-executor.ts` (+382), `lib/tool-orchestrator.ts` (+184) and + `lib/model-result.ts` (+4933). +- Public surface absent from `package agent`: `resumeToolResults`, + `ToolTaskAlreadySettledError`, `AsyncToolRegistry`, `ToolTask`, + `Semaphore`/`acquireAll`, `TASK_TOOL_NAME`/`TaskToolInputSchema`, + `AgentTranscriptSource`, `isLongRunningTool`, `isDeferredHandle`, + `isUnifiedTool`, `isAgentTool`, `isToolAsyncStartedEvent`, + `isToolAsyncSettledEvent`, `getAsyncTasks`, `sendToTask`, + `queueUserMessage`, `TOOL_SET_SNAPSHOT`. +- Absent state/loop surface: `ConversationStatus` `awaiting_async_tools`, + `ConversationState.pendingAsyncTools` / `.settledAsyncCallIds`, the + `tool.async_started` / `tool.async_settled` events, and the + `CallModelInput` fields `toolTimeoutMs`, `toolConcurrency`, `asyncTools`. +- The omission *is* honestly documented (`upstreamer-changelog.md` §"Not yet + ported from 0.9.0"; `README.md` closing note), and the reasoning ("a + half-implemented version would be worse than its absence") is sound. That is + why I am not calling this dishonest. It is still a blocker, for one specific + reason: **nothing mechanical tracks it.** The contract's Required Public API + list was not amended, `verify.sh` has no gate for it, and a PASS here + advances `.upstreamer/state.yaml` to `fac0029`. From the next run onward the + delta is `fac0029..HEAD` and #90 never appears in a diff again. That is + precisely the silent-version-lag mechanism `.upstreamer/eval.md` says this + eval exists to prevent. Per the contract, `packages/agent/` is in scope and + "behavioral divergence is a bug unless it appears in the Idiomatic + Divergences section" — a deferral note in the changelog is not that section. +- Concrete user-visible consequence: a TS user writing + `tool({ lifecycle: 'background', run: … })` gets a non-blocking round with a + pending placeholder and a `tool_task_result` injection; a Go user cannot + express it at all, and a durable `deferred` workload has no representation. + +**B2 — `SessionEnd.Reason` never carries `max_turns` or `user`.** + +- File/symbol: `model_result.go:214-221` (the run-teardown `defer`), the only + `finishHooksSession` call site in the package. +- The reason is computed as exactly three values: `complete`, `error`, or + `doom_loop`. `SessionEndReasonMaxTurns` and `SessionEndReasonUser` are + declared in `hooks_schemas.go:205,207` and referenced nowhere outside their + own declaration. +- Upstream: `lib/model-result.ts` sets `sessionEndReason = 'max_turns'` on the + `shouldStopExecution()` exit (the stopWhen halt) and `sessionEndReason = + 'user'` on `checkForInterruption()`. A Go caller whose `stopWhen` halts a run + therefore observes `SessionEnd{Reason: "complete"}` where a TS caller + observes `"max_turns"` — a wrong value on a hook payload, not a missing one, + so a consumer branching on it silently takes the wrong branch. +- No test asserts `SessionEnd.Reason` for any non-doom-loop path + (`hooks_manager_test.go`, `model_result_hooks_test.go` assert `Stop`'s + `StopReasonMaxTurns` but never `SessionEndReason`). This is exactly the + failure mode the contract's Test Quality §preamble names: the symbol is + exported, the presence check passes, nothing exercises it, nothing detects + that it is wrong. Undocumented — not in Idiomatic Divergences, not in the + changelog. Pre-existing rather than delta-introduced, but the eval grades + the repo at the target commit, and hooks are an explicit FAIL area. + +### Warnings + +**W1 — A doom-loop advisor escalation can consume a caller's freshly re-armed +forced `tool_choice` that never reached the wire.** + +- Files/symbols: `tool_choice.go:132 toolChoicePolicy.commit`, + `model_result.go:315-357`, `doom_loop_engine.go:387 + takeDoomLoopEscalationOverrides`. +- Upstream guards this deliberately: `beginToolChoiceDispatch(actualToolChoice)` + canonicalizes the *dispatched* choice against the prepared one and sets + `dispatchedToolChoiceCommit = null` on mismatch, with the comment + "Engine-owned overrides (for example a forced advisor consult) deliberately + produce a different semantic key and therefore cannot commit caller state." +- Go has no such guard. `takeDoomLoopEscalationOverrides` overwrites + `dispatch.ToolChoice` with an engine-owned `allowed_tools`/`required` + advisor pin, and `m.toolChoice.commit(len(calls) > 0)` at + `model_result.go:356` then runs unconditionally. +- Reachable sequence: turn *N* — a tool's `NextTurnParams["toolChoice"]` + returns a new forced choice, so `configure()` sets `pendingCommit = + "consume"`; the same turn's response arms an `escalate` verdict. Turn *N+1* + — the advisor override replaces the caller choice on the wire, the advisor + call satisfies `len(calls) > 0`, and the caller's never-sent forced choice is + marked consumed and relaxed to `auto` from turn *N+2* on. +- No test covers an escalation coinciding with a re-armed forced choice. + +**W2 — A partially-populated `DoomLoopTextOptions` silently disables text +detection.** + +- File/symbol: `doom_loop.go:433-449` in `ResolveDoomLoopOption`. +- `if !config.Text.Enabled { text.Enabled = false }`. Go's zero value for + `Enabled bool` is `false`, so `DoomLoop: DoomLoopConfig{Text: + &DoomLoopTextOptions{MinRepeats: 6}}` turns both text detectors **off**. +- Upstream `resolveDoomLoopOption` uses `textInput.enabled ?? + DEFAULT_TEXT_OPTIONS.enabled` (default `true`), so the same config *tunes* + detection and leaves it on. Every other numeric field in the Go struct uses + a `> 0` "unset" convention; only `Enabled` inverts, and the field's doc + comment ("nil means on with defaults; set Enabled false to disable") does not + warn that supplying an object at all flips the switch. +- `TestResolveDoomLoopOption` (doom_loop_test.go:276) only covers + `&DoomLoopTextOptions{Enabled: false}`; the partial-tune case is untested. + Undocumented divergence. + +**W3 — `ActiveTools` filters the wire tools only; upstream also removes them +from the executor.** + +- Files/symbols: `model_result.go:157` (`FilterToolsByIDs` applied to + `req.Tools` only), `tool_types.go:322 FilterToolsByIDs`. +- Upstream `inner-loop/call-model.ts` computes `filteredTools` and passes it as + `tools:` to `ModelResult`, commented "so the model cannot call filtered tools + and the executor does not carry orphaned definitions." +- In Go a filtered-out tool remains in `m.input.Tools` and therefore remains + executable, so a provider that ignores `active_tools` — or a persisted + `function_call` for a since-deactivated tool — runs the tool here and is + rejected as unknown upstream. +- This is recorded in `upstreamer-changelog.md` ("Tools filtered out of a + turn's request remain resolvable, so a persisted call for one can still be + executed on resume"), so it is a documented divergence rather than a silent + one — but it inverts upstream's stated security-ish guarantee and is not in + the contract's Idiomatic Divergences section. + +**W4 — `IsClaudeStyleMessages` is a Go type assertion, not upstream's +structural heuristic.** + +- File/symbol: `claude.go:15` + `func IsClaudeStyleMessages(v any) bool { _, ok := v.([]ClaudeMessage); return ok }`. +- Upstream `lib/claude-type-guards.ts:66 isClaudeStyleMessages` is a structural + discriminator: it returns `false` for an empty array, `false` when a + non-Claude role (`system`/`developer`/`tool`) is present, `false` when no + message carries a Claude-specific block, and `true` only when it finds a + `tool_result` block, an `image` block with a `source` record, or a `tool_use` + block with a string `id`. +- Go's version returns `true` for an empty `[]ClaudeMessage` (upstream: + `false`) and `false` for JSON-decoded `[]any` Claude messages (upstream: + `true`) — i.e. the two agree on almost none of the interesting inputs. +- Upstream added `tests/unit/claude-type-guards.test.ts` (257 lines, 14 `it` + blocks) in this delta. There is **no Go test referencing + `IsClaudeStyleMessages` at all**. The source did not change in the delta, so + this is a pre-existing divergence newly test-locked upstream, and it is not + in the contract's required-API list — hence warning, not blocker. + +**W5 — `nextTurnParams` composition order and null-clearing semantics differ.** + +- File/symbol: `next_turn_params.go:ExecuteNextTurnParamsFunctions` (uses + `sortedKeys(funcs)`) and `ApplyNextTurnParamsToRequest`. +- Order: upstream `processNextTurnParamsForCall` iterates + `Object.keys(nextParams)` — declaration order. Go sorts alphabetically. When + one tool declares two interdependent keys (upstream's documented composition + guarantee: "each sees the values written by the ones before it"), the two + ports compose in different orders. Go maps have no order so *some* choice is + forced, but the divergence is undocumented and unnoted in the naming map. +- Null-clearing: upstream's `applyNextTurnParamsToRequest` maps every `null` to + `undefined` and spreads, so a function returning `null` clears **any** field. + Go only clears `toolChoice` and `topK`; a `nil` returned for `model`, + `models`, `input` or `instructions` is silently ignored + (`next_turn_params.go` `case "model"`/`"models"`/`"input"`/`"instructions"` + all require a concrete type). + +**W6 — Doom-stop is checked *after* the stopWhen branch, not before.** + +- File/symbol: `model_result.go:352` (`checkDoomLoopForResponse`, which arms + `m.doom.stop` for text/server-tool verdicts) → `:388` (stop-condition + branch, which can issue the forced-final request at `:445`) → `:468` (the + doom-stop seal). +- Upstream orders these the other way: `if (this.doomLoopStop) { await + this.sealDoomLoopStop(...); sessionEndReason = 'doom_loop'; break; }` sits + **above** `if (await this.shouldStopExecution())`, with the comment "Seal + first: this break fires BEFORE this response's tool calls execute." +- Consequence: when a text-repetition `stop` verdict arms on the same response + that trips `stopWhen`, Go executes the pending tools and spends one more + forced-final request; upstream halts immediately. The whole point of the + `stop` rung is to stop spending. Narrow (needs both to fire on one round) and + untested. + +**W7 — The tool name `task` is not reserved.** + +- Upstream `lib/tool.ts:542-547` throws + `Tool name "task" is reserved for the built-in task-interaction tool`. + `NewTool`/`MustNewTool` in `tool.go` accept it (only `shared` is guarded, if + that). Harmless today because the built-in task tool is unported (B1), but it + means code that is legal here is illegal upstream, and it will collide the + moment #90 lands. + +**W8 — `ToolExecuteContext` has no `ConversationID`.** + +- Upstream `lib/tool-context.ts` `ToolExecutionExtras` threads `signal`, + `callId` and `conversationId` onto **every** execute context, including + `lifecycle: 'sync'` tools. Go's `ToolExecuteContext` + (`tool_executor.go:16-23`) exposes `ToolCall` (so `callId` is reachable) and + relies on `context.Context` for cancellation (divergence 1, fine), but has no + conversation id, so a sync tool body cannot correlate itself to the run. + +**W9 — Persisted `ConversationState` blobs are not interchangeable with the TS +port, and `ConsumedForcedToolChoiceKey` is not a canonical key.** + +- `tool_types.go:210-237`: `ConversationState` fields carry no `json` name + tags, so the blob serializes as `{"Version":…,"Messages":…,"Status":…}` where + upstream writes `{"version":…,"messages":…,"status":…}`. `DoomLoop` and + `DoomLoopSerializedState`/`DoomLoopStreak`/`DoomLoopVerdict` *do* carry + correct camelCase tags (`doom_loop.go:186-231`), so the detector sub-blob is + wire-compatible while its envelope is not. +- `tool_choice.go:67 forcedToolChoiceKey` uses `json.Marshal(choice)` rather + than the JCS canonicalizer that `doom_loop.go` already provides, so + `consumedForcedToolChoiceKey` values do not match upstream's + `canonicalizeKeyMaterial` output either. +- The changelog's cross-port claim is scoped to doom-loop *fingerprints*, and + that claim checks out (see §4), so this is a warning about an unstated + limitation rather than a false claim. + +### Notes + +- `lib/reusable-stream.ts`'s new `findLastBuffered`, and + `stream-transformers.ts`'s new `extractCompletionFromBuffer` / + `tryExtractCompletionFromBuffer`, are not ported. They exist upstream to + recover *parked* PostModelCall telemetry when a source close trails the + terminal event. Go's loop materializes each response synchronously in + `consumeCreateResponse` before `emitPostModelCall`, so there is no parked + telemetry and no hole to recover — structurally N/A rather than missing. + `ReusableStream.IsComplete()` is present. +- `#95` (`validateFinalResponse` message detail) is diagnostics-only and lands + on a code path this port deliberately does not have; already documented in + `upstreamer-changelog.md` with a substantive reason (the go-sdk carries + `OutputText` alongside `Output`). +- `tool_context.go:toFunctionCallItem` prefers `call.RawArgs` when present; + upstream always re-serializes with `JSON.stringify`. Go's choice preserves + byte fidelity and is strictly better; behaviour is otherwise identical. +- `doom_loop_engine.go:183 evaluateDoomLoop` passes `toolInputMap(call)` as the + fallback identity unconditionally; upstream passes `null` when the resolved + key material *is* the full arguments, avoiding a guaranteed-identical retry. + Costs one extra canonicalize attempt and one extra log line on the unhashable + path; no behavioural difference. +- `ResolveLadderAction` is exported in Go but not from upstream's `index.ts`. + Extra surface, harmless; worth adding to the naming map. +- `Usage(ctx context.Context)` ignores its `ctx` parameter + (`model_result.go:1450`). +- Upstream `tool-set.test.ts` cases with no Go counterpart: "last-call-wins + semantics", "validates every name in the map before applying" + (`…WhenAll` forms), "supports override IDs and rejects duplicates" for + server tools, and the `__proto__`/`constructor` id cases (Go maps are immune + to the prototype hazard, so those are genuinely N/A). The remaining ~30 + upstream cases are type-level (`test-d.ts`-style compile assertions) and + correctly out of scope under divergence 5. + +### MCP relocation (`787cbf8`) — judgement + +**Treating it as still out of scope is defensible, and I agree with it.** The +contract's Scope section excludes `packages/mcp/` by *package identity* +("`@openrouter/mcp`. Not ported. … Adding it is a deliberate contract change"), +and it excludes upstream JS/TS packaging infrastructure. `787cbf8` moved the +same code from `packages/mcp/src/` to `packages/agent/src/mcp/` and left +`@openrouter/mcp` in place as a re-export facade; the 0.10.0 changelog entry is +explicit that this is a subpath/packaging change with an optional peer +dependency. Nothing about the MCP *behaviour* changed, and a path move upstream +cannot silently expand a downstream contract's scope — that would let an +upstream refactor conscript ~5000 lines of unrelated work. The port already +carries the MCP *discrimination* surface the contract does require (`MarkMcp`, +`IsMcpTool`, `ToolSourceMCP`). The changelog records the relocation as a no-op +here, which is the right disclosure. If MCP should now be in scope, that is a +contract amendment, not a sync side effect. + +## 3. Per-area assessment + +**Public API completeness — PASS.** All 24 contract-required symbols are +exported, reachable from `package agent`, and covered by at least one test +(`verify.sh` presence + per-symbol coverage gates both pass). New surface added +this run is correctly named per the naming map and exercised: +`DoomLoopMonitor`, `ResolveDoomLoopOption`, `ResolveLoopKeyMaterial`, +`ResolveLadderAction`, `DetectTextRepetition`, `CanonicalizeKeyMaterial`, +`FingerprintKeyMaterial`, `FingerprintToolCall`, `DoomLoopAt`/`DoomLoopOff`, +`ModelResult.DoomLoopVerdict`, `ModelResult.Usage`, `CreateToolSet` / +`MustCreateToolSet` / `ToolSet` / `ResolvedToolSnapshot.Apply`, +`CallModelInput.ActiveTools`, `IsForcedToolChoice`, `RelaxForcedToolChoice`, +`ServerToolConfig.ID`, `ToolConfig.Strict` / `.NextTurnParams` / `.LoopKey`, +`HookNameDoomLoopDetected`. New mappings that should be added to the contract's +naming table: `getUsage()` → `Usage(ctx)`, `createToolSet` → `CreateToolSet` / +`MustCreateToolSet`, `activeTools` → `ActiveTools`, `loopKey` → `LoopKey`, +`doomLoop` → `DoomLoop`, `number | false` thresholds → `DoomLoopThreshold` + +`DoomLoopAt`/`DoomLoopOff`. + +**Version honesty — MIXED.** `verify.sh` reports the tree "level with the +0.10.0 release tag (taggable)" and the changelog heading claims +"ported from `@openrouter/agent@0.9.0` and `@openrouter/agent@0.10.0`". The +body of the changelog and the README both disclose that 0.9.0's headline minor +feature is absent, so this is not a concealed overstatement — but "level with +0.10.0, taggable" is not true of behaviour, only of the required-API floor. See +B1. + +**The load-bearing loop — PASS for the sync path, INCOMPLETE overall.** +Accumulated input is preserved across turns in the correct +`base + responseItems + outputs` order (`model_result.go:538-541`), matching +upstream's `makeFollowupRequest`. `previous_response_id` is carried forward on +state (`:552`). Stop conditions, the `Stop` hook's `ForceResume`/`AppendPrompt`, +the force-resume cap, the `AllowFinalResponse` forced-final turn with +`tool_choice: "none"` and retained `tools`, and the empty-final retry are all +intact and tested. `nextTurnParams` is correctly wired *before* the input +accumulation, matching upstream. Gaps: B1 (no async lifecycles), W1, W3, W6. + +**Streaming — PASS.** Turn boundaries (`turn.start` / `response.event` / +`response.completed` / `turn.end`) are pushed in upstream's order; all six +consumer streams are completed with `m.err` in a single `defer` +(`model_result.go:200-209`) so a transport or stream error reaches every +consumer — `TestReusableStreamSubscriberReceivesCompletionError` pins the +fan-out case. Streaming tests build events with the SDK `Create*` constructors +per the contract's rule 7. `go test -race` passes, including +`TestMonitorIsSafeForConcurrentUse`. + +**State — PASS with one caveat.** `ConversationStateVersion` is 1 (unchanged +upstream), round-trips are stable, a foreign version returns +`*UnsupportedStateVersionError` and a malformed blob `*InvalidStateError`; a +version-less legacy blob normalizes to 1. New fields +(`ConsumedForcedToolChoiceKey`, `DoomLoop`) are additive within version 1, +matching upstream's decision. Doom-loop detector state survives +serialize → resume including the fan-out fingerprint set and per-call counts +(`TestFanOutEvidenceSurvivesSaveAndResume`, +`TestDoomLoopStreaksSurviveSerializeAndResume`), a `stop` verdict survives a +decision-only resume and clears on a fresh turn +(`TestStopVerdictSurvivesADecisionResumeButNotAFreshTurn`), pre-#89 blobs +restore cleanly, and a corrupt blob is ignored. Caveat: W9 (cross-port blob +shape). + +**Approval / HITL ordering — PASS.** The mixed-turn case is correct in both +places it occurs (`model_result.go:490-504` for the normal round and +`:424-434` for the stop-condition round): auto-approved calls are executed and +their outputs persisted as `UnsentToolResults` *before* `Status` flips to +`awaiting_approval`. `TestApprovalResumeReplaysFunctionCallBeforeOutput` pins +`function_call` → `function_call_output` resume order. +`ConversationStatusAwaitingClientTools` is distinct. +`TestForcedToolChoiceConsumptionPersistsAcrossApprovalResume` shows the new +toolChoice key surviving a pause. The resume-that-re-pauses save fix is still +in place (`:267-283`). + +**Hooks — FAIL.** `DoomLoopDetected` is a faithful port: payload fields +(`Detector`, `Action`, `Streak`, `Fingerprint`, `ToolName`, `ToolInput`, +`Message`) and result (`OverrideAction`, last-override-wins) match +`hooks-schemas.ts` exactly, it fires at every rung including `observe`, and +both enforced downgrades are implemented (`doom_loop_engine.go:245-249`) and +tested (`TestDoomLoopHookOverrideDowngradesAreEnforced`). Session id is still +threaded per emit; drain is still unconditional on every exit path including +the no-tools stream error path. But `SessionEnd.Reason` is wrong for two of the +five upstream values — see B2. + +**Compatibility helpers — PASS (untouched by this delta).** +`anthropic-compat.ts` / `chat-compat.ts` / `claude-constants.ts` did not change +between `680bceb` and `fac0029`, and `compat_test.go` still covers the +usage-block / `stop_reason` / `thinking` / `unsupported_content` round trip. +The one exception is W4 (`IsClaudeStyleMessages`), whose *upstream* source also +did not change but which upstream newly test-locked. + +**Test parity — MIXED.** Every behaviour this run *did* port has a Go test that +asserts an upstream-observable outcome rather than the port's internal shape: +the doom-loop fingerprint conformance vectors are embedded byte-for-byte from +`tests/vectors/doom-loop-fingerprints.json` and asserted +(`doom_loop_vectors_test.go` + `TestDoomLoopFingerprintConformanceVectors`); +`TestToolSetSnapshotDrivesTheDispatchedRequest` asserts the dispatched request, +not the snapshot struct; `TestActiveToolsFiltersWhatIsSent` and +`TestToolStrictIsPassedThroughToTheWireDefinition` assert the wire shape; +`TestForcedToolChoiceRelaxesToAutoOnFollowUpTurns` and +`TestNextTurnParamsToolChoiceReachesFollowUpRequest` assert the follow-up +request; `usage_test.go` cross-checks `Usage(ctx)` against +`SessionEnd.TotalUsage` from one snapshot. Upstream clusters with **no** Go +counterpart: `async-tool-registry`, `run-cancellation`, `steering`, +`task-tool-actions`, `task-tool-integration`, `tool-cancellation`, `tool-check` +(all #90 → B1); `claude-type-guards` (→ W4). Untested changed/ported +behaviours: W1, W2, W6, and the whole of B2. + +**Documented-vs-undocumented divergences.** Documented and acceptable: the +`DoomLoopThreshold` / `LoopKey` / `AdvisorEnabled` struct decompositions of TS +unions; `ServerToolConfig.ID` collapsing `undefined` and `''`; `ActiveTools` +resolvability (W3); the `validateFinalResponse` carve-out; the #90 deferral +(though see B1 on tracking). **Undocumented**: B2, W1, W2, W5, W6, W7, W8, W9. + +**Repo-owned files intact — PASS.** `.github/`, `LICENSE`, `scripts/upstream`, +`.upstreamer/scripts/verify.sh` are byte-identical to `HEAD`. +`.upstreamer/coverage-floor.txt` moved 72.0 → 74.0 (ratchet raised, not +lowered). `.upstreamer/state.yaml` is unmodified in the working tree — good, +the converter did not hand-advance it. `go.mod` still pins +`go-sdk v0.5.4`; the `service_tier: auto` workaround is still present at +`model_result.go:148-156` with its explanatory comment intact. No TS/JS +artifacts leaked. + +## 4. What is genuinely good here + +Worth stating plainly so the FAIL is read correctly: the doom-loop port is the +best part of this run and it is strong. The RFC 8785 canonicalizer is a real +JCS implementation (UTF-16 code-unit key ordering, lone-surrogate escaping, +ECMAScript number-to-string, a 64-level depth cap, cycle detection) rather than +`json.Marshal`, and the upstream cross-port conformance vectors are asserted +byte-for-byte — including the `1e+21` exponent form, `-0` collapse, the +`\ud800` lone surrogate, and the NFC/NFD non-normalization case. The round-set +vs. per-call streak model, the fan-out-as-one-unit scoring, the +superset-spares-the-new-call rule, the escalation budget consumed at +application time and persisted against resume-reset, and the two enforced hook +downgrades are all present and individually tested. The `RecordOptions` +`DisallowBlock` inversion (so Go's zero value means "blocking allowed") is +exactly the right way to port an inverted TS default. + +## 5. Recommended human action + +1. **Decide B1 explicitly, before `state.yaml` advances.** Two acceptable + outcomes, one unacceptable one: + - *Port #90.* It is the largest single item in the backlog and the contract + already requires `packages/agent/` wholesale. + - *Or amend the contract* — add an "Deliberately Deferred" section naming + upstream #90 with the upstream commit `e8d7d6d`, and add a `verify.sh` + gate that fails until the deferral is either resolved or re-affirmed. Then + advancing `state.yaml` is safe, because the gap survives in a place a + future run must look at. + - *Not acceptable:* advance `state.yaml` to `fac0029` with the gap recorded + only in `upstreamer-changelog.md` prose. After that the delta never shows + it again. +2. **Fix B2** — thread the exit reason out of the loop so `SessionEnd` reports + `max_turns` when `stopWhen` halts and `user` on external interruption, and + add a test asserting `SessionEnd.Reason` for each of the five values. +3. **Fix W1** — add the dispatched-vs-prepared guard: capture the effective + caller choice before `takeDoomLoopEscalationOverrides`, compare it with + `dispatch.ToolChoice`, and skip `commit` on mismatch. Test: a re-armed + `NextTurnParams` toolChoice plus an advisor escalation on the same turn. +4. **Fix W2** — make `DoomLoopTextOptions` follow upstream's `enabled ?? true`, + e.g. `Disabled bool` or `Enabled *bool`, and test the partial-tune case. +5. **Address W5–W9 and W4** as a follow-up batch, or record each in an + Idiomatic Divergences / compatibility-notes section. W5's alphabetical key + order and W9's state-blob field naming are the two most likely to surprise + someone porting a TS integration, and both are cheap to document. --- -## 1. Upstream reference re-confirmed from source - -`tmp/upstreamer/upstream/packages/agent/src/lib/model-result.ts`: - -- `saveStateSafely` (2428-2445): the single choke point that calls - `stateAccessor.save`. It is unconditional — no caller checks the resulting - status before deciding whether to save. -- `processApprovalDecisions` (2735-2881): computes `nextStatus` (which may be - `awaiting_approval`, `awaiting_hitl`, or `in_progress`) and calls - `await this.saveStateSafely(stateUpdates)` at line 2859 **before** the - `if (nextStatus !== 'in_progress') { return; }` check at 2875 — i.e. the - save happens regardless of whether the resume resolves or re-pauses. This - is the exact behavior the Round-3/4 fix needed to match. -- `persistHitlPause` (1711-1727) and `persistClientToolsPause` (1753-1769) - each call `saveStateSafely(stateUpdates)` unconditionally when a HITL or - manual-tool pause happens on the **initial** (non-resume) path — confirming - the initial pause paths, not just resumes, must persist. -- `handleApprovalCheck`'s mixed-approval-turn save (1690) is likewise - unconditional before pausing with `awaiting_approval`. -- `saveResponseToState` (721-758) and `saveToolResultsToState` (776+) are - called after **every** turn's response and after **every** tool-round's - results (call sites at lines 3003, 3155, 3200, 3246, 3286, 3304) — i.e. - upstream persists incrementally per turn, not just at pause/completion - boundaries. This is the source of the warning-level finding in §4. - -## 2. Go port control flow re-traced end to end (`model_result.go` `run()`) - -Traced every `return` and every `break` in `run()` (lines 182-498): - -- **Resume-repause branch** (227-251, this round's fix): `paused, err := - m.prepareResumeRequest(&req); if paused { ...StateAccessor.Save...; return - }`. Confirmed present, calls `Save` before `return`, mirrors upstream's - unconditional `saveStateSafely` in `processApprovalDecisions`. This is a - faithful, minimal, correctly-placed fix. -- **Main loop pause exits** (`ConversationStatusAwaitingApproval` at 380-381 - and 437-439 for the mixed-approval case; `isPausedStatus` break at 391 and - 448-449 for HITL/client-tools/interrupted) all use `break`, not `return` — - which drops out of the `for turn` loop and falls through to the - **unconditional** `StateAccessor.Save` at lines 493-497. Since none of - these pause paths early-`return`, they all reach that save. This is a - structurally different (but equally correct) mechanism than the - resume-repause fix, and it already covered these cases correctly — I did - not find a second instance of the Round-3 bug class on the *initial* pause - paths. -- `executeToolCallsForTurn` (763-822) sets - `ConversationStatusAwaitingHITL`/`ConversationStatusAwaitingClientTools` - directly on `m.state` and returns to the caller, which then `break`s — so - the same unconditional end-of-`run()` save covers both. -- Genuine mid-loop **errors** (`m.err = err; return`, e.g. `SendResponse` - failure, stream-consume failure, marshal failure) skip the end-of-`run()` - save entirely, same as upstream would skip a `saveStateSafely` call it - never reaches because the enclosing `async` function threw first — *except* - upstream also has the extra `saveResponseToState`/`saveToolResultsToState` - calls sprinkled through the loop that persist progress from **earlier, - already-completed** turns before a **later** turn's request throws. Go-agent - has no equivalent per-turn checkpoint; see §4. - -## 3. Fresh probes (written, run, deleted — not the port's own tests) - -Three throwaway tests in `probe_pause_stateaccessor_test.go`, using -`StateAccessor` end-to-end (a `memoryStateAccessor` test double, not -`ModelResult.State()`) and a **third** `CallModel` call reloading from the -same accessor to check for duplicate tool execution, exactly like the -port's own `TestStateAccessorPersistsWhenResumeRepauses`: - -- `TestProbeStateAccessorSavedOnHITLPause`: an auto tool + a HITL tool - (`OnToolCalled` returning `false`) pause the run with `awaiting_hitl`. - `accessor.saves > 0`, `accessor.state.Status == awaiting_hitl`, the - regular tool's output was already in `accessor.state.Messages`. A second - `CallModel` call reloading from the same accessor and approving the HITL - call did **not** re-run the regular tool (`regularRan == 1`). -- `TestProbeStateAccessorSavedOnAwaitingClientToolsPause`: an auto tool + an - unresolved manual tool pause with `awaiting_client_tools`. Accessor saved, - status and pending call correctly reflected, regular tool's output - persisted before the pause. -- `TestProbeStateAccessorSavedOnMixedApprovalPause`: an auto tool + an - approval-required tool pause with `awaiting_approval`. Accessor saved, the - auto tool's unsent result was persisted to the accessor **before** the - pause. A second `CallModel` call reloading from the same accessor and - approving the pending call did not re-run the auto tool, ran the danger - tool exactly once, and both outputs reached the resume request as - `function_call_output` items in the correct order. - -All three passed, including under `-race`. (My first draft of these probes -had a synchronization bug — checking counters immediately after `CallModel` -returns without blocking on `.State(ctx)`/`.Text(ctx)` first, since -`CallModel` starts `run()` in a goroutine and returns immediately. Fixed by -blocking before asserting; this was a bug in my probe, not the port.) - -A fourth, throwaway probe (`probe_midrun_error_test.go`, -`TestProbeStateAccessorSavedAfterEarlierTurnWhenLaterTurnErrors`) is the -source of the §4 warning: a tool executes successfully on turn 1, a second -call errors on turn 2 (simulated transport failure), and -`accessor.saves == 0` — turn 1's already-executed tool work never reached -the accessor. - -All probe files were deleted before finishing. `git status --porcelain` -before and after probing is identical (only `.upstreamer/eval-report.md` is -new/modified across the whole session). - -## 4. Warning: no per-turn incremental `StateAccessor` persistence - -Confirmed via probe (above) and via reading -`tmp/upstreamer/upstream/packages/agent/src/lib/model-result.ts` at both the -0.7.2 baseline (`adc7939`) and the 0.8.0 target (`680bceb`): upstream calls -`saveResponseToState`/`saveToolResultsToState` after **every** model -response and **every** tool-execution round in the main loop, not just at -pause/completion boundaries. `go-agent`'s `run()` only reaches -`StateAccessor.Save` at the very end (line 493-497, after the `for` loop -exits via `break` or falls through normally) or at the Round-4-fixed -resume-repause early return. Any hard error inside the loop (`m.err = err; -return`) skips persistence of everything accumulated in prior, already- -completed turns of the *same* run. - -Impact: a caller who relies **solely** on `StateAccessor` (never manually -re-threading `ModelResult.State()`, which the contract explicitly allows as -an alternative) and who experiences a genuine mid-run turn-level failure -after at least one earlier turn already executed a tool with a real side -effect, has no durable record of that side effect. A naive retry from the -stale accessor state could re-execute that tool a second time — the same -failure mode as the Round-3 bug, but triggered by an exceptional error -condition on an ordinary multi-turn loop, rather than by the normal, -documented, partial-approval-resume pattern the Round-3/4 fix targets. - -This predates the 0.8.0 delta (confirmed present in upstream's own 0.7.2-era -source, so it is not something this round — or even the initial port round — -was tasked with introducing or fixing as part of *this* sync), is not a -violation of eval.md's actual "State" bullet (scoped specifically to -"a pause/resume cycle mid-approval," which is satisfied — see §2-3), and does -not affect the correctness of `ModelResult.State()`/`.Text()`/etc. for a -caller who does use those directly. I'm recording it here as an actionable, -honestly-disclosed finding for a follow-up round rather than a blocker for -this one. Recommend: either add per-turn `saveResponseToState`/ -`saveToolResultsToState`-equivalent checkpoints inside the loop, or add an -explicit compatibility note documenting that `StateAccessor`-only callers -should treat a `CallModel` error as "state as of the last successful pause," -not "state as of the last successful turn." - -## 5. Full test suite - -`go test ./... -v`: 60 tests (57 port + 3 of my probes, later deleted), 1 -skip (`TestE2ESimpleResponsesCall`, requires a live API key), 0 failures. -Read the assertions directly rather than trusting names: -- `TestStateAccessorPersistsWhenResumeRepauses` (the port's own Round-4 test): - pauses on two approval-required calls, resumes approving only one (which - re-pauses), asserts `accessor.saves` increased across that re-pause, then - makes a **third** `CallModel` call from the same accessor and asserts the - first tool did not re-execute. This is real parity coverage, not a - shape-only test — it matches my own independent probes' methodology. -- `TestAwaitingClientToolsStatusForUnresolvedManualTool`, - `TestMixedRegularHITLPreservesRegularOutput`, - `TestMixedApprovalExecutesAutoToolsBeforePausing`, - `TestMixedApprovalRejectStillSendsAutoOutput`: cover the mixed-turn - ordering and manual/HITL pause requirements directly, asserting - `function_call` → `function_call_output` ordering and pre-pause - persistence, not just status strings. -- `TestHooksSessionStartEndDoNotRefireOnApprovalResume`, - `TestPostModelCallTurnTypeDistinguishesResumeFromInitial`: cover the - Round-2/Round-3 fixes; still pass, confirming no regression. -- `TestSDKStreamErrorPropagatesToConsumers`: confirms `Text`, - `FullResponsesStream`, and `ToolStream` consumers all surface a stream - error rather than hanging — required quality "Streaming" satisfied. -- `TestClaudeToFromRoundTripLossless`, - `TestFormatCompatibilityCarriesMetadataAndUnsupportedContent`: confirm the - Claude/Chat round-trip preserves metadata, reasoning, tool use, and - unsupported content — required quality "Compatibility helpers" satisfied. - -## 6. Verifier - -`.upstreamer/scripts/verify.sh`: **PASS, 0 failures** — gofmt clean, `go -build`/`go vet`/`go test` all clean, all 24 required exported symbols -present, hooks manager + versioned state serialization present, go-sdk -pinned at v0.5.4, `service_tier: auto` workaround retained, no leaked -TS/JS artifacts, LICENSE/README/go.mod/scripts/upstream all present. - -## 7. Required Qualities re-checked from scratch (not just state/approval) - -- **Public API completeness**: every symbol in the contract's Required - Public API list (`CallModel`, `NewOpenRouter`, `NewTool`/`MustNewTool`/ - `NewServerTool`, all nine `ModelResult` consumers, `CreateInitialState`/ - `AppendToMessages`/`UpdateState`/`PartitionToolCalls`, all five stop - conditions, `ToClaudeMessage`/`FromClaudeMessages`/`FromChatMessages`/ - `ToChatMessage`, `ExtractUnsupportedContent`/`HasUnsupportedContent`/ - `GetUnsupportedContentSummary`) confirmed present and exported via direct - `grep` against each file (not just the verifier's own list) — - `model_result.go`, `agent.go`, `tool.go`, `conversation_state.go`, - `stop_conditions.go`, `anthropic_compat.go`, `chat_compat.go`, - `stream_transformers.go`. -- **Version honesty**: upstream target commit `680bceb4598f228d3e2ec58e2416e4335cdff059` - has `packages/agent/package.json` version `0.8.0`, matching the - changelog's claim. `HooksManager` and the full nine-hook surface - (`PreToolUse`, `PostToolUse`, `PostToolUseFailure`, `UserPromptSubmit`, - `Stop`, `PermissionRequest`, `SessionStart`, `SessionEnd`, `PostModelCall`) - are present with typed `OnXxx` registration methods in `hooks_manager.go` — - not a hollow claim. -- **The load-bearing loop**: `TestFollowUpRequestPreservesAccumulatedInputHistory` - and `TestApprovalResumeReplaysFunctionCallBeforeOutput` confirm - `previous_response_id` carry-forward and correct accumulated-history - ordering across turns. -- **Streaming**: confirmed above (§5). -- **Approval/HITL ordering**: confirmed above (§3, §5) — mixed-turn - auto-before-pause ordering and `function_call`→`function_call_output` - replay ordering both hold, independently re-verified with my own probes, - not just by trusting the port's test names. -- **Hooks**: all nine present; `TestHooksStopForceResumeAndAppendPrompt`, - `TestHooksPermissionRequestAllowBypassesApprovalGate`/`Deny.../AskUser...`, - `TestHooksUserPromptSubmitMutatesInitialInput`/`Rejection...` each assert - the specific documented behavior, not just that a handler fired. -- **Compatibility helpers**: confirmed above (§5). -- **Divergences documented**: the six Idiomatic Divergences in - `upstreamer.md` are unaffected by this round. The Round-4-specific - changelog entry and README updates (`git diff README.md`, - `git diff upstreamer-changelog.md`) accurately describe the fix without - overclaiming — the changelog explicitly scopes the fix to "an approval/HITL - resume that itself left calls pending" and does not claim broader - incremental-persistence parity, which is honest given the §4 finding. - The §4 finding itself is **not yet documented** anywhere in - `upstreamer-changelog.md`'s Compatibility Notes — recommend adding it in a - follow-up. -- **Repo-owned files intact**: `git diff --stat HEAD -- .github LICENSE - go.mod` is empty — no changes to CI, license, or the substrate pin. - -## 8. Changelog honesty - -The new changelog bullet ("Fixed a `CallModelInput.StateAccessor` -persistence bug: an approval/HITL resume that itself left calls pending...") -accurately and narrowly describes exactly what was fixed, matches what I -verified in the code, and does not overclaim broader persistence guarantees -it doesn't provide (consistent with the §4 finding still being open). No -overclaiming found. - ---- - -## Verdict: PASS WITH WARNINGS - -The Round-4 targeted fix is real, correctly implemented, and I independently -verified — via fresh code tracing and fresh probes exercising -`StateAccessor` end-to-end (not `ModelResult.State()`) — that the same fix -class already correctly covers the two sibling pause boundaries the task -flagged as most at-risk (`awaiting_hitl`, `awaiting_client_tools`) plus the -mixed-approval pause. All Required Qualities re-checked from scratch pass. -`go test ./... -v` and `.upstreamer/scripts/verify.sh` both pass cleanly. - -One real, reproducible, but out-of-this-round's-scope finding remains open -(§4: no per-turn incremental `StateAccessor` persistence, unlike upstream) — -recorded here as an honest warning and a recommended follow-up, not a -blocker, because it predates this delta, does not violate eval.md's actual -"pause/resume cycle mid-approval" requirement (which is now fully satisfied -across all pause types), and does not affect the correctness of the -primary `ModelResult` consumer API. +## Converter addendum (written after the eval, by the porting run) + +The eval above returned **FAIL**. This addendum records what was fixed in +response and what remains, so the report stays an accurate description of the +tree it graded. + +### Fixed after the eval + +- **B2 — `SessionEnd.Reason`.** `ModelResult.sessionEndReason` now maps the + run's real outcome: `error`, `doom_loop`, `user` (interrupted), `max_turns` + (a `StopWhen` condition halting a tool-call turn, or the turn budget running + out) and `complete` only for a genuine final answer. + `SessionEndReasonMaxTurns` / `SessionEndReasonUser` are no longer dead + constants. Covered by `TestSessionEndReasonDistinguishesHowARunEnded` + (table-driven over all five reasons). +- **W — advisor escalation consuming a re-armed forced `tool_choice`.** A + dispatch carrying an engine-owned one-turn override no longer commits the + caller's forced-choice policy; `toolChoicePolicy.abandonDispatch` drops the + prepared transition instead, mirroring upstream's + `beginToolChoiceDispatch`/`commitToolChoiceDispatch` pairing. +- **W — partial `DoomLoopTextOptions` silently disabling text detection.** The + master switch is now spelled `Disabled` rather than `Enabled`, so the zero + value means "on with defaults" and `&DoomLoopTextOptions{MinRepeats: 3}` + tunes without turning the detectors off. Covered in + `TestResolveDoomLoopOption`. +- **W — `ActiveTools` not filtering the executor.** `CallModel` now narrows + `input.Tools` before both API conversion and execution registration, as + upstream does. The previous behavior (and the test asserting it) was an + invented divergence; the test now asserts upstream's: a call naming a + filtered-out tool is unresolvable and surfaces as `awaiting_client_tools`. + +`.upstreamer/scripts/verify.sh` passes after these fixes (0 failures, coverage +74.7% against a floor raised to 74.0%). + +### Remaining blocker: B1 + +Upstream 0.9.0's async tool support (#90) — the unified `run()` interface with +`background`/`deferred` lifecycles, the universal `task` tool, steering, +subagent tools, per-tool cancellation/timeouts and tool-concurrency controls — +is **not ported**, and cannot be brought in as an afterthought: it changes +pause/resume semantics (`awaiting_async_tools`), the persisted state shape +(`pendingAsyncTools`, `settledAsyncCallIds`), the tool-event sequence +(`tool.async_started` / `tool.async_settled`), and `ModelResult.cancel()` +semantics. A partial version would present upstream's API while diverging on +exactly the observable behavior this port's contract calls its parity floor, +which is worse than its documented absence. + +The eval's structural objection is correct and cannot be fixed from inside a +port run: `.upstreamer/upstreamer.md` and `.upstreamer/scripts/verify.sh` are +repo-owned and may not be rewritten by a sync, so this run cannot add a +mechanical gate that would keep the gap visible. + +### Recommended human action + +1. Leave `.upstreamer/state.yaml` at `680bceb…` (this run did **not** advance + it), so the next sync re-derives the same delta and retries the gap. +2. Amend `.upstreamer/upstreamer.md`: add the async-tool surface to the + Required Public API list and record the naming map for it, so the gap is + mechanically tracked rather than resting on a changelog note. +3. Run a dedicated port for upstream #90 alone. It is a release-sized feature + (roughly 2,000 lines of upstream source across seven new modules plus deep + changes to `model-result.ts`, `tool-executor.ts` and `tool-orchestrator.ts`) + and needs its own run and its own review. +4. The remaining warnings and notes in the eval body above are worth working + through in that run; none of them is a blocker on their own. diff --git a/README.md b/README.md index 90e0693..fa84e40 100644 --- a/README.md +++ b/README.md @@ -74,7 +74,9 @@ Additional consumers include `Response`, `FullResponsesStream`, `ReasoningStream - Generator tools use `Generate` and emit preliminary events through a yield callback before returning the final output. - Manual tools set `Manual: true` or omit executable callbacks; they are surfaced as pending calls instead of auto-executed. - HITL tools use `OnToolCalled`; returning `proceed=false` pauses the agent with pending calls. `OnResponseReceived` rewrites fresh human-supplied tool outputs before the next Responses request. -- Server tools wrap SDK `components.ResponsesRequestToolUnion` values and are passed through to OpenRouter. +- Server tools wrap SDK `components.ResponsesRequestToolUnion` values and are passed through to OpenRouter. `ServerToolConfig.ID` overrides the default `server:` tool-set id. + +`ToolConfig.Strict` is forwarded verbatim to the wire tool definition (omitted when nil, which is the provider default). `ToolConfig.NextTurnParams` computes request parameters for the turn that follows the tool's execution — `toolChoice`, `model`, `models`, `input`, `temperature`, `maxOutputTokens`, `topP`, `topK`, `instructions` — which is how a tool-search tool widens an `allowed_tools` choice without touching the `tools` array, keeping the provider's prompt-cache prefix intact. `ToolConfig.LoopKey` declares the tool's doom-loop call identity. Tool input schemas are generated from Go structs with `invopop/jsonschema`, sanitized to remove upstream-internal `~` keys, and checked before execution. Dynamic `map[string]any` remains available at JSON boundaries. @@ -84,15 +86,56 @@ Use `CreateInitialState`, `AppendToMessages`, `UpdateState`, `PartitionToolCalls Use `SerializeConversationState` / `DeserializeConversationState` for a versioned, durable-storage-friendly encoding of `ConversationState` (`ConversationStateVersion`). A version mismatch returns `*UnsupportedStateVersionError`; a malformed blob returns `*InvalidStateError` — callers get an explicit error instead of a silently misinterpreted state. +## Tool Sets And `ActiveTools` + +`CallModelInput.ActiveTools` narrows which of `Tools` are sent to the model for a given call, addressed by client-tool name (server tools always pass). `CreateToolSet` builds a declarative activation layer on top of it: + +```go +set := agent.MustCreateToolSet(agent.ToolSetOptions{Tools: []agent.Tool{listOrders, search}}) +set, _ = set.Deactivate("list_orders") +set, _ = set.ActivateWhen("refund", func(in agent.ActivationInput) bool { + return in.Context["admin"] == true +}) + +snapshot := set.Resolve(agent.ActivationInput{Context: map[string]any{"admin": true}}) +result, err := agent.CallModel(ctx, client, snapshot.Apply(agent.CallModelInput{ + Model: "openai/gpt-4o-mini", + Input: "Issue the refund.", +})) +``` + +Mutators are immutable by default: each returns a new `ToolSet` and leaves the receiver untouched. Pass `Mutable: true` to mutate in place. `DefineSituations` registers named partition overlays resolved with `ResolveSituation`. The snapshot also carries `Enabled`, `Disabled` and an exhaustive `StatusByTool` map explaining every decision. + +## Doom-Loop Detection + +Opt in with `CallModelInput.DoomLoop` (`true` for the defaults, or a `DoomLoopConfig`) to catch runs that stop making progress while continuing to spend — the model re-issuing identical tool calls round after round, repeating server-tool requests, or looping the same text. Detection is deterministic, and responds through a graduated ladder: `observe` → `steer` → `escalate` → `block` → `stop`, defaulting to observe at 2 consecutive identical rounds, block at 3 and stop at 6. + +```go +result, err := agent.CallModel(ctx, client, agent.CallModelInput{ + Model: "openai/gpt-4o-mini", + Input: "Summarize these files.", + Tools: []agent.Tool{readTool}, + DoomLoop: true, +}) +// after the run: nil unless detection stopped it +verdict := result.DoomLoopVerdict(ctx) +``` + +A repeated *fan-out* counts as one unit — `read(a), read(b), read(c)` reissued verbatim is a repeat, and at the block rung every call of the round is refused — while a round that adds new work always executes the new call. A single call repeating inside changing company is caught by its own per-call streak. Declare a tool exempt with `LoopKey: &agent.LoopKey{Exempt: true}`, narrow its identity with `Fields`, or compute it with `Fn` (returning `nil` exempts one call). + +Detector state persists in `ConversationState.DoomLoop`, so streaks survive serialize → resume; a `stop` verdict survives approve/reject resumes and clears on a fresh conversational turn. Fingerprints are RFC 8785 (JCS) canonical JSON hashed with SHA-256, byte-compatible with the TypeScript and Python ports. The `DoomLoopDetected` hook fires at every rung — including `observe`, so you can watch without changing behavior — and may override the action. + ## Stop Conditions Use `StepCountIs`, `HasToolCall`, `MaxTokensUsed`, `MaxCost`, `FinishReasonIs`, and `IsStopConditionMet`. Multiple stop conditions are ORed, matching the TypeScript package. `MaxTokensUsed` compares cumulative `total_tokens` only. +A caller-configured *forced* `tool_choice` (`required`, a specific tool, or `allowed_tools` with `mode: "required"`) relaxes to `auto` on follow-up turns once it has actually produced a tool call, including after an approval, HITL or client-tool pause, so the model can synthesize a final answer instead of being forced to call tools until the step budget runs out. `allowed_tools` keeps its tool set and only loses `mode: "required"`. A choice whose semantic value changes re-arms. + `AllowFinalResponse` is **default-on**: when a stop condition halts the loop mid-tool-call, go-agent executes the pending tool calls and issues one more request with `tool_choice: "none"` (tools stay in the request so the prompt-cache prefix survives) so the run ends with a natural-language answer. Omitting the option, or setting it to `true`, appends `agent.DefaultFinalResponseDirective` as a final user message; a non-empty string overrides that wording; `""` forbids tool calls without appending any message; `false` disables the forced final turn entirely. ## Lifecycle Hooks -`HooksManager` (`agent.NewHooksManager`) supports the nine built-in lifecycle hooks — `PreToolUse`, `PostToolUse`, `PostToolUseFailure`, `UserPromptSubmit`, `Stop`, `PermissionRequest`, `SessionStart`, `SessionEnd`, and `PostModelCall` — plus fully custom hooks via the generic `agent.On`/`agent.Emit`. Register handlers with the typed `OnXxx` methods (e.g. `manager.OnPreToolUse(...)`) and pass the manager on `CallModelInput.Hooks`: +`HooksManager` (`agent.NewHooksManager`) supports the ten built-in lifecycle hooks — `PreToolUse`, `PostToolUse`, `PostToolUseFailure`, `UserPromptSubmit`, `Stop`, `PermissionRequest`, `SessionStart`, `SessionEnd`, `PostModelCall`, and `DoomLoopDetected` — plus fully custom hooks via the generic `agent.On`/`agent.Emit`. Register handlers with the typed `OnXxx` methods (e.g. `manager.OnPreToolUse(...)`) and pass the manager on `CallModelInput.Hooks`: ```go hooks := agent.NewHooksManager() @@ -111,6 +154,10 @@ result, err := agent.CallModel(ctx, client, agent.CallModelInput{ `SessionStart`/`SessionEnd` fire once per non-resuming run (`SessionEnd` carries aggregated token usage across that run's model calls); an approval/HITL resume call is a continuation of the same session and does not get its own pair. `PostModelCall` fires once per model response, tagged `initial`/`resume`/`tool_round`/`final`/`retry`. `PreToolUse`/`PostToolUse`/`PostToolUseFailure` fire around every client-tool execution path, including during a resume. `PermissionRequest` fires before the human-approval pause and can `allow`/`deny`/`ask_user` (default) a gated call. `Stop` fires whenever a stop condition halts the loop mid-tool-call and can force a resume and/or inject a prompt. `UserPromptSubmit` fires once per non-resuming run against the initial user input and can mutate or reject it. Session identity is threaded per emit, so one `HooksManager` is safe to share across concurrent `CallModel` runs. Call `manager.Drain()` to await fire-and-forget handler work; go-agent always drains on every exit path, including no-tools error paths. +## Aggregate Usage + +`result.Usage(ctx)` reports token and cost totals across **every** model call a run made — the initial request, each tool-round follow-up, the empty-final retry, the forced final turn, and approval-resume requests. `Response(ctx)` resolves to the final round's response only, so intermediate tool-round generations are otherwise unreachable. It returns the same `SessionUsageTotals` shape as the `SessionEnd` hook, is accumulated independently of the hook system, and never returns an error. + ## Format Compatibility `ToClaudeMessage` / `FromClaudeMessages` and `ToChatMessage` / `FromChatMessages` convert between OpenRouter Responses output and Claude or Chat-style messages. Unsupported content is carried with the structured `original_type`, `data`, and `reason` shape so it can round-trip without being silently lost. @@ -118,3 +165,5 @@ result, err := agent.CallModel(ctx, client, agent.CallModelInput{ ## Notes The TypeScript package re-exports `SDKHooks`; the Go SDK keeps hooks in an internal package, so go-agent adapts the same intent with `OpenRouterOptions` middleware installed through `openrouter.WithClient`. This keeps request and response interception working for Responses calls without importing internal SDK packages. + +Upstream's async tool support (the unified `run()` interface with `background`/`deferred` lifecycles, the universal `task` tool, steering, and subagent tools) is not yet ported. Every tool here executes synchronously within its round, which is upstream's default lifecycle. See `upstreamer-changelog.md`. diff --git a/async_params.go b/async_params.go index e56158a..4d0631e 100644 --- a/async_params.go +++ b/async_params.go @@ -9,10 +9,16 @@ import ( type DynamicValue[T any] func(context.Context, TurnContext) (T, error) type CallModelInput struct { - Model string - ModelFunc DynamicValue[string] - Input any - Tools []Tool + Model string + ModelFunc DynamicValue[string] + Input any + Tools []Tool + // ActiveTools narrows which of Tools are sent to the model for this call, + // addressed by tool-set id (see ToolSetIDOf). nil means "send them all"; + // an explicitly empty non-nil slice sends none. The Tools array itself is + // unchanged, so a tool that is filtered out this turn can still be + // resolved when a persisted call for it comes back. + ActiveTools []string StopWhen []StopCondition AllowFinalResponse any // StrictFinalResponse: when true, skips the one-shot retry that would @@ -41,6 +47,11 @@ type CallModelInput struct { // Hooks accepts either a *HooksManager or an InlineHookConfig. See // ResolveHooks. nil means no hooks. Hooks any + // DoomLoop opts into doom-loop detection (upstream #73/#89). Accepts + // `true` for the defaults or a DoomLoopConfig to tune the ladder, the + // text detectors and escalation recovery; nil and `false` leave detection + // off. See ResolveDoomLoopOption. + DoomLoop any } type CallModelInputWithState = CallModelInput diff --git a/doom_loop.go b/doom_loop.go new file mode 100644 index 0000000..deb9f22 --- /dev/null +++ b/doom_loop.go @@ -0,0 +1,1816 @@ +package agent + +// Doom-loop detection for the tool-execution loop (upstream #73, #89). +// +// Catches runs that stop making progress while continuing to spend: the model +// re-issuing the same tool call with identical arguments in consecutive +// rounds, repeating identical server-tool requests, or emitting the same text +// tokens over and over. Detection is deterministic — a verdict is a pure +// function of the transcript — and responds through a configurable graduated +// ladder: observe -> steer -> escalate -> block -> stop. +// +// DoomLoopMonitor is a pure state machine over the recorded transcript. It +// holds no engine concerns (hook emission, blocking, steering, stopping live +// in ModelResult), which is what makes its behavior fully exercisable in unit +// tests and portable as a cross-port spec. + +import ( + "crypto/sha256" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "log" + "math" + "reflect" + "sort" + "strconv" + "strings" + "sync" + "unicode/utf16" + "unicode/utf8" +) + +// DoomLoopAction is one rung of the graduated response ladder. +// +// observe — record the detection and emit the DoomLoopDetected hook only. +// steer — inject corrective guidance into the conversation. +// escalate — recover by throwing more intelligence at the NEXT turn (a +// stronger model and/or a forced advisor consult). The call still +// runs. Bounded by the escalation budget. +// block — refuse the tool call and synthesize an error +// function_call_output; the error text is itself steering, +// delivered where the model looks. +// stop — halt the run before the next model request. +type DoomLoopAction string + +const ( + DoomLoopActionObserve DoomLoopAction = "observe" + DoomLoopActionSteer DoomLoopAction = "steer" + DoomLoopActionEscalate DoomLoopAction = "escalate" + DoomLoopActionBlock DoomLoopAction = "block" + DoomLoopActionStop DoomLoopAction = "stop" +) + +// DoomLoopDetectorKind identifies which detector produced a verdict. +type DoomLoopDetectorKind string + +const ( + DoomLoopDetectorToolFingerprint DoomLoopDetectorKind = "tool-fingerprint" + DoomLoopDetectorServerToolFingerprint DoomLoopDetectorKind = "server-tool-fingerprint" + DoomLoopDetectorTextRepetition DoomLoopDetectorKind = "text-repetition" + DoomLoopDetectorTextStreak DoomLoopDetectorKind = "text-streak" +) + +// DoomLoopVerdict is one doom-loop detection event. +// +// Streak is the repetition count that crossed a ladder threshold: for +// fingerprint detectors the number of consecutive rounds in which that tool +// was called with the same fingerprint *set* (so a repeated fan-out counts, +// and every call in the round reports the round's count); for text-repetition +// the number of consecutive repeats of a token block within one response; for +// text-streak the number of consecutive steps with identical assistant text. +type DoomLoopVerdict struct { + Detector DoomLoopDetectorKind `json:"detector"` + Action DoomLoopAction `json:"action"` + Streak int `json:"streak"` + // Fingerprint is the deterministic fingerprint of the repeated unit + // (call identity or text). + Fingerprint string `json:"fingerprint"` + // ToolName is present for fingerprint verdicts. + ToolName string `json:"toolName,omitempty"` + // Message is the human/model-readable explanation, used verbatim for + // block outputs and steer messages. + Message string `json:"message"` +} + +// DoomLoopCallRecord is the result of recording one tool call. +// +// DuplicateInRound is true when the same (ToolName, Fingerprint) was already +// recorded in this round — the streak did NOT increment and the caller should +// reuse the decision it applied for the first occurrence rather than +// re-emitting hooks. +type DoomLoopCallRecord struct { + Fingerprint string + Streak int + DuplicateInRound bool + Verdict *DoomLoopVerdict +} + +// DoomLoopThreshold is one ladder rung's streak threshold. +// +// Upstream spells this `number | false`, where `false` disables the rung and +// `undefined` means "use the default". Go has no such union, so the three +// states are explicit: the zero value is "unset" (take the default), Disabled +// switches the rung off, and Value carries an enabled threshold. Use +// DoomLoopAt / DoomLoopOff to construct one. +type DoomLoopThreshold struct { + Value int + Disabled bool + Set bool +} + +// DoomLoopAt enables a rung at the given streak threshold. +func DoomLoopAt(streak int) DoomLoopThreshold { + return DoomLoopThreshold{Value: streak, Set: true} +} + +// DoomLoopOff disables a rung. +func DoomLoopOff() DoomLoopThreshold { return DoomLoopThreshold{Disabled: true, Set: true} } + +// Enabled reports whether this rung can fire at all. +func (t DoomLoopThreshold) Enabled() bool { return t.Set && !t.Disabled && t.Value >= 1 } + +// Meets reports whether streak has reached this rung. Thresholds compare with +// >=, so a verdict fires on *every* record at or past a rung — escalating as +// the streak grows. +func (t DoomLoopThreshold) Meets(streak int) bool { return t.Enabled() && streak >= t.Value } + +// DoomLoopLadder holds the streak thresholds for each action. When several +// rungs are crossed the strongest wins (stop > block > escalate > steer > +// observe). +type DoomLoopLadder struct { + Observe DoomLoopThreshold + Steer DoomLoopThreshold + Escalate DoomLoopThreshold + Block DoomLoopThreshold + Stop DoomLoopThreshold +} + +// DoomLoopEscalationConfig configures what "throw more intelligence at the +// stuck turn" means for this run. At least one of Model/Advisor must be set +// for the escalate rung to do anything. +type DoomLoopEscalationConfig struct { + // Model is the slug to run the NEXT turn on, replacing the request's + // configured model for one request only. The following turn reverts. + Model string + // Advisor forces an `openrouter:advisor` consult on the next turn. + // AdvisorEnabled distinguishes "not configured" from an explicit opt-out, + // which upstream expresses as `advisor: false`; an explicit opt-out is + // not a mechanism and must read the same as no escalation config at all. + Advisor map[string]any + AdvisorEnabled *bool + // MaxEscalations caps how many times this run may escalate. Zero means + // DefaultMaxEscalations. + MaxEscalations int +} + +// DoomLoopTextOptions tunes the two text detectors. +// +// The master switch is spelled negatively on purpose. Upstream's option is +// `text: false | object`, where an object always leaves detection on; Go has +// no such union, and an `Enabled bool` would make the natural +// `&DoomLoopTextOptions{MinRepeats: 3}` silently disable the detectors +// through its zero value. With Disabled the zero value means "on with +// defaults", which is upstream's behavior for every object form. +type DoomLoopTextOptions struct { + // Disabled turns both text detectors off. + Disabled bool + // MaxPeriodTokens is the longest repeating token block to search for. + MaxPeriodTokens int + // MinRepeats is the minimum consecutive repeats of a block to count. + MinRepeats int + // MinCoveredTokens is the minimum total tokens covered by the repetition. + MinCoveredTokens int + // MaxWindowTokens bounds how many trailing tokens are scanned. + MaxWindowTokens int +} + +// DoomLoopConfig configures doom-loop detection. A zero DoomLoopConfig passed +// to ResolveDoomLoopOption uses all defaults, which is upstream's +// `doomLoop: true`. +type DoomLoopConfig struct { + Ladder DoomLoopLadder + // Text tunes text-loop detection. nil means "on with defaults"; set + // Enabled false to disable. + Text *DoomLoopTextOptions + // Escalation configures the escalate rung. nil leaves it off. + Escalation *DoomLoopEscalationConfig +} + +// DoomLoopStreak is a consecutive-repetition counter for one identity (a +// tool, or step text). This is the persisted shape. +type DoomLoopStreak struct { + Fingerprint string `json:"fingerprint"` + Streak int `json:"streak"` + // RoundFingerprints is the full fingerprint set of the tool's last round, + // present only when that round had more than one distinct call. + // Fingerprint+Streak alone cannot say WHICH set earned a count, so + // without this a fan-out streak either attached to one arbitrary member + // (refusing a lesser resumed call on first appearance) or had to be + // discarded at every save — losing block-level evidence across approval + // pauses and re-baselining on every per-turn resume. Absent for + // single-call rounds and in pre-#89 blobs, where Fingerprint fully + // describes the round. Never present on text streaks. + RoundFingerprints []string `json:"roundFingerprints,omitempty"` + // CallStreaks holds per-call streaks for the tool's last round: + // fingerprint -> number of consecutive rounds that exact call has been + // issued in. This is the evidence for the PER-CALL detector, which + // catches a call repeating inside rounds whose other members keep + // changing ([a,b], [a,c], [a,d] — the round set differs every time, but + // `a` is a 3-peat). Persisted so a repeat spanning a save/resume boundary + // keeps counting. Absent in pre-#89 blobs, where per-call evidence simply + // restarts. Never present on text streaks. + CallStreaks map[string]int `json:"callStreaks,omitempty"` +} + +// DoomLoopSerializedState is the plain-JSON detector state persisted inside +// ConversationState.DoomLoop so loop memory survives serialize -> resume: a +// resumed doom loop is still a doom loop (provided the resuming call passes +// DoomLoop again). +type DoomLoopSerializedState struct { + // Tools holds per-tool streaks keyed by tool name. Interleaved calls to + // other tools do not reset a tool's streak. + Tools map[string]DoomLoopStreak `json:"tools"` + // Text is the cross-step assistant-text streak. + Text *DoomLoopStreak `json:"text,omitempty"` + // StopVerdict is the verdict that condemned this run, when a stop action + // armed. Kept across decision-only resumes (approve/reject) so a + // condemned run stays halted; cleared by a fresh conversational turn + // (operator intervention is new information). Streaks are kept either way. + StopVerdict *DoomLoopVerdict `json:"stopVerdict,omitempty"` + // PendingSteer is steer guidance queued but not yet injected when the run + // paused. Flushed into the conversation on resume. + PendingSteer []string `json:"pendingSteer,omitempty"` + // EscalationsUsed counts how many escalation recoveries this conversation + // has consumed. Persisted so a resumed run cannot reset its budget. + EscalationsUsed int `json:"escalationsUsed,omitempty"` +} + +// LoopKeyResolutionKind discriminates a LoopKeyResolution. +type LoopKeyResolutionKind string + +const ( + // LoopKeyExempt means the tool or this call is exempt from detection. + LoopKeyExempt LoopKeyResolutionKind = "exempt" + // LoopKeyResolved means KeyMaterial is the declared call identity. + LoopKeyResolved LoopKeyResolutionKind = "key" + // LoopKeyFallback means the declaration was degenerate and the full + // arguments are used instead; Warning explains why. + LoopKeyFallback LoopKeyResolutionKind = "fallback" +) + +// LoopKeyResolution is the result of resolving a tool's LoopKey declaration +// against one call's validated arguments. +type LoopKeyResolution struct { + Kind LoopKeyResolutionKind + KeyMaterial any + Warning string +} + +// LoopKey declares a tool's doom-loop call identity. +// +// Upstream spells this `loopKey?: false | string[] | ((args) => unknown)`. +// In Go the three forms are fields: Exempt for the static opt-out, Fields for +// a subset of argument keys, and Fn for a computed identity. A nil *LoopKey +// (the absent declaration) means the full validated arguments are the +// identity. +type LoopKey struct { + // Exempt statically exempts the tool: no call of it is ever recorded. + Exempt bool + // Fields names the argument keys that make up the identity. An empty or + // wholly-absent field list would collapse every call of the tool onto one + // identity, so it falls back to the full arguments with a warning. + Fields []string + // Fn computes the identity from the call's validated arguments. Returning + // nil exempts THIS call. An error falls back to the full arguments — + // detection must never take down a run. + Fn func(args map[string]any) (any, error) +} + +// ToolWithLoopKey is implemented by tools that declare a doom-loop call +// identity. +type ToolWithLoopKey interface { + Tool + ToolLoopKey() *LoopKey +} + +// ToolLoopKeyOf returns t's LoopKey declaration, or nil when it has none. +func ToolLoopKeyOf(t Tool) *LoopKey { + if t == nil { + return nil + } + if withKey, ok := t.(ToolWithLoopKey); ok { + return withKey.ToolLoopKey() + } + return nil +} + +// ResolvedEscalationConfig is an escalation config with its budget resolved. +type ResolvedEscalationConfig struct { + Model string + Advisor map[string]any + AdvisorEnabled bool + MaxEscalations int +} + +// ResolvedDoomLoopConfig is the fully-resolved config the monitor uses. +type ResolvedDoomLoopConfig struct { + Ladder DoomLoopLadder + Text DoomLoopTextOptions + Escalation *ResolvedEscalationConfig +} + +// DefaultDoomLoopLadder observes at 2 consecutive identical rounds, blocks at +// 3 and stops at 6. Steer is off by default: blocking already delivers +// feedback in the tool output, where the model looks first. +func DefaultDoomLoopLadder() DoomLoopLadder { + return DoomLoopLadder{ + Observe: DoomLoopAt(2), + Steer: DoomLoopOff(), + Escalate: DoomLoopOff(), + Block: DoomLoopAt(3), + Stop: DoomLoopAt(6), + } +} + +// DefaultMaxEscalations is the escalation budget when the rung is enabled +// without an explicit cap. +const DefaultMaxEscalations = 2 + +// DefaultDoomLoopTextOptions returns the default text-detector tuning +// (enabled, since Disabled's zero value is false). +func DefaultDoomLoopTextOptions() DoomLoopTextOptions { + return DoomLoopTextOptions{MaxPeriodTokens: 16, MinRepeats: 4, MinCoveredTokens: 12, MaxWindowTokens: 400} +} + +// MaxCanonicalizeDepth is the deepest nesting CanonicalizeKeyMaterial +// accepts. Real tool arguments come from parsed JSON and stay shallow; +// anything deeper is hostile or buggy key material and must fail fast (with a +// catchable error) instead of overflowing the stack. +const MaxCanonicalizeDepth = 64 + +func sanitizeThreshold(value, fallback DoomLoopThreshold) DoomLoopThreshold { + if !value.Set { + return fallback + } + if value.Disabled { + return DoomLoopOff() + } + if value.Value >= 1 { + return DoomLoopAt(value.Value) + } + return fallback +} + +// ladderRung pairs a rung's name with its threshold, weakest first, for +// dead-rung analysis. +func ladderRungs(l DoomLoopLadder) []struct { + name string + threshold DoomLoopThreshold +} { + return []struct { + name string + threshold DoomLoopThreshold + }{ + {"observe", l.Observe}, + {"steer", l.Steer}, + {"escalate", l.Escalate}, + {"block", l.Block}, + {"stop", l.Stop}, + } +} + +// warnOnLadderHazards flags configurations that are accepted but probably not +// what the caller meant: +// +// - block enabled with stop disabled — blocked calls still increment the +// streak, so a model that keeps re-issuing the call produces an unbounded +// block/re-issue ping-pong bounded only by StopWhen. Explicitly allowed, +// loudly flagged. +// - dead rungs — with strongest-wins resolution, a weaker rung whose +// threshold is >= an enabled stronger rung's threshold can never fire +// (e.g. observe 5 with block 2: block wins from streak 2 on). +func warnOnLadderHazards(ladder DoomLoopLadder) { + if ladder.Block.Enabled() && !ladder.Stop.Enabled() { + log.Printf("[DoomLoop] ladder has block enabled with stop disabled: a model that keeps re-issuing a blocked call loops indefinitely (each blocked round still costs a model request). Bound the run with StopWhen, or enable the stop rung.") + } + rungs := ladderRungs(ladder) + for weak := range rungs { + if !rungs[weak].threshold.Enabled() { + continue + } + for strong := weak + 1; strong < len(rungs); strong++ { + if rungs[strong].threshold.Enabled() && rungs[weak].threshold.Value >= rungs[strong].threshold.Value { + log.Printf("[DoomLoop] ladder rung %q (%d) can never fire: stronger rung %q (%d) already wins at that streak (strongest crossed rung is applied).", + rungs[weak].name, rungs[weak].threshold.Value, rungs[strong].name, rungs[strong].threshold.Value) + break // one warning per dead rung is enough + } + } + } +} + +// ResolveDoomLoopOption normalizes the DoomLoop option into a monitor config. +// +// Accepted values mirror upstream's `boolean | DoomLoopConfig`: nil and +// `false` mean detection is off (returns nil — the SDK's default posture is +// explicit control over implicit magic); `true` and a DoomLoopConfig (or +// *DoomLoopConfig) resolve to a full config. +func ResolveDoomLoopOption(option any) *ResolvedDoomLoopConfig { + var config DoomLoopConfig + switch v := option.(type) { + case nil: + return nil + case bool: + if !v { + return nil + } + case DoomLoopConfig: + config = v + case *DoomLoopConfig: + if v == nil { + return nil + } + config = *v + default: + log.Printf("[DoomLoop] unsupported DoomLoop option of type %T; detection disabled", option) + return nil + } + + defaults := DefaultDoomLoopLadder() + ladder := DoomLoopLadder{ + Observe: sanitizeThreshold(config.Ladder.Observe, defaults.Observe), + Steer: sanitizeThreshold(config.Ladder.Steer, defaults.Steer), + Escalate: sanitizeThreshold(config.Ladder.Escalate, defaults.Escalate), + Block: sanitizeThreshold(config.Ladder.Block, defaults.Block), + Stop: sanitizeThreshold(config.Ladder.Stop, defaults.Stop), + } + warnOnLadderHazards(ladder) + + text := DefaultDoomLoopTextOptions() + if config.Text != nil { + text.Disabled = config.Text.Disabled + if config.Text.MaxPeriodTokens > 0 { + text.MaxPeriodTokens = config.Text.MaxPeriodTokens + } + if config.Text.MinRepeats > 0 { + text.MinRepeats = config.Text.MinRepeats + } + if config.Text.MinCoveredTokens > 0 { + text.MinCoveredTokens = config.Text.MinCoveredTokens + } + if config.Text.MaxWindowTokens > 0 { + text.MaxWindowTokens = config.Text.MaxWindowTokens + } + } + + // Escalation recovery is usable only when the config names at least one + // mechanism. An enabled rung without a mechanism (or a mechanism without + // the rung) is probably a config mistake — warn, and treat the rung as + // absent so verdicts fall through to weaker rungs. `advisor: false` is an + // explicit opt-out, not a mechanism: counting it would resolve a rung + // that consumes budget and announces recovery without applying any + // override. + var escalation *ResolvedEscalationConfig + if in := config.Escalation; in != nil { + advisorEnabled := in.AdvisorEnabled == nil && in.Advisor != nil + if in.AdvisorEnabled != nil { + advisorEnabled = *in.AdvisorEnabled + } + if in.Model != "" || advisorEnabled { + cap := in.MaxEscalations + if cap < 1 { + cap = DefaultMaxEscalations + } + escalation = &ResolvedEscalationConfig{Model: in.Model, MaxEscalations: cap} + if advisorEnabled { + escalation.AdvisorEnabled = true + escalation.Advisor = in.Advisor + } + if !ladder.Escalate.Enabled() { + log.Printf("[DoomLoop] escalation config provided but the escalate ladder rung is disabled; set Ladder.Escalate to a streak threshold for recovery to trigger.") + } + } + } + if escalation == nil && ladder.Escalate.Enabled() { + log.Printf("[DoomLoop] Ladder.Escalate is enabled but no escalation config (Model/Advisor) was provided; the rung is skipped and verdicts fall through to weaker rungs.") + } + + return &ResolvedDoomLoopConfig{Ladder: ladder, Text: text, Escalation: escalation} +} + +// ResolveLoopKeyMaterial resolves a tool's LoopKey declaration against one +// call's validated arguments. +// +// - nil declaration -> the full arguments object. +// - Exempt -> the tool is statically exempt. +// - Fields -> the named subset. An empty list, or a list whose +// every field is absent, would give every call of the +// tool the same identity, so it falls back to the full +// arguments with a warning. +// - Fn -> called with the arguments. A nil result exempts +// THIS call; an error falls back to the full arguments +// (detection must never take down a run). +func ResolveLoopKeyMaterial(loopKey *LoopKey, args map[string]any) LoopKeyResolution { + if loopKey == nil { + return LoopKeyResolution{Kind: LoopKeyResolved, KeyMaterial: args} + } + if loopKey.Exempt { + return LoopKeyResolution{Kind: LoopKeyExempt} + } + if loopKey.Fields != nil { + if len(loopKey.Fields) == 0 { + return LoopKeyResolution{Kind: LoopKeyFallback, KeyMaterial: args, + Warning: "LoopKey is an empty field list, which would give every call to this tool the same identity; falling back to full arguments. Use Exempt to exempt the tool."} + } + subset := map[string]any{} + for _, field := range loopKey.Fields { + if value, ok := args[field]; ok { + subset[field] = value + } + } + if len(subset) == 0 { + return LoopKeyResolution{Kind: LoopKeyFallback, KeyMaterial: args, + Warning: fmt.Sprintf("LoopKey fields [%s] are all absent from the arguments, which would give every call the same identity; falling back to full arguments.", strings.Join(loopKey.Fields, ", "))} + } + return LoopKeyResolution{Kind: LoopKeyResolved, KeyMaterial: subset} + } + if loopKey.Fn != nil { + result, err := loopKey.Fn(args) + if err != nil { + return LoopKeyResolution{Kind: LoopKeyFallback, KeyMaterial: args, + Warning: fmt.Sprintf("LoopKey returned an error (%v); falling back to full arguments", err)} + } + if result == nil { + return LoopKeyResolution{Kind: LoopKeyExempt} + } + return LoopKeyResolution{Kind: LoopKeyResolved, KeyMaterial: result} + } + return LoopKeyResolution{Kind: LoopKeyFallback, KeyMaterial: args, + Warning: "LoopKey declares neither Exempt, Fields nor Fn; falling back to full arguments"} +} + +// ErrUnhashableKeyMaterial wraps every canonicalization failure so the engine +// can recognize "fall back to the full-arguments identity" without string +// matching. +var ErrUnhashableKeyMaterial = errors.New("unhashable doom-loop key material") + +// CanonicalizeKeyMaterial produces RFC 8785 (JCS) canonical JSON: object keys +// sorted by UTF-16 code units (recursively), arrays in order, strings and +// finite numbers serialized per ECMAScript JSON.stringify (which IS the JCS +// serialization — -0 canonicalizes to 0, 1e21 to 1e+21, lone surrogates to +// \udXXX escapes). Insensitive to key insertion order. +// +// This is the cross-port contract: the TypeScript, Python and Go ports must +// produce byte-identical canonical strings for the same key material, which +// is why this is hand-written rather than delegated to encoding/json (whose +// HTML escaping, key ordering by Go string bytes, and float formatting all +// disagree with JCS). +// +// Returns an error wrapping ErrUnhashableKeyMaterial for values RFC 8785 +// cannot represent — non-finite numbers — plus cycles and nesting deeper than +// MaxCanonicalizeDepth. Real tool arguments (parsed JSON) can never trigger +// these; only computed LoopKey material can, and the engine catches the error +// and falls back to the full-arguments identity. +func CanonicalizeKeyMaterial(value any) (string, error) { + var sb strings.Builder + seen := map[uintptr]bool{} + if err := canonicalize(&sb, value, 0, seen); err != nil { + return "", err + } + return sb.String(), nil +} + +func canonicalize(sb *strings.Builder, value any, depth int, seen map[uintptr]bool) error { + if depth > MaxCanonicalizeDepth { + return fmt.Errorf("%w: nested deeper than %d levels", ErrUnhashableKeyMaterial, MaxCanonicalizeDepth) + } + if value == nil { + sb.WriteString("null") + return nil + } + switch v := value.(type) { + case bool: + if v { + sb.WriteString("true") + } else { + sb.WriteString("false") + } + return nil + case string: + sb.WriteString(canonicalJSONString(v)) + return nil + case json.Number: + f, err := v.Float64() + if err != nil { + return fmt.Errorf("%w: %v", ErrUnhashableKeyMaterial, err) + } + return canonicalizeNumber(sb, f) + case float64: + return canonicalizeNumber(sb, v) + case float32: + return canonicalizeNumber(sb, float64(v)) + case int: + return canonicalizeNumber(sb, float64(v)) + case int8: + return canonicalizeNumber(sb, float64(v)) + case int16: + return canonicalizeNumber(sb, float64(v)) + case int32: + return canonicalizeNumber(sb, float64(v)) + case int64: + return canonicalizeNumber(sb, float64(v)) + case uint: + return canonicalizeNumber(sb, float64(v)) + case uint8: + return canonicalizeNumber(sb, float64(v)) + case uint16: + return canonicalizeNumber(sb, float64(v)) + case uint32: + return canonicalizeNumber(sb, float64(v)) + case uint64: + return canonicalizeNumber(sb, float64(v)) + case map[string]any: + return canonicalizeMap(sb, v, depth, seen) + case []any: + return canonicalizeSlice(sb, v, depth, seen) + } + return canonicalizeReflect(sb, value, depth, seen) +} + +// canonicalizeReflect handles concrete Go types that are not the parsed-JSON +// shapes (a typed struct or a []string handed to LoopKey, for instance) by +// projecting them through encoding/json into the JSON data model and +// canonicalizing that. The projection is the only faithful route: JCS is +// defined over JSON values, not over Go types. +func canonicalizeReflect(sb *strings.Builder, value any, depth int, seen map[uintptr]bool) error { + rv := reflect.ValueOf(value) + switch rv.Kind() { + case reflect.Ptr, reflect.Interface: + if rv.IsNil() { + sb.WriteString("null") + return nil + } + ptr := rv.Pointer() + if rv.Kind() == reflect.Ptr { + if seen[ptr] { + return fmt.Errorf("%w: circular reference", ErrUnhashableKeyMaterial) + } + seen[ptr] = true + defer delete(seen, ptr) + } + return canonicalize(sb, rv.Elem().Interface(), depth, seen) + case reflect.Slice, reflect.Array: + items := make([]any, rv.Len()) + for i := range items { + items[i] = rv.Index(i).Interface() + } + return canonicalizeSlice(sb, items, depth, seen) + case reflect.Map: + if rv.Type().Key().Kind() != reflect.String { + return fmt.Errorf("%w: map key type %s is not a JSON object key", ErrUnhashableKeyMaterial, rv.Type().Key()) + } + entries := map[string]any{} + for _, key := range rv.MapKeys() { + entries[key.String()] = rv.MapIndex(key).Interface() + } + return canonicalizeMap(sb, entries, depth, seen) + case reflect.Struct: + // Round-trip through encoding/json so struct tags are honored, then + // canonicalize the resulting JSON data model. json.Number keeps + // integer literals exact through the hop. + b, err := json.Marshal(value) + if err != nil { + return fmt.Errorf("%w: %v", ErrUnhashableKeyMaterial, err) + } + decoder := json.NewDecoder(strings.NewReader(string(b))) + decoder.UseNumber() + var decoded any + if err := decoder.Decode(&decoded); err != nil { + return fmt.Errorf("%w: %v", ErrUnhashableKeyMaterial, err) + } + return canonicalize(sb, decoded, depth, seen) + case reflect.Complex64, reflect.Complex128, reflect.Chan, reflect.Func, reflect.UnsafePointer: + // JSON semantics: functions and symbols serialize as null. + sb.WriteString("null") + return nil + } + return fmt.Errorf("%w: unsupported Go kind %s", ErrUnhashableKeyMaterial, rv.Kind()) +} + +func canonicalizeMap(sb *strings.Builder, entries map[string]any, depth int, seen map[uintptr]bool) error { + keys := make([]string, 0, len(entries)) + for k, v := range entries { + // JSON.stringify drops undefined/function/symbol object entries. Go's + // nearest equivalent is a nil func value; nil interface entries are + // JSON null and are kept. + if isDroppedEntry(v) { + continue + } + keys = append(keys, k) + } + sortByUTF16CodeUnits(keys) + sb.WriteByte('{') + for i, k := range keys { + if i > 0 { + sb.WriteByte(',') + } + sb.WriteString(canonicalJSONString(k)) + sb.WriteByte(':') + if err := canonicalize(sb, entries[k], depth+1, seen); err != nil { + return err + } + } + sb.WriteByte('}') + return nil +} + +func canonicalizeSlice(sb *strings.Builder, items []any, depth int, seen map[uintptr]bool) error { + sb.WriteByte('[') + for i, item := range items { + if i > 0 { + sb.WriteByte(',') + } + // In arrays, JSON.stringify serializes dropped values as null. + if isDroppedEntry(item) { + sb.WriteString("null") + continue + } + if err := canonicalize(sb, item, depth+1, seen); err != nil { + return err + } + } + sb.WriteByte(']') + return nil +} + +func isDroppedEntry(v any) bool { + if v == nil { + return false // JSON null, which IS representable + } + switch reflect.ValueOf(v).Kind() { + case reflect.Func, reflect.Chan, reflect.UnsafePointer: + return true + } + return false +} + +// sortByUTF16CodeUnits sorts keys the way ECMAScript does, by UTF-16 code +// unit. This differs from Go's byte-wise string comparison for characters +// outside the BMP: a surrogate pair sorts below U+E000..U+FFFF in UTF-16 but +// above it in UTF-8. Real tool argument keys are ASCII, but the cross-port +// fingerprint contract is exact, so the comparison has to be too. +func sortByUTF16CodeUnits(keys []string) { + sort.Slice(keys, func(i, j int) bool { + return compareUTF16(keys[i], keys[j]) < 0 + }) +} + +func compareUTF16(a, b string) int { + ua, ub := utf16.Encode([]rune(a)), utf16.Encode([]rune(b)) + for i := 0; i < len(ua) && i < len(ub); i++ { + if ua[i] != ub[i] { + if ua[i] < ub[i] { + return -1 + } + return 1 + } + } + switch { + case len(ua) < len(ub): + return -1 + case len(ua) > len(ub): + return 1 + } + return 0 +} + +// canonicalJSONString serializes s the way ECMAScript JSON.stringify does: +// the two-character escapes for \b \t \n \f \r " \\, \u00XX for the remaining +// control characters, everything else literal UTF-8 — and lone surrogates as +// \udXXX escapes, which is exactly what the conformance vectors pin. +func canonicalJSONString(s string) string { + var sb strings.Builder + sb.WriteByte('"') + for i := 0; i < len(s); { + r, size := utf8.DecodeRuneInString(s[i:]) + if r == utf8.RuneError && size == 1 { + // Not valid UTF-8. Go strings can hold WTF-8-encoded lone + // surrogates (Go's decoder rejects them, JS strings carry them + // natively), so recover the code unit and emit the \udXXX escape + // JSON.stringify would. + if unit, width, ok := decodeLoneSurrogate(s[i:]); ok { + fmt.Fprintf(&sb, "\\u%04x", unit) + i += width + continue + } + // A genuinely invalid byte: emit the replacement character, which + // is what every JSON encoder does with it. + sb.WriteRune(utf8.RuneError) + i++ + continue + } + switch r { + case '"': + sb.WriteString(`\"`) + case '\\': + sb.WriteString(`\\`) + case '\b': + sb.WriteString(`\b`) + case '\f': + sb.WriteString(`\f`) + case '\n': + sb.WriteString(`\n`) + case '\r': + sb.WriteString(`\r`) + case '\t': + sb.WriteString(`\t`) + default: + if r < 0x20 { + fmt.Fprintf(&sb, "\\u%04x", r) + } else if r >= 0xD800 && r <= 0xDFFF { + fmt.Fprintf(&sb, "\\u%04x", r) + } else { + sb.WriteRune(r) + } + } + i += size + } + sb.WriteByte('"') + return sb.String() +} + +// decodeLoneSurrogate recognizes a WTF-8-encoded surrogate code point +// (ED A0 80 .. ED BF BF), which is how a lone surrogate from a JS string +// survives a trip through a Go string. +func decodeLoneSurrogate(s string) (uint16, int, bool) { + if len(s) < 3 || s[0] != 0xED { + return 0, 0, false + } + b1, b2 := s[1], s[2] + if b1 < 0xA0 || b1 > 0xBF || b2 < 0x80 || b2 > 0xBF { + return 0, 0, false + } + code := 0xD000 | (rune(b1&0x3F) << 6) | rune(b2&0x3F) + if code < 0xD800 || code > 0xDFFF { + return 0, 0, false + } + return uint16(code), 3, true +} + +// canonicalizeNumber formats f the way ECMAScript Number::toString does, +// which is the JCS number serialization: -0 becomes 0, integers up to 1e21 +// print without an exponent, and larger magnitudes use the e+NN form. +func canonicalizeNumber(sb *strings.Builder, f float64) error { + if math.IsNaN(f) || math.IsInf(f, 0) { + return fmt.Errorf("%w: non-finite number (NaN/Infinity) has no RFC 8785 representation", ErrUnhashableKeyMaterial) + } + if f == 0 { + // Collapses -0 to 0, as JSON.stringify does. + sb.WriteString("0") + return nil + } + sb.WriteString(ecmaNumberToString(f)) + return nil +} + +// ecmaNumberToString renders f per ECMAScript's Number::toString. Go's +// strconv 'g' shortest representation agrees on the digits but not on the +// exponent formatting or thresholds, so the exponent form is rebuilt here. +func ecmaNumberToString(f float64) string { + // Shortest round-trip decimal digits and decimal exponent. + mantissa := strconv.FormatFloat(f, 'e', -1, 64) + negative := false + if strings.HasPrefix(mantissa, "-") { + negative = true + mantissa = mantissa[1:] + } + parts := strings.SplitN(mantissa, "e", 2) + digits := strings.Replace(parts[0], ".", "", 1) + exp10, err := strconv.Atoi(parts[1]) + if err != nil { + return strconv.FormatFloat(f, 'g', -1, 64) + } + // n is the position of the decimal point relative to the digit string: + // value = 0.digits * 10^n, matching the ECMAScript spec's variables. + k := len(digits) + n := exp10 + 1 + + var out string + switch { + case k <= n && n <= 21: + out = digits + strings.Repeat("0", n-k) + case 0 < n && n <= 21: + out = digits[:n] + "." + digits[n:] + case -6 < n && n <= 0: + out = "0." + strings.Repeat("0", -n) + digits + default: + e := n - 1 + sign := "+" + if e < 0 { + sign = "-" + e = -e + } + if k == 1 { + out = digits + "e" + sign + strconv.Itoa(e) + } else { + out = digits[:1] + "." + digits[1:] + "e" + sign + strconv.Itoa(e) + } + } + if negative { + return "-" + out + } + return out +} + +func sha256Hex(canonical string) string { + sum := sha256.Sum256([]byte(canonical)) + return hex.EncodeToString(sum[:]) +} + +// FingerprintKeyMaterial fingerprints arbitrary key material: +// sha256(utf8(jcs(keyMaterial))), lowercase hex (64 chars). +func FingerprintKeyMaterial(keyMaterial any) (string, error) { + canonical, err := CanonicalizeKeyMaterial(keyMaterial) + if err != nil { + return "", err + } + return sha256Hex(canonical), nil +} + +// FingerprintToolCall fingerprints a tool call: +// sha256(utf8(toolName + "\n" + jcs(keyMaterial))), lowercase hex. The tool +// name participates so search({q}) and fetch({q}) with equal arguments never +// share a fingerprint. This exact construction is the cross-port contract — +// see the conformance vectors in doom_loop_test.go. +func FingerprintToolCall(toolName string, keyMaterial any) (string, error) { + canonical, err := CanonicalizeKeyMaterial(keyMaterial) + if err != nil { + return "", err + } + return sha256Hex(toolName + "\n" + canonical), nil +} + +// TextRepetitionResult describes a repeating token block at the tail of a +// response. +type TextRepetitionResult struct { + // Repeats is the consecutive repeat count, which feeds the ladder. + Repeats int + // PeriodTokens is the block length in whitespace-delimited tokens. + PeriodTokens int + // CoveredTokens is Repeats * PeriodTokens. + CoveredTokens int + // Sample is the repeating block itself. + Sample string +} + +// DetectTextRepetition detects a period-p token block repeating at the *tail* +// of text — the canonical shape of an in-response token doom loop ("I am +// stuck. I am stuck. I am stuck. …"). Pure and deterministic: whitespace +// tokenization, suffix comparison, no heuristics beyond the thresholds. +// +// Only the trailing region is examined: a fixed character budget (64 chars +// per window token) is sliced off the tail before tokenization, so +// multi-megabyte responses do not pay a full-text token split. The budget is +// part of the deterministic contract (same text => same slice => same result). +// +// Ties on covered tokens prefer the smallest period (the most repeats), so +// "no no no no no no" reports p=1 x 6, not p=2 x 3 — the repeat count is what +// feeds the ladder. +// +// Returns nil when no block meets MinRepeats and MinCoveredTokens within +// MaxPeriodTokens / MaxWindowTokens. +// +// KNOWN LIMIT: token blocks must repeat *exactly*. Paraphrased loops ("I am +// stuck" / "I appear to be stuck" / …) do not trip this detector. +func DetectTextRepetition(text string, options *DoomLoopTextOptions) *TextRepetitionResult { + opts := DefaultDoomLoopTextOptions() + if options != nil { + if options.MaxPeriodTokens > 0 { + opts.MaxPeriodTokens = options.MaxPeriodTokens + } + if options.MinRepeats > 0 { + opts.MinRepeats = options.MinRepeats + } + if options.MinCoveredTokens > 0 { + opts.MinCoveredTokens = options.MinCoveredTokens + } + if options.MaxWindowTokens > 0 { + opts.MaxWindowTokens = options.MaxWindowTokens + } + } + + charBudget := opts.MaxWindowTokens * 64 + tail := text + if len(tail) > charBudget { + tail = tail[len(tail)-charBudget:] + } + allTokens := strings.Fields(tail) + tokens := allTokens + if len(tokens) > opts.MaxWindowTokens { + tokens = tokens[len(tokens)-opts.MaxWindowTokens:] + } + floor := opts.MinCoveredTokens + if floor < 2 { + floor = 2 + } + if len(tokens) < floor { + return nil + } + + var best *TextRepetitionResult + maxPeriod := opts.MaxPeriodTokens + if half := len(tokens) / 2; maxPeriod > half { + maxPeriod = half + } + for period := 1; period <= maxPeriod; period++ { + repeats := 1 + outer: + for start := len(tokens) - 2*period; start >= 0; start -= period { + for i := 0; i < period; i++ { + if tokens[start+i] != tokens[len(tokens)-period+i] { + break outer + } + } + repeats++ + } + covered := repeats * period + if repeats >= opts.MinRepeats && covered >= opts.MinCoveredTokens { + // Strictly greater, so a tie keeps the smaller period already found. + if best == nil || covered > best.CoveredTokens { + best = &TextRepetitionResult{ + Repeats: repeats, + PeriodTokens: period, + CoveredTokens: covered, + Sample: strings.Join(tokens[len(tokens)-period:], " "), + } + } + } + } + return best +} + +// LadderOptions tunes one ladder resolution. +type LadderOptions struct { + // AllowBlock is false for text and server-tool verdicts: the tokens are + // already emitted / the tool already ran server-side, so there is nothing + // to block. A block-level streak then falls through to escalate/steer (or + // observe), while stop still stops. + AllowBlock bool + // AllowEscalate disables the escalate rung for this resolution — used + // when no escalation mechanism is configured or the budget is exhausted; + // the streak falls through to the weaker rungs. + AllowEscalate bool +} + +// ResolveLadderAction maps a streak onto the strongest crossed ladder rung, +// returning "" when no rung is crossed. +func ResolveLadderAction(ladder DoomLoopLadder, streak int, options LadderOptions) DoomLoopAction { + if ladder.Stop.Meets(streak) { + return DoomLoopActionStop + } + if options.AllowBlock && ladder.Block.Meets(streak) { + return DoomLoopActionBlock + } + if options.AllowEscalate && ladder.Escalate.Meets(streak) { + return DoomLoopActionEscalate + } + if ladder.Steer.Meets(streak) { + return DoomLoopActionSteer + } + if ladder.Observe.Meets(streak) { + return DoomLoopActionObserve + } + return "" +} + +// streakEntry is the in-memory streak state for one tool: the serialized +// shape plus per-round bookkeeping that is deliberately never persisted, +// because rounds are meaningful only within one run. +type streakEntry struct { + fingerprint string + streak int + // round is the round of the most recent record. Same-round re-records are + // duplicates (the streak does not increment). + round int + hasRound bool + // roundFingerprints is the fingerprint set this tool was called with + // during round, sorted. A round's identity is the whole set, not its last + // call, so a fan-out of *distinct* arguments reissued verbatim is a + // repeat. Comparing only the last call let each round's first call reset + // the streak, so a repeating fan-out never accumulated evidence. + roundFingerprints []string + // seenThisRound collapses in-round duplicates: one decision per + // (tool, fingerprint) per round. + seenThisRound map[string]bool + // priorRoundFingerprints / priorStreak are the set the previous round was + // called with and the streak it earned, fixed at the round transition so + // arrival order within the current round cannot affect a score. + priorRoundFingerprints []string + hasPriorSet bool + priorStreak int + // priorCallStreaks is the PREVIOUS round's per-call streaks, fixed at the + // round transition for the same reason. The CURRENT round's counts live + // in callStreaks and become this field at the next transition. + priorCallStreaks map[string]int + callStreaks map[string]int +} + +// DoomLoopMonitor is a pure state machine over the recorded transcript: feed +// it tool calls and assistant texts, get verdicts back. +// +// State is bounded, plain JSON (one streak entry per distinct tool name plus +// one text streak) and round-trips through ConversationState. +// +// Safe for concurrent use: the engine records a round's calls from the +// goroutine that drives the loop while a consumer may snapshot State, and a +// data race in streak bookkeeping would silently corrupt detection. +type DoomLoopMonitor struct { + mu sync.Mutex + config ResolvedDoomLoopConfig + tools map[string]*streakEntry + text *streakEntry + // escalationsUsed counts recoveries consumed by this conversation. + // Persisted so a resumed run cannot reset its budget. + escalationsUsed int + // declaredRound holds the current round's declared per-tool fingerprint + // sets. Run-local and never serialized. + declaredRound map[string][]string + declaredRoundNum int + hasDeclaredRound bool + fingerprintMemoed map[string]string +} + +// NewDoomLoopMonitor builds a monitor. initialState, when non-nil, is +// restored (see Restore); pass a DoomLoopSerializedState, a *DoomLoopSerializedState, +// or the raw JSON-decoded map a persisted ConversationState carries. +func NewDoomLoopMonitor(config ResolvedDoomLoopConfig, initialState any) *DoomLoopMonitor { + m := &DoomLoopMonitor{config: config, tools: map[string]*streakEntry{}, fingerprintMemoed: map[string]string{}} + if initialState != nil { + m.Restore(initialState) + } + return m +} + +// DoomLoopRoundCall is one call in a declared round. +type DoomLoopRoundCall struct { + ToolName string + KeyMaterial any +} + +// DeclareRound declares the complete set of calls a round will make, before +// any of them is recorded. The monitor groups them per tool into that tool's +// round set. +// +// A round's identity is the *set* of fingerprints a tool was called with, so +// that set has to be known up front. Accumulating it call by call meant a +// round that is a strict superset of the previous one transiently equaled it +// while filling — [a,b], [a,b], [a,b,c] scored a verdict on the `b` of the +// third round and refused a call that represented real progress. It was also +// emission-order dependent: [c,a,b] never formed the matching prefix and +// scored nothing. Declaring the whole set removes both. +// +// Idempotent per round, and safe to skip: PER-CALL detection needs no +// declaration — every repeated (tool, arguments) identity accumulates its own +// consecutive-round count regardless, so an undeclared repeating fan-out +// still flags each repeated member (server-tool records take that path). What +// the declaration adds is round-set evidence: the fan-out scored as one unit, +// with a shared verdict and one steer message, instead of member by member. +// Unhashable key material is skipped here with a warning; the caller's own +// fallback chain handles it at record time. Never serialized. +func (m *DoomLoopMonitor) DeclareRound(round int, calls []DoomLoopRoundCall) { + collected := map[string]map[string]bool{} + m.mu.Lock() + defer m.mu.Unlock() + for _, call := range calls { + fingerprint, err := m.fingerprintOnceLocked(call.ToolName, call.KeyMaterial) + if err != nil { + // Unhashable: leave it out of the declared set — RecordToolCall's + // fallback chain decides this call's identity on its own — but say + // so, like every other fail-open path. + log.Printf("[DoomLoop] could not fingerprint a %q call while declaring round %d; excluding it from the round set: %v", call.ToolName, round, err) + continue + } + set := collected[call.ToolName] + if set == nil { + set = map[string]bool{} + collected[call.ToolName] = set + } + set[fingerprint] = true + } + declared := map[string][]string{} + for toolName, set := range collected { + members := make([]string, 0, len(set)) + for fingerprint := range set { + members = append(members, fingerprint) + } + sort.Strings(members) + declared[toolName] = members + } + m.declaredRound, m.declaredRoundNum, m.hasDeclaredRound = declared, round, true +} + +// fingerprintOnceLocked fingerprints with a per-canonical-form memo. The +// engine hands the same key material to DeclareRound and then to +// RecordToolCall, so without this every checked call is canonicalized and +// SHA-256'd twice per round. Upstream memoizes on object identity via a +// WeakMap; Go has no weak map, so the memo is keyed by the canonical string +// (which is the expensive part to recompute) and bounded by clearing it when +// it grows past a round's plausible width. +func (m *DoomLoopMonitor) fingerprintOnceLocked(toolName string, keyMaterial any) (string, error) { + canonical, err := CanonicalizeKeyMaterial(keyMaterial) + if err != nil { + return "", err + } + key := toolName + "\n" + canonical + if hit, ok := m.fingerprintMemoed[key]; ok { + return hit, nil + } + if len(m.fingerprintMemoed) > 1024 { + m.fingerprintMemoed = map[string]string{} + } + fingerprint := sha256Hex(key) + m.fingerprintMemoed[key] = fingerprint + return fingerprint, nil +} + +// CanEscalate reports whether the escalate rung can still fire: a mechanism +// is configured and the budget is not exhausted. +func (m *DoomLoopMonitor) CanEscalate() bool { + m.mu.Lock() + defer m.mu.Unlock() + return m.canEscalateLocked() +} + +func (m *DoomLoopMonitor) canEscalateLocked() bool { + return m.config.Escalation != nil && m.escalationsUsed < m.config.Escalation.MaxEscalations +} + +// ConsumeEscalation consumes one escalation from the budget. The ENGINE calls +// this when it actually applies the recovery (model swap / advisor forcing) — +// not at verdict time, so a verdict the engine ends up not honoring (a hook +// override, say) does not burn budget. +func (m *DoomLoopMonitor) ConsumeEscalation() { + m.mu.Lock() + m.escalationsUsed++ + m.mu.Unlock() +} + +// EscalationsUsed reports how many recoveries this conversation has consumed. +func (m *DoomLoopMonitor) EscalationsUsed() int { + m.mu.Lock() + defer m.mu.Unlock() + return m.escalationsUsed +} + +// callStreaksReconstructible reports whether an entry's per-call counts carry +// no information beyond its round set and round streak, so State may omit +// them and Restore can rebuild them exactly. Holds in the steady state of a +// repeating fan-out — every member has been issued in the same consecutive +// rounds, so every count equals the round streak. The counts must also cover +// exactly the set members: a shrunk round (count > streak) or an extra +// recorded non-member is real evidence and is still persisted verbatim. +func callStreaksReconstructible(entry *streakEntry) bool { + if entry.callStreaks == nil { + return true + } + set := entry.roundFingerprints + if len(set) == 0 { + set = []string{entry.fingerprint} + } + if len(entry.callStreaks) != len(set) { + return false + } + member := map[string]bool{} + for _, value := range set { + member[value] = true + } + for fingerprint, count := range entry.callStreaks { + if !member[fingerprint] || count != entry.streak { + return false + } + } + return true +} + +// State snapshots the serializable detector state (deep copy). Round markers +// are deliberately dropped: rounds are meaningful only within one run. +// StopVerdict and PendingSteer are owned by the engine, which merges them +// into the persisted blob alongside this snapshot. +func (m *DoomLoopMonitor) State() DoomLoopSerializedState { + m.mu.Lock() + defer m.mu.Unlock() + state := DoomLoopSerializedState{Tools: map[string]DoomLoopStreak{}} + for name, entry := range m.tools { + persisted := DoomLoopStreak{Fingerprint: entry.fingerprint, Streak: entry.streak} + // A multi-call round persists its full set, so a resumed run knows + // WHICH set earned the count. Copied, not aliased: the snapshot is + // handed to the caller's StateAccessor and an in-place mutation must + // not corrupt the detector still running against it. + if len(entry.roundFingerprints) > 1 { + persisted.RoundFingerprints = append([]string{}, entry.roundFingerprints...) + } + // Per-call counts persist so a repeat spanning a save/resume boundary + // keeps counting. Omitted only when fingerprint+streak already carry + // the identical information. The count-mismatch check matters: when a + // round SHRINKS to one call the round streak resets while the + // per-call count keeps climbing, and dropping it would hand the + // repeat a fresh grace window on resume. + if entry.callStreaks != nil && !callStreaksReconstructible(entry) { + counts := make(map[string]int, len(entry.callStreaks)) + for k, v := range entry.callStreaks { + counts[k] = v + } + persisted.CallStreaks = counts + } + state.Tools[name] = persisted + } + if m.text != nil { + state.Text = &DoomLoopStreak{Fingerprint: m.text.fingerprint, Streak: m.text.streak} + } + if m.escalationsUsed > 0 { + state.EscalationsUsed = m.escalationsUsed + } + return state +} + +// maxRestoredStreak bounds a restored streak. The blob is caller-writable +// JSON, so a negative count would hold the guardrail below every rung for +// that tool (thresholds compare with >=) and an absurd one would feed the +// ladder garbage. Clamped rather than rejected: a corrupt streak must degrade +// the entry, not drop the tool's evidence entirely. +const maxRestoredStreak = 1_000_000 + +// Restore restores persisted state (from ConversationState.DoomLoop). +// Invalid blobs are ignored with a warning — a corrupt detector state must +// never take down a resumed run. Engine-owned fields (StopVerdict, +// PendingSteer) are ignored here; the engine restores them itself. +func (m *DoomLoopMonitor) Restore(state any) { + decoded, ok := coerceDoomLoopState(state) + if !ok { + log.Printf("[DoomLoop] ignoring invalid persisted doom-loop state of type %T", state) + return + } + m.mu.Lock() + defer m.mu.Unlock() + m.tools = map[string]*streakEntry{} + for name, entry := range decoded.Tools { + if entry.Fingerprint == "" { + continue + } + m.tools[name] = restoreStreakEntry(entry) + } + if decoded.Text != nil && decoded.Text.Fingerprint != "" { + m.text = &streakEntry{fingerprint: decoded.Text.Fingerprint, streak: clampStreak(decoded.Text.Streak)} + } else { + m.text = nil + } + m.escalationsUsed = 0 + if decoded.EscalationsUsed > 0 { + m.escalationsUsed = decoded.EscalationsUsed + } + // Rounds are meaningful only within one run: the first resumed record is + // always a new round, whatever the numbering. + m.declaredRound, m.hasDeclaredRound = nil, false +} + +func coerceDoomLoopState(state any) (DoomLoopSerializedState, bool) { + switch v := state.(type) { + case DoomLoopSerializedState: + return v, true + case *DoomLoopSerializedState: + if v == nil { + return DoomLoopSerializedState{}, false + } + return *v, true + case string: + var decoded DoomLoopSerializedState + if err := json.Unmarshal([]byte(v), &decoded); err != nil { + return DoomLoopSerializedState{}, false + } + return decoded, true + case map[string]any: + b, err := json.Marshal(v) + if err != nil { + return DoomLoopSerializedState{}, false + } + var decoded DoomLoopSerializedState + if err := json.Unmarshal(b, &decoded); err != nil { + return DoomLoopSerializedState{}, false + } + return decoded, true + } + return DoomLoopSerializedState{}, false +} + +func clampStreak(streak int) int { + if streak < 1 { + return 1 + } + if streak > maxRestoredStreak { + return maxRestoredStreak + } + return streak +} + +// restoreStreakEntry rebuilds one tool's in-memory streak entry from its +// persisted shape. +// +// The persisted set (when the last round was multi-call) restores the round's +// identity exactly, so a resumed fan-out continues its streak and a resumed +// SUBSET cannot match it — the false-positive and false-negative failure +// modes of a single-fingerprint save. Pre-#89 blobs (and single-call rounds) +// carry no set; the lone fingerprint fully describes those rounds. +// +// Per-call counts restore into the CURRENT-round slot: the first resumed +// record is a new round, so RecordToolCall rolls them into its baseline +// exactly as a live round transition would. Absent counts mean State proved +// them reconstructible, so they are rebuilt for the WHOLE set — otherwise a +// width-W fan-out would resume with W-1 members' evidence reset. +func restoreStreakEntry(entry DoomLoopStreak) *streakEntry { + set := []string{entry.Fingerprint} + if len(entry.RoundFingerprints) > 1 { + valid := true + for _, value := range entry.RoundFingerprints { + if value == "" { + valid = false + break + } + } + if valid { + set = append([]string{}, entry.RoundFingerprints...) + sort.Strings(set) + } + } + streak := clampStreak(entry.Streak) + counts := map[string]int{} + for fingerprint, count := range entry.CallStreaks { + if count >= 1 { + counts[fingerprint] = count + } + } + if len(counts) == 0 { + for _, member := range set { + counts[member] = streak + } + } + return &streakEntry{fingerprint: entry.Fingerprint, streak: streak, roundFingerprints: set, callStreaks: counts} +} + +// RecordOptions tunes one RecordToolCall. +type RecordOptions struct { + // DisallowBlock is set for post-execution records (server tools), where + // blocking is meaningless because the tool already ran. + DisallowBlock bool + // Detector overrides the verdict's detector label. Defaults to + // DoomLoopDetectorToolFingerprint. + Detector DoomLoopDetectorKind +} + +// RecordToolCall records one tool call and returns the streak record plus any +// verdict. +// +// Streak semantics: +// - Per tool: interleaved calls to *other* tools do not reset a tool's +// streak (so search X, read file, search X, read file still trips). +// - Per round: the same (tool, fingerprint) recorded again in the SAME round +// is a duplicate — the streak does not increment (DuplicateInRound true) +// and the caller should reuse the decision it applied for the first +// occurrence. A streak measures the model re-issuing a call *after seeing +// its result*, which requires a round trip; N parallel identical calls in +// one turn are one piece of evidence, not N. +// - A round's identity is the *set* of fingerprints the tool was called +// with, not its last call, so a fan-out of distinct arguments reissued +// verbatim accumulates. A round whose set differs from the previous +// round's — in either direction, including a superset — resets to 1. +// Every call in a round reports that round's streak, so the ladder applies +// to the round as a unit. See DeclareRound: the set must be declared +// before the round's first call for this to hold for multi-call rounds. +// +// Blocked calls are recorded like any other — a model re-issuing a blocked +// call in a later round is stronger loop evidence, not progress. +// +// Returns an error wrapping ErrUnhashableKeyMaterial when the key material +// cannot be canonicalized. Callers (the engine) must catch it and fall back +// to the full-arguments identity — see ResolveLoopKeyMaterial. +func (m *DoomLoopMonitor) RecordToolCall(toolName string, keyMaterial any, round int, options RecordOptions) (DoomLoopCallRecord, error) { + m.mu.Lock() + defer m.mu.Unlock() + + fingerprint, err := m.fingerprintOnceLocked(toolName, keyMaterial) + if err != nil { + return DoomLoopCallRecord{}, err + } + previous := m.tools[toolName] + isSameRound := previous != nil && previous.hasRound && previous.round == round + + // The round's declared membership, when the engine announced it. A + // declaration only speaks for the calls it contains: a call dropped as + // unhashable still records here via the caller's fallback chain, but as a + // NON-member — it cannot inherit or move the ROUND's counters. Its own + // repetition still counts via the per-call detector below. + var declared []string + declaredKnown := false + if m.hasDeclaredRound && m.declaredRoundNum == round { + declared, declaredKnown = m.declaredRound[toolName], m.declaredRound[toolName] != nil + } + declaredMember := declaredKnown && containsString(declared, fingerprint) + + var seen map[string]bool + if isSameRound { + seen = previous.seenThisRound + } + duplicateInRound := seen[fingerprint] + + // The baseline every call of this round is measured against: the set the + // PREVIOUS round was called with and the streak it earned. Fixed at the + // round transition and carried unchanged for the round's length, so + // arrival order within a round can never affect a score. + var priorSet []string + hasPriorSet := false + priorStreak := 0 + if isSameRound { + priorSet, hasPriorSet, priorStreak = previous.priorRoundFingerprints, previous.hasPriorSet, previous.priorStreak + } else if previous != nil { + if len(previous.roundFingerprints) > 0 { + priorSet = previous.roundFingerprints + } else { + priorSet = []string{previous.fingerprint} + } + hasPriorSet, priorStreak = true, previous.streak + } + score := func(set []string) int { + if hasPriorSet && setsMatch(priorSet, set) { + return priorStreak + 1 + } + return 1 + } + + // PER-CALL evidence, alongside the round-set streak: the number of + // consecutive rounds THIS exact fingerprint has been issued in, whatever + // its round-mates did. Round identity treats any membership change as + // progress, which is right for the fan-out as a unit but blind to one + // call repeating inside varying company ([a,b], [a,c], [a,d] — the set + // differs every round, yet `a` is a 3-peat) and to a paused HITL member + // changing the resumed round's identity. + var priorCallStreaks map[string]int + if isSameRound { + priorCallStreaks = previous.priorCallStreaks + } else if previous != nil { + priorCallStreaks = previous.callStreaks + } + callStreak := priorCallStreaks[fingerprint] + 1 + + // Two identities, one scoring rule. What this CALL reports: the declared + // set when it is a member (every member of a repeating fan-out shares the + // round's streak, so the ladder applies to the round as a unit), its own + // singleton otherwise. What the ROUND stores: the declared set when there + // is one — a non-member must not overwrite the round's identity with its + // singleton, or the next round's members would compare against it and + // reset forever. + callSet := []string{fingerprint} + if declaredMember { + callSet = declared + } + streak := score(callSet) + roundSet := []string{fingerprint} + if declaredKnown { + roundSet = declared + } + + next := &streakEntry{ + // The identity that pairs with streak in persisted state: a + // non-member must not become it, or the saved count would attach to a + // call that never earned it (blocked on first appearance after a + // resume, while the real repeat lost its evidence). + fingerprint: fingerprint, + round: round, + hasRound: true, + roundFingerprints: roundSet, + streak: score(roundSet), + priorRoundFingerprints: priorSet, + hasPriorSet: hasPriorSet, + priorStreak: priorStreak, + priorCallStreaks: priorCallStreaks, + } + if declaredKnown && !declaredMember && previous != nil { + next.fingerprint = previous.fingerprint + } + if isSameRound && seen != nil { + seen[fingerprint] = true + next.seenThisRound = seen + } else { + next.seenThisRound = map[string]bool{fingerprint: true} + } + // This round's per-call accumulator, grown in place. Safe because + // priorCallStreaks aliases the PREVIOUS round's map (a fresh map is + // created at each round transition), so mutating this round's accumulator + // never disturbs the baseline; State copies before persisting either. + if isSameRound && previous.callStreaks != nil { + previous.callStreaks[fingerprint] = callStreak + next.callStreaks = previous.callStreaks + } else { + next.callStreaks = map[string]int{fingerprint: callStreak} + } + m.tools[toolName] = next + + // The stronger of the two detectors decides. The round streak covers the + // reissued fan-out as a unit; the per-call streak covers a repeat whose + // round-mates keep changing. For an exactly-repeating round both counts + // are equal, so nothing double-fires. + effective := streak + if callStreak > effective { + effective = callStreak + } + action := ResolveLadderAction(m.config.Ladder, effective, LadderOptions{ + AllowBlock: !options.DisallowBlock, + AllowEscalate: m.canEscalateLocked(), + }) + record := DoomLoopCallRecord{Fingerprint: fingerprint, Streak: effective, DuplicateInRound: duplicateInRound} + if action == "" { + return record, nil + } + detector := options.Detector + if detector == "" { + detector = DoomLoopDetectorToolFingerprint + } + record.Verdict = &DoomLoopVerdict{ + Detector: detector, + Action: action, + Streak: effective, + Fingerprint: fingerprint, + ToolName: toolName, + Message: buildToolVerdictMessage(toolName, fingerprint, callSet, streak, callStreak, + // An undeclared record's callSet is just this one call, which says + // nothing about the round's real width — so it may not be + // described as a set, nor have its fingerprint quoted as the + // round's identity. + declaredMember), + } + return record, nil +} + +// ResetTextStreak clears the cross-step text streak. The engine calls this +// when a late async-tool result is injected into the conversation: the run +// made observable forward progress, so a model that said "still waiting…" +// between deliveries must not accumulate a repetition streak toward a +// false-positive stop. Tool streaks are deliberately kept — re-issuing an +// identical tool call after a delivery is still loop evidence. +func (m *DoomLoopMonitor) ResetTextStreak() { + m.mu.Lock() + m.text = nil + m.mu.Unlock() +} + +// RecordAssistantText records one step's assistant text and returns the +// strongest verdict from the two text detectors: +// +// - text-repetition: a token block repeating within THIS response; the +// repeat count feeds the ladder directly, so a single response spinning +// "I am stuck." dozens of times can stop the run immediately. +// - text-streak: byte-identical (whitespace-normalized) text across +// consecutive steps. KNOWN LIMIT: paraphrased repetition does not trip. +// +// Empty/whitespace-only text (typical for tool-only turns) is a no-op: it +// neither counts nor resets the cross-step streak, mirroring how interleaved +// other-tool calls don't reset a tool streak. +func (m *DoomLoopMonitor) RecordAssistantText(text string) *DoomLoopVerdict { + m.mu.Lock() + defer m.mu.Unlock() + if m.config.Text.Disabled { + return nil + } + normalized := strings.Join(strings.Fields(text), " ") + if normalized == "" { + return nil + } + + textOpts := m.config.Text + var withinResponse *DoomLoopVerdict + if repetition := DetectTextRepetition(normalized, &textOpts); repetition != nil { + action := ResolveLadderAction(m.config.Ladder, repetition.Repeats, LadderOptions{AllowBlock: false, AllowEscalate: m.canEscalateLocked()}) + if action != "" { + fingerprint, err := FingerprintKeyMaterial(repetition.Sample) + if err == nil { + withinResponse = &DoomLoopVerdict{ + Detector: DoomLoopDetectorTextRepetition, + Action: action, + Streak: repetition.Repeats, + Fingerprint: fingerprint, + Message: fmt.Sprintf("Doom loop suspected: the response repeats %q %d times in a row. Stop repeating and take a different approach.", + repetition.Sample, repetition.Repeats), + } + } + } + } + + fingerprint, err := FingerprintKeyMaterial(normalized) + if err != nil { + return withinResponse + } + streak := 1 + if m.text != nil && m.text.fingerprint == fingerprint { + streak = m.text.streak + 1 + } + m.text = &streakEntry{fingerprint: fingerprint, streak: streak} + var crossStep *DoomLoopVerdict + if action := ResolveLadderAction(m.config.Ladder, streak, LadderOptions{AllowBlock: false, AllowEscalate: m.canEscalateLocked()}); action != "" { + crossStep = &DoomLoopVerdict{ + Detector: DoomLoopDetectorTextStreak, + Action: action, + Streak: streak, + Fingerprint: fingerprint, + Message: fmt.Sprintf("Doom loop suspected: the assistant produced identical text for %d consecutive turns. Stop repeating and take a different approach.", + streak), + } + } + return strongerVerdict(withinResponse, crossStep) +} + +var actionStrength = map[DoomLoopAction]int{ + DoomLoopActionObserve: 0, + DoomLoopActionSteer: 1, + DoomLoopActionEscalate: 2, + DoomLoopActionBlock: 3, + DoomLoopActionStop: 4, +} + +// strongerVerdict picks the stronger of two verdicts. Within-response wins +// ties: its message names the repeated block. +func strongerVerdict(a, b *DoomLoopVerdict) *DoomLoopVerdict { + if a == nil { + return b + } + if b == nil { + return a + } + if actionStrength[b.Action] > actionStrength[a.Action] { + return b + } + return a +} + +func containsString(values []string, want string) bool { + for _, value := range values { + if value == want { + return true + } + } + return false +} + +// setsMatch reports set equality over two sorted fingerprint lists. +func setsMatch(left, right []string) bool { + if len(left) != len(right) { + return false + } + for i := range left { + if left[i] != right[i] { + return false + } + } + return true +} + +// summarizeRound is a short, stable identity for a round's fingerprint set: +// the first members' prefixes, so every call of one round produces the same +// string (the steer rung dedupes queued guidance on exact text). +func summarizeRound(fingerprints []string) string { + shown := make([]string, 0, 3) + for i, value := range fingerprints { + if i == 3 { + break + } + if len(value) > 8 { + value = value[:8] + } + shown = append(shown, value) + } + if len(fingerprints) > 3 { + return fmt.Sprintf("%s+%d more…", strings.Join(shown, "+"), len(fingerprints)-3) + } + return strings.Join(shown, "+") + "…" +} + +// buildToolVerdictMessage renders the verdict text for a tool-fingerprint +// detection, naming what actually repeated. +// +// When the round streak decides (or ties), a multi-call round quotes the +// ROUND's identity — identical text for every call, deliberately, because the +// steer rung dedupes queued guidance by exact message text and one round of +// evidence must not queue N near-identical corrections. When the PER-CALL +// streak alone decides, the text carries neither the call's fingerprint nor +// its exact count: both vary between members of one round (an expanding +// fan-out holds counts 4, 3, 2 at once), and either would split one piece of +// guidance into per-member strings. The bound is therefore at most TWO +// distinct messages per tool per round; exact counts live in the verdict's +// Streak. +// +// roundDeclared says whether the detector was told the round's true +// membership. Only then does callSet describe the round, so only then may the +// text name a set or quote an argument fingerprint. +func buildToolVerdictMessage(toolName, fingerprint string, callSet []string, roundStreak, callStreak int, roundDeclared bool) string { + if callStreak > roundStreak || !roundDeclared { + return fmt.Sprintf("Doom loop suspected: this exact %q call has been repeated across consecutive rounds. Repeating it will not change the result. Take a different approach, or explain why repetition is required.", toolName) + } + if len(callSet) > 1 { + return fmt.Sprintf("Doom loop suspected: tool %q was invoked in %d consecutive rounds with the same set of %d parallel calls (round identity %s). Reissuing the same fan-out will not change the results. Take a different approach, or explain why repetition is required.", + toolName, roundStreak, len(callSet), summarizeRound(callSet)) + } + short := fingerprint + if len(short) > 16 { + short = short[:16] + } + return fmt.Sprintf("Doom loop suspected: tool %q was invoked in %d consecutive rounds with identical arguments (fingerprint %s…). Repeating the call will not change the result. Take a different approach, or explain why repetition is required.", + toolName, roundStreak, short) +} diff --git a/doom_loop_engine.go b/doom_loop_engine.go new file mode 100644 index 0000000..a83515c --- /dev/null +++ b/doom_loop_engine.go @@ -0,0 +1,478 @@ +package agent + +// Engine wiring for doom-loop detection (upstream #73, #89). +// +// The detector itself lives in doom_loop.go as a pure state machine; this file +// is the ModelResult side: where each checkpoint runs in the turn, how a +// verdict's action turns into a blocked tool output / a steer message / a +// one-turn escalation override / a halted run, and how the whole thing +// round-trips through ConversationState. + +import ( + "context" + "encoding/json" + "fmt" + "log" + + openrouter "github.com/OpenRouterTeam/go-sdk" + "github.com/OpenRouterTeam/go-sdk/models/components" +) + +// doomLoopState is the ModelResult-side doom-loop bookkeeping. +type doomLoopState struct { + monitor *DoomLoopMonitor + escalation *ResolvedEscalationConfig + // stop is the armed stop verdict. Persisted (as StopVerdict) so a + // condemned run stays halted across a decision-only resume. + stop *DoomLoopVerdict + // steer is guidance queued but not yet injected, deduplicated by exact + // text: one verdict can repeat across the calls of a round, and the model + // needs the guidance once. + steer []string + // pendingEscalation latches the FIRST escalation verdict for the next + // request; a second verdict in the same window (tool and text detectors + // both firing) must not double-spend the budget. + pendingEscalation *DoomLoopVerdict + // round counts the detector's rounds, which are what streaks are measured + // in. Incremented once per declared batch. + round int + // roundKeyMaterial caches the loopKey resolution per call id for the + // current round. loopKey is user code — it may count, log, or return a + // fresh value each time — so it must run at most once per call, and the + // declared identity and the recorded identity must agree. + roundKeyMaterial map[string]LoopKeyResolution + // roundDecisions collapses in-round duplicates so a repeated + // (tool, fingerprint) in one round reuses the first occurrence's decision + // instead of re-emitting the hook. + roundDecisions map[string]doomLoopDecision + // blocked maps a call id to the reason the detector refused it, filled by + // the pre-execution checkpoint and consumed by the executor. + blocked map[string]string +} + +type doomLoopDecision struct { + action DoomLoopAction + message string +} + +// initDoomLoop resolves the DoomLoop option and rehydrates persisted detector +// state. +// +// Condemnation rule: a doom-stopped conversation STAYS stopped across a +// decision-only resume (ApproveToolCalls / RejectToolCalls — approving a call +// is not new conversation). A fresh conversational turn clears the verdict: +// operator input is new information. Streaks are kept either way, so renewed +// repetition re-condemns quickly. +func (m *ModelResult) initDoomLoop() { + config := ResolveDoomLoopOption(m.input.DoomLoop) + if config == nil { + return + } + state := &doomLoopState{ + monitor: NewDoomLoopMonitor(*config, nil), + escalation: config.Escalation, + roundKeyMaterial: map[string]LoopKeyResolution{}, + roundDecisions: map[string]doomLoopDecision{}, + blocked: map[string]string{}, + } + if persisted := m.state.DoomLoop; persisted != nil { + state.monitor.Restore(*persisted) + state.steer = append(state.steer, persisted.PendingSteer...) + if persisted.StopVerdict != nil && (len(m.input.ApproveToolCalls) > 0 || len(m.input.RejectToolCalls) > 0) { + verdict := *persisted.StopVerdict + state.stop = &verdict + } + } + m.doom = state +} + +// syncDoomLoopState folds the detector's snapshot plus the engine-owned stop +// verdict and queued steer guidance into m.state, so a StateAccessor save +// carries them. +func (m *ModelResult) syncDoomLoopState() { + if m.doom == nil { + return + } + snapshot := m.doom.monitor.State() + if m.doom.stop != nil { + verdict := *m.doom.stop + snapshot.StopVerdict = &verdict + } + if len(m.doom.steer) > 0 { + snapshot.PendingSteer = append([]string{}, m.doom.steer...) + } + m.state.DoomLoop = &snapshot +} + +// beginDoomLoopRound declares a round's complete call set before any of its +// calls is scored, so a repeating fan-out is measured as one unit and the +// scoring is independent of emission order (upstream #89). +// +// Also the single place loopKey runs for the round: the resolution is cached +// per call id and reused by the pre-execution checkpoint. +func (m *ModelResult) beginDoomLoopRound(calls []ParsedToolCall) { + if m.doom == nil { + return + } + m.doom.round++ + m.doom.roundKeyMaterial = map[string]LoopKeyResolution{} + m.doom.roundDecisions = map[string]doomLoopDecision{} + m.doom.blocked = map[string]string{} + + declared := make([]DoomLoopRoundCall, 0, len(calls)) + for _, call := range calls { + tool := FindToolByName(m.input.Tools, call.Name) + resolution := ResolveLoopKeyMaterial(ToolLoopKeyOf(tool), toolInputMap(call)) + if resolution.Warning != "" { + log.Printf("[DoomLoop] tool %q: %s", call.Name, resolution.Warning) + } + m.doom.roundKeyMaterial[callKey(call)] = resolution + if resolution.Kind == LoopKeyExempt { + continue + } + declared = append(declared, DoomLoopRoundCall{ToolName: call.Name, KeyMaterial: resolution.KeyMaterial}) + } + m.doom.monitor.DeclareRound(m.doom.round, declared) +} + +// checkDoomLoopBeforeExecution is the pre-execution checkpoint for one call. +// It returns the refusal reason when the detector condemned the call, and "" +// when execution should proceed. +// +// - observe/steer/escalate — the call still runs; the side effects are +// applied for the next turn. +// - block — the caller synthesizes an error output without executing. The +// model sees the explanation in the tool result, which is where it looks. +// - stop — arms the stop verdict AND blocks this call: executing a call the +// detector just condemned and then stopping would be incoherent. +func (m *ModelResult) checkDoomLoopBeforeExecution(call ParsedToolCall) string { + if m.doom == nil { + return "" + } + resolution, cached := m.doom.roundKeyMaterial[callKey(call)] + if !cached { + tool := FindToolByName(m.input.Tools, call.Name) + resolution = ResolveLoopKeyMaterial(ToolLoopKeyOf(tool), toolInputMap(call)) + if resolution.Warning != "" { + log.Printf("[DoomLoop] tool %q: %s", call.Name, resolution.Warning) + } + } + if resolution.Kind == LoopKeyExempt { + return "" + } + decision := m.evaluateDoomLoop(call.Name, resolution.KeyMaterial, toolInputMap(call), RecordOptions{}, &call) + if decision.action == DoomLoopActionBlock || decision.action == DoomLoopActionStop { + reason := decision.message + if reason == "" { + reason = fmt.Sprintf("Doom loop suspected: tool %q is repeating identical calls. Repeating the call will not change the result. Take a different approach.", call.Name) + } + m.doom.blocked[callKey(call)] = reason + return reason + } + return "" +} + +// evaluateDoomLoop records one identity with the monitor, applies the +// DoomLoopDetected hook's override, and applies the action's shared side +// effects. In-round duplicates reuse the first occurrence's decision rather +// than re-emitting the hook. +// +// An unhashable identity falls back to the full arguments; if that is also +// unhashable, detection is skipped for the call — a detector must never fail +// a run. +func (m *ModelResult) evaluateDoomLoop(toolName string, keyMaterial, fallback any, options RecordOptions, call *ParsedToolCall) doomLoopDecision { + record, err := m.doom.monitor.RecordToolCall(toolName, keyMaterial, m.doom.round, options) + if err != nil { + log.Printf("[DoomLoop] could not fingerprint call to %q; retrying with the full-arguments identity: %v", toolName, err) + if fallback == nil { + return doomLoopDecision{} + } + record, err = m.doom.monitor.RecordToolCall(toolName, fallback, m.doom.round, options) + if err != nil { + log.Printf("[DoomLoop] fallback identity for %q also unhashable; skipping detection for this call: %v", toolName, err) + return doomLoopDecision{} + } + } + decisionKey := toolName + "\n" + record.Fingerprint + if record.DuplicateInRound { + if previous, ok := m.doom.roundDecisions[decisionKey]; ok { + return previous + } + } + if record.Verdict == nil { + decision := doomLoopDecision{} + m.doom.roundDecisions[decisionKey] = decision + return decision + } + action := m.applyDoomLoopVerdict(*record.Verdict, call) + decision := doomLoopDecision{action: action, message: record.Verdict.Message} + m.doom.roundDecisions[decisionKey] = decision + m.applyDoomLoopSideEffects(action, *record.Verdict) + return decision +} + +// applyDoomLoopVerdict emits the DoomLoopDetected hook and resolves the final +// action. A handler may override in either direction (last override wins) — +// with two downgrades enforced rather than trusted: a non-tool verdict cannot +// be blocked (the tokens are already emitted / the server tool already ran), +// and an escalate override needs config and budget. Both downgrade to +// observe, never silently to a stronger action. +func (m *ModelResult) applyDoomLoopVerdict(verdict DoomLoopVerdict, call *ParsedToolCall) DoomLoopAction { + action := verdict.Action + if m.hooksManager != nil { + payload := DoomLoopDetectedPayload{ + Detector: verdict.Detector, + Action: verdict.Action, + Streak: verdict.Streak, + Fingerprint: verdict.Fingerprint, + ToolName: verdict.ToolName, + Message: verdict.Message, + } + if call != nil { + payload.ToolInput = toolInputMap(*call) + } + emit, err := m.hooksManager.EmitDoomLoopDetected(payload, m.hookEmitOptions(verdict.ToolName)) + if err != nil { + log.Printf("[DoomLoopDetected] hook error: %v", err) + } else { + for _, result := range emit.Results { + if result.OverrideAction != "" { + action = result.OverrideAction + } + } + } + } + if verdict.Detector != DoomLoopDetectorToolFingerprint && action == DoomLoopActionBlock { + action = DoomLoopActionObserve + } + if action == DoomLoopActionEscalate && !m.doom.monitor.CanEscalate() { + action = DoomLoopActionObserve + } + return action +} + +// applyDoomLoopSideEffects applies the side effects shared by every +// checkpoint. `block` has no shared side effect — the calling checkpoint +// synthesizes the blocked output itself. +func (m *ModelResult) applyDoomLoopSideEffects(action DoomLoopAction, verdict DoomLoopVerdict) { + switch action { + case DoomLoopActionSteer: + m.queueDoomLoopSteer(verdict.Message) + case DoomLoopActionEscalate: + if m.doom.pendingEscalation == nil { + latched := verdict + m.doom.pendingEscalation = &latched + // The model should know why its next turn looks different. Rides + // the persisted steer path so the transcript stays well-formed. + m.queueDoomLoopSteer(verdict.Message + " An escalated turn follows: use the additional guidance to change course.") + } + case DoomLoopActionStop: + condemned := verdict + m.doom.stop = &condemned + } +} + +// queueDoomLoopSteer queues guidance, deduplicating identical messages. +func (m *ModelResult) queueDoomLoopSteer(message string) { + for _, existing := range m.doom.steer { + if existing == message { + return + } + } + m.doom.steer = append(m.doom.steer, message) +} + +// flushDoomLoopSteer injects queued guidance as a user message before the next +// model turn, reusing the Stop-hook append-prompt path so state and messages +// advance observably. Called before each follow-up request AND before every +// pause-persist, so guidance queued right before a pause lands in the +// persisted conversation instead of being dropped. +func (m *ModelResult) flushDoomLoopSteer(req *components.ResponsesRequest) { + if m.doom == nil || len(m.doom.steer) == 0 { + return + } + prompt := joinLines(m.doom.steer) + m.doom.steer = nil + m.injectAppendPromptMessage(req, prompt) +} + +func joinLines(values []string) string { + out := "" + for i, value := range values { + if i > 0 { + out += "\n" + } + out += value + } + return out +} + +// checkDoomLoopForResponse is the step-level checkpoint on a fresh model +// response: +// +// 1. Text detectors — within-response token repetition plus the cross-step +// identical-text streak. +// 2. Server-tool fingerprints — server tools (web_search_call and friends) +// never pass through the client-tool executor, so their repetition is +// detected here, post-execution, from the echoed call fields on the +// output item. Post-execution means block is meaningless. +func (m *ModelResult) checkDoomLoopForResponse(resp components.OpenResponsesResult) { + if m.doom == nil { + return + } + if verdict := m.doom.monitor.RecordAssistantText(ExtractTextFromResponse(resp)); verdict != nil { + action := m.applyDoomLoopVerdict(*verdict, nil) + m.applyDoomLoopSideEffects(action, *verdict) + } + for _, item := range resp.Output { + toolName, identity, ok := serverToolIdentity(item) + if !ok { + continue + } + m.evaluateDoomLoop(toolName, identity, nil, RecordOptions{DisallowBlock: true, Detector: DoomLoopDetectorServerToolFingerprint}, nil) + } +} + +// serverToolIdentity extracts a server-tool output item's echoed call +// identity: its item type as the tool name and the request-shaped fields +// (action, query, …) as the key material. Returns false for items that are +// not server-tool results or that echo nothing identifying — a server tool +// whose output carries no request echo cannot be fingerprinted, and guessing +// would collapse every one of its calls onto one identity. +func serverToolIdentity(item components.OutputItems) (string, any, bool) { + b, err := json.Marshal(item) + if err != nil { + return "", nil, false + } + var raw map[string]any + if err := json.Unmarshal(b, &raw); err != nil { + return "", nil, false + } + itemType, _ := raw["type"].(string) + // Only *_call items are server-tool invocations; message / reasoning / + // function_call items are handled elsewhere. + if itemType == "" || itemType == "function_call" || !hasSuffix(itemType, "_call") { + return "", nil, false + } + identity := map[string]any{} + for _, field := range []string{"action", "query", "queries", "arguments", "input", "code", "container_id", "name", "prompt"} { + if value, ok := raw[field]; ok && value != nil { + identity[field] = value + } + } + if len(identity) == 0 { + return "", nil, false + } + return "server:" + itemType, identity, true +} + +func hasSuffix(s, suffix string) bool { + return len(s) >= len(suffix) && s[len(s)-len(suffix):] == suffix +} + +// takeDoomLoopEscalationOverrides consumes a pending escalation into ONE-TURN +// request overrides: +// +// - Model — replace the request's model for this dispatch only. The base +// request is never mutated, so the following turn reverts automatically. +// A single-model override also clears any fallback `models` list, which +// would otherwise shadow it. +// - Advisor — append an `openrouter:advisor` server tool (transcript +// forwarded, instructions describing the detected loop) and pin +// ToolChoice to it via allowed_tools/required, so the stuck model must +// consult the advisor before doing anything else this turn. +// +// Burns one unit of the escalation budget at APPLICATION time — verdicts the +// engine never got to apply (run stopped, paused, or overridden) do not spend. +func (m *ModelResult) takeDoomLoopEscalationOverrides(req components.ResponsesRequest) (components.ResponsesRequest, bool) { + if m.doom == nil { + return req, false + } + verdict, config := m.doom.pendingEscalation, m.doom.escalation + m.doom.pendingEscalation = nil + if verdict == nil || config == nil || !m.doom.monitor.CanEscalate() { + return req, false + } + m.doom.monitor.ConsumeEscalation() + + if config.Model != "" { + req.Model = openrouter.Pointer(config.Model) + req.Models = nil + } + if config.AdvisorEnabled { + forward := true + instructions := "You are an escalation advisor. The executing model appears stuck in a loop: " + + verdict.Message + " Diagnose why its approach is failing and give concrete, specific instructions for a DIFFERENT approach. Do not restate the problem." + params := &components.AdvisorServerToolConfig{ForwardTranscript: &forward, Instructions: &instructions} + if raw, ok := config.Advisor["instructions"].(string); ok && raw != "" { + custom := raw + params.Instructions = &custom + } + if raw, ok := config.Advisor["forwardTranscript"].(bool); ok { + custom := raw + params.ForwardTranscript = &custom + } + if raw, ok := config.Advisor["model"].(string); ok && raw != "" { + custom := raw + params.Model = &custom + } + tools := append([]components.ResponsesRequestToolUnion{}, req.Tools...) + tools = append(tools, components.CreateResponsesRequestToolUnionOpenrouterAdvisor(components.AdvisorServerToolOpenRouter{ + Type: components.AdvisorServerToolOpenRouterTypeOpenrouterAdvisor, Parameters: params, + })) + req.Tools = tools + // Force the consult: constrain this turn's tool surface to the + // advisor and require a call. + choice := components.CreateOpenAIResponsesToolChoiceUnionToolChoiceAllowed(components.ToolChoiceAllowed{ + Mode: components.CreateModeModeRequired(components.ModeRequiredRequired), + Tools: []map[string]any{{"type": "openrouter:advisor"}}, + Type: components.ToolChoiceAllowedTypeAllowedTools, + }) + req.ToolChoice = &choice + } + return req, true +} + +// sealDoomLoopStop halts the run for an armed stop verdict while keeping the +// persisted history well-formed: every function_call in the current response +// that has no output yet gets a synthesized halt-error output, so a stateful +// resume never sends a dangling function_call (providers reject those). +func (m *ModelResult) sealDoomLoopStop(resp components.OpenResponsesResult, resolved map[string]bool) { + if m.doom == nil || m.doom.stop == nil { + return + } + halt := "Run halted by doom-loop detection." + if m.doom.stop.Message != "" { + halt = "Run halted by doom-loop detection: " + m.doom.stop.Message + } + for _, call := range ExtractToolCallsFromResponse(resp) { + if resolved[callKey(call)] { + continue + } + output, err := FormatToolOutputWithError(ToolExecutionResult{CallID: call.CallID, Name: call.Name, Error: errorString(halt)}) + if err != nil { + log.Printf("[DoomLoop] could not synthesize a halt output for %q: %v", call.Name, err) + continue + } + m.state = AppendToMessages(m.state, components.CreateInputsUnion1FunctionCallOutputItem(output)) + } +} + +// errorString is a tiny error wrapper for synthesized tool outputs. +type errorString string + +func (e errorString) Error() string { return string(e) } + +// DoomLoopVerdict reports the verdict that stopped this run, or nil when the +// run was not stopped by doom-loop detection (including when detection is +// off). For per-event observation use the DoomLoopDetected hook. +func (m *ModelResult) DoomLoopVerdict(ctx context.Context) *DoomLoopVerdict { + m.ensure() + m.mu.RLock() + defer m.mu.RUnlock() + if m.doom == nil || m.doom.stop == nil { + return nil + } + verdict := *m.doom.stop + return &verdict +} diff --git a/doom_loop_engine_test.go b/doom_loop_engine_test.go new file mode 100644 index 0000000..c78db85 --- /dev/null +++ b/doom_loop_engine_test.go @@ -0,0 +1,651 @@ +package agent + +// Doom-loop detection through the callModel loop (upstream #73, #89). +// +// The unit tests in doom_loop_test.go pin the detector; these pin the engine: +// which requests are dispatched, what lands in the conversation, whether the +// tool actually ran, and what survives a save/resume. That is the observable +// contract — a detector that is right but wired wrong spends exactly as much +// money as no detector at all. + +import ( + "context" + "encoding/json" + "strings" + "testing" + + "github.com/OpenRouterTeam/go-sdk/models/components" + "github.com/OpenRouterTeam/go-sdk/models/operations" +) + +// repeatingSender returns the same tool-call response for every request, which +// is exactly the shape of a model stuck in a loop. +func repeatingSender(resp components.OpenResponsesResult) *fakeSender { + created := operations.CreateCreateResponsesResponseOpenResponsesResult(resp) + return &fakeSender{responses: []*operations.CreateResponsesResponse{&created}} +} + +// callWith builds a response that calls `search` with the given arguments. +func callWith(id string, args ...string) components.OpenResponsesResult { + items := make([]components.OutputItems, 0, len(args)) + for i, raw := range args { + items = append(items, components.CreateOutputItemsFunctionCall(components.OutputFunctionCallItem{ + CallID: id + "_" + string(rune('a'+i)), Name: "search", Arguments: raw, + })) + } + return components.OpenResponsesResult{ID: id, Output: items} +} + +// outputErrorTexts returns the error strings of every function_call_output in +// a request's accumulated input. +func outputErrorTexts(req components.ResponsesRequest) []string { + var out []string + for _, item := range requestInputItems(req.Input) { + if item.FunctionCallOutputItem == nil { + continue + } + raw := item.FunctionCallOutputItem.Output.Str + if raw == nil { + continue + } + var decoded struct { + Error string `json:"error"` + } + if err := json.Unmarshal([]byte(*raw), &decoded); err == nil && decoded.Error != "" { + out = append(out, decoded.Error) + } + } + return out +} + +// userMessageTexts returns every plain-string user message in a request. +func userMessageTexts(req components.ResponsesRequest) []string { + var out []string + for _, item := range requestInputItems(req.Input) { + msg := item.EasyInputMessage + if msg == nil || msg.Role.EasyInputMessageRoleUser == nil { + continue + } + if content, ok := msg.Content.GetOrZero(); ok && content.Str != nil { + out = append(out, *content.Str) + } + } + return out +} + +// The default ladder gives the loop one free round, observes at 2 and refuses +// at 3 — with the refusal delivered as the tool's own error output, which is +// where the model looks. +func TestRepeatedToolCallIsRefusedAtTheBlockRung(t *testing.T) { + executions := 0 + tool := MustNewTool(ToolConfig[sampleInput]{Name: "search", Execute: func(context.Context, sampleInput, ToolExecuteContext) (any, error) { + executions++ + return "same answer", nil + }}) + sender := repeatingSender(callWith("resp", `{"query":"go"}`)) + result, err := CallModel(context.Background(), sender, CallModelInput{ + Model: "openai/test", Input: "hi", Tools: []Tool{tool}, MaxTurns: 4, DoomLoop: true, + }) + if err != nil { + t.Fatal(err) + } + if _, err := result.Text(context.Background()); err != nil { + t.Fatal(err) + } + + // Rounds 1 and 2 execute (free round, then observe); rounds 3 and 4 are + // refused before execution. + if executions != 2 { + t.Fatalf("tool executed %d times across 4 identical rounds, want 2 (blocked from round 3)", executions) + } + errs := outputErrorTexts(sender.requests[len(sender.requests)-1]) + if len(errs) == 0 { + t.Fatal("a refused call must still produce a function_call_output carrying the explanation") + } + if !strings.Contains(errs[len(errs)-1], "Doom loop suspected") { + t.Fatalf("refusal output = %q, want the verdict message", errs[len(errs)-1]) + } +} + +// Detection is opt-in: without the option the same loop runs to the turn +// budget untouched. +func TestDoomLoopIsOffByDefault(t *testing.T) { + executions := 0 + tool := MustNewTool(ToolConfig[sampleInput]{Name: "search", Execute: func(context.Context, sampleInput, ToolExecuteContext) (any, error) { + executions++ + return "same answer", nil + }}) + sender := repeatingSender(callWith("resp", `{"query":"go"}`)) + result, err := CallModel(context.Background(), sender, CallModelInput{ + Model: "openai/test", Input: "hi", Tools: []Tool{tool}, MaxTurns: 4, + }) + if err != nil { + t.Fatal(err) + } + if _, err := result.Text(context.Background()); err != nil { + t.Fatal(err) + } + if executions != 4 { + t.Fatalf("tool executed %d times, want 4: detection must be opt-in", executions) + } +} + +// The stop rung halts before the next model request — the whole point is to +// stop spending — reports the verdict, and ends the session as doom_loop. +func TestStopRungHaltsBeforeTheNextModelRequest(t *testing.T) { + tool := MustNewTool(ToolConfig[sampleInput]{Name: "search", Execute: func(context.Context, sampleInput, ToolExecuteContext) (any, error) { + return "same answer", nil + }}) + var endReason SessionEndReason + manager := NewHooksManager() + manager.OnSessionEnd(HookEntry[SessionEndPayload, EmptyHookResult]{ + Handler: func(payload SessionEndPayload, _ LifecycleHookContext) (HookHandlerResult[EmptyHookResult], error) { + endReason = payload.Reason + return VoidResult[EmptyHookResult](), nil + }, + }) + sender := repeatingSender(callWith("resp", `{"query":"go"}`)) + result, err := CallModel(context.Background(), sender, CallModelInput{ + Model: "openai/test", Input: "hi", Tools: []Tool{tool}, MaxTurns: 20, Hooks: manager, + DoomLoop: DoomLoopConfig{Ladder: DoomLoopLadder{Observe: DoomLoopAt(2), Block: DoomLoopOff(), Stop: DoomLoopAt(3)}}, + }) + if err != nil { + t.Fatal(err) + } + if _, err := result.Text(context.Background()); err != nil { + t.Fatal(err) + } + + // Rounds 1, 2 and 3 each cost one request; the stop verdict armed on + // round 3 prevents a fourth. + if sender.calls != 3 { + t.Fatalf("dispatched %d requests, want 3: a stop verdict must halt before the next one", sender.calls) + } + verdict := result.DoomLoopVerdict(context.Background()) + if verdict == nil { + t.Fatal("DoomLoopVerdict must report the verdict that stopped the run") + } + if verdict.Action != DoomLoopActionStop || verdict.Streak != 3 { + t.Fatalf("verdict = %+v, want a stop at streak 3", verdict) + } + if endReason != SessionEndReasonDoomLoop { + t.Fatalf("SessionEnd reason = %q, want doom_loop", endReason) + } + // Persisted history must stay well-formed: the halted round's + // function_call needs a paired output or a stateful resume would send a + // dangling call. + state, err := result.State(context.Background()) + if err != nil { + t.Fatal(err) + } + calls, outputs := 0, 0 + for _, item := range state.Messages { + if item.FunctionCallItem != nil || item.OutputFunctionCallItem != nil { + calls++ + } + if item.FunctionCallOutputItem != nil { + outputs++ + } + } + if calls != outputs { + t.Fatalf("persisted history has %d function_calls and %d outputs; every call must be paired", calls, outputs) + } +} + +// A run with no verdict reports nil rather than a zero verdict. +func TestDoomLoopVerdictIsNilOnAHealthyRun(t *testing.T) { + created := operations.CreateCreateResponsesResponseOpenResponsesResult(textTurn("resp_1", "done")) + sender := &fakeSender{responses: []*operations.CreateResponsesResponse{&created}} + result, err := CallModel(context.Background(), sender, CallModelInput{Model: "openai/test", Input: "hi", DoomLoop: true}) + if err != nil { + t.Fatal(err) + } + if verdict := result.DoomLoopVerdict(context.Background()); verdict != nil { + t.Fatalf("healthy run verdict = %+v, want nil", verdict) + } +} + +// The #89 fix through the engine: a repeated three-call fan-out is scored as +// one unit, so at the block rung EVERY member is refused rather than the loop +// spinning forever. +func TestRepeatedFanOutIsRefusedAsAWholeRound(t *testing.T) { + executions := 0 + tool := MustNewTool(ToolConfig[sampleInput]{Name: "search", Execute: func(context.Context, sampleInput, ToolExecuteContext) (any, error) { + executions++ + return "same answer", nil + }}) + sender := repeatingSender(callWith("resp", `{"query":"a"}`, `{"query":"b"}`, `{"query":"c"}`)) + result, err := CallModel(context.Background(), sender, CallModelInput{ + Model: "openai/test", Input: "hi", Tools: []Tool{tool}, MaxTurns: 4, DoomLoop: true, + }) + if err != nil { + t.Fatal(err) + } + if _, err := result.Text(context.Background()); err != nil { + t.Fatal(err) + } + // Rounds 1 and 2 run all three calls; rounds 3 and 4 refuse all three. + if executions != 6 { + t.Fatalf("tool executed %d times, want 6 (two full rounds, then the whole fan-out refused)", executions) + } + errs := outputErrorTexts(sender.requests[len(sender.requests)-1]) + if len(errs) < 3 { + t.Fatalf("refused outputs = %d, want one per member of the round: %v", len(errs), errs) + } + if !strings.Contains(errs[len(errs)-1], "parallel calls") { + t.Fatalf("fan-out refusal should quote the round identity: %q", errs[len(errs)-1]) + } +} + +// A tool declaring itself exempt is never refused — this is the documented +// escape hatch for a legitimately repeated poller or anchor read. +func TestLoopKeyExemptToolIsNeverRefused(t *testing.T) { + executions := 0 + tool := MustNewTool(ToolConfig[sampleInput]{Name: "search", LoopKey: &LoopKey{Exempt: true}, + Execute: func(context.Context, sampleInput, ToolExecuteContext) (any, error) { + executions++ + return "same answer", nil + }}) + sender := repeatingSender(callWith("resp", `{"query":"go"}`)) + result, err := CallModel(context.Background(), sender, CallModelInput{ + Model: "openai/test", Input: "hi", Tools: []Tool{tool}, MaxTurns: 5, DoomLoop: true, + }) + if err != nil { + t.Fatal(err) + } + if _, err := result.Text(context.Background()); err != nil { + t.Fatal(err) + } + if executions != 5 { + t.Fatalf("exempt tool executed %d times, want 5 (never refused)", executions) + } +} + +// A loopKey that narrows identity to one field makes calls differing only in +// other fields count as repeats. +func TestLoopKeyNarrowsCallIdentity(t *testing.T) { + executions := 0 + tool := MustNewTool(ToolConfig[sampleInput]{Name: "search", LoopKey: &LoopKey{Fields: []string{"query"}}, + Execute: func(context.Context, sampleInput, ToolExecuteContext) (any, error) { + executions++ + return "same answer", nil + }}) + // The nonce changes every turn, so the full-arguments identity would + // never repeat; the declared loopKey ignores it. + first := callWith("r1", `{"query":"go","nonce":"1"}`) + second := callWith("r2", `{"query":"go","nonce":"2"}`) + third := callWith("r3", `{"query":"go","nonce":"3"}`) + c1 := operations.CreateCreateResponsesResponseOpenResponsesResult(first) + c2 := operations.CreateCreateResponsesResponseOpenResponsesResult(second) + c3 := operations.CreateCreateResponsesResponseOpenResponsesResult(third) + sender := &fakeSender{responses: []*operations.CreateResponsesResponse{&c1, &c2, &c3}} + + result, err := CallModel(context.Background(), sender, CallModelInput{ + Model: "openai/test", Input: "hi", Tools: []Tool{tool}, MaxTurns: 3, DoomLoop: true, + }) + if err != nil { + t.Fatal(err) + } + if _, err := result.Text(context.Background()); err != nil { + t.Fatal(err) + } + if executions != 2 { + t.Fatalf("tool executed %d times, want 2: a narrowed loopKey must see through the varying nonce", executions) + } +} + +// The steer rung injects the guidance as a user message on the next turn, so +// the correction reaches the model and the transcript stays well-formed. +func TestSteerRungInjectsGuidanceIntoTheNextTurn(t *testing.T) { + tool := MustNewTool(ToolConfig[sampleInput]{Name: "search", Execute: func(context.Context, sampleInput, ToolExecuteContext) (any, error) { + return "same answer", nil + }}) + sender := repeatingSender(callWith("resp", `{"query":"go"}`)) + result, err := CallModel(context.Background(), sender, CallModelInput{ + Model: "openai/test", Input: "hi", Tools: []Tool{tool}, MaxTurns: 3, + DoomLoop: DoomLoopConfig{Ladder: DoomLoopLadder{Observe: DoomLoopOff(), Steer: DoomLoopAt(2), Block: DoomLoopOff(), Stop: DoomLoopOff()}}, + }) + if err != nil { + t.Fatal(err) + } + if _, err := result.Text(context.Background()); err != nil { + t.Fatal(err) + } + messages := userMessageTexts(sender.requests[len(sender.requests)-1]) + var steered int + for _, message := range messages { + if strings.Contains(message, "Doom loop suspected") { + steered++ + } + } + if steered == 0 { + t.Fatalf("no steer guidance reached the model; user messages = %v", messages) + } + // Deduped by exact text: one verdict must not queue the same correction + // twice within a round. + if steered > 2 { + t.Fatalf("steer guidance injected %d times, want it deduplicated", steered) + } +} + +// The DoomLoopDetected hook fires at every rung — including observe, which +// exists precisely so a caller can watch without changing behavior — and a +// handler may de-escalate a would-be block. +func TestDoomLoopDetectedHookFiresAndCanDeEscalate(t *testing.T) { + executions := 0 + tool := MustNewTool(ToolConfig[sampleInput]{Name: "search", Execute: func(context.Context, sampleInput, ToolExecuteContext) (any, error) { + executions++ + return "same answer", nil + }}) + var seen []DoomLoopDetectedPayload + manager := NewHooksManager() + manager.OnDoomLoopDetected(HookEntry[DoomLoopDetectedPayload, DoomLoopDetectedResult]{ + Handler: func(payload DoomLoopDetectedPayload, _ LifecycleHookContext) (HookHandlerResult[DoomLoopDetectedResult], error) { + seen = append(seen, payload) + // Never block: this run knows better. + return SyncResult(DoomLoopDetectedResult{OverrideAction: DoomLoopActionObserve}), nil + }, + }) + sender := repeatingSender(callWith("resp", `{"query":"go"}`)) + result, err := CallModel(context.Background(), sender, CallModelInput{ + Model: "openai/test", Input: "hi", Tools: []Tool{tool}, MaxTurns: 4, Hooks: manager, DoomLoop: true, + }) + if err != nil { + t.Fatal(err) + } + if _, err := result.Text(context.Background()); err != nil { + t.Fatal(err) + } + if len(seen) == 0 { + t.Fatal("DoomLoopDetected never fired") + } + first := seen[0] + if first.Detector != DoomLoopDetectorToolFingerprint || first.ToolName != "search" || first.Streak != 2 { + t.Fatalf("first payload = %+v, want a tool-fingerprint observe at streak 2", first) + } + if first.ToolInput["query"] != "go" { + t.Fatalf("payload toolInput = %v, want the repeated call's arguments", first.ToolInput) + } + if first.Message == "" || first.Fingerprint == "" { + t.Fatalf("payload must carry the verdict message and fingerprint: %+v", first) + } + // The override held: every round executed. + if executions != 4 { + t.Fatalf("tool executed %d times, want 4: the handler de-escalated every block", executions) + } +} + +// A hook cannot make a text verdict blocking (the tokens are already +// emitted), and it cannot escalate without config or budget — both downgrade +// to observe rather than silently acting stronger. +func TestDoomLoopHookOverrideDowngradesAreEnforced(t *testing.T) { + config := ResolveDoomLoopOption(true) + m := &ModelResult{doom: &doomLoopState{monitor: NewDoomLoopMonitor(*config, nil)}} + manager := NewHooksManager() + manager.OnDoomLoopDetected(HookEntry[DoomLoopDetectedPayload, DoomLoopDetectedResult]{ + Handler: func(_ DoomLoopDetectedPayload, _ LifecycleHookContext) (HookHandlerResult[DoomLoopDetectedResult], error) { + return SyncResult(DoomLoopDetectedResult{OverrideAction: DoomLoopActionBlock}), nil + }, + }) + m.hooksManager = manager + + textVerdict := DoomLoopVerdict{Detector: DoomLoopDetectorTextStreak, Action: DoomLoopActionObserve, Streak: 2, Message: "text"} + if got := m.applyDoomLoopVerdict(textVerdict, nil); got != DoomLoopActionObserve { + t.Fatalf("block override on a text verdict = %q, want observe", got) + } + + escalating := NewHooksManager() + escalating.OnDoomLoopDetected(HookEntry[DoomLoopDetectedPayload, DoomLoopDetectedResult]{ + Handler: func(_ DoomLoopDetectedPayload, _ LifecycleHookContext) (HookHandlerResult[DoomLoopDetectedResult], error) { + return SyncResult(DoomLoopDetectedResult{OverrideAction: DoomLoopActionEscalate}), nil + }, + }) + m.hooksManager = escalating + toolVerdict := DoomLoopVerdict{Detector: DoomLoopDetectorToolFingerprint, Action: DoomLoopActionObserve, Streak: 2, Message: "tool"} + if got := m.applyDoomLoopVerdict(toolVerdict, nil); got != DoomLoopActionObserve { + t.Fatalf("escalate override without config/budget = %q, want observe", got) + } +} + +// Streaks persist across the save/resume boundary, so a per-turn-resume +// topology (one CallModel per user turn, state persisted between) accumulates +// evidence instead of re-baselining on every turn. +func TestDoomLoopStreaksSurviveSerializeAndResume(t *testing.T) { + executions := 0 + newTool := func() Tool { + return MustNewTool(ToolConfig[sampleInput]{Name: "search", Execute: func(context.Context, sampleInput, ToolExecuteContext) (any, error) { + executions++ + return "same answer", nil + }}) + } + sender := repeatingSender(callWith("resp", `{"query":"go"}`)) + first, err := CallModel(context.Background(), sender, CallModelInput{ + Model: "openai/test", Input: "hi", Tools: []Tool{newTool()}, MaxTurns: 2, DoomLoop: true, + }) + if err != nil { + t.Fatal(err) + } + state, err := first.State(context.Background()) + if err != nil { + t.Fatal(err) + } + if executions != 2 { + t.Fatalf("first run executed %d times, want 2 (both rounds below the block rung)", executions) + } + if state.DoomLoop == nil || state.DoomLoop.Tools["search"].Streak != 2 { + t.Fatalf("detector state must persist the streak, got %+v", state.DoomLoop) + } + + // Round-trip the whole conversation, as a durable store would. + blob, err := SerializeConversationState(state) + if err != nil { + t.Fatal(err) + } + restored, err := DeserializeConversationState(blob) + if err != nil { + t.Fatal(err) + } + if restored.DoomLoop == nil || restored.DoomLoop.Tools["search"].Streak != 2 { + t.Fatalf("detector state did not round-trip: %+v", restored.DoomLoop) + } + + // The next user turn re-issues the same call: the accumulated evidence + // makes it round 3, which the default ladder refuses. + executions = 0 + resumeSender := repeatingSender(callWith("resp2", `{"query":"go"}`)) + resumed, err := CallModel(context.Background(), resumeSender, CallModelInput{ + Model: "openai/test", Input: "again", Tools: []Tool{newTool()}, MaxTurns: 1, State: &restored, DoomLoop: true, + }) + if err != nil { + t.Fatal(err) + } + if _, err := resumed.Text(context.Background()); err != nil { + t.Fatal(err) + } + if executions != 0 { + t.Fatalf("resumed run executed the repeat %d times, want 0: the streak must not re-baseline on resume", executions) + } +} + +// Condemnation rule: a doom-stopped conversation stays stopped across a +// decision-only resume (approving a call is not new conversation), and a +// fresh conversational turn clears the verdict. +func TestStopVerdictSurvivesADecisionResumeButNotAFreshTurn(t *testing.T) { + tool := MustNewTool(ToolConfig[sampleInput]{Name: "search", Execute: func(context.Context, sampleInput, ToolExecuteContext) (any, error) { + return "ok", nil + }}) + verdict := DoomLoopVerdict{Detector: DoomLoopDetectorToolFingerprint, Action: DoomLoopActionStop, Streak: 6, Fingerprint: "abc", ToolName: "search", Message: "condemned"} + condemned := CreateInitialState() + condemned.Status = ConversationStatusAwaitingApproval + condemned.PendingToolCalls = []ParsedToolCall{{CallID: "call_1", Name: "search", RawArgs: `{"query":"go"}`}} + condemned.DoomLoop = &DoomLoopSerializedState{Tools: map[string]DoomLoopStreak{}, StopVerdict: &verdict} + + // Decision-only resume: the run stays halted and dispatches nothing. + resumeState := condemned + resumeSender := &fakeSender{responses: []*operations.CreateResponsesResponse{}} + resumed, err := CallModel(context.Background(), resumeSender, CallModelInput{ + Model: "openai/test", Tools: []Tool{tool}, State: &resumeState, + ApproveToolCalls: []string{"call_1"}, DoomLoop: true, + }) + if err != nil { + t.Fatal(err) + } + if got := resumed.DoomLoopVerdict(context.Background()); got == nil || got.Message != "condemned" { + t.Fatalf("a decision-only resume must keep the stop verdict, got %+v", got) + } + if resumeSender.calls != 0 { + t.Fatalf("a condemned run dispatched %d requests, want 0", resumeSender.calls) + } + + // Fresh conversational turn: operator input is new information, so the + // verdict clears and the run proceeds. + freshState := condemned + freshState.Status = ConversationStatusInProgress + freshState.PendingToolCalls = nil + created := operations.CreateCreateResponsesResponseOpenResponsesResult(textTurn("resp_1", "fresh")) + freshSender := &fakeSender{responses: []*operations.CreateResponsesResponse{&created}} + fresh, err := CallModel(context.Background(), freshSender, CallModelInput{ + Model: "openai/test", Input: "try again differently", Tools: []Tool{tool}, State: &freshState, DoomLoop: true, + }) + if err != nil { + t.Fatal(err) + } + text, err := fresh.Text(context.Background()) + if err != nil { + t.Fatal(err) + } + if text != "fresh" { + t.Fatalf("text = %q, want the fresh turn to run", text) + } + if got := fresh.DoomLoopVerdict(context.Background()); got != nil { + t.Fatalf("a fresh conversational turn must clear the verdict, got %+v", got) + } +} + +// The escalate rung swaps the model for exactly one dispatch and reverts +// afterwards, and the budget bounds how often that can happen. +func TestEscalateRungAppliesAOneTurnModelOverride(t *testing.T) { + tool := MustNewTool(ToolConfig[sampleInput]{Name: "search", Execute: func(context.Context, sampleInput, ToolExecuteContext) (any, error) { + return "same answer", nil + }}) + sender := repeatingSender(callWith("resp", `{"query":"go"}`)) + result, err := CallModel(context.Background(), sender, CallModelInput{ + Model: "openai/weak", Input: "hi", Tools: []Tool{tool}, MaxTurns: 5, + DoomLoop: DoomLoopConfig{ + Ladder: DoomLoopLadder{Observe: DoomLoopOff(), Escalate: DoomLoopAt(2), Block: DoomLoopOff(), Stop: DoomLoopOff()}, + Escalation: &DoomLoopEscalationConfig{Model: "openai/strong", MaxEscalations: 1}, + }, + }) + if err != nil { + t.Fatal(err) + } + if _, err := result.Text(context.Background()); err != nil { + t.Fatal(err) + } + + var models []string + for _, req := range sender.requests { + if req.Model != nil { + models = append(models, *req.Model) + } + } + if len(models) < 4 { + t.Fatalf("expected several dispatches, got models %v", models) + } + // Round 1 is free; the round-2 verdict escalates the round-3 dispatch, + // then the budget of 1 is spent and the model reverts. + if models[0] != "openai/weak" || models[1] != "openai/weak" { + t.Fatalf("the first two dispatches must use the configured model, got %v", models) + } + if models[2] != "openai/strong" { + t.Fatalf("dispatch 3 model = %q, want the escalated model", models[2]) + } + if models[3] != "openai/weak" { + t.Fatalf("dispatch 4 model = %q, want the configured model back (one-turn override)", models[3]) + } +} + +// The advisor mechanism pins the escalated turn to an `openrouter:advisor` +// consult so the stuck model must ask for guidance before anything else. +func TestEscalateRungCanForceAnAdvisorConsult(t *testing.T) { + tool := MustNewTool(ToolConfig[sampleInput]{Name: "search", Execute: func(context.Context, sampleInput, ToolExecuteContext) (any, error) { + return "same answer", nil + }}) + enabled := true + sender := repeatingSender(callWith("resp", `{"query":"go"}`)) + result, err := CallModel(context.Background(), sender, CallModelInput{ + Model: "openai/test", Input: "hi", Tools: []Tool{tool}, MaxTurns: 3, + DoomLoop: DoomLoopConfig{ + Ladder: DoomLoopLadder{Observe: DoomLoopOff(), Escalate: DoomLoopAt(2), Block: DoomLoopOff(), Stop: DoomLoopOff()}, + Escalation: &DoomLoopEscalationConfig{AdvisorEnabled: &enabled}, + }, + }) + if err != nil { + t.Fatal(err) + } + if _, err := result.Text(context.Background()); err != nil { + t.Fatal(err) + } + escalated := sender.requests[2] + var sawAdvisor bool + for _, apiTool := range escalated.Tools { + if apiTool.AdvisorServerToolOpenRouter != nil { + sawAdvisor = true + params := apiTool.AdvisorServerToolOpenRouter.Parameters + if params == nil || params.ForwardTranscript == nil || !*params.ForwardTranscript { + t.Fatalf("advisor must see the loop it is diagnosing: %+v", params) + } + if params.Instructions == nil || !strings.Contains(*params.Instructions, "Doom loop suspected") { + t.Fatalf("advisor instructions should describe the detected loop: %+v", params.Instructions) + } + } + } + if !sawAdvisor { + t.Fatal("the escalated dispatch must append the openrouter:advisor server tool") + } + if escalated.ToolChoice == nil || escalated.ToolChoice.ToolChoiceAllowed == nil || escalated.ToolChoice.ToolChoiceAllowed.Mode.ModeRequired == nil { + t.Fatalf("the escalated turn must pin tool_choice to a required advisor consult, got %#v", escalated.ToolChoice) + } + // The override is one-turn: the next dispatch is back to the caller's + // tool surface. + if len(sender.requests) > 3 { + for _, apiTool := range sender.requests[3].Tools { + if apiTool.AdvisorServerToolOpenRouter != nil { + t.Fatal("the advisor override must not persist past its one turn") + } + } + } +} + +// A repeated text response trips the text detectors even with no tools in +// play, and block is unavailable there so the streak falls through. +func TestRepeatedAssistantTextIsDetected(t *testing.T) { + var actions []DoomLoopAction + manager := NewHooksManager() + manager.OnDoomLoopDetected(HookEntry[DoomLoopDetectedPayload, DoomLoopDetectedResult]{ + Handler: func(payload DoomLoopDetectedPayload, _ LifecycleHookContext) (HookHandlerResult[DoomLoopDetectedResult], error) { + actions = append(actions, payload.Action) + if payload.Detector != DoomLoopDetectorTextRepetition { + t.Errorf("detector = %q, want text-repetition", payload.Detector) + } + return VoidResult[DoomLoopDetectedResult](), nil + }, + }) + created := operations.CreateCreateResponsesResponseOpenResponsesResult(textTurn("resp_1", strings.Repeat("I am stuck. ", 8))) + sender := &fakeSender{responses: []*operations.CreateResponsesResponse{&created}} + result, err := CallModel(context.Background(), sender, CallModelInput{ + Model: "openai/test", Input: "hi", Hooks: manager, DoomLoop: true, + }) + if err != nil { + t.Fatal(err) + } + if _, err := result.Text(context.Background()); err != nil { + t.Fatal(err) + } + if len(actions) == 0 { + t.Fatal("a response repeating one block eight times must be detected") + } + if actions[0] != DoomLoopActionStop { + t.Fatalf("action = %q, want stop: block is unavailable for text so the streak falls through", actions[0]) + } +} diff --git a/doom_loop_test.go b/doom_loop_test.go new file mode 100644 index 0000000..b07caf6 --- /dev/null +++ b/doom_loop_test.go @@ -0,0 +1,1067 @@ +package agent + +// Doom-loop detector tests (upstream #73, #89). +// +// The cases here are upstream's: the fingerprint conformance vectors, the +// fan-out fixes from #89 (round-set identity, per-call streaks, subset and +// superset rounds, save/resume of both), the ladder's strongest-wins +// resolution and its allowBlock fall-through, the escalation budget, and the +// text detectors' tie-breaking. Assertions are on the verdict payload, the +// serialized state shape and restore semantics — never on the detector's +// private bookkeeping, which is exactly the sort of test that keeps passing +// while the port is wrong. + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "math" + "regexp" + "strconv" + "strings" + "sync" + "testing" +) + +// testLadder is a deterministic ladder for streak-threshold assertions. +func testLadder(observe, steer, escalate, block, stop DoomLoopThreshold) DoomLoopLadder { + return DoomLoopLadder{Observe: observe, Steer: steer, Escalate: escalate, Block: block, Stop: stop} +} + +func defaultMonitor(t *testing.T) *DoomLoopMonitor { + t.Helper() + config := ResolveDoomLoopOption(true) + if config == nil { + t.Fatal("ResolveDoomLoopOption(true) must resolve a config") + } + return NewDoomLoopMonitor(*config, nil) +} + +func mustRecord(t *testing.T, m *DoomLoopMonitor, toolName string, args any, round int) DoomLoopCallRecord { + t.Helper() + record, err := m.RecordToolCall(toolName, args, round, RecordOptions{}) + if err != nil { + t.Fatalf("RecordToolCall(%s, round %d): %v", toolName, round, err) + } + return record +} + +// --------------------------------------------------------------------------- +// Cross-port fingerprint contract +// --------------------------------------------------------------------------- + +type fingerprintVectorFile struct { + ToolCallVectors []struct { + Name string `json:"name"` + ToolName string `json:"toolName"` + KeyMaterial json.RawMessage `json:"keyMaterial"` + JCS string `json:"jcs"` + Fingerprint string `json:"fingerprint"` + } `json:"toolCallVectors"` + KeyMaterialVectors []struct { + Name string `json:"name"` + KeyMaterial json.RawMessage `json:"keyMaterial"` + JCS string `json:"jcs"` + Fingerprint string `json:"fingerprint"` + } `json:"keyMaterialVectors"` +} + +// loneSurrogateEscape matches an unpaired \udXXX escape in a vector's raw +// JSON. A high surrogate followed by a low one is a legitimate pair and is +// left alone. +var loneSurrogateEscape = regexp.MustCompile(`\\u[dD]([89abAB][0-9a-fA-F]{2})(?:\\u[dD][c-fC-F])?`) + +// decodeVectorMaterial parses a vector's key material into the JSON data +// model, keeping numeric literals exact via json.Number so 1e+21 survives. +// +// Lone surrogates need a detour. A JS string carries an unpaired surrogate +// natively, so JSON.parse round-trips "\ud800" and JSON.stringify escapes it +// back — which is what the conformance vector pins. Go's json decoder instead +// substitutes U+FFFD, so decoding the vector normally would silently change +// the key material and test the wrong input. The escape is therefore parked +// behind a control-character sentinel across the decode and restored +// afterwards as the WTF-8 bytes a lone surrogate occupies in a Go string, +// which is exactly what CanonicalizeKeyMaterial is written to re-escape. +func decodeVectorMaterial(t *testing.T, raw json.RawMessage) any { + t.Helper() + parked := loneSurrogateEscape.ReplaceAllStringFunc(string(raw), func(match string) string { + if strings.Count(strings.ToLower(match), "\\u") > 1 { + return match // a real surrogate pair + } + return `\u0001` + match[2:] + `\u0001` + }) + decoder := json.NewDecoder(strings.NewReader(parked)) + decoder.UseNumber() + var value any + if err := decoder.Decode(&value); err != nil { + t.Fatalf("decode vector key material %s: %v", raw, err) + } + return restoreLoneSurrogates(value) +} + +var parkedSurrogate = regexp.MustCompile("\x01([dD][0-9a-fA-F]{3})\x01") + +func restoreLoneSurrogates(value any) any { + switch v := value.(type) { + case string: + return parkedSurrogate.ReplaceAllStringFunc(v, func(match string) string { + code, err := strconv.ParseUint(match[1:len(match)-1], 16, 32) + if err != nil { + return match + } + // WTF-8: the three-byte UTF-8 form of a surrogate code point. + return string([]byte{byte(0xE0 | (code >> 12)), byte(0x80 | ((code >> 6) & 0x3F)), byte(0x80 | (code & 0x3F))}) + }) + case map[string]any: + for k, child := range v { + v[k] = restoreLoneSurrogates(child) + } + return v + case []any: + for i, child := range v { + v[i] = restoreLoneSurrogates(child) + } + return v + } + return value +} + +// TestDoomLoopFingerprintConformanceVectors is the cross-port gate: the JCS +// canonical form AND the digest must match upstream byte for byte. A +// divergence here means persisted detector state does not transfer between +// the TypeScript, Python and Go ports. +func TestDoomLoopFingerprintConformanceVectors(t *testing.T) { + var file fingerprintVectorFile + if err := json.Unmarshal([]byte(doomLoopFingerprintVectors), &file); err != nil { + t.Fatalf("parse embedded vectors: %v", err) + } + if len(file.ToolCallVectors) == 0 || len(file.KeyMaterialVectors) == 0 { + t.Fatal("embedded vectors are empty; the conformance gate would pass vacuously") + } + + for _, vector := range file.ToolCallVectors { + t.Run("toolCall/"+vector.Name, func(t *testing.T) { + material := decodeVectorMaterial(t, vector.KeyMaterial) + canonical, err := CanonicalizeKeyMaterial(material) + if err != nil { + t.Fatalf("canonicalize: %v", err) + } + if canonical != vector.JCS { + t.Fatalf("jcs =\n %s\nwant\n %s", canonical, vector.JCS) + } + got, err := FingerprintToolCall(vector.ToolName, material) + if err != nil { + t.Fatalf("fingerprint: %v", err) + } + if got != vector.Fingerprint { + t.Fatalf("fingerprint = %s, want %s", got, vector.Fingerprint) + } + }) + } + for _, vector := range file.KeyMaterialVectors { + t.Run("keyMaterial/"+vector.Name, func(t *testing.T) { + material := decodeVectorMaterial(t, vector.KeyMaterial) + canonical, err := CanonicalizeKeyMaterial(material) + if err != nil { + t.Fatalf("canonicalize: %v", err) + } + if canonical != vector.JCS { + t.Fatalf("jcs = %s, want %s", canonical, vector.JCS) + } + got, err := FingerprintKeyMaterial(material) + if err != nil { + t.Fatalf("fingerprint: %v", err) + } + if got != vector.Fingerprint { + t.Fatalf("fingerprint = %s, want %s", got, vector.Fingerprint) + } + }) + } +} + +// The vectors' "rejected" section: values RFC 8785 cannot represent must +// produce a catchable error so the engine can fall back to the +// full-arguments identity rather than failing the run. +func TestCanonicalizeRejectsUnrepresentableKeyMaterial(t *testing.T) { + deep := any(map[string]any{}) + cursor := deep.(map[string]any) + for i := 0; i < MaxCanonicalizeDepth+2; i++ { + next := map[string]any{} + cursor["n"] = next + cursor = next + } + cyclic := map[string]any{} + cyclic["self"] = &cyclic + + for _, tc := range []struct { + name string + value any + }{ + {"NaN", map[string]any{"v": math.NaN()}}, + {"positive infinity", map[string]any{"v": math.Inf(1)}}, + {"negative infinity", map[string]any{"v": math.Inf(-1)}}, + {"nesting beyond the depth cap", deep}, + } { + t.Run(tc.name, func(t *testing.T) { + if _, err := CanonicalizeKeyMaterial(tc.value); err == nil { + t.Fatal("expected an error") + } else if !errors.Is(err, ErrUnhashableKeyMaterial) { + t.Fatalf("error %v must wrap ErrUnhashableKeyMaterial so the engine can fall back", err) + } + }) + } +} + +// Key insertion order must not matter, and the same arguments under two tool +// names must never collide. +func TestFingerprintIdentityRules(t *testing.T) { + a, err := FingerprintToolCall("bash", map[string]any{"command": "ls", "cwd": "/tmp"}) + if err != nil { + t.Fatal(err) + } + b, err := FingerprintToolCall("bash", map[string]any{"cwd": "/tmp", "command": "ls"}) + if err != nil { + t.Fatal(err) + } + if a != b { + t.Fatalf("key order changed the fingerprint: %s vs %s", a, b) + } + other, err := FingerprintToolCall("fetch", map[string]any{"command": "ls", "cwd": "/tmp"}) + if err != nil { + t.Fatal(err) + } + if other == a { + t.Fatal("the tool name must participate in the fingerprint") + } +} + +// --------------------------------------------------------------------------- +// Config resolution +// --------------------------------------------------------------------------- + +func TestResolveDoomLoopOption(t *testing.T) { + if ResolveDoomLoopOption(nil) != nil { + t.Fatal("nil must leave detection off") + } + if ResolveDoomLoopOption(false) != nil { + t.Fatal("false must leave detection off") + } + config := ResolveDoomLoopOption(true) + if config == nil { + t.Fatal("true must resolve defaults") + } + want := DefaultDoomLoopLadder() + if config.Ladder != want { + t.Fatalf("default ladder = %+v, want %+v", config.Ladder, want) + } + if config.Text.Disabled || config.Text.MinRepeats != 4 || config.Text.MaxPeriodTokens != 16 { + t.Fatalf("default text options = %+v", config.Text) + } + if config.Escalation != nil { + t.Fatal("escalation must be off unless configured") + } + + // A sub-1 threshold is nonsense and falls back to the default. + partial := ResolveDoomLoopOption(DoomLoopConfig{Ladder: DoomLoopLadder{Block: DoomLoopAt(0)}}) + if partial.Ladder.Block.Value != want.Block.Value { + t.Fatalf("a sub-1 threshold must fall back to the default, got %+v", partial.Ladder.Block) + } + // An explicit off survives. + off := ResolveDoomLoopOption(DoomLoopConfig{Ladder: DoomLoopLadder{Block: DoomLoopOff()}}) + if off.Ladder.Block.Enabled() { + t.Fatal("an explicitly disabled rung must stay disabled") + } + // Text disabled explicitly. + noText := ResolveDoomLoopOption(DoomLoopConfig{Text: &DoomLoopTextOptions{Disabled: true}}) + if !noText.Text.Disabled { + t.Fatal("Text.Disabled must disable the text detectors") + } + // A partially-specified tuning object must NOT disable them: the zero + // value of the master switch has to mean "on", or the natural + // `&DoomLoopTextOptions{MinRepeats: 3}` would silently turn detection off. + tuned := ResolveDoomLoopOption(DoomLoopConfig{Text: &DoomLoopTextOptions{MinRepeats: 3}}) + if tuned.Text.Disabled { + t.Fatal("a partial text-options object must leave the detectors enabled") + } + if tuned.Text.MinRepeats != 3 || tuned.Text.MaxPeriodTokens != 16 { + t.Fatalf("partial tuning = %+v, want MinRepeats 3 with the other defaults kept", tuned.Text) + } +} + +func TestResolveDoomLoopEscalationRequiresAMechanism(t *testing.T) { + no := false + for _, tc := range []struct { + name string + escalation *DoomLoopEscalationConfig + want bool + }{ + {"no config", nil, false}, + {"advisor explicitly off is not a mechanism", &DoomLoopEscalationConfig{AdvisorEnabled: &no}, false}, + {"model is a mechanism", &DoomLoopEscalationConfig{Model: "openai/strong"}, true}, + {"advisor is a mechanism", &DoomLoopEscalationConfig{Advisor: map[string]any{"instructions": "diagnose"}}, true}, + } { + t.Run(tc.name, func(t *testing.T) { + config := ResolveDoomLoopOption(DoomLoopConfig{ + Ladder: DoomLoopLadder{Escalate: DoomLoopAt(2)}, + Escalation: tc.escalation, + }) + if (config.Escalation != nil) != tc.want { + t.Fatalf("escalation resolved = %v, want %v", config.Escalation != nil, tc.want) + } + if tc.want && config.Escalation.MaxEscalations != DefaultMaxEscalations { + t.Fatalf("default budget = %d, want %d", config.Escalation.MaxEscalations, DefaultMaxEscalations) + } + }) + } +} + +// --------------------------------------------------------------------------- +// Ladder +// --------------------------------------------------------------------------- + +func TestResolveLadderAction(t *testing.T) { + ladder := testLadder(DoomLoopAt(2), DoomLoopAt(3), DoomLoopAt(4), DoomLoopAt(5), DoomLoopAt(6)) + for _, tc := range []struct { + streak int + allowBlock bool + allowEscalate bool + want DoomLoopAction + }{ + {1, true, true, ""}, + {2, true, true, DoomLoopActionObserve}, + {3, true, true, DoomLoopActionSteer}, + {4, true, true, DoomLoopActionEscalate}, + {5, true, true, DoomLoopActionBlock}, + {6, true, true, DoomLoopActionStop}, + // allowBlock false: a block-level streak falls through to escalate, + // but stop still stops. + {5, false, true, DoomLoopActionEscalate}, + {6, false, true, DoomLoopActionStop}, + // allowEscalate false: falls through to steer. + {4, true, false, DoomLoopActionSteer}, + {5, false, false, DoomLoopActionSteer}, + } { + t.Run(fmt.Sprintf("streak=%d block=%v escalate=%v", tc.streak, tc.allowBlock, tc.allowEscalate), func(t *testing.T) { + got := ResolveLadderAction(ladder, tc.streak, LadderOptions{AllowBlock: tc.allowBlock, AllowEscalate: tc.allowEscalate}) + if got != tc.want { + t.Fatalf("action = %q, want %q", got, tc.want) + } + }) + } +} + +// --------------------------------------------------------------------------- +// loopKey resolution +// --------------------------------------------------------------------------- + +func TestResolveLoopKeyMaterial(t *testing.T) { + args := map[string]any{"command": "ls", "cwd": "/tmp", "verbose": true} + + if got := ResolveLoopKeyMaterial(nil, args); got.Kind != LoopKeyResolved { + t.Fatalf("absent declaration = %+v, want the full arguments", got) + } + if got := ResolveLoopKeyMaterial(&LoopKey{Exempt: true}, args); got.Kind != LoopKeyExempt { + t.Fatalf("Exempt = %+v, want exempt", got) + } + subset := ResolveLoopKeyMaterial(&LoopKey{Fields: []string{"command", "cwd"}}, args) + if subset.Kind != LoopKeyResolved { + t.Fatalf("field subset = %+v", subset) + } + if picked, ok := subset.KeyMaterial.(map[string]any); !ok || len(picked) != 2 || picked["command"] != "ls" { + t.Fatalf("field subset material = %#v", subset.KeyMaterial) + } + // An empty or wholly-absent field list would collapse every call onto one + // identity, so it falls back rather than being trusted. + for _, fields := range [][]string{{}, {"nope", "missing"}} { + got := ResolveLoopKeyMaterial(&LoopKey{Fields: fields}, args) + if got.Kind != LoopKeyFallback || got.Warning == "" { + t.Fatalf("degenerate field list %v = %+v, want a warned fallback", fields, got) + } + } + // nil from the function exempts THIS call. + exempt := ResolveLoopKeyMaterial(&LoopKey{Fn: func(map[string]any) (any, error) { return nil, nil }}, args) + if exempt.Kind != LoopKeyExempt { + t.Fatalf("nil result = %+v, want exempt", exempt) + } + // An error falls back: detection must never take down a run. + failed := ResolveLoopKeyMaterial(&LoopKey{Fn: func(map[string]any) (any, error) { return nil, errors.New("boom") }}, args) + if failed.Kind != LoopKeyFallback || failed.Warning == "" { + t.Fatalf("erroring loopKey = %+v, want a warned fallback", failed) + } +} + +// loopKey is invoked exactly once per checked call. +func TestLoopKeyInvokedOncePerCheckedCall(t *testing.T) { + calls := 0 + key := &LoopKey{Fn: func(args map[string]any) (any, error) { + calls++ + return args["command"], nil + }} + for i := 0; i < 3; i++ { + ResolveLoopKeyMaterial(key, map[string]any{"command": "ls"}) + } + if calls != 3 { + t.Fatalf("loopKey invoked %d times for 3 calls, want 3", calls) + } +} + +// --------------------------------------------------------------------------- +// Single-call streaks +// --------------------------------------------------------------------------- + +func TestIdenticalCallAcrossRoundsAccumulatesAndTripsTheLadder(t *testing.T) { + m := defaultMonitor(t) + args := map[string]any{"query": "go"} + + // The engine declares each round, so a single-call round is a declared + // round of one — which is what lets the verdict quote the arguments' + // fingerprint as the round's identity. + declare(t, m, 0, "search", args) + first := mustRecord(t, m, "search", args, 0) + if first.Streak != 1 || first.Verdict != nil { + t.Fatalf("round 0 = %+v, want streak 1 with no verdict (the free round)", first) + } + declare(t, m, 1, "search", args) + second := mustRecord(t, m, "search", args, 1) + if second.Streak != 2 || second.Verdict == nil || second.Verdict.Action != DoomLoopActionObserve { + t.Fatalf("round 1 = %+v, want streak 2 observing", second) + } + declare(t, m, 2, "search", args) + third := mustRecord(t, m, "search", args, 2) + if third.Streak != 3 || third.Verdict == nil || third.Verdict.Action != DoomLoopActionBlock { + t.Fatalf("round 2 = %+v, want streak 3 blocking", third) + } + if third.Verdict.Detector != DoomLoopDetectorToolFingerprint || third.Verdict.ToolName != "search" { + t.Fatalf("verdict metadata = %+v", third.Verdict) + } + if !strings.Contains(third.Verdict.Message, "identical arguments") { + t.Fatalf("single-call verdict message should name identical arguments: %q", third.Verdict.Message) + } +} + +// Interleaved calls to other tools do not reset a tool's streak. +func TestInterleavedOtherToolsDoNotResetAStreak(t *testing.T) { + m := defaultMonitor(t) + search := map[string]any{"query": "go"} + for round := 0; round < 3; round++ { + mustRecord(t, m, "search", search, round) + mustRecord(t, m, "read", map[string]any{"path": fmt.Sprintf("f%d", round)}, round) + } + final := mustRecord(t, m, "search", search, 3) + if final.Streak != 4 { + t.Fatalf("streak = %d, want 4: interleaved other-tool calls must not reset it", final.Streak) + } +} + +// A round's N identical parallel calls are one piece of evidence, not N. +func TestInRoundDuplicatesCollapseToOneDecision(t *testing.T) { + m := defaultMonitor(t) + args := map[string]any{"query": "go"} + first := mustRecord(t, m, "search", args, 0) + dup := mustRecord(t, m, "search", args, 0) + if !dup.DuplicateInRound { + t.Fatal("the same (tool, fingerprint) in one round must report DuplicateInRound") + } + if dup.Streak != first.Streak { + t.Fatalf("duplicate streak = %d, want the first occurrence's %d", dup.Streak, first.Streak) + } +} + +// Changing the arguments is progress: the streak resets. +func TestDifferentArgumentsResetTheStreak(t *testing.T) { + m := defaultMonitor(t) + mustRecord(t, m, "search", map[string]any{"query": "a"}, 0) + mustRecord(t, m, "search", map[string]any{"query": "a"}, 1) + changed := mustRecord(t, m, "search", map[string]any{"query": "b"}, 2) + if changed.Streak != 1 || changed.Verdict != nil { + t.Fatalf("changed arguments = %+v, want a reset to streak 1", changed) + } +} + +// --------------------------------------------------------------------------- +// #89: fan-out rounds +// --------------------------------------------------------------------------- + +func declare(t *testing.T, m *DoomLoopMonitor, round int, toolName string, argSets ...any) { + t.Helper() + calls := make([]DoomLoopRoundCall, 0, len(argSets)) + for _, args := range argSets { + calls = append(calls, DoomLoopRoundCall{ToolName: toolName, KeyMaterial: args}) + } + m.DeclareRound(round, calls) +} + +// The #89 headline: read(a), read(b), read(c) reissued verbatim used to +// produce zero detections because each round's first call reset the streak. +// Now the round's fingerprint *set* is the identity. +func TestRepeatedFanOutAccumulatesAsOneUnit(t *testing.T) { + m := defaultMonitor(t) + a := map[string]any{"path": "a"} + b := map[string]any{"path": "b"} + c := map[string]any{"path": "c"} + + for round := 0; round < 3; round++ { + declare(t, m, round, "read", a, b, c) + var records []DoomLoopCallRecord + for _, args := range []any{a, b, c} { + records = append(records, mustRecord(t, m, "read", args, round)) + } + for i, record := range records { + if record.Streak != round+1 { + t.Fatalf("round %d call %d streak = %d, want %d: every call reports the round's streak", round, i, record.Streak, round+1) + } + } + switch round { + case 0: + for _, record := range records { + if record.Verdict != nil { + t.Fatalf("round 0 must be free, got %+v", record.Verdict) + } + } + case 1: + for _, record := range records { + if record.Verdict == nil || record.Verdict.Action != DoomLoopActionObserve { + t.Fatalf("round 1 = %+v, want observe", record.Verdict) + } + } + case 2: + // At the block rung EVERY call of the repeating round is refused, + // so the fan-out stops spending — not just its last member. + for i, record := range records { + if record.Verdict == nil || record.Verdict.Action != DoomLoopActionBlock { + t.Fatalf("round 2 call %d = %+v, want block", i, record.Verdict) + } + } + // One shared message per round, so the steer rung's exact-text + // dedupe collapses a round to one correction. + if records[0].Verdict.Message != records[2].Verdict.Message { + t.Fatalf("fan-out members must share one verdict message:\n %q\n %q", records[0].Verdict.Message, records[2].Verdict.Message) + } + if !strings.Contains(records[0].Verdict.Message, "same set of 3 parallel calls") { + t.Fatalf("fan-out message should name the round set: %q", records[0].Verdict.Message) + } + } + } +} + +// Declaration order within a round must not affect scoring. +func TestFanOutScoringIsOrderIndependent(t *testing.T) { + m := defaultMonitor(t) + a := map[string]any{"path": "a"} + b := map[string]any{"path": "b"} + + declare(t, m, 0, "read", a, b) + mustRecord(t, m, "read", a, 0) + mustRecord(t, m, "read", b, 0) + + // Same set, reversed emission order. + declare(t, m, 1, "read", b, a) + first := mustRecord(t, m, "read", b, 1) + second := mustRecord(t, m, "read", a, 1) + if first.Streak != 2 || second.Streak != 2 { + t.Fatalf("reversed order streaks = %d/%d, want 2/2", first.Streak, second.Streak) + } +} + +// A superset round adds work, so the ROUND streak resets — but each repeated +// member keeps its own per-call count, and the new call always executes. +func TestSupersetRoundFlagsRepeatedMembersAndSparesTheNewCall(t *testing.T) { + config := ResolveDoomLoopOption(true) + m := NewDoomLoopMonitor(*config, nil) + a := map[string]any{"path": "a"} + b := map[string]any{"path": "b"} + c := map[string]any{"path": "c"} + + for round := 0; round < 2; round++ { + declare(t, m, round, "read", a, b) + mustRecord(t, m, "read", a, round) + mustRecord(t, m, "read", b, round) + } + declare(t, m, 2, "read", a, b, c) + recA := mustRecord(t, m, "read", a, 2) + recB := mustRecord(t, m, "read", b, 2) + recC := mustRecord(t, m, "read", c, 2) + + if recA.Streak != 3 || recB.Streak != 3 { + t.Fatalf("repeated members streaks = %d/%d, want 3/3 from the per-call detector", recA.Streak, recB.Streak) + } + if recA.Verdict == nil || recA.Verdict.Action != DoomLoopActionBlock { + t.Fatalf("repeated member = %+v, want block", recA.Verdict) + } + if recC.Streak != 1 || recC.Verdict != nil { + t.Fatalf("the new call = %+v, want streak 1 and no verdict — it is real progress", recC) + } + // The per-call text carries neither a fingerprint nor an exact count, so + // every per-call verdict of one tool renders identically. + if recA.Verdict.Message != recB.Verdict.Message { + t.Fatalf("per-call verdicts must share one message:\n %q\n %q", recA.Verdict.Message, recB.Verdict.Message) + } + if strings.Contains(recA.Verdict.Message, recA.Fingerprint[:16]) { + t.Fatalf("a per-call verdict must not quote the call's fingerprint: %q", recA.Verdict.Message) + } +} + +// [a,b], [a,c], [a,d]: every round set differs, yet `a` is a 3-peat. This is +// the case the round-set detector alone cannot see. +func TestPerCallStreakCatchesARepeatInsideVaryingCompany(t *testing.T) { + config := ResolveDoomLoopOption(true) + m := NewDoomLoopMonitor(*config, nil) + anchor := map[string]any{"path": "README"} + companions := []any{ + map[string]any{"path": "b"}, + map[string]any{"path": "c"}, + map[string]any{"path": "d"}, + } + var last DoomLoopCallRecord + for round, companion := range companions { + declare(t, m, round, "read", anchor, companion) + last = mustRecord(t, m, "read", anchor, round) + fresh := mustRecord(t, m, "read", companion, round) + if fresh.Streak != 1 { + t.Fatalf("round %d companion streak = %d, want 1", round, fresh.Streak) + } + } + if last.Streak != 3 { + t.Fatalf("anchor streak = %d, want 3 (its own consecutive rounds)", last.Streak) + } + if last.Verdict == nil || last.Verdict.Action != DoomLoopActionBlock { + t.Fatalf("anchor verdict = %+v, want block at the default rung", last.Verdict) + } +} + +// A partial repeat ([a,b,c] then [a,b]) flags the re-issued calls rather than +// being invisible. +func TestPartialRepeatFlagsTheReIssuedCalls(t *testing.T) { + config := ResolveDoomLoopOption(true) + m := NewDoomLoopMonitor(*config, nil) + a := map[string]any{"path": "a"} + b := map[string]any{"path": "b"} + c := map[string]any{"path": "c"} + + declare(t, m, 0, "read", a, b, c) + for _, args := range []any{a, b, c} { + mustRecord(t, m, "read", args, 0) + } + declare(t, m, 1, "read", a, b) + recA := mustRecord(t, m, "read", a, 1) + if recA.Streak != 2 || recA.Verdict == nil || recA.Verdict.Action != DoomLoopActionObserve { + t.Fatalf("re-issued call in a shrunk round = %+v, want streak 2 observing", recA) + } +} + +// An exactly-repeating round has equal round and per-call counts, so nothing +// double-fires and the round-set message wins. +func TestExactlyRepeatingRoundDoesNotDoubleFire(t *testing.T) { + m := defaultMonitor(t) + a := map[string]any{"path": "a"} + b := map[string]any{"path": "b"} + for round := 0; round < 2; round++ { + declare(t, m, round, "read", a, b) + mustRecord(t, m, "read", a, round) + mustRecord(t, m, "read", b, round) + } + record := mustRecord(t, m, "read", a, 1) + if !record.DuplicateInRound { + t.Fatal("re-recording a member of the same round is a duplicate") + } + if record.Verdict != nil && !strings.Contains(record.Verdict.Message, "parallel calls") { + t.Fatalf("the round-set message should win a tie: %q", record.Verdict.Message) + } +} + +// An undeclared multi-call round still gets order-independent PER-CALL +// detection: that is the path server-tool records and direct consumers take. +func TestUndeclaredRoundStillGetsPerCallDetection(t *testing.T) { + config := ResolveDoomLoopOption(true) + m := NewDoomLoopMonitor(*config, nil) + a := map[string]any{"path": "a"} + b := map[string]any{"path": "b"} + var last DoomLoopCallRecord + for round := 0; round < 3; round++ { + mustRecord(t, m, "read", a, round) + last = mustRecord(t, m, "read", b, round) + } + if last.Streak < 3 { + t.Fatalf("undeclared repeat streak = %d, want at least 3 from per-call evidence", last.Streak) + } + if last.Verdict == nil || !strings.Contains(last.Verdict.Message, "this exact") { + t.Fatalf("undeclared verdict should use the per-call message: %+v", last.Verdict) + } +} + +// A call the declaration could not include (unhashable key material) cannot +// inherit or move the round's counters, but its own repetition still counts. +func TestUnhashableCallIsANonMemberOfItsRound(t *testing.T) { + config := ResolveDoomLoopOption(true) + m := NewDoomLoopMonitor(*config, nil) + good := map[string]any{"path": "a"} + + for round := 0; round < 3; round++ { + // The engine's declaration drops the unhashable call with a warning. + m.DeclareRound(round, []DoomLoopRoundCall{ + {ToolName: "read", KeyMaterial: good}, + {ToolName: "read", KeyMaterial: map[string]any{"v": math.NaN()}}, + }) + mustRecord(t, m, "read", good, round) + } + // The declared member kept a clean round-set streak despite the excluded + // sibling. + record := mustRecord(t, m, "read", good, 3) + if record.Streak != 4 { + t.Fatalf("declared member streak = %d, want 4", record.Streak) + } + // And the unhashable call itself surfaces as an error the engine must + // catch and fall back from. + if _, err := m.RecordToolCall("read", map[string]any{"v": math.NaN()}, 3, RecordOptions{}); !errors.Is(err, ErrUnhashableKeyMaterial) { + t.Fatalf("recording unhashable material must return a catchable error, got %v", err) + } +} + +// --------------------------------------------------------------------------- +// Persistence +// --------------------------------------------------------------------------- + +// A repeating fan-out keeps its evidence across a save/resume boundary, so an +// approval pause no longer resets a fan-out sitting at the block rung. +func TestFanOutEvidenceSurvivesSaveAndResume(t *testing.T) { + config := ResolveDoomLoopOption(true) + m := NewDoomLoopMonitor(*config, nil) + a := map[string]any{"path": "a"} + b := map[string]any{"path": "b"} + for round := 0; round < 2; round++ { + declare(t, m, round, "read", a, b) + mustRecord(t, m, "read", a, round) + mustRecord(t, m, "read", b, round) + } + state := m.State() + entry, ok := state.Tools["read"] + if !ok { + t.Fatal("state must carry the read tool's streak") + } + if entry.Streak != 2 { + t.Fatalf("persisted streak = %d, want 2", entry.Streak) + } + if len(entry.RoundFingerprints) != 2 { + t.Fatalf("a multi-call round must persist its full set, got %v", entry.RoundFingerprints) + } + + // Round-trip through JSON, as a StateAccessor would. + blob, err := json.Marshal(state) + if err != nil { + t.Fatal(err) + } + var restored DoomLoopSerializedState + if err := json.Unmarshal(blob, &restored); err != nil { + t.Fatal(err) + } + resumed := NewDoomLoopMonitor(*config, restored) + declare(t, resumed, 0, "read", a, b) + record := mustRecord(t, resumed, "read", a, 0) + if record.Streak != 3 { + t.Fatalf("resumed streak = %d, want 3: the fan-out's evidence must survive the pause", record.Streak) + } + if record.Verdict == nil || record.Verdict.Action != DoomLoopActionBlock { + t.Fatalf("resumed verdict = %+v, want block", record.Verdict) + } +} + +// Because the streak travels with the exact set that earned it, a resumed +// round containing only a SUBSET is a different round and starts at 1 — a +// lesser call can never inherit a fan-out's evidence. +func TestResumedSubsetRoundCannotInheritAFanOutStreak(t *testing.T) { + config := ResolveDoomLoopOption(true) + m := NewDoomLoopMonitor(*config, nil) + a := map[string]any{"path": "a"} + b := map[string]any{"path": "b"} + c := map[string]any{"path": "c"} + for round := 0; round < 3; round++ { + declare(t, m, round, "read", a, b, c) + for _, args := range []any{a, b, c} { + mustRecord(t, m, "read", args, round) + } + } + resumed := NewDoomLoopMonitor(*config, m.State()) + // The resumed round is a strict subset plus a brand-new call, so its set + // differs from the one that earned the streak. + d := map[string]any{"path": "d"} + declare(t, resumed, 0, "read", a, d) + subsetMember := mustRecord(t, resumed, "read", a, 0) + fresh := mustRecord(t, resumed, "read", d, 0) + + // A call that was never in the persisted set starts clean: it cannot + // inherit a fan-out's evidence. + if fresh.Streak != 1 || fresh.Verdict != nil { + t.Fatalf("a call absent from the persisted set = %+v, want streak 1 with no verdict", fresh) + } + // The surviving member keeps only its OWN per-call evidence (4 + // consecutive rounds), and its verdict says so — it must not be + // attributed to the 3-call round set, which this round is not. + if subsetMember.Streak != 4 { + t.Fatalf("surviving member streak = %d, want its own per-call count of 4", subsetMember.Streak) + } + if subsetMember.Verdict == nil || strings.Contains(subsetMember.Verdict.Message, "parallel calls") { + t.Fatalf("a subset round must not quote the fan-out's round identity: %+v", subsetMember.Verdict) + } +} + +// A pre-#89 blob (no RoundFingerprints, no CallStreaks) restores with its old +// single-call semantics rather than being dropped. +func TestRestoreOfAPre89BlobKeepsSingleCallSemantics(t *testing.T) { + config := ResolveDoomLoopOption(true) + args := map[string]any{"query": "go"} + fingerprint, err := FingerprintToolCall("search", args) + if err != nil { + t.Fatal(err) + } + legacy := fmt.Sprintf(`{"tools":{"search":{"fingerprint":%q,"streak":2}}}`, fingerprint) + m := NewDoomLoopMonitor(*config, legacy) + record := mustRecord(t, m, "search", args, 0) + if record.Streak != 3 { + t.Fatalf("restored legacy streak continued at %d, want 3", record.Streak) + } +} + +// A corrupt blob must degrade rather than take down a resumed run. +func TestRestoreOfACorruptBlobIsIgnored(t *testing.T) { + config := ResolveDoomLoopOption(true) + for _, blob := range []any{"not json", []string{"array"}, 42} { + m := NewDoomLoopMonitor(*config, blob) + if got := m.State(); len(got.Tools) != 0 { + t.Fatalf("corrupt blob %v restored state %+v, want an empty detector", blob, got) + } + } + // A negative streak is caller-writable garbage: clamped, not trusted to + // hold the guardrail below every rung. + m := NewDoomLoopMonitor(*config, `{"tools":{"search":{"fingerprint":"abc","streak":-5}}}`) + if got := m.State().Tools["search"].Streak; got < 1 { + t.Fatalf("restored streak = %d, want it clamped to at least 1", got) + } +} + +// --------------------------------------------------------------------------- +// Escalation budget +// --------------------------------------------------------------------------- + +func TestEscalationBudgetIsConsumedOnApplicationAndPersists(t *testing.T) { + config := ResolveDoomLoopOption(DoomLoopConfig{ + Ladder: testLadder(DoomLoopAt(2), DoomLoopOff(), DoomLoopAt(3), DoomLoopOff(), DoomLoopOff()), + Escalation: &DoomLoopEscalationConfig{Model: "openai/strong", MaxEscalations: 1}, + }) + m := NewDoomLoopMonitor(*config, nil) + if !m.CanEscalate() { + t.Fatal("a configured escalation with budget left must be available") + } + args := map[string]any{"query": "go"} + mustRecord(t, m, "search", args, 0) + mustRecord(t, m, "search", args, 1) + third := mustRecord(t, m, "search", args, 2) + if third.Verdict == nil || third.Verdict.Action != DoomLoopActionEscalate { + t.Fatalf("streak 3 = %+v, want escalate", third.Verdict) + } + + // Budget is consumed when the ENGINE applies the recovery, not at verdict + // time — a verdict the engine does not honor must not burn budget. + m.ConsumeEscalation() + if m.CanEscalate() { + t.Fatal("the budget of 1 must be exhausted after one applied recovery") + } + fourth := mustRecord(t, m, "search", args, 3) + if fourth.Verdict == nil || fourth.Verdict.Action != DoomLoopActionObserve { + t.Fatalf("exhausted escalation must fall through to the weaker rung, got %+v", fourth.Verdict) + } + + // A resume cannot reset the budget. + state := m.State() + if state.EscalationsUsed != 1 { + t.Fatalf("EscalationsUsed = %d, want 1 persisted", state.EscalationsUsed) + } + resumed := NewDoomLoopMonitor(*config, state) + if resumed.CanEscalate() { + t.Fatal("a resumed run must not regain escalation budget") + } +} + +// --------------------------------------------------------------------------- +// Text detectors +// --------------------------------------------------------------------------- + +func TestDetectTextRepetition(t *testing.T) { + for _, tc := range []struct { + name string + text string + wantNil bool + wantRepeats int + wantPeriod int + wantSample string + customConfig *DoomLoopTextOptions + }{ + { + // Ties on covered tokens prefer the smallest period, so the + // repeat count that feeds the ladder is the larger one. + name: "single-token repetition prefers period 1", text: "no no no no no no", + wantRepeats: 6, wantPeriod: 1, wantSample: "no", + customConfig: &DoomLoopTextOptions{MinCoveredTokens: 6}, + }, + { + name: "phrase repetition", text: "I am stuck. I am stuck. I am stuck. I am stuck.", + wantRepeats: 4, wantPeriod: 3, wantSample: "I am stuck.", + }, + { + name: "no repetition", text: "a coherent answer with plenty of distinct tokens in it indeed", + wantNil: true, + }, + { + name: "too few repeats", text: "loop loop and then something else entirely different here now", + wantNil: true, + }, + } { + t.Run(tc.name, func(t *testing.T) { + got := DetectTextRepetition(tc.text, tc.customConfig) + if tc.wantNil { + if got != nil { + t.Fatalf("expected no repetition, got %+v", got) + } + return + } + if got == nil { + t.Fatal("expected a repetition result") + } + if got.Repeats != tc.wantRepeats || got.PeriodTokens != tc.wantPeriod { + t.Fatalf("repeats=%d period=%d, want %d/%d (%+v)", got.Repeats, got.PeriodTokens, tc.wantRepeats, tc.wantPeriod, got) + } + if got.Sample != tc.wantSample { + t.Fatalf("sample = %q, want %q", got.Sample, tc.wantSample) + } + }) + } +} + +// The within-response detector can stop a run immediately: a single response +// spinning the same block dozens of times is a loop on its own. +func TestTextRepetitionVerdictFallsThroughBlockToStop(t *testing.T) { + config := ResolveDoomLoopOption(true) + m := NewDoomLoopMonitor(*config, nil) + verdict := m.RecordAssistantText(strings.Repeat("I am stuck. ", 8)) + if verdict == nil { + t.Fatal("expected a text verdict") + } + if verdict.Detector != DoomLoopDetectorTextRepetition { + t.Fatalf("detector = %q, want text-repetition", verdict.Detector) + } + // Block is meaningless for text (the tokens are already emitted), so the + // streak of 8 lands on stop rather than block. + if verdict.Action != DoomLoopActionStop { + t.Fatalf("action = %q, want stop (block is not available for text)", verdict.Action) + } +} + +func TestCrossStepTextStreakAccumulatesAndResets(t *testing.T) { + config := ResolveDoomLoopOption(true) + m := NewDoomLoopMonitor(*config, nil) + // Whitespace-normalized identity: the same sentence with different + // spacing is the same text. + if v := m.RecordAssistantText("Retrying the same plan."); v != nil { + t.Fatalf("first occurrence = %+v, want no verdict", v) + } + v := m.RecordAssistantText("Retrying the same\nplan.") + if v == nil || v.Detector != DoomLoopDetectorTextStreak || v.Streak != 2 { + t.Fatalf("second occurrence = %+v, want a text-streak verdict at streak 2", v) + } + // Empty/whitespace-only text is a no-op: it neither counts nor resets. + if got := m.RecordAssistantText(" "); got != nil { + t.Fatalf("whitespace-only text = %+v, want nil", got) + } + again := m.RecordAssistantText("Retrying the same plan.") + if again == nil || again.Streak != 3 { + t.Fatalf("streak after a whitespace-only turn = %+v, want 3", again) + } + // A late async-tool delivery is forward progress, so the engine resets + // the text streak (tool streaks are deliberately kept). + m.ResetTextStreak() + if got := m.RecordAssistantText("Retrying the same plan."); got != nil { + t.Fatalf("after ResetTextStreak = %+v, want a fresh streak with no verdict", got) + } +} + +func TestTextDetectorsRespectTheMasterSwitch(t *testing.T) { + config := ResolveDoomLoopOption(DoomLoopConfig{Text: &DoomLoopTextOptions{Disabled: true}}) + m := NewDoomLoopMonitor(*config, nil) + if v := m.RecordAssistantText(strings.Repeat("stuck ", 20)); v != nil { + t.Fatalf("text detection is off, got %+v", v) + } +} + +// --------------------------------------------------------------------------- +// Concurrency +// --------------------------------------------------------------------------- + +// The engine records a round's calls while a consumer may snapshot state, so +// the monitor is documented as concurrency-safe. A data race here would +// silently corrupt detection, which `go test -race` is the only thing that +// catches. +func TestMonitorIsSafeForConcurrentUse(t *testing.T) { + config := ResolveDoomLoopOption(true) + m := NewDoomLoopMonitor(*config, nil) + var wg sync.WaitGroup + for worker := 0; worker < 8; worker++ { + wg.Add(1) + go func(worker int) { + defer wg.Done() + for round := 0; round < 20; round++ { + if _, err := m.RecordToolCall("read", map[string]any{"path": worker}, round, RecordOptions{}); err != nil { + t.Errorf("record: %v", err) + return + } + _ = m.State() + m.RecordAssistantText(fmt.Sprintf("worker %d step %d", worker, round)) + _ = m.CanEscalate() + } + }(worker) + } + wg.Wait() + if len(m.State().Tools) != 1 { + t.Fatalf("state = %+v, want exactly one tool entry", m.State().Tools) + } +} + +// --------------------------------------------------------------------------- +// Tool declaration surface +// --------------------------------------------------------------------------- + +// A tool's LoopKey declaration must be reachable from the plain Tool +// interface, which is how the engine consults it. +func TestToolLoopKeyOfReadsTheToolDeclaration(t *testing.T) { + plain := MustNewTool(ToolConfig[sampleInput]{Name: "plain", Execute: func(context.Context, sampleInput, ToolExecuteContext) (any, error) { return "ok", nil }}) + if got := ToolLoopKeyOf(plain); got != nil { + t.Fatalf("a tool with no declaration = %+v, want nil (full arguments are the identity)", got) + } + + exempt := MustNewTool(ToolConfig[sampleInput]{Name: "exempt", LoopKey: &LoopKey{Exempt: true}, + Execute: func(context.Context, sampleInput, ToolExecuteContext) (any, error) { return "ok", nil }}) + declared := ToolLoopKeyOf(exempt) + if declared == nil || !declared.Exempt { + t.Fatalf("declared LoopKey = %+v, want the statically exempt declaration", declared) + } + if got := ResolveLoopKeyMaterial(declared, map[string]any{"query": "go"}); got.Kind != LoopKeyExempt { + t.Fatalf("resolution = %+v, want exempt", got) + } + + var seen map[string]any + computed := MustNewTool(ToolConfig[sampleInput]{Name: "computed", + LoopKey: &LoopKey{Fn: func(args map[string]any) (any, error) { seen = args; return args["query"], nil }}, + Execute: func(context.Context, sampleInput, ToolExecuteContext) (any, error) { return "ok", nil }}) + resolution := ResolveLoopKeyMaterial(ToolLoopKeyOf(computed), map[string]any{"query": "go"}) + if resolution.Kind != LoopKeyResolved || resolution.KeyMaterial != "go" { + t.Fatalf("computed resolution = %+v, want the returned identity", resolution) + } + if seen["query"] != "go" { + t.Fatalf("LoopKey saw %v, want the validated arguments", seen) + } + + // ToolLoopKeyOf must not panic on a nil tool: PartitionToolCalls hands + // unresolved names through as nil. + if got := ToolLoopKeyOf(nil); got != nil { + t.Fatalf("ToolLoopKeyOf(nil) = %+v, want nil", got) + } +} diff --git a/doom_loop_vectors_test.go b/doom_loop_vectors_test.go new file mode 100644 index 0000000..2263d39 --- /dev/null +++ b/doom_loop_vectors_test.go @@ -0,0 +1,145 @@ +package agent + +// Cross-port doom-loop fingerprint conformance vectors, copied verbatim from +// upstream `packages/agent/tests/vectors/doom-loop-fingerprints.json`. +// +// Embedded as a Go raw string rather than shipped as a testdata file: the +// port's output shape is a flat, Go-only package at the repo root, and the +// verifier's leaked-upstream-artifact check exists to keep it that way. The +// vectors are the whole point of the file — the TypeScript, Python and Go +// ports must produce byte-identical canonical forms and digests for the same +// key material, so if this diverges from upstream the ports have silently +// forked their persisted detector state. +const doomLoopFingerprintVectors = `{ + "description": "Cross-port doom-loop fingerprint conformance vectors. fingerprint = sha256(utf8(toolName + \"\\n\" + jcs(keyMaterial))) for tool calls, sha256(utf8(jcs(keyMaterial))) for bare key material. jcs = RFC 8785 canonical JSON. Ports MUST use an RFC 8785 implementation (pip jcs, cyberphone/json-canonicalization), NOT their stdlib JSON serializer. All hex lowercase.", + "toolCallVectors": [ + { + "name": "basic bash identity", + "toolName": "bash", + "keyMaterial": { + "command": "ls -la", + "cwd": "/tmp" + }, + "jcs": "{\"command\":\"ls -la\",\"cwd\":\"/tmp\"}", + "fingerprint": "c3430466c9a1a7c11c8e23328fcf3057189c44d86839527ac6e61bb4574410d4" + }, + { + "name": "key order insensitivity (same fingerprint as basic bash identity)", + "toolName": "bash", + "keyMaterial": { + "cwd": "/tmp", + "command": "ls -la" + }, + "jcs": "{\"command\":\"ls -la\",\"cwd\":\"/tmp\"}", + "fingerprint": "c3430466c9a1a7c11c8e23328fcf3057189c44d86839527ac6e61bb4574410d4" + }, + { + "name": "empty object (repeated empty call)", + "toolName": "list_tasks", + "keyMaterial": {}, + "jcs": "{}", + "fingerprint": "b84a3e324e9dc95335f72cfee5e5898465349be72cb1348fe7927c49c8d65d92" + }, + { + "name": "non-ascii + emoji", + "toolName": "web_search", + "keyMaterial": { + "query": "café 🎉" + }, + "jcs": "{\"query\":\"café 🎉\"}", + "fingerprint": "a8bd7e02e874cd1d68db436c3228a42ed2d656ddd30d3195db0e72378479b160" + }, + { + "name": "negative zero collapses to 0 (JCS)", + "toolName": "calc", + "keyMaterial": { + "value": 0 + }, + "jcs": "{\"value\":0}", + "fingerprint": "2c0292f2f8e9099f2879c189b8bd0d61fdc24dea98f06ec1d96f17f8ccb86f7d" + }, + { + "name": "large magnitude number 1e21 (JCS exponent form)", + "toolName": "calc", + "keyMaterial": { + "value": 1e+21 + }, + "jcs": "{\"value\":1e+21}", + "fingerprint": "53f17cc32c78fee57f8e6aba17c4d5f75ca2aa5a1d06d359880c584993dcaaff" + }, + { + "name": "nested structures with arrays", + "toolName": "query", + "keyMaterial": { + "filter": { + "tags": [ + "b", + "a" + ], + "depth": 2 + }, + "sort": null + }, + "jcs": "{\"filter\":{\"depth\":2,\"tags\":[\"b\",\"a\"]},\"sort\":null}", + "fingerprint": "b42e583c626c3b2bcbd5adf86b7099d784ef15398f2e0c530262372e73850f4e" + }, + { + "name": "lone surrogate escapes as \\ud800 (JSON.stringify)", + "toolName": "echo", + "keyMaterial": { + "s": "\ud800" + }, + "jcs": "{\"s\":\"\\ud800\"}", + "fingerprint": "9d901512fe3c9139d48aeaec555bc91c5b8069950bc17a6d014c85bf03c0d8eb" + }, + { + "name": "unicode normalization NOT applied (NFC vs NFD differ)", + "toolName": "echo", + "keyMaterial": { + "s": "é" + }, + "jcs": "{\"s\":\"é\"}", + "fingerprint": "a281d1b32556677dee1b6039b78cde21229a6777874c6dc8eafc4bff64058568" + }, + { + "name": "string primitive key material", + "toolName": "web_search", + "keyMaterial": "openrouter agent sdk", + "jcs": "\"openrouter agent sdk\"", + "fingerprint": "dd066597b3c639cd2c905159c3ee5bda98c29add4deef7d20f723583c1e7e177" + } + ], + "keyMaterialVectors": [ + { + "name": "plain phrase", + "keyMaterial": "I am stuck.", + "jcs": "\"I am stuck.\"", + "fingerprint": "37c38ab53f4c3570d8baf87392cb035ef8b2ad98ca1c1e083ea84184173911aa" + }, + { + "name": "whitespace-normalized cross-step text", + "keyMaterial": "Retrying the same plan.", + "jcs": "\"Retrying the same plan.\"", + "fingerprint": "764fb75d0e16f75f0265b68d579c571e704b6ce9ea2b5dea784200cb701fe3df" + } + ], + "rejected": [ + { + "name": "bigint", + "reason": "RFC 8785 has no representation; canonicalize throws, engine falls back to full arguments" + }, + { + "name": "NaN / Infinity", + "reason": "non-finite numbers unrepresentable; throws, engine falls back" + }, + { + "name": "circular reference", + "reason": "throws, engine falls back" + }, + { + "name": "nesting > 64 levels", + "reason": "depth cap; throws, engine falls back" + } + ] +} +` diff --git a/hooks_manager.go b/hooks_manager.go index 889ae7a..97fb35f 100644 --- a/hooks_manager.go +++ b/hooks_manager.go @@ -390,4 +390,12 @@ func (m *HooksManager) EmitPostModelCall(payload PostModelCallPayload, opts Emit return emitTyped[PostModelCallPayload, EmptyHookResult](m, string(HookNamePostModelCall), payload, opts) } +func (m *HooksManager) OnDoomLoopDetected(entry HookEntry[DoomLoopDetectedPayload, DoomLoopDetectedResult]) func() { + return On(m, string(HookNameDoomLoopDetected), entry) +} + +func (m *HooksManager) EmitDoomLoopDetected(payload DoomLoopDetectedPayload, opts EmitOptions) (EmitResult[DoomLoopDetectedResult, DoomLoopDetectedPayload], error) { + return emitTyped[DoomLoopDetectedPayload, DoomLoopDetectedResult](m, string(HookNameDoomLoopDetected), payload, opts) +} + //#endregion diff --git a/hooks_resolve.go b/hooks_resolve.go index c1efe1e..6818d5e 100644 --- a/hooks_resolve.go +++ b/hooks_resolve.go @@ -16,6 +16,7 @@ type InlineHookConfig struct { SessionStart []HookEntry[SessionStartPayload, EmptyHookResult] SessionEnd []HookEntry[SessionEndPayload, EmptyHookResult] PostModelCall []HookEntry[PostModelCallPayload, EmptyHookResult] + DoomLoopDetected []HookEntry[DoomLoopDetectedPayload, DoomLoopDetectedResult] } // ResolveHooks normalizes a CallModelInput.Hooks value into a *HooksManager. @@ -62,6 +63,9 @@ func ResolveHooks(hooks any) *HooksManager { for _, e := range v.PostModelCall { manager.OnPostModelCall(e) } + for _, e := range v.DoomLoopDetected { + manager.OnDoomLoopDetected(e) + } return manager default: log.Printf("[ResolveHooks] Ignoring CallModelInput.Hooks of unsupported type %T; expected *HooksManager or InlineHookConfig.", hooks) diff --git a/hooks_schemas.go b/hooks_schemas.go index 0e167e8..d808102 100644 --- a/hooks_schemas.go +++ b/hooks_schemas.go @@ -16,6 +16,7 @@ const ( HookNameSessionStart HookName = "SessionStart" HookNameSessionEnd HookName = "SessionEnd" HookNamePostModelCall HookName = "PostModelCall" + HookNameDoomLoopDetected HookName = "DoomLoopDetected" ) // builtInHookNames mirrors upstream's BUILT_IN_HOOK_NAMES set, used by @@ -31,9 +32,10 @@ var builtInHookNames = map[string]bool{ string(HookNameSessionStart): true, string(HookNameSessionEnd): true, string(HookNamePostModelCall): true, + string(HookNameDoomLoopDetected): true, } -// IsBuiltInHookName reports whether name is one of the nine built-in hooks. +// IsBuiltInHookName reports whether name is one of the built-in hooks. func IsBuiltInHookName(name string) bool { return builtInHookNames[name] } //#region Payload & Result types @@ -204,6 +206,9 @@ const ( SessionEndReasonError SessionEndReason = "error" SessionEndReasonMaxTurns SessionEndReason = "max_turns" SessionEndReasonComplete SessionEndReason = "complete" + // SessionEndReasonDoomLoop means doom-loop detection halted the run at + // the `stop` ladder rung (upstream #73). + SessionEndReasonDoomLoop SessionEndReason = "doom_loop" ) // SessionEndPayload is delivered exactly once per run that reached @@ -242,3 +247,35 @@ type PostModelCallPayload struct { type EmptyHookResult struct{} //#endregion + +// DoomLoopDetectedPayload is delivered every time a doom-loop detector fires, +// at every ladder rung — including `observe`, which exists precisely so a +// caller can watch for loops without changing the run's behavior. +type DoomLoopDetectedPayload struct { + // Detector says which detector fired. + Detector DoomLoopDetectorKind + // Action is the ladder action the engine resolved for this streak. + Action DoomLoopAction + // Streak is the consecutive repetition count that crossed a rung. + Streak int + // Fingerprint is the deterministic fingerprint of the repeated unit. + Fingerprint string + // ToolName and ToolInput are present for tool-fingerprint verdicts: + // ToolInput carries the repeated call's arguments. + ToolName string + ToolInput map[string]any + // Message is the explanation used for block outputs and steer messages. + Message string +} + +// DoomLoopDetectedResult lets a handler override the engine's resolved action +// for THIS event — de-escalate (observe on a would-be block) or escalate +// (stop immediately). When several handlers override, the last one wins. +// +// Two downgrades are enforced rather than trusted: a text verdict cannot be +// blocked (the tokens are already emitted), and an escalate override is +// honored only when an escalation config exists and budget remains. Both +// downgrade to observe — never silently to a *stronger* action. +type DoomLoopDetectedResult struct { + OverrideAction DoomLoopAction +} diff --git a/model_result.go b/model_result.go index 35dd14f..3a49d9c 100644 --- a/model_result.go +++ b/model_result.go @@ -65,6 +65,13 @@ type ModelResult struct { sessionEndEmitted bool sessionUsage sessionUsageAggregate toolRoundsExecuted int + toolChoice toolChoicePolicy + // haltedByTurnBudget records that the loop stopped because it ran out of + // turns or a StopWhen condition fired, which SessionEnd reports as + // `max_turns` rather than `complete`. + haltedByTurnBudget bool + roundResults map[string]ToolExecutionResult + doom *doomLoopState } func newModelResult(ctx context.Context, client ResponseSender, input CallModelInput, req components.ResponsesRequest, state ConversationState) *ModelResult { @@ -151,10 +158,22 @@ func CallModel(ctx context.Context, client ResponseSender, input CallModelInput) tier := components.ResponsesRequestServiceTierAuto req.ServiceTier = optionalnullable.From(&tier) } + // Narrow tools to the active subset before API conversion *and* before + // they are registered for execution, mirroring upstream: the model must + // not be offered a filtered tool, and the executor must not carry a + // definition the request never advertised. + input.Tools = FilterToolsByIDs(input.Tools, input.ActiveTools) for _, t := range input.Tools { req.Tools = append(req.Tools, t.ToAPITool()) } - return newModelResult(ctx, client, input, req, state), nil + result := newModelResult(ctx, client, input, req, state) + result.initDoomLoop() + // Re-arm the forced-tool-choice policy from persisted state so a resumed + // run does not force the model back into tool calls with a choice it has + // already spent (upstream #100). + result.toolChoice.consumedKey = state.ConsumedForcedToolChoiceKey + result.req.ToolChoice = result.toolChoice.configure(req.ToolChoice) + return result, nil } func inputItemsOnly(input any) ([]components.InputsUnion1, error) { @@ -202,11 +221,7 @@ func (m *ModelResult) run() { // any tool ever executes (a "no-tools" path) is still drained. Never // masks m.err: finishHooksSession only logs its own failures. defer func() { - reason := SessionEndReasonComplete - if m.err != nil { - reason = SessionEndReasonError - } - m.finishHooksSession(reason) + m.finishHooksSession(m.sessionEndReason()) }() if m.client == nil { return @@ -228,6 +243,21 @@ func (m *ModelResult) run() { // PostModelCall/PreToolUse/PostToolUse still fire during the resume via // their own call sites below, and finishHooksSession's unconditional // defer still drains any pending async hook work. + // A conversation condemned by doom-loop detection stays halted: the stop + // rung exists to stop spending, so a resume under an armed verdict must + // not dispatch — not even the approval-resume request. Only a fresh + // conversational turn clears the verdict (see initDoomLoop's + // condemnation rule), and that path never reaches here with one armed. + if m.doom != nil && m.doom.stop != nil { + m.state.Status = ConversationStatusComplete + m.syncDoomLoopState() + if m.input.StateAccessor != nil { + if err := m.input.StateAccessor.Save(m.ctx, m.state); err != nil { + m.err = err + } + } + return + } if isAwaitingResume(m.state) && (len(m.input.ApproveToolCalls) > 0 || len(m.input.RejectToolCalls) > 0) { if m.hooksManager != nil { m.hooksManager.SetSessionID(m.state.ID) @@ -246,6 +276,7 @@ func (m *ModelResult) run() { // calls pending. Without this, a StateAccessor-backed caller // that reloads on the next CallModel call would see the stale // pre-resume state and could re-execute an already-decided call. + m.syncDoomLoopState() if m.input.StateAccessor != nil { if err := m.input.StateAccessor.Save(m.ctx, m.state); err != nil { m.err = err @@ -261,7 +292,17 @@ func (m *ModelResult) run() { return } } + turnsUsed := 0 for turn := 0; turn < maxTurns; turn++ { + turnsUsed = turn + 1 + // A condemned run halts *before* the next model request: the whole + // point of the stop rung is to stop spending. An armed verdict + // restored from state (a decision-only resume of a doom-stopped + // conversation) therefore never dispatches. + if m.doom != nil && m.doom.stop != nil { + m.state.Status = ConversationStatusComplete + break + } if m.input.BeforeTurn != nil { if err := m.input.BeforeTurn(m.ctx, BuildTurnContext(nil, turn, &req)); err != nil { m.err = err @@ -269,8 +310,20 @@ func (m *ModelResult) run() { } } m.fullStream.Push(ResponseStreamEvent{Type: "turn.start", Turn: turn}) + m.mu.Lock() + m.roundResults = map[string]ToolExecutionResult{} + m.mu.Unlock() turnStartedAt := time.Now() - res, err := m.client.SendResponse(m.ctx, req, m.input.MetadataLevel, operations.WithSetHeaders(map[string]string{"x-openrouter-callmodel": "true"})) + // One-turn escalation overrides (a stronger model and/or a forced + // advisor consult) apply to THIS dispatch only; req itself keeps the + // configured model and tools, so the next turn reverts automatically. + dispatch, escalated := m.takeDoomLoopEscalationOverrides(req) + // An engine-owned override (a forced advisor consult) replaces the + // caller's tool choice on the wire, so that dispatch cannot speak for + // the caller's policy: it may neither consume a forced choice nor + // clear a consumed one. + callerChoiceDispatched := !escalated || dispatch.ToolChoice == req.ToolChoice + res, err := m.client.SendResponse(m.ctx, dispatch, m.input.MetadataLevel, operations.WithSetHeaders(map[string]string{"x-openrouter-callmodel": "true"})) if err != nil { m.err = err return @@ -303,7 +356,19 @@ func (m *ModelResult) run() { } m.fullStream.Push(ResponseStreamEvent{Type: "response.completed", Turn: turn, Response: &resp}) m.fullStream.Push(ResponseStreamEvent{Type: "turn.end", Turn: turn}) + // Step-level doom-loop checkpoint: text repetition, the cross-step + // text streak, and server-tool fingerprints (post-execution, so + // blocking one is meaningless). + m.checkDoomLoopForResponse(resp) calls := ExtractToolCallsFromResponse(resp) + // A forced tool choice is spent only by a response that actually + // called a tool; commit before any pause so a resumed run agrees. + if callerChoiceDispatched { + m.toolChoice.commit(len(calls) > 0) + } else { + m.toolChoice.abandonDispatch() + } + m.state.ConsumedForcedToolChoiceKey = m.toolChoice.consumedKey for _, call := range calls { m.toolCallStream.Push(call) } @@ -352,6 +417,10 @@ func (m *ModelResult) run() { forceResumeCount++ } else { forceResumeCount = 0 + // A StopWhen condition halting a tool-call turn is upstream's + // `max_turns` session end, whether or not the forced final + // response turn runs afterwards. + m.haltedByTurnBudget = true if !allowFinalResponseEnabled(m.input.AllowFinalResponse) { m.state.Status = ConversationStatusComplete break @@ -410,6 +479,18 @@ func (m *ModelResult) run() { break } } + // Declare the round's complete call set before any of it is scored, so + // a repeating fan-out is measured as one unit and scoring does not + // depend on emission order (upstream #89). + m.beginDoomLoopRound(calls) + if m.doom != nil && m.doom.stop != nil { + // A stop armed by the step checkpoint (text/server-tool) halts + // before this round executes; unresolved calls get synthesized + // halt outputs so the persisted history stays well-formed. + m.sealDoomLoopStop(resp, nil) + m.state.Status = ConversationStatusComplete + break + } approved, pending, err := PartitionToolCalls(m.ctx, m.input.Tools, calls, BuildTurnContext(nil, turn, &req), m.input.Approval) if err != nil { m.err = err @@ -447,6 +528,13 @@ func (m *ModelResult) run() { } outputs = append(outputs, deniedOutputs...) if isPausedStatus(m.state.Status) { + m.flushDoomLoopSteer(&req) + break + } + if m.doom != nil && m.doom.stop != nil { + // A stop armed while this round executed: every call already has + // an output (blocked calls included), so nothing is dangling. + m.state.Status = ConversationStatusComplete break } if len(outputs) > 0 { @@ -455,10 +543,30 @@ func (m *ModelResult) run() { // against a later, independent one (mirrors upstream). forceResumeCount = 0 } + // Tool-computed next-turn parameters (upstream #114). A tool-computed + // toolChoice becomes the new caller-level policy rather than a + // one-turn override, so it is re-stamped through the policy — which + // re-arms it as a fresh forced choice when its semantic value changed. + nextReq, choiceChanged, err := m.applyNextTurnParams(calls, req) + if err != nil { + m.err = err + return + } + req = nextReq base := requestInputItems(req.Input) base = append(base, responseItems...) base = append(base, outputs...) req.Input = openrouter.Pointer(components.CreateInputsUnionArrayOfInputsUnion1(base)) + // Queued steer guidance rides into the next turn as a user message. + m.flushDoomLoopSteer(&req) + // A forced tool choice has served its purpose once a tool round has + // executed; keeping it on follow-up turns would forbid the model from + // ever answering in text (upstream #100 / DEV-785). + if choiceChanged { + req.ToolChoice = m.toolChoice.configure(req.ToolChoice) + } else { + req.ToolChoice = m.toolChoice.effective() + } if resp.ID != "" { m.state.PreviousResponseID = &resp.ID } @@ -473,7 +581,15 @@ func (m *ModelResult) run() { // is not on its own a reliable "invalid response" signal the way it is // upstream, where `output` is the sole content carrier. See // upstreamer-changelog.md for this compatibility note. - if m.state.Status == ConversationStatusComplete && m.toolRoundsExecuted > 0 && !m.input.StrictFinalResponse && isEmptyFinalResponse(m.resp) { + // Exhausting the turn budget with the conversation still mid-tool-call is + // the other `max_turns` exit: the loop ran out of turns rather than + // reaching a natural end. + if turnsUsed >= maxTurns && m.state.Status != ConversationStatusComplete && !isPausedStatus(m.state.Status) { + m.haltedByTurnBudget = true + } + // A doom-stopped run must not spend one more request on a cosmetic retry. + doomStopped := m.doom != nil && m.doom.stop != nil + if m.state.Status == ConversationStatusComplete && m.toolRoundsExecuted > 0 && !doomStopped && !m.input.StrictFinalResponse && isEmptyFinalResponse(m.resp) { retryResp, err := m.retryCurrentRequest(req, len(m.steps)+1) if err != nil { m.err = err @@ -490,6 +606,7 @@ func (m *ModelResult) run() { } m.state = appendResponseItemsToState(m.state, retryResp, retryItems) } + m.syncDoomLoopState() if m.input.StateAccessor != nil { if err := m.input.StateAccessor.Save(m.ctx, m.state); err != nil { m.err = err @@ -729,6 +846,7 @@ func (m *ModelResult) executeAutoApproveTools(calls []ParsedToolCall, turn int, return nil, err } m.toolRoundsExecuted++ + m.recordRoundResult(effectiveCall, result) if blocked { results = append(results, CreateRejectedResult(effectiveCall, result.Error.Error())) continue @@ -775,6 +893,20 @@ func (m *ModelResult) executeToolCallsForTurn(calls []ParsedToolCall, turn int, if IsMcpTool(t) { source = ToolSourceMCP } + // Doom-loop pre-execution checkpoint: a condemned call is refused + // with an explanatory error output instead of running, which is both + // the refusal and the steering — delivered where the model looks. + if reason := m.checkDoomLoopBeforeExecution(call); reason != "" { + out, ferr := FormatToolOutputWithError(ToolExecutionResult{CallID: call.CallID, Name: call.Name, Error: errorString(reason)}) + if ferr != nil { + return nil, ferr + } + m.toolStream.Push(ToolStreamEvent{Type: "tool.result", CallID: call.CallID, Name: call.Name, Source: source, Error: reason, Turn: turn}) + outputItem := components.CreateInputsUnion1FunctionCallOutputItem(out) + outputs = append(outputs, outputItem) + m.state = AppendToMessages(m.state, outputItem) + continue + } execCtx := BuildToolExecuteContext(call, BuildTurnContext(nil, turn, req), m.store, func(v any) { m.toolStream.Push(ToolStreamEvent{Type: "tool.preliminary", CallID: call.CallID, Name: call.Name, Source: source, Event: v, Turn: turn}) }) @@ -783,6 +915,7 @@ func (m *ModelResult) executeToolCallsForTurn(calls []ParsedToolCall, turn int, return nil, err } m.toolRoundsExecuted++ + m.recordRoundResult(effectiveCall, result) if blocked { out, ferr := FormatToolOutputWithError(result) if ferr != nil { @@ -974,20 +1107,9 @@ func (m *ModelResult) emitSessionEndOnce(reason SessionEndReason) { m.sessionEndEmitted = true payload := SessionEndPayload{Reason: reason} if m.sessionUsage.modelCalls > 0 { - totals := SessionUsageTotals{ - ModelCallUsage: ModelCallUsage{ - InputTokens: m.sessionUsage.inputTokens, - OutputTokens: m.sessionUsage.outputTokens, - TotalTokens: m.sessionUsage.totalTokens, - CachedTokens: m.sessionUsage.cachedTokens, - ReasoningTokens: m.sessionUsage.reasoningTokens, - }, - ModelCalls: m.sessionUsage.modelCalls, - } - if m.sessionUsage.hasCost { - cost := m.sessionUsage.cost - totals.Cost = &cost - } + m.mu.RLock() + totals := m.snapshotSessionUsage() + m.mu.RUnlock() payload.TotalUsage = &totals } if _, err := m.hooksManager.EmitSessionEnd(payload, m.hookEmitOptions("")); err != nil { @@ -995,6 +1117,28 @@ func (m *ModelResult) emitSessionEndOnce(reason SessionEndReason) { } } +// sessionEndReason maps the run's outcome onto the SessionEnd reason: +// +// error — the run failed. +// doom_loop — doom-loop detection halted it at the stop rung. +// user — the conversation was interrupted. +// max_turns — a StopWhen condition fired, or the turn budget ran out. +// complete — the model produced a final answer. +func (m *ModelResult) sessionEndReason() SessionEndReason { + switch { + case m.err != nil: + return SessionEndReasonError + case m.doom != nil && m.doom.stop != nil: + return SessionEndReasonDoomLoop + case m.state.Status == ConversationStatusInterrupted: + return SessionEndReasonUser + case m.haltedByTurnBudget: + return SessionEndReasonMaxTurns + default: + return SessionEndReasonComplete + } +} + // finishHooksSession emits SessionEnd (if not already emitted) and drains // pending hook work. Never panics: teardown must not mask the run's own // error. Call unconditionally on every exit path (success or error) so @@ -1009,12 +1153,15 @@ func (m *ModelResult) finishHooksSession(reason SessionEndReason) { } // emitPostModelCall emits PostModelCall for a completed model response and -// folds its usage into the session aggregate. One emit per materialized response. +// folds its usage into the session aggregate. One emit per materialized +// response. +// +// Accumulation runs unconditionally — *before* the hooks short-circuit — +// because Usage surfaces these totals to callers who configured no hooks at +// all (upstream #97). Only the hook emit itself is gated on a HooksManager. func (m *ModelResult) emitPostModelCall(resp components.OpenResponsesResult, startedAt time.Time, turnType ModelCallTurnType, turnNumber int) { - if m.hooksManager == nil { - return - } usage := extractModelCallUsage(resp) + m.mu.Lock() m.sessionUsage.modelCalls++ if usage != nil { m.sessionUsage.inputTokens += usage.InputTokens @@ -1027,6 +1174,10 @@ func (m *ModelResult) emitPostModelCall(resp components.OpenResponsesResult, sta m.sessionUsage.hasCost = true } } + m.mu.Unlock() + if m.hooksManager == nil { + return + } payload := PostModelCallPayload{ SessionID: m.state.ID, ResponseID: resp.ID, @@ -1293,6 +1444,82 @@ func toolInputMap(call ParsedToolCall) map[string]any { return map[string]any{} } +// applyNextTurnParams runs the nextTurnParams functions of every tool called +// this turn and returns the request they computed (upstream #114). The second +// return value reports whether the tools set `toolChoice`, which the caller +// re-stamps through the forced-choice policy so a tool-computed choice +// becomes the new caller-level policy rather than a one-turn override. +func (m *ModelResult) applyNextTurnParams(calls []ParsedToolCall, req components.ResponsesRequest) (components.ResponsesRequest, bool, error) { + if len(calls) == 0 || len(m.input.Tools) == 0 { + return req, false, nil + } + m.mu.RLock() + results := m.roundResults + m.mu.RUnlock() + computed, err := ExecuteNextTurnParamsFunctions(m.ctx, m.input.Tools, calls, results, req) + if err != nil { + return req, false, err + } + if len(computed) == 0 { + return req, false, nil + } + _, choiceChanged := computed["toolChoice"] + return ApplyNextTurnParamsToRequest(req, computed), choiceChanged, nil +} + +// recordRoundResult remembers a tool execution result for this turn so +// nextTurnParams functions can read it. +func (m *ModelResult) recordRoundResult(call ParsedToolCall, result ToolExecutionResult) { + m.mu.Lock() + if m.roundResults == nil { + m.roundResults = map[string]ToolExecutionResult{} + } + m.roundResults[call.CallID] = result + m.mu.Unlock() +} + +// Usage reports aggregate token and cost usage across **every** model call +// this run made: the initial request, each tool-round follow-up, the +// empty-final retry, the forced final turn, and approval-resume requests +// (upstream #97 `getUsage()`). +// +// This is the pull-based counterpart to the SessionEnd hook's TotalUsage — +// same shape, same numbers, read from the same snapshot, so the two cannot +// drift. Accumulation is independent of the hook system, so the totals are +// correct for a caller that configured no hooks at all. +// +// Unlike Response it never returns an error: a failed run still consumed +// tokens, and cost accounting usually runs where a second error would mask +// the run's original one. Await the run itself if you need the failure. +// Cost is non-nil only when the server reported cost for at least one call, +// since a 0 would be indistinguishable from "free". +func (m *ModelResult) Usage(ctx context.Context) SessionUsageTotals { + m.ensure() + m.mu.RLock() + defer m.mu.RUnlock() + return m.snapshotSessionUsage() +} + +// snapshotSessionUsage materializes the running aggregate. Callers must hold +// at least a read lock. +func (m *ModelResult) snapshotSessionUsage() SessionUsageTotals { + totals := SessionUsageTotals{ + ModelCallUsage: ModelCallUsage{ + InputTokens: m.sessionUsage.inputTokens, + OutputTokens: m.sessionUsage.outputTokens, + TotalTokens: m.sessionUsage.totalTokens, + CachedTokens: m.sessionUsage.cachedTokens, + ReasoningTokens: m.sessionUsage.reasoningTokens, + }, + ModelCalls: m.sessionUsage.modelCalls, + } + if m.sessionUsage.hasCost { + cost := m.sessionUsage.cost + totals.Cost = &cost + } + return totals +} + func (m *ModelResult) Text(ctx context.Context) (string, error) { m.ensure() return ExtractTextFromResponse(m.resp), m.err diff --git a/model_result_hooks_test.go b/model_result_hooks_test.go index 10c19ac..ed96746 100644 --- a/model_result_hooks_test.go +++ b/model_result_hooks_test.go @@ -862,3 +862,100 @@ func TestStateAccessorPersistsWhenResumeRepauses(t *testing.T) { t.Fatalf("danger2 should have executed exactly once, got %d", d2Executions) } } + +// captureSessionEndReason runs input and returns the reason SessionEnd +// reported. +func captureSessionEndReason(t *testing.T, sender ResponseSender, input CallModelInput) SessionEndReason { + t.Helper() + var reason SessionEndReason + manager := NewHooksManager() + manager.OnSessionEnd(HookEntry[SessionEndPayload, EmptyHookResult]{ + Handler: func(payload SessionEndPayload, _ LifecycleHookContext) (HookHandlerResult[EmptyHookResult], error) { + reason = payload.Reason + return VoidResult[EmptyHookResult](), nil + }, + }) + input.Hooks = manager + result, err := CallModel(context.Background(), sender, input) + if err != nil { + t.Fatal(err) + } + if _, err := result.State(context.Background()); err != nil { + t.Fatal(err) + } + return reason +} + +// SessionEnd.Reason is part of the hook payload contract, and reporting +// "complete" for a run that actually ran out of turns is a wrong value, not a +// missing one: a caller watching for runaway agents would never see them. +func TestSessionEndReasonDistinguishesHowARunEnded(t *testing.T) { + call := components.OutputFunctionCallItem{CallID: "call_1", Name: "search", Arguments: `{"query":"go"}`} + toolTurn := components.OpenResponsesResult{ID: "resp_1", Output: []components.OutputItems{components.CreateOutputItemsFunctionCall(call)}} + + t.Run("complete when the model produces a final answer", func(t *testing.T) { + created := operations.CreateCreateResponsesResponseOpenResponsesResult( + components.OpenResponsesResult{ID: "resp_1", OutputText: openrouter.String("done")}) + sender := &fakeSender{responses: []*operations.CreateResponsesResponse{&created}} + got := captureSessionEndReason(t, sender, CallModelInput{Model: "openai/test", Input: "hi"}) + if got != SessionEndReasonComplete { + t.Fatalf("reason = %q, want complete", got) + } + }) + + t.Run("max_turns when a StopWhen condition halts a tool-call turn", func(t *testing.T) { + sender := twoTurnSender(toolTurn, components.OpenResponsesResult{ID: "resp_2", OutputText: openrouter.String("final")}) + got := captureSessionEndReason(t, sender, CallModelInput{ + Model: "openai/test", Input: "hi", Tools: []Tool{searchTool(t)}, + StopWhen: []StopCondition{StepCountIs(1)}, + }) + if got != SessionEndReasonMaxTurns { + t.Fatalf("reason = %q, want max_turns", got) + } + }) + + t.Run("max_turns when the turn budget runs out mid-loop", func(t *testing.T) { + created := operations.CreateCreateResponsesResponseOpenResponsesResult(toolTurn) + sender := &fakeSender{responses: []*operations.CreateResponsesResponse{&created}} + got := captureSessionEndReason(t, sender, CallModelInput{ + Model: "openai/test", Input: "hi", Tools: []Tool{searchTool(t)}, MaxTurns: 2, + }) + if got != SessionEndReasonMaxTurns { + t.Fatalf("reason = %q, want max_turns", got) + } + }) + + t.Run("error when the run fails", func(t *testing.T) { + created := operations.CreateCreateResponsesResponseOpenResponsesResult(components.OpenResponsesResult{}) + broken := &fakeSender{responses: []*operations.CreateResponsesResponse{&created}} + broken.responses[0] = nil + var reason SessionEndReason + manager := NewHooksManager() + manager.OnSessionEnd(HookEntry[SessionEndPayload, EmptyHookResult]{ + Handler: func(payload SessionEndPayload, _ LifecycleHookContext) (HookHandlerResult[EmptyHookResult], error) { + reason = payload.Reason + return VoidResult[EmptyHookResult](), nil + }, + }) + result, err := CallModel(context.Background(), broken, CallModelInput{Model: "openai/test", Input: "hi", Hooks: manager}) + if err != nil { + t.Fatal(err) + } + if _, err := result.Text(context.Background()); err == nil { + t.Fatal("expected the run to fail") + } + if reason != SessionEndReasonError { + t.Fatalf("reason = %q, want error", reason) + } + }) + + t.Run("user when the conversation was interrupted", func(t *testing.T) { + // A manual tool with no execute path and an interrupted status is the + // caller-driven exit; assert the mapping directly since the engine has + // no synthetic interrupt trigger. + m := &ModelResult{state: ConversationState{Status: ConversationStatusInterrupted}} + if got := m.sessionEndReason(); got != SessionEndReasonUser { + t.Fatalf("reason = %q, want user", got) + } + }) +} diff --git a/next_turn_params.go b/next_turn_params.go index babb2dd..53cc774 100644 --- a/next_turn_params.go +++ b/next_turn_params.go @@ -2,34 +2,241 @@ package agent import ( "context" + "encoding/json" + "fmt" + "log" + openrouter "github.com/OpenRouterTeam/go-sdk" "github.com/OpenRouterTeam/go-sdk/models/components" + "github.com/OpenRouterTeam/go-sdk/optionalnullable" ) +// NextTurnParamsContext is the view of the current request that a tool's +// nextTurnParams functions see, plus the executed call and its result. +// +// Composition matters: within one turn the functions run in tools-array order +// and each one observes the values written by the earlier ones, so a later +// tool can build on an earlier tool's contribution. type NextTurnParamsContext struct { + Input []components.InputsUnion1 + ToolChoice *components.OpenAIResponsesToolChoiceUnion + Model string + Models []string + Temperature *float64 + MaxOutputTokens *int64 + TopP *float64 + TopK *int64 + Instructions *string + + // ToolCall is the call whose nextTurnParams function is running, and + // Result its execution result. Upstream passes the call's validated + // arguments as a separate first parameter; Go carries them here on + // ToolCall.Arguments. ToolCall ParsedToolCall Result ToolExecutionResult - Request components.ResponsesRequest + // Request is the full request the context was derived from, for fields + // nextTurnParams does not model. + Request components.ResponsesRequest } + +// NextTurnParamsFunctions maps a request parameter name to the function that +// computes its value for the turn after a tool executes. Valid keys are the +// ones NextTurnParamsContext models: "input", "toolChoice", "model", +// "models", "temperature", "maxOutputTokens", "topP", "topK" and +// "instructions". Any other key is ignored with a warning. type NextTurnParamsFunctions map[string]func(context.Context, NextTurnParamsContext) (any, error) +// nextTurnParamKeys is the set of request parameters nextTurnParams may set. +var nextTurnParamKeys = map[string]bool{ + "input": true, "toolChoice": true, "model": true, "models": true, + "temperature": true, "maxOutputTokens": true, "topP": true, "topK": true, + "instructions": true, +} + func BuildNextTurnParamsContext(call ParsedToolCall, result ToolExecutionResult, request components.ResponsesRequest) NextTurnParamsContext { - return NextTurnParamsContext{ToolCall: call, Result: result, Request: request} -} -func ExecuteNextTurnParamsFunctions(ctx context.Context, funcs NextTurnParamsFunctions, ntctx NextTurnParamsContext) (map[string]any, error) { - out := map[string]any{} - for k, fn := range funcs { - v, err := fn(ctx, ntctx) - if err != nil { - return nil, err + ntctx := NextTurnParamsContext{ + Input: requestInputItems(request.Input), + ToolChoice: request.ToolChoice, + Models: request.Models, + TopK: request.TopK, + ToolCall: call, + Result: result, + Request: request, + } + if request.Model != nil { + ntctx.Model = *request.Model + } + if v, ok := request.Temperature.GetOrZero(); ok { + ntctx.Temperature = &v + } + if v, ok := request.MaxOutputTokens.GetOrZero(); ok { + ntctx.MaxOutputTokens = &v + } + if v, ok := request.TopP.GetOrZero(); ok { + ntctx.TopP = &v + } + if v, ok := request.Instructions.GetOrZero(); ok { + ntctx.Instructions = &v + } + return ntctx +} + +// ExecuteNextTurnParamsFunctions runs the nextTurnParams functions of every +// tool that was called in this turn, in tools-array order, composing their +// results: each function sees the values written by the ones before it. +// +// Returns the computed parameters keyed by parameter name; an empty map means +// no tool contributed anything. +func ExecuteNextTurnParamsFunctions(ctx context.Context, tools []Tool, calls []ParsedToolCall, results map[string]ToolExecutionResult, request components.ResponsesRequest) (map[string]any, error) { + computed := map[string]any{} + for _, t := range tools { + if t == nil || IsServerTool(t) { + // Server tools have no client-side nextTurnParams hooks. + continue + } + funcs := ToolNextTurnParamsOf(t) + if len(funcs) == 0 { + continue + } + for _, call := range calls { + if call.Name != t.ToolName() { + continue + } + if err := validateNextTurnParamsArguments(t.ToolName(), call); err != nil { + return nil, err + } + working := BuildNextTurnParamsContext(call, results[call.CallID], ApplyNextTurnParamsToRequest(request, computed)) + for _, key := range sortedKeys(funcs) { + fn := funcs[key] + if fn == nil { + continue + } + if !nextTurnParamKeys[key] { + log.Printf("[nextTurnParams] invalid key %q in tool %q; valid keys: input, toolChoice, model, models, temperature, maxOutputTokens, topP, topK, instructions", key, t.ToolName()) + continue + } + value, err := fn(ctx, working) + if err != nil { + return nil, fmt.Errorf("nextTurnParams %s for tool %s: %w", key, t.ToolName(), err) + } + computed[key] = value + // Re-derive so the next function in this turn composes on top + // of what this one just produced. + working = BuildNextTurnParamsContext(call, working.Result, ApplyNextTurnParamsToRequest(request, computed)) + } } - out[k] = v } - return out, nil + return computed, nil } + +// validateNextTurnParamsArguments mirrors upstream's guard that a call's +// arguments must be an object before they are handed to a nextTurnParams +// function. +func validateNextTurnParamsArguments(toolName string, call ParsedToolCall) error { + switch v := call.Arguments.(type) { + case nil: + if call.RawArgs == "" { + return nil + } + var decoded any + if err := json.Unmarshal([]byte(call.RawArgs), &decoded); err != nil { + return nil + } + if _, ok := decoded.(map[string]any); !ok { + return fmt.Errorf("tool call arguments for %s must be an object, got %s", toolName, describeJSONType(decoded)) + } + case map[string]any: + return nil + default: + return fmt.Errorf("tool call arguments for %s must be an object, got %s", toolName, describeJSONType(v)) + } + return nil +} + +// ApplyNextTurnParamsToRequest returns a copy of request with the computed +// nextTurnParams applied. A nil value clears the field (upstream strips nulls +// to undefined). Values of an unexpected Go type for a key are ignored rather +// than corrupting the request. func ApplyNextTurnParamsToRequest(request components.ResponsesRequest, params map[string]any) components.ResponsesRequest { - if v, ok := params["model"].(string); ok { - request.Model = &v + for _, key := range sortedKeys(params) { + value := params[key] + switch key { + case "model": + if v, ok := value.(string); ok { + request.Model = openrouter.Pointer(v) + } + case "models": + if v, ok := value.([]string); ok { + request.Models = v + } + case "toolChoice": + switch v := value.(type) { + case nil: + request.ToolChoice = nil + case *components.OpenAIResponsesToolChoiceUnion: + request.ToolChoice = v + case components.OpenAIResponsesToolChoiceUnion: + choice := v + request.ToolChoice = &choice + } + case "input": + switch v := value.(type) { + case []components.InputsUnion1: + request.Input = openrouter.Pointer(components.CreateInputsUnionArrayOfInputsUnion1(v)) + case string: + request.Input = openrouter.Pointer(components.CreateInputsUnionStr(v)) + } + case "temperature": + request.Temperature = optionalFloat(value) + case "topP": + request.TopP = optionalFloat(value) + case "maxOutputTokens": + request.MaxOutputTokens = optionalInt(value) + case "topK": + if v, ok := intValue(value); ok { + request.TopK = &v + } else if value == nil { + request.TopK = nil + } + case "instructions": + if v, ok := value.(string); ok { + request.Instructions = optionalnullable.From(openrouter.String(v)) + } + } } return request } + +func optionalFloat(value any) optionalnullable.OptionalNullable[float64] { + var f float64 + switch v := value.(type) { + case float64: + f = v + case float32: + f = float64(v) + case int: + f = float64(v) + default: + return optionalnullable.OptionalNullable[float64]{} + } + return optionalnullable.From(&f) +} + +func optionalInt(value any) optionalnullable.OptionalNullable[int64] { + if v, ok := intValue(value); ok { + return optionalnullable.From(&v) + } + return optionalnullable.OptionalNullable[int64]{} +} + +func intValue(value any) (int64, bool) { + switch v := value.(type) { + case int64: + return v, true + case int: + return int64(v), true + case float64: + return int64(v), true + } + return 0, false +} diff --git a/tool.go b/tool.go index 76f9154..6dc61f5 100644 --- a/tool.go +++ b/tool.go @@ -27,6 +27,20 @@ type ToolConfig[In any] struct { Approval ToolApprovalCheck OnToolCalled func(context.Context, In, ToolExecuteContext) (any, bool, error) OnResponseReceived func(context.Context, any, ToolExecuteContext) (any, error) + // Strict requests provider-side strict schema adherence for this tool's + // generated arguments and is passed through to the wire tool definition + // verbatim (upstream #99). nil omits the field entirely, which is the + // provider default (non-strict). OpenAI-style strict schemas require + // every declared object property to appear in `required`. + Strict *bool + // NextTurnParams computes request parameters for the turn that follows + // this tool's execution, keyed by parameter name ("model", "toolChoice", + // "temperature", …). See NextTurnParamsFunctions. + NextTurnParams NextTurnParamsFunctions + // LoopKey declares this tool's doom-loop call identity: which part of a + // call's validated arguments makes two calls "the same call". nil means + // the full arguments; see LoopKey for the exempt and computed forms. + LoopKey *LoopKey } type TypedTool[In any] struct { @@ -105,11 +119,28 @@ func (t *TypedTool[In]) ToAPITool() components.ResponsesRequestToolUnion { if t.config.Description != "" { fn.Description = optionalnullable.From(openrouter.String(t.config.Description)) } - strict := true - fn.Strict = optionalnullable.From(&strict) + // `strict` is passed through verbatim (upstream #99): nil omits the field + // so the provider default applies, rather than pinning a value the caller + // never asked for. + if t.config.Strict != nil { + strict := *t.config.Strict + fn.Strict = optionalnullable.From(&strict) + } return components.CreateResponsesRequestToolUnionFunction(fn) } +// ToolStrict reports the strict flag a tool declared for its wire definition, +// or nil when it declared none. +func (t *TypedTool[In]) ToolStrict() *bool { return t.config.Strict } + +// ToolNextTurnParams returns the tool's nextTurnParams functions, if any. +func (t *TypedTool[In]) ToolNextTurnParams() NextTurnParamsFunctions { + return t.config.NextTurnParams +} + +// ToolLoopKey returns the tool's doom-loop identity declaration, if any. +func (t *TypedTool[In]) ToolLoopKey() *LoopKey { return t.config.LoopKey } + func (t *TypedTool[In]) Execute(ctx context.Context, raw json.RawMessage, execCtx ToolExecuteContext) (ToolExecutionResult, error) { var input In if len(raw) == 0 || string(raw) == "null" { @@ -183,6 +214,12 @@ func validateAny(v any, schema map[string]any) error { type ServerToolConfig struct { Name string Config components.ResponsesRequestToolUnion + // ID overrides the default tool-set identity (`server:`) so two + // server tools of the same type can carry distinct activation ids + // (upstream `serverTool(config, { id })`). Go's zero value collapses + // upstream's `undefined` and `''`, so an empty ID means "use the + // default" rather than the error upstream raises for `id: ''`. + ID string } type serverToolImpl struct{ config ServerToolConfig } @@ -191,6 +228,15 @@ func NewServerTool(config ServerToolConfig) Tool { return serverToolImpl{config: config} } +// ToolSetID is the stable activation identity used by ToolSet. Defaults to +// `server:` when no override was supplied. +func (s serverToolImpl) ToolSetID() string { + if s.config.ID != "" { + return s.config.ID + } + return "server:" + s.config.Name +} + func (s serverToolImpl) ToolName() string { return s.config.Name } func (s serverToolImpl) ToolDescription() string { return "" } func (s serverToolImpl) ToolType() ToolType { return ToolTypeServer } diff --git a/tool_choice.go b/tool_choice.go new file mode 100644 index 0000000..9e01542 --- /dev/null +++ b/tool_choice.go @@ -0,0 +1,149 @@ +package agent + +import ( + "encoding/json" + + "github.com/OpenRouterTeam/go-sdk/models/components" +) + +// autoToolChoice is the API tool_choice value that lets the model decide +// whether to call a tool. +func autoToolChoice() *components.OpenAIResponsesToolChoiceUnion { + choice := components.CreateOpenAIResponsesToolChoiceUnionOpenAIResponsesToolChoiceAuto(components.OpenAIResponsesToolChoiceAutoAuto) + return &choice +} + +// IsForcedToolChoice reports whether a caller-configured tool choice requires +// the turn it is sent on to call a tool. +// +// nil, "auto", "none" -> false +// "required" -> true +// allowed_tools with mode "required" -> true +// allowed_tools with mode "auto" -> false +// a specific tool (function/shell/…) -> true +func IsForcedToolChoice(choice *components.OpenAIResponsesToolChoiceUnion) bool { + if choice == nil { + return false + } + if choice.OpenAIResponsesToolChoiceAuto != nil || choice.OpenAIResponsesToolChoiceNone != nil { + return false + } + if allowed := choice.ToolChoiceAllowed; allowed != nil { + return allowed.Mode.ModeRequired != nil + } + return true +} + +// RelaxForcedToolChoice relaxes a tool choice that *forces* a tool call, for +// the follow-up turns that come after a tool round has executed (upstream +// #100 / DEV-785). +// +// "required" -> "auto" +// a specific tool object -> "auto" +// allowed_tools mode "required" -> the same tool set with mode "auto" +// (the restriction on *which* tools may +// be called still applies; only the +// force-a-call part is spent) +// "auto", "none", allowed_tools/auto, and nil pass through unchanged. +func RelaxForcedToolChoice(choice *components.OpenAIResponsesToolChoiceUnion) *components.OpenAIResponsesToolChoiceUnion { + if choice == nil { + return nil + } + if !IsForcedToolChoice(choice) { + return choice + } + if allowed := choice.ToolChoiceAllowed; allowed != nil { + relaxed := *allowed + relaxed.Mode = components.CreateModeModeAuto(components.ModeAutoAuto) + out := components.CreateOpenAIResponsesToolChoiceUnionToolChoiceAllowed(relaxed) + return &out + } + return autoToolChoice() +} + +// forcedToolChoiceKey is a stable identity for a concrete forced choice. +// An empty string means "tools are not forced", so a caller can distinguish +// "no forced choice configured" from "this particular forced choice". +func forcedToolChoiceKey(choice *components.OpenAIResponsesToolChoiceUnion) string { + if !IsForcedToolChoice(choice) { + return "" + } + b, err := json.Marshal(choice) + if err != nil { + // A choice that cannot be canonicalized still forces a call; fall back + // to a constant identity so it is consumed once rather than re-arming + // on every turn. + return "forced" + } + return string(b) +} + +// toolChoicePolicy tracks a caller-configured forced tool choice across the +// turns of one run so that, once the forced choice has actually produced a +// tool call, later turns are relaxed to `auto` instead of forcing the model +// to keep calling tools until the step budget runs out (upstream #100). +// +// A *different* forced choice re-arms immediately, and an unforced turn +// clears the consumed key so the same forced value re-arms if it comes back +// (which is what makes a dynamically recomputed choice work). +type toolChoicePolicy struct { + configured *components.OpenAIResponsesToolChoiceUnion + configuredKey string + consumedKey string + // pendingCommit describes what the in-flight dispatch will do to + // consumedKey once its response materializes: "" (nothing), "clear", or + // "consume". + pendingCommit string + pendingKey string +} + +// configure records a freshly resolved caller choice and returns the effective +// wire choice for the next dispatch. +func (p *toolChoicePolicy) configure(choice *components.OpenAIResponsesToolChoiceUnion) *components.OpenAIResponsesToolChoiceUnion { + p.configured = choice + p.configuredKey = forcedToolChoiceKey(choice) + switch { + case p.configuredKey == "": + if p.consumedKey == "" { + p.pendingCommit, p.pendingKey = "", "" + } else { + p.pendingCommit, p.pendingKey = "clear", "" + } + case p.configuredKey == p.consumedKey: + p.pendingCommit, p.pendingKey = "", "" + default: + p.pendingCommit, p.pendingKey = "consume", p.configuredKey + } + return p.effective() +} + +// effective derives the wire tool choice for the active run: the configured +// choice, relaxed once that exact forced choice has been consumed. +func (p *toolChoicePolicy) effective() *components.OpenAIResponsesToolChoiceUnion { + if p.configuredKey != "" && p.configuredKey == p.consumedKey { + return RelaxForcedToolChoice(p.configured) + } + return p.configured +} + +// abandonDispatch drops the prepared transition without applying it. Used +// when an engine-owned one-turn override (a forced doom-loop advisor consult) +// replaced the caller's tool choice on the wire: that dispatch does not +// represent the caller's policy, so it can neither consume nor clear it. +func (p *toolChoicePolicy) abandonDispatch() { + p.pendingCommit, p.pendingKey = "", "" +} + +// commit binds the prepared transition to a materialized response. A forced +// choice is consumed only when the response it was sent with actually called +// a tool; a pause before dispatch leaves the policy untouched. +func (p *toolChoicePolicy) commit(hasToolCalls bool) { + commit, key := p.pendingCommit, p.pendingKey + p.pendingCommit, p.pendingKey = "", "" + switch { + case commit == "clear": + p.consumedKey = "" + case commit == "consume" && hasToolCalls: + p.consumedKey = key + } +} diff --git a/tool_choice_test.go b/tool_choice_test.go new file mode 100644 index 0000000..18788bf --- /dev/null +++ b/tool_choice_test.go @@ -0,0 +1,483 @@ +package agent + +// Tests for the 0.9.0/0.10.0 request-shaping delta: +// +// - upstream #100: an unchanged forced toolChoice relaxes to `auto` on +// follow-up turns, including after an approval resume, and re-arms when a +// recomputed choice changes semantic value. +// - upstream #114: a tool's nextTurnParams may set `toolChoice` for the +// following turn without touching the `tools` array. +// - upstream #31: `activeTools` narrows which tools are sent. +// - upstream #99: `strict` is passed through to the wire tool definition. +// +// Assertions read the *dispatched requests* rather than the port's internals: +// what reaches the provider is the observable contract these changes exist to +// fix. + +import ( + "context" + "testing" + + openrouter "github.com/OpenRouterTeam/go-sdk" + "github.com/OpenRouterTeam/go-sdk/models/components" + "github.com/OpenRouterTeam/go-sdk/models/operations" +) + +func requiredChoice() *components.OpenAIResponsesToolChoiceUnion { + choice := components.CreateOpenAIResponsesToolChoiceUnionOpenAIResponsesToolChoiceRequired(components.OpenAIResponsesToolChoiceRequiredRequired) + return &choice +} + +func functionChoice(name string) *components.OpenAIResponsesToolChoiceUnion { + choice := components.CreateOpenAIResponsesToolChoiceUnionOpenAIResponsesToolChoiceFunction(components.OpenAIResponsesToolChoiceFunction{ + Name: name, Type: components.OpenAIResponsesToolChoiceTypeFunctionFunction, + }) + return &choice +} + +func allowedToolsChoice(mode components.Mode, names ...string) *components.OpenAIResponsesToolChoiceUnion { + tools := make([]map[string]any, 0, len(names)) + for _, name := range names { + tools = append(tools, map[string]any{"type": "function", "name": name}) + } + choice := components.CreateOpenAIResponsesToolChoiceUnionToolChoiceAllowed(components.ToolChoiceAllowed{ + Mode: mode, Tools: tools, Type: components.ToolChoiceAllowedTypeAllowedTools, + }) + return &choice +} + +func requiredMode() components.Mode { + return components.CreateModeModeRequired(components.ModeRequiredRequired) +} +func autoMode() components.Mode { return components.CreateModeModeAuto(components.ModeAutoAuto) } + +// toolCallTurn is a response that calls `search` once. +func toolCallTurn(id, callID string) components.OpenResponsesResult { + call := components.OutputFunctionCallItem{CallID: callID, Name: "search", Arguments: `{"query":"go"}`} + return components.OpenResponsesResult{ID: id, Output: []components.OutputItems{components.CreateOutputItemsFunctionCall(call)}} +} + +func textTurn(id, text string) components.OpenResponsesResult { + return components.OpenResponsesResult{ID: id, OutputText: openrouter.String(text)} +} + +func TestForcedToolChoiceRelaxesToAutoOnFollowUpTurns(t *testing.T) { + for _, tc := range []struct { + name string + configure *components.OpenAIResponsesToolChoiceUnion + check func(*testing.T, *components.OpenAIResponsesToolChoiceUnion) + }{ + { + name: "required", + configure: requiredChoice(), + check: func(t *testing.T, got *components.OpenAIResponsesToolChoiceUnion) { + if got == nil || got.OpenAIResponsesToolChoiceAuto == nil { + t.Fatalf("follow-up tool_choice = %#v, want auto", got) + } + }, + }, + { + name: "specific function", + configure: functionChoice("search"), + check: func(t *testing.T, got *components.OpenAIResponsesToolChoiceUnion) { + if got == nil || got.OpenAIResponsesToolChoiceAuto == nil { + t.Fatalf("follow-up tool_choice = %#v, want auto", got) + } + }, + }, + { + name: "allowed_tools mode required", + configure: allowedToolsChoice(requiredMode(), "search"), + check: func(t *testing.T, got *components.OpenAIResponsesToolChoiceUnion) { + // The restriction on *which* tools may be called survives; + // only the force-a-call part is spent. + if got == nil || got.ToolChoiceAllowed == nil { + t.Fatalf("follow-up tool_choice = %#v, want allowed_tools", got) + } + if got.ToolChoiceAllowed.Mode.ModeAuto == nil { + t.Fatalf("allowed_tools mode = %#v, want auto", got.ToolChoiceAllowed.Mode) + } + if len(got.ToolChoiceAllowed.Tools) != 1 { + t.Fatalf("allowed_tools set changed: %#v", got.ToolChoiceAllowed.Tools) + } + }, + }, + } { + t.Run(tc.name, func(t *testing.T) { + sender := twoTurnSender(toolCallTurn("resp_1", "call_1"), textTurn("resp_2", "done")) + result, err := CallModel(context.Background(), sender, CallModelInput{ + Model: "openai/test", Input: "hi", Tools: []Tool{searchTool(t)}, + Request: components.ResponsesRequest{ToolChoice: tc.configure}, + }) + if err != nil { + t.Fatal(err) + } + if _, err := result.Text(context.Background()); err != nil { + t.Fatal(err) + } + if sender.calls < 2 { + t.Fatalf("expected a follow-up request, got %d calls", sender.calls) + } + if first := sender.requests[0].ToolChoice; !IsForcedToolChoice(first) { + t.Fatalf("first request must keep the caller's forced choice, got %#v", first) + } + tc.check(t, sender.requests[1].ToolChoice) + }) + } +} + +func TestUnforcedToolChoicePassesThroughUnchanged(t *testing.T) { + for _, tc := range []struct { + name string + choice *components.OpenAIResponsesToolChoiceUnion + assert func(*testing.T, *components.OpenAIResponsesToolChoiceUnion) + }{ + {"auto", autoToolChoice(), func(t *testing.T, got *components.OpenAIResponsesToolChoiceUnion) { + if got == nil || got.OpenAIResponsesToolChoiceAuto == nil { + t.Fatalf("tool_choice = %#v, want auto", got) + } + }}, + {"allowed_tools mode auto", allowedToolsChoice(autoMode(), "search"), func(t *testing.T, got *components.OpenAIResponsesToolChoiceUnion) { + if got == nil || got.ToolChoiceAllowed == nil || got.ToolChoiceAllowed.Mode.ModeAuto == nil { + t.Fatalf("tool_choice = %#v, want allowed_tools/auto", got) + } + }}, + } { + t.Run(tc.name, func(t *testing.T) { + sender := twoTurnSender(toolCallTurn("resp_1", "call_1"), textTurn("resp_2", "done")) + result, err := CallModel(context.Background(), sender, CallModelInput{ + Model: "openai/test", Input: "hi", Tools: []Tool{searchTool(t)}, + Request: components.ResponsesRequest{ToolChoice: tc.choice}, + }) + if err != nil { + t.Fatal(err) + } + if _, err := result.Text(context.Background()); err != nil { + t.Fatal(err) + } + tc.assert(t, sender.requests[1].ToolChoice) + }) + } +} + +// A forced choice is only spent by a response that actually called a tool: a +// text-only first turn must leave it armed. +func TestForcedToolChoiceIsNotConsumedByAToollessTurn(t *testing.T) { + policy := &toolChoicePolicy{} + effective := policy.configure(requiredChoice()) + if !IsForcedToolChoice(effective) { + t.Fatalf("first dispatch must stay forced, got %#v", effective) + } + policy.commit(false) + if got := policy.effective(); !IsForcedToolChoice(got) { + t.Fatalf("a turn with no tool calls must not consume the forced choice, got %#v", got) + } + // Next dispatch of the same choice: this one does call a tool. + policy.configure(requiredChoice()) + policy.commit(true) + if got := policy.effective(); IsForcedToolChoice(got) { + t.Fatalf("a turn that called a tool must consume the forced choice, got %#v", got) + } +} + +// A dynamically recomputed choice re-arms when its semantic value changes, and +// an unforced turn clears the consumed key so the same value re-arms later. +func TestForcedToolChoiceReArmsOnSemanticChange(t *testing.T) { + policy := &toolChoicePolicy{} + policy.configure(functionChoice("plan")) + policy.commit(true) + if got := policy.effective(); IsForcedToolChoice(got) { + t.Fatalf("consumed choice should be relaxed, got %#v", got) + } + + // A different forced choice is a fresh instruction. + if got := policy.configure(functionChoice("submit")); !IsForcedToolChoice(got) { + t.Fatalf("a changed forced choice must re-arm, got %#v", got) + } + policy.commit(true) + + // An unforced turn clears the consumption, so the same forced value works + // again next time it appears. + policy.configure(autoToolChoice()) + policy.commit(false) + if got := policy.configure(functionChoice("submit")); !IsForcedToolChoice(got) { + t.Fatalf("the same forced choice must re-arm after an unforced turn, got %#v", got) + } +} + +// Consumption survives the approval pause: a resumed run must not force the +// model back into tool calls with a choice it has already spent. +func TestForcedToolChoiceConsumptionPersistsAcrossApprovalResume(t *testing.T) { + tool := MustNewTool(ToolConfig[sampleInput]{Name: "search", RequireApproval: true, Execute: func(context.Context, sampleInput, ToolExecuteContext) (any, error) { + return "ok", nil + }}) + pause := toolCallTurn("resp_1", "call_1") + createdPause := operations.CreateCreateResponsesResponseOpenResponsesResult(pause) + pauseSender := &fakeSender{responses: []*operations.CreateResponsesResponse{&createdPause}} + paused, err := CallModel(context.Background(), pauseSender, CallModelInput{ + Model: "openai/test", Input: "hi", Tools: []Tool{tool}, + Request: components.ResponsesRequest{ToolChoice: requiredChoice()}, + }) + if err != nil { + t.Fatal(err) + } + state, err := paused.State(context.Background()) + if err != nil { + t.Fatal(err) + } + if state.ConsumedForcedToolChoiceKey == "" { + t.Fatal("a forced choice that produced a tool call must be recorded as consumed in state") + } + + final := textTurn("resp_2", "done") + createdFinal := operations.CreateCreateResponsesResponseOpenResponsesResult(final) + resumeSender := &fakeSender{responses: []*operations.CreateResponsesResponse{&createdFinal}} + resumed, err := CallModel(context.Background(), resumeSender, CallModelInput{ + Model: "openai/test", Tools: []Tool{tool}, State: &state, ApproveToolCalls: []string{"call_1"}, + Request: components.ResponsesRequest{ToolChoice: requiredChoice()}, + }) + if err != nil { + t.Fatal(err) + } + if _, err := resumed.Text(context.Background()); err != nil { + t.Fatal(err) + } + if got := resumeSender.requests[0].ToolChoice; IsForcedToolChoice(got) { + t.Fatalf("resume request must relax the already-consumed forced choice, got %#v", got) + } +} + +// upstream #114: a tool's nextTurnParams toolChoice reaches the dispatched +// follow-up request, and the `tools` array is byte-identical across turns so +// the provider's prompt-cache prefix survives. +func TestNextTurnParamsToolChoiceReachesFollowUpRequest(t *testing.T) { + widened := allowedToolsChoice(autoMode(), "tool_search", "get_weather") + toolSearch := MustNewTool(ToolConfig[sampleInput]{ + Name: "search", + Execute: func(context.Context, sampleInput, ToolExecuteContext) (any, error) { + return []string{"get_weather"}, nil + }, + NextTurnParams: NextTurnParamsFunctions{ + "toolChoice": func(_ context.Context, ntctx NextTurnParamsContext) (any, error) { + if ntctx.ToolChoice == nil || ntctx.ToolChoice.ToolChoiceAllowed == nil { + t.Errorf("nextTurnParams should see the current allowed_tools choice, got %#v", ntctx.ToolChoice) + } + if ntctx.ToolCall.Name != "search" { + t.Errorf("nextTurnParams context tool call = %q, want search", ntctx.ToolCall.Name) + } + return widened, nil + }, + }, + }) + sender := twoTurnSender(toolCallTurn("resp_1", "call_1"), textTurn("resp_2", "done")) + result, err := CallModel(context.Background(), sender, CallModelInput{ + Model: "openai/test", Input: "hi", Tools: []Tool{toolSearch}, + Request: components.ResponsesRequest{ToolChoice: allowedToolsChoice(autoMode(), "tool_search")}, + }) + if err != nil { + t.Fatal(err) + } + if _, err := result.Text(context.Background()); err != nil { + t.Fatal(err) + } + if sender.calls != 2 { + t.Fatalf("expected two requests, got %d", sender.calls) + } + got := sender.requests[1].ToolChoice + if got == nil || got.ToolChoiceAllowed == nil || len(got.ToolChoiceAllowed.Tools) != 2 { + t.Fatalf("follow-up tool_choice = %#v, want the widened allowed_tools set", got) + } + if len(sender.requests[0].Tools) != len(sender.requests[1].Tools) { + t.Fatalf("tools array must be unchanged across turns: %d then %d", len(sender.requests[0].Tools), len(sender.requests[1].Tools)) + } +} + +func TestNextTurnParamsLeavesToolChoiceAloneWhenNoToolComputesOne(t *testing.T) { + sender := twoTurnSender(toolCallTurn("resp_1", "call_1"), textTurn("resp_2", "done")) + configured := allowedToolsChoice(autoMode(), "search") + result, err := CallModel(context.Background(), sender, CallModelInput{ + Model: "openai/test", Input: "hi", Tools: []Tool{searchTool(t)}, + Request: components.ResponsesRequest{ToolChoice: configured}, + }) + if err != nil { + t.Fatal(err) + } + if _, err := result.Text(context.Background()); err != nil { + t.Fatal(err) + } + got := sender.requests[1].ToolChoice + if got == nil || got.ToolChoiceAllowed == nil || len(got.ToolChoiceAllowed.Tools) != 1 { + t.Fatalf("tool_choice should be untouched, got %#v", got) + } +} + +// A nextTurnParams model override reaches the follow-up request too — the +// pre-existing "model" key must keep working now that toolChoice joined it. +func TestNextTurnParamsModelReachesFollowUpRequest(t *testing.T) { + tool := MustNewTool(ToolConfig[sampleInput]{ + Name: "search", + Execute: func(context.Context, sampleInput, ToolExecuteContext) (any, error) { return "ok", nil }, + NextTurnParams: NextTurnParamsFunctions{ + "model": func(_ context.Context, ntctx NextTurnParamsContext) (any, error) { + if ntctx.Model != "openai/test" { + t.Errorf("context model = %q, want openai/test", ntctx.Model) + } + return "openai/stronger", nil + }, + }, + }) + sender := twoTurnSender(toolCallTurn("resp_1", "call_1"), textTurn("resp_2", "done")) + result, err := CallModel(context.Background(), sender, CallModelInput{Model: "openai/test", Input: "hi", Tools: []Tool{tool}}) + if err != nil { + t.Fatal(err) + } + if _, err := result.Text(context.Background()); err != nil { + t.Fatal(err) + } + if sender.requests[1].Model == nil || *sender.requests[1].Model != "openai/stronger" { + t.Fatalf("follow-up model = %v, want openai/stronger", sender.requests[1].Model) + } +} + +// upstream #31: activeTools narrows what is sent, addressed by tool-set id. +func TestActiveToolsFiltersWhatIsSent(t *testing.T) { + alpha := MustNewTool(ToolConfig[sampleInput]{Name: "alpha", Execute: func(context.Context, sampleInput, ToolExecuteContext) (any, error) { return "a", nil }}) + beta := MustNewTool(ToolConfig[sampleInput]{Name: "beta", Execute: func(context.Context, sampleInput, ToolExecuteContext) (any, error) { return "b", nil }}) + search := NewServerTool(ServerToolConfig{Name: "web_search", Config: components.CreateResponsesRequestToolUnionFunction(components.ResponsesRequestToolFunction{Name: "web_search", Type: components.ResponsesRequestTypeFunction})}) + + for _, tc := range []struct { + name string + active []string + want int + }{ + {"nil sends every tool", nil, 3}, + // A server tool always passes the filter: `active_tools` addresses + // client tools by name, so a server tool is not nameable there. + {"named client subset keeps the server tool", []string{"alpha"}, 2}, + {"unknown names are ignored", []string{"alpha", "nope"}, 2}, + {"explicitly empty leaves only server tools", []string{}, 1}, + } { + t.Run(tc.name, func(t *testing.T) { + created := operations.CreateCreateResponsesResponseOpenResponsesResult(textTurn("resp_1", "done")) + sender := &fakeSender{responses: []*operations.CreateResponsesResponse{&created}} + result, err := CallModel(context.Background(), sender, CallModelInput{ + Model: "openai/test", Input: "hi", Tools: []Tool{alpha, beta, search}, ActiveTools: tc.active, + }) + if err != nil { + t.Fatal(err) + } + if _, err := result.Text(context.Background()); err != nil { + t.Fatal(err) + } + if got := len(sender.requests[0].Tools); got != tc.want { + t.Fatalf("sent %d tools, want %d", got, tc.want) + } + }) + } +} + +// A tool filtered out of the request is filtered out of the executor too, so +// the executor never carries a definition the request never advertised +// (upstream filters before both API conversion and registration). A call for +// one therefore has nothing to run it, and the run pauses for the client. +func TestActiveToolsAlsoNarrowsTheExecutor(t *testing.T) { + executed := false + tool := MustNewTool(ToolConfig[sampleInput]{Name: "search", Execute: func(context.Context, sampleInput, ToolExecuteContext) (any, error) { + executed = true + return "ok", nil + }}) + sender := twoTurnSender(toolCallTurn("resp_1", "call_1"), textTurn("resp_2", "done")) + result, err := CallModel(context.Background(), sender, CallModelInput{ + Model: "openai/test", Input: "hi", Tools: []Tool{tool}, ActiveTools: []string{}, + }) + if err != nil { + t.Fatal(err) + } + state, err := result.State(context.Background()) + if err != nil { + t.Fatal(err) + } + if executed { + t.Fatal("a tool filtered out by ActiveTools must not be executable this call") + } + if state.Status != ConversationStatusAwaitingClientTools { + t.Fatalf("status = %q, want awaiting_client_tools for an unresolvable call", state.Status) + } + if len(state.PendingToolCalls) != 1 || state.PendingToolCalls[0].Name != "search" { + t.Fatalf("pending calls = %+v, want the unresolvable call surfaced to the caller", state.PendingToolCalls) + } +} + +// upstream #99: strict is forwarded verbatim; nil omits the field rather than +// pinning a value the caller never asked for. +func TestToolStrictIsPassedThroughToTheWireDefinition(t *testing.T) { + yes, no := true, false + for _, tc := range []struct { + name string + strict *bool + want *bool + present bool + }{ + {"omitted", nil, nil, false}, + {"true", &yes, &yes, true}, + {"false", &no, &no, true}, + } { + t.Run(tc.name, func(t *testing.T) { + tool := MustNewTool(ToolConfig[sampleInput]{Name: "search", Strict: tc.strict, Execute: func(context.Context, sampleInput, ToolExecuteContext) (any, error) { return "ok", nil }}) + api := tool.ToAPITool() + if api.ResponsesRequestToolFunction == nil { + t.Fatalf("expected a function tool, got %#v", api) + } + got, present := api.ResponsesRequestToolFunction.Strict.GetOrZero() + if present != tc.present { + t.Fatalf("strict present = %v, want %v", present, tc.present) + } + if tc.present && got != *tc.want { + t.Fatalf("strict = %v, want %v", got, *tc.want) + } + }) + } +} + +// upstream #91: the executed tool call must reach the execute context's turn +// context on the streaming path too, not just through the orchestrator. +func TestExecutedToolCallIsThreadedIntoTheExecuteContext(t *testing.T) { + var seen *components.OutputFunctionCallItem + tool := MustNewTool(ToolConfig[sampleInput]{Name: "search", Execute: func(_ context.Context, _ sampleInput, execCtx ToolExecuteContext) (any, error) { + seen = execCtx.Turn.ToolCall + return "ok", nil + }}) + sender := twoTurnSender(toolCallTurn("resp_1", "call_1"), textTurn("resp_2", "done")) + result, err := CallModel(context.Background(), sender, CallModelInput{Model: "openai/test", Input: "hi", Tools: []Tool{tool}}) + if err != nil { + t.Fatal(err) + } + if _, err := result.Text(context.Background()); err != nil { + t.Fatal(err) + } + if seen == nil { + t.Fatal("execute context turn.ToolCall was nil on the streaming path (upstream #91)") + } + if seen.Name != "search" || seen.CallID != "call_1" { + t.Fatalf("threaded call = %+v, want name=search call_id=call_1", seen) + } + if seen.Arguments != `{"query":"go"}` { + t.Fatalf("threaded arguments = %q", seen.Arguments) + } + if seen.Type != components.OutputFunctionCallItemTypeFunctionCall { + t.Fatalf("threaded item type = %q, want function_call", seen.Type) + } +} + +// A caller-supplied turn context wins: the orchestrator's carries `status`. +func TestCallerSuppliedTurnToolCallWins(t *testing.T) { + provided := &components.OutputFunctionCallItem{CallID: "orchestrated", Name: "search", Arguments: "{}", Type: components.OutputFunctionCallItemTypeFunctionCall} + execCtx := BuildToolExecuteContext( + ParsedToolCall{CallID: "streamed", Name: "search", RawArgs: `{"query":"go"}`}, + TurnContext{ToolCall: provided, NumberOfTurns: 2}, + nil, nil, + ) + if execCtx.Turn.ToolCall != provided { + t.Fatalf("caller-provided turn tool call must win, got %+v", execCtx.Turn.ToolCall) + } +} diff --git a/tool_context.go b/tool_context.go index db2f4fa..8fb82df 100644 --- a/tool_context.go +++ b/tool_context.go @@ -1,6 +1,11 @@ package agent -import "sync" +import ( + "encoding/json" + "sync" + + "github.com/OpenRouterTeam/go-sdk/models/components" +) const SharedContextKey = "shared" @@ -62,9 +67,50 @@ func BuildToolExecuteContext(call ParsedToolCall, turn TurnContext, store *ToolC local = store.Get(call.Name) shared = store.Get(SharedContextKey) } + // `Turn.ToolCall` is part of the tool-facing contract, but only the + // non-streaming orchestrator populates it — the streaming loop builds its + // turn context with just the turn number. Fill the gap from the executed + // call so `execute`/`OnToolCalled` see it on every path (upstream #91). A + // caller-provided turn context wins: the orchestrator's carries `status`. + if turn.ToolCall == nil { + turn.ToolCall = toFunctionCallItem(call) + } return ToolExecuteContext{ToolCall: call, Turn: turn, Context: local, Shared: shared, Store: store, Emit: emit} } +// toFunctionCallItem converts an executor-shaped ParsedToolCall back into the +// wire-shaped function_call item the execute context declares. Arguments are +// re-serialized from the parsed value when no raw string survived, so the +// result is semantically equal to the wire arguments but not guaranteed +// byte-identical (key order, whitespace). Returns nil for a zero call, which +// is what an approval/HITL response-received path legitimately has. +func toFunctionCallItem(call ParsedToolCall) *components.OutputFunctionCallItem { + if call.Name == "" && call.CallID == "" && call.ID == "" { + return nil + } + args := call.RawArgs + if args == "" { + if call.Arguments == nil { + args = "{}" + } else if b, err := json.Marshal(call.Arguments); err == nil { + args = string(b) + } else { + args = "{}" + } + } + item := &components.OutputFunctionCallItem{ + Arguments: args, + CallID: call.CallID, + Name: call.Name, + Type: components.OutputFunctionCallItemTypeFunctionCall, + } + if call.ID != "" { + id := call.ID + item.ID = &id + } + return item +} + func (tc ToolExecuteContext) LocalContext() map[string]any { if tc.Store == nil { return copyAnyMap(tc.Context) diff --git a/tool_set.go b/tool_set.go new file mode 100644 index 0000000..5926216 --- /dev/null +++ b/tool_set.go @@ -0,0 +1,468 @@ +package agent + +import ( + "errors" + "fmt" + "sync" +) + +// ToolSet is a declarative, immutable-by-default activation layer over a set +// of tools (port of upstream `@openrouter/agent/tool-set`, itself a port of +// ai-tool-set v1.0.0, MIT © Chris Cook). +// +// Tools are addressed by their tool-set id: a client tool's wire name, or a +// server tool's `server:` id (overridable — see ServerToolConfig.ID). +// Every tool starts active; Activate / Deactivate pin a static decision and +// ActivateWhen / DeactivateWhen defer it to a predicate evaluated at Resolve +// time against the conversation state and shared context. +// +// Resolve returns the active tools in construction order together with the +// `ActiveTools` id list, which is exactly what CallModelInput.ActiveTools +// expects. +// +// Mutators are immutable by default: each returns a new ToolSet and leaves +// the receiver untouched. A ToolSet created with Mutable applies mutations in +// place and returns itself, so aliases share one live object. +type ToolSet struct { + mu sync.Mutex + index indexedTools + activation map[string]activationEntry + situations map[string]situationRuntime + mutable bool +} + +// ActivationInput is what a ToolSet predicate sees. +type ActivationInput struct { + State *ConversationState + Context map[string]any +} + +// ActivationPredicate decides whether a conditional tool is active. +type ActivationPredicate func(ActivationInput) bool + +// StatusReason explains why a tool resolved active or inactive. +type StatusReason string + +const ( + StatusReasonDefault StatusReason = "default" + StatusReasonActivate StatusReason = "activate" + StatusReasonDeactivate StatusReason = "deactivate" + StatusReasonActivateWhen StatusReason = "activateWhen" + StatusReasonDeactivateWhen StatusReason = "deactivateWhen" + StatusReasonSituation StatusReason = "situation" +) + +// ToolStatusEntry is one tool's entry in a resolved snapshot's status map. +type ToolStatusEntry struct { + Enabled bool + Reason StatusReason + // Directive is the last applicable directive before predicates ran + // ("activate", "deactivate", "activateWhen", "deactivateWhen"), or "" when + // the tool is still at its construction default. + Directive string + // Predicate is true when the outcome depended on evaluating a runtime + // predicate. + Predicate bool +} + +// ResolvedToolSnapshot is what Resolve and ResolveSituation return. +type ResolvedToolSnapshot struct { + // Tools are the active tools only, in construction order. + Tools []Tool + // ActiveTools are the active *client* tool names — the CallModelInput + // wire format. Server ids are deliberately omitted: they are not + // addressable by `active_tools`. + ActiveTools []string + // Enabled and Disabled are the ids that resolved active / inactive, + // including server ids. + Enabled []string + Disabled []string + // StatusByTool has an entry for every known id. + StatusByTool map[string]ToolStatusEntry +} + +// SituationConditionalRule is one conditional entry of a named situation. +// Mode defaults to "activateWhen" when empty. +type SituationConditionalRule struct { + Mode string + Predicate ActivationPredicate +} + +// SituationConfig is a declarative partition overlay for one named +// situation. Ids it does not mention keep whatever the base set declares. +type SituationConfig struct { + Enabled []string + Disabled []string + Conditional map[string]SituationConditionalRule +} + +// ToolSetOptions configures CreateToolSet. +type ToolSetOptions struct { + Tools []Tool + // Mutable makes every mutator apply in place and return the receiver. + Mutable bool +} + +type activationEntry struct { + // kind is "static", "activateWhen" or "deactivateWhen". + kind string + active bool + predicate ActivationPredicate + // source is "default", "activate", "deactivate", "activateWhen", + // "deactivateWhen" or "situation". + source string +} + +type situationConditional struct { + id string + mode string + predicate ActivationPredicate +} + +type situationRuntime struct { + enabled []string + disabled []string + conditional []situationConditional +} + +type indexedTools struct { + orderedTools []Tool + orderedIDs []string + toolByID map[string]Tool +} + +// ErrUnknownTool is returned by a ToolSet mutator addressed at an id the set +// does not contain. +var ErrUnknownTool = errors.New("unknown tool") + +// CreateToolSet builds a ToolSet. Duplicate tool ids are an error: the set +// addresses tools by id, so two tools sharing one would be unaddressable. +func CreateToolSet(opts ToolSetOptions) (*ToolSet, error) { + index := indexedTools{toolByID: map[string]Tool{}} + for _, t := range opts.Tools { + if t == nil { + return nil, errors.New("tool set contains a nil tool") + } + id := ToolSetIDOf(t) + if _, exists := index.toolByID[id]; exists { + return nil, fmt.Errorf("duplicate tool ID: %q", id) + } + index.toolByID[id] = t + index.orderedIDs = append(index.orderedIDs, id) + index.orderedTools = append(index.orderedTools, t) + } + return &ToolSet{index: index, activation: map[string]activationEntry{}, situations: map[string]situationRuntime{}, mutable: opts.Mutable}, nil +} + +// MustCreateToolSet is CreateToolSet, panicking on error. For package-level +// tool-set declarations where a duplicate id is a programming mistake. +func MustCreateToolSet(opts ToolSetOptions) *ToolSet { + ts, err := CreateToolSet(opts) + if err != nil { + panic(err) + } + return ts +} + +// Tools returns every tool in construction order, regardless of activation. +func (s *ToolSet) Tools() []Tool { + return append([]Tool{}, s.index.orderedTools...) +} + +func (s *ToolSet) assertKnown(id string) error { + if _, ok := s.index.toolByID[id]; !ok { + return fmt.Errorf("%w: %q", ErrUnknownTool, id) + } + return nil +} + +// withMutation applies mutate to this set's activation map in place (mutable +// mode) or to a copy carried by a fresh ToSet (the immutable default). +func (s *ToolSet) withMutation(mutate func(map[string]activationEntry)) *ToolSet { + s.mu.Lock() + defer s.mu.Unlock() + if s.mutable { + mutate(s.activation) + return s + } + next := make(map[string]activationEntry, len(s.activation)) + for k, v := range s.activation { + next[k] = v + } + mutate(next) + return &ToolSet{index: s.index, activation: next, situations: s.situations, mutable: false} +} + +// Activate pins the named tools active. +func (s *ToolSet) Activate(names ...string) (*ToolSet, error) { + if err := s.assertAllKnown(names); err != nil { + return nil, err + } + return s.withMutation(func(activation map[string]activationEntry) { + for _, n := range names { + activation[n] = activationEntry{kind: "static", active: true, source: "activate"} + } + }), nil +} + +// Deactivate pins the named tools inactive. +func (s *ToolSet) Deactivate(names ...string) (*ToolSet, error) { + if err := s.assertAllKnown(names); err != nil { + return nil, err + } + return s.withMutation(func(activation map[string]activationEntry) { + for _, n := range names { + activation[n] = activationEntry{kind: "static", active: false, source: "deactivate"} + } + }), nil +} + +// ActivateWhen makes name active only while predicate reports true. +func (s *ToolSet) ActivateWhen(name string, predicate ActivationPredicate) (*ToolSet, error) { + return s.conditional("activateWhen", map[string]ActivationPredicate{name: predicate}) +} + +// ActivateWhenAll is ActivateWhen for several tools at once. +func (s *ToolSet) ActivateWhenAll(predicates map[string]ActivationPredicate) (*ToolSet, error) { + return s.conditional("activateWhen", predicates) +} + +// DeactivateWhen makes name inactive while predicate reports true. +func (s *ToolSet) DeactivateWhen(name string, predicate ActivationPredicate) (*ToolSet, error) { + return s.conditional("deactivateWhen", map[string]ActivationPredicate{name: predicate}) +} + +// DeactivateWhenAll is DeactivateWhen for several tools at once. +func (s *ToolSet) DeactivateWhenAll(predicates map[string]ActivationPredicate) (*ToolSet, error) { + return s.conditional("deactivateWhen", predicates) +} + +func (s *ToolSet) conditional(kind string, predicates map[string]ActivationPredicate) (*ToolSet, error) { + if len(predicates) == 0 { + return nil, fmt.Errorf("%s requires at least one name and predicate", kind) + } + names := sortedKeys(predicates) + for _, name := range names { + if predicates[name] == nil { + return nil, fmt.Errorf("%s requires a predicate for %q", kind, name) + } + if err := s.assertKnown(name); err != nil { + return nil, err + } + } + return s.withMutation(func(activation map[string]activationEntry) { + for _, name := range names { + activation[name] = activationEntry{kind: kind, predicate: predicates[name], source: kind} + } + }), nil +} + +func (s *ToolSet) assertAllKnown(names []string) error { + if len(names) == 0 { + return errors.New("no tool names given") + } + for _, n := range names { + if err := s.assertKnown(n); err != nil { + return err + } + } + return nil +} + +// DefineSituations registers named declarative overlays, replacing any +// previously defined ones (last call wins at the registry level). A situation +// that lists the same tool in more than one bucket is an error. +func (s *ToolSet) DefineSituations(situations map[string]SituationConfig) (*ToolSet, error) { + next := make(map[string]situationRuntime, len(situations)) + for _, name := range sortedKeys(situations) { + config := situations[name] + seen := map[string]bool{} + record := func(id string) error { + if err := s.assertKnown(id); err != nil { + return err + } + if seen[id] { + return fmt.Errorf("situation %q lists tool %q more than once (across enabled/disabled/conditional)", name, id) + } + seen[id] = true + return nil + } + runtime := situationRuntime{} + for _, id := range config.Enabled { + if err := record(id); err != nil { + return nil, err + } + runtime.enabled = append(runtime.enabled, id) + } + for _, id := range config.Disabled { + if err := record(id); err != nil { + return nil, err + } + runtime.disabled = append(runtime.disabled, id) + } + for _, id := range sortedKeys(config.Conditional) { + rule := config.Conditional[id] + if err := record(id); err != nil { + return nil, err + } + if rule.Predicate == nil { + return nil, fmt.Errorf("situation %q: conditional rule for tool %q needs a predicate", name, id) + } + mode := rule.Mode + switch mode { + case "": + mode = "activateWhen" + case "activateWhen", "deactivateWhen": + default: + return nil, fmt.Errorf("situation %q: conditional rule for tool %q has invalid mode %q", name, id, rule.Mode) + } + runtime.conditional = append(runtime.conditional, situationConditional{id: id, mode: mode, predicate: rule.Predicate}) + } + next[name] = runtime + } + + s.mu.Lock() + defer s.mu.Unlock() + if s.mutable { + s.situations = next + return s, nil + } + activation := make(map[string]activationEntry, len(s.activation)) + for k, v := range s.activation { + activation[k] = v + } + return &ToolSet{index: s.index, activation: activation, situations: next, mutable: false}, nil +} + +// Resolve evaluates the base partition (no situation overlay). +func (s *ToolSet) Resolve(input ActivationInput) ResolvedToolSnapshot { + s.mu.Lock() + activation := make(map[string]activationEntry, len(s.activation)) + for k, v := range s.activation { + activation[k] = v + } + s.mu.Unlock() + return s.resolveWithActivation(activation, input) +} + +// ResolveSituation evaluates the base partition with a named situation's +// overlay applied on top. +func (s *ToolSet) ResolveSituation(name string, input ActivationInput) (ResolvedToolSnapshot, error) { + s.mu.Lock() + situation, ok := s.situations[name] + activation := make(map[string]activationEntry, len(s.activation)) + for k, v := range s.activation { + activation[k] = v + } + s.mu.Unlock() + if !ok { + return ResolvedToolSnapshot{}, fmt.Errorf("unknown situation: %q", name) + } + for _, id := range situation.enabled { + activation[id] = activationEntry{kind: "static", active: true, source: "situation"} + } + for _, id := range situation.disabled { + activation[id] = activationEntry{kind: "static", active: false, source: "situation"} + } + for _, entry := range situation.conditional { + activation[entry.id] = activationEntry{kind: entry.mode, predicate: entry.predicate, source: "situation"} + } + return s.resolveWithActivation(activation, input), nil +} + +func (s *ToolSet) resolveWithActivation(activation map[string]activationEntry, input ActivationInput) ResolvedToolSnapshot { + snapshot := ResolvedToolSnapshot{StatusByTool: map[string]ToolStatusEntry{}} + for _, id := range s.index.orderedIDs { + tool := s.index.toolByID[id] + if tool == nil { + continue + } + entry, hasEntry := activation[id] + active := evaluateActivation(entry, hasEntry, input) + snapshot.StatusByTool[id] = toStatusEntry(active, entry, hasEntry) + if active { + snapshot.Tools = append(snapshot.Tools, tool) + snapshot.Enabled = append(snapshot.Enabled, id) + if !IsServerTool(tool) { + snapshot.ActiveTools = append(snapshot.ActiveTools, id) + } + } else { + snapshot.Disabled = append(snapshot.Disabled, id) + } + } + // A snapshot with nothing active must still filter tools down to nothing + // rather than reading as "no filter", so ActiveTools is non-nil whenever + // the set has tools at all. + if snapshot.ActiveTools == nil { + snapshot.ActiveTools = []string{} + } + if snapshot.Tools == nil { + snapshot.Tools = []Tool{} + } + return snapshot +} + +func evaluateActivation(entry activationEntry, hasEntry bool, input ActivationInput) bool { + if !hasEntry { + return true + } + switch entry.kind { + case "static": + return entry.active + case "activateWhen": + return entry.predicate(input) + default: + return !entry.predicate(input) + } +} + +func toStatusEntry(active bool, entry activationEntry, hasEntry bool) ToolStatusEntry { + if !hasEntry { + return ToolStatusEntry{Enabled: active, Reason: StatusReasonDefault} + } + if entry.kind == "static" { + directive := "deactivate" + if entry.active { + directive = "activate" + } + reason := StatusReason(directive) + switch entry.source { + case "situation": + reason = StatusReasonSituation + case "default": + reason = StatusReasonDefault + } + return ToolStatusEntry{Enabled: active, Reason: reason, Directive: directive} + } + reason := StatusReason(entry.kind) + if entry.source == "situation" { + reason = StatusReasonSituation + } + return ToolStatusEntry{Enabled: active, Reason: reason, Directive: entry.kind, Predicate: true} +} + +// Clone copies this set's state into a fresh, independent instance. A nil +// mutable inherits the source's mode. +func (s *ToolSet) Clone(mutable *bool) *ToolSet { + s.mu.Lock() + defer s.mu.Unlock() + next := &ToolSet{index: s.index, activation: make(map[string]activationEntry, len(s.activation)), situations: make(map[string]situationRuntime, len(s.situations)), mutable: s.mutable} + if mutable != nil { + next.mutable = *mutable + } + for k, v := range s.activation { + next.activation[k] = v + } + for k, v := range s.situations { + next.situations[k] = v + } + return next +} + +// Apply writes a snapshot's tools and active-id list onto a CallModelInput, +// the Go equivalent of spreading upstream's `resolved.callModel`. +func (r ResolvedToolSnapshot) Apply(input CallModelInput) CallModelInput { + input.Tools = append([]Tool{}, r.Tools...) + input.ActiveTools = append([]string{}, r.ActiveTools...) + return input +} diff --git a/tool_set_test.go b/tool_set_test.go new file mode 100644 index 0000000..b172292 --- /dev/null +++ b/tool_set_test.go @@ -0,0 +1,264 @@ +package agent + +// ToolSet tests (upstream #31, `@openrouter/agent/tool-set`). +// +// Assertions target the observable snapshot — which tools resolve active, in +// what order, and the id list that reaches CallModelInput.ActiveTools — +// because that snapshot is the whole product surface of the feature. + +import ( + "context" + "testing" + + "github.com/OpenRouterTeam/go-sdk/models/components" + "github.com/OpenRouterTeam/go-sdk/models/operations" +) + +func tsTool(t *testing.T, name string) Tool { + t.Helper() + return MustNewTool(ToolConfig[sampleInput]{Name: name, Execute: func(context.Context, sampleInput, ToolExecuteContext) (any, error) { return name, nil }}) +} + +func tsServerTool(name, id string) Tool { + return NewServerTool(ServerToolConfig{Name: name, ID: id, Config: components.CreateResponsesRequestToolUnionFunction(components.ResponsesRequestToolFunction{Name: name, Type: components.ResponsesRequestTypeFunction})}) +} + +func ids(tools []Tool) []string { + out := make([]string, 0, len(tools)) + for _, t := range tools { + out = append(out, ToolSetIDOf(t)) + } + return out +} + +func equalStrings(a, b []string) bool { + if len(a) != len(b) { + return false + } + for i := range a { + if a[i] != b[i] { + return false + } + } + return true +} + +func TestToolSetEveryToolStartsActiveInConstructionOrder(t *testing.T) { + set := MustCreateToolSet(ToolSetOptions{Tools: []Tool{tsTool(t, "alpha"), tsServerTool("web_search", ""), tsTool(t, "beta")}}) + snap := set.Resolve(ActivationInput{}) + if want := []string{"alpha", "server:web_search", "beta"}; !equalStrings(ids(snap.Tools), want) { + t.Fatalf("resolved tools = %v, want %v", ids(snap.Tools), want) + } + // activeTools carries client names only: a server id is not addressable + // by the wire `active_tools` field. + if want := []string{"alpha", "beta"}; !equalStrings(snap.ActiveTools, want) { + t.Fatalf("activeTools = %v, want %v", snap.ActiveTools, want) + } + if len(snap.StatusByTool) != 3 { + t.Fatalf("statusByTool must be exhaustive, got %d entries", len(snap.StatusByTool)) + } + if got := snap.StatusByTool["alpha"]; got.Reason != StatusReasonDefault || !got.Enabled || got.Directive != "" { + t.Fatalf("untouched tool status = %+v, want default/enabled/no directive", got) + } +} + +func TestToolSetActivateDeactivateAreImmutableByDefault(t *testing.T) { + base := MustCreateToolSet(ToolSetOptions{Tools: []Tool{tsTool(t, "alpha"), tsTool(t, "beta")}}) + off, err := base.Deactivate("alpha") + if err != nil { + t.Fatal(err) + } + if len(base.Resolve(ActivationInput{}).Tools) != 2 { + t.Fatal("Deactivate must not mutate the receiver of an immutable set") + } + snap := off.Resolve(ActivationInput{}) + if want := []string{"beta"}; !equalStrings(ids(snap.Tools), want) { + t.Fatalf("after deactivate = %v, want %v", ids(snap.Tools), want) + } + if got := snap.StatusByTool["alpha"]; got.Enabled || got.Reason != StatusReasonDeactivate || got.Directive != "deactivate" { + t.Fatalf("deactivated status = %+v", got) + } + back, err := off.Activate("alpha") + if err != nil { + t.Fatal(err) + } + if len(back.Resolve(ActivationInput{}).Tools) != 2 { + t.Fatal("Activate must restore a deactivated tool") + } + if got := back.Resolve(ActivationInput{}).StatusByTool["alpha"]; got.Reason != StatusReasonActivate { + t.Fatalf("reactivated status = %+v, want reason activate", got) + } +} + +func TestToolSetMutableAppliesInPlace(t *testing.T) { + set := MustCreateToolSet(ToolSetOptions{Tools: []Tool{tsTool(t, "alpha"), tsTool(t, "beta")}, Mutable: true}) + returned, err := set.Deactivate("alpha") + if err != nil { + t.Fatal(err) + } + if returned != set { + t.Fatal("a mutable set must return itself so aliases stay consistent") + } + if want := []string{"beta"}; !equalStrings(ids(set.Resolve(ActivationInput{}).Tools), want) { + t.Fatalf("mutable set did not mutate in place: %v", ids(set.Resolve(ActivationInput{}).Tools)) + } +} + +func TestToolSetConditionalPredicatesDecideAtResolveTime(t *testing.T) { + set := MustCreateToolSet(ToolSetOptions{Tools: []Tool{tsTool(t, "alpha"), tsTool(t, "beta")}}) + set, err := set.ActivateWhen("alpha", func(in ActivationInput) bool { return in.Context["admin"] == true }) + if err != nil { + t.Fatal(err) + } + set, err = set.DeactivateWhen("beta", func(in ActivationInput) bool { return in.State != nil && in.State.Status == ConversationStatusComplete }) + if err != nil { + t.Fatal(err) + } + + off := set.Resolve(ActivationInput{}) + if want := []string{"beta"}; !equalStrings(ids(off.Tools), want) { + t.Fatalf("with predicates false/false = %v, want %v", ids(off.Tools), want) + } + if got := off.StatusByTool["alpha"]; got.Reason != StatusReasonActivateWhen || !got.Predicate || got.Enabled { + t.Fatalf("activateWhen status = %+v", got) + } + + done := CreateInitialState() + done.Status = ConversationStatusComplete + on := set.Resolve(ActivationInput{Context: map[string]any{"admin": true}, State: &done}) + if want := []string{"alpha"}; !equalStrings(ids(on.Tools), want) { + t.Fatalf("with predicates true/true = %v, want %v", ids(on.Tools), want) + } + if got := on.StatusByTool["beta"]; got.Enabled || got.Reason != StatusReasonDeactivateWhen { + t.Fatalf("deactivateWhen status = %+v", got) + } +} + +func TestToolSetSituationsOverlayTheBasePartition(t *testing.T) { + set := MustCreateToolSet(ToolSetOptions{Tools: []Tool{tsTool(t, "read"), tsTool(t, "write"), tsTool(t, "deploy")}}) + set, err := set.Deactivate("write") + if err != nil { + t.Fatal(err) + } + set, err = set.DefineSituations(map[string]SituationConfig{ + "release": { + Enabled: []string{"write"}, + Disabled: []string{"read"}, + Conditional: map[string]SituationConditionalRule{ + "deploy": {Predicate: func(in ActivationInput) bool { return in.Context["approved"] == true }}, + }, + }, + }) + if err != nil { + t.Fatal(err) + } + + // The base partition is untouched by defining a situation. + if want := []string{"read", "deploy"}; !equalStrings(ids(set.Resolve(ActivationInput{}).Tools), want) { + t.Fatalf("base resolve = %v, want %v", ids(set.Resolve(ActivationInput{}).Tools), want) + } + + snap, err := set.ResolveSituation("release", ActivationInput{}) + if err != nil { + t.Fatal(err) + } + if want := []string{"write"}; !equalStrings(ids(snap.Tools), want) { + t.Fatalf("situation resolve = %v, want %v", ids(snap.Tools), want) + } + for _, id := range []string{"read", "write", "deploy"} { + if got := snap.StatusByTool[id]; got.Reason != StatusReasonSituation { + t.Fatalf("%s status reason = %q, want situation", id, got.Reason) + } + } + + approved, err := set.ResolveSituation("release", ActivationInput{Context: map[string]any{"approved": true}}) + if err != nil { + t.Fatal(err) + } + if want := []string{"write", "deploy"}; !equalStrings(ids(approved.Tools), want) { + t.Fatalf("approved situation resolve = %v, want %v", ids(approved.Tools), want) + } + + if _, err := set.ResolveSituation("nope", ActivationInput{}); err == nil { + t.Fatal("an unknown situation must be an error") + } +} + +func TestToolSetSituationRejectsDuplicateBuckets(t *testing.T) { + set := MustCreateToolSet(ToolSetOptions{Tools: []Tool{tsTool(t, "alpha")}}) + if _, err := set.DefineSituations(map[string]SituationConfig{ + "bad": {Enabled: []string{"alpha"}, Disabled: []string{"alpha"}}, + }); err == nil { + t.Fatal("a situation listing one tool in two buckets must be an error") + } + if _, err := set.DefineSituations(map[string]SituationConfig{ + "bad": {Conditional: map[string]SituationConditionalRule{"alpha": {Mode: "sometimes", Predicate: func(ActivationInput) bool { return true }}}}, + }); err == nil { + t.Fatal("an invalid conditional mode must be an error") + } +} + +func TestToolSetRejectsUnknownIDsAndDuplicates(t *testing.T) { + set := MustCreateToolSet(ToolSetOptions{Tools: []Tool{tsTool(t, "alpha")}}) + if _, err := set.Activate("nope"); err == nil { + t.Fatal("activating an unknown id must be an error") + } + if _, err := set.ActivateWhen("nope", func(ActivationInput) bool { return true }); err == nil { + t.Fatal("activateWhen on an unknown id must be an error") + } + if _, err := set.ActivateWhen("alpha", nil); err == nil { + t.Fatal("activateWhen without a predicate must be an error") + } + if _, err := CreateToolSet(ToolSetOptions{Tools: []Tool{tsTool(t, "alpha"), tsTool(t, "alpha")}}); err == nil { + t.Fatal("duplicate tool ids must be an error") + } +} + +func TestToolSetCloneIsIndependent(t *testing.T) { + mutable := true + base := MustCreateToolSet(ToolSetOptions{Tools: []Tool{tsTool(t, "alpha"), tsTool(t, "beta")}}) + clone := base.Clone(&mutable) + if _, err := clone.Deactivate("alpha"); err != nil { + t.Fatal(err) + } + if len(base.Resolve(ActivationInput{}).Tools) != 2 { + t.Fatal("mutating a clone must not affect the source") + } + if len(clone.Resolve(ActivationInput{}).Tools) != 1 { + t.Fatal("the clone should have mutated in place") + } +} + +// The whole point of the snapshot: it feeds callModel and the request that +// reaches the provider carries exactly the active tools. +func TestToolSetSnapshotDrivesTheDispatchedRequest(t *testing.T) { + set := MustCreateToolSet(ToolSetOptions{Tools: []Tool{tsTool(t, "list_orders"), tsServerTool("web_search", "public_search")}}) + set, err := set.Deactivate("list_orders") + if err != nil { + t.Fatal(err) + } + snap := set.Resolve(ActivationInput{}) + + created := operations.CreateCreateResponsesResponseOpenResponsesResult(textTurn("resp_1", "done")) + sender := &fakeSender{responses: []*operations.CreateResponsesResponse{&created}} + input := snap.Apply(CallModelInput{Model: "openai/test", Input: "hi"}) + result, err := CallModel(context.Background(), sender, input) + if err != nil { + t.Fatal(err) + } + if _, err := result.Text(context.Background()); err != nil { + t.Fatal(err) + } + if got := len(sender.requests[0].Tools); got != 1 { + t.Fatalf("sent %d tools, want only the active server tool", got) + } + if len(snap.ActiveTools) != 0 { + t.Fatalf("activeTools = %v, want empty (the only active tool is a server tool)", snap.ActiveTools) + } + if want := []string{"public_search"}; !equalStrings(snap.Enabled, want) { + t.Fatalf("enabled = %v, want %v (overridden server id)", snap.Enabled, want) + } + if want := []string{"list_orders"}; !equalStrings(snap.Disabled, want) { + t.Fatalf("disabled = %v, want %v", snap.Disabled, want) + } +} diff --git a/tool_types.go b/tool_types.go index b17ebd0..5936417 100644 --- a/tool_types.go +++ b/tool_types.go @@ -3,6 +3,7 @@ package agent import ( "context" "encoding/json" + "sort" "github.com/OpenRouterTeam/go-sdk/models/components" ) @@ -220,8 +221,19 @@ type ConversationState struct { PartialResponse *PartialResponse InterruptedBy *string Status ConversationStatus - CreatedAt int64 - UpdatedAt int64 + // ConsumedForcedToolChoiceKey records that a caller-configured *forced* + // tool choice has already produced a tool call, so a resumed run keeps it + // relaxed to `auto` instead of forcing the model back into tool calls + // (upstream #100). Only the semantic key is persisted, never a callback + // or a copied effective choice. + ConsumedForcedToolChoiceKey string `json:",omitempty"` + // DoomLoop is the persisted doom-loop detector state (upstream #73/#89): + // streak counters, the stop verdict that condemned the run, queued steer + // guidance, and the escalation budget consumed so far. Streaks survive + // serialize -> resume, because a resumed doom loop is still a doom loop. + DoomLoop *DoomLoopSerializedState `json:",omitempty"` + CreatedAt int64 + UpdatedAt int64 } type StateAccessor interface { @@ -265,3 +277,73 @@ func IsToolPreliminaryResultEvent(e ToolStreamEvent) bool { return e.Type == "to func IsToolResultEvent(e ToolStreamEvent) bool { return e.Type == "tool.result" } func IsTurnStartEvent(e ResponseStreamEvent) bool { return e.Type == "turn.start" } func IsTurnEndEvent(e ResponseStreamEvent) bool { return e.Type == "turn.end" } + +// ToolSetIdentified is implemented by tools that carry a stable tool-set +// activation id distinct from their wire name — server tools, whose default +// id is `server:`. +type ToolSetIdentified interface{ ToolSetID() string } + +// ToolSetIDOf returns the identity a ToolSet uses to address t: its +// ToolSetID when it declares one, else its wire name. +func ToolSetIDOf(t Tool) string { + if t == nil { + return "" + } + if identified, ok := t.(ToolSetIdentified); ok { + return identified.ToolSetID() + } + return t.ToolName() +} + +// ToolWithNextTurnParams is implemented by tools that compute request +// parameters for the turn following their execution. +type ToolWithNextTurnParams interface { + Tool + ToolNextTurnParams() NextTurnParamsFunctions +} + +// ToolNextTurnParamsOf returns t's nextTurnParams functions, or nil. +func ToolNextTurnParamsOf(t Tool) NextTurnParamsFunctions { + if t == nil { + return nil + } + if withParams, ok := t.(ToolWithNextTurnParams); ok { + return withParams.ToolNextTurnParams() + } + return nil +} + +// FilterToolsByIDs narrows tools to the ones named in active, preserving the +// original tools order (upstream `activeTools` on callModel). A nil `active` +// slice means "no filter". Server tools always pass: `active_tools` addresses +// client tools by wire name and a server tool is not nameable there, so +// filtering one out would silently drop provider-executed capability the +// caller never deactivated. +func FilterToolsByIDs(tools []Tool, active []string) []Tool { + if active == nil { + return tools + } + allowed := make(map[string]bool, len(active)) + for _, id := range active { + allowed[id] = true + } + out := make([]Tool, 0, len(tools)) + for _, t := range tools { + if IsServerTool(t) || allowed[t.ToolName()] || allowed[ToolSetIDOf(t)] { + out = append(out, t) + } + } + return out +} + +// sortedKeys returns m's keys in a deterministic (sorted) order. Go map +// iteration is randomized, so anywhere upstream relies on JS object +// insertion order this port uses a stable sort instead. +func sortedKeys[V any](m map[string]V) []string { + keys := make([]string, 0, len(m)) + for k := range m { + keys = append(keys, k) + } + sort.Strings(keys) + return keys +} diff --git a/upstreamer-changelog.md b/upstreamer-changelog.md index df94e6d..cba4e12 100644 --- a/upstreamer-changelog.md +++ b/upstreamer-changelog.md @@ -12,6 +12,194 @@ `stream_guards.go` predicates are now used by the streaming loop instead of sitting alongside duplicate inline logic. +## Doom-Loop Detection, Tool Sets, And Request Shaping (ported from `@openrouter/agent@0.9.0` and `@openrouter/agent@0.10.0`) + +### Doom-loop detection (opt-in) + +- Added doom-loop detection for the tool-execution loop, opted into with + `CallModelInput.DoomLoop` (`true` for the defaults, or a `DoomLoopConfig`). + It catches runs that stop making progress while continuing to spend: the + model re-issuing the same tool call with identical arguments in consecutive + rounds, repeating identical server-tool requests, or emitting the same text + tokens over and over. +- Detection is deterministic — a verdict is a pure function of the transcript — + and responds through a configurable graduated ladder: `observe` (emit the new + `DoomLoopDetected` hook) → `steer` (inject corrective guidance as a user + message) → `escalate` (run the next turn on a stronger model and/or force an + `openrouter:advisor` consult) → `block` (refuse the call with an explanatory + tool error, before execution) → `stop` (halt before any further model + request; unresolved calls get synthesized halt-error outputs so persisted + history stays well-formed, and `SessionEnd.Reason` is `doom_loop`). + The default ladder is observe at 2, block at 3, stop at 6. +- Streaks are round-scoped: N identical calls fanned out in parallel within one + round count once. A round's identity for one tool is the *set* of + fingerprints it was called with, so a repeated fan-out + (`read(a), read(b), read(c)` reissued verbatim) is detected — and at the + block rung every call of the repeating round is refused, so the fan-out stops + spending rather than only its last call being refused. Per-call streaks + accumulate alongside the round-set streak and the stronger evidence decides, + so a call repeating inside varying company (`[a,b]`, `[a,c]`, `[a,d]`) is + flagged too. A round that adds new work is progress: the new call always + executes. +- Tools declare call identity with `ToolConfig.LoopKey`: `Exempt: true` opts a + tool out statically, `Fields` narrows identity to named arguments, and `Fn` + computes it from the validated arguments (returning `nil` exempts that one + call). Absent means the full validated arguments. An empty or wholly-absent + field list, or a `Fn` that errors, falls back to the full arguments with a + warning rather than collapsing every call onto one identity. +- **Newly reachable false positive.** The detector compares arguments, not + results, so an agent that legitimately re-reads the same context files each + turn — or re-reads one anchor file while exploring new ones — now accumulates + evidence and is refused at the default `block` rung from round 3. Exempt such + tools with `LoopKey: &LoopKey{Exempt: true}` (or a `Fn` returning `nil`). The + ladder gives every shape a free round and an `observe` warning first. +- Fingerprints are a cross-port contract: RFC 8785 (JCS) canonical JSON + + SHA-256 over UTF-8, lowercase hex, with the tool name participating + (`sha256(toolName + "\n" + jcs(keyMaterial))`). `CanonicalizeKeyMaterial`, + `FingerprintKeyMaterial` and `FingerprintToolCall` are exported, and the + upstream conformance vectors are asserted byte-for-byte, so persisted + detector state transfers between the TypeScript, Python and Go ports. + Unhashable key material (non-finite numbers, cycles, nesting past 64 levels) + falls back to the full-arguments identity — detection never fails a run. +- Detector state persists inside `ConversationState.DoomLoop`: streaks survive + serialize → resume (including the fan-out's fingerprint set and per-call + counts), a `stop` verdict survives decision-only resumes (approve/reject) and + clears on a fresh conversational turn, and queued steer guidance is delivered + on resume. Pre-existing state blobs restore cleanly. +- New public surface: `DoomLoopMonitor` (with `DeclareRound`, `RecordToolCall`, + `RecordAssistantText`, `State`, `Restore`, `CanEscalate`, + `ConsumeEscalation`), `ResolveDoomLoopOption`, `ResolvedDoomLoopConfig`, + `ResolveLadderAction`, `ResolveLoopKeyMaterial`, `DetectTextRepetition`, the + `DoomLoopAt`/`DoomLoopOff` threshold constructors, and + `ModelResult.DoomLoopVerdict(ctx)` which reports a stopping verdict. + Escalations are budgeted (`MaxEscalations`, default 2), consumed when a + recovery is *applied*, and the budget persists so a resume cannot reset it. +- Added the `DoomLoopDetected` lifecycle hook. It fires at every rung, + including `observe`, so you can watch for loops without changing behavior; a + handler may override the action in either direction. Two downgrades are + enforced rather than trusted: a text or server-tool verdict cannot be blocked + (the tokens are already emitted / the tool already ran), and an `escalate` + override needs an escalation config with budget remaining. Both downgrade to + `observe`, never silently to a stronger action. + +### Aggregate usage + +- Added `ModelResult.Usage(ctx)`: aggregate token and cost totals across + **every** model call a run made — the initial request, each tool-round + follow-up, the empty-final retry, the forced final turn, and approval-resume + requests. `Response(ctx)` resolves to the *final* round's response, so in a + multi-round tool loop the intermediate generations' tokens were previously + unreachable. It returns the same `SessionUsageTotals` shape as the + `SessionEnd` hook's `TotalUsage`, read from one snapshot so the two cannot + drift, and it never returns an error — a failed run still consumed tokens. + Accumulation is independent of the hook system, so the totals are correct for + callers who configured no hooks at all. `Cost` is non-nil only when the + server reported cost for at least one call. + +### Forced tool choice relaxes after it is spent + +- A caller-configured *forced* `tool_choice` (`required`, a specific tool, or + `allowed_tools` with `mode: "required"`) is now relaxed to `auto` on + follow-up turns once it has actually produced a tool call — including + follow-ups resumed after an approval, HITL, or client-tool pause. Previously + the model could be forced to keep calling tools until the step budget ran out + instead of ever synthesizing a final text answer. + `allowed_tools` keeps its tool set and only loses `mode: "required"`. + A choice whose semantic value changes re-arms immediately, and an unforced + turn clears the consumption so the same forced value works again later. + The consumed identity persists as + `ConversationState.ConsumedForcedToolChoiceKey`, so a resume agrees with the + run that paused. `IsForcedToolChoice` and `RelaxForcedToolChoice` are + exported. + +### Tool-computed next-turn parameters + +- `ToolConfig.NextTurnParams` is now wired into the loop and supports + `toolChoice` alongside `model`, `models`, `input`, `temperature`, + `maxOutputTokens`, `topP`, `topK` and `instructions`. This is what a + tool-search tool needs: declare every tool up front, keep the not-yet-needed + ones out of reach behind an `allowed_tools` choice, and widen that choice as + the model discovers what it wants. Because the `tools` array is unchanged + across turns, the provider's prompt-cache prefix survives. A tool-computed + `toolChoice` becomes the new caller-level policy (and re-arms the + forced-choice relaxation above) rather than a one-turn override. Functions + for one turn compose: each sees the values written by the ones before it. + +### Tool sets and `ActiveTools` + +- Added `CreateToolSet` / `MustCreateToolSet` (`ToolSet`), a declarative + activation layer over a set of tools: `Activate`, `Deactivate`, + `ActivateWhen`, `DeactivateWhen` (plus the `…All` map forms), + `DefineSituations` for named partition overlays, `Resolve` / + `ResolveSituation` returning a snapshot, and `Clone`. Mutators are immutable + by default — each returns a new `ToolSet` and leaves the receiver untouched — + while a set created with `Mutable: true` applies changes in place and returns + itself. Predicates are evaluated at `Resolve` time against the conversation + state and a shared context map. The resolved snapshot carries the active + tools in construction order, the `ActiveTools` id list, the enabled/disabled + id lists, and an exhaustive per-tool status map explaining each decision. +- Added `CallModelInput.ActiveTools`, which narrows which of `Tools` are sent + to the model for a given call. `ResolvedToolSnapshot.Apply` writes a + snapshot's tools and active ids onto a `CallModelInput` in one step. Server + tools always pass the filter, because `active_tools` addresses client tools by + wire name and a server tool is not nameable there. The filter applies to the + executor as well as the request, so the model is never offered a filtered + tool and the executor never carries a definition the request did not + advertise; a call naming a filtered-out tool therefore has nothing to run it + and surfaces as `awaiting_client_tools`. +- Server tools now carry a stable tool-set id: `ServerToolConfig.ID` overrides + the default `server:`, so two server tools of the same type can have + distinct activation ids. Note that Go's zero value collapses upstream's + `undefined` and `''`, so an empty `ID` means "use the default" rather than + the error upstream raises for `id: ''`. + +### Other behavior changes + +- `ToolConfig.Strict` is now passed through to the wire tool definition + verbatim, so providers can enforce structured-outputs-style schema adherence + on tool-call arguments. **Behavior change:** previously every function tool + was serialized with `strict: true` regardless of what the caller asked for; + the field is now omitted unless you set it, which is the provider default. + Set `Strict: &yes` to restore the old wire shape. OpenAI-style strict schemas + require every declared object property to appear in `required`. +- Fixed `SessionEnd.Reason`, which previously reported `complete` for every + run that did not error. It now reports `max_turns` when a `StopWhen` + condition halts a tool-call turn or the turn budget runs out, `user` for an + interrupted conversation, `doom_loop` for a doom-loop stop, `error` for a + failed run, and `complete` only when the model actually produced a final + answer. A caller watching for runaway agents could not previously see them. +- The executed tool call is now threaded into the tool execute context's turn + context (`ToolExecuteContext.Turn.ToolCall`) on the streaming path too. It + was previously populated only by the non-streaming orchestrator, so + `Execute` / `OnToolCalled` saw a nil call during a normal streaming run. A + caller-supplied turn context still wins. + +### Not yet ported from 0.9.0 + +- **Async tool support (upstream #90) is not ported.** The unified `run()` tool + interface with `lifecycle: 'sync' | 'background' | 'deferred'`, model-side + task check-ins and the universal `task` tool, steering + (`sendToTask`/`queueUserMessage`), subagent tools (`tool.agent()`), per-tool + cancellation and timeouts, and tool-concurrency controls are all absent from + this port. The `awaiting_async_tools` conversation status, the + `PendingAsyncTools`/`SettledAsyncCallIDs` state fields, the + `tool.async_started`/`tool.async_settled` events, `GetAsyncTasks` and + `ResumeToolResults` are likewise absent. Every tool in this port executes + synchronously within its round, which is upstream's `lifecycle: 'sync'` + default — so existing code is unaffected, but a workload that needs durable + external work or a long-running background task cannot be expressed here yet. + A half-implemented version of this feature would be worse than its absence: + it would present upstream's API while diverging on pause/resume semantics and + event ordering. Tracked as the next port's headline. +- The MCP integration upstream moved under the `@openrouter/agent/mcp` subpath + (upstream #102). MCP remains out of this port's scope, so the relocation is a + no-op here. +- Upstream's `validateFinalResponse` diagnostics (upstream #95) have no + analogue: this port deliberately never turns an empty final response into a + hard error, because the Go SDK's response type carries a separate + `OutputText` field alongside `Output` items, so an empty `Output` array is not + on its own a reliable invalidity signal. See `StrictFinalResponse`. + ## Lifecycle Hooks, Versioned State, And 0.8.0 Parity (ported from `@openrouter/agent@0.8.0`) - Added `HooksManager` (`NewHooksManager`), a typed lifecycle-hook system with the nine built-in hooks — `PreToolUse`, `PostToolUse`, `PostToolUseFailure`, `UserPromptSubmit`, `Stop`, `PermissionRequest`, `SessionStart`, `SessionEnd`, and `PostModelCall` — plus fully custom hooks via the generic `On`/`Emit` functions. Register built-ins with the typed `OnXxx`/`EmitXxx` methods (`manager.OnPreToolUse(...)`, etc.). @@ -68,6 +256,21 @@ | `result.getToolCalls()` | `result.ToolCalls(ctx)` | | `result.getFullResponsesStream()` | `result.FullResponsesStream(ctx)` | | `result.getNewMessagesStream()` | `result.NewMessagesStream(ctx)` | +| `result.getUsage()` | `result.Usage(ctx)` | +| `result.getDoomLoopVerdict()` | `result.DoomLoopVerdict(ctx)` | +| `doomLoop` option | `CallModelInput.DoomLoop` | +| `activeTools` option | `CallModelInput.ActiveTools` | +| `createToolSet(...)` | `CreateToolSet(ToolSetOptions)` / `MustCreateToolSet` | +| `toolSet.defineSituations(...)` | `ToolSet.DefineSituations(map[string]SituationConfig)` | +| `toolSet.resolve(...)` | `ToolSet.Resolve(ActivationInput)` | +| `toolSet.inferTools(...)` | `ToolSet.Resolve(...)` + `ResolvedToolSnapshot.Apply` | +| `resolveDoomLoopOption(...)` | `ResolveDoomLoopOption(any) *ResolvedDoomLoopConfig` | +| `monitor.declareRound(...)` | `DoomLoopMonitor.DeclareRound(round, []DoomLoopRoundCall)` | +| `monitor.recordToolCall(...)` | `DoomLoopMonitor.RecordToolCall(name, keyMaterial, round, RecordOptions)` | +| `loopKey` on a tool | `ToolConfig.LoopKey` (`*LoopKey`) | +| `strict` on a tool | `ToolConfig.Strict` (`*bool`) | +| `nextTurnParams` on a tool | `ToolConfig.NextTurnParams` | +| `serverTool(config, { id })` | `ServerToolConfig.ID` | | `createInitialState()` | `CreateInitialState()` | | `appendToMessages(...)` | `AppendToMessages(...)` | | `updateState(...)` | `UpdateState(...)` | diff --git a/usage_test.go b/usage_test.go new file mode 100644 index 0000000..fbd301f --- /dev/null +++ b/usage_test.go @@ -0,0 +1,213 @@ +package agent + +// ModelResult.Usage tests (upstream #97 `getUsage()`). +// +// The bug this feature fixes is that Response resolves to the FINAL round's +// response, so a multi-round tool loop's intermediate generations were +// unreachable. The tests therefore assert *sums across rounds*, and that the +// aggregate is correct with no hooks configured at all — previously it only +// advanced as a side effect of PostModelCall emission. + +import ( + "context" + "testing" + + "github.com/OpenRouterTeam/go-sdk/models/components" + "github.com/OpenRouterTeam/go-sdk/models/operations" + "github.com/OpenRouterTeam/go-sdk/optionalnullable" +) + +// usageOf builds a response carrying a usage block. +func usageOf(resp components.OpenResponsesResult, in, out, total, cached, reasoning int64, cost *float64) components.OpenResponsesResult { + usage := components.Usage{ + InputTokens: in, + OutputTokens: out, + TotalTokens: total, + InputTokensDetails: components.InputTokensDetails{CachedTokens: cached}, + OutputTokensDetails: components.OutputTokensDetails{ReasoningTokens: reasoning}, + } + if cost != nil { + usage.Cost = optionalnullable.From(cost) + } + resp.Usage = optionalnullable.From(&usage) + return resp +} + +func TestUsageSumsAcrossEveryRoundOfAToolLoop(t *testing.T) { + firstCost, secondCost := 0.001, 0.002 + first := usageOf(toolCallTurn("resp_1", "call_1"), 10, 5, 15, 2, 1, &firstCost) + second := usageOf(textTurn("resp_2", "done"), 20, 7, 27, 3, 4, &secondCost) + sender := twoTurnSender(first, second) + + result, err := CallModel(context.Background(), sender, CallModelInput{Model: "openai/test", Input: "hi", Tools: []Tool{searchTool(t)}}) + if err != nil { + t.Fatal(err) + } + if _, err := result.Text(context.Background()); err != nil { + t.Fatal(err) + } + + usage := result.Usage(context.Background()) + if usage.ModelCalls != 2 { + t.Fatalf("modelCalls = %d, want 2", usage.ModelCalls) + } + if usage.InputTokens != 30 || usage.OutputTokens != 12 || usage.TotalTokens != 42 { + t.Fatalf("token totals = %+v, want in=30 out=12 total=42", usage.ModelCallUsage) + } + if usage.CachedTokens != 5 || usage.ReasoningTokens != 5 { + t.Fatalf("cached=%d reasoning=%d, want 5/5", usage.CachedTokens, usage.ReasoningTokens) + } + if usage.Cost == nil || *usage.Cost < 0.0029 || *usage.Cost > 0.0031 { + t.Fatalf("cost = %v, want ~0.003", usage.Cost) + } + + // The point of the accessor: the final response alone reports only the + // last round. + resp, err := result.Response(context.Background()) + if err != nil { + t.Fatal(err) + } + finalUsage, ok := resp.Usage.GetOrZero() + if !ok || finalUsage.TotalTokens != 27 { + t.Fatalf("final response usage = %+v, want the last round's 27 total tokens", finalUsage) + } +} + +// Accumulation must not depend on the hook system. +func TestUsageAccumulatesWithNoHooksConfigured(t *testing.T) { + created := operations.CreateCreateResponsesResponseOpenResponsesResult(usageOf(textTurn("resp_1", "hi"), 4, 6, 10, 0, 0, nil)) + sender := &fakeSender{responses: []*operations.CreateResponsesResponse{&created}} + result, err := CallModel(context.Background(), sender, CallModelInput{Model: "openai/test", Input: "hi"}) + if err != nil { + t.Fatal(err) + } + usage := result.Usage(context.Background()) + if usage.ModelCalls != 1 || usage.TotalTokens != 10 { + t.Fatalf("usage without hooks = %+v", usage) + } + // cost is absent, not zero: a 0 would be indistinguishable from "free". + if usage.Cost != nil { + t.Fatalf("cost = %v, want nil when the server reported none", *usage.Cost) + } +} + +// Awaiting Usage first must drive the run to completion, exactly like +// Response does. +func TestUsageAwaitsRunCompletion(t *testing.T) { + first := usageOf(toolCallTurn("resp_1", "call_1"), 10, 0, 10, 0, 0, nil) + second := usageOf(textTurn("resp_2", "done"), 5, 0, 5, 0, 0, nil) + sender := twoTurnSender(first, second) + result, err := CallModel(context.Background(), sender, CallModelInput{Model: "openai/test", Input: "hi", Tools: []Tool{searchTool(t)}}) + if err != nil { + t.Fatal(err) + } + if usage := result.Usage(context.Background()); usage.ModelCalls != 2 || usage.TotalTokens != 15 { + t.Fatalf("usage read first = %+v, want both rounds counted", usage) + } +} + +// A response with no usage block still counts as a model call, and leaves +// cost absent. +func TestUsageCountsUsagelessResponsesInModelCalls(t *testing.T) { + created := operations.CreateCreateResponsesResponseOpenResponsesResult(textTurn("resp_1", "hi")) + sender := &fakeSender{responses: []*operations.CreateResponsesResponse{&created}} + result, err := CallModel(context.Background(), sender, CallModelInput{Model: "openai/test", Input: "hi"}) + if err != nil { + t.Fatal(err) + } + usage := result.Usage(context.Background()) + if usage.ModelCalls != 1 { + t.Fatalf("modelCalls = %d, want 1", usage.ModelCalls) + } + if usage.TotalTokens != 0 || usage.Cost != nil { + t.Fatalf("usage = %+v, want zeroed tokens and no cost", usage) + } +} + +// A run that failed before any model call completed reports zeroes rather +// than an error: a failed run still consumed tokens, and cost accounting +// usually runs where a second error would mask the original one. +func TestUsageNeverFailsAndReportsWhatAccrued(t *testing.T) { + created := operations.CreateCreateResponsesResponseEventStream( + eventStreamFrom(t, []components.StreamEvents{textDelta("partial", 1)}, true), + ) + sender := &fakeSender{responses: []*operations.CreateResponsesResponse{&created}} + result, err := CallModel(context.Background(), sender, CallModelInput{Model: "openai/test", Input: "hi"}) + if err != nil { + t.Fatal(err) + } + if _, err := result.Text(context.Background()); err == nil { + t.Fatal("expected the incomplete stream to fail the run") + } + usage := result.Usage(context.Background()) + if usage.ModelCalls != 0 || usage.TotalTokens != 0 { + t.Fatalf("usage after a failed run = %+v, want zeroes", usage) + } +} + +// Usage and the SessionEnd hook payload read from one snapshot, so they can +// never drift. +func TestUsageAgreesWithSessionEndTotalUsage(t *testing.T) { + cost := 0.005 + first := usageOf(toolCallTurn("resp_1", "call_1"), 11, 3, 14, 1, 2, &cost) + second := usageOf(textTurn("resp_2", "done"), 9, 4, 13, 0, 1, nil) + sender := twoTurnSender(first, second) + + var hooked *SessionUsageTotals + manager := NewHooksManager() + manager.OnSessionEnd(HookEntry[SessionEndPayload, EmptyHookResult]{ + Handler: func(payload SessionEndPayload, _ LifecycleHookContext) (HookHandlerResult[EmptyHookResult], error) { + if payload.TotalUsage != nil { + totals := *payload.TotalUsage + hooked = &totals + } + return VoidResult[EmptyHookResult](), nil + }, + }) + + result, err := CallModel(context.Background(), sender, CallModelInput{ + Model: "openai/test", Input: "hi", Tools: []Tool{searchTool(t)}, Hooks: manager, + }) + if err != nil { + t.Fatal(err) + } + if _, err := result.Text(context.Background()); err != nil { + t.Fatal(err) + } + usage := result.Usage(context.Background()) + if hooked == nil { + t.Fatal("SessionEnd did not carry TotalUsage") + } + if hooked.ModelCalls != usage.ModelCalls || hooked.TotalTokens != usage.TotalTokens { + t.Fatalf("SessionEnd totals %+v disagree with Usage %+v", *hooked, usage) + } + if (hooked.Cost == nil) != (usage.Cost == nil) { + t.Fatalf("cost presence disagrees: hook=%v accessor=%v", hooked.Cost, usage.Cost) + } +} + +// A run paused for approval reports only what completed before the pause. +func TestUsageOnAPausedRunReportsOnlyCompletedCalls(t *testing.T) { + tool := MustNewTool(ToolConfig[sampleInput]{Name: "search", RequireApproval: true, Execute: func(context.Context, sampleInput, ToolExecuteContext) (any, error) { + return "ok", nil + }}) + paused := usageOf(toolCallTurn("resp_1", "call_1"), 8, 2, 10, 0, 0, nil) + created := operations.CreateCreateResponsesResponseOpenResponsesResult(paused) + sender := &fakeSender{responses: []*operations.CreateResponsesResponse{&created}} + result, err := CallModel(context.Background(), sender, CallModelInput{Model: "openai/test", Input: "hi", Tools: []Tool{tool}}) + if err != nil { + t.Fatal(err) + } + if _, err := result.State(context.Background()); err != nil { + t.Fatal(err) + } + usage := result.Usage(context.Background()) + if usage.ModelCalls != 1 || usage.TotalTokens != 10 { + t.Fatalf("paused-run usage = %+v, want the single pre-pause call", usage) + } + // Idempotent: reading it again drives no further requests. + _ = result.Usage(context.Background()) + if sender.calls != 1 { + t.Fatalf("reading usage drove %d requests, want 1", sender.calls) + } +}