diff --git a/packages/prompt-variance/src/index.ts b/packages/prompt-variance/src/index.ts index b46dd9fe4..c535243e2 100644 --- a/packages/prompt-variance/src/index.ts +++ b/packages/prompt-variance/src/index.ts @@ -10,3 +10,16 @@ export { type PromptVarianceFamily, type PromptVarianceRow, } from "./rows.js"; + +// Astra tool-evasion residual (CL-9027): forensics on the repro trace showed +// evasion, not waste — the gpt-6-astra leaf re-issues the same effective tool +// call with trivial argument deltas (path prefix, default offset/limit, +// padding whitespace), so no exact tool fingerprint repeats 3x and the shared +// threshold guard stays silent. This forbids that specific variation +// (muse-rule precedent, CL-7869); no signature normalization ships in +// first-party code. Named export rather than a family row: it travels the +// ModelFamilyPolicy.promptResidual seam, composed over the gpt narrate note. +export const astraResidual: string = + "Tool discipline (gpt-6-astra worker):\n" + + "- Never re-issue a tool call that repeats a prior call with only trivial argument changes (path prefix, offset, limit, whitespace).\n" + + "- Never re-read a file you have already read this session."; diff --git a/src/agent/model-family-policy.test.ts b/src/agent/model-family-policy.test.ts index 789bfd0c0..b800e2805 100644 --- a/src/agent/model-family-policy.test.ts +++ b/src/agent/model-family-policy.test.ts @@ -200,4 +200,200 @@ describe("resolveModelFamilyPolicy", () => { expect(gpt.toolDisciplineRules).toBeUndefined(); expect(gpt.advertisedToolDeny).toEqual([]); }); + + describe("astra repro trace (CL-9027)", () => { + // Minimal failing session-trace fixture: 8 consecutive tool-only turns + // from a gpt-6-astra leaf (tool names + args + result sizes per turn). + // Fingerprint and repeat-count semantics mirror + // scripts/tool-fingerprint-forensics.ts (stableJson exact signatures, + // largest exact-repeat count per period 1-6): the shared threshold guard + // fires only on exact repeats, so a loop that varies trivial argument + // details escapes it. That is evasion, not threshold-tolerated waste — + // and the Step-3 residual forbids exactly this variation. The + // near-identical grouping below is test-only forensics; no signature + // normalization ships in first-party code. + interface ReproTurn { + tool: string; + args: Record; + resultTokens: number; + } + const ASTRA_REPRO_TRACE: readonly ReproTurn[] = [ + { + tool: "read", + args: { path: "src/agent/prompts.ts" }, + resultTokens: 3200, + }, + { + tool: "read", + args: { path: "./src/agent/prompts.ts" }, + resultTokens: 3200, + }, + { + tool: "read", + args: { path: "src/agent/prompts.ts", offset: 1 }, + resultTokens: 3180, + }, + { + tool: "grep", + args: { pattern: "narrate", path: "src/agent" }, + resultTokens: 420, + }, + { + tool: "read", + args: { path: "src/agent/prompts.ts" }, + resultTokens: 3200, + }, + { + tool: "grep", + args: { pattern: "narrate ", path: "src/agent" }, + resultTokens: 420, + }, + { + tool: "read", + args: { path: "./src/agent/prompts.ts", limit: 2000 }, + resultTokens: 3200, + }, + { + tool: "read", + args: { path: "src/agent/prompts.ts", offset: 0 }, + resultTokens: 3200, + }, + ]; + + function stableJson(value: unknown): string { + if (value === null || typeof value !== "object") + return JSON.stringify(value); + if (Array.isArray(value)) return `[${value.map(stableJson).join(",")}]`; + const obj = value as Record; + const keys = Object.keys(obj).sort(); + return `{${keys.map((k) => `${JSON.stringify(k)}:${stableJson(obj[k])}`).join(",")}}`; + } + + function fingerprint(turn: ReproTurn): string { + return `${turn.tool}:${stableJson(turn.args)}`; + } + + function maxExactRepeats(fps: readonly string[]): number { + let best = 1; + for (let period = 1; period <= 6; period++) { + for (let end = 1; end <= fps.length; end++) { + const prefix = fps.slice(0, end); + let i = prefix.length - 1; + let j = i - period; + let matched = 0; + while (j >= 0 && prefix[i] === prefix[j]) { + matched++; + i--; + j--; + } + const reps = Math.floor((matched + period) / period); + if (reps > best) best = reps; + } + } + return best; + } + + function evasionKey(turn: ReproTurn): string { + const args = { ...turn.args }; + // The trivial deltas this trace varies: a leading ./ prefix, default + // offset/limit values, and padding whitespace. + if (typeof args.path === "string") + args.path = args.path.replace(/^\.\//, ""); + if (args.offset === 0 || args.offset === 1) delete args.offset; + if (typeof args.limit === "number") delete args.limit; + if (typeof args.pattern === "string") args.pattern = args.pattern.trim(); + return `${turn.tool}:${stableJson(args)}`; + } + + function classifyAstraTrace(trace: readonly ReproTurn[]): string { + if (maxExactRepeats(trace.map(fingerprint)) >= 3) return "waste"; + const groups = new Map(); + for (const turn of trace) { + const key = evasionKey(turn); + groups.set(key, (groups.get(key) ?? 0) + 1); + } + return Math.max(...groups.values()) >= 3 ? "evasion" : "waste"; + } + + test("classifies as evasion: no exact repeat trips the shared guard", () => { + expect(ASTRA_REPRO_TRACE).toHaveLength(8); + expect(maxExactRepeats(ASTRA_REPRO_TRACE.map(fingerprint))).toBeLessThan( + 3, + ); + expect(classifyAstraTrace(ASTRA_REPRO_TRACE)).toBe("evasion"); + }); + + test("pins the offending pattern and the wasted turn/token delta", () => { + const groups = new Map(); + for (const turn of ASTRA_REPRO_TRACE) { + const key = evasionKey(turn); + groups.set(key, (groups.get(key) ?? 0) + 1); + } + // Six re-reads of one file plus two re-greps of one pattern, each run + // differing only by a trivial argument delta. + expect(groups.get('read:{"path":"src/agent/prompts.ts"}')).toBe(6); + expect(groups.get('grep:{"path":"src/agent","pattern":"narrate"}')).toBe( + 2, + ); + const wastedTokens = ASTRA_REPRO_TRACE.reduce( + (sum, turn) => sum + turn.resultTokens, + 0, + ); + expect(wastedTokens).toBe(20020); + }); + }); + + test("astra resolves to astra with the composed residual; thresholds stay default (CL-9027)", () => { + const base = resolveModelFamilyPolicy({ + providerName: "unknown-provider", + model: "unknown-model", + }); + const gpt = resolveModelFamilyPolicy({ + providerName: "openai", + model: "gpt-5.6", + }); + for (const orchestrator of [false, true]) { + const astra = resolveModelFamilyPolicy({ + providerName: "codex/default", + model: "gpt-6-astra", + orchestrator, + }); + expect(astra.family).toBe("astra"); + // Keeps the gpt narrate nudge and adds the evasion-specific rules. + expect(astra.promptResidual).toContain( + "Narrate before tools (GPT worker):", + ); + expect(astra.promptResidual).toContain("trivial argument changes"); + expect(astra.promptResidual).not.toBe(gpt.promptResidual); + // Evasion earns a forbidding residual, not tighter thresholds. + expect(astra.toolOnlyTurnNudgeAt).toBe(base.toolOnlyTurnNudgeAt); + expect(astra.subAgentStallTimeoutMs).toBe(base.subAgentStallTimeoutMs); + expect(astra.applyGrokFinishBias).toBe(false); + expect(astra.toolDisciplineRules).toBeUndefined(); + expect(astra.advertisedToolDeny).toEqual([]); + } + }); + + test("sol and generic gpt stay gpt with the byte-identical narrate residual", () => { + const generic = resolveModelFamilyPolicy({ + providerName: "openai", + model: "gpt-5.6", + }); + for (const model of [ + "gpt-5.6-sol", + "gpt-5.5", + "gpt-5.6-luna", + "gpt-5.6-terra", + ] as const) { + for (const orchestrator of [false, true]) { + const policy = resolveModelFamilyPolicy({ + providerName: "codex/default", + model, + orchestrator, + }); + expect(policy.family).toBe("gpt"); + expect(policy.promptResidual).toBe(generic.promptResidual); + } + } + }); }); diff --git a/src/agent/model-family-policy.ts b/src/agent/model-family-policy.ts index ca0ad69f0..5301e8bed 100644 --- a/src/agent/model-family-policy.ts +++ b/src/agent/model-family-policy.ts @@ -3,6 +3,7 @@ import { type ModelFamily, } from "../subagent/provider-family.js"; import { + astraResidual, claudeRow, gptRow, grokRow, @@ -164,9 +165,12 @@ const CLAUDE_POLICY: Omit = { // Deliberately not manage_tasks ceremony — that is CL-7769, not this text. // The text lives here (policy owns data); buildGptNarrateBeforeToolsNote // (prompts.ts) returns it verbatim so the prompt carries exactly one copy. -// Served cells (astra/sol/terra/…) are never named here — CL-8265 -// characterizes them later. Single-sourced from the versioned -// prompt-variance package (CL-8269); the name stays for existing importers. +// Served cells (sol/terra/…) are never named here — CL-8265 characterizes +// them later. Astra is the one exception: forensics (CL-9027) showed the +// gpt-6-astra cell evading the shared threshold guard via trivial argument +// deltas, so it carries its own residual below. Single-sourced from the +// versioned prompt-variance package (CL-8269); the name stays for existing +// importers. export const GPT_NARRATE_BEFORE_TOOLS_NOTE = gptRow.residual; // GPT (Codex / gpt-*) thresholds are provisional: we have no eval @@ -181,6 +185,21 @@ const GPT_POLICY: Omit = { promptResidual: GPT_NARRATE_BEFORE_TOOLS_NOTE, }; +// Astra (served gpt-6-astra cell) is GPT plus the evasion residual. Forensics +// on the repro trace (see model-family-policy.test.ts) classified the loop as +// evasion — near-identical re-issued calls whose trivial argument deltas keep +// every exact fingerprint under the shared threshold — so the residual forbids +// that specific variation (muse-rule precedent, CL-7869) instead of tightening +// thresholds: toolOnlyTurnNudgeAt stays at the permissive default. No +// signature normalization ships in first-party code. +export const ASTRA_PROMPT_RESIDUAL = `${GPT_NARRATE_BEFORE_TOOLS_NOTE}\n${astraResidual}`; + +const ASTRA_POLICY: Omit = { + ...GPT_POLICY, + // Like gpt, primary and leaf alike: no orchestrator carve-out. + promptResidual: ASTRA_PROMPT_RESIDUAL, +}; + export function resolveModelFamilyPolicy(input: { providerName: string; model?: string; @@ -217,6 +236,10 @@ export function resolveModelFamilyPolicy(input: { case "gpt": // Primary and leaf alike: no orchestrator carve-out. return { family, ...GPT_POLICY }; + case "astra": + // Primary and leaf alike, like gpt: no orchestrator carve-out. Generic + // gpt prompts are byte-identical — only astra carries the residual. + return { family, ...ASTRA_POLICY }; default: return { family: "default", ...DEFAULT_POLICY }; } diff --git a/src/subagent/provider-family.test.ts b/src/subagent/provider-family.test.ts index 4970d8f34..0eae482d3 100644 --- a/src/subagent/provider-family.test.ts +++ b/src/subagent/provider-family.test.ts @@ -1,6 +1,7 @@ import { describe, expect, test } from "bun:test"; import { detectModelFamily, + isAstraLeafProvider, isClaudeLeafProvider, isGptProvider, isKimiLeafProvider, @@ -229,16 +230,23 @@ describe("isGptProvider (CL-8310)", () => { ).toBe(true); }); - test("covers every in-tree codex catalog model without naming cells", () => { - // No terra/sol/astra special-casing: every served codex id resolves via - // the generic codex-provider / gpt-* match, so future cells ride along. + test("astra branches to its own family; other cells ride the generic gpt match", () => { + // CL-9027 reverses the no-special-casing rule for astra only: the + // gpt-6-astra cell doom-loops, so it resolves to the astra family while + // sol/terra/luna and generic gpt ids keep the generic gpt match. for (const model of CODEX_DEFAULT_MODELS) { expect(isGptProvider({ providerName: "codex/default", model })).toBe( true, ); - expect(detectModelFamily({ providerName: "codex/default", model })).toBe( - "gpt", - ); + const family = detectModelFamily({ + providerName: "codex/default", + model, + }); + if (model.toLowerCase().startsWith("gpt-6-astra")) { + expect(family).toBe("astra"); + } else { + expect(family).toBe("gpt"); + } } }); @@ -264,3 +272,49 @@ describe("isGptProvider (CL-8310)", () => { expect(isGptProvider({ providerName: "anthropic" })).toBe(false); }); }); + +describe("isAstraLeafProvider (CL-9027)", () => { + test("matches the served gpt-6-astra cell id on any provider", () => { + expect( + isAstraLeafProvider({ + providerName: "codex/default", + model: "gpt-6-astra", + }), + ).toBe(true); + expect( + isAstraLeafProvider({ + providerName: "openai-compat", + model: "GPT-6-ASTRA", + }), + ).toBe(true); + expect( + detectModelFamily({ + providerName: "codex/default", + model: "gpt-6-astra", + }), + ).toBe("astra"); + }); + + test("rejects sol, generic gpt, and other families", () => { + for (const input of [ + { providerName: "codex/default", model: "gpt-5.6-sol" }, + { providerName: "codex/default", model: "gpt-5.6-terra" }, + { providerName: "codex/default", model: "gpt-5.6-luna" }, + { providerName: "openai", model: "gpt-5.6" }, + { providerName: "codex", model: "gpt-5.1" }, + { providerName: "xai/default", model: "grok-4.6" }, + { providerName: "anthropic", model: "claude-sonnet-4" }, + ] as const) { + expect(isAstraLeafProvider(input)).toBe(false); + } + expect( + detectModelFamily({ + providerName: "codex/default", + model: "gpt-5.6-sol", + }), + ).toBe("gpt"); + expect( + detectModelFamily({ providerName: "openai", model: "gpt-5.6" }), + ).toBe("gpt"); + }); +}); diff --git a/src/subagent/provider-family.ts b/src/subagent/provider-family.ts index c1ac21076..8a42497e4 100644 --- a/src/subagent/provider-family.ts +++ b/src/subagent/provider-family.ts @@ -50,12 +50,26 @@ export function isClaudeLeafProvider(input: { return false; } +/** + * True when the model id is the served gpt-6-astra cell. Forensics (CL-9027) + * showed that cell doom-looping via trivial argument deltas, so it resolves + * to its own family with an evasion-specific residual; other served cells + * (sol/terra/luna) keep riding the generic gpt match below. + */ +export function isAstraLeafProvider(input: { + providerName: string; + model?: string; +}): boolean { + return input.model !== undefined && /^gpt-6-astra/i.test(input.model.trim()); +} + /** * True when the inference path is the GPT family: a Codex provider name * (codex/ OAuth profiles, the codex-responses adapter, bare codex) or a - * gpt-* model id on any provider. Served codex cells (astra/sol/terra/luna) + * gpt-* model id on any provider. Served codex cells (sol/terra/luna) * all match the generic gpt-* model shape — never name them here; CL-8265 - * characterizes cells later. + * characterizes cells later. Astra is the one exception: it branches to its + * own family in detectModelFamily below (CL-9027). */ export function isGptProvider(input: { providerName: string; @@ -76,6 +90,7 @@ export type ModelFamily = | "muse" | "claude" | "gpt" + | "astra" | "default"; /** @@ -92,6 +107,7 @@ export function detectModelFamily(input: { if (isKimiLeafProvider(input)) return "kimi"; if (isMuseSparkLeafProvider(input)) return "muse"; if (isClaudeLeafProvider(input)) return "claude"; + if (isAstraLeafProvider(input)) return "astra"; if (isGptProvider(input)) return "gpt"; return "default"; }