From ccf1307cc16761192a591608412a0798d7e274d0 Mon Sep 17 00:00:00 2001 From: Sawyer Date: Sat, 19 Sep 2026 10:22:00 -0700 Subject: [PATCH] feat: director prompt render-style switch (markdown default, xml experiment) --- src/agent/directors/prompt-sections.test.ts | 124 ++++++++++ src/agent/directors/prompt-sections.ts | 242 ++++++++++++++++++++ src/agent/directors/registry.ts | 4 +- src/exec/runner.ts | 4 +- src/subagent/agent-fleet.ts | 10 +- 5 files changed, 374 insertions(+), 10 deletions(-) create mode 100644 src/agent/directors/prompt-sections.test.ts create mode 100644 src/agent/directors/prompt-sections.ts diff --git a/src/agent/directors/prompt-sections.test.ts b/src/agent/directors/prompt-sections.test.ts new file mode 100644 index 000000000..a36f1ed3d --- /dev/null +++ b/src/agent/directors/prompt-sections.test.ts @@ -0,0 +1,124 @@ +import { afterEach, describe, expect, test } from "bun:test"; + +import { DIRECTOR_REGISTRY } from "./registry.js"; +import { DIRECTOR_IDS } from "./types.js"; +import { formatDirectorSystemPrompt } from "./identity.js"; +import { + FALLBACK_SECTION_TAG, + normalizeSectionTag, + PROMPT_RENDER_STYLE_ENV_VAR, + PROMPT_SECTION_TAG_VOCABULARY, + renderDirectorSystemPrompt, + renderPromptBody, + resetCachedPromptRenderStyleForTests, + resolvePromptRenderStyle, + splitPromptSections, + tagForSectionTitle, +} from "./prompt-sections.js"; + +const savedEnv = process.env[PROMPT_RENDER_STYLE_ENV_VAR]; + +afterEach(() => { + // "" resolves like unset (markdown); `delete` on a computed key is + // lint-banned, so an empty assignment stands in for a missing var. + process.env[PROMPT_RENDER_STYLE_ENV_VAR] = savedEnv ?? ""; + resetCachedPromptRenderStyleForTests(); +}); + +function tagNames(xml: string): string[] { + return [...xml.matchAll(/<([a-z][a-z0-9_]*)>/g)].map((m) => m[1] ?? ""); +} + +describe("director prompt render style", () => { + test("default arm is byte-identical to formatDirectorSystemPrompt for every director", () => { + process.env[PROMPT_RENDER_STYLE_ENV_VAR] = ""; + resetCachedPromptRenderStyleForTests(); + for (const id of DIRECTOR_IDS) { + const pkg = DIRECTOR_REGISTRY[id]; + expect(renderDirectorSystemPrompt(pkg)).toBe( + formatDirectorSystemPrompt(pkg), + ); + expect(renderDirectorSystemPrompt(pkg, "markdown")).toBe( + formatDirectorSystemPrompt(pkg), + ); + expect(renderPromptBody(pkg.systemPrompt, "markdown")).toBe( + pkg.systemPrompt, + ); + } + }); + + test("xml arm keeps balanced tags with one counterpart per markdown section", () => { + for (const id of DIRECTOR_IDS) { + const pkg = DIRECTOR_REGISTRY[id]; + const full = formatDirectorSystemPrompt(pkg); + const xml = renderDirectorSystemPrompt(pkg, "xml"); + const { sections } = splitPromptSections(full); + if (sections.length === 0) { + // Section-less directors render identically in both arms. + expect(xml).toBe(full); + continue; + } + // Every markdown section has exactly one xml counterpart. + expect(tagNames(xml)).toHaveLength(sections.length); + // Balanced: every open has its close, no heading lines survive. + for (const tag of new Set(tagNames(xml))) { + const opens = xml.split(`<${tag}>`).length - 1; + const closes = xml.split(``).length - 1; + expect(opens).toBeGreaterThan(0); + expect(closes).toBe(opens); + } + expect(xml).not.toMatch(/^#{1,2} /m); + // Fixed vocabulary: every emitted tag is known or the fallback. + const known = new Set([ + ...PROMPT_SECTION_TAG_VOCABULARY, + FALLBACK_SECTION_TAG, + ]); + for (const tag of tagNames(xml)) expect(known.has(tag)).toBe(true); + } + }); + + test("fixed vocabulary covers every shipped section title", () => { + const known: ReadonlySet = new Set(PROMPT_SECTION_TAG_VOCABULARY); + for (const id of DIRECTOR_IDS) { + const { sections } = splitPromptSections( + formatDirectorSystemPrompt(DIRECTOR_REGISTRY[id]), + ); + for (const section of sections) { + expect(known.has(tagForSectionTitle(section.title))).toBe(true); + } + } + }); + + test("style resolves once per session and defaults to markdown", () => { + process.env[PROMPT_RENDER_STYLE_ENV_VAR] = ""; + resetCachedPromptRenderStyleForTests(); + expect(resolvePromptRenderStyle()).toBe("markdown"); + + process.env[PROMPT_RENDER_STYLE_ENV_VAR] = "xml"; + resetCachedPromptRenderStyleForTests(); + expect(resolvePromptRenderStyle()).toBe("xml"); + // Mid-session flips do not take effect: the cache prefix holds. + process.env[PROMPT_RENDER_STYLE_ENV_VAR] = "markdown"; + expect(resolvePromptRenderStyle()).toBe("xml"); + + process.env[PROMPT_RENDER_STYLE_ENV_VAR] = "no-such-style"; + resetCachedPromptRenderStyleForTests(); + expect(resolvePromptRenderStyle()).toBe("markdown"); + }); + + test("normalizeSectionTag handles digits, punctuation, and empties", () => { + expect(normalizeSectionTag("Prerequisites")).toBe("prerequisites"); + expect(normalizeSectionTag("0. Document discovery")).toBe( + "sec_0_document_discovery", + ); + expect( + normalizeSectionTag( + "If IMPLEMENTATION → DIY when tiny; spawn when substantial", + ), + ).toBe("if_implementation_diy_when_tiny_spawn_when_substantial"); + expect(normalizeSectionTag("")).toBe(FALLBACK_SECTION_TAG); + expect(tagForSectionTitle("a title no package uses")).toBe( + FALLBACK_SECTION_TAG, + ); + }); +}); diff --git a/src/agent/directors/prompt-sections.ts b/src/agent/directors/prompt-sections.ts new file mode 100644 index 000000000..2ce4273ca --- /dev/null +++ b/src/agent/directors/prompt-sections.ts @@ -0,0 +1,242 @@ +// Section registry for director prompt assembly (CL-8685 render-style experiment). +// +// Every director system prompt is a preamble plus `#`/`##` sections. The +// markdown arm renders sections as `## Title` (byte-identical to the shipped +// prompts — the markdown path returns `formatDirectorSystemPrompt` untouched); +// the xml arm renders those same sections as `body` with a tag +// from the fixed vocabulary below. Tags wrap whole sections only: inline +// prose, deeper sub-headings (`###` and below pass through verbatim), and the +// identity header (no `#` headings, so it stays preamble) are never tagged. +// +// Switch: `CORBITS_PROMPT_RENDER_STYLE=markdown|xml` (default `markdown`). +// The style resolves once per process and is cached, so the prompt-cache +// prefix cannot split mid-session. Tests reset the cache via +// `resetCachedPromptRenderStyleForTests`. + +import type { DirectorPackage } from "./types.js"; +import { formatDirectorSystemPrompt } from "./identity.js"; + +export type PromptRenderStyle = "markdown" | "xml"; + +/** Environment variable selecting the director-prompt render style. */ +export const PROMPT_RENDER_STYLE_ENV_VAR = "CORBITS_PROMPT_RENDER_STYLE"; + +// Fixed tag vocabulary: one tag per `#`/`##` section title shipped in +// `src/agent/directors/*/package.ts`, normalized as `normalizeSectionTag` +// does. Titles outside this set (future packages) fall back to `section`. +export const PROMPT_SECTION_TAG_VOCABULARY = [ + "acknowledgment", + "actually_overall_assessment", + "anti_cascade_stall_dig_diagnose", + "architecture_recommendations_all_terrible", + "architecture_review_criteria", + "blockers", + "build_a_shared_glossary", + "build_gate", + "capabilities", + "cargo_cult_programming_patterns_you_re_missing", + "case_template", + "common_neckbeard_phrases", + "corbits_report_shape", + "critical_reminder", + "cross_document_analysis", + "design_in_report_workflow", + "document_discovery", + "document_types", + "documents_not_found", + "effort_scaling_implementation_orchestration", + "error_handling", + "execution_steps", + "fetch_urls_primary_mounted", + "final_recommendation", + "findings", + "findings_route_to_follow_ups_never_silent_retunes", + "guidelines", + "harness_consume_do_not_build_a_second_one", + "hold_the_line_on_scope", + "how_you_work", + "if_communication_answer_directly", + "if_implementation_diy_when_tiny_spawn_when_substantial", + "if_orchestration_coordinate", + "implement_and_test", + "implementation_review_criteria", + "incomplete_document_set", + "insufferable_details_x_found", + "insufferable_mode_default", + "key_suggestions_all_terrible", + "maddening_nitpicks_x_found", + "malformed_documents", + "neckbeard_perspective", + "neckbeard_review_project_name", + "neckbeard_review_project_name_maximum_pedantry_edition", + "non_negotiables", + "operator_updates_mandatory_while_fleet_is_live", + "out_of_lane", + "parent_tools", + "paths", + "peak_neckbeard_issues_x_found", + "plan", + "prerequisites", + "product_review_criteria", + "protocol_in_order_no_shortcuts", + "push_back_on_the_vision_itself_gently", + "recommendation", + "remember", + "report", + "report_contract", + "report_shape", + "report_when_dispatched_as_a_worker", + "reporting_back", + "request_shape_implementation_orchestration_communication", + "review_framework", + "review_modes", + "riff_first", + "risk_prioritization", + "rules", + "sec_0_document_discovery", + "sec_1_analyze_and_classify_input", + "sec_2_route_and_deepen", + "sec_3_update_document", + "sec_4_cross_document_consistency_significant_only", + "sec_5_gap_detection_significant_only", + "sec_6_report", + "session_initialization", + "spawn_graph", + "spawn_handoff", + "stay_in_lane", + "step_1_load_prerequisites", + "step_2_discover_documents_and_code_when_asked", + "step_3_determine_review_mode", + "step_4_read_all_targets", + "step_5_perform_analysis", + "step_6_synthesize_nitpicks", + "step_7_report_nitpicks", + "summary", + "tools", + "unbearable_issues_x_found", + "utterly_unbearable_mode", + "verify_after_ship", + "voice", + "what_to_measure", + "what_you_do_not_do", + "who_you_are", + "workflow_scribe_core", + "write_the_brief_when_it_is_ready", + "your_role", +] as const; + +const SECTION_TAG_SET: ReadonlySet = new Set( + PROMPT_SECTION_TAG_VOCABULARY, +); + +/** Fallback tag for section titles outside the fixed vocabulary. */ +export const FALLBACK_SECTION_TAG = "section"; + +export interface PromptSection { + readonly title: string; + readonly level: 1 | 2; + readonly body: string; +} + +export interface SplitPrompt { + readonly preamble: string; + readonly sections: readonly PromptSection[]; +} + +// Section boundary: an ATX `#` or `##` heading line. `###` and deeper are +// sub-section detail and never split — tags stay at section granularity. +const SECTION_HEADING_PATTERN = /^#{1,2} ([^\n]*)(?:\n|$)/gm; + +/** Split prompt text into its preamble and `#`/`##` sections (offsets preserved). */ +export function splitPromptSections(text: string): SplitPrompt { + const matches = [...text.matchAll(SECTION_HEADING_PATTERN)]; + if (matches.length === 0) return { preamble: text, sections: [] }; + const first = matches[0]; + if (first === undefined || first.index === undefined) + return { preamble: text, sections: [] }; + const preamble = text.slice(0, first.index); + const sections = matches.map((match, i) => { + const level = match[0].startsWith("## ") ? 2 : 1; + const next = matches[i + 1]; + const bodyStart = (match.index ?? 0) + match[0].length; + const bodyEnd = next?.index ?? text.length; + return { + title: (match[1] ?? "").trim(), + level: level as 1 | 2, + body: text.slice(bodyStart, bodyEnd), + }; + }); + return { preamble, sections }; +} + +// Normalize a section title to its tag: lowercase, runs of non-alphanumerics +// to `_`, trimmed. Leading-digit results take a `sec_` prefix (bare digits +// are not valid XML tag starts); empty results take the fallback tag. +export function normalizeSectionTag(title: string): string { + const base = title + .toLowerCase() + .replace(/[^a-z0-9]+/g, "_") + .replace(/^_+|_+$/g, ""); + if (base === "") return FALLBACK_SECTION_TAG; + return /^[0-9]/.test(base) ? `sec_${base}` : base; +} + +/** Map a section title to its render tag: vocabulary hit, else the fallback. */ +export function tagForSectionTitle(title: string): string { + const tag = normalizeSectionTag(title); + return SECTION_TAG_SET.has(tag) ? tag : FALLBACK_SECTION_TAG; +} + +/** + * Render prompt text in a style. Markdown returns the input untouched + * (byte-identical by construction); xml wraps each section body in its tag. + */ +export function renderPromptBody( + text: string, + style: PromptRenderStyle, +): string { + if (style === "markdown") return text; + const { preamble, sections } = splitPromptSections(text); + if (sections.length === 0) return text; + const parts = preamble.trim() === "" ? [] : [preamble.trim()]; + for (const section of sections) { + const tag = tagForSectionTitle(section.title); + parts.push(`<${tag}>\n${section.body.trim()}\n`); + } + return parts.join("\n\n"); +} + +let cachedStyle: PromptRenderStyle | undefined; + +/** + * Resolve the render style once per process and cache it: the director prompt + * prefix must not change mid-session or the prompt cache splits. Unknown and + * absent values fall back to markdown. + */ +export function resolvePromptRenderStyle( + env: Record = process.env, +): PromptRenderStyle { + if (cachedStyle === undefined) { + const raw = env[PROMPT_RENDER_STYLE_ENV_VAR]?.trim().toLowerCase(); + cachedStyle = raw === "xml" ? "xml" : "markdown"; + } + return cachedStyle; +} + +/** Test-only reset for the once-per-session style cache. */ +export function resetCachedPromptRenderStyleForTests(): void { + cachedStyle = undefined; +} + +/** + * Render a director package system prompt (identity header + body) in a + * style. The default arm resolves the once-per-session style; markdown is + * `formatDirectorSystemPrompt` verbatim. + */ +export function renderDirectorSystemPrompt( + pkg: DirectorPackage, + style: PromptRenderStyle = resolvePromptRenderStyle(), +): string { + const full = formatDirectorSystemPrompt(pkg); + return style === "xml" ? renderPromptBody(full, "xml") : full; +} diff --git a/src/agent/directors/registry.ts b/src/agent/directors/registry.ts index b8bb11288..e9827893b 100644 --- a/src/agent/directors/registry.ts +++ b/src/agent/directors/registry.ts @@ -19,7 +19,7 @@ import { testsmithPackage } from "./testsmith/package.js"; import { gauntletPackage } from "./gauntlet/package.js"; import { proberPackage } from "./prober/package.js"; import { wardenPackage } from "./warden/package.js"; -import { formatDirectorSystemPrompt } from "./identity.js"; +import { renderDirectorSystemPrompt } from "./prompt-sections.js"; import { DIRECTOR_IDS, type DirectorId, @@ -144,7 +144,7 @@ export function packageToProfile(pkg: DirectorPackage): AgentProfile { return { id: pkg.id, description: `${pkg.description} (agent id: ${pkg.id})`, - systemPromptRole: formatDirectorSystemPrompt(pkg), + systemPromptRole: renderDirectorSystemPrompt(pkg), // Nested spawn is still gated by allowOrchestrator on the parent fleet tools. // Skywalker maySpawn marks intent; leaves stay non-orchestrator. orchestrator: pkg.spawn.maySpawn, diff --git a/src/exec/runner.ts b/src/exec/runner.ts index 1a3dc85e9..fe904673a 100644 --- a/src/exec/runner.ts +++ b/src/exec/runner.ts @@ -18,7 +18,7 @@ import { peekSourceCredentialSecret, registerSourceCredential, } from "../config/source-credentials.js"; -import { formatDirectorSystemPrompt } from "../agent/directors/identity.js"; +import { renderDirectorSystemPrompt } from "../agent/directors/prompt-sections.js"; import { DIRECTOR_REGISTRY } from "../agent/directors/registry.js"; import type { DirectorId, DirectorPackage } from "../agent/directors/types.js"; import { submitOutputDefinition } from "../agent/director.js"; @@ -284,7 +284,7 @@ export function resolveExecDirectorOverlayForPackage( ) : undefined; return { - systemPrompt: formatDirectorSystemPrompt(pkg), + systemPrompt: renderDirectorSystemPrompt(pkg), ...(advertisedAllow !== undefined ? { advertisedAllow } : {}), mountFleet: pkg.spawn.maySpawn, }; diff --git a/src/subagent/agent-fleet.ts b/src/subagent/agent-fleet.ts index 640b3bd58..b14b6012e 100644 --- a/src/subagent/agent-fleet.ts +++ b/src/subagent/agent-fleet.ts @@ -54,10 +54,8 @@ import { packageToCapabilities, resolveDirector, } from "../agent/directors/registry.js"; -import { - defaultEffortForDirector, - formatDirectorSystemPrompt, -} from "../agent/directors/identity.js"; +import { defaultEffortForDirector } from "../agent/directors/identity.js"; +import { renderDirectorSystemPrompt } from "../agent/directors/prompt-sections.js"; import type { Settings } from "../config/settings.js"; import { resolveInferenceWithPolicy } from "../config/settings.js"; import { @@ -749,7 +747,7 @@ function resolveAgentDispatch(input: { return { directorId: pkg.id, agentLabel: pkg.id, - systemPromptRole: formatDirectorSystemPrompt(pkg), + systemPromptRole: renderDirectorSystemPrompt(pkg), ...(capabilities !== undefined ? { capabilities } : {}), roleDefault: defaultEffortForDirector(pkg), pkg, @@ -824,7 +822,7 @@ function resolveAgentDispatch(input: { return { directorId: pkg.id, agentLabel: pkg.id, - systemPromptRole: formatDirectorSystemPrompt(pkg), + systemPromptRole: renderDirectorSystemPrompt(pkg), ...(capabilities !== undefined ? { capabilities } : {}), roleDefault: defaultEffortForDirector(pkg), pkg,