diff --git a/.changeset/run-results-skill-reference.md b/.changeset/run-results-skill-reference.md new file mode 100644 index 000000000..8732a9ad8 --- /dev/null +++ b/.changeset/run-results-skill-reference.md @@ -0,0 +1,5 @@ +--- +"@qawolf/cli": minor +--- + +The `qawolf-cli` skill has a new reference file, `references/run-results.md`, for reading what `qawolf run get` returns. It explains the fields that a passing run does not show, such as a flow's failure diagnosis; the rules for the artifact URLs, which expire and can give a 404; and how to read the Playwright trace that `traceUrl` downloads. A trace is newline-delimited JSON, so an agent in a shell can pair each call with its result and find the failure without the trace viewer. The field list is generated from the contract, so it cannot drift from the installed version. Command help is unchanged. diff --git a/scripts/genSkillMd.ts b/scripts/genSkillMd.ts index bd3a6cf78..2f4e07c9d 100644 --- a/scripts/genSkillMd.ts +++ b/scripts/genSkillMd.ts @@ -4,8 +4,14 @@ import { readFileSync, writeFileSync } from "node:fs"; import { join } from "node:path"; +import { publicContractsV1 } from "@qawolf/api-contracts/v1"; + import { createProgram } from "~/commands/program.js"; import { renderCommandsTable, spliceCommandsTable } from "~/commands/skill.js"; +import { + renderResponseFields, + spliceResponseFields, +} from "~/commands/skillRunResults.js"; import { makeNoopSignals } from "~/shell/signals/createSignalRegistry.fixtures.js"; const skillMdPath = join(import.meta.dirname, "../skills/qawolf-cli/SKILL.md"); @@ -13,6 +19,14 @@ const skillTemplatePath = join( import.meta.dirname, "../src/commands/qawolfCliSkill.template.md", ); +const runResultsMdPath = join( + import.meta.dirname, + "../skills/qawolf-cli/references/run-results.md", +); +const runResultsTemplatePath = join( + import.meta.dirname, + "../src/commands/qawolfCliRunResults.template.md", +); const skillTemplate = readFileSync(skillTemplatePath, "utf8"); const table = renderCommandsTable( @@ -24,3 +38,12 @@ if (skillMd !== currentSkillMd) { writeFileSync(skillMdPath, skillMd); console.log("Updated skills/qawolf-cli/SKILL.md"); } + +const runResultsMd = spliceResponseFields( + readFileSync(runResultsTemplatePath, "utf8"), + renderResponseFields(publicContractsV1.run.get), +); +if (runResultsMd !== readFileSync(runResultsMdPath, "utf8")) { + writeFileSync(runResultsMdPath, runResultsMd); + console.log("Updated skills/qawolf-cli/references/run-results.md"); +} diff --git a/skills/qawolf-cli/SKILL.md b/skills/qawolf-cli/SKILL.md index 807c9c30d..68a3b4b63 100644 --- a/skills/qawolf-cli/SKILL.md +++ b/skills/qawolf-cli/SKILL.md @@ -72,6 +72,15 @@ each printed line from the payload alone to the whole envelope (`sequence`, `recordedAt`, `payload`). Both are JSON. Pass it when you want to page by sequence, omit it when you want the payloads themselves. +A `--json` response shows you most of its own shape, so read it first. + +`qawolf run get` is the exception worth reading about before you use it. Its +artifact URLs expire, its failure fields are absent from a passing run, and its +`traceUrl` downloads a Playwright trace that you can read as JSON without +opening the trace viewer. **Read +[`references/run-results.md`](references/run-results.md) before reporting on a +run's outcome or opening its trace.** + ## Safety: reads vs writes Read commands do not change team data, but some have operational effects noted diff --git a/skills/qawolf-cli/references/run-results.md b/skills/qawolf-cli/references/run-results.md new file mode 100644 index 000000000..da9b37e2a --- /dev/null +++ b/skills/qawolf-cli/references/run-results.md @@ -0,0 +1,169 @@ +# Reading a run's results + +How to use what `qawolf run get --run-id --json` returns, and how to read +the Playwright trace it links to. + +Call `run get` with `--json` and you see most of the response immediately. Read +this file for the parts a single response cannot show you: fields that appear +only when something fails, rules about the artifact URLs, and how to read a +trace without opening the trace viewer. + +## The shape + +A run holds flows, a flow holds attempts, and artifacts hang off an attempt: + +```text +run +└── flows[] + ├── failure only when the flow failed + └── attempts[] oldest first + ├── logsUrl + ├── traceUrl + └── videoUrl +``` + +`runId` in the response is canonical and can differ from the id you asked for. +Use the returned value for follow-up calls. + +Poll `status` until it reaches `passed`, `failed` or `canceled`. The other +values mean the run is still going. + +## Fields a passing run does not show you + +- `flows[].failure` exists only when a flow failed. Every flow passing means + there is no failure object at all, so its diagnosis and issue id are invisible + until something breaks. Do not conclude the field does not exist. +- `git` is populated only when a deploy notification started the run. A run + started manually or with `run create` has an empty object here. +- An attempt's `kind` and `status` select which other fields it has. Only + automated attempts that reached a verdict carry artifact URLs; canceled + attempts and manual Wolf Browser attempts carry none. +- A flow that passed after a retry still lists its failed attempts. Read the + last attempt for the outcome, and the earlier ones to see what went wrong. + +## Artifact URLs + +Each automated attempt links `logsUrl` (execution logs), `videoUrl` (screen +recording) and `traceUrl` (a Playwright `trace.zip`). + +1. They are signed URLs with a limited life. The contract guarantees at least a + day. Call `run get` again for fresh ones instead of storing them; a stored + URL becomes a dead link. +2. A URL can return 404 when that attempt did not produce that artifact. Handle + the 404 rather than treating the URL's presence as a guarantee of content. +3. Download with a plain HTTP GET. The signature is in the URL, so no + authentication header is needed and no QA Wolf credentials are involved. + +```bash +qawolf run get --run-id "$RUN_ID" --json \ + | jq -r '.flows[].attempts[-1].traceUrl // empty' \ + | head -1 \ + | xargs -r curl -sS -o trace.zip +``` + +## Reading the Playwright trace + +The usual advice is `npx playwright show-trace trace.zip`, which opens a +browser window. That is useless in a shell and unnecessary: the zip holds +newline-delimited JSON files, and reading them directly is faster than +downloading a viewer. + +The zip holds `trace.trace` (the events), `trace.network` (one request and +response per line) and a `resources/` directory of screencast frames. The +frames are most of the size, so extract only what you need. + +### The event types + +Every line of `trace.trace` is one JSON object with a `type`: + +- `before` — a call started. Carries `callId`, `startTime`, `class`, `method` + and `params`. `params.selector` or `params.url` is usually the target. +- `after` — that call finished. Matched to its `before` by `callId`. Carries + `endTime` and `result`, and an `error` when the call failed. +- `console` — a browser console message, with `messageType` and `text`. +- `log` — Playwright's own progress notes for a call. +- `screencast-frame`, `frame-snapshot` — the filmstrip and DOM snapshots the + viewer renders. Usually not worth reading directly. + +Two details cost time if you miss them: + +- **Times are monotonic milliseconds, not seconds.** A `goto` whose `startTime` + and `endTime` differ by `161.6` took 161 milliseconds. Subtract the smallest + `startTime` to get an offset from the start of the trace. +- **Return values use a serialized envelope.** `{"value":{"s":"passed"}}` is + the string `passed`, `{"n":640}` is the number `640`, and `{"o":[...]}` is an + object as a list of key and value pairs. + +### A worked example + +Pairing `before` with `after` gives an action timeline with durations and +failures: + +```python +import json, sys, zipfile + +with zipfile.ZipFile(sys.argv[1]) as z: + events = [json.loads(line) for line in z.read("trace.trace").decode().splitlines()] + +starts = {e["callId"]: e for e in events if e["type"] == "before"} +ends = {e["callId"]: e for e in events if e["type"] == "after"} +t0 = min(e["startTime"] for e in starts.values()) + +for call_id, before in starts.items(): + after = ends.get(call_id, {}) + params = before.get("params", {}) + target = params.get("selector") or params.get("url") or "" + error = after.get("error") + print( + f'{(before["startTime"] - t0) / 1000:7.2f}s' + f' {after.get("endTime", before["startTime"]) - before["startTime"]:7.1f}ms' + f' {before["class"]}.{before["method"]:<18} {target[:40]}' + f'{" FAILED: " + json.dumps(error)[:60] if error else ""}' + ) + +for e in events: + if e["type"] == "console" and e["messageType"] == "error": + print(f'console error: {e["text"][:70]}') +``` + +It prints one line per call, in order: + +```text + 0.00s 161.6ms Frame.goto https://example.com/ + 0.17s 28.3ms Frame.waitForSelector #screen + 0.20s 4.2ms Frame.innerText #fps_stats +console error: Failed to load resource: the server responded with a status of 404 () +``` + +To find why an attempt failed, read the last `after` that carries an `error`, +then the `console` errors near it in time. To see what the page did, read +`trace.network`. + +## Response fields + +Every documented field of the `run.get` response. `[]` marks an array, so +`flows[].attempts[].traceUrl` is the trace URL of one attempt of one flow. + + + +- `completedAt` — When the run finished executing. Absent while queued or running, and also absent for a terminal run that never completed execution (e.g. every flow was canceled or skipped). +- `git` — The branch and commit under test. The fields are present when a deploy notification started the run, and absent for runs started another way, for example manually or with run.create. +- `git.commitUrl` — Link to the commit on the code host. +- `runId` — The run this response describes. Treat it as canonical: it can differ from the id you asked for. A deploy notification returns a run id before the run exists, and if a second notification for the same commit is folded into an earlier run, that id resolves to the earlier run instead. +- `status` — One of: queued, running, passed, failed, canceled +- `flows` — The run's flows, ordered alphabetically by name. +- `flows[].attempts` — The flow's finished execution attempts, oldest first, including manual Wolf Browser attempts. Present once at least one attempt has finished, so a flow that passed after retries also lists its failed attempts. Artifact URLs appear only on automated attempts that reached a verdict, stay valid for at least a day (call run.get again for fresh ones), and can return 404 when the attempt did not produce that artifact. +- `flows[].attempts[].logsUrl` — Signed URL for the attempt's execution logs. +- `flows[].attempts[].traceUrl` — Signed URL for the attempt's Playwright trace (a trace.zip; open it with `npx playwright show-trace`). +- `flows[].attempts[].videoUrl` — Signed URL for the attempt's screen recording. +- `flows[].attempts[].kind` — One of: automated, manual +- `flows[].attempts[].startedAt` — Absent when the attempt failed before it could start. +- `flows[].attempts[].status` — One of: passed, failed, canceled +- `flows[].failure.diagnosis` — QA Wolf's investigation verdict for the failure: `bug` means the application is broken, `maintenance` means the test needed an update and the failure does not indicate an application problem. Absent until the investigation reaches a verdict. Pass issueId to issue.get for details. +- `flows[].failure.diagnosis.issueId` — The id of the issue. +- `flows[].failure.diagnosis.type` — One of: bug, maintenance +- `flows[].flowId` — The id of the flow. +- `flows[].status` — One of: failed, queued, running, passed, canceled +- `url` — Absolute URL of the run page. + + diff --git a/src/commands/qawolfCliRunResults.template.md b/src/commands/qawolfCliRunResults.template.md new file mode 100644 index 000000000..8b8e5afdf --- /dev/null +++ b/src/commands/qawolfCliRunResults.template.md @@ -0,0 +1,148 @@ +# Reading a run's results + +How to use what `qawolf run get --run-id --json` returns, and how to read +the Playwright trace it links to. + +Call `run get` with `--json` and you see most of the response immediately. Read +this file for the parts a single response cannot show you: fields that appear +only when something fails, rules about the artifact URLs, and how to read a +trace without opening the trace viewer. + +## The shape + +A run holds flows, a flow holds attempts, and artifacts hang off an attempt: + +```text +run +└── flows[] + ├── failure only when the flow failed + └── attempts[] oldest first + ├── logsUrl + ├── traceUrl + └── videoUrl +``` + +`runId` in the response is canonical and can differ from the id you asked for. +Use the returned value for follow-up calls. + +Poll `status` until it reaches `passed`, `failed` or `canceled`. The other +values mean the run is still going. + +## Fields a passing run does not show you + +- `flows[].failure` exists only when a flow failed. Every flow passing means + there is no failure object at all, so its diagnosis and issue id are invisible + until something breaks. Do not conclude the field does not exist. +- `git` is populated only when a deploy notification started the run. A run + started manually or with `run create` has an empty object here. +- An attempt's `kind` and `status` select which other fields it has. Only + automated attempts that reached a verdict carry artifact URLs; canceled + attempts and manual Wolf Browser attempts carry none. +- A flow that passed after a retry still lists its failed attempts. Read the + last attempt for the outcome, and the earlier ones to see what went wrong. + +## Artifact URLs + +Each automated attempt links `logsUrl` (execution logs), `videoUrl` (screen +recording) and `traceUrl` (a Playwright `trace.zip`). + +1. They are signed URLs with a limited life. The contract guarantees at least a + day. Call `run get` again for fresh ones instead of storing them; a stored + URL becomes a dead link. +2. A URL can return 404 when that attempt did not produce that artifact. Handle + the 404 rather than treating the URL's presence as a guarantee of content. +3. Download with a plain HTTP GET. The signature is in the URL, so no + authentication header is needed and no QA Wolf credentials are involved. + +```bash +qawolf run get --run-id "$RUN_ID" --json \ + | jq -r '.flows[].attempts[-1].traceUrl // empty' \ + | head -1 \ + | xargs -r curl -sS -o trace.zip +``` + +## Reading the Playwright trace + +The usual advice is `npx playwright show-trace trace.zip`, which opens a +browser window. That is useless in a shell and unnecessary: the zip holds +newline-delimited JSON files, and reading them directly is faster than +downloading a viewer. + +The zip holds `trace.trace` (the events), `trace.network` (one request and +response per line) and a `resources/` directory of screencast frames. The +frames are most of the size, so extract only what you need. + +### The event types + +Every line of `trace.trace` is one JSON object with a `type`: + +- `before` — a call started. Carries `callId`, `startTime`, `class`, `method` + and `params`. `params.selector` or `params.url` is usually the target. +- `after` — that call finished. Matched to its `before` by `callId`. Carries + `endTime` and `result`, and an `error` when the call failed. +- `console` — a browser console message, with `messageType` and `text`. +- `log` — Playwright's own progress notes for a call. +- `screencast-frame`, `frame-snapshot` — the filmstrip and DOM snapshots the + viewer renders. Usually not worth reading directly. + +Two details cost time if you miss them: + +- **Times are monotonic milliseconds, not seconds.** A `goto` whose `startTime` + and `endTime` differ by `161.6` took 161 milliseconds. Subtract the smallest + `startTime` to get an offset from the start of the trace. +- **Return values use a serialized envelope.** `{"value":{"s":"passed"}}` is + the string `passed`, `{"n":640}` is the number `640`, and `{"o":[...]}` is an + object as a list of key and value pairs. + +### A worked example + +Pairing `before` with `after` gives an action timeline with durations and +failures: + +```python +import json, sys, zipfile + +with zipfile.ZipFile(sys.argv[1]) as z: + events = [json.loads(line) for line in z.read("trace.trace").decode().splitlines()] + +starts = {e["callId"]: e for e in events if e["type"] == "before"} +ends = {e["callId"]: e for e in events if e["type"] == "after"} +t0 = min(e["startTime"] for e in starts.values()) + +for call_id, before in starts.items(): + after = ends.get(call_id, {}) + params = before.get("params", {}) + target = params.get("selector") or params.get("url") or "" + error = after.get("error") + print( + f'{(before["startTime"] - t0) / 1000:7.2f}s' + f' {after.get("endTime", before["startTime"]) - before["startTime"]:7.1f}ms' + f' {before["class"]}.{before["method"]:<18} {target[:40]}' + f'{" FAILED: " + json.dumps(error)[:60] if error else ""}' + ) + +for e in events: + if e["type"] == "console" and e["messageType"] == "error": + print(f'console error: {e["text"][:70]}') +``` + +It prints one line per call, in order: + +```text + 0.00s 161.6ms Frame.goto https://example.com/ + 0.17s 28.3ms Frame.waitForSelector #screen + 0.20s 4.2ms Frame.innerText #fps_stats +console error: Failed to load resource: the server responded with a status of 404 () +``` + +To find why an attempt failed, read the last `after` that carries an `error`, +then the `console` errors near it in time. To see what the page did, read +`trace.network`. + +## Response fields + +Every documented field of the `run.get` response. `[]` marks an array, so +`flows[].attempts[].traceUrl` is the trace URL of one attempt of one flow. + + + diff --git a/src/commands/qawolfCliSkill.template.md b/src/commands/qawolfCliSkill.template.md index a5f096d29..c3c58381a 100644 --- a/src/commands/qawolfCliSkill.template.md +++ b/src/commands/qawolfCliSkill.template.md @@ -72,6 +72,15 @@ each printed line from the payload alone to the whole envelope (`sequence`, `recordedAt`, `payload`). Both are JSON. Pass it when you want to page by sequence, omit it when you want the payloads themselves. +A `--json` response shows you most of its own shape, so read it first. + +`qawolf run get` is the exception worth reading about before you use it. Its +artifact URLs expire, its failure fields are absent from a passing run, and its +`traceUrl` downloads a Playwright trace that you can read as JSON without +opening the trace viewer. **Read +[`references/run-results.md`](references/run-results.md) before reporting on a +run's outcome or opening its trace.** + ## Safety: reads vs writes Read commands do not change team data, but some have operational effects noted diff --git a/src/commands/skillRunResults.test.ts b/src/commands/skillRunResults.test.ts new file mode 100644 index 000000000..443e4c692 --- /dev/null +++ b/src/commands/skillRunResults.test.ts @@ -0,0 +1,103 @@ +import { describe, expect, it } from "bun:test"; +import { publicContractsV1 } from "@qawolf/api-contracts/v1"; +import { join } from "node:path"; +import { z } from "zod"; + +import { + renderResponseFields, + spliceResponseFields, +} from "./skillRunResults.js"; + +const runResultsMdPath = join( + import.meta.dirname, + "../../skills/qawolf-cli/references/run-results.md", +); +const runResultsTemplatePath = join( + import.meta.dirname, + "qawolfCliRunResults.template.md", +); + +describe("renderResponseFields", () => { + it("renders one bullet per documented field", () => { + const contract = { + description: "Look up a run.", + input: z.object({ runId: z.string() }), + kind: "read", + name: "run.get", + output: z.object({ + runId: z.string().describe("The id of the run."), + internal: z.string(), + }), + } as const; + + expect(renderResponseFields(contract)).toBe( + "- `runId` — The id of the run.", + ); + }); + + it("throws when the contract documents nothing, rather than emitting an empty reference", () => { + const contract = { + description: "Look up a run.", + input: z.object({ runId: z.string() }), + kind: "read", + name: "run.get", + output: z.object({ runId: z.string() }), + } as const; + + expect(() => renderResponseFields(contract)).toThrow( + 'Contract "run.get" documents no response field', + ); + }); + + it("documents the artifact urls and the nested attempt paths", () => { + const rendered = renderResponseFields(publicContractsV1.run.get); + + expect(rendered).toContain( + "- `flows[].attempts[].traceUrl` — Signed URL for the attempt's Playwright trace", + ); + expect(rendered).toContain("- `flows[].attempts[].logsUrl`"); + expect(rendered).toContain("- `flows[].attempts[].videoUrl`"); + expect(rendered).toContain("- `flows[].attempts[].status` — One of:"); + }); +}); + +describe("spliceResponseFields", () => { + it("replaces the marked region, keeping the surrounding prose", () => { + const template = [ + "# Run results", + "", + "stale", + "", + "trailing prose", + ].join("\n"); + + const spliced = spliceResponseFields(template, "- `runId` — The id."); + + expect(spliced).toContain("# Run results"); + expect(spliced).toContain("- `runId` — The id."); + expect(spliced).toContain("trailing prose"); + expect(spliced).not.toContain("stale"); + }); + + it("throws when the template has no markers", () => { + expect(() => spliceResponseFields("# Run results", "x")).toThrow( + "is missing the fields markers", + ); + }); +}); + +describe("run-results reference", () => { + // Fails the moment @qawolf/api-contracts changes run.get's documented + // fields, which is what keeps this file from drifting from the installed CLI. + it("matches its template and the published contract", async () => { + const runResultsMd = await Bun.file(runResultsMdPath).text(); + const template = await Bun.file(runResultsTemplatePath).text(); + + expect(runResultsMd).toBe( + spliceResponseFields( + template, + renderResponseFields(publicContractsV1.run.get), + ), + ); + }); +}); diff --git a/src/commands/skillRunResults.ts b/src/commands/skillRunResults.ts new file mode 100644 index 000000000..a6261c62c --- /dev/null +++ b/src/commands/skillRunResults.ts @@ -0,0 +1,37 @@ +import type { AnyPublicApiContract } from "@qawolf/api-contracts/v1"; + +import { buildOutputFieldDocs } from "~/core/publicApi/outputFields.js"; + +const fieldsStartMarker = + ""; +const fieldsEndMarker = ""; + +// Bullets rather than a table: descriptions are full sentences, so table pipes +// and alignment would cost tokens on every read without aiding a reader. +export function renderResponseFields(contract: AnyPublicApiContract): string { + const fields = buildOutputFieldDocs(contract.output); + if (fields.length === 0) { + throw new Error( + `Contract "${contract.name}" documents no response field; the reference would be empty.`, + ); + } + return fields + .map((field) => `- \`${field.path}\` — ${field.description}`) + .join("\n"); +} + +export function spliceResponseFields( + template: string, + rendered: string, +): string { + const start = template.indexOf(fieldsStartMarker); + const end = template.indexOf(fieldsEndMarker); + if (start === -1 || end === -1 || end < start) { + throw new Error( + "skills/qawolf-cli/references/run-results.md is missing the fields markers.", + ); + } + const before = template.slice(0, start + fieldsStartMarker.length); + const after = template.slice(end); + return `${before}\n\n${rendered}\n\n${after}`; +} diff --git a/src/core/publicApi/flagKind.ts b/src/core/publicApi/flagKind.ts index bfce10ca4..9e25f7a39 100644 --- a/src/core/publicApi/flagKind.ts +++ b/src/core/publicApi/flagKind.ts @@ -18,6 +18,7 @@ export type JsonSchema = { oneOf?: JsonSchema[]; anyOf?: JsonSchema[]; const?: unknown; + enum?: unknown[]; }; export function flagKind( diff --git a/src/core/publicApi/outputFields.test.ts b/src/core/publicApi/outputFields.test.ts new file mode 100644 index 000000000..4d1a53cfa --- /dev/null +++ b/src/core/publicApi/outputFields.test.ts @@ -0,0 +1,114 @@ +import { describe, expect, it } from "bun:test"; +import { z } from "zod"; + +import { buildOutputFieldDocs } from "./outputFields.js"; + +describe("buildOutputFieldDocs", () => { + it("flattens nested arrays into dotted paths marking each array", () => { + const schema = z.object({ + flows: z + .array( + z.object({ + attempts: z + .array(z.object({ traceUrl: z.string().describe("The trace.") })) + .describe("The attempts."), + }), + ) + .describe("The flows."), + }); + + expect(buildOutputFieldDocs(schema)).toEqual([ + { path: "flows", description: "The flows." }, + { path: "flows[].attempts", description: "The attempts." }, + { path: "flows[].attempts[].traceUrl", description: "The trace." }, + ]); + }); + + it("lists a field shared by union branches once", () => { + const schema = z.object({ + attempt: z.discriminatedUnion("status", [ + z.object({ + status: z.literal("passed"), + traceUrl: z.string().describe("Signed URL for the trace."), + }), + z.object({ + status: z.literal("failed"), + traceUrl: z.string().describe("Signed URL for the trace."), + }), + ]), + }); + + const paths = buildOutputFieldDocs(schema).map((field) => field.path); + expect(paths.filter((path) => path === "attempt.traceUrl")).toHaveLength(1); + }); + + // One branch's prose describes that branch, not the field. Showing it alone + // reads as the field's meaning: the canceled variant's "terminated without + // reaching a verdict" would describe every attempt's status. + it("enumerates a literal field's values rather than one branch's prose", () => { + const schema = z.object({ + attempt: z.discriminatedUnion("kind", [ + z.object({ kind: z.literal("automated").describe("Ran on a runner.") }), + z.object({ kind: z.literal("manual").describe("Ran in a browser.") }), + ]), + }); + + expect(buildOutputFieldDocs(schema)).toEqual([ + { path: "attempt.kind", description: "One of: automated, manual" }, + ]); + }); + + it("documents a field whose branches are bare literals", () => { + const schema = z.object({ + mode: z + .union([z.literal("fast"), z.literal("slow")]) + .describe("How it ran."), + bare: z.union([z.literal("a"), z.literal("b")]), + }); + + expect(buildOutputFieldDocs(schema)).toEqual([ + { path: "mode", description: "How it ran. One of: fast, slow" }, + { path: "bare", description: "One of: a, b" }, + ]); + }); + + // A flow's status is a literal in one response shape and an enum in another. + it("counts enum members among a field's values", () => { + const schema = z.object({ + status: z.union([ + z.object({ status: z.literal("failed") }), + z.object({ status: z.enum(["passed", "canceled"]) }), + ]), + }); + + expect(buildOutputFieldDocs(schema)).toEqual([ + { + path: "status.status", + description: "One of: failed, passed, canceled", + }, + ]); + }); + + it("omits fields that carry no description", () => { + const schema = z.object({ + documented: z.string().describe("Documented."), + bare: z.string(), + }); + + expect(buildOutputFieldDocs(schema)).toEqual([ + { path: "documented", description: "Documented." }, + ]); + }); + + // Every issue.* contract returns dates, which have no JSON Schema form and + // throw under zod's default policy. They must still be documented. + it("documents a field whose type JSON Schema cannot represent", () => { + const schema = z.object({ + createdAt: z.date().describe("When it was created."), + }); + + expect(buildOutputFieldDocs(schema)).toEqual([ + { path: "createdAt", description: "When it was created." }, + ]); + }); +}); diff --git a/src/core/publicApi/outputFields.ts b/src/core/publicApi/outputFields.ts new file mode 100644 index 000000000..3d6d754b6 --- /dev/null +++ b/src/core/publicApi/outputFields.ts @@ -0,0 +1,121 @@ +import { z } from "zod"; + +import type { JsonSchema } from "./flagKind.js"; + +export type OutputFieldDoc = { + // Dotted path into the response, with [] marking an array, + // e.g. "flows[].attempts[].traceUrl". + path: string; + description: string; +}; + +// Deep enough for the nested response shapes contracts actually use, and a +// backstop against a self-referential schema walking forever. +const maxDepth = 12; + +type FieldEntry = { + description: string | undefined; + // Literal values this field takes, one per union branch it appears in. + constValues: string[]; +}; + +const branchesOf = (schema: JsonSchema): JsonSchema[] | undefined => + schema.allOf ?? schema.oneOf ?? schema.anyOf; + +const literalValues = (schema: JsonSchema): string[] => [ + ...(typeof schema.const === "string" ? [schema.const] : []), + ...(schema.enum ?? []).filter((value) => typeof value === "string"), +]; + +function record( + found: Map, + path: string, + schema: JsonSchema, +): void { + const entry = found.get(path) ?? { description: undefined, constValues: [] }; + const values = literalValues(schema); + // Only a schema that pins no literal describes the field itself. A branch + // that pins one describes that branch, and reading its prose as the field's + // meaning is what made an attempt's status read as "terminated without + // reaching a verdict". + if ( + values.length === 0 && + entry.description === undefined && + schema.description !== undefined + ) { + entry.description = schema.description; + } + for (const value of values) { + if (!entry.constValues.includes(value)) entry.constValues.push(value); + } + found.set(path, entry); +} + +function collect( + schema: JsonSchema, + path: string, + depth: number, + found: Map, +): void { + if (depth > maxDepth) return; + + // A union describes one response shape per branch. Walking every branch and + // keeping the first description per path lists a field that appears in + // several branches once, rather than once per branch: an artifact URL is + // documented on both the passed and the failed attempt variant. + const branches = branchesOf(schema); + if (branches) { + for (const branch of branches) { + // A branch that is a bare literal has no properties to walk, so its value + // is only seen here. Without this the whole field goes undocumented. + record(found, path, branch); + collect(branch, path, depth + 1, found); + } + return; + } + + if (schema.type === "array") { + if (schema.items) collect(schema.items, `${path}[]`, depth + 1, found); + return; + } + + if (!schema.properties) return; + for (const [field, fieldSchema] of Object.entries(schema.properties)) { + const fieldPath = path === "" ? field : `${path}.${field}`; + record(found, fieldPath, fieldSchema); + collect(fieldSchema, fieldPath, depth + 1, found); + } +} + +// Flattens a contract's output schema into the described fields worth showing +// in --help. Fields without a description are omitted: the path alone tells a +// reader nothing the JSON would not. +// +// Conversion failures are left to throw. This runs at generation time, so a +// schema it cannot read should fail the build loudly rather than quietly +// produce a reference with fields missing from it. +export function buildOutputFieldDocs( + outputSchema: z.ZodType, +): OutputFieldDoc[] { + const jsonSchema = z.toJSONSchema(outputSchema, { + io: "output", + // z.date() and z.transform() have no JSON Schema form and throw under the + // default "throw" policy. Degrading keeps their descriptions. + unrepresentable: "any", + }) as JsonSchema; + + const found = new Map(); + collect(jsonSchema, "", 0, found); + return [...found].flatMap(([path, entry]) => { + // A field the branches pin to different literals is documented by the set + // of values it takes, after its own description when it has one. + const values = + entry.constValues.length > 1 + ? `One of: ${entry.constValues.join(", ")}` + : undefined; + const description = [entry.description, values] + .filter((part) => part !== undefined) + .join(" "); + return description === "" ? [] : [{ path, description }]; + }); +}