diff --git a/.github/scripts/cli-bench/github.ts b/.github/scripts/cli-bench/github.ts new file mode 100644 index 0000000000..4859009a68 --- /dev/null +++ b/.github/scripts/cli-bench/github.ts @@ -0,0 +1,69 @@ +/** Small GitHub REST helpers shared by the CLI benchmark scripts. */ + +export function requireEnv(name: string): string { + const value = process.env[name]; + if (!value) throw new Error(`Missing required environment variable: ${name}`); + return value; +} + +/** A GitHub request that did not succeed, with the HTTP status and GitHub's own message. */ +export class GithubRequestError extends Error { + constructor( + readonly status: number, + readonly detail: string, + url: string, + ) { + super(`GitHub request failed (${status}) for ${url}`); + this.name = "GithubRequestError"; + } +} + +export async function githubFetch( + url: string, + token: string, + init: RequestInit = {}, +): Promise { + const response = await fetch(url, { + ...init, + headers: { + Authorization: `Bearer ${token}`, + Accept: "application/vnd.github+json", + "X-GitHub-Api-Version": "2022-11-28", + "Content-Type": "application/json", + }, + }); + if (!response.ok) { + const payload = (await response.json().catch(() => ({}))) as { message?: unknown }; + const detail = typeof payload.message === "string" ? payload.message : response.statusText; + throw new GithubRequestError(response.status, detail, url); + } + return response; +} + +export interface IssueComment { + id: number; + body: string; + authorLogin: string; +} + +/** Every comment on an issue or pull request, oldest first. */ +export async function listIssueComments( + token: string, + base: string, + issue: number, +): Promise> { + const comments: Array = []; + for (let page = 1; ; page++) { + const batch = (await ( + await githubFetch(`${base}/issues/${issue}/comments?per_page=100&page=${page}`, token) + ).json()) as Array<{ id: number; body: string | null; user: { login: string } | null }>; + comments.push( + ...batch.map((entry) => ({ + id: entry.id, + body: entry.body ?? "", + authorLogin: entry.user?.login ?? "", + })), + ); + if (batch.length < 100) return comments; + } +} diff --git a/.github/scripts/cli-bench/history.test.ts b/.github/scripts/cli-bench/history.test.ts new file mode 100644 index 0000000000..0d096bce41 --- /dev/null +++ b/.github/scripts/cli-bench/history.test.ts @@ -0,0 +1,114 @@ +import { describe, expect, test } from "bun:test"; +import { + decodeHistory, + headlineOf, + makeEntry, + MAX_PREVIOUS_RUNS, + mergeHistory, + renderHistory, + type RunEntry, +} from "./history.ts"; + +const entry = (n: number, overrides: Partial = {}): RunEntry => ({ + sha: n.toString(16).padStart(7, "0"), + at: "2026-10-28T09:05Z", + headline: "no changes found", + url: `https://github.com/o/r/actions/runs/${n}`, + ...overrides, +}); + +describe("headlineOf", () => { + test("reads the title of the rendered report", () => { + expect(headlineOf("\n\n## CLI benchmark: 2 things to check\n\nbody")).toBe( + "2 things to check", + ); + }); + + test("falls back when the report has no title", () => { + expect(headlineOf("nothing here")).toBe("unknown"); + }); +}); + +describe("makeEntry", () => { + test("shortens the commit and rounds the time down to the minute in UTC", () => { + const made = makeEntry({ + sha: "0123456789abcdef0123456789abcdef01234567", + now: new Date("2026-10-28T09:05:59Z"), + headline: "no changes found", + url: "https://github.com/o/r/actions/runs/9/attempts/2", + }); + + expect(made).toMatchObject({ sha: "0123456", at: "2026-10-28T09:05Z" }); + }); + + test("refuses a run link that is not a workflow run", () => { + const made = makeEntry({ + sha: "abcdef1", + now: new Date(), + headline: "h", + url: "https://evil.test/x", + }); + + expect(made).toBeUndefined(); + }); +}); + +describe("mergeHistory", () => { + test("puts the new run first", () => { + expect(mergeHistory([entry(1)], entry(2)).map((e) => e.sha)).toEqual([ + entry(2).sha, + entry(1).sha, + ]); + }); + + test("keeps the current run plus at most 20 earlier ones, dropping the oldest", () => { + const previous = Array.from({ length: 30 }, (_, i) => entry(i + 1)); + + const merged = mergeHistory(previous, entry(100)); + + expect(merged).toHaveLength(MAX_PREVIOUS_RUNS + 1); + expect(merged[0]).toEqual(entry(100)); + expect(merged.at(-1)).toEqual(entry(MAX_PREVIOUS_RUNS)); + }); +}); + +describe("renderHistory and decodeHistory", () => { + test("round-trips the entries through the hidden marker", () => { + const entries = [entry(3), entry(2), entry(1)]; + + expect(decodeHistory(`body\n${renderHistory(entries, (t) => t)}\n`)).toEqual(entries); + }); + + test("lists earlier runs newest first and leaves the current run out of the list", () => { + const rendered = renderHistory( + [entry(3), entry(2, { headline: "1 thing to check" })], + (t) => t, + ); + + expect(rendered).toContain("Previous runs"); + expect(rendered).toContain( + "- `0000002` · 2026-10-28 09:05 UTC · 1 thing to check · [run](https://github.com/o/r/actions/runs/2)", + ); + expect(rendered).not.toContain("0000003` ·"); + }); + + test("shows no list for a first run", () => { + expect(renderHistory([entry(1)], (t) => t)).not.toContain("Previous runs"); + }); + + test.each([ + ["no marker", "plain comment"], + ["not base64 json", ""], + ["wrong shape", ``], + [ + "a forged run link", + ``, + ], + [ + "too many entries", + ``, + ], + ])("treats %s as empty history", (_name, body) => { + expect(decodeHistory(body)).toEqual([]); + }); +}); diff --git a/.github/scripts/cli-bench/history.ts b/.github/scripts/cli-bench/history.ts new file mode 100644 index 0000000000..9602383119 --- /dev/null +++ b/.github/scripts/cli-bench/history.ts @@ -0,0 +1,106 @@ +/** + * Run history kept inside the benchmark comment. The newest run is the comment itself; earlier + * runs sit in a collapsed list, backed by a hidden marker so the next update can carry them on. + * The marker is read back only from the workflow bot's own comment, and anything that does not + * decode cleanly counts as no history. + */ + +/** Earlier runs kept below the current one. */ +export const MAX_PREVIOUS_RUNS = 20; +const HISTORY_MARKER = //u; +const MAX_ENCODED_CHARS = 30_000; +const MAX_HEADLINE_CHARS = 100; +const SHA = /^[0-9a-f]{7,40}$/u; +const AT = /^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}Z$/u; +const RUN_URL = /^https:\/\/[^\s)<>]+\/actions\/runs\/\d+(?:\/attempts\/\d+)?$/u; + +export interface RunEntry { + /** Short commit SHA. */ + sha: string; + /** UTC time to the minute, `YYYY-MM-DDTHH:MMZ`. */ + at: string; + headline: string; + url: string; +} + +const isEntry = (value: unknown): value is RunEntry => { + if (typeof value !== "object" || value === null) return false; + const { sha, at, headline, url } = value as Record; + return ( + typeof sha === "string" && + SHA.test(sha) && + typeof at === "string" && + AT.test(at) && + typeof headline === "string" && + headline.length <= MAX_HEADLINE_CHARS && + !/[\r\n]/u.test(headline) && + typeof url === "string" && + RUN_URL.test(url) + ); +}; + +/** The title the renderer puts after `## CLI benchmark:`, flattened to one bounded line. */ +export function headlineOf(markdown: string): string { + const title = /^## CLI benchmark: (.+)$/mu.exec(markdown)?.[1] ?? "unknown"; + return title.replaceAll(/\s+/gu, " ").trim().slice(0, MAX_HEADLINE_CHARS); +} + +/** An entry for a run, or undefined when the commit or run link is not in the expected shape. */ +export function makeEntry(input: { + sha: string; + now: Date; + headline: string; + url: string; +}): RunEntry | undefined { + const entry = { + sha: input.sha.slice(0, 7), + at: `${input.now.toISOString().slice(0, 16)}Z`, + headline: input.headline, + url: input.url, + }; + return isEntry(entry) ? entry : undefined; +} + +/** The runs recorded in a comment body, newest first; empty when the marker is absent or invalid. */ +export function decodeHistory(body: string): Array { + const encoded = HISTORY_MARKER.exec(body)?.[1]; + if (encoded === undefined || encoded.length > MAX_ENCODED_CHARS) return []; + try { + const value: unknown = JSON.parse(Buffer.from(encoded, "base64").toString("utf8")); + return Array.isArray(value) && value.length <= MAX_PREVIOUS_RUNS + 1 && value.every(isEntry) + ? value + : []; + } catch { + return []; + } +} + +/** The new run first, then the earlier ones, with at most {@link MAX_PREVIOUS_RUNS} behind it. */ +export function mergeHistory( + previous: ReadonlyArray, + current: RunEntry, +): Array { + return [current, ...previous].slice(0, MAX_PREVIOUS_RUNS + 1); +} + +const day = (at: string) => `${at.slice(0, 10)} ${at.slice(11, 16)} UTC`; + +/** + * The hidden marker for `entries` (current run first), preceded by a collapsed list of the + * earlier ones when there are any. `clean` makes a headline safe to render. + */ +export function renderHistory( + entries: ReadonlyArray, + clean: (text: string) => string, +): string { + const marker = ``; + const earlier = entries.slice(1); + if (earlier.length === 0) return marker; + const lines = earlier.map( + (entry) => + `- \`${entry.sha}\` · ${day(entry.at)} · ${clean(entry.headline)} · [run](${entry.url})`, + ); + return ["
Previous runs", "", ...lines, "", "
", marker].join( + "\n", + ); +} diff --git a/.github/scripts/cli-bench/post-comment.test.ts b/.github/scripts/cli-bench/post-comment.test.ts new file mode 100644 index 0000000000..fb890a8952 --- /dev/null +++ b/.github/scripts/cli-bench/post-comment.test.ts @@ -0,0 +1,143 @@ +import { describe, expect, test } from "bun:test"; +import { + BENCH_MARKER, + FAILURE_TITLE, + MAX_BODY_CHARS, + planComment, + prepareBody, + sanitize, +} from "./post-comment.ts"; +import { decodeHistory, type RunEntry } from "./history.ts"; + +const BOT = "github-actions[bot]"; + +describe("planComment", () => { + test("creates the comment when none carries the marker", () => { + expect(planComment("create-if-missing", [{ id: 1, body: "hi", authorLogin: BOT }])).toEqual({ + kind: "create", + }); + }); + + test("leaves an existing marker comment alone on a later push", () => { + const existing = { id: 5, body: `${BENCH_MARKER}\nold`, authorLogin: BOT }; + + expect(planComment("create-if-missing", [existing])).toEqual({ kind: "skip" }); + }); + + test("replaces a first-run failure notice on a later push", () => { + const failed = { + id: 5, + body: `${BENCH_MARKER}\n${FAILURE_TITLE}\nbuild failed`, + authorLogin: BOT, + }; + + expect(planComment("create-if-missing", [failed])).toEqual({ + kind: "update", + id: 5, + previousBody: failed.body, + }); + }); + + test("updates the marker comment in place for a re-run", () => { + const existing = { id: 5, body: `${BENCH_MARKER}\nold`, authorLogin: BOT }; + + expect(planComment("upsert", [existing])).toEqual({ + kind: "update", + id: 5, + previousBody: existing.body, + }); + }); + + test("ignores a marker pasted by someone else", () => { + const forged = { id: 6, body: BENCH_MARKER, authorLogin: "mallory" }; + + expect(planComment("create-if-missing", [forged])).toEqual({ kind: "create" }); + }); +}); + +describe("prepareBody", () => { + test("puts the marker first exactly once", () => { + const body = prepareBody(`text ${BENCH_MARKER} more`); + + expect(body.startsWith(`${BENCH_MARKER}\n`)).toBe(true); + expect(body.split(BENCH_MARKER)).toHaveLength(2); + }); + + test("cuts an oversized report below GitHub's limit and says so", () => { + const body = prepareBody("x".repeat(200_000)); + + expect(body.length).toBeLessThanOrEqual(MAX_BODY_CHARS); + expect(body).toContain("truncated"); + }); +}); + +describe("prepareBody history", () => { + const run = (n: number): RunEntry => ({ + sha: n.toString(16).padStart(7, "0"), + at: "2026-10-28T09:05Z", + headline: "no changes found", + url: `https://github.com/o/r/actions/runs/${n}`, + }); + + test("lists earlier runs after the run link and carries them in the marker", () => { + const body = prepareBody("report", "https://github.com/o/r/actions/runs/3", [run(3), run(2)]); + + expect(body.indexOf("[Workflow run]")).toBeLessThan(body.indexOf("Previous runs")); + expect(decodeHistory(body)).toEqual([run(3), run(2)]); + }); + + test("drops the oldest runs before cutting the report when the body is too long", () => { + const history = Array.from({ length: 21 }, (_, i) => run(21 - i)); + const report = "x".repeat(MAX_BODY_CHARS - 4_000); + + const body = prepareBody(report, "https://github.com/o/r/actions/runs/21", history); + + expect(body.length).toBeLessThanOrEqual(MAX_BODY_CHARS); + expect(body).not.toContain("truncated"); + const kept = decodeHistory(body); + expect(kept.length).toBeGreaterThan(0); + expect(kept.length).toBeLessThan(history.length); + expect(kept[0]).toEqual(history[0]); + }); + + test("renders an earlier run's headline through the sanitizer", () => { + const hostile = { ...run(1), headline: "@team " }; + + const body = prepareBody("report", undefined, [run(2), hostile]); + + expect(body).not.toContain(" { + test("neutralizes mentions, links, images and unknown HTML but keeps the report tags", () => { + const hostile = + "@org/team [x](https://evil.test) ![i](https://evil.test/a.png) @u [a][r] www.evil.test
"; + + const clean = sanitize(hostile); + + expect(clean).not.toMatch(/@(?!\u200b)/u); + expect(clean).not.toContain("@"); + expect(clean).not.toMatch(/(?"); + }); + + test("pins the failure title and marker literals the renderer emits", () => { + expect(FAILURE_TITLE).toBe("## CLI benchmark: could not run"); + expect(BENCH_MARKER).toBe(""); + }); +}); + +describe("prepareBody run link", () => { + test("appends the trusted run link after the report", () => { + const body = prepareBody("report", "https://github.com/o/r/actions/runs/1"); + + expect(body.trimEnd().endsWith("[Workflow run](https://github.com/o/r/actions/runs/1)")).toBe( + true, + ); + }); +}); diff --git a/.github/scripts/cli-bench/post-comment.ts b/.github/scripts/cli-bench/post-comment.ts new file mode 100644 index 0000000000..d46572ea3c --- /dev/null +++ b/.github/scripts/cli-bench/post-comment.ts @@ -0,0 +1,153 @@ +/** + * Posts the CLI benchmark summary as the pull request's single marker comment. + * + * Runs in the one job that can write comments and treats the rendered markdown as untrusted data + * (pull request code produced it): it reads a bounded regular file, neutralizes mentions, links and + * HTML, adds the marker and the run link itself, truncates to GitHub's body limit, and creates or + * updates the comment. Run in CI as: `bun .github/scripts/cli-bench/post-comment.ts`. + * + * Modes: `create-if-missing` leaves an existing marker comment alone unless it reports a failed + * run (first run on a pull request); `upsert` updates it in place or creates it (a re-run). + * Every post also records the run in the comment's history of earlier runs. + */ +import { lstatSync, readFileSync } from "node:fs"; + +import { githubFetch, listIssueComments, requireEnv, type IssueComment } from "./github.ts"; +import { decodeHistory, headlineOf, makeEntry, mergeHistory, renderHistory } from "./history.ts"; +import type { RunEntry } from "./history.ts"; + +/** Hidden tag identifying the benchmark comment; the renderer in `bench-summary.ts` emits the same text. */ +export const BENCH_MARKER = ""; +/** Login of comments posted with the workflow's `GITHUB_TOKEN`. */ +const WORKFLOW_BOT_LOGIN = "github-actions[bot]"; +/** GitHub rejects comment bodies longer than 65536 characters. */ +export const MAX_BODY_CHARS = 65_000; +const TRUNCATION_NOTICE = + "\n\n
\n\n_The report was truncated; see the workflow run for the rest._\n"; + +export type Mode = "create-if-missing" | "upsert"; + +export type ExistingComment = IssueComment; + +export type CommentPlan = + | { kind: "skip" } + | { kind: "create" } + | { kind: "update"; id: number; previousBody: string }; + +const ALLOWED_TAGS = new Set(["
", "
", "", "", "
"]); +const ZERO_WIDTH_SPACE = "\u200b"; +/** Title the renderer gives a run that produced no comparison. */ +export const FAILURE_TITLE = "## CLI benchmark: could not run"; +export const MAX_FILE_BYTES = 1_000_000; + +/** Keeps mentions, links, images and raw HTML in untrusted markdown from rendering. */ +export function sanitize(markdown: string): string { + return markdown + .replaceAll("&", "&") + .replaceAll(/<[^>\n]*>?/gu, (tag) => + ALLOWED_TAGS.has(tag) ? tag : tag.replaceAll("<", "<"), + ) + .replaceAll("@", `@${ZERO_WIDTH_SPACE}`) + .replaceAll("://", `:${ZERO_WIDTH_SPACE}//`) + .replaceAll(/\b(www)\./giu, `$1${ZERO_WIDTH_SPACE}.`) + .replaceAll("[", "\\[") + .replaceAll("]", "\\]"); +} + +/** + * The comment body: the marker first, the sanitized report, the trusted run link, then the run + * history. `history` lists the current run first; the oldest runs are dropped until the body fits. + */ +export function prepareBody( + markdown: string, + runUrl?: string, + history: ReadonlyArray = [], +): string { + const link = runUrl === undefined || runUrl === "" ? "" : `\n\n[Workflow run](${runUrl})\n`; + const content = sanitize(markdown.replaceAll(BENCH_MARKER, "")).trim(); + const body = `${BENCH_MARKER}\n${content}\n`; + for (let kept = history.length; kept >= 1; kept--) { + const tail = `${link}\n${renderHistory(history.slice(0, kept), sanitize)}\n`; + if (body.length + tail.length <= MAX_BODY_CHARS) return body + tail; + } + const tail = `${link}\n${history.length === 0 ? "" : `${renderHistory(history.slice(0, 1), sanitize)}\n`}`; + const limit = MAX_BODY_CHARS - tail.length; + const fitted = + body.length <= limit + ? body + : body.slice(0, limit - TRUNCATION_NOTICE.length) + TRUNCATION_NOTICE; + return fitted + tail; +} + +/** The workflow bot's benchmark comment, if any. Only its marker counts. */ +export function findBenchComment( + comments: ReadonlyArray, +): ExistingComment | undefined { + return comments.find( + (comment) => comment.authorLogin === WORKFLOW_BOT_LOGIN && comment.body.includes(BENCH_MARKER), + ); +} + +/** What to do with the pull request's benchmark comment. */ +export function planComment(mode: Mode, comments: ReadonlyArray): CommentPlan { + const existing = findBenchComment(comments); + if (existing === undefined) return { kind: "create" }; + // A first run that could not compare anything must not block the next push's real result. + const replaceable = mode === "upsert" || existing.body.includes(FAILURE_TITLE); + return replaceable + ? { kind: "update", id: existing.id, previousBody: existing.body } + : { kind: "skip" }; +} + +async function main(): Promise { + const token = requireEnv("GITHUB_TOKEN"); + const base = `https://api.github.com/repos/${requireEnv("GITHUB_REPOSITORY")}`; + const pr = Number(requireEnv("PR_NUMBER")); + const mode = requireEnv("MODE"); + if (mode !== "create-if-missing" && mode !== "upsert") { + throw new Error(`Unsupported MODE "${mode}".`); + } + const file = requireEnv("BODY_FILE"); + const stat = lstatSync(file); + if (!stat.isFile() || stat.size > MAX_FILE_BYTES) { + throw new Error("The benchmark comment file is not a regular file under the size limit."); + } + const markdown = readFileSync(file, "utf8"); + const runUrl = process.env.RUN_URL; + const entry = makeEntry({ + sha: requireEnv("HEAD_SHA"), + now: new Date(), + headline: headlineOf(markdown), + url: runUrl ?? "", + }); + + const plan = planComment(mode, await listIssueComments(token, base, pr)); + const earlier = plan.kind === "update" ? decodeHistory(plan.previousBody) : []; + const body = prepareBody( + markdown, + runUrl, + entry === undefined ? [] : mergeHistory(earlier, entry), + ); + if (plan.kind === "skip") { + console.log(`PR #${pr} already has a benchmark comment; leaving it untouched.`); + } else if (plan.kind === "update") { + await githubFetch(`${base}/issues/comments/${plan.id}`, token, { + method: "PATCH", + body: JSON.stringify({ body }), + }); + console.log(`Updated benchmark comment ${plan.id} on PR #${pr}.`); + } else { + await githubFetch(`${base}/issues/${pr}/comments`, token, { + method: "POST", + body: JSON.stringify({ body }), + }); + console.log(`Created benchmark comment on PR #${pr}.`); + } +} + +if (import.meta.main) { + main().catch((error) => { + console.error(error); + process.exit(1); + }); +} diff --git a/.github/scripts/cli-bench/rerun.test.ts b/.github/scripts/cli-bench/rerun.test.ts new file mode 100644 index 0000000000..fa37cad648 --- /dev/null +++ b/.github/scripts/cli-bench/rerun.test.ts @@ -0,0 +1,184 @@ +import { describe, expect, test } from "bun:test"; +import { GithubRequestError } from "./github.ts"; +import { + type BenchPr, + type CommandIo, + handleBenchCommand, + RERUN_WINDOW_MS, + type WorkflowRun, +} from "./rerun.ts"; + +const REPO = "supabase/cli"; +const HEAD = "a".repeat(40); +const NOW = new Date("2026-10-28T12:00:00Z"); +const pr = (overrides: Partial = {}): BenchPr => ({ + state: "open", + draft: false, + headSha: HEAD, + headRepoFullName: REPO, + baseRepoFullName: REPO, + baseRef: "develop", + ...overrides, +}); +const run = (overrides: Partial = {}): WorkflowRun => ({ + id: 11, + status: "completed", + createdAt: "2026-10-27T12:00:00Z", + ...overrides, +}); + +function setup( + options: { + permissions?: Record; + pr?: BenchPr; + runs?: ReadonlyArray; + rerunError?: Error; + } = {}, +) { + const calls = { + reactions: [] as Array, + replies: [] as Array, + reruns: [] as Array, + }; + const io: CommandIo = { + fetchPr: async () => options.pr ?? pr(), + fetchPermission: async (login) => (options.permissions ?? { maintainer: "write" })[login], + listRuns: async () => options.runs ?? [run()], + rerun: async (id) => { + if (options.rerunError !== undefined) throw options.rerunError; + calls.reruns.push(id); + }, + reactToComment: async (id) => { + calls.reactions.push(id); + }, + replyToPr: async (_n, body) => { + calls.replies.push(body); + }, + }; + return { io, calls }; +} + +const command = (body: string, association = "COLLABORATOR", login = "maintainer") => ({ + prNumber: 7, + comment: { id: 99, authorLogin: login, authorAssociation: association, body }, + now: NOW, +}); + +describe("handleBenchCommand", () => { + test("re-runs the latest run for the head commit and reacts only after GitHub accepts it", async () => { + const { io, calls } = setup({ + runs: [run({ id: 1, createdAt: "2026-10-20T00:00:00Z" }), run({ id: 2 })], + }); + + const outcome = await handleBenchCommand(command("/bench\n"), io); + + expect(outcome).toEqual({ kind: "rerun", runId: 2 }); + expect(calls.reruns).toEqual([2]); + expect(calls.reactions).toEqual([99]); + expect(calls.replies).toEqual([]); + }); + + test.each(["/bench full", "/benchmark", "please /bench", "/Bench"])( + "ignores %p without a reply", + async (body) => { + const { io, calls } = setup(); + + const outcome = await handleBenchCommand(command(body), io); + + expect(outcome.kind).toBe("ignored"); + expect(calls).toEqual({ reactions: [], replies: [], reruns: [] }); + }, + ); + + test("rejects a commenter without write permission even as a MEMBER", async () => { + const { io, calls } = setup({ permissions: { maintainer: "read" } }); + + const outcome = await handleBenchCommand(command("/bench", "MEMBER"), io); + + expect(outcome).toMatchObject({ kind: "ignored" }); + expect(calls.reruns).toEqual([]); + }); + + test("lets the repository owner run it without a permission lookup result", async () => { + const { io } = setup({ permissions: {} }); + + expect((await handleBenchCommand(command("/bench", "OWNER"), io)).kind).toBe("rerun"); + }); + + test.each([ + ["a closed pull request", pr({ state: "closed" })], + ["a fork", pr({ headRepoFullName: "someone/cli" })], + ["a deleted head repository", pr({ headRepoFullName: "" })], + ["a pull request into another branch", pr({ baseRef: "feature/stack-base" })], + ])("ignores /bench on %s", async (_name, head) => { + const { io, calls } = setup({ pr: head }); + + const outcome = await handleBenchCommand(command("/bench"), io); + + expect(outcome.kind).toBe("ignored"); + expect(calls).toEqual({ reactions: [], replies: [], reruns: [] }); + }); + + test("tells the pull request there is no run when none exists for the head commit", async () => { + const { io, calls } = setup({ runs: [] }); + + const outcome = await handleBenchCommand(command("/bench"), io); + + expect(outcome.kind).toBe("replied"); + expect(calls.replies).toHaveLength(1); + expect(calls.replies[0]).toContain("no benchmark run for commit `aaaaaaa`"); + expect(calls.reruns).toEqual([]); + expect(calls.reactions).toEqual([]); + }); + + test("says a benchmark is already running instead of re-running it", async () => { + const { io, calls } = setup({ runs: [run({ status: "in_progress" })] }); + + await handleBenchCommand(command("/bench"), io); + + expect(calls.replies[0]).toContain("already running"); + expect(calls.reruns).toEqual([]); + expect(calls.reactions).toEqual([]); + }); + + test("says the re-run could not start when the run is older than 30 days", async () => { + const old = new Date(NOW.getTime() - RERUN_WINDOW_MS - 1000).toISOString(); + const { io, calls } = setup({ runs: [run({ createdAt: old })] }); + + await handleBenchCommand(command("/bench"), io); + + expect(calls.replies[0]).toContain("older than 30 days"); + expect(calls.reruns).toEqual([]); + }); + + test("surfaces GitHub's refusal in a reply and does not react", async () => { + const { io, calls } = setup({ + rerunError: new GithubRequestError(403, "Resource not accessible by integration", "u"), + }); + + const outcome = await handleBenchCommand(command("/bench"), io); + + expect(outcome.kind).toBe("replied"); + expect(calls.replies[0]).toContain("403"); + expect(calls.replies[0]).toContain("Resource not accessible by integration"); + expect(calls.reactions).toEqual([]); + }); + + test("keeps an accepted re-run when the reaction fails", async () => { + const { io } = setup(); + io.reactToComment = async () => { + throw new Error("boom"); + }; + + expect(await handleBenchCommand(command("/bench"), io)).toEqual({ kind: "rerun", runId: 11 }); + }); + + test("explains that a draft pull request is not benchmarked", async () => { + const { io, calls } = setup({ pr: pr({ draft: true }) }); + + await handleBenchCommand(command("/bench"), io); + + expect(calls.replies[0]).toContain("draft"); + expect(calls.reruns).toEqual([]); + }); +}); diff --git a/.github/scripts/cli-bench/rerun.ts b/.github/scripts/cli-bench/rerun.ts new file mode 100644 index 0000000000..b0020185b8 --- /dev/null +++ b/.github/scripts/cli-bench/rerun.ts @@ -0,0 +1,231 @@ +/** + * `/bench` handler: re-runs the benchmark workflow run of a pull request's current head commit. + * + * Runs default branch code and never checks out or runs pull request code. The commenter must be + * the repository owner or hold write access, and the pull request must be an open, ready one from + * this repository into `develop`. The command re-runs the existing `pull_request` run, which + * benchmarks and updates the comment; the pull request gets a reply when no run can be re-run. + * Run in CI as: `bun .github/scripts/cli-bench/rerun.ts`. + */ +import { appendFileSync } from "node:fs"; + +import { fetchAuthorPermission, WRITE_PERMISSIONS } from "../contribution-gate.ts"; +import { githubFetch, GithubRequestError, requireEnv } from "./github.ts"; + +export const BENCH_COMMAND = "/bench"; +/** Base branches the benchmark runs for; keep in sync with the workflow's `branches:` filter. */ +export const BENCH_BASE_BRANCHES: ReadonlyArray = ["develop"]; +/** The workflow file whose runs `/bench` re-runs. */ +export const WORKFLOW_FILE = "cli-bench.yml"; +/** GitHub refuses to re-run a workflow run older than this. */ +export const RERUN_WINDOW_MS = 30 * 24 * 60 * 60 * 1000; + +export interface BenchPr { + state: "open" | "closed"; + draft: boolean; + headSha: string; + /** `owner/name` of the head repository, empty when it was deleted. */ + headRepoFullName: string; + baseRepoFullName: string; + baseRef: string; +} + +export interface WorkflowRun { + id: number; + status: string; + createdAt: string; +} + +export interface CommandInput { + prNumber: number; + comment: { id: number; authorLogin: string; authorAssociation: string; body: string }; + now: Date; +} + +export interface CommandIo { + fetchPr: (prNumber: number) => Promise; + fetchPermission: (login: string) => Promise; + /** Runs of the benchmark workflow triggered by `pull_request` for this commit, any order. */ + listRuns: (headSha: string) => Promise>; + rerun: (runId: number) => Promise; + reactToComment: (commentId: number) => Promise; + replyToPr: (prNumber: number, body: string) => Promise; +} + +export type CommandOutcome = + | { kind: "ignored"; reason: string } + | { kind: "replied"; reason: string } + | { kind: "rerun"; runId: number }; + +const short = (sha: string) => sha.slice(0, 7); + +/** A GitHub error message as inline code: one short line with no backticks. */ +const quoted = (detail: string) => + `\`${detail.replaceAll("`", "'").replaceAll(/\s+/gu, " ").trim().slice(0, 200)}\``; + +export const replies = { + noRun: (sha: string) => + `There is no benchmark run for commit \`${short(sha)}\`. Runs start on a push that changes the ` + + "CLI, its packages, or the benchmark workflow, and GitHub's path filter looks at only the " + + "first 300 changed files, so this pull request may not have one.", + running: (sha: string) => + `A benchmark is already running for commit \`${short(sha)}\`. The comment updates when it finishes.`, + draft: () => `This pull request is a draft. The benchmark runs once it is ready for review.`, + expired: (sha: string) => + `The benchmark could not be re-run for commit \`${short(sha)}\`: its run is older than 30 days. ` + + "Push a new commit to benchmark again.", + refused: (sha: string, status: number, detail: string) => + `The benchmark could not be re-run for commit \`${short(sha)}\`: GitHub refused the request ` + + `(${status} ${quoted(detail)}).`, +}; + +/** Handles one `/bench` comment; a rejected command is ignored without a reply. */ +export async function handleBenchCommand( + input: CommandInput, + io: CommandIo, +): Promise { + const { comment, prNumber } = input; + // The workflow's `if:` only pre-filters on `startsWith`, so `/bench full` must be rejected here. + if (comment.body.trim() !== BENCH_COMMAND) { + return { kind: "ignored", reason: `Comment is not exactly ${BENCH_COMMAND}.` }; + } + // Only the repository OWNER skips the lookup: MEMBER and COLLABORATOR do not imply push access. + const permission = await io.fetchPermission(comment.authorLogin); + const authorized = + comment.authorAssociation === "OWNER" || + (permission !== undefined && WRITE_PERMISSIONS.has(permission)); + if (!authorized) { + return { + kind: "ignored", + reason: + `Commenter @${comment.authorLogin} is not authorized to run ${BENCH_COMMAND} ` + + `(author_association=${comment.authorAssociation}, permission=${permission ?? "n/a"}).`, + }; + } + const pr = await io.fetchPr(prNumber); + if (pr.state === "closed") return { kind: "ignored", reason: `PR #${prNumber} is closed.` }; + if (pr.headRepoFullName !== pr.baseRepoFullName) { + return { + kind: "ignored", + reason: `${BENCH_COMMAND} runs only on pull requests from this repository.`, + }; + } + if (!BENCH_BASE_BRANCHES.includes(pr.baseRef)) { + return { + kind: "ignored", + reason: `${BENCH_COMMAND} runs only on pull requests into ${BENCH_BASE_BRANCHES.join(" or ")}.`, + }; + } + + const reply = async (text: string): Promise => { + await io.replyToPr(prNumber, text); + return { kind: "replied", reason: text }; + }; + if (pr.draft) return reply(replies.draft()); + + const latest = [...(await io.listRuns(pr.headSha))].sort((a, b) => + b.createdAt.localeCompare(a.createdAt), + )[0]; + if (latest === undefined) return reply(replies.noRun(pr.headSha)); + if (latest.status !== "completed") return reply(replies.running(pr.headSha)); + if (input.now.getTime() - Date.parse(latest.createdAt) > RERUN_WINDOW_MS) { + return reply(replies.expired(pr.headSha)); + } + + try { + await io.rerun(latest.id); + } catch (error) { + if (error instanceof GithubRequestError) { + return reply(replies.refused(pr.headSha, error.status, error.detail)); + } + throw error; + } + // Cosmetic: a failed reaction must never hide an accepted re-run. + try { + await io.reactToComment(comment.id); + } catch (error) { + console.warn(`Could not react to comment ${comment.id}: ${String(error)}`); + } + return { kind: "rerun", runId: latest.id }; +} + +async function main(): Promise { + const token = requireEnv("GITHUB_TOKEN"); + const [owner = "", repo = ""] = requireEnv("GITHUB_REPOSITORY").split("/"); + const base = `https://api.github.com/repos/${owner}/${repo}`; + const prNumber = Number(requireEnv("PR_NUMBER")); + + const io: CommandIo = { + fetchPr: async (n) => { + const value = (await (await githubFetch(`${base}/pulls/${n}`, token)).json()) as { + state: "open" | "closed"; + draft: boolean; + head: { sha: string; repo: { full_name: string } | null }; + base: { ref: string; repo: { full_name: string } }; + }; + return { + state: value.state, + draft: value.draft, + headSha: value.head.sha, + headRepoFullName: value.head.repo?.full_name ?? "", + baseRepoFullName: value.base.repo.full_name, + baseRef: value.base.ref, + }; + }, + fetchPermission: (login) => fetchAuthorPermission(token, owner, repo, login), + listRuns: async (headSha) => { + const url = + `${base}/actions/workflows/${WORKFLOW_FILE}/runs` + + `?event=pull_request&head_sha=${headSha}&per_page=100`; + const payload = (await (await githubFetch(url, token)).json()) as { + workflow_runs: Array<{ id: number; status: string; created_at: string; head_sha: string }>; + }; + return payload.workflow_runs + .filter((run) => run.head_sha === headSha) + .map((run) => ({ id: run.id, status: run.status, createdAt: run.created_at })); + }, + rerun: async (runId) => { + await githubFetch(`${base}/actions/runs/${runId}/rerun`, token, { method: "POST" }); + }, + reactToComment: async (commentId) => { + await githubFetch(`${base}/issues/comments/${commentId}/reactions`, token, { + method: "POST", + body: JSON.stringify({ content: "eyes" }), + }); + }, + replyToPr: async (n, body) => { + await githubFetch(`${base}/issues/${n}/comments`, token, { + method: "POST", + body: JSON.stringify({ body }), + }); + }, + }; + + const outcome = await handleBenchCommand( + { + prNumber, + comment: { + id: Number(requireEnv("COMMENT_ID")), + authorLogin: requireEnv("COMMENT_AUTHOR_LOGIN"), + authorAssociation: requireEnv("COMMENT_AUTHOR_ASSOCIATION"), + body: requireEnv("COMMENT_BODY"), + }, + now: new Date(), + }, + io, + ); + const message = + outcome.kind === "rerun" + ? `Re-running benchmark run ${outcome.runId} for PR #${prNumber}.` + : `${outcome.kind === "replied" ? "Replied on" : "Ignored command on"} PR #${prNumber}: ${outcome.reason}`; + console.log(message); + const summaryFile = process.env.GITHUB_STEP_SUMMARY; + if (summaryFile) appendFileSync(summaryFile, `${message}\n`); +} + +if (import.meta.main) { + main().catch((error) => { + console.error(error); + process.exit(1); + }); +} diff --git a/.github/scripts/cli-bench/resolve.test.ts b/.github/scripts/cli-bench/resolve.test.ts new file mode 100644 index 0000000000..7ac55287b2 --- /dev/null +++ b/.github/scripts/cli-bench/resolve.test.ts @@ -0,0 +1,47 @@ +import { describe, expect, test } from "bun:test"; +import { BENCH_MARKER, FAILURE_TITLE } from "./post-comment.ts"; +import { resolveBench, SKIP_REASON } from "./resolve.ts"; + +const BOT = "github-actions[bot]"; +const reported = { + id: 5, + body: `${BENCH_MARKER}\n## CLI benchmark: no changes found`, + authorLogin: BOT, +}; + +describe("resolveBench", () => { + test("benchmarks the first push of a pull request that has no comment yet", () => { + expect(resolveBench({ runAttempt: 1, isFork: false, comments: [] }).shouldRun).toBe(true); + }); + + test("skips a later push while the comment reports a result, and says how to refresh it", () => { + const result = resolveBench({ runAttempt: 1, isFork: false, comments: [reported] }); + + expect(result).toEqual({ shouldRun: false, skipReason: SKIP_REASON }); + expect(SKIP_REASON).toContain("`/bench`"); + }); + + test("benchmarks again when the comment reports a failed run", () => { + const failed = { ...reported, body: `${BENCH_MARKER}\n${FAILURE_TITLE}` }; + + expect(resolveBench({ runAttempt: 1, isFork: false, comments: [failed] }).shouldRun).toBe(true); + }); + + test("benchmarks every re-run even when the comment reports a result", () => { + expect(resolveBench({ runAttempt: 2, isFork: false, comments: [reported] }).shouldRun).toBe( + true, + ); + }); + + test("ignores a marker pasted by someone else", () => { + const forged = { ...reported, authorLogin: "mallory" }; + + expect(resolveBench({ runAttempt: 1, isFork: false, comments: [forged] }).shouldRun).toBe(true); + }); + + test("benchmarks every push of a fork pull request, which never has a comment", () => { + expect(resolveBench({ runAttempt: 1, isFork: true, comments: [reported] }).shouldRun).toBe( + true, + ); + }); +}); diff --git a/.github/scripts/cli-bench/resolve.ts b/.github/scripts/cli-bench/resolve.ts new file mode 100644 index 0000000000..03882d2450 --- /dev/null +++ b/.github/scripts/cli-bench/resolve.ts @@ -0,0 +1,68 @@ +/** + * CLI benchmark resolver for `pull_request` runs: decides whether the benchmark job runs. + * + * The first run on a pull request benchmarks and comments. Later pushes skip while the comment + * exists and reports a result; a re-run (`run_attempt` above 1), a missing comment, a failed first + * run, or a fork pull request, which never has a comment, benchmarks. Needs read access only. + * Run in CI as: `bun .github/scripts/cli-bench/resolve.ts`. + */ +import { appendFileSync } from "node:fs"; + +import { listIssueComments, requireEnv, type IssueComment } from "./github.ts"; +import { FAILURE_TITLE, findBenchComment } from "./post-comment.ts"; + +export interface ResolveInput { + runAttempt: number; + isFork: boolean; + comments: ReadonlyArray; +} + +export interface ResolveResult { + shouldRun: boolean; + /** Shown in the job summary when the benchmark is skipped. */ + skipReason?: string; +} + +export const SKIP_REASON = + "A benchmark comment already exists on this pull request. Comment `/bench` to refresh it."; + +/** Skips only a first attempt on a pull request whose benchmark comment reports a result. */ +export function resolveBench(input: ResolveInput): ResolveResult { + if (input.isFork || input.runAttempt > 1) return { shouldRun: true }; + const existing = findBenchComment(input.comments); + if (existing === undefined || existing.body.includes(FAILURE_TITLE)) return { shouldRun: true }; + return { shouldRun: false, skipReason: SKIP_REASON }; +} + +async function main(): Promise { + const token = requireEnv("GITHUB_TOKEN"); + const base = `https://api.github.com/repos/${requireEnv("GITHUB_REPOSITORY")}`; + const prNumber = Number(requireEnv("PR_NUMBER")); + const runAttempt = Number(requireEnv("RUN_ATTEMPT")); + const isFork = requireEnv("HEAD_REPO") !== requireEnv("GITHUB_REPOSITORY"); + + let comments: ReadonlyArray = []; + if (!isFork && runAttempt === 1) { + // A comment that cannot be listed must not hide a benchmark: run it instead. + comments = await listIssueComments(token, base, prNumber).catch((error: unknown) => { + console.warn(`Could not list comments on PR #${prNumber}: ${String(error)}`); + return []; + }); + } + + const result = resolveBench({ runAttempt, isFork, comments }); + console.log( + `CLI benchmark resolve for PR #${prNumber}: should_run=${result.shouldRun} ` + + `attempt=${runAttempt} fork=${isFork}`, + ); + appendFileSync(requireEnv("GITHUB_OUTPUT"), `should_run=${result.shouldRun}\n`); + const summaryFile = process.env.GITHUB_STEP_SUMMARY; + if (result.skipReason && summaryFile) appendFileSync(summaryFile, `${result.skipReason}\n`); +} + +if (import.meta.main) { + main().catch((error) => { + console.error(error); + process.exit(1); + }); +} diff --git a/.github/workflows/cli-bench.yml b/.github/workflows/cli-bench.yml new file mode 100644 index 0000000000..73f76a6ac5 --- /dev/null +++ b/.github/workflows/cli-bench.yml @@ -0,0 +1,301 @@ +name: CLI benchmark +run-name: "CLI benchmark #${{ github.event.pull_request.number || github.event.issue.number }}" + +# Report-only comparison of a pull request's CLI build with a build of its merge base. Every +# workload runs against the replay server, so no Docker, database, or network is involved. The +# job never blocks a pull request and is not a required check. +# +# The first push to a pull request from this repository benchmarks and posts one comment tagged +# ``; later pushes skip while that comment reports a result. A maintainer +# commenting exactly `/bench` re-runs the run for the pull request's current head commit, as does +# "Re-run all jobs" in the Actions UI: a re-run benchmarks again and updates the comment, keeping +# earlier runs in a collapsed list. Pull requests from forks benchmark every push and write a job +# summary only. A pull request that changes no benchmarked path, or more than the 300 files +# GitHub's path filter inspects, has no run to re-run, and `/bench` says so. +# See apps/cli/docs/tracing-monitoring.md. +on: + pull_request: + types: + - opened + - synchronize + - reopened + - ready_for_review + branches: + - develop + paths: + - "apps/cli/**" + - "apps/cli-e2e/**" + - "packages/**" + - "pnpm-lock.yaml" + - "turbo.json" + - ".github/workflows/cli-bench.yml" + - ".github/scripts/cli-bench/**" + # `/bench` runs default branch code only: it never checks out or runs pull request code. + # jactionlint ignore=dangerous-triggers + issue_comment: + types: + - created + +permissions: {} + +# Comment runs get a group of their own, so a comment can never cancel a pull request's benchmark +# nor be cancelled by one. +concurrency: + group: >- + cli-bench-${{ github.event.pull_request.number || github.event.issue.number }}${{ + github.event_name == 'issue_comment' && format('-comment-{0}', github.run_id) || '' }} + cancel-in-progress: true + +jobs: + # Decides whether this push benchmarks. Read-only: it only lists the pull request's comments. + resolve: + name: Resolve + if: github.event_name == 'pull_request' && github.event.pull_request.draft == false + runs-on: ubuntu-latest + timeout-minutes: 5 + permissions: + contents: read + pull-requests: read + outputs: + should_run: ${{ steps.resolve.outputs.should_run }} + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2.2.0 + with: + bun-version-file: ".bun-version" + no-cache: true + - name: Resolve + id: resolve + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + GITHUB_REPOSITORY: ${{ github.repository }} + PR_NUMBER: ${{ github.event.pull_request.number }} + HEAD_REPO: ${{ github.event.pull_request.head.repo.full_name }} + RUN_ATTEMPT: ${{ github.run_attempt }} + run: bun .github/scripts/cli-bench/resolve.ts + + # `/bench`: re-runs this workflow's run for the pull request's current head commit. The only job + # with `actions: write`; it runs default branch code and no pull request code. + rerun: + name: Re-run benchmark + # Cheap, non-authoritative pre-filter; rerun.ts does the exact command match and the + # effective-permission lookup. + if: > + github.event_name == 'issue_comment' && + github.event.issue.pull_request != null && + startsWith(github.event.comment.body, '/bench') && + contains(fromJSON('["OWNER", "MEMBER", "COLLABORATOR"]'), github.event.comment.author_association) + runs-on: ubuntu-latest + timeout-minutes: 5 + permissions: + actions: write + contents: read + pull-requests: write + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2.2.0 + with: + bun-version-file: ".bun-version" + no-cache: true + - name: Re-run + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + GITHUB_REPOSITORY: ${{ github.repository }} + PR_NUMBER: ${{ github.event.issue.number }} + COMMENT_ID: ${{ github.event.comment.id }} + COMMENT_AUTHOR_LOGIN: ${{ github.event.comment.user.login }} + COMMENT_AUTHOR_ASSOCIATION: ${{ github.event.comment.author_association }} + COMMENT_BODY: ${{ github.event.comment.body }} + run: bun .github/scripts/cli-bench/rerun.ts + + bench: + name: Benchmark CLI against merge base + needs: resolve + if: needs.resolve.outputs.should_run == 'true' + runs-on: blacksmith-4vcpu-ubuntu-2404 + timeout-minutes: 30 + permissions: + contents: read + steps: + - name: Checkout pull request head + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + ref: ${{ github.event.pull_request.head.sha }} + fetch-depth: 0 + persist-credentials: false + + - name: Setup + uses: ./.github/actions/setup + with: + dependency-firewall-token: ${{ secrets.DF_FIREWALL_TOKEN }} + + - name: Prepare benchmark directory + shell: bash + run: | + mkdir -p "$RUNNER_TEMP/bench" + echo "BENCH_DIR=$RUNNER_TEMP/bench" >> "$GITHUB_ENV" + + - name: Build branch binary + id: branch_build + continue-on-error: true + shell: bash + run: | + pnpm --filter supabase run build:binary + mkdir -p "$BENCH_DIR/branch" + cp apps/cli/dist/supabase "$BENCH_DIR/branch/supabase" + + - name: Build merge-base binary + id: base_build + continue-on-error: true + if: steps.branch_build.outcome == 'success' + env: + HUSKY: "0" + BASE_REF: ${{ github.event.pull_request.base.ref }} + shell: bash + run: | + merge_base="$(git merge-base HEAD "origin/$BASE_REF")" + echo "Merge base: $merge_base" + echo "merge_base=$merge_base" >> "$GITHUB_OUTPUT" + git worktree add "$RUNNER_TEMP/base-src" "$merge_base" + cd "$RUNNER_TEMP/base-src" + # The setup action writes this file only when the Dependency Firewall token is available. + if [ -f "$RUNNER_TEMP/dependency-firewall.npmrc" ]; then + export NPM_CONFIG_USERCONFIG="$RUNNER_TEMP/dependency-firewall.npmrc" + fi + pnpm install --frozen-lockfile + pnpm --filter supabase run build:binary + mkdir -p "$BENCH_DIR/base" + cp apps/cli/dist/supabase "$BENCH_DIR/base/supabase" + + - name: Start replay server + id: replay + continue-on-error: true + if: steps.branch_build.outcome == 'success' && steps.base_build.outcome == 'success' + shell: bash + run: | + bun apps/cli-e2e/scripts/replay-serve.ts > "$BENCH_DIR/replay.log" 2>&1 & + echo "REPLAY_PID=$!" >> "$GITHUB_ENV" + for _ in $(seq 1 60); do + url="$(grep -o 'http://[^"]*' "$BENCH_DIR/replay.log" | head -n 1 || true)" + if [ -n "$url" ]; then + echo "REPLAY_URL=$url" >> "$GITHUB_ENV" + exit 0 + fi + sleep 0.5 + done + echo "replay server did not report a URL" >&2 + cat "$BENCH_DIR/replay.log" >&2 + exit 1 + + - name: Run benchmark + id: bench + continue-on-error: true + if: steps.replay.outcome == 'success' + shell: bash + run: | + bun apps/cli/scripts/bench-cli.ts \ + --base "$BENCH_DIR/base/supabase" \ + --branch "$BENCH_DIR/branch/supabase" \ + --workloads apps/cli/bench/workloads.pr.json \ + --api-url "$REPLAY_URL" \ + --trace \ + --out "$BENCH_DIR/report.json" | tee "$BENCH_DIR/report.txt" + + - name: Stop replay server + if: always() && env.REPLAY_PID != '' + shell: bash + run: kill "$REPLAY_PID" || true + + - name: Write job summary + if: always() + env: + BRANCH_BUILD: ${{ steps.branch_build.outcome == 'success' && 'ok' || 'failed' }} + BASE_BUILD: ${{ steps.base_build.outcome == 'success' && 'ok' || 'failed' }} + REPLAY: ${{ steps.replay.outcome == 'success' && 'ok' || 'failed' }} + MERGE_BASE: ${{ steps.base_build.outputs.merge_base }} + HEAD_SHA: ${{ github.event.pull_request.head.sha }} + BASE_REF: ${{ github.event.pull_request.base.ref }} + shell: bash + run: | + bun apps/cli/scripts/bench-summary.ts \ + --report "$BENCH_DIR/report.json" \ + --branch-build "$BRANCH_BUILD" \ + --base-build "$BASE_BUILD" \ + --replay "$REPLAY" \ + --head-sha "$HEAD_SHA" \ + --base-ref "$BASE_REF" \ + --merge-base "$MERGE_BASE" \ + --comment-out "$BENCH_DIR/comment.md" + + - name: Upload report + if: always() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: cli-bench-report + path: | + ${{ runner.temp }}/bench/report.json + ${{ runner.temp }}/bench/report.txt + retention-days: 14 + if-no-files-found: ignore + + - name: Upload comment + if: always() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: cli-bench-comment + path: ${{ runner.temp }}/bench/comment.md + retention-days: 1 + if-no-files-found: ignore + + comment: + name: Post pull request comment + needs: + - resolve + - bench + # A pull_request run from a fork has a read-only token, so it posts no comment. Failing to + # post never fails the workflow. + if: >- + ${{ !cancelled() && needs.resolve.outputs.should_run == 'true' && needs.bench.result == 'success' && + github.event.pull_request.head.repo.full_name == github.repository }} + runs-on: ubuntu-latest + timeout-minutes: 5 + continue-on-error: true + # The only pull_request job that can write to the pull request. It checks out the merge commit, + # which its author can already run with this token by editing the workflow. It never builds or + # runs the benchmarked binaries and reads the rendered markdown as data. + permissions: + contents: read + pull-requests: write + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2.2.0 + with: + bun-version-file: ".bun-version" + no-cache: true + - name: Download comment + continue-on-error: true + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + name: cli-bench-comment + path: ${{ runner.temp }}/bench-comment + - name: Post comment + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + GITHUB_REPOSITORY: ${{ github.repository }} + PR_NUMBER: ${{ github.event.pull_request.number }} + MODE: ${{ github.run_attempt > 1 && 'upsert' || 'create-if-missing' }} + HEAD_SHA: ${{ github.event.pull_request.head.sha }} + BODY_FILE: ${{ runner.temp }}/bench-comment/comment.md + RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}/attempts/${{ github.run_attempt }} + run: | + if [ ! -f "$BODY_FILE" ]; then + echo "The benchmark job produced no comment." + exit 0 + fi + bun .github/scripts/cli-bench/post-comment.ts diff --git a/apps/cli-e2e/package.json b/apps/cli-e2e/package.json index 28e474eaa3..d5caeef214 100644 --- a/apps/cli-e2e/package.json +++ b/apps/cli-e2e/package.json @@ -7,6 +7,7 @@ "test": "pnpm run test:e2e", "test:e2e": "pnpm exec turbo run supabase#build && pnpm exec turbo run @supabase/cli-e2e#test:e2e:run --only --", "test:e2e:run": "bun --bun vitest run", + "replay:serve": "bun scripts/replay-serve.ts", "record": "RECORD=true pnpm run test:e2e:run", "types:check": "tsc --noEmit" }, diff --git a/apps/cli-e2e/scripts/replay-serve.ts b/apps/cli-e2e/scripts/replay-serve.ts new file mode 100644 index 0000000000..f1ea4a6e88 --- /dev/null +++ b/apps/cli-e2e/scripts/replay-serve.ts @@ -0,0 +1,36 @@ +import { startReplayServer } from "../src/server/replay-server.ts"; + +const FIXTURES_DIR = new URL("../fixtures", import.meta.url).pathname; + +function parsePort(argv: readonly string[]): number { + const index = argv.indexOf("--port"); + if (index === -1) return 0; + const port = Number(argv[index + 1]); + if (!Number.isInteger(port) || port < 0 || port > 65535) { + throw new Error(`--port expects an integer between 0 and 65535, got "${argv[index + 1]}"`); + } + return port; +} + +// Recording mode wipes the committed fixtures, so a standalone server never runs in it. +if (process.env["RECORD"] === "true") { + throw new Error("replay-serve does not support RECORD; unset it to serve the fixtures"); +} + +const server = await startReplayServer({ + fixturesDir: FIXTURES_DIR, + hostname: "127.0.0.1", + port: parsePort(process.argv.slice(2)), +}); + +let stopping = false; +const shutdown = async () => { + if (stopping) return; + stopping = true; + await server.stop(); + process.exit(0); +}; +process.on("SIGTERM", shutdown); +process.on("SIGINT", shutdown); + +console.log(JSON.stringify({ url: server.url })); diff --git a/apps/cli-e2e/src/server/replay-server.ts b/apps/cli-e2e/src/server/replay-server.ts index 8706bc8736..4d85198cac 100644 --- a/apps/cli-e2e/src/server/replay-server.ts +++ b/apps/cli-e2e/src/server/replay-server.ts @@ -153,6 +153,8 @@ interface ReplayServerOptions { fixturesDir: string; /** Port to listen on (0 = random). */ port?: number; + /** Interface to bind; Bun binds all interfaces when omitted. */ + hostname?: string; /** Optional Postgres mock server to control via /_ctrl/pg-* endpoints. */ pgMock?: PgMockHandle; } @@ -193,6 +195,7 @@ export async function startReplayServer(options: ReplayServerOptions): Promise` to also write the raw per-run samples, and `--cpu-prof` to capture a Bun CPU profile of one untimed @@ -84,6 +84,152 @@ rm -f /tmp/cli-base/mise.toml /tmp/cli-base/mise.lock # if mise refuses the untr (cd /tmp/cli-base && pnpm install && pnpm exec turbo run supabase#build) ``` +### Pull request benchmark + +`.github/workflows/cli-bench.yml` runs on every ready pull request into `develop` that touches the CLI, its packages, or +the e2e replay fixtures. It builds the pull request and the base tip it merges into with `pnpm --filter supabase run build:binary`, runs both +through `bench-cli.ts` against a local replay server, and writes a job summary with `bench-summary.ts`. +The job is report only: it never blocks a pull request, and a benchmark problem is reported in the +summary instead of failing the job: a failed branch or base build or a replay server that does not +start replaces the comparison with a notice naming the step. The workloads use no Docker, database, or network, and no +prompts. + +The first push to a pull request from this repository benchmarks and posts one comment, tagged with +the hidden marker ``. Later pushes skip the benchmark, and the job summary says +so, while that comment reports a result; a comment that reports a failed run is benchmarked again +on the next push. Pull requests from forks have a read-only token, so they get the job summary only +and benchmark every push. + +A maintainer can comment exactly `/bench` to run the benchmark again on the pull request's current +head commit. The command re-runs that commit's `pull_request` workflow run, as "Re-run all jobs" in +the Actions UI does, and the re-run updates the comment in place, or creates it if it is missing. +The comment keeps a collapsed "Previous runs" list below the report with the commit, time (UTC), +headline, and link of each earlier run, newest first and up to 20, stored in a hidden marker in the +comment. Only the repository owner and people with write access can run it, and only on open, +non-draft pull requests into `develop` from this repository; the command must be the whole comment, +so `/bench full` and anything else is ignored. An eyes reaction on the comment means GitHub accepted +the re-run and a benchmark is running; the result arrives as the updated benchmark comment, whether +it succeeded or failed. When no re-run can start, the pull request gets a reply saying why: a +benchmark is already running for the commit, the run is older than 30 days, GitHub refused the +re-run (the reply includes the status and message), or there is no run for the commit. + +Known gap: a pull request whose changes match none of the workflow's path filters, or that changes +more files than GitHub's path filter inspects (300), gets no run, so there is nothing for `/bench` +to re-run and it replies that there is none. + +Workflows triggered by comments run from the default branch, so `/bench` works only once the +workflow is on `develop`. The `/bench` job runs default branch code only: it never checks out or +runs pull request code, because a comment-triggered run shares the default branch's cache scope. +Benchmarks always run in the `pull_request` run. `resolve` (read-only) decides whether a push +benchmarks; `rerun` is the only job with `actions: write`. `bench` has read-only permissions, and +`comment` is the only `pull_request` job that can write to the pull request. It never builds or +runs the benchmarked binaries: it downloads the rendered markdown, sanitizes it, and posts it. + +To run it locally with two binaries (for example builds from two commits): + +```sh +pnpm --filter @supabase/cli-e2e run replay:serve # prints {"url":"http://127.0.0.1:"} +bun apps/cli/scripts/bench-cli.ts --base --branch \ + --workloads apps/cli/bench/workloads.pr.json --api-url --trace --out /tmp/report.json +bun apps/cli/scripts/bench-summary.ts --report /tmp/report.json +``` + +`--workloads` replaces repeated `--command` flags. `apps/cli/bench/workloads.pr.json` lists entries +with an `id`, an `args` array (so quoting works), a `cwdSetup` of `none` or `init`, and optionally +a replay `scenario`. `init` runs `supabase init` in the launch's project directory first. Setup runs +outside the timed region. + +`--api-url` (which requires `--trace`) points every launch at the replay server by writing a profile +file into the launch's fresh `SUPABASE_HOME` and setting `SUPABASE_PROFILE`, plus a dummy access +token. The profile is the CLI's only API redirect: `SUPABASE_API_URL` does not change where the +Management API client sends requests, so it is never set. Every launch gets the profile, including +`--version` and `init`, so an unexpected new API call reaches the server and is counted. Before each +launch the harness clears the server's request log and scenario, then selects the workload's +scenario, so scenarios that are used up as they are served start fresh every time. A failing control +call aborts the benchmark. A launch that runs longer than 60 seconds is killed and counted as a +failed launch (exit code 124), so that workload is reported as not compared and the rest finish. + +API calls are counted from the traced launch's HTTP client spans: spans to the `--api-url` host and +port are API calls, and spans to any other host are `externalCalls`. The count is cross-checked +against the replay server's request log for the same launch. The `--out` file stores raw per-run +data; the summary derives medians, flags, and verdicts from it, so thresholds can change without +re-running. + +The report (the comment and the job summary share one renderer) leads with the answer, "no changes +found" or "N things to check", then one plain-words line for each flag, a table of the median time +per command before and after the change, API calls, and the binary size. A collapsed section holds +the timing ranges, API calls per command, and the slowest phases. It reports: + +- Flags: an API call count that goes from 0 to more than 0 (for example `migration new` starting to + call the API) or from more than 0 to 0, calls to hosts other than the replay server, counts that + differ between runs of one build, spans that disagree with the request log, 4xx/5xx replay + responses (the build did not follow the recorded scenario), non-zero or differing exit codes, + missing traces, and too few runs for a timing verdict. Other count changes, such as 1 to 2 calls, appear in the API calls table with the + changed endpoints but are not flagged. +- Wall time: base ("before") and branch ("after") medians and ranges. A verdict of slower or faster needs the branch + median outside the base min-max range, a change of at least 5 percent and 5 ms, and at least 5 + measured runs; otherwise it reads "within noise", or "not comparable" when a build failed. + Thresholds are named constants in `bench-report.ts`. +- API calls per workload, base and branch. +- Phase deltas: the three largest span deltas for each workload with a slower or faster verdict. +- Binary size of `dist/supabase`. + +The baseline API calls per workload on `develop` (a project-scoped command +also refreshes the linked-project cache, so it makes two calls): + +| Workload | API calls | +| ------------------------------------------ | ---------------------------------------------------------------------- | +| `version`, `help`, `init`, `migration-new` | 0 | +| `projects-list` | 1: `GET /v1/projects` | +| `gen-types` | 2: `GET /v1/projects/{ref}/types/typescript`, `GET /v1/projects/{ref}` | +| `functions-list` | 2: `GET /v1/projects/{ref}/functions`, `GET /v1/projects/{ref}` | + +`SUPABASE_NO_KEYRING=1` disables the keyring (`CommandCredentials.load` reports +`keyring.enabled=false`); the value `true` is not recognized. Counts come from the untimed traced +launch, so a launch that behaves differently when timed shows up only as an exit-code difference. + +#### How the workloads were chosen + +The list comes from `cli_command_executed` usage in PostHog over the last 30 days, for stable CLI +versions only (prereleases excluded). Commands are ranked by distinct `device_id`, then by run +count; the p50 and p90 duration of successful runs, the CI share, and the agent share are context. +Only commands that run with no Docker, no database, no real network, and no prompts are kept, and +every Go-proxied command is dropped. One command is taken from each class (startup, local files, +API read without a ref, API calls with a ref, API read of a large response) rather than the top N +overall, capped to fit the time budget. Refresh the list each quarter, or after the +v3 release with the version regex updated: + +```sql +SELECT properties.command cmd, uniq(properties.device_id) users, count() runs, + round(100*countIf(properties.exit_code != 0)/count(),1) err_pct, + round(quantileIf(0.5)(properties.duration_ms, properties.exit_code = 0)) p50_ms, + round(quantileIf(0.9)(properties.duration_ms, properties.exit_code = 0)) p90_ms, + round(100*countIf(properties.is_ci = true)/count(),1) ci_pct, + round(100*countIf(properties.is_agent = true)/count(),1) agent_pct +FROM events WHERE event='cli_command_executed' AND timestamp >= now() - INTERVAL 30 DAY + AND match(properties.cli_version,'^2\\.(119|120|121|122)\\.[0-9]+$') +GROUP BY cmd ORDER BY users DESC LIMIT 40 +``` + +Caveats: + +- Telemetry records the command path and the flags set on the command line, with values redacted + unless the flag is boolean, a choice, or listed in the command's `safeFlags` (see + [analytics.md](analytics.md)); positional arguments and free-form values are not recorded, so + `bench` and the TypeScript target only approximate real use. +- `device_id` counts machines, not people, short-lived CI runners inflate it, and a large share of + runs come from agents. Telemetry durations include network, Docker, and wait time, so they guided + the choice but are not what the benchmark measures. +- Excluded high-usage commands: `start`, `status`, `stop`, `db start`, `db reset`, and `test db` + need Docker; `db push`, `db lint`, `db dump`, `db diff`, `migration list`, `migration up`, + `migration repair`, and `db schema declarative sync` need a database (a nightly tier); + `link` probes `https://.`, which never reaches the replay server; + `functions deploy` uses Docker by default and its API path has no success scenario; `login` is + interactive unless a token is passed; `functions serve` needs Docker. +- The first launch of a process is slower (credential loading); the warm-up runs absorb it. +- Launches write a profile file, so numbers from runs with `--api-url` are not comparable with runs + without it. + ## Grafana, Tempo, or Jaeger Run a local OpenTelemetry stack, for example Grafana's all-in-one image: diff --git a/apps/cli/scripts/__fixtures__/api-call-trace.jsonl b/apps/cli/scripts/__fixtures__/api-call-trace.jsonl new file mode 100644 index 0000000000..da0943e55b --- /dev/null +++ b/apps/cli/scripts/__fixtures__/api-call-trace.jsonl @@ -0,0 +1,2 @@ +{"resourceSpans":[{"scopeSpans":[{"scope":{"name":"effect","version":"4.0.2"},"spans":[{"traceId":"8ced2df9b0dcb856be8a39ecde5572f1","spanId":"b4b54720ec88d11a","parentSpanId":"894f97211df62fba","name":"GET","kind":3,"startTimeUnixNano":"1793178856745192708","endTimeUnixNano":"1793178856748612583","attributes":[{"key":"http.request.method","value":{"stringValue":"GET"}},{"key":"url.full","value":{"stringValue":"http://127.0.0.1:62824/v1/projects"}},{"key":"url.path","value":{"stringValue":"/v1/projects"}},{"key":"url.scheme","value":{"stringValue":"http"}},{"key":"server.address","value":{"stringValue":"127.0.0.1"}},{"key":"server.port","value":{"intValue":62824}},{"key":"http.response.status_code","value":{"intValue":200}}],"droppedAttributesCount":0,"events":[],"droppedEventsCount":0,"status":{"code":1},"links":[],"droppedLinksCount":0},{"traceId":"8ced2df9b0dcb856be8a39ecde5572f1","spanId":"894664602ca4b246","name":"cli.run","kind":1,"startTimeUnixNano":"1793178856730765750","endTimeUnixNano":"1793178856751636708","attributes":[{"key":"process.boot_ms","value":{"intValue":100}},{"key":"process.exit_code","value":{"intValue":0}}],"droppedAttributesCount":0,"events":[],"droppedEventsCount":0,"status":{"code":1},"links":[],"droppedLinksCount":0}]}]}]} +{"resourceSpans":[{"scopeSpans":[{"scope":{"name":"effect","version":"4.0.2"},"spans":[{"traceId":"ab4e0d434b5b64d740d8c6b20839d5bc","spanId":"ba8b1abcbb3ace23","parentSpanId":"22c8e0027cbedc65","name":"GET","kind":3,"startTimeUnixNano":"1793178857188826250","endTimeUnixNano":"1793178857192480042","attributes":[{"key":"http.request.method","value":{"stringValue":"GET"}},{"key":"url.full","value":{"stringValue":"http://127.0.0.1:62824/v1/projects"}},{"key":"url.path","value":{"stringValue":"/v1/projects"}},{"key":"url.scheme","value":{"stringValue":"http"}},{"key":"server.address","value":{"stringValue":"127.0.0.1"}},{"key":"server.port","value":{"intValue":62824}},{"key":"http.response.status_code","value":{"intValue":200}}],"droppedAttributesCount":0,"events":[],"droppedEventsCount":0,"status":{"code":1},"links":[],"droppedLinksCount":0},{"traceId":"ab4e0d434b5b64d740d8c6b20839d5bc","spanId":"8330e018c95efc64","name":"cli.run","kind":1,"startTimeUnixNano":"1793178857174426458","endTimeUnixNano":"1793178857195383917","attributes":[{"key":"process.boot_ms","value":{"intValue":118}},{"key":"process.exit_code","value":{"intValue":0}}],"droppedAttributesCount":0,"events":[],"droppedEventsCount":0,"status":{"code":1},"links":[],"droppedLinksCount":0}]}]}]} diff --git a/apps/cli/scripts/bench-cli.integration.test.ts b/apps/cli/scripts/bench-cli.integration.test.ts new file mode 100644 index 0000000000..6caeed1fcb --- /dev/null +++ b/apps/cli/scripts/bench-cli.integration.test.ts @@ -0,0 +1,109 @@ +import { BunServices } from "@effect/platform-bun"; +import { describe, expect, it } from "@effect/vitest"; +import { Duration, Effect, FileSystem, Layer, Ref } from "effect"; +import { FetchHttpClient } from "effect/http"; +import { startReplayServer } from "../../cli-e2e/src/server/replay-server.ts"; +import { + LAUNCH_TIMEOUT_EXIT_CODE, + runLaunch, + runWorkloadBench, + type BenchOptions, +} from "./bench-cli.ts"; + +const fixturesDir = new URL("../../cli-e2e/fixtures", import.meta.url).pathname; + +// Stands in for a CLI build: every launch makes one request to the profile's API and, when asked +// to trace, records it as an HTTP client span. +const FAKE_CLI = `#!/bin/sh +URL=$(sed -n 's/^api_url: "\\(.*\\)"$/\\1/p' "$SUPABASE_PROFILE") +curl -s -o /dev/null "$URL/v1/projects" +if [ -n "$SUPABASE_TRACE_FILE" ]; then + PORT=\${URL##*:} + printf '{"resourceSpans":[{"scopeSpans":[{"spans":[{"traceId":"t","spanId":"s","name":"GET","startTimeUnixNano":"0","endTimeUnixNano":"1000000","status":{"code":1},"attributes":[{"key":"http.request.method","value":{"stringValue":"GET"}},{"key":"url.path","value":{"stringValue":"/v1/projects"}},{"key":"server.address","value":{"stringValue":"127.0.0.1"}},{"key":"server.port","value":{"intValue":%s}},{"key":"http.response.status_code","value":{"intValue":200}}]}]}]}]}\\n' "$PORT" >> "$SUPABASE_TRACE_FILE" +fi +`; + +describe("runWorkloadBench", () => { + it.effect("counts only the measured launch's API calls, not setup or earlier launches", () => + Effect.scoped( + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const server = yield* Effect.acquireRelease( + Effect.promise(() => startReplayServer({ fixturesDir })), + (handle) => Effect.promise(() => handle.stop()), + ); + const dir = yield* fs.makeTempDirectoryScoped({ prefix: "bench-fake-cli-" }); + const binary = `${dir}/supabase`; + yield* fs.writeFileString(binary, FAKE_CLI); + yield* fs.chmod(binary, 0o755); + const options: BenchOptions = { + base: binary, + branch: binary, + runs: 2, + warmup: 1, + commands: [], + workloadsFile: undefined, + cwdSetupInit: false, + apiUrl: server.url, + trace: true, + json: false, + out: undefined, + cpuProf: false, + updateCheck: false, + }; + + const { report } = yield* runWorkloadBench( + options, + { + id: "w", + args: ["projects", "list"], + cwdSetup: "init", + scenario: "projects-list-renders-project-list", + }, + yield* Ref.make(false), + ); + + const samples = [...report.base, ...report.branch]; + expect(samples).toHaveLength(4); + for (const sample of samples) { + expect(sample.exitCode).toBe(0); + expect(sample.trace?.api?.apiCalls).toBe(1); + expect(sample.trace?.api?.requestLogCount).toBe(1); + expect(sample.trace?.api?.externalCalls).toBe(0); + } + }), + ).pipe(Effect.provide(Layer.mergeAll(BunServices.layer, FetchHttpClient.layer))), + ); +}); + +describe("runLaunch", () => { + it.live("kills a launch that outlives the timeout and records a failing exit code", () => + Effect.scoped( + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const dir = yield* fs.makeTempDirectoryScoped({ prefix: "bench-hung-cli-" }); + const pidFile = `${dir}/pid`; + const binary = `${dir}/supabase`; + yield* fs.writeFileString( + binary, + `#!/bin/sh\necho $$ > "${pidFile}"\nexec /bin/sleep 60\n`, + ); + yield* fs.chmod(binary, 0o755); + + const result = yield* runLaunch({ + binary, + args: [], + cwd: dir, + env: {}, + timeout: Duration.millis(500), + }); + + expect(result.exitCode).toBe(LAUNCH_TIMEOUT_EXIT_CODE); + expect(result.wallMs).toBeLessThan(30_000); + const pid = Number((yield* fs.readFileString(pidFile)).trim()); + expect(Number.isInteger(pid)).toBe(true); + expect(() => process.kill(pid, 0)).toThrow(); + }), + ).pipe(Effect.provide(BunServices.layer)), + ); +}); diff --git a/apps/cli/scripts/bench-cli.ts b/apps/cli/scripts/bench-cli.ts index 6e1df06c16..5699c7aba8 100644 --- a/apps/cli/scripts/bench-cli.ts +++ b/apps/cli/scripts/bench-cli.ts @@ -1,32 +1,65 @@ /** * A/B wall-time benchmark for two CLI binaries. Alternates launches between `--base` and * `--branch` so neither build runs consistently warmer, isolates each launch's environment, and - * optionally compares per-span trace totals when `SUPABASE_TRACE_FILE` tracing is available. Run - * as `bun scripts/bench-cli.ts --base --branch [options]`. + * optionally records per-span trace totals and Management API call counts from + * `SUPABASE_TRACE_FILE` tracing. Run as + * `bun scripts/bench-cli.ts --base --branch [options]`. */ import { randomUUID } from "node:crypto"; import { tmpdir } from "node:os"; import process from "node:process"; import { parseArgs } from "node:util"; import { BunPath, BunRuntime, BunServices } from "@effect/platform-bun"; -import { Clock, Console, Effect, FileSystem, Path, Ref, Schema, Stdio } from "effect"; +import { + Clock, + Console, + Duration, + Effect, + FileSystem, + Layer, + Option, + Path, + Ref, + Schema, + Stdio, +} from "effect"; +import { FetchHttpClient } from "effect/http"; import { ChildProcess, ChildProcessSpawner } from "effect/process"; -import { readSpans, type ReportSpan } from "./trace-report.ts"; +import { + analyzeWorkload, + BenchReport, + classifyHttpSpans, + encodeBenchReport, + collectFlags, + parseWorkloadManifest, + summarizeSpanRun, + validateWorkloads, + type Build, + type RunSample, + type SpanComparison, + type Workload, + type WorkloadAnalysis, + type WorkloadReport, +} from "./bench-report.ts"; +import { readRequestLogCount, resetReplay } from "./replay-control.ts"; +import { readSpans } from "./trace-report.ts"; const USAGE = - 'usage: bun scripts/bench-cli.ts --base --branch [--runs N] [--warmup N] [--command ""]... [--cwd-setup init] [--trace] [--json] [--out ] [--cpu-prof] [--update-check]\n' + - " --command splits on whitespace; it does not support shell quoting.\n" + + 'usage: bun scripts/bench-cli.ts --base --branch [--runs N] [--warmup N] [--command ""]... [--workloads ] [--api-url ] [--cwd-setup init] [--trace] [--json] [--out ] [--cpu-prof] [--update-check]\n' + + " --command splits on whitespace; it does not support shell quoting. Use --workloads for quoted arguments.\n" + + " --workloads reads a manifest of workloads and cannot be combined with --command or --cwd-setup.\n" + + " --api-url points every launch at a replay server through a profile file, and counts Management API calls from the traced launches; it requires --trace.\n" + " --update-check keeps the GitHub release check, which otherwise adds network time to every launch."; -export type Build = "base" | "branch"; - export interface BenchOptions { readonly base: string; readonly branch: string; readonly runs: number; readonly warmup: number; readonly commands: ReadonlyArray>; + readonly workloadsFile: string | undefined; readonly cwdSetupInit: boolean; + readonly apiUrl: string | undefined; readonly trace: boolean; readonly json: boolean; readonly out: string | undefined; @@ -34,10 +67,11 @@ export interface BenchOptions { readonly updateCheck: boolean; } -const ReportJson = Schema.fromJsonString(Schema.Unknown, { space: 2 }); - const DEFAULT_COMMANDS: ReadonlyArray> = [["--version"], ["--help"]]; +/** Dummy token for launches pointed at a replay server; it never reaches a real API. */ +const BENCH_ACCESS_TOKEN = `sbp_${"0".repeat(40)}`; + /** Whether `--help`/`-h` is passed as a flag rather than as a `--command` value. */ export function wantsHelp(argv: ReadonlyArray): boolean { return argv.some( @@ -45,7 +79,7 @@ export function wantsHelp(argv: ReadonlyArray): boolean { ); } -/** Parses argv into {@link BenchOptions}; throws on missing `--base`/`--branch` or bad numbers. */ +/** Parses argv into {@link BenchOptions}; throws on missing `--base`/`--branch` or bad values. */ export function parseBenchArgs(argv: ReadonlyArray): BenchOptions { // `parseArgs` rejects a separate value that starts with `-`, but CLI commands usually do. const args: Array = []; @@ -67,6 +101,8 @@ export function parseBenchArgs(argv: ReadonlyArray): BenchOptions { runs: { type: "string", default: "10" }, warmup: { type: "string", default: "2" }, command: { type: "string", multiple: true }, + workloads: { type: "string" }, + "api-url": { type: "string" }, "cwd-setup": { type: "string" }, trace: { type: "boolean", default: false }, json: { type: "boolean", default: false }, @@ -90,6 +126,15 @@ export function parseBenchArgs(argv: ReadonlyArray): BenchOptions { if (values["cwd-setup"] !== undefined && values["cwd-setup"] !== "init") { throw new Error('--cwd-setup only supports "init"'); } + const hasCommands = values.command !== undefined && values.command.length > 0; + if (values.workloads !== undefined && (hasCommands || values["cwd-setup"] !== undefined)) { + throw new Error("--workloads cannot be combined with --command or --cwd-setup"); + } + if (values["api-url"] !== undefined) { + if (!URL.canParse(values["api-url"])) throw new Error("--api-url must be a valid URL"); + if (!values.trace) + throw new Error("--api-url requires --trace, which supplies the call counts"); + } /* The host `Path` service (`node:path` itself), read synchronously for this sync parser. */ const hostPath = Effect.runSync( @@ -102,11 +147,12 @@ export function parseBenchArgs(argv: ReadonlyArray): BenchOptions { branch: hostPath.resolve(values.branch), runs, warmup, - commands: - values.command === undefined || values.command.length === 0 - ? DEFAULT_COMMANDS - : values.command.map((raw) => raw.split(/\s+/u).filter((part) => part.length > 0)), + commands: hasCommands + ? values.command!.map((raw) => raw.split(/\s+/u).filter((part) => part.length > 0)) + : DEFAULT_COMMANDS, + workloadsFile: values.workloads, cwdSetupInit: values["cwd-setup"] === "init", + apiUrl: values["api-url"], trace: values.trace ?? false, json: values.json ?? false, out: values.out, @@ -139,185 +185,68 @@ export interface PlannedRun { readonly iteration: number; } -/** Alternates base/branch launches per iteration; the first `warmup` iterations are discarded. */ +/** Launches base and branch once per iteration, swapping which goes first on every other measured iteration. */ export function buildRunPlan(runs: number, warmup: number): ReadonlyArray { const plan: Array = []; for (let iteration = 0; iteration < runs + warmup; iteration++) { const isWarmup = iteration < warmup; - plan.push({ build: "base", warmup: isWarmup, iteration }); - plan.push({ build: "branch", warmup: isWarmup, iteration }); + const order: ReadonlyArray = + Math.abs(iteration - warmup) % 2 === 0 ? ["base", "branch"] : ["branch", "base"]; + for (const build of order) plan.push({ build, warmup: isWarmup, iteration }); } return plan; } -const round = (value: number): number => Math.round(value * 100) / 100; - -export function median(values: ReadonlyArray): number { - if (values.length === 0) return 0; - const sorted = [...values].sort((a, b) => a - b); - const mid = Math.floor(sorted.length / 2); - return sorted.length % 2 === 0 ? (sorted[mid - 1]! + sorted[mid]!) / 2 : sorted[mid]!; -} - -export function percentile(values: ReadonlyArray, p: number): number { - if (values.length === 0) return 0; - const sorted = [...values].sort((a, b) => a - b); - const rank = Math.min(Math.max(Math.ceil((p / 100) * sorted.length) - 1, 0), sorted.length - 1); - return sorted[rank]!; -} - -export interface TimingSummary { - readonly median: number; - readonly min: number; - readonly max: number; - readonly p90: number; -} - -export function summarizeTimings(values: ReadonlyArray): TimingSummary { - const sorted = [...values].sort((a, b) => a - b); - return { - median: round(median(sorted)), - min: round(sorted[0] ?? 0), - max: round(sorted[sorted.length - 1] ?? 0), - p90: round(percentile(sorted, 90)), - }; -} - -export interface Delta { - readonly ms: number; - readonly pct: number; -} - -/** `branch - base`; `pct` is `0` when `baseMedian` is `0` rather than `Infinity`. */ -export function deltaOf(baseMedian: number, branchMedian: number): Delta { - const ms = branchMedian - baseMedian; - return { ms: round(ms), pct: baseMedian === 0 ? 0 : round((ms / baseMedian) * 100) }; -} - -export interface SpanRunSummary { - readonly totalMs: number; - readonly count: number; -} - -/** Sums duration and counts occurrences of each span name within one run's decoded spans. */ -export function summarizeSpanRun(spans: ReadonlyArray): Record { - const result: Record = {}; - for (const span of spans) { - const entry = result[span.name] ?? { totalMs: 0, count: 0 }; - result[span.name] = { - totalMs: entry.totalMs + (span.endMs - span.startMs), - count: entry.count + 1, - }; +export class WorkloadValidationError extends Schema.TaggedError()( + "WorkloadValidationError", + { problems: Schema.Array(Schema.String) }, +) { + override get message(): string { + return this.problems.join("\n"); } - return result; -} - -export interface SpanAggregate { - readonly medianMs: number; - readonly medianCount: number; -} - -/** Medians each span name's per-run total duration and count across runs; missing runs count as 0. */ -export function aggregateSpanRuns( - runs: ReadonlyArray>, -): Record { - const names = new Set(); - for (const run of runs) for (const name of Object.keys(run)) names.add(name); - const result: Record = {}; - for (const name of names) { - result[name] = { - medianMs: round(median(runs.map((run) => run[name]?.totalMs ?? 0))), - medianCount: round(median(runs.map((run) => run[name]?.count ?? 0))), - }; - } - return result; -} - -export interface SpanComparison { - readonly name: string; - readonly base: SpanAggregate | undefined; - readonly branch: SpanAggregate | undefined; - readonly deltaMs: number | undefined; - readonly onlyIn: Build | undefined; -} - -function spanSortKey(entry: SpanComparison): number { - return entry.deltaMs !== undefined - ? Math.abs(entry.deltaMs) - : (entry.branch?.medianMs ?? entry.base?.medianMs ?? 0); -} - -/** Pairs base/branch span aggregates by name, sorted by `|delta|` descending; flags one-sided spans. */ -export function compareSpans( - base: Record, - branch: Record, -): ReadonlyArray { - const names = new Set([...Object.keys(base), ...Object.keys(branch)]); - const comparisons = [...names].map((name): SpanComparison => { - const baseEntry = base[name]; - const branchEntry = branch[name]; - return { - name, - base: baseEntry, - branch: branchEntry, - deltaMs: - baseEntry !== undefined && branchEntry !== undefined - ? round(branchEntry.medianMs - baseEntry.medianMs) - : undefined, - onlyIn: baseEntry === undefined ? "branch" : branchEntry === undefined ? "base" : undefined, - }; - }); - return comparisons.sort((a, b) => spanSortKey(b) - spanSortKey(a)); -} - -/** Whether the builds exited differently, which makes their timings measure different work. */ -export function exitCodesDiffer( - base: ReadonlyArray, - branch: ReadonlyArray, -): boolean { - const codes = (values: ReadonlyArray) => - [...new Set(values)].sort((a, b) => a - b).join(","); - return codes(base) !== codes(branch); -} - -interface BuildTimingSummary extends TimingSummary { - readonly exitCodes: ReadonlyArray; -} - -export interface CommandReport { - readonly command: ReadonlyArray; - readonly base: BuildTimingSummary; - readonly branch: BuildTimingSummary; - readonly deltaMs: number; - readonly deltaPct: number; - readonly spans: ReadonlyArray | undefined; - readonly traceEmptyFor: ReadonlyArray; - readonly exitCodesDiffer: boolean; } -interface RawSample { - readonly command: ReadonlyArray; - readonly build: Build; - readonly wallMs: number; - readonly exitCode: number; - readonly spans: Record | undefined; -} +/** The workloads for a run: the manifest entries, or one per `--command` with `--cwd-setup`. */ +export const resolveWorkloads = Effect.fn("BenchCli.resolveWorkloads")(function* ( + options: BenchOptions, +) { + const workloads: ReadonlyArray = + options.workloadsFile === undefined + ? options.commands.map((args) => ({ + id: args.join(" ") || "(no args)", + args, + cwdSetup: options.cwdSetupInit ? ("init" as const) : ("none" as const), + })) + : yield* Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + return yield* parseWorkloadManifest(yield* fs.readFileString(options.workloadsFile!)); + }); + const problems = validateWorkloads(workloads, { apiUrl: options.apiUrl }); + if (problems.length > 0) return yield* new WorkloadValidationError({ problems }); + return workloads; +}); interface LaunchResult { readonly wallMs: number; readonly exitCode: number; } -const runLaunch = (params: { +/** Longest a single launch may run before it is killed and counted as a failed launch. */ +export const LAUNCH_TIMEOUT = Duration.seconds(60); +/** Exit code recorded for a launch that hit {@link LAUNCH_TIMEOUT}. */ +export const LAUNCH_TIMEOUT_EXIT_CODE = 124; + +export const runLaunch = Effect.fn("BenchCli.runLaunch")(function* (params: { readonly binary: string; readonly args: ReadonlyArray; readonly cwd: string; readonly env: Record; -}) => - Effect.gen(function* () { - const spawner = yield* ChildProcessSpawner.ChildProcessSpawner; - const start = yield* Clock.currentTimeNanos; - const exitCode = yield* spawner.exitCode( + readonly timeout?: Duration.Duration; +}) { + const spawner = yield* ChildProcessSpawner.ChildProcessSpawner; + const start = yield* Clock.currentTimeNanos; + const exit = yield* spawner + .exitCode( ChildProcess.make(params.binary, params.args, { cwd: params.cwd, env: params.env, @@ -326,33 +255,72 @@ const runLaunch = (params: { stdout: "ignore", stderr: "ignore", }), + ) + .pipe(Effect.timeoutOption(params.timeout ?? LAUNCH_TIMEOUT)); + const end = yield* Clock.currentTimeNanos; + if (Option.isNone(exit)) { + yield* Console.error( + `warning: ${params.binary} ${params.args.join(" ")} did not exit within ${Duration.format(params.timeout ?? LAUNCH_TIMEOUT)}; it was killed`, ); - const end = yield* Clock.currentTimeNanos; - return { - wallMs: Number(end - start) / 1_000_000, - exitCode: Number(exitCode), - } satisfies LaunchResult; - }); + } + return { + wallMs: Number(end - start) / 1_000_000, + exitCode: Option.isSome(exit) ? Number(exit.value) : LAUNCH_TIMEOUT_EXIT_CODE, + } satisfies LaunchResult; +}); + +const PROFILE_FILE = "bench-profile.yaml"; + +/** A profile naming `apiUrl` as the Management API; the profile is the CLI's only API redirect. */ +export function profileYaml(apiUrl: string): string { + return [ + "name: bench", + `api_url: "${apiUrl}"`, + `dashboard_url: "${apiUrl}"`, + "project_host: localhost", + "", + ].join("\n"); +} -const baseLaunchEnv = (supabaseHome: string, updateCheck: boolean): Record => ({ +const baseLaunchEnv = ( + options: BenchOptions, + supabaseHome: string, + profilePath: string, +): Record => ({ SUPABASE_HOME: supabaseHome, DO_NOT_TRACK: "1", SUPABASE_NO_KEYRING: "1", - ...(updateCheck ? {} : { SUPABASE_NO_UPDATE_NOTIFIER: "1" }), + ...(options.updateCheck ? {} : { SUPABASE_NO_UPDATE_NOTIFIER: "1" }), + ...(options.apiUrl === undefined + ? {} + : { + SUPABASE_PROFILE: profilePath, + SUPABASE_ACCESS_TOKEN: BENCH_ACCESS_TOKEN, + SUPABASE_TELEMETRY_DISABLED: "1", + }), }); -/** A fresh project directory and `SUPABASE_HOME`, initialized when `--cwd-setup init` is set. */ -const prepareLaunch = (options: BenchOptions, binary: string) => +/** + * A fresh project directory and `SUPABASE_HOME`, with the replay profile written and the + * workload's `cwdSetup` applied. + */ +const prepareLaunch = (options: BenchOptions, workload: Workload, binary: string) => Effect.gen(function* () { const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; const projectDir = yield* fs.makeTempDirectoryScoped({ prefix: "supabase-bench-cwd-" }); const supabaseHome = yield* fs.makeTempDirectoryScoped({ prefix: "supabase-bench-home-" }); - if (options.cwdSetupInit) { + const profilePath = path.join(supabaseHome, PROFILE_FILE); + if (options.apiUrl !== undefined) { + yield* fs.writeFileString(profilePath, profileYaml(options.apiUrl)); + } + const env = baseLaunchEnv(options, supabaseHome, profilePath); + if (workload.cwdSetup !== "none") { const init = yield* runLaunch({ binary, args: ["init"], cwd: projectDir, - env: isolatedEnv(process.env, baseLaunchEnv(supabaseHome, options.updateCheck)), + env: isolatedEnv(process.env, env), }); if (init.exitCode !== 0) { yield* Console.error( @@ -360,12 +328,12 @@ const prepareLaunch = (options: BenchOptions, binary: string) => ); } } - return { projectDir, supabaseHome }; + return { projectDir, env }; }); -const runCommandBench = ( +export const runWorkloadBench = ( options: BenchOptions, - command: ReadonlyArray, + workload: Workload, profiled: Ref.Ref, ) => Effect.gen(function* () { @@ -373,30 +341,30 @@ const runCommandBench = ( const path = yield* Path.Path; const plan = buildRunPlan(options.runs, options.warmup); - const samples: Record> = { base: [], branch: [] }; - const exitCodes: Record> = { base: [], branch: [] }; - const spanRuns: Record>> = { base: [], branch: [] }; - const raw: Array = []; + const samples: Record> = { base: [], branch: [] }; let cpuProfilePath: string | undefined; + // Resets the replay server after setup, so setup's own requests and a previous launch's + // consumed scenario never reach the launch that follows. + const resetServer = + options.apiUrl === undefined ? Effect.void : resetReplay(options.apiUrl, workload.scenario); + for (const planned of plan) { const binary = planned.build === "base" ? options.base : options.branch; yield* Effect.scoped( Effect.gen(function* () { - const { projectDir, supabaseHome } = yield* prepareLaunch(options, binary); + const prepared = yield* prepareLaunch(options, workload, binary); + yield* resetServer; const result = yield* runLaunch({ binary, - args: command, - cwd: projectDir, - env: isolatedEnv(process.env, baseLaunchEnv(supabaseHome, options.updateCheck)), + args: workload.args, + cwd: prepared.projectDir, + env: isolatedEnv(process.env, prepared.env), }); if (planned.warmup) return; - samples[planned.build].push(result.wallMs); - exitCodes[planned.build].push(result.exitCode); - // Tracing and profiling slow the launch they observe, so they run in an extra, untimed // launch in fresh directories. Both builds get traced launches; only one branch launch // is profiled. @@ -413,81 +381,66 @@ const runCommandBench = ( bunOptions = `--cpu-prof --cpu-prof-dir=${profileDir} --cpu-prof-name=${profileName}`; } } + + let trace: RunSample["trace"]; + let tracedExitCode: number | undefined; if (traceFile !== undefined || bunOptions !== undefined) { - const observed = yield* prepareLaunch(options, binary); - yield* runLaunch({ + const observed = yield* prepareLaunch(options, workload, binary); + yield* resetServer; + const traced = yield* runLaunch({ binary, - args: command, + args: workload.args, cwd: observed.projectDir, env: isolatedEnv(process.env, { - ...baseLaunchEnv(observed.supabaseHome, options.updateCheck), + ...observed.env, ...(traceFile === undefined ? {} : { SUPABASE_TRACE_FILE: traceFile }), ...(bunOptions === undefined ? {} : { BUN_OPTIONS: bunOptions }), }), }); + tracedExitCode = traced.exitCode; + if (traceFile !== undefined) { + const requestLogCount = + options.apiUrl === undefined + ? undefined + : yield* readRequestLogCount(options.apiUrl); + // Trace data is optional; a missing or unreadable file must not discard timings. + const decoded = yield* readSpans(traceFile).pipe(Effect.option); + if (Option.isSome(decoded) && decoded.value.spans.length > 0) { + const { spans, skippedLines } = decoded.value; + const http = + options.apiUrl === undefined + ? undefined + : classifyHttpSpans(spans, options.apiUrl); + trace = { + skippedLines, + spans: summarizeSpanRun(spans), + ...(http === undefined || requestLogCount === undefined + ? {} + : { api: { ...http, requestLogCount } }), + }; + } + } } - const spans = - traceFile === undefined - ? undefined - : summarizeSpanRun( - // Trace data is optional; a missing or unreadable file must not discard timings. - (yield* readSpans(traceFile).pipe( - Effect.orElseSucceed(() => ({ spans: [], skippedLines: 0 })), - )).spans, - ); - if (spans !== undefined) spanRuns[planned.build].push(spans); - - raw.push({ - command, - build: planned.build, + samples[planned.build].push({ wallMs: result.wallMs, exitCode: result.exitCode, - spans, + ...(tracedExitCode === undefined ? {} : { tracedExitCode }), + ...(trace === undefined ? {} : { trace }), }); }), ); } - const base = summarizeTimings(samples.base); - const branch = summarizeTimings(samples.branch); - const delta = deltaOf(base.median, branch.median); - - const spans = options.trace - ? compareSpans(aggregateSpanRuns(spanRuns.base), aggregateSpanRuns(spanRuns.branch)) - : undefined; - const traceEmptyFor: ReadonlyArray = options.trace - ? (["base", "branch"] as const).filter( - (build) => - spanRuns[build].length > 0 && - spanRuns[build].every((run) => Object.keys(run).length === 0), - ) - : []; - - const report: CommandReport = { - command, - base: { ...base, exitCodes: exitCodes.base }, - branch: { ...branch, exitCodes: exitCodes.branch }, - deltaMs: delta.ms, - deltaPct: delta.pct, - spans, - traceEmptyFor, - exitCodesDiffer: exitCodesDiffer(exitCodes.base, exitCodes.branch), + const report: WorkloadReport = { + id: workload.id, + args: workload.args, + base: samples.base, + branch: samples.branch, }; - return { report, raw, cpuProfilePath }; + return { report, cpuProfilePath }; }); -interface Report { - readonly base: string; - readonly branch: string; - readonly runs: number; - readonly warmup: number; - readonly trace: boolean; - readonly cwdSetup: "init" | undefined; - readonly commands: ReadonlyArray; - readonly cpuProfilePath: string | undefined; -} - function formatSpanLine(span: SpanComparison): string { if (span.onlyIn !== undefined) { const only = span.onlyIn === "base" ? span.base : span.branch; @@ -500,52 +453,57 @@ function formatSpanLine(span: SpanComparison): string { ); } -function formatCommandReport(commandReport: CommandReport): ReadonlyArray { +function formatWorkload(analysis: WorkloadAnalysis): ReadonlyArray { const sign = (value: number) => (value >= 0 ? "+" : ""); + const line = (label: string, build: WorkloadAnalysis["base"]) => + ` ${label} median ${build.timing.median}ms min ${build.timing.min}ms max ${build.timing.max}ms p90 ${build.timing.p90}ms exits [${build.exitCodes.join(",")}]`; const lines = [ - `command: ${commandReport.command.join(" ") || "(no args)"}`, - ` base: median ${commandReport.base.median}ms min ${commandReport.base.min}ms max ${commandReport.base.max}ms p90 ${commandReport.base.p90}ms exits [${commandReport.base.exitCodes.join(",")}]`, - ` branch: median ${commandReport.branch.median}ms min ${commandReport.branch.min}ms max ${commandReport.branch.max}ms p90 ${commandReport.branch.p90}ms exits [${commandReport.branch.exitCodes.join(",")}]`, - ` delta: ${sign(commandReport.deltaMs)}${commandReport.deltaMs}ms (${sign(commandReport.deltaPct)}${commandReport.deltaPct}%)`, + `command: ${analysis.args.join(" ") || "(no args)"} [${analysis.id}]`, + line("base: ", analysis.base), + line("branch:", analysis.branch), + ` delta: ${sign(analysis.deltaMs)}${analysis.deltaMs}ms (${sign(analysis.deltaPct)}${analysis.deltaPct}%) ${analysis.verdict}`, ]; - if (commandReport.exitCodesDiffer) { - lines.push(" warning: exit codes differ between builds, so the timings are not comparable"); - } - for (const build of ["base", "branch"] as const) { - const codes = commandReport[build].exitCodes; - const failed = codes.filter((code) => code !== 0).length; - if (failed > 0) { - lines.push( - ` warning: ${build} exited non-zero in ${failed}/${codes.length} runs, so its timings measure a failing invocation`, - ); + if (analysis.base.api !== undefined || analysis.branch.api !== undefined) { + lines.push( + ` api calls: base ${analysis.base.api?.medianCalls ?? "n/a"} branch ${analysis.branch.api?.medianCalls ?? "n/a"}`, + ); + for (const change of analysis.endpointChanges) { + lines.push(` ${change.endpoint} base ${change.base} branch ${change.branch}`); } } - if (commandReport.traceEmptyFor.length > 0) { - lines.push(` trace: no spans captured for ${commandReport.traceEmptyFor.join(", ")}`); - } - if (commandReport.spans !== undefined && commandReport.spans.length > 0) { + for (const flag of collectFlags(analysis)) lines.push(` warning: ${flag.detail}`); + if (analysis.spans !== undefined && analysis.spans.length > 0) { lines.push(" spans (delta ms, or one-sided median ms x count):"); - for (const span of commandReport.spans) lines.push(formatSpanLine(span)); + for (const span of analysis.spans) lines.push(formatSpanLine(span)); } lines.push(""); return lines; } -function formatReport(report: Report): string { +function formatReport(report: BenchReport): string { const lines = [ `base: ${report.base}`, `branch: ${report.branch}`, `runs: ${report.runs} warmup: ${report.warmup}` + - (report.cwdSetup === undefined ? "" : ` cwd-setup: ${report.cwdSetup}`), + (report.apiUrl === undefined ? "" : ` api-url: ${report.apiUrl}`), "", ]; - for (const commandReport of report.commands) lines.push(...formatCommandReport(commandReport)); + for (const workload of report.workloads) { + lines.push(...formatWorkload(analyzeWorkload(workload, report.trace))); + } if (report.cpuProfilePath !== undefined) { lines.push(`cpu profile (one untimed branch launch): ${report.cpuProfilePath}`); } return lines.join("\n"); } +const fileSize = (binary: string) => + FileSystem.FileSystem.use((fs) => fs.stat(binary)).pipe( + Effect.map((info) => Number(info.size)), + Effect.option, + Effect.map(Option.getOrUndefined), + ); + const main = Effect.gen(function* () { const stdio = yield* Stdio.Stdio; const argv = yield* stdio.args; @@ -570,42 +528,42 @@ const main = Effect.gen(function* () { ); if (options === undefined) return; + const workloads = yield* resolveWorkloads(options); const profiled = yield* Ref.make(false); - const commands: Array = []; - const samples: Array = []; + const reports: Array = []; let cpuProfilePath: string | undefined; - for (const command of options.commands) { - const result = yield* runCommandBench(options, command, profiled); - commands.push(result.report); - samples.push(...result.raw); + for (const workload of workloads) { + const result = yield* runWorkloadBench(options, workload, profiled); + reports.push(result.report); if (result.cpuProfilePath !== undefined) cpuProfilePath = result.cpuProfilePath; } - const report: Report = { + const baseSizeBytes = yield* fileSize(options.base); + const branchSizeBytes = yield* fileSize(options.branch); + const report: BenchReport = { base: options.base, branch: options.branch, + ...(baseSizeBytes === undefined ? {} : { baseSizeBytes }), + ...(branchSizeBytes === undefined ? {} : { branchSizeBytes }), runs: options.runs, warmup: options.warmup, trace: options.trace, - cwdSetup: options.cwdSetupInit ? "init" : undefined, - commands, - cpuProfilePath, + ...(options.apiUrl === undefined ? {} : { apiUrl: options.apiUrl }), + workloads: reports, + ...(cpuProfilePath === undefined ? {} : { cpuProfilePath }), }; if (options.out !== undefined) { const fs = yield* FileSystem.FileSystem; - yield* fs.writeFileString( - options.out, - yield* Schema.encodeEffect(ReportJson)({ report, samples }), - ); + yield* fs.writeFileString(options.out, yield* encodeBenchReport(report)); } - yield* Console.log( - options.json ? yield* Schema.encodeEffect(ReportJson)(report) : formatReport(report), - ); + yield* Console.log(options.json ? yield* encodeBenchReport(report) : formatReport(report)); }); if (import.meta.main) { - BunRuntime.runMain(main.pipe(Effect.provide(BunServices.layer))); + BunRuntime.runMain( + main.pipe(Effect.provide(Layer.mergeAll(FetchHttpClient.layer, BunServices.layer))), + ); } diff --git a/apps/cli/scripts/bench-cli.unit.test.ts b/apps/cli/scripts/bench-cli.unit.test.ts index d991afdc32..8906f3a380 100644 --- a/apps/cli/scripts/bench-cli.unit.test.ts +++ b/apps/cli/scripts/bench-cli.unit.test.ts @@ -1,37 +1,21 @@ -import { describe, expect, it } from "vitest"; +import { describe, expect, it } from "@effect/vitest"; +import { BunServices } from "@effect/platform-bun"; +import { Effect } from "effect"; import { - aggregateSpanRuns, buildRunPlan, - compareSpans, - deltaOf, - exitCodesDiffer, isolatedEnv, parseBenchArgs, + profileYaml, + resolveWorkloads, wantsHelp, - summarizeSpanRun, - summarizeTimings, } from "./bench-cli.ts"; -import type { SpanRunSummary } from "./bench-cli.ts"; -import type { ReportSpan } from "./trace-report.ts"; - -const span = (name: string, startMs: number, endMs: number): ReportSpan => ({ - traceId: "t", - spanId: `${name}-${startMs}`, - parentSpanId: undefined, - name, - startMs, - endMs, - failed: false, - errorType: undefined, -}); + +const required = ["--base", "/bin/a", "--branch", "/bin/b"]; describe("parseBenchArgs", () => { it("accepts commands that start with a dash and keeps update checks off by default", () => { const options = parseBenchArgs([ - "--base", - "/bin/a", - "--branch", - "/bin/b", + ...required, "--command", "--version", "--command", @@ -41,115 +25,96 @@ describe("parseBenchArgs", () => { expect(options.commands).toEqual([["--version"], ["db", "diff", "--local"]]); expect(options.updateCheck).toBe(false); }); + + it("rejects a manifest combined with --command or --cwd-setup", () => { + expect(() => + parseBenchArgs([...required, "--workloads", "w.json", "--command", "init"]), + ).toThrow("--workloads cannot be combined"); + expect(() => + parseBenchArgs([...required, "--workloads", "w.json", "--cwd-setup", "init"]), + ).toThrow("--workloads cannot be combined"); + }); + + it("requires --trace with --api-url because call counts come from the trace", () => { + expect(() => parseBenchArgs([...required, "--api-url", "http://127.0.0.1:1"])).toThrow( + "--api-url requires --trace", + ); + expect(parseBenchArgs([...required, "--api-url", "http://127.0.0.1:1", "--trace"]).apiUrl).toBe( + "http://127.0.0.1:1", + ); + }); +}); + +describe("resolveWorkloads", () => { + it.effect.each([ + { cwdSetup: ["--cwd-setup", "init"], expected: "init" }, + { cwdSetup: [], expected: "none" }, + ])("turns --command entries into workloads with cwdSetup $expected", ({ cwdSetup, expected }) => + Effect.gen(function* () { + const options = parseBenchArgs([ + ...required, + "--command", + "migration new bench", + ...cwdSetup, + ]); + + const workloads = yield* resolveWorkloads(options); + + expect(workloads).toEqual([ + { id: "migration new bench", args: ["migration", "new", "bench"], cwdSetup: expected }, + ]); + }).pipe(Effect.provide(BunServices.layer)), + ); +}); + +describe("resolveWorkloads validation", () => { + it.effect("fails with a WorkloadValidationError carrying the problems", () => + Effect.gen(function* () { + const options = parseBenchArgs([ + ...required, + "--command", + "migration new bench", + "--command", + "migration new bench", + ]); + + const error = yield* resolveWorkloads(options).pipe(Effect.flip); + + const problems = error._tag === "WorkloadValidationError" ? error.problems : undefined; + expect(problems).toEqual(['duplicate workload id "migration new bench"']); + }).pipe(Effect.provide(BunServices.layer)), + ); }); describe("wantsHelp", () => { it("treats --help as a flag only when it is not a --command value", () => { expect(wantsHelp(["--help"])).toBe(true); - expect(wantsHelp(["--base", "/a", "--branch", "/b", "--command", "--help"])).toBe(false); - }); -}); - -describe("summarizeTimings", () => { - it("computes median, min, max, and p90", () => { - expect(summarizeTimings([10, 20, 30, 40, 50])).toEqual({ - median: 30, - min: 10, - max: 50, - p90: 50, - }); + expect(wantsHelp([...required, "--command", "--help"])).toBe(false); }); }); describe("buildRunPlan", () => { - it("alternates base and branch and discards the first warmup iterations", () => { + it("swaps the launch order every other iteration and discards the first warmup iterations", () => { const plan = buildRunPlan(2, 1); expect(plan.map((run) => [run.build, run.warmup])).toEqual([ - ["base", true], ["branch", true], + ["base", true], ["base", false], ["branch", false], - ["base", false], ["branch", false], + ["base", false], ]); }); }); -describe("exitCodesDiffer", () => { - it("flags builds that exit differently but not shared failures", () => { - expect(exitCodesDiffer([1, 1], [1, 1])).toBe(false); - expect(exitCodesDiffer([0, 0], [1, 0])).toBe(true); - }); -}); - -describe("deltaOf", () => { - it("reports branch minus base in ms and percent", () => { - expect(deltaOf(100, 110)).toEqual({ ms: 10, pct: 10 }); - }); - - it("reports a zero percent delta instead of Infinity when base is zero", () => { - expect(deltaOf(0, 5)).toEqual({ ms: 5, pct: 0 }); - }); -}); - -describe("span aggregation", () => { - it("sums duration and counts per span name within one run", () => { - const run = [span("Config.load", 0, 10), span("Db.query", 10, 20), span("Db.query", 20, 50)]; - - expect(summarizeSpanRun(run)).toEqual({ - "Config.load": { totalMs: 10, count: 1 }, - "Db.query": { totalMs: 40, count: 2 }, - }); - }); +describe("profileYaml", () => { + it("points the API and dashboard at the replay server and keeps project hosts local", () => { + const yaml = profileYaml("http://127.0.0.1:9"); - it("medians per-span totals and counts across runs, treating an absent span as zero", () => { - const runs: ReadonlyArray> = [ - { "Db.query": { totalMs: 40, count: 2 } }, - { "Db.query": { totalMs: 60, count: 2 } }, - {}, - ]; - - expect(aggregateSpanRuns(runs)).toEqual({ - "Db.query": { medianMs: 40, medianCount: 2 }, - }); - }); -}); - -describe("compareSpans", () => { - it("flags spans found in only one build and sorts by absolute delta", () => { - const base = { - "Config.load": { medianMs: 10, medianCount: 1 }, - "Credentials.load": { medianMs: 5, medianCount: 1 }, - }; - const branch = { - "Config.load": { medianMs: 40, medianCount: 1 }, - "Otlp.flush": { medianMs: 3, medianCount: 1 }, - }; - - expect(compareSpans(base, branch)).toEqual([ - { - name: "Config.load", - base: { medianMs: 10, medianCount: 1 }, - branch: { medianMs: 40, medianCount: 1 }, - deltaMs: 30, - onlyIn: undefined, - }, - { - name: "Credentials.load", - base: { medianMs: 5, medianCount: 1 }, - branch: undefined, - deltaMs: undefined, - onlyIn: "base", - }, - { - name: "Otlp.flush", - base: undefined, - branch: { medianMs: 3, medianCount: 1 }, - deltaMs: undefined, - onlyIn: "branch", - }, - ]); + expect(yaml).toContain('api_url: "http://127.0.0.1:9"'); + expect(yaml).toContain('dashboard_url: "http://127.0.0.1:9"'); + expect(yaml).toContain("project_host: localhost"); }); }); @@ -159,6 +124,7 @@ describe("isolatedEnv", () => { HOME: "/home/person", PATH: "/usr/bin", SUPABASE_ACCESS_TOKEN: "secret", + SUPABASE_API_URL: "https://elsewhere", OTEL_EXPORTER_OTLP_ENDPOINT: "http://collector", TRACEPARENT: "00-...-01", }; diff --git a/apps/cli/scripts/bench-report.ts b/apps/cli/scripts/bench-report.ts new file mode 100644 index 0000000000..8e48662b76 --- /dev/null +++ b/apps/cli/scripts/bench-report.ts @@ -0,0 +1,523 @@ +/** + * Report schemas and the pure analysis shared by `bench-cli.ts` (which records raw per-run data) + * and `bench-summary.ts` (which derives flags and verdicts), so thresholds can change without + * re-running a benchmark. + */ +import { Effect, Schema } from "effect"; +import type { ReportSpan } from "./trace-report.ts"; + +export type Build = "base" | "branch"; + +const round = (value: number): number => Math.round(value * 100) / 100; + +export function median(values: ReadonlyArray): number { + if (values.length === 0) return 0; + const sorted = [...values].sort((a, b) => a - b); + const mid = Math.floor(sorted.length / 2); + return sorted.length % 2 === 0 ? (sorted[mid - 1]! + sorted[mid]!) / 2 : sorted[mid]!; +} + +export function percentile(values: ReadonlyArray, p: number): number { + if (values.length === 0) return 0; + const sorted = [...values].sort((a, b) => a - b); + const rank = Math.min(Math.max(Math.ceil((p / 100) * sorted.length) - 1, 0), sorted.length - 1); + return sorted[rank]!; +} + +export interface TimingSummary { + readonly median: number; + readonly min: number; + readonly max: number; + readonly p90: number; +} + +export function summarizeTimings(values: ReadonlyArray): TimingSummary { + const sorted = [...values].sort((a, b) => a - b); + return { + median: round(median(sorted)), + min: round(sorted[0] ?? 0), + max: round(sorted[sorted.length - 1] ?? 0), + p90: round(percentile(sorted, 90)), + }; +} + +export interface Delta { + readonly ms: number; + readonly pct: number; +} + +/** `branch - base`; `pct` is `0` when `baseMedian` is `0` rather than `Infinity`. */ +export function deltaOf(baseMedian: number, branchMedian: number): Delta { + const ms = branchMedian - baseMedian; + return { ms: round(ms), pct: baseMedian === 0 ? 0 : round((ms / baseMedian) * 100) }; +} + +const SpanRunSummary = Schema.Struct({ totalMs: Schema.Finite, count: Schema.Finite }); +export type SpanRunSummary = typeof SpanRunSummary.Type; + +/** + * Sums duration and counts occurrences of each span name within one run's decoded spans. HTTP + * client spans are left out: the API-call section reports them per endpoint. + */ +export function summarizeSpanRun(spans: ReadonlyArray): Record { + const result: Record = {}; + for (const span of spans) { + if (span.http !== undefined) continue; + const entry = result[span.name] ?? { totalMs: 0, count: 0 }; + result[span.name] = { + totalMs: entry.totalMs + (span.endMs - span.startMs), + count: entry.count + 1, + }; + } + return result; +} + +export interface SpanAggregate { + readonly medianMs: number; + readonly medianCount: number; +} + +/** Medians each span name's per-run total duration and count across runs; missing runs count as 0. */ +export function aggregateSpanRuns( + runs: ReadonlyArray>, +): Record { + const names = new Set(); + for (const run of runs) for (const name of Object.keys(run)) names.add(name); + const result: Record = {}; + for (const name of names) { + result[name] = { + medianMs: round(median(runs.map((run) => run[name]?.totalMs ?? 0))), + medianCount: round(median(runs.map((run) => run[name]?.count ?? 0))), + }; + } + return result; +} + +export interface SpanComparison { + readonly name: string; + readonly base: SpanAggregate | undefined; + readonly branch: SpanAggregate | undefined; + readonly deltaMs: number | undefined; + readonly onlyIn: Build | undefined; +} + +function spanSortKey(entry: SpanComparison): number { + return entry.deltaMs !== undefined + ? Math.abs(entry.deltaMs) + : (entry.branch?.medianMs ?? entry.base?.medianMs ?? 0); +} + +/** Pairs base/branch span aggregates by name, sorted by `|delta|` descending; flags one-sided spans. */ +export function compareSpans( + base: Record, + branch: Record, +): ReadonlyArray { + const names = new Set([...Object.keys(base), ...Object.keys(branch)]); + const comparisons = [...names].map((name): SpanComparison => { + const baseEntry = base[name]; + const branchEntry = branch[name]; + return { + name, + base: baseEntry, + branch: branchEntry, + deltaMs: + baseEntry !== undefined && branchEntry !== undefined + ? round(branchEntry.medianMs - baseEntry.medianMs) + : undefined, + onlyIn: baseEntry === undefined ? "branch" : branchEntry === undefined ? "base" : undefined, + }; + }); + return comparisons.sort((a, b) => spanSortKey(b) - spanSortKey(a)); +} + +/** Whether the builds exited differently, which makes their timings measure different work. */ +export function exitCodesDiffer( + base: ReadonlyArray, + branch: ReadonlyArray, +): boolean { + const codes = (values: ReadonlyArray) => + [...new Set(values)].sort((a, b) => a - b).join(","); + return codes(base) !== codes(branch); +} + +/* Workload manifest */ + +const Workload = Schema.Struct({ + id: Schema.NonEmptyString, + args: Schema.Array(Schema.String), + cwdSetup: Schema.Literals(["none", "init"]), + scenario: Schema.optionalKey(Schema.String), +}); +export type Workload = typeof Workload.Type; + +const WorkloadManifest = Schema.Struct({ workloads: Schema.NonEmptyArray(Workload) }); + +const decodeManifest = Schema.decodeUnknownEffect(Schema.fromJsonString(WorkloadManifest)); + +/** Rules the schema cannot express; returns one message per violation. */ +export function validateWorkloads( + workloads: ReadonlyArray, + context: { readonly apiUrl: string | undefined }, +): ReadonlyArray { + const problems: Array = []; + const seen = new Set(); + for (const workload of workloads) { + if (seen.has(workload.id)) problems.push(`duplicate workload id "${workload.id}"`); + seen.add(workload.id); + if (workload.scenario !== undefined && context.apiUrl === undefined) { + problems.push(`workload "${workload.id}" names a scenario and needs --api-url`); + } + } + return problems; +} + +/** Decodes the text of a workload manifest file. */ +export const parseWorkloadManifest = (text: string) => + decodeManifest(text).pipe(Effect.map((manifest) => manifest.workloads)); + +/* HTTP call classification */ + +export interface HttpCallSummary { + readonly apiCalls: number; + readonly externalCalls: number; + readonly errorStatuses: number; + readonly endpoints: Record; +} + +/** + * Splits a run's HTTP client spans into calls to `apiUrl` (same hostname and port) and calls to + * any other host. `errorStatuses` counts API calls that got a 4xx or 5xx response. + */ +export function classifyHttpSpans( + spans: ReadonlyArray, + apiUrl: string, +): HttpCallSummary { + const url = new URL(apiUrl); + const apiHost = url.hostname.replace(/^\[|\]$/gu, ""); + const apiPort = url.port === "" ? (url.protocol === "https:" ? 443 : 80) : Number(url.port); + let apiCalls = 0; + let externalCalls = 0; + let errorStatuses = 0; + const endpoints: Record = {}; + for (const span of spans) { + const http = span.http; + if (http === undefined) continue; + if (http.address !== apiHost || (http.port !== undefined && http.port !== apiPort)) { + externalCalls += 1; + continue; + } + apiCalls += 1; + if (http.status !== undefined && http.status >= 400) errorStatuses += 1; + const endpoint = `${http.method} ${http.path ?? "(unknown path)"}`; + endpoints[endpoint] = (endpoints[endpoint] ?? 0) + 1; + } + return { apiCalls, externalCalls, errorStatuses, endpoints }; +} + +/* Report (raw, as written by `bench-cli.ts --out`) */ + +const ApiSample = Schema.Struct({ + apiCalls: Schema.Finite, + externalCalls: Schema.Finite, + errorStatuses: Schema.Finite, + endpoints: Schema.Record(Schema.String, Schema.Finite), + requestLogCount: Schema.Finite, +}); + +const TraceSample = Schema.Struct({ + skippedLines: Schema.Finite, + spans: Schema.Record(Schema.String, SpanRunSummary), + api: Schema.optionalKey(ApiSample), +}); + +const RunSample = Schema.Struct({ + wallMs: Schema.Finite, + exitCode: Schema.Finite, + /** Exit code of the traced launch, recorded even when no spans could be decoded. */ + tracedExitCode: Schema.optionalKey(Schema.Finite), + trace: Schema.optionalKey(TraceSample), +}); +export type RunSample = typeof RunSample.Type; + +const WorkloadReport = Schema.Struct({ + id: Schema.String, + args: Schema.Array(Schema.String), + base: Schema.Array(RunSample), + branch: Schema.Array(RunSample), +}); +export type WorkloadReport = typeof WorkloadReport.Type; + +export const BenchReport = Schema.Struct({ + base: Schema.String, + branch: Schema.String, + baseSizeBytes: Schema.optionalKey(Schema.Finite), + branchSizeBytes: Schema.optionalKey(Schema.Finite), + runs: Schema.Finite, + warmup: Schema.Finite, + trace: Schema.Boolean, + apiUrl: Schema.optionalKey(Schema.String), + workloads: Schema.Array(WorkloadReport), + cpuProfilePath: Schema.optionalKey(Schema.String), +}); +export type BenchReport = typeof BenchReport.Type; + +const BenchReportJson = Schema.fromJsonString(BenchReport, { space: 2 }); +export const decodeBenchReport = Schema.decodeUnknownEffect(BenchReportJson); +export const encodeBenchReport = Schema.encodeEffect(BenchReportJson); + +/* Analysis */ + +export const VERDICT_MIN_PCT = 5; +export const VERDICT_MIN_MS = 5; +export const VERDICT_MIN_SAMPLES = 5; +export const PHASE_ROWS = 3; + +export type TimingVerdict = "slower" | "faster" | "within noise" | "not comparable"; + +/** + * "slower" or "faster" only when the branch median is outside the base min-max range and the + * change is at least {@link VERDICT_MIN_PCT} percent and {@link VERDICT_MIN_MS} ms. + */ +export function timingVerdict( + base: TimingSummary, + branch: TimingSummary, + context: { readonly comparable: boolean; readonly samples: number }, +): TimingVerdict { + if (!context.comparable || context.samples < VERDICT_MIN_SAMPLES) return "not comparable"; + const delta = deltaOf(base.median, branch.median); + if (Math.abs(delta.pct) < VERDICT_MIN_PCT || Math.abs(delta.ms) < VERDICT_MIN_MS) { + return "within noise"; + } + if (branch.median > base.max) return "slower"; + if (branch.median < base.min) return "faster"; + return "within noise"; +} + +export interface ApiSummary { + readonly medianCalls: number; + readonly runCalls: ReadonlyArray; + readonly nonDeterministic: boolean; + /** Median per-run count for each `METHOD path`; a run without the endpoint counts as 0. */ + readonly endpoints: Record; + readonly externalCalls: number; + /** Runs whose span-derived API call count differs from the replay server's request log. */ + readonly logMismatchRuns: number; + readonly errorStatuses: number; +} + +export interface BuildAnalysis { + readonly timing: TimingSummary; + readonly samples: number; + readonly exitCodes: ReadonlyArray; + readonly tracedExitCodes: ReadonlyArray; + readonly spans: Record | undefined; + readonly api: ApiSummary | undefined; + /** Measured runs without a usable trace when tracing was requested. */ + readonly traceMissingRuns: number; + readonly skippedTraceLines: number; +} + +function summarizeApi(samples: ReadonlyArray): ApiSummary | undefined { + const apis = samples.flatMap((sample) => + sample.trace?.api === undefined ? [] : [sample.trace.api], + ); + if (apis.length === 0) return undefined; + const runCalls = apis.map((api) => api.apiCalls); + const keys = new Set(apis.flatMap((api) => Object.keys(api.endpoints))); + const endpoints: Record = {}; + for (const key of keys) endpoints[key] = median(apis.map((api) => api.endpoints[key] ?? 0)); + const shape = (api: (typeof apis)[number]) => + JSON.stringify(Object.entries(api.endpoints).sort(([a], [b]) => a.localeCompare(b))); + return { + medianCalls: median(runCalls), + runCalls, + nonDeterministic: new Set(apis.map(shape)).size > 1, + endpoints, + externalCalls: Math.max(...apis.map((api) => api.externalCalls)), + logMismatchRuns: apis.filter((api) => api.apiCalls !== api.requestLogCount).length, + errorStatuses: Math.max(...apis.map((api) => api.errorStatuses)), + }; +} + +function summarizeBuild(samples: ReadonlyArray, traced: boolean): BuildAnalysis { + const withTrace = samples.flatMap((sample) => (sample.trace === undefined ? [] : [sample.trace])); + return { + timing: summarizeTimings(samples.map((sample) => sample.wallMs)), + samples: samples.length, + exitCodes: samples.map((sample) => sample.exitCode), + tracedExitCodes: samples.flatMap((sample) => + sample.tracedExitCode === undefined ? [] : [sample.tracedExitCode], + ), + spans: traced ? aggregateSpanRuns(withTrace.map((trace) => trace.spans)) : undefined, + api: summarizeApi(samples), + traceMissingRuns: traced ? samples.length - withTrace.length : 0, + skippedTraceLines: withTrace.reduce((sum, trace) => sum + trace.skippedLines, 0), + }; +} + +export interface EndpointChange { + readonly endpoint: string; + readonly base: number; + readonly branch: number; +} + +export interface WorkloadAnalysis { + readonly id: string; + readonly args: ReadonlyArray; + readonly base: BuildAnalysis; + readonly branch: BuildAnalysis; + readonly deltaMs: number; + readonly deltaPct: number; + readonly verdict: TimingVerdict; + readonly exitCodesDiffer: boolean; + readonly spans: ReadonlyArray | undefined; + /** Top phase deltas; empty unless the verdict is "slower" or "faster". */ + readonly phaseRows: ReadonlyArray; + readonly endpointChanges: ReadonlyArray; +} + +/** Whether every launch of the workload exited 0 and every API call got a non-error response. */ +function runsCleanly(build: BuildAnalysis): boolean { + return ( + [...build.exitCodes, ...build.tracedExitCodes].every((code) => code === 0) && + (build.api?.errorStatuses ?? 0) === 0 + ); +} + +function endpointChanges( + base: BuildAnalysis, + branch: BuildAnalysis, +): ReadonlyArray { + if (base.api === undefined || branch.api === undefined) return []; + const baseEndpoints = base.api.endpoints; + const branchEndpoints = branch.api.endpoints; + const keys = new Set([...Object.keys(baseEndpoints), ...Object.keys(branchEndpoints)]); + return [...keys] + .sort() + .map((endpoint) => ({ + endpoint, + base: baseEndpoints[endpoint] ?? 0, + branch: branchEndpoints[endpoint] ?? 0, + })) + .filter((change) => change.base !== change.branch); +} + +/** Derives medians, comparisons and the timing verdict for one workload from its raw samples. */ +export function analyzeWorkload(workload: WorkloadReport, traced: boolean): WorkloadAnalysis { + const base = summarizeBuild(workload.base, traced); + const branch = summarizeBuild(workload.branch, traced); + const delta = deltaOf(base.timing.median, branch.timing.median); + const differ = exitCodesDiffer(base.exitCodes, branch.exitCodes); + const verdict = timingVerdict(base.timing, branch.timing, { + comparable: !differ && runsCleanly(base) && runsCleanly(branch), + samples: Math.min(base.samples, branch.samples), + }); + const spans = + base.spans === undefined || branch.spans === undefined + ? undefined + : compareSpans(base.spans, branch.spans); + return { + id: workload.id, + args: workload.args, + base, + branch, + deltaMs: delta.ms, + deltaPct: delta.pct, + verdict, + exitCodesDiffer: differ, + spans, + phaseRows: + verdict === "slower" || verdict === "faster" ? (spans ?? []).slice(0, PHASE_ROWS) : [], + endpointChanges: endpointChanges(base, branch), + }; +} + +export type FlagKind = + | "new-api-calls" + | "api-calls-dropped" + | "external-calls" + | "non-deterministic" + | "request-log-mismatch" + | "error-status" + | "trace-incomplete" + | "non-zero-exit" + | "exit-codes-differ" + | "traced-exit-differs" + | "insufficient-samples"; + +export interface Flag { + readonly workload: string; + readonly kind: FlagKind; + readonly detail: string; + /** The build the flag is about, absent when it concerns the comparison. */ + readonly build?: Build; +} + +/** Everything about a workload that makes its comparison suspect or notable beyond timing. */ +export function collectFlags(analysis: WorkloadAnalysis): ReadonlyArray { + const flags: Array = []; + const add = (kind: FlagKind, detail: string, build?: Build) => + flags.push({ workload: analysis.id, kind, detail, ...(build === undefined ? {} : { build }) }); + const baseCalls = analysis.base.api?.medianCalls; + const branchCalls = analysis.branch.api?.medianCalls; + if (baseCalls === 0 && branchCalls !== undefined && branchCalls > 0) { + add("new-api-calls", `base makes 0 API calls, branch makes ${branchCalls}`); + } + if (baseCalls !== undefined && baseCalls > 0 && branchCalls === 0) { + add("api-calls-dropped", `base makes ${baseCalls} API calls, branch makes 0`); + } + if (Math.min(analysis.base.samples, analysis.branch.samples) < VERDICT_MIN_SAMPLES) { + add( + "insufficient-samples", + `fewer than ${VERDICT_MIN_SAMPLES} measured runs, so no timing verdict`, + ); + } + if (analysis.exitCodesDiffer) { + add("exit-codes-differ", "exit codes differ between builds, so timings are not comparable"); + } + for (const build of ["base", "branch"] as const) { + const summary = analysis[build]; + const failed = summary.exitCodes.filter((code) => code !== 0).length; + if (failed > 0) { + add("non-zero-exit", `${build} exited non-zero in ${failed}/${summary.samples} runs`, build); + } + const timed = new Set(summary.exitCodes); + if (summary.tracedExitCodes.some((code) => !timed.has(code))) { + add( + "traced-exit-differs", + `${build} traced launch exited differently from its timed launches`, + build, + ); + } + if (summary.traceMissingRuns > 0 || summary.skippedTraceLines > 0) { + add( + "trace-incomplete", + `${build} has ${summary.traceMissingRuns} runs without a trace and ${summary.skippedTraceLines} undecodable trace lines`, + build, + ); + } + const api = summary.api; + if (api === undefined) continue; + if (api.externalCalls > 0) { + add("external-calls", `${build} made calls to hosts other than the replay server`, build); + } + if (api.nonDeterministic) { + add("non-deterministic", `${build} API calls differ between runs`, build); + } + if (api.logMismatchRuns > 0) { + add( + "request-log-mismatch", + `${build} spans disagree with the replay request log in ${api.logMismatchRuns} runs`, + build, + ); + } + if (api.errorStatuses > 0) { + add( + "error-status", + `${build} got 4xx/5xx replay responses, so it did not follow the recorded scenario`, + build, + ); + } + } + return flags; +} diff --git a/apps/cli/scripts/bench-report.unit.test.ts b/apps/cli/scripts/bench-report.unit.test.ts new file mode 100644 index 0000000000..9f41a41d98 --- /dev/null +++ b/apps/cli/scripts/bench-report.unit.test.ts @@ -0,0 +1,474 @@ +import { describe, expect, it } from "@effect/vitest"; +import { Effect } from "effect"; +import { + aggregateSpanRuns, + analyzeWorkload, + classifyHttpSpans, + collectFlags, + decodeBenchReport, + encodeBenchReport, + compareSpans, + deltaOf, + exitCodesDiffer, + parseWorkloadManifest, + summarizeSpanRun, + summarizeTimings, + timingVerdict, + validateWorkloads, + type BenchReport, + type RunSample, + type SpanRunSummary, + type WorkloadReport, +} from "./bench-report.ts"; +import type { ReportSpan } from "./trace-report.ts"; + +const span = (name: string, startMs: number, endMs: number): ReportSpan => ({ + traceId: "t", + spanId: `${name}-${startMs}`, + parentSpanId: undefined, + name, + startMs, + endMs, + failed: false, + errorType: undefined, +}); + +const httpSpan = ( + method: string, + path: string, + port: number, + status = 200, + address = "127.0.0.1", +): ReportSpan => ({ + ...span(method, 0, 5), + http: { method, path, address, port, status }, +}); + +describe("summarizeTimings", () => { + it("computes median, min, max, and p90", () => { + expect(summarizeTimings([10, 20, 30, 40, 50])).toEqual({ + median: 30, + min: 10, + max: 50, + p90: 50, + }); + }); +}); + +describe("exitCodesDiffer", () => { + it("flags builds that exit differently but not shared failures", () => { + expect(exitCodesDiffer([1, 1], [1, 1])).toBe(false); + expect(exitCodesDiffer([0, 0], [1, 0])).toBe(true); + }); +}); + +describe("deltaOf", () => { + it("reports branch minus base in ms and percent", () => { + expect(deltaOf(100, 110)).toEqual({ ms: 10, pct: 10 }); + }); + + it("reports a zero percent delta instead of Infinity when base is zero", () => { + expect(deltaOf(0, 5)).toEqual({ ms: 5, pct: 0 }); + }); +}); + +describe("span aggregation", () => { + it("sums duration and counts per span name within one run, leaving HTTP spans to the API section", () => { + const run = [ + span("Config.load", 0, 10), + span("Db.query", 10, 20), + span("Db.query", 20, 50), + httpSpan("GET", "/v1/projects", 1), + ]; + + expect(summarizeSpanRun(run)).toEqual({ + "Config.load": { totalMs: 10, count: 1 }, + "Db.query": { totalMs: 40, count: 2 }, + }); + }); + + it("medians per-span totals and counts across runs, treating an absent span as zero", () => { + const runs: ReadonlyArray> = [ + { "Db.query": { totalMs: 40, count: 2 } }, + { "Db.query": { totalMs: 60, count: 2 } }, + {}, + ]; + + expect(aggregateSpanRuns(runs)).toEqual({ "Db.query": { medianMs: 40, medianCount: 2 } }); + }); +}); + +describe("compareSpans", () => { + it("flags spans found in only one build and sorts by absolute delta", () => { + const base = { + "Config.load": { medianMs: 10, medianCount: 1 }, + "Credentials.load": { medianMs: 5, medianCount: 1 }, + }; + const branch = { + "Config.load": { medianMs: 40, medianCount: 1 }, + "Otlp.flush": { medianMs: 3, medianCount: 1 }, + }; + + expect( + compareSpans(base, branch).map((entry) => [entry.name, entry.deltaMs, entry.onlyIn]), + ).toEqual([ + ["Config.load", 30, undefined], + ["Credentials.load", undefined, "base"], + ["Otlp.flush", undefined, "branch"], + ]); + }); +}); + +describe("workload manifest", () => { + const entry = { id: "w", args: ["init"], cwdSetup: "none" } as const; + + it.effect("decodes entries and rejects an unknown cwdSetup", () => + Effect.gen(function* () { + const parsed = yield* parseWorkloadManifest( + '{"workloads":[{"id":"w","args":["init"],"cwdSetup":"none"}]}', + ); + expect(parsed).toEqual([entry]); + + const exit = yield* Effect.exit( + parseWorkloadManifest('{"workloads":[{"id":"w","args":["init"],"cwdSetup":"bogus"}]}'), + ); + expect(exit._tag).toBe("Failure"); + }), + ); + + it("requires --api-url for scenarios and unique ids", () => { + const problems = validateWorkloads( + [ + { id: "b", args: [], cwdSetup: "none", scenario: "s" }, + { id: "b", args: [], cwdSetup: "none" }, + ], + { apiUrl: undefined }, + ); + + expect(problems).toEqual([ + 'workload "b" names a scenario and needs --api-url', + 'duplicate workload id "b"', + ]); + expect( + validateWorkloads([{ ...entry, scenario: "s" }], { apiUrl: "http://127.0.0.1:1" }), + ).toEqual([]); + }); +}); + +describe("classifyHttpSpans", () => { + const apiUrl = "http://127.0.0.1:4010"; + + it("counts calls to the API host and port per endpoint and separates other hosts and ports", () => { + const summary = classifyHttpSpans( + [ + httpSpan("GET", "/v1/projects", 4010), + httpSpan("GET", "/v1/projects", 4010), + httpSpan("POST", "/v1/x", 4010, 502), + httpSpan("GET", "/v1/projects", 4011), + httpSpan("GET", "/", 443, 200, "api.supabase.com"), + span("Config.load", 0, 1), + ], + apiUrl, + ); + + expect(summary).toEqual({ + apiCalls: 3, + externalCalls: 2, + errorStatuses: 1, + endpoints: { "GET /v1/projects": 2, "POST /v1/x": 1 }, + }); + }); +}); + +describe("timingVerdict", () => { + const base = { median: 100, min: 95, max: 108, p90: 107 }; + const context = { comparable: true, samples: 10 }; + + it.each([ + { branch: { median: 120, min: 115, max: 125, p90: 124 }, expected: "slower" }, + { branch: { median: 80, min: 78, max: 90, p90: 88 }, expected: "faster" }, + { branch: { median: 103, min: 99, max: 110, p90: 109 }, expected: "within noise" }, + { branch: { median: 107, min: 99, max: 118, p90: 117 }, expected: "within noise" }, + ])("calls a branch median of $branch.median $expected", ({ branch, expected }) => { + expect(timingVerdict(base, branch, context)).toBe(expected); + }); + + it("ignores a large percentage change that is under the absolute floor", () => { + expect( + timingVerdict( + { median: 10, min: 9, max: 11, p90: 11 }, + { median: 14, min: 13, max: 15, p90: 15 }, + context, + ), + ).toBe("within noise"); + }); + + it("declines to compare when runs are not comparable or too few", () => { + const slow = { median: 200, min: 190, max: 210, p90: 209 }; + expect(timingVerdict(base, slow, { comparable: false, samples: 10 })).toBe("not comparable"); + expect(timingVerdict(base, slow, { comparable: true, samples: 3 })).toBe("not comparable"); + }); +}); + +const sample = ( + wallMs: number, + api?: { + calls: number; + endpoints?: Record; + log?: number; + external?: number; + errors?: number; + }, + exitCode = 0, +): RunSample => ({ + wallMs, + exitCode, + tracedExitCode: exitCode, + trace: { + skippedLines: 0, + spans: { "cli.run": { totalMs: wallMs / 2, count: 1 } }, + ...(api === undefined + ? {} + : { + api: { + apiCalls: api.calls, + externalCalls: api.external ?? 0, + errorStatuses: api.errors ?? 0, + endpoints: api.endpoints ?? {}, + requestLogCount: api.log ?? api.calls, + }, + }), + }, +}); + +const workload = ( + base: ReadonlyArray, + branch: ReadonlyArray, +): WorkloadReport => ({ + id: "w", + args: ["migration", "new"], + base, + branch, +}); + +const times = (n: number, make: (i: number) => RunSample) => + Array.from({ length: n }, (_, i) => make(i)); + +describe("analyzeWorkload and collectFlags", () => { + it("flags a workload whose base makes no API calls and whose branch makes one", () => { + const analysis = analyzeWorkload( + workload( + times(5, () => sample(50, { calls: 0 })), + times(5, () => sample(51, { calls: 1, endpoints: { "GET /v1/projects/abc": 1 } })), + ), + true, + ); + + expect(collectFlags(analysis).map((flag) => flag.kind)).toEqual(["new-api-calls"]); + expect(analysis.endpointChanges).toEqual([ + { endpoint: "GET /v1/projects/abc", base: 0, branch: 1 }, + ]); + }); + + it("lists a count change from one call to two without a zero flag", () => { + const analysis = analyzeWorkload( + workload( + times(5, () => sample(50, { calls: 1, endpoints: { "GET /v1/projects": 1 } })), + times(5, () => sample(50, { calls: 2, endpoints: { "GET /v1/projects": 2 } })), + ), + true, + ); + + expect(collectFlags(analysis)).toEqual([]); + expect(analysis.branch.api?.medianCalls).toBe(2); + expect(analysis.endpointChanges).toEqual([ + { endpoint: "GET /v1/projects", base: 1, branch: 2 }, + ]); + }); + + it("flags external calls, non-determinism, and spans that disagree with the request log", () => { + const noisy = [ + sample(50, { calls: 1, endpoints: { "GET /a": 1 }, external: 1 }), + sample(50, { calls: 2, endpoints: { "GET /a": 2 }, log: 1 }), + sample(50, { calls: 1, endpoints: { "GET /a": 1 } }), + sample(50, { calls: 1, endpoints: { "GET /a": 1 } }), + sample(50, { calls: 1, endpoints: { "GET /a": 1 } }), + ]; + const steady = times(5, () => sample(50, { calls: 1, endpoints: { "GET /a": 1 } })); + + const kinds = collectFlags(analyzeWorkload(workload(steady, noisy), true)).map( + (flag) => flag.kind, + ); + + expect(kinds).toEqual(["external-calls", "non-deterministic", "request-log-mismatch"]); + }); + + it("flags a branch that lost all API calls and treats a missing trace as incomplete", () => { + const analysis = analyzeWorkload( + workload( + times(5, () => sample(50, { calls: 1 })), + [...times(4, () => sample(50, { calls: 0 })), { wallMs: 50, exitCode: 0 }], + ), + true, + ); + + expect(collectFlags(analysis).map((flag) => flag.kind)).toEqual([ + "api-calls-dropped", + "trace-incomplete", + ]); + }); + + it("withholds the verdict and flags failures when a build exits non-zero or gets error responses", () => { + const failing = times(5, () => sample(200, { calls: 1, errors: 1 }, 1)); + const analysis = analyzeWorkload( + workload( + times(5, () => sample(50, { calls: 1 })), + failing, + ), + true, + ); + + expect(analysis.verdict).toBe("not comparable"); + expect(collectFlags(analysis).map((flag) => flag.kind)).toEqual([ + "exit-codes-differ", + "non-zero-exit", + "error-status", + ]); + }); + + it("reports the heaviest phase deltas only when the timing verdict is slower or faster", () => { + const slow = analyzeWorkload( + workload( + times(5, (i) => sample(100 + i)), + times(5, (i) => sample(200 + i)), + ), + true, + ); + const same = analyzeWorkload( + workload( + times(5, (i) => sample(100 + i)), + times(5, (i) => sample(101 + i)), + ), + true, + ); + + expect(slow.verdict).toBe("slower"); + expect(slow.phaseRows.map((row) => row.name)).toEqual(["cli.run"]); + expect(same.verdict).toBe("within noise"); + expect(same.phaseRows).toEqual([]); + }); +}); + +describe("classifyHttpSpans host matching", () => { + it("matches a bracketed IPv6 API URL against the unbracketed span address", () => { + const summary = classifyHttpSpans( + [httpSpan("GET", "/v1/x", 4010, 200, "::1")], + "http://[::1]:4010", + ); + + expect(summary.apiCalls).toBe(1); + expect(summary.externalCalls).toBe(0); + }); + + it("counts a call to another host as external even when its port is unknown", () => { + const other: ReportSpan = { + ...span("GET", 0, 1), + http: { method: "GET", path: "/", address: "api.example.com", port: undefined, status: 200 }, + }; + + expect(classifyHttpSpans([other], "http://127.0.0.1:4010").externalCalls).toBe(1); + }); +}); + +describe("more flags", () => { + it("explains a missing verdict when there are too few runs", () => { + const few = times(3, () => sample(50, { calls: 1 })); + + const flags = collectFlags(analyzeWorkload(workload(few, few), true)); + + expect(flags.map((flag) => flag.kind)).toEqual(["insufficient-samples"]); + }); + + it("flags a traced launch that exits differently from the timed launches", () => { + const traced = sample(50, { calls: 1 }); + const odd: RunSample = { ...traced, tracedExitCode: 1 }; + + const flags = collectFlags( + analyzeWorkload( + workload( + times(5, () => sample(50, { calls: 1 })), + times(5, () => odd), + ), + true, + ), + ); + + expect(flags.map((flag) => flag.kind)).toContain("traced-exit-differs"); + }); +}); + +describe("traced launch failures", () => { + it("keeps a failed traced launch visible when no spans were decoded", () => { + const failedWithoutSpans: RunSample = { wallMs: 50, exitCode: 0, tracedExitCode: 1 }; + + const analysis = analyzeWorkload( + workload( + times(5, () => sample(50)), + times(5, () => failedWithoutSpans), + ), + true, + ); + + expect(analysis.verdict).toBe("not comparable"); + expect(collectFlags(analysis).map((flag) => flag.kind)).toEqual( + expect.arrayContaining(["traced-exit-differs", "trace-incomplete"]), + ); + }); +}); + +describe("endpoint changes without an API measurement", () => { + it("reports none when one build has no API measurement", () => { + const analysis = analyzeWorkload( + workload( + times(5, () => ({ wallMs: 50, exitCode: 0 })), + times(5, () => sample(50, { calls: 1, endpoints: { "GET /v1/projects": 1 } })), + ), + true, + ); + + expect(analysis.endpointChanges).toEqual([]); + }); +}); + +describe("report JSON", () => { + const minimal: BenchReport = { + base: "/b", + branch: "/c", + runs: 5, + warmup: 1, + trace: false, + workloads: [{ id: "w", args: ["x"], base: [{ wallMs: 1, exitCode: 0 }], branch: [] }], + }; + const full: BenchReport = { + ...minimal, + baseSizeBytes: 10, + branchSizeBytes: 11, + trace: true, + apiUrl: "http://127.0.0.1:1", + cpuProfilePath: "/p", + workloads: [ + { ...minimal.workloads[0]!, base: [sample(1, { calls: 1, endpoints: { "GET /a": 1 } })] }, + ], + }; + + it.effect.each([ + { name: "omitted optional fields", report: minimal }, + { name: "every field", report: full }, + ])("round-trips a report with $name", ({ report }) => + Effect.gen(function* () { + const decoded = yield* decodeBenchReport(yield* encodeBenchReport(report)); + + expect(decoded).toEqual(report); + }), + ); +}); diff --git a/apps/cli/scripts/bench-summary.integration.test.ts b/apps/cli/scripts/bench-summary.integration.test.ts new file mode 100644 index 0000000000..98d45acc1a --- /dev/null +++ b/apps/cli/scripts/bench-summary.integration.test.ts @@ -0,0 +1,25 @@ +import { BunServices } from "@effect/platform-bun"; +import { describe, expect, it } from "@effect/vitest"; +import { Effect, FileSystem } from "effect"; +import { loadReport } from "./bench-summary.ts"; + +describe("loadReport", () => { + it.effect("tells an absent report from an unreadable one", () => + Effect.scoped( + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const dir = yield* fs.makeTempDirectoryScoped({ prefix: "bench-summary-" }); + const corrupt = `${dir}/report.json`; + yield* fs.writeFileString(corrupt, "{ not a report"); + + const absent = yield* loadReport(fs, `${dir}/missing.json`); + const unreadable = yield* loadReport(fs, corrupt); + + expect(absent.report).toBeUndefined(); + expect(absent.notice).toContain("did not produce a report"); + expect(unreadable.report).toBeUndefined(); + expect(unreadable.notice).toContain("could not be read"); + }), + ).pipe(Effect.provide(BunServices.layer)), + ); +}); diff --git a/apps/cli/scripts/bench-summary.ts b/apps/cli/scripts/bench-summary.ts new file mode 100644 index 0000000000..21b94399ee --- /dev/null +++ b/apps/cli/scripts/bench-summary.ts @@ -0,0 +1,450 @@ +/** + * Renders a `bench-cli.ts --out` report as Markdown: the answer first, then a timing table, API + * calls, binary size, and collapsed details. Run as `bun scripts/bench-summary.ts --report ` + * with optional `--branch-build|--base-build|--replay ok|failed` step outcomes, `--head-sha`, + * `--base-ref`, `--merge-base`, and `--comment-out ` for the pull request + * comment variant; the job summary goes to `GITHUB_STEP_SUMMARY` when set, otherwise to stdout. + */ +import process from "node:process"; +import { parseArgs } from "node:util"; +import { BunRuntime, BunServices } from "@effect/platform-bun"; +import { Config, Console, Effect, FileSystem, Option, Stdio } from "effect"; +import { + analyzeWorkload, + collectFlags, + decodeBenchReport, + VERDICT_MIN_PCT, + type BenchReport, + type Flag, + type WorkloadAnalysis, +} from "./bench-report.ts"; + +const USAGE = + "usage: bun scripts/bench-summary.ts --report [--branch-build ok|failed] [--base-build ok|failed] [--replay ok|failed] [--head-sha ] [--base-ref ] [--merge-base ] [--comment-out ]"; + +type Outcome = "ok" | "failed"; + +export interface SummaryOptions { + readonly report: string | undefined; + readonly branchBuild: Outcome; + readonly baseBuild: Outcome; + readonly replay: Outcome; + readonly headSha: string | undefined; + readonly baseRef: string | undefined; + readonly mergeBase: string | undefined; + readonly commentOut: string | undefined; +} + +const OUTCOME_FLAGS = ["branch-build", "base-build", "replay"] as const; + +/** Parses argv; throws on an unknown outcome or a missing report when every step succeeded. */ +export function parseSummaryArgs(argv: ReadonlyArray): SummaryOptions { + const { values } = parseArgs({ + args: [...argv], + options: { + report: { type: "string" }, + "branch-build": { type: "string", default: "ok" }, + "base-build": { type: "string", default: "ok" }, + replay: { type: "string", default: "ok" }, + "head-sha": { type: "string" }, + "base-ref": { type: "string" }, + "merge-base": { type: "string" }, + "comment-out": { type: "string" }, + }, + }); + for (const flag of OUTCOME_FLAGS) { + if (values[flag] !== "ok" && values[flag] !== "failed") { + throw new Error(`--${flag} must be "ok" or "failed"`); + } + } + const options: SummaryOptions = { + report: values.report, + branchBuild: values["branch-build"] === "failed" ? "failed" : "ok", + baseBuild: values["base-build"] === "failed" ? "failed" : "ok", + replay: values.replay === "failed" ? "failed" : "ok", + headSha: values["head-sha"], + baseRef: values["base-ref"], + mergeBase: values["merge-base"], + commentOut: values["comment-out"], + }; + if (values.report === undefined && summaryNotice(options) === undefined) throw new Error(USAGE); + return options; +} + +const NO_REPORT = + "The benchmark did not produce a report. See the Run benchmark step in the workflow run."; +const UNREADABLE_REPORT = + "The benchmark report could not be read. See the Write job summary step in the workflow run."; + +/** The reason there is nothing to compare, in the order the steps run, or undefined when all ran. */ +export function summaryNotice( + options: Pick, +): string | undefined { + if (options.branchBuild === "failed") { + return "The build of this commit failed, so there is nothing to compare."; + } + if (options.baseBuild === "failed") { + return "The build of the base branch failed, so there is nothing to compare."; + } + if (options.replay === "failed") { + return "The local test server failed to start, so the benchmark did not run. See the Start replay server step in the workflow run."; + } + return undefined; +} + +const sign = (value: number) => (value >= 0 ? "+" : ""); +const ms = (value: number) => `${Math.round(value)} ms`; +const range = (build: WorkloadAnalysis["base"]) => + `${Math.round(build.timing.min)}-${Math.round(build.timing.max)} ms`; +const signedMs = (value: number) => `${sign(value)}${Math.round(value)} ms`; +const signedPct = (value: number) => `${sign(value)}${Math.round(value)}%`; + +const cell = (value: string) => value.replaceAll("|", "\\|"); +/** Inline code for text that comes from the benchmarked build or its workload file. */ +const code = (value: string) => `\`${value.replaceAll("`", "'").replaceAll(/\s+/gu, " ")}\``; +const command = (analysis: Pick) => + code(["supabase", ...analysis.args].join(" ")); +const plural = (count: number, one: string, many: string) => (count === 1 ? one : many); +const callCount = (count: number | undefined) => + `${count ?? "some"} ${count === 1 ? "call" : "calls"}`; +const MIB = 1024 * 1024; +const mb = (bytes: number) => `${(bytes / MIB).toFixed(1)} MB`; + +function table(header: ReadonlyArray, rows: ReadonlyArray>): string { + return [ + `| ${header.join(" | ")} |`, + `| ${header.map(() => "---").join(" | ")} |`, + ...rows.map((row) => `| ${row.map(cell).join(" | ")} |`), + ].join("\n"); +} + +const when = (build: Flag["build"]) => (build === "base" ? "before" : "after"); + +/** One plain-words sentence for a flag. */ +function describeFlag(flag: Flag, analysis: WorkloadAnalysis): string { + const name = command(analysis); + const at = when(flag.build); + switch (flag.kind) { + case "new-api-calls": { + const added = analysis.endpointChanges + .filter((change) => change.branch > change.base) + .slice(0, 3) + .map((change) => code(change.endpoint)); + return ( + `${name} now calls the Management API. It made no calls before and now makes ` + + `${analysis.branch.api?.medianCalls ?? "some"}${added.length === 0 ? "" : ` (${added.join(", ")})`}.` + ); + } + case "api-calls-dropped": + return `${name} no longer calls the Management API. It made ${callCount(analysis.base.api?.medianCalls)} before and now makes none.`; + case "external-calls": + return `${name} made calls to hosts other than the local test server ${at}, so its timing may be skewed.`; + case "non-deterministic": + return `${name} makes a different number of API calls from run to run ${at}, so its results are less reliable.`; + case "request-log-mismatch": + return `The API calls counted for ${name} ${at} do not match what the local test server saw, so the counts may be wrong.`; + case "error-status": + return `${name} got error responses from the local test server ${at}, so it did not follow the recorded scenario.`; + case "trace-incomplete": + return `Some runs of ${name} ${at} produced no usable trace, so call counts and phase timings may be incomplete.`; + case "non-zero-exit": + return `${name} exited with an error ${at}; its timing is not compared.`; + case "exit-codes-differ": + return `${name} exits with different codes before and after, so its timing is not compared.`; + case "traced-exit-differs": + return `${name} exited differently when traced ${at}, so its timing is not compared.`; + case "insufficient-samples": + return `${name} had too few timed runs for a timing verdict.`; + } +} + +function changeText(analysis: WorkloadAnalysis): string { + switch (analysis.verdict) { + case "not comparable": + return "not compared"; + case "slower": + case "faster": { + const delta = Math.abs(Math.round(analysis.deltaMs)); + return `${analysis.verdict} by ${delta} ms (${signedPct(analysis.deltaPct)})`; + } + case "within noise": + return Math.abs(analysis.deltaPct) >= VERDICT_MIN_PCT + ? `no real change (${signedPct(analysis.deltaPct)}, within normal variation)` + : "no real change"; + } +} + +function timingTable(analyses: ReadonlyArray): string { + return table( + ["Command", "Before", "After", "Change"], + analyses.map((analysis) => [ + command(analysis), + ms(analysis.base.timing.median), + ms(analysis.branch.timing.median), + changeText(analysis), + ]), + ); +} + +interface ApiOverview { + readonly line: string; + /** Ids of the commands whose API calls changed. */ + readonly changedIds: ReadonlyArray; +} + +function apiOverview(analyses: ReadonlyArray): ApiOverview { + const measured = analyses.filter( + (analysis) => analysis.base.api !== undefined && analysis.branch.api !== undefined, + ); + if (measured.length === 0) return { line: "API calls were not counted.", changedIds: [] }; + const changed = measured.filter( + (analysis) => + analysis.base.api?.medianCalls !== analysis.branch.api?.medianCalls || + analysis.endpointChanges.length > 0, + ); + const missing = analyses.length - measured.length; + const suffix = missing === 0 ? "" : ` (not measured for ${missing})`; + if (changed.length === 0) { + return { + line: `API calls: unchanged for ${measured.length === 1 ? "the command" : `all ${measured.length} commands`}${suffix}.`, + changedIds: [], + }; + } + const lines = changed.map((analysis) => { + const endpoints = analysis.endpointChanges + .map((change) => `${code(change.endpoint)} ${change.base} -> ${change.branch}`) + .join(", "); + return `- ${command(analysis)}: ${callCount(analysis.base.api?.medianCalls)} before, ${analysis.branch.api?.medianCalls} now${endpoints === "" ? "" : ` (${endpoints})`}`; + }); + return { + line: [ + `API calls changed for ${changed.length} ${plural(changed.length, "command", "commands")}${suffix}:`, + ...lines, + ].join("\n"), + changedIds: changed.map((analysis) => analysis.id), + }; +} + +function sizeLine(report: BenchReport): string { + const { baseSizeBytes: base, branchSizeBytes: branch } = report; + if (base === undefined || branch === undefined) return "Binary size: not recorded."; + const delta = branch - base; + if (delta === 0) return `Binary size: ${mb(branch)}, unchanged.`; + const abs = Math.abs(delta); + const size = abs < 1024 ? `${abs} bytes` : abs < MIB ? `${(abs / 1024).toFixed(1)} KB` : mb(abs); + return `Binary size: ${mb(branch)}, ${delta > 0 ? "up" : "down"} ${size} from ${mb(base)}.`; +} + +function apiDetailTable(analyses: ReadonlyArray): string { + const rows = analyses + .filter((analysis) => analysis.base.api !== undefined || analysis.branch.api !== undefined) + .map((analysis) => [ + command(analysis), + String(analysis.base.api?.medianCalls ?? "n/a"), + String(analysis.branch.api?.medianCalls ?? "n/a"), + analysis.endpointChanges + .map((change) => `${code(change.endpoint)} ${change.base} -> ${change.branch}`) + .join("
"), + ]); + if (rows.length === 0) return "API calls were not counted."; + return table(["Command", "Before", "After", "Endpoint changes"], rows); +} + +function rangeTable(analyses: ReadonlyArray): string { + return table( + ["Command", "Before range", "After range", "Change", "Verdict"], + analyses.map((analysis) => [ + command(analysis), + range(analysis.base), + range(analysis.branch), + `${signedMs(analysis.deltaMs)} (${signedPct(analysis.deltaPct)})`, + analysis.verdict, + ]), + ); +} + +function phaseDetail(analyses: ReadonlyArray): string { + const rows = analyses.flatMap((analysis) => + analysis.phaseRows.map((row) => { + const only = row.onlyIn === undefined ? undefined : row[row.onlyIn]; + return [ + command(analysis), + code(row.name), + row.deltaMs === undefined + ? `${row.onlyIn === "base" ? "before" : "after"} only, ${Math.round(only?.medianMs ?? 0)} ms` + : signedMs(row.deltaMs), + `${row.base === undefined ? "-" : Math.round(row.base.medianMs)} -> ${row.branch === undefined ? "-" : Math.round(row.branch.medianMs)} ms`, + ]; + }), + ); + if (rows.length === 0) { + return analyses.some((analysis) => analysis.verdict === "not comparable") + ? "No phase deltas are available; some commands were not comparable." + : "No command moved beyond noise, so there are no phase deltas."; + } + return table(["Command", "Span", "Change", "Before -> after"], rows); +} + +export const BENCH_MARKER = ""; + +export interface RenderContext { + /** `comment` adds the marker and the re-run footer; `summary` is the job summary. */ + readonly target: "summary" | "comment"; + readonly headSha?: string | undefined; + readonly baseRef?: string | undefined; + readonly mergeBase?: string | undefined; +} + +const short = (sha: string) => sha.slice(0, 7); + +function contextLine(report: BenchReport | undefined, context: RenderContext): string { + const parts: Array = []; + if (context.headSha !== undefined && context.headSha !== "") { + const against = + context.baseRef === undefined || context.baseRef === "" + ? "" + : ` against ${code(context.baseRef)}${ + context.mergeBase === undefined || context.mergeBase === "" + ? "" + : ` at ${code(short(context.mergeBase))}` + }`; + parts.push(`Commit ${code(short(context.headSha))}${against}`); + } + if (report !== undefined) parts.push(`${report.runs} ${plural(report.runs, "run", "runs")} each`); + parts.push("report only, never blocks merging"); + return parts.join(" · "); +} + +function frame( + title: string, + body: ReadonlyArray, + context: RenderContext, + report?: BenchReport, +): string { + const footer = + context.target === "comment" ? ["Comment `/bench` to run it again on the latest commit."] : []; + return ( + [ + ...(context.target === "comment" ? [BENCH_MARKER] : []), + `## CLI benchmark: ${title}`, + contextLine(report, context), + ...body, + ...footer, + ].join("\n\n") + "\n" + ); +} + +/** The Markdown report, or `notice` in plain words when there is no report to compare. */ +export function renderSummary( + report: BenchReport | undefined, + notice: string, + context: RenderContext = { target: "summary" }, +): string { + if (report === undefined) return frame("could not run", [notice], context); + const analyses = report.workloads.map((workload) => analyzeWorkload(workload, report.trace)); + const flags = analyses.flatMap((analysis) => + collectFlags(analysis).map((flag) => ({ flag, analysis })), + ); + if (analyses.length === 0) return frame("no commands were benchmarked", [], context, report); + const api = apiOverview(analyses); + const moved = analyses + .filter((analysis) => analysis.verdict === "slower" || analysis.verdict === "faster") + .map((analysis) => analysis.id); + const things = new Set([...flags.map(({ analysis }) => analysis.id), ...moved, ...api.changedIds]) + .size; + const title = + things === 0 ? "no changes found" : `${things} ${plural(things, "thing", "things")} to check`; + return frame( + title, + [ + ...flags.map(({ flag, analysis }) => `⚠️ ${describeFlag(flag, analysis)}`), + timingTable(analyses), + api.line, + sizeLine(report), + [ + "
Details: timing ranges, API calls per command, slowest phases", + "#### Timing ranges", + rangeTable(analyses), + "#### API calls per command", + apiDetailTable(analyses), + "#### Slowest phases", + phaseDetail(analyses), + "
", + ].join("\n\n"), + ], + context, + report, + ); +} + +export const loadReport = Effect.fn("BenchSummary.loadReport")(function* ( + fs: FileSystem.FileSystem, + file: string, +) { + if (!(yield* fs.exists(file).pipe(Effect.orElseSucceed(() => false)))) { + return { report: undefined, notice: NO_REPORT }; + } + return yield* fs.readFileString(file).pipe( + Effect.flatMap(decodeBenchReport), + Effect.map((report) => ({ report, notice: NO_REPORT })), + Effect.catch((cause) => + Console.error(`could not read benchmark report ${file}: ${String(cause)}`).pipe( + Effect.as({ + report: undefined, + notice: UNREADABLE_REPORT, + }), + ), + ), + ); +}); + +const main = Effect.gen(function* () { + const stdio = yield* Stdio.Stdio; + const argv = yield* stdio.args; + const options = yield* Effect.try({ + try: () => parseSummaryArgs(argv), + catch: (cause) => (cause instanceof Error ? cause.message : String(cause)), + }).pipe( + Effect.catch((message) => + Console.error(message).pipe( + Effect.andThen( + Effect.sync(() => { + process.exitCode = 2; + }), + ), + Effect.as(undefined), + ), + ), + ); + if (options === undefined) return; + + const fs = yield* FileSystem.FileSystem; + const outcomeNotice = summaryNotice(options); + const loaded = + outcomeNotice === undefined && options.report !== undefined + ? yield* loadReport(fs, options.report) + : { report: undefined, notice: outcomeNotice ?? NO_REPORT }; + const context = { + headSha: options.headSha, + baseRef: options.baseRef, + mergeBase: options.mergeBase, + }; + const markdown = renderSummary(loaded.report, loaded.notice, { target: "summary", ...context }); + const summaryFile = yield* Config.option(Config.String("GITHUB_STEP_SUMMARY")); + if (Option.isSome(summaryFile) && summaryFile.value !== "") { + yield* fs.writeFileString(summaryFile.value, markdown, { flag: "a" }); + } else { + yield* Console.log(markdown); + } + + if (options.commentOut !== undefined) { + yield* fs.writeFileString( + options.commentOut, + renderSummary(loaded.report, loaded.notice, { target: "comment", ...context }), + ); + } +}); + +if (import.meta.main) { + BunRuntime.runMain(main.pipe(Effect.provide(BunServices.layer))); +} diff --git a/apps/cli/scripts/bench-summary.unit.test.ts b/apps/cli/scripts/bench-summary.unit.test.ts new file mode 100644 index 0000000000..3d5e369c6a --- /dev/null +++ b/apps/cli/scripts/bench-summary.unit.test.ts @@ -0,0 +1,224 @@ +import { describe, expect, it } from "vitest"; +import type { BenchReport, RunSample } from "./bench-report.ts"; +import { BENCH_MARKER, parseSummaryArgs, renderSummary, summaryNotice } from "./bench-summary.ts"; + +const run = (wallMs: number, calls: number, endpoints: Record): RunSample => ({ + wallMs, + exitCode: 0, + tracedExitCode: 0, + trace: { + skippedLines: 0, + spans: { "cli.run": { totalMs: wallMs / 2, count: 1 } }, + api: { apiCalls: calls, externalCalls: 0, errorStatuses: 0, endpoints, requestLogCount: calls }, + }, +}); + +const runs = (n: number, make: (i: number) => RunSample) => + Array.from({ length: n }, (_, i) => make(i)); + +const report = (workloads: BenchReport["workloads"]): BenchReport => ({ + base: "/base/supabase", + branch: "/branch/supabase", + baseSizeBytes: 1000, + branchSizeBytes: 1500, + runs: 5, + warmup: 1, + trace: true, + apiUrl: "http://127.0.0.1:1", + workloads, +}); + +const steady = (id: string, args: ReadonlyArray) => ({ + id, + args, + base: runs(5, (i) => run(30 + i, 0, {})), + branch: runs(5, (i) => run(30 + i, 0, {})), +}); + +const migrationNew = { + id: "migration-new", + args: ["migration", "new", "bench"], + base: runs(5, (i) => run(50 + i, 0, {})), + branch: runs(5, (i) => run(120 + i, 1, { "GET /v1/projects/abc": 1 })), +}; + +describe("renderSummary", () => { + it("leads with no changes found and shows commands, not workload ids", () => { + const sized = { + ...report([steady("version", ["--version"])]), + baseSizeBytes: 140269744, + branchSizeBytes: 140269744, + }; + const markdown = renderSummary(sized, "unused", { + target: "summary", + headSha: "abcdef1234567890", + baseRef: "develop", + mergeBase: "1234567890abcdef", + }); + + expect(markdown).toContain("## CLI benchmark: no changes found"); + expect(markdown).toContain( + "Commit `abcdef1` against `develop` at `1234567` · 5 runs each · report only, never blocks merging", + ); + expect(markdown).toContain("| `supabase --version` | 32 ms | 32 ms | no real change |"); + expect(markdown).toContain("API calls: unchanged for the command."); + expect(markdown).toContain("Binary size: 133.8 MB, unchanged."); + expect(markdown).not.toContain("⚠️"); + }); + + it("shows a binary size change in MB, or KB when it is small", () => { + const grown = { + ...report([steady("version", ["--version"])]), + baseSizeBytes: 140269744, + branchSizeBytes: 140269744 + 3 * 1048576, + }; + const nudged = { ...grown, branchSizeBytes: 140269744 + 2048 }; + + expect(renderSummary(grown, "unused")).toContain( + "Binary size: 136.8 MB, up 3.0 MB from 133.8 MB.", + ); + expect(renderSummary(nudged, "unused")).toContain("up 2.0 KB from 133.8 MB"); + }); + + it("explains a new API call in plain words and counts each command once", () => { + const markdown = renderSummary(report([migrationNew]), "unused"); + + expect(markdown).toContain("## CLI benchmark: 1 thing to check"); + expect(markdown).toContain( + "⚠️ `supabase migration new bench` now calls the Management API. It made no calls before and now makes 1 (`GET /v1/projects/abc`).", + ); + expect(markdown).toContain("slower by 70 ms (+135%)"); + }); + + it("keeps command text from the benchmarked build inside code spans", () => { + const markdown = renderSummary(report([steady("x", ["a`@someone`", "b|c"])]), "unused"); + + expect(markdown).toContain("`supabase a'@someone' b\\|c`"); + }); + + it("says nothing was benchmarked instead of a clean pass when there are no commands", () => { + const markdown = renderSummary(report([]), "unused"); + + expect(markdown).toContain("## CLI benchmark: no commands were benchmarked"); + expect(markdown).not.toContain("no changes found"); + }); + + it("describes a dropped API call with a singular count", () => { + const dropped = { + id: "list", + args: ["projects", "list"], + base: runs(5, (i) => run(30 + i, 1, { "GET /v1/projects": 1 })), + branch: runs(5, (i) => run(30 + i, 0, {})), + }; + + const markdown = renderSummary(report([dropped]), "unused"); + + expect(markdown).toContain( + "⚠️ `supabase projects list` no longer calls the Management API. It made 1 call before and now makes none.", + ); + }); + + it("names the failed step and stays a notice when there is no report", () => { + const markdown = renderSummary(undefined, "The build of this commit failed.", { + target: "summary", + }); + + expect(markdown).toContain("## CLI benchmark: could not run"); + expect(markdown).toContain("The build of this commit failed."); + expect(markdown).not.toContain("/bench"); + }); + + it("adds the marker, re-run hint only to the comment", () => { + const input = report([steady("version", ["--version"])]); + const context = { headSha: "abcdef1234" } as const; + + const comment = renderSummary(input, "unused", { target: "comment", ...context }); + const summary = renderSummary(input, "unused", { target: "summary", ...context }); + + expect(BENCH_MARKER).toBe(""); + expect(comment.startsWith(`${BENCH_MARKER}\n`)).toBe(true); + expect(comment).toContain("Comment `/bench` to run it again on the latest commit."); + expect(summary).not.toContain(BENCH_MARKER); + expect(summary).not.toContain("/bench"); + }); + + it("does not claim a missing timing verdict when commands were not comparable", () => { + const failing = (wallMs: number): RunSample => ({ ...run(wallMs, 0, {}), exitCode: 1 }); + + const markdown = renderSummary( + report([ + { + id: "broken", + args: ["x"], + base: runs(5, (i) => run(30 + i, 0, {})), + branch: runs(5, (i) => failing(30 + i)), + }, + ]), + "unused", + ); + + expect(markdown).toContain("No phase deltas are available; some commands were not comparable."); + expect(markdown).toContain("not compared"); + }); + + it("says API calls were not counted when no call data exists", () => { + const noTrace = (wallMs: number): RunSample => ({ wallMs, exitCode: 0 }); + const markdown = renderSummary( + { + ...report([ + { + id: "a", + args: ["x"], + base: runs(5, (i) => noTrace(30 + i)), + branch: runs(5, (i) => noTrace(30 + i)), + }, + ]), + trace: false, + }, + "unused", + ); + + expect(markdown).toContain("API calls were not counted."); + }); +}); + +describe("summaryNotice", () => { + const ok = { branchBuild: "ok", baseBuild: "ok", replay: "ok" } as const; + + it("is undefined when every step succeeded", () => { + expect(summaryNotice(ok)).toBeUndefined(); + }); + + it.each([ + { failed: { branchBuild: "failed" }, expected: "build of this commit failed" }, + { failed: { baseBuild: "failed" }, expected: "build of the base branch failed" }, + { failed: { replay: "failed" }, expected: "test server failed to start" }, + ] as const)("names the failed step: $expected", ({ failed, expected }) => { + expect(summaryNotice({ ...ok, ...failed })).toContain(expected); + }); + + it("reports the earliest failed step when later ones were skipped", () => { + expect( + summaryNotice({ branchBuild: "failed", baseBuild: "failed", replay: "failed" }), + ).toContain("build of this commit failed"); + }); +}); + +describe("parseSummaryArgs", () => { + it("rejects an unknown step outcome", () => { + expect(() => parseSummaryArgs(["--report", "r.json", "--replay", "maybe"])).toThrow("--replay"); + }); + + it("needs a report unless a step failed", () => { + expect(() => parseSummaryArgs([])).toThrow(); + expect(parseSummaryArgs(["--replay", "failed"])).toMatchObject({ + report: undefined, + branchBuild: "ok", + baseBuild: "ok", + replay: "failed", + }); + expect( + parseSummaryArgs(["--report", "r.json", "--head-sha", "abc", "--comment-out", "c.md"]), + ).toMatchObject({ report: "r.json", replay: "ok", headSha: "abc", commentOut: "c.md" }); + }); +}); diff --git a/apps/cli/scripts/bench-workloads.unit.test.ts b/apps/cli/scripts/bench-workloads.unit.test.ts new file mode 100644 index 0000000000..403ce9664f --- /dev/null +++ b/apps/cli/scripts/bench-workloads.unit.test.ts @@ -0,0 +1,39 @@ +import { BunServices } from "@effect/platform-bun"; +import { describe, expect, it } from "@effect/vitest"; +import { Effect, FileSystem } from "effect"; +import { parseWorkloadManifest, validateWorkloads } from "./bench-report.ts"; + +const manifestPath = new URL("../bench/workloads.pr.json", import.meta.url).pathname; +const scenariosDir = new URL("../../cli-e2e/fixtures/scenarios/", import.meta.url).pathname; + +const loadManifest = Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + return yield* parseWorkloadManifest(yield* fs.readFileString(manifestPath)); +}); + +describe("PR workload manifest", () => { + it.effect("decodes and passes the structural rules when a replay server is configured", () => + Effect.gen(function* () { + const workloads = yield* loadManifest; + + expect(workloads.length).toBeGreaterThan(0); + expect(validateWorkloads(workloads, { apiUrl: "http://127.0.0.1:1" })).toEqual([]); + }).pipe(Effect.provide(BunServices.layer)), + ); + + it.effect("names only replay scenarios that exist and serve the project ref it uses", () => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const workloads = yield* loadManifest; + + for (const workload of workloads) { + if (workload.scenario === undefined) continue; + const fixture = yield* fs.readFileString( + `${scenariosDir}${workload.scenario}/interactions.json`, + ); + const ref = workload.args.find((arg) => arg.startsWith("replay")); + if (ref !== undefined) expect(fixture).toContain(`/v1/projects/${ref}`); + } + }).pipe(Effect.provide(BunServices.layer)), + ); +}); diff --git a/apps/cli/scripts/replay-control.integration.test.ts b/apps/cli/scripts/replay-control.integration.test.ts new file mode 100644 index 0000000000..ee393ead1e --- /dev/null +++ b/apps/cli/scripts/replay-control.integration.test.ts @@ -0,0 +1,60 @@ +import { describe, expect, it } from "@effect/vitest"; +import { Effect } from "effect"; +import { FetchHttpClient } from "effect/http"; +import * as HttpClient from "effect/http/HttpClient"; +import { startReplayServer } from "../../cli-e2e/src/server/replay-server.ts"; +import { readRequestLogCount, resetReplay } from "./replay-control.ts"; + +const fixturesDir = new URL("../../cli-e2e/fixtures", import.meta.url).pathname; +const SCENARIO = "projects-list-renders-project-list"; + +const replayServer = Effect.acquireRelease( + Effect.promise(() => startReplayServer({ fixturesDir })), + (server) => Effect.promise(() => server.stop()), +); + +const callApi = (url: string, path: string) => + HttpClient.HttpClient.use((client) => client.get(new URL(path, url).toString())); + +describe("replay control against the replay server", () => { + it.effect("serves a consumed scenario again and starts each launch with an empty log", () => + Effect.scoped( + Effect.gen(function* () { + const server = yield* replayServer; + + yield* resetReplay(server.url, SCENARIO); + const first = yield* callApi(server.url, "/v1/projects"); + const logged = yield* readRequestLogCount(server.url); + + yield* resetReplay(server.url, SCENARIO); + const cleared = yield* readRequestLogCount(server.url); + const second = yield* callApi(server.url, "/v1/projects"); + + expect([first.status, second.status]).toEqual([200, 200]); + expect(logged).toBe(1); + expect(cleared).toBe(0); + }), + ).pipe(Effect.provide(FetchHttpClient.layer)), + ); + + it.effect("fails when the scenario does not exist instead of keeping the previous one", () => + Effect.scoped( + Effect.gen(function* () { + const server = yield* replayServer; + + const failure = yield* resetReplay(server.url, "no-such-scenario").pipe(Effect.flip); + + expect(failure._tag).toBe("ReplayControlError"); + expect(failure.step).toBe("POST /_ctrl/scenario"); + }), + ).pipe(Effect.provide(FetchHttpClient.layer)), + ); + + it.effect("fails when the server is unreachable", () => + Effect.gen(function* () { + const failure = yield* resetReplay("http://127.0.0.1:1", undefined).pipe(Effect.flip); + + expect(failure._tag).toBe("ReplayControlError"); + }).pipe(Effect.provide(FetchHttpClient.layer)), + ); +}); diff --git a/apps/cli/scripts/replay-control.ts b/apps/cli/scripts/replay-control.ts new file mode 100644 index 0000000000..b8b72e052b --- /dev/null +++ b/apps/cli/scripts/replay-control.ts @@ -0,0 +1,80 @@ +/** + * Client for the replay server's `/_ctrl/*` HTTP API, used by `bench-cli.ts` to start each launch + * from a known server state. Any failed control call aborts the benchmark, because a launch that + * runs against stale or missing scenario state measures the wrong work. + */ +import { Effect, Schema } from "effect"; +import * as HttpClient from "effect/http/HttpClient"; +import * as HttpClientRequest from "effect/http/HttpClientRequest"; +import * as HttpClientResponse from "effect/http/HttpClientResponse"; + +export class ReplayControlError extends Schema.TaggedError()( + "ReplayControlError", + { step: Schema.String, message: Schema.String }, +) {} + +export interface ControlCall { + readonly method: "DELETE" | "POST"; + readonly path: string; + readonly body?: { readonly name: string }; +} + +/** The calls that clear the request log and select `scenario`, or no scenario when it is absent. */ +export function controlPlan(scenario: string | undefined): ReadonlyArray { + return [ + { method: "DELETE", path: "/_ctrl/requests" }, + { method: "DELETE", path: "/_ctrl/scenario" }, + ...(scenario === undefined + ? [] + : [{ method: "POST", path: "/_ctrl/scenario", body: { name: scenario } } as const]), + ]; +} + +const controlUrl = (apiUrl: string, path: string) => new URL(path, apiUrl).toString(); + +const failure = (step: string) => (cause: unknown) => + new ReplayControlError({ + step, + message: `${step} failed: ${cause instanceof Error ? cause.message : String(cause)}`, + }); + +const send = Effect.fn("ReplayControl.send")(function* ( + apiUrl: string, + method: "GET" | "DELETE" | "POST", + path: string, + body?: { readonly name: string }, +) { + const step = `${method} ${path}`; + const client = (yield* HttpClient.HttpClient).pipe(HttpClient.filterStatusOk); + const base = + method === "GET" + ? HttpClientRequest.get(controlUrl(apiUrl, path)) + : method === "DELETE" + ? HttpClientRequest.delete(controlUrl(apiUrl, path)) + : HttpClientRequest.post(controlUrl(apiUrl, path)); + const request = body === undefined ? base : HttpClientRequest.bodyJsonUnsafe(base, body); + return yield* client.execute(request).pipe(Effect.mapError(failure(step))); +}); + +/** Resets the request log and scenario so the next launch sees a freshly loaded scenario. */ +export const resetReplay = Effect.fn("ReplayControl.reset")(function* ( + apiUrl: string, + scenario: string | undefined, +) { + for (const call of controlPlan(scenario)) { + yield* send(apiUrl, call.method, call.path, call.body); + } +}); + +const RequestLog = Schema.Array(Schema.Struct({ method: Schema.String, pathname: Schema.String })); + +/** Number of API requests the server has logged since the last reset. */ +export const readRequestLogCount = Effect.fn("ReplayControl.readRequestLogCount")(function* ( + apiUrl: string, +) { + const response = yield* send(apiUrl, "GET", "/_ctrl/requests"); + const json = yield* HttpClientResponse.schemaBodyJson(RequestLog)(response).pipe( + Effect.mapError(failure("GET /_ctrl/requests body")), + ); + return json.length; +}); diff --git a/apps/cli/scripts/trace-report.ts b/apps/cli/scripts/trace-report.ts index a16430839c..41cc789ece 100644 --- a/apps/cli/scripts/trace-report.ts +++ b/apps/cli/scripts/trace-report.ts @@ -8,13 +8,23 @@ import { Console, Effect, FileSystem, Schema, Stdio } from "effect"; const KeyValues = Schema.Array( Schema.Struct({ key: Schema.String, - value: Schema.Struct({ stringValue: Schema.optional(Schema.String) }), + value: Schema.Struct({ + stringValue: Schema.optional(Schema.String), + // OTLP/JSON encodes 64-bit integers as strings; this exporter writes numbers. + intValue: Schema.optional(Schema.Union([Schema.Finite, Schema.String])), + }), }), ); const stringAttribute = (attributes: typeof KeyValues.Type | undefined, key: string) => attributes?.find((attribute) => attribute.key === key)?.value.stringValue; +const intAttribute = (attributes: typeof KeyValues.Type | undefined, key: string) => { + const value = attributes?.find((attribute) => attribute.key === key)?.value.intValue; + const number = value === undefined ? Number.NaN : Number(value); + return Number.isInteger(number) ? number : undefined; +}; + /** Labels container CLI spans with their verb so `pull` and `run` report separately. */ export function reportSpanName(name: string, subcommand: string | undefined): string { return subcommand === undefined ? name : `${name} (${subcommand})`; @@ -47,6 +57,15 @@ const TraceBatch = Schema.fromJsonString( const ReportJson = Schema.fromJsonString(Schema.Unknown, { space: 2 }); +/** Attributes of an HTTP client span; `path` has no query string. */ +export interface HttpSpanAttributes { + readonly method: string; + readonly path: string | undefined; + readonly address: string | undefined; + readonly port: number | undefined; + readonly status: number | undefined; +} + export interface ReportSpan { readonly traceId: string; readonly spanId: string; @@ -56,6 +75,8 @@ export interface ReportSpan { readonly endMs: number; readonly failed: boolean; readonly errorType: string | undefined; + /** Present only on HTTP client spans. */ + readonly http?: HttpSpanAttributes; } export interface TraceReport { @@ -74,6 +95,19 @@ export interface TraceReport { } const STATUS_ERROR = 2; + +function httpAttributes(attributes: typeof KeyValues.Type | undefined) { + const method = stringAttribute(attributes, "http.request.method"); + if (method === undefined) return undefined; + return { + method, + path: stringAttribute(attributes, "url.path")?.split("?")[0], + address: stringAttribute(attributes, "server.address"), + port: intAttribute(attributes, "server.port"), + status: intAttribute(attributes, "http.response.status_code"), + } satisfies HttpSpanAttributes; +} + const nanosToMs = (value: string) => Number(BigInt(value) / 1_000n) / 1_000; const round = (value: number) => Math.round(value * 100) / 100; @@ -192,6 +226,7 @@ export const readSpans = Effect.fnUntraced(function* (file: string) { for (const resource of batch.resourceSpans) { for (const scope of resource.scopeSpans) { for (const span of scope.spans) { + const http = httpAttributes(span.attributes); spans.push({ traceId: span.traceId, spanId: span.spanId, @@ -204,6 +239,7 @@ export const readSpans = Effect.fnUntraced(function* (file: string) { span.events?.find((event) => event.name === "exception")?.attributes, "exception.type", ), + ...(http === undefined ? {} : { http }), }); } } diff --git a/apps/cli/scripts/trace-report.unit.test.ts b/apps/cli/scripts/trace-report.unit.test.ts index c2fff6938e..b9195f141f 100644 --- a/apps/cli/scripts/trace-report.unit.test.ts +++ b/apps/cli/scripts/trace-report.unit.test.ts @@ -149,6 +149,74 @@ describe("readSpans", () => { }); }); +describe("readSpans HTTP attributes", () => { + const fixture = new URL("./__fixtures__/api-call-trace.jsonl", import.meta.url).pathname; + + it.effect("reads method, path, host, port and status from traces of both CLI generations", () => + Effect.gen(function* () { + const { spans, skippedLines } = yield* readSpans(fixture); + const http = spans.flatMap((span) => (span.http === undefined ? [] : [span.http])); + + expect(skippedLines).toBe(0); + expect(http).toHaveLength(2); + for (const call of http) { + expect(call).toEqual({ + method: "GET", + path: "/v1/projects", + address: "127.0.0.1", + port: 62824, + status: 200, + }); + } + expect(spans.find((span) => span.name === "cli.run")?.http).toBeUndefined(); + }).pipe(Effect.provide(BunServices.layer)), + ); + + it.effect("accepts OTLP string-encoded integers and drops the query string from the path", () => { + const batch = JSON.stringify({ + resourceSpans: [ + { + scopeSpans: [ + { + spans: [ + { + traceId: "t", + spanId: "s", + name: "POST", + startTimeUnixNano: "0", + endTimeUnixNano: "1000000", + status: { code: 1 }, + attributes: [ + { key: "http.request.method", value: { stringValue: "POST" } }, + { key: "url.path", value: { stringValue: "/v1/x?limit=1" } }, + { key: "server.port", value: { intValue: "8443" } }, + { key: "http.response.status_code", value: { intValue: "503" } }, + ], + }, + ], + }, + ], + }, + ], + }); + + return Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const file = yield* fs.makeTempFileScoped({ suffix: ".jsonl" }); + yield* fs.writeFileString(file, `${batch}\n`); + const { spans } = yield* readSpans(file); + + expect(spans[0]?.http).toEqual({ + method: "POST", + path: "/v1/x", + address: undefined, + port: 8443, + status: 503, + }); + }).pipe(Effect.provide(BunServices.layer)); + }); +}); + describe("selectTrace", () => { const twoRuns = [ { ...span("a1", undefined, "cli.run", 0, 100), traceId: "first" },